update vendor

Signed-off-by: Roland.Ma <rolandma@yunify.com>

vendor/k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/integrator.go

@@ -0,0 +1,180 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fairqueuing
+
+import (
+	"math"
+	"sync"
+	"time"
+
+	"k8s.io/apimachinery/pkg/util/clock"
+	"k8s.io/apiserver/pkg/util/flowcontrol/metrics"
+)
+
+// Integrator computes the moments of some variable X over time as
+// read from a particular clock.  The integrals start when the
+// Integrator is created, and ends at the latest operation on the
+// Integrator.  As a `metrics.TimedObserver` this fixes X1=1 and
+// ignores attempts to change X1.
+type Integrator interface {
+	metrics.TimedObserver
+
+	GetResults() IntegratorResults
+
+	// Return the results of integrating to now, and reset integration to start now
+	Reset() IntegratorResults
+}
+
+// IntegratorResults holds statistical abstracts of the integration
+type IntegratorResults struct {
+	Duration  float64 //seconds
+	Average   float64 //time-weighted
+	Deviation float64 //standard deviation: sqrt(avg((value-avg)^2))
+	Min, Max  float64
+}
+
+// Equal tests for semantic equality.
+// This considers all NaN values to be equal to each other.
+func (x *IntegratorResults) Equal(y *IntegratorResults) bool {
+	return x == y || x != nil && y != nil && x.Duration == y.Duration && x.Min == y.Min && x.Max == y.Max && (x.Average == y.Average || math.IsNaN(x.Average) && math.IsNaN(y.Average)) && (x.Deviation == y.Deviation || math.IsNaN(x.Deviation) && math.IsNaN(y.Deviation))
+}
+
+type integrator struct {
+	clock clock.PassiveClock
+	sync.Mutex
+	lastTime time.Time
+	x        float64
+	moments  Moments
+	min, max float64
+}
+
+// NewIntegrator makes one that uses the given clock
+func NewIntegrator(clock clock.PassiveClock) Integrator {
+	return &integrator{
+		clock:    clock,
+		lastTime: clock.Now(),
+	}
+}
+
+func (igr *integrator) SetX1(x1 float64) {
+}
+
+func (igr *integrator) Set(x float64) {
+	igr.Lock()
+	igr.setLocked(x)
+	igr.Unlock()
+}
+
+func (igr *integrator) setLocked(x float64) {
+	igr.updateLocked()
+	igr.x = x
+	if x < igr.min {
+		igr.min = x
+	}
+	if x > igr.max {
+		igr.max = x
+	}
+}
+
+func (igr *integrator) Add(deltaX float64) {
+	igr.Lock()
+	igr.setLocked(igr.x + deltaX)
+	igr.Unlock()
+}
+
+func (igr *integrator) updateLocked() {
+	now := igr.clock.Now()
+	dt := now.Sub(igr.lastTime).Seconds()
+	igr.lastTime = now
+	igr.moments = igr.moments.Add(ConstantMoments(dt, igr.x))
+}
+
+func (igr *integrator) GetResults() IntegratorResults {
+	igr.Lock()
+	defer igr.Unlock()
+	return igr.getResultsLocked()
+}
+
+func (igr *integrator) Reset() IntegratorResults {
+	igr.Lock()
+	defer igr.Unlock()
+	results := igr.getResultsLocked()
+	igr.moments = Moments{}
+	igr.min = igr.x
+	igr.max = igr.x
+	return results
+}
+
+func (igr *integrator) getResultsLocked() (results IntegratorResults) {
+	igr.updateLocked()
+	results.Min, results.Max = igr.min, igr.max
+	results.Duration = igr.moments.ElapsedSeconds
+	results.Average, results.Deviation = igr.moments.AvgAndStdDev()
+	return
+}
+
+// Moments are the integrals of the 0, 1, and 2 powers of some
+// variable X over some range of time.
+type Moments struct {
+	ElapsedSeconds float64 // integral of dt
+	IntegralX      float64 // integral of x dt
+	IntegralXX     float64 // integral of x*x dt
+}
+
+// ConstantMoments is for a constant X
+func ConstantMoments(dt, x float64) Moments {
+	return Moments{
+		ElapsedSeconds: dt,
+		IntegralX:      x * dt,
+		IntegralXX:     x * x * dt,
+	}
+}
+
+// Add combines over two ranges of time
+func (igr Moments) Add(ogr Moments) Moments {
+	return Moments{
+		ElapsedSeconds: igr.ElapsedSeconds + ogr.ElapsedSeconds,
+		IntegralX:      igr.IntegralX + ogr.IntegralX,
+		IntegralXX:     igr.IntegralXX + ogr.IntegralXX,
+	}
+}
+
+// Sub finds the difference between a range of time and a subrange
+func (igr Moments) Sub(ogr Moments) Moments {
+	return Moments{
+		ElapsedSeconds: igr.ElapsedSeconds - ogr.ElapsedSeconds,
+		IntegralX:      igr.IntegralX - ogr.IntegralX,
+		IntegralXX:     igr.IntegralXX - ogr.IntegralXX,
+	}
+}
+
+// AvgAndStdDev returns the average and standard devation
+func (igr Moments) AvgAndStdDev() (float64, float64) {
+	if igr.ElapsedSeconds <= 0 {
+		return math.NaN(), math.NaN()
+	}
+	avg := igr.IntegralX / igr.ElapsedSeconds
+	// standard deviation is sqrt( average( (x - xbar)^2 ) )
+	// = sqrt( Integral( x^2 + xbar^2 -2*x*xbar dt ) / Duration )
+	// = sqrt( ( Integral( x^2 dt ) + Duration * xbar^2 - 2*xbar*Integral(x dt) ) / Duration)
+	// = sqrt( Integral(x^2 dt)/Duration - xbar^2 )
+	variance := igr.IntegralXX/igr.ElapsedSeconds - avg*avg
+	if variance >= 0 {
+		return avg, math.Sqrt(variance)
+	}
+	return avg, math.NaN()
+}

vendor/k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/interface.go

@@ -19,6 +19,9 @@ package fairqueuing
 import (
 	"context"
 	"time"
+
+	"k8s.io/apiserver/pkg/util/flowcontrol/debug"
+	"k8s.io/apiserver/pkg/util/flowcontrol/metrics"
 )
 
 // QueueSetFactory is used to create QueueSet objects.  Creation, like
@@ -28,7 +31,7 @@ import (
 // before committing to a concurrency allotment for the second.
 type QueueSetFactory interface {
 	// BeginConstruction does the first phase of creating a QueueSet
-	BeginConstruction(QueuingConfig) (QueueSetCompleter, error)
+	BeginConstruction(QueuingConfig, metrics.TimedObserverPair) (QueueSetCompleter, error)
 }
 
 // QueueSetCompleter finishes the two-step process of creating or
@@ -43,7 +46,7 @@ type QueueSetCompleter interface {
 // functionality of one non-exempt priority level.  It covers the
 // functionality described in the "Assignment to a Queue", "Queuing",
 // and "Dispatching" sections of
-// https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20190228-priority-and-fairness.md
+// https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/1040-priority-and-fairness/README.md
 // .  Some day we may have connections between priority levels, but
 // today is not that day.
 type QueueSet interface {
@@ -66,19 +69,33 @@ type QueueSet interface {
 	IsIdle() bool
 
 	// StartRequest begins the process of handling a request.  If the
-	// request gets queued and the number of queues is greater than
-	// 1 then Wait uses the given hashValue as the source of entropy
-	// as it shuffle-shards the request into a queue.  The descr1 and
-	// descr2 values play no role in the logic but appear in log
-	// messages.  This method always returns quickly (without waiting
-	// for the request to be dequeued).  If this method returns a nil
-	// Request value then caller should reject the request and the
-	// returned bool indicates whether the QueueSet was idle at the
-	// moment of the return.  Otherwise idle==false and the client
-	// must call the Wait method of the Request exactly once.
-	StartRequest(ctx context.Context, hashValue uint64, fsName string, descr1, descr2 interface{}) (req Request, idle bool)
+	// request gets queued and the number of queues is greater than 1
+	// then StartRequest uses the given hashValue as the source of
+	// entropy as it shuffle-shards the request into a queue.  The
+	// descr1 and descr2 values play no role in the logic but appear
+	// in log messages.  This method always returns quickly (without
+	// waiting for the request to be dequeued).  If this method
+	// returns a nil Request value then caller should reject the
+	// request and the returned bool indicates whether the QueueSet
+	// was idle at the moment of the return.  Otherwise idle==false
+	// and the client must call the Finish method of the Request
+	// exactly once.
+	StartRequest(ctx context.Context, hashValue uint64, flowDistinguisher, fsName string, descr1, descr2 interface{}, queueNoteFn QueueNoteFn) (req Request, idle bool)
+
+	// UpdateObservations makes sure any time-based statistics have
+	// caught up with the current clock reading
+	UpdateObservations()
+
+	// Dump saves and returns the instant internal state of the queue-set.
+	// Note that dumping process will stop the queue-set from proceeding
+	// any requests.
+	// For debugging only.
+	Dump(includeRequestDetails bool) debug.QueueSetDump
 }
 
+// QueueNoteFn is called when a request enters and leaves a queue
+type QueueNoteFn func(inQueue bool)
+
 // Request represents the remainder of the handling of one request
 type Request interface {
 	// Finish determines whether to execute or reject the request and
@@ -116,10 +133,3 @@ type DispatchingConfig struct {
 	// ConcurrencyLimit is the maximum number of requests of this QueueSet that may be executing at a time
 	ConcurrencyLimit int
 }
-
-// EmptyHandler is used to notify the callee when all the queues
-// of a QueueSet have been drained.
-type EmptyHandler interface {
-	// HandleEmpty is called to deliver the notification
-	HandleEmpty()
-}

vendor/k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/queueset/queueset.go

@@ -28,11 +28,12 @@ import (
 	"k8s.io/apimachinery/pkg/util/clock"
 	"k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apiserver/pkg/util/flowcontrol/counter"
+	"k8s.io/apiserver/pkg/util/flowcontrol/debug"
 	fq "k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing"
 	"k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/promise/lockingpromise"
 	"k8s.io/apiserver/pkg/util/flowcontrol/metrics"
 	"k8s.io/apiserver/pkg/util/shufflesharding"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 const nsTimeFmt = "2006-01-02 15:04:05.000000000"
@@ -48,6 +49,7 @@ type queueSetFactory struct {
 // the fields `factory` and `theSet` is non-nil.
 type queueSetCompleter struct {
 	factory *queueSetFactory
+	obsPair metrics.TimedObserverPair
 	theSet  *queueSet
 	qCfg    fq.QueuingConfig
 	dealer  *shufflesharding.Dealer
@@ -66,6 +68,7 @@ type queueSet struct {
 	clock                clock.PassiveClock
 	counter              counter.GoRoutineCounter
 	estimatedServiceTime float64
+	obsPair              metrics.TimedObserverPair
 
 	lock sync.Mutex
 
@@ -115,13 +118,14 @@ func NewQueueSetFactory(c clock.PassiveClock, counter counter.GoRoutineCounter)
 	}
 }
 
-func (qsf *queueSetFactory) BeginConstruction(qCfg fq.QueuingConfig) (fq.QueueSetCompleter, error) {
+func (qsf *queueSetFactory) BeginConstruction(qCfg fq.QueuingConfig, obsPair metrics.TimedObserverPair) (fq.QueueSetCompleter, error) {
 	dealer, err := checkConfig(qCfg)
 	if err != nil {
 		return nil, err
 	}
 	return &queueSetCompleter{
 		factory: qsf,
+		obsPair: obsPair,
 		qCfg:    qCfg,
 		dealer:  dealer}, nil
 }
@@ -147,6 +151,7 @@ func (qsc *queueSetCompleter) Complete(dCfg fq.DispatchingConfig) fq.QueueSet {
 			clock:                qsc.factory.clock,
 			counter:              qsc.factory.counter,
 			estimatedServiceTime: 60,
+			obsPair:              qsc.obsPair,
 			qCfg:                 qsc.qCfg,
 			virtualTime:          0,
 			lastRealTime:         qsc.factory.clock.Now(),
@@ -202,6 +207,12 @@ func (qs *queueSet) setConfiguration(qCfg fq.QueuingConfig, dealer *shuffleshard
 	qs.qCfg = qCfg
 	qs.dCfg = dCfg
 	qs.dealer = dealer
+	qll := qCfg.QueueLengthLimit
+	if qll < 1 {
+		qll = 1
+	}
+	qs.obsPair.RequestsWaiting.SetX1(float64(qll))
+	qs.obsPair.RequestsExecuting.SetX1(float64(dCfg.ConcurrencyLimit))
 
 	qs.dispatchAsMuchAsPossibleLocked()
 }
@@ -221,7 +232,7 @@ const (
 // executing at each point where there is a change in that quantity,
 // because the metrics --- and only the metrics --- track that
 // quantity per FlowSchema.
-func (qs *queueSet) StartRequest(ctx context.Context, hashValue uint64, fsName string, descr1, descr2 interface{}) (fq.Request, bool) {
+func (qs *queueSet) StartRequest(ctx context.Context, hashValue uint64, flowDistinguisher, fsName string, descr1, descr2 interface{}, queueNoteFn fq.QueueNoteFn) (fq.Request, bool) {
 	qs.lockAndSyncTime()
 	defer qs.lock.Unlock()
 	var req *request
@@ -232,10 +243,10 @@ func (qs *queueSet) StartRequest(ctx context.Context, hashValue uint64, fsName s
 	if qs.qCfg.DesiredNumQueues < 1 {
 		if qs.totRequestsExecuting >= qs.dCfg.ConcurrencyLimit {
 			klog.V(5).Infof("QS(%s): rejecting request %q %#+v %#+v because %d are executing and the limit is %d", qs.qCfg.Name, fsName, descr1, descr2, qs.totRequestsExecuting, qs.dCfg.ConcurrencyLimit)
-			metrics.AddReject(qs.qCfg.Name, fsName, "concurrency-limit")
+			metrics.AddReject(ctx, qs.qCfg.Name, fsName, "concurrency-limit")
 			return nil, qs.isIdleLocked()
 		}
-		req = qs.dispatchSansQueueLocked(ctx, fsName, descr1, descr2)
+		req = qs.dispatchSansQueueLocked(ctx, flowDistinguisher, fsName, descr1, descr2)
 		return req, false
 	}
 
@@ -246,12 +257,12 @@ func (qs *queueSet) StartRequest(ctx context.Context, hashValue uint64, fsName s
 	// 3) Reject current request if there is not enough concurrency shares and
 	// we are at max queue length
 	// 4) If not rejected, create a request and enqueue
-	req = qs.timeoutOldRequestsAndRejectOrEnqueueLocked(ctx, hashValue, fsName, descr1, descr2)
+	req = qs.timeoutOldRequestsAndRejectOrEnqueueLocked(ctx, hashValue, flowDistinguisher, fsName, descr1, descr2, queueNoteFn)
 	// req == nil means that the request was rejected - no remaining
 	// concurrency shares and at max queue length already
 	if req == nil {
 		klog.V(5).Infof("QS(%s): rejecting request %q %#+v %#+v due to queue full", qs.qCfg.Name, fsName, descr1, descr2)
-		metrics.AddReject(qs.qCfg.Name, fsName, "queue-full")
+		metrics.AddReject(ctx, qs.qCfg.Name, fsName, "queue-full")
 		return nil, qs.isIdleLocked()
 	}
 
@@ -274,6 +285,11 @@ func (qs *queueSet) StartRequest(ctx context.Context, hashValue uint64, fsName s
 	// request's context's Done channel gets closed by the time
 	// the request is done being processed.
 	doneCh := ctx.Done()
+
+	// Retrieve the queueset configuration name while we have the lock
+	// and use it in the goroutine below.
+	configName := qs.qCfg.Name
+
 	if doneCh != nil {
 		qs.preCreateOrUnblockGoroutine()
 		go func() {
@@ -286,7 +302,7 @@ func (qs *queueSet) StartRequest(ctx context.Context, hashValue uint64, fsName s
 			// known that the count does not need to be accurate.
 			// BTW, the count only needs to be accurate in a test that
 			// uses FakeEventClock::Run().
-			klog.V(6).Infof("QS(%s): Context of request %q %#+v %#+v is Done", qs.qCfg.Name, fsName, descr1, descr2)
+			klog.V(6).Infof("QS(%s): Context of request %q %#+v %#+v is Done", configName, fsName, descr1, descr2)
 			qs.cancelWait(req)
 			qs.goroutineDoneOrBlocked()
 		}()
@@ -294,13 +310,26 @@ func (qs *queueSet) StartRequest(ctx context.Context, hashValue uint64, fsName s
 	return req, false
 }
 
+func (req *request) NoteQueued(inQueue bool) {
+	if req.queueNoteFn != nil {
+		req.queueNoteFn(inQueue)
+	}
+}
+
 func (req *request) Finish(execFn func()) bool {
 	exec, idle := req.wait()
 	if !exec {
 		return idle
 	}
-	execFn()
-	return req.qs.finishRequestAndDispatchAsMuchAsPossible(req)
+	func() {
+		defer func() {
+			idle = req.qs.finishRequestAndDispatchAsMuchAsPossible(req)
+		}()
+
+		execFn()
+	}()
+
+	return idle
 }
 
 func (req *request) wait() (bool, bool) {
@@ -327,7 +356,7 @@ func (req *request) wait() (bool, bool) {
 	switch decision {
 	case decisionReject:
 		klog.V(5).Infof("QS(%s): request %#+v %#+v timed out after being enqueued\n", qs.qCfg.Name, req.descr1, req.descr2)
-		metrics.AddReject(qs.qCfg.Name, req.fsName, "time-out")
+		metrics.AddReject(req.ctx, qs.qCfg.Name, req.fsName, "time-out")
 		return false, qs.isIdleLocked()
 	case decisionCancel:
 		// TODO(aaron-prindle) add metrics for this case
@@ -398,7 +427,7 @@ func (qs *queueSet) getVirtualTimeRatioLocked() float64 {
 // returns the enqueud request on a successful enqueue
 // returns nil in the case that there is no available concurrency or
 // the queuelengthlimit has been reached
-func (qs *queueSet) timeoutOldRequestsAndRejectOrEnqueueLocked(ctx context.Context, hashValue uint64, fsName string, descr1, descr2 interface{}) *request {
+func (qs *queueSet) timeoutOldRequestsAndRejectOrEnqueueLocked(ctx context.Context, hashValue uint64, flowDistinguisher, fsName string, descr1, descr2 interface{}, queueNoteFn fq.QueueNoteFn) *request {
 	//	Start with the shuffle sharding, to pick a queue.
 	queueIdx := qs.chooseQueueIndexLocked(hashValue, descr1, descr2)
 	queue := qs.queues[queueIdx]
@@ -410,19 +439,21 @@ func (qs *queueSet) timeoutOldRequestsAndRejectOrEnqueueLocked(ctx context.Conte
 
 	// Create a request and enqueue
 	req := &request{
-		qs:          qs,
-		fsName:      fsName,
-		ctx:         ctx,
-		decision:    lockingpromise.NewWriteOnce(&qs.lock, qs.counter),
-		arrivalTime: qs.clock.Now(),
-		queue:       queue,
-		descr1:      descr1,
-		descr2:      descr2,
+		qs:                qs,
+		fsName:            fsName,
+		flowDistinguisher: flowDistinguisher,
+		ctx:               ctx,
+		decision:          lockingpromise.NewWriteOnce(&qs.lock, qs.counter),
+		arrivalTime:       qs.clock.Now(),
+		queue:             queue,
+		descr1:            descr1,
+		descr2:            descr2,
+		queueNoteFn:       queueNoteFn,
 	}
 	if ok := qs.rejectOrEnqueueLocked(req); !ok {
 		return nil
 	}
-	metrics.ObserveQueueLength(qs.qCfg.Name, fsName, len(queue.requests))
+	metrics.ObserveQueueLength(ctx, qs.qCfg.Name, fsName, len(queue.requests))
 	return req
 }
 
@@ -439,7 +470,7 @@ func (qs *queueSet) chooseQueueIndexLocked(hashValue uint64, descr1, descr2 inte
 			bestQueueIdx, bestQueueLen = queueIdx, thisLen
 		}
 	})
-	klog.V(6).Infof("QS(%s): For request %#+v %#+v chose queue %d, had %d waiting & %d executing", qs.qCfg.Name, descr1, descr2, bestQueueIdx, bestQueueLen, qs.queues[bestQueueIdx].requestsExecuting)
+	klog.V(6).Infof("QS(%s) at r=%s v=%.9fs: For request %#+v %#+v chose queue %d, had %d waiting & %d executing", qs.qCfg.Name, qs.clock.Now().Format(nsTimeFmt), qs.virtualTime, descr1, descr2, bestQueueIdx, bestQueueLen, qs.queues[bestQueueIdx].requestsExecuting)
 	return bestQueueIdx
 }
 
@@ -460,7 +491,8 @@ func (qs *queueSet) removeTimedOutRequestsFromQueueLocked(queue *queue, fsName s
 			req.decision.SetLocked(decisionReject)
 			// get index for timed out requests
 			timeoutIdx = i
-			metrics.AddRequestsInQueues(qs.qCfg.Name, req.fsName, -1)
+			metrics.AddRequestsInQueues(req.ctx, qs.qCfg.Name, req.fsName, -1)
+			req.NoteQueued(false)
 		} else {
 			break
 		}
@@ -473,6 +505,7 @@ func (qs *queueSet) removeTimedOutRequestsFromQueueLocked(queue *queue, fsName s
 		queue.requests = reqs[removeIdx:]
 		// decrement the # of requestsEnqueued
 		qs.totRequestsWaiting -= removeIdx
+		qs.obsPair.RequestsWaiting.Add(float64(-removeIdx))
 	}
 }
 
@@ -496,16 +529,19 @@ func (qs *queueSet) rejectOrEnqueueLocked(request *request) bool {
 // enqueues a request into its queue.
 func (qs *queueSet) enqueueLocked(request *request) {
 	queue := request.queue
+	now := qs.clock.Now()
 	if len(queue.requests) == 0 && queue.requestsExecuting == 0 {
 		// the queue’s virtual start time is set to the virtual time.
 		queue.virtualStart = qs.virtualTime
-		if klog.V(6) {
-			klog.Infof("QS(%s) at r=%s v=%.9fs: initialized queue %d virtual start time due to request %#+v %#+v", qs.qCfg.Name, qs.clock.Now().Format(nsTimeFmt), queue.virtualStart, queue.index, request.descr1, request.descr2)
+		if klog.V(6).Enabled() {
+			klog.Infof("QS(%s) at r=%s v=%.9fs: initialized queue %d virtual start time due to request %#+v %#+v", qs.qCfg.Name, now.Format(nsTimeFmt), queue.virtualStart, queue.index, request.descr1, request.descr2)
 		}
 	}
 	queue.Enqueue(request)
 	qs.totRequestsWaiting++
-	metrics.AddRequestsInQueues(qs.qCfg.Name, request.fsName, 1)
+	metrics.AddRequestsInQueues(request.ctx, qs.qCfg.Name, request.fsName, 1)
+	request.NoteQueued(true)
+	qs.obsPair.RequestsWaiting.Add(1)
 }
 
 // dispatchAsMuchAsPossibleLocked runs a loop, as long as there
@@ -523,22 +559,24 @@ func (qs *queueSet) dispatchAsMuchAsPossibleLocked() {
 	}
 }
 
-func (qs *queueSet) dispatchSansQueueLocked(ctx context.Context, fsName string, descr1, descr2 interface{}) *request {
+func (qs *queueSet) dispatchSansQueueLocked(ctx context.Context, flowDistinguisher, fsName string, descr1, descr2 interface{}) *request {
 	now := qs.clock.Now()
 	req := &request{
-		qs:          qs,
-		fsName:      fsName,
-		ctx:         ctx,
-		startTime:   now,
-		decision:    lockingpromise.NewWriteOnce(&qs.lock, qs.counter),
-		arrivalTime: now,
-		descr1:      descr1,
-		descr2:      descr2,
+		qs:                qs,
+		fsName:            fsName,
+		flowDistinguisher: flowDistinguisher,
+		ctx:               ctx,
+		startTime:         now,
+		decision:          lockingpromise.NewWriteOnce(&qs.lock, qs.counter),
+		arrivalTime:       now,
+		descr1:            descr1,
+		descr2:            descr2,
 	}
 	req.decision.SetLocked(decisionExecute)
 	qs.totRequestsExecuting++
-	metrics.AddRequestsExecuting(qs.qCfg.Name, fsName, 1)
-	if klog.V(5) {
+	metrics.AddRequestsExecuting(ctx, qs.qCfg.Name, fsName, 1)
+	qs.obsPair.RequestsExecuting.Add(1)
+	if klog.V(5).Enabled() {
 		klog.Infof("QS(%s) at r=%s v=%.9fs: immediate dispatch of request %q %#+v %#+v, qs will have %d executing", qs.qCfg.Name, now.Format(nsTimeFmt), qs.virtualTime, fsName, descr1, descr2, qs.totRequestsExecuting)
 	}
 	return req
@@ -566,9 +604,12 @@ func (qs *queueSet) dispatchLocked() bool {
 	qs.totRequestsWaiting--
 	qs.totRequestsExecuting++
 	queue.requestsExecuting++
-	metrics.AddRequestsInQueues(qs.qCfg.Name, request.fsName, -1)
-	metrics.AddRequestsExecuting(qs.qCfg.Name, request.fsName, 1)
-	if klog.V(6) {
+	metrics.AddRequestsInQueues(request.ctx, qs.qCfg.Name, request.fsName, -1)
+	request.NoteQueued(false)
+	metrics.AddRequestsExecuting(request.ctx, qs.qCfg.Name, request.fsName, 1)
+	qs.obsPair.RequestsWaiting.Add(-1)
+	qs.obsPair.RequestsExecuting.Add(1)
+	if klog.V(6).Enabled() {
 		klog.Infof("QS(%s) at r=%s v=%.9fs: dispatching request %#+v %#+v from queue %d with virtual start time %.9fs, queue will have %d waiting & %d executing", qs.qCfg.Name, request.startTime.Format(nsTimeFmt), qs.virtualTime, request.descr1, request.descr2, queue.index, queue.virtualStart, len(queue.requests), queue.requestsExecuting)
 	}
 	// When a request is dequeued for service -> qs.virtualStart += G
@@ -595,11 +636,12 @@ func (qs *queueSet) cancelWait(req *request) {
 			// remove the request
 			queue.requests = append(queue.requests[:i], queue.requests[i+1:]...)
 			qs.totRequestsWaiting--
-			metrics.AddRequestsInQueues(qs.qCfg.Name, req.fsName, -1)
+			metrics.AddRequestsInQueues(req.ctx, qs.qCfg.Name, req.fsName, -1)
+			req.NoteQueued(false)
+			qs.obsPair.RequestsWaiting.Add(-1)
 			break
 		}
 	}
-	return
 }
 
 // selectQueueLocked examines the queues in round robin order and
@@ -614,6 +656,7 @@ func (qs *queueSet) selectQueueLocked() *queue {
 		qs.robinIndex = (qs.robinIndex + 1) % nq
 		queue := qs.queues[qs.robinIndex]
 		if len(queue.requests) != 0 {
+
 			currentVirtualFinish := queue.GetVirtualFinish(0, qs.estimatedServiceTime)
 			if currentVirtualFinish < minVirtualFinish {
 				minVirtualFinish = currentVirtualFinish
@@ -626,6 +669,23 @@ func (qs *queueSet) selectQueueLocked() *queue {
 	// for the next round.  This way the non-selected queues
 	// win in the case that the virtual finish times are the same
 	qs.robinIndex = minIndex
+	// according to the original FQ formula:
+	//
+	//   Si = MAX(R(t), Fi-1)
+	//
+	// the virtual start (excluding the estimated cost) of the chose
+	// queue should always be greater or equal to the global virtual
+	// time.
+	//
+	// hence we're refreshing the per-queue virtual time for the chosen
+	// queue here. if the last virtual start time (excluded estimated cost)
+	// falls behind the global virtual time, we update the latest virtual
+	// start by: <latest global virtual time> + <previously estimated cost>
+	previouslyEstimatedServiceTime := float64(minQueue.requestsExecuting) * qs.estimatedServiceTime
+	if qs.virtualTime > minQueue.virtualStart-previouslyEstimatedServiceTime {
+		// per-queue virtual time should not fall behind the global
+		minQueue.virtualStart = qs.virtualTime + previouslyEstimatedServiceTime
+	}
 	return minQueue
 }
 
@@ -647,17 +707,19 @@ func (qs *queueSet) finishRequestAndDispatchAsMuchAsPossible(req *request) bool
 // previously dispatched request has completed it's service.  This
 // callback updates important state in the queueSet
 func (qs *queueSet) finishRequestLocked(r *request) {
+	now := qs.clock.Now()
 	qs.totRequestsExecuting--
-	metrics.AddRequestsExecuting(qs.qCfg.Name, r.fsName, -1)
+	metrics.AddRequestsExecuting(r.ctx, qs.qCfg.Name, r.fsName, -1)
+	qs.obsPair.RequestsExecuting.Add(-1)
 
 	if r.queue == nil {
-		if klog.V(6) {
-			klog.Infof("QS(%s) at r=%s v=%.9fs: request %#+v %#+v finished, qs will have %d executing", qs.qCfg.Name, qs.clock.Now().Format(nsTimeFmt), qs.virtualTime, r.descr1, r.descr2, qs.totRequestsExecuting)
+		if klog.V(6).Enabled() {
+			klog.Infof("QS(%s) at r=%s v=%.9fs: request %#+v %#+v finished, qs will have %d executing", qs.qCfg.Name, now.Format(nsTimeFmt), qs.virtualTime, r.descr1, r.descr2, qs.totRequestsExecuting)
 		}
 		return
 	}
 
-	S := qs.clock.Since(r.startTime).Seconds()
+	S := now.Sub(r.startTime).Seconds()
 
 	// When a request finishes being served, and the actual service time was S,
 	// the queue’s virtual start time is decremented by G - S.
@@ -666,8 +728,8 @@ func (qs *queueSet) finishRequestLocked(r *request) {
 	// request has finished, remove from requests executing
 	r.queue.requestsExecuting--
 
-	if klog.V(6) {
-		klog.Infof("QS(%s) at r=%s v=%.9fs: request %#+v %#+v finished, adjusted queue %d virtual start time to %.9fs due to service time %.9fs, queue will have %d waiting & %d executing", qs.qCfg.Name, qs.clock.Now().Format(nsTimeFmt), qs.virtualTime, r.descr1, r.descr2, r.queue.index, r.queue.virtualStart, S, len(r.queue.requests), r.queue.requestsExecuting)
+	if klog.V(6).Enabled() {
+		klog.Infof("QS(%s) at r=%s v=%.9fs: request %#+v %#+v finished, adjusted queue %d virtual start time to %.9fs due to service time %.9fs, queue will have %d waiting & %d executing", qs.qCfg.Name, now.Format(nsTimeFmt), qs.virtualTime, r.descr1, r.descr2, r.queue.index, r.queue.virtualStart, S, len(r.queue.requests), r.queue.requestsExecuting)
 	}
 
 	// If there are more queues than desired and this one has no
@@ -709,3 +771,22 @@ func (qs *queueSet) preCreateOrUnblockGoroutine() {
 func (qs *queueSet) goroutineDoneOrBlocked() {
 	qs.counter.Add(-1)
 }
+
+func (qs *queueSet) UpdateObservations() {
+	qs.obsPair.RequestsWaiting.Add(0)
+	qs.obsPair.RequestsExecuting.Add(0)
+}
+
+func (qs *queueSet) Dump(includeRequestDetails bool) debug.QueueSetDump {
+	qs.lock.Lock()
+	defer qs.lock.Unlock()
+	d := debug.QueueSetDump{
+		Queues:    make([]debug.QueueDump, len(qs.queues)),
+		Waiting:   qs.totRequestsWaiting,
+		Executing: qs.totRequestsExecuting,
+	}
+	for i, q := range qs.queues {
+		d.Queues[i] = q.dump(includeRequestDetails)
+	}
+	return d
+}

vendor/k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/queueset/types.go

@@ -20,15 +20,21 @@ import (
 	"context"
 	"time"
 
+	genericrequest "k8s.io/apiserver/pkg/endpoints/request"
+	"k8s.io/apiserver/pkg/util/flowcontrol/debug"
+	fq "k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing"
 	"k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/promise"
 )
 
 // request is a temporary container for "requests" with additional
 // tracking fields required for the functionality FQScheduler
 type request struct {
-	qs     *queueSet
-	fsName string
-	ctx    context.Context
+	ctx context.Context
+
+	qs *queueSet
+
+	flowDistinguisher string
+	fsName            string
 
 	// The relevant queue.  Is nil if this request did not go through
 	// a queue.
@@ -53,6 +59,8 @@ type request struct {
 
 	// Indicates whether client has called Request::Wait()
 	waitStarted bool
+
+	queueNoteFn fq.QueueNoteFn
 }
 
 // queue is an array of requests with additional metadata required for
@@ -94,3 +102,27 @@ func (q *queue) GetVirtualFinish(J int, G float64) float64 {
 	jg := float64(J+1) * float64(G)
 	return jg + q.virtualStart
 }
+
+func (q *queue) dump(includeDetails bool) debug.QueueDump {
+	digest := make([]debug.RequestDump, len(q.requests))
+	for i, r := range q.requests {
+		// dump requests.
+		digest[i].MatchedFlowSchema = r.fsName
+		digest[i].FlowDistinguisher = r.flowDistinguisher
+		digest[i].ArriveTime = r.arrivalTime
+		digest[i].StartTime = r.startTime
+		if includeDetails {
+			userInfo, _ := genericrequest.UserFrom(r.ctx)
+			digest[i].UserName = userInfo.GetName()
+			requestInfo, ok := genericrequest.RequestInfoFrom(r.ctx)
+			if ok {
+				digest[i].RequestInfo = *requestInfo
+			}
+		}
+	}
+	return debug.QueueDump{
+		VirtualStart:      q.virtualStart,
+		Requests:          digest,
+		ExecutingRequests: q.requestsExecuting,
+	}
+}