feat: kubesphere 4.0 (#6115)
* feat: kubesphere 4.0 Signed-off-by: ci-bot <ci-bot@kubesphere.io> * feat: kubesphere 4.0 Signed-off-by: ci-bot <ci-bot@kubesphere.io> --------- Signed-off-by: ci-bot <ci-bot@kubesphere.io> Co-authored-by: ks-ci-bot <ks-ci-bot@example.com> Co-authored-by: joyceliu <joyceliu@yunify.com>
This commit is contained in:
KubeSphere CI Bot
committed by
GitHub
GitHub
parent
b5015ec7b9
commit
447a51f08b
258
vendor/k8s.io/apiserver/pkg/util/flowcontrol/apf_controller.go
generated
vendored
258
vendor/k8s.io/apiserver/pkg/util/flowcontrol/apf_controller.go
generated
vendored
@@ -50,14 +50,19 @@ import (
|
||||
"k8s.io/klog/v2"
|
||||
"k8s.io/utils/clock"
|
||||
|
||||
flowcontrol "k8s.io/api/flowcontrol/v1beta3"
|
||||
flowcontrolapplyconfiguration "k8s.io/client-go/applyconfigurations/flowcontrol/v1beta3"
|
||||
flowcontrolclient "k8s.io/client-go/kubernetes/typed/flowcontrol/v1beta3"
|
||||
flowcontrollister "k8s.io/client-go/listers/flowcontrol/v1beta3"
|
||||
flowcontrol "k8s.io/api/flowcontrol/v1"
|
||||
flowcontrolapplyconfiguration "k8s.io/client-go/applyconfigurations/flowcontrol/v1"
|
||||
flowcontrolclient "k8s.io/client-go/kubernetes/typed/flowcontrol/v1"
|
||||
flowcontrollister "k8s.io/client-go/listers/flowcontrol/v1"
|
||||
)
|
||||
|
||||
const timeFmt = "2006-01-02T15:04:05.999"
|
||||
|
||||
const (
|
||||
// priorityLevelMaxSeatsPercent is the percentage of the nominalCL used as max seats allocatable from work estimator
|
||||
priorityLevelMaxSeatsPercent = float64(0.15)
|
||||
)
|
||||
|
||||
// This file contains a simple local (to the apiserver) controller
|
||||
// that digests API Priority and Fairness config objects (FlowSchema
|
||||
// and PriorityLevelConfiguration) into the data structure that the
|
||||
@@ -138,19 +143,22 @@ type configController struct {
|
||||
fsLister flowcontrollister.FlowSchemaLister
|
||||
fsInformerSynced cache.InformerSynced
|
||||
|
||||
flowcontrolClient flowcontrolclient.FlowcontrolV1beta3Interface
|
||||
flowcontrolClient flowcontrolclient.FlowcontrolV1Interface
|
||||
|
||||
// serverConcurrencyLimit is the limit on the server's total
|
||||
// number of non-exempt requests being served at once. This comes
|
||||
// from server configuration.
|
||||
serverConcurrencyLimit int
|
||||
|
||||
// requestWaitLimit comes from server configuration.
|
||||
requestWaitLimit time.Duration
|
||||
|
||||
// watchTracker implements the necessary WatchTracker interface.
|
||||
WatchTracker
|
||||
|
||||
// MaxSeatsTracker tracks the maximum seats that should be allocatable from the
|
||||
// work estimator for a given priority level. This controller does not enforce
|
||||
// any limits on max seats stored in this tracker, it is up to the work estimator
|
||||
// to set lower/upper limits on max seats (currently min=1, max=10).
|
||||
MaxSeatsTracker
|
||||
|
||||
// the most recent update attempts, ordered by increasing age.
|
||||
// Consumer trims to keep only the last minute's worth of entries.
|
||||
// The controller uses this to limit itself to at most six updates
|
||||
@@ -197,16 +205,15 @@ type priorityLevelState struct {
|
||||
pl *flowcontrol.PriorityLevelConfiguration
|
||||
|
||||
// qsCompleter holds the QueueSetCompleter derived from `config`
|
||||
// and `queues` if config is not exempt, nil otherwise.
|
||||
// and `queues`.
|
||||
qsCompleter fq.QueueSetCompleter
|
||||
|
||||
// The QueueSet for this priority level. This is nil if and only
|
||||
// if the priority level is exempt.
|
||||
// The QueueSet for this priority level.
|
||||
// Never nil.
|
||||
queues fq.QueueSet
|
||||
|
||||
// quiescing==true indicates that this priority level should be
|
||||
// removed when its queues have all drained. May be true only if
|
||||
// queues is non-nil.
|
||||
// removed when its queues have all drained.
|
||||
quiescing bool
|
||||
|
||||
// number of goroutines between Controller::Match and calling the
|
||||
@@ -253,9 +260,15 @@ type seatDemandStats struct {
|
||||
}
|
||||
|
||||
func (stats *seatDemandStats) update(obs fq.IntegratorResults) {
|
||||
stats.highWatermark = obs.Max
|
||||
if obs.Duration <= 0 {
|
||||
return
|
||||
}
|
||||
if math.IsNaN(obs.Deviation) {
|
||||
obs.Deviation = 0
|
||||
}
|
||||
stats.avg = obs.Average
|
||||
stats.stdDev = obs.Deviation
|
||||
stats.highWatermark = obs.Max
|
||||
envelope := obs.Average + obs.Deviation
|
||||
stats.smoothed = math.Max(envelope, seatDemandSmoothingCoefficient*stats.smoothed+(1-seatDemandSmoothingCoefficient)*envelope)
|
||||
}
|
||||
@@ -271,18 +284,18 @@ func newTestableController(config TestableConfig) *configController {
|
||||
asFieldManager: config.AsFieldManager,
|
||||
foundToDangling: config.FoundToDangling,
|
||||
serverConcurrencyLimit: config.ServerConcurrencyLimit,
|
||||
requestWaitLimit: config.RequestWaitLimit,
|
||||
flowcontrolClient: config.FlowcontrolClient,
|
||||
priorityLevelStates: make(map[string]*priorityLevelState),
|
||||
WatchTracker: NewWatchTracker(),
|
||||
MaxSeatsTracker: NewMaxSeatsTracker(),
|
||||
}
|
||||
klog.V(2).Infof("NewTestableController %q with serverConcurrencyLimit=%d, requestWaitLimit=%s, name=%s, asFieldManager=%q", cfgCtlr.name, cfgCtlr.serverConcurrencyLimit, cfgCtlr.requestWaitLimit, cfgCtlr.name, cfgCtlr.asFieldManager)
|
||||
klog.V(2).Infof("NewTestableController %q with serverConcurrencyLimit=%d, name=%s, asFieldManager=%q", cfgCtlr.name, cfgCtlr.serverConcurrencyLimit, cfgCtlr.name, cfgCtlr.asFieldManager)
|
||||
// Start with longish delay because conflicts will be between
|
||||
// different processes, so take some time to go away.
|
||||
cfgCtlr.configQueue = workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(200*time.Millisecond, 8*time.Hour), "priority_and_fairness_config_queue")
|
||||
// ensure the data structure reflects the mandatory config
|
||||
cfgCtlr.lockAndDigestConfigObjects(nil, nil)
|
||||
fci := config.InformerFactory.Flowcontrol().V1beta3()
|
||||
fci := config.InformerFactory.Flowcontrol().V1()
|
||||
pli := fci.PriorityLevelConfigurations()
|
||||
fsi := fci.FlowSchemas()
|
||||
cfgCtlr.plLister = pli.Lister()
|
||||
@@ -384,9 +397,6 @@ func (cfgCtlr *configController) updateBorrowingLocked(setCompleters bool, plSta
|
||||
items := make([]allocProblemItem, 0, len(plStates))
|
||||
plNames := make([]string, 0, len(plStates))
|
||||
for plName, plState := range plStates {
|
||||
if plState.pl.Spec.Limited == nil {
|
||||
continue
|
||||
}
|
||||
obs := plState.seatDemandIntegrator.Reset()
|
||||
plState.seatDemandStats.update(obs)
|
||||
// Lower bound on this priority level's adjusted concurreny limit is the lesser of:
|
||||
@@ -403,7 +413,7 @@ func (cfgCtlr *configController) updateBorrowingLocked(setCompleters bool, plSta
|
||||
})
|
||||
}
|
||||
if len(items) == 0 && cfgCtlr.nominalCLSum > 0 {
|
||||
klog.ErrorS(nil, "Impossible: no non-exempt priority levels", "plStates", cfgCtlr.priorityLevelStates)
|
||||
klog.ErrorS(nil, "Impossible: no priority levels", "plStates", cfgCtlr.priorityLevelStates)
|
||||
return
|
||||
}
|
||||
allocs, fairFrac, err := computeConcurrencyAllocation(cfgCtlr.nominalCLSum, items)
|
||||
@@ -412,20 +422,14 @@ func (cfgCtlr *configController) updateBorrowingLocked(setCompleters bool, plSta
|
||||
allocs = make([]float64, len(items))
|
||||
for idx, plName := range plNames {
|
||||
plState := plStates[plName]
|
||||
if plState.pl.Spec.Limited == nil {
|
||||
continue
|
||||
}
|
||||
allocs[idx] = float64(plState.currentCL)
|
||||
}
|
||||
}
|
||||
for idx, plName := range plNames {
|
||||
plState := plStates[plName]
|
||||
if plState.pl.Spec.Limited == nil {
|
||||
continue
|
||||
}
|
||||
if setCompleters {
|
||||
qsCompleter, err := queueSetCompleterForPL(cfgCtlr.queueSetFactory, plState.queues,
|
||||
plState.pl, cfgCtlr.requestWaitLimit, plState.reqsGaugePair, plState.execSeatsObs,
|
||||
plState.pl, plState.reqsGaugePair, plState.execSeatsObs,
|
||||
metrics.NewUnionGauge(plState.seatDemandIntegrator, plState.seatDemandRatioedGauge))
|
||||
if err != nil {
|
||||
klog.ErrorS(err, "Inconceivable! Configuration error in existing priority level", "pl", plState.pl)
|
||||
@@ -441,8 +445,15 @@ func (cfgCtlr *configController) updateBorrowingLocked(setCompleters bool, plSta
|
||||
if relChange >= 0.05 {
|
||||
logLevel = 2
|
||||
}
|
||||
klog.V(logLevel).InfoS("Update CurrentCL", "plName", plName, "seatDemandHighWatermark", plState.seatDemandStats.highWatermark, "seatDemandAvg", plState.seatDemandStats.avg, "seatDemandStdev", plState.seatDemandStats.stdDev, "seatDemandSmoothed", plState.seatDemandStats.smoothed, "fairFrac", fairFrac, "currentCL", currentCL, "backstop", err != nil)
|
||||
plState.queues = plState.qsCompleter.Complete(fq.DispatchingConfig{ConcurrencyLimit: currentCL})
|
||||
var concurrencyDenominator int
|
||||
if currentCL > 0 {
|
||||
concurrencyDenominator = currentCL
|
||||
} else {
|
||||
concurrencyDenominator = int(math.Max(1, math.Round(float64(cfgCtlr.serverConcurrencyLimit)/10)))
|
||||
}
|
||||
plState.seatDemandRatioedGauge.SetDenominator(float64(concurrencyDenominator))
|
||||
klog.V(logLevel).InfoS("Update CurrentCL", "plName", plName, "seatDemandHighWatermark", plState.seatDemandStats.highWatermark, "seatDemandAvg", plState.seatDemandStats.avg, "seatDemandStdev", plState.seatDemandStats.stdDev, "seatDemandSmoothed", plState.seatDemandStats.smoothed, "fairFrac", fairFrac, "currentCL", currentCL, "concurrencyDenominator", concurrencyDenominator, "backstop", err != nil)
|
||||
plState.queues = plState.qsCompleter.Complete(fq.DispatchingConfig{ConcurrencyLimit: currentCL, ConcurrencyDenominator: concurrencyDenominator})
|
||||
}
|
||||
metrics.SetFairFrac(float64(fairFrac))
|
||||
}
|
||||
@@ -642,10 +653,10 @@ func (cfgCtlr *configController) lockAndDigestConfigObjects(newPLs []*flowcontro
|
||||
|
||||
// Supply missing mandatory PriorityLevelConfiguration objects
|
||||
if !meal.haveExemptPL {
|
||||
meal.imaginePL(fcboot.MandatoryPriorityLevelConfigurationExempt, cfgCtlr.requestWaitLimit)
|
||||
meal.imaginePL(fcboot.MandatoryPriorityLevelConfigurationExempt)
|
||||
}
|
||||
if !meal.haveCatchAllPL {
|
||||
meal.imaginePL(fcboot.MandatoryPriorityLevelConfigurationCatchAll, cfgCtlr.requestWaitLimit)
|
||||
meal.imaginePL(fcboot.MandatoryPriorityLevelConfigurationCatchAll)
|
||||
}
|
||||
|
||||
meal.finishQueueSetReconfigsLocked()
|
||||
@@ -677,7 +688,7 @@ func (meal *cfgMeal) digestNewPLsLocked(newPLs []*flowcontrol.PriorityLevelConfi
|
||||
}
|
||||
}
|
||||
qsCompleter, err := queueSetCompleterForPL(meal.cfgCtlr.queueSetFactory, state.queues,
|
||||
pl, meal.cfgCtlr.requestWaitLimit, state.reqsGaugePair, state.execSeatsObs,
|
||||
pl, state.reqsGaugePair, state.execSeatsObs,
|
||||
metrics.NewUnionGauge(state.seatDemandIntegrator, state.seatDemandRatioedGauge))
|
||||
if err != nil {
|
||||
klog.Warningf("Ignoring PriorityLevelConfiguration object %s because its spec (%s) is broken: %s", pl.Name, fcfmt.Fmt(pl.Spec), err)
|
||||
@@ -690,9 +701,8 @@ func (meal *cfgMeal) digestNewPLsLocked(newPLs []*flowcontrol.PriorityLevelConfi
|
||||
klog.V(3).Infof("Priority level %q was undesired and has become desired again", pl.Name)
|
||||
state.quiescing = false
|
||||
}
|
||||
if state.pl.Spec.Limited != nil {
|
||||
meal.shareSum += float64(state.pl.Spec.Limited.NominalConcurrencyShares)
|
||||
}
|
||||
nominalConcurrencyShares, _, _ := plSpecCommons(state.pl)
|
||||
meal.shareSum += float64(*nominalConcurrencyShares)
|
||||
meal.haveExemptPL = meal.haveExemptPL || pl.Name == flowcontrol.PriorityLevelConfigurationNameExempt
|
||||
meal.haveCatchAllPL = meal.haveCatchAllPL || pl.Name == flowcontrol.PriorityLevelConfigurationNameCatchAll
|
||||
}
|
||||
@@ -765,15 +775,16 @@ func (meal *cfgMeal) processOldPLsLocked() {
|
||||
continue
|
||||
}
|
||||
if plName == flowcontrol.PriorityLevelConfigurationNameExempt && !meal.haveExemptPL || plName == flowcontrol.PriorityLevelConfigurationNameCatchAll && !meal.haveCatchAllPL {
|
||||
// BTW, we know the Spec has not changed because the
|
||||
// mandatory objects have immutable Specs
|
||||
// BTW, we know the Spec has not changed what is says about queuing because the
|
||||
// mandatory objects have immutable Specs as far as queuing is concerned.
|
||||
klog.V(3).Infof("Retaining mandatory priority level %q despite lack of API object", plName)
|
||||
} else {
|
||||
if plState.queues == nil || plState.numPending == 0 && plState.queues.IsIdle() {
|
||||
// Either there are no queues or they are done
|
||||
if plState.numPending == 0 && plState.queues.IsIdle() {
|
||||
// The QueueSet is done
|
||||
// draining and no use is coming from another
|
||||
// goroutine
|
||||
klog.V(3).Infof("Removing undesired priority level %q (nilQueues=%v), Type=%v", plName, plState.queues == nil, plState.pl.Spec.Type)
|
||||
klog.V(3).Infof("Removing undesired priority level %q, Type=%v", plName, plState.pl.Spec.Type)
|
||||
meal.cfgCtlr.MaxSeatsTracker.ForgetPriorityLevel(plName)
|
||||
continue
|
||||
}
|
||||
if !plState.quiescing {
|
||||
@@ -783,21 +794,20 @@ func (meal *cfgMeal) processOldPLsLocked() {
|
||||
}
|
||||
var err error
|
||||
plState.qsCompleter, err = queueSetCompleterForPL(meal.cfgCtlr.queueSetFactory, plState.queues,
|
||||
plState.pl, meal.cfgCtlr.requestWaitLimit, plState.reqsGaugePair, plState.execSeatsObs,
|
||||
plState.pl, plState.reqsGaugePair, plState.execSeatsObs,
|
||||
metrics.NewUnionGauge(plState.seatDemandIntegrator, plState.seatDemandRatioedGauge))
|
||||
if err != nil {
|
||||
// This can not happen because queueSetCompleterForPL already approved this config
|
||||
panic(fmt.Sprintf("%s from name=%q spec=%s", err, plName, fcfmt.Fmt(plState.pl.Spec)))
|
||||
}
|
||||
if plState.pl.Spec.Limited != nil {
|
||||
// We deliberately include the lingering priority levels
|
||||
// here so that their queues get some concurrency and they
|
||||
// continue to drain. During this interim a lingering
|
||||
// priority level continues to get a concurrency
|
||||
// allocation determined by all the share values in the
|
||||
// regular way.
|
||||
meal.shareSum += float64(plState.pl.Spec.Limited.NominalConcurrencyShares)
|
||||
}
|
||||
// We deliberately include the lingering priority levels
|
||||
// here so that their queues get some concurrency and they
|
||||
// continue to drain. During this interim a lingering
|
||||
// priority level continues to get a concurrency
|
||||
// allocation determined by all the share values in the
|
||||
// regular way.
|
||||
nominalConcurrencyShares, _, _ := plSpecCommons(plState.pl)
|
||||
meal.shareSum += float64(*nominalConcurrencyShares)
|
||||
meal.haveExemptPL = meal.haveExemptPL || plName == flowcontrol.PriorityLevelConfigurationNameExempt
|
||||
meal.haveCatchAllPL = meal.haveCatchAllPL || plName == flowcontrol.PriorityLevelConfigurationNameCatchAll
|
||||
meal.newPLStates[plName] = plState
|
||||
@@ -809,41 +819,46 @@ func (meal *cfgMeal) processOldPLsLocked() {
|
||||
// QueueSets.
|
||||
func (meal *cfgMeal) finishQueueSetReconfigsLocked() {
|
||||
for plName, plState := range meal.newPLStates {
|
||||
if plState.pl.Spec.Limited == nil {
|
||||
klog.V(5).Infof("Using exempt priority level %q: quiescing=%v", plName, plState.quiescing)
|
||||
continue
|
||||
}
|
||||
|
||||
limited := plState.pl.Spec.Limited
|
||||
nominalConcurrencyShares, lendablePercent, borrowingLimitPercent := plSpecCommons(plState.pl)
|
||||
// The use of math.Ceil here means that the results might sum
|
||||
// to a little more than serverConcurrencyLimit but the
|
||||
// difference will be negligible.
|
||||
concurrencyLimit := int(math.Ceil(float64(meal.cfgCtlr.serverConcurrencyLimit) * float64(limited.NominalConcurrencyShares) / meal.shareSum))
|
||||
concurrencyLimit := int(math.Ceil(float64(meal.cfgCtlr.serverConcurrencyLimit) * float64(*nominalConcurrencyShares) / meal.shareSum))
|
||||
var lendableCL, borrowingCL int
|
||||
if limited.LendablePercent != nil {
|
||||
lendableCL = int(math.Round(float64(concurrencyLimit) * float64(*limited.LendablePercent) / 100))
|
||||
if lendablePercent != nil {
|
||||
lendableCL = int(math.Round(float64(concurrencyLimit) * float64(*lendablePercent) / 100))
|
||||
}
|
||||
if limited.BorrowingLimitPercent != nil {
|
||||
borrowingCL = int(math.Round(float64(concurrencyLimit) * float64(*limited.BorrowingLimitPercent) / 100))
|
||||
if borrowingLimitPercent != nil {
|
||||
borrowingCL = int(math.Round(float64(concurrencyLimit) * float64(*borrowingLimitPercent) / 100))
|
||||
} else {
|
||||
borrowingCL = meal.cfgCtlr.serverConcurrencyLimit
|
||||
}
|
||||
|
||||
metrics.SetPriorityLevelConfiguration(plName, concurrencyLimit, concurrencyLimit-lendableCL, concurrencyLimit+borrowingCL)
|
||||
plState.seatDemandRatioedGauge.SetDenominator(float64(concurrencyLimit))
|
||||
cfgChanged := plState.nominalCL != concurrencyLimit || plState.minCL != concurrencyLimit-lendableCL || plState.maxCL != concurrencyLimit+borrowingCL
|
||||
plState.nominalCL = concurrencyLimit
|
||||
plState.minCL = concurrencyLimit - lendableCL
|
||||
plState.maxCL = concurrencyLimit + borrowingCL
|
||||
meal.maxExecutingRequests += concurrencyLimit
|
||||
var waitLimit int
|
||||
if qCfg := limited.LimitResponse.Queuing; qCfg != nil {
|
||||
waitLimit = int(qCfg.Queues * qCfg.QueueLengthLimit)
|
||||
}
|
||||
meal.maxWaitingRequests += waitLimit
|
||||
if limited := plState.pl.Spec.Limited; limited != nil {
|
||||
if qCfg := limited.LimitResponse.Queuing; qCfg != nil {
|
||||
meal.maxWaitingRequests += int(qCfg.Queues * qCfg.QueueLengthLimit)
|
||||
|
||||
// Max seats allocatable from work estimator is calculated as MAX(1, MIN(0.15 * nominalCL, nominalCL/handSize)).
|
||||
// This is to keep max seats relative to total available concurrency with a minimum value of 1.
|
||||
// 15% of nominal concurrency was chosen since it preserved the previous max seats of 10 for default priority levels
|
||||
// when using apiserver's default total server concurrency of 600 (--max-requests-inflight=400, --max-mutating-requests-inflight=200).
|
||||
// This ensures that clusters with relatively high inflight requests will continue to use a max seats of 10
|
||||
// while clusters with lower inflight requests will use max seats no greater than nominalCL/handSize.
|
||||
// Calculated max seats can return arbitrarily high values but work estimator currently limits max seats at 10.
|
||||
handSize := plState.pl.Spec.Limited.LimitResponse.Queuing.HandSize
|
||||
maxSeats := uint64(math.Max(1, math.Min(math.Ceil(float64(concurrencyLimit)*priorityLevelMaxSeatsPercent), float64(int32(concurrencyLimit)/handSize))))
|
||||
meal.cfgCtlr.MaxSeatsTracker.SetMaxSeats(plName, maxSeats)
|
||||
}
|
||||
}
|
||||
if plState.queues == nil {
|
||||
initialCL := concurrencyLimit - lendableCL/2
|
||||
klog.V(2).Infof("Introducing queues for priority level %q: config=%s, nominalCL=%d, lendableCL=%d, borrowingCL=%d, currentCL=%d, quiescing=%v (shares=%v, shareSum=%v)", plName, fcfmt.Fmt(plState.pl.Spec), concurrencyLimit, lendableCL, borrowingCL, initialCL, plState.quiescing, plState.pl.Spec.Limited.NominalConcurrencyShares, meal.shareSum)
|
||||
klog.V(2).Infof("Introducing queues for priority level %q: config=%s, nominalCL=%d, lendableCL=%d, borrowingCL=%d, currentCL=%d, quiescing=%v (shares=%v, shareSum=%v)", plName, fcfmt.Fmt(plState.pl.Spec), concurrencyLimit, lendableCL, borrowingCL, initialCL, plState.quiescing, nominalConcurrencyShares, meal.shareSum)
|
||||
plState.seatDemandStats = seatDemandStats{}
|
||||
plState.currentCL = initialCL
|
||||
} else {
|
||||
@@ -851,7 +866,7 @@ func (meal *cfgMeal) finishQueueSetReconfigsLocked() {
|
||||
if cfgChanged {
|
||||
logLevel = 2
|
||||
}
|
||||
klog.V(logLevel).Infof("Retaining queues for priority level %q: config=%s, nominalCL=%d, lendableCL=%d, borrowingCL=%d, currentCL=%d, quiescing=%v, numPending=%d (shares=%v, shareSum=%v)", plName, fcfmt.Fmt(plState.pl.Spec), concurrencyLimit, lendableCL, borrowingCL, plState.currentCL, plState.quiescing, plState.numPending, plState.pl.Spec.Limited.NominalConcurrencyShares, meal.shareSum)
|
||||
klog.V(logLevel).Infof("Retaining queues for priority level %q: config=%s, nominalCL=%d, lendableCL=%d, borrowingCL=%d, currentCL=%d, quiescing=%v, numPending=%d (shares=%v, shareSum=%v)", plName, fcfmt.Fmt(plState.pl.Spec), concurrencyLimit, lendableCL, borrowingCL, plState.currentCL, plState.quiescing, plState.numPending, nominalConcurrencyShares, meal.shareSum)
|
||||
}
|
||||
}
|
||||
meal.cfgCtlr.nominalCLSum = meal.maxExecutingRequests
|
||||
@@ -859,32 +874,34 @@ func (meal *cfgMeal) finishQueueSetReconfigsLocked() {
|
||||
}
|
||||
|
||||
// queueSetCompleterForPL returns an appropriate QueueSetCompleter for the
|
||||
// given priority level configuration. Returns nil if that config
|
||||
// does not call for limiting. Returns nil and an error if the given
|
||||
// given priority level configuration. Returns nil and an error if the given
|
||||
// object is malformed in a way that is a problem for this package.
|
||||
func queueSetCompleterForPL(qsf fq.QueueSetFactory, queues fq.QueueSet, pl *flowcontrol.PriorityLevelConfiguration, requestWaitLimit time.Duration, reqsIntPair metrics.RatioedGaugePair, execSeatsObs metrics.RatioedGauge, seatDemandGauge metrics.Gauge) (fq.QueueSetCompleter, error) {
|
||||
if (pl.Spec.Type == flowcontrol.PriorityLevelEnablementExempt) != (pl.Spec.Limited == nil) {
|
||||
return nil, errors.New("broken union structure at the top")
|
||||
func queueSetCompleterForPL(qsf fq.QueueSetFactory, queues fq.QueueSet, pl *flowcontrol.PriorityLevelConfiguration, reqsIntPair metrics.RatioedGaugePair, execSeatsObs metrics.RatioedGauge, seatDemandGauge metrics.Gauge) (fq.QueueSetCompleter, error) {
|
||||
if (pl.Spec.Type == flowcontrol.PriorityLevelEnablementLimited) != (pl.Spec.Limited != nil) {
|
||||
return nil, errors.New("broken union structure at the top, for Limited")
|
||||
}
|
||||
if (pl.Spec.Type == flowcontrol.PriorityLevelEnablementExempt) != (pl.Spec.Exempt != nil) {
|
||||
return nil, errors.New("broken union structure at the top, for Exempt")
|
||||
}
|
||||
if (pl.Spec.Type == flowcontrol.PriorityLevelEnablementExempt) != (pl.Name == flowcontrol.PriorityLevelConfigurationNameExempt) {
|
||||
// This package does not attempt to cope with a priority level dynamically switching between exempt and not.
|
||||
return nil, errors.New("non-alignment between name and type")
|
||||
}
|
||||
if pl.Spec.Limited == nil {
|
||||
return nil, nil
|
||||
}
|
||||
if (pl.Spec.Limited.LimitResponse.Type == flowcontrol.LimitResponseTypeReject) != (pl.Spec.Limited.LimitResponse.Queuing == nil) {
|
||||
return nil, errors.New("broken union structure for limit response")
|
||||
}
|
||||
qcAPI := pl.Spec.Limited.LimitResponse.Queuing
|
||||
qcQS := fq.QueuingConfig{Name: pl.Name}
|
||||
if qcAPI != nil {
|
||||
qcQS = fq.QueuingConfig{Name: pl.Name,
|
||||
DesiredNumQueues: int(qcAPI.Queues),
|
||||
QueueLengthLimit: int(qcAPI.QueueLengthLimit),
|
||||
HandSize: int(qcAPI.HandSize),
|
||||
RequestWaitLimit: requestWaitLimit,
|
||||
if pl.Spec.Limited != nil {
|
||||
if (pl.Spec.Limited.LimitResponse.Type == flowcontrol.LimitResponseTypeReject) != (pl.Spec.Limited.LimitResponse.Queuing == nil) {
|
||||
return nil, errors.New("broken union structure for limit response")
|
||||
}
|
||||
qcAPI := pl.Spec.Limited.LimitResponse.Queuing
|
||||
if qcAPI != nil {
|
||||
qcQS = fq.QueuingConfig{Name: pl.Name,
|
||||
DesiredNumQueues: int(qcAPI.Queues),
|
||||
QueueLengthLimit: int(qcAPI.QueueLengthLimit),
|
||||
HandSize: int(qcAPI.HandSize),
|
||||
}
|
||||
}
|
||||
} else {
|
||||
qcQS = fq.QueuingConfig{Name: pl.Name, DesiredNumQueues: -1}
|
||||
}
|
||||
var qsc fq.QueueSetCompleter
|
||||
var err error
|
||||
@@ -894,7 +911,7 @@ func queueSetCompleterForPL(qsf fq.QueueSetFactory, queues fq.QueueSet, pl *flow
|
||||
qsc, err = qsf.BeginConstruction(qcQS, reqsIntPair, execSeatsObs, seatDemandGauge)
|
||||
}
|
||||
if err != nil {
|
||||
err = fmt.Errorf("priority level %q has QueuingConfiguration %#+v, which is invalid: %w", pl.Name, qcAPI, err)
|
||||
err = fmt.Errorf("priority level %q has QueuingConfiguration %#+v, which is invalid: %w", pl.Name, qcQS, err)
|
||||
}
|
||||
return qsc, err
|
||||
}
|
||||
@@ -934,16 +951,15 @@ func (meal *cfgMeal) presyncFlowSchemaStatus(fs *flowcontrol.FlowSchema, isDangl
|
||||
|
||||
// imaginePL adds a priority level based on one of the mandatory ones
|
||||
// that does not actually exist (right now) as a real API object.
|
||||
func (meal *cfgMeal) imaginePL(proto *flowcontrol.PriorityLevelConfiguration, requestWaitLimit time.Duration) {
|
||||
func (meal *cfgMeal) imaginePL(proto *flowcontrol.PriorityLevelConfiguration) {
|
||||
klog.V(3).Infof("No %s PriorityLevelConfiguration found, imagining one", proto.Name)
|
||||
labelValues := []string{proto.Name}
|
||||
reqsGaugePair := metrics.RatioedGaugeVecPhasedElementPair(meal.cfgCtlr.reqsGaugeVec, 1, 1, labelValues)
|
||||
execSeatsObs := meal.cfgCtlr.execSeatsGaugeVec.NewForLabelValuesSafe(0, 1, labelValues)
|
||||
seatDemandIntegrator := fq.NewNamedIntegrator(meal.cfgCtlr.clock, proto.Name)
|
||||
seatDemandRatioedGauge := metrics.ApiserverSeatDemands.NewForLabelValuesSafe(0, 1, []string{proto.Name})
|
||||
qsCompleter, err := queueSetCompleterForPL(meal.cfgCtlr.queueSetFactory, nil, proto,
|
||||
requestWaitLimit, reqsGaugePair, execSeatsObs,
|
||||
metrics.NewUnionGauge(seatDemandIntegrator, seatDemandRatioedGauge))
|
||||
qsCompleter, err := queueSetCompleterForPL(meal.cfgCtlr.queueSetFactory, nil, proto, reqsGaugePair,
|
||||
execSeatsObs, metrics.NewUnionGauge(seatDemandIntegrator, seatDemandRatioedGauge))
|
||||
if err != nil {
|
||||
// This can not happen because proto is one of the mandatory
|
||||
// objects and these are not erroneous
|
||||
@@ -957,16 +973,8 @@ func (meal *cfgMeal) imaginePL(proto *flowcontrol.PriorityLevelConfiguration, re
|
||||
seatDemandIntegrator: seatDemandIntegrator,
|
||||
seatDemandRatioedGauge: seatDemandRatioedGauge,
|
||||
}
|
||||
if proto.Spec.Limited != nil {
|
||||
meal.shareSum += float64(proto.Spec.Limited.NominalConcurrencyShares)
|
||||
}
|
||||
}
|
||||
|
||||
type immediateRequest struct{}
|
||||
|
||||
func (immediateRequest) Finish(execute func()) bool {
|
||||
execute()
|
||||
return false
|
||||
nominalConcurrencyShares, _, _ := plSpecCommons(proto)
|
||||
meal.shareSum += float64(*nominalConcurrencyShares)
|
||||
}
|
||||
|
||||
// startRequest classifies and, if appropriate, enqueues the request.
|
||||
@@ -1007,32 +1015,31 @@ func (cfgCtlr *configController) startRequest(ctx context.Context, rd RequestDig
|
||||
}
|
||||
plName := selectedFlowSchema.Spec.PriorityLevelConfiguration.Name
|
||||
plState := cfgCtlr.priorityLevelStates[plName]
|
||||
if plState.pl.Spec.Type == flowcontrol.PriorityLevelEnablementExempt {
|
||||
noteFn(selectedFlowSchema, plState.pl, "")
|
||||
klog.V(7).Infof("startRequest(%#+v) => fsName=%q, distMethod=%#+v, plName=%q, immediate", rd, selectedFlowSchema.Name, selectedFlowSchema.Spec.DistinguisherMethod, plName)
|
||||
return selectedFlowSchema, plState.pl, true, immediateRequest{}, time.Time{}
|
||||
}
|
||||
var numQueues int32
|
||||
if plState.pl.Spec.Limited.LimitResponse.Type == flowcontrol.LimitResponseTypeQueue {
|
||||
numQueues = plState.pl.Spec.Limited.LimitResponse.Queuing.Queues
|
||||
}
|
||||
var flowDistinguisher string
|
||||
var hashValue uint64
|
||||
if numQueues > 1 {
|
||||
flowDistinguisher = computeFlowDistinguisher(rd, selectedFlowSchema.Spec.DistinguisherMethod)
|
||||
hashValue = hashFlowID(selectedFlowSchema.Name, flowDistinguisher)
|
||||
var flowDistinguisher string
|
||||
if plState.pl.Spec.Type != flowcontrol.PriorityLevelEnablementExempt {
|
||||
if plState.pl.Spec.Limited.LimitResponse.Type == flowcontrol.LimitResponseTypeQueue {
|
||||
numQueues = plState.pl.Spec.Limited.LimitResponse.Queuing.Queues
|
||||
}
|
||||
if numQueues > 1 {
|
||||
flowDistinguisher = computeFlowDistinguisher(rd, selectedFlowSchema.Spec.DistinguisherMethod)
|
||||
hashValue = hashFlowID(selectedFlowSchema.Name, flowDistinguisher)
|
||||
}
|
||||
}
|
||||
|
||||
noteFn(selectedFlowSchema, plState.pl, flowDistinguisher)
|
||||
workEstimate := workEstimator()
|
||||
|
||||
startWaitingTime = time.Now()
|
||||
if plState.pl.Spec.Type != flowcontrol.PriorityLevelEnablementExempt {
|
||||
startWaitingTime = cfgCtlr.clock.Now()
|
||||
}
|
||||
klog.V(7).Infof("startRequest(%#+v) => fsName=%q, distMethod=%#+v, plName=%q, numQueues=%d", rd, selectedFlowSchema.Name, selectedFlowSchema.Spec.DistinguisherMethod, plName, numQueues)
|
||||
req, idle := plState.queues.StartRequest(ctx, &workEstimate, hashValue, flowDistinguisher, selectedFlowSchema.Name, rd.RequestInfo, rd.User, queueNoteFn)
|
||||
if idle {
|
||||
cfgCtlr.maybeReapReadLocked(plName, plState)
|
||||
}
|
||||
return selectedFlowSchema, plState.pl, false, req, startWaitingTime
|
||||
return selectedFlowSchema, plState.pl, plState.pl.Spec.Type == flowcontrol.PriorityLevelEnablementExempt, req, startWaitingTime
|
||||
}
|
||||
|
||||
// maybeReap will remove the last internal traces of the named
|
||||
@@ -1046,10 +1053,6 @@ func (cfgCtlr *configController) maybeReap(plName string) {
|
||||
klog.V(7).Infof("plName=%s, plState==nil", plName)
|
||||
return
|
||||
}
|
||||
if plState.queues == nil {
|
||||
klog.V(7).Infof("plName=%s, plState.queues==nil", plName)
|
||||
return
|
||||
}
|
||||
useless := plState.quiescing && plState.numPending == 0 && plState.queues.IsIdle()
|
||||
klog.V(7).Infof("plState.quiescing=%v, plState.numPending=%d, useless=%v", plState.quiescing, plState.numPending, useless)
|
||||
if !useless {
|
||||
@@ -1107,3 +1110,16 @@ func relDiff(x, y float64) float64 {
|
||||
}
|
||||
return diff / den
|
||||
}
|
||||
|
||||
// plSpecCommons returns the (NominalConcurrencyShares, LendablePercent, BorrowingLimitPercent) of the given priority level config
|
||||
func plSpecCommons(pl *flowcontrol.PriorityLevelConfiguration) (*int32, *int32, *int32) {
|
||||
if limiter := pl.Spec.Limited; limiter != nil {
|
||||
return limiter.NominalConcurrencyShares, limiter.LendablePercent, limiter.BorrowingLimitPercent
|
||||
}
|
||||
limiter := pl.Spec.Exempt
|
||||
var nominalConcurrencyShares int32
|
||||
if limiter.NominalConcurrencyShares != nil {
|
||||
nominalConcurrencyShares = *limiter.NominalConcurrencyShares
|
||||
}
|
||||
return &nominalConcurrencyShares, limiter.LendablePercent, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user