Upgrade k8s package verison (#5358)

* upgrade k8s package version

Signed-off-by: hongzhouzi <hongzhouzi@kubesphere.io>

* Script upgrade and code formatting.

Signed-off-by: hongzhouzi <hongzhouzi@kubesphere.io>

Signed-off-by: hongzhouzi <hongzhouzi@kubesphere.io>
This commit is contained in:
hongzhouzi
2022-11-15 14:56:38 +08:00
committed by GitHub
parent 5f91c1663a
commit 44167aa47a
3106 changed files with 321340 additions and 172080 deletions

View File

@@ -43,8 +43,9 @@ var leadingDigits = regexp.MustCompile(`^(\d+)`)
// MajorMinor parses a numeric major/minor version from the provided version info.
// The minor version drops all characters after the first non-digit character:
// version.Info{Major:"1", Minor:"2+"} -> 1,2
// version.Info{Major:"1", Minor:"2.3-build4"} -> 1,2
//
// version.Info{Major:"1", Minor:"2+"} -> 1,2
// version.Info{Major:"1", Minor:"2.3-build4"} -> 1,2
func MajorMinor(v version.Info) (int, int, error) {
major, err := strconv.Atoi(v.Major)
if err != nil {

View File

@@ -19,7 +19,7 @@ package discovery
import (
"net/http"
"github.com/emicklei/go-restful"
"github.com/emicklei/go-restful/v3"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"

View File

@@ -19,7 +19,7 @@ package discovery
import (
"net/http"
"github.com/emicklei/go-restful"
"github.com/emicklei/go-restful/v3"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"

View File

@@ -20,7 +20,7 @@ import (
"net/http"
"sync"
restful "github.com/emicklei/go-restful"
restful "github.com/emicklei/go-restful/v3"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"

View File

@@ -35,6 +35,5 @@ func StorageVersionHash(group, version, kind string) string {
// the chance of colliding hash P(N,X) approximates to 1-e^(-(N^2)/2^(8X+1)).
// P(10,000, 8) ~= 2.7*10^(-12), which is low enough.
// See https://en.wikipedia.org/wiki/Birthday_problem#Approximations.
return base64.StdEncoding.EncodeToString(
bytes[:8])
return base64.StdEncoding.EncodeToString(bytes[:8])
}

View File

@@ -19,7 +19,7 @@ package discovery
import (
"net/http"
restful "github.com/emicklei/go-restful"
restful "github.com/emicklei/go-restful/v3"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"

View File

@@ -18,12 +18,15 @@ package filterlatency
import (
"context"
"fmt"
"net/http"
"time"
utilclock "k8s.io/apimachinery/pkg/util/clock"
"k8s.io/apiserver/pkg/endpoints/metrics"
apirequest "k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/server/httplog"
"k8s.io/klog/v2"
"k8s.io/utils/clock"
)
type requestFilterRecordKeyType int
@@ -31,6 +34,8 @@ type requestFilterRecordKeyType int
// requestFilterRecordKey is the context key for a request filter record struct.
const requestFilterRecordKey requestFilterRecordKeyType = iota
const minFilterLatencyToLog = 100 * time.Millisecond
type requestFilterRecord struct {
name string
startedTimestamp time.Time
@@ -50,18 +55,22 @@ func requestFilterRecordFrom(ctx context.Context) *requestFilterRecord {
// TrackStarted measures the timestamp the given handler has started execution
// by attaching a handler to the chain.
func TrackStarted(handler http.Handler, name string) http.Handler {
return trackStarted(handler, name, utilclock.RealClock{})
return trackStarted(handler, name, clock.RealClock{})
}
// TrackCompleted measures the timestamp the given handler has completed execution and then
// it updates the corresponding metric with the filter latency duration.
func TrackCompleted(handler http.Handler) http.Handler {
return trackCompleted(handler, utilclock.RealClock{}, func(ctx context.Context, fr *requestFilterRecord, completedAt time.Time) {
metrics.RecordFilterLatency(ctx, fr.name, completedAt.Sub(fr.startedTimestamp))
return trackCompleted(handler, clock.RealClock{}, func(ctx context.Context, fr *requestFilterRecord, completedAt time.Time) {
latency := completedAt.Sub(fr.startedTimestamp)
metrics.RecordFilterLatency(ctx, fr.name, latency)
if klog.V(3).Enabled() && latency > minFilterLatencyToLog {
httplog.AddKeyValue(ctx, fmt.Sprintf("fl_%s", fr.name), latency.String())
}
})
}
func trackStarted(handler http.Handler, name string, clock utilclock.PassiveClock) http.Handler {
func trackStarted(handler http.Handler, name string, clock clock.PassiveClock) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
if fr := requestFilterRecordFrom(ctx); fr != nil {
@@ -81,7 +90,7 @@ func trackStarted(handler http.Handler, name string, clock utilclock.PassiveCloc
})
}
func trackCompleted(handler http.Handler, clock utilclock.PassiveClock, action func(context.Context, *requestFilterRecord, time.Time)) http.Handler {
func trackCompleted(handler http.Handler, clock clock.PassiveClock, action func(context.Context, *requestFilterRecord, time.Time)) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// The previous filter has just completed.
completedAt := clock.Now()

View File

@@ -1,6 +1,6 @@
# See the OWNERS docs at https://go.k8s.io/owners
reviewers:
- deads2k
- sttts
- soltysh
- deads2k
- sttts
- soltysh

View File

@@ -30,32 +30,38 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
auditinternal "k8s.io/apiserver/pkg/apis/audit"
"k8s.io/apiserver/pkg/audit"
"k8s.io/apiserver/pkg/audit/policy"
"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/endpoints/responsewriter"
)
// WithAudit decorates a http.Handler with audit logging information for all the
// requests coming to the server. Audit level is decided according to requests'
// attributes and audit policy. Logs are emitted to the audit sink to
// process events. If sink or audit policy is nil, no decoration takes place.
func WithAudit(handler http.Handler, sink audit.Sink, policy policy.Checker, longRunningCheck request.LongRunningRequestCheck) http.Handler {
func WithAudit(handler http.Handler, sink audit.Sink, policy audit.PolicyRuleEvaluator, longRunningCheck request.LongRunningRequestCheck) http.Handler {
if sink == nil || policy == nil {
return handler
}
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
req, ev, omitStages, err := createAuditEventAndAttachToContext(req, policy)
auditContext, err := evaluatePolicyAndCreateAuditEvent(req, policy)
if err != nil {
utilruntime.HandleError(fmt.Errorf("failed to create audit event: %v", err))
responsewriters.InternalError(w, req, errors.New("failed to create audit event"))
return
}
ctx := req.Context()
if ev == nil || ctx == nil {
ev := auditContext.Event
if ev == nil || req.Context() == nil {
handler.ServeHTTP(w, req)
return
}
req = req.WithContext(audit.WithAuditContext(req.Context(), auditContext))
ctx := req.Context()
omitStages := auditContext.RequestAuditConfig.OmitStages
ev.Stage = auditinternal.StageRequestReceived
if processed := processAuditEvent(ctx, sink, ev, omitStages); !processed {
audit.ApiserverAuditDroppedCounter.WithContext(ctx).Inc()
@@ -112,38 +118,67 @@ func WithAudit(handler http.Handler, sink audit.Sink, policy policy.Checker, lon
})
}
// createAuditEventAndAttachToContext is responsible for creating the audit event
// and attaching it to the appropriate request context. It returns:
// - context with audit event attached to it
// - created audit event
// evaluatePolicyAndCreateAuditEvent is responsible for evaluating the audit
// policy configuration applicable to the request and create a new audit
// event that will be written to the API audit log.
// - error if anything bad happened
func createAuditEventAndAttachToContext(req *http.Request, policy policy.Checker) (*http.Request, *auditinternal.Event, []auditinternal.Stage, error) {
func evaluatePolicyAndCreateAuditEvent(req *http.Request, policy audit.PolicyRuleEvaluator) (*audit.AuditContext, error) {
ctx := req.Context()
attribs, err := GetAuthorizerAttributes(ctx)
if err != nil {
return req, nil, nil, fmt.Errorf("failed to GetAuthorizerAttributes: %v", err)
return nil, fmt.Errorf("failed to GetAuthorizerAttributes: %v", err)
}
level, omitStages := policy.LevelAndStages(attribs)
audit.ObservePolicyLevel(ctx, level)
if level == auditinternal.LevelNone {
ls := policy.EvaluatePolicyRule(attribs)
audit.ObservePolicyLevel(ctx, ls.Level)
if ls.Level == auditinternal.LevelNone {
// Don't audit.
return req, nil, nil, nil
return &audit.AuditContext{
RequestAuditConfig: ls.RequestAuditConfig,
}, nil
}
requestReceivedTimestamp, ok := request.ReceivedTimestampFrom(ctx)
if !ok {
requestReceivedTimestamp = time.Now()
}
ev, err := audit.NewEventFromRequest(req, requestReceivedTimestamp, level, attribs)
ev, err := audit.NewEventFromRequest(req, requestReceivedTimestamp, ls.Level, attribs)
if err != nil {
return req, nil, nil, fmt.Errorf("failed to complete audit event from request: %v", err)
return nil, fmt.Errorf("failed to complete audit event from request: %v", err)
}
req = req.WithContext(request.WithAuditEvent(ctx, ev))
return &audit.AuditContext{
RequestAuditConfig: ls.RequestAuditConfig,
Event: ev,
}, nil
}
return req, ev, omitStages, nil
// writeLatencyToAnnotation writes the latency incurred in different
// layers of the apiserver to the annotations of the audit object.
// it should be invoked after ev.StageTimestamp has been set appropriately.
func writeLatencyToAnnotation(ctx context.Context, ev *auditinternal.Event) {
// we will track latency in annotation only when the total latency
// of the given request exceeds 500ms, this is in keeping with the
// traces in rest/handlers for create, delete, update,
// get, list, and deletecollection.
const threshold = 500 * time.Millisecond
latency := ev.StageTimestamp.Time.Sub(ev.RequestReceivedTimestamp.Time)
if latency <= threshold {
return
}
// if we are tracking latency incurred inside different layers within
// the apiserver, add these as annotation to the audit event object.
layerLatencies := request.AuditAnnotationsFromLatencyTrackers(ctx)
if len(layerLatencies) == 0 {
// latency tracking is not enabled for this request
return
}
// record the total latency for this request, for convenience.
layerLatencies["apiserver.latency.k8s.io/total"] = latency.String()
audit.AddAuditAnnotationsMap(ctx, layerLatencies)
}
func processAuditEvent(ctx context.Context, sink audit.Sink, ev *auditinternal.Event, omitStages []auditinternal.Stage) bool {
@@ -153,11 +188,16 @@ func processAuditEvent(ctx context.Context, sink audit.Sink, ev *auditinternal.E
}
}
if ev.Stage == auditinternal.StageRequestReceived {
switch {
case ev.Stage == auditinternal.StageRequestReceived:
ev.StageTimestamp = metav1.NewMicroTime(ev.RequestReceivedTimestamp.Time)
} else {
case ev.Stage == auditinternal.StageResponseComplete:
ev.StageTimestamp = metav1.NewMicroTime(time.Now())
writeLatencyToAnnotation(ctx, ev)
default:
ev.StageTimestamp = metav1.NewMicroTime(time.Now())
}
audit.ObserveEvent(ctx)
return sink.ProcessEvents(ev)
}
@@ -171,18 +211,11 @@ func decorateResponseWriter(ctx context.Context, responseWriter http.ResponseWri
omitStages: omitStages,
}
// check if the ResponseWriter we're wrapping is the fancy one we need
// or if the basic is sufficient
_, cn := responseWriter.(http.CloseNotifier)
_, fl := responseWriter.(http.Flusher)
_, hj := responseWriter.(http.Hijacker)
if cn && fl && hj {
return &fancyResponseWriterDelegator{delegate}
}
return delegate
return responsewriter.WrapForHTTP1Or2(delegate)
}
var _ http.ResponseWriter = &auditResponseWriter{}
var _ responsewriter.UserProvidedDecorator = &auditResponseWriter{}
// auditResponseWriter intercepts WriteHeader, sets it in the event. If the sink is set, it will
// create immediately an event (for long running requests).
@@ -195,8 +228,8 @@ type auditResponseWriter struct {
omitStages []auditinternal.Stage
}
func (a *auditResponseWriter) setHttpHeader() {
a.ResponseWriter.Header().Set(auditinternal.HeaderAuditID, string(a.event.AuditID))
func (a *auditResponseWriter) Unwrap() http.ResponseWriter {
return a.ResponseWriter
}
func (a *auditResponseWriter) processCode(code int) {
@@ -216,44 +249,19 @@ func (a *auditResponseWriter) processCode(code int) {
func (a *auditResponseWriter) Write(bs []byte) (int, error) {
// the Go library calls WriteHeader internally if no code was written yet. But this will go unnoticed for us
a.processCode(http.StatusOK)
a.setHttpHeader()
return a.ResponseWriter.Write(bs)
}
func (a *auditResponseWriter) WriteHeader(code int) {
a.processCode(code)
a.setHttpHeader()
a.ResponseWriter.WriteHeader(code)
}
// fancyResponseWriterDelegator implements http.CloseNotifier, http.Flusher and
// http.Hijacker which are needed to make certain http operation (e.g. watch, rsh, etc)
// working.
type fancyResponseWriterDelegator struct {
*auditResponseWriter
}
func (f *fancyResponseWriterDelegator) CloseNotify() <-chan bool {
return f.ResponseWriter.(http.CloseNotifier).CloseNotify()
}
func (f *fancyResponseWriterDelegator) Flush() {
f.ResponseWriter.(http.Flusher).Flush()
}
func (f *fancyResponseWriterDelegator) Hijack() (net.Conn, *bufio.ReadWriter, error) {
func (a *auditResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
// fake a response status before protocol switch happens
f.processCode(http.StatusSwitchingProtocols)
a.processCode(http.StatusSwitchingProtocols)
// This will be ignored if WriteHeader() function has already been called.
// It's not guaranteed Audit-ID http header is sent for all requests.
// For example, when user run "kubectl exec", apiserver uses a proxy handler
// to deal with the request, users can only get http headers returned by kubelet node.
f.setHttpHeader()
return f.ResponseWriter.(http.Hijacker).Hijack()
// the outer ResponseWriter object returned by WrapForHTTP1Or2 implements
// http.Hijacker if the inner object (a.ResponseWriter) implements http.Hijacker.
return a.ResponseWriter.(http.Hijacker).Hijack()
}
var _ http.CloseNotifier = &fancyResponseWriterDelegator{}
var _ http.Flusher = &fancyResponseWriterDelegator{}
var _ http.Hijacker = &fancyResponseWriterDelegator{}

View File

@@ -20,14 +20,13 @@ import (
"net/http"
"k8s.io/apiserver/pkg/audit"
"k8s.io/apiserver/pkg/audit/policy"
)
// WithAuditAnnotations decorates a http.Handler with a []{key, value} that is merged
// with the audit.Event.Annotations map. This allows layers that run before WithAudit
// (such as authentication) to assert annotations.
// If sink or audit policy is nil, no decoration takes place.
func WithAuditAnnotations(handler http.Handler, sink audit.Sink, policy policy.Checker) http.Handler {
func WithAuditAnnotations(handler http.Handler, sink audit.Sink, policy audit.PolicyRuleEvaluator) http.Handler {
// no need to wrap if auditing is disabled
if sink == nil || policy == nil {
return handler

View File

@@ -26,33 +26,36 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
auditinternal "k8s.io/apiserver/pkg/apis/audit"
"k8s.io/apiserver/pkg/audit"
"k8s.io/apiserver/pkg/audit/policy"
"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
)
// WithFailedAuthenticationAudit decorates a failed http.Handler used in WithAuthentication handler.
// It is meant to log only failed authentication requests.
func WithFailedAuthenticationAudit(failedHandler http.Handler, sink audit.Sink, policy policy.Checker) http.Handler {
func WithFailedAuthenticationAudit(failedHandler http.Handler, sink audit.Sink, policy audit.PolicyRuleEvaluator) http.Handler {
if sink == nil || policy == nil {
return failedHandler
}
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
req, ev, omitStages, err := createAuditEventAndAttachToContext(req, policy)
a, err := evaluatePolicyAndCreateAuditEvent(req, policy)
if err != nil {
utilruntime.HandleError(fmt.Errorf("failed to create audit event: %v", err))
responsewriters.InternalError(w, req, errors.New("failed to create audit event"))
return
}
ev := a.Event
if ev == nil {
failedHandler.ServeHTTP(w, req)
return
}
req = req.WithContext(audit.WithAuditContext(req.Context(), a))
ev.ResponseStatus = &metav1.Status{}
ev.ResponseStatus.Message = getAuthMethods(req)
ev.Stage = auditinternal.StageResponseStarted
rw := decorateResponseWriter(req.Context(), w, ev, sink, omitStages)
rw := decorateResponseWriter(req.Context(), w, ev, sink, a.RequestAuditConfig.OmitStages)
failedHandler.ServeHTTP(rw, req)
})
}

View File

@@ -49,7 +49,6 @@ func WithAuthorization(handler http.Handler, a authorizer.Authorizer, s runtime.
}
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
ctx := req.Context()
ae := request.AuditEventFrom(ctx)
attributes, err := GetAuthorizerAttributes(ctx)
if err != nil {
@@ -59,20 +58,22 @@ func WithAuthorization(handler http.Handler, a authorizer.Authorizer, s runtime.
authorized, reason, err := a.Authorize(ctx, attributes)
// an authorizer like RBAC could encounter evaluation errors and still allow the request, so authorizer decision is checked before error here.
if authorized == authorizer.DecisionAllow {
audit.LogAnnotation(ae, decisionAnnotationKey, decisionAllow)
audit.LogAnnotation(ae, reasonAnnotationKey, reason)
audit.AddAuditAnnotations(ctx,
decisionAnnotationKey, decisionAllow,
reasonAnnotationKey, reason)
handler.ServeHTTP(w, req)
return
}
if err != nil {
audit.LogAnnotation(ae, reasonAnnotationKey, reasonError)
audit.AddAuditAnnotation(ctx, reasonAnnotationKey, reasonError)
responsewriters.InternalError(w, req, err)
return
}
klog.V(4).InfoS("Forbidden", "URI", req.RequestURI, "Reason", reason)
audit.LogAnnotation(ae, decisionAnnotationKey, decisionForbid)
audit.LogAnnotation(ae, reasonAnnotationKey, reason)
audit.AddAuditAnnotations(ctx,
decisionAnnotationKey, decisionForbid,
reasonAnnotationKey, reason)
responsewriters.Forbidden(ctx, attributes, w, req, reason, s)
})
}

View File

@@ -67,6 +67,7 @@ func WithImpersonation(handler http.Handler, a authorizer.Authorizer, s runtime.
username := ""
groups := []string{}
userExtra := map[string][]string{}
uid := ""
for _, impersonationRequest := range impersonationRequests {
gvk := impersonationRequest.GetObjectKind().GroupVersionKind()
actingAsAttributes := &authorizer.AttributesRecord{
@@ -103,6 +104,10 @@ func WithImpersonation(handler http.Handler, a authorizer.Authorizer, s runtime.
actingAsAttributes.Subresource = extraKey
userExtra[extraKey] = append(userExtra[extraKey], extraValue)
case authenticationv1.SchemeGroupVersion.WithKind("UID").GroupKind():
uid = string(impersonationRequest.Name)
actingAsAttributes.Resource = "uids"
default:
klog.V(4).InfoS("unknown impersonation request type", "Request", impersonationRequest)
responsewriters.Forbidden(ctx, actingAsAttributes, w, req, fmt.Sprintf("unknown impersonation request type: %v", impersonationRequest), s)
@@ -154,18 +159,20 @@ func WithImpersonation(handler http.Handler, a authorizer.Authorizer, s runtime.
Name: username,
Groups: groups,
Extra: userExtra,
UID: uid,
}
req = req.WithContext(request.WithUser(ctx, newUser))
oldUser, _ := request.UserFrom(ctx)
httplog.LogOf(req, w).Addf("%v is acting as %v", oldUser, newUser)
ae := request.AuditEventFrom(ctx)
ae := audit.AuditEventFrom(ctx)
audit.LogImpersonatedUser(ae, newUser)
// clear all the impersonation headers from the request
req.Header.Del(authenticationv1.ImpersonateUserHeader)
req.Header.Del(authenticationv1.ImpersonateGroupHeader)
req.Header.Del(authenticationv1.ImpersonateUIDHeader)
for headerName := range req.Header {
if strings.HasPrefix(headerName, authenticationv1.ImpersonateUserExtraHeaderPrefix) {
req.Header.Del(headerName)
@@ -231,7 +238,17 @@ func buildImpersonationRequests(headers http.Header) ([]v1.ObjectReference, erro
}
}
if (hasGroups || hasUserExtra) && !hasUser {
requestedUID := headers.Get(authenticationv1.ImpersonateUIDHeader)
hasUID := len(requestedUID) > 0
if hasUID {
impersonationRequests = append(impersonationRequests, v1.ObjectReference{
Kind: "UID",
Name: requestedUID,
APIVersion: authenticationv1.SchemeGroupVersion.String(),
})
}
if (hasGroups || hasUserExtra || hasUID) && !hasUser {
return nil, fmt.Errorf("requested %v without impersonating a user", impersonationRequests)
}

View File

@@ -28,7 +28,7 @@ import (
/*
* By default, all the following metrics are defined as falling under
* ALPHA stability level https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/1209-metrics-stability/20190404-kubernetes-control-plane-metrics-stability.md#stability-classes)
* ALPHA stability level https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/1209-metrics-stability/kubernetes-control-plane-metrics-stability.md#stability-classes)
*
* Promoting the stability level of the metric is a responsibility of the component owner, since it
* involves explicitly acknowledging support for the metric across multiple releases, in accordance with

View File

@@ -0,0 +1,64 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package filters
import (
"context"
"net/http"
)
type muxAndDiscoveryIncompleteKeyType int
const (
// muxAndDiscoveryIncompleteKey is a key under which a protection signal for all requests made before the server have installed all known HTTP paths is stored in the request's context
muxAndDiscoveryIncompleteKey muxAndDiscoveryIncompleteKeyType = iota
)
// NoMuxAndDiscoveryIncompleteKey checks if the context contains muxAndDiscoveryIncompleteKey.
// The presence of the key indicates the request has been made when the HTTP paths weren't installed.
func NoMuxAndDiscoveryIncompleteKey(ctx context.Context) bool {
muxAndDiscoveryCompleteProtectionKeyValue, _ := ctx.Value(muxAndDiscoveryIncompleteKey).(string)
return len(muxAndDiscoveryCompleteProtectionKeyValue) == 0
}
// WithMuxAndDiscoveryComplete puts the muxAndDiscoveryIncompleteKey in the context if a request has been made before muxAndDiscoveryCompleteSignal has been ready.
// Putting the key protect us from returning a 404 response instead of a 503.
// It is especially important for controllers like GC and NS since they act on 404s.
//
// The presence of the key is checked in the NotFoundHandler (staging/src/k8s.io/apiserver/pkg/util/notfoundhandler/not_found_handler.go)
//
// The primary reason this filter exists is to protect from a potential race between the client's requests reaching the NotFoundHandler and the server becoming ready.
// Without the protection key a request could still get a 404 response when the registered signals changed their status just slightly before reaching the new handler.
// In that case, the presence of the key will make the handler return a 503 instead of a 404.
func WithMuxAndDiscoveryComplete(handler http.Handler, muxAndDiscoveryCompleteSignal <-chan struct{}) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
if muxAndDiscoveryCompleteSignal != nil && !isClosed(muxAndDiscoveryCompleteSignal) {
req = req.WithContext(context.WithValue(req.Context(), muxAndDiscoveryIncompleteKey, "MuxAndDiscoveryInstallationNotComplete"))
}
handler.ServeHTTP(w, req)
})
}
// isClosed is a convenience function that simply check if the given chan has been closed
func isClosed(ch <-chan struct{}) bool {
select {
case <-ch:
return true
default:
return false
}
}

View File

@@ -27,14 +27,13 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
utilclock "k8s.io/apimachinery/pkg/util/clock"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
auditinternal "k8s.io/apiserver/pkg/apis/audit"
"k8s.io/apiserver/pkg/audit"
"k8s.io/apiserver/pkg/audit/policy"
"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/klog/v2"
"k8s.io/utils/clock"
)
const (
@@ -47,13 +46,13 @@ const (
// auditWrapper provides an http.Handler that audits a failed request.
// longRunning returns true if he given request is a long running request.
// requestTimeoutMaximum specifies the default request timeout value.
func WithRequestDeadline(handler http.Handler, sink audit.Sink, policy policy.Checker, longRunning request.LongRunningRequestCheck,
func WithRequestDeadline(handler http.Handler, sink audit.Sink, policy audit.PolicyRuleEvaluator, longRunning request.LongRunningRequestCheck,
negotiatedSerializer runtime.NegotiatedSerializer, requestTimeoutMaximum time.Duration) http.Handler {
return withRequestDeadline(handler, sink, policy, longRunning, negotiatedSerializer, requestTimeoutMaximum, utilclock.RealClock{})
return withRequestDeadline(handler, sink, policy, longRunning, negotiatedSerializer, requestTimeoutMaximum, clock.RealClock{})
}
func withRequestDeadline(handler http.Handler, sink audit.Sink, policy policy.Checker, longRunning request.LongRunningRequestCheck,
negotiatedSerializer runtime.NegotiatedSerializer, requestTimeoutMaximum time.Duration, clock utilclock.PassiveClock) http.Handler {
func withRequestDeadline(handler http.Handler, sink audit.Sink, policy audit.PolicyRuleEvaluator, longRunning request.LongRunningRequestCheck,
negotiatedSerializer runtime.NegotiatedSerializer, requestTimeoutMaximum time.Duration, clock clock.PassiveClock) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
ctx := req.Context()
@@ -104,29 +103,33 @@ func withRequestDeadline(handler http.Handler, sink audit.Sink, policy policy.Ch
// withFailedRequestAudit decorates a failed http.Handler and is used to audit a failed request.
// statusErr is used to populate the Message property of ResponseStatus.
func withFailedRequestAudit(failedHandler http.Handler, statusErr *apierrors.StatusError, sink audit.Sink, policy policy.Checker) http.Handler {
func withFailedRequestAudit(failedHandler http.Handler, statusErr *apierrors.StatusError, sink audit.Sink, policy audit.PolicyRuleEvaluator) http.Handler {
if sink == nil || policy == nil {
return failedHandler
}
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
req, ev, omitStages, err := createAuditEventAndAttachToContext(req, policy)
a, err := evaluatePolicyAndCreateAuditEvent(req, policy)
if err != nil {
utilruntime.HandleError(fmt.Errorf("failed to create audit event: %v", err))
responsewriters.InternalError(w, req, errors.New("failed to create audit event"))
return
}
ev := a.Event
if ev == nil {
failedHandler.ServeHTTP(w, req)
return
}
req = req.WithContext(audit.WithAuditContext(req.Context(), a))
ev.ResponseStatus = &metav1.Status{}
ev.Stage = auditinternal.StageResponseStarted
if statusErr != nil {
ev.ResponseStatus.Message = statusErr.Error()
}
rw := decorateResponseWriter(req.Context(), w, ev, sink, omitStages)
rw := decorateResponseWriter(req.Context(), w, ev, sink, a.RequestAuditConfig.OmitStages)
failedHandler.ServeHTTP(rw, req)
})
}

View File

@@ -19,18 +19,18 @@ package filters
import (
"net/http"
utilclock "k8s.io/apimachinery/pkg/util/clock"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/utils/clock"
)
// WithRequestReceivedTimestamp attaches the ReceivedTimestamp (the time the request reached
// the apiserver) to the context.
func WithRequestReceivedTimestamp(handler http.Handler) http.Handler {
return withRequestReceivedTimestampWithClock(handler, utilclock.RealClock{})
return withRequestReceivedTimestampWithClock(handler, clock.RealClock{})
}
// The clock is passed as a parameter, handy for unit testing.
func withRequestReceivedTimestampWithClock(handler http.Handler, clock utilclock.PassiveClock) http.Handler {
func withRequestReceivedTimestampWithClock(handler http.Handler, clock clock.PassiveClock) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
ctx := req.Context()
req = req.WithContext(request.WithReceivedTimestamp(ctx, clock.Now()))

View File

@@ -0,0 +1,38 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package filters
import (
"net/http"
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
"go.opentelemetry.io/otel/trace"
tracing "k8s.io/component-base/tracing"
)
// WithTracing adds tracing to requests if the incoming request is sampled
func WithTracing(handler http.Handler, tp trace.TracerProvider) http.Handler {
opts := []otelhttp.Option{
otelhttp.WithPropagators(tracing.Propagators()),
otelhttp.WithPublicEndpoint(),
otelhttp.WithTracerProvider(tp),
}
// With Noop TracerProvider, the otelhttp still handles context propagation.
// See https://github.com/open-telemetry/opentelemetry-go/tree/main/example/passthrough
return otelhttp.NewHandler(handler, "KubernetesAPI", opts...)
}

View File

@@ -0,0 +1,80 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package filters
import (
"context"
"fmt"
"net/http"
"time"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/endpoints/responsewriter"
)
var (
watchVerbs = sets.NewString("watch")
)
// WithLatencyTrackers adds a LatencyTrackers instance to the
// context associated with a request so that we can measure latency
// incurred in various components within the apiserver.
func WithLatencyTrackers(handler http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
ctx := req.Context()
requestInfo, ok := request.RequestInfoFrom(ctx)
if !ok {
handleError(w, req, http.StatusInternalServerError, fmt.Errorf("no RequestInfo found in context, handler chain must be wrong"))
return
}
if watchVerbs.Has(requestInfo.Verb) {
handler.ServeHTTP(w, req)
return
}
req = req.WithContext(request.WithLatencyTrackers(ctx))
w = responsewriter.WrapForHTTP1Or2(&writeLatencyTracker{
ResponseWriter: w,
ctx: req.Context(),
})
handler.ServeHTTP(w, req)
})
}
var _ http.ResponseWriter = &writeLatencyTracker{}
var _ responsewriter.UserProvidedDecorator = &writeLatencyTracker{}
type writeLatencyTracker struct {
http.ResponseWriter
ctx context.Context
}
func (wt *writeLatencyTracker) Unwrap() http.ResponseWriter {
return wt.ResponseWriter
}
func (wt *writeLatencyTracker) Write(bs []byte) (int, error) {
startedAt := time.Now()
defer func() {
request.TrackResponseWriteLatency(wt.ctx, time.Since(startedAt))
}()
return wt.ResponseWriter.Write(bs)
}

View File

@@ -0,0 +1,68 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package filters
import (
"net/http"
"k8s.io/apimachinery/pkg/types"
auditinternal "k8s.io/apiserver/pkg/apis/audit"
"k8s.io/apiserver/pkg/endpoints/request"
"github.com/google/uuid"
)
// WithAuditID attaches the Audit-ID associated with a request to the context.
//
// a. If the caller does not specify a value for Audit-ID in the request header, we generate a new audit ID
// b. We echo the Audit-ID value to the caller via the response Header 'Audit-ID'.
func WithAuditID(handler http.Handler) http.Handler {
return withAuditID(handler, func() string {
return uuid.New().String()
})
}
func withAuditID(handler http.Handler, newAuditIDFunc func() string) http.Handler {
if newAuditIDFunc == nil {
return handler
}
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
auditID := r.Header.Get(auditinternal.HeaderAuditID)
if len(auditID) == 0 {
auditID = newAuditIDFunc()
}
// Note: we save the user specified value of the Audit-ID header as is, no truncation is performed.
r = r.WithContext(request.WithAuditID(ctx, types.UID(auditID)))
// We echo the Audit-ID in to the response header.
// It's not guaranteed Audit-ID http header is sent for all requests.
// For example, when user run "kubectl exec", apiserver uses a proxy handler
// to deal with the request, users can only get http headers returned by kubelet node.
//
// This filter will also be used by other aggregated api server(s). For an aggregated API
// we don't want to see the same audit ID appearing more than once.
if value := w.Header().Get(auditinternal.HeaderAuditID); len(value) == 0 {
w.Header().Set(auditinternal.HeaderAuditID, auditID)
}
handler.ServeHTTP(w, r)
})
}

View File

@@ -20,7 +20,7 @@ import (
"path"
"time"
restful "github.com/emicklei/go-restful"
restful "github.com/emicklei/go-restful/v3"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
@@ -79,7 +79,7 @@ type APIGroupVersion struct {
Convertor runtime.ObjectConvertor
ConvertabilityChecker ConvertabilityChecker
Defaulter runtime.ObjectDefaulter
Linker runtime.SelfLinker
Namer runtime.Namer
UnsafeConvertor runtime.ObjectConvertor
TypeConverter fieldmanager.TypeConverter

View File

@@ -36,16 +36,18 @@ import (
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/audit"
"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
"k8s.io/apiserver/pkg/endpoints/handlers/finisher"
"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/features"
"k8s.io/apiserver/pkg/registry/rest"
"k8s.io/apiserver/pkg/util/dryrun"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/klog/v2"
utiltrace "k8s.io/utils/trace"
)
var namespaceGVK = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Namespace"}
var namespaceGVR = schema.GroupVersionResource{Group: "", Version: "v1", Resource: "namespaces"}
func createHandler(r rest.NamedCreater, scope *RequestScope, admit admission.Interface, includeName bool) http.HandlerFunc {
return func(w http.ResponseWriter, req *http.Request) {
@@ -91,9 +93,8 @@ func createHandler(r rest.NamedCreater, scope *RequestScope, admit admission.Int
return
}
decoder := scope.Serializer.DecoderToVersion(s.Serializer, scope.HubGroupVersion)
body, err := limitedReadBody(req, scope.MaxRequestBodyBytes)
trace.Step("limitedReadBody done", utiltrace.Field{"len", len(body)}, utiltrace.Field{"err", err})
if err != nil {
scope.err(err, w, req)
return
@@ -115,15 +116,34 @@ func createHandler(r rest.NamedCreater, scope *RequestScope, admit admission.Int
defaultGVK := scope.Kind
original := r.New()
validationDirective := fieldValidation(options.FieldValidation)
decodeSerializer := s.Serializer
if validationDirective == metav1.FieldValidationWarn || validationDirective == metav1.FieldValidationStrict {
decodeSerializer = s.StrictSerializer
}
decoder := scope.Serializer.DecoderToVersion(decodeSerializer, scope.HubGroupVersion)
trace.Step("About to convert to expected version")
obj, gvk, err := decoder.Decode(body, &defaultGVK, original)
if err != nil {
err = transformDecodeError(scope.Typer, err, original, gvk, body)
scope.err(err, w, req)
return
strictError, isStrictError := runtime.AsStrictDecodingError(err)
switch {
case isStrictError && obj != nil && validationDirective == metav1.FieldValidationWarn:
addStrictDecodingWarnings(req.Context(), strictError.Errors())
case isStrictError && validationDirective == metav1.FieldValidationIgnore:
klog.Warningf("unexpected strict error when field validation is set to ignore")
fallthrough
default:
err = transformDecodeError(scope.Typer, err, original, gvk, body)
scope.err(err, w, req)
return
}
}
if gvk.GroupVersion() != gv {
err = errors.NewBadRequest(fmt.Sprintf("the API version in the data (%s) does not match the expected API version (%v)", gvk.GroupVersion().String(), gv.String()))
objGV := gvk.GroupVersion()
if !scope.AcceptsGroupVersion(objGV) {
err = errors.NewBadRequest(fmt.Sprintf("the API version in the data (%s) does not match the expected API version (%v)", objGV.String(), gv.String()))
scope.err(err, w, req)
return
}
@@ -133,17 +153,27 @@ func createHandler(r rest.NamedCreater, scope *RequestScope, admit admission.Int
if len(name) == 0 {
_, name, _ = scope.Namer.ObjectName(obj)
}
if len(namespace) == 0 && *gvk == namespaceGVK {
if len(namespace) == 0 && scope.Resource == namespaceGVR {
namespace = name
}
ctx = request.WithNamespace(ctx, namespace)
ae := request.AuditEventFrom(ctx)
admit = admission.WithAudit(admit, ae)
audit.LogRequestObject(ae, obj, scope.Resource, scope.Subresource, scope.Serializer)
admit = admission.WithAudit(admit)
audit.LogRequestObject(req.Context(), obj, objGV, scope.Resource, scope.Subresource, scope.Serializer)
userInfo, _ := request.UserFrom(ctx)
if objectMeta, err := meta.Accessor(obj); err == nil {
// Wipe fields which cannot take user-provided values
rest.WipeObjectMetaSystemFields(objectMeta)
// ensure namespace on the object is correct, or error if a conflicting namespace was set in the object
if err := rest.EnsureObjectNamespaceMatchesRequestNamespace(rest.ExpectedNamespaceForResource(namespace, scope.Resource), objectMeta); err != nil {
scope.err(err, w, req)
return
}
}
trace.Step("About to store object in database")
admissionAttributes := admission.NewAttributesRecord(obj, nil, scope.Kind, namespace, name, scope.Resource, scope.Subresource, admission.Create, options, dryrun.IsDryRun(options.DryRun), userInfo)
requestFunc := func() (runtime.Object, error) {
@@ -157,7 +187,7 @@ func createHandler(r rest.NamedCreater, scope *RequestScope, admit admission.Int
}
// Dedup owner references before updating managed fields
dedupOwnerReferencesAndAddWarning(obj, req.Context(), false)
result, err := finishRequest(ctx, func() (runtime.Object, error) {
result, err := finisher.FinishRequest(ctx, func() (runtime.Object, error) {
if scope.FieldManager != nil {
liveObj, err := scope.Creater.New(scope.Kind)
if err != nil {
@@ -184,18 +214,20 @@ func createHandler(r rest.NamedCreater, scope *RequestScope, admit admission.Int
}
return result, err
})
trace.Step("Write to database call finished", utiltrace.Field{"len", len(body)}, utiltrace.Field{"err", err})
if err != nil {
scope.err(err, w, req)
return
}
trace.Step("Object stored in database")
code := http.StatusCreated
status, ok := result.(*metav1.Status)
if ok && err == nil && status.Code == 0 {
if ok && status.Code == 0 {
status.Code = int32(code)
}
trace.Step("About to write a response")
defer trace.Step("Writing http response done")
transformResponseObject(ctx, scope, trace, req, w, code, outputMediaType, result)
}
}

View File

@@ -32,6 +32,7 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/audit"
"k8s.io/apiserver/pkg/endpoints/handlers/finisher"
"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/features"
@@ -66,8 +67,7 @@ func DeleteResource(r rest.GracefulDeleter, allowsOptions bool, scope *RequestSc
defer cancel()
ctx = request.WithNamespace(ctx, namespace)
ae := request.AuditEventFrom(ctx)
admit = admission.WithAudit(admit, ae)
admit = admission.WithAudit(admit)
outputMediaType, _, err := negotiation.NegotiateOutputMediaType(req, scope.Serializer, scope)
if err != nil {
@@ -91,7 +91,7 @@ func DeleteResource(r rest.GracefulDeleter, allowsOptions bool, scope *RequestSc
// For backwards compatibility, we need to allow existing clients to submit per group DeleteOptions
// It is also allowed to pass a body with meta.k8s.io/v1.DeleteOptions
defaultGVK := scope.MetaGroupVersion.WithKind("DeleteOptions")
obj, _, err := metainternalversionscheme.Codecs.DecoderToVersion(s.Serializer, defaultGVK.GroupVersion()).Decode(body, &defaultGVK, options)
obj, gvk, err := metainternalversionscheme.Codecs.DecoderToVersion(s.Serializer, defaultGVK.GroupVersion()).Decode(body, &defaultGVK, options)
if err != nil {
scope.err(err, w, req)
return
@@ -102,8 +102,8 @@ func DeleteResource(r rest.GracefulDeleter, allowsOptions bool, scope *RequestSc
}
trace.Step("Decoded delete options")
ae := request.AuditEventFrom(ctx)
audit.LogRequestObject(ae, obj, scope.Resource, scope.Subresource, scope.Serializer)
objGV := gvk.GroupVersion()
audit.LogRequestObject(req.Context(), obj, objGV, scope.Resource, scope.Subresource, metainternalversionscheme.Codecs)
trace.Step("Recorded the audit event")
} else {
if err := metainternalversionscheme.ParameterCodec.DecodeParameters(req.URL.Query(), scope.MetaGroupVersion, options); err != nil {
@@ -124,7 +124,7 @@ func DeleteResource(r rest.GracefulDeleter, allowsOptions bool, scope *RequestSc
wasDeleted := true
userInfo, _ := request.UserFrom(ctx)
staticAdmissionAttrs := admission.NewAttributesRecord(nil, nil, scope.Kind, namespace, name, scope.Resource, scope.Subresource, admission.Delete, options, dryrun.IsDryRun(options.DryRun), userInfo)
result, err := finishRequest(ctx, func() (runtime.Object, error) {
result, err := finisher.FinishRequest(ctx, func() (runtime.Object, error) {
obj, deleted, err := r.Delete(ctx, name, rest.AdmissionToValidateObjectDeleteFunc(admit, staticAdmissionAttrs, scope), options)
wasDeleted = deleted
return obj, err
@@ -142,7 +142,8 @@ func DeleteResource(r rest.GracefulDeleter, allowsOptions bool, scope *RequestSc
// that will break existing clients.
// Other cases where resource is not instantly deleted are: namespace deletion
// and pod graceful deletion.
//lint:ignore SA1019 backwards compatibility
//nolint:staticcheck // SA1019 backwards compatibility
//nolint: staticcheck
if !wasDeleted && options.OrphanDependents != nil && !*options.OrphanDependents {
status = http.StatusAccepted
}
@@ -159,6 +160,8 @@ func DeleteResource(r rest.GracefulDeleter, allowsOptions bool, scope *RequestSc
}
}
trace.Step("About to write a response")
defer trace.Step("Writing http response done")
transformResponseObject(ctx, scope, trace, req, w, status, outputMediaType, result)
}
}
@@ -186,7 +189,6 @@ func DeleteCollection(r rest.CollectionDeleter, checkBody bool, scope *RequestSc
defer cancel()
ctx = request.WithNamespace(ctx, namespace)
ae := request.AuditEventFrom(ctx)
outputMediaType, _, err := negotiation.NegotiateOutputMediaType(req, scope.Serializer, scope)
if err != nil {
@@ -236,8 +238,8 @@ func DeleteCollection(r rest.CollectionDeleter, checkBody bool, scope *RequestSc
}
// For backwards compatibility, we need to allow existing clients to submit per group DeleteOptions
// It is also allowed to pass a body with meta.k8s.io/v1.DeleteOptions
defaultGVK := scope.Kind.GroupVersion().WithKind("DeleteOptions")
obj, _, err := scope.Serializer.DecoderToVersion(s.Serializer, defaultGVK.GroupVersion()).Decode(body, &defaultGVK, options)
defaultGVK := scope.MetaGroupVersion.WithKind("DeleteOptions")
obj, gvk, err := metainternalversionscheme.Codecs.DecoderToVersion(s.Serializer, defaultGVK.GroupVersion()).Decode(body, &defaultGVK, options)
if err != nil {
scope.err(err, w, req)
return
@@ -247,8 +249,8 @@ func DeleteCollection(r rest.CollectionDeleter, checkBody bool, scope *RequestSc
return
}
ae := request.AuditEventFrom(ctx)
audit.LogRequestObject(ae, obj, scope.Resource, scope.Subresource, scope.Serializer)
objGV := gvk.GroupVersion()
audit.LogRequestObject(req.Context(), obj, objGV, scope.Resource, scope.Subresource, metainternalversionscheme.Codecs)
} else {
if err := metainternalversionscheme.ParameterCodec.DecodeParameters(req.URL.Query(), scope.MetaGroupVersion, options); err != nil {
err = errors.NewBadRequest(err.Error())
@@ -264,10 +266,10 @@ func DeleteCollection(r rest.CollectionDeleter, checkBody bool, scope *RequestSc
}
options.TypeMeta.SetGroupVersionKind(metav1.SchemeGroupVersion.WithKind("DeleteOptions"))
admit = admission.WithAudit(admit, ae)
admit = admission.WithAudit(admit)
userInfo, _ := request.UserFrom(ctx)
staticAdmissionAttrs := admission.NewAttributesRecord(nil, nil, scope.Kind, namespace, "", scope.Resource, scope.Subresource, admission.Delete, options, dryrun.IsDryRun(options.DryRun), userInfo)
result, err := finishRequest(ctx, func() (runtime.Object, error) {
result, err := finisher.FinishRequest(ctx, func() (runtime.Object, error) {
return r.DeleteCollection(ctx, rest.AdmissionToValidateObjectDeleteFunc(admit, staticAdmissionAttrs, scope), options, &listOptions)
})
if err != nil {
@@ -287,6 +289,8 @@ func DeleteCollection(r rest.CollectionDeleter, checkBody bool, scope *RequestSc
}
}
trace.Step("About to write a response")
defer trace.Step("Writing http response done")
transformResponseObject(ctx, scope, trace, req, w, http.StatusOK, outputMediaType, result)
}
}

View File

@@ -1,5 +1,6 @@
approvers:
- jennybuckley
- apelisse
- apelisse
reviewers:
- kwiesmueller
- kwiesmueller
emeritus_approvers:
- jennybuckley

View File

@@ -60,7 +60,10 @@ func (admit *managedFieldsValidatingAdmissionController) Admit(ctx context.Conte
}
objectMeta, err := meta.Accessor(a.GetObject())
if err != nil {
return err
// the object we are dealing with doesn't have object metadata defined
// in that case we don't have to keep track of the managedField
// just call the wrapped admission
return mutationInterface.Admit(ctx, a, o)
}
managedFieldsBeforeAdmission := objectMeta.GetManagedFields()
if err := mutationInterface.Admit(ctx, a, o); err != nil {

View File

@@ -28,16 +28,18 @@ import (
type buildManagerInfoManager struct {
fieldManager Manager
groupVersion schema.GroupVersion
subresource string
}
var _ Manager = &buildManagerInfoManager{}
// NewBuildManagerInfoManager creates a new Manager that converts the manager name into a unique identifier
// combining operation and version for update requests, and just operation for apply requests.
func NewBuildManagerInfoManager(f Manager, gv schema.GroupVersion) Manager {
func NewBuildManagerInfoManager(f Manager, gv schema.GroupVersion, subresource string) Manager {
return &buildManagerInfoManager{
fieldManager: f,
groupVersion: gv,
subresource: subresource,
}
}
@@ -61,9 +63,10 @@ func (f *buildManagerInfoManager) Apply(liveObj, appliedObj runtime.Object, mana
func (f *buildManagerInfoManager) buildManagerInfo(prefix string, operation metav1.ManagedFieldsOperationType) (string, error) {
managerInfo := metav1.ManagedFieldsEntry{
Manager: prefix,
Operation: operation,
APIVersion: f.groupVersion.String(),
Manager: prefix,
Operation: operation,
APIVersion: f.groupVersion.String(),
Subresource: f.subresource,
}
if managerInfo.Manager == "" {
managerInfo.Manager = "unknown"

View File

@@ -0,0 +1,180 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package fieldmanager
import (
"context"
"fmt"
"os"
"reflect"
"strconv"
"time"
"k8s.io/apimachinery/pkg/api/equality"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/conversion"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apiserver/pkg/endpoints/metrics"
"k8s.io/klog/v2"
)
func determineAvoidNoopTimestampUpdatesEnabled() bool {
if avoidNoopTimestampUpdatesString, exists := os.LookupEnv("KUBE_APISERVER_AVOID_NOOP_SSA_TIMESTAMP_UPDATES"); exists {
if ret, err := strconv.ParseBool(avoidNoopTimestampUpdatesString); err == nil {
return ret
} else {
klog.Errorf("failed to parse envar KUBE_APISERVER_AVOID_NOOP_SSA_TIMESTAMP_UPDATES: %v", err)
}
}
// enabled by default
return true
}
var (
avoidNoopTimestampUpdatesEnabled = determineAvoidNoopTimestampUpdatesEnabled()
)
var avoidTimestampEqualities = func() conversion.Equalities {
var eqs = equality.Semantic.Copy()
err := eqs.AddFunc(
func(a, b metav1.ManagedFieldsEntry) bool {
// Two objects' managed fields are equivalent if, ignoring timestamp,
// the objects are deeply equal.
a.Time = nil
b.Time = nil
return reflect.DeepEqual(a, b)
},
)
if err != nil {
panic(err)
}
return eqs
}()
// IgnoreManagedFieldsTimestampsTransformer reverts timestamp updates
// if the non-managed parts of the object are equivalent
func IgnoreManagedFieldsTimestampsTransformer(
_ context.Context,
newObj runtime.Object,
oldObj runtime.Object,
) (res runtime.Object, err error) {
if !avoidNoopTimestampUpdatesEnabled {
return newObj, nil
}
outcome := "unequal_objects_fast"
start := time.Now()
err = nil
res = nil
defer func() {
if err != nil {
outcome = "error"
}
metrics.RecordTimestampComparisonLatency(outcome, time.Since(start))
}()
// If managedFields modulo timestamps are unchanged
// and
// rest of object is unchanged
// then
// revert any changes to timestamps in managed fields
// (to prevent spurious ResourceVersion bump)
//
// Procecure:
// Do a quicker check to see if just managed fields modulo timestamps are
// unchanged. If so, then do the full, slower check.
//
// In most cases which actually update the object, the managed fields modulo
// timestamp check will fail, and we will be able to return early.
//
// In other cases, the managed fields may be exactly the same,
// except for timestamp, but the objects are the different. This is the
// slow path which checks the full object.
oldAccessor, err := meta.Accessor(oldObj)
if err != nil {
return nil, fmt.Errorf("failed to acquire accessor for oldObj: %v", err)
}
accessor, err := meta.Accessor(newObj)
if err != nil {
return nil, fmt.Errorf("failed to acquire accessor for newObj: %v", err)
}
oldManagedFields := oldAccessor.GetManagedFields()
newManagedFields := accessor.GetManagedFields()
if len(oldManagedFields) != len(newManagedFields) {
// Return early if any managed fields entry was added/removed.
// We want to retain user expectation that even if they write to a field
// whose value did not change, they will still result as the field
// manager at the end.
return newObj, nil
} else if len(newManagedFields) == 0 {
// This transformation only makes sense when managedFields are
// non-empty
return newObj, nil
}
// This transformation only makes sense if the managed fields has at least one
// changed timestamp; and are otherwise equal. Return early if there are no
// changed timestamps.
allTimesUnchanged := true
for i, e := range newManagedFields {
if !e.Time.Equal(oldManagedFields[i].Time) {
allTimesUnchanged = false
break
}
}
if allTimesUnchanged {
return newObj, nil
}
// This condition ensures the managed fields are always compared first. If
// this check fails, the if statement will short circuit. If the check
// succeeds the slow path is taken which compares entire objects.
if !avoidTimestampEqualities.DeepEqualWithNilDifferentFromEmpty(oldManagedFields, newManagedFields) {
return newObj, nil
}
if avoidTimestampEqualities.DeepEqualWithNilDifferentFromEmpty(newObj, oldObj) {
// Remove any changed timestamps, so that timestamp is not the only
// change seen by etcd.
//
// newManagedFields is known to be exactly pairwise equal to
// oldManagedFields except for timestamps.
//
// Simply replace possibly changed new timestamps with their old values.
for idx := 0; idx < len(oldManagedFields); idx++ {
newManagedFields[idx].Time = oldManagedFields[idx].Time
}
accessor.SetManagedFields(newManagedFields)
outcome = "equal_objects"
return newObj, nil
}
outcome = "unequal_objects_slow"
return newObj, nil
}

View File

@@ -56,58 +56,71 @@ type Manager interface {
// Update is used when the object has already been merged (non-apply
// use-case), and simply updates the managed fields in the output
// object.
// * `liveObj` is not mutated by this function
// * `newObj` may be mutated by this function
// Returns the new object with managedFields removed, and the object's new
// proposed managedFields separately.
Update(liveObj, newObj runtime.Object, managed Managed, manager string) (runtime.Object, Managed, error)
// Apply is used when server-side apply is called, as it merges the
// object and updates the managed fields.
// * `liveObj` is not mutated by this function
// * `newObj` may be mutated by this function
// Returns the new object with managedFields removed, and the object's new
// proposed managedFields separately.
Apply(liveObj, appliedObj runtime.Object, managed Managed, fieldManager string, force bool) (runtime.Object, Managed, error)
}
// FieldManager updates the managed fields and merge applied
// configurations.
type FieldManager struct {
fieldManager Manager
ignoreManagedFieldsFromRequestObject bool
fieldManager Manager
subresource string
}
// NewFieldManager creates a new FieldManager that decodes, manages, then re-encodes managedFields
// on update and apply requests.
func NewFieldManager(f Manager, ignoreManagedFieldsFromRequestObject bool) *FieldManager {
return &FieldManager{fieldManager: f, ignoreManagedFieldsFromRequestObject: ignoreManagedFieldsFromRequestObject}
func NewFieldManager(f Manager, subresource string) *FieldManager {
return &FieldManager{fieldManager: f, subresource: subresource}
}
// NewDefaultFieldManager creates a new FieldManager that merges apply requests
// and update managed fields for other types of requests.
func NewDefaultFieldManager(typeConverter TypeConverter, objectConverter runtime.ObjectConvertor, objectDefaulter runtime.ObjectDefaulter, objectCreater runtime.ObjectCreater, kind schema.GroupVersionKind, hub schema.GroupVersion, ignoreManagedFieldsFromRequestObject bool, resetFields map[fieldpath.APIVersion]*fieldpath.Set) (*FieldManager, error) {
func NewDefaultFieldManager(typeConverter TypeConverter, objectConverter runtime.ObjectConvertor, objectDefaulter runtime.ObjectDefaulter, objectCreater runtime.ObjectCreater, kind schema.GroupVersionKind, hub schema.GroupVersion, subresource string, resetFields map[fieldpath.APIVersion]*fieldpath.Set) (*FieldManager, error) {
f, err := NewStructuredMergeManager(typeConverter, objectConverter, objectDefaulter, kind.GroupVersion(), hub, resetFields)
if err != nil {
return nil, fmt.Errorf("failed to create field manager: %v", err)
}
return newDefaultFieldManager(f, typeConverter, objectConverter, objectCreater, kind, ignoreManagedFieldsFromRequestObject), nil
return newDefaultFieldManager(f, typeConverter, objectConverter, objectCreater, kind, subresource), nil
}
// NewDefaultCRDFieldManager creates a new FieldManager specifically for
// CRDs. This allows for the possibility of fields which are not defined
// in models, as well as having no models defined at all.
func NewDefaultCRDFieldManager(typeConverter TypeConverter, objectConverter runtime.ObjectConvertor, objectDefaulter runtime.ObjectDefaulter, objectCreater runtime.ObjectCreater, kind schema.GroupVersionKind, hub schema.GroupVersion, ignoreManagedFieldsFromRequestObject bool, resetFields map[fieldpath.APIVersion]*fieldpath.Set) (_ *FieldManager, err error) {
func NewDefaultCRDFieldManager(typeConverter TypeConverter, objectConverter runtime.ObjectConvertor, objectDefaulter runtime.ObjectDefaulter, objectCreater runtime.ObjectCreater, kind schema.GroupVersionKind, hub schema.GroupVersion, subresource string, resetFields map[fieldpath.APIVersion]*fieldpath.Set) (_ *FieldManager, err error) {
f, err := NewCRDStructuredMergeManager(typeConverter, objectConverter, objectDefaulter, kind.GroupVersion(), hub, resetFields)
if err != nil {
return nil, fmt.Errorf("failed to create field manager: %v", err)
}
return newDefaultFieldManager(f, typeConverter, objectConverter, objectCreater, kind, ignoreManagedFieldsFromRequestObject), nil
return newDefaultFieldManager(f, typeConverter, objectConverter, objectCreater, kind, subresource), nil
}
// newDefaultFieldManager is a helper function which wraps a Manager with certain default logic.
func newDefaultFieldManager(f Manager, typeConverter TypeConverter, objectConverter runtime.ObjectConvertor, objectCreater runtime.ObjectCreater, kind schema.GroupVersionKind, ignoreManagedFieldsFromRequestObject bool) *FieldManager {
f = NewStripMetaManager(f)
f = NewManagedFieldsUpdater(f)
f = NewBuildManagerInfoManager(f, kind.GroupVersion())
f = NewCapManagersManager(f, DefaultMaxUpdateManagers)
f = NewProbabilisticSkipNonAppliedManager(f, objectCreater, kind, DefaultTrackOnCreateProbability)
f = NewLastAppliedManager(f, typeConverter, objectConverter, kind.GroupVersion())
f = NewLastAppliedUpdater(f)
return NewFieldManager(f, ignoreManagedFieldsFromRequestObject)
func newDefaultFieldManager(f Manager, typeConverter TypeConverter, objectConverter runtime.ObjectConvertor, objectCreater runtime.ObjectCreater, kind schema.GroupVersionKind, subresource string) *FieldManager {
return NewFieldManager(
NewLastAppliedUpdater(
NewLastAppliedManager(
NewProbabilisticSkipNonAppliedManager(
NewCapManagersManager(
NewBuildManagerInfoManager(
NewManagedFieldsUpdater(
NewStripMetaManager(f),
), kind.GroupVersion(), subresource,
), DefaultMaxUpdateManagers,
), objectCreater, kind, DefaultTrackOnCreateProbability,
), typeConverter, objectConverter, kind.GroupVersion()),
), subresource,
)
}
// DecodeManagedFields converts ManagedFields from the wire format (api format)
@@ -162,12 +175,12 @@ func emptyManagedFieldsOnErr(managed Managed, err error) (Managed, error) {
func (f *FieldManager) Update(liveObj, newObj runtime.Object, manager string) (object runtime.Object, err error) {
// First try to decode the managed fields provided in the update,
// This is necessary to allow directly updating managed fields.
managed, err := decodeLiveOrNew(liveObj, newObj, f.ignoreManagedFieldsFromRequestObject)
isSubresource := f.subresource != ""
managed, err := decodeLiveOrNew(liveObj, newObj, isSubresource)
if err != nil {
return newObj, nil
}
internal.RemoveObjectManagedFields(liveObj)
internal.RemoveObjectManagedFields(newObj)
if object, managed, err = f.fieldManager.Update(liveObj, newObj, managed, manager); err != nil {
@@ -188,8 +201,15 @@ func (f *FieldManager) UpdateNoErrors(liveObj, newObj runtime.Object, manager st
obj, err := f.Update(liveObj, newObj, manager)
if err != nil {
atMostEverySecond.Do(func() {
ns, name := "unknown", "unknown"
accessor, err := meta.Accessor(newObj)
if err == nil {
ns = accessor.GetNamespace()
name = accessor.GetName()
}
klog.ErrorS(err, "[SHOULD NOT HAPPEN] failed to update managedFields", "VersionKind",
newObj.GetObjectKind().GroupVersionKind())
newObj.GetObjectKind().GroupVersionKind(), "namespace", ns, "name", name)
})
// Explicitly remove managedFields on failure, so that
// we can't have garbage in it.
@@ -229,8 +249,6 @@ func (f *FieldManager) Apply(liveObj, appliedObj runtime.Object, manager string,
return nil, fmt.Errorf("failed to decode managed fields: %v", err)
}
internal.RemoveObjectManagedFields(liveObj)
object, managed, err = f.fieldManager.Apply(liveObj, appliedObj, managed, manager, force)
if err != nil {
if conflicts, ok := err.(merge.Conflicts); ok {

View File

@@ -75,11 +75,15 @@ func printManager(manager string) string {
if err := json.Unmarshal([]byte(manager), encodedManager); err != nil {
return fmt.Sprintf("%q", manager)
}
managerStr := fmt.Sprintf("%q", encodedManager.Manager)
if encodedManager.Subresource != "" {
managerStr = fmt.Sprintf("%s with subresource %q", managerStr, encodedManager.Subresource)
}
if encodedManager.Operation == metav1.ManagedFieldsOperationUpdate {
if encodedManager.Time == nil {
return fmt.Sprintf("%q using %v", encodedManager.Manager, encodedManager.APIVersion)
return fmt.Sprintf("%s using %v", managerStr, encodedManager.APIVersion)
}
return fmt.Sprintf("%q using %v at %v", encodedManager.Manager, encodedManager.APIVersion, encodedManager.Time.UTC().Format(time.RFC3339))
return fmt.Sprintf("%s using %v at %v", managerStr, encodedManager.APIVersion, encodedManager.Time.UTC().Format(time.RFC3339))
}
return fmt.Sprintf("%q", encodedManager.Manager)
return managerStr
}

View File

@@ -1,127 +0,0 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package internal
import (
"fmt"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/kube-openapi/pkg/schemaconv"
"k8s.io/kube-openapi/pkg/util/proto"
"sigs.k8s.io/structured-merge-diff/v4/typed"
)
// groupVersionKindExtensionKey is the key used to lookup the
// GroupVersionKind value for an object definition from the
// definition's "extensions" map.
const groupVersionKindExtensionKey = "x-kubernetes-group-version-kind"
// GvkParser contains a Parser that allows introspecting the schema.
type GvkParser struct {
gvks map[schema.GroupVersionKind]string
parser typed.Parser
}
// Type returns a helper which can produce objects of the given type. Any
// errors are deferred until a further function is called.
func (p *GvkParser) Type(gvk schema.GroupVersionKind) *typed.ParseableType {
typeName, ok := p.gvks[gvk]
if !ok {
return nil
}
t := p.parser.Type(typeName)
return &t
}
// NewGVKParser builds a GVKParser from a proto.Models. This
// will automatically find the proper version of the object, and the
// corresponding schema information.
func NewGVKParser(models proto.Models, preserveUnknownFields bool) (*GvkParser, error) {
typeSchema, err := schemaconv.ToSchemaWithPreserveUnknownFields(models, preserveUnknownFields)
if err != nil {
return nil, fmt.Errorf("failed to convert models to schema: %v", err)
}
parser := GvkParser{
gvks: map[schema.GroupVersionKind]string{},
}
parser.parser = typed.Parser{Schema: *typeSchema}
for _, modelName := range models.ListModels() {
model := models.LookupModel(modelName)
if model == nil {
panic(fmt.Sprintf("ListModels returns a model that can't be looked-up for: %v", modelName))
}
gvkList := parseGroupVersionKind(model)
for _, gvk := range gvkList {
if len(gvk.Kind) > 0 {
_, ok := parser.gvks[gvk]
if ok {
return nil, fmt.Errorf("duplicate entry for %v", gvk)
}
parser.gvks[gvk] = modelName
}
}
}
return &parser, nil
}
// Get and parse GroupVersionKind from the extension. Returns empty if it doesn't have one.
func parseGroupVersionKind(s proto.Schema) []schema.GroupVersionKind {
extensions := s.GetExtensions()
gvkListResult := []schema.GroupVersionKind{}
// Get the extensions
gvkExtension, ok := extensions[groupVersionKindExtensionKey]
if !ok {
return []schema.GroupVersionKind{}
}
// gvk extension must be a list of at least 1 element.
gvkList, ok := gvkExtension.([]interface{})
if !ok {
return []schema.GroupVersionKind{}
}
for _, gvk := range gvkList {
// gvk extension list must be a map with group, version, and
// kind fields
gvkMap, ok := gvk.(map[interface{}]interface{})
if !ok {
continue
}
group, ok := gvkMap["group"].(string)
if !ok {
continue
}
version, ok := gvkMap["version"].(string)
if !ok {
continue
}
kind, ok := gvkMap["kind"].(string)
if !ok {
continue
}
gvkListResult = append(gvkListResult, schema.GroupVersionKind{
Group: group,
Version: version,
Kind: kind,
})
}
return gvkListResult
}

View File

@@ -213,7 +213,11 @@ func sortEncodedManagedFields(encodedManagedFields []metav1.ManagedFieldsEntry)
if p.Manager != q.Manager {
return p.Manager < q.Manager
}
return p.APIVersion < q.APIVersion
if p.APIVersion != q.APIVersion {
return p.APIVersion < q.APIVersion
}
return p.Subresource < q.Subresource
})
return encodedManagedFields, nil

View File

@@ -21,12 +21,11 @@ import (
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/meta"
apimachineryvalidation "k8s.io/apimachinery/pkg/api/validation"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
)
const totalAnnotationSizeLimitB int64 = 256 * (1 << 10) // 256 kB
type lastAppliedUpdater struct {
fieldManager Manager
}
@@ -94,7 +93,7 @@ func setLastApplied(obj runtime.Object, value string) error {
annotations = map[string]string{}
}
annotations[corev1.LastAppliedConfigAnnotation] = value
if isAnnotationsValid(annotations) != nil {
if err := apimachineryvalidation.ValidateAnnotationsSize(annotations); err != nil {
delete(annotations, corev1.LastAppliedConfigAnnotation)
}
accessor.SetAnnotations(annotations)
@@ -120,14 +119,3 @@ func buildLastApplied(obj runtime.Object) (string, error) {
}
return string(lastApplied), nil
}
func isAnnotationsValid(annotations map[string]string) error {
var totalSize int64
for k, v := range annotations {
totalSize += (int64)(len(k)) + (int64)(len(v))
}
if totalSize > (int64)(totalAnnotationSizeLimitB) {
return fmt.Errorf("annotations size %d is larger than limit %d", totalSize, totalAnnotationSizeLimitB)
}
return nil
}

View File

@@ -21,6 +21,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
)
@@ -44,7 +45,6 @@ func NewManagedFieldsUpdater(fieldManager Manager) Manager {
// Update implements Manager.
func (f *managedFieldsUpdater) Update(liveObj, newObj runtime.Object, managed Managed, manager string) (runtime.Object, Managed, error) {
self := "current-operation"
formerSet := managed.Fields()[manager]
object, managed, err := f.fieldManager.Update(liveObj, newObj, managed, self)
if err != nil {
return object, managed, err
@@ -60,10 +60,8 @@ func (f *managedFieldsUpdater) Update(liveObj, newObj runtime.Object, managed Ma
} else {
managed.Fields()[manager] = vs
}
// Update the time only if the manager's fieldSet has changed.
if formerSet == nil || !managed.Fields()[manager].Set().Equals(formerSet.Set()) {
managed.Times()[manager] = &metav1.Time{Time: time.Now().UTC()}
}
managed.Times()[manager] = &metav1.Time{Time: time.Now().UTC()}
}
return object, managed, nil
@@ -71,16 +69,15 @@ func (f *managedFieldsUpdater) Update(liveObj, newObj runtime.Object, managed Ma
// Apply implements Manager.
func (f *managedFieldsUpdater) Apply(liveObj, appliedObj runtime.Object, managed Managed, fieldManager string, force bool) (runtime.Object, Managed, error) {
formerManaged := managed.Fields().Copy()
object, managed, err := f.fieldManager.Apply(liveObj, appliedObj, managed, fieldManager, force)
if err != nil {
return object, managed, err
}
if object != nil || !managed.Fields().Equals(formerManaged) {
if object != nil {
managed.Times()[fieldManager] = &metav1.Time{Time: time.Now().UTC()}
}
if object == nil {
object = liveObj
} else {
object = liveObj.DeepCopyObject()
internal.RemoveObjectManagedFields(object)
}
return object, managed, nil
}

View File

@@ -133,8 +133,8 @@ status:
- grafana/grafana:4.4.2
sizeBytes: 287008013
- names:
- k8s.gcr.io/node-problem-detector@sha256:f95cab985c26b2f46e9bd43283e0bfa88860c14e0fb0649266babe8b65e9eb2b
- k8s.gcr.io/node-problem-detector:v0.4.1
- registry.k8s.io/node-problem-detector@sha256:f95cab985c26b2f46e9bd43283e0bfa88860c14e0fb0649266babe8b65e9eb2b
- registry.k8s.io/node-problem-detector:v0.4.1
sizeBytes: 286572743
- names:
- grafana/grafana@sha256:7ff7f9b2501a5d55b55ce3f58d21771b1c5af1f2a4ab7dbf11bef7142aae7033
@@ -153,76 +153,76 @@ status:
- nginx:1.10.1
sizeBytes: 180708613
- names:
- k8s.gcr.io/fluentd-elasticsearch@sha256:b8c94527b489fb61d3d81ce5ad7f3ddbb7be71e9620a3a36e2bede2f2e487d73
- k8s.gcr.io/fluentd-elasticsearch:v2.0.4
- registry.k8s.io/fluentd-elasticsearch@sha256:b8c94527b489fb61d3d81ce5ad7f3ddbb7be71e9620a3a36e2bede2f2e487d73
- registry.k8s.io/fluentd-elasticsearch:v2.0.4
sizeBytes: 135716379
- names:
- nginx@sha256:00be67d6ba53d5318cd91c57771530f5251cfbe028b7be2c4b70526f988cfc9f
- nginx:latest
sizeBytes: 109357355
- names:
- k8s.gcr.io/kubernetes-dashboard-amd64@sha256:dc4026c1b595435ef5527ca598e1e9c4343076926d7d62b365c44831395adbd0
- k8s.gcr.io/kubernetes-dashboard-amd64:v1.8.3
- registry.k8s.io/kubernetes-dashboard-amd64@sha256:dc4026c1b595435ef5527ca598e1e9c4343076926d7d62b365c44831395adbd0
- registry.k8s.io/kubernetes-dashboard-amd64:v1.8.3
sizeBytes: 102319441
- names:
- gcr.io/google_containers/kube-proxy:v1.11.10-gke.5
- k8s.gcr.io/kube-proxy:v1.11.10-gke.5
- registry.k8s.io/kube-proxy:v1.11.10-gke.5
sizeBytes: 102279340
- names:
- k8s.gcr.io/event-exporter@sha256:7f9cd7cb04d6959b0aa960727d04fa86759008048c785397b7b0d9dff0007516
- k8s.gcr.io/event-exporter:v0.2.3
- registry.k8s.io/event-exporter@sha256:7f9cd7cb04d6959b0aa960727d04fa86759008048c785397b7b0d9dff0007516
- registry.k8s.io/event-exporter:v0.2.3
sizeBytes: 94171943
- names:
- k8s.gcr.io/prometheus-to-sd@sha256:6c0c742475363d537ff059136e5d5e4ab1f512ee0fd9b7ca42ea48bc309d1662
- k8s.gcr.io/prometheus-to-sd:v0.3.1
- registry.k8s.io/prometheus-to-sd@sha256:6c0c742475363d537ff059136e5d5e4ab1f512ee0fd9b7ca42ea48bc309d1662
- registry.k8s.io/prometheus-to-sd:v0.3.1
sizeBytes: 88077694
- names:
- k8s.gcr.io/fluentd-gcp-scaler@sha256:a5ace7506d393c4ed65eb2cbb6312c64ab357fcea16dff76b9055bc6e498e5ff
- k8s.gcr.io/fluentd-gcp-scaler:0.5.1
- registry.k8s.io/fluentd-gcp-scaler@sha256:a5ace7506d393c4ed65eb2cbb6312c64ab357fcea16dff76b9055bc6e498e5ff
- registry.k8s.io/fluentd-gcp-scaler:0.5.1
sizeBytes: 86637208
- names:
- k8s.gcr.io/heapster-amd64@sha256:9fae0af136ce0cf4f88393b3670f7139ffc464692060c374d2ae748e13144521
- k8s.gcr.io/heapster-amd64:v1.6.0-beta.1
- registry.k8s.io/heapster-amd64@sha256:9fae0af136ce0cf4f88393b3670f7139ffc464692060c374d2ae748e13144521
- registry.k8s.io/heapster-amd64:v1.6.0-beta.1
sizeBytes: 76016169
- names:
- k8s.gcr.io/ingress-glbc-amd64@sha256:31d36bbd9c44caffa135fc78cf0737266fcf25e3cf0cd1c2fcbfbc4f7309cc52
- k8s.gcr.io/ingress-glbc-amd64:v1.1.1
- registry.k8s.io/ingress-glbc-amd64@sha256:31d36bbd9c44caffa135fc78cf0737266fcf25e3cf0cd1c2fcbfbc4f7309cc52
- registry.k8s.io/ingress-glbc-amd64:v1.1.1
sizeBytes: 67801919
- names:
- k8s.gcr.io/kube-addon-manager@sha256:d53486c3a0b49ebee019932878dc44232735d5622a51dbbdcec7124199020d09
- k8s.gcr.io/kube-addon-manager:v8.7
- registry.k8s.io/kube-addon-manager@sha256:d53486c3a0b49ebee019932878dc44232735d5622a51dbbdcec7124199020d09
- registry.k8s.io/kube-addon-manager:v8.7
sizeBytes: 63322109
- names:
- nginx@sha256:4aacdcf186934dcb02f642579314075910f1855590fd3039d8fa4c9f96e48315
- nginx:1.10-alpine
sizeBytes: 54042627
- names:
- k8s.gcr.io/cpvpa-amd64@sha256:cfe7b0a11c9c8e18c87b1eb34fef9a7cbb8480a8da11fc2657f78dbf4739f869
- k8s.gcr.io/cpvpa-amd64:v0.6.0
- registry.k8s.io/cpvpa-amd64@sha256:cfe7b0a11c9c8e18c87b1eb34fef9a7cbb8480a8da11fc2657f78dbf4739f869
- registry.k8s.io/cpvpa-amd64:v0.6.0
sizeBytes: 51785854
- names:
- k8s.gcr.io/cluster-proportional-autoscaler-amd64@sha256:003f98d9f411ddfa6ff6d539196355e03ddd69fa4ed38c7ffb8fec6f729afe2d
- k8s.gcr.io/cluster-proportional-autoscaler-amd64:1.1.2-r2
- registry.k8s.io/cluster-proportional-autoscaler-amd64@sha256:003f98d9f411ddfa6ff6d539196355e03ddd69fa4ed38c7ffb8fec6f729afe2d
- registry.k8s.io/cluster-proportional-autoscaler-amd64:1.1.2-r2
sizeBytes: 49648481
- names:
- k8s.gcr.io/ip-masq-agent-amd64@sha256:1ffda57d87901bc01324c82ceb2145fe6a0448d3f0dd9cb65aa76a867cd62103
- k8s.gcr.io/ip-masq-agent-amd64:v2.1.1
- registry.k8s.io/ip-masq-agent-amd64@sha256:1ffda57d87901bc01324c82ceb2145fe6a0448d3f0dd9cb65aa76a867cd62103
- registry.k8s.io/ip-masq-agent-amd64:v2.1.1
sizeBytes: 49612505
- names:
- k8s.gcr.io/k8s-dns-kube-dns-amd64@sha256:b99fc3eee2a9f052f7eb4cc00f15eb12fc405fa41019baa2d6b79847ae7284a8
- k8s.gcr.io/k8s-dns-kube-dns-amd64:1.14.10
- registry.k8s.io/k8s-dns-kube-dns-amd64@sha256:b99fc3eee2a9f052f7eb4cc00f15eb12fc405fa41019baa2d6b79847ae7284a8
- registry.k8s.io/k8s-dns-kube-dns-amd64:1.14.10
sizeBytes: 49549457
- names:
- k8s.gcr.io/rescheduler@sha256:156cfbfd05a5a815206fd2eeb6cbdaf1596d71ea4b415d3a6c43071dd7b99450
- k8s.gcr.io/rescheduler:v0.4.0
- registry.k8s.io/rescheduler@sha256:156cfbfd05a5a815206fd2eeb6cbdaf1596d71ea4b415d3a6c43071dd7b99450
- registry.k8s.io/rescheduler:v0.4.0
sizeBytes: 48973149
- names:
- k8s.gcr.io/event-exporter@sha256:16ca66e2b5dc7a1ce6a5aafcb21d0885828b75cdfc08135430480f7ad2364adc
- k8s.gcr.io/event-exporter:v0.2.4
- registry.k8s.io/event-exporter@sha256:16ca66e2b5dc7a1ce6a5aafcb21d0885828b75cdfc08135430480f7ad2364adc
- registry.k8s.io/event-exporter:v0.2.4
sizeBytes: 47261019
- names:
- k8s.gcr.io/coredns@sha256:db2bf53126ed1c761d5a41f24a1b82a461c85f736ff6e90542e9522be4757848
- k8s.gcr.io/coredns:1.1.3
- registry.k8s.io/coredns@sha256:db2bf53126ed1c761d5a41f24a1b82a461c85f736ff6e90542e9522be4757848
- registry.k8s.io/coredns:1.1.3
sizeBytes: 45587362
- names:
- prom/prometheus@sha256:483f4c9d7733699ba79facca9f8bcce1cef1af43dfc3e7c5a1882aa85f53cb74

View File

@@ -0,0 +1,174 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package fieldmanager
import (
"fmt"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
)
var (
scaleGroupVersion = schema.GroupVersion{Group: "autoscaling", Version: "v1"}
replicasPathInScale = fieldpath.MakePathOrDie("spec", "replicas")
)
// ResourcePathMappings maps a group/version to its replicas path. The
// assumption is that all the paths correspond to leaf fields.
type ResourcePathMappings map[string]fieldpath.Path
// ScaleHandler manages the conversion of managed fields between a main
// resource and the scale subresource
type ScaleHandler struct {
parentEntries []metav1.ManagedFieldsEntry
groupVersion schema.GroupVersion
mappings ResourcePathMappings
}
// NewScaleHandler creates a new ScaleHandler
func NewScaleHandler(parentEntries []metav1.ManagedFieldsEntry, groupVersion schema.GroupVersion, mappings ResourcePathMappings) *ScaleHandler {
return &ScaleHandler{
parentEntries: parentEntries,
groupVersion: groupVersion,
mappings: mappings,
}
}
// ToSubresource filter the managed fields of the main resource and convert
// them so that they can be handled by scale.
// For the managed fields that have a replicas path it performs two changes:
// 1. APIVersion is changed to the APIVersion of the scale subresource
// 2. Replicas path of the main resource is transformed to the replicas path of
// the scale subresource
func (h *ScaleHandler) ToSubresource() ([]metav1.ManagedFieldsEntry, error) {
managed, err := DecodeManagedFields(h.parentEntries)
if err != nil {
return nil, err
}
f := fieldpath.ManagedFields{}
t := map[string]*metav1.Time{}
for manager, versionedSet := range managed.Fields() {
path, ok := h.mappings[string(versionedSet.APIVersion())]
// Skip the entry if the APIVersion is unknown
if !ok || path == nil {
continue
}
if versionedSet.Set().Has(path) {
newVersionedSet := fieldpath.NewVersionedSet(
fieldpath.NewSet(replicasPathInScale),
fieldpath.APIVersion(scaleGroupVersion.String()),
versionedSet.Applied(),
)
f[manager] = newVersionedSet
t[manager] = managed.Times()[manager]
}
}
return managedFieldsEntries(internal.NewManaged(f, t))
}
// ToParent merges `scaleEntries` with the entries of the main resource and
// transforms them accordingly
func (h *ScaleHandler) ToParent(scaleEntries []metav1.ManagedFieldsEntry) ([]metav1.ManagedFieldsEntry, error) {
decodedParentEntries, err := DecodeManagedFields(h.parentEntries)
if err != nil {
return nil, err
}
parentFields := decodedParentEntries.Fields()
decodedScaleEntries, err := DecodeManagedFields(scaleEntries)
if err != nil {
return nil, err
}
scaleFields := decodedScaleEntries.Fields()
f := fieldpath.ManagedFields{}
t := map[string]*metav1.Time{}
for manager, versionedSet := range parentFields {
// Get the main resource "replicas" path
path, ok := h.mappings[string(versionedSet.APIVersion())]
// Drop the entry if the APIVersion is unknown.
if !ok {
continue
}
// If the parent entry does not have the replicas path or it is nil, just
// keep it as it is. The path is nil for Custom Resources without scale
// subresource.
if path == nil || !versionedSet.Set().Has(path) {
f[manager] = versionedSet
t[manager] = decodedParentEntries.Times()[manager]
continue
}
if _, ok := scaleFields[manager]; !ok {
// "Steal" the replicas path from the main resource entry
newSet := versionedSet.Set().Difference(fieldpath.NewSet(path))
if !newSet.Empty() {
newVersionedSet := fieldpath.NewVersionedSet(
newSet,
versionedSet.APIVersion(),
versionedSet.Applied(),
)
f[manager] = newVersionedSet
t[manager] = decodedParentEntries.Times()[manager]
}
} else {
// Field wasn't stolen, let's keep the entry as it is.
f[manager] = versionedSet
t[manager] = decodedParentEntries.Times()[manager]
delete(scaleFields, manager)
}
}
for manager, versionedSet := range scaleFields {
if !versionedSet.Set().Has(replicasPathInScale) {
continue
}
newVersionedSet := fieldpath.NewVersionedSet(
fieldpath.NewSet(h.mappings[h.groupVersion.String()]),
fieldpath.APIVersion(h.groupVersion.String()),
versionedSet.Applied(),
)
f[manager] = newVersionedSet
t[manager] = decodedParentEntries.Times()[manager]
}
return managedFieldsEntries(internal.NewManaged(f, t))
}
func managedFieldsEntries(entries internal.ManagedInterface) ([]metav1.ManagedFieldsEntry, error) {
obj := &unstructured.Unstructured{Object: map[string]interface{}{}}
if err := internal.EncodeObjectManagedFields(obj, entries); err != nil {
return nil, err
}
accessor, err := meta.Accessor(obj)
if err != nil {
panic(fmt.Sprintf("couldn't get accessor: %v", err))
}
return accessor.GetManagedFields(), nil
}

View File

@@ -72,30 +72,41 @@ func NewCRDStructuredMergeManager(typeConverter TypeConverter, objectConverter r
}, nil
}
func objectGVKNN(obj runtime.Object) string {
name := "<unknown>"
namespace := "<unknown>"
if accessor, err := meta.Accessor(obj); err == nil {
name = accessor.GetName()
namespace = accessor.GetNamespace()
}
return fmt.Sprintf("%v/%v; %v", namespace, name, obj.GetObjectKind().GroupVersionKind())
}
// Update implements Manager.
func (f *structuredMergeManager) Update(liveObj, newObj runtime.Object, managed Managed, manager string) (runtime.Object, Managed, error) {
newObjVersioned, err := f.toVersioned(newObj)
if err != nil {
return nil, nil, fmt.Errorf("failed to convert new object to proper version: %v", err)
return nil, nil, fmt.Errorf("failed to convert new object (%v) to proper version (%v): %v", objectGVKNN(newObj), f.groupVersion, err)
}
liveObjVersioned, err := f.toVersioned(liveObj)
if err != nil {
return nil, nil, fmt.Errorf("failed to convert live object to proper version: %v", err)
return nil, nil, fmt.Errorf("failed to convert live object (%v) to proper version: %v", objectGVKNN(liveObj), err)
}
newObjTyped, err := f.typeConverter.ObjectToTyped(newObjVersioned)
if err != nil {
return nil, nil, fmt.Errorf("failed to convert new object (%v) to smd typed: %v", newObjVersioned.GetObjectKind().GroupVersionKind(), err)
return nil, nil, fmt.Errorf("failed to convert new object (%v) to smd typed: %v", objectGVKNN(newObjVersioned), err)
}
liveObjTyped, err := f.typeConverter.ObjectToTyped(liveObjVersioned)
if err != nil {
return nil, nil, fmt.Errorf("failed to convert live object (%v) to smd typed: %v", liveObjVersioned.GetObjectKind().GroupVersionKind(), err)
return nil, nil, fmt.Errorf("failed to convert live object (%v) to smd typed: %v", objectGVKNN(liveObjVersioned), err)
}
apiVersion := fieldpath.APIVersion(f.groupVersion.String())
// TODO(apelisse) use the first return value when unions are implemented
_, managedFields, err := f.updater.Update(liveObjTyped, newObjTyped, apiVersion, managed.Fields(), manager)
if err != nil {
return nil, nil, fmt.Errorf("failed to update ManagedFields: %v", err)
return nil, nil, fmt.Errorf("failed to update ManagedFields (%v): %v", objectGVKNN(newObjVersioned), err)
}
managed = internal.NewManaged(managedFields, managed.Times())
@@ -123,16 +134,16 @@ func (f *structuredMergeManager) Apply(liveObj, patchObj runtime.Object, managed
liveObjVersioned, err := f.toVersioned(liveObj)
if err != nil {
return nil, nil, fmt.Errorf("failed to convert live object to proper version: %v", err)
return nil, nil, fmt.Errorf("failed to convert live object (%v) to proper version: %v", objectGVKNN(liveObj), err)
}
patchObjTyped, err := f.typeConverter.ObjectToTyped(patchObj)
if err != nil {
return nil, nil, fmt.Errorf("failed to create typed patch object: %v", err)
return nil, nil, fmt.Errorf("failed to create typed patch object (%v): %v", objectGVKNN(patchObj), err)
}
liveObjTyped, err := f.typeConverter.ObjectToTyped(liveObjVersioned)
if err != nil {
return nil, nil, fmt.Errorf("failed to create typed live object: %v", err)
return nil, nil, fmt.Errorf("failed to create typed live object (%v): %v", objectGVKNN(liveObjVersioned), err)
}
apiVersion := fieldpath.APIVersion(f.groupVersion.String())
@@ -148,18 +159,18 @@ func (f *structuredMergeManager) Apply(liveObj, patchObj runtime.Object, managed
newObj, err := f.typeConverter.TypedToObject(newObjTyped)
if err != nil {
return nil, nil, fmt.Errorf("failed to convert new typed object to object: %v", err)
return nil, nil, fmt.Errorf("failed to convert new typed object (%v) to object: %v", objectGVKNN(patchObj), err)
}
newObjVersioned, err := f.toVersioned(newObj)
if err != nil {
return nil, nil, fmt.Errorf("failed to convert new object to proper version: %v", err)
return nil, nil, fmt.Errorf("failed to convert new object (%v) to proper version: %v", objectGVKNN(patchObj), err)
}
f.objectDefaulter.Default(newObjVersioned)
newObjUnversioned, err := f.toUnversioned(newObjVersioned)
if err != nil {
return nil, nil, fmt.Errorf("failed to convert to unversioned: %v", err)
return nil, nil, fmt.Errorf("failed to convert to unversioned (%v): %v", objectGVKNN(patchObj), err)
}
return newObjUnversioned, managed, nil
}

View File

@@ -22,7 +22,7 @@ import (
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
"k8s.io/apimachinery/pkg/util/managedfields"
"k8s.io/kube-openapi/pkg/util/proto"
"sigs.k8s.io/structured-merge-diff/v4/typed"
"sigs.k8s.io/structured-merge-diff/v4/value"
@@ -65,7 +65,7 @@ func (DeducedTypeConverter) TypedToObject(value *typed.TypedValue) (runtime.Obje
}
type typeConverter struct {
parser *internal.GvkParser
parser *managedfields.GvkParser
}
var _ TypeConverter = &typeConverter{}
@@ -74,7 +74,7 @@ var _ TypeConverter = &typeConverter{}
// will automatically find the proper version of the object, and the
// corresponding schema information.
func NewTypeConverter(models proto.Models, preserveUnknownFields bool) (TypeConverter, error) {
parser, err := internal.NewGVKParser(models, preserveUnknownFields)
parser, err := managedfields.NewGVKParser(models, preserveUnknownFields)
if err != nil {
return nil, err
}

View File

@@ -0,0 +1,176 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package finisher
import (
"context"
"fmt"
"net/http"
goruntime "runtime"
"time"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apiserver/pkg/endpoints/metrics"
"k8s.io/klog/v2"
)
// ResultFunc is a function that returns a rest result and can be run in a goroutine
type ResultFunc func() (runtime.Object, error)
// result stores the return values or panic from a ResultFunc function
type result struct {
// object stores the response returned by the ResultFunc function
object runtime.Object
// err stores the error returned by the ResultFunc function
err error
// reason stores the reason from a panic thrown by the ResultFunc function
reason interface{}
}
// Return processes the result returned by a ResultFunc function
func (r *result) Return() (runtime.Object, error) {
switch {
case r.reason != nil:
// panic has higher precedence, the goroutine executing ResultFunc has panic'd,
// so propagate a panic to the caller.
panic(r.reason)
case r.err != nil:
return nil, r.err
default:
// if we are here, it means neither a panic, nor an error
if status, ok := r.object.(*metav1.Status); ok {
// An api.Status object with status != success is considered an "error",
// which interrupts the normal response flow.
if status.Status != metav1.StatusSuccess {
return nil, errors.FromObject(status)
}
}
return r.object, nil
}
}
// PostTimeoutLoggerFunc is a function that can be used to log the result returned
// by a ResultFunc after the request had timed out.
// timedOutAt is the time the request had been timed out.
// r is the result returned by the child goroutine.
type PostTimeoutLoggerFunc func(timedOutAt time.Time, r *result)
const (
// how much time the post-timeout receiver goroutine will wait for the sender
// (child goroutine executing ResultFunc) to send a result after the request.
// had timed out.
postTimeoutLoggerWait = 5 * time.Minute
)
// FinishRequest makes a given ResultFunc asynchronous and handles errors returned by the response.
func FinishRequest(ctx context.Context, fn ResultFunc) (runtime.Object, error) {
return finishRequest(ctx, fn, postTimeoutLoggerWait, logPostTimeoutResult)
}
func finishRequest(ctx context.Context, fn ResultFunc, postTimeoutWait time.Duration, postTimeoutLogger PostTimeoutLoggerFunc) (runtime.Object, error) {
// the channel needs to be buffered since the post-timeout receiver goroutine
// waits up to 5 minutes for the child goroutine to return.
resultCh := make(chan *result, 1)
go func() {
result := &result{}
// panics don't cross goroutine boundaries, so we have to handle ourselves
defer func() {
reason := recover()
if reason != nil {
// do not wrap the sentinel ErrAbortHandler panic value
if reason != http.ErrAbortHandler {
// Same as stdlib http server code. Manually allocate stack
// trace buffer size to prevent excessively large logs
const size = 64 << 10
buf := make([]byte, size)
buf = buf[:goruntime.Stack(buf, false)]
reason = fmt.Sprintf("%v\n%s", reason, buf)
}
// store the panic reason into the result.
result.reason = reason
}
// Propagate the result to the parent goroutine
resultCh <- result
}()
if object, err := fn(); err != nil {
result.err = err
} else {
result.object = object
}
}()
select {
case result := <-resultCh:
return result.Return()
case <-ctx.Done():
// we are going to send a timeout response to the caller, but the asynchronous goroutine
// (sender) is still executing the ResultFunc function.
// kick off a goroutine (receiver) here to wait for the sender (goroutine executing ResultFunc)
// to send the result and then log details of the result.
defer func() {
go func() {
timedOutAt := time.Now()
var result *result
select {
case result = <-resultCh:
case <-time.After(postTimeoutWait):
// we will not wait forever, if we are here then we know that some sender
// goroutines are taking longer than postTimeoutWait.
}
postTimeoutLogger(timedOutAt, result)
}()
}()
return nil, errors.NewTimeoutError(fmt.Sprintf("request did not complete within requested timeout - %s", ctx.Err()), 0)
}
}
// logPostTimeoutResult logs a panic or an error from the result that the sender (goroutine that is
// executing the ResultFunc function) has sent to the receiver after the request had timed out.
// timedOutAt is the time the request had been timed out
func logPostTimeoutResult(timedOutAt time.Time, r *result) {
if r == nil {
// we are using r == nil to indicate that the child goroutine never returned a result.
metrics.RecordRequestPostTimeout(metrics.PostTimeoutSourceRestHandler, metrics.PostTimeoutHandlerPending)
klog.Errorf("FinishRequest: post-timeout activity, waited for %s, child goroutine has not returned yet", time.Since(timedOutAt))
return
}
var status string
switch {
case r.reason != nil:
// a non empty reason inside a result object indicates that there was a panic.
status = metrics.PostTimeoutHandlerPanic
case r.err != nil:
status = metrics.PostTimeoutHandlerError
default:
status = metrics.PostTimeoutHandlerOK
}
metrics.RecordRequestPostTimeout(metrics.PostTimeoutSourceRestHandler, status)
err := fmt.Errorf("FinishRequest: post-timeout activity - time-elapsed: %s, panicked: %t, err: %v, panic-reason: %v",
time.Since(timedOutAt), r.reason != nil, r.err, r.reason)
utilruntime.HandleError(err)
}

View File

@@ -76,8 +76,8 @@ func getResourceHandler(scope *RequestScope, getter getterFunc) http.HandlerFunc
}
trace.Step("About to write a response")
defer trace.Step("Writing http response done")
transformResponseObject(ctx, scope, trace, req, w, http.StatusOK, outputMediaType, result)
trace.Step("Transformed response object")
}
}
@@ -281,8 +281,7 @@ func ListResource(r rest.Lister, rw rest.Watcher, scope *RequestScope, forceWatc
return
}
trace.Step("Listing from storage done")
defer trace.Step("Writing http response done", utiltrace.Field{"count", meta.LenList(result)})
transformResponseObject(ctx, scope, trace, req, w, http.StatusOK, outputMediaType, result)
trace.Step("Writing http response done", utiltrace.Field{"count", meta.LenList(result)})
}
}

View File

@@ -20,6 +20,7 @@ import (
"net/http"
utilnet "k8s.io/apimachinery/pkg/util/net"
"k8s.io/apiserver/pkg/endpoints/request"
)
const (
@@ -73,3 +74,17 @@ func (lazy *lazyAccept) String() string {
return "unknown"
}
// lazyAuditID implements Stringer interface to lazily retrieve
// the audit ID associated with the request.
type lazyAuditID struct {
req *http.Request
}
func (lazy *lazyAuditID) String() string {
if lazy.req != nil {
return request.GetAuditIDTruncated(lazy.req.Context())
}
return "unknown"
}

View File

@@ -19,8 +19,6 @@ package handlers
import (
"fmt"
"net/http"
"net/url"
"strings"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
@@ -38,31 +36,16 @@ type ScopeNamer interface {
// ObjectName returns the namespace and name from an object if they exist, or an error if the object
// does not support names.
ObjectName(obj runtime.Object) (namespace, name string, err error)
// SetSelfLink sets the provided URL onto the object. The method should return nil if the object
// does not support selfLinks.
SetSelfLink(obj runtime.Object, url string) error
// GenerateLink creates an encoded URI for a given runtime object that represents the canonical path
// and query.
GenerateLink(requestInfo *request.RequestInfo, obj runtime.Object) (uri string, err error)
// GenerateListLink creates an encoded URI for a list that represents the canonical path and query.
GenerateListLink(req *http.Request) (uri string, err error)
}
type ContextBasedNaming struct {
SelfLinker runtime.SelfLinker
Namer runtime.Namer
ClusterScoped bool
SelfLinkPathPrefix string
SelfLinkPathSuffix string
}
// ContextBasedNaming implements ScopeNamer
var _ ScopeNamer = ContextBasedNaming{}
func (n ContextBasedNaming) SetSelfLink(obj runtime.Object, url string) error {
return n.SelfLinker.SetSelfLink(obj, url)
}
func (n ContextBasedNaming) Namespace(req *http.Request) (namespace string, err error) {
requestInfo, ok := request.RequestInfoFrom(req.Context())
if !ok {
@@ -76,75 +59,22 @@ func (n ContextBasedNaming) Name(req *http.Request) (namespace, name string, err
if !ok {
return "", "", fmt.Errorf("missing requestInfo")
}
ns, err := n.Namespace(req)
if err != nil {
return "", "", err
}
if len(requestInfo.Name) == 0 {
return "", "", errEmptyName
}
return ns, requestInfo.Name, nil
}
// fastURLPathEncode encodes the provided path as a URL path
func fastURLPathEncode(path string) string {
for _, r := range []byte(path) {
switch {
case r >= '-' && r <= '9', r >= 'A' && r <= 'Z', r >= 'a' && r <= 'z':
// characters within this range do not require escaping
default:
var u url.URL
u.Path = path
return u.EscapedPath()
}
}
return path
}
func (n ContextBasedNaming) GenerateLink(requestInfo *request.RequestInfo, obj runtime.Object) (uri string, err error) {
namespace, name, err := n.ObjectName(obj)
if err == errEmptyName && len(requestInfo.Name) > 0 {
name = requestInfo.Name
} else if err != nil {
return "", err
}
if len(namespace) == 0 && len(requestInfo.Namespace) > 0 {
namespace = requestInfo.Namespace
}
if n.ClusterScoped {
return n.SelfLinkPathPrefix + url.QueryEscape(name) + n.SelfLinkPathSuffix, nil
}
builder := strings.Builder{}
builder.Grow(len(n.SelfLinkPathPrefix) + len(namespace) + len(requestInfo.Resource) + len(name) + len(n.SelfLinkPathSuffix) + 8)
builder.WriteString(n.SelfLinkPathPrefix)
builder.WriteString(namespace)
builder.WriteByte('/')
builder.WriteString(requestInfo.Resource)
builder.WriteByte('/')
builder.WriteString(name)
builder.WriteString(n.SelfLinkPathSuffix)
return fastURLPathEncode(builder.String()), nil
}
func (n ContextBasedNaming) GenerateListLink(req *http.Request) (uri string, err error) {
if len(req.URL.RawPath) > 0 {
return req.URL.RawPath, nil
}
return fastURLPathEncode(req.URL.Path), nil
return requestInfo.Namespace, requestInfo.Name, nil
}
func (n ContextBasedNaming) ObjectName(obj runtime.Object) (namespace, name string, err error) {
name, err = n.SelfLinker.Name(obj)
name, err = n.Namer.Name(obj)
if err != nil {
return "", "", err
}
if len(name) == 0 {
return "", "", errEmptyName
}
namespace, err = n.SelfLinker.Namespace(obj)
namespace, err = n.Namer.Namespace(obj)
if err != nil {
return "", "", err
}

View File

@@ -23,6 +23,8 @@ import (
"strings"
"time"
kjson "sigs.k8s.io/json"
jsonpatch "github.com/evanphx/json-patch"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
@@ -33,7 +35,6 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/json"
"k8s.io/apimachinery/pkg/util/mergepatch"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/strategicpatch"
@@ -43,6 +44,7 @@ import (
"k8s.io/apiserver/pkg/audit"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
"k8s.io/apiserver/pkg/endpoints/handlers/finisher"
"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/features"
@@ -104,6 +106,7 @@ func PatchResource(r rest.Patcher, scope *RequestScope, admit admission.Interfac
}
patchBytes, err := limitedReadBody(req, scope.MaxRequestBodyBytes)
trace.Step("limitedReadBody done", utiltrace.Field{"len", len(patchBytes)}, utiltrace.Field{"err", err})
if err != nil {
scope.err(err, w, req)
return
@@ -122,10 +125,9 @@ func PatchResource(r rest.Patcher, scope *RequestScope, admit admission.Interfac
}
options.TypeMeta.SetGroupVersionKind(metav1.SchemeGroupVersion.WithKind("PatchOptions"))
ae := request.AuditEventFrom(ctx)
admit = admission.WithAudit(admit, ae)
admit = admission.WithAudit(admit)
audit.LogRequestPatch(ae, patchBytes)
audit.LogRequestPatch(req.Context(), patchBytes)
trace.Step("Recorded the audit event")
baseContentType := runtime.ContentTypeJSON
@@ -139,9 +141,15 @@ func PatchResource(r rest.Patcher, scope *RequestScope, admit admission.Interfac
}
gv := scope.Kind.GroupVersion()
validationDirective := fieldValidation(options.FieldValidation)
decodeSerializer := s.Serializer
if validationDirective == metav1.FieldValidationWarn || validationDirective == metav1.FieldValidationStrict {
decodeSerializer = s.StrictSerializer
}
codec := runtime.NewCodec(
scope.Serializer.EncoderForVersion(s.Serializer, gv),
scope.Serializer.DecoderToVersion(s.Serializer, scope.HubGroupVersion),
scope.Serializer.DecoderToVersion(decodeSerializer, scope.HubGroupVersion),
)
userInfo, _ := request.UserFrom(ctx)
@@ -189,15 +197,16 @@ func PatchResource(r rest.Patcher, scope *RequestScope, admit admission.Interfac
}
p := patcher{
namer: scope.Namer,
creater: scope.Creater,
defaulter: scope.Defaulter,
typer: scope.Typer,
unsafeConvertor: scope.UnsafeConvertor,
kind: scope.Kind,
resource: scope.Resource,
subresource: scope.Subresource,
dryRun: dryrun.IsDryRun(options.DryRun),
namer: scope.Namer,
creater: scope.Creater,
defaulter: scope.Defaulter,
typer: scope.Typer,
unsafeConvertor: scope.UnsafeConvertor,
kind: scope.Kind,
resource: scope.Resource,
subresource: scope.Subresource,
dryRun: dryrun.IsDryRun(options.DryRun),
validationDirective: validationDirective,
objectInterfaces: scope,
@@ -227,16 +236,13 @@ func PatchResource(r rest.Patcher, scope *RequestScope, admit admission.Interfac
}
trace.Step("Object stored in database")
if err := setObjectSelfLink(ctx, result, req, scope.Namer); err != nil {
scope.err(err, w, req)
return
}
trace.Step("Self-link added")
status := http.StatusOK
if wasCreated {
status = http.StatusCreated
}
trace.Step("About to write a response")
defer trace.Step("Writing http response done")
transformResponseObject(ctx, scope, trace, req, w, status, outputMediaType, result)
}
}
@@ -250,15 +256,16 @@ type mutateObjectUpdateFunc func(ctx context.Context, obj, old runtime.Object) e
// moved into this type.
type patcher struct {
// Pieces of RequestScope
namer ScopeNamer
creater runtime.ObjectCreater
defaulter runtime.ObjectDefaulter
typer runtime.ObjectTyper
unsafeConvertor runtime.ObjectConvertor
resource schema.GroupVersionResource
kind schema.GroupVersionKind
subresource string
dryRun bool
namer ScopeNamer
creater runtime.ObjectCreater
defaulter runtime.ObjectDefaulter
typer runtime.ObjectTyper
unsafeConvertor runtime.ObjectConvertor
resource schema.GroupVersionResource
kind schema.GroupVersionKind
subresource string
dryRun bool
validationDirective string
objectInterfaces admission.ObjectInterfaces
@@ -290,8 +297,8 @@ type patcher struct {
}
type patchMechanism interface {
applyPatchToCurrentObject(currentObject runtime.Object) (runtime.Object, error)
createNewObject() (runtime.Object, error)
applyPatchToCurrentObject(requextContext context.Context, currentObject runtime.Object) (runtime.Object, error)
createNewObject(requestContext context.Context) (runtime.Object, error)
}
type jsonPatcher struct {
@@ -300,7 +307,7 @@ type jsonPatcher struct {
fieldManager *fieldmanager.FieldManager
}
func (p *jsonPatcher) applyPatchToCurrentObject(currentObject runtime.Object) (runtime.Object, error) {
func (p *jsonPatcher) applyPatchToCurrentObject(requestContext context.Context, currentObject runtime.Object) (runtime.Object, error) {
// Encode will convert & return a versioned object in JSON.
currentObjJS, err := runtime.Encode(p.codec, currentObject)
if err != nil {
@@ -308,7 +315,7 @@ func (p *jsonPatcher) applyPatchToCurrentObject(currentObject runtime.Object) (r
}
// Apply the patch.
patchedObjJS, err := p.applyJSPatch(currentObjJS)
patchedObjJS, appliedStrictErrs, err := p.applyJSPatch(currentObjJS)
if err != nil {
return nil, err
}
@@ -316,9 +323,32 @@ func (p *jsonPatcher) applyPatchToCurrentObject(currentObject runtime.Object) (r
// Construct the resulting typed, unversioned object.
objToUpdate := p.restPatcher.New()
if err := runtime.DecodeInto(p.codec, patchedObjJS, objToUpdate); err != nil {
return nil, errors.NewInvalid(schema.GroupKind{}, "", field.ErrorList{
field.Invalid(field.NewPath("patch"), string(patchedObjJS), err.Error()),
})
strictError, isStrictError := runtime.AsStrictDecodingError(err)
switch {
case !isStrictError:
// disregard any appliedStrictErrs, because it's an incomplete
// list of strict errors given that we don't know what fields were
// unknown because DecodeInto failed. Non-strict errors trump in this case.
return nil, errors.NewInvalid(schema.GroupKind{}, "", field.ErrorList{
field.Invalid(field.NewPath("patch"), string(patchedObjJS), err.Error()),
})
case p.validationDirective == metav1.FieldValidationWarn:
addStrictDecodingWarnings(requestContext, append(appliedStrictErrs, strictError.Errors()...))
default:
strictDecodingError := runtime.NewStrictDecodingError(append(appliedStrictErrs, strictError.Errors()...))
return nil, errors.NewInvalid(schema.GroupKind{}, "", field.ErrorList{
field.Invalid(field.NewPath("patch"), string(patchedObjJS), strictDecodingError.Error()),
})
}
} else if len(appliedStrictErrs) > 0 {
switch {
case p.validationDirective == metav1.FieldValidationWarn:
addStrictDecodingWarnings(requestContext, appliedStrictErrs)
default:
return nil, errors.NewInvalid(schema.GroupKind{}, "", field.ErrorList{
field.Invalid(field.NewPath("patch"), string(patchedObjJS), runtime.NewStrictDecodingError(appliedStrictErrs).Error()),
})
}
}
if p.fieldManager != nil {
@@ -327,52 +357,65 @@ func (p *jsonPatcher) applyPatchToCurrentObject(currentObject runtime.Object) (r
return objToUpdate, nil
}
func (p *jsonPatcher) createNewObject() (runtime.Object, error) {
func (p *jsonPatcher) createNewObject(_ context.Context) (runtime.Object, error) {
return nil, errors.NewNotFound(p.resource.GroupResource(), p.name)
}
type jsonPatchOp struct {
Op string `json:"op"`
Path string `json:"path"`
From string `json:"from"`
Value interface{} `json:"value"`
}
// applyJSPatch applies the patch. Input and output objects must both have
// the external version, since that is what the patch must have been constructed against.
func (p *jsonPatcher) applyJSPatch(versionedJS []byte) (patchedJS []byte, retErr error) {
func (p *jsonPatcher) applyJSPatch(versionedJS []byte) (patchedJS []byte, strictErrors []error, retErr error) {
switch p.patchType {
case types.JSONPatchType:
// sanity check potentially abusive patches
// TODO(liggitt): drop this once golang json parser limits stack depth (https://github.com/golang/go/issues/31789)
if len(p.patchBytes) > 1024*1024 {
v := []interface{}{}
if err := json.Unmarshal(p.patchBytes, &v); err != nil {
return nil, errors.NewBadRequest(fmt.Sprintf("error decoding patch: %v", err))
if p.validationDirective == metav1.FieldValidationStrict || p.validationDirective == metav1.FieldValidationWarn {
var v []jsonPatchOp
var err error
if strictErrors, err = kjson.UnmarshalStrict(p.patchBytes, &v); err != nil {
return nil, nil, errors.NewBadRequest(fmt.Sprintf("error decoding patch: %v", err))
}
for i, e := range strictErrors {
strictErrors[i] = fmt.Errorf("json patch %v", e)
}
}
patchObj, err := jsonpatch.DecodePatch(p.patchBytes)
if err != nil {
return nil, errors.NewBadRequest(err.Error())
return nil, nil, errors.NewBadRequest(err.Error())
}
if len(patchObj) > maxJSONPatchOperations {
return nil, errors.NewRequestEntityTooLargeError(
return nil, nil, errors.NewRequestEntityTooLargeError(
fmt.Sprintf("The allowed maximum operations in a JSON patch is %d, got %d",
maxJSONPatchOperations, len(patchObj)))
}
patchedJS, err := patchObj.Apply(versionedJS)
if err != nil {
return nil, errors.NewGenericServerResponse(http.StatusUnprocessableEntity, "", schema.GroupResource{}, "", err.Error(), 0, false)
return nil, nil, errors.NewGenericServerResponse(http.StatusUnprocessableEntity, "", schema.GroupResource{}, "", err.Error(), 0, false)
}
return patchedJS, nil
return patchedJS, strictErrors, nil
case types.MergePatchType:
// sanity check potentially abusive patches
// TODO(liggitt): drop this once golang json parser limits stack depth (https://github.com/golang/go/issues/31789)
if len(p.patchBytes) > 1024*1024 {
if p.validationDirective == metav1.FieldValidationStrict || p.validationDirective == metav1.FieldValidationWarn {
v := map[string]interface{}{}
if err := json.Unmarshal(p.patchBytes, &v); err != nil {
return nil, errors.NewBadRequest(fmt.Sprintf("error decoding patch: %v", err))
var err error
strictErrors, err = kjson.UnmarshalStrict(p.patchBytes, &v)
if err != nil {
return nil, nil, errors.NewBadRequest(fmt.Sprintf("error decoding patch: %v", err))
}
}
return jsonpatch.MergePatch(versionedJS, p.patchBytes)
patchedJS, retErr = jsonpatch.MergePatch(versionedJS, p.patchBytes)
if retErr == jsonpatch.ErrBadJSONPatch {
return nil, nil, errors.NewBadRequest(retErr.Error())
}
return patchedJS, strictErrors, retErr
default:
// only here as a safety net - go-restful filters content-type
return nil, fmt.Errorf("unknown Content-Type header for patch: %v", p.patchType)
return nil, nil, fmt.Errorf("unknown Content-Type header for patch: %v", p.patchType)
}
}
@@ -384,7 +427,7 @@ type smpPatcher struct {
fieldManager *fieldmanager.FieldManager
}
func (p *smpPatcher) applyPatchToCurrentObject(currentObject runtime.Object) (runtime.Object, error) {
func (p *smpPatcher) applyPatchToCurrentObject(requestContext context.Context, currentObject runtime.Object) (runtime.Object, error) {
// Since the patch is applied on versioned objects, we need to convert the
// current object to versioned representation first.
currentVersionedObject, err := p.unsafeConvertor.ConvertToVersion(currentObject, p.kind.GroupVersion())
@@ -395,7 +438,7 @@ func (p *smpPatcher) applyPatchToCurrentObject(currentObject runtime.Object) (ru
if err != nil {
return nil, err
}
if err := strategicPatchObject(p.defaulter, currentVersionedObject, p.patchBytes, versionedObjToUpdate, p.schemaReferenceObj); err != nil {
if err := strategicPatchObject(requestContext, p.defaulter, currentVersionedObject, p.patchBytes, versionedObjToUpdate, p.schemaReferenceObj, p.validationDirective); err != nil {
return nil, err
}
// Convert the object back to the hub version
@@ -410,20 +453,21 @@ func (p *smpPatcher) applyPatchToCurrentObject(currentObject runtime.Object) (ru
return newObj, nil
}
func (p *smpPatcher) createNewObject() (runtime.Object, error) {
func (p *smpPatcher) createNewObject(_ context.Context) (runtime.Object, error) {
return nil, errors.NewNotFound(p.resource.GroupResource(), p.name)
}
type applyPatcher struct {
patch []byte
options *metav1.PatchOptions
creater runtime.ObjectCreater
kind schema.GroupVersionKind
fieldManager *fieldmanager.FieldManager
userAgent string
patch []byte
options *metav1.PatchOptions
creater runtime.ObjectCreater
kind schema.GroupVersionKind
fieldManager *fieldmanager.FieldManager
userAgent string
validationDirective string
}
func (p *applyPatcher) applyPatchToCurrentObject(obj runtime.Object) (runtime.Object, error) {
func (p *applyPatcher) applyPatchToCurrentObject(requestContext context.Context, obj runtime.Object) (runtime.Object, error) {
force := false
if p.options.Force != nil {
force = *p.options.Force
@@ -437,28 +481,45 @@ func (p *applyPatcher) applyPatchToCurrentObject(obj runtime.Object) (runtime.Ob
return nil, errors.NewBadRequest(fmt.Sprintf("error decoding YAML: %v", err))
}
return p.fieldManager.Apply(obj, patchObj, p.options.FieldManager, force)
obj, err := p.fieldManager.Apply(obj, patchObj, p.options.FieldManager, force)
if err != nil {
return obj, err
}
// TODO: spawn something to track deciding whether a fieldValidation=Strict
// fatal error should return before an error from the apply operation
if p.validationDirective == metav1.FieldValidationStrict || p.validationDirective == metav1.FieldValidationWarn {
if err := yaml.UnmarshalStrict(p.patch, &map[string]interface{}{}); err != nil {
if p.validationDirective == metav1.FieldValidationStrict {
return nil, errors.NewBadRequest(fmt.Sprintf("error strict decoding YAML: %v", err))
}
addStrictDecodingWarnings(requestContext, []error{err})
}
}
return obj, nil
}
func (p *applyPatcher) createNewObject() (runtime.Object, error) {
func (p *applyPatcher) createNewObject(requestContext context.Context) (runtime.Object, error) {
obj, err := p.creater.New(p.kind)
if err != nil {
return nil, fmt.Errorf("failed to create new object: %v", err)
}
return p.applyPatchToCurrentObject(obj)
return p.applyPatchToCurrentObject(requestContext, obj)
}
// strategicPatchObject applies a strategic merge patch of <patchBytes> to
// <originalObject> and stores the result in <objToUpdate>.
// strategicPatchObject applies a strategic merge patch of `patchBytes` to
// `originalObject` and stores the result in `objToUpdate`.
// It additionally returns the map[string]interface{} representation of the
// <originalObject> and <patchBytes>.
// NOTE: Both <originalObject> and <objToUpdate> are supposed to be versioned.
// `originalObject` and `patchBytes`.
// NOTE: Both `originalObject` and `objToUpdate` are supposed to be versioned.
func strategicPatchObject(
requestContext context.Context,
defaulter runtime.ObjectDefaulter,
originalObject runtime.Object,
patchBytes []byte,
objToUpdate runtime.Object,
schemaReferenceObj runtime.Object,
validationDirective string,
) error {
originalObjMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(originalObject)
if err != nil {
@@ -466,11 +527,19 @@ func strategicPatchObject(
}
patchMap := make(map[string]interface{})
if err := json.Unmarshal(patchBytes, &patchMap); err != nil {
return errors.NewBadRequest(err.Error())
var strictErrs []error
if validationDirective == metav1.FieldValidationWarn || validationDirective == metav1.FieldValidationStrict {
strictErrs, err = kjson.UnmarshalStrict(patchBytes, &patchMap)
if err != nil {
return errors.NewBadRequest(err.Error())
}
} else {
if err = kjson.UnmarshalCaseSensitivePreserveInts(patchBytes, &patchMap); err != nil {
return errors.NewBadRequest(err.Error())
}
}
if err := applyPatchToObject(defaulter, originalObjMap, patchMap, objToUpdate, schemaReferenceObj); err != nil {
if err := applyPatchToObject(requestContext, defaulter, originalObjMap, patchMap, objToUpdate, schemaReferenceObj, strictErrs, validationDirective); err != nil {
return err
}
return nil
@@ -479,16 +548,16 @@ func strategicPatchObject(
// applyPatch is called every time GuaranteedUpdate asks for the updated object,
// and is given the currently persisted object as input.
// TODO: rename this function because the name implies it is related to applyPatcher
func (p *patcher) applyPatch(_ context.Context, _, currentObject runtime.Object) (objToUpdate runtime.Object, patchErr error) {
func (p *patcher) applyPatch(ctx context.Context, _, currentObject runtime.Object) (objToUpdate runtime.Object, patchErr error) {
// Make sure we actually have a persisted currentObject
p.trace.Step("About to apply patch")
currentObjectHasUID, err := hasUID(currentObject)
if err != nil {
return nil, err
} else if !currentObjectHasUID {
objToUpdate, patchErr = p.mechanism.createNewObject()
objToUpdate, patchErr = p.mechanism.createNewObject(ctx)
} else {
objToUpdate, patchErr = p.mechanism.applyPatchToCurrentObject(currentObject)
objToUpdate, patchErr = p.mechanism.applyPatchToCurrentObject(ctx, currentObject)
}
if patchErr != nil {
@@ -507,6 +576,14 @@ func (p *patcher) applyPatch(_ context.Context, _, currentObject runtime.Object)
return nil, errors.NewConflict(p.resource.GroupResource(), p.name, fmt.Errorf("uid mismatch: the provided object specified uid %s, and no existing object was found", accessor.GetUID()))
}
// if this object supports namespace info
if objectMeta, err := meta.Accessor(objToUpdate); err == nil {
// ensure namespace on the object is correct, or error if a conflicting namespace was set in the object
if err := rest.EnsureObjectNamespaceMatchesRequestNamespace(rest.ExpectedNamespaceForResource(p.namespace, p.resource), objectMeta); err != nil {
return nil, err
}
}
if err := checkName(objToUpdate, p.name, p.namespace, p.namer); err != nil {
return nil, err
}
@@ -564,12 +641,13 @@ func (p *patcher) patchResource(ctx context.Context, scope *RequestScope) (runti
// this case is unreachable if ServerSideApply is not enabled because we will have already rejected the content type
case types.ApplyPatchType:
p.mechanism = &applyPatcher{
fieldManager: scope.FieldManager,
patch: p.patchBytes,
options: p.options,
creater: p.creater,
kind: p.kind,
userAgent: p.userAgent,
fieldManager: scope.FieldManager,
patch: p.patchBytes,
options: p.options,
creater: p.creater,
kind: p.kind,
userAgent: p.userAgent,
validationDirective: p.validationDirective,
}
p.forceAllowCreate = true
default:
@@ -581,8 +659,13 @@ func (p *patcher) patchResource(ctx context.Context, scope *RequestScope) (runti
return obj, nil
}
transformers := []rest.TransformFunc{p.applyPatch, p.applyAdmission, dedupOwnerReferencesTransformer}
if scope.FieldManager != nil {
transformers = append(transformers, fieldmanager.IgnoreManagedFieldsTimestampsTransformer)
}
wasCreated := false
p.updatedObjectInfo = rest.DefaultUpdatedObjectInfo(nil, p.applyPatch, p.applyAdmission, dedupOwnerReferencesTransformer)
p.updatedObjectInfo = rest.DefaultUpdatedObjectInfo(nil, transformers...)
requestFunc := func() (runtime.Object, error) {
// Pass in UpdateOptions to override UpdateStrategy.AllowUpdateOnCreate
options := patchToUpdateOptions(p.options)
@@ -590,7 +673,8 @@ func (p *patcher) patchResource(ctx context.Context, scope *RequestScope) (runti
wasCreated = created
return updateObject, updateErr
}
result, err := finishRequest(ctx, func() (runtime.Object, error) {
result, err := finisher.FinishRequest(ctx, func() (runtime.Object, error) {
result, err := requestFunc()
// If the object wasn't committed to storage because it's serialized size was too large,
// it is safe to remove managedFields (which can be large) and try again.
@@ -617,11 +701,14 @@ func (p *patcher) patchResource(ctx context.Context, scope *RequestScope) (runti
// <originalMap> and stores the result in <objToUpdate>.
// NOTE: <objToUpdate> must be a versioned object.
func applyPatchToObject(
requestContext context.Context,
defaulter runtime.ObjectDefaulter,
originalMap map[string]interface{},
patchMap map[string]interface{},
objToUpdate runtime.Object,
schemaReferenceObj runtime.Object,
strictErrs []error,
validationDirective string,
) error {
patchedObjMap, err := strategicpatch.StrategicMergeMapPatch(originalMap, patchMap, schemaReferenceObj)
if err != nil {
@@ -629,11 +716,38 @@ func applyPatchToObject(
}
// Rather than serialize the patched map to JSON, then decode it to an object, we go directly from a map to an object
if err := runtime.DefaultUnstructuredConverter.FromUnstructured(patchedObjMap, objToUpdate); err != nil {
return errors.NewInvalid(schema.GroupKind{}, "", field.ErrorList{
field.Invalid(field.NewPath("patch"), fmt.Sprintf("%+v", patchMap), err.Error()),
})
converter := runtime.DefaultUnstructuredConverter
returnUnknownFields := validationDirective == metav1.FieldValidationWarn || validationDirective == metav1.FieldValidationStrict
if err := converter.FromUnstructuredWithValidation(patchedObjMap, objToUpdate, returnUnknownFields); err != nil {
strictError, isStrictError := runtime.AsStrictDecodingError(err)
switch {
case !isStrictError:
// disregard any sttrictErrs, because it's an incomplete
// list of strict errors given that we don't know what fields were
// unknown because StrategicMergeMapPatch failed.
// Non-strict errors trump in this case.
return errors.NewInvalid(schema.GroupKind{}, "", field.ErrorList{
field.Invalid(field.NewPath("patch"), fmt.Sprintf("%+v", patchMap), err.Error()),
})
case validationDirective == metav1.FieldValidationWarn:
addStrictDecodingWarnings(requestContext, append(strictErrs, strictError.Errors()...))
default:
strictDecodingError := runtime.NewStrictDecodingError(append(strictErrs, strictError.Errors()...))
return errors.NewInvalid(schema.GroupKind{}, "", field.ErrorList{
field.Invalid(field.NewPath("patch"), fmt.Sprintf("%+v", patchMap), strictDecodingError.Error()),
})
}
} else if len(strictErrs) > 0 {
switch {
case validationDirective == metav1.FieldValidationWarn:
addStrictDecodingWarnings(requestContext, strictErrs)
default:
return errors.NewInvalid(schema.GroupKind{}, "", field.ErrorList{
field.Invalid(field.NewPath("patch"), fmt.Sprintf("%+v", patchMap), runtime.NewStrictDecodingError(strictErrs).Error()),
})
}
}
// Decoding from JSON to a versioned object would apply defaults, so we do the same here
defaulter.Default(objToUpdate)
@@ -658,8 +772,9 @@ func patchToUpdateOptions(po *metav1.PatchOptions) *metav1.UpdateOptions {
return nil
}
uo := &metav1.UpdateOptions{
DryRun: po.DryRun,
FieldManager: po.FieldManager,
DryRun: po.DryRun,
FieldManager: po.FieldManager,
FieldValidation: po.FieldValidation,
}
uo.TypeMeta.SetGroupVersionKind(metav1.SchemeGroupVersion.WithKind("UpdateOptions"))
return uo
@@ -671,8 +786,9 @@ func patchToCreateOptions(po *metav1.PatchOptions) *metav1.CreateOptions {
return nil
}
co := &metav1.CreateOptions{
DryRun: po.DryRun,
FieldManager: po.FieldManager,
DryRun: po.DryRun,
FieldManager: po.FieldManager,
FieldValidation: po.FieldValidation,
}
co.TypeMeta.SetGroupVersionKind(metav1.SchemeGroupVersion.WithKind("CreateOptions"))
return co

View File

@@ -31,6 +31,7 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
endpointsrequest "k8s.io/apiserver/pkg/endpoints/request"
utiltrace "k8s.io/utils/trace"
)
@@ -59,8 +60,14 @@ func doTransformObject(ctx context.Context, obj runtime.Object, opts interface{}
if _, ok := obj.(*metav1.Status); ok {
return obj, nil
}
if err := setObjectSelfLink(ctx, obj, req, scope.Namer); err != nil {
return nil, err
// ensure that for empty lists we don't return <nil> items.
// This is safe to modify without deep-copying the object, as
// List objects themselves are never cached.
if meta.IsListType(obj) && meta.LenList(obj) == 0 {
if err := meta.SetList(obj, []runtime.Object{}); err != nil {
return nil, err
}
}
switch target := mediaType.Convert; {
@@ -128,7 +135,13 @@ func transformResponseObject(ctx context.Context, scope *RequestScope, trace *ut
scope.err(err, w, req)
return
}
obj, err := transformObject(ctx, result, options, mediaType, scope, req)
var obj runtime.Object
do := func() {
obj, err = transformObject(ctx, result, options, mediaType, scope, req)
}
endpointsrequest.TrackTransformResponseObjectLatency(ctx, do)
if err != nil {
scope.err(err, w, req)
return
@@ -244,9 +257,9 @@ func asPartialObjectMetadataList(result runtime.Object, groupVersion schema.Grou
if err != nil {
return nil, err
}
list.SelfLink = li.GetSelfLink()
list.ResourceVersion = li.GetResourceVersion()
list.Continue = li.GetContinue()
list.RemainingItemCount = li.GetRemainingItemCount()
return list, nil
case groupVersion == metav1.SchemeGroupVersion:
@@ -264,9 +277,9 @@ func asPartialObjectMetadataList(result runtime.Object, groupVersion schema.Grou
if err != nil {
return nil, err
}
list.SelfLink = li.GetSelfLink()
list.ResourceVersion = li.GetResourceVersion()
list.Continue = li.GetContinue()
list.RemainingItemCount = li.GetRemainingItemCount()
return list, nil
default:

View File

@@ -88,6 +88,7 @@ func StreamObject(statusCode int, gv schema.GroupVersion, s runtime.NegotiatedSe
// a client and the feature gate for APIResponseCompression is enabled.
func SerializeObject(mediaType string, encoder runtime.Encoder, hw http.ResponseWriter, req *http.Request, statusCode int, object runtime.Object) {
trace := utiltrace.New("SerializeObject",
utiltrace.Field{"audit-id", request.GetAuditIDTruncated(req.Context())},
utiltrace.Field{"method", req.Method},
utiltrace.Field{"url", req.URL.Path},
utiltrace.Field{"protocol", req.Proto},
@@ -143,8 +144,10 @@ var gzipPool = &sync.Pool{
}
const (
// defaultGzipContentEncodingLevel is set to 4 which uses less CPU than the default level
defaultGzipContentEncodingLevel = 4
// defaultGzipContentEncodingLevel is set to 1 which uses least CPU compared to higher levels, yet offers
// similar compression ratios (off by at most 1.5x, but typically within 1.1x-1.3x). For further details see -
// https://github.com/kubernetes/kubernetes/issues/112296
defaultGzipContentEncodingLevel = 1
// defaultGzipThresholdBytes is compared to the size of the first write from the stream
// (usually the entire object), and if the size is smaller no gzipping will be performed
// if the client requests it.
@@ -201,7 +204,8 @@ func (w *deferredResponseWriter) Write(p []byte) (n int, err error) {
w.trace.Step("Write call finished",
utiltrace.Field{"writer", fmt.Sprintf("%T", w.w)},
utiltrace.Field{"size", len(p)},
utiltrace.Field{"firstWrite", firstWrite})
utiltrace.Field{"firstWrite", firstWrite},
utiltrace.Field{"err", err})
}()
}
if w.hasWritten {
@@ -267,12 +271,12 @@ func WriteObjectNegotiated(s runtime.NegotiatedSerializer, restrictions negotiat
return
}
if ae := request.AuditEventFrom(req.Context()); ae != nil {
audit.LogResponseObject(ae, object, gv, s)
}
audit.LogResponseObject(req.Context(), object, gv, s)
encoder := s.EncoderForVersion(serializer.Serializer, gv)
SerializeObject(serializer.MediaType, encoder, w, req, statusCode, object)
request.TrackSerializeResponseObjectLatency(req.Context(), func() {
SerializeObject(serializer.MediaType, encoder, w, req, statusCode, object)
})
}
// ErrorNegotiated renders an error to the response. Returns the HTTP status code of the error.

View File

@@ -24,7 +24,6 @@ import (
"io/ioutil"
"net/http"
"net/url"
goruntime "runtime"
"strings"
"time"
@@ -49,7 +48,6 @@ import (
"k8s.io/apiserver/pkg/registry/rest"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/apiserver/pkg/warning"
"k8s.io/klog/v2"
)
const (
@@ -64,6 +62,10 @@ const (
// NOTE: For CREATE and UPDATE requests the API server dedups both before and after mutating admission.
// For PATCH request the API server only dedups after mutating admission.
DuplicateOwnerReferencesAfterMutatingAdmissionWarningFormat = ".metadata.ownerReferences contains duplicate entries after mutating admission happens; API server dedups owner references in 1.20+, and may reject such requests as early as 1.24; please fix your requests; duplicate UID(s) observed: %v"
// shortPrefix is one possible beginning of yaml unmarshal strict errors.
shortPrefix = "yaml: unmarshal errors:\n"
// longPrefix is the other possible beginning of yaml unmarshal strict errors.
longPrefix = "error converting YAML to JSON: yaml: unmarshal errors:\n"
)
// RequestScope encapsulates common fields across all RESTful handler methods.
@@ -90,8 +92,14 @@ type RequestScope struct {
TableConvertor rest.TableConvertor
FieldManager *fieldmanager.FieldManager
Resource schema.GroupVersionResource
Kind schema.GroupVersionKind
Resource schema.GroupVersionResource
Kind schema.GroupVersionKind
// AcceptsGroupVersionDelegate is an optional delegate that can be queried about whether a given GVK
// can be accepted in create or update requests. If nil, only scope.Kind is accepted.
// Note that this does not enable multi-version support for reads from a single endpoint.
AcceptsGroupVersionDelegate rest.GroupVersionAcceptor
Subresource string
MetaGroupVersion schema.GroupVersion
@@ -106,6 +114,17 @@ func (scope *RequestScope) err(err error, w http.ResponseWriter, req *http.Reque
responsewriters.ErrorNegotiated(err, scope.Serializer, scope.Kind.GroupVersion(), w, req)
}
// AcceptsGroupVersion returns true if the specified GroupVersion is allowed
// in create and update requests.
func (scope *RequestScope) AcceptsGroupVersion(gv schema.GroupVersion) bool {
// If there's a custom acceptor, delegate to it. This is extremely rare.
if scope.AcceptsGroupVersionDelegate != nil {
return scope.AcceptsGroupVersionDelegate.AcceptsGroupVersion(gv)
}
// Fall back to only allowing the singular Kind. This is the typical behavior.
return gv == scope.Kind.GroupVersion()
}
func (scope *RequestScope) AllowsMediaTypeTransform(mimeType, mimeSubType string, gvk *schema.GroupVersionKind) bool {
// some handlers like CRDs can't serve all the mime types that PartialObjectMetadata or Table can - if
// gvk is nil (no conversion) allow StandardSerializers to further restrict the set of mime types.
@@ -171,8 +190,7 @@ func ConnectResource(connecter rest.Connecter, scope *RequestScope, admit admiss
}
ctx := req.Context()
ctx = request.WithNamespace(ctx, namespace)
ae := request.AuditEventFrom(ctx)
admit = admission.WithAudit(admit, ae)
admit = admission.WithAudit(admit)
opts, subpath, subpathKey := connecter.NewConnectOptions()
if err := getRequestOptions(req, scope, opts, subpath, subpathKey, isSubresource); err != nil {
@@ -225,60 +243,6 @@ func (r *responder) Error(err error) {
r.scope.err(err, r.w, r.req)
}
// resultFunc is a function that returns a rest result and can be run in a goroutine
type resultFunc func() (runtime.Object, error)
// finishRequest makes a given resultFunc asynchronous and handles errors returned by the response.
// An api.Status object with status != success is considered an "error", which interrupts the normal response flow.
func finishRequest(ctx context.Context, fn resultFunc) (result runtime.Object, err error) {
// these channels need to be buffered to prevent the goroutine below from hanging indefinitely
// when the select statement reads something other than the one the goroutine sends on.
ch := make(chan runtime.Object, 1)
errCh := make(chan error, 1)
panicCh := make(chan interface{}, 1)
go func() {
// panics don't cross goroutine boundaries, so we have to handle ourselves
defer func() {
panicReason := recover()
if panicReason != nil {
// do not wrap the sentinel ErrAbortHandler panic value
if panicReason != http.ErrAbortHandler {
// Same as stdlib http server code. Manually allocate stack
// trace buffer size to prevent excessively large logs
const size = 64 << 10
buf := make([]byte, size)
buf = buf[:goruntime.Stack(buf, false)]
panicReason = fmt.Sprintf("%v\n%s", panicReason, buf)
}
// Propagate to parent goroutine
panicCh <- panicReason
}
}()
if result, err := fn(); err != nil {
errCh <- err
} else {
ch <- result
}
}()
select {
case result = <-ch:
if status, ok := result.(*metav1.Status); ok {
if status.Status != metav1.StatusSuccess {
return nil, errors.FromObject(status)
}
}
return result, nil
case err = <-errCh:
return nil, err
case p := <-panicCh:
panic(p)
case <-ctx.Done():
return nil, errors.NewTimeoutError(fmt.Sprintf("request did not complete within requested timeout %s", ctx.Err()), 0)
}
}
// transformDecodeError adds additional information into a bad-request api error when a decode fails.
func transformDecodeError(typer runtime.ObjectTyper, baseErr error, into runtime.Object, gvk *schema.GroupVersionKind, body []byte) error {
objGVKs, _, err := typer.ObjectKinds(into)
@@ -293,18 +257,6 @@ func transformDecodeError(typer runtime.ObjectTyper, baseErr error, into runtime
return errors.NewBadRequest(fmt.Sprintf("the object provided is unrecognized (must be of type %s): %v (%s)", objGVK.Kind, baseErr, summary))
}
// setSelfLink sets the self link of an object (or the child items in a list) to the base URL of the request
// plus the path and query generated by the provided linkFunc
func setSelfLink(obj runtime.Object, requestInfo *request.RequestInfo, namer ScopeNamer) error {
// TODO: SelfLink generation should return a full URL?
uri, err := namer.GenerateLink(requestInfo, obj)
if err != nil {
return nil
}
return namer.SetSelfLink(obj, uri)
}
func hasUID(obj runtime.Object) (bool, error) {
if obj == nil {
return false, nil
@@ -402,58 +354,6 @@ func dedupOwnerReferencesAndAddWarning(obj runtime.Object, requestContext contex
}
}
// setObjectSelfLink sets the self link of an object as needed.
// TODO: remove the need for the namer LinkSetters by requiring objects implement either Object or List
// interfaces
func setObjectSelfLink(ctx context.Context, obj runtime.Object, req *http.Request, namer ScopeNamer) error {
if utilfeature.DefaultFeatureGate.Enabled(features.RemoveSelfLink) {
// Ensure that for empty lists we don't return <nil> items.
if meta.IsListType(obj) && meta.LenList(obj) == 0 {
if err := meta.SetList(obj, []runtime.Object{}); err != nil {
return err
}
}
return nil
}
// We only generate list links on objects that implement ListInterface - historically we duck typed this
// check via reflection, but as we move away from reflection we require that you not only carry Items but
// ListMeta into order to be identified as a list.
if !meta.IsListType(obj) {
requestInfo, ok := request.RequestInfoFrom(ctx)
if !ok {
return fmt.Errorf("missing requestInfo")
}
return setSelfLink(obj, requestInfo, namer)
}
uri, err := namer.GenerateListLink(req)
if err != nil {
return err
}
if err := namer.SetSelfLink(obj, uri); err != nil {
klog.V(4).InfoS("Unable to set self link on object", "error", err)
}
requestInfo, ok := request.RequestInfoFrom(ctx)
if !ok {
return fmt.Errorf("missing requestInfo")
}
count := 0
err = meta.EachListItem(obj, func(obj runtime.Object) error {
count++
return setSelfLink(obj, requestInfo, namer)
})
if count == 0 {
if err := meta.SetList(obj, []runtime.Object{}); err != nil {
return err
}
}
return err
}
func summarizeData(data []byte, maxLength int) string {
switch {
case len(data) == 0:
@@ -494,6 +394,53 @@ func isDryRun(url *url.URL) bool {
return len(url.Query()["dryRun"]) != 0
}
// fieldValidation checks that the field validation feature is enabled
// and returns a valid directive of either
// - Ignore (default when feature is disabled)
// - Warn (default when feature is enabled)
// - Strict
func fieldValidation(directive string) string {
if !utilfeature.DefaultFeatureGate.Enabled(features.ServerSideFieldValidation) {
return metav1.FieldValidationIgnore
}
if directive == "" {
return metav1.FieldValidationWarn
}
return directive
}
// parseYAMLWarnings takes the strict decoding errors from the yaml decoder's output
// and parses each individual warnings, or leaves the warning as is if
// it does not look like a yaml strict decoding error.
func parseYAMLWarnings(errString string) []string {
var trimmedString string
if trimmedShortString := strings.TrimPrefix(errString, shortPrefix); len(trimmedShortString) < len(errString) {
trimmedString = trimmedShortString
} else if trimmedLongString := strings.TrimPrefix(errString, longPrefix); len(trimmedLongString) < len(errString) {
trimmedString = trimmedLongString
} else {
// not a yaml error, return as-is
return []string{errString}
}
splitStrings := strings.Split(trimmedString, "\n")
for i, s := range splitStrings {
splitStrings[i] = strings.TrimSpace(s)
}
return splitStrings
}
// addStrictDecodingWarnings confirms that the error is a strict decoding error
// and if so adds a warning for each strict decoding violation.
func addStrictDecodingWarnings(requestContext context.Context, errs []error) {
for _, e := range errs {
yamlWarnings := parseYAMLWarnings(e.Error())
for _, w := range yamlWarnings {
warning.AddWarning(requestContext, "", w)
}
}
}
type etcdError interface {
Code() grpccodes.Code
Error() string

View File

@@ -26,6 +26,7 @@ func traceFields(req *http.Request) []utiltrace.Field {
return []utiltrace.Field{
{Key: "url", Value: req.URL.Path},
{Key: "user-agent", Value: &lazyTruncatedUserAgent{req: req}},
{Key: "audit-id", Value: &lazyAuditID{req: req}},
{Key: "client", Value: &lazyClientIP{req: req}},
{Key: "accept", Value: &lazyAccept{req: req}},
{Key: "protocol", Value: req.Proto}}

View File

@@ -34,12 +34,14 @@ import (
"k8s.io/apiserver/pkg/audit"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
"k8s.io/apiserver/pkg/endpoints/handlers/finisher"
"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/features"
"k8s.io/apiserver/pkg/registry/rest"
"k8s.io/apiserver/pkg/util/dryrun"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/klog/v2"
utiltrace "k8s.io/utils/trace"
)
@@ -75,6 +77,7 @@ func UpdateResource(r rest.Updater, scope *RequestScope, admit admission.Interfa
}
body, err := limitedReadBody(req, scope.MaxRequestBodyBytes)
trace.Step("limitedReadBody done", utiltrace.Field{"len", len(body)}, utiltrace.Field{"err", err})
if err != nil {
scope.err(err, w, req)
return
@@ -101,24 +104,49 @@ func UpdateResource(r rest.Updater, scope *RequestScope, admit admission.Interfa
defaultGVK := scope.Kind
original := r.New()
validationDirective := fieldValidation(options.FieldValidation)
decodeSerializer := s.Serializer
if validationDirective == metav1.FieldValidationWarn || validationDirective == metav1.FieldValidationStrict {
decodeSerializer = s.StrictSerializer
}
decoder := scope.Serializer.DecoderToVersion(decodeSerializer, scope.HubGroupVersion)
trace.Step("About to convert to expected version")
decoder := scope.Serializer.DecoderToVersion(s.Serializer, scope.HubGroupVersion)
obj, gvk, err := decoder.Decode(body, &defaultGVK, original)
if err != nil {
err = transformDecodeError(scope.Typer, err, original, gvk, body)
scope.err(err, w, req)
return
strictError, isStrictError := runtime.AsStrictDecodingError(err)
switch {
case isStrictError && obj != nil && validationDirective == metav1.FieldValidationWarn:
addStrictDecodingWarnings(req.Context(), strictError.Errors())
case isStrictError && validationDirective == metav1.FieldValidationIgnore:
klog.Warningf("unexpected strict error when field validation is set to ignore")
fallthrough
default:
err = transformDecodeError(scope.Typer, err, original, gvk, body)
scope.err(err, w, req)
return
}
}
if gvk.GroupVersion() != defaultGVK.GroupVersion() {
err = errors.NewBadRequest(fmt.Sprintf("the API version in the data (%s) does not match the expected API version (%s)", gvk.GroupVersion(), defaultGVK.GroupVersion()))
objGV := gvk.GroupVersion()
if !scope.AcceptsGroupVersion(objGV) {
err = errors.NewBadRequest(fmt.Sprintf("the API version in the data (%s) does not match the expected API version (%s)", objGV, defaultGVK.GroupVersion()))
scope.err(err, w, req)
return
}
trace.Step("Conversion done")
ae := request.AuditEventFrom(ctx)
audit.LogRequestObject(ae, obj, scope.Resource, scope.Subresource, scope.Serializer)
admit = admission.WithAudit(admit, ae)
audit.LogRequestObject(req.Context(), obj, objGV, scope.Resource, scope.Subresource, scope.Serializer)
admit = admission.WithAudit(admit)
// if this object supports namespace info
if objectMeta, err := meta.Accessor(obj); err == nil {
// ensure namespace on the object is correct, or error if a conflicting namespace was set in the object
if err := rest.EnsureObjectNamespaceMatchesRequestNamespace(rest.ExpectedNamespaceForResource(namespace, scope.Resource), objectMeta); err != nil {
scope.err(err, w, req)
return
}
}
if err := checkName(obj, name, namespace, scope.Namer); err != nil {
scope.err(err, w, req)
@@ -163,6 +191,15 @@ func UpdateResource(r rest.Updater, scope *RequestScope, admit admission.Interfa
})
}
// Ignore changes that only affect managed fields
// timestamps. FieldManager can't know about changes
// like normalized fields, defaulted fields and other
// mutations.
// Only makes sense when SSA field manager is being used
if scope.FieldManager != nil {
transformers = append(transformers, fieldmanager.IgnoreManagedFieldsTimestampsTransformer)
}
createAuthorizerAttributes := authorizer.AttributesRecord{
User: userInfo,
ResourceRequest: true,
@@ -198,7 +235,7 @@ func UpdateResource(r rest.Updater, scope *RequestScope, admit admission.Interfa
}
// Dedup owner references before updating managed fields
dedupOwnerReferencesAndAddWarning(obj, req.Context(), false)
result, err := finishRequest(ctx, func() (runtime.Object, error) {
result, err := finisher.FinishRequest(ctx, func() (runtime.Object, error) {
result, err := requestFunc()
// If the object wasn't committed to storage because it's serialized size was too large,
// it is safe to remove managedFields (which can be large) and try again.
@@ -211,17 +248,19 @@ func UpdateResource(r rest.Updater, scope *RequestScope, admit admission.Interfa
}
return result, err
})
trace.Step("Write to database call finished", utiltrace.Field{"len", len(body)}, utiltrace.Field{"err", err})
if err != nil {
scope.err(err, w, req)
return
}
trace.Step("Object stored in database")
status := http.StatusOK
if wasCreated {
status = http.StatusCreated
}
trace.Step("About to write a response")
defer trace.Step("Writing http response done")
transformResponseObject(ctx, scope, trace, req, w, status, outputMediaType, result)
}
}
@@ -264,8 +303,9 @@ func updateToCreateOptions(uo *metav1.UpdateOptions) *metav1.CreateOptions {
return nil
}
co := &metav1.CreateOptions{
DryRun: uo.DryRun,
FieldManager: uo.FieldManager,
DryRun: uo.DryRun,
FieldManager: uo.FieldManager,
FieldValidation: uo.FieldValidation,
}
co.TypeMeta.SetGroupVersionKind(metav1.SchemeGroupVersion.WithKind("CreateOptions"))
return co

View File

@@ -19,10 +19,13 @@ package handlers
import (
"bytes"
"fmt"
"io"
"net/http"
"reflect"
"time"
"golang.org/x/net/websocket"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
@@ -31,10 +34,7 @@ import (
"k8s.io/apimachinery/pkg/watch"
"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
"k8s.io/apiserver/pkg/endpoints/metrics"
"k8s.io/apiserver/pkg/server/httplog"
"k8s.io/apiserver/pkg/util/wsstream"
"golang.org/x/net/websocket"
)
// nothing will ever be sent down this channel
@@ -163,10 +163,6 @@ type WatchServer struct {
// or over a websocket connection.
func (s *WatchServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
kind := s.Scope.Kind
metrics.RegisteredWatchers.WithContext(req.Context()).WithLabelValues(kind.Group, kind.Version, kind.Kind).Inc()
defer metrics.RegisteredWatchers.WithContext(req.Context()).WithLabelValues(kind.Group, kind.Version, kind.Kind).Dec()
w = httplog.Unlogged(req, w)
if wsstream.IsWebSocketRequest(req) {
w.Header().Set("Content-Type", s.MediaType)
@@ -190,7 +186,17 @@ func (s *WatchServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
s.Scope.err(errors.NewBadRequest(err.Error()), w, req)
return
}
e := streaming.NewEncoder(framer, s.Encoder)
var e streaming.Encoder
var memoryAllocator runtime.MemoryAllocator
if encoder, supportsAllocator := s.Encoder.(runtime.EncoderWithAllocator); supportsAllocator {
memoryAllocator = runtime.AllocatorPool.Get().(*runtime.Allocator)
defer runtime.AllocatorPool.Put(memoryAllocator)
e = streaming.NewEncoderWithAllocator(framer, encoder, memoryAllocator)
} else {
e = streaming.NewEncoder(framer, s.Encoder)
}
// ensure the connection times out
timeoutCh, cleanup := s.TimeoutFactory.TimeoutCh()
@@ -209,6 +215,19 @@ func (s *WatchServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
ch := s.Watching.ResultChan()
done := req.Context().Done()
embeddedEncodeFn := s.EmbeddedEncoder.Encode
if encoder, supportsAllocator := s.EmbeddedEncoder.(runtime.EncoderWithAllocator); supportsAllocator {
if memoryAllocator == nil {
// don't put the allocator inside the embeddedEncodeFn as that would allocate memory on every call.
// instead, we allocate the buffer for the entire watch session and release it when we close the connection.
memoryAllocator = runtime.AllocatorPool.Get().(*runtime.Allocator)
defer runtime.AllocatorPool.Put(memoryAllocator)
}
embeddedEncodeFn = func(obj runtime.Object, w io.Writer) error {
return encoder.EncodeWithAllocator(obj, w, memoryAllocator)
}
}
for {
select {
case <-done:
@@ -223,7 +242,7 @@ func (s *WatchServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
metrics.WatchEvents.WithContext(req.Context()).WithLabelValues(kind.Group, kind.Version, kind.Kind).Inc()
obj := s.Fixup(event.Object)
if err := s.EmbeddedEncoder.Encode(obj, buf); err != nil {
if err := embeddedEncodeFn(obj, buf); err != nil {
// unexpected error
utilruntime.HandleError(fmt.Errorf("unable to encode watch object %T: %v", obj, err))
return

View File

@@ -19,14 +19,13 @@ package endpoints
import (
"fmt"
"net/http"
gpath "path"
"reflect"
"sort"
"strings"
"time"
"unicode"
restful "github.com/emicklei/go-restful"
restful "github.com/emicklei/go-restful/v3"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/conversion"
"k8s.io/apimachinery/pkg/runtime"
@@ -217,7 +216,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
isSubresource := len(subresource) > 0
// If there is a subresource, namespace scoping is defined by the parent resource
namespaceScoped := true
var namespaceScoped bool
if isSubresource {
parentStorage, ok := a.group.Storage[resource]
if !ok {
@@ -251,6 +250,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
connecter, isConnecter := storage.(rest.Connecter)
storageMeta, isMetadata := storage.(rest.StorageMetadata)
storageVersionProvider, isStorageVersionProvider := storage.(rest.StorageVersionProvider)
gvAcceptor, _ := storage.(rest.GroupVersionAcceptor)
if !isMetadata {
storageMeta = defaultStorageMetadata{}
}
@@ -427,10 +427,8 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
apiResource.Namespaced = false
apiResource.Kind = resourceKind
namer := handlers.ContextBasedNaming{
SelfLinker: a.group.Linker,
ClusterScoped: true,
SelfLinkPathPrefix: gpath.Join(a.prefix, resource) + "/",
SelfLinkPathSuffix: suffix,
Namer: a.group.Namer,
ClusterScoped: true,
}
// Handler for standard REST verbs (GET, PUT, POST and DELETE).
@@ -476,10 +474,8 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
apiResource.Namespaced = true
apiResource.Kind = resourceKind
namer := handlers.ContextBasedNaming{
SelfLinker: a.group.Linker,
ClusterScoped: false,
SelfLinkPathPrefix: gpath.Join(a.prefix, namespaceParamName) + "/",
SelfLinkPathSuffix: itemPathSuffix,
Namer: a.group.Namer,
ClusterScoped: false,
}
actions = appendIf(actions, action{"LIST", resourcePath, resourceParams, namer, false}, isLister)
@@ -540,6 +536,11 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
}
}
var disabledParams []string
if !utilfeature.DefaultFeatureGate.Enabled(features.ServerSideFieldValidation) {
disabledParams = []string{"fieldValidation"}
}
// Create Routes for the actions.
// TODO: Add status documentation using Returns()
// Errors (see api/errors/errors.go as well as go-restful router):
@@ -587,6 +588,8 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
Subresource: subresource,
Kind: fqKindToRegister,
AcceptsGroupVersionDelegate: gvAcceptor,
HubGroupVersion: schema.GroupVersion{Group: fqKindToRegister.Group, Version: runtime.APIVersionInternal},
MetaGroupVersion: metav1.SchemeGroupVersion,
@@ -604,7 +607,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
a.group.Creater,
fqKindToRegister,
reqScope.HubGroupVersion,
isSubresource,
subresource,
resetFields,
)
if err != nil {
@@ -666,8 +669,6 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
// accumulate endpoint-level warnings
var (
enableWarningHeaders = utilfeature.DefaultFeatureGate.Enabled(features.WarningHeaders)
warnings []string
deprecated bool
removedRelease string
@@ -699,9 +700,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
} else {
handler = metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, handler)
}
if enableWarningHeaders {
handler = utilwarning.AddWarningsHandler(handler, warnings)
}
handler = utilwarning.AddWarningsHandler(handler, warnings)
doc := "read the specified " + kind
if isSubresource {
@@ -727,9 +726,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
doc = "list " + subresource + " of objects of kind " + kind
}
handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, restfulListResource(lister, watcher, reqScope, false, a.minRequestTimeout))
if enableWarningHeaders {
handler = utilwarning.AddWarningsHandler(handler, warnings)
}
handler = utilwarning.AddWarningsHandler(handler, warnings)
route := ws.GET(action.Path).To(handler).
Doc(doc).
Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
@@ -762,9 +759,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
doc = "replace " + subresource + " of the specified " + kind
}
handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, restfulUpdateResource(updater, reqScope, admit))
if enableWarningHeaders {
handler = utilwarning.AddWarningsHandler(handler, warnings)
}
handler = utilwarning.AddWarningsHandler(handler, warnings)
route := ws.PUT(action.Path).To(handler).
Doc(doc).
Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
@@ -776,7 +771,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
Returns(http.StatusCreated, "Created", producedObject).
Reads(defaultVersionedObject).
Writes(producedObject)
if err := AddObjectParams(ws, route, versionedUpdateOptions); err != nil {
if err := AddObjectParams(ws, route, versionedUpdateOptions, disabledParams...); err != nil {
return nil, nil, err
}
addParams(route, action.Params)
@@ -795,9 +790,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
supportedTypes = append(supportedTypes, string(types.ApplyPatchType))
}
handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, restfulPatchResource(patcher, reqScope, admit, supportedTypes))
if enableWarningHeaders {
handler = utilwarning.AddWarningsHandler(handler, warnings)
}
handler = utilwarning.AddWarningsHandler(handler, warnings)
route := ws.PATCH(action.Path).To(handler).
Doc(doc).
Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
@@ -805,9 +798,11 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
Operation("patch"+namespaced+kind+strings.Title(subresource)+operationSuffix).
Produces(append(storageMeta.ProducesMIMETypes(action.Verb), mediaTypes...)...).
Returns(http.StatusOK, "OK", producedObject).
// Patch can return 201 when a server side apply is requested
Returns(http.StatusCreated, "Created", producedObject).
Reads(metav1.Patch{}).
Writes(producedObject)
if err := AddObjectParams(ws, route, versionedPatchOptions); err != nil {
if err := AddObjectParams(ws, route, versionedPatchOptions, disabledParams...); err != nil {
return nil, nil, err
}
addParams(route, action.Params)
@@ -820,9 +815,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
handler = restfulCreateResource(creater, reqScope, admit)
}
handler = metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, handler)
if enableWarningHeaders {
handler = utilwarning.AddWarningsHandler(handler, warnings)
}
handler = utilwarning.AddWarningsHandler(handler, warnings)
article := GetArticleForNoun(kind, " ")
doc := "create" + article + kind
if isSubresource {
@@ -840,7 +833,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
Returns(http.StatusAccepted, "Accepted", producedObject).
Reads(defaultVersionedObject).
Writes(producedObject)
if err := AddObjectParams(ws, route, versionedCreateOptions); err != nil {
if err := AddObjectParams(ws, route, versionedCreateOptions, disabledParams...); err != nil {
return nil, nil, err
}
addParams(route, action.Params)
@@ -856,9 +849,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
deleteReturnType = producedObject
}
handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, restfulDeleteResource(gracefulDeleter, isGracefulDeleter, reqScope, admit))
if enableWarningHeaders {
handler = utilwarning.AddWarningsHandler(handler, warnings)
}
handler = utilwarning.AddWarningsHandler(handler, warnings)
route := ws.DELETE(action.Path).To(handler).
Doc(doc).
Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
@@ -882,9 +873,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
doc = "delete collection of " + subresource + " of a " + kind
}
handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, restfulDeleteCollection(collectionDeleter, isCollectionDeleter, reqScope, admit))
if enableWarningHeaders {
handler = utilwarning.AddWarningsHandler(handler, warnings)
}
handler = utilwarning.AddWarningsHandler(handler, warnings)
route := ws.DELETE(action.Path).To(handler).
Doc(doc).
Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
@@ -912,9 +901,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
}
doc += ". deprecated: use the 'watch' parameter with a list operation instead, filtered to a single item with the 'fieldSelector' parameter."
handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, restfulListResource(lister, watcher, reqScope, true, a.minRequestTimeout))
if enableWarningHeaders {
handler = utilwarning.AddWarningsHandler(handler, warnings)
}
handler = utilwarning.AddWarningsHandler(handler, warnings)
route := ws.GET(action.Path).To(handler).
Doc(doc).
Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
@@ -935,9 +922,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
}
doc += ". deprecated: use the 'watch' parameter with a list operation instead."
handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, restfulListResource(lister, watcher, reqScope, true, a.minRequestTimeout))
if enableWarningHeaders {
handler = utilwarning.AddWarningsHandler(handler, warnings)
}
handler = utilwarning.AddWarningsHandler(handler, warnings)
route := ws.GET(action.Path).To(handler).
Doc(doc).
Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
@@ -961,9 +946,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
doc = "connect " + method + " requests to " + subresource + " of " + kind
}
handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, restfulConnectResource(connecter, reqScope, admit, path, isSubresource))
if enableWarningHeaders {
handler = utilwarning.AddWarningsHandler(handler, warnings)
}
handler = utilwarning.AddWarningsHandler(handler, warnings)
route := ws.Method(method).Path(action.Path).
To(handler).
Doc(doc).
@@ -1067,7 +1050,7 @@ func AddObjectParams(ws *restful.WebService, route *restful.RouteBuilder, obj in
}
switch sf.Type.Kind() {
case reflect.Interface, reflect.Struct:
case reflect.Ptr:
case reflect.Pointer:
// TODO: This is a hack to let metav1.Time through. This needs to be fixed in a more generic way eventually. bug #36191
if (sf.Type.Elem().Kind() == reflect.Interface || sf.Type.Elem().Kind() == reflect.Struct) && strings.TrimPrefix(sf.Type.String(), "*") != "metav1.Time" {
continue

View File

@@ -1,8 +1,6 @@
# See the OWNERS docs at https://go.k8s.io/owners
reviewers:
- wojtek-t
- jimmidyson
- wojtek-t
approvers:
- logicalhan
- logicalhan

View File

@@ -17,9 +17,7 @@ limitations under the License.
package metrics
import (
"bufio"
"context"
"net"
"net/http"
"net/url"
"strconv"
@@ -27,13 +25,14 @@ import (
"sync"
"time"
restful "github.com/emicklei/go-restful"
restful "github.com/emicklei/go-restful/v3"
"k8s.io/apimachinery/pkg/apis/meta/v1/validation"
"k8s.io/apimachinery/pkg/types"
utilsets "k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apiserver/pkg/audit"
"k8s.io/apiserver/pkg/authentication/user"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/endpoints/responsewriter"
"k8s.io/apiserver/pkg/features"
utilfeature "k8s.io/apiserver/pkg/util/feature"
compbasemetrics "k8s.io/component-base/metrics"
@@ -54,7 +53,7 @@ const (
/*
* By default, all the following metrics are defined as falling under
* ALPHA stability level https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/1209-metrics-stability/20190404-kubernetes-control-plane-metrics-stability.md#stability-classes)
* ALPHA stability level https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/1209-metrics-stability/kubernetes-control-plane-metrics-stability.md#stability-classes)
*
* Promoting the stability level of the metric is a responsibility of the component owner, since it
* involves explicitly acknowledging support for the metric across multiple releases, in accordance with
@@ -65,7 +64,7 @@ var (
&compbasemetrics.GaugeOpts{
Name: "apiserver_requested_deprecated_apis",
Help: "Gauge of deprecated APIs that have been requested, broken out by API group, version, resource, subresource, and removed_release.",
StabilityLevel: compbasemetrics.ALPHA,
StabilityLevel: compbasemetrics.STABLE,
},
[]string{"group", "version", "resource", "subresource", "removed_release"},
)
@@ -80,11 +79,11 @@ var (
},
[]string{"verb", "dry_run", "group", "version", "resource", "subresource", "scope", "component", "code"},
)
longRunningRequestGauge = compbasemetrics.NewGaugeVec(
longRunningRequestsGauge = compbasemetrics.NewGaugeVec(
&compbasemetrics.GaugeOpts{
Name: "apiserver_longrunning_gauge",
Name: "apiserver_longrunning_requests",
Help: "Gauge of all active long-running apiserver requests broken out by verb, group, version, resource, scope and component. Not all requests are tracked this way.",
StabilityLevel: compbasemetrics.ALPHA,
StabilityLevel: compbasemetrics.STABLE,
},
[]string{"verb", "group", "version", "resource", "subresource", "scope", "component"},
)
@@ -95,31 +94,48 @@ var (
// This metric is used for verifying api call latencies SLO,
// as well as tracking regressions in this aspects.
// Thus we customize buckets significantly, to empower both usecases.
Buckets: []float64{0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0,
1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5, 6, 7, 8, 9, 10, 15, 20, 25, 30, 40, 50, 60},
Buckets: []float64{0.005, 0.025, 0.05, 0.1, 0.2, 0.4, 0.6, 0.8, 1.0, 1.25, 1.5, 2, 3,
4, 5, 6, 8, 10, 15, 20, 30, 45, 60},
StabilityLevel: compbasemetrics.STABLE,
},
[]string{"verb", "dry_run", "group", "version", "resource", "subresource", "scope", "component"},
)
requestSloLatencies = compbasemetrics.NewHistogramVec(
&compbasemetrics.HistogramOpts{
Name: "apiserver_request_slo_duration_seconds",
Help: "Response latency distribution (not counting webhook duration) in seconds for each verb, group, version, resource, subresource, scope and component.",
// This metric is supplementary to the requestLatencies metric.
// It measures request duration excluding webhooks as they are mostly
// dependant on user configuration.
Buckets: []float64{0.05, 0.1, 0.2, 0.4, 0.6, 0.8, 1.0, 1.25, 1.5, 2, 3,
4, 5, 6, 8, 10, 15, 20, 30, 45, 60},
StabilityLevel: compbasemetrics.ALPHA,
},
[]string{"verb", "group", "version", "resource", "subresource", "scope", "component"},
)
fieldValidationRequestLatencies = compbasemetrics.NewHistogramVec(
&compbasemetrics.HistogramOpts{
Name: "field_validation_request_duration_seconds",
Help: "Response latency distribution in seconds for each field validation value and whether field validation is enabled or not",
// This metric is supplementary to the requestLatencies metric.
// It measures request durations for the various field validation
// values.
Buckets: []float64{0.05, 0.1, 0.2, 0.4, 0.6, 0.8, 1.0, 1.25, 1.5, 2, 3,
4, 5, 6, 8, 10, 15, 20, 30, 45, 60},
StabilityLevel: compbasemetrics.ALPHA,
},
[]string{"field_validation", "enabled"},
)
responseSizes = compbasemetrics.NewHistogramVec(
&compbasemetrics.HistogramOpts{
Name: "apiserver_response_sizes",
Help: "Response size distribution in bytes for each group, version, verb, resource, subresource, scope and component.",
// Use buckets ranging from 1000 bytes (1KB) to 10^9 bytes (1GB).
Buckets: compbasemetrics.ExponentialBuckets(1000, 10.0, 7),
StabilityLevel: compbasemetrics.ALPHA,
StabilityLevel: compbasemetrics.STABLE,
},
[]string{"verb", "group", "version", "resource", "subresource", "scope", "component"},
)
// DroppedRequests is a number of requests dropped with 'Try again later' response"
DroppedRequests = compbasemetrics.NewCounterVec(
&compbasemetrics.CounterOpts{
Name: "apiserver_dropped_requests_total",
Help: "Number of requests dropped with 'Try again later' response",
StabilityLevel: compbasemetrics.ALPHA,
},
[]string{"request_kind"},
)
// TLSHandshakeErrors is a number of requests dropped with 'TLS handshake error from' error
TLSHandshakeErrors = compbasemetrics.NewCounter(
&compbasemetrics.CounterOpts{
@@ -128,15 +144,6 @@ var (
StabilityLevel: compbasemetrics.ALPHA,
},
)
// RegisteredWatchers is a number of currently registered watchers splitted by resource.
RegisteredWatchers = compbasemetrics.NewGaugeVec(
&compbasemetrics.GaugeOpts{
Name: "apiserver_registered_watchers",
Help: "Number of currently registered watchers for a given resources",
StabilityLevel: compbasemetrics.ALPHA,
},
[]string{"group", "version", "kind"},
)
WatchEvents = compbasemetrics.NewCounterVec(
&compbasemetrics.CounterOpts{
Name: "apiserver_watch_events_total",
@@ -160,7 +167,7 @@ var (
&compbasemetrics.GaugeOpts{
Name: "apiserver_current_inflight_requests",
Help: "Maximal number of currently used inflight request limit of this apiserver per request kind in last second.",
StabilityLevel: compbasemetrics.ALPHA,
StabilityLevel: compbasemetrics.STABLE,
},
[]string{"request_kind"},
)
@@ -211,15 +218,47 @@ var (
[]string{"verb", "group", "version", "resource", "subresource", "scope"},
)
// requestPostTimeoutTotal tracks the activity of the executing request handler after the associated request
// has been timed out by the apiserver.
// source: the name of the handler that is recording this metric. Currently, we have two:
// - timeout-handler: the "executing" handler returns after the timeout filter times out the request.
// - rest-handler: the "executing" handler returns after the rest layer times out the request.
// status: whether the handler panicked or threw an error, possible values:
// - 'panic': the handler panicked
// - 'error': the handler return an error
// - 'ok': the handler returned a result (no error and no panic)
// - 'pending': the handler is still running in the background and it did not return
// within the wait threshold.
requestPostTimeoutTotal = compbasemetrics.NewCounterVec(
&compbasemetrics.CounterOpts{
Name: "apiserver_request_post_timeout_total",
Help: "Tracks the activity of the request handlers after the associated requests have been timed out by the apiserver",
StabilityLevel: compbasemetrics.ALPHA,
},
[]string{"source", "status"},
)
requestTimestampComparisonDuration = compbasemetrics.NewHistogramVec(
&compbasemetrics.HistogramOpts{
Name: "apiserver_request_timestamp_comparison_time",
Help: "Time taken for comparison of old vs new objects in UPDATE or PATCH requests",
Buckets: []float64{0.0001, 0.0003, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1.0, 5.0},
StabilityLevel: compbasemetrics.ALPHA,
},
// Path the code takes to reach a conclusion:
// i.e. unequalObjectsFast, unequalObjectsSlow, equalObjectsSlow
[]string{"code_path"},
)
metrics = []resettableCollector{
deprecatedRequestGauge,
requestCounter,
longRunningRequestGauge,
longRunningRequestsGauge,
requestLatencies,
requestSloLatencies,
fieldValidationRequestLatencies,
responseSizes,
DroppedRequests,
TLSHandshakeErrors,
RegisteredWatchers,
WatchEvents,
WatchEventsSizes,
currentInflightRequests,
@@ -228,6 +267,8 @@ var (
apiSelfRequestCounter,
requestFilterDuration,
requestAbortsTotal,
requestPostTimeoutTotal,
requestTimestampComparisonDuration,
}
// these are the valid request methods which we report in our metrics. Any other request methods
@@ -271,6 +312,36 @@ const (
removedReleaseAnnotationKey = "k8s.io/removed-release"
)
const (
// The source that is recording the apiserver_request_post_timeout_total metric.
// The "executing" request handler returns after the timeout filter times out the request.
PostTimeoutSourceTimeoutHandler = "timeout-handler"
// The source that is recording the apiserver_request_post_timeout_total metric.
// The "executing" request handler returns after the rest layer times out the request.
PostTimeoutSourceRestHandler = "rest-handler"
)
const (
// The executing request handler panicked after the request had
// been timed out by the apiserver.
PostTimeoutHandlerPanic = "panic"
// The executing request handler has returned an error to the post-timeout
// receiver after the request had been timed out by the apiserver.
PostTimeoutHandlerError = "error"
// The executing request handler has returned a result to the post-timeout
// receiver after the request had been timed out by the apiserver.
PostTimeoutHandlerOK = "ok"
// The executing request handler has not panicked or returned any error/result to
// the post-timeout receiver yet after the request had been timed out by the apiserver.
// The post-timeout receiver gives up after waiting for certain threshold and if the
// executing request handler has not returned yet we use the following label.
PostTimeoutHandlerPending = "pending"
)
var registerMetrics sync.Once
// Register all metrics.
@@ -308,6 +379,14 @@ func RecordFilterLatency(ctx context.Context, name string, elapsed time.Duration
requestFilterDuration.WithContext(ctx).WithLabelValues(name).Observe(elapsed.Seconds())
}
func RecordTimestampComparisonLatency(codePath string, elapsed time.Duration) {
requestTimestampComparisonDuration.WithLabelValues(codePath).Observe(elapsed.Seconds())
}
func RecordRequestPostTimeout(source string, status string) {
requestPostTimeoutTotal.WithLabelValues(source, status).Inc()
}
// RecordRequestAbort records that the request was aborted possibly due to a timeout.
func RecordRequestAbort(req *http.Request, requestInfo *request.RequestInfo) {
if requestInfo == nil {
@@ -315,7 +394,7 @@ func RecordRequestAbort(req *http.Request, requestInfo *request.RequestInfo) {
}
scope := CleanScope(requestInfo)
reportedVerb := cleanVerb(canonicalVerb(strings.ToUpper(req.Method), scope), req)
reportedVerb := cleanVerb(CanonicalVerb(strings.ToUpper(req.Method), scope), getVerbIfWatch(req), req)
resource := requestInfo.Resource
subresource := requestInfo.Subresource
group := requestInfo.APIGroup
@@ -324,6 +403,27 @@ func RecordRequestAbort(req *http.Request, requestInfo *request.RequestInfo) {
requestAbortsTotal.WithContext(req.Context()).WithLabelValues(reportedVerb, group, version, resource, subresource, scope).Inc()
}
// RecordDroppedRequest records that the request was rejected via http.TooManyRequests.
func RecordDroppedRequest(req *http.Request, requestInfo *request.RequestInfo, component string, isMutatingRequest bool) {
if requestInfo == nil {
requestInfo = &request.RequestInfo{Verb: req.Method, Path: req.URL.Path}
}
scope := CleanScope(requestInfo)
dryRun := cleanDryRun(req.URL)
// We don't use verb from <requestInfo>, as this may be propagated from
// InstrumentRouteFunc which is registered in installer.go with predefined
// list of verbs (different than those translated to RequestInfo).
// However, we need to tweak it e.g. to differentiate GET from LIST.
reportedVerb := cleanVerb(CanonicalVerb(strings.ToUpper(req.Method), scope), getVerbIfWatch(req), req)
if requestInfo.IsResourceRequest {
requestCounter.WithContext(req.Context()).WithLabelValues(reportedVerb, dryRun, requestInfo.APIGroup, requestInfo.APIVersion, requestInfo.Resource, requestInfo.Subresource, scope, component, codeToString(http.StatusTooManyRequests)).Inc()
} else {
requestCounter.WithContext(req.Context()).WithLabelValues(reportedVerb, dryRun, "", "", "", requestInfo.Subresource, scope, component, codeToString(http.StatusTooManyRequests)).Inc()
}
}
// RecordRequestTermination records that the request was terminated early as part of a resource
// preservation or apiserver self-defense mechanism (e.g. timeouts, maxinflight throttling,
// proxyHandler errors). RecordRequestTermination should only be called zero or one times
@@ -338,7 +438,7 @@ func RecordRequestTermination(req *http.Request, requestInfo *request.RequestInf
// InstrumentRouteFunc which is registered in installer.go with predefined
// list of verbs (different than those translated to RequestInfo).
// However, we need to tweak it e.g. to differentiate GET from LIST.
reportedVerb := cleanVerb(canonicalVerb(strings.ToUpper(req.Method), scope), req)
reportedVerb := cleanVerb(CanonicalVerb(strings.ToUpper(req.Method), scope), getVerbIfWatch(req), req)
if requestInfo.IsResourceRequest {
requestTerminationsTotal.WithContext(req.Context()).WithLabelValues(reportedVerb, requestInfo.APIGroup, requestInfo.APIVersion, requestInfo.Resource, requestInfo.Subresource, scope, component, codeToString(code)).Inc()
@@ -360,12 +460,12 @@ func RecordLongRunning(req *http.Request, requestInfo *request.RequestInfo, comp
// InstrumentRouteFunc which is registered in installer.go with predefined
// list of verbs (different than those translated to RequestInfo).
// However, we need to tweak it e.g. to differentiate GET from LIST.
reportedVerb := cleanVerb(canonicalVerb(strings.ToUpper(req.Method), scope), req)
reportedVerb := cleanVerb(CanonicalVerb(strings.ToUpper(req.Method), scope), getVerbIfWatch(req), req)
if requestInfo.IsResourceRequest {
g = longRunningRequestGauge.WithContext(req.Context()).WithLabelValues(reportedVerb, requestInfo.APIGroup, requestInfo.APIVersion, requestInfo.Resource, requestInfo.Subresource, scope, component)
g = longRunningRequestsGauge.WithContext(req.Context()).WithLabelValues(reportedVerb, requestInfo.APIGroup, requestInfo.APIVersion, requestInfo.Resource, requestInfo.Subresource, scope, component)
} else {
g = longRunningRequestGauge.WithContext(req.Context()).WithLabelValues(reportedVerb, "", "", "", requestInfo.Path, scope, component)
g = longRunningRequestsGauge.WithContext(req.Context()).WithLabelValues(reportedVerb, "", "", "", requestInfo.Path, scope, component)
}
g.Inc()
defer g.Dec()
@@ -379,7 +479,7 @@ func MonitorRequest(req *http.Request, verb, group, version, resource, subresour
// InstrumentRouteFunc which is registered in installer.go with predefined
// list of verbs (different than those translated to RequestInfo).
// However, we need to tweak it e.g. to differentiate GET from LIST.
reportedVerb := cleanVerb(canonicalVerb(strings.ToUpper(req.Method), scope), req)
reportedVerb := cleanVerb(CanonicalVerb(strings.ToUpper(req.Method), scope), verb, req)
dryRun := cleanDryRun(req.URL)
elapsedSeconds := elapsed.Seconds()
@@ -397,6 +497,14 @@ func MonitorRequest(req *http.Request, verb, group, version, resource, subresour
}
}
requestLatencies.WithContext(req.Context()).WithLabelValues(reportedVerb, dryRun, group, version, resource, subresource, scope, component).Observe(elapsedSeconds)
fieldValidation := cleanFieldValidation(req.URL)
fieldValidationEnabled := strconv.FormatBool(utilfeature.DefaultFeatureGate.Enabled(features.ServerSideFieldValidation))
fieldValidationRequestLatencies.WithContext(req.Context()).WithLabelValues(fieldValidation, fieldValidationEnabled)
if wd, ok := request.LatencyTrackersFrom(req.Context()); ok {
sloLatency := elapsedSeconds - (wd.MutatingWebhookTracker.GetLatency() + wd.ValidatingWebhookTracker.GetLatency()).Seconds()
requestSloLatencies.WithContext(req.Context()).WithLabelValues(reportedVerb, group, version, resource, subresource, scope, component).Observe(sloLatency)
}
// We are only interested in response sizes of read requests.
if verb == "GET" || verb == "LIST" {
responseSizes.WithContext(req.Context()).WithLabelValues(reportedVerb, group, version, resource, subresource, scope, component).Observe(float64(respSize))
@@ -414,15 +522,7 @@ func InstrumentRouteFunc(verb, group, version, resource, subresource, scope, com
delegate := &ResponseWriterDelegator{ResponseWriter: response.ResponseWriter}
_, cn := response.ResponseWriter.(http.CloseNotifier)
_, fl := response.ResponseWriter.(http.Flusher)
_, hj := response.ResponseWriter.(http.Hijacker)
var rw http.ResponseWriter
if cn && fl && hj {
rw = &fancyResponseWriterDelegator{delegate}
} else {
rw = delegate
}
rw := responsewriter.WrapForHTTP1Or2(delegate)
response.ResponseWriter = rw
routeFunc(req, response)
@@ -440,15 +540,7 @@ func InstrumentHandlerFunc(verb, group, version, resource, subresource, scope, c
}
delegate := &ResponseWriterDelegator{ResponseWriter: w}
_, cn := w.(http.CloseNotifier)
_, fl := w.(http.Flusher)
_, hj := w.(http.Hijacker)
if cn && fl && hj {
w = &fancyResponseWriterDelegator{delegate}
} else {
w = delegate
}
w = responsewriter.WrapForHTTP1Or2(delegate)
handler(w, req)
@@ -458,12 +550,12 @@ func InstrumentHandlerFunc(verb, group, version, resource, subresource, scope, c
// CleanScope returns the scope of the request.
func CleanScope(requestInfo *request.RequestInfo) string {
if requestInfo.Name != "" || requestInfo.Verb == "create" {
return "resource"
}
if requestInfo.Namespace != "" {
return "namespace"
}
if requestInfo.Name != "" {
return "resource"
}
if requestInfo.IsResourceRequest {
return "cluster"
}
@@ -471,7 +563,9 @@ func CleanScope(requestInfo *request.RequestInfo) string {
return ""
}
func canonicalVerb(verb string, scope string) string {
// CanonicalVerb distinguishes LISTs from GETs (and HEADs). It assumes verb is
// UPPERCASE.
func CanonicalVerb(verb string, scope string) string {
switch verb {
case "GET", "HEAD":
if scope != "resource" && scope != "" {
@@ -483,15 +577,12 @@ func canonicalVerb(verb string, scope string) string {
}
}
func cleanVerb(verb string, request *http.Request) string {
// CleanVerb returns a normalized verb, so that it is easy to tell WATCH from
// LIST and APPLY from PATCH.
func CleanVerb(verb string, request *http.Request) string {
reportedVerb := verb
if verb == "LIST" {
// see apimachinery/pkg/runtime/conversion.go Convert_Slice_string_To_bool
if values := request.URL.Query()["watch"]; len(values) > 0 {
if value := strings.ToLower(values[0]); value != "0" && value != "false" {
reportedVerb = "WATCH"
}
}
if suggestedVerb := getVerbIfWatch(request); suggestedVerb == "WATCH" {
reportedVerb = "WATCH"
}
// normalize the legacy WATCHLIST to WATCH to ensure users aren't surprised by metrics
if verb == "WATCHLIST" {
@@ -500,12 +591,40 @@ func cleanVerb(verb string, request *http.Request) string {
if verb == "PATCH" && request.Header.Get("Content-Type") == string(types.ApplyPatchType) && utilfeature.DefaultFeatureGate.Enabled(features.ServerSideApply) {
reportedVerb = "APPLY"
}
return reportedVerb
}
// cleanVerb additionally ensures that unknown verbs don't clog up the metrics.
func cleanVerb(verb, suggestedVerb string, request *http.Request) string {
// CanonicalVerb (being an input for this function) doesn't handle correctly the
// deprecated path pattern for watch of:
// GET /api/{version}/watch/{resource}
// We correct it manually based on the pass verb from the installer.
var reportedVerb string
if suggestedVerb == "WATCH" || suggestedVerb == "WATCHLIST" {
reportedVerb = "WATCH"
} else {
reportedVerb = CleanVerb(verb, request)
}
if validRequestMethods.Has(reportedVerb) {
return reportedVerb
}
return OtherRequestMethod
}
// getVerbIfWatch additionally ensures that GET or List would be transformed to WATCH
func getVerbIfWatch(req *http.Request) string {
if strings.ToUpper(req.Method) == "GET" || strings.ToUpper(req.Method) == "LIST" {
// see apimachinery/pkg/runtime/conversion.go Convert_Slice_string_To_bool
if values := req.URL.Query()["watch"]; len(values) > 0 {
if value := strings.ToLower(values[0]); value != "0" && value != "false" {
return "WATCH"
}
}
}
return ""
}
func cleanDryRun(u *url.URL) string {
// avoid allocating when we don't see dryRun in the query
if !strings.Contains(u.RawQuery, "dryRun") {
@@ -522,6 +641,24 @@ func cleanDryRun(u *url.URL) string {
return strings.Join(utilsets.NewString(dryRun...).List(), ",")
}
func cleanFieldValidation(u *url.URL) string {
// avoid allocating when we don't see dryRun in the query
if !strings.Contains(u.RawQuery, "fieldValidation") {
return ""
}
fieldValidation := u.Query()["fieldValidation"]
if len(fieldValidation) != 1 {
return "invalid"
}
if errs := validation.ValidateFieldValidation(nil, fieldValidation[0]); len(errs) > 0 {
return "invalid"
}
return fieldValidation[0]
}
var _ http.ResponseWriter = (*ResponseWriterDelegator)(nil)
var _ responsewriter.UserProvidedDecorator = (*ResponseWriterDelegator)(nil)
// ResponseWriterDelegator interface wraps http.ResponseWriter to additionally record content-length, status-code, etc.
type ResponseWriterDelegator struct {
http.ResponseWriter
@@ -531,6 +668,10 @@ type ResponseWriterDelegator struct {
wroteHeader bool
}
func (r *ResponseWriterDelegator) Unwrap() http.ResponseWriter {
return r.ResponseWriter
}
func (r *ResponseWriterDelegator) WriteHeader(code int) {
r.status = code
r.wroteHeader = true
@@ -554,22 +695,6 @@ func (r *ResponseWriterDelegator) ContentLength() int {
return int(r.written)
}
type fancyResponseWriterDelegator struct {
*ResponseWriterDelegator
}
func (f *fancyResponseWriterDelegator) CloseNotify() <-chan bool {
return f.ResponseWriter.(http.CloseNotifier).CloseNotify()
}
func (f *fancyResponseWriterDelegator) Flush() {
f.ResponseWriter.(http.Flusher).Flush()
}
func (f *fancyResponseWriterDelegator) Hijack() (net.Conn, *bufio.ReadWriter, error) {
return f.ResponseWriter.(http.Hijacker).Hijack()
}
// Small optimization over Itoa
func codeToString(s int) string {
switch s {

View File

@@ -24,13 +24,13 @@ import (
"strings"
"unicode"
restful "github.com/emicklei/go-restful"
"github.com/go-openapi/spec"
restful "github.com/emicklei/go-restful/v3"
"k8s.io/apimachinery/pkg/apis/meta/v1"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/kube-openapi/pkg/util"
"k8s.io/kube-openapi/pkg/validation/spec"
)
var verbs = util.NewTrie([]string{"get", "log", "read", "replace", "patch", "delete", "deletecollection", "watch", "connect", "proxy", "list", "create", "patch"})

View File

@@ -1,4 +1,4 @@
# See the OWNERS docs at https://go.k8s.io/owners
reviewers:
- sttts
- sttts

View File

@@ -0,0 +1,65 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package request
import (
"context"
"k8s.io/apimachinery/pkg/types"
)
type auditIDKeyType int
// auditIDKey is the key to associate the Audit-ID value of a request.
const auditIDKey auditIDKeyType = iota
// WithAuditID returns a copy of the parent context into which the Audit-ID
// associated with the request is set.
//
// If the specified auditID is empty, no value is set and the parent context is returned as is.
func WithAuditID(parent context.Context, auditID types.UID) context.Context {
if auditID == "" {
return parent
}
return WithValue(parent, auditIDKey, auditID)
}
// AuditIDFrom returns the value of the audit ID from the request context.
func AuditIDFrom(ctx context.Context) (types.UID, bool) {
auditID, ok := ctx.Value(auditIDKey).(types.UID)
return auditID, ok
}
// GetAuditIDTruncated returns the audit ID (truncated) from the request context.
// If the length of the Audit-ID value exceeds the limit, we truncate it to keep
// the first N (maxAuditIDLength) characters.
// This is intended to be used in logging only.
func GetAuditIDTruncated(ctx context.Context) string {
auditID, ok := AuditIDFrom(ctx)
if !ok {
return ""
}
// if the user has specified a very long audit ID then we will use the first N characters
// Note: assuming Audit-ID header is in ASCII
const maxAuditIDLength = 64
if len(auditID) > maxAuditIDLength {
auditID = auditID[0:maxAuditIDLength]
}
return string(auditID)
}

View File

@@ -20,7 +20,6 @@ import (
"context"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apiserver/pkg/apis/audit"
"k8s.io/apiserver/pkg/authentication/user"
)
@@ -33,9 +32,6 @@ const (
// userKey is the context key for the request user.
userKey
// auditKey is the context key for the audit event.
auditKey
)
// NewContext instantiates a base context object for request flows.
@@ -80,14 +76,3 @@ func UserFrom(ctx context.Context) (user.Info, bool) {
user, ok := ctx.Value(userKey).(user.Info)
return user, ok
}
// WithAuditEvent returns set audit event struct.
func WithAuditEvent(parent context.Context, ev *audit.Event) context.Context {
return WithValue(parent, auditKey, ev)
}
// AuditEventFrom returns the audit event struct on the ctx
func AuditEventFrom(ctx context.Context) *audit.Event {
ev, _ := ctx.Value(auditKey).(*audit.Event)
return ev
}

View File

@@ -0,0 +1,272 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package request
import (
"context"
"sync"
"time"
"k8s.io/utils/clock"
)
func sumDuration(d1 time.Duration, d2 time.Duration) time.Duration {
return d1 + d2
}
func maxDuration(d1 time.Duration, d2 time.Duration) time.Duration {
if d1 > d2 {
return d1
}
return d2
}
// DurationTracker is a simple interface for tracking functions duration,
// it is safe for concurrent use by multiple goroutines.
type DurationTracker interface {
// Track measures time spent in the given function f and
// aggregates measured duration using aggregateFunction.
// if Track is invoked with f from multiple goroutines concurrently,
// then f must be safe to be invoked concurrently by multiple goroutines.
Track(f func())
// TrackDuration tracks latency from the specified duration
// and aggregate it using aggregateFunction
TrackDuration(time.Duration)
// GetLatency returns the total latency incurred so far
GetLatency() time.Duration
}
// durationTracker implements DurationTracker by measuring function time
// using given clock and aggregates the duration using given aggregate function
type durationTracker struct {
clock clock.Clock
latency time.Duration
mu sync.Mutex
aggregateFunction func(time.Duration, time.Duration) time.Duration
}
// Track measures time spent in given function and aggregates measured
// duration using aggregateFunction
func (t *durationTracker) Track(f func()) {
startedAt := t.clock.Now()
defer func() {
duration := t.clock.Since(startedAt)
t.mu.Lock()
defer t.mu.Unlock()
t.latency = t.aggregateFunction(t.latency, duration)
}()
f()
}
// TrackDuration tracks latency from the given duration
// using aggregateFunction
func (t *durationTracker) TrackDuration(d time.Duration) {
t.mu.Lock()
defer t.mu.Unlock()
t.latency = t.aggregateFunction(t.latency, d)
}
// GetLatency returns aggregated latency tracked by a tracker
func (t *durationTracker) GetLatency() time.Duration {
t.mu.Lock()
defer t.mu.Unlock()
return t.latency
}
func newSumLatencyTracker(c clock.Clock) DurationTracker {
return &durationTracker{
clock: c,
aggregateFunction: sumDuration,
}
}
func newMaxLatencyTracker(c clock.Clock) DurationTracker {
return &durationTracker{
clock: c,
aggregateFunction: maxDuration,
}
}
// LatencyTrackers stores trackers used to measure latecny incurred in
// components within the apiserver.
type LatencyTrackers struct {
// MutatingWebhookTracker tracks the latency incurred in mutating webhook(s).
// Since mutating webhooks are done sequentially, latency
// is aggregated using sum function.
MutatingWebhookTracker DurationTracker
// ValidatingWebhookTracker tracks the latency incurred in validating webhook(s).
// Validate webhooks are done in parallel, so max function is used.
ValidatingWebhookTracker DurationTracker
// StorageTracker tracks the latency incurred inside the storage layer,
// it accounts for the time it takes to send data to the underlying
// storage layer (etcd) and get the complete response back.
// If a request involves N (N>=1) round trips to the underlying
// stogare layer, the latency will account for the total duration
// from these N round trips.
// It does not include the time incurred in admission, or validation.
StorageTracker DurationTracker
// TransformTracker tracks the latency incurred in transforming the
// response object(s) returned from the underlying storage layer.
// This includes transforming the object to user's desired form
// (ie. as Table), and also setting appropriate API level fields.
// This does not include the latency incurred in serialization
// (json or protobuf) of the response object or writing
// of it to the http ResponseWriter object.
TransformTracker DurationTracker
// SerializationTracker tracks the latency incurred in serialization
// (json or protobuf) of the response object.
// NOTE: serialization and writing of the serialized raw bytes to the
// associated http ResponseWriter object are interleaved, and hence
// the latency measured here will include the time spent writing the
// serialized raw bytes to the http ResponseWriter object.
SerializationTracker DurationTracker
// ResponseWriteTracker tracks the latency incurred in writing the
// serialized raw bytes to the http ResponseWriter object (via the
// Write method) associated with the request.
// The Write method can be invoked multiple times, so we use a
// latency tracker that sums up the duration from each call.
ResponseWriteTracker DurationTracker
}
type latencyTrackersKeyType int
// latencyTrackersKey is the key that associates a LatencyTrackers
// instance with the request context.
const latencyTrackersKey latencyTrackersKeyType = iota
// WithLatencyTrackers returns a copy of parent context to which an
// instance of LatencyTrackers is added.
func WithLatencyTrackers(parent context.Context) context.Context {
return WithLatencyTrackersAndCustomClock(parent, clock.RealClock{})
}
// WithLatencyTrackersAndCustomClock returns a copy of parent context to which
// an instance of LatencyTrackers is added. Tracers use given clock.
func WithLatencyTrackersAndCustomClock(parent context.Context, c clock.Clock) context.Context {
return WithValue(parent, latencyTrackersKey, &LatencyTrackers{
MutatingWebhookTracker: newSumLatencyTracker(c),
ValidatingWebhookTracker: newMaxLatencyTracker(c),
StorageTracker: newSumLatencyTracker(c),
TransformTracker: newSumLatencyTracker(c),
SerializationTracker: newSumLatencyTracker(c),
ResponseWriteTracker: newSumLatencyTracker(c),
})
}
// LatencyTrackersFrom returns the associated LatencyTrackers instance
// from the specified context.
func LatencyTrackersFrom(ctx context.Context) (*LatencyTrackers, bool) {
wd, ok := ctx.Value(latencyTrackersKey).(*LatencyTrackers)
return wd, ok && wd != nil
}
// TrackTransformResponseObjectLatency is used to track latency incurred
// inside the function that takes an object returned from the underlying
// storage layer (etcd) and performs any necessary transformations
// of the response object. This does not include the latency incurred in
// serialization (json or protobuf) of the response object or writing of
// it to the http ResponseWriter object.
// When called multiple times, the latency incurred inside the
// transform func each time will be summed up.
func TrackTransformResponseObjectLatency(ctx context.Context, transform func()) {
if tracker, ok := LatencyTrackersFrom(ctx); ok {
tracker.TransformTracker.Track(transform)
return
}
transform()
}
// TrackStorageLatency is used to track latency incurred
// inside the underlying storage layer.
// When called multiple times, the latency provided will be summed up.
func TrackStorageLatency(ctx context.Context, d time.Duration) {
if tracker, ok := LatencyTrackersFrom(ctx); ok {
tracker.StorageTracker.TrackDuration(d)
}
}
// TrackSerializeResponseObjectLatency is used to track latency incurred in
// serialization (json or protobuf) of the response object.
// When called multiple times, the latency provided will be summed up.
func TrackSerializeResponseObjectLatency(ctx context.Context, f func()) {
if tracker, ok := LatencyTrackersFrom(ctx); ok {
tracker.SerializationTracker.Track(f)
return
}
f()
}
// TrackResponseWriteLatency is used to track latency incurred in writing
// the serialized raw bytes to the http ResponseWriter object (via the
// Write method) associated with the request.
// When called multiple times, the latency provided will be summed up.
func TrackResponseWriteLatency(ctx context.Context, d time.Duration) {
if tracker, ok := LatencyTrackersFrom(ctx); ok {
tracker.ResponseWriteTracker.TrackDuration(d)
}
}
// AuditAnnotationsFromLatencyTrackers will inspect each latency tracker
// associated with the request context and return a set of audit
// annotations that can be added to the API audit entry.
func AuditAnnotationsFromLatencyTrackers(ctx context.Context) map[string]string {
const (
transformLatencyKey = "apiserver.latency.k8s.io/transform-response-object"
storageLatencyKey = "apiserver.latency.k8s.io/etcd"
serializationLatencyKey = "apiserver.latency.k8s.io/serialize-response-object"
responseWriteLatencyKey = "apiserver.latency.k8s.io/response-write"
mutatingWebhookLatencyKey = "apiserver.latency.k8s.io/mutating-webhook"
validatingWebhookLatencyKey = "apiserver.latency.k8s.io/validating-webhook"
)
tracker, ok := LatencyTrackersFrom(ctx)
if !ok {
return nil
}
annotations := map[string]string{}
if latency := tracker.TransformTracker.GetLatency(); latency != 0 {
annotations[transformLatencyKey] = latency.String()
}
if latency := tracker.StorageTracker.GetLatency(); latency != 0 {
annotations[storageLatencyKey] = latency.String()
}
if latency := tracker.SerializationTracker.GetLatency(); latency != 0 {
annotations[serializationLatencyKey] = latency.String()
}
if latency := tracker.ResponseWriteTracker.GetLatency(); latency != 0 {
annotations[responseWriteLatencyKey] = latency.String()
}
if latency := tracker.MutatingWebhookTracker.GetLatency(); latency != 0 {
annotations[mutatingWebhookLatencyKey] = latency.String()
}
if latency := tracker.ValidatingWebhookTracker.GetLatency(); latency != 0 {
annotations[validatingWebhookLatencyKey] = latency.String()
}
return annotations
}

View File

@@ -0,0 +1,54 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package responsewriter
import (
"bufio"
"net"
"net/http"
)
var _ http.ResponseWriter = &FakeResponseWriter{}
// FakeResponseWriter implements http.ResponseWriter,
// it is used for testing purpose only
type FakeResponseWriter struct{}
func (fw *FakeResponseWriter) Header() http.Header { return http.Header{} }
func (fw *FakeResponseWriter) WriteHeader(code int) {}
func (fw *FakeResponseWriter) Write(bs []byte) (int, error) { return len(bs), nil }
// For HTTP2 an http.ResponseWriter object implements
// http.Flusher and http.CloseNotifier.
// It is used for testing purpose only
type FakeResponseWriterFlusherCloseNotifier struct {
*FakeResponseWriter
}
func (fw *FakeResponseWriterFlusherCloseNotifier) Flush() {}
func (fw *FakeResponseWriterFlusherCloseNotifier) CloseNotify() <-chan bool { return nil }
// For HTTP/1.x an http.ResponseWriter object implements
// http.Flusher, http.CloseNotifier and http.Hijacker.
// It is used for testing purpose only
type FakeResponseWriterFlusherCloseNotifierHijacker struct {
*FakeResponseWriterFlusherCloseNotifier
}
func (fw *FakeResponseWriterFlusherCloseNotifierHijacker) Hijack() (net.Conn, *bufio.ReadWriter, error) {
return nil, nil, nil
}

View File

@@ -0,0 +1,180 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package responsewriter
import (
"bufio"
"net"
"net/http"
)
// UserProvidedDecorator represensts a user (client that uses this package)
// provided decorator that wraps an inner http.ResponseWriter object.
// The user-provided decorator object must return the inner (decorated)
// http.ResponseWriter object via the Unwrap function.
type UserProvidedDecorator interface {
http.ResponseWriter
// Unwrap returns the inner http.ResponseWriter object associated
// with the user-provided decorator.
Unwrap() http.ResponseWriter
}
// WrapForHTTP1Or2 accepts a user-provided decorator of an "inner" http.responseWriter
// object and potentially wraps the user-provided decorator with a new http.ResponseWriter
// object that implements http.CloseNotifier, http.Flusher, and/or http.Hijacker by
// delegating to the user-provided decorator (if it implements the relevant method) or
// the inner http.ResponseWriter (otherwise), so that the returned http.ResponseWriter
// object implements the same subset of those interfaces as the inner http.ResponseWriter.
//
// This function handles the following three casses.
// - The inner ResponseWriter implements `http.CloseNotifier`, `http.Flusher`,
// and `http.Hijacker` (an HTTP/1.1 sever provides such a ResponseWriter).
// - The inner ResponseWriter implements `http.CloseNotifier` and `http.Flusher`
// but not `http.Hijacker` (an HTTP/2 server provides such a ResponseWriter).
// - All the other cases collapse to this one, in which the given ResponseWriter is returned.
//
// There are three applicable terms:
// - "outer": this is the ResponseWriter object returned by the WrapForHTTP1Or2 function.
// - "user-provided decorator" or "middle": this is the user-provided decorator
// that decorates an inner ResponseWriter object. A user-provided decorator
// implements the UserProvidedDecorator interface. A user-provided decorator
// may or may not implement http.CloseNotifier, http.Flusher or http.Hijacker.
// - "inner": the ResponseWriter that the user-provided decorator extends.
func WrapForHTTP1Or2(decorator UserProvidedDecorator) http.ResponseWriter {
// from go net/http documentation:
// The default HTTP/1.x and HTTP/2 ResponseWriter implementations support Flusher
// Handlers should always test for this ability at runtime.
//
// The Hijacker interface is implemented by ResponseWriters that allow an HTTP handler
// to take over the connection.
// The default ResponseWriter for HTTP/1.x connections supports Hijacker, but HTTP/2 connections
// intentionally do not. ResponseWriter wrappers may also not support Hijacker.
// Handlers should always test for this ability at runtime
//
// The CloseNotifier interface is implemented by ResponseWriters which allow detecting
// when the underlying connection has gone away.
// Deprecated: the CloseNotifier interface predates Go's context package.
// New code should use Request.Context instead.
inner := decorator.Unwrap()
if innerNotifierFlusher, ok := inner.(CloseNotifierFlusher); ok {
// for HTTP/2 request, the default ResponseWriter object (http2responseWriter)
// implements Flusher and CloseNotifier.
outerHTTP2 := outerWithCloseNotifyAndFlush{
UserProvidedDecorator: decorator,
InnerCloseNotifierFlusher: innerNotifierFlusher,
}
if innerHijacker, hijackable := inner.(http.Hijacker); hijackable {
// for HTTP/1.x request the default implementation of ResponseWriter
// also implement CloseNotifier, Flusher and Hijacker
return &outerWithCloseNotifyFlushAndHijack{
outerWithCloseNotifyAndFlush: outerHTTP2,
InnerHijacker: innerHijacker,
}
}
return outerHTTP2
}
// we should never be here for either http/1.x or http2 request
return decorator
}
// CloseNotifierFlusher is a combination of http.CloseNotifier and http.Flusher
// This applies to both http/1.x and http2 requests.
type CloseNotifierFlusher interface {
http.CloseNotifier
http.Flusher
}
// GetOriginal goes through the chain of wrapped http.ResponseWriter objects
// and returns the original http.ResponseWriter object provided to the first
// request handler in the filter chain.
func GetOriginal(w http.ResponseWriter) http.ResponseWriter {
decorator, ok := w.(UserProvidedDecorator)
if !ok {
return w
}
inner := decorator.Unwrap()
if inner == w {
// infinite cycle here, we should never be here though.
panic("http.ResponseWriter decorator chain has a cycle")
}
return GetOriginal(inner)
}
//nolint:staticcheck // SA1019
var _ http.CloseNotifier = outerWithCloseNotifyAndFlush{}
var _ http.Flusher = outerWithCloseNotifyAndFlush{}
var _ http.ResponseWriter = outerWithCloseNotifyAndFlush{}
var _ UserProvidedDecorator = outerWithCloseNotifyAndFlush{}
// outerWithCloseNotifyAndFlush is the outer object that extends the
// user provied decorator with http.CloseNotifier and http.Flusher only.
type outerWithCloseNotifyAndFlush struct {
// UserProvidedDecorator is the user-provided object, it decorates
// an inner ResponseWriter object.
UserProvidedDecorator
// http.CloseNotifier and http.Flusher for the inner object
InnerCloseNotifierFlusher CloseNotifierFlusher
}
func (wr outerWithCloseNotifyAndFlush) CloseNotify() <-chan bool {
if notifier, ok := wr.UserProvidedDecorator.(http.CloseNotifier); ok {
return notifier.CloseNotify()
}
return wr.InnerCloseNotifierFlusher.CloseNotify()
}
func (wr outerWithCloseNotifyAndFlush) Flush() {
if flusher, ok := wr.UserProvidedDecorator.(http.Flusher); ok {
flusher.Flush()
return
}
wr.InnerCloseNotifierFlusher.Flush()
}
//lint:file-ignore SA1019 Keep supporting deprecated http.CloseNotifier
var _ http.CloseNotifier = outerWithCloseNotifyFlushAndHijack{}
var _ http.Flusher = outerWithCloseNotifyFlushAndHijack{}
var _ http.Hijacker = outerWithCloseNotifyFlushAndHijack{}
var _ http.ResponseWriter = outerWithCloseNotifyFlushAndHijack{}
var _ UserProvidedDecorator = outerWithCloseNotifyFlushAndHijack{}
// outerWithCloseNotifyFlushAndHijack is the outer object that extends the
// user-provided decorator with http.CloseNotifier, http.Flusher and http.Hijacker.
// This applies to http/1.x requests only.
type outerWithCloseNotifyFlushAndHijack struct {
outerWithCloseNotifyAndFlush
// http.Hijacker for the inner object
InnerHijacker http.Hijacker
}
func (wr outerWithCloseNotifyFlushAndHijack) Hijack() (net.Conn, *bufio.ReadWriter, error) {
if hijacker, ok := wr.UserProvidedDecorator.(http.Hijacker); ok {
return hijacker.Hijack()
}
return wr.InnerHijacker.Hijack()
}

View File

@@ -17,7 +17,7 @@ limitations under the License.
package warning
import (
restful "github.com/emicklei/go-restful"
restful "github.com/emicklei/go-restful/v3"
"k8s.io/apiserver/pkg/warning"
)