feat: kubesphere 4.0 (#6115)

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

---------

Signed-off-by: ci-bot <ci-bot@kubesphere.io>
Co-authored-by: ks-ci-bot <ks-ci-bot@example.com>
Co-authored-by: joyceliu <joyceliu@yunify.com>

kube/pkg/openapi/apiservice.go

@@ -0,0 +1,193 @@
+/*
+ * Please refer to the LICENSE file in the root directory of the project.
+ * https://github.com/kubesphere/kubesphere/blob/master/LICENSE
+ */
+
+package openapi
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/httpstream"
+	utilnet "k8s.io/apimachinery/pkg/util/net"
+	"k8s.io/apimachinery/pkg/util/proxy"
+	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
+	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/klog/v2"
+
+	extensionsv1alpha1 "kubesphere.io/api/extensions/v1alpha1"
+)
+
+type certKeyFunc func() ([]byte, []byte)
+
+const (
+	aggregatedDiscoveryTimeout = 5 * time.Second
+)
+
+type ApiService interface {
+	Name() string
+	ResolveEndpoint() (*url.URL, error)
+	UpdateAPIService() error
+	ServeHTTP(w http.ResponseWriter, req *http.Request)
+}
+
+type defaultApiService struct {
+	apiService                 *extensionsv1alpha1.APIService
+	proxyTransport             *http.Transport
+	restConfig                 *restclient.Config
+	proxyRoundTripper          http.RoundTripper
+	proxyCurrentCertKeyContent certKeyFunc
+}
+
+func NewApiService(apiService *extensionsv1alpha1.APIService) ApiService {
+	return &defaultApiService{
+		apiService:                 apiService,
+		proxyCurrentCertKeyContent: func() (bytes []byte, bytes2 []byte) { return nil, nil },
+	}
+}
+
+func (d *defaultApiService) Name() string {
+	return d.apiService.Name
+}
+
+func (d *defaultApiService) ResolveEndpoint() (*url.URL, error) {
+
+	if d.apiService.Spec.Service != nil &&
+		d.apiService.Spec.Service.Name != "" &&
+		d.apiService.Spec.Service.Namespace != "" &&
+		*d.apiService.Spec.Service.Port != 0 {
+		return &url.URL{Scheme: "https", Host: fmt.Sprintf("%s.%s.svc:%d",
+			d.apiService.Spec.Service.Name, d.apiService.Spec.Service.Namespace, d.apiService.Spec.Service.Port)}, nil
+	}
+	if d.apiService.Spec.URL != nil && *d.apiService.Spec.URL != "" {
+		u, err := url.Parse(*d.apiService.Spec.URL)
+		if err != nil {
+			return nil, err
+		}
+		if d.apiService.Spec.InsecureSkipVerify {
+			u.Scheme = "http"
+		}
+		return u, nil
+	}
+	return nil, fmt.Errorf("cannot resolve an apiservice %s", d.Name())
+}
+
+func (d *defaultApiService) UpdateAPIService() error {
+	proxyClientCert, proxyClientKey := d.proxyCurrentCertKeyContent()
+
+	tlsConfig := restclient.TLSClientConfig{
+		Insecure: d.apiService.Spec.InsecureSkipVerify,
+	}
+
+	if !d.apiService.Spec.InsecureSkipVerify && len(d.apiService.Spec.CABundle) > 0 {
+		caData, err := base64.StdEncoding.DecodeString(string(d.apiService.Spec.CABundle))
+		if err != nil {
+			klog.Warning(err.Error())
+			return err
+		}
+		tlsConfig.ServerName = d.apiService.Spec.Service.Name + "." + d.apiService.Spec.Service.Namespace + ".svc"
+		tlsConfig.CertData = proxyClientCert
+		tlsConfig.KeyData = proxyClientKey
+		tlsConfig.CAData = caData
+	}
+	d.restConfig = &restclient.Config{
+		TLSClientConfig: tlsConfig,
+	}
+
+	if d.proxyTransport != nil && d.proxyTransport.DialContext != nil {
+		d.restConfig.Dial = d.proxyTransport.DialContext
+	}
+	proxyRoundTripper, err := restclient.TransportFor(d.restConfig)
+	if err != nil {
+		klog.Warning(err.Error())
+		return err
+	}
+	d.proxyRoundTripper = proxyRoundTripper
+	return nil
+}
+
+func (d *defaultApiService) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	//user, ok := genericapirequest.UserFrom(req.Context())
+	//if !ok {
+	//	proxyError(w, req, "missing user", http.StatusInternalServerError)
+	//	return
+	//}
+
+	// write a new location based on the existing request pointed at the target service
+	location, err := d.ResolveEndpoint()
+	if err != nil {
+		klog.Errorf("error resolving %s: %v", d.Name(), err)
+		proxyError(w, req, "service unavailable", http.StatusServiceUnavailable)
+	}
+	location.Path = req.URL.Path
+	location.RawQuery = req.URL.Query().Encode()
+
+	newReq, cancelFn := newRequestForProxy(location, req)
+	defer cancelFn()
+
+	if d.proxyRoundTripper == nil {
+		proxyError(w, req, "", http.StatusNotFound)
+		return
+	}
+
+	proxyRoundTripper := d.proxyRoundTripper
+	upgrade := httpstream.IsUpgradeRequest(req)
+
+	//proxyRoundTripper = transport.NewAuthProxyRoundTripper(user.GetName(), user.GetGroups(), user.GetExtra(), proxyRoundTripper)
+
+	//if upgrade {
+	//transport.SetAuthProxyHeaders(newReq, user.GetName(), user.GetGroups(), user.GetExtra())
+	//}
+
+	handler := proxy.NewUpgradeAwareHandler(location, proxyRoundTripper, true, upgrade, &responder{w: w})
+	handler.ServeHTTP(w, newReq)
+}
+
+type responder struct {
+	w http.ResponseWriter
+}
+
+func (r *responder) Object(statusCode int, obj runtime.Object) {
+	responsewriters.WriteRawJSON(statusCode, obj, r.w)
+}
+
+func (r *responder) Error(_ http.ResponseWriter, _ *http.Request, err error) {
+	http.Error(r.w, err.Error(), http.StatusServiceUnavailable)
+}
+
+func proxyError(w http.ResponseWriter, req *http.Request, error string, code int) {
+	http.Error(w, error, code)
+}
+
+// newRequestForProxy returns a shallow copy of the original request with a context that may include a timeout for discovery requests
+func newRequestForProxy(location *url.URL, req *http.Request) (*http.Request, context.CancelFunc) {
+	newCtx := req.Context()
+	cancelFn := func() {}
+
+	if requestInfo, ok := genericapirequest.RequestInfoFrom(req.Context()); ok {
+		// trim leading and trailing slashes. Then "/apis/group/version" requests are for discovery, so if we have exactly three
+		// segments that we are going to proxy, we have a discovery request.
+		if !requestInfo.IsResourceRequest && len(strings.Split(strings.Trim(requestInfo.Path, "/"), "/")) == 3 {
+			// discovery requests are used by kubectl and others to determine which resources a server has.  This is a cheap call that
+			// should be fast for every aggregated apiserver.  Latency for aggregation is expected to be low (as for all extensions)
+			// so forcing a short timeout here helps responsiveness of all clients.
+			newCtx, cancelFn = context.WithTimeout(newCtx, aggregatedDiscoveryTimeout)
+		}
+	}
+
+	// WithContext creates a shallow clone of the request with the same context.
+	newReq := req.WithContext(newCtx)
+	newReq.Header = utilnet.CloneHeader(req.Header)
+	newReq.URL = location
+	newReq.Host = location.Host
+
+	return newReq, cancelFn
+}

kube/pkg/openapi/downloader.go

@@ -0,0 +1,129 @@
+/*
+ * Please refer to the LICENSE file in the root directory of the project.
+ * https://github.com/kubesphere/kubesphere/blob/master/LICENSE
+ */
+
+package openapi
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+)
+
+const (
+	specDownloadTimeout = time.Minute
+)
+
+type CacheableDownloader interface {
+	Name() string
+	GetV2() ([]byte, error)
+	GetV3() ([]byte, error)
+	UpdateDownloader(apiService ApiService) error
+}
+
+type Downloader struct{}
+
+func NewDownloader() *Downloader {
+	return &Downloader{}
+}
+
+func (s *Downloader) Download(handler http.Handler, req *http.Request) (data []byte, err error) {
+	handler = http.TimeoutHandler(handler, specDownloadTimeout, "request timed out")
+
+	writer := newInMemoryResponseWriter()
+	handler.ServeHTTP(writer, req)
+
+	switch writer.respCode {
+	case http.StatusNotModified:
+		return nil, nil
+	case http.StatusNotFound:
+		return nil, ErrAPIServiceNotFound
+	case http.StatusOK:
+		return writer.data, nil
+	default:
+		return nil, fmt.Errorf("failed to retrieve openAPI spec, http error: %s", writer.String())
+	}
+}
+
+type inMemoryResponseWriter struct {
+	writeHeaderCalled bool
+	header            http.Header
+	respCode          int
+	data              []byte
+}
+
+func newInMemoryResponseWriter() *inMemoryResponseWriter {
+	return &inMemoryResponseWriter{header: http.Header{}}
+}
+
+func (r *inMemoryResponseWriter) Header() http.Header {
+	return r.header
+}
+
+func (r *inMemoryResponseWriter) WriteHeader(code int) {
+	r.writeHeaderCalled = true
+	r.respCode = code
+}
+
+func (r *inMemoryResponseWriter) Write(in []byte) (int, error) {
+	if !r.writeHeaderCalled {
+		r.WriteHeader(http.StatusOK)
+	}
+	r.data = append(r.data, in...)
+	return len(in), nil
+}
+
+func (r *inMemoryResponseWriter) String() string {
+	s := fmt.Sprintf("ResponseCode: %d", r.respCode)
+	if r.data != nil {
+		s += fmt.Sprintf(", Body: %s", string(r.data))
+	}
+	if r.header != nil {
+		s += fmt.Sprintf(", Header: %s", r.header)
+	}
+	return s
+}
+
+type cacheableDownloader struct {
+	apiService ApiService
+	downloader *Downloader
+}
+
+func NewCacheableDownloader(apiService ApiService, downloader *Downloader) (CacheableDownloader, error) {
+	c := &cacheableDownloader{
+		apiService: apiService,
+		downloader: downloader,
+	}
+	if err := c.apiService.UpdateAPIService(); err != nil {
+		return nil, err
+	}
+	return c, nil
+}
+
+func (c *cacheableDownloader) Name() string {
+	return c.apiService.Name()
+}
+
+func (c *cacheableDownloader) Get(url string) ([]byte, error) {
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Add("Accept", "application/json")
+
+	return c.downloader.Download(c.apiService, req)
+}
+
+func (c *cacheableDownloader) GetV2() ([]byte, error) {
+	return c.Get("/openapi/v2")
+}
+
+func (c *cacheableDownloader) GetV3() ([]byte, error) {
+	return c.Get("/openapi/v3")
+}
+
+func (c *cacheableDownloader) UpdateDownloader(apiService ApiService) error {
+	c.apiService = apiService
+	return c.apiService.UpdateAPIService()
+}

kube/pkg/openapi/error.go

@@ -0,0 +1,14 @@
+/*
+ * Please refer to the LICENSE file in the root directory of the project.
+ * https://github.com/kubesphere/kubesphere/blob/master/LICENSE
+ */
+
+package openapi
+
+import (
+	"errors"
+)
+
+var (
+	ErrAPIServiceNotFound = errors.New("resource not found")
+)

kube/pkg/openapi/merge/default_pruning.go

@@ -0,0 +1,208 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package merge
+
+import "github.com/go-openapi/spec"
+
+// PruneDefaults remove all the defaults recursively from all the
+// schemas in the definitions, and does not modify the definitions in
+// place.
+func PruneDefaults(definitions spec.Definitions) spec.Definitions {
+	definitionsCloned := false
+	for k, v := range definitions {
+		if s := PruneDefaultsSchema(&v); s != &v {
+			if !definitionsCloned {
+				definitionsCloned = true
+				orig := definitions
+				definitions = make(spec.Definitions, len(orig))
+				for k2, v2 := range orig {
+					definitions[k2] = v2
+				}
+			}
+			definitions[k] = *s
+		}
+	}
+	return definitions
+}
+
+// PruneDefaultsSchema remove all the defaults recursively from the
+// schema in place.
+func PruneDefaultsSchema(schema *spec.Schema) *spec.Schema {
+	if schema == nil {
+		return nil
+	}
+
+	orig := schema
+	clone := func() {
+		if orig == schema {
+			schema = &spec.Schema{}
+			*schema = *orig
+		}
+	}
+
+	if schema.Default != nil {
+		clone()
+		schema.Default = nil
+	}
+
+	definitionsCloned := false
+	for k, v := range schema.Definitions {
+		if s := PruneDefaultsSchema(&v); s != &v {
+			if !definitionsCloned {
+				definitionsCloned = true
+				clone()
+				schema.Definitions = make(spec.Definitions, len(orig.Definitions))
+				for k2, v2 := range orig.Definitions {
+					schema.Definitions[k2] = v2
+				}
+			}
+			schema.Definitions[k] = *s
+		}
+	}
+
+	propertiesCloned := false
+	for k, v := range schema.Properties {
+		if s := PruneDefaultsSchema(&v); s != &v {
+			if !propertiesCloned {
+				propertiesCloned = true
+				clone()
+				schema.Properties = make(map[string]spec.Schema, len(orig.Properties))
+				for k2, v2 := range orig.Properties {
+					schema.Properties[k2] = v2
+				}
+			}
+			schema.Properties[k] = *s
+		}
+	}
+
+	patternPropertiesCloned := false
+	for k, v := range schema.PatternProperties {
+		if s := PruneDefaultsSchema(&v); s != &v {
+			if !patternPropertiesCloned {
+				patternPropertiesCloned = true
+				clone()
+				schema.PatternProperties = make(map[string]spec.Schema, len(orig.PatternProperties))
+				for k2, v2 := range orig.PatternProperties {
+					schema.PatternProperties[k2] = v2
+				}
+			}
+			schema.PatternProperties[k] = *s
+		}
+	}
+
+	dependenciesCloned := false
+	for k, v := range schema.Dependencies {
+		if s := PruneDefaultsSchema(v.Schema); s != v.Schema {
+			if !dependenciesCloned {
+				dependenciesCloned = true
+				clone()
+				schema.Dependencies = make(spec.Dependencies, len(orig.Dependencies))
+				for k2, v2 := range orig.Dependencies {
+					schema.Dependencies[k2] = v2
+				}
+			}
+			v.Schema = s
+			schema.Dependencies[k] = v
+		}
+	}
+
+	allOfCloned := false
+	for i := range schema.AllOf {
+		if s := PruneDefaultsSchema(&schema.AllOf[i]); s != &schema.AllOf[i] {
+			if !allOfCloned {
+				allOfCloned = true
+				clone()
+				schema.AllOf = make([]spec.Schema, len(orig.AllOf))
+				copy(schema.AllOf, orig.AllOf)
+			}
+			schema.AllOf[i] = *s
+		}
+	}
+
+	anyOfCloned := false
+	for i := range schema.AnyOf {
+		if s := PruneDefaultsSchema(&schema.AnyOf[i]); s != &schema.AnyOf[i] {
+			if !anyOfCloned {
+				anyOfCloned = true
+				clone()
+				schema.AnyOf = make([]spec.Schema, len(orig.AnyOf))
+				copy(schema.AnyOf, orig.AnyOf)
+			}
+			schema.AnyOf[i] = *s
+		}
+	}
+
+	oneOfCloned := false
+	for i := range schema.OneOf {
+		if s := PruneDefaultsSchema(&schema.OneOf[i]); s != &schema.OneOf[i] {
+			if !oneOfCloned {
+				oneOfCloned = true
+				clone()
+				schema.OneOf = make([]spec.Schema, len(orig.OneOf))
+				copy(schema.OneOf, orig.OneOf)
+			}
+			schema.OneOf[i] = *s
+		}
+	}
+
+	if schema.Not != nil {
+		if s := PruneDefaultsSchema(schema.Not); s != schema.Not {
+			clone()
+			schema.Not = s
+		}
+	}
+
+	if schema.AdditionalProperties != nil && schema.AdditionalProperties.Schema != nil {
+		if s := PruneDefaultsSchema(schema.AdditionalProperties.Schema); s != schema.AdditionalProperties.Schema {
+			clone()
+			schema.AdditionalProperties = &spec.SchemaOrBool{Schema: s, Allows: schema.AdditionalProperties.Allows}
+		}
+	}
+
+	if schema.AdditionalItems != nil && schema.AdditionalItems.Schema != nil {
+		if s := PruneDefaultsSchema(schema.AdditionalItems.Schema); s != schema.AdditionalItems.Schema {
+			clone()
+			schema.AdditionalItems = &spec.SchemaOrBool{Schema: s, Allows: schema.AdditionalItems.Allows}
+		}
+	}
+
+	if schema.Items != nil {
+		if schema.Items.Schema != nil {
+			if s := PruneDefaultsSchema(schema.Items.Schema); s != schema.Items.Schema {
+				clone()
+				schema.Items = &spec.SchemaOrArray{Schema: s}
+			}
+		} else {
+			itemsCloned := false
+			for i := range schema.Items.Schemas {
+				if s := PruneDefaultsSchema(&schema.Items.Schemas[i]); s != &schema.Items.Schemas[i] {
+					if !itemsCloned {
+						clone()
+						schema.Items = &spec.SchemaOrArray{
+							Schemas: make([]spec.Schema, len(orig.Items.Schemas)),
+						}
+						itemsCloned = true
+						copy(schema.Items.Schemas, orig.Items.Schemas)
+					}
+					schema.Items.Schemas[i] = *s
+				}
+			}
+		}
+	}
+
+	return schema
+}

kube/pkg/openapi/merge/merge.go

@@ -0,0 +1,920 @@
+package merge
+
+import (
+	"fmt"
+	"reflect"
+	"sort"
+	"strings"
+
+	"github.com/go-openapi/spec"
+	"k8s.io/kube-openapi/pkg/util"
+)
+
+const gvkKey = "x-kubesphere-group-version-kind"
+
+// usedDefinitionForSpec returns a map with all used definitions in the provided spec as keys and true as values.
+func usedDefinitionForSpec(root *spec.Swagger) map[string]bool {
+	usedDefinitions := map[string]bool{}
+	walkOnAllReferences(func(ref *spec.Ref) {
+		if refStr := ref.String(); refStr != "" && strings.HasPrefix(refStr, definitionPrefix) {
+			usedDefinitions[refStr[len(definitionPrefix):]] = true
+		}
+	}, root)
+	return usedDefinitions
+}
+
+// FilterSpecByPathsWithoutSideEffects removes unnecessary paths and definitions used by those paths.
+// i.e. if a Path removed by this function, all definitions used by it and not used
+// anywhere else will also be removed.
+// It does not modify the input, but the output shares data structures with the input.
+func FilterSpecByPathsWithoutSideEffects(sp *spec.Swagger, keepPathPrefixes []string) *spec.Swagger {
+	if sp.Paths == nil {
+		return sp
+	}
+
+	// Walk all references to find all used definitions. This function
+	// want to only deal with unused definitions resulted from filtering paths.
+	// Thus a definition will be removed only if it has been used before but
+	// it is unused because of a path prune.
+	initialUsedDefinitions := usedDefinitionForSpec(sp)
+
+	// First remove unwanted paths
+	prefixes := util.NewTrie(keepPathPrefixes)
+	ret := *sp
+	ret.Paths = &spec.Paths{
+		VendorExtensible: sp.Paths.VendorExtensible,
+		Paths:            map[string]spec.PathItem{},
+	}
+	for path, pathItem := range sp.Paths.Paths {
+		if !prefixes.HasPrefix(path) {
+			continue
+		}
+		ret.Paths.Paths[path] = pathItem
+	}
+
+	// Walk all references to find all definition references.
+	usedDefinitions := usedDefinitionForSpec(&ret)
+
+	// Remove unused definitions
+	ret.Definitions = spec.Definitions{}
+	for k, v := range sp.Definitions {
+		if usedDefinitions[k] || !initialUsedDefinitions[k] {
+			ret.Definitions[k] = v
+		}
+	}
+
+	return &ret
+}
+
+// renameDefinitions renames definition references, without mutating the input.
+// The output might share data structures with the input.
+func renameDefinitions(s *spec.Swagger, renames map[string]string) *spec.Swagger {
+	refRenames := make(map[string]string, len(renames))
+	foundOne := false
+	for k, v := range renames {
+		refRenames[definitionPrefix+k] = definitionPrefix + v
+		if _, ok := s.Definitions[k]; ok {
+			foundOne = true
+		}
+	}
+
+	if !foundOne {
+		return s
+	}
+
+	ret := &spec.Swagger{}
+	*ret = *s
+
+	ret = ReplaceReferences(func(ref *spec.Ref) *spec.Ref {
+		refName := ref.String()
+		if newRef, found := refRenames[refName]; found {
+			ret := spec.MustCreateRef(newRef)
+			return &ret
+		}
+		return ref
+	}, ret)
+
+	renamedDefinitions := make(spec.Definitions, len(ret.Definitions))
+	for k, v := range ret.Definitions {
+		if newRef, found := renames[k]; found {
+			k = newRef
+		}
+		renamedDefinitions[k] = v
+	}
+	ret.Definitions = renamedDefinitions
+
+	return ret
+}
+
+// renameParameters renames parameter references, without mutating the input.
+// The output might share data structures with the input.
+func renameParameters(s *spec.Swagger, renames map[string]string) *spec.Swagger {
+	refRenames := make(map[string]string, len(renames))
+	foundOne := false
+	for k, v := range renames {
+		refRenames[parameterPrefix+k] = parameterPrefix + v
+		if _, ok := s.Parameters[k]; ok {
+			foundOne = true
+		}
+	}
+
+	if !foundOne {
+		return s
+	}
+
+	ret := &spec.Swagger{}
+	*ret = *s
+
+	ret = ReplaceReferences(func(ref *spec.Ref) *spec.Ref {
+		refName := ref.String()
+		if newRef, found := refRenames[refName]; found {
+			ret := spec.MustCreateRef(newRef)
+			return &ret
+		}
+		return ref
+	}, ret)
+
+	renamed := make(map[string]spec.Parameter, len(ret.Parameters))
+	for k, v := range ret.Parameters {
+		if newRef, found := renames[k]; found {
+			k = newRef
+		}
+		renamed[k] = v
+	}
+	ret.Parameters = renamed
+
+	return ret
+}
+
+// MergeSpecsIgnorePathConflictRenamingDefinitionsAndParameters is the same as
+// MergeSpecs except it will ignore any path conflicts by keeping the paths of
+// destination. It will rename definition and parameter conflicts.
+func MergeSpecsIgnorePathConflictRenamingDefinitionsAndParameters(dest, source *spec.Swagger) error {
+	return mergeSpecs(dest, source, true, true, true)
+}
+
+// mergeSpecs merges source into dest while resolving conflicts.
+// The source is not mutated.
+func mergeSpecs(dest, source *spec.Swagger, renameModelConflicts, renameParameterConflicts, ignorePathConflicts bool) (err error) {
+	// Paths may be empty, due to [ACL constraints](http://goo.gl/8us55a#securityFiltering).
+	if source.Paths == nil {
+		// When a source spec does not have any path, that means none of the definitions
+		// are used thus we should not do anything
+		return nil
+	}
+	if dest.Paths == nil {
+		dest.Paths = &spec.Paths{}
+	}
+	if ignorePathConflicts {
+		keepPaths := []string{}
+		hasConflictingPath := false
+		for k := range source.Paths.Paths {
+			if _, found := dest.Paths.Paths[k]; !found {
+				keepPaths = append(keepPaths, k)
+			} else {
+				hasConflictingPath = true
+			}
+		}
+		if len(keepPaths) == 0 {
+			// There is nothing to merge. All paths are conflicting.
+			return nil
+		}
+		if hasConflictingPath {
+			source = FilterSpecByPathsWithoutSideEffects(source, keepPaths)
+		}
+	}
+
+	// Check for model conflicts and rename to make definitions conflict-free (modulo different GVKs)
+	usedNames := map[string]bool{}
+	for k := range dest.Definitions {
+		usedNames[k] = true
+	}
+	renames := map[string]string{}
+DEFINITIONLOOP:
+	for k, v := range source.Definitions {
+		existing, found := dest.Definitions[k]
+		if !found || deepEqualDefinitionsModuloGVKs(&existing, &v) {
+			// skip for now, we copy them after the rename loop
+			continue
+		}
+
+		if !renameModelConflicts {
+			return fmt.Errorf("model name conflict in merging OpenAPI spec: %s", k)
+		}
+
+		// Reuse previously renamed model if one exists
+		var newName string
+		i := 1
+		for found {
+			i++
+			newName = fmt.Sprintf("%s_v%d", k, i)
+			existing, found = dest.Definitions[newName]
+			if found && deepEqualDefinitionsModuloGVKs(&existing, &v) {
+				renames[k] = newName
+				continue DEFINITIONLOOP
+			}
+		}
+
+		_, foundInSource := source.Definitions[newName]
+		for usedNames[newName] || foundInSource {
+			i++
+			newName = fmt.Sprintf("%s_v%d", k, i)
+			_, foundInSource = source.Definitions[newName]
+		}
+		renames[k] = newName
+		usedNames[newName] = true
+	}
+	source = renameDefinitions(source, renames)
+
+	// Check for parameter conflicts and rename to make parameters conflict-free
+	usedNames = map[string]bool{}
+	for k := range dest.Parameters {
+		usedNames[k] = true
+	}
+	renames = map[string]string{}
+PARAMETERLOOP:
+	for k, p := range source.Parameters {
+		existing, found := dest.Parameters[k]
+		if !found || reflect.DeepEqual(&existing, &p) {
+			// skip for now, we copy them after the rename loop
+			continue
+		}
+
+		if !renameParameterConflicts {
+			return fmt.Errorf("parameter name conflict in merging OpenAPI spec: %s", k)
+		}
+
+		// Reuse previously renamed parameter if one exists
+		var newName string
+		i := 1
+		for found {
+			i++
+			newName = fmt.Sprintf("%s_v%d", k, i)
+			existing, found = dest.Parameters[newName]
+			if found && reflect.DeepEqual(&existing, &p) {
+				renames[k] = newName
+				continue PARAMETERLOOP
+			}
+		}
+
+		_, foundInSource := source.Parameters[newName]
+		for usedNames[newName] || foundInSource {
+			i++
+			newName = fmt.Sprintf("%s_v%d", k, i)
+			_, foundInSource = source.Parameters[newName]
+		}
+		renames[k] = newName
+		usedNames[newName] = true
+	}
+	source = renameParameters(source, renames)
+
+	// Now without conflict (modulo different GVKs), copy definitions to dest
+	for k, v := range source.Definitions {
+		if existing, found := dest.Definitions[k]; !found {
+			if dest.Definitions == nil {
+				dest.Definitions = make(spec.Definitions, len(source.Definitions))
+			}
+			dest.Definitions[k] = v
+		} else if merged, changed, err := mergedGVKs(&existing, &v); err != nil {
+			return err
+		} else if changed {
+			existing.Extensions[gvkKey] = merged
+		}
+	}
+
+	// Now without conflict, copy parameters to dest
+	for k, v := range source.Parameters {
+		if _, found := dest.Parameters[k]; !found {
+			if dest.Parameters == nil {
+				dest.Parameters = make(map[string]spec.Parameter, len(source.Parameters))
+			}
+			dest.Parameters[k] = v
+		}
+	}
+
+	// Check for path conflicts
+	for k, v := range source.Paths.Paths {
+		if _, found := dest.Paths.Paths[k]; found {
+			return fmt.Errorf("unable to merge: duplicated path %s", k)
+		}
+		// PathItem may be empty, due to [ACL constraints](http://goo.gl/8us55a#securityFiltering).
+		if dest.Paths.Paths == nil {
+			dest.Paths.Paths = map[string]spec.PathItem{}
+		}
+		dest.Paths.Paths[k] = v
+	}
+
+	return nil
+}
+
+// deepEqualDefinitionsModuloGVKs compares s1 and s2, but ignores the x-kubernetes-group-version-kind extension.
+func deepEqualDefinitionsModuloGVKs(s1, s2 *spec.Schema) bool {
+	if s1 == nil {
+		return s2 == nil
+	} else if s2 == nil {
+		return false
+	}
+	if !reflect.DeepEqual(s1.Extensions, s2.Extensions) {
+		for k, v := range s1.Extensions {
+			if k == gvkKey {
+				continue
+			}
+			if !reflect.DeepEqual(v, s2.Extensions[k]) {
+				return false
+			}
+		}
+		len1 := len(s1.Extensions)
+		len2 := len(s2.Extensions)
+		if _, found := s1.Extensions[gvkKey]; found {
+			len1--
+		}
+		if _, found := s2.Extensions[gvkKey]; found {
+			len2--
+		}
+		if len1 != len2 {
+			return false
+		}
+
+		if s1.Extensions != nil {
+			shallowCopy := *s1
+			s1 = &shallowCopy
+			s1.Extensions = nil
+		}
+		if s2.Extensions != nil {
+			shallowCopy := *s2
+			s2 = &shallowCopy
+			s2.Extensions = nil
+		}
+	}
+
+	return reflect.DeepEqual(s1, s2)
+}
+
+// mergedGVKs merges the x-kubernetes-group-version-kind slices and returns the result, and whether
+// s1's x-kubernetes-group-version-kind slice was changed at all.
+func mergedGVKs(s1, s2 *spec.Schema) (interface{}, bool, error) {
+	gvk1, found1 := s1.Extensions[gvkKey]
+	gvk2, found2 := s2.Extensions[gvkKey]
+
+	if !found1 {
+		return gvk2, found2, nil
+	}
+	if !found2 {
+		return gvk1, false, nil
+	}
+
+	slice1, ok := gvk1.([]interface{})
+	if !ok {
+		return nil, false, fmt.Errorf("expected slice of GroupVersionKinds, got: %+v", slice1)
+	}
+	slice2, ok := gvk2.([]interface{})
+	if !ok {
+		return nil, false, fmt.Errorf("expected slice of GroupVersionKinds, got: %+v", slice2)
+	}
+
+	ret := make([]interface{}, len(slice1), len(slice1)+len(slice2))
+	keys := make([]string, 0, len(slice1)+len(slice2))
+	copy(ret, slice1)
+	seen := make(map[string]bool, len(slice1))
+	for _, x := range slice1 {
+		gvk, ok := x.(map[string]interface{})
+		if !ok {
+			return nil, false, fmt.Errorf(`expected {"group": <group>, "kind": <kind>, "version": <version>}, got: %#v`, x)
+		}
+		k := fmt.Sprintf("%s/%s.%s", gvk["group"], gvk["version"], gvk["kind"])
+		keys = append(keys, k)
+		seen[k] = true
+	}
+	changed := false
+	for _, x := range slice2 {
+		gvk, ok := x.(map[string]interface{})
+		if !ok {
+			return nil, false, fmt.Errorf(`expected {"group": <group>, "kind": <kind>, "version": <version>}, got: %#v`, x)
+		}
+		k := fmt.Sprintf("%s/%s.%s", gvk["group"], gvk["version"], gvk["kind"])
+		if seen[k] {
+			continue
+		}
+		ret = append(ret, x)
+		keys = append(keys, k)
+		changed = true
+	}
+
+	if changed {
+		sort.Sort(byKeys{ret, keys})
+	}
+
+	return ret, changed, nil
+}
+
+type byKeys struct {
+	values []interface{}
+	keys   []string
+}
+
+func (b byKeys) Len() int {
+	return len(b.values)
+}
+
+func (b byKeys) Less(i, j int) bool {
+	return b.keys[i] < b.keys[j]
+}
+
+func (b byKeys) Swap(i, j int) {
+	b.values[i], b.values[j] = b.values[j], b.values[i]
+	b.keys[i], b.keys[j] = b.keys[j], b.keys[i]
+}
+
+func ReplaceReferences(walkRef func(ref *spec.Ref) *spec.Ref, sp *spec.Swagger) *spec.Swagger {
+	walker := &Walker{RefCallback: walkRef, SchemaCallback: SchemaCallBackNoop}
+	return walker.WalkRoot(sp)
+}
+
+type Walker struct {
+	// SchemaCallback will be called on each schema, taking the original schema,
+	// and before any other callbacks of the Walker.
+	// If the schema needs to be mutated, DO NOT mutate it in-place,
+	// always create a copy, mutate, and return it.
+	SchemaCallback func(schema *spec.Schema) *spec.Schema
+
+	// RefCallback will be called on each ref.
+	// If the ref needs to be mutated, DO NOT mutate it in-place,
+	// always create a copy, mutate, and return it.
+	RefCallback func(ref *spec.Ref) *spec.Ref
+}
+
+type SchemaCallbackFunc func(schema *spec.Schema) *spec.Schema
+type RefCallbackFunc func(ref *spec.Ref) *spec.Ref
+
+var SchemaCallBackNoop SchemaCallbackFunc = func(schema *spec.Schema) *spec.Schema {
+	return schema
+}
+var RefCallbackNoop RefCallbackFunc = func(ref *spec.Ref) *spec.Ref {
+	return ref
+}
+
+func (w *Walker) WalkRoot(swagger *spec.Swagger) *spec.Swagger {
+	if swagger == nil {
+		return nil
+	}
+
+	orig := swagger
+	cloned := false
+	clone := func() {
+		if !cloned {
+			cloned = true
+			swagger = &spec.Swagger{}
+			*swagger = *orig
+		}
+	}
+
+	parametersCloned := false
+	for k, v := range swagger.Parameters {
+		if p := w.walkParameter(&v); p != &v {
+			if !parametersCloned {
+				parametersCloned = true
+				clone()
+				swagger.Parameters = make(map[string]spec.Parameter, len(orig.Parameters))
+				for k2, v2 := range orig.Parameters {
+					swagger.Parameters[k2] = v2
+				}
+			}
+			swagger.Parameters[k] = *p
+		}
+	}
+
+	responsesCloned := false
+	for k, v := range swagger.Responses {
+		if r := w.walkResponse(&v); r != &v {
+			if !responsesCloned {
+				responsesCloned = true
+				clone()
+				swagger.Responses = make(map[string]spec.Response, len(orig.Responses))
+				for k2, v2 := range orig.Responses {
+					swagger.Responses[k2] = v2
+				}
+			}
+			swagger.Responses[k] = *r
+		}
+	}
+
+	definitionsCloned := false
+	for k, v := range swagger.Definitions {
+		if s := w.WalkSchema(&v); s != &v {
+			if !definitionsCloned {
+				definitionsCloned = true
+				clone()
+				swagger.Definitions = make(spec.Definitions, len(orig.Definitions))
+				for k2, v2 := range orig.Definitions {
+					swagger.Definitions[k2] = v2
+				}
+			}
+			swagger.Definitions[k] = *s
+		}
+	}
+
+	if swagger.Paths != nil {
+		if p := w.walkPaths(swagger.Paths); p != swagger.Paths {
+			clone()
+			swagger.Paths = p
+		}
+	}
+
+	return swagger
+}
+
+func (w *Walker) WalkSchema(schema *spec.Schema) *spec.Schema {
+	if schema == nil {
+		return nil
+	}
+
+	orig := schema
+	clone := func() {
+		if orig == schema {
+			schema = &spec.Schema{}
+			*schema = *orig
+		}
+	}
+
+	// Always run callback on the whole schema first
+	// so that SchemaCallback can take the original schema as input.
+	schema = w.SchemaCallback(schema)
+
+	if r := w.RefCallback(&schema.Ref); r != &schema.Ref {
+		clone()
+		schema.Ref = *r
+	}
+
+	definitionsCloned := false
+	for k, v := range schema.Definitions {
+		if s := w.WalkSchema(&v); s != &v {
+			if !definitionsCloned {
+				definitionsCloned = true
+				clone()
+				schema.Definitions = make(spec.Definitions, len(orig.Definitions))
+				for k2, v2 := range orig.Definitions {
+					schema.Definitions[k2] = v2
+				}
+			}
+			schema.Definitions[k] = *s
+		}
+	}
+
+	propertiesCloned := false
+	for k, v := range schema.Properties {
+		if s := w.WalkSchema(&v); s != &v {
+			if !propertiesCloned {
+				propertiesCloned = true
+				clone()
+				schema.Properties = make(map[string]spec.Schema, len(orig.Properties))
+				for k2, v2 := range orig.Properties {
+					schema.Properties[k2] = v2
+				}
+			}
+			schema.Properties[k] = *s
+		}
+	}
+
+	patternPropertiesCloned := false
+	for k, v := range schema.PatternProperties {
+		if s := w.WalkSchema(&v); s != &v {
+			if !patternPropertiesCloned {
+				patternPropertiesCloned = true
+				clone()
+				schema.PatternProperties = make(map[string]spec.Schema, len(orig.PatternProperties))
+				for k2, v2 := range orig.PatternProperties {
+					schema.PatternProperties[k2] = v2
+				}
+			}
+			schema.PatternProperties[k] = *s
+		}
+	}
+
+	allOfCloned := false
+	for i := range schema.AllOf {
+		if s := w.WalkSchema(&schema.AllOf[i]); s != &schema.AllOf[i] {
+			if !allOfCloned {
+				allOfCloned = true
+				clone()
+				schema.AllOf = make([]spec.Schema, len(orig.AllOf))
+				copy(schema.AllOf, orig.AllOf)
+			}
+			schema.AllOf[i] = *s
+		}
+	}
+
+	anyOfCloned := false
+	for i := range schema.AnyOf {
+		if s := w.WalkSchema(&schema.AnyOf[i]); s != &schema.AnyOf[i] {
+			if !anyOfCloned {
+				anyOfCloned = true
+				clone()
+				schema.AnyOf = make([]spec.Schema, len(orig.AnyOf))
+				copy(schema.AnyOf, orig.AnyOf)
+			}
+			schema.AnyOf[i] = *s
+		}
+	}
+
+	oneOfCloned := false
+	for i := range schema.OneOf {
+		if s := w.WalkSchema(&schema.OneOf[i]); s != &schema.OneOf[i] {
+			if !oneOfCloned {
+				oneOfCloned = true
+				clone()
+				schema.OneOf = make([]spec.Schema, len(orig.OneOf))
+				copy(schema.OneOf, orig.OneOf)
+			}
+			schema.OneOf[i] = *s
+		}
+	}
+
+	if schema.Not != nil {
+		if s := w.WalkSchema(schema.Not); s != schema.Not {
+			clone()
+			schema.Not = s
+		}
+	}
+
+	if schema.AdditionalProperties != nil && schema.AdditionalProperties.Schema != nil {
+		if s := w.WalkSchema(schema.AdditionalProperties.Schema); s != schema.AdditionalProperties.Schema {
+			clone()
+			schema.AdditionalProperties = &spec.SchemaOrBool{Schema: s, Allows: schema.AdditionalProperties.Allows}
+		}
+	}
+
+	if schema.AdditionalItems != nil && schema.AdditionalItems.Schema != nil {
+		if s := w.WalkSchema(schema.AdditionalItems.Schema); s != schema.AdditionalItems.Schema {
+			clone()
+			schema.AdditionalItems = &spec.SchemaOrBool{Schema: s, Allows: schema.AdditionalItems.Allows}
+		}
+	}
+
+	if schema.Items != nil {
+		if schema.Items.Schema != nil {
+			if s := w.WalkSchema(schema.Items.Schema); s != schema.Items.Schema {
+				clone()
+				schema.Items = &spec.SchemaOrArray{Schema: s}
+			}
+		} else {
+			itemsCloned := false
+			for i := range schema.Items.Schemas {
+				if s := w.WalkSchema(&schema.Items.Schemas[i]); s != &schema.Items.Schemas[i] {
+					if !itemsCloned {
+						clone()
+						schema.Items = &spec.SchemaOrArray{
+							Schemas: make([]spec.Schema, len(orig.Items.Schemas)),
+						}
+						itemsCloned = true
+						copy(schema.Items.Schemas, orig.Items.Schemas)
+					}
+					schema.Items.Schemas[i] = *s
+				}
+			}
+		}
+	}
+
+	return schema
+}
+
+func (w *Walker) walkParameter(param *spec.Parameter) *spec.Parameter {
+	if param == nil {
+		return nil
+	}
+
+	orig := param
+	cloned := false
+	clone := func() {
+		if !cloned {
+			cloned = true
+			param = &spec.Parameter{}
+			*param = *orig
+		}
+	}
+
+	if r := w.RefCallback(&param.Ref); r != &param.Ref {
+		clone()
+		param.Ref = *r
+	}
+	if s := w.WalkSchema(param.Schema); s != param.Schema {
+		clone()
+		param.Schema = s
+	}
+	if param.Items != nil {
+		if r := w.RefCallback(&param.Items.Ref); r != &param.Items.Ref {
+			param.Items.Ref = *r
+		}
+	}
+
+	return param
+}
+
+func (w *Walker) walkParameters(params []spec.Parameter) ([]spec.Parameter, bool) {
+	if params == nil {
+		return nil, false
+	}
+
+	orig := params
+	cloned := false
+	clone := func() {
+		if !cloned {
+			cloned = true
+			params = make([]spec.Parameter, len(params))
+			copy(params, orig)
+		}
+	}
+
+	for i := range params {
+		if s := w.walkParameter(&params[i]); s != &params[i] {
+			clone()
+			params[i] = *s
+		}
+	}
+
+	return params, cloned
+}
+
+func (w *Walker) walkResponse(resp *spec.Response) *spec.Response {
+	if resp == nil {
+		return nil
+	}
+
+	orig := resp
+	cloned := false
+	clone := func() {
+		if !cloned {
+			cloned = true
+			resp = &spec.Response{}
+			*resp = *orig
+		}
+	}
+
+	if r := w.RefCallback(&resp.Ref); r != &resp.Ref {
+		clone()
+		resp.Ref = *r
+	}
+	if s := w.WalkSchema(resp.Schema); s != resp.Schema {
+		clone()
+		resp.Schema = s
+	}
+
+	return resp
+}
+
+func (w *Walker) walkResponses(resps *spec.Responses) *spec.Responses {
+	if resps == nil {
+		return nil
+	}
+
+	orig := resps
+	cloned := false
+	clone := func() {
+		if !cloned {
+			cloned = true
+			resps = &spec.Responses{}
+			*resps = *orig
+		}
+	}
+
+	if r := w.walkResponse(resps.ResponsesProps.Default); r != resps.ResponsesProps.Default {
+		clone()
+		resps.Default = r
+	}
+
+	responsesCloned := false
+	for k, v := range resps.ResponsesProps.StatusCodeResponses {
+		if r := w.walkResponse(&v); r != &v {
+			if !responsesCloned {
+				responsesCloned = true
+				clone()
+				resps.ResponsesProps.StatusCodeResponses = make(map[int]spec.Response, len(orig.StatusCodeResponses))
+				for k2, v2 := range orig.StatusCodeResponses {
+					resps.ResponsesProps.StatusCodeResponses[k2] = v2
+				}
+			}
+			resps.ResponsesProps.StatusCodeResponses[k] = *r
+		}
+	}
+
+	return resps
+}
+
+func (w *Walker) walkOperation(op *spec.Operation) *spec.Operation {
+	if op == nil {
+		return nil
+	}
+
+	orig := op
+	cloned := false
+	clone := func() {
+		if !cloned {
+			cloned = true
+			op = &spec.Operation{}
+			*op = *orig
+		}
+	}
+
+	parametersCloned := false
+	for i := range op.Parameters {
+		if s := w.walkParameter(&op.Parameters[i]); s != &op.Parameters[i] {
+			if !parametersCloned {
+				parametersCloned = true
+				clone()
+				op.Parameters = make([]spec.Parameter, len(orig.Parameters))
+				copy(op.Parameters, orig.Parameters)
+			}
+			op.Parameters[i] = *s
+		}
+	}
+
+	if r := w.walkResponses(op.Responses); r != op.Responses {
+		clone()
+		op.Responses = r
+	}
+
+	return op
+}
+
+func (w *Walker) walkPathItem(pathItem *spec.PathItem) *spec.PathItem {
+	if pathItem == nil {
+		return nil
+	}
+
+	orig := pathItem
+	cloned := false
+	clone := func() {
+		if !cloned {
+			cloned = true
+			pathItem = &spec.PathItem{}
+			*pathItem = *orig
+		}
+	}
+
+	if p, changed := w.walkParameters(pathItem.Parameters); changed {
+		clone()
+		pathItem.Parameters = p
+	}
+	if op := w.walkOperation(pathItem.Get); op != pathItem.Get {
+		clone()
+		pathItem.Get = op
+	}
+	if op := w.walkOperation(pathItem.Head); op != pathItem.Head {
+		clone()
+		pathItem.Head = op
+	}
+	if op := w.walkOperation(pathItem.Delete); op != pathItem.Delete {
+		clone()
+		pathItem.Delete = op
+	}
+	if op := w.walkOperation(pathItem.Options); op != pathItem.Options {
+		clone()
+		pathItem.Options = op
+	}
+	if op := w.walkOperation(pathItem.Patch); op != pathItem.Patch {
+		clone()
+		pathItem.Patch = op
+	}
+	if op := w.walkOperation(pathItem.Post); op != pathItem.Post {
+		clone()
+		pathItem.Post = op
+	}
+	if op := w.walkOperation(pathItem.Put); op != pathItem.Put {
+		clone()
+		pathItem.Put = op
+	}
+
+	return pathItem
+}
+
+func (w *Walker) walkPaths(paths *spec.Paths) *spec.Paths {
+	if paths == nil {
+		return nil
+	}
+
+	orig := paths
+	cloned := false
+	clone := func() {
+		if !cloned {
+			cloned = true
+			paths = &spec.Paths{}
+			*paths = *orig
+		}
+	}
+
+	pathsCloned := false
+	for k, v := range paths.Paths {
+		if p := w.walkPathItem(&v); p != &v {
+			if !pathsCloned {
+				pathsCloned = true
+				clone()
+				paths.Paths = make(map[string]spec.PathItem, len(orig.Paths))
+				for k2, v2 := range orig.Paths {
+					paths.Paths[k2] = v2
+				}
+			}
+			paths.Paths[k] = *p
+		}
+	}
+
+	return paths
+}

kube/pkg/openapi/merge/walker.go

@@ -0,0 +1,163 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package merge
+
+import (
+	"strings"
+
+	"github.com/go-openapi/spec"
+)
+
+const (
+	definitionPrefix = "#/definitions/"
+	parameterPrefix  = "#/parameters/"
+)
+
+// Run a readonlyReferenceWalker method on all references of an OpenAPI spec
+type readonlyReferenceWalker struct {
+	// walkRefCallback will be called on each reference. The input will never be nil.
+	walkRefCallback func(ref *spec.Ref)
+
+	// The spec to walk through.
+	root *spec.Swagger
+}
+
+// walkOnAllReferences recursively walks on all references, while following references into definitions.
+// it calls walkRef on each found reference.
+func walkOnAllReferences(walkRef func(ref *spec.Ref), root *spec.Swagger) {
+	alreadyVisited := map[string]bool{}
+
+	walker := &readonlyReferenceWalker{
+		root: root,
+	}
+	walker.walkRefCallback = func(ref *spec.Ref) {
+		walkRef(ref)
+
+		refStr := ref.String()
+		if refStr == "" || !strings.HasPrefix(refStr, definitionPrefix) {
+			return
+		}
+		defName := refStr[len(definitionPrefix):]
+
+		if _, found := root.Definitions[defName]; found && !alreadyVisited[refStr] {
+			alreadyVisited[refStr] = true
+			def := root.Definitions[defName]
+			walker.walkSchema(&def)
+		}
+	}
+	walker.Start()
+}
+
+func (s *readonlyReferenceWalker) walkSchema(schema *spec.Schema) {
+	if schema == nil {
+		return
+	}
+	s.walkRefCallback(&schema.Ref)
+	var v *spec.Schema
+	if len(schema.Definitions)+len(schema.Properties)+len(schema.PatternProperties) > 0 {
+		v = &spec.Schema{}
+	}
+	for k := range schema.Definitions {
+		*v = schema.Definitions[k]
+		s.walkSchema(v)
+	}
+	for k := range schema.Properties {
+		*v = schema.Properties[k]
+		s.walkSchema(v)
+	}
+	for k := range schema.PatternProperties {
+		*v = schema.PatternProperties[k]
+		s.walkSchema(v)
+	}
+	for i := range schema.AllOf {
+		s.walkSchema(&schema.AllOf[i])
+	}
+	for i := range schema.AnyOf {
+		s.walkSchema(&schema.AnyOf[i])
+	}
+	for i := range schema.OneOf {
+		s.walkSchema(&schema.OneOf[i])
+	}
+	if schema.Not != nil {
+		s.walkSchema(schema.Not)
+	}
+	if schema.AdditionalProperties != nil && schema.AdditionalProperties.Schema != nil {
+		s.walkSchema(schema.AdditionalProperties.Schema)
+	}
+	if schema.AdditionalItems != nil && schema.AdditionalItems.Schema != nil {
+		s.walkSchema(schema.AdditionalItems.Schema)
+	}
+	if schema.Items != nil {
+		if schema.Items.Schema != nil {
+			s.walkSchema(schema.Items.Schema)
+		}
+		for i := range schema.Items.Schemas {
+			s.walkSchema(&schema.Items.Schemas[i])
+		}
+	}
+}
+
+func (s *readonlyReferenceWalker) walkParams(params []spec.Parameter) {
+	if params == nil {
+		return
+	}
+	for _, param := range params {
+		s.walkRefCallback(&param.Ref)
+		s.walkSchema(param.Schema)
+		if param.Items != nil {
+			s.walkRefCallback(&param.Items.Ref)
+		}
+	}
+}
+
+func (s *readonlyReferenceWalker) walkResponse(resp *spec.Response) {
+	if resp == nil {
+		return
+	}
+	s.walkRefCallback(&resp.Ref)
+	s.walkSchema(resp.Schema)
+}
+
+func (s *readonlyReferenceWalker) walkOperation(op *spec.Operation) {
+	if op == nil {
+		return
+	}
+	s.walkParams(op.Parameters)
+	if op.Responses == nil {
+		return
+	}
+	s.walkResponse(op.Responses.Default)
+	for _, r := range op.Responses.StatusCodeResponses {
+		s.walkResponse(&r)
+	}
+}
+
+func (s *readonlyReferenceWalker) Start() {
+	if s.root.Paths == nil {
+		return
+	}
+	for _, pathItem := range s.root.Paths.Paths {
+		s.walkParams(pathItem.Parameters)
+		s.walkOperation(pathItem.Delete)
+		s.walkOperation(pathItem.Get)
+		s.walkOperation(pathItem.Head)
+		s.walkOperation(pathItem.Options)
+		s.walkOperation(pathItem.Patch)
+		s.walkOperation(pathItem.Post)
+		s.walkOperation(pathItem.Put)
+	}
+}

kube/pkg/openapi/openapi.go

@@ -0,0 +1,102 @@
+/*
+ * Please refer to the LICENSE file in the root directory of the project.
+ * https://github.com/kubesphere/kubesphere/blob/master/LICENSE
+ */
+
+package openapi
+
+import (
+	"fmt"
+	"net/http"
+	"sync/atomic"
+
+	extensionsv1alpha1 "kubesphere.io/api/extensions/v1alpha1"
+)
+
+type Cache[T any] struct {
+	value atomic.Pointer[T]
+}
+
+func (c *Cache[T]) Store(v T) {
+	c.value.Store(&v)
+}
+
+func (c *Cache[T]) Load() T {
+	return *c.value.Load()
+}
+
+type PathHandler interface {
+	Handle(path string, handler http.Handler)
+}
+
+type APIServiceManager interface {
+	AddUpdateApiService(apiService *extensionsv1alpha1.APIService) error
+	UpdateOpenApiSpec(apiServiceName string) error
+	RemoveApiService(apiServiceName string)
+}
+
+type OpenApiAggregatorServices struct {
+	apiService    map[string]ApiService
+	downloaderMap map[string]CacheableDownloader
+	downloader    *Downloader
+}
+
+func NewOpenApiAggregatorServices() *OpenApiAggregatorServices {
+	return &OpenApiAggregatorServices{
+		apiService:    make(map[string]ApiService),
+		downloaderMap: make(map[string]CacheableDownloader),
+		downloader:    NewDownloader(),
+	}
+}
+
+func (o *OpenApiAggregatorServices) AddUpdateApiService(apiService *extensionsv1alpha1.APIService) error {
+	openapiService := NewApiService(apiService)
+	o.apiService[apiService.Name] = openapiService
+
+	if d, ok := o.downloaderMap[apiService.Name]; ok {
+		if err := d.UpdateDownloader(openapiService); err != nil {
+			return err
+		}
+	} else {
+		cacheDownloader, err := NewCacheableDownloader(openapiService, o.downloader)
+		if err != nil {
+			return err
+		}
+		o.downloaderMap[apiService.Name] = cacheDownloader
+	}
+	return nil
+}
+
+func (o *OpenApiAggregatorServices) GetOpenApiSpecV2(apiServiceName string) ([]byte, error) {
+	if d, ok := o.downloaderMap[apiServiceName]; ok {
+		data, err := d.GetV2()
+		if err != nil {
+			return nil, fmt.Errorf("fetch ApiService %s openapi-v2 error: %s", apiServiceName, err)
+		}
+		return data, nil
+	}
+	return nil, fmt.Errorf("update OpenApiSpec failed beaseuse of apiService %s not found", apiServiceName)
+}
+
+func (o *OpenApiAggregatorServices) GetOpenApiSpecV3(apiServiceName string) ([]byte, error) {
+	if d, ok := o.downloaderMap[apiServiceName]; ok {
+		data, err := d.GetV3()
+		if err != nil {
+			return nil, fmt.Errorf("fetch ApiService %s openapi-v3 error: %s", apiServiceName, err)
+		}
+		return data, nil
+	}
+	return nil, fmt.Errorf("update OpenApiSpec failed beaseuse of apiService %s not found", apiServiceName)
+}
+
+func (o *OpenApiAggregatorServices) AddLocalApiService(name string) {
+	apiService := extensionsv1alpha1.APIService{}
+	apiService.Name = name
+
+	o.apiService[apiService.Name] = NewApiService(&apiService)
+}
+
+func (o *OpenApiAggregatorServices) RemoveApiService(apiServiceName string) {
+	delete(o.apiService, apiServiceName)
+	delete(o.downloaderMap, apiServiceName)
+}

kube/pkg/openapi/v2/services.go

@@ -0,0 +1,143 @@
+/*
+ * Please refer to the LICENSE file in the root directory of the project.
+ * https://github.com/kubesphere/kubesphere/blob/master/LICENSE
+ */
+
+package v2
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/NYTimes/gziphandler"
+	restfulspec "github.com/emicklei/go-restful-openapi/v2"
+	"github.com/go-openapi/spec"
+	"k8s.io/klog/v2"
+
+	extensionsv1alpha1 "kubesphere.io/api/extensions/v1alpha1"
+
+	"kubesphere.io/kubesphere/kube/pkg/openapi"
+	"kubesphere.io/kubesphere/kube/pkg/openapi/merge"
+)
+
+var OpenApiPath = "/openapi/v2"
+
+type OpenApiV2Services struct {
+	openApiSpecCache         map[string]*openapi.Cache[*spec.Swagger]
+	openApiAggregatorService *openapi.OpenApiAggregatorServices
+}
+
+func NewOpenApiV2Services() *OpenApiV2Services {
+	return &OpenApiV2Services{
+		openApiSpecCache:         make(map[string]*openapi.Cache[*spec.Swagger]),
+		openApiAggregatorService: openapi.NewOpenApiAggregatorServices(),
+	}
+}
+
+func (s *OpenApiV2Services) AddUpdateApiService(apiService *extensionsv1alpha1.APIService) error {
+	c := &openapi.Cache[*spec.Swagger]{}
+	c.Store(&spec.Swagger{})
+	s.openApiSpecCache[apiService.Name] = c
+	if err := s.openApiAggregatorService.AddUpdateApiService(apiService); err != nil {
+		return err
+	}
+	return s.UpdateOpenApiSpec(apiService.Name)
+}
+
+func (s *OpenApiV2Services) UpdateOpenApiSpec(apiServiceName string) error {
+	data, err := s.openApiAggregatorService.GetOpenApiSpecV2(apiServiceName)
+	if err != nil {
+		return err
+	}
+	openAPISpec := &spec.Swagger{}
+	if err := openAPISpec.UnmarshalJSON(data); err != nil {
+		return err
+	}
+
+	if cache, ok := s.openApiSpecCache[apiServiceName]; ok {
+		cache.Store(openAPISpec)
+	} else {
+		c := openapi.Cache[*spec.Swagger]{}
+		c.Store(openAPISpec)
+		s.openApiSpecCache[apiServiceName] = &c
+	}
+	return nil
+}
+
+func (s *OpenApiV2Services) RemoveApiService(apiServiceName string) {
+	s.openApiAggregatorService.RemoveApiService(apiServiceName)
+	delete(s.openApiSpecCache, apiServiceName)
+}
+
+func (s *OpenApiV2Services) MergeSpecCache() (*spec.Swagger, error) {
+	var merged *spec.Swagger
+	for i := range s.openApiSpecCache {
+		if cacheValue, ok := s.openApiSpecCache[i]; ok {
+			cacheSpec := cacheValue.Load()
+			if merged == nil {
+				merged = &spec.Swagger{}
+				*merged = *cacheSpec
+				merged.Paths = nil
+				merged.Definitions = nil
+				merged.Parameters = nil
+			}
+			if err := merge.MergeSpecsIgnorePathConflictRenamingDefinitionsAndParameters(merged, cacheSpec); err != nil {
+				return nil, fmt.Errorf("failed to build merge specs: %v", err)
+			}
+		}
+	}
+	return merged, nil
+}
+
+func (s *OpenApiV2Services) RegisterOpenAPIVersionedService(servePath string, handler openapi.PathHandler) {
+	handler.Handle(servePath, gziphandler.GzipHandler(http.HandlerFunc(
+		func(w http.ResponseWriter, r *http.Request) {
+
+			result, err := s.MergeSpecCache()
+			if err != nil {
+				klog.Errorf("Error in OpenAPI handler: %s", err)
+				// only return a 503 if we have no older cache data to serve
+				if result == nil {
+					w.WriteHeader(http.StatusServiceUnavailable)
+					return
+				}
+			}
+			data, err := (*result).MarshalJSON()
+			if err != nil {
+				klog.Errorf("Error in OpenAPI handler: %s", err)
+				if data == nil {
+					w.WriteHeader(http.StatusServiceUnavailable)
+					return
+				}
+			}
+			w.Write(data)
+		}),
+	))
+}
+
+func (s *OpenApiV2Services) AddLocalApiService(name string, val *spec.Swagger) {
+	s.openApiAggregatorService.AddLocalApiService(name)
+	c := &openapi.Cache[*spec.Swagger]{}
+	c.Store(val)
+	s.openApiSpecCache[name] = c
+}
+
+func BuildAndRegisterAggregator(
+	config *restfulspec.Config, pathHandler openapi.PathHandler) (*OpenApiV2Services, error) {
+
+	aggregatorOpenAPISpec := restfulspec.BuildSwagger(*config)
+	aggregatorOpenAPISpec.Definitions = merge.PruneDefaults(aggregatorOpenAPISpec.Definitions)
+
+	s := buildAndRegisterOpenApiV2ForLocalServices(OpenApiPath, aggregatorOpenAPISpec, pathHandler)
+	return s, nil
+}
+
+func buildAndRegisterOpenApiV2ForLocalServices(path string, aggregatorSpec *spec.Swagger, pathHandler openapi.PathHandler) *OpenApiV2Services {
+	s := NewOpenApiV2Services()
+	s.AddLocalApiService("kubeSphere_internal_local_delegation", aggregatorSpec)
+	if path == "" {
+		path = OpenApiPath
+	}
+	s.RegisterOpenAPIVersionedService(path, pathHandler)
+	return s
+}

kube/pkg/openapi/v2/services_test.go

@@ -0,0 +1,43 @@
+/*
+ * Please refer to the LICENSE file in the root directory of the project.
+ * https://github.com/kubesphere/kubesphere/blob/master/LICENSE
+ */
+
+package v2
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	extensionsv1alpha1 "kubesphere.io/api/extensions/v1alpha1"
+)
+
+func TestServiceAddUpdateApiService(t *testing.T) {
+	uri := "http://172.31.188.161:8080"
+	apiServer := extensionsv1alpha1.APIService{
+		TypeMeta: metav1.TypeMeta{},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "v1alpha1.local.kubesphere.io",
+		},
+		Spec: extensionsv1alpha1.APIServiceSpec{
+			Group:   "local.kubesphere.io",
+			Version: "v1alpha1",
+			Endpoint: extensionsv1alpha1.Endpoint{
+				URL:                &uri,
+				Service:            nil,
+				CABundle:           nil,
+				InsecureSkipVerify: false,
+			},
+		},
+		Status: extensionsv1alpha1.APIServiceStatus{},
+	}
+
+	openApiV2Services := NewOpenApiV2Services()
+	err := openApiV2Services.AddUpdateApiService(&apiServer)
+	assert.Equal(t, err, nil)
+	val, err := openApiV2Services.MergeSpecCache()
+	assert.Equal(t, err, nil)
+	t.Log(val)
+}

kube/pkg/openapi/v3/services.go

@@ -0,0 +1,154 @@
+/*
+ * Please refer to the LICENSE file in the root directory of the project.
+ * https://github.com/kubesphere/kubesphere/blob/master/LICENSE
+ */
+
+package v3
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/NYTimes/gziphandler"
+	restfulspec "github.com/emicklei/go-restful-openapi/v2"
+	openapibuilder "k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder"
+	"k8s.io/klog/v2"
+	"k8s.io/kube-openapi/pkg/openapiconv"
+	"k8s.io/kube-openapi/pkg/spec3"
+	spec2 "k8s.io/kube-openapi/pkg/validation/spec"
+
+	extensionsv1alpha1 "kubesphere.io/api/extensions/v1alpha1"
+
+	"kubesphere.io/kubesphere/kube/pkg/openapi"
+	"kubesphere.io/kubesphere/kube/pkg/openapi/merge"
+)
+
+var OpenApiPath = "/openapi/v3"
+
+type OpenApiV3Services struct {
+	openApiSpecCache         map[string]*openapi.Cache[*spec3.OpenAPI]
+	openApiAggregatorService *openapi.OpenApiAggregatorServices
+}
+
+func NewOpenApiV3Services() *OpenApiV3Services {
+	return &OpenApiV3Services{
+		openApiSpecCache:         make(map[string]*openapi.Cache[*spec3.OpenAPI]),
+		openApiAggregatorService: openapi.NewOpenApiAggregatorServices(),
+	}
+}
+
+func (s *OpenApiV3Services) AddUpdateApiService(apiService *extensionsv1alpha1.APIService) error {
+	c := &openapi.Cache[*spec3.OpenAPI]{}
+	c.Store(&spec3.OpenAPI{})
+	s.openApiSpecCache[apiService.Name] = c
+	if err := s.openApiAggregatorService.AddUpdateApiService(apiService); err != nil {
+		return err
+	}
+	return s.UpdateOpenApiSpec(apiService.Name)
+}
+
+func (s *OpenApiV3Services) UpdateOpenApiSpec(apiServiceName string) error {
+	data, err := s.openApiAggregatorService.GetOpenApiSpecV3(apiServiceName)
+	if err != nil {
+		return err
+	}
+	openAPISpec := &spec3.OpenAPI{}
+	if err := openAPISpec.UnmarshalJSON(data); err != nil {
+		return err
+	}
+
+	if cache, ok := s.openApiSpecCache[apiServiceName]; ok {
+		cache.Store(openAPISpec)
+		return nil
+	} else {
+		c := openapi.Cache[*spec3.OpenAPI]{}
+		c.Store(openAPISpec)
+		s.openApiSpecCache[apiServiceName] = &c
+	}
+	return nil
+}
+
+func (s *OpenApiV3Services) RemoveApiService(apiServiceName string) {
+	s.openApiAggregatorService.RemoveApiService(apiServiceName)
+	delete(s.openApiSpecCache, apiServiceName)
+}
+
+func (s *OpenApiV3Services) MergeSpecCache() (*spec3.OpenAPI, error) {
+	var merged *spec3.OpenAPI
+	var err error
+	for i := range s.openApiSpecCache {
+		if cacheValue, ok := s.openApiSpecCache[i]; ok {
+			cacheSpec := cacheValue.Load()
+			if merged == nil {
+				merged = &spec3.OpenAPI{}
+				*merged = *cacheSpec
+				merged.Paths = nil
+			}
+			if merged, err = openapibuilder.MergeSpecsV3(merged, cacheSpec); err != nil {
+				return nil, fmt.Errorf("failed to build merge specs: %v", err)
+			}
+		}
+	}
+	return merged, nil
+}
+
+func (s *OpenApiV3Services) RegisterOpenAPIVersionedService(servePath string, handler openapi.PathHandler) {
+	handler.Handle(servePath, gziphandler.GzipHandler(http.HandlerFunc(
+		func(w http.ResponseWriter, r *http.Request) {
+
+			result, err := s.MergeSpecCache()
+			if err != nil {
+				klog.Errorf("Error in OpenAPI handler: %s", err)
+				// only return a 503 if we have no older cache data to serve
+				if result == nil {
+					w.WriteHeader(http.StatusServiceUnavailable)
+					return
+				}
+			}
+			data, err := (*result).MarshalJSON()
+			if err != nil {
+				klog.Errorf("Error in OpenAPI handler: %s", err)
+				if data == nil {
+					w.WriteHeader(http.StatusServiceUnavailable)
+					return
+				}
+			}
+			w.Write(data)
+		}),
+	))
+}
+
+func (s *OpenApiV3Services) AddLocalApiService(name string, val *spec3.OpenAPI) {
+	s.openApiAggregatorService.AddLocalApiService(name)
+	c := &openapi.Cache[*spec3.OpenAPI]{}
+	c.Store(val)
+	s.openApiSpecCache[name] = c
+}
+
+func BuildAndRegisterAggregator(
+	config *restfulspec.Config, pathHandler openapi.PathHandler) (*OpenApiV3Services, error) {
+
+	aggregatorOpenAPISpec := restfulspec.BuildSwagger(*config)
+	aggregatorOpenAPISpec.Definitions = merge.PruneDefaults(aggregatorOpenAPISpec.Definitions)
+	swaggerData, err := aggregatorOpenAPISpec.MarshalJSON()
+	if err != nil {
+		return nil, err
+	}
+	spec2Swagger := spec2.Swagger{}
+	if err = spec2Swagger.UnmarshalJSON(swaggerData); err != nil {
+		return nil, err
+	}
+	convertedOpenAPIV3 := openapiconv.ConvertV2ToV3(&spec2Swagger)
+	s := buildAndRegisterOpenApiV3ForLocalServices(OpenApiPath, convertedOpenAPIV3, pathHandler)
+	return s, nil
+}
+
+func buildAndRegisterOpenApiV3ForLocalServices(path string, aggregatorSpec *spec3.OpenAPI, pathHandler openapi.PathHandler) *OpenApiV3Services {
+	s := NewOpenApiV3Services()
+	s.AddLocalApiService("kubeSphere_internal_local_delegation", aggregatorSpec)
+	if path == "" {
+		path = OpenApiPath
+	}
+	s.RegisterOpenAPIVersionedService(path, pathHandler)
+	return s
+}

kube/pkg/openapi/v3/services_test.go

@@ -0,0 +1,6 @@
+/*
+ * Please refer to the LICENSE file in the root directory of the project.
+ * https://github.com/kubesphere/kubesphere/blob/master/LICENSE
+ */
+
+package v3