admin/kubesphere
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: kubesphere 4.0 (#6115)

Browse Source 
* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

---------

Signed-off-by: ci-bot <ci-bot@kubesphere.io>
Co-authored-by: ks-ci-bot <ks-ci-bot@example.com>
Co-authored-by: joyceliu <joyceliu@yunify.com>
This commit is contained in:
KubeSphere CI Bot
2024-09-06 11:05:52 +08:00
committed by GitHub
parent b5015ec7b9
commit 447a51f08b
8557 changed files with 546695 additions and 1146174 deletions
Download Patch File Download Diff File Expand all files Collapse all files

286
pkg/utils/clusterclient/clusterclient.go
View File

@@ -1,215 +1,195 @@
/*
Copyright 2020 KubeSphere Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package clusterclient
import (
"context"
"fmt"
"net/http"
"net/url"
"reflect"
"sync"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/klog/v2"
clusterv1alpha1 "kubesphere.io/api/cluster/v1alpha1"
runtimecache "sigs.k8s.io/controller-runtime/pkg/cache"
runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
kubesphere "kubesphere.io/kubesphere/pkg/client/clientset/versioned"
clusterinformer "kubesphere.io/kubesphere/pkg/client/informers/externalversions/cluster/v1alpha1"
clusterlister "kubesphere.io/kubesphere/pkg/client/listers/cluster/v1alpha1"
clusterutils "kubesphere.io/kubesphere/pkg/controller/cluster/utils"
"kubesphere.io/kubesphere/pkg/scheme"
)
type innerCluster struct {
KubernetesURL *url.URL
KubesphereURL *url.URL
Transport http.RoundTripper
type Interface interface {
Get(string) (*clusterv1alpha1.Cluster, error)
ListClusters(ctx context.Context) ([]clusterv1alpha1.Cluster, error)
GetClusterClient(string) (*ClusterClient, error)
GetRuntimeClient(string) (runtimeclient.Client, error)
}
type ClusterClient struct {
KubernetesURL *url.URL
KubeSphereURL *url.URL
KubernetesVersion string
RestConfig *rest.Config
Transport http.RoundTripper
Client runtimeclient.Client
KubernetesClient kubernetes.Interface
}
type clusterClients struct {
clusterLister clusterlister.ClusterLister
// build an in memory cluster cache to speed things up
innerClusters sync.Map
clients *sync.Map
cache runtimecache.Cache
}
type ClusterClients interface {
IsHostCluster(cluster *clusterv1alpha1.Cluster) bool
IsClusterReady(cluster *clusterv1alpha1.Cluster) bool
GetClusterKubeconfig(string) (string, error)
Get(string) (*clusterv1alpha1.Cluster, error)
GetInnerCluster(string) *innerCluster
GetKubernetesClientSet(string) (*kubernetes.Clientset, error)
GetKubeSphereClientSet(string) (*kubesphere.Clientset, error)
}
func NewClusterClient(clusterInformer clusterinformer.ClusterInformer) ClusterClients {
func NewClusterClientSet(runtimeCache runtimecache.Cache) (Interface, error) {
c := &clusterClients{
clusterLister: clusterInformer.Lister(),
clients: &sync.Map{},
cache: runtimeCache,
}
clusterInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
clusterInformer, err := runtimeCache.GetInformerForKind(context.Background(), clusterv1alpha1.SchemeGroupVersion.WithKind(clusterv1alpha1.ResourceKindCluster))
if err != nil {
return nil, err
}
if _, err = clusterInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
c.addCluster(obj)
if _, err = c.addCluster(obj); err != nil {
klog.Error(err)
}
},
UpdateFunc: func(oldObj, newObj interface{}) {
oldCluster := oldObj.(*clusterv1alpha1.Cluster)
newCluster := newObj.(*clusterv1alpha1.Cluster)
if !reflect.DeepEqual(oldCluster.Spec, newCluster.Spec) {
c.addCluster(newObj)
c.removeCluster(oldCluster)
if _, err = c.addCluster(newObj); err != nil {
klog.Error(err)
}
}
},
DeleteFunc: c.removeCluster,
}); err != nil {
return nil, err
}
return c, nil
}
func (c *clusterClients) addCluster(obj interface{}) (*ClusterClient, error) {
cluster := obj.(*clusterv1alpha1.Cluster)
klog.V(4).Infof("add new cluster %s", cluster.Name)
kubernetesEndpoint, err := url.Parse(cluster.Spec.Connection.KubernetesAPIEndpoint)
if err != nil {
return nil, fmt.Errorf("parse kubernetes apiserver endpoint %s failed: %v", cluster.Spec.Connection.KubernetesAPIEndpoint, err)
}
kubesphereEndpoint, err := url.Parse(cluster.Spec.Connection.KubeSphereAPIEndpoint)
if err != nil {
return nil, fmt.Errorf("parse kubesphere apiserver endpoint %s failed: %v", cluster.Spec.Connection.KubeSphereAPIEndpoint, err)
}
restConfig, err := clientcmd.RESTConfigFromKubeConfig(cluster.Spec.Connection.KubeConfig)
if err != nil {
return nil, err
}
// It also applies saner defaults for QPS and burst based on the Kubernetes
// controller manager defaults (20 QPS, 30 burst)
if restConfig.QPS == 0.0 {
restConfig.QPS = 20.0
}
if restConfig.Burst == 0 {
restConfig.Burst = 30
}
kubernetesClient, err := kubernetes.NewForConfig(restConfig)
if err != nil {
return nil, err
}
serverVersion, err := kubernetesClient.Discovery().ServerVersion()
if err != nil {
return nil, err
}
httpClient, err := rest.HTTPClientFor(restConfig)
if err != nil {
return nil, err
}
mapper, err := apiutil.NewDynamicRESTMapper(restConfig, httpClient)
if err != nil {
return nil, err
}
if err != nil {
return nil, err
}
client, err := runtimeclient.New(restConfig, runtimeclient.Options{
HTTPClient: httpClient,
Scheme: scheme.Scheme,
Mapper: mapper,
Cache: nil,
})
return c
if err != nil {
return nil, err
}
clusterClient := &ClusterClient{
KubernetesURL: kubernetesEndpoint,
KubeSphereURL: kubesphereEndpoint,
KubernetesVersion: serverVersion.GitVersion,
RestConfig: restConfig,
Transport: httpClient.Transport,
Client: client,
KubernetesClient: kubernetesClient,
}
c.clients.Store(cluster.Name, clusterClient)
return clusterClient, nil
}
func (c *clusterClients) removeCluster(obj interface{}) {
cluster := obj.(*clusterv1alpha1.Cluster)
klog.V(4).Infof("remove cluster %s", cluster.Name)
c.innerClusters.Delete(cluster.Name)
}
func newInnerCluster(cluster *clusterv1alpha1.Cluster) *innerCluster {
kubernetesEndpoint, err := url.Parse(cluster.Spec.Connection.KubernetesAPIEndpoint)
if err != nil {
klog.Errorf("Parse kubernetes apiserver endpoint %s failed, %v", cluster.Spec.Connection.KubernetesAPIEndpoint, err)
return nil
if _, ok := c.clients.Load(cluster.Name); ok {
klog.V(4).Infof("remove cluster %s", cluster.Name)
c.clients.Delete(cluster.Name)
}
kubesphereEndpoint, err := url.Parse(cluster.Spec.Connection.KubeSphereAPIEndpoint)
if err != nil {
klog.Errorf("Parse kubesphere apiserver endpoint %s failed, %v", cluster.Spec.Connection.KubeSphereAPIEndpoint, err)
return nil
}
// prepare for
clientConfig, err := clientcmd.NewClientConfigFromBytes(cluster.Spec.Connection.KubeConfig)
if err != nil {
klog.Errorf("Unable to create client config from kubeconfig bytes, %#v", err)
return nil
}
clusterConfig, err := clientConfig.ClientConfig()
if err != nil {
klog.Errorf("Failed to get client config, %#v", err)
return nil
}
transport, err := rest.TransportFor(clusterConfig)
if err != nil {
klog.Errorf("Create transport failed, %v", err)
return nil
}
return &innerCluster{
KubernetesURL: kubernetesEndpoint,
KubesphereURL: kubesphereEndpoint,
Transport: transport,
}
}
func (c *clusterClients) addCluster(obj interface{}) *innerCluster {
cluster := obj.(*clusterv1alpha1.Cluster)
klog.V(4).Infof("add new cluster %s", cluster.Name)
_, err := url.Parse(cluster.Spec.Connection.KubernetesAPIEndpoint)
if err != nil {
klog.Errorf("Parse kubernetes apiserver endpoint %s failed, %v", cluster.Spec.Connection.KubernetesAPIEndpoint, err)
return nil
}
inner := newInnerCluster(cluster)
c.innerClusters.Store(cluster.Name, inner)
return inner
}
func (c *clusterClients) Get(clusterName string) (*clusterv1alpha1.Cluster, error) {
return c.clusterLister.Get(clusterName)
cluster := &clusterv1alpha1.Cluster{}
err := c.cache.Get(context.Background(), types.NamespacedName{Name: clusterName}, cluster)
return cluster, err
}
func (c *clusterClients) GetClusterKubeconfig(clusterName string) (string, error) {
cluster, err := c.clusterLister.Get(clusterName)
if err != nil {
return "", err
func (c *clusterClients) ListClusters(ctx context.Context) ([]clusterv1alpha1.Cluster, error) {
clusterList := &clusterv1alpha1.ClusterList{}
if err := c.cache.List(ctx, clusterList); err != nil {
return nil, err
}
return string(cluster.Spec.Connection.KubeConfig), nil
return clusterList.Items, nil
}
func (c *clusterClients) GetInnerCluster(name string) *innerCluster {
if inner, ok := c.innerClusters.Load(name); ok {
return inner.(*innerCluster)
} else if cluster, err := c.clusterLister.Get(name); err == nil {
// double check if the cluster exists but is not cached
return c.addCluster(cluster)
func (c *clusterClients) GetClusterClient(name string) (*ClusterClient, error) {
if client, ok := c.clients.Load(name); ok {
return client.(*ClusterClient), nil
}
return nil
}
func (c *clusterClients) IsClusterReady(cluster *clusterv1alpha1.Cluster) bool {
return clusterutils.IsClusterReady(cluster)
}
func (c *clusterClients) IsHostCluster(cluster *clusterv1alpha1.Cluster) bool {
if _, ok := cluster.Labels[clusterv1alpha1.HostCluster]; ok {
return true
}
return false
}
func (c *clusterClients) GetKubeSphereClientSet(name string) (*kubesphere.Clientset, error) {
kubeconfig, err := c.GetClusterKubeconfig(name)
// double check if the cluster exists but is not cached
cluster, err := c.Get(name)
if err != nil {
return nil, err
}
restConfig, err := newRestConfigFromString(kubeconfig)
if err != nil {
return nil, err
}
clientSet, err := kubesphere.NewForConfig(restConfig)
if err != nil {
return nil, err
}
return clientSet, nil
return c.addCluster(cluster)
}
func (c *clusterClients) GetKubernetesClientSet(name string) (*kubernetes.Clientset, error) {
kubeconfig, err := c.GetClusterKubeconfig(name)
func (c *clusterClients) GetRuntimeClient(name string) (runtimeclient.Client, error) {
clusterClient, err := c.GetClusterClient(name)
if err != nil {
return nil, err
}
restConfig, err := newRestConfigFromString(kubeconfig)
if err != nil {
return nil, err
}
clientSet, err := kubernetes.NewForConfig(restConfig)
if err != nil {
return nil, err
}
return clientSet, nil
}
func newRestConfigFromString(kubeconfig string) (*rest.Config, error) {
bytes, err := clientcmd.NewClientConfigFromBytes([]byte(kubeconfig))
if err != nil {
return nil, err
}
return bytes.ClientConfig()
return clusterClient.Client, nil
}

238
pkg/utils/directives/match.go Normal file
View File

@@ -0,0 +1,238 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package directives
import (
"net/http"
"net/url"
"path"
"runtime"
"strings"
)
// Following code copied from github.com/caddyserver/caddy/modules/caddyhttp/matchers.go
type (
MatchPath []string
)
func (m MatchPath) Match(req *http.Request) bool {
// Even though RFC 9110 says that path matching is case-sensitive
// (https://www.rfc-editor.org/rfc/rfc9110.html#section-4.2.3),
// we do case-insensitive matching to mitigate security issues
// related to differences between operating systems, applications,
// etc; if case-sensitive matching is needed, the regex matcher
// can be used instead.
reqPath := strings.ToLower(req.URL.Path)
// See #2917; Windows ignores trailing dots and spaces
// when accessing files (sigh), potentially causing a
// security risk (cry) if PHP files end up being served
// as static files, exposing the source code, instead of
// being matched by *.php to be treated as PHP scripts.
if runtime.GOOS == "windows" { // issue #5613
reqPath = strings.TrimRight(reqPath, ". ")
}
repl := req.Context().Value(ReplacerCtxKey).(*Replacer)
for _, matchPattern := range m {
matchPattern = repl.ReplaceAll(matchPattern, "")
// special case: whole path is wildcard; this is unnecessary
// as it matches all requests, which is the same as no matcher
if matchPattern == "*" {
return true
}
// Clean the path, merge doubled slashes, etc.
// This ensures maliciously crafted requests can't bypass
// the path matcher. See #4407. Good security posture
// requires that we should do all we can to reduce any
// funny-looking paths into "normalized" forms such that
// weird variants can't sneak by.
//
// How we clean the path depends on the kind of pattern:
// we either merge slashes or we don't. If the pattern
// has double slashes, we preserve them in the path.
//
// TODO: Despite the fact that the *vast* majority of path
// matchers have only 1 pattern, a possible optimization is
// to remember the cleaned form of the path for future
// iterations; it's just that the way we clean depends on
// the kind of pattern.
mergeSlashes := !strings.Contains(matchPattern, "//")
// if '%' appears in the match pattern, we interpret that to mean
// the intent is to compare that part of the path in raw/escaped
// space; i.e. "%40"=="%40", not "@", and "%2F"=="%2F", not "/"
if strings.Contains(matchPattern, "%") {
reqPathForPattern := CleanPath(req.URL.EscapedPath(), mergeSlashes)
if m.matchPatternWithEscapeSequence(reqPathForPattern, matchPattern) {
return true
}
// doing prefix/suffix/substring matches doesn't make sense
continue
}
reqPathForPattern := CleanPath(reqPath, mergeSlashes)
// for substring, prefix, and suffix matching, only perform those
// special, fast matches if they are the only wildcards in the pattern;
// otherwise we assume a globular match if any * appears in the middle
// special case: first and last characters are wildcard,
// treat it as a fast substring match
if strings.Count(matchPattern, "*") == 2 &&
strings.HasPrefix(matchPattern, "*") &&
strings.HasSuffix(matchPattern, "*") &&
strings.Count(matchPattern, "*") == 2 {
if strings.Contains(reqPathForPattern, matchPattern[1:len(matchPattern)-1]) {
return true
}
continue
}
// only perform prefix/suffix match if it is the only wildcard...
// I think that is more correct most of the time
if strings.Count(matchPattern, "*") == 1 {
// special case: first character is a wildcard,
// treat it as a fast suffix match
if strings.HasPrefix(matchPattern, "*") {
if strings.HasSuffix(reqPathForPattern, matchPattern[1:]) {
return true
}
continue
}
// special case: last character is a wildcard,
// treat it as a fast prefix match
if strings.HasSuffix(matchPattern, "*") {
if strings.HasPrefix(reqPathForPattern, matchPattern[:len(matchPattern)-1]) {
return true
}
continue
}
}
// at last, use globular matching, which also is exact matching
// if there are no glob/wildcard chars; we ignore the error here
// because we can't handle it anyway
matches, _ := path.Match(matchPattern, reqPathForPattern)
if matches {
return true
}
}
return false
}
func (MatchPath) matchPatternWithEscapeSequence(escapedPath, matchPath string) bool {
// We would just compare the pattern against r.URL.Path,
// but the pattern contains %, indicating that we should
// compare at least some part of the path in raw/escaped
// space, not normalized space; so we build the string we
// will compare against by adding the normalized parts
// of the path, then switching to the escaped parts where
// the pattern hints to us wherever % is present.
var sb strings.Builder
// iterate the pattern and escaped path in lock-step;
// increment iPattern every time we consume a char from the pattern,
// increment iPath every time we consume a char from the path;
// iPattern and iPath are our cursors/iterator positions for each string
var iPattern, iPath int
for {
if iPattern >= len(matchPath) || iPath >= len(escapedPath) {
break
}
// get the next character from the request path
pathCh := string(escapedPath[iPath])
var escapedPathCh string
// normalize (decode) escape sequences
if pathCh == "%" && len(escapedPath) >= iPath+3 {
// hold onto this in case we find out the intent is to match in escaped space here;
// we lowercase it even though technically the spec says: "For consistency, URI
// producers and normalizers should use uppercase hexadecimal digits for all percent-
// encodings" (RFC 3986 section 2.1) - we lowercased the matcher pattern earlier in
// provisioning so we do the same here to gain case-insensitivity in equivalence;
// besides, this string is never shown visibly
escapedPathCh = strings.ToLower(escapedPath[iPath : iPath+3])
var err error
pathCh, err = url.PathUnescape(escapedPathCh)
if err != nil {
// should be impossible unless EscapedPath() is giving us an invalid sequence!
return false
}
iPath += 2 // escape sequence is 2 bytes longer than normal char
}
// now get the next character from the pattern
normalize := true
switch matchPath[iPattern] {
case '%':
// escape sequence
// if not a wildcard ("%*"), compare literally; consume next two bytes of pattern
if len(matchPath) >= iPattern+3 && matchPath[iPattern+1] != '*' {
sb.WriteString(escapedPathCh)
iPath++
iPattern += 2
break
}
// escaped wildcard sequence; consume next byte only ('*')
iPattern++
normalize = false
fallthrough
case '*':
// wildcard, so consume until next matching character
remaining := escapedPath[iPath:]
until := len(escapedPath) - iPath // go until end of string...
if iPattern < len(matchPath)-1 { // ...unless the * is not at the end
nextCh := matchPath[iPattern+1]
until = strings.IndexByte(remaining, nextCh)
if until == -1 {
// terminating char of wildcard span not found, so definitely no match
return false
}
}
if until == 0 {
// empty span; nothing to add on this iteration
break
}
next := remaining[:until]
if normalize {
var err error
next, err = url.PathUnescape(next)
if err != nil {
return false // should be impossible anyway
}
}
sb.WriteString(next)
iPath += until
default:
sb.WriteString(pathCh)
iPath++
}
iPattern++
}
// we can now treat rawpath globs (%*) as regular globs (*)
matchPath = strings.ReplaceAll(matchPath, "%*", "*")
// ignore error here because we can't handle it anyway=
matches, _ := path.Match(matchPath, sb.String())
return matches
}

269
pkg/utils/directives/match_test.go Normal file
View File

@@ -0,0 +1,269 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package directives
import (
"context"
"net/http"
"net/url"
"testing"
)
func TestPathMatcher(t *testing.T) {
for i, tc := range []struct {
match MatchPath // not URI-encoded because not parsing from a URI
input string // should be valid URI encoding (escaped) since it will become part of a request
expect bool
provisionErr bool
}{
{
match: MatchPath{},
input: "/",
expect: false,
},
{
match: MatchPath{"/"},
input: "/",
expect: true,
},
{
match: MatchPath{"/foo/bar"},
input: "/",
expect: false,
},
{
match: MatchPath{"/foo/bar"},
input: "/foo/bar",
expect: true,
},
{
match: MatchPath{"/foo/bar/"},
input: "/foo/bar",
expect: false,
},
{
match: MatchPath{"/foo/bar/"},
input: "/foo/bar/",
expect: true,
},
{
match: MatchPath{"/foo/bar/", "/other"},
input: "/other/",
expect: false,
},
{
match: MatchPath{"/foo/bar/", "/other"},
input: "/other",
expect: true,
},
{
match: MatchPath{"*.ext"},
input: "/foo/bar.ext",
expect: true,
},
{
match: MatchPath{"*.php"},
input: "/index.PHP",
expect: true,
},
{
match: MatchPath{"*.ext"},
input: "/foo/bar.ext",
expect: true,
},
{
match: MatchPath{"/foo/*/baz"},
input: "/foo/bar/baz",
expect: true,
},
{
match: MatchPath{"/foo/*/baz/bam"},
input: "/foo/bar/bam",
expect: false,
},
{
match: MatchPath{"*substring*"},
input: "/foo/substring/bar.txt",
expect: true,
},
{
match: MatchPath{"/foo"},
input: "/foo/bar",
expect: false,
},
{
match: MatchPath{"/foo"},
input: "/foo/bar",
expect: false,
},
{
match: MatchPath{"/foo"},
input: "/FOO",
expect: true,
},
{
match: MatchPath{"/foo*"},
input: "/FOOOO",
expect: true,
},
{
match: MatchPath{"/foo/bar.txt"},
input: "/foo/BAR.txt",
expect: true,
},
{
match: MatchPath{"/foo*"},
input: "//foo/bar",
expect: true,
},
{
match: MatchPath{"/foo"},
input: "//foo",
expect: true,
},
{
match: MatchPath{"//foo"},
input: "/foo",
expect: false,
},
{
match: MatchPath{"//foo"},
input: "//foo",
expect: true,
},
{
match: MatchPath{"/foo//*"},
input: "/foo//bar",
expect: true,
},
{
match: MatchPath{"/foo//*"},
input: "/foo/%2Fbar",
expect: true,
},
{
match: MatchPath{"/foo/%2F*"},
input: "/foo//bar",
expect: false,
},
{
match: MatchPath{"/foo//bar"},
input: "/foo//bar",
expect: true,
},
{
match: MatchPath{"/foo/*//bar"},
input: "/foo///bar",
expect: true,
},
{
match: MatchPath{"/foo/%*//bar"},
input: "/foo///bar",
expect: true,
},
{
match: MatchPath{"/foo/%*//bar"},
input: "/foo//%2Fbar",
expect: true,
},
{
match: MatchPath{"/foo*"},
input: "/%2F/foo",
expect: true,
},
{
match: MatchPath{"*"},
input: "/",
expect: true,
},
{
match: MatchPath{"*"},
input: "/foo/bar",
expect: true,
},
{
match: MatchPath{"**"},
input: "/",
expect: true,
},
{
match: MatchPath{"**"},
input: "/foo/bar",
expect: true,
},
// notice these next three test cases are the same normalized path but are written differently
{
match: MatchPath{"/%25@.txt"},
input: "/%25@.txt",
expect: true,
},
{
match: MatchPath{"/%25@.txt"},
input: "/%25%40.txt",
expect: true,
},
{
match: MatchPath{"/%25%40.txt"},
input: "/%25%40.txt",
expect: true,
},
{
match: MatchPath{"/bands/*/*"},
input: "/bands/AC%2FDC/T.N.T",
expect: false, // because * operates in normalized space
},
{
match: MatchPath{"/bands/%*/%*"},
input: "/bands/AC%2FDC/T.N.T",
expect: true,
},
{
match: MatchPath{"/bands/%*/%*"},
input: "/bands/AC/DC/T.N.T",
expect: false,
},
{
match: MatchPath{"/bands/%*"},
input: "/bands/AC/DC",
expect: false, // not a suffix match
},
{
match: MatchPath{"/bands/%*"},
input: "/bands/AC%2FDC",
expect: true,
},
{
match: MatchPath{"/foo%2fbar/baz"},
input: "/foo%2Fbar/baz",
expect: true,
},
{
match: MatchPath{"/foo%2fbar/baz"},
input: "/foo/bar/baz",
expect: false,
},
{
match: MatchPath{"/foo/bar/baz"},
input: "/foo%2fbar/baz",
expect: true,
},
} {
u, err := url.ParseRequestURI(tc.input)
if err != nil {
t.Fatalf("Test %d (%v): Invalid request URI (should be rejected by Go's HTTP server): %v", i, tc.input, err)
}
req := &http.Request{URL: u}
repl := NewReplacer()
ctx := context.WithValue(req.Context(), ReplacerCtxKey, repl)
req = req.WithContext(ctx)
actual := tc.match.Match(req)
if actual != tc.expect {
t.Errorf("Test %d %v: Expected %t, got %t for '%s'", i, tc.match, tc.expect, actual, tc.input)
continue
}
}
}

338
pkg/utils/directives/replace.go Normal file
View File

@@ -0,0 +1,338 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package directives
import (
"fmt"
"net/http"
"os"
"path/filepath"
"regexp"
"runtime"
"strconv"
"strings"
"time"
)
// Following code copied from github.com/caddyserver/caddy/modules/caddyhttp/rewrite/rewrite.go
type substrReplacer struct {
Find string `json:"find,omitempty"`
Replace string `json:"replace,omitempty"`
Limit int `json:"limit,omitempty"`
}
func (rep substrReplacer) do(r *http.Request, repl *Replacer) {
if rep.Find == "" {
return
}
lim := rep.Limit
if lim == 0 {
lim = -1
}
find := repl.ReplaceAll(rep.Find, "")
replace := repl.ReplaceAll(rep.Replace, "")
mergeSlashes := !strings.Contains(rep.Find, "//")
changePath(r, func(pathOrRawPath string) string {
return strings.Replace(CleanPath(pathOrRawPath, mergeSlashes), find, replace, lim)
})
r.URL.RawQuery = strings.Replace(r.URL.RawQuery, find, replace, lim)
}
type regexReplacer struct {
Find string `json:"find,omitempty"`
Replace string `json:"replace,omitempty"`
re *regexp.Regexp
}
func (rep regexReplacer) do(r *http.Request, repl *Replacer) {
if rep.Find == "" || rep.re == nil {
return
}
replace := repl.ReplaceAll(rep.Replace, "")
changePath(r, func(pathOrRawPath string) string {
return rep.re.ReplaceAllString(pathOrRawPath, replace)
})
}
func NewReplacer() *Replacer {
rep := &Replacer{
static: make(map[string]any),
}
rep.providers = []ReplacerFunc{
globalDefaultReplacements,
rep.fromStatic,
}
return rep
}
type Replacer struct {
providers []ReplacerFunc
static map[string]any
}
func (r *Replacer) Map(mapFunc ReplacerFunc) {
r.providers = append(r.providers, mapFunc)
}
func (r *Replacer) Set(variable string, value any) {
r.static[variable] = value
}
func (r *Replacer) Get(variable string) (any, bool) {
for _, mapFunc := range r.providers {
if val, ok := mapFunc(variable); ok {
return val, true
}
}
return nil, false
}
func (r *Replacer) GetString(variable string) (string, bool) {
s, found := r.Get(variable)
return ToString(s), found
}
func (r *Replacer) Delete(variable string) {
delete(r.static, variable)
}
func (r *Replacer) fromStatic(key string) (any, bool) {
val, ok := r.static[key]
return val, ok
}
func (r *Replacer) ReplaceOrErr(input string, errOnEmpty, errOnUnknown bool) (string, error) {
return r.replace(input, "", false, errOnEmpty, errOnUnknown, nil)
}
func (r *Replacer) ReplaceKnown(input, empty string) string {
out, _ := r.replace(input, empty, false, false, false, nil)
return out
}
func (r *Replacer) ReplaceAll(input, empty string) string {
out, _ := r.replace(input, empty, true, false, false, nil)
return out
}
func (r *Replacer) ReplaceFunc(input string, f ReplacementFunc) (string, error) {
return r.replace(input, "", true, false, false, f)
}
func (r *Replacer) replace(input, empty string,
treatUnknownAsEmpty, errOnEmpty, errOnUnknown bool,
f ReplacementFunc,
) (string, error) {
if !strings.Contains(input, string(phOpen)) {
return input, nil
}
var sb strings.Builder
// it is reasonable to assume that the output
// will be approximately as long as the input
sb.Grow(len(input))
// iterate the input to find each placeholder
var lastWriteCursor int
// fail fast if too many placeholders are unclosed
var unclosedCount int
scan:
for i := 0; i < len(input); i++ {
// check for escaped braces
if i > 0 && input[i-1] == phEscape && (input[i] == phClose || input[i] == phOpen) {
sb.WriteString(input[lastWriteCursor : i-1])
lastWriteCursor = i
continue
}
if input[i] != phOpen {
continue
}
// our iterator is now on an unescaped open brace (start of placeholder)
// too many unclosed placeholders in absolutely ridiculous input can be extremely slow (issue #4170)
if unclosedCount > 100 {
return "", fmt.Errorf("too many unclosed placeholders")
}
// find the end of the placeholder
end := strings.Index(input[i:], string(phClose)) + i
if end < i {
unclosedCount++
continue
}
// if necessary look for the first closing brace that is not escaped
for end > 0 && end < len(input)-1 && input[end-1] == phEscape {
nextEnd := strings.Index(input[end+1:], string(phClose))
if nextEnd < 0 {
unclosedCount++
continue scan
}
end += nextEnd + 1
}
// write the substring from the last cursor to this point
sb.WriteString(input[lastWriteCursor:i])
// trim opening bracket
key := input[i+1 : end]
// try to get a value for this key, handle empty values accordingly
val, found := r.Get(key)
if !found {
// placeholder is unknown (unrecognized); handle accordingly
if errOnUnknown {
return "", fmt.Errorf("unrecognized placeholder %s%s%s",
string(phOpen), key, string(phClose))
} else if !treatUnknownAsEmpty {
// if treatUnknownAsEmpty is true, we'll handle an empty
// val later; so only continue otherwise
lastWriteCursor = i
continue
}
}
// apply any transformations
if f != nil {
var err error
val, err = f(key, val)
if err != nil {
return "", err
}
}
valStr := ToString(val)
if valStr == "" {
if errOnEmpty {
return "", fmt.Errorf("evaluated placeholder %s%s%s is empty",
string(phOpen), key, string(phClose))
} else if empty != "" {
sb.WriteString(empty)
}
} else {
sb.WriteString(valStr)
}
i = end
lastWriteCursor = i + 1
}
sb.WriteString(input[lastWriteCursor:])
return sb.String(), nil
}
func ToString(val any) string {
switch v := val.(type) {
case nil:
return ""
case string:
return v
case fmt.Stringer:
return v.String()
case error:
return v.Error()
case byte:
return string(v)
case []byte:
return string(v)
case []rune:
return string(v)
case int:
return strconv.Itoa(v)
case int32:
return strconv.Itoa(int(v))
case int64:
return strconv.Itoa(int(v))
case uint:
return strconv.Itoa(int(v))
case uint32:
return strconv.Itoa(int(v))
case uint64:
return strconv.Itoa(int(v))
case float32:
return strconv.FormatFloat(float64(v), 'f', -1, 32)
case float64:
return strconv.FormatFloat(v, 'f', -1, 64)
case bool:
if v {
return "true"
}
return "false"
default:
return fmt.Sprintf("%+v", v)
}
}
type ReplacerFunc func(key string) (any, bool)
func globalDefaultReplacements(key string) (any, bool) {
const envPrefix = "env."
if strings.HasPrefix(key, envPrefix) {
return os.Getenv(key[len(envPrefix):]), true
}
switch key {
case "system.hostname":
name, _ := os.Hostname()
return name, true
case "system.slash":
return string(filepath.Separator), true
case "system.os":
return runtime.GOOS, true
case "system.wd":
// OK if there is an error; just return empty string
wd, _ := os.Getwd()
return wd, true
case "system.arch":
return runtime.GOARCH, true
case "time.now":
return nowFunc(), true
case "time.now.http":
return nowFunc().UTC().Format(http.TimeFormat), true
case "time.now.common_log":
return nowFunc().Format("02/Jan/2006:15:04:05 -0700"), true
case "time.now.year":
return strconv.Itoa(nowFunc().Year()), true
case "time.now.unix":
return strconv.FormatInt(nowFunc().Unix(), 10), true
case "time.now.unix_ms":
return strconv.FormatInt(nowFunc().UnixNano()/int64(time.Millisecond), 10), true
}
return nil, false
}
// ReplacementFunc is a function that is called when a
// replacement is being performed. It receives the
// variable (i.e. placeholder name) and the value that
// will be the replacement, and returns the value that
// will actually be the replacement, or an error. Note
// that errors are sometimes ignored by replacers.
type ReplacementFunc func(variable string, val any) (any, error)
// nowFunc is a variable so tests can change it
// in order to obtain a deterministic time.
var nowFunc = time.Now
type ContextKey string
// ReplacerCtxKey is the context key for a replacer.
const ReplacerCtxKey ContextKey = "replacer"
const phOpen, phClose, phEscape = '{', '}', '\\'

472
pkg/utils/directives/rewrite.go Normal file
View File

@@ -0,0 +1,472 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package directives
import (
"context"
"fmt"
"net/http"
"net/url"
"path"
"regexp"
"strconv"
"strings"
)
// Following code copied from github.com/caddyserver/caddy/modules/caddyhttp/rewrite/rewrite.go
type Rewrite struct {
Method string `json:"method,omitempty"`
URI string `json:"uri,omitempty"`
StripPathPrefix string `json:"strip_path_prefix,omitempty"`
StripPathSuffix string `json:"strip_path_suffix,omitempty"`
URISubstring []substrReplacer `json:"uri_substring,omitempty"`
PathRegexp []*regexReplacer `json:"path_regexp,omitempty"`
}
func (r *Rewrite) Rewrite(req *http.Request, repl *Replacer) bool {
oldMethod := req.Method
oldURI := req.RequestURI
// method
if r.Method != "" {
req.Method = strings.ToUpper(repl.ReplaceAll(r.Method, ""))
}
// uri (path, query string and... fragment, because why not)
if uri := r.URI; uri != "" {
// find the bounds of each part of the URI that exist
pathStart, qsStart, fragStart := -1, -1, -1
pathEnd, qsEnd := -1, -1
loop:
for i, ch := range uri {
switch {
case ch == '?' && qsStart < 0:
pathEnd, qsStart = i, i+1
case ch == '#' && fragStart < 0: // everything after fragment is fragment (very clear in RFC 3986 section 4.2)
if qsStart < 0 {
pathEnd = i
} else {
qsEnd = i
}
fragStart = i + 1
break loop
case pathStart < 0 && qsStart < 0:
pathStart = i
}
}
if pathStart >= 0 && pathEnd < 0 {
pathEnd = len(uri)
}
if qsStart >= 0 && qsEnd < 0 {
qsEnd = len(uri)
}
// isolate the three main components of the URI
var path, query, frag string
if pathStart > -1 {
path = uri[pathStart:pathEnd]
}
if qsStart > -1 {
query = uri[qsStart:qsEnd]
}
if fragStart > -1 {
frag = uri[fragStart:]
}
// build components which are specified, and store them
// in a temporary variable so that they all read the
// same version of the URI
var newPath, newQuery, newFrag string
if path != "" {
// replace the `path` placeholder to escaped path
pathPlaceholder := "{http.request.uri.path}"
if strings.Contains(path, pathPlaceholder) {
path = strings.ReplaceAll(path, pathPlaceholder, req.URL.EscapedPath())
}
newPath = repl.ReplaceAll(path, "")
}
// before continuing, we need to check if a query string
// snuck into the path component during replacements
if before, after, found := strings.Cut(newPath, "?"); found {
// recompute; new path contains a query string
var injectedQuery string
newPath, injectedQuery = before, after
// don't overwrite explicitly-configured query string
if query == "" {
query = injectedQuery
}
}
if query != "" {
newQuery = buildQueryString(query, repl)
}
if frag != "" {
newFrag = repl.ReplaceAll(frag, "")
}
// update the URI with the new components
// only after building them
if pathStart >= 0 {
if path, err := url.PathUnescape(newPath); err != nil {
req.URL.Path = newPath
} else {
req.URL.Path = path
}
}
if qsStart >= 0 {
req.URL.RawQuery = newQuery
}
if fragStart >= 0 {
req.URL.Fragment = newFrag
}
}
// strip path prefix or suffix
if r.StripPathPrefix != "" {
prefix := repl.ReplaceAll(r.StripPathPrefix, "")
mergeSlashes := !strings.Contains(prefix, "//")
changePath(req, func(escapedPath string) string {
escapedPath = CleanPath(escapedPath, mergeSlashes)
return trimPathPrefix(escapedPath, prefix)
})
}
if r.StripPathSuffix != "" {
suffix := repl.ReplaceAll(r.StripPathSuffix, "")
mergeSlashes := !strings.Contains(suffix, "//")
changePath(req, func(escapedPath string) string {
escapedPath = CleanPath(escapedPath, mergeSlashes)
return reverse(trimPathPrefix(reverse(escapedPath), reverse(suffix)))
})
}
// substring replacements in URI
for _, rep := range r.URISubstring {
rep.do(req, repl)
}
// regular expression replacements on the path
for _, rep := range r.PathRegexp {
rep.do(req, repl)
}
// update the encoded copy of the URI
req.RequestURI = req.URL.RequestURI()
// return true if anything changed
return req.Method != oldMethod || req.RequestURI != oldURI
}
func buildQueryString(qs string, repl *Replacer) string {
var sb strings.Builder
// first component must be key, which is the same
// as if we just wrote a value in previous iteration
wroteVal := true
for len(qs) > 0 {
// determine the end of this component, which will be at
// the next equal sign or ampersand, whichever comes first
nextEq, nextAmp := strings.Index(qs, "="), strings.Index(qs, "&")
ampIsNext := nextAmp >= 0 && (nextAmp < nextEq || nextEq < 0)
end := len(qs) // assume no delimiter remains...
if ampIsNext {
end = nextAmp // ...unless ampersand is first...
} else if nextEq >= 0 && (nextEq < nextAmp || nextAmp < 0) {
end = nextEq // ...or unless equal is first.
}
// consume the component and write the result
comp := qs[:end]
comp, _ = repl.ReplaceFunc(comp, func(name string, val any) (any, error) {
if name == "http.request.uri.query" && wroteVal {
return val, nil // already escaped
}
var valStr string
switch v := val.(type) {
case string:
valStr = v
case fmt.Stringer:
valStr = v.String()
case int:
valStr = strconv.Itoa(v)
default:
valStr = fmt.Sprintf("%+v", v)
}
return url.QueryEscape(valStr), nil
})
if end < len(qs) {
end++ // consume delimiter
}
qs = qs[end:]
// if previous iteration wrote a value,
// that means we are writing a key
if wroteVal {
if sb.Len() > 0 && len(comp) > 0 {
sb.WriteRune('&')
}
} else {
sb.WriteRune('=')
}
sb.WriteString(comp)
// remember for the next iteration that we just wrote a value,
// which means the next iteration MUST write a key
wroteVal = ampIsNext
}
return sb.String()
}
func trimPathPrefix(escapedPath, prefix string) string {
var iPath, iPrefix int
for {
if iPath >= len(escapedPath) || iPrefix >= len(prefix) {
break
}
prefixCh := prefix[iPrefix]
ch := string(escapedPath[iPath])
if ch == "%" && prefixCh != '%' && len(escapedPath) >= iPath+3 {
var err error
ch, err = url.PathUnescape(escapedPath[iPath : iPath+3])
if err != nil {
// should be impossible unless EscapedPath() is returning invalid values!
return escapedPath
}
iPath += 2
}
// prefix comparisons are case-insensitive to consistency with
// path matcher, which is case-insensitive for good reasons
if !strings.EqualFold(ch, string(prefixCh)) {
return escapedPath
}
iPath++
iPrefix++
}
// if we iterated through the entire prefix, we found it, so trim it
if iPath >= len(prefix) {
return escapedPath[iPath:]
}
// otherwise we did not find the prefix
return escapedPath
}
func reverse(s string) string {
r := []rune(s)
for i, j := 0, len(r)-1; i < len(r)/2; i, j = i+1, j-1 {
r[i], r[j] = r[j], r[i]
}
return string(r)
}
func changePath(req *http.Request, newVal func(pathOrRawPath string) string) {
req.URL.RawPath = newVal(req.URL.EscapedPath())
if p, err := url.PathUnescape(req.URL.RawPath); err == nil && p != "" {
req.URL.Path = p
} else {
req.URL.Path = newVal(req.URL.Path)
}
// RawPath is only set if it's different from the normalized Path (std lib)
if req.URL.RawPath == req.URL.Path {
req.URL.RawPath = ""
}
}
func CleanPath(p string, collapseSlashes bool) string {
if collapseSlashes {
return cleanPath(p)
}
// insert an invalid/impossible URI character into each two consecutive
// slashes to expand empty path segments; then clean the path as usual,
// and then remove the remaining temporary characters.
const tmpCh = 0xff
var sb strings.Builder
for i, ch := range p {
if ch == '/' && i > 0 && p[i-1] == '/' {
sb.WriteByte(tmpCh)
}
sb.WriteRune(ch)
}
halfCleaned := cleanPath(sb.String())
halfCleaned = strings.ReplaceAll(halfCleaned, string([]byte{tmpCh}), "")
return halfCleaned
}
// cleanPath does path.Clean(p) but preserves any trailing slash.
func cleanPath(p string) string {
cleaned := path.Clean(p)
if cleaned != "/" && strings.HasSuffix(p, "/") {
cleaned = cleaned + "/"
}
return cleaned
}
type RewriteRule struct {
Match MatchPath
Rewrite Rewrite
}
func (rr *RewriteRule) Exec(req *http.Request) (change bool, err error) {
var repl *Replacer
defer func() {
if panicErr := recover(); panicErr != nil {
err = fmt.Errorf("RewriteRule Err %v", panicErr)
}
}()
replCtx := req.Context().Value(ReplacerCtxKey)
if replCtx == nil || replCtx.(*Replacer) == nil {
repl := NewReplacer()
repl.Set("query", req.URL.RawQuery)
repl.Set("path", req.URL.Path)
ctx := context.WithValue(req.Context(), ReplacerCtxKey, repl)
req = req.WithContext(ctx)
} else {
repl = replCtx.(*Replacer)
}
if rr.Match == nil || rr.Match.Match(req) {
return rr.Rewrite.Rewrite(req, repl), nil
}
return
}
type DirectiveFilter func(rr *[]RewriteRule, expr []string, exprLen int)
type WithDirectiveFilter func(wf *[]DirectiveFilter)
func NewRewriteRulesWithOptions(rules []string, directiveFilters ...WithDirectiveFilter) []RewriteRule {
var rewriteRules = make([]RewriteRule, 0, 1)
if rules == nil {
return rewriteRules
}
var filter = make([]DirectiveFilter, 0, 1)
// inject directiveFilter for filter rewrite/replace/path_regexp
for _, directiveFilter := range directiveFilters {
directiveFilter(&filter)
}
for _, rule := range rules {
expr := strings.Split(rule, " ")
exprLen := len(expr)
for _, directiveFilter := range filter {
directiveFilter(&rewriteRules, expr, exprLen)
}
}
return rewriteRules
}
func WithRewriteFilter(df *[]DirectiveFilter) {
filterFn := func(rr *[]RewriteRule, expr []string, exprLen int) {
if exprLen >= 2 {
rewrite := Rewrite{
URI: expr[1],
}
matchRule := []string{
expr[0],
}
*rr = append(*rr, RewriteRule{
matchRule,
rewrite,
})
}
}
*df = append(*df, filterFn)
}
func WithReplaceFilter(df *[]DirectiveFilter) {
filterFn := func(rr *[]RewriteRule, expr []string, exprLen int) {
if exprLen >= 2 {
rewrite := Rewrite{
URISubstring: []substrReplacer{
{
Find: expr[0],
Replace: expr[1],
Limit: 0,
},
},
}
*rr = append(*rr, RewriteRule{
nil,
rewrite,
})
}
}
*df = append(*df, filterFn)
}
func WithPathRegexpFilter(df *[]DirectiveFilter) {
filterFn := func(rr *[]RewriteRule, expr []string, exprLen int) {
if exprLen >= 2 {
re, err := regexp.Compile(expr[0])
if err != nil {
return
}
rewrite := Rewrite{
PathRegexp: []*regexReplacer{
{
Find: expr[0],
Replace: expr[1],
re: re,
},
},
}
*rr = append(*rr, RewriteRule{
nil,
rewrite,
})
}
}
*df = append(*df, filterFn)
}
func WithStripPrefixFilter(df *[]DirectiveFilter) {
filterFn := func(rr *[]RewriteRule, expr []string, exprLen int) {
if exprLen >= 1 {
*rr = append(*rr, RewriteRule{
nil,
Rewrite{
StripPathPrefix: expr[0],
},
})
}
}
*df = append(*df, filterFn)
}
func WithStripSuffixFilter(df *[]DirectiveFilter) {
filterFn := func(rr *[]RewriteRule, expr []string, exprLen int) {
if exprLen >= 1 {
*rr = append(*rr, RewriteRule{
nil,
Rewrite{
StripPathSuffix: expr[0],
},
})
}
}
*df = append(*df, filterFn)
}
func HandlerRequest(req *http.Request, rules []string, directiveFilters ...WithDirectiveFilter) error {
rewriteRules := NewRewriteRulesWithOptions(rules, directiveFilters...)
for _, rewriteRule := range rewriteRules {
if _, err := rewriteRule.Exec(req); err != nil {
return err
}
}
return nil
}

225
pkg/utils/directives/rewrite_test.go Normal file
View File

@@ -0,0 +1,225 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package directives
import (
"context"
"net/http"
"net/url"
"regexp"
"testing"
)
func TestRewrite(t *testing.T) {
repl := NewReplacer()
for i, tc := range []struct {
input, expect *http.Request
rule Rewrite
}{
{
rule: Rewrite{StripPathPrefix: "/api"},
input: newRequest(t, "GET", "/api"),
expect: newRequest(t, "GET", "/"),
},
{
rule: Rewrite{StripPathSuffix: ".html"},
input: newRequest(t, "GET", "/index.html"),
expect: newRequest(t, "GET", "/index"),
},
{
rule: Rewrite{URISubstring: []substrReplacer{
{
Find: "/docs/",
Replace: "/v1/docs/",
Limit: 0,
},
}},
input: newRequest(t, "GET", "/docs/"),
expect: newRequest(t, "GET", "/v1/docs/"),
},
{
rule: Rewrite{PathRegexp: []*regexReplacer{
{
Find: "/{2,}",
Replace: "/",
},
}},
input: newRequest(t, "GET", "/doc//readme.md"),
expect: newRequest(t, "GET", "/doc/readme.md"),
},
} {
// copy the original input just enough so that we can
// compare it after the rewrite to see if it changed
urlCopy := *tc.input.URL
originalInput := &http.Request{
Method: tc.input.Method,
RequestURI: tc.input.RequestURI,
URL: &urlCopy,
}
// populate the replacer just enough for our tests
repl.Set("http.request.uri", tc.input.RequestURI)
repl.Set("http.request.uri.path", tc.input.URL.Path)
repl.Set("http.request.uri.query", tc.input.URL.RawQuery)
for _, rep := range tc.rule.PathRegexp {
re, err := regexp.Compile(rep.Find)
if err != nil {
t.Fatal(err)
}
rep.re = re
}
changed := tc.rule.Rewrite(tc.input, repl)
if expected, actual := !reqEqual(originalInput, tc.input), changed; expected != actual {
t.Errorf("Test %d: Expected changed=%t but was %t", i, expected, actual)
}
if tc.rule.StripPathPrefix != "" {
t.Logf("Test UriRule \"uri strip_prefix %v\" ==> rewrite \"%v\" to \"%v\"", tc.rule.StripPathPrefix, originalInput.URL, tc.input.URL)
} else if tc.rule.StripPathSuffix != "" {
t.Logf("Test UriRule \"uri strip_suffix %v\" ==> rewrite \"%v\" to \"%v\"", tc.rule.StripPathSuffix, originalInput.URL, tc.input.URL)
} else if tc.rule.URISubstring != nil {
t.Logf("Test UriRule \"uri replace %s %s\" ==> rewrite \"%v\" to \"%v\"", tc.rule.URISubstring[0].Find, tc.rule.URISubstring[0].Replace, originalInput.URL, tc.input.URL)
} else if tc.rule.PathRegexp != nil {
t.Logf("Test UriRule \"uri path_regexp %s %s\" ==> rewrite \"%v\" to \"%v\"", (*tc.rule.PathRegexp[0]).Find, (*tc.rule.PathRegexp[0]).Replace, originalInput.URL, tc.input.URL)
}
}
}
func TestPathRewriteRule(t *testing.T) {
for i, tc := range []struct {
rr []RewriteRule // not URI-encoded because not parsing from a URI
input string // should be valid URI encoding (escaped) since it will become part of a request
expect bool
provisionErr bool
}{
{
rr: NewRewriteRulesWithOptions([]string{
"* /foo.html",
}, WithRewriteFilter),
input: "/",
expect: true,
},
{
rr: NewRewriteRulesWithOptions([]string{
"/api/* ?a=b",
}, WithRewriteFilter),
input: "/api/abc",
expect: true,
},
{
rr: NewRewriteRulesWithOptions([]string{
"/api/* ?{query}&a=b",
}, WithRewriteFilter),
input: "/api/abc",
expect: true,
},
{
rr: NewRewriteRulesWithOptions([]string{
"* /index.php?{query}&p={path}",
}, WithRewriteFilter),
input: "/foo/bar",
expect: true,
},
{
rr: NewRewriteRulesWithOptions([]string{
"/api",
}, WithStripPrefixFilter),
input: "/api/v1",
expect: true,
},
{
rr: NewRewriteRulesWithOptions([]string{
".html",
}, WithStripSuffixFilter),
input: "/index.html",
expect: true,
},
{
rr: NewRewriteRulesWithOptions([]string{
"/docs/ /v1/docs/",
}, WithReplaceFilter),
input: "/docs/go",
expect: true,
},
{
rr: NewRewriteRulesWithOptions([]string{
"/{2,} /",
}, WithPathRegexpFilter),
input: "/doc//readme.md",
expect: true,
},
} {
u, err := url.ParseRequestURI(tc.input)
if err != nil {
t.Fatalf("Test %d (%v): Invalid request URI (should be rejected by Go's HTTP server): %v", i, tc.input, err)
}
req := &http.Request{URL: u}
repl := NewReplacer()
repl.Set("query", req.URL.RawQuery)
repl.Set("path", req.URL.Path)
//t.Logf("Init ENV with: {\"query\":\"%v\", \"path\": \"%v\"}", req.URL.RawQuery, req.URL.Path)
ctx := context.WithValue(req.Context(), ReplacerCtxKey, repl)
req = req.WithContext(ctx)
for _, r := range tc.rr {
oldRUL := req.URL.Path
actual, err := r.Exec(req)
if err != nil {
t.Errorf("Test RewriteRule \"rewrite %v %v\" ==> Err %v", r.Match[0], r.Rewrite.URI, err)
continue
}
if actual != tc.expect {
t.Errorf("Test RewriteRule \"rewrite %v %v\" ==> Expected %t, got %t for '%s'", r.Match[0], r.Rewrite.URI, tc.expect, actual, tc.input)
continue
}
if r.Rewrite.StripPathPrefix != "" {
t.Logf("Test RewriteRule \"strip_prefix %v\" ==> rewrite \"%v\" to \"%v\"", r.Rewrite.StripPathPrefix, oldRUL, req.URL)
} else if r.Rewrite.StripPathSuffix != "" {
t.Logf("Test RewriteRule \"strip_suffix %v\" ==> rewrite \"%v\" to \"%v\"", r.Rewrite.StripPathSuffix, oldRUL, req.URL)
} else if r.Rewrite.URISubstring != nil {
t.Logf("Test RewriteRule \"replace %s %s\" ==> rewrite \"%v\" to \"%v\"", r.Rewrite.URISubstring[0].Find, r.Rewrite.URISubstring[0].Replace, oldRUL, req.URL)
} else if r.Rewrite.PathRegexp != nil {
t.Logf("Test RewriteRule \"path_regexp %s %s\" ==> rewrite \"%v\" to \"%v\"", (*r.Rewrite.PathRegexp[0]).Find, (*r.Rewrite.PathRegexp[0]).Replace, oldRUL, req.URL)
} else if r.Rewrite.URI != "" {
t.Logf("Test RewriteRule \"rewrite %s %s\" ==> rewrite \"%v\" to \"%v\"", r.Match[0], r.Rewrite.URI, oldRUL, req.URL)
}
}
}
}
func newRequest(t *testing.T, method, uri string) *http.Request {
req, err := http.NewRequest(method, uri, nil)
if err != nil {
t.Fatalf("error creating request: %v", err)
}
req.RequestURI = req.URL.RequestURI() // simulate incoming request
return req
}
func reqEqual(r1, r2 *http.Request) bool {
if r1.Method != r2.Method {
return false
}
if r1.RequestURI != r2.RequestURI {
return false
}
if (r1.URL == nil && r2.URL != nil) || (r1.URL != nil && r2.URL == nil) {
return false
}
if r1.URL == nil && r2.URL == nil {
return true
}
return r1.URL.Scheme == r2.URL.Scheme &&
r1.URL.Host == r2.URL.Host &&
r1.URL.Path == r2.URL.Path &&
r1.URL.RawPath == r2.URL.RawPath &&
r1.URL.RawQuery == r2.URL.RawQuery &&
r1.URL.Fragment == r2.URL.Fragment
}

17
pkg/utils/esutil/esutil.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2020 KubeSphere Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
// TODO: refactor
package esutil

17
pkg/utils/esutil/esutil_test.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2020 KubeSphere Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package esutil

48
pkg/utils/hashutil/MD5.go
View File

@@ -1,48 +0,0 @@
/*
Copyright 2019 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package hashutil
import (
"encoding/hex"
"io"
"code.cloudfoundry.org/bytefmt"
"k8s.io/klog/v2"
"kubesphere.io/kubesphere/pkg/utils/readerutils"
)
func GetMD5(reader io.ReadCloser) (string, error) {
md5reader := readerutils.NewMD5Reader(reader)
data := make([]byte, bytefmt.KILOBYTE)
for {
_, err := md5reader.Read(data)
if err != nil {
if err == io.EOF {
break
}
klog.Error(err)
return "", err
}
}
err := reader.Close()
if err != nil {
return "", err
}
return hex.EncodeToString(md5reader.MD5()), nil
}

46
pkg/utils/hashutil/hashutil.go Normal file
View File

@@ -0,0 +1,46 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package hashutil
import (
"encoding/hex"
"hash/fnv"
"io"
"code.cloudfoundry.org/bytefmt"
"k8s.io/klog/v2"
"kubesphere.io/kubesphere/pkg/utils/readerutils"
)
func GetMD5(reader io.ReadCloser) (string, error) {
md5reader := readerutils.NewMD5Reader(reader)
data := make([]byte, bytefmt.KILOBYTE)
for {
_, err := md5reader.Read(data)
if err != nil {
if err == io.EOF {
break
}
klog.Error(err)
return "", err
}
}
err := reader.Close()
if err != nil {
return "", err
}
return hex.EncodeToString(md5reader.MD5()), nil
}
func FNVString(text []byte) string {
h := fnv.New64a()
if _, err := h.Write(text); err != nil {
klog.Error(err)
}
return hex.EncodeToString(h.Sum(nil))
}

17
pkg/utils/idutils/id_utils.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2018 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package idutils

17
pkg/utils/idutils/id_utils_test.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2018 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package idutils

17
pkg/utils/iputil/iputils.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2019 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package iputil

5
pkg/utils/josnpatchutil/jsonpatchutil.go
View File

@@ -1,3 +1,8 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package josnpatchutil
import (

17
pkg/utils/jsonutil/jsonutils.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2019 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package jsonutil

48
pkg/utils/k8sutil/k8sutil.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2019 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package k8sutil
@@ -20,9 +9,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
tenantv1alpha1 "kubesphere.io/api/tenant/v1alpha1"
tenantv1alpha2 "kubesphere.io/api/tenant/v1alpha2"
tenantv1beta1 "kubesphere.io/api/tenant/v1beta1"
)
// IsControlledBy returns whether the ownerReferences contains the specified resource kind
@@ -39,8 +26,8 @@ func IsControlledBy(ownerReferences []metav1.OwnerReference, kind string, name s
func RemoveWorkspaceOwnerReference(ownerReferences []metav1.OwnerReference) []metav1.OwnerReference {
tmp := make([]metav1.OwnerReference, 0)
for _, owner := range ownerReferences {
if owner.Kind != tenantv1alpha1.ResourceKindWorkspace &&
owner.Kind != tenantv1alpha2.ResourceKindWorkspaceTemplate {
if owner.Kind != tenantv1beta1.ResourceKindWorkspace &&
owner.Kind != tenantv1beta1.ResourceKindWorkspaceTemplate {
tmp = append(tmp, owner)
}
}
@@ -50,8 +37,8 @@ func RemoveWorkspaceOwnerReference(ownerReferences []metav1.OwnerReference) []me
// GetWorkspaceOwnerName return workspace kind owner name
func GetWorkspaceOwnerName(ownerReferences []metav1.OwnerReference) string {
for _, owner := range ownerReferences {
if owner.Kind == tenantv1alpha1.ResourceKindWorkspace ||
owner.Kind == tenantv1alpha2.ResourceKindWorkspaceTemplate {
if owner.Kind == tenantv1beta1.ResourceKindWorkspace ||
owner.Kind == tenantv1beta1.ResourceKindWorkspaceTemplate {
return owner.Name
}
}
@@ -72,3 +59,22 @@ func LoadKubeConfigFromBytes(kubeconfig []byte) (*rest.Config, error) {
return config, nil
}
func GetObjectMeta(obj metav1.Object) metav1.ObjectMeta {
return metav1.ObjectMeta{
Name: obj.GetName(),
GenerateName: obj.GetGenerateName(),
Namespace: obj.GetNamespace(),
UID: obj.GetUID(),
ResourceVersion: obj.GetResourceVersion(),
Generation: obj.GetGeneration(),
CreationTimestamp: obj.GetCreationTimestamp(),
DeletionTimestamp: obj.GetDeletionTimestamp(),
DeletionGracePeriodSeconds: obj.GetDeletionGracePeriodSeconds(),
Labels: obj.GetLabels(),
Annotations: obj.GetAnnotations(),
OwnerReferences: obj.GetOwnerReferences(),
Finalizers: obj.GetFinalizers(),
ManagedFields: obj.GetManagedFields(),
}
}

45
pkg/utils/k8sutil/k8sutil_test.go
View File

@@ -1,20 +1,7 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package k8sutil
@@ -22,10 +9,10 @@ import (
"reflect"
"testing"
"kubesphere.io/api/tenant/v1beta1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
tenantv1alpha1 "kubesphere.io/api/tenant/v1alpha1"
)
func TestIsControlledBy(t *testing.T) {
@@ -43,11 +30,11 @@ func TestIsControlledBy(t *testing.T) {
name: "controlled by Workspace",
args: args{
ownerReferences: []metav1.OwnerReference{{
APIVersion: tenantv1alpha1.SchemeGroupVersion.String(),
Kind: tenantv1alpha1.ResourceKindWorkspace,
APIVersion: v1beta1.SchemeGroupVersion.String(),
Kind: v1beta1.ResourceKindWorkspace,
Name: "workspace-test",
}},
kind: tenantv1alpha1.ResourceKindWorkspace,
kind: v1beta1.ResourceKindWorkspace,
},
want: true,
},
@@ -55,11 +42,11 @@ func TestIsControlledBy(t *testing.T) {
name: "controlled by workspace-test",
args: args{
ownerReferences: []metav1.OwnerReference{{
APIVersion: tenantv1alpha1.SchemeGroupVersion.String(),
Kind: tenantv1alpha1.ResourceKindWorkspace,
APIVersion: v1beta1.SchemeGroupVersion.String(),
Kind: v1beta1.ResourceKindWorkspace,
Name: "workspace-test",
}},
kind: tenantv1alpha1.ResourceKindWorkspace,
kind: v1beta1.ResourceKindWorkspace,
name: "workspace-test",
},
want: true,
@@ -68,11 +55,11 @@ func TestIsControlledBy(t *testing.T) {
name: "not controlled by workspace-test",
args: args{
ownerReferences: []metav1.OwnerReference{{
APIVersion: tenantv1alpha1.SchemeGroupVersion.String(),
Kind: tenantv1alpha1.ResourceKindWorkspace,
APIVersion: v1beta1.SchemeGroupVersion.String(),
Kind: v1beta1.ResourceKindWorkspace,
Name: "workspace",
}},
kind: tenantv1alpha1.ResourceKindWorkspace,
kind: v1beta1.ResourceKindWorkspace,
name: "workspace-test",
},
want: false,
@@ -99,8 +86,8 @@ func TestRemoveWorkspaceOwnerReference(t *testing.T) {
{
name: "remove workspace owner reference",
args: args{ownerReferences: []metav1.OwnerReference{{
APIVersion: tenantv1alpha1.SchemeGroupVersion.String(),
Kind: tenantv1alpha1.ResourceKindWorkspace,
APIVersion: v1beta1.SchemeGroupVersion.String(),
Kind: v1beta1.ResourceKindWorkspace,
Name: "workspace-test",
}}},
want: []metav1.OwnerReference{},

20
pkg/utils/k8sutil/version.go Normal file
View File

@@ -0,0 +1,20 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package k8sutil
import (
"github.com/Masterminds/semver/v3"
)
func ServeBatchV1beta1(k8sVersion *semver.Version) bool {
c, _ := semver.NewConstraint("< 1.21")
return c.Check(k8sVersion)
}
func ServeAutoscalingV2beta2(k8sVersion *semver.Version) bool {
c, _ := semver.NewConstraint("< 1.23")
return c.Check(k8sVersion)
}

92
pkg/utils/metrics/metrics.go
View File

@@ -1,92 +0,0 @@
// /*
// Copyright 2020 The KubeSphere Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// */
//
package metrics
import (
"net/http"
"sync"
"github.com/emicklei/go-restful/v3"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
apimachineryversion "k8s.io/apimachinery/pkg/version"
compbasemetrics "k8s.io/component-base/metrics"
ksVersion "kubesphere.io/kubesphere/pkg/version"
)
var (
registerOnce sync.Once
Defaults DefaultMetrics
defaultRegistry compbasemetrics.KubeRegistry
// MustRegister registers registerable metrics but uses the defaultRegistry, panic upon the first registration that causes an error
MustRegister func(...compbasemetrics.Registerable)
// Register registers a collectable metric but uses the defaultRegistry
Register func(compbasemetrics.Registerable) error
RawMustRegister func(...prometheus.Collector)
)
func init() {
compbasemetrics.BuildVersion = versionGet
defaultRegistry = compbasemetrics.NewKubeRegistry()
MustRegister = defaultRegistry.MustRegister
Register = defaultRegistry.Register
RawMustRegister = defaultRegistry.RawMustRegister
}
// DefaultMetrics installs the default prometheus metrics handler
type DefaultMetrics struct{}
// Install adds the DefaultMetrics handler
func (m DefaultMetrics) Install(c *restful.Container) {
registerOnce.Do(m.registerMetrics)
c.Handle("/kapis/metrics", Handler())
}
func (m DefaultMetrics) registerMetrics() {
//nolint:staticcheck
RawMustRegister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}))
//nolint:staticcheck
RawMustRegister(prometheus.NewGoCollector())
}
// Overwrite version.Get
func versionGet() apimachineryversion.Info {
info := ksVersion.Get()
return apimachineryversion.Info{
Major: info.GitMajor,
Minor: info.GitMinor,
GitVersion: info.GitVersion,
GitCommit: info.GitCommit,
GitTreeState: info.GitTreeState,
BuildDate: info.BuildDate,
GoVersion: info.GoVersion,
Compiler: info.Compiler,
Platform: info.Platform,
}
}
// Handler returns an HTTP handler for the DefaultGatherer. It is
// already instrumented with InstrumentHandler (using "prometheus" as handler
// name).
func Handler() http.Handler {
return promhttp.InstrumentMetricHandler(prometheus.NewRegistry(), promhttp.HandlerFor(defaultRegistry, promhttp.HandlerOpts{}))
}

17
pkg/utils/net/net.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2019 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package net

17
pkg/utils/pkiutil/pkiutil.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package pkiutil

14
pkg/utils/rbac/name.go Normal file
View File

@@ -0,0 +1,14 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package rbac
import "fmt"
const iamPrefix = "kubesphere:iam"
func RelatedK8sResourceName(name string) string {
return fmt.Sprintf("%s:%s", iamPrefix, name)
}

17
pkg/utils/readerutils/MD5Reader.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2019 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package readerutils

19
pkg/utils/reflectutils/deep.go
View File

@@ -1,20 +1,9 @@
/*
Copyright 2018 The KubeSphere Authors.
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package deep provides function deep.Equal which is like reflect.DeepEqual but
// Package reflectutils provides function deep.Equal which is like reflect.DeepEqual but
// returns a list of differences. This is helpful when comparing complex types
// like structures and maps.
package reflectutils

17
pkg/utils/reflectutils/reflect.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2018 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package reflectutils

444
pkg/utils/reposcache/repo_cahes.go
View File

@@ -1,444 +0,0 @@
// /*
// Copyright 2020 The KubeSphere Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// */
//
package reposcache
import (
"context"
"errors"
"strings"
"sync"
"k8s.io/client-go/tools/cache"
"github.com/Masterminds/semver/v3"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/klog/v2"
"kubesphere.io/api/application/v1alpha1"
"kubesphere.io/kubesphere/pkg/constants"
"kubesphere.io/kubesphere/pkg/simple/client/openpitrix/helmrepoindex"
)
const (
CategoryIndexer = "category_indexer"
CategoryAnnotationKey = "app.kubesphere.io/category"
)
var WorkDir string
func NewReposCache() ReposCache {
return &cachedRepos{
chartsInRepo: map[workspace]map[string]int{},
repos: map[string]*v1alpha1.HelmRepo{},
apps: map[string]*v1alpha1.HelmApplication{},
versions: map[string]*v1alpha1.HelmApplicationVersion{},
builtinCategoryCounts: map[string]int{},
}
}
type ReposCache interface {
AddRepo(repo *v1alpha1.HelmRepo) error
DeleteRepo(repo *v1alpha1.HelmRepo) error
UpdateRepo(old, new *v1alpha1.HelmRepo) error
GetApplication(string) (*v1alpha1.HelmApplication, bool)
GetAppVersion(string) (*v1alpha1.HelmApplicationVersion, bool, error)
GetAppVersionWithData(string) (*v1alpha1.HelmApplicationVersion, bool, error)
ListAppVersionsByAppId(appId string) (ret []*v1alpha1.HelmApplicationVersion, exists bool)
ListApplicationsInRepo(repoId string) (ret []*v1alpha1.HelmApplication, exists bool)
ListApplicationsInBuiltinRepo(selector labels.Selector) (ret []*v1alpha1.HelmApplication, exists bool)
SetCategoryIndexer(indexer cache.Indexer)
CopyCategoryCount() map[string]int
}
type workspace string
type cachedRepos struct {
sync.RWMutex
chartsInRepo map[workspace]map[string]int
// builtinCategoryCounts saves the count of every category in the built-in repo.
builtinCategoryCounts map[string]int
repos map[string]*v1alpha1.HelmRepo
apps map[string]*v1alpha1.HelmApplication
versions map[string]*v1alpha1.HelmApplicationVersion
// indexerOfHelmCtg is the indexer of HelmCategory, used to query the category id from category name.
indexerOfHelmCtg cache.Indexer
}
func (c *cachedRepos) deleteRepo(repo *v1alpha1.HelmRepo) {
if len(repo.Status.Data) == 0 {
return
}
index, err := helmrepoindex.ByteArrayToSavedIndex([]byte(repo.Status.Data))
if err != nil {
klog.Errorf("json unmarshal repo %s failed, error: %s", repo.Name, err)
return
}
klog.V(2).Infof("delete repo %s from cache", repo.Name)
repoId := repo.GetHelmRepoId()
ws := workspace(repo.GetWorkspace())
if _, exists := c.chartsInRepo[ws]; exists {
delete(c.chartsInRepo[ws], repoId)
}
delete(c.repos, repoId)
for _, app := range index.Applications {
if _, exists := c.apps[app.ApplicationId]; !exists {
continue
}
if helmrepoindex.IsBuiltInRepo(repo.Name) {
ctgId := c.apps[app.ApplicationId].Labels[constants.CategoryIdLabelKey]
if ctgId != "" {
c.builtinCategoryCounts[ctgId] -= 1
}
}
delete(c.apps, app.ApplicationId)
for _, ver := range app.Charts {
delete(c.versions, ver.ApplicationVersionId)
}
}
}
func (c *cachedRepos) DeleteRepo(repo *v1alpha1.HelmRepo) error {
c.Lock()
defer c.Unlock()
c.deleteRepo(repo)
return nil
}
// CopyCategoryCount copies the internal map to avoid `concurrent map iteration and map write`.
func (c *cachedRepos) CopyCategoryCount() map[string]int {
c.RLock()
defer c.RUnlock()
ret := make(map[string]int, len(c.builtinCategoryCounts))
for k, v := range c.builtinCategoryCounts {
ret[k] = v
}
return ret
}
func (c *cachedRepos) SetCategoryIndexer(indexer cache.Indexer) {
c.Lock()
c.indexerOfHelmCtg = indexer
c.Unlock()
}
// translateCategoryNameToId translate a category-name to a category-id.
// The caller should hold the lock
func (c *cachedRepos) translateCategoryNameToId(ctgName string) string {
if c.indexerOfHelmCtg == nil || ctgName == "" {
return v1alpha1.UncategorizedId
}
if items, err := c.indexerOfHelmCtg.ByIndex(CategoryIndexer, ctgName); len(items) == 0 || err != nil {
return v1alpha1.UncategorizedId
} else {
obj, _ := items[0].(*v1alpha1.HelmCategory)
return obj.Name
}
}
func (c *cachedRepos) GetApplication(appId string) (app *v1alpha1.HelmApplication, exists bool) {
c.RLock()
defer c.RUnlock()
if app, exists := c.apps[appId]; exists {
return app, true
}
return
}
func (c *cachedRepos) UpdateRepo(old, new *v1alpha1.HelmRepo) error {
if old.Status.Data == new.Status.Data {
return nil
}
c.Lock()
defer c.Unlock()
c.deleteRepo(old)
return c.addRepo(new, false)
}
func (c *cachedRepos) AddRepo(repo *v1alpha1.HelmRepo) error {
c.Lock()
defer c.Unlock()
return c.addRepo(repo, false)
}
// Add a new Repo to cachedRepos
func (c *cachedRepos) addRepo(repo *v1alpha1.HelmRepo, builtin bool) error {
if len(repo.Status.Data) == 0 {
return nil
}
index, err := helmrepoindex.ByteArrayToSavedIndex([]byte(repo.Status.Data))
if err != nil {
klog.Errorf("json unmarshal repo %s failed, error: %s", repo.Name, err)
return err
}
klog.V(2).Infof("add repo %s to cache", repo.Name)
ws := workspace(repo.GetWorkspace())
if _, exists := c.chartsInRepo[ws]; !exists {
c.chartsInRepo[ws] = make(map[string]int)
}
repoId := repo.GetHelmRepoId()
c.repos[repoId] = repo
var appName string
chartsCount := 0
for key, app := range index.Applications {
appName = app.ApplicationId
appLabels := make(map[string]string)
if helmrepoindex.IsBuiltInRepo(repo.Name) {
appLabels[constants.WorkspaceLabelKey] = "system-workspace"
}
appLabels[constants.ChartRepoIdLabelKey] = repoId
helmApp := v1alpha1.HelmApplication{
ObjectMeta: metav1.ObjectMeta{
Name: appName,
Annotations: map[string]string{
constants.CreatorAnnotationKey: repo.GetCreator(),
},
Labels: appLabels,
CreationTimestamp: metav1.Time{Time: app.Created},
},
Spec: v1alpha1.HelmApplicationSpec{
Name: key,
Description: app.Description,
Icon: app.Icon,
},
Status: v1alpha1.HelmApplicationStatus{
State: v1alpha1.StateActive,
},
}
c.apps[app.ApplicationId] = &helmApp
var ctg, appVerName string
var chartData []byte
var latestVersionName string
var latestSemver *semver.Version
// build all the versions of this app
for _, chartVersion := range app.Charts {
chartsCount += 1
hvw := helmrepoindex.HelmVersionWrapper{ChartVersion: &chartVersion.ChartVersion}
appVerName = chartVersion.ApplicationVersionId
version := &v1alpha1.HelmApplicationVersion{
ObjectMeta: metav1.ObjectMeta{
Name: appVerName,
Annotations: map[string]string{constants.CreatorAnnotationKey: repo.GetCreator()},
Labels: map[string]string{
constants.ChartApplicationIdLabelKey: appName,
constants.ChartRepoIdLabelKey: repo.GetHelmRepoId(),
},
CreationTimestamp: metav1.Time{Time: chartVersion.Created},
},
Spec: v1alpha1.HelmApplicationVersionSpec{
Metadata: &v1alpha1.Metadata{
Name: hvw.GetName(),
AppVersion: hvw.GetAppVersion(),
Version: hvw.GetVersion(),
Description: hvw.GetDescription(),
Home: hvw.GetHome(),
Icon: hvw.GetIcon(),
Maintainers: hvw.GetRawMaintainers(),
Sources: hvw.GetRawSources(),
Keywords: hvw.GetRawKeywords(),
},
URLs: chartVersion.URLs,
Digest: chartVersion.Digest,
Data: chartData,
},
Status: v1alpha1.HelmApplicationVersionStatus{
State: v1alpha1.StateActive,
},
}
// It is not necessary to store these pieces of information when this is not a built-in repo.
if helmrepoindex.IsBuiltInRepo(repo.Name) {
version.Spec.Sources = hvw.GetRawSources()
version.Spec.Maintainers = hvw.GetRawMaintainers()
version.Spec.Home = hvw.GetHome()
}
c.versions[chartVersion.ApplicationVersionId] = version
// Find the latest version.
currSemver, err := semver.NewVersion(version.GetSemver())
if err == nil {
if latestSemver == nil {
// the first valid semver
latestSemver = currSemver
latestVersionName = version.GetVersionName()
// Use the category of the latest version as the category of the app.
ctg = chartVersion.Annotations[CategoryAnnotationKey]
} else if latestSemver.LessThan(currSemver) {
// find a newer valid semver
latestSemver = currSemver
latestVersionName = version.GetVersionName()
ctg = chartVersion.Annotations[CategoryAnnotationKey]
}
} else {
// If the semver is invalid, just ignore it.
klog.V(2).Infof("parse version failed, id: %s, err: %s", version.Name, err)
}
}
helmApp.Status.LatestVersion = latestVersionName
if helmrepoindex.IsBuiltInRepo(repo.Name) {
// Add category id to the apps in the built-in repo
ctgId := c.translateCategoryNameToId(ctg)
if helmApp.Labels == nil {
helmApp.Labels = map[string]string{}
}
helmApp.Labels[constants.CategoryIdLabelKey] = ctgId
c.builtinCategoryCounts[ctgId] += 1
}
}
c.chartsInRepo[ws][repo.GetHelmRepoId()] = chartsCount
return nil
}
func (c *cachedRepos) ListApplicationsInRepo(repoId string) (ret []*v1alpha1.HelmApplication, exists bool) {
c.RLock()
defer c.RUnlock()
if repo, exists := c.repos[repoId]; !exists {
return nil, false
} else {
ret = make([]*v1alpha1.HelmApplication, 0, 10)
for _, app := range c.apps {
if app.GetHelmRepoId() == repo.Name {
ret = append(ret, app)
}
}
}
return ret, true
}
func (c *cachedRepos) ListApplicationsInBuiltinRepo(selector labels.Selector) (ret []*v1alpha1.HelmApplication, exists bool) {
c.RLock()
defer c.RUnlock()
ret = make([]*v1alpha1.HelmApplication, 0, 20)
for _, app := range c.apps {
if strings.HasPrefix(app.GetHelmRepoId(), v1alpha1.BuiltinRepoPrefix) {
if selector != nil && !selector.Empty() &&
(app.Labels == nil || !selector.Matches(labels.Set(app.Labels))) { // If the selector is not empty, we must check whether the labels of the app match the selector.
continue
}
ret = append(ret, app)
}
}
return ret, true
}
func (c *cachedRepos) ListAppVersionsByAppId(appId string) (ret []*v1alpha1.HelmApplicationVersion, exists bool) {
c.RLock()
defer c.RUnlock()
if _, exists := c.apps[appId]; !exists {
return nil, false
}
ret = make([]*v1alpha1.HelmApplicationVersion, 0, 10)
for _, ver := range c.versions {
if ver.GetHelmApplicationId() == appId {
ret = append(ret, ver)
}
}
return ret, true
}
func (c *cachedRepos) getAppVersion(versionId string, withData bool) (ret *v1alpha1.HelmApplicationVersion, exists bool, err error) {
c.RLock()
if version, exists := c.versions[versionId]; exists {
//builtin chart data
if withData {
if len(version.Spec.Data) != 0 {
c.RUnlock()
return version, true, nil
}
if len(version.Spec.URLs) == 0 {
c.RUnlock()
return nil, true, errors.New("invalid chart spec")
}
var repo *v1alpha1.HelmRepo
var exists bool
if repo, exists = c.repos[version.GetHelmRepoId()]; !exists {
c.RUnlock()
klog.Errorf("load repo for app version: %s/%s failed",
version.GetWorkspace(), version.GetTrueName())
return nil, true, err
}
c.RUnlock()
url := version.Spec.URLs[0]
if !(strings.HasPrefix(url, "https://") || strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "s3://")) {
url = repo.Spec.Url + "/" + url
}
buf, err := helmrepoindex.LoadChart(context.TODO(), url, &repo.Spec.Credential)
if err != nil {
klog.Errorf("load chart data for app version: %s/%s failed, error : %s", version.GetTrueName(),
version.GetTrueName(), err)
return nil, true, err
}
version.Spec.Data = buf.Bytes()
return version, true, nil
} else {
c.RUnlock()
return version, true, nil
}
} else {
c.RUnlock()
//version does not exists
return nil, false, nil
}
}
func (c *cachedRepos) GetAppVersion(versionId string) (ret *v1alpha1.HelmApplicationVersion, exists bool, err error) {
return c.getAppVersion(versionId, false)
}
func (c *cachedRepos) GetAppVersionWithData(versionId string) (ret *v1alpha1.HelmApplicationVersion, exists bool, err error) {
return c.getAppVersion(versionId, true)
}

20
pkg/utils/resourceparse/resource_parse.go
View File

@@ -1,19 +1,7 @@
// /*
// Copyright 2020 The KubeSphere Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// */
//
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package resourceparse

66
pkg/utils/serviceaccount/serviceaccount_util.go Normal file
View File

@@ -0,0 +1,66 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package serviceaccount
import (
"strings"
"k8s.io/apiserver/pkg/authentication/user"
corev1alpha1 "kubesphere.io/api/core/v1alpha1"
)
func IsServiceAccountToken(subjectName string) bool {
if !strings.HasPrefix(subjectName, corev1alpha1.ServiceAccountTokenPrefix) {
return false
}
split := strings.Split(subjectName, ":")
return len(split) == 4
}
func GetSecretName(info user.Info) (name, namespace string) {
extra := info.GetExtra()
if value, ok := extra[corev1alpha1.ServiceAccountTokenExtraSecretName]; ok {
name = value[0]
}
if value, ok := extra[corev1alpha1.ServiceAccountTokenExtraSecretNamespace]; ok {
namespace = value[0]
}
return
}
func SplitUsername(username string) (name, namespace string) {
if !strings.HasPrefix(username, corev1alpha1.ServiceAccountTokenPrefix) {
return "", ""
}
split := strings.Split(username, ":")
if len(split) != 4 {
return "", ""
}
return split[3], split[2]
}
// MatchesUsername checks whether the provided username matches the namespace and name without
// allocating. Use this when checking a service account namespace and name against a known string.
func MatchesUsername(namespace, name string, username string) bool {
if !strings.HasPrefix(username, corev1alpha1.ServiceAccountTokenPrefix) {
return false
}
username = username[len(corev1alpha1.ServiceAccountTokenPrefix):]
if !strings.HasPrefix(username, namespace) {
return false
}
username = username[len(namespace):]
if !strings.HasPrefix(username, ":") {
return false
}
username = username[len(":"):]
return username == name
}

35
pkg/utils/sliceutil/sliceutils.go
View File

@@ -1,21 +1,12 @@
/*
Copyright 2019 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package sliceutil
import "sort"
func RemoveString(slice []string, remove func(item string) bool) []string {
for i := 0; i < len(slice); i++ {
if remove(slice[i]) {
@@ -34,3 +25,19 @@ func HasString(slice []string, str string) bool {
}
return false
}
func Equal(slice1, slice2 []string) bool {
if len(slice1) != len(slice2) {
return false
}
sort.Strings(slice1)
sort.Strings(slice2)
for i, s := range slice1 {
if s != slice2[i] {
return false
}
}
return true
}

17
pkg/utils/stringutils/string.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2018 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package stringutils

17
pkg/utils/term/term.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2018 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package term