admin/kubesphere
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: kubesphere 4.0 (#6115)

Browse Source 
* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

---------

Signed-off-by: ci-bot <ci-bot@kubesphere.io>
Co-authored-by: ks-ci-bot <ks-ci-bot@example.com>
Co-authored-by: joyceliu <joyceliu@yunify.com>
This commit is contained in:
KubeSphere CI Bot
2024-09-06 11:05:52 +08:00
committed by GitHub
parent b5015ec7b9
commit 447a51f08b
8557 changed files with 546695 additions and 1146174 deletions
Download Patch File Download Diff File Expand all files Collapse all files

216
pkg/apiserver/auditing/backend.go
View File

@@ -1,216 +0,0 @@
/*
Copyright 2020 KubeSphere Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package auditing
import (
"bytes"
"context"
"crypto/tls"
"encoding/json"
"net/http"
"time"
"k8s.io/klog/v2"
"kubesphere.io/kubesphere/pkg/apiserver/auditing/v1alpha1"
options "kubesphere.io/kubesphere/pkg/simple/client/auditing"
)
const (
GetSenderTimeout = time.Second
SendTimeout = time.Second * 3
DefaultSendersNum = 100
DefaultBatchSize = 100
DefaultBatchInterval = time.Second * 3
WebhookURL = "https://kube-auditing-webhook-svc.kubesphere-logging-system.svc:6443/audit/webhook/event"
)
type Backend struct {
url string
senderCh chan interface{}
cache chan *v1alpha1.Event
client http.Client
sendTimeout time.Duration
getSenderTimeout time.Duration
eventBatchSize int
eventBatchInterval time.Duration
stopCh <-chan struct{}
}
func NewBackend(opts *options.Options, cache chan *v1alpha1.Event, stopCh <-chan struct{}) *Backend {
b := Backend{
url: opts.WebhookUrl,
getSenderTimeout: GetSenderTimeout,
cache: cache,
sendTimeout: SendTimeout,
eventBatchSize: opts.EventBatchSize,
eventBatchInterval: opts.EventBatchInterval,
stopCh: stopCh,
}
if len(b.url) == 0 {
b.url = WebhookURL
}
if b.eventBatchInterval == 0 {
b.eventBatchInterval = DefaultBatchInterval
}
if b.eventBatchSize == 0 {
b.eventBatchSize = DefaultBatchSize
}
sendersNum := opts.EventSendersNum
if sendersNum == 0 {
sendersNum = DefaultSendersNum
}
b.senderCh = make(chan interface{}, sendersNum)
b.client = http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
},
Timeout: b.sendTimeout,
}
go b.worker()
return &b
}
func (b *Backend) worker() {
for {
events := b.getEvents()
if events == nil {
break
}
if len(events.Items) == 0 {
continue
}
go b.sendEvents(events)
}
}
func (b *Backend) getEvents() *v1alpha1.EventList {
ctx, cancel := context.WithTimeout(context.Background(), b.eventBatchInterval)
defer cancel()
events := &v1alpha1.EventList{}
for {
select {
case event := <-b.cache:
if event == nil {
break
}
events.Items = append(events.Items, *event)
if len(events.Items) >= b.eventBatchSize {
return events
}
case <-ctx.Done():
return events
case <-b.stopCh:
return nil
}
}
}
func (b *Backend) sendEvents(events *v1alpha1.EventList) {
ctx, cancel := context.WithTimeout(context.Background(), b.sendTimeout)
defer cancel()
stopCh := make(chan struct{})
skipReturnSender := false
send := func() {
ctx, cancel := context.WithTimeout(context.Background(), b.getSenderTimeout)
defer cancel()
select {
case <-ctx.Done():
klog.Error("Get auditing event sender timeout")
skipReturnSender = true
return
case b.senderCh <- struct{}{}:
}
start := time.Now()
defer func() {
stopCh <- struct{}{}
klog.V(8).Infof("send %d auditing logs used %d", len(events.Items), time.Since(start).Milliseconds())
}()
bs, err := b.eventToBytes(events)
if err != nil {
klog.Errorf("json marshal error, %s", err)
return
}
klog.V(8).Infof("%s", string(bs))
response, err := b.client.Post(b.url, "application/json", bytes.NewBuffer(bs))
if err != nil {
klog.Errorf("send audit events error, %s", err)
return
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
klog.Errorf("send audit events error[%d]", response.StatusCode)
return
}
}
go send()
defer func() {
if !skipReturnSender {
<-b.senderCh
}
}()
select {
case <-ctx.Done():
klog.Error("send audit events timeout")
case <-stopCh:
}
}
func (b *Backend) eventToBytes(event *v1alpha1.EventList) ([]byte, error) {
bs, err := json.Marshal(event)
if err != nil {
// Normally, the serialization failure is caused by the failure of ResponseObject serialization.
// To ensure the integrity of the auditing event to the greatest extent,
// it is necessary to delete ResponseObject and and then try to serialize again.
if event.Items[0].ResponseObject != nil {
event.Items[0].ResponseObject = nil
return json.Marshal(event)
}
return nil, err
}
return bs, err
}

298
pkg/apiserver/auditing/types.go → pkg/apiserver/auditing/client.go
View File

@@ -1,92 +1,149 @@
/*
Copyright 2020 KubeSphere Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package auditing
import (
"bufio"
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"os"
"time"
"github.com/google/uuid"
"github.com/modern-go/reflect2"
v1 "k8s.io/api/authentication/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apiserver/pkg/apis/audit"
"k8s.io/klog/v2"
devopsv1alpha3 "kubesphere.io/api/devops/v1alpha3"
"kubesphere.io/api/iam/v1alpha2"
clusterv1alpha1 "kubesphere.io/api/cluster/v1alpha1"
"kubesphere.io/api/iam/v1beta1"
auditv1alpha1 "kubesphere.io/kubesphere/pkg/apiserver/auditing/v1alpha1"
"kubesphere.io/kubesphere/pkg/apiserver/query"
"kubesphere.io/kubesphere/pkg/apiserver/auditing/internal"
"kubesphere.io/kubesphere/pkg/apiserver/auditing/log"
"kubesphere.io/kubesphere/pkg/apiserver/auditing/webhook"
"kubesphere.io/kubesphere/pkg/apiserver/request"
"kubesphere.io/kubesphere/pkg/client/listers/auditing/v1alpha1"
"kubesphere.io/kubesphere/pkg/informers"
"kubesphere.io/kubesphere/pkg/models/resources/v1alpha3"
"kubesphere.io/kubesphere/pkg/models/resources/v1alpha3/devops"
options "kubesphere.io/kubesphere/pkg/simple/client/auditing"
"kubesphere.io/kubesphere/pkg/constants"
"kubesphere.io/kubesphere/pkg/simple/client/k8s"
"kubesphere.io/kubesphere/pkg/utils/iputil"
)
const (
DefaultWebhook = "kube-auditing-webhook"
DefaultCacheCapacity = 10000
CacheTimeout = time.Second
DefaultBatchSize = 100
DefaultBatchInterval = time.Second * 3
)
type Auditing interface {
Enabled() bool
K8sAuditingEnabled() bool
LogRequestObject(req *http.Request, info *request.RequestInfo) *auditv1alpha1.Event
LogResponseObject(e *auditv1alpha1.Event, resp *ResponseCapture)
LogRequestObject(req *http.Request, info *request.RequestInfo) *Event
LogResponseObject(e *Event, resp *ResponseCapture)
}
type auditing struct {
webhookLister v1alpha1.WebhookLister
devopsGetter v1alpha3.Interface
cache chan *auditv1alpha1.Event
backend *Backend
k8sClient k8s.Client
stopCh <-chan struct{}
auditLevel audit.Level
events chan *Event
backend []internal.Backend
hostname string
hostIP string
cluster string
eventBatchSize int
eventBatchInterval time.Duration
}
func NewAuditing(informers informers.InformerFactory, opts *options.Options, stopCh <-chan struct{}) Auditing {
func NewAuditing(kubernetesClient k8s.Client, opts *Options, stopCh <-chan struct{}) Auditing {
a := &auditing{
webhookLister: informers.KubeSphereSharedInformerFactory().Auditing().V1alpha1().Webhooks().Lister(),
devopsGetter: devops.New(informers.KubeSphereSharedInformerFactory()),
cache: make(chan *auditv1alpha1.Event, DefaultCacheCapacity),
k8sClient: kubernetesClient,
stopCh: stopCh,
auditLevel: opts.AuditLevel,
events: make(chan *Event, DefaultCacheCapacity),
hostname: os.Getenv("HOSTNAME"),
hostIP: getHostIP(),
eventBatchInterval: opts.EventBatchInterval,
eventBatchSize: opts.EventBatchSize,
}
a.backend = NewBackend(opts, a.cache, stopCh)
if a.eventBatchInterval == 0 {
a.eventBatchInterval = DefaultBatchInterval
}
if a.eventBatchSize == 0 {
a.eventBatchSize = DefaultBatchSize
}
a.cluster = a.getClusterName()
if opts.WebhookOptions.WebhookUrl != "" {
a.backend = append(a.backend, webhook.NewBackend(opts.WebhookOptions.WebhookUrl,
opts.WebhookOptions.EventSendersNum))
}
if opts.LogOptions.Path != "" {
a.backend = append(a.backend, log.NewBackend(opts.LogOptions.Path,
opts.LogOptions.MaxAge,
opts.LogOptions.MaxBackups,
opts.LogOptions.MaxSize))
}
go a.Start()
return a
}
func (a *auditing) getAuditLevel() audit.Level {
wh, err := a.webhookLister.Get(DefaultWebhook)
if err != nil {
klog.V(8).Info(err)
return audit.LevelNone
func getHostIP() string {
addrs, err := net.InterfaceAddrs()
hostip := ""
if err == nil {
for _, addr := range addrs {
if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
if ipnet.IP.To4() != nil {
hostip = ipnet.IP.String()
break
}
}
}
}
return (audit.Level)(wh.Spec.AuditLevel)
return hostip
}
func (a *auditing) getClusterName() string {
ns, err := a.k8sClient.CoreV1().Namespaces().Get(context.Background(), constants.KubeSphereNamespace, metav1.GetOptions{})
if err != nil {
klog.Errorf("get %s error: %s", constants.KubeSphereNamespace, err)
return ""
}
if ns.Annotations != nil {
return ns.Annotations[clusterv1alpha1.AnnotationClusterName]
}
return ""
}
func (a *auditing) getAuditLevel() audit.Level {
if a.auditLevel != "" {
return a.auditLevel
}
return audit.LevelMetadata
}
func (a *auditing) Enabled() bool {
@@ -95,16 +152,6 @@ func (a *auditing) Enabled() bool {
return !level.Less(audit.LevelMetadata)
}
func (a *auditing) K8sAuditingEnabled() bool {
wh, err := a.webhookLister.Get(DefaultWebhook)
if err != nil {
klog.V(8).Info(err)
return false
}
return wh.Spec.K8sAuditingEnabled
}
// If the request is not a standard request, or a resource request,
// or part of the audit information cannot be obtained through url,
// the function that handles the request can obtain Event from
@@ -116,7 +163,8 @@ func (a *auditing) K8sAuditingEnabled() bool {
// info.Verb = "post"
// info.Name = created.Name
// }
func (a *auditing) LogRequestObject(req *http.Request, info *request.RequestInfo) *auditv1alpha1.Event {
func (a *auditing) LogRequestObject(req *http.Request, info *request.RequestInfo) *Event {
// Ignore the dryRun k8s request.
if info.IsKubernetesRequest {
@@ -126,10 +174,11 @@ func (a *auditing) LogRequestObject(req *http.Request, info *request.RequestInfo
}
}
e := &auditv1alpha1.Event{
Devops: info.DevOps,
e := &Event{
HostName: a.hostname,
HostIP: a.hostIP,
Workspace: info.Workspace,
Cluster: info.Cluster,
Cluster: a.cluster,
Event: audit.Event{
RequestURI: info.Path,
Verb: info.Verb,
@@ -153,25 +202,6 @@ func (a *auditing) LogRequestObject(req *http.Request, info *request.RequestInfo
},
}
// Get the workspace which the devops project be in.
if len(e.Devops) > 0 && len(e.Workspace) == 0 {
res, err := a.devopsGetter.List("", query.New())
if err != nil {
klog.Error(err)
}
for _, obj := range res.Items {
d := obj.(*devopsv1alpha3.DevOpsProject)
if d.Name == e.Devops {
e.Workspace = d.Labels["kubesphere.io/workspace"]
} else if d.Status.AdminNamespace == e.Devops {
e.Workspace = d.Labels["kubesphere.io/workspace"]
e.Devops = d.Name
}
}
}
ips := make([]string, 1)
ips[0] = iputil.RemoteIp(req)
e.SourceIPs = ips
@@ -203,7 +233,7 @@ func (a *auditing) LogRequestObject(req *http.Request, info *request.RequestInfo
// For resource creating request, get resource name from the request body.
if info.Verb == "create" {
obj := &auditv1alpha1.Object{}
obj := &Object{}
if err := json.Unmarshal(body, obj); err == nil {
e.ObjectRef.Name = obj.Name
}
@@ -211,21 +241,47 @@ func (a *auditing) LogRequestObject(req *http.Request, info *request.RequestInfo
// for recording disable and enable user
if e.ObjectRef.Resource == "users" && e.Verb == "update" {
u := &v1alpha2.User{}
u := &v1beta1.User{}
if err := json.Unmarshal(body, u); err == nil {
if u.Status.State == v1alpha2.UserActive {
if u.Status.State == v1beta1.UserActive {
e.Verb = "enable"
} else if u.Status.State == v1alpha2.UserDisabled {
} else if u.Status.State == v1beta1.UserDisabled {
e.Verb = "disable"
}
}
}
}
a.getWorkspace(e)
return e
}
func (a *auditing) needAnalyzeRequestBody(e *auditv1alpha1.Event, req *http.Request) bool {
func (a *auditing) getWorkspace(e *Event) {
if e.Workspace != "" {
return
}
ns := e.ObjectRef.Namespace
if e.ObjectRef.Resource == "namespaces" {
ns = e.ObjectRef.Name
}
if ns == "" {
return
}
namespace, err := a.k8sClient.CoreV1().Namespaces().Get(context.Background(), ns, metav1.GetOptions{})
if err != nil && !apierrors.IsNotFound(err) {
klog.Errorf("get %s error: %s", ns, err)
return
}
if namespace.Labels != nil {
e.Workspace = namespace.Labels[constants.WorkspaceLabelKey]
}
}
func (a *auditing) needAnalyzeRequestBody(e *Event, req *http.Request) bool {
if req.ContentLength <= 0 {
return false
@@ -247,7 +303,7 @@ func (a *auditing) needAnalyzeRequestBody(e *auditv1alpha1.Event, req *http.Requ
return false
}
func (a *auditing) LogResponseObject(e *auditv1alpha1.Event, resp *ResponseCapture) {
func (a *auditing) LogResponseObject(e *Event, resp *ResponseCapture) {
e.StageTimestamp = metav1.NowMicro()
e.ResponseStatus = &metav1.Status{Code: int32(resp.StatusCode())}
@@ -258,10 +314,9 @@ func (a *auditing) LogResponseObject(e *auditv1alpha1.Event, resp *ResponseCaptu
a.cacheEvent(*e)
}
func (a *auditing) cacheEvent(e auditv1alpha1.Event) {
func (a *auditing) cacheEvent(e Event) {
select {
case a.cache <- &e:
case a.events <- &e:
return
case <-time.After(CacheTimeout):
klog.V(8).Infof("cache audit event %s timeout", e.AuditID)
@@ -269,6 +324,80 @@ func (a *auditing) cacheEvent(e auditv1alpha1.Event) {
}
}
func (a *auditing) Start() {
for {
events, exit := a.getEvents()
if exit {
break
}
if len(events) == 0 {
continue
}
byteEvents := a.eventToBytes(events)
if len(byteEvents) == 0 {
continue
}
for _, b := range a.backend {
if reflect2.IsNil(b) {
continue
}
b.ProcessEvents(byteEvents...)
}
}
}
func (a *auditing) getEvents() ([]*Event, bool) {
ctx, cancel := context.WithTimeout(context.Background(), a.eventBatchInterval)
defer cancel()
var events []*Event
for {
select {
case event := <-a.events:
if event == nil {
break
}
events = append(events, event)
if len(events) >= a.eventBatchSize {
return events, false
}
case <-ctx.Done():
return events, false
case <-a.stopCh:
return nil, true
}
}
}
func (a *auditing) eventToBytes(events []*Event) [][]byte {
var res [][]byte
for _, event := range events {
bs, err := json.Marshal(event)
if err != nil {
// Normally, the serialization failure is caused by the failure of ResponseObject serialization.
// To ensure the integrity of the auditing event to the greatest extent,
// it is necessary to delete ResponseObject and and then try to serialize again.
if event.ResponseObject != nil {
event.ResponseObject = nil
bs, err = json.Marshal(event)
}
}
if err != nil {
klog.Errorf("serialize audit event error: %s", err)
continue
}
res = append(res, bs)
}
return res
}
type ResponseCapture struct {
http.ResponseWriter
wroteHeader bool
@@ -289,7 +418,6 @@ func (c *ResponseCapture) Header() http.Header {
}
func (c *ResponseCapture) Write(data []byte) (int, error) {
c.WriteHeader(http.StatusOK)
c.body.Write(data)
return c.ResponseWriter.Write(data)

28
pkg/apiserver/auditing/event.go Normal file
View File

@@ -0,0 +1,28 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package auditing
import (
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apiserver/pkg/apis/audit"
)
type Event struct {
// The workspace which this audit event happened
Workspace string
// The cluster which this audit event happened
Cluster string
// Message send to user.
Message string
HostName string
HostIP string
audit.Event
}
type Object struct {
v1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
}

10
pkg/apiserver/auditing/internal/backend.go Normal file
View File

@@ -0,0 +1,10 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package internal
type Backend interface {
ProcessEvents(events ...[]byte)
}

94
pkg/apiserver/auditing/log/backend.go Normal file
View File

@@ -0,0 +1,94 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package log
import (
"fmt"
"io"
"os"
"path/filepath"
"time"
"gopkg.in/natefinch/lumberjack.v2"
"k8s.io/klog/v2"
"kubesphere.io/kubesphere/pkg/apiserver/auditing/internal"
)
const (
WriteTimeout = time.Second * 3
DefaultMaxAge = 7
DefaultMaxBackups = 10
DefaultMaxSize = 100
)
type backend struct {
path string
maxAge int
maxBackups int
maxSize int
timeout time.Duration
writer io.Writer
}
func NewBackend(path string, maxAge, maxBackups, maxSize int) internal.Backend {
b := backend{
path: path,
maxAge: maxAge,
maxBackups: maxBackups,
maxSize: maxSize,
timeout: WriteTimeout,
}
if b.maxAge == 0 {
b.maxAge = DefaultMaxAge
}
if b.maxBackups == 0 {
b.maxBackups = DefaultMaxBackups
}
if b.maxSize == 0 {
b.maxSize = DefaultMaxSize
}
if err := b.ensureLogFile(); err != nil {
klog.Errorf("ensure audit log file error, %s", err)
return nil
}
b.writer = &lumberjack.Logger{
Filename: b.path,
MaxAge: b.maxAge,
MaxBackups: b.maxBackups,
MaxSize: b.maxSize,
Compress: false,
}
return &b
}
func (b *backend) ensureLogFile() error {
if err := os.MkdirAll(filepath.Dir(b.path), 0700); err != nil {
return err
}
mode := os.FileMode(0600)
f, err := os.OpenFile(b.path, os.O_CREATE|os.O_APPEND|os.O_RDWR, mode)
if err != nil {
return err
}
return f.Close()
}
func (b *backend) ProcessEvents(events ...[]byte) {
for _, event := range events {
if _, err := fmt.Fprint(b.writer, string(event)+"\n"); err != nil {
klog.Errorf("Log audit event error, %s. affecting audit event: %v\nImpacted event:\n", err, event)
klog.Error(string(event))
}
}
}

69
pkg/apiserver/auditing/options.go Normal file
View File

@@ -0,0 +1,69 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package auditing
import (
"time"
"k8s.io/apiserver/pkg/apis/audit"
"github.com/spf13/pflag"
)
type WebhookOptions struct {
WebhookUrl string `json:"webhookUrl" yaml:"webhookUrl"`
// The maximum concurrent senders which send auditing events to the auditing webhook.
EventSendersNum int `json:"eventSendersNum" yaml:"eventSendersNum"`
}
type LogOptions struct {
Path string `json:"path" yaml:"path"`
MaxAge int `json:"maxAge" yaml:"maxAge"`
MaxBackups int `json:"maxBackups" yaml:"maxBackups"`
MaxSize int `json:"maxSize" yaml:"maxSize"`
}
type Options struct {
Enable bool `json:"enable" yaml:"enable"`
AuditLevel audit.Level `json:"auditLevel" yaml:"auditLevel"`
// The batch size of auditing events.
EventBatchSize int `json:"eventBatchSize" yaml:"eventBatchSize"`
// The batch interval of auditing events.
EventBatchInterval time.Duration `json:"eventBatchInterval" yaml:"eventBatchInterval"`
WebhookOptions WebhookOptions `json:"webhookOptions" yaml:"webhookOptions"`
LogOptions LogOptions `json:"logOptions" yaml:"logOptions"`
}
func NewAuditingOptions() *Options {
return &Options{}
}
func (s *Options) Validate() []error {
errs := make([]error, 0)
return errs
}
func (s *Options) AddFlags(fs *pflag.FlagSet, c *Options) {
fs.BoolVar(&s.Enable, "auditing-enabled", c.Enable, "Enable auditing component or not. ")
fs.IntVar(&s.EventBatchSize, "auditing-event-batch-size", c.EventBatchSize,
"The batch size of auditing events.")
fs.DurationVar(&s.EventBatchInterval, "auditing-event-batch-interval", c.EventBatchInterval,
"The batch interval of auditing events.")
fs.StringVar(&s.WebhookOptions.WebhookUrl, "auditing-webhook-url", c.WebhookOptions.WebhookUrl, "Auditing wehook url")
fs.IntVar(&s.WebhookOptions.EventSendersNum, "auditing-event-senders-num", c.WebhookOptions.EventSendersNum,
"The maximum concurrent senders which send auditing events to the auditing webhook.")
fs.StringVar(&s.LogOptions.Path, "audit-log-path", s.LogOptions.Path,
"If set, all requests coming to the apiserver will be logged to this file. '-' means standard out.")
fs.IntVar(&s.LogOptions.MaxAge, "audit-log-maxage", s.LogOptions.MaxAge,
"The maximum number of days to retain old audit log files based on the timestamp encoded in their filename.")
fs.IntVar(&s.LogOptions.MaxBackups, "audit-log-maxbackup", s.LogOptions.MaxBackups,
"The maximum number of old audit log files to retain. Setting a value of 0 will mean there's no restriction on the number of files.")
fs.IntVar(&s.LogOptions.MaxSize, "audit-log-maxsize", s.LogOptions.MaxSize,
"The maximum size in megabytes of the audit log file before it gets rotated.")
}

351
pkg/apiserver/auditing/types_test.go
View File

@@ -1,351 +0,0 @@
/*
Copyright 2020 KubeSphere Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package auditing
import (
"encoding/json"
"net/http"
"net/http/httptest"
"net/url"
"testing"
"github.com/stretchr/testify/assert"
v1 "k8s.io/api/authentication/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apiserver/pkg/apis/audit"
"k8s.io/apiserver/pkg/authentication/user"
k8srequest "k8s.io/apiserver/pkg/endpoints/request"
fakek8s "k8s.io/client-go/kubernetes/fake"
auditingv1alpha1 "kubesphere.io/api/auditing/v1alpha1"
v1alpha12 "kubesphere.io/kubesphere/pkg/apiserver/auditing/v1alpha1"
"kubesphere.io/kubesphere/pkg/apiserver/request"
"kubesphere.io/kubesphere/pkg/client/clientset/versioned/fake"
"kubesphere.io/kubesphere/pkg/informers"
"kubesphere.io/kubesphere/pkg/utils/iputil"
)
func TestGetAuditLevel(t *testing.T) {
webhook := &auditingv1alpha1.Webhook{
TypeMeta: metav1.TypeMeta{
APIVersion: auditingv1alpha1.SchemeGroupVersion.String(),
},
ObjectMeta: metav1.ObjectMeta{
Name: "kube-auditing-webhook",
},
Spec: auditingv1alpha1.WebhookSpec{
AuditLevel: auditingv1alpha1.LevelRequestResponse,
},
}
ksClient := fake.NewSimpleClientset()
k8sClient := fakek8s.NewSimpleClientset()
fakeInformerFactory := informers.NewInformerFactories(k8sClient, ksClient, nil, nil, nil, nil)
a := auditing{
webhookLister: fakeInformerFactory.KubeSphereSharedInformerFactory().Auditing().V1alpha1().Webhooks().Lister(),
}
err := fakeInformerFactory.KubeSphereSharedInformerFactory().Auditing().V1alpha1().Webhooks().Informer().GetIndexer().Add(webhook)
if err != nil {
panic(err)
}
assert.Equal(t, string(webhook.Spec.AuditLevel), string(a.getAuditLevel()))
}
func TestAuditing_Enabled(t *testing.T) {
webhook := &auditingv1alpha1.Webhook{
TypeMeta: metav1.TypeMeta{
APIVersion: auditingv1alpha1.SchemeGroupVersion.String(),
},
ObjectMeta: metav1.ObjectMeta{
Name: "kube-auditing-webhook",
},
Spec: auditingv1alpha1.WebhookSpec{
AuditLevel: auditingv1alpha1.LevelNone,
},
}
ksClient := fake.NewSimpleClientset()
k8sClient := fakek8s.NewSimpleClientset()
fakeInformerFactory := informers.NewInformerFactories(k8sClient, ksClient, nil, nil, nil, nil)
a := auditing{
webhookLister: fakeInformerFactory.KubeSphereSharedInformerFactory().Auditing().V1alpha1().Webhooks().Lister(),
}
err := fakeInformerFactory.KubeSphereSharedInformerFactory().Auditing().V1alpha1().Webhooks().Informer().GetIndexer().Add(webhook)
if err != nil {
panic(err)
}
assert.Equal(t, false, a.Enabled())
}
func TestAuditing_K8sAuditingEnabled(t *testing.T) {
webhook := &auditingv1alpha1.Webhook{
TypeMeta: metav1.TypeMeta{
APIVersion: auditingv1alpha1.SchemeGroupVersion.String(),
},
ObjectMeta: metav1.ObjectMeta{
Name: "kube-auditing-webhook",
},
Spec: auditingv1alpha1.WebhookSpec{
AuditLevel: auditingv1alpha1.LevelNone,
K8sAuditingEnabled: true,
},
}
ksClient := fake.NewSimpleClientset()
k8sClient := fakek8s.NewSimpleClientset()
fakeInformerFactory := informers.NewInformerFactories(k8sClient, ksClient, nil, nil, nil, nil)
a := auditing{
webhookLister: fakeInformerFactory.KubeSphereSharedInformerFactory().Auditing().V1alpha1().Webhooks().Lister(),
}
err := fakeInformerFactory.KubeSphereSharedInformerFactory().Auditing().V1alpha1().Webhooks().Informer().GetIndexer().Add(webhook)
if err != nil {
panic(err)
}
assert.Equal(t, true, a.K8sAuditingEnabled())
}
func TestAuditing_LogRequestObject(t *testing.T) {
webhook := &auditingv1alpha1.Webhook{
TypeMeta: metav1.TypeMeta{
APIVersion: auditingv1alpha1.SchemeGroupVersion.String(),
},
ObjectMeta: metav1.ObjectMeta{
Name: "kube-auditing-webhook",
},
Spec: auditingv1alpha1.WebhookSpec{
AuditLevel: auditingv1alpha1.LevelRequestResponse,
K8sAuditingEnabled: true,
},
}
ksClient := fake.NewSimpleClientset()
k8sClient := fakek8s.NewSimpleClientset()
fakeInformerFactory := informers.NewInformerFactories(k8sClient, ksClient, nil, nil, nil, nil)
a := auditing{
webhookLister: fakeInformerFactory.KubeSphereSharedInformerFactory().Auditing().V1alpha1().Webhooks().Lister(),
}
err := fakeInformerFactory.KubeSphereSharedInformerFactory().Auditing().V1alpha1().Webhooks().Informer().GetIndexer().Add(webhook)
if err != nil {
panic(err)
}
req := &http.Request{}
u, err := url.Parse("http://139.198.121.143:32306//kapis/tenant.kubesphere.io/v1alpha2/workspaces")
if err != nil {
panic(err)
}
req.URL = u
req.Header = http.Header{}
req.Header.Add(iputil.XClientIP, "192.168.0.2")
req = req.WithContext(request.WithUser(req.Context(), &user.DefaultInfo{
Name: "admin",
Groups: []string{
"system",
},
}))
info := &request.RequestInfo{
RequestInfo: &k8srequest.RequestInfo{
IsResourceRequest: false,
Path: "/kapis/tenant.kubesphere.io/v1alpha2/workspaces",
Verb: "create",
APIGroup: "tenant.kubesphere.io",
APIVersion: "v1alpha2",
Resource: "workspaces",
Name: "test",
},
}
e := a.LogRequestObject(req, info)
expectedEvent := &v1alpha12.Event{
Event: audit.Event{
AuditID: e.AuditID,
Level: "RequestResponse",
Verb: "create",
Stage: "ResponseComplete",
User: v1.UserInfo{
Username: "admin",
Groups: []string{
"system",
},
Extra: make(map[string]v1.ExtraValue),
},
SourceIPs: []string{
"192.168.0.2",
},
RequestURI: "/kapis/tenant.kubesphere.io/v1alpha2/workspaces",
RequestReceivedTimestamp: e.RequestReceivedTimestamp,
ObjectRef: &audit.ObjectReference{
Resource: "workspaces",
Namespace: "",
Name: "test",
UID: "",
APIGroup: "tenant.kubesphere.io",
APIVersion: "v1alpha2",
ResourceVersion: "",
Subresource: "",
},
},
}
assert.Equal(t, expectedEvent, e)
}
func TestAuditing_LogResponseObject(t *testing.T) {
webhook := &auditingv1alpha1.Webhook{
TypeMeta: metav1.TypeMeta{
APIVersion: auditingv1alpha1.SchemeGroupVersion.String(),
},
ObjectMeta: metav1.ObjectMeta{
Name: "kube-auditing-webhook",
},
Spec: auditingv1alpha1.WebhookSpec{
AuditLevel: auditingv1alpha1.LevelMetadata,
K8sAuditingEnabled: true,
},
}
ksClient := fake.NewSimpleClientset()
k8sClient := fakek8s.NewSimpleClientset()
fakeInformerFactory := informers.NewInformerFactories(k8sClient, ksClient, nil, nil, nil, nil)
a := auditing{
webhookLister: fakeInformerFactory.KubeSphereSharedInformerFactory().Auditing().V1alpha1().Webhooks().Lister(),
}
err := fakeInformerFactory.KubeSphereSharedInformerFactory().Auditing().V1alpha1().Webhooks().Informer().GetIndexer().Add(webhook)
if err != nil {
panic(err)
}
req := &http.Request{}
u, err := url.Parse("http://139.198.121.143:32306//kapis/tenant.kubesphere.io/v1alpha2/workspaces")
if err != nil {
panic(err)
}
req.URL = u
req.Header = http.Header{}
req.Header.Add(iputil.XClientIP, "192.168.0.2")
req = req.WithContext(request.WithUser(req.Context(), &user.DefaultInfo{
Name: "admin",
Groups: []string{
"system",
},
}))
info := &request.RequestInfo{
RequestInfo: &k8srequest.RequestInfo{
IsResourceRequest: false,
Path: "/kapis/tenant.kubesphere.io/v1alpha2/workspaces",
Verb: "create",
APIGroup: "tenant.kubesphere.io",
APIVersion: "v1alpha2",
Resource: "workspaces",
Name: "test",
},
}
e := a.LogRequestObject(req, info)
resp := NewResponseCapture(httptest.NewRecorder())
resp.WriteHeader(200)
a.LogResponseObject(e, resp)
expectedEvent := &v1alpha12.Event{
Event: audit.Event{
Verb: "create",
AuditID: e.AuditID,
Level: "Metadata",
Stage: "ResponseComplete",
User: v1.UserInfo{
Username: "admin",
Groups: []string{
"system",
},
},
SourceIPs: []string{
"192.168.0.2",
},
ObjectRef: &audit.ObjectReference{
Resource: "workspaces",
Name: "test",
APIGroup: "tenant.kubesphere.io",
APIVersion: "v1alpha2",
},
RequestReceivedTimestamp: e.RequestReceivedTimestamp,
StageTimestamp: e.StageTimestamp,
RequestURI: "/kapis/tenant.kubesphere.io/v1alpha2/workspaces",
ResponseStatus: &metav1.Status{
Code: 200,
},
},
}
expectedBs, err := json.Marshal(expectedEvent)
if err != nil {
panic(err)
}
bs, err := json.Marshal(e)
if err != nil {
panic(err)
}
assert.EqualValues(t, string(expectedBs), string(bs))
}
func TestResponseCapture_WriteHeader(t *testing.T) {
record := httptest.NewRecorder()
resp := NewResponseCapture(record)
resp.WriteHeader(404)
assert.EqualValues(t, 404, resp.StatusCode())
assert.EqualValues(t, 404, record.Code)
}
func TestResponseCapture_Write(t *testing.T) {
record := httptest.NewRecorder()
resp := NewResponseCapture(record)
body := []byte("123")
_, err := resp.Write(body)
if err != nil {
panic(err)
}
assert.EqualValues(t, body, resp.Bytes())
assert.EqualValues(t, body, record.Body.Bytes())
}

43
pkg/apiserver/auditing/v1alpha1/event.go
View File

@@ -1,43 +0,0 @@
/*
Copyright 2020 KubeSphere Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package v1alpha1
import (
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apiserver/pkg/apis/audit"
)
type Event struct {
// Devops project
Devops string
// The workspace which this audit event happened
Workspace string
// The cluster which this audit event happened
Cluster string
// Message send to user.
Message string
audit.Event
}
type EventList struct {
Items []Event
}
type Object struct {
v1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
}

129
pkg/apiserver/auditing/webhook/backend.go Normal file
View File

@@ -0,0 +1,129 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package webhook
import (
"bytes"
"context"
"crypto/tls"
"net/http"
"time"
"k8s.io/klog/v2"
"kubesphere.io/kubesphere/pkg/apiserver/auditing/internal"
)
const (
GetSenderTimeout = time.Second
SendTimeout = time.Second * 3
DefaultSendersNum = 100
WebhookURL = "https://kube-auditing-webhook-svc.kubesphere-logging-system.svc:6443/audit/webhook/event"
)
type backend struct {
url string
senderCh chan interface{}
client http.Client
sendTimeout time.Duration
getSenderTimeout time.Duration
}
func NewBackend(url string, sendersNum int) internal.Backend {
b := backend{
url: url,
getSenderTimeout: GetSenderTimeout,
sendTimeout: SendTimeout,
}
if len(b.url) == 0 {
b.url = WebhookURL
}
num := sendersNum
if num == 0 {
num = DefaultSendersNum
}
b.senderCh = make(chan interface{}, num)
b.client = http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
},
Timeout: b.sendTimeout,
}
return &b
}
func (b *backend) ProcessEvents(events ...[]byte) {
go b.sendEvents(events...)
}
func (b *backend) sendEvents(events ...[]byte) {
ctx, cancel := context.WithTimeout(context.Background(), b.sendTimeout)
defer cancel()
stopCh := make(chan struct{})
skipReturnSender := false
send := func() {
ctx, cancel := context.WithTimeout(context.Background(), b.getSenderTimeout)
defer cancel()
select {
case <-ctx.Done():
klog.Error("Get auditing event sender timeout")
skipReturnSender = true
return
case b.senderCh <- struct{}{}:
}
start := time.Now()
defer func() {
stopCh <- struct{}{}
klog.V(8).Infof("send %d auditing events used %d", len(events), time.Since(start).Milliseconds())
}()
var body bytes.Buffer
for _, event := range events {
if _, err := body.Write(event); err != nil {
klog.Errorf("send auditing event error %s", err)
return
}
}
response, err := b.client.Post(b.url, "application/json", &body)
if err != nil {
klog.Errorf("send audit events error, %s", err)
return
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
klog.Errorf("send audit events error[%d]", response.StatusCode)
return
}
}
go send()
defer func() {
if !skipReturnSender {
<-b.senderCh
}
}()
select {
case <-ctx.Done():
klog.Error("send audit events timeout")
case <-stopCh:
}
}