This is a huge commit, it does following things:

1. refactor kubesphere dependency service client creation, we can
disable dependency by config
2. dependencies can be configured by configuration file
3. refactor cmd package using cobra.Command, so we can use hypersphere
to invoke command sepearately. Later we only need to build one image to
contains all kubesphere core components. One command to rule them all!
4. live reloading configuration currently not implemented

pkg/simple/client/mysql/OWNERS

@@ -0,0 +1,10 @@
+approvers:
+    - runzexia
+    - soulseen
+
+reviewers:
+    - runzexia
+    - soulseen
+
+labels:
+    - area/devops

pkg/simple/client/mysql/condition.go

@@ -0,0 +1,109 @@
+/*
+Copyright 2018 The KubeSphere Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mysql
+
+import (
+	"strings"
+
+	"github.com/gocraft/dbr"
+)
+
+const (
+	placeholder = "?"
+)
+
+type EqCondition struct {
+	dbr.Builder
+	Column string
+	Value  interface{}
+}
+
+// Copy From vendor/github.com/gocraft/dbr/condition.go:36
+func buildCmp(d dbr.Dialect, buf dbr.Buffer, pred string, column string, value interface{}) error {
+	buf.WriteString(d.QuoteIdent(column))
+	buf.WriteString(" ")
+	buf.WriteString(pred)
+	buf.WriteString(" ")
+	buf.WriteString(placeholder)
+
+	buf.WriteValue(value)
+	return nil
+}
+
+// And creates AND from a list of conditions
+func And(cond ...dbr.Builder) dbr.Builder {
+	return dbr.And(cond...)
+}
+
+// Or creates OR from a list of conditions
+func Or(cond ...dbr.Builder) dbr.Builder {
+	return dbr.Or(cond...)
+}
+
+func escape(str string) string {
+	return strings.Map(func(r rune) rune {
+		switch r {
+		case '%', '\'', '^', '[', ']', '!', '_':
+			return ' '
+		}
+		return r
+	}, str)
+}
+
+func Like(column string, value string) dbr.Builder {
+	value = "%" + strings.TrimSpace(escape(value)) + "%"
+	return dbr.BuildFunc(func(d dbr.Dialect, buf dbr.Buffer) error {
+		return buildCmp(d, buf, "LIKE", column, value)
+	})
+}
+
+// Eq is `=`.
+// When value is nil, it will be translated to `IS NULL`.
+// When value is a slice, it will be translated to `IN`.
+// Otherwise it will be translated to `=`.
+func Eq(column string, value interface{}) dbr.Builder {
+	return &EqCondition{
+		Builder: dbr.Eq(column, value),
+		Column:  column,
+		Value:   value,
+	}
+}
+
+// Neq is `!=`.
+// When value is nil, it will be translated to `IS NOT NULL`.
+// When value is a slice, it will be translated to `NOT IN`.
+// Otherwise it will be translated to `!=`.
+func Neq(column string, value interface{}) dbr.Builder {
+	return dbr.Neq(column, value)
+}
+
+// Gt is `>`.
+func Gt(column string, value interface{}) dbr.Builder {
+	return dbr.Gt(column, value)
+}
+
+// Gte is '>='.
+func Gte(column string, value interface{}) dbr.Builder {
+	return dbr.Gte(column, value)
+}
+
+// Lt is '<'.
+func Lt(column string, value interface{}) dbr.Builder {
+	return dbr.Lt(column, value)
+}
+
+// Lte is `<=`.
+func Lte(column string, value interface{}) dbr.Builder {
+	return dbr.Lte(column, value)
+}

pkg/simple/client/mysql/condition_test.go

@@ -0,0 +1,88 @@
+/*
+Copyright 2018 The KubeSphere Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mysql
+
+import (
+	"testing"
+
+	"github.com/gocraft/dbr"
+	"github.com/gocraft/dbr/dialect"
+	"github.com/stretchr/testify/assert"
+)
+
+// Ref: https://github.com/gocraft/dbr/blob/5d59a8b3aa915660960329efb3af5513e7a0db07/condition_test.go
+func TestCondition(t *testing.T) {
+	for _, test := range []struct {
+		cond  dbr.Builder
+		query string
+		value []interface{}
+	}{
+		{
+			cond:  Eq("col", 1),
+			query: "`col` = ?",
+			value: []interface{}{1},
+		},
+		{
+			cond:  Eq("col", nil),
+			query: "`col` IS NULL",
+			value: nil,
+		},
+		{
+			cond:  Eq("col", []int{}),
+			query: "0",
+			value: nil,
+		},
+		{
+			cond:  Neq("col", 1),
+			query: "`col` != ?",
+			value: []interface{}{1},
+		},
+		{
+			cond:  Neq("col", nil),
+			query: "`col` IS NOT NULL",
+			value: nil,
+		},
+		{
+			cond:  Gt("col", 1),
+			query: "`col` > ?",
+			value: []interface{}{1},
+		},
+		{
+			cond:  Gte("col", 1),
+			query: "`col` >= ?",
+			value: []interface{}{1},
+		},
+		{
+			cond:  Lt("col", 1),
+			query: "`col` < ?",
+			value: []interface{}{1},
+		},
+		{
+			cond:  Lte("col", 1),
+			query: "`col` <= ?",
+			value: []interface{}{1},
+		},
+		{
+			cond:  And(Lt("a", 1), Or(Gt("b", 2), Neq("c", 3))),
+			query: "(`a` < ?) AND ((`b` > ?) OR (`c` != ?))",
+			value: []interface{}{1, 2, 3},
+		},
+	} {
+		buf := dbr.NewBuffer()
+		err := test.cond.Build(dialect.MySQL, buf)
+		assert.NoError(t, err)
+		assert.Equal(t, test.query, buf.String())
+		assert.Equal(t, test.value, buf.Value())
+	}
+}

pkg/simple/client/mysql/db.go

@@ -0,0 +1,283 @@
+/*
+Copyright 2018 The KubeSphere Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mysql
+
+import (
+	"database/sql"
+	"fmt"
+	"strings"
+
+	_ "github.com/go-sql-driver/mysql"
+	"github.com/gocraft/dbr"
+)
+
+const (
+	DefaultSelectLimit = 200
+)
+
+func GetLimit(n uint64) uint64 {
+	if n < 0 {
+		n = 0
+	}
+	if n > DefaultSelectLimit {
+		n = DefaultSelectLimit
+	}
+	return n
+}
+
+func GetOffset(n uint64) uint64 {
+	if n < 0 {
+		n = 0
+	}
+	return n
+}
+
+type InsertHook func(query *InsertQuery)
+type UpdateHook func(query *UpdateQuery)
+type DeleteHook func(query *DeleteQuery)
+
+type Database struct {
+	*dbr.Session
+	InsertHook InsertHook
+	UpdateHook UpdateHook
+	DeleteHook DeleteHook
+}
+
+type SelectQuery struct {
+	*dbr.SelectBuilder
+	JoinCount int // for join filter
+}
+
+type InsertQuery struct {
+	*dbr.InsertBuilder
+	Hook InsertHook
+}
+
+type DeleteQuery struct {
+	*dbr.DeleteBuilder
+	Hook DeleteHook
+}
+
+type UpdateQuery struct {
+	*dbr.UpdateBuilder
+	Hook UpdateHook
+}
+
+type UpsertQuery struct {
+	table string
+	*dbr.Session
+	whereConds   map[string]string
+	upsertValues map[string]interface{}
+}
+
+// SelectQuery
+// Example: Select().From().Where().Limit().Offset().OrderDir().Load()
+//          Select().From().Where().Limit().Offset().OrderDir().LoadOne()
+//          Select().From().Where().Count()
+//          SelectAll().From().Where().Limit().Offset().OrderDir().Load()
+//          SelectAll().From().Where().Limit().Offset().OrderDir().LoadOne()
+//          SelectAll().From().Where().Count()
+
+func (db *Database) Select(columns ...string) *SelectQuery {
+	return &SelectQuery{db.Session.Select(columns...), 0}
+}
+
+func (db *Database) SelectBySql(query string, value ...interface{}) *SelectQuery {
+	return &SelectQuery{db.Session.SelectBySql(query, value...), 0}
+}
+
+func (db *Database) SelectAll(columns ...string) *SelectQuery {
+	return &SelectQuery{db.Session.Select("*"), 0}
+}
+
+func (b *SelectQuery) Join(table, on interface{}) *SelectQuery {
+	b.SelectBuilder.Join(table, on)
+	return b
+}
+
+func (b *SelectQuery) JoinAs(table string, alias string, on interface{}) *SelectQuery {
+	b.SelectBuilder.Join(dbr.I(table).As(alias), on)
+	return b
+}
+
+func (b *SelectQuery) From(table string) *SelectQuery {
+	b.SelectBuilder.From(table)
+	return b
+}
+
+func (b *SelectQuery) Where(query interface{}, value ...interface{}) *SelectQuery {
+	b.SelectBuilder.Where(query, value...)
+	return b
+}
+
+func (b *SelectQuery) GroupBy(col ...string) *SelectQuery {
+	b.SelectBuilder.GroupBy(col...)
+	return b
+}
+
+func (b *SelectQuery) Distinct() *SelectQuery {
+	b.SelectBuilder.Distinct()
+	return b
+}
+
+func (b *SelectQuery) Limit(n uint64) *SelectQuery {
+	n = GetLimit(n)
+	b.SelectBuilder.Limit(n)
+	return b
+}
+
+func (b *SelectQuery) Offset(n uint64) *SelectQuery {
+	n = GetLimit(n)
+	b.SelectBuilder.Offset(n)
+	return b
+}
+
+func (b *SelectQuery) OrderDir(col string, isAsc bool) *SelectQuery {
+	b.SelectBuilder.OrderDir(col, isAsc)
+	return b
+}
+
+func (b *SelectQuery) Load(value interface{}) (int, error) {
+	return b.SelectBuilder.Load(value)
+}
+
+func (b *SelectQuery) LoadOne(value interface{}) error {
+	return b.SelectBuilder.LoadOne(value)
+}
+
+func getColumns(dbrColumns []interface{}) string {
+	var columns []string
+	for _, column := range dbrColumns {
+		if c, ok := column.(string); ok {
+			columns = append(columns, c)
+		}
+	}
+	return strings.Join(columns, ", ")
+}
+
+func (b *SelectQuery) Count() (count uint32, err error) {
+	// cache SelectStmt
+	selectStmt := b.SelectStmt
+
+	limit := selectStmt.LimitCount
+	offset := selectStmt.OffsetCount
+	column := selectStmt.Column
+	isDistinct := selectStmt.IsDistinct
+	order := selectStmt.Order
+
+	b.SelectStmt.LimitCount = -1
+	b.SelectStmt.OffsetCount = -1
+	b.SelectStmt.Column = []interface{}{"COUNT(*)"}
+	b.SelectStmt.Order = []dbr.Builder{}
+
+	if isDistinct {
+		b.SelectStmt.Column = []interface{}{fmt.Sprintf("COUNT(DISTINCT %s)", getColumns(column))}
+		b.SelectStmt.IsDistinct = false
+	}
+
+	err = b.LoadOne(&count)
+	// fallback SelectStmt
+	selectStmt.LimitCount = limit
+	selectStmt.OffsetCount = offset
+	selectStmt.Column = column
+	selectStmt.IsDistinct = isDistinct
+	selectStmt.Order = order
+	b.SelectStmt = selectStmt
+	return
+}
+
+// InsertQuery
+// Example: InsertInto().Columns().Record().Exec()
+
+func (db *Database) InsertInto(table string) *InsertQuery {
+	return &InsertQuery{db.Session.InsertInto(table), db.InsertHook}
+}
+
+func (b *InsertQuery) Exec() (sql.Result, error) {
+	result, err := b.InsertBuilder.Exec()
+	if b.Hook != nil && err == nil {
+		defer b.Hook(b)
+	}
+	return result, err
+}
+
+func (b *InsertQuery) Columns(columns ...string) *InsertQuery {
+	b.InsertBuilder.Columns(columns...)
+	return b
+}
+
+func (b *InsertQuery) Record(structValue interface{}) *InsertQuery {
+	b.InsertBuilder.Record(structValue)
+	return b
+}
+
+// DeleteQuery
+// Example: DeleteFrom().Where().Limit().Exec()
+
+func (db *Database) DeleteFrom(table string) *DeleteQuery {
+	return &DeleteQuery{db.Session.DeleteFrom(table), db.DeleteHook}
+}
+
+func (b *DeleteQuery) Where(query interface{}, value ...interface{}) *DeleteQuery {
+	b.DeleteBuilder.Where(query, value...)
+	return b
+}
+
+func (b *DeleteQuery) Limit(n uint64) *DeleteQuery {
+	b.DeleteBuilder.Limit(n)
+	return b
+}
+
+func (b *DeleteQuery) Exec() (sql.Result, error) {
+	result, err := b.DeleteBuilder.Exec()
+	if b.Hook != nil && err == nil {
+		defer b.Hook(b)
+	}
+	return result, err
+}
+
+// UpdateQuery
+// Example: Update().Set().Where().Exec()
+
+func (db *Database) Update(table string) *UpdateQuery {
+	return &UpdateQuery{db.Session.Update(table), db.UpdateHook}
+}
+
+func (b *UpdateQuery) Exec() (sql.Result, error) {
+	result, err := b.UpdateBuilder.Exec()
+	if b.Hook != nil && err == nil {
+		defer b.Hook(b)
+	}
+	return result, err
+}
+
+func (b *UpdateQuery) Set(column string, value interface{}) *UpdateQuery {
+	b.UpdateBuilder.Set(column, value)
+	return b
+}
+
+func (b *UpdateQuery) SetMap(m map[string]interface{}) *UpdateQuery {
+	b.UpdateBuilder.SetMap(m)
+	return b
+}
+
+func (b *UpdateQuery) Where(query interface{}, value ...interface{}) *UpdateQuery {
+	b.UpdateBuilder.Where(query, value...)
+	return b
+}
+
+func (b *UpdateQuery) Limit(n uint64) *UpdateQuery {
+	b.UpdateBuilder.Limit(n)
+	return b
+}

pkg/simple/client/mysql/error.go

@@ -0,0 +1,31 @@
+/*
+Copyright 2018 The KubeSphere Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mysql
+
+import (
+	"github.com/gocraft/dbr"
+)
+
+// package errors
+var (
+	ErrNotFound           = dbr.ErrNotFound
+	ErrNotSupported       = dbr.ErrNotSupported
+	ErrTableNotSpecified  = dbr.ErrTableNotSpecified
+	ErrColumnNotSpecified = dbr.ErrColumnNotSpecified
+	ErrInvalidPointer     = dbr.ErrInvalidPointer
+	ErrPlaceholderCount   = dbr.ErrPlaceholderCount
+	ErrInvalidSliceLength = dbr.ErrInvalidSliceLength
+	ErrCantConvertToTime  = dbr.ErrCantConvertToTime
+	ErrInvalidTimestring  = dbr.ErrInvalidTimestring
+)

pkg/simple/client/mysql/event.go

@@ -0,0 +1,55 @@
+/*
+Copyright 2018 The KubeSphere Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mysql
+
+import (
+	"github.com/golang/glog"
+)
+
+// EventReceiver is a sentinel EventReceiver; use it if the caller doesn't supply one
+type EventReceiver struct{}
+
+// Event receives a simple notification when various events occur
+func (n *EventReceiver) Event(eventName string) {
+
+}
+
+// EventKv receives a notification when various events occur along with
+// optional key/value data
+func (n *EventReceiver) EventKv(eventName string, kvs map[string]string) {
+}
+
+// EventErr receives a notification of an error if one occurs
+func (n *EventReceiver) EventErr(eventName string, err error) error {
+	return err
+}
+
+// EventErrKv receives a notification of an error if one occurs along with
+// optional key/value data
+func (n *EventReceiver) EventErrKv(eventName string, err error, kvs map[string]string) error {
+	glog.Errorf("%+v", err)
+	glog.Errorf("%s: %+v", eventName, kvs)
+	return err
+}
+
+// Timing receives the time an event took to happen
+func (n *EventReceiver) Timing(eventName string, nanoseconds int64) {
+
+}
+
+// TimingKv receives the time an event took to happen along with optional key/value data
+func (n *EventReceiver) TimingKv(eventName string, nanoseconds int64, kvs map[string]string) {
+	// TODO: Change logger level to debug
+	glog.Infof("%s spend %.2fms: %+v", eventName, float32(nanoseconds)/1000000, kvs)
+}

pkg/simple/client/mysql/mysql.go

@@ -0,0 +1,82 @@
+/*
+Copyright 2018 The KubeSphere Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mysql
+
+import (
+	"fmt"
+	"github.com/gocraft/dbr"
+	"k8s.io/klog"
+)
+
+type MySQLClient struct {
+	database *Database
+}
+
+func NewMySQLClient(options *MySQLOptions, stopCh <-chan struct{}) (*MySQLClient, error) {
+	var m MySQLClient
+
+	conn, err := dbr.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/devops?parseTime=1&multiStatements=1&charset=utf8mb4&collation=utf8mb4_unicode_ci", options.Username, options.Password, options.Host), nil)
+	if err != nil {
+		klog.Error("unable to connect to mysql", err)
+		return nil, err
+	}
+
+	conn.SetMaxIdleConns(options.MaxIdleConnections)
+	conn.SetConnMaxLifetime(options.MaxConnectionLifeTime)
+	conn.SetMaxOpenConns(options.MaxOpenConnections)
+
+	m.database = &Database{
+		Session: conn.NewSession(nil),
+	}
+
+	go func() {
+		<-stopCh
+		if err := conn.Close(); err != nil {
+			klog.Warning("error happened during closing mysql connection", err)
+		}
+	}()
+
+	return &m, nil
+}
+
+func NewMySQLClientOrDie(options *MySQLOptions, stopCh <-chan struct{}) *MySQLClient {
+	var m MySQLClient
+
+	conn, err := dbr.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/devops?parseTime=1&multiStatements=1&charset=utf8mb4&collation=utf8mb4_unicode_ci", options.Username, options.Password, options.Host), nil)
+	if err != nil {
+		klog.Error("unable to connect to mysql", err)
+		panic(err)
+	}
+
+	conn.SetMaxIdleConns(options.MaxIdleConnections)
+	conn.SetConnMaxLifetime(options.MaxConnectionLifeTime)
+	conn.SetMaxOpenConns(options.MaxOpenConnections)
+
+	m.database = &Database{
+		Session: conn.NewSession(nil),
+	}
+
+	go func() {
+		<-stopCh
+		if err := conn.Close(); err != nil {
+			klog.Warning("error happened during closing mysql connection", err)
+		}
+	}()
+
+	return &m
+}
+
+func (m *MySQLClient) Database() *Database {
+	return m.database
+}

pkg/simple/client/mysql/options.go

@@ -0,0 +1,43 @@
+package mysql
+
+import (
+	"github.com/spf13/pflag"
+	reflectutils "kubesphere.io/kubesphere/pkg/utils/reflectutils"
+	"time"
+)
+
+type MySQLOptions struct {
+	Host                  string        `json:"host,omitempty" yaml:"host,omitempty" description:"MySQL service host address"`
+	Username              string        `json:"username,omitempty" yaml:"username,omitempty"`
+	Password              string        `json:"-" yaml:"password,omitempty"`
+	MaxIdleConnections    int           `json:"maxIdleConnections,omitempty" yaml:"maxIdleConnections,omitempty"`
+	MaxOpenConnections    int           `json:"maxOpenConnections,omitempty" yaml:"maxOpenConnections,omitempty"`
+	MaxConnectionLifeTime time.Duration `json:"maxConnectionLifeTime,omitempty" yaml:"maxConnectionLifeTime,omitempty"`
+}
+
+// NewMySQLOptions create a `zero` value instance
+func NewMySQLOptions() *MySQLOptions {
+	return &MySQLOptions{}
+}
+
+func (m *MySQLOptions) Validate() []error {
+	errors := []error{}
+
+	return errors
+}
+
+func (m *MySQLOptions) ApplyTo(options *MySQLOptions) {
+	reflectutils.Override(options, m)
+}
+
+func (m *MySQLOptions) AddFlags(fs *pflag.FlagSet) {
+
+	fs.StringVar(&m.Host, "mysql-host", m.Host, ""+
+		"MySQL service host address. If left blank, following options will be ignored.")
+
+	fs.StringVar(&m.Username, "mysql-username", m.Username, ""+
+		"Username for access to mysql service.")
+
+	fs.StringVar(&m.Password, "mysql-password", m.Password, ""+
+		"Password for access to mysql, should be used pair with password.")
+}