diff --git a/.travis.yml b/.travis.yml index 6dbe0187f..a535ccfde 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,12 @@ before_script: - bash hack/install_kubebuilder.sh script: - - make all + - diff -u <(echo -n) <(gofmt -d ./pkg ./cmd ./tools) + - make openapi + - make all + +install: + - go get golang.org/x/lint/golint deploy: skip_cleanup: true diff --git a/Makefile b/Makefile index 4812290dc..73bc09dfd 100644 --- a/Makefile +++ b/Makefile @@ -39,7 +39,7 @@ define ALL_HELP_INFO # debugging tools like delve. endef .PHONY: all -all: test ks-apiserver ks-apigateway ks-iam controller-manager +all: hypersphere ks-apiserver ks-apigateway ks-iam controller-manager # Build ks-apiserver binary ks-apiserver: test @@ -57,9 +57,13 @@ ks-iam: test controller-manager: test hack/gobuild.sh cmd/controller-manager +# Build hypersphere binary +hypersphere: test + hack/gobuild.sh cmd/hypersphere + # Run go fmt against code fmt: generate - go fmt ./pkg/... ./cmd/... + gofmt -w ./pkg ./cmd ./tools ./api # Run go vet against code vet: generate diff --git a/build/hypersphere/Dockerfile b/build/hypersphere/Dockerfile new file mode 100644 index 000000000..937a94152 --- /dev/null +++ b/build/hypersphere/Dockerfile @@ -0,0 +1,18 @@ +# Copyright 2018 The KubeSphere Authors. All rights reserved. +# Use of this source code is governed by a Apache license +# that can be found in the LICENSE file. + +# Copyright 2018 The KubeSphere Authors. All rights reserved. +# Use of this source code is governed by a Apache license +# that can be found in the LICENSE file. +FROM golang:1.12 as hypersphere-builder + +COPY / /go/src/kubesphere.io/kubesphere + +WORKDIR /go/src/kubesphere.io/kubesphere +RUN CGO_ENABLED=0 GO111MODULE=on GOOS=linux GOARCH=amd64 GOFLAGS=-mod=vendor go build -i -ldflags '-w -s' -o hypersphere cmd/hypersphere/hypersphere.go + +FROM alpine:3.9 +RUN apk add --update ca-certificates && update-ca-certificates +COPY --from=ks-apiserver-builder /go/src/kubesphere.io/kubesphere/hypersphere /usr/local/bin/ +CMD ["sh"] diff --git a/cmd/controller-manager/app/controllers.go b/cmd/controller-manager/app/controllers.go index a121681a5..afc568385 100644 --- a/cmd/controller-manager/app/controllers.go +++ b/cmd/controller-manager/app/controllers.go @@ -25,7 +25,6 @@ import ( "kubesphere.io/kubesphere/pkg/controller/destinationrule" "kubesphere.io/kubesphere/pkg/controller/job" "kubesphere.io/kubesphere/pkg/controller/s2ibinary" - "kubesphere.io/kubesphere/pkg/controller/s2irun" //"kubesphere.io/kubesphere/pkg/controller/job" @@ -112,7 +111,9 @@ func AddControllers(mgr manager.Manager, cfg *rest.Config, stopCh <-chan struct{ jobController := job.NewJobController(informerFactory.Batch().V1().Jobs(), kubeClient) - s2iBinaryController := s2ibinary.NewController(kubesphereclient, kubeClient, kubesphereInformer.Devops().V1alpha1().S2iBinaries()) + s2iBinaryController := s2ibinary.NewController(kubesphereclient, + kubeClient, + kubesphereInformer.Devops().V1alpha1().S2iBinaries()) s2iRunController := s2irun.NewController(kubesphereclient, s2iclient, kubeClient, kubesphereInformer.Devops().V1alpha1().S2iBinaries(), diff --git a/cmd/controller-manager/app/options/options.go b/cmd/controller-manager/app/options/options.go new file mode 100644 index 000000000..08f13baae --- /dev/null +++ b/cmd/controller-manager/app/options/options.go @@ -0,0 +1,63 @@ +package options + +import ( + "flag" + cliflag "k8s.io/component-base/cli/flag" + "k8s.io/klog" + kubesphereconfig "kubesphere.io/kubesphere/pkg/server/config" + "kubesphere.io/kubesphere/pkg/simple/client/devops" + "kubesphere.io/kubesphere/pkg/simple/client/k8s" + "kubesphere.io/kubesphere/pkg/simple/client/s2is3" + "strings" +) + +type KubeSphereControllerManagerOptions struct { + KubernetesOptions *k8s.KubernetesOptions + DevopsOptions *devops.DevopsOptions + S3Options *s2is3.S3Options +} + +func NewKubeSphereControllerManagerOptions() *KubeSphereControllerManagerOptions { + s := &KubeSphereControllerManagerOptions{ + KubernetesOptions: k8s.NewKubernetesOptions(), + DevopsOptions: devops.NewDevopsOptions(), + S3Options: s2is3.NewS3Options(), + } + + return s +} + +func (s *KubeSphereControllerManagerOptions) ApplyTo(conf *kubesphereconfig.Config) { + s.S3Options.ApplyTo(conf.S3Options) + s.KubernetesOptions.ApplyTo(conf.KubernetesOptions) + s.DevopsOptions.ApplyTo(conf.DevopsOptions) + +} + +func (s *KubeSphereControllerManagerOptions) Flags() cliflag.NamedFlagSets { + fss := cliflag.NamedFlagSets{} + + s.KubernetesOptions.AddFlags(fss.FlagSet("kubernetes")) + s.DevopsOptions.AddFlags(fss.FlagSet("devops")) + s.S3Options.AddFlags(fss.FlagSet("s3")) + + fs := fss.FlagSet("klog") + local := flag.NewFlagSet("klog", flag.ExitOnError) + klog.InitFlags(local) + local.VisitAll(func(fl *flag.Flag) { + fl.Name = strings.Replace(fl.Name, "_", "-", -1) + fs.AddGoFlag(fl) + }) + + return fss +} + +func (s *KubeSphereControllerManagerOptions) Validate() []error { + var errs []error + + errs = append(errs, s.DevopsOptions.Validate()...) + errs = append(errs, s.KubernetesOptions.Validate()...) + errs = append(errs, s.S3Options.Validate()...) + + return errs +} diff --git a/cmd/controller-manager/app/server.go b/cmd/controller-manager/app/server.go new file mode 100644 index 000000000..4b9a9c5e0 --- /dev/null +++ b/cmd/controller-manager/app/server.go @@ -0,0 +1,153 @@ +/* + + Copyright 2019 The KubeSphere Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +*/ + +package app + +import ( + "fmt" + "github.com/spf13/cobra" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + cliflag "k8s.io/component-base/cli/flag" + "k8s.io/klog" + "kubesphere.io/kubesphere/cmd/controller-manager/app/options" + "kubesphere.io/kubesphere/pkg/apis" + "kubesphere.io/kubesphere/pkg/controller" + controllerconfig "kubesphere.io/kubesphere/pkg/server/config" + "kubesphere.io/kubesphere/pkg/simple/client" + "kubesphere.io/kubesphere/pkg/utils/term" + "os" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/runtime/signals" +) + +func NewControllerManagerCommand() *cobra.Command { + s := options.NewKubeSphereControllerManagerOptions() + + cmd := &cobra.Command{ + Use: "controller-manager", + Long: `KubeSphere controller manager is a daemon that`, + Run: func(cmd *cobra.Command, args []string) { + + err := controllerconfig.Load() + if err != nil { + klog.Fatal(err) + os.Exit(1) + } + + err = Complete(s) + if err != nil { + os.Exit(1) + } + + if errs := s.Validate(); len(errs) != 0 { + klog.Error(utilerrors.NewAggregate(errs)) + os.Exit(1) + } + + if err = Run(s, signals.SetupSignalHandler()); err != nil { + os.Exit(1) + } + }, + } + + fs := cmd.Flags() + namedFlagSets := s.Flags() + + for _, f := range namedFlagSets.FlagSets { + fs.AddFlagSet(f) + } + + usageFmt := "Usage:\n %s\n" + cols, _, _ := term.TerminalSize(cmd.OutOrStdout()) + cmd.SetHelpFunc(func(cmd *cobra.Command, args []string) { + fmt.Fprintf(cmd.OutOrStdout(), "%s\n\n"+usageFmt, cmd.Long, cmd.UseLine()) + cliflag.PrintSections(cmd.OutOrStdout(), namedFlagSets, cols) + }) + return cmd +} + +func Complete(s *options.KubeSphereControllerManagerOptions) error { + conf := controllerconfig.Get() + + conf.Apply(&controllerconfig.Config{ + DevopsOptions: s.DevopsOptions, + KubernetesOptions: s.KubernetesOptions, + S3Options: s.S3Options, + }) + + s = &options.KubeSphereControllerManagerOptions{ + KubernetesOptions: conf.KubernetesOptions, + DevopsOptions: conf.DevopsOptions, + S3Options: conf.S3Options, + } + + return nil +} + +func CreateClientSet(s *options.KubeSphereControllerManagerOptions, stopCh <-chan struct{}) error { + csop := &client.ClientSetOptions{} + + csop.SetKubernetesOptions(s.KubernetesOptions). + SetDevopsOptions(s.DevopsOptions). + SetS3Options(s.S3Options) + client.NewClientSetFactory(csop, stopCh) + + return nil +} + +func Run(s *options.KubeSphereControllerManagerOptions, stopCh <-chan struct{}) error { + err := CreateClientSet(s, stopCh) + if err != nil { + klog.Error(err) + return err + } + + config := client.ClientSets().K8s().Config() + + klog.Info("setting up manager") + mgr, err := manager.New(config, manager.Options{}) + if err != nil { + klog.Error(err, "unable to set up overall controller manager") + return err + } + + klog.Info("setting up scheme") + if err := apis.AddToScheme(mgr.GetScheme()); err != nil { + klog.Error(err, "unable add APIs to scheme") + return err + } + + klog.Info("Setting up controllers") + if err := controller.AddToManager(mgr); err != nil { + klog.Error(err, "unable to register controllers to the manager") + return err + } + + if err := AddControllers(mgr, config, stopCh); err != nil { + klog.Error(err, "unable to register controllers to the manager") + return err + } + + klog.Info("Starting the Cmd.") + if err := mgr.Start(stopCh); err != nil { + klog.Error(err, "unable to run the manager") + return err + } + + return nil +} diff --git a/cmd/controller-manager/controller-manager.go b/cmd/controller-manager/controller-manager.go index bc61811ae..62035008a 100644 --- a/cmd/controller-manager/controller-manager.go +++ b/cmd/controller-manager/controller-manager.go @@ -1,83 +1,14 @@ -/* - - Copyright 2019 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -*/ - package main import ( - "flag" - "k8s.io/klog" "kubesphere.io/kubesphere/cmd/controller-manager/app" - "kubesphere.io/kubesphere/pkg/apis" - "kubesphere.io/kubesphere/pkg/controller" - "kubesphere.io/kubesphere/pkg/simple/client/k8s" "os" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/runtime/signals" ) -var ( - masterURL string - metricsAddr string -) - -func init() { - flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.") -} - func main() { - flag.Parse() + command := app.NewControllerManagerCommand() - cfg, err := k8s.Config() - if err != nil { - klog.Error(err, "failed to build kubeconfig") + if err := command.Execute(); err != nil { os.Exit(1) } - - stopCh := signals.SetupSignalHandler() - - klog.Info("setting up manager") - mgr, err := manager.New(cfg, manager.Options{}) - if err != nil { - klog.Error(err, "unable to set up overall controller manager") - os.Exit(1) - } - - klog.Info("setting up scheme") - if err := apis.AddToScheme(mgr.GetScheme()); err != nil { - klog.Error(err, "unable add APIs to scheme") - os.Exit(1) - } - - klog.Info("Setting up controllers") - if err := controller.AddToManager(mgr); err != nil { - klog.Error(err, "unable to register controllers to the manager") - os.Exit(1) - } - - if err := app.AddControllers(mgr, cfg, stopCh); err != nil { - klog.Error(err, "unable to register controllers to the manager") - os.Exit(1) - } - - klog.Info("Starting the Cmd.") - if err := mgr.Start(stopCh); err != nil { - klog.Error(err, "unable to run the manager") - os.Exit(1) - } - } diff --git a/cmd/hypersphere/hypersphere.go b/cmd/hypersphere/hypersphere.go new file mode 100644 index 000000000..13ad5cdfe --- /dev/null +++ b/cmd/hypersphere/hypersphere.go @@ -0,0 +1,72 @@ +package main + +import ( + goflag "flag" + cliflag "k8s.io/component-base/cli/flag" + "path/filepath" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + controllermanager "kubesphere.io/kubesphere/cmd/controller-manager/app" + ksapiserver "kubesphere.io/kubesphere/cmd/ks-apiserver/app" + ksaiam "kubesphere.io/kubesphere/cmd/ks-iam/app" + "os" +) + +func main() { + hypersphereCommand, allCommandFns := NewHyperSphereCommand() + + pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) + pflag.CommandLine.AddGoFlagSet(goflag.CommandLine) + + basename := filepath.Base(os.Args[0]) + if err := commandFor(basename, hypersphereCommand, allCommandFns).Execute(); err != nil { + os.Exit(1) + } +} + +func commandFor(basename string, defaultCommand *cobra.Command, commands []func() *cobra.Command) *cobra.Command { + for _, commandFn := range commands { + command := commandFn() + if command.Name() == basename { + return command + } + + for _, alias := range command.Aliases { + if alias == basename { + return command + } + } + } + + return defaultCommand +} + +func NewHyperSphereCommand() (*cobra.Command, []func() *cobra.Command) { + apiserver := func() *cobra.Command { return ksapiserver.NewAPIServerCommand() } + controllermanager := func() *cobra.Command { return controllermanager.NewControllerManagerCommand() } + iam := func() *cobra.Command { return ksaiam.NewAPIServerCommand() } + + commandFns := []func() *cobra.Command{ + apiserver, + controllermanager, + iam, + } + + cmd := &cobra.Command{ + Use: "hypersphere", + Short: "Request a new project", + Run: func(cmd *cobra.Command, args []string) { + if len(args) != 0 { + cmd.Help() + os.Exit(0) + } + }, + } + + for i := range commandFns { + cmd.AddCommand(commandFns[i]()) + } + + return cmd, commandFns +} diff --git a/cmd/ks-apiserver/app/options/options.go b/cmd/ks-apiserver/app/options/options.go index c22998fe8..9cbdf60e4 100644 --- a/cmd/ks-apiserver/app/options/options.go +++ b/cmd/ks-apiserver/app/options/options.go @@ -1,45 +1,66 @@ package options import ( - "github.com/spf13/pflag" + "flag" + cliflag "k8s.io/component-base/cli/flag" + "k8s.io/klog" genericoptions "kubesphere.io/kubesphere/pkg/options" + "kubesphere.io/kubesphere/pkg/simple/client/devops" + "kubesphere.io/kubesphere/pkg/simple/client/k8s" + "kubesphere.io/kubesphere/pkg/simple/client/mysql" + "kubesphere.io/kubesphere/pkg/simple/client/prometheus" + "kubesphere.io/kubesphere/pkg/simple/client/servicemesh" + "kubesphere.io/kubesphere/pkg/simple/client/sonarqube" + "strings" ) type ServerRunOptions struct { GenericServerRunOptions *genericoptions.ServerRunOptions - // istio pilot discovery service url - IstioPilotServiceURL string + KubernetesOptions *k8s.KubernetesOptions - // jaeger query service url - JaegerQueryServiceUrl string + DevopsOptions *devops.DevopsOptions - // prometheus service url for servicemesh metrics - ServicemeshPrometheusServiceUrl string + SonarQubeOptions *sonarqube.SonarQubeOptions - // openpitrix api gateway service url - OpenPitrixServer string + ServiceMeshOptions *servicemesh.ServiceMeshOptions - // openpitrix service token - OpenPitrixProxyToken string + MySQLOptions *mysql.MySQLOptions + + MonitoringOptions *prometheus.PrometheusOptions } func NewServerRunOptions() *ServerRunOptions { s := ServerRunOptions{ GenericServerRunOptions: genericoptions.NewServerRunOptions(), - IstioPilotServiceURL: "http://istio-pilot.istio-system.svc:8080/version", - JaegerQueryServiceUrl: "http://jaeger-query.istio-system.svc:16686/jaeger", + KubernetesOptions: k8s.NewKubernetesOptions(), + DevopsOptions: devops.NewDevopsOptions(), + SonarQubeOptions: sonarqube.NewSonarQubeOptions(), + ServiceMeshOptions: servicemesh.NewServiceMeshOptions(), + MySQLOptions: mysql.NewMySQLOptions(), + MonitoringOptions: prometheus.NewPrometheusOptions(), } return &s } -func (s *ServerRunOptions) AddFlags(fs *pflag.FlagSet) { +func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) { - s.GenericServerRunOptions.AddFlags(fs) + s.GenericServerRunOptions.AddFlags(fss.FlagSet("generic")) + s.KubernetesOptions.AddFlags(fss.FlagSet("kubernetes")) + s.DevopsOptions.AddFlags(fss.FlagSet("devops")) + s.SonarQubeOptions.AddFlags(fss.FlagSet("sonarqube")) + s.ServiceMeshOptions.AddFlags(fss.FlagSet("servicemesh")) + s.MonitoringOptions.AddFlags(fss.FlagSet("monitoring")) - fs.StringVar(&s.IstioPilotServiceURL, "istio-pilot-service-url", "http://istio-pilot.istio-system.svc:8080/version", "istio pilot discovery service url") - fs.StringVar(&s.JaegerQueryServiceUrl, "jaeger-query-service-url", "http://jaeger-query.istio-system.svc:16686/jaeger", "jaeger query service url") - fs.StringVar(&s.ServicemeshPrometheusServiceUrl, "servicemesh-prometheus-service-url", "http://prometheus-k8s-system.kubesphere-monitoring-system.svc:9090", "prometheus service for servicemesh") + fs := fss.FlagSet("klog") + local := flag.NewFlagSet("klog", flag.ExitOnError) + klog.InitFlags(local) + local.VisitAll(func(fl *flag.Flag) { + fl.Name = strings.Replace(fl.Name, "_", "-", -1) + fs.AddGoFlag(fl) + }) + + return fss } diff --git a/cmd/ks-apiserver/app/options/validation.go b/cmd/ks-apiserver/app/options/validation.go new file mode 100644 index 000000000..fa677a592 --- /dev/null +++ b/cmd/ks-apiserver/app/options/validation.go @@ -0,0 +1,14 @@ +package options + +func (s *ServerRunOptions) Validate() []error { + var errors []error + + errors = append(errors, s.DevopsOptions.Validate()...) + errors = append(errors, s.KubernetesOptions.Validate()...) + errors = append(errors, s.MySQLOptions.Validate()...) + errors = append(errors, s.ServiceMeshOptions.Validate()...) + errors = append(errors, s.MonitoringOptions.Validate()...) + errors = append(errors, s.SonarQubeOptions.Validate()...) + + return errors +} diff --git a/cmd/ks-apiserver/app/server.go b/cmd/ks-apiserver/app/server.go index 2d9d1f9d7..7df2c7fa0 100644 --- a/cmd/ks-apiserver/app/server.go +++ b/cmd/ks-apiserver/app/server.go @@ -18,113 +18,110 @@ package app import ( - goflag "flag" "fmt" - "github.com/golang/glog" - "github.com/json-iterator/go" kconfig "github.com/kiali/kiali/config" "github.com/spf13/cobra" - "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/runtime/schema" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + cliflag "k8s.io/component-base/cli/flag" + "k8s.io/klog" "kubesphere.io/kubesphere/cmd/ks-apiserver/app/options" "kubesphere.io/kubesphere/pkg/apiserver/runtime" "kubesphere.io/kubesphere/pkg/apiserver/servicemesh/tracing" "kubesphere.io/kubesphere/pkg/filter" "kubesphere.io/kubesphere/pkg/informers" - "kubesphere.io/kubesphere/pkg/models/devops" logging "kubesphere.io/kubesphere/pkg/models/log" "kubesphere.io/kubesphere/pkg/server" - "kubesphere.io/kubesphere/pkg/signals" - "kubesphere.io/kubesphere/pkg/simple/client/admin_jenkins" - "kubesphere.io/kubesphere/pkg/simple/client/devops_mysql" - "log" + apiserverconfig "kubesphere.io/kubesphere/pkg/server/config" + "kubesphere.io/kubesphere/pkg/simple/client" + "kubesphere.io/kubesphere/pkg/utils/signals" + "kubesphere.io/kubesphere/pkg/utils/term" "net/http" ) -var jsonIter = jsoniter.ConfigCompatibleWithStandardLibrary - func NewAPIServerCommand() *cobra.Command { s := options.NewServerRunOptions() cmd := &cobra.Command{ Use: "ks-apiserver", - Long: `The KubeSphere API server validates and configures data -for the api objects. The API Server services REST operations and provides the frontend to the + Long: `The KubeSphere API server validates and configures data for the api objects. +The API Server services REST operations and provides the frontend to the cluster's shared state through which all other components interact.`, RunE: func(cmd *cobra.Command, args []string) error { - return Run(s) + err := apiserverconfig.Load() + if err != nil { + return err + } + + err = Complete(s) + if err != nil { + return err + } + + if errs := s.Validate(); len(errs) != 0 { + return utilerrors.NewAggregate(errs) + } + + return Run(s, signals.SetupSignalHandler()) }, } - s.AddFlags(cmd.Flags()) - cmd.Flags().AddGoFlagSet(goflag.CommandLine) - glog.CopyStandardLogTo("INFO") + fs := cmd.Flags() + namedFlagSets := s.Flags() + for _, f := range namedFlagSets.FlagSets { + fs.AddFlagSet(f) + } + + usageFmt := "Usage:\n %s\n" + cols, _, _ := term.TerminalSize(cmd.OutOrStdout()) + cmd.SetHelpFunc(func(cmd *cobra.Command, args []string) { + fmt.Fprintf(cmd.OutOrStdout(), "%s\n\n"+usageFmt, cmd.Long, cmd.UseLine()) + cliflag.PrintSections(cmd.OutOrStdout(), namedFlagSets, cols) + }) return cmd } -func Run(s *options.ServerRunOptions) error { +func Run(s *options.ServerRunOptions, stopCh <-chan struct{}) error { - pflag.VisitAll(func(flag *pflag.Flag) { - log.Printf("FLAG: --%s=%q", flag.Name, flag.Value) - }) - - var err error - - waitForResourceSync() - - container := runtime.Container - container.DoNotRecover(false) - container.Filter(filter.Logging) - container.RecoverHandler(server.LogStackOnRecover) - for _, webservice := range container.RegisteredWebServices() { - for _, route := range webservice.Routes() { - log.Println(route.Method, route.Path) - } + err := CreateClientSet(apiserverconfig.Get(), stopCh) + if err != nil { + return err + } + + err = WaitForResourceSync(stopCh) + if err != nil { + return err } - initializeAdminJenkins() - initializeDevOpsDatabase() - initializeESClientConfig() initializeServicemeshConfig(s) - if s.GenericServerRunOptions.InsecurePort != 0 { - log.Printf("Server listening on %d.", s.GenericServerRunOptions.InsecurePort) - err = http.ListenAndServe(fmt.Sprintf("%s:%d", s.GenericServerRunOptions.BindAddress, s.GenericServerRunOptions.InsecurePort), container) + initializeESClientConfig() + + err = CreateAPIServer(s) + if err != nil { + return err } - if s.GenericServerRunOptions.SecurePort != 0 && len(s.GenericServerRunOptions.TlsCertFile) > 0 && len(s.GenericServerRunOptions.TlsPrivateKey) > 0 { - log.Printf("Server listening on %d.", s.GenericServerRunOptions.SecurePort) - err = http.ListenAndServeTLS(fmt.Sprintf("%s:%d", s.GenericServerRunOptions.BindAddress, s.GenericServerRunOptions.SecurePort), s.GenericServerRunOptions.TlsCertFile, s.GenericServerRunOptions.TlsPrivateKey, container) - } - - return err -} - -func initializeAdminJenkins() { - devops.JenkinsInit() - admin_jenkins.Client() -} - -func initializeDevOpsDatabase() { - devops_mysql.OpenDatabase() + return nil } func initializeServicemeshConfig(s *options.ServerRunOptions) { // Initialize kiali config config := kconfig.NewConfig() - tracing.JaegerQueryUrl = s.JaegerQueryServiceUrl + tracing.JaegerQueryUrl = s.ServiceMeshOptions.JaegerQueryHost // Exclude system namespaces config.API.Namespaces.Exclude = []string{"istio-system", "kubesphere*", "kube*"} config.InCluster = true // Set default prometheus service url - config.ExternalServices.PrometheusServiceURL = s.ServicemeshPrometheusServiceUrl + config.ExternalServices.PrometheusServiceURL = s.ServiceMeshOptions.ServicemeshPrometheusHost config.ExternalServices.PrometheusCustomMetricsURL = config.ExternalServices.PrometheusServiceURL // Set istio pilot discovery service url - config.ExternalServices.Istio.UrlServiceVersion = s.IstioPilotServiceURL + config.ExternalServices.Istio.UrlServiceVersion = s.ServiceMeshOptions.IstioPilotHost kconfig.Set(config) } @@ -134,7 +131,7 @@ func initializeESClientConfig() { // List all outputs outputs, err := logging.GetFluentbitOutputFromConfigMap() if err != nil { - glog.Errorln(err) + klog.Errorln(err) return } @@ -147,54 +144,164 @@ func initializeESClientConfig() { } } -func waitForResourceSync() { - stopChan := signals.SetupSignalHandler() +// +func CreateAPIServer(s *options.ServerRunOptions) error { + var err error + + container := runtime.Container + container.DoNotRecover(false) + container.Filter(filter.Logging) + container.RecoverHandler(server.LogStackOnRecover) + + // install config api + apiserverconfig.InstallAPI(container) + + for _, webservice := range container.RegisteredWebServices() { + for _, route := range webservice.Routes() { + klog.V(0).Info(route.Method, route.Path) + } + } + + if s.GenericServerRunOptions.InsecurePort != 0 { + err = http.ListenAndServe(fmt.Sprintf("%s:%d", s.GenericServerRunOptions.BindAddress, s.GenericServerRunOptions.InsecurePort), container) + if err != nil { + klog.Infof("Server listening on %d.", s.GenericServerRunOptions.InsecurePort) + } + } + + if s.GenericServerRunOptions.SecurePort != 0 && len(s.GenericServerRunOptions.TlsCertFile) > 0 && len(s.GenericServerRunOptions.TlsPrivateKey) > 0 { + klog.Infof("Server listening on %d.", s.GenericServerRunOptions.SecurePort) + err = http.ListenAndServeTLS(fmt.Sprintf("%s:%d", s.GenericServerRunOptions.BindAddress, s.GenericServerRunOptions.SecurePort), s.GenericServerRunOptions.TlsCertFile, s.GenericServerRunOptions.TlsPrivateKey, container) + } + + return err +} + +func CreateClientSet(conf *apiserverconfig.Config, stopCh <-chan struct{}) error { + csop := &client.ClientSetOptions{} + + csop.SetDevopsOptions(conf.DevopsOptions). + SetKubernetesOptions(conf.KubernetesOptions). + SetMySQLOptions(conf.MySQLOptions) + + client.NewClientSetFactory(csop, stopCh) + + return nil +} + +func WaitForResourceSync(stopCh <-chan struct{}) error { + + //apis.AddToScheme(scheme.Scheme) informerFactory := informers.SharedInformerFactory() - informerFactory.Rbac().V1().Roles().Lister() - informerFactory.Rbac().V1().RoleBindings().Lister() - informerFactory.Rbac().V1().ClusterRoles().Lister() - informerFactory.Rbac().V1().ClusterRoleBindings().Lister() - informerFactory.Storage().V1().StorageClasses().Lister() + // resources we have to create informer first + k8sGVRs := []schema.GroupVersionResource{ + {Group: "", Version: "v1", Resource: "namespaces"}, + {Group: "", Version: "v1", Resource: "nodes"}, + {Group: "", Version: "v1", Resource: "resourcequotas"}, + {Group: "", Version: "v1", Resource: "pods"}, + {Group: "", Version: "v1", Resource: "services"}, + {Group: "", Version: "v1", Resource: "persistentvolumeclaims"}, + {Group: "", Version: "v1", Resource: "secrets"}, + {Group: "", Version: "v1", Resource: "configmaps"}, - informerFactory.Core().V1().Namespaces().Lister() - informerFactory.Core().V1().Nodes().Lister() - informerFactory.Core().V1().ResourceQuotas().Lister() - informerFactory.Core().V1().Pods().Lister() - informerFactory.Core().V1().Services().Lister() - informerFactory.Core().V1().PersistentVolumeClaims().Lister() - informerFactory.Core().V1().Secrets().Lister() - informerFactory.Core().V1().ConfigMaps().Lister() + {Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "roles"}, + {Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "rolebindings"}, + {Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "clusterroles"}, + {Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "clusterrolebindings"}, - informerFactory.Apps().V1().ControllerRevisions().Lister() - informerFactory.Apps().V1().StatefulSets().Lister() - informerFactory.Apps().V1().Deployments().Lister() - informerFactory.Apps().V1().DaemonSets().Lister() - informerFactory.Apps().V1().ReplicaSets().Lister() + {Group: "apps", Version: "v1", Resource: "deployments"}, + {Group: "apps", Version: "v1", Resource: "daemonsets"}, + {Group: "apps", Version: "v1", Resource: "replicasets"}, + {Group: "apps", Version: "v1", Resource: "statefulsets"}, + {Group: "apps", Version: "v1", Resource: "controllerrevisions"}, - informerFactory.Batch().V1().Jobs().Lister() - informerFactory.Batch().V1beta1().CronJobs().Lister() - informerFactory.Extensions().V1beta1().Ingresses().Lister() - informerFactory.Autoscaling().V2beta2().HorizontalPodAutoscalers().Lister() + {Group: "batch", Version: "v1", Resource: "jobs"}, + {Group: "batch", Version: "v1beta1", Resource: "cronjobs"}, - informerFactory.Start(stopChan) - informerFactory.WaitForCacheSync(stopChan) + {Group: "extensions", Version: "v1beta1", Resource: "ingresses"}, + + {Group: "autoscaling", Version: "v2beta2", Resource: "horizontalpodautoscalers"}, + } + + for _, gvr := range k8sGVRs { + _, err := informerFactory.ForResource(gvr) + if err != nil { + klog.Errorf("cannot create informer for %s", gvr) + return err + } + } + + informerFactory.Start(stopCh) + informerFactory.WaitForCacheSync(stopCh) s2iInformerFactory := informers.S2iSharedInformerFactory() - s2iInformerFactory.Devops().V1alpha1().S2iBuilderTemplates().Lister() - s2iInformerFactory.Devops().V1alpha1().S2iRuns().Lister() - s2iInformerFactory.Devops().V1alpha1().S2iBuilders().Lister() - s2iInformerFactory.Start(stopChan) - s2iInformerFactory.WaitForCacheSync(stopChan) + s2iGVRs := []schema.GroupVersionResource{ + {Group: "devops.kubesphere.io", Version: "v1alpha1", Resource: "s2ibuildertemplates"}, + {Group: "devops.kubesphere.io", Version: "v1alpha1", Resource: "s2iruns"}, + {Group: "devops.kubesphere.io", Version: "v1alpha1", Resource: "s2ibuilders"}, + } + + for _, gvr := range s2iGVRs { + _, err := s2iInformerFactory.ForResource(gvr) + if err != nil { + return err + } + } + + s2iInformerFactory.Start(stopCh) + s2iInformerFactory.WaitForCacheSync(stopCh) ksInformerFactory := informers.KsSharedInformerFactory() - ksInformerFactory.Tenant().V1alpha1().Workspaces().Lister() - ksInformerFactory.Devops().V1alpha1().S2iBinaries().Lister() - ksInformerFactory.Start(stopChan) - ksInformerFactory.WaitForCacheSync(stopChan) + ksGVRs := []schema.GroupVersionResource{ + {Group: "tenant.kubesphere.io", Version: "v1alpha1", Resource: "workspaces"}, + {Group: "devops.kubesphere.io", Version: "v1alpha1", Resource: "s2ibinaries"}, + + {Group: "servicemesh.kubesphere.io", Version: "v1alpha2", Resource: "strategies"}, + {Group: "servicemesh.kubesphere.io", Version: "v1alpha2", Resource: "servicepolicies"}, + } + + for _, gvr := range ksGVRs { + _, err := ksInformerFactory.ForResource(gvr) + if err != nil { + return err + } + } + + ksInformerFactory.Start(stopCh) + ksInformerFactory.WaitForCacheSync(stopCh) + + return nil - log.Println("resources sync success") +} + +// apply server run options to configuration +func Complete(s *options.ServerRunOptions) error { + + // loading configuration file + conf := apiserverconfig.Get() + + conf.Apply(&apiserverconfig.Config{ + MySQLOptions: s.MySQLOptions, + DevopsOptions: s.DevopsOptions, + SonarQubeOptions: s.SonarQubeOptions, + KubernetesOptions: s.KubernetesOptions, + ServiceMeshOptions: s.ServiceMeshOptions, + MonitoringOptions: s.MonitoringOptions, + }) + + s = &options.ServerRunOptions{ + GenericServerRunOptions: s.GenericServerRunOptions, + KubernetesOptions: conf.KubernetesOptions, + DevopsOptions: conf.DevopsOptions, + SonarQubeOptions: conf.SonarQubeOptions, + ServiceMeshOptions: conf.ServiceMeshOptions, + MySQLOptions: conf.MySQLOptions, + MonitoringOptions: conf.MonitoringOptions, + } + + return nil } diff --git a/cmd/ks-iam/app/server.go b/cmd/ks-iam/app/server.go index 17239a7ba..149062ac8 100644 --- a/cmd/ks-iam/app/server.go +++ b/cmd/ks-iam/app/server.go @@ -29,10 +29,8 @@ import ( "kubesphere.io/kubesphere/pkg/informers" "kubesphere.io/kubesphere/pkg/models/iam" "kubesphere.io/kubesphere/pkg/server" - "kubesphere.io/kubesphere/pkg/signals" - "kubesphere.io/kubesphere/pkg/simple/client/admin_jenkins" - "kubesphere.io/kubesphere/pkg/simple/client/devops_mysql" "kubesphere.io/kubesphere/pkg/utils/jwtutil" + "kubesphere.io/kubesphere/pkg/utils/signals" "log" "net/http" "time" @@ -72,9 +70,6 @@ func Run(s *options.ServerRunOptions) error { waitForResourceSync() - initializeAdminJenkins() - initializeDevOpsDatabase() - err = iam.Init(s.AdminEmail, s.AdminPassword, expireTime, s.AuthRateLimit) jwtutil.Setup(s.JWTSecret) @@ -127,11 +122,3 @@ func waitForResourceSync() { ksInformerFactory.WaitForCacheSync(stopChan) log.Println("resources sync success") } - -func initializeAdminJenkins() { - admin_jenkins.Client() -} - -func initializeDevOpsDatabase() { - devops_mysql.OpenDatabase() -} diff --git a/go.mod b/go.mod index d8407f83f..b92239386 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,6 @@ require ( github.com/coreos/etcd v3.3.13+incompatible // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect - github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/deckarep/golang-set v1.7.1 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/docker/distribution v2.7.1+incompatible @@ -59,8 +58,6 @@ require ( github.com/gocraft/dbr v0.0.0-20180507214907-a0fd650918f6 github.com/golang/example v0.0.0-20170904185048-46695d81d1fa github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b - github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect - github.com/google/btree v1.0.0 // indirect github.com/google/go-querystring v1.0.0 // indirect github.com/google/gofuzz v1.0.0 // indirect github.com/google/uuid v1.1.1 @@ -68,15 +65,12 @@ require ( github.com/gophercloud/gophercloud v0.3.0 // indirect github.com/gorilla/mux v1.7.1 // indirect github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc // indirect - github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 // indirect - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.9.5 // indirect github.com/hashicorp/go-version v1.2.0 // indirect github.com/hashicorp/golang-lru v0.5.1 // indirect github.com/igm/sockjs-go v2.0.1+incompatible // indirect github.com/imdario/mergo v0.3.7 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/jonboulle/clockwork v0.1.0 // indirect github.com/json-iterator/go v1.1.6 github.com/kelseyhightower/envconfig v1.4.0 // indirect github.com/kiali/kiali v0.15.1-0.20190407071308-6b5b818211c3 @@ -114,22 +108,16 @@ require ( github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 // indirect github.com/russross/blackfriday v1.5.2 // indirect github.com/satori/go.uuid v1.2.0 // indirect - github.com/soheilhy/cmux v0.1.4 // indirect github.com/sony/sonyflake v0.0.0-20181109022403-6d5bd6181009 github.com/speps/go-hashids v2.0.0+incompatible github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.3 + github.com/spf13/viper v1.4.0 github.com/stretchr/testify v1.3.0 - github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect github.com/xanzy/ssh-agent v0.2.1 // indirect - github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect go.etcd.io/bbolt v1.3.3 // indirect - go.uber.org/atomic v1.4.0 // indirect - go.uber.org/multierr v1.1.0 // indirect - go.uber.org/zap v1.10.0 // indirect golang.org/x/net v0.0.0-20190620200207-3b0461eec859 golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a // indirect - golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect golang.org/x/tools v0.0.0-20190710153321-831012c29e42 // indirect google.golang.org/appengine v1.5.0 // indirect google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7 // indirect @@ -145,15 +133,15 @@ require ( gotest.tools v2.2.0+incompatible // indirect k8s.io/api v0.0.0-20181213150558-05914d821849 k8s.io/apiextensions-apiserver v0.0.0-20181213153335-0fe22c71c476 - k8s.io/apimachinery v0.0.0-20190221213512-86fb29eff628 + k8s.io/apimachinery v0.0.0-20190831074630-461753078381 k8s.io/apiserver v0.0.0-20190507070644-e9c02aff496d - k8s.io/client-go v0.0.0-20181213151034-8d9ed539ba31 + k8s.io/client-go v0.0.0-20190831074946-3fe2abece89e k8s.io/code-generator v0.0.0-20181117043124-c2090bec4d9b + k8s.io/component-base v0.0.0-20190831075413-37a093468564 k8s.io/gengo v0.0.0-20190327210449-e17681d19d3a // indirect - k8s.io/klog v0.3.1 + k8s.io/klog v0.4.0 k8s.io/kube-openapi v0.0.0-20181109181836-c59034cc13d5 k8s.io/kubernetes v1.13.6 - k8s.io/utils v0.0.0-20190506122338-8fab8cb257d5 // indirect sigs.k8s.io/application v0.0.0-20190404151855-67ae7f915d4e sigs.k8s.io/controller-runtime v0.1.10 sigs.k8s.io/controller-tools v0.1.12 @@ -188,10 +176,12 @@ replace ( github.com/anmitsu/go-shlex => github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 github.com/apache/thrift => github.com/apache/thrift v0.12.0 github.com/appscode/jsonpatch => github.com/appscode/jsonpatch v0.0.0-20190108182946-7c0e3b262f30 + github.com/armon/consul-api => github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6 github.com/asaskevich/govalidator => github.com/asaskevich/govalidator v0.0.0-20180315120708-ccb8e960c48f github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.22.2 github.com/beevik/etree => github.com/beevik/etree v1.1.0 github.com/beorn7/perks => github.com/beorn7/perks v1.0.0 + github.com/blang/semver => github.com/blang/semver v3.5.0+incompatible github.com/cenkalti/backoff => github.com/cenkalti/backoff v2.2.1+incompatible github.com/census-instrumentation/opencensus-proto => github.com/census-instrumentation/opencensus-proto v0.2.0 github.com/cheekybits/genny => github.com/cheekybits/genny v1.0.0 @@ -270,6 +260,7 @@ replace ( github.com/hashicorp/go-syslog => github.com/hashicorp/go-syslog v1.0.0 github.com/hashicorp/go-version => github.com/hashicorp/go-version v1.2.0 github.com/hashicorp/golang-lru => github.com/hashicorp/golang-lru v0.5.1 + github.com/hashicorp/hcl => github.com/hashicorp/hcl v1.0.0 github.com/hpcloud/tail => github.com/hpcloud/tail v1.0.0 github.com/igm/sockjs-go => github.com/igm/sockjs-go v2.0.1+incompatible github.com/imdario/mergo => github.com/imdario/mergo v0.3.7 @@ -298,6 +289,7 @@ replace ( github.com/leodido/go-urn => github.com/leodido/go-urn v1.1.0 github.com/lib/pq => github.com/lib/pq v1.2.0 github.com/lucas-clemente/quic-go => github.com/lucas-clemente/quic-go v0.11.1 + github.com/magiconair/properties => github.com/magiconair/properties v1.8.0 github.com/mailru/easyjson => github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 github.com/marten-seemann/qtls => github.com/marten-seemann/qtls v0.2.3 github.com/mattn/go-sqlite3 => github.com/mattn/go-sqlite3 v1.11.0 @@ -306,6 +298,7 @@ replace ( github.com/mholt/certmagic => github.com/mholt/certmagic v0.5.1 github.com/miekg/dns => github.com/miekg/dns v1.1.9 github.com/mitchellh/go-homedir => github.com/mitchellh/go-homedir v1.1.0 + github.com/mitchellh/mapstructure => github.com/mitchellh/mapstructure v1.1.2 github.com/modern-go/concurrent => github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd github.com/modern-go/reflect2 => github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 github.com/morikuni/aec => github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c @@ -320,6 +313,7 @@ replace ( github.com/openzipkin/zipkin-go => github.com/openzipkin/zipkin-go v0.1.6 github.com/pborman/uuid => github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709 github.com/pelletier/go-buffruneio => github.com/pelletier/go-buffruneio v0.2.0 + github.com/pelletier/go-toml => github.com/pelletier/go-toml v1.2.0 github.com/peterbourgon/diskv => github.com/peterbourgon/diskv v2.0.1+incompatible github.com/pierrec/lz4 => github.com/pierrec/lz4 v2.0.5+incompatible github.com/pkg/errors => github.com/pkg/errors v0.8.1 @@ -343,14 +337,19 @@ replace ( github.com/sony/sonyflake => github.com/sony/sonyflake v0.0.0-20181109022403-6d5bd6181009 github.com/speps/go-hashids => github.com/speps/go-hashids v2.0.0+incompatible github.com/spf13/afero => github.com/spf13/afero v1.2.2 + github.com/spf13/cast => github.com/spf13/cast v1.3.0 github.com/spf13/cobra => github.com/spf13/cobra v0.0.3 + github.com/spf13/jwalterweatherman => github.com/spf13/jwalterweatherman v1.0.0 github.com/spf13/pflag => github.com/spf13/pflag v1.0.3 + github.com/spf13/viper => github.com/spf13/viper v1.4.0 github.com/src-d/gcfg => github.com/src-d/gcfg v1.4.0 github.com/stretchr/objx => github.com/stretchr/objx v0.1.1 github.com/stretchr/testify => github.com/stretchr/testify v1.3.0 github.com/tmc/grpc-websocket-proxy => github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 + github.com/ugorji/go => github.com/ugorji/go v1.1.4 github.com/xanzy/ssh-agent => github.com/xanzy/ssh-agent v0.2.1 github.com/xiang90/probing => github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 + github.com/xordataexchange/crypt => github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77 go.etcd.io/bbolt => go.etcd.io/bbolt v1.3.3 go.opencensus.io => go.opencensus.io v0.20.2 go.uber.org/atomic => go.uber.org/atomic v1.4.0 @@ -396,8 +395,9 @@ replace ( k8s.io/apiserver => k8s.io/apiserver v0.0.0-20190507070644-e9c02aff496d k8s.io/client-go => k8s.io/client-go v0.0.0-20181213151034-8d9ed539ba31 k8s.io/code-generator => k8s.io/code-generator v0.0.0-20181117043124-c2090bec4d9b + k8s.io/component-base => k8s.io/component-base v0.0.0-20190831075413-37a093468564 k8s.io/gengo => k8s.io/gengo v0.0.0-20190327210449-e17681d19d3a - k8s.io/klog => k8s.io/klog v0.3.1 + k8s.io/klog => k8s.io/klog v0.4.0 k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20181109181836-c59034cc13d5 k8s.io/kubernetes => k8s.io/kubernetes v1.13.6 k8s.io/utils => k8s.io/utils v0.0.0-20190506122338-8fab8cb257d5 diff --git a/go.sum b/go.sum index 39eb84376..fdcef51fd 100644 --- a/go.sum +++ b/go.sum @@ -47,6 +47,7 @@ github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYU github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/appscode/jsonpatch v0.0.0-20190108182946-7c0e3b262f30 h1:Kn3rqvbUFqSepE2OqVu0Pn1CbDw9IuMlONapol0zuwk= github.com/appscode/jsonpatch v0.0.0-20190108182946-7c0e3b262f30/go.mod h1:4AJxUpXUhv4N+ziTvIcWWXgeorXpxPZOfk9HdEVr96M= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/govalidator v0.0.0-20180315120708-ccb8e960c48f h1:y2hSFdXeA1y5z5f0vfNO0Dg5qVY036qzlz3Pds0B92o= github.com/asaskevich/govalidator v0.0.0-20180315120708-ccb8e960c48f/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.22.2 h1:uYP58k2Cd9y1qBy8CxTe5ADmdi4kANm8Ul8ch3kkIcQ= @@ -55,6 +56,7 @@ github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs= github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A= github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.0 h1:LzQXZOgg4CQfE6bFvXGM30YZL1WW/M337pXml+GrcZ4= @@ -202,6 +204,8 @@ github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+d github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/igm/sockjs-go v2.0.1+incompatible h1:iyv0auU1Xh1KC8N+GIiLPa3zZXwRsfRZTIzo09UzeUU= @@ -254,6 +258,8 @@ github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lucas-clemente/quic-go v0.11.1 h1:zasajC848Dqq/+WqfqBCkmPw+YHNe1MBts/z7y7nXf4= github.com/lucas-clemente/quic-go v0.11.1/go.mod h1:PpMmPfPKO9nKJ/psF49ESTAGQSdfXxlg1otPbEB2nOw= +github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 h1:2gxZ0XQIU/5z3Z3bUBu+FXuk2pFbkN6tcwi/pjyaDic= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/marten-seemann/qtls v0.2.3 h1:0yWJ43C62LsZt08vuQJDK1uC1czUc3FJeCLPoNAI4vA= @@ -270,6 +276,8 @@ github.com/miekg/dns v1.1.9 h1:OIdC9wT96RzuZMf2PfKRhFgsStHUUBZLM/lo1LqiM9E= github.com/miekg/dns v1.1.9/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= @@ -296,6 +304,8 @@ github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709 h1:zNBQb37RGLmJybyMcs github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= github.com/pelletier/go-buffruneio v0.2.0 h1:U4t4R6YkofJ5xHm3dJzuRpPZ0mr5MMCoAWooScCR7aA= github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= @@ -338,10 +348,16 @@ github.com/speps/go-hashids v2.0.0+incompatible h1:kSfxGfESueJKTx0mpER9Y/1XHl+FV github.com/speps/go-hashids v2.0.0+incompatible/go.mod h1:P7hqPzMdnZOfyIk+xrlG1QaSMw+gCBdHKsBDnhpaZvc= github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= @@ -350,10 +366,12 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.20.2 h1:NAfh7zF0/3/HqtMvJNZ/RFrSlCE6ZTlHmKfhL/Dm1Jk= @@ -440,10 +458,12 @@ k8s.io/client-go v0.0.0-20181213151034-8d9ed539ba31 h1:OH3z6khCtxnJBAc0C5CMYWLl1 k8s.io/client-go v0.0.0-20181213151034-8d9ed539ba31/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= k8s.io/code-generator v0.0.0-20181117043124-c2090bec4d9b h1:KH0fUlgdFZH8UMxJ/FDCYHpczfSQKefetq5NjL6BVF0= k8s.io/code-generator v0.0.0-20181117043124-c2090bec4d9b/go.mod h1:MYiN+ZJZ9HkETbgVZdWw2AsuAi9PZ4V80cwfuf2axe8= +k8s.io/component-base v0.0.0-20190831075413-37a093468564 h1:mY4AxuX1h/hbjrwVkBBiTGnWeh41YGfEcFIFGb9Iabs= +k8s.io/component-base v0.0.0-20190831075413-37a093468564/go.mod h1:pB3zmhcOR5xextKMKdxRr2XUCERS2UNFA/6Tr2WmSJs= k8s.io/gengo v0.0.0-20190327210449-e17681d19d3a h1:QoHVuRquf80YZ+/bovwxoMO3Q/A3nt3yTgS0/0nejuk= k8s.io/gengo v0.0.0-20190327210449-e17681d19d3a/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68= -k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.4.0 h1:lCJCxf/LIowc2IGS9TPjWDyXY4nOmdGdfcwwDQCOURQ= +k8s.io/klog v0.4.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/kube-openapi v0.0.0-20181109181836-c59034cc13d5 h1:MH8SvyTlIiLt8b1oHy4Dtp1zPpLGp6lTOjvfzPTkoQE= k8s.io/kube-openapi v0.0.0-20181109181836-c59034cc13d5/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= k8s.io/kubernetes v1.13.6 h1:eUAUryzMLFmi4ZY8kMOUtLG5lHp2PUx5WOmy4RVaobk= diff --git a/hack/docker_build.sh b/hack/docker_build.sh index 0163be392..83fa96ae6 100755 --- a/hack/docker_build.sh +++ b/hack/docker_build.sh @@ -9,7 +9,7 @@ TAG=latest # check if build was triggered by a travis cronjob if [[ -z "$TRAVIS_EVENT_TYPE" ]]; then echo "TRAVIS_EVENT_TYPE is empty, also normaly build" -elif [ $TRAVIS_EVENT_TYPE == "cron" ]; then +elif [[ $TRAVIS_EVENT_TYPE == "cron" ]]; then TAG=dev-$(date +%Y%m%d) fi @@ -18,6 +18,7 @@ docker build -f build/ks-apigateway/Dockerfile -t $REPO/ks-apigateway:$TAG . docker build -f build/ks-apiserver/Dockerfile -t $REPO/ks-apiserver:$TAG . docker build -f build/ks-iam/Dockerfile -t $REPO/ks-account:$TAG . docker build -f build/ks-controller-manager/Dockerfile -t $REPO/ks-controller-manager:$TAG . +docker build -f build/hypersphere/Dockerfile -t $REPO/hypersphere:$TAG . docker build -f ./pkg/db/Dockerfile -t $REPO/ks-devops:flyway-$TAG ./pkg/db/ # Push image to dockerhub, need to support multiple push @@ -27,4 +28,5 @@ docker push $REPO/ks-apigateway:$TAG docker push $REPO/ks-apiserver:$TAG docker push $REPO/ks-account:$TAG docker push $REPO/ks-controller-manager:$TAG +docker push $REPO/hypersphere:$TAG docker push $REPO/ks-devops:flyway-$TAG diff --git a/pkg/apis/servicemesh/metrics/v1alpha2/register.go b/pkg/apis/servicemesh/metrics/v1alpha2/register.go index b86595c76..40d9b7dce 100644 --- a/pkg/apis/servicemesh/metrics/v1alpha2/register.go +++ b/pkg/apis/servicemesh/metrics/v1alpha2/register.go @@ -168,7 +168,7 @@ func addWebService(c *restful.Container) error { Param(webservice.PathParameter("workload", "workload name").Required(true)). Param(webservice.QueryParameter("rateInterval", "the rate interval used for fetching error rate").DefaultValue("10m").Required(true)). Param(webservice.QueryParameter("queryTime", "the time to use for query")). - ReturnsError(http.StatusOK, "ok", workloadHealthResponse{}). + Returns(http.StatusOK, "ok", workloadHealthResponse{}). Writes(workloadHealthResponse{})).Produces(restful.MIME_JSON) // Get app health diff --git a/pkg/apiserver/monitoring/monitoring.go b/pkg/apiserver/monitoring/monitoring.go index 468adb5e8..e42a4c07e 100644 --- a/pkg/apiserver/monitoring/monitoring.go +++ b/pkg/apiserver/monitoring/monitoring.go @@ -20,8 +20,14 @@ package monitoring import ( "fmt" "github.com/emicklei/go-restful" + "k8s.io/klog" + "kubesphere.io/kubesphere/pkg/informers" "kubesphere.io/kubesphere/pkg/models/metrics" - "kubesphere.io/kubesphere/pkg/simple/client/prometheus" + "kubesphere.io/kubesphere/pkg/simple/client" + "net/url" + "strconv" + "strings" + "time" ) func MonitorAllPodsOfSpecificNamespace(request *restful.Request, response *restful.Response) { @@ -41,7 +47,7 @@ func MonitorSpecificPodOnSpecificNode(request *restful.Request, response *restfu } func MonitorPod(request *restful.Request, response *restful.Response) { - requestParams := prometheus.ParseMonitoringRequestParams(request) + requestParams := ParseMonitoringRequestParams(request) podName := requestParams.PodName if podName != "" { requestParams.ResourcesFilter = fmt.Sprintf("^%s$", requestParams.PodName) @@ -68,7 +74,7 @@ func MonitorSpecificContainerOfSpecificNamespace(request *restful.Request, respo } func MonitorContainer(request *restful.Request, response *restful.Response) { - requestParams := prometheus.ParseMonitoringRequestParams(request) + requestParams := ParseMonitoringRequestParams(request) rawMetrics := metrics.GetContainerLevelMetrics(requestParams) // sorting sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics) @@ -91,7 +97,7 @@ func MonitorAllWorkloadsOfSpecificNamespace(request *restful.Request, response * } func MonitorWorkload(request *restful.Request, response *restful.Response) { - requestParams := prometheus.ParseMonitoringRequestParams(request) + requestParams := ParseMonitoringRequestParams(request) rawMetrics := metrics.GetWorkloadLevelMetrics(requestParams) @@ -107,7 +113,7 @@ func MonitorWorkload(request *restful.Request, response *restful.Response) { func MonitorAllWorkspaces(request *restful.Request, response *restful.Response) { - requestParams := prometheus.ParseMonitoringRequestParams(request) + requestParams := ParseMonitoringRequestParams(request) tp := requestParams.Tp if tp == "statistics" { @@ -129,7 +135,7 @@ func MonitorAllWorkspaces(request *restful.Request, response *restful.Response) } func MonitorSpecificWorkspace(request *restful.Request, response *restful.Response) { - requestParams := prometheus.ParseMonitoringRequestParams(request) + requestParams := ParseMonitoringRequestParams(request) tp := requestParams.Tp if tp == "rank" { @@ -164,7 +170,7 @@ func MonitorSpecificNamespace(request *restful.Request, response *restful.Respon } func MonitorNamespace(request *restful.Request, response *restful.Response) { - requestParams := prometheus.ParseMonitoringRequestParams(request) + requestParams := ParseMonitoringRequestParams(request) // multiple rawMetrics := metrics.GetNamespaceLevelMetrics(requestParams) @@ -176,13 +182,23 @@ func MonitorNamespace(request *restful.Request, response *restful.Response) { } func MonitorCluster(request *restful.Request, response *restful.Response) { - requestParams := prometheus.ParseMonitoringRequestParams(request) + requestParams := ParseMonitoringRequestParams(request) metricName := requestParams.MetricsName if metricName != "" { + prometheusClient, err := client.ClientSets().Prometheus() + if err != nil { + if _, ok := err.(client.ClientSetNotEnabledError); ok { + klog.Error("monitoring is not enabled") + return + } else { + klog.Errorf("get prometheus client failed %+v", err) + } + } + // single queryType, params := metrics.AssembleClusterMetricRequestInfo(requestParams, metricName) - metricsStr := prometheus.SendMonitoringRequest(prometheus.PrometheusEndpoint, queryType, params) + metricsStr := prometheusClient.SendMonitoringRequest(queryType, params) res := metrics.ReformatJson(metricsStr, metricName, map[string]string{metrics.MetricLevelCluster: "local"}) response.WriteAsJson(res) @@ -202,13 +218,22 @@ func MonitorSpecificNode(request *restful.Request, response *restful.Response) { } func MonitorNode(request *restful.Request, response *restful.Response) { - requestParams := prometheus.ParseMonitoringRequestParams(request) + requestParams := ParseMonitoringRequestParams(request) metricName := requestParams.MetricsName if metricName != "" { + prometheusClient, err := client.ClientSets().Prometheus() + if err != nil { + if _, ok := err.(client.ClientSetNotEnabledError); ok { + klog.Error("monitoring is not enabled") + return + } else { + klog.Errorf("get prometheus client failed %+v", err) + } + } // single queryType, params := metrics.AssembleNodeMetricRequestInfo(requestParams, metricName) - metricsStr := prometheus.SendMonitoringRequest(prometheus.PrometheusEndpoint, queryType, params) + metricsStr := prometheusClient.SendMonitoringRequest(queryType, params) res := metrics.ReformatJson(metricsStr, metricName, map[string]string{metrics.MetricLevelNode: ""}) // The raw node-exporter result doesn't include ip address information // Thereby, append node ip address to .data.result[].metric @@ -248,7 +273,7 @@ func MonitorSpecificPVCofSpecificNamespace(request *restful.Request, response *r } func MonitorPVC(request *restful.Request, response *restful.Response) { - requestParams := prometheus.ParseMonitoringRequestParams(request) + requestParams := ParseMonitoringRequestParams(request) pvcName := requestParams.PVCName if pvcName != "" { requestParams.ResourcesFilter = fmt.Sprintf("^%s$", requestParams.PVCName) @@ -263,7 +288,7 @@ func MonitorPVC(request *restful.Request, response *restful.Response) { } func MonitorComponent(request *restful.Request, response *restful.Response) { - requestParams := prometheus.ParseMonitoringRequestParams(request) + requestParams := ParseMonitoringRequestParams(request) if requestParams.MetricsFilter == "" { requestParams.MetricsFilter = requestParams.ComponentName + "_.*" @@ -273,3 +298,113 @@ func MonitorComponent(request *restful.Request, response *restful.Response) { response.WriteAsJson(rawMetrics) } + +func ParseMonitoringRequestParams(request *restful.Request) *metrics.MonitoringRequestParams { + instantTime := strings.Trim(request.QueryParameter("time"), " ") + start := strings.Trim(request.QueryParameter("start"), " ") + end := strings.Trim(request.QueryParameter("end"), " ") + step := strings.Trim(request.QueryParameter("step"), " ") + timeout := strings.Trim(request.QueryParameter("timeout"), " ") + + sortMetricName := strings.Trim(request.QueryParameter("sort_metric"), " ") + sortType := strings.Trim(request.QueryParameter("sort_type"), " ") + pageNum := strings.Trim(request.QueryParameter("page"), " ") + limitNum := strings.Trim(request.QueryParameter("limit"), " ") + tp := strings.Trim(request.QueryParameter("type"), " ") + + metricsFilter := strings.Trim(request.QueryParameter("metrics_filter"), " ") + resourcesFilter := strings.Trim(request.QueryParameter("resources_filter"), " ") + + metricsName := strings.Trim(request.QueryParameter("metrics_name"), " ") + workloadName := strings.Trim(request.PathParameter("workload"), " ") + + nodeId := strings.Trim(request.PathParameter("node"), " ") + wsName := strings.Trim(request.PathParameter("workspace"), " ") + nsName := strings.Trim(request.PathParameter("namespace"), " ") + podName := strings.Trim(request.PathParameter("pod"), " ") + pvcName := strings.Trim(request.PathParameter("pvc"), " ") + storageClassName := strings.Trim(request.PathParameter("storageclass"), " ") + containerName := strings.Trim(request.PathParameter("container"), " ") + workloadKind := strings.Trim(request.PathParameter("kind"), " ") + componentName := strings.Trim(request.PathParameter("component"), " ") + + var requestParams = metrics.MonitoringRequestParams{ + SortMetricName: sortMetricName, + SortType: sortType, + PageNum: pageNum, + LimitNum: limitNum, + Tp: tp, + MetricsFilter: metricsFilter, + ResourcesFilter: resourcesFilter, + MetricsName: metricsName, + WorkloadName: workloadName, + NodeId: nodeId, + WsName: wsName, + NsName: nsName, + PodName: podName, + PVCName: pvcName, + StorageClassName: storageClassName, + ContainerName: containerName, + WorkloadKind: workloadKind, + ComponentName: componentName, + } + + if timeout == "" { + timeout = metrics.DefaultQueryTimeout + } + if step == "" { + step = metrics.DefaultQueryStep + } + // Whether query or query_range request + u := url.Values{} + + if start != "" && end != "" { + + u.Set("start", convertTimeGranularity(start)) + u.Set("end", convertTimeGranularity(end)) + u.Set("step", step) + u.Set("timeout", timeout) + + // range query start time must be greater than the namespace creation time + if nsName != "" { + nsLister := informers.SharedInformerFactory().Core().V1().Namespaces().Lister() + ns, err := nsLister.Get(nsName) + if err == nil { + queryStartTime := u.Get("start") + nsCreationTime := strconv.FormatInt(ns.CreationTimestamp.Unix(), 10) + if nsCreationTime > queryStartTime { + u.Set("start", nsCreationTime) + } + } + } + + requestParams.QueryType = metrics.RangeQueryType + requestParams.Params = u + + return &requestParams + } + if instantTime != "" { + u.Set("time", instantTime) + u.Set("timeout", timeout) + requestParams.QueryType = metrics.DefaultQueryType + requestParams.Params = u + return &requestParams + } else { + u.Set("timeout", timeout) + requestParams.QueryType = metrics.DefaultQueryType + requestParams.Params = u + return &requestParams + } +} + +func convertTimeGranularity(ts string) string { + timeFloat, err := strconv.ParseFloat(ts, 64) + if err != nil { + klog.Errorf("convert second timestamp %s to minute timestamp failed", ts) + return strconv.FormatInt(int64(time.Now().Unix()), 10) + } + timeInt := int64(timeFloat) + // convert second timestamp to minute timestamp + secondTime := time.Unix(timeInt, 0).Truncate(time.Minute).Unix() + return strconv.FormatInt(secondTime, 10) +} diff --git a/pkg/controller/application/application_controller.go b/pkg/controller/application/application_controller.go index 299067df7..aa5223565 100644 --- a/pkg/controller/application/application_controller.go +++ b/pkg/controller/application/application_controller.go @@ -22,11 +22,11 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" + log "k8s.io/klog" "k8s.io/kubernetes/pkg/util/metrics" servicemeshinformers "kubesphere.io/kubesphere/pkg/client/informers/externalversions/servicemesh/v1alpha2" servicemeshlisters "kubesphere.io/kubesphere/pkg/client/listers/servicemesh/v1alpha2" "kubesphere.io/kubesphere/pkg/controller/virtualservice/util" - logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" ) const ( @@ -38,8 +38,6 @@ const ( maxRetries = 15 ) -var log = logf.Log.WithName("application-controller") - type ApplicationController struct { client clientset.Interface diff --git a/pkg/controller/clusterrolebinding/clusterrolebinding_controller.go b/pkg/controller/clusterrolebinding/clusterrolebinding_controller.go index e16c76088..e10f956fe 100644 --- a/pkg/controller/clusterrolebinding/clusterrolebinding_controller.go +++ b/pkg/controller/clusterrolebinding/clusterrolebinding_controller.go @@ -27,6 +27,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + log "k8s.io/klog" "kubesphere.io/kubesphere/pkg/constants" "kubesphere.io/kubesphere/pkg/utils/k8sutil" "reflect" @@ -35,14 +36,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" - logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" "sigs.k8s.io/controller-runtime/pkg/source" ) -var ( - log = logf.Log.WithName("clusterrolebinding-controller") -) - /** * USER ACTION REQUIRED: This is a scaffold file intended for the user to modify with their own Controller * business logic. Delete these comments after modifying this file.* @@ -139,7 +135,7 @@ func (r *ReconcileClusterRoleBinding) updateRoleBindings(clusterRoleBinding *rba err := r.Get(context.TODO(), types.NamespacedName{Namespace: namespace.Name, Name: adminBinding.Name}, found) if errors.IsNotFound(err) { - log.Info("Creating default role binding", "namespace", namespace.Name, "name", adminBinding.Name) + log.V(4).Info("Creating default role binding", "namespace", namespace.Name, "name", adminBinding.Name) err = r.Create(context.TODO(), adminBinding) if err != nil { log.Error(err, "default role binding create failed", "namespace", namespace.Name, "name", adminBinding.Name) @@ -151,7 +147,7 @@ func (r *ReconcileClusterRoleBinding) updateRoleBindings(clusterRoleBinding *rba } if !reflect.DeepEqual(found.RoleRef, adminBinding.RoleRef) { - log.Info("Deleting conflict role binding", "namespace", namespace.Name, "name", adminBinding.Name) + log.V(4).Info("Deleting conflict role binding", "namespace", namespace.Name, "name", adminBinding.Name) err = r.Delete(context.TODO(), found) if err != nil { return err @@ -161,7 +157,7 @@ func (r *ReconcileClusterRoleBinding) updateRoleBindings(clusterRoleBinding *rba if !reflect.DeepEqual(found.Subjects, adminBinding.Subjects) { found.Subjects = adminBinding.Subjects - log.Info("Updating role binding", "namespace", namespace.Name, "name", adminBinding.Name) + log.V(4).Info("Updating role binding", "namespace", namespace.Name, "name", adminBinding.Name) err = r.Update(context.TODO(), found) if err != nil { return err diff --git a/pkg/controller/destinationrule/destinationrule_controller.go b/pkg/controller/destinationrule/destinationrule_controller.go index c3c1c54c0..756c5c8ba 100644 --- a/pkg/controller/destinationrule/destinationrule_controller.go +++ b/pkg/controller/destinationrule/destinationrule_controller.go @@ -16,10 +16,10 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes/scheme" v1core "k8s.io/client-go/kubernetes/typed/core/v1" + log "k8s.io/klog" "k8s.io/kubernetes/pkg/util/metrics" servicemeshv1alpha2 "kubesphere.io/kubesphere/pkg/apis/servicemesh/v1alpha2" "kubesphere.io/kubesphere/pkg/controller/virtualservice/util" - logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" "time" @@ -49,8 +49,6 @@ const ( maxRetries = 15 ) -var log = logf.Log.WithName("destinationrule-controller") - type DestinationRuleController struct { client clientset.Interface diff --git a/pkg/controller/job/job_controller.go b/pkg/controller/job/job_controller.go index 8d7e0cc2d..eb1908f69 100644 --- a/pkg/controller/job/job_controller.go +++ b/pkg/controller/job/job_controller.go @@ -30,8 +30,8 @@ import ( "k8s.io/apimachinery/pkg/util/wait" batchv1informers "k8s.io/client-go/informers/batch/v1" batchv1listers "k8s.io/client-go/listers/batch/v1" + log "k8s.io/klog" "k8s.io/kubernetes/pkg/util/metrics" - logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" "time" @@ -51,8 +51,6 @@ const ( revisionsAnnotationKey = "revisions" ) -var log = logf.Log.WithName("job-controller") - type JobController struct { client clientset.Interface eventBroadcaster record.EventBroadcaster diff --git a/pkg/controller/namespace/namespace_controller.go b/pkg/controller/namespace/namespace_controller.go index c7739c2d0..ea64370c9 100644 --- a/pkg/controller/namespace/namespace_controller.go +++ b/pkg/controller/namespace/namespace_controller.go @@ -33,6 +33,7 @@ import ( "k8s.io/kubernetes/pkg/apis/core" "kubesphere.io/kubesphere/pkg/apis/tenant/v1alpha1" "kubesphere.io/kubesphere/pkg/constants" + cs "kubesphere.io/kubesphere/pkg/simple/client" "kubesphere.io/kubesphere/pkg/simple/client/openpitrix" "kubesphere.io/kubesphere/pkg/utils/k8sutil" "kubesphere.io/kubesphere/pkg/utils/sliceutil" @@ -356,6 +357,10 @@ func (r *ReconcileNamespace) checkAndCreateRoleBindings(namespace *corev1.Namesp // Create openpitrix runtime func (r *ReconcileNamespace) checkAndCreateRuntime(namespace *corev1.Namespace) error { + openPitrixClient, err := cs.ClientSets().OpenPitrix() + if err != nil { + return err + } if runtimeId := namespace.Annotations[constants.OpenPitrixRuntimeAnnotationKey]; runtimeId != "" { return nil @@ -363,7 +368,7 @@ func (r *ReconcileNamespace) checkAndCreateRuntime(namespace *corev1.Namespace) cm := &corev1.ConfigMap{} configName := fmt.Sprintf("kubeconfig-%s", constants.AdminUserName) - err := r.Get(context.TODO(), types.NamespacedName{Namespace: constants.KubeSphereControlNamespace, Name: configName}, cm) + err = r.Get(context.TODO(), types.NamespacedName{Namespace: constants.KubeSphereControlNamespace, Name: configName}, cm) if err != nil { return err @@ -371,7 +376,7 @@ func (r *ReconcileNamespace) checkAndCreateRuntime(namespace *corev1.Namespace) runtime := &openpitrix.RunTime{Name: namespace.Name, Zone: namespace.Name, Provider: "kubernetes", RuntimeCredential: cm.Data["config"]} - if err := openpitrix.Client().CreateRuntime(runtime); err != nil { + if err := openPitrixClient.CreateRuntime(runtime); err != nil { klog.Errorf("creating openpitrix runtime namespace: %s, error: %s", namespace.Name, err) return err } @@ -387,7 +392,12 @@ func (r *ReconcileNamespace) deleteRuntime(namespace *corev1.Namespace) error { for i := float64(0); i < maxRetries; i++ { time.Sleep(time.Duration(i*math.Pow(2, i)) * time.Second) - err := openpitrix.Client().DeleteRuntime(runtimeId) + openPitrixClient, err := cs.ClientSets().OpenPitrix() + if err != nil { + return err + } + + err = openPitrixClient.DeleteRuntime(runtimeId) if err == nil || openpitrix.IsNotFound(err) || openpitrix.IsDeleted(err) { return nil diff --git a/pkg/controller/network/wsnetworkpolicy/wsnetworkpolicy_test.go b/pkg/controller/network/wsnetworkpolicy/wsnetworkpolicy_test.go index 6e9905de1..189d30906 100644 --- a/pkg/controller/network/wsnetworkpolicy/wsnetworkpolicy_test.go +++ b/pkg/controller/network/wsnetworkpolicy/wsnetworkpolicy_test.go @@ -71,9 +71,9 @@ var _ = Describe("Wsnetworkpolicy", func() { wsnp := newWorkspaceNP(testName) wsnp.Spec.PolicyTypes = []k8snetwork.PolicyType{k8snetwork.PolicyTypeIngress} wsnp.Spec.Ingress = []v1alpha1.WorkspaceNetworkPolicyIngressRule{ - v1alpha1.WorkspaceNetworkPolicyIngressRule{ + { From: []v1alpha1.WorkspaceNetworkPolicyPeer{ - v1alpha1.WorkspaceNetworkPolicyPeer{ + { WorkspaceSelector: &metav1.LabelSelector{ MatchLabels: label, }, diff --git a/pkg/controller/s2ibinary/s2ibinary_controller.go b/pkg/controller/s2ibinary/s2ibinary_controller.go index 4b38ab950..38408bd09 100644 --- a/pkg/controller/s2ibinary/s2ibinary_controller.go +++ b/pkg/controller/s2ibinary/s2ibinary_controller.go @@ -17,7 +17,7 @@ import ( "k8s.io/client-go/util/workqueue" "k8s.io/klog" "k8s.io/kubernetes/pkg/util/metrics" - "kubesphere.io/kubesphere/pkg/simple/client/s2is3" + "kubesphere.io/kubesphere/pkg/simple/client" "kubesphere.io/kubesphere/pkg/utils/sliceutil" "time" @@ -210,12 +210,16 @@ func (c *S2iBinaryController) syncHandler(key string) error { } func (c *S2iBinaryController) deleteBinaryInS3(s2ibin *devopsv1alpha1.S2iBinary) error { - s3client := s2is3.Client() + s3Client, err := client.ClientSets().S3() + if err != nil { + return err + } + input := &s3.DeleteObjectInput{ - Bucket: s2is3.Bucket(), + Bucket: s3Client.Bucket(), Key: aws.String(fmt.Sprintf("%s-%s", s2ibin.Namespace, s2ibin.Name)), } - _, err := s3client.DeleteObject(input) + _, err = s3Client.Client().DeleteObject(input) if err != nil { if aerr, ok := err.(awserr.Error); ok { switch aerr.Code() { diff --git a/pkg/controller/virtualservice/virtualservice_controller.go b/pkg/controller/virtualservice/virtualservice_controller.go index 92caaf993..d6a29d119 100644 --- a/pkg/controller/virtualservice/virtualservice_controller.go +++ b/pkg/controller/virtualservice/virtualservice_controller.go @@ -16,9 +16,9 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes/scheme" v1core "k8s.io/client-go/kubernetes/typed/core/v1" + log "k8s.io/klog" "k8s.io/kubernetes/pkg/util/metrics" "kubesphere.io/kubesphere/pkg/controller/virtualservice/util" - logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" istioclient "github.com/knative/pkg/client/clientset/versioned" istioinformers "github.com/knative/pkg/client/informers/externalversions/istio/v1alpha3" @@ -46,8 +46,6 @@ const ( maxRetries = 15 ) -var log = logf.Log.WithName("virtualservice-controller") - type VirtualServiceController struct { client clientset.Interface @@ -152,7 +150,7 @@ func (v *VirtualServiceController) Run(workers int, stopCh <-chan struct{}) erro defer utilruntime.HandleCrash() defer v.queue.ShutDown() - log.Info("starting virtualservice controller") + log.V(0).Info("starting virtualservice controller") defer log.Info("shutting down virtualservice controller") if !cache.WaitForCacheSync(stopCh, v.serviceSynced, v.virtualServiceSynced, v.destinationRuleSynced, v.strategySynced) { diff --git a/pkg/informers/informers.go b/pkg/informers/informers.go index d737e4d5f..9e72f2226 100644 --- a/pkg/informers/informers.go +++ b/pkg/informers/informers.go @@ -18,15 +18,12 @@ package informers import ( + s2iinformers "github.com/kubesphere/s2ioperator/pkg/client/informers/externalversions" + k8sinformers "k8s.io/client-go/informers" + ksinformers "kubesphere.io/kubesphere/pkg/client/informers/externalversions" + "kubesphere.io/kubesphere/pkg/simple/client" "sync" "time" - - s2iInformers "github.com/kubesphere/s2ioperator/pkg/client/informers/externalversions" - - "k8s.io/client-go/informers" - ksInformers "kubesphere.io/kubesphere/pkg/client/informers/externalversions" - - "kubesphere.io/kubesphere/pkg/simple/client/k8s" ) const defaultResync = 600 * time.Second @@ -35,31 +32,31 @@ var ( k8sOnce sync.Once s2iOnce sync.Once ksOnce sync.Once - informerFactory informers.SharedInformerFactory - s2iInformerFactory s2iInformers.SharedInformerFactory - ksInformerFactory ksInformers.SharedInformerFactory + informerFactory k8sinformers.SharedInformerFactory + s2iInformerFactory s2iinformers.SharedInformerFactory + ksInformerFactory ksinformers.SharedInformerFactory ) -func SharedInformerFactory() informers.SharedInformerFactory { +func SharedInformerFactory() k8sinformers.SharedInformerFactory { k8sOnce.Do(func() { - k8sClient := k8s.Client() - informerFactory = informers.NewSharedInformerFactory(k8sClient, defaultResync) + k8sClient := client.ClientSets().K8s().Kubernetes() + informerFactory = k8sinformers.NewSharedInformerFactory(k8sClient, defaultResync) }) return informerFactory } -func S2iSharedInformerFactory() s2iInformers.SharedInformerFactory { +func S2iSharedInformerFactory() s2iinformers.SharedInformerFactory { s2iOnce.Do(func() { - k8sClient := k8s.S2iClient() - s2iInformerFactory = s2iInformers.NewSharedInformerFactory(k8sClient, defaultResync) + k8sClient := client.ClientSets().K8s().S2i() + s2iInformerFactory = s2iinformers.NewSharedInformerFactory(k8sClient, defaultResync) }) return s2iInformerFactory } -func KsSharedInformerFactory() ksInformers.SharedInformerFactory { +func KsSharedInformerFactory() ksinformers.SharedInformerFactory { ksOnce.Do(func() { - k8sClient := k8s.KsClient() - ksInformerFactory = ksInformers.NewSharedInformerFactory(k8sClient, defaultResync) + k8sClient := client.ClientSets().K8s().KubeSphere() + ksInformerFactory = ksinformers.NewSharedInformerFactory(k8sClient, defaultResync) }) return ksInformerFactory } diff --git a/pkg/models/applications/applications.go b/pkg/models/applications/applications.go index a800ffd9d..022ecb06b 100644 --- a/pkg/models/applications/applications.go +++ b/pkg/models/applications/applications.go @@ -31,7 +31,7 @@ import ( "kubesphere.io/kubesphere/pkg/models" "kubesphere.io/kubesphere/pkg/models/resources" "kubesphere.io/kubesphere/pkg/params" - "kubesphere.io/kubesphere/pkg/simple/client/k8s" + "kubesphere.io/kubesphere/pkg/simple/client" "kubesphere.io/kubesphere/pkg/simple/client/openpitrix" "strings" "time" @@ -63,7 +63,11 @@ type workLoads struct { } func ListApplication(runtimeId string, conditions *params.Conditions, limit, offset int) (*models.PageableResponse, error) { - clusterList, err := openpitrix.ListClusters(runtimeId, conditions.Match["keyword"], conditions.Match["status"], limit, offset) + openPitrixClient, err := client.ClientSets().OpenPitrix() + if err != nil { + return nil, err + } + clusterList, err := openPitrixClient.ListClusters(runtimeId, conditions.Match["keyword"], conditions.Match["status"], limit, offset) if err != nil { return nil, err } @@ -76,13 +80,13 @@ func ListApplication(runtimeId string, conditions *params.Conditions, limit, off app.ClusterID = item.ClusterID app.UpdateTime = item.UpdateTime app.Status = item.Status - versionInfo, _ := openpitrix.GetVersion(item.VersionID) + versionInfo, _ := openPitrixClient.GetVersion(item.VersionID) app.Version = versionInfo app.VersionId = item.VersionID - runtimeInfo, _ := openpitrix.GetRuntime(item.RunTimeId) + runtimeInfo, _ := openPitrixClient.GetRuntime(item.RunTimeId) app.Runtime = runtimeInfo app.RuntimeId = item.RunTimeId - appInfo, _, appId, _ := openpitrix.GetAppInfo(item.AppID) + appInfo, _, appId, _ := openPitrixClient.GetAppInfo(item.AppID) app.App = appInfo app.AppId = appId app.Description = item.Description @@ -94,8 +98,11 @@ func ListApplication(runtimeId string, conditions *params.Conditions, limit, off } func GetApp(clusterId string) (*Application, error) { - - item, err := openpitrix.GetCluster(clusterId) + openPitrixClient, err := client.ClientSets().OpenPitrix() + if err != nil { + return nil, err + } + item, err := openPitrixClient.GetCluster(clusterId) if err != nil { glog.Error(err) @@ -109,19 +116,19 @@ func GetApp(clusterId string) (*Application, error) { app.UpdateTime = item.UpdateTime app.CreateTime = item.CreateTime app.Status = item.Status - versionInfo, _ := openpitrix.GetVersion(item.VersionID) + versionInfo, _ := openPitrixClient.GetVersion(item.VersionID) app.Version = versionInfo app.VersionId = item.VersionID - runtimeInfo, _ := openpitrix.GetRuntime(item.RunTimeId) + runtimeInfo, _ := openPitrixClient.GetRuntime(item.RunTimeId) app.Runtime = runtimeInfo app.RuntimeId = item.RunTimeId - appInfo, repoId, appId, _ := openpitrix.GetAppInfo(item.AppID) + appInfo, repoId, appId, _ := openPitrixClient.GetAppInfo(item.AppID) app.App = appInfo app.AppId = appId app.Description = item.Description - app.RepoName, _ = openpitrix.GetRepo(repoId) + app.RepoName, _ = openPitrixClient.GetRepo(repoId) workloads, err := getWorkLoads(app.Runtime, item.ClusterRoleSets) if err != nil { @@ -195,7 +202,7 @@ func getWorkLoads(namespace string, clusterRoles []openpitrix.ClusterRole) (*wor } func getLabels(namespace string, workloads *workLoads) *[]map[string]string { - k8sClient := k8s.Client() + k8sClient := client.ClientSets().K8s().Kubernetes() var workloadLables []map[string]string if workloads == nil { @@ -242,7 +249,7 @@ func getSvcs(namespace string, workLoadLabels *[]map[string]string) []v1.Service if len(*workLoadLabels) == 0 { return nil } - k8sClient := k8s.Client() + k8sClient := client.ClientSets().K8s().Kubernetes() var services []v1.Service for _, label := range *workLoadLabels { labelSelector := labels.Set(label).AsSelector().String() @@ -302,6 +309,11 @@ func getIng(namespace string, services []v1.Service) []v1beta1.Ingress { } func DeployApplication(namespace string, app openpitrix.CreateClusterRequest) error { + openPitrixClient, err := client.ClientSets().OpenPitrix() + if err != nil { + return err + } + ns, err := informers.SharedInformerFactory().Core().V1().Namespaces().Lister().Get(namespace) if err != nil { glog.Errorf("deploy application failed: %+v", err) @@ -313,9 +325,14 @@ func DeployApplication(namespace string, app openpitrix.CreateClusterRequest) er } else { return fmt.Errorf("runtime not init: namespace %s", namespace) } - return openpitrix.CreateCluster(app) + return openPitrixClient.CreateCluster(app) } func DeleteApplication(clusterId string) error { - return openpitrix.DeleteCluster(openpitrix.DeleteClusterRequest{ClusterId: []string{clusterId}}) + openPitrixClient, err := client.ClientSets().OpenPitrix() + if err != nil { + return err + } + + return openPitrixClient.DeleteCluster(openpitrix.DeleteClusterRequest{ClusterId: []string{clusterId}}) } diff --git a/pkg/models/components/components.go b/pkg/models/components/components.go index 42f4b3f3a..05eaf3fab 100644 --- a/pkg/models/components/components.go +++ b/pkg/models/components/components.go @@ -19,9 +19,9 @@ package components import ( "fmt" - meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "kubesphere.io/kubesphere/pkg/models" - "kubesphere.io/kubesphere/pkg/simple/client/k8s" + "kubesphere.io/kubesphere/pkg/simple/client" "kubesphere.io/kubesphere/pkg/informers" @@ -93,7 +93,7 @@ func GetSystemHealthStatus() (*models.HealthStatus, error) { status := &models.HealthStatus{} - componentStatuses, err := k8s.Client().CoreV1().ComponentStatuses().List(meta_v1.ListOptions{}) + componentStatuses, err := client.ClientSets().K8s().Kubernetes().CoreV1().ComponentStatuses().List(metav1.ListOptions{}) if err != nil { return nil, err } diff --git a/pkg/models/devops/common.go b/pkg/models/devops/common.go index f515c775c..7d0af8385 100644 --- a/pkg/models/devops/common.go +++ b/pkg/models/devops/common.go @@ -16,9 +16,10 @@ package devops import ( "fmt" "github.com/fatih/structs" + "k8s.io/klog" "kubesphere.io/kubesphere/pkg/db" "kubesphere.io/kubesphere/pkg/gojenkins" - "kubesphere.io/kubesphere/pkg/simple/client/devops_mysql" + "kubesphere.io/kubesphere/pkg/simple/client" "kubesphere.io/kubesphere/pkg/utils/reflectutils" "kubesphere.io/kubesphere/pkg/utils/stringutils" ) @@ -126,7 +127,7 @@ var JenkinsOwnerProjectPermissionIds = &gojenkins.ProjectPermissionIds{ } var JenkinsProjectPermissionMap = map[string]gojenkins.ProjectPermissionIds{ - ProjectOwner: gojenkins.ProjectPermissionIds{ + ProjectOwner: { CredentialCreate: true, CredentialDelete: true, CredentialManageDomains: true, @@ -146,7 +147,7 @@ var JenkinsProjectPermissionMap = map[string]gojenkins.ProjectPermissionIds{ RunUpdate: true, SCMTag: true, }, - ProjectMaintainer: gojenkins.ProjectPermissionIds{ + ProjectMaintainer: { CredentialCreate: true, CredentialDelete: true, CredentialManageDomains: true, @@ -166,7 +167,7 @@ var JenkinsProjectPermissionMap = map[string]gojenkins.ProjectPermissionIds{ RunUpdate: true, SCMTag: true, }, - ProjectDeveloper: gojenkins.ProjectPermissionIds{ + ProjectDeveloper: { CredentialCreate: false, CredentialDelete: false, CredentialManageDomains: false, @@ -186,7 +187,7 @@ var JenkinsProjectPermissionMap = map[string]gojenkins.ProjectPermissionIds{ RunUpdate: true, SCMTag: false, }, - ProjectReporter: gojenkins.ProjectPermissionIds{ + ProjectReporter: { CredentialCreate: false, CredentialDelete: false, CredentialManageDomains: false, @@ -209,7 +210,7 @@ var JenkinsProjectPermissionMap = map[string]gojenkins.ProjectPermissionIds{ } var JenkinsPipelinePermissionMap = map[string]gojenkins.ProjectPermissionIds{ - ProjectOwner: gojenkins.ProjectPermissionIds{ + ProjectOwner: { CredentialCreate: true, CredentialDelete: true, CredentialManageDomains: true, @@ -229,7 +230,7 @@ var JenkinsPipelinePermissionMap = map[string]gojenkins.ProjectPermissionIds{ RunUpdate: true, SCMTag: true, }, - ProjectMaintainer: gojenkins.ProjectPermissionIds{ + ProjectMaintainer: { CredentialCreate: true, CredentialDelete: true, CredentialManageDomains: true, @@ -249,7 +250,7 @@ var JenkinsPipelinePermissionMap = map[string]gojenkins.ProjectPermissionIds{ RunUpdate: true, SCMTag: true, }, - ProjectDeveloper: gojenkins.ProjectPermissionIds{ + ProjectDeveloper: { CredentialCreate: false, CredentialDelete: false, CredentialManageDomains: false, @@ -269,7 +270,7 @@ var JenkinsPipelinePermissionMap = map[string]gojenkins.ProjectPermissionIds{ RunUpdate: true, SCMTag: false, }, - ProjectReporter: gojenkins.ProjectPermissionIds{ + ProjectReporter: { CredentialCreate: false, CredentialDelete: false, CredentialManageDomains: false, @@ -311,9 +312,18 @@ func CheckProjectUserInRole(username, projectId string, roles []string) error { if username == KS_ADMIN { return nil } - dbconn := devops_mysql.OpenDatabase() + dbconn, err := client.ClientSets().MySQL() + if err != nil { + if _, ok := err.(client.ClientSetNotEnabledError); ok { + klog.Error("mysql is not enabled") + } else { + klog.Error("error creating mysql client", err) + } + return nil + } + membership := &DevOpsProjectMembership{} - err := dbconn.Select(DevOpsProjectMembershipColumns...). + err = dbconn.Select(DevOpsProjectMembershipColumns...). From(DevOpsProjectMembershipTableName). Where(db.And( db.Eq(DevOpsProjectMembershipUsernameColumn, username), @@ -331,9 +341,17 @@ func GetProjectUserRole(username, projectId string) (string, error) { if username == KS_ADMIN { return ProjectOwner, nil } - dbconn := devops_mysql.OpenDatabase() + dbconn, err := client.ClientSets().MySQL() + if err != nil { + if _, ok := err.(client.ClientSetNotEnabledError); ok { + klog.Error("mysql is not enabled") + } else { + klog.Error("error creating mysql client", err) + } + return "", err + } membership := &DevOpsProjectMembership{} - err := dbconn.Select(DevOpsProjectMembershipColumns...). + err = dbconn.Select(DevOpsProjectMembershipColumns...). From(DevOpsProjectMembershipTableName). Where(db.And( db.Eq(DevOpsProjectMembershipUsernameColumn, username), diff --git a/pkg/models/devops/devops.go b/pkg/models/devops/devops.go index a9c1f8b58..ee179db7e 100644 --- a/pkg/models/devops/devops.go +++ b/pkg/models/devops/devops.go @@ -23,11 +23,11 @@ import ( "encoding/json" "fmt" "github.com/PuerkitoBio/goquery" - log "github.com/golang/glog" + "github.com/emicklei/go-restful" "io" "io/ioutil" - "kubesphere.io/kubesphere/pkg/gojenkins" - "kubesphere.io/kubesphere/pkg/simple/client/admin_jenkins" + log "k8s.io/klog" + cs "kubesphere.io/kubesphere/pkg/simple/client" "net/http" "net/url" "strings" @@ -40,15 +40,13 @@ const ( cronJobLayout = "Monday, January 2, 2006 15:04:05 PM" ) -var jenkins *gojenkins.Jenkins - -func JenkinsInit() { - jenkins = admin_jenkins.GetJenkins() -} - func GetPipeline(projectName, pipelineName string, req *http.Request) ([]byte, error) { - baseUrl := fmt.Sprintf(jenkins.Server+GetPipelineUrl, projectName, pipelineName) - log.Infof("Jenkins-url: " + baseUrl) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + baseUrl := fmt.Sprintf(devops.Jenkins().Server+GetPipelineUrl, projectName, pipelineName) res, err := sendJenkinsRequest(baseUrl, req) if err != nil { @@ -60,8 +58,12 @@ func GetPipeline(projectName, pipelineName string, req *http.Request) ([]byte, e } func SearchPipelines(req *http.Request) ([]byte, error) { - baseUrl := jenkins.Server + SearchPipelineUrl + req.URL.RawQuery - log.Infof("Jenkins-url: " + baseUrl) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + baseUrl := devops.Jenkins().Server + SearchPipelineUrl + req.URL.RawQuery res, err := sendJenkinsRequest(baseUrl, req) if err != nil { @@ -73,9 +75,12 @@ func SearchPipelines(req *http.Request) ([]byte, error) { } func SearchPipelineRuns(projectName, pipelineName string, req *http.Request) ([]byte, error) { - baseUrl := fmt.Sprintf(jenkins.Server+SearchPipelineRunUrl, projectName, pipelineName) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } - log.Info("Jenkins-url: " + baseUrl) + baseUrl := fmt.Sprintf(devops.Jenkins().Server+SearchPipelineRunUrl, projectName, pipelineName) res, err := sendJenkinsRequest(baseUrl+req.URL.RawQuery, req) if err != nil { @@ -87,8 +92,12 @@ func SearchPipelineRuns(projectName, pipelineName string, req *http.Request) ([] } func GetBranchPipelineRun(projectName, pipelineName, branchName, runId string, req *http.Request) ([]byte, error) { - baseUrl := fmt.Sprintf(jenkins.Server+GetPipeBranchRunUrl, projectName, pipelineName, branchName, runId) - log.Info("Jenkins-url: " + baseUrl) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + baseUrl := fmt.Sprintf(devops.Jenkins().Server+GetPipeBranchRunUrl, projectName, pipelineName, branchName, runId) res, err := sendJenkinsRequest(baseUrl, req) if err != nil { @@ -100,7 +109,12 @@ func GetBranchPipelineRun(projectName, pipelineName, branchName, runId string, r } func GetPipelineRunNodesbyBranch(projectName, pipelineName, branchName, runId string, req *http.Request) ([]byte, error) { - baseUrl := fmt.Sprintf(jenkins.Server+GetBranchPipeRunNodesUrl+req.URL.RawQuery, projectName, pipelineName, branchName, runId) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + baseUrl := fmt.Sprintf(devops.Jenkins().Server+GetBranchPipeRunNodesUrl+req.URL.RawQuery, projectName, pipelineName, branchName, runId) log.Info("Jenkins-url: " + baseUrl) res, err := sendJenkinsRequest(baseUrl, req) @@ -113,8 +127,12 @@ func GetPipelineRunNodesbyBranch(projectName, pipelineName, branchName, runId st } func GetBranchStepLog(projectName, pipelineName, branchName, runId, nodeId, stepId string, req *http.Request) ([]byte, http.Header, error) { - baseUrl := fmt.Sprintf(jenkins.Server+GetBranchStepLogUrl+req.URL.RawQuery, projectName, pipelineName, branchName, runId, nodeId, stepId) - log.Info("Jenkins-url: " + baseUrl) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + baseUrl := fmt.Sprintf(devops.Jenkins().Server+GetBranchStepLogUrl+req.URL.RawQuery, projectName, pipelineName, branchName, runId, nodeId, stepId) resBody, header, err := jenkinsClient(baseUrl, req) if err != nil { @@ -126,8 +144,12 @@ func GetBranchStepLog(projectName, pipelineName, branchName, runId, nodeId, step } func GetStepLog(projectName, pipelineName, runId, nodeId, stepId string, req *http.Request) ([]byte, http.Header, error) { - baseUrl := fmt.Sprintf(jenkins.Server+GetStepLogUrl+req.URL.RawQuery, projectName, pipelineName, runId, nodeId, stepId) - log.Info("Jenkins-url: " + baseUrl) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + baseUrl := fmt.Sprintf(devops.Jenkins().Server+GetStepLogUrl+req.URL.RawQuery, projectName, pipelineName, runId, nodeId, stepId) resBody, header, err := jenkinsClient(baseUrl, req) if err != nil { @@ -136,12 +158,15 @@ func GetStepLog(projectName, pipelineName, runId, nodeId, stepId string, req *ht } return resBody, header, err - } func GetSCMServers(scmId string, req *http.Request) ([]byte, error) { - baseUrl := fmt.Sprintf(jenkins.Server+GetSCMServersUrl, scmId) - log.Info("Jenkins-url: " + baseUrl) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + baseUrl := fmt.Sprintf(devops.Jenkins().Server+GetSCMServersUrl, scmId) req.Method = http.MethodGet resBody, err := sendJenkinsRequest(baseUrl, req) if err != nil { @@ -152,6 +177,11 @@ func GetSCMServers(scmId string, req *http.Request) ([]byte, error) { } func CreateSCMServers(scmId string, req *http.Request) ([]byte, error) { + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + requestBody, err := ioutil.ReadAll(req.Body) if err != nil { log.Error(err) @@ -169,6 +199,7 @@ func CreateSCMServers(scmId string, req *http.Request) ([]byte, error) { log.Error(err) return nil, err } + var servers []*SCMServer _ = json.Unmarshal(byteServers, &servers) for _, server := range servers { @@ -177,8 +208,9 @@ func CreateSCMServers(scmId string, req *http.Request) ([]byte, error) { } } req.Body = ioutil.NopCloser(bytes.NewReader(requestBody)) - baseUrl := fmt.Sprintf(jenkins.Server+CreateSCMServersUrl, scmId) - log.Info("Jenkins-url: " + baseUrl) + + baseUrl := fmt.Sprintf(devops.Jenkins().Server+CreateSCMServersUrl, scmId) + req.Method = http.MethodPost resBody, err := sendJenkinsRequest(baseUrl, req) if err != nil { @@ -189,8 +221,12 @@ func CreateSCMServers(scmId string, req *http.Request) ([]byte, error) { } func Validate(scmId string, req *http.Request) ([]byte, error) { - baseUrl := fmt.Sprintf(jenkins.Server+ValidateUrl, scmId) - log.Info("Jenkins-url: " + baseUrl) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + baseUrl := fmt.Sprintf(devops.Jenkins().Server+ValidateUrl, scmId) req.Method = http.MethodPut resBody, err := sendJenkinsRequest(baseUrl, req) @@ -203,8 +239,12 @@ func Validate(scmId string, req *http.Request) ([]byte, error) { } func GetSCMOrg(scmId string, req *http.Request) ([]byte, error) { - baseUrl := fmt.Sprintf(jenkins.Server+GetSCMOrgUrl+req.URL.RawQuery, scmId) - log.Info("Jenkins-url: " + baseUrl) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + baseUrl := fmt.Sprintf(devops.Jenkins().Server+GetSCMOrgUrl+req.URL.RawQuery, scmId) res, err := sendJenkinsRequest(baseUrl, req) if err != nil { @@ -216,8 +256,12 @@ func GetSCMOrg(scmId string, req *http.Request) ([]byte, error) { } func GetOrgRepo(scmId, organizationId string, req *http.Request) ([]byte, error) { - baseUrl := fmt.Sprintf(jenkins.Server+GetOrgRepoUrl+req.URL.RawQuery, scmId, organizationId) - log.Info("Jenkins-url: " + baseUrl) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + baseUrl := fmt.Sprintf(devops.Jenkins().Server+GetOrgRepoUrl+req.URL.RawQuery, scmId, organizationId) res, err := sendJenkinsRequest(baseUrl, req) if err != nil { @@ -229,8 +273,12 @@ func GetOrgRepo(scmId, organizationId string, req *http.Request) ([]byte, error) } func StopBranchPipeline(projectName, pipelineName, branchName, runId string, req *http.Request) ([]byte, error) { - baseUrl := fmt.Sprintf(jenkins.Server+StopBranchPipelineUrl+req.URL.RawQuery, projectName, pipelineName, branchName, runId) - log.Info("Jenkins-url: " + baseUrl) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + baseUrl := fmt.Sprintf(devops.Jenkins().Server+StopBranchPipelineUrl+req.URL.RawQuery, projectName, pipelineName, branchName, runId) req.Method = http.MethodPut res, err := sendJenkinsRequest(baseUrl, req) @@ -243,8 +291,12 @@ func StopBranchPipeline(projectName, pipelineName, branchName, runId string, req } func StopPipeline(projectName, pipelineName, runId string, req *http.Request) ([]byte, error) { - baseUrl := fmt.Sprintf(jenkins.Server+StopPipelineUrl+req.URL.RawQuery, projectName, pipelineName, runId) - log.Info("Jenkins-url: " + baseUrl) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + baseUrl := fmt.Sprintf(devops.Jenkins().Server+StopPipelineUrl+req.URL.RawQuery, projectName, pipelineName, runId) req.Method = http.MethodPut res, err := sendJenkinsRequest(baseUrl, req) @@ -257,8 +309,12 @@ func StopPipeline(projectName, pipelineName, runId string, req *http.Request) ([ } func ReplayBranchPipeline(projectName, pipelineName, branchName, runId string, req *http.Request) ([]byte, error) { - baseUrl := fmt.Sprintf(jenkins.Server+ReplayBranchPipelineUrl+req.URL.RawQuery, projectName, pipelineName, branchName, runId) - log.Info("Jenkins-url: " + baseUrl) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + baseUrl := fmt.Sprintf(devops.Jenkins().Server+ReplayBranchPipelineUrl+req.URL.RawQuery, projectName, pipelineName, branchName, runId) res, err := sendJenkinsRequest(baseUrl, req) if err != nil { @@ -270,8 +326,12 @@ func ReplayBranchPipeline(projectName, pipelineName, branchName, runId string, r } func ReplayPipeline(projectName, pipelineName, runId string, req *http.Request) ([]byte, error) { - baseUrl := fmt.Sprintf(jenkins.Server+ReplayPipelineUrl+req.URL.RawQuery, projectName, pipelineName, runId) - log.Info("Jenkins-url: " + baseUrl) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + baseUrl := fmt.Sprintf(devops.Jenkins().Server+ReplayPipelineUrl+req.URL.RawQuery, projectName, pipelineName, runId) res, err := sendJenkinsRequest(baseUrl, req) if err != nil { @@ -283,8 +343,12 @@ func ReplayPipeline(projectName, pipelineName, runId string, req *http.Request) } func GetBranchRunLog(projectName, pipelineName, branchName, runId string, req *http.Request) ([]byte, error) { - baseUrl := fmt.Sprintf(jenkins.Server+GetBranchRunLogUrl+req.URL.RawQuery, projectName, pipelineName, branchName, runId) - log.Info("Jenkins-url: " + baseUrl) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + baseUrl := fmt.Sprintf(devops.Jenkins().Server+GetBranchRunLogUrl+req.URL.RawQuery, projectName, pipelineName, branchName, runId) res, err := sendJenkinsRequest(baseUrl, req) if err != nil { @@ -296,8 +360,12 @@ func GetBranchRunLog(projectName, pipelineName, branchName, runId string, req *h } func GetRunLog(projectName, pipelineName, runId string, req *http.Request) ([]byte, error) { - baseUrl := fmt.Sprintf(jenkins.Server+GetRunLogUrl+req.URL.RawQuery, projectName, pipelineName, runId) - log.Info("Jenkins-url: " + baseUrl) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + baseUrl := fmt.Sprintf(devops.Jenkins().Server+GetRunLogUrl+req.URL.RawQuery, projectName, pipelineName, runId) res, err := sendJenkinsRequest(baseUrl, req) if err != nil { @@ -309,8 +377,12 @@ func GetRunLog(projectName, pipelineName, runId string, req *http.Request) ([]by } func GetBranchArtifacts(projectName, pipelineName, branchName, runId string, req *http.Request) ([]byte, error) { - baseUrl := fmt.Sprintf(jenkins.Server+GetBranchArtifactsUrl+req.URL.RawQuery, projectName, pipelineName, branchName, runId) - log.Info("Jenkins-url: " + baseUrl) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + baseUrl := fmt.Sprintf(devops.Jenkins().Server+GetBranchArtifactsUrl+req.URL.RawQuery, projectName, pipelineName, branchName, runId) res, err := sendJenkinsRequest(baseUrl, req) if err != nil { @@ -322,8 +394,12 @@ func GetBranchArtifacts(projectName, pipelineName, branchName, runId string, req } func GetArtifacts(projectName, pipelineName, runId string, req *http.Request) ([]byte, error) { - baseUrl := fmt.Sprintf(jenkins.Server+GetArtifactsUrl+req.URL.RawQuery, projectName, pipelineName, runId) - log.Info("Jenkins-url: " + baseUrl) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + baseUrl := fmt.Sprintf(devops.Jenkins().Server+GetArtifactsUrl+req.URL.RawQuery, projectName, pipelineName, runId) res, err := sendJenkinsRequest(baseUrl, req) if err != nil { @@ -335,8 +411,12 @@ func GetArtifacts(projectName, pipelineName, runId string, req *http.Request) ([ } func GetPipeBranch(projectName, pipelineName string, req *http.Request) ([]byte, error) { - baseUrl := fmt.Sprintf(jenkins.Server+GetPipeBranchUrl, projectName, pipelineName) - log.Info("Jenkins-url: " + baseUrl) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + baseUrl := fmt.Sprintf(devops.Jenkins().Server+GetPipeBranchUrl, projectName, pipelineName) res, err := sendJenkinsRequest(baseUrl+req.URL.RawQuery, req) if err != nil { @@ -348,8 +428,12 @@ func GetPipeBranch(projectName, pipelineName string, req *http.Request) ([]byte, } func SubmitBranchInputStep(projectName, pipelineName, branchName, runId, nodeId, stepId string, req *http.Request) ([]byte, error) { - baseUrl := fmt.Sprintf(jenkins.Server+CheckBranchPipelineUrl+req.URL.RawQuery, projectName, pipelineName, branchName, runId, nodeId, stepId) - log.Info("Jenkins-url: " + baseUrl) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + baseUrl := fmt.Sprintf(devops.Jenkins().Server+CheckBranchPipelineUrl+req.URL.RawQuery, projectName, pipelineName, branchName, runId, nodeId, stepId) newBody, err := getInputReqBody(req.Body) if err != nil { @@ -367,8 +451,12 @@ func SubmitBranchInputStep(projectName, pipelineName, branchName, runId, nodeId, } func SubmitInputStep(projectName, pipelineName, runId, nodeId, stepId string, req *http.Request) ([]byte, error) { - baseUrl := fmt.Sprintf(jenkins.Server+CheckPipelineUrl+req.URL.RawQuery, projectName, pipelineName, runId, nodeId, stepId) - log.Info("Jenkins-url: " + baseUrl) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + baseUrl := fmt.Sprintf(devops.Jenkins().Server+CheckPipelineUrl+req.URL.RawQuery, projectName, pipelineName, runId, nodeId, stepId) newBody, err := getInputReqBody(req.Body) if err != nil { @@ -425,8 +513,12 @@ func parseBody(body io.Reader) (newReqBody io.ReadCloser) { } func GetConsoleLog(projectName, pipelineName string, req *http.Request) ([]byte, error) { - baseUrl := fmt.Sprintf(jenkins.Server+GetConsoleLogUrl+req.URL.RawQuery, projectName, pipelineName) - log.Info("Jenkins-url: " + baseUrl) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + baseUrl := fmt.Sprintf(devops.Jenkins().Server+GetConsoleLogUrl+req.URL.RawQuery, projectName, pipelineName) resBody, err := sendJenkinsRequest(baseUrl, req) if err != nil { @@ -438,8 +530,12 @@ func GetConsoleLog(projectName, pipelineName string, req *http.Request) ([]byte, } func ScanBranch(projectName, pipelineName string, req *http.Request) ([]byte, error) { - baseUrl := fmt.Sprintf(jenkins.Server+ScanBranchUrl+req.URL.RawQuery, projectName, pipelineName) - log.Info("Jenkins-url: " + baseUrl) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + baseUrl := fmt.Sprintf(devops.Jenkins().Server+ScanBranchUrl+req.URL.RawQuery, projectName, pipelineName) resBody, err := sendJenkinsRequest(baseUrl, req) if err != nil { @@ -451,8 +547,12 @@ func ScanBranch(projectName, pipelineName string, req *http.Request) ([]byte, er } func RunBranchPipeline(projectName, pipelineName, branchName string, req *http.Request) ([]byte, error) { - baseUrl := fmt.Sprintf(jenkins.Server+RunBranchPipelineUrl+req.URL.RawQuery, projectName, pipelineName, branchName) - log.Info("Jenkins-url: " + baseUrl) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + baseUrl := fmt.Sprintf(devops.Jenkins().Server+RunBranchPipelineUrl+req.URL.RawQuery, projectName, pipelineName, branchName) res, err := sendJenkinsRequest(baseUrl, req) if err != nil { @@ -464,8 +564,11 @@ func RunBranchPipeline(projectName, pipelineName, branchName string, req *http.R } func RunPipeline(projectName, pipelineName string, req *http.Request) ([]byte, error) { - baseUrl := fmt.Sprintf(jenkins.Server+RunPipelineUrl+req.URL.RawQuery, projectName, pipelineName) - log.Info("Jenkins-url: " + baseUrl) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + baseUrl := fmt.Sprintf(devops.Jenkins().Server+RunPipelineUrl+req.URL.RawQuery, projectName, pipelineName) res, err := sendJenkinsRequest(baseUrl, req) if err != nil { @@ -477,8 +580,12 @@ func RunPipeline(projectName, pipelineName string, req *http.Request) ([]byte, e } func GetCrumb(req *http.Request) ([]byte, error) { - baseUrl := fmt.Sprintf(jenkins.Server + GetCrumbUrl) - log.Info("Jenkins-url: " + baseUrl) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + baseUrl := fmt.Sprintf(devops.Jenkins().Server + GetCrumbUrl) res, err := sendJenkinsRequest(baseUrl, req) if err != nil { @@ -490,8 +597,12 @@ func GetCrumb(req *http.Request) ([]byte, error) { } func CheckScriptCompile(projectName, pipelineName string, req *http.Request) ([]byte, error) { - baseUrl := fmt.Sprintf(jenkins.Server+CheckScriptCompileUrl, projectName, pipelineName) - log.Info("Jenkins-url: " + baseUrl) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + baseUrl := fmt.Sprintf(devops.Jenkins().Server+CheckScriptCompileUrl, projectName, pipelineName) resBody, err := sendJenkinsRequest(baseUrl, req) if err != nil { @@ -503,6 +614,13 @@ func CheckScriptCompile(projectName, pipelineName string, req *http.Request) ([] } func CheckCron(projectName string, req *http.Request) (*CheckCronRes, error) { + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + jenkins := devops.Jenkins() + var res = new(CheckCronRes) var cron = new(CronData) var reader io.ReadCloser @@ -518,24 +636,24 @@ func CheckCron(projectName string, req *http.Request) (*CheckCronRes, error) { baseUrl = fmt.Sprintf(jenkins.Server+CheckCronUrl, projectName, cron.Cron) } - log.Info("Jenkins-url: " + baseUrl) - newurl, err := url.Parse(baseUrl) + newUrl, err := url.Parse(baseUrl) if err != nil { log.Error(err) return nil, err } - newurl.RawQuery = newurl.Query().Encode() + newUrl.RawQuery = newUrl.Query().Encode() reqJenkins := &http.Request{ Method: http.MethodGet, - URL: newurl, + URL: newUrl, Header: req.Header, } client := &http.Client{Timeout: 30 * time.Second} resp, err := client.Do(reqJenkins) - if resp.StatusCode != http.StatusOK { + + if resp != nil && resp.StatusCode != http.StatusOK { resBody, _ := getRespBody(resp) return &CheckCronRes{ Result: "error", @@ -569,6 +687,7 @@ func CheckCron(projectName string, req *http.Request) (*CheckCronRes, error) { } func parseCronJobTime(msg string) (string, string, error) { + times := strings.Split(msg, ";") lastTmp := strings.SplitN(times[0], " ", 2) @@ -593,8 +712,12 @@ func parseCronJobTime(msg string) (string, string, error) { } func GetPipelineRun(projectName, pipelineName, runId string, req *http.Request) ([]byte, error) { - baseUrl := fmt.Sprintf(jenkins.Server+GetPipelineRunUrl, projectName, pipelineName, runId) - log.Info("Jenkins-url: " + baseUrl) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + baseUrl := fmt.Sprintf(devops.Jenkins().Server+GetPipelineRunUrl, projectName, pipelineName, runId) res, err := sendJenkinsRequest(baseUrl, req) if err != nil { @@ -606,8 +729,12 @@ func GetPipelineRun(projectName, pipelineName, runId string, req *http.Request) } func GetBranchPipeline(projectName, pipelineName, branchName string, req *http.Request) ([]byte, error) { - baseUrl := fmt.Sprintf(jenkins.Server+GetBranchPipeUrl, projectName, pipelineName, branchName) - log.Info("Jenkins-url: " + baseUrl) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + baseUrl := fmt.Sprintf(devops.Jenkins().Server+GetBranchPipeUrl, projectName, pipelineName, branchName) res, err := sendJenkinsRequest(baseUrl, req) if err != nil { @@ -619,8 +746,12 @@ func GetBranchPipeline(projectName, pipelineName, branchName string, req *http.R } func GetPipelineRunNodes(projectName, pipelineName, runId string, req *http.Request) ([]byte, error) { - baseUrl := fmt.Sprintf(jenkins.Server+GetPipeRunNodesUrl+req.URL.RawQuery, projectName, pipelineName, runId) - log.Info("Jenkins-url: " + baseUrl) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + baseUrl := fmt.Sprintf(devops.Jenkins().Server+GetPipeRunNodesUrl+req.URL.RawQuery, projectName, pipelineName, runId) res, err := sendJenkinsRequest(baseUrl, req) if err != nil { @@ -632,8 +763,12 @@ func GetPipelineRunNodes(projectName, pipelineName, runId string, req *http.Requ } func GetBranchNodeSteps(projectName, pipelineName, branchName, runId, nodeId string, req *http.Request) ([]byte, error) { - baseUrl := fmt.Sprintf(jenkins.Server+GetBranchNodeStepsUrl+req.URL.RawQuery, projectName, pipelineName, branchName, runId, nodeId) - log.Info("Jenkins-url: " + baseUrl) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + baseUrl := fmt.Sprintf(devops.Jenkins().Server+GetBranchNodeStepsUrl+req.URL.RawQuery, projectName, pipelineName, branchName, runId, nodeId) res, err := sendJenkinsRequest(baseUrl, req) if err != nil { @@ -645,8 +780,12 @@ func GetBranchNodeSteps(projectName, pipelineName, branchName, runId, nodeId str } func GetNodeSteps(projectName, pipelineName, runId, nodeId string, req *http.Request) ([]byte, error) { - baseUrl := fmt.Sprintf(jenkins.Server+GetNodeStepsUrl+req.URL.RawQuery, projectName, pipelineName, runId, nodeId) - log.Info("Jenkins-url: " + baseUrl) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + baseUrl := fmt.Sprintf(devops.Jenkins().Server+GetNodeStepsUrl+req.URL.RawQuery, projectName, pipelineName, runId, nodeId) res, err := sendJenkinsRequest(baseUrl, req) if err != nil { @@ -658,8 +797,12 @@ func GetNodeSteps(projectName, pipelineName, runId, nodeId string, req *http.Req } func ToJenkinsfile(req *http.Request) ([]byte, error) { - baseUrl := fmt.Sprintf(jenkins.Server + ToJenkinsfileUrl) - log.Info("Jenkins-url: " + baseUrl) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + baseUrl := fmt.Sprintf(devops.Jenkins().Server + ToJenkinsfileUrl) res, err := sendJenkinsRequest(baseUrl, req) if err != nil { @@ -671,8 +814,12 @@ func ToJenkinsfile(req *http.Request) ([]byte, error) { } func ToJson(req *http.Request) ([]byte, error) { - baseUrl := fmt.Sprintf(jenkins.Server + ToJsonUrl) - log.Info("Jenkins-url: " + baseUrl) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + baseUrl := fmt.Sprintf(devops.Jenkins().Server + ToJsonUrl) res, err := sendJenkinsRequest(baseUrl, req) if err != nil { @@ -684,8 +831,12 @@ func ToJson(req *http.Request) ([]byte, error) { } func GetNotifyCommit(req *http.Request) ([]byte, error) { - baseUrl := fmt.Sprint(jenkins.Server, GetNotifyCommitUrl, req.URL.RawQuery) - log.Info("Jenkins-url: " + baseUrl) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + baseUrl := fmt.Sprint(devops.Jenkins().Server, GetNotifyCommitUrl, req.URL.RawQuery) req.Method = "GET" res, err := sendJenkinsRequest(baseUrl, req) @@ -698,8 +849,12 @@ func GetNotifyCommit(req *http.Request) ([]byte, error) { } func GithubWebhook(req *http.Request) ([]byte, error) { - baseUrl := fmt.Sprint(jenkins.Server, GithubWebhookUrl, req.URL.RawQuery) - log.Info("Jenkins-url: " + baseUrl) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + baseUrl := fmt.Sprint(devops.Jenkins().Server, GithubWebhookUrl, req.URL.RawQuery) res, err := sendJenkinsRequest(baseUrl, req) if err != nil { @@ -711,8 +866,6 @@ func GithubWebhook(req *http.Request) ([]byte, error) { } func GetBranchNodesDetail(projectName, pipelineName, branchName, runId string, req *http.Request) ([]NodesDetail, error) { - getNodesUrl := fmt.Sprintf(jenkins.Server+GetBranchPipeRunNodesUrl+req.URL.RawQuery, projectName, pipelineName, branchName, runId) - log.Info("getNodesUrl: " + getNodesUrl) var wg sync.WaitGroup var nodesDetails []NodesDetail stepChan := make(chan *NodesStepsIndex, channelMaxCapacity) @@ -758,8 +911,6 @@ func GetBranchNodesDetail(projectName, pipelineName, branchName, runId string, r } func GetNodesDetail(projectName, pipelineName, runId string, req *http.Request) ([]NodesDetail, error) { - getNodesUrl := fmt.Sprintf(jenkins.Server+GetPipeRunNodesUrl+req.URL.RawQuery, projectName, pipelineName, runId) - log.Info("getNodesUrl: " + getNodesUrl) var wg sync.WaitGroup var nodesDetails []NodesDetail stepChan := make(chan *NodesStepsIndex, channelMaxCapacity) diff --git a/pkg/models/devops/project_credential_handler.go b/pkg/models/devops/project_credential_handler.go index cdf92342e..23559defe 100644 --- a/pkg/models/devops/project_credential_handler.go +++ b/pkg/models/devops/project_credential_handler.go @@ -24,25 +24,25 @@ import ( "kubesphere.io/kubesphere/pkg/db" "kubesphere.io/kubesphere/pkg/gojenkins" "kubesphere.io/kubesphere/pkg/gojenkins/utils" - "kubesphere.io/kubesphere/pkg/simple/client/admin_jenkins" - "kubesphere.io/kubesphere/pkg/simple/client/devops_mysql" + cs "kubesphere.io/kubesphere/pkg/simple/client" "net/http" "strings" ) func CreateProjectCredential(projectId, username string, credentialRequest *JenkinsCredential) (string, error) { - jenkinsClient := admin_jenkins.Client() - if jenkinsClient == nil { - err := fmt.Errorf("could not connect to jenkins") - glog.Error(err) + devops, err := cs.ClientSets().Devops() + if err != nil { return "", restful.NewError(http.StatusServiceUnavailable, err.Error()) } - err := checkJenkinsCredentialExists(projectId, credentialRequest.Domain, credentialRequest.Id) + jenkinsClient := devops.Jenkins() + + err = checkJenkinsCredentialExists(projectId, credentialRequest.Domain, credentialRequest.Id) if err != nil { glog.Errorf("%+v", err) return "", err } + switch credentialRequest.Type { case CredentialTypeUsernamePassword: if credentialRequest.UsernamePasswordCredential == nil { @@ -144,12 +144,12 @@ func CreateProjectCredential(projectId, username string, credentialRequest *Jenk } func UpdateProjectCredential(projectId, credentialId string, credentialRequest *JenkinsCredential) (string, error) { - jenkinsClient := admin_jenkins.Client() - if jenkinsClient == nil { - err := fmt.Errorf("could not connect to jenkins") - glog.Error(err) + devops, err := cs.ClientSets().Devops() + if err != nil { return "", restful.NewError(http.StatusServiceUnavailable, err.Error()) } + jenkinsClient := devops.Jenkins() + jenkinsCredential, err := jenkinsClient.GetCredentialInFolder(credentialRequest.Domain, credentialId, projectId) @@ -238,20 +238,25 @@ func UpdateProjectCredential(projectId, credentialId string, credentialRequest * } func DeleteProjectCredential(projectId, credentialId string, credentialRequest *JenkinsCredential) (string, error) { - jenkinsClient := admin_jenkins.Client() - if jenkinsClient == nil { - err := fmt.Errorf("could not connect to jenkins") - glog.Error(err) + devops, err := cs.ClientSets().Devops() + if err != nil { return "", restful.NewError(http.StatusServiceUnavailable, err.Error()) } - dbClient := devops_mysql.OpenDatabase() - _, err := jenkinsClient.GetCredentialInFolder(credentialRequest.Domain, + jenkinsClient := devops.Jenkins() + + dbClient, err := cs.ClientSets().MySQL() + if err != nil { + return "", restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + _, err = jenkinsClient.GetCredentialInFolder(credentialRequest.Domain, credentialId, projectId) if err != nil { glog.Errorf("%+v", err) return "", restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } + id, err := jenkinsClient.DeleteCredentialInFolder(credentialRequest.Domain, credentialId, projectId) if err != nil { glog.Errorf("%+v", err) @@ -277,13 +282,16 @@ func DeleteProjectCredential(projectId, credentialId string, credentialRequest * } func GetProjectCredential(projectId, credentialId, domain, getContent string) (*JenkinsCredential, error) { - jenkinsClient := admin_jenkins.Client() - if jenkinsClient == nil { - err := fmt.Errorf("could not connect to jenkins") - glog.Error(err) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + jenkinsClient := devops.Jenkins() + + dbClient, err := cs.ClientSets().MySQL() + if err != nil { return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) } - dbClient := devops_mysql.OpenDatabase() jenkinsResponse, err := jenkinsClient.GetCredentialInFolder(domain, credentialId, projectId) @@ -352,13 +360,16 @@ func GetProjectCredential(projectId, credentialId, domain, getContent string) (* } func GetProjectCredentials(projectId, domain string) ([]*JenkinsCredential, error) { - jenkinsClient := admin_jenkins.Client() - if jenkinsClient == nil { - err := fmt.Errorf("could not connect to jenkins") - glog.Error(err) + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + jenkinsClient := devops.Jenkins() + + dbClient, err := cs.ClientSets().MySQL() + if err != nil { return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) } - dbClient := devops_mysql.OpenDatabase() jenkinsCredentialResponses, err := jenkinsClient.GetCredentialsInFolder(domain, projectId) if err != nil { glog.Errorf("%+v", err) @@ -380,9 +391,13 @@ func GetProjectCredentials(projectId, domain string) ([]*JenkinsCredential, erro } func insertCredentialToDb(projectId, credentialId, domain, username string) error { - dbClient := devops_mysql.OpenDatabase() + dbClient, err := cs.ClientSets().MySQL() + if err != nil { + return err + } + projectCredential := NewProjectCredential(projectId, credentialId, domain, username) - _, err := dbClient.InsertInto(ProjectCredentialTableName).Columns(ProjectCredentialColumns...). + _, err = dbClient.InsertInto(ProjectCredentialTableName).Columns(ProjectCredentialColumns...). Record(projectCredential).Exec() if err != nil { glog.Errorf("%+v", err) @@ -392,12 +407,13 @@ func insertCredentialToDb(projectId, credentialId, domain, username string) erro } func checkJenkinsCredentialExists(projectId, domain, credentialId string) error { - jenkinsClient := admin_jenkins.Client() - if jenkinsClient == nil { - err := fmt.Errorf("could not connect to jenkins") - glog.Error(err) + devops, err := cs.ClientSets().Devops() + if err != nil { return restful.NewError(http.StatusServiceUnavailable, err.Error()) } + + jenkinsClient := devops.Jenkins() + credential, err := jenkinsClient.GetCredentialInFolder(domain, credentialId, projectId) if credential != nil { err := fmt.Errorf("credential id [%s] has been used", credential.Id) diff --git a/pkg/models/devops/project_handler.go b/pkg/models/devops/project_handler.go index d2639d256..c0e71fb01 100644 --- a/pkg/models/devops/project_handler.go +++ b/pkg/models/devops/project_handler.go @@ -19,14 +19,17 @@ import ( "github.com/gocraft/dbr" "github.com/golang/glog" "kubesphere.io/kubesphere/pkg/db" - "kubesphere.io/kubesphere/pkg/simple/client/devops_mysql" + cs "kubesphere.io/kubesphere/pkg/simple/client" "net/http" ) func GetProject(projectId string) (*DevOpsProject, error) { - dbconn := devops_mysql.OpenDatabase() + dbconn, err := cs.ClientSets().MySQL() + if err != nil { + return nil, err + } project := &DevOpsProject{} - err := dbconn.Select(DevOpsProjectColumns...). + err = dbconn.Select(DevOpsProjectColumns...). From(DevOpsProjectTableName). Where(db.Eq(DevOpsProjectIdColumn, projectId)). LoadOne(project) @@ -43,7 +46,11 @@ func GetProject(projectId string) (*DevOpsProject, error) { } func UpdateProject(project *DevOpsProject) (*DevOpsProject, error) { - dbconn := devops_mysql.OpenDatabase() + dbconn, err := cs.ClientSets().MySQL() + if err != nil { + return nil, err + } + query := dbconn.Update(DevOpsProjectTableName) if !govalidator.IsNull(project.Description) { query.Set(DevOpsProjectDescriptionColumn, project.Description) @@ -65,7 +72,7 @@ func UpdateProject(project *DevOpsProject) (*DevOpsProject, error) { } } newProject := &DevOpsProject{} - err := dbconn.Select(DevOpsProjectColumns...). + err = dbconn.Select(DevOpsProjectColumns...). From(DevOpsProjectTableName). Where(db.Eq(DevOpsProjectIdColumn, project.ProjectId)). LoadOne(newProject) diff --git a/pkg/models/devops/project_member_handler.go b/pkg/models/devops/project_member_handler.go index 1974dc583..8537c610d 100644 --- a/pkg/models/devops/project_member_handler.go +++ b/pkg/models/devops/project_member_handler.go @@ -14,22 +14,24 @@ package devops import ( "fmt" + "k8s.io/klog" "net/http" "github.com/emicklei/go-restful" "github.com/gocraft/dbr" - "github.com/golang/glog" "kubesphere.io/kubesphere/pkg/db" "kubesphere.io/kubesphere/pkg/gojenkins" "kubesphere.io/kubesphere/pkg/gojenkins/utils" "kubesphere.io/kubesphere/pkg/models" "kubesphere.io/kubesphere/pkg/params" - "kubesphere.io/kubesphere/pkg/simple/client/admin_jenkins" - "kubesphere.io/kubesphere/pkg/simple/client/devops_mysql" + cs "kubesphere.io/kubesphere/pkg/simple/client" ) func GetProjectMembers(projectId string, conditions *params.Conditions, orderBy string, reverse bool, limit int, offset int) (*models.PageableResponse, error) { - dbconn := devops_mysql.OpenDatabase() + dbconn, err := cs.ClientSets().MySQL() + if err != nil { + return nil, err + } memberships := make([]*DevOpsProjectMembership, 0) var sqconditions []dbr.Builder sqconditions = append(sqconditions, db.Eq(DevOpsProjectMembershipProjectIdColumn, projectId)) @@ -59,14 +61,14 @@ func GetProjectMembers(projectId string, conditions *params.Conditions, orderBy } else { query.Where(sqconditions[0]) } - _, err := query.Load(&memberships) + _, err = query.Load(&memberships) if err != nil && err != dbr.ErrNotFound { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(http.StatusInternalServerError, err.Error()) } count, err := query.Count() if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(http.StatusInternalServerError, err.Error()) } result := make([]interface{}, 0) @@ -78,35 +80,46 @@ func GetProjectMembers(projectId string, conditions *params.Conditions, orderBy } func GetProjectMember(projectId, username string) (*DevOpsProjectMembership, error) { - dbconn := devops_mysql.OpenDatabase() + dbconn, err := cs.ClientSets().MySQL() + if err != nil { + return nil, err + } + member := &DevOpsProjectMembership{} - err := dbconn.Select(DevOpsProjectMembershipColumns...). + err = dbconn.Select(DevOpsProjectMembershipColumns...). From(DevOpsProjectMembershipTableName). Where(db.And(db.Eq(DevOpsProjectMembershipProjectIdColumn, projectId), db.Eq(DevOpsProjectMembershipUsernameColumn, username))). LoadOne(&member) if err != nil && err != dbr.ErrNotFound { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(http.StatusInternalServerError, err.Error()) } if err == dbr.ErrNotFound { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(http.StatusNotFound, err.Error()) } return member, nil } func AddProjectMember(projectId, operator string, member *DevOpsProjectMembership) (*DevOpsProjectMembership, error) { - dbconn := devops_mysql.OpenDatabase() - jenkinsClient := admin_jenkins.Client() + dbconn, err := cs.ClientSets().MySQL() + if err != nil { + return nil, err + } + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, err + } + jenkinsClient := devops.Jenkins() if jenkinsClient == nil { err := fmt.Errorf("could not connect to jenkins") - glog.Error(err) + klog.Error(err) return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) } membership := &DevOpsProjectMembership{} - err := dbconn.Select(DevOpsProjectMembershipColumns...). + err = dbconn.Select(DevOpsProjectMembershipColumns...). From(DevOpsProjectMembershipTableName). Where(db.And( db.Eq(DevOpsProjectMembershipUsernameColumn, member.Username), @@ -114,18 +127,18 @@ func AddProjectMember(projectId, operator string, member *DevOpsProjectMembershi // if user could be founded in db, user have been added to project if err == nil { err = fmt.Errorf("user [%s] have been added to project", member.Username) - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(http.StatusBadRequest, err.Error()) } - if err != nil && err != db.ErrNotFound { - glog.Errorf("%+v", err) + if err != db.ErrNotFound { + klog.Errorf("%+v", err) return nil, restful.NewError(http.StatusInternalServerError, err.Error()) } globalRole, err := jenkinsClient.GetGlobalRole(JenkinsAllUserRoleName) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } if globalRole == nil { @@ -133,33 +146,33 @@ func AddProjectMember(projectId, operator string, member *DevOpsProjectMembershi GlobalRead: true, }, true) if err != nil { - glog.Errorf("failed to create jenkins global role %+v", err) + klog.Errorf("failed to create jenkins global role %+v", err) return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } } err = globalRole.AssignRole(member.Username) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } projectRole, err := jenkinsClient.GetProjectRole(GetProjectRoleName(projectId, member.Role)) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } err = projectRole.AssignRole(member.Username) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } pipelineRole, err := jenkinsClient.GetProjectRole(GetPipelineRoleName(projectId, member.Role)) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } err = pipelineRole.AssignRole(member.Username) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } projectMembership := NewDevOpsProjectMemberShip(member.Username, projectId, member.Role, operator) @@ -168,15 +181,15 @@ func AddProjectMember(projectId, operator string, member *DevOpsProjectMembershi Columns(DevOpsProjectMembershipColumns...). Record(projectMembership).Exec() if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) err = projectRole.UnAssignRole(member.Username) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } err = pipelineRole.UnAssignRole(member.Username) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } return nil, restful.NewError(http.StatusInternalServerError, err.Error()) @@ -185,66 +198,82 @@ func AddProjectMember(projectId, operator string, member *DevOpsProjectMembershi } func UpdateProjectMember(projectId, operator string, member *DevOpsProjectMembership) (*DevOpsProjectMembership, error) { - dbconn := devops_mysql.OpenDatabase() - jenkinsClient := admin_jenkins.Client() - if jenkinsClient == nil { - err := fmt.Errorf("could not connect to jenkins") - glog.Error(err) + dbconn, err := cs.ClientSets().MySQL() + if err != nil { return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) } + devops, err := cs.ClientSets().Devops() + if err != nil { + return nil, err + } + + jenkinsClient := devops.Jenkins() + if jenkinsClient == nil { + err := fmt.Errorf("could not connect to jenkins") + klog.Error(err) + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + oldMembership := &DevOpsProjectMembership{} - err := dbconn.Select(DevOpsProjectMembershipColumns...). + err = dbconn.Select(DevOpsProjectMembershipColumns...). From(DevOpsProjectMembershipTableName). Where(db.And( db.Eq(DevOpsProjectMembershipUsernameColumn, member.Username), db.Eq(DevOpsProjectMembershipProjectIdColumn, projectId), )).LoadOne(oldMembership) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(http.StatusBadRequest, err.Error()) } oldProjectRole, err := jenkinsClient.GetProjectRole(GetProjectRoleName(projectId, oldMembership.Role)) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } + err = oldProjectRole.UnAssignRole(member.Username) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } + oldPipelineRole, err := jenkinsClient.GetProjectRole(GetPipelineRoleName(projectId, oldMembership.Role)) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } + err = oldPipelineRole.UnAssignRole(member.Username) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } projectRole, err := jenkinsClient.GetProjectRole(GetProjectRoleName(projectId, member.Role)) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } + err = projectRole.AssignRole(member.Username) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } + pipelineRole, err := jenkinsClient.GetProjectRole(GetPipelineRoleName(projectId, member.Role)) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } + err = pipelineRole.AssignRole(member.Username) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } + _, err = dbconn.Update(DevOpsProjectMembershipTableName). Set(DevOpsProjectMembershipRoleColumn, member.Role). Where(db.And( @@ -252,9 +281,10 @@ func UpdateProjectMember(projectId, operator string, member *DevOpsProjectMember db.Eq(DevOpsProjectMembershipUsernameColumn, member.Username), )).Exec() if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(http.StatusInternalServerError, err.Error()) } + responseMembership := &DevOpsProjectMembership{} err = dbconn.Select(DevOpsProjectMembershipColumns...). From(DevOpsProjectMembershipTableName). @@ -263,35 +293,42 @@ func UpdateProjectMember(projectId, operator string, member *DevOpsProjectMember db.Eq(DevOpsProjectMembershipProjectIdColumn, projectId), )).LoadOne(responseMembership) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(http.StatusInternalServerError, err.Error()) } return responseMembership, nil } func DeleteProjectMember(projectId, username string) (string, error) { - dbconn := devops_mysql.OpenDatabase() - jenkinsClient := admin_jenkins.Client() - if jenkinsClient == nil { - err := fmt.Errorf("could not connect to jenkins") - glog.Error(err) + dbconn, err := cs.ClientSets().MySQL() + if err != nil { return "", restful.NewError(http.StatusServiceUnavailable, err.Error()) } + + devops, err := cs.ClientSets().Devops() + if err != nil { + return "", restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + jenkinsClient := devops.Jenkins() + oldMembership := &DevOpsProjectMembership{} - err := dbconn.Select(DevOpsProjectMembershipColumns...). + err = dbconn.Select(DevOpsProjectMembershipColumns...). From(DevOpsProjectMembershipTableName). Where(db.And( db.Eq(DevOpsProjectMembershipUsernameColumn, username), db.Eq(DevOpsProjectMembershipProjectIdColumn, projectId), )).LoadOne(oldMembership) - if err != nil && err != db.ErrNotFound { - glog.Errorf("%+v", err) - return "", restful.NewError(http.StatusInternalServerError, err.Error()) - } - if err == db.ErrNotFound { - glog.Warningf("user [%s] not found in project", username) - return username, nil + if err != nil { + if err != db.ErrNotFound { + klog.Errorf("%+v", err) + return "", restful.NewError(http.StatusInternalServerError, err.Error()) + } else if err == db.ErrNotFound { + klog.Warningf("user [%s] not found in project", username) + return username, nil + } } + if oldMembership.Role == ProjectOwner { count, err := dbconn.Select(DevOpsProjectMembershipProjectIdColumn). From(DevOpsProjectMembershipTableName). @@ -299,34 +336,35 @@ func DeleteProjectMember(projectId, username string) (string, error) { db.Eq(DevOpsProjectMembershipProjectIdColumn, projectId), db.Eq(DevOpsProjectMembershipRoleColumn, ProjectOwner))).Count() if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return "", restful.NewError(http.StatusInternalServerError, err.Error()) } if count == 1 { err = fmt.Errorf("project must has at least one admin") - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return "", restful.NewError(http.StatusBadRequest, err.Error()) } } + oldProjectRole, err := jenkinsClient.GetProjectRole(GetProjectRoleName(projectId, oldMembership.Role)) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return "", restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } err = oldProjectRole.UnAssignRole(username) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return "", restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } oldPipelineRole, err := jenkinsClient.GetProjectRole(GetPipelineRoleName(projectId, oldMembership.Role)) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return "", restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } err = oldPipelineRole.UnAssignRole(username) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return "", restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } @@ -336,7 +374,7 @@ func DeleteProjectMember(projectId, username string) (string, error) { db.Eq(DevOpsProjectMembershipUsernameColumn, username), )).Exec() if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return "", restful.NewError(http.StatusInternalServerError, err.Error()) } return username, nil diff --git a/pkg/models/devops/project_pipeline.go b/pkg/models/devops/project_pipeline.go index f50f7aa23..2f9d4de0e 100644 --- a/pkg/models/devops/project_pipeline.go +++ b/pkg/models/devops/project_pipeline.go @@ -19,7 +19,7 @@ import ( "github.com/golang/glog" "github.com/kubesphere/sonargo/sonar" "kubesphere.io/kubesphere/pkg/gojenkins" - "kubesphere.io/kubesphere/pkg/simple/client/sonarqube" + "kubesphere.io/kubesphere/pkg/simple/client" "strconv" "strings" "time" @@ -175,7 +175,7 @@ type RemoteTrigger struct { } func replaceXmlVersion(config, oldVersion, targetVersion string) string { - lines := strings.Split(string(config), "\n") + lines := strings.Split(config, "\n") lines[0] = strings.Replace(lines[0], oldVersion, targetVersion, -1) output := strings.Join(lines, "\n") return output @@ -1077,7 +1077,12 @@ func toCrontab(millis int64) string { } func getBuildSonarResults(build *gojenkins.Build) ([]*SonarStatus, error) { - sonarClient := sonarqube.Client() + + sonarClient, err := client.ClientSets().SonarQube() + if err != nil { + return nil, err + } + actions := build.GetActions() sonarStatuses := make([]*SonarStatus, 0) for _, action := range actions { @@ -1086,7 +1091,7 @@ func getBuildSonarResults(build *gojenkins.Build) ([]*SonarStatus, error) { taskOptions := &sonargo.CeTaskOption{ Id: action.SonarTaskId, } - ceTask, _, err := sonarClient.Ce.Task(taskOptions) + ceTask, _, err := sonarClient.SonarQube().Ce.Task(taskOptions) if err != nil { glog.Errorf("get sonar task error [%+v]", err) continue @@ -1097,7 +1102,7 @@ func getBuildSonarResults(build *gojenkins.Build) ([]*SonarStatus, error) { AdditionalFields: SonarAdditionalFields, MetricKeys: SonarMetricKeys, } - measures, _, err := sonarClient.Measures.Component(measuresComponentOption) + measures, _, err := sonarClient.SonarQube().Measures.Component(measuresComponentOption) if err != nil { glog.Errorf("get sonar task error [%+v]", err) continue @@ -1112,7 +1117,7 @@ func getBuildSonarResults(build *gojenkins.Build) ([]*SonarStatus, error) { S: "FILE_LINE", Facets: "severities,types", } - issuesSearch, _, err := sonarClient.Issues.Search(issuesSearchOption) + issuesSearch, _, err := sonarClient.SonarQube().Issues.Search(issuesSearchOption) sonarStatus.Issues = issuesSearch jenkinsAction := action sonarStatus.JenkinsAction = &jenkinsAction diff --git a/pkg/models/devops/project_pipeline_handler.go b/pkg/models/devops/project_pipeline_handler.go index 1bbc2eee2..96f6986d9 100644 --- a/pkg/models/devops/project_pipeline_handler.go +++ b/pkg/models/devops/project_pipeline_handler.go @@ -16,43 +16,43 @@ package devops import ( "fmt" "github.com/emicklei/go-restful" - "github.com/golang/glog" + "k8s.io/klog" "kubesphere.io/kubesphere/pkg/gojenkins/utils" - "kubesphere.io/kubesphere/pkg/simple/client/admin_jenkins" + cs "kubesphere.io/kubesphere/pkg/simple/client" "net/http" ) func CreateProjectPipeline(projectId string, pipeline *ProjectPipeline) (string, error) { - jenkinsClient := admin_jenkins.Client() - if jenkinsClient == nil { - err := fmt.Errorf("could not connect to jenkins") - glog.Error(err) + devops, err := cs.ClientSets().Devops() + if err != nil { return "", restful.NewError(http.StatusServiceUnavailable, err.Error()) } + jenkinsClient := devops.Jenkins() + switch pipeline.Type { case NoScmPipelineType: config, err := createPipelineConfigXml(pipeline.Pipeline) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return "", restful.NewError(http.StatusInternalServerError, err.Error()) } job, err := jenkinsClient.GetJob(pipeline.Pipeline.Name, projectId) if job != nil { err := fmt.Errorf("job name [%s] has been used", job.GetName()) - glog.Warning(err.Error()) + klog.Warning(err.Error()) return "", restful.NewError(http.StatusConflict, err.Error()) } if err != nil && utils.GetJenkinsStatusCode(err) != http.StatusNotFound { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return "", restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } _, err = jenkinsClient.CreateJobInFolder(config, pipeline.Pipeline.Name, projectId) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return "", restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } @@ -60,7 +60,7 @@ func CreateProjectPipeline(projectId string, pipeline *ProjectPipeline) (string, case MultiBranchPipelineType: config, err := createMultiBranchPipelineConfigXml(projectId, pipeline.MultiBranchPipeline) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return "", restful.NewError(http.StatusInternalServerError, err.Error()) } @@ -68,18 +68,18 @@ func CreateProjectPipeline(projectId string, pipeline *ProjectPipeline) (string, job, err := jenkinsClient.GetJob(pipeline.MultiBranchPipeline.Name, projectId) if job != nil { err := fmt.Errorf("job name [%s] has been used", job.GetName()) - glog.Warning(err.Error()) + klog.Warning(err.Error()) return "", restful.NewError(http.StatusConflict, err.Error()) } if err != nil && utils.GetJenkinsStatusCode(err) != http.StatusNotFound { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return "", restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } _, err = jenkinsClient.CreateJobInFolder(config, pipeline.MultiBranchPipeline.Name, projectId) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return "", restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } @@ -87,52 +87,52 @@ func CreateProjectPipeline(projectId string, pipeline *ProjectPipeline) (string, default: err := fmt.Errorf("error unsupport job type") - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return "", restful.NewError(http.StatusBadRequest, err.Error()) } } func DeleteProjectPipeline(projectId string, pipelineId string) (string, error) { - jenkinsClient := admin_jenkins.Client() - if jenkinsClient == nil { - err := fmt.Errorf("could not connect to jenkins") - glog.Error(err) + devops, err := cs.ClientSets().Devops() + if err != nil { return "", restful.NewError(http.StatusServiceUnavailable, err.Error()) } - _, err := jenkinsClient.DeleteJob(pipelineId, projectId) + jenkinsClient := devops.Jenkins() + + _, err = jenkinsClient.DeleteJob(pipelineId, projectId) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return "", restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } return pipelineId, nil } func UpdateProjectPipeline(projectId, pipelineId string, pipeline *ProjectPipeline) (string, error) { - jenkinsClient := admin_jenkins.Client() - if jenkinsClient == nil { - err := fmt.Errorf("could not connect to jenkins") - glog.Error(err) + devops, err := cs.ClientSets().Devops() + if err != nil { return "", restful.NewError(http.StatusServiceUnavailable, err.Error()) } + jenkinsClient := devops.Jenkins() + switch pipeline.Type { case NoScmPipelineType: config, err := createPipelineConfigXml(pipeline.Pipeline) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return "", restful.NewError(http.StatusInternalServerError, err.Error()) } job, err := jenkinsClient.GetJob(pipelineId, projectId) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return "", restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } err = job.UpdateConfig(config) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return "", restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } @@ -141,7 +141,7 @@ func UpdateProjectPipeline(projectId, pipelineId string, pipeline *ProjectPipeli config, err := createMultiBranchPipelineConfigXml(projectId, pipeline.MultiBranchPipeline) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return "", restful.NewError(http.StatusInternalServerError, err.Error()) } @@ -149,13 +149,13 @@ func UpdateProjectPipeline(projectId, pipelineId string, pipeline *ProjectPipeli job, err := jenkinsClient.GetJob(pipelineId, projectId) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return "", restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } err = job.UpdateConfig(config) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return "", restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } @@ -163,33 +163,33 @@ func UpdateProjectPipeline(projectId, pipelineId string, pipeline *ProjectPipeli default: err := fmt.Errorf("error unsupport job type") - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return "", restful.NewError(http.StatusBadRequest, err.Error()) } } func GetProjectPipeline(projectId, pipelineId string) (*ProjectPipeline, error) { - jenkinsClient := admin_jenkins.Client() - if jenkinsClient == nil { - err := fmt.Errorf("could not connect to jenkins") - glog.Error(err) + devops, err := cs.ClientSets().Devops() + if err != nil { return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) } + jenkinsClient := devops.Jenkins() + job, err := jenkinsClient.GetJob(pipelineId, projectId) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } switch job.Raw.Class { case "org.jenkinsci.plugins.workflow.job.WorkflowJob": config, err := job.GetConfig() if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } pipeline, err := parsePipelineConfigXml(config) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } pipeline.Name = pipelineId @@ -201,12 +201,12 @@ func GetProjectPipeline(projectId, pipelineId string) (*ProjectPipeline, error) case "org.jenkinsci.plugins.workflow.multibranch.WorkflowMultiBranchProject": config, err := job.GetConfig() if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } pipeline, err := parseMultiBranchPipelineConfigXml(config) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } pipeline.Name = pipelineId @@ -216,50 +216,50 @@ func GetProjectPipeline(projectId, pipelineId string) (*ProjectPipeline, error) }, nil default: err := fmt.Errorf("error unsupport job type") - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(http.StatusBadRequest, err.Error()) } } func GetPipelineSonar(projectId, pipelineId string) ([]*SonarStatus, error) { - jenkinsClient := admin_jenkins.Client() - if jenkinsClient == nil { - err := fmt.Errorf("could not connect to jenkins") - glog.Error(err) + devops, err := cs.ClientSets().Devops() + if err != nil { return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) } + jenkinsClient := devops.Jenkins() + job, err := jenkinsClient.GetJob(pipelineId, projectId) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } build, err := job.GetLastBuild() if err != nil && utils.GetJenkinsStatusCode(err) != http.StatusNotFound { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } else if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, nil } sonarStatus, err := getBuildSonarResults(build) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(http.StatusBadRequest, err.Error()) } if len(sonarStatus) == 0 { build, err := job.GetLastCompletedBuild() if err != nil && utils.GetJenkinsStatusCode(err) != http.StatusNotFound { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } else if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, nil } sonarStatus, err = getBuildSonarResults(build) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(http.StatusBadRequest, err.Error()) } } @@ -267,43 +267,43 @@ func GetPipelineSonar(projectId, pipelineId string) ([]*SonarStatus, error) { } func GetMultiBranchPipelineSonar(projectId, pipelineId, branchId string) ([]*SonarStatus, error) { - jenkinsClient := admin_jenkins.Client() - if jenkinsClient == nil { - err := fmt.Errorf("could not connect to jenkins") - glog.Error(err) + devops, err := cs.ClientSets().Devops() + if err != nil { return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) } + jenkinsClient := devops.Jenkins() + job, err := jenkinsClient.GetJob(branchId, projectId, pipelineId) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } build, err := job.GetLastBuild() if err != nil && utils.GetJenkinsStatusCode(err) != http.StatusNotFound { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } else if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, nil } sonarStatus, err := getBuildSonarResults(build) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(http.StatusBadRequest, err.Error()) } if len(sonarStatus) == 0 { build, err := job.GetLastCompletedBuild() if err != nil && utils.GetJenkinsStatusCode(err) != http.StatusNotFound { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } else if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, nil } sonarStatus, err = getBuildSonarResults(build) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(http.StatusBadRequest, err.Error()) } } diff --git a/pkg/models/devops/project_pipeline_test.go b/pkg/models/devops/project_pipeline_test.go index 99514758a..4f572b554 100644 --- a/pkg/models/devops/project_pipeline_test.go +++ b/pkg/models/devops/project_pipeline_test.go @@ -20,17 +20,17 @@ import ( func Test_NoScmPipelineConfig(t *testing.T) { inputs := []*NoScmPipeline{ - &NoScmPipeline{ + { Name: "", Description: "for test", Jenkinsfile: "node{echo 'hello'}", }, - &NoScmPipeline{ + { Name: "", Description: "", Jenkinsfile: "node{echo 'hello'}", }, - &NoScmPipeline{ + { Name: "", Description: "", Jenkinsfile: "node{echo 'hello'}", @@ -55,7 +55,7 @@ func Test_NoScmPipelineConfig(t *testing.T) { func Test_NoScmPipelineConfig_Discarder(t *testing.T) { inputs := []*NoScmPipeline{ - &NoScmPipeline{ + { Name: "", Description: "for test", Jenkinsfile: "node{echo 'hello'}", @@ -63,7 +63,7 @@ func Test_NoScmPipelineConfig_Discarder(t *testing.T) { "3", "5", }, }, - &NoScmPipeline{ + { Name: "", Description: "for test", Jenkinsfile: "node{echo 'hello'}", @@ -71,7 +71,7 @@ func Test_NoScmPipelineConfig_Discarder(t *testing.T) { "3", "", }, }, - &NoScmPipeline{ + { Name: "", Description: "for test", Jenkinsfile: "node{echo 'hello'}", @@ -79,7 +79,7 @@ func Test_NoScmPipelineConfig_Discarder(t *testing.T) { "", "21321", }, }, - &NoScmPipeline{ + { Name: "", Description: "for test", Jenkinsfile: "node{echo 'hello'}", @@ -106,7 +106,7 @@ func Test_NoScmPipelineConfig_Discarder(t *testing.T) { func Test_NoScmPipelineConfig_Param(t *testing.T) { inputs := []*NoScmPipeline{ - &NoScmPipeline{ + { Name: "", Description: "for test", Jenkinsfile: "node{echo 'hello'}", @@ -119,7 +119,7 @@ func Test_NoScmPipelineConfig_Param(t *testing.T) { }, }, }, - &NoScmPipeline{ + { Name: "", Description: "for test", Jenkinsfile: "node{echo 'hello'}", @@ -169,7 +169,7 @@ func Test_NoScmPipelineConfig_Param(t *testing.T) { func Test_NoScmPipelineConfig_Trigger(t *testing.T) { inputs := []*NoScmPipeline{ - &NoScmPipeline{ + { Name: "", Description: "for test", Jenkinsfile: "node{echo 'hello'}", @@ -178,7 +178,7 @@ func Test_NoScmPipelineConfig_Trigger(t *testing.T) { }, }, - &NoScmPipeline{ + { Name: "", Description: "for test", Jenkinsfile: "node{echo 'hello'}", @@ -186,7 +186,7 @@ func Test_NoScmPipelineConfig_Trigger(t *testing.T) { Token: "abc", }, }, - &NoScmPipeline{ + { Name: "", Description: "for test", Jenkinsfile: "node{echo 'hello'}", @@ -218,28 +218,28 @@ func Test_NoScmPipelineConfig_Trigger(t *testing.T) { func Test_MultiBranchPipelineConfig(t *testing.T) { inputs := []*MultiBranchPipeline{ - &MultiBranchPipeline{ + { Name: "", Description: "for test", ScriptPath: "Jenkinsfile", SourceType: "git", GitSource: &GitSource{}, }, - &MultiBranchPipeline{ + { Name: "", Description: "for test", ScriptPath: "Jenkinsfile", SourceType: "github", GitHubSource: &GithubSource{}, }, - &MultiBranchPipeline{ + { Name: "", Description: "for test", ScriptPath: "Jenkinsfile", SourceType: "single_svn", SingleSvnSource: &SingleSvnSource{}, }, - &MultiBranchPipeline{ + { Name: "", Description: "for test", ScriptPath: "Jenkinsfile", @@ -266,7 +266,7 @@ func Test_MultiBranchPipelineConfig(t *testing.T) { func Test_MultiBranchPipelineConfig_Discarder(t *testing.T) { inputs := []*MultiBranchPipeline{ - &MultiBranchPipeline{ + { Name: "", Description: "for test", ScriptPath: "Jenkinsfile", @@ -296,7 +296,7 @@ func Test_MultiBranchPipelineConfig_Discarder(t *testing.T) { func Test_MultiBranchPipelineConfig_TimerTrigger(t *testing.T) { inputs := []*MultiBranchPipeline{ - &MultiBranchPipeline{ + { Name: "", Description: "for test", ScriptPath: "Jenkinsfile", @@ -326,7 +326,7 @@ func Test_MultiBranchPipelineConfig_TimerTrigger(t *testing.T) { func Test_MultiBranchPipelineConfig_Source(t *testing.T) { inputs := []*MultiBranchPipeline{ - &MultiBranchPipeline{ + { Name: "", Description: "for test", ScriptPath: "Jenkinsfile", @@ -340,7 +340,7 @@ func Test_MultiBranchPipelineConfig_Source(t *testing.T) { DiscoverBranches: true, }, }, - &MultiBranchPipeline{ + { Name: "", Description: "for test", ScriptPath: "Jenkinsfile", @@ -361,7 +361,7 @@ func Test_MultiBranchPipelineConfig_Source(t *testing.T) { }, }, }, - &MultiBranchPipeline{ + { Name: "", Description: "for test", ScriptPath: "Jenkinsfile", @@ -383,7 +383,7 @@ func Test_MultiBranchPipelineConfig_Source(t *testing.T) { }, }, - &MultiBranchPipeline{ + { Name: "", Description: "for test", ScriptPath: "Jenkinsfile", @@ -398,7 +398,7 @@ func Test_MultiBranchPipelineConfig_Source(t *testing.T) { Includes: "tag/*", }, }, - &MultiBranchPipeline{ + { Name: "", Description: "for test", ScriptPath: "Jenkinsfile", @@ -432,7 +432,7 @@ func Test_MultiBranchPipelineConfig_Source(t *testing.T) { func Test_MultiBranchPipelineCloneConfig(t *testing.T) { inputs := []*MultiBranchPipeline{ - &MultiBranchPipeline{ + { Name: "", Description: "for test", ScriptPath: "Jenkinsfile", @@ -448,7 +448,7 @@ func Test_MultiBranchPipelineCloneConfig(t *testing.T) { }, }, }, - &MultiBranchPipeline{ + { Name: "", Description: "for test", ScriptPath: "Jenkinsfile", @@ -493,7 +493,7 @@ func Test_MultiBranchPipelineCloneConfig(t *testing.T) { func Test_MultiBranchPipelineRegexFilter(t *testing.T) { inputs := []*MultiBranchPipeline{ - &MultiBranchPipeline{ + { Name: "", Description: "for test", ScriptPath: "Jenkinsfile", @@ -505,7 +505,7 @@ func Test_MultiBranchPipelineRegexFilter(t *testing.T) { RegexFilter: ".*", }, }, - &MultiBranchPipeline{ + { Name: "", Description: "for test", ScriptPath: "Jenkinsfile", @@ -546,7 +546,7 @@ func Test_MultiBranchPipelineRegexFilter(t *testing.T) { func Test_MultiBranchPipelineMultibranchTrigger(t *testing.T) { inputs := []*MultiBranchPipeline{ - &MultiBranchPipeline{ + { Name: "", Description: "for test", ScriptPath: "Jenkinsfile", @@ -569,7 +569,7 @@ func Test_MultiBranchPipelineMultibranchTrigger(t *testing.T) { DeleteActionJobsToTrigger: "ddd", }, }, - &MultiBranchPipeline{ + { Name: "", Description: "for test", ScriptPath: "Jenkinsfile", @@ -591,7 +591,7 @@ func Test_MultiBranchPipelineMultibranchTrigger(t *testing.T) { CreateActionJobsToTrigger: "abc", }, }, - &MultiBranchPipeline{ + { Name: "", Description: "for test", ScriptPath: "Jenkinsfile", diff --git a/pkg/models/devops/s2ibinary_handler.go b/pkg/models/devops/s2ibinary_handler.go index 1c7db5cdb..eec84a0cb 100644 --- a/pkg/models/devops/s2ibinary_handler.go +++ b/pkg/models/devops/s2ibinary_handler.go @@ -13,8 +13,7 @@ import ( "k8s.io/klog" "kubesphere.io/kubesphere/pkg/apis/devops/v1alpha1" "kubesphere.io/kubesphere/pkg/informers" - "kubesphere.io/kubesphere/pkg/simple/client/k8s" - "kubesphere.io/kubesphere/pkg/simple/client/s2is3" + "kubesphere.io/kubesphere/pkg/simple/client" "mime/multipart" "net/http" "reflect" @@ -26,6 +25,11 @@ const ( ) func UploadS2iBinary(namespace, name, md5 string, fileHeader *multipart.FileHeader) (*v1alpha1.S2iBinary, error) { + s3Client, err := client.ClientSets().S3() + if err != nil { + return nil, err + } + binFile, err := fileHeader.Open() if err != nil { klog.Errorf("%+v", err) @@ -66,7 +70,8 @@ func UploadS2iBinary(namespace, name, md5 string, fileHeader *multipart.FileHead copy.Spec.Size = bytefmt.ByteSize(uint64(fileHeader.Size)) copy.Spec.FileName = fileHeader.Filename copy.Spec.DownloadURL = fmt.Sprintf(GetS2iBinaryURL, namespace, name, copy.Spec.FileName) - s3session := s2is3.Session() + + s3session := s3Client.Session() if s3session == nil { err := fmt.Errorf("could not connect to s2i s3") klog.Error(err) @@ -82,7 +87,7 @@ func UploadS2iBinary(namespace, name, md5 string, fileHeader *multipart.FileHead uploader.LeavePartsOnError = true }) _, err = uploader.Upload(&s3manager.UploadInput{ - Bucket: s2is3.Bucket(), + Bucket: s3Client.Bucket(), Key: aws.String(fmt.Sprintf("%s-%s", namespace, name)), Body: binFile, ContentMD5: aws.String(md5), @@ -116,7 +121,7 @@ func UploadS2iBinary(namespace, name, md5 string, fileHeader *multipart.FileHead copy.Spec.UploadTimeStamp = new(metav1.Time) } *copy.Spec.UploadTimeStamp = metav1.Now() - copy, err = k8s.KsClient().DevopsV1alpha1().S2iBinaries(namespace).Update(copy) + copy, err = client.ClientSets().K8s().KubeSphere().DevopsV1alpha1().S2iBinaries(namespace).Update(copy) if err != nil { klog.Error(err) return nil, err @@ -131,6 +136,11 @@ func UploadS2iBinary(namespace, name, md5 string, fileHeader *multipart.FileHead } func DownloadS2iBinary(namespace, name, fileName string) (string, error) { + s3Client, err := client.ClientSets().S3() + if err != nil { + return "", err + } + origin, err := informers.KsSharedInformerFactory().Devops().V1alpha1().S2iBinaries().Lister().S2iBinaries(namespace).Get(name) if err != nil { klog.Errorf("%+v", err) @@ -146,14 +156,9 @@ func DownloadS2iBinary(namespace, name, fileName string) (string, error) { klog.Error(err) return "", err } - s3Client := s2is3.Client() - if s3Client == nil { - err := fmt.Errorf("could not get s3 client") - klog.Error(err) - return "", err - } - req, _ := s3Client.GetObjectRequest(&s3.GetObjectInput{ - Bucket: s2is3.Bucket(), + + req, _ := s3Client.Client().GetObjectRequest(&s3.GetObjectInput{ + Bucket: s3Client.Bucket(), Key: aws.String(fmt.Sprintf("%s-%s", namespace, name)), ResponseContentDisposition: aws.String(fmt.Sprintf("attachment; filename=\"%s\"", origin.Spec.FileName)), }) @@ -169,7 +174,7 @@ func DownloadS2iBinary(namespace, name, fileName string) (string, error) { func SetS2iBinaryStatus(s2ibin *v1alpha1.S2iBinary, status string) (*v1alpha1.S2iBinary, error) { copy := s2ibin.DeepCopy() copy.Status.Phase = status - copy, err := k8s.KsClient().DevopsV1alpha1().S2iBinaries(s2ibin.Namespace).Update(copy) + copy, err := client.ClientSets().K8s().KubeSphere().DevopsV1alpha1().S2iBinaries(s2ibin.Namespace).Update(copy) if err != nil { klog.Error(err) return nil, err @@ -188,7 +193,7 @@ func SetS2iBinaryStatusWithRetry(s2ibin *v1alpha1.S2iBinary, status string) (*v1 return err } bin.Status.Phase = status - bin, err = k8s.KsClient().DevopsV1alpha1().S2iBinaries(s2ibin.Namespace).Update(bin) + bin, err = client.ClientSets().K8s().KubeSphere().DevopsV1alpha1().S2iBinaries(s2ibin.Namespace).Update(bin) if err != nil { klog.Error(err) return err diff --git a/pkg/models/iam/am.go b/pkg/models/iam/am.go index 17f6b03f7..cb4732db6 100644 --- a/pkg/models/iam/am.go +++ b/pkg/models/iam/am.go @@ -33,7 +33,7 @@ import ( "kubesphere.io/kubesphere/pkg/models/kubectl" "kubesphere.io/kubesphere/pkg/models/resources" "kubesphere.io/kubesphere/pkg/params" - "kubesphere.io/kubesphere/pkg/simple/client/k8s" + "kubesphere.io/kubesphere/pkg/simple/client" "kubesphere.io/kubesphere/pkg/utils/k8sutil" "kubesphere.io/kubesphere/pkg/utils/sliceutil" "sort" @@ -650,7 +650,7 @@ func CreateClusterRoleBinding(username string, clusterRoleName string) error { found, err := clusterRoleBindingLister.Get(username) if apierrors.IsNotFound(err) { - _, err = k8s.Client().RbacV1().ClusterRoleBindings().Create(clusterRoleBinding) + _, err = client.ClientSets().K8s().Kubernetes().RbacV1().ClusterRoleBindings().Create(clusterRoleBinding) if err != nil { glog.Errorln("create cluster role binding", err) return err @@ -665,14 +665,14 @@ func CreateClusterRoleBinding(username string, clusterRoleName string) error { deletePolicy := metav1.DeletePropagationForeground gracePeriodSeconds := int64(0) deleteOption := &metav1.DeleteOptions{PropagationPolicy: &deletePolicy, GracePeriodSeconds: &gracePeriodSeconds} - err = k8s.Client().RbacV1().ClusterRoleBindings().Delete(found.Name, deleteOption) + err = client.ClientSets().K8s().Kubernetes().RbacV1().ClusterRoleBindings().Delete(found.Name, deleteOption) if err != nil { glog.Errorln("delete cluster role binding", err) return err } maxRetries := 3 for i := 0; i < maxRetries; i++ { - _, err = k8s.Client().RbacV1().ClusterRoleBindings().Create(clusterRoleBinding) + _, err = client.ClientSets().K8s().Kubernetes().RbacV1().ClusterRoleBindings().Create(clusterRoleBinding) if err == nil { return nil } @@ -684,7 +684,7 @@ func CreateClusterRoleBinding(username string, clusterRoleName string) error { if !k8sutil.ContainsUser(found.Subjects, username) { found.Subjects = clusterRoleBinding.Subjects - _, err = k8s.Client().RbacV1().ClusterRoleBindings().Update(found) + _, err = client.ClientSets().K8s().Kubernetes().RbacV1().ClusterRoleBindings().Update(found) if err != nil { glog.Errorln("update cluster role binding", err) return err diff --git a/pkg/models/iam/im.go b/pkg/models/iam/im.go index 6feeef0a0..f3bc69e8c 100644 --- a/pkg/models/iam/im.go +++ b/pkg/models/iam/im.go @@ -23,6 +23,7 @@ import ( "fmt" "github.com/emicklei/go-restful" "io/ioutil" + "k8s.io/klog" "kubesphere.io/kubesphere/pkg/constants" "kubesphere.io/kubesphere/pkg/db" "kubesphere.io/kubesphere/pkg/informers" @@ -30,10 +31,7 @@ import ( "kubesphere.io/kubesphere/pkg/models/kubeconfig" "kubesphere.io/kubesphere/pkg/models/kubectl" "kubesphere.io/kubesphere/pkg/params" - "kubesphere.io/kubesphere/pkg/simple/client/admin_jenkins" - "kubesphere.io/kubesphere/pkg/simple/client/devops_mysql" - "kubesphere.io/kubesphere/pkg/simple/client/k8s" - "kubesphere.io/kubesphere/pkg/simple/client/redis" + clientset "kubesphere.io/kubesphere/pkg/simple/client" "kubesphere.io/kubesphere/pkg/utils/k8sutil" "kubesphere.io/kubesphere/pkg/utils/sliceutil" "net/http" @@ -45,9 +43,8 @@ import ( "github.com/dgrijalva/jwt-go" "github.com/go-ldap/ldap" - "github.com/golang/glog" rbacv1 "k8s.io/api/rbac/v1" - meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" ldapclient "kubesphere.io/kubesphere/pkg/simple/client/ldap" @@ -81,25 +78,22 @@ func Init(email, password string, expireTime time.Duration, authRateLimit string adminPassword = password tokenExpireTime = expireTime maxAuthFailed, authTimeInterval = parseAuthRateLimit(authRateLimit) - conn, err := ldapclient.Client() - + conn, err := clientset.ClientSets().Ldap() if err != nil { return err } - defer conn.Close() - err = checkAndCreateDefaultUser(conn) if err != nil { - glog.Errorln("create default users", err) + klog.Errorln("create default users", err) return err } err = checkAndCreateDefaultGroup(conn) if err != nil { - glog.Errorln("create default groups", err) + klog.Errorln("create default groups", err) return err } @@ -117,28 +111,28 @@ func parseAuthRateLimit(authRateLimit string) (int, time.Duration) { maxCount, _ = strconv.Atoi(groups[1]) timeInterval, _ = time.ParseDuration(groups[2]) } else { - glog.Warning("invalid auth rate limit", authRateLimit) + klog.Warning("invalid auth rate limit", authRateLimit) } return maxCount, timeInterval } -func checkAndCreateDefaultGroup(conn ldap.Client) error { +func checkAndCreateDefaultGroup(conn *ldapclient.LdapClient) error { groupSearchRequest := ldap.NewSearchRequest( - ldapclient.GroupSearchBase, + conn.GroupSearchBase(), ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false, "(&(objectClass=posixGroup))", nil, nil, ) - _, err := conn.Search(groupSearchRequest) + _, err := conn.Ldap().Search(groupSearchRequest) if ldap.IsErrorWithCode(err, ldap.LDAPResultNoSuchObject) { err = createGroupsBaseDN(conn) if err != nil { - return fmt.Errorf("GroupBaseDN %s create failed: %s\n", ldapclient.GroupSearchBase, err) + return fmt.Errorf("GroupBaseDN %s create failed: %s\n", conn.GroupSearchBase(), err) } } @@ -149,22 +143,22 @@ func checkAndCreateDefaultGroup(conn ldap.Client) error { return nil } -func checkAndCreateDefaultUser(conn ldap.Client) error { +func checkAndCreateDefaultUser(conn *ldapclient.LdapClient) error { userSearchRequest := ldap.NewSearchRequest( - ldapclient.UserSearchBase, + conn.GroupSearchBase(), ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false, "(&(objectClass=inetOrgPerson))", []string{"uid"}, nil, ) - result, err := conn.Search(userSearchRequest) + result, err := conn.Ldap().Search(userSearchRequest) if ldap.IsErrorWithCode(err, ldap.LDAPResultNoSuchObject) { err = createUserBaseDN(conn) if err != nil { - return fmt.Errorf("UserBaseDN %s create failed: %s\n", ldapclient.UserSearchBase, err) + return fmt.Errorf("UserBaseDN %s create failed: %s\n", conn.UserSearchBase(), err) } } @@ -182,7 +176,7 @@ func checkAndCreateDefaultUser(conn ldap.Client) error { if result == nil || !containsUser(result.Entries, user) { _, err = CreateUser(&user.User) if err != nil && !ldap.IsErrorWithCode(err, ldap.LDAPResultEntryAlreadyExists) { - glog.Errorln("user init failed", user.Username, err) + klog.Errorln("user init failed", user.Username, err) return fmt.Errorf("user %s init failed: %s\n", user.Username, err) } } @@ -201,36 +195,33 @@ func containsUser(entries []*ldap.Entry, user initUser) bool { return false } -func createUserBaseDN(conn ldap.Client) error { +func createUserBaseDN(conn *ldapclient.LdapClient) error { - conn, err := ldapclient.Client() - if err != nil { - return err - } - defer conn.Close() - - groupsCreateRequest := ldap.NewAddRequest(ldapclient.UserSearchBase, nil) + groupsCreateRequest := ldap.NewAddRequest(conn.UserSearchBase(), nil) groupsCreateRequest.Attribute("objectClass", []string{"organizationalUnit", "top"}) groupsCreateRequest.Attribute("ou", []string{"Users"}) - return conn.Add(groupsCreateRequest) + return conn.Ldap().Add(groupsCreateRequest) } -func createGroupsBaseDN(conn ldap.Client) error { - groupsCreateRequest := ldap.NewAddRequest(ldapclient.GroupSearchBase, nil) +func createGroupsBaseDN(conn *ldapclient.LdapClient) error { + groupsCreateRequest := ldap.NewAddRequest(conn.GroupSearchBase(), nil) groupsCreateRequest.Attribute("objectClass", []string{"organizationalUnit", "top"}) groupsCreateRequest.Attribute("ou", []string{"Groups"}) - return conn.Add(groupsCreateRequest) + return conn.Ldap().Add(groupsCreateRequest) } // User login func Login(username string, password string, ip string) (*models.Token, error) { - redisClient := redis.Client() + redisClient, err := clientset.ClientSets().Redis() + if err != nil { + return nil, err + } records, err := redisClient.Keys(fmt.Sprintf("kubesphere:authfailed:%s:*", username)).Result() if err != nil { - glog.Error(err) + klog.Error(err) return nil, err } @@ -238,24 +229,21 @@ func Login(username string, password string, ip string) (*models.Token, error) { return nil, restful.NewError(http.StatusTooManyRequests, "auth rate limit exceeded") } - conn, err := ldapclient.Client() - + conn, err := clientset.ClientSets().Ldap() if err != nil { - glog.Error(err) return nil, err } - - defer conn.Close() + defer conn.Ldap().Close() userSearchRequest := ldap.NewSearchRequest( - ldapclient.UserSearchBase, + conn.UserSearchBase(), ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false, fmt.Sprintf("(&(objectClass=inetOrgPerson)(|(uid=%s)(mail=%s)))", username, username), []string{"uid", "mail"}, nil, ) - result, err := conn.Search(userSearchRequest) + result, err := conn.Ldap().Search(userSearchRequest) if err != nil { return nil, err @@ -270,10 +258,10 @@ func Login(username string, password string, ip string) (*models.Token, error) { dn := result.Entries[0].DN // bind as the user to verify their password - err = conn.Bind(dn, password) + err = conn.Ldap().Bind(dn, password) if err != nil { - glog.Infoln("auth failed", username, err) + klog.Infoln("auth failed", username, err) if ldap.IsErrorWithCode(err, ldap.LDAPResultInvalidCredentials) { loginFailedRecord := fmt.Sprintf("kubesphere:authfailed:%s:%d", username, time.Now().UnixNano()) @@ -300,14 +288,20 @@ func Login(username string, password string, ip string) (*models.Token, error) { func loginLog(uid, ip string) { if ip != "" { - redisClient := redis.Client() + redisClient, err := clientset.ClientSets().Redis() + if err != nil { + return + } redisClient.RPush(fmt.Sprintf("kubesphere:users:%s:login-log", uid), fmt.Sprintf("%s,%s", time.Now().UTC().Format("2006-01-02T15:04:05Z"), ip)) redisClient.LTrim(fmt.Sprintf("kubesphere:users:%s:login-log", uid), -10, -1) } } func LoginLog(username string) ([]string, error) { - redisClient := redis.Client() + redisClient, err := clientset.ClientSets().Redis() + if err != nil { + return nil, err + } data, err := redisClient.LRange(fmt.Sprintf("kubesphere:users:%s:login-log", username), -10, -1).Result() @@ -320,13 +314,11 @@ func LoginLog(username string) ([]string, error) { func ListUsers(conditions *params.Conditions, orderBy string, reverse bool, limit, offset int) (*models.PageableResponse, error) { - conn, err := ldapclient.Client() - + conn, err := clientset.ClientSets().Ldap() if err != nil { return nil, err } - - defer conn.Close() + defer conn.Ldap().Close() pageControl := ldap.NewControlPaging(1000) @@ -356,17 +348,17 @@ func ListUsers(conditions *params.Conditions, orderBy string, reverse bool, limi for { userSearchRequest := ldap.NewSearchRequest( - ldapclient.UserSearchBase, + conn.UserSearchBase(), ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false, filter, []string{"uid", "mail", "description", "preferredLanguage", "createTimestamp"}, []ldap.Control{pageControl}, ) - response, err := conn.Search(userSearchRequest) + response, err := conn.Ldap().Search(userSearchRequest) if err != nil { - glog.Errorln("search user", err) + klog.Errorln("search user", err) return nil, err } @@ -461,26 +453,24 @@ func DescribeUser(username string) (*models.User, error) { // Get user info only included email description & lang func GetUserInfo(username string) (*models.User, error) { - conn, err := ldapclient.Client() - + conn, err := clientset.ClientSets().Ldap() if err != nil { return nil, err } - - defer conn.Close() + defer conn.Ldap().Close() userSearchRequest := ldap.NewSearchRequest( - ldapclient.UserSearchBase, + conn.UserSearchBase(), ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false, fmt.Sprintf("(&(objectClass=inetOrgPerson)(uid=%s))", username), []string{"mail", "description", "preferredLanguage", "createTimestamp"}, nil, ) - result, err := conn.Search(userSearchRequest) + result, err := conn.Ldap().Search(userSearchRequest) if err != nil { - glog.Errorln("search user", err) + klog.Errorln("search user", err) return nil, err } @@ -500,23 +490,22 @@ func GetUserInfo(username string) (*models.User, error) { } func GetUserGroups(username string) ([]string, error) { - conn, err := ldapclient.Client() - + conn, err := clientset.ClientSets().Ldap() if err != nil { return nil, err } - defer conn.Close() + defer conn.Ldap().Close() groupSearchRequest := ldap.NewSearchRequest( - ldapclient.GroupSearchBase, + conn.GroupSearchBase(), ldap.ScopeBaseObject, ldap.NeverDerefAliases, 0, 0, false, fmt.Sprintf("(&(objectClass=posixGroup)(memberUid=%s))", username), nil, nil, ) - result, err := conn.Search(groupSearchRequest) + result, err := conn.Ldap().Search(groupSearchRequest) if err != nil { return nil, err @@ -533,7 +522,12 @@ func GetUserGroups(username string) ([]string, error) { } func getLastLoginTime(username string) string { - lastLogin, err := redis.Client().LRange(fmt.Sprintf("kubesphere:users:%s:login-log", username), -1, -1).Result() + redis, err := clientset.ClientSets().Redis() + if err != nil { + return "" + } + + lastLogin, err := redis.LRange(fmt.Sprintf("kubesphere:users:%s:login-log", username), -1, -1).Result() if err != nil { return "" @@ -547,13 +541,22 @@ func getLastLoginTime(username string) string { } func setAvatar(username, avatar string) error { - _, err := redis.Client().HMSet("kubesphere:users:avatar", map[string]interface{}{"username": avatar}).Result() + redis, err := clientset.ClientSets().Redis() + if err != nil { + return err + } + + _, err = redis.HMSet("kubesphere:users:avatar", map[string]interface{}{"username": avatar}).Result() return err } func getAvatar(username string) string { + redis, err := clientset.ClientSets().Redis() + if err != nil { + return "" + } - avatar, err := redis.Client().HMGet("kubesphere:users:avatar", username).Result() + avatar, err := redis.HMGet("kubesphere:users:avatar", username).Result() if err != nil { return "" @@ -570,53 +573,54 @@ func getAvatar(username string) string { func DeleteUser(username string) error { - conn, err := ldapclient.Client() - + conn, err := clientset.ClientSets().Ldap() if err != nil { return err } + defer conn.Ldap().Close() - defer conn.Close() + deleteRequest := ldap.NewDelRequest(fmt.Sprintf("uid=%s,%s", username, conn.UserSearchBase()), nil) - deleteRequest := ldap.NewDelRequest(fmt.Sprintf("uid=%s,%s", username, ldapclient.UserSearchBase), nil) - - if err = conn.Del(deleteRequest); err != nil { - glog.Errorln("delete user", err) + if err = conn.Ldap().Del(deleteRequest); err != nil { + klog.Errorln("delete user", err) return err } if err = deleteRoleBindings(username); err != nil { - glog.Errorln("delete user role bindings failed", username, err) + klog.Errorln("delete user role bindings failed", username, err) } if err := kubeconfig.DelKubeConfig(username); err != nil { - glog.Errorln("delete user kubeconfig failed", username, err) + klog.Errorln("delete user kubeconfig failed", username, err) } if err := kubectl.DelKubectlDeploy(username); err != nil { - glog.Errorln("delete user terminal pod failed", username, err) + klog.Errorln("delete user terminal pod failed", username, err) } - devopsDb := devops_mysql.OpenDatabase() - - jenkinsClient := admin_jenkins.Client() - if jenkinsClient == nil { - err := fmt.Errorf("could not connect to jenkins") - glog.Error(err) - return restful.NewError(http.StatusServiceUnavailable, err.Error()) + devopsDb, err := clientset.ClientSets().MySQL() + if err != nil { + return err } + + dp, err := clientset.ClientSets().Devops() + if err != nil { + return err + } + jenkinsClient := dp.Jenkins() + _, err = devopsDb.DeleteFrom(devops.DevOpsProjectMembershipTableName). Where(db.And( db.Eq(devops.DevOpsProjectMembershipUsernameColumn, username), )).Exec() if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return err } err = jenkinsClient.DeleteUserInProject(username) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return err } @@ -646,17 +650,17 @@ func deleteRoleBindings(username string) error { length2 := len(roleBinding.Subjects) if length2 == 0 { - deletePolicy := meta_v1.DeletePropagationForeground - err = k8s.Client().RbacV1().RoleBindings(roleBinding.Namespace).Delete(roleBinding.Name, &meta_v1.DeleteOptions{PropagationPolicy: &deletePolicy}) + deletePolicy := metav1.DeletePropagationForeground + err = clientset.ClientSets().K8s().Kubernetes().RbacV1().RoleBindings(roleBinding.Namespace).Delete(roleBinding.Name, &metav1.DeleteOptions{PropagationPolicy: &deletePolicy}) if err != nil { - glog.Errorf("delete role binding %s %s %s failed: %v", username, roleBinding.Namespace, roleBinding.Name, err) + klog.Errorf("delete role binding %s %s %s failed: %v", username, roleBinding.Namespace, roleBinding.Name, err) } } else if length2 < length1 { - _, err = k8s.Client().RbacV1().RoleBindings(roleBinding.Namespace).Update(roleBinding) + _, err = clientset.ClientSets().K8s().Kubernetes().RbacV1().RoleBindings(roleBinding.Namespace).Update(roleBinding) if err != nil { - glog.Errorf("update role binding %s %s %s failed: %v", username, roleBinding.Namespace, roleBinding.Name, err) + klog.Errorf("update role binding %s %s %s failed: %v", username, roleBinding.Namespace, roleBinding.Name, err) } } } @@ -679,19 +683,19 @@ func deleteRoleBindings(username string) error { if length2 == 0 { // delete if it's not workspace role binding if isWorkspaceRoleBinding(clusterRoleBinding) { - _, err = k8s.Client().RbacV1().ClusterRoleBindings().Update(clusterRoleBinding) + _, err = clientset.ClientSets().K8s().Kubernetes().RbacV1().ClusterRoleBindings().Update(clusterRoleBinding) } else { - deletePolicy := meta_v1.DeletePropagationForeground - err = k8s.Client().RbacV1().ClusterRoleBindings().Delete(clusterRoleBinding.Name, &meta_v1.DeleteOptions{PropagationPolicy: &deletePolicy}) + deletePolicy := metav1.DeletePropagationForeground + err = clientset.ClientSets().K8s().Kubernetes().RbacV1().ClusterRoleBindings().Delete(clusterRoleBinding.Name, &metav1.DeleteOptions{PropagationPolicy: &deletePolicy}) } if err != nil { - glog.Errorf("update cluster role binding %s failed:%s", clusterRoleBinding.Name, err) + klog.Errorf("update cluster role binding %s failed:%s", clusterRoleBinding.Name, err) } } else if length2 < length1 { - _, err = k8s.Client().RbacV1().ClusterRoleBindings().Update(clusterRoleBinding) + _, err = clientset.ClientSets().K8s().Kubernetes().RbacV1().ClusterRoleBindings().Update(clusterRoleBinding) if err != nil { - glog.Errorf("update cluster role binding %s failed:%s", clusterRoleBinding.Name, err) + klog.Errorf("update cluster role binding %s failed:%s", clusterRoleBinding.Name, err) } } @@ -707,27 +711,26 @@ func isWorkspaceRoleBinding(clusterRoleBinding *rbacv1.ClusterRoleBinding) bool func UserCreateCheck(check string) (exist bool, err error) { // bind root DN - conn, err := ldapclient.Client() - + conn, err := clientset.ClientSets().Ldap() if err != nil { return false, err } - defer conn.Close() + defer conn.Ldap().Close() // search for the given username userSearchRequest := ldap.NewSearchRequest( - ldapclient.UserSearchBase, + conn.UserSearchBase(), ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false, fmt.Sprintf("(&(objectClass=inetOrgPerson)(|(uid=%s)(mail=%s)))", check, check), []string{"uid", "mail"}, nil, ) - result, err := conn.Search(userSearchRequest) + result, err := conn.Ldap().Search(userSearchRequest) if err != nil { - glog.Errorln("search user", err) + klog.Errorln("search user", err) return false, err } @@ -740,26 +743,24 @@ func CreateUser(user *models.User) (*models.User, error) { user.Password = strings.TrimSpace(user.Password) user.Description = strings.TrimSpace(user.Description) - conn, err := ldapclient.Client() - + conn, err := clientset.ClientSets().Ldap() if err != nil { return nil, err } - - defer conn.Close() + defer conn.Ldap().Close() userSearchRequest := ldap.NewSearchRequest( - ldapclient.UserSearchBase, + conn.UserSearchBase(), ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false, fmt.Sprintf("(&(objectClass=inetOrgPerson)(|(uid=%s)(mail=%s)))", user.Username, user.Email), []string{"uid", "mail"}, nil, ) - result, err := conn.Search(userSearchRequest) + result, err := conn.Ldap().Search(userSearchRequest) if err != nil { - glog.Errorln("search user", err) + klog.Errorln("search user", err) return nil, err } @@ -770,13 +771,13 @@ func CreateUser(user *models.User) (*models.User, error) { maxUid, err := getMaxUid(conn) if err != nil { - glog.Errorln("get max uid", err) + klog.Errorln("get max uid", err) return nil, err } maxUid += 1 - userCreateRequest := ldap.NewAddRequest(fmt.Sprintf("uid=%s,%s", user.Username, ldapclient.UserSearchBase), nil) + userCreateRequest := ldap.NewAddRequest(fmt.Sprintf("uid=%s,%s", user.Username, conn.UserSearchBase()), nil) userCreateRequest.Attribute("objectClass", []string{"inetOrgPerson", "posixAccount", "top"}) userCreateRequest.Attribute("cn", []string{user.Username}) // RFC4519: common name(s) for which the entity is known by userCreateRequest.Attribute("sn", []string{" "}) // RFC2256: last (family) name(s) for which the entity is known by @@ -794,14 +795,14 @@ func CreateUser(user *models.User) (*models.User, error) { } if err := kubeconfig.CreateKubeConfig(user.Username); err != nil { - glog.Errorln("create user kubeconfig failed", user.Username, err) + klog.Errorln("create user kubeconfig failed", user.Username, err) return nil, err } - err = conn.Add(userCreateRequest) + err = conn.Ldap().Add(userCreateRequest) if err != nil { - glog.Errorln("create user", err) + klog.Errorln("create user", err) return nil, err } @@ -813,7 +814,7 @@ func CreateUser(user *models.User) (*models.User, error) { err := CreateClusterRoleBinding(user.Username, user.ClusterRole) if err != nil { - glog.Errorln("create cluster role binding filed", err) + klog.Errorln("create cluster role binding filed", err) return nil, err } } @@ -821,14 +822,14 @@ func CreateUser(user *models.User) (*models.User, error) { return DescribeUser(user.Username) } -func getMaxUid(conn ldap.Client) (int, error) { - userSearchRequest := ldap.NewSearchRequest(ldapclient.UserSearchBase, +func getMaxUid(conn *ldapclient.LdapClient) (int, error) { + userSearchRequest := ldap.NewSearchRequest(conn.UserSearchBase(), ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false, "(&(objectClass=inetOrgPerson))", []string{"uidNumber"}, nil) - result, err := conn.Search(userSearchRequest) + result, err := conn.Ldap().Search(userSearchRequest) if err != nil { return 0, err @@ -850,15 +851,15 @@ func getMaxUid(conn ldap.Client) (int, error) { return maxUid, nil } -func getMaxGid(conn ldap.Client) (int, error) { +func getMaxGid(conn *ldapclient.LdapClient) (int, error) { - groupSearchRequest := ldap.NewSearchRequest(ldapclient.GroupSearchBase, + groupSearchRequest := ldap.NewSearchRequest(conn.GroupSearchBase(), ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false, "(&(objectClass=posixGroup))", []string{"gidNumber"}, nil) - result, err := conn.Search(groupSearchRequest) + result, err := conn.Ldap().Search(groupSearchRequest) if err != nil { return 0, err @@ -882,38 +883,36 @@ func getMaxGid(conn ldap.Client) (int, error) { func UpdateUser(user *models.User) (*models.User, error) { - conn, err := ldapclient.Client() - + conn, err := clientset.ClientSets().Ldap() if err != nil { - glog.Error(err) + klog.Error(err) return nil, err } + defer conn.Ldap().Close() - defer conn.Close() - - dn := fmt.Sprintf("uid=%s,%s", user.Username, ldapclient.UserSearchBase) + dn := fmt.Sprintf("uid=%s,%s", user.Username, conn.UserSearchBase()) userModifyRequest := ldap.NewModifyRequest(dn, nil) if user.Email != "" { userSearchRequest := ldap.NewSearchRequest( - ldapclient.UserSearchBase, + conn.UserSearchBase(), ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false, fmt.Sprintf("(&(objectClass=inetOrgPerson)(mail=%s))", user.Email), []string{"uid", "mail"}, nil, ) - result, err := conn.Search(userSearchRequest) + result, err := conn.Ldap().Search(userSearchRequest) if err != nil { - glog.Error(err) + klog.Error(err) return nil, err } if len(result.Entries) > 1 { err = ldap.NewError(ldap.ErrorDebugging, fmt.Errorf("email is duplicated: %s", user.Email)) - glog.Error(err) + klog.Error(err) return nil, err } if len(result.Entries) == 1 && result.Entries[0].GetAttributeValue("uid") != user.Username { err = ldap.NewError(ldap.LDAPResultEntryAlreadyExists, fmt.Errorf("email is duplicated: %s", user.Email)) - glog.Error(err) + klog.Error(err) return nil, err } userModifyRequest.Replace("mail", []string{user.Email}) @@ -935,27 +934,30 @@ func UpdateUser(user *models.User) (*models.User, error) { } if err != nil { - glog.Error(err) + klog.Error(err) return nil, err } - err = conn.Modify(userModifyRequest) + err = conn.Ldap().Modify(userModifyRequest) if err != nil { - glog.Error(err) + klog.Error(err) return nil, err } err = CreateClusterRoleBinding(user.Username, user.ClusterRole) if err != nil { - glog.Errorln("create cluster role binding filed", err) + klog.Errorln("create cluster role binding filed", err) return nil, err } // clear auth failed record if user.Password != "" { - redisClient := redis.Client() + redisClient, err := clientset.ClientSets().Redis() + if err != nil { + return nil, err + } records, err := redisClient.Keys(fmt.Sprintf("kubesphere:authfailed:%s:*", user.Username)).Result() @@ -969,19 +971,19 @@ func UpdateUser(user *models.User) (*models.User, error) { func DeleteGroup(path string) error { // bind root DN - conn, err := ldapclient.Client() + conn, err := clientset.ClientSets().Ldap() if err != nil { return err } - defer conn.Close() + defer conn.Ldap().Close() searchBase, cn := splitPath(path) groupDeleteRequest := ldap.NewDelRequest(fmt.Sprintf("cn=%s,%s", cn, searchBase), nil) - err = conn.Del(groupDeleteRequest) + err = conn.Ldap().Del(groupDeleteRequest) if err != nil { - glog.Errorln("delete user group", err) + klog.Errorln("delete user group", err) return err } @@ -990,18 +992,16 @@ func DeleteGroup(path string) error { func CreateGroup(group *models.Group) (*models.Group, error) { - conn, err := ldapclient.Client() - + conn, err := clientset.ClientSets().Ldap() if err != nil { return nil, err } - - defer conn.Close() + defer conn.Ldap().Close() maxGid, err := getMaxGid(conn) if err != nil { - glog.Errorln("get max gid", err) + klog.Errorln("get max gid", err) return nil, err } @@ -1026,10 +1026,10 @@ func CreateGroup(group *models.Group) (*models.Group, error) { groupCreateRequest.Attribute("memberUid", group.Members) } - err = conn.Add(groupCreateRequest) + err = conn.Ldap().Add(groupCreateRequest) if err != nil { - glog.Errorln("create group", err) + klog.Errorln("create group", err) return nil, err } @@ -1041,11 +1041,11 @@ func CreateGroup(group *models.Group) (*models.Group, error) { func UpdateGroup(group *models.Group) (*models.Group, error) { // bind root DN - conn, err := ldapclient.Client() + conn, err := clientset.ClientSets().Ldap() if err != nil { return nil, err } - defer conn.Close() + defer conn.Ldap().Close() old, err := DescribeGroup(group.Path) @@ -1073,10 +1073,10 @@ func UpdateGroup(group *models.Group) (*models.Group, error) { groupUpdateRequest.Replace("memberUid", group.Members) } - err = conn.Modify(groupUpdateRequest) + err = conn.Ldap().Modify(groupUpdateRequest) if err != nil { - glog.Errorln("update group", err) + klog.Errorln("update group", err) return nil, err } @@ -1086,17 +1086,15 @@ func UpdateGroup(group *models.Group) (*models.Group, error) { func ChildList(path string) ([]models.Group, error) { // bind root DN - conn, err := ldapclient.Client() - + conn, err := clientset.ClientSets().Ldap() if err != nil { return nil, err } - - defer conn.Close() + defer conn.Ldap().Close() var groupSearchRequest *ldap.SearchRequest if path == "" { - groupSearchRequest = ldap.NewSearchRequest(ldapclient.GroupSearchBase, + groupSearchRequest = ldap.NewSearchRequest(conn.GroupSearchBase(), ldap.ScopeSingleLevel, ldap.NeverDerefAliases, 0, 0, false, "(&(objectClass=posixGroup))", []string{"cn", "gidNumber", "memberUid", "description"}, @@ -1110,7 +1108,7 @@ func ChildList(path string) ([]models.Group, error) { nil) } - result, err := conn.Search(groupSearchRequest) + result, err := conn.Ldap().Search(groupSearchRequest) if err != nil { return nil, err @@ -1133,7 +1131,7 @@ func ChildList(path string) ([]models.Group, error) { []string{""}, nil) - result, err = conn.Search(childSearchRequest) + result, err = conn.Ldap().Search(childSearchRequest) if err != nil { return nil, err @@ -1158,13 +1156,11 @@ func DescribeGroup(path string) (*models.Group, error) { searchBase, cn := splitPath(path) - conn, err := ldapclient.Client() - + conn, err := clientset.ClientSets().Ldap() if err != nil { return nil, err } - - defer conn.Close() + defer conn.Ldap().Close() groupSearchRequest := ldap.NewSearchRequest(searchBase, ldap.ScopeSingleLevel, ldap.NeverDerefAliases, 0, 0, false, @@ -1172,10 +1168,10 @@ func DescribeGroup(path string) (*models.Group, error) { []string{"cn", "gidNumber", "memberUid", "description"}, nil) - result, err := conn.Search(groupSearchRequest) + result, err := conn.Ldap().Search(groupSearchRequest) if err != nil { - glog.Errorln("search group", err) + klog.Errorln("search group", err) return nil, err } diff --git a/pkg/models/iam/path.go b/pkg/models/iam/path.go index 272c5d58c..f7ff2f1ac 100644 --- a/pkg/models/iam/path.go +++ b/pkg/models/iam/path.go @@ -19,7 +19,7 @@ package iam import ( "fmt" - ldapclient "kubesphere.io/kubesphere/pkg/simple/client/ldap" + "kubesphere.io/kubesphere/pkg/simple/client" "regexp" "strings" ) @@ -44,6 +44,10 @@ func convertDNToPath(dn string) string { } func splitPath(path string) (searchBase string, cn string) { + ldapClient, err := client.ClientSets().Ldap() + if err != nil { + return "", "" + } paths := strings.Split(path, ":") length := len(paths) @@ -60,12 +64,12 @@ func splitPath(path string) (searchBase string, cn string) { basePath[i], basePath[j] = basePath[j], basePath[i] } - searchBase = fmt.Sprintf("%s,%s", strings.Join(basePath, ","), ldapclient.GroupSearchBase) + searchBase = fmt.Sprintf("%s,%s", strings.Join(basePath, ","), ldapClient.GroupSearchBase()) } else if length == 2 { - searchBase = fmt.Sprintf("cn=%s,%s", paths[0], ldapclient.GroupSearchBase) + searchBase = fmt.Sprintf("cn=%s,%s", paths[0], ldapClient.GroupSearchBase()) cn = paths[1] } else { - searchBase = ldapclient.GroupSearchBase + searchBase = ldapClient.GroupSearchBase() if paths[0] == "" { cn = "*" } else { diff --git a/pkg/models/kubeconfig/kubeconfig.go b/pkg/models/kubeconfig/kubeconfig.go index 724423f49..5babbe346 100644 --- a/pkg/models/kubeconfig/kubeconfig.go +++ b/pkg/models/kubeconfig/kubeconfig.go @@ -29,7 +29,7 @@ import ( "fmt" "gopkg.in/yaml.v2" "io/ioutil" - "kubesphere.io/kubesphere/pkg/simple/client/k8s" + "kubesphere.io/kubesphere/pkg/simple/client" "math/big" rd "math/rand" "time" @@ -217,7 +217,7 @@ func createKubeConfig(username string) (string, error) { return "", err } base64ServerCa := base64.StdEncoding.EncodeToString(serverCa) - tmpClusterInfo := clusterInfo{CertificateAuthorityData: base64ServerCa, Server: k8s.KubeConfig.Host} + tmpClusterInfo := clusterInfo{CertificateAuthorityData: base64ServerCa, Server: client.ClientSets().K8s().Master()} tmpCluster := cluster{Cluster: tmpClusterInfo, Name: clusterName} tmpKubeConfig.Clusters = append(tmpKubeConfig.Clusters, tmpCluster) @@ -244,7 +244,7 @@ func createKubeConfig(username string) (string, error) { } func CreateKubeConfig(username string) error { - k8sClient := k8s.Client() + k8sClient := client.ClientSets().K8s().Kubernetes() configName := fmt.Sprintf("kubeconfig-%s", username) _, err := k8sClient.CoreV1().ConfigMaps(constants.KubeSphereControlNamespace).Get(configName, metaV1.GetOptions{}) @@ -255,7 +255,7 @@ func CreateKubeConfig(username string) error { return err } - data := map[string]string{"config": string(config)} + data := map[string]string{"config": config} configMap := v1.ConfigMap{TypeMeta: metaV1.TypeMeta{Kind: "Configmap", APIVersion: "v1"}, ObjectMeta: metaV1.ObjectMeta{Name: configName}, Data: data} _, err = k8sClient.CoreV1().ConfigMaps(constants.KubeSphereControlNamespace).Create(&configMap) if err != nil && !errors.IsAlreadyExists(err) { @@ -269,7 +269,7 @@ func CreateKubeConfig(username string) error { } func GetKubeConfig(username string) (string, error) { - k8sClient := k8s.Client() + k8sClient := client.ClientSets().K8s().Kubernetes() configName := fmt.Sprintf("kubeconfig-%s", username) configMap, err := k8sClient.CoreV1().ConfigMaps(constants.KubeSphereControlNamespace).Get(configName, metaV1.GetOptions{}) if err != nil { @@ -284,10 +284,7 @@ func GetKubeConfig(username string) (string, error) { glog.Error(err) return "", err } - masterURL := k8s.KubeConfig.Host - if host := k8s.MasterURL; host != "" { - masterURL = host - } + masterURL := client.ClientSets().K8s().Master() for i, cluster := range kubeConfig.Clusters { cluster.Cluster.Server = masterURL kubeConfig.Clusters[i] = cluster @@ -301,7 +298,7 @@ func GetKubeConfig(username string) (string, error) { } func DelKubeConfig(username string) error { - k8sClient := k8s.Client() + k8sClient := client.ClientSets().K8s().Kubernetes() configName := fmt.Sprintf("kubeconfig-%s", username) _, err := k8sClient.CoreV1().ConfigMaps(constants.KubeSphereControlNamespace).Get(configName, metaV1.GetOptions{}) if errors.IsNotFound(err) { diff --git a/pkg/models/kubectl/kubectl.go b/pkg/models/kubectl/kubectl.go index b3cc00a6f..f6d3bfa1b 100644 --- a/pkg/models/kubectl/kubectl.go +++ b/pkg/models/kubectl/kubectl.go @@ -21,7 +21,7 @@ package kubectl import ( "fmt" "kubesphere.io/kubesphere/pkg/models" - "kubesphere.io/kubesphere/pkg/simple/client/k8s" + "kubesphere.io/kubesphere/pkg/simple/client" "math/rand" "os" @@ -49,7 +49,7 @@ func init() { } func GetKubectlPod(username string) (models.PodInfo, error) { - k8sClient := k8s.Client() + k8sClient := client.ClientSets().K8s().Kubernetes() deployName := fmt.Sprintf("kubectl-%s", username) deploy, err := k8sClient.AppsV1().Deployments(namespace).Get(deployName, metav1.GetOptions{}) if err != nil { @@ -97,7 +97,7 @@ func selectCorrectPod(namespace string, pods []v1.Pod) (kubectlPod v1.Pod, err e } func CreateKubectlDeploy(username string) error { - k8sClient := k8s.Client() + k8sClient := client.ClientSets().K8s().Kubernetes() deployName := fmt.Sprintf("kubectl-%s", username) configName := fmt.Sprintf("kubeconfig-%s", username) _, err := k8sClient.AppsV1().Deployments(namespace).Get(deployName, metav1.GetOptions{}) @@ -140,7 +140,7 @@ func CreateKubectlDeploy(username string) error { } func DelKubectlDeploy(username string) error { - k8sClient := k8s.Client() + k8sClient := client.ClientSets().K8s().Kubernetes() deployName := fmt.Sprintf("kubectl-%s", username) _, err := k8sClient.AppsV1().Deployments(namespace).Get(deployName, metav1.GetOptions{}) if errors.IsNotFound(err) { diff --git a/pkg/models/metrics/metrics.go b/pkg/models/metrics/metrics.go index 230ce1af7..1e49a4574 100644 --- a/pkg/models/metrics/metrics.go +++ b/pkg/models/metrics/metrics.go @@ -20,6 +20,7 @@ package metrics import ( "github.com/golang/glog" + "k8s.io/klog" "kubesphere.io/kubesphere/pkg/informers" "kubesphere.io/kubesphere/pkg/simple/client/kubesphere" "net/url" @@ -36,7 +37,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "kubesphere.io/kubesphere/pkg/models/workspaces" - client "kubesphere.io/kubesphere/pkg/simple/client/prometheus" + cs "kubesphere.io/kubesphere/pkg/simple/client" ) var jsonIter = jsoniter.ConfigCompatibleWithStandardLibrary @@ -136,10 +137,15 @@ func getAllWorkspaceNames(formatedMetric *FormatedMetric) map[string]int { func getAllWorkspaces() map[string]int { + client, err := cs.ClientSets().Prometheus() + if err != nil { + return nil + } + paramValues := make(url.Values) paramValues.Set("query", WorkspaceNamespaceLabelRule) params := paramValues.Encode() - res := client.SendMonitoringRequest(client.PrometheusEndpoint, client.DefaultQueryType, params) + res := client.SendSecondaryMonitoringRequest(DefaultQueryType, params) metric := ReformatJson(res, "", map[string]string{"workspace": "workspace"}) @@ -234,7 +240,13 @@ func unifyMetricHistoryTimeRange(fmtMetrics *FormatedMetric) { } } -func AssembleSpecificWorkloadMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string, bool) { +func AssembleSpecificWorkloadMetricRequestInfo(monitoringRequest *MonitoringRequestParams, metricName string) (string, string, bool) { + + client, err := cs.ClientSets().Prometheus() + if err != nil { + klog.Error(err) + return "", "", false + } nsName := monitoringRequest.NsName wlName := monitoringRequest.WorkloadName @@ -244,7 +256,7 @@ func AssembleSpecificWorkloadMetricRequestInfo(monitoringRequest *client.Monitor paramValues := monitoringRequest.Params params := makeRequestParamString(rule, paramValues) - res := client.SendMonitoringRequest(client.PrometheusEndpoint, client.DefaultQueryType, params) + res := client.SendMonitoringRequest(DefaultQueryType, params) podNamesFilter := getPodNameRegexInWorkload(res, podsFilter) @@ -255,7 +267,7 @@ func AssembleSpecificWorkloadMetricRequestInfo(monitoringRequest *client.Monitor return queryType, params, rule == "" } -func AssembleAllWorkloadMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) { +func AssembleAllWorkloadMetricRequestInfo(monitoringRequest *MonitoringRequestParams, metricName string) (string, string) { queryType := monitoringRequest.QueryType paramValues := monitoringRequest.Params @@ -266,7 +278,7 @@ func AssembleAllWorkloadMetricRequestInfo(monitoringRequest *client.MonitoringRe return queryType, params } -func AssemblePodMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string, bool) { +func AssemblePodMetricRequestInfo(monitoringRequest *MonitoringRequestParams, metricName string) (string, string, bool) { queryType := monitoringRequest.QueryType paramValues := monitoringRequest.Params @@ -276,7 +288,7 @@ func AssemblePodMetricRequestInfo(monitoringRequest *client.MonitoringRequestPar return queryType, params, rule == "" } -func AssemblePVCMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string, bool) { +func AssemblePVCMetricRequestInfo(monitoringRequest *MonitoringRequestParams, metricName string) (string, string, bool) { queryType := monitoringRequest.QueryType paramValues := monitoringRequest.Params @@ -318,14 +330,20 @@ func AddNodeAddressMetric(nodeMetric *FormatedMetric, nodeAddress *map[string][] } } -func MonitorContainer(monitoringRequest *client.MonitoringRequestParams, metricName string) *FormatedMetric { +func MonitorContainer(monitoringRequest *MonitoringRequestParams, metricName string) *FormatedMetric { + client, err := cs.ClientSets().Prometheus() + if err != nil { + klog.Error(err) + return nil + } + queryType, params := AssembleContainerMetricRequestInfo(monitoringRequest, metricName) - metricsStr := client.SendMonitoringRequest(client.PrometheusEndpoint, queryType, params) + metricsStr := client.SendMonitoringRequest(queryType, params) res := ReformatJson(metricsStr, metricName, map[string]string{MetricLevelContainerName: ""}) return res } -func AssembleContainerMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) { +func AssembleContainerMetricRequestInfo(monitoringRequest *MonitoringRequestParams, metricName string) (string, string) { queryType := monitoringRequest.QueryType paramValues := monitoringRequest.Params @@ -335,7 +353,7 @@ func AssembleContainerMetricRequestInfo(monitoringRequest *client.MonitoringRequ return queryType, params } -func AssembleNamespaceMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) { +func AssembleNamespaceMetricRequestInfo(monitoringRequest *MonitoringRequestParams, metricName string) (string, string) { queryType := monitoringRequest.QueryType paramValues := monitoringRequest.Params @@ -345,7 +363,7 @@ func AssembleNamespaceMetricRequestInfo(monitoringRequest *client.MonitoringRequ return queryType, params } -func AssembleNamespaceMetricRequestInfoByNamesapce(monitoringRequest *client.MonitoringRequestParams, namespace string, metricName string) (string, string) { +func AssembleNamespaceMetricRequestInfoByNamesapce(monitoringRequest *MonitoringRequestParams, namespace string, metricName string) (string, string) { queryType := monitoringRequest.QueryType paramValues := monitoringRequest.Params @@ -356,7 +374,7 @@ func AssembleNamespaceMetricRequestInfoByNamesapce(monitoringRequest *client.Mon return queryType, params } -func AssembleSpecificWorkspaceMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, namespaceList []string, workspace string, metricName string) (string, string) { +func AssembleSpecificWorkspaceMetricRequestInfo(monitoringRequest *MonitoringRequestParams, namespaceList []string, workspace string, metricName string) (string, string) { nsFilter := "^(" + strings.Join(namespaceList, "|") + ")$" @@ -368,7 +386,7 @@ func AssembleSpecificWorkspaceMetricRequestInfo(monitoringRequest *client.Monito return queryType, params } -func AssembleAllWorkspaceMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, namespaceList []string, metricName string) (string, string) { +func AssembleAllWorkspaceMetricRequestInfo(monitoringRequest *MonitoringRequestParams, namespaceList []string, metricName string) (string, string) { var nsFilter = "^()$" if namespaceList != nil { @@ -418,7 +436,7 @@ func filterNamespace(nsFilter string, namespaceList []string) []string { return newNSlist } -func MonitorAllWorkspaces(monitoringRequest *client.MonitoringRequestParams) *FormatedLevelMetric { +func MonitorAllWorkspaces(monitoringRequest *MonitoringRequestParams) *FormatedLevelMetric { metricsFilter := monitoringRequest.MetricsFilter if strings.Trim(metricsFilter, " ") == "" { metricsFilter = ".*" @@ -479,7 +497,13 @@ func MonitorAllWorkspaces(monitoringRequest *client.MonitoringRequestParams) *Fo } } -func collectWorkspaceMetric(monitoringRequest *client.MonitoringRequestParams, ws string, filterMetricsName []string, wgAll *sync.WaitGroup, wsAllch chan *[]FormatedMetric) { +func collectWorkspaceMetric(monitoringRequest *MonitoringRequestParams, ws string, filterMetricsName []string, wgAll *sync.WaitGroup, wsAllch chan *[]FormatedMetric) { + client, err := cs.ClientSets().Prometheus() + if err != nil { + klog.Error(err) + return + } + defer wgAll.Done() var wg sync.WaitGroup var ch = make(chan *FormatedMetric, ChannelMaxCapacity) @@ -494,7 +518,7 @@ func collectWorkspaceMetric(monitoringRequest *client.MonitoringRequestParams, w go func(metricName string) { queryType, params := AssembleSpecificWorkspaceMetricRequestInfo(monitoringRequest, namespaceArray, ws, metricName) - metricsStr := client.SendMonitoringRequest(client.PrometheusEndpoint, queryType, params) + metricsStr := client.SendMonitoringRequest(queryType, params) ch <- ReformatJson(metricsStr, metricName, map[string]string{ResultItemMetricResourceName: ws}) wg.Done() }(metricName) @@ -521,7 +545,12 @@ func collectWorkspaceMetric(monitoringRequest *client.MonitoringRequestParams, w wsAllch <- &metricsArray } -func GetClusterLevelMetrics(monitoringRequest *client.MonitoringRequestParams) *FormatedLevelMetric { +func GetClusterLevelMetrics(monitoringRequest *MonitoringRequestParams) *FormatedLevelMetric { + client, err := cs.ClientSets().Prometheus() + if err != nil { + return nil + } + metricsFilter := monitoringRequest.MetricsFilter if metricsFilter == "" { metricsFilter = ".*" @@ -536,7 +565,7 @@ func GetClusterLevelMetrics(monitoringRequest *client.MonitoringRequestParams) * wg.Add(1) go func(metricName string) { queryType, params := AssembleClusterMetricRequestInfo(monitoringRequest, metricName) - metricsStr := client.SendMonitoringRequest(client.PrometheusEndpoint, queryType, params) + metricsStr := client.SendMonitoringRequest(queryType, params) ch <- ReformatJson(metricsStr, metricName, map[string]string{MetricLevelCluster: "local"}) wg.Done() }(metricName) @@ -560,7 +589,12 @@ func GetClusterLevelMetrics(monitoringRequest *client.MonitoringRequestParams) * } } -func GetNodeLevelMetrics(monitoringRequest *client.MonitoringRequestParams) *FormatedLevelMetric { +func GetNodeLevelMetrics(monitoringRequest *MonitoringRequestParams) *FormatedLevelMetric { + client, err := cs.ClientSets().Prometheus() + if err != nil { + return nil + } + metricsFilter := monitoringRequest.MetricsFilter if metricsFilter == "" { metricsFilter = ".*" @@ -575,7 +609,7 @@ func GetNodeLevelMetrics(monitoringRequest *client.MonitoringRequestParams) *For wg.Add(1) go func(metricName string) { queryType, params := AssembleNodeMetricRequestInfo(monitoringRequest, metricName) - metricsStr := client.SendMonitoringRequest(client.PrometheusEndpoint, queryType, params) + metricsStr := client.SendMonitoringRequest(queryType, params) ch <- ReformatJson(metricsStr, metricName, map[string]string{MetricLevelNode: ""}) wg.Done() }(metricName) @@ -599,7 +633,13 @@ func GetNodeLevelMetrics(monitoringRequest *client.MonitoringRequestParams) *For } } -func GetWorkspaceLevelMetrics(monitoringRequest *client.MonitoringRequestParams) *FormatedLevelMetric { +func GetWorkspaceLevelMetrics(monitoringRequest *MonitoringRequestParams) *FormatedLevelMetric { + client, err := cs.ClientSets().Prometheus() + if err != nil { + klog.Error(err) + return nil + } + metricsFilter := monitoringRequest.MetricsFilter if metricsFilter == "" { metricsFilter = ".*" @@ -638,7 +678,7 @@ func GetWorkspaceLevelMetrics(monitoringRequest *client.MonitoringRequestParams) go func(metricName string, namespace string) { queryType, params := AssembleNamespaceMetricRequestInfoByNamesapce(monitoringRequest, namespace, metricName) - metricsStr := client.SendMonitoringRequest(client.PrometheusEndpoint, queryType, params) + metricsStr := client.SendMonitoringRequest(queryType, params) chForOneMetric <- ReformatJson(metricsStr, metricName, map[string]string{ResultItemMetricResourceName: namespace}) wgForOneMetric.Done() }(metricName, ns) @@ -682,7 +722,7 @@ func GetWorkspaceLevelMetrics(monitoringRequest *client.MonitoringRequestParams) wg.Add(1) go func(metricName string, workspace string) { queryType, params := AssembleSpecificWorkspaceMetricRequestInfo(monitoringRequest, namespaceArray, workspace, metricName) - metricsStr := client.SendMonitoringRequest(client.PrometheusEndpoint, queryType, params) + metricsStr := client.SendMonitoringRequest(queryType, params) ch <- ReformatJson(metricsStr, metricName, map[string]string{ResultItemMetricResourceName: workspace}) wg.Done() }(metricName, workspace) @@ -699,7 +739,7 @@ func GetWorkspaceLevelMetrics(monitoringRequest *client.MonitoringRequestParams) go func(metricName string) { queryType, params := AssembleAllWorkspaceMetricRequestInfo(monitoringRequest, nil, metricName) - metricsStr := client.SendMonitoringRequest(client.PrometheusEndpoint, queryType, params) + metricsStr := client.SendMonitoringRequest(queryType, params) ch <- ReformatJson(metricsStr, metricName, map[string]string{MetricLevelWorkspace: "workspaces"}) wg.Done() @@ -725,7 +765,13 @@ func GetWorkspaceLevelMetrics(monitoringRequest *client.MonitoringRequestParams) } } -func GetNamespaceLevelMetrics(monitoringRequest *client.MonitoringRequestParams) *FormatedLevelMetric { +func GetNamespaceLevelMetrics(monitoringRequest *MonitoringRequestParams) *FormatedLevelMetric { + client, err := cs.ClientSets().Prometheus() + if err != nil { + klog.Error(err) + return nil + } + metricsFilter := monitoringRequest.MetricsFilter if metricsFilter == "" { metricsFilter = ".*" @@ -741,7 +787,7 @@ func GetNamespaceLevelMetrics(monitoringRequest *client.MonitoringRequestParams) go func(metricName string) { queryType, params := AssembleNamespaceMetricRequestInfo(monitoringRequest, metricName) - metricsStr := client.SendMonitoringRequest(client.PrometheusEndpoint, queryType, params) + metricsStr := client.SendMonitoringRequest(queryType, params) rawResult := ReformatJson(metricsStr, metricName, map[string]string{MetricLevelNamespace: ""}) ch <- rawResult @@ -768,7 +814,13 @@ func GetNamespaceLevelMetrics(monitoringRequest *client.MonitoringRequestParams) } } -func GetWorkloadLevelMetrics(monitoringRequest *client.MonitoringRequestParams) *FormatedLevelMetric { +func GetWorkloadLevelMetrics(monitoringRequest *MonitoringRequestParams) *FormatedLevelMetric { + client, err := cs.ClientSets().Prometheus() + if err != nil { + klog.Error(err) + return nil + } + metricsFilter := monitoringRequest.MetricsFilter if metricsFilter == "" { metricsFilter = ".*" @@ -784,7 +836,7 @@ func GetWorkloadLevelMetrics(monitoringRequest *client.MonitoringRequestParams) wg.Add(1) go func(metricName string) { queryType, params := AssembleAllWorkloadMetricRequestInfo(monitoringRequest, metricName) - metricsStr := client.SendMonitoringRequest(client.PrometheusEndpoint, queryType, params) + metricsStr := client.SendMonitoringRequest(queryType, params) reformattedResult := ReformatJson(metricsStr, metricName, map[string]string{MetricLevelWorkload: ""}) // no need to append a null result ch <- reformattedResult @@ -801,7 +853,7 @@ func GetWorkloadLevelMetrics(monitoringRequest *client.MonitoringRequestParams) metricName = strings.TrimLeft(metricName, "workload_") queryType, params, nullRule := AssembleSpecificWorkloadMetricRequestInfo(monitoringRequest, metricName) if !nullRule { - metricsStr := client.SendMonitoringRequest(client.PrometheusEndpoint, queryType, params) + metricsStr := client.SendMonitoringRequest(queryType, params) fmtMetrics := ReformatJson(metricsStr, metricName, map[string]string{MetricLevelPodName: ""}) unifyMetricHistoryTimeRange(fmtMetrics) ch <- fmtMetrics @@ -829,7 +881,13 @@ func GetWorkloadLevelMetrics(monitoringRequest *client.MonitoringRequestParams) } } -func GetPodLevelMetrics(monitoringRequest *client.MonitoringRequestParams) *FormatedLevelMetric { +func GetPodLevelMetrics(monitoringRequest *MonitoringRequestParams) *FormatedLevelMetric { + client, err := cs.ClientSets().Prometheus() + if err != nil { + klog.Error(err) + return nil + } + metricsFilter := monitoringRequest.MetricsFilter if metricsFilter == "" { metricsFilter = ".*" @@ -845,7 +903,7 @@ func GetPodLevelMetrics(monitoringRequest *client.MonitoringRequestParams) *Form go func(metricName string) { queryType, params, nullRule := AssemblePodMetricRequestInfo(monitoringRequest, metricName) if !nullRule { - metricsStr := client.SendMonitoringRequest(client.PrometheusEndpoint, queryType, params) + metricsStr := client.SendMonitoringRequest(queryType, params) ch <- ReformatJson(metricsStr, metricName, map[string]string{MetricLevelPodName: ""}) } else { ch <- nil @@ -872,7 +930,13 @@ func GetPodLevelMetrics(monitoringRequest *client.MonitoringRequestParams) *Form } } -func GetContainerLevelMetrics(monitoringRequest *client.MonitoringRequestParams) *FormatedLevelMetric { +func GetContainerLevelMetrics(monitoringRequest *MonitoringRequestParams) *FormatedLevelMetric { + client, err := cs.ClientSets().Prometheus() + if err != nil { + klog.Error(err) + return nil + } + metricsFilter := monitoringRequest.MetricsFilter if metricsFilter == "" { metricsFilter = ".*" @@ -887,7 +951,7 @@ func GetContainerLevelMetrics(monitoringRequest *client.MonitoringRequestParams) wg.Add(1) go func(metricName string) { queryType, params := AssembleContainerMetricRequestInfo(monitoringRequest, metricName) - metricsStr := client.SendMonitoringRequest(client.PrometheusEndpoint, queryType, params) + metricsStr := client.SendMonitoringRequest(queryType, params) ch <- ReformatJson(metricsStr, metricName, map[string]string{MetricLevelContainerName: ""}) wg.Done() }(metricName) @@ -911,7 +975,13 @@ func GetContainerLevelMetrics(monitoringRequest *client.MonitoringRequestParams) } } -func GetPVCLevelMetrics(monitoringRequest *client.MonitoringRequestParams) *FormatedLevelMetric { +func GetPVCLevelMetrics(monitoringRequest *MonitoringRequestParams) *FormatedLevelMetric { + client, err := cs.ClientSets().Prometheus() + if err != nil { + klog.Error(err) + return nil + } + metricsFilter := monitoringRequest.MetricsFilter if metricsFilter == "" { metricsFilter = ".*" @@ -927,7 +997,7 @@ func GetPVCLevelMetrics(monitoringRequest *client.MonitoringRequestParams) *Form go func(metricName string) { queryType, params, nullRule := AssemblePVCMetricRequestInfo(monitoringRequest, metricName) if !nullRule { - metricsStr := client.SendMonitoringRequest(client.PrometheusEndpoint, queryType, params) + metricsStr := client.SendMonitoringRequest(queryType, params) ch <- ReformatJson(metricsStr, metricName, map[string]string{MetricLevelPVC: ""}) } else { ch <- nil @@ -954,7 +1024,13 @@ func GetPVCLevelMetrics(monitoringRequest *client.MonitoringRequestParams) *Form } } -func GetComponentLevelMetrics(monitoringRequest *client.MonitoringRequestParams) *FormatedLevelMetric { +func GetComponentLevelMetrics(monitoringRequest *MonitoringRequestParams) *FormatedLevelMetric { + client, err := cs.ClientSets().Prometheus() + if err != nil { + klog.Error(err) + return nil + } + metricsFilter := monitoringRequest.MetricsFilter if metricsFilter == "" { metricsFilter = ".*" @@ -969,7 +1045,7 @@ func GetComponentLevelMetrics(monitoringRequest *client.MonitoringRequestParams) wg.Add(1) go func(metricName string) { queryType, params := AssembleComponentRequestInfo(monitoringRequest, metricName) - metricsStr := client.SendMonitoringRequest(client.SecondaryPrometheusEndpoint, queryType, params) + metricsStr := client.SendMonitoringRequest(queryType, params) formattedJson := ReformatJson(metricsStr, metricName, map[string]string{ResultItemMetricResourceName: monitoringRequest.ComponentName}) if metricName == EtcdServerList { @@ -1175,7 +1251,7 @@ func getSpecificMetricItem(timestamp int64, metricName string, resource string, return &nsMetrics } -func AssembleClusterMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) { +func AssembleClusterMetricRequestInfo(monitoringRequest *MonitoringRequestParams, metricName string) (string, string) { queryType := monitoringRequest.QueryType paramValues := monitoringRequest.Params rule := MakeClusterRule(metricName) @@ -1184,7 +1260,7 @@ func AssembleClusterMetricRequestInfo(monitoringRequest *client.MonitoringReques return queryType, params } -func AssembleNodeMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) { +func AssembleNodeMetricRequestInfo(monitoringRequest *MonitoringRequestParams, metricName string) (string, string) { queryType := monitoringRequest.QueryType paramValues := monitoringRequest.Params rule := MakeNodeRule(monitoringRequest.NodeId, monitoringRequest.ResourcesFilter, metricName) @@ -1193,7 +1269,7 @@ func AssembleNodeMetricRequestInfo(monitoringRequest *client.MonitoringRequestPa return queryType, params } -func AssembleComponentRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) { +func AssembleComponentRequestInfo(monitoringRequest *MonitoringRequestParams, metricName string) (string, string) { queryType := monitoringRequest.QueryType paramValues := monitoringRequest.Params rule := MakeComponentRule(metricName) diff --git a/pkg/models/metrics/namespaces.go b/pkg/models/metrics/namespaces.go index ca48b4b14..1c2ecac57 100644 --- a/pkg/models/metrics/namespaces.go +++ b/pkg/models/metrics/namespaces.go @@ -22,8 +22,6 @@ import ( "strings" "k8s.io/api/core/v1" - - prom "kubesphere.io/kubesphere/pkg/simple/client/prometheus" ) func GetNamespacesWithMetrics(namespaces []*v1.Namespace) []*v1.Namespace { @@ -34,10 +32,10 @@ func GetNamespacesWithMetrics(namespaces []*v1.Namespace) []*v1.Namespace { nsFilter := "^(" + strings.Join(nsNameList, "|") + ")$" var timeRelateParams = make(url.Values) - params := prom.MonitoringRequestParams{ + params := MonitoringRequestParams{ ResourcesFilter: nsFilter, Params: timeRelateParams, - QueryType: prom.DefaultQueryType, + QueryType: DefaultQueryType, MetricsFilter: "namespace_cpu_usage|namespace_memory_usage_wo_cache|namespace_pod_count", } diff --git a/pkg/models/metrics/types.go b/pkg/models/metrics/types.go new file mode 100644 index 000000000..480a4ebd0 --- /dev/null +++ b/pkg/models/metrics/types.go @@ -0,0 +1,33 @@ +package metrics + +import "net/url" + +const ( + DefaultQueryStep = "10m" + DefaultQueryTimeout = "10s" + RangeQueryType = "query_range?" + DefaultQueryType = "query?" +) + +type MonitoringRequestParams struct { + Params url.Values + QueryType string + SortMetricName string + SortType string + PageNum string + LimitNum string + Tp string + MetricsFilter string + ResourcesFilter string + MetricsName string + WorkloadName string + NodeId string + WsName string + NsName string + PodName string + PVCName string + StorageClassName string + ContainerName string + WorkloadKind string + ComponentName string +} diff --git a/pkg/models/nodes/nodes.go b/pkg/models/nodes/nodes.go index fdf0d6bc4..cd0b85ea7 100644 --- a/pkg/models/nodes/nodes.go +++ b/pkg/models/nodes/nodes.go @@ -19,7 +19,7 @@ package nodes import ( "fmt" - "kubesphere.io/kubesphere/pkg/simple/client/k8s" + "kubesphere.io/kubesphere/pkg/simple/client" "math" "strings" "time" @@ -36,7 +36,7 @@ import ( func DrainNode(nodename string) (err error) { - k8sclient := k8s.Client() + k8sclient := client.ClientSets().K8s().Kubernetes() node, err := k8sclient.CoreV1().Nodes().Get(nodename, metav1.GetOptions{}) if err != nil { return err @@ -68,7 +68,7 @@ func DrainNode(nodename string) (err error) { func drainEviction(nodename string, donech chan bool, errch chan error) { - k8sclient := k8s.Client() + k8sclient := client.ClientSets().K8s().Kubernetes() var options metav1.ListOptions pods := make([]v1.Pod, 0) options.FieldSelector = "spec.nodeName=" + nodename @@ -107,7 +107,6 @@ func drainEviction(nodename string, donech chan bool, errch chan error) { //create eviction getPodFn := func(namespace, name string) (*v1.Pod, error) { - k8sclient := k8s.Client() return k8sclient.CoreV1().Pods(namespace).Get(name, metav1.GetOptions{}) } evicerr := evictPods(pods, 0, getPodFn) @@ -118,9 +117,7 @@ func drainEviction(nodename string, donech chan bool, errch chan error) { glog.Fatal(evicerr) errch <- err } - } - } func getPodSource(pod *v1.Pod) (string, error) { @@ -159,7 +156,7 @@ func containDaemonset(pod v1.Pod, daemonsetList appsv1.DaemonSetList) bool { func evictPod(pod v1.Pod, GracePeriodSeconds int) error { - k8sclient := k8s.Client() + k8sclient := client.ClientSets().K8s().Kubernetes() deleteOptions := &metav1.DeleteOptions{} if GracePeriodSeconds >= 0 { gracePeriodSeconds := int64(GracePeriodSeconds) diff --git a/pkg/models/routers/routers.go b/pkg/models/routers/routers.go index 4cf156170..1a01b1aa4 100644 --- a/pkg/models/routers/routers.go +++ b/pkg/models/routers/routers.go @@ -24,7 +24,7 @@ import ( "io/ioutil" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" - "kubesphere.io/kubesphere/pkg/simple/client/k8s" + "kubesphere.io/kubesphere/pkg/simple/client" "sort" "k8s.io/apimachinery/pkg/labels" @@ -253,7 +253,7 @@ func createRouterService(namespace string, routerType corev1.ServiceType, annota return nil, fmt.Errorf("service template not loaded") } - k8sClient := k8s.Client() + k8sClient := client.ClientSets().K8s().Kubernetes() service := obj.(*corev1.Service) @@ -277,7 +277,7 @@ func createRouterService(namespace string, routerType corev1.ServiceType, annota func updateRouterService(namespace string, routerType corev1.ServiceType, annotations map[string]string) (*corev1.Service, error) { - k8sClient := k8s.Client() + k8sClient := client.ClientSets().K8s().Kubernetes() service, err := getRouterService(namespace) if err != nil { @@ -309,7 +309,7 @@ func deleteRouterService(namespace string) (*corev1.Service, error) { return service, err } - k8sClient := k8s.Client() + k8sClient := client.ClientSets().K8s().Kubernetes() // delete controller service serviceName := constants.IngressControllerPrefix + namespace @@ -333,7 +333,7 @@ func createOrUpdateRouterWorkload(namespace string, publishService bool, service deployName := constants.IngressControllerPrefix + namespace - k8sClient := k8s.Client() + k8sClient := client.ClientSets().K8s().Kubernetes() deployment, err := k8sClient.ExtensionsV1beta1().Deployments(constants.IngressControllerNamespace).Get(deployName, meta_v1.GetOptions{}) createDeployment := true @@ -403,7 +403,7 @@ func createOrUpdateRouterWorkload(namespace string, publishService bool, service } func deleteRouterWorkload(namespace string) error { - k8sClient := k8s.Client() + k8sClient := client.ClientSets().K8s().Kubernetes() deleteOptions := meta_v1.DeleteOptions{} // delete controller deployment diff --git a/pkg/models/tenant/devops.go b/pkg/models/tenant/devops.go index ba287e326..6a5c2afae 100644 --- a/pkg/models/tenant/devops.go +++ b/pkg/models/tenant/devops.go @@ -21,7 +21,6 @@ import ( "fmt" "github.com/emicklei/go-restful" "github.com/gocraft/dbr" - "github.com/golang/glog" "k8s.io/klog" "kubesphere.io/kubesphere/pkg/db" "kubesphere.io/kubesphere/pkg/gojenkins" @@ -29,8 +28,7 @@ import ( "kubesphere.io/kubesphere/pkg/models" "kubesphere.io/kubesphere/pkg/models/devops" "kubesphere.io/kubesphere/pkg/params" - "kubesphere.io/kubesphere/pkg/simple/client/admin_jenkins" - "kubesphere.io/kubesphere/pkg/simple/client/devops_mysql" + cs "kubesphere.io/kubesphere/pkg/simple/client" "net/http" "sync" ) @@ -42,7 +40,11 @@ type DevOpsProjectRoleResponse struct { func ListDevopsProjects(workspace, username string, conditions *params.Conditions, orderBy string, reverse bool, limit int, offset int) (*models.PageableResponse, error) { - dbconn := devops_mysql.OpenDatabase() + dbconn, err := cs.ClientSets().MySQL() + if err != nil { + klog.Error(err) + return nil, err + } query := dbconn.Select(devops.GetColumnsFromStructWithPrefix(devops.DevOpsProjectTableName, devops.DevOpsProject{})...). From(devops.DevOpsProjectTableName) @@ -87,14 +89,14 @@ func ListDevopsProjects(workspace, username string, conditions *params.Condition } query.Limit(uint64(limit)) query.Offset(uint64(offset)) - _, err := query.Load(&projects) + _, err = query.Load(&projects) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(http.StatusInternalServerError, err.Error()) } count, err := query.Count() if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(http.StatusInternalServerError, err.Error()) } @@ -107,7 +109,11 @@ func ListDevopsProjects(workspace, username string, conditions *params.Condition } func GetDevOpsProjectsCount(username string) (uint32, error) { - dbconn := devops_mysql.OpenDatabase() + dbconn, err := cs.ClientSets().MySQL() + if err != nil { + klog.Error(err) + return 0, err + } query := dbconn.Select(devops.GetColumnsFromStructWithPrefix(devops.DevOpsProjectTableName, devops.DevOpsProject{})...). From(devops.DevOpsProjectTableName) @@ -137,20 +143,27 @@ func GetDevOpsProjectsCount(username string) (uint32, error) { func DeleteDevOpsProject(projectId, username string) error { err := devops.CheckProjectUserInRole(username, projectId, []string{devops.ProjectOwner}) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return restful.NewError(http.StatusForbidden, err.Error()) } - gojenkins := admin_jenkins.Client() - if gojenkins == nil { - err := fmt.Errorf("could not connect to jenkins") - glog.Error(err) + + dp, err := cs.ClientSets().Devops() + if err != nil { + klog.Error(err) return restful.NewError(http.StatusServiceUnavailable, err.Error()) } - devopsdb := devops_mysql.OpenDatabase() - _, err = gojenkins.DeleteJob(projectId) + jenkins := dp.Jenkins() + + devopsdb, err := cs.ClientSets().MySQL() + if err != nil { + klog.Error(err) + return restful.NewError(http.StatusServiceUnavailable, err.Error()) + } + + _, err = jenkins.DeleteJob(projectId) if err != nil && utils.GetJenkinsStatusCode(err) != http.StatusNotFound { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } @@ -159,22 +172,22 @@ func DeleteDevOpsProject(projectId, username string) error { roleNames = append(roleNames, devops.GetProjectRoleName(projectId, role)) roleNames = append(roleNames, devops.GetPipelineRoleName(projectId, role)) } - err = gojenkins.DeleteProjectRoles(roleNames...) + err = jenkins.DeleteProjectRoles(roleNames...) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } _, err = devopsdb.DeleteFrom(devops.DevOpsProjectMembershipTableName). Where(db.Eq(devops.DevOpsProjectMembershipProjectIdColumn, projectId)).Exec() if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } _, err = devopsdb.Update(devops.DevOpsProjectTableName). Set(devops.StatusColumn, devops.StatusDeleted). Where(db.Eq(devops.DevOpsProjectIdColumn, projectId)).Exec() if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } project := &devops.DevOpsProject{} @@ -183,7 +196,7 @@ func DeleteDevOpsProject(projectId, username string) error { Where(db.Eq(devops.DevOpsProjectIdColumn, projectId)). LoadOne(project) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } return nil @@ -191,17 +204,24 @@ func DeleteDevOpsProject(projectId, username string) error { func CreateDevopsProject(username string, workspace string, req *devops.DevOpsProject) (*devops.DevOpsProject, error) { - jenkinsClient := admin_jenkins.Client() - if jenkinsClient == nil { - err := fmt.Errorf("could not connect to jenkins") - glog.Error(err) + dp, err := cs.ClientSets().Devops() + if err != nil { + klog.Error(err) + return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) + + } + jenkinsClient := dp.Jenkins() + + devopsdb, err := cs.ClientSets().MySQL() + if err != nil { + klog.Error(err) return nil, restful.NewError(http.StatusServiceUnavailable, err.Error()) } - devopsdb := devops_mysql.OpenDatabase() + project := devops.NewDevOpsProject(req.Name, req.Description, username, req.Extra, workspace) - _, err := jenkinsClient.CreateFolder(project.ProjectId, project.Description) + _, err = jenkinsClient.CreateFolder(project.ProjectId, project.Description) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } @@ -229,14 +249,14 @@ func CreateDevopsProject(username string, workspace string, req *devops.DevOpsPr close(addRoleCh) for addRoleResponse := range addRoleCh { if addRoleResponse.Err != nil { - glog.Errorf("%+v", addRoleResponse.Err) + klog.Errorf("%+v", addRoleResponse.Err) return nil, restful.NewError(utils.GetJenkinsStatusCode(addRoleResponse.Err), addRoleResponse.Err.Error()) } } globalRole, err := jenkinsClient.GetGlobalRole(devops.JenkinsAllUserRoleName) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } if globalRole == nil { @@ -244,41 +264,41 @@ func CreateDevopsProject(username string, workspace string, req *devops.DevOpsPr GlobalRead: true, }, true) if err != nil { - glog.Error("failed to create jenkins global role") + klog.Error("failed to create jenkins global role") return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } } err = globalRole.AssignRole(username) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } projectRole, err := jenkinsClient.GetProjectRole(devops.GetProjectRoleName(project.ProjectId, devops.ProjectOwner)) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } err = projectRole.AssignRole(username) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } pipelineRole, err := jenkinsClient.GetProjectRole(devops.GetPipelineRoleName(project.ProjectId, devops.ProjectOwner)) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } err = pipelineRole.AssignRole(username) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(utils.GetJenkinsStatusCode(err), err.Error()) } _, err = devopsdb.InsertInto(devops.DevOpsProjectTableName). Columns(devops.DevOpsProjectColumns...).Record(project).Exec() if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(http.StatusInternalServerError, err.Error()) } @@ -286,7 +306,7 @@ func CreateDevopsProject(username string, workspace string, req *devops.DevOpsPr _, err = devopsdb.InsertInto(devops.DevOpsProjectMembershipTableName). Columns(devops.DevOpsProjectMembershipColumns...).Record(projectMembership).Exec() if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(http.StatusInternalServerError, err.Error()) } return project, nil @@ -295,7 +315,7 @@ func CreateDevopsProject(username string, workspace string, req *devops.DevOpsPr func GetUserDevopsSimpleRules(username, projectId string) ([]models.SimpleRule, error) { role, err := devops.GetProjectUserRole(username, projectId) if err != nil { - glog.Errorf("%+v", err) + klog.Errorf("%+v", err) return nil, restful.NewError(http.StatusForbidden, err.Error()) } return GetDevopsRoleSimpleRules(role), nil diff --git a/pkg/models/tenant/tenant.go b/pkg/models/tenant/tenant.go index a695e3245..c867c1ac3 100644 --- a/pkg/models/tenant/tenant.go +++ b/pkg/models/tenant/tenant.go @@ -25,7 +25,7 @@ import ( "kubesphere.io/kubesphere/pkg/models" ws "kubesphere.io/kubesphere/pkg/models/workspaces" "kubesphere.io/kubesphere/pkg/params" - "kubesphere.io/kubesphere/pkg/simple/client/k8s" + "kubesphere.io/kubesphere/pkg/simple/client" "strconv" ) @@ -44,7 +44,7 @@ func CreateNamespace(workspaceName string, namespace *v1.Namespace, username str namespace.Labels[constants.WorkspaceLabelKey] = workspaceName - return k8s.Client().CoreV1().Namespaces().Create(namespace) + return client.ClientSets().K8s().Kubernetes().CoreV1().Namespaces().Create(namespace) } func DescribeWorkspace(username, workspaceName string) (*v1alpha1.Workspace, error) { diff --git a/pkg/models/terminal/terminal.go b/pkg/models/terminal/terminal.go index 67b2a21c1..978c3b113 100644 --- a/pkg/models/terminal/terminal.go +++ b/pkg/models/terminal/terminal.go @@ -29,7 +29,7 @@ import ( "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/remotecommand" - "kubesphere.io/kubesphere/pkg/simple/client/k8s" + "kubesphere.io/kubesphere/pkg/simple/client" ) // PtyHandler is what remotecommand expects from a pty @@ -179,11 +179,9 @@ func HandleTerminalSession(session sockjs.Session) { // Executed cmd in the container specified in request and connects it up with the ptyHandler (a session) func startProcess(namespace, podName, containerName string, cmd []string, ptyHandler PtyHandler) error { - k8sClient := k8s.Client() - cfg, err := k8s.Config() - if err != nil { - return err - } + k8sClient := client.ClientSets().K8s().Kubernetes() + + cfg := client.ClientSets().K8s().Config() req := k8sClient.CoreV1().RESTClient().Post(). Resource("pods"). @@ -290,10 +288,6 @@ func NewSession(shell, namespace, podName, containerName string) (string, error) sizeChan: make(chan remotecommand.TerminalSize), } - if err != nil { - return "", err - } - go WaitingForConnection(shell, namespace, podName, containerName, sessionId) return sessionId, nil diff --git a/pkg/models/workloads/jobs.go b/pkg/models/workloads/jobs.go index 43eeee09a..66f5e6b08 100644 --- a/pkg/models/workloads/jobs.go +++ b/pkg/models/workloads/jobs.go @@ -19,7 +19,7 @@ package workloads import ( "fmt" - "kubesphere.io/kubesphere/pkg/simple/client/k8s" + "kubesphere.io/kubesphere/pkg/simple/client" "strings" "time" @@ -31,7 +31,7 @@ import ( const retryTimes = 3 func JobReRun(namespace, jobName string) error { - k8sClient := k8s.Client() + k8sClient := client.ClientSets().K8s().Kubernetes() job, err := k8sClient.BatchV1().Jobs(namespace).Get(jobName, metav1.GetOptions{}) if err != nil { return err @@ -71,7 +71,7 @@ func JobReRun(namespace, jobName string) error { } func deleteJob(namespace, job string) error { - k8sClient := k8s.Client() + k8sClient := client.ClientSets().K8s().Kubernetes() deletePolicy := metav1.DeletePropagationBackground err := k8sClient.BatchV1().Jobs(namespace).Delete(job, &metav1.DeleteOptions{PropagationPolicy: &deletePolicy}) return err diff --git a/pkg/models/workspaces/workspaces.go b/pkg/models/workspaces/workspaces.go index 58795e2b6..4d2f97e71 100644 --- a/pkg/models/workspaces/workspaces.go +++ b/pkg/models/workspaces/workspaces.go @@ -30,8 +30,7 @@ import ( "kubesphere.io/kubesphere/pkg/models/iam" "kubesphere.io/kubesphere/pkg/models/resources" "kubesphere.io/kubesphere/pkg/params" - "kubesphere.io/kubesphere/pkg/simple/client/devops_mysql" - "kubesphere.io/kubesphere/pkg/simple/client/k8s" + clientset "kubesphere.io/kubesphere/pkg/simple/client" "kubesphere.io/kubesphere/pkg/utils/k8sutil" "kubesphere.io/kubesphere/pkg/utils/sliceutil" "strings" @@ -41,7 +40,7 @@ import ( "errors" "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" ) @@ -67,13 +66,13 @@ func Namespaces(workspaceName string) ([]*core.Namespace, error) { } func DeleteNamespace(workspace string, namespaceName string) error { - namespace, err := k8s.Client().CoreV1().Namespaces().Get(namespaceName, meta_v1.GetOptions{}) + namespace, err := clientset.ClientSets().K8s().Kubernetes().CoreV1().Namespaces().Get(namespaceName, metav1.GetOptions{}) if err != nil { return err } if namespace.Labels[constants.WorkspaceLabelKey] == workspace { - deletePolicy := meta_v1.DeletePropagationForeground - return k8s.Client().CoreV1().Namespaces().Delete(namespaceName, &meta_v1.DeleteOptions{PropagationPolicy: &deletePolicy}) + deletePolicy := metav1.DeletePropagationForeground + return clientset.ClientSets().K8s().Kubernetes().CoreV1().Namespaces().Delete(namespaceName, &metav1.DeleteOptions{PropagationPolicy: &deletePolicy}) } else { return errors.New("resource not found") } @@ -134,7 +133,7 @@ func CreateWorkspaceRoleBinding(workspace, username string, role string) error { if !k8sutil.ContainsUser(workspaceRoleBinding.Subjects, username) { workspaceRoleBinding = workspaceRoleBinding.DeepCopy() workspaceRoleBinding.Subjects = append(workspaceRoleBinding.Subjects, v1.Subject{APIGroup: "rbac.authorization.k8s.io", Kind: "User", Name: username}) - _, err = k8s.Client().RbacV1().ClusterRoleBindings().Update(workspaceRoleBinding) + _, err = clientset.ClientSets().K8s().Kubernetes().RbacV1().ClusterRoleBindings().Update(workspaceRoleBinding) if err != nil { log.Errorf("update workspace role binding failed: %+v", err) return err @@ -165,14 +164,17 @@ func DeleteWorkspaceRoleBinding(workspace, username string, role string) error { } } - workspaceRoleBinding, err = k8s.Client().RbacV1().ClusterRoleBindings().Update(workspaceRoleBinding) + workspaceRoleBinding, err = clientset.ClientSets().K8s().Kubernetes().RbacV1().ClusterRoleBindings().Update(workspaceRoleBinding) return err } func GetDevOpsProjects(workspaceName string) ([]string, error) { - dbconn := devops_mysql.OpenDatabase() + dbconn, err := clientset.ClientSets().MySQL() + if err != nil { + return nil, err + } query := dbconn.Select(devops.DevOpsProjectIdColumn). From(devops.DevOpsProjectTableName). @@ -236,7 +238,10 @@ func GetAllProjectNums() (int, error) { } func GetAllDevOpsProjectsNums() (int, error) { - dbconn := devops_mysql.OpenDatabase() + dbconn, err := clientset.ClientSets().MySQL() + if err != nil { + return 0, err + } query := dbconn.Select(devops.DevOpsProjectIdColumn). From(devops.DevOpsProjectTableName). diff --git a/pkg/options/options.go b/pkg/options/options.go index 152b08a9c..c53d15106 100644 --- a/pkg/options/options.go +++ b/pkg/options/options.go @@ -17,7 +17,11 @@ */ package options -import "github.com/spf13/pflag" +import ( + "fmt" + "github.com/spf13/pflag" + "kubesphere.io/kubesphere/pkg/utils/net" +) type ServerRunOptions struct { // server bind address @@ -49,6 +53,26 @@ func NewServerRunOptions() *ServerRunOptions { return &s } +func (s *ServerRunOptions) Validate() []error { + errs := []error{} + + if s.SecurePort == 0 && s.InsecurePort == 0 { + errs = append(errs, fmt.Errorf("insecure and secure port can not be disabled at the same time")) + } + + if net.IsValidPort(s.SecurePort) { + if s.TlsCertFile == "" { + errs = append(errs, fmt.Errorf("tls cert file is empty while secure serving")) + } + + if s.TlsPrivateKey == "" { + errs = append(errs, fmt.Errorf("tls private key file is empty while secure serving")) + } + } + + return errs +} + func (s *ServerRunOptions) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&s.BindAddress, "bind-address", "0.0.0.0", "server bind address") diff --git a/pkg/server/config/config.go b/pkg/server/config/config.go new file mode 100644 index 000000000..d299f4026 --- /dev/null +++ b/pkg/server/config/config.go @@ -0,0 +1,157 @@ +package config + +import ( + "fmt" + "github.com/emicklei/go-restful" + "github.com/spf13/viper" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/klog" + "kubesphere.io/kubesphere/pkg/apiserver/runtime" + "kubesphere.io/kubesphere/pkg/simple/client/devops" + "kubesphere.io/kubesphere/pkg/simple/client/k8s" + "kubesphere.io/kubesphere/pkg/simple/client/ldap" + "kubesphere.io/kubesphere/pkg/simple/client/mysql" + "kubesphere.io/kubesphere/pkg/simple/client/openpitrix" + "kubesphere.io/kubesphere/pkg/simple/client/prometheus" + "kubesphere.io/kubesphere/pkg/simple/client/redis" + "kubesphere.io/kubesphere/pkg/simple/client/s2is3" + "kubesphere.io/kubesphere/pkg/simple/client/servicemesh" + "kubesphere.io/kubesphere/pkg/simple/client/sonarqube" + "net/http" +) + +// install api for config +func InstallAPI(c *restful.Container) { + ws := runtime.NewWebService(schema.GroupVersion{ + Group: "", + Version: "v1alpha1", + }) + + ws.Route(ws.GET("/configz"). + To(func(request *restful.Request, response *restful.Response) { + response.WriteAsJson(sharedConfig) + }). + Doc("Get system components configuration"). + Produces(restful.MIME_JSON). + Writes(Config{}). + Returns(http.StatusOK, "ok", Config{})) + + c.Add(ws) +} + +// load configuration after setup +func Load() error { + sharedConfig = newConfig() + + viper.SetConfigName(DefaultConfigurationName) + viper.AddConfigPath(DefaultConfigurationPath) + viper.AddConfigPath(".") + + if err := viper.ReadInConfig(); err != nil { + if _, ok := err.(viper.ConfigFileNotFoundError); ok { + klog.Warning("configuration file not found") + return nil + } else { + panic(fmt.Errorf("error parsing configuration file %s", err)) + } + } + + conf := &Config{} + if err := viper.Unmarshal(&conf); err != nil { + klog.Error(fmt.Errorf("error unmarshal configuration %v", err)) + return err + } else { + conf.Apply(shadowConfig) + sharedConfig = conf + } + + return nil +} + +const ( + DefaultConfigurationName = "kubesphere" + DefaultConfigurationPath = "/etc/kubesphere" +) + +var ( + // sharedConfig holds configuration across kubesphere + sharedConfig *Config + + // shadowConfig contains options from commandline options + shadowConfig = &Config{} +) + +type Config struct { + MySQLOptions *mysql.MySQLOptions `json:"mysql,omitempty" yaml:"mysql,omitempty" mapstructure:"mysql"` + DevopsOptions *devops.DevopsOptions `json:"devops,omitempty" yaml:"devops,omitempty" mapstructure:"devops"` + SonarQubeOptions *sonarqube.SonarQubeOptions `json:"sonarqube,omitempty" yaml:"sonarQube,omitempty" mapstructure:"sonarqube"` + KubernetesOptions *k8s.KubernetesOptions `json:"kubernetes,omitempty" yaml:"kubernetes,omitempty" mapstructure:"kubernetes"` + ServiceMeshOptions *servicemesh.ServiceMeshOptions `json:"servicemesh,omitempty" yaml:"servicemesh,omitempty" mapstructure:"servicemesh"` + LdapOptions *ldap.LdapOptions `json:"ldap,omitempty" yaml:"ldap,omitempty" mapstructure:"ldap"` + RedisOptions *redis.RedisOptions `json:"redis,omitempty" yaml:"redis,omitempty" mapstructure:"redis"` + S3Options *s2is3.S3Options `json:"s3,omitempty" yaml:"s3,omitempty" mapstructure:"s3"` + OpenPitrixOptions *openpitrix.OpenPitrixOptions `json:"openpitrix,omitempty" yaml:"openpitrix,omitempty" mapstructure:"openpitrix"` + MonitoringOptions *prometheus.PrometheusOptions `json:"monitoring,omitempty" yaml:"monitoring,omitempty" mapstructure:"monitoring"` +} + +func newConfig() *Config { + return &Config{ + MySQLOptions: mysql.NewMySQLOptions(), + DevopsOptions: devops.NewDevopsOptions(), + SonarQubeOptions: sonarqube.NewSonarQubeOptions(), + KubernetesOptions: k8s.NewKubernetesOptions(), + ServiceMeshOptions: servicemesh.NewServiceMeshOptions(), + LdapOptions: ldap.NewLdapOptions(), + RedisOptions: redis.NewRedisOptions(), + S3Options: s2is3.NewS3Options(), + OpenPitrixOptions: openpitrix.NewOpenPitrixOptions(), + MonitoringOptions: prometheus.NewPrometheusOptions(), + } +} + +func Get() *Config { + return sharedConfig +} + +func (c *Config) Apply(conf *Config) { + shadowConfig = conf + + if conf.MonitoringOptions != nil { + conf.MonitoringOptions.ApplyTo(c.MonitoringOptions) + } + if conf.OpenPitrixOptions != nil { + conf.OpenPitrixOptions.ApplyTo(c.OpenPitrixOptions) + } + + if conf.S3Options != nil { + conf.S3Options.ApplyTo(c.S3Options) + } + + if conf.RedisOptions != nil { + conf.RedisOptions.ApplyTo(c.RedisOptions) + } + + if conf.LdapOptions != nil { + conf.LdapOptions.ApplyTo(c.LdapOptions) + } + + if conf.ServiceMeshOptions != nil { + conf.ServiceMeshOptions.ApplyTo(c.ServiceMeshOptions) + } + + if conf.KubernetesOptions != nil { + conf.KubernetesOptions.ApplyTo(c.KubernetesOptions) + } + + if conf.SonarQubeOptions != nil { + conf.SonarQubeOptions.ApplyTo(c.SonarQubeOptions) + } + + if conf.DevopsOptions != nil { + conf.DevopsOptions.ApplyTo(c.DevopsOptions) + } + + if conf.MySQLOptions != nil { + conf.MySQLOptions.ApplyTo(c.MySQLOptions) + } +} diff --git a/pkg/server/config/config_test.go b/pkg/server/config/config_test.go new file mode 100644 index 000000000..61d36d784 --- /dev/null +++ b/pkg/server/config/config_test.go @@ -0,0 +1,129 @@ +package config + +import ( + "fmt" + "gopkg.in/yaml.v2" + "io/ioutil" + "kubesphere.io/kubesphere/pkg/simple/client/devops" + "kubesphere.io/kubesphere/pkg/simple/client/k8s" + "kubesphere.io/kubesphere/pkg/simple/client/ldap" + "kubesphere.io/kubesphere/pkg/simple/client/mysql" + "kubesphere.io/kubesphere/pkg/simple/client/openpitrix" + "kubesphere.io/kubesphere/pkg/simple/client/prometheus" + "kubesphere.io/kubesphere/pkg/simple/client/redis" + "kubesphere.io/kubesphere/pkg/simple/client/s2is3" + "kubesphere.io/kubesphere/pkg/simple/client/servicemesh" + "kubesphere.io/kubesphere/pkg/simple/client/sonarqube" + "os" + "reflect" + "testing" + "time" +) + +func newTestConfig() *Config { + conf := &Config{ + MySQLOptions: &mysql.MySQLOptions{ + Host: "10.68.96.5:3306", + Username: "root", + Password: "admin", + MaxIdleConnections: 10, + MaxOpenConnections: 20, + MaxConnectionLifeTime: time.Duration(10) * time.Second, + }, + DevopsOptions: &devops.DevopsOptions{ + Host: "http://ks-devops.kubesphere-devops-system.svc", + Username: "jenkins", + Password: "kubesphere", + MaxConnections: 10, + }, + SonarQubeOptions: &sonarqube.SonarQubeOptions{ + Host: "http://sonarqube.kubesphere-devops-system.svc", + Token: "ABCDEFG", + }, + KubernetesOptions: &k8s.KubernetesOptions{ + KubeConfig: "/Users/zry/.kube/config", + Master: "https://127.0.0.1:6443", + QPS: 1e6, + Burst: 1e6, + }, + ServiceMeshOptions: &servicemesh.ServiceMeshOptions{ + IstioPilotHost: "http://istio-pilot.istio-system.svc:9090", + JaegerQueryHost: "http://jaeger-query.istio-system.svc:80", + ServicemeshPrometheusHost: "http://prometheus-k8s.kubesphere-monitoring-system.svc", + }, + LdapOptions: &ldap.LdapOptions{ + Host: "http://openldap.kubesphere-system.svc", + ManagerDN: "cn=admin,dc=example,dc=org", + ManagerPassword: "P@88w0rd", + UserSearchBase: "ou=Users,dc=example,dc=org", + GroupSearchBase: "ou=Groups,dc=example,dc=org", + }, + RedisOptions: &redis.RedisOptions{ + Host: "10.10.111.110", + Port: 6379, + Password: "", + DB: 0, + }, + S3Options: &s2is3.S3Options{ + Endpoint: "http://minio.openpitrix-system.svc", + Region: "us-east-1", + DisableSSL: true, + ForcePathStyle: false, + AccessKeyID: "ABCDEFGHIJKLMN", + SecretAccessKey: "OPQRSTUVWXYZ", + SessionToken: "abcdefghijklmn", + Bucket: "ssss", + }, + OpenPitrixOptions: &openpitrix.OpenPitrixOptions{ + APIServer: "http://api-gateway.openpitrix-system.svc", + Token: "ABCDEFGHIJKLMN", + }, + MonitoringOptions: &prometheus.PrometheusOptions{ + Endpoint: "http://prometheus.kubesphere-monitoring-system.svc", + SecondaryEndpoint: "http://prometheus.kubesphere-monitoring-system.svc", + }, + } + return conf +} + +func saveTestConfig(t *testing.T, conf *Config) { + content, err := yaml.Marshal(conf) + if err != nil { + t.Fatalf("error marshal config. %v", err) + } + + err = ioutil.WriteFile(fmt.Sprintf("%s.yaml", DefaultConfigurationName), content, 0640) + if err != nil { + t.Fatalf("error write configuration file, %v", err) + } +} + +func cleanTestConfig(t *testing.T) { + file := fmt.Sprintf("%s.yaml", DefaultConfigurationName) + if _, err := os.Stat(file); os.IsNotExist(err) { + t.Log("file not exists, skipping") + return + } + + err := os.Remove(file) + if err != nil { + t.Fatalf("remove %s file failed", file) + } + +} + +func TestGet(t *testing.T) { + conf := newTestConfig() + saveTestConfig(t, conf) + + err := Load() + if err != nil { + t.Fatal(err) + } + conf2 := Get() + + if !reflect.DeepEqual(conf2, conf) { + t.Fatalf("Get %v\n expected %v\n", conf2, conf) + } + cleanTestConfig(t) +} diff --git a/pkg/simple/client/admin_jenkins/jenkins.go b/pkg/simple/client/admin_jenkins/jenkins.go deleted file mode 100644 index 2ebe1c43a..000000000 --- a/pkg/simple/client/admin_jenkins/jenkins.go +++ /dev/null @@ -1,86 +0,0 @@ -/* -Copyright 2018 The KubeSphere Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package admin_jenkins - -import ( - "flag" - "github.com/golang/glog" - "kubesphere.io/kubesphere/pkg/gojenkins" - "sync" -) - -var ( - jenkinsInitMutex sync.Mutex - jenkinsClient *gojenkins.Jenkins - jenkinsAdminAddress string - jenkinsAdminUsername string - jenkinsAdminPassword string - jenkinsMaxConn int -) - -const ( - JenkinsAllUserRoleName = "kubesphere-user" -) - -func init() { - flag.StringVar(&jenkinsAdminAddress, "jenkins-address", "http://ks-jenkins.kubesphere-devops-system.svc/", "data source name") - flag.StringVar(&jenkinsAdminUsername, "jenkins-username", "admin", "username of jenkins") - flag.StringVar(&jenkinsAdminPassword, "jenkins-password", "passw0rd", "password of jenkins") - flag.IntVar(&jenkinsMaxConn, "jenkins-max-conn", 20, "max conn to jenkins") -} - -func GetJenkins() *gojenkins.Jenkins { - jenkins := gojenkins.CreateJenkins(nil, jenkinsAdminAddress, jenkinsMaxConn, jenkinsAdminUsername, jenkinsAdminPassword) - return jenkins -} - -func Client() *gojenkins.Jenkins { - if jenkinsClient == nil { - jenkinsInitMutex.Lock() - defer jenkinsInitMutex.Unlock() - if jenkinsClient == nil { - jenkins := GetJenkins() - jenkins, err := jenkins.Init() - if err != nil { - glog.Errorf("failed to connect jenkins, %+v", err) - return nil - } - globalRole, err := jenkins.GetGlobalRole(JenkinsAllUserRoleName) - if err != nil { - glog.Errorf("failed to get jenkins role, %+v", err) - return nil - } - if globalRole == nil { - _, err := jenkins.AddGlobalRole(JenkinsAllUserRoleName, gojenkins.GlobalPermissionIds{ - GlobalRead: true, - }, true) - if err != nil { - glog.Errorf("failed to create jenkins global role, %+v", err) - return nil - } - } - _, err = jenkins.AddProjectRole(JenkinsAllUserRoleName, "\\n\\s*\\r", gojenkins.ProjectPermissionIds{ - SCMTag: true, - }, true) - if err != nil { - glog.Errorf("failed to create jenkins project role, %+v", err) - return nil - } - jenkinsClient = jenkins - } - } - - return jenkinsClient - -} diff --git a/pkg/simple/client/devops/devops.go b/pkg/simple/client/devops/devops.go new file mode 100644 index 000000000..6b5952499 --- /dev/null +++ b/pkg/simple/client/devops/devops.go @@ -0,0 +1,109 @@ +/* +Copyright 2018 The KubeSphere Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package devops + +import ( + "fmt" + "k8s.io/klog" + "kubesphere.io/kubesphere/pkg/gojenkins" + "sync" +) + +const ( + JenkinsAllUserRoleName = "kubesphere-user" +) + +type DevopsClient struct { + jenkinsClient *gojenkins.Jenkins +} + +func NewDevopsClient(options *DevopsOptions) (*DevopsClient, error) { + var d DevopsClient + + jenkins := gojenkins.CreateJenkins(nil, options.Host, options.MaxConnections, options.Username, options.Password) + jenkins, err := jenkins.Init() + if err != nil { + klog.Errorf("failed to connecto to jenkins role, %+v", err) + return nil, err + } + + d.jenkinsClient = jenkins + + err = d.initializeJenkins() + if err != nil { + klog.Error(err) + return nil, err + } + + return &d, nil +} + +func NewDevopsClientOrDie(options *DevopsOptions) *DevopsClient { + jenkins := gojenkins.CreateJenkins(nil, options.Host, options.MaxConnections, options.Username, options.Password) + jenkins, err := jenkins.Init() + if err != nil { + klog.Errorf("failed to connecto to jenkins role, %+v", err) + panic(err) + } + + d := &DevopsClient{ + jenkinsClient: jenkins, + } + + err = d.initializeJenkins() + if err != nil { + klog.Error(err) + panic(err) + } + + return d +} + +func (c *DevopsClient) Jenkins() *gojenkins.Jenkins { + return c.jenkinsClient +} + +var mutex = sync.Mutex{} + +func (c *DevopsClient) initializeJenkins() error { + mutex.Lock() + defer mutex.Unlock() + + if c.jenkinsClient == nil { + return fmt.Errorf("jenkins intialization failed") + } + + globalRole, err := c.jenkinsClient.GetGlobalRole(JenkinsAllUserRoleName) + if err != nil { + klog.Error(err) + return err + } + + // Jenkins uninitialized, create global role + if globalRole == nil { + _, err := c.jenkinsClient.AddGlobalRole(JenkinsAllUserRoleName, gojenkins.GlobalPermissionIds{GlobalRead: true}, true) + if err != nil { + klog.Error(err) + return err + } + } + + _, err = c.jenkinsClient.AddProjectRole(JenkinsAllUserRoleName, "\\n\\s*\\r", gojenkins.ProjectPermissionIds{SCMTag: true}, true) + if err != nil { + klog.Error(err) + return err + } + + return nil +} diff --git a/pkg/simple/client/devops/options.go b/pkg/simple/client/devops/options.go new file mode 100644 index 000000000..81e2a9794 --- /dev/null +++ b/pkg/simple/client/devops/options.go @@ -0,0 +1,81 @@ +package devops + +import ( + "fmt" + "github.com/spf13/pflag" +) + +type DevopsOptions struct { + Host string `json:",omitempty" yaml:",omitempty" description:"Jenkins service host address"` + Username string `json:",omitempty" yaml:",omitempty" description:"Jenkins admin username"` + Password string `json:",omitempty" yaml:",omitempty" description:"Jenkins admin password"` + MaxConnections int `json:"maxConnections,omitempty" yaml:"maxConnections,omitempty" description:"Maximum connections allowed to connect to Jenkins"` +} + +// NewDevopsOptions returns a `zero` instance +func NewDevopsOptions() *DevopsOptions { + return &DevopsOptions{ + Host: "", + Username: "", + Password: "", + MaxConnections: 100, + } +} + +func (s *DevopsOptions) ApplyTo(options *DevopsOptions) { + if options == nil { + return + } + + if s.Host != "" { + options.Host = s.Host + } + + if s.Username != "" { + options.Username = s.Username + } + + if s.Password != "" { + options.Password = s.Password + } + + if s.MaxConnections > 0 { + options.MaxConnections = s.MaxConnections + } +} + +// +func (s *DevopsOptions) Validate() []error { + errors := []error{} + + // devops is not needed, ignore rest options + if s.Host == "" { + return errors + } + + if s.Username == "" || s.Password == "" { + errors = append(errors, fmt.Errorf("jenkins's username or password is empty")) + } + + if s.MaxConnections <= 0 { + errors = append(errors, fmt.Errorf("jenkins's maximum connections should be greater than 0")) + } + + return errors +} + +func (s *DevopsOptions) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&s.Host, "jenkins-host", s.Host, ""+ + "Jenkins service host address. If left blank, means Jenkins "+ + "is unnecessary.") + + fs.StringVar(&s.Username, "jenkins-username", s.Username, ""+ + "Username for access to Jenkins service. Leave it blank if there isn't any.") + + fs.StringVar(&s.Password, "jenkins-password", s.Password, ""+ + "Password for access to Jenkins service, used pair with username.") + + fs.IntVar(&s.MaxConnections, "jenkins-max-connections", s.MaxConnections, ""+ + "Maximum allowed connections to Jenkins. ") + +} diff --git a/pkg/simple/client/devops_mysql/mysql.go b/pkg/simple/client/devops_mysql/mysql.go deleted file mode 100644 index ab85b34ac..000000000 --- a/pkg/simple/client/devops_mysql/mysql.go +++ /dev/null @@ -1,56 +0,0 @@ -/* -Copyright 2018 The KubeSphere Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package devops_mysql - -import ( - "flag" - "github.com/gocraft/dbr" - "github.com/golang/glog" - - "kubesphere.io/kubesphere/pkg/db" - "sync" - "time" -) - -var ( - dbClientOnce sync.Once - dsn string - dbClient *db.Database -) - -func init() { - flag.StringVar(&dsn, "devops-database-connection", "root:password@tcp(openpitrix-db.openpitrix-system.svc:3306)/devops", "data source name") -} - -var defaultEventReceiver = db.EventReceiver{} - -func OpenDatabase() *db.Database { - dbClientOnce.Do(func() { - conn, err := dbr.Open("mysql", dsn+"?parseTime=1&multiStatements=1&charset=utf8mb4&collation=utf8mb4_unicode_ci", &defaultEventReceiver) - if err != nil { - glog.Fatal(err) - } - conn.SetMaxIdleConns(100) - conn.SetMaxOpenConns(100) - conn.SetConnMaxLifetime(10 * time.Second) - dbClient = &db.Database{ - Session: conn.NewSession(nil), - } - err = dbClient.Ping() - if err != nil { - glog.Error(err) - } - }) - return dbClient -} diff --git a/pkg/simple/client/factory.go b/pkg/simple/client/factory.go new file mode 100644 index 000000000..8edce34bd --- /dev/null +++ b/pkg/simple/client/factory.go @@ -0,0 +1,315 @@ +package client + +import ( + "fmt" + goredis "github.com/go-redis/redis" + "kubesphere.io/kubesphere/pkg/simple/client/devops" + "kubesphere.io/kubesphere/pkg/simple/client/k8s" + "kubesphere.io/kubesphere/pkg/simple/client/ldap" + "kubesphere.io/kubesphere/pkg/simple/client/mysql" + "kubesphere.io/kubesphere/pkg/simple/client/openpitrix" + "kubesphere.io/kubesphere/pkg/simple/client/prometheus" + "kubesphere.io/kubesphere/pkg/simple/client/redis" + "kubesphere.io/kubesphere/pkg/simple/client/s2is3" + "kubesphere.io/kubesphere/pkg/simple/client/sonarqube" + "sync" +) + +type ClientSetNotEnabledError struct { + err error +} + +func (e ClientSetNotEnabledError) Error() string { + return fmt.Sprintf("client set not enabled: %s", e.err.Error()) +} + +type ClientSetOptions struct { + mySQLOptions *mysql.MySQLOptions + redisOptions *redis.RedisOptions + kubernetesOptions *k8s.KubernetesOptions + devopsOptions *devops.DevopsOptions + sonarqubeOptions *sonarqube.SonarQubeOptions + ldapOptions *ldap.LdapOptions + s3Options *s2is3.S3Options + openPitrixOptions *openpitrix.OpenPitrixOptions + prometheusOptions *prometheus.PrometheusOptions +} + +func NewClientSetOptions() *ClientSetOptions { + return &ClientSetOptions{ + mySQLOptions: mysql.NewMySQLOptions(), + redisOptions: redis.NewRedisOptions(), + kubernetesOptions: k8s.NewKubernetesOptions(), + ldapOptions: ldap.NewLdapOptions(), + devopsOptions: devops.NewDevopsOptions(), + sonarqubeOptions: sonarqube.NewSonarQubeOptions(), + s3Options: s2is3.NewS3Options(), + openPitrixOptions: openpitrix.NewOpenPitrixOptions(), + prometheusOptions: prometheus.NewPrometheusOptions(), + } +} + +func (c *ClientSetOptions) SetMySQLOptions(options *mysql.MySQLOptions) *ClientSetOptions { + c.mySQLOptions = options + return c +} + +func (c *ClientSetOptions) SetRedisOptions(options *redis.RedisOptions) *ClientSetOptions { + c.redisOptions = options + return c +} + +func (c *ClientSetOptions) SetKubernetesOptions(options *k8s.KubernetesOptions) *ClientSetOptions { + c.kubernetesOptions = options + return c +} + +func (c *ClientSetOptions) SetDevopsOptions(options *devops.DevopsOptions) *ClientSetOptions { + c.devopsOptions = options + return c +} + +func (c *ClientSetOptions) SetLdapOptions(options *ldap.LdapOptions) *ClientSetOptions { + c.ldapOptions = options + return c +} + +func (c *ClientSetOptions) SetS3Options(options *s2is3.S3Options) *ClientSetOptions { + c.s3Options = options + return c +} + +func (c *ClientSetOptions) SetOpenPitrixOptions(options *openpitrix.OpenPitrixOptions) *ClientSetOptions { + c.openPitrixOptions = options + return c +} + +func (c *ClientSetOptions) SetPrometheusOptions(options *prometheus.PrometheusOptions) *ClientSetOptions { + c.prometheusOptions = options + return c +} + +// ClientSet provide best of effort service to initialize clients, +// but there is no guarantee to return a valid client instance, +// so do validity check before use +type ClientSet struct { + csoptions *ClientSetOptions + stopCh <-chan struct{} + + mySQLClient *mysql.MySQLClient + + k8sClient *k8s.KubernetesClient + ldapClient *ldap.LdapClient + devopsClient *devops.DevopsClient + sonarQubeClient *sonarqube.SonarQubeClient + redisClient *redis.RedisClient + s3Client *s2is3.S3Client + prometheusClient *prometheus.PrometheusClient + openpitrixClient *openpitrix.OpenPitrixClient +} + +var mutex sync.Mutex + +// global clientsets instance +var sharedClientSet *ClientSet + +func ClientSets() *ClientSet { + return sharedClientSet +} + +func NewClientSetFactory(c *ClientSetOptions, stopCh <-chan struct{}) *ClientSet { + sharedClientSet = &ClientSet{csoptions: c, stopCh: stopCh} + + if c.kubernetesOptions != nil { + sharedClientSet.k8sClient = k8s.NewKubernetesClientOrDie(c.kubernetesOptions) + } + + return sharedClientSet +} + +// lazy creating +func (cs *ClientSet) MySQL() (*mysql.Database, error) { + var err error + + if cs.csoptions.mySQLOptions == nil || cs.csoptions.mySQLOptions.Host == "" { + return nil, ClientSetNotEnabledError{} + } + + if cs.mySQLClient != nil { + return cs.mySQLClient.Database(), nil + } else { + mutex.Lock() + defer mutex.Unlock() + if cs.mySQLClient == nil { + cs.mySQLClient, err = mysql.NewMySQLClient(cs.csoptions.mySQLOptions, cs.stopCh) + if err != nil { + return nil, err + } + } + + return cs.mySQLClient.Database(), nil + } +} + +func (cs *ClientSet) Redis() (*goredis.Client, error) { + var err error + + if cs.csoptions.redisOptions == nil || cs.csoptions.redisOptions.Host == "" { + return nil, ClientSetNotEnabledError{} + } + + if cs.redisClient != nil { + return cs.redisClient.Redis(), nil + } else { + mutex.Lock() + defer mutex.Unlock() + if cs.redisClient == nil { + cs.redisClient, err = redis.NewRedisClient(cs.csoptions.redisOptions, cs.stopCh) + if err != nil { + return nil, err + } + } + + return cs.redisClient.Redis(), nil + } +} + +func (cs *ClientSet) Devops() (*devops.DevopsClient, error) { + var err error + + if cs.csoptions.devopsOptions == nil || cs.csoptions.devopsOptions.Host == "" { + return nil, ClientSetNotEnabledError{} + } + + if cs.devopsClient != nil { + return cs.devopsClient, nil + } else { + mutex.Lock() + defer mutex.Unlock() + + if cs.devopsClient == nil { + cs.devopsClient, err = devops.NewDevopsClient(cs.csoptions.devopsOptions) + if err != nil { + return nil, err + } + } + return cs.devopsClient, nil + } +} + +func (cs *ClientSet) SonarQube() (*sonarqube.SonarQubeClient, error) { + var err error + + if cs.csoptions.sonarqubeOptions == nil || cs.csoptions.sonarqubeOptions.Host == "" { + return nil, ClientSetNotEnabledError{} + } + + if cs.sonarQubeClient != nil { + return cs.sonarQubeClient, nil + } else { + mutex.Lock() + defer mutex.Unlock() + + if cs.sonarQubeClient == nil { + cs.sonarQubeClient, err = sonarqube.NewSonarQubeClient(cs.csoptions.sonarqubeOptions) + if err != nil { + return nil, err + } + } + return cs.sonarQubeClient, nil + } +} + +func (cs *ClientSet) Ldap() (*ldap.LdapClient, error) { + var err error + + if cs.csoptions.ldapOptions == nil || cs.csoptions.ldapOptions.Host == "" { + return nil, ClientSetNotEnabledError{} + } + + if cs.ldapClient != nil { + return cs.ldapClient, nil + } else { + mutex.Lock() + defer mutex.Unlock() + + if cs.ldapClient == nil { + cs.ldapClient, err = ldap.NewLdapClient(cs.csoptions.ldapOptions, cs.stopCh) + if err != nil { + return nil, err + } + } + return cs.ldapClient, nil + } +} + +// since kubernetes client is required, we will +// create it on setup +func (cs *ClientSet) K8s() *k8s.KubernetesClient { + return cs.k8sClient +} + +func (cs *ClientSet) S3() (*s2is3.S3Client, error) { + var err error + + if cs.csoptions.s3Options == nil || cs.csoptions.s3Options.Endpoint == "" { + return nil, ClientSetNotEnabledError{} + } + + if cs.s3Client != nil { + return cs.s3Client, nil + } else { + mutex.Lock() + defer mutex.Unlock() + + if cs.s3Client == nil { + cs.s3Client, err = s2is3.NewS3Client(cs.csoptions.s3Options) + if err != nil { + return nil, err + } + } + return cs.s3Client, nil + } +} + +func (cs *ClientSet) OpenPitrix() (*openpitrix.OpenPitrixClient, error) { + var err error + + if cs.csoptions.openPitrixOptions == nil || cs.csoptions.openPitrixOptions.APIServer == "" { + return nil, ClientSetNotEnabledError{} + } + + if cs.openpitrixClient != nil { + return cs.openpitrixClient, nil + } else { + cs.openpitrixClient, err = openpitrix.NewOpenPitrixClient(cs.csoptions.openPitrixOptions) + if err != nil { + return nil, err + } + + return cs.openpitrixClient, nil + } +} + +func (cs *ClientSet) Prometheus() (*prometheus.PrometheusClient, error) { + var err error + + if cs.csoptions.prometheusOptions == nil || cs.csoptions.prometheusOptions.Endpoint == "" { + return nil, ClientSetNotEnabledError{} + } + + if cs.prometheusClient != nil { + return cs.prometheusClient, nil + } else { + mutex.Lock() + defer mutex.Unlock() + + if cs.prometheusClient == nil { + cs.prometheusClient, err = prometheus.NewPrometheusClient(cs.csoptions.prometheusOptions) + if err != nil { + return nil, err + } + } + return cs.prometheusClient, nil + } +} diff --git a/pkg/simple/client/k8s/k8sclient.go b/pkg/simple/client/k8s/k8sclient.go deleted file mode 100644 index 044d1c05b..000000000 --- a/pkg/simple/client/k8s/k8sclient.go +++ /dev/null @@ -1,79 +0,0 @@ -/* - - Copyright 2019 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -*/ - -package k8s - -import ( - "flag" - "log" - "os" - "sync" - - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" -) - -var ( - kubeConfigFile string - k8sClient *kubernetes.Clientset - k8sClientOnce sync.Once - KubeConfig *rest.Config - MasterURL string -) - -func init() { - flag.StringVar(&kubeConfigFile, "kubeconfig-path", "", "path to kubeconfig file") - flag.StringVar(&MasterURL, "master-url", "", "kube-apiserver url, only needed when out of cluster") -} - -func Client() *kubernetes.Clientset { - - k8sClientOnce.Do(func() { - - config, err := Config() - - if err != nil { - log.Fatalln(err) - } - - k8sClient = kubernetes.NewForConfigOrDie(config) - - KubeConfig = config - }) - - return k8sClient -} - -func Config() (kubeConfig *rest.Config, err error) { - - if _, err = os.Stat(kubeConfigFile); err == nil { - kubeConfig, err = clientcmd.BuildConfigFromFlags(MasterURL, kubeConfigFile) - } else { - kubeConfig, err = rest.InClusterConfig() - } - - if err != nil { - return nil, err - } - - kubeConfig.QPS = 1e6 - kubeConfig.Burst = 1e6 - - return kubeConfig, nil -} diff --git a/pkg/simple/client/k8s/ksclient.go b/pkg/simple/client/k8s/ksclient.go deleted file mode 100644 index 064dd7365..000000000 --- a/pkg/simple/client/k8s/ksclient.go +++ /dev/null @@ -1,46 +0,0 @@ -/* - - Copyright 2019 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -*/ -package k8s - -import ( - "log" - "sync" - - ks "kubesphere.io/kubesphere/pkg/client/clientset/versioned" -) - -var ( - ksClient *ks.Clientset - ksClientOnce sync.Once -) - -func KsClient() *ks.Clientset { - - ksClientOnce.Do(func() { - - config, err := Config() - - if err != nil { - log.Fatalln(err) - } - - ksClient = ks.NewForConfigOrDie(config) - }) - - return ksClient -} diff --git a/pkg/simple/client/k8s/kubernetes.go b/pkg/simple/client/k8s/kubernetes.go new file mode 100644 index 000000000..c9971a90e --- /dev/null +++ b/pkg/simple/client/k8s/kubernetes.go @@ -0,0 +1,100 @@ +package k8s + +import ( + s2i "github.com/kubesphere/s2ioperator/pkg/client/clientset/versioned" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + kubesphere "kubesphere.io/kubesphere/pkg/client/clientset/versioned" +) + +type KubernetesClient struct { + // kubernetes client interface + k8s *kubernetes.Clientset + + // generated clientset + ks *kubesphere.Clientset + + s2i *s2i.Clientset + + master string + + config *rest.Config +} + +// NewKubernetesClient +func NewKubernetesClientOrDie(options *KubernetesOptions) *KubernetesClient { + config, err := clientcmd.BuildConfigFromFlags("", options.KubeConfig) + if err != nil { + panic(err) + } + + config.QPS = options.QPS + config.Burst = options.Burst + + k := &KubernetesClient{ + k8s: kubernetes.NewForConfigOrDie(config), + ks: kubesphere.NewForConfigOrDie(config), + s2i: s2i.NewForConfigOrDie(config), + master: config.Host, + config: config, + } + + if options.Master != "" { + k.master = options.Master + } + + return k +} + +func NewKubernetesClient(options *KubernetesOptions) (*KubernetesClient, error) { + config, err := clientcmd.BuildConfigFromFlags("", options.KubeConfig) + if err != nil { + return nil, err + } + + config.QPS = options.QPS + config.Burst = options.Burst + + var k KubernetesClient + k.k8s, err = kubernetes.NewForConfig(config) + if err != nil { + return nil, err + } + + k.ks, err = kubesphere.NewForConfig(config) + if err != nil { + return nil, err + } + + k.s2i, err = s2i.NewForConfig(config) + if err != nil { + return nil, err + } + + k.master = options.Master + k.config = config + + return &k, nil +} + +func (k *KubernetesClient) Kubernetes() kubernetes.Interface { + return k.k8s +} + +func (k *KubernetesClient) KubeSphere() kubesphere.Interface { + return k.ks +} + +func (k *KubernetesClient) S2i() s2i.Interface { + return k.s2i +} + +// master address used to generate kubeconfig for downloading +func (k *KubernetesClient) Master() string { + return k.master +} + +func (k *KubernetesClient) Config() *rest.Config { + return k.config +} diff --git a/pkg/simple/client/k8s/options.go b/pkg/simple/client/k8s/options.go new file mode 100644 index 000000000..98ba19ab2 --- /dev/null +++ b/pkg/simple/client/k8s/options.go @@ -0,0 +1,59 @@ +package k8s + +import ( + "github.com/spf13/pflag" + "kubesphere.io/kubesphere/pkg/utils/reflectutils" + "os" +) + +type KubernetesOptions struct { + // kubeconfig path, if not specified, will use + // in cluster way to create clientset + KubeConfig string `json:"kubeconfig" yaml:"kubeconfig"` + + // kubernetes apiserver public address, used to generate kubeconfig + // for downloading, default to host defined in kubeconfig + // +optional + Master string `json:"master,omitempty" yaml:"master,omitempty"` + + // kubernetes clientset qps + // +optional + QPS float32 `json:"qps,omitemtpy" yaml:"qps,omitempty"` + + // kubernetes clientset burst + // +optional + Burst int `json:"burst,omitempty" yaml:"burst,omitempty"` +} + +// NewKubernetesOptions returns a `zero` instance +func NewKubernetesOptions() *KubernetesOptions { + return &KubernetesOptions{ + KubeConfig: "", + QPS: 1e6, + Burst: 1e6, + } +} + +func (k *KubernetesOptions) Validate() []error { + errors := []error{} + + if k.KubeConfig != "" { + if _, err := os.Stat(k.KubeConfig); err != nil { + errors = append(errors, err) + } + } + return errors +} + +func (k *KubernetesOptions) ApplyTo(options *KubernetesOptions) { + reflectutils.Override(options, k) +} + +func (k *KubernetesOptions) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&k.KubeConfig, "kubeconfig", k.KubeConfig, ""+ + "Path for kubernetes kubeconfig file, if left blank, will use "+ + "in cluster way.") + + fs.StringVar(&k.Master, "master", k.Master, ""+ + "Used to generate kubeconfig for downloading, if not specified, will use host in kubeconfig.") +} diff --git a/pkg/simple/client/k8s/s2iclient.go b/pkg/simple/client/k8s/s2iclient.go deleted file mode 100644 index 910710f79..000000000 --- a/pkg/simple/client/k8s/s2iclient.go +++ /dev/null @@ -1,46 +0,0 @@ -/* - - Copyright 2019 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -*/ -package k8s - -import ( - "log" - "sync" - - s2i "github.com/kubesphere/s2ioperator/pkg/client/clientset/versioned" -) - -var ( - s2iClient *s2i.Clientset - s2iClientOnce sync.Once -) - -func S2iClient() *s2i.Clientset { - - s2iClientOnce.Do(func() { - - config, err := Config() - - if err != nil { - log.Fatalln(err) - } - - s2iClient = s2i.NewForConfigOrDie(config) - }) - - return s2iClient -} diff --git a/pkg/simple/client/kubesphere/options.go b/pkg/simple/client/kubesphere/options.go new file mode 100644 index 000000000..3d8743f5b --- /dev/null +++ b/pkg/simple/client/kubesphere/options.go @@ -0,0 +1 @@ +package kubesphere diff --git a/pkg/simple/client/ldap/ldap.go b/pkg/simple/client/ldap/ldap.go new file mode 100644 index 000000000..e44c1f213 --- /dev/null +++ b/pkg/simple/client/ldap/ldap.go @@ -0,0 +1,87 @@ +/* + + Copyright 2019 The KubeSphere Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +*/ +package ldap + +import ( + "github.com/go-ldap/ldap" + "k8s.io/klog" +) + +type LdapClient struct { + pool Pool + options *LdapOptions +} + +// panic if cannot connect to ldap service +func NewLdapClient(options *LdapOptions, stopCh <-chan struct{}) (*LdapClient, error) { + pool, err := NewChannelPool(8, 64, "kubesphere", func(s string) (ldap.Client, error) { + conn, err := ldap.Dial("tcp", options.Host) + if err != nil { + return nil, err + } + return conn, nil + }, []uint16{ldap.LDAPResultAdminLimitExceeded, ldap.ErrorNetwork}) + + if err != nil { + klog.Error(err) + pool.Close() + return nil, err + } + + client := &LdapClient{ + pool: pool, + options: options, + } + + go func() { + <-stopCh + if client.pool != nil { + client.pool.Close() + } + }() + + return client, nil +} + +func (l *LdapClient) Ldap() ldap.Client { + if l.pool != nil { + conn, err := l.pool.Get() + if err != nil { + klog.Error(err) + return nil + } + + err = conn.Bind(l.options.ManagerDN, l.options.ManagerPassword) + if err != nil { + conn.Close() + klog.Error(err) + return nil + } + return conn + } + + return nil +} + +func (l *LdapClient) GroupSearchBase() string { + return l.options.GroupSearchBase +} + +func (l *LdapClient) UserSearchBase() string { + return l.options.UserSearchBase +} diff --git a/pkg/simple/client/ldap/ldapclient.go b/pkg/simple/client/ldap/ldapclient.go deleted file mode 100644 index 9c47a89ba..000000000 --- a/pkg/simple/client/ldap/ldapclient.go +++ /dev/null @@ -1,84 +0,0 @@ -/* - - Copyright 2019 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -*/ -package ldap - -import ( - "flag" - "github.com/go-ldap/ldap" - "github.com/golang/glog" - "log" - "sync" -) - -var ( - once sync.Once - pool Pool - ldapHost string - ManagerDN string - ManagerPassword string - UserSearchBase string - GroupSearchBase string - poolSize int -) - -func init() { - flag.StringVar(&ldapHost, "ldap-server", "localhost:389", "ldap server host") - flag.StringVar(&ManagerDN, "ldap-manager-dn", "cn=admin,dc=example,dc=org", "ldap manager dn") - flag.StringVar(&ManagerPassword, "ldap-manager-password", "admin", "ldap manager password") - flag.StringVar(&UserSearchBase, "ldap-user-search-base", "ou=Users,dc=example,dc=org", "ldap user search base") - flag.StringVar(&GroupSearchBase, "ldap-group-search-base", "ou=Groups,dc=example,dc=org", "ldap group search base") - flag.IntVar(&poolSize, "ldap-pool-size", 64, "ldap connection pool size") -} - -func ldapClientPool() Pool { - - once.Do(func() { - var err error - pool, err = NewChannelPool(8, poolSize, "kubesphere", func(s string) (ldap.Client, error) { - conn, err := ldap.Dial("tcp", ldapHost) - if err != nil { - return nil, err - } - return conn, nil - }, []uint16{ldap.LDAPResultTimeLimitExceeded, ldap.ErrorNetwork}) - - if err != nil { - log.Fatalln(err) - } - }) - return pool -} - -func Client() (ldap.Client, error) { - conn, err := ldapClientPool().Get() - - if err != nil { - glog.Errorln("get ldap connection from pool", err) - return nil, err - } - - err = conn.Bind(ManagerDN, ManagerPassword) - - if err != nil { - conn.Close() - glog.Errorln("bind manager dn", err) - return nil, err - } - - return conn, nil -} diff --git a/pkg/simple/client/ldap/options.go b/pkg/simple/client/ldap/options.go new file mode 100644 index 000000000..16a8bbdc6 --- /dev/null +++ b/pkg/simple/client/ldap/options.go @@ -0,0 +1,53 @@ +package ldap + +import ( + "github.com/spf13/pflag" + "kubesphere.io/kubesphere/pkg/utils/reflectutils" +) + +type LdapOptions struct { + Host string `json:"host,omitempty" yaml:"host,omitempty"` + ManagerDN string `json:"managerDN,omitempty" yaml:"managerDN,omitempty"` + ManagerPassword string `json:"managerPassword,omitempty" yaml:"managerPassword,omitempty"` + UserSearchBase string `json:"userSearchBase,omitempty" yaml:"userSearchBase,omitempty"` + GroupSearchBase string `json:"groupSearchBase,omitempty" yaml:"groupSearchBase,omitempty"` +} + +// NewLdapOptions return a default option +// which host field point to nowhere. +func NewLdapOptions() *LdapOptions { + return &LdapOptions{ + Host: "", + ManagerDN: "cn=admin,dc=example,dc=org", + UserSearchBase: "ou=Users,dc=example,dc=org", + GroupSearchBase: "ou=Groups,dc=example,dc=org", + } +} + +func (l *LdapOptions) Validate() []error { + errors := []error{} + + return errors +} + +func (l *LdapOptions) ApplyTo(options *LdapOptions) { + reflectutils.Override(options, l) +} + +func (l *LdapOptions) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&l.Host, "ldap-host", l.Host, ""+ + "Ldap service host, if left blank, all of the following options will "+ + "be ignored and ldap will be disabled.") + + fs.StringVar(&l.ManagerDN, "ldap-manager-dn", l.ManagerDN, ""+ + "Ldap manager account domain name.") + + fs.StringVar(&l.ManagerPassword, "ldap-manager-password", l.ManagerPassword, ""+ + "Ldap manager account password.") + + fs.StringVar(&l.UserSearchBase, "ldap-user-search-base", l.UserSearchBase, ""+ + "Ldap user search base.") + + fs.StringVar(&l.GroupSearchBase, "ldap-group-search-base", l.GroupSearchBase, ""+ + "Ldap group search base.") +} diff --git a/pkg/simple/client/devops_mysql/OWNERS b/pkg/simple/client/mysql/OWNERS similarity index 100% rename from pkg/simple/client/devops_mysql/OWNERS rename to pkg/simple/client/mysql/OWNERS diff --git a/pkg/simple/client/mysql/condition.go b/pkg/simple/client/mysql/condition.go new file mode 100644 index 000000000..1f1beddcd --- /dev/null +++ b/pkg/simple/client/mysql/condition.go @@ -0,0 +1,109 @@ +/* +Copyright 2018 The KubeSphere Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mysql + +import ( + "strings" + + "github.com/gocraft/dbr" +) + +const ( + placeholder = "?" +) + +type EqCondition struct { + dbr.Builder + Column string + Value interface{} +} + +// Copy From vendor/github.com/gocraft/dbr/condition.go:36 +func buildCmp(d dbr.Dialect, buf dbr.Buffer, pred string, column string, value interface{}) error { + buf.WriteString(d.QuoteIdent(column)) + buf.WriteString(" ") + buf.WriteString(pred) + buf.WriteString(" ") + buf.WriteString(placeholder) + + buf.WriteValue(value) + return nil +} + +// And creates AND from a list of conditions +func And(cond ...dbr.Builder) dbr.Builder { + return dbr.And(cond...) +} + +// Or creates OR from a list of conditions +func Or(cond ...dbr.Builder) dbr.Builder { + return dbr.Or(cond...) +} + +func escape(str string) string { + return strings.Map(func(r rune) rune { + switch r { + case '%', '\'', '^', '[', ']', '!', '_': + return ' ' + } + return r + }, str) +} + +func Like(column string, value string) dbr.Builder { + value = "%" + strings.TrimSpace(escape(value)) + "%" + return dbr.BuildFunc(func(d dbr.Dialect, buf dbr.Buffer) error { + return buildCmp(d, buf, "LIKE", column, value) + }) +} + +// Eq is `=`. +// When value is nil, it will be translated to `IS NULL`. +// When value is a slice, it will be translated to `IN`. +// Otherwise it will be translated to `=`. +func Eq(column string, value interface{}) dbr.Builder { + return &EqCondition{ + Builder: dbr.Eq(column, value), + Column: column, + Value: value, + } +} + +// Neq is `!=`. +// When value is nil, it will be translated to `IS NOT NULL`. +// When value is a slice, it will be translated to `NOT IN`. +// Otherwise it will be translated to `!=`. +func Neq(column string, value interface{}) dbr.Builder { + return dbr.Neq(column, value) +} + +// Gt is `>`. +func Gt(column string, value interface{}) dbr.Builder { + return dbr.Gt(column, value) +} + +// Gte is '>='. +func Gte(column string, value interface{}) dbr.Builder { + return dbr.Gte(column, value) +} + +// Lt is '<'. +func Lt(column string, value interface{}) dbr.Builder { + return dbr.Lt(column, value) +} + +// Lte is `<=`. +func Lte(column string, value interface{}) dbr.Builder { + return dbr.Lte(column, value) +} diff --git a/pkg/simple/client/mysql/condition_test.go b/pkg/simple/client/mysql/condition_test.go new file mode 100644 index 000000000..d031e6f64 --- /dev/null +++ b/pkg/simple/client/mysql/condition_test.go @@ -0,0 +1,88 @@ +/* +Copyright 2018 The KubeSphere Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mysql + +import ( + "testing" + + "github.com/gocraft/dbr" + "github.com/gocraft/dbr/dialect" + "github.com/stretchr/testify/assert" +) + +// Ref: https://github.com/gocraft/dbr/blob/5d59a8b3aa915660960329efb3af5513e7a0db07/condition_test.go +func TestCondition(t *testing.T) { + for _, test := range []struct { + cond dbr.Builder + query string + value []interface{} + }{ + { + cond: Eq("col", 1), + query: "`col` = ?", + value: []interface{}{1}, + }, + { + cond: Eq("col", nil), + query: "`col` IS NULL", + value: nil, + }, + { + cond: Eq("col", []int{}), + query: "0", + value: nil, + }, + { + cond: Neq("col", 1), + query: "`col` != ?", + value: []interface{}{1}, + }, + { + cond: Neq("col", nil), + query: "`col` IS NOT NULL", + value: nil, + }, + { + cond: Gt("col", 1), + query: "`col` > ?", + value: []interface{}{1}, + }, + { + cond: Gte("col", 1), + query: "`col` >= ?", + value: []interface{}{1}, + }, + { + cond: Lt("col", 1), + query: "`col` < ?", + value: []interface{}{1}, + }, + { + cond: Lte("col", 1), + query: "`col` <= ?", + value: []interface{}{1}, + }, + { + cond: And(Lt("a", 1), Or(Gt("b", 2), Neq("c", 3))), + query: "(`a` < ?) AND ((`b` > ?) OR (`c` != ?))", + value: []interface{}{1, 2, 3}, + }, + } { + buf := dbr.NewBuffer() + err := test.cond.Build(dialect.MySQL, buf) + assert.NoError(t, err) + assert.Equal(t, test.query, buf.String()) + assert.Equal(t, test.value, buf.Value()) + } +} diff --git a/pkg/simple/client/mysql/db.go b/pkg/simple/client/mysql/db.go new file mode 100644 index 000000000..3a2cdc082 --- /dev/null +++ b/pkg/simple/client/mysql/db.go @@ -0,0 +1,283 @@ +/* +Copyright 2018 The KubeSphere Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mysql + +import ( + "database/sql" + "fmt" + "strings" + + _ "github.com/go-sql-driver/mysql" + "github.com/gocraft/dbr" +) + +const ( + DefaultSelectLimit = 200 +) + +func GetLimit(n uint64) uint64 { + if n < 0 { + n = 0 + } + if n > DefaultSelectLimit { + n = DefaultSelectLimit + } + return n +} + +func GetOffset(n uint64) uint64 { + if n < 0 { + n = 0 + } + return n +} + +type InsertHook func(query *InsertQuery) +type UpdateHook func(query *UpdateQuery) +type DeleteHook func(query *DeleteQuery) + +type Database struct { + *dbr.Session + InsertHook InsertHook + UpdateHook UpdateHook + DeleteHook DeleteHook +} + +type SelectQuery struct { + *dbr.SelectBuilder + JoinCount int // for join filter +} + +type InsertQuery struct { + *dbr.InsertBuilder + Hook InsertHook +} + +type DeleteQuery struct { + *dbr.DeleteBuilder + Hook DeleteHook +} + +type UpdateQuery struct { + *dbr.UpdateBuilder + Hook UpdateHook +} + +type UpsertQuery struct { + table string + *dbr.Session + whereConds map[string]string + upsertValues map[string]interface{} +} + +// SelectQuery +// Example: Select().From().Where().Limit().Offset().OrderDir().Load() +// Select().From().Where().Limit().Offset().OrderDir().LoadOne() +// Select().From().Where().Count() +// SelectAll().From().Where().Limit().Offset().OrderDir().Load() +// SelectAll().From().Where().Limit().Offset().OrderDir().LoadOne() +// SelectAll().From().Where().Count() + +func (db *Database) Select(columns ...string) *SelectQuery { + return &SelectQuery{db.Session.Select(columns...), 0} +} + +func (db *Database) SelectBySql(query string, value ...interface{}) *SelectQuery { + return &SelectQuery{db.Session.SelectBySql(query, value...), 0} +} + +func (db *Database) SelectAll(columns ...string) *SelectQuery { + return &SelectQuery{db.Session.Select("*"), 0} +} + +func (b *SelectQuery) Join(table, on interface{}) *SelectQuery { + b.SelectBuilder.Join(table, on) + return b +} + +func (b *SelectQuery) JoinAs(table string, alias string, on interface{}) *SelectQuery { + b.SelectBuilder.Join(dbr.I(table).As(alias), on) + return b +} + +func (b *SelectQuery) From(table string) *SelectQuery { + b.SelectBuilder.From(table) + return b +} + +func (b *SelectQuery) Where(query interface{}, value ...interface{}) *SelectQuery { + b.SelectBuilder.Where(query, value...) + return b +} + +func (b *SelectQuery) GroupBy(col ...string) *SelectQuery { + b.SelectBuilder.GroupBy(col...) + return b +} + +func (b *SelectQuery) Distinct() *SelectQuery { + b.SelectBuilder.Distinct() + return b +} + +func (b *SelectQuery) Limit(n uint64) *SelectQuery { + n = GetLimit(n) + b.SelectBuilder.Limit(n) + return b +} + +func (b *SelectQuery) Offset(n uint64) *SelectQuery { + n = GetLimit(n) + b.SelectBuilder.Offset(n) + return b +} + +func (b *SelectQuery) OrderDir(col string, isAsc bool) *SelectQuery { + b.SelectBuilder.OrderDir(col, isAsc) + return b +} + +func (b *SelectQuery) Load(value interface{}) (int, error) { + return b.SelectBuilder.Load(value) +} + +func (b *SelectQuery) LoadOne(value interface{}) error { + return b.SelectBuilder.LoadOne(value) +} + +func getColumns(dbrColumns []interface{}) string { + var columns []string + for _, column := range dbrColumns { + if c, ok := column.(string); ok { + columns = append(columns, c) + } + } + return strings.Join(columns, ", ") +} + +func (b *SelectQuery) Count() (count uint32, err error) { + // cache SelectStmt + selectStmt := b.SelectStmt + + limit := selectStmt.LimitCount + offset := selectStmt.OffsetCount + column := selectStmt.Column + isDistinct := selectStmt.IsDistinct + order := selectStmt.Order + + b.SelectStmt.LimitCount = -1 + b.SelectStmt.OffsetCount = -1 + b.SelectStmt.Column = []interface{}{"COUNT(*)"} + b.SelectStmt.Order = []dbr.Builder{} + + if isDistinct { + b.SelectStmt.Column = []interface{}{fmt.Sprintf("COUNT(DISTINCT %s)", getColumns(column))} + b.SelectStmt.IsDistinct = false + } + + err = b.LoadOne(&count) + // fallback SelectStmt + selectStmt.LimitCount = limit + selectStmt.OffsetCount = offset + selectStmt.Column = column + selectStmt.IsDistinct = isDistinct + selectStmt.Order = order + b.SelectStmt = selectStmt + return +} + +// InsertQuery +// Example: InsertInto().Columns().Record().Exec() + +func (db *Database) InsertInto(table string) *InsertQuery { + return &InsertQuery{db.Session.InsertInto(table), db.InsertHook} +} + +func (b *InsertQuery) Exec() (sql.Result, error) { + result, err := b.InsertBuilder.Exec() + if b.Hook != nil && err == nil { + defer b.Hook(b) + } + return result, err +} + +func (b *InsertQuery) Columns(columns ...string) *InsertQuery { + b.InsertBuilder.Columns(columns...) + return b +} + +func (b *InsertQuery) Record(structValue interface{}) *InsertQuery { + b.InsertBuilder.Record(structValue) + return b +} + +// DeleteQuery +// Example: DeleteFrom().Where().Limit().Exec() + +func (db *Database) DeleteFrom(table string) *DeleteQuery { + return &DeleteQuery{db.Session.DeleteFrom(table), db.DeleteHook} +} + +func (b *DeleteQuery) Where(query interface{}, value ...interface{}) *DeleteQuery { + b.DeleteBuilder.Where(query, value...) + return b +} + +func (b *DeleteQuery) Limit(n uint64) *DeleteQuery { + b.DeleteBuilder.Limit(n) + return b +} + +func (b *DeleteQuery) Exec() (sql.Result, error) { + result, err := b.DeleteBuilder.Exec() + if b.Hook != nil && err == nil { + defer b.Hook(b) + } + return result, err +} + +// UpdateQuery +// Example: Update().Set().Where().Exec() + +func (db *Database) Update(table string) *UpdateQuery { + return &UpdateQuery{db.Session.Update(table), db.UpdateHook} +} + +func (b *UpdateQuery) Exec() (sql.Result, error) { + result, err := b.UpdateBuilder.Exec() + if b.Hook != nil && err == nil { + defer b.Hook(b) + } + return result, err +} + +func (b *UpdateQuery) Set(column string, value interface{}) *UpdateQuery { + b.UpdateBuilder.Set(column, value) + return b +} + +func (b *UpdateQuery) SetMap(m map[string]interface{}) *UpdateQuery { + b.UpdateBuilder.SetMap(m) + return b +} + +func (b *UpdateQuery) Where(query interface{}, value ...interface{}) *UpdateQuery { + b.UpdateBuilder.Where(query, value...) + return b +} + +func (b *UpdateQuery) Limit(n uint64) *UpdateQuery { + b.UpdateBuilder.Limit(n) + return b +} diff --git a/pkg/simple/client/mysql/error.go b/pkg/simple/client/mysql/error.go new file mode 100644 index 000000000..e3eafd507 --- /dev/null +++ b/pkg/simple/client/mysql/error.go @@ -0,0 +1,31 @@ +/* +Copyright 2018 The KubeSphere Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mysql + +import ( + "github.com/gocraft/dbr" +) + +// package errors +var ( + ErrNotFound = dbr.ErrNotFound + ErrNotSupported = dbr.ErrNotSupported + ErrTableNotSpecified = dbr.ErrTableNotSpecified + ErrColumnNotSpecified = dbr.ErrColumnNotSpecified + ErrInvalidPointer = dbr.ErrInvalidPointer + ErrPlaceholderCount = dbr.ErrPlaceholderCount + ErrInvalidSliceLength = dbr.ErrInvalidSliceLength + ErrCantConvertToTime = dbr.ErrCantConvertToTime + ErrInvalidTimestring = dbr.ErrInvalidTimestring +) diff --git a/pkg/simple/client/mysql/event.go b/pkg/simple/client/mysql/event.go new file mode 100644 index 000000000..3a2f9f57a --- /dev/null +++ b/pkg/simple/client/mysql/event.go @@ -0,0 +1,55 @@ +/* +Copyright 2018 The KubeSphere Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mysql + +import ( + "github.com/golang/glog" +) + +// EventReceiver is a sentinel EventReceiver; use it if the caller doesn't supply one +type EventReceiver struct{} + +// Event receives a simple notification when various events occur +func (n *EventReceiver) Event(eventName string) { + +} + +// EventKv receives a notification when various events occur along with +// optional key/value data +func (n *EventReceiver) EventKv(eventName string, kvs map[string]string) { +} + +// EventErr receives a notification of an error if one occurs +func (n *EventReceiver) EventErr(eventName string, err error) error { + return err +} + +// EventErrKv receives a notification of an error if one occurs along with +// optional key/value data +func (n *EventReceiver) EventErrKv(eventName string, err error, kvs map[string]string) error { + glog.Errorf("%+v", err) + glog.Errorf("%s: %+v", eventName, kvs) + return err +} + +// Timing receives the time an event took to happen +func (n *EventReceiver) Timing(eventName string, nanoseconds int64) { + +} + +// TimingKv receives the time an event took to happen along with optional key/value data +func (n *EventReceiver) TimingKv(eventName string, nanoseconds int64, kvs map[string]string) { + // TODO: Change logger level to debug + glog.Infof("%s spend %.2fms: %+v", eventName, float32(nanoseconds)/1000000, kvs) +} diff --git a/pkg/simple/client/mysql/mysql.go b/pkg/simple/client/mysql/mysql.go new file mode 100644 index 000000000..dfedf2beb --- /dev/null +++ b/pkg/simple/client/mysql/mysql.go @@ -0,0 +1,82 @@ +/* +Copyright 2018 The KubeSphere Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mysql + +import ( + "fmt" + "github.com/gocraft/dbr" + "k8s.io/klog" +) + +type MySQLClient struct { + database *Database +} + +func NewMySQLClient(options *MySQLOptions, stopCh <-chan struct{}) (*MySQLClient, error) { + var m MySQLClient + + conn, err := dbr.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/devops?parseTime=1&multiStatements=1&charset=utf8mb4&collation=utf8mb4_unicode_ci", options.Username, options.Password, options.Host), nil) + if err != nil { + klog.Error("unable to connect to mysql", err) + return nil, err + } + + conn.SetMaxIdleConns(options.MaxIdleConnections) + conn.SetConnMaxLifetime(options.MaxConnectionLifeTime) + conn.SetMaxOpenConns(options.MaxOpenConnections) + + m.database = &Database{ + Session: conn.NewSession(nil), + } + + go func() { + <-stopCh + if err := conn.Close(); err != nil { + klog.Warning("error happened during closing mysql connection", err) + } + }() + + return &m, nil +} + +func NewMySQLClientOrDie(options *MySQLOptions, stopCh <-chan struct{}) *MySQLClient { + var m MySQLClient + + conn, err := dbr.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/devops?parseTime=1&multiStatements=1&charset=utf8mb4&collation=utf8mb4_unicode_ci", options.Username, options.Password, options.Host), nil) + if err != nil { + klog.Error("unable to connect to mysql", err) + panic(err) + } + + conn.SetMaxIdleConns(options.MaxIdleConnections) + conn.SetConnMaxLifetime(options.MaxConnectionLifeTime) + conn.SetMaxOpenConns(options.MaxOpenConnections) + + m.database = &Database{ + Session: conn.NewSession(nil), + } + + go func() { + <-stopCh + if err := conn.Close(); err != nil { + klog.Warning("error happened during closing mysql connection", err) + } + }() + + return &m +} + +func (m *MySQLClient) Database() *Database { + return m.database +} diff --git a/pkg/simple/client/mysql/options.go b/pkg/simple/client/mysql/options.go new file mode 100644 index 000000000..9eb229b61 --- /dev/null +++ b/pkg/simple/client/mysql/options.go @@ -0,0 +1,43 @@ +package mysql + +import ( + "github.com/spf13/pflag" + reflectutils "kubesphere.io/kubesphere/pkg/utils/reflectutils" + "time" +) + +type MySQLOptions struct { + Host string `json:"host,omitempty" yaml:"host,omitempty" description:"MySQL service host address"` + Username string `json:"username,omitempty" yaml:"username,omitempty"` + Password string `json:"-" yaml:"password,omitempty"` + MaxIdleConnections int `json:"maxIdleConnections,omitempty" yaml:"maxIdleConnections,omitempty"` + MaxOpenConnections int `json:"maxOpenConnections,omitempty" yaml:"maxOpenConnections,omitempty"` + MaxConnectionLifeTime time.Duration `json:"maxConnectionLifeTime,omitempty" yaml:"maxConnectionLifeTime,omitempty"` +} + +// NewMySQLOptions create a `zero` value instance +func NewMySQLOptions() *MySQLOptions { + return &MySQLOptions{} +} + +func (m *MySQLOptions) Validate() []error { + errors := []error{} + + return errors +} + +func (m *MySQLOptions) ApplyTo(options *MySQLOptions) { + reflectutils.Override(options, m) +} + +func (m *MySQLOptions) AddFlags(fs *pflag.FlagSet) { + + fs.StringVar(&m.Host, "mysql-host", m.Host, ""+ + "MySQL service host address. If left blank, following options will be ignored.") + + fs.StringVar(&m.Username, "mysql-username", m.Username, ""+ + "Username for access to mysql service.") + + fs.StringVar(&m.Password, "mysql-password", m.Password, ""+ + "Password for access to mysql, should be used pair with password.") +} diff --git a/pkg/simple/client/openpitrix/applications.go b/pkg/simple/client/openpitrix/applications.go index 1326ba54c..d6d458c28 100644 --- a/pkg/simple/client/openpitrix/applications.go +++ b/pkg/simple/client/openpitrix/applications.go @@ -25,7 +25,6 @@ import ( "net/http" "strconv" "strings" - "time" ) const ( @@ -35,86 +34,9 @@ const ( StateSuffix = "-StatefulSet" ) -type Cluster struct { - ClusterID string `json:"cluster_id"` - Name string `json:"name"` - AppID string `json:"app_id"` - VersionID string `json:"version_id"` - Status string `json:"status"` - UpdateTime time.Time `json:"status_time"` - CreateTime time.Time `json:"create_time"` - RunTimeId string `json:"runtime_id"` - Description string `json:"description"` - ClusterRoleSets []ClusterRole `json:"cluster_role_set"` -} - -type ClusterRole struct { - ClusterID string `json:"cluster_id"` - Role string `json:"role"` -} - -type ClusterList struct { - Total int `json:"total_count"` - Clusters []Cluster `json:"cluster_set"` -} - -type VersionList struct { - Total int `json:"total_count"` - Versions []version `json:"app_version_set"` -} - -type version struct { - Name string `json:"name"` - VersionID string `json:"version_id"` -} - -type runtime struct { - RuntimeID string `json:"runtime_id"` - Zone string `json:"zone"` -} - -type runtimeList struct { - Total int `json:"total_count"` - Runtimes []runtime `json:"runtime_set"` -} - -type app struct { - AppId string `json:"app_id"` - Name string `json:"name"` - ChartName string `json:"chart_name"` - RepoId string `json:"repo_id"` -} - -type repo struct { - RepoId string `json:"repo_id"` - Name string `json:"name"` - Url string `json:"url"` -} - -type appList struct { - Total int `json:"total_count"` - Apps []app `json:"app_set"` -} - -type repoList struct { - Total int `json:"total_count"` - Repos []repo `json:"repo_set"` -} - -type CreateClusterRequest struct { - AppId string `json:"app_id" description:"ID of app to run in cluster, e.g. app-AA3A3y3zEgEM"` - VersionId string `json:"version_id" description:"app version, e.g. appv-154gXYx5RKRp"` - RuntimeId string `json:"runtime_id" description:"ID of runtime, e.g. runtime-wWwXL0LzWqEr"` - Conf string `json:"conf" description:"conf a json string, include cpu, memory info of cluster"` -} - -type DeleteClusterRequest struct { - ClusterId []string `json:"cluster_id" description:"cluster ID"` -} - -func GetAppInfo(appId string) (string, string, string, error) { - url := fmt.Sprintf("%s/v1/apps?app_id=%s", openpitrixAPIServer, appId) - resp, err := makeHttpRequest("GET", url, "") +func (c *OpenPitrixClient) GetAppInfo(appId string) (string, string, string, error) { + url := fmt.Sprintf("%s/v1/apps?app_id=%s", c.apiServer, appId) + resp, err := c.makeHttpRequest("GET", url, "") if err != nil { glog.Error(err) return Unknown, Unknown, Unknown, err @@ -134,14 +56,10 @@ func GetAppInfo(appId string) (string, string, string, error) { return apps.Apps[0].ChartName, apps.Apps[0].RepoId, apps.Apps[0].AppId, nil } -func GetCluster(clusterId string) (*Cluster, error) { - if strings.HasSuffix(openpitrixAPIServer, "/") { - openpitrixAPIServer = strings.TrimSuffix(openpitrixAPIServer, "/") - } +func (c *OpenPitrixClient) GetCluster(clusterId string) (*Cluster, error) { + url := fmt.Sprintf("%s/v1/clusters?cluster_id=%s", c.apiServer, clusterId) - url := fmt.Sprintf("%s/v1/clusters?cluster_id=%s", openpitrixAPIServer, clusterId) - - resp, err := makeHttpRequest("GET", url, "") + resp, err := c.makeHttpRequest("GET", url, "") if err != nil { glog.Error(err) return nil, err @@ -162,14 +80,11 @@ func GetCluster(clusterId string) (*Cluster, error) { return &clusterList.Clusters[0], nil } -func ListClusters(runtimeId, searchWord, status string, limit, offset int) (*ClusterList, error) { - if strings.HasSuffix(openpitrixAPIServer, "/") { - openpitrixAPIServer = strings.TrimSuffix(openpitrixAPIServer, "/") - } +func (c *OpenPitrixClient) ListClusters(runtimeId, searchWord, status string, limit, offset int) (*ClusterList, error) { defaultStatus := "status=active&status=stopped&status=pending&status=ceased" - url := fmt.Sprintf("%s/v1/clusters?limit=%s&offset=%s", openpitrixAPIServer, strconv.Itoa(limit), strconv.Itoa(offset)) + url := fmt.Sprintf("%s/v1/clusters?limit=%s&offset=%s", c.apiServer, strconv.Itoa(limit), strconv.Itoa(offset)) if searchWord != "" { url = fmt.Sprintf("%s&search_word=%s", url, searchWord) @@ -185,7 +100,7 @@ func ListClusters(runtimeId, searchWord, status string, limit, offset int) (*Clu url = fmt.Sprintf("%s&runtime_id=%s", url, runtimeId) } - resp, err := makeHttpRequest("GET", url, "") + resp, err := c.makeHttpRequest("GET", url, "") if err != nil { glog.Errorf("request %s failed, reason: %s", url, err) return nil, err @@ -201,9 +116,9 @@ func ListClusters(runtimeId, searchWord, status string, limit, offset int) (*Clu return &clusterList, nil } -func GetRepo(repoId string) (string, error) { - url := fmt.Sprintf("%s/v1/repos?repo_id=%s", openpitrixAPIServer, repoId) - resp, err := makeHttpRequest("GET", url, "") +func (c *OpenPitrixClient) GetRepo(repoId string) (string, error) { + url := fmt.Sprintf("%s/v1/repos?repo_id=%s", c.apiServer, repoId) + resp, err := c.makeHttpRequest("GET", url, "") if err != nil { glog.Error(err) return Unknown, err @@ -223,9 +138,9 @@ func GetRepo(repoId string) (string, error) { return repos.Repos[0].Name, nil } -func GetVersion(versionId string) (string, error) { - versionUrl := fmt.Sprintf("%s/v1/app_versions?version_id=%s", openpitrixAPIServer, versionId) - resp, err := makeHttpRequest("GET", versionUrl, "") +func (c *OpenPitrixClient) GetVersion(versionId string) (string, error) { + versionUrl := fmt.Sprintf("%s/v1/app_versions?version_id=%s", c.apiServer, versionId) + resp, err := c.makeHttpRequest("GET", versionUrl, "") if err != nil { glog.Error(err) return Unknown, err @@ -244,10 +159,10 @@ func GetVersion(versionId string) (string, error) { return versions.Versions[0].Name, nil } -func GetRuntime(runtimeId string) (string, error) { +func (c *OpenPitrixClient) GetRuntime(runtimeId string) (string, error) { - versionUrl := fmt.Sprintf("%s/v1/runtimes?runtime_id=%s", openpitrixAPIServer, runtimeId) - resp, err := makeHttpRequest("GET", versionUrl, "") + versionUrl := fmt.Sprintf("%s/v1/runtimes?runtime_id=%s", c.apiServer, runtimeId) + resp, err := c.makeHttpRequest("GET", versionUrl, "") if err != nil { glog.Error(err) return Unknown, err @@ -267,9 +182,9 @@ func GetRuntime(runtimeId string) (string, error) { return runtimes.Runtimes[0].Zone, nil } -func CreateCluster(request CreateClusterRequest) error { +func (c *OpenPitrixClient) CreateCluster(request CreateClusterRequest) error { - versionUrl := fmt.Sprintf("%s/v1/clusters/create", openpitrixAPIServer) + versionUrl := fmt.Sprintf("%s/v1/clusters/create", c.apiServer) data, err := json.Marshal(request) @@ -278,7 +193,7 @@ func CreateCluster(request CreateClusterRequest) error { return err } - data, err = makeHttpRequest("POST", versionUrl, string(data)) + data, err = c.makeHttpRequest("POST", versionUrl, string(data)) if err != nil { glog.Error(err) @@ -288,9 +203,9 @@ func CreateCluster(request CreateClusterRequest) error { return nil } -func DeleteCluster(request DeleteClusterRequest) error { +func (c *OpenPitrixClient) DeleteCluster(request DeleteClusterRequest) error { - versionUrl := fmt.Sprintf("%s/v1/clusters/delete", openpitrixAPIServer) + versionUrl := fmt.Sprintf("%s/v1/clusters/delete", c.apiServer) data, err := json.Marshal(request) @@ -299,7 +214,7 @@ func DeleteCluster(request DeleteClusterRequest) error { return err } - data, err = makeHttpRequest("POST", versionUrl, string(data)) + data, err = c.makeHttpRequest("POST", versionUrl, string(data)) if err != nil { glog.Error(err) @@ -309,7 +224,7 @@ func DeleteCluster(request DeleteClusterRequest) error { return nil } -func makeHttpRequest(method, url, data string) ([]byte, error) { +func (c *OpenPitrixClient) makeHttpRequest(method, url, data string) ([]byte, error) { var req *http.Request var err error @@ -319,7 +234,7 @@ func makeHttpRequest(method, url, data string) ([]byte, error) { req, err = http.NewRequest(method, url, strings.NewReader(data)) } - req.Header.Add("Authorization", openpitrixProxyToken) + req.Header.Add("Authorization", c.token) if err != nil { glog.Error(err) @@ -329,7 +244,7 @@ func makeHttpRequest(method, url, data string) ([]byte, error) { resp, err := http.DefaultClient.Do(req) if err != nil { - err := fmt.Errorf("Request to %s failed, method: %s,token: %s, reason: %s ", url, method, openpitrixProxyToken, err) + err := fmt.Errorf("Request to %s failed, method: %s,token: %s, reason: %s ", url, method, c.apiServer, err) glog.Error(err) return nil, err } diff --git a/pkg/simple/client/openpitrix/openpitrixclient.go b/pkg/simple/client/openpitrix/openpitrix.go similarity index 62% rename from pkg/simple/client/openpitrix/openpitrixclient.go rename to pkg/simple/client/openpitrix/openpitrix.go index 5456f22de..988e7fb77 100644 --- a/pkg/simple/client/openpitrix/openpitrixclient.go +++ b/pkg/simple/client/openpitrix/openpitrix.go @@ -20,105 +20,64 @@ package openpitrix import ( "bytes" "encoding/json" - "flag" "fmt" - "github.com/golang/glog" "io/ioutil" + "k8s.io/klog" "kubesphere.io/kubesphere/pkg/utils/sliceutil" "net/http" "strings" - "sync" + "time" ) -var ( - openpitrixAPIServer string - openpitrixProxyToken string - once sync.Once - c client -) - -type RunTime struct { - RuntimeId string `json:"runtime_id"` - RuntimeUrl string `json:"runtime_url"` - Name string `json:"name"` - Provider string `json:"provider"` - Zone string `json:"zone"` - RuntimeCredential string `json:"runtime_credential"` +func NewOpenPitrixClient(options *OpenPitrixOptions) (*OpenPitrixClient, error) { + return &OpenPitrixClient{ + client: &http.Client{ + Timeout: time.Duration(3) * time.Second, + }, + apiServer: options.APIServer, + token: options.Token, + }, nil } -type Interface interface { - CreateRuntime(runtime *RunTime) error - DeleteRuntime(runtimeId string) error -} -type cluster struct { - Status string `json:"status"` - ClusterId string `json:"cluster_id"` -} - -type Error struct { - status int - message string -} - -func (e Error) Error() string { - return fmt.Sprintf("status: %d,message: %s", e.status, e.message) -} - -type client struct { - client http.Client -} - -func init() { - flag.StringVar(&openpitrixAPIServer, "openpitrix-api-server", "http://openpitrix-api-gateway.openpitrix-system.svc:9100", "openpitrix api server") - flag.StringVar(&openpitrixProxyToken, "openpitrix-proxy-token", "", "openpitrix proxy token") -} - -func Client() Interface { - once.Do(func() { - c = client{client: http.Client{}} - }) - return c -} - -func (c client) CreateRuntime(runtime *RunTime) error { +func (c *OpenPitrixClient) CreateRuntime(runtime *RunTime) error { data, err := json.Marshal(runtime) if err != nil { return err } - req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/v1/runtimes", openpitrixAPIServer), bytes.NewReader(data)) + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/v1/runtimes", c.apiServer), bytes.NewReader(data)) if err != nil { return err } req.Header.Add("Content-Type", "application/json") - req.Header.Add("Authorization", openpitrixProxyToken) + req.Header.Add("Authorization", c.token) resp, err := c.client.Do(req) if err != nil { - glog.Error(err) + klog.Error(err) return err } defer resp.Body.Close() data, err = ioutil.ReadAll(resp.Body) if err != nil { - glog.Error(err) + klog.Error(err) return err } if resp.StatusCode > http.StatusOK { err = Error{resp.StatusCode, string(data)} - glog.Error(err) + klog.Error(err) return err } return nil } -func (c client) deleteClusters(clusters []cluster) error { +func (c *OpenPitrixClient) deleteClusters(clusters []cluster) error { clusterId := make([]string, 0) for _, cluster := range clusters { @@ -137,16 +96,16 @@ func (c client) deleteClusters(clusters []cluster) error { ClusterId: clusterId, } data, _ := json.Marshal(deleteRequest) - req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/v1/clusters/delete", openpitrixAPIServer), bytes.NewReader(data)) + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/v1/clusters/delete", c.apiServer), bytes.NewReader(data)) if err != nil { return err } - req.Header.Add("Authorization", openpitrixProxyToken) + req.Header.Add("Authorization", c.token) resp, err := c.client.Do(req) if err != nil { - glog.Error(err) + klog.Error(err) return err } @@ -154,44 +113,44 @@ func (c client) deleteClusters(clusters []cluster) error { data, err = ioutil.ReadAll(resp.Body) if err != nil { - glog.Error(err) + klog.Error(err) return err } if resp.StatusCode > http.StatusOK { err = Error{resp.StatusCode, string(data)} - glog.Error(err) + klog.Error(err) return err } return nil } -func (c client) listClusters(runtimeId string) ([]cluster, error) { +func (c *OpenPitrixClient) listClusters(runtimeId string) ([]cluster, error) { limit := 200 offset := 0 clusters := make([]cluster, 0) for { - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/v1/clusters?runtime_id=%s&limit=%d&offset=%d", openpitrixAPIServer, runtimeId, limit, offset), nil) + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/v1/clusters?runtime_id=%s&limit=%d&offset=%d", c.apiServer, runtimeId, limit, offset), nil) if err != nil { - glog.Error(err) + klog.Error(err) return nil, err } - req.Header.Add("Authorization", openpitrixProxyToken) + req.Header.Add("Authorization", c.token) resp, err := c.client.Do(req) if err != nil { - glog.Error(err) + klog.Error(err) return nil, err } data, err := ioutil.ReadAll(resp.Body) if err != nil { - glog.Error(err) + klog.Error(err) return nil, err } @@ -199,7 +158,7 @@ func (c client) listClusters(runtimeId string) ([]cluster, error) { if resp.StatusCode > http.StatusOK { err = Error{resp.StatusCode, string(data)} - glog.Error(err) + klog.Error(err) return nil, err } listClusterResponse := struct { @@ -209,7 +168,7 @@ func (c client) listClusters(runtimeId string) ([]cluster, error) { err = json.Unmarshal(data, &listClusterResponse) if err != nil { - glog.Error(err) + klog.Error(err) return nil, err } @@ -225,18 +184,18 @@ func (c client) listClusters(runtimeId string) ([]cluster, error) { return clusters, nil } -func (c client) DeleteRuntime(runtimeId string) error { +func (c *OpenPitrixClient) DeleteRuntime(runtimeId string) error { clusters, err := c.listClusters(runtimeId) if err != nil { - glog.Error(err) + klog.Error(err) return err } err = c.deleteClusters(clusters) if err != nil { - glog.Error(err) + klog.Error(err) return err } diff --git a/pkg/simple/client/openpitrix/options.go b/pkg/simple/client/openpitrix/options.go new file mode 100644 index 000000000..911c45696 --- /dev/null +++ b/pkg/simple/client/openpitrix/options.go @@ -0,0 +1,43 @@ +package openpitrix + +import ( + "fmt" + "github.com/spf13/pflag" + "kubesphere.io/kubesphere/pkg/utils/reflectutils" +) + +type OpenPitrixOptions struct { + APIServer string `json:"apiServer,omitempty" yaml:"apiServer,omitempty"` + Token string `json:"token,omitempty" yaml:"token,omitempty"` +} + +func NewOpenPitrixOptions() *OpenPitrixOptions { + return &OpenPitrixOptions{ + APIServer: "", + Token: "", + } +} + +func (s *OpenPitrixOptions) ApplyTo(options *OpenPitrixOptions) { + reflectutils.Override(s, options) +} + +func (s *OpenPitrixOptions) Validate() []error { + errs := []error{} + + if s.APIServer != "" { + if s.Token == "" { + errs = append(errs, fmt.Errorf("OpenPitrix access token cannot be empty")) + } + } + + return errs +} + +func (s *OpenPitrixOptions) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&s.APIServer, "openpitrix-apiserver", s.APIServer, ""+ + "OpenPitrix api gateway endpoint, if left blank, following options will be ignored.") + + fs.StringVar(&s.Token, "openpitrix-token", s.Token, ""+ + "OpenPitrix api access token.") +} diff --git a/pkg/simple/client/openpitrix/types.go b/pkg/simple/client/openpitrix/types.go new file mode 100644 index 000000000..3e41fa4ff --- /dev/null +++ b/pkg/simple/client/openpitrix/types.go @@ -0,0 +1,117 @@ +package openpitrix + +import ( + "fmt" + "net/http" + "time" +) + +type Cluster struct { + ClusterID string `json:"cluster_id"` + Name string `json:"name"` + AppID string `json:"app_id"` + VersionID string `json:"version_id"` + Status string `json:"status"` + UpdateTime time.Time `json:"status_time"` + CreateTime time.Time `json:"create_time"` + RunTimeId string `json:"runtime_id"` + Description string `json:"description"` + ClusterRoleSets []ClusterRole `json:"cluster_role_set"` +} + +type ClusterRole struct { + ClusterID string `json:"cluster_id"` + Role string `json:"role"` +} + +type ClusterList struct { + Total int `json:"total_count"` + Clusters []Cluster `json:"cluster_set"` +} + +type VersionList struct { + Total int `json:"total_count"` + Versions []version `json:"app_version_set"` +} + +type version struct { + Name string `json:"name"` + VersionID string `json:"version_id"` +} + +type runtime struct { + RuntimeID string `json:"runtime_id"` + Zone string `json:"zone"` +} + +type runtimeList struct { + Total int `json:"total_count"` + Runtimes []runtime `json:"runtime_set"` +} + +type app struct { + AppId string `json:"app_id"` + Name string `json:"name"` + ChartName string `json:"chart_name"` + RepoId string `json:"repo_id"` +} + +type repo struct { + RepoId string `json:"repo_id"` + Name string `json:"name"` + Url string `json:"url"` +} + +type appList struct { + Total int `json:"total_count"` + Apps []app `json:"app_set"` +} + +type repoList struct { + Total int `json:"total_count"` + Repos []repo `json:"repo_set"` +} + +type CreateClusterRequest struct { + AppId string `json:"app_id" description:"ID of app to run in cluster, e.g. app-AA3A3y3zEgEM"` + VersionId string `json:"version_id" description:"app version, e.g. appv-154gXYx5RKRp"` + RuntimeId string `json:"runtime_id" description:"ID of runtime, e.g. runtime-wWwXL0LzWqEr"` + Conf string `json:"conf" description:"conf a json string, include cpu, memory info of cluster"` +} + +type DeleteClusterRequest struct { + ClusterId []string `json:"cluster_id" description:"cluster ID"` +} + +type RunTime struct { + RuntimeId string `json:"runtime_id"` + RuntimeUrl string `json:"runtime_url"` + Name string `json:"name"` + Provider string `json:"provider"` + Zone string `json:"zone"` + RuntimeCredential string `json:"runtime_credential"` +} + +type Interface interface { + CreateRuntime(runtime *RunTime) error + DeleteRuntime(runtimeId string) error +} +type cluster struct { + Status string `json:"status"` + ClusterId string `json:"cluster_id"` +} + +type Error struct { + status int + message string +} + +func (e Error) Error() string { + return fmt.Sprintf("status: %d,message: %s", e.status, e.message) +} + +type OpenPitrixClient struct { + client *http.Client + apiServer string + token string +} diff --git a/pkg/simple/client/prometheus/options.go b/pkg/simple/client/prometheus/options.go new file mode 100644 index 000000000..6c18615be --- /dev/null +++ b/pkg/simple/client/prometheus/options.go @@ -0,0 +1,37 @@ +package prometheus + +import ( + "github.com/spf13/pflag" + "kubesphere.io/kubesphere/pkg/utils/reflectutils" +) + +type PrometheusOptions struct { + Endpoint string `json:"endpoint,omitempty" yaml:"endpoint,omitempty"` + SecondaryEndpoint string `json:"secondaryEndpoint,omitempty" yaml:"secondaryEndpoint,omitempty"` +} + +func NewPrometheusOptions() *PrometheusOptions { + return &PrometheusOptions{ + Endpoint: "", + SecondaryEndpoint: "", + } +} + +func (s *PrometheusOptions) Validate() []error { + errs := []error{} + + return errs +} + +func (s *PrometheusOptions) ApplyTo(options *PrometheusOptions) { + reflectutils.Override(s, options) +} + +func (s *PrometheusOptions) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&s.Endpoint, "prometheus-endpoint", s.Endpoint, ""+ + "Prometheus service endpoint which stores KubeSphere monitoring data, if left "+ + "blank, will use builtin metrics-server as data source.") + + fs.StringVar(&s.SecondaryEndpoint, "prometheus-secondary-endpoint", s.SecondaryEndpoint, ""+ + "Prometheus secondary service endpoint, if left empty and endpoint is set, will use endpoint instead.") +} diff --git a/pkg/simple/client/prometheus/prometheus.go b/pkg/simple/client/prometheus/prometheus.go new file mode 100644 index 000000000..6ababf0d7 --- /dev/null +++ b/pkg/simple/client/prometheus/prometheus.go @@ -0,0 +1,67 @@ +/* + + Copyright 2019 The KubeSphere Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +*/ +package prometheus + +import ( + "github.com/golang/glog" + "io/ioutil" + "net/http" + "time" +) + +type PrometheusClient struct { + client *http.Client + endpoint string + secondaryEndpoint string +} + +func NewPrometheusClient(options *PrometheusOptions) (*PrometheusClient, error) { + return &PrometheusClient{ + client: &http.Client{ + Timeout: time.Duration(3) * time.Second, + }, + endpoint: options.Endpoint, + secondaryEndpoint: options.SecondaryEndpoint, + }, nil +} + +func (c *PrometheusClient) SendMonitoringRequest(queryType string, params string) string { + return c.sendMonitoringRequest(c.endpoint, queryType, params) +} + +func (c *PrometheusClient) SendSecondaryMonitoringRequest(queryType string, params string) string { + return c.sendMonitoringRequest(c.secondaryEndpoint, queryType, params) +} + +func (c *PrometheusClient) sendMonitoringRequest(endpoint string, queryType string, params string) string { + epurl := endpoint + queryType + params + response, err := c.client.Get(epurl) + if err != nil { + glog.Error(err) + } else { + defer response.Body.Close() + + contents, err := ioutil.ReadAll(response.Body) + + if err != nil { + glog.Error(err) + } + return string(contents) + } + return "" +} diff --git a/pkg/simple/client/prometheus/prometheusclient.go b/pkg/simple/client/prometheus/prometheusclient.go deleted file mode 100644 index b75d38993..000000000 --- a/pkg/simple/client/prometheus/prometheusclient.go +++ /dev/null @@ -1,203 +0,0 @@ -/* - - Copyright 2019 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -*/ -package prometheus - -import ( - "flag" - "io/ioutil" - "kubesphere.io/kubesphere/pkg/informers" - "net/http" - "net/url" - "strconv" - "strings" - "time" - - "github.com/emicklei/go-restful" - "github.com/golang/glog" -) - -const ( - DefaultQueryStep = "10m" - DefaultQueryTimeout = "10s" - RangeQueryType = "query_range?" - DefaultQueryType = "query?" -) - -// Kubesphere sets up two Prometheus servers to balance monitoring workloads -var ( - PrometheusEndpoint string // For monitoring node, namespace, pod ... level resources - SecondaryPrometheusEndpoint string // For monitoring components including etcd, apiserver, coredns, etc. -) - -func init() { - flag.StringVar(&PrometheusEndpoint, "prometheus-endpoint", "http://prometheus-k8s.kubesphere-monitoring-system.svc:9090/api/v1/", "For physical and k8s resource monitoring, including node, namespace, pod, etc.") - flag.StringVar(&SecondaryPrometheusEndpoint, "secondary-prometheus-endpoint", "http://prometheus-k8s-system.kubesphere-monitoring-system.svc:9090/api/v1/", "For k8s component monitoring, including etcd, apiserver, coredns, etc.") -} - -type MonitoringRequestParams struct { - Params url.Values - QueryType string - SortMetricName string - SortType string - PageNum string - LimitNum string - Tp string - MetricsFilter string - ResourcesFilter string - MetricsName string - WorkloadName string - NodeId string - WsName string - NsName string - PodName string - PVCName string - StorageClassName string - ContainerName string - WorkloadKind string - ComponentName string -} - -var client = &http.Client{} - -func SendMonitoringRequest(prometheusEndpoint string, queryType string, params string) string { - epurl := prometheusEndpoint + queryType + params - response, err := client.Get(epurl) - if err != nil { - glog.Error(err) - } else { - defer response.Body.Close() - - contents, err := ioutil.ReadAll(response.Body) - - if err != nil { - glog.Error(err) - } - return string(contents) - } - return "" -} - -func ParseMonitoringRequestParams(request *restful.Request) *MonitoringRequestParams { - instantTime := strings.Trim(request.QueryParameter("time"), " ") - start := strings.Trim(request.QueryParameter("start"), " ") - end := strings.Trim(request.QueryParameter("end"), " ") - step := strings.Trim(request.QueryParameter("step"), " ") - timeout := strings.Trim(request.QueryParameter("timeout"), " ") - - sortMetricName := strings.Trim(request.QueryParameter("sort_metric"), " ") - sortType := strings.Trim(request.QueryParameter("sort_type"), " ") - pageNum := strings.Trim(request.QueryParameter("page"), " ") - limitNum := strings.Trim(request.QueryParameter("limit"), " ") - tp := strings.Trim(request.QueryParameter("type"), " ") - - metricsFilter := strings.Trim(request.QueryParameter("metrics_filter"), " ") - resourcesFilter := strings.Trim(request.QueryParameter("resources_filter"), " ") - - metricsName := strings.Trim(request.QueryParameter("metrics_name"), " ") - workloadName := strings.Trim(request.PathParameter("workload"), " ") - - nodeId := strings.Trim(request.PathParameter("node"), " ") - wsName := strings.Trim(request.PathParameter("workspace"), " ") - nsName := strings.Trim(request.PathParameter("namespace"), " ") - podName := strings.Trim(request.PathParameter("pod"), " ") - pvcName := strings.Trim(request.PathParameter("pvc"), " ") - storageClassName := strings.Trim(request.PathParameter("storageclass"), " ") - containerName := strings.Trim(request.PathParameter("container"), " ") - workloadKind := strings.Trim(request.PathParameter("kind"), " ") - componentName := strings.Trim(request.PathParameter("component"), " ") - - var requestParams = MonitoringRequestParams{ - SortMetricName: sortMetricName, - SortType: sortType, - PageNum: pageNum, - LimitNum: limitNum, - Tp: tp, - MetricsFilter: metricsFilter, - ResourcesFilter: resourcesFilter, - MetricsName: metricsName, - WorkloadName: workloadName, - NodeId: nodeId, - WsName: wsName, - NsName: nsName, - PodName: podName, - PVCName: pvcName, - StorageClassName: storageClassName, - ContainerName: containerName, - WorkloadKind: workloadKind, - ComponentName: componentName, - } - - if timeout == "" { - timeout = DefaultQueryTimeout - } - if step == "" { - step = DefaultQueryStep - } - // Whether query or query_range request - u := url.Values{} - - if start != "" && end != "" { - - u.Set("start", convertTimeGranularity(start)) - u.Set("end", convertTimeGranularity(end)) - u.Set("step", step) - u.Set("timeout", timeout) - - // range query start time must be greater than the namespace creation time - if nsName != "" { - nsLister := informers.SharedInformerFactory().Core().V1().Namespaces().Lister() - ns, err := nsLister.Get(nsName) - if err == nil { - queryStartTime := u.Get("start") - nsCreationTime := strconv.FormatInt(ns.CreationTimestamp.Unix(), 10) - if nsCreationTime > queryStartTime { - u.Set("start", nsCreationTime) - } - } - } - - requestParams.QueryType = RangeQueryType - requestParams.Params = u - - return &requestParams - } - if instantTime != "" { - u.Set("time", instantTime) - u.Set("timeout", timeout) - requestParams.QueryType = DefaultQueryType - requestParams.Params = u - return &requestParams - } else { - u.Set("timeout", timeout) - requestParams.QueryType = DefaultQueryType - requestParams.Params = u - return &requestParams - } -} - -func convertTimeGranularity(ts string) string { - timeFloat, err := strconv.ParseFloat(ts, 64) - if err != nil { - glog.Errorf("convert second timestamp %s to minute timestamp failed", ts) - return strconv.FormatInt(int64(time.Now().Unix()), 10) - } - timeInt := int64(timeFloat) - // convert second timestamp to minute timestamp - secondTime := time.Unix(timeInt, 0).Truncate(time.Minute).Unix() - return strconv.FormatInt(secondTime, 10) -} diff --git a/pkg/simple/client/redis/options.go b/pkg/simple/client/redis/options.go new file mode 100644 index 000000000..02fd3ea6e --- /dev/null +++ b/pkg/simple/client/redis/options.go @@ -0,0 +1,57 @@ +package redis + +import ( + "fmt" + "github.com/spf13/pflag" + "kubesphere.io/kubesphere/pkg/utils/net" + "kubesphere.io/kubesphere/pkg/utils/reflectutils" +) + +type RedisOptions struct { + Host string + Port int + Password string + DB int +} + +// NewRedisOptions returns options points to nowhere, +// because redis is not required for some components +func NewRedisOptions() *RedisOptions { + return &RedisOptions{ + Host: "", + Port: 6379, + Password: "", + DB: 0, + } +} + +func (r *RedisOptions) Validate() []error { + errors := make([]error, 0) + + if r.Host != "" { + if !net.IsValidPort(r.Port) { + errors = append(errors, fmt.Errorf("--redis-port is out of range")) + } + } + + return errors +} + +func (r *RedisOptions) ApplyTo(options *RedisOptions) { + reflectutils.Override(options, r) +} + +func (r *RedisOptions) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&r.Host, "--redis-host", r.Host, ""+ + "Redis service host address. If left blank, means redis is unnecessary, "+ + "redis will be disabled") + + fs.IntVar(&r.Port, "--redis-port", r.Port, ""+ + "Redis service port number.") + + fs.StringVar(&r.Password, "--redis-password", r.Password, ""+ + "Redis service password if necessary, default to empty") + + fs.IntVar(&r.DB, "--redis-db", r.DB, ""+ + "Redis service database index, default to 0.") +} diff --git a/pkg/simple/client/redis/redis.go b/pkg/simple/client/redis/redis.go index a97cc9c57..aafade8d5 100644 --- a/pkg/simple/client/redis/redis.go +++ b/pkg/simple/client/redis/redis.go @@ -18,48 +18,49 @@ package redis import ( - "flag" - "log" - "os" - "os/signal" - "sync" - "syscall" - + "fmt" "github.com/go-redis/redis" + "k8s.io/klog" ) -var ( - redisHost string - redisPassword string - redisDB int - redisClientOnce sync.Once - redisClient *redis.Client -) - -func init() { - flag.StringVar(&redisHost, "redis-server", "localhost:6379", "redis server host") - flag.StringVar(&redisPassword, "redis-password", "", "redis password") - flag.IntVar(&redisDB, "redis-db", 0, "redis db") +type RedisClient struct { + client *redis.Client } -func Client() *redis.Client { +func NewRedisClientOrDie(options *RedisOptions, stopCh <-chan struct{}) *RedisClient { + client, err := NewRedisClient(options, stopCh) + if err != nil { + panic(err) + } - redisClientOnce.Do(func() { - redisClient = redis.NewClient(&redis.Options{ - Addr: redisHost, - Password: redisPassword, - DB: redisDB, - }) - if err := redisClient.Ping().Err(); err != nil { - log.Fatalln(err) - } - c := make(chan os.Signal, 0) - signal.Notify(c, os.Interrupt, syscall.SIGTERM) - go func() { - <-c - redisClient.Close() - }() + return client +} + +func NewRedisClient(option *RedisOptions, stopCh <-chan struct{}) (*RedisClient, error) { + var r RedisClient + + r.client = redis.NewClient(&redis.Options{ + Addr: fmt.Sprintf("%s:%d", option.Host, option.Port), + Password: option.Password, + DB: option.DB, }) - return redisClient + if err := r.client.Ping().Err(); err != nil { + klog.Error("unable to reach redis host", err) + r.client.Close() + return nil, err + } + + go func() { + <-stopCh + if err := r.client.Close(); err != nil { + klog.Error(err) + } + }() + + return &r, nil +} + +func (r *RedisClient) Redis() *redis.Client { + return r.client } diff --git a/pkg/simple/client/s2is3/options.go b/pkg/simple/client/s2is3/options.go new file mode 100644 index 000000000..7e0551b79 --- /dev/null +++ b/pkg/simple/client/s2is3/options.go @@ -0,0 +1,61 @@ +package s2is3 + +import ( + "github.com/spf13/pflag" + "kubesphere.io/kubesphere/pkg/utils/reflectutils" +) + +type S3Options struct { + Endpoint string `json:"endpoint,omitempty" yaml:"endpoint,omitempty"` + Region string `json:"region,omitempty" yaml:"region,omitempty"` + DisableSSL bool `json:"disableSSL,omitempty" yaml:"disableSSL,omitempty"` + ForcePathStyle bool `json:"forcePathStyle,omitempty" yaml:"forePathStyle,omitempty"` + AccessKeyID string `json:"accessKeyID,omitempty" yaml:"accessKeyID,omitempty"` + SecretAccessKey string `json:"secretAccessKey,omitempty" yaml:"secretAccessKey,omitempty"` + SessionToken string `json:"sessionToken,omitempty" yaml:"sessionToken,omitempty"` + Bucket string `json:"bucket,omitempty" yaml:"bucket,omitempty"` +} + +func NewS3Options() *S3Options { + return &S3Options{ + Endpoint: "", + Region: "", + DisableSSL: true, + ForcePathStyle: true, + AccessKeyID: "", + SecretAccessKey: "", + SessionToken: "", + Bucket: "", + } +} + +func (s *S3Options) Validate() []error { + errors := []error{} + + return errors +} + +func (s *S3Options) ApplyTo(options *S3Options) { + reflectutils.Override(options, s) +} + +func (s *S3Options) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&s.Endpoint, "s3-endpoint", s.Endpoint, ""+ + "Endpoint to access to s3 object storage service, if left blank, the following options "+ + "will be ignored.") + + fs.StringVar(&s.Region, "s3-region", s.Region, ""+ + "Region of s3 that will access to, like us-east-1.") + + fs.StringVar(&s.AccessKeyID, "s3-access-key-id", "AKIAIOSFODNN7EXAMPLE", "access key of s2i s3") + + fs.StringVar(&s.SecretAccessKey, "s3-secret-access-key", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "secret access key of s2i s3") + + fs.StringVar(&s.SessionToken, "s3-session-token", s.SessionToken, "session token of s2i s3") + + fs.StringVar(&s.Bucket, "s3-bucket", "s2i-binaries", "bucket name of s2i s3") + + fs.BoolVar(&s.DisableSSL, "s3-disable-SSL", s.DisableSSL, "disable ssl") + + fs.BoolVar(&s.ForcePathStyle, "s3-force-path-style", true, "force path style") +} diff --git a/pkg/simple/client/s2is3/s3.go b/pkg/simple/client/s2is3/s3.go index 820fea4a4..e9f7ec09f 100644 --- a/pkg/simple/client/s2is3/s3.go +++ b/pkg/simple/client/s2is3/s3.go @@ -1,83 +1,79 @@ package s2is3 import ( - "flag" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "k8s.io/klog" - "sync" ) -var ( - s3Region string - s3Endpoint string - s3DisableSSL bool - s3ForcePathStyle bool - s3AccessKeyID string - s3SecretAccessKey string - s3SessionToken string - s3Bucket string -) -var ( - s2iS3 *s3.S3 - s2iS3Session *session.Session - sessionInitMutex sync.Mutex - clientInitMutex sync.Mutex -) - -func init() { - flag.StringVar(&s3Region, "s2i-s3-region", "us-east-1", "region of s2i s3") - flag.StringVar(&s3Endpoint, "s2i-s3-endpoint", "http://ks-minio.kubesphere-system.svc", "endpoint of s2i s3") - flag.StringVar(&s3AccessKeyID, "s2i-s3-access-key-id", "AKIAIOSFODNN7EXAMPLE", "access key of s2i s3") - flag.StringVar(&s3SecretAccessKey, "s2i-s3-secret-access-key", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "secret access key of s2i s3") - flag.StringVar(&s3SessionToken, "s2i-s3-session-token", "", "session token of s2i s3") - flag.StringVar(&s3Bucket, "s2i-s3-bucket", "s2i-binaries", "bucket name of s2i s3") - flag.BoolVar(&s3DisableSSL, "s2i-s3-disable-SSL", true, "disable ssl") - flag.BoolVar(&s3ForcePathStyle, "s2i-s3-force-path-style", true, "force path style") +type S3Client struct { + s3Client *s3.S3 + s3Session *session.Session + bucket string } -func Client() *s3.S3 { - if s2iS3 != nil { - return s2iS3 +func NewS3Client(options *S3Options) (*S3Client, error) { + cred := credentials.NewStaticCredentials(options.AccessKeyID, options.SecretAccessKey, options.SessionToken) + + config := aws.Config{ + Region: aws.String(options.Region), + Endpoint: aws.String(options.Endpoint), + DisableSSL: aws.Bool(options.DisableSSL), + S3ForcePathStyle: aws.Bool(options.ForcePathStyle), + Credentials: cred, } - clientInitMutex.Lock() - defer clientInitMutex.Unlock() - if s2iS3Session == nil { - if sess := Session(); sess != nil { - klog.Error("failed to connect to s2i s3") - return nil - } - } - s2iS3 = s3.New(s2iS3Session) - return s2iS3 -} -func Session() *session.Session { - if s2iS3Session != nil { - return s2iS3Session - } - sessionInitMutex.Lock() - defer sessionInitMutex.Unlock() - creds := credentials.NewStaticCredentials( - s3AccessKeyID, s3SecretAccessKey, s3SessionToken, - ) - config := &aws.Config{ - Region: aws.String(s3Region), - Endpoint: aws.String(s3Endpoint), - DisableSSL: aws.Bool(s3DisableSSL), - S3ForcePathStyle: aws.Bool(s3ForcePathStyle), - Credentials: creds, - } - sess, err := session.NewSession(config) + + s, err := session.NewSession(&config) if err != nil { - klog.Errorf("failed to connect to s2i s3: %+v", err) - return nil + klog.Error(err) + return nil, err } - s2iS3Session = sess - return s2iS3Session + + var c S3Client + + c.s3Client = s3.New(s) + c.s3Session = s + c.bucket = options.Bucket + + return &c, nil } -func Bucket() *string { - return aws.String(s3Bucket) +// NewS3ClientOrDie creates S3Client and panics if there is an error +func NewS3ClientOrDie(options *S3Options) *S3Client { + cred := credentials.NewStaticCredentials(options.AccessKeyID, options.SecretAccessKey, options.SessionToken) + + config := aws.Config{ + Region: aws.String(options.Region), + Endpoint: aws.String(options.Endpoint), + DisableSSL: aws.Bool(options.DisableSSL), + S3ForcePathStyle: aws.Bool(options.ForcePathStyle), + Credentials: cred, + } + + s, err := session.NewSession(&config) + if err != nil { + panic(err) + } + + client := s3.New(s) + + return &S3Client{ + s3Client: client, + s3Session: s, + bucket: options.Bucket, + } +} + +func (s *S3Client) Client() *s3.S3 { + + return s.s3Client +} +func (s *S3Client) Session() *session.Session { + return s.s3Session +} + +func (s *S3Client) Bucket() *string { + return aws.String(s.bucket) } diff --git a/pkg/simple/client/servicemesh/options.go b/pkg/simple/client/servicemesh/options.go new file mode 100644 index 000000000..a8e395254 --- /dev/null +++ b/pkg/simple/client/servicemesh/options.go @@ -0,0 +1,59 @@ +package servicemesh + +import "github.com/spf13/pflag" + +type ServiceMeshOptions struct { + + // istio pilot discovery service url + IstioPilotHost string `json:"istioPilotHost,omitempty" yaml:"istioPilotHost,omitempty"` + + // jaeger query service url + JaegerQueryHost string `json:"jaegerQueryHost,omitempty" yaml:"jaegerQueryHost,omitempty"` + + // prometheus service url for servicemesh metrics + ServicemeshPrometheusHost string `json:"servicemeshPrometheusHost,omitempty" yaml:"servicemeshPrometheusHost,omitempty"` +} + +// NewServiceMeshOptions returns a `zero` instance +func NewServiceMeshOptions() *ServiceMeshOptions { + return &ServiceMeshOptions{ + IstioPilotHost: "", + JaegerQueryHost: "", + ServicemeshPrometheusHost: "", + } +} + +func (s *ServiceMeshOptions) Validate() []error { + errors := []error{} + + return errors +} + +func (s *ServiceMeshOptions) ApplyTo(options *ServiceMeshOptions) { + if options == nil { + return + } + + if s.ServicemeshPrometheusHost != "" { + options.ServicemeshPrometheusHost = s.ServicemeshPrometheusHost + } + + if s.JaegerQueryHost != "" { + options.JaegerQueryHost = s.JaegerQueryHost + } + + if s.IstioPilotHost != "" { + options.IstioPilotHost = s.IstioPilotHost + } +} + +func (s *ServiceMeshOptions) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&s.IstioPilotHost, "istio-pilot-host", s.IstioPilotHost, ""+ + "istio pilot discovery service url") + + fs.StringVar(&s.JaegerQueryHost, "jaeger-query-host", s.JaegerQueryHost, ""+ + "jaeger query service url") + + fs.StringVar(&s.ServicemeshPrometheusHost, "servicemesh-prometheus-host", s.ServicemeshPrometheusHost, ""+ + "prometheus service for servicemesh") +} diff --git a/pkg/simple/client/sonarqube/options.go b/pkg/simple/client/sonarqube/options.go new file mode 100644 index 000000000..bc0fc4171 --- /dev/null +++ b/pkg/simple/client/sonarqube/options.go @@ -0,0 +1,49 @@ +package sonarqube + +import ( + "github.com/spf13/pflag" +) + +type SonarQubeOptions struct { + Host string `json:",omitempty" yaml:",omitempty" description:"SonarQube service host address"` + Token string `json:",omitempty" yaml:",omitempty" description:"SonarQube service token"` +} + +func NewSonarQubeOptions() *SonarQubeOptions { + return &SonarQubeOptions{ + Host: "", + Token: "", + } +} + +func NewDefaultSonarQubeOptions() *SonarQubeOptions { + return NewSonarQubeOptions() +} + +func (s *SonarQubeOptions) Validate() []error { + errors := []error{} + + return errors +} + +func (s *SonarQubeOptions) ApplyTo(options *SonarQubeOptions) { + if options == nil { + return + } + + if s.Host != "" { + options.Host = s.Host + } + + if s.Token != "" { + options.Token = s.Token + } +} + +func (s *SonarQubeOptions) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&s.Host, "sonarqube-host", s.Host, ""+ + "Sonarqube service address if enabled.") + + fs.StringVar(&s.Token, "sonarqube-token", s.Token, ""+ + "Sonarqube service access token.") +} diff --git a/pkg/simple/client/sonarqube/sonar.go b/pkg/simple/client/sonarqube/sonar.go deleted file mode 100644 index 20664337e..000000000 --- a/pkg/simple/client/sonarqube/sonar.go +++ /dev/null @@ -1,49 +0,0 @@ -package sonarqube - -import ( - "flag" - "github.com/golang/glog" - "github.com/kubesphere/sonargo/sonar" - "strings" - "sync" -) - -var ( - sonarAddress string - sonarToken string - sonarOnce sync.Once - sonarClient *sonargo.Client -) - -func init() { - flag.StringVar(&sonarAddress, "sonar-address", "", "sonar server host") - flag.StringVar(&sonarToken, "sonar-token", "", "sonar token") -} - -func Client() *sonargo.Client { - - sonarOnce.Do(func() { - if sonarAddress == "" { - sonarClient = nil - glog.Info("skip sonar init") - return - } - if !strings.HasSuffix(sonarAddress, "/") { - sonarAddress += "/" - } - client, err := sonargo.NewClientWithToken(sonarAddress+"api/", sonarToken) - if err != nil { - glog.Error("failed to connect to sonar") - return - } - _, _, err = client.Projects.Search(nil) - if err != nil { - glog.Errorf("failed to search sonar projects [%+v]", err) - return - } - glog.Info("init sonar client success") - sonarClient = client - }) - - return sonarClient -} diff --git a/pkg/simple/client/sonarqube/sonarqube.go b/pkg/simple/client/sonarqube/sonarqube.go new file mode 100644 index 000000000..a6453fe8a --- /dev/null +++ b/pkg/simple/client/sonarqube/sonarqube.go @@ -0,0 +1,54 @@ +package sonarqube + +import ( + "fmt" + "github.com/kubesphere/sonargo/sonar" + "k8s.io/klog" + "strings" +) + +type SonarQubeClient struct { + client *sonargo.Client +} + +func NewSonarQubeClient(options *SonarQubeOptions) (*SonarQubeClient, error) { + var endpoint string + + if strings.HasSuffix(options.Host, "/") { + endpoint = fmt.Sprintf("%sapi/", options.Host) + } else { + endpoint = fmt.Sprintf("%s/api/", options.Host) + } + + sonar, err := sonargo.NewClientWithToken(endpoint, options.Token) + if err != nil { + klog.Errorf("failed to connect to sonarqube service, %+v", err) + return nil, err + } + + return &SonarQubeClient{client: sonar}, err +} + +func NewSonarQubeClientOrDie(options *SonarQubeOptions) *SonarQubeClient { + var endpoint string + + if strings.HasSuffix(options.Host, "/") { + endpoint = fmt.Sprintf("%sapi/", options.Host) + } else { + endpoint = fmt.Sprintf("%s/api/", options.Host) + } + + sonar, err := sonargo.NewClientWithToken(endpoint, options.Token) + if err != nil { + klog.Errorf("failed to connect to sonarqube service, %+v", err) + panic(err) + } + + return &SonarQubeClient{client: sonar} +} + +// return sonarqube client +// Also we can wrap some methods to avoid direct use sonar client +func (s *SonarQubeClient) SonarQube() *sonargo.Client { + return s.client +} diff --git a/pkg/tools.go b/pkg/tools.go index 2d7b8b9ab..83ed03382 100644 --- a/pkg/tools.go +++ b/pkg/tools.go @@ -3,10 +3,10 @@ package tools import ( - _ "k8s.io/kube-openapi/cmd/openapi-gen/" _ "k8s.io/code-generator/cmd/client-gen" + _ "k8s.io/code-generator/cmd/deepcopy-gen" _ "k8s.io/code-generator/cmd/informer-gen" _ "k8s.io/code-generator/cmd/lister-gen" - _ "k8s.io/code-generator/cmd/deepcopy-gen" + _ "k8s.io/kube-openapi/cmd/openapi-gen/" _ "sigs.k8s.io/controller-tools/cmd/controller-gen" ) diff --git a/pkg/utils/net/net.go b/pkg/utils/net/net.go new file mode 100644 index 000000000..e8482c412 --- /dev/null +++ b/pkg/utils/net/net.go @@ -0,0 +1,6 @@ +package net + +// 0 is considered as a non valid port +func IsValidPort(port int) bool { + return port > 0 && port < 65535 +} diff --git a/pkg/utils/reflectutils/reflect.go b/pkg/utils/reflectutils/reflect.go index 9d8e1269e..f8ba4c66b 100644 --- a/pkg/utils/reflectutils/reflect.go +++ b/pkg/utils/reflectutils/reflect.go @@ -13,7 +13,9 @@ limitations under the License. package reflectutils -import "reflect" +import ( + "reflect" +) func In(value interface{}, container interface{}) bool { containerValue := reflect.ValueOf(container) @@ -33,3 +35,24 @@ func In(value interface{}, container interface{}) bool { } return false } + +func Override(left interface{}, right interface{}) { + if left == nil || right == nil { + return + } + + if reflect.ValueOf(left).Type().Kind() != reflect.Ptr || + reflect.ValueOf(right).Type().Kind() != reflect.Ptr { + return + } + + old := reflect.ValueOf(left).Elem() + new := reflect.ValueOf(right).Elem() + + for i := 0; i < old.NumField(); i++ { + val := new.Field(i).Interface() + if !reflect.DeepEqual(val, reflect.Zero(reflect.TypeOf(val)).Interface()) { + old.Field(i).Set(reflect.ValueOf(val)) + } + } +} diff --git a/pkg/signals/signal.go b/pkg/utils/signals/signal.go similarity index 100% rename from pkg/signals/signal.go rename to pkg/utils/signals/signal.go diff --git a/pkg/signals/signal_posix.go b/pkg/utils/signals/signal_posix.go similarity index 100% rename from pkg/signals/signal_posix.go rename to pkg/utils/signals/signal_posix.go diff --git a/pkg/signals/signal_windows.go b/pkg/utils/signals/signal_windows.go similarity index 100% rename from pkg/signals/signal_windows.go rename to pkg/utils/signals/signal_windows.go diff --git a/pkg/utils/term/term.go b/pkg/utils/term/term.go new file mode 100644 index 000000000..bbe39f21e --- /dev/null +++ b/pkg/utils/term/term.go @@ -0,0 +1,22 @@ +package term + +import ( + "fmt" + "github.com/docker/docker/pkg/term" + "io" +) + +// TerminalSize returns the current width and height of the user's terminal. If it isn't a terminal, +// nil is returned. On error, zero values are returned for width and height. +// Usually w must be the stdout of the process. Stderr won't work. +func TerminalSize(w io.Writer) (int, int, error) { + outFd, isTerminal := term.GetFdInfo(w) + if !isTerminal { + return 0, 0, fmt.Errorf("given writer is no terminal") + } + winsize, err := term.GetWinsize(outFd) + if err != nil { + return 0, 0, err + } + return int(winsize.Width), int(winsize.Height), nil +} diff --git a/tools/tools.go b/tools/tools.go index 2dc7c13ae..a6d2bb85d 100644 --- a/tools/tools.go +++ b/tools/tools.go @@ -3,9 +3,9 @@ package tools import ( - _ "k8s.io/kube-openapi/cmd/openapi-gen/" _ "k8s.io/code-generator/cmd/client-gen" + _ "k8s.io/code-generator/cmd/deepcopy-gen" _ "k8s.io/code-generator/cmd/informer-gen" _ "k8s.io/code-generator/cmd/lister-gen" - _ "k8s.io/code-generator/cmd/deepcopy-gen" + _ "k8s.io/kube-openapi/cmd/openapi-gen/" ) diff --git a/vendor/github.com/fsnotify/fsnotify/.editorconfig b/vendor/github.com/fsnotify/fsnotify/.editorconfig new file mode 100644 index 000000000..ba49e3c23 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/.editorconfig @@ -0,0 +1,5 @@ +root = true + +[*] +indent_style = tab +indent_size = 4 diff --git a/vendor/github.com/fsnotify/fsnotify/.gitignore b/vendor/github.com/fsnotify/fsnotify/.gitignore new file mode 100644 index 000000000..4cd0cbaf4 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/.gitignore @@ -0,0 +1,6 @@ +# Setup a Global .gitignore for OS and editor generated files: +# https://help.github.com/articles/ignoring-files +# git config --global core.excludesfile ~/.gitignore_global + +.vagrant +*.sublime-project diff --git a/vendor/github.com/fsnotify/fsnotify/.travis.yml b/vendor/github.com/fsnotify/fsnotify/.travis.yml new file mode 100644 index 000000000..981d1bb81 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/.travis.yml @@ -0,0 +1,30 @@ +sudo: false +language: go + +go: + - 1.8.x + - 1.9.x + - tip + +matrix: + allow_failures: + - go: tip + fast_finish: true + +before_script: + - go get -u github.com/golang/lint/golint + +script: + - go test -v --race ./... + +after_script: + - test -z "$(gofmt -s -l -w . | tee /dev/stderr)" + - test -z "$(golint ./... | tee /dev/stderr)" + - go vet ./... + +os: + - linux + - osx + +notifications: + email: false diff --git a/vendor/github.com/fsnotify/fsnotify/AUTHORS b/vendor/github.com/fsnotify/fsnotify/AUTHORS new file mode 100644 index 000000000..5ab5d41c5 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/AUTHORS @@ -0,0 +1,52 @@ +# Names should be added to this file as +# Name or Organization +# The email address is not required for organizations. + +# You can update this list using the following command: +# +# $ git shortlog -se | awk '{print $2 " " $3 " " $4}' + +# Please keep the list sorted. + +Aaron L +Adrien Bustany +Amit Krishnan +Anmol Sethi +Bjørn Erik Pedersen +Bruno Bigras +Caleb Spare +Case Nelson +Chris Howey +Christoffer Buchholz +Daniel Wagner-Hall +Dave Cheney +Evan Phoenix +Francisco Souza +Hari haran +John C Barstow +Kelvin Fo +Ken-ichirou MATSUZAWA +Matt Layher +Nathan Youngman +Nickolai Zeldovich +Patrick +Paul Hammond +Pawel Knap +Pieter Droogendijk +Pursuit92 +Riku Voipio +Rob Figueiredo +Rodrigo Chiossi +Slawek Ligus +Soge Zhang +Tiffany Jernigan +Tilak Sharma +Tom Payne +Travis Cline +Tudor Golubenco +Vahe Khachikyan +Yukang +bronze1man +debrando +henrikedwards +铁哥 diff --git a/vendor/github.com/fsnotify/fsnotify/CHANGELOG.md b/vendor/github.com/fsnotify/fsnotify/CHANGELOG.md new file mode 100644 index 000000000..be4d7ea2c --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/CHANGELOG.md @@ -0,0 +1,317 @@ +# Changelog + +## v1.4.7 / 2018-01-09 + +* BSD/macOS: Fix possible deadlock on closing the watcher on kqueue (thanks @nhooyr and @glycerine) +* Tests: Fix missing verb on format string (thanks @rchiossi) +* Linux: Fix deadlock in Remove (thanks @aarondl) +* Linux: Watch.Add improvements (avoid race, fix consistency, reduce garbage) (thanks @twpayne) +* Docs: Moved FAQ into the README (thanks @vahe) +* Linux: Properly handle inotify's IN_Q_OVERFLOW event (thanks @zeldovich) +* Docs: replace references to OS X with macOS + +## v1.4.2 / 2016-10-10 + +* Linux: use InotifyInit1 with IN_CLOEXEC to stop leaking a file descriptor to a child process when using fork/exec [#178](https://github.com/fsnotify/fsnotify/pull/178) (thanks @pattyshack) + +## v1.4.1 / 2016-10-04 + +* Fix flaky inotify stress test on Linux [#177](https://github.com/fsnotify/fsnotify/pull/177) (thanks @pattyshack) + +## v1.4.0 / 2016-10-01 + +* add a String() method to Event.Op [#165](https://github.com/fsnotify/fsnotify/pull/165) (thanks @oozie) + +## v1.3.1 / 2016-06-28 + +* Windows: fix for double backslash when watching the root of a drive [#151](https://github.com/fsnotify/fsnotify/issues/151) (thanks @brunoqc) + +## v1.3.0 / 2016-04-19 + +* Support linux/arm64 by [patching](https://go-review.googlesource.com/#/c/21971/) x/sys/unix and switching to to it from syscall (thanks @suihkulokki) [#135](https://github.com/fsnotify/fsnotify/pull/135) + +## v1.2.10 / 2016-03-02 + +* Fix golint errors in windows.go [#121](https://github.com/fsnotify/fsnotify/pull/121) (thanks @tiffanyfj) + +## v1.2.9 / 2016-01-13 + +kqueue: Fix logic for CREATE after REMOVE [#111](https://github.com/fsnotify/fsnotify/pull/111) (thanks @bep) + +## v1.2.8 / 2015-12-17 + +* kqueue: fix race condition in Close [#105](https://github.com/fsnotify/fsnotify/pull/105) (thanks @djui for reporting the issue and @ppknap for writing a failing test) +* inotify: fix race in test +* enable race detection for continuous integration (Linux, Mac, Windows) + +## v1.2.5 / 2015-10-17 + +* inotify: use epoll_create1 for arm64 support (requires Linux 2.6.27 or later) [#100](https://github.com/fsnotify/fsnotify/pull/100) (thanks @suihkulokki) +* inotify: fix path leaks [#73](https://github.com/fsnotify/fsnotify/pull/73) (thanks @chamaken) +* kqueue: watch for rename events on subdirectories [#83](https://github.com/fsnotify/fsnotify/pull/83) (thanks @guotie) +* kqueue: avoid infinite loops from symlinks cycles [#101](https://github.com/fsnotify/fsnotify/pull/101) (thanks @illicitonion) + +## v1.2.1 / 2015-10-14 + +* kqueue: don't watch named pipes [#98](https://github.com/fsnotify/fsnotify/pull/98) (thanks @evanphx) + +## v1.2.0 / 2015-02-08 + +* inotify: use epoll to wake up readEvents [#66](https://github.com/fsnotify/fsnotify/pull/66) (thanks @PieterD) +* inotify: closing watcher should now always shut down goroutine [#63](https://github.com/fsnotify/fsnotify/pull/63) (thanks @PieterD) +* kqueue: close kqueue after removing watches, fixes [#59](https://github.com/fsnotify/fsnotify/issues/59) + +## v1.1.1 / 2015-02-05 + +* inotify: Retry read on EINTR [#61](https://github.com/fsnotify/fsnotify/issues/61) (thanks @PieterD) + +## v1.1.0 / 2014-12-12 + +* kqueue: rework internals [#43](https://github.com/fsnotify/fsnotify/pull/43) + * add low-level functions + * only need to store flags on directories + * less mutexes [#13](https://github.com/fsnotify/fsnotify/issues/13) + * done can be an unbuffered channel + * remove calls to os.NewSyscallError +* More efficient string concatenation for Event.String() [#52](https://github.com/fsnotify/fsnotify/pull/52) (thanks @mdlayher) +* kqueue: fix regression in rework causing subdirectories to be watched [#48](https://github.com/fsnotify/fsnotify/issues/48) +* kqueue: cleanup internal watch before sending remove event [#51](https://github.com/fsnotify/fsnotify/issues/51) + +## v1.0.4 / 2014-09-07 + +* kqueue: add dragonfly to the build tags. +* Rename source code files, rearrange code so exported APIs are at the top. +* Add done channel to example code. [#37](https://github.com/fsnotify/fsnotify/pull/37) (thanks @chenyukang) + +## v1.0.3 / 2014-08-19 + +* [Fix] Windows MOVED_TO now translates to Create like on BSD and Linux. [#36](https://github.com/fsnotify/fsnotify/issues/36) + +## v1.0.2 / 2014-08-17 + +* [Fix] Missing create events on macOS. [#14](https://github.com/fsnotify/fsnotify/issues/14) (thanks @zhsso) +* [Fix] Make ./path and path equivalent. (thanks @zhsso) + +## v1.0.0 / 2014-08-15 + +* [API] Remove AddWatch on Windows, use Add. +* Improve documentation for exported identifiers. [#30](https://github.com/fsnotify/fsnotify/issues/30) +* Minor updates based on feedback from golint. + +## dev / 2014-07-09 + +* Moved to [github.com/fsnotify/fsnotify](https://github.com/fsnotify/fsnotify). +* Use os.NewSyscallError instead of returning errno (thanks @hariharan-uno) + +## dev / 2014-07-04 + +* kqueue: fix incorrect mutex used in Close() +* Update example to demonstrate usage of Op. + +## dev / 2014-06-28 + +* [API] Don't set the Write Op for attribute notifications [#4](https://github.com/fsnotify/fsnotify/issues/4) +* Fix for String() method on Event (thanks Alex Brainman) +* Don't build on Plan 9 or Solaris (thanks @4ad) + +## dev / 2014-06-21 + +* Events channel of type Event rather than *Event. +* [internal] use syscall constants directly for inotify and kqueue. +* [internal] kqueue: rename events to kevents and fileEvent to event. + +## dev / 2014-06-19 + +* Go 1.3+ required on Windows (uses syscall.ERROR_MORE_DATA internally). +* [internal] remove cookie from Event struct (unused). +* [internal] Event struct has the same definition across every OS. +* [internal] remove internal watch and removeWatch methods. + +## dev / 2014-06-12 + +* [API] Renamed Watch() to Add() and RemoveWatch() to Remove(). +* [API] Pluralized channel names: Events and Errors. +* [API] Renamed FileEvent struct to Event. +* [API] Op constants replace methods like IsCreate(). + +## dev / 2014-06-12 + +* Fix data race on kevent buffer (thanks @tilaks) [#98](https://github.com/howeyc/fsnotify/pull/98) + +## dev / 2014-05-23 + +* [API] Remove current implementation of WatchFlags. + * current implementation doesn't take advantage of OS for efficiency + * provides little benefit over filtering events as they are received, but has extra bookkeeping and mutexes + * no tests for the current implementation + * not fully implemented on Windows [#93](https://github.com/howeyc/fsnotify/issues/93#issuecomment-39285195) + +## v0.9.3 / 2014-12-31 + +* kqueue: cleanup internal watch before sending remove event [#51](https://github.com/fsnotify/fsnotify/issues/51) + +## v0.9.2 / 2014-08-17 + +* [Backport] Fix missing create events on macOS. [#14](https://github.com/fsnotify/fsnotify/issues/14) (thanks @zhsso) + +## v0.9.1 / 2014-06-12 + +* Fix data race on kevent buffer (thanks @tilaks) [#98](https://github.com/howeyc/fsnotify/pull/98) + +## v0.9.0 / 2014-01-17 + +* IsAttrib() for events that only concern a file's metadata [#79][] (thanks @abustany) +* [Fix] kqueue: fix deadlock [#77][] (thanks @cespare) +* [NOTICE] Development has moved to `code.google.com/p/go.exp/fsnotify` in preparation for inclusion in the Go standard library. + +## v0.8.12 / 2013-11-13 + +* [API] Remove FD_SET and friends from Linux adapter + +## v0.8.11 / 2013-11-02 + +* [Doc] Add Changelog [#72][] (thanks @nathany) +* [Doc] Spotlight and double modify events on macOS [#62][] (reported by @paulhammond) + +## v0.8.10 / 2013-10-19 + +* [Fix] kqueue: remove file watches when parent directory is removed [#71][] (reported by @mdwhatcott) +* [Fix] kqueue: race between Close and readEvents [#70][] (reported by @bernerdschaefer) +* [Doc] specify OS-specific limits in README (thanks @debrando) + +## v0.8.9 / 2013-09-08 + +* [Doc] Contributing (thanks @nathany) +* [Doc] update package path in example code [#63][] (thanks @paulhammond) +* [Doc] GoCI badge in README (Linux only) [#60][] +* [Doc] Cross-platform testing with Vagrant [#59][] (thanks @nathany) + +## v0.8.8 / 2013-06-17 + +* [Fix] Windows: handle `ERROR_MORE_DATA` on Windows [#49][] (thanks @jbowtie) + +## v0.8.7 / 2013-06-03 + +* [API] Make syscall flags internal +* [Fix] inotify: ignore event changes +* [Fix] race in symlink test [#45][] (reported by @srid) +* [Fix] tests on Windows +* lower case error messages + +## v0.8.6 / 2013-05-23 + +* kqueue: Use EVT_ONLY flag on Darwin +* [Doc] Update README with full example + +## v0.8.5 / 2013-05-09 + +* [Fix] inotify: allow monitoring of "broken" symlinks (thanks @tsg) + +## v0.8.4 / 2013-04-07 + +* [Fix] kqueue: watch all file events [#40][] (thanks @ChrisBuchholz) + +## v0.8.3 / 2013-03-13 + +* [Fix] inoitfy/kqueue memory leak [#36][] (reported by @nbkolchin) +* [Fix] kqueue: use fsnFlags for watching a directory [#33][] (reported by @nbkolchin) + +## v0.8.2 / 2013-02-07 + +* [Doc] add Authors +* [Fix] fix data races for map access [#29][] (thanks @fsouza) + +## v0.8.1 / 2013-01-09 + +* [Fix] Windows path separators +* [Doc] BSD License + +## v0.8.0 / 2012-11-09 + +* kqueue: directory watching improvements (thanks @vmirage) +* inotify: add `IN_MOVED_TO` [#25][] (requested by @cpisto) +* [Fix] kqueue: deleting watched directory [#24][] (reported by @jakerr) + +## v0.7.4 / 2012-10-09 + +* [Fix] inotify: fixes from https://codereview.appspot.com/5418045/ (ugorji) +* [Fix] kqueue: preserve watch flags when watching for delete [#21][] (reported by @robfig) +* [Fix] kqueue: watch the directory even if it isn't a new watch (thanks @robfig) +* [Fix] kqueue: modify after recreation of file + +## v0.7.3 / 2012-09-27 + +* [Fix] kqueue: watch with an existing folder inside the watched folder (thanks @vmirage) +* [Fix] kqueue: no longer get duplicate CREATE events + +## v0.7.2 / 2012-09-01 + +* kqueue: events for created directories + +## v0.7.1 / 2012-07-14 + +* [Fix] for renaming files + +## v0.7.0 / 2012-07-02 + +* [Feature] FSNotify flags +* [Fix] inotify: Added file name back to event path + +## v0.6.0 / 2012-06-06 + +* kqueue: watch files after directory created (thanks @tmc) + +## v0.5.1 / 2012-05-22 + +* [Fix] inotify: remove all watches before Close() + +## v0.5.0 / 2012-05-03 + +* [API] kqueue: return errors during watch instead of sending over channel +* kqueue: match symlink behavior on Linux +* inotify: add `DELETE_SELF` (requested by @taralx) +* [Fix] kqueue: handle EINTR (reported by @robfig) +* [Doc] Godoc example [#1][] (thanks @davecheney) + +## v0.4.0 / 2012-03-30 + +* Go 1 released: build with go tool +* [Feature] Windows support using winfsnotify +* Windows does not have attribute change notifications +* Roll attribute notifications into IsModify + +## v0.3.0 / 2012-02-19 + +* kqueue: add files when watch directory + +## v0.2.0 / 2011-12-30 + +* update to latest Go weekly code + +## v0.1.0 / 2011-10-19 + +* kqueue: add watch on file creation to match inotify +* kqueue: create file event +* inotify: ignore `IN_IGNORED` events +* event String() +* linux: common FileEvent functions +* initial commit + +[#79]: https://github.com/howeyc/fsnotify/pull/79 +[#77]: https://github.com/howeyc/fsnotify/pull/77 +[#72]: https://github.com/howeyc/fsnotify/issues/72 +[#71]: https://github.com/howeyc/fsnotify/issues/71 +[#70]: https://github.com/howeyc/fsnotify/issues/70 +[#63]: https://github.com/howeyc/fsnotify/issues/63 +[#62]: https://github.com/howeyc/fsnotify/issues/62 +[#60]: https://github.com/howeyc/fsnotify/issues/60 +[#59]: https://github.com/howeyc/fsnotify/issues/59 +[#49]: https://github.com/howeyc/fsnotify/issues/49 +[#45]: https://github.com/howeyc/fsnotify/issues/45 +[#40]: https://github.com/howeyc/fsnotify/issues/40 +[#36]: https://github.com/howeyc/fsnotify/issues/36 +[#33]: https://github.com/howeyc/fsnotify/issues/33 +[#29]: https://github.com/howeyc/fsnotify/issues/29 +[#25]: https://github.com/howeyc/fsnotify/issues/25 +[#24]: https://github.com/howeyc/fsnotify/issues/24 +[#21]: https://github.com/howeyc/fsnotify/issues/21 diff --git a/vendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md b/vendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md new file mode 100644 index 000000000..828a60b24 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md @@ -0,0 +1,77 @@ +# Contributing + +## Issues + +* Request features and report bugs using the [GitHub Issue Tracker](https://github.com/fsnotify/fsnotify/issues). +* Please indicate the platform you are using fsnotify on. +* A code example to reproduce the problem is appreciated. + +## Pull Requests + +### Contributor License Agreement + +fsnotify is derived from code in the [golang.org/x/exp](https://godoc.org/golang.org/x/exp) package and it may be included [in the standard library](https://github.com/fsnotify/fsnotify/issues/1) in the future. Therefore fsnotify carries the same [LICENSE](https://github.com/fsnotify/fsnotify/blob/master/LICENSE) as Go. Contributors retain their copyright, so you need to fill out a short form before we can accept your contribution: [Google Individual Contributor License Agreement](https://developers.google.com/open-source/cla/individual). + +Please indicate that you have signed the CLA in your pull request. + +### How fsnotify is Developed + +* Development is done on feature branches. +* Tests are run on BSD, Linux, macOS and Windows. +* Pull requests are reviewed and [applied to master][am] using [hub][]. + * Maintainers may modify or squash commits rather than asking contributors to. +* To issue a new release, the maintainers will: + * Update the CHANGELOG + * Tag a version, which will become available through gopkg.in. + +### How to Fork + +For smooth sailing, always use the original import path. Installing with `go get` makes this easy. + +1. Install from GitHub (`go get -u github.com/fsnotify/fsnotify`) +2. Create your feature branch (`git checkout -b my-new-feature`) +3. Ensure everything works and the tests pass (see below) +4. Commit your changes (`git commit -am 'Add some feature'`) + +Contribute upstream: + +1. Fork fsnotify on GitHub +2. Add your remote (`git remote add fork git@github.com:mycompany/repo.git`) +3. Push to the branch (`git push fork my-new-feature`) +4. Create a new Pull Request on GitHub + +This workflow is [thoroughly explained by Katrina Owen](https://splice.com/blog/contributing-open-source-git-repositories-go/). + +### Testing + +fsnotify uses build tags to compile different code on Linux, BSD, macOS, and Windows. + +Before doing a pull request, please do your best to test your changes on multiple platforms, and list which platforms you were able/unable to test on. + +To aid in cross-platform testing there is a Vagrantfile for Linux and BSD. + +* Install [Vagrant](http://www.vagrantup.com/) and [VirtualBox](https://www.virtualbox.org/) +* Setup [Vagrant Gopher](https://github.com/nathany/vagrant-gopher) in your `src` folder. +* Run `vagrant up` from the project folder. You can also setup just one box with `vagrant up linux` or `vagrant up bsd` (note: the BSD box doesn't support Windows hosts at this time, and NFS may prompt for your host OS password) +* Once setup, you can run the test suite on a given OS with a single command `vagrant ssh linux -c 'cd fsnotify/fsnotify; go test'`. +* When you're done, you will want to halt or destroy the Vagrant boxes. + +Notice: fsnotify file system events won't trigger in shared folders. The tests get around this limitation by using the /tmp directory. + +Right now there is no equivalent solution for Windows and macOS, but there are Windows VMs [freely available from Microsoft](http://www.modern.ie/en-us/virtualization-tools#downloads). + +### Maintainers + +Help maintaining fsnotify is welcome. To be a maintainer: + +* Submit a pull request and sign the CLA as above. +* You must be able to run the test suite on Mac, Windows, Linux and BSD. + +To keep master clean, the fsnotify project uses the "apply mail" workflow outlined in Nathaniel Talbott's post ["Merge pull request" Considered Harmful][am]. This requires installing [hub][]. + +All code changes should be internal pull requests. + +Releases are tagged using [Semantic Versioning](http://semver.org/). + +[hub]: https://github.com/github/hub +[am]: http://blog.spreedly.com/2014/06/24/merge-pull-request-considered-harmful/#.VGa5yZPF_Zs diff --git a/vendor/github.com/fsnotify/fsnotify/LICENSE b/vendor/github.com/fsnotify/fsnotify/LICENSE new file mode 100644 index 000000000..f21e54080 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/LICENSE @@ -0,0 +1,28 @@ +Copyright (c) 2012 The Go Authors. All rights reserved. +Copyright (c) 2012 fsnotify Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/fsnotify/fsnotify/README.md b/vendor/github.com/fsnotify/fsnotify/README.md new file mode 100644 index 000000000..399320741 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/README.md @@ -0,0 +1,79 @@ +# File system notifications for Go + +[![GoDoc](https://godoc.org/github.com/fsnotify/fsnotify?status.svg)](https://godoc.org/github.com/fsnotify/fsnotify) [![Go Report Card](https://goreportcard.com/badge/github.com/fsnotify/fsnotify)](https://goreportcard.com/report/github.com/fsnotify/fsnotify) + +fsnotify utilizes [golang.org/x/sys](https://godoc.org/golang.org/x/sys) rather than `syscall` from the standard library. Ensure you have the latest version installed by running: + +```console +go get -u golang.org/x/sys/... +``` + +Cross platform: Windows, Linux, BSD and macOS. + +|Adapter |OS |Status | +|----------|----------|----------| +|inotify |Linux 2.6.27 or later, Android\*|Supported [![Build Status](https://travis-ci.org/fsnotify/fsnotify.svg?branch=master)](https://travis-ci.org/fsnotify/fsnotify)| +|kqueue |BSD, macOS, iOS\*|Supported [![Build Status](https://travis-ci.org/fsnotify/fsnotify.svg?branch=master)](https://travis-ci.org/fsnotify/fsnotify)| +|ReadDirectoryChangesW|Windows|Supported [![Build status](https://ci.appveyor.com/api/projects/status/ivwjubaih4r0udeh/branch/master?svg=true)](https://ci.appveyor.com/project/NathanYoungman/fsnotify/branch/master)| +|FSEvents |macOS |[Planned](https://github.com/fsnotify/fsnotify/issues/11)| +|FEN |Solaris 11 |[In Progress](https://github.com/fsnotify/fsnotify/issues/12)| +|fanotify |Linux 2.6.37+ | | +|USN Journals |Windows |[Maybe](https://github.com/fsnotify/fsnotify/issues/53)| +|Polling |*All* |[Maybe](https://github.com/fsnotify/fsnotify/issues/9)| + +\* Android and iOS are untested. + +Please see [the documentation](https://godoc.org/github.com/fsnotify/fsnotify) and consult the [FAQ](#faq) for usage information. + +## API stability + +fsnotify is a fork of [howeyc/fsnotify](https://godoc.org/github.com/howeyc/fsnotify) with a new API as of v1.0. The API is based on [this design document](http://goo.gl/MrYxyA). + +All [releases](https://github.com/fsnotify/fsnotify/releases) are tagged based on [Semantic Versioning](http://semver.org/). Further API changes are [planned](https://github.com/fsnotify/fsnotify/milestones), and will be tagged with a new major revision number. + +Go 1.6 supports dependencies located in the `vendor/` folder. Unless you are creating a library, it is recommended that you copy fsnotify into `vendor/github.com/fsnotify/fsnotify` within your project, and likewise for `golang.org/x/sys`. + +## Contributing + +Please refer to [CONTRIBUTING][] before opening an issue or pull request. + +## Example + +See [example_test.go](https://github.com/fsnotify/fsnotify/blob/master/example_test.go). + +## FAQ + +**When a file is moved to another directory is it still being watched?** + +No (it shouldn't be, unless you are watching where it was moved to). + +**When I watch a directory, are all subdirectories watched as well?** + +No, you must add watches for any directory you want to watch (a recursive watcher is on the roadmap [#18][]). + +**Do I have to watch the Error and Event channels in a separate goroutine?** + +As of now, yes. Looking into making this single-thread friendly (see [howeyc #7][#7]) + +**Why am I receiving multiple events for the same file on OS X?** + +Spotlight indexing on OS X can result in multiple events (see [howeyc #62][#62]). A temporary workaround is to add your folder(s) to the *Spotlight Privacy settings* until we have a native FSEvents implementation (see [#11][]). + +**How many files can be watched at once?** + +There are OS-specific limits as to how many watches can be created: +* Linux: /proc/sys/fs/inotify/max_user_watches contains the limit, reaching this limit results in a "no space left on device" error. +* BSD / OSX: sysctl variables "kern.maxfiles" and "kern.maxfilesperproc", reaching these limits results in a "too many open files" error. + +[#62]: https://github.com/howeyc/fsnotify/issues/62 +[#18]: https://github.com/fsnotify/fsnotify/issues/18 +[#11]: https://github.com/fsnotify/fsnotify/issues/11 +[#7]: https://github.com/howeyc/fsnotify/issues/7 + +[contributing]: https://github.com/fsnotify/fsnotify/blob/master/CONTRIBUTING.md + +## Related Projects + +* [notify](https://github.com/rjeczalik/notify) +* [fsevents](https://github.com/fsnotify/fsevents) + diff --git a/vendor/github.com/fsnotify/fsnotify/fen.go b/vendor/github.com/fsnotify/fsnotify/fen.go new file mode 100644 index 000000000..ced39cb88 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/fen.go @@ -0,0 +1,37 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build solaris + +package fsnotify + +import ( + "errors" +) + +// Watcher watches a set of files, delivering events to a channel. +type Watcher struct { + Events chan Event + Errors chan error +} + +// NewWatcher establishes a new watcher with the underlying OS and begins waiting for events. +func NewWatcher() (*Watcher, error) { + return nil, errors.New("FEN based watcher not yet supported for fsnotify\n") +} + +// Close removes all watches and closes the events channel. +func (w *Watcher) Close() error { + return nil +} + +// Add starts watching the named file or directory (non-recursively). +func (w *Watcher) Add(name string) error { + return nil +} + +// Remove stops watching the the named file or directory (non-recursively). +func (w *Watcher) Remove(name string) error { + return nil +} diff --git a/vendor/github.com/fsnotify/fsnotify/fsnotify.go b/vendor/github.com/fsnotify/fsnotify/fsnotify.go new file mode 100644 index 000000000..190bf0de5 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/fsnotify.go @@ -0,0 +1,66 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !plan9 + +// Package fsnotify provides a platform-independent interface for file system notifications. +package fsnotify + +import ( + "bytes" + "errors" + "fmt" +) + +// Event represents a single file system notification. +type Event struct { + Name string // Relative path to the file or directory. + Op Op // File operation that triggered the event. +} + +// Op describes a set of file operations. +type Op uint32 + +// These are the generalized file operations that can trigger a notification. +const ( + Create Op = 1 << iota + Write + Remove + Rename + Chmod +) + +func (op Op) String() string { + // Use a buffer for efficient string concatenation + var buffer bytes.Buffer + + if op&Create == Create { + buffer.WriteString("|CREATE") + } + if op&Remove == Remove { + buffer.WriteString("|REMOVE") + } + if op&Write == Write { + buffer.WriteString("|WRITE") + } + if op&Rename == Rename { + buffer.WriteString("|RENAME") + } + if op&Chmod == Chmod { + buffer.WriteString("|CHMOD") + } + if buffer.Len() == 0 { + return "" + } + return buffer.String()[1:] // Strip leading pipe +} + +// String returns a string representation of the event in the form +// "file: REMOVE|WRITE|..." +func (e Event) String() string { + return fmt.Sprintf("%q: %s", e.Name, e.Op.String()) +} + +// Common errors that can be reported by a watcher +var ErrEventOverflow = errors.New("fsnotify queue overflow") diff --git a/vendor/github.com/fsnotify/fsnotify/inotify.go b/vendor/github.com/fsnotify/fsnotify/inotify.go new file mode 100644 index 000000000..d9fd1b88a --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/inotify.go @@ -0,0 +1,337 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux + +package fsnotify + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + "unsafe" + + "golang.org/x/sys/unix" +) + +// Watcher watches a set of files, delivering events to a channel. +type Watcher struct { + Events chan Event + Errors chan error + mu sync.Mutex // Map access + fd int + poller *fdPoller + watches map[string]*watch // Map of inotify watches (key: path) + paths map[int]string // Map of watched paths (key: watch descriptor) + done chan struct{} // Channel for sending a "quit message" to the reader goroutine + doneResp chan struct{} // Channel to respond to Close +} + +// NewWatcher establishes a new watcher with the underlying OS and begins waiting for events. +func NewWatcher() (*Watcher, error) { + // Create inotify fd + fd, errno := unix.InotifyInit1(unix.IN_CLOEXEC) + if fd == -1 { + return nil, errno + } + // Create epoll + poller, err := newFdPoller(fd) + if err != nil { + unix.Close(fd) + return nil, err + } + w := &Watcher{ + fd: fd, + poller: poller, + watches: make(map[string]*watch), + paths: make(map[int]string), + Events: make(chan Event), + Errors: make(chan error), + done: make(chan struct{}), + doneResp: make(chan struct{}), + } + + go w.readEvents() + return w, nil +} + +func (w *Watcher) isClosed() bool { + select { + case <-w.done: + return true + default: + return false + } +} + +// Close removes all watches and closes the events channel. +func (w *Watcher) Close() error { + if w.isClosed() { + return nil + } + + // Send 'close' signal to goroutine, and set the Watcher to closed. + close(w.done) + + // Wake up goroutine + w.poller.wake() + + // Wait for goroutine to close + <-w.doneResp + + return nil +} + +// Add starts watching the named file or directory (non-recursively). +func (w *Watcher) Add(name string) error { + name = filepath.Clean(name) + if w.isClosed() { + return errors.New("inotify instance already closed") + } + + const agnosticEvents = unix.IN_MOVED_TO | unix.IN_MOVED_FROM | + unix.IN_CREATE | unix.IN_ATTRIB | unix.IN_MODIFY | + unix.IN_MOVE_SELF | unix.IN_DELETE | unix.IN_DELETE_SELF + + var flags uint32 = agnosticEvents + + w.mu.Lock() + defer w.mu.Unlock() + watchEntry := w.watches[name] + if watchEntry != nil { + flags |= watchEntry.flags | unix.IN_MASK_ADD + } + wd, errno := unix.InotifyAddWatch(w.fd, name, flags) + if wd == -1 { + return errno + } + + if watchEntry == nil { + w.watches[name] = &watch{wd: uint32(wd), flags: flags} + w.paths[wd] = name + } else { + watchEntry.wd = uint32(wd) + watchEntry.flags = flags + } + + return nil +} + +// Remove stops watching the named file or directory (non-recursively). +func (w *Watcher) Remove(name string) error { + name = filepath.Clean(name) + + // Fetch the watch. + w.mu.Lock() + defer w.mu.Unlock() + watch, ok := w.watches[name] + + // Remove it from inotify. + if !ok { + return fmt.Errorf("can't remove non-existent inotify watch for: %s", name) + } + + // We successfully removed the watch if InotifyRmWatch doesn't return an + // error, we need to clean up our internal state to ensure it matches + // inotify's kernel state. + delete(w.paths, int(watch.wd)) + delete(w.watches, name) + + // inotify_rm_watch will return EINVAL if the file has been deleted; + // the inotify will already have been removed. + // watches and pathes are deleted in ignoreLinux() implicitly and asynchronously + // by calling inotify_rm_watch() below. e.g. readEvents() goroutine receives IN_IGNORE + // so that EINVAL means that the wd is being rm_watch()ed or its file removed + // by another thread and we have not received IN_IGNORE event. + success, errno := unix.InotifyRmWatch(w.fd, watch.wd) + if success == -1 { + // TODO: Perhaps it's not helpful to return an error here in every case. + // the only two possible errors are: + // EBADF, which happens when w.fd is not a valid file descriptor of any kind. + // EINVAL, which is when fd is not an inotify descriptor or wd is not a valid watch descriptor. + // Watch descriptors are invalidated when they are removed explicitly or implicitly; + // explicitly by inotify_rm_watch, implicitly when the file they are watching is deleted. + return errno + } + + return nil +} + +type watch struct { + wd uint32 // Watch descriptor (as returned by the inotify_add_watch() syscall) + flags uint32 // inotify flags of this watch (see inotify(7) for the list of valid flags) +} + +// readEvents reads from the inotify file descriptor, converts the +// received events into Event objects and sends them via the Events channel +func (w *Watcher) readEvents() { + var ( + buf [unix.SizeofInotifyEvent * 4096]byte // Buffer for a maximum of 4096 raw events + n int // Number of bytes read with read() + errno error // Syscall errno + ok bool // For poller.wait + ) + + defer close(w.doneResp) + defer close(w.Errors) + defer close(w.Events) + defer unix.Close(w.fd) + defer w.poller.close() + + for { + // See if we have been closed. + if w.isClosed() { + return + } + + ok, errno = w.poller.wait() + if errno != nil { + select { + case w.Errors <- errno: + case <-w.done: + return + } + continue + } + + if !ok { + continue + } + + n, errno = unix.Read(w.fd, buf[:]) + // If a signal interrupted execution, see if we've been asked to close, and try again. + // http://man7.org/linux/man-pages/man7/signal.7.html : + // "Before Linux 3.8, reads from an inotify(7) file descriptor were not restartable" + if errno == unix.EINTR { + continue + } + + // unix.Read might have been woken up by Close. If so, we're done. + if w.isClosed() { + return + } + + if n < unix.SizeofInotifyEvent { + var err error + if n == 0 { + // If EOF is received. This should really never happen. + err = io.EOF + } else if n < 0 { + // If an error occurred while reading. + err = errno + } else { + // Read was too short. + err = errors.New("notify: short read in readEvents()") + } + select { + case w.Errors <- err: + case <-w.done: + return + } + continue + } + + var offset uint32 + // We don't know how many events we just read into the buffer + // While the offset points to at least one whole event... + for offset <= uint32(n-unix.SizeofInotifyEvent) { + // Point "raw" to the event in the buffer + raw := (*unix.InotifyEvent)(unsafe.Pointer(&buf[offset])) + + mask := uint32(raw.Mask) + nameLen := uint32(raw.Len) + + if mask&unix.IN_Q_OVERFLOW != 0 { + select { + case w.Errors <- ErrEventOverflow: + case <-w.done: + return + } + } + + // If the event happened to the watched directory or the watched file, the kernel + // doesn't append the filename to the event, but we would like to always fill the + // the "Name" field with a valid filename. We retrieve the path of the watch from + // the "paths" map. + w.mu.Lock() + name, ok := w.paths[int(raw.Wd)] + // IN_DELETE_SELF occurs when the file/directory being watched is removed. + // This is a sign to clean up the maps, otherwise we are no longer in sync + // with the inotify kernel state which has already deleted the watch + // automatically. + if ok && mask&unix.IN_DELETE_SELF == unix.IN_DELETE_SELF { + delete(w.paths, int(raw.Wd)) + delete(w.watches, name) + } + w.mu.Unlock() + + if nameLen > 0 { + // Point "bytes" at the first byte of the filename + bytes := (*[unix.PathMax]byte)(unsafe.Pointer(&buf[offset+unix.SizeofInotifyEvent])) + // The filename is padded with NULL bytes. TrimRight() gets rid of those. + name += "/" + strings.TrimRight(string(bytes[0:nameLen]), "\000") + } + + event := newEvent(name, mask) + + // Send the events that are not ignored on the events channel + if !event.ignoreLinux(mask) { + select { + case w.Events <- event: + case <-w.done: + return + } + } + + // Move to the next event in the buffer + offset += unix.SizeofInotifyEvent + nameLen + } + } +} + +// Certain types of events can be "ignored" and not sent over the Events +// channel. Such as events marked ignore by the kernel, or MODIFY events +// against files that do not exist. +func (e *Event) ignoreLinux(mask uint32) bool { + // Ignore anything the inotify API says to ignore + if mask&unix.IN_IGNORED == unix.IN_IGNORED { + return true + } + + // If the event is not a DELETE or RENAME, the file must exist. + // Otherwise the event is ignored. + // *Note*: this was put in place because it was seen that a MODIFY + // event was sent after the DELETE. This ignores that MODIFY and + // assumes a DELETE will come or has come if the file doesn't exist. + if !(e.Op&Remove == Remove || e.Op&Rename == Rename) { + _, statErr := os.Lstat(e.Name) + return os.IsNotExist(statErr) + } + return false +} + +// newEvent returns an platform-independent Event based on an inotify mask. +func newEvent(name string, mask uint32) Event { + e := Event{Name: name} + if mask&unix.IN_CREATE == unix.IN_CREATE || mask&unix.IN_MOVED_TO == unix.IN_MOVED_TO { + e.Op |= Create + } + if mask&unix.IN_DELETE_SELF == unix.IN_DELETE_SELF || mask&unix.IN_DELETE == unix.IN_DELETE { + e.Op |= Remove + } + if mask&unix.IN_MODIFY == unix.IN_MODIFY { + e.Op |= Write + } + if mask&unix.IN_MOVE_SELF == unix.IN_MOVE_SELF || mask&unix.IN_MOVED_FROM == unix.IN_MOVED_FROM { + e.Op |= Rename + } + if mask&unix.IN_ATTRIB == unix.IN_ATTRIB { + e.Op |= Chmod + } + return e +} diff --git a/vendor/github.com/fsnotify/fsnotify/inotify_poller.go b/vendor/github.com/fsnotify/fsnotify/inotify_poller.go new file mode 100644 index 000000000..cc7db4b22 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/inotify_poller.go @@ -0,0 +1,187 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux + +package fsnotify + +import ( + "errors" + + "golang.org/x/sys/unix" +) + +type fdPoller struct { + fd int // File descriptor (as returned by the inotify_init() syscall) + epfd int // Epoll file descriptor + pipe [2]int // Pipe for waking up +} + +func emptyPoller(fd int) *fdPoller { + poller := new(fdPoller) + poller.fd = fd + poller.epfd = -1 + poller.pipe[0] = -1 + poller.pipe[1] = -1 + return poller +} + +// Create a new inotify poller. +// This creates an inotify handler, and an epoll handler. +func newFdPoller(fd int) (*fdPoller, error) { + var errno error + poller := emptyPoller(fd) + defer func() { + if errno != nil { + poller.close() + } + }() + poller.fd = fd + + // Create epoll fd + poller.epfd, errno = unix.EpollCreate1(0) + if poller.epfd == -1 { + return nil, errno + } + // Create pipe; pipe[0] is the read end, pipe[1] the write end. + errno = unix.Pipe2(poller.pipe[:], unix.O_NONBLOCK) + if errno != nil { + return nil, errno + } + + // Register inotify fd with epoll + event := unix.EpollEvent{ + Fd: int32(poller.fd), + Events: unix.EPOLLIN, + } + errno = unix.EpollCtl(poller.epfd, unix.EPOLL_CTL_ADD, poller.fd, &event) + if errno != nil { + return nil, errno + } + + // Register pipe fd with epoll + event = unix.EpollEvent{ + Fd: int32(poller.pipe[0]), + Events: unix.EPOLLIN, + } + errno = unix.EpollCtl(poller.epfd, unix.EPOLL_CTL_ADD, poller.pipe[0], &event) + if errno != nil { + return nil, errno + } + + return poller, nil +} + +// Wait using epoll. +// Returns true if something is ready to be read, +// false if there is not. +func (poller *fdPoller) wait() (bool, error) { + // 3 possible events per fd, and 2 fds, makes a maximum of 6 events. + // I don't know whether epoll_wait returns the number of events returned, + // or the total number of events ready. + // I decided to catch both by making the buffer one larger than the maximum. + events := make([]unix.EpollEvent, 7) + for { + n, errno := unix.EpollWait(poller.epfd, events, -1) + if n == -1 { + if errno == unix.EINTR { + continue + } + return false, errno + } + if n == 0 { + // If there are no events, try again. + continue + } + if n > 6 { + // This should never happen. More events were returned than should be possible. + return false, errors.New("epoll_wait returned more events than I know what to do with") + } + ready := events[:n] + epollhup := false + epollerr := false + epollin := false + for _, event := range ready { + if event.Fd == int32(poller.fd) { + if event.Events&unix.EPOLLHUP != 0 { + // This should not happen, but if it does, treat it as a wakeup. + epollhup = true + } + if event.Events&unix.EPOLLERR != 0 { + // If an error is waiting on the file descriptor, we should pretend + // something is ready to read, and let unix.Read pick up the error. + epollerr = true + } + if event.Events&unix.EPOLLIN != 0 { + // There is data to read. + epollin = true + } + } + if event.Fd == int32(poller.pipe[0]) { + if event.Events&unix.EPOLLHUP != 0 { + // Write pipe descriptor was closed, by us. This means we're closing down the + // watcher, and we should wake up. + } + if event.Events&unix.EPOLLERR != 0 { + // If an error is waiting on the pipe file descriptor. + // This is an absolute mystery, and should never ever happen. + return false, errors.New("Error on the pipe descriptor.") + } + if event.Events&unix.EPOLLIN != 0 { + // This is a regular wakeup, so we have to clear the buffer. + err := poller.clearWake() + if err != nil { + return false, err + } + } + } + } + + if epollhup || epollerr || epollin { + return true, nil + } + return false, nil + } +} + +// Close the write end of the poller. +func (poller *fdPoller) wake() error { + buf := make([]byte, 1) + n, errno := unix.Write(poller.pipe[1], buf) + if n == -1 { + if errno == unix.EAGAIN { + // Buffer is full, poller will wake. + return nil + } + return errno + } + return nil +} + +func (poller *fdPoller) clearWake() error { + // You have to be woken up a LOT in order to get to 100! + buf := make([]byte, 100) + n, errno := unix.Read(poller.pipe[0], buf) + if n == -1 { + if errno == unix.EAGAIN { + // Buffer is empty, someone else cleared our wake. + return nil + } + return errno + } + return nil +} + +// Close all poller file descriptors, but not the one passed to it. +func (poller *fdPoller) close() { + if poller.pipe[1] != -1 { + unix.Close(poller.pipe[1]) + } + if poller.pipe[0] != -1 { + unix.Close(poller.pipe[0]) + } + if poller.epfd != -1 { + unix.Close(poller.epfd) + } +} diff --git a/vendor/github.com/fsnotify/fsnotify/kqueue.go b/vendor/github.com/fsnotify/fsnotify/kqueue.go new file mode 100644 index 000000000..86e76a3d6 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/kqueue.go @@ -0,0 +1,521 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build freebsd openbsd netbsd dragonfly darwin + +package fsnotify + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "sync" + "time" + + "golang.org/x/sys/unix" +) + +// Watcher watches a set of files, delivering events to a channel. +type Watcher struct { + Events chan Event + Errors chan error + done chan struct{} // Channel for sending a "quit message" to the reader goroutine + + kq int // File descriptor (as returned by the kqueue() syscall). + + mu sync.Mutex // Protects access to watcher data + watches map[string]int // Map of watched file descriptors (key: path). + externalWatches map[string]bool // Map of watches added by user of the library. + dirFlags map[string]uint32 // Map of watched directories to fflags used in kqueue. + paths map[int]pathInfo // Map file descriptors to path names for processing kqueue events. + fileExists map[string]bool // Keep track of if we know this file exists (to stop duplicate create events). + isClosed bool // Set to true when Close() is first called +} + +type pathInfo struct { + name string + isDir bool +} + +// NewWatcher establishes a new watcher with the underlying OS and begins waiting for events. +func NewWatcher() (*Watcher, error) { + kq, err := kqueue() + if err != nil { + return nil, err + } + + w := &Watcher{ + kq: kq, + watches: make(map[string]int), + dirFlags: make(map[string]uint32), + paths: make(map[int]pathInfo), + fileExists: make(map[string]bool), + externalWatches: make(map[string]bool), + Events: make(chan Event), + Errors: make(chan error), + done: make(chan struct{}), + } + + go w.readEvents() + return w, nil +} + +// Close removes all watches and closes the events channel. +func (w *Watcher) Close() error { + w.mu.Lock() + if w.isClosed { + w.mu.Unlock() + return nil + } + w.isClosed = true + + // copy paths to remove while locked + var pathsToRemove = make([]string, 0, len(w.watches)) + for name := range w.watches { + pathsToRemove = append(pathsToRemove, name) + } + w.mu.Unlock() + // unlock before calling Remove, which also locks + + for _, name := range pathsToRemove { + w.Remove(name) + } + + // send a "quit" message to the reader goroutine + close(w.done) + + return nil +} + +// Add starts watching the named file or directory (non-recursively). +func (w *Watcher) Add(name string) error { + w.mu.Lock() + w.externalWatches[name] = true + w.mu.Unlock() + _, err := w.addWatch(name, noteAllEvents) + return err +} + +// Remove stops watching the the named file or directory (non-recursively). +func (w *Watcher) Remove(name string) error { + name = filepath.Clean(name) + w.mu.Lock() + watchfd, ok := w.watches[name] + w.mu.Unlock() + if !ok { + return fmt.Errorf("can't remove non-existent kevent watch for: %s", name) + } + + const registerRemove = unix.EV_DELETE + if err := register(w.kq, []int{watchfd}, registerRemove, 0); err != nil { + return err + } + + unix.Close(watchfd) + + w.mu.Lock() + isDir := w.paths[watchfd].isDir + delete(w.watches, name) + delete(w.paths, watchfd) + delete(w.dirFlags, name) + w.mu.Unlock() + + // Find all watched paths that are in this directory that are not external. + if isDir { + var pathsToRemove []string + w.mu.Lock() + for _, path := range w.paths { + wdir, _ := filepath.Split(path.name) + if filepath.Clean(wdir) == name { + if !w.externalWatches[path.name] { + pathsToRemove = append(pathsToRemove, path.name) + } + } + } + w.mu.Unlock() + for _, name := range pathsToRemove { + // Since these are internal, not much sense in propagating error + // to the user, as that will just confuse them with an error about + // a path they did not explicitly watch themselves. + w.Remove(name) + } + } + + return nil +} + +// Watch all events (except NOTE_EXTEND, NOTE_LINK, NOTE_REVOKE) +const noteAllEvents = unix.NOTE_DELETE | unix.NOTE_WRITE | unix.NOTE_ATTRIB | unix.NOTE_RENAME + +// keventWaitTime to block on each read from kevent +var keventWaitTime = durationToTimespec(100 * time.Millisecond) + +// addWatch adds name to the watched file set. +// The flags are interpreted as described in kevent(2). +// Returns the real path to the file which was added, if any, which may be different from the one passed in the case of symlinks. +func (w *Watcher) addWatch(name string, flags uint32) (string, error) { + var isDir bool + // Make ./name and name equivalent + name = filepath.Clean(name) + + w.mu.Lock() + if w.isClosed { + w.mu.Unlock() + return "", errors.New("kevent instance already closed") + } + watchfd, alreadyWatching := w.watches[name] + // We already have a watch, but we can still override flags. + if alreadyWatching { + isDir = w.paths[watchfd].isDir + } + w.mu.Unlock() + + if !alreadyWatching { + fi, err := os.Lstat(name) + if err != nil { + return "", err + } + + // Don't watch sockets. + if fi.Mode()&os.ModeSocket == os.ModeSocket { + return "", nil + } + + // Don't watch named pipes. + if fi.Mode()&os.ModeNamedPipe == os.ModeNamedPipe { + return "", nil + } + + // Follow Symlinks + // Unfortunately, Linux can add bogus symlinks to watch list without + // issue, and Windows can't do symlinks period (AFAIK). To maintain + // consistency, we will act like everything is fine. There will simply + // be no file events for broken symlinks. + // Hence the returns of nil on errors. + if fi.Mode()&os.ModeSymlink == os.ModeSymlink { + name, err = filepath.EvalSymlinks(name) + if err != nil { + return "", nil + } + + w.mu.Lock() + _, alreadyWatching = w.watches[name] + w.mu.Unlock() + + if alreadyWatching { + return name, nil + } + + fi, err = os.Lstat(name) + if err != nil { + return "", nil + } + } + + watchfd, err = unix.Open(name, openMode, 0700) + if watchfd == -1 { + return "", err + } + + isDir = fi.IsDir() + } + + const registerAdd = unix.EV_ADD | unix.EV_CLEAR | unix.EV_ENABLE + if err := register(w.kq, []int{watchfd}, registerAdd, flags); err != nil { + unix.Close(watchfd) + return "", err + } + + if !alreadyWatching { + w.mu.Lock() + w.watches[name] = watchfd + w.paths[watchfd] = pathInfo{name: name, isDir: isDir} + w.mu.Unlock() + } + + if isDir { + // Watch the directory if it has not been watched before, + // or if it was watched before, but perhaps only a NOTE_DELETE (watchDirectoryFiles) + w.mu.Lock() + + watchDir := (flags&unix.NOTE_WRITE) == unix.NOTE_WRITE && + (!alreadyWatching || (w.dirFlags[name]&unix.NOTE_WRITE) != unix.NOTE_WRITE) + // Store flags so this watch can be updated later + w.dirFlags[name] = flags + w.mu.Unlock() + + if watchDir { + if err := w.watchDirectoryFiles(name); err != nil { + return "", err + } + } + } + return name, nil +} + +// readEvents reads from kqueue and converts the received kevents into +// Event values that it sends down the Events channel. +func (w *Watcher) readEvents() { + eventBuffer := make([]unix.Kevent_t, 10) + +loop: + for { + // See if there is a message on the "done" channel + select { + case <-w.done: + break loop + default: + } + + // Get new events + kevents, err := read(w.kq, eventBuffer, &keventWaitTime) + // EINTR is okay, the syscall was interrupted before timeout expired. + if err != nil && err != unix.EINTR { + select { + case w.Errors <- err: + case <-w.done: + break loop + } + continue + } + + // Flush the events we received to the Events channel + for len(kevents) > 0 { + kevent := &kevents[0] + watchfd := int(kevent.Ident) + mask := uint32(kevent.Fflags) + w.mu.Lock() + path := w.paths[watchfd] + w.mu.Unlock() + event := newEvent(path.name, mask) + + if path.isDir && !(event.Op&Remove == Remove) { + // Double check to make sure the directory exists. This can happen when + // we do a rm -fr on a recursively watched folders and we receive a + // modification event first but the folder has been deleted and later + // receive the delete event + if _, err := os.Lstat(event.Name); os.IsNotExist(err) { + // mark is as delete event + event.Op |= Remove + } + } + + if event.Op&Rename == Rename || event.Op&Remove == Remove { + w.Remove(event.Name) + w.mu.Lock() + delete(w.fileExists, event.Name) + w.mu.Unlock() + } + + if path.isDir && event.Op&Write == Write && !(event.Op&Remove == Remove) { + w.sendDirectoryChangeEvents(event.Name) + } else { + // Send the event on the Events channel. + select { + case w.Events <- event: + case <-w.done: + break loop + } + } + + if event.Op&Remove == Remove { + // Look for a file that may have overwritten this. + // For example, mv f1 f2 will delete f2, then create f2. + if path.isDir { + fileDir := filepath.Clean(event.Name) + w.mu.Lock() + _, found := w.watches[fileDir] + w.mu.Unlock() + if found { + // make sure the directory exists before we watch for changes. When we + // do a recursive watch and perform rm -fr, the parent directory might + // have gone missing, ignore the missing directory and let the + // upcoming delete event remove the watch from the parent directory. + if _, err := os.Lstat(fileDir); err == nil { + w.sendDirectoryChangeEvents(fileDir) + } + } + } else { + filePath := filepath.Clean(event.Name) + if fileInfo, err := os.Lstat(filePath); err == nil { + w.sendFileCreatedEventIfNew(filePath, fileInfo) + } + } + } + + // Move to next event + kevents = kevents[1:] + } + } + + // cleanup + err := unix.Close(w.kq) + if err != nil { + // only way the previous loop breaks is if w.done was closed so we need to async send to w.Errors. + select { + case w.Errors <- err: + default: + } + } + close(w.Events) + close(w.Errors) +} + +// newEvent returns an platform-independent Event based on kqueue Fflags. +func newEvent(name string, mask uint32) Event { + e := Event{Name: name} + if mask&unix.NOTE_DELETE == unix.NOTE_DELETE { + e.Op |= Remove + } + if mask&unix.NOTE_WRITE == unix.NOTE_WRITE { + e.Op |= Write + } + if mask&unix.NOTE_RENAME == unix.NOTE_RENAME { + e.Op |= Rename + } + if mask&unix.NOTE_ATTRIB == unix.NOTE_ATTRIB { + e.Op |= Chmod + } + return e +} + +func newCreateEvent(name string) Event { + return Event{Name: name, Op: Create} +} + +// watchDirectoryFiles to mimic inotify when adding a watch on a directory +func (w *Watcher) watchDirectoryFiles(dirPath string) error { + // Get all files + files, err := ioutil.ReadDir(dirPath) + if err != nil { + return err + } + + for _, fileInfo := range files { + filePath := filepath.Join(dirPath, fileInfo.Name()) + filePath, err = w.internalWatch(filePath, fileInfo) + if err != nil { + return err + } + + w.mu.Lock() + w.fileExists[filePath] = true + w.mu.Unlock() + } + + return nil +} + +// sendDirectoryEvents searches the directory for newly created files +// and sends them over the event channel. This functionality is to have +// the BSD version of fsnotify match Linux inotify which provides a +// create event for files created in a watched directory. +func (w *Watcher) sendDirectoryChangeEvents(dirPath string) { + // Get all files + files, err := ioutil.ReadDir(dirPath) + if err != nil { + select { + case w.Errors <- err: + case <-w.done: + return + } + } + + // Search for new files + for _, fileInfo := range files { + filePath := filepath.Join(dirPath, fileInfo.Name()) + err := w.sendFileCreatedEventIfNew(filePath, fileInfo) + + if err != nil { + return + } + } +} + +// sendFileCreatedEvent sends a create event if the file isn't already being tracked. +func (w *Watcher) sendFileCreatedEventIfNew(filePath string, fileInfo os.FileInfo) (err error) { + w.mu.Lock() + _, doesExist := w.fileExists[filePath] + w.mu.Unlock() + if !doesExist { + // Send create event + select { + case w.Events <- newCreateEvent(filePath): + case <-w.done: + return + } + } + + // like watchDirectoryFiles (but without doing another ReadDir) + filePath, err = w.internalWatch(filePath, fileInfo) + if err != nil { + return err + } + + w.mu.Lock() + w.fileExists[filePath] = true + w.mu.Unlock() + + return nil +} + +func (w *Watcher) internalWatch(name string, fileInfo os.FileInfo) (string, error) { + if fileInfo.IsDir() { + // mimic Linux providing delete events for subdirectories + // but preserve the flags used if currently watching subdirectory + w.mu.Lock() + flags := w.dirFlags[name] + w.mu.Unlock() + + flags |= unix.NOTE_DELETE | unix.NOTE_RENAME + return w.addWatch(name, flags) + } + + // watch file to mimic Linux inotify + return w.addWatch(name, noteAllEvents) +} + +// kqueue creates a new kernel event queue and returns a descriptor. +func kqueue() (kq int, err error) { + kq, err = unix.Kqueue() + if kq == -1 { + return kq, err + } + return kq, nil +} + +// register events with the queue +func register(kq int, fds []int, flags int, fflags uint32) error { + changes := make([]unix.Kevent_t, len(fds)) + + for i, fd := range fds { + // SetKevent converts int to the platform-specific types: + unix.SetKevent(&changes[i], fd, unix.EVFILT_VNODE, flags) + changes[i].Fflags = fflags + } + + // register the events + success, err := unix.Kevent(kq, changes, nil, nil) + if success == -1 { + return err + } + return nil +} + +// read retrieves pending events, or waits until an event occurs. +// A timeout of nil blocks indefinitely, while 0 polls the queue. +func read(kq int, events []unix.Kevent_t, timeout *unix.Timespec) ([]unix.Kevent_t, error) { + n, err := unix.Kevent(kq, nil, events, timeout) + if err != nil { + return nil, err + } + return events[0:n], nil +} + +// durationToTimespec prepares a timeout value +func durationToTimespec(d time.Duration) unix.Timespec { + return unix.NsecToTimespec(d.Nanoseconds()) +} diff --git a/vendor/github.com/fsnotify/fsnotify/open_mode_bsd.go b/vendor/github.com/fsnotify/fsnotify/open_mode_bsd.go new file mode 100644 index 000000000..7d8de1451 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/open_mode_bsd.go @@ -0,0 +1,11 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build freebsd openbsd netbsd dragonfly + +package fsnotify + +import "golang.org/x/sys/unix" + +const openMode = unix.O_NONBLOCK | unix.O_RDONLY diff --git a/vendor/github.com/fsnotify/fsnotify/open_mode_darwin.go b/vendor/github.com/fsnotify/fsnotify/open_mode_darwin.go new file mode 100644 index 000000000..9139e1716 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/open_mode_darwin.go @@ -0,0 +1,12 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin + +package fsnotify + +import "golang.org/x/sys/unix" + +// note: this constant is not defined on BSD +const openMode = unix.O_EVTONLY diff --git a/vendor/github.com/fsnotify/fsnotify/windows.go b/vendor/github.com/fsnotify/fsnotify/windows.go new file mode 100644 index 000000000..09436f31d --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/windows.go @@ -0,0 +1,561 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build windows + +package fsnotify + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "sync" + "syscall" + "unsafe" +) + +// Watcher watches a set of files, delivering events to a channel. +type Watcher struct { + Events chan Event + Errors chan error + isClosed bool // Set to true when Close() is first called + mu sync.Mutex // Map access + port syscall.Handle // Handle to completion port + watches watchMap // Map of watches (key: i-number) + input chan *input // Inputs to the reader are sent on this channel + quit chan chan<- error +} + +// NewWatcher establishes a new watcher with the underlying OS and begins waiting for events. +func NewWatcher() (*Watcher, error) { + port, e := syscall.CreateIoCompletionPort(syscall.InvalidHandle, 0, 0, 0) + if e != nil { + return nil, os.NewSyscallError("CreateIoCompletionPort", e) + } + w := &Watcher{ + port: port, + watches: make(watchMap), + input: make(chan *input, 1), + Events: make(chan Event, 50), + Errors: make(chan error), + quit: make(chan chan<- error, 1), + } + go w.readEvents() + return w, nil +} + +// Close removes all watches and closes the events channel. +func (w *Watcher) Close() error { + if w.isClosed { + return nil + } + w.isClosed = true + + // Send "quit" message to the reader goroutine + ch := make(chan error) + w.quit <- ch + if err := w.wakeupReader(); err != nil { + return err + } + return <-ch +} + +// Add starts watching the named file or directory (non-recursively). +func (w *Watcher) Add(name string) error { + if w.isClosed { + return errors.New("watcher already closed") + } + in := &input{ + op: opAddWatch, + path: filepath.Clean(name), + flags: sysFSALLEVENTS, + reply: make(chan error), + } + w.input <- in + if err := w.wakeupReader(); err != nil { + return err + } + return <-in.reply +} + +// Remove stops watching the the named file or directory (non-recursively). +func (w *Watcher) Remove(name string) error { + in := &input{ + op: opRemoveWatch, + path: filepath.Clean(name), + reply: make(chan error), + } + w.input <- in + if err := w.wakeupReader(); err != nil { + return err + } + return <-in.reply +} + +const ( + // Options for AddWatch + sysFSONESHOT = 0x80000000 + sysFSONLYDIR = 0x1000000 + + // Events + sysFSACCESS = 0x1 + sysFSALLEVENTS = 0xfff + sysFSATTRIB = 0x4 + sysFSCLOSE = 0x18 + sysFSCREATE = 0x100 + sysFSDELETE = 0x200 + sysFSDELETESELF = 0x400 + sysFSMODIFY = 0x2 + sysFSMOVE = 0xc0 + sysFSMOVEDFROM = 0x40 + sysFSMOVEDTO = 0x80 + sysFSMOVESELF = 0x800 + + // Special events + sysFSIGNORED = 0x8000 + sysFSQOVERFLOW = 0x4000 +) + +func newEvent(name string, mask uint32) Event { + e := Event{Name: name} + if mask&sysFSCREATE == sysFSCREATE || mask&sysFSMOVEDTO == sysFSMOVEDTO { + e.Op |= Create + } + if mask&sysFSDELETE == sysFSDELETE || mask&sysFSDELETESELF == sysFSDELETESELF { + e.Op |= Remove + } + if mask&sysFSMODIFY == sysFSMODIFY { + e.Op |= Write + } + if mask&sysFSMOVE == sysFSMOVE || mask&sysFSMOVESELF == sysFSMOVESELF || mask&sysFSMOVEDFROM == sysFSMOVEDFROM { + e.Op |= Rename + } + if mask&sysFSATTRIB == sysFSATTRIB { + e.Op |= Chmod + } + return e +} + +const ( + opAddWatch = iota + opRemoveWatch +) + +const ( + provisional uint64 = 1 << (32 + iota) +) + +type input struct { + op int + path string + flags uint32 + reply chan error +} + +type inode struct { + handle syscall.Handle + volume uint32 + index uint64 +} + +type watch struct { + ov syscall.Overlapped + ino *inode // i-number + path string // Directory path + mask uint64 // Directory itself is being watched with these notify flags + names map[string]uint64 // Map of names being watched and their notify flags + rename string // Remembers the old name while renaming a file + buf [4096]byte +} + +type indexMap map[uint64]*watch +type watchMap map[uint32]indexMap + +func (w *Watcher) wakeupReader() error { + e := syscall.PostQueuedCompletionStatus(w.port, 0, 0, nil) + if e != nil { + return os.NewSyscallError("PostQueuedCompletionStatus", e) + } + return nil +} + +func getDir(pathname string) (dir string, err error) { + attr, e := syscall.GetFileAttributes(syscall.StringToUTF16Ptr(pathname)) + if e != nil { + return "", os.NewSyscallError("GetFileAttributes", e) + } + if attr&syscall.FILE_ATTRIBUTE_DIRECTORY != 0 { + dir = pathname + } else { + dir, _ = filepath.Split(pathname) + dir = filepath.Clean(dir) + } + return +} + +func getIno(path string) (ino *inode, err error) { + h, e := syscall.CreateFile(syscall.StringToUTF16Ptr(path), + syscall.FILE_LIST_DIRECTORY, + syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE, + nil, syscall.OPEN_EXISTING, + syscall.FILE_FLAG_BACKUP_SEMANTICS|syscall.FILE_FLAG_OVERLAPPED, 0) + if e != nil { + return nil, os.NewSyscallError("CreateFile", e) + } + var fi syscall.ByHandleFileInformation + if e = syscall.GetFileInformationByHandle(h, &fi); e != nil { + syscall.CloseHandle(h) + return nil, os.NewSyscallError("GetFileInformationByHandle", e) + } + ino = &inode{ + handle: h, + volume: fi.VolumeSerialNumber, + index: uint64(fi.FileIndexHigh)<<32 | uint64(fi.FileIndexLow), + } + return ino, nil +} + +// Must run within the I/O thread. +func (m watchMap) get(ino *inode) *watch { + if i := m[ino.volume]; i != nil { + return i[ino.index] + } + return nil +} + +// Must run within the I/O thread. +func (m watchMap) set(ino *inode, watch *watch) { + i := m[ino.volume] + if i == nil { + i = make(indexMap) + m[ino.volume] = i + } + i[ino.index] = watch +} + +// Must run within the I/O thread. +func (w *Watcher) addWatch(pathname string, flags uint64) error { + dir, err := getDir(pathname) + if err != nil { + return err + } + if flags&sysFSONLYDIR != 0 && pathname != dir { + return nil + } + ino, err := getIno(dir) + if err != nil { + return err + } + w.mu.Lock() + watchEntry := w.watches.get(ino) + w.mu.Unlock() + if watchEntry == nil { + if _, e := syscall.CreateIoCompletionPort(ino.handle, w.port, 0, 0); e != nil { + syscall.CloseHandle(ino.handle) + return os.NewSyscallError("CreateIoCompletionPort", e) + } + watchEntry = &watch{ + ino: ino, + path: dir, + names: make(map[string]uint64), + } + w.mu.Lock() + w.watches.set(ino, watchEntry) + w.mu.Unlock() + flags |= provisional + } else { + syscall.CloseHandle(ino.handle) + } + if pathname == dir { + watchEntry.mask |= flags + } else { + watchEntry.names[filepath.Base(pathname)] |= flags + } + if err = w.startRead(watchEntry); err != nil { + return err + } + if pathname == dir { + watchEntry.mask &= ^provisional + } else { + watchEntry.names[filepath.Base(pathname)] &= ^provisional + } + return nil +} + +// Must run within the I/O thread. +func (w *Watcher) remWatch(pathname string) error { + dir, err := getDir(pathname) + if err != nil { + return err + } + ino, err := getIno(dir) + if err != nil { + return err + } + w.mu.Lock() + watch := w.watches.get(ino) + w.mu.Unlock() + if watch == nil { + return fmt.Errorf("can't remove non-existent watch for: %s", pathname) + } + if pathname == dir { + w.sendEvent(watch.path, watch.mask&sysFSIGNORED) + watch.mask = 0 + } else { + name := filepath.Base(pathname) + w.sendEvent(filepath.Join(watch.path, name), watch.names[name]&sysFSIGNORED) + delete(watch.names, name) + } + return w.startRead(watch) +} + +// Must run within the I/O thread. +func (w *Watcher) deleteWatch(watch *watch) { + for name, mask := range watch.names { + if mask&provisional == 0 { + w.sendEvent(filepath.Join(watch.path, name), mask&sysFSIGNORED) + } + delete(watch.names, name) + } + if watch.mask != 0 { + if watch.mask&provisional == 0 { + w.sendEvent(watch.path, watch.mask&sysFSIGNORED) + } + watch.mask = 0 + } +} + +// Must run within the I/O thread. +func (w *Watcher) startRead(watch *watch) error { + if e := syscall.CancelIo(watch.ino.handle); e != nil { + w.Errors <- os.NewSyscallError("CancelIo", e) + w.deleteWatch(watch) + } + mask := toWindowsFlags(watch.mask) + for _, m := range watch.names { + mask |= toWindowsFlags(m) + } + if mask == 0 { + if e := syscall.CloseHandle(watch.ino.handle); e != nil { + w.Errors <- os.NewSyscallError("CloseHandle", e) + } + w.mu.Lock() + delete(w.watches[watch.ino.volume], watch.ino.index) + w.mu.Unlock() + return nil + } + e := syscall.ReadDirectoryChanges(watch.ino.handle, &watch.buf[0], + uint32(unsafe.Sizeof(watch.buf)), false, mask, nil, &watch.ov, 0) + if e != nil { + err := os.NewSyscallError("ReadDirectoryChanges", e) + if e == syscall.ERROR_ACCESS_DENIED && watch.mask&provisional == 0 { + // Watched directory was probably removed + if w.sendEvent(watch.path, watch.mask&sysFSDELETESELF) { + if watch.mask&sysFSONESHOT != 0 { + watch.mask = 0 + } + } + err = nil + } + w.deleteWatch(watch) + w.startRead(watch) + return err + } + return nil +} + +// readEvents reads from the I/O completion port, converts the +// received events into Event objects and sends them via the Events channel. +// Entry point to the I/O thread. +func (w *Watcher) readEvents() { + var ( + n, key uint32 + ov *syscall.Overlapped + ) + runtime.LockOSThread() + + for { + e := syscall.GetQueuedCompletionStatus(w.port, &n, &key, &ov, syscall.INFINITE) + watch := (*watch)(unsafe.Pointer(ov)) + + if watch == nil { + select { + case ch := <-w.quit: + w.mu.Lock() + var indexes []indexMap + for _, index := range w.watches { + indexes = append(indexes, index) + } + w.mu.Unlock() + for _, index := range indexes { + for _, watch := range index { + w.deleteWatch(watch) + w.startRead(watch) + } + } + var err error + if e := syscall.CloseHandle(w.port); e != nil { + err = os.NewSyscallError("CloseHandle", e) + } + close(w.Events) + close(w.Errors) + ch <- err + return + case in := <-w.input: + switch in.op { + case opAddWatch: + in.reply <- w.addWatch(in.path, uint64(in.flags)) + case opRemoveWatch: + in.reply <- w.remWatch(in.path) + } + default: + } + continue + } + + switch e { + case syscall.ERROR_MORE_DATA: + if watch == nil { + w.Errors <- errors.New("ERROR_MORE_DATA has unexpectedly null lpOverlapped buffer") + } else { + // The i/o succeeded but the buffer is full. + // In theory we should be building up a full packet. + // In practice we can get away with just carrying on. + n = uint32(unsafe.Sizeof(watch.buf)) + } + case syscall.ERROR_ACCESS_DENIED: + // Watched directory was probably removed + w.sendEvent(watch.path, watch.mask&sysFSDELETESELF) + w.deleteWatch(watch) + w.startRead(watch) + continue + case syscall.ERROR_OPERATION_ABORTED: + // CancelIo was called on this handle + continue + default: + w.Errors <- os.NewSyscallError("GetQueuedCompletionPort", e) + continue + case nil: + } + + var offset uint32 + for { + if n == 0 { + w.Events <- newEvent("", sysFSQOVERFLOW) + w.Errors <- errors.New("short read in readEvents()") + break + } + + // Point "raw" to the event in the buffer + raw := (*syscall.FileNotifyInformation)(unsafe.Pointer(&watch.buf[offset])) + buf := (*[syscall.MAX_PATH]uint16)(unsafe.Pointer(&raw.FileName)) + name := syscall.UTF16ToString(buf[:raw.FileNameLength/2]) + fullname := filepath.Join(watch.path, name) + + var mask uint64 + switch raw.Action { + case syscall.FILE_ACTION_REMOVED: + mask = sysFSDELETESELF + case syscall.FILE_ACTION_MODIFIED: + mask = sysFSMODIFY + case syscall.FILE_ACTION_RENAMED_OLD_NAME: + watch.rename = name + case syscall.FILE_ACTION_RENAMED_NEW_NAME: + if watch.names[watch.rename] != 0 { + watch.names[name] |= watch.names[watch.rename] + delete(watch.names, watch.rename) + mask = sysFSMOVESELF + } + } + + sendNameEvent := func() { + if w.sendEvent(fullname, watch.names[name]&mask) { + if watch.names[name]&sysFSONESHOT != 0 { + delete(watch.names, name) + } + } + } + if raw.Action != syscall.FILE_ACTION_RENAMED_NEW_NAME { + sendNameEvent() + } + if raw.Action == syscall.FILE_ACTION_REMOVED { + w.sendEvent(fullname, watch.names[name]&sysFSIGNORED) + delete(watch.names, name) + } + if w.sendEvent(fullname, watch.mask&toFSnotifyFlags(raw.Action)) { + if watch.mask&sysFSONESHOT != 0 { + watch.mask = 0 + } + } + if raw.Action == syscall.FILE_ACTION_RENAMED_NEW_NAME { + fullname = filepath.Join(watch.path, watch.rename) + sendNameEvent() + } + + // Move to the next event in the buffer + if raw.NextEntryOffset == 0 { + break + } + offset += raw.NextEntryOffset + + // Error! + if offset >= n { + w.Errors <- errors.New("Windows system assumed buffer larger than it is, events have likely been missed.") + break + } + } + + if err := w.startRead(watch); err != nil { + w.Errors <- err + } + } +} + +func (w *Watcher) sendEvent(name string, mask uint64) bool { + if mask == 0 { + return false + } + event := newEvent(name, uint32(mask)) + select { + case ch := <-w.quit: + w.quit <- ch + case w.Events <- event: + } + return true +} + +func toWindowsFlags(mask uint64) uint32 { + var m uint32 + if mask&sysFSACCESS != 0 { + m |= syscall.FILE_NOTIFY_CHANGE_LAST_ACCESS + } + if mask&sysFSMODIFY != 0 { + m |= syscall.FILE_NOTIFY_CHANGE_LAST_WRITE + } + if mask&sysFSATTRIB != 0 { + m |= syscall.FILE_NOTIFY_CHANGE_ATTRIBUTES + } + if mask&(sysFSMOVE|sysFSCREATE|sysFSDELETE) != 0 { + m |= syscall.FILE_NOTIFY_CHANGE_FILE_NAME | syscall.FILE_NOTIFY_CHANGE_DIR_NAME + } + return m +} + +func toFSnotifyFlags(action uint32) uint64 { + switch action { + case syscall.FILE_ACTION_ADDED: + return sysFSCREATE + case syscall.FILE_ACTION_REMOVED: + return sysFSDELETE + case syscall.FILE_ACTION_MODIFIED: + return sysFSMODIFY + case syscall.FILE_ACTION_RENAMED_OLD_NAME: + return sysFSMOVEDFROM + case syscall.FILE_ACTION_RENAMED_NEW_NAME: + return sysFSMOVEDTO + } + return 0 +} diff --git a/vendor/github.com/hashicorp/hcl/.gitignore b/vendor/github.com/hashicorp/hcl/.gitignore new file mode 100644 index 000000000..15586a2b5 --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/.gitignore @@ -0,0 +1,9 @@ +y.output + +# ignore intellij files +.idea +*.iml +*.ipr +*.iws + +*.test diff --git a/vendor/github.com/hashicorp/hcl/.travis.yml b/vendor/github.com/hashicorp/hcl/.travis.yml new file mode 100644 index 000000000..cb63a3216 --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/.travis.yml @@ -0,0 +1,13 @@ +sudo: false + +language: go + +go: + - 1.x + - tip + +branches: + only: + - master + +script: make test diff --git a/vendor/github.com/hashicorp/hcl/LICENSE b/vendor/github.com/hashicorp/hcl/LICENSE new file mode 100644 index 000000000..c33dcc7c9 --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/LICENSE @@ -0,0 +1,354 @@ +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. “Contributor” + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. “Contributor Version” + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor’s Contribution. + +1.3. “Contribution” + + means Covered Software of a particular Contributor. + +1.4. “Covered Software” + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. “Incompatible With Secondary Licenses” + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of version + 1.1 or earlier of the License, but not also under the terms of a + Secondary License. + +1.6. “Executable Form” + + means any form of the work other than Source Code Form. + +1.7. “Larger Work” + + means a work that combines Covered Software with other material, in a separate + file or files, that is not Covered Software. + +1.8. “License” + + means this document. + +1.9. “Licensable” + + means having the right to grant, to the maximum extent possible, whether at the + time of the initial grant or subsequently, any and all of the rights conveyed by + this License. + +1.10. “Modifications” + + means any of the following: + + a. any file in Source Code Form that results from an addition to, deletion + from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. “Patent Claims” of a Contributor + + means any patent claim(s), including without limitation, method, process, + and apparatus claims, in any patent Licensable by such Contributor that + would be infringed, but for the grant of the License, by the making, + using, selling, offering for sale, having made, import, or transfer of + either its Contributions or its Contributor Version. + +1.12. “Secondary License” + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. “Source Code Form” + + means the form of the work preferred for making modifications. + +1.14. “You” (or “Your”) + + means an individual or a legal entity exercising rights under this + License. For legal entities, “You” includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, “control” means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or as + part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its Contributions + or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution become + effective for each Contribution on the date the Contributor first distributes + such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under this + License. No additional rights or licenses will be implied from the distribution + or licensing of Covered Software under this License. Notwithstanding Section + 2.1(b) above, no patent license is granted by a Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party’s + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of its + Contributions. + + This License does not grant any rights in the trademarks, service marks, or + logos of any Contributor (except as may be necessary to comply with the + notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this License + (see Section 10.2) or under the terms of a Secondary License (if permitted + under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its Contributions + are its original creation(s) or it has sufficient rights to grant the + rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under applicable + copyright doctrines of fair use, fair dealing, or other equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under the + terms of this License. You must inform recipients that the Source Code Form + of the Covered Software is governed by the terms of this License, and how + they can obtain a copy of this License. You may not attempt to alter or + restrict the recipients’ rights in the Source Code Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this License, + or sublicense it under different terms, provided that the license for + the Executable Form does not attempt to limit or alter the recipients’ + rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for the + Covered Software. If the Larger Work is a combination of Covered Software + with a work governed by one or more Secondary Licenses, and the Covered + Software is not Incompatible With Secondary Licenses, this License permits + You to additionally distribute such Covered Software under the terms of + such Secondary License(s), so that the recipient of the Larger Work may, at + their option, further distribute the Covered Software under the terms of + either this License or such Secondary License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices (including + copyright notices, patent notices, disclaimers of warranty, or limitations + of liability) contained within the Source Code Form of the Covered + Software, except that You may alter any license notices to the extent + required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on behalf + of any Contributor. You must make it absolutely clear that any such + warranty, support, indemnity, or liability obligation is offered by You + alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, judicial + order, or regulation then You must: (a) comply with the terms of this License + to the maximum extent possible; and (b) describe the limitations and the code + they affect. Such description must be placed in a text file included with all + distributions of the Covered Software under this License. Except to the + extent prohibited by statute or regulation, such description must be + sufficiently detailed for a recipient of ordinary skill to be able to + understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing basis, + if such Contributor fails to notify You of the non-compliance by some + reasonable means prior to 60 days after You have come back into compliance. + Moreover, Your grants from a particular Contributor are reinstated on an + ongoing basis if such Contributor notifies You of the non-compliance by + some reasonable means, this is the first time You have received notice of + non-compliance with this License from such Contributor, and You become + compliant prior to 30 days after Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, counter-claims, + and cross-claims) alleging that a Contributor Version directly or + indirectly infringes any patent, then the rights granted to You by any and + all Contributors for the Covered Software under Section 2.1 of this License + shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an “as is” basis, without + warranty of any kind, either expressed, implied, or statutory, including, + without limitation, warranties that the Covered Software is free of defects, + merchantable, fit for a particular purpose or non-infringing. The entire + risk as to the quality and performance of the Covered Software is with You. + Should any Covered Software prove defective in any respect, You (not any + Contributor) assume the cost of any necessary servicing, repair, or + correction. This disclaimer of warranty constitutes an essential part of this + License. No use of any Covered Software is authorized under this License + except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from such + party’s negligence to the extent applicable law prohibits such limitation. + Some jurisdictions do not allow the exclusion or limitation of incidental or + consequential damages, so this exclusion and limitation may not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts of + a jurisdiction where the defendant maintains its principal place of business + and such litigation shall be governed by laws of that jurisdiction, without + reference to its conflict-of-law provisions. Nothing in this Section shall + prevent a party’s ability to bring cross-claims or counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject matter + hereof. If any provision of this License is held to be unenforceable, such + provision shall be reformed only to the extent necessary to make it + enforceable. Any law or regulation which provides that the language of a + contract shall be construed against the drafter shall not be used to construe + this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version of + the License under which You originally received the Covered Software, or + under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a modified + version of this License if you rename the license and remove any + references to the name of the license steward (except to note that such + modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses + If You choose to distribute Source Code Form that is Incompatible With + Secondary Licenses under the terms of this version of the License, the + notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, then +You may include the notice in a location (such as a LICENSE file in a relevant +directory) where a recipient would be likely to look for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - “Incompatible With Secondary Licenses” Notice + + This Source Code Form is “Incompatible + With Secondary Licenses”, as defined by + the Mozilla Public License, v. 2.0. + diff --git a/vendor/github.com/hashicorp/hcl/Makefile b/vendor/github.com/hashicorp/hcl/Makefile new file mode 100644 index 000000000..84fd743f5 --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/Makefile @@ -0,0 +1,18 @@ +TEST?=./... + +default: test + +fmt: generate + go fmt ./... + +test: generate + go get -t ./... + go test $(TEST) $(TESTARGS) + +generate: + go generate ./... + +updatedeps: + go get -u golang.org/x/tools/cmd/stringer + +.PHONY: default generate test updatedeps diff --git a/vendor/github.com/hashicorp/hcl/README.md b/vendor/github.com/hashicorp/hcl/README.md new file mode 100644 index 000000000..c8223326d --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/README.md @@ -0,0 +1,125 @@ +# HCL + +[![GoDoc](https://godoc.org/github.com/hashicorp/hcl?status.png)](https://godoc.org/github.com/hashicorp/hcl) [![Build Status](https://travis-ci.org/hashicorp/hcl.svg?branch=master)](https://travis-ci.org/hashicorp/hcl) + +HCL (HashiCorp Configuration Language) is a configuration language built +by HashiCorp. The goal of HCL is to build a structured configuration language +that is both human and machine friendly for use with command-line tools, but +specifically targeted towards DevOps tools, servers, etc. + +HCL is also fully JSON compatible. That is, JSON can be used as completely +valid input to a system expecting HCL. This helps makes systems +interoperable with other systems. + +HCL is heavily inspired by +[libucl](https://github.com/vstakhov/libucl), +nginx configuration, and others similar. + +## Why? + +A common question when viewing HCL is to ask the question: why not +JSON, YAML, etc.? + +Prior to HCL, the tools we built at [HashiCorp](http://www.hashicorp.com) +used a variety of configuration languages from full programming languages +such as Ruby to complete data structure languages such as JSON. What we +learned is that some people wanted human-friendly configuration languages +and some people wanted machine-friendly languages. + +JSON fits a nice balance in this, but is fairly verbose and most +importantly doesn't support comments. With YAML, we found that beginners +had a really hard time determining what the actual structure was, and +ended up guessing more often than not whether to use a hyphen, colon, etc. +in order to represent some configuration key. + +Full programming languages such as Ruby enable complex behavior +a configuration language shouldn't usually allow, and also forces +people to learn some set of Ruby. + +Because of this, we decided to create our own configuration language +that is JSON-compatible. Our configuration language (HCL) is designed +to be written and modified by humans. The API for HCL allows JSON +as an input so that it is also machine-friendly (machines can generate +JSON instead of trying to generate HCL). + +Our goal with HCL is not to alienate other configuration languages. +It is instead to provide HCL as a specialized language for our tools, +and JSON as the interoperability layer. + +## Syntax + +For a complete grammar, please see the parser itself. A high-level overview +of the syntax and grammar is listed here. + + * Single line comments start with `#` or `//` + + * Multi-line comments are wrapped in `/*` and `*/`. Nested block comments + are not allowed. A multi-line comment (also known as a block comment) + terminates at the first `*/` found. + + * Values are assigned with the syntax `key = value` (whitespace doesn't + matter). The value can be any primitive: a string, number, boolean, + object, or list. + + * Strings are double-quoted and can contain any UTF-8 characters. + Example: `"Hello, World"` + + * Multi-line strings start with `<- + echo %Path% + + go version + + go env + + go get -t ./... + +build_script: +- cmd: go test -v ./... diff --git a/vendor/github.com/hashicorp/hcl/decoder.go b/vendor/github.com/hashicorp/hcl/decoder.go new file mode 100644 index 000000000..bed9ebbe1 --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/decoder.go @@ -0,0 +1,729 @@ +package hcl + +import ( + "errors" + "fmt" + "reflect" + "sort" + "strconv" + "strings" + + "github.com/hashicorp/hcl/hcl/ast" + "github.com/hashicorp/hcl/hcl/parser" + "github.com/hashicorp/hcl/hcl/token" +) + +// This is the tag to use with structures to have settings for HCL +const tagName = "hcl" + +var ( + // nodeType holds a reference to the type of ast.Node + nodeType reflect.Type = findNodeType() +) + +// Unmarshal accepts a byte slice as input and writes the +// data to the value pointed to by v. +func Unmarshal(bs []byte, v interface{}) error { + root, err := parse(bs) + if err != nil { + return err + } + + return DecodeObject(v, root) +} + +// Decode reads the given input and decodes it into the structure +// given by `out`. +func Decode(out interface{}, in string) error { + obj, err := Parse(in) + if err != nil { + return err + } + + return DecodeObject(out, obj) +} + +// DecodeObject is a lower-level version of Decode. It decodes a +// raw Object into the given output. +func DecodeObject(out interface{}, n ast.Node) error { + val := reflect.ValueOf(out) + if val.Kind() != reflect.Ptr { + return errors.New("result must be a pointer") + } + + // If we have the file, we really decode the root node + if f, ok := n.(*ast.File); ok { + n = f.Node + } + + var d decoder + return d.decode("root", n, val.Elem()) +} + +type decoder struct { + stack []reflect.Kind +} + +func (d *decoder) decode(name string, node ast.Node, result reflect.Value) error { + k := result + + // If we have an interface with a valid value, we use that + // for the check. + if result.Kind() == reflect.Interface { + elem := result.Elem() + if elem.IsValid() { + k = elem + } + } + + // Push current onto stack unless it is an interface. + if k.Kind() != reflect.Interface { + d.stack = append(d.stack, k.Kind()) + + // Schedule a pop + defer func() { + d.stack = d.stack[:len(d.stack)-1] + }() + } + + switch k.Kind() { + case reflect.Bool: + return d.decodeBool(name, node, result) + case reflect.Float32, reflect.Float64: + return d.decodeFloat(name, node, result) + case reflect.Int, reflect.Int32, reflect.Int64: + return d.decodeInt(name, node, result) + case reflect.Interface: + // When we see an interface, we make our own thing + return d.decodeInterface(name, node, result) + case reflect.Map: + return d.decodeMap(name, node, result) + case reflect.Ptr: + return d.decodePtr(name, node, result) + case reflect.Slice: + return d.decodeSlice(name, node, result) + case reflect.String: + return d.decodeString(name, node, result) + case reflect.Struct: + return d.decodeStruct(name, node, result) + default: + return &parser.PosError{ + Pos: node.Pos(), + Err: fmt.Errorf("%s: unknown kind to decode into: %s", name, k.Kind()), + } + } +} + +func (d *decoder) decodeBool(name string, node ast.Node, result reflect.Value) error { + switch n := node.(type) { + case *ast.LiteralType: + if n.Token.Type == token.BOOL { + v, err := strconv.ParseBool(n.Token.Text) + if err != nil { + return err + } + + result.Set(reflect.ValueOf(v)) + return nil + } + } + + return &parser.PosError{ + Pos: node.Pos(), + Err: fmt.Errorf("%s: unknown type %T", name, node), + } +} + +func (d *decoder) decodeFloat(name string, node ast.Node, result reflect.Value) error { + switch n := node.(type) { + case *ast.LiteralType: + if n.Token.Type == token.FLOAT || n.Token.Type == token.NUMBER { + v, err := strconv.ParseFloat(n.Token.Text, 64) + if err != nil { + return err + } + + result.Set(reflect.ValueOf(v).Convert(result.Type())) + return nil + } + } + + return &parser.PosError{ + Pos: node.Pos(), + Err: fmt.Errorf("%s: unknown type %T", name, node), + } +} + +func (d *decoder) decodeInt(name string, node ast.Node, result reflect.Value) error { + switch n := node.(type) { + case *ast.LiteralType: + switch n.Token.Type { + case token.NUMBER: + v, err := strconv.ParseInt(n.Token.Text, 0, 0) + if err != nil { + return err + } + + if result.Kind() == reflect.Interface { + result.Set(reflect.ValueOf(int(v))) + } else { + result.SetInt(v) + } + return nil + case token.STRING: + v, err := strconv.ParseInt(n.Token.Value().(string), 0, 0) + if err != nil { + return err + } + + if result.Kind() == reflect.Interface { + result.Set(reflect.ValueOf(int(v))) + } else { + result.SetInt(v) + } + return nil + } + } + + return &parser.PosError{ + Pos: node.Pos(), + Err: fmt.Errorf("%s: unknown type %T", name, node), + } +} + +func (d *decoder) decodeInterface(name string, node ast.Node, result reflect.Value) error { + // When we see an ast.Node, we retain the value to enable deferred decoding. + // Very useful in situations where we want to preserve ast.Node information + // like Pos + if result.Type() == nodeType && result.CanSet() { + result.Set(reflect.ValueOf(node)) + return nil + } + + var set reflect.Value + redecode := true + + // For testing types, ObjectType should just be treated as a list. We + // set this to a temporary var because we want to pass in the real node. + testNode := node + if ot, ok := node.(*ast.ObjectType); ok { + testNode = ot.List + } + + switch n := testNode.(type) { + case *ast.ObjectList: + // If we're at the root or we're directly within a slice, then we + // decode objects into map[string]interface{}, otherwise we decode + // them into lists. + if len(d.stack) == 0 || d.stack[len(d.stack)-1] == reflect.Slice { + var temp map[string]interface{} + tempVal := reflect.ValueOf(temp) + result := reflect.MakeMap( + reflect.MapOf( + reflect.TypeOf(""), + tempVal.Type().Elem())) + + set = result + } else { + var temp []map[string]interface{} + tempVal := reflect.ValueOf(temp) + result := reflect.MakeSlice( + reflect.SliceOf(tempVal.Type().Elem()), 0, len(n.Items)) + set = result + } + case *ast.ObjectType: + // If we're at the root or we're directly within a slice, then we + // decode objects into map[string]interface{}, otherwise we decode + // them into lists. + if len(d.stack) == 0 || d.stack[len(d.stack)-1] == reflect.Slice { + var temp map[string]interface{} + tempVal := reflect.ValueOf(temp) + result := reflect.MakeMap( + reflect.MapOf( + reflect.TypeOf(""), + tempVal.Type().Elem())) + + set = result + } else { + var temp []map[string]interface{} + tempVal := reflect.ValueOf(temp) + result := reflect.MakeSlice( + reflect.SliceOf(tempVal.Type().Elem()), 0, 1) + set = result + } + case *ast.ListType: + var temp []interface{} + tempVal := reflect.ValueOf(temp) + result := reflect.MakeSlice( + reflect.SliceOf(tempVal.Type().Elem()), 0, 0) + set = result + case *ast.LiteralType: + switch n.Token.Type { + case token.BOOL: + var result bool + set = reflect.Indirect(reflect.New(reflect.TypeOf(result))) + case token.FLOAT: + var result float64 + set = reflect.Indirect(reflect.New(reflect.TypeOf(result))) + case token.NUMBER: + var result int + set = reflect.Indirect(reflect.New(reflect.TypeOf(result))) + case token.STRING, token.HEREDOC: + set = reflect.Indirect(reflect.New(reflect.TypeOf(""))) + default: + return &parser.PosError{ + Pos: node.Pos(), + Err: fmt.Errorf("%s: cannot decode into interface: %T", name, node), + } + } + default: + return fmt.Errorf( + "%s: cannot decode into interface: %T", + name, node) + } + + // Set the result to what its supposed to be, then reset + // result so we don't reflect into this method anymore. + result.Set(set) + + if redecode { + // Revisit the node so that we can use the newly instantiated + // thing and populate it. + if err := d.decode(name, node, result); err != nil { + return err + } + } + + return nil +} + +func (d *decoder) decodeMap(name string, node ast.Node, result reflect.Value) error { + if item, ok := node.(*ast.ObjectItem); ok { + node = &ast.ObjectList{Items: []*ast.ObjectItem{item}} + } + + if ot, ok := node.(*ast.ObjectType); ok { + node = ot.List + } + + n, ok := node.(*ast.ObjectList) + if !ok { + return &parser.PosError{ + Pos: node.Pos(), + Err: fmt.Errorf("%s: not an object type for map (%T)", name, node), + } + } + + // If we have an interface, then we can address the interface, + // but not the slice itself, so get the element but set the interface + set := result + if result.Kind() == reflect.Interface { + result = result.Elem() + } + + resultType := result.Type() + resultElemType := resultType.Elem() + resultKeyType := resultType.Key() + if resultKeyType.Kind() != reflect.String { + return &parser.PosError{ + Pos: node.Pos(), + Err: fmt.Errorf("%s: map must have string keys", name), + } + } + + // Make a map if it is nil + resultMap := result + if result.IsNil() { + resultMap = reflect.MakeMap( + reflect.MapOf(resultKeyType, resultElemType)) + } + + // Go through each element and decode it. + done := make(map[string]struct{}) + for _, item := range n.Items { + if item.Val == nil { + continue + } + + // github.com/hashicorp/terraform/issue/5740 + if len(item.Keys) == 0 { + return &parser.PosError{ + Pos: node.Pos(), + Err: fmt.Errorf("%s: map must have string keys", name), + } + } + + // Get the key we're dealing with, which is the first item + keyStr := item.Keys[0].Token.Value().(string) + + // If we've already processed this key, then ignore it + if _, ok := done[keyStr]; ok { + continue + } + + // Determine the value. If we have more than one key, then we + // get the objectlist of only these keys. + itemVal := item.Val + if len(item.Keys) > 1 { + itemVal = n.Filter(keyStr) + done[keyStr] = struct{}{} + } + + // Make the field name + fieldName := fmt.Sprintf("%s.%s", name, keyStr) + + // Get the key/value as reflection values + key := reflect.ValueOf(keyStr) + val := reflect.Indirect(reflect.New(resultElemType)) + + // If we have a pre-existing value in the map, use that + oldVal := resultMap.MapIndex(key) + if oldVal.IsValid() { + val.Set(oldVal) + } + + // Decode! + if err := d.decode(fieldName, itemVal, val); err != nil { + return err + } + + // Set the value on the map + resultMap.SetMapIndex(key, val) + } + + // Set the final map if we can + set.Set(resultMap) + return nil +} + +func (d *decoder) decodePtr(name string, node ast.Node, result reflect.Value) error { + // Create an element of the concrete (non pointer) type and decode + // into that. Then set the value of the pointer to this type. + resultType := result.Type() + resultElemType := resultType.Elem() + val := reflect.New(resultElemType) + if err := d.decode(name, node, reflect.Indirect(val)); err != nil { + return err + } + + result.Set(val) + return nil +} + +func (d *decoder) decodeSlice(name string, node ast.Node, result reflect.Value) error { + // If we have an interface, then we can address the interface, + // but not the slice itself, so get the element but set the interface + set := result + if result.Kind() == reflect.Interface { + result = result.Elem() + } + // Create the slice if it isn't nil + resultType := result.Type() + resultElemType := resultType.Elem() + if result.IsNil() { + resultSliceType := reflect.SliceOf(resultElemType) + result = reflect.MakeSlice( + resultSliceType, 0, 0) + } + + // Figure out the items we'll be copying into the slice + var items []ast.Node + switch n := node.(type) { + case *ast.ObjectList: + items = make([]ast.Node, len(n.Items)) + for i, item := range n.Items { + items[i] = item + } + case *ast.ObjectType: + items = []ast.Node{n} + case *ast.ListType: + items = n.List + default: + return &parser.PosError{ + Pos: node.Pos(), + Err: fmt.Errorf("unknown slice type: %T", node), + } + } + + for i, item := range items { + fieldName := fmt.Sprintf("%s[%d]", name, i) + + // Decode + val := reflect.Indirect(reflect.New(resultElemType)) + + // if item is an object that was decoded from ambiguous JSON and + // flattened, make sure it's expanded if it needs to decode into a + // defined structure. + item := expandObject(item, val) + + if err := d.decode(fieldName, item, val); err != nil { + return err + } + + // Append it onto the slice + result = reflect.Append(result, val) + } + + set.Set(result) + return nil +} + +// expandObject detects if an ambiguous JSON object was flattened to a List which +// should be decoded into a struct, and expands the ast to properly deocode. +func expandObject(node ast.Node, result reflect.Value) ast.Node { + item, ok := node.(*ast.ObjectItem) + if !ok { + return node + } + + elemType := result.Type() + + // our target type must be a struct + switch elemType.Kind() { + case reflect.Ptr: + switch elemType.Elem().Kind() { + case reflect.Struct: + //OK + default: + return node + } + case reflect.Struct: + //OK + default: + return node + } + + // A list value will have a key and field name. If it had more fields, + // it wouldn't have been flattened. + if len(item.Keys) != 2 { + return node + } + + keyToken := item.Keys[0].Token + item.Keys = item.Keys[1:] + + // we need to un-flatten the ast enough to decode + newNode := &ast.ObjectItem{ + Keys: []*ast.ObjectKey{ + &ast.ObjectKey{ + Token: keyToken, + }, + }, + Val: &ast.ObjectType{ + List: &ast.ObjectList{ + Items: []*ast.ObjectItem{item}, + }, + }, + } + + return newNode +} + +func (d *decoder) decodeString(name string, node ast.Node, result reflect.Value) error { + switch n := node.(type) { + case *ast.LiteralType: + switch n.Token.Type { + case token.NUMBER: + result.Set(reflect.ValueOf(n.Token.Text).Convert(result.Type())) + return nil + case token.STRING, token.HEREDOC: + result.Set(reflect.ValueOf(n.Token.Value()).Convert(result.Type())) + return nil + } + } + + return &parser.PosError{ + Pos: node.Pos(), + Err: fmt.Errorf("%s: unknown type for string %T", name, node), + } +} + +func (d *decoder) decodeStruct(name string, node ast.Node, result reflect.Value) error { + var item *ast.ObjectItem + if it, ok := node.(*ast.ObjectItem); ok { + item = it + node = it.Val + } + + if ot, ok := node.(*ast.ObjectType); ok { + node = ot.List + } + + // Handle the special case where the object itself is a literal. Previously + // the yacc parser would always ensure top-level elements were arrays. The new + // parser does not make the same guarantees, thus we need to convert any + // top-level literal elements into a list. + if _, ok := node.(*ast.LiteralType); ok && item != nil { + node = &ast.ObjectList{Items: []*ast.ObjectItem{item}} + } + + list, ok := node.(*ast.ObjectList) + if !ok { + return &parser.PosError{ + Pos: node.Pos(), + Err: fmt.Errorf("%s: not an object type for struct (%T)", name, node), + } + } + + // This slice will keep track of all the structs we'll be decoding. + // There can be more than one struct if there are embedded structs + // that are squashed. + structs := make([]reflect.Value, 1, 5) + structs[0] = result + + // Compile the list of all the fields that we're going to be decoding + // from all the structs. + type field struct { + field reflect.StructField + val reflect.Value + } + fields := []field{} + for len(structs) > 0 { + structVal := structs[0] + structs = structs[1:] + + structType := structVal.Type() + for i := 0; i < structType.NumField(); i++ { + fieldType := structType.Field(i) + tagParts := strings.Split(fieldType.Tag.Get(tagName), ",") + + // Ignore fields with tag name "-" + if tagParts[0] == "-" { + continue + } + + if fieldType.Anonymous { + fieldKind := fieldType.Type.Kind() + if fieldKind != reflect.Struct { + return &parser.PosError{ + Pos: node.Pos(), + Err: fmt.Errorf("%s: unsupported type to struct: %s", + fieldType.Name, fieldKind), + } + } + + // We have an embedded field. We "squash" the fields down + // if specified in the tag. + squash := false + for _, tag := range tagParts[1:] { + if tag == "squash" { + squash = true + break + } + } + + if squash { + structs = append( + structs, result.FieldByName(fieldType.Name)) + continue + } + } + + // Normal struct field, store it away + fields = append(fields, field{fieldType, structVal.Field(i)}) + } + } + + usedKeys := make(map[string]struct{}) + decodedFields := make([]string, 0, len(fields)) + decodedFieldsVal := make([]reflect.Value, 0) + unusedKeysVal := make([]reflect.Value, 0) + for _, f := range fields { + field, fieldValue := f.field, f.val + if !fieldValue.IsValid() { + // This should never happen + panic("field is not valid") + } + + // If we can't set the field, then it is unexported or something, + // and we just continue onwards. + if !fieldValue.CanSet() { + continue + } + + fieldName := field.Name + + tagValue := field.Tag.Get(tagName) + tagParts := strings.SplitN(tagValue, ",", 2) + if len(tagParts) >= 2 { + switch tagParts[1] { + case "decodedFields": + decodedFieldsVal = append(decodedFieldsVal, fieldValue) + continue + case "key": + if item == nil { + return &parser.PosError{ + Pos: node.Pos(), + Err: fmt.Errorf("%s: %s asked for 'key', impossible", + name, fieldName), + } + } + + fieldValue.SetString(item.Keys[0].Token.Value().(string)) + continue + case "unusedKeys": + unusedKeysVal = append(unusedKeysVal, fieldValue) + continue + } + } + + if tagParts[0] != "" { + fieldName = tagParts[0] + } + + // Determine the element we'll use to decode. If it is a single + // match (only object with the field), then we decode it exactly. + // If it is a prefix match, then we decode the matches. + filter := list.Filter(fieldName) + + prefixMatches := filter.Children() + matches := filter.Elem() + if len(matches.Items) == 0 && len(prefixMatches.Items) == 0 { + continue + } + + // Track the used key + usedKeys[fieldName] = struct{}{} + + // Create the field name and decode. We range over the elements + // because we actually want the value. + fieldName = fmt.Sprintf("%s.%s", name, fieldName) + if len(prefixMatches.Items) > 0 { + if err := d.decode(fieldName, prefixMatches, fieldValue); err != nil { + return err + } + } + for _, match := range matches.Items { + var decodeNode ast.Node = match.Val + if ot, ok := decodeNode.(*ast.ObjectType); ok { + decodeNode = &ast.ObjectList{Items: ot.List.Items} + } + + if err := d.decode(fieldName, decodeNode, fieldValue); err != nil { + return err + } + } + + decodedFields = append(decodedFields, field.Name) + } + + if len(decodedFieldsVal) > 0 { + // Sort it so that it is deterministic + sort.Strings(decodedFields) + + for _, v := range decodedFieldsVal { + v.Set(reflect.ValueOf(decodedFields)) + } + } + + return nil +} + +// findNodeType returns the type of ast.Node +func findNodeType() reflect.Type { + var nodeContainer struct { + Node ast.Node + } + value := reflect.ValueOf(nodeContainer).FieldByName("Node") + return value.Type() +} diff --git a/vendor/github.com/hashicorp/hcl/go.mod b/vendor/github.com/hashicorp/hcl/go.mod new file mode 100644 index 000000000..4debbbe35 --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/go.mod @@ -0,0 +1,3 @@ +module github.com/hashicorp/hcl + +require github.com/davecgh/go-spew v1.1.1 diff --git a/vendor/github.com/hashicorp/hcl/go.sum b/vendor/github.com/hashicorp/hcl/go.sum new file mode 100644 index 000000000..b5e2922e8 --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/go.sum @@ -0,0 +1,2 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/vendor/github.com/hashicorp/hcl/hcl.go b/vendor/github.com/hashicorp/hcl/hcl.go new file mode 100644 index 000000000..575a20b50 --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/hcl.go @@ -0,0 +1,11 @@ +// Package hcl decodes HCL into usable Go structures. +// +// hcl input can come in either pure HCL format or JSON format. +// It can be parsed into an AST, and then decoded into a structure, +// or it can be decoded directly from a string into a structure. +// +// If you choose to parse HCL into a raw AST, the benefit is that you +// can write custom visitor implementations to implement custom +// semantic checks. By default, HCL does not perform any semantic +// checks. +package hcl diff --git a/vendor/github.com/hashicorp/hcl/hcl/ast/ast.go b/vendor/github.com/hashicorp/hcl/hcl/ast/ast.go new file mode 100644 index 000000000..6e5ef654b --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/hcl/ast/ast.go @@ -0,0 +1,219 @@ +// Package ast declares the types used to represent syntax trees for HCL +// (HashiCorp Configuration Language) +package ast + +import ( + "fmt" + "strings" + + "github.com/hashicorp/hcl/hcl/token" +) + +// Node is an element in the abstract syntax tree. +type Node interface { + node() + Pos() token.Pos +} + +func (File) node() {} +func (ObjectList) node() {} +func (ObjectKey) node() {} +func (ObjectItem) node() {} +func (Comment) node() {} +func (CommentGroup) node() {} +func (ObjectType) node() {} +func (LiteralType) node() {} +func (ListType) node() {} + +// File represents a single HCL file +type File struct { + Node Node // usually a *ObjectList + Comments []*CommentGroup // list of all comments in the source +} + +func (f *File) Pos() token.Pos { + return f.Node.Pos() +} + +// ObjectList represents a list of ObjectItems. An HCL file itself is an +// ObjectList. +type ObjectList struct { + Items []*ObjectItem +} + +func (o *ObjectList) Add(item *ObjectItem) { + o.Items = append(o.Items, item) +} + +// Filter filters out the objects with the given key list as a prefix. +// +// The returned list of objects contain ObjectItems where the keys have +// this prefix already stripped off. This might result in objects with +// zero-length key lists if they have no children. +// +// If no matches are found, an empty ObjectList (non-nil) is returned. +func (o *ObjectList) Filter(keys ...string) *ObjectList { + var result ObjectList + for _, item := range o.Items { + // If there aren't enough keys, then ignore this + if len(item.Keys) < len(keys) { + continue + } + + match := true + for i, key := range item.Keys[:len(keys)] { + key := key.Token.Value().(string) + if key != keys[i] && !strings.EqualFold(key, keys[i]) { + match = false + break + } + } + if !match { + continue + } + + // Strip off the prefix from the children + newItem := *item + newItem.Keys = newItem.Keys[len(keys):] + result.Add(&newItem) + } + + return &result +} + +// Children returns further nested objects (key length > 0) within this +// ObjectList. This should be used with Filter to get at child items. +func (o *ObjectList) Children() *ObjectList { + var result ObjectList + for _, item := range o.Items { + if len(item.Keys) > 0 { + result.Add(item) + } + } + + return &result +} + +// Elem returns items in the list that are direct element assignments +// (key length == 0). This should be used with Filter to get at elements. +func (o *ObjectList) Elem() *ObjectList { + var result ObjectList + for _, item := range o.Items { + if len(item.Keys) == 0 { + result.Add(item) + } + } + + return &result +} + +func (o *ObjectList) Pos() token.Pos { + // always returns the uninitiliazed position + return o.Items[0].Pos() +} + +// ObjectItem represents a HCL Object Item. An item is represented with a key +// (or keys). It can be an assignment or an object (both normal and nested) +type ObjectItem struct { + // keys is only one length long if it's of type assignment. If it's a + // nested object it can be larger than one. In that case "assign" is + // invalid as there is no assignments for a nested object. + Keys []*ObjectKey + + // assign contains the position of "=", if any + Assign token.Pos + + // val is the item itself. It can be an object,list, number, bool or a + // string. If key length is larger than one, val can be only of type + // Object. + Val Node + + LeadComment *CommentGroup // associated lead comment + LineComment *CommentGroup // associated line comment +} + +func (o *ObjectItem) Pos() token.Pos { + // I'm not entirely sure what causes this, but removing this causes + // a test failure. We should investigate at some point. + if len(o.Keys) == 0 { + return token.Pos{} + } + + return o.Keys[0].Pos() +} + +// ObjectKeys are either an identifier or of type string. +type ObjectKey struct { + Token token.Token +} + +func (o *ObjectKey) Pos() token.Pos { + return o.Token.Pos +} + +// LiteralType represents a literal of basic type. Valid types are: +// token.NUMBER, token.FLOAT, token.BOOL and token.STRING +type LiteralType struct { + Token token.Token + + // comment types, only used when in a list + LeadComment *CommentGroup + LineComment *CommentGroup +} + +func (l *LiteralType) Pos() token.Pos { + return l.Token.Pos +} + +// ListStatement represents a HCL List type +type ListType struct { + Lbrack token.Pos // position of "[" + Rbrack token.Pos // position of "]" + List []Node // the elements in lexical order +} + +func (l *ListType) Pos() token.Pos { + return l.Lbrack +} + +func (l *ListType) Add(node Node) { + l.List = append(l.List, node) +} + +// ObjectType represents a HCL Object Type +type ObjectType struct { + Lbrace token.Pos // position of "{" + Rbrace token.Pos // position of "}" + List *ObjectList // the nodes in lexical order +} + +func (o *ObjectType) Pos() token.Pos { + return o.Lbrace +} + +// Comment node represents a single //, # style or /*- style commment +type Comment struct { + Start token.Pos // position of / or # + Text string +} + +func (c *Comment) Pos() token.Pos { + return c.Start +} + +// CommentGroup node represents a sequence of comments with no other tokens and +// no empty lines between. +type CommentGroup struct { + List []*Comment // len(List) > 0 +} + +func (c *CommentGroup) Pos() token.Pos { + return c.List[0].Pos() +} + +//------------------------------------------------------------------- +// GoStringer +//------------------------------------------------------------------- + +func (o *ObjectKey) GoString() string { return fmt.Sprintf("*%#v", *o) } +func (o *ObjectList) GoString() string { return fmt.Sprintf("*%#v", *o) } diff --git a/vendor/github.com/hashicorp/hcl/hcl/ast/walk.go b/vendor/github.com/hashicorp/hcl/hcl/ast/walk.go new file mode 100644 index 000000000..ba07ad42b --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/hcl/ast/walk.go @@ -0,0 +1,52 @@ +package ast + +import "fmt" + +// WalkFunc describes a function to be called for each node during a Walk. The +// returned node can be used to rewrite the AST. Walking stops the returned +// bool is false. +type WalkFunc func(Node) (Node, bool) + +// Walk traverses an AST in depth-first order: It starts by calling fn(node); +// node must not be nil. If fn returns true, Walk invokes fn recursively for +// each of the non-nil children of node, followed by a call of fn(nil). The +// returned node of fn can be used to rewrite the passed node to fn. +func Walk(node Node, fn WalkFunc) Node { + rewritten, ok := fn(node) + if !ok { + return rewritten + } + + switch n := node.(type) { + case *File: + n.Node = Walk(n.Node, fn) + case *ObjectList: + for i, item := range n.Items { + n.Items[i] = Walk(item, fn).(*ObjectItem) + } + case *ObjectKey: + // nothing to do + case *ObjectItem: + for i, k := range n.Keys { + n.Keys[i] = Walk(k, fn).(*ObjectKey) + } + + if n.Val != nil { + n.Val = Walk(n.Val, fn) + } + case *LiteralType: + // nothing to do + case *ListType: + for i, l := range n.List { + n.List[i] = Walk(l, fn) + } + case *ObjectType: + n.List = Walk(n.List, fn).(*ObjectList) + default: + // should we panic here? + fmt.Printf("unknown type: %T\n", n) + } + + fn(nil) + return rewritten +} diff --git a/vendor/github.com/hashicorp/hcl/hcl/parser/error.go b/vendor/github.com/hashicorp/hcl/hcl/parser/error.go new file mode 100644 index 000000000..5c99381df --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/hcl/parser/error.go @@ -0,0 +1,17 @@ +package parser + +import ( + "fmt" + + "github.com/hashicorp/hcl/hcl/token" +) + +// PosError is a parse error that contains a position. +type PosError struct { + Pos token.Pos + Err error +} + +func (e *PosError) Error() string { + return fmt.Sprintf("At %s: %s", e.Pos, e.Err) +} diff --git a/vendor/github.com/hashicorp/hcl/hcl/parser/parser.go b/vendor/github.com/hashicorp/hcl/hcl/parser/parser.go new file mode 100644 index 000000000..64c83bcfb --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/hcl/parser/parser.go @@ -0,0 +1,532 @@ +// Package parser implements a parser for HCL (HashiCorp Configuration +// Language) +package parser + +import ( + "bytes" + "errors" + "fmt" + "strings" + + "github.com/hashicorp/hcl/hcl/ast" + "github.com/hashicorp/hcl/hcl/scanner" + "github.com/hashicorp/hcl/hcl/token" +) + +type Parser struct { + sc *scanner.Scanner + + // Last read token + tok token.Token + commaPrev token.Token + + comments []*ast.CommentGroup + leadComment *ast.CommentGroup // last lead comment + lineComment *ast.CommentGroup // last line comment + + enableTrace bool + indent int + n int // buffer size (max = 1) +} + +func newParser(src []byte) *Parser { + return &Parser{ + sc: scanner.New(src), + } +} + +// Parse returns the fully parsed source and returns the abstract syntax tree. +func Parse(src []byte) (*ast.File, error) { + // normalize all line endings + // since the scanner and output only work with "\n" line endings, we may + // end up with dangling "\r" characters in the parsed data. + src = bytes.Replace(src, []byte("\r\n"), []byte("\n"), -1) + + p := newParser(src) + return p.Parse() +} + +var errEofToken = errors.New("EOF token found") + +// Parse returns the fully parsed source and returns the abstract syntax tree. +func (p *Parser) Parse() (*ast.File, error) { + f := &ast.File{} + var err, scerr error + p.sc.Error = func(pos token.Pos, msg string) { + scerr = &PosError{Pos: pos, Err: errors.New(msg)} + } + + f.Node, err = p.objectList(false) + if scerr != nil { + return nil, scerr + } + if err != nil { + return nil, err + } + + f.Comments = p.comments + return f, nil +} + +// objectList parses a list of items within an object (generally k/v pairs). +// The parameter" obj" tells this whether to we are within an object (braces: +// '{', '}') or just at the top level. If we're within an object, we end +// at an RBRACE. +func (p *Parser) objectList(obj bool) (*ast.ObjectList, error) { + defer un(trace(p, "ParseObjectList")) + node := &ast.ObjectList{} + + for { + if obj { + tok := p.scan() + p.unscan() + if tok.Type == token.RBRACE { + break + } + } + + n, err := p.objectItem() + if err == errEofToken { + break // we are finished + } + + // we don't return a nil node, because might want to use already + // collected items. + if err != nil { + return node, err + } + + node.Add(n) + + // object lists can be optionally comma-delimited e.g. when a list of maps + // is being expressed, so a comma is allowed here - it's simply consumed + tok := p.scan() + if tok.Type != token.COMMA { + p.unscan() + } + } + return node, nil +} + +func (p *Parser) consumeComment() (comment *ast.Comment, endline int) { + endline = p.tok.Pos.Line + + // count the endline if it's multiline comment, ie starting with /* + if len(p.tok.Text) > 1 && p.tok.Text[1] == '*' { + // don't use range here - no need to decode Unicode code points + for i := 0; i < len(p.tok.Text); i++ { + if p.tok.Text[i] == '\n' { + endline++ + } + } + } + + comment = &ast.Comment{Start: p.tok.Pos, Text: p.tok.Text} + p.tok = p.sc.Scan() + return +} + +func (p *Parser) consumeCommentGroup(n int) (comments *ast.CommentGroup, endline int) { + var list []*ast.Comment + endline = p.tok.Pos.Line + + for p.tok.Type == token.COMMENT && p.tok.Pos.Line <= endline+n { + var comment *ast.Comment + comment, endline = p.consumeComment() + list = append(list, comment) + } + + // add comment group to the comments list + comments = &ast.CommentGroup{List: list} + p.comments = append(p.comments, comments) + + return +} + +// objectItem parses a single object item +func (p *Parser) objectItem() (*ast.ObjectItem, error) { + defer un(trace(p, "ParseObjectItem")) + + keys, err := p.objectKey() + if len(keys) > 0 && err == errEofToken { + // We ignore eof token here since it is an error if we didn't + // receive a value (but we did receive a key) for the item. + err = nil + } + if len(keys) > 0 && err != nil && p.tok.Type == token.RBRACE { + // This is a strange boolean statement, but what it means is: + // We have keys with no value, and we're likely in an object + // (since RBrace ends an object). For this, we set err to nil so + // we continue and get the error below of having the wrong value + // type. + err = nil + + // Reset the token type so we don't think it completed fine. See + // objectType which uses p.tok.Type to check if we're done with + // the object. + p.tok.Type = token.EOF + } + if err != nil { + return nil, err + } + + o := &ast.ObjectItem{ + Keys: keys, + } + + if p.leadComment != nil { + o.LeadComment = p.leadComment + p.leadComment = nil + } + + switch p.tok.Type { + case token.ASSIGN: + o.Assign = p.tok.Pos + o.Val, err = p.object() + if err != nil { + return nil, err + } + case token.LBRACE: + o.Val, err = p.objectType() + if err != nil { + return nil, err + } + default: + keyStr := make([]string, 0, len(keys)) + for _, k := range keys { + keyStr = append(keyStr, k.Token.Text) + } + + return nil, &PosError{ + Pos: p.tok.Pos, + Err: fmt.Errorf( + "key '%s' expected start of object ('{') or assignment ('=')", + strings.Join(keyStr, " ")), + } + } + + // key=#comment + // val + if p.lineComment != nil { + o.LineComment, p.lineComment = p.lineComment, nil + } + + // do a look-ahead for line comment + p.scan() + if len(keys) > 0 && o.Val.Pos().Line == keys[0].Pos().Line && p.lineComment != nil { + o.LineComment = p.lineComment + p.lineComment = nil + } + p.unscan() + return o, nil +} + +// objectKey parses an object key and returns a ObjectKey AST +func (p *Parser) objectKey() ([]*ast.ObjectKey, error) { + keyCount := 0 + keys := make([]*ast.ObjectKey, 0) + + for { + tok := p.scan() + switch tok.Type { + case token.EOF: + // It is very important to also return the keys here as well as + // the error. This is because we need to be able to tell if we + // did parse keys prior to finding the EOF, or if we just found + // a bare EOF. + return keys, errEofToken + case token.ASSIGN: + // assignment or object only, but not nested objects. this is not + // allowed: `foo bar = {}` + if keyCount > 1 { + return nil, &PosError{ + Pos: p.tok.Pos, + Err: fmt.Errorf("nested object expected: LBRACE got: %s", p.tok.Type), + } + } + + if keyCount == 0 { + return nil, &PosError{ + Pos: p.tok.Pos, + Err: errors.New("no object keys found!"), + } + } + + return keys, nil + case token.LBRACE: + var err error + + // If we have no keys, then it is a syntax error. i.e. {{}} is not + // allowed. + if len(keys) == 0 { + err = &PosError{ + Pos: p.tok.Pos, + Err: fmt.Errorf("expected: IDENT | STRING got: %s", p.tok.Type), + } + } + + // object + return keys, err + case token.IDENT, token.STRING: + keyCount++ + keys = append(keys, &ast.ObjectKey{Token: p.tok}) + case token.ILLEGAL: + return keys, &PosError{ + Pos: p.tok.Pos, + Err: fmt.Errorf("illegal character"), + } + default: + return keys, &PosError{ + Pos: p.tok.Pos, + Err: fmt.Errorf("expected: IDENT | STRING | ASSIGN | LBRACE got: %s", p.tok.Type), + } + } + } +} + +// object parses any type of object, such as number, bool, string, object or +// list. +func (p *Parser) object() (ast.Node, error) { + defer un(trace(p, "ParseType")) + tok := p.scan() + + switch tok.Type { + case token.NUMBER, token.FLOAT, token.BOOL, token.STRING, token.HEREDOC: + return p.literalType() + case token.LBRACE: + return p.objectType() + case token.LBRACK: + return p.listType() + case token.COMMENT: + // implement comment + case token.EOF: + return nil, errEofToken + } + + return nil, &PosError{ + Pos: tok.Pos, + Err: fmt.Errorf("Unknown token: %+v", tok), + } +} + +// objectType parses an object type and returns a ObjectType AST +func (p *Parser) objectType() (*ast.ObjectType, error) { + defer un(trace(p, "ParseObjectType")) + + // we assume that the currently scanned token is a LBRACE + o := &ast.ObjectType{ + Lbrace: p.tok.Pos, + } + + l, err := p.objectList(true) + + // if we hit RBRACE, we are good to go (means we parsed all Items), if it's + // not a RBRACE, it's an syntax error and we just return it. + if err != nil && p.tok.Type != token.RBRACE { + return nil, err + } + + // No error, scan and expect the ending to be a brace + if tok := p.scan(); tok.Type != token.RBRACE { + return nil, &PosError{ + Pos: tok.Pos, + Err: fmt.Errorf("object expected closing RBRACE got: %s", tok.Type), + } + } + + o.List = l + o.Rbrace = p.tok.Pos // advanced via parseObjectList + return o, nil +} + +// listType parses a list type and returns a ListType AST +func (p *Parser) listType() (*ast.ListType, error) { + defer un(trace(p, "ParseListType")) + + // we assume that the currently scanned token is a LBRACK + l := &ast.ListType{ + Lbrack: p.tok.Pos, + } + + needComma := false + for { + tok := p.scan() + if needComma { + switch tok.Type { + case token.COMMA, token.RBRACK: + default: + return nil, &PosError{ + Pos: tok.Pos, + Err: fmt.Errorf( + "error parsing list, expected comma or list end, got: %s", + tok.Type), + } + } + } + switch tok.Type { + case token.BOOL, token.NUMBER, token.FLOAT, token.STRING, token.HEREDOC: + node, err := p.literalType() + if err != nil { + return nil, err + } + + // If there is a lead comment, apply it + if p.leadComment != nil { + node.LeadComment = p.leadComment + p.leadComment = nil + } + + l.Add(node) + needComma = true + case token.COMMA: + // get next list item or we are at the end + // do a look-ahead for line comment + p.scan() + if p.lineComment != nil && len(l.List) > 0 { + lit, ok := l.List[len(l.List)-1].(*ast.LiteralType) + if ok { + lit.LineComment = p.lineComment + l.List[len(l.List)-1] = lit + p.lineComment = nil + } + } + p.unscan() + + needComma = false + continue + case token.LBRACE: + // Looks like a nested object, so parse it out + node, err := p.objectType() + if err != nil { + return nil, &PosError{ + Pos: tok.Pos, + Err: fmt.Errorf( + "error while trying to parse object within list: %s", err), + } + } + l.Add(node) + needComma = true + case token.LBRACK: + node, err := p.listType() + if err != nil { + return nil, &PosError{ + Pos: tok.Pos, + Err: fmt.Errorf( + "error while trying to parse list within list: %s", err), + } + } + l.Add(node) + case token.RBRACK: + // finished + l.Rbrack = p.tok.Pos + return l, nil + default: + return nil, &PosError{ + Pos: tok.Pos, + Err: fmt.Errorf("unexpected token while parsing list: %s", tok.Type), + } + } + } +} + +// literalType parses a literal type and returns a LiteralType AST +func (p *Parser) literalType() (*ast.LiteralType, error) { + defer un(trace(p, "ParseLiteral")) + + return &ast.LiteralType{ + Token: p.tok, + }, nil +} + +// scan returns the next token from the underlying scanner. If a token has +// been unscanned then read that instead. In the process, it collects any +// comment groups encountered, and remembers the last lead and line comments. +func (p *Parser) scan() token.Token { + // If we have a token on the buffer, then return it. + if p.n != 0 { + p.n = 0 + return p.tok + } + + // Otherwise read the next token from the scanner and Save it to the buffer + // in case we unscan later. + prev := p.tok + p.tok = p.sc.Scan() + + if p.tok.Type == token.COMMENT { + var comment *ast.CommentGroup + var endline int + + // fmt.Printf("p.tok.Pos.Line = %+v prev: %d endline %d \n", + // p.tok.Pos.Line, prev.Pos.Line, endline) + if p.tok.Pos.Line == prev.Pos.Line { + // The comment is on same line as the previous token; it + // cannot be a lead comment but may be a line comment. + comment, endline = p.consumeCommentGroup(0) + if p.tok.Pos.Line != endline { + // The next token is on a different line, thus + // the last comment group is a line comment. + p.lineComment = comment + } + } + + // consume successor comments, if any + endline = -1 + for p.tok.Type == token.COMMENT { + comment, endline = p.consumeCommentGroup(1) + } + + if endline+1 == p.tok.Pos.Line && p.tok.Type != token.RBRACE { + switch p.tok.Type { + case token.RBRACE, token.RBRACK: + // Do not count for these cases + default: + // The next token is following on the line immediately after the + // comment group, thus the last comment group is a lead comment. + p.leadComment = comment + } + } + + } + + return p.tok +} + +// unscan pushes the previously read token back onto the buffer. +func (p *Parser) unscan() { + p.n = 1 +} + +// ---------------------------------------------------------------------------- +// Parsing support + +func (p *Parser) printTrace(a ...interface{}) { + if !p.enableTrace { + return + } + + const dots = ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " + const n = len(dots) + fmt.Printf("%5d:%3d: ", p.tok.Pos.Line, p.tok.Pos.Column) + + i := 2 * p.indent + for i > n { + fmt.Print(dots) + i -= n + } + // i <= n + fmt.Print(dots[0:i]) + fmt.Println(a...) +} + +func trace(p *Parser, msg string) *Parser { + p.printTrace(msg, "(") + p.indent++ + return p +} + +// Usage pattern: defer un(trace(p, "...")) +func un(p *Parser) { + p.indent-- + p.printTrace(")") +} diff --git a/vendor/github.com/hashicorp/hcl/hcl/printer/nodes.go b/vendor/github.com/hashicorp/hcl/hcl/printer/nodes.go new file mode 100644 index 000000000..7c038d12a --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/hcl/printer/nodes.go @@ -0,0 +1,789 @@ +package printer + +import ( + "bytes" + "fmt" + "sort" + + "github.com/hashicorp/hcl/hcl/ast" + "github.com/hashicorp/hcl/hcl/token" +) + +const ( + blank = byte(' ') + newline = byte('\n') + tab = byte('\t') + infinity = 1 << 30 // offset or line +) + +var ( + unindent = []byte("\uE123") // in the private use space +) + +type printer struct { + cfg Config + prev token.Pos + + comments []*ast.CommentGroup // may be nil, contains all comments + standaloneComments []*ast.CommentGroup // contains all standalone comments (not assigned to any node) + + enableTrace bool + indentTrace int +} + +type ByPosition []*ast.CommentGroup + +func (b ByPosition) Len() int { return len(b) } +func (b ByPosition) Swap(i, j int) { b[i], b[j] = b[j], b[i] } +func (b ByPosition) Less(i, j int) bool { return b[i].Pos().Before(b[j].Pos()) } + +// collectComments comments all standalone comments which are not lead or line +// comment +func (p *printer) collectComments(node ast.Node) { + // first collect all comments. This is already stored in + // ast.File.(comments) + ast.Walk(node, func(nn ast.Node) (ast.Node, bool) { + switch t := nn.(type) { + case *ast.File: + p.comments = t.Comments + return nn, false + } + return nn, true + }) + + standaloneComments := make(map[token.Pos]*ast.CommentGroup, 0) + for _, c := range p.comments { + standaloneComments[c.Pos()] = c + } + + // next remove all lead and line comments from the overall comment map. + // This will give us comments which are standalone, comments which are not + // assigned to any kind of node. + ast.Walk(node, func(nn ast.Node) (ast.Node, bool) { + switch t := nn.(type) { + case *ast.LiteralType: + if t.LeadComment != nil { + for _, comment := range t.LeadComment.List { + if _, ok := standaloneComments[comment.Pos()]; ok { + delete(standaloneComments, comment.Pos()) + } + } + } + + if t.LineComment != nil { + for _, comment := range t.LineComment.List { + if _, ok := standaloneComments[comment.Pos()]; ok { + delete(standaloneComments, comment.Pos()) + } + } + } + case *ast.ObjectItem: + if t.LeadComment != nil { + for _, comment := range t.LeadComment.List { + if _, ok := standaloneComments[comment.Pos()]; ok { + delete(standaloneComments, comment.Pos()) + } + } + } + + if t.LineComment != nil { + for _, comment := range t.LineComment.List { + if _, ok := standaloneComments[comment.Pos()]; ok { + delete(standaloneComments, comment.Pos()) + } + } + } + } + + return nn, true + }) + + for _, c := range standaloneComments { + p.standaloneComments = append(p.standaloneComments, c) + } + + sort.Sort(ByPosition(p.standaloneComments)) +} + +// output prints creates b printable HCL output and returns it. +func (p *printer) output(n interface{}) []byte { + var buf bytes.Buffer + + switch t := n.(type) { + case *ast.File: + // File doesn't trace so we add the tracing here + defer un(trace(p, "File")) + return p.output(t.Node) + case *ast.ObjectList: + defer un(trace(p, "ObjectList")) + + var index int + for { + // Determine the location of the next actual non-comment + // item. If we're at the end, the next item is at "infinity" + var nextItem token.Pos + if index != len(t.Items) { + nextItem = t.Items[index].Pos() + } else { + nextItem = token.Pos{Offset: infinity, Line: infinity} + } + + // Go through the standalone comments in the file and print out + // the comments that we should be for this object item. + for _, c := range p.standaloneComments { + // Go through all the comments in the group. The group + // should be printed together, not separated by double newlines. + printed := false + newlinePrinted := false + for _, comment := range c.List { + // We only care about comments after the previous item + // we've printed so that comments are printed in the + // correct locations (between two objects for example). + // And before the next item. + if comment.Pos().After(p.prev) && comment.Pos().Before(nextItem) { + // if we hit the end add newlines so we can print the comment + // we don't do this if prev is invalid which means the + // beginning of the file since the first comment should + // be at the first line. + if !newlinePrinted && p.prev.IsValid() && index == len(t.Items) { + buf.Write([]byte{newline, newline}) + newlinePrinted = true + } + + // Write the actual comment. + buf.WriteString(comment.Text) + buf.WriteByte(newline) + + // Set printed to true to note that we printed something + printed = true + } + } + + // If we're not at the last item, write a new line so + // that there is a newline separating this comment from + // the next object. + if printed && index != len(t.Items) { + buf.WriteByte(newline) + } + } + + if index == len(t.Items) { + break + } + + buf.Write(p.output(t.Items[index])) + if index != len(t.Items)-1 { + // Always write a newline to separate us from the next item + buf.WriteByte(newline) + + // Need to determine if we're going to separate the next item + // with a blank line. The logic here is simple, though there + // are a few conditions: + // + // 1. The next object is more than one line away anyways, + // so we need an empty line. + // + // 2. The next object is not a "single line" object, so + // we need an empty line. + // + // 3. This current object is not a single line object, + // so we need an empty line. + current := t.Items[index] + next := t.Items[index+1] + if next.Pos().Line != t.Items[index].Pos().Line+1 || + !p.isSingleLineObject(next) || + !p.isSingleLineObject(current) { + buf.WriteByte(newline) + } + } + index++ + } + case *ast.ObjectKey: + buf.WriteString(t.Token.Text) + case *ast.ObjectItem: + p.prev = t.Pos() + buf.Write(p.objectItem(t)) + case *ast.LiteralType: + buf.Write(p.literalType(t)) + case *ast.ListType: + buf.Write(p.list(t)) + case *ast.ObjectType: + buf.Write(p.objectType(t)) + default: + fmt.Printf(" unknown type: %T\n", n) + } + + return buf.Bytes() +} + +func (p *printer) literalType(lit *ast.LiteralType) []byte { + result := []byte(lit.Token.Text) + switch lit.Token.Type { + case token.HEREDOC: + // Clear the trailing newline from heredocs + if result[len(result)-1] == '\n' { + result = result[:len(result)-1] + } + + // Poison lines 2+ so that we don't indent them + result = p.heredocIndent(result) + case token.STRING: + // If this is a multiline string, poison lines 2+ so we don't + // indent them. + if bytes.IndexRune(result, '\n') >= 0 { + result = p.heredocIndent(result) + } + } + + return result +} + +// objectItem returns the printable HCL form of an object item. An object type +// starts with one/multiple keys and has a value. The value might be of any +// type. +func (p *printer) objectItem(o *ast.ObjectItem) []byte { + defer un(trace(p, fmt.Sprintf("ObjectItem: %s", o.Keys[0].Token.Text))) + var buf bytes.Buffer + + if o.LeadComment != nil { + for _, comment := range o.LeadComment.List { + buf.WriteString(comment.Text) + buf.WriteByte(newline) + } + } + + // If key and val are on different lines, treat line comments like lead comments. + if o.LineComment != nil && o.Val.Pos().Line != o.Keys[0].Pos().Line { + for _, comment := range o.LineComment.List { + buf.WriteString(comment.Text) + buf.WriteByte(newline) + } + } + + for i, k := range o.Keys { + buf.WriteString(k.Token.Text) + buf.WriteByte(blank) + + // reach end of key + if o.Assign.IsValid() && i == len(o.Keys)-1 && len(o.Keys) == 1 { + buf.WriteString("=") + buf.WriteByte(blank) + } + } + + buf.Write(p.output(o.Val)) + + if o.LineComment != nil && o.Val.Pos().Line == o.Keys[0].Pos().Line { + buf.WriteByte(blank) + for _, comment := range o.LineComment.List { + buf.WriteString(comment.Text) + } + } + + return buf.Bytes() +} + +// objectType returns the printable HCL form of an object type. An object type +// begins with a brace and ends with a brace. +func (p *printer) objectType(o *ast.ObjectType) []byte { + defer un(trace(p, "ObjectType")) + var buf bytes.Buffer + buf.WriteString("{") + + var index int + var nextItem token.Pos + var commented, newlinePrinted bool + for { + // Determine the location of the next actual non-comment + // item. If we're at the end, the next item is the closing brace + if index != len(o.List.Items) { + nextItem = o.List.Items[index].Pos() + } else { + nextItem = o.Rbrace + } + + // Go through the standalone comments in the file and print out + // the comments that we should be for this object item. + for _, c := range p.standaloneComments { + printed := false + var lastCommentPos token.Pos + for _, comment := range c.List { + // We only care about comments after the previous item + // we've printed so that comments are printed in the + // correct locations (between two objects for example). + // And before the next item. + if comment.Pos().After(p.prev) && comment.Pos().Before(nextItem) { + // If there are standalone comments and the initial newline has not + // been printed yet, do it now. + if !newlinePrinted { + newlinePrinted = true + buf.WriteByte(newline) + } + + // add newline if it's between other printed nodes + if index > 0 { + commented = true + buf.WriteByte(newline) + } + + // Store this position + lastCommentPos = comment.Pos() + + // output the comment itself + buf.Write(p.indent(p.heredocIndent([]byte(comment.Text)))) + + // Set printed to true to note that we printed something + printed = true + + /* + if index != len(o.List.Items) { + buf.WriteByte(newline) // do not print on the end + } + */ + } + } + + // Stuff to do if we had comments + if printed { + // Always write a newline + buf.WriteByte(newline) + + // If there is another item in the object and our comment + // didn't hug it directly, then make sure there is a blank + // line separating them. + if nextItem != o.Rbrace && nextItem.Line != lastCommentPos.Line+1 { + buf.WriteByte(newline) + } + } + } + + if index == len(o.List.Items) { + p.prev = o.Rbrace + break + } + + // At this point we are sure that it's not a totally empty block: print + // the initial newline if it hasn't been printed yet by the previous + // block about standalone comments. + if !newlinePrinted { + buf.WriteByte(newline) + newlinePrinted = true + } + + // check if we have adjacent one liner items. If yes we'll going to align + // the comments. + var aligned []*ast.ObjectItem + for _, item := range o.List.Items[index:] { + // we don't group one line lists + if len(o.List.Items) == 1 { + break + } + + // one means a oneliner with out any lead comment + // two means a oneliner with lead comment + // anything else might be something else + cur := lines(string(p.objectItem(item))) + if cur > 2 { + break + } + + curPos := item.Pos() + + nextPos := token.Pos{} + if index != len(o.List.Items)-1 { + nextPos = o.List.Items[index+1].Pos() + } + + prevPos := token.Pos{} + if index != 0 { + prevPos = o.List.Items[index-1].Pos() + } + + // fmt.Println("DEBUG ----------------") + // fmt.Printf("prev = %+v prevPos: %s\n", prev, prevPos) + // fmt.Printf("cur = %+v curPos: %s\n", cur, curPos) + // fmt.Printf("next = %+v nextPos: %s\n", next, nextPos) + + if curPos.Line+1 == nextPos.Line { + aligned = append(aligned, item) + index++ + continue + } + + if curPos.Line-1 == prevPos.Line { + aligned = append(aligned, item) + index++ + + // finish if we have a new line or comment next. This happens + // if the next item is not adjacent + if curPos.Line+1 != nextPos.Line { + break + } + continue + } + + break + } + + // put newlines if the items are between other non aligned items. + // newlines are also added if there is a standalone comment already, so + // check it too + if !commented && index != len(aligned) { + buf.WriteByte(newline) + } + + if len(aligned) >= 1 { + p.prev = aligned[len(aligned)-1].Pos() + + items := p.alignedItems(aligned) + buf.Write(p.indent(items)) + } else { + p.prev = o.List.Items[index].Pos() + + buf.Write(p.indent(p.objectItem(o.List.Items[index]))) + index++ + } + + buf.WriteByte(newline) + } + + buf.WriteString("}") + return buf.Bytes() +} + +func (p *printer) alignedItems(items []*ast.ObjectItem) []byte { + var buf bytes.Buffer + + // find the longest key and value length, needed for alignment + var longestKeyLen int // longest key length + var longestValLen int // longest value length + for _, item := range items { + key := len(item.Keys[0].Token.Text) + val := len(p.output(item.Val)) + + if key > longestKeyLen { + longestKeyLen = key + } + + if val > longestValLen { + longestValLen = val + } + } + + for i, item := range items { + if item.LeadComment != nil { + for _, comment := range item.LeadComment.List { + buf.WriteString(comment.Text) + buf.WriteByte(newline) + } + } + + for i, k := range item.Keys { + keyLen := len(k.Token.Text) + buf.WriteString(k.Token.Text) + for i := 0; i < longestKeyLen-keyLen+1; i++ { + buf.WriteByte(blank) + } + + // reach end of key + if i == len(item.Keys)-1 && len(item.Keys) == 1 { + buf.WriteString("=") + buf.WriteByte(blank) + } + } + + val := p.output(item.Val) + valLen := len(val) + buf.Write(val) + + if item.Val.Pos().Line == item.Keys[0].Pos().Line && item.LineComment != nil { + for i := 0; i < longestValLen-valLen+1; i++ { + buf.WriteByte(blank) + } + + for _, comment := range item.LineComment.List { + buf.WriteString(comment.Text) + } + } + + // do not print for the last item + if i != len(items)-1 { + buf.WriteByte(newline) + } + } + + return buf.Bytes() +} + +// list returns the printable HCL form of an list type. +func (p *printer) list(l *ast.ListType) []byte { + if p.isSingleLineList(l) { + return p.singleLineList(l) + } + + var buf bytes.Buffer + buf.WriteString("[") + buf.WriteByte(newline) + + var longestLine int + for _, item := range l.List { + // for now we assume that the list only contains literal types + if lit, ok := item.(*ast.LiteralType); ok { + lineLen := len(lit.Token.Text) + if lineLen > longestLine { + longestLine = lineLen + } + } + } + + haveEmptyLine := false + for i, item := range l.List { + // If we have a lead comment, then we want to write that first + leadComment := false + if lit, ok := item.(*ast.LiteralType); ok && lit.LeadComment != nil { + leadComment = true + + // Ensure an empty line before every element with a + // lead comment (except the first item in a list). + if !haveEmptyLine && i != 0 { + buf.WriteByte(newline) + } + + for _, comment := range lit.LeadComment.List { + buf.Write(p.indent([]byte(comment.Text))) + buf.WriteByte(newline) + } + } + + // also indent each line + val := p.output(item) + curLen := len(val) + buf.Write(p.indent(val)) + + // if this item is a heredoc, then we output the comma on + // the next line. This is the only case this happens. + comma := []byte{','} + if lit, ok := item.(*ast.LiteralType); ok && lit.Token.Type == token.HEREDOC { + buf.WriteByte(newline) + comma = p.indent(comma) + } + + buf.Write(comma) + + if lit, ok := item.(*ast.LiteralType); ok && lit.LineComment != nil { + // if the next item doesn't have any comments, do not align + buf.WriteByte(blank) // align one space + for i := 0; i < longestLine-curLen; i++ { + buf.WriteByte(blank) + } + + for _, comment := range lit.LineComment.List { + buf.WriteString(comment.Text) + } + } + + buf.WriteByte(newline) + + // Ensure an empty line after every element with a + // lead comment (except the first item in a list). + haveEmptyLine = leadComment && i != len(l.List)-1 + if haveEmptyLine { + buf.WriteByte(newline) + } + } + + buf.WriteString("]") + return buf.Bytes() +} + +// isSingleLineList returns true if: +// * they were previously formatted entirely on one line +// * they consist entirely of literals +// * there are either no heredoc strings or the list has exactly one element +// * there are no line comments +func (printer) isSingleLineList(l *ast.ListType) bool { + for _, item := range l.List { + if item.Pos().Line != l.Lbrack.Line { + return false + } + + lit, ok := item.(*ast.LiteralType) + if !ok { + return false + } + + if lit.Token.Type == token.HEREDOC && len(l.List) != 1 { + return false + } + + if lit.LineComment != nil { + return false + } + } + + return true +} + +// singleLineList prints a simple single line list. +// For a definition of "simple", see isSingleLineList above. +func (p *printer) singleLineList(l *ast.ListType) []byte { + buf := &bytes.Buffer{} + + buf.WriteString("[") + for i, item := range l.List { + if i != 0 { + buf.WriteString(", ") + } + + // Output the item itself + buf.Write(p.output(item)) + + // The heredoc marker needs to be at the end of line. + if lit, ok := item.(*ast.LiteralType); ok && lit.Token.Type == token.HEREDOC { + buf.WriteByte(newline) + } + } + + buf.WriteString("]") + return buf.Bytes() +} + +// indent indents the lines of the given buffer for each non-empty line +func (p *printer) indent(buf []byte) []byte { + var prefix []byte + if p.cfg.SpacesWidth != 0 { + for i := 0; i < p.cfg.SpacesWidth; i++ { + prefix = append(prefix, blank) + } + } else { + prefix = []byte{tab} + } + + var res []byte + bol := true + for _, c := range buf { + if bol && c != '\n' { + res = append(res, prefix...) + } + + res = append(res, c) + bol = c == '\n' + } + return res +} + +// unindent removes all the indentation from the tombstoned lines +func (p *printer) unindent(buf []byte) []byte { + var res []byte + for i := 0; i < len(buf); i++ { + skip := len(buf)-i <= len(unindent) + if !skip { + skip = !bytes.Equal(unindent, buf[i:i+len(unindent)]) + } + if skip { + res = append(res, buf[i]) + continue + } + + // We have a marker. we have to backtrace here and clean out + // any whitespace ahead of our tombstone up to a \n + for j := len(res) - 1; j >= 0; j-- { + if res[j] == '\n' { + break + } + + res = res[:j] + } + + // Skip the entire unindent marker + i += len(unindent) - 1 + } + + return res +} + +// heredocIndent marks all the 2nd and further lines as unindentable +func (p *printer) heredocIndent(buf []byte) []byte { + var res []byte + bol := false + for _, c := range buf { + if bol && c != '\n' { + res = append(res, unindent...) + } + res = append(res, c) + bol = c == '\n' + } + return res +} + +// isSingleLineObject tells whether the given object item is a single +// line object such as "obj {}". +// +// A single line object: +// +// * has no lead comments (hence multi-line) +// * has no assignment +// * has no values in the stanza (within {}) +// +func (p *printer) isSingleLineObject(val *ast.ObjectItem) bool { + // If there is a lead comment, can't be one line + if val.LeadComment != nil { + return false + } + + // If there is assignment, we always break by line + if val.Assign.IsValid() { + return false + } + + // If it isn't an object type, then its not a single line object + ot, ok := val.Val.(*ast.ObjectType) + if !ok { + return false + } + + // If the object has no items, it is single line! + return len(ot.List.Items) == 0 +} + +func lines(txt string) int { + endline := 1 + for i := 0; i < len(txt); i++ { + if txt[i] == '\n' { + endline++ + } + } + return endline +} + +// ---------------------------------------------------------------------------- +// Tracing support + +func (p *printer) printTrace(a ...interface{}) { + if !p.enableTrace { + return + } + + const dots = ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " + const n = len(dots) + i := 2 * p.indentTrace + for i > n { + fmt.Print(dots) + i -= n + } + // i <= n + fmt.Print(dots[0:i]) + fmt.Println(a...) +} + +func trace(p *printer, msg string) *printer { + p.printTrace(msg, "(") + p.indentTrace++ + return p +} + +// Usage pattern: defer un(trace(p, "...")) +func un(p *printer) { + p.indentTrace-- + p.printTrace(")") +} diff --git a/vendor/github.com/hashicorp/hcl/hcl/printer/printer.go b/vendor/github.com/hashicorp/hcl/hcl/printer/printer.go new file mode 100644 index 000000000..6617ab8e7 --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/hcl/printer/printer.go @@ -0,0 +1,66 @@ +// Package printer implements printing of AST nodes to HCL format. +package printer + +import ( + "bytes" + "io" + "text/tabwriter" + + "github.com/hashicorp/hcl/hcl/ast" + "github.com/hashicorp/hcl/hcl/parser" +) + +var DefaultConfig = Config{ + SpacesWidth: 2, +} + +// A Config node controls the output of Fprint. +type Config struct { + SpacesWidth int // if set, it will use spaces instead of tabs for alignment +} + +func (c *Config) Fprint(output io.Writer, node ast.Node) error { + p := &printer{ + cfg: *c, + comments: make([]*ast.CommentGroup, 0), + standaloneComments: make([]*ast.CommentGroup, 0), + // enableTrace: true, + } + + p.collectComments(node) + + if _, err := output.Write(p.unindent(p.output(node))); err != nil { + return err + } + + // flush tabwriter, if any + var err error + if tw, _ := output.(*tabwriter.Writer); tw != nil { + err = tw.Flush() + } + + return err +} + +// Fprint "pretty-prints" an HCL node to output +// It calls Config.Fprint with default settings. +func Fprint(output io.Writer, node ast.Node) error { + return DefaultConfig.Fprint(output, node) +} + +// Format formats src HCL and returns the result. +func Format(src []byte) ([]byte, error) { + node, err := parser.Parse(src) + if err != nil { + return nil, err + } + + var buf bytes.Buffer + if err := DefaultConfig.Fprint(&buf, node); err != nil { + return nil, err + } + + // Add trailing newline to result + buf.WriteString("\n") + return buf.Bytes(), nil +} diff --git a/vendor/github.com/hashicorp/hcl/hcl/scanner/scanner.go b/vendor/github.com/hashicorp/hcl/hcl/scanner/scanner.go new file mode 100644 index 000000000..624a18fe3 --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/hcl/scanner/scanner.go @@ -0,0 +1,652 @@ +// Package scanner implements a scanner for HCL (HashiCorp Configuration +// Language) source text. +package scanner + +import ( + "bytes" + "fmt" + "os" + "regexp" + "unicode" + "unicode/utf8" + + "github.com/hashicorp/hcl/hcl/token" +) + +// eof represents a marker rune for the end of the reader. +const eof = rune(0) + +// Scanner defines a lexical scanner +type Scanner struct { + buf *bytes.Buffer // Source buffer for advancing and scanning + src []byte // Source buffer for immutable access + + // Source Position + srcPos token.Pos // current position + prevPos token.Pos // previous position, used for peek() method + + lastCharLen int // length of last character in bytes + lastLineLen int // length of last line in characters (for correct column reporting) + + tokStart int // token text start position + tokEnd int // token text end position + + // Error is called for each error encountered. If no Error + // function is set, the error is reported to os.Stderr. + Error func(pos token.Pos, msg string) + + // ErrorCount is incremented by one for each error encountered. + ErrorCount int + + // tokPos is the start position of most recently scanned token; set by + // Scan. The Filename field is always left untouched by the Scanner. If + // an error is reported (via Error) and Position is invalid, the scanner is + // not inside a token. + tokPos token.Pos +} + +// New creates and initializes a new instance of Scanner using src as +// its source content. +func New(src []byte) *Scanner { + // even though we accept a src, we read from a io.Reader compatible type + // (*bytes.Buffer). So in the future we might easily change it to streaming + // read. + b := bytes.NewBuffer(src) + s := &Scanner{ + buf: b, + src: src, + } + + // srcPosition always starts with 1 + s.srcPos.Line = 1 + return s +} + +// next reads the next rune from the bufferred reader. Returns the rune(0) if +// an error occurs (or io.EOF is returned). +func (s *Scanner) next() rune { + ch, size, err := s.buf.ReadRune() + if err != nil { + // advance for error reporting + s.srcPos.Column++ + s.srcPos.Offset += size + s.lastCharLen = size + return eof + } + + // remember last position + s.prevPos = s.srcPos + + s.srcPos.Column++ + s.lastCharLen = size + s.srcPos.Offset += size + + if ch == utf8.RuneError && size == 1 { + s.err("illegal UTF-8 encoding") + return ch + } + + if ch == '\n' { + s.srcPos.Line++ + s.lastLineLen = s.srcPos.Column + s.srcPos.Column = 0 + } + + if ch == '\x00' { + s.err("unexpected null character (0x00)") + return eof + } + + if ch == '\uE123' { + s.err("unicode code point U+E123 reserved for internal use") + return utf8.RuneError + } + + // debug + // fmt.Printf("ch: %q, offset:column: %d:%d\n", ch, s.srcPos.Offset, s.srcPos.Column) + return ch +} + +// unread unreads the previous read Rune and updates the source position +func (s *Scanner) unread() { + if err := s.buf.UnreadRune(); err != nil { + panic(err) // this is user fault, we should catch it + } + s.srcPos = s.prevPos // put back last position +} + +// peek returns the next rune without advancing the reader. +func (s *Scanner) peek() rune { + peek, _, err := s.buf.ReadRune() + if err != nil { + return eof + } + + s.buf.UnreadRune() + return peek +} + +// Scan scans the next token and returns the token. +func (s *Scanner) Scan() token.Token { + ch := s.next() + + // skip white space + for isWhitespace(ch) { + ch = s.next() + } + + var tok token.Type + + // token text markings + s.tokStart = s.srcPos.Offset - s.lastCharLen + + // token position, initial next() is moving the offset by one(size of rune + // actually), though we are interested with the starting point + s.tokPos.Offset = s.srcPos.Offset - s.lastCharLen + if s.srcPos.Column > 0 { + // common case: last character was not a '\n' + s.tokPos.Line = s.srcPos.Line + s.tokPos.Column = s.srcPos.Column + } else { + // last character was a '\n' + // (we cannot be at the beginning of the source + // since we have called next() at least once) + s.tokPos.Line = s.srcPos.Line - 1 + s.tokPos.Column = s.lastLineLen + } + + switch { + case isLetter(ch): + tok = token.IDENT + lit := s.scanIdentifier() + if lit == "true" || lit == "false" { + tok = token.BOOL + } + case isDecimal(ch): + tok = s.scanNumber(ch) + default: + switch ch { + case eof: + tok = token.EOF + case '"': + tok = token.STRING + s.scanString() + case '#', '/': + tok = token.COMMENT + s.scanComment(ch) + case '.': + tok = token.PERIOD + ch = s.peek() + if isDecimal(ch) { + tok = token.FLOAT + ch = s.scanMantissa(ch) + ch = s.scanExponent(ch) + } + case '<': + tok = token.HEREDOC + s.scanHeredoc() + case '[': + tok = token.LBRACK + case ']': + tok = token.RBRACK + case '{': + tok = token.LBRACE + case '}': + tok = token.RBRACE + case ',': + tok = token.COMMA + case '=': + tok = token.ASSIGN + case '+': + tok = token.ADD + case '-': + if isDecimal(s.peek()) { + ch := s.next() + tok = s.scanNumber(ch) + } else { + tok = token.SUB + } + default: + s.err("illegal char") + } + } + + // finish token ending + s.tokEnd = s.srcPos.Offset + + // create token literal + var tokenText string + if s.tokStart >= 0 { + tokenText = string(s.src[s.tokStart:s.tokEnd]) + } + s.tokStart = s.tokEnd // ensure idempotency of tokenText() call + + return token.Token{ + Type: tok, + Pos: s.tokPos, + Text: tokenText, + } +} + +func (s *Scanner) scanComment(ch rune) { + // single line comments + if ch == '#' || (ch == '/' && s.peek() != '*') { + if ch == '/' && s.peek() != '/' { + s.err("expected '/' for comment") + return + } + + ch = s.next() + for ch != '\n' && ch >= 0 && ch != eof { + ch = s.next() + } + if ch != eof && ch >= 0 { + s.unread() + } + return + } + + // be sure we get the character after /* This allows us to find comment's + // that are not erminated + if ch == '/' { + s.next() + ch = s.next() // read character after "/*" + } + + // look for /* - style comments + for { + if ch < 0 || ch == eof { + s.err("comment not terminated") + break + } + + ch0 := ch + ch = s.next() + if ch0 == '*' && ch == '/' { + break + } + } +} + +// scanNumber scans a HCL number definition starting with the given rune +func (s *Scanner) scanNumber(ch rune) token.Type { + if ch == '0' { + // check for hexadecimal, octal or float + ch = s.next() + if ch == 'x' || ch == 'X' { + // hexadecimal + ch = s.next() + found := false + for isHexadecimal(ch) { + ch = s.next() + found = true + } + + if !found { + s.err("illegal hexadecimal number") + } + + if ch != eof { + s.unread() + } + + return token.NUMBER + } + + // now it's either something like: 0421(octal) or 0.1231(float) + illegalOctal := false + for isDecimal(ch) { + ch = s.next() + if ch == '8' || ch == '9' { + // this is just a possibility. For example 0159 is illegal, but + // 0159.23 is valid. So we mark a possible illegal octal. If + // the next character is not a period, we'll print the error. + illegalOctal = true + } + } + + if ch == 'e' || ch == 'E' { + ch = s.scanExponent(ch) + return token.FLOAT + } + + if ch == '.' { + ch = s.scanFraction(ch) + + if ch == 'e' || ch == 'E' { + ch = s.next() + ch = s.scanExponent(ch) + } + return token.FLOAT + } + + if illegalOctal { + s.err("illegal octal number") + } + + if ch != eof { + s.unread() + } + return token.NUMBER + } + + s.scanMantissa(ch) + ch = s.next() // seek forward + if ch == 'e' || ch == 'E' { + ch = s.scanExponent(ch) + return token.FLOAT + } + + if ch == '.' { + ch = s.scanFraction(ch) + if ch == 'e' || ch == 'E' { + ch = s.next() + ch = s.scanExponent(ch) + } + return token.FLOAT + } + + if ch != eof { + s.unread() + } + return token.NUMBER +} + +// scanMantissa scans the mantissa beginning from the rune. It returns the next +// non decimal rune. It's used to determine wheter it's a fraction or exponent. +func (s *Scanner) scanMantissa(ch rune) rune { + scanned := false + for isDecimal(ch) { + ch = s.next() + scanned = true + } + + if scanned && ch != eof { + s.unread() + } + return ch +} + +// scanFraction scans the fraction after the '.' rune +func (s *Scanner) scanFraction(ch rune) rune { + if ch == '.' { + ch = s.peek() // we peek just to see if we can move forward + ch = s.scanMantissa(ch) + } + return ch +} + +// scanExponent scans the remaining parts of an exponent after the 'e' or 'E' +// rune. +func (s *Scanner) scanExponent(ch rune) rune { + if ch == 'e' || ch == 'E' { + ch = s.next() + if ch == '-' || ch == '+' { + ch = s.next() + } + ch = s.scanMantissa(ch) + } + return ch +} + +// scanHeredoc scans a heredoc string +func (s *Scanner) scanHeredoc() { + // Scan the second '<' in example: '<= len(identBytes) && identRegexp.Match(s.src[lineStart:s.srcPos.Offset-s.lastCharLen]) { + break + } + + // Not an anchor match, record the start of a new line + lineStart = s.srcPos.Offset + } + + if ch == eof { + s.err("heredoc not terminated") + return + } + } + + return +} + +// scanString scans a quoted string +func (s *Scanner) scanString() { + braces := 0 + for { + // '"' opening already consumed + // read character after quote + ch := s.next() + + if (ch == '\n' && braces == 0) || ch < 0 || ch == eof { + s.err("literal not terminated") + return + } + + if ch == '"' && braces == 0 { + break + } + + // If we're going into a ${} then we can ignore quotes for awhile + if braces == 0 && ch == '$' && s.peek() == '{' { + braces++ + s.next() + } else if braces > 0 && ch == '{' { + braces++ + } + if braces > 0 && ch == '}' { + braces-- + } + + if ch == '\\' { + s.scanEscape() + } + } + + return +} + +// scanEscape scans an escape sequence +func (s *Scanner) scanEscape() rune { + // http://en.cppreference.com/w/cpp/language/escape + ch := s.next() // read character after '/' + switch ch { + case 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\', '"': + // nothing to do + case '0', '1', '2', '3', '4', '5', '6', '7': + // octal notation + ch = s.scanDigits(ch, 8, 3) + case 'x': + // hexademical notation + ch = s.scanDigits(s.next(), 16, 2) + case 'u': + // universal character name + ch = s.scanDigits(s.next(), 16, 4) + case 'U': + // universal character name + ch = s.scanDigits(s.next(), 16, 8) + default: + s.err("illegal char escape") + } + return ch +} + +// scanDigits scans a rune with the given base for n times. For example an +// octal notation \184 would yield in scanDigits(ch, 8, 3) +func (s *Scanner) scanDigits(ch rune, base, n int) rune { + start := n + for n > 0 && digitVal(ch) < base { + ch = s.next() + if ch == eof { + // If we see an EOF, we halt any more scanning of digits + // immediately. + break + } + + n-- + } + if n > 0 { + s.err("illegal char escape") + } + + if n != start && ch != eof { + // we scanned all digits, put the last non digit char back, + // only if we read anything at all + s.unread() + } + + return ch +} + +// scanIdentifier scans an identifier and returns the literal string +func (s *Scanner) scanIdentifier() string { + offs := s.srcPos.Offset - s.lastCharLen + ch := s.next() + for isLetter(ch) || isDigit(ch) || ch == '-' || ch == '.' { + ch = s.next() + } + + if ch != eof { + s.unread() // we got identifier, put back latest char + } + + return string(s.src[offs:s.srcPos.Offset]) +} + +// recentPosition returns the position of the character immediately after the +// character or token returned by the last call to Scan. +func (s *Scanner) recentPosition() (pos token.Pos) { + pos.Offset = s.srcPos.Offset - s.lastCharLen + switch { + case s.srcPos.Column > 0: + // common case: last character was not a '\n' + pos.Line = s.srcPos.Line + pos.Column = s.srcPos.Column + case s.lastLineLen > 0: + // last character was a '\n' + // (we cannot be at the beginning of the source + // since we have called next() at least once) + pos.Line = s.srcPos.Line - 1 + pos.Column = s.lastLineLen + default: + // at the beginning of the source + pos.Line = 1 + pos.Column = 1 + } + return +} + +// err prints the error of any scanning to s.Error function. If the function is +// not defined, by default it prints them to os.Stderr +func (s *Scanner) err(msg string) { + s.ErrorCount++ + pos := s.recentPosition() + + if s.Error != nil { + s.Error(pos, msg) + return + } + + fmt.Fprintf(os.Stderr, "%s: %s\n", pos, msg) +} + +// isHexadecimal returns true if the given rune is a letter +func isLetter(ch rune) bool { + return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_' || ch >= 0x80 && unicode.IsLetter(ch) +} + +// isDigit returns true if the given rune is a decimal digit +func isDigit(ch rune) bool { + return '0' <= ch && ch <= '9' || ch >= 0x80 && unicode.IsDigit(ch) +} + +// isDecimal returns true if the given rune is a decimal number +func isDecimal(ch rune) bool { + return '0' <= ch && ch <= '9' +} + +// isHexadecimal returns true if the given rune is an hexadecimal number +func isHexadecimal(ch rune) bool { + return '0' <= ch && ch <= '9' || 'a' <= ch && ch <= 'f' || 'A' <= ch && ch <= 'F' +} + +// isWhitespace returns true if the rune is a space, tab, newline or carriage return +func isWhitespace(ch rune) bool { + return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' +} + +// digitVal returns the integer value of a given octal,decimal or hexadecimal rune +func digitVal(ch rune) int { + switch { + case '0' <= ch && ch <= '9': + return int(ch - '0') + case 'a' <= ch && ch <= 'f': + return int(ch - 'a' + 10) + case 'A' <= ch && ch <= 'F': + return int(ch - 'A' + 10) + } + return 16 // larger than any legal digit val +} diff --git a/vendor/github.com/hashicorp/hcl/hcl/strconv/quote.go b/vendor/github.com/hashicorp/hcl/hcl/strconv/quote.go new file mode 100644 index 000000000..5f981eaa2 --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/hcl/strconv/quote.go @@ -0,0 +1,241 @@ +package strconv + +import ( + "errors" + "unicode/utf8" +) + +// ErrSyntax indicates that a value does not have the right syntax for the target type. +var ErrSyntax = errors.New("invalid syntax") + +// Unquote interprets s as a single-quoted, double-quoted, +// or backquoted Go string literal, returning the string value +// that s quotes. (If s is single-quoted, it would be a Go +// character literal; Unquote returns the corresponding +// one-character string.) +func Unquote(s string) (t string, err error) { + n := len(s) + if n < 2 { + return "", ErrSyntax + } + quote := s[0] + if quote != s[n-1] { + return "", ErrSyntax + } + s = s[1 : n-1] + + if quote != '"' { + return "", ErrSyntax + } + if !contains(s, '$') && !contains(s, '{') && contains(s, '\n') { + return "", ErrSyntax + } + + // Is it trivial? Avoid allocation. + if !contains(s, '\\') && !contains(s, quote) && !contains(s, '$') { + switch quote { + case '"': + return s, nil + case '\'': + r, size := utf8.DecodeRuneInString(s) + if size == len(s) && (r != utf8.RuneError || size != 1) { + return s, nil + } + } + } + + var runeTmp [utf8.UTFMax]byte + buf := make([]byte, 0, 3*len(s)/2) // Try to avoid more allocations. + for len(s) > 0 { + // If we're starting a '${}' then let it through un-unquoted. + // Specifically: we don't unquote any characters within the `${}` + // section. + if s[0] == '$' && len(s) > 1 && s[1] == '{' { + buf = append(buf, '$', '{') + s = s[2:] + + // Continue reading until we find the closing brace, copying as-is + braces := 1 + for len(s) > 0 && braces > 0 { + r, size := utf8.DecodeRuneInString(s) + if r == utf8.RuneError { + return "", ErrSyntax + } + + s = s[size:] + + n := utf8.EncodeRune(runeTmp[:], r) + buf = append(buf, runeTmp[:n]...) + + switch r { + case '{': + braces++ + case '}': + braces-- + } + } + if braces != 0 { + return "", ErrSyntax + } + if len(s) == 0 { + // If there's no string left, we're done! + break + } else { + // If there's more left, we need to pop back up to the top of the loop + // in case there's another interpolation in this string. + continue + } + } + + if s[0] == '\n' { + return "", ErrSyntax + } + + c, multibyte, ss, err := unquoteChar(s, quote) + if err != nil { + return "", err + } + s = ss + if c < utf8.RuneSelf || !multibyte { + buf = append(buf, byte(c)) + } else { + n := utf8.EncodeRune(runeTmp[:], c) + buf = append(buf, runeTmp[:n]...) + } + if quote == '\'' && len(s) != 0 { + // single-quoted must be single character + return "", ErrSyntax + } + } + return string(buf), nil +} + +// contains reports whether the string contains the byte c. +func contains(s string, c byte) bool { + for i := 0; i < len(s); i++ { + if s[i] == c { + return true + } + } + return false +} + +func unhex(b byte) (v rune, ok bool) { + c := rune(b) + switch { + case '0' <= c && c <= '9': + return c - '0', true + case 'a' <= c && c <= 'f': + return c - 'a' + 10, true + case 'A' <= c && c <= 'F': + return c - 'A' + 10, true + } + return +} + +func unquoteChar(s string, quote byte) (value rune, multibyte bool, tail string, err error) { + // easy cases + switch c := s[0]; { + case c == quote && (quote == '\'' || quote == '"'): + err = ErrSyntax + return + case c >= utf8.RuneSelf: + r, size := utf8.DecodeRuneInString(s) + return r, true, s[size:], nil + case c != '\\': + return rune(s[0]), false, s[1:], nil + } + + // hard case: c is backslash + if len(s) <= 1 { + err = ErrSyntax + return + } + c := s[1] + s = s[2:] + + switch c { + case 'a': + value = '\a' + case 'b': + value = '\b' + case 'f': + value = '\f' + case 'n': + value = '\n' + case 'r': + value = '\r' + case 't': + value = '\t' + case 'v': + value = '\v' + case 'x', 'u', 'U': + n := 0 + switch c { + case 'x': + n = 2 + case 'u': + n = 4 + case 'U': + n = 8 + } + var v rune + if len(s) < n { + err = ErrSyntax + return + } + for j := 0; j < n; j++ { + x, ok := unhex(s[j]) + if !ok { + err = ErrSyntax + return + } + v = v<<4 | x + } + s = s[n:] + if c == 'x' { + // single-byte string, possibly not UTF-8 + value = v + break + } + if v > utf8.MaxRune { + err = ErrSyntax + return + } + value = v + multibyte = true + case '0', '1', '2', '3', '4', '5', '6', '7': + v := rune(c) - '0' + if len(s) < 2 { + err = ErrSyntax + return + } + for j := 0; j < 2; j++ { // one digit already; two more + x := rune(s[j]) - '0' + if x < 0 || x > 7 { + err = ErrSyntax + return + } + v = (v << 3) | x + } + s = s[2:] + if v > 255 { + err = ErrSyntax + return + } + value = v + case '\\': + value = '\\' + case '\'', '"': + if c != quote { + err = ErrSyntax + return + } + value = rune(c) + default: + err = ErrSyntax + return + } + tail = s + return +} diff --git a/vendor/github.com/hashicorp/hcl/hcl/token/position.go b/vendor/github.com/hashicorp/hcl/hcl/token/position.go new file mode 100644 index 000000000..59c1bb72d --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/hcl/token/position.go @@ -0,0 +1,46 @@ +package token + +import "fmt" + +// Pos describes an arbitrary source position +// including the file, line, and column location. +// A Position is valid if the line number is > 0. +type Pos struct { + Filename string // filename, if any + Offset int // offset, starting at 0 + Line int // line number, starting at 1 + Column int // column number, starting at 1 (character count) +} + +// IsValid returns true if the position is valid. +func (p *Pos) IsValid() bool { return p.Line > 0 } + +// String returns a string in one of several forms: +// +// file:line:column valid position with file name +// line:column valid position without file name +// file invalid position with file name +// - invalid position without file name +func (p Pos) String() string { + s := p.Filename + if p.IsValid() { + if s != "" { + s += ":" + } + s += fmt.Sprintf("%d:%d", p.Line, p.Column) + } + if s == "" { + s = "-" + } + return s +} + +// Before reports whether the position p is before u. +func (p Pos) Before(u Pos) bool { + return u.Offset > p.Offset || u.Line > p.Line +} + +// After reports whether the position p is after u. +func (p Pos) After(u Pos) bool { + return u.Offset < p.Offset || u.Line < p.Line +} diff --git a/vendor/github.com/hashicorp/hcl/hcl/token/token.go b/vendor/github.com/hashicorp/hcl/hcl/token/token.go new file mode 100644 index 000000000..e37c0664e --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/hcl/token/token.go @@ -0,0 +1,219 @@ +// Package token defines constants representing the lexical tokens for HCL +// (HashiCorp Configuration Language) +package token + +import ( + "fmt" + "strconv" + "strings" + + hclstrconv "github.com/hashicorp/hcl/hcl/strconv" +) + +// Token defines a single HCL token which can be obtained via the Scanner +type Token struct { + Type Type + Pos Pos + Text string + JSON bool +} + +// Type is the set of lexical tokens of the HCL (HashiCorp Configuration Language) +type Type int + +const ( + // Special tokens + ILLEGAL Type = iota + EOF + COMMENT + + identifier_beg + IDENT // literals + literal_beg + NUMBER // 12345 + FLOAT // 123.45 + BOOL // true,false + STRING // "abc" + HEREDOC // < 0 { + // Pop the current item + n := len(frontier) + item := frontier[n-1] + frontier = frontier[:n-1] + + switch v := item.Val.(type) { + case *ast.ObjectType: + items, frontier = flattenObjectType(v, item, items, frontier) + case *ast.ListType: + items, frontier = flattenListType(v, item, items, frontier) + default: + items = append(items, item) + } + } + + // Reverse the list since the frontier model runs things backwards + for i := len(items)/2 - 1; i >= 0; i-- { + opp := len(items) - 1 - i + items[i], items[opp] = items[opp], items[i] + } + + // Done! Set the original items + list.Items = items + return n, true + }) +} + +func flattenListType( + ot *ast.ListType, + item *ast.ObjectItem, + items []*ast.ObjectItem, + frontier []*ast.ObjectItem) ([]*ast.ObjectItem, []*ast.ObjectItem) { + // If the list is empty, keep the original list + if len(ot.List) == 0 { + items = append(items, item) + return items, frontier + } + + // All the elements of this object must also be objects! + for _, subitem := range ot.List { + if _, ok := subitem.(*ast.ObjectType); !ok { + items = append(items, item) + return items, frontier + } + } + + // Great! We have a match go through all the items and flatten + for _, elem := range ot.List { + // Add it to the frontier so that we can recurse + frontier = append(frontier, &ast.ObjectItem{ + Keys: item.Keys, + Assign: item.Assign, + Val: elem, + LeadComment: item.LeadComment, + LineComment: item.LineComment, + }) + } + + return items, frontier +} + +func flattenObjectType( + ot *ast.ObjectType, + item *ast.ObjectItem, + items []*ast.ObjectItem, + frontier []*ast.ObjectItem) ([]*ast.ObjectItem, []*ast.ObjectItem) { + // If the list has no items we do not have to flatten anything + if ot.List.Items == nil { + items = append(items, item) + return items, frontier + } + + // All the elements of this object must also be objects! + for _, subitem := range ot.List.Items { + if _, ok := subitem.Val.(*ast.ObjectType); !ok { + items = append(items, item) + return items, frontier + } + } + + // Great! We have a match go through all the items and flatten + for _, subitem := range ot.List.Items { + // Copy the new key + keys := make([]*ast.ObjectKey, len(item.Keys)+len(subitem.Keys)) + copy(keys, item.Keys) + copy(keys[len(item.Keys):], subitem.Keys) + + // Add it to the frontier so that we can recurse + frontier = append(frontier, &ast.ObjectItem{ + Keys: keys, + Assign: item.Assign, + Val: subitem.Val, + LeadComment: item.LeadComment, + LineComment: item.LineComment, + }) + } + + return items, frontier +} diff --git a/vendor/github.com/hashicorp/hcl/json/parser/parser.go b/vendor/github.com/hashicorp/hcl/json/parser/parser.go new file mode 100644 index 000000000..125a5f072 --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/json/parser/parser.go @@ -0,0 +1,313 @@ +package parser + +import ( + "errors" + "fmt" + + "github.com/hashicorp/hcl/hcl/ast" + hcltoken "github.com/hashicorp/hcl/hcl/token" + "github.com/hashicorp/hcl/json/scanner" + "github.com/hashicorp/hcl/json/token" +) + +type Parser struct { + sc *scanner.Scanner + + // Last read token + tok token.Token + commaPrev token.Token + + enableTrace bool + indent int + n int // buffer size (max = 1) +} + +func newParser(src []byte) *Parser { + return &Parser{ + sc: scanner.New(src), + } +} + +// Parse returns the fully parsed source and returns the abstract syntax tree. +func Parse(src []byte) (*ast.File, error) { + p := newParser(src) + return p.Parse() +} + +var errEofToken = errors.New("EOF token found") + +// Parse returns the fully parsed source and returns the abstract syntax tree. +func (p *Parser) Parse() (*ast.File, error) { + f := &ast.File{} + var err, scerr error + p.sc.Error = func(pos token.Pos, msg string) { + scerr = fmt.Errorf("%s: %s", pos, msg) + } + + // The root must be an object in JSON + object, err := p.object() + if scerr != nil { + return nil, scerr + } + if err != nil { + return nil, err + } + + // We make our final node an object list so it is more HCL compatible + f.Node = object.List + + // Flatten it, which finds patterns and turns them into more HCL-like + // AST trees. + flattenObjects(f.Node) + + return f, nil +} + +func (p *Parser) objectList() (*ast.ObjectList, error) { + defer un(trace(p, "ParseObjectList")) + node := &ast.ObjectList{} + + for { + n, err := p.objectItem() + if err == errEofToken { + break // we are finished + } + + // we don't return a nil node, because might want to use already + // collected items. + if err != nil { + return node, err + } + + node.Add(n) + + // Check for a followup comma. If it isn't a comma, then we're done + if tok := p.scan(); tok.Type != token.COMMA { + break + } + } + + return node, nil +} + +// objectItem parses a single object item +func (p *Parser) objectItem() (*ast.ObjectItem, error) { + defer un(trace(p, "ParseObjectItem")) + + keys, err := p.objectKey() + if err != nil { + return nil, err + } + + o := &ast.ObjectItem{ + Keys: keys, + } + + switch p.tok.Type { + case token.COLON: + pos := p.tok.Pos + o.Assign = hcltoken.Pos{ + Filename: pos.Filename, + Offset: pos.Offset, + Line: pos.Line, + Column: pos.Column, + } + + o.Val, err = p.objectValue() + if err != nil { + return nil, err + } + } + + return o, nil +} + +// objectKey parses an object key and returns a ObjectKey AST +func (p *Parser) objectKey() ([]*ast.ObjectKey, error) { + keyCount := 0 + keys := make([]*ast.ObjectKey, 0) + + for { + tok := p.scan() + switch tok.Type { + case token.EOF: + return nil, errEofToken + case token.STRING: + keyCount++ + keys = append(keys, &ast.ObjectKey{ + Token: p.tok.HCLToken(), + }) + case token.COLON: + // If we have a zero keycount it means that we never got + // an object key, i.e. `{ :`. This is a syntax error. + if keyCount == 0 { + return nil, fmt.Errorf("expected: STRING got: %s", p.tok.Type) + } + + // Done + return keys, nil + case token.ILLEGAL: + return nil, errors.New("illegal") + default: + return nil, fmt.Errorf("expected: STRING got: %s", p.tok.Type) + } + } +} + +// object parses any type of object, such as number, bool, string, object or +// list. +func (p *Parser) objectValue() (ast.Node, error) { + defer un(trace(p, "ParseObjectValue")) + tok := p.scan() + + switch tok.Type { + case token.NUMBER, token.FLOAT, token.BOOL, token.NULL, token.STRING: + return p.literalType() + case token.LBRACE: + return p.objectType() + case token.LBRACK: + return p.listType() + case token.EOF: + return nil, errEofToken + } + + return nil, fmt.Errorf("Expected object value, got unknown token: %+v", tok) +} + +// object parses any type of object, such as number, bool, string, object or +// list. +func (p *Parser) object() (*ast.ObjectType, error) { + defer un(trace(p, "ParseType")) + tok := p.scan() + + switch tok.Type { + case token.LBRACE: + return p.objectType() + case token.EOF: + return nil, errEofToken + } + + return nil, fmt.Errorf("Expected object, got unknown token: %+v", tok) +} + +// objectType parses an object type and returns a ObjectType AST +func (p *Parser) objectType() (*ast.ObjectType, error) { + defer un(trace(p, "ParseObjectType")) + + // we assume that the currently scanned token is a LBRACE + o := &ast.ObjectType{} + + l, err := p.objectList() + + // if we hit RBRACE, we are good to go (means we parsed all Items), if it's + // not a RBRACE, it's an syntax error and we just return it. + if err != nil && p.tok.Type != token.RBRACE { + return nil, err + } + + o.List = l + return o, nil +} + +// listType parses a list type and returns a ListType AST +func (p *Parser) listType() (*ast.ListType, error) { + defer un(trace(p, "ParseListType")) + + // we assume that the currently scanned token is a LBRACK + l := &ast.ListType{} + + for { + tok := p.scan() + switch tok.Type { + case token.NUMBER, token.FLOAT, token.STRING: + node, err := p.literalType() + if err != nil { + return nil, err + } + + l.Add(node) + case token.COMMA: + continue + case token.LBRACE: + node, err := p.objectType() + if err != nil { + return nil, err + } + + l.Add(node) + case token.BOOL: + // TODO(arslan) should we support? not supported by HCL yet + case token.LBRACK: + // TODO(arslan) should we support nested lists? Even though it's + // written in README of HCL, it's not a part of the grammar + // (not defined in parse.y) + case token.RBRACK: + // finished + return l, nil + default: + return nil, fmt.Errorf("unexpected token while parsing list: %s", tok.Type) + } + + } +} + +// literalType parses a literal type and returns a LiteralType AST +func (p *Parser) literalType() (*ast.LiteralType, error) { + defer un(trace(p, "ParseLiteral")) + + return &ast.LiteralType{ + Token: p.tok.HCLToken(), + }, nil +} + +// scan returns the next token from the underlying scanner. If a token has +// been unscanned then read that instead. +func (p *Parser) scan() token.Token { + // If we have a token on the buffer, then return it. + if p.n != 0 { + p.n = 0 + return p.tok + } + + p.tok = p.sc.Scan() + return p.tok +} + +// unscan pushes the previously read token back onto the buffer. +func (p *Parser) unscan() { + p.n = 1 +} + +// ---------------------------------------------------------------------------- +// Parsing support + +func (p *Parser) printTrace(a ...interface{}) { + if !p.enableTrace { + return + } + + const dots = ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " + const n = len(dots) + fmt.Printf("%5d:%3d: ", p.tok.Pos.Line, p.tok.Pos.Column) + + i := 2 * p.indent + for i > n { + fmt.Print(dots) + i -= n + } + // i <= n + fmt.Print(dots[0:i]) + fmt.Println(a...) +} + +func trace(p *Parser, msg string) *Parser { + p.printTrace(msg, "(") + p.indent++ + return p +} + +// Usage pattern: defer un(trace(p, "...")) +func un(p *Parser) { + p.indent-- + p.printTrace(")") +} diff --git a/vendor/github.com/hashicorp/hcl/json/scanner/scanner.go b/vendor/github.com/hashicorp/hcl/json/scanner/scanner.go new file mode 100644 index 000000000..fe3f0f095 --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/json/scanner/scanner.go @@ -0,0 +1,451 @@ +package scanner + +import ( + "bytes" + "fmt" + "os" + "unicode" + "unicode/utf8" + + "github.com/hashicorp/hcl/json/token" +) + +// eof represents a marker rune for the end of the reader. +const eof = rune(0) + +// Scanner defines a lexical scanner +type Scanner struct { + buf *bytes.Buffer // Source buffer for advancing and scanning + src []byte // Source buffer for immutable access + + // Source Position + srcPos token.Pos // current position + prevPos token.Pos // previous position, used for peek() method + + lastCharLen int // length of last character in bytes + lastLineLen int // length of last line in characters (for correct column reporting) + + tokStart int // token text start position + tokEnd int // token text end position + + // Error is called for each error encountered. If no Error + // function is set, the error is reported to os.Stderr. + Error func(pos token.Pos, msg string) + + // ErrorCount is incremented by one for each error encountered. + ErrorCount int + + // tokPos is the start position of most recently scanned token; set by + // Scan. The Filename field is always left untouched by the Scanner. If + // an error is reported (via Error) and Position is invalid, the scanner is + // not inside a token. + tokPos token.Pos +} + +// New creates and initializes a new instance of Scanner using src as +// its source content. +func New(src []byte) *Scanner { + // even though we accept a src, we read from a io.Reader compatible type + // (*bytes.Buffer). So in the future we might easily change it to streaming + // read. + b := bytes.NewBuffer(src) + s := &Scanner{ + buf: b, + src: src, + } + + // srcPosition always starts with 1 + s.srcPos.Line = 1 + return s +} + +// next reads the next rune from the bufferred reader. Returns the rune(0) if +// an error occurs (or io.EOF is returned). +func (s *Scanner) next() rune { + ch, size, err := s.buf.ReadRune() + if err != nil { + // advance for error reporting + s.srcPos.Column++ + s.srcPos.Offset += size + s.lastCharLen = size + return eof + } + + if ch == utf8.RuneError && size == 1 { + s.srcPos.Column++ + s.srcPos.Offset += size + s.lastCharLen = size + s.err("illegal UTF-8 encoding") + return ch + } + + // remember last position + s.prevPos = s.srcPos + + s.srcPos.Column++ + s.lastCharLen = size + s.srcPos.Offset += size + + if ch == '\n' { + s.srcPos.Line++ + s.lastLineLen = s.srcPos.Column + s.srcPos.Column = 0 + } + + // debug + // fmt.Printf("ch: %q, offset:column: %d:%d\n", ch, s.srcPos.Offset, s.srcPos.Column) + return ch +} + +// unread unreads the previous read Rune and updates the source position +func (s *Scanner) unread() { + if err := s.buf.UnreadRune(); err != nil { + panic(err) // this is user fault, we should catch it + } + s.srcPos = s.prevPos // put back last position +} + +// peek returns the next rune without advancing the reader. +func (s *Scanner) peek() rune { + peek, _, err := s.buf.ReadRune() + if err != nil { + return eof + } + + s.buf.UnreadRune() + return peek +} + +// Scan scans the next token and returns the token. +func (s *Scanner) Scan() token.Token { + ch := s.next() + + // skip white space + for isWhitespace(ch) { + ch = s.next() + } + + var tok token.Type + + // token text markings + s.tokStart = s.srcPos.Offset - s.lastCharLen + + // token position, initial next() is moving the offset by one(size of rune + // actually), though we are interested with the starting point + s.tokPos.Offset = s.srcPos.Offset - s.lastCharLen + if s.srcPos.Column > 0 { + // common case: last character was not a '\n' + s.tokPos.Line = s.srcPos.Line + s.tokPos.Column = s.srcPos.Column + } else { + // last character was a '\n' + // (we cannot be at the beginning of the source + // since we have called next() at least once) + s.tokPos.Line = s.srcPos.Line - 1 + s.tokPos.Column = s.lastLineLen + } + + switch { + case isLetter(ch): + lit := s.scanIdentifier() + if lit == "true" || lit == "false" { + tok = token.BOOL + } else if lit == "null" { + tok = token.NULL + } else { + s.err("illegal char") + } + case isDecimal(ch): + tok = s.scanNumber(ch) + default: + switch ch { + case eof: + tok = token.EOF + case '"': + tok = token.STRING + s.scanString() + case '.': + tok = token.PERIOD + ch = s.peek() + if isDecimal(ch) { + tok = token.FLOAT + ch = s.scanMantissa(ch) + ch = s.scanExponent(ch) + } + case '[': + tok = token.LBRACK + case ']': + tok = token.RBRACK + case '{': + tok = token.LBRACE + case '}': + tok = token.RBRACE + case ',': + tok = token.COMMA + case ':': + tok = token.COLON + case '-': + if isDecimal(s.peek()) { + ch := s.next() + tok = s.scanNumber(ch) + } else { + s.err("illegal char") + } + default: + s.err("illegal char: " + string(ch)) + } + } + + // finish token ending + s.tokEnd = s.srcPos.Offset + + // create token literal + var tokenText string + if s.tokStart >= 0 { + tokenText = string(s.src[s.tokStart:s.tokEnd]) + } + s.tokStart = s.tokEnd // ensure idempotency of tokenText() call + + return token.Token{ + Type: tok, + Pos: s.tokPos, + Text: tokenText, + } +} + +// scanNumber scans a HCL number definition starting with the given rune +func (s *Scanner) scanNumber(ch rune) token.Type { + zero := ch == '0' + pos := s.srcPos + + s.scanMantissa(ch) + ch = s.next() // seek forward + if ch == 'e' || ch == 'E' { + ch = s.scanExponent(ch) + return token.FLOAT + } + + if ch == '.' { + ch = s.scanFraction(ch) + if ch == 'e' || ch == 'E' { + ch = s.next() + ch = s.scanExponent(ch) + } + return token.FLOAT + } + + if ch != eof { + s.unread() + } + + // If we have a larger number and this is zero, error + if zero && pos != s.srcPos { + s.err("numbers cannot start with 0") + } + + return token.NUMBER +} + +// scanMantissa scans the mantissa beginning from the rune. It returns the next +// non decimal rune. It's used to determine wheter it's a fraction or exponent. +func (s *Scanner) scanMantissa(ch rune) rune { + scanned := false + for isDecimal(ch) { + ch = s.next() + scanned = true + } + + if scanned && ch != eof { + s.unread() + } + return ch +} + +// scanFraction scans the fraction after the '.' rune +func (s *Scanner) scanFraction(ch rune) rune { + if ch == '.' { + ch = s.peek() // we peek just to see if we can move forward + ch = s.scanMantissa(ch) + } + return ch +} + +// scanExponent scans the remaining parts of an exponent after the 'e' or 'E' +// rune. +func (s *Scanner) scanExponent(ch rune) rune { + if ch == 'e' || ch == 'E' { + ch = s.next() + if ch == '-' || ch == '+' { + ch = s.next() + } + ch = s.scanMantissa(ch) + } + return ch +} + +// scanString scans a quoted string +func (s *Scanner) scanString() { + braces := 0 + for { + // '"' opening already consumed + // read character after quote + ch := s.next() + + if ch == '\n' || ch < 0 || ch == eof { + s.err("literal not terminated") + return + } + + if ch == '"' { + break + } + + // If we're going into a ${} then we can ignore quotes for awhile + if braces == 0 && ch == '$' && s.peek() == '{' { + braces++ + s.next() + } else if braces > 0 && ch == '{' { + braces++ + } + if braces > 0 && ch == '}' { + braces-- + } + + if ch == '\\' { + s.scanEscape() + } + } + + return +} + +// scanEscape scans an escape sequence +func (s *Scanner) scanEscape() rune { + // http://en.cppreference.com/w/cpp/language/escape + ch := s.next() // read character after '/' + switch ch { + case 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\', '"': + // nothing to do + case '0', '1', '2', '3', '4', '5', '6', '7': + // octal notation + ch = s.scanDigits(ch, 8, 3) + case 'x': + // hexademical notation + ch = s.scanDigits(s.next(), 16, 2) + case 'u': + // universal character name + ch = s.scanDigits(s.next(), 16, 4) + case 'U': + // universal character name + ch = s.scanDigits(s.next(), 16, 8) + default: + s.err("illegal char escape") + } + return ch +} + +// scanDigits scans a rune with the given base for n times. For example an +// octal notation \184 would yield in scanDigits(ch, 8, 3) +func (s *Scanner) scanDigits(ch rune, base, n int) rune { + for n > 0 && digitVal(ch) < base { + ch = s.next() + n-- + } + if n > 0 { + s.err("illegal char escape") + } + + // we scanned all digits, put the last non digit char back + s.unread() + return ch +} + +// scanIdentifier scans an identifier and returns the literal string +func (s *Scanner) scanIdentifier() string { + offs := s.srcPos.Offset - s.lastCharLen + ch := s.next() + for isLetter(ch) || isDigit(ch) || ch == '-' { + ch = s.next() + } + + if ch != eof { + s.unread() // we got identifier, put back latest char + } + + return string(s.src[offs:s.srcPos.Offset]) +} + +// recentPosition returns the position of the character immediately after the +// character or token returned by the last call to Scan. +func (s *Scanner) recentPosition() (pos token.Pos) { + pos.Offset = s.srcPos.Offset - s.lastCharLen + switch { + case s.srcPos.Column > 0: + // common case: last character was not a '\n' + pos.Line = s.srcPos.Line + pos.Column = s.srcPos.Column + case s.lastLineLen > 0: + // last character was a '\n' + // (we cannot be at the beginning of the source + // since we have called next() at least once) + pos.Line = s.srcPos.Line - 1 + pos.Column = s.lastLineLen + default: + // at the beginning of the source + pos.Line = 1 + pos.Column = 1 + } + return +} + +// err prints the error of any scanning to s.Error function. If the function is +// not defined, by default it prints them to os.Stderr +func (s *Scanner) err(msg string) { + s.ErrorCount++ + pos := s.recentPosition() + + if s.Error != nil { + s.Error(pos, msg) + return + } + + fmt.Fprintf(os.Stderr, "%s: %s\n", pos, msg) +} + +// isHexadecimal returns true if the given rune is a letter +func isLetter(ch rune) bool { + return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_' || ch >= 0x80 && unicode.IsLetter(ch) +} + +// isHexadecimal returns true if the given rune is a decimal digit +func isDigit(ch rune) bool { + return '0' <= ch && ch <= '9' || ch >= 0x80 && unicode.IsDigit(ch) +} + +// isHexadecimal returns true if the given rune is a decimal number +func isDecimal(ch rune) bool { + return '0' <= ch && ch <= '9' +} + +// isHexadecimal returns true if the given rune is an hexadecimal number +func isHexadecimal(ch rune) bool { + return '0' <= ch && ch <= '9' || 'a' <= ch && ch <= 'f' || 'A' <= ch && ch <= 'F' +} + +// isWhitespace returns true if the rune is a space, tab, newline or carriage return +func isWhitespace(ch rune) bool { + return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' +} + +// digitVal returns the integer value of a given octal,decimal or hexadecimal rune +func digitVal(ch rune) int { + switch { + case '0' <= ch && ch <= '9': + return int(ch - '0') + case 'a' <= ch && ch <= 'f': + return int(ch - 'a' + 10) + case 'A' <= ch && ch <= 'F': + return int(ch - 'A' + 10) + } + return 16 // larger than any legal digit val +} diff --git a/vendor/github.com/hashicorp/hcl/json/token/position.go b/vendor/github.com/hashicorp/hcl/json/token/position.go new file mode 100644 index 000000000..59c1bb72d --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/json/token/position.go @@ -0,0 +1,46 @@ +package token + +import "fmt" + +// Pos describes an arbitrary source position +// including the file, line, and column location. +// A Position is valid if the line number is > 0. +type Pos struct { + Filename string // filename, if any + Offset int // offset, starting at 0 + Line int // line number, starting at 1 + Column int // column number, starting at 1 (character count) +} + +// IsValid returns true if the position is valid. +func (p *Pos) IsValid() bool { return p.Line > 0 } + +// String returns a string in one of several forms: +// +// file:line:column valid position with file name +// line:column valid position without file name +// file invalid position with file name +// - invalid position without file name +func (p Pos) String() string { + s := p.Filename + if p.IsValid() { + if s != "" { + s += ":" + } + s += fmt.Sprintf("%d:%d", p.Line, p.Column) + } + if s == "" { + s = "-" + } + return s +} + +// Before reports whether the position p is before u. +func (p Pos) Before(u Pos) bool { + return u.Offset > p.Offset || u.Line > p.Line +} + +// After reports whether the position p is after u. +func (p Pos) After(u Pos) bool { + return u.Offset < p.Offset || u.Line < p.Line +} diff --git a/vendor/github.com/hashicorp/hcl/json/token/token.go b/vendor/github.com/hashicorp/hcl/json/token/token.go new file mode 100644 index 000000000..95a0c3eee --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/json/token/token.go @@ -0,0 +1,118 @@ +package token + +import ( + "fmt" + "strconv" + + hcltoken "github.com/hashicorp/hcl/hcl/token" +) + +// Token defines a single HCL token which can be obtained via the Scanner +type Token struct { + Type Type + Pos Pos + Text string +} + +// Type is the set of lexical tokens of the HCL (HashiCorp Configuration Language) +type Type int + +const ( + // Special tokens + ILLEGAL Type = iota + EOF + + identifier_beg + literal_beg + NUMBER // 12345 + FLOAT // 123.45 + BOOL // true,false + STRING // "abc" + NULL // null + literal_end + identifier_end + + operator_beg + LBRACK // [ + LBRACE // { + COMMA // , + PERIOD // . + COLON // : + + RBRACK // ] + RBRACE // } + + operator_end +) + +var tokens = [...]string{ + ILLEGAL: "ILLEGAL", + + EOF: "EOF", + + NUMBER: "NUMBER", + FLOAT: "FLOAT", + BOOL: "BOOL", + STRING: "STRING", + NULL: "NULL", + + LBRACK: "LBRACK", + LBRACE: "LBRACE", + COMMA: "COMMA", + PERIOD: "PERIOD", + COLON: "COLON", + + RBRACK: "RBRACK", + RBRACE: "RBRACE", +} + +// String returns the string corresponding to the token tok. +func (t Type) String() string { + s := "" + if 0 <= t && t < Type(len(tokens)) { + s = tokens[t] + } + if s == "" { + s = "token(" + strconv.Itoa(int(t)) + ")" + } + return s +} + +// IsIdentifier returns true for tokens corresponding to identifiers and basic +// type literals; it returns false otherwise. +func (t Type) IsIdentifier() bool { return identifier_beg < t && t < identifier_end } + +// IsLiteral returns true for tokens corresponding to basic type literals; it +// returns false otherwise. +func (t Type) IsLiteral() bool { return literal_beg < t && t < literal_end } + +// IsOperator returns true for tokens corresponding to operators and +// delimiters; it returns false otherwise. +func (t Type) IsOperator() bool { return operator_beg < t && t < operator_end } + +// String returns the token's literal text. Note that this is only +// applicable for certain token types, such as token.IDENT, +// token.STRING, etc.. +func (t Token) String() string { + return fmt.Sprintf("%s %s %s", t.Pos.String(), t.Type.String(), t.Text) +} + +// HCLToken converts this token to an HCL token. +// +// The token type must be a literal type or this will panic. +func (t Token) HCLToken() hcltoken.Token { + switch t.Type { + case BOOL: + return hcltoken.Token{Type: hcltoken.BOOL, Text: t.Text} + case FLOAT: + return hcltoken.Token{Type: hcltoken.FLOAT, Text: t.Text} + case NULL: + return hcltoken.Token{Type: hcltoken.STRING, Text: ""} + case NUMBER: + return hcltoken.Token{Type: hcltoken.NUMBER, Text: t.Text} + case STRING: + return hcltoken.Token{Type: hcltoken.STRING, Text: t.Text, JSON: true} + default: + panic(fmt.Sprintf("unimplemented HCLToken for type: %s", t.Type)) + } +} diff --git a/vendor/github.com/hashicorp/hcl/lex.go b/vendor/github.com/hashicorp/hcl/lex.go new file mode 100644 index 000000000..d9993c292 --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/lex.go @@ -0,0 +1,38 @@ +package hcl + +import ( + "unicode" + "unicode/utf8" +) + +type lexModeValue byte + +const ( + lexModeUnknown lexModeValue = iota + lexModeHcl + lexModeJson +) + +// lexMode returns whether we're going to be parsing in JSON +// mode or HCL mode. +func lexMode(v []byte) lexModeValue { + var ( + r rune + w int + offset int + ) + + for { + r, w = utf8.DecodeRune(v[offset:]) + offset += w + if unicode.IsSpace(r) { + continue + } + if r == '{' { + return lexModeJson + } + break + } + + return lexModeHcl +} diff --git a/vendor/github.com/hashicorp/hcl/parse.go b/vendor/github.com/hashicorp/hcl/parse.go new file mode 100644 index 000000000..1fca53c4c --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/parse.go @@ -0,0 +1,39 @@ +package hcl + +import ( + "fmt" + + "github.com/hashicorp/hcl/hcl/ast" + hclParser "github.com/hashicorp/hcl/hcl/parser" + jsonParser "github.com/hashicorp/hcl/json/parser" +) + +// ParseBytes accepts as input byte slice and returns ast tree. +// +// Input can be either JSON or HCL +func ParseBytes(in []byte) (*ast.File, error) { + return parse(in) +} + +// ParseString accepts input as a string and returns ast tree. +func ParseString(input string) (*ast.File, error) { + return parse([]byte(input)) +} + +func parse(in []byte) (*ast.File, error) { + switch lexMode(in) { + case lexModeHcl: + return hclParser.Parse(in) + case lexModeJson: + return jsonParser.Parse(in) + } + + return nil, fmt.Errorf("unknown config format") +} + +// Parse parses the given input and returns the root object. +// +// The input format can be either HCL or JSON. +func Parse(input string) (*ast.File, error) { + return parse([]byte(input)) +} diff --git a/vendor/github.com/magiconair/properties/.gitignore b/vendor/github.com/magiconair/properties/.gitignore new file mode 100644 index 000000000..e7081ff52 --- /dev/null +++ b/vendor/github.com/magiconair/properties/.gitignore @@ -0,0 +1,6 @@ +*.sublime-project +*.sublime-workspace +*.un~ +*.swp +.idea/ +*.iml diff --git a/vendor/github.com/magiconair/properties/.travis.yml b/vendor/github.com/magiconair/properties/.travis.yml new file mode 100644 index 000000000..3e7c3d2c8 --- /dev/null +++ b/vendor/github.com/magiconair/properties/.travis.yml @@ -0,0 +1,10 @@ +language: go +go: + - 1.4.x + - 1.5.x + - 1.6.x + - 1.7.x + - 1.8.x + - 1.9.x + - "1.10.x" + - tip diff --git a/vendor/github.com/magiconair/properties/CHANGELOG.md b/vendor/github.com/magiconair/properties/CHANGELOG.md new file mode 100644 index 000000000..f83adc205 --- /dev/null +++ b/vendor/github.com/magiconair/properties/CHANGELOG.md @@ -0,0 +1,131 @@ +## Changelog + +### [1.8](https://github.com/magiconair/properties/tree/v1.8) - 15 May 2018 + + * [PR #26](https://github.com/magiconair/properties/pull/26): Disable expansion during loading + + This adds the option to disable property expansion during loading. + + Thanks to [@kmala](https://github.com/kmala) for the patch. + +### [1.7.6](https://github.com/magiconair/properties/tree/v1.7.6) - 14 Feb 2018 + + * [PR #29](https://github.com/magiconair/properties/pull/29): Reworked expansion logic to handle more complex cases. + + See PR for an example. + + Thanks to [@yobert](https://github.com/yobert) for the fix. + +### [1.7.5](https://github.com/magiconair/properties/tree/v1.7.5) - 13 Feb 2018 + + * [PR #28](https://github.com/magiconair/properties/pull/28): Support duplicate expansions in the same value + + Values which expand the same key multiple times (e.g. `key=${a} ${a}`) will no longer fail + with a `circular reference error`. + + Thanks to [@yobert](https://github.com/yobert) for the fix. + +### [1.7.4](https://github.com/magiconair/properties/tree/v1.7.4) - 31 Oct 2017 + + * [Issue #23](https://github.com/magiconair/properties/issues/23): Ignore blank lines with whitespaces + + * [PR #24](https://github.com/magiconair/properties/pull/24): Update keys when DisableExpansion is enabled + + Thanks to [@mgurov](https://github.com/mgurov) for the fix. + +### [1.7.3](https://github.com/magiconair/properties/tree/v1.7.3) - 10 Jul 2017 + + * [Issue #17](https://github.com/magiconair/properties/issues/17): Add [SetValue()](http://godoc.org/github.com/magiconair/properties#Properties.SetValue) method to set values generically + * [Issue #22](https://github.com/magiconair/properties/issues/22): Add [LoadMap()](http://godoc.org/github.com/magiconair/properties#LoadMap) function to load properties from a string map + +### [1.7.2](https://github.com/magiconair/properties/tree/v1.7.2) - 20 Mar 2017 + + * [Issue #15](https://github.com/magiconair/properties/issues/15): Drop gocheck dependency + * [PR #21](https://github.com/magiconair/properties/pull/21): Add [Map()](http://godoc.org/github.com/magiconair/properties#Properties.Map) and [FilterFunc()](http://godoc.org/github.com/magiconair/properties#Properties.FilterFunc) + +### [1.7.1](https://github.com/magiconair/properties/tree/v1.7.1) - 13 Jan 2017 + + * [Issue #14](https://github.com/magiconair/properties/issues/14): Decouple TestLoadExpandedFile from `$USER` + * [PR #12](https://github.com/magiconair/properties/pull/12): Load from files and URLs + * [PR #16](https://github.com/magiconair/properties/pull/16): Keep gofmt happy + * [PR #18](https://github.com/magiconair/properties/pull/18): Fix Delete() function + +### [1.7.0](https://github.com/magiconair/properties/tree/v1.7.0) - 20 Mar 2016 + + * [Issue #10](https://github.com/magiconair/properties/issues/10): Add [LoadURL,LoadURLs,MustLoadURL,MustLoadURLs](http://godoc.org/github.com/magiconair/properties#LoadURL) method to load properties from a URL. + * [Issue #11](https://github.com/magiconair/properties/issues/11): Add [LoadString,MustLoadString](http://godoc.org/github.com/magiconair/properties#LoadString) method to load properties from an UTF8 string. + * [PR #8](https://github.com/magiconair/properties/pull/8): Add [MustFlag](http://godoc.org/github.com/magiconair/properties#Properties.MustFlag) method to provide overrides via command line flags. (@pascaldekloe) + +### [1.6.0](https://github.com/magiconair/properties/tree/v1.6.0) - 11 Dec 2015 + + * Add [Decode](http://godoc.org/github.com/magiconair/properties#Properties.Decode) method to populate struct from properties via tags. + +### [1.5.6](https://github.com/magiconair/properties/tree/v1.5.6) - 18 Oct 2015 + + * Vendored in gopkg.in/check.v1 + +### [1.5.5](https://github.com/magiconair/properties/tree/v1.5.5) - 31 Jul 2015 + + * [PR #6](https://github.com/magiconair/properties/pull/6): Add [Delete](http://godoc.org/github.com/magiconair/properties#Properties.Delete) method to remove keys including comments. (@gerbenjacobs) + +### [1.5.4](https://github.com/magiconair/properties/tree/v1.5.4) - 23 Jun 2015 + + * [Issue #5](https://github.com/magiconair/properties/issues/5): Allow disabling of property expansion [DisableExpansion](http://godoc.org/github.com/magiconair/properties#Properties.DisableExpansion). When property expansion is disabled Properties become a simple key/value store and don't check for circular references. + +### [1.5.3](https://github.com/magiconair/properties/tree/v1.5.3) - 02 Jun 2015 + + * [Issue #4](https://github.com/magiconair/properties/issues/4): Maintain key order in [Filter()](http://godoc.org/github.com/magiconair/properties#Properties.Filter), [FilterPrefix()](http://godoc.org/github.com/magiconair/properties#Properties.FilterPrefix) and [FilterRegexp()](http://godoc.org/github.com/magiconair/properties#Properties.FilterRegexp) + +### [1.5.2](https://github.com/magiconair/properties/tree/v1.5.2) - 10 Apr 2015 + + * [Issue #3](https://github.com/magiconair/properties/issues/3): Don't print comments in [WriteComment()](http://godoc.org/github.com/magiconair/properties#Properties.WriteComment) if they are all empty + * Add clickable links to README + +### [1.5.1](https://github.com/magiconair/properties/tree/v1.5.1) - 08 Dec 2014 + + * Added [GetParsedDuration()](http://godoc.org/github.com/magiconair/properties#Properties.GetParsedDuration) and [MustGetParsedDuration()](http://godoc.org/github.com/magiconair/properties#Properties.MustGetParsedDuration) for values specified compatible with + [time.ParseDuration()](http://golang.org/pkg/time/#ParseDuration). + +### [1.5.0](https://github.com/magiconair/properties/tree/v1.5.0) - 18 Nov 2014 + + * Added support for single and multi-line comments (reading, writing and updating) + * The order of keys is now preserved + * Calling [Set()](http://godoc.org/github.com/magiconair/properties#Properties.Set) with an empty key now silently ignores the call and does not create a new entry + * Added a [MustSet()](http://godoc.org/github.com/magiconair/properties#Properties.MustSet) method + * Migrated test library from launchpad.net/gocheck to [gopkg.in/check.v1](http://gopkg.in/check.v1) + +### [1.4.2](https://github.com/magiconair/properties/tree/v1.4.2) - 15 Nov 2014 + + * [Issue #2](https://github.com/magiconair/properties/issues/2): Fixed goroutine leak in parser which created two lexers but cleaned up only one + +### [1.4.1](https://github.com/magiconair/properties/tree/v1.4.1) - 13 Nov 2014 + + * [Issue #1](https://github.com/magiconair/properties/issues/1): Fixed bug in Keys() method which returned an empty string + +### [1.4.0](https://github.com/magiconair/properties/tree/v1.4.0) - 23 Sep 2014 + + * Added [Keys()](http://godoc.org/github.com/magiconair/properties#Properties.Keys) to get the keys + * Added [Filter()](http://godoc.org/github.com/magiconair/properties#Properties.Filter), [FilterRegexp()](http://godoc.org/github.com/magiconair/properties#Properties.FilterRegexp) and [FilterPrefix()](http://godoc.org/github.com/magiconair/properties#Properties.FilterPrefix) to get a subset of the properties + +### [1.3.0](https://github.com/magiconair/properties/tree/v1.3.0) - 18 Mar 2014 + +* Added support for time.Duration +* Made MustXXX() failure beha[ior configurable (log.Fatal, panic](https://github.com/magiconair/properties/tree/vior configurable (log.Fatal, panic) - custom) +* Changed default of MustXXX() failure from panic to log.Fatal + +### [1.2.0](https://github.com/magiconair/properties/tree/v1.2.0) - 05 Mar 2014 + +* Added MustGet... functions +* Added support for int and uint with range checks on 32 bit platforms + +### [1.1.0](https://github.com/magiconair/properties/tree/v1.1.0) - 20 Jan 2014 + +* Renamed from goproperties to properties +* Added support for expansion of environment vars in + filenames and value expressions +* Fixed bug where value expressions were not at the + start of the string + +### [1.0.0](https://github.com/magiconair/properties/tree/v1.0.0) - 7 Jan 2014 + +* Initial release diff --git a/vendor/github.com/magiconair/properties/LICENSE b/vendor/github.com/magiconair/properties/LICENSE new file mode 100644 index 000000000..b387087c5 --- /dev/null +++ b/vendor/github.com/magiconair/properties/LICENSE @@ -0,0 +1,25 @@ +goproperties - properties file decoder for Go + +Copyright (c) 2013-2018 - Frank Schroeder + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/magiconair/properties/README.md b/vendor/github.com/magiconair/properties/README.md new file mode 100644 index 000000000..2c05f290f --- /dev/null +++ b/vendor/github.com/magiconair/properties/README.md @@ -0,0 +1,129 @@ +[![](https://img.shields.io/github/tag/magiconair/properties.svg?style=flat-square&label=release)](https://github.com/magiconair/properties/releases) +[![Travis CI Status](https://img.shields.io/travis/magiconair/properties.svg?branch=master&style=flat-square&label=travis)](https://travis-ci.org/magiconair/properties) +[![Codeship CI Status](https://img.shields.io/codeship/16aaf660-f615-0135-b8f0-7e33b70920c0/master.svg?label=codeship&style=flat-square)](https://app.codeship.com/projects/274177") +[![License](https://img.shields.io/badge/License-BSD%202--Clause-orange.svg?style=flat-square)](https://raw.githubusercontent.com/magiconair/properties/master/LICENSE) +[![GoDoc](http://img.shields.io/badge/godoc-reference-5272B4.svg?style=flat-square)](http://godoc.org/github.com/magiconair/properties) + +# Overview + +#### Please run `git pull --tags` to update the tags. See [below](#updated-git-tags) why. + +properties is a Go library for reading and writing properties files. + +It supports reading from multiple files or URLs and Spring style recursive +property expansion of expressions like `${key}` to their corresponding value. +Value expressions can refer to other keys like in `${key}` or to environment +variables like in `${USER}`. Filenames can also contain environment variables +like in `/home/${USER}/myapp.properties`. + +Properties can be decoded into structs, maps, arrays and values through +struct tags. + +Comments and the order of keys are preserved. Comments can be modified +and can be written to the output. + +The properties library supports both ISO-8859-1 and UTF-8 encoded data. + +Starting from version 1.3.0 the behavior of the MustXXX() functions is +configurable by providing a custom `ErrorHandler` function. The default has +changed from `panic` to `log.Fatal` but this is configurable and custom +error handling functions can be provided. See the package documentation for +details. + +Read the full documentation on [GoDoc](https://godoc.org/github.com/magiconair/properties) [![GoDoc](https://godoc.org/github.com/magiconair/properties?status.png)](https://godoc.org/github.com/magiconair/properties) + +## Getting Started + +```go +import ( + "flag" + "github.com/magiconair/properties" +) + +func main() { + // init from a file + p := properties.MustLoadFile("${HOME}/config.properties", properties.UTF8) + + // or multiple files + p = properties.MustLoadFiles([]string{ + "${HOME}/config.properties", + "${HOME}/config-${USER}.properties", + }, properties.UTF8, true) + + // or from a map + p = properties.LoadMap(map[string]string{"key": "value", "abc": "def"}) + + // or from a string + p = properties.MustLoadString("key=value\nabc=def") + + // or from a URL + p = properties.MustLoadURL("http://host/path") + + // or from multiple URLs + p = properties.MustLoadURL([]string{ + "http://host/config", + "http://host/config-${USER}", + }, true) + + // or from flags + p.MustFlag(flag.CommandLine) + + // get values through getters + host := p.MustGetString("host") + port := p.GetInt("port", 8080) + + // or through Decode + type Config struct { + Host string `properties:"host"` + Port int `properties:"port,default=9000"` + Accept []string `properties:"accept,default=image/png;image;gif"` + Timeout time.Duration `properties:"timeout,default=5s"` + } + var cfg Config + if err := p.Decode(&cfg); err != nil { + log.Fatal(err) + } +} + +``` + +## Installation and Upgrade + +``` +$ go get -u github.com/magiconair/properties +``` + +## License + +2 clause BSD license. See [LICENSE](https://github.com/magiconair/properties/blob/master/LICENSE) file for details. + +## ToDo + +* Dump contents with passwords and secrets obscured + +## Updated Git tags + +#### 13 Feb 2018 + +I realized that all of the git tags I had pushed before v1.7.5 were lightweight tags +and I've only recently learned that this doesn't play well with `git describe` 😞 + +I have replaced all lightweight tags with signed tags using this script which should +retain the commit date, name and email address. Please run `git pull --tags` to update them. + +Worst case you have to reclone the repo. + +```shell +#!/bin/bash +tag=$1 +echo "Updating $tag" +date=$(git show ${tag}^0 --format=%aD | head -1) +email=$(git show ${tag}^0 --format=%aE | head -1) +name=$(git show ${tag}^0 --format=%aN | head -1) +GIT_COMMITTER_DATE="$date" GIT_COMMITTER_NAME="$name" GIT_COMMITTER_EMAIL="$email" git tag -s -f ${tag} ${tag}^0 -m ${tag} +``` + +I apologize for the inconvenience. + +Frank + diff --git a/vendor/github.com/magiconair/properties/decode.go b/vendor/github.com/magiconair/properties/decode.go new file mode 100644 index 000000000..3ebf8049c --- /dev/null +++ b/vendor/github.com/magiconair/properties/decode.go @@ -0,0 +1,289 @@ +// Copyright 2018 Frank Schroeder. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package properties + +import ( + "fmt" + "reflect" + "strconv" + "strings" + "time" +) + +// Decode assigns property values to exported fields of a struct. +// +// Decode traverses v recursively and returns an error if a value cannot be +// converted to the field type or a required value is missing for a field. +// +// The following type dependent decodings are used: +// +// String, boolean, numeric fields have the value of the property key assigned. +// The property key name is the name of the field. A different key and a default +// value can be set in the field's tag. Fields without default value are +// required. If the value cannot be converted to the field type an error is +// returned. +// +// time.Duration fields have the result of time.ParseDuration() assigned. +// +// time.Time fields have the vaule of time.Parse() assigned. The default layout +// is time.RFC3339 but can be set in the field's tag. +// +// Arrays and slices of string, boolean, numeric, time.Duration and time.Time +// fields have the value interpreted as a comma separated list of values. The +// individual values are trimmed of whitespace and empty values are ignored. A +// default value can be provided as a semicolon separated list in the field's +// tag. +// +// Struct fields are decoded recursively using the field name plus "." as +// prefix. The prefix (without dot) can be overridden in the field's tag. +// Default values are not supported in the field's tag. Specify them on the +// fields of the inner struct instead. +// +// Map fields must have a key of type string and are decoded recursively by +// using the field's name plus ".' as prefix and the next element of the key +// name as map key. The prefix (without dot) can be overridden in the field's +// tag. Default values are not supported. +// +// Examples: +// +// // Field is ignored. +// Field int `properties:"-"` +// +// // Field is assigned value of 'Field'. +// Field int +// +// // Field is assigned value of 'myName'. +// Field int `properties:"myName"` +// +// // Field is assigned value of key 'myName' and has a default +// // value 15 if the key does not exist. +// Field int `properties:"myName,default=15"` +// +// // Field is assigned value of key 'Field' and has a default +// // value 15 if the key does not exist. +// Field int `properties:",default=15"` +// +// // Field is assigned value of key 'date' and the date +// // is in format 2006-01-02 +// Field time.Time `properties:"date,layout=2006-01-02"` +// +// // Field is assigned the non-empty and whitespace trimmed +// // values of key 'Field' split by commas. +// Field []string +// +// // Field is assigned the non-empty and whitespace trimmed +// // values of key 'Field' split by commas and has a default +// // value ["a", "b", "c"] if the key does not exist. +// Field []string `properties:",default=a;b;c"` +// +// // Field is decoded recursively with "Field." as key prefix. +// Field SomeStruct +// +// // Field is decoded recursively with "myName." as key prefix. +// Field SomeStruct `properties:"myName"` +// +// // Field is decoded recursively with "Field." as key prefix +// // and the next dotted element of the key as map key. +// Field map[string]string +// +// // Field is decoded recursively with "myName." as key prefix +// // and the next dotted element of the key as map key. +// Field map[string]string `properties:"myName"` +func (p *Properties) Decode(x interface{}) error { + t, v := reflect.TypeOf(x), reflect.ValueOf(x) + if t.Kind() != reflect.Ptr || v.Elem().Type().Kind() != reflect.Struct { + return fmt.Errorf("not a pointer to struct: %s", t) + } + if err := dec(p, "", nil, nil, v); err != nil { + return err + } + return nil +} + +func dec(p *Properties, key string, def *string, opts map[string]string, v reflect.Value) error { + t := v.Type() + + // value returns the property value for key or the default if provided. + value := func() (string, error) { + if val, ok := p.Get(key); ok { + return val, nil + } + if def != nil { + return *def, nil + } + return "", fmt.Errorf("missing required key %s", key) + } + + // conv converts a string to a value of the given type. + conv := func(s string, t reflect.Type) (val reflect.Value, err error) { + var v interface{} + + switch { + case isDuration(t): + v, err = time.ParseDuration(s) + + case isTime(t): + layout := opts["layout"] + if layout == "" { + layout = time.RFC3339 + } + v, err = time.Parse(layout, s) + + case isBool(t): + v, err = boolVal(s), nil + + case isString(t): + v, err = s, nil + + case isFloat(t): + v, err = strconv.ParseFloat(s, 64) + + case isInt(t): + v, err = strconv.ParseInt(s, 10, 64) + + case isUint(t): + v, err = strconv.ParseUint(s, 10, 64) + + default: + return reflect.Zero(t), fmt.Errorf("unsupported type %s", t) + } + if err != nil { + return reflect.Zero(t), err + } + return reflect.ValueOf(v).Convert(t), nil + } + + // keydef returns the property key and the default value based on the + // name of the struct field and the options in the tag. + keydef := func(f reflect.StructField) (string, *string, map[string]string) { + _key, _opts := parseTag(f.Tag.Get("properties")) + + var _def *string + if d, ok := _opts["default"]; ok { + _def = &d + } + if _key != "" { + return _key, _def, _opts + } + return f.Name, _def, _opts + } + + switch { + case isDuration(t) || isTime(t) || isBool(t) || isString(t) || isFloat(t) || isInt(t) || isUint(t): + s, err := value() + if err != nil { + return err + } + val, err := conv(s, t) + if err != nil { + return err + } + v.Set(val) + + case isPtr(t): + return dec(p, key, def, opts, v.Elem()) + + case isStruct(t): + for i := 0; i < v.NumField(); i++ { + fv := v.Field(i) + fk, def, opts := keydef(t.Field(i)) + if !fv.CanSet() { + return fmt.Errorf("cannot set %s", t.Field(i).Name) + } + if fk == "-" { + continue + } + if key != "" { + fk = key + "." + fk + } + if err := dec(p, fk, def, opts, fv); err != nil { + return err + } + } + return nil + + case isArray(t): + val, err := value() + if err != nil { + return err + } + vals := split(val, ";") + a := reflect.MakeSlice(t, 0, len(vals)) + for _, s := range vals { + val, err := conv(s, t.Elem()) + if err != nil { + return err + } + a = reflect.Append(a, val) + } + v.Set(a) + + case isMap(t): + valT := t.Elem() + m := reflect.MakeMap(t) + for postfix := range p.FilterStripPrefix(key + ".").m { + pp := strings.SplitN(postfix, ".", 2) + mk, mv := pp[0], reflect.New(valT) + if err := dec(p, key+"."+mk, nil, nil, mv); err != nil { + return err + } + m.SetMapIndex(reflect.ValueOf(mk), mv.Elem()) + } + v.Set(m) + + default: + return fmt.Errorf("unsupported type %s", t) + } + return nil +} + +// split splits a string on sep, trims whitespace of elements +// and omits empty elements +func split(s string, sep string) []string { + var a []string + for _, v := range strings.Split(s, sep) { + if v = strings.TrimSpace(v); v != "" { + a = append(a, v) + } + } + return a +} + +// parseTag parses a "key,k=v,k=v,..." +func parseTag(tag string) (key string, opts map[string]string) { + opts = map[string]string{} + for i, s := range strings.Split(tag, ",") { + if i == 0 { + key = s + continue + } + + pp := strings.SplitN(s, "=", 2) + if len(pp) == 1 { + opts[pp[0]] = "" + } else { + opts[pp[0]] = pp[1] + } + } + return key, opts +} + +func isArray(t reflect.Type) bool { return t.Kind() == reflect.Array || t.Kind() == reflect.Slice } +func isBool(t reflect.Type) bool { return t.Kind() == reflect.Bool } +func isDuration(t reflect.Type) bool { return t == reflect.TypeOf(time.Second) } +func isMap(t reflect.Type) bool { return t.Kind() == reflect.Map } +func isPtr(t reflect.Type) bool { return t.Kind() == reflect.Ptr } +func isString(t reflect.Type) bool { return t.Kind() == reflect.String } +func isStruct(t reflect.Type) bool { return t.Kind() == reflect.Struct } +func isTime(t reflect.Type) bool { return t == reflect.TypeOf(time.Time{}) } +func isFloat(t reflect.Type) bool { + return t.Kind() == reflect.Float32 || t.Kind() == reflect.Float64 +} +func isInt(t reflect.Type) bool { + return t.Kind() == reflect.Int || t.Kind() == reflect.Int8 || t.Kind() == reflect.Int16 || t.Kind() == reflect.Int32 || t.Kind() == reflect.Int64 +} +func isUint(t reflect.Type) bool { + return t.Kind() == reflect.Uint || t.Kind() == reflect.Uint8 || t.Kind() == reflect.Uint16 || t.Kind() == reflect.Uint32 || t.Kind() == reflect.Uint64 +} diff --git a/vendor/github.com/magiconair/properties/doc.go b/vendor/github.com/magiconair/properties/doc.go new file mode 100644 index 000000000..f8822da2b --- /dev/null +++ b/vendor/github.com/magiconair/properties/doc.go @@ -0,0 +1,156 @@ +// Copyright 2018 Frank Schroeder. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package properties provides functions for reading and writing +// ISO-8859-1 and UTF-8 encoded .properties files and has +// support for recursive property expansion. +// +// Java properties files are ISO-8859-1 encoded and use Unicode +// literals for characters outside the ISO character set. Unicode +// literals can be used in UTF-8 encoded properties files but +// aren't necessary. +// +// To load a single properties file use MustLoadFile(): +// +// p := properties.MustLoadFile(filename, properties.UTF8) +// +// To load multiple properties files use MustLoadFiles() +// which loads the files in the given order and merges the +// result. Missing properties files can be ignored if the +// 'ignoreMissing' flag is set to true. +// +// Filenames can contain environment variables which are expanded +// before loading. +// +// f1 := "/etc/myapp/myapp.conf" +// f2 := "/home/${USER}/myapp.conf" +// p := MustLoadFiles([]string{f1, f2}, properties.UTF8, true) +// +// All of the different key/value delimiters ' ', ':' and '=' are +// supported as well as the comment characters '!' and '#' and +// multi-line values. +// +// ! this is a comment +// # and so is this +// +// # the following expressions are equal +// key value +// key=value +// key:value +// key = value +// key : value +// key = val\ +// ue +// +// Properties stores all comments preceding a key and provides +// GetComments() and SetComments() methods to retrieve and +// update them. The convenience functions GetComment() and +// SetComment() allow access to the last comment. The +// WriteComment() method writes properties files including +// the comments and with the keys in the original order. +// This can be used for sanitizing properties files. +// +// Property expansion is recursive and circular references +// and malformed expressions are not allowed and cause an +// error. Expansion of environment variables is supported. +// +// # standard property +// key = value +// +// # property expansion: key2 = value +// key2 = ${key} +// +// # recursive expansion: key3 = value +// key3 = ${key2} +// +// # circular reference (error) +// key = ${key} +// +// # malformed expression (error) +// key = ${ke +// +// # refers to the users' home dir +// home = ${HOME} +// +// # local key takes precedence over env var: u = foo +// USER = foo +// u = ${USER} +// +// The default property expansion format is ${key} but can be +// changed by setting different pre- and postfix values on the +// Properties object. +// +// p := properties.NewProperties() +// p.Prefix = "#[" +// p.Postfix = "]#" +// +// Properties provides convenience functions for getting typed +// values with default values if the key does not exist or the +// type conversion failed. +// +// # Returns true if the value is either "1", "on", "yes" or "true" +// # Returns false for every other value and the default value if +// # the key does not exist. +// v = p.GetBool("key", false) +// +// # Returns the value if the key exists and the format conversion +// # was successful. Otherwise, the default value is returned. +// v = p.GetInt64("key", 999) +// v = p.GetUint64("key", 999) +// v = p.GetFloat64("key", 123.0) +// v = p.GetString("key", "def") +// v = p.GetDuration("key", 999) +// +// As an alternative properties may be applied with the standard +// library's flag implementation at any time. +// +// # Standard configuration +// v = flag.Int("key", 999, "help message") +// flag.Parse() +// +// # Merge p into the flag set +// p.MustFlag(flag.CommandLine) +// +// Properties provides several MustXXX() convenience functions +// which will terminate the app if an error occurs. The behavior +// of the failure is configurable and the default is to call +// log.Fatal(err). To have the MustXXX() functions panic instead +// of logging the error set a different ErrorHandler before +// you use the Properties package. +// +// properties.ErrorHandler = properties.PanicHandler +// +// # Will panic instead of logging an error +// p := properties.MustLoadFile("config.properties") +// +// You can also provide your own ErrorHandler function. The only requirement +// is that the error handler function must exit after handling the error. +// +// properties.ErrorHandler = func(err error) { +// fmt.Println(err) +// os.Exit(1) +// } +// +// # Will write to stdout and then exit +// p := properties.MustLoadFile("config.properties") +// +// Properties can also be loaded into a struct via the `Decode` +// method, e.g. +// +// type S struct { +// A string `properties:"a,default=foo"` +// D time.Duration `properties:"timeout,default=5s"` +// E time.Time `properties:"expires,layout=2006-01-02,default=2015-01-01"` +// } +// +// See `Decode()` method for the full documentation. +// +// The following documents provide a description of the properties +// file format. +// +// http://en.wikipedia.org/wiki/.properties +// +// http://docs.oracle.com/javase/7/docs/api/java/util/Properties.html#load%28java.io.Reader%29 +// +package properties diff --git a/vendor/github.com/magiconair/properties/integrate.go b/vendor/github.com/magiconair/properties/integrate.go new file mode 100644 index 000000000..74d38dc67 --- /dev/null +++ b/vendor/github.com/magiconair/properties/integrate.go @@ -0,0 +1,34 @@ +// Copyright 2018 Frank Schroeder. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package properties + +import "flag" + +// MustFlag sets flags that are skipped by dst.Parse when p contains +// the respective key for flag.Flag.Name. +// +// It's use is recommended with command line arguments as in: +// flag.Parse() +// p.MustFlag(flag.CommandLine) +func (p *Properties) MustFlag(dst *flag.FlagSet) { + m := make(map[string]*flag.Flag) + dst.VisitAll(func(f *flag.Flag) { + m[f.Name] = f + }) + dst.Visit(func(f *flag.Flag) { + delete(m, f.Name) // overridden + }) + + for name, f := range m { + v, ok := p.Get(name) + if !ok { + continue + } + + if err := f.Value.Set(v); err != nil { + ErrorHandler(err) + } + } +} diff --git a/vendor/github.com/magiconair/properties/lex.go b/vendor/github.com/magiconair/properties/lex.go new file mode 100644 index 000000000..367166d58 --- /dev/null +++ b/vendor/github.com/magiconair/properties/lex.go @@ -0,0 +1,407 @@ +// Copyright 2018 Frank Schroeder. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +// +// Parts of the lexer are from the template/text/parser package +// For these parts the following applies: +// +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file of the go 1.2 +// distribution. + +package properties + +import ( + "fmt" + "strconv" + "strings" + "unicode/utf8" +) + +// item represents a token or text string returned from the scanner. +type item struct { + typ itemType // The type of this item. + pos int // The starting position, in bytes, of this item in the input string. + val string // The value of this item. +} + +func (i item) String() string { + switch { + case i.typ == itemEOF: + return "EOF" + case i.typ == itemError: + return i.val + case len(i.val) > 10: + return fmt.Sprintf("%.10q...", i.val) + } + return fmt.Sprintf("%q", i.val) +} + +// itemType identifies the type of lex items. +type itemType int + +const ( + itemError itemType = iota // error occurred; value is text of error + itemEOF + itemKey // a key + itemValue // a value + itemComment // a comment +) + +// defines a constant for EOF +const eof = -1 + +// permitted whitespace characters space, FF and TAB +const whitespace = " \f\t" + +// stateFn represents the state of the scanner as a function that returns the next state. +type stateFn func(*lexer) stateFn + +// lexer holds the state of the scanner. +type lexer struct { + input string // the string being scanned + state stateFn // the next lexing function to enter + pos int // current position in the input + start int // start position of this item + width int // width of last rune read from input + lastPos int // position of most recent item returned by nextItem + runes []rune // scanned runes for this item + items chan item // channel of scanned items +} + +// next returns the next rune in the input. +func (l *lexer) next() rune { + if l.pos >= len(l.input) { + l.width = 0 + return eof + } + r, w := utf8.DecodeRuneInString(l.input[l.pos:]) + l.width = w + l.pos += l.width + return r +} + +// peek returns but does not consume the next rune in the input. +func (l *lexer) peek() rune { + r := l.next() + l.backup() + return r +} + +// backup steps back one rune. Can only be called once per call of next. +func (l *lexer) backup() { + l.pos -= l.width +} + +// emit passes an item back to the client. +func (l *lexer) emit(t itemType) { + i := item{t, l.start, string(l.runes)} + l.items <- i + l.start = l.pos + l.runes = l.runes[:0] +} + +// ignore skips over the pending input before this point. +func (l *lexer) ignore() { + l.start = l.pos +} + +// appends the rune to the current value +func (l *lexer) appendRune(r rune) { + l.runes = append(l.runes, r) +} + +// accept consumes the next rune if it's from the valid set. +func (l *lexer) accept(valid string) bool { + if strings.ContainsRune(valid, l.next()) { + return true + } + l.backup() + return false +} + +// acceptRun consumes a run of runes from the valid set. +func (l *lexer) acceptRun(valid string) { + for strings.ContainsRune(valid, l.next()) { + } + l.backup() +} + +// acceptRunUntil consumes a run of runes up to a terminator. +func (l *lexer) acceptRunUntil(term rune) { + for term != l.next() { + } + l.backup() +} + +// hasText returns true if the current parsed text is not empty. +func (l *lexer) isNotEmpty() bool { + return l.pos > l.start +} + +// lineNumber reports which line we're on, based on the position of +// the previous item returned by nextItem. Doing it this way +// means we don't have to worry about peek double counting. +func (l *lexer) lineNumber() int { + return 1 + strings.Count(l.input[:l.lastPos], "\n") +} + +// errorf returns an error token and terminates the scan by passing +// back a nil pointer that will be the next state, terminating l.nextItem. +func (l *lexer) errorf(format string, args ...interface{}) stateFn { + l.items <- item{itemError, l.start, fmt.Sprintf(format, args...)} + return nil +} + +// nextItem returns the next item from the input. +func (l *lexer) nextItem() item { + i := <-l.items + l.lastPos = i.pos + return i +} + +// lex creates a new scanner for the input string. +func lex(input string) *lexer { + l := &lexer{ + input: input, + items: make(chan item), + runes: make([]rune, 0, 32), + } + go l.run() + return l +} + +// run runs the state machine for the lexer. +func (l *lexer) run() { + for l.state = lexBeforeKey(l); l.state != nil; { + l.state = l.state(l) + } +} + +// state functions + +// lexBeforeKey scans until a key begins. +func lexBeforeKey(l *lexer) stateFn { + switch r := l.next(); { + case isEOF(r): + l.emit(itemEOF) + return nil + + case isEOL(r): + l.ignore() + return lexBeforeKey + + case isComment(r): + return lexComment + + case isWhitespace(r): + l.ignore() + return lexBeforeKey + + default: + l.backup() + return lexKey + } +} + +// lexComment scans a comment line. The comment character has already been scanned. +func lexComment(l *lexer) stateFn { + l.acceptRun(whitespace) + l.ignore() + for { + switch r := l.next(); { + case isEOF(r): + l.ignore() + l.emit(itemEOF) + return nil + case isEOL(r): + l.emit(itemComment) + return lexBeforeKey + default: + l.appendRune(r) + } + } +} + +// lexKey scans the key up to a delimiter +func lexKey(l *lexer) stateFn { + var r rune + +Loop: + for { + switch r = l.next(); { + + case isEscape(r): + err := l.scanEscapeSequence() + if err != nil { + return l.errorf(err.Error()) + } + + case isEndOfKey(r): + l.backup() + break Loop + + case isEOF(r): + break Loop + + default: + l.appendRune(r) + } + } + + if len(l.runes) > 0 { + l.emit(itemKey) + } + + if isEOF(r) { + l.emit(itemEOF) + return nil + } + + return lexBeforeValue +} + +// lexBeforeValue scans the delimiter between key and value. +// Leading and trailing whitespace is ignored. +// We expect to be just after the key. +func lexBeforeValue(l *lexer) stateFn { + l.acceptRun(whitespace) + l.accept(":=") + l.acceptRun(whitespace) + l.ignore() + return lexValue +} + +// lexValue scans text until the end of the line. We expect to be just after the delimiter. +func lexValue(l *lexer) stateFn { + for { + switch r := l.next(); { + case isEscape(r): + if isEOL(l.peek()) { + l.next() + l.acceptRun(whitespace) + } else { + err := l.scanEscapeSequence() + if err != nil { + return l.errorf(err.Error()) + } + } + + case isEOL(r): + l.emit(itemValue) + l.ignore() + return lexBeforeKey + + case isEOF(r): + l.emit(itemValue) + l.emit(itemEOF) + return nil + + default: + l.appendRune(r) + } + } +} + +// scanEscapeSequence scans either one of the escaped characters +// or a unicode literal. We expect to be after the escape character. +func (l *lexer) scanEscapeSequence() error { + switch r := l.next(); { + + case isEscapedCharacter(r): + l.appendRune(decodeEscapedCharacter(r)) + return nil + + case atUnicodeLiteral(r): + return l.scanUnicodeLiteral() + + case isEOF(r): + return fmt.Errorf("premature EOF") + + // silently drop the escape character and append the rune as is + default: + l.appendRune(r) + return nil + } +} + +// scans a unicode literal in the form \uXXXX. We expect to be after the \u. +func (l *lexer) scanUnicodeLiteral() error { + // scan the digits + d := make([]rune, 4) + for i := 0; i < 4; i++ { + d[i] = l.next() + if d[i] == eof || !strings.ContainsRune("0123456789abcdefABCDEF", d[i]) { + return fmt.Errorf("invalid unicode literal") + } + } + + // decode the digits into a rune + r, err := strconv.ParseInt(string(d), 16, 0) + if err != nil { + return err + } + + l.appendRune(rune(r)) + return nil +} + +// decodeEscapedCharacter returns the unescaped rune. We expect to be after the escape character. +func decodeEscapedCharacter(r rune) rune { + switch r { + case 'f': + return '\f' + case 'n': + return '\n' + case 'r': + return '\r' + case 't': + return '\t' + default: + return r + } +} + +// atUnicodeLiteral reports whether we are at a unicode literal. +// The escape character has already been consumed. +func atUnicodeLiteral(r rune) bool { + return r == 'u' +} + +// isComment reports whether we are at the start of a comment. +func isComment(r rune) bool { + return r == '#' || r == '!' +} + +// isEndOfKey reports whether the rune terminates the current key. +func isEndOfKey(r rune) bool { + return strings.ContainsRune(" \f\t\r\n:=", r) +} + +// isEOF reports whether we are at EOF. +func isEOF(r rune) bool { + return r == eof +} + +// isEOL reports whether we are at a new line character. +func isEOL(r rune) bool { + return r == '\n' || r == '\r' +} + +// isEscape reports whether the rune is the escape character which +// prefixes unicode literals and other escaped characters. +func isEscape(r rune) bool { + return r == '\\' +} + +// isEscapedCharacter reports whether we are at one of the characters that need escaping. +// The escape character has already been consumed. +func isEscapedCharacter(r rune) bool { + return strings.ContainsRune(" :=fnrt", r) +} + +// isWhitespace reports whether the rune is a whitespace character. +func isWhitespace(r rune) bool { + return strings.ContainsRune(whitespace, r) +} diff --git a/vendor/github.com/magiconair/properties/load.go b/vendor/github.com/magiconair/properties/load.go new file mode 100644 index 000000000..c8e1b5804 --- /dev/null +++ b/vendor/github.com/magiconair/properties/load.go @@ -0,0 +1,292 @@ +// Copyright 2018 Frank Schroeder. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package properties + +import ( + "fmt" + "io/ioutil" + "net/http" + "os" + "strings" +) + +// Encoding specifies encoding of the input data. +type Encoding uint + +const ( + // utf8Default is a private placeholder for the zero value of Encoding to + // ensure that it has the correct meaning. UTF8 is the default encoding but + // was assigned a non-zero value which cannot be changed without breaking + // existing code. Clients should continue to use the public constants. + utf8Default Encoding = iota + + // UTF8 interprets the input data as UTF-8. + UTF8 + + // ISO_8859_1 interprets the input data as ISO-8859-1. + ISO_8859_1 +) + +type Loader struct { + // Encoding determines how the data from files and byte buffers + // is interpreted. For URLs the Content-Type header is used + // to determine the encoding of the data. + Encoding Encoding + + // DisableExpansion configures the property expansion of the + // returned property object. When set to true, the property values + // will not be expanded and the Property object will not be checked + // for invalid expansion expressions. + DisableExpansion bool + + // IgnoreMissing configures whether missing files or URLs which return + // 404 are reported as errors. When set to true, missing files and 404 + // status codes are not reported as errors. + IgnoreMissing bool +} + +// Load reads a buffer into a Properties struct. +func (l *Loader) LoadBytes(buf []byte) (*Properties, error) { + return l.loadBytes(buf, l.Encoding) +} + +// LoadAll reads the content of multiple URLs or files in the given order into +// a Properties struct. If IgnoreMissing is true then a 404 status code or +// missing file will not be reported as error. Encoding sets the encoding for +// files. For the URLs see LoadURL for the Content-Type header and the +// encoding. +func (l *Loader) LoadAll(names []string) (*Properties, error) { + all := NewProperties() + for _, name := range names { + n, err := expandName(name) + if err != nil { + return nil, err + } + + var p *Properties + switch { + case strings.HasPrefix(n, "http://"): + p, err = l.LoadURL(n) + case strings.HasPrefix(n, "https://"): + p, err = l.LoadURL(n) + default: + p, err = l.LoadFile(n) + } + if err != nil { + return nil, err + } + all.Merge(p) + } + + all.DisableExpansion = l.DisableExpansion + if all.DisableExpansion { + return all, nil + } + return all, all.check() +} + +// LoadFile reads a file into a Properties struct. +// If IgnoreMissing is true then a missing file will not be +// reported as error. +func (l *Loader) LoadFile(filename string) (*Properties, error) { + data, err := ioutil.ReadFile(filename) + if err != nil { + if l.IgnoreMissing && os.IsNotExist(err) { + LogPrintf("properties: %s not found. skipping", filename) + return NewProperties(), nil + } + return nil, err + } + return l.loadBytes(data, l.Encoding) +} + +// LoadURL reads the content of the URL into a Properties struct. +// +// The encoding is determined via the Content-Type header which +// should be set to 'text/plain'. If the 'charset' parameter is +// missing, 'iso-8859-1' or 'latin1' the encoding is set to +// ISO-8859-1. If the 'charset' parameter is set to 'utf-8' the +// encoding is set to UTF-8. A missing content type header is +// interpreted as 'text/plain; charset=utf-8'. +func (l *Loader) LoadURL(url string) (*Properties, error) { + resp, err := http.Get(url) + if err != nil { + return nil, fmt.Errorf("properties: error fetching %q. %s", url, err) + } + + if resp.StatusCode == 404 && l.IgnoreMissing { + LogPrintf("properties: %s returned %d. skipping", url, resp.StatusCode) + return NewProperties(), nil + } + + if resp.StatusCode != 200 { + return nil, fmt.Errorf("properties: %s returned %d", url, resp.StatusCode) + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("properties: %s error reading response. %s", url, err) + } + defer resp.Body.Close() + + ct := resp.Header.Get("Content-Type") + var enc Encoding + switch strings.ToLower(ct) { + case "text/plain", "text/plain; charset=iso-8859-1", "text/plain; charset=latin1": + enc = ISO_8859_1 + case "", "text/plain; charset=utf-8": + enc = UTF8 + default: + return nil, fmt.Errorf("properties: invalid content type %s", ct) + } + + return l.loadBytes(body, enc) +} + +func (l *Loader) loadBytes(buf []byte, enc Encoding) (*Properties, error) { + p, err := parse(convert(buf, enc)) + if err != nil { + return nil, err + } + p.DisableExpansion = l.DisableExpansion + if p.DisableExpansion { + return p, nil + } + return p, p.check() +} + +// Load reads a buffer into a Properties struct. +func Load(buf []byte, enc Encoding) (*Properties, error) { + l := &Loader{Encoding: enc} + return l.LoadBytes(buf) +} + +// LoadString reads an UTF8 string into a properties struct. +func LoadString(s string) (*Properties, error) { + l := &Loader{Encoding: UTF8} + return l.LoadBytes([]byte(s)) +} + +// LoadMap creates a new Properties struct from a string map. +func LoadMap(m map[string]string) *Properties { + p := NewProperties() + for k, v := range m { + p.Set(k, v) + } + return p +} + +// LoadFile reads a file into a Properties struct. +func LoadFile(filename string, enc Encoding) (*Properties, error) { + l := &Loader{Encoding: enc} + return l.LoadAll([]string{filename}) +} + +// LoadFiles reads multiple files in the given order into +// a Properties struct. If 'ignoreMissing' is true then +// non-existent files will not be reported as error. +func LoadFiles(filenames []string, enc Encoding, ignoreMissing bool) (*Properties, error) { + l := &Loader{Encoding: enc, IgnoreMissing: ignoreMissing} + return l.LoadAll(filenames) +} + +// LoadURL reads the content of the URL into a Properties struct. +// See Loader#LoadURL for details. +func LoadURL(url string) (*Properties, error) { + l := &Loader{Encoding: UTF8} + return l.LoadAll([]string{url}) +} + +// LoadURLs reads the content of multiple URLs in the given order into a +// Properties struct. If IgnoreMissing is true then a 404 status code will +// not be reported as error. See Loader#LoadURL for the Content-Type header +// and the encoding. +func LoadURLs(urls []string, ignoreMissing bool) (*Properties, error) { + l := &Loader{Encoding: UTF8, IgnoreMissing: ignoreMissing} + return l.LoadAll(urls) +} + +// LoadAll reads the content of multiple URLs or files in the given order into a +// Properties struct. If 'ignoreMissing' is true then a 404 status code or missing file will +// not be reported as error. Encoding sets the encoding for files. For the URLs please see +// LoadURL for the Content-Type header and the encoding. +func LoadAll(names []string, enc Encoding, ignoreMissing bool) (*Properties, error) { + l := &Loader{Encoding: enc, IgnoreMissing: ignoreMissing} + return l.LoadAll(names) +} + +// MustLoadString reads an UTF8 string into a Properties struct and +// panics on error. +func MustLoadString(s string) *Properties { + return must(LoadString(s)) +} + +// MustLoadFile reads a file into a Properties struct and +// panics on error. +func MustLoadFile(filename string, enc Encoding) *Properties { + return must(LoadFile(filename, enc)) +} + +// MustLoadFiles reads multiple files in the given order into +// a Properties struct and panics on error. If 'ignoreMissing' +// is true then non-existent files will not be reported as error. +func MustLoadFiles(filenames []string, enc Encoding, ignoreMissing bool) *Properties { + return must(LoadFiles(filenames, enc, ignoreMissing)) +} + +// MustLoadURL reads the content of a URL into a Properties struct and +// panics on error. +func MustLoadURL(url string) *Properties { + return must(LoadURL(url)) +} + +// MustLoadURLs reads the content of multiple URLs in the given order into a +// Properties struct and panics on error. If 'ignoreMissing' is true then a 404 +// status code will not be reported as error. +func MustLoadURLs(urls []string, ignoreMissing bool) *Properties { + return must(LoadURLs(urls, ignoreMissing)) +} + +// MustLoadAll reads the content of multiple URLs or files in the given order into a +// Properties struct. If 'ignoreMissing' is true then a 404 status code or missing file will +// not be reported as error. Encoding sets the encoding for files. For the URLs please see +// LoadURL for the Content-Type header and the encoding. It panics on error. +func MustLoadAll(names []string, enc Encoding, ignoreMissing bool) *Properties { + return must(LoadAll(names, enc, ignoreMissing)) +} + +func must(p *Properties, err error) *Properties { + if err != nil { + ErrorHandler(err) + } + return p +} + +// expandName expands ${ENV_VAR} expressions in a name. +// If the environment variable does not exist then it will be replaced +// with an empty string. Malformed expressions like "${ENV_VAR" will +// be reported as error. +func expandName(name string) (string, error) { + return expand(name, []string{}, "${", "}", make(map[string]string)) +} + +// Interprets a byte buffer either as an ISO-8859-1 or UTF-8 encoded string. +// For ISO-8859-1 we can convert each byte straight into a rune since the +// first 256 unicode code points cover ISO-8859-1. +func convert(buf []byte, enc Encoding) string { + switch enc { + case utf8Default, UTF8: + return string(buf) + case ISO_8859_1: + runes := make([]rune, len(buf)) + for i, b := range buf { + runes[i] = rune(b) + } + return string(runes) + default: + ErrorHandler(fmt.Errorf("unsupported encoding %v", enc)) + } + panic("ErrorHandler should exit") +} diff --git a/vendor/github.com/magiconair/properties/parser.go b/vendor/github.com/magiconair/properties/parser.go new file mode 100644 index 000000000..cdc4a8034 --- /dev/null +++ b/vendor/github.com/magiconair/properties/parser.go @@ -0,0 +1,95 @@ +// Copyright 2018 Frank Schroeder. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package properties + +import ( + "fmt" + "runtime" +) + +type parser struct { + lex *lexer +} + +func parse(input string) (properties *Properties, err error) { + p := &parser{lex: lex(input)} + defer p.recover(&err) + + properties = NewProperties() + key := "" + comments := []string{} + + for { + token := p.expectOneOf(itemComment, itemKey, itemEOF) + switch token.typ { + case itemEOF: + goto done + case itemComment: + comments = append(comments, token.val) + continue + case itemKey: + key = token.val + if _, ok := properties.m[key]; !ok { + properties.k = append(properties.k, key) + } + } + + token = p.expectOneOf(itemValue, itemEOF) + if len(comments) > 0 { + properties.c[key] = comments + comments = []string{} + } + switch token.typ { + case itemEOF: + properties.m[key] = "" + goto done + case itemValue: + properties.m[key] = token.val + } + } + +done: + return properties, nil +} + +func (p *parser) errorf(format string, args ...interface{}) { + format = fmt.Sprintf("properties: Line %d: %s", p.lex.lineNumber(), format) + panic(fmt.Errorf(format, args...)) +} + +func (p *parser) expect(expected itemType) (token item) { + token = p.lex.nextItem() + if token.typ != expected { + p.unexpected(token) + } + return token +} + +func (p *parser) expectOneOf(expected ...itemType) (token item) { + token = p.lex.nextItem() + for _, v := range expected { + if token.typ == v { + return token + } + } + p.unexpected(token) + panic("unexpected token") +} + +func (p *parser) unexpected(token item) { + p.errorf(token.String()) +} + +// recover is the handler that turns panics into returns from the top level of Parse. +func (p *parser) recover(errp *error) { + e := recover() + if e != nil { + if _, ok := e.(runtime.Error); ok { + panic(e) + } + *errp = e.(error) + } + return +} diff --git a/vendor/github.com/magiconair/properties/properties.go b/vendor/github.com/magiconair/properties/properties.go new file mode 100644 index 000000000..cb3d1a332 --- /dev/null +++ b/vendor/github.com/magiconair/properties/properties.go @@ -0,0 +1,833 @@ +// Copyright 2018 Frank Schroeder. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package properties + +// BUG(frank): Set() does not check for invalid unicode literals since this is currently handled by the lexer. +// BUG(frank): Write() does not allow to configure the newline character. Therefore, on Windows LF is used. + +import ( + "fmt" + "io" + "log" + "os" + "regexp" + "strconv" + "strings" + "time" + "unicode/utf8" +) + +const maxExpansionDepth = 64 + +// ErrorHandlerFunc defines the type of function which handles failures +// of the MustXXX() functions. An error handler function must exit +// the application after handling the error. +type ErrorHandlerFunc func(error) + +// ErrorHandler is the function which handles failures of the MustXXX() +// functions. The default is LogFatalHandler. +var ErrorHandler ErrorHandlerFunc = LogFatalHandler + +// LogHandlerFunc defines the function prototype for logging errors. +type LogHandlerFunc func(fmt string, args ...interface{}) + +// LogPrintf defines a log handler which uses log.Printf. +var LogPrintf LogHandlerFunc = log.Printf + +// LogFatalHandler handles the error by logging a fatal error and exiting. +func LogFatalHandler(err error) { + log.Fatal(err) +} + +// PanicHandler handles the error by panicking. +func PanicHandler(err error) { + panic(err) +} + +// ----------------------------------------------------------------------------- + +// A Properties contains the key/value pairs from the properties input. +// All values are stored in unexpanded form and are expanded at runtime +type Properties struct { + // Pre-/Postfix for property expansion. + Prefix string + Postfix string + + // DisableExpansion controls the expansion of properties on Get() + // and the check for circular references on Set(). When set to + // true Properties behaves like a simple key/value store and does + // not check for circular references on Get() or on Set(). + DisableExpansion bool + + // Stores the key/value pairs + m map[string]string + + // Stores the comments per key. + c map[string][]string + + // Stores the keys in order of appearance. + k []string +} + +// NewProperties creates a new Properties struct with the default +// configuration for "${key}" expressions. +func NewProperties() *Properties { + return &Properties{ + Prefix: "${", + Postfix: "}", + m: map[string]string{}, + c: map[string][]string{}, + k: []string{}, + } +} + +// Load reads a buffer into the given Properties struct. +func (p *Properties) Load(buf []byte, enc Encoding) error { + l := &Loader{Encoding: enc, DisableExpansion: p.DisableExpansion} + newProperties, err := l.LoadBytes(buf) + if err != nil { + return err + } + p.Merge(newProperties) + return nil +} + +// Get returns the expanded value for the given key if exists. +// Otherwise, ok is false. +func (p *Properties) Get(key string) (value string, ok bool) { + v, ok := p.m[key] + if p.DisableExpansion { + return v, ok + } + if !ok { + return "", false + } + + expanded, err := p.expand(key, v) + + // we guarantee that the expanded value is free of + // circular references and malformed expressions + // so we panic if we still get an error here. + if err != nil { + ErrorHandler(fmt.Errorf("%s in %q", err, key+" = "+v)) + } + + return expanded, true +} + +// MustGet returns the expanded value for the given key if exists. +// Otherwise, it panics. +func (p *Properties) MustGet(key string) string { + if v, ok := p.Get(key); ok { + return v + } + ErrorHandler(invalidKeyError(key)) + panic("ErrorHandler should exit") +} + +// ---------------------------------------------------------------------------- + +// ClearComments removes the comments for all keys. +func (p *Properties) ClearComments() { + p.c = map[string][]string{} +} + +// ---------------------------------------------------------------------------- + +// GetComment returns the last comment before the given key or an empty string. +func (p *Properties) GetComment(key string) string { + comments, ok := p.c[key] + if !ok || len(comments) == 0 { + return "" + } + return comments[len(comments)-1] +} + +// ---------------------------------------------------------------------------- + +// GetComments returns all comments that appeared before the given key or nil. +func (p *Properties) GetComments(key string) []string { + if comments, ok := p.c[key]; ok { + return comments + } + return nil +} + +// ---------------------------------------------------------------------------- + +// SetComment sets the comment for the key. +func (p *Properties) SetComment(key, comment string) { + p.c[key] = []string{comment} +} + +// ---------------------------------------------------------------------------- + +// SetComments sets the comments for the key. If the comments are nil then +// all comments for this key are deleted. +func (p *Properties) SetComments(key string, comments []string) { + if comments == nil { + delete(p.c, key) + return + } + p.c[key] = comments +} + +// ---------------------------------------------------------------------------- + +// GetBool checks if the expanded value is one of '1', 'yes', +// 'true' or 'on' if the key exists. The comparison is case-insensitive. +// If the key does not exist the default value is returned. +func (p *Properties) GetBool(key string, def bool) bool { + v, err := p.getBool(key) + if err != nil { + return def + } + return v +} + +// MustGetBool checks if the expanded value is one of '1', 'yes', +// 'true' or 'on' if the key exists. The comparison is case-insensitive. +// If the key does not exist the function panics. +func (p *Properties) MustGetBool(key string) bool { + v, err := p.getBool(key) + if err != nil { + ErrorHandler(err) + } + return v +} + +func (p *Properties) getBool(key string) (value bool, err error) { + if v, ok := p.Get(key); ok { + return boolVal(v), nil + } + return false, invalidKeyError(key) +} + +func boolVal(v string) bool { + v = strings.ToLower(v) + return v == "1" || v == "true" || v == "yes" || v == "on" +} + +// ---------------------------------------------------------------------------- + +// GetDuration parses the expanded value as an time.Duration (in ns) if the +// key exists. If key does not exist or the value cannot be parsed the default +// value is returned. In almost all cases you want to use GetParsedDuration(). +func (p *Properties) GetDuration(key string, def time.Duration) time.Duration { + v, err := p.getInt64(key) + if err != nil { + return def + } + return time.Duration(v) +} + +// MustGetDuration parses the expanded value as an time.Duration (in ns) if +// the key exists. If key does not exist or the value cannot be parsed the +// function panics. In almost all cases you want to use MustGetParsedDuration(). +func (p *Properties) MustGetDuration(key string) time.Duration { + v, err := p.getInt64(key) + if err != nil { + ErrorHandler(err) + } + return time.Duration(v) +} + +// ---------------------------------------------------------------------------- + +// GetParsedDuration parses the expanded value with time.ParseDuration() if the key exists. +// If key does not exist or the value cannot be parsed the default +// value is returned. +func (p *Properties) GetParsedDuration(key string, def time.Duration) time.Duration { + s, ok := p.Get(key) + if !ok { + return def + } + v, err := time.ParseDuration(s) + if err != nil { + return def + } + return v +} + +// MustGetParsedDuration parses the expanded value with time.ParseDuration() if the key exists. +// If key does not exist or the value cannot be parsed the function panics. +func (p *Properties) MustGetParsedDuration(key string) time.Duration { + s, ok := p.Get(key) + if !ok { + ErrorHandler(invalidKeyError(key)) + } + v, err := time.ParseDuration(s) + if err != nil { + ErrorHandler(err) + } + return v +} + +// ---------------------------------------------------------------------------- + +// GetFloat64 parses the expanded value as a float64 if the key exists. +// If key does not exist or the value cannot be parsed the default +// value is returned. +func (p *Properties) GetFloat64(key string, def float64) float64 { + v, err := p.getFloat64(key) + if err != nil { + return def + } + return v +} + +// MustGetFloat64 parses the expanded value as a float64 if the key exists. +// If key does not exist or the value cannot be parsed the function panics. +func (p *Properties) MustGetFloat64(key string) float64 { + v, err := p.getFloat64(key) + if err != nil { + ErrorHandler(err) + } + return v +} + +func (p *Properties) getFloat64(key string) (value float64, err error) { + if v, ok := p.Get(key); ok { + value, err = strconv.ParseFloat(v, 64) + if err != nil { + return 0, err + } + return value, nil + } + return 0, invalidKeyError(key) +} + +// ---------------------------------------------------------------------------- + +// GetInt parses the expanded value as an int if the key exists. +// If key does not exist or the value cannot be parsed the default +// value is returned. If the value does not fit into an int the +// function panics with an out of range error. +func (p *Properties) GetInt(key string, def int) int { + v, err := p.getInt64(key) + if err != nil { + return def + } + return intRangeCheck(key, v) +} + +// MustGetInt parses the expanded value as an int if the key exists. +// If key does not exist or the value cannot be parsed the function panics. +// If the value does not fit into an int the function panics with +// an out of range error. +func (p *Properties) MustGetInt(key string) int { + v, err := p.getInt64(key) + if err != nil { + ErrorHandler(err) + } + return intRangeCheck(key, v) +} + +// ---------------------------------------------------------------------------- + +// GetInt64 parses the expanded value as an int64 if the key exists. +// If key does not exist or the value cannot be parsed the default +// value is returned. +func (p *Properties) GetInt64(key string, def int64) int64 { + v, err := p.getInt64(key) + if err != nil { + return def + } + return v +} + +// MustGetInt64 parses the expanded value as an int if the key exists. +// If key does not exist or the value cannot be parsed the function panics. +func (p *Properties) MustGetInt64(key string) int64 { + v, err := p.getInt64(key) + if err != nil { + ErrorHandler(err) + } + return v +} + +func (p *Properties) getInt64(key string) (value int64, err error) { + if v, ok := p.Get(key); ok { + value, err = strconv.ParseInt(v, 10, 64) + if err != nil { + return 0, err + } + return value, nil + } + return 0, invalidKeyError(key) +} + +// ---------------------------------------------------------------------------- + +// GetUint parses the expanded value as an uint if the key exists. +// If key does not exist or the value cannot be parsed the default +// value is returned. If the value does not fit into an int the +// function panics with an out of range error. +func (p *Properties) GetUint(key string, def uint) uint { + v, err := p.getUint64(key) + if err != nil { + return def + } + return uintRangeCheck(key, v) +} + +// MustGetUint parses the expanded value as an int if the key exists. +// If key does not exist or the value cannot be parsed the function panics. +// If the value does not fit into an int the function panics with +// an out of range error. +func (p *Properties) MustGetUint(key string) uint { + v, err := p.getUint64(key) + if err != nil { + ErrorHandler(err) + } + return uintRangeCheck(key, v) +} + +// ---------------------------------------------------------------------------- + +// GetUint64 parses the expanded value as an uint64 if the key exists. +// If key does not exist or the value cannot be parsed the default +// value is returned. +func (p *Properties) GetUint64(key string, def uint64) uint64 { + v, err := p.getUint64(key) + if err != nil { + return def + } + return v +} + +// MustGetUint64 parses the expanded value as an int if the key exists. +// If key does not exist or the value cannot be parsed the function panics. +func (p *Properties) MustGetUint64(key string) uint64 { + v, err := p.getUint64(key) + if err != nil { + ErrorHandler(err) + } + return v +} + +func (p *Properties) getUint64(key string) (value uint64, err error) { + if v, ok := p.Get(key); ok { + value, err = strconv.ParseUint(v, 10, 64) + if err != nil { + return 0, err + } + return value, nil + } + return 0, invalidKeyError(key) +} + +// ---------------------------------------------------------------------------- + +// GetString returns the expanded value for the given key if exists or +// the default value otherwise. +func (p *Properties) GetString(key, def string) string { + if v, ok := p.Get(key); ok { + return v + } + return def +} + +// MustGetString returns the expanded value for the given key if exists or +// panics otherwise. +func (p *Properties) MustGetString(key string) string { + if v, ok := p.Get(key); ok { + return v + } + ErrorHandler(invalidKeyError(key)) + panic("ErrorHandler should exit") +} + +// ---------------------------------------------------------------------------- + +// Filter returns a new properties object which contains all properties +// for which the key matches the pattern. +func (p *Properties) Filter(pattern string) (*Properties, error) { + re, err := regexp.Compile(pattern) + if err != nil { + return nil, err + } + + return p.FilterRegexp(re), nil +} + +// FilterRegexp returns a new properties object which contains all properties +// for which the key matches the regular expression. +func (p *Properties) FilterRegexp(re *regexp.Regexp) *Properties { + pp := NewProperties() + for _, k := range p.k { + if re.MatchString(k) { + // TODO(fs): we are ignoring the error which flags a circular reference. + // TODO(fs): since we are just copying a subset of keys this cannot happen (fingers crossed) + pp.Set(k, p.m[k]) + } + } + return pp +} + +// FilterPrefix returns a new properties object with a subset of all keys +// with the given prefix. +func (p *Properties) FilterPrefix(prefix string) *Properties { + pp := NewProperties() + for _, k := range p.k { + if strings.HasPrefix(k, prefix) { + // TODO(fs): we are ignoring the error which flags a circular reference. + // TODO(fs): since we are just copying a subset of keys this cannot happen (fingers crossed) + pp.Set(k, p.m[k]) + } + } + return pp +} + +// FilterStripPrefix returns a new properties object with a subset of all keys +// with the given prefix and the prefix removed from the keys. +func (p *Properties) FilterStripPrefix(prefix string) *Properties { + pp := NewProperties() + n := len(prefix) + for _, k := range p.k { + if len(k) > len(prefix) && strings.HasPrefix(k, prefix) { + // TODO(fs): we are ignoring the error which flags a circular reference. + // TODO(fs): since we are modifying keys I am not entirely sure whether we can create a circular reference + // TODO(fs): this function should probably return an error but the signature is fixed + pp.Set(k[n:], p.m[k]) + } + } + return pp +} + +// Len returns the number of keys. +func (p *Properties) Len() int { + return len(p.m) +} + +// Keys returns all keys in the same order as in the input. +func (p *Properties) Keys() []string { + keys := make([]string, len(p.k)) + copy(keys, p.k) + return keys +} + +// Set sets the property key to the corresponding value. +// If a value for key existed before then ok is true and prev +// contains the previous value. If the value contains a +// circular reference or a malformed expression then +// an error is returned. +// An empty key is silently ignored. +func (p *Properties) Set(key, value string) (prev string, ok bool, err error) { + if key == "" { + return "", false, nil + } + + // if expansion is disabled we allow circular references + if p.DisableExpansion { + prev, ok = p.Get(key) + p.m[key] = value + if !ok { + p.k = append(p.k, key) + } + return prev, ok, nil + } + + // to check for a circular reference we temporarily need + // to set the new value. If there is an error then revert + // to the previous state. Only if all tests are successful + // then we add the key to the p.k list. + prev, ok = p.Get(key) + p.m[key] = value + + // now check for a circular reference + _, err = p.expand(key, value) + if err != nil { + + // revert to the previous state + if ok { + p.m[key] = prev + } else { + delete(p.m, key) + } + + return "", false, err + } + + if !ok { + p.k = append(p.k, key) + } + + return prev, ok, nil +} + +// SetValue sets property key to the default string value +// as defined by fmt.Sprintf("%v"). +func (p *Properties) SetValue(key string, value interface{}) error { + _, _, err := p.Set(key, fmt.Sprintf("%v", value)) + return err +} + +// MustSet sets the property key to the corresponding value. +// If a value for key existed before then ok is true and prev +// contains the previous value. An empty key is silently ignored. +func (p *Properties) MustSet(key, value string) (prev string, ok bool) { + prev, ok, err := p.Set(key, value) + if err != nil { + ErrorHandler(err) + } + return prev, ok +} + +// String returns a string of all expanded 'key = value' pairs. +func (p *Properties) String() string { + var s string + for _, key := range p.k { + value, _ := p.Get(key) + s = fmt.Sprintf("%s%s = %s\n", s, key, value) + } + return s +} + +// Write writes all unexpanded 'key = value' pairs to the given writer. +// Write returns the number of bytes written and any write error encountered. +func (p *Properties) Write(w io.Writer, enc Encoding) (n int, err error) { + return p.WriteComment(w, "", enc) +} + +// WriteComment writes all unexpanced 'key = value' pairs to the given writer. +// If prefix is not empty then comments are written with a blank line and the +// given prefix. The prefix should be either "# " or "! " to be compatible with +// the properties file format. Otherwise, the properties parser will not be +// able to read the file back in. It returns the number of bytes written and +// any write error encountered. +func (p *Properties) WriteComment(w io.Writer, prefix string, enc Encoding) (n int, err error) { + var x int + + for _, key := range p.k { + value := p.m[key] + + if prefix != "" { + if comments, ok := p.c[key]; ok { + // don't print comments if they are all empty + allEmpty := true + for _, c := range comments { + if c != "" { + allEmpty = false + break + } + } + + if !allEmpty { + // add a blank line between entries but not at the top + if len(comments) > 0 && n > 0 { + x, err = fmt.Fprintln(w) + if err != nil { + return + } + n += x + } + + for _, c := range comments { + x, err = fmt.Fprintf(w, "%s%s\n", prefix, encode(c, "", enc)) + if err != nil { + return + } + n += x + } + } + } + } + + x, err = fmt.Fprintf(w, "%s = %s\n", encode(key, " :", enc), encode(value, "", enc)) + if err != nil { + return + } + n += x + } + return +} + +// Map returns a copy of the properties as a map. +func (p *Properties) Map() map[string]string { + m := make(map[string]string) + for k, v := range p.m { + m[k] = v + } + return m +} + +// FilterFunc returns a copy of the properties which includes the values which passed all filters. +func (p *Properties) FilterFunc(filters ...func(k, v string) bool) *Properties { + pp := NewProperties() +outer: + for k, v := range p.m { + for _, f := range filters { + if !f(k, v) { + continue outer + } + pp.Set(k, v) + } + } + return pp +} + +// ---------------------------------------------------------------------------- + +// Delete removes the key and its comments. +func (p *Properties) Delete(key string) { + delete(p.m, key) + delete(p.c, key) + newKeys := []string{} + for _, k := range p.k { + if k != key { + newKeys = append(newKeys, k) + } + } + p.k = newKeys +} + +// Merge merges properties, comments and keys from other *Properties into p +func (p *Properties) Merge(other *Properties) { + for k, v := range other.m { + p.m[k] = v + } + for k, v := range other.c { + p.c[k] = v + } + +outer: + for _, otherKey := range other.k { + for _, key := range p.k { + if otherKey == key { + continue outer + } + } + p.k = append(p.k, otherKey) + } +} + +// ---------------------------------------------------------------------------- + +// check expands all values and returns an error if a circular reference or +// a malformed expression was found. +func (p *Properties) check() error { + for key, value := range p.m { + if _, err := p.expand(key, value); err != nil { + return err + } + } + return nil +} + +func (p *Properties) expand(key, input string) (string, error) { + // no pre/postfix -> nothing to expand + if p.Prefix == "" && p.Postfix == "" { + return input, nil + } + + return expand(input, []string{key}, p.Prefix, p.Postfix, p.m) +} + +// expand recursively expands expressions of '(prefix)key(postfix)' to their corresponding values. +// The function keeps track of the keys that were already expanded and stops if it +// detects a circular reference or a malformed expression of the form '(prefix)key'. +func expand(s string, keys []string, prefix, postfix string, values map[string]string) (string, error) { + if len(keys) > maxExpansionDepth { + return "", fmt.Errorf("expansion too deep") + } + + for { + start := strings.Index(s, prefix) + if start == -1 { + return s, nil + } + + keyStart := start + len(prefix) + keyLen := strings.Index(s[keyStart:], postfix) + if keyLen == -1 { + return "", fmt.Errorf("malformed expression") + } + + end := keyStart + keyLen + len(postfix) - 1 + key := s[keyStart : keyStart+keyLen] + + // fmt.Printf("s:%q pp:%q start:%d end:%d keyStart:%d keyLen:%d key:%q\n", s, prefix + "..." + postfix, start, end, keyStart, keyLen, key) + + for _, k := range keys { + if key == k { + return "", fmt.Errorf("circular reference") + } + } + + val, ok := values[key] + if !ok { + val = os.Getenv(key) + } + new_val, err := expand(val, append(keys, key), prefix, postfix, values) + if err != nil { + return "", err + } + s = s[:start] + new_val + s[end+1:] + } + return s, nil +} + +// encode encodes a UTF-8 string to ISO-8859-1 and escapes some characters. +func encode(s string, special string, enc Encoding) string { + switch enc { + case UTF8: + return encodeUtf8(s, special) + case ISO_8859_1: + return encodeIso(s, special) + default: + panic(fmt.Sprintf("unsupported encoding %v", enc)) + } +} + +func encodeUtf8(s string, special string) string { + v := "" + for pos := 0; pos < len(s); { + r, w := utf8.DecodeRuneInString(s[pos:]) + pos += w + v += escape(r, special) + } + return v +} + +func encodeIso(s string, special string) string { + var r rune + var w int + var v string + for pos := 0; pos < len(s); { + switch r, w = utf8.DecodeRuneInString(s[pos:]); { + case r < 1<<8: // single byte rune -> escape special chars only + v += escape(r, special) + case r < 1<<16: // two byte rune -> unicode literal + v += fmt.Sprintf("\\u%04x", r) + default: // more than two bytes per rune -> can't encode + v += "?" + } + pos += w + } + return v +} + +func escape(r rune, special string) string { + switch r { + case '\f': + return "\\f" + case '\n': + return "\\n" + case '\r': + return "\\r" + case '\t': + return "\\t" + default: + if strings.ContainsRune(special, r) { + return "\\" + string(r) + } + return string(r) + } +} + +func invalidKeyError(key string) error { + return fmt.Errorf("unknown property: %s", key) +} diff --git a/vendor/github.com/magiconair/properties/rangecheck.go b/vendor/github.com/magiconair/properties/rangecheck.go new file mode 100644 index 000000000..b013a2e5e --- /dev/null +++ b/vendor/github.com/magiconair/properties/rangecheck.go @@ -0,0 +1,31 @@ +// Copyright 2018 Frank Schroeder. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package properties + +import ( + "fmt" + "math" +) + +// make this a var to overwrite it in a test +var is32Bit = ^uint(0) == math.MaxUint32 + +// intRangeCheck checks if the value fits into the int type and +// panics if it does not. +func intRangeCheck(key string, v int64) int { + if is32Bit && (v < math.MinInt32 || v > math.MaxInt32) { + panic(fmt.Sprintf("Value %d for key %s out of range", v, key)) + } + return int(v) +} + +// uintRangeCheck checks if the value fits into the uint type and +// panics if it does not. +func uintRangeCheck(key string, v uint64) uint { + if is32Bit && v > math.MaxUint32 { + panic(fmt.Sprintf("Value %d for key %s out of range", v, key)) + } + return uint(v) +} diff --git a/vendor/github.com/mitchellh/mapstructure/.travis.yml b/vendor/github.com/mitchellh/mapstructure/.travis.yml new file mode 100644 index 000000000..1689c7d73 --- /dev/null +++ b/vendor/github.com/mitchellh/mapstructure/.travis.yml @@ -0,0 +1,8 @@ +language: go + +go: + - "1.11.x" + - tip + +script: + - go test diff --git a/vendor/github.com/mitchellh/mapstructure/CHANGELOG.md b/vendor/github.com/mitchellh/mapstructure/CHANGELOG.md new file mode 100644 index 000000000..3b3cb723f --- /dev/null +++ b/vendor/github.com/mitchellh/mapstructure/CHANGELOG.md @@ -0,0 +1,21 @@ +## 1.1.2 + +* Fix error when decode hook decodes interface implementation into interface + type. [GH-140] + +## 1.1.1 + +* Fix panic that can happen in `decodePtr` + +## 1.1.0 + +* Added `StringToIPHookFunc` to convert `string` to `net.IP` and `net.IPNet` [GH-133] +* Support struct to struct decoding [GH-137] +* If source map value is nil, then destination map value is nil (instead of empty) +* If source slice value is nil, then destination slice value is nil (instead of empty) +* If source pointer is nil, then destination pointer is set to nil (instead of + allocated zero value of type) + +## 1.0.0 + +* Initial tagged stable release. diff --git a/vendor/github.com/mitchellh/mapstructure/LICENSE b/vendor/github.com/mitchellh/mapstructure/LICENSE new file mode 100644 index 000000000..f9c841a51 --- /dev/null +++ b/vendor/github.com/mitchellh/mapstructure/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2013 Mitchell Hashimoto + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/mitchellh/mapstructure/README.md b/vendor/github.com/mitchellh/mapstructure/README.md new file mode 100644 index 000000000..0018dc7d9 --- /dev/null +++ b/vendor/github.com/mitchellh/mapstructure/README.md @@ -0,0 +1,46 @@ +# mapstructure [![Godoc](https://godoc.org/github.com/mitchellh/mapstructure?status.svg)](https://godoc.org/github.com/mitchellh/mapstructure) + +mapstructure is a Go library for decoding generic map values to structures +and vice versa, while providing helpful error handling. + +This library is most useful when decoding values from some data stream (JSON, +Gob, etc.) where you don't _quite_ know the structure of the underlying data +until you read a part of it. You can therefore read a `map[string]interface{}` +and use this library to decode it into the proper underlying native Go +structure. + +## Installation + +Standard `go get`: + +``` +$ go get github.com/mitchellh/mapstructure +``` + +## Usage & Example + +For usage and examples see the [Godoc](http://godoc.org/github.com/mitchellh/mapstructure). + +The `Decode` function has examples associated with it there. + +## But Why?! + +Go offers fantastic standard libraries for decoding formats such as JSON. +The standard method is to have a struct pre-created, and populate that struct +from the bytes of the encoded format. This is great, but the problem is if +you have configuration or an encoding that changes slightly depending on +specific fields. For example, consider this JSON: + +```json +{ + "type": "person", + "name": "Mitchell" +} +``` + +Perhaps we can't populate a specific structure without first reading +the "type" field from the JSON. We could always do two passes over the +decoding of the JSON (reading the "type" first, and the rest later). +However, it is much simpler to just decode this into a `map[string]interface{}` +structure, read the "type" key, then use something like this library +to decode it into the proper structure. diff --git a/vendor/github.com/mitchellh/mapstructure/decode_hooks.go b/vendor/github.com/mitchellh/mapstructure/decode_hooks.go new file mode 100644 index 000000000..1f0abc65a --- /dev/null +++ b/vendor/github.com/mitchellh/mapstructure/decode_hooks.go @@ -0,0 +1,217 @@ +package mapstructure + +import ( + "errors" + "fmt" + "net" + "reflect" + "strconv" + "strings" + "time" +) + +// typedDecodeHook takes a raw DecodeHookFunc (an interface{}) and turns +// it into the proper DecodeHookFunc type, such as DecodeHookFuncType. +func typedDecodeHook(h DecodeHookFunc) DecodeHookFunc { + // Create variables here so we can reference them with the reflect pkg + var f1 DecodeHookFuncType + var f2 DecodeHookFuncKind + + // Fill in the variables into this interface and the rest is done + // automatically using the reflect package. + potential := []interface{}{f1, f2} + + v := reflect.ValueOf(h) + vt := v.Type() + for _, raw := range potential { + pt := reflect.ValueOf(raw).Type() + if vt.ConvertibleTo(pt) { + return v.Convert(pt).Interface() + } + } + + return nil +} + +// DecodeHookExec executes the given decode hook. This should be used +// since it'll naturally degrade to the older backwards compatible DecodeHookFunc +// that took reflect.Kind instead of reflect.Type. +func DecodeHookExec( + raw DecodeHookFunc, + from reflect.Type, to reflect.Type, + data interface{}) (interface{}, error) { + switch f := typedDecodeHook(raw).(type) { + case DecodeHookFuncType: + return f(from, to, data) + case DecodeHookFuncKind: + return f(from.Kind(), to.Kind(), data) + default: + return nil, errors.New("invalid decode hook signature") + } +} + +// ComposeDecodeHookFunc creates a single DecodeHookFunc that +// automatically composes multiple DecodeHookFuncs. +// +// The composed funcs are called in order, with the result of the +// previous transformation. +func ComposeDecodeHookFunc(fs ...DecodeHookFunc) DecodeHookFunc { + return func( + f reflect.Type, + t reflect.Type, + data interface{}) (interface{}, error) { + var err error + for _, f1 := range fs { + data, err = DecodeHookExec(f1, f, t, data) + if err != nil { + return nil, err + } + + // Modify the from kind to be correct with the new data + f = nil + if val := reflect.ValueOf(data); val.IsValid() { + f = val.Type() + } + } + + return data, nil + } +} + +// StringToSliceHookFunc returns a DecodeHookFunc that converts +// string to []string by splitting on the given sep. +func StringToSliceHookFunc(sep string) DecodeHookFunc { + return func( + f reflect.Kind, + t reflect.Kind, + data interface{}) (interface{}, error) { + if f != reflect.String || t != reflect.Slice { + return data, nil + } + + raw := data.(string) + if raw == "" { + return []string{}, nil + } + + return strings.Split(raw, sep), nil + } +} + +// StringToTimeDurationHookFunc returns a DecodeHookFunc that converts +// strings to time.Duration. +func StringToTimeDurationHookFunc() DecodeHookFunc { + return func( + f reflect.Type, + t reflect.Type, + data interface{}) (interface{}, error) { + if f.Kind() != reflect.String { + return data, nil + } + if t != reflect.TypeOf(time.Duration(5)) { + return data, nil + } + + // Convert it by parsing + return time.ParseDuration(data.(string)) + } +} + +// StringToIPHookFunc returns a DecodeHookFunc that converts +// strings to net.IP +func StringToIPHookFunc() DecodeHookFunc { + return func( + f reflect.Type, + t reflect.Type, + data interface{}) (interface{}, error) { + if f.Kind() != reflect.String { + return data, nil + } + if t != reflect.TypeOf(net.IP{}) { + return data, nil + } + + // Convert it by parsing + ip := net.ParseIP(data.(string)) + if ip == nil { + return net.IP{}, fmt.Errorf("failed parsing ip %v", data) + } + + return ip, nil + } +} + +// StringToIPNetHookFunc returns a DecodeHookFunc that converts +// strings to net.IPNet +func StringToIPNetHookFunc() DecodeHookFunc { + return func( + f reflect.Type, + t reflect.Type, + data interface{}) (interface{}, error) { + if f.Kind() != reflect.String { + return data, nil + } + if t != reflect.TypeOf(net.IPNet{}) { + return data, nil + } + + // Convert it by parsing + _, net, err := net.ParseCIDR(data.(string)) + return net, err + } +} + +// StringToTimeHookFunc returns a DecodeHookFunc that converts +// strings to time.Time. +func StringToTimeHookFunc(layout string) DecodeHookFunc { + return func( + f reflect.Type, + t reflect.Type, + data interface{}) (interface{}, error) { + if f.Kind() != reflect.String { + return data, nil + } + if t != reflect.TypeOf(time.Time{}) { + return data, nil + } + + // Convert it by parsing + return time.Parse(layout, data.(string)) + } +} + +// WeaklyTypedHook is a DecodeHookFunc which adds support for weak typing to +// the decoder. +// +// Note that this is significantly different from the WeaklyTypedInput option +// of the DecoderConfig. +func WeaklyTypedHook( + f reflect.Kind, + t reflect.Kind, + data interface{}) (interface{}, error) { + dataVal := reflect.ValueOf(data) + switch t { + case reflect.String: + switch f { + case reflect.Bool: + if dataVal.Bool() { + return "1", nil + } + return "0", nil + case reflect.Float32: + return strconv.FormatFloat(dataVal.Float(), 'f', -1, 64), nil + case reflect.Int: + return strconv.FormatInt(dataVal.Int(), 10), nil + case reflect.Slice: + dataType := dataVal.Type() + elemKind := dataType.Elem().Kind() + if elemKind == reflect.Uint8 { + return string(dataVal.Interface().([]uint8)), nil + } + case reflect.Uint: + return strconv.FormatUint(dataVal.Uint(), 10), nil + } + } + + return data, nil +} diff --git a/vendor/github.com/mitchellh/mapstructure/error.go b/vendor/github.com/mitchellh/mapstructure/error.go new file mode 100644 index 000000000..47a99e5af --- /dev/null +++ b/vendor/github.com/mitchellh/mapstructure/error.go @@ -0,0 +1,50 @@ +package mapstructure + +import ( + "errors" + "fmt" + "sort" + "strings" +) + +// Error implements the error interface and can represents multiple +// errors that occur in the course of a single decode. +type Error struct { + Errors []string +} + +func (e *Error) Error() string { + points := make([]string, len(e.Errors)) + for i, err := range e.Errors { + points[i] = fmt.Sprintf("* %s", err) + } + + sort.Strings(points) + return fmt.Sprintf( + "%d error(s) decoding:\n\n%s", + len(e.Errors), strings.Join(points, "\n")) +} + +// WrappedErrors implements the errwrap.Wrapper interface to make this +// return value more useful with the errwrap and go-multierror libraries. +func (e *Error) WrappedErrors() []error { + if e == nil { + return nil + } + + result := make([]error, len(e.Errors)) + for i, e := range e.Errors { + result[i] = errors.New(e) + } + + return result +} + +func appendErrors(errors []string, err error) []string { + switch e := err.(type) { + case *Error: + return append(errors, e.Errors...) + default: + return append(errors, e.Error()) + } +} diff --git a/vendor/github.com/mitchellh/mapstructure/go.mod b/vendor/github.com/mitchellh/mapstructure/go.mod new file mode 100644 index 000000000..d2a712562 --- /dev/null +++ b/vendor/github.com/mitchellh/mapstructure/go.mod @@ -0,0 +1 @@ +module github.com/mitchellh/mapstructure diff --git a/vendor/github.com/mitchellh/mapstructure/mapstructure.go b/vendor/github.com/mitchellh/mapstructure/mapstructure.go new file mode 100644 index 000000000..256ee63fb --- /dev/null +++ b/vendor/github.com/mitchellh/mapstructure/mapstructure.go @@ -0,0 +1,1149 @@ +// Package mapstructure exposes functionality to convert an arbitrary +// map[string]interface{} into a native Go structure. +// +// The Go structure can be arbitrarily complex, containing slices, +// other structs, etc. and the decoder will properly decode nested +// maps and so on into the proper structures in the native Go struct. +// See the examples to see what the decoder is capable of. +package mapstructure + +import ( + "encoding/json" + "errors" + "fmt" + "reflect" + "sort" + "strconv" + "strings" +) + +// DecodeHookFunc is the callback function that can be used for +// data transformations. See "DecodeHook" in the DecoderConfig +// struct. +// +// The type should be DecodeHookFuncType or DecodeHookFuncKind. +// Either is accepted. Types are a superset of Kinds (Types can return +// Kinds) and are generally a richer thing to use, but Kinds are simpler +// if you only need those. +// +// The reason DecodeHookFunc is multi-typed is for backwards compatibility: +// we started with Kinds and then realized Types were the better solution, +// but have a promise to not break backwards compat so we now support +// both. +type DecodeHookFunc interface{} + +// DecodeHookFuncType is a DecodeHookFunc which has complete information about +// the source and target types. +type DecodeHookFuncType func(reflect.Type, reflect.Type, interface{}) (interface{}, error) + +// DecodeHookFuncKind is a DecodeHookFunc which knows only the Kinds of the +// source and target types. +type DecodeHookFuncKind func(reflect.Kind, reflect.Kind, interface{}) (interface{}, error) + +// DecoderConfig is the configuration that is used to create a new decoder +// and allows customization of various aspects of decoding. +type DecoderConfig struct { + // DecodeHook, if set, will be called before any decoding and any + // type conversion (if WeaklyTypedInput is on). This lets you modify + // the values before they're set down onto the resulting struct. + // + // If an error is returned, the entire decode will fail with that + // error. + DecodeHook DecodeHookFunc + + // If ErrorUnused is true, then it is an error for there to exist + // keys in the original map that were unused in the decoding process + // (extra keys). + ErrorUnused bool + + // ZeroFields, if set to true, will zero fields before writing them. + // For example, a map will be emptied before decoded values are put in + // it. If this is false, a map will be merged. + ZeroFields bool + + // If WeaklyTypedInput is true, the decoder will make the following + // "weak" conversions: + // + // - bools to string (true = "1", false = "0") + // - numbers to string (base 10) + // - bools to int/uint (true = 1, false = 0) + // - strings to int/uint (base implied by prefix) + // - int to bool (true if value != 0) + // - string to bool (accepts: 1, t, T, TRUE, true, True, 0, f, F, + // FALSE, false, False. Anything else is an error) + // - empty array = empty map and vice versa + // - negative numbers to overflowed uint values (base 10) + // - slice of maps to a merged map + // - single values are converted to slices if required. Each + // element is weakly decoded. For example: "4" can become []int{4} + // if the target type is an int slice. + // + WeaklyTypedInput bool + + // Metadata is the struct that will contain extra metadata about + // the decoding. If this is nil, then no metadata will be tracked. + Metadata *Metadata + + // Result is a pointer to the struct that will contain the decoded + // value. + Result interface{} + + // The tag name that mapstructure reads for field names. This + // defaults to "mapstructure" + TagName string +} + +// A Decoder takes a raw interface value and turns it into structured +// data, keeping track of rich error information along the way in case +// anything goes wrong. Unlike the basic top-level Decode method, you can +// more finely control how the Decoder behaves using the DecoderConfig +// structure. The top-level Decode method is just a convenience that sets +// up the most basic Decoder. +type Decoder struct { + config *DecoderConfig +} + +// Metadata contains information about decoding a structure that +// is tedious or difficult to get otherwise. +type Metadata struct { + // Keys are the keys of the structure which were successfully decoded + Keys []string + + // Unused is a slice of keys that were found in the raw value but + // weren't decoded since there was no matching field in the result interface + Unused []string +} + +// Decode takes an input structure and uses reflection to translate it to +// the output structure. output must be a pointer to a map or struct. +func Decode(input interface{}, output interface{}) error { + config := &DecoderConfig{ + Metadata: nil, + Result: output, + } + + decoder, err := NewDecoder(config) + if err != nil { + return err + } + + return decoder.Decode(input) +} + +// WeakDecode is the same as Decode but is shorthand to enable +// WeaklyTypedInput. See DecoderConfig for more info. +func WeakDecode(input, output interface{}) error { + config := &DecoderConfig{ + Metadata: nil, + Result: output, + WeaklyTypedInput: true, + } + + decoder, err := NewDecoder(config) + if err != nil { + return err + } + + return decoder.Decode(input) +} + +// DecodeMetadata is the same as Decode, but is shorthand to +// enable metadata collection. See DecoderConfig for more info. +func DecodeMetadata(input interface{}, output interface{}, metadata *Metadata) error { + config := &DecoderConfig{ + Metadata: metadata, + Result: output, + } + + decoder, err := NewDecoder(config) + if err != nil { + return err + } + + return decoder.Decode(input) +} + +// WeakDecodeMetadata is the same as Decode, but is shorthand to +// enable both WeaklyTypedInput and metadata collection. See +// DecoderConfig for more info. +func WeakDecodeMetadata(input interface{}, output interface{}, metadata *Metadata) error { + config := &DecoderConfig{ + Metadata: metadata, + Result: output, + WeaklyTypedInput: true, + } + + decoder, err := NewDecoder(config) + if err != nil { + return err + } + + return decoder.Decode(input) +} + +// NewDecoder returns a new decoder for the given configuration. Once +// a decoder has been returned, the same configuration must not be used +// again. +func NewDecoder(config *DecoderConfig) (*Decoder, error) { + val := reflect.ValueOf(config.Result) + if val.Kind() != reflect.Ptr { + return nil, errors.New("result must be a pointer") + } + + val = val.Elem() + if !val.CanAddr() { + return nil, errors.New("result must be addressable (a pointer)") + } + + if config.Metadata != nil { + if config.Metadata.Keys == nil { + config.Metadata.Keys = make([]string, 0) + } + + if config.Metadata.Unused == nil { + config.Metadata.Unused = make([]string, 0) + } + } + + if config.TagName == "" { + config.TagName = "mapstructure" + } + + result := &Decoder{ + config: config, + } + + return result, nil +} + +// Decode decodes the given raw interface to the target pointer specified +// by the configuration. +func (d *Decoder) Decode(input interface{}) error { + return d.decode("", input, reflect.ValueOf(d.config.Result).Elem()) +} + +// Decodes an unknown data type into a specific reflection value. +func (d *Decoder) decode(name string, input interface{}, outVal reflect.Value) error { + var inputVal reflect.Value + if input != nil { + inputVal = reflect.ValueOf(input) + + // We need to check here if input is a typed nil. Typed nils won't + // match the "input == nil" below so we check that here. + if inputVal.Kind() == reflect.Ptr && inputVal.IsNil() { + input = nil + } + } + + if input == nil { + // If the data is nil, then we don't set anything, unless ZeroFields is set + // to true. + if d.config.ZeroFields { + outVal.Set(reflect.Zero(outVal.Type())) + + if d.config.Metadata != nil && name != "" { + d.config.Metadata.Keys = append(d.config.Metadata.Keys, name) + } + } + return nil + } + + if !inputVal.IsValid() { + // If the input value is invalid, then we just set the value + // to be the zero value. + outVal.Set(reflect.Zero(outVal.Type())) + if d.config.Metadata != nil && name != "" { + d.config.Metadata.Keys = append(d.config.Metadata.Keys, name) + } + return nil + } + + if d.config.DecodeHook != nil { + // We have a DecodeHook, so let's pre-process the input. + var err error + input, err = DecodeHookExec( + d.config.DecodeHook, + inputVal.Type(), outVal.Type(), input) + if err != nil { + return fmt.Errorf("error decoding '%s': %s", name, err) + } + } + + var err error + outputKind := getKind(outVal) + switch outputKind { + case reflect.Bool: + err = d.decodeBool(name, input, outVal) + case reflect.Interface: + err = d.decodeBasic(name, input, outVal) + case reflect.String: + err = d.decodeString(name, input, outVal) + case reflect.Int: + err = d.decodeInt(name, input, outVal) + case reflect.Uint: + err = d.decodeUint(name, input, outVal) + case reflect.Float32: + err = d.decodeFloat(name, input, outVal) + case reflect.Struct: + err = d.decodeStruct(name, input, outVal) + case reflect.Map: + err = d.decodeMap(name, input, outVal) + case reflect.Ptr: + err = d.decodePtr(name, input, outVal) + case reflect.Slice: + err = d.decodeSlice(name, input, outVal) + case reflect.Array: + err = d.decodeArray(name, input, outVal) + case reflect.Func: + err = d.decodeFunc(name, input, outVal) + default: + // If we reached this point then we weren't able to decode it + return fmt.Errorf("%s: unsupported type: %s", name, outputKind) + } + + // If we reached here, then we successfully decoded SOMETHING, so + // mark the key as used if we're tracking metainput. + if d.config.Metadata != nil && name != "" { + d.config.Metadata.Keys = append(d.config.Metadata.Keys, name) + } + + return err +} + +// This decodes a basic type (bool, int, string, etc.) and sets the +// value to "data" of that type. +func (d *Decoder) decodeBasic(name string, data interface{}, val reflect.Value) error { + if val.IsValid() && val.Elem().IsValid() { + return d.decode(name, data, val.Elem()) + } + + dataVal := reflect.ValueOf(data) + + // If the input data is a pointer, and the assigned type is the dereference + // of that exact pointer, then indirect it so that we can assign it. + // Example: *string to string + if dataVal.Kind() == reflect.Ptr && dataVal.Type().Elem() == val.Type() { + dataVal = reflect.Indirect(dataVal) + } + + if !dataVal.IsValid() { + dataVal = reflect.Zero(val.Type()) + } + + dataValType := dataVal.Type() + if !dataValType.AssignableTo(val.Type()) { + return fmt.Errorf( + "'%s' expected type '%s', got '%s'", + name, val.Type(), dataValType) + } + + val.Set(dataVal) + return nil +} + +func (d *Decoder) decodeString(name string, data interface{}, val reflect.Value) error { + dataVal := reflect.Indirect(reflect.ValueOf(data)) + dataKind := getKind(dataVal) + + converted := true + switch { + case dataKind == reflect.String: + val.SetString(dataVal.String()) + case dataKind == reflect.Bool && d.config.WeaklyTypedInput: + if dataVal.Bool() { + val.SetString("1") + } else { + val.SetString("0") + } + case dataKind == reflect.Int && d.config.WeaklyTypedInput: + val.SetString(strconv.FormatInt(dataVal.Int(), 10)) + case dataKind == reflect.Uint && d.config.WeaklyTypedInput: + val.SetString(strconv.FormatUint(dataVal.Uint(), 10)) + case dataKind == reflect.Float32 && d.config.WeaklyTypedInput: + val.SetString(strconv.FormatFloat(dataVal.Float(), 'f', -1, 64)) + case dataKind == reflect.Slice && d.config.WeaklyTypedInput, + dataKind == reflect.Array && d.config.WeaklyTypedInput: + dataType := dataVal.Type() + elemKind := dataType.Elem().Kind() + switch elemKind { + case reflect.Uint8: + var uints []uint8 + if dataKind == reflect.Array { + uints = make([]uint8, dataVal.Len(), dataVal.Len()) + for i := range uints { + uints[i] = dataVal.Index(i).Interface().(uint8) + } + } else { + uints = dataVal.Interface().([]uint8) + } + val.SetString(string(uints)) + default: + converted = false + } + default: + converted = false + } + + if !converted { + return fmt.Errorf( + "'%s' expected type '%s', got unconvertible type '%s'", + name, val.Type(), dataVal.Type()) + } + + return nil +} + +func (d *Decoder) decodeInt(name string, data interface{}, val reflect.Value) error { + dataVal := reflect.Indirect(reflect.ValueOf(data)) + dataKind := getKind(dataVal) + dataType := dataVal.Type() + + switch { + case dataKind == reflect.Int: + val.SetInt(dataVal.Int()) + case dataKind == reflect.Uint: + val.SetInt(int64(dataVal.Uint())) + case dataKind == reflect.Float32: + val.SetInt(int64(dataVal.Float())) + case dataKind == reflect.Bool && d.config.WeaklyTypedInput: + if dataVal.Bool() { + val.SetInt(1) + } else { + val.SetInt(0) + } + case dataKind == reflect.String && d.config.WeaklyTypedInput: + i, err := strconv.ParseInt(dataVal.String(), 0, val.Type().Bits()) + if err == nil { + val.SetInt(i) + } else { + return fmt.Errorf("cannot parse '%s' as int: %s", name, err) + } + case dataType.PkgPath() == "encoding/json" && dataType.Name() == "Number": + jn := data.(json.Number) + i, err := jn.Int64() + if err != nil { + return fmt.Errorf( + "error decoding json.Number into %s: %s", name, err) + } + val.SetInt(i) + default: + return fmt.Errorf( + "'%s' expected type '%s', got unconvertible type '%s'", + name, val.Type(), dataVal.Type()) + } + + return nil +} + +func (d *Decoder) decodeUint(name string, data interface{}, val reflect.Value) error { + dataVal := reflect.Indirect(reflect.ValueOf(data)) + dataKind := getKind(dataVal) + + switch { + case dataKind == reflect.Int: + i := dataVal.Int() + if i < 0 && !d.config.WeaklyTypedInput { + return fmt.Errorf("cannot parse '%s', %d overflows uint", + name, i) + } + val.SetUint(uint64(i)) + case dataKind == reflect.Uint: + val.SetUint(dataVal.Uint()) + case dataKind == reflect.Float32: + f := dataVal.Float() + if f < 0 && !d.config.WeaklyTypedInput { + return fmt.Errorf("cannot parse '%s', %f overflows uint", + name, f) + } + val.SetUint(uint64(f)) + case dataKind == reflect.Bool && d.config.WeaklyTypedInput: + if dataVal.Bool() { + val.SetUint(1) + } else { + val.SetUint(0) + } + case dataKind == reflect.String && d.config.WeaklyTypedInput: + i, err := strconv.ParseUint(dataVal.String(), 0, val.Type().Bits()) + if err == nil { + val.SetUint(i) + } else { + return fmt.Errorf("cannot parse '%s' as uint: %s", name, err) + } + default: + return fmt.Errorf( + "'%s' expected type '%s', got unconvertible type '%s'", + name, val.Type(), dataVal.Type()) + } + + return nil +} + +func (d *Decoder) decodeBool(name string, data interface{}, val reflect.Value) error { + dataVal := reflect.Indirect(reflect.ValueOf(data)) + dataKind := getKind(dataVal) + + switch { + case dataKind == reflect.Bool: + val.SetBool(dataVal.Bool()) + case dataKind == reflect.Int && d.config.WeaklyTypedInput: + val.SetBool(dataVal.Int() != 0) + case dataKind == reflect.Uint && d.config.WeaklyTypedInput: + val.SetBool(dataVal.Uint() != 0) + case dataKind == reflect.Float32 && d.config.WeaklyTypedInput: + val.SetBool(dataVal.Float() != 0) + case dataKind == reflect.String && d.config.WeaklyTypedInput: + b, err := strconv.ParseBool(dataVal.String()) + if err == nil { + val.SetBool(b) + } else if dataVal.String() == "" { + val.SetBool(false) + } else { + return fmt.Errorf("cannot parse '%s' as bool: %s", name, err) + } + default: + return fmt.Errorf( + "'%s' expected type '%s', got unconvertible type '%s'", + name, val.Type(), dataVal.Type()) + } + + return nil +} + +func (d *Decoder) decodeFloat(name string, data interface{}, val reflect.Value) error { + dataVal := reflect.Indirect(reflect.ValueOf(data)) + dataKind := getKind(dataVal) + dataType := dataVal.Type() + + switch { + case dataKind == reflect.Int: + val.SetFloat(float64(dataVal.Int())) + case dataKind == reflect.Uint: + val.SetFloat(float64(dataVal.Uint())) + case dataKind == reflect.Float32: + val.SetFloat(dataVal.Float()) + case dataKind == reflect.Bool && d.config.WeaklyTypedInput: + if dataVal.Bool() { + val.SetFloat(1) + } else { + val.SetFloat(0) + } + case dataKind == reflect.String && d.config.WeaklyTypedInput: + f, err := strconv.ParseFloat(dataVal.String(), val.Type().Bits()) + if err == nil { + val.SetFloat(f) + } else { + return fmt.Errorf("cannot parse '%s' as float: %s", name, err) + } + case dataType.PkgPath() == "encoding/json" && dataType.Name() == "Number": + jn := data.(json.Number) + i, err := jn.Float64() + if err != nil { + return fmt.Errorf( + "error decoding json.Number into %s: %s", name, err) + } + val.SetFloat(i) + default: + return fmt.Errorf( + "'%s' expected type '%s', got unconvertible type '%s'", + name, val.Type(), dataVal.Type()) + } + + return nil +} + +func (d *Decoder) decodeMap(name string, data interface{}, val reflect.Value) error { + valType := val.Type() + valKeyType := valType.Key() + valElemType := valType.Elem() + + // By default we overwrite keys in the current map + valMap := val + + // If the map is nil or we're purposely zeroing fields, make a new map + if valMap.IsNil() || d.config.ZeroFields { + // Make a new map to hold our result + mapType := reflect.MapOf(valKeyType, valElemType) + valMap = reflect.MakeMap(mapType) + } + + // Check input type and based on the input type jump to the proper func + dataVal := reflect.Indirect(reflect.ValueOf(data)) + switch dataVal.Kind() { + case reflect.Map: + return d.decodeMapFromMap(name, dataVal, val, valMap) + + case reflect.Struct: + return d.decodeMapFromStruct(name, dataVal, val, valMap) + + case reflect.Array, reflect.Slice: + if d.config.WeaklyTypedInput { + return d.decodeMapFromSlice(name, dataVal, val, valMap) + } + + fallthrough + + default: + return fmt.Errorf("'%s' expected a map, got '%s'", name, dataVal.Kind()) + } +} + +func (d *Decoder) decodeMapFromSlice(name string, dataVal reflect.Value, val reflect.Value, valMap reflect.Value) error { + // Special case for BC reasons (covered by tests) + if dataVal.Len() == 0 { + val.Set(valMap) + return nil + } + + for i := 0; i < dataVal.Len(); i++ { + err := d.decode( + fmt.Sprintf("%s[%d]", name, i), + dataVal.Index(i).Interface(), val) + if err != nil { + return err + } + } + + return nil +} + +func (d *Decoder) decodeMapFromMap(name string, dataVal reflect.Value, val reflect.Value, valMap reflect.Value) error { + valType := val.Type() + valKeyType := valType.Key() + valElemType := valType.Elem() + + // Accumulate errors + errors := make([]string, 0) + + // If the input data is empty, then we just match what the input data is. + if dataVal.Len() == 0 { + if dataVal.IsNil() { + if !val.IsNil() { + val.Set(dataVal) + } + } else { + // Set to empty allocated value + val.Set(valMap) + } + + return nil + } + + for _, k := range dataVal.MapKeys() { + fieldName := fmt.Sprintf("%s[%s]", name, k) + + // First decode the key into the proper type + currentKey := reflect.Indirect(reflect.New(valKeyType)) + if err := d.decode(fieldName, k.Interface(), currentKey); err != nil { + errors = appendErrors(errors, err) + continue + } + + // Next decode the data into the proper type + v := dataVal.MapIndex(k).Interface() + currentVal := reflect.Indirect(reflect.New(valElemType)) + if err := d.decode(fieldName, v, currentVal); err != nil { + errors = appendErrors(errors, err) + continue + } + + valMap.SetMapIndex(currentKey, currentVal) + } + + // Set the built up map to the value + val.Set(valMap) + + // If we had errors, return those + if len(errors) > 0 { + return &Error{errors} + } + + return nil +} + +func (d *Decoder) decodeMapFromStruct(name string, dataVal reflect.Value, val reflect.Value, valMap reflect.Value) error { + typ := dataVal.Type() + for i := 0; i < typ.NumField(); i++ { + // Get the StructField first since this is a cheap operation. If the + // field is unexported, then ignore it. + f := typ.Field(i) + if f.PkgPath != "" { + continue + } + + // Next get the actual value of this field and verify it is assignable + // to the map value. + v := dataVal.Field(i) + if !v.Type().AssignableTo(valMap.Type().Elem()) { + return fmt.Errorf("cannot assign type '%s' to map value field of type '%s'", v.Type(), valMap.Type().Elem()) + } + + tagValue := f.Tag.Get(d.config.TagName) + tagParts := strings.Split(tagValue, ",") + + // Determine the name of the key in the map + keyName := f.Name + if tagParts[0] != "" { + if tagParts[0] == "-" { + continue + } + keyName = tagParts[0] + } + + // If "squash" is specified in the tag, we squash the field down. + squash := false + for _, tag := range tagParts[1:] { + if tag == "squash" { + squash = true + break + } + } + if squash && v.Kind() != reflect.Struct { + return fmt.Errorf("cannot squash non-struct type '%s'", v.Type()) + } + + switch v.Kind() { + // this is an embedded struct, so handle it differently + case reflect.Struct: + x := reflect.New(v.Type()) + x.Elem().Set(v) + + vType := valMap.Type() + vKeyType := vType.Key() + vElemType := vType.Elem() + mType := reflect.MapOf(vKeyType, vElemType) + vMap := reflect.MakeMap(mType) + + err := d.decode(keyName, x.Interface(), vMap) + if err != nil { + return err + } + + if squash { + for _, k := range vMap.MapKeys() { + valMap.SetMapIndex(k, vMap.MapIndex(k)) + } + } else { + valMap.SetMapIndex(reflect.ValueOf(keyName), vMap) + } + + default: + valMap.SetMapIndex(reflect.ValueOf(keyName), v) + } + } + + if val.CanAddr() { + val.Set(valMap) + } + + return nil +} + +func (d *Decoder) decodePtr(name string, data interface{}, val reflect.Value) error { + // If the input data is nil, then we want to just set the output + // pointer to be nil as well. + isNil := data == nil + if !isNil { + switch v := reflect.Indirect(reflect.ValueOf(data)); v.Kind() { + case reflect.Chan, + reflect.Func, + reflect.Interface, + reflect.Map, + reflect.Ptr, + reflect.Slice: + isNil = v.IsNil() + } + } + if isNil { + if !val.IsNil() && val.CanSet() { + nilValue := reflect.New(val.Type()).Elem() + val.Set(nilValue) + } + + return nil + } + + // Create an element of the concrete (non pointer) type and decode + // into that. Then set the value of the pointer to this type. + valType := val.Type() + valElemType := valType.Elem() + if val.CanSet() { + realVal := val + if realVal.IsNil() || d.config.ZeroFields { + realVal = reflect.New(valElemType) + } + + if err := d.decode(name, data, reflect.Indirect(realVal)); err != nil { + return err + } + + val.Set(realVal) + } else { + if err := d.decode(name, data, reflect.Indirect(val)); err != nil { + return err + } + } + return nil +} + +func (d *Decoder) decodeFunc(name string, data interface{}, val reflect.Value) error { + // Create an element of the concrete (non pointer) type and decode + // into that. Then set the value of the pointer to this type. + dataVal := reflect.Indirect(reflect.ValueOf(data)) + if val.Type() != dataVal.Type() { + return fmt.Errorf( + "'%s' expected type '%s', got unconvertible type '%s'", + name, val.Type(), dataVal.Type()) + } + val.Set(dataVal) + return nil +} + +func (d *Decoder) decodeSlice(name string, data interface{}, val reflect.Value) error { + dataVal := reflect.Indirect(reflect.ValueOf(data)) + dataValKind := dataVal.Kind() + valType := val.Type() + valElemType := valType.Elem() + sliceType := reflect.SliceOf(valElemType) + + valSlice := val + if valSlice.IsNil() || d.config.ZeroFields { + if d.config.WeaklyTypedInput { + switch { + // Slice and array we use the normal logic + case dataValKind == reflect.Slice, dataValKind == reflect.Array: + break + + // Empty maps turn into empty slices + case dataValKind == reflect.Map: + if dataVal.Len() == 0 { + val.Set(reflect.MakeSlice(sliceType, 0, 0)) + return nil + } + // Create slice of maps of other sizes + return d.decodeSlice(name, []interface{}{data}, val) + + case dataValKind == reflect.String && valElemType.Kind() == reflect.Uint8: + return d.decodeSlice(name, []byte(dataVal.String()), val) + + // All other types we try to convert to the slice type + // and "lift" it into it. i.e. a string becomes a string slice. + default: + // Just re-try this function with data as a slice. + return d.decodeSlice(name, []interface{}{data}, val) + } + } + + // Check input type + if dataValKind != reflect.Array && dataValKind != reflect.Slice { + return fmt.Errorf( + "'%s': source data must be an array or slice, got %s", name, dataValKind) + + } + + // If the input value is empty, then don't allocate since non-nil != nil + if dataVal.Len() == 0 { + return nil + } + + // Make a new slice to hold our result, same size as the original data. + valSlice = reflect.MakeSlice(sliceType, dataVal.Len(), dataVal.Len()) + } + + // Accumulate any errors + errors := make([]string, 0) + + for i := 0; i < dataVal.Len(); i++ { + currentData := dataVal.Index(i).Interface() + for valSlice.Len() <= i { + valSlice = reflect.Append(valSlice, reflect.Zero(valElemType)) + } + currentField := valSlice.Index(i) + + fieldName := fmt.Sprintf("%s[%d]", name, i) + if err := d.decode(fieldName, currentData, currentField); err != nil { + errors = appendErrors(errors, err) + } + } + + // Finally, set the value to the slice we built up + val.Set(valSlice) + + // If there were errors, we return those + if len(errors) > 0 { + return &Error{errors} + } + + return nil +} + +func (d *Decoder) decodeArray(name string, data interface{}, val reflect.Value) error { + dataVal := reflect.Indirect(reflect.ValueOf(data)) + dataValKind := dataVal.Kind() + valType := val.Type() + valElemType := valType.Elem() + arrayType := reflect.ArrayOf(valType.Len(), valElemType) + + valArray := val + + if valArray.Interface() == reflect.Zero(valArray.Type()).Interface() || d.config.ZeroFields { + // Check input type + if dataValKind != reflect.Array && dataValKind != reflect.Slice { + if d.config.WeaklyTypedInput { + switch { + // Empty maps turn into empty arrays + case dataValKind == reflect.Map: + if dataVal.Len() == 0 { + val.Set(reflect.Zero(arrayType)) + return nil + } + + // All other types we try to convert to the array type + // and "lift" it into it. i.e. a string becomes a string array. + default: + // Just re-try this function with data as a slice. + return d.decodeArray(name, []interface{}{data}, val) + } + } + + return fmt.Errorf( + "'%s': source data must be an array or slice, got %s", name, dataValKind) + + } + if dataVal.Len() > arrayType.Len() { + return fmt.Errorf( + "'%s': expected source data to have length less or equal to %d, got %d", name, arrayType.Len(), dataVal.Len()) + + } + + // Make a new array to hold our result, same size as the original data. + valArray = reflect.New(arrayType).Elem() + } + + // Accumulate any errors + errors := make([]string, 0) + + for i := 0; i < dataVal.Len(); i++ { + currentData := dataVal.Index(i).Interface() + currentField := valArray.Index(i) + + fieldName := fmt.Sprintf("%s[%d]", name, i) + if err := d.decode(fieldName, currentData, currentField); err != nil { + errors = appendErrors(errors, err) + } + } + + // Finally, set the value to the array we built up + val.Set(valArray) + + // If there were errors, we return those + if len(errors) > 0 { + return &Error{errors} + } + + return nil +} + +func (d *Decoder) decodeStruct(name string, data interface{}, val reflect.Value) error { + dataVal := reflect.Indirect(reflect.ValueOf(data)) + + // If the type of the value to write to and the data match directly, + // then we just set it directly instead of recursing into the structure. + if dataVal.Type() == val.Type() { + val.Set(dataVal) + return nil + } + + dataValKind := dataVal.Kind() + switch dataValKind { + case reflect.Map: + return d.decodeStructFromMap(name, dataVal, val) + + case reflect.Struct: + // Not the most efficient way to do this but we can optimize later if + // we want to. To convert from struct to struct we go to map first + // as an intermediary. + m := make(map[string]interface{}) + mval := reflect.Indirect(reflect.ValueOf(&m)) + if err := d.decodeMapFromStruct(name, dataVal, mval, mval); err != nil { + return err + } + + result := d.decodeStructFromMap(name, mval, val) + return result + + default: + return fmt.Errorf("'%s' expected a map, got '%s'", name, dataVal.Kind()) + } +} + +func (d *Decoder) decodeStructFromMap(name string, dataVal, val reflect.Value) error { + dataValType := dataVal.Type() + if kind := dataValType.Key().Kind(); kind != reflect.String && kind != reflect.Interface { + return fmt.Errorf( + "'%s' needs a map with string keys, has '%s' keys", + name, dataValType.Key().Kind()) + } + + dataValKeys := make(map[reflect.Value]struct{}) + dataValKeysUnused := make(map[interface{}]struct{}) + for _, dataValKey := range dataVal.MapKeys() { + dataValKeys[dataValKey] = struct{}{} + dataValKeysUnused[dataValKey.Interface()] = struct{}{} + } + + errors := make([]string, 0) + + // This slice will keep track of all the structs we'll be decoding. + // There can be more than one struct if there are embedded structs + // that are squashed. + structs := make([]reflect.Value, 1, 5) + structs[0] = val + + // Compile the list of all the fields that we're going to be decoding + // from all the structs. + type field struct { + field reflect.StructField + val reflect.Value + } + fields := []field{} + for len(structs) > 0 { + structVal := structs[0] + structs = structs[1:] + + structType := structVal.Type() + + for i := 0; i < structType.NumField(); i++ { + fieldType := structType.Field(i) + fieldKind := fieldType.Type.Kind() + + // If "squash" is specified in the tag, we squash the field down. + squash := false + tagParts := strings.Split(fieldType.Tag.Get(d.config.TagName), ",") + for _, tag := range tagParts[1:] { + if tag == "squash" { + squash = true + break + } + } + + if squash { + if fieldKind != reflect.Struct { + errors = appendErrors(errors, + fmt.Errorf("%s: unsupported type for squash: %s", fieldType.Name, fieldKind)) + } else { + structs = append(structs, structVal.FieldByName(fieldType.Name)) + } + continue + } + + // Normal struct field, store it away + fields = append(fields, field{fieldType, structVal.Field(i)}) + } + } + + // for fieldType, field := range fields { + for _, f := range fields { + field, fieldValue := f.field, f.val + fieldName := field.Name + + tagValue := field.Tag.Get(d.config.TagName) + tagValue = strings.SplitN(tagValue, ",", 2)[0] + if tagValue != "" { + fieldName = tagValue + } + + rawMapKey := reflect.ValueOf(fieldName) + rawMapVal := dataVal.MapIndex(rawMapKey) + if !rawMapVal.IsValid() { + // Do a slower search by iterating over each key and + // doing case-insensitive search. + for dataValKey := range dataValKeys { + mK, ok := dataValKey.Interface().(string) + if !ok { + // Not a string key + continue + } + + if strings.EqualFold(mK, fieldName) { + rawMapKey = dataValKey + rawMapVal = dataVal.MapIndex(dataValKey) + break + } + } + + if !rawMapVal.IsValid() { + // There was no matching key in the map for the value in + // the struct. Just ignore. + continue + } + } + + // Delete the key we're using from the unused map so we stop tracking + delete(dataValKeysUnused, rawMapKey.Interface()) + + if !fieldValue.IsValid() { + // This should never happen + panic("field is not valid") + } + + // If we can't set the field, then it is unexported or something, + // and we just continue onwards. + if !fieldValue.CanSet() { + continue + } + + // If the name is empty string, then we're at the root, and we + // don't dot-join the fields. + if name != "" { + fieldName = fmt.Sprintf("%s.%s", name, fieldName) + } + + if err := d.decode(fieldName, rawMapVal.Interface(), fieldValue); err != nil { + errors = appendErrors(errors, err) + } + } + + if d.config.ErrorUnused && len(dataValKeysUnused) > 0 { + keys := make([]string, 0, len(dataValKeysUnused)) + for rawKey := range dataValKeysUnused { + keys = append(keys, rawKey.(string)) + } + sort.Strings(keys) + + err := fmt.Errorf("'%s' has invalid keys: %s", name, strings.Join(keys, ", ")) + errors = appendErrors(errors, err) + } + + if len(errors) > 0 { + return &Error{errors} + } + + // Add the unused keys to the list of unused keys if we're tracking metadata + if d.config.Metadata != nil { + for rawKey := range dataValKeysUnused { + key := rawKey.(string) + if name != "" { + key = fmt.Sprintf("%s.%s", name, key) + } + + d.config.Metadata.Unused = append(d.config.Metadata.Unused, key) + } + } + + return nil +} + +func getKind(val reflect.Value) reflect.Kind { + kind := val.Kind() + + switch { + case kind >= reflect.Int && kind <= reflect.Int64: + return reflect.Int + case kind >= reflect.Uint && kind <= reflect.Uint64: + return reflect.Uint + case kind >= reflect.Float32 && kind <= reflect.Float64: + return reflect.Float32 + default: + return kind + } +} diff --git a/vendor/github.com/pelletier/go-toml/.gitignore b/vendor/github.com/pelletier/go-toml/.gitignore new file mode 100644 index 000000000..99e38bbc5 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/.gitignore @@ -0,0 +1,2 @@ +test_program/test_program_bin +fuzz/ diff --git a/vendor/github.com/pelletier/go-toml/.travis.yml b/vendor/github.com/pelletier/go-toml/.travis.yml new file mode 100644 index 000000000..c9fbf304b --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/.travis.yml @@ -0,0 +1,23 @@ +sudo: false +language: go +go: + - 1.8.x + - 1.9.x + - 1.10.x + - tip +matrix: + allow_failures: + - go: tip + fast_finish: true +script: + - if [ -n "$(go fmt ./...)" ]; then exit 1; fi + - ./test.sh + - ./benchmark.sh $TRAVIS_BRANCH https://github.com/$TRAVIS_REPO_SLUG.git +before_install: + - go get github.com/axw/gocov/gocov + - go get github.com/mattn/goveralls + - if ! go get code.google.com/p/go.tools/cmd/cover; then go get golang.org/x/tools/cmd/cover; fi +branches: + only: [master] +after_success: + - $HOME/gopath/bin/goveralls -service=travis-ci -coverprofile=coverage.out -repotoken $COVERALLS_TOKEN diff --git a/vendor/github.com/pelletier/go-toml/LICENSE b/vendor/github.com/pelletier/go-toml/LICENSE new file mode 100644 index 000000000..583bdae62 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2013 - 2017 Thomas Pelletier, Eric Anderton + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/pelletier/go-toml/README.md b/vendor/github.com/pelletier/go-toml/README.md new file mode 100644 index 000000000..0d357acf3 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/README.md @@ -0,0 +1,131 @@ +# go-toml + +Go library for the [TOML](https://github.com/mojombo/toml) format. + +This library supports TOML version +[v0.4.0](https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.4.0.md) + +[![GoDoc](https://godoc.org/github.com/pelletier/go-toml?status.svg)](http://godoc.org/github.com/pelletier/go-toml) +[![license](https://img.shields.io/github/license/pelletier/go-toml.svg)](https://github.com/pelletier/go-toml/blob/master/LICENSE) +[![Build Status](https://travis-ci.org/pelletier/go-toml.svg?branch=master)](https://travis-ci.org/pelletier/go-toml) +[![Coverage Status](https://coveralls.io/repos/github/pelletier/go-toml/badge.svg?branch=master)](https://coveralls.io/github/pelletier/go-toml?branch=master) +[![Go Report Card](https://goreportcard.com/badge/github.com/pelletier/go-toml)](https://goreportcard.com/report/github.com/pelletier/go-toml) + +## Features + +Go-toml provides the following features for using data parsed from TOML documents: + +* Load TOML documents from files and string data +* Easily navigate TOML structure using Tree +* Mashaling and unmarshaling to and from data structures +* Line & column position data for all parsed elements +* [Query support similar to JSON-Path](query/) +* Syntax errors contain line and column numbers + +## Import + +```go +import "github.com/pelletier/go-toml" +``` + +## Usage example + +Read a TOML document: + +```go +config, _ := toml.Load(` +[postgres] +user = "pelletier" +password = "mypassword"`) +// retrieve data directly +user := config.Get("postgres.user").(string) + +// or using an intermediate object +postgresConfig := config.Get("postgres").(*toml.Tree) +password := postgresConfig.Get("password").(string) +``` + +Or use Unmarshal: + +```go +type Postgres struct { + User string + Password string +} +type Config struct { + Postgres Postgres +} + +doc := []byte(` +[Postgres] +User = "pelletier" +Password = "mypassword"`) + +config := Config{} +toml.Unmarshal(doc, &config) +fmt.Println("user=", config.Postgres.User) +``` + +Or use a query: + +```go +// use a query to gather elements without walking the tree +q, _ := query.Compile("$..[user,password]") +results := q.Execute(config) +for ii, item := range results.Values() { + fmt.Println("Query result %d: %v", ii, item) +} +``` + +## Documentation + +The documentation and additional examples are available at +[godoc.org](http://godoc.org/github.com/pelletier/go-toml). + +## Tools + +Go-toml provides two handy command line tools: + +* `tomll`: Reads TOML files and lint them. + + ``` + go install github.com/pelletier/go-toml/cmd/tomll + tomll --help + ``` +* `tomljson`: Reads a TOML file and outputs its JSON representation. + + ``` + go install github.com/pelletier/go-toml/cmd/tomljson + tomljson --help + ``` + +## Contribute + +Feel free to report bugs and patches using GitHub's pull requests system on +[pelletier/go-toml](https://github.com/pelletier/go-toml). Any feedback would be +much appreciated! + +### Run tests + +You have to make sure two kind of tests run: + +1. The Go unit tests +2. The TOML examples base + +You can run both of them using `./test.sh`. + +### Fuzzing + +The script `./fuzz.sh` is available to +run [go-fuzz](https://github.com/dvyukov/go-fuzz) on go-toml. + +## Versioning + +Go-toml follows [Semantic Versioning](http://semver.org/). The supported version +of [TOML](https://github.com/toml-lang/toml) is indicated at the beginning of +this document. The last two major versions of Go are supported +(see [Go Release Policy](https://golang.org/doc/devel/release.html#policy)). + +## License + +The MIT License (MIT). Read [LICENSE](LICENSE). diff --git a/vendor/github.com/pelletier/go-toml/benchmark.json b/vendor/github.com/pelletier/go-toml/benchmark.json new file mode 100644 index 000000000..86f99c6a8 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/benchmark.json @@ -0,0 +1,164 @@ +{ + "array": { + "key1": [ + 1, + 2, + 3 + ], + "key2": [ + "red", + "yellow", + "green" + ], + "key3": [ + [ + 1, + 2 + ], + [ + 3, + 4, + 5 + ] + ], + "key4": [ + [ + 1, + 2 + ], + [ + "a", + "b", + "c" + ] + ], + "key5": [ + 1, + 2, + 3 + ], + "key6": [ + 1, + 2 + ] + }, + "boolean": { + "False": false, + "True": true + }, + "datetime": { + "key1": "1979-05-27T07:32:00Z", + "key2": "1979-05-27T00:32:00-07:00", + "key3": "1979-05-27T00:32:00.999999-07:00" + }, + "float": { + "both": { + "key": 6.626e-34 + }, + "exponent": { + "key1": 5e+22, + "key2": 1000000, + "key3": -0.02 + }, + "fractional": { + "key1": 1, + "key2": 3.1415, + "key3": -0.01 + }, + "underscores": { + "key1": 9224617.445991227, + "key2": 1e+100 + } + }, + "fruit": [{ + "name": "apple", + "physical": { + "color": "red", + "shape": "round" + }, + "variety": [{ + "name": "red delicious" + }, + { + "name": "granny smith" + } + ] + }, + { + "name": "banana", + "variety": [{ + "name": "plantain" + }] + } + ], + "integer": { + "key1": 99, + "key2": 42, + "key3": 0, + "key4": -17, + "underscores": { + "key1": 1000, + "key2": 5349221, + "key3": 12345 + } + }, + "products": [{ + "name": "Hammer", + "sku": 738594937 + }, + {}, + { + "color": "gray", + "name": "Nail", + "sku": 284758393 + } + ], + "string": { + "basic": { + "basic": "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF." + }, + "literal": { + "multiline": { + "lines": "The first newline is\ntrimmed in raw strings.\n All other whitespace\n is preserved.\n", + "regex2": "I [dw]on't need \\d{2} apples" + }, + "quoted": "Tom \"Dubs\" Preston-Werner", + "regex": "\u003c\\i\\c*\\s*\u003e", + "winpath": "C:\\Users\\nodejs\\templates", + "winpath2": "\\\\ServerX\\admin$\\system32\\" + }, + "multiline": { + "continued": { + "key1": "The quick brown fox jumps over the lazy dog.", + "key2": "The quick brown fox jumps over the lazy dog.", + "key3": "The quick brown fox jumps over the lazy dog." + }, + "key1": "One\nTwo", + "key2": "One\nTwo", + "key3": "One\nTwo" + } + }, + "table": { + "inline": { + "name": { + "first": "Tom", + "last": "Preston-Werner" + }, + "point": { + "x": 1, + "y": 2 + } + }, + "key": "value", + "subtable": { + "key": "another value" + } + }, + "x": { + "y": { + "z": { + "w": {} + } + } + } +} diff --git a/vendor/github.com/pelletier/go-toml/benchmark.sh b/vendor/github.com/pelletier/go-toml/benchmark.sh new file mode 100644 index 000000000..8b8bb528e --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/benchmark.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -e + +reference_ref=${1:-master} +reference_git=${2:-.} + +if ! `hash benchstat 2>/dev/null`; then + echo "Installing benchstat" + go get golang.org/x/perf/cmd/benchstat + go install golang.org/x/perf/cmd/benchstat +fi + +tempdir=`mktemp -d /tmp/go-toml-benchmark-XXXXXX` +ref_tempdir="${tempdir}/ref" +ref_benchmark="${ref_tempdir}/benchmark-`echo -n ${reference_ref}|tr -s '/' '-'`.txt" +local_benchmark="`pwd`/benchmark-local.txt" + +echo "=== ${reference_ref} (${ref_tempdir})" +git clone ${reference_git} ${ref_tempdir} >/dev/null 2>/dev/null +pushd ${ref_tempdir} >/dev/null +git checkout ${reference_ref} >/dev/null 2>/dev/null +go test -bench=. -benchmem | tee ${ref_benchmark} +popd >/dev/null + +echo "" +echo "=== local" +go test -bench=. -benchmem | tee ${local_benchmark} + +echo "" +echo "=== diff" +benchstat -delta-test=none ${ref_benchmark} ${local_benchmark} \ No newline at end of file diff --git a/vendor/github.com/pelletier/go-toml/benchmark.toml b/vendor/github.com/pelletier/go-toml/benchmark.toml new file mode 100644 index 000000000..dfd77e096 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/benchmark.toml @@ -0,0 +1,244 @@ +################################################################################ +## Comment + +# Speak your mind with the hash symbol. They go from the symbol to the end of +# the line. + + +################################################################################ +## Table + +# Tables (also known as hash tables or dictionaries) are collections of +# key/value pairs. They appear in square brackets on a line by themselves. + +[table] + +key = "value" # Yeah, you can do this. + +# Nested tables are denoted by table names with dots in them. Name your tables +# whatever crap you please, just don't use #, ., [ or ]. + +[table.subtable] + +key = "another value" + +# You don't need to specify all the super-tables if you don't want to. TOML +# knows how to do it for you. + +# [x] you +# [x.y] don't +# [x.y.z] need these +[x.y.z.w] # for this to work + + +################################################################################ +## Inline Table + +# Inline tables provide a more compact syntax for expressing tables. They are +# especially useful for grouped data that can otherwise quickly become verbose. +# Inline tables are enclosed in curly braces `{` and `}`. No newlines are +# allowed between the curly braces unless they are valid within a value. + +[table.inline] + +name = { first = "Tom", last = "Preston-Werner" } +point = { x = 1, y = 2 } + + +################################################################################ +## String + +# There are four ways to express strings: basic, multi-line basic, literal, and +# multi-line literal. All strings must contain only valid UTF-8 characters. + +[string.basic] + +basic = "I'm a string. \"You can quote me\". Name\tJos\u00E9\nLocation\tSF." + +[string.multiline] + +# The following strings are byte-for-byte equivalent: +key1 = "One\nTwo" +key2 = """One\nTwo""" +key3 = """ +One +Two""" + +[string.multiline.continued] + +# The following strings are byte-for-byte equivalent: +key1 = "The quick brown fox jumps over the lazy dog." + +key2 = """ +The quick brown \ + + + fox jumps over \ + the lazy dog.""" + +key3 = """\ + The quick brown \ + fox jumps over \ + the lazy dog.\ + """ + +[string.literal] + +# What you see is what you get. +winpath = 'C:\Users\nodejs\templates' +winpath2 = '\\ServerX\admin$\system32\' +quoted = 'Tom "Dubs" Preston-Werner' +regex = '<\i\c*\s*>' + + +[string.literal.multiline] + +regex2 = '''I [dw]on't need \d{2} apples''' +lines = ''' +The first newline is +trimmed in raw strings. + All other whitespace + is preserved. +''' + + +################################################################################ +## Integer + +# Integers are whole numbers. Positive numbers may be prefixed with a plus sign. +# Negative numbers are prefixed with a minus sign. + +[integer] + +key1 = +99 +key2 = 42 +key3 = 0 +key4 = -17 + +[integer.underscores] + +# For large numbers, you may use underscores to enhance readability. Each +# underscore must be surrounded by at least one digit. +key1 = 1_000 +key2 = 5_349_221 +key3 = 1_2_3_4_5 # valid but inadvisable + + +################################################################################ +## Float + +# A float consists of an integer part (which may be prefixed with a plus or +# minus sign) followed by a fractional part and/or an exponent part. + +[float.fractional] + +key1 = +1.0 +key2 = 3.1415 +key3 = -0.01 + +[float.exponent] + +key1 = 5e+22 +key2 = 1e6 +key3 = -2E-2 + +[float.both] + +key = 6.626e-34 + +[float.underscores] + +key1 = 9_224_617.445_991_228_313 +key2 = 1e1_00 + + +################################################################################ +## Boolean + +# Booleans are just the tokens you're used to. Always lowercase. + +[boolean] + +True = true +False = false + + +################################################################################ +## Datetime + +# Datetimes are RFC 3339 dates. + +[datetime] + +key1 = 1979-05-27T07:32:00Z +key2 = 1979-05-27T00:32:00-07:00 +key3 = 1979-05-27T00:32:00.999999-07:00 + + +################################################################################ +## Array + +# Arrays are square brackets with other primitives inside. Whitespace is +# ignored. Elements are separated by commas. Data types may not be mixed. + +[array] + +key1 = [ 1, 2, 3 ] +key2 = [ "red", "yellow", "green" ] +key3 = [ [ 1, 2 ], [3, 4, 5] ] +#key4 = [ [ 1, 2 ], ["a", "b", "c"] ] # this is ok + +# Arrays can also be multiline. So in addition to ignoring whitespace, arrays +# also ignore newlines between the brackets. Terminating commas are ok before +# the closing bracket. + +key5 = [ + 1, 2, 3 +] +key6 = [ + 1, + 2, # this is ok +] + + +################################################################################ +## Array of Tables + +# These can be expressed by using a table name in double brackets. Each table +# with the same double bracketed name will be an element in the array. The +# tables are inserted in the order encountered. + +[[products]] + +name = "Hammer" +sku = 738594937 + +[[products]] + +[[products]] + +name = "Nail" +sku = 284758393 +color = "gray" + + +# You can create nested arrays of tables as well. + +[[fruit]] + name = "apple" + + [fruit.physical] + color = "red" + shape = "round" + + [[fruit.variety]] + name = "red delicious" + + [[fruit.variety]] + name = "granny smith" + +[[fruit]] + name = "banana" + + [[fruit.variety]] + name = "plantain" diff --git a/vendor/github.com/pelletier/go-toml/benchmark.yml b/vendor/github.com/pelletier/go-toml/benchmark.yml new file mode 100644 index 000000000..0bd19f08a --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/benchmark.yml @@ -0,0 +1,121 @@ +--- +array: + key1: + - 1 + - 2 + - 3 + key2: + - red + - yellow + - green + key3: + - - 1 + - 2 + - - 3 + - 4 + - 5 + key4: + - - 1 + - 2 + - - a + - b + - c + key5: + - 1 + - 2 + - 3 + key6: + - 1 + - 2 +boolean: + 'False': false + 'True': true +datetime: + key1: '1979-05-27T07:32:00Z' + key2: '1979-05-27T00:32:00-07:00' + key3: '1979-05-27T00:32:00.999999-07:00' +float: + both: + key: 6.626e-34 + exponent: + key1: 5.0e+22 + key2: 1000000 + key3: -0.02 + fractional: + key1: 1 + key2: 3.1415 + key3: -0.01 + underscores: + key1: 9224617.445991227 + key2: 1.0e+100 +fruit: +- name: apple + physical: + color: red + shape: round + variety: + - name: red delicious + - name: granny smith +- name: banana + variety: + - name: plantain +integer: + key1: 99 + key2: 42 + key3: 0 + key4: -17 + underscores: + key1: 1000 + key2: 5349221 + key3: 12345 +products: +- name: Hammer + sku: 738594937 +- {} +- color: gray + name: Nail + sku: 284758393 +string: + basic: + basic: "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF." + literal: + multiline: + lines: | + The first newline is + trimmed in raw strings. + All other whitespace + is preserved. + regex2: I [dw]on't need \d{2} apples + quoted: Tom "Dubs" Preston-Werner + regex: "<\\i\\c*\\s*>" + winpath: C:\Users\nodejs\templates + winpath2: "\\\\ServerX\\admin$\\system32\\" + multiline: + continued: + key1: The quick brown fox jumps over the lazy dog. + key2: The quick brown fox jumps over the lazy dog. + key3: The quick brown fox jumps over the lazy dog. + key1: |- + One + Two + key2: |- + One + Two + key3: |- + One + Two +table: + inline: + name: + first: Tom + last: Preston-Werner + point: + x: 1 + y: 2 + key: value + subtable: + key: another value +x: + y: + z: + w: {} diff --git a/vendor/github.com/pelletier/go-toml/doc.go b/vendor/github.com/pelletier/go-toml/doc.go new file mode 100644 index 000000000..d5fd98c02 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/doc.go @@ -0,0 +1,23 @@ +// Package toml is a TOML parser and manipulation library. +// +// This version supports the specification as described in +// https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.4.0.md +// +// Marshaling +// +// Go-toml can marshal and unmarshal TOML documents from and to data +// structures. +// +// TOML document as a tree +// +// Go-toml can operate on a TOML document as a tree. Use one of the Load* +// functions to parse TOML data and obtain a Tree instance, then one of its +// methods to manipulate the tree. +// +// JSONPath-like queries +// +// The package github.com/pelletier/go-toml/query implements a system +// similar to JSONPath to quickly retrieve elements of a TOML document using a +// single expression. See the package documentation for more information. +// +package toml diff --git a/vendor/github.com/pelletier/go-toml/example-crlf.toml b/vendor/github.com/pelletier/go-toml/example-crlf.toml new file mode 100644 index 000000000..12950a163 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/example-crlf.toml @@ -0,0 +1,29 @@ +# This is a TOML document. Boom. + +title = "TOML Example" + +[owner] +name = "Tom Preston-Werner" +organization = "GitHub" +bio = "GitHub Cofounder & CEO\nLikes tater tots and beer." +dob = 1979-05-27T07:32:00Z # First class dates? Why not? + +[database] +server = "192.168.1.1" +ports = [ 8001, 8001, 8002 ] +connection_max = 5000 +enabled = true + +[servers] + + # You can indent as you please. Tabs or spaces. TOML don't care. + [servers.alpha] + ip = "10.0.0.1" + dc = "eqdc10" + + [servers.beta] + ip = "10.0.0.2" + dc = "eqdc10" + +[clients] +data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it diff --git a/vendor/github.com/pelletier/go-toml/example.toml b/vendor/github.com/pelletier/go-toml/example.toml new file mode 100644 index 000000000..3d902f282 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/example.toml @@ -0,0 +1,29 @@ +# This is a TOML document. Boom. + +title = "TOML Example" + +[owner] +name = "Tom Preston-Werner" +organization = "GitHub" +bio = "GitHub Cofounder & CEO\nLikes tater tots and beer." +dob = 1979-05-27T07:32:00Z # First class dates? Why not? + +[database] +server = "192.168.1.1" +ports = [ 8001, 8001, 8002 ] +connection_max = 5000 +enabled = true + +[servers] + + # You can indent as you please. Tabs or spaces. TOML don't care. + [servers.alpha] + ip = "10.0.0.1" + dc = "eqdc10" + + [servers.beta] + ip = "10.0.0.2" + dc = "eqdc10" + +[clients] +data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it diff --git a/vendor/github.com/pelletier/go-toml/fuzz.go b/vendor/github.com/pelletier/go-toml/fuzz.go new file mode 100644 index 000000000..14570c8d3 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/fuzz.go @@ -0,0 +1,31 @@ +// +build gofuzz + +package toml + +func Fuzz(data []byte) int { + tree, err := LoadBytes(data) + if err != nil { + if tree != nil { + panic("tree must be nil if there is an error") + } + return 0 + } + + str, err := tree.ToTomlString() + if err != nil { + if str != "" { + panic(`str must be "" if there is an error`) + } + panic(err) + } + + tree, err = Load(str) + if err != nil { + if tree != nil { + panic("tree must be nil if there is an error") + } + return 0 + } + + return 1 +} diff --git a/vendor/github.com/pelletier/go-toml/fuzz.sh b/vendor/github.com/pelletier/go-toml/fuzz.sh new file mode 100644 index 000000000..3204b4c44 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/fuzz.sh @@ -0,0 +1,15 @@ +#! /bin/sh +set -eu + +go get github.com/dvyukov/go-fuzz/go-fuzz +go get github.com/dvyukov/go-fuzz/go-fuzz-build + +if [ ! -e toml-fuzz.zip ]; then + go-fuzz-build github.com/pelletier/go-toml +fi + +rm -fr fuzz +mkdir -p fuzz/corpus +cp *.toml fuzz/corpus + +go-fuzz -bin=toml-fuzz.zip -workdir=fuzz diff --git a/vendor/github.com/pelletier/go-toml/keysparsing.go b/vendor/github.com/pelletier/go-toml/keysparsing.go new file mode 100644 index 000000000..284db6467 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/keysparsing.go @@ -0,0 +1,85 @@ +// Parsing keys handling both bare and quoted keys. + +package toml + +import ( + "bytes" + "errors" + "fmt" + "unicode" +) + +// Convert the bare key group string to an array. +// The input supports double quotation to allow "." inside the key name, +// but escape sequences are not supported. Lexers must unescape them beforehand. +func parseKey(key string) ([]string, error) { + groups := []string{} + var buffer bytes.Buffer + inQuotes := false + wasInQuotes := false + ignoreSpace := true + expectDot := false + + for _, char := range key { + if ignoreSpace { + if char == ' ' { + continue + } + ignoreSpace = false + } + switch char { + case '"': + if inQuotes { + groups = append(groups, buffer.String()) + buffer.Reset() + wasInQuotes = true + } + inQuotes = !inQuotes + expectDot = false + case '.': + if inQuotes { + buffer.WriteRune(char) + } else { + if !wasInQuotes { + if buffer.Len() == 0 { + return nil, errors.New("empty table key") + } + groups = append(groups, buffer.String()) + buffer.Reset() + } + ignoreSpace = true + expectDot = false + wasInQuotes = false + } + case ' ': + if inQuotes { + buffer.WriteRune(char) + } else { + expectDot = true + } + default: + if !inQuotes && !isValidBareChar(char) { + return nil, fmt.Errorf("invalid bare character: %c", char) + } + if !inQuotes && expectDot { + return nil, errors.New("what?") + } + buffer.WriteRune(char) + expectDot = false + } + } + if inQuotes { + return nil, errors.New("mismatched quotes") + } + if buffer.Len() > 0 { + groups = append(groups, buffer.String()) + } + if len(groups) == 0 { + return nil, errors.New("empty key") + } + return groups, nil +} + +func isValidBareChar(r rune) bool { + return isAlphanumeric(r) || r == '-' || unicode.IsNumber(r) +} diff --git a/vendor/github.com/pelletier/go-toml/lexer.go b/vendor/github.com/pelletier/go-toml/lexer.go new file mode 100644 index 000000000..d11de4285 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/lexer.go @@ -0,0 +1,750 @@ +// TOML lexer. +// +// Written using the principles developed by Rob Pike in +// http://www.youtube.com/watch?v=HxaD_trXwRE + +package toml + +import ( + "bytes" + "errors" + "fmt" + "regexp" + "strconv" + "strings" +) + +var dateRegexp *regexp.Regexp + +// Define state functions +type tomlLexStateFn func() tomlLexStateFn + +// Define lexer +type tomlLexer struct { + inputIdx int + input []rune // Textual source + currentTokenStart int + currentTokenStop int + tokens []token + depth int + line int + col int + endbufferLine int + endbufferCol int +} + +// Basic read operations on input + +func (l *tomlLexer) read() rune { + r := l.peek() + if r == '\n' { + l.endbufferLine++ + l.endbufferCol = 1 + } else { + l.endbufferCol++ + } + l.inputIdx++ + return r +} + +func (l *tomlLexer) next() rune { + r := l.read() + + if r != eof { + l.currentTokenStop++ + } + return r +} + +func (l *tomlLexer) ignore() { + l.currentTokenStart = l.currentTokenStop + l.line = l.endbufferLine + l.col = l.endbufferCol +} + +func (l *tomlLexer) skip() { + l.next() + l.ignore() +} + +func (l *tomlLexer) fastForward(n int) { + for i := 0; i < n; i++ { + l.next() + } +} + +func (l *tomlLexer) emitWithValue(t tokenType, value string) { + l.tokens = append(l.tokens, token{ + Position: Position{l.line, l.col}, + typ: t, + val: value, + }) + l.ignore() +} + +func (l *tomlLexer) emit(t tokenType) { + l.emitWithValue(t, string(l.input[l.currentTokenStart:l.currentTokenStop])) +} + +func (l *tomlLexer) peek() rune { + if l.inputIdx >= len(l.input) { + return eof + } + return l.input[l.inputIdx] +} + +func (l *tomlLexer) peekString(size int) string { + maxIdx := len(l.input) + upperIdx := l.inputIdx + size // FIXME: potential overflow + if upperIdx > maxIdx { + upperIdx = maxIdx + } + return string(l.input[l.inputIdx:upperIdx]) +} + +func (l *tomlLexer) follow(next string) bool { + return next == l.peekString(len(next)) +} + +// Error management + +func (l *tomlLexer) errorf(format string, args ...interface{}) tomlLexStateFn { + l.tokens = append(l.tokens, token{ + Position: Position{l.line, l.col}, + typ: tokenError, + val: fmt.Sprintf(format, args...), + }) + return nil +} + +// State functions + +func (l *tomlLexer) lexVoid() tomlLexStateFn { + for { + next := l.peek() + switch next { + case '[': + return l.lexTableKey + case '#': + return l.lexComment(l.lexVoid) + case '=': + return l.lexEqual + case '\r': + fallthrough + case '\n': + l.skip() + continue + } + + if isSpace(next) { + l.skip() + } + + if l.depth > 0 { + return l.lexRvalue + } + + if isKeyStartChar(next) { + return l.lexKey + } + + if next == eof { + l.next() + break + } + } + + l.emit(tokenEOF) + return nil +} + +func (l *tomlLexer) lexRvalue() tomlLexStateFn { + for { + next := l.peek() + switch next { + case '.': + return l.errorf("cannot start float with a dot") + case '=': + return l.lexEqual + case '[': + l.depth++ + return l.lexLeftBracket + case ']': + l.depth-- + return l.lexRightBracket + case '{': + return l.lexLeftCurlyBrace + case '}': + return l.lexRightCurlyBrace + case '#': + return l.lexComment(l.lexRvalue) + case '"': + return l.lexString + case '\'': + return l.lexLiteralString + case ',': + return l.lexComma + case '\r': + fallthrough + case '\n': + l.skip() + if l.depth == 0 { + return l.lexVoid + } + return l.lexRvalue + case '_': + return l.errorf("cannot start number with underscore") + } + + if l.follow("true") { + return l.lexTrue + } + + if l.follow("false") { + return l.lexFalse + } + + if l.follow("inf") { + return l.lexInf + } + + if l.follow("nan") { + return l.lexNan + } + + if isSpace(next) { + l.skip() + continue + } + + if next == eof { + l.next() + break + } + + possibleDate := l.peekString(35) + dateMatch := dateRegexp.FindString(possibleDate) + if dateMatch != "" { + l.fastForward(len(dateMatch)) + return l.lexDate + } + + if next == '+' || next == '-' || isDigit(next) { + return l.lexNumber + } + + if isAlphanumeric(next) { + return l.lexKey + } + + return l.errorf("no value can start with %c", next) + } + + l.emit(tokenEOF) + return nil +} + +func (l *tomlLexer) lexLeftCurlyBrace() tomlLexStateFn { + l.next() + l.emit(tokenLeftCurlyBrace) + return l.lexRvalue +} + +func (l *tomlLexer) lexRightCurlyBrace() tomlLexStateFn { + l.next() + l.emit(tokenRightCurlyBrace) + return l.lexRvalue +} + +func (l *tomlLexer) lexDate() tomlLexStateFn { + l.emit(tokenDate) + return l.lexRvalue +} + +func (l *tomlLexer) lexTrue() tomlLexStateFn { + l.fastForward(4) + l.emit(tokenTrue) + return l.lexRvalue +} + +func (l *tomlLexer) lexFalse() tomlLexStateFn { + l.fastForward(5) + l.emit(tokenFalse) + return l.lexRvalue +} + +func (l *tomlLexer) lexInf() tomlLexStateFn { + l.fastForward(3) + l.emit(tokenInf) + return l.lexRvalue +} + +func (l *tomlLexer) lexNan() tomlLexStateFn { + l.fastForward(3) + l.emit(tokenNan) + return l.lexRvalue +} + +func (l *tomlLexer) lexEqual() tomlLexStateFn { + l.next() + l.emit(tokenEqual) + return l.lexRvalue +} + +func (l *tomlLexer) lexComma() tomlLexStateFn { + l.next() + l.emit(tokenComma) + return l.lexRvalue +} + +// Parse the key and emits its value without escape sequences. +// bare keys, basic string keys and literal string keys are supported. +func (l *tomlLexer) lexKey() tomlLexStateFn { + growingString := "" + + for r := l.peek(); isKeyChar(r) || r == '\n' || r == '\r'; r = l.peek() { + if r == '"' { + l.next() + str, err := l.lexStringAsString(`"`, false, true) + if err != nil { + return l.errorf(err.Error()) + } + growingString += str + l.next() + continue + } else if r == '\'' { + l.next() + str, err := l.lexLiteralStringAsString(`'`, false) + if err != nil { + return l.errorf(err.Error()) + } + growingString += str + l.next() + continue + } else if r == '\n' { + return l.errorf("keys cannot contain new lines") + } else if isSpace(r) { + break + } else if !isValidBareChar(r) { + return l.errorf("keys cannot contain %c character", r) + } + growingString += string(r) + l.next() + } + l.emitWithValue(tokenKey, growingString) + return l.lexVoid +} + +func (l *tomlLexer) lexComment(previousState tomlLexStateFn) tomlLexStateFn { + return func() tomlLexStateFn { + for next := l.peek(); next != '\n' && next != eof; next = l.peek() { + if next == '\r' && l.follow("\r\n") { + break + } + l.next() + } + l.ignore() + return previousState + } +} + +func (l *tomlLexer) lexLeftBracket() tomlLexStateFn { + l.next() + l.emit(tokenLeftBracket) + return l.lexRvalue +} + +func (l *tomlLexer) lexLiteralStringAsString(terminator string, discardLeadingNewLine bool) (string, error) { + growingString := "" + + if discardLeadingNewLine { + if l.follow("\r\n") { + l.skip() + l.skip() + } else if l.peek() == '\n' { + l.skip() + } + } + + // find end of string + for { + if l.follow(terminator) { + return growingString, nil + } + + next := l.peek() + if next == eof { + break + } + growingString += string(l.next()) + } + + return "", errors.New("unclosed string") +} + +func (l *tomlLexer) lexLiteralString() tomlLexStateFn { + l.skip() + + // handle special case for triple-quote + terminator := "'" + discardLeadingNewLine := false + if l.follow("''") { + l.skip() + l.skip() + terminator = "'''" + discardLeadingNewLine = true + } + + str, err := l.lexLiteralStringAsString(terminator, discardLeadingNewLine) + if err != nil { + return l.errorf(err.Error()) + } + + l.emitWithValue(tokenString, str) + l.fastForward(len(terminator)) + l.ignore() + return l.lexRvalue +} + +// Lex a string and return the results as a string. +// Terminator is the substring indicating the end of the token. +// The resulting string does not include the terminator. +func (l *tomlLexer) lexStringAsString(terminator string, discardLeadingNewLine, acceptNewLines bool) (string, error) { + growingString := "" + + if discardLeadingNewLine { + if l.follow("\r\n") { + l.skip() + l.skip() + } else if l.peek() == '\n' { + l.skip() + } + } + + for { + if l.follow(terminator) { + return growingString, nil + } + + if l.follow("\\") { + l.next() + switch l.peek() { + case '\r': + fallthrough + case '\n': + fallthrough + case '\t': + fallthrough + case ' ': + // skip all whitespace chars following backslash + for strings.ContainsRune("\r\n\t ", l.peek()) { + l.next() + } + case '"': + growingString += "\"" + l.next() + case 'n': + growingString += "\n" + l.next() + case 'b': + growingString += "\b" + l.next() + case 'f': + growingString += "\f" + l.next() + case '/': + growingString += "/" + l.next() + case 't': + growingString += "\t" + l.next() + case 'r': + growingString += "\r" + l.next() + case '\\': + growingString += "\\" + l.next() + case 'u': + l.next() + code := "" + for i := 0; i < 4; i++ { + c := l.peek() + if !isHexDigit(c) { + return "", errors.New("unfinished unicode escape") + } + l.next() + code = code + string(c) + } + intcode, err := strconv.ParseInt(code, 16, 32) + if err != nil { + return "", errors.New("invalid unicode escape: \\u" + code) + } + growingString += string(rune(intcode)) + case 'U': + l.next() + code := "" + for i := 0; i < 8; i++ { + c := l.peek() + if !isHexDigit(c) { + return "", errors.New("unfinished unicode escape") + } + l.next() + code = code + string(c) + } + intcode, err := strconv.ParseInt(code, 16, 64) + if err != nil { + return "", errors.New("invalid unicode escape: \\U" + code) + } + growingString += string(rune(intcode)) + default: + return "", errors.New("invalid escape sequence: \\" + string(l.peek())) + } + } else { + r := l.peek() + + if 0x00 <= r && r <= 0x1F && !(acceptNewLines && (r == '\n' || r == '\r')) { + return "", fmt.Errorf("unescaped control character %U", r) + } + l.next() + growingString += string(r) + } + + if l.peek() == eof { + break + } + } + + return "", errors.New("unclosed string") +} + +func (l *tomlLexer) lexString() tomlLexStateFn { + l.skip() + + // handle special case for triple-quote + terminator := `"` + discardLeadingNewLine := false + acceptNewLines := false + if l.follow(`""`) { + l.skip() + l.skip() + terminator = `"""` + discardLeadingNewLine = true + acceptNewLines = true + } + + str, err := l.lexStringAsString(terminator, discardLeadingNewLine, acceptNewLines) + + if err != nil { + return l.errorf(err.Error()) + } + + l.emitWithValue(tokenString, str) + l.fastForward(len(terminator)) + l.ignore() + return l.lexRvalue +} + +func (l *tomlLexer) lexTableKey() tomlLexStateFn { + l.next() + + if l.peek() == '[' { + // token '[[' signifies an array of tables + l.next() + l.emit(tokenDoubleLeftBracket) + return l.lexInsideTableArrayKey + } + // vanilla table key + l.emit(tokenLeftBracket) + return l.lexInsideTableKey +} + +// Parse the key till "]]", but only bare keys are supported +func (l *tomlLexer) lexInsideTableArrayKey() tomlLexStateFn { + for r := l.peek(); r != eof; r = l.peek() { + switch r { + case ']': + if l.currentTokenStop > l.currentTokenStart { + l.emit(tokenKeyGroupArray) + } + l.next() + if l.peek() != ']' { + break + } + l.next() + l.emit(tokenDoubleRightBracket) + return l.lexVoid + case '[': + return l.errorf("table array key cannot contain ']'") + default: + l.next() + } + } + return l.errorf("unclosed table array key") +} + +// Parse the key till "]" but only bare keys are supported +func (l *tomlLexer) lexInsideTableKey() tomlLexStateFn { + for r := l.peek(); r != eof; r = l.peek() { + switch r { + case ']': + if l.currentTokenStop > l.currentTokenStart { + l.emit(tokenKeyGroup) + } + l.next() + l.emit(tokenRightBracket) + return l.lexVoid + case '[': + return l.errorf("table key cannot contain ']'") + default: + l.next() + } + } + return l.errorf("unclosed table key") +} + +func (l *tomlLexer) lexRightBracket() tomlLexStateFn { + l.next() + l.emit(tokenRightBracket) + return l.lexRvalue +} + +type validRuneFn func(r rune) bool + +func isValidHexRune(r rune) bool { + return r >= 'a' && r <= 'f' || + r >= 'A' && r <= 'F' || + r >= '0' && r <= '9' || + r == '_' +} + +func isValidOctalRune(r rune) bool { + return r >= '0' && r <= '7' || r == '_' +} + +func isValidBinaryRune(r rune) bool { + return r == '0' || r == '1' || r == '_' +} + +func (l *tomlLexer) lexNumber() tomlLexStateFn { + r := l.peek() + + if r == '0' { + follow := l.peekString(2) + if len(follow) == 2 { + var isValidRune validRuneFn + switch follow[1] { + case 'x': + isValidRune = isValidHexRune + case 'o': + isValidRune = isValidOctalRune + case 'b': + isValidRune = isValidBinaryRune + default: + if follow[1] >= 'a' && follow[1] <= 'z' || follow[1] >= 'A' && follow[1] <= 'Z' { + return l.errorf("unknown number base: %s. possible options are x (hex) o (octal) b (binary)", string(follow[1])) + } + } + + if isValidRune != nil { + l.next() + l.next() + digitSeen := false + for { + next := l.peek() + if !isValidRune(next) { + break + } + digitSeen = true + l.next() + } + + if !digitSeen { + return l.errorf("number needs at least one digit") + } + + l.emit(tokenInteger) + + return l.lexRvalue + } + } + } + + if r == '+' || r == '-' { + l.next() + if l.follow("inf") { + return l.lexInf + } + if l.follow("nan") { + return l.lexNan + } + } + + pointSeen := false + expSeen := false + digitSeen := false + for { + next := l.peek() + if next == '.' { + if pointSeen { + return l.errorf("cannot have two dots in one float") + } + l.next() + if !isDigit(l.peek()) { + return l.errorf("float cannot end with a dot") + } + pointSeen = true + } else if next == 'e' || next == 'E' { + expSeen = true + l.next() + r := l.peek() + if r == '+' || r == '-' { + l.next() + } + } else if isDigit(next) { + digitSeen = true + l.next() + } else if next == '_' { + l.next() + } else { + break + } + if pointSeen && !digitSeen { + return l.errorf("cannot start float with a dot") + } + } + + if !digitSeen { + return l.errorf("no digit in that number") + } + if pointSeen || expSeen { + l.emit(tokenFloat) + } else { + l.emit(tokenInteger) + } + return l.lexRvalue +} + +func (l *tomlLexer) run() { + for state := l.lexVoid; state != nil; { + state = state() + } +} + +func init() { + dateRegexp = regexp.MustCompile(`^\d{1,4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,9})?(Z|[+-]\d{2}:\d{2})`) +} + +// Entry point +func lexToml(inputBytes []byte) []token { + runes := bytes.Runes(inputBytes) + l := &tomlLexer{ + input: runes, + tokens: make([]token, 0, 256), + line: 1, + col: 1, + endbufferLine: 1, + endbufferCol: 1, + } + l.run() + return l.tokens +} diff --git a/vendor/github.com/pelletier/go-toml/marshal.go b/vendor/github.com/pelletier/go-toml/marshal.go new file mode 100644 index 000000000..671da5564 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/marshal.go @@ -0,0 +1,609 @@ +package toml + +import ( + "bytes" + "errors" + "fmt" + "io" + "reflect" + "strconv" + "strings" + "time" +) + +const tagKeyMultiline = "multiline" + +type tomlOpts struct { + name string + comment string + commented bool + multiline bool + include bool + omitempty bool +} + +type encOpts struct { + quoteMapKeys bool + arraysOneElementPerLine bool +} + +var encOptsDefaults = encOpts{ + quoteMapKeys: false, +} + +var timeType = reflect.TypeOf(time.Time{}) +var marshalerType = reflect.TypeOf(new(Marshaler)).Elem() + +// Check if the given marshall type maps to a Tree primitive +func isPrimitive(mtype reflect.Type) bool { + switch mtype.Kind() { + case reflect.Ptr: + return isPrimitive(mtype.Elem()) + case reflect.Bool: + return true + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return true + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return true + case reflect.Float32, reflect.Float64: + return true + case reflect.String: + return true + case reflect.Struct: + return mtype == timeType || isCustomMarshaler(mtype) + default: + return false + } +} + +// Check if the given marshall type maps to a Tree slice +func isTreeSlice(mtype reflect.Type) bool { + switch mtype.Kind() { + case reflect.Slice: + return !isOtherSlice(mtype) + default: + return false + } +} + +// Check if the given marshall type maps to a non-Tree slice +func isOtherSlice(mtype reflect.Type) bool { + switch mtype.Kind() { + case reflect.Ptr: + return isOtherSlice(mtype.Elem()) + case reflect.Slice: + return isPrimitive(mtype.Elem()) || isOtherSlice(mtype.Elem()) + default: + return false + } +} + +// Check if the given marshall type maps to a Tree +func isTree(mtype reflect.Type) bool { + switch mtype.Kind() { + case reflect.Map: + return true + case reflect.Struct: + return !isPrimitive(mtype) + default: + return false + } +} + +func isCustomMarshaler(mtype reflect.Type) bool { + return mtype.Implements(marshalerType) +} + +func callCustomMarshaler(mval reflect.Value) ([]byte, error) { + return mval.Interface().(Marshaler).MarshalTOML() +} + +// Marshaler is the interface implemented by types that +// can marshal themselves into valid TOML. +type Marshaler interface { + MarshalTOML() ([]byte, error) +} + +/* +Marshal returns the TOML encoding of v. Behavior is similar to the Go json +encoder, except that there is no concept of a Marshaler interface or MarshalTOML +function for sub-structs, and currently only definite types can be marshaled +(i.e. no `interface{}`). + +The following struct annotations are supported: + + toml:"Field" Overrides the field's name to output. + omitempty When set, empty values and groups are not emitted. + comment:"comment" Emits a # comment on the same line. This supports new lines. + commented:"true" Emits the value as commented. + +Note that pointers are automatically assigned the "omitempty" option, as TOML +explicitly does not handle null values (saying instead the label should be +dropped). + +Tree structural types and corresponding marshal types: + + *Tree (*)struct, (*)map[string]interface{} + []*Tree (*)[](*)struct, (*)[](*)map[string]interface{} + []interface{} (as interface{}) (*)[]primitive, (*)[]([]interface{}) + interface{} (*)primitive + +Tree primitive types and corresponding marshal types: + + uint64 uint, uint8-uint64, pointers to same + int64 int, int8-uint64, pointers to same + float64 float32, float64, pointers to same + string string, pointers to same + bool bool, pointers to same + time.Time time.Time{}, pointers to same +*/ +func Marshal(v interface{}) ([]byte, error) { + return NewEncoder(nil).marshal(v) +} + +// Encoder writes TOML values to an output stream. +type Encoder struct { + w io.Writer + encOpts +} + +// NewEncoder returns a new encoder that writes to w. +func NewEncoder(w io.Writer) *Encoder { + return &Encoder{ + w: w, + encOpts: encOptsDefaults, + } +} + +// Encode writes the TOML encoding of v to the stream. +// +// See the documentation for Marshal for details. +func (e *Encoder) Encode(v interface{}) error { + b, err := e.marshal(v) + if err != nil { + return err + } + if _, err := e.w.Write(b); err != nil { + return err + } + return nil +} + +// QuoteMapKeys sets up the encoder to encode +// maps with string type keys with quoted TOML keys. +// +// This relieves the character limitations on map keys. +func (e *Encoder) QuoteMapKeys(v bool) *Encoder { + e.quoteMapKeys = v + return e +} + +// ArraysWithOneElementPerLine sets up the encoder to encode arrays +// with more than one element on multiple lines instead of one. +// +// For example: +// +// A = [1,2,3] +// +// Becomes +// +// A = [ +// 1, +// 2, +// 3, +// ] +func (e *Encoder) ArraysWithOneElementPerLine(v bool) *Encoder { + e.arraysOneElementPerLine = v + return e +} + +func (e *Encoder) marshal(v interface{}) ([]byte, error) { + mtype := reflect.TypeOf(v) + if mtype.Kind() != reflect.Struct { + return []byte{}, errors.New("Only a struct can be marshaled to TOML") + } + sval := reflect.ValueOf(v) + if isCustomMarshaler(mtype) { + return callCustomMarshaler(sval) + } + t, err := e.valueToTree(mtype, sval) + if err != nil { + return []byte{}, err + } + + var buf bytes.Buffer + _, err = t.writeTo(&buf, "", "", 0, e.arraysOneElementPerLine) + + return buf.Bytes(), err +} + +// Convert given marshal struct or map value to toml tree +func (e *Encoder) valueToTree(mtype reflect.Type, mval reflect.Value) (*Tree, error) { + if mtype.Kind() == reflect.Ptr { + return e.valueToTree(mtype.Elem(), mval.Elem()) + } + tval := newTree() + switch mtype.Kind() { + case reflect.Struct: + for i := 0; i < mtype.NumField(); i++ { + mtypef, mvalf := mtype.Field(i), mval.Field(i) + opts := tomlOptions(mtypef) + if opts.include && (!opts.omitempty || !isZero(mvalf)) { + val, err := e.valueToToml(mtypef.Type, mvalf) + if err != nil { + return nil, err + } + + tval.SetWithOptions(opts.name, SetOptions{ + Comment: opts.comment, + Commented: opts.commented, + Multiline: opts.multiline, + }, val) + } + } + case reflect.Map: + for _, key := range mval.MapKeys() { + mvalf := mval.MapIndex(key) + val, err := e.valueToToml(mtype.Elem(), mvalf) + if err != nil { + return nil, err + } + if e.quoteMapKeys { + keyStr, err := tomlValueStringRepresentation(key.String(), "", e.arraysOneElementPerLine) + if err != nil { + return nil, err + } + tval.SetPath([]string{keyStr}, val) + } else { + tval.Set(key.String(), val) + } + } + } + return tval, nil +} + +// Convert given marshal slice to slice of Toml trees +func (e *Encoder) valueToTreeSlice(mtype reflect.Type, mval reflect.Value) ([]*Tree, error) { + tval := make([]*Tree, mval.Len(), mval.Len()) + for i := 0; i < mval.Len(); i++ { + val, err := e.valueToTree(mtype.Elem(), mval.Index(i)) + if err != nil { + return nil, err + } + tval[i] = val + } + return tval, nil +} + +// Convert given marshal slice to slice of toml values +func (e *Encoder) valueToOtherSlice(mtype reflect.Type, mval reflect.Value) (interface{}, error) { + tval := make([]interface{}, mval.Len(), mval.Len()) + for i := 0; i < mval.Len(); i++ { + val, err := e.valueToToml(mtype.Elem(), mval.Index(i)) + if err != nil { + return nil, err + } + tval[i] = val + } + return tval, nil +} + +// Convert given marshal value to toml value +func (e *Encoder) valueToToml(mtype reflect.Type, mval reflect.Value) (interface{}, error) { + if mtype.Kind() == reflect.Ptr { + return e.valueToToml(mtype.Elem(), mval.Elem()) + } + switch { + case isCustomMarshaler(mtype): + return callCustomMarshaler(mval) + case isTree(mtype): + return e.valueToTree(mtype, mval) + case isTreeSlice(mtype): + return e.valueToTreeSlice(mtype, mval) + case isOtherSlice(mtype): + return e.valueToOtherSlice(mtype, mval) + default: + switch mtype.Kind() { + case reflect.Bool: + return mval.Bool(), nil + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return mval.Int(), nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return mval.Uint(), nil + case reflect.Float32, reflect.Float64: + return mval.Float(), nil + case reflect.String: + return mval.String(), nil + case reflect.Struct: + return mval.Interface().(time.Time), nil + default: + return nil, fmt.Errorf("Marshal can't handle %v(%v)", mtype, mtype.Kind()) + } + } +} + +// Unmarshal attempts to unmarshal the Tree into a Go struct pointed by v. +// Neither Unmarshaler interfaces nor UnmarshalTOML functions are supported for +// sub-structs, and only definite types can be unmarshaled. +func (t *Tree) Unmarshal(v interface{}) error { + d := Decoder{tval: t} + return d.unmarshal(v) +} + +// Marshal returns the TOML encoding of Tree. +// See Marshal() documentation for types mapping table. +func (t *Tree) Marshal() ([]byte, error) { + var buf bytes.Buffer + err := NewEncoder(&buf).Encode(t) + return buf.Bytes(), err +} + +// Unmarshal parses the TOML-encoded data and stores the result in the value +// pointed to by v. Behavior is similar to the Go json encoder, except that there +// is no concept of an Unmarshaler interface or UnmarshalTOML function for +// sub-structs, and currently only definite types can be unmarshaled to (i.e. no +// `interface{}`). +// +// The following struct annotations are supported: +// +// toml:"Field" Overrides the field's name to map to. +// +// See Marshal() documentation for types mapping table. +func Unmarshal(data []byte, v interface{}) error { + t, err := LoadReader(bytes.NewReader(data)) + if err != nil { + return err + } + return t.Unmarshal(v) +} + +// Decoder reads and decodes TOML values from an input stream. +type Decoder struct { + r io.Reader + tval *Tree + encOpts +} + +// NewDecoder returns a new decoder that reads from r. +func NewDecoder(r io.Reader) *Decoder { + return &Decoder{ + r: r, + encOpts: encOptsDefaults, + } +} + +// Decode reads a TOML-encoded value from it's input +// and unmarshals it in the value pointed at by v. +// +// See the documentation for Marshal for details. +func (d *Decoder) Decode(v interface{}) error { + var err error + d.tval, err = LoadReader(d.r) + if err != nil { + return err + } + return d.unmarshal(v) +} + +func (d *Decoder) unmarshal(v interface{}) error { + mtype := reflect.TypeOf(v) + if mtype.Kind() != reflect.Ptr || mtype.Elem().Kind() != reflect.Struct { + return errors.New("Only a pointer to struct can be unmarshaled from TOML") + } + + sval, err := d.valueFromTree(mtype.Elem(), d.tval) + if err != nil { + return err + } + reflect.ValueOf(v).Elem().Set(sval) + return nil +} + +// Convert toml tree to marshal struct or map, using marshal type +func (d *Decoder) valueFromTree(mtype reflect.Type, tval *Tree) (reflect.Value, error) { + if mtype.Kind() == reflect.Ptr { + return d.unwrapPointer(mtype, tval) + } + var mval reflect.Value + switch mtype.Kind() { + case reflect.Struct: + mval = reflect.New(mtype).Elem() + for i := 0; i < mtype.NumField(); i++ { + mtypef := mtype.Field(i) + opts := tomlOptions(mtypef) + if opts.include { + baseKey := opts.name + keysToTry := []string{baseKey, strings.ToLower(baseKey), strings.ToTitle(baseKey)} + for _, key := range keysToTry { + exists := tval.Has(key) + if !exists { + continue + } + val := tval.Get(key) + mvalf, err := d.valueFromToml(mtypef.Type, val) + if err != nil { + return mval, formatError(err, tval.GetPosition(key)) + } + mval.Field(i).Set(mvalf) + break + } + } + } + case reflect.Map: + mval = reflect.MakeMap(mtype) + for _, key := range tval.Keys() { + // TODO: path splits key + val := tval.GetPath([]string{key}) + mvalf, err := d.valueFromToml(mtype.Elem(), val) + if err != nil { + return mval, formatError(err, tval.GetPosition(key)) + } + mval.SetMapIndex(reflect.ValueOf(key), mvalf) + } + } + return mval, nil +} + +// Convert toml value to marshal struct/map slice, using marshal type +func (d *Decoder) valueFromTreeSlice(mtype reflect.Type, tval []*Tree) (reflect.Value, error) { + mval := reflect.MakeSlice(mtype, len(tval), len(tval)) + for i := 0; i < len(tval); i++ { + val, err := d.valueFromTree(mtype.Elem(), tval[i]) + if err != nil { + return mval, err + } + mval.Index(i).Set(val) + } + return mval, nil +} + +// Convert toml value to marshal primitive slice, using marshal type +func (d *Decoder) valueFromOtherSlice(mtype reflect.Type, tval []interface{}) (reflect.Value, error) { + mval := reflect.MakeSlice(mtype, len(tval), len(tval)) + for i := 0; i < len(tval); i++ { + val, err := d.valueFromToml(mtype.Elem(), tval[i]) + if err != nil { + return mval, err + } + mval.Index(i).Set(val) + } + return mval, nil +} + +// Convert toml value to marshal value, using marshal type +func (d *Decoder) valueFromToml(mtype reflect.Type, tval interface{}) (reflect.Value, error) { + if mtype.Kind() == reflect.Ptr { + return d.unwrapPointer(mtype, tval) + } + + switch tval.(type) { + case *Tree: + if isTree(mtype) { + return d.valueFromTree(mtype, tval.(*Tree)) + } + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to a tree", tval, tval) + case []*Tree: + if isTreeSlice(mtype) { + return d.valueFromTreeSlice(mtype, tval.([]*Tree)) + } + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to trees", tval, tval) + case []interface{}: + if isOtherSlice(mtype) { + return d.valueFromOtherSlice(mtype, tval.([]interface{})) + } + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to a slice", tval, tval) + default: + switch mtype.Kind() { + case reflect.Bool, reflect.Struct: + val := reflect.ValueOf(tval) + // if this passes for when mtype is reflect.Struct, tval is a time.Time + if !val.Type().ConvertibleTo(mtype) { + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to %v", tval, tval, mtype.String()) + } + + return val.Convert(mtype), nil + case reflect.String: + val := reflect.ValueOf(tval) + // stupidly, int64 is convertible to string. So special case this. + if !val.Type().ConvertibleTo(mtype) || val.Kind() == reflect.Int64 { + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to %v", tval, tval, mtype.String()) + } + + return val.Convert(mtype), nil + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + val := reflect.ValueOf(tval) + if !val.Type().ConvertibleTo(mtype) { + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to %v", tval, tval, mtype.String()) + } + if reflect.Indirect(reflect.New(mtype)).OverflowInt(val.Int()) { + return reflect.ValueOf(nil), fmt.Errorf("%v(%T) would overflow %v", tval, tval, mtype.String()) + } + + return val.Convert(mtype), nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + val := reflect.ValueOf(tval) + if !val.Type().ConvertibleTo(mtype) { + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to %v", tval, tval, mtype.String()) + } + if val.Int() < 0 { + return reflect.ValueOf(nil), fmt.Errorf("%v(%T) is negative so does not fit in %v", tval, tval, mtype.String()) + } + if reflect.Indirect(reflect.New(mtype)).OverflowUint(uint64(val.Int())) { + return reflect.ValueOf(nil), fmt.Errorf("%v(%T) would overflow %v", tval, tval, mtype.String()) + } + + return val.Convert(mtype), nil + case reflect.Float32, reflect.Float64: + val := reflect.ValueOf(tval) + if !val.Type().ConvertibleTo(mtype) { + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to %v", tval, tval, mtype.String()) + } + if reflect.Indirect(reflect.New(mtype)).OverflowFloat(val.Float()) { + return reflect.ValueOf(nil), fmt.Errorf("%v(%T) would overflow %v", tval, tval, mtype.String()) + } + + return val.Convert(mtype), nil + default: + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to %v(%v)", tval, tval, mtype, mtype.Kind()) + } + } +} + +func (d *Decoder) unwrapPointer(mtype reflect.Type, tval interface{}) (reflect.Value, error) { + val, err := d.valueFromToml(mtype.Elem(), tval) + if err != nil { + return reflect.ValueOf(nil), err + } + mval := reflect.New(mtype.Elem()) + mval.Elem().Set(val) + return mval, nil +} + +func tomlOptions(vf reflect.StructField) tomlOpts { + tag := vf.Tag.Get("toml") + parse := strings.Split(tag, ",") + var comment string + if c := vf.Tag.Get("comment"); c != "" { + comment = c + } + commented, _ := strconv.ParseBool(vf.Tag.Get("commented")) + multiline, _ := strconv.ParseBool(vf.Tag.Get(tagKeyMultiline)) + result := tomlOpts{name: vf.Name, comment: comment, commented: commented, multiline: multiline, include: true, omitempty: false} + if parse[0] != "" { + if parse[0] == "-" && len(parse) == 1 { + result.include = false + } else { + result.name = strings.Trim(parse[0], " ") + } + } + if vf.PkgPath != "" { + result.include = false + } + if len(parse) > 1 && strings.Trim(parse[1], " ") == "omitempty" { + result.omitempty = true + } + if vf.Type.Kind() == reflect.Ptr { + result.omitempty = true + } + return result +} + +func isZero(val reflect.Value) bool { + switch val.Type().Kind() { + case reflect.Map: + fallthrough + case reflect.Array: + fallthrough + case reflect.Slice: + return val.Len() == 0 + default: + return reflect.DeepEqual(val.Interface(), reflect.Zero(val.Type()).Interface()) + } +} + +func formatError(err error, pos Position) error { + if err.Error()[0] == '(' { // Error already contains position information + return err + } + return fmt.Errorf("%s: %s", pos, err) +} diff --git a/vendor/github.com/pelletier/go-toml/marshal_test.toml b/vendor/github.com/pelletier/go-toml/marshal_test.toml new file mode 100644 index 000000000..1c5f98e7a --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/marshal_test.toml @@ -0,0 +1,38 @@ +title = "TOML Marshal Testing" + +[basic] + bool = true + date = 1979-05-27T07:32:00Z + float = 123.4 + int = 5000 + string = "Bite me" + uint = 5001 + +[basic_lists] + bools = [true,false,true] + dates = [1979-05-27T07:32:00Z,1980-05-27T07:32:00Z] + floats = [12.3,45.6,78.9] + ints = [8001,8001,8002] + strings = ["One","Two","Three"] + uints = [5002,5003] + +[basic_map] + one = "one" + two = "two" + +[subdoc] + + [subdoc.first] + name = "First" + + [subdoc.second] + name = "Second" + +[[subdoclist]] + name = "List.First" + +[[subdoclist]] + name = "List.Second" + +[[subdocptrs]] + name = "Second" diff --git a/vendor/github.com/pelletier/go-toml/parser.go b/vendor/github.com/pelletier/go-toml/parser.go new file mode 100644 index 000000000..2d27599a9 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/parser.go @@ -0,0 +1,430 @@ +// TOML Parser. + +package toml + +import ( + "errors" + "fmt" + "math" + "reflect" + "regexp" + "strconv" + "strings" + "time" +) + +type tomlParser struct { + flowIdx int + flow []token + tree *Tree + currentTable []string + seenTableKeys []string +} + +type tomlParserStateFn func() tomlParserStateFn + +// Formats and panics an error message based on a token +func (p *tomlParser) raiseError(tok *token, msg string, args ...interface{}) { + panic(tok.Position.String() + ": " + fmt.Sprintf(msg, args...)) +} + +func (p *tomlParser) run() { + for state := p.parseStart; state != nil; { + state = state() + } +} + +func (p *tomlParser) peek() *token { + if p.flowIdx >= len(p.flow) { + return nil + } + return &p.flow[p.flowIdx] +} + +func (p *tomlParser) assume(typ tokenType) { + tok := p.getToken() + if tok == nil { + p.raiseError(tok, "was expecting token %s, but token stream is empty", tok) + } + if tok.typ != typ { + p.raiseError(tok, "was expecting token %s, but got %s instead", typ, tok) + } +} + +func (p *tomlParser) getToken() *token { + tok := p.peek() + if tok == nil { + return nil + } + p.flowIdx++ + return tok +} + +func (p *tomlParser) parseStart() tomlParserStateFn { + tok := p.peek() + + // end of stream, parsing is finished + if tok == nil { + return nil + } + + switch tok.typ { + case tokenDoubleLeftBracket: + return p.parseGroupArray + case tokenLeftBracket: + return p.parseGroup + case tokenKey: + return p.parseAssign + case tokenEOF: + return nil + default: + p.raiseError(tok, "unexpected token") + } + return nil +} + +func (p *tomlParser) parseGroupArray() tomlParserStateFn { + startToken := p.getToken() // discard the [[ + key := p.getToken() + if key.typ != tokenKeyGroupArray { + p.raiseError(key, "unexpected token %s, was expecting a table array key", key) + } + + // get or create table array element at the indicated part in the path + keys, err := parseKey(key.val) + if err != nil { + p.raiseError(key, "invalid table array key: %s", err) + } + p.tree.createSubTree(keys[:len(keys)-1], startToken.Position) // create parent entries + destTree := p.tree.GetPath(keys) + var array []*Tree + if destTree == nil { + array = make([]*Tree, 0) + } else if target, ok := destTree.([]*Tree); ok && target != nil { + array = destTree.([]*Tree) + } else { + p.raiseError(key, "key %s is already assigned and not of type table array", key) + } + p.currentTable = keys + + // add a new tree to the end of the table array + newTree := newTree() + newTree.position = startToken.Position + array = append(array, newTree) + p.tree.SetPath(p.currentTable, array) + + // remove all keys that were children of this table array + prefix := key.val + "." + found := false + for ii := 0; ii < len(p.seenTableKeys); { + tableKey := p.seenTableKeys[ii] + if strings.HasPrefix(tableKey, prefix) { + p.seenTableKeys = append(p.seenTableKeys[:ii], p.seenTableKeys[ii+1:]...) + } else { + found = (tableKey == key.val) + ii++ + } + } + + // keep this key name from use by other kinds of assignments + if !found { + p.seenTableKeys = append(p.seenTableKeys, key.val) + } + + // move to next parser state + p.assume(tokenDoubleRightBracket) + return p.parseStart +} + +func (p *tomlParser) parseGroup() tomlParserStateFn { + startToken := p.getToken() // discard the [ + key := p.getToken() + if key.typ != tokenKeyGroup { + p.raiseError(key, "unexpected token %s, was expecting a table key", key) + } + for _, item := range p.seenTableKeys { + if item == key.val { + p.raiseError(key, "duplicated tables") + } + } + + p.seenTableKeys = append(p.seenTableKeys, key.val) + keys, err := parseKey(key.val) + if err != nil { + p.raiseError(key, "invalid table array key: %s", err) + } + if err := p.tree.createSubTree(keys, startToken.Position); err != nil { + p.raiseError(key, "%s", err) + } + p.assume(tokenRightBracket) + p.currentTable = keys + return p.parseStart +} + +func (p *tomlParser) parseAssign() tomlParserStateFn { + key := p.getToken() + p.assume(tokenEqual) + + value := p.parseRvalue() + var tableKey []string + if len(p.currentTable) > 0 { + tableKey = p.currentTable + } else { + tableKey = []string{} + } + + // find the table to assign, looking out for arrays of tables + var targetNode *Tree + switch node := p.tree.GetPath(tableKey).(type) { + case []*Tree: + targetNode = node[len(node)-1] + case *Tree: + targetNode = node + default: + p.raiseError(key, "Unknown table type for path: %s", + strings.Join(tableKey, ".")) + } + + // assign value to the found table + keyVals := []string{key.val} + if len(keyVals) != 1 { + p.raiseError(key, "Invalid key") + } + keyVal := keyVals[0] + localKey := []string{keyVal} + finalKey := append(tableKey, keyVal) + if targetNode.GetPath(localKey) != nil { + p.raiseError(key, "The following key was defined twice: %s", + strings.Join(finalKey, ".")) + } + var toInsert interface{} + + switch value.(type) { + case *Tree, []*Tree: + toInsert = value + default: + toInsert = &tomlValue{value: value, position: key.Position} + } + targetNode.values[keyVal] = toInsert + return p.parseStart +} + +var numberUnderscoreInvalidRegexp *regexp.Regexp +var hexNumberUnderscoreInvalidRegexp *regexp.Regexp + +func numberContainsInvalidUnderscore(value string) error { + if numberUnderscoreInvalidRegexp.MatchString(value) { + return errors.New("invalid use of _ in number") + } + return nil +} + +func hexNumberContainsInvalidUnderscore(value string) error { + if hexNumberUnderscoreInvalidRegexp.MatchString(value) { + return errors.New("invalid use of _ in hex number") + } + return nil +} + +func cleanupNumberToken(value string) string { + cleanedVal := strings.Replace(value, "_", "", -1) + return cleanedVal +} + +func (p *tomlParser) parseRvalue() interface{} { + tok := p.getToken() + if tok == nil || tok.typ == tokenEOF { + p.raiseError(tok, "expecting a value") + } + + switch tok.typ { + case tokenString: + return tok.val + case tokenTrue: + return true + case tokenFalse: + return false + case tokenInf: + if tok.val[0] == '-' { + return math.Inf(-1) + } + return math.Inf(1) + case tokenNan: + return math.NaN() + case tokenInteger: + cleanedVal := cleanupNumberToken(tok.val) + var err error + var val int64 + if len(cleanedVal) >= 3 && cleanedVal[0] == '0' { + switch cleanedVal[1] { + case 'x': + err = hexNumberContainsInvalidUnderscore(tok.val) + if err != nil { + p.raiseError(tok, "%s", err) + } + val, err = strconv.ParseInt(cleanedVal[2:], 16, 64) + case 'o': + err = numberContainsInvalidUnderscore(tok.val) + if err != nil { + p.raiseError(tok, "%s", err) + } + val, err = strconv.ParseInt(cleanedVal[2:], 8, 64) + case 'b': + err = numberContainsInvalidUnderscore(tok.val) + if err != nil { + p.raiseError(tok, "%s", err) + } + val, err = strconv.ParseInt(cleanedVal[2:], 2, 64) + default: + panic("invalid base") // the lexer should catch this first + } + } else { + err = numberContainsInvalidUnderscore(tok.val) + if err != nil { + p.raiseError(tok, "%s", err) + } + val, err = strconv.ParseInt(cleanedVal, 10, 64) + } + if err != nil { + p.raiseError(tok, "%s", err) + } + return val + case tokenFloat: + err := numberContainsInvalidUnderscore(tok.val) + if err != nil { + p.raiseError(tok, "%s", err) + } + cleanedVal := cleanupNumberToken(tok.val) + val, err := strconv.ParseFloat(cleanedVal, 64) + if err != nil { + p.raiseError(tok, "%s", err) + } + return val + case tokenDate: + val, err := time.ParseInLocation(time.RFC3339Nano, tok.val, time.UTC) + if err != nil { + p.raiseError(tok, "%s", err) + } + return val + case tokenLeftBracket: + return p.parseArray() + case tokenLeftCurlyBrace: + return p.parseInlineTable() + case tokenEqual: + p.raiseError(tok, "cannot have multiple equals for the same key") + case tokenError: + p.raiseError(tok, "%s", tok) + } + + p.raiseError(tok, "never reached") + + return nil +} + +func tokenIsComma(t *token) bool { + return t != nil && t.typ == tokenComma +} + +func (p *tomlParser) parseInlineTable() *Tree { + tree := newTree() + var previous *token +Loop: + for { + follow := p.peek() + if follow == nil || follow.typ == tokenEOF { + p.raiseError(follow, "unterminated inline table") + } + switch follow.typ { + case tokenRightCurlyBrace: + p.getToken() + break Loop + case tokenKey: + if !tokenIsComma(previous) && previous != nil { + p.raiseError(follow, "comma expected between fields in inline table") + } + key := p.getToken() + p.assume(tokenEqual) + value := p.parseRvalue() + tree.Set(key.val, value) + case tokenComma: + if previous == nil { + p.raiseError(follow, "inline table cannot start with a comma") + } + if tokenIsComma(previous) { + p.raiseError(follow, "need field between two commas in inline table") + } + p.getToken() + default: + p.raiseError(follow, "unexpected token type in inline table: %s", follow.String()) + } + previous = follow + } + if tokenIsComma(previous) { + p.raiseError(previous, "trailing comma at the end of inline table") + } + return tree +} + +func (p *tomlParser) parseArray() interface{} { + var array []interface{} + arrayType := reflect.TypeOf(nil) + for { + follow := p.peek() + if follow == nil || follow.typ == tokenEOF { + p.raiseError(follow, "unterminated array") + } + if follow.typ == tokenRightBracket { + p.getToken() + break + } + val := p.parseRvalue() + if arrayType == nil { + arrayType = reflect.TypeOf(val) + } + if reflect.TypeOf(val) != arrayType { + p.raiseError(follow, "mixed types in array") + } + array = append(array, val) + follow = p.peek() + if follow == nil || follow.typ == tokenEOF { + p.raiseError(follow, "unterminated array") + } + if follow.typ != tokenRightBracket && follow.typ != tokenComma { + p.raiseError(follow, "missing comma") + } + if follow.typ == tokenComma { + p.getToken() + } + } + // An array of Trees is actually an array of inline + // tables, which is a shorthand for a table array. If the + // array was not converted from []interface{} to []*Tree, + // the two notations would not be equivalent. + if arrayType == reflect.TypeOf(newTree()) { + tomlArray := make([]*Tree, len(array)) + for i, v := range array { + tomlArray[i] = v.(*Tree) + } + return tomlArray + } + return array +} + +func parseToml(flow []token) *Tree { + result := newTree() + result.position = Position{1, 1} + parser := &tomlParser{ + flowIdx: 0, + flow: flow, + tree: result, + currentTable: make([]string, 0), + seenTableKeys: make([]string, 0), + } + parser.run() + return result +} + +func init() { + numberUnderscoreInvalidRegexp = regexp.MustCompile(`([^\d]_|_[^\d])|_$|^_`) + hexNumberUnderscoreInvalidRegexp = regexp.MustCompile(`(^0x_)|([^\da-f]_|_[^\da-f])|_$|^_`) +} diff --git a/vendor/github.com/pelletier/go-toml/position.go b/vendor/github.com/pelletier/go-toml/position.go new file mode 100644 index 000000000..c17bff87b --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/position.go @@ -0,0 +1,29 @@ +// Position support for go-toml + +package toml + +import ( + "fmt" +) + +// Position of a document element within a TOML document. +// +// Line and Col are both 1-indexed positions for the element's line number and +// column number, respectively. Values of zero or less will cause Invalid(), +// to return true. +type Position struct { + Line int // line within the document + Col int // column within the line +} + +// String representation of the position. +// Displays 1-indexed line and column numbers. +func (p Position) String() string { + return fmt.Sprintf("(%d, %d)", p.Line, p.Col) +} + +// Invalid returns whether or not the position is valid (i.e. with negative or +// null values) +func (p Position) Invalid() bool { + return p.Line <= 0 || p.Col <= 0 +} diff --git a/vendor/github.com/pelletier/go-toml/test.sh b/vendor/github.com/pelletier/go-toml/test.sh new file mode 100644 index 000000000..ba6adf3fc --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/test.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# fail out of the script if anything here fails +set -e +set -o pipefail + +# set the path to the present working directory +export GOPATH=`pwd` + +function git_clone() { + path=$1 + branch=$2 + version=$3 + if [ ! -d "src/$path" ]; then + mkdir -p src/$path + git clone https://$path.git src/$path + fi + pushd src/$path + git checkout "$branch" + git reset --hard "$version" + popd +} + +# Remove potential previous runs +rm -rf src test_program_bin toml-test + +go get github.com/pelletier/go-buffruneio +go get github.com/davecgh/go-spew/spew +go get gopkg.in/yaml.v2 +go get github.com/BurntSushi/toml + +# get code for BurntSushi TOML validation +# pinning all to 'HEAD' for version 0.3.x work (TODO: pin to commit hash when tests stabilize) +git_clone github.com/BurntSushi/toml master HEAD +git_clone github.com/BurntSushi/toml-test master HEAD #was: 0.2.0 HEAD + +# build the BurntSushi test application +go build -o toml-test github.com/BurntSushi/toml-test + +# vendorize the current lib for testing +# NOTE: this basically mocks an install without having to go back out to github for code +mkdir -p src/github.com/pelletier/go-toml/cmd +mkdir -p src/github.com/pelletier/go-toml/query +cp *.go *.toml src/github.com/pelletier/go-toml +cp -R cmd/* src/github.com/pelletier/go-toml/cmd +cp -R query/* src/github.com/pelletier/go-toml/query +go build -o test_program_bin src/github.com/pelletier/go-toml/cmd/test_program.go + +# Run basic unit tests +go test github.com/pelletier/go-toml -covermode=count -coverprofile=coverage.out +go test github.com/pelletier/go-toml/cmd/tomljson +go test github.com/pelletier/go-toml/query + +# run the entire BurntSushi test suite +if [[ $# -eq 0 ]] ; then + echo "Running all BurntSushi tests" + ./toml-test ./test_program_bin | tee test_out +else + # run a specific test + test=$1 + test_path='src/github.com/BurntSushi/toml-test/tests' + valid_test="$test_path/valid/$test" + invalid_test="$test_path/invalid/$test" + + if [ -e "$valid_test.toml" ]; then + echo "Valid Test TOML for $test:" + echo "====" + cat "$valid_test.toml" + + echo "Valid Test JSON for $test:" + echo "====" + cat "$valid_test.json" + + echo "Go-TOML Output for $test:" + echo "====" + cat "$valid_test.toml" | ./test_program_bin + fi + + if [ -e "$invalid_test.toml" ]; then + echo "Invalid Test TOML for $test:" + echo "====" + cat "$invalid_test.toml" + + echo "Go-TOML Output for $test:" + echo "====" + echo "go-toml Output:" + cat "$invalid_test.toml" | ./test_program_bin + fi +fi diff --git a/vendor/github.com/pelletier/go-toml/token.go b/vendor/github.com/pelletier/go-toml/token.go new file mode 100644 index 000000000..1a9081346 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/token.go @@ -0,0 +1,144 @@ +package toml + +import ( + "fmt" + "strconv" + "unicode" +) + +// Define tokens +type tokenType int + +const ( + eof = -(iota + 1) +) + +const ( + tokenError tokenType = iota + tokenEOF + tokenComment + tokenKey + tokenString + tokenInteger + tokenTrue + tokenFalse + tokenFloat + tokenInf + tokenNan + tokenEqual + tokenLeftBracket + tokenRightBracket + tokenLeftCurlyBrace + tokenRightCurlyBrace + tokenLeftParen + tokenRightParen + tokenDoubleLeftBracket + tokenDoubleRightBracket + tokenDate + tokenKeyGroup + tokenKeyGroupArray + tokenComma + tokenColon + tokenDollar + tokenStar + tokenQuestion + tokenDot + tokenDotDot + tokenEOL +) + +var tokenTypeNames = []string{ + "Error", + "EOF", + "Comment", + "Key", + "String", + "Integer", + "True", + "False", + "Float", + "Inf", + "NaN", + "=", + "[", + "]", + "{", + "}", + "(", + ")", + "]]", + "[[", + "Date", + "KeyGroup", + "KeyGroupArray", + ",", + ":", + "$", + "*", + "?", + ".", + "..", + "EOL", +} + +type token struct { + Position + typ tokenType + val string +} + +func (tt tokenType) String() string { + idx := int(tt) + if idx < len(tokenTypeNames) { + return tokenTypeNames[idx] + } + return "Unknown" +} + +func (t token) Int() int { + if result, err := strconv.Atoi(t.val); err != nil { + panic(err) + } else { + return result + } +} + +func (t token) String() string { + switch t.typ { + case tokenEOF: + return "EOF" + case tokenError: + return t.val + } + + return fmt.Sprintf("%q", t.val) +} + +func isSpace(r rune) bool { + return r == ' ' || r == '\t' +} + +func isAlphanumeric(r rune) bool { + return unicode.IsLetter(r) || r == '_' +} + +func isKeyChar(r rune) bool { + // Keys start with the first character that isn't whitespace or [ and end + // with the last non-whitespace character before the equals sign. Keys + // cannot contain a # character." + return !(r == '\r' || r == '\n' || r == eof || r == '=') +} + +func isKeyStartChar(r rune) bool { + return !(isSpace(r) || r == '\r' || r == '\n' || r == eof || r == '[') +} + +func isDigit(r rune) bool { + return unicode.IsNumber(r) +} + +func isHexDigit(r rune) bool { + return isDigit(r) || + (r >= 'a' && r <= 'f') || + (r >= 'A' && r <= 'F') +} diff --git a/vendor/github.com/pelletier/go-toml/toml.go b/vendor/github.com/pelletier/go-toml/toml.go new file mode 100644 index 000000000..98c185ad0 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/toml.go @@ -0,0 +1,367 @@ +package toml + +import ( + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "runtime" + "strings" +) + +type tomlValue struct { + value interface{} // string, int64, uint64, float64, bool, time.Time, [] of any of this list + comment string + commented bool + multiline bool + position Position +} + +// Tree is the result of the parsing of a TOML file. +type Tree struct { + values map[string]interface{} // string -> *tomlValue, *Tree, []*Tree + comment string + commented bool + position Position +} + +func newTree() *Tree { + return &Tree{ + values: make(map[string]interface{}), + position: Position{}, + } +} + +// TreeFromMap initializes a new Tree object using the given map. +func TreeFromMap(m map[string]interface{}) (*Tree, error) { + result, err := toTree(m) + if err != nil { + return nil, err + } + return result.(*Tree), nil +} + +// Position returns the position of the tree. +func (t *Tree) Position() Position { + return t.position +} + +// Has returns a boolean indicating if the given key exists. +func (t *Tree) Has(key string) bool { + if key == "" { + return false + } + return t.HasPath(strings.Split(key, ".")) +} + +// HasPath returns true if the given path of keys exists, false otherwise. +func (t *Tree) HasPath(keys []string) bool { + return t.GetPath(keys) != nil +} + +// Keys returns the keys of the toplevel tree (does not recurse). +func (t *Tree) Keys() []string { + keys := make([]string, len(t.values)) + i := 0 + for k := range t.values { + keys[i] = k + i++ + } + return keys +} + +// Get the value at key in the Tree. +// Key is a dot-separated path (e.g. a.b.c) without single/double quoted strings. +// If you need to retrieve non-bare keys, use GetPath. +// Returns nil if the path does not exist in the tree. +// If keys is of length zero, the current tree is returned. +func (t *Tree) Get(key string) interface{} { + if key == "" { + return t + } + return t.GetPath(strings.Split(key, ".")) +} + +// GetPath returns the element in the tree indicated by 'keys'. +// If keys is of length zero, the current tree is returned. +func (t *Tree) GetPath(keys []string) interface{} { + if len(keys) == 0 { + return t + } + subtree := t + for _, intermediateKey := range keys[:len(keys)-1] { + value, exists := subtree.values[intermediateKey] + if !exists { + return nil + } + switch node := value.(type) { + case *Tree: + subtree = node + case []*Tree: + // go to most recent element + if len(node) == 0 { + return nil + } + subtree = node[len(node)-1] + default: + return nil // cannot navigate through other node types + } + } + // branch based on final node type + switch node := subtree.values[keys[len(keys)-1]].(type) { + case *tomlValue: + return node.value + default: + return node + } +} + +// GetPosition returns the position of the given key. +func (t *Tree) GetPosition(key string) Position { + if key == "" { + return t.position + } + return t.GetPositionPath(strings.Split(key, ".")) +} + +// GetPositionPath returns the element in the tree indicated by 'keys'. +// If keys is of length zero, the current tree is returned. +func (t *Tree) GetPositionPath(keys []string) Position { + if len(keys) == 0 { + return t.position + } + subtree := t + for _, intermediateKey := range keys[:len(keys)-1] { + value, exists := subtree.values[intermediateKey] + if !exists { + return Position{0, 0} + } + switch node := value.(type) { + case *Tree: + subtree = node + case []*Tree: + // go to most recent element + if len(node) == 0 { + return Position{0, 0} + } + subtree = node[len(node)-1] + default: + return Position{0, 0} + } + } + // branch based on final node type + switch node := subtree.values[keys[len(keys)-1]].(type) { + case *tomlValue: + return node.position + case *Tree: + return node.position + case []*Tree: + // go to most recent element + if len(node) == 0 { + return Position{0, 0} + } + return node[len(node)-1].position + default: + return Position{0, 0} + } +} + +// GetDefault works like Get but with a default value +func (t *Tree) GetDefault(key string, def interface{}) interface{} { + val := t.Get(key) + if val == nil { + return def + } + return val +} + +// SetOptions arguments are supplied to the SetWithOptions and SetPathWithOptions functions to modify marshalling behaviour. +// The default values within the struct are valid default options. +type SetOptions struct { + Comment string + Commented bool + Multiline bool +} + +// SetWithOptions is the same as Set, but allows you to provide formatting +// instructions to the key, that will be used by Marshal(). +func (t *Tree) SetWithOptions(key string, opts SetOptions, value interface{}) { + t.SetPathWithOptions(strings.Split(key, "."), opts, value) +} + +// SetPathWithOptions is the same as SetPath, but allows you to provide +// formatting instructions to the key, that will be reused by Marshal(). +func (t *Tree) SetPathWithOptions(keys []string, opts SetOptions, value interface{}) { + subtree := t + for _, intermediateKey := range keys[:len(keys)-1] { + nextTree, exists := subtree.values[intermediateKey] + if !exists { + nextTree = newTree() + subtree.values[intermediateKey] = nextTree // add new element here + } + switch node := nextTree.(type) { + case *Tree: + subtree = node + case []*Tree: + // go to most recent element + if len(node) == 0 { + // create element if it does not exist + subtree.values[intermediateKey] = append(node, newTree()) + } + subtree = node[len(node)-1] + } + } + + var toInsert interface{} + + switch value.(type) { + case *Tree: + tt := value.(*Tree) + tt.comment = opts.Comment + toInsert = value + case []*Tree: + toInsert = value + case *tomlValue: + tt := value.(*tomlValue) + tt.comment = opts.Comment + toInsert = tt + default: + toInsert = &tomlValue{value: value, comment: opts.Comment, commented: opts.Commented, multiline: opts.Multiline} + } + + subtree.values[keys[len(keys)-1]] = toInsert +} + +// Set an element in the tree. +// Key is a dot-separated path (e.g. a.b.c). +// Creates all necessary intermediate trees, if needed. +func (t *Tree) Set(key string, value interface{}) { + t.SetWithComment(key, "", false, value) +} + +// SetWithComment is the same as Set, but allows you to provide comment +// information to the key, that will be reused by Marshal(). +func (t *Tree) SetWithComment(key string, comment string, commented bool, value interface{}) { + t.SetPathWithComment(strings.Split(key, "."), comment, commented, value) +} + +// SetPath sets an element in the tree. +// Keys is an array of path elements (e.g. {"a","b","c"}). +// Creates all necessary intermediate trees, if needed. +func (t *Tree) SetPath(keys []string, value interface{}) { + t.SetPathWithComment(keys, "", false, value) +} + +// SetPathWithComment is the same as SetPath, but allows you to provide comment +// information to the key, that will be reused by Marshal(). +func (t *Tree) SetPathWithComment(keys []string, comment string, commented bool, value interface{}) { + subtree := t + for _, intermediateKey := range keys[:len(keys)-1] { + nextTree, exists := subtree.values[intermediateKey] + if !exists { + nextTree = newTree() + subtree.values[intermediateKey] = nextTree // add new element here + } + switch node := nextTree.(type) { + case *Tree: + subtree = node + case []*Tree: + // go to most recent element + if len(node) == 0 { + // create element if it does not exist + subtree.values[intermediateKey] = append(node, newTree()) + } + subtree = node[len(node)-1] + } + } + + var toInsert interface{} + + switch value.(type) { + case *Tree: + tt := value.(*Tree) + tt.comment = comment + toInsert = value + case []*Tree: + toInsert = value + case *tomlValue: + tt := value.(*tomlValue) + tt.comment = comment + toInsert = tt + default: + toInsert = &tomlValue{value: value, comment: comment, commented: commented} + } + + subtree.values[keys[len(keys)-1]] = toInsert +} + +// createSubTree takes a tree and a key and create the necessary intermediate +// subtrees to create a subtree at that point. In-place. +// +// e.g. passing a.b.c will create (assuming tree is empty) tree[a], tree[a][b] +// and tree[a][b][c] +// +// Returns nil on success, error object on failure +func (t *Tree) createSubTree(keys []string, pos Position) error { + subtree := t + for _, intermediateKey := range keys { + nextTree, exists := subtree.values[intermediateKey] + if !exists { + tree := newTree() + tree.position = pos + subtree.values[intermediateKey] = tree + nextTree = tree + } + + switch node := nextTree.(type) { + case []*Tree: + subtree = node[len(node)-1] + case *Tree: + subtree = node + default: + return fmt.Errorf("unknown type for path %s (%s): %T (%#v)", + strings.Join(keys, "."), intermediateKey, nextTree, nextTree) + } + } + return nil +} + +// LoadBytes creates a Tree from a []byte. +func LoadBytes(b []byte) (tree *Tree, err error) { + defer func() { + if r := recover(); r != nil { + if _, ok := r.(runtime.Error); ok { + panic(r) + } + err = errors.New(r.(string)) + } + }() + tree = parseToml(lexToml(b)) + return +} + +// LoadReader creates a Tree from any io.Reader. +func LoadReader(reader io.Reader) (tree *Tree, err error) { + inputBytes, err := ioutil.ReadAll(reader) + if err != nil { + return + } + tree, err = LoadBytes(inputBytes) + return +} + +// Load creates a Tree from a string. +func Load(content string) (tree *Tree, err error) { + return LoadBytes([]byte(content)) +} + +// LoadFile creates a Tree from a file. +func LoadFile(path string) (tree *Tree, err error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer file.Close() + return LoadReader(file) +} diff --git a/vendor/github.com/pelletier/go-toml/tomltree_create.go b/vendor/github.com/pelletier/go-toml/tomltree_create.go new file mode 100644 index 000000000..79610e9b3 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/tomltree_create.go @@ -0,0 +1,142 @@ +package toml + +import ( + "fmt" + "reflect" + "time" +) + +var kindToType = [reflect.String + 1]reflect.Type{ + reflect.Bool: reflect.TypeOf(true), + reflect.String: reflect.TypeOf(""), + reflect.Float32: reflect.TypeOf(float64(1)), + reflect.Float64: reflect.TypeOf(float64(1)), + reflect.Int: reflect.TypeOf(int64(1)), + reflect.Int8: reflect.TypeOf(int64(1)), + reflect.Int16: reflect.TypeOf(int64(1)), + reflect.Int32: reflect.TypeOf(int64(1)), + reflect.Int64: reflect.TypeOf(int64(1)), + reflect.Uint: reflect.TypeOf(uint64(1)), + reflect.Uint8: reflect.TypeOf(uint64(1)), + reflect.Uint16: reflect.TypeOf(uint64(1)), + reflect.Uint32: reflect.TypeOf(uint64(1)), + reflect.Uint64: reflect.TypeOf(uint64(1)), +} + +// typeFor returns a reflect.Type for a reflect.Kind, or nil if none is found. +// supported values: +// string, bool, int64, uint64, float64, time.Time, int, int8, int16, int32, uint, uint8, uint16, uint32, float32 +func typeFor(k reflect.Kind) reflect.Type { + if k > 0 && int(k) < len(kindToType) { + return kindToType[k] + } + return nil +} + +func simpleValueCoercion(object interface{}) (interface{}, error) { + switch original := object.(type) { + case string, bool, int64, uint64, float64, time.Time: + return original, nil + case int: + return int64(original), nil + case int8: + return int64(original), nil + case int16: + return int64(original), nil + case int32: + return int64(original), nil + case uint: + return uint64(original), nil + case uint8: + return uint64(original), nil + case uint16: + return uint64(original), nil + case uint32: + return uint64(original), nil + case float32: + return float64(original), nil + case fmt.Stringer: + return original.String(), nil + default: + return nil, fmt.Errorf("cannot convert type %T to Tree", object) + } +} + +func sliceToTree(object interface{}) (interface{}, error) { + // arrays are a bit tricky, since they can represent either a + // collection of simple values, which is represented by one + // *tomlValue, or an array of tables, which is represented by an + // array of *Tree. + + // holding the assumption that this function is called from toTree only when value.Kind() is Array or Slice + value := reflect.ValueOf(object) + insideType := value.Type().Elem() + length := value.Len() + if length > 0 { + insideType = reflect.ValueOf(value.Index(0).Interface()).Type() + } + if insideType.Kind() == reflect.Map { + // this is considered as an array of tables + tablesArray := make([]*Tree, 0, length) + for i := 0; i < length; i++ { + table := value.Index(i) + tree, err := toTree(table.Interface()) + if err != nil { + return nil, err + } + tablesArray = append(tablesArray, tree.(*Tree)) + } + return tablesArray, nil + } + + sliceType := typeFor(insideType.Kind()) + if sliceType == nil { + sliceType = insideType + } + + arrayValue := reflect.MakeSlice(reflect.SliceOf(sliceType), 0, length) + + for i := 0; i < length; i++ { + val := value.Index(i).Interface() + simpleValue, err := simpleValueCoercion(val) + if err != nil { + return nil, err + } + arrayValue = reflect.Append(arrayValue, reflect.ValueOf(simpleValue)) + } + return &tomlValue{value: arrayValue.Interface(), position: Position{}}, nil +} + +func toTree(object interface{}) (interface{}, error) { + value := reflect.ValueOf(object) + + if value.Kind() == reflect.Map { + values := map[string]interface{}{} + keys := value.MapKeys() + for _, key := range keys { + if key.Kind() != reflect.String { + if _, ok := key.Interface().(string); !ok { + return nil, fmt.Errorf("map key needs to be a string, not %T (%v)", key.Interface(), key.Kind()) + } + } + + v := value.MapIndex(key) + newValue, err := toTree(v.Interface()) + if err != nil { + return nil, err + } + values[key.String()] = newValue + } + return &Tree{values: values, position: Position{}}, nil + } + + if value.Kind() == reflect.Array || value.Kind() == reflect.Slice { + return sliceToTree(object) + } + + simpleValue, err := simpleValueCoercion(object) + if err != nil { + return nil, err + } + return &tomlValue{value: simpleValue, position: Position{}}, nil +} diff --git a/vendor/github.com/pelletier/go-toml/tomltree_write.go b/vendor/github.com/pelletier/go-toml/tomltree_write.go new file mode 100644 index 000000000..e4049e29f --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/tomltree_write.go @@ -0,0 +1,333 @@ +package toml + +import ( + "bytes" + "fmt" + "io" + "math" + "reflect" + "sort" + "strconv" + "strings" + "time" +) + +// Encodes a string to a TOML-compliant multi-line string value +// This function is a clone of the existing encodeTomlString function, except that whitespace characters +// are preserved. Quotation marks and backslashes are also not escaped. +func encodeMultilineTomlString(value string) string { + var b bytes.Buffer + + for _, rr := range value { + switch rr { + case '\b': + b.WriteString(`\b`) + case '\t': + b.WriteString("\t") + case '\n': + b.WriteString("\n") + case '\f': + b.WriteString(`\f`) + case '\r': + b.WriteString("\r") + case '"': + b.WriteString(`"`) + case '\\': + b.WriteString(`\`) + default: + intRr := uint16(rr) + if intRr < 0x001F { + b.WriteString(fmt.Sprintf("\\u%0.4X", intRr)) + } else { + b.WriteRune(rr) + } + } + } + return b.String() +} + +// Encodes a string to a TOML-compliant string value +func encodeTomlString(value string) string { + var b bytes.Buffer + + for _, rr := range value { + switch rr { + case '\b': + b.WriteString(`\b`) + case '\t': + b.WriteString(`\t`) + case '\n': + b.WriteString(`\n`) + case '\f': + b.WriteString(`\f`) + case '\r': + b.WriteString(`\r`) + case '"': + b.WriteString(`\"`) + case '\\': + b.WriteString(`\\`) + default: + intRr := uint16(rr) + if intRr < 0x001F { + b.WriteString(fmt.Sprintf("\\u%0.4X", intRr)) + } else { + b.WriteRune(rr) + } + } + } + return b.String() +} + +func tomlValueStringRepresentation(v interface{}, indent string, arraysOneElementPerLine bool) (string, error) { + // this interface check is added to dereference the change made in the writeTo function. + // That change was made to allow this function to see formatting options. + tv, ok := v.(*tomlValue) + if ok { + v = tv.value + } else { + tv = &tomlValue{} + } + + switch value := v.(type) { + case uint64: + return strconv.FormatUint(value, 10), nil + case int64: + return strconv.FormatInt(value, 10), nil + case float64: + // Ensure a round float does contain a decimal point. Otherwise feeding + // the output back to the parser would convert to an integer. + if math.Trunc(value) == value { + return strings.ToLower(strconv.FormatFloat(value, 'f', 1, 32)), nil + } + return strings.ToLower(strconv.FormatFloat(value, 'f', -1, 32)), nil + case string: + if tv.multiline { + return "\"\"\"\n" + encodeMultilineTomlString(value) + "\"\"\"", nil + } + return "\"" + encodeTomlString(value) + "\"", nil + case []byte: + b, _ := v.([]byte) + return tomlValueStringRepresentation(string(b), indent, arraysOneElementPerLine) + case bool: + if value { + return "true", nil + } + return "false", nil + case time.Time: + return value.Format(time.RFC3339), nil + case nil: + return "", nil + } + + rv := reflect.ValueOf(v) + + if rv.Kind() == reflect.Slice { + var values []string + for i := 0; i < rv.Len(); i++ { + item := rv.Index(i).Interface() + itemRepr, err := tomlValueStringRepresentation(item, indent, arraysOneElementPerLine) + if err != nil { + return "", err + } + values = append(values, itemRepr) + } + if arraysOneElementPerLine && len(values) > 1 { + stringBuffer := bytes.Buffer{} + valueIndent := indent + ` ` // TODO: move that to a shared encoder state + + stringBuffer.WriteString("[\n") + + for _, value := range values { + stringBuffer.WriteString(valueIndent) + stringBuffer.WriteString(value) + stringBuffer.WriteString(`,`) + stringBuffer.WriteString("\n") + } + + stringBuffer.WriteString(indent + "]") + + return stringBuffer.String(), nil + } + return "[" + strings.Join(values, ",") + "]", nil + } + return "", fmt.Errorf("unsupported value type %T: %v", v, v) +} + +func (t *Tree) writeTo(w io.Writer, indent, keyspace string, bytesCount int64, arraysOneElementPerLine bool) (int64, error) { + simpleValuesKeys := make([]string, 0) + complexValuesKeys := make([]string, 0) + + for k := range t.values { + v := t.values[k] + switch v.(type) { + case *Tree, []*Tree: + complexValuesKeys = append(complexValuesKeys, k) + default: + simpleValuesKeys = append(simpleValuesKeys, k) + } + } + + sort.Strings(simpleValuesKeys) + sort.Strings(complexValuesKeys) + + for _, k := range simpleValuesKeys { + v, ok := t.values[k].(*tomlValue) + if !ok { + return bytesCount, fmt.Errorf("invalid value type at %s: %T", k, t.values[k]) + } + + repr, err := tomlValueStringRepresentation(v, indent, arraysOneElementPerLine) + if err != nil { + return bytesCount, err + } + + if v.comment != "" { + comment := strings.Replace(v.comment, "\n", "\n"+indent+"#", -1) + start := "# " + if strings.HasPrefix(comment, "#") { + start = "" + } + writtenBytesCountComment, errc := writeStrings(w, "\n", indent, start, comment, "\n") + bytesCount += int64(writtenBytesCountComment) + if errc != nil { + return bytesCount, errc + } + } + + var commented string + if v.commented { + commented = "# " + } + writtenBytesCount, err := writeStrings(w, indent, commented, k, " = ", repr, "\n") + bytesCount += int64(writtenBytesCount) + if err != nil { + return bytesCount, err + } + } + + for _, k := range complexValuesKeys { + v := t.values[k] + + combinedKey := k + if keyspace != "" { + combinedKey = keyspace + "." + combinedKey + } + var commented string + if t.commented { + commented = "# " + } + + switch node := v.(type) { + // node has to be of those two types given how keys are sorted above + case *Tree: + tv, ok := t.values[k].(*Tree) + if !ok { + return bytesCount, fmt.Errorf("invalid value type at %s: %T", k, t.values[k]) + } + if tv.comment != "" { + comment := strings.Replace(tv.comment, "\n", "\n"+indent+"#", -1) + start := "# " + if strings.HasPrefix(comment, "#") { + start = "" + } + writtenBytesCountComment, errc := writeStrings(w, "\n", indent, start, comment) + bytesCount += int64(writtenBytesCountComment) + if errc != nil { + return bytesCount, errc + } + } + writtenBytesCount, err := writeStrings(w, "\n", indent, commented, "[", combinedKey, "]\n") + bytesCount += int64(writtenBytesCount) + if err != nil { + return bytesCount, err + } + bytesCount, err = node.writeTo(w, indent+" ", combinedKey, bytesCount, arraysOneElementPerLine) + if err != nil { + return bytesCount, err + } + case []*Tree: + for _, subTree := range node { + writtenBytesCount, err := writeStrings(w, "\n", indent, commented, "[[", combinedKey, "]]\n") + bytesCount += int64(writtenBytesCount) + if err != nil { + return bytesCount, err + } + + bytesCount, err = subTree.writeTo(w, indent+" ", combinedKey, bytesCount, arraysOneElementPerLine) + if err != nil { + return bytesCount, err + } + } + } + } + + return bytesCount, nil +} + +func writeStrings(w io.Writer, s ...string) (int, error) { + var n int + for i := range s { + b, err := io.WriteString(w, s[i]) + n += b + if err != nil { + return n, err + } + } + return n, nil +} + +// WriteTo encode the Tree as Toml and writes it to the writer w. +// Returns the number of bytes written in case of success, or an error if anything happened. +func (t *Tree) WriteTo(w io.Writer) (int64, error) { + return t.writeTo(w, "", "", 0, false) +} + +// ToTomlString generates a human-readable representation of the current tree. +// Output spans multiple lines, and is suitable for ingest by a TOML parser. +// If the conversion cannot be performed, ToString returns a non-nil error. +func (t *Tree) ToTomlString() (string, error) { + var buf bytes.Buffer + _, err := t.WriteTo(&buf) + if err != nil { + return "", err + } + return buf.String(), nil +} + +// String generates a human-readable representation of the current tree. +// Alias of ToString. Present to implement the fmt.Stringer interface. +func (t *Tree) String() string { + result, _ := t.ToTomlString() + return result +} + +// ToMap recursively generates a representation of the tree using Go built-in structures. +// The following types are used: +// +// * bool +// * float64 +// * int64 +// * string +// * uint64 +// * time.Time +// * map[string]interface{} (where interface{} is any of this list) +// * []interface{} (where interface{} is any of this list) +func (t *Tree) ToMap() map[string]interface{} { + result := map[string]interface{}{} + + for k, v := range t.values { + switch node := v.(type) { + case []*Tree: + var array []interface{} + for _, item := range node { + array = append(array, item.ToMap()) + } + result[k] = array + case *Tree: + result[k] = node.ToMap() + case *tomlValue: + result[k] = node.value + } + } + return result +} diff --git a/vendor/github.com/spf13/cast/.gitignore b/vendor/github.com/spf13/cast/.gitignore new file mode 100644 index 000000000..53053a8ac --- /dev/null +++ b/vendor/github.com/spf13/cast/.gitignore @@ -0,0 +1,25 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test + +*.bench diff --git a/vendor/github.com/spf13/cast/.travis.yml b/vendor/github.com/spf13/cast/.travis.yml new file mode 100644 index 000000000..6420d1c27 --- /dev/null +++ b/vendor/github.com/spf13/cast/.travis.yml @@ -0,0 +1,15 @@ +language: go +env: + - GO111MODULE=on +sudo: required +go: + - "1.11.x" + - tip +os: + - linux +matrix: + allow_failures: + - go: tip + fast_finish: true +script: + - make check diff --git a/vendor/github.com/spf13/cast/LICENSE b/vendor/github.com/spf13/cast/LICENSE new file mode 100644 index 000000000..4527efb9c --- /dev/null +++ b/vendor/github.com/spf13/cast/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Steve Francia + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/vendor/github.com/spf13/cast/Makefile b/vendor/github.com/spf13/cast/Makefile new file mode 100644 index 000000000..7ccf8930b --- /dev/null +++ b/vendor/github.com/spf13/cast/Makefile @@ -0,0 +1,38 @@ +# A Self-Documenting Makefile: http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html + +.PHONY: check fmt lint test test-race vet test-cover-html help +.DEFAULT_GOAL := help + +check: test-race fmt vet lint ## Run tests and linters + +test: ## Run tests + go test ./... + +test-race: ## Run tests with race detector + go test -race ./... + +fmt: ## Run gofmt linter + @for d in `go list` ; do \ + if [ "`gofmt -l -s $$GOPATH/src/$$d | tee /dev/stderr`" ]; then \ + echo "^ improperly formatted go files" && echo && exit 1; \ + fi \ + done + +lint: ## Run golint linter + @for d in `go list` ; do \ + if [ "`golint $$d | tee /dev/stderr`" ]; then \ + echo "^ golint errors!" && echo && exit 1; \ + fi \ + done + +vet: ## Run go vet linter + @if [ "`go vet | tee /dev/stderr`" ]; then \ + echo "^ go vet errors!" && echo && exit 1; \ + fi + +test-cover-html: ## Generate test coverage report + go test -coverprofile=coverage.out -covermode=count + go tool cover -func=coverage.out + +help: + @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' diff --git a/vendor/github.com/spf13/cast/README.md b/vendor/github.com/spf13/cast/README.md new file mode 100644 index 000000000..e6939397d --- /dev/null +++ b/vendor/github.com/spf13/cast/README.md @@ -0,0 +1,75 @@ +cast +==== +[![GoDoc](https://godoc.org/github.com/spf13/cast?status.svg)](https://godoc.org/github.com/spf13/cast) +[![Build Status](https://api.travis-ci.org/spf13/cast.svg?branch=master)](https://travis-ci.org/spf13/cast) +[![Go Report Card](https://goreportcard.com/badge/github.com/spf13/cast)](https://goreportcard.com/report/github.com/spf13/cast) + +Easy and safe casting from one type to another in Go + +Don’t Panic! ... Cast + +## What is Cast? + +Cast is a library to convert between different go types in a consistent and easy way. + +Cast provides simple functions to easily convert a number to a string, an +interface into a bool, etc. Cast does this intelligently when an obvious +conversion is possible. It doesn’t make any attempts to guess what you meant, +for example you can only convert a string to an int when it is a string +representation of an int such as “8”. Cast was developed for use in +[Hugo](http://hugo.spf13.com), a website engine which uses YAML, TOML or JSON +for meta data. + +## Why use Cast? + +When working with dynamic data in Go you often need to cast or convert the data +from one type into another. Cast goes beyond just using type assertion (though +it uses that when possible) to provide a very straightforward and convenient +library. + +If you are working with interfaces to handle things like dynamic content +you’ll need an easy way to convert an interface into a given type. This +is the library for you. + +If you are taking in data from YAML, TOML or JSON or other formats which lack +full types, then Cast is the library for you. + +## Usage + +Cast provides a handful of To_____ methods. These methods will always return +the desired type. **If input is provided that will not convert to that type, the +0 or nil value for that type will be returned**. + +Cast also provides identical methods To_____E. These return the same result as +the To_____ methods, plus an additional error which tells you if it successfully +converted. Using these methods you can tell the difference between when the +input matched the zero value or when the conversion failed and the zero value +was returned. + +The following examples are merely a sample of what is available. Please review +the code for a complete set. + +### Example ‘ToString’: + + cast.ToString("mayonegg") // "mayonegg" + cast.ToString(8) // "8" + cast.ToString(8.31) // "8.31" + cast.ToString([]byte("one time")) // "one time" + cast.ToString(nil) // "" + + var foo interface{} = "one more time" + cast.ToString(foo) // "one more time" + + +### Example ‘ToInt’: + + cast.ToInt(8) // 8 + cast.ToInt(8.31) // 8 + cast.ToInt("8") // 8 + cast.ToInt(true) // 1 + cast.ToInt(false) // 0 + + var eight interface{} = 8 + cast.ToInt(eight) // 8 + cast.ToInt(nil) // 0 + diff --git a/vendor/github.com/spf13/cast/cast.go b/vendor/github.com/spf13/cast/cast.go new file mode 100644 index 000000000..9fba638d4 --- /dev/null +++ b/vendor/github.com/spf13/cast/cast.go @@ -0,0 +1,171 @@ +// Copyright © 2014 Steve Francia . +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +// Package cast provides easy and safe casting in Go. +package cast + +import "time" + +// ToBool casts an interface to a bool type. +func ToBool(i interface{}) bool { + v, _ := ToBoolE(i) + return v +} + +// ToTime casts an interface to a time.Time type. +func ToTime(i interface{}) time.Time { + v, _ := ToTimeE(i) + return v +} + +// ToDuration casts an interface to a time.Duration type. +func ToDuration(i interface{}) time.Duration { + v, _ := ToDurationE(i) + return v +} + +// ToFloat64 casts an interface to a float64 type. +func ToFloat64(i interface{}) float64 { + v, _ := ToFloat64E(i) + return v +} + +// ToFloat32 casts an interface to a float32 type. +func ToFloat32(i interface{}) float32 { + v, _ := ToFloat32E(i) + return v +} + +// ToInt64 casts an interface to an int64 type. +func ToInt64(i interface{}) int64 { + v, _ := ToInt64E(i) + return v +} + +// ToInt32 casts an interface to an int32 type. +func ToInt32(i interface{}) int32 { + v, _ := ToInt32E(i) + return v +} + +// ToInt16 casts an interface to an int16 type. +func ToInt16(i interface{}) int16 { + v, _ := ToInt16E(i) + return v +} + +// ToInt8 casts an interface to an int8 type. +func ToInt8(i interface{}) int8 { + v, _ := ToInt8E(i) + return v +} + +// ToInt casts an interface to an int type. +func ToInt(i interface{}) int { + v, _ := ToIntE(i) + return v +} + +// ToUint casts an interface to a uint type. +func ToUint(i interface{}) uint { + v, _ := ToUintE(i) + return v +} + +// ToUint64 casts an interface to a uint64 type. +func ToUint64(i interface{}) uint64 { + v, _ := ToUint64E(i) + return v +} + +// ToUint32 casts an interface to a uint32 type. +func ToUint32(i interface{}) uint32 { + v, _ := ToUint32E(i) + return v +} + +// ToUint16 casts an interface to a uint16 type. +func ToUint16(i interface{}) uint16 { + v, _ := ToUint16E(i) + return v +} + +// ToUint8 casts an interface to a uint8 type. +func ToUint8(i interface{}) uint8 { + v, _ := ToUint8E(i) + return v +} + +// ToString casts an interface to a string type. +func ToString(i interface{}) string { + v, _ := ToStringE(i) + return v +} + +// ToStringMapString casts an interface to a map[string]string type. +func ToStringMapString(i interface{}) map[string]string { + v, _ := ToStringMapStringE(i) + return v +} + +// ToStringMapStringSlice casts an interface to a map[string][]string type. +func ToStringMapStringSlice(i interface{}) map[string][]string { + v, _ := ToStringMapStringSliceE(i) + return v +} + +// ToStringMapBool casts an interface to a map[string]bool type. +func ToStringMapBool(i interface{}) map[string]bool { + v, _ := ToStringMapBoolE(i) + return v +} + +// ToStringMapInt casts an interface to a map[string]int type. +func ToStringMapInt(i interface{}) map[string]int { + v, _ := ToStringMapIntE(i) + return v +} + +// ToStringMapInt64 casts an interface to a map[string]int64 type. +func ToStringMapInt64(i interface{}) map[string]int64 { + v, _ := ToStringMapInt64E(i) + return v +} + +// ToStringMap casts an interface to a map[string]interface{} type. +func ToStringMap(i interface{}) map[string]interface{} { + v, _ := ToStringMapE(i) + return v +} + +// ToSlice casts an interface to a []interface{} type. +func ToSlice(i interface{}) []interface{} { + v, _ := ToSliceE(i) + return v +} + +// ToBoolSlice casts an interface to a []bool type. +func ToBoolSlice(i interface{}) []bool { + v, _ := ToBoolSliceE(i) + return v +} + +// ToStringSlice casts an interface to a []string type. +func ToStringSlice(i interface{}) []string { + v, _ := ToStringSliceE(i) + return v +} + +// ToIntSlice casts an interface to a []int type. +func ToIntSlice(i interface{}) []int { + v, _ := ToIntSliceE(i) + return v +} + +// ToDurationSlice casts an interface to a []time.Duration type. +func ToDurationSlice(i interface{}) []time.Duration { + v, _ := ToDurationSliceE(i) + return v +} diff --git a/vendor/github.com/spf13/cast/caste.go b/vendor/github.com/spf13/cast/caste.go new file mode 100644 index 000000000..a4859fb0a --- /dev/null +++ b/vendor/github.com/spf13/cast/caste.go @@ -0,0 +1,1249 @@ +// Copyright © 2014 Steve Francia . +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +package cast + +import ( + "encoding/json" + "errors" + "fmt" + "html/template" + "reflect" + "strconv" + "strings" + "time" +) + +var errNegativeNotAllowed = errors.New("unable to cast negative value") + +// ToTimeE casts an interface to a time.Time type. +func ToTimeE(i interface{}) (tim time.Time, err error) { + i = indirect(i) + + switch v := i.(type) { + case time.Time: + return v, nil + case string: + return StringToDate(v) + case int: + return time.Unix(int64(v), 0), nil + case int64: + return time.Unix(v, 0), nil + case int32: + return time.Unix(int64(v), 0), nil + case uint: + return time.Unix(int64(v), 0), nil + case uint64: + return time.Unix(int64(v), 0), nil + case uint32: + return time.Unix(int64(v), 0), nil + default: + return time.Time{}, fmt.Errorf("unable to cast %#v of type %T to Time", i, i) + } +} + +// ToDurationE casts an interface to a time.Duration type. +func ToDurationE(i interface{}) (d time.Duration, err error) { + i = indirect(i) + + switch s := i.(type) { + case time.Duration: + return s, nil + case int, int64, int32, int16, int8, uint, uint64, uint32, uint16, uint8: + d = time.Duration(ToInt64(s)) + return + case float32, float64: + d = time.Duration(ToFloat64(s)) + return + case string: + if strings.ContainsAny(s, "nsuµmh") { + d, err = time.ParseDuration(s) + } else { + d, err = time.ParseDuration(s + "ns") + } + return + default: + err = fmt.Errorf("unable to cast %#v of type %T to Duration", i, i) + return + } +} + +// ToBoolE casts an interface to a bool type. +func ToBoolE(i interface{}) (bool, error) { + i = indirect(i) + + switch b := i.(type) { + case bool: + return b, nil + case nil: + return false, nil + case int: + if i.(int) != 0 { + return true, nil + } + return false, nil + case string: + return strconv.ParseBool(i.(string)) + default: + return false, fmt.Errorf("unable to cast %#v of type %T to bool", i, i) + } +} + +// ToFloat64E casts an interface to a float64 type. +func ToFloat64E(i interface{}) (float64, error) { + i = indirect(i) + + switch s := i.(type) { + case float64: + return s, nil + case float32: + return float64(s), nil + case int: + return float64(s), nil + case int64: + return float64(s), nil + case int32: + return float64(s), nil + case int16: + return float64(s), nil + case int8: + return float64(s), nil + case uint: + return float64(s), nil + case uint64: + return float64(s), nil + case uint32: + return float64(s), nil + case uint16: + return float64(s), nil + case uint8: + return float64(s), nil + case string: + v, err := strconv.ParseFloat(s, 64) + if err == nil { + return v, nil + } + return 0, fmt.Errorf("unable to cast %#v of type %T to float64", i, i) + case bool: + if s { + return 1, nil + } + return 0, nil + default: + return 0, fmt.Errorf("unable to cast %#v of type %T to float64", i, i) + } +} + +// ToFloat32E casts an interface to a float32 type. +func ToFloat32E(i interface{}) (float32, error) { + i = indirect(i) + + switch s := i.(type) { + case float64: + return float32(s), nil + case float32: + return s, nil + case int: + return float32(s), nil + case int64: + return float32(s), nil + case int32: + return float32(s), nil + case int16: + return float32(s), nil + case int8: + return float32(s), nil + case uint: + return float32(s), nil + case uint64: + return float32(s), nil + case uint32: + return float32(s), nil + case uint16: + return float32(s), nil + case uint8: + return float32(s), nil + case string: + v, err := strconv.ParseFloat(s, 32) + if err == nil { + return float32(v), nil + } + return 0, fmt.Errorf("unable to cast %#v of type %T to float32", i, i) + case bool: + if s { + return 1, nil + } + return 0, nil + default: + return 0, fmt.Errorf("unable to cast %#v of type %T to float32", i, i) + } +} + +// ToInt64E casts an interface to an int64 type. +func ToInt64E(i interface{}) (int64, error) { + i = indirect(i) + + switch s := i.(type) { + case int: + return int64(s), nil + case int64: + return s, nil + case int32: + return int64(s), nil + case int16: + return int64(s), nil + case int8: + return int64(s), nil + case uint: + return int64(s), nil + case uint64: + return int64(s), nil + case uint32: + return int64(s), nil + case uint16: + return int64(s), nil + case uint8: + return int64(s), nil + case float64: + return int64(s), nil + case float32: + return int64(s), nil + case string: + v, err := strconv.ParseInt(s, 0, 0) + if err == nil { + return v, nil + } + return 0, fmt.Errorf("unable to cast %#v of type %T to int64", i, i) + case bool: + if s { + return 1, nil + } + return 0, nil + case nil: + return 0, nil + default: + return 0, fmt.Errorf("unable to cast %#v of type %T to int64", i, i) + } +} + +// ToInt32E casts an interface to an int32 type. +func ToInt32E(i interface{}) (int32, error) { + i = indirect(i) + + switch s := i.(type) { + case int: + return int32(s), nil + case int64: + return int32(s), nil + case int32: + return s, nil + case int16: + return int32(s), nil + case int8: + return int32(s), nil + case uint: + return int32(s), nil + case uint64: + return int32(s), nil + case uint32: + return int32(s), nil + case uint16: + return int32(s), nil + case uint8: + return int32(s), nil + case float64: + return int32(s), nil + case float32: + return int32(s), nil + case string: + v, err := strconv.ParseInt(s, 0, 0) + if err == nil { + return int32(v), nil + } + return 0, fmt.Errorf("unable to cast %#v of type %T to int32", i, i) + case bool: + if s { + return 1, nil + } + return 0, nil + case nil: + return 0, nil + default: + return 0, fmt.Errorf("unable to cast %#v of type %T to int32", i, i) + } +} + +// ToInt16E casts an interface to an int16 type. +func ToInt16E(i interface{}) (int16, error) { + i = indirect(i) + + switch s := i.(type) { + case int: + return int16(s), nil + case int64: + return int16(s), nil + case int32: + return int16(s), nil + case int16: + return s, nil + case int8: + return int16(s), nil + case uint: + return int16(s), nil + case uint64: + return int16(s), nil + case uint32: + return int16(s), nil + case uint16: + return int16(s), nil + case uint8: + return int16(s), nil + case float64: + return int16(s), nil + case float32: + return int16(s), nil + case string: + v, err := strconv.ParseInt(s, 0, 0) + if err == nil { + return int16(v), nil + } + return 0, fmt.Errorf("unable to cast %#v of type %T to int16", i, i) + case bool: + if s { + return 1, nil + } + return 0, nil + case nil: + return 0, nil + default: + return 0, fmt.Errorf("unable to cast %#v of type %T to int16", i, i) + } +} + +// ToInt8E casts an interface to an int8 type. +func ToInt8E(i interface{}) (int8, error) { + i = indirect(i) + + switch s := i.(type) { + case int: + return int8(s), nil + case int64: + return int8(s), nil + case int32: + return int8(s), nil + case int16: + return int8(s), nil + case int8: + return s, nil + case uint: + return int8(s), nil + case uint64: + return int8(s), nil + case uint32: + return int8(s), nil + case uint16: + return int8(s), nil + case uint8: + return int8(s), nil + case float64: + return int8(s), nil + case float32: + return int8(s), nil + case string: + v, err := strconv.ParseInt(s, 0, 0) + if err == nil { + return int8(v), nil + } + return 0, fmt.Errorf("unable to cast %#v of type %T to int8", i, i) + case bool: + if s { + return 1, nil + } + return 0, nil + case nil: + return 0, nil + default: + return 0, fmt.Errorf("unable to cast %#v of type %T to int8", i, i) + } +} + +// ToIntE casts an interface to an int type. +func ToIntE(i interface{}) (int, error) { + i = indirect(i) + + switch s := i.(type) { + case int: + return s, nil + case int64: + return int(s), nil + case int32: + return int(s), nil + case int16: + return int(s), nil + case int8: + return int(s), nil + case uint: + return int(s), nil + case uint64: + return int(s), nil + case uint32: + return int(s), nil + case uint16: + return int(s), nil + case uint8: + return int(s), nil + case float64: + return int(s), nil + case float32: + return int(s), nil + case string: + v, err := strconv.ParseInt(s, 0, 0) + if err == nil { + return int(v), nil + } + return 0, fmt.Errorf("unable to cast %#v of type %T to int", i, i) + case bool: + if s { + return 1, nil + } + return 0, nil + case nil: + return 0, nil + default: + return 0, fmt.Errorf("unable to cast %#v of type %T to int", i, i) + } +} + +// ToUintE casts an interface to a uint type. +func ToUintE(i interface{}) (uint, error) { + i = indirect(i) + + switch s := i.(type) { + case string: + v, err := strconv.ParseUint(s, 0, 0) + if err == nil { + return uint(v), nil + } + return 0, fmt.Errorf("unable to cast %#v to uint: %s", i, err) + case int: + if s < 0 { + return 0, errNegativeNotAllowed + } + return uint(s), nil + case int64: + if s < 0 { + return 0, errNegativeNotAllowed + } + return uint(s), nil + case int32: + if s < 0 { + return 0, errNegativeNotAllowed + } + return uint(s), nil + case int16: + if s < 0 { + return 0, errNegativeNotAllowed + } + return uint(s), nil + case int8: + if s < 0 { + return 0, errNegativeNotAllowed + } + return uint(s), nil + case uint: + return s, nil + case uint64: + return uint(s), nil + case uint32: + return uint(s), nil + case uint16: + return uint(s), nil + case uint8: + return uint(s), nil + case float64: + if s < 0 { + return 0, errNegativeNotAllowed + } + return uint(s), nil + case float32: + if s < 0 { + return 0, errNegativeNotAllowed + } + return uint(s), nil + case bool: + if s { + return 1, nil + } + return 0, nil + case nil: + return 0, nil + default: + return 0, fmt.Errorf("unable to cast %#v of type %T to uint", i, i) + } +} + +// ToUint64E casts an interface to a uint64 type. +func ToUint64E(i interface{}) (uint64, error) { + i = indirect(i) + + switch s := i.(type) { + case string: + v, err := strconv.ParseUint(s, 0, 64) + if err == nil { + return v, nil + } + return 0, fmt.Errorf("unable to cast %#v to uint64: %s", i, err) + case int: + if s < 0 { + return 0, errNegativeNotAllowed + } + return uint64(s), nil + case int64: + if s < 0 { + return 0, errNegativeNotAllowed + } + return uint64(s), nil + case int32: + if s < 0 { + return 0, errNegativeNotAllowed + } + return uint64(s), nil + case int16: + if s < 0 { + return 0, errNegativeNotAllowed + } + return uint64(s), nil + case int8: + if s < 0 { + return 0, errNegativeNotAllowed + } + return uint64(s), nil + case uint: + return uint64(s), nil + case uint64: + return s, nil + case uint32: + return uint64(s), nil + case uint16: + return uint64(s), nil + case uint8: + return uint64(s), nil + case float32: + if s < 0 { + return 0, errNegativeNotAllowed + } + return uint64(s), nil + case float64: + if s < 0 { + return 0, errNegativeNotAllowed + } + return uint64(s), nil + case bool: + if s { + return 1, nil + } + return 0, nil + case nil: + return 0, nil + default: + return 0, fmt.Errorf("unable to cast %#v of type %T to uint64", i, i) + } +} + +// ToUint32E casts an interface to a uint32 type. +func ToUint32E(i interface{}) (uint32, error) { + i = indirect(i) + + switch s := i.(type) { + case string: + v, err := strconv.ParseUint(s, 0, 32) + if err == nil { + return uint32(v), nil + } + return 0, fmt.Errorf("unable to cast %#v to uint32: %s", i, err) + case int: + if s < 0 { + return 0, errNegativeNotAllowed + } + return uint32(s), nil + case int64: + if s < 0 { + return 0, errNegativeNotAllowed + } + return uint32(s), nil + case int32: + if s < 0 { + return 0, errNegativeNotAllowed + } + return uint32(s), nil + case int16: + if s < 0 { + return 0, errNegativeNotAllowed + } + return uint32(s), nil + case int8: + if s < 0 { + return 0, errNegativeNotAllowed + } + return uint32(s), nil + case uint: + return uint32(s), nil + case uint64: + return uint32(s), nil + case uint32: + return s, nil + case uint16: + return uint32(s), nil + case uint8: + return uint32(s), nil + case float64: + if s < 0 { + return 0, errNegativeNotAllowed + } + return uint32(s), nil + case float32: + if s < 0 { + return 0, errNegativeNotAllowed + } + return uint32(s), nil + case bool: + if s { + return 1, nil + } + return 0, nil + case nil: + return 0, nil + default: + return 0, fmt.Errorf("unable to cast %#v of type %T to uint32", i, i) + } +} + +// ToUint16E casts an interface to a uint16 type. +func ToUint16E(i interface{}) (uint16, error) { + i = indirect(i) + + switch s := i.(type) { + case string: + v, err := strconv.ParseUint(s, 0, 16) + if err == nil { + return uint16(v), nil + } + return 0, fmt.Errorf("unable to cast %#v to uint16: %s", i, err) + case int: + if s < 0 { + return 0, errNegativeNotAllowed + } + return uint16(s), nil + case int64: + if s < 0 { + return 0, errNegativeNotAllowed + } + return uint16(s), nil + case int32: + if s < 0 { + return 0, errNegativeNotAllowed + } + return uint16(s), nil + case int16: + if s < 0 { + return 0, errNegativeNotAllowed + } + return uint16(s), nil + case int8: + if s < 0 { + return 0, errNegativeNotAllowed + } + return uint16(s), nil + case uint: + return uint16(s), nil + case uint64: + return uint16(s), nil + case uint32: + return uint16(s), nil + case uint16: + return s, nil + case uint8: + return uint16(s), nil + case float64: + if s < 0 { + return 0, errNegativeNotAllowed + } + return uint16(s), nil + case float32: + if s < 0 { + return 0, errNegativeNotAllowed + } + return uint16(s), nil + case bool: + if s { + return 1, nil + } + return 0, nil + case nil: + return 0, nil + default: + return 0, fmt.Errorf("unable to cast %#v of type %T to uint16", i, i) + } +} + +// ToUint8E casts an interface to a uint type. +func ToUint8E(i interface{}) (uint8, error) { + i = indirect(i) + + switch s := i.(type) { + case string: + v, err := strconv.ParseUint(s, 0, 8) + if err == nil { + return uint8(v), nil + } + return 0, fmt.Errorf("unable to cast %#v to uint8: %s", i, err) + case int: + if s < 0 { + return 0, errNegativeNotAllowed + } + return uint8(s), nil + case int64: + if s < 0 { + return 0, errNegativeNotAllowed + } + return uint8(s), nil + case int32: + if s < 0 { + return 0, errNegativeNotAllowed + } + return uint8(s), nil + case int16: + if s < 0 { + return 0, errNegativeNotAllowed + } + return uint8(s), nil + case int8: + if s < 0 { + return 0, errNegativeNotAllowed + } + return uint8(s), nil + case uint: + return uint8(s), nil + case uint64: + return uint8(s), nil + case uint32: + return uint8(s), nil + case uint16: + return uint8(s), nil + case uint8: + return s, nil + case float64: + if s < 0 { + return 0, errNegativeNotAllowed + } + return uint8(s), nil + case float32: + if s < 0 { + return 0, errNegativeNotAllowed + } + return uint8(s), nil + case bool: + if s { + return 1, nil + } + return 0, nil + case nil: + return 0, nil + default: + return 0, fmt.Errorf("unable to cast %#v of type %T to uint8", i, i) + } +} + +// From html/template/content.go +// Copyright 2011 The Go Authors. All rights reserved. +// indirect returns the value, after dereferencing as many times +// as necessary to reach the base type (or nil). +func indirect(a interface{}) interface{} { + if a == nil { + return nil + } + if t := reflect.TypeOf(a); t.Kind() != reflect.Ptr { + // Avoid creating a reflect.Value if it's not a pointer. + return a + } + v := reflect.ValueOf(a) + for v.Kind() == reflect.Ptr && !v.IsNil() { + v = v.Elem() + } + return v.Interface() +} + +// From html/template/content.go +// Copyright 2011 The Go Authors. All rights reserved. +// indirectToStringerOrError returns the value, after dereferencing as many times +// as necessary to reach the base type (or nil) or an implementation of fmt.Stringer +// or error, +func indirectToStringerOrError(a interface{}) interface{} { + if a == nil { + return nil + } + + var errorType = reflect.TypeOf((*error)(nil)).Elem() + var fmtStringerType = reflect.TypeOf((*fmt.Stringer)(nil)).Elem() + + v := reflect.ValueOf(a) + for !v.Type().Implements(fmtStringerType) && !v.Type().Implements(errorType) && v.Kind() == reflect.Ptr && !v.IsNil() { + v = v.Elem() + } + return v.Interface() +} + +// ToStringE casts an interface to a string type. +func ToStringE(i interface{}) (string, error) { + i = indirectToStringerOrError(i) + + switch s := i.(type) { + case string: + return s, nil + case bool: + return strconv.FormatBool(s), nil + case float64: + return strconv.FormatFloat(s, 'f', -1, 64), nil + case float32: + return strconv.FormatFloat(float64(s), 'f', -1, 32), nil + case int: + return strconv.Itoa(s), nil + case int64: + return strconv.FormatInt(s, 10), nil + case int32: + return strconv.Itoa(int(s)), nil + case int16: + return strconv.FormatInt(int64(s), 10), nil + case int8: + return strconv.FormatInt(int64(s), 10), nil + case uint: + return strconv.FormatInt(int64(s), 10), nil + case uint64: + return strconv.FormatInt(int64(s), 10), nil + case uint32: + return strconv.FormatInt(int64(s), 10), nil + case uint16: + return strconv.FormatInt(int64(s), 10), nil + case uint8: + return strconv.FormatInt(int64(s), 10), nil + case []byte: + return string(s), nil + case template.HTML: + return string(s), nil + case template.URL: + return string(s), nil + case template.JS: + return string(s), nil + case template.CSS: + return string(s), nil + case template.HTMLAttr: + return string(s), nil + case nil: + return "", nil + case fmt.Stringer: + return s.String(), nil + case error: + return s.Error(), nil + default: + return "", fmt.Errorf("unable to cast %#v of type %T to string", i, i) + } +} + +// ToStringMapStringE casts an interface to a map[string]string type. +func ToStringMapStringE(i interface{}) (map[string]string, error) { + var m = map[string]string{} + + switch v := i.(type) { + case map[string]string: + return v, nil + case map[string]interface{}: + for k, val := range v { + m[ToString(k)] = ToString(val) + } + return m, nil + case map[interface{}]string: + for k, val := range v { + m[ToString(k)] = ToString(val) + } + return m, nil + case map[interface{}]interface{}: + for k, val := range v { + m[ToString(k)] = ToString(val) + } + return m, nil + case string: + err := jsonStringToObject(v, &m) + return m, err + default: + return m, fmt.Errorf("unable to cast %#v of type %T to map[string]string", i, i) + } +} + +// ToStringMapStringSliceE casts an interface to a map[string][]string type. +func ToStringMapStringSliceE(i interface{}) (map[string][]string, error) { + var m = map[string][]string{} + + switch v := i.(type) { + case map[string][]string: + return v, nil + case map[string][]interface{}: + for k, val := range v { + m[ToString(k)] = ToStringSlice(val) + } + return m, nil + case map[string]string: + for k, val := range v { + m[ToString(k)] = []string{val} + } + case map[string]interface{}: + for k, val := range v { + switch vt := val.(type) { + case []interface{}: + m[ToString(k)] = ToStringSlice(vt) + case []string: + m[ToString(k)] = vt + default: + m[ToString(k)] = []string{ToString(val)} + } + } + return m, nil + case map[interface{}][]string: + for k, val := range v { + m[ToString(k)] = ToStringSlice(val) + } + return m, nil + case map[interface{}]string: + for k, val := range v { + m[ToString(k)] = ToStringSlice(val) + } + return m, nil + case map[interface{}][]interface{}: + for k, val := range v { + m[ToString(k)] = ToStringSlice(val) + } + return m, nil + case map[interface{}]interface{}: + for k, val := range v { + key, err := ToStringE(k) + if err != nil { + return m, fmt.Errorf("unable to cast %#v of type %T to map[string][]string", i, i) + } + value, err := ToStringSliceE(val) + if err != nil { + return m, fmt.Errorf("unable to cast %#v of type %T to map[string][]string", i, i) + } + m[key] = value + } + case string: + err := jsonStringToObject(v, &m) + return m, err + default: + return m, fmt.Errorf("unable to cast %#v of type %T to map[string][]string", i, i) + } + return m, nil +} + +// ToStringMapBoolE casts an interface to a map[string]bool type. +func ToStringMapBoolE(i interface{}) (map[string]bool, error) { + var m = map[string]bool{} + + switch v := i.(type) { + case map[interface{}]interface{}: + for k, val := range v { + m[ToString(k)] = ToBool(val) + } + return m, nil + case map[string]interface{}: + for k, val := range v { + m[ToString(k)] = ToBool(val) + } + return m, nil + case map[string]bool: + return v, nil + case string: + err := jsonStringToObject(v, &m) + return m, err + default: + return m, fmt.Errorf("unable to cast %#v of type %T to map[string]bool", i, i) + } +} + +// ToStringMapE casts an interface to a map[string]interface{} type. +func ToStringMapE(i interface{}) (map[string]interface{}, error) { + var m = map[string]interface{}{} + + switch v := i.(type) { + case map[interface{}]interface{}: + for k, val := range v { + m[ToString(k)] = val + } + return m, nil + case map[string]interface{}: + return v, nil + case string: + err := jsonStringToObject(v, &m) + return m, err + default: + return m, fmt.Errorf("unable to cast %#v of type %T to map[string]interface{}", i, i) + } +} + +// ToStringMapIntE casts an interface to a map[string]int{} type. +func ToStringMapIntE(i interface{}) (map[string]int, error) { + var m = map[string]int{} + if i == nil { + return m, fmt.Errorf("unable to cast %#v of type %T to map[string]int", i, i) + } + + switch v := i.(type) { + case map[interface{}]interface{}: + for k, val := range v { + m[ToString(k)] = ToInt(val) + } + return m, nil + case map[string]interface{}: + for k, val := range v { + m[k] = ToInt(val) + } + return m, nil + case map[string]int: + return v, nil + case string: + err := jsonStringToObject(v, &m) + return m, err + } + + if reflect.TypeOf(i).Kind() != reflect.Map { + return m, fmt.Errorf("unable to cast %#v of type %T to map[string]int", i, i) + } + + mVal := reflect.ValueOf(m) + v := reflect.ValueOf(i) + for _, keyVal := range v.MapKeys() { + val, err := ToIntE(v.MapIndex(keyVal).Interface()) + if err != nil { + return m, fmt.Errorf("unable to cast %#v of type %T to map[string]int", i, i) + } + mVal.SetMapIndex(keyVal, reflect.ValueOf(val)) + } + return m, nil +} + +// ToStringMapInt64E casts an interface to a map[string]int64{} type. +func ToStringMapInt64E(i interface{}) (map[string]int64, error) { + var m = map[string]int64{} + if i == nil { + return m, fmt.Errorf("unable to cast %#v of type %T to map[string]int64", i, i) + } + + switch v := i.(type) { + case map[interface{}]interface{}: + for k, val := range v { + m[ToString(k)] = ToInt64(val) + } + return m, nil + case map[string]interface{}: + for k, val := range v { + m[k] = ToInt64(val) + } + return m, nil + case map[string]int64: + return v, nil + case string: + err := jsonStringToObject(v, &m) + return m, err + } + + if reflect.TypeOf(i).Kind() != reflect.Map { + return m, fmt.Errorf("unable to cast %#v of type %T to map[string]int64", i, i) + } + mVal := reflect.ValueOf(m) + v := reflect.ValueOf(i) + for _, keyVal := range v.MapKeys() { + val, err := ToInt64E(v.MapIndex(keyVal).Interface()) + if err != nil { + return m, fmt.Errorf("unable to cast %#v of type %T to map[string]int64", i, i) + } + mVal.SetMapIndex(keyVal, reflect.ValueOf(val)) + } + return m, nil +} + +// ToSliceE casts an interface to a []interface{} type. +func ToSliceE(i interface{}) ([]interface{}, error) { + var s []interface{} + + switch v := i.(type) { + case []interface{}: + return append(s, v...), nil + case []map[string]interface{}: + for _, u := range v { + s = append(s, u) + } + return s, nil + default: + return s, fmt.Errorf("unable to cast %#v of type %T to []interface{}", i, i) + } +} + +// ToBoolSliceE casts an interface to a []bool type. +func ToBoolSliceE(i interface{}) ([]bool, error) { + if i == nil { + return []bool{}, fmt.Errorf("unable to cast %#v of type %T to []bool", i, i) + } + + switch v := i.(type) { + case []bool: + return v, nil + } + + kind := reflect.TypeOf(i).Kind() + switch kind { + case reflect.Slice, reflect.Array: + s := reflect.ValueOf(i) + a := make([]bool, s.Len()) + for j := 0; j < s.Len(); j++ { + val, err := ToBoolE(s.Index(j).Interface()) + if err != nil { + return []bool{}, fmt.Errorf("unable to cast %#v of type %T to []bool", i, i) + } + a[j] = val + } + return a, nil + default: + return []bool{}, fmt.Errorf("unable to cast %#v of type %T to []bool", i, i) + } +} + +// ToStringSliceE casts an interface to a []string type. +func ToStringSliceE(i interface{}) ([]string, error) { + var a []string + + switch v := i.(type) { + case []interface{}: + for _, u := range v { + a = append(a, ToString(u)) + } + return a, nil + case []string: + return v, nil + case string: + return strings.Fields(v), nil + case interface{}: + str, err := ToStringE(v) + if err != nil { + return a, fmt.Errorf("unable to cast %#v of type %T to []string", i, i) + } + return []string{str}, nil + default: + return a, fmt.Errorf("unable to cast %#v of type %T to []string", i, i) + } +} + +// ToIntSliceE casts an interface to a []int type. +func ToIntSliceE(i interface{}) ([]int, error) { + if i == nil { + return []int{}, fmt.Errorf("unable to cast %#v of type %T to []int", i, i) + } + + switch v := i.(type) { + case []int: + return v, nil + } + + kind := reflect.TypeOf(i).Kind() + switch kind { + case reflect.Slice, reflect.Array: + s := reflect.ValueOf(i) + a := make([]int, s.Len()) + for j := 0; j < s.Len(); j++ { + val, err := ToIntE(s.Index(j).Interface()) + if err != nil { + return []int{}, fmt.Errorf("unable to cast %#v of type %T to []int", i, i) + } + a[j] = val + } + return a, nil + default: + return []int{}, fmt.Errorf("unable to cast %#v of type %T to []int", i, i) + } +} + +// ToDurationSliceE casts an interface to a []time.Duration type. +func ToDurationSliceE(i interface{}) ([]time.Duration, error) { + if i == nil { + return []time.Duration{}, fmt.Errorf("unable to cast %#v of type %T to []time.Duration", i, i) + } + + switch v := i.(type) { + case []time.Duration: + return v, nil + } + + kind := reflect.TypeOf(i).Kind() + switch kind { + case reflect.Slice, reflect.Array: + s := reflect.ValueOf(i) + a := make([]time.Duration, s.Len()) + for j := 0; j < s.Len(); j++ { + val, err := ToDurationE(s.Index(j).Interface()) + if err != nil { + return []time.Duration{}, fmt.Errorf("unable to cast %#v of type %T to []time.Duration", i, i) + } + a[j] = val + } + return a, nil + default: + return []time.Duration{}, fmt.Errorf("unable to cast %#v of type %T to []time.Duration", i, i) + } +} + +// StringToDate attempts to parse a string into a time.Time type using a +// predefined list of formats. If no suitable format is found, an error is +// returned. +func StringToDate(s string) (time.Time, error) { + return parseDateWith(s, []string{ + time.RFC3339, + "2006-01-02T15:04:05", // iso8601 without timezone + time.RFC1123Z, + time.RFC1123, + time.RFC822Z, + time.RFC822, + time.RFC850, + time.ANSIC, + time.UnixDate, + time.RubyDate, + "2006-01-02 15:04:05.999999999 -0700 MST", // Time.String() + "2006-01-02", + "02 Jan 2006", + "2006-01-02T15:04:05-0700", // RFC3339 without timezone hh:mm colon + "2006-01-02 15:04:05 -07:00", + "2006-01-02 15:04:05 -0700", + "2006-01-02 15:04:05Z07:00", // RFC3339 without T + "2006-01-02 15:04:05Z0700", // RFC3339 without T or timezone hh:mm colon + "2006-01-02 15:04:05", + time.Kitchen, + time.Stamp, + time.StampMilli, + time.StampMicro, + time.StampNano, + }) +} + +func parseDateWith(s string, dates []string) (d time.Time, e error) { + for _, dateType := range dates { + if d, e = time.Parse(dateType, s); e == nil { + return + } + } + return d, fmt.Errorf("unable to parse date: %s", s) +} + +// jsonStringToObject attempts to unmarshall a string as JSON into +// the object passed as pointer. +func jsonStringToObject(s string, v interface{}) error { + data := []byte(s) + return json.Unmarshal(data, v) +} diff --git a/vendor/github.com/spf13/cast/go.mod b/vendor/github.com/spf13/cast/go.mod new file mode 100644 index 000000000..c1c0232dd --- /dev/null +++ b/vendor/github.com/spf13/cast/go.mod @@ -0,0 +1,7 @@ +module github.com/spf13/cast + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stretchr/testify v1.2.2 +) diff --git a/vendor/github.com/spf13/cast/go.sum b/vendor/github.com/spf13/cast/go.sum new file mode 100644 index 000000000..e03ee77d9 --- /dev/null +++ b/vendor/github.com/spf13/cast/go.sum @@ -0,0 +1,6 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= diff --git a/vendor/github.com/spf13/jwalterweatherman/.gitignore b/vendor/github.com/spf13/jwalterweatherman/.gitignore new file mode 100644 index 000000000..00268614f --- /dev/null +++ b/vendor/github.com/spf13/jwalterweatherman/.gitignore @@ -0,0 +1,22 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe diff --git a/vendor/github.com/spf13/jwalterweatherman/LICENSE b/vendor/github.com/spf13/jwalterweatherman/LICENSE new file mode 100644 index 000000000..4527efb9c --- /dev/null +++ b/vendor/github.com/spf13/jwalterweatherman/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Steve Francia + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/vendor/github.com/spf13/jwalterweatherman/README.md b/vendor/github.com/spf13/jwalterweatherman/README.md new file mode 100644 index 000000000..932a23fc6 --- /dev/null +++ b/vendor/github.com/spf13/jwalterweatherman/README.md @@ -0,0 +1,148 @@ +jWalterWeatherman +================= + +Seamless printing to the terminal (stdout) and logging to a io.Writer +(file) that’s as easy to use as fmt.Println. + +![and_that__s_why_you_always_leave_a_note_by_jonnyetc-d57q7um](https://cloud.githubusercontent.com/assets/173412/11002937/ccd01654-847d-11e5-828e-12ebaf582eaf.jpg) +Graphic by [JonnyEtc](http://jonnyetc.deviantart.com/art/And-That-s-Why-You-Always-Leave-a-Note-315311422) + +JWW is primarily a wrapper around the excellent standard log library. It +provides a few advantages over using the standard log library alone. + +1. Ready to go out of the box. +2. One library for both printing to the terminal and logging (to files). +3. Really easy to log to either a temp file or a file you specify. + + +I really wanted a very straightforward library that could seamlessly do +the following things. + +1. Replace all the println, printf, etc statements thoughout my code with + something more useful +2. Allow the user to easily control what levels are printed to stdout +3. Allow the user to easily control what levels are logged +4. Provide an easy mechanism (like fmt.Println) to print info to the user + which can be easily logged as well +5. Due to 2 & 3 provide easy verbose mode for output and logs +6. Not have any unnecessary initialization cruft. Just use it. + +# Usage + +## Step 1. Use it +Put calls throughout your source based on type of feedback. +No initialization or setup needs to happen. Just start calling things. + +Available Loggers are: + + * TRACE + * DEBUG + * INFO + * WARN + * ERROR + * CRITICAL + * FATAL + +These each are loggers based on the log standard library and follow the +standard usage. Eg. + +```go + import ( + jww "github.com/spf13/jwalterweatherman" + ) + + ... + + if err != nil { + + // This is a pretty serious error and the user should know about + // it. It will be printed to the terminal as well as logged under the + // default thresholds. + + jww.ERROR.Println(err) + } + + if err2 != nil { + // This error isn’t going to materially change the behavior of the + // application, but it’s something that may not be what the user + // expects. Under the default thresholds, Warn will be logged, but + // not printed to the terminal. + + jww.WARN.Println(err2) + } + + // Information that’s relevant to what’s happening, but not very + // important for the user. Under the default thresholds this will be + // discarded. + + jww.INFO.Printf("information %q", response) + +``` + +NOTE: You can also use the library in a non-global setting by creating an instance of a Notebook: + +```go +notepad = jww.NewNotepad(jww.LevelInfo, jww.LevelTrace, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime) +notepad.WARN.Println("Some warning"") +``` + +_Why 7 levels?_ + +Maybe you think that 7 levels are too much for any application... and you +are probably correct. Just because there are seven levels doesn’t mean +that you should be using all 7 levels. Pick the right set for your needs. +Remember they only have to mean something to your project. + +## Step 2. Optionally configure JWW + +Under the default thresholds : + + * Debug, Trace & Info goto /dev/null + * Warn and above is logged (when a log file/io.Writer is provided) + * Error and above is printed to the terminal (stdout) + +### Changing the thresholds + +The threshold can be changed at any time, but will only affect calls that +execute after the change was made. + +This is very useful if your application has a verbose mode. Of course you +can decide what verbose means to you or even have multiple levels of +verbosity. + + +```go + import ( + jww "github.com/spf13/jwalterweatherman" + ) + + if Verbose { + jww.SetLogThreshold(jww.LevelTrace) + jww.SetStdoutThreshold(jww.LevelInfo) + } +``` + +Note that JWW's own internal output uses log levels as well, so set the log +level before making any other calls if you want to see what it's up to. + + +### Setting a log file + +JWW can log to any `io.Writer`: + + +```go + + jww.SetLogOutput(customWriter) + +``` + + +# More information + +This is an early release. I’ve been using it for a while and this is the +third interface I’ve tried. I like this one pretty well, but no guarantees +that it won’t change a bit. + +I wrote this for use in [hugo](https://gohugo.io). If you are looking +for a static website engine that’s super fast please checkout Hugo. diff --git a/vendor/github.com/spf13/jwalterweatherman/default_notepad.go b/vendor/github.com/spf13/jwalterweatherman/default_notepad.go new file mode 100644 index 000000000..bcb763403 --- /dev/null +++ b/vendor/github.com/spf13/jwalterweatherman/default_notepad.go @@ -0,0 +1,113 @@ +// Copyright © 2016 Steve Francia . +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +package jwalterweatherman + +import ( + "io" + "io/ioutil" + "log" + "os" +) + +var ( + TRACE *log.Logger + DEBUG *log.Logger + INFO *log.Logger + WARN *log.Logger + ERROR *log.Logger + CRITICAL *log.Logger + FATAL *log.Logger + + LOG *log.Logger + FEEDBACK *Feedback + + defaultNotepad *Notepad +) + +func reloadDefaultNotepad() { + TRACE = defaultNotepad.TRACE + DEBUG = defaultNotepad.DEBUG + INFO = defaultNotepad.INFO + WARN = defaultNotepad.WARN + ERROR = defaultNotepad.ERROR + CRITICAL = defaultNotepad.CRITICAL + FATAL = defaultNotepad.FATAL + + LOG = defaultNotepad.LOG + FEEDBACK = defaultNotepad.FEEDBACK +} + +func init() { + defaultNotepad = NewNotepad(LevelError, LevelWarn, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime) + reloadDefaultNotepad() +} + +// SetLogThreshold set the log threshold for the default notepad. Trace by default. +func SetLogThreshold(threshold Threshold) { + defaultNotepad.SetLogThreshold(threshold) + reloadDefaultNotepad() +} + +// SetLogOutput set the log output for the default notepad. Discarded by default. +func SetLogOutput(handle io.Writer) { + defaultNotepad.SetLogOutput(handle) + reloadDefaultNotepad() +} + +// SetStdoutThreshold set the standard output threshold for the default notepad. +// Info by default. +func SetStdoutThreshold(threshold Threshold) { + defaultNotepad.SetStdoutThreshold(threshold) + reloadDefaultNotepad() +} + +// SetPrefix set the prefix for the default logger. Empty by default. +func SetPrefix(prefix string) { + defaultNotepad.SetPrefix(prefix) + reloadDefaultNotepad() +} + +// SetFlags set the flags for the default logger. "log.Ldate | log.Ltime" by default. +func SetFlags(flags int) { + defaultNotepad.SetFlags(flags) + reloadDefaultNotepad() +} + +// Level returns the current global log threshold. +func LogThreshold() Threshold { + return defaultNotepad.logThreshold +} + +// Level returns the current global output threshold. +func StdoutThreshold() Threshold { + return defaultNotepad.stdoutThreshold +} + +// GetStdoutThreshold returns the defined Treshold for the log logger. +func GetLogThreshold() Threshold { + return defaultNotepad.GetLogThreshold() +} + +// GetStdoutThreshold returns the Treshold for the stdout logger. +func GetStdoutThreshold() Threshold { + return defaultNotepad.GetStdoutThreshold() +} + +// LogCountForLevel returns the number of log invocations for a given threshold. +func LogCountForLevel(l Threshold) uint64 { + return defaultNotepad.LogCountForLevel(l) +} + +// LogCountForLevelsGreaterThanorEqualTo returns the number of log invocations +// greater than or equal to a given threshold. +func LogCountForLevelsGreaterThanorEqualTo(threshold Threshold) uint64 { + return defaultNotepad.LogCountForLevelsGreaterThanorEqualTo(threshold) +} + +// ResetLogCounters resets the invocation counters for all levels. +func ResetLogCounters() { + defaultNotepad.ResetLogCounters() +} diff --git a/vendor/github.com/spf13/jwalterweatherman/go.mod b/vendor/github.com/spf13/jwalterweatherman/go.mod new file mode 100644 index 000000000..bce549c04 --- /dev/null +++ b/vendor/github.com/spf13/jwalterweatherman/go.mod @@ -0,0 +1 @@ +module github.com/spf13/jwalterweatherman diff --git a/vendor/github.com/spf13/jwalterweatherman/log_counter.go b/vendor/github.com/spf13/jwalterweatherman/log_counter.go new file mode 100644 index 000000000..11423ac41 --- /dev/null +++ b/vendor/github.com/spf13/jwalterweatherman/log_counter.go @@ -0,0 +1,55 @@ +// Copyright © 2016 Steve Francia . +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +package jwalterweatherman + +import ( + "sync/atomic" +) + +type logCounter struct { + counter uint64 +} + +func (c *logCounter) incr() { + atomic.AddUint64(&c.counter, 1) +} + +func (c *logCounter) resetCounter() { + atomic.StoreUint64(&c.counter, 0) +} + +func (c *logCounter) getCount() uint64 { + return atomic.LoadUint64(&c.counter) +} + +func (c *logCounter) Write(p []byte) (n int, err error) { + c.incr() + return len(p), nil +} + +// LogCountForLevel returns the number of log invocations for a given threshold. +func (n *Notepad) LogCountForLevel(l Threshold) uint64 { + return n.logCounters[l].getCount() +} + +// LogCountForLevelsGreaterThanorEqualTo returns the number of log invocations +// greater than or equal to a given threshold. +func (n *Notepad) LogCountForLevelsGreaterThanorEqualTo(threshold Threshold) uint64 { + var cnt uint64 + + for i := int(threshold); i < len(n.logCounters); i++ { + cnt += n.LogCountForLevel(Threshold(i)) + } + + return cnt +} + +// ResetLogCounters resets the invocation counters for all levels. +func (n *Notepad) ResetLogCounters() { + for _, np := range n.logCounters { + np.resetCounter() + } +} diff --git a/vendor/github.com/spf13/jwalterweatherman/notepad.go b/vendor/github.com/spf13/jwalterweatherman/notepad.go new file mode 100644 index 000000000..ae5aaf711 --- /dev/null +++ b/vendor/github.com/spf13/jwalterweatherman/notepad.go @@ -0,0 +1,194 @@ +// Copyright © 2016 Steve Francia . +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +package jwalterweatherman + +import ( + "fmt" + "io" + "log" +) + +type Threshold int + +func (t Threshold) String() string { + return prefixes[t] +} + +const ( + LevelTrace Threshold = iota + LevelDebug + LevelInfo + LevelWarn + LevelError + LevelCritical + LevelFatal +) + +var prefixes map[Threshold]string = map[Threshold]string{ + LevelTrace: "TRACE", + LevelDebug: "DEBUG", + LevelInfo: "INFO", + LevelWarn: "WARN", + LevelError: "ERROR", + LevelCritical: "CRITICAL", + LevelFatal: "FATAL", +} + +// Notepad is where you leave a note! +type Notepad struct { + TRACE *log.Logger + DEBUG *log.Logger + INFO *log.Logger + WARN *log.Logger + ERROR *log.Logger + CRITICAL *log.Logger + FATAL *log.Logger + + LOG *log.Logger + FEEDBACK *Feedback + + loggers [7]**log.Logger + logHandle io.Writer + outHandle io.Writer + logThreshold Threshold + stdoutThreshold Threshold + prefix string + flags int + + // One per Threshold + logCounters [7]*logCounter +} + +// NewNotepad create a new notepad. +func NewNotepad(outThreshold Threshold, logThreshold Threshold, outHandle, logHandle io.Writer, prefix string, flags int) *Notepad { + n := &Notepad{} + + n.loggers = [7]**log.Logger{&n.TRACE, &n.DEBUG, &n.INFO, &n.WARN, &n.ERROR, &n.CRITICAL, &n.FATAL} + n.outHandle = outHandle + n.logHandle = logHandle + n.stdoutThreshold = outThreshold + n.logThreshold = logThreshold + + if len(prefix) != 0 { + n.prefix = "[" + prefix + "] " + } else { + n.prefix = "" + } + + n.flags = flags + + n.LOG = log.New(n.logHandle, + "LOG: ", + n.flags) + n.FEEDBACK = &Feedback{out: log.New(outHandle, "", 0), log: n.LOG} + + n.init() + return n +} + +// init creates the loggers for each level depending on the notepad thresholds. +func (n *Notepad) init() { + logAndOut := io.MultiWriter(n.outHandle, n.logHandle) + + for t, logger := range n.loggers { + threshold := Threshold(t) + counter := &logCounter{} + n.logCounters[t] = counter + prefix := n.prefix + threshold.String() + " " + + switch { + case threshold >= n.logThreshold && threshold >= n.stdoutThreshold: + *logger = log.New(io.MultiWriter(counter, logAndOut), prefix, n.flags) + + case threshold >= n.logThreshold: + *logger = log.New(io.MultiWriter(counter, n.logHandle), prefix, n.flags) + + case threshold >= n.stdoutThreshold: + *logger = log.New(io.MultiWriter(counter, n.outHandle), prefix, n.flags) + + default: + // counter doesn't care about prefix and flags, so don't use them + // for performance. + *logger = log.New(counter, "", 0) + } + } +} + +// SetLogThreshold changes the threshold above which messages are written to the +// log file. +func (n *Notepad) SetLogThreshold(threshold Threshold) { + n.logThreshold = threshold + n.init() +} + +// SetLogOutput changes the file where log messages are written. +func (n *Notepad) SetLogOutput(handle io.Writer) { + n.logHandle = handle + n.init() +} + +// GetStdoutThreshold returns the defined Treshold for the log logger. +func (n *Notepad) GetLogThreshold() Threshold { + return n.logThreshold +} + +// SetStdoutThreshold changes the threshold above which messages are written to the +// standard output. +func (n *Notepad) SetStdoutThreshold(threshold Threshold) { + n.stdoutThreshold = threshold + n.init() +} + +// GetStdoutThreshold returns the Treshold for the stdout logger. +func (n *Notepad) GetStdoutThreshold() Threshold { + return n.stdoutThreshold +} + +// SetPrefix changes the prefix used by the notepad. Prefixes are displayed between +// brackets at the beginning of the line. An empty prefix won't be displayed at all. +func (n *Notepad) SetPrefix(prefix string) { + if len(prefix) != 0 { + n.prefix = "[" + prefix + "] " + } else { + n.prefix = "" + } + n.init() +} + +// SetFlags choose which flags the logger will display (after prefix and message +// level). See the package log for more informations on this. +func (n *Notepad) SetFlags(flags int) { + n.flags = flags + n.init() +} + +// Feedback writes plainly to the outHandle while +// logging with the standard extra information (date, file, etc). +type Feedback struct { + out *log.Logger + log *log.Logger +} + +func (fb *Feedback) Println(v ...interface{}) { + fb.output(fmt.Sprintln(v...)) +} + +func (fb *Feedback) Printf(format string, v ...interface{}) { + fb.output(fmt.Sprintf(format, v...)) +} + +func (fb *Feedback) Print(v ...interface{}) { + fb.output(fmt.Sprint(v...)) +} + +func (fb *Feedback) output(s string) { + if fb.out != nil { + fb.out.Output(2, s) + } + if fb.log != nil { + fb.log.Output(2, s) + } +} diff --git a/vendor/github.com/spf13/viper/.gitignore b/vendor/github.com/spf13/viper/.gitignore new file mode 100644 index 000000000..01b5c44b9 --- /dev/null +++ b/vendor/github.com/spf13/viper/.gitignore @@ -0,0 +1,29 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.bench + +.vscode + +# exclude dependencies in the `/vendor` folder +vendor diff --git a/vendor/github.com/spf13/viper/.travis.yml b/vendor/github.com/spf13/viper/.travis.yml new file mode 100644 index 000000000..bb83057ba --- /dev/null +++ b/vendor/github.com/spf13/viper/.travis.yml @@ -0,0 +1,31 @@ +go_import_path: github.com/spf13/viper + +language: go + +env: + global: + - GO111MODULE="on" + +go: + - 1.11.x + - tip + +os: + - linux + - osx + +matrix: + allow_failures: + - go: tip + fast_finish: true + +script: + - go install ./... + - diff -u <(echo -n) <(gofmt -d .) + - go test -v ./... + +after_success: + - go get -u -d github.com/spf13/hugo + - cd $GOPATH/src/github.com/spf13/hugo && make && ./hugo -s docs && cd - + +sudo: false diff --git a/vendor/github.com/spf13/viper/LICENSE b/vendor/github.com/spf13/viper/LICENSE new file mode 100644 index 000000000..4527efb9c --- /dev/null +++ b/vendor/github.com/spf13/viper/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Steve Francia + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/vendor/github.com/spf13/viper/README.md b/vendor/github.com/spf13/viper/README.md new file mode 100644 index 000000000..0208eac84 --- /dev/null +++ b/vendor/github.com/spf13/viper/README.md @@ -0,0 +1,691 @@ +![viper logo](https://cloud.githubusercontent.com/assets/173412/10886745/998df88a-8151-11e5-9448-4736db51020d.png) + +Go configuration with fangs! + +Many Go projects are built using Viper including: + +* [Hugo](http://gohugo.io) +* [EMC RexRay](http://rexray.readthedocs.org/en/stable/) +* [Imgur’s Incus](https://github.com/Imgur/incus) +* [Nanobox](https://github.com/nanobox-io/nanobox)/[Nanopack](https://github.com/nanopack) +* [Docker Notary](https://github.com/docker/Notary) +* [BloomApi](https://www.bloomapi.com/) +* [doctl](https://github.com/digitalocean/doctl) +* [Clairctl](https://github.com/jgsqware/clairctl) + +[![Build Status](https://travis-ci.org/spf13/viper.svg)](https://travis-ci.org/spf13/viper) [![Join the chat at https://gitter.im/spf13/viper](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/spf13/viper?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![GoDoc](https://godoc.org/github.com/spf13/viper?status.svg)](https://godoc.org/github.com/spf13/viper) + + +## What is Viper? + +Viper is a complete configuration solution for Go applications including 12-Factor apps. It is designed +to work within an application, and can handle all types of configuration needs +and formats. It supports: + +* setting defaults +* reading from JSON, TOML, YAML, HCL, and Java properties config files +* live watching and re-reading of config files (optional) +* reading from environment variables +* reading from remote config systems (etcd or Consul), and watching changes +* reading from command line flags +* reading from buffer +* setting explicit values + +Viper can be thought of as a registry for all of your applications +configuration needs. + +## Why Viper? + +When building a modern application, you don’t want to worry about +configuration file formats; you want to focus on building awesome software. +Viper is here to help with that. + +Viper does the following for you: + +1. Find, load, and unmarshal a configuration file in JSON, TOML, YAML, HCL, or Java properties formats. +2. Provide a mechanism to set default values for your different + configuration options. +3. Provide a mechanism to set override values for options specified through + command line flags. +4. Provide an alias system to easily rename parameters without breaking existing + code. +5. Make it easy to tell the difference between when a user has provided a + command line or config file which is the same as the default. + +Viper uses the following precedence order. Each item takes precedence over the +item below it: + + * explicit call to Set + * flag + * env + * config + * key/value store + * default + +Viper configuration keys are case insensitive. + +## Putting Values into Viper + +### Establishing Defaults + +A good configuration system will support default values. A default value is not +required for a key, but it’s useful in the event that a key hasn’t been set via +config file, environment variable, remote configuration or flag. + +Examples: + +```go +viper.SetDefault("ContentDir", "content") +viper.SetDefault("LayoutDir", "layouts") +viper.SetDefault("Taxonomies", map[string]string{"tag": "tags", "category": "categories"}) +``` + +### Reading Config Files + +Viper requires minimal configuration so it knows where to look for config files. +Viper supports JSON, TOML, YAML, HCL, and Java Properties files. Viper can search multiple paths, but +currently a single Viper instance only supports a single configuration file. +Viper does not default to any configuration search paths leaving defaults decision +to an application. + +Here is an example of how to use Viper to search for and read a configuration file. +None of the specific paths are required, but at least one path should be provided +where a configuration file is expected. + +```go +viper.SetConfigName("config") // name of config file (without extension) +viper.AddConfigPath("/etc/appname/") // path to look for the config file in +viper.AddConfigPath("$HOME/.appname") // call multiple times to add many search paths +viper.AddConfigPath(".") // optionally look for config in the working directory +err := viper.ReadInConfig() // Find and read the config file +if err != nil { // Handle errors reading the config file + panic(fmt.Errorf("Fatal error config file: %s \n", err)) +} +``` + +### Watching and re-reading config files + +Viper supports the ability to have your application live read a config file while running. + +Gone are the days of needing to restart a server to have a config take effect, +viper powered applications can read an update to a config file while running and +not miss a beat. + +Simply tell the viper instance to watchConfig. +Optionally you can provide a function for Viper to run each time a change occurs. + +**Make sure you add all of the configPaths prior to calling `WatchConfig()`** + +```go +viper.WatchConfig() +viper.OnConfigChange(func(e fsnotify.Event) { + fmt.Println("Config file changed:", e.Name) +}) +``` + +### Reading Config from io.Reader + +Viper predefines many configuration sources such as files, environment +variables, flags, and remote K/V store, but you are not bound to them. You can +also implement your own required configuration source and feed it to viper. + +```go +viper.SetConfigType("yaml") // or viper.SetConfigType("YAML") + +// any approach to require this configuration into your program. +var yamlExample = []byte(` +Hacker: true +name: steve +hobbies: +- skateboarding +- snowboarding +- go +clothing: + jacket: leather + trousers: denim +age: 35 +eyes : brown +beard: true +`) + +viper.ReadConfig(bytes.NewBuffer(yamlExample)) + +viper.Get("name") // this would be "steve" +``` + +### Setting Overrides + +These could be from a command line flag, or from your own application logic. + +```go +viper.Set("Verbose", true) +viper.Set("LogFile", LogFile) +``` + +### Registering and Using Aliases + +Aliases permit a single value to be referenced by multiple keys + +```go +viper.RegisterAlias("loud", "Verbose") + +viper.Set("verbose", true) // same result as next line +viper.Set("loud", true) // same result as prior line + +viper.GetBool("loud") // true +viper.GetBool("verbose") // true +``` + +### Working with Environment Variables + +Viper has full support for environment variables. This enables 12 factor +applications out of the box. There are five methods that exist to aid working +with ENV: + + * `AutomaticEnv()` + * `BindEnv(string...) : error` + * `SetEnvPrefix(string)` + * `SetEnvKeyReplacer(string...) *strings.Replacer` + * `AllowEmptyEnvVar(bool)` + +_When working with ENV variables, it’s important to recognize that Viper +treats ENV variables as case sensitive._ + +Viper provides a mechanism to try to ensure that ENV variables are unique. By +using `SetEnvPrefix`, you can tell Viper to use a prefix while reading from +the environment variables. Both `BindEnv` and `AutomaticEnv` will use this +prefix. + +`BindEnv` takes one or two parameters. The first parameter is the key name, the +second is the name of the environment variable. The name of the environment +variable is case sensitive. If the ENV variable name is not provided, then +Viper will automatically assume that the key name matches the ENV variable name, +but the ENV variable is IN ALL CAPS. When you explicitly provide the ENV +variable name, it **does not** automatically add the prefix. + +One important thing to recognize when working with ENV variables is that the +value will be read each time it is accessed. Viper does not fix the value when +the `BindEnv` is called. + +`AutomaticEnv` is a powerful helper especially when combined with +`SetEnvPrefix`. When called, Viper will check for an environment variable any +time a `viper.Get` request is made. It will apply the following rules. It will +check for a environment variable with a name matching the key uppercased and +prefixed with the `EnvPrefix` if set. + +`SetEnvKeyReplacer` allows you to use a `strings.Replacer` object to rewrite Env +keys to an extent. This is useful if you want to use `-` or something in your +`Get()` calls, but want your environmental variables to use `_` delimiters. An +example of using it can be found in `viper_test.go`. + +By default empty environment variables are considered unset and will fall back to +the next configuration source. To treat empty environment variables as set, use +the `AllowEmptyEnv` method. + +#### Env example + +```go +SetEnvPrefix("spf") // will be uppercased automatically +BindEnv("id") + +os.Setenv("SPF_ID", "13") // typically done outside of the app + +id := Get("id") // 13 +``` + +### Working with Flags + +Viper has the ability to bind to flags. Specifically, Viper supports `Pflags` +as used in the [Cobra](https://github.com/spf13/cobra) library. + +Like `BindEnv`, the value is not set when the binding method is called, but when +it is accessed. This means you can bind as early as you want, even in an +`init()` function. + +For individual flags, the `BindPFlag()` method provides this functionality. + +Example: + +```go +serverCmd.Flags().Int("port", 1138, "Port to run Application server on") +viper.BindPFlag("port", serverCmd.Flags().Lookup("port")) +``` + +You can also bind an existing set of pflags (pflag.FlagSet): + +Example: + +```go +pflag.Int("flagname", 1234, "help message for flagname") + +pflag.Parse() +viper.BindPFlags(pflag.CommandLine) + +i := viper.GetInt("flagname") // retrieve values from viper instead of pflag +``` + +The use of [pflag](https://github.com/spf13/pflag/) in Viper does not preclude +the use of other packages that use the [flag](https://golang.org/pkg/flag/) +package from the standard library. The pflag package can handle the flags +defined for the flag package by importing these flags. This is accomplished +by a calling a convenience function provided by the pflag package called +AddGoFlagSet(). + +Example: + +```go +package main + +import ( + "flag" + "github.com/spf13/pflag" +) + +func main() { + + // using standard library "flag" package + flag.Int("flagname", 1234, "help message for flagname") + + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) + pflag.Parse() + viper.BindPFlags(pflag.CommandLine) + + i := viper.GetInt("flagname") // retrieve value from viper + + ... +} +``` + +#### Flag interfaces + +Viper provides two Go interfaces to bind other flag systems if you don’t use `Pflags`. + +`FlagValue` represents a single flag. This is a very simple example on how to implement this interface: + +```go +type myFlag struct {} +func (f myFlag) HasChanged() bool { return false } +func (f myFlag) Name() string { return "my-flag-name" } +func (f myFlag) ValueString() string { return "my-flag-value" } +func (f myFlag) ValueType() string { return "string" } +``` + +Once your flag implements this interface, you can simply tell Viper to bind it: + +```go +viper.BindFlagValue("my-flag-name", myFlag{}) +``` + +`FlagValueSet` represents a group of flags. This is a very simple example on how to implement this interface: + +```go +type myFlagSet struct { + flags []myFlag +} + +func (f myFlagSet) VisitAll(fn func(FlagValue)) { + for _, flag := range flags { + fn(flag) + } +} +``` + +Once your flag set implements this interface, you can simply tell Viper to bind it: + +```go +fSet := myFlagSet{ + flags: []myFlag{myFlag{}, myFlag{}}, +} +viper.BindFlagValues("my-flags", fSet) +``` + +### Remote Key/Value Store Support + +To enable remote support in Viper, do a blank import of the `viper/remote` +package: + +`import _ "github.com/spf13/viper/remote"` + +Viper will read a config string (as JSON, TOML, YAML or HCL) retrieved from a path +in a Key/Value store such as etcd or Consul. These values take precedence over +default values, but are overridden by configuration values retrieved from disk, +flags, or environment variables. + +Viper uses [crypt](https://github.com/xordataexchange/crypt) to retrieve +configuration from the K/V store, which means that you can store your +configuration values encrypted and have them automatically decrypted if you have +the correct gpg keyring. Encryption is optional. + +You can use remote configuration in conjunction with local configuration, or +independently of it. + +`crypt` has a command-line helper that you can use to put configurations in your +K/V store. `crypt` defaults to etcd on http://127.0.0.1:4001. + +```bash +$ go get github.com/xordataexchange/crypt/bin/crypt +$ crypt set -plaintext /config/hugo.json /Users/hugo/settings/config.json +``` + +Confirm that your value was set: + +```bash +$ crypt get -plaintext /config/hugo.json +``` + +See the `crypt` documentation for examples of how to set encrypted values, or +how to use Consul. + +### Remote Key/Value Store Example - Unencrypted + +#### etcd +```go +viper.AddRemoteProvider("etcd", "http://127.0.0.1:4001","/config/hugo.json") +viper.SetConfigType("json") // because there is no file extension in a stream of bytes, supported extensions are "json", "toml", "yaml", "yml", "properties", "props", "prop" +err := viper.ReadRemoteConfig() +``` + +#### Consul +You need to set a key to Consul key/value storage with JSON value containing your desired config. +For example, create a Consul key/value store key `MY_CONSUL_KEY` with value: + +```json +{ + "port": 8080, + "hostname": "myhostname.com" +} +``` + +```go +viper.AddRemoteProvider("consul", "localhost:8500", "MY_CONSUL_KEY") +viper.SetConfigType("json") // Need to explicitly set this to json +err := viper.ReadRemoteConfig() + +fmt.Println(viper.Get("port")) // 8080 +fmt.Println(viper.Get("hostname")) // myhostname.com +``` + +### Remote Key/Value Store Example - Encrypted + +```go +viper.AddSecureRemoteProvider("etcd","http://127.0.0.1:4001","/config/hugo.json","/etc/secrets/mykeyring.gpg") +viper.SetConfigType("json") // because there is no file extension in a stream of bytes, supported extensions are "json", "toml", "yaml", "yml", "properties", "props", "prop" +err := viper.ReadRemoteConfig() +``` + +### Watching Changes in etcd - Unencrypted + +```go +// alternatively, you can create a new viper instance. +var runtime_viper = viper.New() + +runtime_viper.AddRemoteProvider("etcd", "http://127.0.0.1:4001", "/config/hugo.yml") +runtime_viper.SetConfigType("yaml") // because there is no file extension in a stream of bytes, supported extensions are "json", "toml", "yaml", "yml", "properties", "props", "prop" + +// read from remote config the first time. +err := runtime_viper.ReadRemoteConfig() + +// unmarshal config +runtime_viper.Unmarshal(&runtime_conf) + +// open a goroutine to watch remote changes forever +go func(){ + for { + time.Sleep(time.Second * 5) // delay after each request + + // currently, only tested with etcd support + err := runtime_viper.WatchRemoteConfig() + if err != nil { + log.Errorf("unable to read remote config: %v", err) + continue + } + + // unmarshal new config into our runtime config struct. you can also use channel + // to implement a signal to notify the system of the changes + runtime_viper.Unmarshal(&runtime_conf) + } +}() +``` + +## Getting Values From Viper + +In Viper, there are a few ways to get a value depending on the value’s type. +The following functions and methods exist: + + * `Get(key string) : interface{}` + * `GetBool(key string) : bool` + * `GetFloat64(key string) : float64` + * `GetInt(key string) : int` + * `GetString(key string) : string` + * `GetStringMap(key string) : map[string]interface{}` + * `GetStringMapString(key string) : map[string]string` + * `GetStringSlice(key string) : []string` + * `GetTime(key string) : time.Time` + * `GetDuration(key string) : time.Duration` + * `IsSet(key string) : bool` + * `AllSettings() : map[string]interface{}` + +One important thing to recognize is that each Get function will return a zero +value if it’s not found. To check if a given key exists, the `IsSet()` method +has been provided. + +Example: +```go +viper.GetString("logfile") // case-insensitive Setting & Getting +if viper.GetBool("verbose") { + fmt.Println("verbose enabled") +} +``` +### Accessing nested keys + +The accessor methods also accept formatted paths to deeply nested keys. For +example, if the following JSON file is loaded: + +```json +{ + "host": { + "address": "localhost", + "port": 5799 + }, + "datastore": { + "metric": { + "host": "127.0.0.1", + "port": 3099 + }, + "warehouse": { + "host": "198.0.0.1", + "port": 2112 + } + } +} + +``` + +Viper can access a nested field by passing a `.` delimited path of keys: + +```go +GetString("datastore.metric.host") // (returns "127.0.0.1") +``` + +This obeys the precedence rules established above; the search for the path +will cascade through the remaining configuration registries until found. + +For example, given this configuration file, both `datastore.metric.host` and +`datastore.metric.port` are already defined (and may be overridden). If in addition +`datastore.metric.protocol` was defined in the defaults, Viper would also find it. + +However, if `datastore.metric` was overridden (by a flag, an environment variable, +the `Set()` method, …) with an immediate value, then all sub-keys of +`datastore.metric` become undefined, they are “shadowed” by the higher-priority +configuration level. + +Lastly, if there exists a key that matches the delimited key path, its value +will be returned instead. E.g. + +```json +{ + "datastore.metric.host": "0.0.0.0", + "host": { + "address": "localhost", + "port": 5799 + }, + "datastore": { + "metric": { + "host": "127.0.0.1", + "port": 3099 + }, + "warehouse": { + "host": "198.0.0.1", + "port": 2112 + } + } +} + +GetString("datastore.metric.host") // returns "0.0.0.0" +``` + +### Extract sub-tree + +Extract sub-tree from Viper. + +For example, `viper` represents: + +```json +app: + cache1: + max-items: 100 + item-size: 64 + cache2: + max-items: 200 + item-size: 80 +``` + +After executing: + +```go +subv := viper.Sub("app.cache1") +``` + +`subv` represents: + +```json +max-items: 100 +item-size: 64 +``` + +Suppose we have: + +```go +func NewCache(cfg *Viper) *Cache {...} +``` + +which creates a cache based on config information formatted as `subv`. +Now it’s easy to create these 2 caches separately as: + +```go +cfg1 := viper.Sub("app.cache1") +cache1 := NewCache(cfg1) + +cfg2 := viper.Sub("app.cache2") +cache2 := NewCache(cfg2) +``` + +### Unmarshaling + +You also have the option of Unmarshaling all or a specific value to a struct, map, +etc. + +There are two methods to do this: + + * `Unmarshal(rawVal interface{}) : error` + * `UnmarshalKey(key string, rawVal interface{}) : error` + +Example: + +```go +type config struct { + Port int + Name string + PathMap string `mapstructure:"path_map"` +} + +var C config + +err := Unmarshal(&C) +if err != nil { + t.Fatalf("unable to decode into struct, %v", err) +} +``` + +### Marshalling to string + +You may need to marhsal all the settings held in viper into a string rather than write them to a file. +You can use your favorite format's marshaller with the config returned by `AllSettings()`. + +```go +import ( + yaml "gopkg.in/yaml.v2" + // ... +) + +func yamlStringSettings() string { + c := viper.AllSettings() + bs, err := yaml.Marshal(c) + if err != nil { + t.Fatalf("unable to marshal config to YAML: %v", err) + } + return string(bs) +} +``` + +## Viper or Vipers? + +Viper comes ready to use out of the box. There is no configuration or +initialization needed to begin using Viper. Since most applications will want +to use a single central repository for their configuration, the viper package +provides this. It is similar to a singleton. + +In all of the examples above, they demonstrate using viper in its singleton +style approach. + +### Working with multiple vipers + +You can also create many different vipers for use in your application. Each will +have its own unique set of configurations and values. Each can read from a +different config file, key value store, etc. All of the functions that viper +package supports are mirrored as methods on a viper. + +Example: + +```go +x := viper.New() +y := viper.New() + +x.SetDefault("ContentDir", "content") +y.SetDefault("ContentDir", "foobar") + +//... +``` + +When working with multiple vipers, it is up to the user to keep track of the +different vipers. + +## Q & A + +Q: Why not INI files? + +A: Ini files are pretty awful. There’s no standard format, and they are hard to +validate. Viper is designed to work with JSON, TOML or YAML files. If someone +really wants to add this feature, I’d be happy to merge it. It’s easy to specify +which formats your application will permit. + +Q: Why is it called “Viper”? + +A: Viper is designed to be a [companion](http://en.wikipedia.org/wiki/Viper_(G.I._Joe)) +to [Cobra](https://github.com/spf13/cobra). While both can operate completely +independently, together they make a powerful pair to handle much of your +application foundation needs. + +Q: Why is it called “Cobra”? + +A: Is there a better name for a [commander](http://en.wikipedia.org/wiki/Cobra_Commander)? diff --git a/vendor/github.com/spf13/viper/flags.go b/vendor/github.com/spf13/viper/flags.go new file mode 100644 index 000000000..dd32f4e1c --- /dev/null +++ b/vendor/github.com/spf13/viper/flags.go @@ -0,0 +1,57 @@ +package viper + +import "github.com/spf13/pflag" + +// FlagValueSet is an interface that users can implement +// to bind a set of flags to viper. +type FlagValueSet interface { + VisitAll(fn func(FlagValue)) +} + +// FlagValue is an interface that users can implement +// to bind different flags to viper. +type FlagValue interface { + HasChanged() bool + Name() string + ValueString() string + ValueType() string +} + +// pflagValueSet is a wrapper around *pflag.ValueSet +// that implements FlagValueSet. +type pflagValueSet struct { + flags *pflag.FlagSet +} + +// VisitAll iterates over all *pflag.Flag inside the *pflag.FlagSet. +func (p pflagValueSet) VisitAll(fn func(flag FlagValue)) { + p.flags.VisitAll(func(flag *pflag.Flag) { + fn(pflagValue{flag}) + }) +} + +// pflagValue is a wrapper aroung *pflag.flag +// that implements FlagValue +type pflagValue struct { + flag *pflag.Flag +} + +// HasChanges returns whether the flag has changes or not. +func (p pflagValue) HasChanged() bool { + return p.flag.Changed +} + +// Name returns the name of the flag. +func (p pflagValue) Name() string { + return p.flag.Name +} + +// ValueString returns the value of the flag as a string. +func (p pflagValue) ValueString() string { + return p.flag.Value.String() +} + +// ValueType returns the type of the flag as a string. +func (p pflagValue) ValueType() string { + return p.flag.Value.Type() +} diff --git a/vendor/github.com/spf13/viper/go.mod b/vendor/github.com/spf13/viper/go.mod new file mode 100644 index 000000000..279430055 --- /dev/null +++ b/vendor/github.com/spf13/viper/go.mod @@ -0,0 +1,43 @@ +module github.com/spf13/viper + +require ( + github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6 // indirect + github.com/coreos/bbolt v1.3.2 // indirect + github.com/coreos/etcd v3.3.10+incompatible // indirect + github.com/coreos/go-semver v0.2.0 // indirect + github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e // indirect + github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect + github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect + github.com/fsnotify/fsnotify v1.4.7 + github.com/gogo/protobuf v1.2.1 // indirect + github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect + github.com/google/btree v1.0.0 // indirect + github.com/gorilla/websocket v1.4.0 // indirect + github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 // indirect + github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect + github.com/grpc-ecosystem/grpc-gateway v1.9.0 // indirect + github.com/hashicorp/hcl v1.0.0 + github.com/jonboulle/clockwork v0.1.0 // indirect + github.com/magiconair/properties v1.8.0 + github.com/mitchellh/mapstructure v1.1.2 + github.com/pelletier/go-toml v1.2.0 + github.com/prometheus/client_golang v0.9.3 // indirect + github.com/soheilhy/cmux v0.1.4 // indirect + github.com/spf13/afero v1.1.2 + github.com/spf13/cast v1.3.0 + github.com/spf13/jwalterweatherman v1.0.0 + github.com/spf13/pflag v1.0.3 + github.com/stretchr/testify v1.2.2 + github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect + github.com/ugorji/go v1.1.4 // indirect + github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect + github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77 + go.etcd.io/bbolt v1.3.2 // indirect + go.uber.org/atomic v1.4.0 // indirect + go.uber.org/multierr v1.1.0 // indirect + go.uber.org/zap v1.10.0 // indirect + golang.org/x/net v0.0.0-20190522155817-f3200d17e092 // indirect + golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect + google.golang.org/grpc v1.21.0 // indirect + gopkg.in/yaml.v2 v2.2.2 +) diff --git a/vendor/github.com/spf13/viper/go.sum b/vendor/github.com/spf13/viper/go.sum new file mode 100644 index 000000000..97afaffe2 --- /dev/null +++ b/vendor/github.com/spf13/viper/go.sum @@ -0,0 +1,178 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6 h1:G1bPvciwNyF7IUmKXNt9Ak3m6u9DE1rF+RmtIkBpVdA= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible h1:jFneRYjIvLMLhDLCzuTuU4rSJUjRplcJQ7pD7MnhC04= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0 h1:bM6ZAFZmc/wPFaRDi0d5L7hGEZEx/2u+Tmr2evNHDiI= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.1.4 h1:j4s+tAvLfL3bZyefP2SEWmhBzmuIlH/eqNuPdFPgngw= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77 h1:ESFSdwYZvkeru3RtdrYueztKhOBCSAAzS4Gf+k0tEow= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.21.0 h1:G+97AoqBnmZIT91cLG/EkCoK9NSelj64P8bOHHNmGn0= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/vendor/github.com/spf13/viper/util.go b/vendor/github.com/spf13/viper/util.go new file mode 100644 index 000000000..952cad44c --- /dev/null +++ b/vendor/github.com/spf13/viper/util.go @@ -0,0 +1,221 @@ +// Copyright © 2014 Steve Francia . +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +// Viper is a application configuration system. +// It believes that applications can be configured a variety of ways +// via flags, ENVIRONMENT variables, configuration files retrieved +// from the file system, or a remote key/value store. + +package viper + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "unicode" + + "github.com/spf13/afero" + "github.com/spf13/cast" + jww "github.com/spf13/jwalterweatherman" +) + +// ConfigParseError denotes failing to parse configuration file. +type ConfigParseError struct { + err error +} + +// Error returns the formatted configuration error. +func (pe ConfigParseError) Error() string { + return fmt.Sprintf("While parsing config: %s", pe.err.Error()) +} + +// toCaseInsensitiveValue checks if the value is a map; +// if so, create a copy and lower-case the keys recursively. +func toCaseInsensitiveValue(value interface{}) interface{} { + switch v := value.(type) { + case map[interface{}]interface{}: + value = copyAndInsensitiviseMap(cast.ToStringMap(v)) + case map[string]interface{}: + value = copyAndInsensitiviseMap(v) + } + + return value +} + +// copyAndInsensitiviseMap behaves like insensitiviseMap, but creates a copy of +// any map it makes case insensitive. +func copyAndInsensitiviseMap(m map[string]interface{}) map[string]interface{} { + nm := make(map[string]interface{}) + + for key, val := range m { + lkey := strings.ToLower(key) + switch v := val.(type) { + case map[interface{}]interface{}: + nm[lkey] = copyAndInsensitiviseMap(cast.ToStringMap(v)) + case map[string]interface{}: + nm[lkey] = copyAndInsensitiviseMap(v) + default: + nm[lkey] = v + } + } + + return nm +} + +func insensitiviseMap(m map[string]interface{}) { + for key, val := range m { + switch val.(type) { + case map[interface{}]interface{}: + // nested map: cast and recursively insensitivise + val = cast.ToStringMap(val) + insensitiviseMap(val.(map[string]interface{})) + case map[string]interface{}: + // nested map: recursively insensitivise + insensitiviseMap(val.(map[string]interface{})) + } + + lower := strings.ToLower(key) + if key != lower { + // remove old key (not lower-cased) + delete(m, key) + } + // update map + m[lower] = val + } +} + +func absPathify(inPath string) string { + jww.INFO.Println("Trying to resolve absolute path to", inPath) + + if strings.HasPrefix(inPath, "$HOME") { + inPath = userHomeDir() + inPath[5:] + } + + if strings.HasPrefix(inPath, "$") { + end := strings.Index(inPath, string(os.PathSeparator)) + inPath = os.Getenv(inPath[1:end]) + inPath[end:] + } + + if filepath.IsAbs(inPath) { + return filepath.Clean(inPath) + } + + p, err := filepath.Abs(inPath) + if err == nil { + return filepath.Clean(p) + } + + jww.ERROR.Println("Couldn't discover absolute path") + jww.ERROR.Println(err) + return "" +} + +// Check if File / Directory Exists +func exists(fs afero.Fs, path string) (bool, error) { + _, err := fs.Stat(path) + if err == nil { + return true, nil + } + if os.IsNotExist(err) { + return false, nil + } + return false, err +} + +func stringInSlice(a string, list []string) bool { + for _, b := range list { + if b == a { + return true + } + } + return false +} + +func userHomeDir() string { + if runtime.GOOS == "windows" { + home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH") + if home == "" { + home = os.Getenv("USERPROFILE") + } + return home + } + return os.Getenv("HOME") +} + +func safeMul(a, b uint) uint { + c := a * b + if a > 1 && b > 1 && c/b != a { + return 0 + } + return c +} + +// parseSizeInBytes converts strings like 1GB or 12 mb into an unsigned integer number of bytes +func parseSizeInBytes(sizeStr string) uint { + sizeStr = strings.TrimSpace(sizeStr) + lastChar := len(sizeStr) - 1 + multiplier := uint(1) + + if lastChar > 0 { + if sizeStr[lastChar] == 'b' || sizeStr[lastChar] == 'B' { + if lastChar > 1 { + switch unicode.ToLower(rune(sizeStr[lastChar-1])) { + case 'k': + multiplier = 1 << 10 + sizeStr = strings.TrimSpace(sizeStr[:lastChar-1]) + case 'm': + multiplier = 1 << 20 + sizeStr = strings.TrimSpace(sizeStr[:lastChar-1]) + case 'g': + multiplier = 1 << 30 + sizeStr = strings.TrimSpace(sizeStr[:lastChar-1]) + default: + multiplier = 1 + sizeStr = strings.TrimSpace(sizeStr[:lastChar]) + } + } + } + } + + size := cast.ToInt(sizeStr) + if size < 0 { + size = 0 + } + + return safeMul(uint(size), multiplier) +} + +// deepSearch scans deep maps, following the key indexes listed in the +// sequence "path". +// The last value is expected to be another map, and is returned. +// +// In case intermediate keys do not exist, or map to a non-map value, +// a new map is created and inserted, and the search continues from there: +// the initial map "m" may be modified! +func deepSearch(m map[string]interface{}, path []string) map[string]interface{} { + for _, k := range path { + m2, ok := m[k] + if !ok { + // intermediate key does not exist + // => create it and continue from there + m3 := make(map[string]interface{}) + m[k] = m3 + m = m3 + continue + } + m3, ok := m2.(map[string]interface{}) + if !ok { + // intermediate key is a value + // => replace with a new map + m3 = make(map[string]interface{}) + m[k] = m3 + } + // continue search from here + m = m3 + } + return m +} diff --git a/vendor/github.com/spf13/viper/viper.go b/vendor/github.com/spf13/viper/viper.go new file mode 100644 index 000000000..a3d37f8c2 --- /dev/null +++ b/vendor/github.com/spf13/viper/viper.go @@ -0,0 +1,1868 @@ +// Copyright © 2014 Steve Francia . +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +// Viper is a application configuration system. +// It believes that applications can be configured a variety of ways +// via flags, ENVIRONMENT variables, configuration files retrieved +// from the file system, or a remote key/value store. + +// Each item takes precedence over the item below it: + +// overrides +// flag +// env +// config +// key/value store +// default + +package viper + +import ( + "bytes" + "encoding/csv" + "encoding/json" + "fmt" + "io" + "log" + "os" + "path/filepath" + "reflect" + "strings" + "sync" + "time" + + yaml "gopkg.in/yaml.v2" + + "github.com/fsnotify/fsnotify" + "github.com/hashicorp/hcl" + "github.com/hashicorp/hcl/hcl/printer" + "github.com/magiconair/properties" + "github.com/mitchellh/mapstructure" + toml "github.com/pelletier/go-toml" + "github.com/spf13/afero" + "github.com/spf13/cast" + jww "github.com/spf13/jwalterweatherman" + "github.com/spf13/pflag" +) + +// ConfigMarshalError happens when failing to marshal the configuration. +type ConfigMarshalError struct { + err error +} + +// Error returns the formatted configuration error. +func (e ConfigMarshalError) Error() string { + return fmt.Sprintf("While marshaling config: %s", e.err.Error()) +} + +var v *Viper + +type RemoteResponse struct { + Value []byte + Error error +} + +func init() { + v = New() +} + +type remoteConfigFactory interface { + Get(rp RemoteProvider) (io.Reader, error) + Watch(rp RemoteProvider) (io.Reader, error) + WatchChannel(rp RemoteProvider) (<-chan *RemoteResponse, chan bool) +} + +// RemoteConfig is optional, see the remote package +var RemoteConfig remoteConfigFactory + +// UnsupportedConfigError denotes encountering an unsupported +// configuration filetype. +type UnsupportedConfigError string + +// Error returns the formatted configuration error. +func (str UnsupportedConfigError) Error() string { + return fmt.Sprintf("Unsupported Config Type %q", string(str)) +} + +// UnsupportedRemoteProviderError denotes encountering an unsupported remote +// provider. Currently only etcd and Consul are supported. +type UnsupportedRemoteProviderError string + +// Error returns the formatted remote provider error. +func (str UnsupportedRemoteProviderError) Error() string { + return fmt.Sprintf("Unsupported Remote Provider Type %q", string(str)) +} + +// RemoteConfigError denotes encountering an error while trying to +// pull the configuration from the remote provider. +type RemoteConfigError string + +// Error returns the formatted remote provider error +func (rce RemoteConfigError) Error() string { + return fmt.Sprintf("Remote Configurations Error: %s", string(rce)) +} + +// ConfigFileNotFoundError denotes failing to find configuration file. +type ConfigFileNotFoundError struct { + name, locations string +} + +// Error returns the formatted configuration error. +func (fnfe ConfigFileNotFoundError) Error() string { + return fmt.Sprintf("Config File %q Not Found in %q", fnfe.name, fnfe.locations) +} + +// A DecoderConfigOption can be passed to viper.Unmarshal to configure +// mapstructure.DecoderConfig options +type DecoderConfigOption func(*mapstructure.DecoderConfig) + +// DecodeHook returns a DecoderConfigOption which overrides the default +// DecoderConfig.DecodeHook value, the default is: +// +// mapstructure.ComposeDecodeHookFunc( +// mapstructure.StringToTimeDurationHookFunc(), +// mapstructure.StringToSliceHookFunc(","), +// ) +func DecodeHook(hook mapstructure.DecodeHookFunc) DecoderConfigOption { + return func(c *mapstructure.DecoderConfig) { + c.DecodeHook = hook + } +} + +// Viper is a prioritized configuration registry. It +// maintains a set of configuration sources, fetches +// values to populate those, and provides them according +// to the source's priority. +// The priority of the sources is the following: +// 1. overrides +// 2. flags +// 3. env. variables +// 4. config file +// 5. key/value store +// 6. defaults +// +// For example, if values from the following sources were loaded: +// +// Defaults : { +// "secret": "", +// "user": "default", +// "endpoint": "https://localhost" +// } +// Config : { +// "user": "root" +// "secret": "defaultsecret" +// } +// Env : { +// "secret": "somesecretkey" +// } +// +// The resulting config will have the following values: +// +// { +// "secret": "somesecretkey", +// "user": "root", +// "endpoint": "https://localhost" +// } +type Viper struct { + // Delimiter that separates a list of keys + // used to access a nested value in one go + keyDelim string + + // A set of paths to look for the config file in + configPaths []string + + // The filesystem to read config from. + fs afero.Fs + + // A set of remote providers to search for the configuration + remoteProviders []*defaultRemoteProvider + + // Name of file to look for inside the path + configName string + configFile string + configType string + configPermissions os.FileMode + envPrefix string + + automaticEnvApplied bool + envKeyReplacer *strings.Replacer + allowEmptyEnv bool + + config map[string]interface{} + override map[string]interface{} + defaults map[string]interface{} + kvstore map[string]interface{} + pflags map[string]FlagValue + env map[string]string + aliases map[string]string + typeByDefValue bool + + // Store read properties on the object so that we can write back in order with comments. + // This will only be used if the configuration read is a properties file. + properties *properties.Properties + + onConfigChange func(fsnotify.Event) +} + +// New returns an initialized Viper instance. +func New() *Viper { + v := new(Viper) + v.keyDelim = "." + v.configName = "config" + v.configPermissions = os.FileMode(0644) + v.fs = afero.NewOsFs() + v.config = make(map[string]interface{}) + v.override = make(map[string]interface{}) + v.defaults = make(map[string]interface{}) + v.kvstore = make(map[string]interface{}) + v.pflags = make(map[string]FlagValue) + v.env = make(map[string]string) + v.aliases = make(map[string]string) + v.typeByDefValue = false + + return v +} + +// Intended for testing, will reset all to default settings. +// In the public interface for the viper package so applications +// can use it in their testing as well. +func Reset() { + v = New() + SupportedExts = []string{"json", "toml", "yaml", "yml", "properties", "props", "prop", "hcl"} + SupportedRemoteProviders = []string{"etcd", "consul"} +} + +type defaultRemoteProvider struct { + provider string + endpoint string + path string + secretKeyring string +} + +func (rp defaultRemoteProvider) Provider() string { + return rp.provider +} + +func (rp defaultRemoteProvider) Endpoint() string { + return rp.endpoint +} + +func (rp defaultRemoteProvider) Path() string { + return rp.path +} + +func (rp defaultRemoteProvider) SecretKeyring() string { + return rp.secretKeyring +} + +// RemoteProvider stores the configuration necessary +// to connect to a remote key/value store. +// Optional secretKeyring to unencrypt encrypted values +// can be provided. +type RemoteProvider interface { + Provider() string + Endpoint() string + Path() string + SecretKeyring() string +} + +// SupportedExts are universally supported extensions. +var SupportedExts = []string{"json", "toml", "yaml", "yml", "properties", "props", "prop", "hcl"} + +// SupportedRemoteProviders are universally supported remote providers. +var SupportedRemoteProviders = []string{"etcd", "consul"} + +func OnConfigChange(run func(in fsnotify.Event)) { v.OnConfigChange(run) } +func (v *Viper) OnConfigChange(run func(in fsnotify.Event)) { + v.onConfigChange = run +} + +func WatchConfig() { v.WatchConfig() } + +func (v *Viper) WatchConfig() { + initWG := sync.WaitGroup{} + initWG.Add(1) + go func() { + watcher, err := fsnotify.NewWatcher() + if err != nil { + log.Fatal(err) + } + defer watcher.Close() + // we have to watch the entire directory to pick up renames/atomic saves in a cross-platform way + filename, err := v.getConfigFile() + if err != nil { + log.Printf("error: %v\n", err) + return + } + + configFile := filepath.Clean(filename) + configDir, _ := filepath.Split(configFile) + realConfigFile, _ := filepath.EvalSymlinks(filename) + + eventsWG := sync.WaitGroup{} + eventsWG.Add(1) + go func() { + for { + select { + case event, ok := <-watcher.Events: + if !ok { // 'Events' channel is closed + eventsWG.Done() + return + } + currentConfigFile, _ := filepath.EvalSymlinks(filename) + // we only care about the config file with the following cases: + // 1 - if the config file was modified or created + // 2 - if the real path to the config file changed (eg: k8s ConfigMap replacement) + const writeOrCreateMask = fsnotify.Write | fsnotify.Create + if (filepath.Clean(event.Name) == configFile && + event.Op&writeOrCreateMask != 0) || + (currentConfigFile != "" && currentConfigFile != realConfigFile) { + realConfigFile = currentConfigFile + err := v.ReadInConfig() + if err != nil { + log.Printf("error reading config file: %v\n", err) + } + if v.onConfigChange != nil { + v.onConfigChange(event) + } + } else if filepath.Clean(event.Name) == configFile && + event.Op&fsnotify.Remove&fsnotify.Remove != 0 { + eventsWG.Done() + return + } + + case err, ok := <-watcher.Errors: + if ok { // 'Errors' channel is not closed + log.Printf("watcher error: %v\n", err) + } + eventsWG.Done() + return + } + } + }() + watcher.Add(configDir) + initWG.Done() // done initalizing the watch in this go routine, so the parent routine can move on... + eventsWG.Wait() // now, wait for event loop to end in this go-routine... + }() + initWG.Wait() // make sure that the go routine above fully ended before returning +} + +// SetConfigFile explicitly defines the path, name and extension of the config file. +// Viper will use this and not check any of the config paths. +func SetConfigFile(in string) { v.SetConfigFile(in) } +func (v *Viper) SetConfigFile(in string) { + if in != "" { + v.configFile = in + } +} + +// SetEnvPrefix defines a prefix that ENVIRONMENT variables will use. +// E.g. if your prefix is "spf", the env registry will look for env +// variables that start with "SPF_". +func SetEnvPrefix(in string) { v.SetEnvPrefix(in) } +func (v *Viper) SetEnvPrefix(in string) { + if in != "" { + v.envPrefix = in + } +} + +func (v *Viper) mergeWithEnvPrefix(in string) string { + if v.envPrefix != "" { + return strings.ToUpper(v.envPrefix + "_" + in) + } + + return strings.ToUpper(in) +} + +// AllowEmptyEnv tells Viper to consider set, +// but empty environment variables as valid values instead of falling back. +// For backward compatibility reasons this is false by default. +func AllowEmptyEnv(allowEmptyEnv bool) { v.AllowEmptyEnv(allowEmptyEnv) } +func (v *Viper) AllowEmptyEnv(allowEmptyEnv bool) { + v.allowEmptyEnv = allowEmptyEnv +} + +// TODO: should getEnv logic be moved into find(). Can generalize the use of +// rewriting keys many things, Ex: Get('someKey') -> some_key +// (camel case to snake case for JSON keys perhaps) + +// getEnv is a wrapper around os.Getenv which replaces characters in the original +// key. This allows env vars which have different keys than the config object +// keys. +func (v *Viper) getEnv(key string) (string, bool) { + if v.envKeyReplacer != nil { + key = v.envKeyReplacer.Replace(key) + } + + val, ok := os.LookupEnv(key) + + return val, ok && (v.allowEmptyEnv || val != "") +} + +// ConfigFileUsed returns the file used to populate the config registry. +func ConfigFileUsed() string { return v.ConfigFileUsed() } +func (v *Viper) ConfigFileUsed() string { return v.configFile } + +// AddConfigPath adds a path for Viper to search for the config file in. +// Can be called multiple times to define multiple search paths. +func AddConfigPath(in string) { v.AddConfigPath(in) } +func (v *Viper) AddConfigPath(in string) { + if in != "" { + absin := absPathify(in) + jww.INFO.Println("adding", absin, "to paths to search") + if !stringInSlice(absin, v.configPaths) { + v.configPaths = append(v.configPaths, absin) + } + } +} + +// AddRemoteProvider adds a remote configuration source. +// Remote Providers are searched in the order they are added. +// provider is a string value, "etcd" or "consul" are currently supported. +// endpoint is the url. etcd requires http://ip:port consul requires ip:port +// path is the path in the k/v store to retrieve configuration +// To retrieve a config file called myapp.json from /configs/myapp.json +// you should set path to /configs and set config name (SetConfigName()) to +// "myapp" +func AddRemoteProvider(provider, endpoint, path string) error { + return v.AddRemoteProvider(provider, endpoint, path) +} +func (v *Viper) AddRemoteProvider(provider, endpoint, path string) error { + if !stringInSlice(provider, SupportedRemoteProviders) { + return UnsupportedRemoteProviderError(provider) + } + if provider != "" && endpoint != "" { + jww.INFO.Printf("adding %s:%s to remote provider list", provider, endpoint) + rp := &defaultRemoteProvider{ + endpoint: endpoint, + provider: provider, + path: path, + } + if !v.providerPathExists(rp) { + v.remoteProviders = append(v.remoteProviders, rp) + } + } + return nil +} + +// AddSecureRemoteProvider adds a remote configuration source. +// Secure Remote Providers are searched in the order they are added. +// provider is a string value, "etcd" or "consul" are currently supported. +// endpoint is the url. etcd requires http://ip:port consul requires ip:port +// secretkeyring is the filepath to your openpgp secret keyring. e.g. /etc/secrets/myring.gpg +// path is the path in the k/v store to retrieve configuration +// To retrieve a config file called myapp.json from /configs/myapp.json +// you should set path to /configs and set config name (SetConfigName()) to +// "myapp" +// Secure Remote Providers are implemented with github.com/xordataexchange/crypt +func AddSecureRemoteProvider(provider, endpoint, path, secretkeyring string) error { + return v.AddSecureRemoteProvider(provider, endpoint, path, secretkeyring) +} + +func (v *Viper) AddSecureRemoteProvider(provider, endpoint, path, secretkeyring string) error { + if !stringInSlice(provider, SupportedRemoteProviders) { + return UnsupportedRemoteProviderError(provider) + } + if provider != "" && endpoint != "" { + jww.INFO.Printf("adding %s:%s to remote provider list", provider, endpoint) + rp := &defaultRemoteProvider{ + endpoint: endpoint, + provider: provider, + path: path, + secretKeyring: secretkeyring, + } + if !v.providerPathExists(rp) { + v.remoteProviders = append(v.remoteProviders, rp) + } + } + return nil +} + +func (v *Viper) providerPathExists(p *defaultRemoteProvider) bool { + for _, y := range v.remoteProviders { + if reflect.DeepEqual(y, p) { + return true + } + } + return false +} + +// searchMap recursively searches for a value for path in source map. +// Returns nil if not found. +// Note: This assumes that the path entries and map keys are lower cased. +func (v *Viper) searchMap(source map[string]interface{}, path []string) interface{} { + if len(path) == 0 { + return source + } + + next, ok := source[path[0]] + if ok { + // Fast path + if len(path) == 1 { + return next + } + + // Nested case + switch next.(type) { + case map[interface{}]interface{}: + return v.searchMap(cast.ToStringMap(next), path[1:]) + case map[string]interface{}: + // Type assertion is safe here since it is only reached + // if the type of `next` is the same as the type being asserted + return v.searchMap(next.(map[string]interface{}), path[1:]) + default: + // got a value but nested key expected, return "nil" for not found + return nil + } + } + return nil +} + +// searchMapWithPathPrefixes recursively searches for a value for path in source map. +// +// While searchMap() considers each path element as a single map key, this +// function searches for, and prioritizes, merged path elements. +// e.g., if in the source, "foo" is defined with a sub-key "bar", and "foo.bar" +// is also defined, this latter value is returned for path ["foo", "bar"]. +// +// This should be useful only at config level (other maps may not contain dots +// in their keys). +// +// Note: This assumes that the path entries and map keys are lower cased. +func (v *Viper) searchMapWithPathPrefixes(source map[string]interface{}, path []string) interface{} { + if len(path) == 0 { + return source + } + + // search for path prefixes, starting from the longest one + for i := len(path); i > 0; i-- { + prefixKey := strings.ToLower(strings.Join(path[0:i], v.keyDelim)) + + next, ok := source[prefixKey] + if ok { + // Fast path + if i == len(path) { + return next + } + + // Nested case + var val interface{} + switch next.(type) { + case map[interface{}]interface{}: + val = v.searchMapWithPathPrefixes(cast.ToStringMap(next), path[i:]) + case map[string]interface{}: + // Type assertion is safe here since it is only reached + // if the type of `next` is the same as the type being asserted + val = v.searchMapWithPathPrefixes(next.(map[string]interface{}), path[i:]) + default: + // got a value but nested key expected, do nothing and look for next prefix + } + if val != nil { + return val + } + } + } + + // not found + return nil +} + +// isPathShadowedInDeepMap makes sure the given path is not shadowed somewhere +// on its path in the map. +// e.g., if "foo.bar" has a value in the given map, it “shadows” +// "foo.bar.baz" in a lower-priority map +func (v *Viper) isPathShadowedInDeepMap(path []string, m map[string]interface{}) string { + var parentVal interface{} + for i := 1; i < len(path); i++ { + parentVal = v.searchMap(m, path[0:i]) + if parentVal == nil { + // not found, no need to add more path elements + return "" + } + switch parentVal.(type) { + case map[interface{}]interface{}: + continue + case map[string]interface{}: + continue + default: + // parentVal is a regular value which shadows "path" + return strings.Join(path[0:i], v.keyDelim) + } + } + return "" +} + +// isPathShadowedInFlatMap makes sure the given path is not shadowed somewhere +// in a sub-path of the map. +// e.g., if "foo.bar" has a value in the given map, it “shadows” +// "foo.bar.baz" in a lower-priority map +func (v *Viper) isPathShadowedInFlatMap(path []string, mi interface{}) string { + // unify input map + var m map[string]interface{} + switch mi.(type) { + case map[string]string, map[string]FlagValue: + m = cast.ToStringMap(mi) + default: + return "" + } + + // scan paths + var parentKey string + for i := 1; i < len(path); i++ { + parentKey = strings.Join(path[0:i], v.keyDelim) + if _, ok := m[parentKey]; ok { + return parentKey + } + } + return "" +} + +// isPathShadowedInAutoEnv makes sure the given path is not shadowed somewhere +// in the environment, when automatic env is on. +// e.g., if "foo.bar" has a value in the environment, it “shadows” +// "foo.bar.baz" in a lower-priority map +func (v *Viper) isPathShadowedInAutoEnv(path []string) string { + var parentKey string + for i := 1; i < len(path); i++ { + parentKey = strings.Join(path[0:i], v.keyDelim) + if _, ok := v.getEnv(v.mergeWithEnvPrefix(parentKey)); ok { + return parentKey + } + } + return "" +} + +// SetTypeByDefaultValue enables or disables the inference of a key value's +// type when the Get function is used based upon a key's default value as +// opposed to the value returned based on the normal fetch logic. +// +// For example, if a key has a default value of []string{} and the same key +// is set via an environment variable to "a b c", a call to the Get function +// would return a string slice for the key if the key's type is inferred by +// the default value and the Get function would return: +// +// []string {"a", "b", "c"} +// +// Otherwise the Get function would return: +// +// "a b c" +func SetTypeByDefaultValue(enable bool) { v.SetTypeByDefaultValue(enable) } +func (v *Viper) SetTypeByDefaultValue(enable bool) { + v.typeByDefValue = enable +} + +// GetViper gets the global Viper instance. +func GetViper() *Viper { + return v +} + +// Get can retrieve any value given the key to use. +// Get is case-insensitive for a key. +// Get has the behavior of returning the value associated with the first +// place from where it is set. Viper will check in the following order: +// override, flag, env, config file, key/value store, default +// +// Get returns an interface. For a specific value use one of the Get____ methods. +func Get(key string) interface{} { return v.Get(key) } +func (v *Viper) Get(key string) interface{} { + lcaseKey := strings.ToLower(key) + val := v.find(lcaseKey) + if val == nil { + return nil + } + + if v.typeByDefValue { + // TODO(bep) this branch isn't covered by a single test. + valType := val + path := strings.Split(lcaseKey, v.keyDelim) + defVal := v.searchMap(v.defaults, path) + if defVal != nil { + valType = defVal + } + + switch valType.(type) { + case bool: + return cast.ToBool(val) + case string: + return cast.ToString(val) + case int32, int16, int8, int: + return cast.ToInt(val) + case uint: + return cast.ToUint(val) + case uint32: + return cast.ToUint32(val) + case uint64: + return cast.ToUint64(val) + case int64: + return cast.ToInt64(val) + case float64, float32: + return cast.ToFloat64(val) + case time.Time: + return cast.ToTime(val) + case time.Duration: + return cast.ToDuration(val) + case []string: + return cast.ToStringSlice(val) + } + } + + return val +} + +// Sub returns new Viper instance representing a sub tree of this instance. +// Sub is case-insensitive for a key. +func Sub(key string) *Viper { return v.Sub(key) } +func (v *Viper) Sub(key string) *Viper { + subv := New() + data := v.Get(key) + if data == nil { + return nil + } + + if reflect.TypeOf(data).Kind() == reflect.Map { + subv.config = cast.ToStringMap(data) + return subv + } + return nil +} + +// GetString returns the value associated with the key as a string. +func GetString(key string) string { return v.GetString(key) } +func (v *Viper) GetString(key string) string { + return cast.ToString(v.Get(key)) +} + +// GetBool returns the value associated with the key as a boolean. +func GetBool(key string) bool { return v.GetBool(key) } +func (v *Viper) GetBool(key string) bool { + return cast.ToBool(v.Get(key)) +} + +// GetInt returns the value associated with the key as an integer. +func GetInt(key string) int { return v.GetInt(key) } +func (v *Viper) GetInt(key string) int { + return cast.ToInt(v.Get(key)) +} + +// GetInt32 returns the value associated with the key as an integer. +func GetInt32(key string) int32 { return v.GetInt32(key) } +func (v *Viper) GetInt32(key string) int32 { + return cast.ToInt32(v.Get(key)) +} + +// GetInt64 returns the value associated with the key as an integer. +func GetInt64(key string) int64 { return v.GetInt64(key) } +func (v *Viper) GetInt64(key string) int64 { + return cast.ToInt64(v.Get(key)) +} + +// GetUint returns the value associated with the key as an unsigned integer. +func GetUint(key string) uint { return v.GetUint(key) } +func (v *Viper) GetUint(key string) uint { + return cast.ToUint(v.Get(key)) +} + +// GetUint32 returns the value associated with the key as an unsigned integer. +func GetUint32(key string) uint32 { return v.GetUint32(key) } +func (v *Viper) GetUint32(key string) uint32 { + return cast.ToUint32(v.Get(key)) +} + +// GetUint64 returns the value associated with the key as an unsigned integer. +func GetUint64(key string) uint64 { return v.GetUint64(key) } +func (v *Viper) GetUint64(key string) uint64 { + return cast.ToUint64(v.Get(key)) +} + +// GetFloat64 returns the value associated with the key as a float64. +func GetFloat64(key string) float64 { return v.GetFloat64(key) } +func (v *Viper) GetFloat64(key string) float64 { + return cast.ToFloat64(v.Get(key)) +} + +// GetTime returns the value associated with the key as time. +func GetTime(key string) time.Time { return v.GetTime(key) } +func (v *Viper) GetTime(key string) time.Time { + return cast.ToTime(v.Get(key)) +} + +// GetDuration returns the value associated with the key as a duration. +func GetDuration(key string) time.Duration { return v.GetDuration(key) } +func (v *Viper) GetDuration(key string) time.Duration { + return cast.ToDuration(v.Get(key)) +} + +// GetStringSlice returns the value associated with the key as a slice of strings. +func GetStringSlice(key string) []string { return v.GetStringSlice(key) } +func (v *Viper) GetStringSlice(key string) []string { + return cast.ToStringSlice(v.Get(key)) +} + +// GetStringMap returns the value associated with the key as a map of interfaces. +func GetStringMap(key string) map[string]interface{} { return v.GetStringMap(key) } +func (v *Viper) GetStringMap(key string) map[string]interface{} { + return cast.ToStringMap(v.Get(key)) +} + +// GetStringMapString returns the value associated with the key as a map of strings. +func GetStringMapString(key string) map[string]string { return v.GetStringMapString(key) } +func (v *Viper) GetStringMapString(key string) map[string]string { + return cast.ToStringMapString(v.Get(key)) +} + +// GetStringMapStringSlice returns the value associated with the key as a map to a slice of strings. +func GetStringMapStringSlice(key string) map[string][]string { return v.GetStringMapStringSlice(key) } +func (v *Viper) GetStringMapStringSlice(key string) map[string][]string { + return cast.ToStringMapStringSlice(v.Get(key)) +} + +// GetSizeInBytes returns the size of the value associated with the given key +// in bytes. +func GetSizeInBytes(key string) uint { return v.GetSizeInBytes(key) } +func (v *Viper) GetSizeInBytes(key string) uint { + sizeStr := cast.ToString(v.Get(key)) + return parseSizeInBytes(sizeStr) +} + +// UnmarshalKey takes a single key and unmarshals it into a Struct. +func UnmarshalKey(key string, rawVal interface{}, opts ...DecoderConfigOption) error { + return v.UnmarshalKey(key, rawVal, opts...) +} +func (v *Viper) UnmarshalKey(key string, rawVal interface{}, opts ...DecoderConfigOption) error { + err := decode(v.Get(key), defaultDecoderConfig(rawVal, opts...)) + + if err != nil { + return err + } + + return nil +} + +// Unmarshal unmarshals the config into a Struct. Make sure that the tags +// on the fields of the structure are properly set. +func Unmarshal(rawVal interface{}, opts ...DecoderConfigOption) error { + return v.Unmarshal(rawVal, opts...) +} +func (v *Viper) Unmarshal(rawVal interface{}, opts ...DecoderConfigOption) error { + err := decode(v.AllSettings(), defaultDecoderConfig(rawVal, opts...)) + + if err != nil { + return err + } + + return nil +} + +// defaultDecoderConfig returns default mapsstructure.DecoderConfig with suppot +// of time.Duration values & string slices +func defaultDecoderConfig(output interface{}, opts ...DecoderConfigOption) *mapstructure.DecoderConfig { + c := &mapstructure.DecoderConfig{ + Metadata: nil, + Result: output, + WeaklyTypedInput: true, + DecodeHook: mapstructure.ComposeDecodeHookFunc( + mapstructure.StringToTimeDurationHookFunc(), + mapstructure.StringToSliceHookFunc(","), + ), + } + for _, opt := range opts { + opt(c) + } + return c +} + +// A wrapper around mapstructure.Decode that mimics the WeakDecode functionality +func decode(input interface{}, config *mapstructure.DecoderConfig) error { + decoder, err := mapstructure.NewDecoder(config) + if err != nil { + return err + } + return decoder.Decode(input) +} + +// UnmarshalExact unmarshals the config into a Struct, erroring if a field is nonexistent +// in the destination struct. +func (v *Viper) UnmarshalExact(rawVal interface{}) error { + config := defaultDecoderConfig(rawVal) + config.ErrorUnused = true + + err := decode(v.AllSettings(), config) + + if err != nil { + return err + } + + return nil +} + +// BindPFlags binds a full flag set to the configuration, using each flag's long +// name as the config key. +func BindPFlags(flags *pflag.FlagSet) error { return v.BindPFlags(flags) } +func (v *Viper) BindPFlags(flags *pflag.FlagSet) error { + return v.BindFlagValues(pflagValueSet{flags}) +} + +// BindPFlag binds a specific key to a pflag (as used by cobra). +// Example (where serverCmd is a Cobra instance): +// +// serverCmd.Flags().Int("port", 1138, "Port to run Application server on") +// Viper.BindPFlag("port", serverCmd.Flags().Lookup("port")) +// +func BindPFlag(key string, flag *pflag.Flag) error { return v.BindPFlag(key, flag) } +func (v *Viper) BindPFlag(key string, flag *pflag.Flag) error { + return v.BindFlagValue(key, pflagValue{flag}) +} + +// BindFlagValues binds a full FlagValue set to the configuration, using each flag's long +// name as the config key. +func BindFlagValues(flags FlagValueSet) error { return v.BindFlagValues(flags) } +func (v *Viper) BindFlagValues(flags FlagValueSet) (err error) { + flags.VisitAll(func(flag FlagValue) { + if err = v.BindFlagValue(flag.Name(), flag); err != nil { + return + } + }) + return nil +} + +// BindFlagValue binds a specific key to a FlagValue. +// Example (where serverCmd is a Cobra instance): +// +// serverCmd.Flags().Int("port", 1138, "Port to run Application server on") +// Viper.BindFlagValue("port", serverCmd.Flags().Lookup("port")) +// +func BindFlagValue(key string, flag FlagValue) error { return v.BindFlagValue(key, flag) } +func (v *Viper) BindFlagValue(key string, flag FlagValue) error { + if flag == nil { + return fmt.Errorf("flag for %q is nil", key) + } + v.pflags[strings.ToLower(key)] = flag + return nil +} + +// BindEnv binds a Viper key to a ENV variable. +// ENV variables are case sensitive. +// If only a key is provided, it will use the env key matching the key, uppercased. +// EnvPrefix will be used when set when env name is not provided. +func BindEnv(input ...string) error { return v.BindEnv(input...) } +func (v *Viper) BindEnv(input ...string) error { + var key, envkey string + if len(input) == 0 { + return fmt.Errorf("BindEnv missing key to bind to") + } + + key = strings.ToLower(input[0]) + + if len(input) == 1 { + envkey = v.mergeWithEnvPrefix(key) + } else { + envkey = input[1] + } + + v.env[key] = envkey + + return nil +} + +// Given a key, find the value. +// Viper will check in the following order: +// flag, env, config file, key/value store, default. +// Viper will check to see if an alias exists first. +// Note: this assumes a lower-cased key given. +func (v *Viper) find(lcaseKey string) interface{} { + + var ( + val interface{} + exists bool + path = strings.Split(lcaseKey, v.keyDelim) + nested = len(path) > 1 + ) + + // compute the path through the nested maps to the nested value + if nested && v.isPathShadowedInDeepMap(path, castMapStringToMapInterface(v.aliases)) != "" { + return nil + } + + // if the requested key is an alias, then return the proper key + lcaseKey = v.realKey(lcaseKey) + path = strings.Split(lcaseKey, v.keyDelim) + nested = len(path) > 1 + + // Set() override first + val = v.searchMap(v.override, path) + if val != nil { + return val + } + if nested && v.isPathShadowedInDeepMap(path, v.override) != "" { + return nil + } + + // PFlag override next + flag, exists := v.pflags[lcaseKey] + if exists && flag.HasChanged() { + switch flag.ValueType() { + case "int", "int8", "int16", "int32", "int64": + return cast.ToInt(flag.ValueString()) + case "bool": + return cast.ToBool(flag.ValueString()) + case "stringSlice": + s := strings.TrimPrefix(flag.ValueString(), "[") + s = strings.TrimSuffix(s, "]") + res, _ := readAsCSV(s) + return res + default: + return flag.ValueString() + } + } + if nested && v.isPathShadowedInFlatMap(path, v.pflags) != "" { + return nil + } + + // Env override next + if v.automaticEnvApplied { + // even if it hasn't been registered, if automaticEnv is used, + // check any Get request + if val, ok := v.getEnv(v.mergeWithEnvPrefix(lcaseKey)); ok { + return val + } + if nested && v.isPathShadowedInAutoEnv(path) != "" { + return nil + } + } + envkey, exists := v.env[lcaseKey] + if exists { + if val, ok := v.getEnv(envkey); ok { + return val + } + } + if nested && v.isPathShadowedInFlatMap(path, v.env) != "" { + return nil + } + + // Config file next + val = v.searchMapWithPathPrefixes(v.config, path) + if val != nil { + return val + } + if nested && v.isPathShadowedInDeepMap(path, v.config) != "" { + return nil + } + + // K/V store next + val = v.searchMap(v.kvstore, path) + if val != nil { + return val + } + if nested && v.isPathShadowedInDeepMap(path, v.kvstore) != "" { + return nil + } + + // Default next + val = v.searchMap(v.defaults, path) + if val != nil { + return val + } + if nested && v.isPathShadowedInDeepMap(path, v.defaults) != "" { + return nil + } + + // last chance: if no other value is returned and a flag does exist for the value, + // get the flag's value even if the flag's value has not changed + if flag, exists := v.pflags[lcaseKey]; exists { + switch flag.ValueType() { + case "int", "int8", "int16", "int32", "int64": + return cast.ToInt(flag.ValueString()) + case "bool": + return cast.ToBool(flag.ValueString()) + case "stringSlice": + s := strings.TrimPrefix(flag.ValueString(), "[") + s = strings.TrimSuffix(s, "]") + res, _ := readAsCSV(s) + return res + default: + return flag.ValueString() + } + } + // last item, no need to check shadowing + + return nil +} + +func readAsCSV(val string) ([]string, error) { + if val == "" { + return []string{}, nil + } + stringReader := strings.NewReader(val) + csvReader := csv.NewReader(stringReader) + return csvReader.Read() +} + +// IsSet checks to see if the key has been set in any of the data locations. +// IsSet is case-insensitive for a key. +func IsSet(key string) bool { return v.IsSet(key) } +func (v *Viper) IsSet(key string) bool { + lcaseKey := strings.ToLower(key) + val := v.find(lcaseKey) + return val != nil +} + +// AutomaticEnv has Viper check ENV variables for all. +// keys set in config, default & flags +func AutomaticEnv() { v.AutomaticEnv() } +func (v *Viper) AutomaticEnv() { + v.automaticEnvApplied = true +} + +// SetEnvKeyReplacer sets the strings.Replacer on the viper object +// Useful for mapping an environmental variable to a key that does +// not match it. +func SetEnvKeyReplacer(r *strings.Replacer) { v.SetEnvKeyReplacer(r) } +func (v *Viper) SetEnvKeyReplacer(r *strings.Replacer) { + v.envKeyReplacer = r +} + +// Aliases provide another accessor for the same key. +// This enables one to change a name without breaking the application +func RegisterAlias(alias string, key string) { v.RegisterAlias(alias, key) } +func (v *Viper) RegisterAlias(alias string, key string) { + v.registerAlias(alias, strings.ToLower(key)) +} + +func (v *Viper) registerAlias(alias string, key string) { + alias = strings.ToLower(alias) + if alias != key && alias != v.realKey(key) { + _, exists := v.aliases[alias] + + if !exists { + // if we alias something that exists in one of the maps to another + // name, we'll never be able to get that value using the original + // name, so move the config value to the new realkey. + if val, ok := v.config[alias]; ok { + delete(v.config, alias) + v.config[key] = val + } + if val, ok := v.kvstore[alias]; ok { + delete(v.kvstore, alias) + v.kvstore[key] = val + } + if val, ok := v.defaults[alias]; ok { + delete(v.defaults, alias) + v.defaults[key] = val + } + if val, ok := v.override[alias]; ok { + delete(v.override, alias) + v.override[key] = val + } + v.aliases[alias] = key + } + } else { + jww.WARN.Println("Creating circular reference alias", alias, key, v.realKey(key)) + } +} + +func (v *Viper) realKey(key string) string { + newkey, exists := v.aliases[key] + if exists { + jww.DEBUG.Println("Alias", key, "to", newkey) + return v.realKey(newkey) + } + return key +} + +// InConfig checks to see if the given key (or an alias) is in the config file. +func InConfig(key string) bool { return v.InConfig(key) } +func (v *Viper) InConfig(key string) bool { + // if the requested key is an alias, then return the proper key + key = v.realKey(key) + + _, exists := v.config[key] + return exists +} + +// SetDefault sets the default value for this key. +// SetDefault is case-insensitive for a key. +// Default only used when no value is provided by the user via flag, config or ENV. +func SetDefault(key string, value interface{}) { v.SetDefault(key, value) } +func (v *Viper) SetDefault(key string, value interface{}) { + // If alias passed in, then set the proper default + key = v.realKey(strings.ToLower(key)) + value = toCaseInsensitiveValue(value) + + path := strings.Split(key, v.keyDelim) + lastKey := strings.ToLower(path[len(path)-1]) + deepestMap := deepSearch(v.defaults, path[0:len(path)-1]) + + // set innermost value + deepestMap[lastKey] = value +} + +// Set sets the value for the key in the override register. +// Set is case-insensitive for a key. +// Will be used instead of values obtained via +// flags, config file, ENV, default, or key/value store. +func Set(key string, value interface{}) { v.Set(key, value) } +func (v *Viper) Set(key string, value interface{}) { + // If alias passed in, then set the proper override + key = v.realKey(strings.ToLower(key)) + value = toCaseInsensitiveValue(value) + + path := strings.Split(key, v.keyDelim) + lastKey := strings.ToLower(path[len(path)-1]) + deepestMap := deepSearch(v.override, path[0:len(path)-1]) + + // set innermost value + deepestMap[lastKey] = value +} + +// ReadInConfig will discover and load the configuration file from disk +// and key/value stores, searching in one of the defined paths. +func ReadInConfig() error { return v.ReadInConfig() } +func (v *Viper) ReadInConfig() error { + jww.INFO.Println("Attempting to read in config file") + filename, err := v.getConfigFile() + if err != nil { + return err + } + + if !stringInSlice(v.getConfigType(), SupportedExts) { + return UnsupportedConfigError(v.getConfigType()) + } + + jww.DEBUG.Println("Reading file: ", filename) + file, err := afero.ReadFile(v.fs, filename) + if err != nil { + return err + } + + config := make(map[string]interface{}) + + err = v.unmarshalReader(bytes.NewReader(file), config) + if err != nil { + return err + } + + v.config = config + return nil +} + +// MergeInConfig merges a new configuration with an existing config. +func MergeInConfig() error { return v.MergeInConfig() } +func (v *Viper) MergeInConfig() error { + jww.INFO.Println("Attempting to merge in config file") + filename, err := v.getConfigFile() + if err != nil { + return err + } + + if !stringInSlice(v.getConfigType(), SupportedExts) { + return UnsupportedConfigError(v.getConfigType()) + } + + file, err := afero.ReadFile(v.fs, filename) + if err != nil { + return err + } + + return v.MergeConfig(bytes.NewReader(file)) +} + +// ReadConfig will read a configuration file, setting existing keys to nil if the +// key does not exist in the file. +func ReadConfig(in io.Reader) error { return v.ReadConfig(in) } +func (v *Viper) ReadConfig(in io.Reader) error { + v.config = make(map[string]interface{}) + return v.unmarshalReader(in, v.config) +} + +// MergeConfig merges a new configuration with an existing config. +func MergeConfig(in io.Reader) error { return v.MergeConfig(in) } +func (v *Viper) MergeConfig(in io.Reader) error { + cfg := make(map[string]interface{}) + if err := v.unmarshalReader(in, cfg); err != nil { + return err + } + return v.MergeConfigMap(cfg) +} + +// MergeConfigMap merges the configuration from the map given with an existing config. +// Note that the map given may be modified. +func MergeConfigMap(cfg map[string]interface{}) error { return v.MergeConfigMap(cfg) } +func (v *Viper) MergeConfigMap(cfg map[string]interface{}) error { + if v.config == nil { + v.config = make(map[string]interface{}) + } + insensitiviseMap(cfg) + mergeMaps(cfg, v.config, nil) + return nil +} + +// WriteConfig writes the current configuration to a file. +func WriteConfig() error { return v.WriteConfig() } +func (v *Viper) WriteConfig() error { + filename, err := v.getConfigFile() + if err != nil { + return err + } + return v.writeConfig(filename, true) +} + +// SafeWriteConfig writes current configuration to file only if the file does not exist. +func SafeWriteConfig() error { return v.SafeWriteConfig() } +func (v *Viper) SafeWriteConfig() error { + filename, err := v.getConfigFile() + if err != nil { + return err + } + return v.writeConfig(filename, false) +} + +// WriteConfigAs writes current configuration to a given filename. +func WriteConfigAs(filename string) error { return v.WriteConfigAs(filename) } +func (v *Viper) WriteConfigAs(filename string) error { + return v.writeConfig(filename, true) +} + +// SafeWriteConfigAs writes current configuration to a given filename if it does not exist. +func SafeWriteConfigAs(filename string) error { return v.SafeWriteConfigAs(filename) } +func (v *Viper) SafeWriteConfigAs(filename string) error { + return v.writeConfig(filename, false) +} + +func writeConfig(filename string, force bool) error { return v.writeConfig(filename, force) } +func (v *Viper) writeConfig(filename string, force bool) error { + jww.INFO.Println("Attempting to write configuration to file.") + ext := filepath.Ext(filename) + if len(ext) <= 1 { + return fmt.Errorf("Filename: %s requires valid extension.", filename) + } + configType := ext[1:] + if !stringInSlice(configType, SupportedExts) { + return UnsupportedConfigError(configType) + } + if v.config == nil { + v.config = make(map[string]interface{}) + } + var flags int + if force == true { + flags = os.O_CREATE | os.O_TRUNC | os.O_WRONLY + } else { + if _, err := os.Stat(filename); os.IsNotExist(err) { + flags = os.O_WRONLY + } else { + return fmt.Errorf("File: %s exists. Use WriteConfig to overwrite.", filename) + } + } + f, err := v.fs.OpenFile(filename, flags, v.configPermissions) + if err != nil { + return err + } + return v.marshalWriter(f, configType) +} + +// Unmarshal a Reader into a map. +// Should probably be an unexported function. +func unmarshalReader(in io.Reader, c map[string]interface{}) error { + return v.unmarshalReader(in, c) +} +func (v *Viper) unmarshalReader(in io.Reader, c map[string]interface{}) error { + buf := new(bytes.Buffer) + buf.ReadFrom(in) + + switch strings.ToLower(v.getConfigType()) { + case "yaml", "yml": + if err := yaml.Unmarshal(buf.Bytes(), &c); err != nil { + return ConfigParseError{err} + } + + case "json": + if err := json.Unmarshal(buf.Bytes(), &c); err != nil { + return ConfigParseError{err} + } + + case "hcl": + obj, err := hcl.Parse(string(buf.Bytes())) + if err != nil { + return ConfigParseError{err} + } + if err = hcl.DecodeObject(&c, obj); err != nil { + return ConfigParseError{err} + } + + case "toml": + tree, err := toml.LoadReader(buf) + if err != nil { + return ConfigParseError{err} + } + tmap := tree.ToMap() + for k, v := range tmap { + c[k] = v + } + + case "properties", "props", "prop": + v.properties = properties.NewProperties() + var err error + if v.properties, err = properties.Load(buf.Bytes(), properties.UTF8); err != nil { + return ConfigParseError{err} + } + for _, key := range v.properties.Keys() { + value, _ := v.properties.Get(key) + // recursively build nested maps + path := strings.Split(key, ".") + lastKey := strings.ToLower(path[len(path)-1]) + deepestMap := deepSearch(c, path[0:len(path)-1]) + // set innermost value + deepestMap[lastKey] = value + } + } + + insensitiviseMap(c) + return nil +} + +// Marshal a map into Writer. +func marshalWriter(f afero.File, configType string) error { + return v.marshalWriter(f, configType) +} +func (v *Viper) marshalWriter(f afero.File, configType string) error { + c := v.AllSettings() + switch configType { + case "json": + b, err := json.MarshalIndent(c, "", " ") + if err != nil { + return ConfigMarshalError{err} + } + _, err = f.WriteString(string(b)) + if err != nil { + return ConfigMarshalError{err} + } + + case "hcl": + b, err := json.Marshal(c) + ast, err := hcl.Parse(string(b)) + if err != nil { + return ConfigMarshalError{err} + } + err = printer.Fprint(f, ast.Node) + if err != nil { + return ConfigMarshalError{err} + } + + case "prop", "props", "properties": + if v.properties == nil { + v.properties = properties.NewProperties() + } + p := v.properties + for _, key := range v.AllKeys() { + _, _, err := p.Set(key, v.GetString(key)) + if err != nil { + return ConfigMarshalError{err} + } + } + _, err := p.WriteComment(f, "#", properties.UTF8) + if err != nil { + return ConfigMarshalError{err} + } + + case "toml": + t, err := toml.TreeFromMap(c) + if err != nil { + return ConfigMarshalError{err} + } + s := t.String() + if _, err := f.WriteString(s); err != nil { + return ConfigMarshalError{err} + } + + case "yaml", "yml": + b, err := yaml.Marshal(c) + if err != nil { + return ConfigMarshalError{err} + } + if _, err = f.WriteString(string(b)); err != nil { + return ConfigMarshalError{err} + } + } + return nil +} + +func keyExists(k string, m map[string]interface{}) string { + lk := strings.ToLower(k) + for mk := range m { + lmk := strings.ToLower(mk) + if lmk == lk { + return mk + } + } + return "" +} + +func castToMapStringInterface( + src map[interface{}]interface{}) map[string]interface{} { + tgt := map[string]interface{}{} + for k, v := range src { + tgt[fmt.Sprintf("%v", k)] = v + } + return tgt +} + +func castMapStringToMapInterface(src map[string]string) map[string]interface{} { + tgt := map[string]interface{}{} + for k, v := range src { + tgt[k] = v + } + return tgt +} + +func castMapFlagToMapInterface(src map[string]FlagValue) map[string]interface{} { + tgt := map[string]interface{}{} + for k, v := range src { + tgt[k] = v + } + return tgt +} + +// mergeMaps merges two maps. The `itgt` parameter is for handling go-yaml's +// insistence on parsing nested structures as `map[interface{}]interface{}` +// instead of using a `string` as the key for nest structures beyond one level +// deep. Both map types are supported as there is a go-yaml fork that uses +// `map[string]interface{}` instead. +func mergeMaps( + src, tgt map[string]interface{}, itgt map[interface{}]interface{}) { + for sk, sv := range src { + tk := keyExists(sk, tgt) + if tk == "" { + jww.TRACE.Printf("tk=\"\", tgt[%s]=%v", sk, sv) + tgt[sk] = sv + if itgt != nil { + itgt[sk] = sv + } + continue + } + + tv, ok := tgt[tk] + if !ok { + jww.TRACE.Printf("tgt[%s] != ok, tgt[%s]=%v", tk, sk, sv) + tgt[sk] = sv + if itgt != nil { + itgt[sk] = sv + } + continue + } + + svType := reflect.TypeOf(sv) + tvType := reflect.TypeOf(tv) + if svType != tvType { + jww.ERROR.Printf( + "svType != tvType; key=%s, st=%v, tt=%v, sv=%v, tv=%v", + sk, svType, tvType, sv, tv) + continue + } + + jww.TRACE.Printf("processing key=%s, st=%v, tt=%v, sv=%v, tv=%v", + sk, svType, tvType, sv, tv) + + switch ttv := tv.(type) { + case map[interface{}]interface{}: + jww.TRACE.Printf("merging maps (must convert)") + tsv := sv.(map[interface{}]interface{}) + ssv := castToMapStringInterface(tsv) + stv := castToMapStringInterface(ttv) + mergeMaps(ssv, stv, ttv) + case map[string]interface{}: + jww.TRACE.Printf("merging maps") + mergeMaps(sv.(map[string]interface{}), ttv, nil) + default: + jww.TRACE.Printf("setting value") + tgt[tk] = sv + if itgt != nil { + itgt[tk] = sv + } + } + } +} + +// ReadRemoteConfig attempts to get configuration from a remote source +// and read it in the remote configuration registry. +func ReadRemoteConfig() error { return v.ReadRemoteConfig() } +func (v *Viper) ReadRemoteConfig() error { + return v.getKeyValueConfig() +} + +func WatchRemoteConfig() error { return v.WatchRemoteConfig() } +func (v *Viper) WatchRemoteConfig() error { + return v.watchKeyValueConfig() +} + +func (v *Viper) WatchRemoteConfigOnChannel() error { + return v.watchKeyValueConfigOnChannel() +} + +// Retrieve the first found remote configuration. +func (v *Viper) getKeyValueConfig() error { + if RemoteConfig == nil { + return RemoteConfigError("Enable the remote features by doing a blank import of the viper/remote package: '_ github.com/spf13/viper/remote'") + } + + for _, rp := range v.remoteProviders { + val, err := v.getRemoteConfig(rp) + if err != nil { + continue + } + v.kvstore = val + return nil + } + return RemoteConfigError("No Files Found") +} + +func (v *Viper) getRemoteConfig(provider RemoteProvider) (map[string]interface{}, error) { + reader, err := RemoteConfig.Get(provider) + if err != nil { + return nil, err + } + err = v.unmarshalReader(reader, v.kvstore) + return v.kvstore, err +} + +// Retrieve the first found remote configuration. +func (v *Viper) watchKeyValueConfigOnChannel() error { + for _, rp := range v.remoteProviders { + respc, _ := RemoteConfig.WatchChannel(rp) + //Todo: Add quit channel + go func(rc <-chan *RemoteResponse) { + for { + b := <-rc + reader := bytes.NewReader(b.Value) + v.unmarshalReader(reader, v.kvstore) + } + }(respc) + return nil + } + return RemoteConfigError("No Files Found") +} + +// Retrieve the first found remote configuration. +func (v *Viper) watchKeyValueConfig() error { + for _, rp := range v.remoteProviders { + val, err := v.watchRemoteConfig(rp) + if err != nil { + continue + } + v.kvstore = val + return nil + } + return RemoteConfigError("No Files Found") +} + +func (v *Viper) watchRemoteConfig(provider RemoteProvider) (map[string]interface{}, error) { + reader, err := RemoteConfig.Watch(provider) + if err != nil { + return nil, err + } + err = v.unmarshalReader(reader, v.kvstore) + return v.kvstore, err +} + +// AllKeys returns all keys holding a value, regardless of where they are set. +// Nested keys are returned with a v.keyDelim (= ".") separator +func AllKeys() []string { return v.AllKeys() } +func (v *Viper) AllKeys() []string { + m := map[string]bool{} + // add all paths, by order of descending priority to ensure correct shadowing + m = v.flattenAndMergeMap(m, castMapStringToMapInterface(v.aliases), "") + m = v.flattenAndMergeMap(m, v.override, "") + m = v.mergeFlatMap(m, castMapFlagToMapInterface(v.pflags)) + m = v.mergeFlatMap(m, castMapStringToMapInterface(v.env)) + m = v.flattenAndMergeMap(m, v.config, "") + m = v.flattenAndMergeMap(m, v.kvstore, "") + m = v.flattenAndMergeMap(m, v.defaults, "") + + // convert set of paths to list + a := []string{} + for x := range m { + a = append(a, x) + } + return a +} + +// flattenAndMergeMap recursively flattens the given map into a map[string]bool +// of key paths (used as a set, easier to manipulate than a []string): +// - each path is merged into a single key string, delimited with v.keyDelim (= ".") +// - if a path is shadowed by an earlier value in the initial shadow map, +// it is skipped. +// The resulting set of paths is merged to the given shadow set at the same time. +func (v *Viper) flattenAndMergeMap(shadow map[string]bool, m map[string]interface{}, prefix string) map[string]bool { + if shadow != nil && prefix != "" && shadow[prefix] { + // prefix is shadowed => nothing more to flatten + return shadow + } + if shadow == nil { + shadow = make(map[string]bool) + } + + var m2 map[string]interface{} + if prefix != "" { + prefix += v.keyDelim + } + for k, val := range m { + fullKey := prefix + k + switch val.(type) { + case map[string]interface{}: + m2 = val.(map[string]interface{}) + case map[interface{}]interface{}: + m2 = cast.ToStringMap(val) + default: + // immediate value + shadow[strings.ToLower(fullKey)] = true + continue + } + // recursively merge to shadow map + shadow = v.flattenAndMergeMap(shadow, m2, fullKey) + } + return shadow +} + +// mergeFlatMap merges the given maps, excluding values of the second map +// shadowed by values from the first map. +func (v *Viper) mergeFlatMap(shadow map[string]bool, m map[string]interface{}) map[string]bool { + // scan keys +outer: + for k, _ := range m { + path := strings.Split(k, v.keyDelim) + // scan intermediate paths + var parentKey string + for i := 1; i < len(path); i++ { + parentKey = strings.Join(path[0:i], v.keyDelim) + if shadow[parentKey] { + // path is shadowed, continue + continue outer + } + } + // add key + shadow[strings.ToLower(k)] = true + } + return shadow +} + +// AllSettings merges all settings and returns them as a map[string]interface{}. +func AllSettings() map[string]interface{} { return v.AllSettings() } +func (v *Viper) AllSettings() map[string]interface{} { + m := map[string]interface{}{} + // start from the list of keys, and construct the map one value at a time + for _, k := range v.AllKeys() { + value := v.Get(k) + if value == nil { + // should not happen, since AllKeys() returns only keys holding a value, + // check just in case anything changes + continue + } + path := strings.Split(k, v.keyDelim) + lastKey := strings.ToLower(path[len(path)-1]) + deepestMap := deepSearch(m, path[0:len(path)-1]) + // set innermost value + deepestMap[lastKey] = value + } + return m +} + +// SetFs sets the filesystem to use to read configuration. +func SetFs(fs afero.Fs) { v.SetFs(fs) } +func (v *Viper) SetFs(fs afero.Fs) { + v.fs = fs +} + +// SetConfigName sets name for the config file. +// Does not include extension. +func SetConfigName(in string) { v.SetConfigName(in) } +func (v *Viper) SetConfigName(in string) { + if in != "" { + v.configName = in + v.configFile = "" + } +} + +// SetConfigType sets the type of the configuration returned by the +// remote source, e.g. "json". +func SetConfigType(in string) { v.SetConfigType(in) } +func (v *Viper) SetConfigType(in string) { + if in != "" { + v.configType = in + } +} + +// SetConfigPermissions sets the permissions for the config file. +func SetConfigPermissions(perm os.FileMode) { v.SetConfigPermissions(perm) } +func (v *Viper) SetConfigPermissions(perm os.FileMode) { + v.configPermissions = perm.Perm() +} + +func (v *Viper) getConfigType() string { + if v.configType != "" { + return v.configType + } + + cf, err := v.getConfigFile() + if err != nil { + return "" + } + + ext := filepath.Ext(cf) + + if len(ext) > 1 { + return ext[1:] + } + + return "" +} + +func (v *Viper) getConfigFile() (string, error) { + if v.configFile == "" { + cf, err := v.findConfigFile() + if err != nil { + return "", err + } + v.configFile = cf + } + return v.configFile, nil +} + +func (v *Viper) searchInPath(in string) (filename string) { + jww.DEBUG.Println("Searching for config in ", in) + for _, ext := range SupportedExts { + jww.DEBUG.Println("Checking for", filepath.Join(in, v.configName+"."+ext)) + if b, _ := exists(v.fs, filepath.Join(in, v.configName+"."+ext)); b { + jww.DEBUG.Println("Found: ", filepath.Join(in, v.configName+"."+ext)) + return filepath.Join(in, v.configName+"."+ext) + } + } + + return "" +} + +// Search all configPaths for any config file. +// Returns the first path that exists (and is a config file). +func (v *Viper) findConfigFile() (string, error) { + jww.INFO.Println("Searching for config in ", v.configPaths) + + for _, cp := range v.configPaths { + file := v.searchInPath(cp) + if file != "" { + return file, nil + } + } + return "", ConfigFileNotFoundError{v.configName, fmt.Sprintf("%s", v.configPaths)} +} + +// Debug prints all configuration registries for debugging +// purposes. +func Debug() { v.Debug() } +func (v *Viper) Debug() { + fmt.Printf("Aliases:\n%#v\n", v.aliases) + fmt.Printf("Override:\n%#v\n", v.override) + fmt.Printf("PFlags:\n%#v\n", v.pflags) + fmt.Printf("Env:\n%#v\n", v.env) + fmt.Printf("Key/Value Store:\n%#v\n", v.kvstore) + fmt.Printf("Config:\n%#v\n", v.config) + fmt.Printf("Defaults:\n%#v\n", v.defaults) +} diff --git a/vendor/k8s.io/component-base/LICENSE b/vendor/k8s.io/component-base/LICENSE new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/vendor/k8s.io/component-base/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/k8s.io/component-base/cli/flag/ciphersuites_flag.go b/vendor/k8s.io/component-base/cli/flag/ciphersuites_flag.go new file mode 100644 index 000000000..4fecd1732 --- /dev/null +++ b/vendor/k8s.io/component-base/cli/flag/ciphersuites_flag.go @@ -0,0 +1,106 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flag + +import ( + "crypto/tls" + "fmt" + + "k8s.io/apimachinery/pkg/util/sets" +) + +// ciphers maps strings into tls package cipher constants in +// https://golang.org/pkg/crypto/tls/#pkg-constants +var ciphers = map[string]uint16{ + "TLS_RSA_WITH_RC4_128_SHA": tls.TLS_RSA_WITH_RC4_128_SHA, + "TLS_RSA_WITH_3DES_EDE_CBC_SHA": tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, + "TLS_RSA_WITH_AES_128_CBC_SHA": tls.TLS_RSA_WITH_AES_128_CBC_SHA, + "TLS_RSA_WITH_AES_256_CBC_SHA": tls.TLS_RSA_WITH_AES_256_CBC_SHA, + "TLS_RSA_WITH_AES_128_CBC_SHA256": tls.TLS_RSA_WITH_AES_128_CBC_SHA256, + "TLS_RSA_WITH_AES_128_GCM_SHA256": tls.TLS_RSA_WITH_AES_128_GCM_SHA256, + "TLS_RSA_WITH_AES_256_GCM_SHA384": tls.TLS_RSA_WITH_AES_256_GCM_SHA384, + "TLS_ECDHE_ECDSA_WITH_RC4_128_SHA": tls.TLS_ECDHE_ECDSA_WITH_RC4_128_SHA, + "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA": tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, + "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA": tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, + "TLS_ECDHE_RSA_WITH_RC4_128_SHA": tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA, + "TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA": tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, + "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA": tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, + "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA": tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, + "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256": tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256, + "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256": tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256": tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256": tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384": tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384": tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305": tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, + "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305": tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, +} + +func TLSCipherPossibleValues() []string { + cipherKeys := sets.NewString() + for key := range ciphers { + cipherKeys.Insert(key) + } + return cipherKeys.List() +} + +func TLSCipherSuites(cipherNames []string) ([]uint16, error) { + if len(cipherNames) == 0 { + return nil, nil + } + ciphersIntSlice := make([]uint16, 0) + for _, cipher := range cipherNames { + intValue, ok := ciphers[cipher] + if !ok { + return nil, fmt.Errorf("Cipher suite %s not supported or doesn't exist", cipher) + } + ciphersIntSlice = append(ciphersIntSlice, intValue) + } + return ciphersIntSlice, nil +} + +var versions = map[string]uint16{ + "VersionTLS10": tls.VersionTLS10, + "VersionTLS11": tls.VersionTLS11, + "VersionTLS12": tls.VersionTLS12, + "VersionTLS13": tls.VersionTLS13, +} + +func TLSPossibleVersions() []string { + versionsKeys := sets.NewString() + for key := range versions { + versionsKeys.Insert(key) + } + return versionsKeys.List() +} + +func TLSVersion(versionName string) (uint16, error) { + if len(versionName) == 0 { + return DefaultTLSVersion(), nil + } + if version, ok := versions[versionName]; ok { + return version, nil + } + return 0, fmt.Errorf("unknown tls version %q", versionName) +} + +func DefaultTLSVersion() uint16 { + // Can't use SSLv3 because of POODLE and BEAST + // Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher + // Can't use TLSv1.1 because of RC4 cipher usage + return tls.VersionTLS12 +} diff --git a/vendor/k8s.io/component-base/cli/flag/colon_separated_multimap_string_string.go b/vendor/k8s.io/component-base/cli/flag/colon_separated_multimap_string_string.go new file mode 100644 index 000000000..bd2cf5f87 --- /dev/null +++ b/vendor/k8s.io/component-base/cli/flag/colon_separated_multimap_string_string.go @@ -0,0 +1,102 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flag + +import ( + "fmt" + "sort" + "strings" +) + +// ColonSeparatedMultimapStringString supports setting a map[string][]string from an encoding +// that separates keys from values with ':' and separates key-value pairs with ','. +// A key can be repeated multiple times, in which case the values are appended to a +// slice of strings associated with that key. Items in the list associated with a given +// key will appear in the order provided. +// For example: `a:hello,b:again,c:world,b:beautiful` results in `{"a": ["hello"], "b": ["again", "beautiful"], "c": ["world"]}` +// The first call to Set will clear the map before adding entries; subsequent calls will simply append to the map. +// This makes it possible to override default values with a command-line option rather than appending to defaults, +// while still allowing the distribution of key-value pairs across multiple flag invocations. +// For example: `--flag "a:hello" --flag "b:again" --flag "b:beautiful" --flag "c:world"` results in `{"a": ["hello"], "b": ["again", "beautiful"], "c": ["world"]}` +type ColonSeparatedMultimapStringString struct { + Multimap *map[string][]string + initialized bool // set to true after the first Set call +} + +// NewColonSeparatedMultimapStringString takes a pointer to a map[string][]string and returns the +// ColonSeparatedMultimapStringString flag parsing shim for that map. +func NewColonSeparatedMultimapStringString(m *map[string][]string) *ColonSeparatedMultimapStringString { + return &ColonSeparatedMultimapStringString{Multimap: m} +} + +// Set implements github.com/spf13/pflag.Value +func (m *ColonSeparatedMultimapStringString) Set(value string) error { + if m.Multimap == nil { + return fmt.Errorf("no target (nil pointer to map[string][]string)") + } + if !m.initialized || *m.Multimap == nil { + // clear default values, or allocate if no existing map + *m.Multimap = make(map[string][]string) + m.initialized = true + } + for _, pair := range strings.Split(value, ",") { + if len(pair) == 0 { + continue + } + kv := strings.SplitN(pair, ":", 2) + if len(kv) != 2 { + return fmt.Errorf("malformed pair, expect string:string") + } + k := strings.TrimSpace(kv[0]) + v := strings.TrimSpace(kv[1]) + (*m.Multimap)[k] = append((*m.Multimap)[k], v) + } + return nil +} + +// String implements github.com/spf13/pflag.Value +func (m *ColonSeparatedMultimapStringString) String() string { + type kv struct { + k string + v string + } + kvs := make([]kv, 0, len(*m.Multimap)) + for k, vs := range *m.Multimap { + for i := range vs { + kvs = append(kvs, kv{k: k, v: vs[i]}) + } + } + // stable sort by keys, order of values should be preserved + sort.SliceStable(kvs, func(i, j int) bool { + return kvs[i].k < kvs[j].k + }) + pairs := make([]string, 0, len(kvs)) + for i := range kvs { + pairs = append(pairs, fmt.Sprintf("%s:%s", kvs[i].k, kvs[i].v)) + } + return strings.Join(pairs, ",") +} + +// Type implements github.com/spf13/pflag.Value +func (m *ColonSeparatedMultimapStringString) Type() string { + return "colonSeparatedMultimapStringString" +} + +// Empty implements OmitEmpty +func (m *ColonSeparatedMultimapStringString) Empty() bool { + return len(*m.Multimap) == 0 +} diff --git a/vendor/k8s.io/component-base/cli/flag/configuration_map.go b/vendor/k8s.io/component-base/cli/flag/configuration_map.go new file mode 100644 index 000000000..911b05ec6 --- /dev/null +++ b/vendor/k8s.io/component-base/cli/flag/configuration_map.go @@ -0,0 +1,53 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flag + +import ( + "fmt" + "sort" + "strings" +) + +type ConfigurationMap map[string]string + +func (m *ConfigurationMap) String() string { + pairs := []string{} + for k, v := range *m { + pairs = append(pairs, fmt.Sprintf("%s=%s", k, v)) + } + sort.Strings(pairs) + return strings.Join(pairs, ",") +} + +func (m *ConfigurationMap) Set(value string) error { + for _, s := range strings.Split(value, ",") { + if len(s) == 0 { + continue + } + arr := strings.SplitN(s, "=", 2) + if len(arr) == 2 { + (*m)[strings.TrimSpace(arr[0])] = strings.TrimSpace(arr[1]) + } else { + (*m)[strings.TrimSpace(arr[0])] = "" + } + } + return nil +} + +func (*ConfigurationMap) Type() string { + return "mapStringString" +} diff --git a/vendor/k8s.io/component-base/cli/flag/flags.go b/vendor/k8s.io/component-base/cli/flag/flags.go new file mode 100644 index 000000000..d0fff8db2 --- /dev/null +++ b/vendor/k8s.io/component-base/cli/flag/flags.go @@ -0,0 +1,54 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flag + +import ( + goflag "flag" + "strings" + + "github.com/spf13/pflag" + "k8s.io/klog" +) + +// WordSepNormalizeFunc changes all flags that contain "_" separators +func WordSepNormalizeFunc(f *pflag.FlagSet, name string) pflag.NormalizedName { + if strings.Contains(name, "_") { + return pflag.NormalizedName(strings.Replace(name, "_", "-", -1)) + } + return pflag.NormalizedName(name) +} + +// WarnWordSepNormalizeFunc changes and warns for flags that contain "_" separators +func WarnWordSepNormalizeFunc(f *pflag.FlagSet, name string) pflag.NormalizedName { + if strings.Contains(name, "_") { + nname := strings.Replace(name, "_", "-", -1) + klog.Warningf("%s is DEPRECATED and will be removed in a future version. Use %s instead.", name, nname) + + return pflag.NormalizedName(nname) + } + return pflag.NormalizedName(name) +} + +// InitFlags normalizes, parses, then logs the command line flags +func InitFlags() { + pflag.CommandLine.SetNormalizeFunc(WordSepNormalizeFunc) + pflag.CommandLine.AddGoFlagSet(goflag.CommandLine) + pflag.Parse() + pflag.VisitAll(func(flag *pflag.Flag) { + klog.V(2).Infof("FLAG: --%s=%q", flag.Name, flag.Value) + }) +} diff --git a/vendor/k8s.io/component-base/cli/flag/langle_separated_map_string_string.go b/vendor/k8s.io/component-base/cli/flag/langle_separated_map_string_string.go new file mode 100644 index 000000000..bf8dbfb9b --- /dev/null +++ b/vendor/k8s.io/component-base/cli/flag/langle_separated_map_string_string.go @@ -0,0 +1,82 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flag + +import ( + "fmt" + "sort" + "strings" +) + +// LangleSeparatedMapStringString can be set from the command line with the format `--flag "string 0 { + s = s + ":" + strings.Join(nkc.Names, ",") + } + return s +} + +func (nkc *NamedCertKey) Set(value string) error { + cs := strings.SplitN(value, ":", 2) + var keycert string + if len(cs) == 2 { + var names string + keycert, names = strings.TrimSpace(cs[0]), strings.TrimSpace(cs[1]) + if names == "" { + return errors.New("empty names list is not allowed") + } + nkc.Names = nil + for _, name := range strings.Split(names, ",") { + nkc.Names = append(nkc.Names, strings.TrimSpace(name)) + } + } else { + nkc.Names = nil + keycert = strings.TrimSpace(cs[0]) + } + cs = strings.Split(keycert, ",") + if len(cs) != 2 { + return errors.New("expected comma separated certificate and key file paths") + } + nkc.CertFile = strings.TrimSpace(cs[0]) + nkc.KeyFile = strings.TrimSpace(cs[1]) + return nil +} + +func (*NamedCertKey) Type() string { + return "namedCertKey" +} + +// NamedCertKeyArray is a flag value parsing NamedCertKeys, each passed with its own +// flag instance (in contrast to comma separated slices). +type NamedCertKeyArray struct { + value *[]NamedCertKey + changed bool +} + +var _ flag.Value = &NamedCertKey{} + +// NewNamedKeyCertArray creates a new NamedCertKeyArray with the internal value +// pointing to p. +func NewNamedCertKeyArray(p *[]NamedCertKey) *NamedCertKeyArray { + return &NamedCertKeyArray{ + value: p, + } +} + +func (a *NamedCertKeyArray) Set(val string) error { + nkc := NamedCertKey{} + err := nkc.Set(val) + if err != nil { + return err + } + if !a.changed { + *a.value = []NamedCertKey{nkc} + a.changed = true + } else { + *a.value = append(*a.value, nkc) + } + return nil +} + +func (a *NamedCertKeyArray) Type() string { + return "namedCertKey" +} + +func (a *NamedCertKeyArray) String() string { + nkcs := make([]string, 0, len(*a.value)) + for i := range *a.value { + nkcs = append(nkcs, (*a.value)[i].String()) + } + return "[" + strings.Join(nkcs, ";") + "]" +} diff --git a/vendor/k8s.io/component-base/cli/flag/noop.go b/vendor/k8s.io/component-base/cli/flag/noop.go new file mode 100644 index 000000000..03f7f14c0 --- /dev/null +++ b/vendor/k8s.io/component-base/cli/flag/noop.go @@ -0,0 +1,41 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flag + +import ( + goflag "flag" + "github.com/spf13/pflag" +) + +// NoOp implements goflag.Value and plfag.Value, +// but has a noop Set implementation +type NoOp struct{} + +var _ goflag.Value = NoOp{} +var _ pflag.Value = NoOp{} + +func (NoOp) String() string { + return "" +} + +func (NoOp) Set(val string) error { + return nil +} + +func (NoOp) Type() string { + return "NoOp" +} diff --git a/vendor/k8s.io/component-base/cli/flag/omitempty.go b/vendor/k8s.io/component-base/cli/flag/omitempty.go new file mode 100644 index 000000000..c354754ea --- /dev/null +++ b/vendor/k8s.io/component-base/cli/flag/omitempty.go @@ -0,0 +1,24 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flag + +// OmitEmpty is an interface for flags to report whether their underlying value +// is "empty." If a flag implements OmitEmpty and returns true for a call to Empty(), +// it is assumed that flag may be omitted from the command line. +type OmitEmpty interface { + Empty() bool +} diff --git a/vendor/k8s.io/component-base/cli/flag/sectioned.go b/vendor/k8s.io/component-base/cli/flag/sectioned.go new file mode 100644 index 000000000..493a6c0f0 --- /dev/null +++ b/vendor/k8s.io/component-base/cli/flag/sectioned.go @@ -0,0 +1,79 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flag + +import ( + "bytes" + "fmt" + "io" + "strings" + + "github.com/spf13/pflag" +) + +// NamedFlagSets stores named flag sets in the order of calling FlagSet. +type NamedFlagSets struct { + // Order is an ordered list of flag set names. + Order []string + // FlagSets stores the flag sets by name. + FlagSets map[string]*pflag.FlagSet +} + +// FlagSet returns the flag set with the given name and adds it to the +// ordered name list if it is not in there yet. +func (nfs *NamedFlagSets) FlagSet(name string) *pflag.FlagSet { + if nfs.FlagSets == nil { + nfs.FlagSets = map[string]*pflag.FlagSet{} + } + if _, ok := nfs.FlagSets[name]; !ok { + nfs.FlagSets[name] = pflag.NewFlagSet(name, pflag.ExitOnError) + nfs.Order = append(nfs.Order, name) + } + return nfs.FlagSets[name] +} + +// PrintSections prints the given names flag sets in sections, with the maximal given column number. +// If cols is zero, lines are not wrapped. +func PrintSections(w io.Writer, fss NamedFlagSets, cols int) { + for _, name := range fss.Order { + fs := fss.FlagSets[name] + if !fs.HasFlags() { + continue + } + + wideFS := pflag.NewFlagSet("", pflag.ExitOnError) + wideFS.AddFlagSet(fs) + + var zzz string + if cols > 24 { + zzz = strings.Repeat("z", cols-24) + wideFS.Int(zzz, 0, strings.Repeat("z", cols-24)) + } + + var buf bytes.Buffer + fmt.Fprintf(&buf, "\n%s flags:\n\n%s", strings.ToUpper(name[:1])+name[1:], wideFS.FlagUsagesWrapped(cols)) + + if cols > 24 { + i := strings.Index(buf.String(), zzz) + lines := strings.Split(buf.String()[:i], "\n") + fmt.Fprint(w, strings.Join(lines[:len(lines)-1], "\n")) + fmt.Fprintln(w) + } else { + fmt.Fprint(w, buf.String()) + } + } +} diff --git a/vendor/k8s.io/component-base/cli/flag/string_flag.go b/vendor/k8s.io/component-base/cli/flag/string_flag.go new file mode 100644 index 000000000..331bdb66e --- /dev/null +++ b/vendor/k8s.io/component-base/cli/flag/string_flag.go @@ -0,0 +1,56 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flag + +// StringFlag is a string flag compatible with flags and pflags that keeps track of whether it had a value supplied or not. +type StringFlag struct { + // If Set has been invoked this value is true + provided bool + // The exact value provided on the flag + value string +} + +func NewStringFlag(defaultVal string) StringFlag { + return StringFlag{value: defaultVal} +} + +func (f *StringFlag) Default(value string) { + f.value = value +} + +func (f StringFlag) String() string { + return f.value +} + +func (f StringFlag) Value() string { + return f.value +} + +func (f *StringFlag) Set(value string) error { + f.value = value + f.provided = true + + return nil +} + +func (f StringFlag) Provided() bool { + return f.provided +} + +func (f *StringFlag) Type() string { + return "string" +} diff --git a/vendor/k8s.io/component-base/cli/flag/tristate.go b/vendor/k8s.io/component-base/cli/flag/tristate.go new file mode 100644 index 000000000..cf16376bf --- /dev/null +++ b/vendor/k8s.io/component-base/cli/flag/tristate.go @@ -0,0 +1,83 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flag + +import ( + "fmt" + "strconv" +) + +// Tristate is a flag compatible with flags and pflags that +// keeps track of whether it had a value supplied or not. +type Tristate int + +const ( + Unset Tristate = iota // 0 + True + False +) + +func (f *Tristate) Default(value bool) { + *f = triFromBool(value) +} + +func (f Tristate) String() string { + b := boolFromTri(f) + return fmt.Sprintf("%t", b) +} + +func (f Tristate) Value() bool { + b := boolFromTri(f) + return b +} + +func (f *Tristate) Set(value string) error { + boolVal, err := strconv.ParseBool(value) + if err != nil { + return err + } + + *f = triFromBool(boolVal) + return nil +} + +func (f Tristate) Provided() bool { + if f != Unset { + return true + } + return false +} + +func (f *Tristate) Type() string { + return "tristate" +} + +func boolFromTri(t Tristate) bool { + if t == True { + return true + } else { + return false + } +} + +func triFromBool(b bool) Tristate { + if b { + return True + } else { + return False + } +} diff --git a/vendor/k8s.io/klog/.travis.yml b/vendor/k8s.io/klog/.travis.yml index 0f508dae6..5677664c2 100644 --- a/vendor/k8s.io/klog/.travis.yml +++ b/vendor/k8s.io/klog/.travis.yml @@ -5,11 +5,12 @@ go: - 1.9.x - 1.10.x - 1.11.x + - 1.12.x script: - go get -t -v ./... - diff -u <(echo -n) <(gofmt -d .) - diff -u <(echo -n) <(golint $(go list -e ./...)) - - go tool vet . + - go tool vet . || go vet . - go test -v -race ./... install: - go get golang.org/x/lint/golint diff --git a/vendor/k8s.io/klog/README.md b/vendor/k8s.io/klog/README.md index bee306f39..841468b4b 100644 --- a/vendor/k8s.io/klog/README.md +++ b/vendor/k8s.io/klog/README.md @@ -31,7 +31,7 @@ How to use klog - Use `klog.InitFlags(nil)` explicitly for initializing global flags as we no longer use `init()` method to register the flags - You can now use `log-file` instead of `log-dir` for logging to a single file (See `examples/log_file/usage_log_file.go`) - If you want to redirect everything logged using klog somewhere else (say syslog!), you can use `klog.SetOutput()` method and supply a `io.Writer`. (See `examples/set_output/usage_set_output.go`) -- For more logging conventions (See [Logging Conventions](https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md)) +- For more logging conventions (See [Logging Conventions](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/logging.md)) ### Coexisting with glog This package can be used side by side with glog. [This example](examples/coexist_glog/coexist_glog.go) shows how to initialize and syncronize flags from the global `flag.CommandLine` FlagSet. In addition, the example makes use of stderr as combined output by setting `alsologtostderr` (or `logtostderr`) to `true`. diff --git a/vendor/k8s.io/klog/go.mod b/vendor/k8s.io/klog/go.mod new file mode 100644 index 000000000..3877d8546 --- /dev/null +++ b/vendor/k8s.io/klog/go.mod @@ -0,0 +1,5 @@ +module k8s.io/klog + +go 1.12 + +require github.com/go-logr/logr v0.1.0 diff --git a/vendor/k8s.io/klog/go.sum b/vendor/k8s.io/klog/go.sum new file mode 100644 index 000000000..fb64d277a --- /dev/null +++ b/vendor/k8s.io/klog/go.sum @@ -0,0 +1,2 @@ +github.com/go-logr/logr v0.1.0 h1:M1Tv3VzNlEHg6uyACnRdtrploV2P7wZqH8BoQMtz0cg= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= diff --git a/vendor/k8s.io/klog/klog.go b/vendor/k8s.io/klog/klog.go index a3dddeadf..2520ebdaa 100644 --- a/vendor/k8s.io/klog/klog.go +++ b/vendor/k8s.io/klog/klog.go @@ -20,26 +20,26 @@ // // Basic examples: // -// glog.Info("Prepare to repel boarders") +// klog.Info("Prepare to repel boarders") // -// glog.Fatalf("Initialization failed: %s", err) +// klog.Fatalf("Initialization failed: %s", err) // // See the documentation for the V function for an explanation of these examples: // -// if glog.V(2) { -// glog.Info("Starting transaction...") +// if klog.V(2) { +// klog.Info("Starting transaction...") // } // -// glog.V(2).Infoln("Processed", nItems, "elements") +// klog.V(2).Infoln("Processed", nItems, "elements") // // Log output is buffered and written periodically using Flush. Programs // should call Flush before exiting to guarantee all log output is written. // -// By default, all log statements write to files in a temporary directory. +// By default, all log statements write to standard error. // This package provides several flags that modify this behavior. // As a result, flag.Parse must be called before any logging is done. // -// -logtostderr=false +// -logtostderr=true // Logs are written to standard error instead of to files. // -alsologtostderr=false // Logs are written to standard error as well as to files. @@ -417,6 +417,7 @@ func InitFlags(flagset *flag.FlagSet) { logging.toStderr = true logging.alsoToStderr = false logging.skipHeaders = false + logging.addDirHeader = false logging.skipLogHeaders = false }) @@ -432,6 +433,7 @@ func InitFlags(flagset *flag.FlagSet) { flagset.BoolVar(&logging.toStderr, "logtostderr", logging.toStderr, "log to standard error instead of files") flagset.BoolVar(&logging.alsoToStderr, "alsologtostderr", logging.alsoToStderr, "log to standard error as well as files") flagset.Var(&logging.verbosity, "v", "number for the log level verbosity") + flagset.BoolVar(&logging.skipHeaders, "add_dir_header", logging.addDirHeader, "If true, adds the file directory to the header") flagset.BoolVar(&logging.skipHeaders, "skip_headers", logging.skipHeaders, "If true, avoid header prefixes in the log messages") flagset.BoolVar(&logging.skipLogHeaders, "skip_log_headers", logging.skipLogHeaders, "If true, avoid headers when opening log files") flagset.Var(&logging.stderrThreshold, "stderrthreshold", "logs at or above this threshold go to stderr") @@ -500,6 +502,9 @@ type loggingT struct { // If true, do not add the headers to log files skipLogHeaders bool + + // If true, add the file directory to the header + addDirHeader bool } // buffer holds a byte Buffer for reuse. The zero value is ready for use. @@ -585,9 +590,14 @@ func (l *loggingT) header(s severity, depth int) (*buffer, string, int) { file = "???" line = 1 } else { - slash := strings.LastIndex(file, "/") - if slash >= 0 { - file = file[slash+1:] + if slash := strings.LastIndex(file, "/"); slash >= 0 { + path := file + file = path[slash+1:] + if l.addDirHeader { + if dirsep := strings.LastIndex(path[:slash], "/"); dirsep >= 0 { + file = path[dirsep+1:] + } + } } } return l.formatHeader(s, file, line), file, line @@ -736,6 +746,8 @@ func (rb *redirectBuffer) Write(bytes []byte) (n int, err error) { // SetOutput sets the output destination for all severities func SetOutput(w io.Writer) { + logging.mu.Lock() + defer logging.mu.Unlock() for s := fatalLog; s >= infoLog; s-- { rb := &redirectBuffer{ w: w, @@ -746,6 +758,8 @@ func SetOutput(w io.Writer) { // SetOutputBySeverity sets the output destination for specific severity func SetOutputBySeverity(name string, w io.Writer) { + logging.mu.Lock() + defer logging.mu.Unlock() sev, ok := severityByName(name) if !ok { panic(fmt.Sprintf("SetOutputBySeverity(%q): unrecognized severity name", name)) @@ -771,24 +785,38 @@ func (l *loggingT) output(s severity, buf *buffer, file string, line int, alsoTo if alsoToStderr || l.alsoToStderr || s >= l.stderrThreshold.get() { os.Stderr.Write(data) } - if l.file[s] == nil { - if err := l.createFiles(s); err != nil { - os.Stderr.Write(data) // Make sure the message appears somewhere. - l.exit(err) + + if logging.logFile != "" { + // Since we are using a single log file, all of the items in l.file array + // will point to the same file, so just use one of them to write data. + if l.file[infoLog] == nil { + if err := l.createFiles(infoLog); err != nil { + os.Stderr.Write(data) // Make sure the message appears somewhere. + l.exit(err) + } } - } - switch s { - case fatalLog: - l.file[fatalLog].Write(data) - fallthrough - case errorLog: - l.file[errorLog].Write(data) - fallthrough - case warningLog: - l.file[warningLog].Write(data) - fallthrough - case infoLog: l.file[infoLog].Write(data) + } else { + if l.file[s] == nil { + if err := l.createFiles(s); err != nil { + os.Stderr.Write(data) // Make sure the message appears somewhere. + l.exit(err) + } + } + + switch s { + case fatalLog: + l.file[fatalLog].Write(data) + fallthrough + case errorLog: + l.file[errorLog].Write(data) + fallthrough + case warningLog: + l.file[warningLog].Write(data) + fallthrough + case infoLog: + l.file[infoLog].Write(data) + } } } if s == fatalLog { @@ -827,7 +855,7 @@ func (l *loggingT) output(s severity, buf *buffer, file string, line int, alsoTo // timeoutFlush calls Flush and returns when it completes or after timeout // elapses, whichever happens first. This is needed because the hooks invoked -// by Flush may deadlock when glog.Fatal is called from a hook that holds +// by Flush may deadlock when klog.Fatal is called from a hook that holds // a lock. func timeoutFlush(timeout time.Duration) { done := make(chan bool, 1) @@ -838,7 +866,7 @@ func timeoutFlush(timeout time.Duration) { select { case <-done: case <-time.After(timeout): - fmt.Fprintln(os.Stderr, "glog: Flush took longer than", timeout) + fmt.Fprintln(os.Stderr, "klog: Flush took longer than", timeout) } } @@ -1094,9 +1122,9 @@ type Verbose bool // The returned value is a boolean of type Verbose, which implements Info, Infoln // and Infof. These methods will write to the Info log if called. // Thus, one may write either -// if glog.V(2) { glog.Info("log this") } +// if klog.V(2) { klog.Info("log this") } // or -// glog.V(2).Info("log this") +// klog.V(2).Info("log this") // The second form is shorter but the first is cheaper if logging is off because it does // not evaluate its arguments. // @@ -1170,7 +1198,7 @@ func InfoDepth(depth int, args ...interface{}) { } // Infoln logs to the INFO log. -// Arguments are handled in the manner of fmt.Println; a newline is appended if missing. +// Arguments are handled in the manner of fmt.Println; a newline is always appended. func Infoln(args ...interface{}) { logging.println(infoLog, args...) } @@ -1194,7 +1222,7 @@ func WarningDepth(depth int, args ...interface{}) { } // Warningln logs to the WARNING and INFO logs. -// Arguments are handled in the manner of fmt.Println; a newline is appended if missing. +// Arguments are handled in the manner of fmt.Println; a newline is always appended. func Warningln(args ...interface{}) { logging.println(warningLog, args...) } @@ -1218,7 +1246,7 @@ func ErrorDepth(depth int, args ...interface{}) { } // Errorln logs to the ERROR, WARNING, and INFO logs. -// Arguments are handled in the manner of fmt.Println; a newline is appended if missing. +// Arguments are handled in the manner of fmt.Println; a newline is always appended. func Errorln(args ...interface{}) { logging.println(errorLog, args...) } @@ -1244,7 +1272,7 @@ func FatalDepth(depth int, args ...interface{}) { // Fatalln logs to the FATAL, ERROR, WARNING, and INFO logs, // including a stack trace of all running goroutines, then calls os.Exit(255). -// Arguments are handled in the manner of fmt.Println; a newline is appended if missing. +// Arguments are handled in the manner of fmt.Println; a newline is always appended. func Fatalln(args ...interface{}) { logging.println(fatalLog, args...) } diff --git a/vendor/modules.txt b/vendor/modules.txt index d7cd975f8..ecf863210 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -187,6 +187,8 @@ github.com/evanphx/json-patch github.com/fatih/structs # github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 => github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 github.com/flynn/go-shlex +# github.com/fsnotify/fsnotify v1.4.7 => github.com/fsnotify/fsnotify v1.4.7 +github.com/fsnotify/fsnotify # github.com/ghodss/yaml v1.0.0 => github.com/ghodss/yaml v1.0.0 github.com/ghodss/yaml # github.com/go-acme/lego v2.5.0+incompatible => github.com/go-acme/lego v2.5.0+incompatible @@ -240,7 +242,7 @@ github.com/gobuffalo/flect # github.com/gocraft/dbr v0.0.0-20180507214907-a0fd650918f6 => github.com/gocraft/dbr v0.0.0-20180507214907-a0fd650918f6 github.com/gocraft/dbr github.com/gocraft/dbr/dialect -# github.com/gogo/protobuf v1.2.0 => github.com/gogo/protobuf v1.2.0 +# github.com/gogo/protobuf v1.2.1 => github.com/gogo/protobuf v1.2.0 github.com/gogo/protobuf/gogoproto github.com/gogo/protobuf/proto github.com/gogo/protobuf/protoc-gen-gogo/descriptor @@ -304,6 +306,17 @@ github.com/hashicorp/go-version # github.com/hashicorp/golang-lru v0.5.1 => github.com/hashicorp/golang-lru v0.5.1 github.com/hashicorp/golang-lru github.com/hashicorp/golang-lru/simplelru +# github.com/hashicorp/hcl v1.0.0 => github.com/hashicorp/hcl v1.0.0 +github.com/hashicorp/hcl +github.com/hashicorp/hcl/hcl/ast +github.com/hashicorp/hcl/hcl/parser +github.com/hashicorp/hcl/hcl/printer +github.com/hashicorp/hcl/hcl/scanner +github.com/hashicorp/hcl/hcl/strconv +github.com/hashicorp/hcl/hcl/token +github.com/hashicorp/hcl/json/parser +github.com/hashicorp/hcl/json/scanner +github.com/hashicorp/hcl/json/token # github.com/hpcloud/tail v1.0.0 => github.com/hpcloud/tail v1.0.0 github.com/hpcloud/tail github.com/hpcloud/tail/ratelimiter @@ -411,6 +424,8 @@ github.com/lucas-clemente/quic-go/internal/protocol github.com/lucas-clemente/quic-go/internal/qerr github.com/lucas-clemente/quic-go/internal/utils github.com/lucas-clemente/quic-go/internal/wire +# github.com/magiconair/properties v1.8.0 => github.com/magiconair/properties v1.8.0 +github.com/magiconair/properties # github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 => github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 github.com/mailru/easyjson/buffer github.com/mailru/easyjson/jlexer @@ -464,6 +479,8 @@ github.com/mholt/certmagic github.com/miekg/dns # github.com/mitchellh/go-homedir v1.1.0 => github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/go-homedir +# github.com/mitchellh/mapstructure v1.1.2 => github.com/mitchellh/mapstructure v1.1.2 +github.com/mitchellh/mapstructure # github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd => github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd github.com/modern-go/concurrent # github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 => github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 @@ -522,6 +539,8 @@ github.com/openshift/api/project/v1 github.com/pborman/uuid # github.com/pelletier/go-buffruneio v0.2.0 => github.com/pelletier/go-buffruneio v0.2.0 github.com/pelletier/go-buffruneio +# github.com/pelletier/go-toml v1.2.0 => github.com/pelletier/go-toml v1.2.0 +github.com/pelletier/go-toml # github.com/peterbourgon/diskv v2.0.1+incompatible => github.com/peterbourgon/diskv v2.0.1+incompatible github.com/peterbourgon/diskv # github.com/pkg/errors v0.8.1 => github.com/pkg/errors v0.8.1 @@ -563,7 +582,7 @@ github.com/projectcalico/libcalico-go/lib/selector/tokenizer github.com/projectcalico/libcalico-go/lib/set github.com/projectcalico/libcalico-go/lib/validator/v3 github.com/projectcalico/libcalico-go/lib/watch -# github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 => github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 +# github.com/prometheus/client_golang v0.9.3 => github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 github.com/prometheus/client_golang/api github.com/prometheus/client_golang/api/prometheus/v1 github.com/prometheus/client_golang/prometheus @@ -594,10 +613,16 @@ github.com/speps/go-hashids # github.com/spf13/afero v1.2.2 => github.com/spf13/afero v1.2.2 github.com/spf13/afero github.com/spf13/afero/mem +# github.com/spf13/cast v1.3.0 => github.com/spf13/cast v1.3.0 +github.com/spf13/cast # github.com/spf13/cobra v0.0.3 => github.com/spf13/cobra v0.0.3 github.com/spf13/cobra +# github.com/spf13/jwalterweatherman v1.0.0 => github.com/spf13/jwalterweatherman v1.0.0 +github.com/spf13/jwalterweatherman # github.com/spf13/pflag v1.0.3 => github.com/spf13/pflag v1.0.3 github.com/spf13/pflag +# github.com/spf13/viper v1.4.0 => github.com/spf13/viper v1.4.0 +github.com/spf13/viper # github.com/src-d/gcfg v1.4.0 => github.com/src-d/gcfg v1.4.0 github.com/src-d/gcfg github.com/src-d/gcfg/scanner @@ -750,7 +775,7 @@ google.golang.org/appengine/urlfetch google.golang.org/genproto/googleapis/api/httpbody google.golang.org/genproto/googleapis/rpc/status google.golang.org/genproto/protobuf/field_mask -# google.golang.org/grpc v1.19.1 => google.golang.org/grpc v1.19.1 +# google.golang.org/grpc v1.21.0 => google.golang.org/grpc v1.19.1 google.golang.org/grpc google.golang.org/grpc/balancer google.golang.org/grpc/balancer/base @@ -895,7 +920,7 @@ k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1 k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1 -# k8s.io/apimachinery v0.0.0-20190221213512-86fb29eff628 => k8s.io/apimachinery v0.0.0-20190221213512-86fb29eff628 +# k8s.io/apimachinery v0.0.0-20190831074630-461753078381 => k8s.io/apimachinery v0.0.0-20190221213512-86fb29eff628 k8s.io/apimachinery/pkg/api/equality k8s.io/apimachinery/pkg/api/errors k8s.io/apimachinery/pkg/api/meta @@ -1047,7 +1072,7 @@ k8s.io/apiserver/plugin/pkg/audit/truncate k8s.io/apiserver/plugin/pkg/audit/webhook k8s.io/apiserver/plugin/pkg/authenticator/token/webhook k8s.io/apiserver/plugin/pkg/authorizer/webhook -# k8s.io/client-go v0.0.0-20181213151034-8d9ed539ba31 => k8s.io/client-go v0.0.0-20181213151034-8d9ed539ba31 +# k8s.io/client-go v0.0.0-20190831074946-3fe2abece89e => k8s.io/client-go v0.0.0-20181213151034-8d9ed539ba31 k8s.io/client-go/discovery k8s.io/client-go/discovery/fake k8s.io/client-go/dynamic @@ -1250,6 +1275,8 @@ k8s.io/code-generator/cmd/lister-gen k8s.io/code-generator/cmd/lister-gen/args k8s.io/code-generator/cmd/lister-gen/generators k8s.io/code-generator/pkg/util +# k8s.io/component-base v0.0.0-20190831075413-37a093468564 => k8s.io/component-base v0.0.0-20190831075413-37a093468564 +k8s.io/component-base/cli/flag # k8s.io/gengo v0.0.0-20190327210449-e17681d19d3a => k8s.io/gengo v0.0.0-20190327210449-e17681d19d3a k8s.io/gengo/args k8s.io/gengo/examples/deepcopy-gen/generators @@ -1258,7 +1285,7 @@ k8s.io/gengo/generator k8s.io/gengo/namer k8s.io/gengo/parser k8s.io/gengo/types -# k8s.io/klog v0.3.1 => k8s.io/klog v0.3.1 +# k8s.io/klog v0.4.0 => k8s.io/klog v0.4.0 k8s.io/klog k8s.io/klog/klogr # k8s.io/kube-openapi v0.0.0-20181109181836-c59034cc13d5 => k8s.io/kube-openapi v0.0.0-20181109181836-c59034cc13d5