admin/kubesphere
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update vendor

Browse Source 
Signed-off-by: Roland.Ma <rolandma@yunify.com>
This commit is contained in:
Roland.Ma
2021-08-11 07:10:14 +00:00
parent a18f72b565
commit ea8f47c73a
2901 changed files with 269317 additions and 43103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

294
vendor/sigs.k8s.io/controller-runtime/pkg/envtest/crd.go generated vendored
View File

@@ -20,12 +20,16 @@ import (
"bufio"
"bytes"
"context"
"encoding/base64"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"time"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -34,18 +38,31 @@ import (
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
k8syaml "k8s.io/apimachinery/pkg/util/yaml"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"k8s.io/client-go/util/retry"
"k8s.io/utils/pointer"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/webhook/conversion"
"sigs.k8s.io/yaml"
)
// CRDInstallOptions are the options for installing CRDs
// CRDInstallOptions are the options for installing CRDs.
type CRDInstallOptions struct {
// Scheme is used to determine if conversion webhooks should be enabled
// for a particular CRD / object.
//
// Conversion webhooks are going to be enabled if an object in the scheme
// implements Hub and Spoke conversions.
//
// If nil, scheme.Scheme is used.
Scheme *runtime.Scheme
// Paths is a list of paths to the directories or files containing CRDs
Paths []string
// CRDs is a list of CRDs to install
CRDs []runtime.Object
CRDs []client.Object
// ErrorIfPathMissing will cause an error if a Path does not exist
ErrorIfPathMissing bool
@@ -60,34 +77,45 @@ type CRDInstallOptions struct {
// uninstalled when terminating the test environment.
// Defaults to false.
CleanUpAfterUse bool
// WebhookOptions contains the conversion webhook information to install
// on the CRDs. This field is usually inherited by the EnvTest options.
//
// If you're passing this field manually, you need to make sure that
// the CA information and host port is filled in properly.
WebhookOptions WebhookInstallOptions
}
const defaultPollInterval = 100 * time.Millisecond
const defaultMaxWait = 10 * time.Second
// InstallCRDs installs a collection of CRDs into a cluster by reading the crd yaml files from a directory
func InstallCRDs(config *rest.Config, options CRDInstallOptions) ([]runtime.Object, error) {
// InstallCRDs installs a collection of CRDs into a cluster by reading the crd yaml files from a directory.
func InstallCRDs(config *rest.Config, options CRDInstallOptions) ([]client.Object, error) {
defaultCRDOptions(&options)
// Read the CRD yamls into options.CRDs
if err := readCRDFiles(&options); err != nil {
return nil, fmt.Errorf("unable to read CRD files: %w", err)
}
if err := modifyConversionWebhooks(options.CRDs, options.Scheme, options.WebhookOptions); err != nil {
return nil, err
}
// Create the CRDs in the apiserver
if err := CreateCRDs(config, options.CRDs); err != nil {
return options.CRDs, err
return options.CRDs, fmt.Errorf("unable to create CRD instances: %w", err)
}
// Wait for the CRDs to appear as Resources in the apiserver
if err := WaitForCRDs(config, options.CRDs, options); err != nil {
return options.CRDs, err
return options.CRDs, fmt.Errorf("something went wrong waiting for CRDs to appear as API resources: %w", err)
}
return options.CRDs, nil
}
// readCRDFiles reads the directories of CRDs in options.Paths and adds the CRD structs to options.CRDs
// readCRDFiles reads the directories of CRDs in options.Paths and adds the CRD structs to options.CRDs.
func readCRDFiles(options *CRDInstallOptions) error {
if len(options.Paths) > 0 {
crdList, err := renderCRDs(options)
@@ -100,8 +128,11 @@ func readCRDFiles(options *CRDInstallOptions) error {
return nil
}
// defaultCRDOptions sets the default values for CRDs
// defaultCRDOptions sets the default values for CRDs.
func defaultCRDOptions(o *CRDInstallOptions) {
if o.Scheme == nil {
o.Scheme = scheme.Scheme
}
if o.MaxTime == 0 {
o.MaxTime = defaultMaxWait
}
@@ -110,8 +141,8 @@ func defaultCRDOptions(o *CRDInstallOptions) {
}
}
// WaitForCRDs waits for the CRDs to appear in discovery
func WaitForCRDs(config *rest.Config, crds []runtime.Object, options CRDInstallOptions) error {
// WaitForCRDs waits for the CRDs to appear in discovery.
func WaitForCRDs(config *rest.Config, crds []client.Object, options CRDInstallOptions) error {
// Add each CRD to a map of GroupVersion to Resource
waitingFor := map[schema.GroupVersion]*sets.String{}
for _, crd := range runtimeCRDListToUnstructured(crds) {
@@ -128,14 +159,17 @@ func WaitForCRDs(config *rest.Config, crds []runtime.Object, options CRDInstallO
if err != nil {
return err
}
if crdVersion != "" {
gvs = append(gvs, schema.GroupVersion{Group: crdGroup, Version: crdVersion})
}
versions, _, err := unstructured.NestedSlice(crd.Object, "spec", "versions")
versions, found, err := unstructured.NestedSlice(crd.Object, "spec", "versions")
if err != nil {
return err
}
// gvs should be added here only if single version is found. If multiple version is found we will add those version
// based on the version is served or not.
if crdVersion != "" && !found {
gvs = append(gvs, schema.GroupVersion{Group: crdGroup, Version: crdVersion})
}
for _, version := range versions {
versionMap, ok := version.(map[string]interface{})
if !ok {
@@ -170,7 +204,7 @@ func WaitForCRDs(config *rest.Config, crds []runtime.Object, options CRDInstallO
return wait.PollImmediate(options.PollInterval, options.MaxTime, p.poll)
}
// poller checks if all the resources have been found in discovery, and returns false if not
// poller checks if all the resources have been found in discovery, and returns false if not.
type poller struct {
// config is used to get discovery
config *rest.Config
@@ -179,7 +213,7 @@ type poller struct {
waitingFor map[schema.GroupVersion]*sets.String
}
// poll checks if all the resources have been found in discovery, and returns false if not
// poll checks if all the resources have been found in discovery, and returns false if not.
func (p *poller) poll() (done bool, err error) {
// Create a new clientset to avoid any client caching of discovery
cs, err := clientset.NewForConfig(p.config)
@@ -199,7 +233,7 @@ func (p *poller) poll() (done bool, err error) {
// TODO: Maybe the controller-runtime client should be able to do this...
resourceList, err := cs.Discovery().ServerResourcesForGroupVersion(gv.Group + "/" + gv.Version)
if err != nil {
return false, nil
return false, nil //nolint:nilerr
}
// Remove each found resource from the resources set that we are waiting for
@@ -215,9 +249,8 @@ func (p *poller) poll() (done bool, err error) {
return allFound, nil
}
// UninstallCRDs uninstalls a collection of CRDs by reading the crd yaml files from a directory
// UninstallCRDs uninstalls a collection of CRDs by reading the crd yaml files from a directory.
func UninstallCRDs(config *rest.Config, options CRDInstallOptions) error {
// Read the CRD yamls into options.CRDs
if err := readCRDFiles(&options); err != nil {
return err
@@ -243,11 +276,11 @@ func UninstallCRDs(config *rest.Config, options CRDInstallOptions) error {
return nil
}
// CreateCRDs creates the CRDs
func CreateCRDs(config *rest.Config, crds []runtime.Object) error {
// CreateCRDs creates the CRDs.
func CreateCRDs(config *rest.Config, crds []client.Object) error {
cs, err := client.New(config, client.Options{})
if err != nil {
return err
return fmt.Errorf("unable to create client: %w", err)
}
// Create each CRD
@@ -258,14 +291,19 @@ func CreateCRDs(config *rest.Config, crds []runtime.Object) error {
switch {
case apierrors.IsNotFound(err):
if err := cs.Create(context.TODO(), crd); err != nil {
return err
return fmt.Errorf("unable to create CRD %q: %w", crd.GetName(), err)
}
case err != nil:
return err
return fmt.Errorf("unable to get CRD %q to check if it exists: %w", crd.GetName(), err)
default:
log.V(1).Info("CRD already exists, updating", "crd", crd.GetName())
crd.SetResourceVersion(existingCrd.GetResourceVersion())
if err := cs.Update(context.TODO(), crd); err != nil {
if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
if err := cs.Get(context.TODO(), client.ObjectKey{Name: crd.GetName()}, existingCrd); err != nil {
return err
}
crd.SetResourceVersion(existingCrd.GetResourceVersion())
return cs.Update(context.TODO(), crd)
}); err != nil {
return err
}
}
@@ -274,7 +312,7 @@ func CreateCRDs(config *rest.Config, crds []runtime.Object) error {
}
// renderCRDs iterate through options.Paths and extract all CRD files.
func renderCRDs(options *CRDInstallOptions) ([]runtime.Object, error) {
func renderCRDs(options *CRDInstallOptions) ([]client.Object, error) {
var (
err error
info os.FileInfo
@@ -301,10 +339,8 @@ func renderCRDs(options *CRDInstallOptions) ([]runtime.Object, error) {
if !info.IsDir() {
filePath, files = filepath.Dir(path), []os.FileInfo{info}
} else {
if files, err = ioutil.ReadDir(path); err != nil {
return nil, err
}
} else if files, err = ioutil.ReadDir(path); err != nil {
return nil, err
}
log.V(1).Info("reading CRDs from path", "path", path)
@@ -325,14 +361,200 @@ func renderCRDs(options *CRDInstallOptions) ([]runtime.Object, error) {
}
// Converting map to a list to return
var res []runtime.Object
res := []client.Object{}
for _, obj := range crds {
res = append(res, obj)
}
return res, nil
}
// readCRDs reads the CRDs from files and Unmarshals them into structs
// modifyConversionWebhooks takes all the registered CustomResourceDefinitions and applies modifications
// to conditionally enable webhooks if the type is registered within the scheme.
//
// The complexity of this function is high mostly due to all the edge cases that we need to handle:
// CRDv1beta1, CRDv1, and their unstructured counterpart.
//
// We should be able to simplify this code once we drop support for v1beta1 and standardize around the typed CRDv1 object.
func modifyConversionWebhooks(crds []client.Object, scheme *runtime.Scheme, webhookOptions WebhookInstallOptions) error { //nolint:gocyclo
if len(webhookOptions.LocalServingCAData) == 0 {
return nil
}
// Determine all registered convertible types.
convertibles := map[schema.GroupKind]struct{}{}
for gvk := range scheme.AllKnownTypes() {
obj, err := scheme.New(gvk)
if err != nil {
return err
}
if ok, err := conversion.IsConvertible(scheme, obj); ok && err == nil {
convertibles[gvk.GroupKind()] = struct{}{}
}
}
// generate host port.
hostPort, err := webhookOptions.generateHostPort()
if err != nil {
return err
}
url := pointer.StringPtr(fmt.Sprintf("https://%s/convert", hostPort))
for _, crd := range crds {
switch c := crd.(type) {
case *apiextensionsv1beta1.CustomResourceDefinition:
// Continue if we're preserving unknown fields.
//
// preserveUnknownFields defaults to true if `nil` in v1beta1.
if c.Spec.PreserveUnknownFields == nil || *c.Spec.PreserveUnknownFields {
continue
}
// Continue if the GroupKind isn't registered as being convertible.
if _, ok := convertibles[schema.GroupKind{
Group: c.Spec.Group,
Kind: c.Spec.Names.Kind,
}]; !ok {
continue
}
c.Spec.Conversion.Strategy = apiextensionsv1beta1.WebhookConverter
c.Spec.Conversion.WebhookClientConfig.Service = nil
c.Spec.Conversion.WebhookClientConfig = &apiextensionsv1beta1.WebhookClientConfig{
Service: nil,
URL: url,
CABundle: webhookOptions.LocalServingCAData,
}
case *apiextensionsv1.CustomResourceDefinition:
// Continue if we're preserving unknown fields.
if c.Spec.PreserveUnknownFields {
continue
}
// Continue if the GroupKind isn't registered as being convertible.
if _, ok := convertibles[schema.GroupKind{
Group: c.Spec.Group,
Kind: c.Spec.Names.Kind,
}]; !ok {
continue
}
c.Spec.Conversion.Strategy = apiextensionsv1.WebhookConverter
c.Spec.Conversion.Webhook.ClientConfig.Service = nil
c.Spec.Conversion.Webhook.ClientConfig = &apiextensionsv1.WebhookClientConfig{
Service: nil,
URL: url,
CABundle: webhookOptions.LocalServingCAData,
}
case *unstructured.Unstructured:
webhookClientConfig := map[string]interface{}{
"url": *url,
"caBundle": base64.StdEncoding.EncodeToString(webhookOptions.LocalServingCAData),
}
switch c.GroupVersionKind().Version {
case "v1beta1":
// Continue if we're preserving unknown fields.
//
// preserveUnknownFields defaults to true if `nil` in v1beta1.
if preserve, found, err := unstructured.NestedBool(c.Object, "spec", "preserveUnknownFields"); preserve || !found {
continue
} else if err != nil {
return err
}
// Continue if the GroupKind isn't registered as being convertible.
group, found, err := unstructured.NestedString(c.Object, "spec", "group")
if !found {
continue
} else if err != nil {
return err
}
kind, found, err := unstructured.NestedString(c.Object, "spec", "names", "kind")
if !found {
continue
} else if err != nil {
return err
}
if _, ok := convertibles[schema.GroupKind{
Group: group,
Kind: kind,
}]; !ok {
continue
}
// Set the strategy.
if err := unstructured.SetNestedField(
c.Object,
string(apiextensionsv1beta1.WebhookConverter),
"spec", "conversion", "strategy"); err != nil {
return err
}
// Set the conversion review versions.
if err := unstructured.SetNestedStringSlice(
c.Object,
[]string{"v1beta1"},
"spec", "conversion", "webhook", "clientConfig"); err != nil {
return err
}
// Set the client configuration.
if err := unstructured.SetNestedMap(
c.Object,
webhookClientConfig,
"spec", "conversion", "webhookClientConfig"); err != nil {
return err
}
case "v1":
if preserve, _, err := unstructured.NestedBool(c.Object, "spec", "preserveUnknownFields"); preserve {
continue
} else if err != nil {
return err
}
// Continue if the GroupKind isn't registered as being convertible.
group, found, err := unstructured.NestedString(c.Object, "spec", "group")
if !found {
continue
} else if err != nil {
return err
}
kind, found, err := unstructured.NestedString(c.Object, "spec", "names", "kind")
if !found {
continue
} else if err != nil {
return err
}
if _, ok := convertibles[schema.GroupKind{
Group: group,
Kind: kind,
}]; !ok {
continue
}
// Set the strategy.
if err := unstructured.SetNestedField(
c.Object,
string(apiextensionsv1.WebhookConverter),
"spec", "conversion", "strategy"); err != nil {
return err
}
// Set the conversion review versions.
if err := unstructured.SetNestedStringSlice(
c.Object,
[]string{"v1", "v1beta1"},
"spec", "conversion", "webhook", "conversionReviewVersions"); err != nil {
return err
}
// Set the client configuration.
if err := unstructured.SetNestedMap(
c.Object,
webhookClientConfig,
"spec", "conversion", "webhook", "clientConfig"); err != nil {
return err
}
}
}
}
return nil
}
// readCRDs reads the CRDs from files and Unmarshals them into structs.
func readCRDs(basePath string, files []os.FileInfo) ([]*unstructured.Unstructured, error) {
var crds []*unstructured.Unstructured
@@ -378,9 +600,9 @@ func readCRDs(basePath string, files []os.FileInfo) ([]*unstructured.Unstructure
return crds, nil
}
// readDocuments reads documents from file
// readDocuments reads documents from file.
func readDocuments(fp string) ([][]byte, error) {
b, err := ioutil.ReadFile(fp)
b, err := ioutil.ReadFile(fp) //nolint:gosec
if err != nil {
return nil, err
}

23
vendor/sigs.k8s.io/controller-runtime/pkg/envtest/helper.go generated vendored
View File

@@ -1,3 +1,19 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package envtest
import (
@@ -5,6 +21,7 @@ import (
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
)
var (
@@ -38,7 +55,7 @@ func mergePaths(s1, s2 []string) []string {
// mergeCRDs merges two CRD slices using their names.
// This function makes no guarantees about order of the merged slice.
func mergeCRDs(s1, s2 []runtime.Object) []runtime.Object {
func mergeCRDs(s1, s2 []client.Object) []client.Object {
m := make(map[string]*unstructured.Unstructured)
for _, obj := range runtimeCRDListToUnstructured(s1) {
m[obj.GetName()] = obj
@@ -46,7 +63,7 @@ func mergeCRDs(s1, s2 []runtime.Object) []runtime.Object {
for _, obj := range runtimeCRDListToUnstructured(s2) {
m[obj.GetName()] = obj
}
merged := make([]runtime.Object, len(m))
merged := make([]client.Object, len(m))
i := 0
for _, obj := range m {
merged[i] = obj
@@ -55,7 +72,7 @@ func mergeCRDs(s1, s2 []runtime.Object) []runtime.Object {
return merged
}
func runtimeCRDListToUnstructured(l []runtime.Object) []*unstructured.Unstructured {
func runtimeCRDListToUnstructured(l []client.Object) []*unstructured.Unstructured {
res := []*unstructured.Unstructured{}
for _, obj := range l {
u := &unstructured.Unstructured{}

12
vendor/sigs.k8s.io/controller-runtime/pkg/envtest/printer/ginkgo.go generated vendored
View File

@@ -33,21 +33,21 @@ var _ ginkgo.Reporter = NewlineReporter{}
// See issue https://github.com/jstemmer/go-junit-report/issues/31
type NewlineReporter struct{}
// SpecSuiteWillBegin implements ginkgo.Reporter
// SpecSuiteWillBegin implements ginkgo.Reporter.
func (NewlineReporter) SpecSuiteWillBegin(config config.GinkgoConfigType, summary *types.SuiteSummary) {
}
// BeforeSuiteDidRun implements ginkgo.Reporter
// BeforeSuiteDidRun implements ginkgo.Reporter.
func (NewlineReporter) BeforeSuiteDidRun(setupSummary *types.SetupSummary) {}
// AfterSuiteDidRun implements ginkgo.Reporter
// AfterSuiteDidRun implements ginkgo.Reporter.
func (NewlineReporter) AfterSuiteDidRun(setupSummary *types.SetupSummary) {}
// SpecWillRun implements ginkgo.Reporter
// SpecWillRun implements ginkgo.Reporter.
func (NewlineReporter) SpecWillRun(specSummary *types.SpecSummary) {}
// SpecDidComplete implements ginkgo.Reporter
// SpecDidComplete implements ginkgo.Reporter.
func (NewlineReporter) SpecDidComplete(specSummary *types.SpecSummary) {}
// SpecSuiteDidEnd Prints a newline between "35 Passed | 0 Failed | 0 Pending | 0 Skipped" and "--- PASS:"
// SpecSuiteDidEnd Prints a newline between "35 Passed | 0 Failed | 0 Pending | 0 Skipped" and "--- PASS:".
func (NewlineReporter) SpecSuiteDidEnd(summary *types.SuiteSummary) { fmt.Printf("\n") }

10
vendor/sigs.k8s.io/controller-runtime/pkg/envtest/printer/prow.go generated vendored
View File

@@ -73,35 +73,35 @@ func (pr *prowReporter) SpecSuiteWillBegin(config config.GinkgoConfigType, summa
}
}
// BeforeSuiteDidRun implements ginkgo.Reporter
// BeforeSuiteDidRun implements ginkgo.Reporter.
func (pr *prowReporter) BeforeSuiteDidRun(setupSummary *types.SetupSummary) {
if pr.junitReporter != nil {
pr.junitReporter.BeforeSuiteDidRun(setupSummary)
}
}
// AfterSuiteDidRun implements ginkgo.Reporter
// AfterSuiteDidRun implements ginkgo.Reporter.
func (pr *prowReporter) AfterSuiteDidRun(setupSummary *types.SetupSummary) {
if pr.junitReporter != nil {
pr.junitReporter.AfterSuiteDidRun(setupSummary)
}
}
// SpecWillRun implements ginkgo.Reporter
// SpecWillRun implements ginkgo.Reporter.
func (pr *prowReporter) SpecWillRun(specSummary *types.SpecSummary) {
if pr.junitReporter != nil {
pr.junitReporter.SpecWillRun(specSummary)
}
}
// SpecDidComplete implements ginkgo.Reporter
// SpecDidComplete implements ginkgo.Reporter.
func (pr *prowReporter) SpecDidComplete(specSummary *types.SpecSummary) {
if pr.junitReporter != nil {
pr.junitReporter.SpecDidComplete(specSummary)
}
}
// SpecSuiteDidEnd Prints a newline between "35 Passed | 0 Failed | 0 Pending | 0 Skipped" and "--- PASS:"
// SpecSuiteDidEnd Prints a newline between "35 Passed | 0 Failed | 0 Pending | 0 Skipped" and "--- PASS:".
func (pr *prowReporter) SpecSuiteDidEnd(summary *types.SuiteSummary) {
if pr.junitReporter != nil {
pr.junitReporter.SpecSuiteDidEnd(summary)

227
vendor/sigs.k8s.io/controller-runtime/pkg/envtest/server.go generated vendored
View File

@@ -19,14 +19,16 @@ package envtest
import (
"fmt"
"os"
"path/filepath"
"strings"
"time"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/config"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/integration"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/controlplane"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/process"
logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
)
@@ -46,46 +48,74 @@ It's possible to override some defaults, by setting the following environment va
*/
const (
envUseExistingCluster = "USE_EXISTING_CLUSTER"
envKubeAPIServerBin = "TEST_ASSET_KUBE_APISERVER"
envEtcdBin = "TEST_ASSET_ETCD"
envKubectlBin = "TEST_ASSET_KUBECTL"
envKubebuilderPath = "KUBEBUILDER_ASSETS"
envStartTimeout = "KUBEBUILDER_CONTROLPLANE_START_TIMEOUT"
envStopTimeout = "KUBEBUILDER_CONTROLPLANE_STOP_TIMEOUT"
envAttachOutput = "KUBEBUILDER_ATTACH_CONTROL_PLANE_OUTPUT"
defaultKubebuilderPath = "/usr/local/kubebuilder/bin"
StartTimeout = 60
StopTimeout = 60
envUseExistingCluster = "USE_EXISTING_CLUSTER"
envStartTimeout = "KUBEBUILDER_CONTROLPLANE_START_TIMEOUT"
envStopTimeout = "KUBEBUILDER_CONTROLPLANE_STOP_TIMEOUT"
envAttachOutput = "KUBEBUILDER_ATTACH_CONTROL_PLANE_OUTPUT"
StartTimeout = 60
StopTimeout = 60
defaultKubebuilderControlPlaneStartTimeout = 20 * time.Second
defaultKubebuilderControlPlaneStopTimeout = 20 * time.Second
)
// Default binary path for test framework
func defaultAssetPath(binary string) string {
assetPath := os.Getenv(envKubebuilderPath)
if assetPath == "" {
assetPath = defaultKubebuilderPath
}
return filepath.Join(assetPath, binary)
// internal types we expose as part of our public API.
type (
// ControlPlane is the re-exported ControlPlane type from the internal testing package.
ControlPlane = controlplane.ControlPlane
}
// APIServer is the re-exported APIServer from the internal testing package.
APIServer = controlplane.APIServer
// ControlPlane is the re-exported ControlPlane type from the internal integration package
type ControlPlane = integration.ControlPlane
// Etcd is the re-exported Etcd from the internal testing package.
Etcd = controlplane.Etcd
// APIServer is the re-exported APIServer type from the internal integration package
type APIServer = integration.APIServer
// User represents a Kubernetes user to provision for auth purposes.
User = controlplane.User
// Etcd is the re-exported Etcd type from the internal integration package
type Etcd = integration.Etcd
// AuthenticatedUser represets a Kubernetes user that's been provisioned.
AuthenticatedUser = controlplane.AuthenticatedUser
// ListenAddr indicates the address and port that the API server should listen on.
ListenAddr = process.ListenAddr
// SecureServing contains details describing how the API server should serve
// its secure endpoint.
SecureServing = controlplane.SecureServing
// Authn is an authentication method that can be used with the control plane to
// provision users.
Authn = controlplane.Authn
// Arguments allows configuring a process's flags.
Arguments = process.Arguments
// Arg is a single flag with one or more values.
Arg = process.Arg
)
var (
// EmptyArguments constructs a new set of flags with nothing set.
//
// This is mostly useful for testing helper methods -- you'll want to call
// Configure on the APIServer (or etcd) to configure their arguments.
EmptyArguments = process.EmptyArguments
)
// Environment creates a Kubernetes test environment that will start / stop the Kubernetes control plane and
// install extension APIs
// install extension APIs.
type Environment struct {
// ControlPlane is the ControlPlane including the apiserver and etcd
ControlPlane integration.ControlPlane
ControlPlane controlplane.ControlPlane
// Scheme is used to determine if conversion webhooks should be enabled
// for a particular CRD / object.
//
// Conversion webhooks are going to be enabled if an object in the scheme
// implements Hub and Spoke conversions.
//
// If nil, scheme.Scheme is used.
Scheme *runtime.Scheme
// Config can be used to talk to the apiserver. It's automatically
// populated if not set using the standard controller-runtime config
@@ -106,14 +136,18 @@ type Environment struct {
// CRDs is a list of CRDs to install.
// If both this field and CRDs field in CRDInstallOptions are specified, the
// values are merged.
CRDs []runtime.Object
CRDs []client.Object
// CRDDirectoryPaths is a list of paths containing CRD yaml or json configs.
// If both this field and Paths field in CRDInstallOptions are specified, the
// values are merged.
CRDDirectoryPaths []string
// UseExisting indicates that this environments should use an
// BinaryAssetsDirectory is the path where the binaries required for the envtest are
// located in the local environment. This field can be overridden by setting KUBEBUILDER_ASSETS.
BinaryAssetsDirectory string
// UseExistingCluster indicates that this environments should use an
// existing kubeconfig, instead of trying to stand up a new control plane.
// This is useful in cases that need aggregated API servers and the like.
UseExistingCluster *bool
@@ -129,6 +163,8 @@ type Environment struct {
ControlPlaneStopTimeout time.Duration
// KubeAPIServerFlags is the set of flags passed while starting the api server.
//
// Deprecated: use ControlPlane.GetAPIServer().Configure() instead.
KubeAPIServerFlags []string
// AttachControlPlaneOutput indicates if control plane output will be attached to os.Stdout and os.Stderr.
@@ -146,38 +182,19 @@ func (te *Environment) Stop() error {
return err
}
}
if err := te.WebhookInstallOptions.Cleanup(); err != nil {
return err
}
if te.useExistingCluster() {
return nil
}
err := te.WebhookInstallOptions.Cleanup()
if err != nil {
return err
}
return te.ControlPlane.Stop()
}
// getAPIServerFlags returns flags to be used with the Kubernetes API server.
// it returns empty slice for api server defined defaults to be applied if no args specified
func (te Environment) getAPIServerFlags() []string {
// Set default API server flags if not set.
if len(te.KubeAPIServerFlags) == 0 {
return []string{}
}
// Check KubeAPIServerFlags contains service-cluster-ip-range, if not, set default value to service-cluster-ip-range
containServiceClusterIPRange := false
for _, flag := range te.KubeAPIServerFlags {
if strings.Contains(flag, "service-cluster-ip-range") {
containServiceClusterIPRange = true
break
}
}
if !containServiceClusterIPRange {
te.KubeAPIServerFlags = append(te.KubeAPIServerFlags, "--service-cluster-ip-range=10.0.0.0/24")
}
return te.KubeAPIServerFlags
}
// Start starts a local Kubernetes server and updates te.ApiserverPort with the port it is listening on
// Start starts a local Kubernetes server and updates te.ApiserverPort with the port it is listening on.
func (te *Environment) Start() (*rest.Config, error) {
if te.useExistingCluster() {
log.V(1).Info("using existing cluster")
@@ -189,25 +206,36 @@ func (te *Environment) Start() (*rest.Config, error) {
var err error
te.Config, err = config.GetConfig()
if err != nil {
return nil, err
return nil, fmt.Errorf("unable to get configuration for existing cluster: %w", err)
}
}
} else {
if te.ControlPlane.APIServer == nil {
te.ControlPlane.APIServer = &integration.APIServer{Args: te.getAPIServerFlags()}
apiServer := te.ControlPlane.GetAPIServer()
if len(apiServer.Args) == 0 { //nolint:staticcheck
// pass these through separately from above in case something like
// AddUser defaults APIServer.
//
// TODO(directxman12): if/when we feel like making a bigger
// breaking change here, just make APIServer and Etcd non-pointers
// in ControlPlane.
// NB(directxman12): we still pass these in so that things work if the
// user manually specifies them, but in most cases we expect them to
// be nil so that we use the new .Configure() logic.
apiServer.Args = te.KubeAPIServerFlags //nolint:staticcheck
}
if te.ControlPlane.Etcd == nil {
te.ControlPlane.Etcd = &integration.Etcd{}
te.ControlPlane.Etcd = &controlplane.Etcd{}
}
if os.Getenv(envAttachOutput) == "true" {
te.AttachControlPlaneOutput = true
}
if te.ControlPlane.APIServer.Out == nil && te.AttachControlPlaneOutput {
te.ControlPlane.APIServer.Out = os.Stdout
if apiServer.Out == nil && te.AttachControlPlaneOutput {
apiServer.Out = os.Stdout
}
if te.ControlPlane.APIServer.Err == nil && te.AttachControlPlaneOutput {
te.ControlPlane.APIServer.Err = os.Stderr
if apiServer.Err == nil && te.AttachControlPlaneOutput {
apiServer.Err = os.Stderr
}
if te.ControlPlane.Etcd.Out == nil && te.AttachControlPlaneOutput {
te.ControlPlane.Etcd.Out = os.Stdout
@@ -216,55 +244,78 @@ func (te *Environment) Start() (*rest.Config, error) {
te.ControlPlane.Etcd.Err = os.Stderr
}
if os.Getenv(envKubeAPIServerBin) == "" {
te.ControlPlane.APIServer.Path = defaultAssetPath("kube-apiserver")
}
if os.Getenv(envEtcdBin) == "" {
te.ControlPlane.Etcd.Path = defaultAssetPath("etcd")
}
if os.Getenv(envKubectlBin) == "" {
// we can't just set the path manually (it's behind a function), so set the environment variable instead
if err := os.Setenv(envKubectlBin, defaultAssetPath("kubectl")); err != nil {
return nil, err
}
}
apiServer.Path = process.BinPathFinder("kube-apiserver", te.BinaryAssetsDirectory)
te.ControlPlane.Etcd.Path = process.BinPathFinder("etcd", te.BinaryAssetsDirectory)
te.ControlPlane.KubectlPath = process.BinPathFinder("kubectl", te.BinaryAssetsDirectory)
if err := te.defaultTimeouts(); err != nil {
return nil, fmt.Errorf("failed to default controlplane timeouts: %w", err)
}
te.ControlPlane.Etcd.StartTimeout = te.ControlPlaneStartTimeout
te.ControlPlane.Etcd.StopTimeout = te.ControlPlaneStopTimeout
te.ControlPlane.APIServer.StartTimeout = te.ControlPlaneStartTimeout
te.ControlPlane.APIServer.StopTimeout = te.ControlPlaneStopTimeout
apiServer.StartTimeout = te.ControlPlaneStartTimeout
apiServer.StopTimeout = te.ControlPlaneStopTimeout
log.V(1).Info("starting control plane", "api server flags", te.ControlPlane.APIServer.Args)
log.V(1).Info("starting control plane")
if err := te.startControlPlane(); err != nil {
return nil, err
return nil, fmt.Errorf("unable to start control plane itself: %w", err)
}
// Create the *rest.Config for creating new clients
te.Config = &rest.Config{
Host: te.ControlPlane.APIURL().Host,
baseConfig := &rest.Config{
// gotta go fast during tests -- we don't really care about overwhelming our test API server
QPS: 1000.0,
Burst: 2000.0,
}
adminInfo := User{Name: "admin", Groups: []string{"system:masters"}}
adminUser, err := te.ControlPlane.AddUser(adminInfo, baseConfig)
if err != nil {
return te.Config, fmt.Errorf("unable to provision admin user: %w", err)
}
te.Config = adminUser.Config()
}
// Set the default scheme if nil.
if te.Scheme == nil {
te.Scheme = scheme.Scheme
}
// Call PrepWithoutInstalling to setup certificates first
// and have them available to patch CRD conversion webhook as well.
if err := te.WebhookInstallOptions.PrepWithoutInstalling(); err != nil {
return nil, err
}
log.V(1).Info("installing CRDs")
te.CRDInstallOptions.CRDs = mergeCRDs(te.CRDInstallOptions.CRDs, te.CRDs)
te.CRDInstallOptions.Paths = mergePaths(te.CRDInstallOptions.Paths, te.CRDDirectoryPaths)
te.CRDInstallOptions.ErrorIfPathMissing = te.ErrorIfCRDPathMissing
te.CRDInstallOptions.WebhookOptions = te.WebhookInstallOptions
crds, err := InstallCRDs(te.Config, te.CRDInstallOptions)
if err != nil {
return te.Config, err
return te.Config, fmt.Errorf("unable to install CRDs onto control plane: %w", err)
}
te.CRDs = crds
log.V(1).Info("installing webhooks")
err = te.WebhookInstallOptions.Install(te.Config)
if err := te.WebhookInstallOptions.Install(te.Config); err != nil {
return nil, fmt.Errorf("unable to install webhooks onto control plane: %w", err)
}
return te.Config, nil
}
return te.Config, err
// AddUser provisions a new user for connecting to this Environment. The user will
// have the specified name & belong to the specified groups.
//
// If you specify a "base" config, the returned REST Config will contain those
// settings as well as any required by the authentication method. You can use
// this to easily specify options like QPS.
//
// This is effectively a convinience alias for ControlPlane.AddUser -- see that
// for more low-level details.
func (te *Environment) AddUser(user User, baseConfig *rest.Config) (*AuthenticatedUser, error) {
return te.ControlPlane.AddUser(user, baseConfig)
}
func (te *Environment) startControlPlane() error {
@@ -319,4 +370,6 @@ func (te *Environment) useExistingCluster() bool {
// DefaultKubeAPIServerFlags exposes the default args for the APIServer so that
// you can use those to append your own additional arguments.
var DefaultKubeAPIServerFlags = integration.APIServerDefaultArgs
//
// Deprecated: use APIServer.Configure() instead.
var DefaultKubeAPIServerFlags = controlplane.APIServerDefaultArgs //nolint:staticcheck

140
vendor/sigs.k8s.io/controller-runtime/pkg/envtest/webhook.go generated vendored
View File

@@ -23,7 +23,6 @@ import (
"path/filepath"
"time"
"k8s.io/apimachinery/pkg/api/errors"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -33,21 +32,21 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/integration"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/integration/addr"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/addr"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/certs"
"sigs.k8s.io/yaml"
)
// WebhookInstallOptions are the options for installing mutating or validating webhooks
// WebhookInstallOptions are the options for installing mutating or validating webhooks.
type WebhookInstallOptions struct {
// Paths is a list of paths to the directories containing the mutating or validating webhooks yaml or json configs.
DirectoryPaths []string
// Paths is a list of paths to the directories or files containing the mutating or validating webhooks yaml or json configs.
Paths []string
// MutatingWebhooks is a list of MutatingWebhookConfigurations to install
MutatingWebhooks []runtime.Object
MutatingWebhooks []client.Object
// ValidatingWebhooks is a list of ValidatingWebhookConfigurations to install
ValidatingWebhooks []runtime.Object
ValidatingWebhooks []client.Object
// IgnoreErrorIfPathMissing will ignore an error if a DirectoryPath does not exist when set to true
IgnoreErrorIfPathMissing bool
@@ -67,6 +66,9 @@ type WebhookInstallOptions struct {
// CAData is the CA that can be used to trust the serving certificates in LocalServingCertDir.
LocalServingCAData []byte
// LocalServingHostExternalName is the hostname to use to reach the webhook server.
LocalServingHostExternalName string
// MaxTime is the max time to wait
MaxTime time.Duration
@@ -76,8 +78,11 @@ type WebhookInstallOptions struct {
// ModifyWebhookDefinitions modifies webhook definitions by:
// - applying CABundle based on the provided tinyca
// - if webhook client config uses service spec, it's removed and replaced with direct url
func (o *WebhookInstallOptions) ModifyWebhookDefinitions(caData []byte) error {
// - if webhook client config uses service spec, it's removed and replaced with direct url.
func (o *WebhookInstallOptions) ModifyWebhookDefinitions() error {
caData := o.LocalServingCAData
// generate host port.
hostPort, err := o.generateHostPort()
if err != nil {
return err
@@ -137,13 +142,19 @@ func modifyWebhook(webhook map[string]interface{}, caData []byte, hostPort strin
}
func (o *WebhookInstallOptions) generateHostPort() (string, error) {
port, host, err := addr.Suggest()
if err != nil {
return "", fmt.Errorf("unable to grab random port for serving webhooks on: %v", err)
if o.LocalServingPort == 0 {
port, host, err := addr.Suggest(o.LocalServingHost)
if err != nil {
return "", fmt.Errorf("unable to grab random port for serving webhooks on: %v", err)
}
o.LocalServingPort = port
o.LocalServingHost = host
}
o.LocalServingPort = port
o.LocalServingHost = host
return net.JoinHostPort(host, fmt.Sprintf("%d", port)), nil
host := o.LocalServingHostExternalName
if host == "" {
host = o.LocalServingHost
}
return net.JoinHostPort(host, fmt.Sprintf("%d", o.LocalServingPort)), nil
}
// PrepWithoutInstalling does the setup parts of Install (populating host-port,
@@ -152,40 +163,33 @@ func (o *WebhookInstallOptions) generateHostPort() (string, error) {
// controller-runtime, where we need a random host-port & caData for webhook
// tests, but may be useful in similar scenarios.
func (o *WebhookInstallOptions) PrepWithoutInstalling() error {
hookCA, err := o.setupCA()
if err != nil {
return err
}
if err := parseWebhookDirs(o); err != nil {
if err := o.setupCA(); err != nil {
return err
}
err = o.ModifyWebhookDefinitions(hookCA)
if err != nil {
if err := parseWebhook(o); err != nil {
return err
}
return nil
return o.ModifyWebhookDefinitions()
}
// Install installs specified webhooks to the API server
// Install installs specified webhooks to the API server.
func (o *WebhookInstallOptions) Install(config *rest.Config) error {
if err := o.PrepWithoutInstalling(); err != nil {
return err
if len(o.LocalServingCAData) == 0 {
if err := o.PrepWithoutInstalling(); err != nil {
return err
}
}
if err := createWebhooks(config, o.MutatingWebhooks, o.ValidatingWebhooks); err != nil {
return err
}
if err := WaitForWebhooks(config, o.MutatingWebhooks, o.ValidatingWebhooks, *o); err != nil {
return err
}
return nil
return WaitForWebhooks(config, o.MutatingWebhooks, o.ValidatingWebhooks, *o)
}
// Cleanup cleans up cert directories
// Cleanup cleans up cert directories.
func (o *WebhookInstallOptions) Cleanup() error {
if o.LocalServingCertDir != "" {
return os.RemoveAll(o.LocalServingCertDir)
@@ -193,12 +197,11 @@ func (o *WebhookInstallOptions) Cleanup() error {
return nil
}
// WaitForWebhooks waits for the Webhooks to be available through API server
// WaitForWebhooks waits for the Webhooks to be available through API server.
func WaitForWebhooks(config *rest.Config,
mutatingWebhooks []runtime.Object,
validatingWebhooks []runtime.Object,
mutatingWebhooks []client.Object,
validatingWebhooks []client.Object,
options WebhookInstallOptions) error {
waitingFor := map[schema.GroupVersionKind]*sets.String{}
for _, hook := range runtimeListToUnstructured(append(validatingWebhooks, mutatingWebhooks...)) {
@@ -213,7 +216,7 @@ func WaitForWebhooks(config *rest.Config,
return wait.PollImmediate(options.PollInterval, options.MaxTime, p.poll)
}
// poller checks if all the resources have been found in discovery, and returns false if not
// poller checks if all the resources have been found in discovery, and returns false if not.
type webhookPoller struct {
// config is used to get discovery
config *rest.Config
@@ -222,7 +225,7 @@ type webhookPoller struct {
waitingFor map[schema.GroupVersionKind]*sets.String
}
// poll checks if all the resources have been found in discovery, and returns false if not
// poll checks if all the resources have been found in discovery, and returns false if not.
func (p *webhookPoller) poll() (done bool, err error) {
// Create a new clientset to avoid any client caching of discovery
c, err := client.New(p.config, client.Options{})
@@ -248,7 +251,7 @@ func (p *webhookPoller) poll() (done bool, err error) {
names.Delete(name)
}
if errors.IsNotFound(err) {
if apierrors.IsNotFound(err) {
allFound = false
}
if err != nil {
@@ -259,41 +262,42 @@ func (p *webhookPoller) poll() (done bool, err error) {
return allFound, nil
}
// setupCA creates CA for testing and writes them to disk
func (o *WebhookInstallOptions) setupCA() ([]byte, error) {
hookCA, err := integration.NewTinyCA()
// setupCA creates CA for testing and writes them to disk.
func (o *WebhookInstallOptions) setupCA() error {
hookCA, err := certs.NewTinyCA()
if err != nil {
return nil, fmt.Errorf("unable to set up webhook CA: %v", err)
return fmt.Errorf("unable to set up webhook CA: %v", err)
}
hookCert, err := hookCA.NewServingCert()
names := []string{"localhost", o.LocalServingHost, o.LocalServingHostExternalName}
hookCert, err := hookCA.NewServingCert(names...)
if err != nil {
return nil, fmt.Errorf("unable to set up webhook serving certs: %v", err)
return fmt.Errorf("unable to set up webhook serving certs: %v", err)
}
localServingCertsDir, err := ioutil.TempDir("", "envtest-serving-certs-")
o.LocalServingCertDir = localServingCertsDir
if err != nil {
return nil, fmt.Errorf("unable to create directory for webhook serving certs: %v", err)
return fmt.Errorf("unable to create directory for webhook serving certs: %v", err)
}
certData, keyData, err := hookCert.AsBytes()
if err != nil {
return nil, fmt.Errorf("unable to marshal webhook serving certs: %v", err)
return fmt.Errorf("unable to marshal webhook serving certs: %v", err)
}
if err := ioutil.WriteFile(filepath.Join(localServingCertsDir, "tls.crt"), certData, 0640); err != nil {
return nil, fmt.Errorf("unable to write webhook serving cert to disk: %v", err)
if err := ioutil.WriteFile(filepath.Join(localServingCertsDir, "tls.crt"), certData, 0640); err != nil { //nolint:gosec
return fmt.Errorf("unable to write webhook serving cert to disk: %v", err)
}
if err := ioutil.WriteFile(filepath.Join(localServingCertsDir, "tls.key"), keyData, 0640); err != nil {
return nil, fmt.Errorf("unable to write webhook serving key to disk: %v", err)
if err := ioutil.WriteFile(filepath.Join(localServingCertsDir, "tls.key"), keyData, 0640); err != nil { //nolint:gosec
return fmt.Errorf("unable to write webhook serving key to disk: %v", err)
}
o.LocalServingCAData = certData
return certData, nil
return err
}
func createWebhooks(config *rest.Config, mutHooks []runtime.Object, valHooks []runtime.Object) error {
func createWebhooks(config *rest.Config, mutHooks []client.Object, valHooks []client.Object) error {
cs, err := client.New(config, client.Options{})
if err != nil {
return err
@@ -315,7 +319,7 @@ func createWebhooks(config *rest.Config, mutHooks []runtime.Object, valHooks []r
return nil
}
// ensureCreated creates or update object if already exists in the cluster
// ensureCreated creates or update object if already exists in the cluster.
func ensureCreated(cs client.Client, obj *unstructured.Unstructured) error {
existing := obj.DeepCopy()
err := cs.Get(context.Background(), client.ObjectKey{Name: obj.GetName()}, existing)
@@ -336,10 +340,10 @@ func ensureCreated(cs client.Client, obj *unstructured.Unstructured) error {
return nil
}
// parseWebhookDirs reads the directories of Webhooks in options.DirectoryPaths and adds the Webhook structs to options
func parseWebhookDirs(options *WebhookInstallOptions) error {
if len(options.DirectoryPaths) > 0 {
for _, path := range options.DirectoryPaths {
// parseWebhook reads the directories or files of Webhooks in options.Paths and adds the Webhook structs to options.
func parseWebhook(options *WebhookInstallOptions) error {
if len(options.Paths) > 0 {
for _, path := range options.Paths {
_, err := os.Stat(path)
if options.IgnoreErrorIfPathMissing && os.IsNotExist(err) {
continue // skip this path
@@ -359,21 +363,27 @@ func parseWebhookDirs(options *WebhookInstallOptions) error {
}
// readWebhooks reads the Webhooks from files and Unmarshals them into structs
// returns slice of mutating and validating webhook configurations
func readWebhooks(path string) ([]runtime.Object, []runtime.Object, error) {
// returns slice of mutating and validating webhook configurations.
func readWebhooks(path string) ([]client.Object, []client.Object, error) {
// Get the webhook files
var files []os.FileInfo
var err error
log.V(1).Info("reading Webhooks from path", "path", path)
if files, err = ioutil.ReadDir(path); err != nil {
info, err := os.Stat(path)
if err != nil {
return nil, nil, err
}
if !info.IsDir() {
path, files = filepath.Dir(path), []os.FileInfo{info}
} else if files, err = ioutil.ReadDir(path); err != nil {
return nil, nil, err
}
// file extensions that may contain Webhooks
resourceExtensions := sets.NewString(".json", ".yaml", ".yml")
var mutHooks []runtime.Object
var valHooks []runtime.Object
var mutHooks []client.Object
var valHooks []client.Object
for _, file := range files {
// Only parse allowlisted file types
if !resourceExtensions.Has(filepath.Ext(file.Name())) {
@@ -425,7 +435,7 @@ func readWebhooks(path string) ([]runtime.Object, []runtime.Object, error) {
return mutHooks, valHooks, nil
}
func runtimeListToUnstructured(l []runtime.Object) []*unstructured.Unstructured {
func runtimeListToUnstructured(l []client.Object) []*unstructured.Unstructured {
res := []*unstructured.Unstructured{}
for _, obj := range l {
m, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj.DeepCopyObject())