feat: kubesphere 4.0 (#6115)

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

---------

Signed-off-by: ci-bot <ci-bot@kubesphere.io>
Co-authored-by: ks-ci-bot <ks-ci-bot@example.com>
Co-authored-by: joyceliu <joyceliu@yunify.com>
This commit is contained in:
KubeSphere CI Bot
2024-09-06 11:05:52 +08:00
committed by GitHub
parent b5015ec7b9
commit 447a51f08b
8557 changed files with 546695 additions and 1146174 deletions

View File

@@ -31,6 +31,16 @@ func Errorf(msg string, args ...interface{}) error {
return goerrors.Wrap(fmt.Errorf(msg, args...), 1)
}
// As finds the targeted error in any wrapped error.
func As(err error, target interface{}) bool {
return goerrors.As(err, target)
}
// Is detects whether the error is equal to a given error.
func Is(err error, target error) bool {
return goerrors.Is(err, target)
}
// GetStack returns a stack trace for the error if it has one
func GetStack(err error) string {
if e, ok := err.(*goerrors.Error); ok {

View File

@@ -4,7 +4,7 @@
package filesys
import (
"io/ioutil"
"os"
"path/filepath"
"strings"
)
@@ -17,12 +17,12 @@ type ConfirmedDir string
// The directory is cleaned, no symlinks, etc. so it's
// returned as a ConfirmedDir.
func NewTmpConfirmedDir() (ConfirmedDir, error) {
n, err := ioutil.TempDir("", "kustomize-")
n, err := os.MkdirTemp("", "kustomize-")
if err != nil {
return "", err
}
// In MacOs `ioutil.TempDir` creates a directory
// In MacOs `os.MkdirTemp` creates a directory
// with root in the `/var` folder, which is in turn
// a symlinked path to `/private/var`.
// Function `filepath.EvalSymlinks`is used to

View File

@@ -5,7 +5,6 @@ package filesys
import (
"fmt"
"io/ioutil"
"log"
"os"
"path/filepath"
@@ -125,12 +124,15 @@ func (fsOnDisk) ReadDir(name string) ([]string, error) {
return result, nil
}
// ReadFile delegates to ioutil.ReadFile.
func (fsOnDisk) ReadFile(name string) ([]byte, error) { return ioutil.ReadFile(name) }
// ReadFile delegates to os.ReadFile.
func (fsOnDisk) ReadFile(name string) ([]byte, error) {
content, err := os.ReadFile(name)
return content, errors.Wrap(err)
}
// WriteFile delegates to ioutil.WriteFile with read/write permissions.
// WriteFile delegates to os.WriteFile with read/write permissions.
func (fsOnDisk) WriteFile(name string, c []byte) error {
return errors.Wrap(ioutil.WriteFile(name, c, 0666)) //nolint:gosec
return errors.Wrap(os.WriteFile(name, c, 0666)) //nolint:gosec
}
// Walk delegates to filepath.Walk.

View File

@@ -74,30 +74,30 @@ func PathJoin(incoming []string) string {
//
// E.g. if part == 'PEACH'
//
// OLD : NEW : POS
// --------------------------------------------------------
// {empty} : PEACH : irrelevant
// / : /PEACH : irrelevant
// pie : PEACH/pie : 0 (or negative)
// /pie : /PEACH/pie : 0 (or negative)
// raw : raw/PEACH : 1 (or larger)
// /raw : /raw/PEACH : 1 (or larger)
// a/nice/warm/pie : a/nice/warm/PEACH/pie : 3
// /a/nice/warm/pie : /a/nice/warm/PEACH/pie : 3
// OLD : NEW : POS
// --------------------------------------------------------
// {empty} : PEACH : irrelevant
// / : /PEACH : irrelevant
// pie : PEACH/pie : 0 (or negative)
// /pie : /PEACH/pie : 0 (or negative)
// raw : raw/PEACH : 1 (or larger)
// /raw : /raw/PEACH : 1 (or larger)
// a/nice/warm/pie : a/nice/warm/PEACH/pie : 3
// /a/nice/warm/pie : /a/nice/warm/PEACH/pie : 3
//
// * An empty part results in no change.
//
// * Absolute paths get their leading '/' stripped, treated like
// relative paths, and the leading '/' is re-added on output.
// The meaning of pos is intentionally the same in either absolute or
// relative paths; if it weren't, this function could convert absolute
// paths to relative paths, which is not desirable.
// - Absolute paths get their leading '/' stripped, treated like
// relative paths, and the leading '/' is re-added on output.
// The meaning of pos is intentionally the same in either absolute or
// relative paths; if it weren't, this function could convert absolute
// paths to relative paths, which is not desirable.
//
// * For robustness (liberal input, conservative output) Pos values that
// that are too small (large) to index the split filepath result in a
// prefix (postfix) rather than an error. Use extreme position values
// to assure a prefix or postfix (e.g. 0 will always prefix, and
// 9999 will presumably always postfix).
// - For robustness (liberal input, conservative output) Pos values
// that are too small (large) to index the split filepath result in a
// prefix (postfix) rather than an error. Use extreme position values
// to assure a prefix or postfix (e.g. 0 will always prefix, and
// 9999 will presumably always postfix).
func InsertPathPart(path string, pos int, part string) string {
if part == "" {
return path
@@ -121,7 +121,7 @@ func InsertPathPart(path string, pos int, part string) string {
result := make([]string, len(parts)+1)
copy(result, parts[0:pos])
result[pos] = part
return PathJoin(append(result, parts[pos:]...)) // nolint: makezero
return PathJoin(append(result, parts[pos:]...)) //nolint: makezero
}
func IsHiddenFilePath(pattern string) bool {

View File

@@ -34,7 +34,7 @@ func (c *Filter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
}
func (c *Filter) Run(reader io.Reader, writer io.Writer) error {
cmd := exec.Command(c.Path, c.Args...) // nolint:gosec
cmd := exec.Command(c.Path, c.Args...) //nolint:gosec
cmd.Stdin = reader
cmd.Stdout = writer
cmd.Stderr = os.Stderr

View File

@@ -10,6 +10,7 @@ import (
"strings"
"sigs.k8s.io/kustomize/kyaml/yaml"
k8syaml "sigs.k8s.io/yaml"
)
const (
@@ -200,50 +201,57 @@ func (s *StorageMount) String() string {
//
// The FunctionSpec is read from the resource metadata.annotation
// "config.kubernetes.io/function"
func GetFunctionSpec(n *yaml.RNode) *FunctionSpec {
func GetFunctionSpec(n *yaml.RNode) (*FunctionSpec, error) {
meta, err := n.GetMeta()
if err != nil {
return nil
return nil, fmt.Errorf("failed to get ResourceMeta: %w", err)
}
if fn := getFunctionSpecFromAnnotation(n, meta); fn != nil {
return fn
fn, err := getFunctionSpecFromAnnotation(n, meta)
if err != nil {
return nil, err
}
if fn != nil {
return fn, nil
}
// legacy function specification for backwards compatibility
container := meta.Annotations["config.kubernetes.io/container"]
if container != "" {
return &FunctionSpec{Container: ContainerSpec{Image: container}}
return &FunctionSpec{Container: ContainerSpec{Image: container}}, nil
}
return nil
return nil, nil
}
// getFunctionSpecFromAnnotation parses the config function from an annotation
// if it is found
func getFunctionSpecFromAnnotation(n *yaml.RNode, meta yaml.ResourceMeta) *FunctionSpec {
func getFunctionSpecFromAnnotation(n *yaml.RNode, meta yaml.ResourceMeta) (*FunctionSpec, error) {
var fs FunctionSpec
for _, s := range functionAnnotationKeys {
fn := meta.Annotations[s]
if fn != "" {
err := yaml.Unmarshal([]byte(fn), &fs)
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
if err := k8syaml.UnmarshalStrict([]byte(fn), &fs); err != nil {
return nil, fmt.Errorf("%s unmarshal error: %w", s, err)
}
return &fs
return &fs, nil
}
}
n, err := n.Pipe(yaml.Lookup("metadata", "configFn"))
if err != nil || yaml.IsMissingOrNull(n) {
return nil
if err != nil {
return nil, fmt.Errorf("failed to look up metadata.configFn: %w", err)
}
if yaml.IsMissingOrNull(n) {
return nil, nil
}
s, err := n.String()
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
fmt.Fprintf(os.Stderr, "configFn parse error: %v\n", err)
return nil, fmt.Errorf("configFn parse error: %w", err)
}
err = yaml.Unmarshal([]byte(s), &fs)
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
if err := k8syaml.UnmarshalStrict([]byte(s), &fs); err != nil {
return nil, fmt.Errorf("%s unmarshal error: %w", "configFn", err)
}
return &fs
return &fs, nil
}
func StringToStorageMount(s string) StorageMount {
@@ -288,7 +296,11 @@ type IsReconcilerFilter struct {
func (c *IsReconcilerFilter) Filter(inputs []*yaml.RNode) ([]*yaml.RNode, error) {
var out []*yaml.RNode
for i := range inputs {
isFnResource := GetFunctionSpec(inputs[i]) != nil
functionSpec, err := GetFunctionSpec(inputs[i])
if err != nil {
return nil, err
}
isFnResource := functionSpec != nil
if isFnResource && !c.ExcludeReconcilers {
out = append(out, inputs[i])
}

View File

@@ -7,7 +7,7 @@ import (
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
"path"
"strings"
@@ -268,7 +268,7 @@ func (c *FunctionFilter) doResults(r *kio.ByteReader) error {
if err != nil {
return err
}
err = ioutil.WriteFile(c.ResultsFile, []byte(results), 0600)
err = os.WriteFile(c.ResultsFile, []byte(results), 0600)
if err != nil {
return err
}

View File

@@ -7,8 +7,8 @@ import (
"bytes"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"go.starlark.net/starlark"
"sigs.k8s.io/kustomize/kyaml/errors"
@@ -57,7 +57,7 @@ func (sf *Filter) setup() error {
// read the program from a file
if sf.Path != "" {
b, err := ioutil.ReadFile(sf.Path)
b, err := os.ReadFile(sf.Path)
if err != nil {
return err
}
@@ -72,7 +72,7 @@ func (sf *Filter) setup() error {
return err
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
b, err := io.ReadAll(resp.Body)
if err != nil {
return err
}

View File

@@ -100,7 +100,10 @@ func (p *parser) peek() yaml_event_type_t {
if p.event.typ != yaml_NO_EVENT {
return p.event.typ
}
if !yaml_parser_parse(&p.parser, &p.event) {
// It's curious choice from the underlying API to generally return a
// positive result on success, but on this case return true in an error
// scenario. This was the source of bugs in the past (issue #666).
if !yaml_parser_parse(&p.parser, &p.event) || p.parser.error != yaml_NO_ERROR {
p.fail()
}
return p.event.typ
@@ -320,6 +323,8 @@ type decoder struct {
decodeCount int
aliasCount int
aliasDepth int
mergedFields map[interface{}]bool
}
var (
@@ -808,6 +813,11 @@ func (d *decoder) mapping(n *Node, out reflect.Value) (good bool) {
}
}
mergedFields := d.mergedFields
d.mergedFields = nil
var mergeNode *Node
mapIsNew := false
if out.IsNil() {
out.Set(reflect.MakeMap(outt))
@@ -815,11 +825,18 @@ func (d *decoder) mapping(n *Node, out reflect.Value) (good bool) {
}
for i := 0; i < l; i += 2 {
if isMerge(n.Content[i]) {
d.merge(n.Content[i+1], out)
mergeNode = n.Content[i+1]
continue
}
k := reflect.New(kt).Elem()
if d.unmarshal(n.Content[i], k) {
if mergedFields != nil {
ki := k.Interface()
if mergedFields[ki] {
continue
}
mergedFields[ki] = true
}
kkind := k.Kind()
if kkind == reflect.Interface {
kkind = k.Elem().Kind()
@@ -833,6 +850,12 @@ func (d *decoder) mapping(n *Node, out reflect.Value) (good bool) {
}
}
}
d.mergedFields = mergedFields
if mergeNode != nil {
d.merge(n, mergeNode, out)
}
d.stringMapType = stringMapType
d.generalMapType = generalMapType
return true
@@ -844,7 +867,8 @@ func isStringMap(n *Node) bool {
}
l := len(n.Content)
for i := 0; i < l; i += 2 {
if n.Content[i].ShortTag() != strTag {
shortTag := n.Content[i].ShortTag()
if shortTag != strTag && shortTag != mergeTag {
return false
}
}
@@ -861,7 +885,6 @@ func (d *decoder) mappingStruct(n *Node, out reflect.Value) (good bool) {
var elemType reflect.Type
if sinfo.InlineMap != -1 {
inlineMap = out.Field(sinfo.InlineMap)
inlineMap.Set(reflect.New(inlineMap.Type()).Elem())
elemType = inlineMap.Type().Elem()
}
@@ -870,6 +893,9 @@ func (d *decoder) mappingStruct(n *Node, out reflect.Value) (good bool) {
d.prepare(n, field)
}
mergedFields := d.mergedFields
d.mergedFields = nil
var mergeNode *Node
var doneFields []bool
if d.uniqueKeys {
doneFields = make([]bool, len(sinfo.FieldsList))
@@ -879,13 +905,20 @@ func (d *decoder) mappingStruct(n *Node, out reflect.Value) (good bool) {
for i := 0; i < l; i += 2 {
ni := n.Content[i]
if isMerge(ni) {
d.merge(n.Content[i+1], out)
mergeNode = n.Content[i+1]
continue
}
if !d.unmarshal(ni, name) {
continue
}
if info, ok := sinfo.FieldsMap[name.String()]; ok {
sname := name.String()
if mergedFields != nil {
if mergedFields[sname] {
continue
}
mergedFields[sname] = true
}
if info, ok := sinfo.FieldsMap[sname]; ok {
if d.uniqueKeys {
if doneFields[info.Id] {
d.terrors = append(d.terrors, fmt.Sprintf("line %d: field %s already set in type %s", ni.Line, name.String(), out.Type()))
@@ -911,6 +944,11 @@ func (d *decoder) mappingStruct(n *Node, out reflect.Value) (good bool) {
d.terrors = append(d.terrors, fmt.Sprintf("line %d: field %s not found in type %s", ni.Line, name.String(), out.Type()))
}
}
d.mergedFields = mergedFields
if mergeNode != nil {
d.merge(n, mergeNode, out)
}
return true
}
@@ -918,19 +956,29 @@ func failWantMap() {
failf("map merge requires map or sequence of maps as the value")
}
func (d *decoder) merge(n *Node, out reflect.Value) {
switch n.Kind {
func (d *decoder) merge(parent *Node, merge *Node, out reflect.Value) {
mergedFields := d.mergedFields
if mergedFields == nil {
d.mergedFields = make(map[interface{}]bool)
for i := 0; i < len(parent.Content); i += 2 {
k := reflect.New(ifaceType).Elem()
if d.unmarshal(parent.Content[i], k) {
d.mergedFields[k.Interface()] = true
}
}
}
switch merge.Kind {
case MappingNode:
d.unmarshal(n, out)
d.unmarshal(merge, out)
case AliasNode:
if n.Alias != nil && n.Alias.Kind != MappingNode {
if merge.Alias != nil && merge.Alias.Kind != MappingNode {
failWantMap()
}
d.unmarshal(n, out)
d.unmarshal(merge, out)
case SequenceNode:
// Step backwards as earlier nodes take precedence.
for i := len(n.Content) - 1; i >= 0; i-- {
ni := n.Content[i]
for i := 0; i < len(merge.Content); i++ {
ni := merge.Content[i]
if ni.Kind == AliasNode {
if ni.Alias != nil && ni.Alias.Kind != MappingNode {
failWantMap()
@@ -943,6 +991,8 @@ func (d *decoder) merge(n *Node, out reflect.Value) {
default:
failWantMap()
}
d.mergedFields = mergedFields
}
func isMerge(n *Node) bool {

View File

@@ -687,6 +687,9 @@ func yaml_parser_parse_node(parser *yaml_parser_t, event *yaml_event_t, block, i
func yaml_parser_parse_block_sequence_entry(parser *yaml_parser_t, event *yaml_event_t, first bool) bool {
if first {
token := peek_token(parser)
if token == nil {
return false
}
parser.marks = append(parser.marks, token.start_mark)
skip_token(parser)
}
@@ -786,7 +789,7 @@ func yaml_parser_split_stem_comment(parser *yaml_parser_t, stem_len int) {
}
token := peek_token(parser)
if token.typ != yaml_BLOCK_SEQUENCE_START_TOKEN && token.typ != yaml_BLOCK_MAPPING_START_TOKEN {
if token == nil || token.typ != yaml_BLOCK_SEQUENCE_START_TOKEN && token.typ != yaml_BLOCK_MAPPING_START_TOKEN {
return
}
@@ -813,6 +816,9 @@ func yaml_parser_split_stem_comment(parser *yaml_parser_t, stem_len int) {
func yaml_parser_parse_block_mapping_key(parser *yaml_parser_t, event *yaml_event_t, first bool) bool {
if first {
token := peek_token(parser)
if token == nil {
return false
}
parser.marks = append(parser.marks, token.start_mark)
skip_token(parser)
}
@@ -922,6 +928,9 @@ func yaml_parser_parse_block_mapping_value(parser *yaml_parser_t, event *yaml_ev
func yaml_parser_parse_flow_sequence_entry(parser *yaml_parser_t, event *yaml_event_t, first bool) bool {
if first {
token := peek_token(parser)
if token == nil {
return false
}
parser.marks = append(parser.marks, token.start_mark)
skip_token(parser)
}

View File

@@ -25,9 +25,9 @@ package util
import (
"fmt"
"github.com/pkg/errors"
"go.starlark.net/starlark"
"go.starlark.net/starlarkstruct"
"sigs.k8s.io/kustomize/kyaml/errors"
)
// // asString unquotes a starlark string value
@@ -44,6 +44,7 @@ func IsEmptyString(s starlark.String) bool {
}
// Unmarshal decodes a starlark.Value into it's golang counterpart
//
//nolint:nakedret
func Unmarshal(x starlark.Value) (val interface{}, err error) {
switch v := x.(type) {
@@ -161,7 +162,7 @@ func Unmarshal(x starlark.Value) (val interface{}, err error) {
if _var, ok := v.Constructor().(Unmarshaler); ok {
err = _var.UnmarshalStarlark(x)
if err != nil {
err = errors.Wrapf(err, "failed marshal %q to Starlark object", v.Constructor().Type())
err = errors.WrapPrefixf(err, "failed marshal %q to Starlark object", v.Constructor().Type())
return
}
val = _var
@@ -176,6 +177,7 @@ func Unmarshal(x starlark.Value) (val interface{}, err error) {
}
// Marshal turns go values into starlark types
//
//nolint:nakedret
func Marshal(data interface{}) (v starlark.Value, err error) {
switch x := data.(type) {

View File

@@ -45,7 +45,7 @@ type ByteWriter struct {
// WrappingAPIVersion is the apiVersion for WrappingKind
WrappingAPIVersion string
// Sort if set, will cause ByteWriter to sort the the nodes before writing them.
// Sort if set, will cause ByteWriter to sort the nodes before writing them.
Sort bool
}

View File

@@ -5,8 +5,8 @@ MYGOBIN = $(shell go env GOBIN)
ifeq ($(MYGOBIN),)
MYGOBIN = $(shell go env GOPATH)/bin
endif
API_VERSION := "v1.21.2"
KIND_VERSION := "v0.11.1"
API_VERSION ?= "v1.21.2"
.PHONY: all
all: \
@@ -28,7 +28,7 @@ nuke: clean
rm -r kubernetesapi/*
$(MYGOBIN)/go-bindata:
go install github.com/go-bindata/go-bindata/v3/go-bindata
go install github.com/go-bindata/go-bindata/v3/go-bindata@latest
$(MYGOBIN)/kind:
( \
@@ -40,22 +40,22 @@ $(MYGOBIN)/kind:
rm -rf $$d; \
)
.PHONY: kubernetesapi/openapiinfo.go
kubernetesapi/openapiinfo.go:
./scripts/makeOpenApiInfoDotGo.sh
kustomizationapi/swagger.go: $(MYGOBIN)/go-bindata kustomizationapi/swagger.json
$(MYGOBIN)/go-bindata \
--pkg kustomizationapi \
-o kustomizationapi/swagger.go \
kustomizationapi/swagger.json
.PHONY: kubernetesapi/openapiinfo.go
kubernetesapi/openapiinfo.go:
./scripts/makeOpenApiInfoDotGo.sh
.PHONY: kubernetesapi/swagger.json
kubernetesapi/swagger.json: $(MYGOBIN)/kind $(MYGOBIN)/kustomize
.PHONY: kubernetesapi/swagger.pb
kubernetesapi/swagger.pb: $(MYGOBIN)/kind $(MYGOBIN)/kustomize
./scripts/fetchSchemaFromCluster.sh $(API_VERSION)
.PHONY: kubernetesapi/swagger.go
kubernetesapi/swagger.go: $(MYGOBIN)/go-bindata kubernetesapi/swagger.json
kubernetesapi/swagger.go: $(MYGOBIN)/go-bindata kubernetesapi/swagger.pb
./scripts/generateSwaggerDotGo.sh $(API_VERSION)
$(MYGOBIN)/kustomize:

View File

@@ -22,22 +22,31 @@ make nuke
The compiled-in schema version should maximize API availability with respect to all actively supported Kubernetes versions. For example, while 1.20, 1.21 and 1.22 are the actively supported versions, 1.21 is the best choice. This is because 1.21 introduces at least one new API and does not remove any, while 1.22 removes a large set of long-deprecated APIs that are still supported in 1.20/1.21.
### Update the built-in schema to a new version
### Generating additional schema
In the Makefile in this directory, update the `API_VERSION` to your desired version.
If you'd like to change the default schema version, then in the Makefile in this directory, update the `API_VERSION` to your desired version.
You may need to update the version of Kind these scripts use by changing `KIND_VERSION` in the Makefile in this directory. You can find compatibility information in the [kind release notes](https://github.com/kubernetes-sigs/kind/releases).
In this directory, fetch the openapi schema and generate the
corresponding swagger.go for the kubernetes api:
In this directory, fetch the openapi schema, generate the
corresponding swagger.go for the kubernetes api, and update `kubernetesapi/openapiinfo.go`:
```
make all
```
The above command will update the [OpenAPI schema] and the [Kustomization schema]. It will
create a directory kubernetesapi/v1212 and store the resulting
swagger.json and swagger.go files there.
If you want to run the steps individually instead of using `make all`, you can run
the following commands:
```
make kustomizationapi/swagger.go
make kubernetesapi/swagger.go
make kubernetesapi/openapiinfo.go
```
You can optionally delete the old `swagger.pb` and `swagger.go` files if we no longer need to support that kubernetes version of
openapi data. Make sure you rerun `make kubernetesapi/openapiinfo.go` after deleting any old schemas.
#### Precomputations
@@ -55,25 +64,6 @@ make prow-presubmit-check >& /tmp/k.txt; echo $?
The exit code should be zero; if not, examine `/tmp/k.txt`.
## Generating additional schemas
Instead of replacing the default version, you can specify a desired version as part of the make invocation:
```
rm kubernetesapi/swagger.go
make kubernetesapi/swagger.go API_VERSION=v1.21.2
```
While the above commands generate the swagger.go files, they
do not make them available for use nor do they update the
info field reported by `kustomize openapi info`. To make the
newly fetched schema and swagger.go available:
```
rm kubernetesapi/openapiinfo.go
make kubernetesapi/openapiinfo.go
```
## Partial regeneration
You can also regenerate the kubernetes api schemas specifically with:
@@ -87,8 +77,8 @@ To fetch the schema without generating the swagger.go, you can
run:
```
rm kubernetesapi/swagger.json
make kubernetesapi/swagger.json
rm kubernetesapi/swagger.pb
make kubernetesapi/swagger.pb
```
Note that generating the swagger.go will re-fetch the schema.

View File

@@ -6,13 +6,13 @@
package kubernetesapi
import (
"sigs.k8s.io/kustomize/kyaml/openapi/kubernetesapi/v1212"
"sigs.k8s.io/kustomize/kyaml/openapi/kubernetesapi/v1_21_2"
)
const Info = "{title:Kubernetes,version:v1.21.2}"
var OpenAPIMustAsset = map[string]func(string) []byte{
"v1212": v1212.MustAsset,
"v1.21.2": v1_21_2.MustAsset,
}
const DefaultOpenAPI = "v1212"
const DefaultOpenAPI = "v1.21.2"

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@@ -11,7 +11,6 @@ import (
"compress/gzip"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"strings"
@@ -215,7 +214,7 @@ func RestoreAsset(dir, name string) error {
if err != nil {
return err
}
err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode())
err = os.WriteFile(_filePath(dir, name), data, info.Mode())
if err != nil {
return err
}

View File

@@ -6,12 +6,13 @@ package openapi
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"reflect"
"strings"
"sync"
openapi_v2 "github.com/google/gnostic/openapiv2"
openapi_v2 "github.com/google/gnostic-models/openapiv2"
"google.golang.org/protobuf/proto"
"k8s.io/kube-openapi/pkg/validation/spec"
"sigs.k8s.io/kustomize/kyaml/errors"
@@ -21,14 +22,27 @@ import (
k8syaml "sigs.k8s.io/yaml"
)
// globalSchema contains global state information about the openapi
var globalSchema openapiData
var (
// schemaLock is the lock for schema related globals.
//
// NOTE: This lock helps with preventing panics that might occur due to the data
// race that concurrent access on this variable might cause but it doesn't
// fully fix the issue described in https://github.com/kubernetes-sigs/kustomize/issues/4824.
// For instance concurrently running goroutines where each of them calls SetSchema()
// and/or GetSchemaVersion might end up received nil errors (success) whereas the
// seconds one would overwrite the global variable that has been written by the
// first one.
schemaLock sync.RWMutex //nolint:gochecknoglobals
// kubernetesOpenAPIVersion specifies which builtin kubernetes schema to use
var kubernetesOpenAPIVersion string
// kubernetesOpenAPIVersion specifies which builtin kubernetes schema to use.
kubernetesOpenAPIVersion string //nolint:gochecknoglobals
// customSchemaFile stores the custom OpenApi schema if it is provided
var customSchema []byte
// globalSchema contains global state information about the openapi
globalSchema openapiData //nolint:gochecknoglobals
// customSchemaFile stores the custom OpenApi schema if it is provided
customSchema []byte //nolint:gochecknoglobals
)
// openapiData contains the parsed openapi state. this is in a struct rather than
// a list of vars so that it can be reset from tests.
@@ -104,7 +118,7 @@ var precomputedIsNamespaceScoped = map[yaml.TypeMeta]bool{
{APIVersion: "node.k8s.io/v1beta1", Kind: "RuntimeClass"}: false,
{APIVersion: "policy/v1", Kind: "PodDisruptionBudget"}: true,
{APIVersion: "policy/v1beta1", Kind: "PodDisruptionBudget"}: true,
{APIVersion: "policy/v1beta1", Kind: "PodSecurityPolicy"}: false,
{APIVersion: "policy/v1beta1", Kind: "PodSecurityPolicy"}: false, // remove after openapi upgrades to v1.25.
{APIVersion: "rbac.authorization.k8s.io/v1", Kind: "ClusterRole"}: false,
{APIVersion: "rbac.authorization.k8s.io/v1", Kind: "ClusterRoleBinding"}: false,
{APIVersion: "rbac.authorization.k8s.io/v1", Kind: "Role"}: true,
@@ -220,7 +234,7 @@ func definitionRefsFromRNode(object *yaml.RNode) ([]string, error) {
// parseOpenAPI reads openAPIPath yaml and converts it to RNode
func parseOpenAPI(openAPIPath string) (*yaml.RNode, error) {
b, err := ioutil.ReadFile(openAPIPath)
b, err := os.ReadFile(openAPIPath)
if err != nil {
return nil, err
}
@@ -278,9 +292,12 @@ func AddSchema(s []byte) error {
// ResetOpenAPI resets the openapi data to empty
func ResetOpenAPI() {
schemaLock.Lock()
defer schemaLock.Unlock()
globalSchema = openapiData{}
kubernetesOpenAPIVersion = ""
customSchema = nil
kubernetesOpenAPIVersion = ""
}
// AddDefinitions adds the definitions to the global schema.
@@ -400,7 +417,7 @@ func SuppressBuiltInSchemaUse() {
// Elements returns the Schema for the elements of an array.
func (rs *ResourceSchema) Elements() *ResourceSchema {
// load the schema from swagger.json
// load the schema from swagger files
initSchema()
if len(rs.Schema.Type) != 1 || rs.Schema.Type[0] != "array" {
@@ -446,7 +463,7 @@ func (rs *ResourceSchema) Lookup(path ...string) *ResourceSchema {
// Field returns the Schema for a field.
func (rs *ResourceSchema) Field(field string) *ResourceSchema {
// load the schema from swagger.json
// load the schema from swagger files
initSchema()
// locate the Schema
@@ -459,7 +476,7 @@ func (rs *ResourceSchema) Field(field string) *ResourceSchema {
// (the key doesn't matter, they all have the same value type)
s = *rs.Schema.AdditionalProperties.Schema
default:
// no Schema found from either swagger.json or line comments
// no Schema found from either swagger files or line comments
return nil
}
@@ -545,24 +562,28 @@ const (
groupKey = "group"
// versionKey is the key to lookup the version from the GVK extension
versionKey = "version"
// kindKey is the the to lookup the kind from the GVK extension
// kindKey is the to lookup the kind from the GVK extension
kindKey = "kind"
)
// SetSchema sets the kubernetes OpenAPI schema version to use
func SetSchema(openAPIField map[string]string, schema []byte, reset bool) error {
schemaLock.Lock()
defer schemaLock.Unlock()
// this should only be set once
schemaIsSet := (kubernetesOpenAPIVersion != "") || customSchema != nil
if schemaIsSet && !reset {
return nil
}
version, exists := openAPIField["version"]
if exists && schema != nil {
return fmt.Errorf("builtin version and custom schema provided, cannot use both")
}
version, versionProvided := openAPIField["version"]
if schema != nil { // use custom schema
// use custom schema
if schema != nil {
if versionProvided {
return fmt.Errorf("builtin version and custom schema provided, cannot use both")
}
customSchema = schema
kubernetesOpenAPIVersion = "custom"
// if the schema is changed, initSchema should parse the new schema
@@ -571,13 +592,14 @@ func SetSchema(openAPIField map[string]string, schema []byte, reset bool) error
}
// use builtin version
kubernetesOpenAPIVersion = strings.ReplaceAll(version, ".", "")
kubernetesOpenAPIVersion = version
if kubernetesOpenAPIVersion == "" {
return nil
}
if _, ok := kubernetesapi.OpenAPIMustAsset[kubernetesOpenAPIVersion]; !ok {
return fmt.Errorf("the specified OpenAPI version is not built in")
}
customSchema = nil
// if the schema is changed, initSchema should parse the new schema
globalSchema.schemaInit = false
@@ -586,6 +608,9 @@ func SetSchema(openAPIField map[string]string, schema []byte, reset bool) error
// GetSchemaVersion returns what kubernetes OpenAPI version is being used
func GetSchemaVersion() string {
schemaLock.RLock()
defer schemaLock.RUnlock()
switch {
case kubernetesOpenAPIVersion == "" && customSchema == nil:
return kubernetesOpenAPIDefaultVersion
@@ -598,6 +623,9 @@ func GetSchemaVersion() string {
// initSchema parses the json schema
func initSchema() {
schemaLock.Lock()
defer schemaLock.Unlock()
if globalSchema.schemaInit {
return
}
@@ -607,7 +635,7 @@ func initSchema() {
if customSchema != nil {
err := parse(customSchema, JsonOrYaml)
if err != nil {
panic("invalid schema file")
panic(fmt.Errorf("invalid schema file: %w", err))
}
} else {
if kubernetesOpenAPIVersion == "" {
@@ -629,11 +657,10 @@ func parseBuiltinSchema(version string) {
// don't parse the built in schema
return
}
// parse the swagger, this should never fail
assetName := filepath.Join(
"kubernetesapi",
version,
strings.ReplaceAll(version, ".", "_"),
"swagger.pb")
if err := parse(kubernetesapi.OpenAPIMustAsset[version](assetName), Proto); err != nil {

View File

@@ -11,7 +11,7 @@ import (
)
// Gvk identifies a Kubernetes API type.
// https://github.com/kubernetes/community/blob/master/contributors/design-proposals/api-machinery/api-group.md
// https://git.k8s.io/design-proposals-archive/api-machinery/api-group.md
type Gvk struct {
Group string `json:"group,omitempty" yaml:"group,omitempty"`
Version string `json:"version,omitempty" yaml:"version,omitempty"`
@@ -97,38 +97,35 @@ func (x Gvk) String() string {
return strings.Join([]string{k, v, g}, fieldSep)
}
// legacySortString returns an older version of String() that LegacyOrderTransformer depends on
// to keep its ordering stable across Kustomize versions
func (x Gvk) legacySortString() string {
legacyNoGroup := "~G"
legacyNoVersion := "~V"
legacyNoKind := "~K"
legacyFieldSeparator := "_"
// stableSortString returns a GVK representation that ensures determinism and
// backwards-compatibility in testing, logging, ...
func (x Gvk) stableSortString() string {
stableNoGroup := "~G"
stableNoVersion := "~V"
stableNoKind := "~K"
stableFieldSeparator := "_"
g := x.Group
if g == "" {
g = legacyNoGroup
g = stableNoGroup
}
v := x.Version
if v == "" {
v = legacyNoVersion
v = stableNoVersion
}
k := x.Kind
if k == "" {
k = legacyNoKind
k = stableNoKind
}
return strings.Join([]string{g, v, k}, legacyFieldSeparator)
return strings.Join([]string{g, v, k}, stableFieldSeparator)
}
// ApiVersion returns the combination of Group and Version
func (x Gvk) ApiVersion() string {
var sb strings.Builder
if x.Group != "" {
sb.WriteString(x.Group)
sb.WriteString("/")
return x.Group + "/" + x.Version
}
sb.WriteString(x.Version)
return sb.String()
return x.Version
}
// StringWoEmptyField returns a string representation of the GVK. Non-exist
@@ -203,7 +200,7 @@ func (x Gvk) IsLessThan(o Gvk) bool {
if indexI != indexJ {
return indexI < indexJ
}
return x.legacySortString() < o.legacySortString()
return x.stableSortString() < o.stableSortString()
}
// IsSelected returns true if `selector` selects `x`; otherwise, false.

View File

@@ -60,25 +60,6 @@ func (id ResId) String() string {
[]string{id.Gvk.String(), strings.Join([]string{nm, ns}, fieldSep)}, separator)
}
// LegacySortString returns an older version of String() that LegacyOrderTransformer depends on
// to keep its ordering stable across Kustomize versions
func (id ResId) LegacySortString() string {
legacyNoNamespace := "~X"
legacyNoName := "~N"
legacySeparator := "|"
ns := id.Namespace
if ns == "" {
ns = legacyNoNamespace
}
nm := id.Name
if nm == "" {
nm = legacyNoName
}
return strings.Join(
[]string{id.Gvk.String(), ns, nm}, legacySeparator)
}
func FromString(s string) ResId {
values := strings.Split(s, separator)
gvk := GvkFromString(values[0])

View File

@@ -308,7 +308,10 @@ func (r RunFns) getFunctionFilters(global bool, fns ...*yaml.RNode) (
var fltrs []kio.Filter
for i := range fns {
api := fns[i]
spec := runtimeutil.GetFunctionSpec(api)
spec, err := runtimeutil.GetFunctionSpec(api)
if err != nil {
return nil, fmt.Errorf("failed to get FunctionSpec: %w", err)
}
if spec == nil {
// resource doesn't have function spec
continue

View File

@@ -10,7 +10,7 @@ func (s String) Len() int {
}
func (s String) List() []string {
var val []string
val := make([]string, 0, len(s))
for k := range s {
val = append(val, k)
}

View File

@@ -11,6 +11,13 @@ import "strings"
func PathSplitter(path string, delimiter string) []string {
ps := strings.Split(path, delimiter)
var res []string
// allow path to start with forward slash
// i.e. /a/b/c
if len(ps) > 1 && ps[0] == "" {
ps = ps[1:]
}
res = append(res, ps[0])
for i := 1; i < len(ps); i++ {
last := len(res) - 1

View File

@@ -87,6 +87,16 @@ var MappingNode yaml.Kind = yaml.MappingNode
var ScalarNode yaml.Kind = yaml.ScalarNode
var SequenceNode yaml.Kind = yaml.SequenceNode
func nodeKindString(k yaml.Kind) string {
return map[yaml.Kind]string{
yaml.SequenceNode: "SequenceNode",
yaml.MappingNode: "MappingNode",
yaml.ScalarNode: "ScalarNode",
yaml.DocumentNode: "DocumentNode",
yaml.AliasNode: "AliasNode",
}[k]
}
var DoubleQuotedStyle yaml.Style = yaml.DoubleQuotedStyle
var FlowStyle yaml.Style = yaml.FlowStyle
var FoldedStyle yaml.Style = yaml.FoldedStyle

View File

@@ -68,7 +68,7 @@ func (y *YFilter) UnmarshalYAML(unmarshal func(interface{}) error) error {
type YFilters []YFilter
func (y YFilters) Filters() []Filter {
var f []Filter
f := make([]Filter, 0, len(y))
for i := range y {
f = append(f, y[i].Filter)
}

View File

@@ -197,36 +197,37 @@ func (c FieldClearer) Filter(rn *RNode) (*RNode, error) {
return nil, err
}
for i := 0; i < len(rn.Content()); i += 2 {
// if name matches, remove these 2 elements from the list because
// they are treated as a fieldName/fieldValue pair.
if rn.Content()[i].Value == c.Name {
if c.IfEmpty {
if len(rn.Content()[i+1].Content) > 0 {
continue
}
}
// save the item we are about to remove
removed := NewRNode(rn.Content()[i+1])
if len(rn.YNode().Content) > i+2 {
l := len(rn.YNode().Content)
// remove from the middle of the list
rn.YNode().Content = rn.Content()[:i]
rn.YNode().Content = append(
rn.YNode().Content,
rn.Content()[i+2:l]...)
} else {
// remove from the end of the list
rn.YNode().Content = rn.Content()[:i]
}
// return the removed field name and value
return removed, nil
var removed *RNode
visitFieldsWhileTrue(rn.Content(), func(key, value *yaml.Node, keyIndex int) bool {
if key.Value != c.Name {
return true
}
}
// nothing removed
return nil, nil
// the name matches: remove these 2 elements from the list because
// they are treated as a fieldName/fieldValue pair.
if c.IfEmpty {
if len(value.Content) > 0 {
return true
}
}
// save the item we are about to remove
removed = NewRNode(value)
if len(rn.YNode().Content) > keyIndex+2 {
l := len(rn.YNode().Content)
// remove from the middle of the list
rn.YNode().Content = rn.Content()[:keyIndex]
rn.YNode().Content = append(
rn.YNode().Content,
rn.Content()[keyIndex+2:l]...)
} else {
// remove from the end of the list
rn.YNode().Content = rn.Content()[:keyIndex]
}
return false
})
return removed, nil
}
func MatchElement(field, value string) ElementMatcher {
@@ -402,14 +403,15 @@ func (f FieldMatcher) Filter(rn *RNode) (*RNode, error) {
return nil, err
}
for i := 0; i < len(rn.Content()); i = IncrementFieldIndex(i) {
isMatchingField := rn.Content()[i].Value == f.Name
if isMatchingField {
requireMatchFieldValue := f.Value != nil
if !requireMatchFieldValue || rn.Content()[i+1].Value == f.Value.YNode().Value {
return NewRNode(rn.Content()[i+1]), nil
}
var returnNode *RNode
requireMatchFieldValue := f.Value != nil
visitMappingNodeFields(rn.Content(), func(key, value *yaml.Node) {
if !requireMatchFieldValue || value.Value == f.Value.YNode().Value {
returnNode = NewRNode(value)
}
}, f.Name)
if returnNode != nil {
return returnNode, nil
}
if f.Create != nil {
@@ -550,7 +552,7 @@ func (l PathGetter) getFilter(part, nextPart string, fieldPath *[]string) (Filte
default:
// mapping node
*fieldPath = append(*fieldPath, part)
return l.fieldFilter(part, l.getKind(nextPart))
return l.fieldFilter(part, getPathPartKind(nextPart, l.Create))
}
}
@@ -590,15 +592,18 @@ func (l PathGetter) fieldFilter(
return FieldMatcher{Name: name, Create: &RNode{value: &yaml.Node{Kind: kind, Style: l.Style}}}, nil
}
func (l PathGetter) getKind(nextPart string) yaml.Kind {
func getPathPartKind(nextPart string, defaultKind yaml.Kind) yaml.Kind {
if IsListIndex(nextPart) {
// if nextPart is of the form [a=b], then it is an index into a Sequence
// so the current part must be a SequenceNode
return yaml.SequenceNode
}
if IsIdxNumber(nextPart) {
return yaml.SequenceNode
}
if nextPart == "" {
// final name in the path, use the l.Create defined Kind
return l.Create
// final name in the path, use the default kind provided
return defaultKind
}
// non-sequence intermediate Node
@@ -640,13 +645,19 @@ func (s MapEntrySetter) Filter(rn *RNode) (*RNode, error) {
if s.Name == "" {
s.Name = GetValue(s.Key)
}
for i := 0; i < len(rn.Content()); i = IncrementFieldIndex(i) {
isMatchingField := rn.Content()[i].Value == s.Name
if isMatchingField {
rn.Content()[i] = s.Key.YNode()
rn.Content()[i+1] = s.Value.YNode()
return rn, nil
content := rn.Content()
fieldStillNotFound := true
visitFieldsWhileTrue(content, func(key, value *yaml.Node, keyIndex int) bool {
if key.Value == s.Name {
content[keyIndex] = s.Key.YNode()
content[keyIndex+1] = s.Value.YNode()
fieldStillNotFound = false
}
return fieldStillNotFound
})
if !fieldStillNotFound {
return rn, nil
}
// create the field
@@ -794,12 +805,22 @@ func ErrorIfAnyInvalidAndNonNull(kind yaml.Kind, rn ...*RNode) error {
return nil
}
var nodeTypeIndex = map[yaml.Kind]string{
yaml.SequenceNode: "SequenceNode",
yaml.MappingNode: "MappingNode",
yaml.ScalarNode: "ScalarNode",
yaml.DocumentNode: "DocumentNode",
yaml.AliasNode: "AliasNode",
type InvalidNodeKindError struct {
expectedKind yaml.Kind
node *RNode
}
func (e *InvalidNodeKindError) Error() string {
msg := fmt.Sprintf("wrong node kind: expected %s but got %s",
nodeKindString(e.expectedKind), nodeKindString(e.node.YNode().Kind))
if content, err := e.node.String(); err == nil {
msg += fmt.Sprintf(": node contents:\n%s", content)
}
return msg
}
func (e *InvalidNodeKindError) ActualNodeKind() Kind {
return e.node.YNode().Kind
}
func ErrorIfInvalid(rn *RNode, kind yaml.Kind) error {
@@ -809,11 +830,7 @@ func ErrorIfInvalid(rn *RNode, kind yaml.Kind) error {
}
if rn.YNode().Kind != kind {
s, _ := rn.String()
return errors.Errorf(
"wrong Node Kind for %s expected: %v was %v: value: {%s}",
strings.Join(rn.FieldPath(), "."),
nodeTypeIndex[kind], nodeTypeIndex[rn.YNode().Kind], strings.TrimSpace(s))
return &InvalidNodeKindError{node: rn, expectedKind: kind}
}
if kind == yaml.MappingNode {
@@ -859,9 +876,3 @@ func SplitIndexNameValue(p string) (string, string, error) {
}
return parts[0], parts[1], nil
}
// IncrementFieldIndex increments i to point to the next field name element in
// a slice of Contents.
func IncrementFieldIndex(i int) int {
return i + 2
}

View File

@@ -4,9 +4,13 @@
package yaml
import (
"fmt"
"regexp"
"strconv"
"strings"
"sigs.k8s.io/kustomize/kyaml/errors"
"sigs.k8s.io/kustomize/kyaml/internal/forked/github.com/go-yaml/yaml"
)
// PathMatcher returns all RNodes matching the path wrapped in a SequenceNode.
@@ -21,7 +25,7 @@ type PathMatcher struct {
// Each path part may be one of:
// * FieldMatcher -- e.g. "spec"
// * Map Key -- e.g. "app.k8s.io/version"
// * List Entry -- e.g. "[name=nginx]" or "[=-jar]"
// * List Entry -- e.g. "[name=nginx]" or "[=-jar]" or "0"
//
// Map Keys and Fields are equivalent.
// See FieldMatcher for more on Fields and Map Keys.
@@ -43,10 +47,18 @@ type PathMatcher struct {
// This is useful for if the nodes are to be printed in FlowStyle.
StripComments bool
val *RNode
field string
matchRegex string
indexNumber int
// Create will cause missing path parts to be created as they are walked.
//
// * The leaf Node (final path) will be created with a Kind matching Create
// * Intermediary Nodes will be created as either a MappingNodes or
// SequenceNodes as appropriate for each's Path location.
// * Nodes identified by an index will only be created if the index indicates
// an append operation (i.e. index=len(list))
Create yaml.Kind `yaml:"create,omitempty"`
val *RNode
field string
matchRegex string
}
func (p *PathMatcher) stripComments(n *Node) {
@@ -109,7 +121,7 @@ func (p *PathMatcher) doMatchEvery(rn *RNode) (*RNode, error) {
func (p *PathMatcher) visitEveryElem(elem *RNode) error {
fieldName := p.Path[0]
// recurse on the matching element
pm := &PathMatcher{Path: p.Path[1:]}
pm := &PathMatcher{Path: p.Path[1:], Create: p.Create}
add, err := pm.filter(elem)
for k, v := range pm.Matches {
p.Matches[k] = v
@@ -125,13 +137,25 @@ func (p *PathMatcher) visitEveryElem(elem *RNode) error {
func (p *PathMatcher) doField(rn *RNode) (*RNode, error) {
// lookup the field
field, err := rn.Pipe(Get(p.Path[0]))
if err != nil || field == nil {
// if the field doesn't exist, return nil
if err != nil || (!IsCreate(p.Create) && field == nil) {
return nil, err
}
if IsCreate(p.Create) && field == nil {
var nextPart string
if len(p.Path) > 1 {
nextPart = p.Path[1]
}
nextPartKind := getPathPartKind(nextPart, p.Create)
field = &RNode{value: &yaml.Node{Kind: nextPartKind}}
err := rn.PipeE(SetField(p.Path[0], field))
if err != nil {
return nil, err
}
}
// recurse on the field, removing the first element of the path
pm := &PathMatcher{Path: p.Path[1:]}
pm := &PathMatcher{Path: p.Path[1:], Create: p.Create}
p.val, err = pm.filter(field)
p.Matches = pm.Matches
return p.val, err
@@ -144,18 +168,33 @@ func (p *PathMatcher) doIndexSeq(rn *RNode) (*RNode, error) {
if err != nil {
return nil, err
}
p.indexNumber = idx
elements, err := rn.Elements()
if err != nil {
return nil, err
}
if len(elements) == idx && IsCreate(p.Create) {
var nextPart string
if len(p.Path) > 1 {
nextPart = p.Path[1]
}
elem := &yaml.Node{Kind: getPathPartKind(nextPart, p.Create)}
err = rn.PipeE(Append(elem))
if err != nil {
return nil, errors.WrapPrefixf(err, "failed to append element for %q", p.Path[0])
}
elements = append(elements, NewRNode(elem))
}
if len(elements) < idx+1 {
return nil, fmt.Errorf("index %d specified but only %d elements found", idx, len(elements))
}
// get target element
element := elements[idx]
// recurse on the matching element
pm := &PathMatcher{Path: p.Path[1:]}
pm := &PathMatcher{Path: p.Path[1:], Create: p.Create}
add, err := pm.filter(element)
for k, v := range pm.Matches {
p.Matches[k] = v
@@ -176,16 +215,39 @@ func (p *PathMatcher) doSeq(rn *RNode) (*RNode, error) {
return nil, err
}
if p.field == "" {
primitiveElement := len(p.field) == 0
if primitiveElement {
err = rn.VisitElements(p.visitPrimitiveElem)
} else {
err = rn.VisitElements(p.visitElem)
}
if err != nil || p.val == nil || len(p.val.YNode().Content) == 0 {
if err != nil {
return nil, err
}
if !p.val.IsNil() && len(p.val.YNode().Content) == 0 {
p.val = nil
}
return p.val, nil
if !IsCreate(p.Create) || p.val != nil {
return p.val, nil
}
var elem *yaml.Node
valueNode := NewScalarRNode(p.matchRegex).YNode()
if primitiveElement {
elem = valueNode
} else {
elem = &yaml.Node{
Kind: yaml.MappingNode,
Content: []*yaml.Node{{Kind: yaml.ScalarNode, Value: p.field}, valueNode},
}
}
err = rn.PipeE(Append(elem))
if err != nil {
return nil, errors.WrapPrefixf(err, "failed to create element for %q", p.Path[0])
}
// re-do the sequence search; this time we'll find the element we just created
return p.doSeq(rn)
}
func (p *PathMatcher) visitPrimitiveElem(elem *RNode) error {
@@ -228,7 +290,7 @@ func (p *PathMatcher) visitElem(elem *RNode) error {
}
// recurse on the matching element
pm := &PathMatcher{Path: p.Path[1:]}
pm := &PathMatcher{Path: p.Path[1:], Create: p.Create}
add, err := pm.filter(elem)
for k, v := range pm.Matches {
p.Matches[k] = v

View File

@@ -1,7 +1,7 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
// Package merge contains libraries for merging fields from one RNode to another
// Package merge2 contains libraries for merging fields from one RNode to another
// RNode
package merge2
@@ -20,7 +20,7 @@ func Merge(src, dest *yaml.RNode, mergeOptions yaml.MergeOptions) (*yaml.RNode,
}.Walk()
}
// Merge parses the arguments, and merges fields from srcStr into destStr.
// MergeStrings parses the arguments, and merges fields from srcStr into destStr.
func MergeStrings(srcStr, destStr string, infer bool, mergeOptions yaml.MergeOptions) (string, error) {
src, err := yaml.Parse(srcStr)
if err != nil {
@@ -64,6 +64,12 @@ func (m Merger) VisitMap(nodes walk.Sources, s *openapi.ResourceSchema) (*yaml.R
return walk.ClearNode, nil
}
// If Origin is missing, preserve explicitly set null in Dest ("null", "~", etc)
if nodes.Origin().IsNil() && !nodes.Dest().IsNil() && len(nodes.Dest().YNode().Value) > 0 {
// Return a new node so that it won't have a "!!null" tag and therefore won't be cleared.
return yaml.NewScalarRNode(nodes.Dest().YNode().Value), nil
}
return nodes.Origin(), nil
}
if nodes.Origin().IsTaggedNull() {

View File

@@ -6,8 +6,8 @@ package yaml
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"os"
"regexp"
"strconv"
"strings"
@@ -53,7 +53,7 @@ func Parse(value string) (*RNode, error) {
// ReadFile parses a single Resource from a yaml file.
// To parse multiple resources, consider a kio.ByteReader
func ReadFile(path string) (*RNode, error) {
b, err := ioutil.ReadFile(path)
b, err := os.ReadFile(path)
if err != nil {
return nil, err
}
@@ -66,7 +66,7 @@ func WriteFile(node *RNode, path string) error {
if err != nil {
return err
}
return ioutil.WriteFile(path, []byte(out), 0600)
return errors.WrapPrefixf(os.WriteFile(path, []byte(out), 0600), "writing RNode to file")
}
// UpdateFile reads the file at path, applies the filter to it, and write the result back.
@@ -242,11 +242,7 @@ func (rn *RNode) IsTaggedNull() bool {
// IsNilOrEmpty is true if the node is nil,
// has no YNode, or has YNode that appears empty.
func (rn *RNode) IsNilOrEmpty() bool {
return rn.IsNil() ||
IsYNodeTaggedNull(rn.YNode()) ||
IsYNodeEmptyMap(rn.YNode()) ||
IsYNodeEmptySeq(rn.YNode()) ||
IsYNodeZero(rn.YNode())
return rn.IsNil() || IsYNodeNilOrEmpty(rn.YNode())
}
// IsStringValue is true if the RNode is not nil and is scalar string node
@@ -420,12 +416,11 @@ func (rn *RNode) SetApiVersion(av string) {
// given field, so this function cannot be used to make distinctions
// between these cases.
func (rn *RNode) getMapFieldValue(field string) *yaml.Node {
for i := 0; i < len(rn.Content()); i = IncrementFieldIndex(i) {
if rn.Content()[i].Value == field {
return rn.Content()[i+1]
}
}
return nil
var result *yaml.Node
visitMappingNodeFields(rn.Content(), func(key, value *yaml.Node) {
result = value
}, field)
return result
}
// GetName returns the name, or empty string if
@@ -440,31 +435,33 @@ func (rn *RNode) getMetaStringField(fName string) string {
if md == nil {
return ""
}
f := md.Field(fName)
if f.IsNilOrEmpty() {
return ""
}
return GetValue(f.Value)
var result string
visitMappingNodeFields(md.Content, func(key, value *yaml.Node) {
if !IsYNodeNilOrEmpty(value) {
result = value.Value
}
}, fName)
return result
}
// getMetaData returns the RNode holding the value of the metadata field.
// getMetaData returns the *yaml.Node of the metadata field.
// Return nil if field not found (no error).
func (rn *RNode) getMetaData() *RNode {
func (rn *RNode) getMetaData() *yaml.Node {
if IsMissingOrNull(rn) {
return nil
}
var n *RNode
content := rn.Content()
if rn.YNode().Kind == DocumentNode {
// get the content if this is the document node
n = NewRNode(rn.Content()[0])
} else {
n = rn
content = content[0].Content
}
mf := n.Field(MetadataField)
if mf.IsNilOrEmpty() {
return nil
}
return mf.Value
var mf *yaml.Node
visitMappingNodeFields(content, func(key, value *yaml.Node) {
if !IsYNodeNilOrEmpty(value) {
mf = value
}
}, MetadataField)
return mf
}
// SetName sets the metadata name field.
@@ -496,14 +493,14 @@ func (rn *RNode) SetNamespace(ns string) error {
}
// GetAnnotations gets the metadata annotations field.
// If the field is missing, returns an empty map.
// If the annotations field is missing, returns an empty map.
// Use another method to check for missing metadata.
func (rn *RNode) GetAnnotations() map[string]string {
meta := rn.getMetaData()
if meta == nil {
return make(map[string]string)
}
return rn.getMapFromMeta(meta, AnnotationsField)
// If specific annotations are provided, then the map is
// restricted to only those entries with keys that match
// one of the specific annotations. If no annotations are
// provided, then the map will contain all entries.
func (rn *RNode) GetAnnotations(annotations ...string) map[string]string {
return rn.getMapFromMeta(AnnotationsField, annotations...)
}
// SetAnnotations tries to set the metadata annotations field.
@@ -512,24 +509,45 @@ func (rn *RNode) SetAnnotations(m map[string]string) error {
}
// GetLabels gets the metadata labels field.
// If the field is missing, returns an empty map.
// If the labels field is missing, returns an empty map.
// Use another method to check for missing metadata.
func (rn *RNode) GetLabels() map[string]string {
// If specific labels are provided, then the map is
// restricted to only those entries with keys that match
// one of the specific labels. If no labels are
// provided, then the map will contain all entries.
func (rn *RNode) GetLabels(labels ...string) map[string]string {
return rn.getMapFromMeta(LabelsField, labels...)
}
// getMapFromMeta returns a map, sometimes empty, from the fName
// field in the node's metadata field.
// If specific fields are provided, then the map is
// restricted to only those entries with keys that match
// one of the specific fields. If no fields are
// provided, then the map will contain all entries.
func (rn *RNode) getMapFromMeta(fName string, fields ...string) map[string]string {
meta := rn.getMetaData()
if meta == nil {
return make(map[string]string)
}
return rn.getMapFromMeta(meta, LabelsField)
}
// getMapFromMeta returns map, sometimes empty, from metadata.
func (rn *RNode) getMapFromMeta(meta *RNode, fName string) map[string]string {
result := make(map[string]string)
if f := meta.Field(fName); !f.IsNilOrEmpty() {
_ = f.Value.VisitFields(func(node *MapNode) error {
result[GetValue(node.Key)] = GetValue(node.Value)
return nil
})
var result map[string]string
visitMappingNodeFields(meta.Content, func(_, fNameValue *yaml.Node) {
// fName is found in metadata; create the map from its content
expectedSize := len(fields)
if expectedSize == 0 {
expectedSize = len(fNameValue.Content) / 2 //nolint: gomnd
}
result = make(map[string]string, expectedSize)
visitMappingNodeFields(fNameValue.Content, func(key, value *yaml.Node) {
result[key.Value] = value.Value
}, fields...)
}, fName)
if result == nil {
return make(map[string]string)
}
return result
}
@@ -696,9 +714,9 @@ func (rn *RNode) Fields() ([]string, error) {
return nil, errors.Wrap(err)
}
var fields []string
for i := 0; i < len(rn.Content()); i += 2 {
fields = append(fields, rn.Content()[i].Value)
}
visitMappingNodeFields(rn.Content(), func(key, value *yaml.Node) {
fields = append(fields, key.Value)
})
return fields, nil
}
@@ -709,13 +727,12 @@ func (rn *RNode) FieldRNodes() ([]*RNode, error) {
return nil, errors.Wrap(err)
}
var fields []*RNode
for i := 0; i < len(rn.Content()); i += 2 {
yNode := rn.Content()[i]
visitMappingNodeFields(rn.Content(), func(key, value *yaml.Node) {
// for each key node in the input mapping node contents create equivalent rNode
rNode := &RNode{}
rNode.SetYNode(yNode)
rNode.SetYNode(key)
fields = append(fields, rNode)
}
})
return fields, nil
}
@@ -725,13 +742,11 @@ func (rn *RNode) Field(field string) *MapNode {
if rn.YNode().Kind != yaml.MappingNode {
return nil
}
for i := 0; i < len(rn.Content()); i = IncrementFieldIndex(i) {
isMatchingField := rn.Content()[i].Value == field
if isMatchingField {
return &MapNode{Key: NewRNode(rn.Content()[i]), Value: NewRNode(rn.Content()[i+1])}
}
}
return nil
var result *MapNode
visitMappingNodeFields(rn.Content(), func(key, value *yaml.Node) {
result = &MapNode{Key: NewRNode(key), Value: NewRNode(value)}
}, field)
return result
}
// VisitFields calls fn for each field in the RNode.
@@ -752,6 +767,59 @@ func (rn *RNode) VisitFields(fn func(node *MapNode) error) error {
return nil
}
// visitMappingNodeFields calls fn for fields in the content, in content order.
// The caller is responsible to ensure the node is a mapping node. If fieldNames
// are specified, then fn is called only for the fields that match the given
// fieldNames.
func visitMappingNodeFields(content []*yaml.Node, fn func(key, value *yaml.Node), fieldNames ...string) {
switch len(fieldNames) {
case 0: // visit all fields
visitFieldsWhileTrue(content, func(key, value *yaml.Node, _ int) bool {
fn(key, value)
return true
})
case 1: // visit single field
visitFieldsWhileTrue(content, func(key, value *yaml.Node, _ int) bool {
if key == nil {
return true
}
if fieldNames[0] == key.Value {
fn(key, value)
return false
}
return true
})
default: // visit specified fields
fieldsStillToVisit := make(map[string]bool, len(fieldNames))
for _, fieldName := range fieldNames {
fieldsStillToVisit[fieldName] = true
}
visitFieldsWhileTrue(content, func(key, value *yaml.Node, _ int) bool {
if key == nil {
return true
}
if fieldsStillToVisit[key.Value] {
fn(key, value)
delete(fieldsStillToVisit, key.Value)
}
return len(fieldsStillToVisit) > 0
})
}
}
// visitFieldsWhileTrue calls fn for the fields in content, in content order,
// until either fn returns false or all fields have been visited. The caller
// should ensure that content is from a mapping node, or fits the same expected
// pattern (consecutive key/value entries in the slice).
func visitFieldsWhileTrue(content []*yaml.Node, fn func(key, value *yaml.Node, keyIndex int) bool) {
for i := 0; i < len(content); i += 2 {
continueVisiting := fn(content[i], content[i+1], i)
if !continueVisiting {
return
}
}
}
// Elements returns the list of elements in the RNode.
// Returns an error for non-SequenceNodes.
func (rn *RNode) Elements() ([]*RNode, error) {
@@ -937,7 +1005,11 @@ func deAnchor(yn *yaml.Node) (res *yaml.Node, err error) {
case yaml.ScalarNode:
return yn, nil
case yaml.AliasNode:
return deAnchor(yn.Alias)
result, err := deAnchor(yn.Alias)
if err != nil {
return nil, err
}
return CopyYNode(result), nil
case yaml.MappingNode:
toMerge, err := removeMergeTags(yn)
if err != nil {
@@ -1003,17 +1075,19 @@ func findMergeValues(yn *yaml.Node) ([]*yaml.Node, error) {
// it fails.
func getMergeTagValue(yn *yaml.Node) (*yaml.Node, error) {
var result *yaml.Node
for i := 0; i < len(yn.Content); i += 2 {
key := yn.Content[i]
value := yn.Content[i+1]
var err error
visitFieldsWhileTrue(yn.Content, func(key, value *yaml.Node, _ int) bool {
if isMerge(key) {
if result != nil {
return nil, fmt.Errorf("duplicate merge key")
err = fmt.Errorf("duplicate merge key")
result = nil
return false
}
result = value
}
}
return result, nil
return true
})
return result, err
}
// removeMergeTags removes all merge tags and returns a ordered list of yaml

View File

@@ -39,11 +39,20 @@ func IsYNodeEmptyMap(n *yaml.Node) bool {
return n != nil && n.Kind == yaml.MappingNode && len(n.Content) == 0
}
// IsYNodeEmptyMap is true if the Node is a non-nil empty sequence.
// IsYNodeEmptySeq is true if the Node is a non-nil empty sequence.
func IsYNodeEmptySeq(n *yaml.Node) bool {
return n != nil && n.Kind == yaml.SequenceNode && len(n.Content) == 0
}
// IsYNodeNilOrEmpty is true if the Node is nil or appears empty.
func IsYNodeNilOrEmpty(n *yaml.Node) bool {
return n == nil ||
IsYNodeTaggedNull(n) ||
IsYNodeEmptyMap(n) ||
IsYNodeEmptySeq(n) ||
IsYNodeZero(n)
}
// IsYNodeEmptyDoc is true if the node is a Document with no content.
// E.g.: "---\n---"
func IsYNodeEmptyDoc(n *yaml.Node) bool {
@@ -238,3 +247,53 @@ type MergeOptions struct {
// source list to destination or append.
ListIncreaseDirection MergeOptionsListIncreaseDirection
}
// Since ObjectMeta and TypeMeta are stable, we manually create DeepCopy funcs for ResourceMeta and ObjectMeta.
// For TypeMeta and NameMeta no DeepCopy funcs are required, as they only contain basic types.
// DeepCopyInto copies the receiver, writing into out. in must be non-nil.
func (in *ObjectMeta) DeepCopyInto(out *ObjectMeta) {
*out = *in
out.NameMeta = in.NameMeta
if in.Labels != nil {
in, out := &in.Labels, &out.Labels
*out = make(map[string]string, len(*in))
for key, val := range *in {
(*out)[key] = val
}
}
if in.Annotations != nil {
in, out := &in.Annotations, &out.Annotations
*out = make(map[string]string, len(*in))
for key, val := range *in {
(*out)[key] = val
}
}
}
// DeepCopy copies the receiver, creating a new ObjectMeta.
func (in *ObjectMeta) DeepCopy() *ObjectMeta {
if in == nil {
return nil
}
out := new(ObjectMeta)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto copies the receiver, writing into out. in must be non-nil.
func (in *ResourceMeta) DeepCopyInto(out *ResourceMeta) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
}
// DeepCopy copies the receiver, creating a new ResourceMeta.
func (in *ResourceMeta) DeepCopy() *ResourceMeta {
if in == nil {
return nil
}
out := new(ResourceMeta)
in.DeepCopyInto(out)
return out
}

View File

@@ -217,7 +217,7 @@ func (l *Walker) setAssociativeSequenceElements(valuesList [][]string, keys []st
// Add the val to the sequence. val will replace the item in the sequence if
// there is an item that matches all key-value pairs. Otherwise val will be appended
// the the sequence.
// the sequence.
_, err = itemsToBeAdded.Pipe(yaml.ElementSetter{
Element: val.YNode(),
Keys: validKeys,