update vendor

Signed-off-by: Roland.Ma <rolandma@yunify.com>
This commit is contained in:
Roland.Ma
2021-08-11 07:10:14 +00:00
parent a18f72b565
commit ea8f47c73a
2901 changed files with 269317 additions and 43103 deletions

View File

@@ -0,0 +1,192 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package container
import (
"fmt"
runtimeexec "sigs.k8s.io/kustomize/kyaml/fn/runtime/exec"
"sigs.k8s.io/kustomize/kyaml/fn/runtime/runtimeutil"
"sigs.k8s.io/kustomize/kyaml/yaml"
)
// Filter filters Resources using a container image.
// The container must start a process that reads the list of
// input Resources from stdin, reads the Configuration from the env
// API_CONFIG, and writes the filtered Resources to stdout.
// If there is a error or validation failure, the process must exit
// non-zero.
// The full set of environment variables from the parent process
// are passed to the container.
//
// Function Scoping:
// Filter applies the function only to Resources to which it is scoped.
//
// Resources are scoped to a function if any of the following are true:
// - the Resource were read from the same directory as the function config
// - the Resource were read from a subdirectory of the function config directory
// - the function config is in a directory named "functions" and
// they were read from a subdirectory of "functions" parent
// - the function config doesn't have a path annotation (considered globally scoped)
// - the Filter has GlobalScope == true
//
// In Scope Examples:
//
// Example 1: deployment.yaml and service.yaml in function.yaml scope
// same directory as the function config directory
// .
// ├── function.yaml
// ├── deployment.yaml
// └── service.yaml
//
// Example 2: apps/deployment.yaml and apps/service.yaml in function.yaml scope
// subdirectory of the function config directory
// .
// ├── function.yaml
// └── apps
//    ├── deployment.yaml
//    └── service.yaml
//
// Example 3: apps/deployment.yaml and apps/service.yaml in functions/function.yaml scope
// function config is in a directory named "functions"
// .
// ├── functions
// │   └── function.yaml
// └── apps
//    ├── deployment.yaml
//    └── service.yaml
//
// Out of Scope Examples:
//
// Example 1: apps/deployment.yaml and apps/service.yaml NOT in stuff/function.yaml scope
// .
// ├── stuff
// │   └── function.yaml
// └── apps
//    ├── deployment.yaml
//    └── service.yaml
//
// Example 2: apps/deployment.yaml and apps/service.yaml NOT in stuff/functions/function.yaml scope
// .
// ├── stuff
// │   └── functions
// │    └── function.yaml
// └── apps
//    ├── deployment.yaml
//    └── service.yaml
//
// Default Paths:
// Resources emitted by functions will have default path applied as annotations
// if none is present.
// The default path will be the function-dir/ (or parent directory in the case of "functions")
// + function-file-name/ + namespace/ + kind_name.yaml
//
// Example 1: Given a function in fn.yaml that produces a Deployment name foo and a Service named bar
// dir
// └── fn.yaml
//
// Would default newly generated Resources to:
//
// dir
// ├── fn.yaml
// └── fn
//    ├── deployment_foo.yaml
//    └── service_bar.yaml
//
// Example 2: Given a function in functions/fn.yaml that produces a Deployment name foo and a Service named bar
// dir
// └── fn.yaml
//
// Would default newly generated Resources to:
//
// dir
// ├── functions
// │   └── fn.yaml
// └── fn
//    ├── deployment_foo.yaml
//    └── service_bar.yaml
//
// Example 3: Given a function in fn.yaml that produces a Deployment name foo, namespace baz and a Service named bar namespace baz
// dir
// └── fn.yaml
//
// Would default newly generated Resources to:
//
// dir
// ├── fn.yaml
// └── fn
// └── baz
//    ├── deployment_foo.yaml
//    └── service_bar.yaml
type Filter struct {
runtimeutil.ContainerSpec `json:",inline" yaml:",inline"`
Exec runtimeexec.Filter
UIDGID string
}
func (c Filter) String() string {
if c.Exec.DeferFailure {
return fmt.Sprintf("%s deferFailure: %v", c.Image, c.Exec.DeferFailure)
}
return c.Image
}
func (c Filter) GetExit() error {
return c.Exec.GetExit()
}
func (c *Filter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
c.setupExec()
return c.Exec.Filter(nodes)
}
func (c *Filter) setupExec() {
// don't init 2x
if c.Exec.Path != "" {
return
}
path, args := c.getCommand()
c.Exec.Path = path
c.Exec.Args = args
}
// getArgs returns the command + args to run to spawn the container
func (c *Filter) getCommand() (string, []string) {
network := runtimeutil.NetworkNameNone
if c.ContainerSpec.Network {
network = runtimeutil.NetworkNameHost
}
// run the container using docker. this is simpler than using the docker
// libraries, and ensures things like auth work the same as if the container
// was run from the cli.
args := []string{"run",
"--rm", // delete the container afterward
"-i", "-a", "STDIN", "-a", "STDOUT", "-a", "STDERR", // attach stdin, stdout, stderr
"--network", string(network),
// added security options
"--user", c.UIDGID,
"--security-opt=no-new-privileges", // don't allow the user to escalate privileges
// note: don't make fs readonly because things like heredoc rely on writing tmp files
}
// TODO(joncwong): Allow StorageMount fields to have default values.
for _, storageMount := range c.StorageMounts {
args = append(args, "--mount", storageMount.String())
}
args = append(args, runtimeutil.NewContainerEnvFromStringSlice(c.Env).GetDockerFlags()...)
a := append(args, c.Image)
return "docker", a
}
// NewContainer returns a new container filter
func NewContainer(spec runtimeutil.ContainerSpec, uidgid string) Filter {
f := Filter{ContainerSpec: spec, UIDGID: uidgid}
return f
}

View File

@@ -0,0 +1,5 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
// Package exec contains the exec function implementation.
package exec

View File

@@ -0,0 +1,36 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package exec
import (
"io"
"os"
"os/exec"
"sigs.k8s.io/kustomize/kyaml/fn/runtime/runtimeutil"
"sigs.k8s.io/kustomize/kyaml/yaml"
)
type Filter struct {
// Path is the path to the executable to run
Path string `yaml:"path,omitempty"`
// Args are the arguments to the executable
Args []string `yaml:"args,omitempty"`
runtimeutil.FunctionFilter
}
func (c *Filter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
c.FunctionFilter.Run = c.Run
return c.FunctionFilter.Filter(nodes)
}
func (c *Filter) Run(reader io.Reader, writer io.Writer) error {
cmd := exec.Command(c.Path, c.Args...)
cmd.Stdin = reader
cmd.Stdout = writer
cmd.Stderr = os.Stderr
return cmd.Run()
}

View File

@@ -0,0 +1,5 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
// Package runtimeutil contains libraries for implementing function runtimes.
package runtimeutil

View File

@@ -0,0 +1,305 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package runtimeutil
import (
"fmt"
"os"
"sort"
"strings"
"sigs.k8s.io/kustomize/kyaml/yaml"
)
const (
FunctionAnnotationKey = "config.kubernetes.io/function"
oldFunctionAnnotationKey = "config.k8s.io/function"
)
var functionAnnotationKeys = []string{FunctionAnnotationKey, oldFunctionAnnotationKey}
// ContainerNetworkName is a type for network name used in container
type ContainerNetworkName string
const (
NetworkNameNone ContainerNetworkName = "none"
NetworkNameHost ContainerNetworkName = "host"
)
const defaultEnvValue string = "true"
// ContainerEnv defines the environment present in a container.
type ContainerEnv struct {
// EnvVars is a key-value map that will be set as env in container
EnvVars map[string]string
// VarsToExport are only env key. Value will be the value in the host system
VarsToExport []string
}
// GetDockerFlags returns docker run style env flags
func (ce *ContainerEnv) GetDockerFlags() []string {
envs := ce.EnvVars
if envs == nil {
envs = make(map[string]string)
}
flags := []string{}
// return in order to keep consistent among different runs
keys := []string{}
for k := range envs {
keys = append(keys, k)
}
sort.Strings(keys)
for _, key := range keys {
flags = append(flags, "-e", key+"="+envs[key])
}
for _, key := range ce.VarsToExport {
flags = append(flags, "-e", key)
}
return flags
}
// AddKeyValue adds a key-value pair into the envs
func (ce *ContainerEnv) AddKeyValue(key, value string) {
if ce.EnvVars == nil {
ce.EnvVars = make(map[string]string)
}
ce.EnvVars[key] = value
}
// HasExportedKey returns true if the key is a exported key
func (ce *ContainerEnv) HasExportedKey(key string) bool {
for _, k := range ce.VarsToExport {
if k == key {
return true
}
}
return false
}
// AddKey adds a key into the envs
func (ce *ContainerEnv) AddKey(key string) {
if !ce.HasExportedKey(key) {
ce.VarsToExport = append(ce.VarsToExport, key)
}
}
// Raw returns a slice of string which represents the envs.
// Example: [foo=bar, baz]
func (ce *ContainerEnv) Raw() []string {
var ret []string
for k, v := range ce.EnvVars {
ret = append(ret, k+"="+v)
}
ret = append(ret, ce.VarsToExport...)
return ret
}
// NewContainerEnv returns a pointer to a new ContainerEnv
func NewContainerEnv() *ContainerEnv {
var ce ContainerEnv
ce.EnvVars = make(map[string]string)
// default envs
ce.EnvVars["LOG_TO_STDERR"] = defaultEnvValue
ce.EnvVars["STRUCTURED_RESULTS"] = defaultEnvValue
return &ce
}
// NewContainerEnvFromStringSlice returns a new ContainerEnv pointer with parsing
// input envStr. envStr example: ["foo=bar", "baz"]
func NewContainerEnvFromStringSlice(envStr []string) *ContainerEnv {
ce := NewContainerEnv()
for _, e := range envStr {
parts := strings.SplitN(e, "=", 2)
if len(parts) == 1 {
ce.AddKey(e)
} else {
ce.AddKeyValue(parts[0], parts[1])
}
}
return ce
}
// FunctionSpec defines a spec for running a function
type FunctionSpec struct {
DeferFailure bool `json:"deferFailure,omitempty" yaml:"deferFailure,omitempty"`
// Container is the spec for running a function as a container
Container ContainerSpec `json:"container,omitempty" yaml:"container,omitempty"`
// Starlark is the spec for running a function as a starlark script
Starlark StarlarkSpec `json:"starlark,omitempty" yaml:"starlark,omitempty"`
// ExecSpec is the spec for running a function as an executable
Exec ExecSpec `json:"exec,omitempty" yaml:"exec,omitempty"`
// Mounts are the storage or directories to mount into the container
StorageMounts []StorageMount `json:"mounts,omitempty" yaml:"mounts,omitempty"`
}
type ExecSpec struct {
Path string `json:"path,omitempty" yaml:"path,omitempty"`
}
// ContainerSpec defines a spec for running a function as a container
type ContainerSpec struct {
// Image is the container image to run
Image string `json:"image,omitempty" yaml:"image,omitempty"`
// Network defines network specific configuration
Network bool `json:"network,omitempty" yaml:"network,omitempty"`
// Mounts are the storage or directories to mount into the container
StorageMounts []StorageMount `json:"mounts,omitempty" yaml:"mounts,omitempty"`
// Env is a slice of env string that will be exposed to container
Env []string `json:"envs,omitempty" yaml:"envs,omitempty"`
}
// StarlarkSpec defines how to run a function as a starlark program
type StarlarkSpec struct {
Name string `json:"name,omitempty" yaml:"name,omitempty"`
// Path specifies a path to a starlark script
Path string `json:"path,omitempty" yaml:"path,omitempty"`
// URL specifies a url containing a starlark script
URL string `json:"url,omitempty" yaml:"url,omitempty"`
}
// StorageMount represents a container's mounted storage option(s)
type StorageMount struct {
// Type of mount e.g. bind mount, local volume, etc.
MountType string `json:"type,omitempty" yaml:"type,omitempty"`
// Source for the storage to be mounted.
// For named volumes, this is the name of the volume.
// For anonymous volumes, this field is omitted (empty string).
// For bind mounts, this is the path to the file or directory on the host.
Src string `json:"src,omitempty" yaml:"src,omitempty"`
// The path where the file or directory is mounted in the container.
DstPath string `json:"dst,omitempty" yaml:"dst,omitempty"`
// Mount in ReadWrite mode if it's explicitly configured
// See https://docs.docker.com/storage/bind-mounts/#use-a-read-only-bind-mount
ReadWriteMode bool `json:"rw,omitempty" yaml:"rw,omitempty"`
}
func (s *StorageMount) String() string {
mode := ""
if !s.ReadWriteMode {
mode = ",readonly"
}
return fmt.Sprintf("type=%s,source=%s,target=%s%s", s.MountType, s.Src, s.DstPath, mode)
}
// GetFunctionSpec returns the FunctionSpec for a resource. Returns
// nil if the resource does not have a FunctionSpec.
//
// The FunctionSpec is read from the resource metadata.annotation
// "config.kubernetes.io/function"
func GetFunctionSpec(n *yaml.RNode) *FunctionSpec {
meta, err := n.GetMeta()
if err != nil {
return nil
}
if fn := getFunctionSpecFromAnnotation(n, meta); fn != nil {
fn.StorageMounts = []StorageMount{}
return fn
}
// legacy function specification for backwards compatibility
container := meta.Annotations["config.kubernetes.io/container"]
if container != "" {
return &FunctionSpec{Container: ContainerSpec{Image: container}}
}
return nil
}
// getFunctionSpecFromAnnotation parses the config function from an annotation
// if it is found
func getFunctionSpecFromAnnotation(n *yaml.RNode, meta yaml.ResourceMeta) *FunctionSpec {
var fs FunctionSpec
for _, s := range functionAnnotationKeys {
fn := meta.Annotations[s]
if fn != "" {
err := yaml.Unmarshal([]byte(fn), &fs)
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
}
return &fs
}
}
n, err := n.Pipe(yaml.Lookup("metadata", "configFn"))
if err != nil || yaml.IsMissingOrNull(n) {
return nil
}
s, err := n.String()
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
}
err = yaml.Unmarshal([]byte(s), &fs)
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
}
return &fs
}
func StringToStorageMount(s string) StorageMount {
m := make(map[string]string)
options := strings.Split(s, ",")
for _, option := range options {
keyVal := strings.SplitN(option, "=", 2)
if len(keyVal) == 2 {
m[keyVal[0]] = keyVal[1]
}
}
var sm StorageMount
for key, value := range m {
switch {
case key == "type":
sm.MountType = value
case key == "src" || key == "source":
sm.Src = value
case key == "dst" || key == "target":
sm.DstPath = value
case key == "rw" && value == "true":
sm.ReadWriteMode = true
}
}
return sm
}
// IsReconcilerFilter filters Resources based on whether or not they are Reconciler Resource.
// Resources with an apiVersion starting with '*.gcr.io', 'gcr.io' or 'docker.io' are considered
// Reconciler Resources.
type IsReconcilerFilter struct {
// ExcludeReconcilers if set to true, then Reconcilers will be excluded -- e.g.
// Resources with a reconcile container through the apiVersion (gcr.io prefix) or
// through the annotations
ExcludeReconcilers bool `yaml:"excludeReconcilers,omitempty"`
// IncludeNonReconcilers if set to true, the NonReconciler will be included.
IncludeNonReconcilers bool `yaml:"includeNonReconcilers,omitempty"`
}
// Filter implements kio.Filter
func (c *IsReconcilerFilter) Filter(inputs []*yaml.RNode) ([]*yaml.RNode, error) {
var out []*yaml.RNode
for i := range inputs {
isFnResource := GetFunctionSpec(inputs[i]) != nil
if isFnResource && !c.ExcludeReconcilers {
out = append(out, inputs[i])
}
if !isFnResource && c.IncludeNonReconcilers {
out = append(out, inputs[i])
}
}
return out, nil
}

View File

@@ -0,0 +1,256 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package runtimeutil
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"path"
"strings"
"sigs.k8s.io/kustomize/kyaml/comments"
"sigs.k8s.io/kustomize/kyaml/errors"
"sigs.k8s.io/kustomize/kyaml/kio"
"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
"sigs.k8s.io/kustomize/kyaml/yaml"
)
// FunctionFilter wraps another filter to be invoked in the context of a function.
// FunctionFilter manages scoping the function, deferring failures, and saving results
// to files.
type FunctionFilter struct {
// Run implements the function.
Run func(reader io.Reader, writer io.Writer) error
// FunctionConfig is passed to the function through ResourceList.functionConfig.
FunctionConfig *yaml.RNode `yaml:"functionConfig,omitempty"`
// GlobalScope explicitly scopes the function to all input resources rather than only those
// resources scoped to it by path.
GlobalScope bool
// ResultsFile is the file to write function ResourceList.results to.
// If unset, results will not be written.
ResultsFile string
// DeferFailure will cause the Filter to return a nil error even if Run returns an error.
// The Run error will be available through GetExit().
DeferFailure bool
// results saves the results emitted from Run
results *yaml.RNode
// exit saves the error returned from Run
exit error
ids map[string]*yaml.RNode
}
// GetExit returns the error from Run
func (c FunctionFilter) GetExit() error {
return c.exit
}
// functionsDirectoryName is keyword directory name for functions scoped 1 directory higher
const functionsDirectoryName = "functions"
// getFunctionScope returns the path of the directory containing the function config,
// or its parent directory if the base directory is named "functions"
func (c *FunctionFilter) getFunctionScope() (string, error) {
m, err := c.FunctionConfig.GetMeta()
if err != nil {
return "", errors.Wrap(err)
}
p, found := m.Annotations[kioutil.PathAnnotation]
if !found {
return "", nil
}
functionDir := path.Clean(path.Dir(p))
if path.Base(functionDir) == functionsDirectoryName {
// the scope of functions in a directory called "functions" is 1 level higher
// this is similar to how the golang "internal" directory scoping works
functionDir = path.Dir(functionDir)
}
return functionDir, nil
}
// scope partitions the input nodes into 2 slices. The first slice contains only Resources
// which are scoped under dir, and the second slice contains the Resources which are not.
func (c *FunctionFilter) scope(dir string, nodes []*yaml.RNode) ([]*yaml.RNode, []*yaml.RNode, error) {
// scope container filtered Resources to Resources under that directory
var input, saved []*yaml.RNode
if c.GlobalScope {
return nodes, nil, nil
}
// global function
if dir == "" || dir == "." {
return nodes, nil, nil
}
// identify Resources read from directories under the function configuration
for i := range nodes {
m, err := nodes[i].GetMeta()
if err != nil {
return nil, nil, err
}
p, found := m.Annotations[kioutil.PathAnnotation]
if !found {
// this Resource isn't scoped under the function -- don't know where it came from
// consider it out of scope
saved = append(saved, nodes[i])
continue
}
resourceDir := path.Clean(path.Dir(p))
if path.Base(resourceDir) == functionsDirectoryName {
// Functions in the `functions` directory are scoped to
// themselves, and should see themselves as input
resourceDir = path.Dir(resourceDir)
}
if !strings.HasPrefix(resourceDir, dir) {
// this Resource doesn't fall under the function scope if it
// isn't in a subdirectory of where the function lives
saved = append(saved, nodes[i])
continue
}
// this input is scoped under the function
input = append(input, nodes[i])
}
return input, saved, nil
}
func (c *FunctionFilter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
in := &bytes.Buffer{}
out := &bytes.Buffer{}
// only process Resources scoped to this function, save the others
functionDir, err := c.getFunctionScope()
if err != nil {
return nil, err
}
input, saved, err := c.scope(functionDir, nodes)
if err != nil {
return nil, err
}
// set ids on each input so it is possible to copy comments from inputs back to outputs
if err := c.setIds(input); err != nil {
return nil, err
}
// write the input
err = kio.ByteWriter{
WrappingAPIVersion: kio.ResourceListAPIVersion,
WrappingKind: kio.ResourceListKind,
Writer: in,
KeepReaderAnnotations: true,
FunctionConfig: c.FunctionConfig}.Write(input)
if err != nil {
return nil, err
}
// capture the command stdout for the return value
r := &kio.ByteReader{Reader: out}
// don't exit immediately if the function fails -- write out the validation
c.exit = c.Run(in, out)
output, err := r.Read()
if err != nil {
return nil, err
}
// copy the comments from the inputs to the outputs
if err := c.setComments(output); err != nil {
return nil, err
}
if err := c.doResults(r); err != nil {
return nil, err
}
if c.exit != nil && !c.DeferFailure {
return append(output, saved...), c.exit
}
// annotate any generated Resources with a path and index if they don't already have one
if err := kioutil.DefaultPathAnnotation(functionDir, output); err != nil {
return nil, err
}
// emit both the Resources output from the function, and the out-of-scope Resources
// which were not provided to the function
return append(output, saved...), nil
}
const idAnnotation = "config.k8s.io/id"
func (c *FunctionFilter) setIds(nodes []*yaml.RNode) error {
// set the id on each node to map inputs to outputs
var id int
c.ids = map[string]*yaml.RNode{}
for i := range nodes {
id++
idStr := fmt.Sprintf("%v", id)
err := nodes[i].PipeE(yaml.SetAnnotation(idAnnotation, idStr))
if err != nil {
return errors.Wrap(err)
}
c.ids[idStr] = nodes[i]
}
return nil
}
func (c *FunctionFilter) setComments(nodes []*yaml.RNode) error {
for i := range nodes {
node := nodes[i]
anID, err := node.Pipe(yaml.GetAnnotation(idAnnotation))
if err != nil {
return errors.Wrap(err)
}
if anID == nil {
continue
}
var in *yaml.RNode
var found bool
if in, found = c.ids[anID.YNode().Value]; !found {
continue
}
if err := comments.CopyComments(in, node); err != nil {
return errors.Wrap(err)
}
if err := node.PipeE(yaml.ClearAnnotation(idAnnotation)); err != nil {
return errors.Wrap(err)
}
}
return nil
}
func (c *FunctionFilter) doResults(r *kio.ByteReader) error {
// Write the results to a file if configured to do so
if c.ResultsFile != "" && r.Results != nil {
results, err := r.Results.String()
if err != nil {
return err
}
err = ioutil.WriteFile(c.ResultsFile, []byte(results), 0600)
if err != nil {
return err
}
}
if r.Results != nil {
c.results = r.Results
}
return nil
}

View File

@@ -0,0 +1,8 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package runtimeutil
type DeferFailureFunction interface {
GetExit() error
}

View File

@@ -0,0 +1,79 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package starlark
import (
"encoding/json"
"os"
"strings"
"go.starlark.net/starlark"
"go.starlark.net/starlarkstruct"
"sigs.k8s.io/kustomize/kyaml/errors"
"sigs.k8s.io/kustomize/kyaml/internal/forked/github.com/qri-io/starlib/util"
"sigs.k8s.io/kustomize/kyaml/openapi"
"sigs.k8s.io/kustomize/kyaml/yaml"
)
type Context struct {
resourceList starlark.Value
}
func (c *Context) predeclared() (starlark.StringDict, error) {
e, err := env()
if err != nil {
return nil, err
}
oa, err := oa()
if err != nil {
return nil, err
}
dict := starlark.StringDict{
"resource_list": c.resourceList,
"open_api": oa,
"environment": e,
}
return starlark.StringDict{
"ctx": starlarkstruct.FromStringDict(starlarkstruct.Default, dict),
}, nil
}
func oa() (starlark.Value, error) {
return interfaceToValue(openapi.Schema())
}
func env() (starlark.Value, error) {
env := map[string]interface{}{}
for _, e := range os.Environ() {
pair := strings.SplitN(e, "=", 2)
if len(pair) < 2 {
continue
}
env[pair[0]] = pair[1]
}
value, err := util.Marshal(env)
if err != nil {
return nil, errors.Wrap(err)
}
return value, nil
}
func interfaceToValue(i interface{}) (starlark.Value, error) {
b, err := json.Marshal(i)
if err != nil {
return nil, err
}
var in map[string]interface{}
if err := yaml.Unmarshal(b, &in); err != nil {
return nil, errors.Wrap(err)
}
value, err := util.Marshal(in)
if err != nil {
return nil, errors.Wrap(err)
}
return value, nil
}

View File

@@ -0,0 +1,36 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
// Package starlark contains a kio.Filter which can be applied to resources to transform
// them through starlark program.
//
// Starlark has become a popular runtime embedding in go programs, especially for Kubernetes
// and data processing.
// Examples: https://github.com/cruise-automation/isopod, https://qri.io/docs/starlark/starlib,
// https://github.com/stripe/skycfg, https://github.com/k14s/ytt
//
// The resources are provided to the starlark program through the global variable "resourceList".
// "resourceList" is a dictionary containing an "items" field with a list of resources.
// The starlark modified "resourceList" is the Filter output.
//
// After being run through the starlark program, the filter will copy the comments from the input
// resources to restore them -- due to them being dropped as a result of serializing the resources
// as starlark values.
//
// "resourceList" may also contain a "functionConfig" entry to configure the starlark script itself.
// Changes made by the starlark program to the "functionConfig" will be reflected in the
// Filter.FunctionConfig value.
//
// The Filter will also format the output so that output has the preferred field ordering
// rather than an alphabetical field ordering.
//
// The resourceList variable adheres to the kustomize function spec as specified by:
// https://github.com/kubernetes-sigs/kustomize/blob/master/cmd/config/docs/api-conventions/functions-spec.md
//
// All items in the resourceList are resources represented as starlark dictionaries/
// The items in the resourceList respect the io spec specified by:
// https://github.com/kubernetes-sigs/kustomize/blob/master/cmd/config/docs/api-conventions/config-io.md
//
// The starlark language spec can be found here:
// https://github.com/google/starlark-go/blob/master/doc/spec.md
package starlark

View File

@@ -0,0 +1,181 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package starlark
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"net/http"
"go.starlark.net/starlark"
"sigs.k8s.io/kustomize/kyaml/errors"
"sigs.k8s.io/kustomize/kyaml/fn/runtime/runtimeutil"
"sigs.k8s.io/kustomize/kyaml/internal/forked/github.com/qri-io/starlib/util"
"sigs.k8s.io/kustomize/kyaml/kio/filters"
"sigs.k8s.io/kustomize/kyaml/yaml"
)
// Filter transforms a set of resources through the provided program
type Filter struct {
Name string
// Program is a starlark script which will be run against the resources
Program string
// URL is the url of a starlark program to fetch and run
URL string
// Path is the path to a starlark program to read and run
Path string
runtimeutil.FunctionFilter
}
func (sf *Filter) String() string {
return fmt.Sprintf(
"name: %v path: %v url: %v program: %v", sf.Name, sf.Path, sf.URL, sf.Program)
}
func (sf *Filter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
err := sf.setup()
if err != nil {
return nil, err
}
sf.FunctionFilter.Run = sf.Run
return sf.FunctionFilter.Filter(nodes)
}
func (sf *Filter) setup() error {
if (sf.URL != "" && sf.Path != "") ||
(sf.URL != "" && sf.Program != "") ||
(sf.Path != "" && sf.Program != "") {
return errors.Errorf("Filter Path, Program and URL are mutually exclusive")
}
// read the program from a file
if sf.Path != "" {
b, err := ioutil.ReadFile(sf.Path)
if err != nil {
return err
}
sf.Program = string(b)
}
// read the program from a URL
if sf.URL != "" {
err := func() error {
resp, err := http.Get(sf.URL)
if err != nil {
return err
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
sf.Program = string(b)
return nil
}()
if err != nil {
return err
}
}
return nil
}
func (sf *Filter) Run(reader io.Reader, writer io.Writer) error {
// retain map of inputs to outputs by id so if the name is changed by the
// starlark program, we are able to match the same resources
value, err := sf.readResourceList(reader)
if err != nil {
return errors.Wrap(err)
}
// run the starlark as program as transformation function
thread := &starlark.Thread{Name: sf.Name}
ctx := &Context{resourceList: value}
pd, err := ctx.predeclared()
if err != nil {
return errors.Wrap(err)
}
_, err = starlark.ExecFile(thread, sf.Name, sf.Program, pd)
if err != nil {
return errors.Wrap(err)
}
return sf.writeResourceList(value, writer)
}
// inputToResourceList transforms input into a starlark.Value
func (sf *Filter) readResourceList(reader io.Reader) (starlark.Value, error) {
// read and parse the inputs
rl := bytes.Buffer{}
_, err := rl.ReadFrom(reader)
if err != nil {
return nil, errors.Wrap(err)
}
rn, err := yaml.Parse(rl.String())
if err != nil {
return nil, errors.Wrap(err)
}
// convert to a starlark value
b, err := yaml.Marshal(rn.Document()) // convert to bytes
if err != nil {
return nil, errors.Wrap(err)
}
var in map[string]interface{}
err = yaml.Unmarshal(b, &in) // convert to map[string]interface{}
if err != nil {
return nil, errors.Wrap(err)
}
return util.Marshal(in) // convert to starlark value
}
// resourceListToOutput converts the output of the starlark program to the filter output
func (sf *Filter) writeResourceList(value starlark.Value, writer io.Writer) error {
// convert the modified resourceList back into a slice of RNodes
// by first converting to a map[string]interface{}
out, err := util.Unmarshal(value)
if err != nil {
return errors.Wrap(err)
}
b, err := yaml.Marshal(out)
if err != nil {
return errors.Wrap(err)
}
rl, err := yaml.Parse(string(b))
if err != nil {
return errors.Wrap(err)
}
// preserve the comments from the input
items, err := rl.Pipe(yaml.Lookup("items"))
if err != nil {
return errors.Wrap(err)
}
err = items.VisitElements(func(node *yaml.RNode) error {
// starlark will serialize the resources sorting the fields alphabetically,
// format them to have a better ordering
_, err := filters.FormatFilter{}.Filter([]*yaml.RNode{node})
return err
})
if err != nil {
return errors.Wrap(err)
}
s, err := rl.String()
if err != nil {
return errors.Wrap(err)
}
_, err = writer.Write([]byte(s))
return err
}