Files
kubesphere/vendor/github.com/projectcalico/libcalico-go/lib/clientv3/ippool.go
2019-08-17 15:34:02 +08:00

621 lines
20 KiB
Go

// Copyright (c) 2017-2019 Tigera, Inc. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package clientv3
import (
"context"
"fmt"
"net"
"time"
log "github.com/sirupsen/logrus"
apiv3 "github.com/projectcalico/libcalico-go/lib/apis/v3"
"github.com/projectcalico/libcalico-go/lib/backend/model"
cerrors "github.com/projectcalico/libcalico-go/lib/errors"
cnet "github.com/projectcalico/libcalico-go/lib/net"
"github.com/projectcalico/libcalico-go/lib/options"
validator "github.com/projectcalico/libcalico-go/lib/validator/v3"
"github.com/projectcalico/libcalico-go/lib/watch"
)
// IPPoolInterface has methods to work with IPPool resources.
type IPPoolInterface interface {
Create(ctx context.Context, res *apiv3.IPPool, opts options.SetOptions) (*apiv3.IPPool, error)
Update(ctx context.Context, res *apiv3.IPPool, opts options.SetOptions) (*apiv3.IPPool, error)
Delete(ctx context.Context, name string, opts options.DeleteOptions) (*apiv3.IPPool, error)
Get(ctx context.Context, name string, opts options.GetOptions) (*apiv3.IPPool, error)
List(ctx context.Context, opts options.ListOptions) (*apiv3.IPPoolList, error)
Watch(ctx context.Context, opts options.ListOptions) (watch.Interface, error)
}
// ipPools implements IPPoolInterface
type ipPools struct {
client client
}
// Create takes the representation of a IPPool and creates it. Returns the stored
// representation of the IPPool, and an error, if there is any.
func (r ipPools) Create(ctx context.Context, res *apiv3.IPPool, opts options.SetOptions) (*apiv3.IPPool, error) {
if res != nil {
// Since we're about to default some fields, take a (shallow) copy of the input data
// before we do so.
resCopy := *res
res = &resCopy
}
// Validate the IPPool before creating the resource.
if err := r.validateAndSetDefaults(ctx, res, nil); err != nil {
return nil, err
}
if err := validator.Validate(res); err != nil {
return nil, err
}
// Check that there are no existing blocks in the pool range that have a different block size.
poolBlockSize := res.Spec.BlockSize
poolIP, poolCIDR, err := net.ParseCIDR(res.Spec.CIDR)
if err != nil {
return nil, cerrors.ErrorParsingDatastoreEntry{
RawKey: "CIDR",
RawValue: string(res.Spec.CIDR),
Err: err,
}
}
ipVersion := 4
if poolIP.To4() == nil {
ipVersion = 6
}
blocks, err := r.client.backend.List(ctx, model.BlockListOptions{IPVersion: ipVersion}, "")
if _, ok := err.(cerrors.ErrorOperationNotSupported); !ok && err != nil {
// There was an error and it wasn't OperationNotSupported - return it.
return nil, err
} else if err == nil {
// Skip the block check if the error is OperationUnsupported - listing blocks is not
// supported with host-local IPAM on KDD.
for _, b := range blocks.KVPairs {
k := b.Key.(model.BlockKey)
ones, _ := k.CIDR.Mask.Size()
// Check if this block has a different size to the pool, and that it overlaps with the pool.
if ones != poolBlockSize && k.CIDR.IsNetOverlap(*poolCIDR) {
return nil, cerrors.ErrorValidation{
ErroredFields: []cerrors.ErroredField{{
Name: "IPPool.Spec.BlockSize",
Reason: "IPPool blocksSize conflicts with existing allocations that use a different blockSize",
Value: res.Spec.BlockSize,
}},
}
}
}
}
// Enable IPIP or VXLAN globally if required. Do this before the Create so if it fails the user
// can retry the same command.
err = r.maybeEnableIPIP(ctx, res)
if err != nil {
return nil, err
}
err = r.maybeEnableVXLAN(ctx, res)
if err != nil {
return nil, err
}
out, err := r.client.resources.Create(ctx, opts, apiv3.KindIPPool, res)
if out != nil {
return out.(*apiv3.IPPool), err
}
return nil, err
}
// Update takes the representation of a IPPool and updates it. Returns the stored
// representation of the IPPool, and an error, if there is any.
func (r ipPools) Update(ctx context.Context, res *apiv3.IPPool, opts options.SetOptions) (*apiv3.IPPool, error) {
if res != nil {
// Since we're about to default some fields, take a (shallow) copy of the input data
// before we do so.
resCopy := *res
res = &resCopy
}
// Get the existing settings, so that we can validate the CIDR and block size have not changed.
old, err := r.Get(ctx, res.Name, options.GetOptions{})
if err != nil {
return nil, err
}
// Validate the IPPool updating the resource.
if err := r.validateAndSetDefaults(ctx, res, old); err != nil {
return nil, err
}
if err := validator.Validate(res); err != nil {
return nil, err
}
// Enable IPIP globally if required. Do this before the Update so if it fails the user
// can retry the same command.
err = r.maybeEnableIPIP(ctx, res)
if err != nil {
return nil, err
}
err = r.maybeEnableVXLAN(ctx, res)
if err != nil {
return nil, err
}
out, err := r.client.resources.Update(ctx, opts, apiv3.KindIPPool, res)
if out != nil {
return out.(*apiv3.IPPool), err
}
return nil, err
}
// Delete takes name of the IPPool and deletes it. Returns an error if one occurs.
func (r ipPools) Delete(ctx context.Context, name string, opts options.DeleteOptions) (*apiv3.IPPool, error) {
// Deleting a pool requires a little care because of existing endpoints
// using IP addresses allocated in the pool. We do the deletion in
// the following steps:
// - disable the pool so no more IPs are assigned from it
// - remove all affinities associated with the pool
// - delete the pool
// Get the pool so that we can find the CIDR associated with it.
pool, err := r.Get(ctx, name, options.GetOptions{})
if err != nil {
return nil, err
}
logCxt := log.WithFields(log.Fields{
"CIDR": pool.Spec.CIDR,
"Name": name,
})
// If the pool is active, set the disabled flag to ensure we stop allocating from this pool.
if !pool.Spec.Disabled {
logCxt.Info("Disabling pool to release affinities")
pool.Spec.Disabled = true
// If the Delete has been called with a ResourceVersion then use that to perform the
// update - that way we'll catch update conflicts (we could actually check here, but
// the most likely scenario is there isn't one - so just pass it through and let the
// Update handle any conflicts).
if opts.ResourceVersion != "" {
pool.ResourceVersion = opts.ResourceVersion
}
if _, err := r.Update(ctx, pool, options.SetOptions{}); err != nil {
return nil, err
}
// Reset the resource version before the actual delete since the version of that resource
// will now have been updated.
opts.ResourceVersion = ""
}
// Release affinities associated with this pool. We do this even if the pool was disabled
// (since it may have been enabled at one time, and if there are no affine blocks created
// then this will be a no-op). We've already validated the CIDR so we know it will parse.
if _, cidrNet, err := cnet.ParseCIDR(pool.Spec.CIDR); err == nil {
logCxt.Info("Releasing pool affinities")
// Pause for a short period before releasing the affinities - this gives any in-progress
// allocations an opportunity to finish.
time.Sleep(500 * time.Millisecond)
err = r.client.IPAM().ReleasePoolAffinities(ctx, *cidrNet)
// Depending on the datastore, IPAM may not be supported. If we get a not supported
// error, then continue. Any other error, fail.
if _, ok := err.(cerrors.ErrorOperationNotSupported); !ok && err != nil {
return nil, err
}
}
// And finally, delete the pool.
logCxt.Info("Deleting pool")
out, err := r.client.resources.Delete(ctx, opts, apiv3.KindIPPool, noNamespace, name)
if out != nil {
return out.(*apiv3.IPPool), err
}
return nil, err
}
// Get takes name of the IPPool, and returns the corresponding IPPool object,
// and an error if there is any.
func (r ipPools) Get(ctx context.Context, name string, opts options.GetOptions) (*apiv3.IPPool, error) {
out, err := r.client.resources.Get(ctx, opts, apiv3.KindIPPool, noNamespace, name)
if out != nil {
convertIpPoolFromStorage(out.(*apiv3.IPPool))
return out.(*apiv3.IPPool), err
}
return nil, err
}
// List returns the list of IPPool objects that match the supplied options.
func (r ipPools) List(ctx context.Context, opts options.ListOptions) (*apiv3.IPPoolList, error) {
res := &apiv3.IPPoolList{}
if err := r.client.resources.List(ctx, opts, apiv3.KindIPPool, apiv3.KindIPPoolList, res); err != nil {
return nil, err
}
// Default values when reading from backend.
for i := range res.Items {
convertIpPoolFromStorage(&res.Items[i])
}
return res, nil
}
// Default pool values when reading from storage
func convertIpPoolFromStorage(pool *apiv3.IPPool) error {
// Default the blockSize if it wasn't previously set
if pool.Spec.BlockSize == 0 {
// Get the IP address of the CIDR to find the IP version
ipAddr, _, err := cnet.ParseCIDR(pool.Spec.CIDR)
if err != nil {
return cerrors.ErrorValidation{
ErroredFields: []cerrors.ErroredField{{
Name: "IPPool.Spec.CIDR",
Reason: "IPPool CIDR must be a valid subnet",
Value: pool.Spec.CIDR,
}},
}
}
if ipAddr.Version() == 4 {
pool.Spec.BlockSize = 26
} else {
pool.Spec.BlockSize = 122
}
}
// Default the nodeSelector if it wasn't previously set.
if pool.Spec.NodeSelector == "" {
pool.Spec.NodeSelector = "all()"
}
return nil
}
// Watch returns a watch.Interface that watches the IPPools that match the
// supplied options.
func (r ipPools) Watch(ctx context.Context, opts options.ListOptions) (watch.Interface, error) {
return r.client.resources.Watch(ctx, opts, apiv3.KindIPPool, nil)
}
// validateAndSetDefaults validates IPPool fields and sets default values that are
// not assigned.
// The old pool will be unassigned for a Create.
func (r ipPools) validateAndSetDefaults(ctx context.Context, new, old *apiv3.IPPool) error {
errFields := []cerrors.ErroredField{}
// Spec.CIDR field must not be empty.
if new.Spec.CIDR == "" {
return cerrors.ErrorValidation{
ErroredFields: []cerrors.ErroredField{{
Name: "IPPool.Spec.CIDR",
Reason: "IPPool CIDR must be specified",
}},
}
}
// Make sure the CIDR is parsable.
ipAddr, cidr, err := cnet.ParseCIDR(new.Spec.CIDR)
if err != nil {
return cerrors.ErrorValidation{
ErroredFields: []cerrors.ErroredField{{
Name: "IPPool.Spec.CIDR",
Reason: "IPPool CIDR must be a valid subnet",
Value: new.Spec.CIDR,
}},
}
}
// Normalize the CIDR before persisting.
new.Spec.CIDR = cidr.String()
// If a nodeSelector is not specified, then this IP pool selects all nodes.
if new.Spec.NodeSelector == "" {
new.Spec.NodeSelector = "all()"
}
// If there was a previous pool then this must be an Update, validate that the
// CIDR has not changed. Since we are using normalized CIDRs we can just do a
// simple string comparison.
if old != nil && old.Spec.CIDR != new.Spec.CIDR {
errFields = append(errFields, cerrors.ErroredField{
Name: "IPPool.Spec.CIDR",
Reason: "IPPool CIDR cannot be modified",
Value: new.Spec.CIDR,
})
}
// Default the blockSize
if new.Spec.BlockSize == 0 {
if ipAddr.Version() == 4 {
new.Spec.BlockSize = 26
} else {
new.Spec.BlockSize = 122
}
}
// Check that the blockSize hasn't changed since updates are not supported.
if old != nil && old.Spec.BlockSize != new.Spec.BlockSize {
errFields = append(errFields, cerrors.ErroredField{
Name: "IPPool.Spec.BlockSize",
Reason: "IPPool BlockSize cannot be modified",
Value: new.Spec.BlockSize,
})
}
if ipAddr.Version() == 4 {
if new.Spec.BlockSize > 32 || new.Spec.BlockSize < 20 {
errFields = append(errFields, cerrors.ErroredField{
Name: "IPPool.Spec.BlockSize",
Reason: "IPv4 block size must be between 20 and 32",
Value: new.Spec.BlockSize,
})
}
} else {
if new.Spec.BlockSize > 128 || new.Spec.BlockSize < 116 {
errFields = append(errFields, cerrors.ErroredField{
Name: "IPPool.Spec.BlockSize",
Reason: "IPv6 block size must be between 116 and 128",
Value: new.Spec.BlockSize,
})
}
}
// The Calico IPAM places restrictions on the minimum IP pool size. If
// the ippool is enabled, check that the pool is at least the minimum size.
if !new.Spec.Disabled {
ones, _ := cidr.Mask.Size()
log.Debugf("Pool CIDR: %s, mask: %d, blockSize: %d", cidr.String(), ones, new.Spec.BlockSize)
if ones > new.Spec.BlockSize {
errFields = append(errFields, cerrors.ErroredField{
Name: "IPPool.Spec.CIDR",
Reason: "IP pool size is too small for use with Calico IPAM. It must be equal to or greater than the block size.",
Value: new.Spec.CIDR,
})
}
}
// If there was no previous pool then this must be a Create. Check that the CIDR
// does not overlap with any other pool CIDRs.
if old == nil {
allPools, err := r.List(ctx, options.ListOptions{})
if err != nil {
return err
}
for _, otherPool := range allPools.Items {
// It's possible that Create is called for a pre-existing pool, so skip our own
// pool and let the generic processing handle the pre-existing resource error case.
if otherPool.Name == new.Name {
continue
}
_, otherCIDR, err := cnet.ParseCIDR(otherPool.Spec.CIDR)
if err != nil {
log.WithField("Name", otherPool.Name).WithError(err).Error("IPPool is configured with an invalid CIDR")
continue
}
if otherCIDR.IsNetOverlap(cidr.IPNet) {
errFields = append(errFields, cerrors.ErroredField{
Name: "IPPool.Spec.CIDR",
Reason: fmt.Sprintf("IPPool(%s) CIDR overlaps with IPPool(%s) CIDR %s", new.Name, otherPool.Name, otherPool.Spec.CIDR),
Value: new.Spec.CIDR,
})
}
}
}
// Make sure IPIPMode is defaulted to "Never".
if len(new.Spec.IPIPMode) == 0 {
new.Spec.IPIPMode = apiv3.IPIPModeNever
}
// Make sure VXLANMode is defaulted to "Never".
if len(new.Spec.VXLANMode) == 0 {
new.Spec.VXLANMode = apiv3.VXLANModeNever
}
// Make sure only one of VXLAN and IPIP is enabled.
if new.Spec.VXLANMode != apiv3.VXLANModeNever && new.Spec.IPIPMode != apiv3.IPIPModeNever {
errFields = append(errFields, cerrors.ErroredField{
Name: "IPPool.Spec.VXLANMode",
Reason: "Cannot enable both VXLAN and IPIP on the same IPPool",
Value: new.Spec.VXLANMode,
})
}
// IPIP cannot be enabled for IPv6.
if cidr.Version() == 6 && new.Spec.IPIPMode != apiv3.IPIPModeNever {
errFields = append(errFields, cerrors.ErroredField{
Name: "IPPool.Spec.IPIPMode",
Reason: "IPIP is not supported on an IPv6 IP pool",
Value: new.Spec.IPIPMode,
})
}
// The Calico CIDR should be strictly masked
log.Debugf("IPPool CIDR: %s, Masked IP: %d", new.Spec.CIDR, cidr.IP)
if cidr.IP.String() != ipAddr.String() {
errFields = append(errFields, cerrors.ErroredField{
Name: "IPPool.Spec.CIDR",
Reason: "IPPool CIDR is not strictly masked",
Value: new.Spec.CIDR,
})
}
// IPv4 link local subnet.
ipv4LinkLocalNet := net.IPNet{
IP: net.ParseIP("169.254.0.0"),
Mask: net.CIDRMask(16, 32),
}
// IPv6 link local subnet.
ipv6LinkLocalNet := net.IPNet{
IP: net.ParseIP("fe80::"),
Mask: net.CIDRMask(10, 128),
}
// IP Pool CIDR cannot overlap with IPv4 or IPv6 link local address range.
if cidr.Version() == 4 && cidr.IsNetOverlap(ipv4LinkLocalNet) {
errFields = append(errFields, cerrors.ErroredField{
Name: "IPPool.Spec.CIDR",
Reason: "IPPool CIDR overlaps with IPv4 Link Local range 169.254.0.0/16",
Value: new.Spec.CIDR,
})
}
if cidr.Version() == 6 && cidr.IsNetOverlap(ipv6LinkLocalNet) {
errFields = append(errFields, cerrors.ErroredField{
Name: "IPPool.Spec.CIDR",
Reason: "IPPool CIDR overlaps with IPv6 Link Local range fe80::/10",
Value: new.Spec.CIDR,
})
}
// Return the errors if we have one or more validation errors.
if len(errFields) > 0 {
return cerrors.ErrorValidation{
ErroredFields: errFields,
}
}
return nil
}
// maybeEnableIPIP enables global IPIP if a default setting is not already configured
// and the pool has IPIP enabled.
func (c ipPools) maybeEnableIPIP(ctx context.Context, pool *apiv3.IPPool) error {
if pool.Spec.IPIPMode == apiv3.IPIPModeNever {
log.Debug("IPIP is not enabled for this pool - no need to check global setting")
return nil
}
var err error
ipEnabled := true
for i := 0; i < maxApplyRetries; i++ {
log.WithField("Retry", i).Debug("Checking global IPIP setting")
res, err := c.client.FelixConfigurations().Get(ctx, "default", options.GetOptions{})
if _, ok := err.(cerrors.ErrorResourceDoesNotExist); !ok && err != nil {
log.WithError(err).Debug("Error getting current FelixConfiguration resource")
return err
}
if res == nil {
log.Debug("Global FelixConfiguration does not exist - creating")
res = apiv3.NewFelixConfiguration()
res.Name = "default"
} else if res.Spec.IPIPEnabled != nil {
// A value for the default config is set so leave unchanged. It may be set to false,
// so log the actual value - but we shouldn't update it if someone has explicitly
// disabled it globally.
log.WithField("IPIPEnabled", res.Spec.IPIPEnabled).Debug("Global IPIPEnabled setting is already configured")
return nil
}
// Enable IpInIp and do the Create or Update.
res.Spec.IPIPEnabled = &ipEnabled
if res.ResourceVersion == "" {
res, err = c.client.FelixConfigurations().Create(ctx, res, options.SetOptions{})
if _, ok := err.(cerrors.ErrorResourceAlreadyExists); ok {
log.Debug("FelixConfiguration already exists - retry update")
continue
}
} else {
res, err = c.client.FelixConfigurations().Update(ctx, res, options.SetOptions{})
if _, ok := err.(cerrors.ErrorResourceUpdateConflict); ok {
log.Debug("FelixConfiguration update conflict - retry update")
continue
}
}
if err == nil {
log.Debug("FelixConfiguration updated successfully")
return nil
}
log.WithError(err).Debug("Error updating FelixConfiguration to enable IPIP")
return err
}
// Return the error from the final Update.
log.WithError(err).Info("Too many conflict failures attempting to update FelixConfiguration to enable IPIP")
return err
}
// maybeEnableVXLAN enables global VXLAN if a default setting is not already configured
// and the pool has VXLAN enabled.
func (c ipPools) maybeEnableVXLAN(ctx context.Context, pool *apiv3.IPPool) error {
if pool.Spec.VXLANMode == apiv3.VXLANModeNever {
log.Debug("VXLAN is not enabled for this pool - no need to check global setting")
return nil
}
var err error
ipEnabled := true
for i := 0; i < maxApplyRetries; i++ {
log.WithField("Retry", i).Debug("Checking global VXLAN setting")
res, err := c.client.FelixConfigurations().Get(ctx, "default", options.GetOptions{})
if _, ok := err.(cerrors.ErrorResourceDoesNotExist); !ok && err != nil {
log.WithError(err).Debug("Error getting current FelixConfiguration resource")
return err
}
if res == nil {
log.Debug("Global FelixConfiguration does not exist - creating")
res = apiv3.NewFelixConfiguration()
res.Name = "default"
} else if res.Spec.VXLANEnabled != nil {
// A value for the default config is set so leave unchanged. It may be set to false,
// so log the actual value - but we shouldn't update it if someone has explicitly
// disabled it globally.
log.WithField("VXLANEnabled", res.Spec.VXLANEnabled).Debug("Global VXLANEnabled setting is already configured")
return nil
}
// Enable IpInIp and do the Create or Update.
res.Spec.VXLANEnabled = &ipEnabled
if res.ResourceVersion == "" {
res, err = c.client.FelixConfigurations().Create(ctx, res, options.SetOptions{})
if _, ok := err.(cerrors.ErrorResourceAlreadyExists); ok {
log.Debug("FelixConfiguration already exists - retry update")
continue
}
} else {
res, err = c.client.FelixConfigurations().Update(ctx, res, options.SetOptions{})
if _, ok := err.(cerrors.ErrorResourceUpdateConflict); ok {
log.Debug("FelixConfiguration update conflict - retry update")
continue
}
}
if err == nil {
log.Debug("FelixConfiguration updated successfully")
return nil
}
log.WithError(err).Debug("Error updating FelixConfiguration to enable VXLAN")
return err
}
// Return the error from the final Update.
log.WithError(err).Info("Too many conflict failures attempting to update FelixConfiguration to enable VXLAN")
return err
}