temp commit

vendor/github.com/projectcalico/libcalico-go/lib/backend/etcdv3/conversion.go

@@ -0,0 +1,121 @@
+// Copyright (c) 2016-2019 Tigera, Inc. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package etcdv3
+
+import (
+	"fmt"
+	"strconv"
+
+	"github.com/coreos/etcd/clientv3"
+	"github.com/coreos/etcd/mvcc/mvccpb"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/projectcalico/libcalico-go/lib/backend/api"
+	"github.com/projectcalico/libcalico-go/lib/backend/model"
+	"github.com/projectcalico/libcalico-go/lib/errors"
+)
+
+// convertListResponse converts etcdv3 Kv to a model.KVPair with parsed values.
+// If the etcdv3 key or value does not represent the resource specified by the ListInterface,
+// or if value cannot be parsed, this method returns nil.
+func convertListResponse(ekv *mvccpb.KeyValue, l model.ListInterface) *model.KVPair {
+	log.WithField("etcdv3-etcdKey", string(ekv.Key)).Debug("Processing etcdv3 entry")
+	if k := l.KeyFromDefaultPath(string(ekv.Key)); k != nil {
+		log.WithField("model-etcdKey", k).Debug("Key is valid and converted to model-etcdKey")
+		if v, err := model.ParseValue(k, ekv.Value); err == nil {
+			log.Debug("Value is valid - return KVPair with parsed value")
+			return &model.KVPair{Key: k, Value: v, Revision: strconv.FormatInt(ekv.ModRevision, 10)}
+		}
+	}
+	return nil
+}
+
+// convertWatchEvent converts an etcdv3 watch event to an api.WatchEvent, or nil if the
+// event did not correspond to an event that we are interested in.
+func convertWatchEvent(e *clientv3.Event, l model.ListInterface) (*api.WatchEvent, error) {
+	log.WithField("etcdv3-etcdKey", string(e.Kv.Key)).Debug("Processing etcdv3 event")
+
+	var eventType api.WatchEventType
+	switch {
+	case e.Type == clientv3.EventTypeDelete:
+		eventType = api.WatchDeleted
+	case e.IsCreate():
+		eventType = api.WatchAdded
+	default:
+		eventType = api.WatchModified
+	}
+
+	var oldKV, newKV *model.KVPair
+	var err error
+	if k := l.KeyFromDefaultPath(string(e.Kv.Key)); k != nil {
+		log.WithField("model-etcdKey", k).Debug("Key is valid and converted to model-etcdKey")
+
+		if eventType != api.WatchDeleted {
+			// Add or modify, parse the new value.
+			if newKV, err = etcdToKVPair(k, e.Kv); err != nil {
+				return nil, err
+			}
+		}
+		if eventType != api.WatchAdded {
+			// Delete or modify, parse the old value.
+			if oldKV, err = etcdToKVPair(k, e.PrevKv); err != nil {
+				if eventType == api.WatchDeleted || err != ErrMissingValue {
+					// Ignore missing value for modified events, but we need them for deletion.
+					return nil, err
+				}
+			}
+		}
+	} else {
+		log.WithField("key", string(e.Kv.Key)).Debug("key filtered")
+		return nil, nil
+	}
+
+	return &api.WatchEvent{
+		Old:  oldKV,
+		New:  newKV,
+		Type: eventType,
+	}, nil
+}
+
+var (
+	ErrMissingValue = fmt.Errorf("missing etcd KV")
+)
+
+// etcdToKVPair converts an etcd KeyValue in to model.KVPair.
+func etcdToKVPair(key model.Key, ekv *mvccpb.KeyValue) (*model.KVPair, error) {
+	if ekv == nil {
+		return nil, ErrMissingValue
+	}
+
+	v, err := model.ParseValue(key, ekv.Value)
+	if err != nil {
+		if len(ekv.Value) == 0 {
+			// We do this check after the ParseValue call because ParseValue has some special-case logic for handling
+			// empty values for some resource types.
+			return nil, ErrMissingValue
+		}
+		return nil, errors.ErrorParsingDatastoreEntry{
+			RawKey:   string(ekv.Key),
+			RawValue: string(ekv.Value),
+			Err:      err,
+		}
+	}
+
+	return &model.KVPair{
+		Key:      key,
+		Value:    v,
+		Revision: strconv.FormatInt(ekv.ModRevision, 10),
+	}, nil
+}

vendor/github.com/projectcalico/libcalico-go/lib/backend/etcdv3/etcdv3.go

@@ -0,0 +1,542 @@
+// Copyright (c) 2016-2019 Tigera, Inc. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package etcdv3
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"crypto/tls"
+
+	"github.com/coreos/etcd/clientv3"
+	"github.com/coreos/etcd/pkg/srv"
+	"github.com/coreos/etcd/pkg/transport"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/projectcalico/libcalico-go/lib/apiconfig"
+	"github.com/projectcalico/libcalico-go/lib/backend/api"
+	"github.com/projectcalico/libcalico-go/lib/backend/model"
+	cerrors "github.com/projectcalico/libcalico-go/lib/errors"
+)
+
+var (
+	clientTimeout    = 10 * time.Second
+	keepaliveTime    = 30 * time.Second
+	keepaliveTimeout = 10 * time.Second
+)
+
+type etcdV3Client struct {
+	etcdClient *clientv3.Client
+}
+
+func NewEtcdV3Client(config *apiconfig.EtcdConfig) (api.Client, error) {
+	if config.EtcdEndpoints != "" && config.EtcdDiscoverySrv != "" {
+		log.Warning("Multiple etcd endpoint discovery methods specified in etcdv3 API config")
+		return nil, errors.New("multiple discovery or bootstrap options specified, use either \"etcdEndpoints\" or \"etcdDiscoverySrv\"")
+	}
+
+	// Split the endpoints into a location slice.
+	etcdLocation := []string{}
+	if config.EtcdEndpoints != "" {
+		etcdLocation = strings.Split(config.EtcdEndpoints, ",")
+	}
+
+	if config.EtcdDiscoverySrv != "" {
+		srvs, srvErr := srv.GetClient("etcd-client", config.EtcdDiscoverySrv)
+		if srvErr != nil {
+			return nil, fmt.Errorf("failed to discover etcd endpoints through SRV discovery: %v", srvErr)
+		}
+		etcdLocation = srvs.Endpoints
+	}
+
+	if len(etcdLocation) == 0 {
+		log.Warning("No etcd endpoints specified in etcdv3 API config")
+		return nil, errors.New("no etcd endpoints specified")
+	}
+
+	// Create the etcd client
+	// If Etcd Certificate and Key are provided inline through command line agrument,
+	// then the inline values take precedence over the ones in the config file.
+	// All the three parametes, Certificate, key and CA certificate are to be provided inline for processing.
+	var tls *tls.Config
+	var err error
+
+	haveInline := config.EtcdCert != "" || config.EtcdKey != "" || config.EtcdCACert != ""
+	haveFiles := config.EtcdCertFile != "" || config.EtcdKeyFile != "" || config.EtcdCACertFile != ""
+
+	if haveInline && haveFiles {
+		return nil, fmt.Errorf("Cannot mix inline certificate-key and certificate / key files")
+	}
+
+	if haveInline {
+		tlsInfo := &TlsInlineCertKey{
+			CACert: config.EtcdCACert,
+			Cert:   config.EtcdCert,
+			Key:    config.EtcdKey,
+		}
+		tls, err = tlsInfo.ClientConfigInlineCertKey()
+	} else {
+		tlsInfo := &transport.TLSInfo{
+			CAFile:   config.EtcdCACertFile,
+			CertFile: config.EtcdCertFile,
+			KeyFile:  config.EtcdKeyFile,
+		}
+		tls, err = tlsInfo.ClientConfig()
+	}
+
+	if err != nil {
+		return nil, fmt.Errorf("could not initialize etcdv3 client: %+v", err)
+	}
+
+	// Build the etcdv3 config.
+	cfg := clientv3.Config{
+		Endpoints:            etcdLocation,
+		TLS:                  tls,
+		DialTimeout:          clientTimeout,
+		DialKeepAliveTime:    keepaliveTime,
+		DialKeepAliveTimeout: keepaliveTimeout,
+	}
+
+	// Plumb through the username and password if both are configured.
+	if config.EtcdUsername != "" && config.EtcdPassword != "" {
+		cfg.Username = config.EtcdUsername
+		cfg.Password = config.EtcdPassword
+	}
+
+	client, err := clientv3.New(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return &etcdV3Client{etcdClient: client}, nil
+}
+
+// Create an entry in the datastore.  If the entry already exists, this will return
+// an ErrorResourceAlreadyExists error and the current entry.
+func (c *etcdV3Client) Create(ctx context.Context, d *model.KVPair) (*model.KVPair, error) {
+	logCxt := log.WithFields(log.Fields{"model-etcdKey": d.Key, "value": d.Value, "ttl": d.TTL, "rev": d.Revision})
+	logCxt.Debug("Processing Create request")
+
+	key, value, err := getKeyValueStrings(d)
+	if err != nil {
+		return nil, err
+	}
+	logCxt = logCxt.WithField("etcdv3-etcdKey", key)
+
+	putOpts, err := c.getTTLOption(ctx, d)
+	if err != nil {
+		return nil, err
+	}
+
+	// Checking for 0 version of the etcdKey, which means it doesn't exists yet,
+	// and if it does, get the current value.
+	logCxt.Debug("Performing etcdv3 transaction for Create request")
+	txnResp, err := c.etcdClient.Txn(ctx).If(
+		clientv3.Compare(clientv3.Version(key), "=", 0),
+	).Then(
+		clientv3.OpPut(key, value, putOpts...),
+	).Else(
+		clientv3.OpGet(key),
+	).Commit()
+	if err != nil {
+		logCxt.WithError(err).Warning("Create failed")
+		return nil, cerrors.ErrorDatastoreError{Err: err}
+	}
+
+	if !txnResp.Succeeded {
+		// The resource must already exist.  Extract the current newValue and
+		// return that if possible.
+		logCxt.Debug("Create transaction failed due to resource already existing")
+		var existing *model.KVPair
+		getResp := (*clientv3.GetResponse)(txnResp.Responses[0].GetResponseRange())
+		if len(getResp.Kvs) != 0 {
+			existing, _ = etcdToKVPair(d.Key, getResp.Kvs[0])
+		}
+		return existing, cerrors.ErrorResourceAlreadyExists{Identifier: d.Key}
+	}
+
+	v, err := model.ParseValue(d.Key, []byte(value))
+	if err != nil {
+		return nil, cerrors.ErrorPartialFailure{Err: fmt.Errorf("Unexpected error parsing stored datastore entry '%v': %+v", value, err)}
+	}
+	d.Value = v
+	d.Revision = strconv.FormatInt(txnResp.Header.Revision, 10)
+
+	return d, nil
+}
+
+// Update an entry in the datastore.  If the entry does not exist, this will return
+// an ErrorResourceDoesNotExist error.  The ResourceVersion must be specified, and if
+// incorrect will return a ErrorResourceUpdateConflict error and the current entry.
+func (c *etcdV3Client) Update(ctx context.Context, d *model.KVPair) (*model.KVPair, error) {
+	logCxt := log.WithFields(log.Fields{"model-etcdKey": d.Key, "value": d.Value, "ttl": d.TTL, "rev": d.Revision})
+	logCxt.Debug("Processing Update request")
+	key, value, err := getKeyValueStrings(d)
+	if err != nil {
+		return nil, err
+	}
+	logCxt = logCxt.WithField("etcdv3-etcdKey", key)
+
+	opts, err := c.getTTLOption(ctx, d)
+	if err != nil {
+		return nil, err
+	}
+
+	// ResourceVersion must be set for an Update.
+	rev, err := parseRevision(d.Revision)
+	if err != nil {
+		return nil, err
+	}
+	conds := []clientv3.Cmp{clientv3.Compare(clientv3.ModRevision(key), "=", rev)}
+
+	logCxt.Debug("Performing etcdv3 transaction for Update request")
+	txnResp, err := c.etcdClient.Txn(ctx).If(
+		conds...,
+	).Then(
+		clientv3.OpPut(key, value, opts...),
+	).Else(
+		clientv3.OpGet(key),
+	).Commit()
+
+	if err != nil {
+		logCxt.WithError(err).Warning("Update failed")
+		return nil, cerrors.ErrorDatastoreError{Err: err}
+	}
+
+	// Etcd V3 does not return a error when compare condition fails we must verify the
+	// response Succeeded field instead.  If the compare did not succeed then check for
+	// a successful get to return either an UpdateConflict or a ResourceDoesNotExist error.
+	if !txnResp.Succeeded {
+		getResp := (*clientv3.GetResponse)(txnResp.Responses[0].GetResponseRange())
+		if len(getResp.Kvs) == 0 {
+			logCxt.Debug("Update transaction failed due to resource not existing")
+			return nil, cerrors.ErrorResourceDoesNotExist{Identifier: d.Key}
+		}
+
+		logCxt.Debug("Update transaction failed due to resource update conflict")
+		existing, _ := etcdToKVPair(d.Key, getResp.Kvs[0])
+		return existing, cerrors.ErrorResourceUpdateConflict{Identifier: d.Key}
+	}
+
+	v, err := model.ParseValue(d.Key, []byte(value))
+	cerrors.PanicIfErrored(err, "Unexpected error parsing stored datastore entry: %v", value)
+	d.Value = v
+	d.Revision = strconv.FormatInt(txnResp.Header.Revision, 10)
+
+	return d, nil
+}
+
+//TODO Remove once we get rid of the v1 client.  Apply should no longer be supported
+// at least in it's current guise.  Apply will need to be handled further up the stack
+// by performing a Get/Create or Update to ensure we don't lose certain read-only Metadata.
+// It's possible that we will just perform that processing in the clients (e.g. calicoctl),
+// but that is to be decided.
+func (c *etcdV3Client) Apply(ctx context.Context, d *model.KVPair) (*model.KVPair, error) {
+	logCxt := log.WithFields(log.Fields{"etcdKey": d.Key, "value": d.Value, "ttl": d.TTL, "rev": d.Revision})
+	logCxt.Debug("Processing Apply request")
+	key, value, err := getKeyValueStrings(d)
+	if err != nil {
+		return nil, err
+	}
+
+	putOpts, err := c.getTTLOption(ctx, d)
+	if err != nil {
+		return nil, err
+	}
+
+	logCxt.Debug("Performing etcdv3 Put for Apply request")
+	resp, err := c.etcdClient.Put(ctx, key, value, putOpts...)
+	if err != nil {
+		logCxt.WithError(err).Warning("Apply failed")
+		return nil, cerrors.ErrorDatastoreError{Err: err}
+	}
+
+	v, err := model.ParseValue(d.Key, []byte(value))
+	cerrors.PanicIfErrored(err, "Unexpected error parsing stored datastore entry: %v", value)
+	d.Value = v
+	d.Revision = strconv.FormatInt(resp.Header.Revision, 10)
+
+	return d, nil
+}
+
+func (c *etcdV3Client) DeleteKVP(ctx context.Context, kvp *model.KVPair) (*model.KVPair, error) {
+	return c.Delete(ctx, kvp.Key, kvp.Revision)
+}
+
+// Delete an entry in the datastore.  This errors if the entry does not exists.
+func (c *etcdV3Client) Delete(ctx context.Context, k model.Key, revision string) (*model.KVPair, error) {
+	logCxt := log.WithFields(log.Fields{"model-etcdKey": k, "rev": revision})
+	logCxt.Debug("Processing Delete request")
+	key, err := model.KeyToDefaultDeletePath(k)
+	if err != nil {
+		return nil, err
+	}
+	logCxt = logCxt.WithField("etcdv3-etcdKey", key)
+
+	conds := []clientv3.Cmp{}
+	if len(revision) != 0 {
+		rev, err := parseRevision(revision)
+		if err != nil {
+			return nil, err
+		}
+		conds = append(conds, clientv3.Compare(clientv3.ModRevision(key), "=", rev))
+	}
+
+	// Perform the delete transaction - note that this is an exact delete, not a prefix delete.
+	logCxt.Debug("Performing etcdv3 transaction for Delete request")
+	txnResp, err := c.etcdClient.Txn(ctx).If(
+		conds...,
+	).Then(
+		clientv3.OpDelete(key, clientv3.WithPrevKV()),
+	).Else(
+		clientv3.OpGet(key),
+	).Commit()
+	if err != nil {
+		logCxt.WithError(err).Warning("Delete failed")
+		return nil, cerrors.ErrorDatastoreError{Err: err, Identifier: k}
+	}
+
+	// Transaction did not succeed - which means the ModifiedIndex check failed.  We can respond
+	// with the latest settings.
+	if !txnResp.Succeeded {
+		logCxt.Debug("Delete transaction failed due to resource update conflict")
+
+		getResp := txnResp.Responses[0].GetResponseRange()
+		if len(getResp.Kvs) == 0 {
+			logCxt.Debug("Delete transaction failed due to resource not existing")
+			return nil, cerrors.ErrorResourceDoesNotExist{Identifier: k}
+		}
+		latestValue, err := etcdToKVPair(k, getResp.Kvs[0])
+		if err != nil {
+			return nil, err
+		}
+		return latestValue, cerrors.ErrorResourceUpdateConflict{Identifier: k}
+	}
+
+	// The delete response should have succeeded since the Get response did.
+	delResp := txnResp.Responses[0].GetResponseDeleteRange()
+	if delResp.Deleted == 0 {
+		logCxt.Debug("Delete transaction failed due to resource not existing")
+		return nil, cerrors.ErrorResourceDoesNotExist{Identifier: k}
+	}
+
+	// Parse the deleted value.  Don't propagate the error in this case since the
+	// delete did succeed.
+	previousValue, _ := etcdToKVPair(k, delResp.PrevKvs[0])
+	return previousValue, nil
+}
+
+// Get an entry from the datastore.  This errors if the entry does not exist.
+func (c *etcdV3Client) Get(ctx context.Context, k model.Key, revision string) (*model.KVPair, error) {
+	logCxt := log.WithFields(log.Fields{"model-etcdKey": k, "rev": revision})
+	logCxt.Debug("Processing Get request")
+
+	key, err := model.KeyToDefaultPath(k)
+	if err != nil {
+		logCxt.Error("Unable to convert model.Key to an etcdv3 etcdKey")
+		return nil, err
+	}
+	logCxt = logCxt.WithField("etcdv3-etcdKey", key)
+
+	ops := []clientv3.OpOption{}
+	if len(revision) != 0 {
+		rev, err := parseRevision(revision)
+		if err != nil {
+			return nil, err
+		}
+		ops = append(ops, clientv3.WithRev(rev))
+	}
+
+	logCxt.Debug("Calling Get on etcdv3 client")
+	resp, err := c.etcdClient.Get(ctx, key, ops...)
+	if err != nil {
+		logCxt.WithError(err).Debug("Error returned from etcdv3 client")
+		return nil, cerrors.ErrorDatastoreError{Err: err}
+	}
+	if len(resp.Kvs) == 0 {
+		logCxt.Debug("No results returned from etcdv3 client")
+		return nil, cerrors.ErrorResourceDoesNotExist{Identifier: k}
+	}
+
+	return etcdToKVPair(k, resp.Kvs[0])
+}
+
+// List entries in the datastore.  This may return an empty list of there are
+// no entries matching the request in the ListInterface.
+func (c *etcdV3Client) List(ctx context.Context, l model.ListInterface, revision string) (*model.KVPairList, error) {
+	logCxt := log.WithFields(log.Fields{"list-interface": l, "rev": revision})
+	logCxt.Debug("Processing List request")
+
+	// To list entries, we enumerate from the common root based on the supplied IDs, and then filter the results.
+	key, ops := calculateListKeyAndOptions(logCxt, l)
+	logCxt = logCxt.WithField("etcdv3-etcdKey", key)
+
+	// We may also need to perform a get based on a particular revision.
+	if len(revision) != 0 {
+		rev, err := parseRevision(revision)
+		if err != nil {
+			return nil, err
+		}
+		ops = append(ops, clientv3.WithRev(rev))
+	}
+
+	logCxt.Debug("Calling Get on etcdv3 client")
+	resp, err := c.etcdClient.Get(ctx, key, ops...)
+	if err != nil {
+		logCxt.WithError(err).Debug("Error returned from etcdv3 client")
+		return nil, cerrors.ErrorDatastoreError{Err: err}
+	}
+	logCxt.WithField("numResults", len(resp.Kvs)).Debug("Processing response from etcdv3")
+
+	// Filter/process the results.
+	list := []*model.KVPair{}
+	for _, p := range resp.Kvs {
+		if kv := convertListResponse(p, l); kv != nil {
+			list = append(list, kv)
+		}
+	}
+
+	return &model.KVPairList{
+		KVPairs:  list,
+		Revision: strconv.FormatInt(resp.Header.Revision, 10),
+	}, nil
+}
+
+func calculateListKeyAndOptions(logCxt *log.Entry, l model.ListInterface) (string, []clientv3.OpOption) {
+	// -  If the final name segment of the name is itself a prefix, then just perform a prefix Get
+	//    using the constructed key.
+	// -  If the etcdKey is actually fully qualified, then perform an exact Get using the constructed
+	//    key.
+	// -  If the etcdKey is not fully qualified then it is a path prefix but the last segment is complete.
+	//    Append a terminating "/" and perform a prefix Get.  The terminating / for a prefix Get ensures
+	//    for a prefix of "/a" we only return "child entries" of "/a" such as "/a/x" and not siblings
+	//    such as "/ab".
+	key := model.ListOptionsToDefaultPathRoot(l)
+	var ops []clientv3.OpOption
+	if model.IsListOptionsLastSegmentPrefix(l) {
+		// The last segment is a prefix, perform a prefix Get without adding a segment
+		// delimiter.
+		logCxt.Debug("List options is a name prefix, don't add a / to the path")
+		ops = append(ops, clientv3.WithPrefix())
+	} else if !model.ListOptionsIsFullyQualified(l) {
+		// The etcdKey not a fully qualified etcdKey - it must be a prefix.
+		logCxt.Debug("List options is a parent prefix, ensure path ends in /")
+		if !strings.HasSuffix(key, "/") {
+			logCxt.Debug("Adding / to path")
+			key += "/"
+		}
+		ops = append(ops, clientv3.WithPrefix())
+	}
+
+	return key, ops
+}
+
+// EnsureInitialized makes sure that the etcd data is initialized for use by
+// Calico.
+func (c *etcdV3Client) EnsureInitialized() error {
+	//TODO - still need to worry about ready flag.
+	return nil
+}
+
+// Clean removes all of the Calico data from the datastore.
+func (c *etcdV3Client) Clean() error {
+	log.Warning("Cleaning etcdv3 datastore of all Calico data")
+	_, err := c.etcdClient.Txn(context.Background()).If().Then(
+		clientv3.OpDelete("/calico/", clientv3.WithPrefix()),
+	).Commit()
+
+	if err != nil {
+		return cerrors.ErrorDatastoreError{Err: err}
+	}
+	return nil
+}
+
+// IsClean() returns true if there are no /calico/ prefixed entries in the
+// datastore.  This is not part of the exposed API, but is public to allow
+// direct consumers of the backend API to access this.
+func (c *etcdV3Client) IsClean() (bool, error) {
+	log.Debug("Calling Get on etcdv3 client")
+	resp, err := c.etcdClient.Get(context.Background(), "/calico/", clientv3.WithPrefix())
+	if err != nil {
+		log.WithError(err).Debug("Error returned from etcdv3 client")
+		return false, cerrors.ErrorDatastoreError{Err: err}
+	}
+
+	// The datastore is clean if no results were enumerated.
+	return len(resp.Kvs) == 0, nil
+}
+
+// getTTLOption returns a OpOption slice containing a Lease granted for the TTL.
+func (c *etcdV3Client) getTTLOption(ctx context.Context, d *model.KVPair) ([]clientv3.OpOption, error) {
+	putOpts := []clientv3.OpOption{}
+
+	if d.TTL != 0 {
+		resp, err := c.etcdClient.Lease.Grant(ctx, int64(d.TTL.Seconds()))
+		if err != nil {
+			log.WithError(err).Error("Failed to grant a lease")
+			return nil, cerrors.ErrorDatastoreError{Err: err}
+		}
+
+		putOpts = append(putOpts, clientv3.WithLease(resp.ID))
+	}
+
+	return putOpts, nil
+}
+
+// getKeyValueStrings returns the etcdv3 etcdKey and serialized value calculated from the
+// KVPair.
+func getKeyValueStrings(d *model.KVPair) (string, string, error) {
+	logCxt := log.WithFields(log.Fields{"model-etcdKey": d.Key, "value": d.Value})
+	key, err := model.KeyToDefaultPath(d.Key)
+	if err != nil {
+		logCxt.WithError(err).Error("Failed to convert model-etcdKey to etcdv3 etcdKey")
+		return "", "", cerrors.ErrorDatastoreError{
+			Err:        err,
+			Identifier: d.Key,
+		}
+	}
+	bytes, err := model.SerializeValue(d)
+	if err != nil {
+		logCxt.WithError(err).Error("Failed to serialize value")
+		return "", "", cerrors.ErrorDatastoreError{
+			Err:        err,
+			Identifier: d.Key,
+		}
+	}
+
+	return key, string(bytes), nil
+}
+
+// parseRevision parses the model.KVPair revision string and converts to the
+// equivalent etcdv3 int64 value.
+func parseRevision(revs string) (int64, error) {
+	rev, err := strconv.ParseInt(revs, 10, 64)
+	if err != nil {
+		log.WithField("Revision", revs).Debug("Unable to parse Revision")
+		return 0, cerrors.ErrorValidation{
+			ErroredFields: []cerrors.ErroredField{
+				{
+					Name:  "ResourceVersion",
+					Value: revs,
+				},
+			},
+		}
+	}
+	return rev, nil
+}

vendor/github.com/projectcalico/libcalico-go/lib/backend/etcdv3/inline_cert_key.go

@@ -0,0 +1,115 @@
+// Copyright (c) 2019 Tigera, Inc. All rights reserved.
+
+// This code has been based on code from etcd repository
+// to provide support for inline certificates and keys for calicoctl.
+// Below are the github links for the files from which the code has been borrowed.
+
+// Copyright 2015 The etcd Authors
+//     https://github.com/etcd-io/etcd/blob/release-3.3/pkg/transport/listener.go
+
+// Copyright 2016 The etcd Authors
+//     https://github.com/etcd-io/etcd/blob/release-3.3/pkg/tlsutil/tlsutil.go
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package etcdv3
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+)
+
+// This struct is used to store the inline cert, key and CA cert data
+type TlsInlineCertKey struct {
+	Cert   string
+	Key    string
+	CACert string
+}
+
+// ClientConfigInlineCertKey() returns a pointer to tls.Config struct object with certificate data
+// for client creation using only the inline certificate, key and CA certificate data.
+func (info TlsInlineCertKey) ClientConfigInlineCertKey() (*tls.Config, error) {
+	var cfg *tls.Config
+	var err error
+
+	if info.Cert == "" || info.Key == "" {
+		return nil, fmt.Errorf("Certificate and Key must both be present inline.")
+	}
+
+	if info.Cert != "" && info.Key != "" {
+		cfg, err = info.baseCertConfig()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if info.CACert != "" {
+		cfg.RootCAs, err = newCertPool(info.CACert)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return cfg, nil
+}
+
+// baseCertConfig() populates tls struct with certificate data
+func (info TlsInlineCertKey) baseCertConfig() (*tls.Config, error) {
+	_, err := newCert([]byte(info.Cert), []byte(info.Key))
+	if err != nil {
+		return nil, err
+	}
+
+	cfg := &tls.Config{
+		MinVersion: tls.VersionTLS12,
+	}
+
+	cfg.GetCertificate = func(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) {
+		return newCert([]byte(info.Cert), []byte(info.Key))
+	}
+	cfg.GetClientCertificate = func(unused *tls.CertificateRequestInfo) (*tls.Certificate, error) {
+		return newCert([]byte(info.Cert), []byte(info.Key))
+	}
+	return cfg, nil
+}
+
+// newCertPool() creates the certificate pool from the CA certificates provided
+func newCertPool(caCert string) (*x509.CertPool, error) {
+	certPool := x509.NewCertPool()
+	if caCert == "" {
+		return nil, nil
+	}
+	var block *pem.Block
+	certByte := []byte(caCert)
+	block, certByte = pem.Decode(certByte)
+	if block == nil {
+		return nil, fmt.Errorf("Cannot decode PEM block containing certificate")
+	}
+	cert, err := x509.ParseCertificate(block.Bytes)
+	if err != nil {
+		return nil, err
+	}
+	certPool.AddCert(cert)
+	return certPool, nil
+}
+
+// newCert() generates TLS cert by using the given cert and key values.
+func newCert(cert, key []byte) (*tls.Certificate, error) {
+	tlsCert, err := tls.X509KeyPair(cert, key)
+	if err != nil {
+		return nil, err
+	}
+	return &tlsCert, nil
+}

vendor/github.com/projectcalico/libcalico-go/lib/backend/etcdv3/watcher.go

@@ -0,0 +1,212 @@
+// Copyright (c) 2016-2019 Tigera, Inc. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package etcdv3
+
+import (
+	"context"
+	goerrors "errors"
+	"strconv"
+	"sync/atomic"
+
+	"github.com/coreos/etcd/clientv3"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/projectcalico/libcalico-go/lib/backend/api"
+	"github.com/projectcalico/libcalico-go/lib/backend/model"
+	"github.com/projectcalico/libcalico-go/lib/errors"
+)
+
+const (
+	resultsBufSize = 100
+)
+
+// Watch entries in the datastore matching the resources specified by the ListInterface.
+func (c *etcdV3Client) Watch(cxt context.Context, l model.ListInterface, revision string) (api.WatchInterface, error) {
+	var rev int64
+	if len(revision) != 0 {
+		var err error
+		rev, err = strconv.ParseInt(revision, 10, 64)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	wc := &watcher{
+		client:     c,
+		list:       l,
+		initialRev: rev,
+		resultChan: make(chan api.WatchEvent, resultsBufSize),
+	}
+	wc.ctx, wc.cancel = context.WithCancel(cxt)
+	go wc.watchLoop()
+	return wc, nil
+}
+
+// watcher implements watch.Interface.
+type watcher struct {
+	client     *etcdV3Client
+	initialRev int64
+	ctx        context.Context
+	cancel     context.CancelFunc
+	resultChan chan api.WatchEvent
+	list       model.ListInterface
+	terminated uint32
+}
+
+// Stop stops the watcher and releases associated resources.
+// This calls through to the context cancel function.
+func (wc *watcher) Stop() {
+	wc.cancel()
+}
+
+// ResultChan returns a channel used to receive WatchEvents.
+func (wc *watcher) ResultChan() <-chan api.WatchEvent {
+	return wc.resultChan
+}
+
+// HasTerminated returns true when the watcher has completed termination processing.
+func (wc *watcher) HasTerminated() bool {
+	return atomic.LoadUint32(&wc.terminated) != 0
+}
+
+// watchLoop starts a watch on the required path prefix and sends a stream of
+// event updates for internal processing.
+func (wc *watcher) watchLoop() {
+	// When this loop exits, make sure we terminate the watcher resources.
+	defer wc.terminateWatcher()
+
+	log.Debug("Starting watcher.watchLoop")
+	if wc.initialRev == 0 {
+		// No initial revision supplied, so perform a list of current configuration
+		// which will also get the current revision we will start our watch from.
+		if err := wc.listCurrent(); err != nil {
+			log.Errorf("failed to list current with latest state: %v", err)
+			wc.sendError(err, true)
+			return
+		}
+	}
+
+	// If we are not watching a specific resource then this is a prefix watch.
+	logCxt := log.WithField("list", wc.list)
+	key, opts := calculateListKeyAndOptions(logCxt, wc.list)
+	opts = append(opts, clientv3.WithRev(wc.initialRev+1), clientv3.WithPrevKV())
+	logCxt = logCxt.WithFields(log.Fields{
+		"etcdv3-etcdKey": key,
+		"rev":            wc.initialRev,
+	})
+	logCxt.Debug("Starting etcdv3 watch")
+	wch := wc.client.etcdClient.Watch(wc.ctx, key, opts...)
+	for wres := range wch {
+		if wres.Err() != nil {
+			// A watch channel error is a terminating event, so exit the loop.
+			err := wres.Err()
+			log.WithError(err).Error("Watch channel error")
+			wc.sendError(err, true)
+			return
+		}
+		for _, e := range wres.Events {
+			// Convert the etcdv3 event to the equivalent Watcher event.  An error
+			// parsing the event is returned as an error, but don't exit the watcher as
+			// restarting the watcher is unlikely to fix the conversion error.
+			if ae, err := convertWatchEvent(e, wc.list); ae != nil {
+				wc.sendEvent(ae)
+			} else if err != nil {
+				wc.sendError(err, false)
+			}
+		}
+	}
+
+	// If we exit the loop, it means the watcher has closed for some reason.
+	// Bubble this up as a watch termination error.
+	log.Warn("etcdv3 watch channel closed")
+	wc.sendError(goerrors.New("etcdv3 watch channel closed"), true)
+}
+
+// listCurrent retrieves the existing entries and sends an event for each listed
+func (wc *watcher) listCurrent() error {
+	log.Info("Performing initial list with no revision")
+	list, err := wc.client.List(wc.ctx, wc.list, "")
+	if err != nil {
+		return err
+	}
+
+	wc.initialRev, err = strconv.ParseInt(list.Revision, 10, 64)
+	if err != nil {
+		log.WithError(err).Error("List returned revision that could not be parsed")
+		return err
+	}
+
+	// We are sending an initial sync of entries to the watcher to provide current
+	// state.  To the perspective of the watcher, these are added entries, so set the
+	// event type to WatchAdded.
+	log.WithField("NumEntries", len(list.KVPairs)).Debug("Sending create events for each existing entry")
+	for _, kv := range list.KVPairs {
+		wc.sendEvent(&api.WatchEvent{
+			Type: api.WatchAdded,
+			New:  kv,
+		})
+	}
+
+	return nil
+}
+
+// terminateWatcher terminates the resources associated with the watcher.
+func (wc *watcher) terminateWatcher() {
+	log.Debug("Terminating etcdv3 watcher")
+	// Cancel the context - which will cancel the etcd Watch, this may have already been
+	// cancelled through an explicit Stop, but it is fine to cancel multiple times.
+	wc.cancel()
+
+	// Close the results channel.
+	close(wc.resultChan)
+
+	// Increment the terminated counter using a goroutine safe operation.
+	atomic.AddUint32(&wc.terminated, 1)
+}
+
+// sendError packages up the error as an event and sends it in the results channel.
+func (wc *watcher) sendError(err error, terminating bool) {
+	// The response from etcd commands may include a context.Canceled error if the context
+	// was cancelled before completion.  Since with our Watcher we don't include that as
+	// an error type skip over the Canceled error, the error processing in the main
+	// watch thread will terminate the watcher.
+	if err == context.Canceled {
+		return
+	}
+
+	// If this is a terminating error, wrap the error up in an errors.ErrorWatchTerminated
+	// error type.
+	if terminating {
+		err = errors.ErrorWatchTerminated{Err: err}
+	}
+
+	// Wrap the error up in a WatchEvent and use sendEvent to send it.
+	errEvent := &api.WatchEvent{
+		Type:  api.WatchError,
+		Error: err,
+	}
+	wc.sendEvent(errEvent)
+}
+
+// sendEvent sends an event in the results channel.
+func (wc *watcher) sendEvent(e *api.WatchEvent) {
+	if len(wc.resultChan) == resultsBufSize {
+		log.Warningf("Watch events backing up: %d events", resultsBufSize)
+	}
+	select {
+	case wc.resultChan <- *e:
+	case <-wc.ctx.Done():
+	}
+}