This PR does the following things:

1. add new registry api under resources.kubesphere.io/v1alpha3
2. deprecate registry api v1alpha2

Registry API v1alpha2 uses docker client to authenticate image registry
secret, which depends on docker.sock. We used to mount host
`/var/run/docker.sock` to deployment. It will prevent us imgrating to
containerd since no `docker.sock` exists. Registry API v1alpha3 comes to
rescure, it wraps library go-containerregistry and compatible with
docker registry, Harbor etc.
This commit is contained in:
Jeff
2021-08-23 19:56:28 +08:00
parent c740fef5b4
commit 3d2fd1b538
88 changed files with 10002 additions and 16 deletions

View File

@@ -131,12 +131,14 @@ func AddToContainer(c *restful.Container, k8sClient kubernetes.Interface, factor
To(handler.handleGetNamespaceQuotas))
webservice.Route(webservice.POST("registry/verify").
Deprecate().
To(handler.handleVerifyRegistryCredential).
Metadata(restfulspec.KeyOpenAPITags, []string{constants.RegistryTag}).
Doc("verify if a user has access to the docker registry").
Reads(api.RegistryCredential{}).
Returns(http.StatusOK, api.StatusOK, errors.Error{}))
webservice.Route(webservice.GET("/registry/blob").
Deprecate().
To(handler.handleGetRegistryEntry).
Param(webservice.QueryParameter("image", "query image, condition for filtering.").
Required(true).

View File

@@ -17,14 +17,19 @@ limitations under the License.
package v1alpha3
import (
"fmt"
"net/http"
"strings"
"github.com/emicklei/go-restful"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/klog"
"kubesphere.io/kubesphere/pkg/api"
"kubesphere.io/kubesphere/pkg/apiserver/query"
"kubesphere.io/kubesphere/pkg/models/components"
v2 "kubesphere.io/kubesphere/pkg/models/registries/v2"
"kubesphere.io/kubesphere/pkg/models/resources/v1alpha2"
resourcev1alpha2 "kubesphere.io/kubesphere/pkg/models/resources/v1alpha2/resource"
resourcev1alpha3 "kubesphere.io/kubesphere/pkg/models/resources/v1alpha3/resource"
@@ -35,6 +40,7 @@ type Handler struct {
resourceGetterV1alpha3 *resourcev1alpha3.ResourceGetter
resourcesGetterV1alpha2 *resourcev1alpha2.ResourceGetter
componentsGetter components.ComponentsGetter
registryHelper v2.RegistryHelper
}
func New(resourceGetterV1alpha3 *resourcev1alpha3.ResourceGetter, resourcesGetterV1alpha2 *resourcev1alpha2.ResourceGetter, componentsGetter components.ComponentsGetter) *Handler {
@@ -42,6 +48,7 @@ func New(resourceGetterV1alpha3 *resourcev1alpha3.ResourceGetter, resourcesGette
resourceGetterV1alpha3: resourceGetterV1alpha3,
resourcesGetterV1alpha2: resourcesGetterV1alpha2,
componentsGetter: componentsGetter,
registryHelper: v2.NewRegistryHelper(),
}
}
@@ -203,3 +210,86 @@ func (h *Handler) handleGetComponents(request *restful.Request, response *restfu
response.WriteEntity(result)
}
// handleVerifyImageRepositorySecret verifies image secret against registry, it takes k8s.io/api/core/v1/types.Secret
// as input, and authenticate registry with credential specified. Returns http.StatusOK if authenticate successfully,
// returns http.StatusUnauthorized if failed.
func (h *Handler) handleVerifyImageRepositorySecret(request *restful.Request, response *restful.Response) {
secret := &v1.Secret{}
err := request.ReadEntity(secret)
if err != nil {
api.HandleBadRequest(response, request, err)
}
ok, err := h.registryHelper.Auth(secret)
if !ok {
klog.Error(err)
api.HandleUnauthorized(response, request, err)
} else {
response.WriteHeaderAndJson(http.StatusOK, secret, restful.MIME_JSON)
}
}
// handleGetImageConfig fetches container image spec described in https://github.com/opencontainers/image-spec/blob/main/manifest.md
func (h *Handler) handleGetImageConfig(request *restful.Request, response *restful.Response) {
secretName := request.QueryParameter("secret")
namespace := request.PathParameter("namespace")
image := request.QueryParameter("image")
var secret *v1.Secret
// empty secret means anoymous fetching
if len(secretName) != 0 {
object, err := h.resourceGetterV1alpha3.Get("secrets", namespace, secretName)
if errors.IsNotFound(err) {
api.HandleNotFound(response, request, err)
}
secret = object.(*v1.Secret)
}
config, err := h.registryHelper.Config(secret, image)
if err != nil {
canonicalizeRegistryError(request, response, err)
return
}
response.WriteHeaderAndJson(http.StatusOK, config, restful.MIME_JSON)
}
// handleGetRepositoryTags fetchs all tags of given repository, no paging.
func (h *Handler) handleGetRepositoryTags(request *restful.Request, response *restful.Response) {
secretName := request.QueryParameter("secret")
namespace := request.PathParameter("namespace")
repository := request.QueryParameter("repository")
var secret *v1.Secret
if len(repository) == 0 {
api.HandleBadRequest(response, request, fmt.Errorf("empty repository name"))
return
}
if len(secretName) != 0 {
object, err := h.resourceGetterV1alpha3.Get("secrets", namespace, secretName)
if errors.IsNotFound(err) {
api.HandleNotFound(response, request, err)
}
secret = object.(*v1.Secret)
}
tags, err := h.registryHelper.ListRepositoryTags(secret, repository)
if err != nil {
canonicalizeRegistryError(request, response, err)
return
}
response.WriteHeaderAndJson(http.StatusOK, tags, restful.MIME_JSON)
}
func canonicalizeRegistryError(request *restful.Request, response *restful.Response, err error) {
if strings.Contains(err.Error(), "Unauthorized") {
api.HandleUnauthorized(response, request, err)
} else if strings.Contains(err.Error(), "MANIFEST_UNKNOWN") {
api.HandleNotFound(response, request, err)
} else {
api.HandleBadRequest(response, request, err)
}
}

View File

@@ -19,6 +19,7 @@ package v1alpha3
import (
"github.com/emicklei/go-restful"
restfulspec "github.com/emicklei/go-restful-openapi"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"sigs.k8s.io/controller-runtime/pkg/cache"
@@ -28,6 +29,7 @@ import (
"kubesphere.io/kubesphere/pkg/apiserver/runtime"
"kubesphere.io/kubesphere/pkg/informers"
"kubesphere.io/kubesphere/pkg/models/components"
v2 "kubesphere.io/kubesphere/pkg/models/registries/v2"
resourcev1alpha2 "kubesphere.io/kubesphere/pkg/models/resources/v1alpha2/resource"
resourcev1alpha3 "kubesphere.io/kubesphere/pkg/models/resources/v1alpha3/resource"
@@ -114,6 +116,34 @@ func AddToContainer(c *restful.Container, informerFactory informers.InformerFact
Doc("Get the health status of system components.").
Returns(http.StatusOK, ok, v1alpha2.HealthStatus{}))
webservice.Route(webservice.POST("/namespaces/{namespace}/registrysecrets/{secret}").
To(handler.handleVerifyImageRepositorySecret).
Param(webservice.PathParameter("namespace", "Namespace of the image repository secret to create.").Required(true)).
Param(webservice.PathParameter("secret", "Secret name of the image repository credential to create").Required(true)).
Param(webservice.BodyParameter("secretSpec", "Secret specification, definition in k8s.io/api/core/v1/types.Secret")).
Reads(v1.Secret{}).
Metadata(restfulspec.KeyOpenAPITags, []string{tagNamespacedResource}).
Doc("Verify image repostiry secret.").
Returns(http.StatusOK, ok, v1.Secret{}))
webservice.Route(webservice.GET("/namespaces/{namespace}/imageconfig").
To(handler.handleGetImageConfig).
Param(webservice.PathParameter("namespace", "Namespace of the image repository secret.").Required(true)).
Param(webservice.QueryParameter("secret", "Secret name of the image repository credential, left empty means anonymous fetch.").Required(false)).
Param(webservice.QueryParameter("image", "Image name to query, e.g. kubesphere/ks-apiserver:v3.1.1").Required(true)).
Metadata(restfulspec.KeyOpenAPITags, []string{tagNamespacedResource}).
Doc("Get image config.").
Returns(http.StatusOK, ok, v2.ImageConfig{}))
webservice.Route(webservice.GET("/namespaces/{namespace}/repositorytags").
To(handler.handleGetRepositoryTags).
Param(webservice.PathParameter("namespace", "Namespace of the image repository secret.").Required(true)).
Param(webservice.QueryParameter("repository", "Repository to query, e.g. calico/cni.").Required(true)).
Param(webservice.QueryParameter("secret", "Secret name of the image repository credential, left empty means anonymous fetch.").Required(false)).
Metadata(restfulspec.KeyOpenAPITags, []string{tagNamespacedResource}).
Doc("List repository tags, this is an experimental API, use it by your own caution.").
Returns(http.StatusOK, ok, v2.RepositoryTags{}))
c.Add(webservice)
return nil

View File

@@ -0,0 +1,83 @@
package v2
import (
"context"
"net/http"
"github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/remote"
)
const (
// DefaultRegistry is the registry name that will be used if no registry
// provided and the default is not overridden.
DefaultRegistry = "index.docker.io"
defaultRegistryAlias = "docker.io"
// DefaultTag is the tag name that will be used if no tag provided and the
// default is not overridden.
DefaultTag = "latest"
)
type options struct {
name []name.Option
remote []remote.Option
platform *v1.Platform
}
func makeOptions(opts ...Option) options {
opt := options{
remote: []remote.Option{
remote.WithAuth(authn.Anonymous),
},
}
for _, o := range opts {
o(&opt)
}
return opt
}
// Option is a functional option
type Option func(*options)
// WithTransport is a functional option for overriding the default transport
// for remote operations.
func WithTransport(t http.RoundTripper) Option {
return func(o *options) {
o.remote = append(o.remote, remote.WithTransport(t))
}
}
// Insecure is an Option that allows image references to be fetched without TLS.
func Insecure(o *options) {
o.name = append(o.name, name.Insecure)
}
// WithAuth is a functional option for overriding the default authenticator
// for remote operations.
//
func WithAuth(auth authn.Authenticator) Option {
return func(o *options) {
// Replace the default keychain at position 0.
o.remote[0] = remote.WithAuth(auth)
}
}
// WithContext is a functional option for setting the context.
func WithContext(ctx context.Context) Option {
return func(o *options) {
o.remote = append(o.remote, remote.WithContext(ctx))
}
}
// WithPlatform is an Option to specify the platform.
func WithPlatform(platform *v1.Platform) Option {
return func(o *options) {
if platform != nil {
o.remote = append(o.remote, remote.WithPlatform(*platform))
}
o.platform = platform
}
}

View File

@@ -0,0 +1,71 @@
package v2
import (
"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/remote"
)
type Registryer interface {
// list repository tags
ListRepositoryTags(image string) (RepositoryTags, error)
// get image config
Config(image string) (*v1.ConfigFile, error)
}
type registryer struct {
opts options
}
func NewRegistryer(opts ...Option) *registryer {
return &registryer{
opts: makeOptions(opts...),
}
}
func (r *registryer) ListRepositoryTags(src string) (RepositoryTags, error) {
repo, err := name.NewRepository(src, r.opts.name...)
if err != nil {
return RepositoryTags{}, err
}
tags, err := remote.List(repo, r.opts.remote...)
if err != nil {
return RepositoryTags{}, err
}
return RepositoryTags{
Registry: repo.RegistryStr(),
Repository: repo.RepositoryStr(),
Tags: tags,
}, nil
}
func (r *registryer) Config(image string) (*v1.ConfigFile, error) {
img, _, err := r.getImage(image)
if err != nil {
return nil, err
}
configFile, err := img.ConfigFile()
if err != nil {
return nil, err
}
return configFile, nil
}
func (r *registryer) getImage(reference string) (v1.Image, name.Reference, error) {
ref, err := name.ParseReference(reference, r.opts.name...)
if err != nil {
return nil, nil, err
}
img, err := remote.Image(ref, r.opts.remote...)
if err != nil {
return nil, nil, err
}
return img, ref, nil
}

View File

@@ -0,0 +1,155 @@
package v2
import (
"testing"
corev1 "k8s.io/api/core/v1"
"github.com/google/go-cmp/cmp"
v1 "github.com/google/go-containerregistry/pkg/v1"
)
func TestRegistryerConfig(t *testing.T) {
testCases := []struct {
name string
secret *corev1.Secret
image string
configFile *v1.ConfigFile
expectErr bool
}{
{
name: "Should fetch image config with public registry",
secret: nil,
image: "kubesphere/ks-apiserver:v3.1.0",
expectErr: false,
configFile: &v1.ConfigFile{
Config: v1.Config{
Image: "sha256:a3006bafb7702494227f7fa69720e65e1324b7bf8ece8ac03d9fe1d0134e7341",
},
},
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
secretAuthenticator, err := NewSecretAuthenticator(testCase.secret)
if err != nil {
t.Error(err)
}
// fix platform to linux/amd64 so we could compare config
platform := &v1.Platform{
OS: "linux",
Architecture: "amd64",
}
options := secretAuthenticator.Options()
options = append(options, WithPlatform(platform))
registryer := NewRegistryer(options...)
config, err := registryer.Config(testCase.image)
if testCase.expectErr && err == nil {
t.Errorf("expected error, but got nil")
}
if !testCase.expectErr && err != nil {
t.Error(err)
}
if diff := cmp.Diff(testCase.configFile.Config.Image, config.Config.Image); len(diff) != 0 {
t.Errorf("expected %v, but got %v", testCase.configFile.Config.Image, config.Config.Image)
}
})
}
}
func TestRegistryerListRepoTags(t *testing.T) {
testCases := []struct {
name string
secret *corev1.Secret
image string
repositoryTags RepositoryTags
expectErr bool
}{
{
name: "Should fetch config with public registry",
secret: nil,
image: "kubesphere/ks-apiserver",
expectErr: false,
repositoryTags: RepositoryTags{
Registry: "index.docker.io",
Tags: []string{
"v3.1.1",
"v3.1.0",
"latest",
},
},
},
{
name: "Should fetch config from public registry with credential",
secret: buildSecret("dockerhub.qingcloud.com", "guest", "guest", false),
image: "dockerhub.qingcloud.com/calico/cni",
expectErr: false,
repositoryTags: RepositoryTags{
Registry: "dockerhub.qingcloud.com",
Tags: []string{
"v1.11.4",
"v3.1.3",
"v3.3.2",
"v3.3.3",
"v3.3.6",
"v3.7.3",
"v3.8.4",
},
},
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
secretAuthenticator, err := NewSecretAuthenticator(testCase.secret)
if err != nil {
t.Error(err)
}
// fix platform to linux/amd64 so we could compare config
platform := &v1.Platform{
OS: "linux",
Architecture: "amd64",
}
options := secretAuthenticator.Options()
options = append(options, WithPlatform(platform))
registryer := NewRegistryer(options...)
tags, err := registryer.ListRepositoryTags(testCase.image)
if testCase.expectErr && err == nil {
t.Errorf("expected error, but got nil")
}
if !testCase.expectErr && err != nil {
t.Error(err)
}
cotains := func(s []string, e string) bool {
for _, a := range s {
if a == e {
return true
}
}
return false
}
for _, tag := range testCase.repositoryTags.Tags {
if !cotains(tags.Tags, tag) {
t.Errorf("no expected tag %s in result %v", tag, tags.Tags)
}
}
})
}
}

View File

@@ -0,0 +1,52 @@
package v2
import (
corev1 "k8s.io/api/core/v1"
)
type RegistryHelper interface {
// check if secret has correct credential to authenticate with remote registry
Auth(secret *corev1.Secret) (bool, error)
// fetch OCI Image Manifest, specification described as in https://github.com/opencontainers/image-spec/blob/main/manifest.md
Config(secret *corev1.Secret, image string) (*ImageConfig, error)
// list all tags of given repository, experimental
ListRepositoryTags(secret *corev1.Secret, repository string) (RepositoryTags, error)
}
type registryHelper struct{}
func NewRegistryHelper() RegistryHelper {
return &registryHelper{}
}
func (r *registryHelper) Auth(secret *corev1.Secret) (bool, error) {
secretAuth, err := NewSecretAuthenticator(secret)
if err != nil {
return false, err
}
return secretAuth.Auth()
}
func (r *registryHelper) Config(secret *corev1.Secret, image string) (*ImageConfig, error) {
secretAuth, err := NewSecretAuthenticator(secret)
if err != nil {
return nil, err
}
registryer := NewRegistryer(secretAuth.Options()...)
config, err := registryer.Config(image)
return &ImageConfig{ConfigFile: config}, err
}
func (r *registryHelper) ListRepositoryTags(secret *corev1.Secret, image string) (RepositoryTags, error) {
secretAuth, err := NewSecretAuthenticator(secret)
if err != nil {
return RepositoryTags{}, err
}
registryer := NewRegistryer(secretAuth.Options()...)
return registryer.ListRepositoryTags(image)
}

View File

@@ -0,0 +1,123 @@
package v2
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/url"
"github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/name"
"github.com/google/go-containerregistry/pkg/v1/remote/transport"
v1 "k8s.io/api/core/v1"
)
const (
forceInsecure = "secret.kubesphere.io/force-insecure"
)
type SecretAuthenticator interface {
Options() []Option
Auth() (bool, error)
Authorization() (*authn.AuthConfig, error)
}
type secretAuthenticator struct {
auths DockerConfig
insecure bool // force using insecure when talk to the remote registry, even registry address starts with https
}
func NewSecretAuthenticator(secret *v1.Secret) (SecretAuthenticator, error) {
if secret == nil {
return &secretAuthenticator{}, nil
}
sa := &secretAuthenticator{
insecure: false,
}
if secret.Type != v1.SecretTypeDockerConfigJson {
return nil, fmt.Errorf("expected secret type: %s, got: %s", v1.SecretTypeDockerConfigJson, secret.Type)
}
// force insecure if secret has annotation forceInsecure
if val, ok := secret.Annotations[forceInsecure]; ok && val == "true" {
sa.insecure = true
}
configJson, ok := secret.Data[v1.DockerConfigJsonKey]
if !ok {
return nil, fmt.Errorf("expected key %s in data, found none", v1.DockerConfigJsonKey)
}
dockerConfigJSON := DockerConfigJSON{}
if err := json.Unmarshal(configJson, &dockerConfigJSON); err != nil {
return nil, err
}
if len(dockerConfigJSON.Auths) == 0 {
return nil, fmt.Errorf("not found valid auth in secret, %v", dockerConfigJSON)
}
sa.auths = dockerConfigJSON.Auths
return sa, nil
}
func (s *secretAuthenticator) Authorization() (*authn.AuthConfig, error) {
for _, v := range s.auths {
return &authn.AuthConfig{
Username: v.Username,
Password: v.Password,
Auth: v.Auth,
}, nil
}
return &authn.AuthConfig{}, nil
}
func (s *secretAuthenticator) Auth() (bool, error) {
for k := range s.auths {
return s.AuthRegistry(k)
}
return false, fmt.Errorf("no registry found in secret")
}
func (s *secretAuthenticator) AuthRegistry(reg string) (bool, error) {
url, err := url.Parse(reg) // in case reg is unformatted like http://docker.index.io
if err != nil {
return false, err
}
options := make([]name.Option, 0)
if url.Scheme == "http" || s.insecure {
options = append(options, name.Insecure)
}
registry, err := name.NewRegistry(url.Host, options...)
if err != nil {
return false, err
}
ctx := context.TODO()
_, err = transport.NewWithContext(ctx, registry, s, http.DefaultTransport, []string{})
if err != nil {
return false, err
}
return true, nil
}
func (s *secretAuthenticator) Options() []Option {
options := make([]Option, 0)
options = append(options, WithAuth(s))
if s.insecure {
options = append(options, Insecure)
}
return options
}

View File

@@ -0,0 +1,111 @@
package v2
import (
"fmt"
"testing"
"encoding/base64"
"github.com/google/go-cmp/cmp"
"github.com/google/go-containerregistry/pkg/authn"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func buildSecret(registry, username, password string, insecure bool) *v1.Secret {
auth := fmt.Sprintf("%s:%s", username, password)
authString := fmt.Sprintf("{\"auths\":{\"%s\":{\"username\":\"%s\",\"password\":\"%s\",\"email\":\"\",\"auth\":\"%s\"}}}", registry, username, password, base64.StdEncoding.EncodeToString([]byte(auth)))
secret := &v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "docker",
Namespace: v1.NamespaceDefault,
},
Data: map[string][]byte{
v1.DockerConfigJsonKey: []byte(authString),
},
Type: v1.SecretTypeDockerConfigJson,
}
if insecure {
secret.Annotations = make(map[string]string)
secret.Annotations[forceInsecure] = "true"
}
return secret
}
func TestSecretAuthenticator(t *testing.T) {
secret := buildSecret("dockerhub.qingcloud.com", "guest", "guest", false)
secretAuthenticator, err := NewSecretAuthenticator(secret)
if err != nil {
t.Fatal(err)
}
auth, err := secretAuthenticator.Authorization()
if err != nil {
t.Fatal(err)
}
expected := &authn.AuthConfig{
Username: "guest",
Password: "guest",
Auth: "Z3Vlc3Q6Z3Vlc3Q=",
}
if diff := cmp.Diff(auth, expected); len(diff) != 0 {
t.Errorf("%T, got+ expected-, %s", expected, diff)
}
}
func TestAuthn(t *testing.T) {
testCases := []struct {
name string
secret *v1.Secret
auth bool
expectErr bool
}{
{
name: "Should authenticate with correct credential",
secret: buildSecret("https://dockerhub.qingcloud.com", "guest", "guest", false),
auth: true,
expectErr: false,
},
{
name: "Shouldn't authenticate with incorrect credentials",
secret: buildSecret("https://index.docker.io", "foo", "bar", false),
auth: false,
expectErr: true,
},
{
name: "Shouldn't authenticate with no credentials",
secret: nil,
auth: false,
expectErr: true,
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
secretAuthenticator, err := NewSecretAuthenticator(testCase.secret)
if err != nil {
t.Errorf("error creating secretAuthenticator, %v", err)
}
ok, err := secretAuthenticator.Auth()
if testCase.auth != ok {
t.Errorf("expected auth result: %v, but got %v", testCase.auth, ok)
}
if testCase.expectErr && err == nil {
t.Errorf("expected error, but got nil")
}
if !testCase.expectErr && err != nil {
t.Errorf("authentication error, %v", err)
}
})
}
}

View File

@@ -0,0 +1,37 @@
package v2
import (
v1 "github.com/google/go-containerregistry/pkg/v1"
)
// DockerConfig represents the config file used by the docker CLI.
// This config that represents the credentials that should be used
// when pulling images from specific image repositories.
type DockerConfig map[string]DockerConfigEntry
// DockerConfigEntry wraps a docker config as a entry
type DockerConfigEntry struct {
Username string
Password string
Email string
Auth string
}
// DockerConfigJSON represents ~/.docker/config.json file info
// see https://github.com/docker/docker/pull/12009
type DockerConfigJSON struct {
Auths DockerConfig `json:"auths"`
// +optional
HTTPHeaders map[string]string `json:"HttpHeaders,omitempty"`
}
type RepositoryTags struct {
Registry string `json:"registry"`
Repository string `json:"repository"`
Tags []string `json:"tags"`
}
// ImageConfig wraps v1.ConfigFile to avoid direct dependency
type ImageConfig struct {
*v1.ConfigFile `json:",inline"`
}