devops refactor (#1739)

* add devops client interface Signed-off-by: runzexia <runzexia@yunify.com> * direct return jenkins Signed-off-by: runzexia <runzexia@yunify.com> * add some interface Signed-off-by: runzexia <runzexia@yunify.com> * update Signed-off-by: runzexia <runzexia@yunify.com> * update interface Signed-off-by: runzexia <runzexia@yunify.com> * update Signed-off-by: runzexia <runzexia@yunify.com> * credential op structs Signed-off-by: runzexia <runzexia@yunify.com> * status Signed-off-by: zhuxiaoyang <sunzhu@yunify.com> * update interface Signed-off-by: zhuxiaoyang <sunzhu@yunify.com> * credential handler Signed-off-by: runzexia <runzexia@yunify.com> * update devopsoperator func Signed-off-by: zhuxiaoyang <sunzhu@yunify.com> * get build sonar Signed-off-by: runzexia <runzexia@yunify.com> * sonar handler * mv code to cilent Signed-off-by: runzexia <runzexia@yunify.com> * update Signed-off-by: runzexia <runzexia@yunify.com> * project member handler Signed-off-by: runzexia <runzexia@yunify.com> * update pipeline operator interface Signed-off-by: zhuxiaoyang <sunzhu@yunify.com> * add tenant devops handler Signed-off-by: runzexia <runzexia@yunify.com> * update merge Signed-off-by: runzexia <runzexia@yunify.com> * clean Signed-off-by: runzexia <runzexia@yunify.com> * fmt Signed-off-by: runzexia <runzexia@yunify.com> * update ListPipelineRuns Signed-off-by: zhuxiaoyang <sunzhu@yunify.com> * complate pipelineOperator interface Signed-off-by: zhuxiaoyang <sunzhu@yunify.com> * update HttpParameters Signed-off-by: zhuxiaoyang <sunzhu@yunify.com> * add pipeline steps interface Signed-off-by: zhuxiaoyang <sunzhu@yunify.com> * update pipeline GetNodesDetail Signed-off-by: zhuxiaoyang <sunzhu@yunify.com> * add s2i api Signed-off-by: runzexia <runzexia@yunify.com> * add branch pipeline interface and update handler Signed-off-by: zhuxiaoyang <sunzhu@yunify.com> * add scan branch interface and update handler Signed-off-by: zhuxiaoyang <sunzhu@yunify.com> * add common interface and update handler Signed-off-by: zhuxiaoyang <sunzhu@yunify.com> * add SCM interface and update handler Signed-off-by: zhuxiaoyang <sunzhu@yunify.com> * add handler Signed-off-by: runzexia <runzexia@yunify.com> * add fake s3 Signed-off-by: runzexia <runzexia@yunify.com> * add webhook&check interface and update handler Signed-off-by: zhuxiaoyang <sunzhu@yunify.com> * clean Signed-off-by: zhuxiaoyang <sunzhu@yunify.com> * clean Signed-off-by: zhuxiaoyang <sunzhu@yunify.com> * format Signed-off-by: zhuxiaoyang <sunzhu@yunify.com> * add some func Signed-off-by: runzexia <runzexia@yunify.com> * clean code Signed-off-by: runzexia <runzexia@yunify.com> * implement interface Signed-off-by: zhuxiaoyang <sunzhu@yunify.com> * fix interface GetBranchArtifacts Signed-off-by: zhuxiaoyang <sunzhu@yunify.com> * add s2ibinary upload test Signed-off-by: runzexia <runzexia@yunify.com> * tenant devops Signed-off-by: runzexia <runzexia@yunify.com> * update tenant Signed-off-by: runzexia <runzexia@yunify.com> * fake Signed-off-by: zhuxiaoyang <sunzhu@yunify.com> * add some unit test Signed-off-by: zhuxiaoyang <sunzhu@yunify.com> * add devops tenant handler Signed-off-by: runzexia <runzexia@yunify.com> * status Signed-off-by: zhuxiaoyang <sunzhu@yunify.com> * status Signed-off-by: zhuxiaoyang <sunzhu@yunify.com> * status Signed-off-by: zhuxiaoyang <sunzhu@yunify.com> * update fake test Signed-off-by: zhuxiaoyang <sunzhu@yunify.com> * update unit test and fake data Signed-off-by: zhuxiaoyang <sunzhu@yunify.com> * update Co-authored-by: Xiaoyang Zhu <sunzhu@yunify.com>
2020-02-04 10:40:36 +08:00
parent 71849f028f
commit c5a340a2b4
101 changed files with 6923 additions and 6972 deletions
--- a/pkg/simple/client/s3/fake/fakes3.go
+++ b/pkg/simple/client/s3/fake/fakes3.go
@@ -0,0 +1,43 @@
+package fake
+
+import (
+	"fmt"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"io"
+)
+
+type FakeS3 struct {
+	Storage map[string]*object
+}
+
+func NewFakeS3() *FakeS3 {
+	return &FakeS3{Storage: map[string]*object{}}
+}
+
+type object struct {
+	key      string
+	fileName string
+	body     io.Reader
+}
+
+func (s *FakeS3) Upload(key, fileName string, body io.Reader) error {
+	s.Storage[key] = &object{
+		key:      key,
+		fileName: fileName,
+		body:     body,
+	}
+	return nil
+}
+
+func (s *FakeS3) GetDownloadURL(key string, fileName string) (string, error) {
+	if o, ok := s.Storage[key]; ok {
+		return fmt.Sprintf("http://%s/%s", o.key, fileName), nil
+	}
+	return "", awserr.New(s3.ErrCodeNoSuchKey, "no such object", nil)
+}
+
+func (s *FakeS3) Delete(key string) error {
+	delete(s.Storage, key)
+	return nil
+}
--- a/pkg/simple/client/s3/fake/fakes3_test.go
+++ b/pkg/simple/client/s3/fake/fakes3_test.go
@@ -0,0 +1,52 @@
+package fake
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestFakeS3(t *testing.T) {
+	s3 := NewFakeS3()
+	key := "hello"
+	fileName := "world"
+	err := s3.Upload(key, fileName, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	o, ok := s3.storage["hello"]
+	if !ok {
+		t.Fatal("should have hello object")
+	}
+	if o.key != key || o.fileName != fileName {
+		t.Fatalf("key should be %s, fileName should be %s", key, fileName)
+	}
+
+	url, err := s3.GetDownloadURL(key, fileName+"1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if url != fmt.Sprintf("http://%s/%s", key, fileName+"1") {
+		t.Fatalf("url should be %s", fmt.Sprintf("http://%s/%s", key, fileName+"1"))
+	}
+
+	url, err = s3.GetDownloadURL(key, fileName+"2")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if url != fmt.Sprintf("http://%s/%s", key, fileName+"2") {
+		t.Fatalf("url should be %s", fmt.Sprintf("http://%s/%s", key, fileName+"2"))
+	}
+
+	err = s3.Delete(key)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, ok = s3.storage["hello"]
+	if ok {
+		t.Fatal("should not have hello object")
+	}
+	err = s3.Delete(key)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
--- a/pkg/simple/client/s3/interface.go
+++ b/pkg/simple/client/s3/interface.go
@@ -2,15 +2,13 @@ package s3

 import (
 	"io"
-	"time"
 )

 type Interface interface {
 	// Upload uploads a object to storage and returns object location if succeeded
-	Upload(key string, body io.Reader) (string, error)
+	Upload(key, fileName string, body io.Reader) error

-	// Get retrieves and object's downloadable location if succeeded
-	Get(key string, fileName string, expire time.Duration) (string, error)
+	GetDownloadURL(key string, fileName string) (string, error)

 	// Delete deletes an object by its key
 	Delete(key string) error
--- a/pkg/simple/client/s3/s3.go
+++ b/pkg/simple/client/s3/s3.go
@@ -1,10 +1,13 @@
 package s3

 import (
+	"code.cloudfoundry.org/bytefmt"
+	"fmt"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	"io"
 	"k8s.io/klog"
 	"time"
@@ -16,16 +19,38 @@ type Client struct {
 	bucket    string
 }

-func (s *Client) Upload(key string, body io.Reader) (string, error) {
-	panic("implement me")
+func (s *Client) Upload(key, fileName string, body io.Reader) error {
+	uploader := s3manager.NewUploader(s.s3Session, func(uploader *s3manager.Uploader) {
+		uploader.PartSize = 5 * bytefmt.MEGABYTE
+		uploader.LeavePartsOnError = true
+	})
+	_, err := uploader.Upload(&s3manager.UploadInput{
+		Bucket:             aws.String(s.bucket),
+		Key:                aws.String(key),
+		Body:               body,
+		ContentDisposition: aws.String(fmt.Sprintf("attachment; filename=\"%s\"", fileName)),
+	})
+	return err
 }

-func (s *Client) Get(key string, fileName string, expire time.Duration) (string, error) {
-	panic("implement me")
+func (s *Client) GetDownloadURL(key string, fileName string) (string, error) {
+	req, _ := s.s3Client.GetObjectRequest(&s3.GetObjectInput{
+		Bucket:                     aws.String(s.bucket),
+		Key:                        aws.String(key),
+		ResponseContentDisposition: aws.String(fmt.Sprintf("attachment; filename=\"%s\"", fileName)),
+	})
+	return req.Presign(5 * time.Minute)
 }

 func (s *Client) Delete(key string) error {
-	panic("implement me")
+	_, err := s.s3Client.DeleteObject(
+		&s3.DeleteObjectInput{Bucket: aws.String(s.bucket),
+			Key: aws.String(key),
+		})
+	if err != nil {
+		return err
+	}
+	return nil
 }

 func NewS3Client(options *Options) (Interface, error) {
@@ -55,7 +80,6 @@ func NewS3Client(options *Options) (Interface, error) {
 }

 func (s *Client) Client() *s3.S3 {
-
 	return s.s3Client
 }
 func (s *Client) Session() *session.Session {