admin/kubesphere
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve the s3 uploader for better performance.

Browse Source 
Signed-off-by: dkkb <dabkb@aol.com>
This commit is contained in:
dkkb
2021-06-15 16:53:02 +08:00
parent d97bdffd94
commit 3ec5a5aea6
9 changed files with 58 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/models/devops/s2ibinary_handler.go
View File

@@ -104,7 +104,7 @@ func (s *s2iBinaryUploader) UploadS2iBinary(namespace, name, md5 string, fileHea
copy.Spec.FileName = fileHeader.Filename
copy.Spec.DownloadURL = fmt.Sprintf(GetS2iBinaryURL, namespace, name, copy.Spec.FileName)
err = s.s3Client.Upload(fmt.Sprintf("%s-%s", namespace, name), copy.Spec.FileName, binFile)
err = s.s3Client.Upload(fmt.Sprintf("%s-%s", namespace, name), copy.Spec.FileName, binFile, int(fileHeader.Size))
if err != nil {
if aerr, ok := err.(awserr.Error); ok {
switch aerr.Code() {

6
pkg/models/openpitrix/applications.go
View File

@@ -116,7 +116,7 @@ func (c *applicationOperator) createApp(app *v1alpha1.HelmApplication, iconData
if len(iconData) != 0 {
// save icon attachment
iconId := idutils.GetUuid(v1alpha1.HelmAttachmentPrefix)
err = c.backingStoreClient.Upload(iconId, iconId, bytes.NewBuffer(iconData))
err = c.backingStoreClient.Upload(iconId, iconId, bytes.NewBuffer(iconData), len(iconData))
if err != nil {
klog.Errorf("save icon attachment failed, error: %s", err)
return nil, err
@@ -499,7 +499,7 @@ func (c *applicationOperator) modifyAppAttachment(app *v1alpha1.HelmApplication,
// add attachment to app
add := idutils.GetUuid("att-")
*attachments = append(*attachments, add)
err = c.backingStoreClient.Upload(add, add, bytes.NewBuffer(request.AttachmentContent))
err = c.backingStoreClient.Upload(add, add, bytes.NewBuffer(request.AttachmentContent), len(request.AttachmentContent))
if err != nil {
return "", err
} else {
@@ -518,7 +518,7 @@ func (c *applicationOperator) modifyAppAttachment(app *v1alpha1.HelmApplication,
}
if len(request.AttachmentContent) != 0 {
add := idutils.GetUuid("att-")
err = c.backingStoreClient.Upload(add, add, bytes.NewBuffer(request.AttachmentContent))
err = c.backingStoreClient.Upload(add, add, bytes.NewBuffer(request.AttachmentContent), len(request.AttachmentContent))
if err != nil {
return "", err
} else {

4
pkg/models/openpitrix/applicationversions.go
View File

@@ -200,7 +200,7 @@ func (c *applicationOperator) ModifyAppVersion(id string, request *ModifyAppVers
spec.Created = &now
// 3. save chart data to s3 storage, just overwrite the legacy data
err = c.backingStoreClient.Upload(dataKeyInStorage(versionCopy.GetWorkspace(), versionCopy.Name), versionCopy.Name, bytes.NewReader(request.Package))
err = c.backingStoreClient.Upload(dataKeyInStorage(versionCopy.GetWorkspace(), versionCopy.Name), versionCopy.Name, bytes.NewBuffer(request.Package), len(request.Package))
if err != nil {
klog.Errorf("upload chart for app version: %s/%s failed, error: %s", versionCopy.GetWorkspace(),
versionCopy.GetTrueName(), err)
@@ -488,7 +488,7 @@ func (c *applicationOperator) createApplicationVersion(ver *v1alpha1.HelmApplica
klog.Errorf("decode error: %s", err)
return nil, err
} else {
err = c.backingStoreClient.Upload(dataKeyInStorage(ver.GetWorkspace(), ver.Name), ver.Name, bytes.NewReader(ver.Spec.Data))
err = c.backingStoreClient.Upload(dataKeyInStorage(ver.GetWorkspace(), ver.Name), ver.Name, bytes.NewBuffer(ver.Spec.Data), len(ver.Spec.Data))
if err != nil {
klog.Errorf("upload chart for app version: %s/%s failed, error: %s", ver.GetWorkspace(),
ver.GetTrueName(), err)

3
pkg/models/openpitrix/attachments.go
View File

@@ -15,7 +15,6 @@ package openpitrix
import (
"bytes"
"github.com/go-openapi/strfmt"
"k8s.io/klog"
@@ -66,7 +65,7 @@ func (c *attachmentOperator) CreateAttachment(data []byte) (*Attachment, error)
}
id := idutils.GetUuid36(v1alpha1.HelmAttachmentPrefix)
err := c.backingStoreClient.Upload(id, id, bytes.NewBuffer(data))
err := c.backingStoreClient.Upload(id, id, bytes.NewBuffer(data), len(data))
if err != nil {
klog.Errorf("upload attachment failed, err: %s", err)
return nil, err