update package dependencies (#5463)

Signed-off-by: fsl <1171313930@qq.com>

Signed-off-by: fsl <1171313930@qq.com>
This commit is contained in:
fsl
2023-01-13 15:33:36 +08:00
committed by GitHub
parent 30d10cac53
commit efac3bde8f
471 changed files with 101081 additions and 36331 deletions

View File

@@ -0,0 +1,23 @@
package s3manager
import (
"fmt"
"github.com/aws/aws-sdk-go/aws/arn"
)
func validateSupportedARNType(bucket string) error {
if !arn.IsARN(bucket) {
return nil
}
parsedARN, err := arn.Parse(bucket)
if err != nil {
return err
}
if parsedARN.Service == "s3-object-lambda" {
return fmt.Errorf("manager does not support s3-object-lambda service ARNs")
}
return nil
}

View File

@@ -3,6 +3,7 @@ package s3manager
import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/client"
"github.com/aws/aws-sdk-go/aws/corehandlers"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/service/s3"
@@ -35,6 +36,30 @@ import (
// }
// fmt.Printf("Bucket %s is in %s region\n", bucket, region)
//
// By default the request will be made to the Amazon S3 endpoint using the Path
// style addressing.
//
// s3.us-west-2.amazonaws.com/bucketname
//
// This is not compatible with Amazon S3's FIPS endpoints. To override this
// behavior to use Virtual Host style addressing, provide a functional option
// that will set the Request's Config.S3ForcePathStyle to aws.Bool(false).
//
// region, err := s3manager.GetBucketRegion(ctx, sess, "bucketname", "us-west-2", func(r *request.Request) {
// r.S3ForcePathStyle = aws.Bool(false)
// })
//
// To configure the GetBucketRegion to make a request via the Amazon
// S3 FIPS endpoints directly when a FIPS region name is not available, (e.g.
// fips-us-gov-west-1) set the Config.Endpoint on the Session, or client the
// utility is called with. The hint region will be ignored if an endpoint URL
// is configured on the session or client.
//
// sess, err := session.NewSession(&aws.Config{
// Endpoint: aws.String("https://s3-fips.us-west-2.amazonaws.com"),
// })
//
// region, err := s3manager.GetBucketRegion(context.Background(), sess, "bucketname", "")
func GetBucketRegion(ctx aws.Context, c client.ConfigProvider, bucket, regionHint string, opts ...request.Option) (string, error) {
var cfg aws.Config
if len(regionHint) != 0 {
@@ -50,12 +75,38 @@ const bucketRegionHeader = "X-Amz-Bucket-Region"
// that it takes a S3 service client instead of a Session. The regionHint is
// derived from the region the S3 service client was created in.
//
// By default the request will be made to the Amazon S3 endpoint using the Path
// style addressing.
//
// s3.us-west-2.amazonaws.com/bucketname
//
// This is not compatible with Amazon S3's FIPS endpoints. To override this
// behavior to use Virtual Host style addressing, provide a functional option
// that will set the Request's Config.S3ForcePathStyle to aws.Bool(false).
//
// region, err := s3manager.GetBucketRegionWithClient(ctx, client, "bucketname", func(r *request.Request) {
// r.S3ForcePathStyle = aws.Bool(false)
// })
//
// To configure the GetBucketRegion to make a request via the Amazon
// S3 FIPS endpoints directly when a FIPS region name is not available, (e.g.
// fips-us-gov-west-1) set the Config.Endpoint on the Session, or client the
// utility is called with. The hint region will be ignored if an endpoint URL
// is configured on the session or client.
//
// region, err := s3manager.GetBucketRegionWithClient(context.Background(),
// s3.New(sess, &aws.Config{
// Endpoint: aws.String("https://s3-fips.us-west-2.amazonaws.com"),
// }),
// "bucketname")
//
// See GetBucketRegion for more information.
func GetBucketRegionWithClient(ctx aws.Context, svc s3iface.S3API, bucket string, opts ...request.Option) (string, error) {
req, _ := svc.HeadBucketRequest(&s3.HeadBucketInput{
Bucket: aws.String(bucket),
})
req.Config.S3ForcePathStyle = aws.Bool(true)
req.Config.Credentials = credentials.AnonymousCredentials
req.SetContext(ctx)
@@ -75,6 +126,16 @@ func GetBucketRegionWithClient(ctx aws.Context, svc s3iface.S3API, bucket string
r.HTTPResponse.Status = "OK"
r.Error = nil
})
// Replace the endpoint validation handler to not require a region if an
// endpoint URL was specified. Since these requests are not authenticated,
// requiring a region is not needed when an endpoint URL is provided.
req.Handlers.Validate.Swap(
corehandlers.ValidateEndpointHandler.Name,
request.NamedHandler{
Name: "validateEndpointWithoutRegion",
Fn: validateEndpointWithoutRegion,
},
)
req.ApplyOptions(opts...)
@@ -86,3 +147,13 @@ func GetBucketRegionWithClient(ctx aws.Context, svc s3iface.S3API, bucket string
return bucketRegion, nil
}
func validateEndpointWithoutRegion(r *request.Request) {
// Check if the caller provided an explicit URL instead of one derived by
// the SDK's endpoint resolver. For GetBucketRegion, with an explicit
// endpoint URL, a region is not needed. If no endpoint URL is provided,
// fallback the SDK's standard endpoint validation handler.
if len(aws.StringValue(r.Config.Endpoint)) == 0 {
corehandlers.ValidateEndpointHandler.Fn(r)
}
}

View File

@@ -1,3 +1,4 @@
//go:build !windows
// +build !windows
package s3manager

View File

@@ -1,3 +1,4 @@
//go:build !windows
// +build !windows
package s3manager

View File

@@ -192,6 +192,10 @@ func (d Downloader) Download(w io.WriterAt, input *s3.GetObjectInput, options ..
// to perform a single GetObjectInput request for that object's range. This will
// caused the part size, and concurrency configurations to be ignored.
func (d Downloader) DownloadWithContext(ctx aws.Context, w io.WriterAt, input *s3.GetObjectInput, options ...func(*Downloader)) (n int64, err error) {
if err := validateSupportedARNType(aws.StringValue(input.Bucket)); err != nil {
return 0, err
}
impl := downloader{w: w, in: input, cfg: d, ctx: ctx}
for _, option := range options {

View File

@@ -60,6 +60,14 @@ func (p *maxSlicePool) Get(ctx aws.Context) (*[]byte, error) {
return nil, errZeroCapacity
}
return bs, nil
case <-ctx.Done():
p.mtx.RUnlock()
return nil, ctx.Err()
default:
// pass
}
select {
case _, ok := <-p.allocations:
p.mtx.RUnlock()
if !ok {

View File

@@ -109,6 +109,9 @@ type UploadOutput struct {
// The ID for a multipart upload to S3. In the case of an error the error
// can be cast to the MultiUploadFailure interface to extract the upload ID.
UploadID string
// Entity tag of the object.
ETag *string
}
// WithUploaderRequestOptions appends to the Uploader's API request options.
@@ -121,6 +124,14 @@ func WithUploaderRequestOptions(opts ...request.Option) func(*Uploader) {
// The Uploader structure that calls Upload(). It is safe to call Upload()
// on this structure for multiple objects and across concurrent goroutines.
// Mutating the Uploader's properties is not safe to be done concurrently.
//
// The ContentMD5 member for pre-computed MD5 checksums will be ignored for
// multipart uploads. Objects that will be uploaded in a single part, the
// ContentMD5 will be used.
//
// The Checksum members for pre-computed checksums will be ignored for
// multipart uploads. Objects that will be uploaded in a single part, will
// include the checksum member in the request.
type Uploader struct {
// The buffer size (in bytes) to use when buffering data into chunks and
// sending them as parts to S3. The minimum allowed part size is 5MB, and
@@ -388,6 +399,10 @@ func (u *uploader) upload() (*UploadOutput, error) {
// init will initialize all default options.
func (u *uploader) init() error {
if err := validateSupportedARNType(aws.StringValue(u.in.Bucket)); err != nil {
return err
}
if u.cfg.Concurrency == 0 {
u.cfg.Concurrency = DefaultUploadConcurrency
}
@@ -527,6 +542,7 @@ func (u *uploader) singlePart(r io.ReadSeeker, cleanup func()) (*UploadOutput, e
return &UploadOutput{
Location: url,
VersionID: out.VersionId,
ETag: out.ETag,
}, nil
}
@@ -632,6 +648,7 @@ func (u *multiuploader) upload(firstBuf io.ReadSeeker, cleanup func()) (*UploadO
Location: uploadLocation,
VersionID: complete.VersionId,
UploadID: u.uploadID,
ETag: complete.ETag,
}, nil
}

View File

@@ -11,33 +11,105 @@ import (
// to an object in an Amazon S3 bucket. This type is similar to the s3
// package's PutObjectInput with the exception that the Body member is an
// io.Reader instead of an io.ReadSeeker.
//
// The ContentMD5 member for pre-computed MD5 checksums will be ignored for
// multipart uploads. Objects that will be uploaded in a single part, the
// ContentMD5 will be used.
//
// The Checksum members for pre-computed checksums will be ignored for
// multipart uploads. Objects that will be uploaded in a single part, will
// include the checksum member in the request.
type UploadInput struct {
_ struct{} `locationName:"PutObjectRequest" type:"structure" payload:"Body"`
// The canned ACL to apply to the object. For more information, see Canned ACL
// (https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#CannedACL).
//
// This action is not supported by Amazon S3 on Outposts.
ACL *string `location:"header" locationName:"x-amz-acl" type:"string" enum:"ObjectCannedACL"`
// The readable body payload to send to S3.
Body io.Reader
// Bucket name to which the PUT operation was initiated.
// The bucket name to which the PUT action was initiated.
//
// When using this API with an access point, you must direct requests to the
// access point hostname. The access point hostname takes the form AccessPointName-AccountId.s3-accesspoint.Region.amazonaws.com.
// When using this operation using an access point through the AWS SDKs, you
// provide the access point ARN in place of the bucket name. For more information
// about access point ARNs, see Using Access Points (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-access-points.html)
// in the Amazon Simple Storage Service Developer Guide.
// When using this action with an access point, you must direct requests to
// the access point hostname. The access point hostname takes the form AccessPointName-AccountId.s3-accesspoint.Region.amazonaws.com.
// When using this action with an access point through the Amazon Web Services
// SDKs, you provide the access point ARN in place of the bucket name. For more
// information about access point ARNs, see Using access points (https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-access-points.html)
// in the Amazon S3 User Guide.
//
// When using this action with Amazon S3 on Outposts, you must direct requests
// to the S3 on Outposts hostname. The S3 on Outposts hostname takes the form
// AccessPointName-AccountId.outpostID.s3-outposts.Region.amazonaws.com. When
// using this action with S3 on Outposts through the Amazon Web Services SDKs,
// you provide the Outposts bucket ARN in place of the bucket name. For more
// information about S3 on Outposts ARNs, see Using Amazon S3 on Outposts (https://docs.aws.amazon.com/AmazonS3/latest/userguide/S3onOutposts.html)
// in the Amazon S3 User Guide.
//
// Bucket is a required field
Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
// Specifies whether Amazon S3 should use an S3 Bucket Key for object encryption
// with server-side encryption using AWS KMS (SSE-KMS). Setting this header
// to true causes Amazon S3 to use an S3 Bucket Key for object encryption with
// SSE-KMS.
//
// Specifying this header with a PUT action doesnt affect bucket-level settings
// for S3 Bucket Key.
BucketKeyEnabled *bool `location:"header" locationName:"x-amz-server-side-encryption-bucket-key-enabled" type:"boolean"`
// Can be used to specify caching behavior along the request/reply chain. For
// more information, see http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9
// (http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9).
CacheControl *string `location:"header" locationName:"Cache-Control" type:"string"`
// Indicates the algorithm used to create the checksum for the object when using
// the SDK. This header will not provide any additional functionality if not
// using the SDK. When sending this header, there must be a corresponding x-amz-checksum
// or x-amz-trailer header sent. Otherwise, Amazon S3 fails the request with
// the HTTP status code 400 Bad Request. For more information, see Checking
// object integrity (https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html)
// in the Amazon S3 User Guide.
//
// If you provide an individual checksum, Amazon S3 ignores any provided ChecksumAlgorithm
// parameter.
//
// The AWS SDK for Go v1 does not support automatic computing request payload
// checksum. This feature is available in the AWS SDK for Go v2. If a value
// is specified for this parameter, the matching algorithm's checksum member
// must be populated with the algorithm's checksum of the request payload.
ChecksumAlgorithm *string `location:"header" locationName:"x-amz-sdk-checksum-algorithm" type:"string" enum:"ChecksumAlgorithm"`
// This header can be used as a data integrity check to verify that the data
// received is the same data that was originally sent. This header specifies
// the base64-encoded, 32-bit CRC32 checksum of the object. For more information,
// see Checking object integrity (https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html)
// in the Amazon S3 User Guide.
ChecksumCRC32 *string `location:"header" locationName:"x-amz-checksum-crc32" type:"string"`
// This header can be used as a data integrity check to verify that the data
// received is the same data that was originally sent. This header specifies
// the base64-encoded, 32-bit CRC32C checksum of the object. For more information,
// see Checking object integrity (https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html)
// in the Amazon S3 User Guide.
ChecksumCRC32C *string `location:"header" locationName:"x-amz-checksum-crc32c" type:"string"`
// This header can be used as a data integrity check to verify that the data
// received is the same data that was originally sent. This header specifies
// the base64-encoded, 160-bit SHA-1 digest of the object. For more information,
// see Checking object integrity (https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html)
// in the Amazon S3 User Guide.
ChecksumSHA1 *string `location:"header" locationName:"x-amz-checksum-sha1" type:"string"`
// This header can be used as a data integrity check to verify that the data
// received is the same data that was originally sent. This header specifies
// the base64-encoded, 256-bit SHA-256 digest of the object. For more information,
// see Checking object integrity (https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html)
// in the Amazon S3 User Guide.
ChecksumSHA256 *string `location:"header" locationName:"x-amz-checksum-sha256" type:"string"`
// Specifies presentational information for the object. For more information,
// see http://www.w3.org/Protocols/rfc2616/rfc2616-sec19.html#sec19.5.1 (http://www.w3.org/Protocols/rfc2616/rfc2616-sec19.html#sec19.5.1).
ContentDisposition *string `location:"header" locationName:"Content-Disposition" type:"string"`
@@ -57,29 +129,45 @@ type UploadInput struct {
// it is optional, we recommend using the Content-MD5 mechanism as an end-to-end
// integrity check. For more information about REST request authentication,
// see REST Authentication (https://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html).
//
// If the ContentMD5 is provided for a multipart upload, it will be ignored.
// Objects that will be uploaded in a single part, the ContentMD5 will be used.
ContentMD5 *string `location:"header" locationName:"Content-MD5" type:"string"`
// A standard MIME type describing the format of the contents. For more information,
// see http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.17 (http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.17).
ContentType *string `location:"header" locationName:"Content-Type" type:"string"`
// The account ID of the expected bucket owner. If the bucket is owned by a
// different account, the request fails with the HTTP status code 403 Forbidden
// (access denied).
ExpectedBucketOwner *string `location:"header" locationName:"x-amz-expected-bucket-owner" type:"string"`
// The date and time at which the object is no longer cacheable. For more information,
// see http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.21 (http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.21).
Expires *time.Time `location:"header" locationName:"Expires" type:"timestamp"`
// Gives the grantee READ, READ_ACP, and WRITE_ACP permissions on the object.
//
// This action is not supported by Amazon S3 on Outposts.
GrantFullControl *string `location:"header" locationName:"x-amz-grant-full-control" type:"string"`
// Allows grantee to read the object data and its metadata.
//
// This action is not supported by Amazon S3 on Outposts.
GrantRead *string `location:"header" locationName:"x-amz-grant-read" type:"string"`
// Allows grantee to read the object ACL.
//
// This action is not supported by Amazon S3 on Outposts.
GrantReadACP *string `location:"header" locationName:"x-amz-grant-read-acp" type:"string"`
// Allows grantee to write the ACL for the applicable object.
//
// This action is not supported by Amazon S3 on Outposts.
GrantWriteACP *string `location:"header" locationName:"x-amz-grant-write-acp" type:"string"`
// Object key for which the PUT operation was initiated.
// Object key for which the PUT action was initiated.
//
// Key is a required field
Key *string `location:"uri" locationName:"Key" min:"1" type:"string" required:"true"`
@@ -94,14 +182,15 @@ type UploadInput struct {
// The Object Lock mode that you want to apply to this object.
ObjectLockMode *string `location:"header" locationName:"x-amz-object-lock-mode" type:"string" enum:"ObjectLockMode"`
// The date and time when you want this object's Object Lock to expire.
// The date and time when you want this object's Object Lock to expire. Must
// be formatted as a timestamp parameter.
ObjectLockRetainUntilDate *time.Time `location:"header" locationName:"x-amz-object-lock-retain-until-date" type:"timestamp" timestampFormat:"iso8601"`
// Confirms that the requester knows that they will be charged for the request.
// Bucket owners need not specify this parameter in their requests. For information
// about downloading objects from requester pays buckets, see Downloading Objects
// in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html)
// in the Amazon S3 Developer Guide.
// about downloading objects from Requester Pays buckets, see Downloading Objects
// in Requester Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html)
// in the Amazon S3 User Guide.
RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"`
// Specifies the algorithm to use to when encrypting the object (for example,
@@ -111,7 +200,7 @@ type UploadInput struct {
// Specifies the customer-provided encryption key for Amazon S3 to use in encrypting
// data. This value is used to store the object and then it is discarded; Amazon
// S3 does not store the encryption key. The key must be appropriate for use
// with the algorithm specified in the x-amz-server-side-encryption-customer-algorithm
// with the algorithm specified in the x-amz-server-side-encryption-customer-algorithm
// header.
SSECustomerKey *string `marshal-as:"blob" location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string" sensitive:"true"`
@@ -120,29 +209,31 @@ type UploadInput struct {
// encryption key was transmitted without error.
SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"`
// Specifies the AWS KMS Encryption Context to use for object encryption. The
// value of this header is a base64-encoded UTF-8 string holding JSON with the
// encryption context key-value pairs.
// Specifies the Amazon Web Services KMS Encryption Context to use for object
// encryption. The value of this header is a base64-encoded UTF-8 string holding
// JSON with the encryption context key-value pairs.
SSEKMSEncryptionContext *string `location:"header" locationName:"x-amz-server-side-encryption-context" type:"string" sensitive:"true"`
// If x-amz-server-side-encryption is present and has the value of aws:kms,
// this header specifies the ID of the AWS Key Management Service (AWS KMS)
// symmetrical customer managed customer master key (CMK) that was used for
// the object.
//
// If the value of x-amz-server-side-encryption is aws:kms, this header specifies
// the ID of the symmetric customer managed AWS KMS CMK that will be used for
// the object. If you specify x-amz-server-side-encryption:aws:kms, but do not
// providex-amz-server-side-encryption-aws-kms-key-id, Amazon S3 uses the AWS
// managed CMK in AWS to protect the data.
// this header specifies the ID of the Amazon Web Services Key Management Service
// (Amazon Web Services KMS) symmetrical customer managed key that was used
// for the object. If you specify x-amz-server-side-encryption:aws:kms, but
// do not providex-amz-server-side-encryption-aws-kms-key-id, Amazon S3 uses
// the Amazon Web Services managed key to protect the data. If the KMS key does
// not exist in the same account issuing the command, you must use the full
// ARN and not just the ID.
SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"`
// The server-side encryption algorithm used when storing this object in Amazon
// S3 (for example, AES256, aws:kms).
ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string" enum:"ServerSideEncryption"`
// If you don't specify, Standard is the default storage class. Amazon S3 supports
// other storage classes.
// By default, Amazon S3 uses the STANDARD Storage Class to store newly created
// objects. The STANDARD storage class provides high durability and high availability.
// Depending on performance needs, you can specify a different Storage Class.
// Amazon S3 on Outposts only uses the OUTPOSTS Storage Class. For more information,
// see Storage Classes (https://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html)
// in the Amazon S3 User Guide.
StorageClass *string `location:"header" locationName:"x-amz-storage-class" type:"string" enum:"StorageClass"`
// The tag-set for the object. The tag-set must be encoded as URL Query parameters.