update package dependencies (#5463)

Signed-off-by: fsl <1171313930@qq.com>

Signed-off-by: fsl <1171313930@qq.com>
This commit is contained in:
fsl
2023-01-13 15:33:36 +08:00
committed by GitHub
parent 30d10cac53
commit efac3bde8f
471 changed files with 101081 additions and 36331 deletions

File diff suppressed because it is too large Load Diff

View File

@@ -13,7 +13,6 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/internal/sdkio"
)
const (
@@ -25,30 +24,6 @@ const (
appendMD5TxEncoding = "append-md5"
)
// contentMD5 computes and sets the HTTP Content-MD5 header for requests that
// require it.
func contentMD5(r *request.Request) {
h := md5.New()
if !aws.IsReaderSeekable(r.Body) {
if r.Config.Logger != nil {
r.Config.Logger.Log(fmt.Sprintf(
"Unable to compute Content-MD5 for unseekable body, S3.%s",
r.Operation.Name))
}
return
}
if _, err := copySeekableBody(h, r.Body); err != nil {
r.Error = awserr.New("ContentMD5", "failed to compute body MD5", err)
return
}
// encode the md5 checksum in base64 and set the request header.
v := base64.StdEncoding.EncodeToString(h.Sum(nil))
r.HTTPRequest.Header.Set(contentMD5Header, v)
}
// computeBodyHashes will add Content MD5 and Content Sha256 hashes to the
// request. If the body is not seekable or S3DisableContentMD5Validation set
// this handler will be ignored.
@@ -90,7 +65,7 @@ func computeBodyHashes(r *request.Request) {
dst = io.MultiWriter(hashers...)
}
if _, err := copySeekableBody(dst, r.Body); err != nil {
if _, err := aws.CopySeekableBody(dst, r.Body); err != nil {
r.Error = awserr.New("BodyHashError", "failed to compute body hashes", err)
return
}
@@ -119,28 +94,6 @@ const (
sha256HexEncLen = sha256.Size * 2 // hex.EncodedLen
)
func copySeekableBody(dst io.Writer, src io.ReadSeeker) (int64, error) {
curPos, err := src.Seek(0, sdkio.SeekCurrent)
if err != nil {
return 0, err
}
// hash the body. seek back to the first position after reading to reset
// the body for transmission. copy errors may be assumed to be from the
// body.
n, err := io.Copy(dst, src)
if err != nil {
return n, err
}
_, err = src.Seek(curPos, sdkio.SeekStart)
if err != nil {
return n, err
}
return n, nil
}
// Adds the x-amz-te: append_md5 header to the request. This requests the service
// responds with a trailing MD5 checksum.
//

View File

@@ -1,10 +1,12 @@
package s3
import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/client"
"github.com/aws/aws-sdk-go/aws/endpoints"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/internal/s3err"
"github.com/aws/aws-sdk-go/service/s3/internal/arn"
"github.com/aws/aws-sdk-go/internal/s3shared/arn"
"github.com/aws/aws-sdk-go/internal/s3shared/s3err"
)
func init() {
@@ -13,6 +15,14 @@ func init() {
}
func defaultInitClientFn(c *client.Client) {
if c.Config.UseDualStackEndpoint == endpoints.DualStackEndpointStateUnset {
if aws.BoolValue(c.Config.UseDualStack) {
c.Config.UseDualStackEndpoint = endpoints.DualStackEndpointStateEnabled
} else {
c.Config.UseDualStackEndpoint = endpoints.DualStackEndpointStateDisabled
}
}
// Support building custom endpoints based on config
c.Handlers.Build.PushFront(endpointHandler)
@@ -33,12 +43,6 @@ func defaultInitRequestFn(r *request.Request) {
platformRequestHandlers(r)
switch r.Operation.Name {
case opPutBucketCors, opPutBucketLifecycle, opPutBucketPolicy,
opPutBucketTagging, opDeleteObjects, opPutBucketLifecycleConfiguration,
opPutObjectLegalHold, opPutObjectRetention, opPutObjectLockConfiguration,
opPutBucketReplication:
// These S3 operations require Content-MD5 to be set
r.Handlers.Build.PushBack(contentMD5)
case opGetBucketLocation:
// GetBucketLocation has custom parsing logic
r.Handlers.Unmarshal.PushFront(buildGetBucketLocation)
@@ -46,7 +50,7 @@ func defaultInitRequestFn(r *request.Request) {
// Auto-populate LocationConstraint with current region
r.Handlers.Validate.PushFront(populateLocationConstraint)
case opCopyObject, opUploadPartCopy, opCompleteMultipartUpload:
r.Handlers.Unmarshal.PushFront(copyMultipartStatusOKUnmarhsalError)
r.Handlers.Unmarshal.PushFront(copyMultipartStatusOKUnmarshalError)
r.Handlers.Unmarshal.PushBackNamed(s3err.RequestFailureWrapperHandler())
case opPutObject, opUploadPart:
r.Handlers.Build.PushBack(computeBodyHashes)
@@ -54,6 +58,8 @@ func defaultInitRequestFn(r *request.Request) {
// case opGetObject:
// r.Handlers.Build.PushBack(askForTxEncodingAppendMD5)
// r.Handlers.Unmarshal.PushBack(useMD5ValidationReader)
case opWriteGetObjectResponse:
r.Handlers.Build.PushFront(buildWriteGetObjectResponseEndpoint)
}
}
@@ -75,6 +81,8 @@ type copySourceSSECustomerKeyGetter interface {
getCopySourceSSECustomerKey() string
}
// endpointARNGetter is an accessor interface to grab the
// the field corresponding to an endpoint ARN input.
type endpointARNGetter interface {
getEndpointARN() (arn.Resource, error)
hasEndpointARN() bool

View File

@@ -104,19 +104,6 @@
// content from S3. The Encryption and Decryption clients can be used concurrently
// once the client is created.
//
// sess := session.Must(session.NewSession())
//
// // Create the decryption client.
// svc := s3crypto.NewDecryptionClient(sess)
//
// // The object will be downloaded from S3 and decrypted locally. By metadata
// // about the object's encryption will instruct the decryption client how
// // decrypt the content of the object. By default KMS is used for keys.
// result, err := svc.GetObject(&s3.GetObjectInput {
// Bucket: aws.String(myBucket),
// Key: aws.String(myKey),
// })
//
// See the s3crypto package documentation for more information.
// https://docs.aws.amazon.com/sdk-for-go/api/service/s3/s3crypto/
//

View File

@@ -1,16 +1,24 @@
package s3
import (
"fmt"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/endpoints"
"net/url"
"strings"
"github.com/aws/aws-sdk-go/aws"
awsarn "github.com/aws/aws-sdk-go/aws/arn"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/endpoints"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/private/protocol"
"github.com/aws/aws-sdk-go/service/s3/internal/arn"
"github.com/aws/aws-sdk-go/internal/s3shared"
"github.com/aws/aws-sdk-go/internal/s3shared/arn"
)
const (
s3Namespace = "s3"
s3AccessPointNamespace = "s3-accesspoint"
s3ObjectsLambdaNamespace = "s3-object-lambda"
s3OutpostsNamespace = "s3-outposts"
)
// Used by shapes with members decorated as endpoint ARN.
@@ -22,12 +30,89 @@ func accessPointResourceParser(a awsarn.ARN) (arn.Resource, error) {
resParts := arn.SplitResource(a.Resource)
switch resParts[0] {
case "accesspoint":
return arn.ParseAccessPointResource(a, resParts[1:])
switch a.Service {
case s3Namespace:
return arn.ParseAccessPointResource(a, resParts[1:])
case s3ObjectsLambdaNamespace:
return parseS3ObjectLambdaAccessPointResource(a, resParts)
default:
return arn.AccessPointARN{}, arn.InvalidARNError{ARN: a, Reason: fmt.Sprintf("service is not %s or %s", s3Namespace, s3ObjectsLambdaNamespace)}
}
case "outpost":
if a.Service != "s3-outposts" {
return arn.OutpostAccessPointARN{}, arn.InvalidARNError{ARN: a, Reason: "service is not s3-outposts"}
}
return parseOutpostAccessPointResource(a, resParts[1:])
default:
return nil, arn.InvalidARNError{ARN: a, Reason: "unknown resource type"}
}
}
// parseOutpostAccessPointResource attempts to parse the ARNs resource as an
// outpost access-point resource.
//
// Supported Outpost AccessPoint ARN format:
// - ARN format: arn:{partition}:s3-outposts:{region}:{accountId}:outpost/{outpostId}/accesspoint/{accesspointName}
// - example: arn:aws:s3-outposts:us-west-2:012345678901:outpost/op-1234567890123456/accesspoint/myaccesspoint
//
func parseOutpostAccessPointResource(a awsarn.ARN, resParts []string) (arn.OutpostAccessPointARN, error) {
// outpost accesspoint arn is only valid if service is s3-outposts
if a.Service != "s3-outposts" {
return arn.OutpostAccessPointARN{}, arn.InvalidARNError{ARN: a, Reason: "service is not s3-outposts"}
}
if len(resParts) == 0 {
return arn.OutpostAccessPointARN{}, arn.InvalidARNError{ARN: a, Reason: "outpost resource-id not set"}
}
if len(resParts) < 3 {
return arn.OutpostAccessPointARN{}, arn.InvalidARNError{
ARN: a, Reason: "access-point resource not set in Outpost ARN",
}
}
resID := strings.TrimSpace(resParts[0])
if len(resID) == 0 {
return arn.OutpostAccessPointARN{}, arn.InvalidARNError{ARN: a, Reason: "outpost resource-id not set"}
}
var outpostAccessPointARN = arn.OutpostAccessPointARN{}
switch resParts[1] {
case "accesspoint":
accessPointARN, err := arn.ParseAccessPointResource(a, resParts[2:])
if err != nil {
return arn.OutpostAccessPointARN{}, err
}
// set access-point arn
outpostAccessPointARN.AccessPointARN = accessPointARN
default:
return arn.OutpostAccessPointARN{}, arn.InvalidARNError{ARN: a, Reason: "access-point resource not set in Outpost ARN"}
}
// set outpost id
outpostAccessPointARN.OutpostID = resID
return outpostAccessPointARN, nil
}
func parseS3ObjectLambdaAccessPointResource(a awsarn.ARN, resParts []string) (arn.S3ObjectLambdaAccessPointARN, error) {
if a.Service != s3ObjectsLambdaNamespace {
return arn.S3ObjectLambdaAccessPointARN{}, arn.InvalidARNError{ARN: a, Reason: fmt.Sprintf("service is not %s", s3ObjectsLambdaNamespace)}
}
accessPointARN, err := arn.ParseAccessPointResource(a, resParts[1:])
if err != nil {
return arn.S3ObjectLambdaAccessPointARN{}, err
}
if len(accessPointARN.Region) == 0 {
return arn.S3ObjectLambdaAccessPointARN{}, arn.InvalidARNError{ARN: a, Reason: fmt.Sprintf("%s region not set", s3ObjectsLambdaNamespace)}
}
return arn.S3ObjectLambdaAccessPointARN{
AccessPointARN: accessPointARN,
}, nil
}
func endpointHandler(req *request.Request) {
endpoint, ok := req.Params.(endpointARNGetter)
if !ok || !endpoint.hasEndpointARN() {
@@ -37,79 +122,55 @@ func endpointHandler(req *request.Request) {
resource, err := endpoint.getEndpointARN()
if err != nil {
req.Error = newInvalidARNError(nil, err)
req.Error = s3shared.NewInvalidARNError(nil, err)
return
}
resReq := resourceRequest{
resReq := s3shared.ResourceRequest{
Resource: resource,
Request: req,
}
if resReq.IsCrossPartition() {
req.Error = newClientPartitionMismatchError(resource,
if len(resReq.Request.ClientInfo.PartitionID) != 0 && resReq.IsCrossPartition() {
req.Error = s3shared.NewClientPartitionMismatchError(resource,
req.ClientInfo.PartitionID, aws.StringValue(req.Config.Region), nil)
return
}
if !resReq.AllowCrossRegion() && resReq.IsCrossRegion() {
req.Error = newClientRegionMismatchError(resource,
req.Error = s3shared.NewClientRegionMismatchError(resource,
req.ClientInfo.PartitionID, aws.StringValue(req.Config.Region), nil)
return
}
if resReq.HasCustomEndpoint() {
req.Error = newInvalidARNWithCustomEndpointError(resource, nil)
return
}
switch tv := resource.(type) {
case arn.AccessPointARN:
err = updateRequestAccessPointEndpoint(req, tv)
if err != nil {
req.Error = err
}
case arn.S3ObjectLambdaAccessPointARN:
err = updateRequestS3ObjectLambdaAccessPointEndpoint(req, tv)
if err != nil {
req.Error = err
}
case arn.OutpostAccessPointARN:
// outposts does not support FIPS regions
if req.Config.UseFIPSEndpoint == endpoints.FIPSEndpointStateEnabled {
req.Error = s3shared.NewFIPSConfigurationError(resource, req.ClientInfo.PartitionID,
aws.StringValue(req.Config.Region), nil)
return
}
err = updateRequestOutpostAccessPointEndpoint(req, tv)
if err != nil {
req.Error = err
}
default:
req.Error = newInvalidARNError(resource, nil)
req.Error = s3shared.NewInvalidARNError(resource, nil)
}
}
type resourceRequest struct {
Resource arn.Resource
Request *request.Request
}
func (r resourceRequest) ARN() awsarn.ARN {
return r.Resource.GetARN()
}
func (r resourceRequest) AllowCrossRegion() bool {
return aws.BoolValue(r.Request.Config.S3UseARNRegion)
}
func (r resourceRequest) UseFIPS() bool {
return isFIPS(aws.StringValue(r.Request.Config.Region))
}
func (r resourceRequest) IsCrossPartition() bool {
return r.Request.ClientInfo.PartitionID != r.Resource.GetARN().Partition
}
func (r resourceRequest) IsCrossRegion() bool {
return isCrossRegion(r.Request, r.Resource.GetARN().Region)
}
func (r resourceRequest) HasCustomEndpoint() bool {
return len(aws.StringValue(r.Request.Config.Endpoint)) > 0
}
func isFIPS(clientRegion string) bool {
return strings.HasPrefix(clientRegion, "fips-") || strings.HasSuffix(clientRegion, "-fips")
}
func isCrossRegion(req *request.Request, otherRegion string) bool {
return req.ClientInfo.SigningRegion != otherRegion
}
func updateBucketEndpointFromParams(r *request.Request) {
bucket, ok := bucketNameFromReqParams(r.Params)
if !ok {
@@ -124,15 +185,14 @@ func updateBucketEndpointFromParams(r *request.Request) {
func updateRequestAccessPointEndpoint(req *request.Request, accessPoint arn.AccessPointARN) error {
// Accelerate not supported
if aws.BoolValue(req.Config.S3UseAccelerate) {
return newClientConfiguredForAccelerateError(accessPoint,
return s3shared.NewClientConfiguredForAccelerateError(accessPoint,
req.ClientInfo.PartitionID, aws.StringValue(req.Config.Region), nil)
}
// Ignore the disable host prefix for access points since custom endpoints
// are not supported.
// Ignore the disable host prefix for access points
req.Config.DisableEndpointHostPrefix = aws.Bool(false)
if err := accessPointEndpointBuilder(accessPoint).Build(req); err != nil {
if err := accessPointEndpointBuilder(accessPoint).build(req); err != nil {
return err
}
@@ -141,6 +201,55 @@ func updateRequestAccessPointEndpoint(req *request.Request, accessPoint arn.Acce
return nil
}
func updateRequestS3ObjectLambdaAccessPointEndpoint(req *request.Request, accessPoint arn.S3ObjectLambdaAccessPointARN) error {
// DualStack not supported
if isUseDualStackEndpoint(req) {
return s3shared.NewClientConfiguredForDualStackError(accessPoint,
req.ClientInfo.PartitionID, aws.StringValue(req.Config.Region), nil)
}
// Accelerate not supported
if aws.BoolValue(req.Config.S3UseAccelerate) {
return s3shared.NewClientConfiguredForAccelerateError(accessPoint,
req.ClientInfo.PartitionID, aws.StringValue(req.Config.Region), nil)
}
// Ignore the disable host prefix for access points
req.Config.DisableEndpointHostPrefix = aws.Bool(false)
if err := s3ObjectLambdaAccessPointEndpointBuilder(accessPoint).build(req); err != nil {
return err
}
removeBucketFromPath(req.HTTPRequest.URL)
return nil
}
func updateRequestOutpostAccessPointEndpoint(req *request.Request, accessPoint arn.OutpostAccessPointARN) error {
// Accelerate not supported
if aws.BoolValue(req.Config.S3UseAccelerate) {
return s3shared.NewClientConfiguredForAccelerateError(accessPoint,
req.ClientInfo.PartitionID, aws.StringValue(req.Config.Region), nil)
}
// Dualstack not supported
if isUseDualStackEndpoint(req) {
return s3shared.NewClientConfiguredForDualStackError(accessPoint,
req.ClientInfo.PartitionID, aws.StringValue(req.Config.Region), nil)
}
// Ignore the disable host prefix for access points
req.Config.DisableEndpointHostPrefix = aws.Bool(false)
if err := outpostAccessPointEndpointBuilder(accessPoint).build(req); err != nil {
return err
}
removeBucketFromPath(req.HTTPRequest.URL)
return nil
}
func removeBucketFromPath(u *url.URL) {
u.Path = strings.Replace(u.Path, "/{Bucket}", "", -1)
if u.Path == "" {
@@ -148,86 +257,43 @@ func removeBucketFromPath(u *url.URL) {
}
}
type accessPointEndpointBuilder arn.AccessPointARN
func buildWriteGetObjectResponseEndpoint(req *request.Request) {
// DualStack not supported
if isUseDualStackEndpoint(req) {
req.Error = awserr.New("ConfigurationError", "client configured for dualstack but not supported for operation", nil)
return
}
const (
accessPointPrefixLabel = "accesspoint"
accountIDPrefixLabel = "accountID"
accesPointPrefixTemplate = "{" + accessPointPrefixLabel + "}-{" + accountIDPrefixLabel + "}."
)
// Accelerate not supported
if aws.BoolValue(req.Config.S3UseAccelerate) {
req.Error = awserr.New("ConfigurationError", "client configured for accelerate but not supported for operation", nil)
return
}
func (a accessPointEndpointBuilder) Build(req *request.Request) error {
resolveRegion := arn.AccessPointARN(a).Region
cfgRegion := aws.StringValue(req.Config.Region)
signingName := s3ObjectsLambdaNamespace
signingRegion := req.ClientInfo.SigningRegion
if isFIPS(cfgRegion) {
if aws.BoolValue(req.Config.S3UseARNRegion) && isCrossRegion(req, resolveRegion) {
// FIPS with cross region is not supported, the SDK must fail
// because there is no well defined method for SDK to construct a
// correct FIPS endpoint.
return newClientConfiguredForCrossRegionFIPSError(arn.AccessPointARN(a),
req.ClientInfo.PartitionID, cfgRegion, nil)
if !hasCustomEndpoint(req) {
endpoint, err := resolveRegionalEndpoint(req, aws.StringValue(req.Config.Region), req.ClientInfo.ResolvedRegion, EndpointsID)
if err != nil {
req.Error = awserr.New(request.ErrCodeSerialization, "failed to resolve endpoint", err)
return
}
resolveRegion = cfgRegion
signingRegion = endpoint.SigningRegion
if err = updateRequestEndpoint(req, endpoint.URL); err != nil {
req.Error = err
return
}
updateS3HostPrefixForS3ObjectLambda(req)
}
endpoint, err := resolveRegionalEndpoint(req, resolveRegion)
if err != nil {
return newFailedToResolveEndpointError(arn.AccessPointARN(a),
req.ClientInfo.PartitionID, cfgRegion, err)
}
if err = updateRequestEndpoint(req, endpoint.URL); err != nil {
return err
}
const serviceEndpointLabel = "s3-accesspoint"
// dualstack provided by endpoint resolver
cfgHost := req.HTTPRequest.URL.Host
if strings.HasPrefix(cfgHost, "s3") {
req.HTTPRequest.URL.Host = serviceEndpointLabel + cfgHost[2:]
}
protocol.HostPrefixBuilder{
Prefix: accesPointPrefixTemplate,
LabelsFn: a.hostPrefixLabelValues,
}.Build(req)
req.ClientInfo.SigningName = endpoint.SigningName
req.ClientInfo.SigningRegion = endpoint.SigningRegion
err = protocol.ValidateEndpointHost(req.Operation.Name, req.HTTPRequest.URL.Host)
if err != nil {
return newInvalidARNError(arn.AccessPointARN(a), err)
}
return nil
redirectSigner(req, signingName, signingRegion)
}
func (a accessPointEndpointBuilder) hostPrefixLabelValues() map[string]string {
return map[string]string{
accessPointPrefixLabel: arn.AccessPointARN(a).AccessPointName,
accountIDPrefixLabel: arn.AccessPointARN(a).AccountID,
func isUseDualStackEndpoint(req *request.Request) bool {
if req.Config.UseDualStackEndpoint != endpoints.DualStackEndpointStateUnset {
return req.Config.UseDualStackEndpoint == endpoints.DualStackEndpointStateEnabled
}
}
func resolveRegionalEndpoint(r *request.Request, region string) (endpoints.ResolvedEndpoint, error) {
return r.Config.EndpointResolver.EndpointFor(EndpointsID, region, func(opts *endpoints.Options) {
opts.DisableSSL = aws.BoolValue(r.Config.DisableSSL)
opts.UseDualStack = aws.BoolValue(r.Config.UseDualStack)
opts.S3UsEast1RegionalEndpoint = endpoints.RegionalS3UsEast1Endpoint
})
}
func updateRequestEndpoint(r *request.Request, endpoint string) (err error) {
endpoint = endpoints.AddScheme(endpoint, aws.BoolValue(r.Config.DisableSSL))
r.HTTPRequest.URL, err = url.Parse(endpoint + r.Operation.HTTPPath)
if err != nil {
return awserr.New(request.ErrCodeSerialization,
"failed to parse endpoint URL", err)
}
return nil
return aws.BoolValue(req.Config.UseDualStack)
}

View File

@@ -0,0 +1,242 @@
package s3
import (
"net/url"
"strings"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/endpoints"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/internal/s3shared"
"github.com/aws/aws-sdk-go/internal/s3shared/arn"
"github.com/aws/aws-sdk-go/private/protocol"
)
const (
accessPointPrefixLabel = "accesspoint"
accountIDPrefixLabel = "accountID"
accessPointPrefixTemplate = "{" + accessPointPrefixLabel + "}-{" + accountIDPrefixLabel + "}."
outpostPrefixLabel = "outpost"
outpostAccessPointPrefixTemplate = accessPointPrefixTemplate + "{" + outpostPrefixLabel + "}."
)
// hasCustomEndpoint returns true if endpoint is a custom endpoint
func hasCustomEndpoint(r *request.Request) bool {
return len(aws.StringValue(r.Config.Endpoint)) > 0
}
// accessPointEndpointBuilder represents the endpoint builder for access point arn
type accessPointEndpointBuilder arn.AccessPointARN
// build builds the endpoint for corresponding access point arn
//
// For building an endpoint from access point arn, format used is:
// - Access point endpoint format : {accesspointName}-{accountId}.s3-accesspoint.{region}.{dnsSuffix}
// - example : myaccesspoint-012345678901.s3-accesspoint.us-west-2.amazonaws.com
//
// Access Point Endpoint requests are signed using "s3" as signing name.
//
func (a accessPointEndpointBuilder) build(req *request.Request) error {
resolveService := arn.AccessPointARN(a).Service
resolveRegion := arn.AccessPointARN(a).Region
endpoint, err := resolveRegionalEndpoint(req, resolveRegion, "", resolveService)
if err != nil {
return s3shared.NewFailedToResolveEndpointError(arn.AccessPointARN(a),
req.ClientInfo.PartitionID, resolveRegion, err)
}
endpoint.URL = endpoints.AddScheme(endpoint.URL, aws.BoolValue(req.Config.DisableSSL))
if !hasCustomEndpoint(req) {
if err = updateRequestEndpoint(req, endpoint.URL); err != nil {
return err
}
// dual stack provided by endpoint resolver
updateS3HostForS3AccessPoint(req)
}
protocol.HostPrefixBuilder{
Prefix: accessPointPrefixTemplate,
LabelsFn: a.hostPrefixLabelValues,
}.Build(req)
// signer redirection
redirectSigner(req, endpoint.SigningName, endpoint.SigningRegion)
err = protocol.ValidateEndpointHost(req.Operation.Name, req.HTTPRequest.URL.Host)
if err != nil {
return s3shared.NewInvalidARNError(arn.AccessPointARN(a), err)
}
return nil
}
func (a accessPointEndpointBuilder) hostPrefixLabelValues() map[string]string {
return map[string]string{
accessPointPrefixLabel: arn.AccessPointARN(a).AccessPointName,
accountIDPrefixLabel: arn.AccessPointARN(a).AccountID,
}
}
// s3ObjectLambdaAccessPointEndpointBuilder represents the endpoint builder for an s3 object lambda access point arn
type s3ObjectLambdaAccessPointEndpointBuilder arn.S3ObjectLambdaAccessPointARN
// build builds the endpoint for corresponding access point arn
//
// For building an endpoint from access point arn, format used is:
// - Access point endpoint format : {accesspointName}-{accountId}.s3-object-lambda.{region}.{dnsSuffix}
// - example : myaccesspoint-012345678901.s3-object-lambda.us-west-2.amazonaws.com
//
// Access Point Endpoint requests are signed using "s3-object-lambda" as signing name.
//
func (a s3ObjectLambdaAccessPointEndpointBuilder) build(req *request.Request) error {
resolveRegion := arn.S3ObjectLambdaAccessPointARN(a).Region
endpoint, err := resolveRegionalEndpoint(req, resolveRegion, "", EndpointsID)
if err != nil {
return s3shared.NewFailedToResolveEndpointError(arn.S3ObjectLambdaAccessPointARN(a),
req.ClientInfo.PartitionID, resolveRegion, err)
}
endpoint.URL = endpoints.AddScheme(endpoint.URL, aws.BoolValue(req.Config.DisableSSL))
endpoint.SigningName = s3ObjectsLambdaNamespace
if !hasCustomEndpoint(req) {
if err = updateRequestEndpoint(req, endpoint.URL); err != nil {
return err
}
updateS3HostPrefixForS3ObjectLambda(req)
}
protocol.HostPrefixBuilder{
Prefix: accessPointPrefixTemplate,
LabelsFn: a.hostPrefixLabelValues,
}.Build(req)
// signer redirection
redirectSigner(req, endpoint.SigningName, endpoint.SigningRegion)
err = protocol.ValidateEndpointHost(req.Operation.Name, req.HTTPRequest.URL.Host)
if err != nil {
return s3shared.NewInvalidARNError(arn.S3ObjectLambdaAccessPointARN(a), err)
}
return nil
}
func (a s3ObjectLambdaAccessPointEndpointBuilder) hostPrefixLabelValues() map[string]string {
return map[string]string{
accessPointPrefixLabel: arn.S3ObjectLambdaAccessPointARN(a).AccessPointName,
accountIDPrefixLabel: arn.S3ObjectLambdaAccessPointARN(a).AccountID,
}
}
// outpostAccessPointEndpointBuilder represents the Endpoint builder for outpost access point arn.
type outpostAccessPointEndpointBuilder arn.OutpostAccessPointARN
// build builds an endpoint corresponding to the outpost access point arn.
//
// For building an endpoint from outpost access point arn, format used is:
// - Outpost access point endpoint format : {accesspointName}-{accountId}.{outpostId}.s3-outposts.{region}.{dnsSuffix}
// - example : myaccesspoint-012345678901.op-01234567890123456.s3-outposts.us-west-2.amazonaws.com
//
// Outpost AccessPoint Endpoint request are signed using "s3-outposts" as signing name.
//
func (o outpostAccessPointEndpointBuilder) build(req *request.Request) error {
resolveRegion := o.Region
resolveService := o.Service
endpointsID := resolveService
if resolveService == s3OutpostsNamespace {
endpointsID = "s3"
}
endpoint, err := resolveRegionalEndpoint(req, resolveRegion, "", endpointsID)
if err != nil {
return s3shared.NewFailedToResolveEndpointError(o,
req.ClientInfo.PartitionID, resolveRegion, err)
}
endpoint.URL = endpoints.AddScheme(endpoint.URL, aws.BoolValue(req.Config.DisableSSL))
if !hasCustomEndpoint(req) {
if err = updateRequestEndpoint(req, endpoint.URL); err != nil {
return err
}
updateHostPrefix(req, endpointsID, resolveService)
}
protocol.HostPrefixBuilder{
Prefix: outpostAccessPointPrefixTemplate,
LabelsFn: o.hostPrefixLabelValues,
}.Build(req)
// set the signing region, name to resolved names from ARN
redirectSigner(req, resolveService, resolveRegion)
err = protocol.ValidateEndpointHost(req.Operation.Name, req.HTTPRequest.URL.Host)
if err != nil {
return s3shared.NewInvalidARNError(o, err)
}
return nil
}
func (o outpostAccessPointEndpointBuilder) hostPrefixLabelValues() map[string]string {
return map[string]string{
accessPointPrefixLabel: o.AccessPointName,
accountIDPrefixLabel: o.AccountID,
outpostPrefixLabel: o.OutpostID,
}
}
func resolveRegionalEndpoint(r *request.Request, region, resolvedRegion, endpointsID string) (endpoints.ResolvedEndpoint, error) {
return r.Config.EndpointResolver.EndpointFor(endpointsID, region, func(opts *endpoints.Options) {
opts.DisableSSL = aws.BoolValue(r.Config.DisableSSL)
opts.UseDualStack = aws.BoolValue(r.Config.UseDualStack)
opts.UseDualStackEndpoint = r.Config.UseDualStackEndpoint
opts.UseFIPSEndpoint = r.Config.UseFIPSEndpoint
opts.S3UsEast1RegionalEndpoint = endpoints.RegionalS3UsEast1Endpoint
opts.ResolvedRegion = resolvedRegion
opts.Logger = r.Config.Logger
opts.LogDeprecated = r.Config.LogLevel.Matches(aws.LogDebugWithDeprecated)
})
}
func updateRequestEndpoint(r *request.Request, endpoint string) (err error) {
r.HTTPRequest.URL, err = url.Parse(endpoint + r.Operation.HTTPPath)
if err != nil {
return awserr.New(request.ErrCodeSerialization,
"failed to parse endpoint URL", err)
}
return nil
}
// redirectSigner sets signing name, signing region for a request
func redirectSigner(req *request.Request, signingName string, signingRegion string) {
req.ClientInfo.SigningName = signingName
req.ClientInfo.SigningRegion = signingRegion
}
func updateS3HostForS3AccessPoint(req *request.Request) {
updateHostPrefix(req, "s3", s3AccessPointNamespace)
}
func updateS3HostPrefixForS3ObjectLambda(req *request.Request) {
updateHostPrefix(req, "s3", s3ObjectsLambdaNamespace)
}
func updateHostPrefix(req *request.Request, oldEndpointPrefix, newEndpointPrefix string) {
host := req.HTTPRequest.URL.Host
if strings.HasPrefix(host, oldEndpointPrefix) {
// replace service hostlabel oldEndpointPrefix to newEndpointPrefix
req.HTTPRequest.URL.Host = newEndpointPrefix + host[len(oldEndpointPrefix):]
}
}

View File

@@ -1,151 +0,0 @@
package s3
import (
"fmt"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/service/s3/internal/arn"
)
const (
invalidARNErrorErrCode = "InvalidARNError"
configurationErrorErrCode = "ConfigurationError"
)
type invalidARNError struct {
message string
resource arn.Resource
origErr error
}
func (e invalidARNError) Error() string {
var extra string
if e.resource != nil {
extra = "ARN: " + e.resource.String()
}
return awserr.SprintError(e.Code(), e.Message(), extra, e.origErr)
}
func (e invalidARNError) Code() string {
return invalidARNErrorErrCode
}
func (e invalidARNError) Message() string {
return e.message
}
func (e invalidARNError) OrigErr() error {
return e.origErr
}
func newInvalidARNError(resource arn.Resource, err error) invalidARNError {
return invalidARNError{
message: "invalid ARN",
origErr: err,
resource: resource,
}
}
func newInvalidARNWithCustomEndpointError(resource arn.Resource, err error) invalidARNError {
return invalidARNError{
message: "resource ARN not supported with custom client endpoints",
origErr: err,
resource: resource,
}
}
// ARN not supported for the target partition
func newInvalidARNWithUnsupportedPartitionError(resource arn.Resource, err error) invalidARNError {
return invalidARNError{
message: "resource ARN not supported for the target ARN partition",
origErr: err,
resource: resource,
}
}
type configurationError struct {
message string
resource arn.Resource
clientPartitionID string
clientRegion string
origErr error
}
func (e configurationError) Error() string {
extra := fmt.Sprintf("ARN: %s, client partition: %s, client region: %s",
e.resource, e.clientPartitionID, e.clientRegion)
return awserr.SprintError(e.Code(), e.Message(), extra, e.origErr)
}
func (e configurationError) Code() string {
return configurationErrorErrCode
}
func (e configurationError) Message() string {
return e.message
}
func (e configurationError) OrigErr() error {
return e.origErr
}
func newClientPartitionMismatchError(resource arn.Resource, clientPartitionID, clientRegion string, err error) configurationError {
return configurationError{
message: "client partition does not match provided ARN partition",
origErr: err,
resource: resource,
clientPartitionID: clientPartitionID,
clientRegion: clientRegion,
}
}
func newClientRegionMismatchError(resource arn.Resource, clientPartitionID, clientRegion string, err error) configurationError {
return configurationError{
message: "client region does not match provided ARN region",
origErr: err,
resource: resource,
clientPartitionID: clientPartitionID,
clientRegion: clientRegion,
}
}
func newFailedToResolveEndpointError(resource arn.Resource, clientPartitionID, clientRegion string, err error) configurationError {
return configurationError{
message: "endpoint resolver failed to find an endpoint for the provided ARN region",
origErr: err,
resource: resource,
clientPartitionID: clientPartitionID,
clientRegion: clientRegion,
}
}
func newClientConfiguredForFIPSError(resource arn.Resource, clientPartitionID, clientRegion string, err error) configurationError {
return configurationError{
message: "client configured for fips but cross-region resource ARN provided",
origErr: err,
resource: resource,
clientPartitionID: clientPartitionID,
clientRegion: clientRegion,
}
}
func newClientConfiguredForAccelerateError(resource arn.Resource, clientPartitionID, clientRegion string, err error) configurationError {
return configurationError{
message: "client configured for S3 Accelerate but is supported with resource ARN",
origErr: err,
resource: resource,
clientPartitionID: clientPartitionID,
clientRegion: clientRegion,
}
}
func newClientConfiguredForCrossRegionFIPSError(resource arn.Resource, clientPartitionID, clientRegion string, err error) configurationError {
return configurationError{
message: "client configured for FIPS with cross-region enabled but is supported with cross-region resource ARN",
origErr: err,
resource: resource,
clientPartitionID: clientPartitionID,
clientRegion: clientRegion,
}
}

View File

@@ -8,19 +8,25 @@ const (
// "BucketAlreadyExists".
//
// The requested bucket name is not available. The bucket namespace is shared
// by all users of the system. Please select a different name and try again.
// by all users of the system. Select a different name and try again.
ErrCodeBucketAlreadyExists = "BucketAlreadyExists"
// ErrCodeBucketAlreadyOwnedByYou for service response error code
// "BucketAlreadyOwnedByYou".
//
// The bucket you tried to create already exists, and you own it. Amazon S3
// returns this error in all AWS Regions except in the North Virginia Region.
// For legacy compatibility, if you re-create an existing bucket that you already
// own in the North Virginia Region, Amazon S3 returns 200 OK and resets the
// bucket access control lists (ACLs).
// returns this error in all Amazon Web Services Regions except in the North
// Virginia Region. For legacy compatibility, if you re-create an existing bucket
// that you already own in the North Virginia Region, Amazon S3 returns 200
// OK and resets the bucket access control lists (ACLs).
ErrCodeBucketAlreadyOwnedByYou = "BucketAlreadyOwnedByYou"
// ErrCodeInvalidObjectState for service response error code
// "InvalidObjectState".
//
// Object is archived and inaccessible until restored.
ErrCodeInvalidObjectState = "InvalidObjectState"
// ErrCodeNoSuchBucket for service response error code
// "NoSuchBucket".
//
@@ -42,13 +48,13 @@ const (
// ErrCodeObjectAlreadyInActiveTierError for service response error code
// "ObjectAlreadyInActiveTierError".
//
// This operation is not allowed against this storage tier.
// This action is not allowed against this storage tier.
ErrCodeObjectAlreadyInActiveTierError = "ObjectAlreadyInActiveTierError"
// ErrCodeObjectNotInActiveTierError for service response error code
// "ObjectNotInActiveTierError".
//
// The source object of the COPY operation is not in the active tier and is
// only stored in Amazon S3 Glacier.
// The source object of the COPY action is not in the active tier and is only
// stored in Amazon S3 Glacier.
ErrCodeObjectNotInActiveTierError = "ObjectNotInActiveTierError"
)

View File

@@ -1,45 +0,0 @@
package arn
import (
"strings"
"github.com/aws/aws-sdk-go/aws/arn"
)
// AccessPointARN provides representation
type AccessPointARN struct {
arn.ARN
AccessPointName string
}
// GetARN returns the base ARN for the Access Point resource
func (a AccessPointARN) GetARN() arn.ARN {
return a.ARN
}
// ParseAccessPointResource attempts to parse the ARN's resource as an
// AccessPoint resource.
func ParseAccessPointResource(a arn.ARN, resParts []string) (AccessPointARN, error) {
if len(a.Region) == 0 {
return AccessPointARN{}, InvalidARNError{a, "region not set"}
}
if len(a.AccountID) == 0 {
return AccessPointARN{}, InvalidARNError{a, "account-id not set"}
}
if len(resParts) == 0 {
return AccessPointARN{}, InvalidARNError{a, "resource-id not set"}
}
if len(resParts) > 1 {
return AccessPointARN{}, InvalidARNError{a, "sub resource not supported"}
}
resID := resParts[0]
if len(strings.TrimSpace(resID)) == 0 {
return AccessPointARN{}, InvalidARNError{a, "resource-id not set"}
}
return AccessPointARN{
ARN: a,
AccessPointName: resID,
}, nil
}

View File

@@ -1,71 +0,0 @@
package arn
import (
"strings"
"github.com/aws/aws-sdk-go/aws/arn"
)
// Resource provides the interfaces abstracting ARNs of specific resource
// types.
type Resource interface {
GetARN() arn.ARN
String() string
}
// ResourceParser provides the function for parsing an ARN's resource
// component into a typed resource.
type ResourceParser func(arn.ARN) (Resource, error)
// ParseResource parses an AWS ARN into a typed resource for the S3 API.
func ParseResource(s string, resParser ResourceParser) (resARN Resource, err error) {
a, err := arn.Parse(s)
if err != nil {
return nil, err
}
if len(a.Partition) == 0 {
return nil, InvalidARNError{a, "partition not set"}
}
if a.Service != "s3" {
return nil, InvalidARNError{a, "service is not S3"}
}
if len(a.Resource) == 0 {
return nil, InvalidARNError{a, "resource not set"}
}
return resParser(a)
}
// SplitResource splits the resource components by the ARN resource delimiters.
func SplitResource(v string) []string {
var parts []string
var offset int
for offset <= len(v) {
idx := strings.IndexAny(v[offset:], "/:")
if idx < 0 {
parts = append(parts, v[offset:])
break
}
parts = append(parts, v[offset:idx+offset])
offset += idx + 1
}
return parts
}
// IsARN returns whether the given string is an ARN
func IsARN(s string) bool {
return arn.IsARN(s)
}
// InvalidARNError provides the error for an invalid ARN error.
type InvalidARNError struct {
ARN arn.ARN
Reason string
}
func (e InvalidARNError) Error() string {
return "invalid Amazon S3 ARN, " + e.Reason + ", " + e.ARN.String()
}

View File

@@ -1,3 +1,4 @@
//go:build !go1.6
// +build !go1.6
package s3

View File

@@ -1,3 +1,4 @@
//go:build go1.6
// +build go1.6
package s3

View File

@@ -96,6 +96,10 @@ type S3API interface {
DeleteBucketEncryptionWithContext(aws.Context, *s3.DeleteBucketEncryptionInput, ...request.Option) (*s3.DeleteBucketEncryptionOutput, error)
DeleteBucketEncryptionRequest(*s3.DeleteBucketEncryptionInput) (*request.Request, *s3.DeleteBucketEncryptionOutput)
DeleteBucketIntelligentTieringConfiguration(*s3.DeleteBucketIntelligentTieringConfigurationInput) (*s3.DeleteBucketIntelligentTieringConfigurationOutput, error)
DeleteBucketIntelligentTieringConfigurationWithContext(aws.Context, *s3.DeleteBucketIntelligentTieringConfigurationInput, ...request.Option) (*s3.DeleteBucketIntelligentTieringConfigurationOutput, error)
DeleteBucketIntelligentTieringConfigurationRequest(*s3.DeleteBucketIntelligentTieringConfigurationInput) (*request.Request, *s3.DeleteBucketIntelligentTieringConfigurationOutput)
DeleteBucketInventoryConfiguration(*s3.DeleteBucketInventoryConfigurationInput) (*s3.DeleteBucketInventoryConfigurationOutput, error)
DeleteBucketInventoryConfigurationWithContext(aws.Context, *s3.DeleteBucketInventoryConfigurationInput, ...request.Option) (*s3.DeleteBucketInventoryConfigurationOutput, error)
DeleteBucketInventoryConfigurationRequest(*s3.DeleteBucketInventoryConfigurationInput) (*request.Request, *s3.DeleteBucketInventoryConfigurationOutput)
@@ -108,6 +112,10 @@ type S3API interface {
DeleteBucketMetricsConfigurationWithContext(aws.Context, *s3.DeleteBucketMetricsConfigurationInput, ...request.Option) (*s3.DeleteBucketMetricsConfigurationOutput, error)
DeleteBucketMetricsConfigurationRequest(*s3.DeleteBucketMetricsConfigurationInput) (*request.Request, *s3.DeleteBucketMetricsConfigurationOutput)
DeleteBucketOwnershipControls(*s3.DeleteBucketOwnershipControlsInput) (*s3.DeleteBucketOwnershipControlsOutput, error)
DeleteBucketOwnershipControlsWithContext(aws.Context, *s3.DeleteBucketOwnershipControlsInput, ...request.Option) (*s3.DeleteBucketOwnershipControlsOutput, error)
DeleteBucketOwnershipControlsRequest(*s3.DeleteBucketOwnershipControlsInput) (*request.Request, *s3.DeleteBucketOwnershipControlsOutput)
DeleteBucketPolicy(*s3.DeleteBucketPolicyInput) (*s3.DeleteBucketPolicyOutput, error)
DeleteBucketPolicyWithContext(aws.Context, *s3.DeleteBucketPolicyInput, ...request.Option) (*s3.DeleteBucketPolicyOutput, error)
DeleteBucketPolicyRequest(*s3.DeleteBucketPolicyInput) (*request.Request, *s3.DeleteBucketPolicyOutput)
@@ -160,6 +168,10 @@ type S3API interface {
GetBucketEncryptionWithContext(aws.Context, *s3.GetBucketEncryptionInput, ...request.Option) (*s3.GetBucketEncryptionOutput, error)
GetBucketEncryptionRequest(*s3.GetBucketEncryptionInput) (*request.Request, *s3.GetBucketEncryptionOutput)
GetBucketIntelligentTieringConfiguration(*s3.GetBucketIntelligentTieringConfigurationInput) (*s3.GetBucketIntelligentTieringConfigurationOutput, error)
GetBucketIntelligentTieringConfigurationWithContext(aws.Context, *s3.GetBucketIntelligentTieringConfigurationInput, ...request.Option) (*s3.GetBucketIntelligentTieringConfigurationOutput, error)
GetBucketIntelligentTieringConfigurationRequest(*s3.GetBucketIntelligentTieringConfigurationInput) (*request.Request, *s3.GetBucketIntelligentTieringConfigurationOutput)
GetBucketInventoryConfiguration(*s3.GetBucketInventoryConfigurationInput) (*s3.GetBucketInventoryConfigurationOutput, error)
GetBucketInventoryConfigurationWithContext(aws.Context, *s3.GetBucketInventoryConfigurationInput, ...request.Option) (*s3.GetBucketInventoryConfigurationOutput, error)
GetBucketInventoryConfigurationRequest(*s3.GetBucketInventoryConfigurationInput) (*request.Request, *s3.GetBucketInventoryConfigurationOutput)
@@ -192,6 +204,10 @@ type S3API interface {
GetBucketNotificationConfigurationWithContext(aws.Context, *s3.GetBucketNotificationConfigurationRequest, ...request.Option) (*s3.NotificationConfiguration, error)
GetBucketNotificationConfigurationRequest(*s3.GetBucketNotificationConfigurationRequest) (*request.Request, *s3.NotificationConfiguration)
GetBucketOwnershipControls(*s3.GetBucketOwnershipControlsInput) (*s3.GetBucketOwnershipControlsOutput, error)
GetBucketOwnershipControlsWithContext(aws.Context, *s3.GetBucketOwnershipControlsInput, ...request.Option) (*s3.GetBucketOwnershipControlsOutput, error)
GetBucketOwnershipControlsRequest(*s3.GetBucketOwnershipControlsInput) (*request.Request, *s3.GetBucketOwnershipControlsOutput)
GetBucketPolicy(*s3.GetBucketPolicyInput) (*s3.GetBucketPolicyOutput, error)
GetBucketPolicyWithContext(aws.Context, *s3.GetBucketPolicyInput, ...request.Option) (*s3.GetBucketPolicyOutput, error)
GetBucketPolicyRequest(*s3.GetBucketPolicyInput) (*request.Request, *s3.GetBucketPolicyOutput)
@@ -228,6 +244,10 @@ type S3API interface {
GetObjectAclWithContext(aws.Context, *s3.GetObjectAclInput, ...request.Option) (*s3.GetObjectAclOutput, error)
GetObjectAclRequest(*s3.GetObjectAclInput) (*request.Request, *s3.GetObjectAclOutput)
GetObjectAttributes(*s3.GetObjectAttributesInput) (*s3.GetObjectAttributesOutput, error)
GetObjectAttributesWithContext(aws.Context, *s3.GetObjectAttributesInput, ...request.Option) (*s3.GetObjectAttributesOutput, error)
GetObjectAttributesRequest(*s3.GetObjectAttributesInput) (*request.Request, *s3.GetObjectAttributesOutput)
GetObjectLegalHold(*s3.GetObjectLegalHoldInput) (*s3.GetObjectLegalHoldOutput, error)
GetObjectLegalHoldWithContext(aws.Context, *s3.GetObjectLegalHoldInput, ...request.Option) (*s3.GetObjectLegalHoldOutput, error)
GetObjectLegalHoldRequest(*s3.GetObjectLegalHoldInput) (*request.Request, *s3.GetObjectLegalHoldOutput)
@@ -264,6 +284,10 @@ type S3API interface {
ListBucketAnalyticsConfigurationsWithContext(aws.Context, *s3.ListBucketAnalyticsConfigurationsInput, ...request.Option) (*s3.ListBucketAnalyticsConfigurationsOutput, error)
ListBucketAnalyticsConfigurationsRequest(*s3.ListBucketAnalyticsConfigurationsInput) (*request.Request, *s3.ListBucketAnalyticsConfigurationsOutput)
ListBucketIntelligentTieringConfigurations(*s3.ListBucketIntelligentTieringConfigurationsInput) (*s3.ListBucketIntelligentTieringConfigurationsOutput, error)
ListBucketIntelligentTieringConfigurationsWithContext(aws.Context, *s3.ListBucketIntelligentTieringConfigurationsInput, ...request.Option) (*s3.ListBucketIntelligentTieringConfigurationsOutput, error)
ListBucketIntelligentTieringConfigurationsRequest(*s3.ListBucketIntelligentTieringConfigurationsInput) (*request.Request, *s3.ListBucketIntelligentTieringConfigurationsOutput)
ListBucketInventoryConfigurations(*s3.ListBucketInventoryConfigurationsInput) (*s3.ListBucketInventoryConfigurationsOutput, error)
ListBucketInventoryConfigurationsWithContext(aws.Context, *s3.ListBucketInventoryConfigurationsInput, ...request.Option) (*s3.ListBucketInventoryConfigurationsOutput, error)
ListBucketInventoryConfigurationsRequest(*s3.ListBucketInventoryConfigurationsInput) (*request.Request, *s3.ListBucketInventoryConfigurationsOutput)
@@ -331,6 +355,10 @@ type S3API interface {
PutBucketEncryptionWithContext(aws.Context, *s3.PutBucketEncryptionInput, ...request.Option) (*s3.PutBucketEncryptionOutput, error)
PutBucketEncryptionRequest(*s3.PutBucketEncryptionInput) (*request.Request, *s3.PutBucketEncryptionOutput)
PutBucketIntelligentTieringConfiguration(*s3.PutBucketIntelligentTieringConfigurationInput) (*s3.PutBucketIntelligentTieringConfigurationOutput, error)
PutBucketIntelligentTieringConfigurationWithContext(aws.Context, *s3.PutBucketIntelligentTieringConfigurationInput, ...request.Option) (*s3.PutBucketIntelligentTieringConfigurationOutput, error)
PutBucketIntelligentTieringConfigurationRequest(*s3.PutBucketIntelligentTieringConfigurationInput) (*request.Request, *s3.PutBucketIntelligentTieringConfigurationOutput)
PutBucketInventoryConfiguration(*s3.PutBucketInventoryConfigurationInput) (*s3.PutBucketInventoryConfigurationOutput, error)
PutBucketInventoryConfigurationWithContext(aws.Context, *s3.PutBucketInventoryConfigurationInput, ...request.Option) (*s3.PutBucketInventoryConfigurationOutput, error)
PutBucketInventoryConfigurationRequest(*s3.PutBucketInventoryConfigurationInput) (*request.Request, *s3.PutBucketInventoryConfigurationOutput)
@@ -359,6 +387,10 @@ type S3API interface {
PutBucketNotificationConfigurationWithContext(aws.Context, *s3.PutBucketNotificationConfigurationInput, ...request.Option) (*s3.PutBucketNotificationConfigurationOutput, error)
PutBucketNotificationConfigurationRequest(*s3.PutBucketNotificationConfigurationInput) (*request.Request, *s3.PutBucketNotificationConfigurationOutput)
PutBucketOwnershipControls(*s3.PutBucketOwnershipControlsInput) (*s3.PutBucketOwnershipControlsOutput, error)
PutBucketOwnershipControlsWithContext(aws.Context, *s3.PutBucketOwnershipControlsInput, ...request.Option) (*s3.PutBucketOwnershipControlsOutput, error)
PutBucketOwnershipControlsRequest(*s3.PutBucketOwnershipControlsInput) (*request.Request, *s3.PutBucketOwnershipControlsOutput)
PutBucketPolicy(*s3.PutBucketPolicyInput) (*s3.PutBucketPolicyOutput, error)
PutBucketPolicyWithContext(aws.Context, *s3.PutBucketPolicyInput, ...request.Option) (*s3.PutBucketPolicyOutput, error)
PutBucketPolicyRequest(*s3.PutBucketPolicyInput) (*request.Request, *s3.PutBucketPolicyOutput)
@@ -427,6 +459,10 @@ type S3API interface {
UploadPartCopyWithContext(aws.Context, *s3.UploadPartCopyInput, ...request.Option) (*s3.UploadPartCopyOutput, error)
UploadPartCopyRequest(*s3.UploadPartCopyInput) (*request.Request, *s3.UploadPartCopyOutput)
WriteGetObjectResponse(*s3.WriteGetObjectResponseInput) (*s3.WriteGetObjectResponseOutput, error)
WriteGetObjectResponseWithContext(aws.Context, *s3.WriteGetObjectResponseInput, ...request.Option) (*s3.WriteGetObjectResponseOutput, error)
WriteGetObjectResponseRequest(*s3.WriteGetObjectResponseInput) (*request.Request, *s3.WriteGetObjectResponseOutput)
WaitUntilBucketExists(*s3.HeadBucketInput) error
WaitUntilBucketExistsWithContext(aws.Context, *s3.HeadBucketInput, ...request.WaiterOption) error

View File

@@ -0,0 +1,23 @@
package s3manager
import (
"fmt"
"github.com/aws/aws-sdk-go/aws/arn"
)
func validateSupportedARNType(bucket string) error {
if !arn.IsARN(bucket) {
return nil
}
parsedARN, err := arn.Parse(bucket)
if err != nil {
return err
}
if parsedARN.Service == "s3-object-lambda" {
return fmt.Errorf("manager does not support s3-object-lambda service ARNs")
}
return nil
}

View File

@@ -3,6 +3,7 @@ package s3manager
import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/client"
"github.com/aws/aws-sdk-go/aws/corehandlers"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/service/s3"
@@ -35,6 +36,30 @@ import (
// }
// fmt.Printf("Bucket %s is in %s region\n", bucket, region)
//
// By default the request will be made to the Amazon S3 endpoint using the Path
// style addressing.
//
// s3.us-west-2.amazonaws.com/bucketname
//
// This is not compatible with Amazon S3's FIPS endpoints. To override this
// behavior to use Virtual Host style addressing, provide a functional option
// that will set the Request's Config.S3ForcePathStyle to aws.Bool(false).
//
// region, err := s3manager.GetBucketRegion(ctx, sess, "bucketname", "us-west-2", func(r *request.Request) {
// r.S3ForcePathStyle = aws.Bool(false)
// })
//
// To configure the GetBucketRegion to make a request via the Amazon
// S3 FIPS endpoints directly when a FIPS region name is not available, (e.g.
// fips-us-gov-west-1) set the Config.Endpoint on the Session, or client the
// utility is called with. The hint region will be ignored if an endpoint URL
// is configured on the session or client.
//
// sess, err := session.NewSession(&aws.Config{
// Endpoint: aws.String("https://s3-fips.us-west-2.amazonaws.com"),
// })
//
// region, err := s3manager.GetBucketRegion(context.Background(), sess, "bucketname", "")
func GetBucketRegion(ctx aws.Context, c client.ConfigProvider, bucket, regionHint string, opts ...request.Option) (string, error) {
var cfg aws.Config
if len(regionHint) != 0 {
@@ -50,12 +75,38 @@ const bucketRegionHeader = "X-Amz-Bucket-Region"
// that it takes a S3 service client instead of a Session. The regionHint is
// derived from the region the S3 service client was created in.
//
// By default the request will be made to the Amazon S3 endpoint using the Path
// style addressing.
//
// s3.us-west-2.amazonaws.com/bucketname
//
// This is not compatible with Amazon S3's FIPS endpoints. To override this
// behavior to use Virtual Host style addressing, provide a functional option
// that will set the Request's Config.S3ForcePathStyle to aws.Bool(false).
//
// region, err := s3manager.GetBucketRegionWithClient(ctx, client, "bucketname", func(r *request.Request) {
// r.S3ForcePathStyle = aws.Bool(false)
// })
//
// To configure the GetBucketRegion to make a request via the Amazon
// S3 FIPS endpoints directly when a FIPS region name is not available, (e.g.
// fips-us-gov-west-1) set the Config.Endpoint on the Session, or client the
// utility is called with. The hint region will be ignored if an endpoint URL
// is configured on the session or client.
//
// region, err := s3manager.GetBucketRegionWithClient(context.Background(),
// s3.New(sess, &aws.Config{
// Endpoint: aws.String("https://s3-fips.us-west-2.amazonaws.com"),
// }),
// "bucketname")
//
// See GetBucketRegion for more information.
func GetBucketRegionWithClient(ctx aws.Context, svc s3iface.S3API, bucket string, opts ...request.Option) (string, error) {
req, _ := svc.HeadBucketRequest(&s3.HeadBucketInput{
Bucket: aws.String(bucket),
})
req.Config.S3ForcePathStyle = aws.Bool(true)
req.Config.Credentials = credentials.AnonymousCredentials
req.SetContext(ctx)
@@ -75,6 +126,16 @@ func GetBucketRegionWithClient(ctx aws.Context, svc s3iface.S3API, bucket string
r.HTTPResponse.Status = "OK"
r.Error = nil
})
// Replace the endpoint validation handler to not require a region if an
// endpoint URL was specified. Since these requests are not authenticated,
// requiring a region is not needed when an endpoint URL is provided.
req.Handlers.Validate.Swap(
corehandlers.ValidateEndpointHandler.Name,
request.NamedHandler{
Name: "validateEndpointWithoutRegion",
Fn: validateEndpointWithoutRegion,
},
)
req.ApplyOptions(opts...)
@@ -86,3 +147,13 @@ func GetBucketRegionWithClient(ctx aws.Context, svc s3iface.S3API, bucket string
return bucketRegion, nil
}
func validateEndpointWithoutRegion(r *request.Request) {
// Check if the caller provided an explicit URL instead of one derived by
// the SDK's endpoint resolver. For GetBucketRegion, with an explicit
// endpoint URL, a region is not needed. If no endpoint URL is provided,
// fallback the SDK's standard endpoint validation handler.
if len(aws.StringValue(r.Config.Endpoint)) == 0 {
corehandlers.ValidateEndpointHandler.Fn(r)
}
}

View File

@@ -1,3 +1,4 @@
//go:build !windows
// +build !windows
package s3manager

View File

@@ -1,3 +1,4 @@
//go:build !windows
// +build !windows
package s3manager

View File

@@ -192,6 +192,10 @@ func (d Downloader) Download(w io.WriterAt, input *s3.GetObjectInput, options ..
// to perform a single GetObjectInput request for that object's range. This will
// caused the part size, and concurrency configurations to be ignored.
func (d Downloader) DownloadWithContext(ctx aws.Context, w io.WriterAt, input *s3.GetObjectInput, options ...func(*Downloader)) (n int64, err error) {
if err := validateSupportedARNType(aws.StringValue(input.Bucket)); err != nil {
return 0, err
}
impl := downloader{w: w, in: input, cfg: d, ctx: ctx}
for _, option := range options {

View File

@@ -60,6 +60,14 @@ func (p *maxSlicePool) Get(ctx aws.Context) (*[]byte, error) {
return nil, errZeroCapacity
}
return bs, nil
case <-ctx.Done():
p.mtx.RUnlock()
return nil, ctx.Err()
default:
// pass
}
select {
case _, ok := <-p.allocations:
p.mtx.RUnlock()
if !ok {

View File

@@ -109,6 +109,9 @@ type UploadOutput struct {
// The ID for a multipart upload to S3. In the case of an error the error
// can be cast to the MultiUploadFailure interface to extract the upload ID.
UploadID string
// Entity tag of the object.
ETag *string
}
// WithUploaderRequestOptions appends to the Uploader's API request options.
@@ -121,6 +124,14 @@ func WithUploaderRequestOptions(opts ...request.Option) func(*Uploader) {
// The Uploader structure that calls Upload(). It is safe to call Upload()
// on this structure for multiple objects and across concurrent goroutines.
// Mutating the Uploader's properties is not safe to be done concurrently.
//
// The ContentMD5 member for pre-computed MD5 checksums will be ignored for
// multipart uploads. Objects that will be uploaded in a single part, the
// ContentMD5 will be used.
//
// The Checksum members for pre-computed checksums will be ignored for
// multipart uploads. Objects that will be uploaded in a single part, will
// include the checksum member in the request.
type Uploader struct {
// The buffer size (in bytes) to use when buffering data into chunks and
// sending them as parts to S3. The minimum allowed part size is 5MB, and
@@ -388,6 +399,10 @@ func (u *uploader) upload() (*UploadOutput, error) {
// init will initialize all default options.
func (u *uploader) init() error {
if err := validateSupportedARNType(aws.StringValue(u.in.Bucket)); err != nil {
return err
}
if u.cfg.Concurrency == 0 {
u.cfg.Concurrency = DefaultUploadConcurrency
}
@@ -527,6 +542,7 @@ func (u *uploader) singlePart(r io.ReadSeeker, cleanup func()) (*UploadOutput, e
return &UploadOutput{
Location: url,
VersionID: out.VersionId,
ETag: out.ETag,
}, nil
}
@@ -632,6 +648,7 @@ func (u *multiuploader) upload(firstBuf io.ReadSeeker, cleanup func()) (*UploadO
Location: uploadLocation,
VersionID: complete.VersionId,
UploadID: u.uploadID,
ETag: complete.ETag,
}, nil
}

View File

@@ -11,33 +11,105 @@ import (
// to an object in an Amazon S3 bucket. This type is similar to the s3
// package's PutObjectInput with the exception that the Body member is an
// io.Reader instead of an io.ReadSeeker.
//
// The ContentMD5 member for pre-computed MD5 checksums will be ignored for
// multipart uploads. Objects that will be uploaded in a single part, the
// ContentMD5 will be used.
//
// The Checksum members for pre-computed checksums will be ignored for
// multipart uploads. Objects that will be uploaded in a single part, will
// include the checksum member in the request.
type UploadInput struct {
_ struct{} `locationName:"PutObjectRequest" type:"structure" payload:"Body"`
// The canned ACL to apply to the object. For more information, see Canned ACL
// (https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#CannedACL).
//
// This action is not supported by Amazon S3 on Outposts.
ACL *string `location:"header" locationName:"x-amz-acl" type:"string" enum:"ObjectCannedACL"`
// The readable body payload to send to S3.
Body io.Reader
// Bucket name to which the PUT operation was initiated.
// The bucket name to which the PUT action was initiated.
//
// When using this API with an access point, you must direct requests to the
// access point hostname. The access point hostname takes the form AccessPointName-AccountId.s3-accesspoint.Region.amazonaws.com.
// When using this operation using an access point through the AWS SDKs, you
// provide the access point ARN in place of the bucket name. For more information
// about access point ARNs, see Using Access Points (https://docs.aws.amazon.com/AmazonS3/latest/dev/using-access-points.html)
// in the Amazon Simple Storage Service Developer Guide.
// When using this action with an access point, you must direct requests to
// the access point hostname. The access point hostname takes the form AccessPointName-AccountId.s3-accesspoint.Region.amazonaws.com.
// When using this action with an access point through the Amazon Web Services
// SDKs, you provide the access point ARN in place of the bucket name. For more
// information about access point ARNs, see Using access points (https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-access-points.html)
// in the Amazon S3 User Guide.
//
// When using this action with Amazon S3 on Outposts, you must direct requests
// to the S3 on Outposts hostname. The S3 on Outposts hostname takes the form
// AccessPointName-AccountId.outpostID.s3-outposts.Region.amazonaws.com. When
// using this action with S3 on Outposts through the Amazon Web Services SDKs,
// you provide the Outposts bucket ARN in place of the bucket name. For more
// information about S3 on Outposts ARNs, see Using Amazon S3 on Outposts (https://docs.aws.amazon.com/AmazonS3/latest/userguide/S3onOutposts.html)
// in the Amazon S3 User Guide.
//
// Bucket is a required field
Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"`
// Specifies whether Amazon S3 should use an S3 Bucket Key for object encryption
// with server-side encryption using AWS KMS (SSE-KMS). Setting this header
// to true causes Amazon S3 to use an S3 Bucket Key for object encryption with
// SSE-KMS.
//
// Specifying this header with a PUT action doesnt affect bucket-level settings
// for S3 Bucket Key.
BucketKeyEnabled *bool `location:"header" locationName:"x-amz-server-side-encryption-bucket-key-enabled" type:"boolean"`
// Can be used to specify caching behavior along the request/reply chain. For
// more information, see http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9
// (http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9).
CacheControl *string `location:"header" locationName:"Cache-Control" type:"string"`
// Indicates the algorithm used to create the checksum for the object when using
// the SDK. This header will not provide any additional functionality if not
// using the SDK. When sending this header, there must be a corresponding x-amz-checksum
// or x-amz-trailer header sent. Otherwise, Amazon S3 fails the request with
// the HTTP status code 400 Bad Request. For more information, see Checking
// object integrity (https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html)
// in the Amazon S3 User Guide.
//
// If you provide an individual checksum, Amazon S3 ignores any provided ChecksumAlgorithm
// parameter.
//
// The AWS SDK for Go v1 does not support automatic computing request payload
// checksum. This feature is available in the AWS SDK for Go v2. If a value
// is specified for this parameter, the matching algorithm's checksum member
// must be populated with the algorithm's checksum of the request payload.
ChecksumAlgorithm *string `location:"header" locationName:"x-amz-sdk-checksum-algorithm" type:"string" enum:"ChecksumAlgorithm"`
// This header can be used as a data integrity check to verify that the data
// received is the same data that was originally sent. This header specifies
// the base64-encoded, 32-bit CRC32 checksum of the object. For more information,
// see Checking object integrity (https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html)
// in the Amazon S3 User Guide.
ChecksumCRC32 *string `location:"header" locationName:"x-amz-checksum-crc32" type:"string"`
// This header can be used as a data integrity check to verify that the data
// received is the same data that was originally sent. This header specifies
// the base64-encoded, 32-bit CRC32C checksum of the object. For more information,
// see Checking object integrity (https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html)
// in the Amazon S3 User Guide.
ChecksumCRC32C *string `location:"header" locationName:"x-amz-checksum-crc32c" type:"string"`
// This header can be used as a data integrity check to verify that the data
// received is the same data that was originally sent. This header specifies
// the base64-encoded, 160-bit SHA-1 digest of the object. For more information,
// see Checking object integrity (https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html)
// in the Amazon S3 User Guide.
ChecksumSHA1 *string `location:"header" locationName:"x-amz-checksum-sha1" type:"string"`
// This header can be used as a data integrity check to verify that the data
// received is the same data that was originally sent. This header specifies
// the base64-encoded, 256-bit SHA-256 digest of the object. For more information,
// see Checking object integrity (https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html)
// in the Amazon S3 User Guide.
ChecksumSHA256 *string `location:"header" locationName:"x-amz-checksum-sha256" type:"string"`
// Specifies presentational information for the object. For more information,
// see http://www.w3.org/Protocols/rfc2616/rfc2616-sec19.html#sec19.5.1 (http://www.w3.org/Protocols/rfc2616/rfc2616-sec19.html#sec19.5.1).
ContentDisposition *string `location:"header" locationName:"Content-Disposition" type:"string"`
@@ -57,29 +129,45 @@ type UploadInput struct {
// it is optional, we recommend using the Content-MD5 mechanism as an end-to-end
// integrity check. For more information about REST request authentication,
// see REST Authentication (https://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html).
//
// If the ContentMD5 is provided for a multipart upload, it will be ignored.
// Objects that will be uploaded in a single part, the ContentMD5 will be used.
ContentMD5 *string `location:"header" locationName:"Content-MD5" type:"string"`
// A standard MIME type describing the format of the contents. For more information,
// see http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.17 (http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.17).
ContentType *string `location:"header" locationName:"Content-Type" type:"string"`
// The account ID of the expected bucket owner. If the bucket is owned by a
// different account, the request fails with the HTTP status code 403 Forbidden
// (access denied).
ExpectedBucketOwner *string `location:"header" locationName:"x-amz-expected-bucket-owner" type:"string"`
// The date and time at which the object is no longer cacheable. For more information,
// see http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.21 (http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.21).
Expires *time.Time `location:"header" locationName:"Expires" type:"timestamp"`
// Gives the grantee READ, READ_ACP, and WRITE_ACP permissions on the object.
//
// This action is not supported by Amazon S3 on Outposts.
GrantFullControl *string `location:"header" locationName:"x-amz-grant-full-control" type:"string"`
// Allows grantee to read the object data and its metadata.
//
// This action is not supported by Amazon S3 on Outposts.
GrantRead *string `location:"header" locationName:"x-amz-grant-read" type:"string"`
// Allows grantee to read the object ACL.
//
// This action is not supported by Amazon S3 on Outposts.
GrantReadACP *string `location:"header" locationName:"x-amz-grant-read-acp" type:"string"`
// Allows grantee to write the ACL for the applicable object.
//
// This action is not supported by Amazon S3 on Outposts.
GrantWriteACP *string `location:"header" locationName:"x-amz-grant-write-acp" type:"string"`
// Object key for which the PUT operation was initiated.
// Object key for which the PUT action was initiated.
//
// Key is a required field
Key *string `location:"uri" locationName:"Key" min:"1" type:"string" required:"true"`
@@ -94,14 +182,15 @@ type UploadInput struct {
// The Object Lock mode that you want to apply to this object.
ObjectLockMode *string `location:"header" locationName:"x-amz-object-lock-mode" type:"string" enum:"ObjectLockMode"`
// The date and time when you want this object's Object Lock to expire.
// The date and time when you want this object's Object Lock to expire. Must
// be formatted as a timestamp parameter.
ObjectLockRetainUntilDate *time.Time `location:"header" locationName:"x-amz-object-lock-retain-until-date" type:"timestamp" timestampFormat:"iso8601"`
// Confirms that the requester knows that they will be charged for the request.
// Bucket owners need not specify this parameter in their requests. For information
// about downloading objects from requester pays buckets, see Downloading Objects
// in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html)
// in the Amazon S3 Developer Guide.
// about downloading objects from Requester Pays buckets, see Downloading Objects
// in Requester Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html)
// in the Amazon S3 User Guide.
RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"`
// Specifies the algorithm to use to when encrypting the object (for example,
@@ -111,7 +200,7 @@ type UploadInput struct {
// Specifies the customer-provided encryption key for Amazon S3 to use in encrypting
// data. This value is used to store the object and then it is discarded; Amazon
// S3 does not store the encryption key. The key must be appropriate for use
// with the algorithm specified in the x-amz-server-side-encryption-customer-algorithm
// with the algorithm specified in the x-amz-server-side-encryption-customer-algorithm
// header.
SSECustomerKey *string `marshal-as:"blob" location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string" sensitive:"true"`
@@ -120,29 +209,31 @@ type UploadInput struct {
// encryption key was transmitted without error.
SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"`
// Specifies the AWS KMS Encryption Context to use for object encryption. The
// value of this header is a base64-encoded UTF-8 string holding JSON with the
// encryption context key-value pairs.
// Specifies the Amazon Web Services KMS Encryption Context to use for object
// encryption. The value of this header is a base64-encoded UTF-8 string holding
// JSON with the encryption context key-value pairs.
SSEKMSEncryptionContext *string `location:"header" locationName:"x-amz-server-side-encryption-context" type:"string" sensitive:"true"`
// If x-amz-server-side-encryption is present and has the value of aws:kms,
// this header specifies the ID of the AWS Key Management Service (AWS KMS)
// symmetrical customer managed customer master key (CMK) that was used for
// the object.
//
// If the value of x-amz-server-side-encryption is aws:kms, this header specifies
// the ID of the symmetric customer managed AWS KMS CMK that will be used for
// the object. If you specify x-amz-server-side-encryption:aws:kms, but do not
// providex-amz-server-side-encryption-aws-kms-key-id, Amazon S3 uses the AWS
// managed CMK in AWS to protect the data.
// this header specifies the ID of the Amazon Web Services Key Management Service
// (Amazon Web Services KMS) symmetrical customer managed key that was used
// for the object. If you specify x-amz-server-side-encryption:aws:kms, but
// do not providex-amz-server-side-encryption-aws-kms-key-id, Amazon S3 uses
// the Amazon Web Services managed key to protect the data. If the KMS key does
// not exist in the same account issuing the command, you must use the full
// ARN and not just the ID.
SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"`
// The server-side encryption algorithm used when storing this object in Amazon
// S3 (for example, AES256, aws:kms).
ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string" enum:"ServerSideEncryption"`
// If you don't specify, Standard is the default storage class. Amazon S3 supports
// other storage classes.
// By default, Amazon S3 uses the STANDARD Storage Class to store newly created
// objects. The STANDARD storage class provides high durability and high availability.
// Depending on performance needs, you can specify a different Storage Class.
// Amazon S3 on Outposts only uses the OUTPOSTS Storage Class. For more information,
// see Storage Classes (https://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html)
// in the Amazon S3 User Guide.
StorageClass *string `location:"header" locationName:"x-amz-storage-class" type:"string" enum:"StorageClass"`
// The tag-set for the object. The tag-set must be encoded as URL Query parameters.

View File

@@ -48,22 +48,26 @@ const (
// svc := s3.New(mySession, aws.NewConfig().WithRegion("us-west-2"))
func New(p client.ConfigProvider, cfgs ...*aws.Config) *S3 {
c := p.ClientConfig(EndpointsID, cfgs...)
return newClient(*c.Config, c.Handlers, c.PartitionID, c.Endpoint, c.SigningRegion, c.SigningName)
if c.SigningNameDerived || len(c.SigningName) == 0 {
c.SigningName = "s3"
}
return newClient(*c.Config, c.Handlers, c.PartitionID, c.Endpoint, c.SigningRegion, c.SigningName, c.ResolvedRegion)
}
// newClient creates, initializes and returns a new service client instance.
func newClient(cfg aws.Config, handlers request.Handlers, partitionID, endpoint, signingRegion, signingName string) *S3 {
func newClient(cfg aws.Config, handlers request.Handlers, partitionID, endpoint, signingRegion, signingName, resolvedRegion string) *S3 {
svc := &S3{
Client: client.New(
cfg,
metadata.ClientInfo{
ServiceName: ServiceName,
ServiceID: ServiceID,
SigningName: signingName,
SigningRegion: signingRegion,
PartitionID: partitionID,
Endpoint: endpoint,
APIVersion: "2006-03-01",
ServiceName: ServiceName,
ServiceID: ServiceID,
SigningName: signingName,
SigningRegion: signingRegion,
PartitionID: partitionID,
Endpoint: endpoint,
APIVersion: "2006-03-01",
ResolvedRegion: resolvedRegion,
},
handlers,
),

View File

@@ -69,7 +69,7 @@ func computeKeyMD5(keyHeader, keyMD5Header, key string, r *http.Request) {
return
}
// In backwards compatiable, the header's value is not base64 encoded,
// In backwards compatible, the header's value is not base64 encoded,
// and needs to be encoded and updated by the SDK's customizations.
b64Key := base64.StdEncoding.EncodeToString([]byte(key))
r.Header.Set(keyHeader, b64Key)

View File

@@ -2,6 +2,7 @@ package s3
import (
"bytes"
"io"
"io/ioutil"
"net/http"
@@ -10,31 +11,37 @@ import (
"github.com/aws/aws-sdk-go/internal/sdkio"
)
func copyMultipartStatusOKUnmarhsalError(r *request.Request) {
func copyMultipartStatusOKUnmarshalError(r *request.Request) {
b, err := ioutil.ReadAll(r.HTTPResponse.Body)
r.HTTPResponse.Body.Close()
if err != nil {
r.Error = awserr.NewRequestFailure(
awserr.New(request.ErrCodeSerialization, "unable to read response body", err),
r.HTTPResponse.StatusCode,
r.RequestID,
)
// Note, some middleware later in the stack like restxml.Unmarshal expect a valid, non-closed Body
// even in case of an error, so we replace it with an empty Reader.
r.HTTPResponse.Body = ioutil.NopCloser(bytes.NewBuffer(nil))
return
}
body := bytes.NewReader(b)
r.HTTPResponse.Body = ioutil.NopCloser(body)
defer body.Seek(0, sdkio.SeekStart)
if body.Len() == 0 {
// If there is no body don't attempt to parse the body.
return
}
unmarshalError(r)
if err, ok := r.Error.(awserr.Error); ok && err != nil {
if err.Code() == request.ErrCodeSerialization {
if err.Code() == request.ErrCodeSerialization &&
err.OrigErr() != io.EOF {
r.Error = nil
return
}
r.HTTPResponse.StatusCode = http.StatusServiceUnavailable
// if empty payload
if err.OrigErr() == io.EOF {
r.HTTPResponse.StatusCode = http.StatusInternalServerError
} else {
r.HTTPResponse.StatusCode = http.StatusServiceUnavailable
}
}
}

View File

@@ -1,6 +1,7 @@
package s3
import (
"bytes"
"encoding/xml"
"fmt"
"io"
@@ -45,17 +46,24 @@ func unmarshalError(r *request.Request) {
// Attempt to parse error from body if it is known
var errResp xmlErrorResponse
err := xmlutil.UnmarshalXMLError(&errResp, r.HTTPResponse.Body)
if err == io.EOF {
// Only capture the error if an unmarshal error occurs that is not EOF,
// because S3 might send an error without a error message which causes
// the XML unmarshal to fail with EOF.
err = nil
var err error
if r.HTTPResponse.StatusCode >= 200 && r.HTTPResponse.StatusCode < 300 {
err = s3unmarshalXMLError(&errResp, r.HTTPResponse.Body)
} else {
err = xmlutil.UnmarshalXMLError(&errResp, r.HTTPResponse.Body)
}
if err != nil {
var errorMsg string
if err == io.EOF {
errorMsg = "empty response payload"
} else {
errorMsg = "failed to unmarshal error message"
}
r.Error = awserr.NewRequestFailure(
awserr.New(request.ErrCodeSerialization,
"failed to unmarshal error message", err),
errorMsg, err),
r.HTTPResponse.StatusCode,
r.RequestID,
)
@@ -86,3 +94,21 @@ type RequestFailure interface {
// Host ID is the S3 Host ID needed for debug, and contacting support
HostID() string
}
// s3unmarshalXMLError is s3 specific xml error unmarshaler
// for 200 OK errors and response payloads.
// This function differs from the xmlUtil.UnmarshalXMLError
// func. It does not ignore the EOF error and passes it up.
// Related to bug fix for `s3 200 OK response with empty payload`
func s3unmarshalXMLError(v interface{}, stream io.Reader) error {
var errBuf bytes.Buffer
body := io.TeeReader(stream, &errBuf)
err := xml.NewDecoder(body).Decode(v)
if err != nil && err != io.EOF {
return awserr.NewUnmarshalError(err,
"failed to unmarshal error message", errBuf.Bytes())
}
return err
}