Skip to content

Commit

Permalink
Implement presign for s3 UploadPart (#1232)
Browse files Browse the repository at this point in the history
  • Loading branch information
@F21
F21 authored May 7, 2021
1 parent 8c4dbc7 commit ea88d01
Show file tree
Hide file tree
Showing 4 changed files with 350 additions and 2 deletions.
3 changes: 2 additions & 1 deletion ...src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpPresignURLClientGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,8 @@ public class AwsHttpPresignURLClientGenerator implements GoIntegration {
private static final Map<ShapeId, Set<ShapeId>> presignedClientMap = MapUtils.of(
ShapeId.from("com.amazonaws.s3#AmazonS3"), SetUtils.of(
ShapeId.from("com.amazonaws.s3#GetObject"),
ShapeId.from("com.amazonaws.s3#PutObject")
ShapeId.from("com.amazonaws.s3#PutObject"),
ShapeId.from("com.amazonaws.s3#UploadPart")
),
ShapeId.from("com.amazonaws.sts#AWSSecurityTokenServiceV20110615"), SetUtils.of(
ShapeId.from("com.amazonaws.sts#GetCallerIdentity"))
Expand Down
125 changes: 125 additions & 0 deletions service/internal/integrationtest/s3/presign_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,131 @@ func TestInteg_PresignURL(t *testing.T) {
}
}

func TestInteg_MultipartPresignURL(t *testing.T) {
cases := map[string]struct {
key string
body io.Reader
expires time.Duration
sha256Header string
expectedSignedHeader http.Header
}{
"standard": {
body: bytes.NewReader([]byte("Hello-world")),
expectedSignedHeader: http.Header{},
},
"special characters": {
key: "some_value_(1).foo",
},
"nil-body": {
expectedSignedHeader: http.Header{},
},
"empty-body": {
body: bytes.NewReader([]byte("")),
expectedSignedHeader: http.Header{},
},
}

for name, c := range cases {
t.Run(name, func(t *testing.T) {
key := c.key
if len(key) == 0 {
key = integrationtest.UniqueID()
}

ctx, cancelFn := context.WithTimeout(context.Background(), 5*time.Second)
defer cancelFn()

cfg, err := integrationtest.LoadConfigWithDefaultRegion("us-west-2")
if err != nil {
t.Fatalf("failed to load config, %v", err)
}

client := s3.NewFromConfig(cfg)

multipartUpload, err := client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
Bucket: &setupMetadata.Buckets.Source.Name,
Key: &key,
})

if err != nil {
t.Fatalf("error creating multipart upload: %v", err)
}

// construct an upload part object
uploadPartInput := &s3.UploadPartInput{
Bucket: &setupMetadata.Buckets.Source.Name,
Key: &key,
PartNumber: 1,
UploadId: multipartUpload.UploadId,
Body: c.body,
}

presignerClient := s3.NewPresignClient(client, func(options *s3.PresignOptions) {
options.Expires = 600 * time.Second
})

presignRequest, err := presignerClient.PresignUploadPart(ctx, uploadPartInput)
if err != nil {
t.Fatalf("expect no error, got %v", err)
}

for k, v := range c.expectedSignedHeader {
value := presignRequest.SignedHeader[k]
if len(value) == 0 {
t.Fatalf("expected %v header to be present in presigned url, got %v", k, presignRequest.SignedHeader)
}

if diff := cmp.Diff(v, value); len(diff) != 0 {
t.Fatalf("expected %v header value to be %v got %v", k, v, value)
}
}

resp, err := sendHTTPRequest(presignRequest, uploadPartInput.Body)
if err != nil {
t.Errorf("expect no error while sending HTTP request using presigned url, got %v", err)
}

defer resp.Body.Close()

if resp.StatusCode != http.StatusOK {
t.Fatalf("failed to upload part, %d:%s", resp.StatusCode, resp.Status)
}

_, err = client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
Bucket: &setupMetadata.Buckets.Source.Name,
Key: &key,
UploadId: multipartUpload.UploadId,
})

if err != nil {
t.Fatalf("error completing multipart upload: %v", err)
}

// construct a get object
getObjectInput := &s3.GetObjectInput{
Bucket: &setupMetadata.Buckets.Source.Name,
Key: &key,
}

presignRequest, err = presignerClient.PresignGetObject(ctx, getObjectInput)
if err != nil {
t.Errorf("expect no error, got %v", err)
}

resp, err = sendHTTPRequest(presignRequest, nil)
if err != nil {
t.Errorf("expect no error while sending HTTP request using presigned url, got %v", err)
}

defer resp.Body.Close()

if resp.StatusCode != http.StatusOK {
t.Fatalf("failed to get S3 object, %d:%s", resp.StatusCode, resp.Status)
}
})
}
}

func sendHTTPRequest(presignRequest *v4.PresignedHTTPRequest, body io.Reader) (*http.Response, error) {
// create a http request
req, err := http.NewRequest(presignRequest.Method, presignRequest.URL, nil)
Expand Down
38 changes: 37 additions & 1 deletion service/s3/api_op_UploadPart.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

186 changes: 186 additions & 0 deletions service/s3/internal/customizations/presign_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -179,3 +179,189 @@ func TestPutObject_PresignURL(t *testing.T) {
})
}
}

func TestUploadPart_PresignURL(t *testing.T) {
cases := map[string]struct {
input s3.UploadPartInput
options s3.PresignOptions
expectPresignedURLHost string
expectRequestURIQuery []string
expectSignedHeader http.Header
expectMethod string
expectError string
}{
"standard case": {
input: s3.UploadPartInput{
Bucket: aws.String("mock-bucket"),
Key: aws.String("mockkey"),
PartNumber: 1,
UploadId: aws.String("123456"),
Body: strings.NewReader("hello-world"),
},
expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?",
expectRequestURIQuery: []string{
"X-Amz-Expires=900",
"X-Amz-Credential",
"X-Amz-Date",
"partNumber=1",
"uploadId=123456",
"x-id=UploadPart",
"X-Amz-Signature",
},
expectMethod: "PUT",
expectSignedHeader: http.Header{
"Content-Length": []string{"11"},
"Content-Type": []string{"application/octet-stream"},
"Host": []string{"mock-bucket.s3.us-west-2.amazonaws.com"},
},
},
"seekable payload": {
input: s3.UploadPartInput{
Bucket: aws.String("mock-bucket"),
Key: aws.String("mockkey"),
PartNumber: 1,
UploadId: aws.String("123456"),
Body: bytes.NewReader([]byte("hello-world")),
},
expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?",
expectRequestURIQuery: []string{
"X-Amz-Expires=900",
"X-Amz-Credential",
"X-Amz-Date",
"partNumber=1",
"uploadId=123456",
"x-id=UploadPart",
"X-Amz-Signature",
},
expectMethod: "PUT",
expectSignedHeader: http.Header{
"Content-Length": []string{"11"},
"Content-Type": []string{"application/octet-stream"},
"Host": []string{"mock-bucket.s3.us-west-2.amazonaws.com"},
},
},
"unseekable payload": {
// unseekable payload succeeds as we disable content sha256 computation for streaming input
input: s3.UploadPartInput{
Bucket: aws.String("mock-bucket"),
Key: aws.String("mockkey"),
PartNumber: 1,
UploadId: aws.String("123456"),
Body: bytes.NewBuffer([]byte(`hello-world`)),
},
expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?",
expectRequestURIQuery: []string{
"X-Amz-Expires=900",
"X-Amz-Credential",
"X-Amz-Date",
"partNumber=1",
"uploadId=123456",
"x-id=UploadPart",
"X-Amz-Signature",
},
expectMethod: "PUT",
expectSignedHeader: http.Header{
"Content-Length": []string{"11"},
"Content-Type": []string{"application/octet-stream"},
"Host": []string{"mock-bucket.s3.us-west-2.amazonaws.com"},
},
},
"empty body": {
input: s3.UploadPartInput{
Bucket: aws.String("mock-bucket"),
Key: aws.String("mockkey"),
PartNumber: 1,
UploadId: aws.String("123456"),
Body: bytes.NewReader([]byte(``)),
},
expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?",
expectRequestURIQuery: []string{
"X-Amz-Expires=900",
"X-Amz-Credential",
"X-Amz-Date",
"partNumber=1",
"uploadId=123456",
"x-id=UploadPart",
"X-Amz-Signature",
},
expectMethod: "PUT",
expectSignedHeader: http.Header{
"Host": []string{"mock-bucket.s3.us-west-2.amazonaws.com"},
},
},
"nil body": {
input: s3.UploadPartInput{
Bucket: aws.String("mock-bucket"),
Key: aws.String("mockkey"),
PartNumber: 1,
UploadId: aws.String("123456"),
},
expectPresignedURLHost: "https://mock-bucket.s3.us-west-2.amazonaws.com/mockkey?",
expectRequestURIQuery: []string{
"X-Amz-Expires=900",
"X-Amz-Credential",
"X-Amz-Date",
"partNumber=1",
"uploadId=123456",
"x-id=UploadPart",
"X-Amz-Signature",
},
expectMethod: "PUT",
expectSignedHeader: http.Header{
"Host": []string{"mock-bucket.s3.us-west-2.amazonaws.com"},
},
},
}

for name, c := range cases {
t.Run(name, func(t *testing.T) {
ctx := context.Background()
cfg := aws.Config{
Region: "us-west-2",
Credentials: unit.StubCredentialsProvider{},
Retryer: func() aws.Retryer {
return aws.NopRetryer{}
},
}
presignClient := s3.NewPresignClient(s3.NewFromConfig(cfg), func(options *s3.PresignOptions) {
options = &c.options
})

req, err := presignClient.PresignUploadPart(ctx, &c.input)
if err != nil {
if len(c.expectError) == 0 {
t.Fatalf("expected no error, got %v", err)
}
// if expect error, match error and skip rest
if e, a := c.expectError, err.Error(); !strings.Contains(a, e) {
t.Fatalf("expected error to be %s, got %s", e, a)
}
} else {
if len(c.expectError) != 0 {
t.Fatalf("expected error to be %v, got none", c.expectError)
}
}

if e, a := c.expectPresignedURLHost, req.URL; !strings.Contains(a, e) {
t.Fatalf("expected presigned url to contain host %s, got %s", e, a)
}

if len(c.expectRequestURIQuery) != 0 {
for _, label := range c.expectRequestURIQuery {
if e, a := label, req.URL; !strings.Contains(a, e) {
t.Fatalf("expected presigned url to contain %v label in url: %v", label, req.URL)
}
}
}

if e, a := c.expectSignedHeader, req.SignedHeader; len(cmp.Diff(e, a)) != 0 {
t.Fatalf("expected signed header to be %s, got %s, \n diff : %s", e, a, cmp.Diff(e, a))
}

if e, a := c.expectMethod, req.Method; !strings.EqualFold(e, a) {
t.Fatalf("expected presigning Method to be %s, got %s", e, a)
}

})
}
}

0 comments on commit ea88d01

Please sign in to comment.