Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add cache-control and expires options #318

Merged
merged 7 commits into from
Jul 22, 2021
Merged

add cache-control and expires options #318

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
91e9798
add cache-control and expires options
tombokombo Jul 9, 2021
62f8df4
Merge branch 'master' into feature/headers_cache_expires
tombokombo Jul 10, 2021
02d6e99
add examples
tombokombo Jul 18, 2021
98c2e9c
return error on invalid expires
tombokombo Jul 19, 2021
70a8cc7
remove else
tombokombo Jul 19, 2021
0cab42a
remove else
tombokombo Jul 19, 2021
8149223
Merge branch 'master' into feature/headers_cache_expires
ilkinulas Jul 22, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 19 additions & 2 deletions command/cp.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,14 @@ var copyCommandFlags = []cli.Flag{
Name: "acl",
Usage: "set acl for target: defines granted accesses and their types on different accounts/groups",
},
&cli.StringFlag{
Name: "cache-control",
Usage: "set cache control for target: defines cache control header for object",
},
&cli.StringFlag{
Name: "expires",
Usage: "set expires for target (uses RFC3339 format): defines expires header for object",
},
&cli.BoolFlag{
Name: "force-glacier-transfer",
Usage: "force transfer of GLACIER objects whether they are restored or not",
Expand Down Expand Up @@ -183,6 +191,8 @@ var copyCommand = &cli.Command{
encryptionKeyID: c.String("sse-kms-key-id"),
acl: c.String("acl"),
forceGlacierTransfer: c.Bool("force-glacier-transfer"),
cacheControl: c.String("cache-control"),
expires: c.String("expires"),
// region settings
srcRegion: c.String("source-region"),
dstRegion: c.String("destination-region"),
Expand Down Expand Up @@ -212,6 +222,9 @@ type Copy struct {
encryptionKeyID string
acl string
forceGlacierTransfer bool
cacheControl string
expires string


// region settings
srcRegion string
Expand Down Expand Up @@ -472,7 +485,9 @@ func (c Copy) doUpload(ctx context.Context, srcurl *url.URL, dsturl *url.URL) er
SetStorageClass(string(c.storageClass)).
SetSSE(c.encryptionMethod).
SetSSEKeyID(c.encryptionKeyID).
SetACL(c.acl)
SetACL(c.acl).
SetCacheControl(c.cacheControl).
SetExpires(c.expires)

err = dstClient.Put(ctx, file, dsturl, metadata, c.concurrency, c.partSize)
if err != nil {
Expand Down Expand Up @@ -518,7 +533,9 @@ func (c Copy) doCopy(ctx context.Context, srcurl, dsturl *url.URL) error {
SetStorageClass(string(c.storageClass)).
SetSSE(c.encryptionMethod).
SetSSEKeyID(c.encryptionKeyID).
SetACL(c.acl)
SetACL(c.acl).
SetCacheControl(c.cacheControl).
SetExpires(c.expires)

err = c.shouldOverride(ctx, srcurl, dsturl)
if err != nil {
Expand Down
3 changes: 3 additions & 0 deletions command/mv.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,9 @@ var moveCommand = &cli.Command{
encryptionMethod: c.String("sse"),
encryptionKeyID: c.String("sse-kms-key-id"),
acl: c.String("acl"),
cacheControl: c.String("cache-control"),
expires: c.String("expires"),


storageOpts: NewStorageOpts(c),
}
Expand Down
32 changes: 32 additions & 0 deletions storage/s3.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -345,6 +345,22 @@ func (s *S3) Copy(ctx context.Context, from, to *url.URL, metadata Metadata) err
input.ACL = aws.String(acl)
}

cacheControl := metadata.CacheControl()
if cacheControl != "" {
input.CacheControl = aws.String(cacheControl)
}

expires := metadata.Expires()
if expires != "" {
t, err := time.Parse(time.RFC3339, expires)
if err == nil {
input.Expires = aws.Time(t)
} else {
msg := log.DebugMessage{Err: err.Error()}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was trying invalid time formats to see s3 behavior, here are my findings:

  • if we upload a new object s3 ignores invalid Expires metadata.
  • if we override an existing s3 objects (which has a valid Expires metadata) with an invalid expires value, s3 deletes the current object's Expires metadata.

So, it would be better to return the error and do not cp/mv any objects in case of unparsable --expires values.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok, done

log.Debug(msg)
}
}

_, err := s.api.CopyObject(input)
return err
}
Expand Down Expand Up @@ -492,6 +508,22 @@ func (s *S3) Put(
input.ACL = aws.String(acl)
}

cacheControl := metadata.CacheControl()
if cacheControl != "" {
input.CacheControl = aws.String(cacheControl)
}

expires := metadata.Expires()
if expires != "" {
t, err := time.Parse(time.RFC3339, expires)
if err == nil {
input.Expires = aws.Time(t)
} else {
msg := log.DebugMessage{Err: err.Error()}
log.Debug(msg)
}
}

sseEncryption := metadata.SSE()
if sseEncryption != "" {
input.ServerSideEncryption = aws.String(sseEncryption)
Expand Down
18 changes: 18 additions & 0 deletions storage/storage.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,24 @@ func (m Metadata) SetACL(acl string) Metadata {
return m
}

func (m Metadata) CacheControl() string {
return m["CacheControl"]
}

func (m Metadata) SetCacheControl(cacheControl string) Metadata {
m["CacheControl"] = cacheControl
return m
}

func (m Metadata) Expires() string {
return m["Expires"]
}

func (m Metadata) SetExpires(expires string) Metadata {
m["Expires"] = expires
return m
}

func (m Metadata) StorageClass() string {
return m["StorageClass"]
}
Expand Down