Add ability to read requester pays buckets

Also add profile flags to dictate which
credentials are used for requester pays.

Author: mattrbianchi

awsutil/aws.go

@@ -26,11 +26,12 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/aws/aws-sdk-go/aws/credentials"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/jacobsa/fuse"
-	"github.com/mattrbianchi/twig"
 	"github.com/pkg/errors"
 )
 
@@ -82,42 +83,41 @@ func GetObjectRange(url, byteRange string) (*http.Response, error) {
 		return nil, err
 	}
 	if resp.StatusCode != http.StatusPartialContent && resp.StatusCode != http.StatusOK {
-		twig.Debugf("status code: %d\n", resp.StatusCode)
 		return nil, parseHTTPError(resp.StatusCode)
 	}
 	return resp, nil
 }
 
 type Client struct {
-	Bucket string
-	Key    string
-	Region string
+	Bucket  string
+	Key     string
+	Region  string
+	Profile string
 }
 
-func NewClient(bucket, key, region string) Client {
+func NewClient(bucket, key, region, profile string) Client {
 	return Client{
-		Bucket: bucket,
-		Key:    key,
-		Region: region,
+		Bucket:  bucket,
+		Key:     key,
+		Region:  region,
+		Profile: profile,
 	}
 }
 
 func (c Client) GetObjectRange(byteRange string) (io.ReadCloser, error) {
 	cfg := (&aws.Config{
-		Region: aws.String(c.Region),
+		Credentials: credentials.NewSharedCredentials("", c.Profile),
+		Region:      aws.String(c.Region),
 	}).WithHTTPClient(newHTTPClient())
 	sess := session.New(cfg)
 	svc := s3.New(sess)
 	input := &s3.GetObjectInput{
-		Bucket: aws.String(c.Bucket),
-		Key:    aws.String(c.Key),
-		Range:  aws.String(byteRange),
+		Bucket:       aws.String(c.Bucket),
+		Key:          aws.String(c.Key),
+		Range:        aws.String(byteRange),
+		RequestPayer: aws.String("requester"),
 	}
 	obj, err := svc.GetObject(input)
-	if err != nil {
-		twig.Debug("error from GetObject")
-		return nil, err
-	}
 	return obj.Body, err
 }
 
@@ -140,11 +140,8 @@ func ReadFile(path string) ([]byte, error) {
 		return nil, errors.Errorf("url did not point to a valid amazon s3 location or follow the virtual-hosted style of https://[bucket].[region].s3.amazonaws.com/[file]: %s", path)
 	}
 	bucket := sections[0]
-	twig.Debugf("bucket: %s", bucket)
 	region := sections[1]
-	twig.Debugf("region: %s", region)
 	file := u.Path
-	twig.Debugf("file: %s", file)
 	cfg := (&aws.Config{
 		Region: &region,
 	}).WithHTTPClient(newHTTPClient())
@@ -156,7 +153,6 @@ func ReadFile(path string) ([]byte, error) {
 	}
 	obj, err := svc.GetObject(input)
 	if err != nil {
-		twig.Debug("error from GetObject")
 		return nil, err
 	}
 	bytes, err := ioutil.ReadAll(obj.Body)
@@ -276,23 +272,17 @@ func newHTTPClient() *http.Client {
 func parseHTTPError(code int) error {
 	switch code {
 	case 400:
-		twig.Debug("converting to EINVAL")
 		return fuse.EINVAL
 	case 403:
-		twig.Debug("converting to EACCES")
 		return syscall.EACCES
 	case 404:
-		twig.Debug("converting to ENOENT")
 		return fuse.ENOENT
 	case 405:
-		twig.Debug("converting to ENOTSUP")
 		return syscall.ENOTSUP
 	case 500:
-		twig.Debug("converting to EAGAIN")
 		return syscall.EAGAIN
 	default:
-		// TODO: log this and re-evaluate whether this is a good move.
-		twig.Debug("converting to EOF")
+		// TODO: re-evaluate whether this is a good move.
 		return io.EOF
 	}
 }

cmd/mount.go

@@ -41,10 +41,12 @@ var (
 	ngcpath   string
 	filetype  string
 
-	endpoint             string
-	awsBatch, awsDefault int = 0, 50
-	gcpBatch, gcpDefault int = 0, 25
-	eager                bool
+	endpoint                      string
+	awsBatch, awsDefault          int    = 0, 50
+	gcpBatch, gcpDefault          int    = 0, 25
+	awsProfile, awsProfileDefault string = "", "default"
+	gcpProfile, gcpProfileDefault string = "", "gcp"
+	eager                         bool
 )
 
 func init() {
@@ -78,11 +80,21 @@ func init() {
 		panic("INTERNAL ERROR: could not bind aws-batch flag to aws-batch environment variable")
 	}
 
+	mountCmd.Flags().StringVarP(&awsProfile, "aws-profile", "", awsProfileDefault, flags.AwsProfileMsg)
+	if err := viper.BindPFlag("aws-profile", mountCmd.Flags().Lookup("aws-profile")); err != nil {
+		panic("INTERNAL ERROR: could not bind aws-profile flag to aws-profile environment variable")
+	}
+
 	mountCmd.Flags().IntVarP(&gcpBatch, "gcp-batch", "", gcpDefault, flags.GcpBatchMsg)
 	if err := viper.BindPFlag("gcp-batch", mountCmd.Flags().Lookup("gcp-batch")); err != nil {
 		panic("INTERNAL ERROR: could not bind gcp-batch flag to gcp-batch environment variable")
 	}
 
+	mountCmd.Flags().StringVarP(&gcpProfile, "gcp-profile", "", gcpProfileDefault, flags.GcpProfileMsg)
+	if err := viper.BindPFlag("gcp-profile", mountCmd.Flags().Lookup("gcp-profile")); err != nil {
+		panic("INTERNAL ERROR: could not bind gcp-profile flag to gcp-profile environment variable")
+	}
+
 	mountCmd.Flags().BoolVarP(&eager, "eager", "", false, "ADVANCED: Have fusera request that urls be signed by the API on start up.\nEnvironment Variable: [$DBGAP_EAGER]")
 	if err := viper.BindPFlag("eager", mountCmd.Flags().Lookup("eager")); err != nil {
 		panic("INTERNAL ERROR: could not bind gcp-batch flag to gcp-batch environment variable")
@@ -204,11 +216,22 @@ func mount(cmd *cobra.Command, args []string) (err error) {
 		fmt.Println("It seems like none of the accessions were successful, fusera is shutting down.")
 		os.Exit(1)
 	}
+	credProfile := ""
+	cloud := location[:2]
+	twig.Debug(cloud)
+	if cloud == "s3" {
+		credProfile = awsProfile
+	}
+	if cloud == "gs" {
+		credProfile = gcpProfile
+	}
+	twig.Debug(credProfile)
 	//
 	opt := &fuseralib.Options{
-		Signer: client,
-		Acc:    accessions,
-		Region: location,
+		Signer:  client,
+		Acc:     accessions,
+		Region:  location[3:],
+		Profile: credProfile,
 
 		UID: uint32(uid),
 		GID: uint32(gid),
@@ -237,6 +260,8 @@ func foldEnvVarsIntoFlagValues() {
 	flags.ResolveString("endpoint", &endpoint)
 	flags.ResolveInt("aws-batch", &awsBatch)
 	flags.ResolveInt("gcp-batch", &gcpBatch)
+	flags.ResolveString("aws-profile", &awsProfile)
+	flags.ResolveString("gcp-profile", &gcpProfile)
 	flags.ResolveBool("eager", &eager)
 	flags.ResolveString("location", &location)
 	flags.ResolveString("accession", &accession)

flags/flags.go

@@ -23,13 +23,15 @@ var (
 	AwsBatchName  = "aws-batch"
 	GcpBatchName  = "gcp-batch"
 
-	LocationMsg  = "Cloud provider and region where files should be located.\nFORMAT: [cloud.region]\nEXAMPLES: [s3.us-east-1 | gs.US]\nNOTE: This can be auto-resolved if running on AWS or GCP.\nEnvironment Variable: [$DBGAP_LOCATION]"
-	AccessionMsg = "A list of accessions to mount or path to accession file.\nEXAMPLES: [\"SRR123,SRR456\" | local/accession/file | https://<bucket>.<region>.s3.amazonaws.com/<accession/file>]\nNOTE: If using an s3 url, the proper aws credentials need to be in place on the machine.\nEnvironment Variable: [$DBGAP_ACCESSION]"
-	NgcMsg       = "A path to an ngc file used to authorize access to accessions in dbGaP.\nEXAMPLES: [local/ngc/file | https://<bucket>.<region>.s3.amazonaws.com/<ngc/file>]\nNOTE: If using an s3 url, the proper aws credentials need to be in place on the machine.\nEnvironment Variable: [$DBGAP_NGC]"
-	FiletypeMsg  = "A list of the only file types to copy.\nEXAMPLES: \"cram,crai,bam,bai\"\nEnvironment Variable: [$DBGAP_FILETYPE]"
-	EndpointMsg  = "ADVANCED: Change the endpoint used to communicate with SDL API.\nEnvironment Variable: [$DBGAP_ENDPOINT]"
-	AwsBatchMsg  = "ADVANCED: Adjust the amount of accessions put in one request to the SDL API when using an AWS location.\nEnvironment Variable: [$DBGAP_AWS-BATCH]"
-	GcpBatchMsg  = "ADVANCED: Adjust the amount of accessions put in one request to the SDL API when using a GCP location.\nEnvironment Variable: [$DBGAP_GCP-BATCH]"
+	LocationMsg   = "Cloud provider and region where files should be located.\nFORMAT: [cloud.region]\nEXAMPLES: [s3.us-east-1 | gs.US]\nNOTE: This can be auto-resolved if running on AWS or GCP.\nEnvironment Variable: [$DBGAP_LOCATION]"
+	AccessionMsg  = "A list of accessions to mount or path to accession file.\nEXAMPLES: [\"SRR123,SRR456\" | local/accession/file | https://<bucket>.<region>.s3.amazonaws.com/<accession/file>]\nNOTE: If using an s3 url, the proper aws credentials need to be in place on the machine.\nEnvironment Variable: [$DBGAP_ACCESSION]"
+	NgcMsg        = "A path to an ngc file used to authorize access to accessions in dbGaP.\nEXAMPLES: [local/ngc/file | https://<bucket>.<region>.s3.amazonaws.com/<ngc/file>]\nNOTE: If using an s3 url, the proper aws credentials need to be in place on the machine.\nEnvironment Variable: [$DBGAP_NGC]"
+	FiletypeMsg   = "A list of the only file types to copy.\nEXAMPLES: \"cram,crai,bam,bai\"\nEnvironment Variable: [$DBGAP_FILETYPE]"
+	EndpointMsg   = "ADVANCED: Change the endpoint used to communicate with SDL API.\nEnvironment Variable: [$DBGAP_ENDPOINT]"
+	AwsBatchMsg   = "ADVANCED: Adjust the amount of accessions put in one request to the SDL API when using an AWS location.\nEnvironment Variable: [$DBGAP_AWS-BATCH]"
+	GcpBatchMsg   = "ADVANCED: Adjust the amount of accessions put in one request to the SDL API when using a GCP location.\nEnvironment Variable: [$DBGAP_GCP-BATCH]"
+	AwsProfileMsg = "The desired AWS credentials profile in ~/.aws/credentials to use for instances when files require the requester (you) to pay for accessing the file.\nEnvironment Variable: [$DBGAP_AWS-PROFILE]\nNOTE: This account will be charged all cost accrued by accessing these certain files through fusera."
+	GcpProfileMsg = "The desired GCP credentials profile in ~/.aws/credentials to use for instances when files require the requester (you) to pay for accessing the file.\nEnvironment Variable: [$DBGAP_GCP-PROFILE]\nNOTE: This account will be charged all cost accrued by accessing these certain files through fusera. These credentials should be in the AWS supported format that Google provides in order to work with their AWS compatible API."
 )
 
 // ResolveLocation attempts to resolve the location on GCP and AWS.

fuseralib/file.go

@@ -255,7 +255,7 @@ func (fh *FileHandle) readFromStream(offset int64, buf []byte) (bytesRead int, e
 			sd, _ := time.ParseDuration("30s")
 			exp := fh.inode.Attributes.ExpirationDate
 			if fh.inode.ReqPays {
-				client := awsutil.NewClient(fh.inode.Bucket, fh.inode.Key, fh.inode.Region)
+				client := awsutil.NewClient(fh.inode.Bucket, fh.inode.Key, fh.inode.Region, fh.inode.fs.opt.Profile)
 				body, err := client.GetObjectRange(byteRange)
 				if err != nil {
 					return 0, syscall.EACCES

fuseralib/system.go

@@ -38,9 +38,10 @@ import (
 // Options is a collection of values that describe how Fusera should behave.
 type Options struct {
 	// The file used to authenticate with the SRA Data Locator API
-	Signer Signer
-	Acc    []*Accession
-	Region string
+	Signer  Signer
+	Acc     []*Accession
+	Region  string
+	Profile string
 
 	// File system
 	MountOptions      map[string]string

mock/cmd/root.go

@@ -71,26 +71,32 @@ type file struct {
 	Md5Hash        string    `json:"md5,omitempty"`
 	Link           string    `json:"link,omitempty"`
 	ExpirationDate time.Time `json:"expirationDate,omitempty"`
+	Bucket         string    `json:"bucket,omitempty"`
+	Key            string    `json:"key,omitempty"`
 	Service        string    `json:"service,omitempty"`
 }
 
 // HomeHandler returns whatever JSON I want.
 func HomeHandler(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusOK)
-	response := make([]payload, 5045, 5045)
+	response := make([]payload, 1, 1)
 	for i := range response {
 		response[i].ID = "a" + fmt.Sprintf("%d", i)
 		response[i].Status = 200
-		response[i].Files = make([]file, 3, 3)
+		response[i].Files = make([]file, 1, 1)
 		for j := range response[i].Files {
-			response[i].Files[j].Name = "a" + fmt.Sprintf("%d", i) + "file" + fmt.Sprintf("%d", j)
-			response[i].Files[j].ExpirationDate = time.Now().Add(time.Hour)
+			response[i].Files[j].Name = "test.txt"
+			response[i].Files[j].Bucket = "matt-first-test-bucket"
+			response[i].Files[j].Key = "test.txt"
+			response[i].Files[j].Size = "51"
+			//response[i].Files[j].ExpirationDate = time.Now().Add(time.Hour)
 		}
 	}
+	js, _ := json.Marshal(&response)
 	if err := json.NewEncoder(w).Encode(&response); err != nil {
 		panic("couldn't encode json")
 	}
-	fmt.Println(response)
+	fmt.Println(string(js))
 }
 
 // Execute runs the root command of mocksdlapi.