Check mountpoint before doing slow operations

This change also has a major refactor of the nr library in part
due to the improvements in the SDL API.

This refactor also cleans up debug statements and removes logging
from the nr library altogether.

Author: mattrbianchi

.travis.yml

@@ -1,10 +1,10 @@
 language: go
 os: osx
 before_deploy:
-  - cd $HOME/gopath/src/github.com/mitre/fusera/cmd/fusera
+  - cd $HOME/gopath/src/github.com/mitre/fusera
   - go build -o fusera-darwin-amd64
   - env GOOS=linux GOARCH=amd64 go build -o fusera-linux-amd64
-  - cd $HOME/gopath/src/github.com/mitre/fusera/cmd/sracp
+  - cd $HOME/gopath/src/github.com/mitre/fusera/sracp
   - go build -o sracp-darwin-amd64
   - env GOOS=linux GOARCH=amd64 go build -o sracp-linux-amd64
 deploy:
@@ -14,9 +14,9 @@ deploy:
   skip_cleanup: true
   overwrite: true
   file:
-    - $HOME/gopath/src/github.com/mitre/fusera/cmd/fusera/fusera-linux-amd64
-    - $HOME/gopath/src/github.com/mitre/fusera/cmd/fusera/fusera-darwin-amd64
-    - $HOME/gopath/src/github.com/mitre/fusera/cmd/sracp/sracp-linux-amd64
-    - $HOME/gopath/src/github.com/mitre/fusera/cmd/sracp/sracp-darwin-amd64
+    - $HOME/gopath/src/github.com/mitre/fusera/fusera-linux-amd64
+    - $HOME/gopath/src/github.com/mitre/fusera/fusera-darwin-amd64
+    - $HOME/gopath/src/github.com/mitre/fusera/sracp/sracp-linux-amd64
+    - $HOME/gopath/src/github.com/mitre/fusera/sracp/sracp-darwin-amd64
   on:
     tags: true

cmd/errors.go

@@ -0,0 +1,83 @@
+// Modifications Copyright 2018 The MITRE Corporation
+// Authors: Matthew Bianchi
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/mattrbianchi/twig"
+)
+
+func prettyPrintError(err error) {
+	// Accession errors
+	if err.Error() == "no accessions provided" {
+		twig.Debug(err)
+		fmt.Println("No accessions provided: Fusera needs accession(s) in order to know what files to provide in its file system.")
+	}
+	if strings.Contains(err.Error(), "couldn't open cart file") {
+		twig.Debug(err)
+		fmt.Println("Bad cart file or path: Fusera interpreted the accession flag as a path to a cart file, but could not open the file at the path specified. Make sure the path leads to a valid cart file and that you have permissions to read that file. If you do and you're still getting this message, run with debug enabled for a more detailed error message and contact your IT administrator with its contents.")
+	}
+	if strings.Contains(err.Error(), "cart file was empty") {
+		twig.Debug(err)
+		fmt.Println("Bad cart file: Fusera interpreted the accession flag as a path to a cart file, but the file seems empty. Make sure the path leads to a valid cart file that has properly formatted contents and isn't corrupted. If you're still getting this message after assuring the file is correct, run with debug enabled for a more detailed error message and contact your IT administrator with its contents.")
+	}
+
+	// Location errors
+	if err.Error() == "no location provided" {
+		twig.Debug(err)
+		fmt.Println("No location provided: A location was not provided so Fusera attempted to resolve the location itself and could not do so. This feature is only supported when Fusera is running on Amazon or Google's cloud platforms. If you are running on a server in either of these two cloud platforms and are still getting this message, run fusera with debug enabled for a more detailed error message and contact your IT administrator with its contents.")
+	}
+
+	// Ngc errors
+	if strings.Contains(err.Error(), "couldn't open ngc file") {
+		twig.Debug(err)
+		fmt.Println("Bad ngc file path: Fusera tried to read the cart file at the path specified and couldn't. Make sure the path leads to a valid ngc file and that you have permissions to read that file. If you do and you're still getting this message, run with debug enabled for a more detailed error message and contact your IT administrator with its contents.")
+	}
+
+	// Filetype errors
+	if err.Error() == "filetype was empty" {
+		twig.Debug(err)
+		fmt.Println("Filetype was empty: Fusera tried to parse the list of filetypes given but couldn't find anything. Example of a well formatted list to the filetype flag: -f \"bai,crai,cram\".")
+	}
+
+	// Mount errors
+	if strings.Contains(err.Error(), "mountpoint doesn't exist") {
+		twig.Debug(err)
+		fmt.Println("Mountpoint doesn't exist: It seems like the directory you want to mount to does not exist. Please create the directory before trying again.")
+	}
+	if strings.Contains(err.Error(), "no such file or directory") {
+		twig.Debug(err)
+		fmt.Println("Failed to mount: It seems like the directory you want to mount to does not exist or you do not have correct permissions to access it. Please create the directory or correct the permissions on it before trying again.")
+	}
+	if strings.Contains(err.Error(), "EOF") {
+		twig.Debug(err)
+		fmt.Println("Failed to mount: It seems like the directory you want to mount to is already mounted by another instance of Fusera or another device. Choose another directory or try using the unmount command to unmount the other instance of Fusera before trying again. Be considerate of the unmount command, if anything is using Fusera while attempting to unmount, the unmount attempt will fail and that instance of Fusera will keep running.")
+	}
+
+	// API errors
+	if strings.Contains(err.Error(), "failed to locate accessions") {
+		twig.Debug(err)
+		fmt.Println("Failed to locate accessions: It seems that Fusera has encountered an error while using the SRA Data Locator API to determine the file locations for accessions. This is an issue between Fusera and the API. In order to get more information, run Fusera with debug enabled and contact your IT administrator with its contents.")
+	}
+
+	// Fatal errors
+	if strings.Contains(err.Error(), "FATAL") {
+		twig.Debug(err)
+		fmt.Println("Fatal: It seems like fusera encountered an internal issue, please run fusera with debug enabled for a more detailed error message and contact your IT administrator with its contents.")
+	}
+}

cmd/flags.go

@@ -14,16 +14,15 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
 package cmd
 
 import (
 	"context"
-	"fmt"
 	"os"
 	"os/signal"
 	"os/user"
 	"strconv"
-	"strings"
 	"syscall"
 
 	"github.com/mattrbianchi/twig"
@@ -97,14 +96,11 @@ var mountCmd = &cobra.Command{
 	RunE:  mount,
 }
 
+// mount locates the files for each accession its given with the SDL API
+// and then mounts a FUSE system.
 func mount(cmd *cobra.Command, args []string) (err error) {
 	setConfig()
-	twig.Debug("got mount command")
-	twig.Debug("args:")
-	twig.Debug(args)
 	foldEnvVarsIntoFlagValues()
-	twig.Debug("location: " + location)
-	twig.Debug("accessions: " + accession)
 	var ngc []byte
 	if ngcpath != "" {
 		ngc, err = flags.ResolveNgcFile(ngcpath)
@@ -113,29 +109,35 @@ func mount(cmd *cobra.Command, args []string) (err error) {
 		}
 	}
 	if accession == "" {
-		return errors.New("No accessions provided: Fusera needs a list of accessions in order to know what files to provide in its file system.")
+		return errors.New("no accessions provided")
 	}
-	// Now resolveAccession's value
 	accs, err := flags.ResolveAccession(accession)
 	if err != nil {
 		return err
 	}
-	// Validate mount point
-	// Do mount stuff
-
+	var types map[string]bool
+	if filetype != "" {
+		types, err = flags.ResolveFileType(filetype)
+		if err != nil {
+			return err
+		}
+	}
+	// Validate the mount point before trying to mount to it.
+	// So it must exist
+	mountpoint := args[0]
+	if !flags.FileExists(mountpoint) {
+		return errors.New("mountpoint doesn't exist")
+	}
+	// So it must be readable
+	if !flags.HavePermissions(mountpoint) {
+		return errors.New("incorrect permissions for mountpoint")
+	}
 	// Location takes longest if there's a failure, so validate it last.
 	if location == "" {
 		location, err = flags.ResolveLocation()
 		if err != nil {
 			twig.Debug(err)
-			return errors.New("No location: A location was not provided so Fusera attempted to resolve the location itself. This feature is only supported when Fusera is running on Amazon or Google's cloud platforms.")
-		}
-	}
-	var types map[string]bool
-	if filetype != "" {
-		types, err = flags.ResolveFileType(filetype)
-		if err != nil {
-			return errors.New("Seems like something was wrong with the format of the filetype flag.")
+			return errors.New("no location provided")
 		}
 	}
 	uid, gid := myUserAndGroup()
@@ -146,41 +148,30 @@ func mount(cmd *cobra.Command, args []string) (err error) {
 		Filetypes: types,
 		Eager:     eager,
 
-		ApiEndpoint: endpoint,
+		APIEndpoint: endpoint,
 		AwsBatch:    awsBatch,
 		GcpBatch:    gcpBatch,
 
 		DirMode:  0555,
 		FileMode: 0444,
-		Uid:      uint32(uid),
-		Gid:      uint32(gid),
+		UID:      uint32(uid),
+		GID:      uint32(gid),
 		// TODO: won't need.
 		MountOptions:  make(map[string]string),
-		MountPoint:    args[0],
-		MountPointArg: args[0],
+		MountPoint:    mountpoint,
+		MountPointArg: mountpoint,
 	}
 	fs, mfs, err := fuseralib.Mount(context.Background(), opt)
 	if err != nil {
-		var msg string
-		if strings.Contains(err.Error(), "no such file or directory") {
-			msg = "Fusera failed to mount the file system.\nIt seems like the directory you want to mount to does not exist or you do not have correct permissions to access it. Please create the directory or correct the permissions on it before trying again."
-		}
-		if strings.Contains(err.Error(), "EOF") {
-			msg = "Fusera failed to mount the file system.\nIt seems like the directory you want to mount to is already mounted by fusera or another device. Choose another directory or try using the unmount command before trying again. Be considerate of the unmount command, if anything is using the device mounted while attempting to unmount, it will fail."
-		}
-		twig.Debugf("%+v\n", err)
-		return errors.New(msg)
+		return err
 	}
-	twig.Debug("File system has been successfully mounted.")
 	// Let the user unmount with Ctrl-C
 	registerSIGINTHandler(fs, opt.MountPoint)
 
 	// Wait for the file system to be unmounted.
 	err = mfs.Join(context.Background())
 	if err != nil {
-		fmt.Println("fusera encountered an internal issue, please rerun with the --debug flag to learn more.")
-		twig.Debugf("FATAL: MountedFileSystem.Join: %+#v\n", err)
-		os.Exit(1)
+		return errors.Wrap(err, "FATAL")
 	}
 
 	return nil

cmd/mount.go

@@ -1,7 +1,5 @@
-// Copyright 2015 - 2017 Ka-Hing Cheung
-// Copyright 2015 - 2017 Google Inc. All Rights Reserved.
 // Modifications Copyright 2018 The MITRE Corporation
-// Authors: Ka-Hing Cheung, Matthew Bianchi
+// Authors: Matthew Bianchi
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -14,10 +12,10 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
 package cmd
 
 import (
-	"fmt"
 	"os"
 
 	"github.com/mattrbianchi/twig"
@@ -47,9 +45,11 @@ var rootCmd = &cobra.Command{
 	Version: version,
 }
 
+// Execute runs the main command of fusera, which has no action of its own,
+// so it evaluates which subcommand should be executed.
 func Execute() {
 	if err := rootCmd.Execute(); err != nil {
-		fmt.Println(err)
+		prettyPrintError(err)
 		os.Exit(1)
 	}
 }

cmd/root.go

@@ -69,25 +69,20 @@ func ResolveAccession(acc string) (map[string]bool, error) {
 		}
 	}
 	if empty {
-		return nil, errors.Errorf("No accessions were found in the content given to the --accession flag. --accession: %s.", acc)
+		return nil, errors.New("cart file was empty")
 	}
 
 	return accessions, nil
 }
 
+// TODO: should be a private function
 func ParseAccessions(r rune) bool {
 	return r == '\n' || r == '\t' || r == ',' || r == ' '
 }
 
-func FileExists(path string) bool {
-	_, err := os.Stat(path)
-	return !os.IsNotExist(err)
-}
-
 // Deduce whether path is on s3 or local.
 // Either way, read all of the file into a byte slice.
 func ResolveNgcFile(ngcpath string) (data []byte, err error) {
-	// we were given a path to an ngc file. Let's read it.
 	if strings.HasPrefix(ngcpath, "http") {
 		// we were given a url on s3.
 		data, err = awsutil.ReadFile(ngcpath)
@@ -107,7 +102,7 @@ func ResolveFileType(filetype string) (map[string]bool, error) {
 	uniqTypes := make(map[string]bool)
 	types := strings.Split(filetype, ",")
 	if len(types) == 1 && types[0] == "" {
-		return nil, errors.New("")
+		return nil, errors.New("filetype was empty")
 	}
 	if len(types) > 0 {
 		for _, t := range types {
@@ -117,7 +112,17 @@ func ResolveFileType(filetype string) (map[string]bool, error) {
 		}
 		return uniqTypes, nil
 	}
-	return nil, errors.New("")
+	return nil, errors.New("filetype was empty")
+}
+
+func FileExists(path string) bool {
+	_, err := os.Stat(path)
+	return !os.IsNotExist(err)
+}
+
+func HavePermissions(path string) bool {
+	_, err := os.Stat(path)
+	return !os.IsPermission(err)
 }
 
 func ResolveString(name string, value *string) {

flags/flags.go

@@ -65,15 +65,15 @@ func NewDirHandle(inode *Inode) (dh *DirHandle) {
 
 func (inode *Inode) OpenDir() (dh *DirHandle) {
 	parent := inode.Parent
-	if parent != nil && inode.fs.opt.TypeCacheTTL != 0 {
+	if parent != nil {
 		parent.mu.Lock()
 		defer parent.mu.Unlock()
 
 		num := len(parent.dir.Children)
 
 		if parent.dir.lastOpenDir == nil && num > 0 && *parent.dir.Children[0].Name == *inode.Name {
 			if parent.dir.seqOpenDirScore < 255 {
-				parent.dir.seqOpenDirScore += 1
+				parent.dir.seqOpenDirScore++
 			}
 			// 2.1) if I open a/a, a/'s score is now 2
 			// ie: handle the depth first search case
@@ -134,7 +134,6 @@ func (p sortedDirents) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
 
 // LOCKS_REQUIRED(dh.mu)
 func (dh *DirHandle) ReadDir(offset fuseops.DirOffset) (en *DirHandleEntry, err error) {
-	twig.Debug("dir.go/ReadDir called")
 	// If the request is for offset zero, we assume that either this is the first
 	// call or rewinddir has been called. Reset state.
 	if offset == 0 {