diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go
index 2592cf54477405..11080053e83357 100644
--- a/src/cmd/go/internal/load/pkg.go
+++ b/src/cmd/go/internal/load/pkg.go
@@ -1857,12 +1857,26 @@ func (p *Package) load(ctx context.Context, opts PackageOpts, path string, stk *
 	if other := foldPath[fold]; other == "" {
 		foldPath[fold] = p.ImportPath
 	} else if other != p.ImportPath {
-		setError(ImportErrorf(p.ImportPath, "case-insensitive import collision: %q and %q", p.ImportPath, other))
+		// It is unnecessary to set an ImportPathError when the error is already
+		// a module.InvalidPathError.
+		// Beside, ImportErrorf may panic when the import path is invalid.
+		// See https://golang.org/issue/49137
+		var invalidPathErr *module.InvalidPathError
+		if !errors.As(err, &invalidPathErr) {
+			setError(ImportErrorf(p.ImportPath, "case-insensitive import collision: %q and %q", p.ImportPath, other))
+		}
 		return
 	}
 
 	if !SafeArg(p.ImportPath) {
-		setError(ImportErrorf(p.ImportPath, "invalid import path %q", p.ImportPath))
+		// It is unnecessary to set an ImportPathError when the error is already
+		// a module.InvalidPathError.
+		// Beside, ImportErrorf may panic when the import path is invalid.
+		// See https://golang.org/issue/49137
+		var invalidPathErr *module.InvalidPathError
+		if !errors.As(err, &invalidPathErr) {
+			setError(ImportErrorf(p.ImportPath, "invalid import path %q", p.ImportPath))
+		}
 		return
 	}
 
diff --git a/src/cmd/go/testdata/script/load_invalid_import_path.txt b/src/cmd/go/testdata/script/load_invalid_import_path.txt
new file mode 100644
index 00000000000000..5a219be770913d
--- /dev/null
+++ b/src/cmd/go/testdata/script/load_invalid_import_path.txt
@@ -0,0 +1,14 @@
+# Tests issue 49137. The following commands should not panic any way.
+
+! go list '"a"'
+stderr '^malformed import path "\\"a\\"": invalid char ''"''$'
+
+! go build '' -ldflags='-X "main.Tags="'
+stderr '^malformed import path "-ldflags=-X \\"main.Tags=\\"": leading dash$'
+
+-- go.mod --
+module m
+
+go 1.16
+-- main.go --
+package main