Skip to content

Commit 4d8c2c2

Browse files
RaduPetreTareanRaduPetreTarean
authored
feat: sanitize-command (#403)
1 parent c45afbb commit 4d8c2c2

File tree

4 files changed

+299
-9
lines changed

4 files changed

+299
-9
lines changed

‎internal/cmd/cmd.go

+1
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ var CLIApp = cli.App{
8383
&LocalizeCommand,
8484
&ResourceCommand,
8585
&ResolveCommand,
86+
&SanitizeCommand,
8687
},
8788
}
8889

‎internal/cmd/sanitize.go

+151
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
package cmd
2+
3+
import (
4+
"fmt"
5+
"os"
6+
"path/filepath"
7+
"sort"
8+
9+
"github.com/urfave/cli/v2"
10+
11+
"github.com/snyk/vervet/v8"
12+
"github.com/snyk/vervet/v8/internal/storage"
13+
)
14+
15+
var SanitizeCommand = cli.Command{
16+
Name: "sanitize",
17+
Usage: "Manually load compiled specs from subfolders, strip sensitive fields, write sanitized output",
18+
ArgsUsage: " ",
19+
Flags: []cli.Flag{
20+
&cli.StringFlag{
21+
Name: "compiled-path",
22+
Required: true,
23+
Usage: "Directory containing subfolders for each version, e.g. internal/rest/api/versions/",
24+
},
25+
&cli.StringFlag{
26+
Name: "out",
27+
Usage: "Output directory for sanitized versions",
28+
Value: "sanitized",
29+
Required: false,
30+
},
31+
&cli.StringSliceFlag{
32+
Name: "exclude-extension",
33+
Usage: "Regex patterns of extension names to remove (repeatable)",
34+
},
35+
&cli.StringSliceFlag{
36+
Name: "exclude-header",
37+
Usage: "Regex patterns of header names to remove (repeatable)",
38+
},
39+
&cli.StringSliceFlag{
40+
Name: "exclude-path",
41+
Usage: "Exact path(s) to remove from the final OpenAPI specs",
42+
},
43+
},
44+
Action: sanitizeAction,
45+
}
46+
47+
func sanitizeAction(c *cli.Context) error {
48+
excludePatterns := vervet.ExcludePatterns{
49+
ExtensionPatterns: c.StringSlice("exclude-extension"),
50+
HeaderPatterns: c.StringSlice("exclude-header"),
51+
Paths: c.StringSlice("exclude-path"),
52+
}
53+
54+
compiledPath := c.String("compiled-path")
55+
56+
outDir := c.String("out")
57+
if err := os.MkdirAll(outDir, 0o755); err != nil {
58+
return fmt.Errorf("failed to create out directory %q: %w", outDir, err)
59+
}
60+
61+
versionDirs, err := findVersionDirs(compiledPath)
62+
if err != nil {
63+
return fmt.Errorf("failed to find version subfolders in %q: %w", compiledPath, err)
64+
}
65+
66+
if len(versionDirs) == 0 {
67+
fmt.Fprintf(c.App.Writer, "No version subfolders found in %q\n", compiledPath)
68+
return nil
69+
}
70+
71+
coll, err := storage.NewCollator(
72+
storage.CollatorExcludePattern(excludePatterns),
73+
)
74+
75+
if err != nil {
76+
return fmt.Errorf("failed to create collator: %w", err)
77+
}
78+
79+
// for each version folder, read openapi.yaml/spec.yaml, parse version, add to Collator
80+
const serviceName = "api" // or any label you prefer
81+
for _, dir := range versionDirs {
82+
specFile := filepath.Join(dir, "spec.yaml")
83+
if _, statErr := os.Stat(specFile); statErr != nil {
84+
// No recognized spec found, skip this version folder
85+
continue
86+
}
87+
88+
blob, err := os.ReadFile(specFile)
89+
if err != nil {
90+
return fmt.Errorf("failed to read spec file %q: %w", specFile, err)
91+
}
92+
93+
versionName := filepath.Base(dir)
94+
v, err := vervet.ParseVersion(versionName)
95+
if err != nil {
96+
fmt.Fprintf(c.App.Writer, "Skipping folder %q: not a valid version: %v\n", dir, err)
97+
continue
98+
}
99+
100+
digest := storage.NewDigest(blob)
101+
rev := storage.ContentRevision{
102+
Service: serviceName,
103+
Version: v,
104+
Digest: digest,
105+
Blob: blob,
106+
}
107+
coll.Add(serviceName, rev)
108+
}
109+
110+
sanitized, err := coll.Collate()
111+
if err != nil {
112+
return fmt.Errorf("collate failed: %w", err)
113+
}
114+
115+
for version, doc := range sanitized {
116+
versionOutDir := filepath.Join(outDir, version.String())
117+
if err := os.MkdirAll(versionOutDir, 0o755); err != nil {
118+
return fmt.Errorf("failed to create version output dir %q: %w", versionOutDir, err)
119+
}
120+
outPath := filepath.Join(versionOutDir, "openapi.yaml")
121+
122+
jsonBytes, err := doc.MarshalJSON()
123+
if err != nil {
124+
return fmt.Errorf("failed to marshal JSON for version %s: %w", version, err)
125+
}
126+
127+
// Write the final sanitized file
128+
if err := os.WriteFile(outPath, jsonBytes, 0o644); err != nil {
129+
return fmt.Errorf("failed to write sanitized spec %q: %w", outPath, err)
130+
}
131+
}
132+
133+
fmt.Fprintf(c.App.Writer, "Wrote sanitized specs to %s\n", outDir)
134+
return nil
135+
}
136+
137+
// findVersionDirs enumerates subdirectories of compiledPath and returns them sorted.
138+
func findVersionDirs(compiledPath string) ([]string, error) {
139+
entries, err := os.ReadDir(compiledPath)
140+
if err != nil {
141+
return nil, err
142+
}
143+
var dirs []string
144+
for _, e := range entries {
145+
if e.IsDir() {
146+
dirs = append(dirs, filepath.Join(compiledPath, e.Name()))
147+
}
148+
}
149+
sort.Strings(dirs)
150+
return dirs, nil
151+
}

‎internal/cmd/sanitize_test.go

+138
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
package cmd_test
2+
3+
import (
4+
"os"
5+
"path/filepath"
6+
"testing"
7+
8+
qt "github.com/frankban/quicktest"
9+
"github.com/getkin/kin-openapi/openapi3"
10+
"gopkg.in/yaml.v3"
11+
12+
"github.com/snyk/vervet/v8"
13+
"github.com/snyk/vervet/v8/internal/cmd"
14+
)
15+
16+
func TestSanitize(t *testing.T) {
17+
c := qt.New(t)
18+
19+
// Create temp directories for source and output
20+
srcDir := c.TempDir()
21+
outDir := c.TempDir()
22+
23+
// Create test OpenAPI specs with sensitive data
24+
tests := []struct {
25+
version string
26+
spec *openapi3.T
27+
}{
28+
{
29+
version: "2024-01-15",
30+
spec: &openapi3.T{
31+
OpenAPI: "3.0.0",
32+
Info: &openapi3.Info{
33+
Title: "Test API",
34+
Version: "1.0.0",
35+
},
36+
Paths: func() *openapi3.Paths {
37+
paths := &openapi3.Paths{}
38+
paths.Set("/public", &openapi3.PathItem{
39+
Get: &openapi3.Operation{
40+
OperationID: "getPublic",
41+
Extensions: map[string]interface{}{
42+
"x-internal-op": "should-be-removed",
43+
"x-public-op": "should-remain",
44+
},
45+
Parameters: []*openapi3.ParameterRef{
46+
{
47+
Value: &openapi3.Parameter{
48+
Name: "X-Internal-Header",
49+
In: "header",
50+
Description: "Should be removed",
51+
},
52+
},
53+
{
54+
Value: &openapi3.Parameter{
55+
Name: "X-Public-Header",
56+
In: "header",
57+
Description: "Should remain",
58+
},
59+
},
60+
},
61+
},
62+
})
63+
paths.Set("/internal", &openapi3.PathItem{
64+
Get: &openapi3.Operation{
65+
OperationID: "getInternal",
66+
},
67+
})
68+
return paths
69+
}(),
70+
},
71+
},
72+
}
73+
74+
// Write test specs to source directory
75+
for _, test := range tests {
76+
versionDir := filepath.Join(srcDir, test.version)
77+
err := os.MkdirAll(versionDir, 0755)
78+
c.Assert(err, qt.IsNil)
79+
80+
specBytes, err := yaml.Marshal(test.spec)
81+
c.Assert(err, qt.IsNil)
82+
83+
err = os.WriteFile(filepath.Join(versionDir, "spec.yaml"), specBytes, 0644)
84+
c.Assert(err, qt.IsNil)
85+
}
86+
87+
// Run sanitize command
88+
err := cmd.Vervet.Run([]string{
89+
"vervet",
90+
"sanitize",
91+
"--compiled-path", srcDir,
92+
"--out", outDir,
93+
"--exclude-extension", "^x-internal-.*$",
94+
"--exclude-header", "^X-Internal-.*$",
95+
"--exclude-path", "/internal",
96+
})
97+
c.Assert(err, qt.IsNil)
98+
99+
// Verify output
100+
for _, test := range tests {
101+
c.Run("sanitized version "+test.version, func(c *qt.C) {
102+
outPath := filepath.Join(outDir, test.version, "openapi.yaml")
103+
doc, err := vervet.NewDocumentFile(outPath)
104+
c.Assert(err, qt.IsNil)
105+
106+
// Check that internal path was removed
107+
internalPath := doc.Paths.Find("/internal")
108+
c.Assert(internalPath, qt.IsNil, qt.Commentf("Internal path should be removed"))
109+
110+
// Check that public path and its proper operations remain
111+
publicPath := doc.Paths.Find("/public")
112+
c.Assert(publicPath, qt.Not(qt.IsNil), qt.Commentf("Public path should remain"))
113+
c.Assert(publicPath.Get, qt.Not(qt.IsNil))
114+
115+
// Check operation extensions
116+
_, hasInternalOpExt := publicPath.Get.Extensions["x-internal-op"]
117+
c.Assert(hasInternalOpExt, qt.IsFalse, qt.Commentf("Internal operation extension should be removed"))
118+
119+
_, hasPublicOpExt := publicPath.Get.Extensions["x-public-op"]
120+
c.Assert(hasPublicOpExt, qt.IsTrue, qt.Commentf("Public operation extension should remain"))
121+
122+
// Check headers in parameters
123+
var foundInternalHeader, foundPublicHeader bool
124+
for _, param := range publicPath.Get.Parameters {
125+
if param.Value.In == "header" {
126+
if param.Value.Name == "X-Internal-Header" {
127+
foundInternalHeader = true
128+
}
129+
if param.Value.Name == "X-Public-Header" {
130+
foundPublicHeader = true
131+
}
132+
}
133+
}
134+
c.Assert(foundInternalHeader, qt.IsFalse, qt.Commentf("Internal header parameter should be removed"))
135+
c.Assert(foundPublicHeader, qt.IsTrue, qt.Commentf("Public header parameter should remain"))
136+
})
137+
}
138+
}

‎internal/storage/disk/disk.go

+9-9
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,8 @@ type Storage struct {
3030
type Option func(*Storage)
3131

3232
type objectMeta struct {
33-
blob []byte
34-
lastMod time.Time
33+
Blob []byte
34+
LastMod time.Time
3535
}
3636

3737
func New(path string, options ...Option) storage.Storage {
@@ -88,7 +88,7 @@ func (s *Storage) CollateVersions(ctx context.Context, serviceFilter map[string]
8888

8989
// all specs are stored as: "service-versions/{service_name}/{version}/{digest}.json"
9090
for _, revKey := range serviceRevisions {
91-
service, version, digest, err := parseServiceVersionRevisionKey(revKey)
91+
service, version, digest, err := ParseServiceVersionRevisionKey(revKey)
9292
if err != nil {
9393
return err
9494
}
@@ -109,9 +109,9 @@ func (s *Storage) CollateVersions(ctx context.Context, serviceFilter map[string]
109109
revision := storage.ContentRevision{
110110
Service: service,
111111
Version: parsedVersion,
112-
Timestamp: rev.lastMod,
112+
Timestamp: rev.LastMod,
113113
Digest: storage.Digest(digest),
114-
Blob: rev.blob,
114+
Blob: rev.Blob,
115115
}
116116
aggregate.Add(service, revision)
117117
}
@@ -179,7 +179,7 @@ func (s *Storage) VersionIndex(ctx context.Context) (vervet.VersionIndex, error)
179179
}
180180
vs := make(vervet.VersionSlice, len(objects))
181181
for idx, obj := range objects {
182-
_, versionStr, _, err := parseServiceVersionRevisionKey(obj)
182+
_, versionStr, _, err := ParseServiceVersionRevisionKey(obj)
183183
if err != nil {
184184
return vervet.VersionIndex{}, err
185185
}
@@ -269,7 +269,7 @@ func (s *Storage) ListObjects(ctx context.Context, key string) ([]string, error)
269269
return objects, err
270270
}
271271

272-
func parseServiceVersionRevisionKey(key string) (string, string, string, error) {
272+
func ParseServiceVersionRevisionKey(key string) (string, string, string, error) {
273273
digestB64 := filepath.Base(key)
274274
digestB64 = strings.TrimSuffix(digestB64, ".json")
275275
digest, err := base64.StdEncoding.DecodeString(digestB64)
@@ -292,8 +292,8 @@ func (s *Storage) GetObjectWithMetadata(key string) (*objectMeta, error) {
292292
lastMod := info.ModTime()
293293
body, err := os.ReadFile(key)
294294
return &objectMeta{
295-
lastMod: lastMod,
296-
blob: body,
295+
LastMod: lastMod,
296+
Blob: body,
297297
}, err
298298
}
299299

0 commit comments

Comments
 (0)