Skip to content

Commit c1ceb3b

Browse files
yihuangtac0turtle
yihuang
and
tac0turtle
authored
feat: add local snapshots management commands (#16067)
Co-authored-by: Marko <[email protected]>
1 parent b28c50f commit c1ceb3b

File tree

16 files changed

+483
-16
lines changed

16 files changed

+483
-16
lines changed

‎CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
110110
* (x/auth) [#15867](https://github.com/cosmos/cosmos-sdk/pull/15867) Support better logging for signature verification failure.
111111
* (types/query) [#16041](https://github.com/cosmos/cosmos-sdk/pull/16041) change pagination max limit to a variable in order to be modifed by application devs
112112
* (server) [#16061](https://github.com/cosmos/cosmos-sdk/pull/16061) add comet bootstrap command
113+
* (store) [#16067](https://github.com/cosmos/cosmos-sdk/pull/16067) Add local snapshots management commands.
113114

114115
### State Machine Breaking
115116

‎client/snapshot/cmd.go

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
package snapshot
2+
3+
import (
4+
servertypes "github.com/cosmos/cosmos-sdk/server/types"
5+
"github.com/spf13/cobra"
6+
)
7+
8+
// Cmd returns the snapshots group command
9+
func Cmd(appCreator servertypes.AppCreator) *cobra.Command {
10+
cmd := &cobra.Command{
11+
Use: "snapshots",
12+
Short: "Manage local snapshots",
13+
Long: "Manage local snapshots",
14+
}
15+
cmd.AddCommand(
16+
ListSnapshotsCmd,
17+
RestoreSnapshotCmd(appCreator),
18+
ExportSnapshotCmd(appCreator),
19+
DumpArchiveCmd(),
20+
LoadArchiveCmd(),
21+
DeleteSnapshotCmd(),
22+
)
23+
return cmd
24+
}

‎client/snapshot/delete.go

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
package snapshot
2+
3+
import (
4+
"strconv"
5+
6+
"github.com/cosmos/cosmos-sdk/server"
7+
"github.com/spf13/cobra"
8+
)
9+
10+
func DeleteSnapshotCmd() *cobra.Command {
11+
return &cobra.Command{
12+
Use: "delete <height> <format>",
13+
Short: "Delete a local snapshot",
14+
Args: cobra.ExactArgs(2),
15+
RunE: func(cmd *cobra.Command, args []string) error {
16+
ctx := server.GetServerContextFromCmd(cmd)
17+
18+
height, err := strconv.ParseUint(args[0], 10, 64)
19+
if err != nil {
20+
return err
21+
}
22+
format, err := strconv.ParseUint(args[1], 10, 32)
23+
if err != nil {
24+
return err
25+
}
26+
27+
snapshotStore, err := server.GetSnapshotStore(ctx.Viper)
28+
if err != nil {
29+
return err
30+
}
31+
32+
return snapshotStore.Delete(height, uint32(format))
33+
},
34+
}
35+
}

‎client/snapshot/dump.go

+119
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
package snapshot
2+
3+
import (
4+
"archive/tar"
5+
"compress/gzip"
6+
"fmt"
7+
"io"
8+
"os"
9+
"strconv"
10+
11+
"github.com/cosmos/cosmos-sdk/server"
12+
"github.com/spf13/cobra"
13+
)
14+
15+
// DumpArchiveCmd returns a command to dump the snapshot as portable archive format
16+
func DumpArchiveCmd() *cobra.Command {
17+
cmd := &cobra.Command{
18+
Use: "dump <height> <format>",
19+
Short: "Dump the snapshot as portable archive format",
20+
Args: cobra.ExactArgs(2),
21+
RunE: func(cmd *cobra.Command, args []string) error {
22+
ctx := server.GetServerContextFromCmd(cmd)
23+
snapshotStore, err := server.GetSnapshotStore(ctx.Viper)
24+
if err != nil {
25+
return err
26+
}
27+
28+
output, err := cmd.Flags().GetString("output")
29+
if err != nil {
30+
return err
31+
}
32+
33+
height, err := strconv.ParseUint(args[0], 10, 64)
34+
if err != nil {
35+
return err
36+
}
37+
format, err := strconv.ParseUint(args[1], 10, 32)
38+
if err != nil {
39+
return err
40+
}
41+
42+
if output == "" {
43+
output = fmt.Sprintf("%d-%d.tar.gz", height, format)
44+
}
45+
46+
snapshot, err := snapshotStore.Get(height, uint32(format))
47+
if err != nil {
48+
return err
49+
}
50+
51+
bz, err := snapshot.Marshal()
52+
if err != nil {
53+
return err
54+
}
55+
56+
fp, err := os.Create(output)
57+
if err != nil {
58+
return err
59+
}
60+
defer fp.Close()
61+
62+
// since the chunk files are already compressed, we just use fastest compression here
63+
gzipWriter, err := gzip.NewWriterLevel(fp, gzip.BestSpeed)
64+
if err != nil {
65+
return err
66+
}
67+
tarWriter := tar.NewWriter(gzipWriter)
68+
if err := tarWriter.WriteHeader(&tar.Header{
69+
Name: SnapshotFileName,
70+
Mode: 0o644,
71+
Size: int64(len(bz)),
72+
}); err != nil {
73+
return fmt.Errorf("failed to write snapshot header to tar: %w", err)
74+
}
75+
if _, err := tarWriter.Write(bz); err != nil {
76+
return fmt.Errorf("failed to write snapshot to tar: %w", err)
77+
}
78+
79+
for i := uint32(0); i < snapshot.Chunks; i++ {
80+
path := snapshotStore.PathChunk(height, uint32(format), i)
81+
file, err := os.Open(path)
82+
if err != nil {
83+
return fmt.Errorf("failed to open chunk file %s: %w", path, err)
84+
}
85+
86+
st, err := file.Stat()
87+
if err != nil {
88+
return fmt.Errorf("failed to stat chunk file %s: %w", path, err)
89+
}
90+
91+
if err := tarWriter.WriteHeader(&tar.Header{
92+
Name: strconv.FormatUint(uint64(i), 10),
93+
Mode: 0o644,
94+
Size: st.Size(),
95+
}); err != nil {
96+
return fmt.Errorf("failed to write chunk header to tar: %w", err)
97+
}
98+
99+
if _, err := io.Copy(tarWriter, file); err != nil {
100+
return fmt.Errorf("failed to write chunk to tar: %w", err)
101+
}
102+
}
103+
104+
if err := tarWriter.Close(); err != nil {
105+
return fmt.Errorf("failed to close tar writer: %w", err)
106+
}
107+
108+
if err := gzipWriter.Close(); err != nil {
109+
return fmt.Errorf("failed to close gzip writer: %w", err)
110+
}
111+
112+
return fp.Close()
113+
},
114+
}
115+
116+
cmd.Flags().StringP("output", "o", "", "output file")
117+
118+
return cmd
119+
}

‎client/snapshot/export.go

+54
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
package snapshot
2+
3+
import (
4+
"fmt"
5+
6+
"cosmossdk.io/log"
7+
"github.com/cosmos/cosmos-sdk/server"
8+
servertypes "github.com/cosmos/cosmos-sdk/server/types"
9+
"github.com/spf13/cobra"
10+
)
11+
12+
// ExportSnapshotCmd returns a command to take a snapshot of the application state
13+
func ExportSnapshotCmd(appCreator servertypes.AppCreator) *cobra.Command {
14+
cmd := &cobra.Command{
15+
Use: "export",
16+
Short: "Export app state to snapshot store",
17+
Args: cobra.NoArgs,
18+
RunE: func(cmd *cobra.Command, args []string) error {
19+
ctx := server.GetServerContextFromCmd(cmd)
20+
21+
height, err := cmd.Flags().GetInt64("height")
22+
if err != nil {
23+
return err
24+
}
25+
26+
home := ctx.Config.RootDir
27+
db, err := openDB(home, server.GetAppDBBackend(ctx.Viper))
28+
if err != nil {
29+
return err
30+
}
31+
logger := log.NewLogger(cmd.OutOrStdout())
32+
app := appCreator(logger, db, nil, ctx.Viper)
33+
34+
if height == 0 {
35+
height = app.CommitMultiStore().LastCommitID().Version
36+
}
37+
38+
fmt.Printf("Exporting snapshot for height %d\n", height)
39+
40+
sm := app.SnapshotManager()
41+
snapshot, err := sm.Create(uint64(height))
42+
if err != nil {
43+
return err
44+
}
45+
46+
fmt.Printf("Snapshot created at height %d, format %d, chunks %d\n", snapshot.Height, snapshot.Format, snapshot.Chunks)
47+
return nil
48+
},
49+
}
50+
51+
cmd.Flags().Int64("height", 0, "Height to export, default to latest state height")
52+
53+
return cmd
54+
}

‎client/snapshot/list.go

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package snapshot
2+
3+
import (
4+
"fmt"
5+
6+
"github.com/cosmos/cosmos-sdk/server"
7+
"github.com/spf13/cobra"
8+
)
9+
10+
// ListSnapshotsCmd returns the command to list local snapshots
11+
var ListSnapshotsCmd = &cobra.Command{
12+
Use: "list",
13+
Short: "List local snapshots",
14+
RunE: func(cmd *cobra.Command, args []string) error {
15+
ctx := server.GetServerContextFromCmd(cmd)
16+
snapshotStore, err := server.GetSnapshotStore(ctx.Viper)
17+
if err != nil {
18+
return err
19+
}
20+
snapshots, err := snapshotStore.List()
21+
if err != nil {
22+
return fmt.Errorf("failed to list snapshots: %w", err)
23+
}
24+
for _, snapshot := range snapshots {
25+
fmt.Println("height:", snapshot.Height, "format:", snapshot.Format, "chunks:", snapshot.Chunks)
26+
}
27+
28+
return nil
29+
},
30+
}

‎client/snapshot/load.go

+113
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
package snapshot
2+
3+
import (
4+
"archive/tar"
5+
"bytes"
6+
"compress/gzip"
7+
"fmt"
8+
"io"
9+
"os"
10+
"reflect"
11+
"strconv"
12+
13+
"github.com/cosmos/cosmos-sdk/server"
14+
"github.com/spf13/cobra"
15+
16+
snapshottypes "cosmossdk.io/store/snapshots/types"
17+
)
18+
19+
const SnapshotFileName = "_snapshot"
20+
21+
// LoadArchiveCmd load a portable archive format snapshot into snapshot store
22+
func LoadArchiveCmd() *cobra.Command {
23+
return &cobra.Command{
24+
Use: "load <archive-file>",
25+
Short: "Load a snapshot archive file into snapshot store",
26+
Args: cobra.ExactArgs(1),
27+
RunE: func(cmd *cobra.Command, args []string) error {
28+
ctx := server.GetServerContextFromCmd(cmd)
29+
snapshotStore, err := server.GetSnapshotStore(ctx.Viper)
30+
if err != nil {
31+
return err
32+
}
33+
34+
path := args[0]
35+
fp, err := os.Open(path)
36+
if err != nil {
37+
return fmt.Errorf("failed to open archive file: %w", err)
38+
}
39+
reader, err := gzip.NewReader(fp)
40+
if err != nil {
41+
return fmt.Errorf("failed to create gzip reader: %w", err)
42+
}
43+
44+
var snapshot snapshottypes.Snapshot
45+
tr := tar.NewReader(reader)
46+
if err != nil {
47+
return fmt.Errorf("failed to create tar reader: %w", err)
48+
}
49+
50+
hdr, err := tr.Next()
51+
if err != nil {
52+
return fmt.Errorf("failed to read snapshot file header: %w", err)
53+
}
54+
if hdr.Name != SnapshotFileName {
55+
return fmt.Errorf("invalid archive, expect file: snapshot, got: %s", hdr.Name)
56+
}
57+
bz, err := io.ReadAll(tr)
58+
if err != nil {
59+
return fmt.Errorf("failed to read snapshot file: %w", err)
60+
}
61+
if err := snapshot.Unmarshal(bz); err != nil {
62+
return fmt.Errorf("failed to unmarshal snapshot: %w", err)
63+
}
64+
65+
// make sure the channel is unbuffered, because the tar reader can't do concurrency
66+
chunks := make(chan io.ReadCloser)
67+
quitChan := make(chan *snapshottypes.Snapshot)
68+
go func() {
69+
defer close(quitChan)
70+
71+
savedSnapshot, err := snapshotStore.Save(snapshot.Height, snapshot.Format, chunks)
72+
if err != nil {
73+
fmt.Println("failed to save snapshot", err)
74+
return
75+
}
76+
quitChan <- savedSnapshot
77+
}()
78+
79+
for i := uint32(0); i < snapshot.Chunks; i++ {
80+
hdr, err = tr.Next()
81+
if err != nil {
82+
if err == io.EOF {
83+
break
84+
}
85+
return err
86+
}
87+
88+
if hdr.Name != strconv.FormatInt(int64(i), 10) {
89+
return fmt.Errorf("invalid archive, expect file: %d, got: %s", i, hdr.Name)
90+
}
91+
92+
bz, err := io.ReadAll(tr)
93+
if err != nil {
94+
return fmt.Errorf("failed to read chunk file: %w", err)
95+
}
96+
chunks <- io.NopCloser(bytes.NewReader(bz))
97+
}
98+
close(chunks)
99+
100+
savedSnapshot := <-quitChan
101+
if savedSnapshot == nil {
102+
return fmt.Errorf("failed to save snapshot")
103+
}
104+
105+
if !reflect.DeepEqual(&snapshot, savedSnapshot) {
106+
_ = snapshotStore.Delete(snapshot.Height, snapshot.Format)
107+
return fmt.Errorf("invalid archive, the saved snapshot is not equal to the original one")
108+
}
109+
110+
return nil
111+
},
112+
}
113+
}

0 commit comments

Comments
 (0)