Merge pull request #9098 from hashicorp/watsonian/kv-size-breakdown

Add detailed key size breakdown to snapshot inspect

.changelog/9098.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+cli: snapshot inspect command provides KV usage breakdown
+```

command/snapshot/inspect/formatter.go

@@ -48,18 +48,31 @@ func (_ *prettyFormatter) Format(info *OutputFormat) (string, error) {
 	fmt.Fprintf(tw, "\n Term\t%d", info.Meta.Term)
 	fmt.Fprintf(tw, "\n Version\t%d", info.Meta.Version)
 	fmt.Fprintf(tw, "\n")
-	fmt.Fprintln(tw, "\n Type\tCount\tSize\t")
+	fmt.Fprintln(tw, "\n Type\tCount\tSize")
-	fmt.Fprintf(tw, " %s\t%s\t%s\t", "----", "----", "----")
+	fmt.Fprintf(tw, " %s\t%s\t%s", "----", "----", "----")
 	// For each different type generate new output
 	for _, s := range info.Stats {
-		fmt.Fprintf(tw, "\n %s\t%d\t%s\t", s.Name, s.Count, ByteSize(uint64(s.Sum)))
+		fmt.Fprintf(tw, "\n %s\t%d\t%s", s.Name, s.Count, ByteSize(uint64(s.Sum)))
+	}
+	fmt.Fprintf(tw, "\n %s\t%s\t%s", "----", "----", "----")
+	fmt.Fprintf(tw, "\n Total\t\t%s", ByteSize(uint64(info.TotalSize)))
+
+	if info.StatsKV != nil {
+		fmt.Fprintf(tw, "\n")
+		fmt.Fprintln(tw, "\n Key Name\tCount\tSize")
+		fmt.Fprintf(tw, " %s\t%s\t%s", "----", "----", "----")
+		// For each different type generate new output
+		for _, s := range info.StatsKV {
+			fmt.Fprintf(tw, "\n %s\t%d\t%s", s.Name, s.Count, ByteSize(uint64(s.Sum)))
+		}
+		fmt.Fprintf(tw, "\n %s\t%s\t%s", "----", "----", "----")
+		fmt.Fprintf(tw, "\n Total\t\t%s", ByteSize(uint64(info.TotalSizeKV)))
 	}
-	fmt.Fprintf(tw, "\n %s\t%s\t%s\t", "----", "----", "----")
-	fmt.Fprintf(tw, "\n Total\t\t%s\t", ByteSize(uint64(info.TotalSize)))
 
 	if err := tw.Flush(); err != nil {
 		return b.String(), err
 	}
+
 	return b.String(), nil
 }
 

command/snapshot/inspect/formatter_test.go

@@ -13,6 +13,11 @@ func TestFormat(t *testing.T) {
 		Sum:   1,
 		Count: 2,
 	}}
+	mkv := []typeStats{{
+		Name:  "msgKV",
+		Sum:   1,
+		Count: 2,
+	}}
 	info := OutputFormat{
 		Meta: &MetadataInfo{
 			ID:      "one",
@@ -22,7 +27,9 @@ func TestFormat(t *testing.T) {
 			Version: 1,
 		},
 		Stats:       m,
+		StatsKV:     mkv,
 		TotalSize:   1,
+		TotalSizeKV: 1,
 	}
 
 	formatters := map[string]Formatter{

command/snapshot/inspect/snapshot_inspect.go

@@ -29,10 +29,21 @@ type cmd struct {
 	flags  *flag.FlagSet
 	help   string
 	format string
+
+	// flags
+	kvDetails bool
+	kvDepth   int
+	kvFilter  string
 }
 
 func (c *cmd) init() {
 	c.flags = flag.NewFlagSet("", flag.ContinueOnError)
+	c.flags.BoolVar(&c.kvDetails, "kvdetails", false,
+		"Provides a detailed KV space usage breakdown for any KV data that's been stored.")
+	c.flags.IntVar(&c.kvDepth, "kvdepth", 2,
+		"Can only be used with -kvdetails. The key prefix depth used to breakdown KV store data. Defaults to 2.")
+	c.flags.StringVar(&c.kvFilter, "kvfilter", "",
+		"Can only be used with -kvdetails. Limits KV key breakdown using this prefix filter.")
 	c.flags.StringVar(
 		&c.format,
 		"format",
@@ -52,12 +63,24 @@ type MetadataInfo struct {
 	Version raft.SnapshotVersion
 }
 
+// SnapshotInfo is used for passing snapshot stat
+// information between functions
+type SnapshotInfo struct {
+	Meta        MetadataInfo
+	Stats       map[structs.MessageType]typeStats
+	StatsKV     map[string]typeStats
+	TotalSize   int
+	TotalSizeKV int
+}
+
 // OutputFormat is used for passing information
 // through the formatter
 type OutputFormat struct {
 	Meta        *MetadataInfo
 	Stats       []typeStats
+	StatsKV     []typeStats
 	TotalSize   int
+	TotalSizeKV int
 }
 
 func (c *cmd) Run(args []string) int {
@@ -101,7 +124,7 @@ func (c *cmd) Run(args []string) int {
 		}
 	}()
 
-	stats, totalSize, err := enhance(readFile)
+	info, err := c.enhance(readFile)
 	if err != nil {
 		c.UI.Error(fmt.Sprintf("Error extracting snapshot data: %s", err))
 		return 1
@@ -122,13 +145,17 @@ func (c *cmd) Run(args []string) int {
 	}
 
 	//Restructures stats given above to be human readable
-	formattedStats := generatetypeStats(stats)
+	formattedStats := generateStats(info)
+	formattedStatsKV := generateKVStats(info)
 
 	in := &OutputFormat{
 		Meta:        metaformat,
 		Stats:       formattedStats,
-		TotalSize: totalSize,
+		StatsKV:     formattedStatsKV,
+		TotalSize:   info.TotalSize,
+		TotalSizeKV: info.TotalSizeKV,
 	}
+
 	out, err := formatter.Format(in)
 	if err != nil {
 		c.UI.Error(err.Error())
@@ -145,19 +172,55 @@ type typeStats struct {
 	Count int
 }
 
-func generatetypeStats(info map[structs.MessageType]typeStats) []typeStats {
+// generateStats formats the stats for the output struct
-	ss := make([]typeStats, 0, len(info))
+// that's used to produce the printed output the user sees.
+func generateStats(info SnapshotInfo) []typeStats {
+	ss := make([]typeStats, 0, len(info.Stats))
 
-	for _, s := range info {
+	for _, s := range info.Stats {
 		ss = append(ss, s)
 	}
 
-	// Sort the stat slice
+	ss = sortTypeStats(ss)
-	sort.Slice(ss, func(i, j int) bool { return ss[i].Sum > ss[j].Sum })
 
 	return ss
 }
 
+// generateKVStats reformats the KV stats to work with
+// the output struct that's used to produce the printed
+// output the user sees.
+func generateKVStats(info SnapshotInfo) []typeStats {
+	kvLen := len(info.StatsKV)
+	if kvLen > 0 {
+		ks := make([]typeStats, 0, kvLen)
+
+		for _, s := range info.StatsKV {
+			ks = append(ks, s)
+		}
+
+		ks = sortTypeStats(ks)
+
+		return ks
+	}
+
+	return nil
+}
+
+// sortTypeStats sorts the stat slice by size and then
+// alphabetically in the case the size is identical
+func sortTypeStats(stats []typeStats) []typeStats {
+	sort.Slice(stats, func(i, j int) bool {
+		// sort alphabetically if size is equal
+		if stats[i].Sum == stats[j].Sum {
+			return stats[i].Name < stats[j].Name
+		}
+
+		return stats[i].Sum > stats[j].Sum
+	})
+
+	return stats
+}
+
 // countingReader helps keep track of the bytes we have read
 // when reading snapshots
 type countingReader struct {
@@ -175,36 +238,89 @@ func (r *countingReader) Read(p []byte) (n int, err error) {
 
 // enhance utilizes ReadSnapshot to populate the struct with
 // all of the snapshot's itemized data
-func enhance(file io.Reader) (map[structs.MessageType]typeStats, int, error) {
+func (c *cmd) enhance(file io.Reader) (SnapshotInfo, error) {
-	stats := make(map[structs.MessageType]typeStats)
+	info := SnapshotInfo{
+		Stats:       make(map[structs.MessageType]typeStats),
+		StatsKV:     make(map[string]typeStats),
+		TotalSize:   0,
+		TotalSizeKV: 0,
+	}
 	cr := &countingReader{wrappedReader: file}
-	totalSize := 0
 	handler := func(header *fsm.SnapshotHeader, msg structs.MessageType, dec *codec.Decoder) error {
 		name := structs.MessageType.String(msg)
-		s := stats[msg]
+		s := info.Stats[msg]
 		if s.Name == "" {
 			s.Name = name
 		}
+
 		var val interface{}
 		err := dec.Decode(&val)
 		if err != nil {
 			return fmt.Errorf("failed to decode msg type %v, error %v", name, err)
 		}
 
-		size := cr.read - totalSize
+		size := cr.read - info.TotalSize
 		s.Sum += size
 		s.Count++
-		totalSize = cr.read
+		info.TotalSize = cr.read
-		stats[msg] = s
+		info.Stats[msg] = s
+
+		c.kvEnhance(s.Name, val, size, &info)
+
 		return nil
 	}
 	if err := fsm.ReadSnapshot(cr, handler); err != nil {
-		return nil, 0, err
+		return info, err
 	}
-	return stats, totalSize, nil
+	return info, nil
 
 }
 
+// kvEnhance populates the struct with all of the snapshot's
+// size information for KV data stored in it
+func (c *cmd) kvEnhance(keyType string, val interface{}, size int, info *SnapshotInfo) {
+	if c.kvDetails {
+		if keyType != "KVS" {
+			return
+		}
+
+		// have to coerce this into a usable type here or this won't work
+		keyVal := val.(map[string]interface{})
+		for k, v := range keyVal {
+			// we only care about the entry on the key specifically
+			// related to the key name, so skip all others
+			if k != "Key" {
+				continue
+			}
+
+			// check for whether a filter is specified. if it is, skip
+			// any keys that don't match.
+			if len(c.kvFilter) > 0 && !strings.HasPrefix(v.(string), c.kvFilter) {
+				break
+			}
+
+			split := strings.Split(v.(string), "/")
+
+			// handle the situation where the key is shorter than
+			// the specified depth.
+			actualDepth := c.kvDepth
+			if c.kvDepth > len(split) {
+				actualDepth = len(split)
+			}
+			prefix := strings.Join(split[0:actualDepth], "/")
+			kvs := info.StatsKV[prefix]
+			if kvs.Name == "" {
+				kvs.Name = prefix
+			}
+
+			kvs.Sum += size
+			kvs.Count++
+			info.TotalSizeKV += size
+			info.StatsKV[prefix] = kvs
+		}
+	}
+}
+
 func (c *cmd) Synopsis() string {
 	return synopsis
 }

command/snapshot/inspect/snapshot_inspect_test.go

@@ -95,3 +95,57 @@ func TestSnapshotInspectCommand(t *testing.T) {
 	want := golden(t, t.Name(), ui.OutputWriter.String())
 	require.Equal(t, want, ui.OutputWriter.String())
 }
+
+func TestSnapshotInspectKVDetailsCommand(t *testing.T) {
+
+	filepath := "./testdata/backupWithKV.snap"
+
+	// Inspect the snapshot
+	ui := cli.NewMockUi()
+	c := New(ui)
+	args := []string{"-kvdetails", filepath}
+
+	code := c.Run(args)
+	if code != 0 {
+		t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String())
+	}
+
+	want := golden(t, t.Name(), ui.OutputWriter.String())
+	require.Equal(t, want, ui.OutputWriter.String())
+}
+
+func TestSnapshotInspectKVDetailsDepthCommand(t *testing.T) {
+
+	filepath := "./testdata/backupWithKV.snap"
+
+	// Inspect the snapshot
+	ui := cli.NewMockUi()
+	c := New(ui)
+	args := []string{"-kvdetails", "-kvdepth", "3", filepath}
+
+	code := c.Run(args)
+	if code != 0 {
+		t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String())
+	}
+
+	want := golden(t, t.Name(), ui.OutputWriter.String())
+	require.Equal(t, want, ui.OutputWriter.String())
+}
+
+func TestSnapshotInspectKVDetailsDepthFilterCommand(t *testing.T) {
+
+	filepath := "./testdata/backupWithKV.snap"
+
+	// Inspect the snapshot
+	ui := cli.NewMockUi()
+	c := New(ui)
+	args := []string{"-kvdetails", "-kvdepth", "3", "-kvfilter", "vault/logical", filepath}
+
+	code := c.Run(args)
+	if code != 0 {
+		t.Fatalf("bad: %d. %#v", code, ui.ErrorWriter.String())
+	}
+
+	want := golden(t, t.Name(), ui.OutputWriter.String())
+	require.Equal(t, want, ui.OutputWriter.String())
+}

command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsCommand.golden

@@ -0,0 +1,27 @@
+ ID           2-12426-1604593650375
+ Size         17228
+ Index        12426
+ Term         2
+ Version      1
+
+ Type                       Count      Size
+ ----                       ----       ----
+ KVS                        27         12.3KB
+ Register                   5          3.4KB
+ Index                      11         285B
+ Autopilot                  1          199B
+ Session                    1          199B
+ CoordinateBatchUpdate      1          166B
+ Tombstone                  2          146B
+ FederationState            1          139B
+ ChunkingState              1          12B
+ ----                       ----       ----
+ Total                                 16.8KB
+
+ Key Name           Count      Size
+ ----               ----       ----
+ vault/core         16         5.9KB
+ vault/sys          7          4.4KB
+ vault/logical      4          2KB
+ ----               ----       ----
+ Total                         12.3KB

command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsDepthCommand.golden

@@ -0,0 +1,44 @@
+ ID           2-12426-1604593650375
+ Size         17228
+ Index        12426
+ Term         2
+ Version      1
+
+ Type                       Count      Size
+ ----                       ----       ----
+ KVS                        27         12.3KB
+ Register                   5          3.4KB
+ Index                      11         285B
+ Autopilot                  1          199B
+ Session                    1          199B
+ CoordinateBatchUpdate      1          166B
+ Tombstone                  2          146B
+ FederationState            1          139B
+ ChunkingState              1          12B
+ ----                       ----       ----
+ Total                                 16.8KB
+
+ Key Name                                                Count      Size
+ ----                                                    ----       ----
+ vault/sys/policy                                        3          3.3KB
+ vault/logical/0989e79e-06cd-5374-c8c0-4c6d675bc1c9      3          1.8KB
+ vault/core/leader                                       1          1.6KB
+ vault/sys/token                                         3          1KB
+ vault/core/mounts                                       1          675B
+ vault/core/wrapping                                     1          633B
+ vault/core/local-mounts                                 1          450B
+ vault/core/auth                                         1          423B
+ vault/core/cluster                                      2          388B
+ vault/core/keyring                                      1          320B
+ vault/core/master                                       1          237B
+ vault/core/seal-config                                  1          211B
+ vault/logical/5c018b68-3573-41d3-0c33-04bce60cd6b0      1          210B
+ vault/core/hsm                                          1          189B
+ vault/core/local-audit                                  1          185B
+ vault/core/local-auth                                   1          183B
+ vault/core/audit                                        1          179B
+ vault/core/lock                                         1          170B
+ vault/core/shamir-kek                                   1          159B
+ vault/sys/counters                                      1          155B
+ ----                                                    ----       ----
+ Total                                                              12.3KB

command/snapshot/inspect/testdata/TestSnapshotInspectKVDetailsDepthFilterCommand.golden

@@ -0,0 +1,26 @@
+ ID           2-12426-1604593650375
+ Size         17228
+ Index        12426
+ Term         2
+ Version      1
+
+ Type                       Count      Size
+ ----                       ----       ----
+ KVS                        27         12.3KB
+ Register                   5          3.4KB
+ Index                      11         285B
+ Autopilot                  1          199B
+ Session                    1          199B
+ CoordinateBatchUpdate      1          166B
+ Tombstone                  2          146B
+ FederationState            1          139B
+ ChunkingState              1          12B
+ ----                       ----       ----
+ Total                                 16.8KB
+
+ Key Name                                                Count      Size
+ ----                                                    ----       ----
+ vault/logical/0989e79e-06cd-5374-c8c0-4c6d675bc1c9      3          1.8KB
+ vault/logical/5c018b68-3573-41d3-0c33-04bce60cd6b0      1          210B
+ ----                                                    ----       ----
+ Total                                                              2KB

command/snapshot/inspect/testdata/json.golden

@@ -13,5 +13,13 @@
          "Count": 2
       }
    ],
-   "TotalSize": 1
+   "StatsKV": [
+      {
+         "Name": "msgKV",
+         "Sum": 1,
+         "Count": 2
+      }
+   ],
+   "TotalSize": 1,
+   "TotalSizeKV": 1
 }

command/snapshot/inspect/testdata/pretty.golden

@@ -9,3 +9,9 @@
  msg        2          1B
  ----       ----       ----
  Total                 1B
+
+ Key Name      Count      Size
+ ----          ----       ----
+ msgKV         2          1B
+ ----          ----       ----
+ Total                    1B

website/pages/commands/snapshot/inspect.mdx

@@ -38,87 +38,70 @@ To inspect a snapshot from the file "backup.snap":
 
 ```shell-session
 $ consul snapshot inspect backup.snap
- ID           2-13-1603221729747
+ ID           2-12426-1604593650375
- Size         5141
+ Size         17228
- Index        13
+ Index        12426
  Term         2
  Version      1
 
  Type                       Count      Size
  ----                       ----       ----
- Register                    3          1.7KB
+ KVS                        27         12.3KB
- ConnectCA                   1          1.2KB
+ Register                   5          3.4KB
- ConnectCAProviderState      1          1.1KB
+ Index                      11         285B
- Index                       12         344B
  Autopilot                  1          199B
- ConnectCAConfig             1          197B
+ Session                    1          199B
+ CoordinateBatchUpdate      1          166B
+ Tombstone                  2          146B
  FederationState            1          139B
- SystemMetadata              1          68B
  ChunkingState              1          12B
  ----                       ----       ----
- Total                                  5KB
+ Total                                 16.8KB
 ```
 
-To enhance a snapshot inespection from "backup.snap":
+To get more details for a snapshot inspection from "backup.snap":
+
 ```shell-session
-$ consul snapshot inspect -format=json backup.snap
+$ consul snapshot inspect -kvdetails -kvdepth 3 -kvfilter vault/core backup.snap
-{
+ ID           2-12426-1604593650375
-   "Meta": {
+ Size         17228
-      "ID": "2-13-1603221729747",
+ Index        12426
-      "Size": 5141,
+ Term         2
-      "Index": 13,
+ Version      1
-      "Term": 2,
+
-      "Version": 1
+ Type                       Count      Size
-   },
+ ----                       ----       ----
-   "Stats": [
+ KVS                        27         12.3KB
-      {
+ Register                   5          3.4KB
-         "Name": "Register",
+ Index                      11         285B
-         "Sum": 1750,
+ Autopilot                  1          199B
-         "Count": 3
+ Session                    1          199B
-      },
+ CoordinateBatchUpdate      1          166B
-      {
+ Tombstone                  2          146B
-         "Name": "ConnectCA",
+ FederationState            1          139B
-         "Sum": 1258,
+ ChunkingState              1          12B
-         "Count": 1
+ ----                       ----       ----
-      },
+ Total                                 16.8KB
-      {
+
-         "Name": "ConnectCAProviderState",
+ Key Name                     Count      Size
-         "Sum": 1174,
+ ----                         ----       ----
-         "Count": 1
+ vault/core/leader            1          1.6KB
-      },
+ vault/core/mounts            1          675B
-      {
+ vault/core/wrapping          1          633B
-         "Name": "Index",
+ vault/core/local-mounts      1          450B
-         "Sum": 344,
+ vault/core/auth              1          423B
-         "Count": 12
+ vault/core/cluster           2          388B
-      },
+ vault/core/keyring           1          320B
-      {
+ vault/core/master            1          237B
-         "Name": "Autopilot",
+ vault/core/seal-config       1          211B
-         "Sum": 199,
+ vault/core/hsm               1          189B
-         "Count": 1
+ vault/core/local-audit       1          185B
-      },
+ vault/core/local-auth        1          183B
-      {
+ vault/core/audit             1          179B
-         "Name": "ConnectCAConfig",
+ vault/core/lock              1          170B
-         "Sum": 197,
+ vault/core/shamir-kek        1          159B
-         "Count": 1
+ ----                         ----       ----
-      },
+ Total                                   5.9KB
-      {
-         "Name": "FederationState",
-         "Sum": 139,
-         "Count": 1
-      },
-      {
-         "Name": "SystemMetadata",
-         "Sum": 68,
-         "Count": 1
-      },
-      {
-         "Name": "ChunkingState",
-         "Sum": 12,
-         "Count": 1
-      }
-   ],
-   "TotalSize": 5141
-}
 ```
 
 Please see the [HTTP API](/api/snapshot) documentation for
@@ -126,4 +109,7 @@ more details about snapshot internals.
 
 #### Command Options
 
+- `-kvdetails` - Optional, provides a space usage breakdown for any KV data stored in Consul.
+- `-kvdepth` - Can only be used with `-kvdetails`. Used to adjust the grouping level of keys. Defaults to 2.
+- `-kvfilter` - Can only be used with `-kvdetails`. Used to specify a key prefix that excludes keys that don't match.
 - `-format` - Optional, allows from changing the output to JSON. Parameters accepted are "pretty" and "JSON".