Backport of snapshot: some improvments to the snapshot process into release/1.15.x (#17277)

* backport of commit 99c3322dc1

* backport of commit 757f22b518

* backport of commit 7eda9c7f22

* backport of commit 8124fb33ae

* backport of commit 164cada247

---------

Co-authored-by: cskh <hui.kang@hashicorp.com>

.changelog/17236.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+logging: change snapshot log header from `agent.server.snapshot` to `agent.server.raft.snapshot`
+```

agent/consul/server.go

@@ -1050,7 +1050,7 @@ func (s *Server) setupRaft() error {
 		log = cacheStore
 
 		// Create the snapshot store.
-		snapshots, err := raft.NewFileSnapshotStoreWithLogger(path, snapshotsRetained, s.logger.Named("snapshot"))
+		snapshots, err := raft.NewFileSnapshotStoreWithLogger(path, snapshotsRetained, s.logger.Named("raft.snapshot"))
 		if err != nil {
 			return err
 		}

snapshot/snapshot.go

@@ -50,6 +50,7 @@ func New(logger hclog.Logger, r *raft.Raft) (*Snapshot, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to create snapshot file: %v", err)
 	}
+	logger.Debug("creating temporary file of snapshot", "path", archive.Name())
 
 	// If anything goes wrong after this point, we will attempt to clean up
 	// the temp file. The happy path will disarm this.
@@ -109,7 +110,7 @@ func (s *Snapshot) Read(p []byte) (n int, err error) {
 }
 
 // Close closes the snapshot and removes any temporary storage associated with
-// it. You must arrange to call this whenever NewSnapshot() has been called
+// it. You must arrange to call this whenever New() has been called
 // successfully. This is safe to call on a nil snapshot.
 func (s *Snapshot) Close() error {
 	if s == nil {

website/content/commands/snapshot/save.mdx

@@ -20,7 +20,8 @@ If ACLs are enabled, a management token must be supplied in order to perform
 a snapshot save.
 
 -> Note that saving a snapshot involves the server process writing the snapshot to a
-temporary file on-disk before sending that file to the CLI client. The default location
+temporary file on-disk before sending that file to the CLI client. Upon successful completion,
+Consul removes the temporary file. The default location of the temporary file
 can vary depending on operating system, but typically is `/tmp`. You can get more detailed
 information on default locations in the Go documentation for [os.TempDir](https://golang.org/pkg/os/#TempDir).
 If you need to change this location, you can do so by setting the `TMPDIR` environment
@@ -28,6 +29,7 @@ variable for the Consul server processes. Keep in mind that setting the environm
 the CLI client attempting to perform a snapshot save will have no effect. It _must_ be set in
 the context of the server process. If you're using Systemd to manage your Consul server
 processes, then adding `Environment=TMPDIR=/path/to/dir` to your Consul unit file will work.
+As a result of the Raft snapshot, Consul also saves one snapshot file at `data_dir/raft/snapshots`.
 
 The table below shows this command's [required ACLs](/consul/api-docs/api-structure#authentication). Configuration of
 [blocking queries](/consul/api-docs/features/blocking) and [agent caching](/consul/api-docs/features/caching)