// Copyright 2021 The etcd Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package verify import ( "fmt" "os" "go.etcd.io/etcd/raft/v3/raftpb" "go.etcd.io/etcd/server/v3/datadir" "go.etcd.io/etcd/server/v3/etcdserver/cindex" "go.etcd.io/etcd/server/v3/mvcc/backend" wal2 "go.etcd.io/etcd/server/v3/wal" "go.etcd.io/etcd/server/v3/wal/walpb" "go.uber.org/zap" ) const ENV_VERIFY = "ETCD_VERIFY" const ENV_VERIFY_ALL_VALUE = "all" type Config struct { // DataDir is a root directory where the data being verified are stored. DataDir string // ExactIndex requires consistent_index in backend exactly match the last committed WAL entry. // Usually backend's consistent_index needs to be <= WAL.commit, but for backups the match // is expected to be exact. ExactIndex bool Logger *zap.Logger } // Verify performs consistency checks of given etcd data-directory. // The errors are reported as the returned error, but for some situations // the function can also panic. // The function is expected to work on not-in-use data model, i.e. // no file-locks should be taken. Verify does not modified the data. func Verify(cfg Config) error { lg := cfg.Logger if lg == nil { lg = zap.NewNop() } var err error lg.Info("verification of persisted state", zap.String("data-dir", cfg.DataDir)) defer func() { if err != nil { lg.Error("verification of persisted state failed", zap.String("data-dir", cfg.DataDir), zap.Error(err)) } else if r := recover(); r != nil { lg.Error("verification of persisted state failed", zap.String("data-dir", cfg.DataDir)) panic(r) } else { lg.Info("verification of persisted state successful", zap.String("data-dir", cfg.DataDir)) } }() beConfig := backend.DefaultBackendConfig() beConfig.Path = datadir.ToBackendFileName(cfg.DataDir) beConfig.Logger = cfg.Logger be := backend.New(beConfig) defer be.Close() snapshot, hardstate, err := validateWal(cfg) if err != nil { return err } // TODO: Perform validation of consistency of membership between // backend/members & WAL confstate (and maybe storev2 if still exists). return validateConsistentIndex(cfg, hardstate, snapshot, be) } // VerifyIfEnabled performs verification according to ETCD_VERIFY env settings. // See Verify for more information. func VerifyIfEnabled(cfg Config) error { if os.Getenv(ENV_VERIFY) == ENV_VERIFY_ALL_VALUE { return Verify(cfg) } return nil } // MustVerifyIfEnabled performs verification according to ETCD_VERIFY env settings // and exits in case of found problems. // See Verify for more information. func MustVerifyIfEnabled(cfg Config) { if err := VerifyIfEnabled(cfg); err != nil { cfg.Logger.Fatal("Verification failed", zap.String("data-dir", cfg.DataDir), zap.Error(err)) } } func validateConsistentIndex(cfg Config, hardstate *raftpb.HardState, snapshot *walpb.Snapshot, be backend.Backend) error { tx := be.BatchTx() index, term := cindex.ReadConsistentIndex(tx) if cfg.ExactIndex && index != hardstate.Commit { return fmt.Errorf("backend.ConsistentIndex (%v) expected == WAL.HardState.commit (%v)", index, hardstate.Commit) } if cfg.ExactIndex && term != hardstate.Term { return fmt.Errorf("backend.Term (%v) expected == WAL.HardState.term, (%v)", term, hardstate.Term) } if index > hardstate.Commit { return fmt.Errorf("backend.ConsistentIndex (%v) must be <= WAL.HardState.commit (%v)", index, hardstate.Commit) } if term > hardstate.Term { return fmt.Errorf("backend.Term (%v) must be <= WAL.HardState.term, (%v)", term, hardstate.Term) } if index < snapshot.Index { return fmt.Errorf("backend.ConsistentIndex (%v) must be >= last snapshot index (%v)", index, snapshot.Index) } cfg.Logger.Info("verification: consistentIndex OK", zap.Uint64("backend-consistent-index", index), zap.Uint64("hardstate-commit", hardstate.Commit)) return nil } func validateWal(cfg Config) (*walpb.Snapshot, *raftpb.HardState, error) { walDir := datadir.ToWalDir(cfg.DataDir) walSnaps, err := wal2.ValidSnapshotEntries(cfg.Logger, walDir) if err != nil { return nil, nil, err } snapshot := walSnaps[len(walSnaps)-1] hardstate, err := wal2.Verify(cfg.Logger, walDir, snapshot) if err != nil { return nil, nil, err } return &snapshot, hardstate, nil }