Have the built in CA store its state in raft

agent/connect/ca_provider.go

@@ -1,23 +1,9 @@
 package connect
 
 import (
-	"bytes"
-	"crypto/ecdsa"
-	"crypto/elliptic"
-	"crypto/rand"
 	"crypto/x509"
-	"crypto/x509/pkix"
-	"encoding/pem"
-	"fmt"
-	"math/big"
-	"net/url"
-	"sync"
-	"sync/atomic"
-	"time"
 
 	"github.com/hashicorp/consul/agent/structs"
-	uuid "github.com/hashicorp/go-uuid"
-	"github.com/mitchellh/mapstructure"
 )
 
 // CAProvider is the interface for Consul to interact with
@@ -27,252 +13,6 @@ type CAProvider interface {
 	SetConfiguration(raw map[string]interface{}) error
 	ActiveRoot() (*structs.CARoot, error)
 	ActiveIntermediate() (*structs.CARoot, error)
-	RotateIntermediate() error
+	GenerateIntermediate() (*structs.CARoot, error)
 	Sign(*SpiffeIDService, *x509.CertificateRequest) (*structs.IssuedCert, error)
 }
-
-type ConsulCAProviderConfig struct {
-	PrivateKey     string
-	RootCert       string
-	RotationPeriod time.Duration
-}
-
-type ConsulCAProvider struct {
-	config *ConsulCAProviderConfig
-
-	// todo(kyhavlov): store these directly in the state store
-	// and pass a reference to the state to this provider instead of
-	// having these values here
-	privateKey string
-	caRoot     *structs.CARoot
-	caIndex    uint64
-	sync.RWMutex
-}
-
-func NewConsulCAProvider(rawConfig map[string]interface{}) (*ConsulCAProvider, error) {
-	provider := &ConsulCAProvider{}
-	provider.SetConfiguration(rawConfig)
-
-	return provider, nil
-}
-
-func (c *ConsulCAProvider) SetConfiguration(raw map[string]interface{}) error {
-	conf, err := decodeConfig(raw)
-	if err != nil {
-		return err
-	}
-
-	c.config = conf
-	return nil
-}
-
-func decodeConfig(raw map[string]interface{}) (*ConsulCAProviderConfig, error) {
-	var config *ConsulCAProviderConfig
-	if err := mapstructure.WeakDecode(raw, &config); err != nil {
-		return nil, fmt.Errorf("error decoding config: %s", err)
-	}
-
-	return config, nil
-}
-
-func (c *ConsulCAProvider) ActiveRoot() (*structs.CARoot, error) {
-	if c.privateKey == "" {
-		pk, err := generatePrivateKey()
-		if err != nil {
-			return nil, err
-		}
-		c.privateKey = pk
-	}
-
-	if c.caRoot == nil {
-		ca, err := c.generateCA()
-		if err != nil {
-			return nil, err
-		}
-		c.caRoot = ca
-	}
-
-	return c.caRoot, nil
-}
-
-func (c *ConsulCAProvider) ActiveIntermediate() (*structs.CARoot, error) {
-	return c.ActiveRoot()
-}
-
-func (c *ConsulCAProvider) RotateIntermediate() error {
-	ca, err := c.generateCA()
-	if err != nil {
-		return err
-	}
-	c.caRoot = ca
-
-	return nil
-}
-
-// Sign returns a new certificate valid for the given SpiffeIDService
-// using the current CA.
-func (c *ConsulCAProvider) Sign(serviceId *SpiffeIDService, csr *x509.CertificateRequest) (*structs.IssuedCert, error) {
-	// The serial number for the cert.
-	// todo(kyhavlov): increment this based on raft index once the provider uses
-	// the state store directly
-	sn, err := rand.Int(rand.Reader, (&big.Int{}).Exp(big.NewInt(2), big.NewInt(159), nil))
-	if err != nil {
-		return nil, fmt.Errorf("error generating serial number: %s", err)
-	}
-
-	// Create the keyId for the cert from the signing public key.
-	signer, err := ParseSigner(c.privateKey)
-	if err != nil {
-		return nil, err
-	}
-	if signer == nil {
-		return nil, fmt.Errorf("error signing cert: Consul CA not initialized yet")
-	}
-	keyId, err := KeyId(signer.Public())
-	if err != nil {
-		return nil, err
-	}
-
-	// Parse the CA cert
-	caCert, err := ParseCert(c.caRoot.RootCert)
-	if err != nil {
-		return nil, fmt.Errorf("error parsing CA cert: %s", err)
-	}
-
-	// Cert template for generation
-	template := x509.Certificate{
-		SerialNumber:          sn,
-		Subject:               pkix.Name{CommonName: serviceId.Service},
-		URIs:                  csr.URIs,
-		Signature:             csr.Signature,
-		SignatureAlgorithm:    csr.SignatureAlgorithm,
-		PublicKeyAlgorithm:    csr.PublicKeyAlgorithm,
-		PublicKey:             csr.PublicKey,
-		BasicConstraintsValid: true,
-		KeyUsage: x509.KeyUsageDataEncipherment |
-			x509.KeyUsageKeyAgreement |
-			x509.KeyUsageDigitalSignature |
-			x509.KeyUsageKeyEncipherment,
-		ExtKeyUsage: []x509.ExtKeyUsage{
-			x509.ExtKeyUsageClientAuth,
-			x509.ExtKeyUsageServerAuth,
-		},
-		NotAfter:       time.Now().Add(3 * 24 * time.Hour),
-		NotBefore:      time.Now(),
-		AuthorityKeyId: keyId,
-		SubjectKeyId:   keyId,
-	}
-
-	// Create the certificate, PEM encode it and return that value.
-	var buf bytes.Buffer
-	bs, err := x509.CreateCertificate(
-		rand.Reader, &template, caCert, signer.Public(), signer)
-	if err != nil {
-		return nil, fmt.Errorf("error generating certificate: %s", err)
-	}
-	err = pem.Encode(&buf, &pem.Block{Type: "CERTIFICATE", Bytes: bs})
-	if err != nil {
-		return nil, fmt.Errorf("error encoding private key: %s", err)
-	}
-
-	// Set the response
-	return &structs.IssuedCert{
-		SerialNumber: HexString(template.SerialNumber.Bytes()),
-		CertPEM:      buf.String(),
-		Service:      serviceId.Service,
-		ServiceURI:   template.URIs[0].String(),
-		ValidAfter:   template.NotBefore,
-		ValidBefore:  template.NotAfter,
-	}, nil
-}
-
-// generatePrivateKey returns a new private key
-func generatePrivateKey() (string, error) {
-	var pk *ecdsa.PrivateKey
-
-	// If we have no key, then create a new one.
-	pk, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
-	if err != nil {
-		return "", fmt.Errorf("error generating private key: %s", err)
-	}
-
-	bs, err := x509.MarshalECPrivateKey(pk)
-	if err != nil {
-		return "", fmt.Errorf("error generating private key: %s", err)
-	}
-
-	var buf bytes.Buffer
-	err = pem.Encode(&buf, &pem.Block{Type: "EC PRIVATE KEY", Bytes: bs})
-	if err != nil {
-		return "", fmt.Errorf("error encoding private key: %s", err)
-	}
-
-	return buf.String(), nil
-}
-
-// generateCA makes a new root CA using the given private key
-func (c *ConsulCAProvider) generateCA() (*structs.CARoot, error) {
-	privKey, err := ParseSigner(c.privateKey)
-	if err != nil {
-		return nil, err
-	}
-
-	name := fmt.Sprintf("Consul CA %d", atomic.AddUint64(&c.caIndex, 1))
-
-	// The serial number for the cert
-	sn, err := testSerialNumber()
-	if err != nil {
-		return nil, err
-	}
-
-	// The URI (SPIFFE compatible) for the cert
-	id := &SpiffeIDSigning{ClusterID: testClusterID, Domain: "consul"}
-	keyId, err := KeyId(privKey.Public())
-	if err != nil {
-		return nil, err
-	}
-
-	// Create the CA cert
-	template := x509.Certificate{
-		SerialNumber: sn,
-		Subject:      pkix.Name{CommonName: name},
-		URIs:         []*url.URL{id.URI()},
-		PermittedDNSDomainsCritical: true,
-		PermittedDNSDomains:         []string{id.URI().Hostname()},
-		BasicConstraintsValid:       true,
-		KeyUsage: x509.KeyUsageCertSign |
-			x509.KeyUsageCRLSign |
-			x509.KeyUsageDigitalSignature,
-		IsCA:           true,
-		NotAfter:       time.Now().Add(10 * 365 * 24 * time.Hour),
-		NotBefore:      time.Now(),
-		AuthorityKeyId: keyId,
-		SubjectKeyId:   keyId,
-	}
-
-	bs, err := x509.CreateCertificate(
-		rand.Reader, &template, &template, privKey.Public(), privKey)
-	if err != nil {
-		return nil, fmt.Errorf("error generating CA certificate: %s", err)
-	}
-
-	var buf bytes.Buffer
-	err = pem.Encode(&buf, &pem.Block{Type: "CERTIFICATE", Bytes: bs})
-	if err != nil {
-		return nil, fmt.Errorf("error encoding private key: %s", err)
-	}
-
-	// Generate an ID for the new intermediate
-	rootId, err := uuid.GenerateUUID()
-	if err != nil {
-		return nil, err
-	}
-
-	return &structs.CARoot{
-		ID:         rootId,
-		Name:       name,
-		RootCert:   buf.String(),
-		SigningKey: c.privateKey,
-		Active:     true,
-	}, nil
-}

agent/consul/connect_ca_endpoint.go

@@ -61,6 +61,7 @@ func (s *ConnectCA) ConfigurationSet(
 	}
 
 	// Commit
+	// todo(kyhavlov): trigger a bootstrap here when the provider changes
 	args.Op = structs.CAOpSetConfig
 	resp, err := s.srv.raftApply(structs.ConnectCARequestType, args)
 	if err != nil {

agent/consul/connect_ca_provider.go

@@ -0,0 +1,322 @@
+package consul
+
+import (
+	"bytes"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"fmt"
+	"math/big"
+	"net/url"
+	"sync"
+	"time"
+
+	"github.com/hashicorp/consul/agent/connect"
+	"github.com/hashicorp/consul/agent/structs"
+	uuid "github.com/hashicorp/go-uuid"
+	"github.com/mitchellh/mapstructure"
+)
+
+type ConsulCAProviderConfig struct {
+	PrivateKey     string
+	RootCert       string
+	RotationPeriod time.Duration
+}
+
+type ConsulCAProvider struct {
+	config *ConsulCAProviderConfig
+
+	// todo(kyhavlov): store these directly in the state store
+	// and pass a reference to the state to this provider instead of
+	// having these values here
+	srv *Server
+	sync.RWMutex
+}
+
+func NewConsulCAProvider(rawConfig map[string]interface{}, srv *Server) (*ConsulCAProvider, error) {
+	provider := &ConsulCAProvider{srv: srv}
+	provider.SetConfiguration(rawConfig)
+
+	return provider, nil
+}
+
+func (c *ConsulCAProvider) SetConfiguration(raw map[string]interface{}) error {
+	conf, err := decodeConfig(raw)
+	if err != nil {
+		return err
+	}
+
+	c.config = conf
+	return nil
+}
+
+func decodeConfig(raw map[string]interface{}) (*ConsulCAProviderConfig, error) {
+	var config *ConsulCAProviderConfig
+	if err := mapstructure.WeakDecode(raw, &config); err != nil {
+		return nil, fmt.Errorf("error decoding config: %s", err)
+	}
+
+	return config, nil
+}
+
+// Return the active root CA and generate a new one if needed
+func (c *ConsulCAProvider) ActiveRoot() (*structs.CARoot, error) {
+	state := c.srv.fsm.State()
+	_, providerState, err := state.CAProviderState()
+	if err != nil {
+		return nil, err
+	}
+
+	var update bool
+	var newState structs.CAConsulProviderState
+	if providerState != nil {
+		newState = *providerState
+	}
+
+	// Generate a private key if needed
+	if providerState == nil || providerState.PrivateKey == "" {
+		pk, err := generatePrivateKey()
+		if err != nil {
+			return nil, err
+		}
+		newState.PrivateKey = pk
+		update = true
+	}
+
+	// Generate a root CA if needed
+	if providerState == nil || providerState.CARoot == nil {
+		ca, err := c.generateCA(newState.PrivateKey, newState.RootIndex+1)
+		if err != nil {
+			return nil, err
+		}
+		newState.CARoot = ca
+		newState.RootIndex += 1
+		update = true
+	}
+
+	// Update the provider state if we generated a new private key/cert
+	if update {
+		args := &structs.CARequest{
+			Op:            structs.CAOpSetProviderState,
+			ProviderState: &newState,
+		}
+		resp, err := c.srv.raftApply(structs.ConnectCARequestType, args)
+		if err != nil {
+			return nil, err
+		}
+		if respErr, ok := resp.(error); ok {
+			return nil, respErr
+		}
+	}
+	return newState.CARoot, nil
+}
+
+func (c *ConsulCAProvider) ActiveIntermediate() (*structs.CARoot, error) {
+	return c.ActiveRoot()
+}
+
+func (c *ConsulCAProvider) GenerateIntermediate() (*structs.CARoot, error) {
+	state := c.srv.fsm.State()
+	_, providerState, err := state.CAProviderState()
+	if err != nil {
+		return nil, err
+	}
+	if providerState == nil {
+		return nil, fmt.Errorf("CA provider not yet initialized")
+	}
+
+	ca, err := c.generateCA(providerState.PrivateKey, providerState.RootIndex+1)
+	if err != nil {
+		return nil, err
+	}
+
+	return ca, nil
+}
+
+// Sign returns a new certificate valid for the given SpiffeIDService
+// using the current CA.
+func (c *ConsulCAProvider) Sign(serviceId *connect.SpiffeIDService, csr *x509.CertificateRequest) (*structs.IssuedCert, error) {
+	// Get the provider state
+	state := c.srv.fsm.State()
+	_, providerState, err := state.CAProviderState()
+	if err != nil {
+		return nil, err
+	}
+
+	// Create the keyId for the cert from the signing public key.
+	signer, err := connect.ParseSigner(providerState.PrivateKey)
+	if err != nil {
+		return nil, err
+	}
+	if signer == nil {
+		return nil, fmt.Errorf("error signing cert: Consul CA not initialized yet")
+	}
+	keyId, err := connect.KeyId(signer.Public())
+	if err != nil {
+		return nil, err
+	}
+
+	// Parse the CA cert
+	caCert, err := connect.ParseCert(providerState.CARoot.RootCert)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing CA cert: %s", err)
+	}
+
+	// Cert template for generation
+	sn := &big.Int{}
+	sn.SetUint64(providerState.LeafIndex + 1)
+	template := x509.Certificate{
+		SerialNumber:          sn,
+		Subject:               pkix.Name{CommonName: serviceId.Service},
+		URIs:                  csr.URIs,
+		Signature:             csr.Signature,
+		SignatureAlgorithm:    csr.SignatureAlgorithm,
+		PublicKeyAlgorithm:    csr.PublicKeyAlgorithm,
+		PublicKey:             csr.PublicKey,
+		BasicConstraintsValid: true,
+		KeyUsage: x509.KeyUsageDataEncipherment |
+			x509.KeyUsageKeyAgreement |
+			x509.KeyUsageDigitalSignature |
+			x509.KeyUsageKeyEncipherment,
+		ExtKeyUsage: []x509.ExtKeyUsage{
+			x509.ExtKeyUsageClientAuth,
+			x509.ExtKeyUsageServerAuth,
+		},
+		NotAfter:       time.Now().Add(3 * 24 * time.Hour),
+		NotBefore:      time.Now(),
+		AuthorityKeyId: keyId,
+		SubjectKeyId:   keyId,
+	}
+
+	// Create the certificate, PEM encode it and return that value.
+	var buf bytes.Buffer
+	bs, err := x509.CreateCertificate(
+		rand.Reader, &template, caCert, signer.Public(), signer)
+	if err != nil {
+		return nil, fmt.Errorf("error generating certificate: %s", err)
+	}
+	err = pem.Encode(&buf, &pem.Block{Type: "CERTIFICATE", Bytes: bs})
+	if err != nil {
+		return nil, fmt.Errorf("error encoding private key: %s", err)
+	}
+
+	// Increment the leaf cert index
+	newState := *providerState
+	newState.LeafIndex += 1
+	args := &structs.CARequest{
+		Op:            structs.CAOpSetProviderState,
+		ProviderState: &newState,
+	}
+	resp, err := c.srv.raftApply(structs.ConnectCARequestType, args)
+	if err != nil {
+		return nil, err
+	}
+	if respErr, ok := resp.(error); ok {
+		return nil, respErr
+	}
+
+	// Set the response
+	return &structs.IssuedCert{
+		SerialNumber: connect.HexString(template.SerialNumber.Bytes()),
+		CertPEM:      buf.String(),
+		Service:      serviceId.Service,
+		ServiceURI:   template.URIs[0].String(),
+		ValidAfter:   template.NotBefore,
+		ValidBefore:  template.NotAfter,
+	}, nil
+}
+
+// generatePrivateKey returns a new private key
+func generatePrivateKey() (string, error) {
+	var pk *ecdsa.PrivateKey
+
+	// If we have no key, then create a new one.
+	pk, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		return "", fmt.Errorf("error generating private key: %s", err)
+	}
+
+	bs, err := x509.MarshalECPrivateKey(pk)
+	if err != nil {
+		return "", fmt.Errorf("error generating private key: %s", err)
+	}
+
+	var buf bytes.Buffer
+	err = pem.Encode(&buf, &pem.Block{Type: "EC PRIVATE KEY", Bytes: bs})
+	if err != nil {
+		return "", fmt.Errorf("error encoding private key: %s", err)
+	}
+
+	return buf.String(), nil
+}
+
+// generateCA makes a new root CA using the current private key
+func (c *ConsulCAProvider) generateCA(privateKey string, sn uint64) (*structs.CARoot, error) {
+	state := c.srv.fsm.State()
+	_, config, err := state.CAConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	privKey, err := connect.ParseSigner(privateKey)
+	if err != nil {
+		return nil, err
+	}
+
+	name := fmt.Sprintf("Consul CA %d", sn)
+
+	// The URI (SPIFFE compatible) for the cert
+	id := &connect.SpiffeIDSigning{ClusterID: config.ClusterSerial, Domain: "consul"}
+	keyId, err := connect.KeyId(privKey.Public())
+	if err != nil {
+		return nil, err
+	}
+
+	// Create the CA cert
+	serialNum := &big.Int{}
+	serialNum.SetUint64(sn)
+	template := x509.Certificate{
+		SerialNumber: serialNum,
+		Subject:      pkix.Name{CommonName: name},
+		URIs:         []*url.URL{id.URI()},
+		PermittedDNSDomainsCritical: true,
+		PermittedDNSDomains:         []string{id.URI().Hostname()},
+		BasicConstraintsValid:       true,
+		KeyUsage: x509.KeyUsageCertSign |
+			x509.KeyUsageCRLSign |
+			x509.KeyUsageDigitalSignature,
+		IsCA:           true,
+		NotAfter:       time.Now().Add(10 * 365 * 24 * time.Hour),
+		NotBefore:      time.Now(),
+		AuthorityKeyId: keyId,
+		SubjectKeyId:   keyId,
+	}
+
+	bs, err := x509.CreateCertificate(
+		rand.Reader, &template, &template, privKey.Public(), privKey)
+	if err != nil {
+		return nil, fmt.Errorf("error generating CA certificate: %s", err)
+	}
+
+	var buf bytes.Buffer
+	err = pem.Encode(&buf, &pem.Block{Type: "CERTIFICATE", Bytes: bs})
+	if err != nil {
+		return nil, fmt.Errorf("error encoding private key: %s", err)
+	}
+
+	// Generate an ID for the new CA cert
+	rootId, err := uuid.GenerateUUID()
+	if err != nil {
+		return nil, err
+	}
+
+	return &structs.CARoot{
+		ID:       rootId,
+		Name:     name,
+		RootCert: buf.String(),
+		Active:   true,
+	}, nil
+}

agent/consul/fsm/commands_oss.go

@@ -300,6 +300,13 @@ func (c *FSM) applyConnectCAOperation(buf []byte, index uint64) interface{} {
 			return err
 		}
 
+		return act
+	case structs.CAOpSetProviderState:
+		act, err := c.state.CASetProviderState(index, req.ProviderState)
+		if err != nil {
+			return err
+		}
+
 		return act
 	default:
 		c.logger.Printf("[WARN] consul.fsm: Invalid CA operation '%s'", req.Op)

agent/consul/leader.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/hashicorp/consul/agent/connect"
+	uuid "github.com/hashicorp/go-uuid"
 
 	"github.com/armon/go-metrics"
 	"github.com/hashicorp/consul/acl"
@@ -377,7 +378,13 @@ func (s *Server) getOrCreateCAConfig() (*structs.CAConfiguration, error) {
 		return config, nil
 	}
 
+	sn, err := uuid.GenerateUUID()
+	if err != nil {
+		return nil, err
+	}
+
 	config = s.config.CAConfig
+	config.ClusterSerial = sn
 	req := structs.CARequest{
 		Op:     structs.CAOpSetConfig,
 		Config: config,
@@ -400,7 +407,7 @@ func (s *Server) bootstrapCA() error {
 	var provider connect.CAProvider
 	switch conf.Provider {
 	case structs.ConsulCAProvider:
-		provider, err = connect.NewConsulCAProvider(conf.Config)
+		provider, err = NewConsulCAProvider(conf.Config, s)
 		if err != nil {
 			return err
 		}
@@ -412,10 +419,10 @@ func (s *Server) bootstrapCA() error {
 	s.caProvider = provider
 	s.caProviderLock.Unlock()
 
-	// Get the intermediate cert from the CA
+	// Get the active root cert from the CA
-	trustedCA, err := provider.ActiveIntermediate()
+	trustedCA, err := provider.ActiveRoot()
 	if err != nil {
-		return fmt.Errorf("error getting intermediate cert: %v", err)
+		return fmt.Errorf("error getting root cert: %v", err)
 	}
 
 	// Check if this CA is already initialized
@@ -435,7 +442,7 @@ func (s *Server) bootstrapCA() error {
 		return err
 	}
 
-	// Store the intermediate in raft
+	// Store the root cert in raft
 	resp, err := s.raftApply(structs.ConnectCARequestType, &structs.CARequest{
 		Op:    structs.CAOpSetRoots,
 		Index: idx,

agent/consul/state/connect_ca.go

@@ -10,6 +10,7 @@ import (
 const (
 	caConfigTableName   = "connect-ca-config"
 	caRootTableName     = "connect-ca-roots"
+	caProviderTableName = "connect-ca-builtin"
 )
 
 // caConfigTableSchema returns a new table schema used for storing
@@ -48,14 +49,34 @@ func caRootTableSchema() *memdb.TableSchema {
 	}
 }
 
+// caProviderTableSchema returns a new table schema used for storing
+// the built-in CA provider's state for connect. This is only used by
+// the internal Consul CA provider.
+func caProviderTableSchema() *memdb.TableSchema {
+	return &memdb.TableSchema{
+		Name: caProviderTableName,
+		Indexes: map[string]*memdb.IndexSchema{
+			"id": &memdb.IndexSchema{
+				Name:         "id",
+				AllowMissing: false,
+				Unique:       true,
+				Indexer: &memdb.ConditionalIndex{
+					Conditional: func(obj interface{}) (bool, error) { return true, nil },
+				},
+			},
+		},
+	}
+}
+
 func init() {
 	registerSchema(caConfigTableSchema)
 	registerSchema(caRootTableSchema)
+	registerSchema(caProviderTableSchema)
 }
 
 // CAConfig is used to pull the CA config from the snapshot.
 func (s *Snapshot) CAConfig() (*structs.CAConfiguration, error) {
-	c, err := s.tx.First("connect-ca-config", "id")
+	c, err := s.tx.First(caConfigTableName, "id")
 	if err != nil {
 		return nil, err
 	}
@@ -70,7 +91,7 @@ func (s *Snapshot) CAConfig() (*structs.CAConfiguration, error) {
 
 // CAConfig is used when restoring from a snapshot.
 func (s *Restore) CAConfig(config *structs.CAConfiguration) error {
-	if err := s.tx.Insert("connect-ca-config", config); err != nil {
+	if err := s.tx.Insert(caConfigTableName, config); err != nil {
 		return fmt.Errorf("failed restoring CA config: %s", err)
 	}
 
@@ -83,7 +104,7 @@ func (s *Store) CAConfig() (uint64, *structs.CAConfiguration, error) {
 	defer tx.Abort()
 
 	// Get the autopilot config
-	c, err := tx.First("connect-ca-config", "id")
+	c, err := tx.First(caConfigTableName, "id")
 	if err != nil {
 		return 0, nil, fmt.Errorf("failed CA config lookup: %s", err)
 	}
@@ -101,7 +122,9 @@ func (s *Store) CASetConfig(idx uint64, config *structs.CAConfiguration) error {
 	tx := s.db.Txn(true)
 	defer tx.Abort()
 
-	s.caSetConfigTxn(idx, tx, config)
+	if err := s.caSetConfigTxn(idx, tx, config); err != nil {
+		return err
+	}
 
 	tx.Commit()
 	return nil
@@ -115,7 +138,7 @@ func (s *Store) CACheckAndSetConfig(idx, cidx uint64, config *structs.CAConfigur
 	defer tx.Abort()
 
 	// Check for an existing config
-	existing, err := tx.First("connect-ca-config", "id")
+	existing, err := tx.First(caConfigTableName, "id")
 	if err != nil {
 		return false, fmt.Errorf("failed CA config lookup: %s", err)
 	}
@@ -128,7 +151,9 @@ func (s *Store) CACheckAndSetConfig(idx, cidx uint64, config *structs.CAConfigur
 		return false, nil
 	}
 
-	s.caSetConfigTxn(idx, tx, config)
+	if err := s.caSetConfigTxn(idx, tx, config); err != nil {
+		return false, err
+	}
 
 	tx.Commit()
 	return true, nil
@@ -136,20 +161,22 @@ func (s *Store) CACheckAndSetConfig(idx, cidx uint64, config *structs.CAConfigur
 
 func (s *Store) caSetConfigTxn(idx uint64, tx *memdb.Txn, config *structs.CAConfiguration) error {
 	// Check for an existing config
-	existing, err := tx.First("connect-ca-config", "id")
+	prev, err := tx.First(caConfigTableName, "id")
 	if err != nil {
 		return fmt.Errorf("failed CA config lookup: %s", err)
 	}
 
-	// Set the indexes.
+	// Set the indexes, prevent the cluster ID from changing.
-	if existing != nil {
+	if prev != nil {
-		config.CreateIndex = existing.(*structs.CAConfiguration).CreateIndex
+		existing := prev.(*structs.CAConfiguration)
+		config.CreateIndex = existing.CreateIndex
+		config.ClusterSerial = existing.ClusterSerial
 	} else {
 		config.CreateIndex = idx
 	}
 	config.ModifyIndex = idx
 
-	if err := tx.Insert("connect-ca-config", config); err != nil {
+	if err := tx.Insert(caConfigTableName, config); err != nil {
 		return fmt.Errorf("failed updating CA config: %s", err)
 	}
 	return nil
@@ -289,3 +316,73 @@ func (s *Store) CARootSetCAS(idx, cidx uint64, rs []*structs.CARoot) (bool, erro
 	tx.Commit()
 	return true, nil
 }
+
+// CAProviderState is used to pull the built-in provider state from the snapshot.
+func (s *Snapshot) CAProviderState() (*structs.CAConsulProviderState, error) {
+	c, err := s.tx.First(caProviderTableName, "id")
+	if err != nil {
+		return nil, err
+	}
+
+	state, ok := c.(*structs.CAConsulProviderState)
+	if !ok {
+		return nil, nil
+	}
+
+	return state, nil
+}
+
+// CAProviderState is used when restoring from a snapshot.
+func (s *Restore) CAProviderState(state *structs.CAConsulProviderState) error {
+	if err := s.tx.Insert(caProviderTableName, state); err != nil {
+		return fmt.Errorf("failed restoring built-in CA state: %s", err)
+	}
+
+	return nil
+}
+
+// CAProviderState is used to get the current Consul CA provider state.
+func (s *Store) CAProviderState() (uint64, *structs.CAConsulProviderState, error) {
+	tx := s.db.Txn(false)
+	defer tx.Abort()
+
+	// Get the autopilot config
+	c, err := tx.First(caProviderTableName, "id")
+	if err != nil {
+		return 0, nil, fmt.Errorf("failed built-in CA state lookup: %s", err)
+	}
+
+	state, ok := c.(*structs.CAConsulProviderState)
+	if !ok {
+		return 0, nil, nil
+	}
+
+	return state.ModifyIndex, state, nil
+}
+
+// CASetProviderState is used to set the current built-in CA provider state.
+func (s *Store) CASetProviderState(idx uint64, state *structs.CAConsulProviderState) (bool, error) {
+	tx := s.db.Txn(true)
+	defer tx.Abort()
+
+	// Check for an existing config
+	existing, err := tx.First(caProviderTableName, "id")
+	if err != nil {
+		return false, fmt.Errorf("failed built-in CA state lookup: %s", err)
+	}
+
+	// Set the indexes.
+	if existing != nil {
+		state.CreateIndex = existing.(*structs.CAConfiguration).CreateIndex
+	} else {
+		state.CreateIndex = idx
+	}
+	state.ModifyIndex = idx
+
+	if err := tx.Insert(caProviderTableName, state); err != nil {
+		return false, fmt.Errorf("failed updating built-in CA state: %s", err)
+	}
+	tx.Commit()
+
+	return true, nil
+}

agent/structs/connect_ca.go

@@ -98,6 +98,7 @@ type CAOp string
 const (
 	CAOpSetRoots         CAOp = "set-roots"
 	CAOpSetConfig        CAOp = "set-config"
+	CAOpSetProviderState CAOp = "set-provider-state"
 )
 
 // CARequest is used to modify connect CA data. This is used by the
@@ -110,7 +111,7 @@ type CARequest struct {
 	// Datacenter is the target for this request.
 	Datacenter string
 
-	// Index is used by CAOpSet for a CAS operation.
+	// Index is used by CAOpSetRoots and CAOpSetConfig for a CAS operation.
 	Index uint64
 
 	// Roots is a list of roots. This is used for CAOpSet. One root must
@@ -120,6 +121,9 @@ type CARequest struct {
 	// Config is the configuration for the current CA plugin.
 	Config *CAConfiguration
 
+	// ProviderState is the state for the builtin CA provider.
+	ProviderState *CAConsulProviderState
+
 	// WriteRequest is a common struct containing ACL tokens and other
 	// write-related common elements for requests.
 	WriteRequest
@@ -136,6 +140,9 @@ const (
 
 // CAConfiguration is the configuration for the current CA plugin.
 type CAConfiguration struct {
+	// Unique identifier for the cluster
+	ClusterSerial string `json:"-"`
+
 	// Provider is the CA provider implementation to use.
 	Provider string
 
@@ -144,7 +151,15 @@ type CAConfiguration struct {
 	// and maps).
 	Config map[string]interface{}
 
-	// CreateIndex/ModifyIndex store the create/modify indexes of this configuration.
+	RaftIndex
-	CreateIndex uint64
+}
-	ModifyIndex uint64
+
+// CAConsulProviderState is used to track the built-in Consul CA provider's state.
+type CAConsulProviderState struct {
+	PrivateKey string
+	CARoot     *CARoot
+	RootIndex  uint64
+	LeafIndex  uint64
+
+	RaftIndex
 }