2020-05-05 21:59:15 +00:00
package cluster
import (
"bytes"
"context"
2021-10-07 19:47:00 +00:00
"encoding/json"
"fmt"
"io"
"io/ioutil"
2021-11-23 14:46:06 +00:00
"net"
2020-05-05 21:59:15 +00:00
"os"
"path/filepath"
2021-11-23 14:46:06 +00:00
"reflect"
2021-12-09 21:54:27 +00:00
"strconv"
2021-10-07 19:47:00 +00:00
"strings"
"time"
2020-05-05 21:59:15 +00:00
2022-03-02 23:47:27 +00:00
"github.com/k3s-io/k3s/pkg/bootstrap"
"github.com/k3s-io/k3s/pkg/clientaccess"
"github.com/k3s-io/k3s/pkg/daemons/config"
"github.com/k3s-io/k3s/pkg/etcd"
"github.com/k3s-io/k3s/pkg/version"
2021-10-07 19:47:00 +00:00
"github.com/k3s-io/kine/pkg/client"
2021-11-10 12:33:42 +00:00
"github.com/k3s-io/kine/pkg/endpoint"
2021-12-09 21:54:27 +00:00
"github.com/otiai10/copy"
2022-01-06 23:32:29 +00:00
"github.com/pkg/errors"
2020-05-05 21:59:15 +00:00
"github.com/sirupsen/logrus"
)
2020-09-24 06:29:25 +00:00
// Bootstrap attempts to load a managed database driver, if one has been initialized or should be created/joined.
2020-12-22 20:35:58 +00:00
// It then checks to see if the cluster needs to load bootstrap data, and if so, loads data into the
2020-09-24 06:29:25 +00:00
// ControlRuntimeBoostrap struct, either via HTTP or from the datastore.
2021-11-10 12:33:42 +00:00
func ( c * Cluster ) Bootstrap ( ctx context . Context , snapshot bool ) error {
2020-05-05 21:59:15 +00:00
if err := c . assignManagedDriver ( ctx ) ; err != nil {
return err
}
2021-11-10 12:33:42 +00:00
shouldBootstrap , isInitialized , err := c . shouldBootstrapLoad ( ctx )
2020-05-05 21:59:15 +00:00
if err != nil {
return err
}
2020-09-24 06:43:53 +00:00
c . shouldBootstrap = shouldBootstrap
2021-11-10 12:33:42 +00:00
if c . managedDB != nil {
if ! snapshot {
2021-12-15 23:38:50 +00:00
isHTTP := c . config . JoinURL != "" && c . config . Token != ""
// For secondary servers, we attempt to connect and reconcile with the datastore.
// If that fails we fallback to the local etcd cluster start
if isInitialized && isHTTP && c . clientAccessInfo != nil {
if err := c . httpBootstrap ( ctx ) ; err == nil {
logrus . Info ( "Successfully reconciled with datastore" )
return nil
}
logrus . Warnf ( "Unable to reconcile with datastore: %v" , err )
}
2021-11-10 12:33:42 +00:00
// In the case of etcd, if the database has been initialized, it doesn't
// need to be bootstrapped however we still need to check the database
// and reconcile the bootstrap data. Below we're starting a temporary
// instance of etcd in the event that etcd certificates are unavailable,
// reading the data, and comparing that to the data on disk, all the while
// starting normal etcd.
2021-12-15 23:38:50 +00:00
if isInitialized {
2022-02-24 22:35:08 +00:00
if err := c . reconcileEtcd ( ctx ) ; err != nil {
logrus . Fatalf ( "Failed to reconcile with temporary etcd: %v" , err )
2021-11-10 12:33:42 +00:00
}
}
}
}
2021-10-07 19:47:00 +00:00
if c . shouldBootstrap {
return c . bootstrap ( ctx )
2020-05-05 21:59:15 +00:00
}
return nil
}
2021-11-10 12:33:42 +00:00
// shouldBootstrapLoad returns true if we need to load ControlRuntimeBootstrap data again and a second boolean
// indicating that the server has or has not been initialized, if etcd. This is controlled by a stamp file on
// disk that records successful bootstrap using a hash of the join token.
func ( c * Cluster ) shouldBootstrapLoad ( ctx context . Context ) ( bool , bool , error ) {
2020-09-24 05:22:15 +00:00
// Non-nil managedDB indicates that the database is either initialized, initializing, or joining
2020-05-05 21:59:15 +00:00
if c . managedDB != nil {
2022-02-24 19:01:14 +00:00
c . config . Runtime . HTTPBootstrap = true
2020-05-05 21:59:15 +00:00
2020-09-24 05:22:15 +00:00
isInitialized , err := c . managedDB . IsInitialized ( ctx , c . config )
2020-05-05 21:59:15 +00:00
if err != nil {
2021-11-10 12:33:42 +00:00
return false , false , err
2020-05-05 21:59:15 +00:00
}
2020-09-24 05:22:15 +00:00
if isInitialized {
2021-12-15 23:38:50 +00:00
// If the database is initialized we skip bootstrapping; if the user wants to rejoin a
// cluster they need to delete the database.
logrus . Infof ( "Managed %s cluster bootstrap already complete and initialized" , c . managedDB . EndpointName ( ) )
2020-09-30 10:37:45 +00:00
// This is a workaround for an issue that can be caused by terminating the cluster bootstrap before
// etcd is promoted from learner. Odds are we won't need this info, and we don't want to fail startup
// due to failure to retrieve it as this will break cold cluster restart, so we ignore any errors.
if c . config . JoinURL != "" && c . config . Token != "" {
c . clientAccessInfo , _ = clientaccess . ParseAndValidateTokenForUser ( c . config . JoinURL , c . config . Token , "server" )
}
2021-11-10 12:33:42 +00:00
return false , true , nil
2020-09-24 05:22:15 +00:00
} else if c . config . JoinURL == "" {
// Not initialized, not joining - must be initializing (cluster-init)
logrus . Infof ( "Managed %s cluster initializing" , c . managedDB . EndpointName ( ) )
2021-11-10 12:33:42 +00:00
return false , false , nil
2020-09-24 05:22:15 +00:00
} else {
// Not initialized, but have a Join URL - fail if there's no token; if there is then validate it.
if c . config . Token == "" {
2021-11-10 12:33:42 +00:00
return false , false , errors . New ( version . ProgramUpper + "_TOKEN is required to join a cluster" )
2020-09-24 05:22:15 +00:00
}
2020-09-24 06:47:17 +00:00
// Fail if the token isn't syntactically valid, or if the CA hash on the remote server doesn't match
// the hash in the token. The password isn't actually checked until later when actually bootstrapping.
info , err := clientaccess . ParseAndValidateTokenForUser ( c . config . JoinURL , c . config . Token , "server" )
2020-09-24 05:22:15 +00:00
if err != nil {
2021-11-10 12:33:42 +00:00
return false , false , err
2020-09-24 05:22:15 +00:00
}
logrus . Infof ( "Managed %s cluster not yet initialized" , c . managedDB . EndpointName ( ) )
c . clientAccessInfo = info
2020-05-05 21:59:15 +00:00
}
}
2020-09-24 05:22:15 +00:00
// No errors and no bootstrap stamp, need to bootstrap.
2021-11-10 12:33:42 +00:00
return true , false , nil
2020-05-05 21:59:15 +00:00
}
2021-10-07 19:47:00 +00:00
// isDirEmpty checks to see if the given directory
// is empty.
func isDirEmpty ( name string ) ( bool , error ) {
f , err := os . Open ( name )
if err != nil {
return false , err
}
defer f . Close ( )
_ , err = f . Readdir ( 1 )
if err == io . EOF {
return true , nil
}
return false , err
}
// certDirsExist checks to see if the directories
// that contain the needed certificates exist.
func ( c * Cluster ) certDirsExist ( ) error {
bootstrapDirs := [ ] string {
2021-11-10 12:33:42 +00:00
"cred" ,
2021-10-07 19:47:00 +00:00
"tls" ,
"tls/etcd" ,
}
const (
missingDir = "missing %s directory from ${data-dir}"
emptyDir = "%s directory is empty"
)
for _ , dir := range bootstrapDirs {
d := filepath . Join ( c . config . DataDir , dir )
if _ , err := os . Stat ( d ) ; os . IsNotExist ( err ) {
errMsg := fmt . Sprintf ( missingDir , d )
logrus . Debug ( errMsg )
return errors . New ( errMsg )
}
ok , err := isDirEmpty ( d )
if err != nil {
return err
}
if ok {
errMsg := fmt . Sprintf ( emptyDir , d )
logrus . Debug ( errMsg )
return errors . New ( errMsg )
}
}
return nil
}
// migrateBootstrapData migrates bootstrap data from the old format to the new format.
2021-10-15 17:17:29 +00:00
func migrateBootstrapData ( ctx context . Context , data io . Reader , files bootstrap . PathsDataformat ) error {
2021-10-07 19:47:00 +00:00
logrus . Info ( "Migrating bootstrap data to new format" )
var oldBootstrapData map [ string ] [ ] byte
if err := json . NewDecoder ( data ) . Decode ( & oldBootstrapData ) ; err != nil {
// if this errors here, we can assume that the error being thrown
// is not related to needing to perform a migration.
2020-05-05 21:59:15 +00:00
return err
}
2021-10-07 19:47:00 +00:00
// iterate through the old bootstrap data structure
// and copy into the new bootstrap data structure
for k , v := range oldBootstrapData {
files [ k ] = bootstrap . File {
Content : v ,
}
2020-05-05 21:59:15 +00:00
}
2021-10-07 19:47:00 +00:00
return nil
}
const systemTimeSkew = int64 ( 3 )
2021-12-13 20:02:30 +00:00
// isMigrated checks to see if the given bootstrap data
// is in the latest format.
2021-12-14 00:04:39 +00:00
func isMigrated ( buf io . ReadSeeker , files * bootstrap . PathsDataformat ) bool {
2021-12-13 20:02:30 +00:00
buf . Seek ( 0 , 0 )
defer buf . Seek ( 0 , 0 )
2021-12-14 00:04:39 +00:00
if err := json . NewDecoder ( buf ) . Decode ( files ) ; err != nil {
2021-12-13 20:02:30 +00:00
// This will fail if data is being pulled from old an cluster since
// older clusters used a map[string][]byte for the data structure.
// Therefore, we need to perform a migration to the newer bootstrap
// format; bootstrap.BootstrapFile.
return false
}
return true
}
2021-10-07 19:47:00 +00:00
// ReconcileBootstrapData is called before any data is saved to the
// datastore or locally. It checks to see if the contents of the
// bootstrap data in the datastore is newer than on disk or different
2021-12-13 20:02:30 +00:00
// and depending on where the difference is. If the datastore is newer,
// then the data will be written to disk. If the data on disk is newer,
// k3s will exit with an error.
2022-02-24 22:35:08 +00:00
func ( c * Cluster ) ReconcileBootstrapData ( ctx context . Context , buf io . ReadSeeker , crb * config . ControlRuntimeBootstrap , isHTTP bool ) error {
2021-10-07 19:47:00 +00:00
logrus . Info ( "Reconciling bootstrap data between datastore and disk" )
if err := c . certDirsExist ( ) ; err != nil {
2021-12-13 20:02:30 +00:00
// we need to see if the data has been migrated before writing to disk. This
// is because the data may have been given to us via the HTTP bootstrap process
// from an older version of k3s. That version might not have the new data format
// and we should write the correct format.
files := make ( bootstrap . PathsDataformat )
2021-12-14 00:04:39 +00:00
if ! isMigrated ( buf , & files ) {
2021-12-13 20:02:30 +00:00
if err := migrateBootstrapData ( ctx , buf , files ) ; err != nil {
return err
}
buf . Seek ( 0 , 0 )
}
2022-02-16 22:19:58 +00:00
logrus . Debugf ( "One or more certificate directories do not exist; writing data to disk from datastore" )
2021-12-13 20:02:30 +00:00
return bootstrap . WriteToDiskFromStorage ( files , crb )
2021-10-07 19:47:00 +00:00
}
2021-11-10 12:33:42 +00:00
var dbRawData [ ] byte
if c . managedDB != nil && ! isHTTP {
token := c . config . Token
if token == "" {
2022-02-24 19:01:14 +00:00
tokenFromFile , err := readTokenFromFile ( c . config . Runtime . ServerToken , c . config . Runtime . ServerCA , c . config . DataDir )
2021-11-10 12:33:42 +00:00
if err != nil {
return err
}
if tokenFromFile == "" {
// at this point this is a fresh start in a non-managed environment
c . saveBootstrap = true
return nil
}
token = tokenFromFile
}
normalizedToken , err := normalizeToken ( token )
2021-10-07 19:47:00 +00:00
if err != nil {
return err
}
2021-11-10 12:33:42 +00:00
var value * client . Value
2022-02-24 22:35:08 +00:00
storageClient , err := client . New ( c . config . Runtime . EtcdConfig )
2021-11-10 12:33:42 +00:00
if err != nil {
return err
}
2021-12-14 21:58:45 +00:00
defer storageClient . Close ( )
2021-10-07 19:47:00 +00:00
2021-11-10 12:33:42 +00:00
ticker := time . NewTicker ( 5 * time . Second )
defer ticker . Stop ( )
2021-10-07 19:47:00 +00:00
2021-11-10 12:33:42 +00:00
RETRY :
for {
2021-12-07 22:31:32 +00:00
value , c . saveBootstrap , err = getBootstrapKeyFromStorage ( ctx , storageClient , normalizedToken , token )
2021-11-10 12:33:42 +00:00
if err != nil {
if strings . Contains ( err . Error ( ) , "not supported for learner" ) {
for range ticker . C {
continue RETRY
}
2021-10-07 19:47:00 +00:00
}
2021-11-10 12:33:42 +00:00
return err
}
if value == nil {
return nil
}
2021-10-07 19:47:00 +00:00
2021-11-10 12:33:42 +00:00
dbRawData , err = decrypt ( normalizedToken , value . Data )
if err != nil {
return err
2021-10-07 19:47:00 +00:00
}
2021-11-10 12:33:42 +00:00
break
2021-10-07 19:47:00 +00:00
}
2021-11-10 12:33:42 +00:00
buf = bytes . NewReader ( dbRawData )
2021-10-07 19:47:00 +00:00
}
paths , err := bootstrap . ObjToMap ( crb )
if err != nil {
return err
}
files := make ( bootstrap . PathsDataformat )
2021-12-14 00:04:39 +00:00
if ! isMigrated ( buf , & files ) {
2021-10-07 19:47:00 +00:00
if err := migrateBootstrapData ( ctx , buf , files ) ; err != nil {
return err
}
2021-12-13 20:02:30 +00:00
buf . Seek ( 0 , 0 )
2021-10-07 19:47:00 +00:00
}
type update struct {
db , disk , conflict bool
}
2021-11-10 12:33:42 +00:00
var updateDisk bool
2021-10-07 19:47:00 +00:00
results := make ( map [ string ] update )
for pathKey , fileData := range files {
path , ok := paths [ pathKey ]
if ! ok {
continue
}
f , err := os . Open ( path )
if err != nil {
if os . IsNotExist ( err ) {
logrus . Warn ( path + " doesn't exist. continuing..." )
updateDisk = true
continue
}
return err
}
defer f . Close ( )
fData , err := ioutil . ReadAll ( f )
if err != nil {
return err
}
if ! bytes . Equal ( fileData . Content , fData ) {
info , err := os . Stat ( path )
if err != nil {
return err
}
switch {
case info . ModTime ( ) . Unix ( ) - files [ pathKey ] . Timestamp . Unix ( ) >= systemTimeSkew :
if _ , ok := results [ path ] ; ! ok {
results [ path ] = update {
db : true ,
}
}
for pk := range files {
p , ok := paths [ pk ]
if ! ok {
continue
}
if filepath . Base ( p ) == info . Name ( ) {
continue
}
i , err := os . Stat ( p )
if err != nil {
return err
}
if i . ModTime ( ) . Unix ( ) - files [ pk ] . Timestamp . Unix ( ) >= systemTimeSkew {
if _ , ok := results [ path ] ; ! ok {
results [ path ] = update {
conflict : true ,
}
}
}
}
case info . ModTime ( ) . Unix ( ) - files [ pathKey ] . Timestamp . Unix ( ) <= systemTimeSkew :
if _ , ok := results [ info . Name ( ) ] ; ! ok {
results [ path ] = update {
disk : true ,
}
}
for pk := range files {
p , ok := paths [ pk ]
if ! ok {
continue
}
if filepath . Base ( p ) == info . Name ( ) {
continue
}
i , err := os . Stat ( p )
if err != nil {
return err
}
if i . ModTime ( ) . Unix ( ) - files [ pk ] . Timestamp . Unix ( ) <= systemTimeSkew {
if _ , ok := results [ path ] ; ! ok {
results [ path ] = update {
conflict : true ,
}
}
}
}
default :
if _ , ok := results [ path ] ; ok {
results [ path ] = update { }
}
}
}
}
2021-12-09 21:54:27 +00:00
if c . config . ClusterReset {
serverTLSDir := filepath . Join ( c . config . DataDir , "tls" )
tlsBackupDir := filepath . Join ( c . config . DataDir , "tls-" + strconv . Itoa ( int ( time . Now ( ) . Unix ( ) ) ) )
logrus . Infof ( "Cluster reset: backing up certificates directory to " + tlsBackupDir )
if _ , err := os . Stat ( serverTLSDir ) ; err != nil {
return err
}
if err := copy . Copy ( serverTLSDir , tlsBackupDir ) ; err != nil {
return err
}
}
2021-12-21 21:28:32 +00:00
var newerOnDisk [ ] string
2021-10-07 19:47:00 +00:00
for path , res := range results {
2021-11-10 12:33:42 +00:00
switch {
case res . disk :
2021-10-07 19:47:00 +00:00
updateDisk = true
logrus . Warn ( "datastore newer than " + path )
2021-11-10 12:33:42 +00:00
case res . db :
2021-12-09 21:54:27 +00:00
if c . config . ClusterReset {
logrus . Infof ( "Cluster reset: replacing file on disk: " + path )
updateDisk = true
continue
}
2021-12-21 21:28:32 +00:00
newerOnDisk = append ( newerOnDisk , path )
2021-11-10 12:33:42 +00:00
case res . conflict :
2021-10-07 19:47:00 +00:00
logrus . Warnf ( "datastore / disk conflict: %s newer than in the datastore" , path )
}
}
2021-12-21 21:28:32 +00:00
if len ( newerOnDisk ) > 0 {
logrus . Fatal ( strings . Join ( newerOnDisk , ", " ) + " newer than datastore and could cause a cluster outage. Remove the file(s) from disk and restart to be recreated from datastore." )
}
2021-11-10 12:33:42 +00:00
if updateDisk {
2021-10-07 19:47:00 +00:00
logrus . Warn ( "updating bootstrap data on disk from datastore" )
2021-12-13 20:02:30 +00:00
return bootstrap . WriteToDiskFromStorage ( files , crb )
2021-10-07 19:47:00 +00:00
}
return nil
2020-05-05 21:59:15 +00:00
}
2020-09-24 06:29:25 +00:00
// httpBootstrap retrieves bootstrap data (certs and keys, etc) from the remote server via HTTP
// and loads it into the ControlRuntimeBootstrap struct. Unlike the storage bootstrap path,
// this data does not need to be decrypted since it is generated on-demand by an existing server.
2021-10-07 19:47:00 +00:00
func ( c * Cluster ) httpBootstrap ( ctx context . Context ) error {
2021-03-06 10:29:57 +00:00
content , err := c . clientAccessInfo . Get ( "/v1-" + version . Program + "/server-bootstrap" )
2020-05-05 21:59:15 +00:00
if err != nil {
return err
}
2022-02-24 22:35:08 +00:00
return c . ReconcileBootstrapData ( ctx , bytes . NewReader ( content ) , & c . config . Runtime . ControlRuntimeBootstrap , true )
2021-11-10 12:33:42 +00:00
}
func ( c * Cluster ) retrieveInitializedDBdata ( ctx context . Context ) ( * bytes . Buffer , error ) {
var buf bytes . Buffer
2022-02-24 19:01:14 +00:00
if err := bootstrap . ReadFromDisk ( & buf , & c . config . Runtime . ControlRuntimeBootstrap ) ; err != nil {
2021-11-10 12:33:42 +00:00
return nil , err
}
return & buf , nil
2020-05-05 21:59:15 +00:00
}
2020-09-24 06:29:25 +00:00
// bootstrap performs cluster bootstrapping, either via HTTP (for managed databases) or direct load from datastore.
2020-05-05 21:59:15 +00:00
func ( c * Cluster ) bootstrap ( ctx context . Context ) error {
c . joining = true
2021-06-22 20:42:34 +00:00
// bootstrap managed database via HTTPS
2022-02-24 19:01:14 +00:00
if c . config . Runtime . HTTPBootstrap {
2021-11-23 14:46:06 +00:00
// Assuming we should just compare on managed databases
if err := c . compareConfig ( ) ; err != nil {
2022-01-06 23:32:29 +00:00
return errors . Wrap ( err , "failed to validate server configuration" )
2021-11-23 14:46:06 +00:00
}
2021-10-07 19:47:00 +00:00
return c . httpBootstrap ( ctx )
2020-05-05 21:59:15 +00:00
}
2020-09-24 05:22:15 +00:00
// Bootstrap directly from datastore
return c . storageBootstrap ( ctx )
2020-05-05 21:59:15 +00:00
}
2021-01-21 21:09:15 +00:00
// Snapshot is a proxy method to call the snapshot method on the managedb
// interface for etcd clusters.
func ( c * Cluster ) Snapshot ( ctx context . Context , config * config . Control ) error {
if c . managedDB == nil {
return errors . New ( "unable to perform etcd snapshot on non-etcd system" )
}
return c . managedDB . Snapshot ( ctx , config )
}
2021-11-23 14:46:06 +00:00
// compareConfig verifies that the config of the joining control plane node coincides with the cluster's config
func ( c * Cluster ) compareConfig ( ) error {
2022-01-06 23:32:29 +00:00
token := c . config . AgentToken
if token == "" {
token = c . config . Token
}
agentClientAccessInfo , err := clientaccess . ParseAndValidateTokenForUser ( c . config . JoinURL , token , "node" )
2021-11-23 14:46:06 +00:00
if err != nil {
return err
}
serverConfig , err := agentClientAccessInfo . Get ( "/v1-" + version . Program + "/config" )
if err != nil {
return err
}
clusterControl := & config . Control { }
if err := json . Unmarshal ( serverConfig , clusterControl ) ; err != nil {
return err
}
// We are saving IPs of ClusterIPRanges and ServiceIPRanges in 4-bytes representation but json decodes in 16-byte
ipsTo16Bytes ( c . config . CriticalControlArgs . ClusterIPRanges )
ipsTo16Bytes ( c . config . CriticalControlArgs . ServiceIPRanges )
if ! reflect . DeepEqual ( clusterControl . CriticalControlArgs , c . config . CriticalControlArgs ) {
logrus . Debugf ( "This is the server CriticalControlArgs: %#v" , clusterControl . CriticalControlArgs )
logrus . Debugf ( "This is the local CriticalControlArgs: %#v" , c . config . CriticalControlArgs )
2022-01-06 23:32:29 +00:00
return errors . New ( "critical configuration value mismatch" )
2021-11-23 14:46:06 +00:00
}
return nil
}
// ipsTo16Bytes makes sure the IPs in the []*net.IPNet slice are represented in 16-byte format
func ipsTo16Bytes ( mySlice [ ] * net . IPNet ) {
for _ , ipNet := range mySlice {
ipNet . IP = ipNet . IP . To16 ( )
}
}
2022-02-24 22:35:08 +00:00
// reconcileEtcd starts a temporary single-member etcd cluster using a copy of the
// etcd database, and uses it to reconcile bootstrap data. This is necessary
// because the full etcd cluster may not have quorum during startup, but we still
// need to extract data from the datastore.
func ( c * Cluster ) reconcileEtcd ( ctx context . Context ) error {
logrus . Info ( "Starting temporary etcd to reconcile with datastore" )
tempConfig := endpoint . ETCDConfig { Endpoints : [ ] string { "http://127.0.0.1:2399" } }
originalConfig := c . config . Runtime . EtcdConfig
c . config . Runtime . EtcdConfig = tempConfig
reconcileCtx , cancel := context . WithCancel ( ctx )
defer func ( ) {
cancel ( )
c . config . Runtime . EtcdConfig = originalConfig
} ( )
e := etcd . NewETCD ( )
if err := e . SetControlConfig ( reconcileCtx , c . config ) ; err != nil {
return err
}
e . StartEmbeddedTemporary ( reconcileCtx )
for {
2022-03-01 23:55:35 +00:00
if err := e . Test ( reconcileCtx ) ; err != nil && ! errors . Is ( err , etcd . ErrNotMember ) {
logrus . Infof ( "Failed to test temporary data store connection: %v" , err )
2022-02-24 22:35:08 +00:00
} else {
2022-03-01 23:55:35 +00:00
logrus . Info ( e . EndpointName ( ) + " temporary data store connection OK" )
2022-02-24 22:35:08 +00:00
break
}
select {
case <- time . After ( 5 * time . Second ) :
case <- reconcileCtx . Done ( ) :
break
}
}
data , err := c . retrieveInitializedDBdata ( reconcileCtx )
if err != nil {
return err
}
return c . ReconcileBootstrapData ( reconcileCtx , bytes . NewReader ( data . Bytes ( ) ) , & c . config . Runtime . ControlRuntimeBootstrap , false )
}