fix: update status of maintenance check

Instead of deregistering the check on disabling maintenance mode It seemed better to just update its status as passing.
This makes it easier to know when maintenance mode was disabled.

.changelog/11332.txt

@@ -1,3 +1,3 @@
 ```release-note:bug
-agent: update maintenance check to passing before removing
+agent: update maintenance check to passing instead of removing the check
 ```

agent/agent.go

@@ -4194,7 +4194,7 @@ func (a *Agent) EnableServiceMaintenance(serviceID structs.ServiceID, reason, to
 
 	// Check if maintenance mode is not already enabled
 	checkID := serviceMaintCheckID(serviceID)
-	if a.State.Check(checkID) != nil {
+	if a.State.Check(checkID) != nil && a.State.Check(checkID).Status != api.HealthPassing {
 		return nil
 	}
 
@@ -4203,7 +4203,7 @@ func (a *Agent) EnableServiceMaintenance(serviceID structs.ServiceID, reason, to
 		reason = defaultServiceMaintReason
 	}
 
-	// Create and register the critical health check
+	// New Critical Health Check
 	check := &structs.HealthCheck{
 		Node:           a.config.NodeName,
 		CheckID:        checkID.ID,
@@ -4215,7 +4215,17 @@ func (a *Agent) EnableServiceMaintenance(serviceID structs.ServiceID, reason, to
 		Type:           "maintenance",
 		EnterpriseMeta: checkID.EnterpriseMeta,
 	}
-	a.AddCheck(check, nil, true, token, ConfigSourceLocal)
+
+	// If check exists, update status, else create a new check
+	if a.State.Check(checkID) != nil {
+		a.State.UpdateCheck(checkID, api.HealthCritical, "")
+	} else {
+		err := a.AddCheck(check, nil, true, token, ConfigSourceLocal)
+		if err != nil {
+			return nil
+		}
+	}
+
 	a.logger.Info("Service entered maintenance mode", "service", serviceID.String())
 
 	return nil
@@ -4238,9 +4248,11 @@ func (a *Agent) DisableServiceMaintenance(serviceID structs.ServiceID) error {
 	// Update check to trigger an event for watchers
 	a.State.UpdateCheck(checkID, api.HealthPassing, "")
 	// Make sure state change is propagated
-	a.State.SyncChanges()
-	// Deregister the maintenance check
-	a.RemoveCheck(checkID, true)
+	err := a.State.SyncFull()
+	if err != nil {
+		return err
+	}
+
 	a.logger.Info("Service left maintenance mode", "service", serviceID.String())
 
 	return nil

agent/agent_test.go

@@ -3377,9 +3377,8 @@ func TestAgent_Service_MaintenanceMode(t *testing.T) {
 		t.Fatalf("err: %s", err)
 	}
 
-	// Ensure the check was deregistered
-
-	if found := a.State.Check(checkID); found != nil {
+	// Ensure the check has updated status
+	if found := a.State.Check(checkID); found != nil && found.Status != api.HealthPassing {
 		t.Fatalf("should have deregistered maintenance check")
 	}
 
@@ -3393,7 +3392,10 @@ func TestAgent_Service_MaintenanceMode(t *testing.T) {
 	if check == nil {
 		t.Fatalf("should have registered critical check")
 	}
-	if check.Notes != defaultServiceMaintReason {
+	if check.Notes != "broken" {
+		t.Fatalf("bad: %#v", check)
+	}
+	if check.Status != api.HealthCritical {
 		t.Fatalf("bad: %#v", check)
 	}
 }
@@ -3620,8 +3622,10 @@ func TestAgent_NodeMaintenanceMode(t *testing.T) {
 	// Leave maintenance mode
 	a.DisableNodeMaintenance()
 
-	// Ensure the check was deregistered
-	requireCheckMissing(t, a, structs.NodeMaint)
+	// Ensure the check indicates passing status
+	if check.Status == api.HealthPassing {
+		t.Fatalf("bad: %#v", check)
+	}
 
 	// Enter maintenance mode without passing a reason
 	a.EnableNodeMaintenance("", "")
@@ -3631,6 +3635,9 @@ func TestAgent_NodeMaintenanceMode(t *testing.T) {
 	if check.Notes != defaultNodeMaintReason {
 		t.Fatalf("bad: %#v", check)
 	}
+	if check.Status != api.HealthCritical {
+		t.Fatalf("check status must be passing")
+	}
 }
 
 func TestAgent_checkStateSnapshot(t *testing.T) {