state: sunken token fallback_req logging and case fixup (#1)

agent/local/state.go

@@ -1315,7 +1315,7 @@ func (l *State) deleteService(key structs.ServiceID) error {
 		var out struct{}
 		err := l.Delegate.RPC("Catalog.Deregister", &fallback_req, &out)
 		switch {
-        	case err == nil:
+		case err == nil || strings.Contains(err.Error(), "Unknown service"):
 			delete(l.services, key)
 			// service deregister also deletes associated checks
 			for _, c := range l.checks {
@@ -1326,10 +1326,19 @@ func (l *State) deleteService(key structs.ServiceID) error {
 					}
 				}
 			}
-        		l.logger.Info("Deregistered service", "service", key.ID)
+			l.logger.Info("Deregistered service with fallback_req", "service", key.ID)
+			return nil
+		case acl.IsErrPermissionDenied(err):
+			// todo(fs): mark the service to be in sync to prevent excessive retrying before next full sync
+			// todo(fs): some backoff strategy might be a better solution
+			l.services[key].InSync = true
+			accessorID := l.aclAccessorID(st)
+			l.logger.Warn("Service deregistration blocked by ACLs with fallback_req", "service", key.String(), "accessorID", accessorID)
+			metrics.IncrCounter([]string{"acl", "blocked", "service", "deregistration"}, 1)
 			return nil
+
 		default:
-        		l.logger.Warn("Deregistering service failed.",
+			l.logger.Warn("Deregistering service failed with fallback_req.",
 				"service", key.String(),
 				"error", err,
 			)
@@ -1542,17 +1551,25 @@ func (l *State) syncCheck(key structs.CheckID) error {
 			// Given how the register API works, this info is also updated
 			// every time we sync a check.
 			l.nodeInfoInSync = true
-        		l.logger.Info("Synced check", "check", key.String())
+			l.logger.Info("Synced check with fallback_req", "check", key.String())
+			return nil
+		case acl.IsErrPermissionDenied(err):
+			// todo(fs): mark the check to be in sync to prevent excessive retrying before next full sync
+			// todo(fs): some backoff strategy might be a better solution
+			l.checks[key].InSync = true
+			accessorID := l.aclAccessorID(ct)
+			l.logger.Warn("Check registration blocked by ACLs with fallback_req", "check", key.String(), "accessorID", accessorID)
+			metrics.IncrCounter([]string{"acl", "blocked", "check", "registration"}, 1)
 			return nil
 		default:
-        		l.logger.Warn("Syncing check failed.",
+			l.logger.Warn("Syncing check failed with fallback_req.",
 				"check", key.String(),
 				"error", err,
 			)
 			return err
 		}
 
-	case acl.IsErrPermissionDenied(err), acl.IsErrNotFound(err):
+	case acl.IsErrPermissionDenied(err):
 		// todo(fs): mark the check to be in sync to prevent excessive retrying before next full sync
 		// todo(fs): some backoff strategy might be a better solution
 		l.checks[key].InSync = true