2021-02-01 19:20:24 +00:00
// Apache License v2.0 (copyright Cloud Native Labs & Rancher Labs)
2021-07-07 15:46:10 +00:00
// - modified from https://github.com/cloudnativelabs/kube-router/blob/73b1b03b32c5755b240f6c077bb097abe3888314/pkg/controllers/netpol/network_policy_controller.go
2021-02-01 19:20:24 +00:00
// +build !windows
2019-10-17 21:46:15 +00:00
package netpol
import (
2021-07-07 15:46:10 +00:00
"bytes"
2019-10-17 21:46:15 +00:00
"crypto/sha256"
"encoding/base32"
"fmt"
"net"
"strconv"
"strings"
"sync"
"time"
"github.com/coreos/go-iptables/iptables"
2021-02-01 19:20:24 +00:00
"github.com/rancher/k3s/pkg/agent/netpol/utils"
"github.com/rancher/k3s/pkg/daemons/config"
2021-02-01 19:03:43 +00:00
2019-10-17 21:46:15 +00:00
"k8s.io/apimachinery/pkg/labels"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache"
2021-07-07 15:46:10 +00:00
"k8s.io/klog/v2"
2019-10-17 21:46:15 +00:00
)
const (
kubePodFirewallChainPrefix = "KUBE-POD-FW-"
kubeNetworkPolicyChainPrefix = "KUBE-NWPLCY-"
kubeSourceIPSetPrefix = "KUBE-SRC-"
kubeDestinationIPSetPrefix = "KUBE-DST-"
2021-02-01 19:03:43 +00:00
kubeInputChainName = "KUBE-ROUTER-INPUT"
kubeForwardChainName = "KUBE-ROUTER-FORWARD"
kubeOutputChainName = "KUBE-ROUTER-OUTPUT"
2021-07-07 15:46:10 +00:00
kubeDefaultNetpolChain = "KUBE-NWPLCY-DEFAULT"
2021-02-01 19:20:24 +00:00
defaultSyncPeriod = 5 * time . Minute
2019-10-17 21:46:15 +00:00
)
// Network policy controller provides both ingress and egress filtering for the pods as per the defined network
// policies. Two different types of iptables chains are used. Each pod running on the node which either
// requires ingress or egress filtering gets a pod specific chains. Each network policy has a iptables chain, which
// has rules expressed through ipsets matching source and destination pod ip's. In the FORWARD chain of the
// filter table a rule is added to jump the traffic originating (in case of egress network policy) from the pod
// or destined (in case of ingress network policy) to the pod specific iptables chain. Each
// pod specific iptables chain has rules to jump to the network polices chains, that pod matches. So packet
2021-01-28 01:26:48 +00:00
// originating/destined from/to pod goes through filter table's, FORWARD chain, followed by pod specific chain,
2019-10-17 21:46:15 +00:00
// followed by one or more network policy chains, till there is a match which will accept the packet, or gets
// dropped by the rule in the pod chain, if there is no match.
2021-07-07 15:46:10 +00:00
// NetworkPolicyController struct to hold information required by NetworkPolicyController
2019-10-17 21:46:15 +00:00
type NetworkPolicyController struct {
2021-02-01 19:03:43 +00:00
nodeIP net . IP
nodeHostName string
serviceClusterIPRange net . IPNet
serviceExternalIPRanges [ ] net . IPNet
serviceNodePortRange string
mu sync . Mutex
syncPeriod time . Duration
fullSyncRequestChan chan struct { }
2021-07-07 15:46:10 +00:00
ipsetMutex * sync . Mutex
2021-02-01 19:03:43 +00:00
ipSetHandler * utils . IPSet
2019-10-17 21:46:15 +00:00
podLister cache . Indexer
npLister cache . Indexer
nsLister cache . Indexer
PodEventHandler cache . ResourceEventHandler
NamespaceEventHandler cache . ResourceEventHandler
NetworkPolicyEventHandler cache . ResourceEventHandler
2021-07-07 15:46:10 +00:00
filterTableRules bytes . Buffer
2019-10-17 21:46:15 +00:00
}
// internal structure to represent a network policy
type networkPolicyInfo struct {
2021-02-01 19:03:43 +00:00
name string
namespace string
podSelector labels . Selector
2019-10-17 21:46:15 +00:00
// set of pods matching network policy spec podselector label selector
targetPods map [ string ] podInfo
// whitelist ingress rules from the network policy spec
ingressRules [ ] ingressRule
// whitelist egress rules from the network policy spec
egressRules [ ] egressRule
// policy type "ingress" or "egress" or "both" as defined by PolicyType in the spec
policyType string
}
// internal structure to represent Pod
type podInfo struct {
ip string
name string
namespace string
labels map [ string ] string
}
// internal structure to represent NetworkPolicyIngressRule in the spec
type ingressRule struct {
matchAllPorts bool
ports [ ] protocolAndPort
namedPorts [ ] endPoints
matchAllSource bool
srcPods [ ] podInfo
srcIPBlocks [ ] [ ] string
}
// internal structure to represent NetworkPolicyEgressRule in the spec
type egressRule struct {
matchAllPorts bool
ports [ ] protocolAndPort
namedPorts [ ] endPoints
matchAllDestinations bool
dstPods [ ] podInfo
dstIPBlocks [ ] [ ] string
}
type protocolAndPort struct {
protocol string
port string
2021-07-07 15:46:10 +00:00
endport string
2019-10-17 21:46:15 +00:00
}
type endPoints struct {
ips [ ] string
protocolAndPort
}
type numericPort2eps map [ string ] * endPoints
type protocol2eps map [ string ] numericPort2eps
type namedPort2eps map [ string ] protocol2eps
2021-01-28 01:26:48 +00:00
// Run runs forever till we receive notification on stopCh
2021-02-01 19:20:24 +00:00
func ( npc * NetworkPolicyController ) Run ( stopCh <- chan struct { } ) {
2019-10-17 21:46:15 +00:00
t := time . NewTicker ( npc . syncPeriod )
defer t . Stop ( )
2021-02-01 19:03:43 +00:00
2021-07-07 15:46:10 +00:00
klog . Info ( "Starting network policy controller" )
2021-02-01 19:03:43 +00:00
2021-07-07 15:46:10 +00:00
// setup kube-router specific top level custom chains (KUBE-ROUTER-INPUT, KUBE-ROUTER-FORWARD, KUBE-ROUTER-OUTPUT)
2021-02-01 19:03:43 +00:00
npc . ensureTopLevelChains ( )
2021-07-07 15:46:10 +00:00
// setup default network policy chain that is applied to traffic from/to the pods that does not match any network policy
npc . ensureDefaultNetworkPolicyChain ( )
2021-02-01 19:03:43 +00:00
// Full syncs of the network policy controller take a lot of time and can only be processed one at a time,
// therefore, we start it in it's own goroutine and request a sync through a single item channel
2021-07-07 15:46:10 +00:00
klog . Info ( "Starting network policy controller full sync goroutine" )
2021-02-01 19:20:24 +00:00
go func ( fullSyncRequest <- chan struct { } , stopCh <- chan struct { } ) {
2021-02-01 19:03:43 +00:00
for {
// Add an additional non-blocking select to ensure that if the stopCh channel is closed it is handled first
select {
case <- stopCh :
2021-07-07 15:46:10 +00:00
klog . Info ( "Shutting down network policies full sync goroutine" )
2021-02-01 19:03:43 +00:00
return
default :
}
select {
case <- stopCh :
2021-07-07 15:46:10 +00:00
klog . Info ( "Shutting down network policies full sync goroutine" )
2021-02-01 19:03:43 +00:00
return
case <- fullSyncRequest :
2021-07-07 15:46:10 +00:00
klog . V ( 3 ) . Info ( "Received request for a full sync, processing" )
2021-02-01 19:03:43 +00:00
npc . fullPolicySync ( ) // fullPolicySync() is a blocking request here
}
}
2021-02-01 19:20:24 +00:00
} ( npc . fullSyncRequestChan , stopCh )
2019-10-17 21:46:15 +00:00
// loop forever till notified to stop on stopCh
for {
2021-07-07 15:46:10 +00:00
klog . V ( 1 ) . Info ( "Requesting periodic sync of iptables to reflect network policies" )
2021-02-01 19:03:43 +00:00
npc . RequestFullSync ( )
2019-10-17 21:46:15 +00:00
select {
case <- stopCh :
2021-07-07 15:46:10 +00:00
klog . Infof ( "Shutting down network policies controller" )
2019-10-17 21:46:15 +00:00
return
case <- t . C :
}
}
}
2021-02-01 19:03:43 +00:00
// RequestFullSync allows the request of a full network policy sync without blocking the callee
func ( npc * NetworkPolicyController ) RequestFullSync ( ) {
select {
case npc . fullSyncRequestChan <- struct { } { } :
2021-07-07 15:46:10 +00:00
klog . V ( 3 ) . Info ( "Full sync request queue was empty so a full sync request was successfully sent" )
2021-02-01 19:03:43 +00:00
default : // Don't block if the buffered channel is full, return quickly so that we don't block callee execution
2021-07-07 15:46:10 +00:00
klog . V ( 1 ) . Info ( "Full sync request queue was full, skipping..." )
2019-10-17 21:46:15 +00:00
}
}
// Sync synchronizes iptables to desired state of network policies
2021-02-01 19:03:43 +00:00
func ( npc * NetworkPolicyController ) fullPolicySync ( ) {
2019-10-17 21:46:15 +00:00
var err error
2021-02-01 19:03:43 +00:00
var networkPoliciesInfo [ ] networkPolicyInfo
2019-10-17 21:46:15 +00:00
npc . mu . Lock ( )
defer npc . mu . Unlock ( )
start := time . Now ( )
syncVersion := strconv . FormatInt ( start . UnixNano ( ) , 10 )
defer func ( ) {
endTime := time . Since ( start )
2021-07-07 15:46:10 +00:00
klog . V ( 1 ) . Infof ( "sync iptables took %v" , endTime )
2019-10-17 21:46:15 +00:00
} ( )
2021-07-07 15:46:10 +00:00
klog . V ( 1 ) . Infof ( "Starting sync of iptables with version: %s" , syncVersion )
2019-10-17 21:46:15 +00:00
2021-02-01 19:03:43 +00:00
// ensure kube-router specific top level chains and corresponding rules exist
npc . ensureTopLevelChains ( )
2019-10-17 21:46:15 +00:00
2021-07-07 15:46:10 +00:00
// ensure default network policy chain that is applied to traffic from/to the pods that does not match any network policy
npc . ensureDefaultNetworkPolicyChain ( )
2021-02-01 19:03:43 +00:00
networkPoliciesInfo , err = npc . buildNetworkPoliciesInfo ( )
2019-10-17 21:46:15 +00:00
if err != nil {
2021-07-07 15:46:10 +00:00
klog . Errorf ( "Aborting sync. Failed to build network policies: %v" , err . Error ( ) )
return
}
npc . filterTableRules . Reset ( )
if err := utils . SaveInto ( "filter" , & npc . filterTableRules ) ; err != nil {
klog . Errorf ( "Aborting sync. Failed to run iptables-save: %v" + err . Error ( ) )
2021-02-01 19:03:43 +00:00
return
2019-10-17 21:46:15 +00:00
}
2021-02-01 19:03:43 +00:00
activePolicyChains , activePolicyIPSets , err := npc . syncNetworkPolicyChains ( networkPoliciesInfo , syncVersion )
2019-10-17 21:46:15 +00:00
if err != nil {
2021-07-07 15:46:10 +00:00
klog . Errorf ( "Aborting sync. Failed to sync network policy chains: %v" + err . Error ( ) )
2021-02-01 19:03:43 +00:00
return
2019-10-17 21:46:15 +00:00
}
2021-02-01 19:03:43 +00:00
activePodFwChains , err := npc . syncPodFirewallChains ( networkPoliciesInfo , syncVersion )
2019-10-17 21:46:15 +00:00
if err != nil {
2021-07-07 15:46:10 +00:00
klog . Errorf ( "Aborting sync. Failed to sync pod firewalls: %v" , err . Error ( ) )
return
}
err = npc . cleanupStaleRules ( activePolicyChains , activePodFwChains )
if err != nil {
klog . Errorf ( "Aborting sync. Failed to cleanup stale iptables rules: %v" , err . Error ( ) )
return
}
if err := utils . Restore ( "filter" , npc . filterTableRules . Bytes ( ) ) ; err != nil {
klog . Errorf ( "Aborting sync. Failed to run iptables-restore: %v\n%s" , err . Error ( ) , npc . filterTableRules . String ( ) )
2021-02-01 19:03:43 +00:00
return
2019-10-17 21:46:15 +00:00
}
2021-07-07 15:46:10 +00:00
err = npc . cleanupStaleIPSets ( activePolicyIPSets )
2019-10-17 21:46:15 +00:00
if err != nil {
2021-07-07 15:46:10 +00:00
klog . Errorf ( "Failed to cleanup stale ipsets: %v" , err . Error ( ) )
2021-02-01 19:03:43 +00:00
return
2019-10-17 21:46:15 +00:00
}
}
2021-02-01 19:03:43 +00:00
// Creates custom chains KUBE-ROUTER-INPUT, KUBE-ROUTER-FORWARD, KUBE-ROUTER-OUTPUT
// and following rules in the filter table to jump from builtin chain to custom chain
// -A INPUT -m comment --comment "kube-router netpol" -j KUBE-ROUTER-INPUT
// -A FORWARD -m comment --comment "kube-router netpol" -j KUBE-ROUTER-FORWARD
// -A OUTPUT -m comment --comment "kube-router netpol" -j KUBE-ROUTER-OUTPUT
func ( npc * NetworkPolicyController ) ensureTopLevelChains ( ) {
2019-10-17 21:46:15 +00:00
iptablesCmdHandler , err := iptables . New ( )
if err != nil {
2021-07-07 15:46:10 +00:00
klog . Fatalf ( "Failed to initialize iptables executor due to %s" , err . Error ( ) )
2019-10-17 21:46:15 +00:00
}
2021-02-01 19:03:43 +00:00
addUUIDForRuleSpec := func ( chain string , ruleSpec * [ ] string ) ( string , error ) {
hash := sha256 . Sum256 ( [ ] byte ( chain + strings . Join ( * ruleSpec , "" ) ) )
encoded := base32 . StdEncoding . EncodeToString ( hash [ : ] ) [ : 16 ]
for idx , part := range * ruleSpec {
if "--comment" == part {
( * ruleSpec ) [ idx + 1 ] = ( * ruleSpec ) [ idx + 1 ] + " - " + encoded
return encoded , nil
2019-10-17 21:46:15 +00:00
}
}
2021-02-01 19:03:43 +00:00
return "" , fmt . Errorf ( "could not find a comment in the ruleSpec string given: %s" , strings . Join ( * ruleSpec , " " ) )
2019-10-17 21:46:15 +00:00
}
2021-02-01 19:03:43 +00:00
ensureRuleAtPosition := func ( chain string , ruleSpec [ ] string , uuid string , position int ) {
exists , err := iptablesCmdHandler . Exists ( "filter" , chain , ruleSpec ... )
2019-10-17 21:46:15 +00:00
if err != nil {
2021-07-07 15:46:10 +00:00
klog . Fatalf ( "Failed to verify rule exists in %s chain due to %s" , chain , err . Error ( ) )
2019-10-17 21:46:15 +00:00
}
if ! exists {
2021-02-01 19:03:43 +00:00
err := iptablesCmdHandler . Insert ( "filter" , chain , position , ruleSpec ... )
2019-10-17 21:46:15 +00:00
if err != nil {
2021-07-07 15:46:10 +00:00
klog . Fatalf ( "Failed to run iptables command to insert in %s chain %s" , chain , err . Error ( ) )
2019-10-17 21:46:15 +00:00
}
2021-02-01 19:03:43 +00:00
return
2019-10-17 21:46:15 +00:00
}
2021-02-01 19:03:43 +00:00
rules , err := iptablesCmdHandler . List ( "filter" , chain )
2019-10-17 21:46:15 +00:00
if err != nil {
2021-07-07 15:46:10 +00:00
klog . Fatalf ( "failed to list rules in filter table %s chain due to %s" , chain , err . Error ( ) )
2019-10-17 21:46:15 +00:00
}
2021-02-01 19:03:43 +00:00
var ruleNo , ruleIndexOffset int
for i , rule := range rules {
rule = strings . Replace ( rule , "\"" , "" , 2 ) //removes quote from comment string
if strings . HasPrefix ( rule , "-P" ) || strings . HasPrefix ( rule , "-N" ) {
// if this chain has a default policy, then it will show as rule #1 from iptablesCmdHandler.List so we
// need to account for this offset
ruleIndexOffset ++
continue
}
if strings . Contains ( rule , uuid ) {
// range uses a 0 index, but iptables uses a 1 index so we need to increase ruleNo by 1
ruleNo = i + 1 - ruleIndexOffset
break
2019-10-17 21:46:15 +00:00
}
}
2021-02-01 19:03:43 +00:00
if ruleNo != position {
err = iptablesCmdHandler . Insert ( "filter" , chain , position , ruleSpec ... )
2019-10-17 21:46:15 +00:00
if err != nil {
2021-07-07 15:46:10 +00:00
klog . Fatalf ( "Failed to run iptables command to insert in %s chain %s" , chain , err . Error ( ) )
2019-10-17 21:46:15 +00:00
}
2021-02-01 19:03:43 +00:00
err = iptablesCmdHandler . Delete ( "filter" , chain , strconv . Itoa ( ruleNo + 1 ) )
2019-10-17 21:46:15 +00:00
if err != nil {
2021-07-07 15:46:10 +00:00
klog . Fatalf ( "Failed to delete incorrect rule in %s chain due to %s" , chain , err . Error ( ) )
2019-10-17 21:46:15 +00:00
}
}
}
2021-02-01 19:03:43 +00:00
chains := map [ string ] string { "INPUT" : kubeInputChainName , "FORWARD" : kubeForwardChainName , "OUTPUT" : kubeOutputChainName }
2019-10-17 21:46:15 +00:00
2021-02-01 19:03:43 +00:00
for builtinChain , customChain := range chains {
err = iptablesCmdHandler . NewChain ( "filter" , customChain )
2019-10-17 21:46:15 +00:00
if err != nil && err . ( * iptables . Error ) . ExitStatus ( ) != 1 {
2021-07-07 15:46:10 +00:00
klog . Fatalf ( "Failed to run iptables command to create %s chain due to %s" , customChain , err . Error ( ) )
2019-10-17 21:46:15 +00:00
}
2021-02-01 19:03:43 +00:00
args := [ ] string { "-m" , "comment" , "--comment" , "kube-router netpol" , "-j" , customChain }
uuid , err := addUUIDForRuleSpec ( builtinChain , & args )
2019-10-17 21:46:15 +00:00
if err != nil {
2021-07-07 15:46:10 +00:00
klog . Fatalf ( "Failed to get uuid for rule: %s" , err . Error ( ) )
2019-10-17 21:46:15 +00:00
}
2021-02-01 19:03:43 +00:00
ensureRuleAtPosition ( builtinChain , args , uuid , 1 )
}
2019-10-17 21:46:15 +00:00
2021-02-01 19:03:43 +00:00
whitelistServiceVips := [ ] string { "-m" , "comment" , "--comment" , "allow traffic to cluster IP" , "-d" , npc . serviceClusterIPRange . String ( ) , "-j" , "RETURN" }
uuid , err := addUUIDForRuleSpec ( kubeInputChainName , & whitelistServiceVips )
if err != nil {
2021-07-07 15:46:10 +00:00
klog . Fatalf ( "Failed to get uuid for rule: %s" , err . Error ( ) )
2021-02-01 19:03:43 +00:00
}
ensureRuleAtPosition ( kubeInputChainName , whitelistServiceVips , uuid , 1 )
2019-10-17 21:46:15 +00:00
2021-02-01 19:03:43 +00:00
whitelistTCPNodeports := [ ] string { "-p" , "tcp" , "-m" , "comment" , "--comment" , "allow LOCAL TCP traffic to node ports" , "-m" , "addrtype" , "--dst-type" , "LOCAL" ,
"-m" , "multiport" , "--dports" , npc . serviceNodePortRange , "-j" , "RETURN" }
uuid , err = addUUIDForRuleSpec ( kubeInputChainName , & whitelistTCPNodeports )
if err != nil {
2021-07-07 15:46:10 +00:00
klog . Fatalf ( "Failed to get uuid for rule: %s" , err . Error ( ) )
2021-02-01 19:03:43 +00:00
}
ensureRuleAtPosition ( kubeInputChainName , whitelistTCPNodeports , uuid , 2 )
whitelistUDPNodeports := [ ] string { "-p" , "udp" , "-m" , "comment" , "--comment" , "allow LOCAL UDP traffic to node ports" , "-m" , "addrtype" , "--dst-type" , "LOCAL" ,
"-m" , "multiport" , "--dports" , npc . serviceNodePortRange , "-j" , "RETURN" }
uuid , err = addUUIDForRuleSpec ( kubeInputChainName , & whitelistUDPNodeports )
if err != nil {
2021-07-07 15:46:10 +00:00
klog . Fatalf ( "Failed to get uuid for rule: %s" , err . Error ( ) )
2021-02-01 19:03:43 +00:00
}
ensureRuleAtPosition ( kubeInputChainName , whitelistUDPNodeports , uuid , 3 )
2019-10-17 21:46:15 +00:00
2021-02-01 19:03:43 +00:00
for externalIPIndex , externalIPRange := range npc . serviceExternalIPRanges {
whitelistServiceVips := [ ] string { "-m" , "comment" , "--comment" , "allow traffic to external IP range: " + externalIPRange . String ( ) , "-d" , externalIPRange . String ( ) , "-j" , "RETURN" }
uuid , err = addUUIDForRuleSpec ( kubeInputChainName , & whitelistServiceVips )
2019-10-17 21:46:15 +00:00
if err != nil {
2021-07-07 15:46:10 +00:00
klog . Fatalf ( "Failed to get uuid for rule: %s" , err . Error ( ) )
2019-10-17 21:46:15 +00:00
}
2021-02-01 19:03:43 +00:00
ensureRuleAtPosition ( kubeInputChainName , whitelistServiceVips , uuid , externalIPIndex + 4 )
2019-10-17 21:46:15 +00:00
}
2021-07-07 15:46:10 +00:00
// for the traffic to/from the local pod's let network policy controller be
// authoritative entity to ACCEPT the traffic if it complies to network policies
for _ , chain := range chains {
comment := "rule to explicitly ACCEPT traffic that comply to network policies"
args := [ ] string { "-m" , "comment" , "--comment" , comment , "-m" , "mark" , "--mark" , "0x20000/0x20000" , "-j" , "ACCEPT" }
err = iptablesCmdHandler . AppendUnique ( "filter" , chain , args ... )
if err != nil {
klog . Fatalf ( "Failed to run iptables command: %s" , err . Error ( ) )
}
}
2019-10-17 21:46:15 +00:00
}
2021-07-07 15:46:10 +00:00
// Creates custom chains KUBE-NWPLCY-DEFAULT
func ( npc * NetworkPolicyController ) ensureDefaultNetworkPolicyChain ( ) {
2019-10-17 21:46:15 +00:00
iptablesCmdHandler , err := iptables . New ( )
if err != nil {
2021-07-07 15:46:10 +00:00
klog . Fatalf ( "Failed to initialize iptables executor due to %s" , err . Error ( ) )
2019-10-17 21:46:15 +00:00
}
2021-07-07 15:46:10 +00:00
markArgs := make ( [ ] string , 0 )
markComment := "rule to mark traffic matching a network policy"
markArgs = append ( markArgs , "-j" , "MARK" , "-m" , "comment" , "--comment" , markComment , "--set-xmark" , "0x10000/0x10000" )
err = iptablesCmdHandler . NewChain ( "filter" , kubeDefaultNetpolChain )
if err != nil && err . ( * iptables . Error ) . ExitStatus ( ) != 1 {
klog . Fatalf ( "Failed to run iptables command to create %s chain due to %s" , kubeDefaultNetpolChain , err . Error ( ) )
}
err = iptablesCmdHandler . AppendUnique ( "filter" , kubeDefaultNetpolChain , markArgs ... )
2019-10-17 21:46:15 +00:00
if err != nil {
2021-07-07 15:46:10 +00:00
klog . Fatalf ( "Failed to run iptables command: %s" , err . Error ( ) )
2021-02-01 19:03:43 +00:00
}
2021-07-07 15:46:10 +00:00
}
func ( npc * NetworkPolicyController ) cleanupStaleRules ( activePolicyChains , activePodFwChains map [ string ] bool ) error {
cleanupPodFwChains := make ( [ ] string , 0 )
cleanupPolicyChains := make ( [ ] string , 0 )
// initialize tool sets for working with iptables and ipset
iptablesCmdHandler , err := iptables . New ( )
2021-02-01 19:03:43 +00:00
if err != nil {
2021-07-07 15:46:10 +00:00
return fmt . Errorf ( "failed to initialize iptables command executor due to %s" , err . Error ( ) )
2019-10-17 21:46:15 +00:00
}
2021-02-01 19:03:43 +00:00
// find iptables chains and ipsets that are no longer used by comparing current to the active maps we were passed
2019-10-17 21:46:15 +00:00
chains , err := iptablesCmdHandler . ListChains ( "filter" )
2021-02-01 19:03:43 +00:00
if err != nil {
2021-07-07 15:46:10 +00:00
return fmt . Errorf ( "unable to list chains: %s" , err )
2021-02-01 19:03:43 +00:00
}
2019-10-17 21:46:15 +00:00
for _ , chain := range chains {
if strings . HasPrefix ( chain , kubeNetworkPolicyChainPrefix ) {
2021-07-07 15:46:10 +00:00
if chain == kubeDefaultNetpolChain {
continue
}
2019-10-17 21:46:15 +00:00
if _ , ok := activePolicyChains [ chain ] ; ! ok {
cleanupPolicyChains = append ( cleanupPolicyChains , chain )
}
}
if strings . HasPrefix ( chain , kubePodFirewallChainPrefix ) {
if _ , ok := activePodFwChains [ chain ] ; ! ok {
cleanupPodFwChains = append ( cleanupPodFwChains , chain )
}
}
}
2021-07-07 15:46:10 +00:00
var newChains , newRules , desiredFilterTable bytes . Buffer
rules := strings . Split ( npc . filterTableRules . String ( ) , "\n" )
if len ( rules ) > 0 && rules [ len ( rules ) - 1 ] == "" {
rules = rules [ : len ( rules ) - 1 ]
}
for _ , rule := range rules {
skipRule := false
for _ , podFWChainName := range cleanupPodFwChains {
if strings . Contains ( rule , podFWChainName ) {
skipRule = true
break
2019-10-17 21:46:15 +00:00
}
2021-07-07 15:46:10 +00:00
}
for _ , policyChainName := range cleanupPolicyChains {
if strings . Contains ( rule , policyChainName ) {
skipRule = true
break
2019-10-17 21:46:15 +00:00
}
}
2021-07-07 15:46:10 +00:00
if strings . Contains ( rule , "COMMIT" ) || strings . HasPrefix ( rule , "# " ) {
skipRule = true
2019-10-17 21:46:15 +00:00
}
2021-07-07 15:46:10 +00:00
if skipRule {
continue
}
if strings . HasPrefix ( rule , ":" ) {
newChains . WriteString ( rule + " - [0:0]\n" )
}
if strings . HasPrefix ( rule , "-" ) {
newRules . WriteString ( rule + "\n" )
2019-10-17 21:46:15 +00:00
}
}
2021-07-07 15:46:10 +00:00
desiredFilterTable . WriteString ( "*filter" + "\n" )
desiredFilterTable . Write ( newChains . Bytes ( ) )
desiredFilterTable . Write ( newRules . Bytes ( ) )
desiredFilterTable . WriteString ( "COMMIT" + "\n" )
npc . filterTableRules = desiredFilterTable
2019-10-17 21:46:15 +00:00
2021-07-07 15:46:10 +00:00
return nil
}
2019-10-17 21:46:15 +00:00
2021-07-07 15:46:10 +00:00
func ( npc * NetworkPolicyController ) cleanupStaleIPSets ( activePolicyIPSets map [ string ] bool ) error {
cleanupPolicyIPSets := make ( [ ] * utils . Set , 0 )
ipsets , err := utils . NewIPSet ( false )
if err != nil {
return fmt . Errorf ( "failed to create ipsets command executor due to %s" , err . Error ( ) )
}
err = ipsets . Save ( )
if err != nil {
klog . Fatalf ( "failed to initialize ipsets command executor due to %s" , err . Error ( ) )
}
for _ , set := range ipsets . Sets {
if strings . HasPrefix ( set . Name , kubeSourceIPSetPrefix ) ||
strings . HasPrefix ( set . Name , kubeDestinationIPSetPrefix ) {
if _ , ok := activePolicyIPSets [ set . Name ] ; ! ok {
cleanupPolicyIPSets = append ( cleanupPolicyIPSets , set )
2019-10-17 21:46:15 +00:00
}
}
}
// cleanup network policy ipsets
for _ , set := range cleanupPolicyIPSets {
err = set . Destroy ( )
if err != nil {
2021-07-07 15:46:10 +00:00
return fmt . Errorf ( "failed to delete ipset %s due to %s" , set . Name , err )
2019-10-17 21:46:15 +00:00
}
}
return nil
}
// Cleanup cleanup configurations done
func ( npc * NetworkPolicyController ) Cleanup ( ) {
2021-07-07 15:46:10 +00:00
klog . Info ( "Cleaning up iptables configuration permanently done by kube-router" )
2019-10-17 21:46:15 +00:00
iptablesCmdHandler , err := iptables . New ( )
if err != nil {
2021-07-07 15:46:10 +00:00
klog . Errorf ( "Failed to initialize iptables executor: %s" , err . Error ( ) )
return
2019-10-17 21:46:15 +00:00
}
// delete jump rules in FORWARD chain to pod specific firewall chain
2021-02-01 19:03:43 +00:00
forwardChainRules , err := iptablesCmdHandler . List ( "filter" , kubeForwardChainName )
2019-10-17 21:46:15 +00:00
if err != nil {
2021-07-07 15:46:10 +00:00
klog . Errorf ( "Failed to delete iptables rules as part of cleanup" )
2019-10-17 21:46:15 +00:00
return
}
2021-01-28 01:26:48 +00:00
// TODO: need a better way to delete rule with out using number
2019-10-17 21:46:15 +00:00
var realRuleNo int
for i , rule := range forwardChainRules {
if strings . Contains ( rule , kubePodFirewallChainPrefix ) {
2021-02-01 19:03:43 +00:00
err = iptablesCmdHandler . Delete ( "filter" , kubeForwardChainName , strconv . Itoa ( i - realRuleNo ) )
if err != nil {
2021-07-07 15:46:10 +00:00
klog . Errorf ( "Failed to delete iptables rule as part of cleanup: %s" , err )
2021-02-01 19:03:43 +00:00
}
2019-10-17 21:46:15 +00:00
realRuleNo ++
}
}
// delete jump rules in OUTPUT chain to pod specific firewall chain
2021-02-01 19:03:43 +00:00
forwardChainRules , err = iptablesCmdHandler . List ( "filter" , kubeOutputChainName )
2019-10-17 21:46:15 +00:00
if err != nil {
2021-07-07 15:46:10 +00:00
klog . Errorf ( "Failed to delete iptables rules as part of cleanup" )
2019-10-17 21:46:15 +00:00
return
}
2021-01-28 01:26:48 +00:00
// TODO: need a better way to delete rule with out using number
2019-10-17 21:46:15 +00:00
realRuleNo = 0
for i , rule := range forwardChainRules {
if strings . Contains ( rule , kubePodFirewallChainPrefix ) {
2021-02-01 19:03:43 +00:00
err = iptablesCmdHandler . Delete ( "filter" , kubeOutputChainName , strconv . Itoa ( i - realRuleNo ) )
if err != nil {
2021-07-07 15:46:10 +00:00
klog . Errorf ( "Failed to delete iptables rule as part of cleanup: %s" , err )
2021-02-01 19:03:43 +00:00
}
2019-10-17 21:46:15 +00:00
realRuleNo ++
}
}
// flush and delete pod specific firewall chain
chains , err := iptablesCmdHandler . ListChains ( "filter" )
2021-02-01 19:03:43 +00:00
if err != nil {
2021-07-07 15:46:10 +00:00
klog . Errorf ( "Unable to list chains: %s" , err )
2021-02-01 19:03:43 +00:00
return
}
2019-10-17 21:46:15 +00:00
for _ , chain := range chains {
if strings . HasPrefix ( chain , kubePodFirewallChainPrefix ) {
err = iptablesCmdHandler . ClearChain ( "filter" , chain )
if err != nil {
2021-07-07 15:46:10 +00:00
klog . Errorf ( "Failed to cleanup iptables rules: " + err . Error ( ) )
2019-10-17 21:46:15 +00:00
return
}
err = iptablesCmdHandler . DeleteChain ( "filter" , chain )
if err != nil {
2021-07-07 15:46:10 +00:00
klog . Errorf ( "Failed to cleanup iptables rules: " + err . Error ( ) )
2019-10-17 21:46:15 +00:00
return
}
}
}
// flush and delete per network policy specific chain
chains , err = iptablesCmdHandler . ListChains ( "filter" )
2021-02-01 19:03:43 +00:00
if err != nil {
2021-07-07 15:46:10 +00:00
klog . Errorf ( "Unable to list chains: %s" , err )
2021-02-01 19:03:43 +00:00
return
}
2019-10-17 21:46:15 +00:00
for _ , chain := range chains {
if strings . HasPrefix ( chain , kubeNetworkPolicyChainPrefix ) {
err = iptablesCmdHandler . ClearChain ( "filter" , chain )
if err != nil {
2021-07-07 15:46:10 +00:00
klog . Errorf ( "Failed to cleanup iptables rules: " + err . Error ( ) )
2019-10-17 21:46:15 +00:00
return
}
err = iptablesCmdHandler . DeleteChain ( "filter" , chain )
if err != nil {
2021-07-07 15:46:10 +00:00
klog . Errorf ( "Failed to cleanup iptables rules: " + err . Error ( ) )
2019-10-17 21:46:15 +00:00
return
}
}
}
// delete all ipsets
2021-07-07 15:46:10 +00:00
klog . V ( 1 ) . Infof ( "Attempting to attain ipset mutex lock" )
npc . ipsetMutex . Lock ( )
klog . V ( 1 ) . Infof ( "Attained ipset mutex lock, continuing..." )
defer func ( ) {
npc . ipsetMutex . Unlock ( )
klog . V ( 1 ) . Infof ( "Returned ipset mutex lock" )
} ( )
2021-02-01 19:03:43 +00:00
ipset , err := utils . NewIPSet ( false )
2019-10-17 21:46:15 +00:00
if err != nil {
2021-07-07 15:46:10 +00:00
klog . Errorf ( "Failed to clean up ipsets: " + err . Error ( ) )
return
2019-10-17 21:46:15 +00:00
}
2021-02-01 19:03:43 +00:00
err = ipset . Save ( )
2019-10-17 21:46:15 +00:00
if err != nil {
2021-07-07 15:46:10 +00:00
klog . Errorf ( "Failed to clean up ipsets: " + err . Error ( ) )
2019-10-17 21:46:15 +00:00
}
2021-02-01 19:03:43 +00:00
err = ipset . DestroyAllWithin ( )
if err != nil {
2021-07-07 15:46:10 +00:00
klog . Errorf ( "Failed to clean up ipsets: " + err . Error ( ) )
2019-10-17 21:46:15 +00:00
}
2021-07-07 15:46:10 +00:00
klog . Infof ( "Successfully cleaned the iptables configuration done by kube-router" )
2019-10-17 21:46:15 +00:00
}
// NewNetworkPolicyController returns new NetworkPolicyController object
2021-02-01 19:03:43 +00:00
func NewNetworkPolicyController ( clientset kubernetes . Interface ,
2021-02-01 19:20:24 +00:00
config * config . Node , podInformer cache . SharedIndexInformer ,
2021-07-07 15:46:10 +00:00
npInformer cache . SharedIndexInformer , nsInformer cache . SharedIndexInformer , ipsetMutex * sync . Mutex ) ( * NetworkPolicyController , error ) {
npc := NetworkPolicyController { ipsetMutex : ipsetMutex }
2019-10-17 21:46:15 +00:00
2021-02-01 19:03:43 +00:00
// Creating a single-item buffered channel to ensure that we only keep a single full sync request at a time,
// additional requests would be pointless to queue since after the first one was processed the system would already
// be up to date with all of the policy changes from any enqueued request after that
npc . fullSyncRequestChan = make ( chan struct { } , 1 )
2019-10-17 21:46:15 +00:00
2021-04-21 22:56:20 +00:00
npc . serviceClusterIPRange = * config . AgentConfig . ServiceCIDR
2021-02-01 19:20:24 +00:00
npc . serviceNodePortRange = strings . ReplaceAll ( config . AgentConfig . ServiceNodePortRange . String ( ) , "-" , ":" )
npc . syncPeriod = defaultSyncPeriod
2021-02-01 19:03:43 +00:00
2021-02-01 19:20:24 +00:00
node , err := utils . GetNodeObject ( clientset , config . AgentConfig . NodeName )
2019-10-17 21:46:15 +00:00
if err != nil {
return nil , err
}
2021-02-01 19:03:43 +00:00
nodeIP , err := utils . GetNodeIP ( node )
2019-10-17 21:46:15 +00:00
if err != nil {
return nil , err
}
npc . nodeIP = nodeIP
npc . podLister = podInformer . GetIndexer ( )
npc . PodEventHandler = npc . newPodEventHandler ( )
npc . nsLister = nsInformer . GetIndexer ( )
npc . NamespaceEventHandler = npc . newNamespaceEventHandler ( )
npc . npLister = npInformer . GetIndexer ( )
npc . NetworkPolicyEventHandler = npc . newNetworkPolicyEventHandler ( )
return & npc , nil
}