Skip to content

Commit

Permalink
Notice that rule for not tracking DNS packet is experimental and that…
Browse files Browse the repository at this point in the history
… rules is disabled by default
  • Loading branch information
ssup2 committed Dec 9, 2020
1 parent 4e242b3 commit 3c57660
Show file tree
Hide file tree
Showing 5 changed files with 49 additions and 44 deletions.
25 changes: 13 additions & 12 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@

network-node-manager is a kubernetes controller that controls the network configuration of a node to resolve network issues of kubernetes. By simply deploying and configuring network-node-manager, you can solve kubernetes network issues that cannot be resolved by kubernetes or resolved by the higher kubernetes Version. Below is a list of kubernetes's issues to be resolved by network-node-manager. network-node-manager is based on [kubebuilder](https://github.com/kubernetes-sigs/kubebuilder).

* [DNS packet dropped issue](issues/DNS_packet_dropped_issue.md)
* [Connection reset issue between pod and out of cluster](issues/connection_reset_issue_pod_out_cluster.md)
* [External-IP access issue with IPVS proxy mode](issues/external_IP_access_issue_IPVS_proxy_mode.md)
* [DNS packet dropped issue](issues/DNS_packet_dropped_issue.md) (Experimental)

## Deploy

Expand Down Expand Up @@ -33,17 +33,6 @@ IPv4 : kubectl -n kube-system set env daemonset/network-node-manager NET_ST
IPv6 : kubectl -n kube-system set env daemonset/network-node-manager NET_STACK=ipv6
IPv4,IPv6 : kubectl -n kube-system set env daemonset/network-node-manager NET_STACK=ipv4,ipv6
```
### Not Track DNS Packet Rule

* Related issue : [DNS packet dropped issue](issues/DNS_packet_dropped_issue.md)
* Default : true
* iptables proxy mode manifest : true
* IPVS proxy mode manifest : true

```
On : kubectl -n kube-system set env daemonset/network-node-manager RULE_NOT_TRACK_DNS=true
Off : kubectl -n kube-system set env daemonset/network-node-manager RULE_NOT_TRACK_DNS=false
```

### Drop Invalid Packet Rule in INPUT chain

Expand All @@ -69,6 +58,18 @@ On : kubectl -n kube-system set env daemonset/network-node-manager RULE_EXTERNA
Off : kubectl -n kube-system set env daemonset/network-node-manager RULE_EXTERNAL_CLUSTER=false
```

### Not Track DNS Packet Rule (Experimental)

* Related issue : [DNS packet dropped issue](issues/DNS_packet_dropped_issue.md)
* Default : false
* iptables proxy mode manifest : false
* IPVS proxy mode manifest : false

```
On : kubectl -n kube-system set env daemonset/network-node-manager RULE_NOT_TRACK_DNS=true
Off : kubectl -n kube-system set env daemonset/network-node-manager RULE_NOT_TRACK_DNS=false
```

## How it works?

![kpexec Architecture](img/network-node-manager_Architecture.PNG)
Expand Down
40 changes: 20 additions & 20 deletions controllers/service_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,9 @@ var (
configIPv4Enabled bool
configIPv6Enabled bool

configRuleNotTrackDNS bool
configRuleDropInvalidInput bool
configRuleExternalCluster bool
configRuleNotTrackDNS bool

initFlag = false
podCIDRIPv4 string
Expand Down Expand Up @@ -93,48 +93,48 @@ func (r *ServiceReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
logger.WithValues("node name", configNodeName).Info("config node name")
logger.WithValues("IPv4", configIPv4Enabled).WithValues("IPv6", configIPv6Enabled).Info("config network stack")

configRuleNotTrackDNS, err = configs.GetConfigRuleNotTrackDNS()
configRuleDropInvalidInput, err = configs.GetConfigRuleDropInvalidInput()
if err != nil {
logger.Error(err, "config error")
os.Exit(1)
}
configRuleDropInvalidInput, err = configs.GetConfigRuleDropInvalidInput()
configRuleExternalCluster, err = configs.GetConfigRuleExternalCluster()
if err != nil {
logger.Error(err, "config error")
os.Exit(1)
}
configRuleExternalCluster, err = configs.GetConfigRuleExternalCluster()
configRuleNotTrackDNS, err = configs.GetConfigRuleNotTrackDNS()
if err != nil {
logger.Error(err, "config error")
os.Exit(1)
}
logger.WithValues("flag", configRuleNotTrackDNS).Info("config rule not tracking DNS packet")
logger.WithValues("flag", configRuleDropInvalidInput).Info("config rule drop invalid packet in INPUT chain")
logger.WithValues("flag", configRuleExternalCluster).Info("config rule externalIP to clusterIP")
logger.WithValues("flag", configRuleNotTrackDNS).Info("config rule not tracking DNS packet")

// Init packages
rules.Init(configIPv4Enabled, configIPv6Enabled)

// Run rules first
if configRuleNotTrackDNS {
if err := rules.CreateRulesNotTrackDNS(logger); err != nil {
logger.Error(err, "failed to create rule not tracking DNS packet")
if configRuleDropInvalidInput {
if err := rules.CreateRulesDropInvalidInput(logger); err != nil {
logger.Error(err, "failed to create rule drop invalid packet in INPUT chain")
os.Exit(1)
}
} else {
if err := rules.DeleteRulesNotTrackDNS(logger); err != nil {
logger.Error(err, "failed to delete rule not trackring DNS packet")
if err := rules.DeleteRulesDropInvalidInput(logger); err != nil {
logger.Error(err, "failed to delete rule drop invalid packet in INPUT chain")
os.Exit(1)
}
}
if configRuleDropInvalidInput {
if err := rules.CreateRulesDropInvalidInput(logger); err != nil {
logger.Error(err, "failed to create rule drop invalid packet in INPUT chain")
if configRuleNotTrackDNS {
if err := rules.CreateRulesNotTrackDNS(logger); err != nil {
logger.Error(err, "failed to create rule not tracking DNS packet")
os.Exit(1)
}
} else {
if err := rules.DeleteRulesDropInvalidInput(logger); err != nil {
logger.Error(err, "failed to delete rule drop invalid packet in INPUT chain")
if err := rules.DeleteRulesNotTrackDNS(logger); err != nil {
logger.Error(err, "failed to delete rule not trackring DNS packet")
os.Exit(1)
}
}
Expand All @@ -145,16 +145,16 @@ func (r *ServiceReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
for {
<-ticker.C

if configRuleNotTrackDNS {
if err := rules.CreateRulesNotTrackDNS(logger); err != nil {
logger.Error(err, "failed to create rule not tracking DNS packet")
}
}
if configRuleDropInvalidInput {
if err := rules.CreateRulesDropInvalidInput(logger); err != nil {
logger.Error(err, "failed to create rule drop invalid packet in INPUT chain")
}
}
if configRuleNotTrackDNS {
if err := rules.CreateRulesNotTrackDNS(logger); err != nil {
logger.Error(err, "failed to create rule not tracking DNS packet")
}
}
}
}()

Expand Down
4 changes: 4 additions & 0 deletions issues/DNS_packet_dropped_issue.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@

There is an issue in which some DNS packets are dropped due to the race condition of linux kernel conntrack. Because of this issue, a phenomenon in which Record Resolve of Service or Pod within Kubernetes Cluster often fails occurs.

## Caution and Preparation

Currently, this solution is in **experimental** stage. This solution only works in environments where systemd-resolved dose not run in kubernetes cluster nodes. **This is because CoreDNS must be run in the host network namespace.** Before applying this solution, configure CoreDNS to run in the host network namespace.

## How to solve it

network-node-manager adds the not tracking rules for DNS packet to avoid the race condition of linux kernel conntrack. Below are rules set by network-node-manager.
Expand Down
20 changes: 10 additions & 10 deletions pkg/configs/configs.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,9 +53,9 @@ func GetConfigNetStack() (bool, bool, error) {
return ipv4, ipv6, nil
}

func GetConfigRuleNotTrackDNS() (bool, error) {
func GetConfigRuleDropInvalidInput() (bool, error) {
// organize configs
config := os.Getenv(EnvRuleDropNotTrackDNS)
config := os.Getenv(EnvRuleDropInvalidInput)
config = strings.ToLower(config)
if config == "" {
return true, nil
Expand All @@ -67,15 +67,15 @@ func GetConfigRuleNotTrackDNS() (bool, error) {
} else if config == EnvConfigTrue {
return true, nil
}
return false, fmt.Errorf("wrong config for not tracking DNS packet : %s", config)
return false, fmt.Errorf("wrong config for drop invalid packet in INPUT chain : %s", config)
}

func GetConfigRuleDropInvalidInput() (bool, error) {
func GetConfigRuleExternalCluster() (bool, error) {
// organize configs
config := os.Getenv(EnvRuleDropInvalidInput)
config := os.Getenv(EnvRuleExternalCluster)
config = strings.ToLower(config)
if config == "" {
return true, nil
return false, nil
}

// return configs
Expand All @@ -84,12 +84,12 @@ func GetConfigRuleDropInvalidInput() (bool, error) {
} else if config == EnvConfigTrue {
return true, nil
}
return false, fmt.Errorf("wrong config for drop invalid packet in INPUT chain : %s", config)
return false, fmt.Errorf("wrong config for externalIP to clusterIP DNAT : %s", config)
}

func GetConfigRuleExternalCluster() (bool, error) {
func GetConfigRuleNotTrackDNS() (bool, error) {
// organize configs
config := os.Getenv(EnvRuleExternalCluster)
config := os.Getenv(EnvRuleDropNotTrackDNS)
config = strings.ToLower(config)
if config == "" {
return false, nil
Expand All @@ -101,5 +101,5 @@ func GetConfigRuleExternalCluster() (bool, error) {
} else if config == EnvConfigTrue {
return true, nil
}
return false, fmt.Errorf("wrong config for externalIP to clusterIP DNAT : %s", config)
return false, fmt.Errorf("wrong config for not tracking DNS packet : %s", config)
}
4 changes: 2 additions & 2 deletions pkg/rules/rules.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,11 @@ const (
ChainBaseInput = "NMANAGER_INPUT"
ChainBaseOutput = "NMANAGER_OUTPUT"

ChainRawNotTrackDNSPrerouting = "NMANAGER_NOT_DNS_PREROUTING"
ChainRawNotTrackDNSOutput = "NMANAGER_NOT_DNS_OUTPUT"
ChainFilterDropInvalidInput = "NMANAGER_DROP_INVALID_INPUT"
ChainNATExternalClusterPrerouting = "NMANAGER_EX_CLUS_PREROUTING"
ChainNATExternalClusterOutput = "NMANAGER_EX_CLUS_OUTPUT"
ChainRawNotTrackDNSPrerouting = "NMANAGER_NOT_DNS_PREROUTING"
ChainRawNotTrackDNSOutput = "NMANAGER_NOT_DNS_OUTPUT"

ChainNATKubeMarkMasq = "KUBE-MARK-MASQ"
)
Expand Down

0 comments on commit 3c57660

Please sign in to comment.