diff --git a/cmd/node-cache/app/cache_app.go b/cmd/node-cache/app/cache_app.go index 098874c3f..ab8156927 100644 --- a/cmd/node-cache/app/cache_app.go +++ b/cmd/node-cache/app/cache_app.go @@ -25,6 +25,7 @@ import ( "k8s.io/dns/cmd/kube-dns/app/options" "k8s.io/dns/pkg/dns/config" "k8s.io/dns/pkg/netif" + "k8s.io/kubernetes/pkg/util/iptables" utiliptables "k8s.io/kubernetes/pkg/util/iptables" utilexec "k8s.io/utils/exec" utilnet "k8s.io/utils/net" @@ -62,13 +63,15 @@ type iptablesRule struct { // CacheApp contains all the config required to run node-cache. type CacheApp struct { - iptables utiliptables.Interface - iptablesRules []iptablesRule - params *ConfigParams - netifHandle *netif.NetifManager - kubednsConfig *options.KubeDNSConfig - exitChan chan struct{} // Channel to terminate background goroutines - clusterDNSIP net.IP + iptablesV4 utiliptables.Interface + iptablesV6 utiliptables.Interface + iptablesRulesV4 []iptablesRule + iptablesRulesV6 []iptablesRule + params *ConfigParams + netifHandle *netif.NetifManager + kubednsConfig *options.KubeDNSConfig + exitChan chan struct{} // Channel to terminate background goroutines + clusterDNSIP net.IP } func isLockedErr(err error) bool { @@ -98,61 +101,86 @@ func (c *CacheApp) Init() { c.params.SetupIptables = setupIptables } -// isIPv6 return if the node-cache is working in IPv6 mode -// LocalIPs are guaranteed to have the same family -func (c *CacheApp) isIPv6() bool { - if len(c.params.LocalIPs) > 0 { - return utilnet.IsIPv6(c.params.LocalIPs[0]) - } - return false -} - func (c *CacheApp) initIptables() { // using the localIPStr param since we need ip strings here for _, localIP := range strings.Split(c.params.LocalIPStr, ",") { - c.iptablesRules = append(c.iptablesRules, []iptablesRule{ - // Match traffic destined for localIp:localPort and set the flows to be NOTRACKED, this skips connection tracking - {utiliptables.Table("raw"), utiliptables.ChainPrerouting, []string{"-p", "tcp", "-d", localIP, - "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, - {utiliptables.Table("raw"), utiliptables.ChainPrerouting, []string{"-p", "udp", "-d", localIP, - "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, - // There are rules in filter table to allow tracked connections to be accepted. Since we skipped connection tracking, - // need these additional filter table rules. - {utiliptables.TableFilter, utiliptables.ChainInput, []string{"-p", "tcp", "-d", localIP, - "--dport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, - {utiliptables.TableFilter, utiliptables.ChainInput, []string{"-p", "udp", "-d", localIP, - "--dport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, - // Match traffic from localIp:localPort and set the flows to be NOTRACKED, this skips connection tracking - {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP, - "--sport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, - {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "udp", "-s", localIP, - "--sport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, - // Additional filter table rules for traffic frpm localIp:localPort - {utiliptables.TableFilter, utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP, - "--sport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, - {utiliptables.TableFilter, utiliptables.ChainOutput, []string{"-p", "udp", "-s", localIP, - "--sport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, - // Skip connection tracking for requests to nodelocalDNS that are locally generated, example - by hostNetwork pods - {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-d", localIP, - "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, - {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "udp", "-d", localIP, - "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, - // skip connection tracking for healthcheck requests generated by liveness probe to health plugin - {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-d", localIP, - "--dport", c.params.HealthPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, - {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP, - "--sport", c.params.HealthPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, - }...) + if utilnet.IsIPv6(net.ParseIP(localIP)) { + c.iptablesRulesV6 = append(c.iptablesRulesV6, []iptablesRule{ + // Match traffic destined for localIp:localPort and set the flows to be NOTRACKED, this skips connection tracking + {utiliptables.Table("raw"), utiliptables.ChainPrerouting, []string{"-p", "tcp", "-d", localIP, + "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + {utiliptables.Table("raw"), utiliptables.ChainPrerouting, []string{"-p", "udp", "-d", localIP, + "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + // There are rules in filter table to allow tracked connections to be accepted. Since we skipped connection tracking, + // need these additional filter table rules. + {utiliptables.TableFilter, utiliptables.ChainInput, []string{"-p", "tcp", "-d", localIP, + "--dport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, + {utiliptables.TableFilter, utiliptables.ChainInput, []string{"-p", "udp", "-d", localIP, + "--dport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, + // Match traffic from localIp:localPort and set the flows to be NOTRACKED, this skips connection tracking + {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP, + "--sport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "udp", "-s", localIP, + "--sport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + // Additional filter table rules for traffic frpm localIp:localPort + {utiliptables.TableFilter, utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP, + "--sport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, + {utiliptables.TableFilter, utiliptables.ChainOutput, []string{"-p", "udp", "-s", localIP, + "--sport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, + // Skip connection tracking for requests to nodelocalDNS that are locally generated, example - by hostNetwork pods + {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-d", localIP, + "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "udp", "-d", localIP, + "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + // skip connection tracking for healthcheck requests generated by liveness probe to health plugin + {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-d", localIP, + "--dport", c.params.HealthPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP, + "--sport", c.params.HealthPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + }...) + } else { + c.iptablesRulesV4 = append(c.iptablesRulesV4, []iptablesRule{ + // Match traffic destined for localIp:localPort and set the flows to be NOTRACKED, this skips connection tracking + {utiliptables.Table("raw"), utiliptables.ChainPrerouting, []string{"-p", "tcp", "-d", localIP, + "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + {utiliptables.Table("raw"), utiliptables.ChainPrerouting, []string{"-p", "udp", "-d", localIP, + "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + // There are rules in filter table to allow tracked connections to be accepted. Since we skipped connection tracking, + // need these additional filter table rules. + {utiliptables.TableFilter, utiliptables.ChainInput, []string{"-p", "tcp", "-d", localIP, + "--dport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, + {utiliptables.TableFilter, utiliptables.ChainInput, []string{"-p", "udp", "-d", localIP, + "--dport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, + // Match traffic from localIp:localPort and set the flows to be NOTRACKED, this skips connection tracking + {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP, + "--sport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "udp", "-s", localIP, + "--sport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + // Additional filter table rules for traffic frpm localIp:localPort + {utiliptables.TableFilter, utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP, + "--sport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, + {utiliptables.TableFilter, utiliptables.ChainOutput, []string{"-p", "udp", "-s", localIP, + "--sport", c.params.LocalPort, "-j", "ACCEPT", "-m", "comment", "--comment", iptablesCommentAllowTraffic}}, + // Skip connection tracking for requests to nodelocalDNS that are locally generated, example - by hostNetwork pods + {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-d", localIP, + "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "udp", "-d", localIP, + "--dport", c.params.LocalPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + // skip connection tracking for healthcheck requests generated by liveness probe to health plugin + {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-d", localIP, + "--dport", c.params.HealthPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + {utiliptables.Table("raw"), utiliptables.ChainOutput, []string{"-p", "tcp", "-s", localIP, + "--sport", c.params.HealthPort, "-j", "NOTRACK", "-m", "comment", "--comment", iptablesCommentSkipConntrack}}, + }...) + } + } - c.iptables = newIPTables(c.isIPv6()) + c.iptablesV4 = newIPTables(iptables.ProtocolIPv4) + c.iptablesV6 = newIPTables(iptables.ProtocolIPv6) } -func newIPTables(isIPv6 bool) utiliptables.Interface { +func newIPTables(protocol iptables.Protocol) utiliptables.Interface { execer := utilexec.New() - protocol := utiliptables.ProtocolIPv4 - if isIPv6 { - protocol = utiliptables.ProtocolIPv6 - } return utiliptables.New(execer, protocol) } @@ -180,23 +208,41 @@ func (c *CacheApp) TeardownNetworking() error { err = c.netifHandle.RemoveDummyDevice(c.params.InterfaceName) } if c.params.SetupIptables { - for _, rule := range c.iptablesRules { + for _, rule := range c.iptablesRulesV4 { + exists := true + for exists == true { + // check in a loop in case the same rule got added multiple times. + err = c.iptablesV4.DeleteRule(rule.table, rule.chain, rule.args...) + if err != nil { + clog.Errorf("Failed deleting iptablesV4 rule %v, error - %v", rule, err) + handleIPTablesError(err) + } + exists, err = c.iptablesV4.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...) + if err != nil { + clog.Errorf("Failed checking iptablesV4 rule after deletion, rule - %v, error - %v", rule, err) + handleIPTablesError(err) + } + } + // Delete the rule one last time since EnsureRule creates the rule if it doesn't exist + err = c.iptablesV4.DeleteRule(rule.table, rule.chain, rule.args...) + } + for _, rule := range c.iptablesRulesV6 { exists := true for exists == true { // check in a loop in case the same rule got added multiple times. - err = c.iptables.DeleteRule(rule.table, rule.chain, rule.args...) + err = c.iptablesV6.DeleteRule(rule.table, rule.chain, rule.args...) if err != nil { - clog.Errorf("Failed deleting iptables rule %v, error - %v", rule, err) + clog.Errorf("Failed deleting iptablesV6 rule %v, error - %v", rule, err) handleIPTablesError(err) } - exists, err = c.iptables.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...) + exists, err = c.iptablesV6.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...) if err != nil { - clog.Errorf("Failed checking iptables rule after deletion, rule - %v, error - %v", rule, err) + clog.Errorf("Failed checking iptablesV6 rule after deletion, rule - %v, error - %v", rule, err) handleIPTablesError(err) } } // Delete the rule one last time since EnsureRule creates the rule if it doesn't exist - err = c.iptables.DeleteRule(rule.table, rule.chain, rule.args...) + err = c.iptablesV6.DeleteRule(rule.table, rule.chain, rule.args...) } } return err @@ -204,8 +250,24 @@ func (c *CacheApp) TeardownNetworking() error { func (c *CacheApp) setupNetworking() { if c.params.SetupIptables { - for _, rule := range c.iptablesRules { - exists, err := c.iptables.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...) + for _, rule := range c.iptablesRulesV4 { + exists, err := c.iptablesV4.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...) + switch { + case exists: + // debug messages can be printed by including "debug" plugin in coreFile. + clog.Debugf("iptables rule %v for nodelocaldns already exists", rule) + continue + case err == nil: + clog.Infof("Added back nodelocaldns rule - %v", rule) + continue + default: + // iptables check/rule add failed with error since control reached here. + clog.Errorf("Error checking/adding iptables rule %v, error - %v", rule, err) + handleIPTablesError(err) + } + } + for _, rule := range c.iptablesRulesV6 { + exists, err := c.iptablesV6.EnsureRule(utiliptables.Prepend, rule.table, rule.chain, rule.args...) switch { case exists: // debug messages can be printed by including "debug" plugin in coreFile. diff --git a/cmd/node-cache/main.go b/cmd/node-cache/main.go index e8f8eab6a..9a8e1e19d 100644 --- a/cmd/node-cache/main.go +++ b/cmd/node-cache/main.go @@ -25,7 +25,6 @@ import ( corednsmain "github.com/coredns/coredns/coremain" clog "github.com/coredns/coredns/plugin/pkg/log" - utilnet "k8s.io/utils/net" "github.com/coredns/caddy" // blank imports to make sure the plugin code is pulled in from vendor when building node-cache image @@ -100,12 +99,6 @@ func parseAndValidateFlags() (*app.ConfigParams, error) { params.LocalIPs = append(params.LocalIPs, newIP) } - // validate all the IPs have the same IP family - for _, ip := range params.LocalIPs { - if utilnet.IsIPv6(params.LocalIPs[0]) != utilnet.IsIPv6(ip) { - return params, fmt.Errorf("unexpected IP Family for localIP - %q, want IPv6=%v", ip, utilnet.IsIPv6(params.LocalIPs[0])) - } - } // lookup specified dns port f := flag.Lookup("dns.port") if f == nil {