Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit b7ad318

Browse files
jessfrazjessfraz
authored
Merge pull request kubernetes#35816 from bprashanth/automated-cherry-pick-of-#31388-kubernetes#35572-upstream-release-1.4
Automated cherry pick of kubernetes#31388 kubernetes#35572
2 parents f7cfbc9 + 75a28c1 commit b7ad318

7 files changed

Lines changed: 388 additions & 34 deletions

File tree

‎pkg/kubelet/network/hostport/hostport.go‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -42,11 +42,11 @@ const (
4242
)
4343

4444
type HostportHandler interface {
45-
OpenPodHostportsAndSync(newPod *RunningPod, natInterfaceName string, runningPods []*RunningPod) error
46-
SyncHostports(natInterfaceName string, runningPods []*RunningPod) error
45+
OpenPodHostportsAndSync(newPod *ActivePod, natInterfaceName string, activePods []*ActivePod) error
46+
SyncHostports(natInterfaceName string, activePods []*ActivePod) error
4747
}
4848

49-
type RunningPod struct {
49+
type ActivePod struct {
5050
Pod *api.Pod
5151
IP net.IP
5252
}
@@ -131,9 +131,9 @@ func (h *handler) openHostports(pod *api.Pod) error {
131131
// gatherAllHostports returns all hostports that should be presented on node,
132132
// given the list of pods running on that node and ignoring host network
133133
// pods (which don't need hostport <-> container port mapping).
134-
func gatherAllHostports(runningPods []*RunningPod) (map[api.ContainerPort]targetPod, error) {
134+
func gatherAllHostports(activePods []*ActivePod) (map[api.ContainerPort]targetPod, error) {
135135
podHostportMap := make(map[api.ContainerPort]targetPod)
136-
for _, r := range runningPods {
136+
for _, r := range activePods {
137137
if r.IP.To4() == nil {
138138
return nil, fmt.Errorf("Invalid or missing pod %s IP", kubecontainer.GetPodFullName(r.Pod))
139139
}
@@ -171,36 +171,37 @@ func hostportChainName(cp api.ContainerPort, podFullName string) utiliptables.Ch
171171
}
172172

173173
// OpenPodHostportsAndSync opens hostports for a new pod, gathers all hostports on
174-
// node, sets up iptables rules enable them. And finally clean up stale hostports
175-
func (h *handler) OpenPodHostportsAndSync(newPod *RunningPod, natInterfaceName string, runningPods []*RunningPod) error {
174+
// node, sets up iptables rules enable them. And finally clean up stale hostports.
175+
// 'newPod' must also be present in 'activePods'.
176+
func (h *handler) OpenPodHostportsAndSync(newPod *ActivePod, natInterfaceName string, activePods []*ActivePod) error {
176177
// try to open pod host port if specified
177178
if err := h.openHostports(newPod.Pod); err != nil {
178179
return err
179180
}
180181

181-
// Add the new pod to running pods if it's not running already (e.g. in rkt's case).
182+
// Add the new pod to active pods if it's not present.
182183
var found bool
183-
for _, p := range runningPods {
184+
for _, p := range activePods {
184185
if p.Pod.UID == newPod.Pod.UID {
185186
found = true
186187
break
187188
}
188189
}
189190
if !found {
190-
runningPods = append(runningPods, newPod)
191+
activePods = append(activePods, newPod)
191192
}
192193

193-
return h.SyncHostports(natInterfaceName, runningPods)
194+
return h.SyncHostports(natInterfaceName, activePods)
194195
}
195196

196197
// SyncHostports gathers all hostports on node and setup iptables rules enable them. And finally clean up stale hostports
197-
func (h *handler) SyncHostports(natInterfaceName string, runningPods []*RunningPod) error {
198+
func (h *handler) SyncHostports(natInterfaceName string, activePods []*ActivePod) error {
198199
start := time.Now()
199200
defer func() {
200201
glog.V(4).Infof("syncHostportsRules took %v", time.Since(start))
201202
}()
202203

203-
containerPortMap, err := gatherAllHostports(runningPods)
204+
containerPortMap, err := gatherAllHostports(activePods)
204205
if err != nil {
205206
return err
206207
}

‎pkg/kubelet/network/hostport/hostport_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,7 @@ func TestOpenPodHostports(t *testing.T) {
158158
},
159159
}
160160

161-
runningPods := make([]*RunningPod, 0)
161+
activePods := make([]*ActivePod, 0)
162162

163163
// Fill in any match rules missing chain names
164164
for _, test := range tests {
@@ -179,13 +179,13 @@ func TestOpenPodHostports(t *testing.T) {
179179
}
180180
}
181181
}
182-
runningPods = append(runningPods, &RunningPod{
182+
activePods = append(activePods, &ActivePod{
183183
Pod: test.pod,
184184
IP: net.ParseIP(test.ip),
185185
})
186186
}
187187

188-
err := h.OpenPodHostportsAndSync(&RunningPod{Pod: tests[0].pod, IP: net.ParseIP(tests[0].ip)}, "br0", runningPods)
188+
err := h.OpenPodHostportsAndSync(&ActivePod{Pod: tests[0].pod, IP: net.ParseIP(tests[0].ip)}, "br0", activePods)
189189
if err != nil {
190190
t.Fatalf("Failed to OpenPodHostportsAndSync: %v", err)
191191
}

‎pkg/kubelet/network/hostport/testing/fake.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,12 +29,12 @@ func NewFakeHostportHandler() hostport.HostportHandler {
2929
return &fakeHandler{}
3030
}
3131

32-
func (h *fakeHandler) OpenPodHostportsAndSync(newPod *hostport.RunningPod, natInterfaceName string, runningPods []*hostport.RunningPod) error {
33-
return h.SyncHostports(natInterfaceName, append(runningPods, newPod))
32+
func (h *fakeHandler) OpenPodHostportsAndSync(newPod *hostport.ActivePod, natInterfaceName string, activePods []*hostport.ActivePod) error {
33+
return h.SyncHostports(natInterfaceName, activePods)
3434
}
3535

36-
func (h *fakeHandler) SyncHostports(natInterfaceName string, runningPods []*hostport.RunningPod) error {
37-
for _, r := range runningPods {
36+
func (h *fakeHandler) SyncHostports(natInterfaceName string, activePods []*hostport.ActivePod) error {
37+
for _, r := range activePods {
3838
if r.IP.To4() == nil {
3939
return fmt.Errorf("Invalid or missing pod %s IP", kubecontainer.GetPodFullName(r.Pod))
4040
}

‎pkg/kubelet/network/kubenet/kubenet_linux.go‎

Lines changed: 128 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,9 @@ package kubenet
2020

2121
import (
2222
"fmt"
23+
"io/ioutil"
2324
"net"
25+
"path/filepath"
2426
"strings"
2527
"sync"
2628
"syscall"
@@ -45,8 +47,9 @@ import (
4547
utilsets "k8s.io/kubernetes/pkg/util/sets"
4648
utilsysctl "k8s.io/kubernetes/pkg/util/sysctl"
4749

48-
"k8s.io/kubernetes/pkg/kubelet/network/hostport"
4950
"strconv"
51+
52+
"k8s.io/kubernetes/pkg/kubelet/network/hostport"
5053
)
5154

5255
const (
@@ -67,6 +70,10 @@ const (
6770

6871
// ebtables Chain to store dedup rules
6972
dedupChain = utilebtables.Chain("KUBE-DEDUP")
73+
74+
// defaultIPAMDir is the default location for the checkpoint files stored by host-local ipam
75+
// https://github.com/containernetworking/cni/tree/master/plugins/ipam/host-local#backends
76+
defaultIPAMDir = "/var/lib/cni/networks"
7077
)
7178

7279
// CNI plugins required by kubenet in /opt/cni/bin or vendor directory
@@ -394,13 +401,13 @@ func (plugin *kubenetNetworkPlugin) setup(namespace string, name string, id kube
394401
plugin.podIPs[id] = ip4.String()
395402

396403
// Open any hostports the pod's containers want
397-
runningPods, err := plugin.getRunningPods()
404+
activePods, err := plugin.getActivePods()
398405
if err != nil {
399406
return err
400407
}
401408

402-
newPod := &hostport.RunningPod{Pod: pod, IP: ip4}
403-
if err := plugin.hostportHandler.OpenPodHostportsAndSync(newPod, BridgeName, runningPods); err != nil {
409+
newPod := &hostport.ActivePod{Pod: pod, IP: ip4}
410+
if err := plugin.hostportHandler.OpenPodHostportsAndSync(newPod, BridgeName, activePods); err != nil {
404411
return err
405412
}
406413

@@ -432,6 +439,16 @@ func (plugin *kubenetNetworkPlugin) SetUpPod(namespace string, name string, id k
432439
// Not a hard error or warning
433440
glog.V(4).Infof("Failed to clean up %s/%s after SetUpPod failure: %v", namespace, name, err)
434441
}
442+
443+
// TODO: Remove this hack once we've figured out how to retrieve the netns
444+
// of an exited container. Currently, restarting docker will leak a bunch of
445+
// ips. This will exhaust available ip space unless we cleanup old ips. At the
446+
// same time we don't want to try GC'ing them periodically as that could lead
447+
// to a performance regression in starting pods. So on each setup failure, try
448+
// GC on the assumption that the kubelet is going to retry pod creation, and
449+
// when it does, there will be ips.
450+
plugin.ipamGarbageCollection()
451+
435452
return err
436453
}
437454

@@ -468,9 +485,9 @@ func (plugin *kubenetNetworkPlugin) teardown(namespace string, name string, id k
468485
}
469486
}
470487

471-
runningPods, err := plugin.getRunningPods()
488+
activePods, err := plugin.getActivePods()
472489
if err == nil {
473-
err = plugin.hostportHandler.SyncHostports(BridgeName, runningPods)
490+
err = plugin.hostportHandler.SyncHostports(BridgeName, activePods)
474491
}
475492
if err != nil {
476493
errList = append(errList, err)
@@ -571,15 +588,31 @@ func (plugin *kubenetNetworkPlugin) checkCNIPluginInDir(dir string) bool {
571588
return true
572589
}
573590

574-
// Returns a list of pods running on this node and each pod's IP address. Assumes
575-
// PodSpecs retrieved from the runtime include the name and ID of containers in
576-
// each pod.
577-
func (plugin *kubenetNetworkPlugin) getRunningPods() ([]*hostport.RunningPod, error) {
578-
pods, err := plugin.host.GetRuntime().GetPods(false)
591+
// getNonExitedPods returns a list of pods that have at least one running container.
592+
func (plugin *kubenetNetworkPlugin) getNonExitedPods() ([]*kubecontainer.Pod, error) {
593+
ret := []*kubecontainer.Pod{}
594+
pods, err := plugin.host.GetRuntime().GetPods(true)
579595
if err != nil {
580596
return nil, fmt.Errorf("Failed to retrieve pods from runtime: %v", err)
581597
}
582-
runningPods := make([]*hostport.RunningPod, 0)
598+
for _, p := range pods {
599+
if podIsExited(p) {
600+
continue
601+
}
602+
ret = append(ret, p)
603+
}
604+
return ret, nil
605+
}
606+
607+
// Returns a list of pods running or ready to run on this node and each pod's IP address.
608+
// Assumes PodSpecs retrieved from the runtime include the name and ID of containers in
609+
// each pod.
610+
func (plugin *kubenetNetworkPlugin) getActivePods() ([]*hostport.ActivePod, error) {
611+
pods, err := plugin.getNonExitedPods()
612+
if err != nil {
613+
return nil, err
614+
}
615+
activePods := make([]*hostport.ActivePod, 0)
583616
for _, p := range pods {
584617
containerID, err := plugin.host.GetRuntime().GetPodContainerID(p)
585618
if err != nil {
@@ -594,13 +627,94 @@ func (plugin *kubenetNetworkPlugin) getRunningPods() ([]*hostport.RunningPod, er
594627
continue
595628
}
596629
if pod, ok := plugin.host.GetPodByName(p.Namespace, p.Name); ok {
597-
runningPods = append(runningPods, &hostport.RunningPod{
630+
activePods = append(activePods, &hostport.ActivePod{
598631
Pod: pod,
599632
IP: podIP,
600633
})
601634
}
602635
}
603-
return runningPods, nil
636+
return activePods, nil
637+
}
638+
639+
// ipamGarbageCollection will release unused IP.
640+
// kubenet uses the CNI bridge plugin, which stores allocated ips on file. Each
641+
// file created under defaultIPAMDir has the format: ip/container-hash. So this
642+
// routine looks for hashes that are not reported by the currently running docker,
643+
// and invokes DelNetwork on each one. Note that this will only work for the
644+
// current CNI bridge plugin, because we have no way of finding the NetNs.
645+
func (plugin *kubenetNetworkPlugin) ipamGarbageCollection() {
646+
glog.V(2).Infof("Starting IP garbage collection")
647+
648+
ipamDir := filepath.Join(defaultIPAMDir, KubenetPluginName)
649+
files, err := ioutil.ReadDir(ipamDir)
650+
if err != nil {
651+
glog.Errorf("Failed to list files in %q: %v", ipamDir, err)
652+
return
653+
}
654+
655+
// gather containerIDs for allocated ips
656+
ipContainerIdMap := make(map[string]string)
657+
for _, file := range files {
658+
// skip non checkpoint file
659+
if ip := net.ParseIP(file.Name()); ip == nil {
660+
continue
661+
}
662+
663+
content, err := ioutil.ReadFile(filepath.Join(ipamDir, file.Name()))
664+
if err != nil {
665+
glog.Errorf("Failed to read file %v: %v", file, err)
666+
}
667+
ipContainerIdMap[file.Name()] = strings.TrimSpace(string(content))
668+
}
669+
670+
// gather infra container IDs of current running Pods
671+
runningContainerIDs := utilsets.String{}
672+
pods, err := plugin.getNonExitedPods()
673+
if err != nil {
674+
glog.Errorf("Failed to get pods: %v", err)
675+
return
676+
}
677+
for _, pod := range pods {
678+
containerID, err := plugin.host.GetRuntime().GetPodContainerID(pod)
679+
if err != nil {
680+
glog.Warningf("Failed to get infra containerID of %q/%q: %v", pod.Namespace, pod.Name, err)
681+
continue
682+
}
683+
684+
runningContainerIDs.Insert(strings.TrimSpace(containerID.ID))
685+
}
686+
687+
// release leaked ips
688+
for ip, containerID := range ipContainerIdMap {
689+
// if the container is not running, release IP
690+
if runningContainerIDs.Has(containerID) {
691+
continue
692+
}
693+
// CNI requires all config to be presented, although only containerID is needed in this case
694+
rt := &libcni.RuntimeConf{
695+
ContainerID: containerID,
696+
IfName: network.DefaultInterfaceName,
697+
// TODO: How do we find the NetNs of an exited container? docker inspect
698+
// doesn't show us the pid, so we probably need to checkpoint
699+
NetNS: "",
700+
}
701+
702+
glog.V(2).Infof("Releasing IP %q allocated to %q.", ip, containerID)
703+
// CNI bridge plugin should try to release IP and then return
704+
if err := plugin.cniConfig.DelNetwork(plugin.netConfig, rt); err != nil {
705+
glog.Errorf("Error while releasing IP: %v", err)
706+
}
707+
}
708+
}
709+
710+
// podIsExited returns true if the pod is exited (all containers inside are exited).
711+
func podIsExited(p *kubecontainer.Pod) bool {
712+
for _, c := range p.Containers {
713+
if c.State != kubecontainer.ContainerStateExited {
714+
return false
715+
}
716+
}
717+
return true
604718
}
605719

606720
func (plugin *kubenetNetworkPlugin) buildCNIRuntimeConf(ifName string, id kubecontainer.ContainerID) (*libcni.RuntimeConf, error) {

0 commit comments

Comments
 (0)