move to cadvisor.MachineInfo

This patch removes GetNUMANodeInfo, cadvisor.MachineInfo will be used instead of it. GetNUMANodeInfo was introduced due to difference of meaning of MachineInfo.Topology. On the arm it was NUMA nodes, but on the x86 it represents sockets (since reading from /proc/cpuinfo). Now it unified and MachineInfo.Topology represents NUMA node. Signed-off-by: Alexey Perevalov <[email protected]>
openshift · damemi · Feb 17, 2021 · Jan 27, 2021 · Feb 17, 2021 · Feb 2, 2021
commit 33547e4d2c79a044e5f61c9d0f5954d4bcfb1e1c
diff --git a/pkg/kubelet/cm/BUILD b/pkg/kubelet/cm/BUILD
@@ -56,7 +56,6 @@ go_library(
             "//pkg/apis/core/v1/helper/qos:go_default_library",
             "//pkg/kubelet/cadvisor:go_default_library",
             "//pkg/kubelet/cm/containermap:go_default_library",
-            "//pkg/kubelet/cm/cpumanager/topology:go_default_library",
             "//pkg/kubelet/cm/devicemanager:go_default_library",
             "//pkg/kubelet/cm/util:go_default_library",
             "//pkg/kubelet/events:go_default_library",
@@ -118,7 +117,6 @@ go_library(
             "//pkg/apis/core/v1/helper/qos:go_default_library",
             "//pkg/kubelet/cadvisor:go_default_library",
             "//pkg/kubelet/cm/containermap:go_default_library",
-            "//pkg/kubelet/cm/cpumanager/topology:go_default_library",
             "//pkg/kubelet/cm/devicemanager:go_default_library",
             "//pkg/kubelet/cm/util:go_default_library",
             "//pkg/kubelet/events:go_default_library",

diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go
@@ -52,7 +52,6 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/cadvisor"
 	"k8s.io/kubernetes/pkg/kubelet/cm/containermap"
 	"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager"
-	cputopology "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
 	"k8s.io/kubernetes/pkg/kubelet/cm/devicemanager"
 	"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
 	cmutil "k8s.io/kubernetes/pkg/kubelet/cm/util"
@@ -238,13 +237,6 @@ func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.I
 	if err != nil {
 		return nil, err
 	}
-	// Correct NUMA information is currently missing from cadvisor's
-	// MachineInfo struct, so we use the CPUManager's internal logic for
-	// gathering NUMANodeInfo to pass to components that care about it.
-	numaNodeInfo, err := cputopology.GetNUMANodeInfo()
-	if err != nil {
-		return nil, err
-	}
 	capacity := cadvisor.CapacityFromMachineInfo(machineInfo)
 	for k, v := range capacity {
 		internalCapacity[k] = v
@@ -300,7 +292,7 @@ func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.I
 
 	if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.TopologyManager) {
 		cm.topologyManager, err = topologymanager.NewManager(
-			numaNodeInfo,
+			machineInfo.Topology,
 			nodeConfig.ExperimentalTopologyManagerPolicy,
 		)
 
@@ -315,7 +307,7 @@ func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.I
 
 	klog.Infof("Creating device plugin manager: %t", devicePluginEnabled)
 	if devicePluginEnabled {
-		cm.deviceManager, err = devicemanager.NewManagerImpl(numaNodeInfo, cm.topologyManager)
+		cm.deviceManager, err = devicemanager.NewManagerImpl(machineInfo.Topology, cm.topologyManager)
 		cm.topologyManager.AddHintProvider(cm.deviceManager)
 	} else {
 		cm.deviceManager, err = devicemanager.NewManagerStub()
@@ -330,7 +322,6 @@ func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.I
 			nodeConfig.ExperimentalCPUManagerPolicy,
 			nodeConfig.ExperimentalCPUManagerReconcilePeriod,
 			machineInfo,
-			numaNodeInfo,
 			nodeConfig.NodeAllocatableConfig.ReservedSystemCPUs,
 			cm.GetNodeAllocatableReservation(),
 			nodeConfig.KubeletRootDir,

diff --git a/pkg/kubelet/cm/cpumanager/cpu_manager.go b/pkg/kubelet/cm/cpumanager/cpu_manager.go
@@ -126,7 +126,7 @@ func (s *sourcesReadyStub) AddSource(source string) {}
 func (s *sourcesReadyStub) AllReady() bool          { return true }
 
 // NewManager creates new cpu manager based on provided policy
-func NewManager(cpuPolicyName string, reconcilePeriod time.Duration, machineInfo *cadvisorapi.MachineInfo, numaNodeInfo topology.NUMANodeInfo, specificCPUs cpuset.CPUSet, nodeAllocatableReservation v1.ResourceList, stateFileDirectory string, affinity topologymanager.Store) (Manager, error) {
+func NewManager(cpuPolicyName string, reconcilePeriod time.Duration, machineInfo *cadvisorapi.MachineInfo, specificCPUs cpuset.CPUSet, nodeAllocatableReservation v1.ResourceList, stateFileDirectory string, affinity topologymanager.Store) (Manager, error) {
 	var topo *topology.CPUTopology
 	var policy Policy
 
@@ -137,7 +137,7 @@ func NewManager(cpuPolicyName string, reconcilePeriod time.Duration, machineInfo
 
 	case PolicyStatic:
 		var err error
-		topo, err = topology.Discover(machineInfo, numaNodeInfo)
+		topo, err = topology.Discover(machineInfo)
 		if err != nil {
 			return nil, err
 		}

diff --git a/pkg/kubelet/cm/cpumanager/cpu_manager_test.go b/pkg/kubelet/cm/cpumanager/cpu_manager_test.go
@@ -629,7 +629,7 @@ func TestCPUManagerGenerate(t *testing.T) {
 			}
 			defer os.RemoveAll(sDir)
 
-			mgr, err := NewManager(testCase.cpuPolicyName, 5*time.Second, machineInfo, nil, cpuset.NewCPUSet(), testCase.nodeAllocatableReservation, sDir, topologymanager.NewFakeManager())
+			mgr, err := NewManager(testCase.cpuPolicyName, 5*time.Second, machineInfo, cpuset.NewCPUSet(), testCase.nodeAllocatableReservation, sDir, topologymanager.NewFakeManager())
 			if testCase.expectedError != nil {
 				if !strings.Contains(err.Error(), testCase.expectedError.Error()) {
 					t.Errorf("Unexpected error message. Have: %s wants %s", err.Error(), testCase.expectedError.Error())

diff --git a/pkg/kubelet/cm/cpumanager/topology/BUILD b/pkg/kubelet/cm/cpumanager/topology/BUILD
@@ -33,8 +33,5 @@ go_test(
     name = "go_default_test",
     srcs = ["topology_test.go"],
     embed = [":go_default_library"],
-    deps = [
-        "//pkg/kubelet/cm/cpuset:go_default_library",
-        "//vendor/github.com/google/cadvisor/info/v1:go_default_library",
-    ],
+    deps = ["//vendor/github.com/google/cadvisor/info/v1:go_default_library"],
 )
diff --git a/pkg/kubelet/cm/cpumanager/topology/topology.go b/pkg/kubelet/cm/cpumanager/topology/topology.go
@@ -18,8 +18,6 @@ package topology
 
 import (
 	"fmt"
-	"io/ioutil"
-	"strings"
 
 	cadvisorapi "github.com/google/cadvisor/info/v1"
 	"k8s.io/klog/v2"
@@ -218,34 +216,28 @@ func (d CPUDetails) CPUsInCores(ids ...int) cpuset.CPUSet {
 }
 
 // Discover returns CPUTopology based on cadvisor node info
-func Discover(machineInfo *cadvisorapi.MachineInfo, numaNodeInfo NUMANodeInfo) (*CPUTopology, error) {
+func Discover(machineInfo *cadvisorapi.MachineInfo) (*CPUTopology, error) {
 	if machineInfo.NumCores == 0 {
 		return nil, fmt.Errorf("could not detect number of cpus")
 	}
 
 	CPUDetails := CPUDetails{}
 	numPhysicalCores := 0
 
-	for _, socket := range machineInfo.Topology {
-		numPhysicalCores += len(socket.Cores)
-		for _, core := range socket.Cores {
+	for _, node := range machineInfo.Topology {
+		numPhysicalCores += len(node.Cores)
+		for _, core := range node.Cores {
 			if coreID, err := getUniqueCoreID(core.Threads); err == nil {
 				for _, cpu := range core.Threads {
-					numaNodeID := 0
-					for id, cset := range numaNodeInfo {
-						if cset.Contains(cpu) {
-							numaNodeID = id
-						}
-					}
 					CPUDetails[cpu] = CPUInfo{
 						CoreID:     coreID,
-						SocketID:   socket.Id,
-						NUMANodeID: numaNodeID,
+						SocketID:   core.SocketID,
+						NUMANodeID: node.Id,
 					}
 				}
 			} else {
 				klog.Errorf("could not get unique coreID for socket: %d core %d threads: %v",
-					socket.Id, core.Id, core.Threads)
+					core.SocketID, core.Id, core.Threads)
 				return nil, err
 			}
 		}
@@ -280,49 +272,3 @@ func getUniqueCoreID(threads []int) (coreID int, err error) {
 
 	return min, nil
 }
-
-// GetNUMANodeInfo uses sysfs to return a map of NUMANode id to the list of
-// CPUs associated with that NUMANode.
-//
-// TODO: This is a temporary workaround until cadvisor provides this
-// information directly in machineInfo. We should remove this once this
-// information is available from cadvisor.
-func GetNUMANodeInfo() (NUMANodeInfo, error) {
-	// Get the possible NUMA nodes on this machine. If reading this file
-	// is not possible, this is not an error. Instead, we just return a
-	// nil NUMANodeInfo, indicating that no NUMA information is available
-	// on this machine. This should implicitly be interpreted as having a
-	// single NUMA node with id 0 for all CPUs.
-	nodelist, err := ioutil.ReadFile("/sys/devices/system/node/online")
-	if err != nil {
-		return nil, nil
-	}
-
-	// Parse the nodelist into a set of Node IDs
-	nodes, err := cpuset.Parse(strings.TrimSpace(string(nodelist)))
-	if err != nil {
-		return nil, err
-	}
-
-	info := make(NUMANodeInfo)
-
-	// For each node...
-	for _, node := range nodes.ToSlice() {
-		// Read the 'cpulist' of the NUMA node from sysfs.
-		path := fmt.Sprintf("/sys/devices/system/node/node%d/cpulist", node)
-		cpulist, err := ioutil.ReadFile(path)
-		if err != nil {
-			return nil, err
-		}
-
-		// Convert the 'cpulist' into a set of CPUs.
-		cpus, err := cpuset.Parse(strings.TrimSpace(string(cpulist)))
-		if err != nil {
-			return nil, err
-		}
-
-		info[node] = cpus
-	}
-
-	return info, nil
-}
diff --git a/pkg/kubelet/cm/cpumanager/topology/topology_test.go b/pkg/kubelet/cm/cpumanager/topology/topology_test.go
@@ -21,26 +21,23 @@ import (
 	"testing"
 
 	cadvisorapi "github.com/google/cadvisor/info/v1"
-	"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
 )
 
 func Test_Discover(t *testing.T) {
 
 	tests := []struct {
-		name         string
-		machineInfo  cadvisorapi.MachineInfo
-		numaNodeInfo NUMANodeInfo
-		want         *CPUTopology
-		wantErr      bool
+		name        string
+		machineInfo cadvisorapi.MachineInfo
+		want        *CPUTopology
+		wantErr     bool
 	}{
 		{
 			name: "FailNumCores",
 			machineInfo: cadvisorapi.MachineInfo{
 				NumCores: 0,
 			},
-			numaNodeInfo: NUMANodeInfo{},
-			want:         &CPUTopology{},
-			wantErr:      true,
+			want:    &CPUTopology{},
+			wantErr: true,
 		},
 		{
 			name: "OneSocketHT",
@@ -49,17 +46,14 @@ func Test_Discover(t *testing.T) {
 				Topology: []cadvisorapi.Node{
 					{Id: 0,
 						Cores: []cadvisorapi.Core{
-							{Id: 0, Threads: []int{0, 4}},
-							{Id: 1, Threads: []int{1, 5}},
-							{Id: 2, Threads: []int{2, 6}},
-							{Id: 3, Threads: []int{3, 7}},
+							{SocketID: 0, Id: 0, Threads: []int{0, 4}},
+							{SocketID: 0, Id: 1, Threads: []int{1, 5}},
+							{SocketID: 0, Id: 2, Threads: []int{2, 6}},
+							{SocketID: 0, Id: 3, Threads: []int{3, 7}},
 						},
 					},
 				},
 			},
-			numaNodeInfo: NUMANodeInfo{
-				0: cpuset.NewCPUSet(0, 1, 2, 3, 4, 5, 6, 7),
-			},
 			want: &CPUTopology{
 				NumCPUs:    8,
 				NumSockets: 1,
@@ -84,22 +78,18 @@ func Test_Discover(t *testing.T) {
 				Topology: []cadvisorapi.Node{
 					{Id: 0,
 						Cores: []cadvisorapi.Core{
-							{Id: 0, Threads: []int{0}},
-							{Id: 2, Threads: []int{2}},
+							{SocketID: 0, Id: 0, Threads: []int{0}},
+							{SocketID: 0, Id: 2, Threads: []int{2}},
 						},
 					},
 					{Id: 1,
 						Cores: []cadvisorapi.Core{
-							{Id: 1, Threads: []int{1}},
-							{Id: 3, Threads: []int{3}},
+							{SocketID: 1, Id: 1, Threads: []int{1}},
+							{SocketID: 1, Id: 3, Threads: []int{3}},
 						},
 					},
 				},
 			},
-			numaNodeInfo: NUMANodeInfo{
-				0: cpuset.NewCPUSet(0, 2),
-				1: cpuset.NewCPUSet(1, 3),
-			},
 			want: &CPUTopology{
 				NumCPUs:    4,
 				NumSockets: 2,
@@ -120,24 +110,20 @@ func Test_Discover(t *testing.T) {
 				Topology: []cadvisorapi.Node{
 					{Id: 0,
 						Cores: []cadvisorapi.Core{
-							{Id: 0, Threads: []int{0, 6}},
-							{Id: 1, Threads: []int{1, 7}},
-							{Id: 2, Threads: []int{2, 8}},
+							{SocketID: 0, Id: 0, Threads: []int{0, 6}},
+							{SocketID: 0, Id: 1, Threads: []int{1, 7}},
+							{SocketID: 0, Id: 2, Threads: []int{2, 8}},
 						},
 					},
 					{Id: 1,
 						Cores: []cadvisorapi.Core{
-							{Id: 0, Threads: []int{3, 9}},
-							{Id: 1, Threads: []int{4, 10}},
-							{Id: 2, Threads: []int{5, 11}},
+							{SocketID: 1, Id: 0, Threads: []int{3, 9}},
+							{SocketID: 1, Id: 1, Threads: []int{4, 10}},
+							{SocketID: 1, Id: 2, Threads: []int{5, 11}},
 						},
 					},
 				},
 			},
-			numaNodeInfo: NUMANodeInfo{
-				0: cpuset.NewCPUSet(0, 6, 1, 7, 2, 8),
-				1: cpuset.NewCPUSet(3, 9, 4, 10, 5, 11),
-			},
 			want: &CPUTopology{
 				NumCPUs:    12,
 				NumSockets: 2,
@@ -166,17 +152,16 @@ func Test_Discover(t *testing.T) {
 				Topology: []cadvisorapi.Node{
 					{Id: 0,
 						Cores: []cadvisorapi.Core{
-							{Id: 0, Threads: []int{0, 4}},
-							{Id: 1, Threads: []int{1, 5}},
-							{Id: 2, Threads: []int{2, 2}}, // Wrong case - should fail here
-							{Id: 3, Threads: []int{3, 7}},
+							{SocketID: 0, Id: 0, Threads: []int{0, 4}},
+							{SocketID: 0, Id: 1, Threads: []int{1, 5}},
+							{SocketID: 0, Id: 2, Threads: []int{2, 2}}, // Wrong case - should fail here
+							{SocketID: 0, Id: 3, Threads: []int{3, 7}},
 						},
 					},
 				},
 			},
-			numaNodeInfo: NUMANodeInfo{},
-			want:         &CPUTopology{},
-			wantErr:      true,
+			want:    &CPUTopology{},
+			wantErr: true,
 		},
 		{
 			name: "OneSocketHT fail",
@@ -185,22 +170,21 @@ func Test_Discover(t *testing.T) {
 				Topology: []cadvisorapi.Node{
 					{Id: 0,
 						Cores: []cadvisorapi.Core{
-							{Id: 0, Threads: []int{0, 4}},
-							{Id: 1, Threads: []int{1, 5}},
-							{Id: 2, Threads: []int{2, 6}},
-							{Id: 3, Threads: []int{}}, // Wrong case - should fail here
+							{SocketID: 0, Id: 0, Threads: []int{0, 4}},
+							{SocketID: 0, Id: 1, Threads: []int{1, 5}},
+							{SocketID: 0, Id: 2, Threads: []int{2, 6}},
+							{SocketID: 0, Id: 3, Threads: []int{}}, // Wrong case - should fail here
 						},
 					},
 				},
 			},
-			numaNodeInfo: NUMANodeInfo{},
-			want:         &CPUTopology{},
-			wantErr:      true,
+			want:    &CPUTopology{},
+			wantErr: true,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := Discover(&tt.machineInfo, tt.numaNodeInfo)
+			got, err := Discover(&tt.machineInfo)
 			if err != nil {
 				if tt.wantErr {
 					t.Logf("Discover() expected error = %v", err)