diff --git a/pkg/controller/volume/attachdetach/attach_detach_controller.go b/pkg/controller/volume/attachdetach/attach_detach_controller.go index fb608ac481a01..2f078a733e8fa 100644 --- a/pkg/controller/volume/attachdetach/attach_detach_controller.go +++ b/pkg/controller/volume/attachdetach/attach_detach_controller.go @@ -391,7 +391,11 @@ func (adc *attachDetachController) populateActualStateOfWorld() error { adc.addNodeToDswp(node, types.NodeName(node.Name)) } } - return nil + err = adc.processVolumeAttachments() + if err != nil { + klog.Errorf("Failed to process volume attachments: %v", err) + } + return err } func (adc *attachDetachController) getNodeVolumeDevicePath( @@ -461,7 +465,12 @@ func (adc *attachDetachController) populateDesiredStateOfWorld() error { err) continue } - if adc.actualStateOfWorld.IsVolumeAttachedToNode(volumeName, nodeName) { + attachState := adc.actualStateOfWorld.GetAttachState(volumeName, nodeName) + if attachState == cache.AttachStateAttached { + klog.V(10).Infof("Volume %q is attached to node %q. Marking as attached in ActualStateOfWorld", + volumeName, + nodeName, + ) devicePath, err := adc.getNodeVolumeDevicePath(volumeName, nodeName) if err != nil { klog.Errorf("Failed to find device path: %v", err) @@ -679,6 +688,67 @@ func (adc *attachDetachController) processVolumesInUse( } } +// Process Volume-Attachment objects. +// Should be called only after populating attached volumes in the ASW. +// For each VA object, this function checks if its present in the ASW. +// If not, adds the volume to ASW as an "uncertain" attachment. +// In the reconciler, the logic checks if the volume is present in the DSW; +// if yes, the reconciler will attempt attach on the volume; +// if not (could be a dangling attachment), the reconciler will detach this volume. +func (adc *attachDetachController) processVolumeAttachments() error { + vas, err := adc.volumeAttachmentLister.List(labels.Everything()) + if err != nil { + klog.Errorf("failed to list VolumeAttachment objects: %v", err) + return err + } + for _, va := range vas { + nodeName := types.NodeName(va.Spec.NodeName) + pvName := va.Spec.Source.PersistentVolumeName + if pvName == nil { + // Currently VA objects are created for CSI volumes only. nil pvName is unexpected, generate a warning + klog.Warningf("Skipping the va as its pvName is nil, va.Name: %q, nodeName: %q", + va.Name, nodeName) + continue + } + pv, err := adc.pvLister.Get(*pvName) + if err != nil { + klog.Errorf("Unable to lookup pv object for: %q, err: %v", *pvName, err) + continue + } + volumeSpec := volume.NewSpecFromPersistentVolume(pv, false) + plugin, err := adc.volumePluginMgr.FindAttachablePluginBySpec(volumeSpec) + if err != nil || plugin == nil { + // Currently VA objects are created for CSI volumes only. nil plugin is unexpected, generate a warning + klog.Warningf( + "Skipping processing the volume %q on nodeName: %q, no attacher interface found. err=%v", + *pvName, + nodeName, + err) + continue + } + volumeName, err := volumeutil.GetUniqueVolumeNameFromSpec(plugin, volumeSpec) + if err != nil { + klog.Errorf( + "Failed to find unique name for volume:%q, va.Name:%q, nodeName:%q: %v", + *pvName, + va.Name, + nodeName, + err) + continue + } + attachState := adc.actualStateOfWorld.GetAttachState(volumeName, nodeName) + if attachState == cache.AttachStateDetached { + klog.V(1).Infof("Marking volume attachment as uncertain as volume:%q (%q) is not attached (%v)", + volumeName, nodeName, attachState) + err = adc.actualStateOfWorld.MarkVolumeAsUncertain(volumeName, volumeSpec, nodeName) + if err != nil { + klog.Errorf("MarkVolumeAsUncertain fail to add the volume %q (%q) to ASW. err: %s", volumeName, nodeName, err) + } + } + } + return nil +} + var _ volume.VolumeHost = &attachDetachController{} var _ volume.AttachDetachVolumeHost = &attachDetachController{} diff --git a/pkg/controller/volume/attachdetach/attach_detach_controller_test.go b/pkg/controller/volume/attachdetach/attach_detach_controller_test.go index a321478d9c52d..36845402e8f3d 100644 --- a/pkg/controller/volume/attachdetach/attach_detach_controller_test.go +++ b/pkg/controller/volume/attachdetach/attach_detach_controller_test.go @@ -72,18 +72,21 @@ func Test_AttachDetachControllerStateOfWolrdPopulators_Positive(t *testing.T) { nodeInformer := informerFactory.Core().V1().Nodes() pvcInformer := informerFactory.Core().V1().PersistentVolumeClaims() pvInformer := informerFactory.Core().V1().PersistentVolumes() + volumeAttachmentInformer := informerFactory.Storage().V1().VolumeAttachments() adc := &attachDetachController{ - kubeClient: fakeKubeClient, - pvcLister: pvcInformer.Lister(), - pvcsSynced: pvcInformer.Informer().HasSynced, - pvLister: pvInformer.Lister(), - pvsSynced: pvInformer.Informer().HasSynced, - podLister: podInformer.Lister(), - podsSynced: podInformer.Informer().HasSynced, - nodeLister: nodeInformer.Lister(), - nodesSynced: nodeInformer.Informer().HasSynced, - cloud: nil, + kubeClient: fakeKubeClient, + pvcLister: pvcInformer.Lister(), + pvcsSynced: pvcInformer.Informer().HasSynced, + pvLister: pvInformer.Lister(), + pvsSynced: pvInformer.Informer().HasSynced, + podLister: podInformer.Lister(), + podsSynced: podInformer.Informer().HasSynced, + nodeLister: nodeInformer.Lister(), + nodesSynced: nodeInformer.Informer().HasSynced, + volumeAttachmentLister: volumeAttachmentInformer.Lister(), + volumeAttachmentSynced: volumeAttachmentInformer.Informer().HasSynced, + cloud: nil, } // Act @@ -116,8 +119,8 @@ func Test_AttachDetachControllerStateOfWolrdPopulators_Positive(t *testing.T) { for _, node := range nodes { nodeName := types.NodeName(node.Name) for _, attachedVolume := range node.Status.VolumesAttached { - found := adc.actualStateOfWorld.IsVolumeAttachedToNode(attachedVolume.Name, nodeName) - if !found { + attachedState := adc.actualStateOfWorld.GetAttachState(attachedVolume.Name, nodeName) + if attachedState != cache.AttachStateAttached { t.Fatalf("Run failed with error. Node %s, volume %s not found", nodeName, attachedVolume.Name) } } @@ -335,3 +338,210 @@ func attachDetachRecoveryTestCase(t *testing.T, extraPods1 []*v1.Pod, extraPods2 } } + +type vaTest struct { + testName string + volName string + podName string + podNodeName string + pvName string + vaName string + vaNodeName string + vaAttachStatus bool + expected_attaches map[string][]string + expected_detaches map[string][]string +} + +func Test_ADC_VolumeAttachmentRecovery(t *testing.T) { + for _, tc := range []vaTest{ + { // pod is scheduled + testName: "Scheduled pod", + volName: "vol1", + podName: "pod1", + podNodeName: "mynode-1", + pvName: "pv1", + vaName: "va1", + vaNodeName: "mynode-1", + vaAttachStatus: false, + expected_attaches: map[string][]string{"mynode-1": {"vol1"}}, + expected_detaches: map[string][]string{}, + }, + { // pod is deleted, attach status:true, verify dangling volume is detached + testName: "VA status is attached", + volName: "vol1", + pvName: "pv1", + vaName: "va1", + vaNodeName: "mynode-1", + vaAttachStatus: true, + expected_attaches: map[string][]string{}, + expected_detaches: map[string][]string{"mynode-1": {"vol1"}}, + }, + { // pod is deleted, attach status:false, verify dangling volume is detached + testName: "VA status is unattached", + volName: "vol1", + pvName: "pv1", + vaName: "va1", + vaNodeName: "mynode-1", + vaAttachStatus: false, + expected_attaches: map[string][]string{}, + expected_detaches: map[string][]string{"mynode-1": {"vol1"}}, + }, + } { + t.Run(tc.testName, func(t *testing.T) { + volumeAttachmentRecoveryTestCase(t, tc) + }) + } +} + +func volumeAttachmentRecoveryTestCase(t *testing.T, tc vaTest) { + fakeKubeClient := controllervolumetesting.CreateTestClient() + informerFactory := informers.NewSharedInformerFactory(fakeKubeClient, time.Second*1) + plugins := controllervolumetesting.CreateTestPlugin() + nodeInformer := informerFactory.Core().V1().Nodes().Informer() + podInformer := informerFactory.Core().V1().Pods().Informer() + pvInformer := informerFactory.Core().V1().PersistentVolumes().Informer() + vaInformer := informerFactory.Storage().V1().VolumeAttachments().Informer() + + // Create the controller + adcObj, err := NewAttachDetachController( + fakeKubeClient, + informerFactory.Core().V1().Pods(), + informerFactory.Core().V1().Nodes(), + informerFactory.Core().V1().PersistentVolumeClaims(), + informerFactory.Core().V1().PersistentVolumes(), + informerFactory.Storage().V1().CSINodes(), + informerFactory.Storage().V1().CSIDrivers(), + informerFactory.Storage().V1().VolumeAttachments(), + nil, /* cloud */ + plugins, + nil, /* prober */ + false, + 1*time.Second, + DefaultTimerConfig, + nil, /* filteredDialOptions */ + ) + if err != nil { + t.Fatalf("NewAttachDetachController failed with error. Expected: Actual: <%v>", err) + } + adc := adcObj.(*attachDetachController) + + // Add existing objects (created by testplugin) to the respective informers + pods, err := fakeKubeClient.CoreV1().Pods(v1.NamespaceAll).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + t.Fatalf("Run failed with error. Expected: Actual: %v", err) + } + for _, pod := range pods.Items { + podToAdd := pod + podInformer.GetIndexer().Add(&podToAdd) + } + nodes, err := fakeKubeClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + t.Fatalf("Run failed with error. Expected: Actual: %v", err) + } + for _, node := range nodes.Items { + nodeToAdd := node + nodeInformer.GetIndexer().Add(&nodeToAdd) + } + + // Create and add objects requested by the test + if tc.podName != "" { + newPod := controllervolumetesting.NewPodWithVolume(tc.podName, tc.volName, tc.podNodeName) + _, err = adc.kubeClient.CoreV1().Pods(newPod.ObjectMeta.Namespace).Create(context.TODO(), newPod, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Run failed with error. Failed to create a new pod: <%v>", err) + } + podInformer.GetIndexer().Add(newPod) + } + if tc.pvName != "" { + newPv := controllervolumetesting.NewPV(tc.pvName, tc.volName) + _, err = adc.kubeClient.CoreV1().PersistentVolumes().Create(context.TODO(), newPv, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Run failed with error. Failed to create a new pv: <%v>", err) + } + pvInformer.GetIndexer().Add(newPv) + } + if tc.vaName != "" { + newVa := controllervolumetesting.NewVolumeAttachment("va1", "pv1", "mynode-1", false) + _, err = adc.kubeClient.StorageV1().VolumeAttachments().Create(context.TODO(), newVa, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Run failed with error. Failed to create a new volumeAttachment: <%v>", err) + } + vaInformer.GetIndexer().Add(newVa) + } + + // Makesure the informer cache is synced + stopCh := make(chan struct{}) + informerFactory.Start(stopCh) + + if !kcache.WaitForNamedCacheSync("attach detach", stopCh, + informerFactory.Core().V1().Pods().Informer().HasSynced, + informerFactory.Core().V1().Nodes().Informer().HasSynced, + informerFactory.Core().V1().PersistentVolumes().Informer().HasSynced, + informerFactory.Storage().V1().VolumeAttachments().Informer().HasSynced) { + t.Fatalf("Error waiting for the informer caches to sync") + } + + // Populate ASW + err = adc.populateActualStateOfWorld() + if err != nil { + t.Fatalf("Run failed with error. Expected: Actual: <%v>", err) + } + + // Populate DSW + err = adc.populateDesiredStateOfWorld() + if err != nil { + t.Fatalf("Run failed with error. Expected: Actual: %v", err) + } + // Run reconciler and DSW populator loops + go adc.reconciler.Run(stopCh) + go adc.desiredStateOfWorldPopulator.Run(stopCh) + defer close(stopCh) + + // Verify if expected attaches and detaches have happened + testPlugin := plugins[0].(*controllervolumetesting.TestPlugin) + for tries := 0; tries <= 10; tries++ { // wait & try few times before failing the test + expected_op_map := tc.expected_attaches + plugin_map := testPlugin.GetAttachedVolumes() + verify_op := "attach" + volFound, nodeFound := false, false + for i := 0; i <= 1; i++ { // verify attaches and detaches + if i == 1 { + expected_op_map = tc.expected_detaches + plugin_map = testPlugin.GetDetachedVolumes() + verify_op = "detach" + } + // Verify every (node, volume) in the expected_op_map is in the + // plugin_map + for expectedNode, expectedVolumeList := range expected_op_map { + var volumeList []string + volumeList, nodeFound = plugin_map[expectedNode] + if !nodeFound && tries == 10 { + t.Fatalf("Expected node not found, node:%v, op: %v, tries: %d", + expectedNode, verify_op, tries) + } + for _, expectedVolume := range expectedVolumeList { + volFound = false + for _, volume := range volumeList { + if expectedVolume == volume { + volFound = true + break + } + } + if !volFound && tries == 10 { + t.Fatalf("Expected %v operation not found, node:%v, volume: %v, tries: %d", + verify_op, expectedNode, expectedVolume, tries) + } + } + } + } + if nodeFound && volFound { + break + } + time.Sleep(time.Second * 1) + } + + if testPlugin.GetErrorEncountered() { + t.Fatalf("Fatal error encountered in the testing volume plugin") + } + +} diff --git a/pkg/controller/volume/attachdetach/cache/actual_state_of_world.go b/pkg/controller/volume/attachdetach/cache/actual_state_of_world.go index 6d597cf875137..9513d09a60341 100644 --- a/pkg/controller/volume/attachdetach/cache/actual_state_of_world.go +++ b/pkg/controller/volume/attachdetach/cache/actual_state_of_world.go @@ -96,10 +96,13 @@ type ActualStateOfWorld interface { // nodes, the volume is also deleted. DeleteVolumeNode(volumeName v1.UniqueVolumeName, nodeName types.NodeName) - // IsVolumeAttachedToNode returns true if the specified volume/node combo exists - // in the underlying store indicating the specified volume is attached to - // the specified node. - IsVolumeAttachedToNode(volumeName v1.UniqueVolumeName, nodeName types.NodeName) bool + // GetAttachState returns the attach state for the given volume-node + // combination. + // Returns AttachStateAttached if the specified volume/node combo exists in + // the underlying store indicating the specified volume is attached to the + // specified node, AttachStateDetached if the combo does not exist, or + // AttachStateUncertain if the attached state is marked as uncertain. + GetAttachState(volumeName v1.UniqueVolumeName, nodeName types.NodeName) AttachState // GetAttachedVolumes generates and returns a list of volumes/node pairs // reflecting which volumes might attached to which nodes based on the @@ -153,6 +156,31 @@ type AttachedVolume struct { DetachRequestedTime time.Time } +// AttachState represents the attach state of a volume to a node known to the +// Actual State of World. +// This type is used as external representation of attach state (specifically +// as the return type of GetAttachState only); the state is represented +// differently in the internal cache implementation. +type AttachState int + +const ( + // AttachStateAttached represents the state in which the volume is attached to + // the node. + AttachStateAttached AttachState = iota + + // AttachStateUncertain represents the state in which the Actual State of World + // does not know whether the volume is attached to the node. + AttachStateUncertain + + // AttachStateDetached represents the state in which the volume is not + // attached to the node. + AttachStateDetached +) + +func (s AttachState) String() string { + return []string{"Attached", "Uncertain", "Detached"}[s] +} + // NewActualStateOfWorld returns a new instance of ActualStateOfWorld. func NewActualStateOfWorld(volumePluginMgr *volume.VolumePluginMgr) ActualStateOfWorld { return &actualStateOfWorld{ @@ -530,19 +558,22 @@ func (asw *actualStateOfWorld) DeleteVolumeNode( asw.removeVolumeFromReportAsAttached(volumeName, nodeName) } -func (asw *actualStateOfWorld) IsVolumeAttachedToNode( - volumeName v1.UniqueVolumeName, nodeName types.NodeName) bool { +func (asw *actualStateOfWorld) GetAttachState( + volumeName v1.UniqueVolumeName, nodeName types.NodeName) AttachState { asw.RLock() defer asw.RUnlock() volumeObj, volumeExists := asw.attachedVolumes[volumeName] if volumeExists { if node, nodeExists := volumeObj.nodesAttachedTo[nodeName]; nodeExists { - return node.attachedConfirmed + if node.attachedConfirmed { + return AttachStateAttached + } + return AttachStateUncertain } } - return false + return AttachStateDetached } func (asw *actualStateOfWorld) GetAttachedVolumes() []AttachedVolume { diff --git a/pkg/controller/volume/attachdetach/cache/actual_state_of_world_test.go b/pkg/controller/volume/attachdetach/cache/actual_state_of_world_test.go index 4a1a3e0b283d7..71406909a9a0a 100644 --- a/pkg/controller/volume/attachdetach/cache/actual_state_of_world_test.go +++ b/pkg/controller/volume/attachdetach/cache/actual_state_of_world_test.go @@ -20,7 +20,7 @@ import ( "testing" "time" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" controllervolumetesting "k8s.io/kubernetes/pkg/controller/volume/attachdetach/testing" volumetesting "k8s.io/kubernetes/pkg/volume/testing" @@ -47,9 +47,9 @@ func Test_AddVolumeNode_Positive_NewVolumeNewNode(t *testing.T) { t.Fatalf("AddVolumeNode failed. Expected: Actual: <%v>", err) } - volumeNodeComboExists := asw.IsVolumeAttachedToNode(generatedVolumeName, nodeName) - if !volumeNodeComboExists { - t.Fatalf("%q/%q volume/node combo does not exist, it should.", generatedVolumeName, nodeName) + volumeNodeComboState := asw.GetAttachState(generatedVolumeName, nodeName) + if volumeNodeComboState != AttachStateAttached { + t.Fatalf("%q/%q volume/node combo is marked %q; expected 'Attached'.", generatedVolumeName, nodeName, volumeNodeComboState) } attachedVolumes := asw.GetAttachedVolumes() @@ -82,9 +82,9 @@ func Test_AddVolumeNode_Positive_NewVolumeNewNodeWithFalseAttached(t *testing.T) t.Fatalf("AddVolumeNode failed. Expected: Actual: <%v>", err) } - volumeNodeComboExists := asw.IsVolumeAttachedToNode(generatedVolumeName, nodeName) - if volumeNodeComboExists { - t.Fatalf("%q/%q volume/node combo does exist, it should not.", generatedVolumeName, nodeName) + volumeNodeComboState := asw.GetAttachState(generatedVolumeName, nodeName) + if volumeNodeComboState != AttachStateUncertain { + t.Fatalf("%q/%q volume/node combo is marked %q, expected 'Uncertain'.", generatedVolumeName, nodeName, volumeNodeComboState) } allVolumes := asw.GetAttachedVolumes() @@ -131,9 +131,9 @@ func Test_AddVolumeNode_Positive_NewVolumeNewNodeWithFalseAttached(t *testing.T) generatedVolumeName2) } - volumeNodeComboExists = asw.IsVolumeAttachedToNode(generatedVolumeName, nodeName) - if !volumeNodeComboExists { - t.Fatalf("%q/%q combo does not exist, it should.", generatedVolumeName, nodeName) + volumeNodeComboState = asw.GetAttachState(generatedVolumeName, nodeName) + if volumeNodeComboState != AttachStateAttached { + t.Fatalf("%q/%q volume/node combo is marked %q; expected 'Attached'.", generatedVolumeName, nodeName, volumeNodeComboState) } attachedVolumes := asw.GetAttachedVolumes() @@ -182,9 +182,9 @@ func Test_AddVolumeNode_Positive_NewVolumeTwoNodesWithFalseAttached(t *testing.T t.Fatalf("AddVolumeNode failed. Expected: Actual: <%v>", err) } - volumeNodeComboExists := asw.IsVolumeAttachedToNode(generatedVolumeName, node1Name) - if volumeNodeComboExists { - t.Fatalf("%q/%q volume/node combo does exist, it should not.", generatedVolumeName, node1Name) + volumeNodeComboState := asw.GetAttachState(generatedVolumeName, node1Name) + if volumeNodeComboState != AttachStateUncertain { + t.Fatalf("%q/%q volume/node combo is marked %q, expected 'Uncertain'.", generatedVolumeName, node1Name, volumeNodeComboState) } generatedVolumeName2, add2Err := asw.AddVolumeNode(volumeName, volumeSpec, node2Name, devicePath, true) @@ -201,9 +201,9 @@ func Test_AddVolumeNode_Positive_NewVolumeTwoNodesWithFalseAttached(t *testing.T generatedVolumeName2) } - volumeNodeComboExists = asw.IsVolumeAttachedToNode(generatedVolumeName, node2Name) - if !volumeNodeComboExists { - t.Fatalf("%q/%q combo does not exist, it should.", generatedVolumeName, node2Name) + volumeNodeComboState = asw.GetAttachState(generatedVolumeName, node2Name) + if volumeNodeComboState != AttachStateAttached { + t.Fatalf("%q/%q volume/node combo is marked %q; expected 'Attached'.", generatedVolumeName, node2Name, volumeNodeComboState) } attachedVolumes := asw.GetAttachedVolumes() @@ -268,14 +268,14 @@ func Test_AddVolumeNode_Positive_ExistingVolumeNewNode(t *testing.T) { generatedVolumeName2) } - volumeNode1ComboExists := asw.IsVolumeAttachedToNode(generatedVolumeName1, node1Name) - if !volumeNode1ComboExists { - t.Fatalf("%q/%q volume/node combo does not exist, it should.", generatedVolumeName1, node1Name) + volumeNode1ComboState := asw.GetAttachState(generatedVolumeName1, node1Name) + if volumeNode1ComboState != AttachStateAttached { + t.Fatalf("%q/%q volume/node combo is marked %q; expected 'Attached'.", generatedVolumeName1, node1Name, volumeNode1ComboState) } - volumeNode2ComboExists := asw.IsVolumeAttachedToNode(generatedVolumeName1, node2Name) - if !volumeNode2ComboExists { - t.Fatalf("%q/%q volume/node combo does not exist, it should.", generatedVolumeName1, node2Name) + volumeNode2ComboState := asw.GetAttachState(generatedVolumeName1, node2Name) + if volumeNode2ComboState != AttachStateAttached { + t.Fatalf("%q/%q volume/node combo is marked %q; expected 'Attached'.", generatedVolumeName1, node2Name, volumeNode2ComboState) } attachedVolumes := asw.GetAttachedVolumes() @@ -317,9 +317,9 @@ func Test_AddVolumeNode_Positive_ExistingVolumeExistingNode(t *testing.T) { generatedVolumeName2) } - volumeNodeComboExists := asw.IsVolumeAttachedToNode(generatedVolumeName1, nodeName) - if !volumeNodeComboExists { - t.Fatalf("%q/%q volume/node combo does not exist, it should.", generatedVolumeName1, nodeName) + volumeNodeComboState := asw.GetAttachState(generatedVolumeName1, nodeName) + if volumeNodeComboState != AttachStateAttached { + t.Fatalf("%q/%q volume/node combo is marked %q; expected 'Attached'.", generatedVolumeName1, nodeName, volumeNodeComboState) } attachedVolumes := asw.GetAttachedVolumes() @@ -350,9 +350,9 @@ func Test_DeleteVolumeNode_Positive_VolumeExistsNodeExists(t *testing.T) { asw.DeleteVolumeNode(generatedVolumeName, nodeName) // Assert - volumeNodeComboExists := asw.IsVolumeAttachedToNode(generatedVolumeName, nodeName) - if volumeNodeComboExists { - t.Fatalf("%q/%q volume/node combo exists, it should not.", generatedVolumeName, nodeName) + volumeNodeComboState := asw.GetAttachState(generatedVolumeName, nodeName) + if volumeNodeComboState != AttachStateDetached { + t.Fatalf("%q/%q volume/node combo is marked %q, expected 'Detached'.", generatedVolumeName, nodeName, volumeNodeComboState) } attachedVolumes := asw.GetAttachedVolumes() @@ -374,9 +374,9 @@ func Test_DeleteVolumeNode_Positive_VolumeDoesntExistNodeDoesntExist(t *testing. asw.DeleteVolumeNode(volumeName, nodeName) // Assert - volumeNodeComboExists := asw.IsVolumeAttachedToNode(volumeName, nodeName) - if volumeNodeComboExists { - t.Fatalf("%q/%q volume/node combo exists, it should not.", volumeName, nodeName) + volumeNodeComboState := asw.GetAttachState(volumeName, nodeName) + if volumeNodeComboState != AttachStateDetached { + t.Fatalf("%q/%q volume/node combo is marked %q, expected 'Detached'.", volumeName, nodeName, volumeNodeComboState) } attachedVolumes := asw.GetAttachedVolumes() @@ -417,14 +417,14 @@ func Test_DeleteVolumeNode_Positive_TwoNodesOneDeleted(t *testing.T) { asw.DeleteVolumeNode(generatedVolumeName1, node1Name) // Assert - volumeNodeComboExists := asw.IsVolumeAttachedToNode(generatedVolumeName1, node1Name) - if volumeNodeComboExists { - t.Fatalf("%q/%q volume/node combo exists, it should not.", generatedVolumeName1, node1Name) + volumeNodeComboState := asw.GetAttachState(generatedVolumeName1, node1Name) + if volumeNodeComboState != AttachStateDetached { + t.Fatalf("%q/%q volume/node combo is marked %q, expected 'Detached'.", generatedVolumeName1, node1Name, volumeNodeComboState) } - volumeNodeComboExists = asw.IsVolumeAttachedToNode(generatedVolumeName1, node2Name) - if !volumeNodeComboExists { - t.Fatalf("%q/%q volume/node combo does not exist, it should.", generatedVolumeName1, node2Name) + volumeNodeComboState = asw.GetAttachState(generatedVolumeName1, node2Name) + if volumeNodeComboState != AttachStateAttached { + t.Fatalf("%q/%q volume/node combo is marked %q; expected 'Attached'.", generatedVolumeName1, node2Name, volumeNodeComboState) } attachedVolumes := asw.GetAttachedVolumes() @@ -436,7 +436,7 @@ func Test_DeleteVolumeNode_Positive_TwoNodesOneDeleted(t *testing.T) { } // Populates data struct with one volume/node entry. -// Calls IsVolumeAttachedToNode() to verify entry. +// Calls GetAttachState() to verify entry. // Verifies the populated volume/node entry exists. func Test_VolumeNodeExists_Positive_VolumeExistsNodeExists(t *testing.T) { // Arrange @@ -452,11 +452,11 @@ func Test_VolumeNodeExists_Positive_VolumeExistsNodeExists(t *testing.T) { } // Act - volumeNodeComboExists := asw.IsVolumeAttachedToNode(generatedVolumeName, nodeName) + volumeNodeComboState := asw.GetAttachState(generatedVolumeName, nodeName) // Assert - if !volumeNodeComboExists { - t.Fatalf("%q/%q volume/node combo does not exist, it should.", generatedVolumeName, nodeName) + if volumeNodeComboState != AttachStateAttached { + t.Fatalf("%q/%q volume/node combo is marked %q; expected 'Attached'.", generatedVolumeName, nodeName, volumeNodeComboState) } attachedVolumes := asw.GetAttachedVolumes() @@ -468,7 +468,7 @@ func Test_VolumeNodeExists_Positive_VolumeExistsNodeExists(t *testing.T) { } // Populates data struct with one volume1/node1 entry. -// Calls IsVolumeAttachedToNode() with volume1/node2. +// Calls GetAttachState() with volume1/node2. // Verifies requested entry does not exist, but populated entry does. func Test_VolumeNodeExists_Positive_VolumeExistsNodeDoesntExist(t *testing.T) { // Arrange @@ -485,11 +485,11 @@ func Test_VolumeNodeExists_Positive_VolumeExistsNodeDoesntExist(t *testing.T) { } // Act - volumeNodeComboExists := asw.IsVolumeAttachedToNode(generatedVolumeName, node2Name) + volumeNodeComboState := asw.GetAttachState(generatedVolumeName, node2Name) // Assert - if volumeNodeComboExists { - t.Fatalf("%q/%q volume/node combo exists, it should not.", generatedVolumeName, node2Name) + if volumeNodeComboState != AttachStateDetached { + t.Fatalf("%q/%q volume/node combo is marked %q, expected 'Detached'.", generatedVolumeName, node2Name, volumeNodeComboState) } attachedVolumes := asw.GetAttachedVolumes() @@ -500,7 +500,7 @@ func Test_VolumeNodeExists_Positive_VolumeExistsNodeDoesntExist(t *testing.T) { verifyAttachedVolume(t, attachedVolumes, generatedVolumeName, string(volumeName), node1Name, devicePath, true /* expectedMountedByNode */, false /* expectNonZeroDetachRequestedTime */) } -// Calls IsVolumeAttachedToNode() on empty data struct. +// Calls GetAttachState() on empty data struct. // Verifies requested entry does not exist. func Test_VolumeNodeExists_Positive_VolumeAndNodeDontExist(t *testing.T) { // Arrange @@ -510,11 +510,11 @@ func Test_VolumeNodeExists_Positive_VolumeAndNodeDontExist(t *testing.T) { nodeName := types.NodeName("node-name") // Act - volumeNodeComboExists := asw.IsVolumeAttachedToNode(volumeName, nodeName) + volumeNodeComboState := asw.GetAttachState(volumeName, nodeName) // Assert - if volumeNodeComboExists { - t.Fatalf("%q/%q volume/node combo exists, it should not.", volumeName, nodeName) + if volumeNodeComboState != AttachStateDetached { + t.Fatalf("%q/%q volume/node combo is marked %q, expected 'Detached'.", volumeName, nodeName, volumeNodeComboState) } attachedVolumes := asw.GetAttachedVolumes() @@ -1374,6 +1374,79 @@ func Test_updateNodeStatusUpdateNeededError(t *testing.T) { } } +// Mark a volume as attached to a node. +// Verify GetAttachState returns AttachedState +// Verify GetAttachedVolumes return this volume +func Test_MarkVolumeAsAttached(t *testing.T) { + // Arrange + volumePluginMgr, _ := volumetesting.GetTestVolumePluginMgr(t) + asw := NewActualStateOfWorld(volumePluginMgr) + volumeName := v1.UniqueVolumeName("volume-name") + volumeSpec := controllervolumetesting.GetTestVolumeSpec(string(volumeName), volumeName) + + nodeName := types.NodeName("node-name") + devicePath := "fake/device/path" + + plugin, err := volumePluginMgr.FindAttachablePluginBySpec(volumeSpec) + if err != nil || plugin == nil { + t.Fatalf("Failed to get volume plugin from spec %v, %v", volumeSpec, err) + } + + // Act + err = asw.MarkVolumeAsAttached(volumeName, volumeSpec, nodeName, devicePath) + + // Assert + if err != nil { + t.Fatalf("MarkVolumeAsAttached failed. Expected: Actual: <%v>", err) + } + + volumeNodeComboState := asw.GetAttachState(volumeName, nodeName) + if volumeNodeComboState != AttachStateAttached { + t.Fatalf("asw says the volume: %q is not attached (%v) to node:%q, it should.", + volumeName, AttachStateAttached, nodeName) + } + attachedVolumes := asw.GetAttachedVolumes() + if len(attachedVolumes) != 1 { + t.Fatalf("len(attachedVolumes) Expected: <1> Actual: <%v>", len(attachedVolumes)) + } + verifyAttachedVolume(t, attachedVolumes, volumeName, string(volumeName), nodeName, devicePath, true /* expectedMountedByNode */, false /* expectNonZeroDetachRequestedTime */) +} + +// Mark a volume as attachment as uncertain. +// Verify GetAttachState returns UncertainState +// Verify GetAttachedVolumes return this volume +func Test_MarkVolumeAsUncertain(t *testing.T) { + // Arrange + volumePluginMgr, _ := volumetesting.GetTestVolumePluginMgr(t) + asw := NewActualStateOfWorld(volumePluginMgr) + volumeName := v1.UniqueVolumeName("volume-name") + volumeSpec := controllervolumetesting.GetTestVolumeSpec(string(volumeName), volumeName) + nodeName := types.NodeName("node-name") + + plugin, err := volumePluginMgr.FindAttachablePluginBySpec(volumeSpec) + if err != nil || plugin == nil { + t.Fatalf("Failed to get volume plugin from spec %v, %v", volumeSpec, err) + } + + // Act + err = asw.MarkVolumeAsUncertain(volumeName, volumeSpec, nodeName) + + // Assert + if err != nil { + t.Fatalf("MarkVolumeAsUncertain failed. Expected: Actual: <%v>", err) + } + volumeNodeComboState := asw.GetAttachState(volumeName, nodeName) + if volumeNodeComboState != AttachStateUncertain { + t.Fatalf("asw says the volume: %q is attached (%v) to node:%q, it should not.", + volumeName, volumeNodeComboState, nodeName) + } + attachedVolumes := asw.GetAttachedVolumes() + if len(attachedVolumes) != 1 { + t.Fatalf("len(attachedVolumes) Expected: <1> Actual: <%v>", len(attachedVolumes)) + } + verifyAttachedVolume(t, attachedVolumes, volumeName, string(volumeName), nodeName, "", true /* expectedMountedByNode */, false /* expectNonZeroDetachRequestedTime */) +} + func verifyAttachedVolume( t *testing.T, attachedVolumes []AttachedVolume, diff --git a/pkg/controller/volume/attachdetach/reconciler/reconciler.go b/pkg/controller/volume/attachdetach/reconciler/reconciler.go index a010630d77e5e..247b49ca64063 100644 --- a/pkg/controller/volume/attachdetach/reconciler/reconciler.go +++ b/pkg/controller/volume/attachdetach/reconciler/reconciler.go @@ -142,6 +142,7 @@ func (rc *reconciler) reconcile() { for _, attachedVolume := range rc.actualStateOfWorld.GetAttachedVolumes() { if !rc.desiredStateOfWorld.VolumeExists( attachedVolume.VolumeName, attachedVolume.NodeName) { + // Check whether there already exist an operation pending, and don't even // try to start an operation if there is already one running. // This check must be done before we do any other checks, as otherwise the other checks @@ -161,6 +162,21 @@ func (rc *reconciler) reconcile() { } } + // Because the detach operation updates the ActualStateOfWorld before + // marking itself complete, it's possible for the volume to be removed + // from the ActualStateOfWorld between the GetAttachedVolumes() check + // and the IsOperationPending() check above. + // Check the ActualStateOfWorld again to avoid issuing an unnecessary + // detach. + // See https://github.com/kubernetes/kubernetes/issues/93902 + attachState := rc.actualStateOfWorld.GetAttachState(attachedVolume.VolumeName, attachedVolume.NodeName) + if attachState == cache.AttachStateDetached { + if klog.V(5).Enabled() { + klog.Infof(attachedVolume.GenerateMsgDetailed("Volume detached--skipping", "")) + } + continue + } + // Set the detach request time elapsedTime, err := rc.actualStateOfWorld.SetDetachRequestTime(attachedVolume.VolumeName, attachedVolume.NodeName) if err != nil { @@ -226,17 +242,7 @@ func (rc *reconciler) reconcile() { func (rc *reconciler) attachDesiredVolumes() { // Ensure volumes that should be attached are attached. for _, volumeToAttach := range rc.desiredStateOfWorld.GetVolumesToAttach() { - if rc.actualStateOfWorld.IsVolumeAttachedToNode(volumeToAttach.VolumeName, volumeToAttach.NodeName) { - // Volume/Node exists, touch it to reset detachRequestedTime - if klog.V(5).Enabled() { - klog.Infof(volumeToAttach.GenerateMsgDetailed("Volume attached--touching", "")) - } - rc.actualStateOfWorld.ResetDetachRequestTime(volumeToAttach.VolumeName, volumeToAttach.NodeName) - continue - } - if util.IsMultiAttachAllowed(volumeToAttach.VolumeSpec) { - // Don't even try to start an operation if there is already one running for the given volume and node. if rc.attacherDetacher.IsOperationPending(volumeToAttach.VolumeName, "" /* podName */, volumeToAttach.NodeName) { if klog.V(10).Enabled() { @@ -244,9 +250,7 @@ func (rc *reconciler) attachDesiredVolumes() { } continue } - } else { - // Don't even try to start an operation if there is already one running for the given volume if rc.attacherDetacher.IsOperationPending(volumeToAttach.VolumeName, "" /* podName */, "" /* nodeName */) { if klog.V(10).Enabled() { @@ -254,7 +258,24 @@ func (rc *reconciler) attachDesiredVolumes() { } continue } + } + // Because the attach operation updates the ActualStateOfWorld before + // marking itself complete, IsOperationPending() must be checked before + // GetAttachState() to guarantee the ActualStateOfWorld is + // up-to-date when it's read. + // See https://github.com/kubernetes/kubernetes/issues/93902 + attachState := rc.actualStateOfWorld.GetAttachState(volumeToAttach.VolumeName, volumeToAttach.NodeName) + if attachState == cache.AttachStateAttached { + // Volume/Node exists, touch it to reset detachRequestedTime + if klog.V(5).Enabled() { + klog.Infof(volumeToAttach.GenerateMsgDetailed("Volume attached--touching", "")) + } + rc.actualStateOfWorld.ResetDetachRequestTime(volumeToAttach.VolumeName, volumeToAttach.NodeName) + continue + } + + if !util.IsMultiAttachAllowed(volumeToAttach.VolumeSpec) { nodes := rc.actualStateOfWorld.GetNodesForAttachedVolume(volumeToAttach.VolumeName) if len(nodes) > 0 { if !volumeToAttach.MultiAttachErrorReported { @@ -263,7 +284,6 @@ func (rc *reconciler) attachDesiredVolumes() { } continue } - } // Volume/Node doesn't exist, spawn a goroutine to attach it diff --git a/pkg/controller/volume/attachdetach/reconciler/reconciler_test.go b/pkg/controller/volume/attachdetach/reconciler/reconciler_test.go index 90023eb0ad209..84ffaf49d134d 100644 --- a/pkg/controller/volume/attachdetach/reconciler/reconciler_test.go +++ b/pkg/controller/volume/attachdetach/reconciler/reconciler_test.go @@ -347,7 +347,7 @@ func Test_Run_Negative_OneDesiredVolumeAttachThenDetachWithUnmountedVolumeUpdate } // Creates a volume with accessMode ReadWriteMany -// Populates desiredStateOfWorld cache with two ode/volume/pod tuples pointing to the created volume +// Populates desiredStateOfWorld cache with two node/volume/pod tuples pointing to the created volume // Calls Run() // Verifies there are two attach calls and no detach calls. // Deletes the first node/volume/pod tuple from desiredStateOfWorld cache without first marking the node/volume as unmounted. @@ -536,7 +536,7 @@ func Test_Run_OneVolumeAttachAndDetachMultipleNodesWithReadWriteOnce(t *testing. // Creates a volume with accessMode ReadWriteOnce // First create a pod which will try to attach the volume to the a node named "uncertain-node". The attach call for this node will // fail for timeout, but the volume will be actually attached to the node after the call. -// Secondly, delete the this pod. +// Secondly, delete this pod. // Lastly, create a pod scheduled to a normal node which will trigger attach volume to the node. The attach should return successfully. func Test_Run_OneVolumeAttachAndDetachUncertainNodesWithReadWriteOnce(t *testing.T) { // Arrange @@ -577,9 +577,9 @@ func Test_Run_OneVolumeAttachAndDetachUncertainNodesWithReadWriteOnce(t *testing } time.Sleep(1 * time.Second) - // Volume is added to asw. Because attach operation fails, volume should not reported as attached to the node. + // Volume is added to asw. Because attach operation fails, volume should not be reported as attached to the node. waitForVolumeAddedToNode(t, generatedVolumeName, nodeName1, asw) - verifyVolumeAttachedToNode(t, generatedVolumeName, nodeName1, true, asw) + verifyVolumeAttachedToNode(t, generatedVolumeName, nodeName1, cache.AttachStateAttached, asw) verifyVolumeReportedAsAttachedToNode(t, generatedVolumeName, nodeName1, true, asw) // When volume is added to the node, it is set to mounted by default. Then the status will be updated by checking node status VolumeInUse. @@ -596,7 +596,7 @@ func Test_Run_OneVolumeAttachAndDetachUncertainNodesWithReadWriteOnce(t *testing t.Fatalf("AddPod failed. Expected: Actual: <%v>", podAddErr) } waitForVolumeAttachedToNode(t, generatedVolumeName, nodeName2, asw) - verifyVolumeAttachedToNode(t, generatedVolumeName, nodeName2, true, asw) + verifyVolumeAttachedToNode(t, generatedVolumeName, nodeName2, cache.AttachStateAttached, asw) } @@ -643,9 +643,9 @@ func Test_Run_OneVolumeAttachAndDetachTimeoutNodesWithReadWriteOnce(t *testing.T t.Fatalf("AddPod failed. Expected: Actual: <%v>", podAddErr) } - // Volume is added to asw. Because attach operation fails, volume should not reported as attached to the node. + // Volume is added to asw. Because attach operation fails, volume should not be reported as attached to the node. waitForVolumeAddedToNode(t, generatedVolumeName, nodeName1, asw) - verifyVolumeAttachedToNode(t, generatedVolumeName, nodeName1, false, asw) + verifyVolumeAttachedToNode(t, generatedVolumeName, nodeName1, cache.AttachStateUncertain, asw) verifyVolumeReportedAsAttachedToNode(t, generatedVolumeName, nodeName1, false, asw) // When volume is added to the node, it is set to mounted by default. Then the status will be updated by checking node status VolumeInUse. @@ -662,7 +662,7 @@ func Test_Run_OneVolumeAttachAndDetachTimeoutNodesWithReadWriteOnce(t *testing.T t.Fatalf("AddPod failed. Expected: Actual: <%v>", podAddErr) } waitForVolumeAttachedToNode(t, generatedVolumeName, nodeName2, asw) - verifyVolumeAttachedToNode(t, generatedVolumeName, nodeName2, true, asw) + verifyVolumeAttachedToNode(t, generatedVolumeName, nodeName2, cache.AttachStateAttached, asw) } @@ -1048,7 +1048,8 @@ func waitForVolumeAttachedToNode( err := retryWithExponentialBackOff( time.Duration(500*time.Millisecond), func() (bool, error) { - if asw.IsVolumeAttachedToNode(volumeName, nodeName) { + attachState := asw.GetAttachState(volumeName, nodeName) + if attachState == cache.AttachStateAttached { return true, nil } t.Logf( @@ -1060,7 +1061,8 @@ func waitForVolumeAttachedToNode( }, ) - if err != nil && !asw.IsVolumeAttachedToNode(volumeName, nodeName) { + attachState := asw.GetAttachState(volumeName, nodeName) + if err != nil && attachState != cache.AttachStateAttached { t.Fatalf( "Volume <%v> is not attached to node <%v>.", volumeName, @@ -1141,19 +1143,17 @@ func verifyVolumeAttachedToNode( t *testing.T, volumeName v1.UniqueVolumeName, nodeName k8stypes.NodeName, - isAttached bool, + expectedAttachState cache.AttachState, asw cache.ActualStateOfWorld, ) { - result := asw.IsVolumeAttachedToNode(volumeName, nodeName) - if result == isAttached { - return + attachState := asw.GetAttachState(volumeName, nodeName) + if attachState != expectedAttachState { + t.Fatalf("Check volume <%v> is attached to node <%v>, got %v, expected %v", + volumeName, + nodeName, + attachState, + expectedAttachState) } - t.Fatalf("Check volume <%v> is attached to node <%v>, got %v, expected %v", - volumeName, - nodeName, - result, - isAttached) - } func verifyVolumeReportedAsAttachedToNode( diff --git a/pkg/controller/volume/attachdetach/testing/testvolumespec.go b/pkg/controller/volume/attachdetach/testing/testvolumespec.go index 907b42188295b..1a176f74fe7a1 100644 --- a/pkg/controller/volume/attachdetach/testing/testvolumespec.go +++ b/pkg/controller/volume/attachdetach/testing/testvolumespec.go @@ -60,6 +60,9 @@ func GetTestVolumeSpec(volumeName string, diskName v1.UniqueVolumeName) *volume. } var extraPods *v1.PodList +var volumeAttachments *storagev1.VolumeAttachmentList +var pvs *v1.PersistentVolumeList +var nodes *v1.NodeList func CreateTestClient() *fake.Clientset { fakeClient := &fake.Clientset{} @@ -143,40 +146,48 @@ func CreateTestClient() *fake.Clientset { } return true, obj, nil }) + nodes = &v1.NodeList{} + nodeNamePrefix := "mynode" + for i := 0; i < 5; i++ { + var nodeName string + if i != 0 { + nodeName = fmt.Sprintf("%s-%d", nodeNamePrefix, i) + } else { + // We want also the "mynode" node since all the testing pods live there + nodeName = nodeNamePrefix + } + attachVolumeToNode("lostVolumeName", nodeName) + } fakeClient.AddReactor("list", "nodes", func(action core.Action) (handled bool, ret runtime.Object, err error) { obj := &v1.NodeList{} - nodeNamePrefix := "mynode" - for i := 0; i < 5; i++ { - var nodeName string - if i != 0 { - nodeName = fmt.Sprintf("%s-%d", nodeNamePrefix, i) - } else { - // We want also the "mynode" node since all the testing pods live there - nodeName = nodeNamePrefix - } - node := v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: nodeName, - Labels: map[string]string{ - "name": nodeName, - }, - Annotations: map[string]string{ - util.ControllerManagedAttachAnnotation: "true", - }, - }, - Status: v1.NodeStatus{ - VolumesAttached: []v1.AttachedVolume{ - { - Name: TestPluginName + "/lostVolumeName", - DevicePath: "fake/path", - }, - }, - }, - } - obj.Items = append(obj.Items, node) - } + obj.Items = append(obj.Items, nodes.Items...) return true, obj, nil }) + volumeAttachments = &storagev1.VolumeAttachmentList{} + fakeClient.AddReactor("list", "volumeattachments", func(action core.Action) (handled bool, ret runtime.Object, err error) { + obj := &storagev1.VolumeAttachmentList{} + obj.Items = append(obj.Items, volumeAttachments.Items...) + return true, obj, nil + }) + fakeClient.AddReactor("create", "volumeattachments", func(action core.Action) (handled bool, ret runtime.Object, err error) { + createAction := action.(core.CreateAction) + va := createAction.GetObject().(*storagev1.VolumeAttachment) + volumeAttachments.Items = append(volumeAttachments.Items, *va) + return true, createAction.GetObject(), nil + }) + + pvs = &v1.PersistentVolumeList{} + fakeClient.AddReactor("list", "persistentvolumes", func(action core.Action) (handled bool, ret runtime.Object, err error) { + obj := &v1.PersistentVolumeList{} + obj.Items = append(obj.Items, pvs.Items...) + return true, obj, nil + }) + fakeClient.AddReactor("create", "persistentvolumes", func(action core.Action) (handled bool, ret runtime.Object, err error) { + createAction := action.(core.CreateAction) + pv := createAction.GetObject().(*v1.PersistentVolume) + pvs.Items = append(pvs.Items, *pv) + return true, createAction.GetObject(), nil + }) fakeWatch := watch.NewFake() fakeClient.AddWatchReactor("*", core.DefaultWatchReactor(fakeWatch, nil)) @@ -237,6 +248,88 @@ func NewPodWithVolume(podName, volumeName, nodeName string) *v1.Pod { } } +// Returns a volumeAttachment object +func NewVolumeAttachment(vaName, pvName, nodeName string, status bool) *storagev1.VolumeAttachment { + return &storagev1.VolumeAttachment{ + + ObjectMeta: metav1.ObjectMeta{ + UID: types.UID(vaName), + Name: vaName, + }, + Spec: storagev1.VolumeAttachmentSpec{ + Attacher: "test.storage.gke.io", + NodeName: nodeName, + Source: storagev1.VolumeAttachmentSource{ + PersistentVolumeName: &pvName, + }, + }, + Status: storagev1.VolumeAttachmentStatus{ + Attached: status, + }, + } +} + +// Returns a persistentVolume object +func NewPV(pvName, volumeName string) *v1.PersistentVolume { + return &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + UID: types.UID(pvName), + Name: pvName, + }, + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{ + GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{ + PDName: volumeName, + }, + }, + }, + } +} + +func attachVolumeToNode(volumeName, nodeName string) { + // if nodeName exists, get the object.. if not create node object + var node *v1.Node + found := false + nodes.Size() + for i := range nodes.Items { + curNode := nodes.Items[i] + if curNode.ObjectMeta.Name == nodeName { + node = &curNode + found = true + break + } + } + if !found { + node = &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Labels: map[string]string{ + "name": nodeName, + }, + Annotations: map[string]string{ + util.ControllerManagedAttachAnnotation: "true", + }, + }, + Status: v1.NodeStatus{ + VolumesAttached: []v1.AttachedVolume{ + { + Name: v1.UniqueVolumeName(TestPluginName + "/" + volumeName), + DevicePath: "fake/path", + }, + }, + }, + } + } else { + volumeAttached := v1.AttachedVolume{ + Name: v1.UniqueVolumeName(TestPluginName + "/" + volumeName), + DevicePath: "fake/path", + } + node.Status.VolumesAttached = append(node.Status.VolumesAttached, volumeAttached) + } + + nodes.Items = append(nodes.Items, *node) +} + type TestPlugin struct { ErrorEncountered bool attachedVolumeMap map[string][]string @@ -258,8 +351,15 @@ func (plugin *TestPlugin) GetVolumeName(spec *volume.Spec) (string, error) { if spec == nil { klog.Errorf("GetVolumeName called with nil volume spec") plugin.ErrorEncountered = true + return "", fmt.Errorf("GetVolumeName called with nil volume spec") + } + if spec.Volume != nil { + return spec.Name(), nil + } else if spec.PersistentVolume != nil { + return spec.PersistentVolume.Spec.PersistentVolumeSource.GCEPersistentDisk.PDName, nil + } else { + return "", nil } - return spec.Name(), nil } func (plugin *TestPlugin) CanSupport(spec *volume.Spec) bool { diff --git a/pkg/volume/testing/BUILD b/pkg/volume/testing/BUILD index 16070d966681e..27cef5aa716e4 100644 --- a/pkg/volume/testing/BUILD +++ b/pkg/volume/testing/BUILD @@ -26,6 +26,7 @@ go_library( "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", diff --git a/pkg/volume/testing/testing.go b/pkg/volume/testing/testing.go index 2b466cc43671a..22e942afe6b58 100644 --- a/pkg/volume/testing/testing.go +++ b/pkg/volume/testing/testing.go @@ -27,6 +27,7 @@ import ( "testing" "time" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/mount-utils" "k8s.io/utils/exec" testingexec "k8s.io/utils/exec/testing" @@ -427,7 +428,7 @@ func (plugin *FakeVolumePlugin) getFakeVolume(list *[]*FakeVolume) *FakeVolume { WaitForAttachHook: plugin.WaitForAttachHook, UnmountDeviceHook: plugin.UnmountDeviceHook, } - volume.VolumesAttached = make(map[string]types.NodeName) + volume.VolumesAttached = make(map[string]sets.String) volume.DeviceMountState = make(map[string]string) volume.VolumeMountState = make(map[string]string) *list = append(*list, volume) @@ -836,7 +837,7 @@ type FakeVolume struct { VolName string Plugin *FakeVolumePlugin MetricsNil - VolumesAttached map[string]types.NodeName + VolumesAttached map[string]sets.String DeviceMountState map[string]string VolumeMountState map[string]string @@ -1155,11 +1156,12 @@ func (fv *FakeVolume) Attach(spec *Spec, nodeName types.NodeName) (string, error fv.Lock() defer fv.Unlock() fv.AttachCallCount++ + volumeName, err := getUniqueVolumeName(spec) if err != nil { return "", err } - volumeNode, exist := fv.VolumesAttached[volumeName] + volumeNodes, exist := fv.VolumesAttached[volumeName] if exist { if nodeName == UncertainAttachNode { return "/dev/vdb-test", nil @@ -1169,13 +1171,14 @@ func (fv *FakeVolume) Attach(spec *Spec, nodeName types.NodeName) (string, error if nodeName == TimeoutAttachNode { return "", fmt.Errorf("Timed out to attach volume %q to node %q", volumeName, nodeName) } - if volumeNode == nodeName || volumeNode == MultiAttachNode || nodeName == MultiAttachNode { + if volumeNodes.Has(string(nodeName)) || volumeNodes.Has(MultiAttachNode) || nodeName == MultiAttachNode { + volumeNodes.Insert(string(nodeName)) return "/dev/vdb-test", nil } - return "", fmt.Errorf("volume %q trying to attach to node %q is already attached to node %q", volumeName, nodeName, volumeNode) + return "", fmt.Errorf("volume %q trying to attach to node %q is already attached to node %q", volumeName, nodeName, volumeNodes) } - fv.VolumesAttached[volumeName] = nodeName + fv.VolumesAttached[volumeName] = sets.NewString(string(nodeName)) if nodeName == UncertainAttachNode || nodeName == TimeoutAttachNode { return "", fmt.Errorf("Timed out to attach volume %q to node %q", volumeName, nodeName) } @@ -1273,10 +1276,18 @@ func (fv *FakeVolume) Detach(volumeName string, nodeName types.NodeName) error { fv.Lock() defer fv.Unlock() fv.DetachCallCount++ - if _, exist := fv.VolumesAttached[volumeName]; !exist { - return fmt.Errorf("Trying to detach volume %q that is not attached to the node %q", volumeName, nodeName) + + node := string(nodeName) + volumeNodes, exist := fv.VolumesAttached[volumeName] + if !exist || !volumeNodes.Has(node) { + return fmt.Errorf("Trying to detach volume %q that is not attached to the node %q", volumeName, node) } - delete(fv.VolumesAttached, volumeName) + + volumeNodes.Delete(node) + if volumeNodes.Len() == 0 { + delete(fv.VolumesAttached, volumeName) + } + return nil }