Merge pull request #55944 from edisonxiang/probeimprovecinder

Automatic merge from submit-queue (batch tested with PRs 55925, 55999, 55944, 55992, 56196). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

probeAttachedVolume improvement in Cinder

What this PR does / why we need it:

During OpenStack Cinder volume attachment, kubelet creates too many udev "change" events.
If volume attachement takes long, then udev event queue gets full,therefore
udev does not answer requests from docker, and makes Kubernetes node unhealthy.

This patch will extend the udevadm trigger period,
and use the udevadm settle to ensure that any device nodes
have been created successfully before proceeding.

Fixes #55007

Release note:

```release-note
None
```

/cc @dims @kabakaev

Author: Kubernetes Submit Queue

pkg/volume/cinder/attacher.go

@@ -44,14 +44,15 @@ var _ volume.Attacher = &cinderDiskAttacher{}
 var _ volume.AttachableVolumePlugin = &cinderPlugin{}
 
 const (
-	checkSleepDuration       = 1 * time.Second
-	operationFinishInitDealy = 1 * time.Second
+	probeVolumeInitDelay     = 1 * time.Second
+	probeVolumeFactor        = 2.0
+	operationFinishInitDelay = 1 * time.Second
 	operationFinishFactor    = 1.1
 	operationFinishSteps     = 10
-	diskAttachInitDealy      = 1 * time.Second
+	diskAttachInitDelay      = 1 * time.Second
 	diskAttachFactor         = 1.2
 	diskAttachSteps          = 15
-	diskDetachInitDealy      = 1 * time.Second
+	diskDetachInitDelay      = 1 * time.Second
 	diskDetachFactor         = 1.2
 	diskDetachSteps          = 13
 )
@@ -74,7 +75,7 @@ func (plugin *cinderPlugin) GetDeviceMountRefs(deviceMountPath string) ([]string
 
 func (attacher *cinderDiskAttacher) waitOperationFinished(volumeID string) error {
 	backoff := wait.Backoff{
-		Duration: operationFinishInitDealy,
+		Duration: operationFinishInitDelay,
 		Factor:   operationFinishFactor,
 		Steps:    operationFinishSteps,
 	}
@@ -99,7 +100,7 @@ func (attacher *cinderDiskAttacher) waitOperationFinished(volumeID string) error
 
 func (attacher *cinderDiskAttacher) waitDiskAttached(instanceID, volumeID string) error {
 	backoff := wait.Backoff{
-		Duration: diskAttachInitDealy,
+		Duration: diskAttachInitDelay,
 		Factor:   diskAttachFactor,
 		Steps:    diskAttachSteps,
 	}
@@ -234,11 +235,12 @@ func (attacher *cinderDiskAttacher) WaitForAttach(spec *volume.Spec, devicePath
 		return "", fmt.Errorf("WaitForAttach failed for Cinder disk %q: devicePath is empty.", volumeID)
 	}
 
-	ticker := time.NewTicker(checkSleepDuration)
+	ticker := time.NewTicker(probeVolumeInitDelay)
 	defer ticker.Stop()
 	timer := time.NewTimer(timeout)
 	defer timer.Stop()
 
+	duration := probeVolumeInitDelay
 	for {
 		select {
 		case <-ticker.C:
@@ -255,6 +257,10 @@ func (attacher *cinderDiskAttacher) WaitForAttach(spec *volume.Spec, devicePath
 			} else {
 				// Log an error, and continue checking periodically
 				glog.Errorf("Error: could not find attached Cinder disk %q (path: %q): %v", volumeID, devicePath, err)
+				// Using exponential backoff instead of linear
+				ticker.Stop()
+				duration = time.Duration(float64(duration) * probeVolumeFactor)
+				ticker = time.NewTicker(duration)
 			}
 		case <-timer.C:
 			return "", fmt.Errorf("Could not find attached Cinder disk %q. Timeout waiting for mount paths to be created.", volumeID)
@@ -328,7 +334,7 @@ func (plugin *cinderPlugin) NewDetacher() (volume.Detacher, error) {
 
 func (detacher *cinderDiskDetacher) waitOperationFinished(volumeID string) error {
 	backoff := wait.Backoff{
-		Duration: operationFinishInitDealy,
+		Duration: operationFinishInitDelay,
 		Factor:   operationFinishFactor,
 		Steps:    operationFinishSteps,
 	}
@@ -353,7 +359,7 @@ func (detacher *cinderDiskDetacher) waitOperationFinished(volumeID string) error
 
 func (detacher *cinderDiskDetacher) waitDiskDetached(instanceID, volumeID string) error {
 	backoff := wait.Backoff{
-		Duration: diskDetachInitDealy,
+		Duration: diskDetachInitDelay,
 		Factor:   diskDetachFactor,
 		Steps:    diskDetachSteps,
 	}

pkg/volume/cinder/cinder_util.go

@@ -224,6 +224,17 @@ func probeAttachedVolume() error {
 	scsiHostRescan()
 
 	executor := exec.New()
+
+	// udevadm settle waits for udevd to process the device creation
+	// events for all hardware devices, thus ensuring that any device
+	// nodes have been created successfully before proceeding.
+	argsSettle := []string{"settle"}
+	cmdSettle := executor.Command("udevadm", argsSettle...)
+	_, errSettle := cmdSettle.CombinedOutput()
+	if errSettle != nil {
+		glog.Errorf("error running udevadm settle %v\n", errSettle)
+	}
+
 	args := []string{"trigger"}
 	cmd := executor.Command("udevadm", args...)
 	_, err := cmd.CombinedOutput()