Merge pull request #52910 from FengyunPan/volume-v3

Automatic merge from submit-queue (batch tested with PRs 56094, 52910, 55953, 56405, 56415). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Support VolumeV3 for OpenStack cloud Provider

Currently OpenStack supports Cinder v3 API, let Kubernetes support
it too.

Fix #52877

**Release note**:
```release-note
OpenStack cloud provider supports Cinder v3 API.
```

Author: Kubernetes Submit Queue

pkg/cloudprovider/providers/openstack/BUILD

@@ -32,6 +32,7 @@ go_library(
         "//vendor/github.com/gophercloud/gophercloud/openstack/blockstorage/extensions/volumeactions:go_default_library",
         "//vendor/github.com/gophercloud/gophercloud/openstack/blockstorage/v1/volumes:go_default_library",
         "//vendor/github.com/gophercloud/gophercloud/openstack/blockstorage/v2/volumes:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/blockstorage/v3/volumes:go_default_library",
         "//vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/attachinterfaces:go_default_library",
         "//vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/volumeattach:go_default_library",
         "//vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/servers:go_default_library",

pkg/cloudprovider/providers/openstack/openstack.go

@@ -660,27 +660,36 @@ func (os *OpenStack) volumeService(forceVersion string) (volumeService, error) {
 		}
 		glog.V(3).Infof("Using Blockstorage API V2")
 		return &VolumesV2{sClient, os.bsOpts}, nil
+	case "v3":
+		sClient, err := os.NewBlockStorageV3()
+		if err != nil {
+			return nil, err
+		}
+		glog.V(3).Infof("Using Blockstorage API V3")
+		return &VolumesV3{sClient, os.bsOpts}, nil
 	case "auto":
-		// Currently kubernetes just support Cinder v1 and Cinder v2.
-		// Choose Cinder v2 firstly, if kubernetes can't initialize cinder v2 client, try to initialize cinder v1 client.
+		// Currently kubernetes support Cinder v1 / Cinder v2 / Cinder v3.
+		// Choose Cinder v3 firstly, if kubernetes can't initialize cinder v3 client, try to initialize cinder v2 client.
+		// If kubernetes can't initialize cinder v2 client, try to initialize cinder v1 client.
 		// Return appropriate message when kubernetes can't initialize them.
-		// TODO(FengyunPan): revisit 'auto' after supporting Cinder v3.
-		sClient, err := os.NewBlockStorageV2()
-		if err != nil {
-			sClient, err = os.NewBlockStorageV1()
-			if err != nil {
-				// Nothing suitable found, failed autodetection, just exit with appropriate message
-				err_txt := "BlockStorage API version autodetection failed. " +
-					"Please set it explicitly in cloud.conf in section [BlockStorage] with key `bs-version`"
-				return nil, errors.New(err_txt)
-			} else {
-				glog.V(3).Infof("Using Blockstorage API V1")
-				return &VolumesV1{sClient, os.bsOpts}, nil
-			}
-		} else {
+		if sClient, err := os.NewBlockStorageV3(); err == nil {
+			glog.V(3).Infof("Using Blockstorage API V3")
+			return &VolumesV3{sClient, os.bsOpts}, nil
+		}
+
+		if sClient, err := os.NewBlockStorageV2(); err == nil {
 			glog.V(3).Infof("Using Blockstorage API V2")
 			return &VolumesV2{sClient, os.bsOpts}, nil
 		}
+
+		if sClient, err := os.NewBlockStorageV1(); err == nil {
+			glog.V(3).Infof("Using Blockstorage API V1")
+			return &VolumesV1{sClient, os.bsOpts}, nil
+		}
+
+		err_txt := "BlockStorage API version autodetection failed. " +
+			"Please set it explicitly in cloud.conf in section [BlockStorage] with key `bs-version`"
+		return nil, errors.New(err_txt)
 	default:
 		err_txt := fmt.Sprintf("Config error: unrecognised bs-version \"%v\"", os.bsOpts.BSVersion)
 		return nil, errors.New(err_txt)

pkg/cloudprovider/providers/openstack/openstack_client.go

@@ -63,6 +63,16 @@ func (os *OpenStack) NewBlockStorageV2() (*gophercloud.ServiceClient, error) {
 	return storage, nil
 }
 
+func (os *OpenStack) NewBlockStorageV3() (*gophercloud.ServiceClient, error) {
+	storage, err := openstack.NewBlockStorageV3(os.provider, gophercloud.EndpointOpts{
+		Region: os.region,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("unable to initialize cinder v3 client for region %s: %v", os.region, err)
+	}
+	return storage, nil
+}
+
 func (os *OpenStack) NewLoadBalancerV2() (*gophercloud.ServiceClient, error) {
 	var lb *gophercloud.ServiceClient
 	var err error

pkg/cloudprovider/providers/openstack/openstack_volumes.go

@@ -31,6 +31,7 @@ import (
 	volumeexpand "github.com/gophercloud/gophercloud/openstack/blockstorage/extensions/volumeactions"
 	volumes_v1 "github.com/gophercloud/gophercloud/openstack/blockstorage/v1/volumes"
 	volumes_v2 "github.com/gophercloud/gophercloud/openstack/blockstorage/v2/volumes"
+	volumes_v3 "github.com/gophercloud/gophercloud/openstack/blockstorage/v3/volumes"
 	"github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/volumeattach"
 	"github.com/prometheus/client_golang/prometheus"
 
@@ -56,6 +57,12 @@ type VolumesV2 struct {
 	opts         BlockStorageOpts
 }
 
+// Volumes implementation for v3
+type VolumesV3 struct {
+	blockstorage *gophercloud.ServiceClient
+	opts         BlockStorageOpts
+}
+
 type Volume struct {
 	// ID of the instance, to which this volume is attached. "" if not attached
 	AttachedServerId string
@@ -131,6 +138,26 @@ func (volumes *VolumesV2) createVolume(opts VolumeCreateOpts) (string, string, e
 	return vol.ID, vol.AvailabilityZone, nil
 }
 
+func (volumes *VolumesV3) createVolume(opts VolumeCreateOpts) (string, string, error) {
+	startTime := time.Now()
+
+	create_opts := volumes_v3.CreateOpts{
+		Name:             opts.Name,
+		Size:             opts.Size,
+		VolumeType:       opts.VolumeType,
+		AvailabilityZone: opts.Availability,
+		Metadata:         opts.Metadata,
+	}
+
+	vol, err := volumes_v3.Create(volumes.blockstorage, create_opts).Extract()
+	timeTaken := time.Since(startTime).Seconds()
+	recordOpenstackOperationMetric("create_v3_volume", timeTaken, err)
+	if err != nil {
+		return "", "", err
+	}
+	return vol.ID, vol.AvailabilityZone, nil
+}
+
 func (volumes *VolumesV1) getVolume(volumeID string) (Volume, error) {
 	startTime := time.Now()
 	volumeV1, err := volumes_v1.Get(volumes.blockstorage, volumeID).Extract()
@@ -179,6 +206,29 @@ func (volumes *VolumesV2) getVolume(volumeID string) (Volume, error) {
 	return volume, nil
 }
 
+func (volumes *VolumesV3) getVolume(volumeID string) (Volume, error) {
+	startTime := time.Now()
+	volumeV3, err := volumes_v3.Get(volumes.blockstorage, volumeID).Extract()
+	timeTaken := time.Since(startTime).Seconds()
+	recordOpenstackOperationMetric("get_v3_volume", timeTaken, err)
+	if err != nil {
+		return Volume{}, fmt.Errorf("error occurred getting volume by ID: %s, err: %v", volumeID, err)
+	}
+
+	volume := Volume{
+		ID:     volumeV3.ID,
+		Name:   volumeV3.Name,
+		Status: volumeV3.Status,
+	}
+
+	if len(volumeV3.Attachments) > 0 {
+		volume.AttachedServerId = volumeV3.Attachments[0].ServerID
+		volume.AttachedDevice = volumeV3.Attachments[0].Device
+	}
+
+	return volume, nil
+}
+
 func (volumes *VolumesV1) deleteVolume(volumeID string) error {
 	startTime := time.Now()
 	err := volumes_v1.Delete(volumes.blockstorage, volumeID).ExtractErr()
@@ -195,6 +245,14 @@ func (volumes *VolumesV2) deleteVolume(volumeID string) error {
 	return err
 }
 
+func (volumes *VolumesV3) deleteVolume(volumeID string) error {
+	startTime := time.Now()
+	err := volumes_v3.Delete(volumes.blockstorage, volumeID).ExtractErr()
+	timeTaken := time.Since(startTime).Seconds()
+	recordOpenstackOperationMetric("delete_v3_volume", timeTaken, err)
+	return err
+}
+
 func (volumes *VolumesV1) expandVolume(volumeID string, newSize int) error {
 	startTime := time.Now()
 	create_opts := volumeexpand.ExtendSizeOpts{
@@ -203,7 +261,6 @@ func (volumes *VolumesV1) expandVolume(volumeID string, newSize int) error {
 	err := volumeexpand.ExtendSize(volumes.blockstorage, volumeID, create_opts).ExtractErr()
 	timeTaken := time.Since(startTime).Seconds()
 	recordOpenstackOperationMetric("expand_volume", timeTaken, err)
-
 	return err
 }
 
@@ -215,7 +272,17 @@ func (volumes *VolumesV2) expandVolume(volumeID string, newSize int) error {
 	err := volumeexpand.ExtendSize(volumes.blockstorage, volumeID, create_opts).ExtractErr()
 	timeTaken := time.Since(startTime).Seconds()
 	recordOpenstackOperationMetric("expand_volume", timeTaken, err)
+	return err
+}
 
+func (volumes *VolumesV3) expandVolume(volumeID string, newSize int) error {
+	startTime := time.Now()
+	create_opts := volumeexpand.ExtendSizeOpts{
+		NewSize: newSize,
+	}
+	err := volumeexpand.ExtendSize(volumes.blockstorage, volumeID, create_opts).ExtractErr()
+	timeTaken := time.Since(startTime).Seconds()
+	recordOpenstackOperationMetric("expand_volume", timeTaken, err)
 	return err
 }