Merge branch 'master' into upgrade_aliases_branch

Author: jingax10

pkg/cloudprovider/providers/openstack/BUILD

@@ -32,6 +32,7 @@ go_library(
         "//vendor/github.com/gophercloud/gophercloud/openstack/blockstorage/extensions/volumeactions:go_default_library",
         "//vendor/github.com/gophercloud/gophercloud/openstack/blockstorage/v1/volumes:go_default_library",
         "//vendor/github.com/gophercloud/gophercloud/openstack/blockstorage/v2/volumes:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/blockstorage/v3/volumes:go_default_library",
         "//vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/attachinterfaces:go_default_library",
         "//vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/volumeattach:go_default_library",
         "//vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/servers:go_default_library",
@@ -75,6 +76,7 @@ go_test(
         "//pkg/cloudprovider:go_default_library",
         "//vendor/github.com/gophercloud/gophercloud:go_default_library",
         "//vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/servers:go_default_library",
+        "//vendor/github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/routers:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",

pkg/cloudprovider/providers/openstack/openstack.go

@@ -660,27 +660,36 @@ func (os *OpenStack) volumeService(forceVersion string) (volumeService, error) {
 		}
 		glog.V(3).Infof("Using Blockstorage API V2")
 		return &VolumesV2{sClient, os.bsOpts}, nil
+	case "v3":
+		sClient, err := os.NewBlockStorageV3()
+		if err != nil {
+			return nil, err
+		}
+		glog.V(3).Infof("Using Blockstorage API V3")
+		return &VolumesV3{sClient, os.bsOpts}, nil
 	case "auto":
-		// Currently kubernetes just support Cinder v1 and Cinder v2.
-		// Choose Cinder v2 firstly, if kubernetes can't initialize cinder v2 client, try to initialize cinder v1 client.
+		// Currently kubernetes support Cinder v1 / Cinder v2 / Cinder v3.
+		// Choose Cinder v3 firstly, if kubernetes can't initialize cinder v3 client, try to initialize cinder v2 client.
+		// If kubernetes can't initialize cinder v2 client, try to initialize cinder v1 client.
 		// Return appropriate message when kubernetes can't initialize them.
-		// TODO(FengyunPan): revisit 'auto' after supporting Cinder v3.
-		sClient, err := os.NewBlockStorageV2()
-		if err != nil {
-			sClient, err = os.NewBlockStorageV1()
-			if err != nil {
-				// Nothing suitable found, failed autodetection, just exit with appropriate message
-				err_txt := "BlockStorage API version autodetection failed. " +
-					"Please set it explicitly in cloud.conf in section [BlockStorage] with key `bs-version`"
-				return nil, errors.New(err_txt)
-			} else {
-				glog.V(3).Infof("Using Blockstorage API V1")
-				return &VolumesV1{sClient, os.bsOpts}, nil
-			}
-		} else {
+		if sClient, err := os.NewBlockStorageV3(); err == nil {
+			glog.V(3).Infof("Using Blockstorage API V3")
+			return &VolumesV3{sClient, os.bsOpts}, nil
+		}
+
+		if sClient, err := os.NewBlockStorageV2(); err == nil {
 			glog.V(3).Infof("Using Blockstorage API V2")
 			return &VolumesV2{sClient, os.bsOpts}, nil
 		}
+
+		if sClient, err := os.NewBlockStorageV1(); err == nil {
+			glog.V(3).Infof("Using Blockstorage API V1")
+			return &VolumesV1{sClient, os.bsOpts}, nil
+		}
+
+		err_txt := "BlockStorage API version autodetection failed. " +
+			"Please set it explicitly in cloud.conf in section [BlockStorage] with key `bs-version`"
+		return nil, errors.New(err_txt)
 	default:
 		err_txt := fmt.Sprintf("Config error: unrecognised bs-version \"%v\"", os.bsOpts.BSVersion)
 		return nil, errors.New(err_txt)

pkg/cloudprovider/providers/openstack/openstack_client.go

@@ -63,6 +63,16 @@ func (os *OpenStack) NewBlockStorageV2() (*gophercloud.ServiceClient, error) {
 	return storage, nil
 }
 
+func (os *OpenStack) NewBlockStorageV3() (*gophercloud.ServiceClient, error) {
+	storage, err := openstack.NewBlockStorageV3(os.provider, gophercloud.EndpointOpts{
+		Region: os.region,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("unable to initialize cinder v3 client for region %s: %v", os.region, err)
+	}
+	return storage, nil
+}
+
 func (os *OpenStack) NewLoadBalancerV2() (*gophercloud.ServiceClient, error) {
 	var lb *gophercloud.ServiceClient
 	var err error

pkg/cloudprovider/providers/openstack/openstack_routes_test.go

@@ -20,6 +20,8 @@ import (
 	"net"
 	"testing"
 
+	"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
+	"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/routers"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/kubernetes/pkg/cloudprovider"
 )
@@ -37,14 +39,18 @@ func TestRoutes(t *testing.T) {
 		t.Fatalf("Failed to construct/authenticate OpenStack: %s", err)
 	}
 
+	// Pick the first router and server to try a test with
+	os.routeOpts.RouterId = getRouters(os)[0].ID
+	servername := getServers(os)[0].Name
+
 	r, ok := os.Routes()
 	if !ok {
-		t.Fatalf("Routes() returned false - perhaps your stack doens't support Neutron?")
+		t.Skip("Routes() returned false - perhaps your stack does not support Neutron extraroute extension?")
 	}
 
 	newroute := cloudprovider.Route{
 		DestinationCIDR: "10.164.2.0/24",
-		TargetNode:      types.NodeName("testinstance"),
+		TargetNode:      types.NodeName(servername),
 	}
 	err = r.CreateRoute(clusterName, "myhint", &newroute)
 	if err != nil {
@@ -69,3 +75,39 @@ func TestRoutes(t *testing.T) {
 		t.Fatalf("DeleteRoute error: %v", err)
 	}
 }
+
+func getServers(os *OpenStack) []servers.Server {
+	c, err := os.NewComputeV2()
+	allPages, err := servers.List(c, servers.ListOpts{}).AllPages()
+	if err != nil {
+		panic(err)
+	}
+	allServers, err := servers.ExtractServers(allPages)
+	if err != nil {
+		panic(err)
+	}
+	if len(allServers) == 0 {
+		panic("No servers to test with")
+	}
+	return allServers
+}
+
+func getRouters(os *OpenStack) []routers.Router {
+	listOpts := routers.ListOpts{}
+	n, err := os.NewNetworkV2()
+	if err != nil {
+		panic(err)
+	}
+	allPages, err := routers.List(n, listOpts).AllPages()
+	if err != nil {
+		panic(err)
+	}
+	allRouters, err := routers.ExtractRouters(allPages)
+	if err != nil {
+		panic(err)
+	}
+	if len(allRouters) == 0 {
+		panic("No routers to test with")
+	}
+	return allRouters
+}

pkg/cloudprovider/providers/openstack/openstack_test.go

@@ -421,6 +421,7 @@ func configFromEnv() (cfg Config, ok bool) {
 			cfg.Global.DomainId != "" || cfg.Global.DomainName != ""))
 
 	cfg.Metadata.SearchOrder = fmt.Sprintf("%s,%s", configDriveID, metadataID)
+	cfg.BlockStorage.BSVersion = "auto"
 
 	return
 }
@@ -443,7 +444,7 @@ func TestLoadBalancer(t *testing.T) {
 		t.Skipf("No config found in environment")
 	}
 
-	versions := []string{"v1", "v2", ""}
+	versions := []string{"v2", ""}
 
 	for _, v := range versions {
 		t.Logf("Trying LBVersion = '%s'\n", v)
@@ -525,30 +526,30 @@ func TestVolumes(t *testing.T) {
 
 	id, err := os.InstanceID()
 	if err != nil {
-		t.Fatalf("Cannot find instance id: %v", err)
-	}
+		t.Logf("Cannot find instance id: %v - perhaps you are running this test outside a VM launched by OpenStack", err)
+	} else {
+		diskId, err := os.AttachDisk(id, vol)
+		if err != nil {
+			t.Fatalf("Cannot AttachDisk Cinder volume %s: %v", vol, err)
+		}
+		t.Logf("Volume (%s) attached, disk ID: %s\n", vol, diskId)
 
-	diskId, err := os.AttachDisk(id, vol)
-	if err != nil {
-		t.Fatalf("Cannot AttachDisk Cinder volume %s: %v", vol, err)
-	}
-	t.Logf("Volume (%s) attached, disk ID: %s\n", vol, diskId)
+		WaitForVolumeStatus(t, os, vol, volumeInUseStatus)
 
-	WaitForVolumeStatus(t, os, vol, volumeInUseStatus)
+		devicePath := os.GetDevicePath(diskId)
+		if diskPathRegexp.FindString(devicePath) == "" {
+			t.Fatalf("GetDevicePath returned and unexpected path for Cinder volume %s, returned %s", vol, devicePath)
+		}
+		t.Logf("Volume (%s) found at path: %s\n", vol, devicePath)
 
-	devicePath := os.GetDevicePath(diskId)
-	if diskPathRegexp.FindString(devicePath) == "" {
-		t.Fatalf("GetDevicePath returned and unexpected path for Cinder volume %s, returned %s", vol, devicePath)
-	}
-	t.Logf("Volume (%s) found at path: %s\n", vol, devicePath)
+		err = os.DetachDisk(id, vol)
+		if err != nil {
+			t.Fatalf("Cannot DetachDisk Cinder volume %s: %v", vol, err)
+		}
+		t.Logf("Volume (%s) detached\n", vol)
 
-	err = os.DetachDisk(id, vol)
-	if err != nil {
-		t.Fatalf("Cannot DetachDisk Cinder volume %s: %v", vol, err)
+		WaitForVolumeStatus(t, os, vol, volumeAvailableStatus)
 	}
-	t.Logf("Volume (%s) detached\n", vol)
-
-	WaitForVolumeStatus(t, os, vol, volumeAvailableStatus)
 
 	err = os.DeleteVolume(vol)
 	if err != nil {

pkg/cloudprovider/providers/openstack/openstack_volumes.go

@@ -31,6 +31,7 @@ import (
 	volumeexpand "github.com/gophercloud/gophercloud/openstack/blockstorage/extensions/volumeactions"
 	volumes_v1 "github.com/gophercloud/gophercloud/openstack/blockstorage/v1/volumes"
 	volumes_v2 "github.com/gophercloud/gophercloud/openstack/blockstorage/v2/volumes"
+	volumes_v3 "github.com/gophercloud/gophercloud/openstack/blockstorage/v3/volumes"
 	"github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/volumeattach"
 	"github.com/prometheus/client_golang/prometheus"
 
@@ -56,6 +57,12 @@ type VolumesV2 struct {
 	opts         BlockStorageOpts
 }
 
+// Volumes implementation for v3
+type VolumesV3 struct {
+	blockstorage *gophercloud.ServiceClient
+	opts         BlockStorageOpts
+}
+
 type Volume struct {
 	// ID of the instance, to which this volume is attached. "" if not attached
 	AttachedServerId string
@@ -131,6 +138,26 @@ func (volumes *VolumesV2) createVolume(opts VolumeCreateOpts) (string, string, e
 	return vol.ID, vol.AvailabilityZone, nil
 }
 
+func (volumes *VolumesV3) createVolume(opts VolumeCreateOpts) (string, string, error) {
+	startTime := time.Now()
+
+	create_opts := volumes_v3.CreateOpts{
+		Name:             opts.Name,
+		Size:             opts.Size,
+		VolumeType:       opts.VolumeType,
+		AvailabilityZone: opts.Availability,
+		Metadata:         opts.Metadata,
+	}
+
+	vol, err := volumes_v3.Create(volumes.blockstorage, create_opts).Extract()
+	timeTaken := time.Since(startTime).Seconds()
+	recordOpenstackOperationMetric("create_v3_volume", timeTaken, err)
+	if err != nil {
+		return "", "", err
+	}
+	return vol.ID, vol.AvailabilityZone, nil
+}
+
 func (volumes *VolumesV1) getVolume(volumeID string) (Volume, error) {
 	startTime := time.Now()
 	volumeV1, err := volumes_v1.Get(volumes.blockstorage, volumeID).Extract()
@@ -179,6 +206,29 @@ func (volumes *VolumesV2) getVolume(volumeID string) (Volume, error) {
 	return volume, nil
 }
 
+func (volumes *VolumesV3) getVolume(volumeID string) (Volume, error) {
+	startTime := time.Now()
+	volumeV3, err := volumes_v3.Get(volumes.blockstorage, volumeID).Extract()
+	timeTaken := time.Since(startTime).Seconds()
+	recordOpenstackOperationMetric("get_v3_volume", timeTaken, err)
+	if err != nil {
+		return Volume{}, fmt.Errorf("error occurred getting volume by ID: %s, err: %v", volumeID, err)
+	}
+
+	volume := Volume{
+		ID:     volumeV3.ID,
+		Name:   volumeV3.Name,
+		Status: volumeV3.Status,
+	}
+
+	if len(volumeV3.Attachments) > 0 {
+		volume.AttachedServerId = volumeV3.Attachments[0].ServerID
+		volume.AttachedDevice = volumeV3.Attachments[0].Device
+	}
+
+	return volume, nil
+}
+
 func (volumes *VolumesV1) deleteVolume(volumeID string) error {
 	startTime := time.Now()
 	err := volumes_v1.Delete(volumes.blockstorage, volumeID).ExtractErr()
@@ -195,6 +245,14 @@ func (volumes *VolumesV2) deleteVolume(volumeID string) error {
 	return err
 }
 
+func (volumes *VolumesV3) deleteVolume(volumeID string) error {
+	startTime := time.Now()
+	err := volumes_v3.Delete(volumes.blockstorage, volumeID).ExtractErr()
+	timeTaken := time.Since(startTime).Seconds()
+	recordOpenstackOperationMetric("delete_v3_volume", timeTaken, err)
+	return err
+}
+
 func (volumes *VolumesV1) expandVolume(volumeID string, newSize int) error {
 	startTime := time.Now()
 	create_opts := volumeexpand.ExtendSizeOpts{
@@ -203,7 +261,6 @@ func (volumes *VolumesV1) expandVolume(volumeID string, newSize int) error {
 	err := volumeexpand.ExtendSize(volumes.blockstorage, volumeID, create_opts).ExtractErr()
 	timeTaken := time.Since(startTime).Seconds()
 	recordOpenstackOperationMetric("expand_volume", timeTaken, err)
-
 	return err
 }
 
@@ -215,7 +272,17 @@ func (volumes *VolumesV2) expandVolume(volumeID string, newSize int) error {
 	err := volumeexpand.ExtendSize(volumes.blockstorage, volumeID, create_opts).ExtractErr()
 	timeTaken := time.Since(startTime).Seconds()
 	recordOpenstackOperationMetric("expand_volume", timeTaken, err)
+	return err
+}
 
+func (volumes *VolumesV3) expandVolume(volumeID string, newSize int) error {
+	startTime := time.Now()
+	create_opts := volumeexpand.ExtendSizeOpts{
+		NewSize: newSize,
+	}
+	err := volumeexpand.ExtendSize(volumes.blockstorage, volumeID, create_opts).ExtractErr()
+	timeTaken := time.Since(startTime).Seconds()
+	recordOpenstackOperationMetric("expand_volume", timeTaken, err)
 	return err
 }