Use volumes when allocating an openstack server

spread/export_openstack_test.go

@@ -9,7 +9,9 @@ import (
 )
 
 var (
-	OpenStackName = openstackName
+	OpenStackName          = openstackName
+	OpenstackDefaultFlavor = openstackDefaultFlavor
+	OpenstackReadyMarker   = openstackReadyMarker
 )
 
 func FakeOpenStackImageClient(p Provider, imageClient glanceImageClient) (restore func()) {
@@ -39,6 +41,11 @@ func FakeOpenStackGooseClient(p Provider, gooseClient gooseclient.Client) (resto
 	}
 }
 
+func FakeOpenStackNeutronClient(p Provider, nc neutronClient) {
+	osp := p.(*openstackProvider)
+	osp.networkClient = nc
+}
+
 func FakeOpenStackProvisionTimeout(timeout, retry time.Duration) (restore func()) {
 	oldTimeout := openstackProvisionTimeout
 	oldRetry := openstackProvisionRetry

spread/openstack.go

@@ -43,6 +43,11 @@ type novaComputeClient interface {
 	DeleteServer(serverId string) error
 }
 
+type neutronClient interface {
+	ListNetworksV2(filter ...*neutron.Filter) ([]neutron.NetworkV2, error)
+	ListSecurityGroupsV2() ([]neutron.SecurityGroupV2, error)
+}
+
 type openstackProvider struct {
 	project *Project
 	backend *Backend
@@ -51,7 +56,7 @@ type openstackProvider struct {
 	region        string
 	osClient      gooseclient.Client
 	computeClient novaComputeClient
-	networkClient *neutron.Client
+	networkClient neutronClient
 	imageClient   glanceImageClient
 
 	mu sync.Mutex
@@ -593,6 +598,12 @@ func (p *openstackProvider) createMachine(ctx context.Context, system *System) (
 		"password": p.options.Password,
 	}
 
+	// halt-timeout is added to the server to determine the time when it
+	// has to be garbage collected
+	if p.backend.HaltTimeout.Duration != 0 {
+		tags["halt-timeout"] = p.backend.HaltTimeout.Duration.String()
+	}
+
 	opts := nova.RunServerOpts{
 		Name:             name,
 		FlavorId:         flavor.Id,
@@ -603,6 +614,35 @@ func (p *openstackProvider) createMachine(ctx context.Context, system *System) (
 		UserData:         []byte(cloudconfig),
 	}
 
+	// When the storage size is defined, then we use the volume generated
+	// with the source image. Otherwise we boot in an ephemeral disk the
+	// image using the size described in the flavor
+	storage := image.MinimumDisk
+	if system.Storage != 0 {
+		storage = int(system.Storage / gb)
+	}
+	// We use 20 GB as default value for the storage size
+	if storage == 0 {
+		storage = 20
+	}
+
+	// By default volumes are deleted on termination
+	deleteOnTermination := false
+	if p.backend.VolumeAutoDelete == nil {
+		deleteOnTermination = true
+	} else {
+		deleteOnTermination = *p.backend.VolumeAutoDelete
+	}
+
+	opts.BlockDeviceMappings = []nova.BlockDeviceMapping{{
+		BootIndex:           0,
+		SourceType:          "image",
+		DestinationType:     "volume",
+		VolumeSize:          storage,
+		UUID:                image.Id,
+		DeleteOnTermination: deleteOnTermination,
+	}}
+
 	if len(system.Groups) > 0 {
 		sgNames, err := p.findSecurityGroupNames(system.Groups)
 		if err != nil {

spread/openstack_test.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/go-goose/goose/v5/glance"
 	goosehttp "github.com/go-goose/goose/v5/http"
+	"github.com/go-goose/goose/v5/neutron"
 	"github.com/go-goose/goose/v5/nova"
 	"golang.org/x/crypto/ssh"
 
@@ -119,6 +120,25 @@ func (cc *fakeNovaComputeClient) DeleteServer(serverId string) error {
 	return cc.deleteServer(serverId)
 }
 
+type fakeNeutronClient struct {
+	listNetworksV2       func(filter ...*neutron.Filter) ([]neutron.NetworkV2, error)
+	listSecurityGroupsV2 func() ([]neutron.SecurityGroupV2, error)
+}
+
+func (f *fakeNeutronClient) ListNetworksV2(filter ...*neutron.Filter) ([]neutron.NetworkV2, error) {
+	if f.listNetworksV2 != nil {
+		return f.listNetworksV2()
+	}
+	return nil, nil
+}
+
+func (f *fakeNeutronClient) ListSecurityGroupsV2() ([]neutron.SecurityGroupV2, error) {
+	if f.listSecurityGroupsV2 != nil {
+		return f.listSecurityGroupsV2()
+	}
+	return nil, nil
+}
+
 type fakeOsClientRequest struct {
 	method      string
 	svcType     string
@@ -163,6 +183,7 @@ type openstackFindImageSuite struct {
 	fakeImageClient   *fakeGlanceImageClient
 	fakeComputeClient *fakeNovaComputeClient
 	fakeOsClient      *fakeOsClient
+	fakeNeutronClient *fakeNeutronClient
 }
 
 var _ = Suite(&openstackFindImageSuite{})
@@ -173,10 +194,12 @@ func (s *openstackFindImageSuite) SetUpTest(c *C) {
 	s.fakeImageClient = &fakeGlanceImageClient{}
 	s.fakeComputeClient = &fakeNovaComputeClient{}
 	s.fakeOsClient = &fakeOsClient{}
+	s.fakeNeutronClient = &fakeNeutronClient{}
 
 	spread.FakeOpenStackImageClient(s.opst, s.fakeImageClient)
 	spread.FakeOpenStackComputeClient(s.opst, s.fakeComputeClient)
 	spread.FakeOpenStackGooseClient(s.opst, s.fakeOsClient)
+	spread.FakeOpenStackNeutronClient(s.opst, s.fakeNeutronClient)
 }
 
 func (s *openstackFindImageSuite) TestOpenStackFindImageNotFound(c *C) {
@@ -464,3 +487,95 @@ func (s *openstackFindImageSuite) TestOpenStackWaitServerBootSSHTimeout(c *C) {
 	err := spread.OpenStackWaitServerBoot(s.opst, context.TODO(), "test-id", "test-server", []string{"net-1"})
 	c.Check(err, ErrorMatches, "cannot connect to server test-server: cannot ssh to the allocated instance: timeout reached")
 }
+
+func (s *openstackFindImageSuite) TestOpenStackCreateMachineVolumeAndOpts(c *C) {
+	// flavors
+	s.fakeComputeClient.listFlavors = func() ([]nova.Entity, error) {
+		return []nova.Entity{{Id: "f1", Name: spread.OpenstackDefaultFlavor}}, nil
+	}
+
+	// availability zones returned
+	s.fakeComputeClient.listAvailabilityZones = func() ([]nova.AvailabilityZone, error) {
+		return []nova.AvailabilityZone{{Name: "zone-1"}}, nil
+	}
+
+	// image with minimal disk
+	imgID := "i1"
+	s.fakeImageClient.res = []glance.ImageDetail{
+		{Id: imgID, Name: "ubuntu-20.04", Status: "ACTIVE", MinimumDisk: 1},
+	}
+
+	// network
+	s.fakeNeutronClient.listNetworksV2 = func(filter ...*neutron.Filter) ([]neutron.NetworkV2, error) {
+		return []neutron.NetworkV2{{Id: "net-id1", Name: "net-1"}}, nil
+	}
+
+	s.fakeNeutronClient.listSecurityGroupsV2 = func() ([]neutron.SecurityGroupV2, error) {
+		return []neutron.SecurityGroupV2{{Id: "sg1", Name: "default"}}, nil
+	}
+
+	// capture the RunServer opts
+	var capturedOpts nova.RunServerOpts
+	s.fakeComputeClient.runServer = func(opts nova.RunServerOpts) (*nova.Entity, error) {
+		capturedOpts = opts
+		return &nova.Entity{Id: "srv1"}, nil
+	}
+
+	// getServer: BUILD then ACTIVE with addresses
+	call := 0
+	s.fakeComputeClient.getServer = func(id string) (*nova.ServerDetail, error) {
+		call++
+		if call == 1 {
+			return &nova.ServerDetail{Id: id, Status: nova.StatusBuild}, nil
+		}
+		return &nova.ServerDetail{
+			Id:     id,
+			Status: nova.StatusActive,
+			Addresses: map[string][]nova.IPAddress{
+				"net-1": {
+					{Address: "1.2.3.4", Version: 4},
+				},
+			},
+		}, nil
+	}
+
+	// ensure serial console returns ready immediately
+	s.fakeOsClient.response = func() interface{} {
+		return map[string]string{"output": spread.OpenstackReadyMarker}
+	}
+
+	// Request a storage of 50 GB (in bytes) and a security group
+	sys := &spread.System{
+		Name:     "test",
+		Image:    "ubuntu-20.04",
+		Networks: []string{"net-1"},
+		Storage:  50 * 1024 * 1024 * 1024,
+		Groups:   []string{"default"},
+	}
+
+	srv, err := s.opst.Allocate(context.TODO(), sys)
+	c.Assert(err, IsNil)
+	c.Assert(srv, NotNil)
+
+	// Verify block device mapping attached with expected volume size and properties.
+	c.Assert(len(capturedOpts.BlockDeviceMappings), Equals, 1)
+	bdm := capturedOpts.BlockDeviceMappings[0]
+	c.Check(bdm.UUID, Equals, imgID)
+	c.Check(bdm.VolumeSize, Equals, 50)
+	c.Check(bdm.DestinationType, Equals, "volume")
+	c.Check(bdm.SourceType, Equals, "image")
+	c.Check(bdm.BootIndex, Equals, 0)
+	// Default behaviour when VolumeAutoDelete is nil is to delete on termination
+	c.Check(bdm.DeleteOnTermination, Equals, true)
+
+	// Check other RunServer options
+	c.Check(capturedOpts.FlavorId, Equals, "f1")
+	c.Check(capturedOpts.ImageId, Equals, imgID)
+	c.Check(len(capturedOpts.Networks), Equals, 1)
+	c.Check(capturedOpts.Networks[0].NetworkId, Equals, "net-id1")
+	c.Check(capturedOpts.AvailabilityZone, Equals, "zone-1")
+
+	// Metadata contains expected keys
+	c.Check(capturedOpts.Metadata["spread"], Equals, "true")
+	c.Check(capturedOpts.Metadata["reuse"], Equals, "false")
+}

spread/project.go

@@ -65,10 +65,11 @@ type Backend struct {
 	Storage  Size
 
 	// Only for OpenStack so far
-	Account  string
-	Endpoint string
-	Networks []string
-	Groups   []string
+	Account          string
+	Endpoint         string
+	Networks         []string
+	Groups           []string
+	VolumeAutoDelete *bool `yaml:"volume-auto-delete"`
 
 	Systems SystemsMap