Cf gvisor deferred start

gardener/gardener.go

@@ -50,6 +50,7 @@ type SysInfoProvider interface {
 
 type Containerizer interface {
 	Create(log lager.Logger, desiredContainerSpec spec.DesiredContainerSpec) error
+	Start(log lager.Logger, handle string) error
 	Handles() ([]string, error)
 
 	StreamIn(log lager.Logger, handle string, streamInSpec garden.StreamInSpec) error
@@ -320,6 +321,10 @@ func (g *Gardener) Create(containerSpec garden.ContainerSpec) (ctr garden.Contai
 		return nil, err
 	}
 
+	if err = g.Containerizer.Start(log, containerSpec.Handle); err != nil {
+		return nil, err
+	}
+
 	if containerSpec.GraceTime != 0 {
 		if err := container.SetGraceTime(containerSpec.GraceTime); err != nil {
 			return nil, err

guardiancmd/command.go

@@ -191,6 +191,7 @@ type CommonCommand struct {
 	Containerd struct {
 		Socket                    string `long:"containerd-socket" description:"Path to a containerd socket."`
 		UseContainerdForProcesses bool   `long:"use-containerd-for-processes" description:"Use containerd to run processes in containers."`
+		RuntimeType               string `long:"containerd-runtime-type" description:"Containerd runtime type (e.g. io.containerd.runc.v2 or io.containerd.runsc.v1)."`
 	} `group:"Containerd"`
 
 	CPUThrottling struct {
@@ -666,15 +667,16 @@ func (cmd *CommonCommand) wireContainerizer(
 
 	peaCleaner := cmd.wirePeaCleaner(factory, volumizer, ociRuntime, peaPidGetter)
 	peaCreator = &peas.PeaCreator{
-		Volumizer:        volumizer,
-		PidGetter:        containersPidGetter,
-		PrivilegedGetter: privilegeChecker,
-		NetworkDepot:     networkDepot,
-		BundleGenerator:  template,
-		ProcessBuilder:   processBuilder,
-		BundleSaver:      bundleSaver,
-		ExecRunner:       peasExecRunner,
-		PeaCleaner:       peaCleaner,
+		Volumizer:         volumizer,
+		PidGetter:         containersPidGetter,
+		PrivilegedGetter:  privilegeChecker,
+		NetworkDepot:      networkDepot,
+		BundleGenerator:   template,
+		ProcessBuilder:    processBuilder,
+		BundleSaver:       bundleSaver,
+		ExecRunner:        peasExecRunner,
+		PeaCleaner:        peaCleaner,
+		SkipUserNamespace: cmd.Containerd.RuntimeType != "" && cmd.Containerd.RuntimeType != "io.containerd.runc.v2",
 	}
 
 	peaUsernameResolver := &peas.PeaUsernameResolver{
@@ -692,7 +694,15 @@ func (cmd *CommonCommand) wireContainerizer(
 		return nil, nil, err
 	}
 
-	return rundmc.New(depot, template, ociRuntime, nstar, processesStopper, eventStore, stateStore, peaCreator, peaUsernameResolver, cpuEntitlementPerShare, runtimeStopper, cpuCgrouper), peaCleaner, nil
+	var streamer rundmc.Streamer
+	if cmd.Containerd.RuntimeType != "" && cmd.Containerd.RuntimeType != "io.containerd.runc.v2" {
+		streamer = rundmc.NewExecStreamer(ociRuntime)
+	} else {
+		streamer = rundmc.NewNstarStreamer(ociRuntime, nstar)
+	}
+
+	execPeas := cmd.Containerd.RuntimeType != "" && cmd.Containerd.RuntimeType != "io.containerd.runc.v2"
+	return rundmc.New(depot, template, ociRuntime, streamer, processesStopper, eventStore, stateStore, peaCreator, peaUsernameResolver, cpuEntitlementPerShare, runtimeStopper, cpuCgrouper, execPeas), peaCleaner, nil
 }
 
 func (cmd *CommonCommand) useContainerd() bool {

guardiancmd/command_linux.go

@@ -140,18 +140,28 @@ func (f *LinuxFactory) WireResolvConfigurer() kawasaki.DnsResolvConfigurer {
 }
 
 func (f *LinuxFactory) WireContainerd(processBuilder *processes.ProcBuilder, userLookupper users.UserLookupper, wireExecer func(pidGetter runrunc.PidGetter) *runrunc.Execer, statser runcontainerd.Statser, log lager.Logger, volumizer peas.Volumizer, peaHandlesGetter runcontainerd.PeaHandlesGetter, metricsProvider *metrics.MetricsProvider) (*runcontainerd.RunContainerd, *runcontainerd.RunContainerPea, *runcontainerd.PidGetter, *privchecker.PrivilegeChecker, peas.BundleLoader, error) {
-	containerdClient, err := client.New(f.config.Containerd.Socket, client.WithDefaultRuntime(plugins.RuntimeRuncV2))
+	runtimeType := f.config.Containerd.RuntimeType
+	if runtimeType == "" {
+		runtimeType = plugins.RuntimeRuncV2
+	}
+	containerdClient, err := client.New(f.config.Containerd.Socket, client.WithDefaultRuntime(runtimeType))
 	if err != nil {
 		return nil, nil, nil, nil, nil, err
 	}
 	ctx := namespaces.WithNamespace(context.Background(), containerdNamespace)
 	ctx = leases.WithLease(ctx, "lease-is-off")
-	nerd := nerdpkg.New(containerdClient, ctx, filepath.Join(containerdRuncRoot(), "fifo"), metricsProvider)
+	nerd := nerdpkg.New(containerdClient, ctx, filepath.Join(containerdRuncRoot(), "fifo"), metricsProvider, runtimeType)
 	nerdStopper := nerdpkg.NewNerdStopper(containerdClient)
 	pidGetter := &runcontainerd.PidGetter{Nerd: nerd}
 
 	cgroupManager := runcontainerd.NewCgroupManager(containerdRuncRoot(), containerdNamespace)
 
+	// For non-runc runtimes (e.g. gVisor/runsc), use containerd task API for metrics
+	// instead of shelling out to runc which has no knowledge of the container.
+	if runtimeType != plugins.RuntimeRuncV2 {
+		statser = runcontainerd.NewContainerdStatser(nerd)
+	}
+
 	containerdManager := runcontainerd.New(nerd, nerd, processBuilder, userLookupper, wireExecer(pidGetter), statser, f.config.Containerd.UseContainerdForProcesses, cgroupManager, f.WireMkdirer(), peaHandlesGetter, f.config.Containers.CleanupProcessDirsOnWait, nerdStopper)
 
 	peaRunner := runcontainerd.NewRunContainerPea(containerdManager, nerd, volumizer, f.config.Containers.CleanupProcessDirsOnWait)

netplugin/external_networker.go

@@ -14,6 +14,8 @@ import (
 	"code.cloudfoundry.org/guardian/gardener"
 	"code.cloudfoundry.org/guardian/kawasaki"
 	"code.cloudfoundry.org/lager/v3"
+	"github.com/vishvananda/netlink"
+	"github.com/vishvananda/netns"
 )
 
 const NetworkPropertyPrefix = "network."
@@ -111,6 +113,15 @@ func (p *externalBinaryNetworker) Network(log lager.Logger, containerSpec garden
 		return err
 	}
 
+	// Ensure loopback interface is up in the container's network namespace.
+	// External network plugins (e.g. silk CNI) typically only create a veth pair.
+	// Runtimes like gVisor that build their own netstack by scraping the netns
+	// need lo to be present for applications binding to 127.0.0.1.
+	if err := setupLoopback(log, pid); err != nil {
+		log.Error("setup-loopback", err)
+		// Non-fatal: runc brings up lo on its own, only gVisor needs this.
+	}
+
 	for k, v := range outputs.Properties {
 		p.configStore.Set(containerSpec.Handle, k, v)
 	}
@@ -151,6 +162,36 @@ func (p *externalBinaryNetworker) Network(log lager.Logger, containerSpec garden
 	return nil
 }
 
+// setupLoopback enters the container's network namespace and brings up the
+// loopback interface. This is needed for runtimes like gVisor that scrape the
+// netns for interfaces rather than creating lo themselves.
+func setupLoopback(log lager.Logger, pid int) error {
+	containerNS, err := netns.GetFromPid(pid)
+	if err != nil {
+		return fmt.Errorf("getting netns for pid %d: %w", pid, err)
+	}
+	defer containerNS.Close()
+
+	// Create a netlink handle in the container's network namespace.
+	nlh, err := netlink.NewHandleAt(containerNS)
+	if err != nil {
+		return fmt.Errorf("creating netlink handle in container netns: %w", err)
+	}
+	defer nlh.Close()
+
+	lo, err := nlh.LinkByName("lo")
+	if err != nil {
+		return fmt.Errorf("finding lo interface: %w", err)
+	}
+
+	if err := nlh.LinkSetUp(lo); err != nil {
+		return fmt.Errorf("bringing up lo: %w", err)
+	}
+
+	log.Info("setup-loopback-success", lager.Data{"pid": pid})
+	return nil
+}
+
 func (p *externalBinaryNetworker) Destroy(log lager.Logger, handle string) error {
 	err := p.exec(log, "down", handle, nil, nil)
 	if err != nil {

rundmc/containerizer.go

@@ -53,6 +53,7 @@ type State struct {
 
 type OCIRuntime interface {
 	Create(log lager.Logger, id string, bundle goci.Bndl, io garden.ProcessIO) error
+	Start(log lager.Logger, id string) error
 	Exec(log lager.Logger, id string, spec garden.ProcessSpec, io garden.ProcessIO) (garden.Process, error)
 	Attach(log lager.Logger, id, processId string, io garden.ProcessIO) (garden.Process, error)
 	Delete(log lager.Logger, id string) error
@@ -108,21 +109,22 @@ type Containerizer struct {
 	bundler                BundleGenerator
 	runtime                OCIRuntime
 	processesStopper       ProcessesStopper
-	nstar                  NstarRunner
+	streamer               Streamer
 	events                 EventStore
 	states                 StateStore
 	peaCreator             PeaCreator
 	peaUsernameResolver    PeaUsernameResolver
 	cpuEntitlementPerShare float64
 	runtimeStopper         RuntimeStopper
 	cpuCgrouper            CPUCgrouper
+	execPeas               bool
 }
 
 func New(
 	depot Depot,
 	bundler BundleGenerator,
 	runtime OCIRuntime,
-	nstarRunner NstarRunner,
+	streamer Streamer,
 	processesStopper ProcessesStopper,
 	events EventStore,
 	states StateStore,
@@ -131,12 +133,13 @@ func New(
 	cpuEntitlementPerShare float64,
 	runtimeStopper RuntimeStopper,
 	cpuCgrouper CPUCgrouper,
+	execPeas bool,
 ) *Containerizer {
 	containerizer := &Containerizer{
 		depot:                  depot,
 		bundler:                bundler,
 		runtime:                runtime,
-		nstar:                  nstarRunner,
+		streamer:               streamer,
 		processesStopper:       processesStopper,
 		events:                 events,
 		states:                 states,
@@ -145,6 +148,7 @@ func New(
 		cpuEntitlementPerShare: cpuEntitlementPerShare,
 		runtimeStopper:         runtimeStopper,
 		cpuCgrouper:            cpuCgrouper,
+		execPeas:               execPeas,
 	}
 	return containerizer
 }
@@ -192,6 +196,11 @@ func (c *Containerizer) Create(log lager.Logger, spec spec.DesiredContainerSpec)
 	return nil
 }
 
+func (c *Containerizer) Start(log lager.Logger, handle string) error {
+	log = log.Session("containerizer-start", lager.Data{"handle": handle})
+	return c.runtime.Start(log, handle)
+}
+
 // Run runs a process inside a running container
 func (c *Containerizer) Run(log lager.Logger, handle string, spec garden.ProcessSpec, io garden.ProcessIO) (garden.Process, error) {
 	log = log.Session("run", lager.Data{"handle": handle, "path": spec.Path})
@@ -200,6 +209,16 @@ func (c *Containerizer) Run(log lager.Logger, handle string, spec garden.Process
 	defer log.Info("finished")
 
 	if isPea(spec) {
+		if c.execPeas {
+			// On gVisor, peas (sidecar containers) cannot share namespaces across
+			// sandboxes. Run the process as exec inside the existing container instead.
+			// The required binaries (healthcheck, etc.) are already bind-mounted in.
+			log.Info("exec-pea-as-process", lager.Data{"path": spec.Path})
+			spec.Image = garden.ImageRef{}
+			spec.BindMounts = nil
+			return c.runtime.Exec(log, handle, spec, io)
+		}
+
 		if shouldResolveUsername(spec.User) {
 			resolvedUID, resolvedGID, err := c.peaUsernameResolver.ResolveUser(log, handle, spec.Image, spec.User)
 			if err != nil {
@@ -248,18 +267,7 @@ func (c *Containerizer) StreamIn(log lager.Logger, handle string, spec garden.St
 		_ = metrics.SendValue("StreamInDuration", float64(time.Since(startedAt).Nanoseconds()), "nanos")
 	}(time.Now())
 
-	state, err := c.runtime.State(log, handle)
-	if err != nil {
-		log.Error("check-pid-failed", err)
-		return fmt.Errorf("stream-in: pid not found for container")
-	}
-
-	if err := c.nstar.StreamIn(log, state.Pid, spec.Path, spec.User, spec.TarStream); err != nil {
-		log.Error("nstar-failed", err)
-		return fmt.Errorf("stream-in: nstar: %s", err)
-	}
-
-	return nil
+	return c.streamer.StreamIn(log, handle, spec)
 }
 
 // StreamOut stream files from the container
@@ -269,19 +277,7 @@ func (c *Containerizer) StreamOut(log lager.Logger, handle string, spec garden.S
 	log.Info("started")
 	defer log.Info("finished")
 
-	state, err := c.runtime.State(log, handle)
-	if err != nil {
-		log.Error("check-pid-failed", err)
-		return nil, fmt.Errorf("stream-out: pid not found for container")
-	}
-
-	stream, err := c.nstar.StreamOut(log, state.Pid, spec.Path, spec.User)
-	if err != nil {
-		log.Error("nstar-failed", err)
-		return nil, fmt.Errorf("stream-out: nstar: %s", err)
-	}
-
-	return stream, nil
+	return c.streamer.StreamOut(log, handle, spec)
 }
 
 // Stop stops all the processes other than the init process in the container