From 41ad7e83891dd710a9d1ae3fad3dba7aea9001d7 Mon Sep 17 00:00:00 2001 From: Vladimir Savchenko Date: Wed, 27 May 2026 10:18:02 +0300 Subject: [PATCH 01/16] Split container Create/Start for gVisor network timing gVisor network=sandbox requires the netns to be configured BEFORE task.Start() because setupNetwork() reads the netns at start time and mirrors it into netstack. Previously Start was called before silk-cni configured networking, leaving gVisor with empty netstack. Split the lifecycle: Create only boots the sandbox (runsc create), then Start is called after Networker.Network() configures the netns. --- gardener/gardener.go | 5 +++++ rundmc/containerizer.go | 6 ++++++ rundmc/runcontainerd/nerd/nerd.go | 10 +++++++--- rundmc/runcontainerd/runcontainerd.go | 5 +++++ 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/gardener/gardener.go b/gardener/gardener.go index 0592d4372..f72f7b934 100644 --- a/gardener/gardener.go +++ b/gardener/gardener.go @@ -50,6 +50,7 @@ type SysInfoProvider interface { type Containerizer interface { Create(log lager.Logger, desiredContainerSpec spec.DesiredContainerSpec) error + Start(log lager.Logger, handle string) error Handles() ([]string, error) StreamIn(log lager.Logger, handle string, streamInSpec garden.StreamInSpec) error @@ -320,6 +321,10 @@ func (g *Gardener) Create(containerSpec garden.ContainerSpec) (ctr garden.Contai return nil, err } + if err = g.Containerizer.Start(log, containerSpec.Handle); err != nil { + return nil, err + } + if containerSpec.GraceTime != 0 { if err := container.SetGraceTime(containerSpec.GraceTime); err != nil { return nil, err diff --git a/rundmc/containerizer.go b/rundmc/containerizer.go index 910b7bcf3..f2153bb83 100644 --- a/rundmc/containerizer.go +++ b/rundmc/containerizer.go @@ -53,6 +53,7 @@ type State struct { type OCIRuntime interface { Create(log lager.Logger, id string, bundle goci.Bndl, io garden.ProcessIO) error + Start(log lager.Logger, id string) error Exec(log lager.Logger, id string, spec garden.ProcessSpec, io garden.ProcessIO) (garden.Process, error) Attach(log lager.Logger, id, processId string, io garden.ProcessIO) (garden.Process, error) Delete(log lager.Logger, id string) error @@ -192,6 +193,11 @@ func (c *Containerizer) Create(log lager.Logger, spec spec.DesiredContainerSpec) return nil } +func (c *Containerizer) Start(log lager.Logger, handle string) error { + log = log.Session("containerizer-start", lager.Data{"handle": handle}) + return c.runtime.Start(log, handle) +} + // Run runs a process inside a running container func (c *Containerizer) Run(log lager.Logger, handle string, spec garden.ProcessSpec, io garden.ProcessIO) (garden.Process, error) { log = log.Session("run", lager.Data{"handle": handle, "path": spec.Path}) diff --git a/rundmc/runcontainerd/nerd/nerd.go b/rundmc/runcontainerd/nerd/nerd.go index 02a8eebe6..45d2428a0 100644 --- a/rundmc/runcontainerd/nerd/nerd.go +++ b/rundmc/runcontainerd/nerd/nerd.go @@ -58,14 +58,18 @@ func (n *Nerd) Create(log lager.Logger, containerID string, spec *specs.Spec, ho } log.Debug("creating-task", lager.Data{"containerID": containerID}) - task, err := container.NewTask(n.context, cio.NewCreator(withProcessIO(noTTYProcessIO(pio), n.ioFifoDir)), client.WithNoNewKeyring, WithUIDAndGID(hostUID, hostGID)) + _, err = container.NewTask(n.context, cio.NewCreator(withProcessIO(noTTYProcessIO(pio), n.ioFifoDir)), client.WithNoNewKeyring, WithUIDAndGID(hostUID, hostGID)) + return err +} + +func (n *Nerd) Start(log lager.Logger, containerID string) error { + _, task, err := n.loadContainerAndTask(log, containerID) if err != nil { return err } log.Debug("starting-task", lager.Data{"containerID": containerID}) - err = task.Start(n.context) - if err != nil { + if err := task.Start(n.context); err != nil { return err } diff --git a/rundmc/runcontainerd/runcontainerd.go b/rundmc/runcontainerd/runcontainerd.go index 1d534c483..f3a76d4d8 100644 --- a/rundmc/runcontainerd/runcontainerd.go +++ b/rundmc/runcontainerd/runcontainerd.go @@ -25,6 +25,7 @@ import ( //counterfeiter:generate . ContainerManager type ContainerManager interface { Create(log lager.Logger, containerID string, spec *specs.Spec, containerRootUID, containerRootGID uint32, processIO func() (io.Reader, io.Writer, io.Writer)) error + Start(log lager.Logger, containerID string) error Delete(log lager.Logger, containerID string) error Exec(log lager.Logger, containerID, processID string, spec *specs.Process, processIO func() (io.Reader, io.Writer, io.Writer, bool)) (BackingProcess, error) State(log lager.Logger, containerID string) (int, string, error) @@ -151,6 +152,10 @@ func (r *RunContainerd) Create(log lager.Logger, id string, bundle goci.Bndl, pi return nil } +func (r *RunContainerd) Start(log lager.Logger, id string) error { + return r.containerManager.Start(log, id) +} + func updateAnnotationsIfNeeded(bundle *goci.Bndl) { if _, ok := bundle.Spec.Annotations["container-type"]; !ok { if bundle.Spec.Annotations == nil { From 18bd18a5cba0dbed642f535590d74555acf2970d Mon Sep 17 00:00:00 2001 From: Vladimir Savchenko Date: Wed, 27 May 2026 12:50:22 +0300 Subject: [PATCH 02/16] Add configurable containerd runtime type for gVisor support Wire a --containerd-runtime-type flag through to the containerd client and Nerd layer. When set to a non-runc runtime (e.g. io.containerd.runsc.v1), Create skips runc-specific task options (WithNoNewKeyring, WithUIDAndGID) that runsc rejects, and nils spec.Windows which gVisor does not support. Also adds a no-op Start() to RunRunc for OCIRuntime interface compliance after the deferred-start split. --- guardiancmd/command.go | 1 + guardiancmd/command_linux.go | 8 ++++++-- rundmc/runcontainerd/nerd/nerd.go | 30 ++++++++++++++++++++---------- rundmc/runrunc/runrunc.go | 4 ++++ 4 files changed, 31 insertions(+), 12 deletions(-) diff --git a/guardiancmd/command.go b/guardiancmd/command.go index 78e930ca6..6b97f9c00 100644 --- a/guardiancmd/command.go +++ b/guardiancmd/command.go @@ -191,6 +191,7 @@ type CommonCommand struct { Containerd struct { Socket string `long:"containerd-socket" description:"Path to a containerd socket."` UseContainerdForProcesses bool `long:"use-containerd-for-processes" description:"Use containerd to run processes in containers."` + RuntimeType string `long:"containerd-runtime-type" description:"Containerd runtime type (e.g. io.containerd.runc.v2 or io.containerd.runsc.v1)."` } `group:"Containerd"` CPUThrottling struct { diff --git a/guardiancmd/command_linux.go b/guardiancmd/command_linux.go index edfa49b72..c664f97b3 100644 --- a/guardiancmd/command_linux.go +++ b/guardiancmd/command_linux.go @@ -140,13 +140,17 @@ func (f *LinuxFactory) WireResolvConfigurer() kawasaki.DnsResolvConfigurer { } func (f *LinuxFactory) WireContainerd(processBuilder *processes.ProcBuilder, userLookupper users.UserLookupper, wireExecer func(pidGetter runrunc.PidGetter) *runrunc.Execer, statser runcontainerd.Statser, log lager.Logger, volumizer peas.Volumizer, peaHandlesGetter runcontainerd.PeaHandlesGetter, metricsProvider *metrics.MetricsProvider) (*runcontainerd.RunContainerd, *runcontainerd.RunContainerPea, *runcontainerd.PidGetter, *privchecker.PrivilegeChecker, peas.BundleLoader, error) { - containerdClient, err := client.New(f.config.Containerd.Socket, client.WithDefaultRuntime(plugins.RuntimeRuncV2)) + runtimeType := f.config.Containerd.RuntimeType + if runtimeType == "" { + runtimeType = plugins.RuntimeRuncV2 + } + containerdClient, err := client.New(f.config.Containerd.Socket, client.WithDefaultRuntime(runtimeType)) if err != nil { return nil, nil, nil, nil, nil, err } ctx := namespaces.WithNamespace(context.Background(), containerdNamespace) ctx = leases.WithLease(ctx, "lease-is-off") - nerd := nerdpkg.New(containerdClient, ctx, filepath.Join(containerdRuncRoot(), "fifo"), metricsProvider) + nerd := nerdpkg.New(containerdClient, ctx, filepath.Join(containerdRuncRoot(), "fifo"), metricsProvider, runtimeType) nerdStopper := nerdpkg.NewNerdStopper(containerdClient) pidGetter := &runcontainerd.PidGetter{Nerd: nerd} diff --git a/rundmc/runcontainerd/nerd/nerd.go b/rundmc/runcontainerd/nerd/nerd.go index 45d2428a0..ff4eba82f 100644 --- a/rundmc/runcontainerd/nerd/nerd.go +++ b/rundmc/runcontainerd/nerd/nerd.go @@ -19,6 +19,7 @@ import ( apievents "github.com/containerd/containerd/api/events" v2types "github.com/containerd/containerd/api/types/runc/options" "github.com/containerd/containerd/v2/client" + "github.com/containerd/containerd/v2/plugins" ctrdevents "github.com/containerd/containerd/v2/core/events" "github.com/containerd/containerd/v2/pkg/cio" "github.com/containerd/errdefs" @@ -28,23 +29,28 @@ import ( ) type Nerd struct { - client *client.Client - context context.Context - ioFifoDir string - mp *metrics.MetricsProvider + client *client.Client + context context.Context + ioFifoDir string + mp *metrics.MetricsProvider + runtimeType string } -func New(client *client.Client, context context.Context, ioFifoDir string, mp *metrics.MetricsProvider) *Nerd { +func New(client *client.Client, context context.Context, ioFifoDir string, mp *metrics.MetricsProvider, runtimeType string) *Nerd { return &Nerd{ - client: client, - context: context, - ioFifoDir: ioFifoDir, - mp: mp, + client: client, + context: context, + ioFifoDir: ioFifoDir, + mp: mp, + runtimeType: runtimeType, } } func (n *Nerd) Create(log lager.Logger, containerID string, spec *specs.Spec, hostUID, hostGID uint32, pio func() (io.Reader, io.Writer, io.Writer)) error { log.Debug("creating-container", lager.Data{"containerID": containerID}) + if n.runtimeType != "" && n.runtimeType != plugins.RuntimeRuncV2 { + spec.Windows = nil + } container, err := n.client.NewContainer(n.context, containerID, client.WithSpec(spec)) if err != nil { return err @@ -58,7 +64,11 @@ func (n *Nerd) Create(log lager.Logger, containerID string, spec *specs.Spec, ho } log.Debug("creating-task", lager.Data{"containerID": containerID}) - _, err = container.NewTask(n.context, cio.NewCreator(withProcessIO(noTTYProcessIO(pio), n.ioFifoDir)), client.WithNoNewKeyring, WithUIDAndGID(hostUID, hostGID)) + var taskOpts []client.NewTaskOpts + if n.runtimeType == "" || n.runtimeType == plugins.RuntimeRuncV2 { + taskOpts = append(taskOpts, client.WithNoNewKeyring, WithUIDAndGID(hostUID, hostGID)) + } + _, err = container.NewTask(n.context, cio.NewCreator(withProcessIO(noTTYProcessIO(pio), n.ioFifoDir)), taskOpts...) return err } diff --git a/rundmc/runrunc/runrunc.go b/rundmc/runrunc/runrunc.go index 7385e0058..80d1f30d6 100644 --- a/rundmc/runrunc/runrunc.go +++ b/rundmc/runrunc/runrunc.go @@ -60,3 +60,7 @@ func New( BundleManager: bundleManager, } } + +func (r *RunRunc) Start(log lager.Logger, id string) error { + return nil +} From 2d6d38a8901457f265fb2a163a19fd4190cb6b06 Mon Sep 17 00:00:00 2001 From: Vladimir Savchenko Date: Wed, 27 May 2026 12:50:55 +0300 Subject: [PATCH 03/16] Use bundle rootfs for user lookup instead of /proc/pid/root For gVisor containers, the sentry process PID on the host does not expose the container filesystem via /proc//root (the sentry runs in its own mount namespace). Look up users against bundle.Spec.Root.Path (the host-side rootfs) which is always accessible regardless of runtime type. --- rundmc/runcontainerd/runcontainerd.go | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/rundmc/runcontainerd/runcontainerd.go b/rundmc/runcontainerd/runcontainerd.go index f3a76d4d8..7f02f58c4 100644 --- a/rundmc/runcontainerd/runcontainerd.go +++ b/rundmc/runcontainerd/runcontainerd.go @@ -4,9 +4,7 @@ import ( "fmt" "io" "os" - "path/filepath" "regexp" - "strconv" "code.cloudfoundry.org/guardian/rundmc" @@ -175,18 +173,13 @@ func (r *RunContainerd) Exec(log lager.Logger, containerID string, gardenProcess return r.execer.ExecWithBndl(log, containerID, bundle, gardenProcessSpec, gardenIO) } - containerPid, err := r.containerManager.GetContainerPID(log, containerID) - if err != nil { - return nil, err - } + rootfsPath := bundle.Spec.Root.Path - resolvedUser, err := r.userLookupper.Lookup(fmt.Sprintf("/proc/%d/root", containerPid), gardenProcessSpec.User) + resolvedUser, err := r.userLookupper.Lookup(rootfsPath, gardenProcessSpec.User) if err != nil { return nil, err } - rootfsPath := filepath.Join("/proc", strconv.FormatInt(int64(containerPid), 10), "root") - hostUID := idmapper.MappingList(bundle.Spec.Linux.UIDMappings).Map(resolvedUser.Uid) hostGID := idmapper.MappingList(bundle.Spec.Linux.GIDMappings).Map(resolvedUser.Gid) From 6d06bafba5ffa10c956fe9c2a646c4d1ead54232 Mon Sep 17 00:00:00 2001 From: Vladimir Savchenko Date: Wed, 27 May 2026 12:51:08 +0300 Subject: [PATCH 04/16] Tolerate missing state.json in SetUseMemoryHierarchy Non-runc runtimes (e.g. runsc/gVisor) do not write state.json to the runc root directory. On cgroups v2 the memory.use_hierarchy knob does not exist either. Treat a missing state.json as a no-op instead of returning an error that would block container creation. --- rundmc/runcontainerd/cgroup_manager.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rundmc/runcontainerd/cgroup_manager.go b/rundmc/runcontainerd/cgroup_manager.go index ce0b35dec..17e9bc51f 100644 --- a/rundmc/runcontainerd/cgroup_manager.go +++ b/rundmc/runcontainerd/cgroup_manager.go @@ -38,6 +38,9 @@ func NewCgroupManager(runcRoot, namespace string) CgroupManager { func (m cgroupManager) SetUseMemoryHierarchy(handle string) error { statePath := filepath.Join(m.runcRoot, m.namespace, handle, "state.json") + if _, statErr := os.Stat(statePath); os.IsNotExist(statErr) { + return nil + } stateFile, err := os.Open(statePath) if err != nil { return err From e1454dd8ba0cae0ee24a0ab00dc135976eefe4c4 Mon Sep 17 00:00:00 2001 From: Vladimir Savchenko Date: Wed, 27 May 2026 12:52:27 +0300 Subject: [PATCH 05/16] Exec-based StreamIn/StreamOut for gVisor containers nstar uses nsenter to enter the container mount namespace, which does not work with gVisor because the sentry PID on the host has a different mount namespace that does not expose the container filesystem. For non-runc runtimes, use runtime.Exec to run tar inside the container through the containerd -> shim -> runsc exec path. This goes through the sentry and keeps its dentry cache consistent with the filesystem state. StreamIn: exec /bin/tar -xf - -C with TarStream on stdin. StreamOut: exec /bin/tar -cf - -C with stdout piped back. Detection is based on the configured runtimeType (from the --containerd-runtime-type flag) rather than runtime probing. --- guardiancmd/command.go | 2 +- rundmc/containerizer.go | 89 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 1 deletion(-) diff --git a/guardiancmd/command.go b/guardiancmd/command.go index 6b97f9c00..72bde8918 100644 --- a/guardiancmd/command.go +++ b/guardiancmd/command.go @@ -693,7 +693,7 @@ func (cmd *CommonCommand) wireContainerizer( return nil, nil, err } - return rundmc.New(depot, template, ociRuntime, nstar, processesStopper, eventStore, stateStore, peaCreator, peaUsernameResolver, cpuEntitlementPerShare, runtimeStopper, cpuCgrouper), peaCleaner, nil + return rundmc.New(depot, template, ociRuntime, nstar, processesStopper, eventStore, stateStore, peaCreator, peaUsernameResolver, cpuEntitlementPerShare, runtimeStopper, cpuCgrouper, cmd.Containerd.RuntimeType), peaCleaner, nil } func (cmd *CommonCommand) useContainerd() bool { diff --git a/rundmc/containerizer.go b/rundmc/containerizer.go index f2153bb83..cc7c7b989 100644 --- a/rundmc/containerizer.go +++ b/rundmc/containerizer.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "os" + "path/filepath" "strconv" "strings" "time" @@ -117,6 +118,7 @@ type Containerizer struct { cpuEntitlementPerShare float64 runtimeStopper RuntimeStopper cpuCgrouper CPUCgrouper + runtimeType string } func New( @@ -132,6 +134,7 @@ func New( cpuEntitlementPerShare float64, runtimeStopper RuntimeStopper, cpuCgrouper CPUCgrouper, + runtimeType string, ) *Containerizer { containerizer := &Containerizer{ depot: depot, @@ -146,6 +149,7 @@ func New( cpuEntitlementPerShare: cpuEntitlementPerShare, runtimeStopper: runtimeStopper, cpuCgrouper: cpuCgrouper, + runtimeType: runtimeType, } return containerizer } @@ -254,6 +258,10 @@ func (c *Containerizer) StreamIn(log lager.Logger, handle string, spec garden.St _ = metrics.SendValue("StreamInDuration", float64(time.Since(startedAt).Nanoseconds()), "nanos") }(time.Now()) + if c.runtimeType != "" && c.runtimeType != "io.containerd.runc.v2" { + return c.streamInViaExec(log, handle, spec) + } + state, err := c.runtime.State(log, handle) if err != nil { log.Error("check-pid-failed", err) @@ -268,6 +276,39 @@ func (c *Containerizer) StreamIn(log lager.Logger, handle string, spec garden.St return nil } +func (c *Containerizer) streamInViaExec(log lager.Logger, handle string, spec garden.StreamInSpec) error { + processSpec := garden.ProcessSpec{ + Path: "/bin/tar", + Args: []string{"-xf", "-", "-C", spec.Path}, + User: spec.User, + } + if processSpec.User == "" { + processSpec.User = "root" + } + + processIO := garden.ProcessIO{ + Stdin: spec.TarStream, + Stdout: io.Discard, + Stderr: io.Discard, + } + + process, err := c.runtime.Exec(log, handle, processSpec, processIO) + if err != nil { + log.Error("exec-tar-failed", err) + return fmt.Errorf("stream-in: exec tar: %s", err) + } + + exitCode, err := process.Wait() + if err != nil { + return fmt.Errorf("stream-in: exec tar wait: %s", err) + } + if exitCode != 0 { + return fmt.Errorf("stream-in: exec tar exited %d", exitCode) + } + + return nil +} + // StreamOut stream files from the container func (c *Containerizer) StreamOut(log lager.Logger, handle string, spec garden.StreamOutSpec) (io.ReadCloser, error) { log = log.Session("stream-out", lager.Data{"handle": handle}) @@ -275,6 +316,10 @@ func (c *Containerizer) StreamOut(log lager.Logger, handle string, spec garden.S log.Info("started") defer log.Info("finished") + if c.runtimeType != "" && c.runtimeType != "io.containerd.runc.v2" { + return c.streamOutViaExec(log, handle, spec) + } + state, err := c.runtime.State(log, handle) if err != nil { log.Error("check-pid-failed", err) @@ -290,6 +335,50 @@ func (c *Containerizer) StreamOut(log lager.Logger, handle string, spec garden.S return stream, nil } +func (c *Containerizer) streamOutViaExec(log lager.Logger, handle string, spec garden.StreamOutSpec) (io.ReadCloser, error) { + path := spec.Path + sourcePath := filepath.Dir(path) + compressPath := filepath.Base(path) + if strings.HasSuffix(path, "/") { + sourcePath = path + compressPath = "." + } + + reader, writer, err := os.Pipe() + if err != nil { + return nil, err + } + + processSpec := garden.ProcessSpec{ + Path: "/bin/tar", + Args: []string{"-cf", "-", "-C", sourcePath, compressPath}, + User: spec.User, + } + if processSpec.User == "" { + processSpec.User = "root" + } + + processIO := garden.ProcessIO{ + Stdout: writer, + Stderr: io.Discard, + } + + process, err := c.runtime.Exec(log, handle, processSpec, processIO) + if err != nil { + writer.Close() + reader.Close() + log.Error("exec-tar-failed", err) + return nil, fmt.Errorf("stream-out: exec tar: %s", err) + } + + go func() { + process.Wait() + writer.Close() + }() + + return reader, nil +} + // Stop stops all the processes other than the init process in the container func (c *Containerizer) Stop(log lager.Logger, handle string, kill bool) error { log = log.Session("stop", lager.Data{"handle": handle, "kill": kill}) From c753dcc6e02251c1fe488aaa8a83470fe2d1f9b1 Mon Sep 17 00:00:00 2001 From: Vladimir Savchenko Date: Wed, 27 May 2026 13:11:57 +0300 Subject: [PATCH 06/16] Create target directory before exec tar in StreamIn The exec-based StreamIn runs tar -xf inside the container, but the target directory (e.g. /tmp/app) may not exist in a freshly created container. The nstar-based approach creates it implicitly. Use mkdir -p before tar to ensure the path exists. --- rundmc/containerizer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rundmc/containerizer.go b/rundmc/containerizer.go index cc7c7b989..039512999 100644 --- a/rundmc/containerizer.go +++ b/rundmc/containerizer.go @@ -278,8 +278,8 @@ func (c *Containerizer) StreamIn(log lager.Logger, handle string, spec garden.St func (c *Containerizer) streamInViaExec(log lager.Logger, handle string, spec garden.StreamInSpec) error { processSpec := garden.ProcessSpec{ - Path: "/bin/tar", - Args: []string{"-xf", "-", "-C", spec.Path}, + Path: "/bin/sh", + Args: []string{"-c", fmt.Sprintf("mkdir -p %q && exec tar -xf - -C %q", spec.Path, spec.Path)}, User: spec.User, } if processSpec.User == "" { From 4afd8d93b6906164aac41c1a8b6e6f5efa4f89de Mon Sep 17 00:00:00 2001 From: Vladimir Savchenko Date: Wed, 27 May 2026 14:11:42 +0300 Subject: [PATCH 07/16] Regenerate fakes for Start() interface addition --- gardener/gardenerfakes/fake_containerizer.go | 76 ++++++++++++++++++++ rundmc/rundmcfakes/fake_ociruntime.go | 76 ++++++++++++++++++++ 2 files changed, 152 insertions(+) diff --git a/gardener/gardenerfakes/fake_containerizer.go b/gardener/gardenerfakes/fake_containerizer.go index 085ebb29c..fee0b8edf 100644 --- a/gardener/gardenerfakes/fake_containerizer.go +++ b/gardener/gardenerfakes/fake_containerizer.go @@ -130,6 +130,18 @@ type FakeContainerizer struct { shutdownReturnsOnCall map[int]struct { result1 error } + StartStub func(lager.Logger, string) error + startMutex sync.RWMutex + startArgsForCall []struct { + arg1 lager.Logger + arg2 string + } + startReturns struct { + result1 error + } + startReturnsOnCall map[int]struct { + result1 error + } StopStub func(lager.Logger, string, bool) error stopMutex sync.RWMutex stopArgsForCall []struct { @@ -745,6 +757,68 @@ func (fake *FakeContainerizer) ShutdownReturnsOnCall(i int, result1 error) { }{result1} } +func (fake *FakeContainerizer) Start(arg1 lager.Logger, arg2 string) error { + fake.startMutex.Lock() + ret, specificReturn := fake.startReturnsOnCall[len(fake.startArgsForCall)] + fake.startArgsForCall = append(fake.startArgsForCall, struct { + arg1 lager.Logger + arg2 string + }{arg1, arg2}) + stub := fake.StartStub + fakeReturns := fake.startReturns + fake.recordInvocation("Start", []interface{}{arg1, arg2}) + fake.startMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeContainerizer) StartCallCount() int { + fake.startMutex.RLock() + defer fake.startMutex.RUnlock() + return len(fake.startArgsForCall) +} + +func (fake *FakeContainerizer) StartCalls(stub func(lager.Logger, string) error) { + fake.startMutex.Lock() + defer fake.startMutex.Unlock() + fake.StartStub = stub +} + +func (fake *FakeContainerizer) StartArgsForCall(i int) (lager.Logger, string) { + fake.startMutex.RLock() + defer fake.startMutex.RUnlock() + argsForCall := fake.startArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeContainerizer) StartReturns(result1 error) { + fake.startMutex.Lock() + defer fake.startMutex.Unlock() + fake.StartStub = nil + fake.startReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeContainerizer) StartReturnsOnCall(i int, result1 error) { + fake.startMutex.Lock() + defer fake.startMutex.Unlock() + fake.StartStub = nil + if fake.startReturnsOnCall == nil { + fake.startReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.startReturnsOnCall[i] = struct { + result1 error + }{result1} +} + func (fake *FakeContainerizer) Stop(arg1 lager.Logger, arg2 string, arg3 bool) error { fake.stopMutex.Lock() ret, specificReturn := fake.stopReturnsOnCall[len(fake.stopArgsForCall)] @@ -1019,6 +1093,8 @@ func (fake *FakeContainerizer) Invocations() map[string][][]interface{} { defer fake.runMutex.RUnlock() fake.shutdownMutex.RLock() defer fake.shutdownMutex.RUnlock() + fake.startMutex.RLock() + defer fake.startMutex.RUnlock() fake.stopMutex.RLock() defer fake.stopMutex.RUnlock() fake.streamInMutex.RLock() diff --git a/rundmc/rundmcfakes/fake_ociruntime.go b/rundmc/rundmcfakes/fake_ociruntime.go index c88770eca..bc8dedee4 100644 --- a/rundmc/rundmcfakes/fake_ociruntime.go +++ b/rundmc/rundmcfakes/fake_ociruntime.go @@ -138,6 +138,18 @@ type FakeOCIRuntime struct { removeBundleReturnsOnCall map[int]struct { result1 error } + StartStub func(lager.Logger, string) error + startMutex sync.RWMutex + startArgsForCall []struct { + arg1 lager.Logger + arg2 string + } + startReturns struct { + result1 error + } + startReturnsOnCall map[int]struct { + result1 error + } StateStub func(lager.Logger, string) (rundmc.State, error) stateMutex sync.RWMutex stateArgsForCall []struct { @@ -745,6 +757,68 @@ func (fake *FakeOCIRuntime) RemoveBundleReturnsOnCall(i int, result1 error) { }{result1} } +func (fake *FakeOCIRuntime) Start(arg1 lager.Logger, arg2 string) error { + fake.startMutex.Lock() + ret, specificReturn := fake.startReturnsOnCall[len(fake.startArgsForCall)] + fake.startArgsForCall = append(fake.startArgsForCall, struct { + arg1 lager.Logger + arg2 string + }{arg1, arg2}) + stub := fake.StartStub + fakeReturns := fake.startReturns + fake.recordInvocation("Start", []interface{}{arg1, arg2}) + fake.startMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeOCIRuntime) StartCallCount() int { + fake.startMutex.RLock() + defer fake.startMutex.RUnlock() + return len(fake.startArgsForCall) +} + +func (fake *FakeOCIRuntime) StartCalls(stub func(lager.Logger, string) error) { + fake.startMutex.Lock() + defer fake.startMutex.Unlock() + fake.StartStub = stub +} + +func (fake *FakeOCIRuntime) StartArgsForCall(i int) (lager.Logger, string) { + fake.startMutex.RLock() + defer fake.startMutex.RUnlock() + argsForCall := fake.startArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeOCIRuntime) StartReturns(result1 error) { + fake.startMutex.Lock() + defer fake.startMutex.Unlock() + fake.StartStub = nil + fake.startReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeOCIRuntime) StartReturnsOnCall(i int, result1 error) { + fake.startMutex.Lock() + defer fake.startMutex.Unlock() + fake.StartStub = nil + if fake.startReturnsOnCall == nil { + fake.startReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.startReturnsOnCall[i] = struct { + result1 error + }{result1} +} + func (fake *FakeOCIRuntime) State(arg1 lager.Logger, arg2 string) (rundmc.State, error) { fake.stateMutex.Lock() ret, specificReturn := fake.stateReturnsOnCall[len(fake.stateArgsForCall)] @@ -896,6 +970,8 @@ func (fake *FakeOCIRuntime) Invocations() map[string][][]interface{} { defer fake.execMutex.RUnlock() fake.removeBundleMutex.RLock() defer fake.removeBundleMutex.RUnlock() + fake.startMutex.RLock() + defer fake.startMutex.RUnlock() fake.stateMutex.RLock() defer fake.stateMutex.RUnlock() fake.statsMutex.RLock() From 6468411278c05d9f3b20c6d1f728bea5e3d5d0c0 Mon Sep 17 00:00:00 2001 From: Vladimir Savchenko Date: Wed, 27 May 2026 14:13:35 +0300 Subject: [PATCH 08/16] Fix containerizer_test: pass runtimeType parameter to New() --- rundmc/containerizer_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rundmc/containerizer_test.go b/rundmc/containerizer_test.go index fd7da03d2..71e9c0524 100644 --- a/rundmc/containerizer_test.go +++ b/rundmc/containerizer_test.go @@ -73,6 +73,7 @@ var _ = Describe("Rundmc", func() { 0, fakeRuntimeStopper, fakeCPUCgrouper, + "", ) }) @@ -664,6 +665,7 @@ var _ = Describe("Rundmc", func() { entitlementPerSharePercent, fakeRuntimeStopper, fakeCPUCgrouper, + "", ) }) From d974506842bc7ff8cd2c2f5335f0231dc2def220 Mon Sep 17 00:00:00 2001 From: Vladimir Savchenko Date: Wed, 27 May 2026 14:17:18 +0300 Subject: [PATCH 09/16] Fix runcontainerd tests for bundle-rootfs user lookup change --- rundmc/runcontainerd/runcontainerd_test.go | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/rundmc/runcontainerd/runcontainerd_test.go b/rundmc/runcontainerd/runcontainerd_test.go index d8190910a..536c5dde6 100644 --- a/rundmc/runcontainerd/runcontainerd_test.go +++ b/rundmc/runcontainerd/runcontainerd_test.go @@ -5,7 +5,6 @@ import ( "errors" "io" "os" - "path/filepath" "code.cloudfoundry.org/guardian/rundmc" @@ -299,6 +298,7 @@ var _ = Describe("Runcontainerd", func() { bundle = goci.Bndl{ Spec: specs.Spec{ Hostname: "test-hostname", + Root: &specs.Root{Path: "/test/rootfs"}, Linux: &specs.Linux{}, }, }.WithUIDMappings(specs.LinuxIDMapping{ @@ -417,13 +417,10 @@ var _ = Describe("Runcontainerd", func() { }) It("creates the process with the resolved user", func() { - _, actualContainerId := containerManager.GetContainerPIDArgsForCall(0) - Expect(actualContainerId).To(Equal("container-id")) - Expect(userLookupper.LookupCallCount()).To(Equal(1)) passedRootfs, passedUserId := userLookupper.LookupArgsForCall(0) Expect(passedUserId).To(Equal("testuser")) - Expect(passedRootfs).To(Equal("/proc/1234/root")) + Expect(passedRootfs).To(Equal("/test/rootfs")) Expect(processBuilder.BuildProcessCallCount()).To(Equal(1)) _, _, user := processBuilder.BuildProcessArgsForCall(0) @@ -435,7 +432,7 @@ var _ = Describe("Runcontainerd", func() { It("sets up the working directory", func() { Expect(mkdirer.MkdirAsCallCount()).To(Equal(1)) rootfsPath, hostUID, hostGID, mode, shouldRecreate, workDir := mkdirer.MkdirAsArgsForCall(0) - Expect(rootfsPath).To(Equal(filepath.Join("/proc", "1234", "root"))) + Expect(rootfsPath).To(Equal("/test/rootfs")) Expect(hostUID).To(Equal(2000)) Expect(hostGID).To(Equal(2001)) Expect(mode).To(Equal(os.FileMode(0755))) @@ -505,16 +502,6 @@ var _ = Describe("Runcontainerd", func() { }) }) - Context("when getting the container PID fails", func() { - BeforeEach(func() { - containerManager.GetContainerPIDReturns(0, errors.New("get-container-pid-failure")) - }) - - It("returns the error", func() { - Expect(execErr).To(MatchError("get-container-pid-failure")) - }) - }) - Context("when containerManager returns an error", func() { BeforeEach(func() { containerManager.ExecReturns(nil, errors.New("error-execing")) From 11ad841bfc313543aab11874f360e9fbbfece4a5 Mon Sep 17 00:00:00 2001 From: Vladimir Savchenko Date: Wed, 27 May 2026 14:18:36 +0300 Subject: [PATCH 10/16] Update cgroup_manager_test: missing state.json now returns nil (gVisor compat) --- rundmc/runcontainerd/cgroup_manager_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rundmc/runcontainerd/cgroup_manager_test.go b/rundmc/runcontainerd/cgroup_manager_test.go index d4c9cfea1..8bae3bf70 100644 --- a/rundmc/runcontainerd/cgroup_manager_test.go +++ b/rundmc/runcontainerd/cgroup_manager_test.go @@ -72,8 +72,8 @@ var _ = Describe("CgroupManager", func() { containerHandle = "foo" }) - It("returns an error", func() { - Expect(setErr).To(HaveOccurred()) + It("returns nil (tolerates missing state for gVisor)", func() { + Expect(setErr).NotTo(HaveOccurred()) }) }) From c230d3b6f89f8edacac9735d598b5ff977f94db6 Mon Sep 17 00:00:00 2001 From: Vladimir Savchenko Date: Wed, 27 May 2026 14:40:39 +0300 Subject: [PATCH 11/16] Address review findings: eliminate shell injection, capture StreamOut errors, add constants - Replace shell-based streamInViaExec (vulnerable to command injection via spec.Path) with two direct exec calls: /bin/mkdir then /bin/tar - Capture stderr in streamOutViaExec and log tar failures instead of silently discarding errors - Extract hardcoded "io.containerd.runc.v2" into defaultRuncRuntime constant - Add debug log to RunRunc.Start no-op for observability --- rundmc/containerizer.go | 47 ++++++++++++++++++++++++++++++++++----- rundmc/runrunc/runrunc.go | 1 + 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/rundmc/containerizer.go b/rundmc/containerizer.go index 039512999..504eea5a4 100644 --- a/rundmc/containerizer.go +++ b/rundmc/containerizer.go @@ -1,6 +1,7 @@ package rundmc import ( + "bytes" "fmt" "io" "os" @@ -47,6 +48,8 @@ const RunningStatus Status = "running" const CreatedStatus Status = "created" const StoppedStatus Status = "stopped" +const defaultRuncRuntime = "io.containerd.runc.v2" + type State struct { Pid int Status Status @@ -258,7 +261,7 @@ func (c *Containerizer) StreamIn(log lager.Logger, handle string, spec garden.St _ = metrics.SendValue("StreamInDuration", float64(time.Since(startedAt).Nanoseconds()), "nanos") }(time.Now()) - if c.runtimeType != "" && c.runtimeType != "io.containerd.runc.v2" { + if c.runtimeType != "" && c.runtimeType != defaultRuncRuntime { return c.streamInViaExec(log, handle, spec) } @@ -277,9 +280,37 @@ func (c *Containerizer) StreamIn(log lager.Logger, handle string, spec garden.St } func (c *Containerizer) streamInViaExec(log lager.Logger, handle string, spec garden.StreamInSpec) error { + // Create target directory (no shell, no stdin needed) + mkdirSpec := garden.ProcessSpec{ + Path: "/bin/mkdir", + Args: []string{"-p", spec.Path}, + User: spec.User, + } + if mkdirSpec.User == "" { + mkdirSpec.User = "root" + } + + mkdirProcess, err := c.runtime.Exec(log, handle, mkdirSpec, garden.ProcessIO{ + Stdout: io.Discard, + Stderr: io.Discard, + }) + if err != nil { + log.Error("exec-mkdir-failed", err) + return fmt.Errorf("stream-in: mkdir: %s", err) + } + + mkdirExit, err := mkdirProcess.Wait() + if err != nil { + return fmt.Errorf("stream-in: mkdir wait: %s", err) + } + if mkdirExit != 0 { + return fmt.Errorf("stream-in: mkdir exited %d", mkdirExit) + } + + // Stream tar content into container processSpec := garden.ProcessSpec{ - Path: "/bin/sh", - Args: []string{"-c", fmt.Sprintf("mkdir -p %q && exec tar -xf - -C %q", spec.Path, spec.Path)}, + Path: "/bin/tar", + Args: []string{"-xf", "-", "-C", spec.Path}, User: spec.User, } if processSpec.User == "" { @@ -316,7 +347,7 @@ func (c *Containerizer) StreamOut(log lager.Logger, handle string, spec garden.S log.Info("started") defer log.Info("finished") - if c.runtimeType != "" && c.runtimeType != "io.containerd.runc.v2" { + if c.runtimeType != "" && c.runtimeType != defaultRuncRuntime { return c.streamOutViaExec(log, handle, spec) } @@ -358,9 +389,10 @@ func (c *Containerizer) streamOutViaExec(log lager.Logger, handle string, spec g processSpec.User = "root" } + var stderrBuf bytes.Buffer processIO := garden.ProcessIO{ Stdout: writer, - Stderr: io.Discard, + Stderr: &stderrBuf, } process, err := c.runtime.Exec(log, handle, processSpec, processIO) @@ -372,7 +404,10 @@ func (c *Containerizer) streamOutViaExec(log lager.Logger, handle string, spec g } go func() { - process.Wait() + exitCode, waitErr := process.Wait() + if waitErr != nil || exitCode != 0 { + log.Error("stream-out-tar-failed", fmt.Errorf("exit=%d stderr=%s", exitCode, stderrBuf.String())) + } writer.Close() }() diff --git a/rundmc/runrunc/runrunc.go b/rundmc/runrunc/runrunc.go index 80d1f30d6..04fd2ee05 100644 --- a/rundmc/runrunc/runrunc.go +++ b/rundmc/runrunc/runrunc.go @@ -62,5 +62,6 @@ func New( } func (r *RunRunc) Start(log lager.Logger, id string) error { + log.Debug("start-noop-for-runc", lager.Data{"id": id}) return nil } From d9eeb38fd34443178c2c0c4ed5b54d1fd537f286 Mon Sep 17 00:00:00 2001 From: Vladimir Savchenko Date: Wed, 27 May 2026 14:57:59 +0300 Subject: [PATCH 12/16] Simplify gVisor patches: extract execAndWait, deduplicate user defaulting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Factor exec→wait→check-exit into execAndWait helper (eliminates repetition) - Set user once at top of each streaming function (was duplicated 3x) - Add stderr capture to streamInViaExec (symmetric with streamOut) - Replace magic constant with isNonRuncRuntime() method - Remove noisy debug log from RunRunc.Start no-op - Add one-line comments for spec.Windows and taskOpts skipping in nerd.go --- rundmc/containerizer.go | 111 ++++++++++++------------------ rundmc/runcontainerd/nerd/nerd.go | 3 +- rundmc/runrunc/runrunc.go | 3 +- 3 files changed, 48 insertions(+), 69 deletions(-) diff --git a/rundmc/containerizer.go b/rundmc/containerizer.go index 504eea5a4..8fd736fdf 100644 --- a/rundmc/containerizer.go +++ b/rundmc/containerizer.go @@ -48,8 +48,6 @@ const RunningStatus Status = "running" const CreatedStatus Status = "created" const StoppedStatus Status = "stopped" -const defaultRuncRuntime = "io.containerd.runc.v2" - type State struct { Pid int Status Status @@ -261,7 +259,7 @@ func (c *Containerizer) StreamIn(log lager.Logger, handle string, spec garden.St _ = metrics.SendValue("StreamInDuration", float64(time.Since(startedAt).Nanoseconds()), "nanos") }(time.Now()) - if c.runtimeType != "" && c.runtimeType != defaultRuncRuntime { + if c.isNonRuncRuntime() { return c.streamInViaExec(log, handle, spec) } @@ -280,61 +278,29 @@ func (c *Containerizer) StreamIn(log lager.Logger, handle string, spec garden.St } func (c *Containerizer) streamInViaExec(log lager.Logger, handle string, spec garden.StreamInSpec) error { - // Create target directory (no shell, no stdin needed) - mkdirSpec := garden.ProcessSpec{ - Path: "/bin/mkdir", - Args: []string{"-p", spec.Path}, - User: spec.User, - } - if mkdirSpec.User == "" { - mkdirSpec.User = "root" + user := spec.User + if user == "" { + user = "root" } - mkdirProcess, err := c.runtime.Exec(log, handle, mkdirSpec, garden.ProcessIO{ - Stdout: io.Discard, - Stderr: io.Discard, - }) - if err != nil { - log.Error("exec-mkdir-failed", err) + if err := c.execAndWait(log, handle, garden.ProcessSpec{ + Path: "/bin/mkdir", + Args: []string{"-p", spec.Path}, + User: user, + }, garden.ProcessIO{Stdout: io.Discard, Stderr: io.Discard}); err != nil { return fmt.Errorf("stream-in: mkdir: %s", err) } - mkdirExit, err := mkdirProcess.Wait() - if err != nil { - return fmt.Errorf("stream-in: mkdir wait: %s", err) - } - if mkdirExit != 0 { - return fmt.Errorf("stream-in: mkdir exited %d", mkdirExit) - } - - // Stream tar content into container - processSpec := garden.ProcessSpec{ + var stderrBuf bytes.Buffer + if err := c.execAndWait(log, handle, garden.ProcessSpec{ Path: "/bin/tar", Args: []string{"-xf", "-", "-C", spec.Path}, - User: spec.User, - } - if processSpec.User == "" { - processSpec.User = "root" - } - - processIO := garden.ProcessIO{ - Stdin: spec.TarStream, - Stdout: io.Discard, - Stderr: io.Discard, - } - - process, err := c.runtime.Exec(log, handle, processSpec, processIO) - if err != nil { - log.Error("exec-tar-failed", err) - return fmt.Errorf("stream-in: exec tar: %s", err) - } - - exitCode, err := process.Wait() - if err != nil { - return fmt.Errorf("stream-in: exec tar wait: %s", err) - } - if exitCode != 0 { - return fmt.Errorf("stream-in: exec tar exited %d", exitCode) + User: user, + }, garden.ProcessIO{Stdin: spec.TarStream, Stdout: io.Discard, Stderr: &stderrBuf}); err != nil { + if stderrBuf.Len() > 0 { + log.Error("stream-in-tar-stderr", fmt.Errorf("%s", stderrBuf.String())) + } + return fmt.Errorf("stream-in: tar: %s", err) } return nil @@ -347,7 +313,7 @@ func (c *Containerizer) StreamOut(log lager.Logger, handle string, spec garden.S log.Info("started") defer log.Info("finished") - if c.runtimeType != "" && c.runtimeType != defaultRuncRuntime { + if c.isNonRuncRuntime() { return c.streamOutViaExec(log, handle, spec) } @@ -380,26 +346,20 @@ func (c *Containerizer) streamOutViaExec(log lager.Logger, handle string, spec g return nil, err } - processSpec := garden.ProcessSpec{ - Path: "/bin/tar", - Args: []string{"-cf", "-", "-C", sourcePath, compressPath}, - User: spec.User, - } - if processSpec.User == "" { - processSpec.User = "root" + user := spec.User + if user == "" { + user = "root" } var stderrBuf bytes.Buffer - processIO := garden.ProcessIO{ - Stdout: writer, - Stderr: &stderrBuf, - } - - process, err := c.runtime.Exec(log, handle, processSpec, processIO) + process, err := c.runtime.Exec(log, handle, garden.ProcessSpec{ + Path: "/bin/tar", + Args: []string{"-cf", "-", "-C", sourcePath, compressPath}, + User: user, + }, garden.ProcessIO{Stdout: writer, Stderr: &stderrBuf}) if err != nil { writer.Close() reader.Close() - log.Error("exec-tar-failed", err) return nil, fmt.Errorf("stream-out: exec tar: %s", err) } @@ -414,6 +374,25 @@ func (c *Containerizer) streamOutViaExec(log lager.Logger, handle string, spec g return reader, nil } +func (c *Containerizer) isNonRuncRuntime() bool { + return c.runtimeType != "" && c.runtimeType != "io.containerd.runc.v2" +} + +func (c *Containerizer) execAndWait(log lager.Logger, handle string, spec garden.ProcessSpec, pio garden.ProcessIO) error { + process, err := c.runtime.Exec(log, handle, spec, pio) + if err != nil { + return err + } + exitCode, err := process.Wait() + if err != nil { + return err + } + if exitCode != 0 { + return fmt.Errorf("exited %d", exitCode) + } + return nil +} + // Stop stops all the processes other than the init process in the container func (c *Containerizer) Stop(log lager.Logger, handle string, kill bool) error { log = log.Session("stop", lager.Data{"handle": handle, "kill": kill}) diff --git a/rundmc/runcontainerd/nerd/nerd.go b/rundmc/runcontainerd/nerd/nerd.go index ff4eba82f..60125952f 100644 --- a/rundmc/runcontainerd/nerd/nerd.go +++ b/rundmc/runcontainerd/nerd/nerd.go @@ -49,7 +49,7 @@ func New(client *client.Client, context context.Context, ioFifoDir string, mp *m func (n *Nerd) Create(log lager.Logger, containerID string, spec *specs.Spec, hostUID, hostGID uint32, pio func() (io.Reader, io.Writer, io.Writer)) error { log.Debug("creating-container", lager.Data{"containerID": containerID}) if n.runtimeType != "" && n.runtimeType != plugins.RuntimeRuncV2 { - spec.Windows = nil + spec.Windows = nil // runsc rejects the Windows section } container, err := n.client.NewContainer(n.context, containerID, client.WithSpec(spec)) if err != nil { @@ -66,6 +66,7 @@ func (n *Nerd) Create(log lager.Logger, containerID string, spec *specs.Spec, ho log.Debug("creating-task", lager.Data{"containerID": containerID}) var taskOpts []client.NewTaskOpts if n.runtimeType == "" || n.runtimeType == plugins.RuntimeRuncV2 { + // runsc manages its own keyring and UID/GID via the OCI spec taskOpts = append(taskOpts, client.WithNoNewKeyring, WithUIDAndGID(hostUID, hostGID)) } _, err = container.NewTask(n.context, cio.NewCreator(withProcessIO(noTTYProcessIO(pio), n.ioFifoDir)), taskOpts...) diff --git a/rundmc/runrunc/runrunc.go b/rundmc/runrunc/runrunc.go index 04fd2ee05..de990b986 100644 --- a/rundmc/runrunc/runrunc.go +++ b/rundmc/runrunc/runrunc.go @@ -61,7 +61,6 @@ func New( } } -func (r *RunRunc) Start(log lager.Logger, id string) error { - log.Debug("start-noop-for-runc", lager.Data{"id": id}) +func (r *RunRunc) Start(_ lager.Logger, _ string) error { return nil } From 5a003d32663af215608fc34d0e2fbd03a42af348 Mon Sep 17 00:00:00 2001 From: Vladimir Savchenko Date: Wed, 27 May 2026 15:31:28 +0300 Subject: [PATCH 13/16] Extract Streamer interface for configurable runtime streaming Move exec-based and nstar-based streaming into separate implementations of a new Streamer interface. Containerizer delegates to the injected streamer without knowing which runtime is in use. - NstarStreamer: existing nstar/nsenter approach for runc containers - ExecStreamer: exec-based tar for runtimes where /proc/pid/ns/mnt is not accessible (e.g. gVisor) Selection happens once at construction time based on --containerd-runtime-type. Containerizer no longer carries runtimeType or any streaming logic. --- guardiancmd/command.go | 9 ++- rundmc/containerizer.go | 136 ++------------------------------ rundmc/containerizer_test.go | 6 +- rundmc/streamer.go | 145 +++++++++++++++++++++++++++++++++++ 4 files changed, 160 insertions(+), 136 deletions(-) create mode 100644 rundmc/streamer.go diff --git a/guardiancmd/command.go b/guardiancmd/command.go index 72bde8918..4476c25c0 100644 --- a/guardiancmd/command.go +++ b/guardiancmd/command.go @@ -693,7 +693,14 @@ func (cmd *CommonCommand) wireContainerizer( return nil, nil, err } - return rundmc.New(depot, template, ociRuntime, nstar, processesStopper, eventStore, stateStore, peaCreator, peaUsernameResolver, cpuEntitlementPerShare, runtimeStopper, cpuCgrouper, cmd.Containerd.RuntimeType), peaCleaner, nil + var streamer rundmc.Streamer + if cmd.Containerd.RuntimeType != "" && cmd.Containerd.RuntimeType != "io.containerd.runc.v2" { + streamer = rundmc.NewExecStreamer(ociRuntime) + } else { + streamer = rundmc.NewNstarStreamer(ociRuntime, nstar) + } + + return rundmc.New(depot, template, ociRuntime, streamer, processesStopper, eventStore, stateStore, peaCreator, peaUsernameResolver, cpuEntitlementPerShare, runtimeStopper, cpuCgrouper), peaCleaner, nil } func (cmd *CommonCommand) useContainerd() bool { diff --git a/rundmc/containerizer.go b/rundmc/containerizer.go index 8fd736fdf..5c86f547a 100644 --- a/rundmc/containerizer.go +++ b/rundmc/containerizer.go @@ -1,11 +1,9 @@ package rundmc import ( - "bytes" "fmt" "io" "os" - "path/filepath" "strconv" "strings" "time" @@ -111,7 +109,7 @@ type Containerizer struct { bundler BundleGenerator runtime OCIRuntime processesStopper ProcessesStopper - nstar NstarRunner + streamer Streamer events EventStore states StateStore peaCreator PeaCreator @@ -119,14 +117,13 @@ type Containerizer struct { cpuEntitlementPerShare float64 runtimeStopper RuntimeStopper cpuCgrouper CPUCgrouper - runtimeType string } func New( depot Depot, bundler BundleGenerator, runtime OCIRuntime, - nstarRunner NstarRunner, + streamer Streamer, processesStopper ProcessesStopper, events EventStore, states StateStore, @@ -135,13 +132,12 @@ func New( cpuEntitlementPerShare float64, runtimeStopper RuntimeStopper, cpuCgrouper CPUCgrouper, - runtimeType string, ) *Containerizer { containerizer := &Containerizer{ depot: depot, bundler: bundler, runtime: runtime, - nstar: nstarRunner, + streamer: streamer, processesStopper: processesStopper, events: events, states: states, @@ -150,7 +146,6 @@ func New( cpuEntitlementPerShare: cpuEntitlementPerShare, runtimeStopper: runtimeStopper, cpuCgrouper: cpuCgrouper, - runtimeType: runtimeType, } return containerizer } @@ -259,51 +254,7 @@ func (c *Containerizer) StreamIn(log lager.Logger, handle string, spec garden.St _ = metrics.SendValue("StreamInDuration", float64(time.Since(startedAt).Nanoseconds()), "nanos") }(time.Now()) - if c.isNonRuncRuntime() { - return c.streamInViaExec(log, handle, spec) - } - - state, err := c.runtime.State(log, handle) - if err != nil { - log.Error("check-pid-failed", err) - return fmt.Errorf("stream-in: pid not found for container") - } - - if err := c.nstar.StreamIn(log, state.Pid, spec.Path, spec.User, spec.TarStream); err != nil { - log.Error("nstar-failed", err) - return fmt.Errorf("stream-in: nstar: %s", err) - } - - return nil -} - -func (c *Containerizer) streamInViaExec(log lager.Logger, handle string, spec garden.StreamInSpec) error { - user := spec.User - if user == "" { - user = "root" - } - - if err := c.execAndWait(log, handle, garden.ProcessSpec{ - Path: "/bin/mkdir", - Args: []string{"-p", spec.Path}, - User: user, - }, garden.ProcessIO{Stdout: io.Discard, Stderr: io.Discard}); err != nil { - return fmt.Errorf("stream-in: mkdir: %s", err) - } - - var stderrBuf bytes.Buffer - if err := c.execAndWait(log, handle, garden.ProcessSpec{ - Path: "/bin/tar", - Args: []string{"-xf", "-", "-C", spec.Path}, - User: user, - }, garden.ProcessIO{Stdin: spec.TarStream, Stdout: io.Discard, Stderr: &stderrBuf}); err != nil { - if stderrBuf.Len() > 0 { - log.Error("stream-in-tar-stderr", fmt.Errorf("%s", stderrBuf.String())) - } - return fmt.Errorf("stream-in: tar: %s", err) - } - - return nil + return c.streamer.StreamIn(log, handle, spec) } // StreamOut stream files from the container @@ -313,84 +264,7 @@ func (c *Containerizer) StreamOut(log lager.Logger, handle string, spec garden.S log.Info("started") defer log.Info("finished") - if c.isNonRuncRuntime() { - return c.streamOutViaExec(log, handle, spec) - } - - state, err := c.runtime.State(log, handle) - if err != nil { - log.Error("check-pid-failed", err) - return nil, fmt.Errorf("stream-out: pid not found for container") - } - - stream, err := c.nstar.StreamOut(log, state.Pid, spec.Path, spec.User) - if err != nil { - log.Error("nstar-failed", err) - return nil, fmt.Errorf("stream-out: nstar: %s", err) - } - - return stream, nil -} - -func (c *Containerizer) streamOutViaExec(log lager.Logger, handle string, spec garden.StreamOutSpec) (io.ReadCloser, error) { - path := spec.Path - sourcePath := filepath.Dir(path) - compressPath := filepath.Base(path) - if strings.HasSuffix(path, "/") { - sourcePath = path - compressPath = "." - } - - reader, writer, err := os.Pipe() - if err != nil { - return nil, err - } - - user := spec.User - if user == "" { - user = "root" - } - - var stderrBuf bytes.Buffer - process, err := c.runtime.Exec(log, handle, garden.ProcessSpec{ - Path: "/bin/tar", - Args: []string{"-cf", "-", "-C", sourcePath, compressPath}, - User: user, - }, garden.ProcessIO{Stdout: writer, Stderr: &stderrBuf}) - if err != nil { - writer.Close() - reader.Close() - return nil, fmt.Errorf("stream-out: exec tar: %s", err) - } - - go func() { - exitCode, waitErr := process.Wait() - if waitErr != nil || exitCode != 0 { - log.Error("stream-out-tar-failed", fmt.Errorf("exit=%d stderr=%s", exitCode, stderrBuf.String())) - } - writer.Close() - }() - - return reader, nil -} - -func (c *Containerizer) isNonRuncRuntime() bool { - return c.runtimeType != "" && c.runtimeType != "io.containerd.runc.v2" -} - -func (c *Containerizer) execAndWait(log lager.Logger, handle string, spec garden.ProcessSpec, pio garden.ProcessIO) error { - process, err := c.runtime.Exec(log, handle, spec, pio) - if err != nil { - return err - } - exitCode, err := process.Wait() - if err != nil { - return err - } - if exitCode != 0 { - return fmt.Errorf("exited %d", exitCode) - } - return nil + return c.streamer.StreamOut(log, handle, spec) } // Stop stops all the processes other than the init process in the container diff --git a/rundmc/containerizer_test.go b/rundmc/containerizer_test.go index 71e9c0524..a1d635892 100644 --- a/rundmc/containerizer_test.go +++ b/rundmc/containerizer_test.go @@ -64,7 +64,7 @@ var _ = Describe("Rundmc", func() { fakeDepot, fakeBundleGenerator, fakeOCIRuntime, - fakeNstarRunner, + rundmc.NewNstarStreamer(fakeOCIRuntime, fakeNstarRunner), fakeProcessesStopper, fakeEventStore, fakeStateStore, @@ -73,7 +73,6 @@ var _ = Describe("Rundmc", func() { 0, fakeRuntimeStopper, fakeCPUCgrouper, - "", ) }) @@ -656,7 +655,7 @@ var _ = Describe("Rundmc", func() { fakeDepot, fakeBundleGenerator, fakeOCIRuntime, - fakeNstarRunner, + rundmc.NewNstarStreamer(fakeOCIRuntime, fakeNstarRunner), fakeProcessesStopper, fakeEventStore, fakeStateStore, @@ -665,7 +664,6 @@ var _ = Describe("Rundmc", func() { entitlementPerSharePercent, fakeRuntimeStopper, fakeCPUCgrouper, - "", ) }) diff --git a/rundmc/streamer.go b/rundmc/streamer.go new file mode 100644 index 000000000..c55f4f2d1 --- /dev/null +++ b/rundmc/streamer.go @@ -0,0 +1,145 @@ +package rundmc + +import ( + "bytes" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "code.cloudfoundry.org/garden" + "code.cloudfoundry.org/lager/v3" +) + +//counterfeiter:generate . Streamer +type Streamer interface { + StreamIn(log lager.Logger, handle string, spec garden.StreamInSpec) error + StreamOut(log lager.Logger, handle string, spec garden.StreamOutSpec) (io.ReadCloser, error) +} + +type NstarStreamer struct { + runtime OCIRuntime + nstar NstarRunner +} + +func NewNstarStreamer(runtime OCIRuntime, nstar NstarRunner) *NstarStreamer { + return &NstarStreamer{runtime: runtime, nstar: nstar} +} + +func (s *NstarStreamer) StreamIn(log lager.Logger, handle string, spec garden.StreamInSpec) error { + state, err := s.runtime.State(log, handle) + if err != nil { + return fmt.Errorf("stream-in: pid not found for container") + } + if err := s.nstar.StreamIn(log, state.Pid, spec.Path, spec.User, spec.TarStream); err != nil { + return fmt.Errorf("stream-in: nstar: %s", err) + } + return nil +} + +func (s *NstarStreamer) StreamOut(log lager.Logger, handle string, spec garden.StreamOutSpec) (io.ReadCloser, error) { + state, err := s.runtime.State(log, handle) + if err != nil { + return nil, fmt.Errorf("stream-out: pid not found for container") + } + stream, err := s.nstar.StreamOut(log, state.Pid, spec.Path, spec.User) + if err != nil { + return nil, fmt.Errorf("stream-out: nstar: %s", err) + } + return stream, nil +} + +type ExecStreamer struct { + runtime OCIRuntime +} + +func NewExecStreamer(runtime OCIRuntime) *ExecStreamer { + return &ExecStreamer{runtime: runtime} +} + +func (s *ExecStreamer) StreamIn(log lager.Logger, handle string, spec garden.StreamInSpec) error { + user := spec.User + if user == "" { + user = "root" + } + + if err := s.execAndWait(log, handle, garden.ProcessSpec{ + Path: "/bin/mkdir", + Args: []string{"-p", spec.Path}, + User: user, + }, garden.ProcessIO{Stdout: io.Discard, Stderr: io.Discard}); err != nil { + return fmt.Errorf("stream-in: mkdir: %s", err) + } + + var stderrBuf bytes.Buffer + if err := s.execAndWait(log, handle, garden.ProcessSpec{ + Path: "/bin/tar", + Args: []string{"-xf", "-", "-C", spec.Path}, + User: user, + }, garden.ProcessIO{Stdin: spec.TarStream, Stdout: io.Discard, Stderr: &stderrBuf}); err != nil { + if stderrBuf.Len() > 0 { + log.Error("stream-in-tar-stderr", fmt.Errorf("%s", stderrBuf.String())) + } + return fmt.Errorf("stream-in: tar: %s", err) + } + + return nil +} + +func (s *ExecStreamer) StreamOut(log lager.Logger, handle string, spec garden.StreamOutSpec) (io.ReadCloser, error) { + path := spec.Path + sourcePath := filepath.Dir(path) + compressPath := filepath.Base(path) + if strings.HasSuffix(path, "/") { + sourcePath = path + compressPath = "." + } + + reader, writer, err := os.Pipe() + if err != nil { + return nil, err + } + + user := spec.User + if user == "" { + user = "root" + } + + var stderrBuf bytes.Buffer + process, err := s.runtime.Exec(log, handle, garden.ProcessSpec{ + Path: "/bin/tar", + Args: []string{"-cf", "-", "-C", sourcePath, compressPath}, + User: user, + }, garden.ProcessIO{Stdout: writer, Stderr: &stderrBuf}) + if err != nil { + writer.Close() + reader.Close() + return nil, fmt.Errorf("stream-out: exec tar: %s", err) + } + + go func() { + exitCode, waitErr := process.Wait() + if waitErr != nil || exitCode != 0 { + log.Error("stream-out-tar-failed", fmt.Errorf("exit=%d stderr=%s", exitCode, stderrBuf.String())) + } + writer.Close() + }() + + return reader, nil +} + +func (s *ExecStreamer) execAndWait(log lager.Logger, handle string, spec garden.ProcessSpec, pio garden.ProcessIO) error { + process, err := s.runtime.Exec(log, handle, spec, pio) + if err != nil { + return err + } + exitCode, err := process.Wait() + if err != nil { + return err + } + if exitCode != 0 { + return fmt.Errorf("exited %d", exitCode) + } + return nil +} From fbb0d75f1dc06e275d8afa60e9b6732beadb5ee1 Mon Sep 17 00:00:00 2001 From: Vladimir Savchenko Date: Mon, 1 Jun 2026 12:49:01 +0300 Subject: [PATCH 14/16] Add gVisor pea-as-exec and loopback fixes for Docker container support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three patches for full Docker container support on gVisor: 1. execPeas (containerizer.go): When gVisor runtime detected, convert pea requests to exec inside existing container. Peas create separate sandboxes that can't share gVisor's per-sandbox netstack. The healthcheck/envoy binaries are already bind-mounted, so exec works directly. 2. SkipUserNamespace (pea_creator.go): Defense-in-depth — skip userns join for gVisor since setns(CLONE_NEWUSER) is rejected with EINVAL. 3. setupLoopback (external_networker.go): After silk CNI 'up', bring up the loopback interface in the container's network namespace. Silk only creates a veth pair; gVisor's netstack scrapes the netns and needs lo present to support 127.0.0.1 binding (required by envoy). Validated end-to-end on lod-aws-0515: - Docker python:3.12-slim with port healthcheck: PASS - Envoy container proxy (mTLS): PASS - HTTP routing through gorouter: PASS --- guardiancmd/command.go | 22 +++-- netplugin/external_networker.go | 163 ++++++++++++++++++++++++++++++++ rundmc/containerizer.go | 13 +++ rundmc/peas/pea_creator.go | 21 ++-- 4 files changed, 199 insertions(+), 20 deletions(-) diff --git a/guardiancmd/command.go b/guardiancmd/command.go index 4476c25c0..6177eb458 100644 --- a/guardiancmd/command.go +++ b/guardiancmd/command.go @@ -667,15 +667,16 @@ func (cmd *CommonCommand) wireContainerizer( peaCleaner := cmd.wirePeaCleaner(factory, volumizer, ociRuntime, peaPidGetter) peaCreator = &peas.PeaCreator{ - Volumizer: volumizer, - PidGetter: containersPidGetter, - PrivilegedGetter: privilegeChecker, - NetworkDepot: networkDepot, - BundleGenerator: template, - ProcessBuilder: processBuilder, - BundleSaver: bundleSaver, - ExecRunner: peasExecRunner, - PeaCleaner: peaCleaner, + Volumizer: volumizer, + PidGetter: containersPidGetter, + PrivilegedGetter: privilegeChecker, + NetworkDepot: networkDepot, + BundleGenerator: template, + ProcessBuilder: processBuilder, + BundleSaver: bundleSaver, + ExecRunner: peasExecRunner, + PeaCleaner: peaCleaner, + SkipUserNamespace: cmd.Containerd.RuntimeType != "" && cmd.Containerd.RuntimeType != "io.containerd.runc.v2", } peaUsernameResolver := &peas.PeaUsernameResolver{ @@ -700,7 +701,8 @@ func (cmd *CommonCommand) wireContainerizer( streamer = rundmc.NewNstarStreamer(ociRuntime, nstar) } - return rundmc.New(depot, template, ociRuntime, streamer, processesStopper, eventStore, stateStore, peaCreator, peaUsernameResolver, cpuEntitlementPerShare, runtimeStopper, cpuCgrouper), peaCleaner, nil + execPeas := cmd.Containerd.RuntimeType != "" && cmd.Containerd.RuntimeType != "io.containerd.runc.v2" + return rundmc.New(depot, template, ociRuntime, streamer, processesStopper, eventStore, stateStore, peaCreator, peaUsernameResolver, cpuEntitlementPerShare, runtimeStopper, cpuCgrouper, execPeas), peaCleaner, nil } func (cmd *CommonCommand) useContainerd() bool { diff --git a/netplugin/external_networker.go b/netplugin/external_networker.go index ae5cf4484..1b77dc131 100644 --- a/netplugin/external_networker.go +++ b/netplugin/external_networker.go @@ -8,12 +8,15 @@ import ( "net" "os/exec" "strings" + "time" "code.cloudfoundry.org/commandrunner" "code.cloudfoundry.org/garden" "code.cloudfoundry.org/guardian/gardener" "code.cloudfoundry.org/guardian/kawasaki" "code.cloudfoundry.org/lager/v3" + "github.com/vishvananda/netlink" + "github.com/vishvananda/netns" ) const NetworkPropertyPrefix = "network." @@ -111,6 +114,15 @@ func (p *externalBinaryNetworker) Network(log lager.Logger, containerSpec garden return err } + // Ensure loopback interface is up in the container's network namespace. + // External network plugins (e.g. silk CNI) typically only create a veth pair. + // Runtimes like gVisor that build their own netstack by scraping the netns + // need lo to be present for applications binding to 127.0.0.1. + if err := setupLoopback(log, pid); err != nil { + log.Error("setup-loopback", err) + // Non-fatal: runc brings up lo on its own, only gVisor needs this. + } + for k, v := range outputs.Properties { p.configStore.Set(containerSpec.Handle, k, v) } @@ -151,6 +163,157 @@ func (p *externalBinaryNetworker) Network(log lager.Logger, containerSpec garden return nil } +// setupLoopback enters the container's network namespace and brings up the +// loopback interface. This is needed for runtimes like gVisor that scrape the +// netns for interfaces rather than creating lo themselves. +func setupLoopback(log lager.Logger, pid int) error { + containerNS, err := netns.GetFromPid(pid) + if err != nil { + return fmt.Errorf("getting netns for pid %d: %w", pid, err) + } + defer containerNS.Close() + + // Create a netlink handle in the container's network namespace. + nlh, err := netlink.NewHandleAt(containerNS) + if err != nil { + return fmt.Errorf("creating netlink handle in container netns: %w", err) + } + defer nlh.Close() + + lo, err := nlh.LinkByName("lo") + if err != nil { + return fmt.Errorf("finding lo interface: %w", err) + } + + if err := nlh.LinkSetUp(lo); err != nil { + return fmt.Errorf("bringing up lo: %w", err) + } + + log.Info("setup-loopback-success", lager.Data{"pid": pid}) + return nil +} + +// warmARPCache ensures the container's network namespace has resolved ARP +// entries for all gateway IPs. gVisor's netstack copies ARP entries from the +// namespace at startup but only those in "live" states (PERMANENT, REACHABLE, +// STALE). On silk CNI with point-to-point links, gateway ARP entries may not +// exist until triggered. This function probes each gateway with a UDP packet +// to force ARP resolution, then promotes entries to PERMANENT so gVisor +// reliably picks them up. +func warmARPCache(log lager.Logger, pid int) error { + containerNS, err := netns.GetFromPid(pid) + if err != nil { + return fmt.Errorf("getting netns for pid %d: %w", pid, err) + } + defer containerNS.Close() + + nlh, err := netlink.NewHandleAt(containerNS) + if err != nil { + return fmt.Errorf("creating netlink handle: %w", err) + } + defer nlh.Close() + + // Find all gateway IPs from routes in the container namespace. + routes, err := nlh.RouteList(nil, netlink.FAMILY_ALL) + if err != nil { + return fmt.Errorf("listing routes: %w", err) + } + + gwSet := map[string]net.IP{} + for _, r := range routes { + if r.Gw != nil && !r.Gw.IsUnspecified() { + gwSet[r.Gw.String()] = r.Gw + } + } + + if len(gwSet) == 0 { + return nil + } + + // Check which gateways already have ARP entries. + links, _ := nlh.LinkList() + var primaryLink netlink.Link + for _, l := range links { + if l.Attrs().Name != "lo" { + primaryLink = l + break + } + } + + if primaryLink == nil { + return nil + } + + neighs, _ := nlh.NeighList(primaryLink.Attrs().Index, netlink.FAMILY_ALL) + hasEntry := func(ip net.IP) bool { + for _, n := range neighs { + if n.IP.Equal(ip) && len(n.HardwareAddr) > 0 { + return true + } + } + return false + } + + // Probe unresolved gateways with a UDP packet to trigger ARP. + probed := false + for _, gw := range gwSet { + if hasEntry(gw) { + continue + } + network := "udp4" + if gw.To4() == nil { + network = "udp6" + } + conn, err := net.DialUDP(network, nil, &net.UDPAddr{IP: gw, Port: 9}) + if err != nil { + log.Info("arp-warm-probe-failed", lager.Data{"gw": gw.String(), "error": err.Error()}) + continue + } + conn.Write([]byte{}) + conn.Close() + probed = true + } + + if probed { + time.Sleep(75 * time.Millisecond) // wait for ARP reply + } + + // Re-read neighbor table and promote entries to PERMANENT. + neighs, err = nlh.NeighList(primaryLink.Attrs().Index, netlink.FAMILY_ALL) + if err != nil { + return fmt.Errorf("re-reading neighbor table: %w", err) + } + + const liveStates = netlink.NUD_REACHABLE | netlink.NUD_STALE | netlink.NUD_DELAY | netlink.NUD_PROBE | netlink.NUD_PERMANENT + promoted := 0 + for _, n := range neighs { + if len(n.HardwareAddr) == 0 || n.State&liveStates == 0 { + continue + } + if _, isGW := gwSet[n.IP.String()]; !isGW { + continue + } + if n.State == netlink.NUD_PERMANENT { + continue // already permanent + } + // Promote to PERMANENT so gVisor's scraper picks it up. + err := nlh.NeighSet(&netlink.Neigh{ + LinkIndex: primaryLink.Attrs().Index, + IP: n.IP, + HardwareAddr: n.HardwareAddr, + State: netlink.NUD_PERMANENT, + }) + if err != nil { + log.Info("arp-promote-failed", lager.Data{"ip": n.IP.String(), "error": err.Error()}) + } else { + promoted++ + } + } + + log.Info("warm-arp-cache-done", lager.Data{"pid": pid, "gateways": len(gwSet), "promoted": promoted}) + return nil +} + func (p *externalBinaryNetworker) Destroy(log lager.Logger, handle string) error { err := p.exec(log, "down", handle, nil, nil) if err != nil { diff --git a/rundmc/containerizer.go b/rundmc/containerizer.go index 5c86f547a..c7aea215a 100644 --- a/rundmc/containerizer.go +++ b/rundmc/containerizer.go @@ -117,6 +117,7 @@ type Containerizer struct { cpuEntitlementPerShare float64 runtimeStopper RuntimeStopper cpuCgrouper CPUCgrouper + execPeas bool } func New( @@ -132,6 +133,7 @@ func New( cpuEntitlementPerShare float64, runtimeStopper RuntimeStopper, cpuCgrouper CPUCgrouper, + execPeas bool, ) *Containerizer { containerizer := &Containerizer{ depot: depot, @@ -146,6 +148,7 @@ func New( cpuEntitlementPerShare: cpuEntitlementPerShare, runtimeStopper: runtimeStopper, cpuCgrouper: cpuCgrouper, + execPeas: execPeas, } return containerizer } @@ -206,6 +209,16 @@ func (c *Containerizer) Run(log lager.Logger, handle string, spec garden.Process defer log.Info("finished") if isPea(spec) { + if c.execPeas { + // On gVisor, peas (sidecar containers) cannot share namespaces across + // sandboxes. Run the process as exec inside the existing container instead. + // The required binaries (healthcheck, etc.) are already bind-mounted in. + log.Info("exec-pea-as-process", lager.Data{"path": spec.Path}) + spec.Image = garden.ImageRef{} + spec.BindMounts = nil + return c.runtime.Exec(log, handle, spec, io) + } + if shouldResolveUsername(spec.User) { resolvedUID, resolvedGID, err := c.peaUsernameResolver.ResolveUser(log, handle, spec.Image, spec.User) if err != nil { diff --git a/rundmc/peas/pea_creator.go b/rundmc/peas/pea_creator.go index 39426d0be..dbbf95faa 100644 --- a/rundmc/peas/pea_creator.go +++ b/rundmc/peas/pea_creator.go @@ -59,15 +59,16 @@ type ExecRunner interface { } type PeaCreator struct { - Volumizer Volumizer - PidGetter PidGetter - PrivilegedGetter PrivilegedGetter - NetworkDepot NetworkDepot - BundleGenerator BundleGenerator - BundleSaver depot.BundleSaver - ProcessBuilder runrunc.ProcessBuilder - ExecRunner ExecRunner - PeaCleaner gardener.PeaCleaner + Volumizer Volumizer + PidGetter PidGetter + PrivilegedGetter PrivilegedGetter + NetworkDepot NetworkDepot + BundleGenerator BundleGenerator + BundleSaver depot.BundleSaver + ProcessBuilder runrunc.ProcessBuilder + ExecRunner ExecRunner + PeaCleaner gardener.PeaCleaner + SkipUserNamespace bool } func (p *PeaCreator) CreatePea(log lager.Logger, processSpec garden.ProcessSpec, procIO garden.ProcessIO, sandboxHandle string) (_ garden.Process, theErr error) { @@ -181,7 +182,7 @@ func (p *PeaCreator) linuxNamespaces(log lager.Logger, sandboxHandle string, pri linuxNamespaces["pid"] = fmt.Sprintf("/proc/%d/ns/pid", originalCtrInitPid) linuxNamespaces["uts"] = fmt.Sprintf("/proc/%d/ns/uts", originalCtrInitPid) - if !privileged { + if !privileged && !p.SkipUserNamespace { linuxNamespaces["user"] = fmt.Sprintf("/proc/%d/ns/user", originalCtrInitPid) } From 1483b1d019052b1d8b978415f59a5b5278d8a709 Mon Sep 17 00:00:00 2001 From: Vladimir Savchenko Date: Sat, 4 Jul 2026 16:30:41 +0000 Subject: [PATCH 15/16] fix: use containerd task metrics API for non-runc runtimes (gVisor) Under gVisor (io.containerd.runsc.v1), runc has no knowledge of containers since they are managed by the runsc shim. The existing Statser shells out to "runc events --stats" which fails for gVisor containers, causing all memory/CPU metrics to report as 0. This commit adds a ContainerdStatser that uses the containerd task Metrics() API to read cgroup stats directly. The containerd runtime properly tracks the cgroup hierarchy for all shim types including runsc. The fix is wired in WireContainerd: when RuntimeType is not runc (e.g. io.containerd.runsc.v1), the ContainerdStatser is used instead of the runc-based one. Verified: cgroupsv2 memory.stat files exist on host for gVisor containers with real data (anon, file, inactive_file, etc). The containerd ctr task metrics command correctly returns this data. --- guardiancmd/command_linux.go | 6 + rundmc/runcontainerd/containerd_statser.go | 122 +++++++++++++++++++++ rundmc/runcontainerd/nerd/nerd.go | 36 ++++++ 3 files changed, 164 insertions(+) create mode 100644 rundmc/runcontainerd/containerd_statser.go diff --git a/guardiancmd/command_linux.go b/guardiancmd/command_linux.go index c664f97b3..55a21942f 100644 --- a/guardiancmd/command_linux.go +++ b/guardiancmd/command_linux.go @@ -156,6 +156,12 @@ func (f *LinuxFactory) WireContainerd(processBuilder *processes.ProcBuilder, use cgroupManager := runcontainerd.NewCgroupManager(containerdRuncRoot(), containerdNamespace) + // For non-runc runtimes (e.g. gVisor/runsc), use containerd task API for metrics + // instead of shelling out to runc which has no knowledge of the container. + if runtimeType != plugins.RuntimeRuncV2 { + statser = runcontainerd.NewContainerdStatser(nerd) + } + containerdManager := runcontainerd.New(nerd, nerd, processBuilder, userLookupper, wireExecer(pidGetter), statser, f.config.Containerd.UseContainerdForProcesses, cgroupManager, f.WireMkdirer(), peaHandlesGetter, f.config.Containers.CleanupProcessDirsOnWait, nerdStopper) peaRunner := runcontainerd.NewRunContainerPea(containerdManager, nerd, volumizer, f.config.Containers.CleanupProcessDirsOnWait) diff --git a/rundmc/runcontainerd/containerd_statser.go b/rundmc/runcontainerd/containerd_statser.go new file mode 100644 index 000000000..f51706b9c --- /dev/null +++ b/rundmc/runcontainerd/containerd_statser.go @@ -0,0 +1,122 @@ +//go:build !windows + +package runcontainerd + +import ( + "fmt" + "time" + + "code.cloudfoundry.org/garden" + "code.cloudfoundry.org/guardian/gardener" + "code.cloudfoundry.org/lager/v3" + cgroup2stats "github.com/containerd/cgroups/v3/cgroup2/stats" + "github.com/containerd/typeurl/v2" +) + +// ContainerMetricsGetter abstracts the containerd task metrics retrieval. +type ContainerMetricsGetter interface { + TaskMetrics(log lager.Logger, containerID string) (*cgroup2stats.Metrics, time.Time, error) +} + +// ContainerdStatser implements the Statser interface using containerd task +// metrics API instead of shelling out to runc. This is required for non-runc +// runtimes (e.g. gVisor/runsc) where "runc events --stats" does not work +// because runc has no knowledge of the container. +type ContainerdStatser struct { + metricsGetter ContainerMetricsGetter +} + +func NewContainerdStatser(metricsGetter ContainerMetricsGetter) *ContainerdStatser { + return &ContainerdStatser{metricsGetter: metricsGetter} +} + +func (s *ContainerdStatser) Stats(log lager.Logger, id string) (gardener.StatsContainerMetrics, error) { + log = log.Session("containerd-statser", lager.Data{"id": id}) + + metrics, createdAt, err := s.metricsGetter.TaskMetrics(log, id) + if err != nil { + return gardener.StatsContainerMetrics{}, fmt.Errorf("containerd task metrics: %w", err) + } + + var memoryStat garden.ContainerMemoryStat + if metrics.Memory != nil { + memoryStat = garden.ContainerMemoryStat{ + Anon: metrics.Memory.Anon, + File: metrics.Memory.File, + ActiveAnon: metrics.Memory.ActiveAnon, + InactiveAnon: metrics.Memory.InactiveAnon, + ActiveFile: metrics.Memory.ActiveFile, + InactiveFile: metrics.Memory.InactiveFile, + Unevictable: metrics.Memory.Unevictable, + MappedFile: metrics.Memory.FileMapped, + Pgfault: metrics.Memory.Pgfault, + Pgmajfault: metrics.Memory.Pgmajfault, + SwapCached: 0, // cgroupsv2 does not expose swapcached separately + // cgroupsv2 does not have hierarchical_memory_limit; + // usage_limit is the closest equivalent + HierarchicalMemoryLimit: metrics.Memory.UsageLimit, + // Populate TotalXxx with the same values (cgroupsv2 unified hierarchy + // means there is no separate "total" vs "local" distinction) + TotalActiveAnon: metrics.Memory.ActiveAnon, + TotalActiveFile: metrics.Memory.ActiveFile, + TotalInactiveAnon: metrics.Memory.InactiveAnon, + TotalInactiveFile: metrics.Memory.InactiveFile, + TotalUnevictable: metrics.Memory.Unevictable, + TotalMappedFile: metrics.Memory.FileMapped, + TotalPgfault: metrics.Memory.Pgfault, + TotalPgmajfault: metrics.Memory.Pgmajfault, + // For cgroupsv2, "Rss" is essentially Anon + swap-backed anon + Rss: metrics.Memory.Anon, + TotalRss: metrics.Memory.Anon, + // Cache = file-backed pages + Cache: metrics.Memory.File, + TotalCache: metrics.Memory.File, + // Swap + Swap: metrics.Memory.SwapUsage, + TotalSwap: metrics.Memory.SwapUsage, + } + // Calculate TotalUsageTowardLimit the same way garden does for cgroupsv2: + // File + Anon + SwapCached - InactiveFile + totalMemoryUsage := memoryStat.File + memoryStat.Anon + memoryStat.SwapCached + if memoryStat.InactiveFile > totalMemoryUsage { + memoryStat.TotalUsageTowardLimit = 0 + } else { + memoryStat.TotalUsageTowardLimit = totalMemoryUsage - memoryStat.InactiveFile + } + } + + var cpuStat garden.ContainerCPUStat + if metrics.CPU != nil { + // cgroupsv2 reports CPU in microseconds; garden expects nanoseconds + cpuStat = garden.ContainerCPUStat{ + Usage: metrics.CPU.UsageUsec * 1000, + User: metrics.CPU.UserUsec * 1000, + System: metrics.CPU.SystemUsec * 1000, + } + } + + var pidStat garden.ContainerPidStat + if metrics.Pids != nil { + pidStat = garden.ContainerPidStat{ + Current: metrics.Pids.Current, + Max: metrics.Pids.Limit, + } + } + + return gardener.StatsContainerMetrics{ + CPU: cpuStat, + Memory: memoryStat, + Pid: pidStat, + Age: time.Since(createdAt), + }, nil +} + +// UnmarshalContainerdMetrics unmarshals the protobuf Any from containerd task +// metrics response into cgroupsv2 Metrics. +func UnmarshalContainerdMetrics(data typeurl.Any) (*cgroup2stats.Metrics, error) { + var m cgroup2stats.Metrics + if err := typeurl.UnmarshalTo(data, &m); err != nil { + return nil, fmt.Errorf("unmarshal cgroupsv2 metrics: %w", err) + } + return &m, nil +} diff --git a/rundmc/runcontainerd/nerd/nerd.go b/rundmc/runcontainerd/nerd/nerd.go index 60125952f..a517004d9 100644 --- a/rundmc/runcontainerd/nerd/nerd.go +++ b/rundmc/runcontainerd/nerd/nerd.go @@ -23,6 +23,7 @@ import ( ctrdevents "github.com/containerd/containerd/v2/core/events" "github.com/containerd/containerd/v2/pkg/cio" "github.com/containerd/errdefs" + cgroup2stats "github.com/containerd/cgroups/v3/cgroup2/stats" "github.com/containerd/typeurl/v2" specs "github.com/opencontainers/runtime-spec/specs-go" @@ -466,3 +467,38 @@ func updateTaskInfoCreateOptions(taskInfo *client.TaskInfo, updateCreateOptions return updateCreateOptions(opts) } + +// TaskMetrics retrieves cgroup metrics for a container via the containerd task +// API. Returns cgroupsv2 Metrics and the container creation time (used for Age). +func (n *Nerd) TaskMetrics(log lager.Logger, containerID string) (*cgroup2stats.Metrics, time.Time, error) { + log = log.Session("task-metrics", lager.Data{"containerID": containerID}) + + container, task, err := n.loadContainerAndTask(log, containerID) + if err != nil { + return nil, time.Time{}, err + } + + metric, err := task.Metrics(n.context) + if err != nil { + return nil, time.Time{}, fmt.Errorf("task.Metrics: %w", err) + } + + if metric.Data == nil { + return nil, time.Time{}, fmt.Errorf("no metrics data for container %s", containerID) + } + + cgMetrics, err := runcontainerd.UnmarshalContainerdMetrics(metric.Data) + if err != nil { + return nil, time.Time{}, err + } + + // Get creation time from container info + info, err := container.Info(n.context) + if err != nil { + // Non-fatal: use zero time (Age will be large but metrics still report) + log.Error("get-info-for-creation-time", err) + return cgMetrics, time.Time{}, nil + } + + return cgMetrics, info.CreatedAt, nil +} From 761fc06552af7e7fbf0b9647a6e7d7cd29aebc74 Mon Sep 17 00:00:00 2001 From: Vladimir Savchenko Date: Thu, 9 Jul 2026 06:27:11 +0300 Subject: [PATCH 16/16] Remove dead code: warmARPCache (never called) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The warmARPCache function in external_networker.go was written as a host-side workaround for silk-CNI ARP resolution but was never wired into any call site. The actual fix lives in gVisor's network.go (collectLinksAndRoutes) where ARP warming runs at sandbox startup — the correct placement since gVisor consumes the entries immediately. setupLoopback (which IS called) is retained. --- netplugin/external_networker.go | 122 -------------------------------- 1 file changed, 122 deletions(-) diff --git a/netplugin/external_networker.go b/netplugin/external_networker.go index 1b77dc131..99013fb14 100644 --- a/netplugin/external_networker.go +++ b/netplugin/external_networker.go @@ -8,7 +8,6 @@ import ( "net" "os/exec" "strings" - "time" "code.cloudfoundry.org/commandrunner" "code.cloudfoundry.org/garden" @@ -193,127 +192,6 @@ func setupLoopback(log lager.Logger, pid int) error { return nil } -// warmARPCache ensures the container's network namespace has resolved ARP -// entries for all gateway IPs. gVisor's netstack copies ARP entries from the -// namespace at startup but only those in "live" states (PERMANENT, REACHABLE, -// STALE). On silk CNI with point-to-point links, gateway ARP entries may not -// exist until triggered. This function probes each gateway with a UDP packet -// to force ARP resolution, then promotes entries to PERMANENT so gVisor -// reliably picks them up. -func warmARPCache(log lager.Logger, pid int) error { - containerNS, err := netns.GetFromPid(pid) - if err != nil { - return fmt.Errorf("getting netns for pid %d: %w", pid, err) - } - defer containerNS.Close() - - nlh, err := netlink.NewHandleAt(containerNS) - if err != nil { - return fmt.Errorf("creating netlink handle: %w", err) - } - defer nlh.Close() - - // Find all gateway IPs from routes in the container namespace. - routes, err := nlh.RouteList(nil, netlink.FAMILY_ALL) - if err != nil { - return fmt.Errorf("listing routes: %w", err) - } - - gwSet := map[string]net.IP{} - for _, r := range routes { - if r.Gw != nil && !r.Gw.IsUnspecified() { - gwSet[r.Gw.String()] = r.Gw - } - } - - if len(gwSet) == 0 { - return nil - } - - // Check which gateways already have ARP entries. - links, _ := nlh.LinkList() - var primaryLink netlink.Link - for _, l := range links { - if l.Attrs().Name != "lo" { - primaryLink = l - break - } - } - - if primaryLink == nil { - return nil - } - - neighs, _ := nlh.NeighList(primaryLink.Attrs().Index, netlink.FAMILY_ALL) - hasEntry := func(ip net.IP) bool { - for _, n := range neighs { - if n.IP.Equal(ip) && len(n.HardwareAddr) > 0 { - return true - } - } - return false - } - - // Probe unresolved gateways with a UDP packet to trigger ARP. - probed := false - for _, gw := range gwSet { - if hasEntry(gw) { - continue - } - network := "udp4" - if gw.To4() == nil { - network = "udp6" - } - conn, err := net.DialUDP(network, nil, &net.UDPAddr{IP: gw, Port: 9}) - if err != nil { - log.Info("arp-warm-probe-failed", lager.Data{"gw": gw.String(), "error": err.Error()}) - continue - } - conn.Write([]byte{}) - conn.Close() - probed = true - } - - if probed { - time.Sleep(75 * time.Millisecond) // wait for ARP reply - } - - // Re-read neighbor table and promote entries to PERMANENT. - neighs, err = nlh.NeighList(primaryLink.Attrs().Index, netlink.FAMILY_ALL) - if err != nil { - return fmt.Errorf("re-reading neighbor table: %w", err) - } - - const liveStates = netlink.NUD_REACHABLE | netlink.NUD_STALE | netlink.NUD_DELAY | netlink.NUD_PROBE | netlink.NUD_PERMANENT - promoted := 0 - for _, n := range neighs { - if len(n.HardwareAddr) == 0 || n.State&liveStates == 0 { - continue - } - if _, isGW := gwSet[n.IP.String()]; !isGW { - continue - } - if n.State == netlink.NUD_PERMANENT { - continue // already permanent - } - // Promote to PERMANENT so gVisor's scraper picks it up. - err := nlh.NeighSet(&netlink.Neigh{ - LinkIndex: primaryLink.Attrs().Index, - IP: n.IP, - HardwareAddr: n.HardwareAddr, - State: netlink.NUD_PERMANENT, - }) - if err != nil { - log.Info("arp-promote-failed", lager.Data{"ip": n.IP.String(), "error": err.Error()}) - } else { - promoted++ - } - } - - log.Info("warm-arp-cache-done", lager.Data{"pid": pid, "gateways": len(gwSet), "promoted": promoted}) - return nil -} - func (p *externalBinaryNetworker) Destroy(log lager.Logger, handle string) error { err := p.exec(log, "down", handle, nil, nil) if err != nil {