From 8400fe2134e274d3602fff6bf2e375c4bb69c421 Mon Sep 17 00:00:00 2001 From: Mayur Das Date: Fri, 3 Jul 2026 10:26:14 +0530 Subject: [PATCH] feat(network): support --aux-address on network create Docker's network create reserves auxiliary addresses so IPAM never hands them out to containers. nerdctl had no equivalent. host-local has no exclude list, but it allocates across every range in a set, so each reserved IP is carved out by splitting the subnet range around it. The reserved name=IP pairs are matched to the subnet that contains them (so dual-stack picks the right family), rejected when they hit the network or gateway address or fall outside every subnet, and recorded so network inspect reports AuxiliaryAddresses the same way Docker does. Signed-off-by: Mayur Das --- cmd/nerdctl/network/network_create.go | 32 ++-- .../network/network_create_linux_test.go | 70 ++++++++ docs/command-reference.md | 3 +- pkg/api/types/network_types.go | 7 +- pkg/cmd/network/create.go | 10 ++ pkg/cmd/network/create_test.go | 35 ++++ pkg/inspecttypes/dockercompat/dockercompat.go | 22 ++- .../dockercompat/dockercompat_test.go | 19 ++ pkg/netutil/cni_plugin.go | 5 + pkg/netutil/netutil.go | 122 ++++++++++++- pkg/netutil/netutil_test.go | 165 +++++++++++++++++- pkg/netutil/netutil_unix.go | 76 ++++++-- pkg/netutil/netutil_unix_test.go | 90 +++++++++- pkg/netutil/netutil_windows.go | 6 +- 14 files changed, 622 insertions(+), 40 deletions(-) create mode 100644 pkg/cmd/network/create_test.go diff --git a/cmd/nerdctl/network/network_create.go b/cmd/nerdctl/network/network_create.go index 596fa5f785f..5923219239c 100644 --- a/cmd/nerdctl/network/network_create.go +++ b/cmd/nerdctl/network/network_create.go @@ -49,6 +49,7 @@ func createCommand() *cobra.Command { cmd.Flags().StringArray("subnet", nil, `Subnet in CIDR format that represents a network segment, e.g. "10.5.0.0/16"`) cmd.Flags().StringArray("gateway", nil, "IPv4 or IPv6 Gateway for the master subnet") cmd.Flags().StringArray("ip-range", nil, `Allocate container ip from a sub-range`) + cmd.Flags().StringArray("aux-address", nil, "Auxiliary IPv4 or IPv6 addresses used by Network driver, as name=IP pairs. The IPs are reserved and never assigned to containers") cmd.Flags().StringArray("label", nil, "Set metadata for a network") cmd.Flags().Bool("ipv4", true, "Enable IPv4 networking (set to false together with --ipv6 for an IPv6-only network)") cmd.Flags().Bool("ipv6", false, "Enable IPv6 networking") @@ -93,6 +94,10 @@ func createAction(cmd *cobra.Command, args []string) error { if err != nil { return err } + auxAddresses, err := cmd.Flags().GetStringArray("aux-address") + if err != nil { + return err + } labels, err := cmd.Flags().GetStringArray("label") if err != nil { return err @@ -112,18 +117,19 @@ func createAction(cmd *cobra.Command, args []string) error { } return network.Create(types.NetworkCreateOptions{ - GOptions: globalOptions, - Name: name, - Driver: driver, - Options: strutil.ConvertKVStringsToMap(opts), - IPAMDriver: ipamDriver, - IPAMOptions: strutil.ConvertKVStringsToMap(ipamOpts), - Subnets: subnets, - Gateway: gateways, - IPRange: ipRanges, - Labels: labels, - IPv6: ipv6, - IPv4: &ipv4, - Internal: internal, + GOptions: globalOptions, + Name: name, + Driver: driver, + Options: strutil.ConvertKVStringsToMap(opts), + IPAMDriver: ipamDriver, + IPAMOptions: strutil.ConvertKVStringsToMap(ipamOpts), + Subnets: subnets, + Gateway: gateways, + IPRange: ipRanges, + AuxAddresses: auxAddresses, + Labels: labels, + IPv6: ipv6, + IPv4: &ipv4, + Internal: internal, }, cmd.OutOrStdout()) } diff --git a/cmd/nerdctl/network/network_create_linux_test.go b/cmd/nerdctl/network/network_create_linux_test.go index d0d113e7e8d..589c6a88eb0 100644 --- a/cmd/nerdctl/network/network_create_linux_test.go +++ b/cmd/nerdctl/network/network_create_linux_test.go @@ -256,6 +256,76 @@ func TestNetworkCreate(t *testing.T) { } }, }, + { + Description: "with aux-address", + Setup: func(data test.Data, helpers test.Helpers) { + helpers.Ensure("network", "create", data.Identifier(), + "--subnet", "10.6.0.0/24", + "--gateway", "10.6.0.1", + "--aux-address", "router=10.6.0.5", + "--aux-address", "dns=10.6.0.6", + ) + }, + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("network", "rm", data.Identifier()) + }, + Command: func(data test.Data, helpers test.Helpers) test.TestableCommand { + return helpers.Command("network", "inspect", data.Identifier()) + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + ExitCode: expect.ExitCodeSuccess, + Output: func(stdout string, t tig.T) { + netw := nerdtest.InspectNetwork(helpers, data.Identifier()) + var aux map[string]string + for _, c := range netw.IPAM.Config { + if c.Subnet == "10.6.0.0/24" { + aux = c.AuxiliaryAddresses + } + } + assert.Equal(t, aux["router"], "10.6.0.5") + assert.Equal(t, aux["dns"], "10.6.0.6") + }, + } + }, + }, + { + Description: "aux-address is reserved", + Setup: func(data test.Data, helpers test.Helpers) { + helpers.Ensure("network", "create", data.Identifier(), + "--subnet", "10.6.1.0/24", + "--aux-address", "reserved=10.6.1.5", + ) + }, + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("network", "rm", data.Identifier()) + }, + Command: func(data test.Data, helpers test.Helpers) test.TestableCommand { + // The reserved address is carved out of the range, so requesting + // it explicitly must fail just as it does on Docker. + return helpers.Command("run", "--rm", "--net", data.Identifier(), "--ip", "10.6.1.5", testutil.CommonImage, "true") + }, + Expected: test.Expects(expect.ExitCodeGenericFail, nil, nil), + }, + { + Description: "an un-reserved address is allocatable", + Setup: func(data test.Data, helpers test.Helpers) { + helpers.Ensure("network", "create", data.Identifier(), + "--subnet", "10.6.2.0/24", + "--aux-address", "reserved=10.6.2.5", + ) + }, + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("network", "rm", data.Identifier()) + }, + Command: func(data test.Data, helpers test.Helpers) test.TestableCommand { + // Positive control: an address in the same subnet that is not + // reserved allocates fine, so the failure above is specific to the + // reserved IP rather than an unrelated --ip problem. + return helpers.Command("run", "--rm", "--net", data.Identifier(), "--ip", "10.6.2.7", testutil.CommonImage, "true") + }, + Expected: test.Expects(expect.ExitCodeSuccess, nil, nil), + }, } testCase.Run(t) diff --git a/docs/command-reference.md b/docs/command-reference.md index 93c481b4c11..28566a5155e 100644 --- a/docs/command-reference.md +++ b/docs/command-reference.md @@ -1279,12 +1279,13 @@ Flags: - :whale: `--subnet`: Subnet in CIDR format that represents a network segment, e.g. "10.5.0.0/16" - :whale: `--gateway`: IPv4 or IPv6 Gateway for the master subnet - :whale: `--ip-range`: Allocate container ip from a sub-range +- :whale: `--aux-address`: Auxiliary IPv4 or IPv6 addresses, as `name=IP` pairs. Each IP is reserved and never assigned to a container. Repeatable, and matched to the subnet that contains it. - :whale: `--label`: Set metadata on a network - :whale: `--ipv4`: Enable IPv4. Enabled by default; set to false with `--ipv6` and an IPv6 subnet for an IPv6-only network. `--ipv4=false` is not supported on Windows. - :whale: `--ipv6`: Enable IPv6. Should be used with a valid subnet. - :whale: `--internal`: Restrict external access to the network. -Unimplemented `docker network create` flags: `--attachable`, `--aux-address`, `--config-from`, `--config-only`, `--ingress`, `--scope` +Unimplemented `docker network create` flags: `--attachable`, `--config-from`, `--config-only`, `--ingress`, `--scope` ### :whale: nerdctl network ls diff --git a/pkg/api/types/network_types.go b/pkg/api/types/network_types.go index 90969d6612e..70b5e6e4aab 100644 --- a/pkg/api/types/network_types.go +++ b/pkg/api/types/network_types.go @@ -33,8 +33,11 @@ type NetworkCreateOptions struct { Subnets []string Gateway []string IPRange []string - Labels []string - IPv6 bool + // AuxAddresses holds "name=IP" auxiliary addresses (docker --aux-address). + // Each IP is reserved so IPAM never hands it out to a container. + AuxAddresses []string + Labels []string + IPv6 bool // IPv4 enables IPv4 on the network. A nil value defaults to enabled, so a // directly-constructed NetworkCreateOptions keeps IPv4 on; setting it to // false together with IPv6 yields an IPv6-only network. diff --git a/pkg/cmd/network/create.go b/pkg/cmd/network/create.go index f64ba519f3a..b93b92d7a1f 100644 --- a/pkg/cmd/network/create.go +++ b/pkg/cmd/network/create.go @@ -41,6 +41,16 @@ func Create(options types.NetworkCreateOptions, stdout io.Writer) error { return fmt.Errorf("IPv6-only network requires an IPv6 subnet, specify --subnet manually") } if len(options.Subnets) == 0 { + // Docker matches each aux-address to a subnet that contains it, so + // without any subnet there is nothing to match. Surface the same + // "no matching subnet for aux-address " error Docker returns. + aux, err := netutil.ParseAuxAddresses(options.AuxAddresses) + if err != nil { + return err + } + for _, ip := range aux { + return fmt.Errorf("no matching subnet for aux-address %s", ip) + } if len(options.Gateway) > 0 || len(options.IPRange) > 0 { return fmt.Errorf("cannot set gateway or ip-range without subnet, specify --subnet manually") } diff --git a/pkg/cmd/network/create_test.go b/pkg/cmd/network/create_test.go new file mode 100644 index 00000000000..74db8b1c292 --- /dev/null +++ b/pkg/cmd/network/create_test.go @@ -0,0 +1,35 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package network + +import ( + "io" + "testing" + + "gotest.tools/v3/assert" + + "github.com/containerd/nerdctl/v2/pkg/api/types" +) + +// TestCreateAuxAddressWithoutSubnet verifies that an aux-address given without +// any subnet is rejected the same way Docker rejects it, before any CNI setup. +func TestCreateAuxAddressWithoutSubnet(t *testing.T) { + err := Create(types.NetworkCreateOptions{ + AuxAddresses: []string{"host=10.9.0.5"}, + }, io.Discard) + assert.ErrorContains(t, err, "no matching subnet for aux-address 10.9.0.5") +} diff --git a/pkg/inspecttypes/dockercompat/dockercompat.go b/pkg/inspecttypes/dockercompat/dockercompat.go index 1e7211307fd..6d7a86133c4 100644 --- a/pkg/inspecttypes/dockercompat/dockercompat.go +++ b/pkg/inspecttypes/dockercompat/dockercompat.go @@ -1052,9 +1052,10 @@ func getUlimitsFromNative(sp *specs.Spec) ([]*units.Ulimit, error) { } type IPAMConfig struct { - Subnet string `json:"Subnet,omitempty"` - Gateway string `json:"Gateway,omitempty"` - IPRange string `json:"IPRange,omitempty"` + Subnet string `json:"Subnet,omitempty"` + Gateway string `json:"Gateway,omitempty"` + IPRange string `json:"IPRange,omitempty"` + AuxiliaryAddresses map[string]string `json:"AuxiliaryAddresses,omitempty"` } type IPAM struct { @@ -1196,7 +1197,20 @@ func NetworkFromNative(n *native.Network) (*Network, error) { res.Name = sCNI.Name for _, plugin := range sCNI.Plugins { for _, ranges := range plugin.Ipam.Ranges { - res.IPAM.Config = append(res.IPAM.Config, ranges...) + // A range-set normally describes one subnet; an aux-address + // reservation splits it into several sub-ranges whose first entry + // carries the subnet, gateway, ip-range and aux-addresses. Report the + // first entry per distinct subnet so a split subnet collapses to one + // IPAM.Config like Docker, without dropping entries for different + // subnets in the same set. + seen := make(map[string]struct{}, len(ranges)) + for _, r := range ranges { + if _, ok := seen[r.Subnet]; ok { + continue + } + seen[r.Subnet] = struct{}{} + res.IPAM.Config = append(res.IPAM.Config, r) + } } } diff --git a/pkg/inspecttypes/dockercompat/dockercompat_test.go b/pkg/inspecttypes/dockercompat/dockercompat_test.go index 4c9986ca3b5..b269898c781 100644 --- a/pkg/inspecttypes/dockercompat/dockercompat_test.go +++ b/pkg/inspecttypes/dockercompat/dockercompat_test.go @@ -605,6 +605,25 @@ func TestGetUlimitsFromNative(t *testing.T) { } } +func TestNetworkFromNative(t *testing.T) { + // The first range-set is one subnet split into sub-ranges by an aux-address + // reservation and must collapse to a single IPAM.Config carrying the aux; the + // second set holds two distinct subnets that must both be kept; the empty set + // contributes nothing. + cni := `{"name":"testnet","plugins":[{"ipam":{"ranges":[` + + `[{"Subnet":"10.6.0.0/24","Gateway":"10.6.0.1","AuxiliaryAddresses":{"router":"10.6.0.5"}},{"Subnet":"10.6.0.0/24"}],` + + `[{"Subnet":"10.7.0.0/24"},{"Subnet":"10.8.0.0/24"}],` + + `[]` + + `]}}]}` + got, err := NetworkFromNative(&native.Network{CNI: []byte(cni)}) + assert.NilError(t, err) + assert.DeepEqual(t, []IPAMConfig{ + {Subnet: "10.6.0.0/24", Gateway: "10.6.0.1", AuxiliaryAddresses: map[string]string{"router": "10.6.0.5"}}, + {Subnet: "10.7.0.0/24"}, + {Subnet: "10.8.0.0/24"}, + }, got.IPAM.Config) +} + func TestNetworkSettingsFromNative(t *testing.T) { tempStateDir, err := os.MkdirTemp(t.TempDir(), "rw") if err != nil { diff --git a/pkg/netutil/cni_plugin.go b/pkg/netutil/cni_plugin.go index b44e76042e2..26dc76cb738 100644 --- a/pkg/netutil/cni_plugin.go +++ b/pkg/netutil/cni_plugin.go @@ -34,6 +34,11 @@ type IPAMRange struct { RangeEnd string `json:"rangeEnd,omitempty"` Gateway string `json:"gateway,omitempty"` IPRange string `json:"ipRange,omitempty"` + // AuxiliaryAddresses records the reserved name=IP pairs for this subnet. + // host-local does not read it (reservation is done by splitting the range + // around each IP); it is nerdctl bookkeeping so `network inspect` can + // report AuxiliaryAddresses the way Docker does. + AuxiliaryAddresses map[string]string `json:"auxiliaryAddresses,omitempty"` } type IPAMRoute struct { diff --git a/pkg/netutil/netutil.go b/pkg/netutil/netutil.go index 026b210df65..5a220cd2bb4 100644 --- a/pkg/netutil/netutil.go +++ b/pkg/netutil/netutil.go @@ -17,6 +17,7 @@ package netutil import ( + "bytes" "context" "crypto/sha256" "encoding/hex" @@ -28,6 +29,7 @@ import ( "path/filepath" "sort" "strconv" + "strings" "github.com/containernetworking/cni/libcni" @@ -341,7 +343,7 @@ func (e *CNIEnv) CreateNetwork(opts types.NetworkCreateOptions) (*NetworkConfig, } // A nil IPv4 defaults to enabled. ipv4 := opts.IPv4 == nil || *opts.IPv4 - ipam, err := e.generateIPAM(opts.IPAMDriver, opts.Subnets, opts.Gateway, opts.IPRange, opts.IPAMOptions, opts.IPv6, ipv4, opts.Internal) + ipam, err := e.generateIPAM(opts.IPAMDriver, opts.Subnets, opts.Gateway, opts.IPRange, opts.AuxAddresses, opts.IPAMOptions, opts.IPv6, ipv4, opts.Internal) if err != nil { return nil, err } @@ -621,6 +623,124 @@ func parseIPAMRange(subnet *net.IPNet, gatewayStr, ipRangeStr string) (*IPAMRang return res, nil } +// ParseAuxAddresses parses Docker-style "name=IP" auxiliary-address pairs into a +// name-to-IP map. An entry with an empty IP (including one with no "=") is +// dropped; a later entry overrides an earlier one with the same name, matching +// Docker; and a non-empty but unparsable IP is an error. +func ParseAuxAddresses(raw []string) (map[string]string, error) { + if len(raw) == 0 { + return nil, nil + } + aux := make(map[string]string, len(raw)) + for _, kv := range raw { + name, ip, _ := strings.Cut(kv, "=") + if ip == "" { + continue + } + if net.ParseIP(ip) == nil { + return nil, fmt.Errorf("invalid aux-address %q", ip) + } + aux[name] = ip + } + if len(aux) == 0 { + return nil, nil + } + return aux, nil +} + +// splitIPAMRange reserves the given IPs inside a subnet's allocation range by +// carving them out. host-local has no exclude list, but it does allocate across +// every range in a set, so the reserved IPs become gaps between sub-ranges and +// are never handed out. Reserved IPs outside the allocation window need no split +// (host-local cannot reach them anyway). The base range's gateway and ip-range +// are kept on the first sub-range so the rest of the pipeline and `network +// inspect` behave exactly as the un-split case. +func splitIPAMRange(subnet *net.IPNet, base *IPAMRange, reserved []net.IP) ([]IPAMRange, error) { + if len(reserved) == 0 { + return []IPAMRange{*base}, nil + } + + // Resolve the allocation window. With an ip-range the base already carries + // its bounds; otherwise it is the whole subnet minus the all-ones address. + start := net.ParseIP(base.RangeStart) + if start == nil { + start, _ = subnetutil.FirstIPInSubnet(subnet) + } + end := net.ParseIP(base.RangeEnd) + if end == nil { + last, _ := subnetutil.LastIPInSubnet(subnet) + end = ipDec(last) + } + + // Keep only the reserved IPs that fall within the window, sorted ascending. + inWindow := make([]net.IP, 0, len(reserved)) + for _, ip := range reserved { + if bytes.Compare(ip.To16(), start.To16()) >= 0 && bytes.Compare(ip.To16(), end.To16()) <= 0 { + inWindow = append(inWindow, ip) + } + } + if len(inWindow) == 0 { + return []IPAMRange{*base}, nil + } + sort.Slice(inWindow, func(i, j int) bool { + return bytes.Compare(inWindow[i].To16(), inWindow[j].To16()) < 0 + }) + + // Walk the window left to right, emitting a sub-range for each gap between + // reserved IPs. + var out []IPAMRange + cur := start + for _, r := range inWindow { + hi := ipDec(r) + if bytes.Compare(cur.To16(), hi.To16()) <= 0 { + out = append(out, IPAMRange{Subnet: subnet.String(), RangeStart: cur.String(), RangeEnd: hi.String()}) + } + cur = ipInc(r) + } + if bytes.Compare(cur.To16(), end.To16()) <= 0 { + out = append(out, IPAMRange{Subnet: subnet.String(), RangeStart: cur.String(), RangeEnd: end.String()}) + } + if len(out) == 0 { + return nil, fmt.Errorf("aux-address reservations leave no allocatable IPs in subnet %s", subnet) + } + + // host-local reserves the gateway only when it is set on the range it lands + // in, and after splitting the gateway can be in any sub-range, so set it on + // all of them. The original ip-range is nerdctl-only bookkeeping for inspect, + // so keep it on the first sub-range alone. + for i := range out { + out[i].Gateway = base.Gateway + } + out[0].IPRange = base.IPRange + return out, nil +} + +// ipInc returns ip+1 and ipDec returns ip-1, both in 16-byte form and on a copy +// so the caller's IP is left untouched. +func ipInc(ip net.IP) net.IP { + out := make(net.IP, net.IPv6len) + copy(out, ip.To16()) + for i := len(out) - 1; i >= 0; i-- { + out[i]++ + if out[i] != 0 { + break + } + } + return out +} + +func ipDec(ip net.IP) net.IP { + out := make(net.IP, net.IPv6len) + copy(out, ip.To16()) + for i := len(out) - 1; i >= 0; i-- { + out[i]-- + if out[i] != 0xff { + break + } + } + return out +} + // convert the struct to a map func structToMap(in interface{}) (map[string]interface{}, error) { out := make(map[string]interface{}) diff --git a/pkg/netutil/netutil_test.go b/pkg/netutil/netutil_test.go index f818c59a1b2..72cd294461c 100644 --- a/pkg/netutil/netutil_test.go +++ b/pkg/netutil/netutil_test.go @@ -128,11 +128,174 @@ func TestParseIPAMRange(t *testing.T) { assert.ErrorContains(t, err, tc.err) } else { assert.NilError(t, err) - assert.Equal(t, *tc.expected, *got) + assert.DeepEqual(t, *tc.expected, *got) } } } +func TestParseAuxAddresses(t *testing.T) { + t.Parallel() + type testCase struct { + raw []string + expected map[string]string + err string + } + testCases := []testCase{ + { + raw: nil, + expected: nil, + }, + { + raw: []string{"router=10.1.100.5", "dns=10.1.100.6"}, + expected: map[string]string{"router": "10.1.100.5", "dns": "10.1.100.6"}, + }, + { + // An empty name is allowed, matching Docker. + raw: []string{"=10.1.100.5"}, + expected: map[string]string{"": "10.1.100.5"}, + }, + { + // An entry with no "=" has an empty IP and is dropped, matching Docker. + raw: []string{"10.1.100.5"}, + expected: nil, + }, + { + // A later value overrides an earlier one with the same name. + raw: []string{"a=10.1.100.5", "a=10.1.100.6"}, + expected: map[string]string{"a": "10.1.100.6"}, + }, + { + raw: []string{"v6=2001:db8::5"}, + expected: map[string]string{"v6": "2001:db8::5"}, + }, + { + raw: []string{"bad=not-an-ip"}, + err: "invalid aux-address", + }, + } + for _, tc := range testCases { + got, err := ParseAuxAddresses(tc.raw) + if tc.err != "" { + assert.ErrorContains(t, err, tc.err) + continue + } + assert.NilError(t, err) + assert.DeepEqual(t, tc.expected, got) + } +} + +func TestSplitIPAMRange(t *testing.T) { + t.Parallel() + ips := func(addrs ...string) []net.IP { + out := make([]net.IP, len(addrs)) + for i, a := range addrs { + out[i] = net.ParseIP(a) + } + return out + } + type testCase struct { + name string + subnet string + base *IPAMRange + reserved []net.IP + expected []IPAMRange + err string + } + testCases := []testCase{ + { + name: "no reservation leaves the range untouched", + subnet: "10.1.100.0/24", + base: &IPAMRange{Subnet: "10.1.100.0/24", Gateway: "10.1.100.1"}, + reserved: nil, + expected: []IPAMRange{{Subnet: "10.1.100.0/24", Gateway: "10.1.100.1"}}, + }, + { + name: "a mid-subnet reservation splits the range in two", + subnet: "10.1.100.0/24", + base: &IPAMRange{Subnet: "10.1.100.0/24", Gateway: "10.1.100.1"}, + reserved: ips("10.1.100.5"), + expected: []IPAMRange{ + {Subnet: "10.1.100.0/24", RangeStart: "10.1.100.1", RangeEnd: "10.1.100.4", Gateway: "10.1.100.1"}, + {Subnet: "10.1.100.0/24", RangeStart: "10.1.100.6", RangeEnd: "10.1.100.254", Gateway: "10.1.100.1"}, + }, + }, + { + name: "two reservations produce three sub-ranges", + subnet: "10.1.100.0/24", + base: &IPAMRange{Subnet: "10.1.100.0/24", Gateway: "10.1.100.1"}, + reserved: ips("10.1.100.6", "10.1.100.5"), + expected: []IPAMRange{ + {Subnet: "10.1.100.0/24", RangeStart: "10.1.100.1", RangeEnd: "10.1.100.4", Gateway: "10.1.100.1"}, + {Subnet: "10.1.100.0/24", RangeStart: "10.1.100.7", RangeEnd: "10.1.100.254", Gateway: "10.1.100.1"}, + }, + }, + { + // The gateway is the first usable address, so reserving the next one + // leaves a gateway-only sub-range; host-local reserves the gateway, so + // allocation still starts after the reservation. + name: "a reservation right after the gateway leaves a gateway-only range", + subnet: "10.1.100.0/24", + base: &IPAMRange{Subnet: "10.1.100.0/24", Gateway: "10.1.100.1"}, + reserved: ips("10.1.100.2"), + expected: []IPAMRange{ + {Subnet: "10.1.100.0/24", RangeStart: "10.1.100.1", RangeEnd: "10.1.100.1", Gateway: "10.1.100.1"}, + {Subnet: "10.1.100.0/24", RangeStart: "10.1.100.3", RangeEnd: "10.1.100.254", Gateway: "10.1.100.1"}, + }, + }, + { + name: "a reservation inside an ip-range splits within its bounds", + subnet: "10.1.100.0/24", + base: &IPAMRange{Subnet: "10.1.100.0/24", Gateway: "10.1.100.1", IPRange: "10.1.100.0/28", RangeStart: "10.1.100.1", RangeEnd: "10.1.100.15"}, + reserved: ips("10.1.100.5"), + expected: []IPAMRange{ + {Subnet: "10.1.100.0/24", RangeStart: "10.1.100.1", RangeEnd: "10.1.100.4", Gateway: "10.1.100.1", IPRange: "10.1.100.0/28"}, + {Subnet: "10.1.100.0/24", RangeStart: "10.1.100.6", RangeEnd: "10.1.100.15", Gateway: "10.1.100.1"}, + }, + }, + { + name: "a reservation outside the ip-range needs no split", + subnet: "10.1.100.0/24", + base: &IPAMRange{Subnet: "10.1.100.0/24", Gateway: "10.1.100.1", IPRange: "10.1.100.0/28", RangeStart: "10.1.100.1", RangeEnd: "10.1.100.15"}, + reserved: ips("10.1.100.200"), + expected: []IPAMRange{ + {Subnet: "10.1.100.0/24", Gateway: "10.1.100.1", IPRange: "10.1.100.0/28", RangeStart: "10.1.100.1", RangeEnd: "10.1.100.15"}, + }, + }, + { + // Reserving every usable address in the window leaves nothing to hand + // out, which is an error rather than an empty range set. + name: "reservations leaving no allocatable address error", + subnet: "10.1.100.0/30", + base: &IPAMRange{Subnet: "10.1.100.0/30", Gateway: "10.1.100.1"}, + reserved: ips("10.1.100.1", "10.1.100.2"), + err: "leave no allocatable", + }, + { + name: "an IPv6 reservation splits the range around it", + subnet: "2001:db8::/64", + base: &IPAMRange{Subnet: "2001:db8::/64", Gateway: "2001:db8::1"}, + reserved: ips("2001:db8::5"), + expected: []IPAMRange{ + {Subnet: "2001:db8::/64", RangeStart: "2001:db8::1", RangeEnd: "2001:db8::4", Gateway: "2001:db8::1"}, + {Subnet: "2001:db8::/64", RangeStart: "2001:db8::6", RangeEnd: "2001:db8::ffff:ffff:ffff:fffe", Gateway: "2001:db8::1"}, + }, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + _, subnet, err := net.ParseCIDR(tc.subnet) + assert.NilError(t, err) + got, err := splitIPAMRange(subnet, tc.base, tc.reserved) + if tc.err != "" { + assert.ErrorContains(t, err, tc.err) + return + } + assert.NilError(t, err) + assert.DeepEqual(t, tc.expected, got) + }) + } +} + // Tests whether nerdctl properly creates the default network when required. // Note that this test will require a CNI driver bearing the same name as // the type of the default network. (denoted by netutil.DefaultNetworkName, diff --git a/pkg/netutil/netutil_unix.go b/pkg/netutil/netutil_unix.go index 7e2549da449..86127033798 100644 --- a/pkg/netutil/netutil_unix.go +++ b/pkg/netutil/netutil_unix.go @@ -215,10 +215,16 @@ func (e *CNIEnv) generateCNIPlugins(driver string, name string, ipam map[string] return plugins, nil } -func (e *CNIEnv) generateIPAM(driver string, subnets []string, gateways []string, ipRanges []string, opts map[string]string, ipv6, ipv4, internal bool) (map[string]interface{}, error) { +func (e *CNIEnv) generateIPAM(driver string, subnets []string, gateways []string, ipRanges []string, auxAddresses []string, opts map[string]string, ipv6, ipv4, internal bool) (map[string]interface{}, error) { var ipamConfig interface{} switch driver { case "default", "host-local": + // Reserved auxiliary addresses are only meaningful for host-local, where + // they are enforced by carving the reserved IPs out of the range below. + aux, err := ParseAuxAddresses(auxAddresses) + if err != nil { + return nil, err + } ipamConf := newHostLocalIPAMConfig() if !internal { // An IPv6-only network has no IPv4 gateway, so its default route @@ -232,7 +238,7 @@ func (e *CNIEnv) generateIPAM(driver string, subnets []string, gateways []string {Dst: defaultRoute}, } } - ranges, findIPv4, err := e.parseIPAMRanges(subnets, gateways, ipRanges, ipv6) + ranges, findIPv4, err := e.parseIPAMRanges(subnets, gateways, ipRanges, aux, ipv6) if err != nil { return nil, err } @@ -243,8 +249,9 @@ func (e *CNIEnv) generateIPAM(driver string, subnets []string, gateways []string if ipv4 && !findIPv4 { // The default IPv4 range uses a computed gateway and no ip-range; // any user-supplied gateway or ip-range belongs to an explicit subnet. + // It also has no user subnet, so no aux-address can match it. // Skipped when IPv4 is disabled, leaving the network IPv6-only. - ranges, _, _ = e.parseIPAMRanges([]string{""}, nil, nil, ipv6) + ranges, _, _ = e.parseIPAMRanges([]string{""}, nil, nil, nil, ipv6) ipamConf.Ranges = append(ipamConf.Ranges, ranges...) } ipamConfig = ipamConf @@ -301,7 +308,7 @@ func (e *CNIEnv) generateIPAM(driver string, subnets []string, gateways []string return ipam, nil } -func (e *CNIEnv) parseIPAMRanges(subnets []string, gateways []string, ipRanges []string, ipv6 bool) ([][]IPAMRange, bool, error) { +func (e *CNIEnv) parseIPAMRanges(subnets []string, gateways []string, ipRanges []string, aux map[string]string, ipv6 bool) ([][]IPAMRange, bool, error) { // Resolve every requested subnet first; parseSubnet also rejects overlaps // with existing networks. The pairing below then works purely on the parsed // CIDRs, so it can be unit-tested without probing the host's networks. @@ -313,13 +320,13 @@ func (e *CNIEnv) parseIPAMRanges(subnets []string, gateways []string, ipRanges [ } parsedSubnets[i] = subnet } - return pairIPAMRanges(parsedSubnets, gateways, ipRanges, ipv6) + return pairIPAMRanges(parsedSubnets, gateways, ipRanges, aux, ipv6) } -// pairIPAMRanges matches each gateway and ip-range to the subnet that contains -// it and builds the per-subnet IPAM ranges. It is split out from subnet -// resolution so the matching can be tested without touching live networks. -func pairIPAMRanges(subnets []*net.IPNet, gateways []string, ipRanges []string, ipv6 bool) ([][]IPAMRange, bool, error) { +// pairIPAMRanges matches each gateway, ip-range and aux-address to the subnet +// that contains it and builds the per-subnet IPAM ranges. It is split out from +// subnet resolution so the matching can be tested without touching live networks. +func pairIPAMRanges(subnets []*net.IPNet, gateways []string, ipRanges []string, aux map[string]string, ipv6 bool) ([][]IPAMRange, bool, error) { // Parse the gateways once up front; matching them to subnets below is then // just a containment check, with no parse error mixed into the loop. parsedGateways := make([]net.IP, len(gateways)) @@ -341,6 +348,17 @@ func pairIPAMRanges(subnets []*net.IPNet, gateways []string, ipRanges []string, parsedRanges[i] = ipNet } + // Parse the aux-addresses once too, so the per-subnet loop only tests + // containment. aux is already validated by ParseAuxAddresses, so every value + // parses. matchedAux records which ones landed in a subnet, both to flag an + // unmatched aux as an error and to attach each aux to only the first subnet + // that contains it. + parsedAux := make(map[string]net.IP, len(aux)) + for name, ipStr := range aux { + parsedAux[name] = net.ParseIP(ipStr) + } + matchedAux := make(map[string]bool, len(parsedAux)) + findIPv4 := false ranges := make([][]IPAMRange, 0, len(subnets)) usedGateways := make([]bool, len(gateways)) @@ -375,10 +393,39 @@ func pairIPAMRanges(subnets []*net.IPNet, gateways []string, ipRanges []string, if err != nil { return nil, findIPv4, err } - ranges = append(ranges, []IPAMRange{*ipamRange}) + // Collect the aux-addresses that fall inside this subnet, rejecting the + // ones Docker also rejects (the network or gateway address), then reserve + // them by splitting the range. + gatewayIP := net.ParseIP(ipamRange.Gateway) + subnetAux := map[string]string{} + var reserved []net.IP + for name, ip := range parsedAux { + // Like gateway/ip-range, an aux-address attaches only to the first + // subnet that contains it. + if matchedAux[name] { + continue + } + if !subnet.Contains(ip) { + continue + } + matchedAux[name] = true + if ip.Equal(subnet.IP) || (gatewayIP != nil && ip.Equal(gatewayIP)) { + return nil, findIPv4, fmt.Errorf("failed to allocate secondary ip address (%s:%s): Address already in use", name, ip) + } + subnetAux[name] = ip.String() + reserved = append(reserved, ip) + } + rangeSet, err := splitIPAMRange(subnet, ipamRange, reserved) + if err != nil { + return nil, findIPv4, err + } + if len(subnetAux) > 0 { + rangeSet[0].AuxiliaryAddresses = subnetAux + } + ranges = append(ranges, rangeSet) } - // Only known after every subnet is seen: a gateway or ip-range that matched - // none is a user error, same as Docker. + // Only known after every subnet is seen: a gateway, ip-range or aux-address + // that matched no subnet is a user error, same as Docker. for j, ok := range usedGateways { if !ok { return nil, findIPv4, fmt.Errorf("no matching subnet for gateway %q", gateways[j]) @@ -389,6 +436,11 @@ func pairIPAMRanges(subnets []*net.IPNet, gateways []string, ipRanges []string, return nil, findIPv4, fmt.Errorf("no matching subnet for ip-range %q", ipRanges[j]) } } + for name, ip := range parsedAux { + if !matchedAux[name] { + return nil, findIPv4, fmt.Errorf("no matching subnet for aux-address %s", ip) + } + } return ranges, findIPv4, nil } diff --git a/pkg/netutil/netutil_unix_test.go b/pkg/netutil/netutil_unix_test.go index 142a3e36b71..c85332fd5cf 100644 --- a/pkg/netutil/netutil_unix_test.go +++ b/pkg/netutil/netutil_unix_test.go @@ -92,7 +92,7 @@ func TestPairIPAMRangesIPRange(t *testing.T) { subnets := parse(t, "10.6.0.0/16", "2001:db8:6::/64") // Given v6-first to prove the pairing is by containment, not by index. ipRanges := []string{"2001:db8:6::/80", "10.6.1.0/24"} - ranges, findIPv4, err := pairIPAMRanges(subnets, nil, ipRanges, true) + ranges, findIPv4, err := pairIPAMRanges(subnets, nil, ipRanges, nil, true) assert.NilError(t, err) assert.Equal(t, true, findIPv4) got := map[string]string{} @@ -104,22 +104,102 @@ func TestPairIPAMRangesIPRange(t *testing.T) { }) t.Run("an ip-range matching no subnet errors", func(t *testing.T) { - _, _, err := pairIPAMRanges(parse(t, "10.6.0.0/16"), nil, []string{"192.168.1.0/24"}, false) + _, _, err := pairIPAMRanges(parse(t, "10.6.0.0/16"), nil, []string{"192.168.1.0/24"}, nil, false) assert.ErrorContains(t, err, `no matching subnet for ip-range "192.168.1.0/24"`) }) t.Run("an IPv4 ip-range with only an IPv6 subnet errors", func(t *testing.T) { - _, _, err := pairIPAMRanges(parse(t, "2001:db8:6::/64"), nil, []string{"10.6.1.0/24"}, true) + _, _, err := pairIPAMRanges(parse(t, "2001:db8:6::/64"), nil, []string{"10.6.1.0/24"}, nil, true) assert.ErrorContains(t, err, `no matching subnet for ip-range "10.6.1.0/24"`) }) t.Run("a second ip-range claiming the same subnet errors", func(t *testing.T) { - _, _, err := pairIPAMRanges(parse(t, "10.6.0.0/16"), nil, []string{"10.6.1.0/24", "10.6.2.0/24"}, false) + _, _, err := pairIPAMRanges(parse(t, "10.6.0.0/16"), nil, []string{"10.6.1.0/24", "10.6.2.0/24"}, nil, false) assert.ErrorContains(t, err, `no matching subnet for ip-range "10.6.2.0/24"`) }) t.Run("a malformed ip-range errors", func(t *testing.T) { - _, _, err := pairIPAMRanges(parse(t, "10.6.0.0/16"), nil, []string{"bogus"}, false) + _, _, err := pairIPAMRanges(parse(t, "10.6.0.0/16"), nil, []string{"bogus"}, nil, false) assert.ErrorContains(t, err, `failed to parse ip-range "bogus"`) }) } + +// TestPairIPAMRangesAuxAddress exercises the aux-address side of pairIPAMRanges: +// each reserved address is matched to the subnet that contains it, recorded for +// inspect, and carved out of the range, while the network/gateway address and an +// address matching no subnet are rejected. +func TestPairIPAMRangesAuxAddress(t *testing.T) { + t.Parallel() + parse := func(t *testing.T, cidrs ...string) []*net.IPNet { + t.Helper() + subnets := make([]*net.IPNet, len(cidrs)) + for i, c := range cidrs { + _, n, err := net.ParseCIDR(c) + assert.NilError(t, err) + subnets[i] = n + } + return subnets + } + + t.Run("a reserved address is recorded and carved out of the range", func(t *testing.T) { + ranges, _, err := pairIPAMRanges(parse(t, "10.7.0.0/24"), nil, nil, map[string]string{"host": "10.7.0.5"}, false) + assert.NilError(t, err) + assert.Equal(t, 1, len(ranges)) + // The reservation is recorded on the first sub-range for inspect, and the + // range is split so .5 falls in the gap between the two sub-ranges. + assert.DeepEqual(t, map[string]string{"host": "10.7.0.5"}, ranges[0][0].AuxiliaryAddresses) + assert.Equal(t, 2, len(ranges[0])) + assert.Equal(t, "10.7.0.4", ranges[0][0].RangeEnd) + assert.Equal(t, "10.7.0.6", ranges[0][1].RangeStart) + }) + + t.Run("a reserved network address is rejected", func(t *testing.T) { + _, _, err := pairIPAMRanges(parse(t, "10.7.0.0/24"), nil, nil, map[string]string{"net": "10.7.0.0"}, false) + assert.ErrorContains(t, err, "Address already in use") + }) + + t.Run("a reserved gateway address is rejected", func(t *testing.T) { + // With no explicit gateway the first address (.1) is the gateway, so an + // aux-address on it collides. + _, _, err := pairIPAMRanges(parse(t, "10.7.0.0/24"), nil, nil, map[string]string{"gw": "10.7.0.1"}, false) + assert.ErrorContains(t, err, "Address already in use") + }) + + t.Run("an aux-address matching no subnet errors", func(t *testing.T) { + _, _, err := pairIPAMRanges(parse(t, "10.7.0.0/24"), nil, nil, map[string]string{"x": "192.168.5.5"}, false) + assert.ErrorContains(t, err, "no matching subnet for aux-address 192.168.5.5") + }) + + t.Run("dual-stack keeps each aux-address on its own family's subnet", func(t *testing.T) { + ranges, _, err := pairIPAMRanges(parse(t, "10.7.0.0/24", "fd00:7::/64"), nil, nil, map[string]string{"v4": "10.7.0.9", "v6": "fd00:7::9"}, true) + assert.NilError(t, err) + assert.Equal(t, 2, len(ranges)) + got := map[string]map[string]string{} + for _, rs := range ranges { + got[rs[0].Subnet] = rs[0].AuxiliaryAddresses + } + assert.DeepEqual(t, map[string]string{"v4": "10.7.0.9"}, got["10.7.0.0/24"]) + assert.DeepEqual(t, map[string]string{"v6": "fd00:7::9"}, got["fd00:7::/64"]) + }) + + t.Run("multiple aux-addresses in one subnet split it into three ranges", func(t *testing.T) { + ranges, _, err := pairIPAMRanges(parse(t, "10.7.0.0/24"), nil, nil, map[string]string{"a": "10.7.0.5", "b": "10.7.0.9"}, false) + assert.NilError(t, err) + assert.Equal(t, 1, len(ranges)) + // Both reservations are recorded on the first sub-range, and the subnet + // is carved into three ranges with .5 and .9 sitting in the two gaps. + assert.DeepEqual(t, map[string]string{"a": "10.7.0.5", "b": "10.7.0.9"}, ranges[0][0].AuxiliaryAddresses) + assert.Equal(t, 3, len(ranges[0])) + assert.Equal(t, "10.7.0.4", ranges[0][0].RangeEnd) + assert.Equal(t, "10.7.0.6", ranges[0][1].RangeStart) + assert.Equal(t, "10.7.0.8", ranges[0][1].RangeEnd) + assert.Equal(t, "10.7.0.10", ranges[0][2].RangeStart) + }) + + t.Run("an aux-address in a subnet filtered out by disabled IPv6 errors", func(t *testing.T) { + // The fd00:7::/64 subnet is skipped because ipv6 is false, so its + // aux-address matches nothing and is reported, not silently dropped. + _, _, err := pairIPAMRanges(parse(t, "10.7.0.0/24", "fd00:7::/64"), nil, nil, map[string]string{"v6": "fd00:7::9"}, false) + assert.ErrorContains(t, err, "no matching subnet for aux-address fd00:7::9") + }) +} diff --git a/pkg/netutil/netutil_windows.go b/pkg/netutil/netutil_windows.go index 6fef605b2f3..b669fd1fbfc 100644 --- a/pkg/netutil/netutil_windows.go +++ b/pkg/netutil/netutil_windows.go @@ -72,7 +72,7 @@ func (e *CNIEnv) generateCNIPlugins(driver string, name string, ipam map[string] return plugins, nil } -func (e *CNIEnv) generateIPAM(driver string, subnets []string, gateways []string, ipRanges []string, opts map[string]string, ipv6, ipv4, internal bool) (map[string]interface{}, error) { +func (e *CNIEnv) generateIPAM(driver string, subnets []string, gateways []string, ipRanges []string, auxAddresses []string, opts map[string]string, ipv6, ipv4, internal bool) (map[string]interface{}, error) { switch driver { case "default": default: @@ -82,6 +82,10 @@ func (e *CNIEnv) generateIPAM(driver string, subnets []string, gateways []string if !ipv4 { return nil, fmt.Errorf("--ipv4=false is not supported on Windows") } + // The Windows nat IPAM has no way to reserve individual addresses. + if len(auxAddresses) > 0 { + return nil, fmt.Errorf("--aux-address is not supported on Windows") + } // Windows is single-subnet, so use at most one gateway and one ip-range. gatewayStr := ""