diff --git a/cmd/nerdctl/network/network_create.go b/cmd/nerdctl/network/network_create.go index 596fa5f785f..5923219239c 100644 --- a/cmd/nerdctl/network/network_create.go +++ b/cmd/nerdctl/network/network_create.go @@ -49,6 +49,7 @@ func createCommand() *cobra.Command { cmd.Flags().StringArray("subnet", nil, `Subnet in CIDR format that represents a network segment, e.g. "10.5.0.0/16"`) cmd.Flags().StringArray("gateway", nil, "IPv4 or IPv6 Gateway for the master subnet") cmd.Flags().StringArray("ip-range", nil, `Allocate container ip from a sub-range`) + cmd.Flags().StringArray("aux-address", nil, "Auxiliary IPv4 or IPv6 addresses used by Network driver, as name=IP pairs. The IPs are reserved and never assigned to containers") cmd.Flags().StringArray("label", nil, "Set metadata for a network") cmd.Flags().Bool("ipv4", true, "Enable IPv4 networking (set to false together with --ipv6 for an IPv6-only network)") cmd.Flags().Bool("ipv6", false, "Enable IPv6 networking") @@ -93,6 +94,10 @@ func createAction(cmd *cobra.Command, args []string) error { if err != nil { return err } + auxAddresses, err := cmd.Flags().GetStringArray("aux-address") + if err != nil { + return err + } labels, err := cmd.Flags().GetStringArray("label") if err != nil { return err @@ -112,18 +117,19 @@ func createAction(cmd *cobra.Command, args []string) error { } return network.Create(types.NetworkCreateOptions{ - GOptions: globalOptions, - Name: name, - Driver: driver, - Options: strutil.ConvertKVStringsToMap(opts), - IPAMDriver: ipamDriver, - IPAMOptions: strutil.ConvertKVStringsToMap(ipamOpts), - Subnets: subnets, - Gateway: gateways, - IPRange: ipRanges, - Labels: labels, - IPv6: ipv6, - IPv4: &ipv4, - Internal: internal, + GOptions: globalOptions, + Name: name, + Driver: driver, + Options: strutil.ConvertKVStringsToMap(opts), + IPAMDriver: ipamDriver, + IPAMOptions: strutil.ConvertKVStringsToMap(ipamOpts), + Subnets: subnets, + Gateway: gateways, + IPRange: ipRanges, + AuxAddresses: auxAddresses, + Labels: labels, + IPv6: ipv6, + IPv4: &ipv4, + Internal: internal, }, cmd.OutOrStdout()) } diff --git a/cmd/nerdctl/network/network_create_linux_test.go b/cmd/nerdctl/network/network_create_linux_test.go index d0d113e7e8d..589c6a88eb0 100644 --- a/cmd/nerdctl/network/network_create_linux_test.go +++ b/cmd/nerdctl/network/network_create_linux_test.go @@ -256,6 +256,76 @@ func TestNetworkCreate(t *testing.T) { } }, }, + { + Description: "with aux-address", + Setup: func(data test.Data, helpers test.Helpers) { + helpers.Ensure("network", "create", data.Identifier(), + "--subnet", "10.6.0.0/24", + "--gateway", "10.6.0.1", + "--aux-address", "router=10.6.0.5", + "--aux-address", "dns=10.6.0.6", + ) + }, + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("network", "rm", data.Identifier()) + }, + Command: func(data test.Data, helpers test.Helpers) test.TestableCommand { + return helpers.Command("network", "inspect", data.Identifier()) + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + ExitCode: expect.ExitCodeSuccess, + Output: func(stdout string, t tig.T) { + netw := nerdtest.InspectNetwork(helpers, data.Identifier()) + var aux map[string]string + for _, c := range netw.IPAM.Config { + if c.Subnet == "10.6.0.0/24" { + aux = c.AuxiliaryAddresses + } + } + assert.Equal(t, aux["router"], "10.6.0.5") + assert.Equal(t, aux["dns"], "10.6.0.6") + }, + } + }, + }, + { + Description: "aux-address is reserved", + Setup: func(data test.Data, helpers test.Helpers) { + helpers.Ensure("network", "create", data.Identifier(), + "--subnet", "10.6.1.0/24", + "--aux-address", "reserved=10.6.1.5", + ) + }, + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("network", "rm", data.Identifier()) + }, + Command: func(data test.Data, helpers test.Helpers) test.TestableCommand { + // The reserved address is carved out of the range, so requesting + // it explicitly must fail just as it does on Docker. + return helpers.Command("run", "--rm", "--net", data.Identifier(), "--ip", "10.6.1.5", testutil.CommonImage, "true") + }, + Expected: test.Expects(expect.ExitCodeGenericFail, nil, nil), + }, + { + Description: "an un-reserved address is allocatable", + Setup: func(data test.Data, helpers test.Helpers) { + helpers.Ensure("network", "create", data.Identifier(), + "--subnet", "10.6.2.0/24", + "--aux-address", "reserved=10.6.2.5", + ) + }, + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("network", "rm", data.Identifier()) + }, + Command: func(data test.Data, helpers test.Helpers) test.TestableCommand { + // Positive control: an address in the same subnet that is not + // reserved allocates fine, so the failure above is specific to the + // reserved IP rather than an unrelated --ip problem. + return helpers.Command("run", "--rm", "--net", data.Identifier(), "--ip", "10.6.2.7", testutil.CommonImage, "true") + }, + Expected: test.Expects(expect.ExitCodeSuccess, nil, nil), + }, } testCase.Run(t) diff --git a/docs/command-reference.md b/docs/command-reference.md index 93c481b4c11..28566a5155e 100644 --- a/docs/command-reference.md +++ b/docs/command-reference.md @@ -1279,12 +1279,13 @@ Flags: - :whale: `--subnet`: Subnet in CIDR format that represents a network segment, e.g. "10.5.0.0/16" - :whale: `--gateway`: IPv4 or IPv6 Gateway for the master subnet - :whale: `--ip-range`: Allocate container ip from a sub-range +- :whale: `--aux-address`: Auxiliary IPv4 or IPv6 addresses, as `name=IP` pairs. Each IP is reserved and never assigned to a container. Repeatable, and matched to the subnet that contains it. - :whale: `--label`: Set metadata on a network - :whale: `--ipv4`: Enable IPv4. Enabled by default; set to false with `--ipv6` and an IPv6 subnet for an IPv6-only network. `--ipv4=false` is not supported on Windows. - :whale: `--ipv6`: Enable IPv6. Should be used with a valid subnet. - :whale: `--internal`: Restrict external access to the network. -Unimplemented `docker network create` flags: `--attachable`, `--aux-address`, `--config-from`, `--config-only`, `--ingress`, `--scope` +Unimplemented `docker network create` flags: `--attachable`, `--config-from`, `--config-only`, `--ingress`, `--scope` ### :whale: nerdctl network ls diff --git a/pkg/api/types/network_types.go b/pkg/api/types/network_types.go index 90969d6612e..70b5e6e4aab 100644 --- a/pkg/api/types/network_types.go +++ b/pkg/api/types/network_types.go @@ -33,8 +33,11 @@ type NetworkCreateOptions struct { Subnets []string Gateway []string IPRange []string - Labels []string - IPv6 bool + // AuxAddresses holds "name=IP" auxiliary addresses (docker --aux-address). + // Each IP is reserved so IPAM never hands it out to a container. + AuxAddresses []string + Labels []string + IPv6 bool // IPv4 enables IPv4 on the network. A nil value defaults to enabled, so a // directly-constructed NetworkCreateOptions keeps IPv4 on; setting it to // false together with IPv6 yields an IPv6-only network. diff --git a/pkg/cmd/network/create.go b/pkg/cmd/network/create.go index f64ba519f3a..b93b92d7a1f 100644 --- a/pkg/cmd/network/create.go +++ b/pkg/cmd/network/create.go @@ -41,6 +41,16 @@ func Create(options types.NetworkCreateOptions, stdout io.Writer) error { return fmt.Errorf("IPv6-only network requires an IPv6 subnet, specify --subnet manually") } if len(options.Subnets) == 0 { + // Docker matches each aux-address to a subnet that contains it, so + // without any subnet there is nothing to match. Surface the same + // "no matching subnet for aux-address " error Docker returns. + aux, err := netutil.ParseAuxAddresses(options.AuxAddresses) + if err != nil { + return err + } + for _, ip := range aux { + return fmt.Errorf("no matching subnet for aux-address %s", ip) + } if len(options.Gateway) > 0 || len(options.IPRange) > 0 { return fmt.Errorf("cannot set gateway or ip-range without subnet, specify --subnet manually") } diff --git a/pkg/cmd/network/create_test.go b/pkg/cmd/network/create_test.go new file mode 100644 index 00000000000..74db8b1c292 --- /dev/null +++ b/pkg/cmd/network/create_test.go @@ -0,0 +1,35 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package network + +import ( + "io" + "testing" + + "gotest.tools/v3/assert" + + "github.com/containerd/nerdctl/v2/pkg/api/types" +) + +// TestCreateAuxAddressWithoutSubnet verifies that an aux-address given without +// any subnet is rejected the same way Docker rejects it, before any CNI setup. +func TestCreateAuxAddressWithoutSubnet(t *testing.T) { + err := Create(types.NetworkCreateOptions{ + AuxAddresses: []string{"host=10.9.0.5"}, + }, io.Discard) + assert.ErrorContains(t, err, "no matching subnet for aux-address 10.9.0.5") +} diff --git a/pkg/inspecttypes/dockercompat/dockercompat.go b/pkg/inspecttypes/dockercompat/dockercompat.go index 1e7211307fd..6d7a86133c4 100644 --- a/pkg/inspecttypes/dockercompat/dockercompat.go +++ b/pkg/inspecttypes/dockercompat/dockercompat.go @@ -1052,9 +1052,10 @@ func getUlimitsFromNative(sp *specs.Spec) ([]*units.Ulimit, error) { } type IPAMConfig struct { - Subnet string `json:"Subnet,omitempty"` - Gateway string `json:"Gateway,omitempty"` - IPRange string `json:"IPRange,omitempty"` + Subnet string `json:"Subnet,omitempty"` + Gateway string `json:"Gateway,omitempty"` + IPRange string `json:"IPRange,omitempty"` + AuxiliaryAddresses map[string]string `json:"AuxiliaryAddresses,omitempty"` } type IPAM struct { @@ -1196,7 +1197,20 @@ func NetworkFromNative(n *native.Network) (*Network, error) { res.Name = sCNI.Name for _, plugin := range sCNI.Plugins { for _, ranges := range plugin.Ipam.Ranges { - res.IPAM.Config = append(res.IPAM.Config, ranges...) + // A range-set normally describes one subnet; an aux-address + // reservation splits it into several sub-ranges whose first entry + // carries the subnet, gateway, ip-range and aux-addresses. Report the + // first entry per distinct subnet so a split subnet collapses to one + // IPAM.Config like Docker, without dropping entries for different + // subnets in the same set. + seen := make(map[string]struct{}, len(ranges)) + for _, r := range ranges { + if _, ok := seen[r.Subnet]; ok { + continue + } + seen[r.Subnet] = struct{}{} + res.IPAM.Config = append(res.IPAM.Config, r) + } } } diff --git a/pkg/inspecttypes/dockercompat/dockercompat_test.go b/pkg/inspecttypes/dockercompat/dockercompat_test.go index 4c9986ca3b5..b269898c781 100644 --- a/pkg/inspecttypes/dockercompat/dockercompat_test.go +++ b/pkg/inspecttypes/dockercompat/dockercompat_test.go @@ -605,6 +605,25 @@ func TestGetUlimitsFromNative(t *testing.T) { } } +func TestNetworkFromNative(t *testing.T) { + // The first range-set is one subnet split into sub-ranges by an aux-address + // reservation and must collapse to a single IPAM.Config carrying the aux; the + // second set holds two distinct subnets that must both be kept; the empty set + // contributes nothing. + cni := `{"name":"testnet","plugins":[{"ipam":{"ranges":[` + + `[{"Subnet":"10.6.0.0/24","Gateway":"10.6.0.1","AuxiliaryAddresses":{"router":"10.6.0.5"}},{"Subnet":"10.6.0.0/24"}],` + + `[{"Subnet":"10.7.0.0/24"},{"Subnet":"10.8.0.0/24"}],` + + `[]` + + `]}}]}` + got, err := NetworkFromNative(&native.Network{CNI: []byte(cni)}) + assert.NilError(t, err) + assert.DeepEqual(t, []IPAMConfig{ + {Subnet: "10.6.0.0/24", Gateway: "10.6.0.1", AuxiliaryAddresses: map[string]string{"router": "10.6.0.5"}}, + {Subnet: "10.7.0.0/24"}, + {Subnet: "10.8.0.0/24"}, + }, got.IPAM.Config) +} + func TestNetworkSettingsFromNative(t *testing.T) { tempStateDir, err := os.MkdirTemp(t.TempDir(), "rw") if err != nil { diff --git a/pkg/netutil/cni_plugin.go b/pkg/netutil/cni_plugin.go index b44e76042e2..26dc76cb738 100644 --- a/pkg/netutil/cni_plugin.go +++ b/pkg/netutil/cni_plugin.go @@ -34,6 +34,11 @@ type IPAMRange struct { RangeEnd string `json:"rangeEnd,omitempty"` Gateway string `json:"gateway,omitempty"` IPRange string `json:"ipRange,omitempty"` + // AuxiliaryAddresses records the reserved name=IP pairs for this subnet. + // host-local does not read it (reservation is done by splitting the range + // around each IP); it is nerdctl bookkeeping so `network inspect` can + // report AuxiliaryAddresses the way Docker does. + AuxiliaryAddresses map[string]string `json:"auxiliaryAddresses,omitempty"` } type IPAMRoute struct { diff --git a/pkg/netutil/netutil.go b/pkg/netutil/netutil.go index 026b210df65..5a220cd2bb4 100644 --- a/pkg/netutil/netutil.go +++ b/pkg/netutil/netutil.go @@ -17,6 +17,7 @@ package netutil import ( + "bytes" "context" "crypto/sha256" "encoding/hex" @@ -28,6 +29,7 @@ import ( "path/filepath" "sort" "strconv" + "strings" "github.com/containernetworking/cni/libcni" @@ -341,7 +343,7 @@ func (e *CNIEnv) CreateNetwork(opts types.NetworkCreateOptions) (*NetworkConfig, } // A nil IPv4 defaults to enabled. ipv4 := opts.IPv4 == nil || *opts.IPv4 - ipam, err := e.generateIPAM(opts.IPAMDriver, opts.Subnets, opts.Gateway, opts.IPRange, opts.IPAMOptions, opts.IPv6, ipv4, opts.Internal) + ipam, err := e.generateIPAM(opts.IPAMDriver, opts.Subnets, opts.Gateway, opts.IPRange, opts.AuxAddresses, opts.IPAMOptions, opts.IPv6, ipv4, opts.Internal) if err != nil { return nil, err } @@ -621,6 +623,124 @@ func parseIPAMRange(subnet *net.IPNet, gatewayStr, ipRangeStr string) (*IPAMRang return res, nil } +// ParseAuxAddresses parses Docker-style "name=IP" auxiliary-address pairs into a +// name-to-IP map. An entry with an empty IP (including one with no "=") is +// dropped; a later entry overrides an earlier one with the same name, matching +// Docker; and a non-empty but unparsable IP is an error. +func ParseAuxAddresses(raw []string) (map[string]string, error) { + if len(raw) == 0 { + return nil, nil + } + aux := make(map[string]string, len(raw)) + for _, kv := range raw { + name, ip, _ := strings.Cut(kv, "=") + if ip == "" { + continue + } + if net.ParseIP(ip) == nil { + return nil, fmt.Errorf("invalid aux-address %q", ip) + } + aux[name] = ip + } + if len(aux) == 0 { + return nil, nil + } + return aux, nil +} + +// splitIPAMRange reserves the given IPs inside a subnet's allocation range by +// carving them out. host-local has no exclude list, but it does allocate across +// every range in a set, so the reserved IPs become gaps between sub-ranges and +// are never handed out. Reserved IPs outside the allocation window need no split +// (host-local cannot reach them anyway). The base range's gateway and ip-range +// are kept on the first sub-range so the rest of the pipeline and `network +// inspect` behave exactly as the un-split case. +func splitIPAMRange(subnet *net.IPNet, base *IPAMRange, reserved []net.IP) ([]IPAMRange, error) { + if len(reserved) == 0 { + return []IPAMRange{*base}, nil + } + + // Resolve the allocation window. With an ip-range the base already carries + // its bounds; otherwise it is the whole subnet minus the all-ones address. + start := net.ParseIP(base.RangeStart) + if start == nil { + start, _ = subnetutil.FirstIPInSubnet(subnet) + } + end := net.ParseIP(base.RangeEnd) + if end == nil { + last, _ := subnetutil.LastIPInSubnet(subnet) + end = ipDec(last) + } + + // Keep only the reserved IPs that fall within the window, sorted ascending. + inWindow := make([]net.IP, 0, len(reserved)) + for _, ip := range reserved { + if bytes.Compare(ip.To16(), start.To16()) >= 0 && bytes.Compare(ip.To16(), end.To16()) <= 0 { + inWindow = append(inWindow, ip) + } + } + if len(inWindow) == 0 { + return []IPAMRange{*base}, nil + } + sort.Slice(inWindow, func(i, j int) bool { + return bytes.Compare(inWindow[i].To16(), inWindow[j].To16()) < 0 + }) + + // Walk the window left to right, emitting a sub-range for each gap between + // reserved IPs. + var out []IPAMRange + cur := start + for _, r := range inWindow { + hi := ipDec(r) + if bytes.Compare(cur.To16(), hi.To16()) <= 0 { + out = append(out, IPAMRange{Subnet: subnet.String(), RangeStart: cur.String(), RangeEnd: hi.String()}) + } + cur = ipInc(r) + } + if bytes.Compare(cur.To16(), end.To16()) <= 0 { + out = append(out, IPAMRange{Subnet: subnet.String(), RangeStart: cur.String(), RangeEnd: end.String()}) + } + if len(out) == 0 { + return nil, fmt.Errorf("aux-address reservations leave no allocatable IPs in subnet %s", subnet) + } + + // host-local reserves the gateway only when it is set on the range it lands + // in, and after splitting the gateway can be in any sub-range, so set it on + // all of them. The original ip-range is nerdctl-only bookkeeping for inspect, + // so keep it on the first sub-range alone. + for i := range out { + out[i].Gateway = base.Gateway + } + out[0].IPRange = base.IPRange + return out, nil +} + +// ipInc returns ip+1 and ipDec returns ip-1, both in 16-byte form and on a copy +// so the caller's IP is left untouched. +func ipInc(ip net.IP) net.IP { + out := make(net.IP, net.IPv6len) + copy(out, ip.To16()) + for i := len(out) - 1; i >= 0; i-- { + out[i]++ + if out[i] != 0 { + break + } + } + return out +} + +func ipDec(ip net.IP) net.IP { + out := make(net.IP, net.IPv6len) + copy(out, ip.To16()) + for i := len(out) - 1; i >= 0; i-- { + out[i]-- + if out[i] != 0xff { + break + } + } + return out +} + // convert the struct to a map func structToMap(in interface{}) (map[string]interface{}, error) { out := make(map[string]interface{}) diff --git a/pkg/netutil/netutil_test.go b/pkg/netutil/netutil_test.go index f818c59a1b2..72cd294461c 100644 --- a/pkg/netutil/netutil_test.go +++ b/pkg/netutil/netutil_test.go @@ -128,11 +128,174 @@ func TestParseIPAMRange(t *testing.T) { assert.ErrorContains(t, err, tc.err) } else { assert.NilError(t, err) - assert.Equal(t, *tc.expected, *got) + assert.DeepEqual(t, *tc.expected, *got) } } } +func TestParseAuxAddresses(t *testing.T) { + t.Parallel() + type testCase struct { + raw []string + expected map[string]string + err string + } + testCases := []testCase{ + { + raw: nil, + expected: nil, + }, + { + raw: []string{"router=10.1.100.5", "dns=10.1.100.6"}, + expected: map[string]string{"router": "10.1.100.5", "dns": "10.1.100.6"}, + }, + { + // An empty name is allowed, matching Docker. + raw: []string{"=10.1.100.5"}, + expected: map[string]string{"": "10.1.100.5"}, + }, + { + // An entry with no "=" has an empty IP and is dropped, matching Docker. + raw: []string{"10.1.100.5"}, + expected: nil, + }, + { + // A later value overrides an earlier one with the same name. + raw: []string{"a=10.1.100.5", "a=10.1.100.6"}, + expected: map[string]string{"a": "10.1.100.6"}, + }, + { + raw: []string{"v6=2001:db8::5"}, + expected: map[string]string{"v6": "2001:db8::5"}, + }, + { + raw: []string{"bad=not-an-ip"}, + err: "invalid aux-address", + }, + } + for _, tc := range testCases { + got, err := ParseAuxAddresses(tc.raw) + if tc.err != "" { + assert.ErrorContains(t, err, tc.err) + continue + } + assert.NilError(t, err) + assert.DeepEqual(t, tc.expected, got) + } +} + +func TestSplitIPAMRange(t *testing.T) { + t.Parallel() + ips := func(addrs ...string) []net.IP { + out := make([]net.IP, len(addrs)) + for i, a := range addrs { + out[i] = net.ParseIP(a) + } + return out + } + type testCase struct { + name string + subnet string + base *IPAMRange + reserved []net.IP + expected []IPAMRange + err string + } + testCases := []testCase{ + { + name: "no reservation leaves the range untouched", + subnet: "10.1.100.0/24", + base: &IPAMRange{Subnet: "10.1.100.0/24", Gateway: "10.1.100.1"}, + reserved: nil, + expected: []IPAMRange{{Subnet: "10.1.100.0/24", Gateway: "10.1.100.1"}}, + }, + { + name: "a mid-subnet reservation splits the range in two", + subnet: "10.1.100.0/24", + base: &IPAMRange{Subnet: "10.1.100.0/24", Gateway: "10.1.100.1"}, + reserved: ips("10.1.100.5"), + expected: []IPAMRange{ + {Subnet: "10.1.100.0/24", RangeStart: "10.1.100.1", RangeEnd: "10.1.100.4", Gateway: "10.1.100.1"}, + {Subnet: "10.1.100.0/24", RangeStart: "10.1.100.6", RangeEnd: "10.1.100.254", Gateway: "10.1.100.1"}, + }, + }, + { + name: "two reservations produce three sub-ranges", + subnet: "10.1.100.0/24", + base: &IPAMRange{Subnet: "10.1.100.0/24", Gateway: "10.1.100.1"}, + reserved: ips("10.1.100.6", "10.1.100.5"), + expected: []IPAMRange{ + {Subnet: "10.1.100.0/24", RangeStart: "10.1.100.1", RangeEnd: "10.1.100.4", Gateway: "10.1.100.1"}, + {Subnet: "10.1.100.0/24", RangeStart: "10.1.100.7", RangeEnd: "10.1.100.254", Gateway: "10.1.100.1"}, + }, + }, + { + // The gateway is the first usable address, so reserving the next one + // leaves a gateway-only sub-range; host-local reserves the gateway, so + // allocation still starts after the reservation. + name: "a reservation right after the gateway leaves a gateway-only range", + subnet: "10.1.100.0/24", + base: &IPAMRange{Subnet: "10.1.100.0/24", Gateway: "10.1.100.1"}, + reserved: ips("10.1.100.2"), + expected: []IPAMRange{ + {Subnet: "10.1.100.0/24", RangeStart: "10.1.100.1", RangeEnd: "10.1.100.1", Gateway: "10.1.100.1"}, + {Subnet: "10.1.100.0/24", RangeStart: "10.1.100.3", RangeEnd: "10.1.100.254", Gateway: "10.1.100.1"}, + }, + }, + { + name: "a reservation inside an ip-range splits within its bounds", + subnet: "10.1.100.0/24", + base: &IPAMRange{Subnet: "10.1.100.0/24", Gateway: "10.1.100.1", IPRange: "10.1.100.0/28", RangeStart: "10.1.100.1", RangeEnd: "10.1.100.15"}, + reserved: ips("10.1.100.5"), + expected: []IPAMRange{ + {Subnet: "10.1.100.0/24", RangeStart: "10.1.100.1", RangeEnd: "10.1.100.4", Gateway: "10.1.100.1", IPRange: "10.1.100.0/28"}, + {Subnet: "10.1.100.0/24", RangeStart: "10.1.100.6", RangeEnd: "10.1.100.15", Gateway: "10.1.100.1"}, + }, + }, + { + name: "a reservation outside the ip-range needs no split", + subnet: "10.1.100.0/24", + base: &IPAMRange{Subnet: "10.1.100.0/24", Gateway: "10.1.100.1", IPRange: "10.1.100.0/28", RangeStart: "10.1.100.1", RangeEnd: "10.1.100.15"}, + reserved: ips("10.1.100.200"), + expected: []IPAMRange{ + {Subnet: "10.1.100.0/24", Gateway: "10.1.100.1", IPRange: "10.1.100.0/28", RangeStart: "10.1.100.1", RangeEnd: "10.1.100.15"}, + }, + }, + { + // Reserving every usable address in the window leaves nothing to hand + // out, which is an error rather than an empty range set. + name: "reservations leaving no allocatable address error", + subnet: "10.1.100.0/30", + base: &IPAMRange{Subnet: "10.1.100.0/30", Gateway: "10.1.100.1"}, + reserved: ips("10.1.100.1", "10.1.100.2"), + err: "leave no allocatable", + }, + { + name: "an IPv6 reservation splits the range around it", + subnet: "2001:db8::/64", + base: &IPAMRange{Subnet: "2001:db8::/64", Gateway: "2001:db8::1"}, + reserved: ips("2001:db8::5"), + expected: []IPAMRange{ + {Subnet: "2001:db8::/64", RangeStart: "2001:db8::1", RangeEnd: "2001:db8::4", Gateway: "2001:db8::1"}, + {Subnet: "2001:db8::/64", RangeStart: "2001:db8::6", RangeEnd: "2001:db8::ffff:ffff:ffff:fffe", Gateway: "2001:db8::1"}, + }, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + _, subnet, err := net.ParseCIDR(tc.subnet) + assert.NilError(t, err) + got, err := splitIPAMRange(subnet, tc.base, tc.reserved) + if tc.err != "" { + assert.ErrorContains(t, err, tc.err) + return + } + assert.NilError(t, err) + assert.DeepEqual(t, tc.expected, got) + }) + } +} + // Tests whether nerdctl properly creates the default network when required. // Note that this test will require a CNI driver bearing the same name as // the type of the default network. (denoted by netutil.DefaultNetworkName, diff --git a/pkg/netutil/netutil_unix.go b/pkg/netutil/netutil_unix.go index 7e2549da449..86127033798 100644 --- a/pkg/netutil/netutil_unix.go +++ b/pkg/netutil/netutil_unix.go @@ -215,10 +215,16 @@ func (e *CNIEnv) generateCNIPlugins(driver string, name string, ipam map[string] return plugins, nil } -func (e *CNIEnv) generateIPAM(driver string, subnets []string, gateways []string, ipRanges []string, opts map[string]string, ipv6, ipv4, internal bool) (map[string]interface{}, error) { +func (e *CNIEnv) generateIPAM(driver string, subnets []string, gateways []string, ipRanges []string, auxAddresses []string, opts map[string]string, ipv6, ipv4, internal bool) (map[string]interface{}, error) { var ipamConfig interface{} switch driver { case "default", "host-local": + // Reserved auxiliary addresses are only meaningful for host-local, where + // they are enforced by carving the reserved IPs out of the range below. + aux, err := ParseAuxAddresses(auxAddresses) + if err != nil { + return nil, err + } ipamConf := newHostLocalIPAMConfig() if !internal { // An IPv6-only network has no IPv4 gateway, so its default route @@ -232,7 +238,7 @@ func (e *CNIEnv) generateIPAM(driver string, subnets []string, gateways []string {Dst: defaultRoute}, } } - ranges, findIPv4, err := e.parseIPAMRanges(subnets, gateways, ipRanges, ipv6) + ranges, findIPv4, err := e.parseIPAMRanges(subnets, gateways, ipRanges, aux, ipv6) if err != nil { return nil, err } @@ -243,8 +249,9 @@ func (e *CNIEnv) generateIPAM(driver string, subnets []string, gateways []string if ipv4 && !findIPv4 { // The default IPv4 range uses a computed gateway and no ip-range; // any user-supplied gateway or ip-range belongs to an explicit subnet. + // It also has no user subnet, so no aux-address can match it. // Skipped when IPv4 is disabled, leaving the network IPv6-only. - ranges, _, _ = e.parseIPAMRanges([]string{""}, nil, nil, ipv6) + ranges, _, _ = e.parseIPAMRanges([]string{""}, nil, nil, nil, ipv6) ipamConf.Ranges = append(ipamConf.Ranges, ranges...) } ipamConfig = ipamConf @@ -301,7 +308,7 @@ func (e *CNIEnv) generateIPAM(driver string, subnets []string, gateways []string return ipam, nil } -func (e *CNIEnv) parseIPAMRanges(subnets []string, gateways []string, ipRanges []string, ipv6 bool) ([][]IPAMRange, bool, error) { +func (e *CNIEnv) parseIPAMRanges(subnets []string, gateways []string, ipRanges []string, aux map[string]string, ipv6 bool) ([][]IPAMRange, bool, error) { // Resolve every requested subnet first; parseSubnet also rejects overlaps // with existing networks. The pairing below then works purely on the parsed // CIDRs, so it can be unit-tested without probing the host's networks. @@ -313,13 +320,13 @@ func (e *CNIEnv) parseIPAMRanges(subnets []string, gateways []string, ipRanges [ } parsedSubnets[i] = subnet } - return pairIPAMRanges(parsedSubnets, gateways, ipRanges, ipv6) + return pairIPAMRanges(parsedSubnets, gateways, ipRanges, aux, ipv6) } -// pairIPAMRanges matches each gateway and ip-range to the subnet that contains -// it and builds the per-subnet IPAM ranges. It is split out from subnet -// resolution so the matching can be tested without touching live networks. -func pairIPAMRanges(subnets []*net.IPNet, gateways []string, ipRanges []string, ipv6 bool) ([][]IPAMRange, bool, error) { +// pairIPAMRanges matches each gateway, ip-range and aux-address to the subnet +// that contains it and builds the per-subnet IPAM ranges. It is split out from +// subnet resolution so the matching can be tested without touching live networks. +func pairIPAMRanges(subnets []*net.IPNet, gateways []string, ipRanges []string, aux map[string]string, ipv6 bool) ([][]IPAMRange, bool, error) { // Parse the gateways once up front; matching them to subnets below is then // just a containment check, with no parse error mixed into the loop. parsedGateways := make([]net.IP, len(gateways)) @@ -341,6 +348,17 @@ func pairIPAMRanges(subnets []*net.IPNet, gateways []string, ipRanges []string, parsedRanges[i] = ipNet } + // Parse the aux-addresses once too, so the per-subnet loop only tests + // containment. aux is already validated by ParseAuxAddresses, so every value + // parses. matchedAux records which ones landed in a subnet, both to flag an + // unmatched aux as an error and to attach each aux to only the first subnet + // that contains it. + parsedAux := make(map[string]net.IP, len(aux)) + for name, ipStr := range aux { + parsedAux[name] = net.ParseIP(ipStr) + } + matchedAux := make(map[string]bool, len(parsedAux)) + findIPv4 := false ranges := make([][]IPAMRange, 0, len(subnets)) usedGateways := make([]bool, len(gateways)) @@ -375,10 +393,39 @@ func pairIPAMRanges(subnets []*net.IPNet, gateways []string, ipRanges []string, if err != nil { return nil, findIPv4, err } - ranges = append(ranges, []IPAMRange{*ipamRange}) + // Collect the aux-addresses that fall inside this subnet, rejecting the + // ones Docker also rejects (the network or gateway address), then reserve + // them by splitting the range. + gatewayIP := net.ParseIP(ipamRange.Gateway) + subnetAux := map[string]string{} + var reserved []net.IP + for name, ip := range parsedAux { + // Like gateway/ip-range, an aux-address attaches only to the first + // subnet that contains it. + if matchedAux[name] { + continue + } + if !subnet.Contains(ip) { + continue + } + matchedAux[name] = true + if ip.Equal(subnet.IP) || (gatewayIP != nil && ip.Equal(gatewayIP)) { + return nil, findIPv4, fmt.Errorf("failed to allocate secondary ip address (%s:%s): Address already in use", name, ip) + } + subnetAux[name] = ip.String() + reserved = append(reserved, ip) + } + rangeSet, err := splitIPAMRange(subnet, ipamRange, reserved) + if err != nil { + return nil, findIPv4, err + } + if len(subnetAux) > 0 { + rangeSet[0].AuxiliaryAddresses = subnetAux + } + ranges = append(ranges, rangeSet) } - // Only known after every subnet is seen: a gateway or ip-range that matched - // none is a user error, same as Docker. + // Only known after every subnet is seen: a gateway, ip-range or aux-address + // that matched no subnet is a user error, same as Docker. for j, ok := range usedGateways { if !ok { return nil, findIPv4, fmt.Errorf("no matching subnet for gateway %q", gateways[j]) @@ -389,6 +436,11 @@ func pairIPAMRanges(subnets []*net.IPNet, gateways []string, ipRanges []string, return nil, findIPv4, fmt.Errorf("no matching subnet for ip-range %q", ipRanges[j]) } } + for name, ip := range parsedAux { + if !matchedAux[name] { + return nil, findIPv4, fmt.Errorf("no matching subnet for aux-address %s", ip) + } + } return ranges, findIPv4, nil } diff --git a/pkg/netutil/netutil_unix_test.go b/pkg/netutil/netutil_unix_test.go index 142a3e36b71..c85332fd5cf 100644 --- a/pkg/netutil/netutil_unix_test.go +++ b/pkg/netutil/netutil_unix_test.go @@ -92,7 +92,7 @@ func TestPairIPAMRangesIPRange(t *testing.T) { subnets := parse(t, "10.6.0.0/16", "2001:db8:6::/64") // Given v6-first to prove the pairing is by containment, not by index. ipRanges := []string{"2001:db8:6::/80", "10.6.1.0/24"} - ranges, findIPv4, err := pairIPAMRanges(subnets, nil, ipRanges, true) + ranges, findIPv4, err := pairIPAMRanges(subnets, nil, ipRanges, nil, true) assert.NilError(t, err) assert.Equal(t, true, findIPv4) got := map[string]string{} @@ -104,22 +104,102 @@ func TestPairIPAMRangesIPRange(t *testing.T) { }) t.Run("an ip-range matching no subnet errors", func(t *testing.T) { - _, _, err := pairIPAMRanges(parse(t, "10.6.0.0/16"), nil, []string{"192.168.1.0/24"}, false) + _, _, err := pairIPAMRanges(parse(t, "10.6.0.0/16"), nil, []string{"192.168.1.0/24"}, nil, false) assert.ErrorContains(t, err, `no matching subnet for ip-range "192.168.1.0/24"`) }) t.Run("an IPv4 ip-range with only an IPv6 subnet errors", func(t *testing.T) { - _, _, err := pairIPAMRanges(parse(t, "2001:db8:6::/64"), nil, []string{"10.6.1.0/24"}, true) + _, _, err := pairIPAMRanges(parse(t, "2001:db8:6::/64"), nil, []string{"10.6.1.0/24"}, nil, true) assert.ErrorContains(t, err, `no matching subnet for ip-range "10.6.1.0/24"`) }) t.Run("a second ip-range claiming the same subnet errors", func(t *testing.T) { - _, _, err := pairIPAMRanges(parse(t, "10.6.0.0/16"), nil, []string{"10.6.1.0/24", "10.6.2.0/24"}, false) + _, _, err := pairIPAMRanges(parse(t, "10.6.0.0/16"), nil, []string{"10.6.1.0/24", "10.6.2.0/24"}, nil, false) assert.ErrorContains(t, err, `no matching subnet for ip-range "10.6.2.0/24"`) }) t.Run("a malformed ip-range errors", func(t *testing.T) { - _, _, err := pairIPAMRanges(parse(t, "10.6.0.0/16"), nil, []string{"bogus"}, false) + _, _, err := pairIPAMRanges(parse(t, "10.6.0.0/16"), nil, []string{"bogus"}, nil, false) assert.ErrorContains(t, err, `failed to parse ip-range "bogus"`) }) } + +// TestPairIPAMRangesAuxAddress exercises the aux-address side of pairIPAMRanges: +// each reserved address is matched to the subnet that contains it, recorded for +// inspect, and carved out of the range, while the network/gateway address and an +// address matching no subnet are rejected. +func TestPairIPAMRangesAuxAddress(t *testing.T) { + t.Parallel() + parse := func(t *testing.T, cidrs ...string) []*net.IPNet { + t.Helper() + subnets := make([]*net.IPNet, len(cidrs)) + for i, c := range cidrs { + _, n, err := net.ParseCIDR(c) + assert.NilError(t, err) + subnets[i] = n + } + return subnets + } + + t.Run("a reserved address is recorded and carved out of the range", func(t *testing.T) { + ranges, _, err := pairIPAMRanges(parse(t, "10.7.0.0/24"), nil, nil, map[string]string{"host": "10.7.0.5"}, false) + assert.NilError(t, err) + assert.Equal(t, 1, len(ranges)) + // The reservation is recorded on the first sub-range for inspect, and the + // range is split so .5 falls in the gap between the two sub-ranges. + assert.DeepEqual(t, map[string]string{"host": "10.7.0.5"}, ranges[0][0].AuxiliaryAddresses) + assert.Equal(t, 2, len(ranges[0])) + assert.Equal(t, "10.7.0.4", ranges[0][0].RangeEnd) + assert.Equal(t, "10.7.0.6", ranges[0][1].RangeStart) + }) + + t.Run("a reserved network address is rejected", func(t *testing.T) { + _, _, err := pairIPAMRanges(parse(t, "10.7.0.0/24"), nil, nil, map[string]string{"net": "10.7.0.0"}, false) + assert.ErrorContains(t, err, "Address already in use") + }) + + t.Run("a reserved gateway address is rejected", func(t *testing.T) { + // With no explicit gateway the first address (.1) is the gateway, so an + // aux-address on it collides. + _, _, err := pairIPAMRanges(parse(t, "10.7.0.0/24"), nil, nil, map[string]string{"gw": "10.7.0.1"}, false) + assert.ErrorContains(t, err, "Address already in use") + }) + + t.Run("an aux-address matching no subnet errors", func(t *testing.T) { + _, _, err := pairIPAMRanges(parse(t, "10.7.0.0/24"), nil, nil, map[string]string{"x": "192.168.5.5"}, false) + assert.ErrorContains(t, err, "no matching subnet for aux-address 192.168.5.5") + }) + + t.Run("dual-stack keeps each aux-address on its own family's subnet", func(t *testing.T) { + ranges, _, err := pairIPAMRanges(parse(t, "10.7.0.0/24", "fd00:7::/64"), nil, nil, map[string]string{"v4": "10.7.0.9", "v6": "fd00:7::9"}, true) + assert.NilError(t, err) + assert.Equal(t, 2, len(ranges)) + got := map[string]map[string]string{} + for _, rs := range ranges { + got[rs[0].Subnet] = rs[0].AuxiliaryAddresses + } + assert.DeepEqual(t, map[string]string{"v4": "10.7.0.9"}, got["10.7.0.0/24"]) + assert.DeepEqual(t, map[string]string{"v6": "fd00:7::9"}, got["fd00:7::/64"]) + }) + + t.Run("multiple aux-addresses in one subnet split it into three ranges", func(t *testing.T) { + ranges, _, err := pairIPAMRanges(parse(t, "10.7.0.0/24"), nil, nil, map[string]string{"a": "10.7.0.5", "b": "10.7.0.9"}, false) + assert.NilError(t, err) + assert.Equal(t, 1, len(ranges)) + // Both reservations are recorded on the first sub-range, and the subnet + // is carved into three ranges with .5 and .9 sitting in the two gaps. + assert.DeepEqual(t, map[string]string{"a": "10.7.0.5", "b": "10.7.0.9"}, ranges[0][0].AuxiliaryAddresses) + assert.Equal(t, 3, len(ranges[0])) + assert.Equal(t, "10.7.0.4", ranges[0][0].RangeEnd) + assert.Equal(t, "10.7.0.6", ranges[0][1].RangeStart) + assert.Equal(t, "10.7.0.8", ranges[0][1].RangeEnd) + assert.Equal(t, "10.7.0.10", ranges[0][2].RangeStart) + }) + + t.Run("an aux-address in a subnet filtered out by disabled IPv6 errors", func(t *testing.T) { + // The fd00:7::/64 subnet is skipped because ipv6 is false, so its + // aux-address matches nothing and is reported, not silently dropped. + _, _, err := pairIPAMRanges(parse(t, "10.7.0.0/24", "fd00:7::/64"), nil, nil, map[string]string{"v6": "fd00:7::9"}, false) + assert.ErrorContains(t, err, "no matching subnet for aux-address fd00:7::9") + }) +} diff --git a/pkg/netutil/netutil_windows.go b/pkg/netutil/netutil_windows.go index 6fef605b2f3..b669fd1fbfc 100644 --- a/pkg/netutil/netutil_windows.go +++ b/pkg/netutil/netutil_windows.go @@ -72,7 +72,7 @@ func (e *CNIEnv) generateCNIPlugins(driver string, name string, ipam map[string] return plugins, nil } -func (e *CNIEnv) generateIPAM(driver string, subnets []string, gateways []string, ipRanges []string, opts map[string]string, ipv6, ipv4, internal bool) (map[string]interface{}, error) { +func (e *CNIEnv) generateIPAM(driver string, subnets []string, gateways []string, ipRanges []string, auxAddresses []string, opts map[string]string, ipv6, ipv4, internal bool) (map[string]interface{}, error) { switch driver { case "default": default: @@ -82,6 +82,10 @@ func (e *CNIEnv) generateIPAM(driver string, subnets []string, gateways []string if !ipv4 { return nil, fmt.Errorf("--ipv4=false is not supported on Windows") } + // The Windows nat IPAM has no way to reserve individual addresses. + if len(auxAddresses) > 0 { + return nil, fmt.Errorf("--aux-address is not supported on Windows") + } // Windows is single-subnet, so use at most one gateway and one ip-range. gatewayStr := ""