Docker.Network.DHCP/pkg/plugin/network.go

package plugin

import (
	"bytes"
	"context"
	"fmt"
	"net"

	dTypes "github.com/docker/docker/api/types"
	"github.com/mitchellh/mapstructure"
	log "github.com/sirupsen/logrus"
	"github.com/vishvananda/netlink"
	"golang.org/x/sys/unix"

	"github.com/devplayer0/docker-net-dhcp/pkg/udhcpc"
	"github.com/devplayer0/docker-net-dhcp/pkg/util"
)

// CLIOptionsKey is the key used in create network options by the CLI for custom options
const CLIOptionsKey string = "com.docker.network.generic"

// Implementations of the endpoints described in
// https://github.com/moby/libnetwork/blob/master/docs/remote.md

// CreateNetwork "creates" a new DHCP network (validates the interface and IPAM settings)
func (p *Plugin) CreateNetwork(r CreateNetworkRequest) error {
	log.WithField("options", r.Options).Debug("CreateNetwork options")

	opts, err := decodeOpts(r.Options[util.OptionsKeyGeneric])
	if err != nil {
		return fmt.Errorf("failed to decode network options: %w", err)
	}

	switch opts.NetMode() {
	case NetworkModeBridge, NetworkModeMacvlan, NetworkModeIPvlan:
		// valid
	default:
		return util.ErrInvalidMode
	}

	if opts.Bridge == "" {
		return util.ErrBridgeRequired
	}

	for _, d := range r.IPv4Data {
		if d.AddressSpace != "null" || d.Pool != "0.0.0.0/0" {
			return util.ErrIPAM
		}
	}

	link, err := netlink.LinkByName(opts.Bridge)
	if err != nil {
		return fmt.Errorf("failed to lookup interface %v: %w", opts.Bridge, err)
	}

	if opts.NetMode() == NetworkModeBridge {
		if link.Type() != "bridge" {
			return util.ErrNotBridge
		}

		if !opts.IgnoreConflicts {
			v4Addrs, err := netlink.AddrList(link, unix.AF_INET)
			if err != nil {
				return fmt.Errorf("failed to retrieve IPv4 addresses for %v: %w", opts.Bridge, err)
			}
			v6Addrs, err := netlink.AddrList(link, unix.AF_INET6)
			if err != nil {
				return fmt.Errorf("failed to retrieve IPv6 addresses for %v: %w", opts.Bridge, err)
			}
			bridgeAddrs := append(v4Addrs, v6Addrs...)

			nets, err := p.docker.NetworkList(context.Background(), dTypes.NetworkListOptions{})
			if err != nil {
				return fmt.Errorf("failed to retrieve list of networks from Docker: %w", err)
			}

			// Make sure the addresses on this bridge aren't used by another network
			for _, n := range nets {
				if IsDHCPPlugin(n.Driver) {
					otherOpts, err := decodeOpts(n.Options)
					if err != nil {
						log.
							WithField("network", n.Name).
							WithError(err).
							Warn("Failed to parse other DHCP network's options")
					} else if otherOpts.Bridge == opts.Bridge {
						return util.ErrBridgeUsed
					}
				}
				if n.IPAM.Driver == "null" {
					// Null driver networks will have 0.0.0.0/0 which covers any address range!
					continue
				}

				for _, c := range n.IPAM.Config {
					_, dockerCIDR, err := net.ParseCIDR(c.Subnet)
					if err != nil {
						return fmt.Errorf("failed to parse subnet %v on Docker network %v: %w", c.Subnet, n.ID, err)
					}
					if bytes.Equal(dockerCIDR.Mask, net.CIDRMask(0, 32)) || bytes.Equal(dockerCIDR.Mask, net.CIDRMask(0, 128)) {
						// Last check to make sure the network isn't 0.0.0.0/0 or ::/0 (which would always pass the check below)
						continue
					}

					for _, bridgeAddr := range bridgeAddrs {
						if bridgeAddr.IPNet.Contains(dockerCIDR.IP) || dockerCIDR.Contains(bridgeAddr.IP) {
							return util.ErrBridgeUsed
						}
					}
				}
			}
		}
	}

	log.WithFields(log.Fields{
		"network": r.NetworkID,
		"bridge":  opts.Bridge,
		"mode":    opts.NetMode(),
		"ipv6":    opts.IPv6,
	}).Info("Network created")

	return nil
}

// DeleteNetwork "deletes" a DHCP network (does nothing, the bridge is managed by the user)
func (p *Plugin) DeleteNetwork(r DeleteNetworkRequest) error {
	log.WithField("network", r.NetworkID).Info("Network deleted")
	return nil
}

func vethPairNames(id string) (string, string) {
	return "dh-" + id[:12], id[:12] + "-dh"
}

func (p *Plugin) netOptions(ctx context.Context, id string) (DHCPNetworkOptions, error) {
	dummy := DHCPNetworkOptions{}

	n, err := p.docker.NetworkInspect(ctx, id, dTypes.NetworkInspectOptions{})
	if err != nil {
		return dummy, fmt.Errorf("failed to get info from Docker: %w", err)
	}

	opts, err := decodeOpts(n.Options)
	if err != nil {
		return dummy, fmt.Errorf("failed to parse options: %w", err)
	}

	return opts, nil
}

// CreateEndpoint creates the container-side network interface and uses udhcpc to acquire an initial IP address.
// In bridge mode a veth pair is used; in macvlan/ipvlan mode a directly-attached sub-interface is created on the
// parent NIC. Docker will move the interface into the container's namespace and apply the address.
func (p *Plugin) CreateEndpoint(ctx context.Context, r CreateEndpointRequest) (CreateEndpointResponse, error) {
	log.WithField("options", r.Options).Debug("CreateEndpoint options")
	res := CreateEndpointResponse{
		Interface: &EndpointInterface{},
	}

	if r.Interface != nil && (r.Interface.Address != "" || r.Interface.AddressIPv6 != "") {
		// TODO: Should we allow static IP's somehow?
		return res, util.ErrIPAM
	}

	opts, err := p.netOptions(ctx, r.NetworkID)
	if err != nil {
		return res, fmt.Errorf("failed to get network options: %w", err)
	}

	timeout := defaultLeaseTimeout
	if opts.LeaseTimeout != 0 {
		timeout = opts.LeaseTimeout
	}

	// initialIP runs udhcpc on ifName to obtain an IP and stores it in joinHints.
	initialIP := func(ifName string, v6 bool) error {
		v6str := ""
		if v6 {
			v6str = "v6"
		}

		timeoutCtx, cancel := context.WithTimeout(ctx, timeout)
		defer cancel()

		info, err := udhcpc.GetIP(timeoutCtx, ifName, &udhcpc.DHCPClientOptions{V6: v6})
		if err != nil {
			return fmt.Errorf("failed to get initial IP%v address via DHCP%v: %w", v6str, v6str, err)
		}
		ip, err := netlink.ParseAddr(info.IP)
		if err != nil {
			return fmt.Errorf("failed to parse initial IP%v address: %w", v6str, err)
		}

		hint := p.joinHints[r.EndpointID]
		if v6 {
			res.Interface.AddressIPv6 = info.IP
			hint.IPv6 = ip
			// No gateways in DHCPv6!
		} else {
			res.Interface.Address = info.IP
			hint.IPv4 = ip
			hint.Gateway = info.Gateway
		}
		p.joinHints[r.EndpointID] = hint

		return nil
	}

	if opts.NetMode() == NetworkModeBridge {
		bridge, err := netlink.LinkByName(opts.Bridge)
		if err != nil {
			return res, fmt.Errorf("failed to get bridge interface: %w", err)
		}

		hostName, ctrName := vethPairNames(r.EndpointID)
		la := netlink.NewLinkAttrs()
		la.Name = hostName
		hostLink := &netlink.Veth{
			LinkAttrs: la,
			PeerName:  ctrName,
		}
		if r.Interface.MacAddress != "" {
			addr, err := net.ParseMAC(r.Interface.MacAddress)
			if err != nil {
				return res, util.ErrMACAddress
			}

			hostLink.PeerHardwareAddr = addr
		}

		if err := netlink.LinkAdd(hostLink); err != nil {
			return res, fmt.Errorf("failed to create veth pair: %w", err)
		}
		if err := func() error {
			if err := netlink.LinkSetUp(hostLink); err != nil {
				return fmt.Errorf("failed to set host side link of veth pair up: %w", err)
			}

			ctrLink, err := netlink.LinkByName(ctrName)
			if err != nil {
				return fmt.Errorf("failed to find container side of veth pair: %w", err)
			}
			if err := netlink.LinkSetUp(ctrLink); err != nil {
				return fmt.Errorf("failed to set container side link of veth pair up: %w", err)
			}

			// Only write back the MAC address if it wasn't provided to us by libnetwork
			if r.Interface.MacAddress == "" {
				// The kernel will often reset a randomly assigned MAC address after actions like LinkSetMaster. We prevent
				// this behaviour by setting it manually to the random value
				if err := netlink.LinkSetHardwareAddr(ctrLink, ctrLink.Attrs().HardwareAddr); err != nil {
					return fmt.Errorf("failed to set container side of veth pair's MAC address: %w", err)
				}

				res.Interface.MacAddress = ctrLink.Attrs().HardwareAddr.String()
			}

			if err := netlink.LinkSetMaster(hostLink, bridge); err != nil {
				return fmt.Errorf("failed to attach host side link of veth peer to bridge: %w", err)
			}

			if err := initialIP(ctrName, false); err != nil {
				return err
			}
			if opts.IPv6 {
				if err := initialIP(ctrName, true); err != nil {
					return err
				}
			}

			return nil
		}(); err != nil {
			// Be sure to clean up the veth pair if any of this fails
			netlink.LinkDel(hostLink)
			return res, err
		}
	} else {
		// macvlan or ipvlan mode: create a sub-interface directly on the parent NIC.
		// The interface is temporarily created in the host namespace so that udhcpc can
		// obtain an initial lease; Docker will move it into the container namespace via Join.
		if opts.NetMode() == NetworkModeIPvlan && r.Interface.MacAddress != "" {
			// ipvlan interfaces share the parent's MAC; a custom MAC is not supported
			return res, util.ErrMACAddress
		}

		parentLink, err := netlink.LinkByName(opts.Bridge)
		if err != nil {
			return res, fmt.Errorf("failed to get parent interface %v: %w", opts.Bridge, err)
		}

		ifName := "dh-" + r.EndpointID[:12]
		la := netlink.NewLinkAttrs()
		la.Name = ifName
		la.ParentIndex = parentLink.Attrs().Index
		if r.Interface.MacAddress != "" {
			addr, err := net.ParseMAC(r.Interface.MacAddress)
			if err != nil {
				return res, util.ErrMACAddress
			}
			la.HardwareAddr = addr
		}

		var newLink netlink.Link
		if opts.NetMode() == NetworkModeMacvlan {
			newLink = &netlink.Macvlan{LinkAttrs: la, Mode: netlink.MACVLAN_MODE_BRIDGE}
		} else {
			newLink = &netlink.IPVlan{LinkAttrs: la, Mode: netlink.IPVLAN_MODE_L2}
		}

		if err := netlink.LinkAdd(newLink); err != nil {
			return res, fmt.Errorf("failed to create %v interface: %w", opts.NetMode(), err)
		}
		if err := func() error {
			// Re-fetch to get the kernel-assigned index and MAC
			ctrLink, err := netlink.LinkByName(ifName)
			if err != nil {
				return fmt.Errorf("failed to get %v interface after creation: %w", opts.NetMode(), err)
			}
			if err := netlink.LinkSetUp(ctrLink); err != nil {
				return fmt.Errorf("failed to set %v interface up: %w", opts.NetMode(), err)
			}

			if opts.NetMode() == NetworkModeMacvlan && r.Interface.MacAddress == "" {
				res.Interface.MacAddress = ctrLink.Attrs().HardwareAddr.String()
			}

			// Store the interface index so the persistent DHCP manager can locate
			// the interface after Docker moves it into the container namespace.
			hint := p.joinHints[r.EndpointID]
			hint.IfIndex = ctrLink.Attrs().Index
			p.joinHints[r.EndpointID] = hint

			if err := initialIP(ifName, false); err != nil {
				return err
			}
			if opts.IPv6 {
				if err := initialIP(ifName, true); err != nil {
					return err
				}
			}

			return nil
		}(); err != nil {
			netlink.LinkDel(newLink)
			return res, err
		}
	}

	log.WithFields(log.Fields{
		"network":     r.NetworkID[:12],
		"endpoint":    r.EndpointID[:12],
		"mode":        opts.NetMode(),
		"mac_address": res.Interface.MacAddress,
		"ip":          res.Interface.Address,
		"ipv6":        res.Interface.AddressIPv6,
		"gateway":     fmt.Sprintf("%#v", p.joinHints[r.EndpointID].Gateway),
	}).Info("Endpoint created")

	return res, nil
}

type operInfo struct {
	Bridge      string `mapstructure:"bridge"`
	HostVEth    string `mapstructure:"veth_host"`
	HostVEthMAC string `mapstructure:"veth_host_mac"`
}

// EndpointOperInfo retrieves some info about an existing endpoint
func (p *Plugin) EndpointOperInfo(ctx context.Context, r InfoRequest) (InfoResponse, error) {
	res := InfoResponse{}

	opts, err := p.netOptions(ctx, r.NetworkID)
	if err != nil {
		return res, fmt.Errorf("failed to get network options: %w", err)
	}

	info := operInfo{Bridge: opts.Bridge}

	if opts.NetMode() == NetworkModeBridge {
		hostName, _ := vethPairNames(r.EndpointID)
		hostLink, err := netlink.LinkByName(hostName)
		if err != nil {
			return res, fmt.Errorf("failed to find host side of veth pair: %w", err)
		}
		info.HostVEth = hostName
		info.HostVEthMAC = hostLink.Attrs().HardwareAddr.String()
	}

	if err := mapstructure.Decode(info, &res.Value); err != nil {
		return res, fmt.Errorf("failed to encode OperInfo: %w", err)
	}

	return res, nil
}

// DeleteEndpoint cleans up the endpoint interface. For bridge mode the host-side veth is deleted (which cascades to
// remove the container side). For macvlan/ipvlan mode the interface lives in the container namespace and is destroyed
// automatically when that namespace is torn down, so nothing needs to be done here.
func (p *Plugin) DeleteEndpoint(ctx context.Context, r DeleteEndpointRequest) error {
	opts, err := p.netOptions(ctx, r.NetworkID)
	if err != nil {
		return fmt.Errorf("failed to get network options: %w", err)
	}

	if opts.NetMode() != NetworkModeBridge {
		log.WithFields(log.Fields{
			"network":  r.NetworkID[:12],
			"endpoint": r.EndpointID[:12],
			"mode":     opts.NetMode(),
		}).Debug("Skipping host-side interface deletion; container namespace teardown handles cleanup")
		return nil
	}

	hostName, _ := vethPairNames(r.EndpointID)
	link, err := netlink.LinkByName(hostName)
	if err != nil {
		return fmt.Errorf("failed to lookup host veth interface %v: %w", hostName, err)
	}

	if err := netlink.LinkDel(link); err != nil {
		return fmt.Errorf("failed to delete veth pair: %w", err)
	}

	log.WithFields(log.Fields{
		"network":  r.NetworkID[:12],
		"endpoint": r.EndpointID[:12],
	}).Info("Endpoint deleted")

	return nil
}

func (p *Plugin) addRoutes(opts *DHCPNetworkOptions, v6 bool, bridge netlink.Link, r JoinRequest, hint joinHint, res *JoinResponse) error {
	family := unix.AF_INET
	if v6 {
		family = unix.AF_INET6
	}

	routes, err := netlink.RouteListFiltered(family, &netlink.Route{
		LinkIndex: bridge.Attrs().Index,
		Type:      unix.RTN_UNICAST,
	}, netlink.RT_FILTER_OIF|netlink.RT_FILTER_TYPE)
	if err != nil {
		return fmt.Errorf("failed to list routes: %w", err)
	}

	logFields := log.Fields{
		"network":  r.NetworkID[:12],
		"endpoint": r.EndpointID[:12],
		"sandbox":  r.SandboxKey,
	}
	for _, route := range routes {
		if route.Dst == nil {
			// Default route
			switch family {
			case unix.AF_INET:
				if res.Gateway == "" {
					res.Gateway = route.Gw.String()
					log.
						WithFields(logFields).
						WithField("gateway", res.Gateway).
						Info("[Join] Setting IPv4 gateway retrieved from bridge interface on host routing table")
				}
			case unix.AF_INET6:
				if res.GatewayIPv6 == "" {
					res.GatewayIPv6 = route.Gw.String()
					log.
						WithFields(logFields).
						WithField("gateway", res.GatewayIPv6).
						Info("[Join] Setting IPv6 gateway retrieved from bridge interface on host routing table")
				}
			}

			continue
		}

		if opts.SkipRoutes {
			// Don't do static routes at all
			continue
		}

		if route.Protocol == unix.RTPROT_KERNEL ||
			(family == unix.AF_INET && route.Dst.Contains(hint.IPv4.IP)) ||
			(family == unix.AF_INET6 && route.Dst.Contains(hint.IPv6.IP)) {
			// Make sure to leave out the default on-link route created automatically for the IP(s) acquired by DHCP
			continue
		}

		staticRoute := &StaticRoute{
			Destination: route.Dst.String(),
			// Default to an on-link route
			RouteType: 1,
		}
		res.StaticRoutes = append(res.StaticRoutes, staticRoute)

		if route.Gw != nil {
			staticRoute.RouteType = 0
			staticRoute.NextHop = route.Gw.String()

			log.
				WithFields(logFields).
				WithField("route", staticRoute.Destination).
				WithField("gateway", staticRoute.NextHop).
				Info("[Join] Adding route (via gateway) retrieved from bridge interface on host routing table")
		} else {
			log.
				WithFields(logFields).
				WithField("route", staticRoute.Destination).
				Info("[Join] Adding on-link route retrieved from bridge interface on host routing table")
		}
	}

	return nil
}

// Join passes the veth name and route information (gateway from DHCP and existing routes on the host bridge) to Docker
// and starts a persistent DHCP client to maintain the lease on the acquired IP
func (p *Plugin) Join(ctx context.Context, r JoinRequest) (JoinResponse, error) {
	log.WithField("options", r.Options).Debug("Join options")
	res := JoinResponse{}

	opts, err := p.netOptions(ctx, r.NetworkID)
	if err != nil {
		return res, fmt.Errorf("failed to get network options: %w", err)
	}

	var srcName, dstPrefix string
	if opts.NetMode() == NetworkModeBridge {
		_, srcName = vethPairNames(r.EndpointID)
		dstPrefix = opts.Bridge
	} else {
		srcName = "dh-" + r.EndpointID[:12]
		dstPrefix = "eth"
	}

	res.InterfaceName = InterfaceName{
		SrcName:   srcName,
		DstPrefix: dstPrefix,
	}

	hint, ok := p.joinHints[r.EndpointID]
	if !ok {
		return res, util.ErrNoHint
	}
	delete(p.joinHints, r.EndpointID)

	if hint.Gateway != "" {
		log.WithFields(log.Fields{
			"network":  r.NetworkID[:12],
			"endpoint": r.EndpointID[:12],
			"sandbox":  r.SandboxKey,
			"gateway":  hint.Gateway,
		}).Info("[Join] Setting IPv4 gateway retrieved from initial DHCP in CreateEndpoint")
		res.Gateway = hint.Gateway
	}

	bridge, err := netlink.LinkByName(opts.Bridge)
	if err != nil {
		return res, fmt.Errorf("failed to get bridge interface: %w", err)
	}

	if err := p.addRoutes(&opts, false, bridge, r, hint, &res); err != nil {
		return res, err
	}
	if opts.IPv6 {
		if err := p.addRoutes(&opts, true, bridge, r, hint, &res); err != nil {
			return res, err
		}
	}

	go func() {
		ctx, cancel := context.WithTimeout(context.Background(), p.awaitTimeout)
		defer cancel()

		m := newDHCPManager(p.docker, r, opts)
		m.LastIP = hint.IPv4
		m.LastIPv6 = hint.IPv6
		m.IfIndex = hint.IfIndex

		if err := m.Start(ctx); err != nil {
			log.WithError(err).WithFields(log.Fields{
				"network":  r.NetworkID[:12],
				"endpoint": r.EndpointID[:12],
				"sandbox":  r.SandboxKey,
			}).Error("Failed to start persistent DHCP client")
			return
		}

		p.persistentDHCP[r.EndpointID] = m
	}()

	log.WithFields(log.Fields{
		"network":  r.NetworkID[:12],
		"endpoint": r.EndpointID[:12],
		"sandbox":  r.SandboxKey,
	}).Info("Joined sandbox to endpoint")

	return res, nil
}

// Leave stops the persistent DHCP client for an endpoint
func (p *Plugin) Leave(ctx context.Context, r LeaveRequest) error {
	manager, ok := p.persistentDHCP[r.EndpointID]
	if !ok {
		return util.ErrNoSandbox
	}
	delete(p.persistentDHCP, r.EndpointID)

	if err := manager.Stop(); err != nil {
		return err
	}

	log.WithFields(log.Fields{
		"network":  r.NetworkID[:12],
		"endpoint": r.EndpointID[:12],
	}).Info("Sandbox left endpoint")

	return nil
}