VRF: Wait for the local/host routes to be added

Without waiting for the local/host routes to be added
by the kernel after the IP address is being added to
an interface. The routes requiring the local/host routes
may failed. This caused flaky e2e tests, but could also
happen during the execution of the VRF plugin when the
IPv6 addresses were being re-added to the interface and
when the route were being moved to the VRF table.

Signed-off-by: Lionel Jouin <[email protected]>

Author: LionelJouin

plugins/meta/vrf/vrf.go

@@ -17,6 +17,8 @@ package main
 import (
 	"fmt"
 	"math"
+	"net"
+	"time"
 
 	"github.com/vishvananda/netlink"
 )
@@ -124,7 +126,7 @@ func addInterface(vrf *netlink.Vrf, intf string) error {
 
 	afterAddresses, err := netlink.AddrList(i, netlink.FAMILY_V6)
 	if err != nil {
-		return fmt.Errorf("failed getting ipv6 new addresses for %s", intf)
+		return fmt.Errorf("failed getting ipv6 new addresses for %s: %v", intf, err)
 	}
 
 	// Since keeping the ipv6 address depends on net.ipv6.conf.all.keep_addr_on_down ,
@@ -141,6 +143,37 @@ CONTINUE:
 		if err != nil {
 			return fmt.Errorf("could not restore address %s to %s @ %s: %v", toFind, intf, vrf.Name, err)
 		}
+
+		// Waits for local/host routes to be added by the kernel.
+		maxRetry := 10
+		for {
+			routesVRFTable, err := netlink.RouteListFiltered(
+				netlink.FAMILY_ALL,
+				&netlink.Route{
+					Dst: &net.IPNet{
+						IP:   toFind.IP,
+						Mask: net.IPMask{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
+					},
+					Table:     int(vrf.Table),
+					LinkIndex: i.Attrs().Index,
+				},
+				netlink.RT_FILTER_OIF|netlink.RT_FILTER_TABLE|netlink.RT_FILTER_DST,
+			)
+			if err != nil {
+				return fmt.Errorf("failed getting routes for %s table %d for dst %s: %v", intf, vrf.Table, toFind.IPNet.String(), err)
+			}
+
+			if len(routesVRFTable) >= 1 {
+				break
+			}
+
+			maxRetry--
+			if maxRetry <= 0 {
+				return fmt.Errorf("failed getting local/host addresses for %s in table %d with dst %s", intf, vrf.Table, toFind.IPNet.String())
+			}
+
+			time.Sleep(10 * time.Millisecond)
+		}
 	}
 
 	// Apply all saved routes for the interface that was moved to the VRF

plugins/meta/vrf/vrf_test.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"net"
 	"strings"
+	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -207,6 +208,18 @@ var _ = Describe("vrf plugin", func() {
 				// Add IP addresses for network reachability
 				netlink.AddrAdd(link, &netlink.Addr{IPNet: ipv4})
 				netlink.AddrAdd(link, &netlink.Addr{IPNet: ipv6})
+				// Wait for the corresponding route to be addeded
+				Eventually(func() bool {
+					ipv6RouteDst := &net.IPNet{
+						IP:   ipv6.IP,
+						Mask: net.IPMask{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
+					}
+					routes, _ := netlink.RouteListFiltered(netlink.FAMILY_ALL, &netlink.Route{
+						Dst:   ipv6RouteDst,
+						Table: 0,
+					}, netlink.RT_FILTER_DST|netlink.RT_FILTER_TABLE)
+					return err == nil && len(routes) >= 1
+				}, time.Second, 500*time.Millisecond).Should(BeTrue())
 
 				ipAddrs, err := netlink.AddrList(link, netlink.FAMILY_V4)
 				Expect(err).NotTo(HaveOccurred())
@@ -304,6 +317,18 @@ var _ = Describe("vrf plugin", func() {
 				// Add IP addresses for network reachability
 				netlink.AddrAdd(link, &netlink.Addr{IPNet: ipv4})
 				netlink.AddrAdd(link, &netlink.Addr{IPNet: ipv6})
+				// Wait for the corresponding route to be addeded
+				Eventually(func() bool {
+					ipv6RouteDst := &net.IPNet{
+						IP:   ipv6.IP,
+						Mask: net.IPMask{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
+					}
+					routes, _ := netlink.RouteListFiltered(netlink.FAMILY_ALL, &netlink.Route{
+						Dst:   ipv6RouteDst,
+						Table: 0,
+					}, netlink.RT_FILTER_DST|netlink.RT_FILTER_TABLE)
+					return err == nil && len(routes) >= 1
+				}, time.Second, 500*time.Millisecond).Should(BeTrue())
 
 				ipAddrs, err := netlink.AddrList(link, netlink.FAMILY_V4)
 				Expect(err).NotTo(HaveOccurred())
@@ -362,6 +387,18 @@ var _ = Describe("vrf plugin", func() {
 				// Add IP addresses for network reachability
 				netlink.AddrAdd(link, &netlink.Addr{IPNet: ipv4})
 				netlink.AddrAdd(link, &netlink.Addr{IPNet: ipv6})
+				// Wait for the corresponding route to be addeded
+				Eventually(func() bool {
+					ipv6RouteDst := &net.IPNet{
+						IP:   ipv6.IP,
+						Mask: net.IPMask{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
+					}
+					routes, _ := netlink.RouteListFiltered(netlink.FAMILY_ALL, &netlink.Route{
+						Dst:   ipv6RouteDst,
+						Table: 0,
+					}, netlink.RT_FILTER_DST|netlink.RT_FILTER_TABLE)
+					return err == nil && len(routes) >= 1
+				}, time.Second, 500*time.Millisecond).Should(BeTrue())
 
 				ipAddrs, err := netlink.AddrList(link, netlink.FAMILY_V4)
 				Expect(err).NotTo(HaveOccurred())