feat(hhfab): give DPDK gateway VMs a virtual IOMMU and e1000e NICs

pkg/hhfab/vlabbuilder.go

@@ -415,7 +415,7 @@ func (b *VLABBuilderDefault) Build(ctx context.Context, l *apiutil.Loader, f fab
 					// }
 				case GatewayDriverDPDK:
 					ifaces[fmt.Sprintf("port%d", i)] = gwapi.GatewayInterface{
-						PCI: fmt.Sprintf("0000:02:%02d.0", i),
+						PCI: VLABGatewayDPDKPCI(uint(i)),
 					}
 				}
 			}

pkg/hhfab/vlabconfig.go

@@ -19,6 +19,7 @@ import (
 
 	"dario.cat/mergo"
 	"github.com/charmbracelet/keygen"
+	gwapi "go.githedgehog.com/fabric/api/gateway/v1alpha1"
 	fmeta "go.githedgehog.com/fabric/api/meta"
 	vpcapi "go.githedgehog.com/fabric/api/vpc/v1beta1"
 	wiringapi "go.githedgehog.com/fabric/api/wiring/v1beta1"
@@ -58,6 +59,12 @@ const (
 	VLABNICsPerPCIBridge = 32
 	VLABPCIBridges       = 2
 	VLABMaxNICs          = VLABNICsPerPCIBridge * VLABPCIBridges
+	// A gateway VM with a virtual IOMMU keeps its management NIC behind the PCI bridge and puts
+	// its other NICs on the root bus from this slot on. Everything behind a conventional PCI
+	// bridge shares one IOMMU group, and vfio-pci can only hand a group to DPDK whole, so a
+	// dataplane NIC there would take the management NIC with it.
+	VLABIOMMUNICBaseSlot = 0x10
+	VLABIOMMUMaxNICs     = 0x1f - VLABIOMMUNICBaseSlot // slot 0x1f is the q35 LPC/SATA/SMBus
 	VLABBaseSSHPort      = 22000
 	VLABBaseDirectPort   = 22100
 	VLABTapPrefix        = "hhtap"
@@ -83,6 +90,7 @@ type VM struct {
 	Restricted    bool
 	NICs          []string
 	Size          VMSize
+	IOMMU         bool
 }
 
 type ExternalAttachCfg struct {
@@ -146,6 +154,7 @@ type VMConfig struct {
 	SwitchMode    VMSwitchMode      `json:"switchMode,omitempty"`
 	SwitchProfile string            `json:"switchProfile,omitempty"`
 	NICs          map[string]string `json:"nics"`
+	IOMMU         bool              `json:"iommu,omitempty"`
 }
 
 type VMType string
@@ -433,6 +442,12 @@ func createVLABConfig(ctx context.Context, controls []fabapi.ControlNode, nodes
 		}
 	}
 
+	// The DPDK driver binds its NICs to vfio-pci, which needs an IOMMU in the VM.
+	dpdkGateways, err := getDPDKGateways(ctx, wiring)
+	if err != nil {
+		return nil, err
+	}
+
 	// TODO deduplicate
 	for _, node := range nodes {
 		if len(node.Spec.Roles) != 1 || node.Spec.Roles[0] != fabapi.NodeRoleGateway {
@@ -476,6 +491,7 @@ func createVLABConfig(ctx context.Context, controls []fabapi.ControlNode, nodes
 			NICs: map[string]string{
 				mgmtIface: mgmt,
 			},
+			IOMMU: dpdkGateways[node.Name],
 		}
 	}
 
@@ -1104,10 +1120,24 @@ func vlabFromConfig(cfg *VLABConfig, opts VLABRunOpts) (*VLAB, error) {
 					// sets what the guest sees, it does not limit the traffic
 					link = ",speed=" + VLABVirtioNetSpeed + ",duplex=full"
 				}
+				// The DPDK driver gives each port a transmit queue per worker and one more for
+				// the control plane. The e1000 (82540EM) has a single transmit ring, though its
+				// DPDK driver advertises two, so what is sent on the second never leaves; the
+				// e1000e (82574L) has two.
+				if vm.IOMMU && nicID > 0 {
+					nic = "e1000e"
+				}
 
 				device = fmt.Sprintf("%s,netdev=eth%02d,mac=%s%s", nic, nicID, mac, link)
 			}
-			device += fmt.Sprintf(",bus=%s%d,addr=0x%x", VLABPCIBridgePrefix, nicID/VLABNICsPerPCIBridge, nicID%VLABNICsPerPCIBridge)
+			if vm.IOMMU && nicID > 0 {
+				if nicID > VLABIOMMUMaxNICs {
+					return nil, fmt.Errorf("too many NICs for VM %q with IOMMU: %d", name, nicID) //nolint:goerr113
+				}
+				device += fmt.Sprintf(",bus=pcie.0,addr=0x%x", VLABIOMMUNICBaseSlot+nicID)
+			} else {
+				device += fmt.Sprintf(",bus=%s%d,addr=0x%x", VLABPCIBridgePrefix, nicID/VLABNICsPerPCIBridge, nicID%VLABNICsPerPCIBridge)
+			}
 
 			nic := ""
 			if netdev != "" {
@@ -1149,6 +1179,7 @@ func vlabFromConfig(cfg *VLABConfig, opts VLABRunOpts) (*VLAB, error) {
 			SwitchProfile: vm.SwitchProfile,
 			NICs:          paddedNICs,
 			Size:          size,
+			IOMMU:         vm.IOMMU,
 		})
 	}
 
@@ -1161,6 +1192,34 @@ func vlabFromConfig(cfg *VLABConfig, opts VLABRunOpts) (*VLAB, error) {
 	}, nil
 }
 
+// VLABGatewayDPDKPCI is the PCI address a DPDK gateway VM's NIC nicID has: on the root bus, see
+// VLABIOMMUNICBaseSlot.
+func VLABGatewayDPDKPCI(nicID uint) string {
+	return fmt.Sprintf("0000:00:%02x.0", VLABIOMMUNICBaseSlot+nicID)
+}
+
+// getDPDKGateways returns the gateways that use the DPDK driver: those with an interface given by
+// PCI address, the same way the gateway API defaults its driver.
+func getDPDKGateways(ctx context.Context, wiring kclient.Reader) (map[string]bool, error) {
+	gws := &gwapi.GatewayList{}
+	if err := wiring.List(ctx, gws); err != nil {
+		return nil, fmt.Errorf("listing gateways: %w", err)
+	}
+
+	dpdk := map[string]bool{}
+	for _, gw := range gws.Items {
+		for _, iface := range gw.Spec.Interfaces {
+			if iface.PCI != "" {
+				dpdk[gw.Name] = true
+
+				break
+			}
+		}
+	}
+
+	return dpdk, nil
+}
+
 func getAsn(obj kclient.Object) uint32 {
 	if obj.GetAnnotations() != nil {
 		annotationLabel := HHFabCfgASNPrefix + HHFabCfgPrefix

pkg/hhfab/vlabconfig_test.go

@@ -13,6 +13,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+	gwapi "go.githedgehog.com/fabric/api/gateway/v1alpha1"
 	"go.githedgehog.com/fabric/api/meta"
 	wiringapi "go.githedgehog.com/fabric/api/wiring/v1beta1"
 	fabapi "go.githedgehog.com/fabricator/api/fabricator/v1beta1"
@@ -274,3 +275,72 @@ func TestCreateVLABConfigHardwareGateway(t *testing.T) {
 		require.Equal(t, NICTypePassthrough+NICTypeSep+pci, cfg.VMs[parts[0]].NICs[parts[1]], swPort)
 	}
 }
+
+func TestCreateVLABConfigDPDKGatewayIOMMU(t *testing.T) {
+	nodes := []fabapi.FabNode{}
+	for _, name := range []string{"gateway-1", "gateway-2"} {
+		nodes = append(nodes, fabapi.FabNode{
+			ObjectMeta: kmetav1.ObjectMeta{Name: name},
+			Spec: fabapi.FabNodeSpec{
+				Roles:      []fabapi.FabNodeRole{fabapi.NodeRoleGateway},
+				Management: fabapi.ControlNodeManagement{Interface: "enp2s0"},
+			},
+		})
+	}
+
+	for _, driver := range GatewayDrivers {
+		t.Run(driver, func(t *testing.T) {
+			l := apiutil.NewLoader()
+			b := &VLABBuilderDefault{
+				GatewayLogLevel: "info",
+				GatewayDriver:   driver,
+				VLABBuilderBase: VLABBuilderBase{DefaultSwitchProfile: meta.SwitchProfileVS},
+			}
+			f := fabapi.Fabricator{Spec: fabapi.FabricatorSpec{Config: fab.DefaultConfig}}
+			f.Default()
+			require.NoError(t, b.Build(t.Context(), l, f, nodes))
+
+			cfg, err := createVLABConfig(t.Context(), nil, nodes, l.GetClient())
+			require.NoError(t, err)
+
+			vlab, err := vlabFromConfig(cfg, VLABRunOpts{})
+			require.NoError(t, err)
+
+			dpdk := driver == GatewayDriverDPDK
+			gws := 0
+			for _, vm := range vlab.VMs {
+				if vm.Type != VMTypeGateway {
+					require.False(t, vm.IOMMU, vm.Name)
+
+					continue
+				}
+				gws++
+				require.Equal(t, dpdk, vm.IOMMU, vm.Name)
+
+				// the management NIC stays behind the PCI bridge, where its name comes from
+				require.Contains(t, vm.NICs[0], ",bus="+VLABPCIBridgePrefix+"0,addr=0x0", vm.Name)
+				require.Contains(t, vm.NICs[0], "-device e1000,", vm.Name)
+				if !dpdk {
+					continue
+				}
+
+				gw := &gwapi.Gateway{}
+				require.NoError(t, l.GetClient().Get(t.Context(), kclient.ObjectKey{Name: vm.Name, Namespace: "default"}, gw))
+				require.NotEmpty(t, gw.Spec.Interfaces, vm.Name)
+				for ifaceName, iface := range gw.Spec.Interfaces {
+					var nicID uint
+					_, err := fmt.Sscanf(ifaceName, "port%d", &nicID)
+					require.NoError(t, err, ifaceName)
+
+					// the address the gateway is told is where the VM's NIC is
+					var slot uint
+					_, err = fmt.Sscanf(iface.PCI, "0000:00:%02x.0", &slot)
+					require.NoError(t, err, iface.PCI)
+					require.Contains(t, vm.NICs[nicID], fmt.Sprintf(",bus=pcie.0,addr=0x%x", slot), vm.Name+"/"+ifaceName)
+					require.Contains(t, vm.NICs[nicID], "-device e1000e,", vm.Name+"/"+ifaceName)
+				}
+			}
+			require.Equal(t, 2, gws)
+		})
+	}
+}

pkg/hhfab/vlabrunner.go

@@ -468,6 +468,15 @@ func (c *Config) VLABRun(ctx context.Context, vlab *VLAB, opts VLABRunOpts) erro
 				"-global", "ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off",
 			}
 
+			// An emulated Intel IOMMU, so the DPDK driver can bind its NICs to vfio-pci. Interrupt
+			// remapping needs the split irqchip. The Flatcar kernel enables the IOMMU by default.
+			// It goes right after the machine, ahead of every PCI device it translates for.
+			if vm.IOMMU {
+				idx := slices.Index(args, "-machine") + 2
+				args[idx-1] += ",kernel-irqchip=split"
+				args = slices.Insert(args, idx, "-device", "intel-iommu,intremap=on")
+			}
+
 			// Mark Flatcar VMs as VLAB in SMBIOS, where VLAB-only host config can match on it
 			// (see flatcar.VLABVirtioNamesLink). The chassis asset tag is set by nothing else and
 			// read by nothing else, unlike the vendor and product fields switch images may use.