From b136de3a7e01322866dd82700579fb69649bd9c0 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Thu, 22 Oct 2026 22:10:33 -0600 Subject: [PATCH 1/2] feat(hhfab): give DPDK gateway VMs a virtual IOMMU The DPDK driver binds the gateway's NICs to vfio-pci, and vfio-pci needs an IOMMU group to bind to. VLAB gateway VMs have no IOMMU, and the Flatcar kernel is built without VFIO no-IOMMU mode, so the bind fails with EINVAL and the dataplane never starts. Give gateway VMs whose Gateway uses the DPDK driver (an interface with a PCI address, as the gateway API decides it) an emulated Intel IOMMU with interrupt remapping. The Flatcar kernel enables the IOMMU by default. The IOMMU alone is not enough: every NIC sits behind the same conventional PCI bridge, which puts them all in one IOMMU group, and vfio-pci takes a group whole. The management NIC would go with the dataplane's, and DPDK refuses the group as not viable while it is bound to e1000. So these VMs put their dataplane NICs on the root bus, one group each, and keep the management NIC behind the bridge where its name, enp2s0, comes from. The Gateway's PCI addresses are generated from the same rule. Co-Authored-By: Claude Opus 5.5 --- pkg/hhfab/vlabbuilder.go | 2 +- pkg/hhfab/vlabconfig.go | 54 +++++++++++++++++++++++++++- pkg/hhfab/vlabconfig_test.go | 68 ++++++++++++++++++++++++++++++++++++ pkg/hhfab/vlabrunner.go | 9 +++++ 4 files changed, 131 insertions(+), 2 deletions(-) diff --git a/pkg/hhfab/vlabbuilder.go b/pkg/hhfab/vlabbuilder.go index 5509cb712..22ee3e49c 100644 --- a/pkg/hhfab/vlabbuilder.go +++ b/pkg/hhfab/vlabbuilder.go @@ -415,7 +415,7 @@ func (b *VLABBuilderDefault) Build(ctx context.Context, l *apiutil.Loader, f fab // } case GatewayDriverDPDK: ifaces[fmt.Sprintf("port%d", i)] = gwapi.GatewayInterface{ - PCI: fmt.Sprintf("0000:02:%02d.0", i), + PCI: VLABGatewayDPDKPCI(uint(i)), } } } diff --git a/pkg/hhfab/vlabconfig.go b/pkg/hhfab/vlabconfig.go index 83a94bb74..532067bb9 100644 --- a/pkg/hhfab/vlabconfig.go +++ b/pkg/hhfab/vlabconfig.go @@ -19,6 +19,7 @@ import ( "dario.cat/mergo" "github.com/charmbracelet/keygen" + gwapi "go.githedgehog.com/fabric/api/gateway/v1alpha1" fmeta "go.githedgehog.com/fabric/api/meta" vpcapi "go.githedgehog.com/fabric/api/vpc/v1beta1" wiringapi "go.githedgehog.com/fabric/api/wiring/v1beta1" @@ -58,6 +59,12 @@ const ( VLABNICsPerPCIBridge = 32 VLABPCIBridges = 2 VLABMaxNICs = VLABNICsPerPCIBridge * VLABPCIBridges + // A gateway VM with a virtual IOMMU keeps its management NIC behind the PCI bridge and puts + // its other NICs on the root bus from this slot on. Everything behind a conventional PCI + // bridge shares one IOMMU group, and vfio-pci can only hand a group to DPDK whole, so a + // dataplane NIC there would take the management NIC with it. + VLABIOMMUNICBaseSlot = 0x10 + VLABIOMMUMaxNICs = 0x1f - VLABIOMMUNICBaseSlot // slot 0x1f is the q35 LPC/SATA/SMBus VLABBaseSSHPort = 22000 VLABBaseDirectPort = 22100 VLABTapPrefix = "hhtap" @@ -83,6 +90,7 @@ type VM struct { Restricted bool NICs []string Size VMSize + IOMMU bool } type ExternalAttachCfg struct { @@ -146,6 +154,7 @@ type VMConfig struct { SwitchMode VMSwitchMode `json:"switchMode,omitempty"` SwitchProfile string `json:"switchProfile,omitempty"` NICs map[string]string `json:"nics"` + IOMMU bool `json:"iommu,omitempty"` } type VMType string @@ -433,6 +442,12 @@ func createVLABConfig(ctx context.Context, controls []fabapi.ControlNode, nodes } } + // The DPDK driver binds its NICs to vfio-pci, which needs an IOMMU in the VM. + dpdkGateways, err := getDPDKGateways(ctx, wiring) + if err != nil { + return nil, err + } + // TODO deduplicate for _, node := range nodes { if len(node.Spec.Roles) != 1 || node.Spec.Roles[0] != fabapi.NodeRoleGateway { @@ -476,6 +491,7 @@ func createVLABConfig(ctx context.Context, controls []fabapi.ControlNode, nodes NICs: map[string]string{ mgmtIface: mgmt, }, + IOMMU: dpdkGateways[node.Name], } } @@ -1107,7 +1123,14 @@ func vlabFromConfig(cfg *VLABConfig, opts VLABRunOpts) (*VLAB, error) { device = fmt.Sprintf("%s,netdev=eth%02d,mac=%s%s", nic, nicID, mac, link) } - device += fmt.Sprintf(",bus=%s%d,addr=0x%x", VLABPCIBridgePrefix, nicID/VLABNICsPerPCIBridge, nicID%VLABNICsPerPCIBridge) + if vm.IOMMU && nicID > 0 { + if nicID > VLABIOMMUMaxNICs { + return nil, fmt.Errorf("too many NICs for VM %q with IOMMU: %d", name, nicID) //nolint:goerr113 + } + device += fmt.Sprintf(",bus=pcie.0,addr=0x%x", VLABIOMMUNICBaseSlot+nicID) + } else { + device += fmt.Sprintf(",bus=%s%d,addr=0x%x", VLABPCIBridgePrefix, nicID/VLABNICsPerPCIBridge, nicID%VLABNICsPerPCIBridge) + } nic := "" if netdev != "" { @@ -1149,6 +1172,7 @@ func vlabFromConfig(cfg *VLABConfig, opts VLABRunOpts) (*VLAB, error) { SwitchProfile: vm.SwitchProfile, NICs: paddedNICs, Size: size, + IOMMU: vm.IOMMU, }) } @@ -1161,6 +1185,34 @@ func vlabFromConfig(cfg *VLABConfig, opts VLABRunOpts) (*VLAB, error) { }, nil } +// VLABGatewayDPDKPCI is the PCI address a DPDK gateway VM's NIC nicID has: on the root bus, see +// VLABIOMMUNICBaseSlot. +func VLABGatewayDPDKPCI(nicID uint) string { + return fmt.Sprintf("0000:00:%02x.0", VLABIOMMUNICBaseSlot+nicID) +} + +// getDPDKGateways returns the gateways that use the DPDK driver: those with an interface given by +// PCI address, the same way the gateway API defaults its driver. +func getDPDKGateways(ctx context.Context, wiring kclient.Reader) (map[string]bool, error) { + gws := &gwapi.GatewayList{} + if err := wiring.List(ctx, gws); err != nil { + return nil, fmt.Errorf("listing gateways: %w", err) + } + + dpdk := map[string]bool{} + for _, gw := range gws.Items { + for _, iface := range gw.Spec.Interfaces { + if iface.PCI != "" { + dpdk[gw.Name] = true + + break + } + } + } + + return dpdk, nil +} + func getAsn(obj kclient.Object) uint32 { if obj.GetAnnotations() != nil { annotationLabel := HHFabCfgASNPrefix + HHFabCfgPrefix diff --git a/pkg/hhfab/vlabconfig_test.go b/pkg/hhfab/vlabconfig_test.go index d8dda2af8..a0c9cdeb6 100644 --- a/pkg/hhfab/vlabconfig_test.go +++ b/pkg/hhfab/vlabconfig_test.go @@ -13,6 +13,7 @@ import ( "testing" "github.com/stretchr/testify/require" + gwapi "go.githedgehog.com/fabric/api/gateway/v1alpha1" "go.githedgehog.com/fabric/api/meta" wiringapi "go.githedgehog.com/fabric/api/wiring/v1beta1" fabapi "go.githedgehog.com/fabricator/api/fabricator/v1beta1" @@ -274,3 +275,70 @@ func TestCreateVLABConfigHardwareGateway(t *testing.T) { require.Equal(t, NICTypePassthrough+NICTypeSep+pci, cfg.VMs[parts[0]].NICs[parts[1]], swPort) } } + +func TestCreateVLABConfigDPDKGatewayIOMMU(t *testing.T) { + nodes := []fabapi.FabNode{} + for _, name := range []string{"gateway-1", "gateway-2"} { + nodes = append(nodes, fabapi.FabNode{ + ObjectMeta: kmetav1.ObjectMeta{Name: name}, + Spec: fabapi.FabNodeSpec{ + Roles: []fabapi.FabNodeRole{fabapi.NodeRoleGateway}, + Management: fabapi.ControlNodeManagement{Interface: "enp2s0"}, + }, + }) + } + + for _, driver := range GatewayDrivers { + t.Run(driver, func(t *testing.T) { + l := apiutil.NewLoader() + b := &VLABBuilderDefault{ + GatewayLogLevel: "info", + GatewayDriver: driver, + VLABBuilderBase: VLABBuilderBase{DefaultSwitchProfile: meta.SwitchProfileVS}, + } + f := fabapi.Fabricator{Spec: fabapi.FabricatorSpec{Config: fab.DefaultConfig}} + f.Default() + require.NoError(t, b.Build(t.Context(), l, f, nodes)) + + cfg, err := createVLABConfig(t.Context(), nil, nodes, l.GetClient()) + require.NoError(t, err) + + vlab, err := vlabFromConfig(cfg, VLABRunOpts{}) + require.NoError(t, err) + + dpdk := driver == GatewayDriverDPDK + gws := 0 + for _, vm := range vlab.VMs { + if vm.Type != VMTypeGateway { + require.False(t, vm.IOMMU, vm.Name) + + continue + } + gws++ + require.Equal(t, dpdk, vm.IOMMU, vm.Name) + + // the management NIC stays behind the PCI bridge, where its name comes from + require.Contains(t, vm.NICs[0], ",bus="+VLABPCIBridgePrefix+"0,addr=0x0", vm.Name) + if !dpdk { + continue + } + + gw := &gwapi.Gateway{} + require.NoError(t, l.GetClient().Get(t.Context(), kclient.ObjectKey{Name: vm.Name, Namespace: "default"}, gw)) + require.NotEmpty(t, gw.Spec.Interfaces, vm.Name) + for ifaceName, iface := range gw.Spec.Interfaces { + var nicID uint + _, err := fmt.Sscanf(ifaceName, "port%d", &nicID) + require.NoError(t, err, ifaceName) + + // the address the gateway is told is where the VM's NIC is + var slot uint + _, err = fmt.Sscanf(iface.PCI, "0000:00:%02x.0", &slot) + require.NoError(t, err, iface.PCI) + require.Contains(t, vm.NICs[nicID], fmt.Sprintf(",bus=pcie.0,addr=0x%x", slot), vm.Name+"/"+ifaceName) + } + } + require.Equal(t, 2, gws) + }) + } +} diff --git a/pkg/hhfab/vlabrunner.go b/pkg/hhfab/vlabrunner.go index 7e8f4fce0..309acc301 100644 --- a/pkg/hhfab/vlabrunner.go +++ b/pkg/hhfab/vlabrunner.go @@ -468,6 +468,15 @@ func (c *Config) VLABRun(ctx context.Context, vlab *VLAB, opts VLABRunOpts) erro "-global", "ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off", } + // An emulated Intel IOMMU, so the DPDK driver can bind its NICs to vfio-pci. Interrupt + // remapping needs the split irqchip. The Flatcar kernel enables the IOMMU by default. + // It goes right after the machine, ahead of every PCI device it translates for. + if vm.IOMMU { + idx := slices.Index(args, "-machine") + 2 + args[idx-1] += ",kernel-irqchip=split" + args = slices.Insert(args, idx, "-device", "intel-iommu,intremap=on") + } + // Mark Flatcar VMs as VLAB in SMBIOS, where VLAB-only host config can match on it // (see flatcar.VLABVirtioNamesLink). The chassis asset tag is set by nothing else and // read by nothing else, unlike the vendor and product fields switch images may use. From e65e59323bebc2c0ce12b1d901033c2aed98d0cf Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Thu, 22 Oct 2026 23:06:23 -0600 Subject: [PATCH 2/2] feat(hhfab): give DPDK gateway VMs e1000e dataplane NICs The DPDK driver gives each port a transmit queue per worker and one more for the control plane. QEMU's e1000 emulates the 82540EM, which has a single transmit ring, but DPDK's em driver advertises two for every em device. The control plane's frames went to the second queue and never left the VM: no ARP, no BGP, no BFD, while the dataplane counted them as sent. The e1000e (82574L) has two rings, and with it BGP, BFD and LLDP come up over the DPDK driver. Co-Authored-By: Claude Opus 5.5 --- pkg/hhfab/vlabconfig.go | 7 +++++++ pkg/hhfab/vlabconfig_test.go | 2 ++ 2 files changed, 9 insertions(+) diff --git a/pkg/hhfab/vlabconfig.go b/pkg/hhfab/vlabconfig.go index 532067bb9..05181d2ff 100644 --- a/pkg/hhfab/vlabconfig.go +++ b/pkg/hhfab/vlabconfig.go @@ -1120,6 +1120,13 @@ func vlabFromConfig(cfg *VLABConfig, opts VLABRunOpts) (*VLAB, error) { // sets what the guest sees, it does not limit the traffic link = ",speed=" + VLABVirtioNetSpeed + ",duplex=full" } + // The DPDK driver gives each port a transmit queue per worker and one more for + // the control plane. The e1000 (82540EM) has a single transmit ring, though its + // DPDK driver advertises two, so what is sent on the second never leaves; the + // e1000e (82574L) has two. + if vm.IOMMU && nicID > 0 { + nic = "e1000e" + } device = fmt.Sprintf("%s,netdev=eth%02d,mac=%s%s", nic, nicID, mac, link) } diff --git a/pkg/hhfab/vlabconfig_test.go b/pkg/hhfab/vlabconfig_test.go index a0c9cdeb6..e6a257a23 100644 --- a/pkg/hhfab/vlabconfig_test.go +++ b/pkg/hhfab/vlabconfig_test.go @@ -319,6 +319,7 @@ func TestCreateVLABConfigDPDKGatewayIOMMU(t *testing.T) { // the management NIC stays behind the PCI bridge, where its name comes from require.Contains(t, vm.NICs[0], ",bus="+VLABPCIBridgePrefix+"0,addr=0x0", vm.Name) + require.Contains(t, vm.NICs[0], "-device e1000,", vm.Name) if !dpdk { continue } @@ -336,6 +337,7 @@ func TestCreateVLABConfigDPDKGatewayIOMMU(t *testing.T) { _, err = fmt.Sscanf(iface.PCI, "0000:00:%02x.0", &slot) require.NoError(t, err, iface.PCI) require.Contains(t, vm.NICs[nicID], fmt.Sprintf(",bus=pcie.0,addr=0x%x", slot), vm.Name+"/"+ifaceName) + require.Contains(t, vm.NICs[nicID], "-device e1000e,", vm.Name+"/"+ifaceName) } } require.Equal(t, 2, gws)