From 927ab369b9897372464417e32cd887191bc2b94d Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 14 Oct 2026 02:10:36 +0000 Subject: [PATCH 01/19] Place vGPU profiles the driver hides from creatable lists In mixed-size mode the driver drops a profile from every VF's creatable_vgpu_types once its GPU holds as many vGPUs as that profile's equal-size instance limit, even while a placement remains. Remember the profiles each GPU has advertised, and when none advertises the requested one, let the driver decide by configuring a clean VF per supporting GPU under the allocation lock. --- lib/devices/GPU.md | 2 + lib/instances/manager.go | 1 + lib/instances/vgpu.go | 121 ++++++++++++++++++++++++++++--- lib/instances/vgpu_linux_test.go | 92 +++++++++++++++++++++++ lib/instances/vgpu_test.go | 110 ++++++++++++++++++++++++++++ 5 files changed, 316 insertions(+), 10 deletions(-) diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 683a506a6..fb08e770e 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -106,6 +106,8 @@ before the hypervisor is signalled. Hypeman reconciles metadata claims once at startup and every minute afterward, skipping hosts without GPUs. A claim whose VMM is confirmed dead is reset before the claim is removed. An ambiguous hypervisor ownership check preserves the claim, logs a warning, and increments `hypeman_instances_vgpu_liveness_uncertain_total`; the create and start cleanup paths preserve and count the same way. mdev hosts also sweep orphaned device-level assignments. Vendor VFIO hosts repair an unclaimed dirty VF when the allocator next selects it; repair checks for open VFIO handles before resetting `current_vgpu_type`. The allocator prefers VFs that are already clean, and a dirty VF that refuses its reset is skipped in favor of another candidate. +In mixed-size (heterogeneous) mode, `creatable_vgpu_types` under-reports. The driver drops a profile from every VF's list once its GPU holds as many vGPUs as that profile's equal-size instance limit (12 for L40S-4Q), even while a placement for it remains, and still accepts a `current_vgpu_type` write for it. The allocator remembers every profile each GPU has advertised. When no GPU advertises the requested profile, it writes the profile to a clean VF on each GPU that supports it, under the allocation lock, until the driver accepts one. GPUs that advertise the profile are always tried first. A GPU whose VFs advertise nothing has no placement left for any size and is skipped. + ### Hypervisor Support Hypervisor selection for vGPU instances is caller policy; hypeman does not enforce it. In practice **QEMU is the only hypervisor with working vGPU support**: diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 2a9bb6dfb..2598b45c5 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -192,6 +192,7 @@ type manager struct { destroyVGPU func(context.Context, devices.VGPUAssignment) error reconcileVGPUDevices func(context.Context, map[string]struct{}) error vgpuAllocationMu sync.Mutex + supportedVGPUProfiles map[string]map[string]devices.VGPUProfileType // guarded by vgpuAllocationMu deleteSnapshotFn func(context.Context, string) error ttlReaperDeleteTimeout time.Duration egressProxy *egressproxy.Service diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 4563398d0..16a8236f0 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -57,6 +57,7 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str if err != nil { return nil, err } + m.rememberVGPUProfiles(vfs, profilesByVF) // Quarantine is read under the allocation lock but mutated under the // devices lock, so this is a snapshot. configure re-checks it under the // devices lock before the VF is touched. @@ -64,7 +65,7 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str if err != nil { return nil, err } - candidates := vendorVFIOCandidates{vfs: vfs, profilesByVF: profilesByVF, claims: allMetadata, quarantined: quarantined} + candidates := vendorVFIOCandidates{vfs: vfs, profilesByVF: profilesByVF, supported: m.supportedVGPUProfiles, claims: allMetadata, quarantined: quarantined} vfAddress, profileType, err := selectVendorVFIOVF(candidates, profileName, m.pickVFIndex) if err != nil { // A dirty unclaimed VF consumes framebuffer, which can make the @@ -111,6 +112,31 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str return nil, fmt.Errorf("repair dirty VF %s before claim: %w", vf.PCIAddress, repairErr) } } + // No VF on the chosen GPU advertises the profile, so only the driver can + // say whether a placement remains. Configure it here, under the + // allocation lock, and move to the next GPU if the driver refuses. A VF + // left configured by a failed claim below is an unclaimed dirty VF, + // which the next allocation repairs. + for { + vf, ok := vfByAddress(candidates.vfs, vfAddress) + if !ok || advertisingGPUs(candidates, profileType)[vf.ParentGPU] { + break + } + probeErr := m.configureClaimedVGPU(ctx, &devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + VFAddress: vfAddress, + ProfileType: profileType, + }) + if probeErr == nil { + log.InfoContext(ctx, "placed vGPU profile its GPU no longer advertises", "vf", vfAddress, "profile", profileName) + break + } + log.InfoContext(ctx, "driver refused vGPU profile on GPU; trying another GPU", "gpu", vf.ParentGPU, "vf", vfAddress, "profile", profileName, "error", probeErr) + candidates.vfs = withoutGPU(candidates.vfs, vf.ParentGPU) + if vfAddress, profileType, err = selectVendorVFIOVF(candidates, profileName, m.pickVFIndex); err != nil { + return nil, fmt.Errorf("%w (driver refused it on GPU %s: %v)", err, vf.ParentGPU, probeErr) + } + } device := &devices.VGPUDevice{ Framework: devices.VGPUFrameworkVendorVFIO, VFAddress: vfAddress, @@ -165,6 +191,28 @@ func (m *manager) resetDirtyUnclaimedVFs(ctx context.Context, vfs []devices.Virt return reset } +// rememberVGPUProfiles records every profile each GPU's VFs advertise. In +// mixed-size mode the driver drops a profile from every creatable list on a +// GPU once that GPU holds as many vGPUs as the profile's equal-size instance +// limit (12 for L40S-4Q), even while a placement for it remains, so the +// current lists alone under-report what a busy GPU can still take. Callers +// must hold vgpuAllocationMu. +func (m *manager) rememberVGPUProfiles(vfs []devices.VirtualFunction, profilesByVF map[string][]devices.VGPUProfileType) { + if m.supportedVGPUProfiles == nil { + m.supportedVGPUProfiles = make(map[string]map[string]devices.VGPUProfileType) + } + for _, vf := range vfs { + for _, profile := range profilesByVF[vf.PCIAddress] { + byName := m.supportedVGPUProfiles[vf.ParentGPU] + if byName == nil { + byName = make(map[string]devices.VGPUProfileType) + m.supportedVGPUProfiles[vf.ParentGPU] = byName + } + byName[profile.Name] = profile + } + } +} + func (m *manager) quarantinedVFAddresses() (map[string]struct{}, error) { quarantined := m.quarantinedVFs if quarantined == nil { @@ -177,7 +225,8 @@ func (m *manager) quarantinedVFAddresses() (map[string]struct{}, error) { type vendorVFIOCandidates struct { vfs []devices.VirtualFunction profilesByVF map[string][]devices.VGPUProfileType - claims []StoredMetadata // every instance; those with a device path hold a VF + supported map[string]map[string]devices.VGPUProfileType // parent GPU -> profile name -> profile, as remembered by rememberVGPUProfiles + claims []StoredMetadata // every instance; those with a device path hold a VF quarantined map[string]struct{} } @@ -186,22 +235,39 @@ type vendorVFIOCandidates struct { // drifts away from cards carrying a wedged VF. Among equally ranked // candidates on the chosen GPU, pick selects the index (nil is uniform // random), so a single VF cannot capture every placement on an idle host. +// +// A GPU that supports the profile but no longer advertises it is still a +// candidate for a clean VF, ranked after every GPU that does advertise it, +// because the driver hides profiles by instance count rather than by free +// placement. The caller must confirm such a choice with the driver. A GPU +// whose VFs advertise nothing at all has no placement left for any size and +// is not a candidate. func selectVendorVFIOVF(c vendorVFIOCandidates, profileName string, pick func(n int) int) (string, string, error) { profilesByName := make(map[string]devices.VGPUProfileType) + for _, byName := range c.supported { + for name, profile := range byName { + profilesByName[name] = profile + } + } + advertisedNames := make(map[string]struct{}) advertises := make(map[string]map[string]struct{}, len(c.profilesByVF)) for vfAddress, profiles := range c.profilesByVF { advertises[vfAddress] = make(map[string]struct{}, len(profiles)) for _, profile := range profiles { profilesByName[profile.Name] = profile + advertisedNames[profile.Name] = struct{}{} advertises[vfAddress][profile.TypeName] = struct{}{} } } + notCreatable := func() error { + if len(advertisedNames) == 0 && len(c.vfs) > 0 { + return fmt.Errorf("no creatable vGPU profiles on any VF (GPUs at capacity or dirty VFs consuming framebuffer): profile %q", profileName) + } + return fmt.Errorf("profile %q is not creatable on any VF (unknown profile or insufficient capacity)", profileName) + } requested, found := profilesByName[profileName] if !found { - if len(profilesByName) == 0 && len(c.vfs) > 0 { - return "", "", fmt.Errorf("no creatable vGPU profiles on any VF (GPUs at capacity or dirty VFs consuming framebuffer): profile %q", profileName) - } - return "", "", fmt.Errorf("profile %q is not creatable on any VF (unknown profile or insufficient capacity)", profileName) + return "", "", notCreatable() } vfsByAddress := make(map[string]devices.VirtualFunction, len(c.vfs)) @@ -230,10 +296,11 @@ func selectVendorVFIOVF(c vendorVFIOCandidates, profileName string, pick func(n usageByGPU[vf.ParentGPU] += profile.FramebufferMB } - parentAdvertises := make(map[string]bool) + parentAdvertises := advertisingGPUs(c, requested.TypeName) + parentAdvertisesAny := make(map[string]bool) for _, vf := range c.vfs { - if _, ok := advertises[vf.PCIAddress][requested.TypeName]; ok { - parentAdvertises[vf.ParentGPU] = true + if len(advertises[vf.PCIAddress]) > 0 { + parentAdvertisesAny[vf.ParentGPU] = true } } quarantinedByGPU := make(map[string]int) @@ -248,7 +315,10 @@ func selectVendorVFIOVF(c vendorVFIOCandidates, profileName string, pick func(n } _, directlyCreatable := advertises[vf.PCIAddress][requested.TypeName] repairable := vf.Allocated && (vf.ProfileType == requested.TypeName || parentAdvertises[vf.ParentGPU]) - if directlyCreatable || repairable { + supported, ok := c.supported[vf.ParentGPU][profileName] + hidden := !vf.Allocated && !parentAdvertises[vf.ParentGPU] && parentAdvertisesAny[vf.ParentGPU] && + ok && supported.TypeName == requested.TypeName + if directlyCreatable || repairable || hidden { freeByGPU[vf.ParentGPU] = append(freeByGPU[vf.ParentGPU], vf) } } @@ -268,6 +338,9 @@ func selectVendorVFIOVF(c vendorVFIOCandidates, profileName string, pick func(n gpus = append(gpus, gpu) } sort.Slice(gpus, func(i, j int) bool { + if parentAdvertises[gpus[i]] != parentAdvertises[gpus[j]] { + return parentAdvertises[gpus[i]] + } if quarantinedByGPU[gpus[i]] != quarantinedByGPU[gpus[j]] { return quarantinedByGPU[gpus[i]] < quarantinedByGPU[gpus[j]] } @@ -280,6 +353,9 @@ func selectVendorVFIOVF(c vendorVFIOCandidates, profileName string, pick func(n return usageByGPU[gpus[i]] < usageByGPU[gpus[j]] }) if len(gpus) == 0 { + if _, ok := advertisedNames[profileName]; !ok { + return "", "", notCreatable() + } return "", "", fmt.Errorf("no available VF for profile %q", profileName) } // Candidates are sorted clean-first, so the leading run with the same @@ -295,6 +371,21 @@ func selectVendorVFIOVF(c vendorVFIOCandidates, profileName string, pick func(n return candidates[pick(n)].PCIAddress, requested.TypeName, nil } +// advertisingGPUs returns the parent GPUs with at least one VF advertising +// profileType. +func advertisingGPUs(c vendorVFIOCandidates, profileType string) map[string]bool { + gpus := make(map[string]bool) + for _, vf := range c.vfs { + for _, profile := range c.profilesByVF[vf.PCIAddress] { + if profile.TypeName == profileType { + gpus[vf.ParentGPU] = true + break + } + } + } + return gpus +} + func (m *manager) configureClaimedVGPU(ctx context.Context, device *devices.VGPUDevice) error { if device.Framework != devices.VGPUFrameworkVendorVFIO { return nil @@ -447,6 +538,16 @@ func withoutVF(vfs []devices.VirtualFunction, address string) []devices.VirtualF return remaining } +func withoutGPU(vfs []devices.VirtualFunction, parentGPU string) []devices.VirtualFunction { + remaining := make([]devices.VirtualFunction, 0, len(vfs)) + for _, vf := range vfs { + if vf.ParentGPU != parentGPU { + remaining = append(remaining, vf) + } + } + return remaining +} + func storedVGPUDevicePath(stored *StoredMetadata) string { if stored.GPUDevicePath != "" { return stored.GPUDevicePath diff --git a/lib/instances/vgpu_linux_test.go b/lib/instances/vgpu_linux_test.go index c93e4b2f5..f21a8278a 100644 --- a/lib/instances/vgpu_linux_test.go +++ b/lib/instances/vgpu_linux_test.go @@ -315,3 +315,95 @@ func TestVGPUClaimFallsBackWhenDirtyVFRefusesReset(t *testing.T) { assert.Equal(t, "0000:82:00.5", device.VFAddress) assert.Equal(t, []string{testVFDevicePath, "/sys/bus/pci/devices/0000:82:00.5"}, resets) } + +// newHiddenProfileManager returns a manager on a two-GPU host whose free VFs +// advertise both profiles until hide4Q is set, after which they advertise +// only 2Q, as the driver does once a GPU holds 12 vGPUs. +func newHiddenProfileManager(t *testing.T, hide4Q *bool) *manager { + t.Helper() + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:e3:00.4", ParentGPU: "0000:e3:00.0"}, + } + return &manager{ + paths: paths.New(t.TempDir()), + discoverVGPU: func() (devices.VGPUFramework, []devices.VirtualFunction, error) { + return devices.VGPUFrameworkVendorVFIO, vfs, nil + }, + vendorVFIOProfiles: func([]devices.VirtualFunction) (map[string][]devices.VGPUProfileType, error) { + profiles := make(map[string][]devices.VGPUProfileType, len(vfs)) + for _, vf := range vfs { + profiles[vf.PCIAddress] = []devices.VGPUProfileType{testSmallProfile} + if !*hide4Q { + profiles[vf.PCIAddress] = append(profiles[vf.PCIAddress], testLargeProfile) + } + } + return profiles, nil + }, + pickVFIndex: pickFirst, + } +} + +func TestVGPUClaimPlacesProfileItsGPUHides(t *testing.T) { + hide4Q := false + m := newHiddenProfileManager(t, &hide4Q) + var configured []string + m.configureVGPU = func(_ context.Context, vfAddress, profileType string) error { + configured = append(configured, vfAddress+"="+profileType) + return nil + } + // An earlier claim on the idle host records that both GPUs support 4Q. + _, err := m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "earlier"), testVGPUProfile) + require.NoError(t, err) + require.Empty(t, configured, "an advertised profile is configured by the caller, not the claim") + + hide4Q = true + meta := saveTestVGPUInstance(t, m, "large") + device, err := m.claimVGPU(context.Background(), meta, testLargeVGPUProfile) + require.NoError(t, err) + assert.Equal(t, "0000:e3:00.4", device.VFAddress) + assert.Equal(t, testLargeProfileType, device.ProfileType) + assert.Equal(t, []string{"0000:e3:00.4=" + testLargeProfileType}, configured) + stored, err := m.loadMetadata("large") + require.NoError(t, err) + assert.Equal(t, "/sys/bus/pci/devices/0000:e3:00.4", stored.GPUDevicePath) +} + +func TestVGPUClaimTriesNextGPUWhenDriverRefusesHiddenProfile(t *testing.T) { + hide4Q := true + m := newHiddenProfileManager(t, &hide4Q) + m.supportedVGPUProfiles = testSupported("0000:82:00.0", "0000:e3:00.0") + var configured []string + m.configureVGPU = func(_ context.Context, vfAddress, profileType string) error { + configured = append(configured, vfAddress) + if vfAddress == "0000:82:00.4" { + return errors.New("configure vGPU on VF 0000:82:00.4: invalid argument") + } + return nil + } + + device, err := m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "large"), testLargeVGPUProfile) + require.NoError(t, err) + assert.Equal(t, "0000:e3:00.4", device.VFAddress) + assert.Equal(t, []string{"0000:82:00.4", "0000:e3:00.4"}, configured, "a refusal rules out the whole GPU, not just the VF") +} + +func TestVGPUClaimFailsWhenDriverRefusesHiddenProfileEverywhere(t *testing.T) { + hide4Q := true + m := newHiddenProfileManager(t, &hide4Q) + m.supportedVGPUProfiles = testSupported("0000:82:00.0", "0000:e3:00.0") + var configured []string + m.configureVGPU = func(_ context.Context, vfAddress, _ string) error { + configured = append(configured, vfAddress) + return errors.New("invalid argument") + } + + _, err := m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "large"), testLargeVGPUProfile) + require.ErrorContains(t, err, "is not creatable on any VF") + require.ErrorContains(t, err, "driver refused it") + assert.Equal(t, []string{"0000:82:00.4", "0000:e3:00.4"}, configured) + stored, loadErr := m.loadMetadata("large") + require.NoError(t, loadErr) + assert.Empty(t, stored.GPUDevicePath) +} diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index be346dee0..93dfc077e 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -241,3 +241,113 @@ func TestSelectVendorVFIOVFPrefersGPUWithKnownLoad(t *testing.T) { require.NoError(t, err) assert.Equal(t, "0000:82:00.5", vf) } + +const ( + testLargeVGPUProfile = "NVIDIA L40S-4Q" + testLargeProfileType = "1150" +) + +var ( + testSmallProfile = devices.VGPUProfileType{TypeName: testVFProfileType, Name: testVGPUProfile, FramebufferMB: 2048} + testLargeProfile = devices.VGPUProfileType{TypeName: testLargeProfileType, Name: testLargeVGPUProfile, FramebufferMB: 4096} +) + +// testSupported reports both profiles as supported on every listed GPU. +func testSupported(gpus ...string) map[string]map[string]devices.VGPUProfileType { + supported := make(map[string]map[string]devices.VGPUProfileType, len(gpus)) + for _, gpu := range gpus { + supported[gpu] = map[string]devices.VGPUProfileType{ + testVGPUProfile: testSmallProfile, + testLargeVGPUProfile: testLargeProfile, + } + } + return supported +} + +func TestSelectVendorVFIOVFOffersSupportedProfileItsGPUHides(t *testing.T) { + // Both GPUs are busy enough that the driver hides 4Q from every + // creatable list, but both have supported it. The less loaded GPU wins. + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0", Allocated: true, ProfileType: testVFProfileType}, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0", Allocated: true, ProfileType: testVFProfileType}, + {PCIAddress: "0000:82:00.6", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:e3:00.4", ParentGPU: "0000:e3:00.0", Allocated: true, ProfileType: testVFProfileType}, + {PCIAddress: "0000:e3:00.5", ParentGPU: "0000:e3:00.0"}, + } + claims := []StoredMetadata{ + {GPUProfile: testVGPUProfile, GPUDevicePath: testVFDevicePath}, + {GPUProfile: testVGPUProfile, GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5"}, + {GPUProfile: testVGPUProfile, GPUDevicePath: "/sys/bus/pci/devices/0000:e3:00.4"}, + } + c := vendorVFIOCandidates{ + vfs: vfs, + profilesByVF: testVFProfiles("0000:82:00.6", "0000:e3:00.5"), + supported: testSupported("0000:82:00.0", "0000:e3:00.0"), + claims: claims, + } + + vf, profileType, err := selectVendorVFIOVF(c, testLargeVGPUProfile, nil) + require.NoError(t, err) + assert.Equal(t, "0000:e3:00.5", vf) + assert.Equal(t, testLargeProfileType, profileType) + + // Without a remembered profile the request fails as before. + c.supported = nil + _, _, err = selectVendorVFIOVF(c, testLargeVGPUProfile, nil) + require.ErrorContains(t, err, "is not creatable on any VF") +} + +func TestSelectVendorVFIOVFPrefersGPUThatAdvertisesProfile(t *testing.T) { + // GPU 82 is idle but hides 4Q in this snapshot; GPU e3 is busier but + // still advertises it. A certain placement beats a lighter load. + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:e3:00.4", ParentGPU: "0000:e3:00.0", Allocated: true, ProfileType: testVFProfileType}, + {PCIAddress: "0000:e3:00.5", ParentGPU: "0000:e3:00.0"}, + } + profiles := testVFProfiles("0000:82:00.4") + profiles["0000:e3:00.5"] = []devices.VGPUProfileType{testSmallProfile, testLargeProfile} + claims := []StoredMetadata{{GPUProfile: testVGPUProfile, GPUDevicePath: "/sys/bus/pci/devices/0000:e3:00.4"}} + + vf, _, err := selectVendorVFIOVF(vendorVFIOCandidates{ + vfs: vfs, + profilesByVF: profiles, + supported: testSupported("0000:82:00.0", "0000:e3:00.0"), + claims: claims, + }, testLargeVGPUProfile, nil) + require.NoError(t, err) + assert.Equal(t, "0000:e3:00.5", vf) +} + +func TestSelectVendorVFIOVFOffersHiddenProfileOnlyWhereSupported(t *testing.T) { + // GPU 82 is idle but has never advertised 4Q, so it is not offered it. + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:e3:00.4", ParentGPU: "0000:e3:00.0", Allocated: true, ProfileType: testVFProfileType}, + {PCIAddress: "0000:e3:00.5", ParentGPU: "0000:e3:00.0"}, + } + vf, _, err := selectVendorVFIOVF(vendorVFIOCandidates{ + vfs: vfs, + profilesByVF: testVFProfiles("0000:82:00.4", "0000:e3:00.5"), + supported: testSupported("0000:e3:00.0"), + claims: []StoredMetadata{{GPUProfile: testVGPUProfile, GPUDevicePath: "/sys/bus/pci/devices/0000:e3:00.4"}}, + }, testLargeVGPUProfile, nil) + require.NoError(t, err) + assert.Equal(t, "0000:e3:00.5", vf) +} + +func TestSelectVendorVFIOVFSkipsGPUThatAdvertisesNothing(t *testing.T) { + // A free VF that advertises no profile at all means the GPU has no + // placement left for any size, so it is not worth asking the driver. + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0", Allocated: true, ProfileType: testVFProfileType}, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0"}, + } + _, _, err := selectVendorVFIOVF(vendorVFIOCandidates{ + vfs: vfs, + profilesByVF: map[string][]devices.VGPUProfileType{"0000:82:00.5": {}}, + supported: testSupported("0000:82:00.0"), + claims: []StoredMetadata{{GPUProfile: testVGPUProfile, GPUDevicePath: testVFDevicePath}}, + }, testLargeVGPUProfile, nil) + require.ErrorContains(t, err, "no creatable vGPU profiles on any VF") +} From d4d32baa5f1d07f54da770402af555719a78e72e Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 14 Oct 2026 02:54:49 +0000 Subject: [PATCH 02/19] Reset dirty VFs and classify driver rejections in hidden vGPU placement Merge the dirty-VF repair and hidden-profile probe into one loop so a VF reselected after the driver refuses a GPU is reset like any other dirty VF, instead of being claimed while it may still hold open VFIO handles. When the driver rejects a hidden profile, reset unclaimed dirty siblings on that GPU and retry once, since one of them may hold the last placement. Only a rejected current_vgpu_type write, now reported as devices.ErrVGPUTypeRejected, moves placement to the next GPU; any other configure error fails the claim. Remember profiles listed after the dirty-VF reset retry, and only offer a hidden profile on a VF whose own creatable list was read and is non-empty. --- lib/devices/GPU.md | 2 +- lib/devices/errors.go | 5 + lib/devices/vendor_vfio_linux.go | 16 ++- lib/devices/vendor_vfio_linux_test.go | 24 ++++ lib/instances/vgpu.go | 97 ++++++++----- lib/instances/vgpu_linux_test.go | 198 +++++++++++++++++++++++++- lib/instances/vgpu_test.go | 51 +++++++ 7 files changed, 349 insertions(+), 44 deletions(-) diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index fb08e770e..0be5354a3 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -106,7 +106,7 @@ before the hypervisor is signalled. Hypeman reconciles metadata claims once at startup and every minute afterward, skipping hosts without GPUs. A claim whose VMM is confirmed dead is reset before the claim is removed. An ambiguous hypervisor ownership check preserves the claim, logs a warning, and increments `hypeman_instances_vgpu_liveness_uncertain_total`; the create and start cleanup paths preserve and count the same way. mdev hosts also sweep orphaned device-level assignments. Vendor VFIO hosts repair an unclaimed dirty VF when the allocator next selects it; repair checks for open VFIO handles before resetting `current_vgpu_type`. The allocator prefers VFs that are already clean, and a dirty VF that refuses its reset is skipped in favor of another candidate. -In mixed-size (heterogeneous) mode, `creatable_vgpu_types` under-reports. The driver drops a profile from every VF's list once its GPU holds as many vGPUs as that profile's equal-size instance limit (12 for L40S-4Q), even while a placement for it remains, and still accepts a `current_vgpu_type` write for it. The allocator remembers every profile each GPU has advertised. When no GPU advertises the requested profile, it writes the profile to a clean VF on each GPU that supports it, under the allocation lock, until the driver accepts one. GPUs that advertise the profile are always tried first. A GPU whose VFs advertise nothing has no placement left for any size and is skipped. +In mixed-size (heterogeneous) mode, `creatable_vgpu_types` under-reports. The driver drops a profile from every VF's list once its GPU holds as many vGPUs as that profile's equal-size instance limit (12 for L40S-4Q), even while a placement for it remains, and still accepts a `current_vgpu_type` write for it. The allocator remembers every profile each GPU has advertised. When no GPU advertises the requested profile, it writes the profile to a clean VF on each GPU that supports it, under the allocation lock, until the driver accepts one. If the driver rejects the write, unclaimed dirty VFs on that GPU are reset and the write is retried once before the next GPU is tried; any other configure error fails the claim. GPUs that advertise the profile are always tried first. A free VF that advertises nothing is skipped (on L40S its GPU has no placement left for any size), as is one whose list cannot be read. ### Hypervisor Support diff --git a/lib/devices/errors.go b/lib/devices/errors.go index 15d68dab6..5b88afe5e 100644 --- a/lib/devices/errors.go +++ b/lib/devices/errors.go @@ -38,4 +38,9 @@ var ( // ErrVGPUNotSupportedOnMacOS is returned for vGPU operations on macOS ErrVGPUNotSupportedOnMacOS = errors.New("vGPU (mdev) is not supported on macOS") + + // ErrVGPUTypeRejected is returned when the driver refuses a vendor VFIO + // current_vgpu_type write, which is how it reports that the VF's GPU has + // no placement left for the type + ErrVGPUTypeRejected = errors.New("driver rejected vGPU type") ) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 2fb552095..d286ce87e 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -27,6 +27,7 @@ type vendorVFIOSysfs struct { procPath string vfioDevicesPath string openVFIOPathsFunc func() (map[string]struct{}, error) + writeFileFunc func(name string, data []byte, perm os.FileMode) error } var hostVendorVFIO = vendorVFIOSysfs{ @@ -188,7 +189,7 @@ func (s vendorVFIOSysfs) configure(ctx context.Context, vfAddress, profileType s if inUse { return fmt.Errorf("vendor VFIO vGPU on VF %s is still in use", vfAddress) } - if err := os.WriteFile(currentTypePath, []byte("0"), 0200); err != nil { + if err := s.writeFile(currentTypePath, []byte("0"), 0200); err != nil { return fmt.Errorf("reset vGPU on VF %s: %w", vfAddress, err) } currentType, err = readCurrentVGPUType(currentTypePath) @@ -199,8 +200,8 @@ func (s vendorVFIOSysfs) configure(ctx context.Context, vfAddress, profileType s return fmt.Errorf("verify reset vGPU on VF %s: type is %s, want 0", vfAddress, currentType) } } - if err := os.WriteFile(currentTypePath, []byte(profileType), 0200); err != nil { - return fmt.Errorf("configure vGPU on VF %s: %w", vfAddress, err) + if err := s.writeFile(currentTypePath, []byte(profileType), 0200); err != nil { + return fmt.Errorf("configure vGPU on VF %s: %w: %w", vfAddress, ErrVGPUTypeRejected, err) } currentType, err = readCurrentVGPUType(currentTypePath) if err != nil { @@ -242,13 +243,20 @@ func (s vendorVFIOSysfs) destroy(ctx context.Context, vfAddress, instanceID stri return fmt.Errorf("vendor VFIO vGPU on VF %s is still in use", vfAddress) } - if err := os.WriteFile(currentTypePath, []byte("0"), 0200); err != nil { + if err := s.writeFile(currentTypePath, []byte("0"), 0200); err != nil { return fmt.Errorf("destroy vGPU on VF %s: %w", vfAddress, err) } log.InfoContext(ctx, "destroyed vendor VFIO vGPU", "vf", vfAddress, "instance_id", instanceID) return nil } +func (s vendorVFIOSysfs) writeFile(name string, data []byte, perm os.FileMode) error { + if s.writeFileFunc != nil { + return s.writeFileFunc(name, data, perm) + } + return os.WriteFile(name, data, perm) +} + func (s vendorVFIOSysfs) readCreatableProfiles(vfAddress string) ([]profileMetadata, error) { path := filepath.Join(s.pciDevicesPath, vfAddress, "nvidia", "creatable_vgpu_types") data, err := os.ReadFile(path) diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 3c7438532..9f2983f9f 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -6,6 +6,7 @@ import ( "context" "os" "path/filepath" + "syscall" "testing" "github.com/stretchr/testify/assert" @@ -64,6 +65,29 @@ func TestVendorVFIOConfigure(t *testing.T) { require.NoError(t, sysfs.configure(context.Background(), vfAddress, "1148"), "configuration must be idempotent") } +func TestVendorVFIOConfigureReportsDriverRejection(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1159", testCreatableTypes) + sysfs.writeFileFunc = func(name string, data []byte, perm os.FileMode) error { + if string(data) != "0" { + return syscall.EINVAL + } + return os.WriteFile(name, data, perm) + } + + err := sysfs.configure(context.Background(), vfAddress, "1148") + require.ErrorIs(t, err, ErrVGPUTypeRejected) + require.ErrorIs(t, err, syscall.EINVAL) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "0") + + sysfs.writeFileFunc = func(string, []byte, os.FileMode) error { return syscall.EIO } + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "1159", testCreatableTypes) + err = sysfs.configure(context.Background(), "0000:82:00.5", "1148") + require.ErrorContains(t, err, "reset vGPU on VF 0000:82:00.5") + assert.NotErrorIs(t, err, ErrVGPUTypeRejected, "a failed reset is not a placement answer") +} + func TestVendorVFIOConfigureRepairsDirtyVF(t *testing.T) { sysfs := newTestVendorVFIOSysfs(t) const vfAddress = "0000:82:00.4" diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 16a8236f0..fbfd5d63f 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -2,6 +2,7 @@ package instances import ( "context" + "errors" "fmt" "math/rand/v2" "path/filepath" @@ -80,6 +81,7 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str if candidates.profilesByVF, err = listProfiles(vfs); err != nil { return nil, err } + m.rememberVGPUProfiles(vfs, candidates.profilesByVF) candidates.vfs = vfs if vfAddress, profileType, err = selectVendorVFIOVF(candidates, profileName, m.pickVFIndex); err != nil { return nil, err @@ -91,46 +93,43 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str // claiming; the allocation lock keeps it unclaimed until then. A VF that // refuses its reset is dropped from the candidates so it cannot block // clean siblings. + // + // When no VF on the chosen GPU advertises the profile, only the driver can + // say whether a placement remains, so the VF is configured here and the + // next GPU is tried if the driver rejects the type. Every reselection + // passes through the dirty-VF reset again. log := logger.FromContext(ctx) for { vf, ok := vfByAddress(candidates.vfs, vfAddress) - if !ok || !vf.Allocated { + if !ok { break } - assignment := devices.VGPUAssignment{ - Framework: devices.VGPUFrameworkVendorVFIO, - DevicePath: filepath.Clean(devices.GetDeviceSysfsPath(vf.PCIAddress)), - } - repairErr := m.destroyVGPUAssignment(ctx, assignment) - if repairErr == nil { + if vf.Allocated { + assignment := devices.VGPUAssignment{ + Framework: devices.VGPUFrameworkVendorVFIO, + DevicePath: filepath.Clean(devices.GetDeviceSysfsPath(vf.PCIAddress)), + } + if repairErr := m.destroyVGPUAssignment(ctx, assignment); repairErr != nil { + log.WarnContext(ctx, "dirty vGPU VF refused reset; trying another VF", "vf", vf.PCIAddress, "error", repairErr) + candidates.vfs = withoutVF(candidates.vfs, vf.PCIAddress) + if vfAddress, profileType, err = selectVendorVFIOVF(candidates, profileName, m.pickVFIndex); err != nil { + return nil, fmt.Errorf("repair dirty VF %s before claim: %w", vf.PCIAddress, repairErr) + } + continue + } log.InfoContext(ctx, "reset dirty vGPU VF before claim", "vf", vf.PCIAddress) - break } - log.WarnContext(ctx, "dirty vGPU VF refused reset; trying another VF", "vf", vf.PCIAddress, "error", repairErr) - candidates.vfs = withoutVF(candidates.vfs, vf.PCIAddress) - if vfAddress, profileType, err = selectVendorVFIOVF(candidates, profileName, m.pickVFIndex); err != nil { - return nil, fmt.Errorf("repair dirty VF %s before claim: %w", vf.PCIAddress, repairErr) - } - } - // No VF on the chosen GPU advertises the profile, so only the driver can - // say whether a placement remains. Configure it here, under the - // allocation lock, and move to the next GPU if the driver refuses. A VF - // left configured by a failed claim below is an unclaimed dirty VF, - // which the next allocation repairs. - for { - vf, ok := vfByAddress(candidates.vfs, vfAddress) - if !ok || advertisingGPUs(candidates, profileType)[vf.ParentGPU] { + if advertisingGPUs(candidates, profileType)[vf.ParentGPU] { break } - probeErr := m.configureClaimedVGPU(ctx, &devices.VGPUDevice{ - Framework: devices.VGPUFrameworkVendorVFIO, - VFAddress: vfAddress, - ProfileType: profileType, - }) + probeErr := m.placeHiddenVGPU(ctx, candidates.vfs, allMetadata, vf, profileType) if probeErr == nil { log.InfoContext(ctx, "placed vGPU profile its GPU no longer advertises", "vf", vfAddress, "profile", profileName) break } + if !errors.Is(probeErr, devices.ErrVGPUTypeRejected) { + return nil, probeErr + } log.InfoContext(ctx, "driver refused vGPU profile on GPU; trying another GPU", "gpu", vf.ParentGPU, "vf", vfAddress, "profile", profileName, "error", probeErr) candidates.vfs = withoutGPU(candidates.vfs, vf.ParentGPU) if vfAddress, profileType, err = selectVendorVFIOVF(candidates, profileName, m.pickVFIndex); err != nil { @@ -191,6 +190,25 @@ func (m *manager) resetDirtyUnclaimedVFs(ctx context.Context, vfs []devices.Virt return reset } +// placeHiddenVGPU configures vf with a profile its GPU no longer advertises, +// so the driver decides whether a placement remains. An unclaimed dirty +// sibling can hold the last placement, so when the driver rejects the type, +// those siblings are reset (in-use gated) and the write is retried once. A VF +// left configured by a claim that fails afterwards is such a sibling for the +// next allocation. Callers must hold vgpuAllocationMu. +func (m *manager) placeHiddenVGPU(ctx context.Context, vfs []devices.VirtualFunction, allMetadata []StoredMetadata, vf devices.VirtualFunction, profileType string) error { + device := &devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + VFAddress: vf.PCIAddress, + ProfileType: profileType, + } + err := m.configureClaimedVGPU(ctx, device) + if errors.Is(err, devices.ErrVGPUTypeRejected) && m.resetDirtyUnclaimedVFs(ctx, siblingVFs(vfs, vf), allMetadata) { + err = m.configureClaimedVGPU(ctx, device) + } + return err +} + // rememberVGPUProfiles records every profile each GPU's VFs advertise. In // mixed-size mode the driver drops a profile from every creatable list on a // GPU once that GPU holds as many vGPUs as the profile's equal-size instance @@ -239,9 +257,10 @@ type vendorVFIOCandidates struct { // A GPU that supports the profile but no longer advertises it is still a // candidate for a clean VF, ranked after every GPU that does advertise it, // because the driver hides profiles by instance count rather than by free -// placement. The caller must confirm such a choice with the driver. A GPU -// whose VFs advertise nothing at all has no placement left for any size and -// is not a candidate. +// placement. The caller must confirm such a choice with the driver. A free +// VF that advertises nothing is not a candidate for this: on L40S an empty +// list means its GPU has no placement left for any size, and a VF whose list +// could not be read is unknown. func selectVendorVFIOVF(c vendorVFIOCandidates, profileName string, pick func(n int) int) (string, string, error) { profilesByName := make(map[string]devices.VGPUProfileType) for _, byName := range c.supported { @@ -297,12 +316,6 @@ func selectVendorVFIOVF(c vendorVFIOCandidates, profileName string, pick func(n } parentAdvertises := advertisingGPUs(c, requested.TypeName) - parentAdvertisesAny := make(map[string]bool) - for _, vf := range c.vfs { - if len(advertises[vf.PCIAddress]) > 0 { - parentAdvertisesAny[vf.ParentGPU] = true - } - } quarantinedByGPU := make(map[string]int) freeByGPU := make(map[string][]devices.VirtualFunction) for _, vf := range c.vfs { @@ -316,7 +329,7 @@ func selectVendorVFIOVF(c vendorVFIOCandidates, profileName string, pick func(n _, directlyCreatable := advertises[vf.PCIAddress][requested.TypeName] repairable := vf.Allocated && (vf.ProfileType == requested.TypeName || parentAdvertises[vf.ParentGPU]) supported, ok := c.supported[vf.ParentGPU][profileName] - hidden := !vf.Allocated && !parentAdvertises[vf.ParentGPU] && parentAdvertisesAny[vf.ParentGPU] && + hidden := !vf.Allocated && !parentAdvertises[vf.ParentGPU] && len(advertises[vf.PCIAddress]) > 0 && ok && supported.TypeName == requested.TypeName if directlyCreatable || repairable || hidden { freeByGPU[vf.ParentGPU] = append(freeByGPU[vf.ParentGPU], vf) @@ -538,6 +551,16 @@ func withoutVF(vfs []devices.VirtualFunction, address string) []devices.VirtualF return remaining } +func siblingVFs(vfs []devices.VirtualFunction, vf devices.VirtualFunction) []devices.VirtualFunction { + siblings := make([]devices.VirtualFunction, 0, len(vfs)) + for _, other := range vfs { + if other.ParentGPU == vf.ParentGPU && other.PCIAddress != vf.PCIAddress { + siblings = append(siblings, other) + } + } + return siblings +} + func withoutGPU(vfs []devices.VirtualFunction, parentGPU string) []devices.VirtualFunction { remaining := make([]devices.VirtualFunction, 0, len(vfs)) for _, vf := range vfs { diff --git a/lib/instances/vgpu_linux_test.go b/lib/instances/vgpu_linux_test.go index f21a8278a..614f04f27 100644 --- a/lib/instances/vgpu_linux_test.go +++ b/lib/instances/vgpu_linux_test.go @@ -5,6 +5,7 @@ package instances import ( "context" "errors" + "fmt" "sync" "testing" "time" @@ -378,7 +379,7 @@ func TestVGPUClaimTriesNextGPUWhenDriverRefusesHiddenProfile(t *testing.T) { m.configureVGPU = func(_ context.Context, vfAddress, profileType string) error { configured = append(configured, vfAddress) if vfAddress == "0000:82:00.4" { - return errors.New("configure vGPU on VF 0000:82:00.4: invalid argument") + return errDriverRejected(vfAddress) } return nil } @@ -396,7 +397,7 @@ func TestVGPUClaimFailsWhenDriverRefusesHiddenProfileEverywhere(t *testing.T) { var configured []string m.configureVGPU = func(_ context.Context, vfAddress, _ string) error { configured = append(configured, vfAddress) - return errors.New("invalid argument") + return errDriverRejected(vfAddress) } _, err := m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "large"), testLargeVGPUProfile) @@ -407,3 +408,196 @@ func TestVGPUClaimFailsWhenDriverRefusesHiddenProfileEverywhere(t *testing.T) { require.NoError(t, loadErr) assert.Empty(t, stored.GPUDevicePath) } + +func TestVGPUClaimFailsOnHiddenProfileConfigureErrorOtherThanRejection(t *testing.T) { + hide4Q := true + m := newHiddenProfileManager(t, &hide4Q) + m.supportedVGPUProfiles = testSupported("0000:82:00.0", "0000:e3:00.0") + var configured []string + m.configureVGPU = func(_ context.Context, vfAddress, _ string) error { + configured = append(configured, vfAddress) + return errors.New("VF health state unavailable: read failed") + } + + _, err := m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "large"), testLargeVGPUProfile) + require.ErrorContains(t, err, "VF health state unavailable") + assert.NotContains(t, err.Error(), "not creatable", "only a driver rejection says the GPU is out of placements") + assert.Equal(t, []string{"0000:82:00.4"}, configured) +} + +func TestVGPUClaimResetsDirtyVFSelectedAfterDriverRefusal(t *testing.T) { + // GPU 82 refuses the hidden 4Q, so the claim moves to GPU e3, whose only + // free VF is dirty with a leftover 4Q and still open. It must go through + // the reset like any other dirty VF instead of being claimed as is. + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:e3:00.4", ParentGPU: "0000:e3:00.0", Allocated: true, ProfileType: testLargeProfileType}, + } + var events []string + m := &manager{ + paths: paths.New(t.TempDir()), + discoverVGPU: func() (devices.VGPUFramework, []devices.VirtualFunction, error) { + return devices.VGPUFrameworkVendorVFIO, vfs, nil + }, + vendorVFIOProfiles: func([]devices.VirtualFunction) (map[string][]devices.VGPUProfileType, error) { + return map[string][]devices.VGPUProfileType{ + "0000:82:00.4": {testSmallProfile}, + "0000:e3:00.4": {}, + }, nil + }, + supportedVGPUProfiles: testSupported("0000:82:00.0", "0000:e3:00.0"), + destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { + events = append(events, "reset "+assignment.DevicePath) + return errors.New("vendor VFIO vGPU on VF 0000:e3:00.4 is still in use") + }, + configureVGPU: func(_ context.Context, vfAddress, _ string) error { + events = append(events, "configure "+vfAddress) + if vfAddress == "0000:82:00.4" { + return errDriverRejected(vfAddress) + } + return nil + }, + pickVFIndex: pickFirst, + } + + _, err := m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "large"), testLargeVGPUProfile) + require.ErrorContains(t, err, "repair dirty VF 0000:e3:00.4 before claim") + assert.Equal(t, []string{"configure 0000:82:00.4", "reset /sys/bus/pci/devices/0000:e3:00.4"}, events) + stored, loadErr := m.loadMetadata("large") + require.NoError(t, loadErr) + assert.Empty(t, stored.GPUDevicePath) +} + +func TestVGPUClaimResetsDirtySiblingHoldingHiddenPlacement(t *testing.T) { + // GPU 82 hides 4Q, and its last 4Q placement is held by an unclaimed + // dirty VF, for example one configured by a claim that failed to save. + // The clean sibling is tried first; when the driver refuses it, the dirty + // VF is reset and the write retried rather than giving up on the GPU. + dirty := true + m := &manager{ + paths: paths.New(t.TempDir()), + discoverVGPU: func() (devices.VGPUFramework, []devices.VirtualFunction, error) { + leftover := devices.VirtualFunction{PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"} + if dirty { + leftover.Allocated, leftover.ProfileType = true, testLargeProfileType + } + return devices.VGPUFrameworkVendorVFIO, []devices.VirtualFunction{ + leftover, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0"}, + }, nil + }, + vendorVFIOProfiles: func([]devices.VirtualFunction) (map[string][]devices.VGPUProfileType, error) { + return map[string][]devices.VGPUProfileType{ + "0000:82:00.4": {}, + "0000:82:00.5": {testSmallProfile}, + }, nil + }, + supportedVGPUProfiles: testSupported("0000:82:00.0"), + pickVFIndex: pickFirst, + } + var events []string + m.destroyVGPU = func(_ context.Context, assignment devices.VGPUAssignment) error { + events = append(events, "reset "+assignment.DevicePath) + dirty = false + return nil + } + m.configureVGPU = func(_ context.Context, vfAddress, _ string) error { + events = append(events, "configure "+vfAddress) + if dirty { + return errDriverRejected(vfAddress) + } + return nil + } + + device, err := m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "large"), testLargeVGPUProfile) + require.NoError(t, err) + assert.Equal(t, "0000:82:00.5", device.VFAddress) + assert.Equal(t, []string{ + "configure 0000:82:00.5", + "reset /sys/bus/pci/devices/0000:82:00.4", + "configure 0000:82:00.5", + }, events) +} + +func TestVGPUClaimProbesRepairedVFOnceWhenDriverRefuses(t *testing.T) { + // The only candidate is a dirty VF left at 4Q on a GPU that hides 4Q. It + // is reset, then probed. A refusal must not count that same VF as a dirty + // sibling to reset and retry. + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0", Allocated: true, ProfileType: testLargeProfileType}, + } + var events []string + m := &manager{ + paths: paths.New(t.TempDir()), + discoverVGPU: func() (devices.VGPUFramework, []devices.VirtualFunction, error) { + return devices.VGPUFrameworkVendorVFIO, vfs, nil + }, + vendorVFIOProfiles: func([]devices.VirtualFunction) (map[string][]devices.VGPUProfileType, error) { + return map[string][]devices.VGPUProfileType{"0000:82:00.4": {}}, nil + }, + supportedVGPUProfiles: testSupported("0000:82:00.0"), + destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { + events = append(events, "reset "+assignment.DevicePath) + return nil + }, + configureVGPU: func(_ context.Context, vfAddress, _ string) error { + events = append(events, "configure "+vfAddress) + return errDriverRejected(vfAddress) + }, + } + + _, err := m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "large"), testLargeVGPUProfile) + require.ErrorContains(t, err, "driver refused it on GPU 0000:82:00.0") + assert.Equal(t, []string{"reset " + testVFDevicePath, "configure 0000:82:00.4"}, events) +} + +func TestVGPUClaimRemembersProfilesListedAfterDirtyReset(t *testing.T) { + // The first claim only sees the GPU's profiles after resetting a dirty + // VF that consumed its framebuffer. 4Q must still be remembered from that + // listing once the GPU later hides it. + dirty := true + hide4Q := false + m := &manager{ + paths: paths.New(t.TempDir()), + discoverVGPU: func() (devices.VGPUFramework, []devices.VirtualFunction, error) { + leftover := devices.VirtualFunction{PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"} + if dirty { + leftover.Allocated, leftover.ProfileType = true, "999" + } + return devices.VGPUFrameworkVendorVFIO, []devices.VirtualFunction{ + leftover, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0"}, + }, nil + }, + vendorVFIOProfiles: func(vfs []devices.VirtualFunction) (map[string][]devices.VGPUProfileType, error) { + profiles := make(map[string][]devices.VGPUProfileType, len(vfs)) + for _, vf := range vfs { + switch { + case dirty: + profiles[vf.PCIAddress] = []devices.VGPUProfileType{} + case hide4Q: + profiles[vf.PCIAddress] = []devices.VGPUProfileType{testSmallProfile} + default: + profiles[vf.PCIAddress] = []devices.VGPUProfileType{testSmallProfile, testLargeProfile} + } + } + return profiles, nil + }, + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + dirty = false + return nil + }, + configureVGPU: func(context.Context, string, string) error { return nil }, + pickVFIndex: pickFirst, + } + _, err := m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "small"), testVGPUProfile) + require.NoError(t, err) + + hide4Q = true + _, err = m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "large"), testLargeVGPUProfile) + require.NoError(t, err) +} + +func errDriverRejected(vfAddress string) error { + return fmt.Errorf("configure vGPU on VF %s: %w: invalid argument", vfAddress, devices.ErrVGPUTypeRejected) +} diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 93dfc077e..893e311ae 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -336,6 +336,57 @@ func TestSelectVendorVFIOVFOffersHiddenProfileOnlyWhereSupported(t *testing.T) { assert.Equal(t, "0000:e3:00.5", vf) } +func TestSelectVendorVFIOVFSkipsUnreadableVFForHiddenProfile(t *testing.T) { + // VF .4's creatable types could not be read, so it is absent from the + // listing. Its readable sibling says the GPU still has placements, but + // that says nothing about the unreadable VF itself. + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0"}, + } + vf, _, err := selectVendorVFIOVF(vendorVFIOCandidates{ + vfs: vfs, + profilesByVF: testVFProfiles("0000:82:00.5"), + supported: testSupported("0000:82:00.0"), + }, testLargeVGPUProfile, pickFirst) + require.NoError(t, err) + assert.Equal(t, "0000:82:00.5", vf) +} + +func TestSelectVendorVFIOVFCountsHiddenProfileClaimsAsKnownLoad(t *testing.T) { + // GPU 82 runs one 4Q that no VF advertises any more; GPU e3 runs three + // 2Q. The remembered 4Q makes 82's load known and lighter (4 GB vs 6 GB), + // instead of ranking 82 last for unknown load. + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0", Allocated: true, ProfileType: testLargeProfileType}, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:e3:00.4", ParentGPU: "0000:e3:00.0", Allocated: true, ProfileType: testVFProfileType}, + {PCIAddress: "0000:e3:00.5", ParentGPU: "0000:e3:00.0", Allocated: true, ProfileType: testVFProfileType}, + {PCIAddress: "0000:e3:00.6", ParentGPU: "0000:e3:00.0", Allocated: true, ProfileType: testVFProfileType}, + {PCIAddress: "0000:e3:00.7", ParentGPU: "0000:e3:00.0"}, + } + c := vendorVFIOCandidates{ + vfs: vfs, + profilesByVF: testVFProfiles("0000:82:00.5", "0000:e3:00.7"), + supported: testSupported("0000:82:00.0", "0000:e3:00.0"), + claims: []StoredMetadata{ + {GPUProfile: testLargeVGPUProfile, GPUDevicePath: testVFDevicePath}, + {GPUProfile: testVGPUProfile, GPUDevicePath: "/sys/bus/pci/devices/0000:e3:00.4"}, + {GPUProfile: testVGPUProfile, GPUDevicePath: "/sys/bus/pci/devices/0000:e3:00.5"}, + {GPUProfile: testVGPUProfile, GPUDevicePath: "/sys/bus/pci/devices/0000:e3:00.6"}, + }, + } + + vf, _, err := selectVendorVFIOVF(c, testVGPUProfile, nil) + require.NoError(t, err) + assert.Equal(t, "0000:82:00.5", vf) + + c.supported = nil + vf, _, err = selectVendorVFIOVF(c, testVGPUProfile, nil) + require.NoError(t, err) + assert.Equal(t, "0000:e3:00.7", vf, "without the remembered profile 82's load is unknown") +} + func TestSelectVendorVFIOVFSkipsGPUThatAdvertisesNothing(t *testing.T) { // A free VF that advertises no profile at all means the GPU has no // placement left for any size, so it is not worth asking the driver. From 67ccc4e227996769ea2f85a1b98ada5364297a00 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:30:05 +0000 Subject: [PATCH 03/19] Configure every vendor VFIO claim under the allocation lock Advertised profiles were configured after claimVGPU released the lock, so a later claim could take the placement a saved claim had picked, and its configure then failed with no fallback. Write the type for every claim inside the claim loop, so a driver rejection moves to the next GPU whether or not the profile is advertised. Reset the VF if saving the claim fails, since it is now configured before the claim exists. --- lib/devices/GPU.md | 6 ++-- lib/instances/vgpu.go | 46 +++++++++++++----------- lib/instances/vgpu_linux_test.go | 60 +++++++++++++++++++++++++++++++- 3 files changed, 87 insertions(+), 25 deletions(-) diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 0be5354a3..28646ae50 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -92,10 +92,10 @@ On an mdev host, the response also includes the assigned mdev UUID: ### Ephemeral vGPU Lifecycle -vGPU assignments are created on instance start and released on stop or delete. Hypeman creates/removes an mdev on mdev hosts. On vendor VFIO hosts, it first reserves a VF by persisting its path in instance metadata, then writes the profile ID to `current_vgpu_type`. Metadata is the authoritative VF claim. +vGPU assignments are created on instance start and released on stop or delete. Hypeman creates/removes an mdev on mdev hosts. On vendor VFIO hosts, it selects a VF and writes the profile ID to `current_vgpu_type` under the allocation lock, so the driver confirms a placement remains before any later claim can take it, then reserves the VF by persisting its path in instance metadata. Metadata is the authoritative VF claim; a configured VF without one is dirty and is reset by the allocator. ``` -Instance Create → Persist VF claim → Configure profile → Attach VF to VM → Instance Running +Instance Create → Configure profile → Persist VF claim → Attach VF to VM → Instance Running Instance Stop/Delete → Reset profile → Remove VF claim → VF available again ``` @@ -106,7 +106,7 @@ before the hypervisor is signalled. Hypeman reconciles metadata claims once at startup and every minute afterward, skipping hosts without GPUs. A claim whose VMM is confirmed dead is reset before the claim is removed. An ambiguous hypervisor ownership check preserves the claim, logs a warning, and increments `hypeman_instances_vgpu_liveness_uncertain_total`; the create and start cleanup paths preserve and count the same way. mdev hosts also sweep orphaned device-level assignments. Vendor VFIO hosts repair an unclaimed dirty VF when the allocator next selects it; repair checks for open VFIO handles before resetting `current_vgpu_type`. The allocator prefers VFs that are already clean, and a dirty VF that refuses its reset is skipped in favor of another candidate. -In mixed-size (heterogeneous) mode, `creatable_vgpu_types` under-reports. The driver drops a profile from every VF's list once its GPU holds as many vGPUs as that profile's equal-size instance limit (12 for L40S-4Q), even while a placement for it remains, and still accepts a `current_vgpu_type` write for it. The allocator remembers every profile each GPU has advertised. When no GPU advertises the requested profile, it writes the profile to a clean VF on each GPU that supports it, under the allocation lock, until the driver accepts one. If the driver rejects the write, unclaimed dirty VFs on that GPU are reset and the write is retried once before the next GPU is tried; any other configure error fails the claim. GPUs that advertise the profile are always tried first. A free VF that advertises nothing is skipped (on L40S its GPU has no placement left for any size), as is one whose list cannot be read. +In mixed-size (heterogeneous) mode, `creatable_vgpu_types` under-reports. The driver drops a profile from every VF's list once its GPU holds as many vGPUs as that profile's equal-size instance limit (12 for L40S-4Q), even while a placement for it remains, and still accepts a `current_vgpu_type` write for it. The allocator remembers every profile each GPU has advertised, and a clean VF on a GPU that supports the requested profile but no longer advertises it stays a candidate, ranked after every GPU with a free VF that advertises it. If the driver rejects the claim's write, unclaimed dirty VFs on that GPU are reset and the write is retried once before the next GPU is tried; any other configure error fails the claim. A free VF that advertises nothing is skipped (on L40S its GPU has no placement left for any size), as is one whose list cannot be read. This assumes every GPU is in mixed-size mode (`nvidia-smi vgpu -i -shm 1`): in equal-size mode a GPU running a smaller profile hides larger ones for good, so each request for one costs a refused write on that GPU. ### Hypervisor Support diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index fbfd5d63f..fa503cd8a 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -94,10 +94,12 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str // refuses its reset is dropped from the candidates so it cannot block // clean siblings. // - // When no VF on the chosen GPU advertises the profile, only the driver can - // say whether a placement remains, so the VF is configured here and the - // next GPU is tried if the driver rejects the type. Every reselection - // passes through the dirty-VF reset again. + // The VF is then configured here, under the allocation lock, so a claim + // cannot lose its placement to a later claim once the lock is released. + // Creatable lists are a snapshot and hide some profiles that still fit, + // so only the driver can say whether a placement remains; the next GPU is + // tried if it rejects the type. Every reselection passes through the + // dirty-VF reset again. log := logger.FromContext(ctx) for { vf, ok := vfByAddress(candidates.vfs, vfAddress) @@ -119,21 +121,20 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str } log.InfoContext(ctx, "reset dirty vGPU VF before claim", "vf", vf.PCIAddress) } - if advertisingGPUs(candidates, profileType)[vf.ParentGPU] { - break - } - probeErr := m.placeHiddenVGPU(ctx, candidates.vfs, allMetadata, vf, profileType) - if probeErr == nil { - log.InfoContext(ctx, "placed vGPU profile its GPU no longer advertises", "vf", vfAddress, "profile", profileName) + placeErr := m.placeVGPU(ctx, candidates.vfs, allMetadata, vf, profileType) + if placeErr == nil { + if !advertisingGPUs(candidates, profileType)[vf.ParentGPU] { + log.InfoContext(ctx, "placed vGPU profile its GPU no longer advertises", "vf", vfAddress, "profile", profileName) + } break } - if !errors.Is(probeErr, devices.ErrVGPUTypeRejected) { - return nil, probeErr + if !errors.Is(placeErr, devices.ErrVGPUTypeRejected) { + return nil, placeErr } - log.InfoContext(ctx, "driver refused vGPU profile on GPU; trying another GPU", "gpu", vf.ParentGPU, "vf", vfAddress, "profile", profileName, "error", probeErr) + log.InfoContext(ctx, "driver refused vGPU profile on GPU; trying another GPU", "gpu", vf.ParentGPU, "vf", vfAddress, "profile", profileName, "error", placeErr) candidates.vfs = withoutGPU(candidates.vfs, vf.ParentGPU) if vfAddress, profileType, err = selectVendorVFIOVF(candidates, profileName, m.pickVFIndex); err != nil { - return nil, fmt.Errorf("%w (driver refused it on GPU %s: %v)", err, vf.ParentGPU, probeErr) + return nil, fmt.Errorf("%w (driver refused it on GPU %s: %v)", err, vf.ParentGPU, placeErr) } } device := &devices.VGPUDevice{ @@ -151,6 +152,9 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str meta.GPUClaimedAt = &claimedAt if err := m.saveMetadata(meta); err != nil { clearStoredVGPUDevice(&meta.StoredMetadata) + if resetErr := m.destroyVGPUAssignment(ctx, devices.VGPUAssignment{Framework: device.Framework, DevicePath: device.SysfsPath}); resetErr != nil { + log.WarnContext(ctx, "failed to reset vGPU VF after claim save failed", "vf", vfAddress, "error", resetErr) + } return nil, fmt.Errorf("save vGPU claim: %w", err) } return device, nil @@ -190,13 +194,13 @@ func (m *manager) resetDirtyUnclaimedVFs(ctx context.Context, vfs []devices.Virt return reset } -// placeHiddenVGPU configures vf with a profile its GPU no longer advertises, -// so the driver decides whether a placement remains. An unclaimed dirty -// sibling can hold the last placement, so when the driver rejects the type, -// those siblings are reset (in-use gated) and the write is retried once. A VF -// left configured by a claim that fails afterwards is such a sibling for the -// next allocation. Callers must hold vgpuAllocationMu. -func (m *manager) placeHiddenVGPU(ctx context.Context, vfs []devices.VirtualFunction, allMetadata []StoredMetadata, vf devices.VirtualFunction, profileType string) error { +// placeVGPU configures vf with profileType, so the driver decides whether a +// placement remains. An unclaimed dirty sibling can hold the last placement, +// so when the driver rejects the type, those siblings are reset (in-use +// gated) and the write is retried once. A VF left configured by a crash before +// its claim was saved is such a sibling for the next allocation. Callers must +// hold vgpuAllocationMu. +func (m *manager) placeVGPU(ctx context.Context, vfs []devices.VirtualFunction, allMetadata []StoredMetadata, vf devices.VirtualFunction, profileType string) error { device := &devices.VGPUDevice{ Framework: devices.VGPUFrameworkVendorVFIO, VFAddress: vf.PCIAddress, diff --git a/lib/instances/vgpu_linux_test.go b/lib/instances/vgpu_linux_test.go index 614f04f27..645c2e61e 100644 --- a/lib/instances/vgpu_linux_test.go +++ b/lib/instances/vgpu_linux_test.go @@ -6,6 +6,7 @@ import ( "context" "errors" "fmt" + "os" "sync" "testing" "time" @@ -34,6 +35,7 @@ func newVGPUAllocationManager(t *testing.T, vfs []devices.VirtualFunction) *mana } return profiles, nil }, + configureVGPU: func(context.Context, string, string) error { return nil }, } } @@ -214,6 +216,7 @@ func TestVGPUClaimRepairsDirtyVFWhenProfileNotAdvertised(t *testing.T) { dirty = false return nil }, + configureVGPU: func(context.Context, string, string) error { return nil }, } meta := saveTestVGPUInstance(t, m, "new") @@ -357,9 +360,10 @@ func TestVGPUClaimPlacesProfileItsGPUHides(t *testing.T) { // An earlier claim on the idle host records that both GPUs support 4Q. _, err := m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "earlier"), testVGPUProfile) require.NoError(t, err) - require.Empty(t, configured, "an advertised profile is configured by the caller, not the claim") + require.Equal(t, []string{"0000:82:00.4=" + testVFProfileType}, configured) hide4Q = true + configured = nil meta := saveTestVGPUInstance(t, m, "large") device, err := m.claimVGPU(context.Background(), meta, testLargeVGPUProfile) require.NoError(t, err) @@ -598,6 +602,60 @@ func TestVGPUClaimRemembersProfilesListedAfterDirtyReset(t *testing.T) { require.NoError(t, err) } +func TestVGPUClaimConfiguresBeforeReleasingAllocationLock(t *testing.T) { + // GPU 82 has one placement left, but its free VFs still list 2Q. The + // first claim must take that placement before releasing the allocation + // lock, so the second is refused by the driver and moves to GPU e3, + // instead of both claims landing on GPU 82 and one failing later. + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:e3:00.4", ParentGPU: "0000:e3:00.0", Allocated: true, ProfileType: testVFProfileType}, + {PCIAddress: "0000:e3:00.5", ParentGPU: "0000:e3:00.0"}, + } + m := newVGPUAllocationManager(t, vfs) + m.pickVFIndex = pickFirst + running := saveTestVGPUInstance(t, m, "running") + running.GPUFramework = devices.VGPUFrameworkVendorVFIO + running.GPUDevicePath = "/sys/bus/pci/devices/0000:e3:00.4" + require.NoError(t, m.saveMetadata(running)) + placementsLeft := map[string]int{"0000:82:00.0": 1, "0000:e3:00.0": 1} + m.configureVGPU = func(_ context.Context, vfAddress, _ string) error { + vf, ok := vfByAddress(vfs, vfAddress) + require.True(t, ok) + if placementsLeft[vf.ParentGPU] == 0 { + return errDriverRejected(vfAddress) + } + placementsLeft[vf.ParentGPU]-- + return nil + } + + first, err := m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "first"), testVGPUProfile) + require.NoError(t, err) + second, err := m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "second"), testVGPUProfile) + require.NoError(t, err) + assert.Equal(t, "0000:82:00.4", first.VFAddress) + assert.Equal(t, "0000:e3:00.5", second.VFAddress) + assert.Equal(t, map[string]int{"0000:82:00.0": 0, "0000:e3:00.0": 0}, placementsLeft) +} + +func TestVGPUClaimResetsConfiguredVFWhenClaimSaveFails(t *testing.T) { + vfs := []devices.VirtualFunction{{PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}} + m := newVGPUAllocationManager(t, vfs) + var resets []string + m.destroyVGPU = func(_ context.Context, assignment devices.VGPUAssignment) error { + resets = append(resets, assignment.DevicePath) + return nil + } + meta := saveTestVGPUInstance(t, m, "new") + require.NoError(t, os.RemoveAll(m.paths.InstanceDir("new"))) + + _, err := m.claimVGPU(context.Background(), meta, testVGPUProfile) + require.ErrorContains(t, err, "save vGPU claim") + assert.Equal(t, []string{testVFDevicePath}, resets) + assert.Empty(t, meta.GPUDevicePath) +} + func errDriverRejected(vfAddress string) error { return fmt.Errorf("configure vGPU on VF %s: %w: invalid argument", vfAddress, devices.ErrVGPUTypeRejected) } From 0b19dbaf7c645a1390e363b295acbdc3a65c1bce Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 14 Oct 2026 21:43:35 +0000 Subject: [PATCH 04/19] Report vGPU driver rejections and drop rejected GPUs from not-creatable check When every GPU that advertised a profile refused its write, the selector still counted their creatable lists and reported "no available VF". Count only VFs still in the candidate set, so the claim fails with the not-creatable error as it does for hidden profiles. Count driver rejections per GPU in hypeman_instances_vgpu_driver_rejections_total, return an error instead of saving an unconfigured claim if the selected VF is somehow not a candidate, and note that the later configure call in create and start only re-checks quarantine and the type. --- lib/devices/GPU.md | 2 +- lib/instances/create.go | 2 ++ lib/instances/metrics.go | 19 +++++++++++++++++++ lib/instances/metrics_test.go | 25 +++++++++++++++++++++++++ lib/instances/start.go | 2 ++ lib/instances/vgpu.go | 14 +++++++++++--- lib/instances/vgpu_linux_test.go | 18 ++++++++++++++++++ 7 files changed, 78 insertions(+), 4 deletions(-) diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 28646ae50..0d59a4714 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -106,7 +106,7 @@ before the hypervisor is signalled. Hypeman reconciles metadata claims once at startup and every minute afterward, skipping hosts without GPUs. A claim whose VMM is confirmed dead is reset before the claim is removed. An ambiguous hypervisor ownership check preserves the claim, logs a warning, and increments `hypeman_instances_vgpu_liveness_uncertain_total`; the create and start cleanup paths preserve and count the same way. mdev hosts also sweep orphaned device-level assignments. Vendor VFIO hosts repair an unclaimed dirty VF when the allocator next selects it; repair checks for open VFIO handles before resetting `current_vgpu_type`. The allocator prefers VFs that are already clean, and a dirty VF that refuses its reset is skipped in favor of another candidate. -In mixed-size (heterogeneous) mode, `creatable_vgpu_types` under-reports. The driver drops a profile from every VF's list once its GPU holds as many vGPUs as that profile's equal-size instance limit (12 for L40S-4Q), even while a placement for it remains, and still accepts a `current_vgpu_type` write for it. The allocator remembers every profile each GPU has advertised, and a clean VF on a GPU that supports the requested profile but no longer advertises it stays a candidate, ranked after every GPU with a free VF that advertises it. If the driver rejects the claim's write, unclaimed dirty VFs on that GPU are reset and the write is retried once before the next GPU is tried; any other configure error fails the claim. A free VF that advertises nothing is skipped (on L40S its GPU has no placement left for any size), as is one whose list cannot be read. This assumes every GPU is in mixed-size mode (`nvidia-smi vgpu -i -shm 1`): in equal-size mode a GPU running a smaller profile hides larger ones for good, so each request for one costs a refused write on that GPU. +In mixed-size (heterogeneous) mode, `creatable_vgpu_types` under-reports. The driver drops a profile from every VF's list once its GPU holds as many vGPUs as that profile's equal-size instance limit (12 for L40S-4Q), even while a placement for it remains, and still accepts a `current_vgpu_type` write for it. The allocator remembers every profile each GPU has advertised, and a clean VF on a GPU that supports the requested profile but no longer advertises it stays a candidate, ranked after every GPU with a free VF that advertises it. If the driver rejects the claim's write, unclaimed dirty VFs on that GPU are reset and the write is retried once before the next GPU is tried, and `hypeman_instances_vgpu_driver_rejections_total` is incremented with the GPU's address; any other configure error fails the claim. A free VF that advertises nothing is skipped (on L40S its GPU has no placement left for any size), as is one whose list cannot be read. This assumes every GPU is in mixed-size mode (`nvidia-smi vgpu -i -shm 1`): in equal-size mode a GPU running a smaller profile hides larger ones for good, so each request for one costs a refused write on that GPU. ### Hypervisor Support diff --git a/lib/instances/create.go b/lib/instances/create.go index 504b58e65..0284d1791 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -403,6 +403,8 @@ func (m *manager) createInstance( "instance_id", id, "device_path", storedVGPUDevicePath(stored)) } }) + // On vendor VFIO hosts claimVGPU already wrote the type, so this only + // re-checks quarantine and confirms the type is still set. if err := m.configureClaimedVGPU(ctx, gpuDevice); err != nil { log.ErrorContext(ctx, "failed to configure vGPU", "profile", gpuProfile, "error", err) return nil, wrapCreateVGPUErr(gpuProfile, err) diff --git a/lib/instances/metrics.go b/lib/instances/metrics.go index 8d86b6cc7..4d2117df9 100644 --- a/lib/instances/metrics.go +++ b/lib/instances/metrics.go @@ -105,6 +105,7 @@ type Metrics struct { vgpuReconcileFailuresTotal metric.Int64Counter vgpuStaleReleaseFailuresTotal metric.Int64Counter vgpuLivenessUncertainTotal metric.Int64Counter + vgpuDriverRejectionsTotal metric.Int64Counter tracer trace.Tracer } @@ -305,6 +306,14 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M return nil, err } + vgpuDriverRejectionsTotal, err := meter.Int64Counter( + "hypeman_instances_vgpu_driver_rejections_total", + metric.WithDescription("Total vendor VFIO current_vgpu_type writes the driver refused during placement, after which the GPU was skipped for that claim"), + ) + if err != nil { + return nil, err + } + vgpuQuarantinedVFs, err := meter.Int64ObservableGauge( "hypeman_instances_vgpu_quarantined_vfs", metric.WithDescription("Number of vGPU virtual functions currently quarantined"), @@ -535,6 +544,7 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M vgpuReconcileFailuresTotal: vgpuReconcileFailuresTotal, vgpuStaleReleaseFailuresTotal: vgpuStaleReleaseFailuresTotal, vgpuLivenessUncertainTotal: vgpuLivenessUncertainTotal, + vgpuDriverRejectionsTotal: vgpuDriverRejectionsTotal, tracer: tracer, }, nil } @@ -657,6 +667,15 @@ func (m *manager) recordVGPULivenessUncertain(ctx context.Context) { m.metrics.vgpuLivenessUncertainTotal.Add(ctx, 1) } +func (m *manager) recordVGPUDriverRejection(ctx context.Context, parentGPU string) { + if m.metrics == nil { + return + } + m.metrics.vgpuDriverRejectionsTotal.Add(ctx, 1, metric.WithAttributes( + attribute.String("gpu", parentGPU), + )) +} + // recordStateTransition records a state transition with hypervisor label. func (m *manager) recordStateTransition(ctx context.Context, fromState, toState string, hvType hypervisor.Type) { if m.metrics == nil { diff --git a/lib/instances/metrics_test.go b/lib/instances/metrics_test.go index 511ff2f92..c6c047466 100644 --- a/lib/instances/metrics_test.go +++ b/lib/instances/metrics_test.go @@ -182,6 +182,31 @@ func TestVGPULivenessUncertainMetric(t *testing.T) { assert.Equal(t, int64(1), uncertain.DataPoints[0].Value) } +func TestVGPUDriverRejectionMetric(t *testing.T) { + t.Parallel() + + reader := otelmetric.NewManualReader() + provider := otelmetric.NewMeterProvider(otelmetric.WithReader(reader)) + m := &manager{paths: paths.New(t.TempDir())} + metrics, err := newInstanceMetrics(provider.Meter("test"), nil, m) + require.NoError(t, err) + m.metrics = metrics + + m.recordVGPUDriverRejection(t.Context(), "0000:82:00.0") + m.recordVGPUDriverRejection(t.Context(), "0000:82:00.0") + + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(t.Context(), &rm)) + metricValue := findMetric(t, rm, "hypeman_instances_vgpu_driver_rejections_total") + rejections, ok := metricValue.Data.(metricdata.Sum[int64]) + require.True(t, ok) + require.Len(t, rejections.DataPoints, 1) + assert.Equal(t, int64(2), rejections.DataPoints[0].Value) + gpu, ok := rejections.DataPoints[0].Attributes.Value("gpu") + require.True(t, ok) + assert.Equal(t, "0000:82:00.0", gpu.AsString()) +} + func TestLifecycleEventMetrics_ObserveSubscribersQueueDepthAndDrops(t *testing.T) { t.Parallel() diff --git a/lib/instances/start.go b/lib/instances/start.go index b162ad90f..f7dee4b9d 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -170,6 +170,8 @@ func (m *manager) startInstance( cu.Add(func() { m.cleanupStartVGPU(ctx, stored, rollbackMeta) }) + // On vendor VFIO hosts claimVGPU already wrote the type, so this only + // re-checks quarantine and confirms the type is still set. if err := m.configureClaimedVGPU(ctx, device); err != nil { log.ErrorContext(ctx, "failed to configure vGPU", "instance_id", id, "profile", stored.GPUProfile, "error", err) return nil, fmt.Errorf("configure vGPU for profile %s: %w", stored.GPUProfile, err) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index fa503cd8a..531671c62 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -104,7 +104,7 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str for { vf, ok := vfByAddress(candidates.vfs, vfAddress) if !ok { - break + return nil, fmt.Errorf("selected VF %s is not a candidate", vfAddress) } if vf.Allocated { assignment := devices.VGPUAssignment{ @@ -132,6 +132,7 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str return nil, placeErr } log.InfoContext(ctx, "driver refused vGPU profile on GPU; trying another GPU", "gpu", vf.ParentGPU, "vf", vfAddress, "profile", profileName, "error", placeErr) + m.recordVGPUDriverRejection(ctx, vf.ParentGPU) candidates.vfs = withoutGPU(candidates.vfs, vf.ParentGPU) if vfAddress, profileType, err = selectVendorVFIOVF(candidates, profileName, m.pickVFIndex); err != nil { return nil, fmt.Errorf("%w (driver refused it on GPU %s: %v)", err, vf.ParentGPU, placeErr) @@ -272,16 +273,23 @@ func selectVendorVFIOVF(c vendorVFIOCandidates, profileName string, pick func(n profilesByName[name] = profile } } - advertisedNames := make(map[string]struct{}) advertises := make(map[string]map[string]struct{}, len(c.profilesByVF)) for vfAddress, profiles := range c.profilesByVF { advertises[vfAddress] = make(map[string]struct{}, len(profiles)) for _, profile := range profiles { profilesByName[profile.Name] = profile - advertisedNames[profile.Name] = struct{}{} advertises[vfAddress][profile.TypeName] = struct{}{} } } + // Only VFs still in c.vfs count as advertising, so a profile whose + // advertising GPUs were all dropped after a driver rejection reports as + // not creatable. + advertisedNames := make(map[string]struct{}) + for _, vf := range c.vfs { + for _, profile := range c.profilesByVF[vf.PCIAddress] { + advertisedNames[profile.Name] = struct{}{} + } + } notCreatable := func() error { if len(advertisedNames) == 0 && len(c.vfs) > 0 { return fmt.Errorf("no creatable vGPU profiles on any VF (GPUs at capacity or dirty VFs consuming framebuffer): profile %q", profileName) diff --git a/lib/instances/vgpu_linux_test.go b/lib/instances/vgpu_linux_test.go index 645c2e61e..d4384a536 100644 --- a/lib/instances/vgpu_linux_test.go +++ b/lib/instances/vgpu_linux_test.go @@ -413,6 +413,24 @@ func TestVGPUClaimFailsWhenDriverRefusesHiddenProfileEverywhere(t *testing.T) { assert.Empty(t, stored.GPUDevicePath) } +func TestVGPUClaimFailsWhenDriverRefusesAdvertisedProfileEverywhere(t *testing.T) { + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:e3:00.4", ParentGPU: "0000:e3:00.0"}, + } + m := newVGPUAllocationManager(t, vfs) + var configured []string + m.configureVGPU = func(_ context.Context, vfAddress, _ string) error { + configured = append(configured, vfAddress) + return errDriverRejected(vfAddress) + } + + _, err := m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "new"), testVGPUProfile) + require.ErrorContains(t, err, "is not creatable on any VF") + require.ErrorContains(t, err, "driver refused it") + assert.Len(t, configured, 2) +} + func TestVGPUClaimFailsOnHiddenProfileConfigureErrorOtherThanRejection(t *testing.T) { hide4Q := true m := newHiddenProfileManager(t, &hide4Q) From 3ad4efa2d5f15520541eaf99b1dc0c23e5434c82 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 14 Oct 2026 22:20:47 +0000 Subject: [PATCH 05/19] Test that a hidden vGPU profile is only offered on a clean VF --- lib/instances/vgpu_test.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 893e311ae..1e5ce7538 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -402,3 +402,21 @@ func TestSelectVendorVFIOVFSkipsGPUThatAdvertisesNothing(t *testing.T) { }, testLargeVGPUProfile, nil) require.ErrorContains(t, err, "no creatable vGPU profiles on any VF") } + +func TestSelectVendorVFIOVFOffersHiddenProfileOnlyOnCleanVF(t *testing.T) { + // GPU 82 hides 4Q and its only free VF is dirty. A configured VF lists + // nothing on L40S, but a hidden profile is only offered on a clean VF + // regardless of what the dirty one lists; claimVGPU resets unclaimed + // dirty VFs and selects again when this fails. + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0", Allocated: true, ProfileType: testVFProfileType}, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0", Allocated: true, ProfileType: testVFProfileType}, + } + _, _, err := selectVendorVFIOVF(vendorVFIOCandidates{ + vfs: vfs, + profilesByVF: testVFProfiles("0000:82:00.5"), + supported: testSupported("0000:82:00.0"), + claims: []StoredMetadata{{GPUProfile: testVGPUProfile, GPUDevicePath: testVFDevicePath}}, + }, testLargeVGPUProfile, nil) + require.ErrorContains(t, err, "is not creatable on any VF") +} From 95995a3a550666bb0d7d44380350efe6afe787d1 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 16 Oct 2026 10:16:16 +0000 Subject: [PATCH 06/19] Reset dirty VFs before failing a claim whose hidden candidates were refused --- lib/devices/GPU.md | 2 +- lib/instances/vgpu.go | 41 ++++++++++++++++---- lib/instances/vgpu_linux_test.go | 64 ++++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 8 deletions(-) diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 0d59a4714..903a555e8 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -106,7 +106,7 @@ before the hypervisor is signalled. Hypeman reconciles metadata claims once at startup and every minute afterward, skipping hosts without GPUs. A claim whose VMM is confirmed dead is reset before the claim is removed. An ambiguous hypervisor ownership check preserves the claim, logs a warning, and increments `hypeman_instances_vgpu_liveness_uncertain_total`; the create and start cleanup paths preserve and count the same way. mdev hosts also sweep orphaned device-level assignments. Vendor VFIO hosts repair an unclaimed dirty VF when the allocator next selects it; repair checks for open VFIO handles before resetting `current_vgpu_type`. The allocator prefers VFs that are already clean, and a dirty VF that refuses its reset is skipped in favor of another candidate. -In mixed-size (heterogeneous) mode, `creatable_vgpu_types` under-reports. The driver drops a profile from every VF's list once its GPU holds as many vGPUs as that profile's equal-size instance limit (12 for L40S-4Q), even while a placement for it remains, and still accepts a `current_vgpu_type` write for it. The allocator remembers every profile each GPU has advertised, and a clean VF on a GPU that supports the requested profile but no longer advertises it stays a candidate, ranked after every GPU with a free VF that advertises it. If the driver rejects the claim's write, unclaimed dirty VFs on that GPU are reset and the write is retried once before the next GPU is tried, and `hypeman_instances_vgpu_driver_rejections_total` is incremented with the GPU's address; any other configure error fails the claim. A free VF that advertises nothing is skipped (on L40S its GPU has no placement left for any size), as is one whose list cannot be read. This assumes every GPU is in mixed-size mode (`nvidia-smi vgpu -i -shm 1`): in equal-size mode a GPU running a smaller profile hides larger ones for good, so each request for one costs a refused write on that GPU. +In mixed-size (heterogeneous) mode, `creatable_vgpu_types` under-reports. The driver drops a profile from every VF's list once its GPU holds as many vGPUs as that profile's equal-size instance limit (12 for L40S-4Q), even while a placement for it remains, and still accepts a `current_vgpu_type` write for it. The allocator remembers every profile each GPU has advertised, and a clean VF on a GPU that supports the requested profile but no longer advertises it stays a candidate, ranked after every GPU with a free VF that advertises it. If the driver rejects the claim's write, unclaimed dirty VFs on that GPU are reset and the write is retried once before the next GPU is tried, and `hypeman_instances_vgpu_driver_rejections_total` is incremented with the GPU's address; any other configure error fails the claim. If no candidate VF is left, unclaimed dirty VFs on the GPUs not yet tried are reset once and selection runs again. A free VF that advertises nothing is skipped (on L40S its GPU has no placement left for any size), as is one whose list cannot be read. This assumes every GPU is in mixed-size mode (`nvidia-smi vgpu -i -shm 1`): in equal-size mode a GPU running a smaller profile hides larger ones for good, so each request for one costs a refused write on that GPU. ### Hypervisor Support diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 531671c62..3eb17be2b 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -6,6 +6,7 @@ import ( "fmt" "math/rand/v2" "path/filepath" + "slices" "sort" "github.com/kernel/hypeman/lib/devices" @@ -67,7 +68,27 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str return nil, err } candidates := vendorVFIOCandidates{vfs: vfs, profilesByVF: profilesByVF, supported: m.supportedVGPUProfiles, claims: allMetadata, quarantined: quarantined} + // refreshCandidates re-reads VF state and creatable lists after dirty VFs + // were reset, keeping only the VFs still in candidates. + refreshCandidates := func() error { + _, vfs, err := m.discoverVGPUDevices() + if err != nil { + return err + } + vfs = slices.DeleteFunc(vfs, func(vf devices.VirtualFunction) bool { + _, ok := vfByAddress(candidates.vfs, vf.PCIAddress) + return !ok + }) + profilesByVF, err := listProfiles(vfs) + if err != nil { + return err + } + m.rememberVGPUProfiles(vfs, profilesByVF) + candidates.vfs, candidates.profilesByVF = vfs, profilesByVF + return nil + } vfAddress, profileType, err := selectVendorVFIOVF(candidates, profileName, m.pickVFIndex) + dirtyReset := err != nil if err != nil { // A dirty unclaimed VF consumes framebuffer, which can make the // requested profile vanish from every creatable list before the @@ -75,14 +96,9 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str if !m.resetDirtyUnclaimedVFs(ctx, vfs, allMetadata) { return nil, err } - if _, vfs, err = m.discoverVGPUDevices(); err != nil { - return nil, err - } - if candidates.profilesByVF, err = listProfiles(vfs); err != nil { + if err = refreshCandidates(); err != nil { return nil, err } - m.rememberVGPUProfiles(vfs, candidates.profilesByVF) - candidates.vfs = vfs if vfAddress, profileType, err = selectVendorVFIOVF(candidates, profileName, m.pickVFIndex); err != nil { return nil, err } @@ -134,7 +150,18 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str log.InfoContext(ctx, "driver refused vGPU profile on GPU; trying another GPU", "gpu", vf.ParentGPU, "vf", vfAddress, "profile", profileName, "error", placeErr) m.recordVGPUDriverRejection(ctx, vf.ParentGPU) candidates.vfs = withoutGPU(candidates.vfs, vf.ParentGPU) - if vfAddress, profileType, err = selectVendorVFIOVF(candidates, profileName, m.pickVFIndex); err != nil { + vfAddress, profileType, err = selectVendorVFIOVF(candidates, profileName, m.pickVFIndex) + // A hidden candidate can satisfy the first selection while dirty VFs + // fill a GPU that has no candidate, so the dirty-VF retry above did + // not run. Run it once before giving up. + if err != nil && !dirtyReset && m.resetDirtyUnclaimedVFs(ctx, candidates.vfs, allMetadata) { + dirtyReset = true + if err = refreshCandidates(); err != nil { + return nil, err + } + vfAddress, profileType, err = selectVendorVFIOVF(candidates, profileName, m.pickVFIndex) + } + if err != nil { return nil, fmt.Errorf("%w (driver refused it on GPU %s: %v)", err, vf.ParentGPU, placeErr) } } diff --git a/lib/instances/vgpu_linux_test.go b/lib/instances/vgpu_linux_test.go index d4384a536..aae7a18d1 100644 --- a/lib/instances/vgpu_linux_test.go +++ b/lib/instances/vgpu_linux_test.go @@ -541,6 +541,70 @@ func TestVGPUClaimResetsDirtySiblingHoldingHiddenPlacement(t *testing.T) { }, events) } +func TestVGPUClaimResetsDirtyVFsOnOtherGPUAfterHiddenProfileRefused(t *testing.T) { + // GPU 41 hides 4Q and refuses it. GPU 82 is filled by an unclaimed dirty + // VF, so its free VF lists nothing and is no candidate. The hidden + // candidate on 41 satisfied the first selection, so the dirty VF must + // still be reset before the claim gives up, without resetting the VF + // another instance claims or retrying GPU 41. + dirty := true + var events []string + m := &manager{ + paths: paths.New(t.TempDir()), + discoverVGPU: func() (devices.VGPUFramework, []devices.VirtualFunction, error) { + leftover := devices.VirtualFunction{PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"} + if dirty { + leftover.Allocated, leftover.ProfileType = true, testVFProfileType + } + return devices.VGPUFrameworkVendorVFIO, []devices.VirtualFunction{ + {PCIAddress: "0000:41:00.4", ParentGPU: "0000:41:00.0"}, + leftover, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:82:00.6", ParentGPU: "0000:82:00.0", Allocated: true, ProfileType: testVFProfileType}, + }, nil + }, + vendorVFIOProfiles: func([]devices.VirtualFunction) (map[string][]devices.VGPUProfileType, error) { + profiles := map[string][]devices.VGPUProfileType{ + "0000:41:00.4": {testSmallProfile}, + "0000:82:00.4": {}, + "0000:82:00.5": {}, + "0000:82:00.6": {}, + } + if !dirty { + profiles["0000:82:00.5"] = []devices.VGPUProfileType{testSmallProfile} + } + return profiles, nil + }, + supportedVGPUProfiles: testSupported("0000:41:00.0", "0000:82:00.0"), + destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { + events = append(events, "reset "+assignment.DevicePath) + dirty = false + return nil + }, + configureVGPU: func(_ context.Context, vfAddress, _ string) error { + events = append(events, "configure "+vfAddress) + if vfAddress == "0000:41:00.4" { + return errDriverRejected(vfAddress) + } + return nil + }, + pickVFIndex: pickFirst, + } + running := saveTestVGPUInstance(t, m, "running") + running.GPUFramework = devices.VGPUFrameworkVendorVFIO + running.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.6" + require.NoError(t, m.saveMetadata(running)) + + device, err := m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "large"), testLargeVGPUProfile) + require.NoError(t, err) + assert.Equal(t, "0000:82:00.5", device.VFAddress) + assert.Equal(t, []string{ + "configure 0000:41:00.4", + "reset " + testVFDevicePath, + "configure 0000:82:00.5", + }, events) +} + func TestVGPUClaimProbesRepairedVFOnceWhenDriverRefuses(t *testing.T) { // The only candidate is a dirty VF left at 4Q on a GPU that hides 4Q. It // is reset, then probed. A refusal must not count that same VF as a dirty From 0c3eaa92824164f49ec3f4b9dc404a1855c11e28 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 16 Oct 2026 10:16:16 +0000 Subject: [PATCH 07/19] Describe the vGPU driver rejection counter as counting skipped GPUs --- lib/instances/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/instances/metrics.go b/lib/instances/metrics.go index 4d2117df9..f7e3d0cfe 100644 --- a/lib/instances/metrics.go +++ b/lib/instances/metrics.go @@ -308,7 +308,7 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M vgpuDriverRejectionsTotal, err := meter.Int64Counter( "hypeman_instances_vgpu_driver_rejections_total", - metric.WithDescription("Total vendor VFIO current_vgpu_type writes the driver refused during placement, after which the GPU was skipped for that claim"), + metric.WithDescription("Total times a GPU was skipped during vendor VFIO vGPU placement because the driver refused the type, counted once per claim and GPU"), ) if err != nil { return nil, err From 6245232ed264342b6d415b35f7528748af653277 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 16 Oct 2026 10:16:16 +0000 Subject: [PATCH 08/19] Test that a claim counts a driver rejection for the skipped GPU --- lib/instances/vgpu_linux_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/instances/vgpu_linux_test.go b/lib/instances/vgpu_linux_test.go index aae7a18d1..44f24f846 100644 --- a/lib/instances/vgpu_linux_test.go +++ b/lib/instances/vgpu_linux_test.go @@ -15,6 +15,8 @@ import ( "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + otelmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" ) func newVGPUAllocationManager(t *testing.T, vfs []devices.VirtualFunction) *manager { @@ -379,6 +381,10 @@ func TestVGPUClaimTriesNextGPUWhenDriverRefusesHiddenProfile(t *testing.T) { hide4Q := true m := newHiddenProfileManager(t, &hide4Q) m.supportedVGPUProfiles = testSupported("0000:82:00.0", "0000:e3:00.0") + reader := otelmetric.NewManualReader() + metrics, err := newInstanceMetrics(otelmetric.NewMeterProvider(otelmetric.WithReader(reader)).Meter("test"), nil, m) + require.NoError(t, err) + m.metrics = metrics var configured []string m.configureVGPU = func(_ context.Context, vfAddress, profileType string) error { configured = append(configured, vfAddress) @@ -392,6 +398,16 @@ func TestVGPUClaimTriesNextGPUWhenDriverRefusesHiddenProfile(t *testing.T) { require.NoError(t, err) assert.Equal(t, "0000:e3:00.4", device.VFAddress) assert.Equal(t, []string{"0000:82:00.4", "0000:e3:00.4"}, configured, "a refusal rules out the whole GPU, not just the VF") + + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(context.Background(), &rm)) + rejections, ok := findMetric(t, rm, "hypeman_instances_vgpu_driver_rejections_total").Data.(metricdata.Sum[int64]) + require.True(t, ok) + require.Len(t, rejections.DataPoints, 1) + assert.Equal(t, int64(1), rejections.DataPoints[0].Value) + gpu, ok := rejections.DataPoints[0].Attributes.Value("gpu") + require.True(t, ok) + assert.Equal(t, "0000:82:00.0", gpu.AsString()) } func TestVGPUClaimFailsWhenDriverRefusesHiddenProfileEverywhere(t *testing.T) { From b43bbd007db15fbcab56f8c3bc61044fbe5fe38f Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 16 Oct 2026 10:16:16 +0000 Subject: [PATCH 09/19] Test that a driver rejection does not reset a claimed sibling VF --- lib/instances/vgpu_linux_test.go | 39 ++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/lib/instances/vgpu_linux_test.go b/lib/instances/vgpu_linux_test.go index 44f24f846..f97171303 100644 --- a/lib/instances/vgpu_linux_test.go +++ b/lib/instances/vgpu_linux_test.go @@ -557,6 +557,45 @@ func TestVGPUClaimResetsDirtySiblingHoldingHiddenPlacement(t *testing.T) { }, events) } +func TestVGPUClaimKeepsClaimedSiblingWhenDriverRefuses(t *testing.T) { + // GPU 82's configured sibling belongs to another instance, so a refusal + // must not reset it to free a placement. + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0", Allocated: true, ProfileType: testVFProfileType}, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0"}, + } + var events []string + m := &manager{ + paths: paths.New(t.TempDir()), + discoverVGPU: func() (devices.VGPUFramework, []devices.VirtualFunction, error) { + return devices.VGPUFrameworkVendorVFIO, vfs, nil + }, + vendorVFIOProfiles: func([]devices.VirtualFunction) (map[string][]devices.VGPUProfileType, error) { + return map[string][]devices.VGPUProfileType{ + "0000:82:00.4": {}, + "0000:82:00.5": {testSmallProfile}, + }, nil + }, + supportedVGPUProfiles: testSupported("0000:82:00.0"), + destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { + events = append(events, "reset "+assignment.DevicePath) + return nil + }, + configureVGPU: func(_ context.Context, vfAddress, _ string) error { + events = append(events, "configure "+vfAddress) + return errDriverRejected(vfAddress) + }, + } + running := saveTestVGPUInstance(t, m, "running") + running.GPUFramework = devices.VGPUFrameworkVendorVFIO + running.GPUDevicePath = testVFDevicePath + require.NoError(t, m.saveMetadata(running)) + + _, err := m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "large"), testLargeVGPUProfile) + require.ErrorContains(t, err, "driver refused it on GPU 0000:82:00.0") + assert.Equal(t, []string{"configure 0000:82:00.5"}, events) +} + func TestVGPUClaimResetsDirtyVFsOnOtherGPUAfterHiddenProfileRefused(t *testing.T) { // GPU 41 hides 4Q and refuses it. GPU 82 is filled by an unclaimed dirty // VF, so its free VF lists nothing and is no candidate. The hidden From 9bd194d56f347afca432f4ceee5003ebd8dbe004 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 16 Oct 2026 10:55:23 +0000 Subject: [PATCH 10/19] Drop restating and change-narrating vGPU comments --- lib/instances/vgpu.go | 2 -- lib/instances/vgpu_test.go | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 3eb17be2b..2c484ea00 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -423,8 +423,6 @@ func selectVendorVFIOVF(c vendorVFIOCandidates, profileName string, pick func(n return candidates[pick(n)].PCIAddress, requested.TypeName, nil } -// advertisingGPUs returns the parent GPUs with at least one VF advertising -// profileType. func advertisingGPUs(c vendorVFIOCandidates, profileType string) map[string]bool { gpus := make(map[string]bool) for _, vf := range c.vfs { diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 1e5ce7538..ee25b6925 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -291,7 +291,7 @@ func TestSelectVendorVFIOVFOffersSupportedProfileItsGPUHides(t *testing.T) { assert.Equal(t, "0000:e3:00.5", vf) assert.Equal(t, testLargeProfileType, profileType) - // Without a remembered profile the request fails as before. + // Without a remembered profile the request fails. c.supported = nil _, _, err = selectVendorVFIOVF(c, testLargeVGPUProfile, nil) require.ErrorContains(t, err, "is not creatable on any VF") From 12a59e3591f9a46364db5499c4b9cb0c09710020 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 16 Oct 2026 10:55:34 +0000 Subject: [PATCH 11/19] Remove vGPU driver rejection tests other tests already cover --- lib/instances/metrics_test.go | 25 ------------------------- lib/instances/vgpu_linux_test.go | 19 ------------------- 2 files changed, 44 deletions(-) diff --git a/lib/instances/metrics_test.go b/lib/instances/metrics_test.go index c6c047466..511ff2f92 100644 --- a/lib/instances/metrics_test.go +++ b/lib/instances/metrics_test.go @@ -182,31 +182,6 @@ func TestVGPULivenessUncertainMetric(t *testing.T) { assert.Equal(t, int64(1), uncertain.DataPoints[0].Value) } -func TestVGPUDriverRejectionMetric(t *testing.T) { - t.Parallel() - - reader := otelmetric.NewManualReader() - provider := otelmetric.NewMeterProvider(otelmetric.WithReader(reader)) - m := &manager{paths: paths.New(t.TempDir())} - metrics, err := newInstanceMetrics(provider.Meter("test"), nil, m) - require.NoError(t, err) - m.metrics = metrics - - m.recordVGPUDriverRejection(t.Context(), "0000:82:00.0") - m.recordVGPUDriverRejection(t.Context(), "0000:82:00.0") - - var rm metricdata.ResourceMetrics - require.NoError(t, reader.Collect(t.Context(), &rm)) - metricValue := findMetric(t, rm, "hypeman_instances_vgpu_driver_rejections_total") - rejections, ok := metricValue.Data.(metricdata.Sum[int64]) - require.True(t, ok) - require.Len(t, rejections.DataPoints, 1) - assert.Equal(t, int64(2), rejections.DataPoints[0].Value) - gpu, ok := rejections.DataPoints[0].Attributes.Value("gpu") - require.True(t, ok) - assert.Equal(t, "0000:82:00.0", gpu.AsString()) -} - func TestLifecycleEventMetrics_ObserveSubscribersQueueDepthAndDrops(t *testing.T) { t.Parallel() diff --git a/lib/instances/vgpu_linux_test.go b/lib/instances/vgpu_linux_test.go index f97171303..f7aa023b9 100644 --- a/lib/instances/vgpu_linux_test.go +++ b/lib/instances/vgpu_linux_test.go @@ -410,25 +410,6 @@ func TestVGPUClaimTriesNextGPUWhenDriverRefusesHiddenProfile(t *testing.T) { assert.Equal(t, "0000:82:00.0", gpu.AsString()) } -func TestVGPUClaimFailsWhenDriverRefusesHiddenProfileEverywhere(t *testing.T) { - hide4Q := true - m := newHiddenProfileManager(t, &hide4Q) - m.supportedVGPUProfiles = testSupported("0000:82:00.0", "0000:e3:00.0") - var configured []string - m.configureVGPU = func(_ context.Context, vfAddress, _ string) error { - configured = append(configured, vfAddress) - return errDriverRejected(vfAddress) - } - - _, err := m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "large"), testLargeVGPUProfile) - require.ErrorContains(t, err, "is not creatable on any VF") - require.ErrorContains(t, err, "driver refused it") - assert.Equal(t, []string{"0000:82:00.4", "0000:e3:00.4"}, configured) - stored, loadErr := m.loadMetadata("large") - require.NoError(t, loadErr) - assert.Empty(t, stored.GPUDevicePath) -} - func TestVGPUClaimFailsWhenDriverRefusesAdvertisedProfileEverywhere(t *testing.T) { vfs := []devices.VirtualFunction{ {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}, From 2c9c09cbb9ea160ed215071354ed1d6efe047234 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 16 Oct 2026 10:57:25 +0000 Subject: [PATCH 12/19] Keep the test for a hidden profile refused on every GPU --- lib/instances/vgpu_linux_test.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/lib/instances/vgpu_linux_test.go b/lib/instances/vgpu_linux_test.go index f7aa023b9..f97171303 100644 --- a/lib/instances/vgpu_linux_test.go +++ b/lib/instances/vgpu_linux_test.go @@ -410,6 +410,25 @@ func TestVGPUClaimTriesNextGPUWhenDriverRefusesHiddenProfile(t *testing.T) { assert.Equal(t, "0000:82:00.0", gpu.AsString()) } +func TestVGPUClaimFailsWhenDriverRefusesHiddenProfileEverywhere(t *testing.T) { + hide4Q := true + m := newHiddenProfileManager(t, &hide4Q) + m.supportedVGPUProfiles = testSupported("0000:82:00.0", "0000:e3:00.0") + var configured []string + m.configureVGPU = func(_ context.Context, vfAddress, _ string) error { + configured = append(configured, vfAddress) + return errDriverRejected(vfAddress) + } + + _, err := m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "large"), testLargeVGPUProfile) + require.ErrorContains(t, err, "is not creatable on any VF") + require.ErrorContains(t, err, "driver refused it") + assert.Equal(t, []string{"0000:82:00.4", "0000:e3:00.4"}, configured) + stored, loadErr := m.loadMetadata("large") + require.NoError(t, loadErr) + assert.Empty(t, stored.GPUDevicePath) +} + func TestVGPUClaimFailsWhenDriverRefusesAdvertisedProfileEverywhere(t *testing.T) { vfs := []devices.VirtualFunction{ {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}, From 188521d627891bec1e74a24172bebd3420be6919 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 14 Oct 2026 09:03:55 +0000 Subject: [PATCH 13/19] Pack mixed-size vGPU placements to keep room for larger profiles The driver puts each new vGPU at the lowest free placement for its type, which on an L40S lands half of the 2Q vGPUs on placements a 4Q needs. After configuring a type, move the vGPU to the free placement that blocks the fewest free placements of larger profiles, read from nvidia-smi and cached per GPU. Ties and any failure keep the driver's placement. --- lib/devices/GPU.md | 2 + .../nvidia-smi-vgpu-supported-l40s.txt | 662 ++++++++++++++++++ lib/devices/vendor_vfio_linux.go | 2 + lib/devices/vendor_vfio_linux_test.go | 4 + lib/devices/vgpu_placement_linux.go | 332 +++++++++ lib/devices/vgpu_placement_linux_test.go | 226 ++++++ 6 files changed, 1228 insertions(+) create mode 100644 lib/devices/testdata/nvidia-smi-vgpu-supported-l40s.txt create mode 100644 lib/devices/vgpu_placement_linux.go create mode 100644 lib/devices/vgpu_placement_linux_test.go diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 903a555e8..ca0288263 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -108,6 +108,8 @@ Hypeman reconciles metadata claims once at startup and every minute afterward, s In mixed-size (heterogeneous) mode, `creatable_vgpu_types` under-reports. The driver drops a profile from every VF's list once its GPU holds as many vGPUs as that profile's equal-size instance limit (12 for L40S-4Q), even while a placement for it remains, and still accepts a `current_vgpu_type` write for it. The allocator remembers every profile each GPU has advertised, and a clean VF on a GPU that supports the requested profile but no longer advertises it stays a candidate, ranked after every GPU with a free VF that advertises it. If the driver rejects the claim's write, unclaimed dirty VFs on that GPU are reset and the write is retried once before the next GPU is tried, and `hypeman_instances_vgpu_driver_rejections_total` is incremented with the GPU's address; any other configure error fails the claim. If no candidate VF is left, unclaimed dirty VFs on the GPUs not yet tried are reset once and selection runs again. A free VF that advertises nothing is skipped (on L40S its GPU has no placement left for any size), as is one whose list cannot be read. This assumes every GPU is in mixed-size mode (`nvidia-smi vgpu -i -shm 1`): in equal-size mode a GPU running a smaller profile hides larger ones for good, so each request for one costs a refused write on that GPU. +In mixed-size mode each vGPU also occupies a fixed run of placement slots, and the driver puts a new vGPU at the lowest free placement for its type. On an L40S that scatters 2Q across the runs 4Q needs: half of the 16 2Q placements overlap a 4Q placement. After writing `current_vgpu_type`, and before any VM attaches, hypeman moves the vGPU by writing `placement_id` to the free placement that blocks the fewest free placements of larger profiles, comparing the smallest larger size first. A tie keeps the driver's placement. On an L40S the first eight 2Q then land on the placements no 4Q overlaps, so a card holds eight 2Q and eight 4Q instead of eight 2Q and four 4Q. The placement table and mode come from `nvidia-smi vgpu -s -v -i ` and `nvidia-smi -q -i `, cached per GPU for 10 minutes. Packing is skipped in equal-size mode, and any failure (no `nvidia-smi`, an unreadable sibling, a refused write) keeps the driver's placement. + ### Hypervisor Support Hypervisor selection for vGPU instances is caller policy; hypeman does not enforce it. In practice **QEMU is the only hypervisor with working vGPU support**: diff --git a/lib/devices/testdata/nvidia-smi-vgpu-supported-l40s.txt b/lib/devices/testdata/nvidia-smi-vgpu-supported-l40s.txt new file mode 100644 index 000000000..90e2e1c1e --- /dev/null +++ b/lib/devices/testdata/nvidia-smi-vgpu-supported-l40s.txt @@ -0,0 +1,662 @@ +GPU 00000000:82:00.0 + vGPU Type ID : 0x479 + Name : NVIDIA L40S-1B + Class : NVS + GPU Instance Profile ID : N/A + Max Instances : 32 + Max Instances Per VM : 1 + Max Instances Per GI : N/A + Multi vGPU Exclusive : True + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x188910de + FB Memory : 1024 MiB + BAR1 size : 256 MB + Display Heads : 4 + Maximum X Resolution : 5120 + Maximum Y Resolution : 2880 + Frame Rate Limit : 45 FPS + Placement Size : 1 + Heterogeneous Placement IDs : 0 5 6 11 12 17 18 23 24 29 30 35 36 41 42 47 + Homogeneous Placement IDs : 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 + GRID License : GRID-Virtual-PC,2.0;Quadro-Virtual-DWS,5.0;GRID-Virtual-WS,2.0;GRID-Virtual-WS-Ext,2.0 + vGPU Type ID : 0x47a + Name : NVIDIA L40S-2B + Class : NVS + GPU Instance Profile ID : N/A + Max Instances : 24 + Max Instances Per VM : 1 + Max Instances Per GI : N/A + Multi vGPU Exclusive : True + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x188a10de + FB Memory : 2048 MiB + BAR1 size : 256 MB + Display Heads : 4 + Maximum X Resolution : 5120 + Maximum Y Resolution : 2880 + Frame Rate Limit : 45 FPS + Placement Size : 2 + Heterogeneous Placement IDs : 0 4 6 10 12 16 18 22 24 28 30 34 36 40 42 46 + Homogeneous Placement IDs : 0 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40 42 44 46 + GRID License : GRID-Virtual-PC,2.0;Quadro-Virtual-DWS,5.0;GRID-Virtual-WS,2.0;GRID-Virtual-WS-Ext,2.0 + vGPU Type ID : 0x47b + Name : NVIDIA L40S-1Q + Class : Quadro + GPU Instance Profile ID : N/A + Max Instances : 32 + Max Instances Per VM : 16 + Max Instances Per GI : N/A + Multi vGPU Exclusive : False + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x188b10de + FB Memory : 1024 MiB + BAR1 size : 256 MB + Display Heads : 4 + Maximum X Resolution : 5120 + Maximum Y Resolution : 2880 + Frame Rate Limit : 60 FPS + Placement Size : 1 + Heterogeneous Placement IDs : 0 5 6 11 12 17 18 23 24 29 30 35 36 41 42 47 + Homogeneous Placement IDs : 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 + GRID License : Quadro-Virtual-DWS,5.0;GRID-Virtual-WS,2.0;GRID-Virtual-WS-Ext,2.0 + vGPU Type ID : 0x47c + Name : NVIDIA L40S-2Q + Class : Quadro + GPU Instance Profile ID : N/A + Max Instances : 24 + Max Instances Per VM : 16 + Max Instances Per GI : N/A + Multi vGPU Exclusive : False + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x188c10de + FB Memory : 2048 MiB + BAR1 size : 256 MB + Display Heads : 4 + Maximum X Resolution : 7680 + Maximum Y Resolution : 4320 + Frame Rate Limit : 60 FPS + Placement Size : 2 + Heterogeneous Placement IDs : 0 4 6 10 12 16 18 22 24 28 30 34 36 40 42 46 + Homogeneous Placement IDs : 0 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40 42 44 46 + GRID License : Quadro-Virtual-DWS,5.0;GRID-Virtual-WS,2.0;GRID-Virtual-WS-Ext,2.0 + vGPU Type ID : 0x47d + Name : NVIDIA L40S-3Q + Class : Quadro + GPU Instance Profile ID : N/A + Max Instances : 16 + Max Instances Per VM : 16 + Max Instances Per GI : N/A + Multi vGPU Exclusive : False + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x188d10de + FB Memory : 3072 MiB + BAR1 size : 256 MB + Display Heads : 4 + Maximum X Resolution : 7680 + Maximum Y Resolution : 4320 + Frame Rate Limit : 60 FPS + Placement Size : 3 + Heterogeneous Placement IDs : 0 3 6 9 12 15 18 21 24 27 30 33 36 39 42 45 + Homogeneous Placement IDs : 0 3 6 9 12 15 18 21 24 27 30 33 36 39 42 45 + GRID License : Quadro-Virtual-DWS,5.0;GRID-Virtual-WS,2.0;GRID-Virtual-WS-Ext,2.0 + vGPU Type ID : 0x47e + Name : NVIDIA L40S-4Q + Class : Quadro + GPU Instance Profile ID : N/A + Max Instances : 12 + Max Instances Per VM : 16 + Max Instances Per GI : N/A + Multi vGPU Exclusive : False + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x188e10de + FB Memory : 4096 MiB + BAR1 size : 256 MB + Display Heads : 4 + Maximum X Resolution : 7680 + Maximum Y Resolution : 4320 + Frame Rate Limit : 60 FPS + Placement Size : 4 + Heterogeneous Placement IDs : 0 8 12 20 24 32 36 44 + Homogeneous Placement IDs : 0 4 8 12 16 20 24 28 32 36 40 44 + GRID License : Quadro-Virtual-DWS,5.0;GRID-Virtual-WS,2.0;GRID-Virtual-WS-Ext,2.0 + vGPU Type ID : 0x47f + Name : NVIDIA L40S-6Q + Class : Quadro + GPU Instance Profile ID : N/A + Max Instances : 8 + Max Instances Per VM : 16 + Max Instances Per GI : N/A + Multi vGPU Exclusive : False + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x188f10de + FB Memory : 6144 MiB + BAR1 size : 256 MB + Display Heads : 4 + Maximum X Resolution : 7680 + Maximum Y Resolution : 4320 + Frame Rate Limit : 60 FPS + Placement Size : 6 + Heterogeneous Placement IDs : 0 6 12 18 24 30 36 42 + Homogeneous Placement IDs : 0 6 12 18 24 30 36 42 + GRID License : Quadro-Virtual-DWS,5.0;GRID-Virtual-WS,2.0;GRID-Virtual-WS-Ext,2.0 + vGPU Type ID : 0x480 + Name : NVIDIA L40S-8Q + Class : Quadro + GPU Instance Profile ID : N/A + Max Instances : 6 + Max Instances Per VM : 16 + Max Instances Per GI : N/A + Multi vGPU Exclusive : False + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x189010de + FB Memory : 8192 MiB + BAR1 size : 256 MB + Display Heads : 4 + Maximum X Resolution : 7680 + Maximum Y Resolution : 4320 + Frame Rate Limit : 60 FPS + Placement Size : 8 + Heterogeneous Placement IDs : 0 16 24 40 + Homogeneous Placement IDs : 0 8 16 24 32 40 + GRID License : Quadro-Virtual-DWS,5.0;GRID-Virtual-WS,2.0;GRID-Virtual-WS-Ext,2.0 + vGPU Type ID : 0x481 + Name : NVIDIA L40S-12Q + Class : Quadro + GPU Instance Profile ID : N/A + Max Instances : 4 + Max Instances Per VM : 16 + Max Instances Per GI : N/A + Multi vGPU Exclusive : False + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x189110de + FB Memory : 12288 MiB + BAR1 size : 256 MB + Display Heads : 4 + Maximum X Resolution : 7680 + Maximum Y Resolution : 4320 + Frame Rate Limit : 60 FPS + Placement Size : 12 + Heterogeneous Placement IDs : 0 12 24 36 + Homogeneous Placement IDs : 0 12 24 36 + GRID License : Quadro-Virtual-DWS,5.0;GRID-Virtual-WS,2.0;GRID-Virtual-WS-Ext,2.0 + vGPU Type ID : 0x482 + Name : NVIDIA L40S-16Q + Class : Quadro + GPU Instance Profile ID : N/A + Max Instances : 3 + Max Instances Per VM : 16 + Max Instances Per GI : N/A + Multi vGPU Exclusive : False + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x189210de + FB Memory : 16384 MiB + BAR1 size : 256 MB + Display Heads : 4 + Maximum X Resolution : 7680 + Maximum Y Resolution : 4320 + Frame Rate Limit : 60 FPS + Placement Size : 16 + Heterogeneous Placement IDs : 0 32 + Homogeneous Placement IDs : 0 16 32 + GRID License : Quadro-Virtual-DWS,5.0;GRID-Virtual-WS,2.0;GRID-Virtual-WS-Ext,2.0 + vGPU Type ID : 0x483 + Name : NVIDIA L40S-24Q + Class : Quadro + GPU Instance Profile ID : N/A + Max Instances : 2 + Max Instances Per VM : 16 + Max Instances Per GI : N/A + Multi vGPU Exclusive : False + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x189310de + FB Memory : 24576 MiB + BAR1 size : 256 MB + Display Heads : 4 + Maximum X Resolution : 7680 + Maximum Y Resolution : 4320 + Frame Rate Limit : 60 FPS + Placement Size : 24 + Heterogeneous Placement IDs : 0 24 + Homogeneous Placement IDs : 0 24 + GRID License : Quadro-Virtual-DWS,5.0;GRID-Virtual-WS,2.0;GRID-Virtual-WS-Ext,2.0 + vGPU Type ID : 0x484 + Name : NVIDIA L40S-48Q + Class : Quadro + GPU Instance Profile ID : N/A + Max Instances : 1 + Max Instances Per VM : 16 + Max Instances Per GI : N/A + Multi vGPU Exclusive : False + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x189410de + FB Memory : 49152 MiB + BAR1 size : 256 MB + Display Heads : 4 + Maximum X Resolution : 7680 + Maximum Y Resolution : 4320 + Frame Rate Limit : 60 FPS + Placement Size : 48 + Heterogeneous Placement IDs : 0 + Homogeneous Placement IDs : 0 + GRID License : Quadro-Virtual-DWS,5.0;GRID-Virtual-WS,2.0;GRID-Virtual-WS-Ext,2.0 + vGPU Type ID : 0x485 + Name : NVIDIA L40S-1A + Class : NVS + GPU Instance Profile ID : N/A + Max Instances : 32 + Max Instances Per VM : 1 + Max Instances Per GI : N/A + Multi vGPU Exclusive : True + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x189510de + FB Memory : 1024 MiB + BAR1 size : 256 MB + Display Heads : 1 + Maximum X Resolution : 1280 + Maximum Y Resolution : 1024 + Frame Rate Limit : 60 FPS + Placement Size : 1 + Heterogeneous Placement IDs : 0 5 6 11 12 17 18 23 24 29 30 35 36 41 42 47 + Homogeneous Placement IDs : 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 + GRID License : GRID-Virtual-Apps,3.0 + vGPU Type ID : 0x486 + Name : NVIDIA L40S-2A + Class : NVS + GPU Instance Profile ID : N/A + Max Instances : 24 + Max Instances Per VM : 1 + Max Instances Per GI : N/A + Multi vGPU Exclusive : True + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x189610de + FB Memory : 2048 MiB + BAR1 size : 256 MB + Display Heads : 1 + Maximum X Resolution : 1280 + Maximum Y Resolution : 1024 + Frame Rate Limit : 60 FPS + Placement Size : 2 + Heterogeneous Placement IDs : 0 4 6 10 12 16 18 22 24 28 30 34 36 40 42 46 + Homogeneous Placement IDs : 0 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40 42 44 46 + GRID License : GRID-Virtual-Apps,3.0 + vGPU Type ID : 0x487 + Name : NVIDIA L40S-3A + Class : NVS + GPU Instance Profile ID : N/A + Max Instances : 16 + Max Instances Per VM : 1 + Max Instances Per GI : N/A + Multi vGPU Exclusive : True + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x189710de + FB Memory : 3072 MiB + BAR1 size : 256 MB + Display Heads : 1 + Maximum X Resolution : 1280 + Maximum Y Resolution : 1024 + Frame Rate Limit : 60 FPS + Placement Size : 3 + Heterogeneous Placement IDs : 0 3 6 9 12 15 18 21 24 27 30 33 36 39 42 45 + Homogeneous Placement IDs : 0 3 6 9 12 15 18 21 24 27 30 33 36 39 42 45 + GRID License : GRID-Virtual-Apps,3.0 + vGPU Type ID : 0x488 + Name : NVIDIA L40S-4A + Class : NVS + GPU Instance Profile ID : N/A + Max Instances : 12 + Max Instances Per VM : 1 + Max Instances Per GI : N/A + Multi vGPU Exclusive : True + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x189810de + FB Memory : 4096 MiB + BAR1 size : 256 MB + Display Heads : 1 + Maximum X Resolution : 1280 + Maximum Y Resolution : 1024 + Frame Rate Limit : 60 FPS + Placement Size : 4 + Heterogeneous Placement IDs : 0 8 12 20 24 32 36 44 + Homogeneous Placement IDs : 0 4 8 12 16 20 24 28 32 36 40 44 + GRID License : GRID-Virtual-Apps,3.0 + vGPU Type ID : 0x489 + Name : NVIDIA L40S-6A + Class : NVS + GPU Instance Profile ID : N/A + Max Instances : 8 + Max Instances Per VM : 1 + Max Instances Per GI : N/A + Multi vGPU Exclusive : True + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x189910de + FB Memory : 6144 MiB + BAR1 size : 256 MB + Display Heads : 1 + Maximum X Resolution : 1280 + Maximum Y Resolution : 1024 + Frame Rate Limit : 60 FPS + Placement Size : 6 + Heterogeneous Placement IDs : 0 6 12 18 24 30 36 42 + Homogeneous Placement IDs : 0 6 12 18 24 30 36 42 + GRID License : GRID-Virtual-Apps,3.0 + vGPU Type ID : 0x48a + Name : NVIDIA L40S-8A + Class : NVS + GPU Instance Profile ID : N/A + Max Instances : 6 + Max Instances Per VM : 1 + Max Instances Per GI : N/A + Multi vGPU Exclusive : True + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x189a10de + FB Memory : 8192 MiB + BAR1 size : 256 MB + Display Heads : 1 + Maximum X Resolution : 1280 + Maximum Y Resolution : 1024 + Frame Rate Limit : 60 FPS + Placement Size : 8 + Heterogeneous Placement IDs : 0 16 24 40 + Homogeneous Placement IDs : 0 8 16 24 32 40 + GRID License : GRID-Virtual-Apps,3.0 + vGPU Type ID : 0x48b + Name : NVIDIA L40S-12A + Class : NVS + GPU Instance Profile ID : N/A + Max Instances : 4 + Max Instances Per VM : 1 + Max Instances Per GI : N/A + Multi vGPU Exclusive : True + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x189b10de + FB Memory : 12288 MiB + BAR1 size : 256 MB + Display Heads : 1 + Maximum X Resolution : 1280 + Maximum Y Resolution : 1024 + Frame Rate Limit : 60 FPS + Placement Size : 12 + Heterogeneous Placement IDs : 0 12 24 36 + Homogeneous Placement IDs : 0 12 24 36 + GRID License : GRID-Virtual-Apps,3.0 + vGPU Type ID : 0x48c + Name : NVIDIA L40S-16A + Class : NVS + GPU Instance Profile ID : N/A + Max Instances : 3 + Max Instances Per VM : 1 + Max Instances Per GI : N/A + Multi vGPU Exclusive : True + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x189c10de + FB Memory : 16384 MiB + BAR1 size : 256 MB + Display Heads : 1 + Maximum X Resolution : 1280 + Maximum Y Resolution : 1024 + Frame Rate Limit : 60 FPS + Placement Size : 16 + Heterogeneous Placement IDs : 0 32 + Homogeneous Placement IDs : 0 16 32 + GRID License : GRID-Virtual-Apps,3.0 + vGPU Type ID : 0x48d + Name : NVIDIA L40S-24A + Class : NVS + GPU Instance Profile ID : N/A + Max Instances : 2 + Max Instances Per VM : 1 + Max Instances Per GI : N/A + Multi vGPU Exclusive : True + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x189d10de + FB Memory : 24576 MiB + BAR1 size : 256 MB + Display Heads : 1 + Maximum X Resolution : 1280 + Maximum Y Resolution : 1024 + Frame Rate Limit : 60 FPS + Placement Size : 24 + Heterogeneous Placement IDs : 0 24 + Homogeneous Placement IDs : 0 24 + GRID License : GRID-Virtual-Apps,3.0 + vGPU Type ID : 0x48e + Name : NVIDIA L40S-48A + Class : NVS + GPU Instance Profile ID : N/A + Max Instances : 1 + Max Instances Per VM : 1 + Max Instances Per GI : N/A + Multi vGPU Exclusive : True + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x189e10de + FB Memory : 49152 MiB + BAR1 size : 256 MB + Display Heads : 1 + Maximum X Resolution : 1280 + Maximum Y Resolution : 1024 + Frame Rate Limit : 60 FPS + Placement Size : 48 + Heterogeneous Placement IDs : 0 + Homogeneous Placement IDs : 0 + GRID License : GRID-Virtual-Apps,3.0 + vGPU Type ID : 0x49b + Name : NVIDIA L40S-4C + Class : Compute + GPU Instance Profile ID : N/A + Max Instances : 12 + Max Instances Per VM : 16 + Max Instances Per GI : N/A + Multi vGPU Exclusive : False + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x18ab10de + FB Memory : 4096 MiB + BAR1 size : 2048 MB + Display Heads : 1 + Maximum X Resolution : 4096 + Maximum Y Resolution : 2400 + Frame Rate Limit : 60 FPS + Placement Size : 4 + Heterogeneous Placement IDs : 0 8 12 20 24 32 36 44 + Homogeneous Placement IDs : 0 4 8 12 16 20 24 28 32 36 40 44 + GRID License : NVIDIA-vComputeServer,9.0 + vGPU Type ID : 0x49c + Name : NVIDIA L40S-6C + Class : Compute + GPU Instance Profile ID : N/A + Max Instances : 8 + Max Instances Per VM : 16 + Max Instances Per GI : N/A + Multi vGPU Exclusive : False + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x18ac10de + FB Memory : 6144 MiB + BAR1 size : 2048 MB + Display Heads : 1 + Maximum X Resolution : 4096 + Maximum Y Resolution : 2400 + Frame Rate Limit : 60 FPS + Placement Size : 6 + Heterogeneous Placement IDs : 0 6 12 18 24 30 36 42 + Homogeneous Placement IDs : 0 6 12 18 24 30 36 42 + GRID License : NVIDIA-vComputeServer,9.0 + vGPU Type ID : 0x49d + Name : NVIDIA L40S-8C + Class : Compute + GPU Instance Profile ID : N/A + Max Instances : 6 + Max Instances Per VM : 16 + Max Instances Per GI : N/A + Multi vGPU Exclusive : False + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x18ad10de + FB Memory : 8192 MiB + BAR1 size : 2048 MB + Display Heads : 1 + Maximum X Resolution : 4096 + Maximum Y Resolution : 2400 + Frame Rate Limit : 60 FPS + Placement Size : 8 + Heterogeneous Placement IDs : 0 16 24 40 + Homogeneous Placement IDs : 0 8 16 24 32 40 + GRID License : NVIDIA-vComputeServer,9.0 + vGPU Type ID : 0x49e + Name : NVIDIA L40S-12C + Class : Compute + GPU Instance Profile ID : N/A + Max Instances : 4 + Max Instances Per VM : 16 + Max Instances Per GI : N/A + Multi vGPU Exclusive : False + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x18ae10de + FB Memory : 12288 MiB + BAR1 size : 2048 MB + Display Heads : 1 + Maximum X Resolution : 4096 + Maximum Y Resolution : 2400 + Frame Rate Limit : 60 FPS + Placement Size : 12 + Heterogeneous Placement IDs : 0 12 24 36 + Homogeneous Placement IDs : 0 12 24 36 + GRID License : NVIDIA-vComputeServer,9.0 + vGPU Type ID : 0x49f + Name : NVIDIA L40S-16C + Class : Compute + GPU Instance Profile ID : N/A + Max Instances : 3 + Max Instances Per VM : 16 + Max Instances Per GI : N/A + Multi vGPU Exclusive : False + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x18af10de + FB Memory : 16384 MiB + BAR1 size : 2048 MB + Display Heads : 1 + Maximum X Resolution : 4096 + Maximum Y Resolution : 2400 + Frame Rate Limit : 60 FPS + Placement Size : 16 + Heterogeneous Placement IDs : 0 32 + Homogeneous Placement IDs : 0 16 32 + GRID License : NVIDIA-vComputeServer,9.0 + vGPU Type ID : 0x4a0 + Name : NVIDIA L40S-24C + Class : Compute + GPU Instance Profile ID : N/A + Max Instances : 2 + Max Instances Per VM : 16 + Max Instances Per GI : N/A + Multi vGPU Exclusive : False + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x18b010de + FB Memory : 24576 MiB + BAR1 size : 2048 MB + Display Heads : 1 + Maximum X Resolution : 4096 + Maximum Y Resolution : 2400 + Frame Rate Limit : 60 FPS + Placement Size : 24 + Heterogeneous Placement IDs : 0 24 + Homogeneous Placement IDs : 0 24 + GRID License : NVIDIA-vComputeServer,9.0 + vGPU Type ID : 0x4a1 + Name : NVIDIA L40S-48C + Class : Compute + GPU Instance Profile ID : N/A + Max Instances : 1 + Max Instances Per VM : 16 + Max Instances Per GI : N/A + Multi vGPU Exclusive : False + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x18b110de + FB Memory : 49152 MiB + BAR1 size : 2048 MB + Display Heads : 1 + Maximum X Resolution : 4096 + Maximum Y Resolution : 2400 + Frame Rate Limit : 60 FPS + Placement Size : 48 + Heterogeneous Placement IDs : 0 + Homogeneous Placement IDs : 0 + GRID License : NVIDIA-vComputeServer,9.0 + vGPU Type ID : 0x874 + Name : NVIDIA L40S-3B + Class : NVS + GPU Instance Profile ID : N/A + Max Instances : 16 + Max Instances Per VM : 1 + Max Instances Per GI : N/A + Multi vGPU Exclusive : True + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x21fe10de + FB Memory : 3072 MiB + BAR1 size : 256 MB + Display Heads : 4 + Maximum X Resolution : 5120 + Maximum Y Resolution : 2880 + Frame Rate Limit : 45 FPS + Placement Size : 3 + Heterogeneous Placement IDs : 0 3 6 9 12 15 18 21 24 27 30 33 36 39 42 45 + Homogeneous Placement IDs : 0 3 6 9 12 15 18 21 24 27 30 33 36 39 42 45 + GRID License : GRID-Virtual-PC,2.0;Quadro-Virtual-DWS,5.0;GRID-Virtual-WS,2.0;GRID-Virtual-WS-Ext,2.0 + diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index d286ce87e..7f080364d 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -28,6 +28,7 @@ type vendorVFIOSysfs struct { vfioDevicesPath string openVFIOPathsFunc func() (map[string]struct{}, error) writeFileFunc func(name string, data []byte, perm os.FileMode) error + nvidiaSMIFunc func(ctx context.Context, args ...string) ([]byte, error) } var hostVendorVFIO = vendorVFIOSysfs{ @@ -211,6 +212,7 @@ func (s vendorVFIOSysfs) configure(ctx context.Context, vfAddress, profileType s return fmt.Errorf("verify vGPU on VF %s: type is %s, want %s", vfAddress, currentType, profileType) } logger.FromContext(ctx).InfoContext(ctx, "configured vendor VFIO vGPU", "profile_type", profileType, "vf", vfAddress) + s.packPlacement(ctx, vfAddress, profileType) return nil } diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 9f2983f9f..3a938ae4d 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -4,6 +4,7 @@ package devices import ( "context" + "errors" "os" "path/filepath" "syscall" @@ -177,6 +178,9 @@ func newTestVendorVFIOSysfs(t *testing.T) testVendorVFIOSysfs { pciDevicesPath: pci, procPath: proc, vfioDevicesPath: vfio, + nvidiaSMIFunc: func(context.Context, ...string) ([]byte, error) { + return nil, errors.New("nvidia-smi not available in tests") + }, }} } diff --git a/lib/devices/vgpu_placement_linux.go b/lib/devices/vgpu_placement_linux.go new file mode 100644 index 000000000..5eff4f2cb --- /dev/null +++ b/lib/devices/vgpu_placement_linux.go @@ -0,0 +1,332 @@ +//go:build linux + +package devices + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + + "github.com/kernel/hypeman/lib/logger" +) + +// In mixed-size (heterogeneous) mode each vGPU occupies a fixed run of its +// GPU's placement slots, and each profile may only start at the driver's +// heterogeneous placement IDs. The driver puts a new vGPU at the lowest free +// placement for its type, which scatters small profiles across the runs larger +// profiles need: on an L40S, half of the 2Q placements overlap a 4Q placement. +// After a type is configured, packPlacement moves the vGPU to the free +// placement that blocks the fewest placements of larger profiles. Packing is +// best effort: any failure keeps the driver's placement. + +// vgpuPlacementTTL bounds how long a GPU's mode and geometry are reused, so a +// mode change made while hypeman runs is picked up. +const vgpuPlacementTTL = 10 * time.Minute + +const nvidiaSMITimeout = 5 * time.Second + +type vgpuPlacementType struct { + size int + ids []int +} + +// vgpuPlacementGeometry holds a GPU's placement table, which is empty unless +// the GPU is in mixed-size mode. +type vgpuPlacementGeometry struct { + types map[string]vgpuPlacementType // keyed by decimal vGPU type ID + loadedAt time.Time +} + +// vgpuPlacements caches geometry by parent GPU sysfs path, failures included, +// so a host without nvidia-smi does not run it on every claim. Guarded by +// vendorVFIOMu. +var vgpuPlacements = make(map[string]vgpuPlacementGeometry) + +type placementSpan struct { + start, size int +} + +func (s vendorVFIOSysfs) packPlacement(ctx context.Context, vfAddress, profileType string) { + log := logger.FromContext(ctx) + vfPath := filepath.Join(s.pciDevicesPath, vfAddress) + target, err := os.Readlink(filepath.Join(vfPath, "physfn")) + if err != nil { + log.WarnContext(ctx, "keeping driver vGPU placement: read parent GPU", "vf", vfAddress, "error", err) + return + } + parentGPU := filepath.Base(target) + geometry := s.placementGeometry(ctx, parentGPU) + requested, ok := geometry.types[profileType] + if !ok { + return + } + placementPath := filepath.Join(vfPath, "nvidia", "placement_id") + current, err := readPlacementID(placementPath) + if err != nil { + log.WarnContext(ctx, "keeping driver vGPU placement: read placement", "vf", vfAddress, "error", err) + return + } + occupied, err := s.occupiedPlacements(parentGPU, vfAddress, geometry) + if err != nil { + log.WarnContext(ctx, "keeping driver vGPU placement: read sibling placements", "vf", vfAddress, "error", err) + return + } + best := choosePlacement(geometry.types, occupied, requested, current) + if best == current { + return + } + if err := s.writeFile(placementPath, []byte(strconv.Itoa(best)), 0200); err != nil { + log.WarnContext(ctx, "driver refused vGPU placement; keeping its own", "vf", vfAddress, "from", current, "to", best, "error", err) + return + } + if moved, err := readPlacementID(placementPath); err != nil || moved != best { + log.WarnContext(ctx, "vGPU placement did not move as written", "vf", vfAddress, "want", best, "got", moved, "error", err) + return + } + log.InfoContext(ctx, "packed vGPU placement", "vf", vfAddress, "profile_type", profileType, "from", current, "to", best) +} + +// placementGeometry returns the cached placement table for a GPU, reloading +// it once it is older than vgpuPlacementTTL. A failed load is cached as an +// empty table, which disables packing on that GPU until the next reload. +// Callers must hold vendorVFIOMu. +func (s vendorVFIOSysfs) placementGeometry(ctx context.Context, parentGPU string) vgpuPlacementGeometry { + key := filepath.Join(s.pciDevicesPath, parentGPU) + if cached, ok := vgpuPlacements[key]; ok && time.Since(cached.loadedAt) < vgpuPlacementTTL { + return cached + } + geometry, err := s.loadPlacementGeometry(ctx, parentGPU) + if err != nil { + logger.FromContext(ctx).WarnContext(ctx, "vGPU placement packing disabled for GPU", "gpu", parentGPU, "error", err) + geometry = vgpuPlacementGeometry{} + } + geometry.loadedAt = time.Now() + vgpuPlacements[key] = geometry + return geometry +} + +func (s vendorVFIOSysfs) loadPlacementGeometry(ctx context.Context, parentGPU string) (vgpuPlacementGeometry, error) { + query, err := s.nvidiaSMI(ctx, "-q", "-i", parentGPU) + if err != nil { + return vgpuPlacementGeometry{}, fmt.Errorf("query GPU: %w", err) + } + heterogeneous, err := parseHeterogeneousMode(string(query)) + if err != nil || !heterogeneous { + return vgpuPlacementGeometry{}, err + } + supported, err := s.nvidiaSMI(ctx, "vgpu", "-s", "-v", "-i", parentGPU) + if err != nil { + return vgpuPlacementGeometry{}, fmt.Errorf("query supported vGPU types: %w", err) + } + types, err := parsePlacementTypes(string(supported)) + if err != nil { + return vgpuPlacementGeometry{}, err + } + return vgpuPlacementGeometry{types: types}, nil +} + +func (s vendorVFIOSysfs) nvidiaSMI(ctx context.Context, args ...string) ([]byte, error) { + if s.nvidiaSMIFunc != nil { + return s.nvidiaSMIFunc(ctx, args...) + } + ctx, cancel := context.WithTimeout(ctx, nvidiaSMITimeout) + defer cancel() + return exec.CommandContext(ctx, "nvidia-smi", args...).Output() +} + +// occupiedPlacements returns the placement spans of every configured VF on +// parentGPU other than vfAddress. An unreadable or unknown sibling fails the +// lookup, since a placement chosen around it could not be trusted. +func (s vendorVFIOSysfs) occupiedPlacements(parentGPU, vfAddress string, geometry vgpuPlacementGeometry) ([]placementSpan, error) { + entries, err := os.ReadDir(s.pciDevicesPath) + if err != nil { + return nil, fmt.Errorf("read PCI devices: %w", err) + } + var occupied []placementSpan + for _, entry := range entries { + if entry.Name() == vfAddress { + continue + } + vfPath := filepath.Join(s.pciDevicesPath, entry.Name()) + target, err := os.Readlink(filepath.Join(vfPath, "physfn")) + if err != nil || filepath.Base(target) != parentGPU { + continue + } + nvidiaPath := filepath.Join(vfPath, "nvidia") + if _, err := os.Stat(nvidiaPath); err != nil { + continue + } + currentType, err := readCurrentVGPUType(filepath.Join(nvidiaPath, "current_vgpu_type")) + if err != nil { + return nil, fmt.Errorf("read current vGPU type for VF %s: %w", entry.Name(), err) + } + if currentType == "0" { + continue + } + placed, ok := geometry.types[currentType] + if !ok { + return nil, fmt.Errorf("VF %s has vGPU type %s with no known placement size", entry.Name(), currentType) + } + start, err := readPlacementID(filepath.Join(nvidiaPath, "placement_id")) + if err != nil { + return nil, fmt.Errorf("read placement for VF %s: %w", entry.Name(), err) + } + occupied = append(occupied, placementSpan{start: start, size: placed.size}) + } + return occupied, nil +} + +// choosePlacement returns the free placement for requested that blocks the +// fewest free placements of larger profiles, comparing the smallest larger +// size first. Ties keep current, so the driver's placement is only overridden +// for a strictly better one. +func choosePlacement(types map[string]vgpuPlacementType, occupied []placementSpan, requested vgpuPlacementType, current int) int { + free := func(span placementSpan) bool { + for _, other := range occupied { + if span.start < other.start+other.size && other.start < span.start+span.size { + return false + } + } + return true + } + freeBySize := make(map[int]map[int]struct{}) + for _, placementType := range types { + if placementType.size <= requested.size { + continue + } + for _, id := range placementType.ids { + if !free(placementSpan{start: id, size: placementType.size}) { + continue + } + if freeBySize[placementType.size] == nil { + freeBySize[placementType.size] = make(map[int]struct{}) + } + freeBySize[placementType.size][id] = struct{}{} + } + } + sizes := make([]int, 0, len(freeBySize)) + for size := range freeBySize { + sizes = append(sizes, size) + } + sort.Ints(sizes) + blocked := func(start int) []int { + counts := make([]int, len(sizes)) + for i, size := range sizes { + for id := range freeBySize[size] { + if start < id+size && id < start+requested.size { + counts[i]++ + } + } + } + return counts + } + + best, bestBlocked := current, blocked(current) + ids := append([]int(nil), requested.ids...) + sort.Ints(ids) + for _, id := range ids { + if !free(placementSpan{start: id, size: requested.size}) { + continue + } + if candidate := blocked(id); fewerBlocked(candidate, bestBlocked) { + best, bestBlocked = id, candidate + } + } + return best +} + +func fewerBlocked(a, b []int) bool { + for i := range a { + if a[i] != b[i] { + return a[i] < b[i] + } + } + return false +} + +// parseHeterogeneousMode reads "vGPU Heterogeneous Mode : Enabled" from +// nvidia-smi -q output for a single GPU. +func parseHeterogeneousMode(output string) (bool, error) { + for _, line := range strings.Split(output, "\n") { + key, value, found := strings.Cut(line, ":") + if !found || strings.TrimSpace(key) != "vGPU Heterogeneous Mode" { + continue + } + switch strings.TrimSpace(value) { + case "Enabled": + return true, nil + case "Disabled": + return false, nil + } + return false, fmt.Errorf("unrecognized vGPU heterogeneous mode %q", strings.TrimSpace(value)) + } + return false, fmt.Errorf("vGPU heterogeneous mode not reported") +} + +// parsePlacementTypes reads each type's placement size and heterogeneous +// placement IDs from nvidia-smi vgpu -s -v output, keyed by decimal type ID +// to match current_vgpu_type. Types without numeric placement information +// are left out. +func parsePlacementTypes(output string) (map[string]vgpuPlacementType, error) { + types := make(map[string]vgpuPlacementType) + typeID := "" + var current vgpuPlacementType + flush := func() { + if typeID != "" && current.size > 0 && len(current.ids) > 0 { + types[typeID] = current + } + } + for _, line := range strings.Split(output, "\n") { + key, value, found := strings.Cut(line, ":") + if !found { + continue + } + key, value = strings.TrimSpace(key), strings.TrimSpace(value) + switch key { + case "vGPU Type ID": + flush() + id, err := strconv.ParseInt(value, 0, 64) + if err != nil { + return nil, fmt.Errorf("parse vGPU type ID %q: %w", value, err) + } + typeID, current = strconv.FormatInt(id, 10), vgpuPlacementType{} + case "Placement Size": + if size, err := strconv.Atoi(value); err == nil { + current.size = size + } + case "Heterogeneous Placement IDs": + for _, field := range strings.Fields(value) { + id, err := strconv.Atoi(field) + if err != nil { + current.ids = nil + break + } + current.ids = append(current.ids, id) + } + } + } + flush() + if len(types) == 0 { + return nil, fmt.Errorf("no vGPU placement information reported") + } + return types, nil +} + +func readPlacementID(path string) (int, error) { + data, err := os.ReadFile(path) + if err != nil { + return 0, err + } + value := strings.TrimSpace(string(data)) + id, err := strconv.Atoi(value) + if err != nil { + return 0, fmt.Errorf("invalid placement ID %q", value) + } + return id, nil +} diff --git a/lib/devices/vgpu_placement_linux_test.go b/lib/devices/vgpu_placement_linux_test.go new file mode 100644 index 000000000..381143f31 --- /dev/null +++ b/lib/devices/vgpu_placement_linux_test.go @@ -0,0 +1,226 @@ +//go:build linux + +package devices + +import ( + "context" + "errors" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "syscall" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + testL40S1Q = "1147" + testL40S2Q = "1148" + testL40S4Q = "1150" +) + +const testHeterogeneousQuery = `GPU 00000000:82:00.0 + GPU Virtualization Mode + Virtualization Mode : Host VGPU + Host VGPU Mode : SR-IOV + vGPU Heterogeneous Mode : Enabled +` + +func loadL40SPlacementTypes(t *testing.T) map[string]vgpuPlacementType { + t.Helper() + output, err := os.ReadFile(filepath.Join("testdata", "nvidia-smi-vgpu-supported-l40s.txt")) + require.NoError(t, err) + types, err := parsePlacementTypes(string(output)) + require.NoError(t, err) + return types +} + +func TestParsePlacementTypes(t *testing.T) { + types := loadL40SPlacementTypes(t) + + assert.Equal(t, vgpuPlacementType{size: 2, ids: []int{0, 4, 6, 10, 12, 16, 18, 22, 24, 28, 30, 34, 36, 40, 42, 46}}, types[testL40S2Q]) + assert.Equal(t, vgpuPlacementType{size: 4, ids: []int{0, 8, 12, 20, 24, 32, 36, 44}}, types[testL40S4Q]) + assert.Equal(t, 1, types[testL40S1Q].size) +} + +func TestParseHeterogeneousMode(t *testing.T) { + enabled, err := parseHeterogeneousMode(testHeterogeneousQuery) + require.NoError(t, err) + assert.True(t, enabled) + + enabled, err = parseHeterogeneousMode(strings.Replace(testHeterogeneousQuery, "Enabled", "Disabled", 1)) + require.NoError(t, err) + assert.False(t, enabled) + + _, err = parseHeterogeneousMode("GPU 00000000:82:00.0\n") + require.Error(t, err) +} + +func placementFree(occupied []placementSpan, span placementSpan) bool { + for _, other := range occupied { + if span.start < other.start+other.size && other.start < span.start+span.size { + return false + } + } + return true +} + +// placeLikeDriver returns the lowest free placement for typeID, which is +// where the driver puts a newly configured vGPU. +func placeLikeDriver(types map[string]vgpuPlacementType, occupied []placementSpan, typeID string) (int, bool) { + requested := types[typeID] + for _, id := range requested.ids { + if placementFree(occupied, placementSpan{start: id, size: requested.size}) { + return id, true + } + } + return 0, false +} + +func free4QPlacements(types map[string]vgpuPlacementType, occupied []placementSpan) int { + count := 0 + for _, id := range types[testL40S4Q].ids { + if placementFree(occupied, placementSpan{start: id, size: 4}) { + count++ + } + } + return count +} + +func TestChoosePlacementKeeps4QRoomOnL40S(t *testing.T) { + types := loadL40SPlacementTypes(t) + + var packed, unpacked []placementSpan + for i := 0; i < 8; i++ { + start, ok := placeLikeDriver(types, unpacked, testL40S2Q) + require.True(t, ok) + unpacked = append(unpacked, placementSpan{start: start, size: 2}) + + start, ok = placeLikeDriver(types, packed, testL40S2Q) + require.True(t, ok) + start = choosePlacement(types, packed, types[testL40S2Q], start) + packed = append(packed, placementSpan{start: start, size: 2}) + } + + starts := make([]int, 0, len(packed)) + for _, span := range packed { + starts = append(starts, span.start) + } + sort.Ints(starts) + assert.Equal(t, []int{4, 6, 16, 18, 28, 30, 40, 42}, starts, "the first eight 2Q fill the placements no 4Q overlaps") + assert.Equal(t, 8, free4QPlacements(types, packed)) + assert.Equal(t, 4, free4QPlacements(types, unpacked), "the driver's lowest-free placement blocks half the 4Q room") + + // Once those are used, every further 2Q costs exactly one 4Q placement. + for i := 0; i < 8; i++ { + start, ok := placeLikeDriver(types, packed, testL40S2Q) + require.True(t, ok) + start = choosePlacement(types, packed, types[testL40S2Q], start) + packed = append(packed, placementSpan{start: start, size: 2}) + assert.Equal(t, 7-i, free4QPlacements(types, packed)) + } +} + +func TestChoosePlacementKeepsCurrentOnTie(t *testing.T) { + types := loadL40SPlacementTypes(t) + // 4Q at 0 and 8 fill the first two 4Q placements; 2Q at 4 and 6 are + // equally good, so the driver's choice stands. + occupied := []placementSpan{{start: 0, size: 4}, {start: 8, size: 4}} + + assert.Equal(t, 6, choosePlacement(types, occupied, types[testL40S2Q], 6)) + assert.Equal(t, 4, choosePlacement(types, occupied, types[testL40S2Q], 4)) +} + +// addPlacedVF adds a VF with a configured type at placement, as the driver +// leaves it after a type write. +func (s testVendorVFIOSysfs) addPlacedVF(t *testing.T, address, vfioID, currentType string, placement int) { + t.Helper() + s.addVF(t, "0000:82:00.0", address, vfioID, currentType, testCreatableTypes) + require.NoError(t, os.WriteFile(filepath.Join(s.pciDevicesPath, address, "nvidia", "placement_id"), []byte(strconv.Itoa(placement)), 0644)) +} + +func newPackingSysfs(t *testing.T, query string) (testVendorVFIOSysfs, *int) { + t.Helper() + supported, err := os.ReadFile(filepath.Join("testdata", "nvidia-smi-vgpu-supported-l40s.txt")) + require.NoError(t, err) + sysfs := newTestVendorVFIOSysfs(t) + calls := 0 + sysfs.nvidiaSMIFunc = func(_ context.Context, args ...string) ([]byte, error) { + calls++ + if args[0] == "-q" { + return []byte(query), nil + } + return supported, nil + } + return sysfs, &calls +} + +func TestVendorVFIOConfigurePacksPlacement(t *testing.T) { + sysfs, calls := newPackingSysfs(t, testHeterogeneousQuery) + // The driver put the new 2Q at the lowest free placement, 0, which + // overlaps the 4Q placement at 0. Placement 6 overlaps none. + sysfs.addPlacedVF(t, "0000:82:00.4", "42", "0", 0) + sysfs.addPlacedVF(t, "0000:82:00.5", "43", testL40S2Q, 4) + + require.NoError(t, sysfs.configure(context.Background(), "0000:82:00.4", testL40S2Q)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "placement_id"), "6") + + sysfs.addPlacedVF(t, "0000:82:00.6", "44", "0", 0) + require.NoError(t, sysfs.configure(context.Background(), "0000:82:00.6", testL40S2Q)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.6", "nvidia", "placement_id"), "16") + assert.Equal(t, 2, *calls, "geometry is loaded once per GPU") +} + +func TestVendorVFIOConfigureKeepsDriverPlacement(t *testing.T) { + tests := []struct { + name string + setup func(sysfs *testVendorVFIOSysfs) + }{ + { + name: "equal-size mode", + setup: func(sysfs *testVendorVFIOSysfs) { + query := strings.Replace(testHeterogeneousQuery, "Enabled", "Disabled", 1) + sysfs.nvidiaSMIFunc = func(context.Context, ...string) ([]byte, error) { return []byte(query), nil } + }, + }, + { + name: "nvidia-smi fails", + setup: func(sysfs *testVendorVFIOSysfs) { + sysfs.nvidiaSMIFunc = func(context.Context, ...string) ([]byte, error) { return nil, errors.New("exit status 9") } + }, + }, + { + name: "sibling of unknown type", + setup: func(sysfs *testVendorVFIOSysfs) { + sysfs.addPlacedVF(t, "0000:82:00.6", "44", "999", 8) + }, + }, + { + name: "driver refuses placement", + setup: func(sysfs *testVendorVFIOSysfs) { + sysfs.writeFileFunc = func(name string, data []byte, perm os.FileMode) error { + if filepath.Base(name) == "placement_id" { + return syscall.EIO + } + return os.WriteFile(name, data, perm) + } + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sysfs, _ := newPackingSysfs(t, testHeterogeneousQuery) + sysfs.addPlacedVF(t, "0000:82:00.4", "42", "0", 0) + sysfs.addPlacedVF(t, "0000:82:00.5", "43", testL40S2Q, 4) + tt.setup(&sysfs) + + require.NoError(t, sysfs.configure(context.Background(), "0000:82:00.4", testL40S2Q)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "current_vgpu_type"), testL40S2Q) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "placement_id"), "0") + }) + } +} From fbe59487644e606c1b89129980b0dce6b4b74733 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 16 Oct 2026 10:10:56 +0000 Subject: [PATCH 14/19] Log when equal-size mode disables vGPU placement packing --- lib/devices/vgpu_placement_linux.go | 6 +++++- lib/devices/vgpu_placement_linux_test.go | 25 +++++++++++++++++------- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/lib/devices/vgpu_placement_linux.go b/lib/devices/vgpu_placement_linux.go index 5eff4f2cb..9ce71ce5b 100644 --- a/lib/devices/vgpu_placement_linux.go +++ b/lib/devices/vgpu_placement_linux.go @@ -4,6 +4,7 @@ package devices import ( "context" + "errors" "fmt" "os" "os/exec" @@ -117,9 +118,12 @@ func (s vendorVFIOSysfs) loadPlacementGeometry(ctx context.Context, parentGPU st return vgpuPlacementGeometry{}, fmt.Errorf("query GPU: %w", err) } heterogeneous, err := parseHeterogeneousMode(string(query)) - if err != nil || !heterogeneous { + if err != nil { return vgpuPlacementGeometry{}, err } + if !heterogeneous { + return vgpuPlacementGeometry{}, errors.New("GPU is in equal-size vGPU mode") + } supported, err := s.nvidiaSMI(ctx, "vgpu", "-s", "-v", "-i", parentGPU) if err != nil { return vgpuPlacementGeometry{}, fmt.Errorf("query supported vGPU types: %w", err) diff --git a/lib/devices/vgpu_placement_linux_test.go b/lib/devices/vgpu_placement_linux_test.go index 381143f31..2e156c853 100644 --- a/lib/devices/vgpu_placement_linux_test.go +++ b/lib/devices/vgpu_placement_linux_test.go @@ -3,8 +3,10 @@ package devices import ( + "bytes" "context" "errors" + "log/slog" "os" "path/filepath" "sort" @@ -13,6 +15,7 @@ import ( "syscall" "testing" + "github.com/kernel/hypeman/lib/logger" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -177,30 +180,35 @@ func TestVendorVFIOConfigurePacksPlacement(t *testing.T) { func TestVendorVFIOConfigureKeepsDriverPlacement(t *testing.T) { tests := []struct { - name string - setup func(sysfs *testVendorVFIOSysfs) + name string + setup func(sysfs *testVendorVFIOSysfs) + logged string }{ { - name: "equal-size mode", + name: "equal-size mode", + logged: "equal-size vGPU mode", setup: func(sysfs *testVendorVFIOSysfs) { query := strings.Replace(testHeterogeneousQuery, "Enabled", "Disabled", 1) sysfs.nvidiaSMIFunc = func(context.Context, ...string) ([]byte, error) { return []byte(query), nil } }, }, { - name: "nvidia-smi fails", + name: "nvidia-smi fails", + logged: "exit status 9", setup: func(sysfs *testVendorVFIOSysfs) { sysfs.nvidiaSMIFunc = func(context.Context, ...string) ([]byte, error) { return nil, errors.New("exit status 9") } }, }, { - name: "sibling of unknown type", + name: "sibling of unknown type", + logged: "no known placement size", setup: func(sysfs *testVendorVFIOSysfs) { sysfs.addPlacedVF(t, "0000:82:00.6", "44", "999", 8) }, }, { - name: "driver refuses placement", + name: "driver refuses placement", + logged: "driver refused vGPU placement", setup: func(sysfs *testVendorVFIOSysfs) { sysfs.writeFileFunc = func(name string, data []byte, perm os.FileMode) error { if filepath.Base(name) == "placement_id" { @@ -217,10 +225,13 @@ func TestVendorVFIOConfigureKeepsDriverPlacement(t *testing.T) { sysfs.addPlacedVF(t, "0000:82:00.4", "42", "0", 0) sysfs.addPlacedVF(t, "0000:82:00.5", "43", testL40S2Q, 4) tt.setup(&sysfs) + var logs bytes.Buffer + ctx := logger.AddToContext(context.Background(), slog.New(slog.NewTextHandler(&logs, nil))) - require.NoError(t, sysfs.configure(context.Background(), "0000:82:00.4", testL40S2Q)) + require.NoError(t, sysfs.configure(ctx, "0000:82:00.4", testL40S2Q)) assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "current_vgpu_type"), testL40S2Q) assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "placement_id"), "0") + assert.Contains(t, logs.String(), tt.logged) }) } } From 9580005ac73acbc3d4a8dd77d4bb2983fff04b22 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 16 Oct 2026 10:11:22 +0000 Subject: [PATCH 15/19] Test that a clean sibling VF does not block packing --- lib/devices/vgpu_placement_linux_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/devices/vgpu_placement_linux_test.go b/lib/devices/vgpu_placement_linux_test.go index 2e156c853..5504d3dde 100644 --- a/lib/devices/vgpu_placement_linux_test.go +++ b/lib/devices/vgpu_placement_linux_test.go @@ -165,9 +165,11 @@ func newPackingSysfs(t *testing.T, query string) (testVendorVFIOSysfs, *int) { func TestVendorVFIOConfigurePacksPlacement(t *testing.T) { sysfs, calls := newPackingSysfs(t, testHeterogeneousQuery) // The driver put the new 2Q at the lowest free placement, 0, which - // overlaps the 4Q placement at 0. Placement 6 overlaps none. + // overlaps the 4Q placement at 0. Placement 6 overlaps none. The clean + // sibling occupies nothing. sysfs.addPlacedVF(t, "0000:82:00.4", "42", "0", 0) sysfs.addPlacedVF(t, "0000:82:00.5", "43", testL40S2Q, 4) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.7", "45", "0", testCreatableTypes) require.NoError(t, sysfs.configure(context.Background(), "0000:82:00.4", testL40S2Q)) assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "placement_id"), "6") From 91af238887b708cc27fdbadb423f7ffba3a12d8a Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 16 Oct 2026 10:11:34 +0000 Subject: [PATCH 16/19] Test that vGPUs on another GPU do not block packing --- lib/devices/vgpu_placement_linux_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/devices/vgpu_placement_linux_test.go b/lib/devices/vgpu_placement_linux_test.go index 5504d3dde..a02d1628c 100644 --- a/lib/devices/vgpu_placement_linux_test.go +++ b/lib/devices/vgpu_placement_linux_test.go @@ -166,10 +166,12 @@ func TestVendorVFIOConfigurePacksPlacement(t *testing.T) { sysfs, calls := newPackingSysfs(t, testHeterogeneousQuery) // The driver put the new 2Q at the lowest free placement, 0, which // overlaps the 4Q placement at 0. Placement 6 overlaps none. The clean - // sibling occupies nothing. + // sibling occupies nothing, and the 2Q at 6 is on another GPU. sysfs.addPlacedVF(t, "0000:82:00.4", "42", "0", 0) sysfs.addPlacedVF(t, "0000:82:00.5", "43", testL40S2Q, 4) sysfs.addVF(t, "0000:82:00.0", "0000:82:00.7", "45", "0", testCreatableTypes) + sysfs.addVF(t, "0000:83:00.0", "0000:83:00.4", "46", testL40S2Q, testCreatableTypes) + require.NoError(t, os.WriteFile(filepath.Join(sysfs.pciDevicesPath, "0000:83:00.4", "nvidia", "placement_id"), []byte("6"), 0644)) require.NoError(t, sysfs.configure(context.Background(), "0000:82:00.4", testL40S2Q)) assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "placement_id"), "6") From ccb02826d8c69720a52828ed04685fa24d180481 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 16 Oct 2026 10:11:49 +0000 Subject: [PATCH 17/19] Test that a failed vGPU placement load is cached --- lib/devices/vgpu_placement_linux_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lib/devices/vgpu_placement_linux_test.go b/lib/devices/vgpu_placement_linux_test.go index a02d1628c..b37b24cb8 100644 --- a/lib/devices/vgpu_placement_linux_test.go +++ b/lib/devices/vgpu_placement_linux_test.go @@ -182,6 +182,21 @@ func TestVendorVFIOConfigurePacksPlacement(t *testing.T) { assert.Equal(t, 2, *calls, "geometry is loaded once per GPU") } +func TestVendorVFIOConfigureCachesFailedPlacementLoad(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + calls := 0 + sysfs.nvidiaSMIFunc = func(context.Context, ...string) ([]byte, error) { + calls++ + return nil, errors.New("exit status 9") + } + sysfs.addPlacedVF(t, "0000:82:00.4", "42", "0", 0) + sysfs.addPlacedVF(t, "0000:82:00.5", "43", "0", 0) + + require.NoError(t, sysfs.configure(context.Background(), "0000:82:00.4", testL40S2Q)) + require.NoError(t, sysfs.configure(context.Background(), "0000:82:00.5", testL40S2Q)) + assert.Equal(t, 1, calls, "a failed load is cached too") +} + func TestVendorVFIOConfigureKeepsDriverPlacement(t *testing.T) { tests := []struct { name string From 8a3fbc71165bbef3a3afcc9453ba9edeba9399a3 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 16 Oct 2026 10:39:02 +0000 Subject: [PATCH 18/19] Drop redundant defensive code from vGPU placement packing --- lib/devices/vgpu_placement_linux.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/devices/vgpu_placement_linux.go b/lib/devices/vgpu_placement_linux.go index 9ce71ce5b..9cd0341aa 100644 --- a/lib/devices/vgpu_placement_linux.go +++ b/lib/devices/vgpu_placement_linux.go @@ -105,7 +105,6 @@ func (s vendorVFIOSysfs) placementGeometry(ctx context.Context, parentGPU string geometry, err := s.loadPlacementGeometry(ctx, parentGPU) if err != nil { logger.FromContext(ctx).WarnContext(ctx, "vGPU placement packing disabled for GPU", "gpu", parentGPU, "error", err) - geometry = vgpuPlacementGeometry{} } geometry.loadedAt = time.Now() vgpuPlacements[key] = geometry @@ -232,9 +231,7 @@ func choosePlacement(types map[string]vgpuPlacementType, occupied []placementSpa } best, bestBlocked := current, blocked(current) - ids := append([]int(nil), requested.ids...) - sort.Ints(ids) - for _, id := range ids { + for _, id := range requested.ids { if !free(placementSpan{start: id, size: requested.size}) { continue } From f87d76fa4c37f0d354de96e2843792953556dfe5 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 16 Oct 2026 10:47:15 +0000 Subject: [PATCH 19/19] Keep sorting placement IDs before choosing one --- lib/devices/vgpu_placement_linux.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/devices/vgpu_placement_linux.go b/lib/devices/vgpu_placement_linux.go index 9cd0341aa..ef78bf161 100644 --- a/lib/devices/vgpu_placement_linux.go +++ b/lib/devices/vgpu_placement_linux.go @@ -231,7 +231,9 @@ func choosePlacement(types map[string]vgpuPlacementType, occupied []placementSpa } best, bestBlocked := current, blocked(current) - for _, id := range requested.ids { + ids := append([]int(nil), requested.ids...) + sort.Ints(ids) + for _, id := range ids { if !free(placementSpan{start: id, size: requested.size}) { continue }