Place vGPU profiles the driver hides from creatable lists

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/distribution/reference v0.6.0
 	github.com/docker/distribution v2.8.3+incompatible
 	github.com/docker/go-units v0.5.0
+	github.com/ebitengine/purego v0.11.2
 	github.com/getkin/kin-openapi v0.133.0
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-chi/chi/v5 v5.3.0

go.sum

@@ -176,6 +176,8 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4
 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/ebitengine/purego v0.11.2 h1:WWiTx8rD8zYdOwSj8O/NiNbsOvSAgJMTlyV1XlbUSeU=
+github.com/ebitengine/purego v0.11.2/go.mod h1:DCHPP08djqhNSoTfImcnHYQRZmd0qhakvrozqaEYhGQ=
 github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA=
 github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU=
 github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ=

lib/devices/GPU.md

@@ -59,6 +59,8 @@ curl -s http://localhost:4973/resources | jq .gpu
 }
 ```
 
+On vendor VFIO hosts, `available` is how many more vGPUs of that profile fit if only that profile is added: per GPU, the smaller of its free placements for the profile and its free, non-quarantined VFs. Counts for different profiles are not additive, because their placements overlap.
+
 ### Creating an Instance with vGPU
 
 Request a vGPU by specifying the profile name:
@@ -92,10 +94,10 @@ On an mdev host, the response also includes the assigned mdev UUID:
 
 ### Ephemeral vGPU Lifecycle
 
-vGPU assignments are created on instance start and released on stop or delete. Hypeman creates/removes an mdev on mdev hosts. On vendor VFIO hosts, it first reserves a VF by persisting its path in instance metadata, then writes the profile ID to `current_vgpu_type`. Metadata is the authoritative VF claim.
+vGPU assignments are created on instance start and released on stop or delete. Hypeman creates/removes an mdev on mdev hosts. On vendor VFIO hosts, it selects a VF on a GPU with a free placement for the profile and writes the profile ID to `current_vgpu_type` under the allocation lock, so the placement is taken before the next claim checks for one, then reserves the VF by persisting its path in instance metadata. Metadata is the authoritative VF claim; a configured VF without one is dirty and is reset by the reconciler or the allocator.
 
 ```
-Instance Create → Persist VF claim → Configure profile → Attach VF to VM → Instance Running
+Instance Create → Configure profile → Persist VF claim → Attach VF to VM → Instance Running
 Instance Stop/Delete → Reset profile → Remove VF claim → VF available again
 ```
 
@@ -104,7 +106,9 @@ itself down and wait for it. Delete normally caps that wait at 2s, but guests
 with a vGPU or passthrough device get 5s so the guest can unload its GPU driver
 before the hypervisor is signalled.
 
-Hypeman reconciles metadata claims once at startup and every minute afterward, skipping hosts without GPUs. A claim whose VMM is confirmed dead is reset before the claim is removed. An ambiguous hypervisor ownership check preserves the claim, logs a warning, and increments `hypeman_instances_vgpu_liveness_uncertain_total`; the create and start cleanup paths preserve and count the same way. mdev hosts also sweep orphaned device-level assignments. Vendor VFIO hosts repair an unclaimed dirty VF when the allocator next selects it; repair checks for open VFIO handles before resetting `current_vgpu_type`. The allocator prefers VFs that are already clean, and a dirty VF that refuses its reset is skipped in favor of another candidate.
+Hypeman reconciles metadata claims once at startup and every minute afterward, skipping hosts without GPUs. A claim whose VMM is confirmed dead is reset before the claim is removed. An ambiguous hypervisor ownership check preserves the claim, logs a warning, and increments `hypeman_instances_vgpu_liveness_uncertain_total`; the create and start cleanup paths preserve and count the same way. mdev hosts also sweep orphaned device-level assignments. Vendor VFIO hosts reset unclaimed dirty VFs on every reconcile pass, which covers a restart between a claim writing the type and saving the claim, or a failed claim whose reset also failed, but also clears a type written to `current_vgpu_type` by hand unless the VF has an open VFIO handle. A reset that is refused is retried on the next pass and increments `hypeman_instances_vgpu_reconcile_failures_total{stage="reset_unclaimed_vfs"}`. The allocator also repairs an unclaimed dirty VF when it next selects it; repair checks for open VFIO handles before resetting `current_vgpu_type`. The allocator prefers VFs that are already clean, and a dirty VF that refuses its reset is skipped in favor of another candidate.
+
+Vendor VFIO placement and `/resources` both read the driver's creatable placements through NVML (`nvmlDeviceGetVgpuTypeCreatablePlacements`, the query behind `nvidia-smi vgpu -c -v`), which list the free placement IDs of every profile each GPU can still create. Hypeman loads `libnvidia-ml.so.1` at runtime on first use and keeps it initialized, which holds the GPU device nodes open (stop hypeman before the SR-IOV cycle below); a query for every profile on two L40S GPUs takes about 20 ms. A free VF can take any profile its GPU lists there, and a claim only writes a type the driver has just reported a free placement for; if that configure still fails, the VF is reset and the claim fails without trying another GPU. The per-VF `creatable_vgpu_types` lists are not used for this because in mixed-size (heterogeneous) mode they under-report: the driver drops a profile from them once its GPU holds as many vGPUs as that profile's equal-size instance limit (12 for L40S-4Q), even while a placement for it remains. If NVML cannot be loaded or initialized or takes longer than 5 s, placement and `/resources` fall back to those lists for every GPU; if it fails for one GPU, or reports a GPU not in mixed-size mode, only that GPU falls back. Profiles the lists hide are refused on a GPU that has fallen back until NVML recovers. Queries run one at a time and wait for each other within the same 5 s, so callers queued behind a hung query never enter the driver. A query that times out is abandoned rather than waited for, since a call stuck in the driver cannot be interrupted; until it returns, queries skip NVML and use the lists directly, so a hung driver costs one timeout rather than one per claim. Each fallback increments `hypeman_resources_gpu_nvml_fallbacks_total` with `reason` `error`, `equal_size` (every GPU NVML failed for is in equal-size mode, as after a driver reload or GPU reset), `timeout` or `stuck`. A warning is logged when a fallback starts or its error changes, and an info line once NVML answers for every GPU again, rather than on every query. Admission only checks free VFs, so it does not query NVML. `/resources` does not query it either: it counts profiles from cached placements, loaded at startup, refreshed in the background after every vendor VFIO configure or reset, and refreshed again on every reconcile pass. Free and quarantined VFs are read fresh on every request, so a claim or release changes the counts at once; the placements it frees or takes on a mixed-size GPU show once the background refresh lands, about 20 ms later.
 
 ### Hypervisor Support
 
@@ -394,7 +398,8 @@ SR-IOV on the parent GPU (this destroys and recreates all of its VFs, so it
 requires no vGPU assignments on that GPU). Quiescing the services that hold
 the GPU open is not optional: with `nv-hostengine`/`dcgm-exporter` or
 `nvidia-persistenced` attached, `sriov-manage -d` fails with `Cannot obtain
-unbindLock` on first contact.
+unbindLock` on first contact. Hypeman holds the same device nodes through
+NVML, so stop it too.
 
 Any manual edit to `vf-health.json` needs an immediate hypeman restart: the
 store loads only at startup, and a failure report landing first re-persists
@@ -414,14 +419,15 @@ GPU; once none remain, run the cycle below.
 
 ```bash
 # 1. Quiesce the services holding the GPU (required for the unbind lock).
-systemctl stop nvidia-dcgm-exporter nvidia-dcgm nvidia-persistenced
+#    Hypeman holds it through NVML; stopping it leaves running VMs alone.
+systemctl stop hypeman nvidia-dcgm-exporter nvidia-dcgm nvidia-persistenced
 
 # 2. Cycle SR-IOV on the parent GPU.
 /usr/lib/nvidia/sriov-manage -d <parent-gpu-pci-addr>
 /usr/lib/nvidia/sriov-manage -e <parent-gpu-pci-addr>
 
 # 3. Restart the quiesced services.
-systemctl start nvidia-persistenced nvidia-dcgm nvidia-dcgm-exporter
+systemctl start nvidia-persistenced nvidia-dcgm nvidia-dcgm-exporter hypeman
 ```
 
 After the cycle, remove the card's entries from `vf-health.json`, restart,
@@ -442,6 +448,7 @@ Check the files for the framework detected on the host:
 cat /sys/class/mdev_bus/*/mdev_supported_types/*/available_instances
 
 # vendor VFIO
+nvidia-smi vgpu -c -v
 cat /sys/bus/pci/devices/*/nvidia/creatable_vgpu_types
 cat /sys/bus/pci/devices/*/nvidia/current_vgpu_type
 ```

lib/devices/mdev_darwin.go

@@ -40,6 +40,14 @@ func ListVendorVFIOProfileTypes(vfs []VirtualFunction) (map[string][]VGPUProfile
 	return nil, ErrVGPUNotSupportedOnMacOS
 }
 
+// RefreshVGPUCapacity does nothing on macOS.
+func RefreshVGPUCapacity() {}
+
+// NVMLFallbacks returns nil on macOS.
+func NVMLFallbacks() map[string]int64 {
+	return nil
+}
+
 func ConfigureVGPU(ctx context.Context, vfAddress, profileType string) error {
 	return ErrVGPUNotSupportedOnMacOS
 }

lib/devices/nvml_linux.go

@@ -0,0 +1,231 @@
+//go:build linux
+
+package devices
+
+import (
+	"errors"
+	"fmt"
+	"runtime"
+	"strconv"
+	"strings"
+	"sync"
+	"unsafe"
+
+	"github.com/ebitengine/purego"
+)
+
+// nvmlLibrary is the NVML library the NVIDIA driver installs. It is loaded
+// at runtime, so hosts without the driver never need it.
+var nvmlLibrary = "libnvidia-ml.so.1"
+
+const (
+	nvmlSuccess          = 0
+	nvmlInsufficientSize = 7
+	nvmlFeatureEnabled   = 1
+	// nvmlHeterogeneousPlacements is NVML_VGPU_PGPU_HETEROGENEOUS_MODE.
+	nvmlHeterogeneousPlacements = 0
+)
+
+// nvmlPlacementList is nvmlVgpuPlacementList_v2_t.
+type nvmlPlacementList struct {
+	version       uint32
+	placementSize uint32
+	count         uint32
+	placementIDs  *uint32
+	mode          uint32
+}
+
+// nvmlHeterogeneousMode is nvmlVgpuHeterogeneousMode_v1_t.
+type nvmlHeterogeneousMode struct {
+	version uint32
+	mode    uint32
+}
+
+// nvmlStructVersion is NVML_STRUCT_VERSION.
+func nvmlStructVersion(size uintptr, version uint32) uint32 {
+	return uint32(size) | version<<24
+}
+
+type nvmlFuncs struct {
+	errorString                  func(rc int32) string
+	deviceGetHandleByPCIBusID    func(busID string, device *uintptr) int32
+	deviceGetHeterogeneousMode   func(device uintptr, mode *nvmlHeterogeneousMode) int32
+	deviceGetSupportedVGPUs      func(device uintptr, count *uint32, typeIDs *uint32) int32
+	deviceGetCreatablePlacements func(device uintptr, typeID uint32, list *nvmlPlacementList) int32
+	vgpuTypeGetName              func(typeID uint32, name *byte, size *uint32) int32
+}
+
+var (
+	nvmlMu sync.Mutex
+	nvml   *nvmlFuncs
+)
+
+// openNVML loads and initializes NVML on first use and keeps it for the life
+// of the process. A failure is not kept, so a driver that loads later is
+// picked up by the next query.
+func openNVML() (*nvmlFuncs, error) {
+	nvmlMu.Lock()
+	defer nvmlMu.Unlock()
+	if nvml != nil {
+		return nvml, nil
+	}
+	lib, err := purego.Dlopen(nvmlLibrary, purego.RTLD_NOW|purego.RTLD_GLOBAL)
+	if err != nil {
+		return nil, fmt.Errorf("load %s: %w", nvmlLibrary, err)
+	}
+	var f nvmlFuncs
+	var initialize func() int32
+	// purego.RegisterLibFunc panics on a missing symbol, which an older
+	// driver could cause, so each symbol is looked up first.
+	for name, fn := range map[string]any{
+		"nvmlInit_v2":                              &initialize,
+		"nvmlErrorString":                          &f.errorString,
+		"nvmlDeviceGetHandleByPciBusId_v2":         &f.deviceGetHandleByPCIBusID,
+		"nvmlDeviceGetVgpuHeterogeneousMode":       &f.deviceGetHeterogeneousMode,
+		"nvmlDeviceGetSupportedVgpus":              &f.deviceGetSupportedVGPUs,
+		"nvmlDeviceGetVgpuTypeCreatablePlacements": &f.deviceGetCreatablePlacements,
+		"nvmlVgpuTypeGetName":                      &f.vgpuTypeGetName,
+	} {
+		sym, err := purego.Dlsym(lib, name)
+		if err != nil {
+			purego.Dlclose(lib)
+			return nil, fmt.Errorf("load %s from %s: %w", name, nvmlLibrary, err)
+		}
+		purego.RegisterFunc(fn, sym)
+	}
+	if rc := initialize(); rc != nvmlSuccess {
+		err := fmt.Errorf("nvmlInit_v2: %s", f.errorString(rc))
+		purego.Dlclose(lib)
+		return nil, err
+	}
+	nvml = &f
+	return nvml, nil
+}
+
+// nvmlCreatableProfiles asks NVML which profiles each GPU can still create
+// and at which placements, keyed by GPU address. This is the query behind
+// nvidia-smi vgpu -c -v. It fails for a GPU not in mixed-size mode, so
+// such a GPU keeps the sysfs lists. A GPU that fails is left out of the
+// result and its error joined into the returned one.
+func nvmlCreatableProfiles(gpus []string) (map[string][]creatableProfile, error) {
+	f, err := openNVML()
+	if err != nil {
+		return nil, err
+	}
+	byGPU := make(map[string][]creatableProfile, len(gpus))
+	var errs []error
+	for _, gpu := range gpus {
+		profiles, err := f.creatableProfiles(gpu)
+		if err != nil {
+			errs = append(errs, fmt.Errorf("GPU %s: %w", gpu, err))
+			continue
+		}
+		byGPU[gpu] = profiles
+	}
+	return byGPU, errors.Join(errs...)
+}
+
+func (f *nvmlFuncs) creatableProfiles(gpu string) ([]creatableProfile, error) {
+	var device uintptr
+	if err := f.check("nvmlDeviceGetHandleByPciBusId_v2", f.deviceGetHandleByPCIBusID(gpu, &device)); err != nil {
+		return nil, err
+	}
+	mode := nvmlHeterogeneousMode{version: nvmlStructVersion(unsafe.Sizeof(nvmlHeterogeneousMode{}), 1)}
+	if err := f.check("nvmlDeviceGetVgpuHeterogeneousMode", f.deviceGetHeterogeneousMode(device, &mode)); err != nil {
+		return nil, err
+	}
+	if mode.mode != nvmlFeatureEnabled {
+		return nil, errNVMLEqualSize
+	}
+	typeIDs, err := f.supportedTypes(device)
+	if err != nil {
+		return nil, err
+	}
+	profiles := make([]creatableProfile, 0)
+	for _, typeID := range typeIDs {
+		placements, err := f.creatablePlacements(device, typeID)
+		if err != nil {
+			return nil, err
+		}
+		if len(placements) == 0 {
+			continue
+		}
+		name, err := f.typeName(typeID)
+		if err != nil {
+			return nil, err
+		}
+		profiles = append(profiles, creatableProfile{
+			VGPUProfileType: VGPUProfileType{
+				TypeName:      strconv.FormatUint(uint64(typeID), 10),
+				Name:          name,
+				FramebufferMB: framebufferFromProfileName(name),
+			},
+			placements: placements,
+		})
+	}
+	return profiles, nil
+}
+
+func (f *nvmlFuncs) supportedTypes(device uintptr) ([]uint32, error) {
+	typeIDs := make([]uint32, 64)
+	count := uint32(len(typeIDs))
+	rc := f.deviceGetSupportedVGPUs(device, &count, &typeIDs[0])
+	if rc == nvmlInsufficientSize {
+		typeIDs = make([]uint32, count)
+		rc = f.deviceGetSupportedVGPUs(device, &count, &typeIDs[0])
+	}
+	if err := f.check("nvmlDeviceGetSupportedVgpus", rc); err != nil {
+		return nil, err
+	}
+	return typeIDs[:count], nil
+}
+
+func (f *nvmlFuncs) creatablePlacements(device uintptr, typeID uint32) ([]int, error) {
+	ids := make([]uint32, 64)
+	list, rc := f.placementList(device, typeID, ids)
+	if rc == nvmlInsufficientSize {
+		ids = make([]uint32, list.count)
+		list, rc = f.placementList(device, typeID, ids)
+	}
+	if err := f.check("nvmlDeviceGetVgpuTypeCreatablePlacements", rc); err != nil {
+		return nil, err
+	}
+	placements := make([]int, 0, list.count)
+	for _, id := range ids[:list.count] {
+		placements = append(placements, int(id))
+	}
+	return placements, nil
+}
+
+func (f *nvmlFuncs) placementList(device uintptr, typeID uint32, ids []uint32) (nvmlPlacementList, int32) {
+	// The list carries a pointer to ids, so ids must not move while NVML
+	// writes to it.
+	var pinner runtime.Pinner
+	pinner.Pin(&ids[0])
+	defer pinner.Unpin()
+	list := nvmlPlacementList{
+		version:      nvmlStructVersion(unsafe.Sizeof(nvmlPlacementList{}), 2),
+		count:        uint32(len(ids)),
+		placementIDs: &ids[0],
+		mode:         nvmlHeterogeneousPlacements,
+	}
+	rc := f.deviceGetCreatablePlacements(device, typeID, &list)
+	return list, rc
+}
+
+func (f *nvmlFuncs) typeName(typeID uint32) (string, error) {
+	name := make([]byte, 96)
+	size := uint32(len(name))
+	if err := f.check("nvmlVgpuTypeGetName", f.vgpuTypeGetName(typeID, &name[0], &size)); err != nil {
+		return "", err
+	}
+	value, _, _ := strings.Cut(string(name), "\x00")
+	return value, nil
+}
+
+func (f *nvmlFuncs) check(call string, rc int32) error {
+	if rc == nvmlSuccess {
+		return nil
+	}
+	return fmt.Errorf("%s: %s", call, f.errorString(rc))
+}

lib/devices/types.go

@@ -121,7 +121,7 @@ type MdevDevice struct {
 type GPUProfile struct {
 	Name          string `json:"name"`           // user-facing name, e.g., "L40S-1Q"
 	FramebufferMB int    `json:"framebuffer_mb"` // frame buffer size in MB
-	Available     int    `json:"available"`      // number of VFs that can create this profile
+	Available     int    `json:"available"`      // more instances that fit if only this profile is added
 }
 
 // PassthroughDevice describes a physical GPU available for passthrough