From ab8ee1e42e600fa4cfc3059c53426a6205fab8f4 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:29:35 +0000 Subject: [PATCH 01/16] Place vGPU profiles the driver hides from creatable lists In mixed-size mode the driver drops a profile from creatable_vgpu_types once its GPU holds as many vGPUs as that profile's equal-size limit (12 for L40S-4Q), even while a placement for it is free, so 4Q claims failed on busy GPUs and /resources under-reported. Ask NVML for each GPU's creatable placements instead (the query behind nvidia-smi vgpu -c -v), which stays accurate in that state. libnvidia-ml is loaded at runtime through purego, so builds need no cgo and hosts without the driver never load it; a query takes about 20 ms. Placement picks a GPU with a free placement for the profile, and /resources counts per GPU the smaller of free placements and free healthy VFs. Both fall back to the sysfs lists if NVML cannot be loaded, fails, times out, or a GPU is not in mixed-size mode. A timed-out query is abandoned rather than waited for, and queries skip NVML until it returns, so a hung driver costs one timeout rather than one per claim. Fallbacks are counted by reason in hypeman_resources_gpu_nvml_fallbacks_total. Admission only needs the free-VF count, so it no longer lists profiles. The type is now written under the allocation lock, before the claim is saved, so the next claim's query sees the placement as taken. A failed configure or claim save resets the VF. --- go.mod | 1 + go.sum | 2 + lib/devices/GPU.md | 9 +- lib/devices/mdev_darwin.go | 5 + lib/devices/nvml_linux.go | 225 +++++++ lib/devices/types.go | 2 +- lib/devices/vendor_vfio_linux.go | 69 +- lib/devices/vendor_vfio_linux_test.go | 4 + lib/devices/vgpu_creatable_linux.go | 121 ++++ lib/devices/vgpu_creatable_linux_test.go | 201 ++++++ lib/instances/create.go | 2 + lib/instances/start.go | 2 + lib/instances/vgpu.go | 14 + lib/instances/vgpu_linux_test.go | 98 +++ lib/oapi/oapi.go | 777 +++++++++++------------ lib/otel/README.md | 3 +- lib/resources/gpu.go | 13 +- lib/resources/gpu_test.go | 28 +- lib/resources/monitoring.go | 16 +- lib/resources/monitoring_test.go | 22 +- lib/resources/resource.go | 8 +- openapi.yaml | 2 +- 22 files changed, 1205 insertions(+), 419 deletions(-) create mode 100644 lib/devices/nvml_linux.go create mode 100644 lib/devices/vgpu_creatable_linux.go create mode 100644 lib/devices/vgpu_creatable_linux_test.go diff --git a/go.mod b/go.mod index 6b7ea0e6e..f16d68c01 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/distribution/reference v0.6.0 github.com/docker/distribution v2.8.3+incompatible github.com/docker/go-units v0.5.0 + github.com/ebitengine/purego v0.11.2 github.com/getkin/kin-openapi v0.133.0 github.com/ghodss/yaml v1.0.0 github.com/go-chi/chi/v5 v5.3.0 diff --git a/go.sum b/go.sum index 9d2531db9..851573b04 100644 --- a/go.sum +++ b/go.sum @@ -176,6 +176,8 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ebitengine/purego v0.11.2 h1:WWiTx8rD8zYdOwSj8O/NiNbsOvSAgJMTlyV1XlbUSeU= +github.com/ebitengine/purego v0.11.2/go.mod h1:DCHPP08djqhNSoTfImcnHYQRZmd0qhakvrozqaEYhGQ= github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 683a506a6..c7d4af289 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -59,6 +59,8 @@ curl -s http://localhost:4973/resources | jq .gpu } ``` +On vendor VFIO hosts, `available` is how many more vGPUs of that profile fit if only that profile is added: per GPU, the smaller of its free placements for the profile and its free, non-quarantined VFs. Counts for different profiles are not additive, because their placements overlap. + ### Creating an Instance with vGPU Request a vGPU by specifying the profile name: @@ -92,10 +94,10 @@ On an mdev host, the response also includes the assigned mdev UUID: ### Ephemeral vGPU Lifecycle -vGPU assignments are created on instance start and released on stop or delete. Hypeman creates/removes an mdev on mdev hosts. On vendor VFIO hosts, it first reserves a VF by persisting its path in instance metadata, then writes the profile ID to `current_vgpu_type`. Metadata is the authoritative VF claim. +vGPU assignments are created on instance start and released on stop or delete. Hypeman creates/removes an mdev on mdev hosts. On vendor VFIO hosts, it selects a VF on a GPU with a free placement for the profile and writes the profile ID to `current_vgpu_type` under the allocation lock, so the placement is taken before the next claim checks for one, then reserves the VF by persisting its path in instance metadata. Metadata is the authoritative VF claim; a configured VF without one is dirty and is reset by the allocator. ``` -Instance Create → Persist VF claim → Configure profile → Attach VF to VM → Instance Running +Instance Create → Configure profile → Persist VF claim → Attach VF to VM → Instance Running Instance Stop/Delete → Reset profile → Remove VF claim → VF available again ``` @@ -106,6 +108,8 @@ before the hypervisor is signalled. Hypeman reconciles metadata claims once at startup and every minute afterward, skipping hosts without GPUs. A claim whose VMM is confirmed dead is reset before the claim is removed. An ambiguous hypervisor ownership check preserves the claim, logs a warning, and increments `hypeman_instances_vgpu_liveness_uncertain_total`; the create and start cleanup paths preserve and count the same way. mdev hosts also sweep orphaned device-level assignments. Vendor VFIO hosts repair an unclaimed dirty VF when the allocator next selects it; repair checks for open VFIO handles before resetting `current_vgpu_type`. The allocator prefers VFs that are already clean, and a dirty VF that refuses its reset is skipped in favor of another candidate. +Vendor VFIO placement and `/resources` both read the driver's creatable placements through NVML (`nvmlDeviceGetVgpuTypeCreatablePlacements`, the query behind `nvidia-smi vgpu -c -v`), which list the free placement IDs of every profile each GPU can still create. Hypeman loads `libnvidia-ml.so.1` at runtime on first use and keeps it initialized; a query for every profile on two L40S GPUs takes about 20 ms. A free VF can take any profile its GPU lists there, and a claim only writes a type the driver has just reported a free placement for; if that configure still fails, the VF is reset and the claim fails without trying another GPU. The per-VF `creatable_vgpu_types` lists are not used for this because in mixed-size (heterogeneous) mode they under-report: the driver drops a profile from them once its GPU holds as many vGPUs as that profile's equal-size instance limit (12 for L40S-4Q), even while a placement for it remains. If NVML cannot be loaded or initialized, fails, takes longer than 5 s, or reports a GPU not in mixed-size mode, placement and `/resources` fall back to those lists, so profiles they hide are refused until it recovers. A query that times out is abandoned rather than waited for, since a call stuck in the driver cannot be interrupted; until it returns, queries skip NVML and use the lists directly, so a hung driver costs one timeout rather than one per claim. Each fallback logs a warning and increments `hypeman_resources_gpu_nvml_fallbacks_total` with `reason` `error`, `timeout` or `stuck`. Admission only checks free VFs, so it does not query NVML. + ### Hypervisor Support Hypervisor selection for vGPU instances is caller policy; hypeman does not enforce it. In practice **QEMU is the only hypervisor with working vGPU support**: @@ -442,6 +446,7 @@ Check the files for the framework detected on the host: cat /sys/class/mdev_bus/*/mdev_supported_types/*/available_instances # vendor VFIO +nvidia-smi vgpu -c -v cat /sys/bus/pci/devices/*/nvidia/creatable_vgpu_types cat /sys/bus/pci/devices/*/nvidia/current_vgpu_type ``` diff --git a/lib/devices/mdev_darwin.go b/lib/devices/mdev_darwin.go index f0f009d34..a5d76c1a0 100644 --- a/lib/devices/mdev_darwin.go +++ b/lib/devices/mdev_darwin.go @@ -40,6 +40,11 @@ func ListVendorVFIOProfileTypes(vfs []VirtualFunction) (map[string][]VGPUProfile return nil, ErrVGPUNotSupportedOnMacOS } +// NVMLFallbacks returns nil on macOS. +func NVMLFallbacks() map[string]int64 { + return nil +} + func ConfigureVGPU(ctx context.Context, vfAddress, profileType string) error { return ErrVGPUNotSupportedOnMacOS } diff --git a/lib/devices/nvml_linux.go b/lib/devices/nvml_linux.go new file mode 100644 index 000000000..20943ec4c --- /dev/null +++ b/lib/devices/nvml_linux.go @@ -0,0 +1,225 @@ +//go:build linux + +package devices + +import ( + "fmt" + "runtime" + "strconv" + "strings" + "sync" + "unsafe" + + "github.com/ebitengine/purego" +) + +// nvmlLibrary is the NVML library the NVIDIA driver installs. It is loaded +// at runtime, so hosts without the driver never need it. +var nvmlLibrary = "libnvidia-ml.so.1" + +const ( + nvmlSuccess = 0 + nvmlInsufficientSize = 7 + nvmlFeatureEnabled = 1 +) + +// nvmlPlacementList is nvmlVgpuPlacementList_v2_t. +type nvmlPlacementList struct { + version uint32 + placementSize uint32 + count uint32 + placementIDs *uint32 + mode uint32 +} + +// nvmlHeterogeneousMode is nvmlVgpuHeterogeneousMode_v1_t. +type nvmlHeterogeneousMode struct { + version uint32 + mode uint32 +} + +// nvmlStructVersion is NVML_STRUCT_VERSION. +func nvmlStructVersion(size uintptr, version uint32) uint32 { + return uint32(size) | version<<24 +} + +type nvmlFuncs struct { + errorString func(rc int32) string + deviceGetHandleByPCIBusID func(busID string, device *uintptr) int32 + deviceGetHeterogeneousMode func(device uintptr, mode *nvmlHeterogeneousMode) int32 + deviceGetSupportedVGPUs func(device uintptr, count *uint32, typeIDs *uint32) int32 + deviceGetCreatablePlacements func(device uintptr, typeID uint32, list *nvmlPlacementList) int32 + vgpuTypeGetName func(typeID uint32, name *byte, size *uint32) int32 +} + +var ( + nvmlMu sync.Mutex + nvml *nvmlFuncs +) + +// openNVML loads and initializes NVML on first use and keeps it for the life +// of the process. A failure is not kept, so a driver that loads later is +// picked up by the next query. +func openNVML() (*nvmlFuncs, error) { + nvmlMu.Lock() + defer nvmlMu.Unlock() + if nvml != nil { + return nvml, nil + } + lib, err := purego.Dlopen(nvmlLibrary, purego.RTLD_NOW|purego.RTLD_GLOBAL) + if err != nil { + return nil, fmt.Errorf("load %s: %w", nvmlLibrary, err) + } + var f nvmlFuncs + var initialize func() int32 + // purego.RegisterLibFunc panics on a missing symbol, which an older + // driver could cause, so each symbol is looked up first. + for name, fn := range map[string]any{ + "nvmlInit_v2": &initialize, + "nvmlErrorString": &f.errorString, + "nvmlDeviceGetHandleByPciBusId_v2": &f.deviceGetHandleByPCIBusID, + "nvmlDeviceGetVgpuHeterogeneousMode": &f.deviceGetHeterogeneousMode, + "nvmlDeviceGetSupportedVgpus": &f.deviceGetSupportedVGPUs, + "nvmlDeviceGetVgpuTypeCreatablePlacements": &f.deviceGetCreatablePlacements, + "nvmlVgpuTypeGetName": &f.vgpuTypeGetName, + } { + sym, err := purego.Dlsym(lib, name) + if err != nil { + purego.Dlclose(lib) + return nil, fmt.Errorf("load %s from %s: %w", name, nvmlLibrary, err) + } + purego.RegisterFunc(fn, sym) + } + if rc := initialize(); rc != nvmlSuccess { + err := fmt.Errorf("nvmlInit_v2: %s", f.errorString(rc)) + purego.Dlclose(lib) + return nil, err + } + nvml = &f + return nvml, nil +} + +// nvmlCreatableProfiles asks NVML which profiles each GPU can still create +// and at which placements, keyed by GPU address. This is the query behind +// nvidia-smi vgpu -c -v. It fails for a GPU not in mixed-size mode, so +// equal-size hosts keep the sysfs lists. +func nvmlCreatableProfiles(gpus []string) (map[string][]creatableProfile, error) { + f, err := openNVML() + if err != nil { + return nil, err + } + byGPU := make(map[string][]creatableProfile, len(gpus)) + for _, gpu := range gpus { + profiles, err := f.creatableProfiles(gpu) + if err != nil { + return nil, fmt.Errorf("GPU %s: %w", gpu, err) + } + byGPU[gpu] = profiles + } + return byGPU, nil +} + +func (f *nvmlFuncs) creatableProfiles(gpu string) ([]creatableProfile, error) { + var device uintptr + // sysfs addresses have a four-digit domain; NVML bus IDs have eight. + if err := f.check("nvmlDeviceGetHandleByPciBusId_v2", f.deviceGetHandleByPCIBusID("0000"+gpu, &device)); err != nil { + return nil, err + } + mode := nvmlHeterogeneousMode{version: nvmlStructVersion(unsafe.Sizeof(nvmlHeterogeneousMode{}), 1)} + if err := f.check("nvmlDeviceGetVgpuHeterogeneousMode", f.deviceGetHeterogeneousMode(device, &mode)); err != nil { + return nil, err + } + if mode.mode != nvmlFeatureEnabled { + return nil, fmt.Errorf("not in mixed-size vGPU mode") + } + typeIDs, err := f.supportedTypes(device) + if err != nil { + return nil, err + } + profiles := make([]creatableProfile, 0) + for _, typeID := range typeIDs { + placements, err := f.creatablePlacements(device, typeID) + if err != nil { + return nil, err + } + if len(placements) == 0 { + continue + } + name, err := f.typeName(typeID) + if err != nil { + return nil, err + } + profiles = append(profiles, creatableProfile{ + VGPUProfileType: VGPUProfileType{ + TypeName: strconv.FormatUint(uint64(typeID), 10), + Name: name, + FramebufferMB: framebufferFromProfileName(name), + }, + placements: placements, + }) + } + return profiles, nil +} + +func (f *nvmlFuncs) supportedTypes(device uintptr) ([]uint32, error) { + typeIDs := make([]uint32, 64) + count := uint32(len(typeIDs)) + rc := f.deviceGetSupportedVGPUs(device, &count, &typeIDs[0]) + if rc == nvmlInsufficientSize { + typeIDs = make([]uint32, count) + rc = f.deviceGetSupportedVGPUs(device, &count, &typeIDs[0]) + } + if err := f.check("nvmlDeviceGetSupportedVgpus", rc); err != nil { + return nil, err + } + return typeIDs[:count], nil +} + +func (f *nvmlFuncs) creatablePlacements(device uintptr, typeID uint32) ([]int, error) { + ids := make([]uint32, 64) + list, rc := f.placementList(device, typeID, ids) + if rc == nvmlInsufficientSize { + ids = make([]uint32, list.count) + list, rc = f.placementList(device, typeID, ids) + } + if err := f.check("nvmlDeviceGetVgpuTypeCreatablePlacements", rc); err != nil { + return nil, err + } + placements := make([]int, 0, list.count) + for _, id := range ids[:list.count] { + placements = append(placements, int(id)) + } + return placements, nil +} + +func (f *nvmlFuncs) placementList(device uintptr, typeID uint32, ids []uint32) (nvmlPlacementList, int32) { + // The list carries a pointer to ids, so ids must not move while NVML + // writes to it. + var pinner runtime.Pinner + pinner.Pin(&ids[0]) + defer pinner.Unpin() + list := nvmlPlacementList{ + version: nvmlStructVersion(unsafe.Sizeof(nvmlPlacementList{}), 2), + count: uint32(len(ids)), + placementIDs: &ids[0], + } + rc := f.deviceGetCreatablePlacements(device, typeID, &list) + return list, rc +} + +func (f *nvmlFuncs) typeName(typeID uint32) (string, error) { + name := make([]byte, 96) + size := uint32(len(name)) + if err := f.check("nvmlVgpuTypeGetName", f.vgpuTypeGetName(typeID, &name[0], &size)); err != nil { + return "", err + } + value, _, _ := strings.Cut(string(name), "\x00") + return value, nil +} + +func (f *nvmlFuncs) check(call string, rc int32) error { + if rc == nvmlSuccess { + return nil + } + return fmt.Errorf("%s: %s", call, f.errorString(rc)) +} diff --git a/lib/devices/types.go b/lib/devices/types.go index 67352132b..049bd3757 100644 --- a/lib/devices/types.go +++ b/lib/devices/types.go @@ -121,7 +121,7 @@ type MdevDevice struct { type GPUProfile struct { Name string `json:"name"` // user-facing name, e.g., "L40S-1Q" FramebufferMB int `json:"framebuffer_mb"` // frame buffer size in MB - Available int `json:"available"` // number of VFs that can create this profile + Available int `json:"available"` // more instances that fit if only this profile is added } // PassthroughDevice describes a physical GPU available for passthrough diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 2fb552095..95d74f1f7 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -27,6 +27,7 @@ type vendorVFIOSysfs struct { procPath string vfioDevicesPath string openVFIOPathsFunc func() (map[string]struct{}, error) + creatableFunc func(gpus []string) (map[string][]creatableProfile, error) } var hostVendorVFIO = vendorVFIOSysfs{ @@ -90,15 +91,42 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { return vfs, nil } -// listProfiles counts each free, non-quarantined VF advertising a type as -// one creatable instance, matching the driver-reported units that mdev sums -// through available_instances. This is a best-effort snapshot because +// listProfiles reports how many more vGPUs of each profile the host can +// take if only that profile is added: per GPU, the smaller of its free +// placements for the profile and its free, non-quarantined VFs. Counts for +// different profiles are not additive, because their placements overlap. +func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction, quarantined map[string]struct{}) ([]GPUProfile, error) { + creatable, err := s.creatableProfiles(vfs) + if err != nil { + recordNVMLFallback(err) + return s.listSysfsProfiles(vfs, quarantined) + } + freeVFs := make(map[string]int) + for _, vf := range vfs { + if _, bad := quarantined[vf.PCIAddress]; !vf.Allocated && !bad { + freeVFs[vf.ParentGPU]++ + } + } + profilesByType := make(map[string]VGPUProfileType) + available := make(map[string]int) + for gpu, profiles := range creatable { + for _, profile := range profiles { + profilesByType[profile.TypeName] = profile.VGPUProfileType + available[profile.TypeName] += min(len(profile.placements), freeVFs[gpu]) + } + } + return gpuProfiles(profilesByType, available), nil +} + +// listSysfsProfiles counts each free, non-quarantined VF advertising a type +// as one creatable instance, matching the driver-reported units that mdev +// sums through available_instances. This is a best-effort snapshot because // creating on one VF may revoke the type from siblings that share its GPU // framebuffer. -func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction, quarantined map[string]struct{}) ([]GPUProfile, error) { +func (s vendorVFIOSysfs) listSysfsProfiles(vfs []VirtualFunction, quarantined map[string]struct{}) ([]GPUProfile, error) { profilesByType := make(map[string]VGPUProfileType) creatableVFs := make(map[string]int) - profilesByVF, err := s.profileTypes(vfs) + profilesByVF, err := s.sysfsProfileTypes(vfs) if err != nil { return nil, err } @@ -111,7 +139,10 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction, quarantined map[str } } } + return gpuProfiles(profilesByType, creatableVFs), nil +} +func gpuProfiles(profilesByType map[string]VGPUProfileType, available map[string]int) []GPUProfile { metadata := make([]VGPUProfileType, 0, len(profilesByType)) for _, profile := range profilesByType { metadata = append(metadata, profile) @@ -123,13 +154,37 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction, quarantined map[str profiles = append(profiles, GPUProfile{ Name: profile.Name, FramebufferMB: profile.FramebufferMB, - Available: creatableVFs[profile.TypeName], + Available: available[profile.TypeName], }) } - return profiles, nil + return profiles } +// profileTypes returns the profiles each VF can be configured with. A free +// VF can take any profile its GPU has a free placement for, as NVML reports +// it, and a configured VF takes none. The sysfs creatable lists are only the +// fallback when NVML fails, because in mixed-size mode they hide profiles +// that still fit on a busy GPU. func (s vendorVFIOSysfs) profileTypes(vfs []VirtualFunction) (map[string][]VGPUProfileType, error) { + creatable, err := s.creatableProfiles(vfs) + if err != nil { + recordNVMLFallback(err) + return s.sysfsProfileTypes(vfs) + } + profilesByVF := make(map[string][]VGPUProfileType, len(vfs)) + for _, vf := range vfs { + profiles := make([]VGPUProfileType, 0) + if !vf.Allocated { + for _, profile := range creatable[vf.ParentGPU] { + profiles = append(profiles, profile.VGPUProfileType) + } + } + profilesByVF[vf.PCIAddress] = profiles + } + return profilesByVF, nil +} + +func (s vendorVFIOSysfs) sysfsProfileTypes(vfs []VirtualFunction) (map[string][]VGPUProfileType, error) { profilesByVF := make(map[string][]VGPUProfileType, len(vfs)) for _, vf := range vfs { profiles, err := s.readCreatableProfiles(vf.PCIAddress) diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 3c7438532..e57769cb9 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -4,6 +4,7 @@ package devices import ( "context" + "errors" "os" "path/filepath" "testing" @@ -153,6 +154,9 @@ func newTestVendorVFIOSysfs(t *testing.T) testVendorVFIOSysfs { pciDevicesPath: pci, procPath: proc, vfioDevicesPath: vfio, + creatableFunc: func([]string) (map[string][]creatableProfile, error) { + return nil, errors.New("NVML unavailable") + }, }} } diff --git a/lib/devices/vgpu_creatable_linux.go b/lib/devices/vgpu_creatable_linux.go new file mode 100644 index 000000000..5463766c1 --- /dev/null +++ b/lib/devices/vgpu_creatable_linux.go @@ -0,0 +1,121 @@ +//go:build linux + +package devices + +import ( + "errors" + "fmt" + "log/slog" + "sort" + "sync/atomic" + "time" +) + +// nvmlTimeout bounds the creatable-placements query, which takes about 20 ms +// on a two-GPU L40S host and stayed under 140 ms with every host core +// saturated. +var nvmlTimeout = 5 * time.Second + +var ( + errNVMLTimeout = errors.New("NVML query timed out") + errNVMLStuck = errors.New("an earlier NVML query is still running after its timeout") +) + +// nvmlStuck counts timed-out NVML queries that have not returned. +var nvmlStuck atomic.Int32 + +// nvmlFallbacks counts creatable-placement queries answered from the sysfs +// lists instead, by reason. +var nvmlFallbacks = map[string]*atomic.Int64{ + "error": {}, + "timeout": {}, + "stuck": {}, +} + +// NVMLFallbacks returns how many creatable-placement queries have fallen back +// to the sysfs lists since startup, by reason: NVML failed (error), timed out +// (timeout), or an earlier timed-out query had not returned yet (stuck). +func NVMLFallbacks() map[string]int64 { + counts := make(map[string]int64, len(nvmlFallbacks)) + for reason, count := range nvmlFallbacks { + counts[reason] = count.Load() + } + return counts +} + +func recordNVMLFallback(err error) { + reason := "error" + switch { + case errors.Is(err, errNVMLStuck): + reason = "stuck" + case errors.Is(err, errNVMLTimeout): + reason = "timeout" + } + nvmlFallbacks[reason].Add(1) + slog.Default().Warn("falling back to sysfs creatable vGPU types", "reason", reason, "error", err) +} + +// creatableProfile is a profile a GPU can still create, with the placement +// IDs a new vGPU of that profile could start at. +type creatableProfile struct { + VGPUProfileType + placements []int +} + +// creatableProfiles asks the driver, through NVML, which profiles the GPU of +// each VF can still create and at which placements, keyed by parent GPU +// address. Unlike the per-VF creatable_vgpu_types lists, this stays accurate +// on a busy GPU in mixed-size mode; see GPU.md. +func (s vendorVFIOSysfs) creatableProfiles(vfs []VirtualFunction) (map[string][]creatableProfile, error) { + query := s.creatableFunc + if query == nil { + query = nvmlCreatableProfiles + } + seen := make(map[string]struct{}) + gpus := make([]string, 0) + for _, vf := range vfs { + if _, ok := seen[vf.ParentGPU]; !ok { + seen[vf.ParentGPU] = struct{}{} + gpus = append(gpus, vf.ParentGPU) + } + } + sort.Strings(gpus) + byGPU, err := runBounded(func() (map[string][]creatableProfile, error) { return query(gpus) }) + if err != nil { + return nil, fmt.Errorf("query creatable vGPU placements: %w", err) + } + return byGPU, nil +} + +// runBounded returns at nvmlTimeout even if query has not: a call stuck in +// the driver cannot be interrupted. Until such a query returns, later calls +// fail at once instead of each waiting out the timeout, which the claim +// would pay while holding the vGPU allocation lock. +func runBounded[T any](query func() (T, error)) (T, error) { + var zero T + if nvmlStuck.Load() > 0 { + return zero, errNVMLStuck + } + type result struct { + value T + err error + } + done := make(chan result, 1) + go func() { + value, err := query() + done <- result{value, err} + }() + timer := time.NewTimer(nvmlTimeout) + defer timer.Stop() + select { + case r := <-done: + return r.value, r.err + case <-timer.C: + } + nvmlStuck.Add(1) + go func() { + <-done + nvmlStuck.Add(-1) + }() + return zero, fmt.Errorf("%w after %s", errNVMLTimeout, nvmlTimeout) +} diff --git a/lib/devices/vgpu_creatable_linux_test.go b/lib/devices/vgpu_creatable_linux_test.go new file mode 100644 index 000000000..08e438b2b --- /dev/null +++ b/lib/devices/vgpu_creatable_linux_test.go @@ -0,0 +1,201 @@ +//go:build linux + +package devices + +import ( + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// testCreatablePlacements is what NVML reported on a host with two L40S GPUs +// in mixed-size mode, trimmed to three profiles. GPU 82 holds twelve vGPUs +// (six 4Q, six 2Q), so its creatable_vgpu_types lists hide 4Q although +// placements 36 and 44 are free. GPU e3 holds four 2Q and two 4Q. +func testCreatablePlacements() map[string][]creatableProfile { + twoQ := VGPUProfileType{TypeName: "1148", Name: "NVIDIA L40S-2Q", FramebufferMB: 2048} + fourQ := VGPUProfileType{TypeName: "1150", Name: "NVIDIA L40S-4Q", FramebufferMB: 4096} + twelveQ := VGPUProfileType{TypeName: "1153", Name: "NVIDIA L40S-12Q", FramebufferMB: 12288} + return map[string][]creatableProfile{ + "0000:82:00.0": { + {twoQ, []int{36, 40, 42, 46}}, + {fourQ, []int{36, 44}}, + {twelveQ, []int{36}}, + }, + "0000:e3:00.0": { + {twoQ, []int{18, 22, 24, 28, 30, 34, 36, 40, 42, 46}}, + {fourQ, []int{20, 24, 32, 36, 44}}, + {twelveQ, []int{24, 36}}, + }, + } +} + +func (s *testVendorVFIOSysfs) useCreatablePlacements(t *testing.T) { + t.Helper() + s.creatableFunc = func(gpus []string) (map[string][]creatableProfile, error) { + byGPU := make(map[string][]creatableProfile, len(gpus)) + for _, gpu := range gpus { + profiles, ok := testCreatablePlacements()[gpu] + require.True(t, ok, "unexpected GPU %s", gpu) + byGPU[gpu] = profiles + } + return byGPU, nil + } +} + +func setNVMLTimeout(t *testing.T, timeout time.Duration) { + t.Helper() + previous := nvmlTimeout + nvmlTimeout = timeout + t.Cleanup(func() { nvmlTimeout = previous }) +} + +func waitForStuckNVML(t *testing.T) { + t.Helper() + require.Eventually(t, func() bool { return nvmlStuck.Load() == 0 }, 10*time.Second, 10*time.Millisecond) +} + +func TestNVMLQueryFailsWithoutUsableLibrary(t *testing.T) { + previous := nvmlLibrary + nvmlMu.Lock() + loaded := nvml + nvml = nil + nvmlMu.Unlock() + t.Cleanup(func() { + nvmlLibrary = previous + nvmlMu.Lock() + nvml = loaded + nvmlMu.Unlock() + }) + + nvmlLibrary = "libnvidia-ml-missing.so.1" + _, err := nvmlCreatableProfiles([]string{"0000:82:00.0"}) + require.ErrorContains(t, err, "load libnvidia-ml-missing.so.1") + + // A library without the NVML symbols, as an old driver could ship, is + // an error rather than a panic. + nvmlLibrary = "libc.so.6" + _, err = nvmlCreatableProfiles([]string{"0000:82:00.0"}) + require.ErrorContains(t, err, "from libc.so.6") + assert.Nil(t, nvml) +} + +func TestVendorVFIOProfileTypesUseCreatablePlacements(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + sysfs.useCreatablePlacements(t) + var queried []string + query := sysfs.creatableFunc + sysfs.creatableFunc = func(gpus []string) (map[string][]creatableProfile, error) { + queried = gpus + return query(gpus) + } + const hiddenFromSysfs = "ID : vGPU Name\n1148 : NVIDIA L40S-2Q\n" + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "ID : vGPU Name\n") + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", hiddenFromSysfs) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + profilesByVF, err := sysfs.profileTypes(vfs) + require.NoError(t, err) + assert.Equal(t, []string{"0000:82:00.0"}, queried, "each parent GPU is queried once") + assert.Empty(t, profilesByVF["0000:82:00.4"], "a configured VF takes no profile") + assert.Len(t, profilesByVF["0000:82:00.5"], 3) + assert.Contains(t, profilesByVF["0000:82:00.5"], VGPUProfileType{TypeName: "1150", Name: "NVIDIA L40S-4Q", FramebufferMB: 4096}) +} + +func TestVendorVFIOProfileTypesFallBackToSysfs(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + before := NVMLFallbacks()["error"] + + sysfs.creatableFunc = func([]string) (map[string][]creatableProfile, error) { + return nil, errors.New("nvmlInit_v2: Driver Not Loaded") + } + profilesByVF, err := sysfs.profileTypes(vfs) + require.NoError(t, err) + assert.Equal(t, []VGPUProfileType{ + {TypeName: "1147", Name: "NVIDIA L40S-1Q", FramebufferMB: 1024}, + {TypeName: "1148", Name: "NVIDIA L40S-2Q", FramebufferMB: 2048}, + {TypeName: "1159", Name: "NVIDIA L40S-48Q", FramebufferMB: 49152}, + }, profilesByVF["0000:82:00.4"]) + assert.Equal(t, before+1, NVMLFallbacks()["error"]) +} + +func TestVendorVFIOProfileTypesSkipNVMLWhileAQueryIsStuck(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + setNVMLTimeout(t, 50*time.Millisecond) + before := NVMLFallbacks() + + release := make(chan struct{}) + var calls atomic.Int32 + sysfs.creatableFunc = func([]string) (map[string][]creatableProfile, error) { + calls.Add(1) + <-release + return nil, errors.New("driver gone") + } + t.Cleanup(func() { + select { + case <-release: + default: + close(release) + } + waitForStuckNVML(t) + }) + + start := time.Now() + profilesByVF, err := sysfs.profileTypes(vfs) + require.NoError(t, err) + assert.Less(t, time.Since(start), time.Second) + assert.Len(t, profilesByVF["0000:82:00.4"], 3, "falls back to the sysfs list") + + _, err = sysfs.profileTypes(vfs) + require.NoError(t, err) + assert.Equal(t, int32(1), calls.Load(), "no second query while the first is stuck") + after := NVMLFallbacks() + assert.Equal(t, before["timeout"]+1, after["timeout"]) + assert.Equal(t, before["stuck"]+1, after["stuck"]) + + close(release) + waitForStuckNVML(t) + sysfs.useCreatablePlacements(t) + profilesByVF, err = sysfs.profileTypes(vfs) + require.NoError(t, err) + assert.Contains(t, profilesByVF["0000:82:00.4"], VGPUProfileType{TypeName: "1153", Name: "NVIDIA L40S-12Q", FramebufferMB: 12288}, + "queries NVML again once the stuck query returns") +} + +func TestVendorVFIOListProfilesCountsFreePlacements(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:82:00.7") + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.useCreatablePlacements(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "ID : vGPU Name\n") + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.6", "44", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.7", "45", "0", testCreatableTypes) + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "46", "0", testCreatableTypes) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + availability, err := GetVGPUAvailability(VGPUFrameworkVendorVFIO, vfs) + require.NoError(t, err) + profiles, err := sysfs.listProfiles(vfs, availability.Quarantined) + require.NoError(t, err) + // GPU 82 has two free healthy VFs; GPU e3 has one. + // 4Q: min(2 placements, 2 VFs) + min(5 placements, 1 VF). + assert.Equal(t, 3, profileAvailability(profiles, "NVIDIA L40S-4Q")) + // 2Q: min(4 placements, 2 VFs) + min(10 placements, 1 VF). + assert.Equal(t, 3, profileAvailability(profiles, "NVIDIA L40S-2Q")) + // 12Q: min(1 placement, 2 VFs) + min(2 placements, 1 VF). + assert.Equal(t, 2, profileAvailability(profiles, "NVIDIA L40S-12Q")) +} diff --git a/lib/instances/create.go b/lib/instances/create.go index 504b58e65..0284d1791 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -403,6 +403,8 @@ func (m *manager) createInstance( "instance_id", id, "device_path", storedVGPUDevicePath(stored)) } }) + // On vendor VFIO hosts claimVGPU already wrote the type, so this only + // re-checks quarantine and confirms the type is still set. if err := m.configureClaimedVGPU(ctx, gpuDevice); err != nil { log.ErrorContext(ctx, "failed to configure vGPU", "profile", gpuProfile, "error", err) return nil, wrapCreateVGPUErr(gpuProfile, err) diff --git a/lib/instances/start.go b/lib/instances/start.go index b162ad90f..f7dee4b9d 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -170,6 +170,8 @@ func (m *manager) startInstance( cu.Add(func() { m.cleanupStartVGPU(ctx, stored, rollbackMeta) }) + // On vendor VFIO hosts claimVGPU already wrote the type, so this only + // re-checks quarantine and confirms the type is still set. if err := m.configureClaimedVGPU(ctx, device); err != nil { log.ErrorContext(ctx, "failed to configure vGPU", "instance_id", id, "profile", stored.GPUProfile, "error", err) return nil, fmt.Errorf("configure vGPU for profile %s: %w", stored.GPUProfile, err) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 4563398d0..80d837e21 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -118,6 +118,17 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str ProfileName: profileName, SysfsPath: filepath.Clean(devices.GetDeviceSysfsPath(vfAddress)), } + assignment := devices.VGPUAssignment{Framework: device.Framework, DevicePath: device.SysfsPath} + // The type is written under the allocation lock, so the placement the + // driver reported as free is taken before the next claim asks again. + // Nothing else resets a VF without a saved claim, and the write may have + // landed before a later check failed, so a failed configure resets it. + if err := m.configureClaimedVGPU(ctx, device); err != nil { + if resetErr := m.destroyVGPUAssignment(ctx, assignment); resetErr != nil { + log.WarnContext(ctx, "failed to reset vGPU VF after configure failed", "vf", vfAddress, "error", resetErr) + } + return nil, err + } setStoredVGPUDevice(&meta.StoredMetadata, device) // Only vendor VFIO claims carry an assignment identity, so the claim time // is set here rather than in setStoredVGPUDevice. clearStoredVGPUDevice @@ -126,6 +137,9 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str meta.GPUClaimedAt = &claimedAt if err := m.saveMetadata(meta); err != nil { clearStoredVGPUDevice(&meta.StoredMetadata) + if resetErr := m.destroyVGPUAssignment(ctx, assignment); resetErr != nil { + log.WarnContext(ctx, "failed to reset vGPU VF after claim save failed", "vf", vfAddress, "error", resetErr) + } return nil, fmt.Errorf("save vGPU claim: %w", err) } return device, nil diff --git a/lib/instances/vgpu_linux_test.go b/lib/instances/vgpu_linux_test.go index c93e4b2f5..8c2147901 100644 --- a/lib/instances/vgpu_linux_test.go +++ b/lib/instances/vgpu_linux_test.go @@ -5,6 +5,8 @@ package instances import ( "context" "errors" + "fmt" + "os" "sync" "testing" "time" @@ -33,6 +35,7 @@ func newVGPUAllocationManager(t *testing.T, vfs []devices.VirtualFunction) *mana } return profiles, nil }, + configureVGPU: func(context.Context, string, string) error { return nil }, } } @@ -213,6 +216,7 @@ func TestVGPUClaimRepairsDirtyVFWhenProfileNotAdvertised(t *testing.T) { dirty = false return nil }, + configureVGPU: func(context.Context, string, string) error { return nil }, } meta := saveTestVGPUInstance(t, m, "new") @@ -315,3 +319,97 @@ func TestVGPUClaimFallsBackWhenDirtyVFRefusesReset(t *testing.T) { assert.Equal(t, "0000:82:00.5", device.VFAddress) assert.Equal(t, []string{testVFDevicePath, "/sys/bus/pci/devices/0000:82:00.5"}, resets) } + +func TestVGPUClaimConfiguresBeforeReleasingAllocationLock(t *testing.T) { + // GPU 82 has one placement left. The first claim must take it before + // releasing the allocation lock, so the second claim sees GPU 82 as full + // and lands on GPU e3, instead of both claims choosing GPU 82 and one + // failing later. + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:e3:00.4", ParentGPU: "0000:e3:00.0", Allocated: true, ProfileType: testVFProfileType}, + {PCIAddress: "0000:e3:00.5", ParentGPU: "0000:e3:00.0"}, + } + m := newVGPUAllocationManager(t, vfs) + m.pickVFIndex = pickFirst + running := saveTestVGPUInstance(t, m, "running") + running.GPUFramework = devices.VGPUFrameworkVendorVFIO + running.GPUDevicePath = "/sys/bus/pci/devices/0000:e3:00.4" + require.NoError(t, m.saveMetadata(running)) + placementsLeft := map[string]int{"0000:82:00.0": 1, "0000:e3:00.0": 1} + // As with the driver's creatable placements, a GPU's VFs list the profile + // only while the GPU has a placement left for it. + m.vendorVFIOProfiles = func(vfs []devices.VirtualFunction) (map[string][]devices.VGPUProfileType, error) { + profiles := make(map[string][]devices.VGPUProfileType, len(vfs)) + for _, vf := range vfs { + if placementsLeft[vf.ParentGPU] > 0 { + profiles[vf.PCIAddress] = []devices.VGPUProfileType{{TypeName: testVFProfileType, Name: testVGPUProfile, FramebufferMB: 2048}} + } + } + return profiles, nil + } + m.configureVGPU = func(_ context.Context, vfAddress, _ string) error { + vf, ok := vfByAddress(vfs, vfAddress) + require.True(t, ok) + require.Positive(t, placementsLeft[vf.ParentGPU], "configured a GPU with no placement left") + placementsLeft[vf.ParentGPU]-- + return nil + } + + first, err := m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "first"), testVGPUProfile) + require.NoError(t, err) + second, err := m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "second"), testVGPUProfile) + require.NoError(t, err) + assert.Equal(t, "0000:82:00.4", first.VFAddress) + assert.Equal(t, "0000:e3:00.5", second.VFAddress) + assert.Equal(t, map[string]int{"0000:82:00.0": 0, "0000:e3:00.0": 0}, placementsLeft) +} + +func TestVGPUClaimResetsConfiguredVFWhenClaimSaveFails(t *testing.T) { + vfs := []devices.VirtualFunction{{PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}} + m := newVGPUAllocationManager(t, vfs) + var resets []string + m.destroyVGPU = func(_ context.Context, assignment devices.VGPUAssignment) error { + resets = append(resets, assignment.DevicePath) + return nil + } + meta := saveTestVGPUInstance(t, m, "new") + require.NoError(t, os.RemoveAll(m.paths.InstanceDir("new"))) + + _, err := m.claimVGPU(context.Background(), meta, testVGPUProfile) + require.ErrorContains(t, err, "save vGPU claim") + assert.Equal(t, []string{testVFDevicePath}, resets) + assert.Empty(t, meta.GPUDevicePath) +} + +func TestVGPUClaimResetsVFWhenConfigureFails(t *testing.T) { + // The selection already used the driver's free placements, so a configure + // error fails the claim instead of trying another GPU. The write may have + // landed before the error and no claim is saved, so the claim must reset + // the VF itself. + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:e3:00.4", ParentGPU: "0000:e3:00.0"}, + } + m := newVGPUAllocationManager(t, vfs) + var configured []string + m.configureVGPU = func(_ context.Context, vfAddress, profileType string) error { + configured = append(configured, vfAddress) + return fmt.Errorf("verify vGPU on VF %s: type is 0, want %s", vfAddress, profileType) + } + var resets []string + m.destroyVGPU = func(_ context.Context, assignment devices.VGPUAssignment) error { + resets = append(resets, assignment.DevicePath) + return nil + } + meta := saveTestVGPUInstance(t, m, "new") + + _, err := m.claimVGPU(context.Background(), meta, testVGPUProfile) + require.Len(t, configured, 1) + require.ErrorContains(t, err, "verify vGPU on VF "+configured[0]) + assert.Equal(t, []string{"/sys/bus/pci/devices/" + configured[0]}, resets) + stored, err := m.loadMetadata("new") + require.NoError(t, err) + assert.Empty(t, stored.GPUDevicePath) +} diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index b1d2067b8..b91b791c7 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -1007,7 +1007,7 @@ type GPUConfig struct { // GPUProfile Available vGPU profile type GPUProfile struct { - // Available Number of virtual functions currently able to create this profile. Best-effort: creating an instance may reduce availability on sibling functions sharing GPU framebuffer. + // Available Number of additional instances of this profile the host can take if only this profile is added. Counts for different profiles are not additive: an instance of one profile can reduce availability of others on the same GPU. Available int `json:"available"` // FramebufferMb Frame buffer size in MB @@ -19241,395 +19241,392 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+y9+3LbOLY3+ioonf1V23skWb7k5q6u77jtdNq7444ndjLf3q0cGSIhCWMSYAOgbKUr", - "VeeveYCp84TzJKewAJAgBUq0E9vxdGqqph2RxHVhYV1/649OxNOMM8KU7Oz/0ZHRjKQY/jxQCkez9zzJ", - "U/KW/J4TqfTPmeAZEYoSeCnlOVOjDKuZ/ldMZCRopihnnf3OKVYzdDUjgqA5tILkjOdJjMYEwXck7nQ7", - "5BqnWUI6+52tlKmtGCvc6XbUItM/SSUom3Y+dTuC4JizZGG6meA8UZ39CU4k6da6PdFNIyyR/qQH3xTt", - "jTlPCGadT9Di7zkVJO7s/+ZP40PxMh//nURKd36QK36mMIvHi1Oe0GixPNnXlOXX0BvCueIpVjRC0nyD", - "MvgIjbEkMeIM4UjROUGUjXnOYnR+eIoizhiJdGNyyPhYEjEnMZoIniI1I2jGpYJ3lMDRJVJ4nJD+kHW6", - "tf0gTD+J16/S32ZEzYgIDJZKZFtBEy6QmlGJKNNPI9L3N0yJnCyvbLdD44SMFE0Jz9XyQv3Mr1DC2RSm", - "5dpFaS4VmuE5QR+J4Oj3HCd0sqBs2rxIYzLhgqCfFxlJMUNZgiMiEVWIMsXdbMwalTT2JA0RF50yLsgo", - "JlJRhnX7o4wLcyKqo38Df+AEee/C0OB9pGZYOSpnXKFLQrLqRPEVvqwu4287O90Xg8HgQ7dDFUnNscLX", - "NM3Tzv7TJ092n3Q7KWXm39vF6ClTZEqEHr79BQuBF950JM9FREYRjcWqmUQJJUyhw+Ojt7ecQGd70If/", - "bT3vdDvbL3b620+fw7+3n3b8aS0tfHXkn1YfvTOFVS6XeZA5TSNLKCOPSJZn/WuejolAfIKiXAjCVLJA", - "cKRI3ILoKtMehLYi4mxCp7lwRzB05CrLOcMSYWaYRq/GL8rGWp27SDOxmF+xkSAppkyv8dIg3rpHSJ9Q", - "ZA+RHlLEmRI8STRTUIqkmZLuFHU1G2cIZ1lCI2A9lUO1lw5kp9theZLoh7URlrtNEjql8EKrpaHS2yT3", - "LVIcEaaIKE54m6WpsMWmjsvlDu5GyRdbdTnjSTzKmaLJcq/v9M92Tf2eqEQzksSITyZdRCcII92K/tnQ", - "eHXZdwY7T3uDvd7g6fn2s/3Bi/3Bk//pdDsTLlKsOvudGCvS07vcZm+AaUvKovDusDqJpPpCEiQyu1Nc", - "WJUNHJOIpwTppleOfO/zR9543RyWm6pfRPbF8mYNbHW/dmW0o+4ES1UwIdgvqhYjHBjTOU2JVDjNNB/S", - "Y/AWs4kLuQbr++BWfuUCb3/WAjNyrUZ2hYLzCdEHuc5IpG9E7rhJIWDo9ix5FyzrXmhcECz1gLWYpC/T", - "3zo5k3mmr24Sj7IEK92ulqmADEYplVJ/WvwQU2n4SLfjiHzEuBqJnDHzIiPqiotL/03byohmnW5nhuVo", - "Ps3yTnfVtVUlauiCJDiT0J7dcTEiQnDRMaLxYjThwm2SvnPLJVzR1NIKyeKKDaxQp9upLEDBzt1c3LiL", - "XQ0ODnoBWhJGqzBqAExmeeB+W8vDLYa2mrGbW8QI0W6bkf1YVjlATPGUcaloJFuxeRAe9PamPA6wzqOi", - "OURjwhSdUCKsXE2Q0PdDSpBrBOlGEGUol7VzUIj+IzLXutpovjdSUba8KDXFxt88TzYpb0TvVi62vzgp", - "a4i0Oveg4jTHFM7kEZlTc7VUZTe7NaNY0DkRAfZdCACGFZr30IY+65qFMM7IZmWl2JzGFLdhBzGMaUQD", - "1HN6eIzMY3R8hDZm5Lrayc6z8fNOc5MMpwFa+DlPMevpA6GH5dqHd/22X+8FVRSepvloKnieLbd8/Obk", - "5B2Ch4iBhOu3+HwnJKlmER3hOBZEyvD83UN/bIPBYLCPd/YHg/4gNMo5YTEXjUtqHoeXdHsQkxVNtlpS", - "2/7Skv76/vjo+AAdcpFxATrb2oPjL48/L59sqrsSov8fc5rEy1Q/1j+ThnUiQlKpNL/60byGBDGqHLqa", - "cUlQhKMZQWOjtIB6Du21IfmiY3d7hUZwRGXGgc+j9yelNAeqIbkmUa4q/X6PYqp14cgKVd7kWowo4nqP", - "9P0bkixgBZB9R6vayolOtxYCIkHwmu70G606Wz7/uSGvUSqbWnevaDaf0iShkkScxdLvgzL1dK95Mt4p", - "NtfmUlcv9c8oJVLiKUEbYJYCdcNweC1tTTBNSLzZTsJumszf+di71ypnDsigh8fR9s5ukKGleEpGMZ1a", - "u2KdBvXvWj7W7SgEb4cnAhJGu3lAl4JMlvv7Ce4T6ESQCRFEU/xndpcJPicMW5XqP6Dfzv+1VRpct6y1", - "dQsW87R8/VO383tOcjLKuKRmhEtswj7RZARLjeCL8Jjh0aq99ihKKixWnw944wucxFLYXLs21vSj5S08", - "XfvJuX6nztCBHRUCjscFGvn2Sy1pBUQWzpR9UDMB8ylKKDNqkObfZi9A2Ftk5IeETzc7X2wdiuVfPvx6", - "3LdgXuaHhtb0s26hFSR86q/mjGChxqSymA33qm2oHF3j8p9Wjk/tAsWSjFZzkFPKGInB5m4PtnlTy9ZB", - "3QdO0SVVo7m+gUNnDob1C1XIvtHYVMKjywlNyGiG5cwaKeOYGoPraWUmARGy4szAYCRwDYJoA0r12c8H", - "O0+eIttBYA2t9Ve/sDwT72vdvBUvFBZjnCRB2mgmt5vf0csUEqaA0uDbdPcUFOgI03C6jt1Nq7zncmb+", - "At6tRwV3n2YDmrwS/fcS4XY71z3dam+OBay6bt4b1l9dT95vP5ader+eFv17P761Q/F++smNyvvtsByg", - "WxWjJgVN4DDpsFTJKVM9ymALtABuLU/m6ijNrNaSgfQ2FhIe3OyfJVgdLolUaOPtT4e7u7svNtsLV1Re", - "jiT9SEbT8Uq52Q7aCsr6M6Q/01fllE7xeKGI7KNDzBhXaExQNMNsSmKEJ0p/ZcdaUcCfDNY5YRqFJCKa", - "RCT1cQ+/eH59jdWLp/RKvviYjsX077tB/ydYFzW/abbAYbub+lUkMAts4u0u6hRfa83fUEnI+Xli3FWo", - "eMnugawPoY9KhX5Cr/WaK7RdWeng4oa1v8KFxTjr5Yz+nsNuZwleLOuAiuC0BxQRWl7DN8wBkuuPj9Tn", - "x3zjZnqFqdInp/CZ2il3EU9ife9MqJCq094bdiPBgIgvJSJVTllYYlqmiPoCNl7qxTgbz0pCJyRaRAlZ", - "Zu6OfWcit+bWmOg7B/4sDIht2bgbSoUTux9Piy4qPx+V/VV+f2k6/9TtHOIMj2lCHWeu8mnrkx9Zw9+6", - "XfIbOzKfvrVffup2JgSrXJDAWp5BfAAyduJeQuYkQfZtTbz76OD0GMlcTMBlDtRq3kXkOuOSyCETZIpF", - "nBAJpH41o9GsMFeOcXQpEWZlWAA6SK7wQqJMEEmY2h+yYWEhl8NOFw2N8mX/NlRi/jahIe4lNhVE2n+Q", - "axINO13dVsKnctjpo5+5VD2tLltZah8NrT1GDjtow5rUMiylmgmeT2dDlmKGpyQlTHURBGmAGVhzpmSx", - "iTCL0bAjuCRK4R5J8wR4v27sIMsSMmRnNKERZyjF0Zsz++0VVTP01nzkXZ4wY31Zd1FmTc8Sp2TIsDGD", - "6p1coEzwMUEJzlk0IxIRNuEiIpt9dEpEzy2x21uEBdGbYYy4KGf6iBBcbob87QMiTAkb6dCeueiZ3IT+", - "9NIXSrS8yZfH5otPhXfkJh//aj/RrMrOOKijiEVpTffo2dyIhanfXUcVT0B3yGBFYVepfuuKocqWTRI8", - "BVLxDpEhRjlkEU9TUPRsL0iSORE4KXZI33Ec7An6brA30pBFCc/j3myRETGnkouuviFIBMEIoot+J2ne", - "hU71X72URoLPU8TZkCWajLdwGj/d2+yjA5RouScu5k8NcVsSA04A9gCqRzvsYGeTH3bMxKhE+uLvo5dM", - "CWpoDklDcGNzk9aIq+3mecxq6W6D/blJY2fmiyUPh/nZEnR3icV6dFMSYEHGHhcN3VkrGPCyDO7WdbUz", - "yo6v2K0IM7tTFercR1QhnGUEC4koG7KSmlgMVGo/ygSB5ZBU2a1LiUIbKVbAXtQMqyEDDvGdRJXd3+wj", - "kBtNfFjJkcDQK0iyMAMiQ+bGfEWTBAyHCAIbDIczBw0sY14EBRwmzArel5TeLuv8sn7RygQqR8T0/deX", - "J++QXEhFUjSmDIvFJnIRaZrXGtJcdsyFRUa7e6C5g6hWnXXMIepJf4s4qwqQ9fO61oVgRdCSLtZR2M88", - "FGKJRTQLhdFJhQ5P3yH9mCoSaSKujBeL9GnQlcRlQ3O6VwxCrFnvSnPAdNbOmesjBSNeN9nj4h4Ji0iF", - "O37Z2wXWnNKPSxISaV5l4mmM/kaQdS4jngJLr22mZaFNS+Qal+t6l2iDyy094U3vZtFnWuSsj94wBGIE", - "qkgRQ2aDKqMkj4lEHj9HpfnWyReVMKiC7IfsX//v/2dkiRjNKS6kDfSeCpXjhH40WuxE4JRople90qIZ", - "iS6HrC6GIN2q5Ai7S6VYZir9C0XQ6Uwhxq/6Q3ZsxlQMeEPyibrCguSZVjNRr2dH3bNy1qYJ4oKGqKqF", - "aP5W2xtvcW4YS1hx5xUb2l2msHWk+msptlRpdYoVucIBhfiVJr3enEoIWQOasO+i49M+gsfgSohm5qnm", - "olqKRRuGKiibDhk8sSKxJi+QaA2ZWddkH71JqSqIn/HierE33ZDNsERjQhh4EZO5CUAuSXVB6hvQ2R70", - "tweD/qC/HToaUz32keKjaR60u7q7LtbCjELvT3qK996fICXwZEIjTUcZEXbU5oKpDzrIz1Mek6Rhpd2H", - "LoQi6WslQ9B4qoUcvVxW9De/mcspI0KP6/zgVF9wsPrfo2GHYWW+KTl9LxN8TmMSo18PzvXVGr0527Sr", - "ZlVT07CWMIJG8G5H5mNGVNP4zVOIwQ1vBYTSrmO+ZomWtmgdeX+eRKMvftPAdytkkpKDaelEC7EFCfbR", - "64r8auVPF7iiRdBlib5owR3i71Ex2h9AoEEpwUxCCKWW9X0hg8ohc9JHReDQkjayK6o/FTkzXMrchz2P", - "+4JYYuQRkMiKIP25/v/eXPLo0oZ4GC2z4LQMp3C6/aVbkqyoaiXkrDUBeApLYEkLicwXPLWOIBnO5Iwr", - "0KrdP7YEkYoLsqmVcxsvpZ9nOJcE/QWlJOViMWTu/a45ZwlWJoah+HaiGVMHbUQJZ8S3I5gAAjxkUvEs", - "I7F1THyvJTU43LgIIuSisFWbl7SWDF3ILSMBSFTMAjag9MAki3KTvZlsgqkBZlNMS885T82wZ1xlST7t", - "2Xl20EZC58ROW0+QfiSmjTFOEs5Zz8ZD6ldL2wk8QgqLKVHW9CyhfaCZYWfINuDcIkNDWr3MGY3gNof3", - "plne800cHbTx6vTdVhmzNGTeY/gkpvKyR3kvoSlVYCmh8hIdb71BQt/Q8DNlUzN6eNlMp5gkvG+n2Efv", - "cZLbcwrUrtdXK8JG+oeR9pTI1WySJ/to/tEKYdX9KHaykIq2d7vW0pSAhalRknl/giSeE0eQ8JVWqaXS", - "xwjHc83DpLXGXqINS0496+cCupOIERIjxofMb0sfZCBTPcS6yI+MIUai2iINWSm21SLJ3dXmXIaFKcIJ", - "Pu0tNqs1mgYfw+drLTdQkc8Kpb52kWS02at6cHpsAodxVC6Tp78S9CYjTL8V8yhPCdMy0YS7V6tRZIP+", - "TlNkWkP3Z/61UnQvZ1woNKVaEdZiHGeWmw07vZgKpY+/zCcTeg1aZA5WICPTGONmV3OoYSdnl4xfsWGn", - "OsxnT/HOJB7vrt0NN+xuZQ2D+wBMz1qlG/P1VvvQDlc6zLSoqSV3ze09zd9S+Bdwl5UpSfrSEz19VhIK", - "kfuOtrsmWndKmFZV9aMJaOxOQmtw+fSCouyde5Vu5Ilp2FETOdu4oWvmME34GC48N5Ey6LSPjqtr10XY", - "WD6orK9ykS3hBYaiDdKf9vXtkkW0NxgMBj280xsMeoM6tSd7PRPynmGliNAD/H9+w72PB73/GfRefCj/", - "HPV7H/7yH0GFvGW0qvNr23luuLPURW6wfghrfaDrwltv7Vnzh998eMG00LjTkSBwDHCydgynuZwdeq83", - "0/rh8XLUm1mpmEeXRPQp30roWGCx2GJTyq73tTgnVXXdVr+7ysASSI4xglFhd8ASWQvLb1tzLChm6oMZ", - "Ixr6xgEtvyG4NfS9fwQjQr2ea8eyLxd5X3EA9NGvXCFsPdJapduHl5b7nQqcphisnXOc0BgOh3WzSRoT", - "Y5xl8Ey/oaUkQVQuGNobDNyTkTcoLTdw6aYy7FjDqYhmyBqEBAJTiMmQiUgG0bc8xrU0Jd9OsnTS9DH7", - "y1bx14b7s//hL5v/+z++KKkDoa2gcWPOuCE/q8V5A5faSPgVERGWBCVEz1Z2UUynVEkjvMUQGgUGte9R", - "ZII9TOii1viYM1HDe1VyThc9nNGetbwYd/drwqZq1tl/uhtc3A37R+/Df7qfGhZW5ElIX3vLc7C8wmM/", - "8dqNoZXzxa1unoDTJaXs2Hy2vSxOft4Ou4ms2mmj1DVutb5jiqygNQNZzsD/1O1EaUiImBMh9FE0fO3w", - "5EirMJfEMjStNaJhPhjsRvAC/EnsL1qI06oM/LZZlXhsDGFA2PmtQ6IZhzDIJOE3MVJ260y9KUpwpY8s", - "tNol91+OKQQnunGLQ7C7fRGNBb8kAlwyeokpkeiSLIy5YFoxaBI2R3MsnGvwHNISjNJqH4E3iIB6rJVY", - "SM6fcQjwMGzR6mwmDYrgpOSWKabM2D57MuKZlvGYfc1Y9C8Im1+gFGdGrRfEGM1TrIigWl0sDYspianm", - "0UNG4GCgDOszH0VcxJADzY0zvVyF7yS6MGriBTR/QZkm6wtzMGum0j86b96d//jm3a9HozenL389OB79", - "8vK/9c/mo87+b390DJhGEXj1I8GCCPQff8B8P5ngWYjp6xzkasaF1Xadix7iVXBG+zwjDNN+xNNOt/Of", - "/j8/fPrgwj1N5t5cH4PAwD4FxUxr+wzE4LsEKIlsUo1L59RLplmUMzt45oaaKd9Ifjc6EqCiUD4aZzKY", - "m1K3W5Ry6PZgcPLjlo0eeeL+sdlH1nULw9c8iAsrHsuZJp8ClwP8aUnCI5uhMWlKwXddhRg8+FozTkMh", - "4jXmVL66zKN6vfLpDVjR1piyLam3oRfdbN2Bbm4dqPySzangDNRzEJfGCZHVs/Lrm6OXo5e/vu/s64sg", - "ziObiHX65u15Z7+zOxgMOiECJdcZFUQGAx8PxpInuSIIXipjTPvoJJcQ3GmZxyQHOyjIZxKlucpBLSLX", - "UZJLzadAGlAqCeUfP+/tbJ9vP98fDPYHg/9pHbCqSX8N8351+s5kqMN5JzhRsxG45tZ9+DO8ewivAqsA", - "2ySo9oFVKvbUWSv16bXfoI1ZVboyk0NAkMPO7qsfzYHafgVnyRGiTQUuWjEN11bv1Y+hdfGsUcuSXmls", - "gww3FjuqXwajMfb6XFozke8J7zEMEDsgXmARozHHIv6+GksDX/715cm77yQCawVOrJ/IiN7wjZElOZGg", - "K7sYn9LYKru1JQDLT2pMiZghAp7SOdUsp5emlLvvqmzJGlMqNtS+52AK2PG8cKFOt6NnZv/jJtjpduYf", - "qznV682BNjSmla64RgnESUYZWaEFfiXC/hUXlwnHMViJvqSs78W71WBozIPqdhfKqSPwJXSnMWbxFY3V", - "bBTzK6aHHLgh7RNUvFxck9d6Jjj51z/++f6kNIhsvxpn9s7c3nnymXdm7ZbUTQdzbYqJ5Fl4Gu+y8CTe", - "n/zrH/90M3nYSRiR8lbiud3/l6aFOkCNxe0y0fgNftdCDivzc3mDL30NhkHoyuVzIhK88K4TO6bO9gB4", - "em1Ughp3o/3OMyGvvlx0a05ce1W3xu0MwtfHN9vRV247EgRY7ygrsPJWHZC35u1SrQ8QXYDmfsSy8L22", - "obSC0LZ3TuyfO8skF6Y4eUkzE0wxwtMi93MVvN7ZJc2s0gxfOFe+YfRxDmr2mHPVHzIDn6PPJlAOuSaR", - "c5AqdHB6LE10gL7S4NJYFoS0Cu7hLsHrxhcqctZF41xpvZorgqyFw+bx67HAy2OC8jJSs6bl2gkuRx/A", - "slwSwUgyMlqsbLky5iNkP2pcHJjqBEsL9yVUnlXX6+iXkzO0cbRgOKUR+sW0esLjPCHozIhqm9XV6w5Z", - "JgBDRXeiyZHafvkE8Vz1+KSnBCFuiCk0Vjg7LJLE/NXpO4tFIjf7Q/aW6IUlLLZRpU6iKMJK2XfK+Jsr", - "zfr91xa9CW/G+dBbHq4z+3p5utqb/bodpZIQglxiZOuETohJVhA8LanP5c91ter1ipeAB+ZA9tE7qQ/k", - "QA47mkFxzVrHXM18FW5CSRJLX80ocSrL17SSF9bjSpWxppDszYLO4SjLq6S7020EDpybqAQtM1QU3CDK", - "is1uWV5Fg6vqG1Ts/V0CLaoqEkVbG7BpGVBQ14ZFWlOukfhbm3I94+KSUddZvv5omS2wuv1j5gay0pRd", - "Gr8+o68z08hSfoH5uetmdotVOi7WpGYA/zLLcyA9Y2ErwFIjwZhrXqKNC5zRvqXjfsTTiy66+M/KD5rH", - "OcVfy8lXyKwG8E2mf/Lbl0shvWsMmLcO68Xy9vtxIBvhptB8GymBmTQB5jOckT76GS4rpEiqRTDA5/Nj", - "6hi/+h5xI5y7T4dMD01WgwydGVvSKaNsugnR4mOCcBwbW7c1VbnAExnC+XUGqCUJ2IzOovUAqq4NM0cX", - "zuZ8UeWJyxbpZYONNVEvaepmSUBDBxuG2kpzpbvXEy4Eap4rg95lp15FVqvZvddFxNixFDa4W+z/WcEu", - "6uDJ84CqDrkgk9JR4XlMmhwTViALO00uycIkCFgHCV5ykfi+kbAHw8ZyW/HC966MAR6a2/jOwrFiXCTW", - "KwJRt7Wg75C/YN1W6PVqvfxVlTeYgNJzky0pxmqxDpjTcSE9OdMfWNskgcUHFXrf5DNddI3OT0DDQkwT", - "S+KC0uvNQzwuRO9d2F/6trWLIjvuS4Btl2kP/tYib2edeGtCmK14pEjcrcoGl4Rkcv2kTFzmrMHfJ8iV", - "oI6ROWTHlmKoTRdJrTL0eQaQl15jKyKh2jaxnB5q1tcbs8P0xTbADPbe7Ac4fipQv3XE+7imnRrIlGqX", - "FzhJLtCGfWkTCfJ3gEO1e8U4K4n9/PDUkUARZ/X+pKspUnOBi5lS2Uj/nxzpU3xRb8x+6054iUb+fAB6", - "5N7ebjVZwQy41mzVThyEgWnemtNczu4ocKnBGu1g341F2t1AWS5naEPhqT46Fu9H3xObn2uppkySKBdk", - "/Vk/APHIYs7AmBJMWe/n8/NTJMiUSsixDUbzm5jwBnAyq8CXcVpuuopXzZU7u3tPnj57/qIfX4o+iUQ/", - "lz2Cpept93GKP3KGr6QWu7bSBc6y/fn2Wj5vtqAYXzPLd9pmMynwVLNkF4nbRnM9LD8pPWWXlMVtG/hF", - "v7s+2tMp1nftb8gE6eXZVGCAu/yS3oZbB/TAajbv6poSIKsjeXOpeOpj326sDuCtLtacJ70YK3yT+F0z", - "3OVQ3XRhmjIqedDM1xQXfVYPhq4A1gQLAXxudJUbS2hbmsB0jSGBxCPFA3mrTnY4PgLsbftuK9RQwP5V", - "fDSfUL4uux1iRio5q0YqseYN3UQvi6i1nrmEDxBWzSKA7vD+xA8q6Q9ZD6SwfXRUdFA0WzSJQcXAsXFv", - "b3BRSZwFJ/J4sYkwen/SR+fFaL+TyHqLLbpwkTKZgycNhKKeEcn8AeSQYUJV/XNrKjRAxJsQO8Pts35R", - "rgSMkoVRC2xYY1qbj0loho2yd4mPudLSSLsKluutuY9WQXNVQiGe9Abbve0n59uDm4ZC3AHWcqitgypv", - "sUHvPvc5fHd8tGNNkZu3RuP64mjMYVZ2VEbro41cEi8TlYtgjL4XCt8Qg3/r0Po7ipQvcSdXvWtW4ly/", - "eRcw0yEgLItUeXMg6DrDXIs26k1u2aCzyMD8UJ4ST5C3WRURDaYcH1F5+aMg+BLg85fvbQBgGJn7LBxo", - "BygZ4wWgQoGRS3CuJhIV4Cml8Xt779ne892ne88HgwCQ8TLB84iOIG+m1QDeHB6jBC8KTL8N8N/HaJzw", - "cZXQn+w+ff5s8GJ7p+04jMe63ToU+rf7Cm3YFfmLK3HmnlQGtbPz7Onu7u7g6dOdvVajsm6DVoNyLoaK", - "SPJs99ne9vOdvVarENLrXjpg6boAH6pycGAKA0Hep8xIRCc0QgDIhvQHkDI/o4wUztnqmRzj2NWCCN8d", - "CtNErgxrM53ZN429Nc0TRbOEmGewIa1cEjDzI2gpWFyLsaL2xs1asnDcawOe3FyKV0Ccick4n04t1l2x", - "dCc2jb0UnihJ4n1UqGwr+RzsZjmwD010YOfQkhpea9XJos55RGCuLgPRIAgq6MRsWmVWLtaBsiwPkkTj", - "Uv6UC4M+AY0iPOa5Ml51Wyyl7MQAuTDwMucsbmfu+ImLy7WJDfomLmrCtDQYuAx3kwFfy233hb7C+21i", - "BOxzid6aL4yhsPw5y6sF8brQkzUoMpeWHxd2Y9tMW+kyLLeAzdyFy5j+St55T6F8vYmJfvqyGraYEqiF", - "pNZKLJpSTDzSGbzeOk9Kf7jWkNJi3Rm5uo9Fh0y/nibbnmQ4u5sVXxUxXNgavDx9buPs+whOFwT5uQT9", - "2kk7c1ATdsn7Q2YTjTy8AnCkAZaACROfESoQF3RKqx1X7az3GLN7E8p0xHVr6vQ/bIrBk4UbMMQDDNLy", - "xITAmgIgPtCr3ZNOt3NWFI2yjKm6NG+LwltLK1JG1i9j7py+u2nkbSb4hIaAcCASyD61ipqLSX29Nzjr", - "bf/VRM1r8gOJjTITPZTyuFbjyb7f7iJ6dfrutGlMRdUl5I9uaU4r4H2Wo10mOTN1nzybCrb1GC3imsHj", - "MZ310Y9Eqh6ZTLhQ+zaQBRxSXs1XvECCxLmmBx+ajDMk6TiBM1r0qgV+/YueEACBjPPJhIgq6sCLkCzt", - "vT1KA8a9n/RzZF4oYA9OfqzK01pub6u1n1aoAdT2CY4om2623u6AQbA2jXW4gq9O3721ZYOaEJf1Uhal", - "hQzYch/9WhTW0kstS1SlfsBSWKMnE0CtXxjJhIeiUX4ShCB4Vhb01GcPPNgGo7eIGsAxYDRx5soKog2I", - "8oqJRL/nWGCmKCMxev+T/B4NrMv8/U/IZNFYBkSlH9hY1RSfhba0MRvudLaQNMKJWRZT2oMy30oJR7q1", - "mnFafmjtuQFlI1xUzrEPtDGfZjks4Nnb3vGb91tpTObdypggGnHGE6LHvenx2LmDZChT9yqsdd5kLnJb", - "VdQ4HJV1FesHAdCgrR2zTh16bwZoTCLsAEXqmxcVZYjHEHsBVq8rQZXSkiu2RmeInikbMaB5Ek3yJOlb", - "BuiGbMzME+DDHoKpuao0+cDzDIMmVYhR1wo6JUjmUURIXAtfqY/ao7d9lLMUCznDSX1y+0WodzTDAkf6", - "Pvzu+rtOuI6QHpxsy+g96ixumtZk6d0rAXr0zl3TEX//U/V4mqveJAdbAzllVAHYGSBcbxSDNJjcfXSC", - "F9aOksGdaeml3DE8mRjIz+IuESQhWEL9b4ne/9RfG5ipuMJJ0xzO9UPLpzb0hPQp0sPsoqzCBoB3eVhb", - "frdPg1dGOZ+Amwk6rPss+uiYhdmeif4BtDwDfCUlnTKt20kbABRhVqzk0t5V786gWaiOLtipLlxlOt3A", - "BRCimNBtZbIZl9Wc5Xqk/LLKpvjl+hqYppHmfg9dwmXNr2V1SXduIS/TRSIh86nlGMY6KkmGBVYkWdRC", - "oqs1DchyUAm5JtENMj5f6tc/mTo2uSAjNRNEznhSDUXa7S7XQpYQ9j8ntvybmZPne1McpVhcwilzujTK", - "mVmBatbI7jqIp5lS2Q0m9fP5+akxsCki5jip5x3JpSCbI5LgBRoTdUUIc1PBEmE/7Lyety0bSnAJNcqI", - "oLy6hp3dQL9nJhUBTQWOCDJfuUrqBVvTR6/tUtpeAqChUUSkbNjf7VX7az+d5Em7PQ4Na3vdDqvoJht8", - "fnjqSkkVpbrdMu8sr/IpET1z5FzN7tVbuyNXFzVzXTGDBb0ks48JVDmzyV5+LrYLMYQqbvrzSv6zxxyk", - "n7Fr+4FTYJaqa875h1YKXv24h2JpUsziUMlzk+Nj0DGmAJkHiQYiB+2LxkZSMneydz/bVCUtaVFGpKxB", - "C0S5SDrdTm9iZ7W/taX5fQKooXu728+3VkdyrwzhtxGLo5iuMvG4uEYT+eYS2Q3kIky6ShJbOMtaGMHN", - "Oq65H4A9LYcMQ5lzfbd5SpbLIRkMllASrnGkXGFGsIpXoh6wf2wB27sqyOgGU1NrSWu7L/zzOQiGoWRY", - "zarkv7VE+xCaBmHFmkaMxbG2jobIPwalUy5C1fe4UDbpdkxc3HFxH7qoXodiXfGPD577s3z65Mnuk3V8", - "CJhN7ZjbcxeYqnm7CitAvOW259c2ABGSVZnDHenVBYT1uqyhKc0Rl0hq9YLyjLAbreeTvd2dm61n24kc", - "u8jMGl8KoTIdnhwZmUjr8pgyIlBKFI6xwlUmA+ZkzWWgvBMmKWTmTb5fzVoaQph8mKXb1qb7UgEwDWUq", - "3zpU9hQzOgElybzp9yxneOfJ031TTDcmk70nT/v9/k3BZ16WaDOttmLLxMl6ODR9Ofu8fbgDjJk2c/mj", - "c3pw/rNmZLkU5tLakmPK9r1/F/8sH8Af5p9jysLYNG3qL9PJUt3lakhobsG/SbyPyhL7Tu5pE6LX4A+C", - "BAEAxAoiPVYCpe8O0rGgcVqt3HGD9PwV6epaXHnDkkXj2ty6lDIrbKHKK6Hs2xValFOmH1eHuDiLM7xj", - "+zSWjqLS9HJwy61qhcuV5VSXqu1lhBUFVJPE/BVxBtjaoWqqlSvSPWtRjA+uEVt1r+jS/7Ho3fvx0B+I", - "97sr5uf9ZMuqfrhhVNpKgfRvy3Loei7kxNE1hzls/i9uhbYlrC0UZDAd44HvwttEblZ7fzP9r9//jzx9", - "9vft31+/f//f81f/dfQr/e/3yembz4IZWo0C+qBQnl8MvRPCFSsQnm1J6QSrKGCj0+pfwwrbJ8bioKIZ", - "FN1FY7I/ZD30mioiTAnHWv7xsIM2CGhK8JUWd6E+lUn93NQfn5qgAv3xH04M/lRvI7a4EMJuSAH3I/Nx", - "zFNM2eaQDZltC7mJSNAL9F8xinBm6jBShrT+u0BjAUUzrZe37LyL/sBZ9mlzyMAqS64NoH2Goe7gpPBN", - "MBejYUdlItHt66RAfjFJwUNW3NYFDKZx9ffLmhmUJPW0vYZFWa2/Wc3p+SAEGAopZXojoS4UqCAFZWsy", - "KnLd0PPB5rI+t0bHKGhoBfnZWBiTc3yQh8zFTXnKRySmEfAVl6o7s8ncRZa0oTRrxMsEv17A3rw1+aMx", - "wrmaaV4UWWyLiPNLSrqwpV3wSEPwFXxpQmpmPOuNF70ZzwqcEyxMwBk2QSlVJfv/9OxEe++JoBPbUxCu", - "QpNIQOiEI2NnZrJ+C+vC0sTOTUkdpkWfObGvm9pM0pTjMdkUKhfMVYwhUDcWAHUK6iMhmfx7FCUUrE5y", - "Bj68GWBeKt1MCLayMyiy1vA4ismk/u9qVNHOk6egwbp/7+60Tho3S7eKyvIkoNOmjvW14NiGTcIAjHgw", - "cobwNXGA+ga0oRRgp1Ac/nuGXEPliSsYifH0mTRWaQuQJNJLYN0MZhraY2BRe0bYnqY299HSKawkLrZo", - "wQQIwWdJC7yglyZh+vz1GVJEpA7CYiPSuwOnxIDD9KiUua2Fd3B48nKz3wlinVVcWrBVKxMbq4MOwJ3Y", - "gKGmOKjSRoNT0kXHR5Cwbq+VUheDDKOfuECJuRXLy2gf8HKq5h5sqmgeH1kBNFmUUUdGbBl2Nl2LWf16", - "20dvCxUQF0MpUo9L2nJNlpcJNGtjUE3601LrtUx18I9Z9c/ex5DsBCVGDS8GvNbG+6u9zdGhv+mLqmYh", - "u/GF5AeCNdq/vL3/0sjmX15G372ZjG690KNshmWIume+VxNeWtp335FdZfeiOWyv0u9I0uDZ+psrtORd", - "Q4roe67yeQjK90lve/t8e+/m5rubglJX0eg8pMoCl7o9oPRdADMHYJapGjXmdyD92GZzOLvI+xM0w5J9", - "p+BhzTqyvfusjVECem2bGeHnRPCJGVLBpRy0XRHRb0D+LmmSGAFG0inDCXqBNs6OX/1y/Pr1JuqhN29O", - "6lux6ovg/twCnxpuAVhHExwVQDeroHKgIn33/Pw1HK6EQAaUkcMvb49avda02ALF2g3u1ek7cPxjOXKx", - "083pwrhMuSfXVCq5DGzYKgXhc1CzzaelYazNJE0bNqhyLfT2zxVs6CBS5eYdYGa7/JGl5XwAOOmHzNP9", - "+qCsV4JPfy6CtLUz3BGAdOOVFgJfroGSPGm63W4PBX0nw6lgOoXYli/hOBCFW2Mvdzs0kEB+YOP40PFp", - "WWytdEa45mtzerHT3376HOoFbw/aMPYURyv6Pjk4bN/5YMfcMvt4vB/F+6Cw39ZnZQnbqCA4ucILqLdp", - "lnbYMRemp916x9Yqkq3ia5Yhrm+HaF0X4xowq0GcdYFLcpSuLPfTIkO4jluY5haVNaVJQiWJOItlVUae", - "YYlkZsCITdmbQoIfMhhgFxXVx0FKQTiKRF6aHq10beX9PLN0D6V3M860DgC1N34hC4lSCk7QonsIfZSo", - "SESLh2xDuKTFIjsRqu7G+gdIAera5JK4C1HDUN5HfzBkcpYrzcQ2++iQM5mnRFirLBpT8BhtIpkblRbG", - "C6ux0AxT0piIIdOvBeCO/yjUk/2ng8Fg0O0Umtyu/vcgRE136vzsWzhvk3YPgJvMAntDtLvIGcpZTERR", - "gp8YcqiHyN3QcfqZON7u83bilf28lKvCB3Md7Hc7PO/PBVGGoTbo5xAdegvl/MntRfRWaX9OfrUJf/ar", - "0U0iGGxKhtb4xvq2MwY5ElszpCQ216PIvXlnyt9Wp25DjxVHv+dELND7k5NK2IPN0Gg3ceASDfvAsxtt", - "w84aG8na0dzEvexBTt8HzHRdUvEkxC8OKu17GB0OgaHQimGrojivsq9prTKYA0WZ2SdNNCsmWKuAbFBb", - "ysBIv1W5kBMLsWpFaZdTYSHfCwSW0qkvF3JrnMutLKJbNntsC+BxngM8zl4QvyAm81Geh1Qj/cjhIb17", - "d3yENuAXgHc2CTKV7jF+uv188PxF7/l4+2lvLx5s9/D27tPezhM8mOxGz3a3d3ZXJBW1SGi9fY5qUGMO", - "xDEXUesjFz0fCmpuyl2oySY2HvuKsphfVa6/YICs37sNvl3X/XJofeshBFOCEiyVMV80cLITuORJpNs2", - "Aek2abqodhY2dD49H2x/rvUHBtdwR5yLnBm3qoH5KFwIqTdgf7Oq47wdy4cBucSXdavld95+0Qb7T17s", - "P/ncRXPJG+vGWCene9zcpogwB4Neyw5xScKeHckZKDtWJjJWfZtM0ul2inwX+BuEgVosdfG4VRJX04Ht", - "htnIqmulAb/guKKvQKSKgcGM97Wk4vQRKKpSoGRoEegw4XmMPFucQQUEP9yxp7voZsAtZk10JnHZJGNA", - "ZiSVNp+PMs2Iwf+oG7FgB/voFbwLj3Bq1Do7CFNCyHe94Xhh4mX0+XJdGyVr9ZDPrH4F32hlC+l/wbT1", - "MliT7eomjHS2j37l8E2h7TFet/2a10HNWn69bifesIj5DrwGOrOi5j76qRAvCwHVCqQbktg/R5ZhlZhR", - "mxXkDrvjHU0t5c55KBTdjlnRTrfjFgrQKpZxK96VVL90/nxSDAWSEZzAWS7T9HNFE4uUDzOhUtFI2uQR", - "vblNYo/NzSTxyChPTTGpJvXVKljFR06qen+CNgAM9S/IGrb1vzaL+NXKXbfzYu/F02c7L562gjwrB7he", - "ND4EZILlwa2Vk6MsH1nbSNPUD0/fGdtHZKwKRezL+xMfYSYTXLMePXPXoN/5i/4LH+kt5vk48RyLFhbS", - "AEvDhgXBDAte1BAH+TtN5nQyYb9/jC53/i5oun39VO6MtxsQrE1HYbPbsR9csGSjJuOeqVQWBuMCghKy", - "Ea/uLZEwA3RGFAL66SEcgXpT5FNbknOodnbFg4S1t7u7+/zZk51WdGVH5x2cERjhApeyHYF3xOBNtPH2", - "7AxteQRn2nSwLlkA5qB2zpAtMz6oCqT97cFuiEoaLu6Samzb87Rxyd9b9dFOyi46JG8XquXSKQ+u9u7u", - "4Nnek+dP2h1jax4eievVHMalLJnlsbUw/J3fAGny/OAUQULwBEdV246LELvRqNSNRgV1XEz9hRsM7Pmz", - "p0/2dne22wEvhoJOLKRo5cBWeVfg0AWIIrAbgaVYZr3dptsiJE4ZAntLogTT9CByKRa128fUWRgJ81q5", - "CW0uBquBL11cLb5tZdwqTFYmQceIBlygnBXVffrrXbJfxLPazLXN9bCeq4fScphePYsQZqoY3mIpM0Hm", - "lOfyCzTElcmZnSScixt926SwvCUyT5Sx2VCJ3p98BzxF0xqSimRVHcpS4woctVtO7kbnuUIiYSJvWqxW", - "u9Fm61dNuNtwarurADUq3KARvTDWnCtn64M/D3ES5VDPChf7qWcFMHwACZBlycLE9icJ5wxFM8zASSI8", - "0DE040ncD0bC6iejSTCqgl+hhBvc9UtCMlvqyQxCf6ZFGDonaMMvcmhIqVZ6+ElqmIwt5lOlxidpuIZq", - "GFOpSIXX64kV9yDBzScVS2jCpxKUQgVZC/16JYoMC5OMgJkpXTZPjS4ZCLgODLHGzEM3qrlJ+cQquFbk", - "gERzs5I4Elx6cGDvT2r5yyty3oos5vUBndXBtiBd49AMXGUGkK51hcPQ/RjI5/mcGxJoGHIGV4RKOuNk", - "ilkOxZ88QraG+H7rcMgZl2pUwFzdcLBSjaCiSy5ICVhZZN0X9iD3TvBedKztNstl445v9fUSVYWbahpg", - "M08Nrmh4tboFDYbIeBlXbiWUXQnwVwezugleZFkChEpolXrIgWgDcl48tuShQG62CZIJq6y6nyVt1Rbo", - "fb03OGuLrLgaSPEUq9kxm/AA9scNPKfOEm2jVTMiHLhhTBglsdMlCxeqNXVBwngiCYpzYlfOyKc+/B2E", - "OGA1A3skfEjZtMbr6x22MQ+bMawu+AL92hfbhDvJcELtuchhrUy8okS4TK1tFQRK5SjszlpuWJBpnmCB", - "LCZqmyHLRZpQdtmmdblIxzyhEdIf1P3iE54k/GqkH8kfYC6brWanPxg1VQc7M4OzeYFmQ2r9llP4Qc9y", - "s5aVDJaYLfP9FjhG20SPBSPFf6IJsdiU7xi99gi9WhFhb2fQlC3f0GglT34ZnPWmnNuSbPDE5zKQW7hS", - "ynF1zUhsy1QYsSfLpSmx1OJWcjjIzgV4O49ONXHk86BJDg2/rgGToDGBvB83tWWu0YIttplKsLpLLmfo", - "73xcNYi2DfsN1AzcYCVEhiCTYHw/7OhKg7R5Y2lNvN29CQYFsFU9UfjohtAO66orlvFVTfzk7VKhwRmx", - "S0bdHE3RwRZFdFz8RwFfYHttj2NQLwkZiFcGlBqpFlBcGSpaLbw6pxKNuRAAAq8lHM7cbAB2Rcs8eq0d", - "7hU6n5EFEiTFlA0ZZYWRFMDUCGJkToSXJcuFVrKmJO6jv3kqHsDmp5la2HoMYDz/TiJ+xYoxDpk/SN14", - "LnU7B8xYFkWeqUrFVt0saH2aUCBrGZxgSkAJU6pmaCKInPlzD5Wt1TLeFRdxYz2wBXKvQJkp8LEixS8J", - "81lZ0UxQNTQNjcxXy1F8puY0PLX6J6qUgUb1Ms+r+8slEWEhsZhS8Uqr0BXvqHjKiQGBAUQUKO9p/zIs", - "vkBBaYF5Ujb/V9dk+dNp0Xj1t9prHq6JQ/o+MGbboAk2Mmk8tWCfqidtbagKpMGtQrNZ9iWgDRdC7Wok", - "VSUBr1ZRq3uyXSZePVnAjWZLkqja+97zJ8+etiwW9VnOOoPe9aVdc/N0hUuuYadO2vh9nj95/uLF7t6T", - "Fzs38rC4vJKG/WnKLfH3B22Qa6UPa/Kvf/zz/UnN6/MEYrAHNxqUySwJD6khu6Q6oPcn//rHP92obj2g", - "EKNZBulv8Ns3Rukk/k66QIGqC6+dk2yFfn9QMRLggs2gDQJQ3HRORmbdeuVgajAg7aRgnOGIqkWAkeMr", - "E+1evFLD2m7jDqoONiTymrYtKqrmXDIfl0mnG65z9J/GN1yjheeta87JfNzkh35T79V4oUuvhR/j0CLE", - "wFBE2MBdzOcKy0pAt/47grwLl2G2nG1j3liNultPhYAoFlta0QsFDNUaqMmT9iN/+2vb6fktK2ad+op/", - "WHEOm4/gjay+gRs5YPSN1qfW1viDvQBv99Vo7FeDXFlus1I6srx1b95vi+zh5VIlxQ128/68hMmbfFjH", - "BAZ6tGOwS1623a2QRAM1ebkwAQMaT0jPK15gSm3J3HgE9Zm3MPOBDM7okk8mVazbJ83Y6AD7A8lerhes", - "lNZMuohcO5tFHVjbYPwMO0/ksKNVgGFnOx12am6rYPpkiq9HtoMqtstgFVh5mf5eG6R0MxgnPLo0hQ6h", - "fn4fDVBKMJMoZ3D4a1617cFq71C3k3l7U0CDExPitMS2YExjMsNzCvVUrE9lWgnEJNdUSQgYhXb2UcwN", - "2lOlyrOdoX7NJDful5OGSwezhW1YN6jf48xFtJbvgoFvArWl2UcieNeCFWiO/ebNSdcEMEDooRlYJb7R", - "TdSMQDPIootaeYXy93D88DghIxh3Ha4/XV5HPycdPKuCSKKkxe8uyaFGBCjiOVN1HP+0nSJXTStbvpJy", - "BsF+NvwDcNls74ZAUEwiOJFy+SxWCf0WxF3LG7ArHUoc2A2RMByKFfV33lqHcH0AxtjgFWg37fhx3cZL", - "OJKK24p+xakekeuIkLgO+Bl+pW2svP0yGCv/GluMoKJ2un0b4p2XZ9e/uwQvGGvTavsx/YyzHqCTuC21", - "SCIGGtBi1VQJrQI97kFajELwqqEX2mRck+vVa/0ruVaAjx7niQG9C5OuZVX2Mlq34rfObGw60FyQtRUy", - "76BypIk3v1XtSBuq/hDlI+1bd1Iycml3zohy755ZMmrcoWqhl4pLywX8u1eqMTaGlLrIXvBoO92skeDe", - "LGwVsaC8LXM0GU7JKBNkQq9XEI95wSjGVViT8iAVGQwGX3Qjxddo7xmU/pK1sTM6nalkUQ3A2QtgKX1W", - "XVVBFGHOUNhm58vddB8uR7vZ7fRbDwnHZx400FJJEyuSjlbhZh+W3jZrnc/wAqw4jU7CZ7t7g8HuzuBW", - "wNluWDdYrsPyE1uFtNpOU0qd95119FeiVP0WiiTr5dLWplyd55SUShCc7kPiTYYjghIyAZC8IqF1vWex", - "3vXqwVuBymbRFvTvNsrum/PBV0vmFF1ZzHE3jY5zLlYxiPznaxyiDWwmWoLUC+Tc7fYGT8+3d/efPN3f", - "3r4LsOtikZqyPZ593L56luzgyV7yfPHs9+3Zs+lOuhvUwy6pqQzUhlZ/0e82RtmUl2QVy6jC0tCGnUNG", - "RL1meb3WvyQJZaQniwyp9WmKK3iB8b+vPf83s/ObGayUHc6qk/RFCKzKxalQ1sPgb9nJrPRd1GdzfLR6", - "FrfKQKoPJExv9aEAebUbDFSo2O58JjJDzlpeQ++8F1tfRCuz4tZdRSEPO5z04C43rHiIvGvADN6sV13g", - "y5dcwHY65YKqWbr6tiheK2DEIW76o1RxFe+pj46njGvtyf+5CJPzlSj9cafbST7uVc+M/b098pdFIC4I", - "0G61LxW0CCNLyJwkq1cBXikVD2Ei2bWursf8w3Zv+wXEISQf934Y9F5UIw66ZrX85dt2b1d+HbRZQ78E", - "oCsdtf3iRhHXbj1XUdAvNFTArryXLTaxpfGyPLy7OlzCbWWDy8dLe1xD8mkUQD9X0rOX28gXmmKS4EUI", - "m94z1Mqa9ugTGRqTKWWyjd12d1AYbp+kw04fHViAcNBlFS/68ZvXtOLTCU1TElMtYxrVvzmDYaelLa6u", - "S9ysNon7KiCt9cPi2ov1EAnrEq7WXZP9z8jH/Sztt53Guwq9A+xqTkUFDDF4sYvoBGFWK1DqilUbpQMS", - "IyFebd8BtZUka3mALOVAZyfpoilXqEyhb2lvy1mzXbAYP7kGe+sKzAxDEDtfBBClABCjq9jX8RHKBI/z", - "qMwfTWDQJeKHyGsQbSuE/PUhuXdp34DE7AkXaL19o8mg0c4+2bTfNdukJtjmrd4erN/qOzGKdDt5Fq/n", - "YealdhzsRsjta1IQAyaa6rLXJEFvMh9acPS3/gou67zGlhxpkSjPnINF09QyJQXcLeBiCMX1HpGE6Gtq", - "uRHEk7jMkqCy5KLrWer20+ezJhcneKSWB/ILIZnWVQD/CPpLMVsEB+bKjhZ3ycbAoX1L4/DqmXJFdrWq", - "g3u2VhJr3CrfhNtUQsFw+ZrN2+ClXHrm7wLj2xfNlhFQHMOvCGlvm0sA2C9d2Fuj/fguzHIPKaS9sa6H", - "GmyrAxUu0NFd/2UssBbrqsS7F3LPh8jiHE+hVt4aP0ojqYTqSoKTAyOFbXErKLCp/4mgiqh0gC1OQIDP", - "XUavLTaIqEJ4iuuVI5rKUhpP9PoYouZCL+fWRNCEyVtPh/XN7we9/zHmdjTq72/98Jf/u/fhP4Nm95oB", - "QRLRi8kEIq4uyaJnqjApPK1eoX+YEhBaq5jaM0NwCsY0QHu3XMkf75NBwT0Xv+J0aQoQquaVUNpeO6G/", - "/EdzoJe3jO/gwlh7dj+7QspdVJJV3N3LGykRUxdU7zLqNvtDBkl6l2QhkVeYzcp27sR+J4tPvFB8dGHI", - "vU/Y/AKNKVS6lEOm1XscRSTTapWt9UNNuXYObFgQnPjt2AJx7rxYz6wJrCDo/ckSnPGbd+c/vnn369Ho", - "zenLXw+OR7+8/G+IdbnqmR7inqa9vSdPbZF2fyW3g4VCbl7voo9ObL6CjXmY5KDZA2CZRGmucoiOIddR", - "kks6d55Sldy+ssVy1vLtK0V8JhSyUkkoPMNCdid0QiDAAe5VG11EpSNGKqG6vbXyUIaWRRdDOMMOXCmK", - "O0kiWFdEb0V4tcuNrS7601m7u8SAxAYOO6T+6sMWsKK+phKAO1wQjPcy2oAUGleC12UQb94MtPagaDAY", - "gvmFKy0NXnyJaqjvVpY/nfOkp/W8hpIRQbO6WYtgCgE0ZVIzOk3el+k4oMxYG/eUTnHA4RJyrHyRqqVu", - "QGtTx5b2v7F8Wzih46heT8McS7NUtfoPNWuJVL3mfI9Ui/cNCMQAsWySeKkXZFjN2E2Z2rLVhUPAITEH", - "VPdVadvlKXMwkT34aH028koF05uZN5LmvTlxelRN01uxQKd6aa5mRBBvI+CDsk7BDZfMJii1gKMx1Rkz", - "IsrgXZfdpMVz8LtLtFGYwNwSFGnXy36B1XUoTvB10QP4lLBccsTCPMo6WNuvfoSaAW9d7U86cU3AMGpa", - "bhghv0pFq9bEUdXyZvhUtTxv837w4FletYL7NZ2tGnGWfVRIM0SPf8NU/cQF6MXN4C93DrQPl39MBIDh", - "1WH0W2HQ05TEI56r1edfv6alR3PlF/Vhy/rCzgaAgYijSl5zEy9w8CTlGJZXWi8HiXJB1eJMr5eNaod8", - "UFfUFxYSOoKfy46hkOqnT2A9nwQyZ14RRgSNoEytPo8pZqAxofcnXrVCU7hyCbgWRKA3h8fW7uKwj0GP", - "pgpIzwWgHpwed7qdORHG9tAZ9Hf7AzjMGWE4o539zm5/uz/ogFY1gylujXOaxDaR3GrUhQZ/HFtJ6Ef3", - "kv5S4JQo+OK3ACQCBGDa10EFwVNPicwwFVaLzBKAajAEQ/XXUHfBXaj75lbummVvbTyGfGtIAyLZG7u5", - "H0BQhrMD09wZDCzCu7LXLyQxmcyJrb/bMNqy31ZSnV2iQBmCJTXPyZbF0n/qdvYG2zca06qhwNkNdfyO", - "YZvNTEA7f3LDhbhVp8fM5CfabHMbF+afOCAk/6z99kHvmczTFIuFWzB/tTIumwRjIhF27xo9TkkUaVYB", - "xZL66A0j5jnCCmETwi1yBjWm3YeaQqunwLTtNrlAa/qRx4svtoSVPpyN4lOVnenj8mmJnr8c7RRkvLyR", - "9pGDGjdUew8E9CMuCqQ/2EnZG7y4+04POZskNFKoVxCwDcymEmKfEgBOdyBMXKDfc64wKvIaHtGRtjLr", - "uCC3bnkVbf1B40/meCck5A84JSLFzGSJmHfWHPql42x8M+VxXnmrOcI/PurYm8qhEZmLCgS56hH1r626", - "MLh8He0FoChsn2Z68QMS/t49nHA72aJG7kMeOahMinJJHtNxsr7GcSmEBGW5V0R9LTQ/uM8ry1ZT+BOe", - "osdCwK9IIeGVu7V0KWxlImdGAQ5KgG/LzE373XdV4e+8fOKFC4FfQzcNdT2UcTDjeNFHbk2N0q8WgDUl", - "CMwzXr5WTvXwvpYTtnMfJwxmXHiKvl1T366pVafcUIubAhxM75S3sEHcyALx57M/3Nj68M320N720Mry", - "wMiVtS78nY/7yIbmRjwmSM54nsRoTJABfnJBOAqL/vQjwiKa0TkBdD+oVpcnimZYQIhNimKssPGhNxom", - "Vpoliua2dHM9F5BZLnAd0EOSEYTAjJqAOMtQTMoYiZH+xEbNlLiKS3XVzdkPGtiLBsurEV3NuCQFsCFT", - "3m0Oed7SaMfQbH/Izi3irV5AiCp3vEaSBHB7V9h/OEN4yOwH3zsW4iLiJE5LzoUFgCdSA9FptmU5x0+P", - "dCQjHgIdOicMM9WTGYnohEZ2WpdkYQNbgw22KkClB+zG+f6kyFxBO5th4DoIXQqjFB8Vz5ClpKr/hkE0", - "eJTkcenkclhKWIxxkgQrlEwTPsbJyKzPJQn4BF/BG3ZRSodL6U1iPCamln62UDPOzN/5OGcqN3+PBb+S", - "RAw7m/0hg4wUu9Yk7pYCIrqCinZpxvU5Ezw1fW6ZIW79cUkWn/pDdhCnlDmKgE9wIjki1/AdxI0BeIjh", - "Xg30YE5T2A9+mEvFUx8C1tGdGSbPVZYrm1ojieqG4E+HTHH0hwO5/LT1R9njJ3AWExxrOvFeMVMC2bpp", - "1HKE9exH8GrA3U5gAYYdfZGaMI+pwEwZ/NICpRNN/S3dKMpEQOnY+gpHmKGMZ6bEBhDVDGuSq7QBoBU4", - "SZCCo+S+1YI77GTDfCwGYTpuBCA0iHG1Y0QZOvnRO0yDvefh8yRJJEgoouS/zt78iuBW1ntgXivDtUxu", - "C9MCA4pzcJ06nvYSRzNkHFVQVXHYofGwU7hz400Yay5tuEyvBz7FH/TQfjDddGn8Q7+vmzLuyn302x+m", - "lX19lrLUAKIOO5+6yHswpWqWj4tnH8IL2oTjdlZhBGjDXHObwEkwBcgd78Y3VyRmMeL2FkgWCKOSA/mB", - "K2PKsFisyqgMLL1dQT4xkYzeYvwxhMjFYWd/6GIXh53usEPYHH6zAY7DzqfwClivZXMJP7jPCudmQURP", - "B4PN9ZDgdn0DPssWjoEvrAM2akVF/VG9gxaP9s/lH/i31j8L1w9muvMSo8ko/s74/ggdEJ7E7muiARdE", - "TezGLCKJE7vXG3ru33mgNysiSXLfBPpQ5Fm4x4qSBY+KHGGzymO00nz/wBQ3uK9LpWK2fxj6fXT284D1", - "3NrOydyFOocLtgAYj1WlkXkZYYnOYEy9M618v4Rf+/a/TvcDcMmLhE8v9o3qjhI+RQllNh/AC1TW4oFd", - "S/jI4PEU31l4Hlctb8NIEv/6xz9hUJRN//WPf1qQ+3/9459w3LcMzhwU276YESzUmGB1sY9+ISTr4YTO", - "iZsMlMMlcyIWaHdgbf7wCHk1/62UJodsyN4SlQvm5U2YwnXSNmhdBXo+lOVEWjwjSBOa2Ko6JrYxYLdx", - "Z9ks5b2e6G4AFxJm4E1A34qOBgBUj5qK41YT7YRNpmbOFaNpPUxzKVhvPX9R5FoZ6u2ZAd6QwcASh84d", - "PLCTRhtnZy83+wi0LUMVUDkJdIeyGatG9L/xpPU8yXCUKkOBVTa8KcIZHtOEOpNjQ9kXcwRTHM0oI2V8", - "cQG67prYdyPVPObg9BjZQMguvDpkb862wMSqSKRyQbqWEwgLtVrWheM2zwV6AP5FFUSH9ey7QzYhGPKE", - "jo8ME/DQyIvEyKJhBogmEONKVaUEXXfIDKSuhXDWBy/lMUngI+h/ihW5wosuKor+ujIxCVZaIZZd/fKQ", - "mVRDuwY9wGxB3jD7wM/MkHouktfmbAkySbRqDBH4pv459L0x4QLZCOdumVfqujPZpmZYetFSHL050/Ob", - "gibIjT0QWnpz5nZjs4skR1FCgRoizIZsCoFADsWYs8quFgllMyziXsT1JeCjWl0yfpWQeNrEYw99IrtD", - "SabST+A4/Vwn18cmXMyWJ6APsUHqW+25O7LvtHPd2Rb/TL47WxHzBs47Y8Elht+Y1f3myGvhyAuvm3Pq", - "hTxrRw6K8u4ifk0XDxTw62hvec3NE2/JHsKihzYcxg94RbhAp4fHCMexIFJu/nvb+/RMDZWW8p++HzUr", - "fojQEzsWLiz6obW3VAnksbCDt3bUCLt51QsN+/fbVqUKUeNNVxQkKq+8u789ap3e5Bophd6S1r7dJGuD", - "bamMONRbLKmlB6JRQgrxpTinPhWtsyqbMN7iylkpLln2fHzkDuT92Zdt1zmr3w33wBSPagzxARlhNdXa", - "Lx/+mKj5XbGLDnZ7hfn56yLNwf1JQfdtig6R+WNSF+PasmkuaIBOGi/QV0QZeJO71NNtD4GJnxHhTrUZ", - "6MLMupiW+RQZnBaYEFhiVuu+x+aVdqqvae/PpPnC8txEYrFL/k1EaaHslmu1SsE9trWw706/rSC53XPY", - "iiWwwCKDFXXs3E5gWd3AcsGizW+RK1+cok1cY6nECjdvEheWbIOmVOhZ9yXXHTC/8LqW6axeSxmaJHQ6", - "s06AmE4gVk/5hcxhlDv3MMqiYLjAitgQxceY93uqF9l6gedEeEiN/pW69QcEra5XlRzzWnm7vnv7ukdY", - "xOPCedIsk9onX1hhMvRfyeW9/1P3CPNZqRMPmgTGz9h/E0yOCujO/7XzkwXv/F87P+Eko4z8r92DBCsi", - "1eadEcvgvm66+1ZgHjHxaf2FVhdtiTVtKTz181CrlOnga29CmRXYWWfsLhJpvhj1fXkhr47V20rCuze6", - "15rUfSOWHLuyBGZLuXDoVTVI4n9voe+sQtD3bq+rdE8ljAAEu0cYpcyFQ13FFraaOuK2vIlNAdd7jTGi", - "eKulPcK9/6cySZhJ38goUazrN7tEG7uEv1wrTRN2K+7UOGH6eCDve0FsodWGR9/gtu7BoWMp0oPbqni4", - "S8CtGZcKHj0+7AV7ldCC4vxro6VnsjyQK68PR7rHR11YSCj7D2WHbGrzPfkp3TjuXfG2/d6/0HOQjuk0", - "57n0s6ZTrKIZkRZRICFVBvzYTALl9dxoFPiKqXRwn1fHvev83+j+jqwR9Q01zNsEG6yT+d1bbWV++76W", - "+Q3askVdsLWRuq5u3mZDEojDW25LxhVY6uXklNC4QroIeqcVlVJdQKBB7A/Z/9b6x2+K4PTDDy69Ox8M", - "dp7C74TNP/zgMrzZiSMVwpSgtszpwa9HEOExhSB+qIRagknUx4HSXCpDeq7kyb+dglQGubTXkBwVftOQ", - "WmlI3nKt1pDsXtytilQtm3TvOpKjt9CC23oHf04t6U/uuq1ocDKfTGhECYPiU2D8k0uxykaT++a1vSVY", - "ArOxEl6gY0USaa1GFlxrjYReFv7/kpGE3cYaFBxhpUiaKTQVOCKTPDFVW5Cc5SrmV8yVpIAJuupmtJxP", - "6Hp3TY1cI+EE2XCJ9raablGW8b5VXdvxI81Q5ZmtMG6Vy1K0adYuH5Z471anbHHV3r9W+ZhJzKhvy0uX", - "aQ0hUGLNuHnS3KTzFl+W6Ix9dH7+2qXuavVEuIJ9irsqfa6S85D5Vfr66GVZ/tC84FrQ6gOJbao/JDTb", - "uncxwXFCGYFcByJDWbbV2poPeiy+vAQcLhx63y7uNsfS1sR+OAn4wVjBvciahU9fWYdscTS9mqLFaXHy", - "JpyaR8WvLAMKMJ6QrLeFc8V7Fgxga8YNQmQYJPc0wRFg5OrXDHyjxV8xeK1+UwCqIniSEGFgObNcOXFr", - "yIrBUaZ4UXjaSmYXuvlRzhRNLrom1BCwlSTCbGGx6Yas0pmV+QAjAfA/YISCZGbEtSq6etCU5xLeAjgD", - "v0uEkyu8kENmURXM51B6XZDIINgmSR/9zAHQxlST9hivKeX6nRyyCxonZGTxaC4QlUjOuFCEkRilfE5k", - "tV+CRUKJgEkcYr1yEqV4AcCQBiPXrA/PiAFfrKDecP1vzGIKRUF1z8WU94cMo53BAKUEM2kxLCSewIVj", - "20AwiMqAvkcY7Q1e2K9q+wbg5W75N/RpEoLMeYTHyQIRTcWm2PYmbGBqi/Saqu96+yZUSLNfhX3TVl+s", - "bCyVruZs3EU5K1E6wNafswJUQ2+XygWDeVovIKGiuAYtMNGYRFivJ+PVfgASlkdRLkIXpN5qr1r0v6Pg", - "6E3vDJYqjIGRgMkgIjHsOeNqBmeaw1Ha/L6Bqkqi+nNcNMFDwgXCyKPr0qJBohxY4wZAqF6UpU+ZK2V+", - "sfm9Ozv6+FpG4I6/ATF9LPcTEBGfTCoHcP3VZA7wqtyzZRL+s57TQ1fz2mdxMcVTxqWikWOGDqHaV5q/", - "KYStFMLVKxuk5gkXl80Bxz9xcdlWA3OBkY9LEfNn+BU6IvTwAAT/4f0RYA03yoommntX0ur0VZxSELqo", - "kkVIMEo4m+pTVFrl791t4Gt1GwbQUl+mwji7C/gxrYSM7I+mbLYX1g0uhsi2+tC8SPd+D86oX7lCNM0S", - "khIoq90zxKY3u4SqGy8slF4B2HYzXqlPlY+rYHRBaeIPuk4cArpyG7YB0vvydgWZasKn6wFRi84d+mcA", - "EXXI3klTquDCuJ4uUMGDtUBryo+gqxmNZoCOCnqrbt+Ap+IsuyiA4Tf30Ss4yD4+PnS+YYqOaFqTPCEG", - "9HSephf7y4Wj35+cwEcGGNWUiL7YR65YdHF/SP2Wj3aqZ5FgqdCvFsN1o1DGYUcvFNb6ZjG/TYuDWgL3", - "D1kIE5WRK9sgnaALDx71ogG7z/Hb13wqvxpXUVluxcxFcWRVR6BNwuJOU5AHTcKOn+3BIFQFoCVKqxnG", - "HYO0Lg3mNZ8WpV4qpIyzrC352mECFc/TdAUNow0P8lGqmOfqL1LFRAj42FJ3E3GjDRzZMn/4UhOqBfh0", - "B3sTyC8YymRqLwSXSjPVTrdDWJ529n+z/5qnaafbsePR312pLPVqN9xAyF+DeltvcDn0Ru+QB237TTy/", - "CWhtlel7qLW1G8Sq1c2S+Vvzwp/ea+hsdw9IhiAn1Iy5X5Mo6o23avhhvEDfhZG9uI+RAUQvihIuScXR", - "83gA/qzBqyY7NhuM3Br39PDi3FVEaxPJcmY/PXNffgU6+LqYETdm5KZ778EjyyN4zGAFcmk2Ey7qqHDr", - "okq+ekL6cluyNNU2FPKNNm9ubWxFmFpfWGYR9oPYVMjEueIpVjSC6mzRjHPpkX0B4W7qKFojckGZYGIx", - "2q7NJLjQpHphzdEXVp3Yt6YzhP1Hto8+fG7zD8JfuEflFz951oGC43edCgAVTCTCaCwomaAM55JoqS5P", - "CYoWkeaKphwfwdEMRThTuSBQaZSglDKa5qmPza93bI4BR+hiO73oonGuUILFFLQz89AF3UQ8TQmLCdjp", - "hmxG8Jxq1VKgBCvCokVPEqhQPifoiovLhOMYTA1ZjMHjAxVOBdEUCIUOUqJwjBUGQedCn/iRSWa6KIqW", - "G/WekeuSGuIhEzn73lRd0c1euIFeIAJlBaicFcVtIxwTFgXh9s++bjb25W3SZ0TVJ/pAEUK34qUPGTLk", - "217dcL6OaKJHCw3Rgs2vEHplswpbzQJxZPTveaTNXN0cH8jRVCzxqlP8dXiYCqL7arxMD+9G4gLFuenO", - "O5VA5n9W31DBUPygK8gwNdt4WwdRUcWzWOYb8bytP9yfx7ew5X0lnLDbqNg31YsrJ/01sFy7qrfiuQ9k", - "xLS2JN8m93As2EV2PZj4xIXH5R6LsbWC0FbwbZ87KYFB++LsG9uus20b+HBbtu1ss0uufY+RU9aDWNEw", - "B7dm3EZWbU0H/6ZZKbXZeSzzwVlk6bm4d7hFxxozvEg4jv8MwcIr/EcRF8LAYACwxmOCiPashn6aANjm", - "ykKUXZe1+f7kZLOJSwi1kkcI9Yg5hJeaoz9L42UD7ps5EYLGDgbz8OTIhu1SiUTO+uhNShVSHF0SkpWZ", - "LZBd2Nfzc4AgtWHXkT+6HcKUWGScMrV2FOWrdzOY8gducVG+QlHS1hz45g5v7Q4Hy/7jY2fAZSB3w0xg", - "tWaqsFpbC5myCRepkcvwmOe6dc2D9DLp/TSIBROaELmQiqQmOnGSJ3DcoH6NrVFuvzO73IXYXH1yTNpc", - "RkRKpaScySGzOSMZEbpv/blu3wu0CjoEFC7466lhkl9HEJ8ejIlbw6pp1QC6CWofd/Y7WzjLtmKscEOg", - "mB3eZwzpJ4jKQ3KRjnlCI5RQdinRRkIvjXqC5hIl+o/NlWF9I/juS1dgv/3J0it9zCY8WN/S0GxBzH+q", - "7C7L1pxj8tGxtVfEPyyO/8BGh9na+hrvguCkBzXTHYAPyhVN6EfD6nQjVCoamdQjXKzd+5OCqfaH7IQo", - "od/BkOKWJAbZALTLrUzwaGuYDwa7UUYBBW6XwOCA4TU/TqHHw9N3Jh2VpFwsukOm/wENnx+cGu/uBFtr", - "gjdQW9wdHW+9WRPofAbL9G8cIWgmuBLFILjh31yCN8caaTxDsuGI8myVqsSzP30Iq5XgvtkVHqddAcCe", - "itlsFABfDpUrbEOY8yRP9T/MH8fr8M0Ujmbv4dWvRto1w1nbjZvgoziUdk4xMfV3H8TpYRbsscas6oVz", - "UwAhphINGLwFDtSfkbq/vPneX8ev0N1pV9TVtv5qztZ933x2DA5pw1+Px3LMDaW5mSi+2vp0hWmz9enH", - "hEeX0kKy+GZDrbcBzrr+scTFti5CEBMgQxRZKCMDmEVkd8hqBkiD/CMRRoqIlDKcbMGcTSOA8O2sWHjO", - "KSRqR5Cn0pM0BuykBGC8AQZPzwYMVa4Bz6MrbfU//x3fGak4GpOIp8Shnm+GVLe/Yap+4qIKYf618MVz", - "b/0BGhBTsLevQW1v7vGzUNxP8DWESse5dSi7EW284uWPxhTURbA3w87uQA47XTTs7KTDjt6BQwwmVKzQ", - "E5RSlisi++jI2LcgFffpAEkScRZLB77uLHi7A9mUmGvIsiHL8yl8d59ij6UqWMq3tpMQe9DvIf09JO2g", - "Df/A2TMZd+HQxYjnypj77bmyb8VEgXlk8959td4Z+abbt+Hkf7PHt8KjYJc1u/S23nD2LJcz0mxye20K", - "GuVqDKDergCynKG/87HsIkaujDVcSNVf4nv661PTwX0UHNBd3aTYgJ37t0oDLSoNlGsVBm00AZb6SnbU", - "YZAbyXXGhQI0R5tzb2gINAlAkIAyhW8Oj4cs0qzIQAwKknLgThYX3dzCB387Qy8P33bRERTjRT/n480+", - "esOSha1hb300Q2YkMcO8IszQ2FAtiUPXsxk7UM9dBovrDh6our05GQHPitsrFyTe7cwIjkEi+aPzmpvO", - "AujDb1/rAwQAwObLYts7K4WPzluixKJ3MFFELDd7YvOkWIGdYS9pB0VnBTcDgKk7lA6BrezTyAYGImN3", - "pxNAzPj0rfjD3Rdxvh8vmYkTMWX3xrl6tPVb4SAWzDHEAv3ruiif0JQlbHnZSgUDumyK/P6KTO4reVcF", - "Y/7f9XTBTB+toymr7JMm4qLsylpPr0sOnhlYZOuoinCGI6oWXYSTxN5R9iYoIlJ6hfg7FgRfxvyK9Yfs", - "bVHwxSb0osPTd13nqEUxlZemBeuL7aM3cyJkPi4Gh+CgGa8xrDmJh0xxFOEkyhMtbpDJhESQiwt1XGSD", - "L7cYSucOz07ZSbDojBfVnj+6WndhmoDdK8miTnFbZqu3BIkSTNNmEHIrqEHAIYQajHWjnCHKJokNqYoE", - "lxLZpnokoVM6TmyAkOyj8xlBEqdkyLIEM0YEyqWJitdD72WCSJmbBG/dAID1GorqohJgMBNc2dCEhHMh", - "TTSBpvD3J0gqkq0gs7em5ROY8x3JtqZx29MDGalrY2g2hdhXkN4QQylmwTUd5YkLYLzXUHQzoIeWEh/L", - "wT8XdDolQp8KbJisCcczx9otpzn0lYzlxrqXZ8Vb7epeFq16WYlext5KgLhRibkdd24W9Rfo/JI2Ygja", - "RzfLIv5Ff9Sy72q2angQ9tFnzjJUwvPfsVrmmZck2NaAVVL4YzMneSOvHNVKou16WK3WmbV3menaGj/r", - "wWCzHjNaFq6kzzYpvF8fIQzuF+XhvoutPW7aqqBdVXTThpT/9aj6XwUF3g2c/gOjnNwCTv+ryrsHvPOH", - "wz8JHtSHyqOv+J5d0d0/PSL+XaXPG1h8gGNrSp83XM8Gr65UlN7bd9qpSbbFP5MEb+MdbyC/u2X/pvW3", - "UBm8xVrngtYET9JMLVxAm/VVlkFnkn4k/QZHcBG3eneu4FuEdH458nB02hjQ+eeskf8gMaO2hCCV6Pgo", - "UHz+kWEM+meucrFs6Vunh0U0o3PSbHSvnmC7RJkgvYxn4FyJzYLZ9XB3mcKiP/2IbPMWc9X+C2pQAlQ/", - "iVFMBYlUsjD1QDVHMH18J5HgWhOA51wsmqNEzBH5SfD0wM5mzX1oz5Q1hpVxhumiF2OFe3PHbVaY0D4j", - "utPFU2qGhyhDr35EG+RaCVPpAk205oPopFhSch0REkugyU1/wNuDBssm/UhG03GbUa6oWfLG1oRBUS4V", - "T93eHx+hDaiBNiVM74UW9ScgyWaCz2lM4soYO3OemFXdbljQm9pdtVBRFLBzyoUZ3IPIMG0upOlHmlXZ", - "QhESM6YMw+DWVgWpnimTxK/7w5S5ABy7R24U364wq/ltOGVHUyLU47SLqDg3EM+b3665x3zN+clQ7k6r", - "3HYuPGe18bpdflTLtKW7KPxQ5M7dr9n6/deT0kPlo8zmsabzeaGQNpnNvy4SHNzf/XDf5vL3jzgF9BVx", - "yrdnKocGdIshgnkNMd0xmZOEZynURYd3O91OLpLOfmemVLa/tQWx3zMu1f7ei2e7nU8fPv3/AQAA//8A", - "qZIc0fkBAA==", + "H4sIAAAAAAAC/+y9+3IbN7Y3+ioonv1VpBmSoi6+KZX6jmM7jnaiWNuyPd/eoQ8FdoMkRt1AB0BTZlKu", + "On/NA0ydJ5wnObUWgL4RTbZkS7YnrqmayGw0GpeFhXX9rT96kUwzKZgwunf8R09HC5ZS/POxMTRavJFJ", + "nrKX7LecaQM/Z0pmTBnOsFEqc2EmGTUL+FfMdKR4ZrgUvePeGTULcrVgipEl9kL0QuZJTKaM4Hss7vV7", + "7B1Ns4T1jnt7qTB7MTW01++ZVQY/aaO4mPfe93uK0ViKZGU/M6N5YnrHM5po1m989hS6JlQTeGWA7xT9", + "TaVMGBW999jjbzlXLO4d/1qdxtuisZz+nUUGPv44N/LcUBFPV2cy4dFqfbI/c5G/w68RmhuZUsMjou07", + "JMOXyJRqFhMpCI0MXzLCxVTmIiavnpyRSArBIuhMj4WcaqaWLCYzJVNiFowspDbYxigaXRJDpwkbjkWv", + "39gPJuBJvH2V/rZgZsFUYLBcE9cLmUlFzIJrwgU8jdiwumFG5Wx9Zfs9HidsYnjKZG7WF+pHeUUSKeY4", + "Ld8vSXNtyIIuGfmdKUl+y2nCZysu5u2LNGUzqRj5cZWxlAqSJTRimnBDuDDSz8auUUlj99IQcfG5kIpN", + "YqYNFxT6n2RSGb0++hf4B01IpS0ODdsTs6DGU7mQhlwyltUnSq/oZX0Zfz046D8ajUZv+z1uWGqPFX3H", + "0zztHd+/d+/wXr+XcmH/vV+MngvD5kz13he/UKXoqjIdLXMVsUnEY7VpJlHCmTDkycnTlzecQG9/NMT/", + "7T3s9Xv7jw6G+/cf4r/37/eq01pb+PrI328+eueGmlyv8yB7miaOUCYVIlmf9S95OmWKyBmJcqWYMMmK", + "4JFicQeiq017FNqKSIoZn+eKxevf9keutpwLqgkVlmkMGvyi7KzTuYuAicXySkwUSykXsMZrg3jpHxE4", + "ocQdIhhSJIVRMkmAKRjD0sxof4r6wMYFoVmW8AhZT+1QHaUj3ev3RJ4kdLo2wnK3WcLnHBt0WhquK5vk", + "3yVGEiYMU8UJ77I0NbbY9uFyuYO7UfLFTp9cyCSe5MLwZP2rr+Fnt6bVL3FNFiyJiZzN+oTPCCXQC/xs", + "aby+7Aejg/uD0dFgdP/V/oPj0aPj0b3/6fV7M6lSanrHvZgaNoBd7rI3yLQ1F1F4d0STRFK4kBSL7O4U", + "F1ZtA6cskikj0PXGkR99+Mhbr5sn5aZCQ+IaljdrYKuHjSujG3UnVJuCCeF+cbOa0MCYXvGUaUPTDPgQ", + "jKGymG1cyHfY3Ae/8hsXeP+DFliwd2biVig4nxB9sHcZi+BGlJ6bFAIG9OfIu2BZd0LjilEthRWT4DL9", + "tZcLnWdwdbN4kiXUQL+9fs+SwSTlWsOrxQ8x15aP9HueyCdCmonKhbANBTNXUl1WW7peJjzr9XsLqifL", + "eZb3+puurTpR4ydYQjON/bkdVxOmlFQ9KxqvJjOp/Cb13laXcENXayukiys2sEK9fq+2AAU793Px4y52", + "NTi4fs/9NVFWq8Du7GTWB17ta324xdA2M3Z7i1gh2m8zcS/rOgeIOZ0LqQ2PdCc2j8IDbG8q4wDrfFp0", + "R3jMhOEzzpSTqxlRcD+kjPhOCHRCuCC5bpyDQvSfsCUTRk+WRxMTZeuL0lBsqptXkU3KG7FyKxfbX5yU", + "LURan3tQcVpSjmfyKVvyiK3Lbm5rJrHiS6YC7LsQACwrtO3IDpx1YCFCCrZbWymx5DGnXdhBjGOa8AD1", + "nD05IfYxOXlKdhbsXf0jBw+mD3vtXQqaBmjhxzylYgAHAobl+8e21b5/Pgr1zGWa5pO5knm23vPJi9PT", + "1wQfEoESbrXHhwchSTWL+ITGsWJah+fvH1bHNhqNRsf04Hg0Go5Co1wyEUvVuqT2cXhJ90cx29BlpyV1", + "/a8t6S9vTp6ePCZPpMqkQp1t68GpLk91XlWyqe9KiP6/z3kSr1P9FH5mLevElObaMGHI97YZUcyqcuRq", + "ITUjEY0WjEyt0oLqOfbXheSLD/vbKzSCp1xnEvk8eXNaSnOoGrJ3LMpN7bvfkphrw0XkhKrK5DqMCMxP", + "CYP7NyRZ4AoQ14ZLQYwXnW4sBESK0S2fgxadPrZ+/nNLXpNUt/XumxAuSMqThGsWSRHr6je4MPeP2idT", + "OcX22lz71DP4maRMazpnZAfNUqhuWA5PuCYzyhMW73aTsNsm83c5rdxrtTOHZDCg02j/4DDI0FI6Z5OY", + "z51dsUmD8DvIx9CPITxtnQhKGN3mgZ9UbLb+vR/wPsGPKDZjionogz+XKblkgjqV6j/wu73/a680uO45", + "a+seLuZZ2fx9v/dbznI2yaTmdoRrbMI9ATLCpSb4RnjM+Cje7URR2lC1+Xxgi49wEkthc+vaONMPyFt0", + "vvWVV9CmydCRHRUCToULtPLtZyBpBUQWKQwTgdX5Wc5JwoVVg5gwbi9Q2Ftl7LtEznd7H20diuVfP/ww", + "7hswL/tDS2/wrF9oBYmcV1dzwagyU1ZbzJZ71XVUjq51+c9qx6dxgVLNJps5yBkXgsVoc3cH27YE2Tqo", + "++ApuuRmsoQbOHTmcFg/cUNci9auEhldznjCJguqF3bENI65Nbie1WYSECFrzgyKRgLfIYo2qFSf//j4", + "4N594j4QWENn/YUG6zOpvA3d27bEUDWlSRKkjXZyu/4dvU4hYQooDb5td09BgZ4wLafrud10ynuuF/Yv", + "5N0wKrz7ev1eBOSVwN9rhNvvvRtAr4MlVbjq0H1lWP/lv1T57fvyo5Vfz4rvV3586YZS+ekHP6rKb0/K", + "AfpVYWr9PDg1DScdliolF2bABW4BCODO8jS1m1doWc6SQaSoSHhMdWHnmwSrJ2siFdl5+cOTw8PDR7vd", + "hSuuLyea/84m8+lGudkN2gnK8BqB1wgXZM7ndLoyTA/JEyqENOCMjBZUzFlM6MzAW26sNQX83mibE6ZV", + "SGKqTUQyvx/RRw/fvaPm0X1+pR/9nk7V/O+HtNdmXcx1y/qiBY663YSmRFER2MSbXdQpfQeav6WSkPPz", + "1LqrSNHI7YFuDmFISoV+xt/BmhuyX1vp4OKGtb/ChSWkGOSC/5bjbmcJXa3rgIbRdIAUEVpeyzfsAdLb", + "j4+G82Pf8TO9otzAySl8pm7KfSKTGO6dGVfa9Lp7w64lGDD1sUSk2ikLS0zrFNFcwNZLvRhn61lJ+IxF", + "qyhh68zds+9M5c7cGjO4c/DPwoDYlY0zFeDE/sez4hO1n5+W36v9/sx+/H2/94RmdMoT7jlznU87n/zE", + "Gf627VK1s6f21Zfuzff93oxRkysWWMtzjA8g1k48SNiSJcS1BuI9Jo/PTojO1Qxd5kitti2Y7aVmeiwU", + "m1MVJ0wjqV8teLQozJVgetCEijIsgDxOruhKk0wxzYQ5HotxYSHX416fjK3y5f62VGL/tqEhvpGYK6bd", + "P8DWMO71oa9EzvW4NyQ/Sm0GoC47WeqYjJ09Ro97ZMf+STKqtVkomc8XY5FSQecsZcL0CQZpoBkYOFOy", + "2iVUxGTcU1IzY+iApXmCvB86e5xlCRuLc57wSAqS0ujFuXv3ipsFeWlfqlyeOGO4rPskc6ZnTVM2FtSa", + "QWEnVyRTcspIQnMRLZgmTMykitjukJwxNfBL7PeWUMVgM6wRl+QCjgij5WboX98SJoxykQ7dmQvM5Dr0", + "B0tfKNH6Om+e2DfeF96R67z8i3sFWJWbcVBHUavSml6hZ3sjFqZ+fx3VPAH9scAVxV3l0OpKkNqWzRI6", + "R1KpHCJLjHosIpmmqOi5rxDNlkzRpNghuOMk2hPgbnA30lhEiczjwWKVMbXkWqo+3BAswmAE1Se/sTTv", + "40fhr0HKIyWXKZFiLBIg4z2axvePdofkMUm4BuLw8+eWuB2JISdAewCH0Y571Nvkxz07Ma4JXPxD8kwY", + "xS3NEW0Jbmpv0gZxdd28CrNau9twf67T2bl9Y83DYX92BN1fY7EVuikJsCDjChcN3VkbGPC6DO7XdbMz", + "yo2v2K2ICrdTNeo8JtwQmmWMKk24GIuSmkSMVOpeyhTD5dDcuK1LmSE7KTXIXsBqOxbIIb7RpLb7u0OC", + "cqONDys5ErxCFEtWdkBsLPyYr3iSoOGQ5BjBgBzOHjS0jFUiKPAwUVHwvqT0djnnl/OL1iZQOyL22//1", + "7PQ10SttWEqmXFC12iVcl7zWkua6Yy4sMrrdQ80dRbX6rGOJUU/wLpGiLkA2z+tWF4ITQUu62EZhP8pQ", + "iCVV0SIURqcNeXL2msBjblgERFwbL1Xp/aArSeqW7uCrFIVYu9617pDpbJ2zhCOFI9422ZPiHgmLSIU7", + "ft3bBW9W/LgsYRHwKhtPY/U3RpxzmcgUWbpg67PZa10i37ne9nVNdqTegwnvVm4WONMqF0PyQhAUI0hN", + "ihgLbMpFlOQx06TCz0lpvvXyRS0MqiD7sfjX//v/WVkiJktOC2mDvOHKQAzJ71aLnSmaMmB69SstWrDo", + "ciyaYgiBXrUk1F8qxTJzXb1QFJ8vDBHyajgWJ3ZMxYB3tJyZK6pYnoGaSQYDN+qBk7N2bRAXdsRNI0Tz", + "18beVBbnmrGENXdesaH9dQrbRqq/lGJLnVbn1LArGlCInwPpDZZcY8ga0oRrS07OhgQfoyshWtinwEVB", + "iiU7liq4mI8FPnEi8S5xEq0lM+eaHJIXKTcF8QtZXC/uphsLCDCcMibQi5gsbQBySaor1twAiObcBwfv", + "cD90NOYw9omRk3ketLv6uy7mikWGvDkdGDl4c0qMorMZj4COMqbcqO0F0xx0kJ9DjEHSstL+RR9CkQxB", + "yVA8noOQA8vlRH/7m72cMqZgXK8en8EFh6v/LRn3BDX2nZLTD8CTw2MWk18ev4KrNXpxvutWzammtuMe", + "XDshI3i/p/OpYKZt/PYpxuCGtwJDabcxX7tEa1u0jbw/TKKBi9928M0GmaTkYCCdgBBbkOCQ/FyTX+0L", + "ReAKiKDrEn3Rgz/E35JitN+hQENSRoUm1Mn6lRERrsfCSx81gQMkbeJWFF5VubBcyt6Hgwr3RbHEyiMo", + "kRVB+kv4/8FSy+jShXhYLbPgtIKmeLqrS7cmWXHTScjZagKoKCyBJfVfrwmeoCNoQTO9kAa1av+PPcW0", + "kYrtgnLu4qXgeUZzzchfScpSqVZj4dv37TlLqLExDMW7M2BMPbITJVKwqh3BBhDQsdBGZhmLnWPiWzJz", + "h5sWQYRSFbZq22gs3Cf0npUANClmgRtQemCSVbnJlZnsoqkBZ1NMC+acp3bYC2myJJ8P3Dx7ZCfhS+am", + "DRPkvzPbB3hRpBQDFw8JTUvbCT4ihqo5M870rLF/pJlxbyx28NwS/Dc4VtJc8Ahvc2w3z/JB1cTRIzvP", + "z17vlTFLY1F5jK+AUW/A5SDhKQf2tgM/kJO9F0TBDY0/czG3o8fGdjrFJLG9m+KQvKFJ7s4pUjus71h4", + "6R9HOjAqN4tZnhyT5e9OCKvvR7GThVS0f9h3lqYELUytksybU6LpknmCxLdApdYGjhGNl0wZrp019pLs", + "OHIaOD8X0p0mgrGYCDkW1b7gICOZwhCbIj+xhhhNGos0FqXY1ogk91ebdxkWpggv+HS32GzWaFp8DB+u", + "tVxDRT4vlPrGRZLxdq8q2CLxoNCoXKaK/srIi4wJaBXLKE+ZMISLmfRN61Fko+FBW2Ray+fPq9dK8Xm9", + "kMqQOTdEMRDjpHDcbNwbxFwZOP46n834O9Qic7QCWZkGe9J94FDjXi4uhbwS4159mA/u04NZPD3cuht+", + "2P3aGgb3AZmes0q35utt9qE92egwA1GTGAm6c1XzdxT+EdxlZUoSXHpqAGcl4Syu0HbfRuvOmWDAuWIb", + "A2qIl9BaXD6DoCh7616la3liWnbURs62buiWOcwTOcULz0+kDDodkpP62vUJxQcoMNdXuciWqASGkh02", + "nA/hdskiPoDI0AE9GIxGg1GT2pOjgQ15z6gxTMEA/59f6eD3x4P/GQ0evS3/nAwHb//6H0GFvGO0qvdr", + "u3nu+LPUJ36w1RDW5kC3hbfe2LNWHX774UXTQutOR4rhMaDJ1jFAxMGTSvN2Wn9ysh71ZlcqlmCHHnK5", + "l/Cpomq1J+ZcvDsGcU6b+rptbrvJwBJIjrGCkW9AqCbOwvLr3pIqToV5a8dIxlXjAMhvBG8NuPef4ojI", + "YOD7cezLR97XHABD8os0hDqPNKh0x9ho/btzRdOUorVzSRMe4+FwbjbNY2aNswKfkaWVkhQzuRLkaDTy", + "TyaVQYHcILWfyrjnDKcqWhBnEFIETSE2QyZiGXzSyJg20pQqS7F+0uCY/XWv+GvH/zl8+9fd//0fH5XU", + "kdA20Lg1Z1yTnzXivKER2UnkFVMR1YwkDGar+yTmc260Fd5iDI1Cg9q3JLLBHjZ0USrChDdRY7s6Oaer", + "Ac34wFlerLv7ZybmZtE7vn8YXNwd98fg7V/8Ty0Lq/IkpK+9lDlaXvFxNfHaj6GT88Wvbp6g0yXl4sS+", + "tr8uTn7YDvuJbNppq9S1bjXcMUVW0JaBrGfgQ8RRGhIilkwpOIqWrz05fQoqzCVzDA20RjLOR6PDCBvg", + "n8z9AkIcqDL4225d4rHdhYSdX3ssWkgMg0wSeR0jZb/J1NuiBDf6yEKrXXL/9ZhCdKJbtzgGu7uGZKrk", + "JVPokoEl5kyTS7ay5oJ5zaDJxJIsqfKuwVeYloBN/CP0BjFUj0GJxeR8yGVlyrFFp7PZNChGk5JbphTc", + "XahX6khmDBiqa4ZTIxdMLC9ISjOr1itmjeYpNUxxUBdLw2LKYg48eiwYHgySUTjzUSQVmFhhb9EGW67C", + "N5pcWDXxAru/4ALI+sIezIap9I/ei9evvn/x+penkxdnz355fDL56dl/w8/2pd7xr3/0LJhGEXj1PaOK", + "KfIff+B839vgWYzpg1T3hVRO2/UueoxXoRkfyowJyoeRTHv93l+q/3z7/q0P97SZe8vecXBg74NiprN9", + "BmLwfQKUJlKVkacUcUGQRXmzQ8Xc0DDlW8nvWkcCVRQuJ9NMB3NTmnaLUg7dH41Ov99z0SP3/D92h8S5", + "bnH4wIOkcuKxXgD5FLgc6E9LEhm5DI1ZWwq+/1SIwaOvNZNcmK3MqWy6zqMGg/LpNVjR3pSLPQ3bMIiu", + "t+5INzcOVH4mllxJgeo5ikvThOn6WfnlxdNnk2e/vOkdw0UQ55hH2Ov3zl68fNU77h2ORqNeiEDZu4wr", + "poOBj4+nWia5YQQblTGmQ3KaawzudMxjlqMdFOUzTdIcDEvJirB3UZJr4FMoDRiThPKPHw4O9l/tPzwe", + "jY5Ho//pHLAKpL+FeT8/e20z1PG8M5qYxQRdc9te/BHbPsGm7/veNomqfWCVij311ko4ve4dsrOoS1d2", + "cgQJctw7fP69PVD7z8e9CiG6VOCiF9txY/Wefx9al4o1al3SK57ZDDcRe6pfB6Ox9vpcOzNR1RM+EBQh", + "dqBlTFVMppKq+Nt6LA2+CYb8bzRBawVNnJ/Iit74jpUlJdOoK/sYn9LYqvuNJUDLT2pNiVQQhp7SJQeW", + "M0hTLv17dbbkjCk1G+qw4mAK2PEq4UK9fg9m5v7jJ9jr95a/13Oqt5sDXWhMJ11xixJIk4wLtkEL/EyE", + "fbApJ5LGg/2PLOtX4t0aMDT2QX27C+XUE/gautOUiviKx2YxgYx4GHLghnRPSNG4uCbfwUxo8q9//PPN", + "aWkQ2X8+zdyduX9w7wPvzMYtCV0Hc22KieRZeBqvs/Ak3pz+6x//9DP5tJOwIuWNxHO3/89sD02AGofb", + "ZaPxW/yuhRxW5ufKFl/6FgyD0JUrl0wldFW5TtyYevsj5OmNUSlu3Y3uvYoJefPlAr15ce150xp3MApf", + "H19tR5+57UgxZL2TrMDK23RAXtrWpVofILoAzX1PdeF77UJpBaHtH5y6Pw/WSS5McfqSZzaYYkLnRe7n", + "Jni980ueOaUZ3/CufMvo4xx6JlMpwQlo4XPgbCLlQNi7d5AaiNTXNjoArjS8NNYFIVDBy2vDNre+UJWL", + "PpnmBvRqaRhxFg6Xxw9jwcZTRvIyUrOh5boJrkcf4LJcMiVYMrFarO64MvYl4l5qXRyc6oxqB/elTJ7V", + "1+vpT6fnZOfpStCUR+Qn2+upjHMIubOi2m599fpjkSnEUIGPADly9105IzI3AzkbGMWYH2KKnRXODock", + "sXx+9tphkWiIBHrJYGGZiF1UqZcoirBS8Y2x/uZat9XvNxa9DW/G+9A7Hq5z17w8Xd3Nfv2eMUkIQS6x", + "sjVk6thkBfAOFdTn8+f6hAvyXJaAB/ZADslrDQdypMc9YFASWOtUmkVVhZtxlsS6qmaUOJVlM1Dywnpc", + "qTI2FJKjRdA5HGV5nXQP+q3AgUsblQAyQ03BDaKsuOyW9VW0uKpVg4q7v4t1pKaORNHVBmx7RhTUrWGR", + "zpRrJf7OptyKcXHNqOstX390zBbY3P+J8APZaMoujV8f8K1z20lziVzffT+zG6zSSbEmDQP4x1mex7pi", + "LOwEWGolGHvNa7JzAfZFR8dgUbzok4u/1H4AHucVf5CTr4hdDeSbAn6q9q/XQnq3GDBvHNZL9c3347Fu", + "hZsiy31iFBXaBpgvaMaG5Ee8rIhhKYhgcHvwakydkFffEmmFc//qWMDQdD3I0JuxNZ8LiLnCaPEpAw+2", + "tXU7U5UPPNEhnF9vgFqTgO3oHFoPouq6MHNy4W3OF3WeuG6RXjfYOBP1mqaOv1sNHW0YZi/NDXweJlwI", + "1DI3Fr3LTb2OrNawe2+LiHFjKWxwN9j/84JdNMGTlwFVHSYnZ6WjouIxaXNMOIEs7DS5ZCvccu8goWsu", + "kqpvJOzBcLHcTryovIpGM3uVcF1xrFgXifOKYNRt/YgG/QXbtgLWq/Py11XeYALKwE+2pBinxXpgTs+F", + "YHL2e2ht0wwXH1XoY5vPdNG3Oj9DDYsIIJbEB6U3u8d4XIzeu3C/DF1vF0V23McA2y7THqpbSyo768Vb", + "7MaLR4bF/bpsALDMevukbFzmosXfp9iV4p6ReWTHjmKoSxdJnTL0YQaQZ5XONkRCde1iPT3Urm9lzI6e", + "LKKxz56w+4GOnxrUbxPxPm5opxYypf7JC5okF2THNdoliv0d4VDdXkFoWbH9AO/qSKCIs3pz2h8LywUu", + "FsZkE/g/PYFTfNHszL3rT3iJRv5whHrk0dFhPVnBDrjRbd1OHISBad8aCDi6pcClFmu0h33Hx8UNBFAv", + "ZMfQORwdh/cD98Tuh1qqudAsyhXbftYfo3jkMGdwTAnlYvDjq1dnRLE515hjG4zmtzHhLeBkToEv47T8", + "dI2smysPDo/u3X/w8NEwvlRDFqlhrgeMajPYH9KU/i4FvdIgdu2lK5plx8v9rXzebkExvnaW77XNdlKQ", + "KbBkH4nbRXN9Ur5SesouuYi7dvATtN0e7ekV69v2N2SKDfJsrmjMPrK34cYBPbia7bu6pQTI5kjeXBsw", + "DpTYtzubA3jri7WUySCmhl4nftcOdz1UN13ZrqxKHuqvNS76vBkMXQOsCRYC+NDoKj+W0La0gelaQwKL", + "J0YG8la97HDyFNbct+2EGorYv0ZOljMut2W3w9DqOav4ujdvQBeDLOLOeuYTPlBYtYuAusOb02pQyXAs", + "BiiFHZOnxQeKbosuKaoYNLbu7R2paomz6ESeAuYHeXM6JK+K0X6jifMW2zGRImUyR08aCkUDK5JVB5Bj", + "hgk3zdedqdACEe9i7Ix0z4ZFuRI0ShZGLbRhTXljPjahGTfK3SVVzJWORtpNsFwv7X20CZqrFgpxbzDa", + "H+zfe7U/um4oxC1gLYf6elznLS7ovcp9nrw+eXrgTJG7N0bj+uhozGFW9rSM1ic7uWaVTFSpgjH6lVD4", + "lhj8G4fW31KkfIk7uamtXYlX0PI2YKZDQFjYpH8DIOgmw9yKNlqZ3LpBZ5Wh+aE8JRVB3u5oFvFgyjHE", + "yn2vGL1E+Pz1exsBGCb2PgsH2iFKxnSFqFBo5FJSmpkmBXhKafzeP3pw9PDw/tHD0SgAZLxO8DLiE8yb", + "6TQAiD1J6KrA9NtB/31Mpomc1gn93uH9hw9Gj/YPuo7Deqy7rUOhf/u3yI5bkb/6Emf+SW1QBwcP7h8e", + "Ho7u3z846jQq21m3Qbm2dZHkweGDo/2HB0edViGk1z3zwNJNAT5U5eCxLQwE/xrojEUcEv0RkI3AC5gy", + "v+CCFc7Z+pmc0tjXggjfHYbyRG8Ma7Mfcy2tvTXNE8OzhNlnuCGdXBI486fYU7C4lhBF7Y3r9eTguLcG", + "PPm5FE1QnInZNJ/PHdZdsXSnLo29FJ44S+JjUqhsG/kc7mY5sLdtdODm0JEafpZXBepchQjs1WUhGhQj", + "BZ3YTavNysc6cJHlQZJoXcofcmXRJ7BTQqcyN9ar7oqllB+xQC4Cvcy5iLuZO36Q6nJrYgPcxEVNmI4G", + "A5/hbjPgG7ntVaGv8H7bGAH3XJOX9g1rKCx/zvJ6Qbw+fskZFIVPy48Lu7Hrpqt0GZZb0Gbuw2Xs90re", + "eUehfIOZjX76uBq2mjOshWS2SixAKTYe6Rybd86Tghe3GlI6rLtgV3ex6JjpNwCyHYAJ5XZWfFPEcGFr", + "KBsR6eLsAZgSZh5XcJUaJ+3cQ024JR+OhUs0Kn7S1pEGL9p1MAvGFZGKz3n9w3U76x3G7F6HMj1x3Zg6", + "qy+2xeDpwg0Y4gEWaXlmQ2BtAZAq0Kvbk16/d14UjXKMqb40/sfAipSR9euYO2evrxt5myk54yEgHIwE", + "ck+douZjUn8+Gp0P9v/LRs0D+aHExoWNHgI/wrBR+Afbd7uInp+9PmsbU1F1iVRHtzanDfA+ZbRLmQ5S", + "LI5Lsuba91ypUEsFMfSSgYUJT1ytmcWtYvGQPJG5MDbNMeYztGgb38ziigADst9esuMaRgzEaQlWdGpd", + "qXEOVFUFOINmIApo7/VEcLTnZ6/rQAWPQuI3Io1McxjYJA3YA3+A58Q2KJASTr+vi+AHR6Guw4z7rEZA", + "qOnPaATRAZ0pJGBDbExjGxTh87PXL12loTaQZiCnohqRxWcekl+KWlywvLoEYhoGjIsNErQx19BgohMZ", + "CmD5QTFG8FlZAxToBp3eFta3CDSgMcI6SeErEULQuwPZ+y2nigrDBYvJmx/0t2TkvOxvfiA28cbxLK6r", + "sZB15fJBaEtbE+jOFisNtj27LLYaCBdVwyZygc6ayVn5ojMBB/STcB06z3HIDlQfxAU8fzk4efFmL43Z", + "sl8bEzy8WsgET8tuhS0vPYpD0bbOjZdtFia/VUVZxElZirF5EBBA2pk+m9QBezMiUxZRj0HS3LyoqFw8", + "xXANNJSBm9swQRR1dmoqap0g6+KazPIkGTqe6YdM0DI9Q9ZdAT21t5tjaoplFJWvQvJ6Z/CjjOg8ihiL", + "GxEvzVFX6O2Y5CKlSi9o0pzccREdHi2oopFhinzz7pteuPQQstKud0OFOovLqTNZVq6iAD1Wzl3bEX/z", + "Q/14WunA5hM7mzoX3CA+GoJi7xSDtDDeQ3JKV870kuE16+il3DE6m1mU0OIeUSxhVGPJcE3e/DDcGstp", + "pKFJ2xxewUPHp3ZgQnCKYJh9ktXYAPxeheeqfvZ+8Moo5xPwTOEHm26OITkRYbZnA4YQYM9iZWnN5wLU", + "Qe1ihuA+9Su5tnf1uzNoSWoCEvbqC1ebTj9wAYQoJnRb2QTIdc1ovYSpvKyzKXm5vWym7aT9u098jmbD", + "FebUT39uMZXTBy8R+6rjGNagqllGFTUsWVl9pKDOehkEth6HAhH810gSfQbN39vSN7liE7NQTEPMWM1A", + "cdhfL5+sMVNgyVzFODunirvOSJJSdYmnzKvfJBd2BeqJJofbUKEWxmTXmBQEVVibnGFqSZNmqpJei8t5", + "yiA9acrMFWPCT4VqQquR6s1Ub91StQtSXJjisr6GvcPAd89t9gKZKxoxYt/yxdcLtgZHr+tS9g5bhgUX", + "jtYt+7u/aX/dq7M86bbHoWHtb9thE11ng189OfPVp4rq3n6ZD9ZXGSoo2CPn3tiytQd6cx00/ylh4aPX", + "ZPYpw8JoLj+smr7toxKx8Bu8XkuZrjAHXU3ydd/BU2CXqm/P+dtOOmHzuIfCbyD7J1QlHR84QI05ouxh", + "boLKUfPisZWU7J1cuZ9ddhNIWlwAIHE97jvKVdLr9wYzN6vjvT3g9wkCjR4d7j/c2xz8vTHq3wU5TmK+", + "ySrkWrlgOZ/7blEacdJ1ktijWdbBbo5vbrsfkD2tbYKtjA53W0XJ8mkn4K9pBC++o5HxtRzRkF4LlKDV", + "Y4tw4HVBBjpMbXkm0HYfVc/nKBi5AmGodfLfW6N9mJmNRAYasUbKxjpaIv89KJ1KFSrYJ5VxebpT5pX2", + "4j70gcAe+LrmUh89rM7y/r17h/e28SFkNo1j7s5dYKq2dR2JgFWW251f1wH8R9dlDvdkM1nhumyhqVdP", + "AiS1eUFlxsS11vPe0eHB9daz60ROfDBngy+FgJwAuQllItDlKRdMkZQZGlND60wGLdDAZbAiFGUpJvPN", + "vt3MWlqinqrITDctZ/exYmZaKlu+9EDuKRV8hkqSbVn9sl7Qg3v3j2393ZjNju7dHw6H18WreVY867YV", + "eza0tgJdM9SLD9uHW4Cl6TKXP3pnj1/9CIws18peWnt6ysVx5d/FP8sH+If955SLMJxNl5LNfLZWqrm2", + "vVnu8MJZfEzKqvxe7ukS1dfiQsKcAsTQCoJD1mKrbw8FsqBxXi/2cY2M/g0Z7iCugEW+dW1uXH1ZFLZQ", + "U6m6XLUrdKjADKGfmyI/vMUZ27hvWktHUZx6PR7mRuXF9cYKrGsF+jImipqrSWL/iqRAOO5QAdbaFemf", + "dajfh9eIK9RXfLL6Y/H1yo9PqgOp/O7r/1V+cpVY314zkG2jQPq3dTl0Oxfy4uiWwxw2/xe3Qteq1w49", + "MpjB8YnvwpsEe9a//mL+n7/9H3324O/7v/385s1/L5//59Nf+H+/Sc5efBAy0Wbg0E+K/vnRAD8xwrGG", + "+tmVlE7BM7NOT6D+taywe2ItDiZaYJ1eMmXHEIL9MzdM2aqPjZRlKFTAUFPCt4gUWJ7PIUzswstnNg4B", + "Xv7Di8Hvm33EDkpCuQ0pEIJ0Po1lSrnYHYuxcH0RPxGNegH8FZOIZrZ0IxdgmIXAboV1Np1juPx4n/xB", + "s+z9LtQCoYawdxYDP6NYqnBW+CaED+two7LB6645K8BibB7xWBS3dYGcaaMDhmWZDc6SZqZfy6Js1t+c", + "5vRwFMIYxSw02EgsJYUqSEHZQEZFehx5ONpd1+e26BgFDW0gPxc+Y9OUIc04QIotqc1PWcwj5Cs+u9e2", + "rCRWW0pzRrxMyXcr3JuXNuU0JjQ3CyaMC4zsk0jKS876uKV9As5ZjNfCN20UzkJmg+lqsJBZAY1S+MJt", + "HEtdyf4/AzfRwRum+Mx9KYhwASQSEDrxyLiZYZvSurA2sVe2Co/Q6JR3zW05J20r+NgEDJMr4YvMMCw1", + "ixg8BfWxkEz+LYkSjlYnvUAf3gJhMg10E0K67I2KRDc6jWI2a/67Hoh0cO8+arD+34cHnfPM7dJtorI8", + "Cei0qWd9HTi2ZZM4ACseTLwhfEvoINyAlhMTtFMYif89J76j8sQVjMR6+mzmq3Y1SxJdyXndDSYnumPg", + "gH4mNDddZ7d+Cmu5jh16sDFF+FrSAWLoGc6DvPr5nBimUo96sRPB7uApsXgyA6517srnPX5y+mx32AvC", + "o9VcWrhVG3Mh64MOIKTYc9AaOlXaaGjK+uTkKea4u2ul1MUwKekHqUhib8XyMjpGiJ26uYfawpsnT50A", + "mqzKQCUrtox7u77HrHm9HZOX/rOEFkMpspVL2vJdlpcJduvCVm3G1FrvjeR29I859c/dx5gfhVVJLS9G", + "iNfW+6u7zdGtODxsWsiufSFVXmy3f1X2/mODoX98Gf3wejK680JPsgXVIepeVL2a2Ght36uO7Dq7V+2R", + "frXvTjQPnq2/+dpMlWvIMLjnaq+H0H/vDfb3X+0fXd98d10c6zqAXQXcsoCy7o5BfRtYzgFkZm4mrSkh", + "BB67BBBvF3lzCtmVgIYGDxvWkf3DB12MEvjVrskUlYcgUuOQCi7l0fCKJACLC3jJk8QKMJrPBU3II7Jz", + "fvL8p5Off94lA/LixWlzKza9EdyfG0Ba4y2A62iDowKAaDUgD1Jk/L569TM8iRKGSVNWDr+8OdD1VtNi", + "B+BrP7jnZ6/hjQXVEx9u3Z5hTMssffaOa6PXsRA7ZS18CNC2fbU0jHWZpO3DBVVuRev+sQYnHQS33L0F", + "mG2fcrK2nJ8AgfpTpvZ+fujXG/GqPxR02tkZbglzuvVKC+E1N3BM7rXdbjdHj76V4dRgoEJsqyrheNyF", + "G8M193s8kHP+2MXxkZOzsj5b6Yzw3Tfm9OhguH//IZYY3h91YewpjTZ8+/Txk+4fHx3YW+aYTo+j+BgV", + "9pv6rBxhWxWEJleQmTv2SuK4Zy/MinZbOba2Tbeci3VU7JuBYDfFuBaYaxRnfeCSnqQbKwR1SCpuQh2m", + "uQNyTXmScM0iKWJdl5EXVBOdWfxiWymnkODHAgfYJ0XBcpRSCI0ilZemRyddO3k/zxzdY7XeTArNXLmO", + "n9hKk5SjE7T4PIY+alLkrsVjsaN8nmOR0IiFemP4AbOG+sSpQn2MGsaKQPDCWOhFboCJ7UICioC6vspZ", + "ZcmUo8dol+jcqrQ4XlyNFTBMzWOmxgKaBRCS/yjUk+P7gIzQ7xWaHFQ1GY1C1HSrzs+hQwDHHjAFhgqH", + "BY7R7ioXJBcxU0XVfmbJoRkid03H6QdCf/vXu4lX7vVSrgofzG1I4d0gwD8UdxmH2qKfY3ToDZTzezcX", + "0TtlCnr51eUIurcm14lgcCkZoPFN4bazBjkWOzOkZi7Xo8i9eW0r5tan7kKPjQSnulqRN6entbAHl6HR", + "beLIJVr2QWbX2oaDLTaSraO5jnu5glJ9F8jUTUmlIiF+dBzqqofRQxdYCq0ZtmqK8yb7GmiVwRwoLuw+", + "AdFsmGCjaLIFeikDI6u96pWeOVRWJ0r7nAqHEl+AtpROfb3Se9Nc72UR37PP9R4i6jxERJ2jIORBzJaT", + "PA+pRvDIfYe8fg2wN/gLIkLbBJna5ym9v/9w9PDR4OF0//7gKB7tD+j+4f3BwT06mh1GDw73Dw43JBV1", + "yIG9eVprUGMOxDEXUesTHz0fCmpuy11oyCYuHvuKi1he1a6/YIBs9esu+Hbb59dD6zsPIZgSlFBtrPmi", + "hZOd4iXPIugb2/k866JAWtjQef/VaP9DrT84uJY74pXKhXWrYoPShZBWBlzdrPo4b8bycUA+8WXbalU/", + "3n3RRsf3Hh3f+9BF88kb28bYJKc73Ny2iDCPnN7IDvFJwhU7kjdQ9pxMZK36tvmq1+/lovo3CgONWOri", + "cackrrYD2w+zkU3XSgvkwUlNX8FIFYucGR+DpOL1EazDUgBrgAj0BGxhpGKLs0CC6Ic7qegu0A26xZyJ", + "ziYu22QMzIzk2uXzcQGMGP2P0InDRzgmz7EtPqKpVevcIGzVoarrjcYrGy8D58t/2ipZm4d87vQrfAeU", + "LQL/wmnDMjiT7eYurHR2TH6R+E6h7QnZtP3a5qhmrTdvtCU7DmTf493gx5yoeUx+KMTLQkB1AumOZu7P", + "iWNYJczUbg3sw+04BBJWdq4CXNHv2RXt9Xt+oeDPANTF65Lq185flRRDgWSMJniWyzT93PDEgevjTLg2", + "PNIueQQ2t03scbmZLJ5Y5aktJtWmvto2pHjJS1VvTskO4qf+lTjDNvxrt4hfrd11B4+OHt1/cPDofieU", + "tHKA20XjJ4hMsD64rXJylOUTZxtpmzpUj8M1j6xVoYh9AW9XSeaZksB6YOa+w+rHHw0fVcHhYpkDHy6G", + "5JAkLRY1blgQ/7DgRS1xkL/xZMlnM/Hb79Hlwd8VT/ff3dcH0/0W0Gv7obDZ7aQaXLBmo2bTgS1uFsbv", + "QoJSuhXi7iXTOANyzgxB+hkQGqF6U+RTO5LLdW3Fg4R1dHh4+PDBvYNOdOVGVzk4EzTCBS5lN4KyJcGW", + "ZOfl+TnZqxCc7dMjwWQBmIPGOSOuMvmoLpAO90eHISppubhLqnF9L9PWJX/j1Ec3KbfomLxdqJZrpzy4", + "2oeHowdH9x7e63aMnXl4ot5t5jCunf2eL59R3fkdlCZfPT4j0Lua0ahu2/ERYtcalbnWqLD0iy3ZcI2B", + "PXxwH3Ku9rthNYaCThwKae3A1nlX4NAFiCKwG4GlWGe9/bbbIiROWQJ7yaKE8vRx5FMsGrePLc0wUbZZ", + "uQldLganga9dXB3e7WTcKkxWNkEH3yFSkVwUBYGG212yH8Wz2s617fWwnauH0nIErJ4DFbOFD2+wlJli", + "Sy5z/RE6ksbmzM4SKdW13m1TWF4yDX43D2X15vQbTRytEW1YVtehHDVugF674eSudZ5rJBIm8rbF6rQb", + "XbZ+04T7Lae2vwlQo8YNWgEPY+Bcudge/PmEJlGOJbBosZ8wK0TuQ0iALEtWNrY/SaQUJFpQgU4S5Qru", + "iTmhBLAThsFIWHgymQWjKuQVSaSFaocSPa46lB0EvAYiDNiDdqp1ES0pNaoV30stk3H1f+rUeC8Nl10N", + "YyoVqfCwntTICoq4faVmCU3kXKNSaDBrYdgsXpFRZZMRqLDVzpap1SUDAdeBITaYeehGxYdwMK2C60QO", + "TDS3K0kjJXUFDuzNqR52zXkrspi3B3TWB9uBdK1DM3CV4RWnOxdFDN2PgXyeD7khkYYxZ3BDqKQ3TqZU", + "5FgvqkLIzhA/7BwOCdboSQFzdc3BajPBIjBgwCs8V0XWfWEP8m2C96JnbTdZLhd3fKO316gq3FXbANt5", + "anBFw6vVL2gwRMbruHIboexKgL8mmNV1ICbLqiFcY6+8ghxIdjDnpcKWKgiQu12CZMIqK3xnTVt1NX3B", + "T9EVWXEzkOIZNYsTMZPrnOA6nlNviXbRqhlTHtwwZoKz2OuShQvVmbq4dtXj4py5lcPP1uDvMMQBfFbC", + "28ggxafO65sf7GIetmPYXCMGv+sadthJrsMJta9UjmuFBATEU6bWdgoC5XoSdmetd6zYPE+oQsC/jkPW", + "qzTh4rJL73qVTmXCIwIvNP3iMwmJXxN4pL/Duex2mh28MGkrKHZuB+fyAu2GNL5bTuE7mOVuIysZLTF7", + "9v09eL9T9FgwUvwHnjCHTfla8HcVQq8XUTg6GLVly7d0WsuTXwdnvS7ndiQbPPG5DuQWbpRyfCk0FrvK", + "FtgaS7rhUetwK3noZO8CvJlHp5448mHQJE8sv24Ak5Apw7wfP7W17rqwxS5TCRaEgRp5f5fTukG0a9hv", + "oMzgjighMhSbBeP7cUc3GqRti7U1qezudTAokK3CRPGla0I7bCvIWMZXtfGTl2u1CRfMLRn3c7R1CjvU", + "3fHxHwV8gftqdxyDZhXJQLwyotRos8J6zFgEa1UpjQrFxZRC3HiQcKTws0HYFansWnvcK8jGXhHFUsrF", + "WHBRGEkRTI0RwZZMVbJkpQIlaw4pGn+rqHjwO0szs3IlHNB4/o0m4ELzYxyL6iChczB+D8ljYS2LKs9M", + "rcgri53WB4SCWcvoBDMKq55ysyAzxfSiOvdQpVuQ8a6kiltLiK2IbwLToOhjJUZeMlFlZUU3QdXQdjSx", + "b61H8dky1fjU6Z+kVjmaNCtDb/5erpkKC4nFlIomnUJXKkelopzggYQBuIqg7i/L4gsUlA6YJ2X3/+W7", + "LH86Kzqv/9ZoVsE18Ujfj63ZNmiCjbJ8fXUanrStoSqYBrcJzWbdl0B2fAi1L6ukd1vKG3W6J7tl4jWT", + "Bfxo9jSL6l8/enjvwf2O9aU+yFln0bs+tmtumW5wybXs1GkXv8/Dew8fPTo8uvfo4FoeFp9X0rI/bbkl", + "1f0B8A8DhzX51z/++ea04fW5hzHYo2sNKs/ah/Q66zCgN6f/+sc//ahuPKAQo1kH6W/x27dG6STVnfSB", + "AnUXXjcn2Qb9/nHNSEALNkN2GEJxQ9CNXbdBOZgGDEg3KZhmNOJmFWDk9MqXpXBNGljbHXpvDDYk8tq+", + "HSoqcC6dT4sWEHniGvzF+oYbtPCwc5k6nU/b/NAvml/FdqT0WlRjHDqEGFiKCBu4i/lcUV0L6I6Z9Tv0", + "iwyz9Wwb22Iz6m4zFQKGgkC6tb6iUK2BhjzpXqpuf2M7K37LmlmnueJvN5zD9iN4Latv4EYOGH2jLO/a", + "USm3wwV4s7cm02oByY0VOmvVJstb9/rf7ZA9vF6qpLjBrv+9SsLkdV5s0JqlRzcGt+Rl3/0aSbRQUyUX", + "JmBAkwkbVIoXYGuic+sRhDPvYOYDGZzRpZzN6li399qx0RH2B5O9/FeoMaCZ9MHq72wWTWBti/Ez7t3T", + "4x6RChIp03Gv4bYKpk+m9N3EfaCO7TLaBFZepr83Bqn9DKaJjC5tbUQsuT8kI5IyKjTJBR7+hldtf7TZ", + "O9TvZZW98euIyt3aUrrNJFO2oEuO9VScT2VeC8Rk77jRGDCK/RyTWFq0p1phaDdDaGaTG4/LSeOlQ8XK", + "dQwdQjspfERr2RYNfDMsRy1+Z0r2HVgBcOwXL077NoABQw/twGrxjX6idgS9fq/8RKO8Qvl7OH4YnBU4", + "7iZcf7q+jtWcdPSsKqaZ0Q6/uySHBhFA0pEwTRz/tJsiV08rW7+ScoHBfi78AxbUf90SCIlZhCdSr5/F", + "OqHfgLgbeQNupUOJA4chEsZDsaH+zkvnEG4OwBobKjXdbT/VuG7rJZxoI7Ne/VRP2LuIsbgJ+Blu0jVW", + "3r0ZjJX/mTqMoKLcumuN8c7rsxveXoIXjrVttasx/UKKAZy6Ykvt4bR4cx6rpk5o1dWvQlpMQvCqoQYd", + "xi/Yu81r/Qt7ZxAfPc4TFq+vbXEkLatyl9G2Fb9xZmPbgZaKbS2qeQvFJm28+Y3KTdpX2aeoOOla3UqV", + "ybXdOWfGtz13ZNS6Q/VCLzWXluuC+Cb1GBtLSn3iLniyn+42SPBoEbaKOFDejjmaYG8B5/uMv9tAPLaB", + "VYzrsCblQfLUQCy+6E5K35GjB1j6SzfGLvh8YZJVPQDnKICl9EGlWBUzTHhDYZedL3fTv7ge7ea2s9p7", + "SDg+r0ADrZU0cSLpZBNu9pOimbfOZ3SFVpxWJ+GDw6PR6PBgdCPgbD+sayzXk/IVV7i03k9bSl3lPcKq", + "HjPLgao9FEnW69Wwbbm6cjWJNorR9BgTbzIaMZKwGYLkFQmt2z2LzU9vHrwTqFwWbUH/fqPcvnkffL1k", + "TvEpMa/8y5o+cK51DKLq8y0O0RY2E61B6gVy7g4Ho/uv9g+P790/3t+/DbDrYpHasj0e/L5/9SA5oLOj", + "5OHqwW/7iwfzg/QwqIddchF3pdWfoG1rlE15SdaxjGosjey4OWRMNcuc161GgMGccMEGusiQ2p6muIEX", + "WP/71vN/PTu/ncFG2eG8PsmqCEEN0UHK+jT4W24yG30XzdmcPN08ixtlIDUHEqa35lCgVcfBYIWK/d4H", + "IjPkouM19LrSsPNFtDErbttVFPKw40kP7nLLiofIuwHMUJn1pgt8/ZIL2E7nUnGzSDffFkWzAkYc46Z/", + "1yau4z0NyclcSMXqPxdhclUlCl7u9XvJ70f1M+N+74785RCICwJ0W12VCjqEkSVsyZLNq4BNSsVD2Uh2", + "qhguxHf7g/1HGIeQ/H703WjwqB5x0LerVV2+fd+69uuoyxpWSwD60lH7j64Vce3XcxMF/cRDBezKe9lh", + "EzsaLyvK+6vDJ9zWNrh8vLbHDSSfVgH0QyU9d7lNqkJTDBbZEDZ9xVCrG9pjlcjIlM250F3stoejwnB7", + "Dwy35LEDCLdl3GXxnWr3QCtVOuFpymJODY5KmQ0ZDAcdbXFNXeJ6tUn8WwFpbRgW1x5th0jYlnC17Zoc", + "fkA+7gdpv9003k3oHWhX8yoqYohhwz7hM7BB1/v3xaqt0oGJkRivduyB2kqSdTxAl3Kgt5P0yVwaUqbQ", + "d7S35aLdLliMHypr5hsxMyxBHHwUQJQCQIxvYl8nT0mmZJxHZf5ogoMuET9ULoZdhfytQ7tV+wYmZoOR", + "Y7t9o82g0c0+2bbfDdskEGz7Vu+Ptm/1rRhF+r08i7fzMNuoGwe7FnL7lhTEgImmvuwNSbAymbcdOPrL", + "6gqu67zWlhyBSJRn3sECNLVOSQF3C7oYQnG9T1nCDAt1QmQSl1kSXJdcdDtL3b//cNHm4kSP1PpAfmIs", + "I9RY/CP8XgrevNDAfNnR4i7ZGXm0b20dXgNbrsitVn1wD7ZKYq1bVTXhtpVQsFy+YfO2eCmXFfO3n01N", + "NFtHQPEMvyakvWwvAeDe9GFvrfbj2zDLfUoh7YVzPTRgWz2ocIGO7r9fxgKDWFcn3qOQez5EFq/oHGvl", + "bfGjtJJKqK4kOjkoMdQVt8ICm/BPglVEtQds8QICvu4zel2xQcINoXParBzRVpbSeqK3xxC1F3p5Recb", + "MXmb6bBV8/vjwf9YczuZDI/3vvvr/z14+5eg2b1hQNBMDWI2w4irS7Ya2CpMYKwY1hFpsQRETxs6d2eG", + "0RSNaYj27rhSdbz3RgX3XP1C07UpYKhapYTS/tYJ/fU/2gO9Ksv4Gi+MrWf3gyuk3EYlWSP9vbyTMjX3", + "QfU+o253OBaYpHfJVppUCrNNV7UT+40uXqmE4pMLS+5DJpYXZMqx0qUeC1DvaRSxDNQqV+uH41iURDas", + "GE2q/bgCcf68OM+sDayA9Ok1OOMXr199/+L1L08nL86e/fL4ZPLTs//uHfcEuxrYL8QDoL2je/ddkfbq", + "Su4HC4Vcv97FkJy6fAUX8zDLUbNHwDLwkUIKWrIi7F2U5BrCJnDpjUluXtliPWv55pUiPhAK2ZgkFJ7h", + "ILsBhhvGbO9VF13EtSdGrrG6vbPycEHWRRdLOOMeXilGekkiWFcEtiK82uXG1hf9/qLbXWJBYgOHHVN/", + "4bAFrKg/c43AHT4IptKY7GAKjS/Ba5/YMNRrgNY+LjoMhmB+5EpLo0cfoxrq643lT5cyGYCe11IyImhW", + "t2sRTCHArmxqRq/N+zKfBpQZZ+Oe8zkNOFxCjpWPUrXUD2hr6tja/reWbwsndDxt1tOwx9IuVaP+Q8Na", + "os2gPd8jBfG+BYEYIZZtEi+vBBnWut9Lhdlz1YVDwCGxRFT3TWnb5SnzMJEDfGl7NvJGBbMys8pI2vfm", + "VOahbdm0QJCPDzxSscpG4AssvuGSuQSlDnA0SG8kY6oM3nUvo3iOfndNdgoTmF+CIu163S+wuQ7FKX1X", + "fAFagDbRdMTiPMo6WPvPv8eaAS/dLgH7dF3gMBpabhghv05Fm9bEU9X6ZlSpan3etn3w4DletYH7tZ2t", + "BnGW36iRZoge/0a5+UEq1IvbwV9uHWgfL/+YKQTDa8Lod8Kg5ymLJzI3m88/NAPp0V75RX3Ysr6wtwFQ", + "JOKoltfcxgs8PEk5hvWVhuVgUa64WYHRyGkmU8wH9UV9cSHxQ/hz+WEspPr+PVrPZ4HMmedMMMUjLFML", + "5zGlAjUmgO0rqxXawpVrwLUoAr14cuLsLh77GPVobpD0fADq47MTcMYzZW0PvdHwcDjCw5wxQTPeO+4d", + "DveHox5qVQuc4t4Uyuy7RHKnURca/EnsJKHvfSN4U9GUGXzj1wAkAgZguuaogtB5RYnMKFdOi8wSGRcE", + "w+FtrLvgL9Rjeyv37bJ3Nh5jvjWmAbHshdvctygo49nBaR6MRg7h3bjrF5OYbObE3t9dGG353U5SnVui", + "QBmCNTXPy5bF0r/v945G+9ca06ah4NkNffi1oC6bmaF2fm80uv2Pngibn+iyzV1cWPXEISFVz9qvb2HP", + "dJ6mVK38glVXK5O6TTBmYNdyba0eZzSJgFVgsSQAtGH2OZhEqQ3hVrnAGtP+RaDQ+imwfftNLtCavpfx", + "6qMtYe0b3kbxvs7O4Li8X6Pnj0c7BRmvb6R75KHGLdXeAQF9T4sC6Z/spByNHt3+R8H2m/AIslU9AbvA", + "bK4x9ilB4HQPwiQV+S2XhpIir+ELOtJOZp0W5NYvr6K9P3j83h7vhIX8AWdMpVTYLBHbZsuhXzvO1jdT", + "HueNt5on/JOnPXdTeTQie1GhIFc/otVrqykMrl9HR73jtm/a6cWfkPCP7uCEu8kWNXI/5ZHDyqTgV/6S", + "jpPzNU5LISQoyz1n5nOh+dFdXlmumsKf8BR9KQT8nBUSXrlba5fCXqZyYRXgoAT4sszcdO99Uxf+XpVP", + "KuFC6NeArrGuh7EOZhqvhsSvqVX6weirLbC+Wtpws/r5OoPhfS4n7OAuThjOuPAUfb2mvl5Tm065pRY/", + "BTyYlVPewQZxLQvEn8/+cG3rw1fbQ3fbQyfLg2BXtjXgQQ6JC82NZMyIXkCVUnA2W+AnH4RjqBrOfydU", + "RQu+ZIjuh9Xq8sTwjCoMsUkJeAmsD73VMLHRLFF0twfdDXxAZrnATUAPzSYYAjNpA+IsQzG5EGCspdpH", + "zZS4imt11e3ZDxrYiw7Lq5FcLaRmBbChMJXbHPO8tdWOsdvhWLxyiLewgBhV7nmNZgni9m6w/0hB6Fi4", + "F771LMRHxGmalpyLKgRP5Bai027L2lRxpBMdyRDo0CsmqDADnbGIzyD6EKd1yVYusDXYYacCVDBgP06o", + "C+MDtQ52w8B1GLoURil+WjwjjpLq/hshjSvNXTq5PJYSVVD7IFihZJ7IKU0mdn0uWcAn+BxbuEUpHS6l", + "N0nImNla+tnKLKSwf+fTXJjc/j1V8kozBT6nscCMFLfWEK1QCIjkCivapZlUxsY54Df37BD3/rhkq/fD", + "sXgcp1zYN7R9hSZagpsC3jPSg4dY7tVCD/Y0hf3gT3JtZFqFgPV0Z4cpc5PlxqXWaGb6IfjTsTCS/OFB", + "Lt/v/VF+8T06ixmNgU4qTeyUULZuG7WeUJj9BJsG3O0MF2Dcg4vUhnnMFRUGx1SidJJ5dUt3ijIRWDq2", + "ucIRFSSTmS2xgUS1oEBytT4QtIImCTF4lPy7ILjjTrbMx2EQptNWAEKLGNc4RlyQ0+8rh2l09DB8njSL", + "FAtFlPzn+YtfCN7KsAe2WRmuhWvEBQgMEEWDtUMcT3sGMEnWUYVVFcc9Ho97hTs33sWx5tqFywwG6FP8", + "Dob2nf1Mn8ffDYfQlXVXHpNf/7C9HMNZylILiDruve+TyoM5N4t8Wjx7G17QNhy38xojIDv2mttFTkI5", + "Qu5Ubnx7RYLpTrpbAAAEScmBqoErUy6oWm3KqAwsvVtBObORjJXF+GOMkYvj3vHYxy6Oe/1xj4kl/uYC", + "HMe99+EVcF7L9hJ+eJ8Vzs2CiO6PRrvbIcHd+gZ8lh0cAx9ZB2zVior6o7CDDo/2z+Uf+LfWP78vNDYB", + "Hy8xmqzi743vX6ADoiKxVzXRgAuiIXZTEbHEi93bDT137zyAzYpYktw1gX4q8izcY0XJgi+KHHGzymO0", + "0Xz/iSludFeXSs1s/2no94uznwes5852zpY+1DlcsAXBeJwqTWxjMFCc45gG50wY8gx/Hbr/et0PwSUv", + "Ejm/OLaqO9QeIAkXLh+gEqgM4oFbS3zJ4vEU79l/FtXydqwk8a9//BMHxcX8X//4pwO5/9c//onHfc/i", + "zGGx7YsFo8pMGTUXxwSy3gY0gTByNxksh8uWTK3I4cjZ/PERqdT8d1KaHouxeMlMrkQlb8IWrtOuQ+cq", + "gPlwkTPt8IygIZ+5qjo2tjFgt/Fn2S7lnZ7ofgAXEmdQmQDcip4GEFSP24rjThPthU2mds41o2kzTHMt", + "WG87fzHsnbHUO7ADvCaDwSUOnTt84CZNds7Pn+0OCWpbliqwchLqDmU3To0YfuVJ23mS5Sh1hoKrbHlT", + "RDM65Qn3JseWsi/2CKY0WgA+UxFfXICu+y6O/UiBx0CopQuE7GPTsXhxvocmVsMikyvWd5xAOajVsi6c", + "dHku+AXkX9xgdNjAtR2LGaPQBzl5aplABY28SIwsOhaIaIIxrtzUStD1x8JC6joIZzh4qYxZgi/h9+fU", + "sCu66pOi6K8vE5NQAwqx7kPjsbCphm4NBojZQirDHCI/s0Ma+Ehel7Ol2AyMozgRV/8cv70DdgUX4dwv", + "80r952y2qR0WLFpKoxfnML85aoLS2gOxpxfnfjd2+0RLEiUcqSGiYizmGAjkUYylqO1qkVC2oCoeRBIu", + "gSqqFUQgJyyet/HYJ5W+ercoydS+EzhOPzbJ9UsTLhbrE4BDbJH6Nnvunro23Vx3rsc/k+/OLtB1nHfW", + "gsssv7Gr+9WR18GRF14379QLedaeeijK24v4tZ/4RAG/nvbW19w+qSzZp7DokR2P8YNeEanI2ZMTQuNY", + "Ma13/73tfTBTS6Wl/Af3I7DiTxF64sYild0Lb2+pE8iXwg5eulET6ufVLDRcvd/2alWIWm+6oiBReeXd", + "/u3R+Oh1rpFS6C1p7etNsjXYlusIUher1DJA0ShhhfhSnNMqFW2zKtsw3uLK2Sgu2VYQI+EO5N3Zl92n", + "c9G8G+6AKT5tMMRPyAjrqdbV8uFfEjW/LnbRzWuT+fnzIs3R3UlBd22KDpH5l6Quxo1lAy5ogU5aL9Dn", + "zFh4k9vU090XAhMHO7c71XagKzvrYlr2VWJxWnBCaInZrPue2CbdVF/b359J88XluY7E4pb8q4jSQdkt", + "12qTgnviamHfnn5bQ3K747AVR2CBRYYHzjaNsEhQXpLqlYh2v0aufHSKxsWuKLHKz5vFhSUbn5d61l3J", + "dY9FtfA614VeywWZJYASa50AMZ9hrJ6pFjLHUR7cwSiLguGKGuZCFL/EvN+zPLHejUiKJVMVpMbqlbr3", + "BwatbleVPPPaeLu+fvnzgIlIxoXzpF0mdU8+ssJk6b+Wy3v3p+4LzGflXjxoExg/YP9tMDkpoDv/18EP", + "Drzzfx38QJOMC/a/Dh8n1DBtdm+NWEZ3ddPdtQLzBRMf6C+8vmhrrGnP0Hk1D7VOmR6+9jqUWYOd9cbu", + "IpHmo1Hfxxfymli9nSS8O6N70KTuGrHkxJclsFsqlUevakAS/3sLfec1gr5ze13t81zjCFCw+wKjlKXy", + "qKvUwVZzT9yON4m5YlpvM0YUrTraI3z7P5VJwk76WkaJYl2/2iW62CWqy7XRNGEb3q5xwn7jE3nfC2IL", + "rTY++gq3dQcOHUeRFbitmoe7BNxaSG3w0ZeHveCuEl5QXPXa6OiZLA/kxuvDk+7J0z4uJJb9P3lapjbf", + "kZ/Sj+POFW/33bsXeh6nUz7PZa6rWdMpNdGCaYcokLA6A/7STALl9dxqFPiMqXR0l1fHnev8X+n+lqwR", + "zQ21zNsGG2yT+X2rrjK/aw8yv0VbdqgLrjZS39fN221JAvF4y13JuAZL/b7fZVwhXQRqTWhSqgsENYjj", + "sfjfoH/8ahhN337n07vz0ejgPv7OxPLtdz7DW5x6UmHCKO7KnD7+5SlGeMwxiB8roZZgEs1xkDTXxpKe", + "L3nyb6cglUEu3TUkT4VfNaROGlJluTZrSLbhLatI9bJJd64jeXoLLbh99ifVkv7krtuaBqfz2YxHnAks", + "PoXGP70Wq2w1ua9e2xuCJQgXK1EJdKxJIp3VyIJrbZHQy8L/HzOSsN9ag0ISagxLMwMgRhGDWrg2Q08v", + "chPLK+FLUuAEfXUzXs4ndL37ria+k3CCbLhEe1dNtyjLeNeqrvvwF5qhKjNXYdwpl6Vo065dflrivV2d", + "ssNVe/da5ZdMYlZ9W1+6DDSEQIk16+ZJc5vOW7xZojMOyatXP/vUXVBPlC/YZ6Sv0ucrOY9FtUrfkDwr", + "yx/aBr4HUB9Y7FL94X1f9y5mNE64YJjrwHQoy7ZeW/OTHouPLwGHC4fetYu7y7G0G/oJJeBPxgruRNYs", + "fPrGOWSLo1mpKVqcFi9v4qn5oviVY0ABxhOS9fZobuTAgQHsLaRFiAyD5J4lNEKMXGhm4Rsd/orFa612", + "haAqSiYJUxaWM8uNF7cAIdcNjgsji8LTTjK7gO4nuTA8uejbUENom2gCRc8tosxY1D7mZD7ESED8Dxyh", + "YpkdcaOKLgyag6kRWiGcQfWThCZXdKXHwqEq2Nex9LpikUWwTZIh+VEioI2tJl1hvLaU6zd6LC54nLCJ", + "w6O5IFwTvZDKMMFiksol0/XvMqoSzhRO4gmFldMkpSsEhrQYuXZ9ZMYs+GIN9UbCv6mIORYFhS8XUz4e", + "C0oORiOSMiq0w7DQdIYXjuuD4CBqA/qWUHI0euTeauwbgpf75d+B06QUW8qITqH+K1CxLba9ixuYuiK9", + "tuo7bN+MK233q7BvuuqLtY3l2tecjfskFyVKB9r6c1GAasB2mVwJnKfzAjKuimvQARNNWURhPYWsfwch", + "YWUU5Sp0QcJWV6pF/zsKjpXpneNShTEwEjQZRBBqLxHHcIFnWuJR2v22hapKovpzXDTBQyIVoaRC16VF", + "g0U5ssYdhFC9KEufCl/K/GL3W3924Pg6RuCPv5h/SfcTEpGczWoHcPvVZA/wptyzdRL+s57TJ47p1Vhc", + "zOlcSG145JmhR6iuKs1fFcJOCuHmlQ1S80yqy/aA4x+kuuyqgfnAyC9LEavO8DN0RMDwEAT/0/sj0Bpu", + "lRUgmjtX0pr0VZxSFLq40UVIMEmkmDNVtcrfudugqtXtWEBLuEyVdXYX8GOghEzcj7ZsdiWsG10Mkev1", + "U/Mi+PodOKN+kYbwNEtYyrCs9sASG2x2CVU3XVnVowRsux6vhFNVxVWwuqC28Qd9Lw4hXfkN20HpfX27", + "gkwV8Dy3AqIWH/fonwFE1LF4rW2pggvrerogBQ8GgdaWHyFXCx4toB/8Dfu34Kk0yy4KYPjdY/IcD3Jl", + "2+zHd2zREaA1LRNmQU+XaXpxvF44+s3pKb6EbRa2RPTFMfHFoov7Q0OrKtopzCKh2pBfHIbrTqGM445e", + "gA2zMr9dh4NaAvePRQgTFSBFbYd8Ri4q8KgXLdh9nt/+LOf6s3EVleVW7FyMJE51RNpkIu61BXnwJOz4", + "2R+NQlUAOqK02mHcMkjr2mB+lvOi1EuNlGmWdSVfN0yk4mWabqBhslNyEKJNLHPzV21iphS+7Ki7jbjJ", + "Do3sPwy9BEJ1AJ/+YO+ORctS2RmGlwqYaq/fYyJPe8e/un8t07TX77nxwHtXJksrtRuuIeRvQb1tdvi+", + "H9qhCrTtV/H8OqC1daZfQa1t3CBOrW6XzF/aBn96r6G33X1CMkQ5oWHM/ZxE0cp464YfIQv0XRzZo7sY", + "GUL0kiiRmtUcPV8OwJ8zeDVkx3aDkV/jAQwvzn1FtC6RLOfu1XP/5megg2+LGfFjJn66dx48sj6CLxms", + "QK/NBixkDVS4bVElnz0hfbwtWZtqFwr5SpvXtzZ2IkzQF9ZZhHshthUyaW5kSg2PsDpbtJBSV8i+gHC3", + "dRSdEbmgTDSxWG3XZRJcAKleOHP0hVMnjp3pjNDqI/eNIb7u8g/Cb/hH5Rs/VKwDBcfvexUAK5hoqGCl", + "OJuRjOaagVSXg0lnFSXMl+Nj4BiOaGZyxbDSKCMpFzzN0yo2P+zYkiKO0MV+etEn09yQBMxEqnjog24i", + "maZMAPbDTKqxWDC65EzBoiTUMBGtBpphhfIlI1AeIJE0RlMDuOhhSFjhVDGgQCx0kDJDY2ooCjoXcOIn", + "NpnpoihabtV7wd6V1ADOz1x8a6uuQLcXfqAXhGFZAa4XRXHbiMZMREG4/fPPm419fJv0OTPNiX6iCKEb", + "8dJPGTJUtb364Xwe0URfLDREBza/QejV7SpsPQvEk9G/55G2c/Vz/ESOpmKJN53iz8PDVBDdZ+Nl+vRu", + "JKlInNvPVU4lkvmf1TdUMJRq0BVmmNptvKmDqKjiWSzztXje3h/+z5Mb2PI+E07Yb1Xs2+rFlZP+HFiu", + "W9Ub8dxPZMR0tqSqTe7TsWA3qE8nPklV4XJfirG1htBW8O0qdzKKovYlxVe23WTbLvDhpmzb22bXXPsV", + "Rs7FAGNFwxzcmXFbWbUzHfybZqU0ZldhmZ+cRZaeizuHW/SsMaMrMJX8GYKFN/iPIqmUhcGghn1ZENEV", + "q2E1TQBtc2Uhyr7P2nxzerrbxiWU2cgjlPmCOUQlNQdeS+N1A+6LJVOKxx4G88npUxe2yzXYRIfkRcoN", + "MZJcMpaVmS2YXQjojgUgSGPYTeSPfo8Jo1aZ5MJsHUXZ9HYGU/4gHS7KZyhKupoDX93hnd3haNn/8tgZ", + "chlCiwls1kwNNVtrIXMxkyq1chmdyhx6Bx4EywT7aRELZjxheqUNS210ImAZwHHD+jWuRrl7z+5yH2Nz", + "4eTYtLmMqZRrzaXQY+FyRjKm4NvwOvRfCbQKOgQMLfjrmWWSn0cQHwzGxq1R07ZqCN2EtY97x709mmV7", + "MTW0JVDMDe8DhvQDRuURvUqnkAEIYX2Xmuwk/NKqJ2SpSQJ/7G4M65vgex+7AvvNTxas9ImYyWB9S0uz", + "BTH/qbK7HFvzjskvjq09Z9XD4vnPTLayte013hWjyQBrpnsAH5IbnvDfLauDTrg2PLKpR7RYuzenBVMd", + "jsUpM4pH1scaQdAyIhugdrmXKRntAfjbYZRxRIE7ZDg4ZHjtj1P84pOz19guZalUq/5YUOE6fvX4zHp3", + "ZzRizYG64u7kZO/FlkDnc1ymf+MIQTvBjSgGwQ3/6hK8PtZI6xnSLUdUZptUJZn96UNYnQT31a7wZdoV", + "gIbL2ewUAF8elStsQ1jKJE/hH/aPk234ZoZGizfY9LORdu1wtn7GT/CLOJRuTjGz9Xc/idPDLtiXGrMK", + "C+engEJMLRoweAs8Nn9G6v745vvqOn6G7k63or629Wdztu765nNj8Egb1fX4Uo65pTQ/EyM3W5+uKG+3", + "Pn2fyOhSO0iWqtmQKEYRZx1+LHGxnYsQxQTMECUOysgCZjHdH4uGAdIi/2hCiWEq5YImezhn2wkifHsr", + "Fl1KjonaEeapDDSYlzOZJAjjjTB4MBtoMfMdVDy62lX/q7apOiONBKwfmTKPer4bUt3+Rrn5Qao6hPnn", + "whdfVdYfoQEpR3v7FtT29i9+EIr7KX2HodJx7hzKfkQ7z2X5ozUF9Qnuzbh3ONLjXp+MewfpuAc78ISi", + "CZUaco+kXOQG4BCfWvsWpuLeHxHNIili7cHXvQXvcKRbZu7IsiXL8z6+d5dij6MqXMqX7iMh9gDtCLyP", + "STtkp3rg3JmM+3joYiJzY8397ly5VjEzaB7ZvXNfbeWMfNXtu3Dyv7njW+NRuMvALitbbzl7lusFaze5", + "/WwLGuVmCutQFEDWC/J3OdV9iPS31nCAWVvje/D2mf3AXRQcgE9dp9iAm/vXSgMdKg2UaxUGbbQBlnAl", + "e+qwyI3sXSaVgd9dsrujIdQkEEECyxS+eHIyFhGwIgsxqFgqkTs5XHR7Cz/+2zl59uRlnzzFYrzkx3y6", + "OyQvILsFe9XORzMWVhKzzCuigMZtZxCHrmc7dqSe2wwWhw98our29mQEPCt+r3yQeL+3YDRGieSP3s/S", + "fiyAPvzyZzhAsH/uzWLbexuFD7Dhq9Xg8cwwtd7tqcuTEgV2hrukPRSdE9zww/hB7RHYym9a2cBCZBwe", + "9AKIGe+/Fn+4/SLOd+Mlw++6snuQQfel1m/Fg1gwxxALrF7XRfmEtixhx8s2Khj4yZOnH0WpuFXP8Cbe", + "VcOY/3c9XTjTL9bRlNX2CYi4KLuy1dPrk4MXFhbZvkYimtGIm1Wf0CRxd5S7CYqIlEEh/k4Vo5dgtx8C", + "dJX7sk/oBV9t3ztqScz1pe3B+WKHBELSdD4tBkfwoFmvMa45pMUaSSKaRHlCDSNsNmMR5uJiHRfd4sst", + "htK7xbNTfiRYdKYS1Z5/cbXuwjSBu1eSRZPi9uxW7ykWJZSn7SDkTlDDgEMMNZhCp1JA7EJiKY5GSmpN", + "XFcDlvA5nyYuQAjKL4CZi6ZsLLKECsEUybWNioehDzLFtM4V8x3A3eUoqk9KgMFMSeNCExIplR4LT+Fv", + "Tok2LNtAZi9tz6c451uSbW3n7kufyEjdGEO7KcQ1IbAhllLsggMdgWnoE4Si2wF9ainxSzn4rxSfQ7xp", + "piS1TNaG49lj7ZfTHvpaxnJr3cvzolW3updFr5WsxErG3kaAuEmJuR33rhf1F/j4JW/FEHSPrpdF/BO8", + "1PHb9WzV8CDcow+cZaiE579jtczzSpJgVwNWSeFfmjmpMvLaUa0l2m6H1eqcWXubma6d8bM+GWzWl4yW", + "RWvps20K7+dHCKO7RXm462JrXzZt1dCuarppS8r/dlT9z4ICbwdO/xOjnNwATv+zyrsH2vmE+CfBg/qp", + "8uhrvmepviLi32r6vIXFRzi2tvR5y/Vc8OpGRemNa9NNTXI9/pkkeLtA15Hf/bJ/1fo7qAyVxdrmggaC", + "Z2lmVu4l76ssg840/50NWxzBRdzq7bmCbxDS+fHIw9Npa0Dnn7NG/ieJGXUlBLmGML/14vNfGMZg9czV", + "LpY9uHUGVEULvmTtRvf6CXZLlCk2yGSGzpXYLphbD3+XGaqG89+J695hrrp/YQ1KhOpnMYm5YpGBaBFh", + "JHIE+41vNFFSGvdcqlV7lIg9Ij8omT52s9lyH7oz5YxhZZxhuhrE1NDB0nObDSa0D4ju9PGUwPAIF+T5", + "92SHvTPKVrogM9B8IMPbLyl7FzEWa6TJ3eqA90fhIULHk/m0yyg31Cx5gX/QhES5NjL1e3/ylOxgDbQ5", + "E7AXIOrPUJLNlFzymMW1MULwvl3V/ZYFva7dFYSKooCdVy7s4D6JDNPlQpr/zrM6WyhCYqZcUBzc1qog", + "9TNlk/jhe5QLH4Dj9siP4usV5jS/Ha/sACViPU63iEZKC/G8+/Wa+5KvuWoylL/TaredD8/ZbLzulh/V", + "MW3pNgo/FLlzd2u2fvP5pPRw/UVm8zjT+bJQSNvM5p8XCY7u7n64a3P5my84BRSM5cvGstkO1DJMMD9j", + "THfMliyRWYp10bFtr9/LVdI77i2MyY739jD2eyG1OT569OCw9/7t+/9/AJc0ExYE+gEA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/otel/README.md b/lib/otel/README.md index ed4613b68..1481ac934 100644 --- a/lib/otel/README.md +++ b/lib/otel/README.md @@ -99,7 +99,8 @@ This keeps pull and push views aligned because both are sourced from the same OT | `hypeman_disk_utilization_bytes` | gauge | component | Actual filesystem bytes allocated by disk component | | `hypeman_resources_image_storage_bytes` | gauge | kind | Current and maximum image storage bytes | | `hypeman_resources_gpu_slots` | gauge | kind | Total and used GPU slots | -| `hypeman_resources_gpu_profile_slots` | gauge | profile, kind | Available GPU slots by profile | +| `hypeman_resources_gpu_profile_slots` | gauge | profile, kind | How many more vGPUs of each profile fit if only that profile is added | +| `hypeman_resources_gpu_nvml_fallbacks_total` | counter | reason | vGPU capacity queries answered from the sysfs creatable lists because NVML failed, timed out, or was stuck | ### Volumes | Metric | Type | Description | diff --git a/lib/resources/gpu.go b/lib/resources/gpu.go index 6dfd537ae..dc64ed753 100644 --- a/lib/resources/gpu.go +++ b/lib/resources/gpu.go @@ -27,8 +27,10 @@ type GPUResourceStatus struct { // GetGPUStatus returns the current GPU resource status and any error that // prevents determining allocatable vGPU capacity. The status is still // returned alongside such an error, with PlacementDisabledReason set. It -// returns nil if no GPU is available or the mode is "none". -func GetGPUStatus(ctx context.Context) (*GPUResourceStatus, error) { +// returns nil if no GPU is available or the mode is "none". Profiles are +// listed only if withProfiles is set: admission uses only the slot counts, +// and on a vendor VFIO host the listing queries NVML. +func GetGPUStatus(ctx context.Context, withProfiles bool) (*GPUResourceStatus, error) { framework, vfs, err := devices.DiscoverVGPU() if err != nil { // Only report passthrough once vGPU discovery confirms no vGPU @@ -39,13 +41,13 @@ func GetGPUStatus(ctx context.Context) (*GPUResourceStatus, error) { return nil, nil } if framework != devices.VGPUFrameworkNone { - return getVGPUStatus(ctx, framework, vfs) + return getVGPUStatus(ctx, framework, vfs, withProfiles) } return getPassthroughStatus(), nil } // getVGPUStatus returns GPU status for vGPU mode (SR-IOV). -func getVGPUStatus(ctx context.Context, framework devices.VGPUFramework, vfs []devices.VirtualFunction) (*GPUResourceStatus, error) { +func getVGPUStatus(ctx context.Context, framework devices.VGPUFramework, vfs []devices.VirtualFunction, withProfiles bool) (*GPUResourceStatus, error) { usedSlots := 0 // Count used VFs (those with a vGPU assigned) for _, vf := range vfs { @@ -68,6 +70,9 @@ func getVGPUStatus(ctx context.Context, framework devices.VGPUFramework, vfs []d } status.AllocatableSlots = availability.AllocatableSlots status.QuarantinedSlots = availability.QuarantinedSlots + if !withProfiles { + return status, nil + } // Get available profiles (reuse VFs to avoid redundant discovery) profiles, err := devices.ListGPUProfilesWithVFs(framework, vfs, availability.Quarantined) diff --git a/lib/resources/gpu_test.go b/lib/resources/gpu_test.go index e15def07f..b579f5c6f 100644 --- a/lib/resources/gpu_test.go +++ b/lib/resources/gpu_test.go @@ -35,7 +35,7 @@ func initVFHealthForTest(t *testing.T, state []byte) { func TestGetVGPUStatusFailsClosedWhenVFHealthIsUnavailable(t *testing.T) { initVFHealthForTest(t, []byte("not json")) - status, err := getVGPUStatus(context.Background(), devices.VGPUFrameworkVendorVFIO, []devices.VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + status, err := getVGPUStatus(context.Background(), devices.VGPUFrameworkVendorVFIO, []devices.VirtualFunction{{PCIAddress: "0000:82:00.4"}}, false) assert.Zero(t, status.AllocatableSlots) assert.Zero(t, status.QuarantinedSlots) require.ErrorContains(t, err, "VF health state unavailable") @@ -53,12 +53,30 @@ func TestGetVGPUStatusReportsQuarantinedSlots(t *testing.T) { {PCIAddress: "0000:82:00.4"}, {PCIAddress: "0000:82:00.5", Allocated: true}, {PCIAddress: "0000:82:00.6"}, - }) + }, false) require.NoError(t, err) assert.Equal(t, 3, status.TotalSlots) assert.Equal(t, 1, status.UsedSlots) assert.Equal(t, 1, status.AllocatableSlots) assert.Equal(t, 1, status.QuarantinedSlots) + assert.Nil(t, status.Profiles) +} + +func TestOnlyFullStatusListsGPUProfiles(t *testing.T) { + var withProfiles []bool + setGPUStatusProvider(func(_ context.Context, profiles bool) (*GPUResourceStatus, error) { + withProfiles = append(withProfiles, profiles) + return &GPUResourceStatus{Mode: string(devices.GPUModeVGPU), TotalSlots: 1, AllocatableSlots: 1}, nil + }) + t.Cleanup(func() { setGPUStatusProvider(nil) }) + ctx := context.Background() + + mgr := NewManager(&config.Config{}, paths.New(t.TempDir())) + require.NoError(t, mgr.ValidateAllocation(ctx, 0, 0, 0, 0, 0, 0, true)) + statusMgr, _, _ := monitoringTestManager(t) + _, err := statusMgr.GetFullStatus(ctx) + require.NoError(t, err) + assert.Equal(t, []bool{false, true}, withProfiles) } func TestReserveAllocationUsesAllocatableGPUSlots(t *testing.T) { @@ -68,7 +86,7 @@ func TestReserveAllocationUsesAllocatableGPUSlots(t *testing.T) { UsedSlots: 1, AllocatableSlots: 0, } - setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { return status, nil }) + setGPUStatusProvider(func(context.Context, bool) (*GPUResourceStatus, error) { return status, nil }) t.Cleanup(func() { setGPUStatusProvider(nil) }) mgr := NewManager(&config.Config{}, paths.New(t.TempDir())) @@ -79,7 +97,7 @@ func TestReserveAllocationUsesAllocatableGPUSlots(t *testing.T) { disabled := *status disabled.PlacementDisabledReason = "VF health state unavailable: read failed" - setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { + setGPUStatusProvider(func(context.Context, bool) (*GPUResourceStatus, error) { return &disabled, errors.New(disabled.PlacementDisabledReason) }) err = mgr.ValidateAllocation(ctx, 0, 0, 0, 0, 0, 0, true) @@ -89,7 +107,7 @@ func TestReserveAllocationUsesAllocatableGPUSlots(t *testing.T) { require.NoError(t, err) require.NotNil(t, full.GPU) assert.Equal(t, "VF health state unavailable: read failed", full.GPU.PlacementDisabledReason) - setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { return status, nil }) + setGPUStatusProvider(func(context.Context, bool) (*GPUResourceStatus, error) { return status, nil }) full, err = statusMgr.GetFullStatus(ctx) require.NoError(t, err) assert.Empty(t, full.GPU.PlacementDisabledReason) diff --git a/lib/resources/monitoring.go b/lib/resources/monitoring.go index 59851658d..c06edcbad 100644 --- a/lib/resources/monitoring.go +++ b/lib/resources/monitoring.go @@ -7,6 +7,7 @@ import ( "sync" "time" + "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/diskutilization" "github.com/kernel/hypeman/lib/logger" "go.opentelemetry.io/otel/attribute" @@ -199,7 +200,15 @@ func newMonitoringMetrics(meter metric.Meter, mgr *Manager) error { gpuProfileSlots, err := meter.Int64ObservableGauge( "hypeman_resources_gpu_profile_slots", - metric.WithDescription("Virtual functions able to create each vGPU profile (best-effort snapshot)"), + metric.WithDescription("How many more vGPUs of each profile fit if only that profile is added"), + ) + if err != nil { + return err + } + + gpuNVMLFallbacks, err := meter.Int64ObservableCounter( + "hypeman_resources_gpu_nvml_fallbacks_total", + metric.WithDescription("vGPU capacity queries answered from the sysfs creatable lists because NVML failed (error), timed out (timeout), or an earlier timed-out query had not returned (stuck)"), ) if err != nil { return err @@ -265,10 +274,13 @@ func newMonitoringMetrics(meter metric.Meter, mgr *Manager) error { ), ) } + for reason, count := range devices.NVMLFallbacks() { + o.ObserveInt64(gpuNVMLFallbacks, count, metric.WithAttributes(attribute.String("reason", reason))) + } } return nil - }, capacity, effectiveLimit, allocated, oversubRatio, diskBreakdown, diskUtilization, imageStorage, gpuSlots, gpuProfileSlots, gpuPlacementDisabled); err != nil { + }, capacity, effectiveLimit, allocated, oversubRatio, diskBreakdown, diskUtilization, imageStorage, gpuSlots, gpuProfileSlots, gpuPlacementDisabled, gpuNVMLFallbacks); err != nil { return err } diff --git a/lib/resources/monitoring_test.go b/lib/resources/monitoring_test.go index 357e1c79c..76aff19ea 100644 --- a/lib/resources/monitoring_test.go +++ b/lib/resources/monitoring_test.go @@ -199,7 +199,7 @@ func TestStartMonitoringPublishesGPUMetrics(t *testing.T) { mgr, _, _ := monitoringTestManager(t) originalProvider := currentGPUStatusProvider() - setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { + setGPUStatusProvider(func(context.Context, bool) (*GPUResourceStatus, error) { return &GPUResourceStatus{ Mode: "vgpu", TotalSlots: 8, @@ -233,13 +233,16 @@ func TestStartMonitoringPublishesGPUMetrics(t *testing.T) { require.Equal(t, int64(5), int64GaugeValue(t, rm, "hypeman_resources_gpu_profile_slots", map[string]string{"profile": "L40S-1Q", "kind": "available"})) require.Equal(t, int64(2), int64GaugeValue(t, rm, "hypeman_resources_gpu_profile_slots", map[string]string{"profile": "L40S-2Q", "kind": "available"})) require.Equal(t, int64(0), int64GaugeValue(t, rm, "hypeman_resources_gpu_placement_disabled", nil)) + for reason, count := range devices.NVMLFallbacks() { + require.Equal(t, count, int64SumValue(t, rm, "hypeman_resources_gpu_nvml_fallbacks_total", map[string]string{"reason": reason})) + } } func TestStartMonitoringPublishesGPUPlacementDisabled(t *testing.T) { mgr, _, _ := monitoringTestManager(t) originalProvider := currentGPUStatusProvider() - setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { + setGPUStatusProvider(func(context.Context, bool) (*GPUResourceStatus, error) { return &GPUResourceStatus{ Mode: "vgpu", TotalSlots: 8, @@ -375,6 +378,21 @@ func int64GaugeValue(t *testing.T, rm metricdata.ResourceMetrics, name string, w return 0 } +func int64SumValue(t *testing.T, rm metricdata.ResourceMetrics, name string, wantAttrs map[string]string) int64 { + t.Helper() + + metric := findMonitoringMetric(t, rm, name) + sum, ok := metric.Data.(metricdata.Sum[int64]) + require.True(t, ok, "expected int64 sum metric data for %s", name) + for _, point := range sum.DataPoints { + if metricAttrsMatch(point.Attributes, wantAttrs) { + return point.Value + } + } + t.Fatalf("metric %s with attrs %v not found", name, wantAttrs) + return 0 +} + func float64GaugeValue(t *testing.T, rm metricdata.ResourceMetrics, name string, wantAttrs map[string]string) float64 { t.Helper() diff --git a/lib/resources/resource.go b/lib/resources/resource.go index f8f624e93..95d089c31 100644 --- a/lib/resources/resource.go +++ b/lib/resources/resource.go @@ -37,13 +37,13 @@ var ( gpuStatusProvider = GetGPUStatus ) -func currentGPUStatusProvider() func(context.Context) (*GPUResourceStatus, error) { +func currentGPUStatusProvider() func(context.Context, bool) (*GPUResourceStatus, error) { gpuStatusProviderMu.RLock() defer gpuStatusProviderMu.RUnlock() return gpuStatusProvider } -func setGPUStatusProvider(fn func(context.Context) (*GPUResourceStatus, error)) { +func setGPUStatusProvider(fn func(context.Context, bool) (*GPUResourceStatus, error)) { if fn == nil { fn = GetGPUStatus } @@ -428,7 +428,7 @@ func (m *Manager) GetFullStatus(ctx context.Context) (*FullResourceStatus, error // A GPU status error only means vGPU placement is disabled. The status // carries the reason, so it is reported rather than failing the read. - gpuStatus, _ := currentGPUStatusProvider()(ctx) + gpuStatus, _ := currentGPUStatusProvider()(ctx, true) return &FullResourceStatus{ CPU: *cpuStatus, @@ -633,7 +633,7 @@ func (m *Manager) gpuAdmissionFor(ctx context.Context, req pendingAllocation) gp if req.GPUSlots == 0 { return gpuAdmission{} } - status, err := currentGPUStatusProvider()(ctx) + status, err := currentGPUStatusProvider()(ctx, false) return gpuAdmission{status: status, err: err} } diff --git a/openapi.yaml b/openapi.yaml index 3621c3e8c..8616face2 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1788,7 +1788,7 @@ components: example: 1024 available: type: integer - description: "Number of virtual functions currently able to create this profile. Best-effort: creating an instance may reduce availability on sibling functions sharing GPU framebuffer." + description: "Number of additional instances of this profile the host can take if only this profile is added. Counts for different profiles are not additive: an instance of one profile can reduce availability of others on the same GPU." example: 59 PassthroughDevice: From b7655f23b700c3ef9602d212e6d3e588b1f7ef5a Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:46:00 +0000 Subject: [PATCH 02/16] Pass sysfs GPU addresses to NVML unchanged and name the placement mode NVML parses the PCI domain numerically, so the sysfs address works as is and the four-digit-domain prefix only broke wider domains. The creatable placements query now sets the heterogeneous mode by name rather than by zero value. --- lib/devices/nvml_linux.go | 6 ++++-- lib/instances/vgpu.go | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/devices/nvml_linux.go b/lib/devices/nvml_linux.go index 20943ec4c..cb81f8d4f 100644 --- a/lib/devices/nvml_linux.go +++ b/lib/devices/nvml_linux.go @@ -21,6 +21,8 @@ const ( nvmlSuccess = 0 nvmlInsufficientSize = 7 nvmlFeatureEnabled = 1 + // nvmlHeterogeneousPlacements is NVML_VGPU_PGPU_HETEROGENEOUS_MODE. + nvmlHeterogeneousPlacements = 0 ) // nvmlPlacementList is nvmlVgpuPlacementList_v2_t. @@ -121,8 +123,7 @@ func nvmlCreatableProfiles(gpus []string) (map[string][]creatableProfile, error) func (f *nvmlFuncs) creatableProfiles(gpu string) ([]creatableProfile, error) { var device uintptr - // sysfs addresses have a four-digit domain; NVML bus IDs have eight. - if err := f.check("nvmlDeviceGetHandleByPciBusId_v2", f.deviceGetHandleByPCIBusID("0000"+gpu, &device)); err != nil { + if err := f.check("nvmlDeviceGetHandleByPciBusId_v2", f.deviceGetHandleByPCIBusID(gpu, &device)); err != nil { return nil, err } mode := nvmlHeterogeneousMode{version: nvmlStructVersion(unsafe.Sizeof(nvmlHeterogeneousMode{}), 1)} @@ -202,6 +203,7 @@ func (f *nvmlFuncs) placementList(device uintptr, typeID uint32, ids []uint32) ( version: nvmlStructVersion(unsafe.Sizeof(nvmlPlacementList{}), 2), count: uint32(len(ids)), placementIDs: &ids[0], + mode: nvmlHeterogeneousPlacements, } rc := f.deviceGetCreatablePlacements(device, typeID, &list) return list, rc diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 80d837e21..7c088da73 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -121,8 +121,9 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str assignment := devices.VGPUAssignment{Framework: device.Framework, DevicePath: device.SysfsPath} // The type is written under the allocation lock, so the placement the // driver reported as free is taken before the next claim asks again. - // Nothing else resets a VF without a saved claim, and the write may have - // landed before a later check failed, so a failed configure resets it. + // The write may have landed before a later check failed, and without a + // saved claim the VF would hold its placement until the allocator next + // repairs it, so a failed configure resets it. if err := m.configureClaimedVGPU(ctx, device); err != nil { if resetErr := m.destroyVGPUAssignment(ctx, assignment); resetErr != nil { log.WarnContext(ctx, "failed to reset vGPU VF after configure failed", "vf", vfAddress, "error", resetErr) From e837c75402bd2e4123b4cf8d7293fda8d5baa04a Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 19 Oct 2026 16:54:05 +0000 Subject: [PATCH 03/16] Fall back to the sysfs lists only for GPUs NVML cannot answer for A GPU that NVML fails for, or that is not in mixed-size mode, no longer sends every GPU on the host back to the sysfs lists. The others keep NVML's creatable placements, so the profiles sysfs hides stay placeable there. A query that cannot load NVML or times out still falls back for every GPU. --- lib/devices/GPU.md | 2 +- lib/devices/nvml_linux.go | 10 ++++--- lib/devices/vendor_vfio_linux.go | 34 +++++++++++++----------- lib/devices/vgpu_creatable_linux.go | 28 +++++++++++++++---- lib/devices/vgpu_creatable_linux_test.go | 33 +++++++++++++++++++++++ lib/otel/README.md | 2 +- lib/resources/monitoring.go | 2 +- 7 files changed, 84 insertions(+), 27 deletions(-) diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index c7d4af289..623cb8b94 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -108,7 +108,7 @@ before the hypervisor is signalled. Hypeman reconciles metadata claims once at startup and every minute afterward, skipping hosts without GPUs. A claim whose VMM is confirmed dead is reset before the claim is removed. An ambiguous hypervisor ownership check preserves the claim, logs a warning, and increments `hypeman_instances_vgpu_liveness_uncertain_total`; the create and start cleanup paths preserve and count the same way. mdev hosts also sweep orphaned device-level assignments. Vendor VFIO hosts repair an unclaimed dirty VF when the allocator next selects it; repair checks for open VFIO handles before resetting `current_vgpu_type`. The allocator prefers VFs that are already clean, and a dirty VF that refuses its reset is skipped in favor of another candidate. -Vendor VFIO placement and `/resources` both read the driver's creatable placements through NVML (`nvmlDeviceGetVgpuTypeCreatablePlacements`, the query behind `nvidia-smi vgpu -c -v`), which list the free placement IDs of every profile each GPU can still create. Hypeman loads `libnvidia-ml.so.1` at runtime on first use and keeps it initialized; a query for every profile on two L40S GPUs takes about 20 ms. A free VF can take any profile its GPU lists there, and a claim only writes a type the driver has just reported a free placement for; if that configure still fails, the VF is reset and the claim fails without trying another GPU. The per-VF `creatable_vgpu_types` lists are not used for this because in mixed-size (heterogeneous) mode they under-report: the driver drops a profile from them once its GPU holds as many vGPUs as that profile's equal-size instance limit (12 for L40S-4Q), even while a placement for it remains. If NVML cannot be loaded or initialized, fails, takes longer than 5 s, or reports a GPU not in mixed-size mode, placement and `/resources` fall back to those lists, so profiles they hide are refused until it recovers. A query that times out is abandoned rather than waited for, since a call stuck in the driver cannot be interrupted; until it returns, queries skip NVML and use the lists directly, so a hung driver costs one timeout rather than one per claim. Each fallback logs a warning and increments `hypeman_resources_gpu_nvml_fallbacks_total` with `reason` `error`, `timeout` or `stuck`. Admission only checks free VFs, so it does not query NVML. +Vendor VFIO placement and `/resources` both read the driver's creatable placements through NVML (`nvmlDeviceGetVgpuTypeCreatablePlacements`, the query behind `nvidia-smi vgpu -c -v`), which list the free placement IDs of every profile each GPU can still create. Hypeman loads `libnvidia-ml.so.1` at runtime on first use and keeps it initialized; a query for every profile on two L40S GPUs takes about 20 ms. A free VF can take any profile its GPU lists there, and a claim only writes a type the driver has just reported a free placement for; if that configure still fails, the VF is reset and the claim fails without trying another GPU. The per-VF `creatable_vgpu_types` lists are not used for this because in mixed-size (heterogeneous) mode they under-report: the driver drops a profile from them once its GPU holds as many vGPUs as that profile's equal-size instance limit (12 for L40S-4Q), even while a placement for it remains. If NVML cannot be loaded or initialized or takes longer than 5 s, placement and `/resources` fall back to those lists for every GPU; if it fails for one GPU, or reports a GPU not in mixed-size mode, only that GPU falls back. Profiles the lists hide are refused on a GPU that has fallen back until NVML recovers. A query that times out is abandoned rather than waited for, since a call stuck in the driver cannot be interrupted; until it returns, queries skip NVML and use the lists directly, so a hung driver costs one timeout rather than one per claim. Each fallback logs a warning and increments `hypeman_resources_gpu_nvml_fallbacks_total` with `reason` `error`, `timeout` or `stuck`. Admission only checks free VFs, so it does not query NVML. ### Hypervisor Support diff --git a/lib/devices/nvml_linux.go b/lib/devices/nvml_linux.go index cb81f8d4f..b6664a21c 100644 --- a/lib/devices/nvml_linux.go +++ b/lib/devices/nvml_linux.go @@ -3,6 +3,7 @@ package devices import ( + "errors" "fmt" "runtime" "strconv" @@ -104,21 +105,24 @@ func openNVML() (*nvmlFuncs, error) { // nvmlCreatableProfiles asks NVML which profiles each GPU can still create // and at which placements, keyed by GPU address. This is the query behind // nvidia-smi vgpu -c -v. It fails for a GPU not in mixed-size mode, so -// equal-size hosts keep the sysfs lists. +// such a GPU keeps the sysfs lists. A GPU that fails is left out of the +// result and its error joined into the returned one. func nvmlCreatableProfiles(gpus []string) (map[string][]creatableProfile, error) { f, err := openNVML() if err != nil { return nil, err } byGPU := make(map[string][]creatableProfile, len(gpus)) + var errs []error for _, gpu := range gpus { profiles, err := f.creatableProfiles(gpu) if err != nil { - return nil, fmt.Errorf("GPU %s: %w", gpu, err) + errs = append(errs, fmt.Errorf("GPU %s: %w", gpu, err)) + continue } byGPU[gpu] = profiles } - return byGPU, nil + return byGPU, errors.Join(errs...) } func (f *nvmlFuncs) creatableProfiles(gpu string) ([]creatableProfile, error) { diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 95d74f1f7..170bcb846 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -95,11 +95,12 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { // take if only that profile is added: per GPU, the smaller of its free // placements for the profile and its free, non-quarantined VFs. Counts for // different profiles are not additive, because their placements overlap. +// GPUs NVML cannot answer for are counted from the sysfs lists instead. func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction, quarantined map[string]struct{}) ([]GPUProfile, error) { - creatable, err := s.creatableProfiles(vfs) + creatable, sysfsVFs := s.creatableOrSysfs(vfs) + profilesByType, available, err := s.sysfsAvailability(sysfsVFs, quarantined) if err != nil { - recordNVMLFallback(err) - return s.listSysfsProfiles(vfs, quarantined) + return nil, err } freeVFs := make(map[string]int) for _, vf := range vfs { @@ -107,8 +108,6 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction, quarantined map[str freeVFs[vf.ParentGPU]++ } } - profilesByType := make(map[string]VGPUProfileType) - available := make(map[string]int) for gpu, profiles := range creatable { for _, profile := range profiles { profilesByType[profile.TypeName] = profile.VGPUProfileType @@ -118,17 +117,17 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction, quarantined map[str return gpuProfiles(profilesByType, available), nil } -// listSysfsProfiles counts each free, non-quarantined VF advertising a type +// sysfsAvailability counts each free, non-quarantined VF advertising a type // as one creatable instance, matching the driver-reported units that mdev // sums through available_instances. This is a best-effort snapshot because // creating on one VF may revoke the type from siblings that share its GPU // framebuffer. -func (s vendorVFIOSysfs) listSysfsProfiles(vfs []VirtualFunction, quarantined map[string]struct{}) ([]GPUProfile, error) { +func (s vendorVFIOSysfs) sysfsAvailability(vfs []VirtualFunction, quarantined map[string]struct{}) (map[string]VGPUProfileType, map[string]int, error) { profilesByType := make(map[string]VGPUProfileType) creatableVFs := make(map[string]int) profilesByVF, err := s.sysfsProfileTypes(vfs) if err != nil { - return nil, err + return nil, nil, err } for _, vf := range vfs { _, bad := quarantined[vf.PCIAddress] @@ -139,7 +138,7 @@ func (s vendorVFIOSysfs) listSysfsProfiles(vfs []VirtualFunction, quarantined ma } } } - return gpuProfiles(profilesByType, creatableVFs), nil + return profilesByType, creatableVFs, nil } func gpuProfiles(profilesByType map[string]VGPUProfileType, available map[string]int) []GPUProfile { @@ -163,19 +162,22 @@ func gpuProfiles(profilesByType map[string]VGPUProfileType, available map[string // profileTypes returns the profiles each VF can be configured with. A free // VF can take any profile its GPU has a free placement for, as NVML reports // it, and a configured VF takes none. The sysfs creatable lists are only the -// fallback when NVML fails, because in mixed-size mode they hide profiles -// that still fit on a busy GPU. +// fallback for GPUs NVML cannot answer for, because in mixed-size mode they +// hide profiles that still fit on a busy GPU. func (s vendorVFIOSysfs) profileTypes(vfs []VirtualFunction) (map[string][]VGPUProfileType, error) { - creatable, err := s.creatableProfiles(vfs) + creatable, sysfsVFs := s.creatableOrSysfs(vfs) + profilesByVF, err := s.sysfsProfileTypes(sysfsVFs) if err != nil { - recordNVMLFallback(err) - return s.sysfsProfileTypes(vfs) + return nil, err } - profilesByVF := make(map[string][]VGPUProfileType, len(vfs)) for _, vf := range vfs { + gpuCreatable, ok := creatable[vf.ParentGPU] + if !ok { + continue + } profiles := make([]VGPUProfileType, 0) if !vf.Allocated { - for _, profile := range creatable[vf.ParentGPU] { + for _, profile := range gpuCreatable { profiles = append(profiles, profile.VGPUProfileType) } } diff --git a/lib/devices/vgpu_creatable_linux.go b/lib/devices/vgpu_creatable_linux.go index 5463766c1..ad8f28a06 100644 --- a/lib/devices/vgpu_creatable_linux.go +++ b/lib/devices/vgpu_creatable_linux.go @@ -25,7 +25,7 @@ var ( var nvmlStuck atomic.Int32 // nvmlFallbacks counts creatable-placement queries answered from the sysfs -// lists instead, by reason. +// lists for at least one GPU, by reason. var nvmlFallbacks = map[string]*atomic.Int64{ "error": {}, "timeout": {}, @@ -33,8 +33,9 @@ var nvmlFallbacks = map[string]*atomic.Int64{ } // NVMLFallbacks returns how many creatable-placement queries have fallen back -// to the sysfs lists since startup, by reason: NVML failed (error), timed out -// (timeout), or an earlier timed-out query had not returned yet (stuck). +// to the sysfs lists for at least one GPU since startup, by reason: NVML +// failed (error), timed out (timeout), or an earlier timed-out query had not +// returned yet (stuck). func NVMLFallbacks() map[string]int64 { counts := make(map[string]int64, len(nvmlFallbacks)) for reason, count := range nvmlFallbacks { @@ -65,7 +66,8 @@ type creatableProfile struct { // creatableProfiles asks the driver, through NVML, which profiles the GPU of // each VF can still create and at which placements, keyed by parent GPU // address. Unlike the per-VF creatable_vgpu_types lists, this stays accurate -// on a busy GPU in mixed-size mode; see GPU.md. +// on a busy GPU in mixed-size mode; see GPU.md. GPUs NVML could not answer +// for are missing from the result, and the error says why. func (s vendorVFIOSysfs) creatableProfiles(vfs []VirtualFunction) (map[string][]creatableProfile, error) { query := s.creatableFunc if query == nil { @@ -82,11 +84,27 @@ func (s vendorVFIOSysfs) creatableProfiles(vfs []VirtualFunction) (map[string][] sort.Strings(gpus) byGPU, err := runBounded(func() (map[string][]creatableProfile, error) { return query(gpus) }) if err != nil { - return nil, fmt.Errorf("query creatable vGPU placements: %w", err) + return byGPU, fmt.Errorf("query creatable vGPU placements: %w", err) } return byGPU, nil } +// creatableOrSysfs returns the NVML creatable profiles by GPU and the VFs +// whose GPU NVML could not answer for, which fall back to the sysfs lists. +func (s vendorVFIOSysfs) creatableOrSysfs(vfs []VirtualFunction) (map[string][]creatableProfile, []VirtualFunction) { + creatable, err := s.creatableProfiles(vfs) + if err != nil { + recordNVMLFallback(err) + } + sysfsVFs := make([]VirtualFunction, 0) + for _, vf := range vfs { + if _, ok := creatable[vf.ParentGPU]; !ok { + sysfsVFs = append(sysfsVFs, vf) + } + } + return creatable, sysfsVFs +} + // runBounded returns at nvmlTimeout even if query has not: a call stuck in // the driver cannot be interrupted. Until such a query returns, later calls // fail at once instead of each waiting out the timeout, which the claim diff --git a/lib/devices/vgpu_creatable_linux_test.go b/lib/devices/vgpu_creatable_linux_test.go index 08e438b2b..e8c0805e4 100644 --- a/lib/devices/vgpu_creatable_linux_test.go +++ b/lib/devices/vgpu_creatable_linux_test.go @@ -199,3 +199,36 @@ func TestVendorVFIOListProfilesCountsFreePlacements(t *testing.T) { // 12Q: min(1 placement, 2 VFs) + min(2 placements, 1 VF). assert.Equal(t, 2, profileAvailability(profiles, "NVIDIA L40S-12Q")) } + +func TestVendorVFIOFallsBackToSysfsOnlyForGPUsNVMLFails(t *testing.T) { + resetVFHealthStore(t) + sysfs := newTestVendorVFIOSysfs(t) + sysfs.creatableFunc = func(gpus []string) (map[string][]creatableProfile, error) { + return map[string][]creatableProfile{"0000:82:00.0": testCreatablePlacements()["0000:82:00.0"]}, + errors.New("GPU 0000:e3:00.0: not in mixed-size vGPU mode") + } + const hiddenFromSysfs = "ID : vGPU Name\n1148 : NVIDIA L40S-2Q\n" + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", hiddenFromSysfs) + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "43", "0", testCreatableTypes) + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + before := NVMLFallbacks()["error"] + + profilesByVF, err := sysfs.profileTypes(vfs) + require.NoError(t, err) + assert.Contains(t, profilesByVF["0000:82:00.4"], VGPUProfileType{TypeName: "1150", Name: "NVIDIA L40S-4Q", FramebufferMB: 4096}, + "the GPU NVML answered for keeps the profile sysfs hides") + assert.Equal(t, []VGPUProfileType{ + {TypeName: "1147", Name: "NVIDIA L40S-1Q", FramebufferMB: 1024}, + {TypeName: "1148", Name: "NVIDIA L40S-2Q", FramebufferMB: 2048}, + {TypeName: "1159", Name: "NVIDIA L40S-48Q", FramebufferMB: 49152}, + }, profilesByVF["0000:e3:00.4"], "the failed GPU uses its sysfs list") + assert.Equal(t, before+1, NVMLFallbacks()["error"]) + + profiles, err := sysfs.listProfiles(vfs, nil) + require.NoError(t, err) + // 2Q: min(4 placements, 1 VF) on GPU 82 plus one advertising VF on e3. + assert.Equal(t, 2, profileAvailability(profiles, "NVIDIA L40S-2Q")) + assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-4Q")) + assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-48Q")) +} diff --git a/lib/otel/README.md b/lib/otel/README.md index 1481ac934..88af2a934 100644 --- a/lib/otel/README.md +++ b/lib/otel/README.md @@ -100,7 +100,7 @@ This keeps pull and push views aligned because both are sourced from the same OT | `hypeman_resources_image_storage_bytes` | gauge | kind | Current and maximum image storage bytes | | `hypeman_resources_gpu_slots` | gauge | kind | Total and used GPU slots | | `hypeman_resources_gpu_profile_slots` | gauge | profile, kind | How many more vGPUs of each profile fit if only that profile is added | -| `hypeman_resources_gpu_nvml_fallbacks_total` | counter | reason | vGPU capacity queries answered from the sysfs creatable lists because NVML failed, timed out, or was stuck | +| `hypeman_resources_gpu_nvml_fallbacks_total` | counter | reason | vGPU capacity queries answered from the sysfs creatable lists for at least one GPU because NVML failed, timed out, or was stuck | ### Volumes | Metric | Type | Description | diff --git a/lib/resources/monitoring.go b/lib/resources/monitoring.go index c06edcbad..7cb1e43fe 100644 --- a/lib/resources/monitoring.go +++ b/lib/resources/monitoring.go @@ -208,7 +208,7 @@ func newMonitoringMetrics(meter metric.Meter, mgr *Manager) error { gpuNVMLFallbacks, err := meter.Int64ObservableCounter( "hypeman_resources_gpu_nvml_fallbacks_total", - metric.WithDescription("vGPU capacity queries answered from the sysfs creatable lists because NVML failed (error), timed out (timeout), or an earlier timed-out query had not returned (stuck)"), + metric.WithDescription("vGPU capacity queries answered from the sysfs creatable lists for at least one GPU because NVML failed (error), timed out (timeout), or an earlier timed-out query had not returned (stuck)"), ) if err != nil { return err From 67cbd805209b1426905b5a029ec9e15cb9d4a66c Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:33:16 +0000 Subject: [PATCH 04/16] Count equal-size GPUs as their own NVML fallback reason A GPU drops to equal-size mode after a driver reload or GPU reset, and NVML then refuses its creatable placements. That fell back under the error reason, the same as NVML failing outright, so a drifted host kept the error count climbing on every claim and resources refresh. Count it as equal_size when every GPU NVML failed for is in equal-size mode, so error stays a signal for NVML itself. --- lib/devices/GPU.md | 2 +- lib/devices/nvml_linux.go | 2 +- lib/devices/vgpu_creatable_linux.go | 35 +++++++++++++++++----- lib/devices/vgpu_creatable_linux_test.go | 38 ++++++++++++++++++++++-- lib/otel/README.md | 2 +- lib/resources/monitoring.go | 2 +- 6 files changed, 67 insertions(+), 14 deletions(-) diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 623cb8b94..b1c3499f4 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -108,7 +108,7 @@ before the hypervisor is signalled. Hypeman reconciles metadata claims once at startup and every minute afterward, skipping hosts without GPUs. A claim whose VMM is confirmed dead is reset before the claim is removed. An ambiguous hypervisor ownership check preserves the claim, logs a warning, and increments `hypeman_instances_vgpu_liveness_uncertain_total`; the create and start cleanup paths preserve and count the same way. mdev hosts also sweep orphaned device-level assignments. Vendor VFIO hosts repair an unclaimed dirty VF when the allocator next selects it; repair checks for open VFIO handles before resetting `current_vgpu_type`. The allocator prefers VFs that are already clean, and a dirty VF that refuses its reset is skipped in favor of another candidate. -Vendor VFIO placement and `/resources` both read the driver's creatable placements through NVML (`nvmlDeviceGetVgpuTypeCreatablePlacements`, the query behind `nvidia-smi vgpu -c -v`), which list the free placement IDs of every profile each GPU can still create. Hypeman loads `libnvidia-ml.so.1` at runtime on first use and keeps it initialized; a query for every profile on two L40S GPUs takes about 20 ms. A free VF can take any profile its GPU lists there, and a claim only writes a type the driver has just reported a free placement for; if that configure still fails, the VF is reset and the claim fails without trying another GPU. The per-VF `creatable_vgpu_types` lists are not used for this because in mixed-size (heterogeneous) mode they under-report: the driver drops a profile from them once its GPU holds as many vGPUs as that profile's equal-size instance limit (12 for L40S-4Q), even while a placement for it remains. If NVML cannot be loaded or initialized or takes longer than 5 s, placement and `/resources` fall back to those lists for every GPU; if it fails for one GPU, or reports a GPU not in mixed-size mode, only that GPU falls back. Profiles the lists hide are refused on a GPU that has fallen back until NVML recovers. A query that times out is abandoned rather than waited for, since a call stuck in the driver cannot be interrupted; until it returns, queries skip NVML and use the lists directly, so a hung driver costs one timeout rather than one per claim. Each fallback logs a warning and increments `hypeman_resources_gpu_nvml_fallbacks_total` with `reason` `error`, `timeout` or `stuck`. Admission only checks free VFs, so it does not query NVML. +Vendor VFIO placement and `/resources` both read the driver's creatable placements through NVML (`nvmlDeviceGetVgpuTypeCreatablePlacements`, the query behind `nvidia-smi vgpu -c -v`), which list the free placement IDs of every profile each GPU can still create. Hypeman loads `libnvidia-ml.so.1` at runtime on first use and keeps it initialized; a query for every profile on two L40S GPUs takes about 20 ms. A free VF can take any profile its GPU lists there, and a claim only writes a type the driver has just reported a free placement for; if that configure still fails, the VF is reset and the claim fails without trying another GPU. The per-VF `creatable_vgpu_types` lists are not used for this because in mixed-size (heterogeneous) mode they under-report: the driver drops a profile from them once its GPU holds as many vGPUs as that profile's equal-size instance limit (12 for L40S-4Q), even while a placement for it remains. If NVML cannot be loaded or initialized or takes longer than 5 s, placement and `/resources` fall back to those lists for every GPU; if it fails for one GPU, or reports a GPU not in mixed-size mode, only that GPU falls back. Profiles the lists hide are refused on a GPU that has fallen back until NVML recovers. A query that times out is abandoned rather than waited for, since a call stuck in the driver cannot be interrupted; until it returns, queries skip NVML and use the lists directly, so a hung driver costs one timeout rather than one per claim. Each fallback logs a warning and increments `hypeman_resources_gpu_nvml_fallbacks_total` with `reason` `error`, `equal_size` (every GPU NVML failed for is in equal-size mode, as after a driver reload or GPU reset), `timeout` or `stuck`. Admission only checks free VFs, so it does not query NVML. ### Hypervisor Support diff --git a/lib/devices/nvml_linux.go b/lib/devices/nvml_linux.go index b6664a21c..b60771e7d 100644 --- a/lib/devices/nvml_linux.go +++ b/lib/devices/nvml_linux.go @@ -135,7 +135,7 @@ func (f *nvmlFuncs) creatableProfiles(gpu string) ([]creatableProfile, error) { return nil, err } if mode.mode != nvmlFeatureEnabled { - return nil, fmt.Errorf("not in mixed-size vGPU mode") + return nil, errNVMLEqualSize } typeIDs, err := f.supportedTypes(device) if err != nil { diff --git a/lib/devices/vgpu_creatable_linux.go b/lib/devices/vgpu_creatable_linux.go index ad8f28a06..a119bd7ce 100644 --- a/lib/devices/vgpu_creatable_linux.go +++ b/lib/devices/vgpu_creatable_linux.go @@ -17,8 +17,9 @@ import ( var nvmlTimeout = 5 * time.Second var ( - errNVMLTimeout = errors.New("NVML query timed out") - errNVMLStuck = errors.New("an earlier NVML query is still running after its timeout") + errNVMLTimeout = errors.New("NVML query timed out") + errNVMLStuck = errors.New("an earlier NVML query is still running after its timeout") + errNVMLEqualSize = errors.New("not in mixed-size vGPU mode") ) // nvmlStuck counts timed-out NVML queries that have not returned. @@ -27,15 +28,17 @@ var nvmlStuck atomic.Int32 // nvmlFallbacks counts creatable-placement queries answered from the sysfs // lists for at least one GPU, by reason. var nvmlFallbacks = map[string]*atomic.Int64{ - "error": {}, - "timeout": {}, - "stuck": {}, + "error": {}, + "equal_size": {}, + "timeout": {}, + "stuck": {}, } // NVMLFallbacks returns how many creatable-placement queries have fallen back // to the sysfs lists for at least one GPU since startup, by reason: NVML -// failed (error), timed out (timeout), or an earlier timed-out query had not -// returned yet (stuck). +// failed (error), every GPU it failed for is in equal-size mode (equal_size), +// it timed out (timeout), or an earlier timed-out query had not returned yet +// (stuck). func NVMLFallbacks() map[string]int64 { counts := make(map[string]int64, len(nvmlFallbacks)) for reason, count := range nvmlFallbacks { @@ -51,11 +54,29 @@ func recordNVMLFallback(err error) { reason = "stuck" case errors.Is(err, errNVMLTimeout): reason = "timeout" + case onlyEqualSize(err): + reason = "equal_size" } nvmlFallbacks[reason].Add(1) slog.Default().Warn("falling back to sysfs creatable vGPU types", "reason", reason, "error", err) } +// onlyEqualSize reports whether every GPU NVML failed for was left out only +// because it is in equal-size mode, which happens when a driver reload or +// GPU reset drops mixed-size mode, rather than because NVML failed. +func onlyEqualSize(err error) bool { + var joined interface{ Unwrap() []error } + if !errors.As(err, &joined) { + return errors.Is(err, errNVMLEqualSize) + } + for _, gpuErr := range joined.Unwrap() { + if !errors.Is(gpuErr, errNVMLEqualSize) { + return false + } + } + return true +} + // creatableProfile is a profile a GPU can still create, with the placement // IDs a new vGPU of that profile could start at. type creatableProfile struct { diff --git a/lib/devices/vgpu_creatable_linux_test.go b/lib/devices/vgpu_creatable_linux_test.go index e8c0805e4..40de27c51 100644 --- a/lib/devices/vgpu_creatable_linux_test.go +++ b/lib/devices/vgpu_creatable_linux_test.go @@ -4,6 +4,7 @@ package devices import ( "errors" + "fmt" "sync/atomic" "testing" "time" @@ -205,14 +206,14 @@ func TestVendorVFIOFallsBackToSysfsOnlyForGPUsNVMLFails(t *testing.T) { sysfs := newTestVendorVFIOSysfs(t) sysfs.creatableFunc = func(gpus []string) (map[string][]creatableProfile, error) { return map[string][]creatableProfile{"0000:82:00.0": testCreatablePlacements()["0000:82:00.0"]}, - errors.New("GPU 0000:e3:00.0: not in mixed-size vGPU mode") + errors.Join(fmt.Errorf("GPU 0000:e3:00.0: %w", errNVMLEqualSize)) } const hiddenFromSysfs = "ID : vGPU Name\n1148 : NVIDIA L40S-2Q\n" sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", hiddenFromSysfs) sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "43", "0", testCreatableTypes) vfs, err := sysfs.discoverVFs() require.NoError(t, err) - before := NVMLFallbacks()["error"] + before := NVMLFallbacks() profilesByVF, err := sysfs.profileTypes(vfs) require.NoError(t, err) @@ -223,7 +224,9 @@ func TestVendorVFIOFallsBackToSysfsOnlyForGPUsNVMLFails(t *testing.T) { {TypeName: "1148", Name: "NVIDIA L40S-2Q", FramebufferMB: 2048}, {TypeName: "1159", Name: "NVIDIA L40S-48Q", FramebufferMB: 49152}, }, profilesByVF["0000:e3:00.4"], "the failed GPU uses its sysfs list") - assert.Equal(t, before+1, NVMLFallbacks()["error"]) + after := NVMLFallbacks() + assert.Equal(t, before["equal_size"]+1, after["equal_size"]) + assert.Equal(t, before["error"], after["error"]) profiles, err := sysfs.listProfiles(vfs, nil) require.NoError(t, err) @@ -232,3 +235,32 @@ func TestVendorVFIOFallsBackToSysfsOnlyForGPUsNVMLFails(t *testing.T) { assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-4Q")) assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-48Q")) } + +func TestNVMLFallbackReasons(t *testing.T) { + equalSize := fmt.Errorf("GPU 0000:e3:00.0: %w", errNVMLEqualSize) + failed := errors.New("GPU 0000:82:00.0: nvmlDeviceGetSupportedVgpus: Unknown Error") + for _, tc := range []struct { + name string + err error + reason string + }{ + {"every failed GPU is in equal-size mode", errors.Join(equalSize, equalSize), "equal_size"}, + {"one failed GPU is not in equal-size mode", errors.Join(equalSize, failed), "error"}, + {"NVML cannot load", errors.New("load libnvidia-ml.so.1: not found"), "error"}, + {"timed out", errNVMLTimeout, "timeout"}, + {"stuck", errNVMLStuck, "stuck"}, + } { + t.Run(tc.name, func(t *testing.T) { + before := NVMLFallbacks() + recordNVMLFallback(fmt.Errorf("query creatable vGPU placements: %w", tc.err)) + after := NVMLFallbacks() + for reason := range after { + want := before[reason] + if reason == tc.reason { + want++ + } + assert.Equal(t, want, after[reason], reason) + } + }) + } +} diff --git a/lib/otel/README.md b/lib/otel/README.md index 88af2a934..70762a5c9 100644 --- a/lib/otel/README.md +++ b/lib/otel/README.md @@ -100,7 +100,7 @@ This keeps pull and push views aligned because both are sourced from the same OT | `hypeman_resources_image_storage_bytes` | gauge | kind | Current and maximum image storage bytes | | `hypeman_resources_gpu_slots` | gauge | kind | Total and used GPU slots | | `hypeman_resources_gpu_profile_slots` | gauge | profile, kind | How many more vGPUs of each profile fit if only that profile is added | -| `hypeman_resources_gpu_nvml_fallbacks_total` | counter | reason | vGPU capacity queries answered from the sysfs creatable lists for at least one GPU because NVML failed, timed out, or was stuck | +| `hypeman_resources_gpu_nvml_fallbacks_total` | counter | reason | vGPU capacity queries answered from the sysfs creatable lists for at least one GPU because NVML failed, a GPU was in equal-size mode, or NVML timed out or was stuck | ### Volumes | Metric | Type | Description | diff --git a/lib/resources/monitoring.go b/lib/resources/monitoring.go index 7cb1e43fe..95aff3868 100644 --- a/lib/resources/monitoring.go +++ b/lib/resources/monitoring.go @@ -208,7 +208,7 @@ func newMonitoringMetrics(meter metric.Meter, mgr *Manager) error { gpuNVMLFallbacks, err := meter.Int64ObservableCounter( "hypeman_resources_gpu_nvml_fallbacks_total", - metric.WithDescription("vGPU capacity queries answered from the sysfs creatable lists for at least one GPU because NVML failed (error), timed out (timeout), or an earlier timed-out query had not returned (stuck)"), + metric.WithDescription("vGPU capacity queries answered from the sysfs creatable lists for at least one GPU because NVML failed (error), every GPU it failed for was in equal-size mode (equal_size), it timed out (timeout), or an earlier timed-out query had not returned (stuck)"), ) if err != nil { return err From 47ab58f68cc97520c1782d78374159098ca1fb37 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:35:35 +0000 Subject: [PATCH 05/16] Reset unclaimed vendor VFIO VFs when the reconciler starts A claim now writes the type before it saves the claim, so a restart between the two leaves a configured VF that no instance claims. It holds a placement and is left out of /resources, and nothing reset it until a later claim happened to pick it or found no VF for its profile. Reset such VFs once at startup, under the allocation lock and behind the same open-VFIO-handle check the allocator's repair uses. --- lib/devices/GPU.md | 4 ++-- lib/instances/vgpu.go | 2 +- lib/instances/vgpu_reconcile.go | 31 ++++++++++++++++++++++----- lib/instances/vgpu_reconcile_test.go | 32 ++++++++++++++++++++++++++++ 4 files changed, 61 insertions(+), 8 deletions(-) diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index b1c3499f4..a389c1b49 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -94,7 +94,7 @@ On an mdev host, the response also includes the assigned mdev UUID: ### Ephemeral vGPU Lifecycle -vGPU assignments are created on instance start and released on stop or delete. Hypeman creates/removes an mdev on mdev hosts. On vendor VFIO hosts, it selects a VF on a GPU with a free placement for the profile and writes the profile ID to `current_vgpu_type` under the allocation lock, so the placement is taken before the next claim checks for one, then reserves the VF by persisting its path in instance metadata. Metadata is the authoritative VF claim; a configured VF without one is dirty and is reset by the allocator. +vGPU assignments are created on instance start and released on stop or delete. Hypeman creates/removes an mdev on mdev hosts. On vendor VFIO hosts, it selects a VF on a GPU with a free placement for the profile and writes the profile ID to `current_vgpu_type` under the allocation lock, so the placement is taken before the next claim checks for one, then reserves the VF by persisting its path in instance metadata. Metadata is the authoritative VF claim; a configured VF without one is dirty and is reset at startup or by the allocator. ``` Instance Create → Configure profile → Persist VF claim → Attach VF to VM → Instance Running @@ -106,7 +106,7 @@ itself down and wait for it. Delete normally caps that wait at 2s, but guests with a vGPU or passthrough device get 5s so the guest can unload its GPU driver before the hypervisor is signalled. -Hypeman reconciles metadata claims once at startup and every minute afterward, skipping hosts without GPUs. A claim whose VMM is confirmed dead is reset before the claim is removed. An ambiguous hypervisor ownership check preserves the claim, logs a warning, and increments `hypeman_instances_vgpu_liveness_uncertain_total`; the create and start cleanup paths preserve and count the same way. mdev hosts also sweep orphaned device-level assignments. Vendor VFIO hosts repair an unclaimed dirty VF when the allocator next selects it; repair checks for open VFIO handles before resetting `current_vgpu_type`. The allocator prefers VFs that are already clean, and a dirty VF that refuses its reset is skipped in favor of another candidate. +Hypeman reconciles metadata claims once at startup and every minute afterward, skipping hosts without GPUs. A claim whose VMM is confirmed dead is reset before the claim is removed. An ambiguous hypervisor ownership check preserves the claim, logs a warning, and increments `hypeman_instances_vgpu_liveness_uncertain_total`; the create and start cleanup paths preserve and count the same way. mdev hosts also sweep orphaned device-level assignments. Vendor VFIO hosts reset unclaimed dirty VFs at startup, which covers a restart between a claim writing the type and saving the claim, and otherwise repair an unclaimed dirty VF when the allocator next selects it; repair checks for open VFIO handles before resetting `current_vgpu_type`. The allocator prefers VFs that are already clean, and a dirty VF that refuses its reset is skipped in favor of another candidate. Vendor VFIO placement and `/resources` both read the driver's creatable placements through NVML (`nvmlDeviceGetVgpuTypeCreatablePlacements`, the query behind `nvidia-smi vgpu -c -v`), which list the free placement IDs of every profile each GPU can still create. Hypeman loads `libnvidia-ml.so.1` at runtime on first use and keeps it initialized; a query for every profile on two L40S GPUs takes about 20 ms. A free VF can take any profile its GPU lists there, and a claim only writes a type the driver has just reported a free placement for; if that configure still fails, the VF is reset and the claim fails without trying another GPU. The per-VF `creatable_vgpu_types` lists are not used for this because in mixed-size (heterogeneous) mode they under-report: the driver drops a profile from them once its GPU holds as many vGPUs as that profile's equal-size instance limit (12 for L40S-4Q), even while a placement for it remains. If NVML cannot be loaded or initialized or takes longer than 5 s, placement and `/resources` fall back to those lists for every GPU; if it fails for one GPU, or reports a GPU not in mixed-size mode, only that GPU falls back. Profiles the lists hide are refused on a GPU that has fallen back until NVML recovers. A query that times out is abandoned rather than waited for, since a call stuck in the driver cannot be interrupted; until it returns, queries skip NVML and use the lists directly, so a hung driver costs one timeout rather than one per claim. Each fallback logs a warning and increments `hypeman_resources_gpu_nvml_fallbacks_total` with `reason` `error`, `equal_size` (every GPU NVML failed for is in equal-size mode, as after a driver reload or GPU reset), `timeout` or `stuck`. Admission only checks free VFs, so it does not query NVML. diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 7c088da73..d1a0ae70b 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -174,7 +174,7 @@ func (m *manager) resetDirtyUnclaimedVFs(ctx context.Context, vfs []devices.Virt log.WarnContext(ctx, "failed to reset dirty vGPU VF", "vf", vf.PCIAddress, "error", err) continue } - log.InfoContext(ctx, "reset dirty vGPU VF during allocation", "vf", vf.PCIAddress) + log.InfoContext(ctx, "reset dirty unclaimed vGPU VF", "vf", vf.PCIAddress) reset = true } return reset diff --git a/lib/instances/vgpu_reconcile.go b/lib/instances/vgpu_reconcile.go index c4d2defdb..63b234cd5 100644 --- a/lib/instances/vgpu_reconcile.go +++ b/lib/instances/vgpu_reconcile.go @@ -11,9 +11,10 @@ import ( const defaultVGPUReconcileInterval = time.Minute -// StartVGPUReconciler runs one reconcile pass and then keeps reconciling -// periodically until ctx is cancelled. Hosts without a vGPU framework skip -// reconciliation entirely. A discovery failure starts the reconciler anyway. +// StartVGPUReconciler resets unclaimed vendor VFIO VFs, runs one reconcile +// pass and then keeps reconciling periodically until ctx is cancelled. Hosts +// without a vGPU framework skip reconciliation entirely. A discovery failure +// starts the reconciler anyway. func (m *manager) StartVGPUReconciler(ctx context.Context) { framework, _, err := m.discoverVGPUDevices() if err == nil && framework == devices.VGPUFrameworkNone { @@ -22,6 +23,7 @@ func (m *manager) StartVGPUReconciler(ctx context.Context) { if err != nil { logger.FromContext(ctx).WarnContext(ctx, "failed to discover vGPU framework; starting vGPU reconciler anyway", "error", err) } + m.resetUnclaimedVendorVFs(ctx) m.ReconcileVGPUs(ctx) m.vgpuReconcileOnce.Do(func() { interval := m.vgpuReconcileInterval @@ -44,8 +46,8 @@ func (m *manager) StartVGPUReconciler(ctx context.Context) { } // ReconcileVGPUs releases metadata claims whose VMM is confirmed dead. The -// device-level pass remains for mdevs; vendor VFIO leftovers are repaired only -// when the allocator claims that VF again. +// device-level pass remains for mdevs; vendor VFIO leftovers are repaired at +// startup and when the allocator claims that VF again. func (m *manager) ReconcileVGPUs(ctx context.Context) { log := logger.FromContext(ctx) protected, err := m.reconcileVGPUAssignments(ctx) @@ -64,6 +66,25 @@ func (m *manager) ReconcileVGPUs(ctx context.Context) { } } +// resetUnclaimedVendorVFs resets vendor VFIO VFs that are configured but +// claimed by no instance. A claim writes the type before it saves the claim, +// so a restart between the two leaves such a VF holding a placement that +// /resources does not count until the allocator next repairs it. +func (m *manager) resetUnclaimedVendorVFs(ctx context.Context) { + m.vgpuAllocationMu.Lock() + defer m.vgpuAllocationMu.Unlock() + framework, vfs, err := m.discoverVGPUDevices() + if err != nil || framework != devices.VGPUFrameworkVendorVFIO { + return + } + allMetadata, err := m.listMetadataForReconcile() + if err != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to list instances for unclaimed vGPU VF reset", "error", err) + return + } + m.resetDirtyUnclaimedVFs(ctx, vfs, allMetadata) +} + func (m *manager) reconcileVGPUAssignments(ctx context.Context) (map[string]struct{}, error) { allMetadata, err := m.listMetadataForReconcile() if err != nil { diff --git a/lib/instances/vgpu_reconcile_test.go b/lib/instances/vgpu_reconcile_test.go index 51d0624fb..0e4519fa6 100644 --- a/lib/instances/vgpu_reconcile_test.go +++ b/lib/instances/vgpu_reconcile_test.go @@ -135,3 +135,35 @@ func TestStartVGPUReconcilerRunsPeriodically(t *testing.T) { m.StartVGPUReconciler(t.Context()) require.Eventually(t, func() bool { return passes.Load() >= 3 }, 5*time.Second, time.Millisecond) } + +func TestStartVGPUReconcilerResetsUnclaimedVendorVFs(t *testing.T) { + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0", Allocated: true, ProfileType: testVFProfileType}, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0", Allocated: true, ProfileType: testVFProfileType}, + {PCIAddress: "0000:82:00.6", ParentGPU: "0000:82:00.0"}, + } + var destroyed []string + m := &manager{ + paths: paths.New(t.TempDir()), + discoverVGPU: func() (devices.VGPUFramework, []devices.VirtualFunction, error) { + return devices.VGPUFrameworkVendorVFIO, vfs, nil + }, + destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { + destroyed = append(destroyed, assignment.DevicePath) + return nil + }, + reconcileVGPUDevices: func(context.Context, map[string]struct{}) error { return nil }, + vgpuReconcileInterval: time.Hour, + } + liveIdentity := HypervisorProcessIdentity{} + liveIdentity.Set(os.Getpid()) + owner := saveTestVGPUInstance(t, m, "owner") + owner.GPUFramework = devices.VGPUFrameworkVendorVFIO + owner.GPUDevicePath = testVFDevicePath + owner.HypervisorProcessIdentity = liveIdentity + require.NoError(t, m.saveMetadata(owner)) + + m.StartVGPUReconciler(t.Context()) + assert.Equal(t, []string{"/sys/bus/pci/devices/0000:82:00.5"}, destroyed, + "only the configured VF no instance claims is reset") +} From fe2a27de64797d6d694c3f5ce8cc7602010accfa Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:43:10 +0000 Subject: [PATCH 06/16] Run the unclaimed VF reset test only on Linux Vendor VFIO is Linux-only, and on macOS GetDeviceSysfsPath returns an empty path, so the test's expected reset path cannot match there. --- lib/instances/vgpu_reconcile_linux_test.go | 34 ++++++++++++++++++++++ lib/instances/vgpu_reconcile_test.go | 32 -------------------- 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/lib/instances/vgpu_reconcile_linux_test.go b/lib/instances/vgpu_reconcile_linux_test.go index 4625dd4c7..87c808e11 100644 --- a/lib/instances/vgpu_reconcile_linux_test.go +++ b/lib/instances/vgpu_reconcile_linux_test.go @@ -5,8 +5,10 @@ package instances import ( "context" "net" + "os" "path/filepath" "testing" + "time" "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/paths" @@ -51,3 +53,35 @@ func TestReconcileVGPUsProtectsSocketOwnerWithoutPersistedPID(t *testing.T) { require.NoError(t, err) assert.Equal(t, "/sys/bus/mdev/devices/test-mdev", stored.GPUDevicePath) } + +func TestStartVGPUReconcilerResetsUnclaimedVendorVFs(t *testing.T) { + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0", Allocated: true, ProfileType: testVFProfileType}, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0", Allocated: true, ProfileType: testVFProfileType}, + {PCIAddress: "0000:82:00.6", ParentGPU: "0000:82:00.0"}, + } + var destroyed []string + m := &manager{ + paths: paths.New(t.TempDir()), + discoverVGPU: func() (devices.VGPUFramework, []devices.VirtualFunction, error) { + return devices.VGPUFrameworkVendorVFIO, vfs, nil + }, + destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { + destroyed = append(destroyed, assignment.DevicePath) + return nil + }, + reconcileVGPUDevices: func(context.Context, map[string]struct{}) error { return nil }, + vgpuReconcileInterval: time.Hour, + } + liveIdentity := HypervisorProcessIdentity{} + liveIdentity.Set(os.Getpid()) + owner := saveTestVGPUInstance(t, m, "owner") + owner.GPUFramework = devices.VGPUFrameworkVendorVFIO + owner.GPUDevicePath = testVFDevicePath + owner.HypervisorProcessIdentity = liveIdentity + require.NoError(t, m.saveMetadata(owner)) + + m.StartVGPUReconciler(t.Context()) + assert.Equal(t, []string{"/sys/bus/pci/devices/0000:82:00.5"}, destroyed, + "only the configured VF no instance claims is reset") +} diff --git a/lib/instances/vgpu_reconcile_test.go b/lib/instances/vgpu_reconcile_test.go index 0e4519fa6..51d0624fb 100644 --- a/lib/instances/vgpu_reconcile_test.go +++ b/lib/instances/vgpu_reconcile_test.go @@ -135,35 +135,3 @@ func TestStartVGPUReconcilerRunsPeriodically(t *testing.T) { m.StartVGPUReconciler(t.Context()) require.Eventually(t, func() bool { return passes.Load() >= 3 }, 5*time.Second, time.Millisecond) } - -func TestStartVGPUReconcilerResetsUnclaimedVendorVFs(t *testing.T) { - vfs := []devices.VirtualFunction{ - {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0", Allocated: true, ProfileType: testVFProfileType}, - {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0", Allocated: true, ProfileType: testVFProfileType}, - {PCIAddress: "0000:82:00.6", ParentGPU: "0000:82:00.0"}, - } - var destroyed []string - m := &manager{ - paths: paths.New(t.TempDir()), - discoverVGPU: func() (devices.VGPUFramework, []devices.VirtualFunction, error) { - return devices.VGPUFrameworkVendorVFIO, vfs, nil - }, - destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { - destroyed = append(destroyed, assignment.DevicePath) - return nil - }, - reconcileVGPUDevices: func(context.Context, map[string]struct{}) error { return nil }, - vgpuReconcileInterval: time.Hour, - } - liveIdentity := HypervisorProcessIdentity{} - liveIdentity.Set(os.Getpid()) - owner := saveTestVGPUInstance(t, m, "owner") - owner.GPUFramework = devices.VGPUFrameworkVendorVFIO - owner.GPUDevicePath = testVFDevicePath - owner.HypervisorProcessIdentity = liveIdentity - require.NoError(t, m.saveMetadata(owner)) - - m.StartVGPUReconciler(t.Context()) - assert.Equal(t, []string{"/sys/bus/pci/devices/0000:82:00.5"}, destroyed, - "only the configured VF no instance claims is reset") -} From 79357c95393fb1b9f843022259e0a69f26caaf8a Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:13:31 +0000 Subject: [PATCH 07/16] Initialize NVML for each creatable-placements query An initialized NVML holds every GPU's device nodes open, the same nodes DCGM holds, so keeping it initialized for hypeman's lifetime would block cycling SR-IOV on a wedged GPU and unloading the driver until hypeman is stopped. Initialize NVML for each query and shut it down after; the library stays loaded. On a two-GPU L40S host the query now takes about 55 ms rather than 20 ms, and no NVML handle is left open between queries. This also means a query never runs on NVML state left over from before a driver event. --- lib/devices/GPU.md | 2 +- lib/devices/nvml_linux.go | 28 ++++++++------ lib/devices/vgpu_creatable_linux.go | 6 +-- lib/devices/vgpu_creatable_linux_test.go | 47 ++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 15 deletions(-) diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index a389c1b49..3176b2ccb 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -108,7 +108,7 @@ before the hypervisor is signalled. Hypeman reconciles metadata claims once at startup and every minute afterward, skipping hosts without GPUs. A claim whose VMM is confirmed dead is reset before the claim is removed. An ambiguous hypervisor ownership check preserves the claim, logs a warning, and increments `hypeman_instances_vgpu_liveness_uncertain_total`; the create and start cleanup paths preserve and count the same way. mdev hosts also sweep orphaned device-level assignments. Vendor VFIO hosts reset unclaimed dirty VFs at startup, which covers a restart between a claim writing the type and saving the claim, and otherwise repair an unclaimed dirty VF when the allocator next selects it; repair checks for open VFIO handles before resetting `current_vgpu_type`. The allocator prefers VFs that are already clean, and a dirty VF that refuses its reset is skipped in favor of another candidate. -Vendor VFIO placement and `/resources` both read the driver's creatable placements through NVML (`nvmlDeviceGetVgpuTypeCreatablePlacements`, the query behind `nvidia-smi vgpu -c -v`), which list the free placement IDs of every profile each GPU can still create. Hypeman loads `libnvidia-ml.so.1` at runtime on first use and keeps it initialized; a query for every profile on two L40S GPUs takes about 20 ms. A free VF can take any profile its GPU lists there, and a claim only writes a type the driver has just reported a free placement for; if that configure still fails, the VF is reset and the claim fails without trying another GPU. The per-VF `creatable_vgpu_types` lists are not used for this because in mixed-size (heterogeneous) mode they under-report: the driver drops a profile from them once its GPU holds as many vGPUs as that profile's equal-size instance limit (12 for L40S-4Q), even while a placement for it remains. If NVML cannot be loaded or initialized or takes longer than 5 s, placement and `/resources` fall back to those lists for every GPU; if it fails for one GPU, or reports a GPU not in mixed-size mode, only that GPU falls back. Profiles the lists hide are refused on a GPU that has fallen back until NVML recovers. A query that times out is abandoned rather than waited for, since a call stuck in the driver cannot be interrupted; until it returns, queries skip NVML and use the lists directly, so a hung driver costs one timeout rather than one per claim. Each fallback logs a warning and increments `hypeman_resources_gpu_nvml_fallbacks_total` with `reason` `error`, `equal_size` (every GPU NVML failed for is in equal-size mode, as after a driver reload or GPU reset), `timeout` or `stuck`. Admission only checks free VFs, so it does not query NVML. +Vendor VFIO placement and `/resources` both read the driver's creatable placements through NVML (`nvmlDeviceGetVgpuTypeCreatablePlacements`, the query behind `nvidia-smi vgpu -c -v`), which list the free placement IDs of every profile each GPU can still create. Hypeman loads `libnvidia-ml.so.1` at runtime on first use, and initializes NVML for each query and shuts it down after, because an initialized NVML holds the GPU device nodes open, which would block the SR-IOV cycle below and driver unloads; a query for every profile on two L40S GPUs takes about 55 ms including the initialization, against about 20 ms with NVML already initialized. A free VF can take any profile its GPU lists there, and a claim only writes a type the driver has just reported a free placement for; if that configure still fails, the VF is reset and the claim fails without trying another GPU. The per-VF `creatable_vgpu_types` lists are not used for this because in mixed-size (heterogeneous) mode they under-report: the driver drops a profile from them once its GPU holds as many vGPUs as that profile's equal-size instance limit (12 for L40S-4Q), even while a placement for it remains. If NVML cannot be loaded or initialized or takes longer than 5 s, placement and `/resources` fall back to those lists for every GPU; if it fails for one GPU, or reports a GPU not in mixed-size mode, only that GPU falls back. Profiles the lists hide are refused on a GPU that has fallen back until NVML recovers. A query that times out is abandoned rather than waited for, since a call stuck in the driver cannot be interrupted; until it returns, queries skip NVML and use the lists directly, so a hung driver costs one timeout rather than one per claim. Each fallback logs a warning and increments `hypeman_resources_gpu_nvml_fallbacks_total` with `reason` `error`, `equal_size` (every GPU NVML failed for is in equal-size mode, as after a driver reload or GPU reset), `timeout` or `stuck`. Admission only checks free VFs, so it does not query NVML. ### Hypervisor Support diff --git a/lib/devices/nvml_linux.go b/lib/devices/nvml_linux.go index b60771e7d..e6903bb16 100644 --- a/lib/devices/nvml_linux.go +++ b/lib/devices/nvml_linux.go @@ -47,6 +47,8 @@ func nvmlStructVersion(size uintptr, version uint32) uint32 { } type nvmlFuncs struct { + initialize func() int32 + shutdown func() int32 errorString func(rc int32) string deviceGetHandleByPCIBusID func(busID string, device *uintptr) int32 deviceGetHeterogeneousMode func(device uintptr, mode *nvmlHeterogeneousMode) int32 @@ -60,10 +62,10 @@ var ( nvml *nvmlFuncs ) -// openNVML loads and initializes NVML on first use and keeps it for the life -// of the process. A failure is not kept, so a driver that loads later is +// loadNVML loads the NVML library on first use and keeps it loaded for the +// life of the process. A failure is not kept, so a driver that loads later is // picked up by the next query. -func openNVML() (*nvmlFuncs, error) { +func loadNVML() (*nvmlFuncs, error) { nvmlMu.Lock() defer nvmlMu.Unlock() if nvml != nil { @@ -74,11 +76,11 @@ func openNVML() (*nvmlFuncs, error) { return nil, fmt.Errorf("load %s: %w", nvmlLibrary, err) } var f nvmlFuncs - var initialize func() int32 // purego.RegisterLibFunc panics on a missing symbol, which an older // driver could cause, so each symbol is looked up first. for name, fn := range map[string]any{ - "nvmlInit_v2": &initialize, + "nvmlInit_v2": &f.initialize, + "nvmlShutdown": &f.shutdown, "nvmlErrorString": &f.errorString, "nvmlDeviceGetHandleByPciBusId_v2": &f.deviceGetHandleByPCIBusID, "nvmlDeviceGetVgpuHeterogeneousMode": &f.deviceGetHeterogeneousMode, @@ -93,11 +95,6 @@ func openNVML() (*nvmlFuncs, error) { } purego.RegisterFunc(fn, sym) } - if rc := initialize(); rc != nvmlSuccess { - err := fmt.Errorf("nvmlInit_v2: %s", f.errorString(rc)) - purego.Dlclose(lib) - return nil, err - } nvml = &f return nvml, nil } @@ -107,11 +104,20 @@ func openNVML() (*nvmlFuncs, error) { // nvidia-smi vgpu -c -v. It fails for a GPU not in mixed-size mode, so // such a GPU keeps the sysfs lists. A GPU that fails is left out of the // result and its error joined into the returned one. +// +// NVML is initialized for each query and shut down after it. Initialized, +// it holds the GPU device nodes open, which blocks cycling SR-IOV on a GPU +// and unloading the driver; NVML counts initializations, so concurrent +// queries each hold their own. func nvmlCreatableProfiles(gpus []string) (map[string][]creatableProfile, error) { - f, err := openNVML() + f, err := loadNVML() if err != nil { return nil, err } + if err := f.check("nvmlInit_v2", f.initialize()); err != nil { + return nil, err + } + defer f.shutdown() byGPU := make(map[string][]creatableProfile, len(gpus)) var errs []error for _, gpu := range gpus { diff --git a/lib/devices/vgpu_creatable_linux.go b/lib/devices/vgpu_creatable_linux.go index a119bd7ce..12c162e93 100644 --- a/lib/devices/vgpu_creatable_linux.go +++ b/lib/devices/vgpu_creatable_linux.go @@ -11,9 +11,9 @@ import ( "time" ) -// nvmlTimeout bounds the creatable-placements query, which takes about 20 ms -// on a two-GPU L40S host and stayed under 140 ms with every host core -// saturated. +// nvmlTimeout bounds the creatable-placements query, which takes about 55 ms +// with NVML initialization on a two-GPU L40S host; the query alone stayed +// under 140 ms with every host core saturated. var nvmlTimeout = 5 * time.Second var ( diff --git a/lib/devices/vgpu_creatable_linux_test.go b/lib/devices/vgpu_creatable_linux_test.go index 40de27c51..0f10dc4e0 100644 --- a/lib/devices/vgpu_creatable_linux_test.go +++ b/lib/devices/vgpu_creatable_linux_test.go @@ -85,6 +85,53 @@ func TestNVMLQueryFailsWithoutUsableLibrary(t *testing.T) { assert.Nil(t, nvml) } +func TestNVMLQueryShutsDownNVMLAfterEachQuery(t *testing.T) { + nvmlMu.Lock() + loaded := nvml + nvmlMu.Unlock() + t.Cleanup(func() { + nvmlMu.Lock() + nvml = loaded + nvmlMu.Unlock() + }) + + var initialized, inits int + initRC := int32(nvmlSuccess) + fake := &nvmlFuncs{ + initialize: func() int32 { + inits++ + if initRC == nvmlSuccess { + initialized++ + } + return initRC + }, + shutdown: func() int32 { + initialized-- + return nvmlSuccess + }, + errorString: func(rc int32) string { return fmt.Sprintf("rc %d", rc) }, + deviceGetHandleByPCIBusID: func(string, *uintptr) int32 { + require.Equal(t, 1, initialized, "queried without NVML initialized") + return 999 + }, + } + nvmlMu.Lock() + nvml = fake + nvmlMu.Unlock() + + for range 2 { + _, err := nvmlCreatableProfiles([]string{"0000:82:00.0", "0000:e3:00.0"}) + require.ErrorContains(t, err, "GPU 0000:e3:00.0: nvmlDeviceGetHandleByPciBusId_v2: rc 999") + } + assert.Equal(t, 2, inits, "NVML is initialized for each query") + assert.Zero(t, initialized, "NVML is shut down after each query") + + initRC = 9 + _, err := nvmlCreatableProfiles([]string{"0000:82:00.0"}) + require.ErrorContains(t, err, "nvmlInit_v2: rc 9") + assert.Zero(t, initialized, "a failed initialization is not shut down") +} + func TestVendorVFIOProfileTypesUseCreatablePlacements(t *testing.T) { sysfs := newTestVendorVFIOSysfs(t) sysfs.useCreatablePlacements(t) From ee2cd60cff7049a707f602df008d631270fadb34 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:13:39 +0000 Subject: [PATCH 08/16] Log NVML fallbacks only when they start or change A host on the sysfs lists, such as one with a GPU drifted to equal-size mode, logged a warning on every claim, resources read and monitoring refresh. Keep counting every fallback, but log only when a fallback starts or its error changes, and log once when NVML answers for every GPU again. --- lib/devices/GPU.md | 2 +- lib/devices/vgpu_creatable_linux.go | 29 ++++++++++++++ lib/devices/vgpu_creatable_linux_test.go | 48 ++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 1 deletion(-) diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 3176b2ccb..5559c671d 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -108,7 +108,7 @@ before the hypervisor is signalled. Hypeman reconciles metadata claims once at startup and every minute afterward, skipping hosts without GPUs. A claim whose VMM is confirmed dead is reset before the claim is removed. An ambiguous hypervisor ownership check preserves the claim, logs a warning, and increments `hypeman_instances_vgpu_liveness_uncertain_total`; the create and start cleanup paths preserve and count the same way. mdev hosts also sweep orphaned device-level assignments. Vendor VFIO hosts reset unclaimed dirty VFs at startup, which covers a restart between a claim writing the type and saving the claim, and otherwise repair an unclaimed dirty VF when the allocator next selects it; repair checks for open VFIO handles before resetting `current_vgpu_type`. The allocator prefers VFs that are already clean, and a dirty VF that refuses its reset is skipped in favor of another candidate. -Vendor VFIO placement and `/resources` both read the driver's creatable placements through NVML (`nvmlDeviceGetVgpuTypeCreatablePlacements`, the query behind `nvidia-smi vgpu -c -v`), which list the free placement IDs of every profile each GPU can still create. Hypeman loads `libnvidia-ml.so.1` at runtime on first use, and initializes NVML for each query and shuts it down after, because an initialized NVML holds the GPU device nodes open, which would block the SR-IOV cycle below and driver unloads; a query for every profile on two L40S GPUs takes about 55 ms including the initialization, against about 20 ms with NVML already initialized. A free VF can take any profile its GPU lists there, and a claim only writes a type the driver has just reported a free placement for; if that configure still fails, the VF is reset and the claim fails without trying another GPU. The per-VF `creatable_vgpu_types` lists are not used for this because in mixed-size (heterogeneous) mode they under-report: the driver drops a profile from them once its GPU holds as many vGPUs as that profile's equal-size instance limit (12 for L40S-4Q), even while a placement for it remains. If NVML cannot be loaded or initialized or takes longer than 5 s, placement and `/resources` fall back to those lists for every GPU; if it fails for one GPU, or reports a GPU not in mixed-size mode, only that GPU falls back. Profiles the lists hide are refused on a GPU that has fallen back until NVML recovers. A query that times out is abandoned rather than waited for, since a call stuck in the driver cannot be interrupted; until it returns, queries skip NVML and use the lists directly, so a hung driver costs one timeout rather than one per claim. Each fallback logs a warning and increments `hypeman_resources_gpu_nvml_fallbacks_total` with `reason` `error`, `equal_size` (every GPU NVML failed for is in equal-size mode, as after a driver reload or GPU reset), `timeout` or `stuck`. Admission only checks free VFs, so it does not query NVML. +Vendor VFIO placement and `/resources` both read the driver's creatable placements through NVML (`nvmlDeviceGetVgpuTypeCreatablePlacements`, the query behind `nvidia-smi vgpu -c -v`), which list the free placement IDs of every profile each GPU can still create. Hypeman loads `libnvidia-ml.so.1` at runtime on first use, and initializes NVML for each query and shuts it down after, because an initialized NVML holds the GPU device nodes open, which would block the SR-IOV cycle below and driver unloads; a query for every profile on two L40S GPUs takes about 55 ms including the initialization, against about 20 ms with NVML already initialized. A free VF can take any profile its GPU lists there, and a claim only writes a type the driver has just reported a free placement for; if that configure still fails, the VF is reset and the claim fails without trying another GPU. The per-VF `creatable_vgpu_types` lists are not used for this because in mixed-size (heterogeneous) mode they under-report: the driver drops a profile from them once its GPU holds as many vGPUs as that profile's equal-size instance limit (12 for L40S-4Q), even while a placement for it remains. If NVML cannot be loaded or initialized or takes longer than 5 s, placement and `/resources` fall back to those lists for every GPU; if it fails for one GPU, or reports a GPU not in mixed-size mode, only that GPU falls back. Profiles the lists hide are refused on a GPU that has fallen back until NVML recovers. A query that times out is abandoned rather than waited for, since a call stuck in the driver cannot be interrupted; until it returns, queries skip NVML and use the lists directly, so a hung driver costs one timeout rather than one per claim. Each fallback increments `hypeman_resources_gpu_nvml_fallbacks_total` with `reason` `error`, `equal_size` (every GPU NVML failed for is in equal-size mode, as after a driver reload or GPU reset), `timeout` or `stuck`. A warning is logged when a fallback starts or its error changes, and an info line once NVML answers for every GPU again, rather than on every query. Admission only checks free VFs, so it does not query NVML. ### Hypervisor Support diff --git a/lib/devices/vgpu_creatable_linux.go b/lib/devices/vgpu_creatable_linux.go index 12c162e93..0180d4779 100644 --- a/lib/devices/vgpu_creatable_linux.go +++ b/lib/devices/vgpu_creatable_linux.go @@ -7,6 +7,7 @@ import ( "fmt" "log/slog" "sort" + "sync" "sync/atomic" "time" ) @@ -47,6 +48,16 @@ func NVMLFallbacks() map[string]int64 { return counts } +// nvmlFallbackLogged is the fallback error last logged, or "" while NVML +// answers for every GPU. +var nvmlFallbackLogged struct { + sync.Mutex + err string +} + +// recordNVMLFallback counts every fallback but logs only when the fallback +// starts or its error changes, since a host stuck on the sysfs lists would +// otherwise log on every claim and resources read. func recordNVMLFallback(err error) { reason := "error" switch { @@ -58,9 +69,25 @@ func recordNVMLFallback(err error) { reason = "equal_size" } nvmlFallbacks[reason].Add(1) + nvmlFallbackLogged.Lock() + defer nvmlFallbackLogged.Unlock() + if nvmlFallbackLogged.err == err.Error() { + return + } + nvmlFallbackLogged.err = err.Error() slog.Default().Warn("falling back to sysfs creatable vGPU types", "reason", reason, "error", err) } +func recordNVMLRecovered() { + nvmlFallbackLogged.Lock() + defer nvmlFallbackLogged.Unlock() + if nvmlFallbackLogged.err == "" { + return + } + nvmlFallbackLogged.err = "" + slog.Default().Info("NVML creatable vGPU placements available for every GPU again") +} + // onlyEqualSize reports whether every GPU NVML failed for was left out only // because it is in equal-size mode, which happens when a driver reload or // GPU reset drops mixed-size mode, rather than because NVML failed. @@ -116,6 +143,8 @@ func (s vendorVFIOSysfs) creatableOrSysfs(vfs []VirtualFunction) (map[string][]c creatable, err := s.creatableProfiles(vfs) if err != nil { recordNVMLFallback(err) + } else { + recordNVMLRecovered() } sysfsVFs := make([]VirtualFunction, 0) for _, vf := range vfs { diff --git a/lib/devices/vgpu_creatable_linux_test.go b/lib/devices/vgpu_creatable_linux_test.go index 0f10dc4e0..9128dcf5f 100644 --- a/lib/devices/vgpu_creatable_linux_test.go +++ b/lib/devices/vgpu_creatable_linux_test.go @@ -3,8 +3,11 @@ package devices import ( + "bytes" "errors" "fmt" + "log/slog" + "strings" "sync/atomic" "testing" "time" @@ -132,6 +135,51 @@ func TestNVMLQueryShutsDownNVMLAfterEachQuery(t *testing.T) { assert.Zero(t, initialized, "a failed initialization is not shut down") } +func TestNVMLFallbackLogsOnlyWhenItChanges(t *testing.T) { + var logs bytes.Buffer + previous := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&logs, nil))) + nvmlFallbackLogged.Lock() + nvmlFallbackLogged.err = "" + nvmlFallbackLogged.Unlock() + t.Cleanup(func() { + slog.SetDefault(previous) + nvmlFallbackLogged.Lock() + nvmlFallbackLogged.err = "" + nvmlFallbackLogged.Unlock() + }) + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + var queryErr error + sysfs.creatableFunc = func(gpus []string) (map[string][]creatableProfile, error) { + if queryErr != nil { + return nil, queryErr + } + return map[string][]creatableProfile{"0000:82:00.0": testCreatablePlacements()["0000:82:00.0"]}, nil + } + before := NVMLFallbacks() + + equalSize := errors.Join(fmt.Errorf("GPU 0000:82:00.0: %w", errNVMLEqualSize)) + failed := errors.Join(errors.New("GPU 0000:82:00.0: nvmlDeviceGetSupportedVgpus: Unknown Error")) + for _, err := range []error{equalSize, equalSize, failed, failed, nil, nil, equalSize} { + queryErr = err + _, err := sysfs.profileTypes(vfs) + require.NoError(t, err) + } + + lines := strings.Split(strings.TrimSpace(logs.String()), "\n") + require.Len(t, lines, 4, logs.String()) + assert.Contains(t, lines[0], "reason=equal_size") + assert.Contains(t, lines[1], "reason=error") + assert.Contains(t, lines[2], "available for every GPU again") + assert.Contains(t, lines[3], "reason=equal_size") + after := NVMLFallbacks() + assert.Equal(t, before["equal_size"]+3, after["equal_size"], "every fallback is counted") + assert.Equal(t, before["error"]+2, after["error"]) +} + func TestVendorVFIOProfileTypesUseCreatablePlacements(t *testing.T) { sysfs := newTestVendorVFIOSysfs(t) sysfs.useCreatablePlacements(t) From 5b9d0caf330350686213e1ed7b212d7e3fa01592 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:13:39 +0000 Subject: [PATCH 09/16] Note that the startup VF reset also clears hand-written types --- lib/devices/GPU.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 5559c671d..21b8cae8d 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -106,7 +106,7 @@ itself down and wait for it. Delete normally caps that wait at 2s, but guests with a vGPU or passthrough device get 5s so the guest can unload its GPU driver before the hypervisor is signalled. -Hypeman reconciles metadata claims once at startup and every minute afterward, skipping hosts without GPUs. A claim whose VMM is confirmed dead is reset before the claim is removed. An ambiguous hypervisor ownership check preserves the claim, logs a warning, and increments `hypeman_instances_vgpu_liveness_uncertain_total`; the create and start cleanup paths preserve and count the same way. mdev hosts also sweep orphaned device-level assignments. Vendor VFIO hosts reset unclaimed dirty VFs at startup, which covers a restart between a claim writing the type and saving the claim, and otherwise repair an unclaimed dirty VF when the allocator next selects it; repair checks for open VFIO handles before resetting `current_vgpu_type`. The allocator prefers VFs that are already clean, and a dirty VF that refuses its reset is skipped in favor of another candidate. +Hypeman reconciles metadata claims once at startup and every minute afterward, skipping hosts without GPUs. A claim whose VMM is confirmed dead is reset before the claim is removed. An ambiguous hypervisor ownership check preserves the claim, logs a warning, and increments `hypeman_instances_vgpu_liveness_uncertain_total`; the create and start cleanup paths preserve and count the same way. mdev hosts also sweep orphaned device-level assignments. Vendor VFIO hosts reset unclaimed dirty VFs at startup, which covers a restart between a claim writing the type and saving the claim but also clears a type written to `current_vgpu_type` by hand unless the VF has an open VFIO handle, and otherwise repair an unclaimed dirty VF when the allocator next selects it; repair checks for open VFIO handles before resetting `current_vgpu_type`. The allocator prefers VFs that are already clean, and a dirty VF that refuses its reset is skipped in favor of another candidate. Vendor VFIO placement and `/resources` both read the driver's creatable placements through NVML (`nvmlDeviceGetVgpuTypeCreatablePlacements`, the query behind `nvidia-smi vgpu -c -v`), which list the free placement IDs of every profile each GPU can still create. Hypeman loads `libnvidia-ml.so.1` at runtime on first use, and initializes NVML for each query and shuts it down after, because an initialized NVML holds the GPU device nodes open, which would block the SR-IOV cycle below and driver unloads; a query for every profile on two L40S GPUs takes about 55 ms including the initialization, against about 20 ms with NVML already initialized. A free VF can take any profile its GPU lists there, and a claim only writes a type the driver has just reported a free placement for; if that configure still fails, the VF is reset and the claim fails without trying another GPU. The per-VF `creatable_vgpu_types` lists are not used for this because in mixed-size (heterogeneous) mode they under-report: the driver drops a profile from them once its GPU holds as many vGPUs as that profile's equal-size instance limit (12 for L40S-4Q), even while a placement for it remains. If NVML cannot be loaded or initialized or takes longer than 5 s, placement and `/resources` fall back to those lists for every GPU; if it fails for one GPU, or reports a GPU not in mixed-size mode, only that GPU falls back. Profiles the lists hide are refused on a GPU that has fallen back until NVML recovers. A query that times out is abandoned rather than waited for, since a call stuck in the driver cannot be interrupted; until it returns, queries skip NVML and use the lists directly, so a hung driver costs one timeout rather than one per claim. Each fallback increments `hypeman_resources_gpu_nvml_fallbacks_total` with `reason` `error`, `equal_size` (every GPU NVML failed for is in equal-size mode, as after a driver reload or GPU reset), `timeout` or `stuck`. A warning is logged when a fallback starts or its error changes, and an info line once NVML answers for every GPU again, rather than on every query. Admission only checks free VFs, so it does not query NVML. From 8f421f36d4fd9fbde03869217b187f293821092f Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:37:55 +0000 Subject: [PATCH 10/16] Serialize NVML queries Queries that each initialize and shut down NVML fail when they overlap: on the dev L40S host (driver 580.178.05), 20 concurrent queries returned Invalid Argument, Insufficient Resources and GPU access blocked errors, which would fall placement and /resources back to the sysfs lists whenever a resources read overlaps a claim. Hold a lock from initialization to shutdown. Concurrent queries on one initialization took no less time than serialized ones, so this costs no throughput. --- lib/devices/nvml_linux.go | 10 ++++-- lib/devices/vgpu_creatable_linux_test.go | 44 ++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/lib/devices/nvml_linux.go b/lib/devices/nvml_linux.go index e6903bb16..2bbec12b6 100644 --- a/lib/devices/nvml_linux.go +++ b/lib/devices/nvml_linux.go @@ -60,6 +60,11 @@ type nvmlFuncs struct { var ( nvmlMu sync.Mutex nvml *nvmlFuncs + // nvmlQueryMu serializes queries. NVML counts initializations, but + // queries that initialize and shut it down concurrently fail with errors + // such as Invalid Argument (driver 580.178.05), and concurrent queries on + // one initialization take no less time than serialized ones. + nvmlQueryMu sync.Mutex ) // loadNVML loads the NVML library on first use and keeps it loaded for the @@ -107,13 +112,14 @@ func loadNVML() (*nvmlFuncs, error) { // // NVML is initialized for each query and shut down after it. Initialized, // it holds the GPU device nodes open, which blocks cycling SR-IOV on a GPU -// and unloading the driver; NVML counts initializations, so concurrent -// queries each hold their own. +// and unloading the driver. func nvmlCreatableProfiles(gpus []string) (map[string][]creatableProfile, error) { f, err := loadNVML() if err != nil { return nil, err } + nvmlQueryMu.Lock() + defer nvmlQueryMu.Unlock() if err := f.check("nvmlInit_v2", f.initialize()); err != nil { return nil, err } diff --git a/lib/devices/vgpu_creatable_linux_test.go b/lib/devices/vgpu_creatable_linux_test.go index 9128dcf5f..a87e9deb1 100644 --- a/lib/devices/vgpu_creatable_linux_test.go +++ b/lib/devices/vgpu_creatable_linux_test.go @@ -8,6 +8,7 @@ import ( "fmt" "log/slog" "strings" + "sync" "sync/atomic" "testing" "time" @@ -135,6 +136,49 @@ func TestNVMLQueryShutsDownNVMLAfterEachQuery(t *testing.T) { assert.Zero(t, initialized, "a failed initialization is not shut down") } +func TestNVMLQueriesDoNotOverlap(t *testing.T) { + nvmlMu.Lock() + loaded := nvml + nvmlMu.Unlock() + t.Cleanup(func() { + nvmlMu.Lock() + nvml = loaded + nvmlMu.Unlock() + }) + + var active, overlaps atomic.Int32 + nvmlMu.Lock() + nvml = &nvmlFuncs{ + initialize: func() int32 { + if active.Add(1) > 1 { + overlaps.Add(1) + } + return nvmlSuccess + }, + shutdown: func() int32 { + active.Add(-1) + return nvmlSuccess + }, + errorString: func(rc int32) string { return fmt.Sprintf("rc %d", rc) }, + deviceGetHandleByPCIBusID: func(string, *uintptr) int32 { + time.Sleep(time.Millisecond) + return 999 + }, + } + nvmlMu.Unlock() + + var wg sync.WaitGroup + for range 20 { + wg.Add(1) + go func() { + defer wg.Done() + _, _ = nvmlCreatableProfiles([]string{"0000:82:00.0"}) + }() + } + wg.Wait() + assert.Zero(t, overlaps.Load(), "a query initialized NVML while another held it") +} + func TestNVMLFallbackLogsOnlyWhenItChanges(t *testing.T) { var logs bytes.Buffer previous := slog.Default() From ddacebf053ebdbbb6dee489db2e2125cd028455d Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:45:14 +0000 Subject: [PATCH 11/16] Keep NVML initialized and stop hypeman for the SR-IOV cycle Initializing NVML for each query held no GPU device nodes between queries, but it nearly tripled the query (about 55 ms against 20 ms), so the last of 20 concurrent creates waited about 1.2 s for the allocation lock instead of 0.4 s, and it needed the queries serialized. The device nodes only matter for the SR-IOV cycle that fixes a wedged VF, which already stops the services holding the GPU, so keep NVML initialized as before and add hypeman to the services that procedure stops. Stopping hypeman leaves running VMs alone. This reverts the code of 79357c9 and 8f421f3. --- lib/devices/GPU.md | 10 +-- lib/devices/nvml_linux.go | 34 +++------ lib/devices/vgpu_creatable_linux.go | 6 +- lib/devices/vgpu_creatable_linux_test.go | 91 ------------------------ 4 files changed, 20 insertions(+), 121 deletions(-) diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 21b8cae8d..39a9bc4fd 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -108,7 +108,7 @@ before the hypervisor is signalled. Hypeman reconciles metadata claims once at startup and every minute afterward, skipping hosts without GPUs. A claim whose VMM is confirmed dead is reset before the claim is removed. An ambiguous hypervisor ownership check preserves the claim, logs a warning, and increments `hypeman_instances_vgpu_liveness_uncertain_total`; the create and start cleanup paths preserve and count the same way. mdev hosts also sweep orphaned device-level assignments. Vendor VFIO hosts reset unclaimed dirty VFs at startup, which covers a restart between a claim writing the type and saving the claim but also clears a type written to `current_vgpu_type` by hand unless the VF has an open VFIO handle, and otherwise repair an unclaimed dirty VF when the allocator next selects it; repair checks for open VFIO handles before resetting `current_vgpu_type`. The allocator prefers VFs that are already clean, and a dirty VF that refuses its reset is skipped in favor of another candidate. -Vendor VFIO placement and `/resources` both read the driver's creatable placements through NVML (`nvmlDeviceGetVgpuTypeCreatablePlacements`, the query behind `nvidia-smi vgpu -c -v`), which list the free placement IDs of every profile each GPU can still create. Hypeman loads `libnvidia-ml.so.1` at runtime on first use, and initializes NVML for each query and shuts it down after, because an initialized NVML holds the GPU device nodes open, which would block the SR-IOV cycle below and driver unloads; a query for every profile on two L40S GPUs takes about 55 ms including the initialization, against about 20 ms with NVML already initialized. A free VF can take any profile its GPU lists there, and a claim only writes a type the driver has just reported a free placement for; if that configure still fails, the VF is reset and the claim fails without trying another GPU. The per-VF `creatable_vgpu_types` lists are not used for this because in mixed-size (heterogeneous) mode they under-report: the driver drops a profile from them once its GPU holds as many vGPUs as that profile's equal-size instance limit (12 for L40S-4Q), even while a placement for it remains. If NVML cannot be loaded or initialized or takes longer than 5 s, placement and `/resources` fall back to those lists for every GPU; if it fails for one GPU, or reports a GPU not in mixed-size mode, only that GPU falls back. Profiles the lists hide are refused on a GPU that has fallen back until NVML recovers. A query that times out is abandoned rather than waited for, since a call stuck in the driver cannot be interrupted; until it returns, queries skip NVML and use the lists directly, so a hung driver costs one timeout rather than one per claim. Each fallback increments `hypeman_resources_gpu_nvml_fallbacks_total` with `reason` `error`, `equal_size` (every GPU NVML failed for is in equal-size mode, as after a driver reload or GPU reset), `timeout` or `stuck`. A warning is logged when a fallback starts or its error changes, and an info line once NVML answers for every GPU again, rather than on every query. Admission only checks free VFs, so it does not query NVML. +Vendor VFIO placement and `/resources` both read the driver's creatable placements through NVML (`nvmlDeviceGetVgpuTypeCreatablePlacements`, the query behind `nvidia-smi vgpu -c -v`), which list the free placement IDs of every profile each GPU can still create. Hypeman loads `libnvidia-ml.so.1` at runtime on first use and keeps it initialized, which holds the GPU device nodes open (stop hypeman before the SR-IOV cycle below); a query for every profile on two L40S GPUs takes about 20 ms. A free VF can take any profile its GPU lists there, and a claim only writes a type the driver has just reported a free placement for; if that configure still fails, the VF is reset and the claim fails without trying another GPU. The per-VF `creatable_vgpu_types` lists are not used for this because in mixed-size (heterogeneous) mode they under-report: the driver drops a profile from them once its GPU holds as many vGPUs as that profile's equal-size instance limit (12 for L40S-4Q), even while a placement for it remains. If NVML cannot be loaded or initialized or takes longer than 5 s, placement and `/resources` fall back to those lists for every GPU; if it fails for one GPU, or reports a GPU not in mixed-size mode, only that GPU falls back. Profiles the lists hide are refused on a GPU that has fallen back until NVML recovers. A query that times out is abandoned rather than waited for, since a call stuck in the driver cannot be interrupted; until it returns, queries skip NVML and use the lists directly, so a hung driver costs one timeout rather than one per claim. Each fallback increments `hypeman_resources_gpu_nvml_fallbacks_total` with `reason` `error`, `equal_size` (every GPU NVML failed for is in equal-size mode, as after a driver reload or GPU reset), `timeout` or `stuck`. A warning is logged when a fallback starts or its error changes, and an info line once NVML answers for every GPU again, rather than on every query. Admission only checks free VFs, so it does not query NVML. ### Hypervisor Support @@ -398,7 +398,8 @@ SR-IOV on the parent GPU (this destroys and recreates all of its VFs, so it requires no vGPU assignments on that GPU). Quiescing the services that hold the GPU open is not optional: with `nv-hostengine`/`dcgm-exporter` or `nvidia-persistenced` attached, `sriov-manage -d` fails with `Cannot obtain -unbindLock` on first contact. +unbindLock` on first contact. Hypeman holds the same device nodes through +NVML, so stop it too. Any manual edit to `vf-health.json` needs an immediate hypeman restart: the store loads only at startup, and a failure report landing first re-persists @@ -418,14 +419,15 @@ GPU; once none remain, run the cycle below. ```bash # 1. Quiesce the services holding the GPU (required for the unbind lock). -systemctl stop nvidia-dcgm-exporter nvidia-dcgm nvidia-persistenced +# Hypeman holds it through NVML; stopping it leaves running VMs alone. +systemctl stop hypeman nvidia-dcgm-exporter nvidia-dcgm nvidia-persistenced # 2. Cycle SR-IOV on the parent GPU. /usr/lib/nvidia/sriov-manage -d /usr/lib/nvidia/sriov-manage -e # 3. Restart the quiesced services. -systemctl start nvidia-persistenced nvidia-dcgm nvidia-dcgm-exporter +systemctl start nvidia-persistenced nvidia-dcgm nvidia-dcgm-exporter hypeman ``` After the cycle, remove the card's entries from `vf-health.json`, restart, diff --git a/lib/devices/nvml_linux.go b/lib/devices/nvml_linux.go index 2bbec12b6..b60771e7d 100644 --- a/lib/devices/nvml_linux.go +++ b/lib/devices/nvml_linux.go @@ -47,8 +47,6 @@ func nvmlStructVersion(size uintptr, version uint32) uint32 { } type nvmlFuncs struct { - initialize func() int32 - shutdown func() int32 errorString func(rc int32) string deviceGetHandleByPCIBusID func(busID string, device *uintptr) int32 deviceGetHeterogeneousMode func(device uintptr, mode *nvmlHeterogeneousMode) int32 @@ -60,17 +58,12 @@ type nvmlFuncs struct { var ( nvmlMu sync.Mutex nvml *nvmlFuncs - // nvmlQueryMu serializes queries. NVML counts initializations, but - // queries that initialize and shut it down concurrently fail with errors - // such as Invalid Argument (driver 580.178.05), and concurrent queries on - // one initialization take no less time than serialized ones. - nvmlQueryMu sync.Mutex ) -// loadNVML loads the NVML library on first use and keeps it loaded for the -// life of the process. A failure is not kept, so a driver that loads later is +// openNVML loads and initializes NVML on first use and keeps it for the life +// of the process. A failure is not kept, so a driver that loads later is // picked up by the next query. -func loadNVML() (*nvmlFuncs, error) { +func openNVML() (*nvmlFuncs, error) { nvmlMu.Lock() defer nvmlMu.Unlock() if nvml != nil { @@ -81,11 +74,11 @@ func loadNVML() (*nvmlFuncs, error) { return nil, fmt.Errorf("load %s: %w", nvmlLibrary, err) } var f nvmlFuncs + var initialize func() int32 // purego.RegisterLibFunc panics on a missing symbol, which an older // driver could cause, so each symbol is looked up first. for name, fn := range map[string]any{ - "nvmlInit_v2": &f.initialize, - "nvmlShutdown": &f.shutdown, + "nvmlInit_v2": &initialize, "nvmlErrorString": &f.errorString, "nvmlDeviceGetHandleByPciBusId_v2": &f.deviceGetHandleByPCIBusID, "nvmlDeviceGetVgpuHeterogeneousMode": &f.deviceGetHeterogeneousMode, @@ -100,6 +93,11 @@ func loadNVML() (*nvmlFuncs, error) { } purego.RegisterFunc(fn, sym) } + if rc := initialize(); rc != nvmlSuccess { + err := fmt.Errorf("nvmlInit_v2: %s", f.errorString(rc)) + purego.Dlclose(lib) + return nil, err + } nvml = &f return nvml, nil } @@ -109,21 +107,11 @@ func loadNVML() (*nvmlFuncs, error) { // nvidia-smi vgpu -c -v. It fails for a GPU not in mixed-size mode, so // such a GPU keeps the sysfs lists. A GPU that fails is left out of the // result and its error joined into the returned one. -// -// NVML is initialized for each query and shut down after it. Initialized, -// it holds the GPU device nodes open, which blocks cycling SR-IOV on a GPU -// and unloading the driver. func nvmlCreatableProfiles(gpus []string) (map[string][]creatableProfile, error) { - f, err := loadNVML() + f, err := openNVML() if err != nil { return nil, err } - nvmlQueryMu.Lock() - defer nvmlQueryMu.Unlock() - if err := f.check("nvmlInit_v2", f.initialize()); err != nil { - return nil, err - } - defer f.shutdown() byGPU := make(map[string][]creatableProfile, len(gpus)) var errs []error for _, gpu := range gpus { diff --git a/lib/devices/vgpu_creatable_linux.go b/lib/devices/vgpu_creatable_linux.go index 0180d4779..7377ac12b 100644 --- a/lib/devices/vgpu_creatable_linux.go +++ b/lib/devices/vgpu_creatable_linux.go @@ -12,9 +12,9 @@ import ( "time" ) -// nvmlTimeout bounds the creatable-placements query, which takes about 55 ms -// with NVML initialization on a two-GPU L40S host; the query alone stayed -// under 140 ms with every host core saturated. +// nvmlTimeout bounds the creatable-placements query, which takes about 20 ms +// on a two-GPU L40S host and stayed under 140 ms with every host core +// saturated. var nvmlTimeout = 5 * time.Second var ( diff --git a/lib/devices/vgpu_creatable_linux_test.go b/lib/devices/vgpu_creatable_linux_test.go index a87e9deb1..854728516 100644 --- a/lib/devices/vgpu_creatable_linux_test.go +++ b/lib/devices/vgpu_creatable_linux_test.go @@ -8,7 +8,6 @@ import ( "fmt" "log/slog" "strings" - "sync" "sync/atomic" "testing" "time" @@ -89,96 +88,6 @@ func TestNVMLQueryFailsWithoutUsableLibrary(t *testing.T) { assert.Nil(t, nvml) } -func TestNVMLQueryShutsDownNVMLAfterEachQuery(t *testing.T) { - nvmlMu.Lock() - loaded := nvml - nvmlMu.Unlock() - t.Cleanup(func() { - nvmlMu.Lock() - nvml = loaded - nvmlMu.Unlock() - }) - - var initialized, inits int - initRC := int32(nvmlSuccess) - fake := &nvmlFuncs{ - initialize: func() int32 { - inits++ - if initRC == nvmlSuccess { - initialized++ - } - return initRC - }, - shutdown: func() int32 { - initialized-- - return nvmlSuccess - }, - errorString: func(rc int32) string { return fmt.Sprintf("rc %d", rc) }, - deviceGetHandleByPCIBusID: func(string, *uintptr) int32 { - require.Equal(t, 1, initialized, "queried without NVML initialized") - return 999 - }, - } - nvmlMu.Lock() - nvml = fake - nvmlMu.Unlock() - - for range 2 { - _, err := nvmlCreatableProfiles([]string{"0000:82:00.0", "0000:e3:00.0"}) - require.ErrorContains(t, err, "GPU 0000:e3:00.0: nvmlDeviceGetHandleByPciBusId_v2: rc 999") - } - assert.Equal(t, 2, inits, "NVML is initialized for each query") - assert.Zero(t, initialized, "NVML is shut down after each query") - - initRC = 9 - _, err := nvmlCreatableProfiles([]string{"0000:82:00.0"}) - require.ErrorContains(t, err, "nvmlInit_v2: rc 9") - assert.Zero(t, initialized, "a failed initialization is not shut down") -} - -func TestNVMLQueriesDoNotOverlap(t *testing.T) { - nvmlMu.Lock() - loaded := nvml - nvmlMu.Unlock() - t.Cleanup(func() { - nvmlMu.Lock() - nvml = loaded - nvmlMu.Unlock() - }) - - var active, overlaps atomic.Int32 - nvmlMu.Lock() - nvml = &nvmlFuncs{ - initialize: func() int32 { - if active.Add(1) > 1 { - overlaps.Add(1) - } - return nvmlSuccess - }, - shutdown: func() int32 { - active.Add(-1) - return nvmlSuccess - }, - errorString: func(rc int32) string { return fmt.Sprintf("rc %d", rc) }, - deviceGetHandleByPCIBusID: func(string, *uintptr) int32 { - time.Sleep(time.Millisecond) - return 999 - }, - } - nvmlMu.Unlock() - - var wg sync.WaitGroup - for range 20 { - wg.Add(1) - go func() { - defer wg.Done() - _, _ = nvmlCreatableProfiles([]string{"0000:82:00.0"}) - }() - } - wg.Wait() - assert.Zero(t, overlaps.Load(), "a query initialized NVML while another held it") -} - func TestNVMLFallbackLogsOnlyWhenItChanges(t *testing.T) { var logs bytes.Buffer previous := slog.Default() From 92067f9bb9291b976f131434cf45bf09a57c7d41 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 21 Oct 2026 15:18:52 +0000 Subject: [PATCH 12/16] Start NVML queries one at a time The stuck check only stopped new queries once one had timed out, so callers arriving before that each entered a hung driver. Take a single slot before starting a query and give it back when the driver returns; waiting for the slot counts against the same timeout. --- lib/devices/GPU.md | 2 +- lib/devices/vgpu_creatable_linux.go | 23 +++++++-- lib/devices/vgpu_creatable_linux_test.go | 60 ++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 6 deletions(-) diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 39a9bc4fd..a975a8d4f 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -108,7 +108,7 @@ before the hypervisor is signalled. Hypeman reconciles metadata claims once at startup and every minute afterward, skipping hosts without GPUs. A claim whose VMM is confirmed dead is reset before the claim is removed. An ambiguous hypervisor ownership check preserves the claim, logs a warning, and increments `hypeman_instances_vgpu_liveness_uncertain_total`; the create and start cleanup paths preserve and count the same way. mdev hosts also sweep orphaned device-level assignments. Vendor VFIO hosts reset unclaimed dirty VFs at startup, which covers a restart between a claim writing the type and saving the claim but also clears a type written to `current_vgpu_type` by hand unless the VF has an open VFIO handle, and otherwise repair an unclaimed dirty VF when the allocator next selects it; repair checks for open VFIO handles before resetting `current_vgpu_type`. The allocator prefers VFs that are already clean, and a dirty VF that refuses its reset is skipped in favor of another candidate. -Vendor VFIO placement and `/resources` both read the driver's creatable placements through NVML (`nvmlDeviceGetVgpuTypeCreatablePlacements`, the query behind `nvidia-smi vgpu -c -v`), which list the free placement IDs of every profile each GPU can still create. Hypeman loads `libnvidia-ml.so.1` at runtime on first use and keeps it initialized, which holds the GPU device nodes open (stop hypeman before the SR-IOV cycle below); a query for every profile on two L40S GPUs takes about 20 ms. A free VF can take any profile its GPU lists there, and a claim only writes a type the driver has just reported a free placement for; if that configure still fails, the VF is reset and the claim fails without trying another GPU. The per-VF `creatable_vgpu_types` lists are not used for this because in mixed-size (heterogeneous) mode they under-report: the driver drops a profile from them once its GPU holds as many vGPUs as that profile's equal-size instance limit (12 for L40S-4Q), even while a placement for it remains. If NVML cannot be loaded or initialized or takes longer than 5 s, placement and `/resources` fall back to those lists for every GPU; if it fails for one GPU, or reports a GPU not in mixed-size mode, only that GPU falls back. Profiles the lists hide are refused on a GPU that has fallen back until NVML recovers. A query that times out is abandoned rather than waited for, since a call stuck in the driver cannot be interrupted; until it returns, queries skip NVML and use the lists directly, so a hung driver costs one timeout rather than one per claim. Each fallback increments `hypeman_resources_gpu_nvml_fallbacks_total` with `reason` `error`, `equal_size` (every GPU NVML failed for is in equal-size mode, as after a driver reload or GPU reset), `timeout` or `stuck`. A warning is logged when a fallback starts or its error changes, and an info line once NVML answers for every GPU again, rather than on every query. Admission only checks free VFs, so it does not query NVML. +Vendor VFIO placement and `/resources` both read the driver's creatable placements through NVML (`nvmlDeviceGetVgpuTypeCreatablePlacements`, the query behind `nvidia-smi vgpu -c -v`), which list the free placement IDs of every profile each GPU can still create. Hypeman loads `libnvidia-ml.so.1` at runtime on first use and keeps it initialized, which holds the GPU device nodes open (stop hypeman before the SR-IOV cycle below); a query for every profile on two L40S GPUs takes about 20 ms. A free VF can take any profile its GPU lists there, and a claim only writes a type the driver has just reported a free placement for; if that configure still fails, the VF is reset and the claim fails without trying another GPU. The per-VF `creatable_vgpu_types` lists are not used for this because in mixed-size (heterogeneous) mode they under-report: the driver drops a profile from them once its GPU holds as many vGPUs as that profile's equal-size instance limit (12 for L40S-4Q), even while a placement for it remains. If NVML cannot be loaded or initialized or takes longer than 5 s, placement and `/resources` fall back to those lists for every GPU; if it fails for one GPU, or reports a GPU not in mixed-size mode, only that GPU falls back. Profiles the lists hide are refused on a GPU that has fallen back until NVML recovers. Queries run one at a time and wait for each other within the same 5 s, so callers queued behind a hung query never enter the driver. A query that times out is abandoned rather than waited for, since a call stuck in the driver cannot be interrupted; until it returns, queries skip NVML and use the lists directly, so a hung driver costs one timeout rather than one per claim. Each fallback increments `hypeman_resources_gpu_nvml_fallbacks_total` with `reason` `error`, `equal_size` (every GPU NVML failed for is in equal-size mode, as after a driver reload or GPU reset), `timeout` or `stuck`. A warning is logged when a fallback starts or its error changes, and an info line once NVML answers for every GPU again, rather than on every query. Admission only checks free VFs, so it does not query NVML. ### Hypervisor Support diff --git a/lib/devices/vgpu_creatable_linux.go b/lib/devices/vgpu_creatable_linux.go index 7377ac12b..c03e82e24 100644 --- a/lib/devices/vgpu_creatable_linux.go +++ b/lib/devices/vgpu_creatable_linux.go @@ -155,15 +155,29 @@ func (s vendorVFIOSysfs) creatableOrSysfs(vfs []VirtualFunction) (map[string][]c return creatable, sysfsVFs } +// nvmlSlot admits one NVML query at a time. A query takes it before it +// starts and gives it back only when the driver returns, so callers queued +// behind a hung query never enter the driver themselves. Concurrent queries +// on one initialization take no less time than serialized ones. +var nvmlSlot = make(chan struct{}, 1) + // runBounded returns at nvmlTimeout even if query has not: a call stuck in -// the driver cannot be interrupted. Until such a query returns, later calls -// fail at once instead of each waiting out the timeout, which the claim -// would pay while holding the vGPU allocation lock. +// the driver cannot be interrupted. The timeout covers waiting for +// nvmlSlot. Until a timed-out query returns, later calls fail at once +// instead of each waiting out the timeout, which the claim would pay while +// holding the vGPU allocation lock. func runBounded[T any](query func() (T, error)) (T, error) { var zero T if nvmlStuck.Load() > 0 { return zero, errNVMLStuck } + timer := time.NewTimer(nvmlTimeout) + defer timer.Stop() + select { + case nvmlSlot <- struct{}{}: + case <-timer.C: + return zero, errNVMLStuck + } type result struct { value T err error @@ -171,10 +185,9 @@ func runBounded[T any](query func() (T, error)) (T, error) { done := make(chan result, 1) go func() { value, err := query() + <-nvmlSlot done <- result{value, err} }() - timer := time.NewTimer(nvmlTimeout) - defer timer.Stop() select { case r := <-done: return r.value, r.err diff --git a/lib/devices/vgpu_creatable_linux_test.go b/lib/devices/vgpu_creatable_linux_test.go index 854728516..6aaac0b6f 100644 --- a/lib/devices/vgpu_creatable_linux_test.go +++ b/lib/devices/vgpu_creatable_linux_test.go @@ -8,6 +8,7 @@ import ( "fmt" "log/slog" "strings" + "sync" "sync/atomic" "testing" "time" @@ -222,6 +223,65 @@ func TestVendorVFIOProfileTypesSkipNVMLWhileAQueryIsStuck(t *testing.T) { "queries NVML again once the stuck query returns") } +func TestNVMLQueriesStartOneAtATime(t *testing.T) { + setNVMLTimeout(t, 100*time.Millisecond) + release := make(chan struct{}) + var entered atomic.Int32 + hung := func() (int, error) { + entered.Add(1) + <-release + return 0, nil + } + t.Cleanup(func() { + close(release) + waitForStuckNVML(t) + }) + + errs := make(chan error, 5) + for range 5 { + go func() { + _, err := runBounded(hung) + errs <- err + }() + } + var timeouts, stuck int + for range 5 { + err := <-errs + switch { + case errors.Is(err, errNVMLTimeout): + timeouts++ + case errors.Is(err, errNVMLStuck): + stuck++ + } + } + assert.Equal(t, 1, timeouts) + assert.Equal(t, 4, stuck) + assert.Equal(t, int32(1), entered.Load(), "callers behind a hung query never enter the driver") +} + +func TestNVMLQueriesWaitForEachOther(t *testing.T) { + var active, overlaps atomic.Int32 + query := func() (int, error) { + if active.Add(1) > 1 { + overlaps.Add(1) + } + time.Sleep(5 * time.Millisecond) + active.Add(-1) + return 1, nil + } + + var wg sync.WaitGroup + for range 10 { + wg.Go(func() { + value, err := runBounded(query) + assert.NoError(t, err) + assert.Equal(t, 1, value) + }) + } + wg.Wait() + assert.Zero(t, overlaps.Load()) +} + func TestVendorVFIOListProfilesCountsFreePlacements(t *testing.T) { resetVFHealthStore(t) quarantineVF(t, "0000:82:00.7") From d602c2f7d73ea889daf5296bf1a9a527e2e28542 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 21 Oct 2026 15:18:52 +0000 Subject: [PATCH 13/16] Serve /resources vGPU profile counts from cached placements /resources and the resource metrics queried NVML on every read. Cache the driver's placement data instead: load it on first use, refresh it in the background after every vendor VFIO configure or reset, and expose a refresh for the reconciler. Reads recompute counts against the caller's VFs and quarantine, so a claim or release counts at once, and refreshes coalesce and run one at a time so an older snapshot never replaces a newer one. --- lib/devices/GPU.md | 2 +- lib/devices/mdev_darwin.go | 3 + lib/devices/vendor_vfio_linux.go | 75 ++++++++++-------- lib/devices/vendor_vfio_linux_test.go | 10 +++ lib/devices/vgpu_capacity_linux.go | 101 ++++++++++++++++++++++++ lib/devices/vgpu_capacity_linux_test.go | 97 +++++++++++++++++++++++ lib/devices/vgpu_linux.go | 11 ++- 7 files changed, 260 insertions(+), 39 deletions(-) create mode 100644 lib/devices/vgpu_capacity_linux.go create mode 100644 lib/devices/vgpu_capacity_linux_test.go diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index a975a8d4f..18463d69f 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -108,7 +108,7 @@ before the hypervisor is signalled. Hypeman reconciles metadata claims once at startup and every minute afterward, skipping hosts without GPUs. A claim whose VMM is confirmed dead is reset before the claim is removed. An ambiguous hypervisor ownership check preserves the claim, logs a warning, and increments `hypeman_instances_vgpu_liveness_uncertain_total`; the create and start cleanup paths preserve and count the same way. mdev hosts also sweep orphaned device-level assignments. Vendor VFIO hosts reset unclaimed dirty VFs at startup, which covers a restart between a claim writing the type and saving the claim but also clears a type written to `current_vgpu_type` by hand unless the VF has an open VFIO handle, and otherwise repair an unclaimed dirty VF when the allocator next selects it; repair checks for open VFIO handles before resetting `current_vgpu_type`. The allocator prefers VFs that are already clean, and a dirty VF that refuses its reset is skipped in favor of another candidate. -Vendor VFIO placement and `/resources` both read the driver's creatable placements through NVML (`nvmlDeviceGetVgpuTypeCreatablePlacements`, the query behind `nvidia-smi vgpu -c -v`), which list the free placement IDs of every profile each GPU can still create. Hypeman loads `libnvidia-ml.so.1` at runtime on first use and keeps it initialized, which holds the GPU device nodes open (stop hypeman before the SR-IOV cycle below); a query for every profile on two L40S GPUs takes about 20 ms. A free VF can take any profile its GPU lists there, and a claim only writes a type the driver has just reported a free placement for; if that configure still fails, the VF is reset and the claim fails without trying another GPU. The per-VF `creatable_vgpu_types` lists are not used for this because in mixed-size (heterogeneous) mode they under-report: the driver drops a profile from them once its GPU holds as many vGPUs as that profile's equal-size instance limit (12 for L40S-4Q), even while a placement for it remains. If NVML cannot be loaded or initialized or takes longer than 5 s, placement and `/resources` fall back to those lists for every GPU; if it fails for one GPU, or reports a GPU not in mixed-size mode, only that GPU falls back. Profiles the lists hide are refused on a GPU that has fallen back until NVML recovers. Queries run one at a time and wait for each other within the same 5 s, so callers queued behind a hung query never enter the driver. A query that times out is abandoned rather than waited for, since a call stuck in the driver cannot be interrupted; until it returns, queries skip NVML and use the lists directly, so a hung driver costs one timeout rather than one per claim. Each fallback increments `hypeman_resources_gpu_nvml_fallbacks_total` with `reason` `error`, `equal_size` (every GPU NVML failed for is in equal-size mode, as after a driver reload or GPU reset), `timeout` or `stuck`. A warning is logged when a fallback starts or its error changes, and an info line once NVML answers for every GPU again, rather than on every query. Admission only checks free VFs, so it does not query NVML. +Vendor VFIO placement and `/resources` both read the driver's creatable placements through NVML (`nvmlDeviceGetVgpuTypeCreatablePlacements`, the query behind `nvidia-smi vgpu -c -v`), which list the free placement IDs of every profile each GPU can still create. Hypeman loads `libnvidia-ml.so.1` at runtime on first use and keeps it initialized, which holds the GPU device nodes open (stop hypeman before the SR-IOV cycle below); a query for every profile on two L40S GPUs takes about 20 ms. A free VF can take any profile its GPU lists there, and a claim only writes a type the driver has just reported a free placement for; if that configure still fails, the VF is reset and the claim fails without trying another GPU. The per-VF `creatable_vgpu_types` lists are not used for this because in mixed-size (heterogeneous) mode they under-report: the driver drops a profile from them once its GPU holds as many vGPUs as that profile's equal-size instance limit (12 for L40S-4Q), even while a placement for it remains. If NVML cannot be loaded or initialized or takes longer than 5 s, placement and `/resources` fall back to those lists for every GPU; if it fails for one GPU, or reports a GPU not in mixed-size mode, only that GPU falls back. Profiles the lists hide are refused on a GPU that has fallen back until NVML recovers. Queries run one at a time and wait for each other within the same 5 s, so callers queued behind a hung query never enter the driver. A query that times out is abandoned rather than waited for, since a call stuck in the driver cannot be interrupted; until it returns, queries skip NVML and use the lists directly, so a hung driver costs one timeout rather than one per claim. Each fallback increments `hypeman_resources_gpu_nvml_fallbacks_total` with `reason` `error`, `equal_size` (every GPU NVML failed for is in equal-size mode, as after a driver reload or GPU reset), `timeout` or `stuck`. A warning is logged when a fallback starts or its error changes, and an info line once NVML answers for every GPU again, rather than on every query. Admission only checks free VFs, so it does not query NVML. `/resources` does not query it either: it counts profiles from cached placements, loaded at startup, refreshed in the background after every vendor VFIO configure or reset, and refreshed again on every reconcile pass. Free and quarantined VFs are read fresh on every request, so a claim or release changes the counts at once; the placements it frees or takes on a mixed-size GPU show once the background refresh lands, about 20 ms later. ### Hypervisor Support diff --git a/lib/devices/mdev_darwin.go b/lib/devices/mdev_darwin.go index a5d76c1a0..60f401725 100644 --- a/lib/devices/mdev_darwin.go +++ b/lib/devices/mdev_darwin.go @@ -40,6 +40,9 @@ func ListVendorVFIOProfileTypes(vfs []VirtualFunction) (map[string][]VGPUProfile return nil, ErrVGPUNotSupportedOnMacOS } +// RefreshVGPUCapacity does nothing on macOS. +func RefreshVGPUCapacity() {} + // NVMLFallbacks returns nil on macOS. func NVMLFallbacks() map[string]int64 { return nil diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 170bcb846..4e0e2c774 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -91,54 +91,59 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { return vfs, nil } -// listProfiles reports how many more vGPUs of each profile the host can -// take if only that profile is added: per GPU, the smaller of its free -// placements for the profile and its free, non-quarantined VFs. Counts for -// different profiles are not additive, because their placements overlap. -// GPUs NVML cannot answer for are counted from the sysfs lists instead. -func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction, quarantined map[string]struct{}) ([]GPUProfile, error) { +// creatableSnapshot is what the driver reports vendor VFIO profile counts +// are computed from: NVML's creatable placements for each GPU it answered +// for, and the sysfs creatable lists of the VFs on every other GPU. +type creatableSnapshot struct { + byGPU map[string][]creatableProfile + sysfsByVF map[string][]VGPUProfileType +} + +func (s vendorVFIOSysfs) creatableSnapshot(vfs []VirtualFunction) (creatableSnapshot, error) { creatable, sysfsVFs := s.creatableOrSysfs(vfs) - profilesByType, available, err := s.sysfsAvailability(sysfsVFs, quarantined) + sysfsByVF, err := s.sysfsProfileTypes(sysfsVFs) if err != nil { - return nil, err + return creatableSnapshot{}, err } + return creatableSnapshot{byGPU: creatable, sysfsByVF: sysfsByVF}, nil +} + +// profiles counts, per GPU, the smaller of its free placements for each +// profile and its free, non-quarantined VFs. Counts for different profiles +// are not additive, because their placements overlap. On a GPU counted from +// the sysfs lists, each free, non-quarantined VF advertising a type is one +// creatable instance, matching the driver-reported units that mdev sums +// through available_instances; that is best-effort, because creating on one +// VF may revoke the type from siblings that share its GPU framebuffer. vfs +// may be newer than the snapshot, so a VF claimed or released since it was +// taken already counts as such. +func (c creatableSnapshot) profiles(vfs []VirtualFunction, quarantined map[string]struct{}) []GPUProfile { + profilesByType := make(map[string]VGPUProfileType) + available := make(map[string]int) freeVFs := make(map[string]int) for _, vf := range vfs { - if _, bad := quarantined[vf.PCIAddress]; !vf.Allocated && !bad { + _, bad := quarantined[vf.PCIAddress] + free := !vf.Allocated && !bad + if free { freeVFs[vf.ParentGPU]++ } + if _, ok := c.byGPU[vf.ParentGPU]; ok { + continue + } + for _, profile := range c.sysfsByVF[vf.PCIAddress] { + profilesByType[profile.TypeName] = profile + if free { + available[profile.TypeName]++ + } + } } - for gpu, profiles := range creatable { + for gpu, profiles := range c.byGPU { for _, profile := range profiles { profilesByType[profile.TypeName] = profile.VGPUProfileType available[profile.TypeName] += min(len(profile.placements), freeVFs[gpu]) } } - return gpuProfiles(profilesByType, available), nil -} - -// sysfsAvailability counts each free, non-quarantined VF advertising a type -// as one creatable instance, matching the driver-reported units that mdev -// sums through available_instances. This is a best-effort snapshot because -// creating on one VF may revoke the type from siblings that share its GPU -// framebuffer. -func (s vendorVFIOSysfs) sysfsAvailability(vfs []VirtualFunction, quarantined map[string]struct{}) (map[string]VGPUProfileType, map[string]int, error) { - profilesByType := make(map[string]VGPUProfileType) - creatableVFs := make(map[string]int) - profilesByVF, err := s.sysfsProfileTypes(vfs) - if err != nil { - return nil, nil, err - } - for _, vf := range vfs { - _, bad := quarantined[vf.PCIAddress] - for _, profile := range profilesByVF[vf.PCIAddress] { - profilesByType[profile.TypeName] = profile - if !vf.Allocated && !bad { - creatableVFs[profile.TypeName]++ - } - } - } - return profilesByType, creatableVFs, nil + return gpuProfiles(profilesByType, available) } func gpuProfiles(profilesByType map[string]VGPUProfileType, available map[string]int) []GPUProfile { diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index e57769cb9..a03df7a9b 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -160,6 +160,16 @@ func newTestVendorVFIOSysfs(t *testing.T) testVendorVFIOSysfs { }} } +// listProfiles counts profiles from a fresh driver query, as a capacity +// cache load does. +func (s testVendorVFIOSysfs) listProfiles(vfs []VirtualFunction, quarantined map[string]struct{}) ([]GPUProfile, error) { + snapshot, err := s.creatableSnapshot(vfs) + if err != nil { + return nil, err + } + return snapshot.profiles(vfs, quarantined), nil +} + func (s testVendorVFIOSysfs) addVF(t *testing.T, parent, address, vfioID, currentType, creatableTypes string) { t.Helper() parentPath := filepath.Join(s.pciDevicesPath, parent) diff --git a/lib/devices/vgpu_capacity_linux.go b/lib/devices/vgpu_capacity_linux.go new file mode 100644 index 000000000..67f0a973a --- /dev/null +++ b/lib/devices/vgpu_capacity_linux.go @@ -0,0 +1,101 @@ +//go:build linux + +package devices + +import ( + "fmt" + "sync" + "sync/atomic" +) + +// vendorVFIOCapacity holds the driver's placement data for vendor VFIO +// profile counts, so /resources reads memory rather than querying NVML. It +// is loaded on first use or by the vGPU reconciler at startup, refreshed in +// the background after every vendor VFIO configure or reset, and refreshed +// again on every reconcile pass. Counts are recomputed on each read against +// the caller's VFs and quarantine, so a claimed or released VF counts at +// once; placements, which only the driver can recompute for mixed sizes, +// catch up when the background refresh lands. +var vendorVFIOCapacity = newCapacityCache(func() (creatableSnapshot, error) { + return hostVendorVFIO.loadCapacity() +}) + +type capacityCache struct { + load func() (creatableSnapshot, error) + + // refreshMu serializes loads, so an older snapshot never replaces a + // newer one. + refreshMu sync.Mutex + // queued is set while a background refresh waits to start its load. A + // change made before the load starts is seen by it, so later requests + // join it. + queued atomic.Bool + + mu sync.RWMutex + loaded bool + snapshot creatableSnapshot + err error +} + +func newCapacityCache(load func() (creatableSnapshot, error)) *capacityCache { + return &capacityCache{load: load} +} + +func (s vendorVFIOSysfs) loadCapacity() (creatableSnapshot, error) { + vfs, err := s.discoverVFs() + if err != nil { + return creatableSnapshot{}, fmt.Errorf("discover vendor VFIO VFs: %w", err) + } + return s.creatableSnapshot(vfs) +} + +// profiles returns the profile counts for vfs from the cached placements, +// loading them first if nothing has been loaded yet. +func (c *capacityCache) profiles(vfs []VirtualFunction, quarantined map[string]struct{}) ([]GPUProfile, error) { + c.mu.RLock() + loaded := c.loaded + c.mu.RUnlock() + if !loaded { + c.refresh() + } + c.mu.RLock() + defer c.mu.RUnlock() + if c.err != nil { + return nil, c.err + } + return c.snapshot.profiles(vfs, quarantined), nil +} + +func (c *capacityCache) refresh() { + c.refreshMu.Lock() + defer c.refreshMu.Unlock() + c.loadLocked() +} + +// refreshAsync reloads the placements in the background. +func (c *capacityCache) refreshAsync() { + if !c.queued.CompareAndSwap(false, true) { + return + } + go func() { + c.refreshMu.Lock() + defer c.refreshMu.Unlock() + c.queued.Store(false) + c.loadLocked() + }() +} + +func (c *capacityCache) loadLocked() { + snapshot, err := c.load() + c.mu.Lock() + defer c.mu.Unlock() + c.snapshot, c.err, c.loaded = snapshot, err, true +} + +// RefreshVGPUCapacity reloads the vendor VFIO placements /resources counts +// from. It does nothing on hosts without vendor VFIO vGPUs. +func RefreshVGPUCapacity() { + if framework, _, err := DiscoverVGPU(); err == nil && framework == VGPUFrameworkVendorVFIO { + vendorVFIOCapacity.refresh() + } +} diff --git a/lib/devices/vgpu_capacity_linux_test.go b/lib/devices/vgpu_capacity_linux_test.go new file mode 100644 index 000000000..d058d1f54 --- /dev/null +++ b/lib/devices/vgpu_capacity_linux_test.go @@ -0,0 +1,97 @@ +//go:build linux + +package devices + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func testCapacityVFs() []VirtualFunction { + return []VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:e3:00.4", ParentGPU: "0000:e3:00.0"}, + } +} + +func TestCapacityCacheReadsDoNotQueryTheDriver(t *testing.T) { + var loads atomic.Int32 + cache := newCapacityCache(func() (creatableSnapshot, error) { + loads.Add(1) + return creatableSnapshot{byGPU: testCreatablePlacements()}, nil + }) + vfs := testCapacityVFs() + + profiles, err := cache.profiles(vfs, nil) + require.NoError(t, err) + // 4Q: min(2 placements, 2 VFs) on GPU 82 + min(5 placements, 1 VF) on e3. + assert.Equal(t, 3, profileAvailability(profiles, "NVIDIA L40S-4Q")) + + vfs[0].Allocated = true + profiles, err = cache.profiles(vfs, map[string]struct{}{"0000:e3:00.4": {}}) + require.NoError(t, err) + assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-4Q"), + "a claimed or quarantined VF counts before the placements are reloaded") + assert.Equal(t, int32(1), loads.Load(), "only the first read loads") +} + +func TestCapacityCacheBackgroundRefreshesCoalesce(t *testing.T) { + started := make(chan struct{}, 10) + release := make(chan struct{}) + var loads atomic.Int32 + cache := newCapacityCache(func() (creatableSnapshot, error) { + n := loads.Add(1) + started <- struct{}{} + <-release + if n == 1 { + return creatableSnapshot{}, nil + } + return creatableSnapshot{byGPU: testCreatablePlacements()}, nil + }) + + cache.refreshAsync() + <-started + // A change made while a load runs needs one more load, however many + // refreshes it requests. + for range 5 { + cache.refreshAsync() + } + close(release) + require.Eventually(t, func() bool { return loads.Load() == 2 }, 5*time.Second, time.Millisecond) + require.Eventually(t, func() bool { + profiles, err := cache.profiles(testCapacityVFs(), nil) + return err == nil && profileAvailability(profiles, "NVIDIA L40S-4Q") == 3 + }, 5*time.Second, time.Millisecond, "the newer load replaces the older one") + time.Sleep(20 * time.Millisecond) + assert.Equal(t, int32(2), loads.Load()) +} + +func TestVendorVFIOConfigureAndDestroyRefreshCapacity(t *testing.T) { + resetVFHealthStore(t) + sysfs := newTestVendorVFIOSysfs(t) + sysfs.useCreatablePlacements(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + previousSysfs, previousCache := hostVendorVFIO, vendorVFIOCapacity + hostVendorVFIO = sysfs.vendorVFIOSysfs + var loads atomic.Int32 + vendorVFIOCapacity = newCapacityCache(func() (creatableSnapshot, error) { + snapshot, err := hostVendorVFIO.loadCapacity() + loads.Add(1) + return snapshot, err + }) + t.Cleanup(func() { hostVendorVFIO, vendorVFIOCapacity = previousSysfs, previousCache }) + + require.NoError(t, ConfigureVGPU(context.Background(), "0000:82:00.4", "1150")) + require.Eventually(t, func() bool { return loads.Load() == 1 }, 5*time.Second, time.Millisecond) + require.NoError(t, DestroyVGPU(context.Background(), VGPUAssignment{ + Framework: VGPUFrameworkVendorVFIO, + DevicePath: GetDeviceSysfsPath("0000:82:00.4"), + })) + require.Eventually(t, func() bool { return loads.Load() == 2 }, 5*time.Second, time.Millisecond) +} diff --git a/lib/devices/vgpu_linux.go b/lib/devices/vgpu_linux.go index d53659efd..b5f69511b 100644 --- a/lib/devices/vgpu_linux.go +++ b/lib/devices/vgpu_linux.go @@ -46,13 +46,14 @@ func ListGPUProfiles() ([]GPUProfile, error) { } // ListGPUProfilesWithVFs returns available profiles for discovered VFs. -// Quarantined VFs are excluded from vendor VFIO counts. +// Quarantined VFs are excluded from vendor VFIO counts, which come from the +// cached placements rather than a driver query. func ListGPUProfilesWithVFs(framework VGPUFramework, vfs []VirtualFunction, quarantined map[string]struct{}) ([]GPUProfile, error) { switch framework { case VGPUFrameworkMdev: return listMdevGPUProfilesWithVFs(vfs) case VGPUFrameworkVendorVFIO: - return hostVendorVFIO.listProfiles(vfs, quarantined) + return vendorVFIOCapacity.profiles(vfs, quarantined) default: return nil, nil } @@ -88,8 +89,11 @@ func ListVendorVFIOProfileTypes(vfs []VirtualFunction) (map[string][]VGPUProfile return hostVendorVFIO.profileTypes(vfs) } -// ConfigureVGPU idempotently configures a claimed vendor VFIO VF. +// ConfigureVGPU idempotently configures a claimed vendor VFIO VF. A failed +// configure may still have written the type, so the cached placements are +// refreshed either way. func ConfigureVGPU(ctx context.Context, vfAddress, profileType string) error { + defer vendorVFIOCapacity.refreshAsync() return hostVendorVFIO.configure(ctx, vfAddress, profileType) } @@ -110,6 +114,7 @@ func DestroyVGPU(ctx context.Context, assignment VGPUAssignment) error { } return DestroyMdev(ctx, mdevUUID) case VGPUFrameworkVendorVFIO: + defer vendorVFIOCapacity.refreshAsync() return hostVendorVFIO.destroy(ctx, filepath.Base(assignment.DevicePath), assignment.InstanceID) case VGPUFrameworkNone: return nil From 0b41f9f387d0b129db69942f0f0d1747b1adda35 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 21 Oct 2026 15:18:52 +0000 Subject: [PATCH 14/16] Reset unclaimed VFs and refresh vGPU capacity on every reconcile pass The unclaimed VF reset ran once at startup, so a reset refused then, or an orphan left later by a failed claim whose reset also failed, held its placement until the allocator next selected that VF, which it may never do if placement skips the host. Run it on every pass, count refusals under the reset_unclaimed_vfs reconcile stage, and reload the cached vGPU placements at the end of each pass. --- lib/devices/GPU.md | 4 +- lib/instances/manager.go | 1 + lib/instances/metrics.go | 7 ++-- lib/instances/vgpu.go | 11 +++--- lib/instances/vgpu_reconcile.go | 36 +++++++++++------ lib/instances/vgpu_reconcile_linux_test.go | 45 ++++++++++++++++++++++ 6 files changed, 83 insertions(+), 21 deletions(-) diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 18463d69f..33e54354b 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -94,7 +94,7 @@ On an mdev host, the response also includes the assigned mdev UUID: ### Ephemeral vGPU Lifecycle -vGPU assignments are created on instance start and released on stop or delete. Hypeman creates/removes an mdev on mdev hosts. On vendor VFIO hosts, it selects a VF on a GPU with a free placement for the profile and writes the profile ID to `current_vgpu_type` under the allocation lock, so the placement is taken before the next claim checks for one, then reserves the VF by persisting its path in instance metadata. Metadata is the authoritative VF claim; a configured VF without one is dirty and is reset at startup or by the allocator. +vGPU assignments are created on instance start and released on stop or delete. Hypeman creates/removes an mdev on mdev hosts. On vendor VFIO hosts, it selects a VF on a GPU with a free placement for the profile and writes the profile ID to `current_vgpu_type` under the allocation lock, so the placement is taken before the next claim checks for one, then reserves the VF by persisting its path in instance metadata. Metadata is the authoritative VF claim; a configured VF without one is dirty and is reset by the reconciler or the allocator. ``` Instance Create → Configure profile → Persist VF claim → Attach VF to VM → Instance Running @@ -106,7 +106,7 @@ itself down and wait for it. Delete normally caps that wait at 2s, but guests with a vGPU or passthrough device get 5s so the guest can unload its GPU driver before the hypervisor is signalled. -Hypeman reconciles metadata claims once at startup and every minute afterward, skipping hosts without GPUs. A claim whose VMM is confirmed dead is reset before the claim is removed. An ambiguous hypervisor ownership check preserves the claim, logs a warning, and increments `hypeman_instances_vgpu_liveness_uncertain_total`; the create and start cleanup paths preserve and count the same way. mdev hosts also sweep orphaned device-level assignments. Vendor VFIO hosts reset unclaimed dirty VFs at startup, which covers a restart between a claim writing the type and saving the claim but also clears a type written to `current_vgpu_type` by hand unless the VF has an open VFIO handle, and otherwise repair an unclaimed dirty VF when the allocator next selects it; repair checks for open VFIO handles before resetting `current_vgpu_type`. The allocator prefers VFs that are already clean, and a dirty VF that refuses its reset is skipped in favor of another candidate. +Hypeman reconciles metadata claims once at startup and every minute afterward, skipping hosts without GPUs. A claim whose VMM is confirmed dead is reset before the claim is removed. An ambiguous hypervisor ownership check preserves the claim, logs a warning, and increments `hypeman_instances_vgpu_liveness_uncertain_total`; the create and start cleanup paths preserve and count the same way. mdev hosts also sweep orphaned device-level assignments. Vendor VFIO hosts reset unclaimed dirty VFs on every reconcile pass, which covers a restart between a claim writing the type and saving the claim, or a failed claim whose reset also failed, but also clears a type written to `current_vgpu_type` by hand unless the VF has an open VFIO handle. A reset that is refused is retried on the next pass and increments `hypeman_instances_vgpu_reconcile_failures_total{stage="reset_unclaimed_vfs"}`. The allocator also repairs an unclaimed dirty VF when it next selects it; repair checks for open VFIO handles before resetting `current_vgpu_type`. The allocator prefers VFs that are already clean, and a dirty VF that refuses its reset is skipped in favor of another candidate. Vendor VFIO placement and `/resources` both read the driver's creatable placements through NVML (`nvmlDeviceGetVgpuTypeCreatablePlacements`, the query behind `nvidia-smi vgpu -c -v`), which list the free placement IDs of every profile each GPU can still create. Hypeman loads `libnvidia-ml.so.1` at runtime on first use and keeps it initialized, which holds the GPU device nodes open (stop hypeman before the SR-IOV cycle below); a query for every profile on two L40S GPUs takes about 20 ms. A free VF can take any profile its GPU lists there, and a claim only writes a type the driver has just reported a free placement for; if that configure still fails, the VF is reset and the claim fails without trying another GPU. The per-VF `creatable_vgpu_types` lists are not used for this because in mixed-size (heterogeneous) mode they under-report: the driver drops a profile from them once its GPU holds as many vGPUs as that profile's equal-size instance limit (12 for L40S-4Q), even while a placement for it remains. If NVML cannot be loaded or initialized or takes longer than 5 s, placement and `/resources` fall back to those lists for every GPU; if it fails for one GPU, or reports a GPU not in mixed-size mode, only that GPU falls back. Profiles the lists hide are refused on a GPU that has fallen back until NVML recovers. Queries run one at a time and wait for each other within the same 5 s, so callers queued behind a hung query never enter the driver. A query that times out is abandoned rather than waited for, since a call stuck in the driver cannot be interrupted; until it returns, queries skip NVML and use the lists directly, so a hung driver costs one timeout rather than one per claim. Each fallback increments `hypeman_resources_gpu_nvml_fallbacks_total` with `reason` `error`, `equal_size` (every GPU NVML failed for is in equal-size mode, as after a driver reload or GPU reset), `timeout` or `stuck`. A warning is logged when a fallback starts or its error changes, and an info line once NVML answers for every GPU again, rather than on every query. Admission only checks free VFs, so it does not query NVML. `/resources` does not query it either: it counts profiles from cached placements, loaded at startup, refreshed in the background after every vendor VFIO configure or reset, and refreshed again on every reconcile pass. Free and quarantined VFs are read fresh on every request, so a claim or release changes the counts at once; the placements it frees or takes on a mixed-size GPU show once the background refresh lands, about 20 ms later. diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 0b39814f0..63acb682a 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -194,6 +194,7 @@ type manager struct { pickVFIndex func(n int) int destroyVGPU func(context.Context, devices.VGPUAssignment) error reconcileVGPUDevices func(context.Context, map[string]struct{}) error + refreshVGPUCapacity func() vgpuAllocationMu sync.Mutex deleteSnapshotFn func(context.Context, string) error ttlReaperDeleteTimeout time.Duration diff --git a/lib/instances/metrics.go b/lib/instances/metrics.go index 8d86b6cc7..02bd0fc8d 100644 --- a/lib/instances/metrics.go +++ b/lib/instances/metrics.go @@ -77,8 +77,9 @@ const lifecycleEventDropReasonBufferFull lifecycleEventDropReason = "buffer_full type vgpuReconcileStage string const ( - vgpuReconcileStageListInstances vgpuReconcileStage = "list_instances" - vgpuReconcileStageReconcileDevices vgpuReconcileStage = "reconcile_devices" + vgpuReconcileStageListInstances vgpuReconcileStage = "list_instances" + vgpuReconcileStageReconcileDevices vgpuReconcileStage = "reconcile_devices" + vgpuReconcileStageResetUnclaimedVFs vgpuReconcileStage = "reset_unclaimed_vfs" ) // Metrics holds the metrics instruments for instance operations. @@ -283,7 +284,7 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M vgpuReconcileFailuresTotal, err := meter.Int64Counter( "hypeman_instances_vgpu_reconcile_failures_total", - metric.WithDescription("Total number of vGPU reconcile pass stages that failed, leaving stale assignments or device leftovers allocated while /resources still advertises the capacity"), + metric.WithDescription("Total number of vGPU reconcile pass stages that failed, leaving stale assignments or device leftovers allocated"), ) if err != nil { return nil, err diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index d1a0ae70b..3d4c4b3f3 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -70,7 +70,7 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str // A dirty unclaimed VF consumes framebuffer, which can make the // requested profile vanish from every creatable list before the // selector's repair path can resolve it. Reset such VFs and retry. - if !m.resetDirtyUnclaimedVFs(ctx, vfs, allMetadata) { + if reset, _ := m.resetDirtyUnclaimedVFs(ctx, vfs, allMetadata); !reset { return nil, err } if _, vfs, err = m.discoverVGPUDevices(); err != nil { @@ -149,8 +149,9 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str // resetDirtyUnclaimedVFs resets allocated VFs that no instance claims, // restoring the framebuffer their leftover type consumes. The reset is // gated on the open-VFIO-handle in-use check inside destroy. Reports -// whether any VF was reset. Callers must hold vgpuAllocationMu. -func (m *manager) resetDirtyUnclaimedVFs(ctx context.Context, vfs []devices.VirtualFunction, allMetadata []StoredMetadata) bool { +// whether any VF was reset and whether any refused. Callers must hold +// vgpuAllocationMu. +func (m *manager) resetDirtyUnclaimedVFs(ctx context.Context, vfs []devices.VirtualFunction, allMetadata []StoredMetadata) (reset, failed bool) { claimed := make(map[string]struct{}, len(allMetadata)) for i := range allMetadata { if path := allMetadata[i].GPUDevicePath; path != "" { @@ -158,7 +159,6 @@ func (m *manager) resetDirtyUnclaimedVFs(ctx context.Context, vfs []devices.Virt } } log := logger.FromContext(ctx) - reset := false for _, vf := range vfs { if !vf.Allocated { continue @@ -172,12 +172,13 @@ func (m *manager) resetDirtyUnclaimedVFs(ctx context.Context, vfs []devices.Virt } if err := m.destroyVGPUAssignment(ctx, assignment); err != nil { log.WarnContext(ctx, "failed to reset dirty vGPU VF", "vf", vf.PCIAddress, "error", err) + failed = true continue } log.InfoContext(ctx, "reset dirty unclaimed vGPU VF", "vf", vf.PCIAddress) reset = true } - return reset + return reset, failed } func (m *manager) quarantinedVFAddresses() (map[string]struct{}, error) { diff --git a/lib/instances/vgpu_reconcile.go b/lib/instances/vgpu_reconcile.go index 63b234cd5..d756401c3 100644 --- a/lib/instances/vgpu_reconcile.go +++ b/lib/instances/vgpu_reconcile.go @@ -11,10 +11,9 @@ import ( const defaultVGPUReconcileInterval = time.Minute -// StartVGPUReconciler resets unclaimed vendor VFIO VFs, runs one reconcile -// pass and then keeps reconciling periodically until ctx is cancelled. Hosts -// without a vGPU framework skip reconciliation entirely. A discovery failure -// starts the reconciler anyway. +// StartVGPUReconciler runs one reconcile pass and then keeps reconciling +// periodically until ctx is cancelled. Hosts without a vGPU framework skip +// reconciliation entirely. A discovery failure starts the reconciler anyway. func (m *manager) StartVGPUReconciler(ctx context.Context) { framework, _, err := m.discoverVGPUDevices() if err == nil && framework == devices.VGPUFrameworkNone { @@ -23,7 +22,6 @@ func (m *manager) StartVGPUReconciler(ctx context.Context) { if err != nil { logger.FromContext(ctx).WarnContext(ctx, "failed to discover vGPU framework; starting vGPU reconciler anyway", "error", err) } - m.resetUnclaimedVendorVFs(ctx) m.ReconcileVGPUs(ctx) m.vgpuReconcileOnce.Do(func() { interval := m.vgpuReconcileInterval @@ -45,10 +43,20 @@ func (m *manager) StartVGPUReconciler(ctx context.Context) { }) } -// ReconcileVGPUs releases metadata claims whose VMM is confirmed dead. The -// device-level pass remains for mdevs; vendor VFIO leftovers are repaired at -// startup and when the allocator claims that VF again. +// ReconcileVGPUs releases metadata claims whose VMM is confirmed dead, sweeps +// orphaned mdevs, resets vendor VFIO VFs no instance claims, and reloads the +// vendor VFIO placements /resources counts from. func (m *manager) ReconcileVGPUs(ctx context.Context) { + m.reconcileVGPUClaims(ctx) + m.resetUnclaimedVendorVFs(ctx) + refresh := m.refreshVGPUCapacity + if refresh == nil { + refresh = devices.RefreshVGPUCapacity + } + refresh() +} + +func (m *manager) reconcileVGPUClaims(ctx context.Context) { log := logger.FromContext(ctx) protected, err := m.reconcileVGPUAssignments(ctx) if err != nil { @@ -68,8 +76,11 @@ func (m *manager) ReconcileVGPUs(ctx context.Context) { // resetUnclaimedVendorVFs resets vendor VFIO VFs that are configured but // claimed by no instance. A claim writes the type before it saves the claim, -// so a restart between the two leaves such a VF holding a placement that -// /resources does not count until the allocator next repairs it. +// so a restart between the two, or a failed claim whose reset also failed, +// leaves such a VF holding a placement that /resources does not count. It +// runs on every pass, so a reset refused once, as while a VFIO handle is +// still open, is retried rather than waiting for the allocator to select +// that VF, which it may never do if placement skips the host. func (m *manager) resetUnclaimedVendorVFs(ctx context.Context) { m.vgpuAllocationMu.Lock() defer m.vgpuAllocationMu.Unlock() @@ -79,10 +90,13 @@ func (m *manager) resetUnclaimedVendorVFs(ctx context.Context) { } allMetadata, err := m.listMetadataForReconcile() if err != nil { + m.recordVGPUReconcileFailure(ctx, vgpuReconcileStageResetUnclaimedVFs) logger.FromContext(ctx).WarnContext(ctx, "failed to list instances for unclaimed vGPU VF reset", "error", err) return } - m.resetDirtyUnclaimedVFs(ctx, vfs, allMetadata) + if _, failed := m.resetDirtyUnclaimedVFs(ctx, vfs, allMetadata); failed { + m.recordVGPUReconcileFailure(ctx, vgpuReconcileStageResetUnclaimedVFs) + } } func (m *manager) reconcileVGPUAssignments(ctx context.Context) (map[string]struct{}, error) { diff --git a/lib/instances/vgpu_reconcile_linux_test.go b/lib/instances/vgpu_reconcile_linux_test.go index 87c808e11..9e130d239 100644 --- a/lib/instances/vgpu_reconcile_linux_test.go +++ b/lib/instances/vgpu_reconcile_linux_test.go @@ -4,6 +4,7 @@ package instances import ( "context" + "errors" "net" "os" "path/filepath" @@ -14,6 +15,8 @@ import ( "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + otelmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" ) func TestReconcileVGPUsProtectsSocketOwnerWithoutPersistedPID(t *testing.T) { @@ -85,3 +88,45 @@ func TestStartVGPUReconcilerResetsUnclaimedVendorVFs(t *testing.T) { assert.Equal(t, []string{"/sys/bus/pci/devices/0000:82:00.5"}, destroyed, "only the configured VF no instance claims is reset") } + +func TestReconcileVGPUsRetriesRefusedUnclaimedVFReset(t *testing.T) { + reader := otelmetric.NewManualReader() + provider := otelmetric.NewMeterProvider(otelmetric.WithReader(reader)) + inUse := true + var destroyed []string + refreshes := 0 + m := &manager{ + paths: paths.New(t.TempDir()), + discoverVGPU: func() (devices.VGPUFramework, []devices.VirtualFunction, error) { + return devices.VGPUFrameworkVendorVFIO, []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0", Allocated: true, ProfileType: testVFProfileType}, + }, nil + }, + destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { + if inUse { + return errors.New("vendor VFIO vGPU on VF 0000:82:00.4 is still in use") + } + destroyed = append(destroyed, assignment.DevicePath) + return nil + }, + reconcileVGPUDevices: func(context.Context, map[string]struct{}) error { return nil }, + refreshVGPUCapacity: func() { refreshes++ }, + } + metrics, err := newInstanceMetrics(provider.Meter("test"), nil, m) + require.NoError(t, err) + m.metrics = metrics + + m.ReconcileVGPUs(t.Context()) + assert.Empty(t, destroyed) + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(t.Context(), &rm)) + failures, ok := findMetric(t, rm, "hypeman_instances_vgpu_reconcile_failures_total").Data.(metricdata.Sum[int64]) + require.True(t, ok) + require.Len(t, failures.DataPoints, 1) + assert.Equal(t, "reset_unclaimed_vfs", metricLabel(t, failures.DataPoints[0].Attributes, "stage")) + + inUse = false + m.ReconcileVGPUs(t.Context()) + assert.Equal(t, []string{"/sys/bus/pci/devices/0000:82:00.4"}, destroyed, "the next pass retries the reset") + assert.Equal(t, 2, refreshes, "every pass reloads the placements") +} From ea6f9ee991ce16470a65ea7b7df1d250cacaab68 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 21 Oct 2026 15:34:31 +0000 Subject: [PATCH 15/16] Reload vGPU capacity on the next read after a failed load --- lib/devices/vgpu_capacity_linux.go | 6 ++++-- lib/devices/vgpu_capacity_linux_test.go | 22 ++++++++++++++++++++++ lib/resources/gpu.go | 4 ++-- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/lib/devices/vgpu_capacity_linux.go b/lib/devices/vgpu_capacity_linux.go index 67f0a973a..74a949f5b 100644 --- a/lib/devices/vgpu_capacity_linux.go +++ b/lib/devices/vgpu_capacity_linux.go @@ -50,10 +50,12 @@ func (s vendorVFIOSysfs) loadCapacity() (creatableSnapshot, error) { } // profiles returns the profile counts for vfs from the cached placements, -// loading them first if nothing has been loaded yet. +// loading them first if nothing has been loaded yet or the last load +// failed, so a transient failure costs one read, not every read until the +// next refresh. func (c *capacityCache) profiles(vfs []VirtualFunction, quarantined map[string]struct{}) ([]GPUProfile, error) { c.mu.RLock() - loaded := c.loaded + loaded := c.loaded && c.err == nil c.mu.RUnlock() if !loaded { c.refresh() diff --git a/lib/devices/vgpu_capacity_linux_test.go b/lib/devices/vgpu_capacity_linux_test.go index d058d1f54..71b96317a 100644 --- a/lib/devices/vgpu_capacity_linux_test.go +++ b/lib/devices/vgpu_capacity_linux_test.go @@ -4,6 +4,7 @@ package devices import ( "context" + "errors" "sync/atomic" "testing" "time" @@ -41,6 +42,27 @@ func TestCapacityCacheReadsDoNotQueryTheDriver(t *testing.T) { assert.Equal(t, int32(1), loads.Load(), "only the first read loads") } +func TestCapacityCacheReloadsAfterFailedLoad(t *testing.T) { + fail := true + var loads atomic.Int32 + cache := newCapacityCache(func() (creatableSnapshot, error) { + loads.Add(1) + if fail { + return creatableSnapshot{}, errors.New("read PCI devices: transient") + } + return creatableSnapshot{byGPU: testCreatablePlacements()}, nil + }) + cache.refresh() + _, err := cache.profiles(testCapacityVFs(), nil) + require.ErrorContains(t, err, "transient") + + fail = false + profiles, err := cache.profiles(testCapacityVFs(), nil) + require.NoError(t, err, "a read after a failed load loads again rather than returning the stale error") + assert.Equal(t, 3, profileAvailability(profiles, "NVIDIA L40S-4Q")) + assert.Equal(t, int32(3), loads.Load()) +} + func TestCapacityCacheBackgroundRefreshesCoalesce(t *testing.T) { started := make(chan struct{}, 10) release := make(chan struct{}) diff --git a/lib/resources/gpu.go b/lib/resources/gpu.go index dc64ed753..fd976a60e 100644 --- a/lib/resources/gpu.go +++ b/lib/resources/gpu.go @@ -28,8 +28,8 @@ type GPUResourceStatus struct { // prevents determining allocatable vGPU capacity. The status is still // returned alongside such an error, with PlacementDisabledReason set. It // returns nil if no GPU is available or the mode is "none". Profiles are -// listed only if withProfiles is set: admission uses only the slot counts, -// and on a vendor VFIO host the listing queries NVML. +// listed only if withProfiles is set, since admission uses only the slot +// counts. func GetGPUStatus(ctx context.Context, withProfiles bool) (*GPUResourceStatus, error) { framework, vfs, err := devices.DiscoverVGPU() if err != nil { From 9fe16b3305d212aae387df05fc7542a8eeb9ac11 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 21 Oct 2026 15:34:31 +0000 Subject: [PATCH 16/16] Refresh vGPU capacity from the reconciler only on vendor VFIO hosts --- lib/instances/vgpu_linux_test.go | 3 ++- lib/instances/vgpu_reconcile.go | 14 +++++++++----- lib/instances/vgpu_reconcile_linux_test.go | 1 + lib/instances/vgpu_reconcile_test.go | 1 + 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/lib/instances/vgpu_linux_test.go b/lib/instances/vgpu_linux_test.go index 8c2147901..ab514d4fa 100644 --- a/lib/instances/vgpu_linux_test.go +++ b/lib/instances/vgpu_linux_test.go @@ -35,7 +35,8 @@ func newVGPUAllocationManager(t *testing.T, vfs []devices.VirtualFunction) *mana } return profiles, nil }, - configureVGPU: func(context.Context, string, string) error { return nil }, + configureVGPU: func(context.Context, string, string) error { return nil }, + refreshVGPUCapacity: func() {}, } } diff --git a/lib/instances/vgpu_reconcile.go b/lib/instances/vgpu_reconcile.go index d756401c3..016b5a91c 100644 --- a/lib/instances/vgpu_reconcile.go +++ b/lib/instances/vgpu_reconcile.go @@ -48,7 +48,9 @@ func (m *manager) StartVGPUReconciler(ctx context.Context) { // vendor VFIO placements /resources counts from. func (m *manager) ReconcileVGPUs(ctx context.Context) { m.reconcileVGPUClaims(ctx) - m.resetUnclaimedVendorVFs(ctx) + if !m.resetUnclaimedVendorVFs(ctx) { + return + } refresh := m.refreshVGPUCapacity if refresh == nil { refresh = devices.RefreshVGPUCapacity @@ -80,23 +82,25 @@ func (m *manager) reconcileVGPUClaims(ctx context.Context) { // leaves such a VF holding a placement that /resources does not count. It // runs on every pass, so a reset refused once, as while a VFIO handle is // still open, is retried rather than waiting for the allocator to select -// that VF, which it may never do if placement skips the host. -func (m *manager) resetUnclaimedVendorVFs(ctx context.Context) { +// that VF, which it may never do if placement skips the host. Reports +// whether the host has vendor VFIO VFs. +func (m *manager) resetUnclaimedVendorVFs(ctx context.Context) bool { m.vgpuAllocationMu.Lock() defer m.vgpuAllocationMu.Unlock() framework, vfs, err := m.discoverVGPUDevices() if err != nil || framework != devices.VGPUFrameworkVendorVFIO { - return + return false } allMetadata, err := m.listMetadataForReconcile() if err != nil { m.recordVGPUReconcileFailure(ctx, vgpuReconcileStageResetUnclaimedVFs) logger.FromContext(ctx).WarnContext(ctx, "failed to list instances for unclaimed vGPU VF reset", "error", err) - return + return true } if _, failed := m.resetDirtyUnclaimedVFs(ctx, vfs, allMetadata); failed { m.recordVGPUReconcileFailure(ctx, vgpuReconcileStageResetUnclaimedVFs) } + return true } func (m *manager) reconcileVGPUAssignments(ctx context.Context) (map[string]struct{}, error) { diff --git a/lib/instances/vgpu_reconcile_linux_test.go b/lib/instances/vgpu_reconcile_linux_test.go index 9e130d239..bda272dcf 100644 --- a/lib/instances/vgpu_reconcile_linux_test.go +++ b/lib/instances/vgpu_reconcile_linux_test.go @@ -74,6 +74,7 @@ func TestStartVGPUReconcilerResetsUnclaimedVendorVFs(t *testing.T) { return nil }, reconcileVGPUDevices: func(context.Context, map[string]struct{}) error { return nil }, + refreshVGPUCapacity: func() {}, vgpuReconcileInterval: time.Hour, } liveIdentity := HypervisorProcessIdentity{} diff --git a/lib/instances/vgpu_reconcile_test.go b/lib/instances/vgpu_reconcile_test.go index 51d0624fb..eca54ef61 100644 --- a/lib/instances/vgpu_reconcile_test.go +++ b/lib/instances/vgpu_reconcile_test.go @@ -129,6 +129,7 @@ func TestStartVGPUReconcilerRunsPeriodically(t *testing.T) { passes.Add(1) return nil }, + refreshVGPUCapacity: func() {}, vgpuReconcileInterval: time.Millisecond, }