diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 683a506a6..ca0288263 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -92,10 +92,10 @@ On an mdev host, the response also includes the assigned mdev UUID: ### Ephemeral vGPU Lifecycle -vGPU assignments are created on instance start and released on stop or delete. Hypeman creates/removes an mdev on mdev hosts. On vendor VFIO hosts, it first reserves a VF by persisting its path in instance metadata, then writes the profile ID to `current_vgpu_type`. Metadata is the authoritative VF claim. +vGPU assignments are created on instance start and released on stop or delete. Hypeman creates/removes an mdev on mdev hosts. On vendor VFIO hosts, it selects a VF and writes the profile ID to `current_vgpu_type` under the allocation lock, so the driver confirms a placement remains before any later claim can take it, then reserves the VF by persisting its path in instance metadata. Metadata is the authoritative VF claim; a configured VF without one is dirty and is reset by the allocator. ``` -Instance Create → Persist VF claim → Configure profile → Attach VF to VM → Instance Running +Instance Create → Configure profile → Persist VF claim → Attach VF to VM → Instance Running Instance Stop/Delete → Reset profile → Remove VF claim → VF available again ``` @@ -106,6 +106,10 @@ before the hypervisor is signalled. Hypeman reconciles metadata claims once at startup and every minute afterward, skipping hosts without GPUs. A claim whose VMM is confirmed dead is reset before the claim is removed. An ambiguous hypervisor ownership check preserves the claim, logs a warning, and increments `hypeman_instances_vgpu_liveness_uncertain_total`; the create and start cleanup paths preserve and count the same way. mdev hosts also sweep orphaned device-level assignments. Vendor VFIO hosts repair an unclaimed dirty VF when the allocator next selects it; repair checks for open VFIO handles before resetting `current_vgpu_type`. The allocator prefers VFs that are already clean, and a dirty VF that refuses its reset is skipped in favor of another candidate. +In mixed-size (heterogeneous) mode, `creatable_vgpu_types` under-reports. The driver drops a profile from every VF's list once its GPU holds as many vGPUs as that profile's equal-size instance limit (12 for L40S-4Q), even while a placement for it remains, and still accepts a `current_vgpu_type` write for it. The allocator remembers every profile each GPU has advertised, and a clean VF on a GPU that supports the requested profile but no longer advertises it stays a candidate, ranked after every GPU with a free VF that advertises it. If the driver rejects the claim's write, unclaimed dirty VFs on that GPU are reset and the write is retried once before the next GPU is tried, and `hypeman_instances_vgpu_driver_rejections_total` is incremented with the GPU's address; any other configure error fails the claim. If no candidate VF is left, unclaimed dirty VFs on the GPUs not yet tried are reset once and selection runs again. A free VF that advertises nothing is skipped (on L40S its GPU has no placement left for any size), as is one whose list cannot be read. This assumes every GPU is in mixed-size mode (`nvidia-smi vgpu -i -shm 1`): in equal-size mode a GPU running a smaller profile hides larger ones for good, so each request for one costs a refused write on that GPU. + +In mixed-size mode each vGPU also occupies a fixed run of placement slots, and the driver puts a new vGPU at the lowest free placement for its type. On an L40S that scatters 2Q across the runs 4Q needs: half of the 16 2Q placements overlap a 4Q placement. After writing `current_vgpu_type`, and before any VM attaches, hypeman moves the vGPU by writing `placement_id` to the free placement that blocks the fewest free placements of larger profiles, comparing the smallest larger size first. A tie keeps the driver's placement. On an L40S the first eight 2Q then land on the placements no 4Q overlaps, so a card holds eight 2Q and eight 4Q instead of eight 2Q and four 4Q. The placement table and mode come from `nvidia-smi vgpu -s -v -i ` and `nvidia-smi -q -i `, cached per GPU for 10 minutes. Packing is skipped in equal-size mode, and any failure (no `nvidia-smi`, an unreadable sibling, a refused write) keeps the driver's placement. + ### Hypervisor Support Hypervisor selection for vGPU instances is caller policy; hypeman does not enforce it. In practice **QEMU is the only hypervisor with working vGPU support**: diff --git a/lib/devices/errors.go b/lib/devices/errors.go index 15d68dab6..5b88afe5e 100644 --- a/lib/devices/errors.go +++ b/lib/devices/errors.go @@ -38,4 +38,9 @@ var ( // ErrVGPUNotSupportedOnMacOS is returned for vGPU operations on macOS ErrVGPUNotSupportedOnMacOS = errors.New("vGPU (mdev) is not supported on macOS") + + // ErrVGPUTypeRejected is returned when the driver refuses a vendor VFIO + // current_vgpu_type write, which is how it reports that the VF's GPU has + // no placement left for the type + ErrVGPUTypeRejected = errors.New("driver rejected vGPU type") ) diff --git a/lib/devices/testdata/nvidia-smi-vgpu-supported-l40s.txt b/lib/devices/testdata/nvidia-smi-vgpu-supported-l40s.txt new file mode 100644 index 000000000..90e2e1c1e --- /dev/null +++ b/lib/devices/testdata/nvidia-smi-vgpu-supported-l40s.txt @@ -0,0 +1,662 @@ +GPU 00000000:82:00.0 + vGPU Type ID : 0x479 + Name : NVIDIA L40S-1B + Class : NVS + GPU Instance Profile ID : N/A + Max Instances : 32 + Max Instances Per VM : 1 + Max Instances Per GI : N/A + Multi vGPU Exclusive : True + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x188910de + FB Memory : 1024 MiB + BAR1 size : 256 MB + Display Heads : 4 + Maximum X Resolution : 5120 + Maximum Y Resolution : 2880 + Frame Rate Limit : 45 FPS + Placement Size : 1 + Heterogeneous Placement IDs : 0 5 6 11 12 17 18 23 24 29 30 35 36 41 42 47 + Homogeneous Placement IDs : 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 + GRID License : GRID-Virtual-PC,2.0;Quadro-Virtual-DWS,5.0;GRID-Virtual-WS,2.0;GRID-Virtual-WS-Ext,2.0 + vGPU Type ID : 0x47a + Name : NVIDIA L40S-2B + Class : NVS + GPU Instance Profile ID : N/A + Max Instances : 24 + Max Instances Per VM : 1 + Max Instances Per GI : N/A + Multi vGPU Exclusive : True + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x188a10de + FB Memory : 2048 MiB + BAR1 size : 256 MB + Display Heads : 4 + Maximum X Resolution : 5120 + Maximum Y Resolution : 2880 + Frame Rate Limit : 45 FPS + Placement Size : 2 + Heterogeneous Placement IDs : 0 4 6 10 12 16 18 22 24 28 30 34 36 40 42 46 + Homogeneous Placement IDs : 0 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40 42 44 46 + GRID License : GRID-Virtual-PC,2.0;Quadro-Virtual-DWS,5.0;GRID-Virtual-WS,2.0;GRID-Virtual-WS-Ext,2.0 + vGPU Type ID : 0x47b + Name : NVIDIA L40S-1Q + Class : Quadro + GPU Instance Profile ID : N/A + Max Instances : 32 + Max Instances Per VM : 16 + Max Instances Per GI : N/A + Multi vGPU Exclusive : False + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x188b10de + FB Memory : 1024 MiB + BAR1 size : 256 MB + Display Heads : 4 + Maximum X Resolution : 5120 + Maximum Y Resolution : 2880 + Frame Rate Limit : 60 FPS + Placement Size : 1 + Heterogeneous Placement IDs : 0 5 6 11 12 17 18 23 24 29 30 35 36 41 42 47 + Homogeneous Placement IDs : 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 + GRID License : Quadro-Virtual-DWS,5.0;GRID-Virtual-WS,2.0;GRID-Virtual-WS-Ext,2.0 + vGPU Type ID : 0x47c + Name : NVIDIA L40S-2Q + Class : Quadro + GPU Instance Profile ID : N/A + Max Instances : 24 + Max Instances Per VM : 16 + Max Instances Per GI : N/A + Multi vGPU Exclusive : False + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x188c10de + FB Memory : 2048 MiB + BAR1 size : 256 MB + Display Heads : 4 + Maximum X Resolution : 7680 + Maximum Y Resolution : 4320 + Frame Rate Limit : 60 FPS + Placement Size : 2 + Heterogeneous Placement IDs : 0 4 6 10 12 16 18 22 24 28 30 34 36 40 42 46 + Homogeneous Placement IDs : 0 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40 42 44 46 + GRID License : Quadro-Virtual-DWS,5.0;GRID-Virtual-WS,2.0;GRID-Virtual-WS-Ext,2.0 + vGPU Type ID : 0x47d + Name : NVIDIA L40S-3Q + Class : Quadro + GPU Instance Profile ID : N/A + Max Instances : 16 + Max Instances Per VM : 16 + Max Instances Per GI : N/A + Multi vGPU Exclusive : False + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x188d10de + FB Memory : 3072 MiB + BAR1 size : 256 MB + Display Heads : 4 + Maximum X Resolution : 7680 + Maximum Y Resolution : 4320 + Frame Rate Limit : 60 FPS + Placement Size : 3 + Heterogeneous Placement IDs : 0 3 6 9 12 15 18 21 24 27 30 33 36 39 42 45 + Homogeneous Placement IDs : 0 3 6 9 12 15 18 21 24 27 30 33 36 39 42 45 + GRID License : Quadro-Virtual-DWS,5.0;GRID-Virtual-WS,2.0;GRID-Virtual-WS-Ext,2.0 + vGPU Type ID : 0x47e + Name : NVIDIA L40S-4Q + Class : Quadro + GPU Instance Profile ID : N/A + Max Instances : 12 + Max Instances Per VM : 16 + Max Instances Per GI : N/A + Multi vGPU Exclusive : False + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x188e10de + FB Memory : 4096 MiB + BAR1 size : 256 MB + Display Heads : 4 + Maximum X Resolution : 7680 + Maximum Y Resolution : 4320 + Frame Rate Limit : 60 FPS + Placement Size : 4 + Heterogeneous Placement IDs : 0 8 12 20 24 32 36 44 + Homogeneous Placement IDs : 0 4 8 12 16 20 24 28 32 36 40 44 + GRID License : Quadro-Virtual-DWS,5.0;GRID-Virtual-WS,2.0;GRID-Virtual-WS-Ext,2.0 + vGPU Type ID : 0x47f + Name : NVIDIA L40S-6Q + Class : Quadro + GPU Instance Profile ID : N/A + Max Instances : 8 + Max Instances Per VM : 16 + Max Instances Per GI : N/A + Multi vGPU Exclusive : False + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x188f10de + FB Memory : 6144 MiB + BAR1 size : 256 MB + Display Heads : 4 + Maximum X Resolution : 7680 + Maximum Y Resolution : 4320 + Frame Rate Limit : 60 FPS + Placement Size : 6 + Heterogeneous Placement IDs : 0 6 12 18 24 30 36 42 + Homogeneous Placement IDs : 0 6 12 18 24 30 36 42 + GRID License : Quadro-Virtual-DWS,5.0;GRID-Virtual-WS,2.0;GRID-Virtual-WS-Ext,2.0 + vGPU Type ID : 0x480 + Name : NVIDIA L40S-8Q + Class : Quadro + GPU Instance Profile ID : N/A + Max Instances : 6 + Max Instances Per VM : 16 + Max Instances Per GI : N/A + Multi vGPU Exclusive : False + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x189010de + FB Memory : 8192 MiB + BAR1 size : 256 MB + Display Heads : 4 + Maximum X Resolution : 7680 + Maximum Y Resolution : 4320 + Frame Rate Limit : 60 FPS + Placement Size : 8 + Heterogeneous Placement IDs : 0 16 24 40 + Homogeneous Placement IDs : 0 8 16 24 32 40 + GRID License : Quadro-Virtual-DWS,5.0;GRID-Virtual-WS,2.0;GRID-Virtual-WS-Ext,2.0 + vGPU Type ID : 0x481 + Name : NVIDIA L40S-12Q + Class : Quadro + GPU Instance Profile ID : N/A + Max Instances : 4 + Max Instances Per VM : 16 + Max Instances Per GI : N/A + Multi vGPU Exclusive : False + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x189110de + FB Memory : 12288 MiB + BAR1 size : 256 MB + Display Heads : 4 + Maximum X Resolution : 7680 + Maximum Y Resolution : 4320 + Frame Rate Limit : 60 FPS + Placement Size : 12 + Heterogeneous Placement IDs : 0 12 24 36 + Homogeneous Placement IDs : 0 12 24 36 + GRID License : Quadro-Virtual-DWS,5.0;GRID-Virtual-WS,2.0;GRID-Virtual-WS-Ext,2.0 + vGPU Type ID : 0x482 + Name : NVIDIA L40S-16Q + Class : Quadro + GPU Instance Profile ID : N/A + Max Instances : 3 + Max Instances Per VM : 16 + Max Instances Per GI : N/A + Multi vGPU Exclusive : False + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x189210de + FB Memory : 16384 MiB + BAR1 size : 256 MB + Display Heads : 4 + Maximum X Resolution : 7680 + Maximum Y Resolution : 4320 + Frame Rate Limit : 60 FPS + Placement Size : 16 + Heterogeneous Placement IDs : 0 32 + Homogeneous Placement IDs : 0 16 32 + GRID License : Quadro-Virtual-DWS,5.0;GRID-Virtual-WS,2.0;GRID-Virtual-WS-Ext,2.0 + vGPU Type ID : 0x483 + Name : NVIDIA L40S-24Q + Class : Quadro + GPU Instance Profile ID : N/A + Max Instances : 2 + Max Instances Per VM : 16 + Max Instances Per GI : N/A + Multi vGPU Exclusive : False + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x189310de + FB Memory : 24576 MiB + BAR1 size : 256 MB + Display Heads : 4 + Maximum X Resolution : 7680 + Maximum Y Resolution : 4320 + Frame Rate Limit : 60 FPS + Placement Size : 24 + Heterogeneous Placement IDs : 0 24 + Homogeneous Placement IDs : 0 24 + GRID License : Quadro-Virtual-DWS,5.0;GRID-Virtual-WS,2.0;GRID-Virtual-WS-Ext,2.0 + vGPU Type ID : 0x484 + Name : NVIDIA L40S-48Q + Class : Quadro + GPU Instance Profile ID : N/A + Max Instances : 1 + Max Instances Per VM : 16 + Max Instances Per GI : N/A + Multi vGPU Exclusive : False + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x189410de + FB Memory : 49152 MiB + BAR1 size : 256 MB + Display Heads : 4 + Maximum X Resolution : 7680 + Maximum Y Resolution : 4320 + Frame Rate Limit : 60 FPS + Placement Size : 48 + Heterogeneous Placement IDs : 0 + Homogeneous Placement IDs : 0 + GRID License : Quadro-Virtual-DWS,5.0;GRID-Virtual-WS,2.0;GRID-Virtual-WS-Ext,2.0 + vGPU Type ID : 0x485 + Name : NVIDIA L40S-1A + Class : NVS + GPU Instance Profile ID : N/A + Max Instances : 32 + Max Instances Per VM : 1 + Max Instances Per GI : N/A + Multi vGPU Exclusive : True + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x189510de + FB Memory : 1024 MiB + BAR1 size : 256 MB + Display Heads : 1 + Maximum X Resolution : 1280 + Maximum Y Resolution : 1024 + Frame Rate Limit : 60 FPS + Placement Size : 1 + Heterogeneous Placement IDs : 0 5 6 11 12 17 18 23 24 29 30 35 36 41 42 47 + Homogeneous Placement IDs : 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 + GRID License : GRID-Virtual-Apps,3.0 + vGPU Type ID : 0x486 + Name : NVIDIA L40S-2A + Class : NVS + GPU Instance Profile ID : N/A + Max Instances : 24 + Max Instances Per VM : 1 + Max Instances Per GI : N/A + Multi vGPU Exclusive : True + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x189610de + FB Memory : 2048 MiB + BAR1 size : 256 MB + Display Heads : 1 + Maximum X Resolution : 1280 + Maximum Y Resolution : 1024 + Frame Rate Limit : 60 FPS + Placement Size : 2 + Heterogeneous Placement IDs : 0 4 6 10 12 16 18 22 24 28 30 34 36 40 42 46 + Homogeneous Placement IDs : 0 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40 42 44 46 + GRID License : GRID-Virtual-Apps,3.0 + vGPU Type ID : 0x487 + Name : NVIDIA L40S-3A + Class : NVS + GPU Instance Profile ID : N/A + Max Instances : 16 + Max Instances Per VM : 1 + Max Instances Per GI : N/A + Multi vGPU Exclusive : True + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x189710de + FB Memory : 3072 MiB + BAR1 size : 256 MB + Display Heads : 1 + Maximum X Resolution : 1280 + Maximum Y Resolution : 1024 + Frame Rate Limit : 60 FPS + Placement Size : 3 + Heterogeneous Placement IDs : 0 3 6 9 12 15 18 21 24 27 30 33 36 39 42 45 + Homogeneous Placement IDs : 0 3 6 9 12 15 18 21 24 27 30 33 36 39 42 45 + GRID License : GRID-Virtual-Apps,3.0 + vGPU Type ID : 0x488 + Name : NVIDIA L40S-4A + Class : NVS + GPU Instance Profile ID : N/A + Max Instances : 12 + Max Instances Per VM : 1 + Max Instances Per GI : N/A + Multi vGPU Exclusive : True + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x189810de + FB Memory : 4096 MiB + BAR1 size : 256 MB + Display Heads : 1 + Maximum X Resolution : 1280 + Maximum Y Resolution : 1024 + Frame Rate Limit : 60 FPS + Placement Size : 4 + Heterogeneous Placement IDs : 0 8 12 20 24 32 36 44 + Homogeneous Placement IDs : 0 4 8 12 16 20 24 28 32 36 40 44 + GRID License : GRID-Virtual-Apps,3.0 + vGPU Type ID : 0x489 + Name : NVIDIA L40S-6A + Class : NVS + GPU Instance Profile ID : N/A + Max Instances : 8 + Max Instances Per VM : 1 + Max Instances Per GI : N/A + Multi vGPU Exclusive : True + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x189910de + FB Memory : 6144 MiB + BAR1 size : 256 MB + Display Heads : 1 + Maximum X Resolution : 1280 + Maximum Y Resolution : 1024 + Frame Rate Limit : 60 FPS + Placement Size : 6 + Heterogeneous Placement IDs : 0 6 12 18 24 30 36 42 + Homogeneous Placement IDs : 0 6 12 18 24 30 36 42 + GRID License : GRID-Virtual-Apps,3.0 + vGPU Type ID : 0x48a + Name : NVIDIA L40S-8A + Class : NVS + GPU Instance Profile ID : N/A + Max Instances : 6 + Max Instances Per VM : 1 + Max Instances Per GI : N/A + Multi vGPU Exclusive : True + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x189a10de + FB Memory : 8192 MiB + BAR1 size : 256 MB + Display Heads : 1 + Maximum X Resolution : 1280 + Maximum Y Resolution : 1024 + Frame Rate Limit : 60 FPS + Placement Size : 8 + Heterogeneous Placement IDs : 0 16 24 40 + Homogeneous Placement IDs : 0 8 16 24 32 40 + GRID License : GRID-Virtual-Apps,3.0 + vGPU Type ID : 0x48b + Name : NVIDIA L40S-12A + Class : NVS + GPU Instance Profile ID : N/A + Max Instances : 4 + Max Instances Per VM : 1 + Max Instances Per GI : N/A + Multi vGPU Exclusive : True + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x189b10de + FB Memory : 12288 MiB + BAR1 size : 256 MB + Display Heads : 1 + Maximum X Resolution : 1280 + Maximum Y Resolution : 1024 + Frame Rate Limit : 60 FPS + Placement Size : 12 + Heterogeneous Placement IDs : 0 12 24 36 + Homogeneous Placement IDs : 0 12 24 36 + GRID License : GRID-Virtual-Apps,3.0 + vGPU Type ID : 0x48c + Name : NVIDIA L40S-16A + Class : NVS + GPU Instance Profile ID : N/A + Max Instances : 3 + Max Instances Per VM : 1 + Max Instances Per GI : N/A + Multi vGPU Exclusive : True + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x189c10de + FB Memory : 16384 MiB + BAR1 size : 256 MB + Display Heads : 1 + Maximum X Resolution : 1280 + Maximum Y Resolution : 1024 + Frame Rate Limit : 60 FPS + Placement Size : 16 + Heterogeneous Placement IDs : 0 32 + Homogeneous Placement IDs : 0 16 32 + GRID License : GRID-Virtual-Apps,3.0 + vGPU Type ID : 0x48d + Name : NVIDIA L40S-24A + Class : NVS + GPU Instance Profile ID : N/A + Max Instances : 2 + Max Instances Per VM : 1 + Max Instances Per GI : N/A + Multi vGPU Exclusive : True + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x189d10de + FB Memory : 24576 MiB + BAR1 size : 256 MB + Display Heads : 1 + Maximum X Resolution : 1280 + Maximum Y Resolution : 1024 + Frame Rate Limit : 60 FPS + Placement Size : 24 + Heterogeneous Placement IDs : 0 24 + Homogeneous Placement IDs : 0 24 + GRID License : GRID-Virtual-Apps,3.0 + vGPU Type ID : 0x48e + Name : NVIDIA L40S-48A + Class : NVS + GPU Instance Profile ID : N/A + Max Instances : 1 + Max Instances Per VM : 1 + Max Instances Per GI : N/A + Multi vGPU Exclusive : True + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x189e10de + FB Memory : 49152 MiB + BAR1 size : 256 MB + Display Heads : 1 + Maximum X Resolution : 1280 + Maximum Y Resolution : 1024 + Frame Rate Limit : 60 FPS + Placement Size : 48 + Heterogeneous Placement IDs : 0 + Homogeneous Placement IDs : 0 + GRID License : GRID-Virtual-Apps,3.0 + vGPU Type ID : 0x49b + Name : NVIDIA L40S-4C + Class : Compute + GPU Instance Profile ID : N/A + Max Instances : 12 + Max Instances Per VM : 16 + Max Instances Per GI : N/A + Multi vGPU Exclusive : False + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x18ab10de + FB Memory : 4096 MiB + BAR1 size : 2048 MB + Display Heads : 1 + Maximum X Resolution : 4096 + Maximum Y Resolution : 2400 + Frame Rate Limit : 60 FPS + Placement Size : 4 + Heterogeneous Placement IDs : 0 8 12 20 24 32 36 44 + Homogeneous Placement IDs : 0 4 8 12 16 20 24 28 32 36 40 44 + GRID License : NVIDIA-vComputeServer,9.0 + vGPU Type ID : 0x49c + Name : NVIDIA L40S-6C + Class : Compute + GPU Instance Profile ID : N/A + Max Instances : 8 + Max Instances Per VM : 16 + Max Instances Per GI : N/A + Multi vGPU Exclusive : False + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x18ac10de + FB Memory : 6144 MiB + BAR1 size : 2048 MB + Display Heads : 1 + Maximum X Resolution : 4096 + Maximum Y Resolution : 2400 + Frame Rate Limit : 60 FPS + Placement Size : 6 + Heterogeneous Placement IDs : 0 6 12 18 24 30 36 42 + Homogeneous Placement IDs : 0 6 12 18 24 30 36 42 + GRID License : NVIDIA-vComputeServer,9.0 + vGPU Type ID : 0x49d + Name : NVIDIA L40S-8C + Class : Compute + GPU Instance Profile ID : N/A + Max Instances : 6 + Max Instances Per VM : 16 + Max Instances Per GI : N/A + Multi vGPU Exclusive : False + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x18ad10de + FB Memory : 8192 MiB + BAR1 size : 2048 MB + Display Heads : 1 + Maximum X Resolution : 4096 + Maximum Y Resolution : 2400 + Frame Rate Limit : 60 FPS + Placement Size : 8 + Heterogeneous Placement IDs : 0 16 24 40 + Homogeneous Placement IDs : 0 8 16 24 32 40 + GRID License : NVIDIA-vComputeServer,9.0 + vGPU Type ID : 0x49e + Name : NVIDIA L40S-12C + Class : Compute + GPU Instance Profile ID : N/A + Max Instances : 4 + Max Instances Per VM : 16 + Max Instances Per GI : N/A + Multi vGPU Exclusive : False + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x18ae10de + FB Memory : 12288 MiB + BAR1 size : 2048 MB + Display Heads : 1 + Maximum X Resolution : 4096 + Maximum Y Resolution : 2400 + Frame Rate Limit : 60 FPS + Placement Size : 12 + Heterogeneous Placement IDs : 0 12 24 36 + Homogeneous Placement IDs : 0 12 24 36 + GRID License : NVIDIA-vComputeServer,9.0 + vGPU Type ID : 0x49f + Name : NVIDIA L40S-16C + Class : Compute + GPU Instance Profile ID : N/A + Max Instances : 3 + Max Instances Per VM : 16 + Max Instances Per GI : N/A + Multi vGPU Exclusive : False + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x18af10de + FB Memory : 16384 MiB + BAR1 size : 2048 MB + Display Heads : 1 + Maximum X Resolution : 4096 + Maximum Y Resolution : 2400 + Frame Rate Limit : 60 FPS + Placement Size : 16 + Heterogeneous Placement IDs : 0 32 + Homogeneous Placement IDs : 0 16 32 + GRID License : NVIDIA-vComputeServer,9.0 + vGPU Type ID : 0x4a0 + Name : NVIDIA L40S-24C + Class : Compute + GPU Instance Profile ID : N/A + Max Instances : 2 + Max Instances Per VM : 16 + Max Instances Per GI : N/A + Multi vGPU Exclusive : False + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x18b010de + FB Memory : 24576 MiB + BAR1 size : 2048 MB + Display Heads : 1 + Maximum X Resolution : 4096 + Maximum Y Resolution : 2400 + Frame Rate Limit : 60 FPS + Placement Size : 24 + Heterogeneous Placement IDs : 0 24 + Homogeneous Placement IDs : 0 24 + GRID License : NVIDIA-vComputeServer,9.0 + vGPU Type ID : 0x4a1 + Name : NVIDIA L40S-48C + Class : Compute + GPU Instance Profile ID : N/A + Max Instances : 1 + Max Instances Per VM : 16 + Max Instances Per GI : N/A + Multi vGPU Exclusive : False + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x18b110de + FB Memory : 49152 MiB + BAR1 size : 2048 MB + Display Heads : 1 + Maximum X Resolution : 4096 + Maximum Y Resolution : 2400 + Frame Rate Limit : 60 FPS + Placement Size : 48 + Heterogeneous Placement IDs : 0 + Homogeneous Placement IDs : 0 + GRID License : NVIDIA-vComputeServer,9.0 + vGPU Type ID : 0x874 + Name : NVIDIA L40S-3B + Class : NVS + GPU Instance Profile ID : N/A + Max Instances : 16 + Max Instances Per VM : 1 + Max Instances Per GI : N/A + Multi vGPU Exclusive : True + vGPU Exclusive Type : False + vGPU Exclusive Size : False + Device ID : 0x26b910de + Sub System ID : 0x21fe10de + FB Memory : 3072 MiB + BAR1 size : 256 MB + Display Heads : 4 + Maximum X Resolution : 5120 + Maximum Y Resolution : 2880 + Frame Rate Limit : 45 FPS + Placement Size : 3 + Heterogeneous Placement IDs : 0 3 6 9 12 15 18 21 24 27 30 33 36 39 42 45 + Homogeneous Placement IDs : 0 3 6 9 12 15 18 21 24 27 30 33 36 39 42 45 + GRID License : GRID-Virtual-PC,2.0;Quadro-Virtual-DWS,5.0;GRID-Virtual-WS,2.0;GRID-Virtual-WS-Ext,2.0 + diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 2fb552095..7f080364d 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -27,6 +27,8 @@ type vendorVFIOSysfs struct { procPath string vfioDevicesPath string openVFIOPathsFunc func() (map[string]struct{}, error) + writeFileFunc func(name string, data []byte, perm os.FileMode) error + nvidiaSMIFunc func(ctx context.Context, args ...string) ([]byte, error) } var hostVendorVFIO = vendorVFIOSysfs{ @@ -188,7 +190,7 @@ func (s vendorVFIOSysfs) configure(ctx context.Context, vfAddress, profileType s if inUse { return fmt.Errorf("vendor VFIO vGPU on VF %s is still in use", vfAddress) } - if err := os.WriteFile(currentTypePath, []byte("0"), 0200); err != nil { + if err := s.writeFile(currentTypePath, []byte("0"), 0200); err != nil { return fmt.Errorf("reset vGPU on VF %s: %w", vfAddress, err) } currentType, err = readCurrentVGPUType(currentTypePath) @@ -199,8 +201,8 @@ func (s vendorVFIOSysfs) configure(ctx context.Context, vfAddress, profileType s return fmt.Errorf("verify reset vGPU on VF %s: type is %s, want 0", vfAddress, currentType) } } - if err := os.WriteFile(currentTypePath, []byte(profileType), 0200); err != nil { - return fmt.Errorf("configure vGPU on VF %s: %w", vfAddress, err) + if err := s.writeFile(currentTypePath, []byte(profileType), 0200); err != nil { + return fmt.Errorf("configure vGPU on VF %s: %w: %w", vfAddress, ErrVGPUTypeRejected, err) } currentType, err = readCurrentVGPUType(currentTypePath) if err != nil { @@ -210,6 +212,7 @@ func (s vendorVFIOSysfs) configure(ctx context.Context, vfAddress, profileType s return fmt.Errorf("verify vGPU on VF %s: type is %s, want %s", vfAddress, currentType, profileType) } logger.FromContext(ctx).InfoContext(ctx, "configured vendor VFIO vGPU", "profile_type", profileType, "vf", vfAddress) + s.packPlacement(ctx, vfAddress, profileType) return nil } @@ -242,13 +245,20 @@ func (s vendorVFIOSysfs) destroy(ctx context.Context, vfAddress, instanceID stri return fmt.Errorf("vendor VFIO vGPU on VF %s is still in use", vfAddress) } - if err := os.WriteFile(currentTypePath, []byte("0"), 0200); err != nil { + if err := s.writeFile(currentTypePath, []byte("0"), 0200); err != nil { return fmt.Errorf("destroy vGPU on VF %s: %w", vfAddress, err) } log.InfoContext(ctx, "destroyed vendor VFIO vGPU", "vf", vfAddress, "instance_id", instanceID) return nil } +func (s vendorVFIOSysfs) writeFile(name string, data []byte, perm os.FileMode) error { + if s.writeFileFunc != nil { + return s.writeFileFunc(name, data, perm) + } + return os.WriteFile(name, data, perm) +} + func (s vendorVFIOSysfs) readCreatableProfiles(vfAddress string) ([]profileMetadata, error) { path := filepath.Join(s.pciDevicesPath, vfAddress, "nvidia", "creatable_vgpu_types") data, err := os.ReadFile(path) diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 3c7438532..3a938ae4d 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -4,8 +4,10 @@ package devices import ( "context" + "errors" "os" "path/filepath" + "syscall" "testing" "github.com/stretchr/testify/assert" @@ -64,6 +66,29 @@ func TestVendorVFIOConfigure(t *testing.T) { require.NoError(t, sysfs.configure(context.Background(), vfAddress, "1148"), "configuration must be idempotent") } +func TestVendorVFIOConfigureReportsDriverRejection(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1159", testCreatableTypes) + sysfs.writeFileFunc = func(name string, data []byte, perm os.FileMode) error { + if string(data) != "0" { + return syscall.EINVAL + } + return os.WriteFile(name, data, perm) + } + + err := sysfs.configure(context.Background(), vfAddress, "1148") + require.ErrorIs(t, err, ErrVGPUTypeRejected) + require.ErrorIs(t, err, syscall.EINVAL) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "0") + + sysfs.writeFileFunc = func(string, []byte, os.FileMode) error { return syscall.EIO } + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "1159", testCreatableTypes) + err = sysfs.configure(context.Background(), "0000:82:00.5", "1148") + require.ErrorContains(t, err, "reset vGPU on VF 0000:82:00.5") + assert.NotErrorIs(t, err, ErrVGPUTypeRejected, "a failed reset is not a placement answer") +} + func TestVendorVFIOConfigureRepairsDirtyVF(t *testing.T) { sysfs := newTestVendorVFIOSysfs(t) const vfAddress = "0000:82:00.4" @@ -153,6 +178,9 @@ func newTestVendorVFIOSysfs(t *testing.T) testVendorVFIOSysfs { pciDevicesPath: pci, procPath: proc, vfioDevicesPath: vfio, + nvidiaSMIFunc: func(context.Context, ...string) ([]byte, error) { + return nil, errors.New("nvidia-smi not available in tests") + }, }} } diff --git a/lib/devices/vgpu_placement_linux.go b/lib/devices/vgpu_placement_linux.go new file mode 100644 index 000000000..ef78bf161 --- /dev/null +++ b/lib/devices/vgpu_placement_linux.go @@ -0,0 +1,335 @@ +//go:build linux + +package devices + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + + "github.com/kernel/hypeman/lib/logger" +) + +// In mixed-size (heterogeneous) mode each vGPU occupies a fixed run of its +// GPU's placement slots, and each profile may only start at the driver's +// heterogeneous placement IDs. The driver puts a new vGPU at the lowest free +// placement for its type, which scatters small profiles across the runs larger +// profiles need: on an L40S, half of the 2Q placements overlap a 4Q placement. +// After a type is configured, packPlacement moves the vGPU to the free +// placement that blocks the fewest placements of larger profiles. Packing is +// best effort: any failure keeps the driver's placement. + +// vgpuPlacementTTL bounds how long a GPU's mode and geometry are reused, so a +// mode change made while hypeman runs is picked up. +const vgpuPlacementTTL = 10 * time.Minute + +const nvidiaSMITimeout = 5 * time.Second + +type vgpuPlacementType struct { + size int + ids []int +} + +// vgpuPlacementGeometry holds a GPU's placement table, which is empty unless +// the GPU is in mixed-size mode. +type vgpuPlacementGeometry struct { + types map[string]vgpuPlacementType // keyed by decimal vGPU type ID + loadedAt time.Time +} + +// vgpuPlacements caches geometry by parent GPU sysfs path, failures included, +// so a host without nvidia-smi does not run it on every claim. Guarded by +// vendorVFIOMu. +var vgpuPlacements = make(map[string]vgpuPlacementGeometry) + +type placementSpan struct { + start, size int +} + +func (s vendorVFIOSysfs) packPlacement(ctx context.Context, vfAddress, profileType string) { + log := logger.FromContext(ctx) + vfPath := filepath.Join(s.pciDevicesPath, vfAddress) + target, err := os.Readlink(filepath.Join(vfPath, "physfn")) + if err != nil { + log.WarnContext(ctx, "keeping driver vGPU placement: read parent GPU", "vf", vfAddress, "error", err) + return + } + parentGPU := filepath.Base(target) + geometry := s.placementGeometry(ctx, parentGPU) + requested, ok := geometry.types[profileType] + if !ok { + return + } + placementPath := filepath.Join(vfPath, "nvidia", "placement_id") + current, err := readPlacementID(placementPath) + if err != nil { + log.WarnContext(ctx, "keeping driver vGPU placement: read placement", "vf", vfAddress, "error", err) + return + } + occupied, err := s.occupiedPlacements(parentGPU, vfAddress, geometry) + if err != nil { + log.WarnContext(ctx, "keeping driver vGPU placement: read sibling placements", "vf", vfAddress, "error", err) + return + } + best := choosePlacement(geometry.types, occupied, requested, current) + if best == current { + return + } + if err := s.writeFile(placementPath, []byte(strconv.Itoa(best)), 0200); err != nil { + log.WarnContext(ctx, "driver refused vGPU placement; keeping its own", "vf", vfAddress, "from", current, "to", best, "error", err) + return + } + if moved, err := readPlacementID(placementPath); err != nil || moved != best { + log.WarnContext(ctx, "vGPU placement did not move as written", "vf", vfAddress, "want", best, "got", moved, "error", err) + return + } + log.InfoContext(ctx, "packed vGPU placement", "vf", vfAddress, "profile_type", profileType, "from", current, "to", best) +} + +// placementGeometry returns the cached placement table for a GPU, reloading +// it once it is older than vgpuPlacementTTL. A failed load is cached as an +// empty table, which disables packing on that GPU until the next reload. +// Callers must hold vendorVFIOMu. +func (s vendorVFIOSysfs) placementGeometry(ctx context.Context, parentGPU string) vgpuPlacementGeometry { + key := filepath.Join(s.pciDevicesPath, parentGPU) + if cached, ok := vgpuPlacements[key]; ok && time.Since(cached.loadedAt) < vgpuPlacementTTL { + return cached + } + geometry, err := s.loadPlacementGeometry(ctx, parentGPU) + if err != nil { + logger.FromContext(ctx).WarnContext(ctx, "vGPU placement packing disabled for GPU", "gpu", parentGPU, "error", err) + } + geometry.loadedAt = time.Now() + vgpuPlacements[key] = geometry + return geometry +} + +func (s vendorVFIOSysfs) loadPlacementGeometry(ctx context.Context, parentGPU string) (vgpuPlacementGeometry, error) { + query, err := s.nvidiaSMI(ctx, "-q", "-i", parentGPU) + if err != nil { + return vgpuPlacementGeometry{}, fmt.Errorf("query GPU: %w", err) + } + heterogeneous, err := parseHeterogeneousMode(string(query)) + if err != nil { + return vgpuPlacementGeometry{}, err + } + if !heterogeneous { + return vgpuPlacementGeometry{}, errors.New("GPU is in equal-size vGPU mode") + } + supported, err := s.nvidiaSMI(ctx, "vgpu", "-s", "-v", "-i", parentGPU) + if err != nil { + return vgpuPlacementGeometry{}, fmt.Errorf("query supported vGPU types: %w", err) + } + types, err := parsePlacementTypes(string(supported)) + if err != nil { + return vgpuPlacementGeometry{}, err + } + return vgpuPlacementGeometry{types: types}, nil +} + +func (s vendorVFIOSysfs) nvidiaSMI(ctx context.Context, args ...string) ([]byte, error) { + if s.nvidiaSMIFunc != nil { + return s.nvidiaSMIFunc(ctx, args...) + } + ctx, cancel := context.WithTimeout(ctx, nvidiaSMITimeout) + defer cancel() + return exec.CommandContext(ctx, "nvidia-smi", args...).Output() +} + +// occupiedPlacements returns the placement spans of every configured VF on +// parentGPU other than vfAddress. An unreadable or unknown sibling fails the +// lookup, since a placement chosen around it could not be trusted. +func (s vendorVFIOSysfs) occupiedPlacements(parentGPU, vfAddress string, geometry vgpuPlacementGeometry) ([]placementSpan, error) { + entries, err := os.ReadDir(s.pciDevicesPath) + if err != nil { + return nil, fmt.Errorf("read PCI devices: %w", err) + } + var occupied []placementSpan + for _, entry := range entries { + if entry.Name() == vfAddress { + continue + } + vfPath := filepath.Join(s.pciDevicesPath, entry.Name()) + target, err := os.Readlink(filepath.Join(vfPath, "physfn")) + if err != nil || filepath.Base(target) != parentGPU { + continue + } + nvidiaPath := filepath.Join(vfPath, "nvidia") + if _, err := os.Stat(nvidiaPath); err != nil { + continue + } + currentType, err := readCurrentVGPUType(filepath.Join(nvidiaPath, "current_vgpu_type")) + if err != nil { + return nil, fmt.Errorf("read current vGPU type for VF %s: %w", entry.Name(), err) + } + if currentType == "0" { + continue + } + placed, ok := geometry.types[currentType] + if !ok { + return nil, fmt.Errorf("VF %s has vGPU type %s with no known placement size", entry.Name(), currentType) + } + start, err := readPlacementID(filepath.Join(nvidiaPath, "placement_id")) + if err != nil { + return nil, fmt.Errorf("read placement for VF %s: %w", entry.Name(), err) + } + occupied = append(occupied, placementSpan{start: start, size: placed.size}) + } + return occupied, nil +} + +// choosePlacement returns the free placement for requested that blocks the +// fewest free placements of larger profiles, comparing the smallest larger +// size first. Ties keep current, so the driver's placement is only overridden +// for a strictly better one. +func choosePlacement(types map[string]vgpuPlacementType, occupied []placementSpan, requested vgpuPlacementType, current int) int { + free := func(span placementSpan) bool { + for _, other := range occupied { + if span.start < other.start+other.size && other.start < span.start+span.size { + return false + } + } + return true + } + freeBySize := make(map[int]map[int]struct{}) + for _, placementType := range types { + if placementType.size <= requested.size { + continue + } + for _, id := range placementType.ids { + if !free(placementSpan{start: id, size: placementType.size}) { + continue + } + if freeBySize[placementType.size] == nil { + freeBySize[placementType.size] = make(map[int]struct{}) + } + freeBySize[placementType.size][id] = struct{}{} + } + } + sizes := make([]int, 0, len(freeBySize)) + for size := range freeBySize { + sizes = append(sizes, size) + } + sort.Ints(sizes) + blocked := func(start int) []int { + counts := make([]int, len(sizes)) + for i, size := range sizes { + for id := range freeBySize[size] { + if start < id+size && id < start+requested.size { + counts[i]++ + } + } + } + return counts + } + + best, bestBlocked := current, blocked(current) + ids := append([]int(nil), requested.ids...) + sort.Ints(ids) + for _, id := range ids { + if !free(placementSpan{start: id, size: requested.size}) { + continue + } + if candidate := blocked(id); fewerBlocked(candidate, bestBlocked) { + best, bestBlocked = id, candidate + } + } + return best +} + +func fewerBlocked(a, b []int) bool { + for i := range a { + if a[i] != b[i] { + return a[i] < b[i] + } + } + return false +} + +// parseHeterogeneousMode reads "vGPU Heterogeneous Mode : Enabled" from +// nvidia-smi -q output for a single GPU. +func parseHeterogeneousMode(output string) (bool, error) { + for _, line := range strings.Split(output, "\n") { + key, value, found := strings.Cut(line, ":") + if !found || strings.TrimSpace(key) != "vGPU Heterogeneous Mode" { + continue + } + switch strings.TrimSpace(value) { + case "Enabled": + return true, nil + case "Disabled": + return false, nil + } + return false, fmt.Errorf("unrecognized vGPU heterogeneous mode %q", strings.TrimSpace(value)) + } + return false, fmt.Errorf("vGPU heterogeneous mode not reported") +} + +// parsePlacementTypes reads each type's placement size and heterogeneous +// placement IDs from nvidia-smi vgpu -s -v output, keyed by decimal type ID +// to match current_vgpu_type. Types without numeric placement information +// are left out. +func parsePlacementTypes(output string) (map[string]vgpuPlacementType, error) { + types := make(map[string]vgpuPlacementType) + typeID := "" + var current vgpuPlacementType + flush := func() { + if typeID != "" && current.size > 0 && len(current.ids) > 0 { + types[typeID] = current + } + } + for _, line := range strings.Split(output, "\n") { + key, value, found := strings.Cut(line, ":") + if !found { + continue + } + key, value = strings.TrimSpace(key), strings.TrimSpace(value) + switch key { + case "vGPU Type ID": + flush() + id, err := strconv.ParseInt(value, 0, 64) + if err != nil { + return nil, fmt.Errorf("parse vGPU type ID %q: %w", value, err) + } + typeID, current = strconv.FormatInt(id, 10), vgpuPlacementType{} + case "Placement Size": + if size, err := strconv.Atoi(value); err == nil { + current.size = size + } + case "Heterogeneous Placement IDs": + for _, field := range strings.Fields(value) { + id, err := strconv.Atoi(field) + if err != nil { + current.ids = nil + break + } + current.ids = append(current.ids, id) + } + } + } + flush() + if len(types) == 0 { + return nil, fmt.Errorf("no vGPU placement information reported") + } + return types, nil +} + +func readPlacementID(path string) (int, error) { + data, err := os.ReadFile(path) + if err != nil { + return 0, err + } + value := strings.TrimSpace(string(data)) + id, err := strconv.Atoi(value) + if err != nil { + return 0, fmt.Errorf("invalid placement ID %q", value) + } + return id, nil +} diff --git a/lib/devices/vgpu_placement_linux_test.go b/lib/devices/vgpu_placement_linux_test.go new file mode 100644 index 000000000..b37b24cb8 --- /dev/null +++ b/lib/devices/vgpu_placement_linux_test.go @@ -0,0 +1,256 @@ +//go:build linux + +package devices + +import ( + "bytes" + "context" + "errors" + "log/slog" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "syscall" + "testing" + + "github.com/kernel/hypeman/lib/logger" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + testL40S1Q = "1147" + testL40S2Q = "1148" + testL40S4Q = "1150" +) + +const testHeterogeneousQuery = `GPU 00000000:82:00.0 + GPU Virtualization Mode + Virtualization Mode : Host VGPU + Host VGPU Mode : SR-IOV + vGPU Heterogeneous Mode : Enabled +` + +func loadL40SPlacementTypes(t *testing.T) map[string]vgpuPlacementType { + t.Helper() + output, err := os.ReadFile(filepath.Join("testdata", "nvidia-smi-vgpu-supported-l40s.txt")) + require.NoError(t, err) + types, err := parsePlacementTypes(string(output)) + require.NoError(t, err) + return types +} + +func TestParsePlacementTypes(t *testing.T) { + types := loadL40SPlacementTypes(t) + + assert.Equal(t, vgpuPlacementType{size: 2, ids: []int{0, 4, 6, 10, 12, 16, 18, 22, 24, 28, 30, 34, 36, 40, 42, 46}}, types[testL40S2Q]) + assert.Equal(t, vgpuPlacementType{size: 4, ids: []int{0, 8, 12, 20, 24, 32, 36, 44}}, types[testL40S4Q]) + assert.Equal(t, 1, types[testL40S1Q].size) +} + +func TestParseHeterogeneousMode(t *testing.T) { + enabled, err := parseHeterogeneousMode(testHeterogeneousQuery) + require.NoError(t, err) + assert.True(t, enabled) + + enabled, err = parseHeterogeneousMode(strings.Replace(testHeterogeneousQuery, "Enabled", "Disabled", 1)) + require.NoError(t, err) + assert.False(t, enabled) + + _, err = parseHeterogeneousMode("GPU 00000000:82:00.0\n") + require.Error(t, err) +} + +func placementFree(occupied []placementSpan, span placementSpan) bool { + for _, other := range occupied { + if span.start < other.start+other.size && other.start < span.start+span.size { + return false + } + } + return true +} + +// placeLikeDriver returns the lowest free placement for typeID, which is +// where the driver puts a newly configured vGPU. +func placeLikeDriver(types map[string]vgpuPlacementType, occupied []placementSpan, typeID string) (int, bool) { + requested := types[typeID] + for _, id := range requested.ids { + if placementFree(occupied, placementSpan{start: id, size: requested.size}) { + return id, true + } + } + return 0, false +} + +func free4QPlacements(types map[string]vgpuPlacementType, occupied []placementSpan) int { + count := 0 + for _, id := range types[testL40S4Q].ids { + if placementFree(occupied, placementSpan{start: id, size: 4}) { + count++ + } + } + return count +} + +func TestChoosePlacementKeeps4QRoomOnL40S(t *testing.T) { + types := loadL40SPlacementTypes(t) + + var packed, unpacked []placementSpan + for i := 0; i < 8; i++ { + start, ok := placeLikeDriver(types, unpacked, testL40S2Q) + require.True(t, ok) + unpacked = append(unpacked, placementSpan{start: start, size: 2}) + + start, ok = placeLikeDriver(types, packed, testL40S2Q) + require.True(t, ok) + start = choosePlacement(types, packed, types[testL40S2Q], start) + packed = append(packed, placementSpan{start: start, size: 2}) + } + + starts := make([]int, 0, len(packed)) + for _, span := range packed { + starts = append(starts, span.start) + } + sort.Ints(starts) + assert.Equal(t, []int{4, 6, 16, 18, 28, 30, 40, 42}, starts, "the first eight 2Q fill the placements no 4Q overlaps") + assert.Equal(t, 8, free4QPlacements(types, packed)) + assert.Equal(t, 4, free4QPlacements(types, unpacked), "the driver's lowest-free placement blocks half the 4Q room") + + // Once those are used, every further 2Q costs exactly one 4Q placement. + for i := 0; i < 8; i++ { + start, ok := placeLikeDriver(types, packed, testL40S2Q) + require.True(t, ok) + start = choosePlacement(types, packed, types[testL40S2Q], start) + packed = append(packed, placementSpan{start: start, size: 2}) + assert.Equal(t, 7-i, free4QPlacements(types, packed)) + } +} + +func TestChoosePlacementKeepsCurrentOnTie(t *testing.T) { + types := loadL40SPlacementTypes(t) + // 4Q at 0 and 8 fill the first two 4Q placements; 2Q at 4 and 6 are + // equally good, so the driver's choice stands. + occupied := []placementSpan{{start: 0, size: 4}, {start: 8, size: 4}} + + assert.Equal(t, 6, choosePlacement(types, occupied, types[testL40S2Q], 6)) + assert.Equal(t, 4, choosePlacement(types, occupied, types[testL40S2Q], 4)) +} + +// addPlacedVF adds a VF with a configured type at placement, as the driver +// leaves it after a type write. +func (s testVendorVFIOSysfs) addPlacedVF(t *testing.T, address, vfioID, currentType string, placement int) { + t.Helper() + s.addVF(t, "0000:82:00.0", address, vfioID, currentType, testCreatableTypes) + require.NoError(t, os.WriteFile(filepath.Join(s.pciDevicesPath, address, "nvidia", "placement_id"), []byte(strconv.Itoa(placement)), 0644)) +} + +func newPackingSysfs(t *testing.T, query string) (testVendorVFIOSysfs, *int) { + t.Helper() + supported, err := os.ReadFile(filepath.Join("testdata", "nvidia-smi-vgpu-supported-l40s.txt")) + require.NoError(t, err) + sysfs := newTestVendorVFIOSysfs(t) + calls := 0 + sysfs.nvidiaSMIFunc = func(_ context.Context, args ...string) ([]byte, error) { + calls++ + if args[0] == "-q" { + return []byte(query), nil + } + return supported, nil + } + return sysfs, &calls +} + +func TestVendorVFIOConfigurePacksPlacement(t *testing.T) { + sysfs, calls := newPackingSysfs(t, testHeterogeneousQuery) + // The driver put the new 2Q at the lowest free placement, 0, which + // overlaps the 4Q placement at 0. Placement 6 overlaps none. The clean + // sibling occupies nothing, and the 2Q at 6 is on another GPU. + sysfs.addPlacedVF(t, "0000:82:00.4", "42", "0", 0) + sysfs.addPlacedVF(t, "0000:82:00.5", "43", testL40S2Q, 4) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.7", "45", "0", testCreatableTypes) + sysfs.addVF(t, "0000:83:00.0", "0000:83:00.4", "46", testL40S2Q, testCreatableTypes) + require.NoError(t, os.WriteFile(filepath.Join(sysfs.pciDevicesPath, "0000:83:00.4", "nvidia", "placement_id"), []byte("6"), 0644)) + + require.NoError(t, sysfs.configure(context.Background(), "0000:82:00.4", testL40S2Q)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "placement_id"), "6") + + sysfs.addPlacedVF(t, "0000:82:00.6", "44", "0", 0) + require.NoError(t, sysfs.configure(context.Background(), "0000:82:00.6", testL40S2Q)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.6", "nvidia", "placement_id"), "16") + assert.Equal(t, 2, *calls, "geometry is loaded once per GPU") +} + +func TestVendorVFIOConfigureCachesFailedPlacementLoad(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + calls := 0 + sysfs.nvidiaSMIFunc = func(context.Context, ...string) ([]byte, error) { + calls++ + return nil, errors.New("exit status 9") + } + sysfs.addPlacedVF(t, "0000:82:00.4", "42", "0", 0) + sysfs.addPlacedVF(t, "0000:82:00.5", "43", "0", 0) + + require.NoError(t, sysfs.configure(context.Background(), "0000:82:00.4", testL40S2Q)) + require.NoError(t, sysfs.configure(context.Background(), "0000:82:00.5", testL40S2Q)) + assert.Equal(t, 1, calls, "a failed load is cached too") +} + +func TestVendorVFIOConfigureKeepsDriverPlacement(t *testing.T) { + tests := []struct { + name string + setup func(sysfs *testVendorVFIOSysfs) + logged string + }{ + { + name: "equal-size mode", + logged: "equal-size vGPU mode", + setup: func(sysfs *testVendorVFIOSysfs) { + query := strings.Replace(testHeterogeneousQuery, "Enabled", "Disabled", 1) + sysfs.nvidiaSMIFunc = func(context.Context, ...string) ([]byte, error) { return []byte(query), nil } + }, + }, + { + name: "nvidia-smi fails", + logged: "exit status 9", + setup: func(sysfs *testVendorVFIOSysfs) { + sysfs.nvidiaSMIFunc = func(context.Context, ...string) ([]byte, error) { return nil, errors.New("exit status 9") } + }, + }, + { + name: "sibling of unknown type", + logged: "no known placement size", + setup: func(sysfs *testVendorVFIOSysfs) { + sysfs.addPlacedVF(t, "0000:82:00.6", "44", "999", 8) + }, + }, + { + name: "driver refuses placement", + logged: "driver refused vGPU placement", + setup: func(sysfs *testVendorVFIOSysfs) { + sysfs.writeFileFunc = func(name string, data []byte, perm os.FileMode) error { + if filepath.Base(name) == "placement_id" { + return syscall.EIO + } + return os.WriteFile(name, data, perm) + } + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sysfs, _ := newPackingSysfs(t, testHeterogeneousQuery) + sysfs.addPlacedVF(t, "0000:82:00.4", "42", "0", 0) + sysfs.addPlacedVF(t, "0000:82:00.5", "43", testL40S2Q, 4) + tt.setup(&sysfs) + var logs bytes.Buffer + ctx := logger.AddToContext(context.Background(), slog.New(slog.NewTextHandler(&logs, nil))) + + require.NoError(t, sysfs.configure(ctx, "0000:82:00.4", testL40S2Q)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "current_vgpu_type"), testL40S2Q) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "placement_id"), "0") + assert.Contains(t, logs.String(), tt.logged) + }) + } +} diff --git a/lib/instances/create.go b/lib/instances/create.go index 504b58e65..0284d1791 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -403,6 +403,8 @@ func (m *manager) createInstance( "instance_id", id, "device_path", storedVGPUDevicePath(stored)) } }) + // On vendor VFIO hosts claimVGPU already wrote the type, so this only + // re-checks quarantine and confirms the type is still set. if err := m.configureClaimedVGPU(ctx, gpuDevice); err != nil { log.ErrorContext(ctx, "failed to configure vGPU", "profile", gpuProfile, "error", err) return nil, wrapCreateVGPUErr(gpuProfile, err) diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 2a9bb6dfb..2598b45c5 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -192,6 +192,7 @@ type manager struct { destroyVGPU func(context.Context, devices.VGPUAssignment) error reconcileVGPUDevices func(context.Context, map[string]struct{}) error vgpuAllocationMu sync.Mutex + supportedVGPUProfiles map[string]map[string]devices.VGPUProfileType // guarded by vgpuAllocationMu deleteSnapshotFn func(context.Context, string) error ttlReaperDeleteTimeout time.Duration egressProxy *egressproxy.Service diff --git a/lib/instances/metrics.go b/lib/instances/metrics.go index 8d86b6cc7..f7e3d0cfe 100644 --- a/lib/instances/metrics.go +++ b/lib/instances/metrics.go @@ -105,6 +105,7 @@ type Metrics struct { vgpuReconcileFailuresTotal metric.Int64Counter vgpuStaleReleaseFailuresTotal metric.Int64Counter vgpuLivenessUncertainTotal metric.Int64Counter + vgpuDriverRejectionsTotal metric.Int64Counter tracer trace.Tracer } @@ -305,6 +306,14 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M return nil, err } + vgpuDriverRejectionsTotal, err := meter.Int64Counter( + "hypeman_instances_vgpu_driver_rejections_total", + metric.WithDescription("Total times a GPU was skipped during vendor VFIO vGPU placement because the driver refused the type, counted once per claim and GPU"), + ) + if err != nil { + return nil, err + } + vgpuQuarantinedVFs, err := meter.Int64ObservableGauge( "hypeman_instances_vgpu_quarantined_vfs", metric.WithDescription("Number of vGPU virtual functions currently quarantined"), @@ -535,6 +544,7 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M vgpuReconcileFailuresTotal: vgpuReconcileFailuresTotal, vgpuStaleReleaseFailuresTotal: vgpuStaleReleaseFailuresTotal, vgpuLivenessUncertainTotal: vgpuLivenessUncertainTotal, + vgpuDriverRejectionsTotal: vgpuDriverRejectionsTotal, tracer: tracer, }, nil } @@ -657,6 +667,15 @@ func (m *manager) recordVGPULivenessUncertain(ctx context.Context) { m.metrics.vgpuLivenessUncertainTotal.Add(ctx, 1) } +func (m *manager) recordVGPUDriverRejection(ctx context.Context, parentGPU string) { + if m.metrics == nil { + return + } + m.metrics.vgpuDriverRejectionsTotal.Add(ctx, 1, metric.WithAttributes( + attribute.String("gpu", parentGPU), + )) +} + // recordStateTransition records a state transition with hypervisor label. func (m *manager) recordStateTransition(ctx context.Context, fromState, toState string, hvType hypervisor.Type) { if m.metrics == nil { diff --git a/lib/instances/start.go b/lib/instances/start.go index b162ad90f..f7dee4b9d 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -170,6 +170,8 @@ func (m *manager) startInstance( cu.Add(func() { m.cleanupStartVGPU(ctx, stored, rollbackMeta) }) + // On vendor VFIO hosts claimVGPU already wrote the type, so this only + // re-checks quarantine and confirms the type is still set. if err := m.configureClaimedVGPU(ctx, device); err != nil { log.ErrorContext(ctx, "failed to configure vGPU", "instance_id", id, "profile", stored.GPUProfile, "error", err) return nil, fmt.Errorf("configure vGPU for profile %s: %w", stored.GPUProfile, err) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 4563398d0..2c484ea00 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -2,9 +2,11 @@ package instances import ( "context" + "errors" "fmt" "math/rand/v2" "path/filepath" + "slices" "sort" "github.com/kernel/hypeman/lib/devices" @@ -57,6 +59,7 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str if err != nil { return nil, err } + m.rememberVGPUProfiles(vfs, profilesByVF) // Quarantine is read under the allocation lock but mutated under the // devices lock, so this is a snapshot. configure re-checks it under the // devices lock before the VF is touched. @@ -64,8 +67,28 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str if err != nil { return nil, err } - candidates := vendorVFIOCandidates{vfs: vfs, profilesByVF: profilesByVF, claims: allMetadata, quarantined: quarantined} + candidates := vendorVFIOCandidates{vfs: vfs, profilesByVF: profilesByVF, supported: m.supportedVGPUProfiles, claims: allMetadata, quarantined: quarantined} + // refreshCandidates re-reads VF state and creatable lists after dirty VFs + // were reset, keeping only the VFs still in candidates. + refreshCandidates := func() error { + _, vfs, err := m.discoverVGPUDevices() + if err != nil { + return err + } + vfs = slices.DeleteFunc(vfs, func(vf devices.VirtualFunction) bool { + _, ok := vfByAddress(candidates.vfs, vf.PCIAddress) + return !ok + }) + profilesByVF, err := listProfiles(vfs) + if err != nil { + return err + } + m.rememberVGPUProfiles(vfs, profilesByVF) + candidates.vfs, candidates.profilesByVF = vfs, profilesByVF + return nil + } vfAddress, profileType, err := selectVendorVFIOVF(candidates, profileName, m.pickVFIndex) + dirtyReset := err != nil if err != nil { // A dirty unclaimed VF consumes framebuffer, which can make the // requested profile vanish from every creatable list before the @@ -73,13 +96,9 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str if !m.resetDirtyUnclaimedVFs(ctx, vfs, allMetadata) { return nil, err } - if _, vfs, err = m.discoverVGPUDevices(); err != nil { - return nil, err - } - if candidates.profilesByVF, err = listProfiles(vfs); err != nil { + if err = refreshCandidates(); err != nil { return nil, err } - candidates.vfs = vfs if vfAddress, profileType, err = selectVendorVFIOVF(candidates, profileName, m.pickVFIndex); err != nil { return nil, err } @@ -90,25 +109,60 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str // claiming; the allocation lock keeps it unclaimed until then. A VF that // refuses its reset is dropped from the candidates so it cannot block // clean siblings. + // + // The VF is then configured here, under the allocation lock, so a claim + // cannot lose its placement to a later claim once the lock is released. + // Creatable lists are a snapshot and hide some profiles that still fit, + // so only the driver can say whether a placement remains; the next GPU is + // tried if it rejects the type. Every reselection passes through the + // dirty-VF reset again. log := logger.FromContext(ctx) for { vf, ok := vfByAddress(candidates.vfs, vfAddress) - if !ok || !vf.Allocated { - break - } - assignment := devices.VGPUAssignment{ - Framework: devices.VGPUFrameworkVendorVFIO, - DevicePath: filepath.Clean(devices.GetDeviceSysfsPath(vf.PCIAddress)), + if !ok { + return nil, fmt.Errorf("selected VF %s is not a candidate", vfAddress) } - repairErr := m.destroyVGPUAssignment(ctx, assignment) - if repairErr == nil { + if vf.Allocated { + assignment := devices.VGPUAssignment{ + Framework: devices.VGPUFrameworkVendorVFIO, + DevicePath: filepath.Clean(devices.GetDeviceSysfsPath(vf.PCIAddress)), + } + if repairErr := m.destroyVGPUAssignment(ctx, assignment); repairErr != nil { + log.WarnContext(ctx, "dirty vGPU VF refused reset; trying another VF", "vf", vf.PCIAddress, "error", repairErr) + candidates.vfs = withoutVF(candidates.vfs, vf.PCIAddress) + if vfAddress, profileType, err = selectVendorVFIOVF(candidates, profileName, m.pickVFIndex); err != nil { + return nil, fmt.Errorf("repair dirty VF %s before claim: %w", vf.PCIAddress, repairErr) + } + continue + } log.InfoContext(ctx, "reset dirty vGPU VF before claim", "vf", vf.PCIAddress) + } + placeErr := m.placeVGPU(ctx, candidates.vfs, allMetadata, vf, profileType) + if placeErr == nil { + if !advertisingGPUs(candidates, profileType)[vf.ParentGPU] { + log.InfoContext(ctx, "placed vGPU profile its GPU no longer advertises", "vf", vfAddress, "profile", profileName) + } break } - log.WarnContext(ctx, "dirty vGPU VF refused reset; trying another VF", "vf", vf.PCIAddress, "error", repairErr) - candidates.vfs = withoutVF(candidates.vfs, vf.PCIAddress) - if vfAddress, profileType, err = selectVendorVFIOVF(candidates, profileName, m.pickVFIndex); err != nil { - return nil, fmt.Errorf("repair dirty VF %s before claim: %w", vf.PCIAddress, repairErr) + if !errors.Is(placeErr, devices.ErrVGPUTypeRejected) { + return nil, placeErr + } + log.InfoContext(ctx, "driver refused vGPU profile on GPU; trying another GPU", "gpu", vf.ParentGPU, "vf", vfAddress, "profile", profileName, "error", placeErr) + m.recordVGPUDriverRejection(ctx, vf.ParentGPU) + candidates.vfs = withoutGPU(candidates.vfs, vf.ParentGPU) + vfAddress, profileType, err = selectVendorVFIOVF(candidates, profileName, m.pickVFIndex) + // A hidden candidate can satisfy the first selection while dirty VFs + // fill a GPU that has no candidate, so the dirty-VF retry above did + // not run. Run it once before giving up. + if err != nil && !dirtyReset && m.resetDirtyUnclaimedVFs(ctx, candidates.vfs, allMetadata) { + dirtyReset = true + if err = refreshCandidates(); err != nil { + return nil, err + } + vfAddress, profileType, err = selectVendorVFIOVF(candidates, profileName, m.pickVFIndex) + } + if err != nil { + return nil, fmt.Errorf("%w (driver refused it on GPU %s: %v)", err, vf.ParentGPU, placeErr) } } device := &devices.VGPUDevice{ @@ -126,6 +180,9 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str meta.GPUClaimedAt = &claimedAt if err := m.saveMetadata(meta); err != nil { clearStoredVGPUDevice(&meta.StoredMetadata) + if resetErr := m.destroyVGPUAssignment(ctx, devices.VGPUAssignment{Framework: device.Framework, DevicePath: device.SysfsPath}); resetErr != nil { + log.WarnContext(ctx, "failed to reset vGPU VF after claim save failed", "vf", vfAddress, "error", resetErr) + } return nil, fmt.Errorf("save vGPU claim: %w", err) } return device, nil @@ -165,6 +222,47 @@ func (m *manager) resetDirtyUnclaimedVFs(ctx context.Context, vfs []devices.Virt return reset } +// placeVGPU configures vf with profileType, so the driver decides whether a +// placement remains. An unclaimed dirty sibling can hold the last placement, +// so when the driver rejects the type, those siblings are reset (in-use +// gated) and the write is retried once. A VF left configured by a crash before +// its claim was saved is such a sibling for the next allocation. Callers must +// hold vgpuAllocationMu. +func (m *manager) placeVGPU(ctx context.Context, vfs []devices.VirtualFunction, allMetadata []StoredMetadata, vf devices.VirtualFunction, profileType string) error { + device := &devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + VFAddress: vf.PCIAddress, + ProfileType: profileType, + } + err := m.configureClaimedVGPU(ctx, device) + if errors.Is(err, devices.ErrVGPUTypeRejected) && m.resetDirtyUnclaimedVFs(ctx, siblingVFs(vfs, vf), allMetadata) { + err = m.configureClaimedVGPU(ctx, device) + } + return err +} + +// rememberVGPUProfiles records every profile each GPU's VFs advertise. In +// mixed-size mode the driver drops a profile from every creatable list on a +// GPU once that GPU holds as many vGPUs as the profile's equal-size instance +// limit (12 for L40S-4Q), even while a placement for it remains, so the +// current lists alone under-report what a busy GPU can still take. Callers +// must hold vgpuAllocationMu. +func (m *manager) rememberVGPUProfiles(vfs []devices.VirtualFunction, profilesByVF map[string][]devices.VGPUProfileType) { + if m.supportedVGPUProfiles == nil { + m.supportedVGPUProfiles = make(map[string]map[string]devices.VGPUProfileType) + } + for _, vf := range vfs { + for _, profile := range profilesByVF[vf.PCIAddress] { + byName := m.supportedVGPUProfiles[vf.ParentGPU] + if byName == nil { + byName = make(map[string]devices.VGPUProfileType) + m.supportedVGPUProfiles[vf.ParentGPU] = byName + } + byName[profile.Name] = profile + } + } +} + func (m *manager) quarantinedVFAddresses() (map[string]struct{}, error) { quarantined := m.quarantinedVFs if quarantined == nil { @@ -177,7 +275,8 @@ func (m *manager) quarantinedVFAddresses() (map[string]struct{}, error) { type vendorVFIOCandidates struct { vfs []devices.VirtualFunction profilesByVF map[string][]devices.VGPUProfileType - claims []StoredMetadata // every instance; those with a device path hold a VF + supported map[string]map[string]devices.VGPUProfileType // parent GPU -> profile name -> profile, as remembered by rememberVGPUProfiles + claims []StoredMetadata // every instance; those with a device path hold a VF quarantined map[string]struct{} } @@ -186,8 +285,21 @@ type vendorVFIOCandidates struct { // drifts away from cards carrying a wedged VF. Among equally ranked // candidates on the chosen GPU, pick selects the index (nil is uniform // random), so a single VF cannot capture every placement on an idle host. +// +// A GPU that supports the profile but no longer advertises it is still a +// candidate for a clean VF, ranked after every GPU that does advertise it, +// because the driver hides profiles by instance count rather than by free +// placement. The caller must confirm such a choice with the driver. A free +// VF that advertises nothing is not a candidate for this: on L40S an empty +// list means its GPU has no placement left for any size, and a VF whose list +// could not be read is unknown. func selectVendorVFIOVF(c vendorVFIOCandidates, profileName string, pick func(n int) int) (string, string, error) { profilesByName := make(map[string]devices.VGPUProfileType) + for _, byName := range c.supported { + for name, profile := range byName { + profilesByName[name] = profile + } + } advertises := make(map[string]map[string]struct{}, len(c.profilesByVF)) for vfAddress, profiles := range c.profilesByVF { advertises[vfAddress] = make(map[string]struct{}, len(profiles)) @@ -196,12 +308,24 @@ func selectVendorVFIOVF(c vendorVFIOCandidates, profileName string, pick func(n advertises[vfAddress][profile.TypeName] = struct{}{} } } + // Only VFs still in c.vfs count as advertising, so a profile whose + // advertising GPUs were all dropped after a driver rejection reports as + // not creatable. + advertisedNames := make(map[string]struct{}) + for _, vf := range c.vfs { + for _, profile := range c.profilesByVF[vf.PCIAddress] { + advertisedNames[profile.Name] = struct{}{} + } + } + notCreatable := func() error { + if len(advertisedNames) == 0 && len(c.vfs) > 0 { + return fmt.Errorf("no creatable vGPU profiles on any VF (GPUs at capacity or dirty VFs consuming framebuffer): profile %q", profileName) + } + return fmt.Errorf("profile %q is not creatable on any VF (unknown profile or insufficient capacity)", profileName) + } requested, found := profilesByName[profileName] if !found { - if len(profilesByName) == 0 && len(c.vfs) > 0 { - return "", "", fmt.Errorf("no creatable vGPU profiles on any VF (GPUs at capacity or dirty VFs consuming framebuffer): profile %q", profileName) - } - return "", "", fmt.Errorf("profile %q is not creatable on any VF (unknown profile or insufficient capacity)", profileName) + return "", "", notCreatable() } vfsByAddress := make(map[string]devices.VirtualFunction, len(c.vfs)) @@ -230,12 +354,7 @@ func selectVendorVFIOVF(c vendorVFIOCandidates, profileName string, pick func(n usageByGPU[vf.ParentGPU] += profile.FramebufferMB } - parentAdvertises := make(map[string]bool) - for _, vf := range c.vfs { - if _, ok := advertises[vf.PCIAddress][requested.TypeName]; ok { - parentAdvertises[vf.ParentGPU] = true - } - } + parentAdvertises := advertisingGPUs(c, requested.TypeName) quarantinedByGPU := make(map[string]int) freeByGPU := make(map[string][]devices.VirtualFunction) for _, vf := range c.vfs { @@ -248,7 +367,10 @@ func selectVendorVFIOVF(c vendorVFIOCandidates, profileName string, pick func(n } _, directlyCreatable := advertises[vf.PCIAddress][requested.TypeName] repairable := vf.Allocated && (vf.ProfileType == requested.TypeName || parentAdvertises[vf.ParentGPU]) - if directlyCreatable || repairable { + supported, ok := c.supported[vf.ParentGPU][profileName] + hidden := !vf.Allocated && !parentAdvertises[vf.ParentGPU] && len(advertises[vf.PCIAddress]) > 0 && + ok && supported.TypeName == requested.TypeName + if directlyCreatable || repairable || hidden { freeByGPU[vf.ParentGPU] = append(freeByGPU[vf.ParentGPU], vf) } } @@ -268,6 +390,9 @@ func selectVendorVFIOVF(c vendorVFIOCandidates, profileName string, pick func(n gpus = append(gpus, gpu) } sort.Slice(gpus, func(i, j int) bool { + if parentAdvertises[gpus[i]] != parentAdvertises[gpus[j]] { + return parentAdvertises[gpus[i]] + } if quarantinedByGPU[gpus[i]] != quarantinedByGPU[gpus[j]] { return quarantinedByGPU[gpus[i]] < quarantinedByGPU[gpus[j]] } @@ -280,6 +405,9 @@ func selectVendorVFIOVF(c vendorVFIOCandidates, profileName string, pick func(n return usageByGPU[gpus[i]] < usageByGPU[gpus[j]] }) if len(gpus) == 0 { + if _, ok := advertisedNames[profileName]; !ok { + return "", "", notCreatable() + } return "", "", fmt.Errorf("no available VF for profile %q", profileName) } // Candidates are sorted clean-first, so the leading run with the same @@ -295,6 +423,19 @@ func selectVendorVFIOVF(c vendorVFIOCandidates, profileName string, pick func(n return candidates[pick(n)].PCIAddress, requested.TypeName, nil } +func advertisingGPUs(c vendorVFIOCandidates, profileType string) map[string]bool { + gpus := make(map[string]bool) + for _, vf := range c.vfs { + for _, profile := range c.profilesByVF[vf.PCIAddress] { + if profile.TypeName == profileType { + gpus[vf.ParentGPU] = true + break + } + } + } + return gpus +} + func (m *manager) configureClaimedVGPU(ctx context.Context, device *devices.VGPUDevice) error { if device.Framework != devices.VGPUFrameworkVendorVFIO { return nil @@ -447,6 +588,26 @@ func withoutVF(vfs []devices.VirtualFunction, address string) []devices.VirtualF return remaining } +func siblingVFs(vfs []devices.VirtualFunction, vf devices.VirtualFunction) []devices.VirtualFunction { + siblings := make([]devices.VirtualFunction, 0, len(vfs)) + for _, other := range vfs { + if other.ParentGPU == vf.ParentGPU && other.PCIAddress != vf.PCIAddress { + siblings = append(siblings, other) + } + } + return siblings +} + +func withoutGPU(vfs []devices.VirtualFunction, parentGPU string) []devices.VirtualFunction { + remaining := make([]devices.VirtualFunction, 0, len(vfs)) + for _, vf := range vfs { + if vf.ParentGPU != parentGPU { + remaining = append(remaining, vf) + } + } + return remaining +} + func storedVGPUDevicePath(stored *StoredMetadata) string { if stored.GPUDevicePath != "" { return stored.GPUDevicePath diff --git a/lib/instances/vgpu_linux_test.go b/lib/instances/vgpu_linux_test.go index c93e4b2f5..f97171303 100644 --- a/lib/instances/vgpu_linux_test.go +++ b/lib/instances/vgpu_linux_test.go @@ -5,6 +5,8 @@ package instances import ( "context" "errors" + "fmt" + "os" "sync" "testing" "time" @@ -13,6 +15,8 @@ import ( "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + otelmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" ) func newVGPUAllocationManager(t *testing.T, vfs []devices.VirtualFunction) *manager { @@ -33,6 +37,7 @@ func newVGPUAllocationManager(t *testing.T, vfs []devices.VirtualFunction) *mana } return profiles, nil }, + configureVGPU: func(context.Context, string, string) error { return nil }, } } @@ -213,6 +218,7 @@ func TestVGPUClaimRepairsDirtyVFWhenProfileNotAdvertised(t *testing.T) { dirty = false return nil }, + configureVGPU: func(context.Context, string, string) error { return nil }, } meta := saveTestVGPUInstance(t, m, "new") @@ -315,3 +321,478 @@ func TestVGPUClaimFallsBackWhenDirtyVFRefusesReset(t *testing.T) { assert.Equal(t, "0000:82:00.5", device.VFAddress) assert.Equal(t, []string{testVFDevicePath, "/sys/bus/pci/devices/0000:82:00.5"}, resets) } + +// newHiddenProfileManager returns a manager on a two-GPU host whose free VFs +// advertise both profiles until hide4Q is set, after which they advertise +// only 2Q, as the driver does once a GPU holds 12 vGPUs. +func newHiddenProfileManager(t *testing.T, hide4Q *bool) *manager { + t.Helper() + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:e3:00.4", ParentGPU: "0000:e3:00.0"}, + } + return &manager{ + paths: paths.New(t.TempDir()), + discoverVGPU: func() (devices.VGPUFramework, []devices.VirtualFunction, error) { + return devices.VGPUFrameworkVendorVFIO, vfs, nil + }, + vendorVFIOProfiles: func([]devices.VirtualFunction) (map[string][]devices.VGPUProfileType, error) { + profiles := make(map[string][]devices.VGPUProfileType, len(vfs)) + for _, vf := range vfs { + profiles[vf.PCIAddress] = []devices.VGPUProfileType{testSmallProfile} + if !*hide4Q { + profiles[vf.PCIAddress] = append(profiles[vf.PCIAddress], testLargeProfile) + } + } + return profiles, nil + }, + pickVFIndex: pickFirst, + } +} + +func TestVGPUClaimPlacesProfileItsGPUHides(t *testing.T) { + hide4Q := false + m := newHiddenProfileManager(t, &hide4Q) + var configured []string + m.configureVGPU = func(_ context.Context, vfAddress, profileType string) error { + configured = append(configured, vfAddress+"="+profileType) + return nil + } + // An earlier claim on the idle host records that both GPUs support 4Q. + _, err := m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "earlier"), testVGPUProfile) + require.NoError(t, err) + require.Equal(t, []string{"0000:82:00.4=" + testVFProfileType}, configured) + + hide4Q = true + configured = nil + meta := saveTestVGPUInstance(t, m, "large") + device, err := m.claimVGPU(context.Background(), meta, testLargeVGPUProfile) + require.NoError(t, err) + assert.Equal(t, "0000:e3:00.4", device.VFAddress) + assert.Equal(t, testLargeProfileType, device.ProfileType) + assert.Equal(t, []string{"0000:e3:00.4=" + testLargeProfileType}, configured) + stored, err := m.loadMetadata("large") + require.NoError(t, err) + assert.Equal(t, "/sys/bus/pci/devices/0000:e3:00.4", stored.GPUDevicePath) +} + +func TestVGPUClaimTriesNextGPUWhenDriverRefusesHiddenProfile(t *testing.T) { + hide4Q := true + m := newHiddenProfileManager(t, &hide4Q) + m.supportedVGPUProfiles = testSupported("0000:82:00.0", "0000:e3:00.0") + reader := otelmetric.NewManualReader() + metrics, err := newInstanceMetrics(otelmetric.NewMeterProvider(otelmetric.WithReader(reader)).Meter("test"), nil, m) + require.NoError(t, err) + m.metrics = metrics + var configured []string + m.configureVGPU = func(_ context.Context, vfAddress, profileType string) error { + configured = append(configured, vfAddress) + if vfAddress == "0000:82:00.4" { + return errDriverRejected(vfAddress) + } + return nil + } + + device, err := m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "large"), testLargeVGPUProfile) + require.NoError(t, err) + assert.Equal(t, "0000:e3:00.4", device.VFAddress) + assert.Equal(t, []string{"0000:82:00.4", "0000:e3:00.4"}, configured, "a refusal rules out the whole GPU, not just the VF") + + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(context.Background(), &rm)) + rejections, ok := findMetric(t, rm, "hypeman_instances_vgpu_driver_rejections_total").Data.(metricdata.Sum[int64]) + require.True(t, ok) + require.Len(t, rejections.DataPoints, 1) + assert.Equal(t, int64(1), rejections.DataPoints[0].Value) + gpu, ok := rejections.DataPoints[0].Attributes.Value("gpu") + require.True(t, ok) + assert.Equal(t, "0000:82:00.0", gpu.AsString()) +} + +func TestVGPUClaimFailsWhenDriverRefusesHiddenProfileEverywhere(t *testing.T) { + hide4Q := true + m := newHiddenProfileManager(t, &hide4Q) + m.supportedVGPUProfiles = testSupported("0000:82:00.0", "0000:e3:00.0") + var configured []string + m.configureVGPU = func(_ context.Context, vfAddress, _ string) error { + configured = append(configured, vfAddress) + return errDriverRejected(vfAddress) + } + + _, err := m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "large"), testLargeVGPUProfile) + require.ErrorContains(t, err, "is not creatable on any VF") + require.ErrorContains(t, err, "driver refused it") + assert.Equal(t, []string{"0000:82:00.4", "0000:e3:00.4"}, configured) + stored, loadErr := m.loadMetadata("large") + require.NoError(t, loadErr) + assert.Empty(t, stored.GPUDevicePath) +} + +func TestVGPUClaimFailsWhenDriverRefusesAdvertisedProfileEverywhere(t *testing.T) { + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:e3:00.4", ParentGPU: "0000:e3:00.0"}, + } + m := newVGPUAllocationManager(t, vfs) + var configured []string + m.configureVGPU = func(_ context.Context, vfAddress, _ string) error { + configured = append(configured, vfAddress) + return errDriverRejected(vfAddress) + } + + _, err := m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "new"), testVGPUProfile) + require.ErrorContains(t, err, "is not creatable on any VF") + require.ErrorContains(t, err, "driver refused it") + assert.Len(t, configured, 2) +} + +func TestVGPUClaimFailsOnHiddenProfileConfigureErrorOtherThanRejection(t *testing.T) { + hide4Q := true + m := newHiddenProfileManager(t, &hide4Q) + m.supportedVGPUProfiles = testSupported("0000:82:00.0", "0000:e3:00.0") + var configured []string + m.configureVGPU = func(_ context.Context, vfAddress, _ string) error { + configured = append(configured, vfAddress) + return errors.New("VF health state unavailable: read failed") + } + + _, err := m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "large"), testLargeVGPUProfile) + require.ErrorContains(t, err, "VF health state unavailable") + assert.NotContains(t, err.Error(), "not creatable", "only a driver rejection says the GPU is out of placements") + assert.Equal(t, []string{"0000:82:00.4"}, configured) +} + +func TestVGPUClaimResetsDirtyVFSelectedAfterDriverRefusal(t *testing.T) { + // GPU 82 refuses the hidden 4Q, so the claim moves to GPU e3, whose only + // free VF is dirty with a leftover 4Q and still open. It must go through + // the reset like any other dirty VF instead of being claimed as is. + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:e3:00.4", ParentGPU: "0000:e3:00.0", Allocated: true, ProfileType: testLargeProfileType}, + } + var events []string + m := &manager{ + paths: paths.New(t.TempDir()), + discoverVGPU: func() (devices.VGPUFramework, []devices.VirtualFunction, error) { + return devices.VGPUFrameworkVendorVFIO, vfs, nil + }, + vendorVFIOProfiles: func([]devices.VirtualFunction) (map[string][]devices.VGPUProfileType, error) { + return map[string][]devices.VGPUProfileType{ + "0000:82:00.4": {testSmallProfile}, + "0000:e3:00.4": {}, + }, nil + }, + supportedVGPUProfiles: testSupported("0000:82:00.0", "0000:e3:00.0"), + destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { + events = append(events, "reset "+assignment.DevicePath) + return errors.New("vendor VFIO vGPU on VF 0000:e3:00.4 is still in use") + }, + configureVGPU: func(_ context.Context, vfAddress, _ string) error { + events = append(events, "configure "+vfAddress) + if vfAddress == "0000:82:00.4" { + return errDriverRejected(vfAddress) + } + return nil + }, + pickVFIndex: pickFirst, + } + + _, err := m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "large"), testLargeVGPUProfile) + require.ErrorContains(t, err, "repair dirty VF 0000:e3:00.4 before claim") + assert.Equal(t, []string{"configure 0000:82:00.4", "reset /sys/bus/pci/devices/0000:e3:00.4"}, events) + stored, loadErr := m.loadMetadata("large") + require.NoError(t, loadErr) + assert.Empty(t, stored.GPUDevicePath) +} + +func TestVGPUClaimResetsDirtySiblingHoldingHiddenPlacement(t *testing.T) { + // GPU 82 hides 4Q, and its last 4Q placement is held by an unclaimed + // dirty VF, for example one configured by a claim that failed to save. + // The clean sibling is tried first; when the driver refuses it, the dirty + // VF is reset and the write retried rather than giving up on the GPU. + dirty := true + m := &manager{ + paths: paths.New(t.TempDir()), + discoverVGPU: func() (devices.VGPUFramework, []devices.VirtualFunction, error) { + leftover := devices.VirtualFunction{PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"} + if dirty { + leftover.Allocated, leftover.ProfileType = true, testLargeProfileType + } + return devices.VGPUFrameworkVendorVFIO, []devices.VirtualFunction{ + leftover, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0"}, + }, nil + }, + vendorVFIOProfiles: func([]devices.VirtualFunction) (map[string][]devices.VGPUProfileType, error) { + return map[string][]devices.VGPUProfileType{ + "0000:82:00.4": {}, + "0000:82:00.5": {testSmallProfile}, + }, nil + }, + supportedVGPUProfiles: testSupported("0000:82:00.0"), + pickVFIndex: pickFirst, + } + var events []string + m.destroyVGPU = func(_ context.Context, assignment devices.VGPUAssignment) error { + events = append(events, "reset "+assignment.DevicePath) + dirty = false + return nil + } + m.configureVGPU = func(_ context.Context, vfAddress, _ string) error { + events = append(events, "configure "+vfAddress) + if dirty { + return errDriverRejected(vfAddress) + } + return nil + } + + device, err := m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "large"), testLargeVGPUProfile) + require.NoError(t, err) + assert.Equal(t, "0000:82:00.5", device.VFAddress) + assert.Equal(t, []string{ + "configure 0000:82:00.5", + "reset /sys/bus/pci/devices/0000:82:00.4", + "configure 0000:82:00.5", + }, events) +} + +func TestVGPUClaimKeepsClaimedSiblingWhenDriverRefuses(t *testing.T) { + // GPU 82's configured sibling belongs to another instance, so a refusal + // must not reset it to free a placement. + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0", Allocated: true, ProfileType: testVFProfileType}, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0"}, + } + var events []string + m := &manager{ + paths: paths.New(t.TempDir()), + discoverVGPU: func() (devices.VGPUFramework, []devices.VirtualFunction, error) { + return devices.VGPUFrameworkVendorVFIO, vfs, nil + }, + vendorVFIOProfiles: func([]devices.VirtualFunction) (map[string][]devices.VGPUProfileType, error) { + return map[string][]devices.VGPUProfileType{ + "0000:82:00.4": {}, + "0000:82:00.5": {testSmallProfile}, + }, nil + }, + supportedVGPUProfiles: testSupported("0000:82:00.0"), + destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { + events = append(events, "reset "+assignment.DevicePath) + return nil + }, + configureVGPU: func(_ context.Context, vfAddress, _ string) error { + events = append(events, "configure "+vfAddress) + return errDriverRejected(vfAddress) + }, + } + running := saveTestVGPUInstance(t, m, "running") + running.GPUFramework = devices.VGPUFrameworkVendorVFIO + running.GPUDevicePath = testVFDevicePath + require.NoError(t, m.saveMetadata(running)) + + _, err := m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "large"), testLargeVGPUProfile) + require.ErrorContains(t, err, "driver refused it on GPU 0000:82:00.0") + assert.Equal(t, []string{"configure 0000:82:00.5"}, events) +} + +func TestVGPUClaimResetsDirtyVFsOnOtherGPUAfterHiddenProfileRefused(t *testing.T) { + // GPU 41 hides 4Q and refuses it. GPU 82 is filled by an unclaimed dirty + // VF, so its free VF lists nothing and is no candidate. The hidden + // candidate on 41 satisfied the first selection, so the dirty VF must + // still be reset before the claim gives up, without resetting the VF + // another instance claims or retrying GPU 41. + dirty := true + var events []string + m := &manager{ + paths: paths.New(t.TempDir()), + discoverVGPU: func() (devices.VGPUFramework, []devices.VirtualFunction, error) { + leftover := devices.VirtualFunction{PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"} + if dirty { + leftover.Allocated, leftover.ProfileType = true, testVFProfileType + } + return devices.VGPUFrameworkVendorVFIO, []devices.VirtualFunction{ + {PCIAddress: "0000:41:00.4", ParentGPU: "0000:41:00.0"}, + leftover, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:82:00.6", ParentGPU: "0000:82:00.0", Allocated: true, ProfileType: testVFProfileType}, + }, nil + }, + vendorVFIOProfiles: func([]devices.VirtualFunction) (map[string][]devices.VGPUProfileType, error) { + profiles := map[string][]devices.VGPUProfileType{ + "0000:41:00.4": {testSmallProfile}, + "0000:82:00.4": {}, + "0000:82:00.5": {}, + "0000:82:00.6": {}, + } + if !dirty { + profiles["0000:82:00.5"] = []devices.VGPUProfileType{testSmallProfile} + } + return profiles, nil + }, + supportedVGPUProfiles: testSupported("0000:41:00.0", "0000:82:00.0"), + destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { + events = append(events, "reset "+assignment.DevicePath) + dirty = false + return nil + }, + configureVGPU: func(_ context.Context, vfAddress, _ string) error { + events = append(events, "configure "+vfAddress) + if vfAddress == "0000:41:00.4" { + return errDriverRejected(vfAddress) + } + return nil + }, + pickVFIndex: pickFirst, + } + running := saveTestVGPUInstance(t, m, "running") + running.GPUFramework = devices.VGPUFrameworkVendorVFIO + running.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.6" + require.NoError(t, m.saveMetadata(running)) + + device, err := m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "large"), testLargeVGPUProfile) + require.NoError(t, err) + assert.Equal(t, "0000:82:00.5", device.VFAddress) + assert.Equal(t, []string{ + "configure 0000:41:00.4", + "reset " + testVFDevicePath, + "configure 0000:82:00.5", + }, events) +} + +func TestVGPUClaimProbesRepairedVFOnceWhenDriverRefuses(t *testing.T) { + // The only candidate is a dirty VF left at 4Q on a GPU that hides 4Q. It + // is reset, then probed. A refusal must not count that same VF as a dirty + // sibling to reset and retry. + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0", Allocated: true, ProfileType: testLargeProfileType}, + } + var events []string + m := &manager{ + paths: paths.New(t.TempDir()), + discoverVGPU: func() (devices.VGPUFramework, []devices.VirtualFunction, error) { + return devices.VGPUFrameworkVendorVFIO, vfs, nil + }, + vendorVFIOProfiles: func([]devices.VirtualFunction) (map[string][]devices.VGPUProfileType, error) { + return map[string][]devices.VGPUProfileType{"0000:82:00.4": {}}, nil + }, + supportedVGPUProfiles: testSupported("0000:82:00.0"), + destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { + events = append(events, "reset "+assignment.DevicePath) + return nil + }, + configureVGPU: func(_ context.Context, vfAddress, _ string) error { + events = append(events, "configure "+vfAddress) + return errDriverRejected(vfAddress) + }, + } + + _, err := m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "large"), testLargeVGPUProfile) + require.ErrorContains(t, err, "driver refused it on GPU 0000:82:00.0") + assert.Equal(t, []string{"reset " + testVFDevicePath, "configure 0000:82:00.4"}, events) +} + +func TestVGPUClaimRemembersProfilesListedAfterDirtyReset(t *testing.T) { + // The first claim only sees the GPU's profiles after resetting a dirty + // VF that consumed its framebuffer. 4Q must still be remembered from that + // listing once the GPU later hides it. + dirty := true + hide4Q := false + m := &manager{ + paths: paths.New(t.TempDir()), + discoverVGPU: func() (devices.VGPUFramework, []devices.VirtualFunction, error) { + leftover := devices.VirtualFunction{PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"} + if dirty { + leftover.Allocated, leftover.ProfileType = true, "999" + } + return devices.VGPUFrameworkVendorVFIO, []devices.VirtualFunction{ + leftover, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0"}, + }, nil + }, + vendorVFIOProfiles: func(vfs []devices.VirtualFunction) (map[string][]devices.VGPUProfileType, error) { + profiles := make(map[string][]devices.VGPUProfileType, len(vfs)) + for _, vf := range vfs { + switch { + case dirty: + profiles[vf.PCIAddress] = []devices.VGPUProfileType{} + case hide4Q: + profiles[vf.PCIAddress] = []devices.VGPUProfileType{testSmallProfile} + default: + profiles[vf.PCIAddress] = []devices.VGPUProfileType{testSmallProfile, testLargeProfile} + } + } + return profiles, nil + }, + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + dirty = false + return nil + }, + configureVGPU: func(context.Context, string, string) error { return nil }, + pickVFIndex: pickFirst, + } + _, err := m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "small"), testVGPUProfile) + require.NoError(t, err) + + hide4Q = true + _, err = m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "large"), testLargeVGPUProfile) + require.NoError(t, err) +} + +func TestVGPUClaimConfiguresBeforeReleasingAllocationLock(t *testing.T) { + // GPU 82 has one placement left, but its free VFs still list 2Q. The + // first claim must take that placement before releasing the allocation + // lock, so the second is refused by the driver and moves to GPU e3, + // instead of both claims landing on GPU 82 and one failing later. + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:e3:00.4", ParentGPU: "0000:e3:00.0", Allocated: true, ProfileType: testVFProfileType}, + {PCIAddress: "0000:e3:00.5", ParentGPU: "0000:e3:00.0"}, + } + m := newVGPUAllocationManager(t, vfs) + m.pickVFIndex = pickFirst + running := saveTestVGPUInstance(t, m, "running") + running.GPUFramework = devices.VGPUFrameworkVendorVFIO + running.GPUDevicePath = "/sys/bus/pci/devices/0000:e3:00.4" + require.NoError(t, m.saveMetadata(running)) + placementsLeft := map[string]int{"0000:82:00.0": 1, "0000:e3:00.0": 1} + m.configureVGPU = func(_ context.Context, vfAddress, _ string) error { + vf, ok := vfByAddress(vfs, vfAddress) + require.True(t, ok) + if placementsLeft[vf.ParentGPU] == 0 { + return errDriverRejected(vfAddress) + } + placementsLeft[vf.ParentGPU]-- + return nil + } + + first, err := m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "first"), testVGPUProfile) + require.NoError(t, err) + second, err := m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "second"), testVGPUProfile) + require.NoError(t, err) + assert.Equal(t, "0000:82:00.4", first.VFAddress) + assert.Equal(t, "0000:e3:00.5", second.VFAddress) + assert.Equal(t, map[string]int{"0000:82:00.0": 0, "0000:e3:00.0": 0}, placementsLeft) +} + +func TestVGPUClaimResetsConfiguredVFWhenClaimSaveFails(t *testing.T) { + vfs := []devices.VirtualFunction{{PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}} + m := newVGPUAllocationManager(t, vfs) + var resets []string + m.destroyVGPU = func(_ context.Context, assignment devices.VGPUAssignment) error { + resets = append(resets, assignment.DevicePath) + return nil + } + meta := saveTestVGPUInstance(t, m, "new") + require.NoError(t, os.RemoveAll(m.paths.InstanceDir("new"))) + + _, err := m.claimVGPU(context.Background(), meta, testVGPUProfile) + require.ErrorContains(t, err, "save vGPU claim") + assert.Equal(t, []string{testVFDevicePath}, resets) + assert.Empty(t, meta.GPUDevicePath) +} + +func errDriverRejected(vfAddress string) error { + return fmt.Errorf("configure vGPU on VF %s: %w: invalid argument", vfAddress, devices.ErrVGPUTypeRejected) +} diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index be346dee0..ee25b6925 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -241,3 +241,182 @@ func TestSelectVendorVFIOVFPrefersGPUWithKnownLoad(t *testing.T) { require.NoError(t, err) assert.Equal(t, "0000:82:00.5", vf) } + +const ( + testLargeVGPUProfile = "NVIDIA L40S-4Q" + testLargeProfileType = "1150" +) + +var ( + testSmallProfile = devices.VGPUProfileType{TypeName: testVFProfileType, Name: testVGPUProfile, FramebufferMB: 2048} + testLargeProfile = devices.VGPUProfileType{TypeName: testLargeProfileType, Name: testLargeVGPUProfile, FramebufferMB: 4096} +) + +// testSupported reports both profiles as supported on every listed GPU. +func testSupported(gpus ...string) map[string]map[string]devices.VGPUProfileType { + supported := make(map[string]map[string]devices.VGPUProfileType, len(gpus)) + for _, gpu := range gpus { + supported[gpu] = map[string]devices.VGPUProfileType{ + testVGPUProfile: testSmallProfile, + testLargeVGPUProfile: testLargeProfile, + } + } + return supported +} + +func TestSelectVendorVFIOVFOffersSupportedProfileItsGPUHides(t *testing.T) { + // Both GPUs are busy enough that the driver hides 4Q from every + // creatable list, but both have supported it. The less loaded GPU wins. + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0", Allocated: true, ProfileType: testVFProfileType}, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0", Allocated: true, ProfileType: testVFProfileType}, + {PCIAddress: "0000:82:00.6", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:e3:00.4", ParentGPU: "0000:e3:00.0", Allocated: true, ProfileType: testVFProfileType}, + {PCIAddress: "0000:e3:00.5", ParentGPU: "0000:e3:00.0"}, + } + claims := []StoredMetadata{ + {GPUProfile: testVGPUProfile, GPUDevicePath: testVFDevicePath}, + {GPUProfile: testVGPUProfile, GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5"}, + {GPUProfile: testVGPUProfile, GPUDevicePath: "/sys/bus/pci/devices/0000:e3:00.4"}, + } + c := vendorVFIOCandidates{ + vfs: vfs, + profilesByVF: testVFProfiles("0000:82:00.6", "0000:e3:00.5"), + supported: testSupported("0000:82:00.0", "0000:e3:00.0"), + claims: claims, + } + + vf, profileType, err := selectVendorVFIOVF(c, testLargeVGPUProfile, nil) + require.NoError(t, err) + assert.Equal(t, "0000:e3:00.5", vf) + assert.Equal(t, testLargeProfileType, profileType) + + // Without a remembered profile the request fails. + c.supported = nil + _, _, err = selectVendorVFIOVF(c, testLargeVGPUProfile, nil) + require.ErrorContains(t, err, "is not creatable on any VF") +} + +func TestSelectVendorVFIOVFPrefersGPUThatAdvertisesProfile(t *testing.T) { + // GPU 82 is idle but hides 4Q in this snapshot; GPU e3 is busier but + // still advertises it. A certain placement beats a lighter load. + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:e3:00.4", ParentGPU: "0000:e3:00.0", Allocated: true, ProfileType: testVFProfileType}, + {PCIAddress: "0000:e3:00.5", ParentGPU: "0000:e3:00.0"}, + } + profiles := testVFProfiles("0000:82:00.4") + profiles["0000:e3:00.5"] = []devices.VGPUProfileType{testSmallProfile, testLargeProfile} + claims := []StoredMetadata{{GPUProfile: testVGPUProfile, GPUDevicePath: "/sys/bus/pci/devices/0000:e3:00.4"}} + + vf, _, err := selectVendorVFIOVF(vendorVFIOCandidates{ + vfs: vfs, + profilesByVF: profiles, + supported: testSupported("0000:82:00.0", "0000:e3:00.0"), + claims: claims, + }, testLargeVGPUProfile, nil) + require.NoError(t, err) + assert.Equal(t, "0000:e3:00.5", vf) +} + +func TestSelectVendorVFIOVFOffersHiddenProfileOnlyWhereSupported(t *testing.T) { + // GPU 82 is idle but has never advertised 4Q, so it is not offered it. + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:e3:00.4", ParentGPU: "0000:e3:00.0", Allocated: true, ProfileType: testVFProfileType}, + {PCIAddress: "0000:e3:00.5", ParentGPU: "0000:e3:00.0"}, + } + vf, _, err := selectVendorVFIOVF(vendorVFIOCandidates{ + vfs: vfs, + profilesByVF: testVFProfiles("0000:82:00.4", "0000:e3:00.5"), + supported: testSupported("0000:e3:00.0"), + claims: []StoredMetadata{{GPUProfile: testVGPUProfile, GPUDevicePath: "/sys/bus/pci/devices/0000:e3:00.4"}}, + }, testLargeVGPUProfile, nil) + require.NoError(t, err) + assert.Equal(t, "0000:e3:00.5", vf) +} + +func TestSelectVendorVFIOVFSkipsUnreadableVFForHiddenProfile(t *testing.T) { + // VF .4's creatable types could not be read, so it is absent from the + // listing. Its readable sibling says the GPU still has placements, but + // that says nothing about the unreadable VF itself. + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0"}, + } + vf, _, err := selectVendorVFIOVF(vendorVFIOCandidates{ + vfs: vfs, + profilesByVF: testVFProfiles("0000:82:00.5"), + supported: testSupported("0000:82:00.0"), + }, testLargeVGPUProfile, pickFirst) + require.NoError(t, err) + assert.Equal(t, "0000:82:00.5", vf) +} + +func TestSelectVendorVFIOVFCountsHiddenProfileClaimsAsKnownLoad(t *testing.T) { + // GPU 82 runs one 4Q that no VF advertises any more; GPU e3 runs three + // 2Q. The remembered 4Q makes 82's load known and lighter (4 GB vs 6 GB), + // instead of ranking 82 last for unknown load. + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0", Allocated: true, ProfileType: testLargeProfileType}, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:e3:00.4", ParentGPU: "0000:e3:00.0", Allocated: true, ProfileType: testVFProfileType}, + {PCIAddress: "0000:e3:00.5", ParentGPU: "0000:e3:00.0", Allocated: true, ProfileType: testVFProfileType}, + {PCIAddress: "0000:e3:00.6", ParentGPU: "0000:e3:00.0", Allocated: true, ProfileType: testVFProfileType}, + {PCIAddress: "0000:e3:00.7", ParentGPU: "0000:e3:00.0"}, + } + c := vendorVFIOCandidates{ + vfs: vfs, + profilesByVF: testVFProfiles("0000:82:00.5", "0000:e3:00.7"), + supported: testSupported("0000:82:00.0", "0000:e3:00.0"), + claims: []StoredMetadata{ + {GPUProfile: testLargeVGPUProfile, GPUDevicePath: testVFDevicePath}, + {GPUProfile: testVGPUProfile, GPUDevicePath: "/sys/bus/pci/devices/0000:e3:00.4"}, + {GPUProfile: testVGPUProfile, GPUDevicePath: "/sys/bus/pci/devices/0000:e3:00.5"}, + {GPUProfile: testVGPUProfile, GPUDevicePath: "/sys/bus/pci/devices/0000:e3:00.6"}, + }, + } + + vf, _, err := selectVendorVFIOVF(c, testVGPUProfile, nil) + require.NoError(t, err) + assert.Equal(t, "0000:82:00.5", vf) + + c.supported = nil + vf, _, err = selectVendorVFIOVF(c, testVGPUProfile, nil) + require.NoError(t, err) + assert.Equal(t, "0000:e3:00.7", vf, "without the remembered profile 82's load is unknown") +} + +func TestSelectVendorVFIOVFSkipsGPUThatAdvertisesNothing(t *testing.T) { + // A free VF that advertises no profile at all means the GPU has no + // placement left for any size, so it is not worth asking the driver. + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0", Allocated: true, ProfileType: testVFProfileType}, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0"}, + } + _, _, err := selectVendorVFIOVF(vendorVFIOCandidates{ + vfs: vfs, + profilesByVF: map[string][]devices.VGPUProfileType{"0000:82:00.5": {}}, + supported: testSupported("0000:82:00.0"), + claims: []StoredMetadata{{GPUProfile: testVGPUProfile, GPUDevicePath: testVFDevicePath}}, + }, testLargeVGPUProfile, nil) + require.ErrorContains(t, err, "no creatable vGPU profiles on any VF") +} + +func TestSelectVendorVFIOVFOffersHiddenProfileOnlyOnCleanVF(t *testing.T) { + // GPU 82 hides 4Q and its only free VF is dirty. A configured VF lists + // nothing on L40S, but a hidden profile is only offered on a clean VF + // regardless of what the dirty one lists; claimVGPU resets unclaimed + // dirty VFs and selects again when this fails. + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0", Allocated: true, ProfileType: testVFProfileType}, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0", Allocated: true, ProfileType: testVFProfileType}, + } + _, _, err := selectVendorVFIOVF(vendorVFIOCandidates{ + vfs: vfs, + profilesByVF: testVFProfiles("0000:82:00.5"), + supported: testSupported("0000:82:00.0"), + claims: []StoredMetadata{{GPUProfile: testVGPUProfile, GPUDevicePath: testVFDevicePath}}, + }, testLargeVGPUProfile, nil) + require.ErrorContains(t, err, "is not creatable on any VF") +}