diff --git a/go.mod b/go.mod index 6b7ea0e6e..f16d68c01 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/distribution/reference v0.6.0 github.com/docker/distribution v2.8.3+incompatible github.com/docker/go-units v0.5.0 + github.com/ebitengine/purego v0.11.2 github.com/getkin/kin-openapi v0.133.0 github.com/ghodss/yaml v1.0.0 github.com/go-chi/chi/v5 v5.3.0 diff --git a/go.sum b/go.sum index 9d2531db9..851573b04 100644 --- a/go.sum +++ b/go.sum @@ -176,6 +176,8 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ebitengine/purego v0.11.2 h1:WWiTx8rD8zYdOwSj8O/NiNbsOvSAgJMTlyV1XlbUSeU= +github.com/ebitengine/purego v0.11.2/go.mod h1:DCHPP08djqhNSoTfImcnHYQRZmd0qhakvrozqaEYhGQ= github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 683a506a6..33e54354b 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -59,6 +59,8 @@ curl -s http://localhost:4973/resources | jq .gpu } ``` +On vendor VFIO hosts, `available` is how many more vGPUs of that profile fit if only that profile is added: per GPU, the smaller of its free placements for the profile and its free, non-quarantined VFs. Counts for different profiles are not additive, because their placements overlap. + ### Creating an Instance with vGPU Request a vGPU by specifying the profile name: @@ -92,10 +94,10 @@ On an mdev host, the response also includes the assigned mdev UUID: ### Ephemeral vGPU Lifecycle -vGPU assignments are created on instance start and released on stop or delete. Hypeman creates/removes an mdev on mdev hosts. On vendor VFIO hosts, it first reserves a VF by persisting its path in instance metadata, then writes the profile ID to `current_vgpu_type`. Metadata is the authoritative VF claim. +vGPU assignments are created on instance start and released on stop or delete. Hypeman creates/removes an mdev on mdev hosts. On vendor VFIO hosts, it selects a VF on a GPU with a free placement for the profile and writes the profile ID to `current_vgpu_type` under the allocation lock, so the placement is taken before the next claim checks for one, then reserves the VF by persisting its path in instance metadata. Metadata is the authoritative VF claim; a configured VF without one is dirty and is reset by the reconciler or the allocator. ``` -Instance Create → Persist VF claim → Configure profile → Attach VF to VM → Instance Running +Instance Create → Configure profile → Persist VF claim → Attach VF to VM → Instance Running Instance Stop/Delete → Reset profile → Remove VF claim → VF available again ``` @@ -104,7 +106,9 @@ itself down and wait for it. Delete normally caps that wait at 2s, but guests with a vGPU or passthrough device get 5s so the guest can unload its GPU driver before the hypervisor is signalled. -Hypeman reconciles metadata claims once at startup and every minute afterward, skipping hosts without GPUs. A claim whose VMM is confirmed dead is reset before the claim is removed. An ambiguous hypervisor ownership check preserves the claim, logs a warning, and increments `hypeman_instances_vgpu_liveness_uncertain_total`; the create and start cleanup paths preserve and count the same way. mdev hosts also sweep orphaned device-level assignments. Vendor VFIO hosts repair an unclaimed dirty VF when the allocator next selects it; repair checks for open VFIO handles before resetting `current_vgpu_type`. The allocator prefers VFs that are already clean, and a dirty VF that refuses its reset is skipped in favor of another candidate. +Hypeman reconciles metadata claims once at startup and every minute afterward, skipping hosts without GPUs. A claim whose VMM is confirmed dead is reset before the claim is removed. An ambiguous hypervisor ownership check preserves the claim, logs a warning, and increments `hypeman_instances_vgpu_liveness_uncertain_total`; the create and start cleanup paths preserve and count the same way. mdev hosts also sweep orphaned device-level assignments. Vendor VFIO hosts reset unclaimed dirty VFs on every reconcile pass, which covers a restart between a claim writing the type and saving the claim, or a failed claim whose reset also failed, but also clears a type written to `current_vgpu_type` by hand unless the VF has an open VFIO handle. A reset that is refused is retried on the next pass and increments `hypeman_instances_vgpu_reconcile_failures_total{stage="reset_unclaimed_vfs"}`. The allocator also repairs an unclaimed dirty VF when it next selects it; repair checks for open VFIO handles before resetting `current_vgpu_type`. The allocator prefers VFs that are already clean, and a dirty VF that refuses its reset is skipped in favor of another candidate. + +Vendor VFIO placement and `/resources` both read the driver's creatable placements through NVML (`nvmlDeviceGetVgpuTypeCreatablePlacements`, the query behind `nvidia-smi vgpu -c -v`), which list the free placement IDs of every profile each GPU can still create. Hypeman loads `libnvidia-ml.so.1` at runtime on first use and keeps it initialized, which holds the GPU device nodes open (stop hypeman before the SR-IOV cycle below); a query for every profile on two L40S GPUs takes about 20 ms. A free VF can take any profile its GPU lists there, and a claim only writes a type the driver has just reported a free placement for; if that configure still fails, the VF is reset and the claim fails without trying another GPU. The per-VF `creatable_vgpu_types` lists are not used for this because in mixed-size (heterogeneous) mode they under-report: the driver drops a profile from them once its GPU holds as many vGPUs as that profile's equal-size instance limit (12 for L40S-4Q), even while a placement for it remains. If NVML cannot be loaded or initialized or takes longer than 5 s, placement and `/resources` fall back to those lists for every GPU; if it fails for one GPU, or reports a GPU not in mixed-size mode, only that GPU falls back. Profiles the lists hide are refused on a GPU that has fallen back until NVML recovers. Queries run one at a time and wait for each other within the same 5 s, so callers queued behind a hung query never enter the driver. A query that times out is abandoned rather than waited for, since a call stuck in the driver cannot be interrupted; until it returns, queries skip NVML and use the lists directly, so a hung driver costs one timeout rather than one per claim. Each fallback increments `hypeman_resources_gpu_nvml_fallbacks_total` with `reason` `error`, `equal_size` (every GPU NVML failed for is in equal-size mode, as after a driver reload or GPU reset), `timeout` or `stuck`. A warning is logged when a fallback starts or its error changes, and an info line once NVML answers for every GPU again, rather than on every query. Admission only checks free VFs, so it does not query NVML. `/resources` does not query it either: it counts profiles from cached placements, loaded at startup, refreshed in the background after every vendor VFIO configure or reset, and refreshed again on every reconcile pass. Free and quarantined VFs are read fresh on every request, so a claim or release changes the counts at once; the placements it frees or takes on a mixed-size GPU show once the background refresh lands, about 20 ms later. ### Hypervisor Support @@ -394,7 +398,8 @@ SR-IOV on the parent GPU (this destroys and recreates all of its VFs, so it requires no vGPU assignments on that GPU). Quiescing the services that hold the GPU open is not optional: with `nv-hostengine`/`dcgm-exporter` or `nvidia-persistenced` attached, `sriov-manage -d` fails with `Cannot obtain -unbindLock` on first contact. +unbindLock` on first contact. Hypeman holds the same device nodes through +NVML, so stop it too. Any manual edit to `vf-health.json` needs an immediate hypeman restart: the store loads only at startup, and a failure report landing first re-persists @@ -414,14 +419,15 @@ GPU; once none remain, run the cycle below. ```bash # 1. Quiesce the services holding the GPU (required for the unbind lock). -systemctl stop nvidia-dcgm-exporter nvidia-dcgm nvidia-persistenced +# Hypeman holds it through NVML; stopping it leaves running VMs alone. +systemctl stop hypeman nvidia-dcgm-exporter nvidia-dcgm nvidia-persistenced # 2. Cycle SR-IOV on the parent GPU. /usr/lib/nvidia/sriov-manage -d /usr/lib/nvidia/sriov-manage -e # 3. Restart the quiesced services. -systemctl start nvidia-persistenced nvidia-dcgm nvidia-dcgm-exporter +systemctl start nvidia-persistenced nvidia-dcgm nvidia-dcgm-exporter hypeman ``` After the cycle, remove the card's entries from `vf-health.json`, restart, @@ -442,6 +448,7 @@ Check the files for the framework detected on the host: cat /sys/class/mdev_bus/*/mdev_supported_types/*/available_instances # vendor VFIO +nvidia-smi vgpu -c -v cat /sys/bus/pci/devices/*/nvidia/creatable_vgpu_types cat /sys/bus/pci/devices/*/nvidia/current_vgpu_type ``` diff --git a/lib/devices/mdev_darwin.go b/lib/devices/mdev_darwin.go index f0f009d34..60f401725 100644 --- a/lib/devices/mdev_darwin.go +++ b/lib/devices/mdev_darwin.go @@ -40,6 +40,14 @@ func ListVendorVFIOProfileTypes(vfs []VirtualFunction) (map[string][]VGPUProfile return nil, ErrVGPUNotSupportedOnMacOS } +// RefreshVGPUCapacity does nothing on macOS. +func RefreshVGPUCapacity() {} + +// NVMLFallbacks returns nil on macOS. +func NVMLFallbacks() map[string]int64 { + return nil +} + func ConfigureVGPU(ctx context.Context, vfAddress, profileType string) error { return ErrVGPUNotSupportedOnMacOS } diff --git a/lib/devices/nvml_linux.go b/lib/devices/nvml_linux.go new file mode 100644 index 000000000..b60771e7d --- /dev/null +++ b/lib/devices/nvml_linux.go @@ -0,0 +1,231 @@ +//go:build linux + +package devices + +import ( + "errors" + "fmt" + "runtime" + "strconv" + "strings" + "sync" + "unsafe" + + "github.com/ebitengine/purego" +) + +// nvmlLibrary is the NVML library the NVIDIA driver installs. It is loaded +// at runtime, so hosts without the driver never need it. +var nvmlLibrary = "libnvidia-ml.so.1" + +const ( + nvmlSuccess = 0 + nvmlInsufficientSize = 7 + nvmlFeatureEnabled = 1 + // nvmlHeterogeneousPlacements is NVML_VGPU_PGPU_HETEROGENEOUS_MODE. + nvmlHeterogeneousPlacements = 0 +) + +// nvmlPlacementList is nvmlVgpuPlacementList_v2_t. +type nvmlPlacementList struct { + version uint32 + placementSize uint32 + count uint32 + placementIDs *uint32 + mode uint32 +} + +// nvmlHeterogeneousMode is nvmlVgpuHeterogeneousMode_v1_t. +type nvmlHeterogeneousMode struct { + version uint32 + mode uint32 +} + +// nvmlStructVersion is NVML_STRUCT_VERSION. +func nvmlStructVersion(size uintptr, version uint32) uint32 { + return uint32(size) | version<<24 +} + +type nvmlFuncs struct { + errorString func(rc int32) string + deviceGetHandleByPCIBusID func(busID string, device *uintptr) int32 + deviceGetHeterogeneousMode func(device uintptr, mode *nvmlHeterogeneousMode) int32 + deviceGetSupportedVGPUs func(device uintptr, count *uint32, typeIDs *uint32) int32 + deviceGetCreatablePlacements func(device uintptr, typeID uint32, list *nvmlPlacementList) int32 + vgpuTypeGetName func(typeID uint32, name *byte, size *uint32) int32 +} + +var ( + nvmlMu sync.Mutex + nvml *nvmlFuncs +) + +// openNVML loads and initializes NVML on first use and keeps it for the life +// of the process. A failure is not kept, so a driver that loads later is +// picked up by the next query. +func openNVML() (*nvmlFuncs, error) { + nvmlMu.Lock() + defer nvmlMu.Unlock() + if nvml != nil { + return nvml, nil + } + lib, err := purego.Dlopen(nvmlLibrary, purego.RTLD_NOW|purego.RTLD_GLOBAL) + if err != nil { + return nil, fmt.Errorf("load %s: %w", nvmlLibrary, err) + } + var f nvmlFuncs + var initialize func() int32 + // purego.RegisterLibFunc panics on a missing symbol, which an older + // driver could cause, so each symbol is looked up first. + for name, fn := range map[string]any{ + "nvmlInit_v2": &initialize, + "nvmlErrorString": &f.errorString, + "nvmlDeviceGetHandleByPciBusId_v2": &f.deviceGetHandleByPCIBusID, + "nvmlDeviceGetVgpuHeterogeneousMode": &f.deviceGetHeterogeneousMode, + "nvmlDeviceGetSupportedVgpus": &f.deviceGetSupportedVGPUs, + "nvmlDeviceGetVgpuTypeCreatablePlacements": &f.deviceGetCreatablePlacements, + "nvmlVgpuTypeGetName": &f.vgpuTypeGetName, + } { + sym, err := purego.Dlsym(lib, name) + if err != nil { + purego.Dlclose(lib) + return nil, fmt.Errorf("load %s from %s: %w", name, nvmlLibrary, err) + } + purego.RegisterFunc(fn, sym) + } + if rc := initialize(); rc != nvmlSuccess { + err := fmt.Errorf("nvmlInit_v2: %s", f.errorString(rc)) + purego.Dlclose(lib) + return nil, err + } + nvml = &f + return nvml, nil +} + +// nvmlCreatableProfiles asks NVML which profiles each GPU can still create +// and at which placements, keyed by GPU address. This is the query behind +// nvidia-smi vgpu -c -v. It fails for a GPU not in mixed-size mode, so +// such a GPU keeps the sysfs lists. A GPU that fails is left out of the +// result and its error joined into the returned one. +func nvmlCreatableProfiles(gpus []string) (map[string][]creatableProfile, error) { + f, err := openNVML() + if err != nil { + return nil, err + } + byGPU := make(map[string][]creatableProfile, len(gpus)) + var errs []error + for _, gpu := range gpus { + profiles, err := f.creatableProfiles(gpu) + if err != nil { + errs = append(errs, fmt.Errorf("GPU %s: %w", gpu, err)) + continue + } + byGPU[gpu] = profiles + } + return byGPU, errors.Join(errs...) +} + +func (f *nvmlFuncs) creatableProfiles(gpu string) ([]creatableProfile, error) { + var device uintptr + if err := f.check("nvmlDeviceGetHandleByPciBusId_v2", f.deviceGetHandleByPCIBusID(gpu, &device)); err != nil { + return nil, err + } + mode := nvmlHeterogeneousMode{version: nvmlStructVersion(unsafe.Sizeof(nvmlHeterogeneousMode{}), 1)} + if err := f.check("nvmlDeviceGetVgpuHeterogeneousMode", f.deviceGetHeterogeneousMode(device, &mode)); err != nil { + return nil, err + } + if mode.mode != nvmlFeatureEnabled { + return nil, errNVMLEqualSize + } + typeIDs, err := f.supportedTypes(device) + if err != nil { + return nil, err + } + profiles := make([]creatableProfile, 0) + for _, typeID := range typeIDs { + placements, err := f.creatablePlacements(device, typeID) + if err != nil { + return nil, err + } + if len(placements) == 0 { + continue + } + name, err := f.typeName(typeID) + if err != nil { + return nil, err + } + profiles = append(profiles, creatableProfile{ + VGPUProfileType: VGPUProfileType{ + TypeName: strconv.FormatUint(uint64(typeID), 10), + Name: name, + FramebufferMB: framebufferFromProfileName(name), + }, + placements: placements, + }) + } + return profiles, nil +} + +func (f *nvmlFuncs) supportedTypes(device uintptr) ([]uint32, error) { + typeIDs := make([]uint32, 64) + count := uint32(len(typeIDs)) + rc := f.deviceGetSupportedVGPUs(device, &count, &typeIDs[0]) + if rc == nvmlInsufficientSize { + typeIDs = make([]uint32, count) + rc = f.deviceGetSupportedVGPUs(device, &count, &typeIDs[0]) + } + if err := f.check("nvmlDeviceGetSupportedVgpus", rc); err != nil { + return nil, err + } + return typeIDs[:count], nil +} + +func (f *nvmlFuncs) creatablePlacements(device uintptr, typeID uint32) ([]int, error) { + ids := make([]uint32, 64) + list, rc := f.placementList(device, typeID, ids) + if rc == nvmlInsufficientSize { + ids = make([]uint32, list.count) + list, rc = f.placementList(device, typeID, ids) + } + if err := f.check("nvmlDeviceGetVgpuTypeCreatablePlacements", rc); err != nil { + return nil, err + } + placements := make([]int, 0, list.count) + for _, id := range ids[:list.count] { + placements = append(placements, int(id)) + } + return placements, nil +} + +func (f *nvmlFuncs) placementList(device uintptr, typeID uint32, ids []uint32) (nvmlPlacementList, int32) { + // The list carries a pointer to ids, so ids must not move while NVML + // writes to it. + var pinner runtime.Pinner + pinner.Pin(&ids[0]) + defer pinner.Unpin() + list := nvmlPlacementList{ + version: nvmlStructVersion(unsafe.Sizeof(nvmlPlacementList{}), 2), + count: uint32(len(ids)), + placementIDs: &ids[0], + mode: nvmlHeterogeneousPlacements, + } + rc := f.deviceGetCreatablePlacements(device, typeID, &list) + return list, rc +} + +func (f *nvmlFuncs) typeName(typeID uint32) (string, error) { + name := make([]byte, 96) + size := uint32(len(name)) + if err := f.check("nvmlVgpuTypeGetName", f.vgpuTypeGetName(typeID, &name[0], &size)); err != nil { + return "", err + } + value, _, _ := strings.Cut(string(name), "\x00") + return value, nil +} + +func (f *nvmlFuncs) check(call string, rc int32) error { + if rc == nvmlSuccess { + return nil + } + return fmt.Errorf("%s: %s", call, f.errorString(rc)) +} diff --git a/lib/devices/types.go b/lib/devices/types.go index 67352132b..049bd3757 100644 --- a/lib/devices/types.go +++ b/lib/devices/types.go @@ -121,7 +121,7 @@ type MdevDevice struct { type GPUProfile struct { Name string `json:"name"` // user-facing name, e.g., "L40S-1Q" FramebufferMB int `json:"framebuffer_mb"` // frame buffer size in MB - Available int `json:"available"` // number of VFs that can create this profile + Available int `json:"available"` // more instances that fit if only this profile is added } // PassthroughDevice describes a physical GPU available for passthrough diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 2fb552095..4e0e2c774 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -27,6 +27,7 @@ type vendorVFIOSysfs struct { procPath string vfioDevicesPath string openVFIOPathsFunc func() (map[string]struct{}, error) + creatableFunc func(gpus []string) (map[string][]creatableProfile, error) } var hostVendorVFIO = vendorVFIOSysfs{ @@ -90,28 +91,62 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { return vfs, nil } -// listProfiles counts each free, non-quarantined VF advertising a type as -// one creatable instance, matching the driver-reported units that mdev sums -// through available_instances. This is a best-effort snapshot because -// creating on one VF may revoke the type from siblings that share its GPU -// framebuffer. -func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction, quarantined map[string]struct{}) ([]GPUProfile, error) { - profilesByType := make(map[string]VGPUProfileType) - creatableVFs := make(map[string]int) - profilesByVF, err := s.profileTypes(vfs) +// creatableSnapshot is what the driver reports vendor VFIO profile counts +// are computed from: NVML's creatable placements for each GPU it answered +// for, and the sysfs creatable lists of the VFs on every other GPU. +type creatableSnapshot struct { + byGPU map[string][]creatableProfile + sysfsByVF map[string][]VGPUProfileType +} + +func (s vendorVFIOSysfs) creatableSnapshot(vfs []VirtualFunction) (creatableSnapshot, error) { + creatable, sysfsVFs := s.creatableOrSysfs(vfs) + sysfsByVF, err := s.sysfsProfileTypes(sysfsVFs) if err != nil { - return nil, err + return creatableSnapshot{}, err } + return creatableSnapshot{byGPU: creatable, sysfsByVF: sysfsByVF}, nil +} + +// profiles counts, per GPU, the smaller of its free placements for each +// profile and its free, non-quarantined VFs. Counts for different profiles +// are not additive, because their placements overlap. On a GPU counted from +// the sysfs lists, each free, non-quarantined VF advertising a type is one +// creatable instance, matching the driver-reported units that mdev sums +// through available_instances; that is best-effort, because creating on one +// VF may revoke the type from siblings that share its GPU framebuffer. vfs +// may be newer than the snapshot, so a VF claimed or released since it was +// taken already counts as such. +func (c creatableSnapshot) profiles(vfs []VirtualFunction, quarantined map[string]struct{}) []GPUProfile { + profilesByType := make(map[string]VGPUProfileType) + available := make(map[string]int) + freeVFs := make(map[string]int) for _, vf := range vfs { _, bad := quarantined[vf.PCIAddress] - for _, profile := range profilesByVF[vf.PCIAddress] { + free := !vf.Allocated && !bad + if free { + freeVFs[vf.ParentGPU]++ + } + if _, ok := c.byGPU[vf.ParentGPU]; ok { + continue + } + for _, profile := range c.sysfsByVF[vf.PCIAddress] { profilesByType[profile.TypeName] = profile - if !vf.Allocated && !bad { - creatableVFs[profile.TypeName]++ + if free { + available[profile.TypeName]++ } } } + for gpu, profiles := range c.byGPU { + for _, profile := range profiles { + profilesByType[profile.TypeName] = profile.VGPUProfileType + available[profile.TypeName] += min(len(profile.placements), freeVFs[gpu]) + } + } + return gpuProfiles(profilesByType, available) +} +func gpuProfiles(profilesByType map[string]VGPUProfileType, available map[string]int) []GPUProfile { metadata := make([]VGPUProfileType, 0, len(profilesByType)) for _, profile := range profilesByType { metadata = append(metadata, profile) @@ -123,13 +158,40 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction, quarantined map[str profiles = append(profiles, GPUProfile{ Name: profile.Name, FramebufferMB: profile.FramebufferMB, - Available: creatableVFs[profile.TypeName], + Available: available[profile.TypeName], }) } - return profiles, nil + return profiles } +// profileTypes returns the profiles each VF can be configured with. A free +// VF can take any profile its GPU has a free placement for, as NVML reports +// it, and a configured VF takes none. The sysfs creatable lists are only the +// fallback for GPUs NVML cannot answer for, because in mixed-size mode they +// hide profiles that still fit on a busy GPU. func (s vendorVFIOSysfs) profileTypes(vfs []VirtualFunction) (map[string][]VGPUProfileType, error) { + creatable, sysfsVFs := s.creatableOrSysfs(vfs) + profilesByVF, err := s.sysfsProfileTypes(sysfsVFs) + if err != nil { + return nil, err + } + for _, vf := range vfs { + gpuCreatable, ok := creatable[vf.ParentGPU] + if !ok { + continue + } + profiles := make([]VGPUProfileType, 0) + if !vf.Allocated { + for _, profile := range gpuCreatable { + profiles = append(profiles, profile.VGPUProfileType) + } + } + profilesByVF[vf.PCIAddress] = profiles + } + return profilesByVF, nil +} + +func (s vendorVFIOSysfs) sysfsProfileTypes(vfs []VirtualFunction) (map[string][]VGPUProfileType, error) { profilesByVF := make(map[string][]VGPUProfileType, len(vfs)) for _, vf := range vfs { profiles, err := s.readCreatableProfiles(vf.PCIAddress) diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 3c7438532..a03df7a9b 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -4,6 +4,7 @@ package devices import ( "context" + "errors" "os" "path/filepath" "testing" @@ -153,9 +154,22 @@ func newTestVendorVFIOSysfs(t *testing.T) testVendorVFIOSysfs { pciDevicesPath: pci, procPath: proc, vfioDevicesPath: vfio, + creatableFunc: func([]string) (map[string][]creatableProfile, error) { + return nil, errors.New("NVML unavailable") + }, }} } +// listProfiles counts profiles from a fresh driver query, as a capacity +// cache load does. +func (s testVendorVFIOSysfs) listProfiles(vfs []VirtualFunction, quarantined map[string]struct{}) ([]GPUProfile, error) { + snapshot, err := s.creatableSnapshot(vfs) + if err != nil { + return nil, err + } + return snapshot.profiles(vfs, quarantined), nil +} + func (s testVendorVFIOSysfs) addVF(t *testing.T, parent, address, vfioID, currentType, creatableTypes string) { t.Helper() parentPath := filepath.Join(s.pciDevicesPath, parent) diff --git a/lib/devices/vgpu_capacity_linux.go b/lib/devices/vgpu_capacity_linux.go new file mode 100644 index 000000000..74a949f5b --- /dev/null +++ b/lib/devices/vgpu_capacity_linux.go @@ -0,0 +1,103 @@ +//go:build linux + +package devices + +import ( + "fmt" + "sync" + "sync/atomic" +) + +// vendorVFIOCapacity holds the driver's placement data for vendor VFIO +// profile counts, so /resources reads memory rather than querying NVML. It +// is loaded on first use or by the vGPU reconciler at startup, refreshed in +// the background after every vendor VFIO configure or reset, and refreshed +// again on every reconcile pass. Counts are recomputed on each read against +// the caller's VFs and quarantine, so a claimed or released VF counts at +// once; placements, which only the driver can recompute for mixed sizes, +// catch up when the background refresh lands. +var vendorVFIOCapacity = newCapacityCache(func() (creatableSnapshot, error) { + return hostVendorVFIO.loadCapacity() +}) + +type capacityCache struct { + load func() (creatableSnapshot, error) + + // refreshMu serializes loads, so an older snapshot never replaces a + // newer one. + refreshMu sync.Mutex + // queued is set while a background refresh waits to start its load. A + // change made before the load starts is seen by it, so later requests + // join it. + queued atomic.Bool + + mu sync.RWMutex + loaded bool + snapshot creatableSnapshot + err error +} + +func newCapacityCache(load func() (creatableSnapshot, error)) *capacityCache { + return &capacityCache{load: load} +} + +func (s vendorVFIOSysfs) loadCapacity() (creatableSnapshot, error) { + vfs, err := s.discoverVFs() + if err != nil { + return creatableSnapshot{}, fmt.Errorf("discover vendor VFIO VFs: %w", err) + } + return s.creatableSnapshot(vfs) +} + +// profiles returns the profile counts for vfs from the cached placements, +// loading them first if nothing has been loaded yet or the last load +// failed, so a transient failure costs one read, not every read until the +// next refresh. +func (c *capacityCache) profiles(vfs []VirtualFunction, quarantined map[string]struct{}) ([]GPUProfile, error) { + c.mu.RLock() + loaded := c.loaded && c.err == nil + c.mu.RUnlock() + if !loaded { + c.refresh() + } + c.mu.RLock() + defer c.mu.RUnlock() + if c.err != nil { + return nil, c.err + } + return c.snapshot.profiles(vfs, quarantined), nil +} + +func (c *capacityCache) refresh() { + c.refreshMu.Lock() + defer c.refreshMu.Unlock() + c.loadLocked() +} + +// refreshAsync reloads the placements in the background. +func (c *capacityCache) refreshAsync() { + if !c.queued.CompareAndSwap(false, true) { + return + } + go func() { + c.refreshMu.Lock() + defer c.refreshMu.Unlock() + c.queued.Store(false) + c.loadLocked() + }() +} + +func (c *capacityCache) loadLocked() { + snapshot, err := c.load() + c.mu.Lock() + defer c.mu.Unlock() + c.snapshot, c.err, c.loaded = snapshot, err, true +} + +// RefreshVGPUCapacity reloads the vendor VFIO placements /resources counts +// from. It does nothing on hosts without vendor VFIO vGPUs. +func RefreshVGPUCapacity() { + if framework, _, err := DiscoverVGPU(); err == nil && framework == VGPUFrameworkVendorVFIO { + vendorVFIOCapacity.refresh() + } +} diff --git a/lib/devices/vgpu_capacity_linux_test.go b/lib/devices/vgpu_capacity_linux_test.go new file mode 100644 index 000000000..71b96317a --- /dev/null +++ b/lib/devices/vgpu_capacity_linux_test.go @@ -0,0 +1,119 @@ +//go:build linux + +package devices + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func testCapacityVFs() []VirtualFunction { + return []VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:e3:00.4", ParentGPU: "0000:e3:00.0"}, + } +} + +func TestCapacityCacheReadsDoNotQueryTheDriver(t *testing.T) { + var loads atomic.Int32 + cache := newCapacityCache(func() (creatableSnapshot, error) { + loads.Add(1) + return creatableSnapshot{byGPU: testCreatablePlacements()}, nil + }) + vfs := testCapacityVFs() + + profiles, err := cache.profiles(vfs, nil) + require.NoError(t, err) + // 4Q: min(2 placements, 2 VFs) on GPU 82 + min(5 placements, 1 VF) on e3. + assert.Equal(t, 3, profileAvailability(profiles, "NVIDIA L40S-4Q")) + + vfs[0].Allocated = true + profiles, err = cache.profiles(vfs, map[string]struct{}{"0000:e3:00.4": {}}) + require.NoError(t, err) + assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-4Q"), + "a claimed or quarantined VF counts before the placements are reloaded") + assert.Equal(t, int32(1), loads.Load(), "only the first read loads") +} + +func TestCapacityCacheReloadsAfterFailedLoad(t *testing.T) { + fail := true + var loads atomic.Int32 + cache := newCapacityCache(func() (creatableSnapshot, error) { + loads.Add(1) + if fail { + return creatableSnapshot{}, errors.New("read PCI devices: transient") + } + return creatableSnapshot{byGPU: testCreatablePlacements()}, nil + }) + cache.refresh() + _, err := cache.profiles(testCapacityVFs(), nil) + require.ErrorContains(t, err, "transient") + + fail = false + profiles, err := cache.profiles(testCapacityVFs(), nil) + require.NoError(t, err, "a read after a failed load loads again rather than returning the stale error") + assert.Equal(t, 3, profileAvailability(profiles, "NVIDIA L40S-4Q")) + assert.Equal(t, int32(3), loads.Load()) +} + +func TestCapacityCacheBackgroundRefreshesCoalesce(t *testing.T) { + started := make(chan struct{}, 10) + release := make(chan struct{}) + var loads atomic.Int32 + cache := newCapacityCache(func() (creatableSnapshot, error) { + n := loads.Add(1) + started <- struct{}{} + <-release + if n == 1 { + return creatableSnapshot{}, nil + } + return creatableSnapshot{byGPU: testCreatablePlacements()}, nil + }) + + cache.refreshAsync() + <-started + // A change made while a load runs needs one more load, however many + // refreshes it requests. + for range 5 { + cache.refreshAsync() + } + close(release) + require.Eventually(t, func() bool { return loads.Load() == 2 }, 5*time.Second, time.Millisecond) + require.Eventually(t, func() bool { + profiles, err := cache.profiles(testCapacityVFs(), nil) + return err == nil && profileAvailability(profiles, "NVIDIA L40S-4Q") == 3 + }, 5*time.Second, time.Millisecond, "the newer load replaces the older one") + time.Sleep(20 * time.Millisecond) + assert.Equal(t, int32(2), loads.Load()) +} + +func TestVendorVFIOConfigureAndDestroyRefreshCapacity(t *testing.T) { + resetVFHealthStore(t) + sysfs := newTestVendorVFIOSysfs(t) + sysfs.useCreatablePlacements(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + previousSysfs, previousCache := hostVendorVFIO, vendorVFIOCapacity + hostVendorVFIO = sysfs.vendorVFIOSysfs + var loads atomic.Int32 + vendorVFIOCapacity = newCapacityCache(func() (creatableSnapshot, error) { + snapshot, err := hostVendorVFIO.loadCapacity() + loads.Add(1) + return snapshot, err + }) + t.Cleanup(func() { hostVendorVFIO, vendorVFIOCapacity = previousSysfs, previousCache }) + + require.NoError(t, ConfigureVGPU(context.Background(), "0000:82:00.4", "1150")) + require.Eventually(t, func() bool { return loads.Load() == 1 }, 5*time.Second, time.Millisecond) + require.NoError(t, DestroyVGPU(context.Background(), VGPUAssignment{ + Framework: VGPUFrameworkVendorVFIO, + DevicePath: GetDeviceSysfsPath("0000:82:00.4"), + })) + require.Eventually(t, func() bool { return loads.Load() == 2 }, 5*time.Second, time.Millisecond) +} diff --git a/lib/devices/vgpu_creatable_linux.go b/lib/devices/vgpu_creatable_linux.go new file mode 100644 index 000000000..c03e82e24 --- /dev/null +++ b/lib/devices/vgpu_creatable_linux.go @@ -0,0 +1,202 @@ +//go:build linux + +package devices + +import ( + "errors" + "fmt" + "log/slog" + "sort" + "sync" + "sync/atomic" + "time" +) + +// nvmlTimeout bounds the creatable-placements query, which takes about 20 ms +// on a two-GPU L40S host and stayed under 140 ms with every host core +// saturated. +var nvmlTimeout = 5 * time.Second + +var ( + errNVMLTimeout = errors.New("NVML query timed out") + errNVMLStuck = errors.New("an earlier NVML query is still running after its timeout") + errNVMLEqualSize = errors.New("not in mixed-size vGPU mode") +) + +// nvmlStuck counts timed-out NVML queries that have not returned. +var nvmlStuck atomic.Int32 + +// nvmlFallbacks counts creatable-placement queries answered from the sysfs +// lists for at least one GPU, by reason. +var nvmlFallbacks = map[string]*atomic.Int64{ + "error": {}, + "equal_size": {}, + "timeout": {}, + "stuck": {}, +} + +// NVMLFallbacks returns how many creatable-placement queries have fallen back +// to the sysfs lists for at least one GPU since startup, by reason: NVML +// failed (error), every GPU it failed for is in equal-size mode (equal_size), +// it timed out (timeout), or an earlier timed-out query had not returned yet +// (stuck). +func NVMLFallbacks() map[string]int64 { + counts := make(map[string]int64, len(nvmlFallbacks)) + for reason, count := range nvmlFallbacks { + counts[reason] = count.Load() + } + return counts +} + +// nvmlFallbackLogged is the fallback error last logged, or "" while NVML +// answers for every GPU. +var nvmlFallbackLogged struct { + sync.Mutex + err string +} + +// recordNVMLFallback counts every fallback but logs only when the fallback +// starts or its error changes, since a host stuck on the sysfs lists would +// otherwise log on every claim and resources read. +func recordNVMLFallback(err error) { + reason := "error" + switch { + case errors.Is(err, errNVMLStuck): + reason = "stuck" + case errors.Is(err, errNVMLTimeout): + reason = "timeout" + case onlyEqualSize(err): + reason = "equal_size" + } + nvmlFallbacks[reason].Add(1) + nvmlFallbackLogged.Lock() + defer nvmlFallbackLogged.Unlock() + if nvmlFallbackLogged.err == err.Error() { + return + } + nvmlFallbackLogged.err = err.Error() + slog.Default().Warn("falling back to sysfs creatable vGPU types", "reason", reason, "error", err) +} + +func recordNVMLRecovered() { + nvmlFallbackLogged.Lock() + defer nvmlFallbackLogged.Unlock() + if nvmlFallbackLogged.err == "" { + return + } + nvmlFallbackLogged.err = "" + slog.Default().Info("NVML creatable vGPU placements available for every GPU again") +} + +// onlyEqualSize reports whether every GPU NVML failed for was left out only +// because it is in equal-size mode, which happens when a driver reload or +// GPU reset drops mixed-size mode, rather than because NVML failed. +func onlyEqualSize(err error) bool { + var joined interface{ Unwrap() []error } + if !errors.As(err, &joined) { + return errors.Is(err, errNVMLEqualSize) + } + for _, gpuErr := range joined.Unwrap() { + if !errors.Is(gpuErr, errNVMLEqualSize) { + return false + } + } + return true +} + +// creatableProfile is a profile a GPU can still create, with the placement +// IDs a new vGPU of that profile could start at. +type creatableProfile struct { + VGPUProfileType + placements []int +} + +// creatableProfiles asks the driver, through NVML, which profiles the GPU of +// each VF can still create and at which placements, keyed by parent GPU +// address. Unlike the per-VF creatable_vgpu_types lists, this stays accurate +// on a busy GPU in mixed-size mode; see GPU.md. GPUs NVML could not answer +// for are missing from the result, and the error says why. +func (s vendorVFIOSysfs) creatableProfiles(vfs []VirtualFunction) (map[string][]creatableProfile, error) { + query := s.creatableFunc + if query == nil { + query = nvmlCreatableProfiles + } + seen := make(map[string]struct{}) + gpus := make([]string, 0) + for _, vf := range vfs { + if _, ok := seen[vf.ParentGPU]; !ok { + seen[vf.ParentGPU] = struct{}{} + gpus = append(gpus, vf.ParentGPU) + } + } + sort.Strings(gpus) + byGPU, err := runBounded(func() (map[string][]creatableProfile, error) { return query(gpus) }) + if err != nil { + return byGPU, fmt.Errorf("query creatable vGPU placements: %w", err) + } + return byGPU, nil +} + +// creatableOrSysfs returns the NVML creatable profiles by GPU and the VFs +// whose GPU NVML could not answer for, which fall back to the sysfs lists. +func (s vendorVFIOSysfs) creatableOrSysfs(vfs []VirtualFunction) (map[string][]creatableProfile, []VirtualFunction) { + creatable, err := s.creatableProfiles(vfs) + if err != nil { + recordNVMLFallback(err) + } else { + recordNVMLRecovered() + } + sysfsVFs := make([]VirtualFunction, 0) + for _, vf := range vfs { + if _, ok := creatable[vf.ParentGPU]; !ok { + sysfsVFs = append(sysfsVFs, vf) + } + } + return creatable, sysfsVFs +} + +// nvmlSlot admits one NVML query at a time. A query takes it before it +// starts and gives it back only when the driver returns, so callers queued +// behind a hung query never enter the driver themselves. Concurrent queries +// on one initialization take no less time than serialized ones. +var nvmlSlot = make(chan struct{}, 1) + +// runBounded returns at nvmlTimeout even if query has not: a call stuck in +// the driver cannot be interrupted. The timeout covers waiting for +// nvmlSlot. Until a timed-out query returns, later calls fail at once +// instead of each waiting out the timeout, which the claim would pay while +// holding the vGPU allocation lock. +func runBounded[T any](query func() (T, error)) (T, error) { + var zero T + if nvmlStuck.Load() > 0 { + return zero, errNVMLStuck + } + timer := time.NewTimer(nvmlTimeout) + defer timer.Stop() + select { + case nvmlSlot <- struct{}{}: + case <-timer.C: + return zero, errNVMLStuck + } + type result struct { + value T + err error + } + done := make(chan result, 1) + go func() { + value, err := query() + <-nvmlSlot + done <- result{value, err} + }() + select { + case r := <-done: + return r.value, r.err + case <-timer.C: + } + nvmlStuck.Add(1) + go func() { + <-done + nvmlStuck.Add(-1) + }() + return zero, fmt.Errorf("%w after %s", errNVMLTimeout, nvmlTimeout) +} diff --git a/lib/devices/vgpu_creatable_linux_test.go b/lib/devices/vgpu_creatable_linux_test.go new file mode 100644 index 000000000..6aaac0b6f --- /dev/null +++ b/lib/devices/vgpu_creatable_linux_test.go @@ -0,0 +1,374 @@ +//go:build linux + +package devices + +import ( + "bytes" + "errors" + "fmt" + "log/slog" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// testCreatablePlacements is what NVML reported on a host with two L40S GPUs +// in mixed-size mode, trimmed to three profiles. GPU 82 holds twelve vGPUs +// (six 4Q, six 2Q), so its creatable_vgpu_types lists hide 4Q although +// placements 36 and 44 are free. GPU e3 holds four 2Q and two 4Q. +func testCreatablePlacements() map[string][]creatableProfile { + twoQ := VGPUProfileType{TypeName: "1148", Name: "NVIDIA L40S-2Q", FramebufferMB: 2048} + fourQ := VGPUProfileType{TypeName: "1150", Name: "NVIDIA L40S-4Q", FramebufferMB: 4096} + twelveQ := VGPUProfileType{TypeName: "1153", Name: "NVIDIA L40S-12Q", FramebufferMB: 12288} + return map[string][]creatableProfile{ + "0000:82:00.0": { + {twoQ, []int{36, 40, 42, 46}}, + {fourQ, []int{36, 44}}, + {twelveQ, []int{36}}, + }, + "0000:e3:00.0": { + {twoQ, []int{18, 22, 24, 28, 30, 34, 36, 40, 42, 46}}, + {fourQ, []int{20, 24, 32, 36, 44}}, + {twelveQ, []int{24, 36}}, + }, + } +} + +func (s *testVendorVFIOSysfs) useCreatablePlacements(t *testing.T) { + t.Helper() + s.creatableFunc = func(gpus []string) (map[string][]creatableProfile, error) { + byGPU := make(map[string][]creatableProfile, len(gpus)) + for _, gpu := range gpus { + profiles, ok := testCreatablePlacements()[gpu] + require.True(t, ok, "unexpected GPU %s", gpu) + byGPU[gpu] = profiles + } + return byGPU, nil + } +} + +func setNVMLTimeout(t *testing.T, timeout time.Duration) { + t.Helper() + previous := nvmlTimeout + nvmlTimeout = timeout + t.Cleanup(func() { nvmlTimeout = previous }) +} + +func waitForStuckNVML(t *testing.T) { + t.Helper() + require.Eventually(t, func() bool { return nvmlStuck.Load() == 0 }, 10*time.Second, 10*time.Millisecond) +} + +func TestNVMLQueryFailsWithoutUsableLibrary(t *testing.T) { + previous := nvmlLibrary + nvmlMu.Lock() + loaded := nvml + nvml = nil + nvmlMu.Unlock() + t.Cleanup(func() { + nvmlLibrary = previous + nvmlMu.Lock() + nvml = loaded + nvmlMu.Unlock() + }) + + nvmlLibrary = "libnvidia-ml-missing.so.1" + _, err := nvmlCreatableProfiles([]string{"0000:82:00.0"}) + require.ErrorContains(t, err, "load libnvidia-ml-missing.so.1") + + // A library without the NVML symbols, as an old driver could ship, is + // an error rather than a panic. + nvmlLibrary = "libc.so.6" + _, err = nvmlCreatableProfiles([]string{"0000:82:00.0"}) + require.ErrorContains(t, err, "from libc.so.6") + assert.Nil(t, nvml) +} + +func TestNVMLFallbackLogsOnlyWhenItChanges(t *testing.T) { + var logs bytes.Buffer + previous := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&logs, nil))) + nvmlFallbackLogged.Lock() + nvmlFallbackLogged.err = "" + nvmlFallbackLogged.Unlock() + t.Cleanup(func() { + slog.SetDefault(previous) + nvmlFallbackLogged.Lock() + nvmlFallbackLogged.err = "" + nvmlFallbackLogged.Unlock() + }) + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + var queryErr error + sysfs.creatableFunc = func(gpus []string) (map[string][]creatableProfile, error) { + if queryErr != nil { + return nil, queryErr + } + return map[string][]creatableProfile{"0000:82:00.0": testCreatablePlacements()["0000:82:00.0"]}, nil + } + before := NVMLFallbacks() + + equalSize := errors.Join(fmt.Errorf("GPU 0000:82:00.0: %w", errNVMLEqualSize)) + failed := errors.Join(errors.New("GPU 0000:82:00.0: nvmlDeviceGetSupportedVgpus: Unknown Error")) + for _, err := range []error{equalSize, equalSize, failed, failed, nil, nil, equalSize} { + queryErr = err + _, err := sysfs.profileTypes(vfs) + require.NoError(t, err) + } + + lines := strings.Split(strings.TrimSpace(logs.String()), "\n") + require.Len(t, lines, 4, logs.String()) + assert.Contains(t, lines[0], "reason=equal_size") + assert.Contains(t, lines[1], "reason=error") + assert.Contains(t, lines[2], "available for every GPU again") + assert.Contains(t, lines[3], "reason=equal_size") + after := NVMLFallbacks() + assert.Equal(t, before["equal_size"]+3, after["equal_size"], "every fallback is counted") + assert.Equal(t, before["error"]+2, after["error"]) +} + +func TestVendorVFIOProfileTypesUseCreatablePlacements(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + sysfs.useCreatablePlacements(t) + var queried []string + query := sysfs.creatableFunc + sysfs.creatableFunc = func(gpus []string) (map[string][]creatableProfile, error) { + queried = gpus + return query(gpus) + } + const hiddenFromSysfs = "ID : vGPU Name\n1148 : NVIDIA L40S-2Q\n" + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "ID : vGPU Name\n") + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", hiddenFromSysfs) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + profilesByVF, err := sysfs.profileTypes(vfs) + require.NoError(t, err) + assert.Equal(t, []string{"0000:82:00.0"}, queried, "each parent GPU is queried once") + assert.Empty(t, profilesByVF["0000:82:00.4"], "a configured VF takes no profile") + assert.Len(t, profilesByVF["0000:82:00.5"], 3) + assert.Contains(t, profilesByVF["0000:82:00.5"], VGPUProfileType{TypeName: "1150", Name: "NVIDIA L40S-4Q", FramebufferMB: 4096}) +} + +func TestVendorVFIOProfileTypesFallBackToSysfs(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + before := NVMLFallbacks()["error"] + + sysfs.creatableFunc = func([]string) (map[string][]creatableProfile, error) { + return nil, errors.New("nvmlInit_v2: Driver Not Loaded") + } + profilesByVF, err := sysfs.profileTypes(vfs) + require.NoError(t, err) + assert.Equal(t, []VGPUProfileType{ + {TypeName: "1147", Name: "NVIDIA L40S-1Q", FramebufferMB: 1024}, + {TypeName: "1148", Name: "NVIDIA L40S-2Q", FramebufferMB: 2048}, + {TypeName: "1159", Name: "NVIDIA L40S-48Q", FramebufferMB: 49152}, + }, profilesByVF["0000:82:00.4"]) + assert.Equal(t, before+1, NVMLFallbacks()["error"]) +} + +func TestVendorVFIOProfileTypesSkipNVMLWhileAQueryIsStuck(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + setNVMLTimeout(t, 50*time.Millisecond) + before := NVMLFallbacks() + + release := make(chan struct{}) + var calls atomic.Int32 + sysfs.creatableFunc = func([]string) (map[string][]creatableProfile, error) { + calls.Add(1) + <-release + return nil, errors.New("driver gone") + } + t.Cleanup(func() { + select { + case <-release: + default: + close(release) + } + waitForStuckNVML(t) + }) + + start := time.Now() + profilesByVF, err := sysfs.profileTypes(vfs) + require.NoError(t, err) + assert.Less(t, time.Since(start), time.Second) + assert.Len(t, profilesByVF["0000:82:00.4"], 3, "falls back to the sysfs list") + + _, err = sysfs.profileTypes(vfs) + require.NoError(t, err) + assert.Equal(t, int32(1), calls.Load(), "no second query while the first is stuck") + after := NVMLFallbacks() + assert.Equal(t, before["timeout"]+1, after["timeout"]) + assert.Equal(t, before["stuck"]+1, after["stuck"]) + + close(release) + waitForStuckNVML(t) + sysfs.useCreatablePlacements(t) + profilesByVF, err = sysfs.profileTypes(vfs) + require.NoError(t, err) + assert.Contains(t, profilesByVF["0000:82:00.4"], VGPUProfileType{TypeName: "1153", Name: "NVIDIA L40S-12Q", FramebufferMB: 12288}, + "queries NVML again once the stuck query returns") +} + +func TestNVMLQueriesStartOneAtATime(t *testing.T) { + setNVMLTimeout(t, 100*time.Millisecond) + release := make(chan struct{}) + var entered atomic.Int32 + hung := func() (int, error) { + entered.Add(1) + <-release + return 0, nil + } + t.Cleanup(func() { + close(release) + waitForStuckNVML(t) + }) + + errs := make(chan error, 5) + for range 5 { + go func() { + _, err := runBounded(hung) + errs <- err + }() + } + var timeouts, stuck int + for range 5 { + err := <-errs + switch { + case errors.Is(err, errNVMLTimeout): + timeouts++ + case errors.Is(err, errNVMLStuck): + stuck++ + } + } + assert.Equal(t, 1, timeouts) + assert.Equal(t, 4, stuck) + assert.Equal(t, int32(1), entered.Load(), "callers behind a hung query never enter the driver") +} + +func TestNVMLQueriesWaitForEachOther(t *testing.T) { + var active, overlaps atomic.Int32 + query := func() (int, error) { + if active.Add(1) > 1 { + overlaps.Add(1) + } + time.Sleep(5 * time.Millisecond) + active.Add(-1) + return 1, nil + } + + var wg sync.WaitGroup + for range 10 { + wg.Go(func() { + value, err := runBounded(query) + assert.NoError(t, err) + assert.Equal(t, 1, value) + }) + } + wg.Wait() + assert.Zero(t, overlaps.Load()) +} + +func TestVendorVFIOListProfilesCountsFreePlacements(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:82:00.7") + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.useCreatablePlacements(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "ID : vGPU Name\n") + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.6", "44", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.7", "45", "0", testCreatableTypes) + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "46", "0", testCreatableTypes) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + availability, err := GetVGPUAvailability(VGPUFrameworkVendorVFIO, vfs) + require.NoError(t, err) + profiles, err := sysfs.listProfiles(vfs, availability.Quarantined) + require.NoError(t, err) + // GPU 82 has two free healthy VFs; GPU e3 has one. + // 4Q: min(2 placements, 2 VFs) + min(5 placements, 1 VF). + assert.Equal(t, 3, profileAvailability(profiles, "NVIDIA L40S-4Q")) + // 2Q: min(4 placements, 2 VFs) + min(10 placements, 1 VF). + assert.Equal(t, 3, profileAvailability(profiles, "NVIDIA L40S-2Q")) + // 12Q: min(1 placement, 2 VFs) + min(2 placements, 1 VF). + assert.Equal(t, 2, profileAvailability(profiles, "NVIDIA L40S-12Q")) +} + +func TestVendorVFIOFallsBackToSysfsOnlyForGPUsNVMLFails(t *testing.T) { + resetVFHealthStore(t) + sysfs := newTestVendorVFIOSysfs(t) + sysfs.creatableFunc = func(gpus []string) (map[string][]creatableProfile, error) { + return map[string][]creatableProfile{"0000:82:00.0": testCreatablePlacements()["0000:82:00.0"]}, + errors.Join(fmt.Errorf("GPU 0000:e3:00.0: %w", errNVMLEqualSize)) + } + const hiddenFromSysfs = "ID : vGPU Name\n1148 : NVIDIA L40S-2Q\n" + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", hiddenFromSysfs) + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "43", "0", testCreatableTypes) + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + before := NVMLFallbacks() + + profilesByVF, err := sysfs.profileTypes(vfs) + require.NoError(t, err) + assert.Contains(t, profilesByVF["0000:82:00.4"], VGPUProfileType{TypeName: "1150", Name: "NVIDIA L40S-4Q", FramebufferMB: 4096}, + "the GPU NVML answered for keeps the profile sysfs hides") + assert.Equal(t, []VGPUProfileType{ + {TypeName: "1147", Name: "NVIDIA L40S-1Q", FramebufferMB: 1024}, + {TypeName: "1148", Name: "NVIDIA L40S-2Q", FramebufferMB: 2048}, + {TypeName: "1159", Name: "NVIDIA L40S-48Q", FramebufferMB: 49152}, + }, profilesByVF["0000:e3:00.4"], "the failed GPU uses its sysfs list") + after := NVMLFallbacks() + assert.Equal(t, before["equal_size"]+1, after["equal_size"]) + assert.Equal(t, before["error"], after["error"]) + + profiles, err := sysfs.listProfiles(vfs, nil) + require.NoError(t, err) + // 2Q: min(4 placements, 1 VF) on GPU 82 plus one advertising VF on e3. + assert.Equal(t, 2, profileAvailability(profiles, "NVIDIA L40S-2Q")) + assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-4Q")) + assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-48Q")) +} + +func TestNVMLFallbackReasons(t *testing.T) { + equalSize := fmt.Errorf("GPU 0000:e3:00.0: %w", errNVMLEqualSize) + failed := errors.New("GPU 0000:82:00.0: nvmlDeviceGetSupportedVgpus: Unknown Error") + for _, tc := range []struct { + name string + err error + reason string + }{ + {"every failed GPU is in equal-size mode", errors.Join(equalSize, equalSize), "equal_size"}, + {"one failed GPU is not in equal-size mode", errors.Join(equalSize, failed), "error"}, + {"NVML cannot load", errors.New("load libnvidia-ml.so.1: not found"), "error"}, + {"timed out", errNVMLTimeout, "timeout"}, + {"stuck", errNVMLStuck, "stuck"}, + } { + t.Run(tc.name, func(t *testing.T) { + before := NVMLFallbacks() + recordNVMLFallback(fmt.Errorf("query creatable vGPU placements: %w", tc.err)) + after := NVMLFallbacks() + for reason := range after { + want := before[reason] + if reason == tc.reason { + want++ + } + assert.Equal(t, want, after[reason], reason) + } + }) + } +} diff --git a/lib/devices/vgpu_linux.go b/lib/devices/vgpu_linux.go index d53659efd..b5f69511b 100644 --- a/lib/devices/vgpu_linux.go +++ b/lib/devices/vgpu_linux.go @@ -46,13 +46,14 @@ func ListGPUProfiles() ([]GPUProfile, error) { } // ListGPUProfilesWithVFs returns available profiles for discovered VFs. -// Quarantined VFs are excluded from vendor VFIO counts. +// Quarantined VFs are excluded from vendor VFIO counts, which come from the +// cached placements rather than a driver query. func ListGPUProfilesWithVFs(framework VGPUFramework, vfs []VirtualFunction, quarantined map[string]struct{}) ([]GPUProfile, error) { switch framework { case VGPUFrameworkMdev: return listMdevGPUProfilesWithVFs(vfs) case VGPUFrameworkVendorVFIO: - return hostVendorVFIO.listProfiles(vfs, quarantined) + return vendorVFIOCapacity.profiles(vfs, quarantined) default: return nil, nil } @@ -88,8 +89,11 @@ func ListVendorVFIOProfileTypes(vfs []VirtualFunction) (map[string][]VGPUProfile return hostVendorVFIO.profileTypes(vfs) } -// ConfigureVGPU idempotently configures a claimed vendor VFIO VF. +// ConfigureVGPU idempotently configures a claimed vendor VFIO VF. A failed +// configure may still have written the type, so the cached placements are +// refreshed either way. func ConfigureVGPU(ctx context.Context, vfAddress, profileType string) error { + defer vendorVFIOCapacity.refreshAsync() return hostVendorVFIO.configure(ctx, vfAddress, profileType) } @@ -110,6 +114,7 @@ func DestroyVGPU(ctx context.Context, assignment VGPUAssignment) error { } return DestroyMdev(ctx, mdevUUID) case VGPUFrameworkVendorVFIO: + defer vendorVFIOCapacity.refreshAsync() return hostVendorVFIO.destroy(ctx, filepath.Base(assignment.DevicePath), assignment.InstanceID) case VGPUFrameworkNone: return nil diff --git a/lib/instances/create.go b/lib/instances/create.go index ee14d15fd..90a268a12 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -410,6 +410,8 @@ func (m *manager) createInstance( "instance_id", id, "device_path", storedVGPUDevicePath(stored)) } }) + // On vendor VFIO hosts claimVGPU already wrote the type, so this only + // re-checks quarantine and confirms the type is still set. if err := m.configureClaimedVGPU(ctx, gpuDevice); err != nil { log.ErrorContext(ctx, "failed to configure vGPU", "profile", gpuProfile, "error", err) return nil, wrapCreateVGPUErr(gpuProfile, err) diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 0b39814f0..63acb682a 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -194,6 +194,7 @@ type manager struct { pickVFIndex func(n int) int destroyVGPU func(context.Context, devices.VGPUAssignment) error reconcileVGPUDevices func(context.Context, map[string]struct{}) error + refreshVGPUCapacity func() vgpuAllocationMu sync.Mutex deleteSnapshotFn func(context.Context, string) error ttlReaperDeleteTimeout time.Duration diff --git a/lib/instances/metrics.go b/lib/instances/metrics.go index 8d86b6cc7..02bd0fc8d 100644 --- a/lib/instances/metrics.go +++ b/lib/instances/metrics.go @@ -77,8 +77,9 @@ const lifecycleEventDropReasonBufferFull lifecycleEventDropReason = "buffer_full type vgpuReconcileStage string const ( - vgpuReconcileStageListInstances vgpuReconcileStage = "list_instances" - vgpuReconcileStageReconcileDevices vgpuReconcileStage = "reconcile_devices" + vgpuReconcileStageListInstances vgpuReconcileStage = "list_instances" + vgpuReconcileStageReconcileDevices vgpuReconcileStage = "reconcile_devices" + vgpuReconcileStageResetUnclaimedVFs vgpuReconcileStage = "reset_unclaimed_vfs" ) // Metrics holds the metrics instruments for instance operations. @@ -283,7 +284,7 @@ func newInstanceMetrics(meter metric.Meter, tracer trace.Tracer, m *manager) (*M vgpuReconcileFailuresTotal, err := meter.Int64Counter( "hypeman_instances_vgpu_reconcile_failures_total", - metric.WithDescription("Total number of vGPU reconcile pass stages that failed, leaving stale assignments or device leftovers allocated while /resources still advertises the capacity"), + metric.WithDescription("Total number of vGPU reconcile pass stages that failed, leaving stale assignments or device leftovers allocated"), ) if err != nil { return nil, err diff --git a/lib/instances/start.go b/lib/instances/start.go index 763ed44c1..203c9f5d6 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -181,6 +181,8 @@ func (m *manager) startInstance( cu.Add(func() { m.cleanupStartVGPU(ctx, stored, rollbackMeta) }) + // On vendor VFIO hosts claimVGPU already wrote the type, so this only + // re-checks quarantine and confirms the type is still set. if err := m.configureClaimedVGPU(ctx, device); err != nil { log.ErrorContext(ctx, "failed to configure vGPU", "instance_id", id, "profile", stored.GPUProfile, "error", err) return nil, fmt.Errorf("configure vGPU for profile %s: %w", stored.GPUProfile, err) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 4563398d0..3d4c4b3f3 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -70,7 +70,7 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str // A dirty unclaimed VF consumes framebuffer, which can make the // requested profile vanish from every creatable list before the // selector's repair path can resolve it. Reset such VFs and retry. - if !m.resetDirtyUnclaimedVFs(ctx, vfs, allMetadata) { + if reset, _ := m.resetDirtyUnclaimedVFs(ctx, vfs, allMetadata); !reset { return nil, err } if _, vfs, err = m.discoverVGPUDevices(); err != nil { @@ -118,6 +118,18 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str ProfileName: profileName, SysfsPath: filepath.Clean(devices.GetDeviceSysfsPath(vfAddress)), } + assignment := devices.VGPUAssignment{Framework: device.Framework, DevicePath: device.SysfsPath} + // The type is written under the allocation lock, so the placement the + // driver reported as free is taken before the next claim asks again. + // The write may have landed before a later check failed, and without a + // saved claim the VF would hold its placement until the allocator next + // repairs it, so a failed configure resets it. + if err := m.configureClaimedVGPU(ctx, device); err != nil { + if resetErr := m.destroyVGPUAssignment(ctx, assignment); resetErr != nil { + log.WarnContext(ctx, "failed to reset vGPU VF after configure failed", "vf", vfAddress, "error", resetErr) + } + return nil, err + } setStoredVGPUDevice(&meta.StoredMetadata, device) // Only vendor VFIO claims carry an assignment identity, so the claim time // is set here rather than in setStoredVGPUDevice. clearStoredVGPUDevice @@ -126,6 +138,9 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str meta.GPUClaimedAt = &claimedAt if err := m.saveMetadata(meta); err != nil { clearStoredVGPUDevice(&meta.StoredMetadata) + if resetErr := m.destroyVGPUAssignment(ctx, assignment); resetErr != nil { + log.WarnContext(ctx, "failed to reset vGPU VF after claim save failed", "vf", vfAddress, "error", resetErr) + } return nil, fmt.Errorf("save vGPU claim: %w", err) } return device, nil @@ -134,8 +149,9 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str // resetDirtyUnclaimedVFs resets allocated VFs that no instance claims, // restoring the framebuffer their leftover type consumes. The reset is // gated on the open-VFIO-handle in-use check inside destroy. Reports -// whether any VF was reset. Callers must hold vgpuAllocationMu. -func (m *manager) resetDirtyUnclaimedVFs(ctx context.Context, vfs []devices.VirtualFunction, allMetadata []StoredMetadata) bool { +// whether any VF was reset and whether any refused. Callers must hold +// vgpuAllocationMu. +func (m *manager) resetDirtyUnclaimedVFs(ctx context.Context, vfs []devices.VirtualFunction, allMetadata []StoredMetadata) (reset, failed bool) { claimed := make(map[string]struct{}, len(allMetadata)) for i := range allMetadata { if path := allMetadata[i].GPUDevicePath; path != "" { @@ -143,7 +159,6 @@ func (m *manager) resetDirtyUnclaimedVFs(ctx context.Context, vfs []devices.Virt } } log := logger.FromContext(ctx) - reset := false for _, vf := range vfs { if !vf.Allocated { continue @@ -157,12 +172,13 @@ func (m *manager) resetDirtyUnclaimedVFs(ctx context.Context, vfs []devices.Virt } if err := m.destroyVGPUAssignment(ctx, assignment); err != nil { log.WarnContext(ctx, "failed to reset dirty vGPU VF", "vf", vf.PCIAddress, "error", err) + failed = true continue } - log.InfoContext(ctx, "reset dirty vGPU VF during allocation", "vf", vf.PCIAddress) + log.InfoContext(ctx, "reset dirty unclaimed vGPU VF", "vf", vf.PCIAddress) reset = true } - return reset + return reset, failed } func (m *manager) quarantinedVFAddresses() (map[string]struct{}, error) { diff --git a/lib/instances/vgpu_linux_test.go b/lib/instances/vgpu_linux_test.go index c93e4b2f5..ab514d4fa 100644 --- a/lib/instances/vgpu_linux_test.go +++ b/lib/instances/vgpu_linux_test.go @@ -5,6 +5,8 @@ package instances import ( "context" "errors" + "fmt" + "os" "sync" "testing" "time" @@ -33,6 +35,8 @@ func newVGPUAllocationManager(t *testing.T, vfs []devices.VirtualFunction) *mana } return profiles, nil }, + configureVGPU: func(context.Context, string, string) error { return nil }, + refreshVGPUCapacity: func() {}, } } @@ -213,6 +217,7 @@ func TestVGPUClaimRepairsDirtyVFWhenProfileNotAdvertised(t *testing.T) { dirty = false return nil }, + configureVGPU: func(context.Context, string, string) error { return nil }, } meta := saveTestVGPUInstance(t, m, "new") @@ -315,3 +320,97 @@ func TestVGPUClaimFallsBackWhenDirtyVFRefusesReset(t *testing.T) { assert.Equal(t, "0000:82:00.5", device.VFAddress) assert.Equal(t, []string{testVFDevicePath, "/sys/bus/pci/devices/0000:82:00.5"}, resets) } + +func TestVGPUClaimConfiguresBeforeReleasingAllocationLock(t *testing.T) { + // GPU 82 has one placement left. The first claim must take it before + // releasing the allocation lock, so the second claim sees GPU 82 as full + // and lands on GPU e3, instead of both claims choosing GPU 82 and one + // failing later. + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:e3:00.4", ParentGPU: "0000:e3:00.0", Allocated: true, ProfileType: testVFProfileType}, + {PCIAddress: "0000:e3:00.5", ParentGPU: "0000:e3:00.0"}, + } + m := newVGPUAllocationManager(t, vfs) + m.pickVFIndex = pickFirst + running := saveTestVGPUInstance(t, m, "running") + running.GPUFramework = devices.VGPUFrameworkVendorVFIO + running.GPUDevicePath = "/sys/bus/pci/devices/0000:e3:00.4" + require.NoError(t, m.saveMetadata(running)) + placementsLeft := map[string]int{"0000:82:00.0": 1, "0000:e3:00.0": 1} + // As with the driver's creatable placements, a GPU's VFs list the profile + // only while the GPU has a placement left for it. + m.vendorVFIOProfiles = func(vfs []devices.VirtualFunction) (map[string][]devices.VGPUProfileType, error) { + profiles := make(map[string][]devices.VGPUProfileType, len(vfs)) + for _, vf := range vfs { + if placementsLeft[vf.ParentGPU] > 0 { + profiles[vf.PCIAddress] = []devices.VGPUProfileType{{TypeName: testVFProfileType, Name: testVGPUProfile, FramebufferMB: 2048}} + } + } + return profiles, nil + } + m.configureVGPU = func(_ context.Context, vfAddress, _ string) error { + vf, ok := vfByAddress(vfs, vfAddress) + require.True(t, ok) + require.Positive(t, placementsLeft[vf.ParentGPU], "configured a GPU with no placement left") + placementsLeft[vf.ParentGPU]-- + return nil + } + + first, err := m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "first"), testVGPUProfile) + require.NoError(t, err) + second, err := m.claimVGPU(context.Background(), saveTestVGPUInstance(t, m, "second"), testVGPUProfile) + require.NoError(t, err) + assert.Equal(t, "0000:82:00.4", first.VFAddress) + assert.Equal(t, "0000:e3:00.5", second.VFAddress) + assert.Equal(t, map[string]int{"0000:82:00.0": 0, "0000:e3:00.0": 0}, placementsLeft) +} + +func TestVGPUClaimResetsConfiguredVFWhenClaimSaveFails(t *testing.T) { + vfs := []devices.VirtualFunction{{PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}} + m := newVGPUAllocationManager(t, vfs) + var resets []string + m.destroyVGPU = func(_ context.Context, assignment devices.VGPUAssignment) error { + resets = append(resets, assignment.DevicePath) + return nil + } + meta := saveTestVGPUInstance(t, m, "new") + require.NoError(t, os.RemoveAll(m.paths.InstanceDir("new"))) + + _, err := m.claimVGPU(context.Background(), meta, testVGPUProfile) + require.ErrorContains(t, err, "save vGPU claim") + assert.Equal(t, []string{testVFDevicePath}, resets) + assert.Empty(t, meta.GPUDevicePath) +} + +func TestVGPUClaimResetsVFWhenConfigureFails(t *testing.T) { + // The selection already used the driver's free placements, so a configure + // error fails the claim instead of trying another GPU. The write may have + // landed before the error and no claim is saved, so the claim must reset + // the VF itself. + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:e3:00.4", ParentGPU: "0000:e3:00.0"}, + } + m := newVGPUAllocationManager(t, vfs) + var configured []string + m.configureVGPU = func(_ context.Context, vfAddress, profileType string) error { + configured = append(configured, vfAddress) + return fmt.Errorf("verify vGPU on VF %s: type is 0, want %s", vfAddress, profileType) + } + var resets []string + m.destroyVGPU = func(_ context.Context, assignment devices.VGPUAssignment) error { + resets = append(resets, assignment.DevicePath) + return nil + } + meta := saveTestVGPUInstance(t, m, "new") + + _, err := m.claimVGPU(context.Background(), meta, testVGPUProfile) + require.Len(t, configured, 1) + require.ErrorContains(t, err, "verify vGPU on VF "+configured[0]) + assert.Equal(t, []string{"/sys/bus/pci/devices/" + configured[0]}, resets) + stored, err := m.loadMetadata("new") + require.NoError(t, err) + assert.Empty(t, stored.GPUDevicePath) +} diff --git a/lib/instances/vgpu_reconcile.go b/lib/instances/vgpu_reconcile.go index c4d2defdb..016b5a91c 100644 --- a/lib/instances/vgpu_reconcile.go +++ b/lib/instances/vgpu_reconcile.go @@ -43,10 +43,22 @@ func (m *manager) StartVGPUReconciler(ctx context.Context) { }) } -// ReconcileVGPUs releases metadata claims whose VMM is confirmed dead. The -// device-level pass remains for mdevs; vendor VFIO leftovers are repaired only -// when the allocator claims that VF again. +// ReconcileVGPUs releases metadata claims whose VMM is confirmed dead, sweeps +// orphaned mdevs, resets vendor VFIO VFs no instance claims, and reloads the +// vendor VFIO placements /resources counts from. func (m *manager) ReconcileVGPUs(ctx context.Context) { + m.reconcileVGPUClaims(ctx) + if !m.resetUnclaimedVendorVFs(ctx) { + return + } + refresh := m.refreshVGPUCapacity + if refresh == nil { + refresh = devices.RefreshVGPUCapacity + } + refresh() +} + +func (m *manager) reconcileVGPUClaims(ctx context.Context) { log := logger.FromContext(ctx) protected, err := m.reconcileVGPUAssignments(ctx) if err != nil { @@ -64,6 +76,33 @@ func (m *manager) ReconcileVGPUs(ctx context.Context) { } } +// resetUnclaimedVendorVFs resets vendor VFIO VFs that are configured but +// claimed by no instance. A claim writes the type before it saves the claim, +// so a restart between the two, or a failed claim whose reset also failed, +// leaves such a VF holding a placement that /resources does not count. It +// runs on every pass, so a reset refused once, as while a VFIO handle is +// still open, is retried rather than waiting for the allocator to select +// that VF, which it may never do if placement skips the host. Reports +// whether the host has vendor VFIO VFs. +func (m *manager) resetUnclaimedVendorVFs(ctx context.Context) bool { + m.vgpuAllocationMu.Lock() + defer m.vgpuAllocationMu.Unlock() + framework, vfs, err := m.discoverVGPUDevices() + if err != nil || framework != devices.VGPUFrameworkVendorVFIO { + return false + } + allMetadata, err := m.listMetadataForReconcile() + if err != nil { + m.recordVGPUReconcileFailure(ctx, vgpuReconcileStageResetUnclaimedVFs) + logger.FromContext(ctx).WarnContext(ctx, "failed to list instances for unclaimed vGPU VF reset", "error", err) + return true + } + if _, failed := m.resetDirtyUnclaimedVFs(ctx, vfs, allMetadata); failed { + m.recordVGPUReconcileFailure(ctx, vgpuReconcileStageResetUnclaimedVFs) + } + return true +} + func (m *manager) reconcileVGPUAssignments(ctx context.Context) (map[string]struct{}, error) { allMetadata, err := m.listMetadataForReconcile() if err != nil { diff --git a/lib/instances/vgpu_reconcile_linux_test.go b/lib/instances/vgpu_reconcile_linux_test.go index 4625dd4c7..bda272dcf 100644 --- a/lib/instances/vgpu_reconcile_linux_test.go +++ b/lib/instances/vgpu_reconcile_linux_test.go @@ -4,14 +4,19 @@ package instances import ( "context" + "errors" "net" + "os" "path/filepath" "testing" + "time" "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + otelmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" ) func TestReconcileVGPUsProtectsSocketOwnerWithoutPersistedPID(t *testing.T) { @@ -51,3 +56,78 @@ func TestReconcileVGPUsProtectsSocketOwnerWithoutPersistedPID(t *testing.T) { require.NoError(t, err) assert.Equal(t, "/sys/bus/mdev/devices/test-mdev", stored.GPUDevicePath) } + +func TestStartVGPUReconcilerResetsUnclaimedVendorVFs(t *testing.T) { + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0", Allocated: true, ProfileType: testVFProfileType}, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0", Allocated: true, ProfileType: testVFProfileType}, + {PCIAddress: "0000:82:00.6", ParentGPU: "0000:82:00.0"}, + } + var destroyed []string + m := &manager{ + paths: paths.New(t.TempDir()), + discoverVGPU: func() (devices.VGPUFramework, []devices.VirtualFunction, error) { + return devices.VGPUFrameworkVendorVFIO, vfs, nil + }, + destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { + destroyed = append(destroyed, assignment.DevicePath) + return nil + }, + reconcileVGPUDevices: func(context.Context, map[string]struct{}) error { return nil }, + refreshVGPUCapacity: func() {}, + vgpuReconcileInterval: time.Hour, + } + liveIdentity := HypervisorProcessIdentity{} + liveIdentity.Set(os.Getpid()) + owner := saveTestVGPUInstance(t, m, "owner") + owner.GPUFramework = devices.VGPUFrameworkVendorVFIO + owner.GPUDevicePath = testVFDevicePath + owner.HypervisorProcessIdentity = liveIdentity + require.NoError(t, m.saveMetadata(owner)) + + m.StartVGPUReconciler(t.Context()) + assert.Equal(t, []string{"/sys/bus/pci/devices/0000:82:00.5"}, destroyed, + "only the configured VF no instance claims is reset") +} + +func TestReconcileVGPUsRetriesRefusedUnclaimedVFReset(t *testing.T) { + reader := otelmetric.NewManualReader() + provider := otelmetric.NewMeterProvider(otelmetric.WithReader(reader)) + inUse := true + var destroyed []string + refreshes := 0 + m := &manager{ + paths: paths.New(t.TempDir()), + discoverVGPU: func() (devices.VGPUFramework, []devices.VirtualFunction, error) { + return devices.VGPUFrameworkVendorVFIO, []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0", Allocated: true, ProfileType: testVFProfileType}, + }, nil + }, + destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { + if inUse { + return errors.New("vendor VFIO vGPU on VF 0000:82:00.4 is still in use") + } + destroyed = append(destroyed, assignment.DevicePath) + return nil + }, + reconcileVGPUDevices: func(context.Context, map[string]struct{}) error { return nil }, + refreshVGPUCapacity: func() { refreshes++ }, + } + metrics, err := newInstanceMetrics(provider.Meter("test"), nil, m) + require.NoError(t, err) + m.metrics = metrics + + m.ReconcileVGPUs(t.Context()) + assert.Empty(t, destroyed) + var rm metricdata.ResourceMetrics + require.NoError(t, reader.Collect(t.Context(), &rm)) + failures, ok := findMetric(t, rm, "hypeman_instances_vgpu_reconcile_failures_total").Data.(metricdata.Sum[int64]) + require.True(t, ok) + require.Len(t, failures.DataPoints, 1) + assert.Equal(t, "reset_unclaimed_vfs", metricLabel(t, failures.DataPoints[0].Attributes, "stage")) + + inUse = false + m.ReconcileVGPUs(t.Context()) + assert.Equal(t, []string{"/sys/bus/pci/devices/0000:82:00.4"}, destroyed, "the next pass retries the reset") + assert.Equal(t, 2, refreshes, "every pass reloads the placements") +} diff --git a/lib/instances/vgpu_reconcile_test.go b/lib/instances/vgpu_reconcile_test.go index 51d0624fb..eca54ef61 100644 --- a/lib/instances/vgpu_reconcile_test.go +++ b/lib/instances/vgpu_reconcile_test.go @@ -129,6 +129,7 @@ func TestStartVGPUReconcilerRunsPeriodically(t *testing.T) { passes.Add(1) return nil }, + refreshVGPUCapacity: func() {}, vgpuReconcileInterval: time.Millisecond, } diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index ce5626905..4c604c755 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -1007,7 +1007,7 @@ type GPUConfig struct { // GPUProfile Available vGPU profile type GPUProfile struct { - // Available Number of virtual functions currently able to create this profile. Best-effort: creating an instance may reduce availability on sibling functions sharing GPU framebuffer. + // Available Number of additional instances of this profile the host can take if only this profile is added. Counts for different profiles are not additive: an instance of one profile can reduce availability of others on the same GPU. Available int `json:"available"` // FramebufferMb Frame buffer size in MB @@ -19326,397 +19326,397 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+y9+3IbN7Y3+ioonv1VpD0kRV18Uyr1Hd/iaMeKtS3b8+0JfSiwGyQx6gY6AFoyk3LV", - "+WseYOo84TzJqbUA9I1osiVbkj1xTdVEZqPRuCwsrOtv/dGLZJpJwYTRvcM/ejpasJTin4+NodHinUzy", - "lL1mv+VMG/g5UzJjynCGjVKZCzPJqFnAv2KmI8Uzw6XoHfZOqFmQywVTjFxgL0QvZJ7EZMoIvsfiXr/H", - "PtA0S1jvsLeTCrMTU0N7/Z5ZZvCTNoqLee9jv6cYjaVIlvYzM5onpnc4o4lm/cZnj6FrQjWBVwb4TtHf", - "VMqEUdH7iD3+lnPF4t7hr9VpvC8ay+nfWWTg449zI08NFfF0eSITHi1XJ/uSi/wDfo3Q3MiUGh4Rbd8h", - "Gb5EplSzmEhBaGT4BSNcTGUuYvLm6QmJpBAsgs70WMipZuqCxWSmZErMgpGF1AbbGEWjc2LoNGHDsej1", - "G/vBBDyJN6/SXxfMLJgKDJZr4nohM6mIWXBNuICnERtWN8yonK2ubL/H44RNDE+ZzM3qQv0kL0kixRyn", - "5fslaa4NWdALRn5nSpLfcprw2ZKLefsiTdlMKkZ+WmYspYJkCY2YJtwQLoz0s7FrVNLYvTREXHwupGKT", - "mGnDBYX+J5lURq+O/hX+QRNSaYtDw/bELKjxVC6kIeeMZfWJ0kt6Xl/GX/f2+o9Go9H7fo8bltpjRT/w", - "NE97h/fv3du/1++lXNh/7xaj58KwOVO9j8UvVCm6rExHy1xFbBLxWK2bSZRwJgx5evTs9TUn0NsdDfF/", - "Ow97/d7uo73h7v2H+O/d+73qtFYWvj7yj+uP3qmhJterPMiepokjlEmFSFZn/UueTpkickaiXCkmTLIk", - "eKRY3IHoatMehbYikmLG57li8eq3/ZGrLeeCakKFZRqDBr8oO+t07iJgYrG8FBPFUsoFrPHKIF77RwRO", - "KHGHCIYUSWGUTBJgCsawNDPan6I+sHFBaJYlPELWUztUB+lI9/o9kScJna6MsNxtlvA5xwadlobryib5", - "d4mRhAnDVHHCuyxNjS22fbhc7uBulHyx0ycXMoknuTA8Wf3qW/jZrWn1S1yTBUtiImezPuEzQgn0Aj9b", - "Gq8v+95o7/5gdDAY3X+z++Bw9OhwdO9vvX5vJlVKTe+wF1PDBrDLXfYGmbbmIgrvjmiSSAoXkmKR3Z3i", - "wqpt4JRFMmUEul478oNPH3nrdfO03FRoSFzD8mYNbPWwcWV0o+6EalMwIdwvbpYTGhjTG54ybWiaAR+C", - "MVQWs40L+Q6b++BXfu0C737SAgv2wUzcCgXnE6IP9iFjEdyI0nOTQsCA/hx5FyzrVmhcMaqlsGISXKa/", - "9nKh8wyubhZPsoQa6LfX71kymKRca3i1+CHm2vKRfs8T+URIM1G5ELahYOZSqvNqS9fLhGe9fm9B9eRi", - "nuW9/rprq07U+AmW0Exjf27H1YQpJVXPisbLyUwqv0m999UlXNPVygrp4ooNrFCv36stQMHO/Vz8uItd", - "DQ6u33N/TZTVKrA7O5nVgVf7Wh1uMbT1jN3eIlaI9ttM3Mu6zgFiTudCasMj3YnNo/AA25vKOMA6nxXd", - "ER4zYfiMM+XkakYU3A8pI74TAp0QLkiuG+egEP0n7IIJoycXBxMTZauL0lBsqptXkU3KG7FyKxfbX5yU", - "DURan3tQcbqgHM/kM3bBI7Yqu7mtmcSKXzAVYN+FAGBZoW1HtuCsAwsRUrDt2kqJCx5z2oUdxDimCQ9Q", - "z8nTI2Ifk6NnZGvBPtQ/svdg+rDX3qWgaYAWfspTKgZwIGBYvn9sW+375UGoZy7TNJ/Mlcyz1Z6PXh0f", - "vyX4kAiUcKs9PtwLSapZxCc0jhXTOjx//7A6ttFoNDqke4ej0XAUGuUFE7FUrUtqH4eXdHcUszVddlpS", - "1//Kkv7y7ujZ0WPyVKpMKtTZNh6c6vJU51Ulm/quhOj/Sc6TeJXqp/Aza1knpjTXhglDnthmRDGrypHL", - "hdSMRDRaMDK1Sguq59hfF5IvPuxvr9AInnGdSeTz5N1xKc2hasg+sCg3te9+T2KuDReRE6oqk+swIjA/", - "JQzu35BkgStAXBsuBTFedLq2EBApRjd8Dlp0+tjq+c8teU1S3da7b0K4IClPEq5ZJEWsq9/gwtw/aJ9M", - "5RTba3PlU8/hZ5IyremckS00S6G6YTk84ZrMKE9YvN1Nwm6bzN/ltHKv1c4cksGATqPdvf0gQ0vpnE1i", - "Pnd2xSYNwu8gH0M/hvC0dSIoYXSbB35Ssdnq937E+wQ/otiMKSaiT/5cpuQFE9SpVP+B3+39XzulwXXH", - "WVt3cDFPyuYf+73fcpazSSY1tyNcYRPuCZARLjXBN8JjxkfxdieK0oaq9ecDW3yGk1gKmxvXxpl+QN6i", - "842vvIE2TYaO7KgQcCpcoJVvPwdJKyCySGGYCKzOSzknCRdWDWLCuL1AYW+ZsR8SOd/ufbZ1KJZ/9fDD", - "uK/BvOwPLb3Bs36hFSRyXl3NBaPKTFltMVvuVddRObrW5T+pHZ/GBUo1m6znICdcCBajzd0dbNsSZOug", - "7oOn6JybyQXcwKEzh8P6mRviWrR2lcjofMYTNllQvbAjpnHMrcH1pDaTgAhZc2ZQNBL4DlG0QaX69KfH", - "e/fuE/eBwBo66y80WJ1J5W3o3rYlhqopTZIgbbST29Xv6FUKCVNAafBtu3sKCvSEaTldz+2mU95zvbB/", - "Ie+GUeHd1+v3IiCvBP5eIdx+78MAeh1cUIWrDt1XhvXf/kuV356UH638elJ8v/LjazeUyk8/+lFVfnta", - "DtCvClOr58GpaTjpsFQpuTADLnALQAB3lqep3bxCy3KWDCJFRcJjqgs7XydYPV0RqcjW6x+f7u/vP9ru", - "LlxxfT7R/Hc2mU/Xys1u0E5QhtcIvEa4IHM+p9OlYXpInlIhpAFnZLSgYs5iQmcG3nJjrSng90abnDCt", - "QhJTbSKS+f2APnr44QM1j+7zS/3o93Sq5n/fp70262KuW9YXLXDU7SY0JYqKwCZe76JO6QfQ/C2VhJyf", - "x9ZdRYpGbg90cwhDUir0M/4B1tyQ3dpKBxc3rP0VLiwhxSAX/LccdztL6HJVBzSMpgOkiNDyWr5hD5De", - "fHw0nB/7jp/pJeUGTk7hM3VT7hOZxHDvzLjSptfdG3YlwYCpzyUi1U5ZWGJapYjmArZe6sU4W89Kwmcs", - "WkYJW2Xunn1nKnfm1pjBnYN/FgbErmycqQAn9j+eFJ+o/fys/F7t9+f24x/7vac0o1OecM+Z63za+eQn", - "zvC3aZeqnT2zr752b37s92aMmlyxwFqeYnwAsXbiQcIuWEJcayDeQ/L45IjoXM3QZY7UatuC2V5qpsdC", - "sTlVccI0kvrlgkeLwlwJpgdNqCjDAsjj5JIuNckU00yYw7EYFxZyPe71ydgqX+5vSyX2bxsa4huJuWLa", - "/QNsDeNeH/pK5FyPe0Pyk9RmAOqyk6UOydjZY/S4R7bsnySjWpuFkvl8MRYpFXTOUiZMn2CQBpqBgTMl", - "y21CRUzGPSU1M4YOWJonyPuhs8dZlrCxOOUJj6QgKY1enbp3L7lZkNf2pcrliTOGy7pPMmd61jRlY0Gt", - "GRR2ckkyJaeMJDQX0YJpwsRMqohtD8kJUwO/xH5vCVUMNsMacUku4IgwWm6G/vU9YcIoF+nQnbnATK5C", - "f7D0hRKtr/LmkX3jY+EducrLv7hXgFW5GQd1FLUsrekVerY3YmHq99dRzRPQHwtcUdxVDq0uBalt2Syh", - "cySVyiGyxKjHIpJpioqe+wrR7IIpmhQ7BHecRHsC3A3uRhqLKJF5PFgsM6YuuJaqDzcEizAYQfXJbyzN", - "+/hR+GuQ8kjJi5RIMRYJkPEOTeP7B9tD8pgkXANx+PlzS9yOxJAToD2Aw2jHPept8uOenRjXBC7+IXku", - "jOKW5oi2BDe1N2mDuLpuXoVZrdxtuD9X6ezUvrHi4bA/O4Lur7DYCt2UBFiQcYWLhu6sNQx4VQb367re", - "GeXGV+xWRIXbqRp1HhJuCM0yRpUmXIxFSU0iRip1L2WK4XJobtzWpcyQrZQaZC9gtR0L5BDfaVLb/e0h", - "QbnRxoeVHAleIYolSzsgNhZ+zJc8SdBwSHKMYEAOZw8aWsYqERR4mKgoeF9Seruc88v5RWsTqB0R++3/", - "fn78luilNiwlUy6oWm4Trktea0lz1TEXFhnd7qHmjqJafdaxxKgneJdIURcgm+d1owvBiaAlXWyisJ9k", - "KMSSqmgRCqPThjw9eUvgMTcsAiKujZeq9H7QlSR1S3fwVYpCrF3vWnfIdDbOWcKRwhFvmuxRcY+ERaTC", - "Hb/q7YI3K35clrAIeJWNp7H6GyPOuUxkiixdsNXZ7LQuke9cb/q6JltS78CEtys3C5xplYsheSUIihGk", - "JkWMBTblIkrymGlS4eekNN96+aIWBlWQ/Vj86//9/6wsEZMLTgtpg7zjykAMye9Wi50pmjJgevUrLVqw", - "6HwsmmIIgV61JNRfKsUyc129UBSfLwwR8nI4Fkd2TMWAt7ScmUuqWJ6BmkkGAzfqgZOztm0QF3bETSNE", - "89fG3lQW54qxhDV3XrGh/VUK20Sqv5RiS51W59SwSxpQiF8A6Q0uuMaQNaQJ15YcnQwJPkZXQrSwT4GL", - "ghRLtixVcDEfC3ziROJt4iRaS2bONTkkr1JuCuIXsrhe3E03FhBgOGVMoBcxubAByCWpLllzAyCacxcc", - "vMPd0NGYw9gnRk7medDu6u+6mCsWGfLueGDk4N0xMYrOZjwCOsqYcqO2F0xz0EF+DjEGSctK+xd9CEUy", - "BCVD8XgOQg4slxP97W/2csqYgnG9eXwCFxyu/vdk3BPU2HdKTj8ATw6PWUx+efwGrtbo1em2WzWnmtqO", - "e3DthIzg/Z7Op4KZtvHbpxiDG94KDKXdxHztEq1s0Sby/jSJBi5+28F3a2SSkoOBdAJCbEGCQ/KyJr/a", - "F4rAFRBBVyX6ogd/iL8nxWh/QIGGpIwKTaiT9SsjIlyPhZc+agIHSNrErSi8qnJhuZS9DwcV7otiiZVH", - "UCIrgvQv4P8HF1pG5y7Ew2qZBacVNMXTXV26FcmKm05CzkYTQEVhCSyp/3pN8AQdQQua6YU0qFX7f+wo", - "po1UbBuUcxcvBc8zmmtG/kJSlkq1HAvfvm/PWUKNjWEo3p0BY+qRrSiRglXtCDaAgI6FNjLLWOwcE9+T", - "mTvctAgilKqwVdtGY+E+oXesBKBJMQvcgNIDkyzLTa7MZBtNDTibYlow5zy1w15IkyX5fODm2SNbCb9g", - "btowQf47s32AF0VKMXDxkNC0tJ3gI2KomjPjTM8a+0eaGffGYgvPLcF/g2MlzQWP8DbHdvMsH1RNHD2y", - "9eLk7U4ZszQWlcf4Chj1BlwOEp5yYG9b8AM52nlFFNzQ+DMXczt6bGynU0wS27spDsk7muTunCK1w/qO", - "hZf+caQDo3KzmOXJIbn43Qlh9f0odrKQinb3+87SlKCFqVWSeXdMNL1gniDxLVCptYFjROMLpgzXzhp7", - "TrYcOQ2cnwvpThPBWEyEHItqX3CQkUxhiE2Rn1hDjCaNRRqLUmxrRJL7q827DAtThBd8ults1ms0LT6G", - "T9darqAinxZKfeMiyXi7VxVskXhQaFQuU0V/ZeRVxgS0imWUp0wYwsVM+qb1KLLRcK8tMq3l86fVa6X4", - "vF5IZcicG6IYiHFSOG427g1irgwcf53PZvwDapE5WoGsTIM96T5wqHEvF+dCXopxrz7MB/fp3iye7m/c", - "DT/sfm0Ng/uATM9ZpVvz9db70J6udZiBqEmMBN25qvk7Cv8M7rIyJQkuPTWAs5JwFldou2+jdedMMOBc", - "sY0BNcRLaC0un0FQlL1xr9KVPDEtO2ojZ1s3dMMc5omc4oXnJ1IGnQ7JUX3t+oTiAxSY66tcZEtUAkPJ", - "FhvOh3C7ZBEfQGTogO4NRqPBqEntycHAhrxn1BimYID/z6908Pvjwd9Gg0fvyz8nw8H7v/xHUCHvGK3q", - "/dpunlv+LPWJH2w1hLU50E3hrdf2rFWH33540bTQutORYngMaLJxDBBx8LTSvJ3Wnx6tRr3ZlYol2KGH", - "XO4kfKqoWu6IORcfDkGc06a+buvbrjOwBJJjrGDkGxCqibOw/LpzQRWnwry3YyTjqnEA5DeCtwbc+89w", - "RGQw8P049uUj72sOgCH5RRpCnUcaVLpDbLT63bmiaUrR2nlBEx7j4XBuNs1jZo2zAp+RCyslKWZyJcjB", - "aOSfTCqDArlBaj+Vcc8ZTlW0IM4gpAiaQmyGTMQy+KSRMW2kKVWWYvWkwTH7y07x15b/c/j+L9v/+z8+", - "K6kjoa2hcWvOuCI/a8R5QyOylchLpiKqGUkYzFb3Sczn3GgrvMUYGoUGte9JZIM9bOiiVIQJb6LGdnVy", - "TpcDmvGBs7xYd/dLJuZm0Tu8vx9c3C33x+D9f/qfWhZW5UlIX3stc7S84uNq4rUfQyfni1/dPEGnS8rF", - "kX1td1Wc/LQd9hNZt9NWqWvdarhjiqygDQNZzcCHiKM0JERcMKXgKFq+9vT4Gagw58wxNNAayTgfjfYj", - "bIB/MvcLCHGgyuBv23WJx3YXEnZ+7bFoITEMMknkVYyU/SZTb4sSXOsjC612yf1XYwrRiW7d4hjs7hqS", - "qZLnTKFLBpaYM03O2dKaC+Y1gyYTF+SCKu8afINpCdjEP0JvEEP1GJRYTM6HXFamHFt0OptNg2I0Kbll", - "SsHdhXqljmTGgKG6Zjg1csbExRlJaWbVesWs0TylhikO6mJpWExZzIFHjwXDg0EyCmc+iqQCEyvsLdpg", - "y1X4TpMzqyaeYfdnXABZn9mD2TCV/tF79fbNk1dvf3k2eXXy/JfHR5Ofn/8P/Gxf6h3++kfPgmkUgVdP", - "GFVMkf/4A+f70QbPYkwfpLovpHLarnfRY7wKzfhQZkxQPoxk2uv3/rP6z/cf3/twT5u5d9E7DA7sY1DM", - "dLbPQAy+T4DSRKoy8pQiLgiyKG92qJgbGqZ8K/ld6UigisLlZJrpYG5K025RyqG7o9Hxkx0XPXLP/2N7", - "SJzrFocPPEgqJx7rBZBPgcuB/rQkkZHL0Ji1peD7T4UYPPpaM8mF2cicyqarPGowKJ9egRXtTLnY0bAN", - "g+hq6450c+1A5efigispUD1HcWmaMF0/K7+8evZ88vyXd71DuAjiHPMIe/3eyavXb3qHvf3RaNQLESj7", - "kHHFdDDw8fFUyyQ3jGCjMsZ0SI5zjcGdjnnMcrSDonymSZqDYSlZEvYhSnINfAqlAWOSUP7xw8He7pvd", - "h4ej0eFo9LfOAatA+huY94uTtzZDHc87o4lZTNA1t+nFn7DtU2z6se9tk6jaB1ap2FNvrYTT694hW4u6", - "dGUnR5Agx739F0/sgdp9Me5VCNGlAhe92I4bq/fiSWhdKtaoVUmveGYz3ETsqX4VjMba63PtzERVT/hA", - "UITYgZYxVTGZSqri7+uxNPgmGPK/0wStFTRxfiIreuM7VpaUTKOu7GN8SmOr7jeWAC0/qTUlUkEYekov", - "OLCcQZpy6d+rsyVnTKnZUIcVB1PAjlcJF+r1ezAz9x8/wV6/d/F7Pad6sznQhcZ00hU3KIE0ybhga7TA", - "L0TYB5tyImk82P3Msn4l3q0BQ2Mf1Le7UE49ga+gO02piC95bBYTyIiHIQduSPeEFI2La/IDzIQm//rH", - "P98dlwaR3RfTzN2Zu3v3PvHObNyS0HUw16aYSJ6Fp/E2C0/i3fG//vFPP5O7nYQVKa8lnrv9f257aALU", - "ONwuG43f4nct5LAyP1e2+NI3YBiErlx5wVRCl5XrxI2ptztCnt4YleLW3ejeq5iQ118u0JsX16rXC7D5", - "GWeJjdGyobeoX2jvIbK+QzwtwJO+0z4DiZGplMaOANaG0RVslPCt9GkmqaG7OSIpDOWCKTfe4sawV4rV", - "WNL7B0Py/EPGFAeBiSYkpuqSCxv04t90OjehAgCBEi7YgKfOg2uXxE4SeTJ0DI42H3CkbcDRqtUejdUx", - "cfejfVmqhkHsJi1LiiFjnmQFkt664/Pati6V/gBJBijyCdWFZ7YLHRZkuLt37P7cswRZJb7icNmzxlVB", - "fPZT34MYCqqzwY9qC2s3ZQS+xKg25IC84E8IF5DKnzByzJ8Q6m4nLsi7v9l9QF6nO8lS+pxnNs5jQudF", - "Wuo65L/Tc545fR7f8FEG9g6Kc+gZz9BwLCyyD7ANpB2IyPe+WwNJBNoGLsBti/fZqowG1oHyRrPNrZtW", - "5aJPprkBlV8aRpzxxUEMwFiw8ZSRvAwibSjgboKrgRG4LOdMCZZMrIKtO66MfYm4l1oXB6c6o9ohkSmT", - "Z/X1evbz8SnZerYUNOUR+dn2eizjHKIBLU/Yrq9efywyhfAu8BE4C9x9V86IzM1AzgZGMeaHmGJnhR/G", - "gVxcvDh562BSNAQpvWawsEzELuDVCztFxKv4zlhXeK3b6vcbi94GhePd+x1P9qlrXh7t7hbJfs+YJARu", - "l1i2BklENo8CHFcF9fnUvj6cwBeyxGKw3GBI3mrgBiMIe5EKoxbIVJpFVbvEa0lXNaASQrNsBvpnWMUs", - "tdnG+T5YBP3WUZbXSXev34ppeGEDJkCc0V051wU0rjAuBFwKcK69jowqCDbjknxWd8zCy1btSnaA5Z5R", - "Uwfk6GoKtz0jGOzG6FBn0cYl6W7RrthYV2zb3gD4R8ekifX9Hwk/kLUW/dIG+AnfOrWdNJfI9d33M7vG", - "Kh0Va9LwA3ye5XmsKzbTTritVuax8owmW2dgZnV0DIbVsz45+8/aD8BPvTwF6sIlsauBPFrAT9X+9Upk", - "8wY77rWjm6m+/n481q2oW+RilxhFhbZx9guasSH5CS9GYlgK0iLcVLwaWijk5fdEWh3FvzoWMDRdj7X0", - "1nzN5wJCzzBoHhhOHFuTv7PY+fgbHYI79na4FYndjs6BFiEvc9H25Myb3s/q/HfVML9qt3KW+hWDBf5u", - "DRVoyjE7aW7g81Z7cT5pmRsLYuam3hDw6ub/TYFBbiyFKfIa+39asIsmhvRFwGIBk5Oz0l9TcRy1+Wec", - "8Bf2HZ2zJW659xPRFU9R1UUUduS4kHYnylReRduhvUq4rviXrKfIOYcw+Lh+RINuk01bAevVefnrmn8w", - "D2fgJ1tSjFPmPT6p50IwOfs9NDpqhouPloRDm9Z11remD4b2RSKAWBIfm9/sHsOSMYjxzP0ydL2dFUmC", - "nwNzvMz+qG4tqeysF6WxGy+KGRb367IBoFPrzZOy4amLFrenYpeKe0bmAS47irwuayZ1iten2YGeVzpb", - "ExDWtYvVLFm7vpUxO3qywM4+icTuB/q/aojHTeD/uKGGW+SY+ifPaJKckS3XaJso9ndEhXV7BRF2xfYD", - "yq0jgSLc7N1xfywsFzhbGJNN4P/0BE7xWbMz964/4SUo+8MRiq4HB/v1nA074Ea3dXN5EA2nfWsg7uqG", - "4rdajPIe/R4fFzcQIN6QLUPncHQc7BHcE9ufarDnQrMoV2zzWX+M4pGD3sExJZSLwU9v3pwQxeZcY6px", - "MKnBhsa3YLQ5Y0EZruana2Tdaru3f3Dv/oOHj4bxuRqySA1zPQBFZrA7pCn9XQp6qUHs2kmXNMsOL3Y3", - "8nm7BcX42lm+12zbSUGmwJJ9QHIXLflp+UrpMDznIu7awc/QdnPQq1fib9rtkik2yLO5ojH7zE6Xa8c1", - "4Wq27+qGSijrA5pzbcAQUUIAb62PY64v1oVMBjE19CphzHa4qxHL6dJ2ZVXyUH+t4eGnzZjwGm5PsB7C", - "pwaZ+bGEtqUNU9gaElg8MTKQvutlh6NnsOa+bSfwVIRANnJyMeNyU5I/DK2euouve/MGdDHIIu4sdd6r", - "wQurvzXWvTuuxtYMx2KAUtgheVZ8oOi26JKiikFj6+XfkqqWP4yOgClAn5B3x0Pyphjtd9o7BeyYSJE5", - "mqNDEYWigRXJqgPIMdGGm+brzixp8Zi3MYRIumfDomoLGkALAxray6a8MR+b140b5e6SKvRMR4PwOnSy", - "1/Y+WodQVosIuTcY7Q52773ZHV01IuQGIKdDfT2u8xYX+1/lPk/fHj3bc2bP7WuDkn12UOowK3tWJi2Q", - "rVyzSkKuVMFUhUpGQEsqwrUzDG4oYaCE31zX1q7EG2h5E2jbITwwbNK/Bh52k2FuBF2tTG7VoLPM0PxQ", - "npKKIG93NIt4MPMaQgafKEbPsYrA6r2NOBQTe5+F4w0RLGS6RHAsNHIpKc1MkwJDpjR+7x48OHi4f//g", - "4WgUwHNeJXgZ8Ql6ZDsNAEJwErosoA23MIwhJtNETuuEfm///sMHo0e7e13HYR333dah0L/9W2TLrchf", - "fKU3/6Q2qL29B/f39/dH9+/vHXQale2s26Bc27pI8mD/wcHuw72DTqsQ0uuee3ztpgAfKvbw2NZHgn8N", - "dMYiPuMRQVw6Ai8gcsAC3PjeC10/k1Ma+5IY4bvDUJ7otdF99mOupbW3pnlieJYw+ww3pJNLAmf+DHsK", - "1hgToihBcrWeHCr5xrgvP5eiCYozMZvm87mD/CuW7thl85fCE2dJfEgKlW0tn8PdLAf2vo0O3Bw6UsNL", - "eVmA71WIwF5dFqlCMVLQid202qx8uhAXWR4kidal/DFXFoQDOyV0KnNjPfiuZkz5EYtnI9CjnYu4m7nj", - "R6nON+Z3wE1clMbpaDDwif4zBAJopPhXhb7C027jEdxzTV7bN6yhsPw5y+t1Afv4JWdQFB6dIC7sxq6b", - "rtJlWG5Bm7kP6rPfK3nnLUU0DmY2COzzathqzrAklNkosQCl2PipU2zeOV0MXtxoSOmw7oJd3saiY8Lj", - "AMh2ACaUm1nxdYHTha2hbESkSzcAfE6YeVyBl2qctFOPuOGWfDgWLt+q+MkFBcCLdh1sIIFUfM7rH67b", - "WW8xdPkqlOmJ69rUWX2xLWZQF27AEA+wgNMzGwlsoxereLduT3r93mlRO8sxpvrS+B8DK1ImGKxCD528", - "vWoAcqbkjIfwgDDqyD11ipoPzX15MDod7P63je4E8kOJjQsbqQR+hGGj/hG273YRvTh5e9I2pqL4FKmO", - "bmVOa1COViNrZrmw5a8qNhXqylI64DkLS2Q/NiRPwPfJZjOpzKFtYR1SJRGkdEkUi3OghypCmxQEPKZ4", - "RouvgsAPv8CEEA9lms9mTNXBFx6FZOlK60kaMO79CM+JbVCgPxw/qcvTewehrsNc+KRGDai2z2gErv7O", - "2x0wCDamsQle8cXJ29euelIb8DQsZVFhyWJOD8kvRX0xWGpdgksNA5bCBj3ZOHJoMNGJDEWj/KgYI/is", - "rGsKZw892BaquIgaoDFCVUnhqytCIL8DDvwtp4oKwwWLybsf9fdk5Fzm734kNpnIMSCuq0GUdU3xQWhL", - "W5MCTxZLDYY6uyy2wgkXVSslHunOasZJ+aKz5waUjXBtPc8+yBZUVMQFPH09OHr1bieN2UW/NiZ4aENu", - "X5y83a7w2AuPTFG0rbPWizZzkd+qotTjpCwv2TwICIrt7JhN6oC9GZEpi6iP0G5uXlRUY55i7AVavcBn", - "bZggijqjMxW1Tix2oCazPEmGjgH6IRM0M8+QD1eAXO1VBeSDzzOKmlQhRn0w+FFGdB5FjMWN8JXmqCv0", - "dkhykVKlFzRpTu6wQEuIFlTRyDBFvvvwXS9cTgkGp7sy+gp1FjdNZ7Ks3CsBeqycu7Yj/u7H+vG0V73N", - "kXYGci64Qcw3BPreKgZpocmH5JgunR0lwzvT0Uu5Y3Q2s8inxV2iWMKoxjLomrz7cXNgppGGJm1zeAMP", - "HZ/aggnBKYJh9klWYwPwexVyrPrZ+8Ero5xPwM2EH2z6LIbkSITZno3+QdBAi/+lNZ8L0O20CwCKqChW", - "cmXv6ndn0CzUBFns1ReuNp1+4AIIUUzotrJJnatqzmpZVnleZ1PyfHMpUNtJ+3ef+rzThl/L6ZL+3GJ6", - "qo9EIvZVxzGsdVSzjCpqWLK0ykVBnfXSDmw1qARC/6+Q+Pocmn+05XxyxSZmoZiGALCatWG/v1oSWmOK", - "wQVzVfDsnCq+NyNJStU5njKvS5Nc2BWoJ7nub0K6WhiTXWFSECFhDWyGqQuaNNOv9EqQzTMGKVdTZi4Z", - "E34qVBNaDXFvpq/rlkpkkJjDFJf1NeztB757atMeyFzRiBH7li8oX7A1OHpdl7K33zIsuHC0btnf3XX7", - "616d5Um3PQ4Na3fTDpvoKhv85umJr6hVVCz3y7y3uspQFcIeOffGhq3d0+tru/lPCQuJvSKzTxkWe3OJ", - "Y9WUdB9iiMXsBKbfVdLAK8xBVxOX3XfwFNil6ttz/r6Tgtc87qFYGkgbClV+xwcu5W6OyIGYaKBy1L54", - "bCUleydX7meXFgWSFhcAslwP4o5ylfT6vcHMzepwZwf4fYLgqQf7uw931kdyrw3hdxGLk5ivM/G4Vi7y", - "zefzW+RJnHSdJHZolnUwguObm+4HZE8rm2CrvcPdVlGyfL4KOF8akYgfaGR8fUq0iteiHmj12CLEeV2Q", - "gQ5TW3IKtN1H1fM5CoahQExpnfx3VmgfZmbDioFGrMWxsY6WyH8PSqdShYoQSmVc7vGU+bjj4j70Ub0e", - "zLvmHx89rM7y/r17+/c28SFkNo1j7s5dYKq2dR1dgVWW251f1wH8R9dlDvdkPVnhumygqTdPAyS1fkFl", - "xsSV1vPewf7e1daz60SOfGRmgy+FwKkAjQplojJNOGWGxtTQOpNBczJwGaxyRVmKWYCz79ezlpYQpira", - "1HVL9H2uAJiWap2vPTh9SgWfoZJkW1a/rBd07979Q1tTOGazg3v3h8PhVTF4nhfPum3Fjo2TrcDxDPXi", - "0/bhBqB2uszlj97J4zc/ASPLtbKX1o6ecnFY+Xfxz/IB/mH/OeUiDNHTpQw1n62Un65tb5Y7DHQWH8JM", - "hMus8pJShxC9Fn8QJgggLlgQ8LIWKH1zyJYFjfN6AZMrIFyuycsHcQXM661rc+2K0qKwhZpKJemqXaFD", - "VWmI41wXxuEtztjGfdNaOoqC26vBLdcqma7XVpVdKTqYMVHUkU0S+1ckBUKMh4rK1q5I/6xDTUK8Rlzx", - "weKT1R+Lr1d+fFodSOV3X9Ow8pOrLvv+ilFpawXSv67KoZu5kBdHNxzmsPm/uBW6VvJ2iJjBdIw7vguv", - "E7lZ//qr+X/99n/0yYO/7/728t27/7l48V/PfuH/8y45efVJaEvrwVDvFNH0s4GYYrhiDcm0Kykdg2dm", - "lZ5A/WtZYffEWhxMtMDaw2TKDiGe+iU3TNlKlo38Yyi+wFBTwreIFFhy0EFTbMPLJzaoAF7+w4vBH5t9", - "xA6DQrkNKVCPdD6NZUq52B6LsXB9ET8RjXoB/BWTiGa2HCUXYJiFKG2FtUOdl7f8eJ/8QbPs4zbUN6GG", - "sA8W1z+jWH5xVvgmhI/RcKOykeiuOSvAk21S8FgUt3WBBmpd/cOydAhnSTNtr2VR1utvTnN6OArhpmJK", - "GWwklsdCFaSgbCCjIteNPBxtr+pzG3SMgobWkJ+LhbE5x5AzHCDFljzlZyzmEfIVn6prW1aypC2lOSNe", - "puSHJe7Na5s/GhOamwUTxkU59kkk5TlnfdzSPnqkMfgK37QhNQuZDabLwUJmBaYKVTbgjNqglLqS/X8G", - "bqKDd0zxmftSEBoDSCQgdOKRcTPDNqV1YWVib2xlIaE5mgttc1uiStuqRDabwuRK+MI5DMvnInJQQX0s", - "JJN/T6KEo9VJL9CHt0DoTwPdhNA7e6Mia41Oo5jNmv+uRxXt3buPGqz/9/5e56Rxu3TrqCxPAjpt6llf", - "B45t2SQOwIoHE28I3xAHCDegC6VAO4WR+N9T4jsqT1zBSKynz6axaleHJdGVBNbtYKahOwYOIWhCc9N1", - "dqunsJa42KEHGyCEryUdsIme4zzIm5enxDCVegiLrQh2B0+JBaIZcK1zVxLw8dPj59vDXhDyrebSwq1a", - "m9hYH3QA7sSeg9Y4qNJGQ1PWJ0fPMGHdXSulLoYZRj9KRRJ7K5aX0SFi89TNPdQWEz165gTQZFlGHVmx", - "Zdzb9j1mzevtkLz2nyW0GEqRelzSlu+yvEywWxeDatOfVnpvZKqjf8ypf+4+xmQnrLRqeTHC1rbeX91t", - "jm7F4WHTQnblC6nyYrv9q7L3nxvg/fPL6PtXk9GdF3qSLagOUfei6tXERiv7XnVk19m9ag/bq313onnw", - "bP3V15uqXEOGwT1Xez2EaHxvsLv7Zvfg6ua7q2Jz12H3KoCdBTx3d1ztm8CnDqBNczNpze8g8Nhlc3i7", - "yLtjSJUEGDV42LCO7O4/6GKUwK92zYyoPASRGodUcCkPo1dE9Fs0w3OeJFaA0XwuaEIeka3Toxc/H718", - "uU0G5NWr4+ZWrHsjuD/XgOnGWwDX0QZHBZDUaqgcpEjfffPmJTyJEoYZUFYOP78+ePdG02IHMG8/uBcn", - "b+GNBdUTHzvdni5My5R79oFro1dBFDulIHwKeLh9tTSMdZmk7cMFVW5EIP+pBpEdhOTcvgHocJ8/srKc", - "d4CqfZd5ul8eovdaDO5PBdJ2doYbwtFuvdJCGNQNUJJ7bbfb9RGxb2Q4NUynENuqSjgeROHaENT9Hg8k", - "kD92cXzk6KSsOVc6I3z3jTk92hvu3n+IZZN3R10Ye0qjNd8+fvy0+8dHe/aWOaTTwyg+RIX9uj4rR9hW", - "BaHJJaTZjr2SOO7ZC7Oi3VaOrW3TLYFiFen7esDeTTGuBWMbxVkfuKQn6dqqRx0yhJu4hWnuEGBTniRc", - "s0iKWNdl5AXVRGcW+NhW/ykk+LHAAfZJUYQdpRRCo0jlpenRSddO3s8zR/dYgTiTQjNXguRnttQk5egE", - "LT6PoY+aFIlo8VhsKZ+0WGQnYvHhGH7AFKC+Sy6J+xg1jFWO4IWx0IvcABPbHhKIe8tTppxVlkw5eoy2", - "ic6tSovjxdVYAsPUPGZqLKBZAFr5j0I9ObwPMAf9XqHJQaWW0ShETTfq/Bw6DHXsQSPgpnC18TDaXeWC", - "5CJmiryWmhlDCbPk0AyRu6Lj9BMBy/3r3cQr93opV4UP5iZ8807C1CcDNuNQW/RzjA69hnJ+7/oieqe0", - "Py+/uoQ/99bkKhEMLiUDNL4p3HbWIMdiZ4bUzOV6FLk3b20V4PrUXeixkeBUV0vy7vi4FvbgMjS6TRy5", - "RMs+yOxK27C3wUaycTRXcS9X4K03Q1p/Osx0U1KpSIifHVS66mH0OASWQmuGrZrivM6+BlplMAeKC7tP", - "QDRrJtgoBG1RW8rAyGqveqlnDmLVidI+p8LByxcILKVTXy/1zjTXO1nEd+xzvYPwOA8RHucgiF8Qs4tJ", - "nodUI3jkvkPevgUMG/wF4Z1tgkzt85Te3304evho8HC6e39wEI92B3R3//5g7x4dzfajB/u7e/trkoo6", - "JLReP0c1qDEH4piLqPWJj54PBTW35S40ZBMXj33JRSwva9dfMEC2+nUXfLvp86uh9Z2HEEwJSqg21nzR", - "wsmO8ZJnEfSN7XzSdFH0LWzovP9mtPup1h8cXMsd8UblwrpVsUHpQkgrA65uVn2c12P5OCCf+LJptaof", - "775oo8N7jw7vfeqi+eSNTWNsktMtbm5bRJiHQW9kh/gk4YodyRsoe04mslZ923zZ6/dyUf0bhYFGLHXx", - "uFMSV9uB7YfZyLprpQW/4Kimr2CkioXBjA9BUvH6CBZwKVAyQAR6CrYwUrHFWVRA9MMdVXQX6AbdYs5E", - "ZxOXbTIGZkZy7fL5uABGjP5H6MSBHRySF9gWH9HUqnVuELYKd9X1RuOljZeB8+U/bZWs9UM+dfoVvgPK", - "FoF/4bRhGZzJdn0XVjo7JL9IfKfQ9oRs2n5tc1SzVps32pIth5jvwWvwY07UPCQ/FuJlIaA6gXRLM/fn", - "xDGsEjNqu4bc4XYcAgkrO1dBoej37Ir2+j2/UPBnALfibUn1K+evSoqhQDJGEzzLZZp+bnjikPJxJlwb", - "HmmXPAKb2yb2uNxMFk+s8tQWk2pTX20bUrzkpap3x2QLwVD/QpxhG/61XcSv1u66vUcHj+4/2Ht0vxPk", - "WTnAzaLxU0QmWB3cRjk5yvKJs420TR0q4uGaR9aqUMS+gLerJPNMSWA9MHPfYfXjj4aPqkhvscyBDxdD", - "crCQFlgaNywIZljwopY4yN94csFnM/Hb79H53t8VT3c/3Nd7090WBGv7obDZ7agaXLBio2bTga3TGQbj", - "QoJSuhWv7jXTOANyygxB+hkQGqF6U+RTO5LLdW3Fg4R1sL+///DBvb1OdOVGVzk4EzTCBS5lN4KyJcGW", - "ZOv16SnZqRCc7dPDumQBmIPGOSOu2vqoLpAOd0f7ISppubhLqnF9X6StS/7OqY9uUm7RMXm7UC1XTnlw", - "tff3Rw8O7j281+0YO/PwRH1Yz2FcO/s9XwujuvNbKE2+eXxCoHc1oxEUpn9s7cGj1vKIuWbkl8dvLE6T", - "kCRjalDeiL6nepqrizW70vzMleaHFWFsJYcNU6xFqjy4D9lbu90gHEPhKw6ctHb061wwcHwD5BXY18BS", - "rDLxftu9ExLMLKm+ZlFCefo48skajXvMVmyYKNus3IQuV4zT5VeuwA7vdjKTFcYvm+qD7xCpSC6KOkHD", - "zc7dz+Kjbef/9qLZfD+EEnwErJ7DGrO1F6+xlJliF1zm+jN0JI3Nvp0lUqorvdum+rxmGjx4aP3hmrw7", - "/k4TR2tEG5bVtTFHjWsQ2a45uSud5xqJhIm8bbE67UaXrV834X7Lqe2vg+aocYNWHMQYOFcuNoeRPqVJ", - "lGNlLFrsJ8wKLwoEF8iyZGmzBJJESkGiBRXoblEV+DICKAzDYEwtPJnMgvEZ8pIk0iK4Q+UeVzTKDgJe", - "A2EILEtb1dKMlpQatZzvpZbJuLJAdWq8l4bLzobRmYqkelhPamQFXNy+UrOpJnKuUb00mP8wbNa0yKiy", - "aQ1U2CJoF6nVSgOh24EhNph56EbFh3AwrarshBdMWbcrSSMldQVY7N2xHnbNnivyoTeHhtYH24F0rWs0", - "cJVZaLvOtRJD92MgM+hTbkikYcw+XBN06c2cKRU5lpGqELIz6Q87B1aCXXtSAGZdcbDaTLA2DJgCCx9Y", - "kb9fWJZ8m+C96FnbdZbLRTBf6+0Vqgp31TbAdp4aXNHwavULGgyR8SpC3VpQvBIqsAmLdRXkybKYCNfY", - "K69gEJItzJ6psKUKnuR2l3CbsPIL31nRe11ZYfB4dMVoXA/JeELN4kjM5ConuIoP1tu0XdxrxpSHSYyZ", - "4Cz2WmnhjHVGM65dUbk4Z27l8LM1ID0MlqCoOTlrGyQL1Xl984NdDM12DOtLx+B3XcMOO8l1ODX3jcpx", - "rZCAgHjKJN1O4aRcT8KOsdWOFZvnCVUIHdhxyHqZJlycd+ldL9OpTHhE4IWmh30mIYVsAo/0DziX7U6z", - "gxcmbXXGTu3gXIah3ZDGd8sp/ACz3G7kN6NNZ8e+vwPvd4pDC8ac/8gT5lAu3wr+oULo9doKB3ujtrz7", - "lk5rGferMK9X5dyOZIMnPteBLMW1Uo6vkMZiV/ACW2OlNzxqHW4lj6jsnYnX8w3VU1A+DeTkqeXXDYgT", - "MmWYQeSnttJdF7bYZSrBOjFQOu/vclo3rXYNIA5UH9wSJdiGYrNgpgDu6FrTtm2xsiaV3b0KmgWyVZgo", - "vnRFkIhNdRrLSK02fvJ6pWThgrkl436Otnxhh3I8PpKkAEJwX+2OiNAsLhmIfEa8G22WWKYZa2MtKxVT", - "oeaYUggnDxKOFH42COAilV1rj6AFed1LolhKuRgLLgpzK8KyMSLYBVOVfFupQMmaQ7LHXysqHvzO0sws", - "XWUHNMN/pwk44/wYx6I6SOgczOhD8lhYy6LKM1Or/cpip/UBofiK94oZhcVQuVmQmWJ6UZ17qAAuyHiX", - "UsWtlcWWxDeBaVD01hIjz5mosrKim6BqaDua2LdW4wFt9Wp86vRPUisoTZoFo9d/L9dMhYXEYkpFk05B", - "MJWjUlFO8EDCAFyhUPeXZfEFnkoH9JSy+//2XZY/nRSd139rNKsgpHjM8MfWbBs0wUZZvro6DZ/cxqAX", - "TKhbh4uz6pUgWz4Y21db0tstVY863ZPdcvqaaQd+NDuaRfWvHzy89+B+x7JTn+T2szhgn9vJd5Guce61", - "7NRxFw/Sw3sPHz3aP7j3aO9KHhafodKyP21ZKtX9ARgRA4c1+dc//vnuuE4te/cwmnt0pUHlWfuQ3mYd", - "BvTu+F//+Kcf1bUHFGI0q3D/LREArfE+SXUnfchB3RnYzUm2Rr9/XDMS0ILNkC2GoN4QvmPXbVAOpgEo", - "0k0KphmNuFkGGDm9tHHzRZMGaneH3huDDYm8tm+HrwqcS+fTogXEsLgG/2m9zA1aeNi5ep3Op20e7VfN", - "r2I7UnotqtESHYIVLEWEDdzFfC6proWGx8z6HfpFrtpq3o5tsR6/t5lUAUNBSN5aX1GoakFDnnQvVbe/", - "sZ0Vv2XNrNNc8fdrzmH7EbyS1TdwIweMvlGWd+2olNvhArzeW5Npta7k2sKdtSKU5a179e92yENeLXpS", - "3GBX/14l9fIqLzZozdKjG4Nb8rLvfo0kWqipklUTMKDJhA0qZRCwNdG59QjCmXeA9YFc0OhczmZ11Nx7", - "7SjrCCCEaWP+K9QY0Ez6YPV3NosmRLdFCxr37ulxj0gFKZnpuNdwWwUTMVP6YeI+UEeJGa2DPS8T6RuD", - "1H4G00RG57ZkIlbiH5IRSRkVmuQCD3/Dq7Y7Wu8d6veyyt74dUTlbmUp3WaSKVvQC46VWZxPZV4L6WQf", - "uNEYeor9HJJYWtyoWr1oN0NoZtMkD8tJ46VDxdJ1DB1COyl8bGzZFg18M6xSLX5nSvYd7AFw7Fevjvs2", - "gAGDGO3AapGSfqJ2BL1+r/xEo1BD+Xs4EhmcFTjuJvB/urqO1ex29KwqppnRDgm8JIcGEUD6kjDNigBp", - "N0WunqC2eiXlAsMGXfgHLKj/uiUQErMIT6RePYt1Qr8GcTcyENxKh1IQ9kMkjIdiTSWf184h3ByANTZU", - "Sr3bfqoR4tZLONFGZr36qZ6wDxFjcRM6NNyka9S9ezMYdf+SOrShogq7a42R06uzG95cqhiOtW21q9kB", - "QooBnLpiS+3htMh1HvWmTmjV1a+CY0xCQK2hBh3GL9iH9Wv9C/tgEGk9zhMWr65tcSQtq3KX0aYVv3aO", - "ZNuBloptrLV5AzUobeT6tapQ2lfZXRSidK1upPjkyu6cMuPbnjoyat2hesmYmkvLdUF8k3qMjSWlPnEX", - "PNlNtxskeLAIW0UcvG/HbE+wt4DzfcY/rCEe28AqxnWAlPIgeWogFql0K6UfyMEDLCKmG2MXfL4wybIe", - "gHMQQGX6pAqtihkmvKGwy86Xu+lfXI12c9tZ7T0kHJ9WQIZWiqM4kXSyDoH7adHMW+czukQrTquT8MH+", - "wWi0vze6FgS3H9YVlutp+YqrZ1rvpy05r/IeYVWPmeVA1R6KdO3VItm28F25mkQbxWh6iCk8GY0YSdgM", - "4faK1NjNnsXmp9cP3glULh+3oH+/UW7fvA++Xnyn+JSYV/5lTR841zqaUfX5BodoC5uJVsD5Atl7+4PR", - "/Te7+4f37h/u7t4EbHaxSG15Iw9+3718kOzR2UHycPngt93Fg/leuh/Uw865iLvS6s/QtjXKprwk66hI", - "NZZGttwcMqaa1c/rViNAc064YANd5FptTnhcwwus/33j+b+and/OYK3scFqfZFWEoIboIGXdDZKXm8xa", - "30VzNkfP1s/iWrlMzYGE6a05FGjVcTBY62K394kYD7noeA29rTTsfBGtza/bdBWFPOx40oO73LLiIfJu", - "QDxUZr3uAl+95AK207lU3CzS9bdF0awAJMe46d+1ievIUUNyNBdSsfrPRZhcVYmCl3v9XvL7Qf3MuN+7", - "Y4g5LOOCAN1WV6WCDmFkCbtgyfpVwCal4qFsJDtVDBfih93B7iOMQ0h+P/hhNHhUjzjo29WqLt+ub137", - "ddRlDavFBH0Rqt1HV4q49uu5joJ+5qFSeOW97FCOHY2Xheb91eFTd2sbXD5e2eMGJlCrAPqpkp673CZV", - "oSkGi2wI5b5iqNUN7bFKZGTK5lzoLnbb/VFhuL0Hhlvy2EGNoy5rZPGdavdAK1U64WnKYk4NjkqZNRkM", - "ex1tcU1d4mpVTvxbAWltGBbXHm0GW9iUcLXpmhx+QmbvJ2m/3TTedTggaFfzKiqikWHDPuEzsEHX+/dl", - "r63SgYmRGK926CHfSpJ1PECXcqC3k/TJXBpSJuN3tLflot0uWIwfanTma9E3LEHsfRZolQKKjK9jX0fP", - "SKZknEdl/miCgy6xQ1Quhl2F/I1Du1H7BqZ4g5Fjs32jzaDRzT7Ztt8N2yQQbPtW7442b/WNGEX6vTyL", - "N/Mw26gbB7sSBvyGFMSAiaa+7A1JsDKZ9x04+uvqCq7qvNaWHIFIlGfewQI0tUpJAXcLuhhCcb3PWMIM", - "C3VCZBKXWRJcl1x0M0vdvf9w0ebiRI/U6kB+Ziwj1FgkJfxeCt680MB8AdPiLtkaedxwbR1eA1v4yK1W", - "fXAPNkpirVtVNeG2FWOwXL5h87bIK+cV87efTU00W8VS8Qy/JqS9bi8m4N70YW+t9uObMMvdpZD2yrke", - "GgCwHp64wFn33y9jgUGsqxPvQcg9HyKLN3SOVfc2+FFaSSVUoRKdHJQY6spkYalO+CfBeqTaQ794AQFf", - "9xm9rmwh4YbQOW3WoGgrcGk90ZtjiNpLxryh87Xovs102Kr5/fHgb9bcTibDw50f/vJ/D97/Z9Ds3jAg", - "aKYGMZthxNU5Ww5sPScwVgzr2LZYTKKnDZ27M8NoisY0xI13XKk63nujgnsuf6HpyhQwVK1SjGl344T+", - "8h/tgV6VZXyLF8bGs/vJtVZuoiatkf5e3kqZmvugep9Rtz0cC0zSO2dLTSol3qbL2on9ThevVELxyZkl", - "9yETF2dkyrFmph4LUO9pFLEM1CpXNYjjWJRENqwYTar9uFJz/rw4z6wNrID06RVg5Fdv3zx59faXZ5NX", - "J89/eXw0+fn5//QOe4JdDuwX4gHQ3sG9+67ce3Uld4MlR65eOWNIjl2+got5mOWo2SP0GfhIIQUtWRL2", - "IUpyDWETuPTGJNevkbGatXz9mhOfCKpsTBIKz3Dg3wDoDWO296qLLuLaEyPXWCffWXm4IKuiiyWccQ+v", - "FCO9JBGsUAJbEV7tcmPri35/0e0usXCzgcOOqb9w2AJW1JdcI3CHD4KpNCZbmELji/naJzYM9Qrwt4+L", - "DoMhmJ+5ZtPo0eeoq/p2bSHVC5kMQM9rKT4RNKvbtQimEGBXNjWj1+Z9mU8Dyoyzcc/5nAYcLiHHymep", - "f+oHtDF1bGX/WwvBhRM6njUrc9hjaZeqUUmiYS3RZtCe75GCeN+CZYxgzTaJl1eCDGvd76TC7Lg6xSHg", - "kFgiPvy6tO3ylHnAyQG+tDkbea2CWZlZZSTte3Ms89C2rFsgyMcHHqlYZSPwBRZfc8lcglIHOBqktzo6", - "mXsZxXP0u2uyVZjA/BIUaderfoH1FS2O6YfiC9ACtImmIxbnUVbU2n3xBKsPvHa7BOzTdYHDaGi5Yaz9", - "OhWtWxNPVaubUaWq1Xnb9sGD53jVGu7XdrYaxFl+o0aaIXr8K+XmR6lQL24Hf7lxyH68/GOmEFavCcjf", - "Cc2epyyeyNysP//QDKRHe+UXlWbLSsXeBkCRiKNaXnMbL/DwJOUYVlcaloNFueJmCUYjp5lMMR/UlwfG", - "hcQP4c/lh7Ek68ePaD2fBTJnXjDBFI+w4K2FGhSoMQFsX1n30JbAXIHARRHo1dMjZ3fxKMqoR3ODpOcD", - "UB+fHIEznilre+iNhvvDER7mjAma8d5hb3+4Oxz1UKta4BR3plCw3yWSO4260OCPYicJPfGN4E1FU2bw", - "jV8DkAgYgOmaowpC5xUlMqNcOS0yS2RcEAyHt7GCg79QD+2t3LfL3tl4jPnWmAbEslduc9+joIxnB6e5", - "Nxo5rHjjrl9MYrKZEzt/d2G05Xc7SXVuiQIFDVbUPC9bFkv/sd87GO1eaUzrhoJnN/Tht4K6bGaG2vm9", - "0ejmP3okbH6iyzZ3cWHVE4eEVD1rv76HPdN5mlK19AtWXa1M6jbBmIFdy7W1epzRJAJWgWWXANCG2edg", - "EqU2hFvlAqtV+xeBQuunwPbtN7lAa3oi4+VnW8LaN7yN4mOdncFx+bhCz5+PdgoyXt1I98iDlluqvQUC", - "ekKLUut3dlIORo9u/qNg+014BNmqnoBdYDbXGPuUIAS7B2GSivyWS0NJkdfwFR1pJ7NOC3Lrl1fRzh88", - "/miPd8JC/oATplIqbJaIbbPh0K8cZ+ubKY/z2lvNE/7Rs567qTwakb2oUJCrH9HqtdUUBlevo4PeYds3", - "7fTiOyT8g1s44W6yRbXduzxyWOMU/Mpf03FyvsZpKYQEZbkXzHwpND+6zSvL1WX4E56ir4WAX7BCwit3", - "a+VS2MlULqwCHJQAX5eZm+697+rC35vySSVcCP0a0DVWCDHWwUzj5ZD4NbVKPxh9tYXoVxc23Kx+vk5g", - "eF/KCdu7jROGMy48Rd+uqW/X1LpTbqnFTwEPZuWUd7BBXMkC8eezP1zZ+vDN9tDd9tDJ8iDYpW0NeJBD", - "4kJzIxkzohdQ7xSczRb4yQfhGKqG898JVdGCXzBE98O6d3lieEYVhtikBLwE1ofeaphYa5YoutuB7gY+", - "ILNc4Cagh2YTDIGZtAFxlqGYXAgw1lLto2ZKXMWVCu327AcN7EWH5dVILhdSswLYUJjKbY553tpqx9jt", - "cCzeOMRbWECMKve8RrMEcXvX2H+kIHQs3AvfexbiI+I0TUvORRWCJ3IL0Wm3ZWWqONKJjmQIdOgNE1SY", - "gc5YxGcQfYjTOmdLF9ga7LBTKSsYsB8n1IXxgVp722HgOgxdCqMUPyueEUdJdf+NkMYV+S6dXB5LiSqo", - "fRCsUDJP5JQmE7s+5yzgE3yBLdyilA6X0pskZMxsVf5saRZS2L/zaS5Mbv+eKnmpmQKf01hgRopba4hW", - "KAREcom18dJMKmPjHPCbO3aIO3+cs+XH4Vg8jlMu7BvavkITLcFNAe8Z6cFDLPdqoQd7msJ+8Ke5NjKt", - "QsB6urPDlLnJcuNSazQz/RD86VgYSf7wIJcfd/4ov/gRncWMxkAnlSZ2Sihbt41aTyjMfoJNA+52hgsw", - "7sFFasM85ooKg2MqUTrJvLqlW0WZCCxC21zhiAqSycyW2ECiWlAguVofCFpBk4QYPEr+XRDccSdb5uMw", - "CNNpKwChRYxrHCMuyPGTymEaHTwMnyfNIsVCESX/dfrqF4K3MuyBbVaGa+EacQECA0TRYO0Qx9OeA0yS", - "dVRhfcZxj8fjXuHOjbdxrLl24TKDAfoUf4Ch/WA/0+fxD8MhdGXdlYfk1z9sL4dwlrLUAqKOex/7pPJg", - "zs0inxbP3ocXtA3H7bTGCMiWvea2kZNQjpA7lRvfXpFgupPuFgAAQVJyoGrgypQLqpbrMioDS+9WUM5s", - "JGNlMf4YY+TiuHc49rGL415/3GPiAn9zAY7j3sfwCjivZXsxQLzPCudmQUT3R6PtzZDgbn0DPssOjoHP", - "rAO2akVFJVPYQYdH++fyD/xb659PCo1NwMdLjCar+Hvj+13L9fD1W9j8X0DsAdaRMowoGVRrssFNgbUE", - "8WrDogPX8oxUVImqihzwjTT0ASoilnh9YLMF6va9GkBFEUuS2z45d3VuCr9dUUvhq3LU4WaV53utX+GO", - "KW50W7ddzZ9wN/T71Rn2A2Z9Z9RnFz4GO1xJBlGCnI5PbGOwnJzimAanTBjyHH8duv96pRRRL88SOT87", - "tDYFKIpAEi5cokIlghrkFreW+JIFCires/8syvhtWRHnX//4Jw6Ki/m//vFPh77/r3/8E4/7jgXAw3ri", - "ZwtGlZkyas4OCaTjDWgC8e1uMljxl10wtST7I+eMwEcwJMGiItpb5oCYORavmcmVqCR02Ip62nXofBgw", - "Hy5yph3QEjTkM1fuxwZdBgxK/izbpbzVE90PAFbiDCoTgFvR0wCi/XFbVN2pyL2wLdfOuWbNbcaPrkQR", - "buYvhn0wlnoHdoBXZDC4xKFzhw/cpMnW6enz7SFBNdBSBZZ0QqWm7MbpN8NvPGkzT7Icpc5QcJUtb4po", - "Rqc84d4W2lKPxh7BlEYLAI4qAp8LNHjfxaEfKfAYiAF1EZp9bDoWr0530PZrWGRyxfqOEyiHAVsWrJMu", - "AQe/gPyLGwxbG7i2YzFjFPogR88sE6jApBcZm0XHAqFWMPiWm1ptvP5YWKxfhy0NBw8k2QRfwu/PqWGX", - "dNknFcnX1q9JqAFNXfeh8VjYHEi3BgMEkyGVYQ6Rn9khDXyIsUsmU2wGVluciCvxjt/eAoOHC73ulwmv", - "/nM2DdYOi0gnjsP85qiiSmuoxJ5enfrd2O4TLUmUcKSGiIqxmGOEkodXlqK2q0Wm24KqeBBJuASqcFsQ", - "Gp2weN7GY59W+urdoCRT+07gOP3UJNevTbhYrE4ADrGFEFzvUnzm2nTzKboe/0xORbtAV/EqWtMys/zG", - "ru43D2MHD2N43by3MeTye+YxMm8uFNl+4o4ikT3tra65fVJZsrswNZItDz6E7hqpyMnTI0LjWDGtt/+9", - "DZEwU0ulpfwH9yOw4ruIiXFjkcruhbe31Anka2EHr92oCfXzalZArt5vO7XySK03XVEpqbzybv72aHz0", - "KtdIKfSWtPbtJtkYBcx1BDmVVWoZoGiUsEJ8Kc5plYo2WZVtfHFx5awVl2wrCN5wB/L27Mvu07lo3g23", - "wBSfNRjiHTLCeg54ta7510TNb4tddPNaZ37+skhzdHtS0G2bokNk/jWpi3Fj2YALWgSW1gv0BTMWd+Um", - "9XT3hcDEwc7tTrUd6NLOupiWfZVYABmcEFpi1uu+R7ZJN9XX9vdn0nxxea4isbgl/yaidFB2y7Vap+Ae", - "uSLdN6ff1iDmbjmexhFYYJHhgbNNI14T1L2keimi7W8hNZ+donGxK0qs8vNmcWHJxuelnnVbch0UXC8r", - "wnNd6LVckFkC8LXWCRDzGQYRmmqFdRzl3i2MsqhkrqhhLnbya0xIPskT692IpLhgqgIhWb1Sd/7AaNrN", - "qpJnXmtv17evXw6YiGRcOE/aZVL35DMrTJb+a0nGt3/qvsJEW+7FgzaB8RP230a5kwJT9H/t/ehQRf/X", - "3o80ybhg/2v/cUIN02b7xohldFs33W0rMF8x8YH+wuuLtsKadgydVxNk65TpcXWvQpk1PFxv7C4yfD4b", - "9X1+Ia8JItxJwrs1ugdN6rahVI58vQS7pVJ5WK0GVvK/t9B3WiPoW7fX1T7PNY4ABbuvEL9FKg8HSx2e", - "NvfE7XiTmCum9SZjRNGqoz3Ct/9TmSTspK9klCjW9Ztdootdorpca00TtuHNGifsN+7I+14QW2i18dE3", - "HLBbcOg4iqzggNU83CUS2EJqg4++DFCIf6dMHF4chep91tFlWnKKtfeaP1NHz/q4w33Y06NnZTL4LTlQ", - "/Thu3SLgvnv70tjjdMrnucx1Nc88pSZaMO0wGBJWvxm+NltFKTe0Wiu+YCod3eadduvGiG90f0NmkuaG", - "WuZtoyA2KSO+VVdlxLUHZcTiUzucCldNqu8rDW63ZKd4hOquZFwD8v7Y7zKukJIE1Tk0KfUYgqrN4Vj8", - "b1CMfjWMpu9/8Anx+Wi0dx9/Z+Li/Q8+J14ce1JhwijuCsM+/uUZhp7MMbsAa8eW8BvNcZA018aSni8S", - "82+nuZXRN91VN0+F31S3TqpbZbnWq2624Q3rbvVCU7euvHl6Cy24ffYnVd/+5D7lmmqp89mMR5wJLNeF", - "Vkm9EkRtVcxv7uTr2UepcEEclQjMmiTSWY0suNYGCb2oAf1ZQxz7rVU7JKHGsDQzAPsUMagebFMH9SI3", - "sbwUvogHTtDXg+PlfELXu+9q4jsJZ+6Gi9p31XSLQpa3req6D3+lqbMyczXZnXJZijbt2uXdEu/N6pQd", - "rtrb1yq/ZhKz6tvq0mWgIQSK0ln/U5rbPOPizRLPckjevHnpc4pBPVG+xKGRvq6hr309FtW6hkPyvCwY", - "aRv4HkB9YLHDIID3faXAmNE44YJhEgbTofTfejXSOz0Wn18CDpdavW3fe5djaTf0DiXgO2MFtyJrFsEG", - "xnmKi6NZqcJanBYvb+Kp+ar4lWNAAcYTkvV2aG7kwKEU7CykxdQMwwqfJDRCVGFoZgEvHTCMRbitdoVo", - "L0omCVMWyDTLjRe3AFPYDY4LI4tS3U4yO4PuJ7kwPDnr2xhIaJtoAmXiLdTNWNQ+5mQ+BG9AYBIcoWKZ", - "HXGj7jAMmoOpEVohzkL1k4Qml3Spx8LBPdjXsVi9YpHF/E2SIflJItKOrb9dYby2+O13eizOeJywiQPK", - "OSNcE72QyjDBYpLKC6br32VUJZwpnMRTCiunSUqXCKVpUYXt+siMWbjKGhyPhH9TEXMsowpfLqZ8OBaU", - "7I1GJGVUaAeuoekMLxzXB8FB1Ab0PaHkYPTIvdXYN4R798u/BadJKXYhIzqFirlAxbY8+TZuYOrKGts6", - "+bB9M6603a/CvunqVdY2lmtfpTfuk1yU8CFo669412C7TK4EztO5JxlXxTXoEJOmLKKwnkLWv4MgujKK", - "chW6IGGrK/W1/x0Fx8r0TnGpwuAcCZoMIsgBkIj8uMAzLfEobX/fQlUlUf05LprgIZGKUFKh69KiwaIc", - "WeMWgs6elcVihS/+frb9vT87cHwdI/DHX8y/pvsJiUjOZrUDuPlqsgd4XVLcKgn/Wc/pU8f0aiwu5nQu", - "pDY88szQY3pXleZvCmEnhXD9ygapeSbVeXsk9I9SnXfVwHzE5teliFVn+AU6ImB4GKRz9/4ItIZbZQWI", - "5taVtCZ9FacUhS5udBGrTBIp5kxVrfK37jaoanVbFmkTLlNlnd0FLhooIRP3oy00Xok3RxdD5Hr90war", - "IbHBZpcxa9OlVT1KJLmr8Uo4VVXAB6sLaht/0PfiENKV37AtlN5XtyvIVAFodCNSa/FxD0sagGodi7fa", - "Fnc4s66nM1LwYBBobcEWcrng0QL6wd+wf4vqSrPsrIDS3z4kL/AgV7bNfnzLlmkBWtMyYRaN9SJNzw5X", - "S22/Oz7Gl7DNwhbVPjskvrx2cX9oaFWFYYVZJFQb8osDl90qlHHc0TOwYVbmt+0AWstSB2MRAmsFrFPb", - "IZ+Rswpu61kLqKDnty/lXH8xrqKyQI2di5HEqY5Im0zEvbYgD56EHT+7o1GobkJH+Fg7jBtGj10ZzEs5", - "L4rj1EiZZllX8nXDRCq+SNM1NEy2Sg5CtIllbv6iTcyUwpcddbcRN9mikf2HoedAqA551B/s7bFoWSo7", - "w/BSAVPt9XtM5Gnv8Ff3r4s07fV7bjzw3qXJ0kq1iysI+RvgeJsdfuyHdqiCufsnFs/v7Ep0hZSKoxJL", - "ZqMjUeJxGhRAueJ5wS29DvRv/YaqYP82rjtnA2hXI17bBn96F6c3NN6a+O40mob4FKSPOzvFKGY1bOFf", - "kiRfGW/dbiZkgaqMI3t0GyND6GUSJVKzmp/s6wFudPbChujdbm/zazyA4cW5L8HXJRDo1L166t/8AkwY", - "m0Ju/JiJn+6tx96sjuBrBqHQK7MB9tdA+9sUlPPFE9Ln25KVqXahkG+0eXVjbSfCBHVrlUW4F2JbkpXm", - "RqbU8AjLAUYLKXWF7Atoflu409ngC8pEC5U1FrhEjDMg1TNnzT9zIuahszwSWn3kvjHE1136RvgN/6h8", - "48eKcaXg+H2vQWFlGg0l0xRnM5LRXDOQM3OwiC2jhPn6jwz86hHNTK4YlrZlJOWCp3larbkAO3ZBER/q", - "bDc965NpbkgCVjZVPPQxS5FMUyZiKyeNxYLRC84ULEpCDRPRcqAZlsS/YATKPiSSxmipyWKKDjNMpFQM", - "KBALWKTM0JgaioLOGZz4ic0FOyuq5FvriGAfSmoA33EuvrfVdKDbMz/QM8KwXATXi6KackRjJqJgGYXT", - "L5uNfX6T/ikzzYneUYDVtXjpXUZcVU3XfjhfRjDWVwv50YHNrxF6dbtSXU+i8WT073mk7Vz9HO/IT1cs", - "8bpT/GU46Aqi+2KcdHfvhZOKxLn9XOVUIpn/We2IBUOpIUJAgq7dxuv614rqrMUyX4nn7fzh/zy6hnXx", - "C+GE/VbFvq0OYDnpL4HlulW9Fs+9I7OqsyVVbXJ3x4LdoO5OfJKqwuW+FmNrDXmv4NtV7mQURe1Lim9s", - "u8m2XdzIddm2t82uREZUGDkXAwy1DXNwZ8ZtZdXOdPBvmtTTmF2FZd45iyw9F7cOo+lZY0aXYCr5M8Ra", - "r/EfRVIpiyJCDfu6oL8rVsNqlgXa5soCo32f9Pru+Hi7jUsos5ZHKPMVc4hKZhO8lsarBtxXF0wpHnt4", - "06fHz0qfrMrFkLxKuSFGknPGsjIxCJMzAbWzwFNpDLsJnNLvMWHUMpNcmI2jKJvezGDKH6SDlfkCRUlX", - "S+KbO7yzOxwt+18fO0MuQ2gxgfWaqaFmY41rLmZSpVYuo1OZQ+/Ag2CZYD9tsMWMJ0wvtWGpDe4EKAg4", - "bliXyNWed+/ZXe5jaDOcHJt1mDGVcq25FHosXMpNxhR8G16H/itxakGHgKEFfz2xTPLLiIGEwdiwP2ra", - "Vg2Rr7Cmde+wt0OzbCemhrbE2bnhfcKQfsSgRqKX6RQSKCEq8lyTrYSfW/WEXGiSwB/ba6MiJ/je566s", - "f/2TBSt9JGYyWLfU0mxBzH+q5DjH1rxj8otga3ekZ35yFOELVj24nhfOZCuL1Rt5rGI0GWBdfo/FRHLD", - "E/67ZbvQCdeGRzaLjBb7+O64YPDDsThmRvHI+nsjiD9HkArUdHcyJaMdwPHbjzKOgH77DAeHzLf9cYpf", - "fHryFtulLJVq2R8LKlzHbx6fWE/zjEasOVDBDHiSydHOqw0x66e4TP/G8ZN2gmsBKYIb/s09eXXYmNYz", - "pFuOqMzWqW0y+9MH+Dpp8puN4+u0cQANl7PZKrDaPMBa2J5xIZM8hX/YP442QdUZGi3eYdMvRvK2w9n4", - "GT/Br+JQujnFzNZ4vhMHjF2wrzV+FhbOTwGFmFpkYvAWeGz+jNT9+V0J1XX8Al2vbkV9/fQv5mzd9s3n", - "xuBBU6rr8bUcc0tpfiZGrreEXVLebgl7ksjoXDt0naoJkyhGETIffiwhzp27EsUETPYlDpXKYp8x3R+L", - "hjHUgjhpQolhKuWCJjs4Z9sJgrV7ixq9kBxz7iPMmRloMHVnMkkQkR0RDWE20GLmO6h4l7WrMFltU3WM", - "GgmwTTJlHsB+O6S6/ZVy86NUdTT6L4UvvqmsP6I8Uo62/w0A/O1f/CRA/mP6AcO249w5t/2Itl7I8kdr", - "luoT3Jtxb3+kx70+Gff20nEPduApRXMuNeQeSbnIDSBbPrO2Nsyqvj8imkVSxNrj6Htr4v5It8zckWVL", - "wu59fO82xR5HVbiUr91HQuwB2hF4HxOIyFb1wLkzGffx0MVE5sa6Hty5cq1iZtA8sn3rfuPKGfmm23fh", - "5H91x7fGo3CXgV1Wtt5y9izXC9Zucntpi2blZgrrUBTZ1gvydznVfcg6sJZ5QMxb4Xvw9on9wG3UjoBP", - "XaVuhJv7t6IRHYpGlGsVxt+0wZ5wJXvqsCCc7EMmlYHfHW6BoyHUJBAMBEthvnp6NBYRsCKLFqlYKpE7", - "OYh7ews//uspef70dZ88w4LP5Kd8uj0kryDTBnvVzl80FlYSs8wrogCsbmcQh65nO3aknpsMXIcPXEmK", - "3/t8nh48GQEvj98rH7De7y0YjVEi+aP3UtqPBYCkX7+EAwT7594str23VvgAG75aDh7PDFOr3R67nC1R", - "wKC4S9qjCjrBDT+MH9QeTK/8ppUNLNrJ/l4vAH7y8Vsdj5svFH47Hjv8rivtCNl8X2uNYDyIBXMMscDq", - "dV1UwmjLWHa8bK2CgZ88evZZlIob9VKv4121cgH/rqcLZ/rVOpqy2j4BERcVdDZ6en2i8sIiXNvXSEQz", - "GnGz7BOaJO6OcjdBER0zKMTfqWL0HOz2Q0Ahc1/2ycXgq+17Ry2JuT63PThf7JBAeJzOp8XgCB406zXG", - "NYcUXSNJRJMoT6hhhM1mLMK8YCzJo1t8ucVQejd4dsqPBGFhKhH2+VdXtjBME7h7JVk0KW7HbvWOYlFC", - "edqOJ+8ENQx+xFCDKXQqBcQuJJbiaKSk1sR1NWAJn3Moq4GRElBJA8xcNGVjkSVUCKZIrm2EPgx9kCmm", - "da6Y7wDuLkdRfVJiRWZKGheakEip9Fh4Cn93TLRh2Roye217PsY535Bsazt3X7ojI3VjDO2mENeEwIZY", - "SrELDnQEpqE7CIu3A7prKfFrOfhvFJ9D7GumJLVM1sYl2WPtl9Me+lr2dGsJ09OiVbcSpkWvlQzJSvbg", - "Wqy/SQmfHveuFoEY+Pg5b4WDdI+ultH8M7zU8dv1zNnwINyjT5xlqBrrv2Ph09NKwmJXA1ZJ4V+bOaky", - "8tpRrSX9bob46pzle5NZt52xvO4MwutrRu6itVTeNoX3yyOE0e0iTtx23byvm7ZqyFs13bQFfmBzgYQv", - "ggJvpjLCHSOuXKMywheFAQC0c4dYLMGDelc5/TXfs1TfihvcaCq/rXCA0HBtqfyW67ng1bWK0jvXppua", - "5Hr8M0nwdoGuIr/7Zf+m9XdQGSqLtckFDQTP0sws3UveV1kGnWn+Oxu2OIKLuNWbcwVfI6Tz85GHp9PW", - "gM47RC+7Q0fOncSMumqQXEOYX1l1zyd2f2V4h9UzV7tYduDWGVAVLfgFaze610+wW6JMsUEmM3SuxHbB", - "3Hr4u8xQNZz/Tlz3Dv/V/QvLiWIhAxaTmCsWGYgWEUYiR7Df+E4TJaVxz6VatkeJ2CPyo5LpYzebDfeh", - "O1POGFbGGabLQUwNHVx4brPGhPYJ0Z0+nhIYHuGCvHhCttgHo2zREjIDzQeyzf2Ssg8RY7FGmtyuDnh3", - "FB4idDyZT7uMck35mVf4B01IlGsjU7/3R8/IFpazmzMBewGi/gwl2UzJCx6zuDZGCN63q7rbsqBXtbuC", - "UFHUIvTKhR3cncgwXS6k+e88q7OFIiRmygXFwW0s8FI/UxZQAL5HufABOG6P/Ci+XWFO89vyyg5QIpZW", - "dYtopLRw09vfrrmv+ZqrJkP5O6122/nwnPXG6275UR3Tlm6iCEWRO3e7Zut3X05KD9dfZTaPM51fFApp", - "m9n8yyLB0e3dD7dtLn/3FaeAgrH8orFstgN1ESaYlxjTHbMLlsgsxRL32LbX7+Uq6R32FsZkhzs7GPu9", - "kNocHjx6sN/7+P7j/z8Agm/bWh/+AQA=", + "H4sIAAAAAAAC/+y9+3IbN7Y3+ioonv1VpD0kRV18Uyr1Hcd2HO1EsbZle749oQ8FdoMkRt1AB0DTZlKu", + "On/NA0ydJ5wnObUWgL4RTbZkS7InrqmayGw0GpeFhXX9rT96kUwzKZgwunf8R09HC5ZS/POxMTRavJFJ", + "nrKX7LecaQM/Z0pmTBnOsFEqc2EmGTUL+FfMdKR4ZrgUvePeGTUL8m7BFCNL7IXohcyTmEwZwfdY3Ov3", + "2HuaZgnrHff2UmH2Ympor98zqwx+0kZxMe996PcUo7EUycp+ZkbzxPSOZzTRrN/47Cl0Tagm8MoA3yn6", + "m0qZMCp6H7DH33KuWNw7/rU6jbdFYzn9O4sMfPxxbuS5oSKers5kwqPV+mR/5iJ/j18jNDcypYZHRNt3", + "SIYvkSnVLCZSEBoZvmSEi6nMRUxePTkjkRSCRdCZHgs51UwtWUxmSqbELBhZSG2wjVE0uiSGThM2HIte", + "v7EfTMCTePsq/XXBzIKpwGC5Jq4XMpOKmAXXhAt4GrFhdcOMytn6yvZ7PE7YxPCUydysL9SP8h1JpJjj", + "tHy/JM21IQu6ZOR3piT5LacJn624mLcv0pTNpGLkx1XGUipIltCIacIN4cJIPxu7RiWN3UtDxMXnQio2", + "iZk2XFDof5JJZfT66F/gHzQhlbY4NGxPzIIaT+VCGnLJWFafKH1HL+vL+OvBQf/RaDR62+9xw1J7rOh7", + "nuZp7/j+vXuH9/q9lAv77/1i9FwYNmeq96H4hSpFV5XpaJmriE0iHqtNM4kSzoQhT06evrzmBHr7oyH+", + "b+9hr9/bf3Qw3L//EP+9f79XndbawtdH/mHz0Ts31OR6nQfZ0zRxhDKpEMn6rH/J0ylTRM5IlCvFhElW", + "BI8UizsQXW3ao9BWRFLM+DxXLF7/tj9yteVcUE2osExj0OAXZWedzl0ETCyW78REsZRyAWu8NoiX/hGB", + "E0rcIYIhRVIYJZMEmIIxLM2M9qeoD2xcEJplCY+Q9dQO1VE60r1+T+RJQqdrIyx3myV8zrFBp6XhurJJ", + "/l1iJGHCMFWc8C5LU2OLbR8ulzu4GyVf7PTJhUziSS4MT9a/+hp+dmta/RLXZMGSmMjZrE/4jFACvcDP", + "lsbry34wOrg/GB0NRvdf7T84Hj06Ht37W6/fm0mVUtM77sXUsAHscpe9QaatuYjCuyOaJJLChaRYZHen", + "uLBqGzhlkUwZga43jvzo40feet08KTcVGhLXsLxZA1s9bFwZ3ag7odoUTAj3i5vVhAbG9IqnTBuaZsCH", + "YAyVxWzjQr7D5j74ld+4wPsftcCCvTcTt0LB+YTog73PWAQ3ovTcpBAwoD9H3gXLuhUaV4xqKayYBJfp", + "r71c6DyDq5vFkyyhBvrt9XuWDCYp1xpeLX6IubZ8pN/zRD4R0kxULoRtKJh5J9VltaXrZcKzXr+3oHqy", + "nGd5r7/p2qoTNX6CJTTT2J/bcTVhSknVs6LxajKTym9S7211CTd0tbZCurhiAyvU6/dqC1Cwcz8XP+5i", + "V4OD6/fcXxNltQrszk5mfeDVvtaHWwxtM2O3t4gVov02E/eyrnOAmNO5kNrwSHdi8yg8wPamMg6wzqdF", + "d4THTBg+40w5uZoRBfdDyojvhEAnhAuS68Y5KET/CVsyYfRkeTQxUba+KA3Fprp5FdmkvBErt3Kx/cVJ", + "2UKk9bkHFacl5Xgmn7Ilj9i67Oa2ZhIrvmQqwL4LAcCyQtuO7MBZBxYipGC7tZUSSx5z2oUdxDimCQ9Q", + "z9mTE2Ifk5OnZGfB3tc/cvBg+rDX3qWgaYAWfsxTKgZwIGBYvn9sW+3756NQz1ymaT6ZK5ln6z2fvDg9", + "fU3wIREo4VZ7fHgQklSziE9oHCumdXj+/mF1bKPRaHRMD45Ho+EoNMolE7FUrUtqH4eXdH8Usw1ddlpS", + "1//akv7y5uTpyWPyRKpMKtTZth6c6vJU51Ulm/quhOj/+5wn8TrVT+Fn1rJOTGmuDROGfG+bEcWsKkfe", + "LaRmJKLRgpGpVVpQPcf+upB88WF/e4VG8JTrTCKfJ29OS2kOVUP2nkW5qX33WxJzbbiInFBVmVyHEYH5", + "KWFw/4YkC1wB4tpwKYjxotO1hYBIMbrlc9Ci08fWz39uyWuS6rbefRPCBUl5knDNIiliXf0GF+b+Uftk", + "KqfYXptrn3oGP5OUaU3njOygWQrVDcvhCddkRnnC4t1uEnbbZP4up5V7rXbmkAwGdBrtHxwGGVpK52wS", + "87mzKzZpEH4H+Rj6MYSnrRNBCaPbPPCTis3Wv/cD3if4EcVmTDERffTnMiWXTFCnUv0Hfrf3f+2VBtc9", + "Z23dw8U8K5t/6Pd+y1nOJpnU3I5wjU24J0BGuNQE3wiPGR/Fu50oShuqNp8PbPEJTmIpbG5dG2f6AXmL", + "zre+8graNBk6sqNCwKlwgVa+/QwkrYDIIoVhIrA6P8s5SbiwahATxu0FCnurjH2XyPlu75OtQ7H864cf", + "xn0N5mV/aOkNnvULrSCR8+pqLhhVZspqi9lyr7qOytG1Lv9Z7fg0LlCq2WQzBznjQrAYbe7uYNuWIFsH", + "dR88RZfcTJZwA4fOHA7rJ26Ia9HaVSKjyxlP2GRB9cKOmMYxtwbXs9pMAiJkzZlB0UjgO0TRBpXq8x8f", + "H9y7T9wHAmvorL/QYH0mlbehe9uWGKqmNEmCtNFOble/o9cpJEwBpcG37e4pKNATpuV0PbebTnnP9cL+", + "hbwbRoV3X6/fi4C8Evh7jXD7vfcD6HWwpApXHbqvDOu//Zcqv31ffrTy61nx/cqPL91QKj/94EdV+e1J", + "OUC/KkytnwenpuGkw1Kl5MIMuMAtAAHcWZ6mdvMKLctZMogUFQmPqS7sfJNg9WRNpCI7L394cnh4+Gi3", + "u3DF9eVE89/ZZD7dKDe7QTtBGV4j8Brhgsz5nE5XhukheUKFkAackdGCijmLCZ0ZeMuNtaaA3xttc8K0", + "CklMtYlI5vcj+ujh+/fUPLrP3+lHv6dTNf/7Ie21WRdz3bK+aIGjbjehKVFUBDbxehd1St+D5m+pJOT8", + "PLXuKlI0cnugm0MYklKhn/H3sOaG7NdWOri4Ye2vcGEJKQa54L/luNtZQlfrOqBhNB0gRYSW1/INe4D0", + "9uOj4fzYd/xM31Fu4OQUPlM35T6RSQz3zowrbXrdvWFXEgyY+lQiUu2UhSWmdYpoLmDrpV6Ms/WsJHzG", + "olWUsHXm7tl3pnJnbo0Z3Dn4Z2FA7MrGmQpwYv/jWfGJ2s9Py+/Vfn9mP/6h33tCMzrlCfecuc6nnU9+", + "4gx/23ap2tlT++pL9+aHfm/GqMkVC6zlOcYHEGsnHiRsyRLiWgPxHpPHZydE52qGLnOkVtsWzPZSMz0W", + "is2pihOmkdTfLXi0KMyVYHrQhIoyLIA8Tt7RlSaZYpoJczwW48JCrse9Phlb5cv9banE/m1DQ3wjMVdM", + "u3+ArWHc60NfiZzrcW9IfpTaDEBddrLUMRk7e4we98iO/ZNkVGuzUDKfL8YipYLOWcqE6RMM0kAzMHCm", + "ZLVLqIjJuKekZsbQAUvzBHk/dPY4yxI2Fuc84ZEUJKXRi3P37jtuFuSlfalyeeKM4bLuk8yZnjVN2VhQ", + "awaFnVyRTMkpIwnNRbRgmjAxkypiu0NyxtTAL7HfW0IVg82wRlySCzgijJaboX99S5gwykU6dGcuMJOr", + "0B8sfaFE66u8eWLf+FB4R67y8i/uFWBVbsZBHUWtSmt6hZ7tjViY+v11VPME9McCVxR3lUOrd4LUtmyW", + "0DmSSuUQWWLUYxHJNEVFz32FaLZkiibFDsEdJ9GeAHeDu5HGIkpkHg8Wq4ypJddS9eGGYBEGI6g++Y2l", + "eR8/Cn8NUh4puUyJFGORABnv0TS+f7Q7JI9JwjUQh58/t8TtSAw5AdoDOIx23KPeJj/u2YlxTeDiH5Jn", + "wihuaY5oS3BTe5M2iKvr5lWY1drdhvtzlc7O7RtrHg77syPo/hqLrdBNSYAFGVe4aOjO2sCA12Vwv66b", + "nVFufMVuRVS4napR5zHhhtAsY1RpwsVYlNQkYqRS91KmGC6H5sZtXcoM2UmpQfYCVtuxQA7xjSa13d8d", + "EpQbbXxYyZHgFaJYsrIDYmPhx/yOJwkaDkmOEQzI4exBQ8tYJYICDxMVBe9LSm+Xc345v2htArUjYr/9", + "389OXxO90oalZMoFVatdwnXJay1prjvmwiKj2z3U3FFUq886lhj1BO8SKeoCZPO8bnUhOBG0pIttFPaj", + "DIVYUhUtQmF02pAnZ68JPOaGRUDEtfFSld4PupKkbukOvkpRiLXrXesOmc7WOUs4UjjibZM9Ke6RsIhU", + "uOPXvV3wZsWPyxIWAa+y8TRWf2PEOZeJTJGlC7Y+m73WJfKd621f12RH6j2Y8G7lZoEzrXIxJC8EQTGC", + "1KSIscCmXERJHjNNKvyclOZbL1/UwqAKsh+Lf/2//5+VJWKy5LSQNsgbrgzEkPxutdiZoikDple/0qIF", + "iy7HoimGEOhVS0L9pVIsM9fVC0Xx+cIQId8Nx+LEjqkY8I6WM/OOKpZnoGaSwcCNeuDkrF0bxIUdcdMI", + "0fy1sTeVxbliLGHNnVdsaH+dwraR6i+l2FKn1Tk17B0NKMTPgfQGS64xZA1pwrUlJ2dDgo/RlRAt7FPg", + "oiDFkh1LFVzMxwKfOJF4lziJ1pKZc00OyYuUm4L4hSyuF3fTjQUEGE4ZE+hFTJY2ALkk1RVrbgBEc+6D", + "g3e4Hzoacxj7xMjJPA/aXf1dF3PFIkPenA6MHLw5JUbR2YxHQEcZU27U9oJpDjrIzyHGIGlZaf+iD6FI", + "hqBkKB7PQciB5XKiv/3NXk4ZUzCuV4/P4ILD1f+WjHuCGvtOyekH4MnhMYvJL49fwdUavTjfdavmVFPb", + "cQ+unZARvN/T+VQw0zZ++xRjcMNbgaG025ivXaK1LdpG3h8n0cDFbzv4ZoNMUnIwkE5AiC1IcEh+rsmv", + "9oUicAVE0HWJvujBH+JvSTHa71CgISmjQhPqZP3KiAjXY+Glj5rAAZI2cSsKr6pcWC5l78NBhfuiWGLl", + "EZTIiiD9Jfz/YKlldOlCPKyWWXBaQVM83dWlW5OsuOkk5Gw1AVQUlsCS+q/XBE/QEbSgmV5Ig1q1/8ee", + "YtpIxXZBOXfxUvA8o7lm5C8kZalUq7Hw7fv2nCXU2BiG4t0ZMKYe2YkSKVjVjmADCOhYaCOzjMXOMfEt", + "mbnDTYsgQqkKW7VtNBbuE3rPSgCaFLPADSg9MMmq3OTKTHbR1ICzKaYFc85TO+yFNFmSzwdunj2yk/Al", + "c9OGCfLfme0DvChSioGLh4Smpe0EHxFD1ZwZZ3rW2D/SzLg3Fjt4bgn+GxwraS54hLc5tptn+aBq4uiR", + "nednr/fKmKWxqDzGV8CoN+BykPCUA3vbgR/Iyd4LouCGxp+5mNvRY2M7nWKS2N5NcUje0CR35xSpHdZ3", + "LLz0jyMdGJWbxSxPjsnydyeE1fej2MlCKto/7DtLU4IWplZJ5s0p0XTJPEHiW6BSawPHiMZLpgzXzhp7", + "SXYcOQ2cnwvpThPBWEyEHItqX3CQkUxhiE2Rn1hDjCaNRRqLUmxrRJL7q827DAtThBd8ultsNms0LT6G", + "j9darqAinxdKfeMiyXi7VxVskXhQaFQuU0V/ZeRFxgS0imWUp0wYwsVM+qb1KLLR8KAtMq3l8+fVa6X4", + "vF5IZcicG6IYiHFSOG427g1irgwcf53PZvw9apE5WoGsTIM96T5wqHEvF5dCvhPjXn2YD+7Tg1k8Pdy6", + "G37Y/doaBvcBmZ6zSrfm6232oT3Z6DADUZMYCbpzVfN3FP4J3GVlShJcemoAZyXhLK7Qdt9G686ZYMC5", + "YhsDaoiX0FpcPoOgKHvjXqUreWJadtRGzrZu6JY5zBM5xQvPT6QMOh2Sk/ra9QnFBygw11e5yJaoBIaS", + "HTacD+F2ySI+gMjQAT0YjEaDUZPak6OBDXnPqDFMwQD/n1/p4PfHg7+NBo/eln9OhoO3f/mPoELeMVrV", + "+7XdPHf8WeoTP9hqCGtzoNvCW6/tWasOv/3wommhdacjxfAY0GTrGCDi4EmleTutPzlZj3qzKxVLsEMP", + "udxL+FRRtdoTcy7eH4M4p0193Ta33WRgCSTHWMHINyBUE2dh+XVvSRWnwry1YyTjqnEA5DeCtwbc+09x", + "RGQw8P049uUj72sOgCH5RRpCnUcaVLpjbLT+3bmiaUrR2rmkCY/xcDg3m+Yxs8ZZgc/I0kpJiplcCXI0", + "Gvknk8qgQG6Q2k9l3HOGUxUtiDMIKYKmEJshE7EMPmlkTBtpSpWlWD9pcMz+slf8teP/HL79y+7//o9P", + "SupIaBto3JozrsjPGnHe0IjsJPIdUxHVjCQMZqv7JOZzbrQV3mIMjUKD2rckssEeNnRRKsKEN1Fjuzo5", + "p6sBzfjAWV6su/tnJuZm0Tu+fxhc3B33x+Dtf/qfWhZW5UlIX3spc7S84uNq4rUfQyfni1/dPEGnS8rF", + "iX1tf12c/Lgd9hPZtNNWqWvdarhjiqygLQNZz8CHiKM0JEQsmVJwFC1fe3L6FFSYS+YYGmiNZJyPRocR", + "NsA/mfsFhDhQZfC33brEY7sLCTu/9li0kBgGmSTyKkbKfpOpt0UJbvSRhVa75P7rMYXoRLducQx2dw3J", + "VMlLptAlA0vMmSaXbGXNBfOaQZOJJVlS5V2DrzAtAZv4R+gNYqgegxKLyfmQy8qUY4tOZ7NpUIwmJbdM", + "Kbi7UK/UkcwYMFTXDKdGLphYXpCUZlatV8wazVNqmOKgLpaGxZTFHHj0WDA8GCSjcOajSCowscLeog22", + "XIVvNLmwauIFdn/BBZD1hT2YDVPpH70Xr199/+L1L08nL86e/fL4ZPLTs/+Bn+1LveNf/+hZMI0i8Op7", + "RhVT5D/+wPl+sMGzGNMHqe4LqZy26130GK9CMz6UGROUDyOZ9vq9/6z+8+2Htz7c02buLXvHwYF9CIqZ", + "zvYZiMH3CVCaSFVGnlLEBUEW5c0OFXNDw5RvJb8rHQlUUbicTDMdzE1p2i1KOXR/NDr9fs9Fj9zz/9gd", + "Eue6xeEDD5LKicd6AeRT4HKgPy1JZOQyNGZtKfj+UyEGj77WTHJhtjKnsuk6jxoMyqdXYEV7Uy72NGzD", + "ILrauiPdXDtQ+ZlYciUFqucoLk0Tputn5ZcXT59Nnv3ypncMF0GcYx5hr987e/HyVe+4dzgajXohAmXv", + "M66YDgY+Pp5qmeSGEWxUxpgOyWmuMbjTMY9ZjnZQlM80SXMwLCUrwt5HSa6BT6E0YEwSyj9+ODjYf7X/", + "8Hg0Oh6N/tY5YBVIfwvzfn722mao43lnNDGLCbrmtr34I7Z9gk0/9L1tElX7wCoVe+qtlXB63TtkZ1GX", + "ruzkCBLkuHf4/Ht7oPafj3sVQnSpwEUvtuPG6j3/PrQuFWvUuqRXPLMZbiL2VL8ORmPt9bl2ZqKqJ3wg", + "KELsQMuYqphMJVXxt/VYGnwTDPnfaILWCpo4P5EVvfEdK0tKplFX9jE+pbFV9xtLgJaf1JoSqSAMPaVL", + "DixnkKZc+vfqbMkZU2o21GHFwRSw41XChXr9HszM/cdPsNfvLX+v51RvNwe60JhOuuIWJZAmGRdsgxb4", + "mQj7YFNOJI0H+59Y1q/EuzVgaOyD+nYXyqkn8DV0pykV8Tsem8UEMuJhyIEb0j0hRePimnwPM6HJv/7x", + "zzenpUFk//k0c3fm/sG9j7wzG7ckdB3MtSkmkmfhabzOwpN4c/qvf/zTz+RuJ2FFymuJ527/n9kemgA1", + "DrfLRuO3+F0LOazMz5UtvvQtGAahK1cumUroqnKduDH19kfI0xujUty6G917FRPy5ssFevPiWvV6ATY/", + "4yyxMVo29Bb1C+09RNZ3iKcFeNI32mcgMTKV0tgRwNowuoaNEr6VPs4kNXQ3RySFoVww5cZb3Bj2SrEa", + "S3r/aEievc+Y4iAw0YTEVL3jwga9+Dedzk2oAECghAs24Knz4NolsZNEngwdg6PNBxxpG3C0brVHY3VM", + "3P1oX5aqYRC7ScuSYsiYJ1mBpLfp+Ly0rUulP0CSAYr8nurCM9uFDgsy3D84dX8eWIKsEl9xuOxZ46og", + "Pvupb0EMBdXZ4Ee1hbWbMgJfYlQbckSe8+8JF5DKnzByyr8n1N1OXJA3f7P7gLxOd5Kl9CXPbJzHhM6L", + "tNRNyH/nlzxz+jy+4aMM7B0U59AznqHhWFhkH2AbSDsQke99twaSCLQNXIDbFu+zdRkNrAPljWabWzet", + "ykWfTHMDKr80jDjji4MYgLFg4ykjeRlE2lDA3QTXAyNwWS6ZEiyZWAVbd1wZ+xJxL7UuDk51RrVDIlMm", + "z+rr9fSn03Oy83QlaMoj8pPt9VTGOUQDWp6wW1+9/lhkCuFd4CNwFrj7rpwRmZuBnA2MYswPMcXOCj+M", + "A7lYPj977WBSNAQpvWSwsEzELuDVCztFxKv4xlhXeK3b6vcbi94GhePd+x1P9rlrXh7t7hbJfs+YJARu", + "l1i2BklENo8CHFcF9fnUvj6cwOeyxGKw3GBIXmvgBiMIe5EKoxbIVJpFVbvEa0lXNaASQrNsBvpnWMUs", + "tdnG+T5aBP3WUZbXSfeg34ppuLQBEyDO6K6cawmNK4wLAZcCnOugI6MKgs24JJ/1HbPwslW7kh1guWfU", + "1AE5uprCbc8IBrs1OtRZtHFJulu0KzbWNdu2NwD+0TFpYnP/J8IPZKNFv7QBfsS3zm0nzSVyfff9zK6x", + "SifFmjT8AJ9meR7ris20E26rlXmsPKPJzgWYWR0dg2H1ok8u/rP2A/BTL0+BuvCO2NVAHi3gp2r/ei2y", + "eYsd99rRzVRffz8e61bULbLcJ0ZRoW2c/YJmbEh+xIuRGJaCtAg3Fa+GFgr57lsirY7iXx0LGJqux1p6", + "a77mcwGhZxg0Dwwnjq3J31nsfPyNDsEdezvcmsRuR+dAi5CXuWh7cuFN7xd1/rtumF+3WzlL/ZrBAn+3", + "hgo05Zi9NDfweau9OJ+0zI0FMXNTbwh4dfP/tsAgN5bCFHmN/T8v2EUTQ3oZsFjA5OSs9NdUHEdt/hkn", + "/IV9R5dshVvu/UR0zVNUdRGFHTkupN2JMpVX0XZorxKuK/4l6ylyziEMPq4f0aDbZNtWwHp1Xv665h/M", + "wxn4yZYU45R5j0/quRBMzn4PjY6a4eKjJeHYpnVd9K3pg6F9kQgglsTH5je7x7BkDGK8cL8MXW8XRZLg", + "p8AcL7M/qltLKjvrRWnsxotihsX9umwA6NR6+6RseOqixe2p2DvFPSPzAJcdRV6XNZM6xevj7EDPKp1t", + "CAjr2sV6lqxd38qYHT1ZYGefRGL3A/1fNcTjJvB/3FDDLXJM/ZMXNEkuyI5rtEsU+zuiwrq9ggi7YvsB", + "5daRQBFu9ua0PxaWC1wsjMkm8H96Aqf4otmZe9ef8BKU/eEIRdejo8N6zoYdcKPburk8iIbTvjUQd3VD", + "8VstRnmPfo+PixsIEG/IjqFzODoO9gjuid2PNdhzoVmUK7b9rD9G8chB7+CYEsrF4MdXr86IYnOuMdU4", + "mNRgQ+NbMNqcsaAMV/PTNbJutT04PLp3/8HDR8P4Ug1ZpIa5HoAiM9gf0pT+LgV9p0Hs2ktXNMuOl/tb", + "+bzdgmJ87Szfa7btpCBTYMk+ILmLlvykfKV0GF5yEXft4Cdouz3o1SvxN+12yRQb5Nlc0Zh9YqfLteOa", + "cDXbd3VLJZTNAc25NmCIKCGAdzbHMdcXaymTQUwNvUoYsx3uesRyurJdWZU81F9rePh5Mya8htsTrIfw", + "sUFmfiyhbWnDFLaGBBZPjAyk73rZ4eQprLlv2wk8FSGQjZwsZ1xuS/KHodVTd/F1b96ALgZZxJ2lzns1", + "eGH1t8a6N6fV2JrhWAxQCjsmT4sPFN0WXVJUMWhsvfw7UtXyh9ERMAXoE/LmdEheFaP9RnungB0TKTJH", + "c3QoolA0sCJZdQA5Jtpw03zdmSUtHvMuhhBJ92xYVG1BA2hhQEN72ZQ35mPzunGj3F1ShZ7paBDehE72", + "0t5HmxDKahEh9waj/cH+vVf7o6tGhNwA5HSor8d13uJi/6vc58nrk6cHzuy5e21Qsk8OSh1mZU/LpAWy", + "k2tWSciVKpiqUMkIaElFuHaGwQ0lDJTwm5va2pV4BS1vAm07hAeGTfrXwMNuMsytoKuVya0bdFYZmh/K", + "U1IR5O2OZhEPZl5DyOD3itFLrCKwfm8jDsXE3mfheEMEC5muEBwLjVxKSjPTpMCQKY3f+0cPjh4e3j96", + "OBoF8JzXCV5GfIIe2U4DgBCchK4KaMMdDGOIyTSR0zqh3zu8//DB6NH+QddxWMd9t3Uo9G//FtlxK/IX", + "X+nNP6kN6uDgwf3Dw8PR/fsHR51GZTvrNijXti6SPDh8cLT/8OCo0yqE9LpnHl+7KcCHij08tvWR4F8D", + "nbGIz3hEEJeOwAuIHLAAN773QtfP5JTGviRG+O4wlCd6Y3Sf/Zhrae2taZ4YniXMPsMN6eSSwJk/xZ6C", + "NcaEKEqQXK0nh0q+Ne7Lz6VoguJMzKb5fO4g/4qlO3XZ/KXwxFkSH5NCZdvI53A3y4G9baMDN4eO1PCz", + "fFeA71WIwF5dFqlCMVLQid202qx8uhAXWR4kidal/CFXFoQDOyV0KnNjPfiuZkz5EYtnI9CjnYu4m7nj", + "B6kut+Z3wE1clMbpaDDwif4zBAJopPhXhb7C027jEdxzTV7aN6yhsPw5y+t1Afv4JWdQFB6dIC7sxq6b", + "rtJlWG5Bm7kP6rPfK3nnLUU0DmY2COzTathqzrAklNkqsQCl2Pipc2zeOV0MXtxqSOmw7oK9u41Fx4TH", + "AZDtAEwoN7PimwKnC1tD2YhIl24A+Jww87gCL9U4aececcMt+XAsXL5V8ZMLCoAX7TrYQAKp+JzXP1y3", + "s95i6PJVKNMT17Wps/piW8ygLtyAIR5gAadnNhLYRi9W8W7dnvT6vfOidpZjTPWl8T8GVqRMMFiHHjp7", + "fdUA5EzJGQ/hAWHUkXvqFDUfmvvz0eh8sP/fNroTyA8lNi5spBL4EYaN+kfYvttF9Pzs9VnbmIriU6Q6", + "urU5bUA5KiNryqyYYnFcrjnXvudKoV4qiKGXDCxMeOJqzSx8F4uH5ImNtUG5hs/Qom18MwuvAgzIfnvJ", + "jmtQORATJljRqXWlxjlQVRXnDZqBKKC91xMx4p6fva7jNTwKid8IuDLNYWCTNGAP/AGeE9ugAIw4/b4u", + "gh8chboOM+6zGgGhpj+jEUQHdKaQgA2xMY1tiIzPz16/dAWX2rCqgZyKokwWpnpIfilKksHy6hKPahgw", + "LjZI0IaeQ4OJTmQogOUHxRjBZ2UpVKAbdHpbdOMi0IDGiG4lhS/ICLH/Dmvwt5wqKgwXLCZvftDfkpHz", + "sr/5gdj8I8ezuK7GXdaVywehLW3NIzxbrDTY9uyy2KIoXFQNm8gFOmsmZ+WLzgQc0E/C5fg8xyE7UIQR", + "F/D85eDkxZu9NGbLfm1M8NBG6T4/e71bYctLD2ZRtK1z42WbhclvVVEdclJWpGweBMTRdqbPJnXA3ozI", + "lEXUB3U3Ny8qCjhPMVwDDWXg5jZMEEWdnZqKWifIuiDwPk+SoeOZfsgELdMzZN0V7Fd7uzmmplhGUfkq", + "JK/3Bj/KiM6jiLG4EfHSHHWF3o5JLlKq9IImzckdFwAL0YIqGhmmyDfvv+mFKzAhK+16N1Sos7icOpNl", + "5SoK0GPl3LUd8Tc/1I+nlQ5sWrWzqXPBDcLEITb4TjFIi2Y+JKd05UwvGV6zjl7KHaOzmQVLLe4RxRJG", + "NVZO1+TND9tjOY00NGmbwyt46PjUDkwIThEMs0+yGhuA36soZdXP3g9eGeV8Ap4p/GDTzTEkJyLM9mzA", + "EOIMWsgwrflcgDqoXcwQ3Kd+Jdf2rn53Bi1JTVzGXn3hatPpBy6AEMWEbiubB7quGa1XcpWXdTYlL7dX", + "D7WdtH/3iU9VbbjCnPrpzy1mtPrgJWJfdRzDGlQ1y6iihiUrq48U1FmvBsHW41AgW+AKubLPoPkHWwEo", + "V2xiFoppiBmrGSgO++tVpDVmJSyZK5xn51Rx1xlJUqou8ZR59Zvkwq5APS/2cBs41sKY7AqTgqAKa5Mz", + "TC1p0szY0mtxOU8ZZGlNmXnHmPBToZrQalR8M+NdtxQvg1weprisr2HvMPDdc5spQeaKRozYt3wN+oKt", + "wdHrupS9w5ZhwYWjdcv+7m/aX/fqLE+67XFoWPvbdthEV9ngV0/OfBGuosi5X+aD9VWGQhL2yLk3tmzt", + "gd5cDs5/SlgU7TWZfcqwPpzLNatmsfuoRKx/JzBjr5I5XmEOuprr7L6Dp8AuVd+e87eddMLmcQ+F30Cm", + "UahYPD5wWXpzBBvE3ASVo+bFYysp2Tu5cj+7TCqQtLgAXOZ63HeUq6TX7w1mblbHe3vA7xPEWz063H+4", + "tzn4e2PUvwtynMR8k1XItXLBch4CwIJV4qTrJLFHs6yD3Rzf3HY/IHta2wRbIB7utoqS5VNcwF/TCF58", + "TyPjS1qiIb0WKEGrxxZR0euCDHSY2ipVoO0+qp7PUTByBcJQ6+S/t0b7MDMbiQw0Yo2UjXW0RP57UDqV", + "KlS3UCrj0pWnzCvtxX3oA4E9/nfNpT56WJ3l/Xv3Du9t40PIbBrH3J27wFRt6zogA6sstzu/rgP4j67L", + "HO7JZrLCddlCU6+eBEhq84LKjIkrree9o8ODq61n14mc+GDOBl8K4VkBgBXKRGVmccoMjamhdSaDFmjg", + "MlgYi7IUEwdn325mLS1RT1WAqutW9ftUMTMtBT5fejz7lAo+QyXJtqx+WS/owb37x7YMccxmR/fuD4fD", + "q8L2PCuedduKPRtaW0HwGerFx+3DDaDzdJnLH72zx69+BEaWa2UvrT095eK48u/in+UD/MP+c8pFGNWn", + "S+VqPlurWF3b3ix3sOksPoaZCJeM5SWlDlF9LS4kzClAKLEgRmYttvrmwDALGuf1midXAMXckMoP4gpY", + "5FvX5tpFqEVhCzWV4tNVu0KHQtQQ+rkp8sNbnLGN+6a1dBQ1utfjYa5VZV1vLES7VqcwY6IoPZsk9q9I", + "CkQlD9WhrV2R/lmHMoZ4jbh6hcUnqz8WX6/8+KQ6kMrvvgxi5SdXkPbtFQPZNgqkf12XQ7dzIS+ObjnM", + "YfN/cSt0Lf7tQDSDGRx3fBdeJ9iz/vUX8//67f/oswd/3//t5zdv/mf5/L+e/sL/501y9uKjAJo246fe", + "KQjqJ8M9xQjHGvhpV1I6Bc/MOj2B+teywu6JtTiYaIHlismUHUMI9s/cMGWLXzZSlqFeA0NNCd8iUmCV", + "QodmsQsvn9k4BHj5Dy8Gf2j2ETvYCuU2pABK0vk0linlYncsxsL1RfxENOoF8FdMIprZCpZcgGEWArsV", + "lht1juHy433yB82yD7tQEoUawt7bUgAZxYqNs8I3IXxYhxuVDV53zVmBt2zziMeiuK0LAFEbHTAsq41w", + "ljQz/VoWZbP+5jSnh6MQ1CpmocFGYkUtVEEKygYyKtLjyMPR7ro+t0XHKGhoA/m58BmbpgxpxgFSbElt", + "fspiHiFf8dm9tmUlsdpSmjPiZUq+X+HevLQppzGhuVkwYVxgZJ9EUl5y1sct7RNwzmK8Fr5po3AWMhtM", + "V4OFzAoYlsIXbuNY6kr2/xm4iQ7eMMVn7ktBNA0gkYDQiUfGzQzblNaFtYm9ssWIhEanvGtuq1ppW8jI", + "JmCYXAlfa4dhxV0EGyqoj4Vk8m9JlHC0OukF+vAWiBZqoJsQ4GdvVCS60WkUs1nz3/VApIN791GD9f8+", + "POicZ26XbhOV5UlAp0096+vAsS2bxAFY8WDiDeFbQgfhBrScmKCdwkj87znxHZUnrmAk1tNnM1+1K92S", + "6ErO624wOdEdAwcqNKG56Tq79VNYy3Xs0IONKcLXkg5wRs9wHuTVz+fEMJV61IudCHYHT4nFrhlwrXNX", + "RfDxk9Nnu8NeECWu5tLCrdqYC1kfdAAhxZ6D1tCp0kZDU9YnJ08xx91dK6UuhklJP0hFEnsrlpfRMcL5", + "1M091NYfPXnqBNBkVQYqWbFl3Nv1PWbN6+2YvPSfJbQYSpGtXNKW77K8TLBbF7ZqM6bWem8kt6N/zKl/", + "7j7G/Cgszmp5MSLdtt5f3W2ObsXhYdNCduULqfJiu/2rsvefGhP+08voh1eT0Z0XepItqA5R96Lq1cRG", + "a/tedWTX2b1qj/SrfXeiefBs/dWXqKpcQ4bBPVd7PQSCfG+wv/9q/+jq5rurwnnXkfoqGJ8Fond3KO6b", + "gLQOAFRzM2lNCSHw2CWAeLvIm1PIrgTkNXjYsI7sHz7oYpTAr3ZNpqg8BJEah1RwKY+8VyQBWADES54k", + "VoDRfC5oQh6RnfOT5z+d/PzzLhmQFy9Om1ux6Y3g/lwD2RtvAVxHGxwVAF+rAXmQIuP31auf4UmUMEya", + "snL45fXxvreaFjvgf/vBPT97DW8sqJ74cOv2DGNaZumz91wbvY672Clr4WPwxu2rpWGsyyRtHy6ocito", + "+Y81VO0giufuDaCN+5STteW8AyDuu0zt/fxAwDfCdn8s9razM9wQ9HbrlRaCrW7gmNxru92uD6J9I8Op", + "wUCF2FZVwvG4C9dGre73eCDn/LGL4yMnZ2WZutIZ4btvzOnRwXD//kOstLw/6sLYUxpt+Pbp4yfdPz46", + "sLfMMZ0eR/ExKuzX9Vk5wrYqCE3eQWbu2CuJ4569MCvabeXY2jbdci7WwcGvhwXeFONaYLlRnPWBS3qS", + "biyU1CGpuAl1mOYONDblScI1i6SIdV1GXlBNdGaxkm3BoEKCHwscYJ8UddtRSiE0ilRemh6ddO3k/Txz", + "dI9FizMpNHNVS35iK01Sjk7Q4vMY+qhJkbsWj8WO8nmORUIj1iuO4QfMGuoTpwr1MWoYCyPBC2OhF7kB", + "JrYLCSgCyhsrZ5UlU44eo12ic6vS4nhxNVbAMDWPmRoLaBZAY/6jUE+O7wMyQr9XaHJQ3GU0ClHTjTo/", + "hw52HXvAFBgqXDk9jHZXuSC5iJkiL6VmxlDCLDk0Q+Su6Dj9SIxz/3o38cq9XspV4YO5DRK9kzD10RjP", + "ONQW/RyjQ6+hnN+7vojeKVPQy68uR9C9NblKBINLyQCNbwq3nTXIsdiZITVzuR5F7s1rWzi4PnUXemwk", + "ONXVirw5Pa2FPbgMjW4TRy7Rsg8yu9I2HGyxkWwdzVXcyxVE7O0o2B+PTN2UVCoS4ifHoa56GD10gaXQ", + "mmGrpjhvsq+BVhnMgeLC7hMQzYYJNmpHW6CXMjCy2qte6ZlDZXWitM+pcIj0BWhL6dTXK703zfVeFvE9", + "+1zvIaLOQ0TUOQpCHsRsOcnzkGoEj9x3yOvXAHuDvyAitE2QqX2e0vv7D0cPHw0eTvfvD47i0f6A7h/e", + "Hxzco6PZYfTgcP/gcENSUYcc2OuntQY15kAccxG1PvHR86Gg5rbchYZs4uKx33ERy3e16y8YIFv9ugu+", + "3fb59dD6zkMIpgQlVBtrvmjhZKd4ybMI+sZ2Ps+6qBMXNnTefzXa/1jrDw6u5Y54pXJh3arYoHQhpJUB", + "VzerPs7rsXwckE982bZa1Y93X7TR8b1Hx/c+dtF88sa2MTbJ6RY3ty0izCOnN7JDfJJwxY7kDZQ9JxNZ", + "q75tvur1e7mo/o3CQCOWunjcKYmr7cD2w2xk07XSAnlwUtNXMFLFImfGxyCpeH0Ea74UwBogAj0BWxip", + "2OIskCD64U4qugt0g24xZ6Kzics2GQMzI7l2+XxcACNG/yN04vARjslzbIuPaGrVOjcIW7i76nqj8crG", + "y8D58p+2StbmIZ87/QrfAWWLwL9w2rAMzmS7uQsrnR2TXyS+U2h7QjZtv7Y5qlnrzRttyY4D2fd4N/gx", + "J2oekx8K8bIQUJ1AuqOZ+3PiGFYJM7VbA/twOw6BhJWdqwBX9Ht2RXv9nl8o+DMAdfG6pPq181clxVAg", + "GaMJnuUyTT83PHHg+jgTrg2PtEsegc1tE3tcbiaLJ1Z5aotJtamvtg0pXvJS1ZtTsoP4qX8hzrAN/9ot", + "4ldrd93Bo6NH9x8cPLrfCSWtHOB20fgJIhOsD26rnBxl+cTZRtqmDkX0cM0ja1UoYl/A21WSeaYksB6Y", + "ue+w+vFHw0dVcLhY5sCHiyE5JEmLRY0bFsQ/LHhRSxzkbzxZ8tlM/PZ7dHnwd8XT/ff39cF0vwX02n4o", + "bHY7qQYXrNmo2XRgS3uG8buQoJRuhbh7yTTOgJwzQ5B+BoRGqN4U+dSO5HJdW/EgYR0dHh4+fHDvoBNd", + "udFVDs4EjXCBS9mNoGxJsCXZeXl+TvYqBGf79EgwWQDmoHHOiCvQPqoLpMP90WGISlou7pJqXN/LtHXJ", + "3zj10U3KLTombxeq5dopD6724eHowdG9h/e6HWNnHp6o95s5jGtnv+fLZ1R3fgelyVePzwj0rmY0glr2", + "j609eNRaUTHXjPzy+JWFdhKSZEwNyhvR91RPc3WxZlean7nS/LCIjC3+sGWKtUiVB/che2u/G+pjKHzF", + "4ZnWjn6dCwaOb4C8AvsaWIp1Jt5vu3dCgpkl1ZcsSihPH0c+WaNxj9kiDxNlm5Wb0OWKcbr82hXY4d1O", + "ZrLC+GVTffAdIhXJRVFaaLjduftJfLTt/N9eNNvvh1CCj4DVc/BktlzjNZYyU2zJZa4/QUfS2OzbWSKl", + "utK7barPS6bBg+dBsd6cfqOJozWiDcvq2pijxg0gbtec3JXOc41EwkTetliddqPL1m+acL/l1PY3QXPU", + "uEErdGIMnCsX28NIn9AkyrGYFi32E2aFFwWCC2RZsrJZAkkipSDRggp0tyhXuk/MCSWAwjAMxtTCk8ks", + "GJ8h35FEWtB3KPbj6kzZQcBrIAyBZWmnWs3RklKj/PO91DIZV0moTo330nCl2jA6U5FUD+tJjazgkdtX", + "ajbVRM41qpcG8x+GzTIYGVU2rYEKWzdtmVqtNBC6HRhig5mHblR8CAfTqspOeMGUdbuSNFJSV4DF3pzq", + "YdfsuSIfentoaH2wHUjXukYDVxlecbpzecXQ/RjIDPqYGxJpGLMPNwRdejNnSkWOlacqhOxM+sPOgZVg", + "154UgFlXHKw2EywnA6bAwgdW5O8XliXfJngvetZ2neVyEczXenuNqsJdtQ2wnacGVzS8Wv2CBkNkvI5Q", + "txEUr4QKbMJiXQWssqw/wjX2yisYhGQHs2cqbKmCJbnbJdwmrPzCd9b0XleJGDweXTEaN0MynlGzOBEz", + "uc4JruKD9TZtF/eaMeVhEmMmOIu9Vlo4Y53RjGtXhy7OmVs5/GwNSA+DJShqTs7aBslCdV7f/GAXQ7Md", + "w+ZqM/hd17DDTnIdTs19pXJcKyQgIJ4ySbdTOCnXk7BjbL1jxeZ5QhVCB3Ycsl6lCReXXXrXq3QqEx4R", + "eKHpYZ9JSCGbwCP9Hc5lt9Ps4IVJW2myczs4l2FoN6Tx3XIK38Esdxv5zWjT2bPv78H7neLQgjHnP/CE", + "OZTL14K/rxB6vRzD0cGoLe++pdNaxv06zOtVObcj2eCJz3UgS3GjlOOLqrHY1cjA1lgcDo9ah1vJgzB7", + "Z+L1fEP1FJSPAzl5Yvl1A+KETBlmEPmprXXXhS12mUqwtAxU2/u7nNZNq10DiAMFC3dECbah2CyYKYA7", + "utG0bVusrUlld6+CZoFsFSaKL10RJGJbaccyUquNn7xcq3K4YG7JuJ+jrXjYoYKPjyQpgBDcV7sjIjTr", + "UQYinxHvRpsVVnbGclqrSpFVKFOmFCLQg4QjhZ8NArhIZdfaI2hBXveKKJZSLsaCi8LcirBsjAi2ZKqS", + "bysVKFlzSPb4a0XFg99ZmpmVKwaBZvhvNAFnnB/jWFQHCZ2DGX1IHgtrWVR5ZmrlYlnstD4gFF8kXzGj", + "sH4qNwsyU0wvqnMP1cwFGe+dVHFrMbIV8U1gGhS9tcTISyaqrKzoJqga2o4m9q31eEBb8BqfOv2T1GpQ", + "k2aN6c3fyzVTYSGxmFLRpFMQTOWoVJQTPJAwAFdb1P1lWXyBp9IBPaXs/r99l+VPZ0Xn9d8azSoIKR4z", + "/LE12wZNsFGWr69Owye3NegFE+o24eKseyXIjg/G9gWa9G5LoaRO92S3nL5m2oEfzZ5mUf3rRw/vPbjf", + "sVLVR7n9LA7Yp3byLdMNzr2WnTrt4kF6eO/ho0eHR/ceHVzJw+IzVFr2py1Lpbo/ACNi4LAm//rHP9+c", + "1qnl4B5Gc4+uNKg8ax/S66zDgN6c/usf//SjuvaAQoxmHe6/JQKgNd4nqe6kDzmoOwO7Ock26PePa0YC", + "WrAZssMQ1BvCd+y6DcrBNABFuknBNKMRN6sAI6fvfIEL16SB2t2h98ZgQyKv7dvhqwLn0vm0aAExLK7B", + "f1ovc4MWHnYueKfzaZtH+0Xzq9iOlF6LarREh2AFSxFhA3cxn3dU10LDY2b9Dv0iV209b8e22Izf20yq", + "gKEgJG+tryhUtaAhT7qXqtvf2M6K37Jm1mmu+NsN57D9CF7J6hu4kQNG3yjLu3ZUyu1wAV7vrcm0Wopy", + "Y63PWt3K8ta9+nc75CGvFz0pbrCrf6+SenmVFxu0ZunRjcEtedl3v0YSLdRUyaoJGNBkwgaVMgjYmujc", + "egThzDvA+kAuaHQpZ7M6au69dpR1BBDCtDH/FWoMaCZ9sPo7m0UTotuiBY179/S4R6SClMx03Gu4rYKJ", + "mCl9P3EfqKPEjDbBnpeJ9I1Baj+DaSKjS1tlEYv3D8mIpIwKTXKBh7/hVdsfbfYO9XtZZW/8OqJyt7aU", + "bjPJlC3okmNlFudTmddCOtl7bjSGnmI/xySWFjeqVmLazRCa2TTJ43LSeOlQsXIdQ4fQTgofG1u2RQPf", + "DAtbi9+Zkn0HewAc+8WL074NYMAgRjuwWqSkn6gdQa/fKz/RKNRQ/h6ORAZnBY67Cfyfrq9jNbsdPauK", + "aWa0QwIvyaFBBJC+JEyzIkDaTZGrJ6itX0m5wLBBF/4BC+q/bgmExCzCE6nXz2Kd0K9B3I0MBLfSoRSE", + "wxAJ46HYUMnnpXMINwdgjQ2V6vC2n2qEuPUSTrSRWa9+qifsfcRY3IQODTfpGnXv3gxG3f9MHdpQUbjd", + "tcbI6fXZDW8uVQzH2rba1ewAIcUATl2xpfZwWuQ6j3pTJ7Tq6lfBMSYhoNZQgw7jF+z95rX+hb03iLQe", + "5wmL19e2OJKWVbnLaNuKXztHsu1AS8W2lue8gbKVNnL9WoUr7avsLmpXulY3Uq9ybXfOmfFtzx0Zte5Q", + "vWRMzaXluiC+ST3GxpJSn7gLnuynuw0SPFqErSIO3rdjtifYW8D5PuPvNxCPbWAV4zpASnmQPDUQi1S6", + "k9L35OgBFhHTjbELPl+YZFUPwDkKoDJ9VFFXxQwT3lDYZefL3fQvrke7ue2s9h4Sjs8rIENrxVGcSDrZ", + "hMD9pGjmrfMZXaEVp9VJ+ODwaDQ6PBhdC4LbD+sKy/WkfMWVQK3305acV3mPsKrHzHKgag9FuvZ6XW1b", + "+K5cTaKNYjQ9xhSejEaMJGyGcHtFaux2z2Lz05sH7wQql49b0L/fKLdv3gdfL75TfErMK/+ypg+cax3N", + "qPp8i0O0hc1Ea+B8gey9w8Ho/qv9w+N794/3928CNrtYpLa8kQe/7797kBzQ2VHycPXgt/3Fg/lBehjU", + "wy65iLvS6k/QtjXKprwk66hINZZGdtwcMqaaBdPrViNAc064YANd5FptT3jcwAus/33r+b+and/OYKPs", + "cF6fZFWEoIboIGXdDZKXm8xG30VzNidPN8/iWrlMzYGE6a05FGjVcTBY62K/95EYD7noeA29rjTsfBFt", + "zK/bdhWFPOx40oO73LLiIfJuQDxUZr3pAl+/5AK207lU3CzSzbdF0awAJMe46d+1ievIUUNyMhdSsfrP", + "RZhcVYmCl3v9XvL7Uf3MuN+7Y4g5LOOCAN1WV6WCDmFkCVuyZPMqYJNS8VA2kp0qhgvx3f5g/xHGISS/", + "H303GjyqRxz07WpVl2/ft679OuqyhtVigr4I1f6jK0Vc+/XcREE/8VApvPJedijHjsbL2vT+6vCpu7UN", + "Lh+v7XEDE6hVAP1YSc9dbpOq0BSDRTaEcl8x1OqG9lglMjJlcy50F7vt4agw3N4Dwy157KDGbUF4WXyn", + "2j3QSpVOeJqymFODo1JmQwbDQUdbXFOXuFqVE/9WQFobhsW1R9vBFrYlXG27Jocfkdn7UdpvN413Ew4I", + "2tW8iopoZNiwT/gMbND1/n3Za6t0YGIkxqsde8i3kmQdD9ClHOjtJH0yl4aUyfgd7W25aLcLFuOHGp35", + "RvQNSxAHnwRapYAi45vY18lTkikZ51GZP5rgoEvsEJWLYVchf+vQbtS+gSneYOTYbt9oM2h0s0+27XfD", + "NgkE277V+6PtW30jRpF+L8/i7TzMNurGwa6EAb8lBTFgoqkve0MSrEzmbQeO/rK6gus6r7UlRyAS5Zl3", + "sABNrVNSwN2CLoZQXO9TljDDQp0QmcRllgTXJRfdzlL37z9ctLk40SO1PpCfGMsINRZJCb+XgjcvNDBf", + "wLS4S3ZGHjdcW4fXwBY+cqtVH9yDrZJY61ZVTbhtxRgsl2/YvC3yymXF/O1nUxPN1rFUPMOvCWkv24sJ", + "uDd92Fur/fgmzHJ3KaS9cK6HBgCshycucNb998tYYBDr6sR7FHLPh8jiFZ1j1b0tfpRWUglVqEQnByWG", + "ujJZWKoT/kmwHqn20C9eQMDXfUavK1tIuCF0Tps1KNoKXFpP9PYYovaSMa/ofCO6bzMdtmp+fzz4mzW3", + "k8nweO+7v/zfg7f/GTS7NwwImqlBzGYYcXXJVgNbzwmMFcM6ti0Wk+hpQ+fuzDCaojENceMdV6qO996o", + "4J6rX2i6NgUMVasUY9rfOqG//Ed7oFdlGV/jhbH17H50rZWbqElrpL+Xd1Km5j6o3mfU7Q7HApP0LtlK", + "k0qJt+mqdmK/0cUrlVB8cmHJfcjE8oJMOdbM1GMB6j2NIpaBWuWqBnEci5LIhhWjSbUfV2rOnxfnmbWB", + "FZA+vQaM/OL1q+9fvP7l6eTF2bNfHp9Mfnr2P73jnmDvBvYL8QBo7+jefVfuvbqS+8GSI1evnDEkpy5f", + "wcU8zHLU7BH6DHykkIKWrAh7HyW5hrAJXHpjkuvXyFjPWr5+zYmPBFU2JgmFZzjwbwD0hjHbe9VFF3Ht", + "iZFrrJPvrDxckHXRxRLOuIdXipFekghWKIGtCK92ubH1Rb+/6HaXWLjZwGHH1F84bAEr6s9cI3CHD4Kp", + "NCY7mELji/naJzYM9Qrwt4+LDoMhmJ+4ZtPo0aeoq/p6YyHVpUwGoOe1FJ8ImtXtWgRTCLArm5rRa/O+", + "zKcBZcbZuOd8TgMOl5Bj5ZPUP/UD2po6trb/rYXgwgkdT5uVOeyxtEvVqCTRsJZoM2jP90hBvG/BMkaw", + "ZpvEyytBhrXu91Jh9lyd4hBwSCwRH35T2nZ5yjzg5ABf2p6NvFHBrMysMpL2vTmVeWhbNi0Q5OMDj1Ss", + "shH4AouvuWQuQakDHA3SWx2dzL2M4jn63TXZKUxgfgmKtOt1v8Dmihan9H3xBWgB2kTTEYvzKCtq7T//", + "HqsPvHS7BOzTdYHDaGi5Yaz9OhVtWhNPVeubUaWq9Xnb9sGD53jVBu7XdrYaxFl+o0aaIXr8K+XmB6lQ", + "L24Hf7lxyH68/GOmEFavCcjfCc2epyyeyNxsPv/QDKRHe+UXlWbLSsXeBkCRiKNaXnMbL/DwJOUY1lca", + "loNFueJmBUYjp5lMMR/UlwfGhcQP4c/lh7Ek64cPaD2fBTJnnjPBFI+w4K2FGhSoMQFsX1n30JbAXIPA", + "RRHoxZMTZ3fxKMqoR3ODpOcDUB+fnYAznilre+iNhofDER7mjAma8d5x73C4Pxz1UKta4BT3plCw3yWS", + "O4260OBPYicJfe8bwZuKpszgG78GIBEwANM1RxWEzitKZEa5clpklsi4IBgOb2MFB3+hHttbuW+XvbPx", + "GPOtMQ2IZS/c5r5FQRnPDk7zYDRyWPHGXb+YxGQzJ/b+7sJoy+92kurcEgUKGqypeV62LJb+Q793NNq/", + "0pg2DQXPbujDrwV12cwMtfN7o9HNf/RE2PxEl23u4sKqJw4JqXrWfn0Le6bzNKVq5ResulqZ1G2CMQO7", + "lmtr9TijSQSsAssuAaANs8/BJEptCLfKBVar9i8ChdZPge3bb3KB1vS9jFefbAlr3/A2ig91dgbH5cMa", + "PX862inIeH0j3SMPWm6p9hYI6HtalFq/s5NyNHp08x8F22/CI8hW9QTsArO5xtinBCHYPQiTVOS3XBpK", + "iryGL+hIO5l1WpBbv7yK9v7g8Qd7vBMW8gecMZVSYbNEbJsth37tOFvfTHmcN95qnvBPnvbcTeXRiOxF", + "hYJc/YhWr62mMLh+HR31jtu+aacX3yHhH93CCXeTLart3uWRwxqn4Ff+ko6T8zVOSyEkKMs9Z+ZzofnR", + "bV5Zri7Dn/AUfSkE/JwVEl65W2uXwl6mcmEV4KAE+LLM3HTvfVMX/l6VTyrhQujXgK6xQoixDmYar4bE", + "r6lV+sHoqy1Ev1racLP6+TqD4X0uJ+zgNk4YzrjwFH29pr5eU5tOuaUWPwU8mJVT3sEGcSULxJ/P/nBl", + "68NX20N320Mny4Ng72xrwIMcEheaG8mYEb2AeqfgbLbATz4Ix1A1nP9OqIoWfMkQ3Q/r3uWJ4RlVGGKT", + "EvASWB96q2Fio1mi6G4Puhv4gMxygZuAHppNMARm0gbEWYZiciHAWEu1j5opcRXXKrTbsx80sBcdllcj", + "ebeQmhXAhsJUbnPM89ZWO8Zuh2PxyiHewgJiVLnnNZoliNu7wf4jBaFj4V741rMQHxGnaVpyLqoQPJFb", + "iE67LWtTxZFOdCRDoEOvmKDCDHTGIj6D6EOc1iVbucDWYIedSlnBgP04oS6MD9Q62A0D12HoUhil+Gnx", + "jDhKqvtvhDSuyHfp5PJYSlRB7YNghZJ5Iqc0mdj1uWQBn+BzbOEWpXS4lN4kIWNmq/JnK7OQwv6dT3Nh", + "cvv3VMl3minwOY0FZqS4tYZohUJAJO+wNl6aSWVsnAN+c88Oce+PS7b6MByLx3HKhX1D21dooiW4KeA9", + "Iz14iOVeLfRgT1PYD/4k10amVQhYT3d2mDI3WW5cao1mph+CPx0LI8kfHuTyw94f5Rc/oLOY0RjopNLE", + "Tgll67ZR6wmF2U+wacDdznABxj24SG2Yx1xRYXBMJUonmVe3dKcoE4FFaJsrHFFBMpnZEhtIVAsKJFfr", + "A0EraJIQg0fJvwuCO+5ky3wcBmE6bQUgtIhxjWPEBTn9vnKYRkcPw+dJs0ixUETJf52/+IXgrQx7YJuV", + "4Vq4RlyAwABRNFg7xPG0ZwCTZB1VWJ9x3OPxuFe4c+NdHGuuXbjMYIA+xe9gaN/Zz/R5/N1wCF1Zd+Ux", + "+fUP28sxnKUstYCo496HPqk8mHOzyKfFs7fhBW3DcTuvMQKyY6+5XeQklCPkTuXGt1ckmO6kuwUAQJCU", + "HKgauDLlgqrVpozKwNK7FZQzG8lYWYw/xhi5OO4dj33s4rjXH/eYWOJvLsBx3PsQXgHntWwvBoj3WeHc", + "LIjo/mi0ux0S3K1vwGfZwTHwiXXAVq2oqGQKO+jwaP9c/oF/a/3z+0JjE/DxEqPJKv7e+H7Xcj18/RY2", + "/xcQe4B1pAwjSgbVmmxwU2AtQbzasOjAtTwjFVWiqiIHfCMNfYCKiCVeH9hugbp9rwZQUcSS5LZPzl2d", + "m8JvV9RS+KIcdbhZ5fne6Fe4Y4ob3dZtV/Mn3A39fnGG/YBZ3xn12dLHYIcrySBKkNPxiW0MlpNzHNPg", + "nAlDnuGvQ/dfr5Qi6uVFIucXx9amAEURSMKFS1SoRFCD3OLWEl+yQEHFe/afRRm/HSvi/Osf/8RBcTH/", + "1z/+6dD3//WPf+Jx37MAeFhP/GLBqDJTRs3FMYF0vAFNIL7dTQYr/rIlUytyOHLOCHwEQxIsKqK9ZQ6I", + "mWPxkplciUpCh62op12HzocB8+EiZ9oBLUFDPnPlfmzQZcCg5M+yXcpbPdH9AGAlzqAyAbgVPQ0g2h+3", + "RdWditwL23LtnGvW3Gb86FoU4Xb+Yth7Y6l3YAd4RQaDSxw6d/jATZrsnJ8/2x0SVAMtVWBJJ1Rqym6c", + "fjP8ypO28yTLUeoMBVfZ8qaIZnTKE+5toS31aOwRTGm0AOCoIvC5QIP3XRz7kQKPgRhQF6HZx6Zj8eJ8", + "D22/hkUmV6zvOIFyGLBlwTrpEnDwC8i/uMGwtYFrOxYzRqEPcvLUMoEKTHqRsVl0LBBqBYNvuanVxuuP", + "hcX6ddjScPBAkk3wJfz+nBr2jq76pCL52vo1CTWgqes+NB4LmwPp1mCAYDKkMswh8jM7pIEPMXbJZIrN", + "wGqLE3El3vHbO2DwcKHX/TLh1X/OpsHaYRHpxHGY3xxVVGkNldjTi3O/G7t9oiWJEo7UEFExFnOMUPLw", + "ylLUdrXIdFtQFQ8iCZdAFW4LQqMTFs/beOyTSl+9G5Rkat8JHKcfm+T6pQkXi/UJwCG2EIKbXYpPXZtu", + "PkXX45/JqWgX6CpeRWtaZpbf2NX96mHs4GEMr5v3NoZcfk89RubNhSLbT9xRJLKnvfU1t08qS3YXpkay", + "48GH0F0jFTl7ckJoHCum9e6/tyESZmqptJT/4H4EVnwXMTFuLFLZvfD2ljqBfCns4KUbNaF+Xs0KyNX7", + "ba9WHqn1pisqJZVX3s3fHo2PXuUaKYXekta+3iRbo4C5jiCnskotAxSNElaIL8U5rVLRNquyjS8urpyN", + "4pJtBcEb7kDenn3ZfToXzbvhFpji0wZDvENGWM8Br9Y1/5Ko+XWxi25em8zPnxdpjm5PCrptU3SIzL8k", + "dTFuLBtwQYvA0nqBPmfG4q7cpJ7uvhCYONi53am2A13ZWRfTsq8SCyCDE0JLzGbd98Q26ab62v7+TJov", + "Ls9VJBa35F9FlA7KbrlWmxTcE1ek++b02xrE3C3H0zgCCywyPHC2acRrgrqXVK9EtPs1pOaTUzQudkWJ", + "VX7eLC4s2fi81LNuS66DgutlRXiuC72WCzJLAL7WOgFiPsMgQlOtsI6jPLiFURaVzBU1zMVOfokJyWd5", + "Yr0bkRRLpioQktUrde8PjKbdrip55rXxdn398ucBE5GMC+dJu0zqnnxihcnSfy3J+PZP3ReYaMu9eNAm", + "MH7E/tsod1Jgiv6vgx8cquj/OviBJhkX7H8dPk6oYdrs3hixjG7rprttBeYLJj7QX3h90dZY056h82qC", + "bJ0yPa7uVSizhofrjd1Fhs8no75PL+Q1QYQ7SXi3RvegSd02lMqJr5dgt1QqD6vVwEr+9xb6zmsEfev2", + "utrnucYRoGD3BeK3SOXhYKnD0+aeuB1vEnPFtN5mjChadbRH+PZ/KpOEnfSVjBLFun61S3SxS1SXa6Np", + "wja8WeOE/cYded8LYgutNj76igN2Cw4dR5EVHLCah7tEAltIbfDR5wEK8e+UicOLo1C9zzq6TEtOsfFe", + "82fq5Gkfd7gPe3rytEwGvyUHqh/HrVsE3HdvXxp7nE75PJe5ruaZp9REC6YdBkPC6jfDl2arKOWGVmvF", + "Z0ylo9u8027dGPGV7m/ITNLcUMu8bRTENmXEt+qqjLj2oIxYfGqHU+GqSfV9pcHdluwUj1DdlYxrQN4f", + "+l3GFVKSoDqHJqUeQ1C1OR6L/w2K0a+G0fTtdz4hPh+NDu7j70ws337nc+LFqScVJozirjDs41+eYujJ", + "HLMLsHZsCb/RHAdJc20s6fkiMf92mlsZfdNddfNU+FV166S6VZZrs+pmG96w7lYvNHXrypunt9CC22d/", + "UvXtT+5TrqmWOp/NeMSZwHJdaJXUa0HUVsX86k6+nn2UChfEUYnArEkindXIgmttkdCLGtCfNMSx31q1", + "QxJqDEszA7BPEYPqwTZ1UC9yE8t3whfxwAn6enC8nE/oevddTXwn4czdcFH7rppuUcjytlVd9+EvNHVW", + "Zq4mu1MuS9GmXbu8W+K9WZ2yw1V7+1rll0xiVn1bX7oMNIRAUTrrf0pzm2dcvFniWQ7Jq1c/+5xiUE+U", + "L3FopK9r6Gtfj0W1ruGQPCsLRtoGvgdQH1jsMAjgfV8pMGY0TrhgmITBdCj9t16N9E6PxaeXgMOlVm/b", + "997lWNoNvUMJ+M5Ywa3ImkWwgXGe4uJoVqqwFqfFy5t4ar4ofuUYUIDxhGS9PZobOXAoBXsLaTE1w7DC", + "ZwmNEFUYmlnASwcMYxFuq10h2ouSScKUBTLNcuPFLcAUdoPjwsiiVLeTzC6g+0kuDE8u+jYGEtommkCZ", + "eAt1Mxa1jzmZD8EbEJgER6hYZkfcqDsMg+ZgaoRWiLNQ/SShyTu60mPh4B7s61isXrHIYv4myZD8KBFp", + "x9bfrjBeW/z2Gz0WFzxO2MQB5VwQroleSGWYYDFJ5ZLp+ncZVQlnCifxhMLKaZLSFUJpWlRhuz4yYxau", + "sgbHI+HfVMQcy6jCl4spH48FJQejEUkZFdqBa2g6wwvH9UFwELUBfUsoORo9cm819g3h3v3y78BpUoot", + "ZUSnUDEXqNiWJ9/FDUxdWWNbJx+2b8aVtvtV2DddvcraxnLtq/TGfZKLEj4Ebf0V7xpsl8mVwHk69yTj", + "qrgGHWLSlEUU1lPI+ncQRFdGUa5CFyRsdaW+9r+j4FiZ3jkuVRicI0GTQQQ5ABKRHxd4piUepd1vW6iq", + "JKo/x0UTPCRSEUoqdF1aNFiUI2vcQdDZi7JYrPDF3y92v/VnB46vYwT++Iv5l3Q/IRHJ2ax2ALdfTfYA", + "b0qKWyfhP+s5feKYXo3FxZzOhdSGR54ZekzvqtL8VSHspBBuXtkgNc+kumyPhP5BqsuuGpiP2PyyFLHq", + "DD9DRwQMD4N07t4fgdZwq6wA0dy6ktakr+KUotDFjS5ilUkixZypqlX+1t0GVa1uxyJtwmWqrLO7wEUD", + "JWTifrSFxivx5uhiiFyvf9pgNSQ22OwyZm26sqpHiSR3NV4Jp6oK+GB1QW3jD/peHEK68hu2g9L7+nYF", + "mSoAjW5Fai0+7mFJA1CtY/Fa2+IOF9b1dEEKHgwCrS3YQt4teLSAfvA37N+iutIsuyig9HePyXM8yJVt", + "sx/fsWVagNa0TJhFY12m6cXxeqntN6en+BK2Wdii2hfHxJfXLu4PDa2qMKwwi4RqQ35x4LI7hTKOO3oB", + "NszK/HYdQGtZ6mAsQmCtgHVqO+QzclHBbb1oARX0/PZnOdefjauoLFBj52Ikcaoj0iYTca8tyIMnYcfP", + "/mgUqpvQET7WDuOG0WPXBvOznBfFcWqkTLOsK/m6YSIVL9N0Aw2TnZKDEG1imZu/aBMzpfBlR91txE12", + "aGT/YeglEKpDHvUHe3csWpbKzjC8VMBUe/0eE3naO/7V/WuZpr1+z40H3ntnsrRS7eIKQv4WON5mhx/6", + "oR2qYO7+icXzO7sSXSGl4qjEktnoSJR4nAYFUK54XnBLrwP9W7+hKti/jevO2QDa1YiXtsGf3sXpDY23", + "Jr47jaYhPgXp485OMYpZDVv45yTJV8Zbt5sJWaAq48ge3cbIEHqZRInUrOYn+3KAG529sCF6t9vb/BoP", + "YHhx7kvwdQkEOnevnvs3PwMTxraQGz9m4qd767E36yP4kkEo9NpsgP010P62BeV89oT06bZkbapdKOQr", + "bV7dWNuJMEHdWmcR7oXYlmSluZEpNTzCcoDRQkpdIfsCmt8W7nQ2+IIy0UJljQUuEeMCSPXCWfMvnIh5", + "7CyPhFYfuW8M8XWXvhF+wz8q3/ihYlwpOH7fa1BYmUZDyTTF2YxkNNcM5MwcLGKrKGG+/iMDv3pEM5Mr", + "hqVtGUm54GmeVmsuwI4tKeJDXeynF30yzQ1JwMqmioc+ZimSacpEbOWksVgwuuRMwaIk1DARrQaaYUn8", + "JSNQ9iGRNEZLTRZTdJhhIqViQIFYwCJlhsbUUBR0LuDET2wu2EVRJd9aRwR7X1ID+I5z8a2tpgPdXviB", + "XhCG5SK4XhTVlCMaMxEFyyicf95s7NOb9M+ZaU70jgKsrsVL7zLiqmq69sP5PIKxvljIjw5sfoPQq9uV", + "6noSjSejf88jbefq53hHfrpiiTed4s/DQVcQ3WfjpLt7L5xUJM7t5yqnEsn8z2pHLBhKDRECEnTtNl7X", + "v1ZUZy2W+Uo8b+8P/+fJNayLnwkn7Lcq9m11AMtJfw4s163qtXjuHZlVnS2papO7OxbsBnV34pNUFS73", + "pRhba8h7Bd+uciejKGpfUnxl20227eJGrsu2vW12LTKiwsi5GGCobZiDOzNuK6t2poN/06SexuwqLPPO", + "WWTpubh1GE3PGjO6AlPJnyHWeoP/KJJKWRQRatiXBf1dsRpWsyzQNlcWGO37pNc3p6e7bVxCmY08Qpkv", + "mENUMpvgtTReN+C+WDKleOzhTZ+cPi19sioXQ/Ii5YYYSS4Zy8rEIEzOBNTOAk+lMewmcEq/x4RRq0xy", + "YbaOomx6M4Mpf5AOVuYzFCVdLYmv7vDO7nC07H957Ay5DKHFBDZrpoaarTWuuZhJlVq5jE5lDr0DD4Jl", + "gv20wRYznjC90oalNrgToCDguGFdIld73r1nd7mPoc1wcmzWYcZUyrXmUuixcCk3GVPwbXgd+q/EqQUd", + "AoYW/PXMMsnPIwYSBmPD/qhpWzVEvsKa1r3j3h7Nsr2YGtoSZ+eG9xFD+gGDGolepVNIoISoyEtNdhJ+", + "adUTstQkgT92N0ZFTvC9T11Z//onC1b6RMxksG6ppdmCmP9UyXGOrXnH5GfB1u5Iz/zoKMLnrHpwPS+c", + "yVYWq7fyWMVoMsC6/B6LieSGJ/x3y3ahE64Nj2wWGS328c1pweCHY3HKjOKR9fdGEH+OIBWo6e5lSkZ7", + "gON3GGUcAf0OGQ4OmW/74xS/+OTsNbZLWSrVqj8WVLiOXz0+s57mGY1Yc6CCGfAkk5O9F1ti1s9xmf6N", + "4yftBDcCUgQ3/Kt78uqwMa1nSLccUZltUttk9qcP8HXS5Fcbx5dp4wAaLmezU2C1eYC1sD1jKZM8hX/Y", + "P062QdUZGi3eYNPPRvK2w9n6GT/BL+JQujnFzNZ4vhMHjF2wLzV+FhbOTwGFmFpkYvAWeGz+jNT96V0J", + "1XX8DF2vbkV9/fTP5mzd9s3nxuBBU6rr8aUcc0tpfiZGbraEvaO83RL2fSKjS+3QdaomTKIYRch8+LGE", + "OHfuShQTMNmXOFQqi33GdH8sGsZQC+KkCSWGqZQLmuzhnG0nCNbuLWp0KTnm3EeYMzPQYOrOZJIgIjsi", + "GsJsoMXMd1DxLmtXYbLapuoYNRJgm2TKPID9bkh1+yvl5gep6mj0nwtffFVZf0R5pBxt/1sA+Nu/+FGA", + "/Kf0PYZtx7lzbvsR7TyX5Y/WLNUnuDfj3uFIj3t9Mu4dpOMe7MATiuZcasg9knKRG0C2fGptbZhVfX9E", + "NIukiLXH0ffWxMORbpm5I8uWhN37+N5tij2OqnApX7qPhNgDtCPwPiYQkZ3qgXNnMu7joYuJzI11Pbhz", + "5VrFzKB5ZPfW/caVM/JVt+/Cyf/qjm+NR+EuA7usbL3l7FmuF6zd5PazLZqVmymsQ1FkWy/I3+VU9yHr", + "wFrmATFvje/B22f2A7dROwI+dZW6EW7uX4tGdCgaUa5VGH/TBnvCleypw4JwsveZVAZ+d7gFjoZQk0Aw", + "ECyF+eLJyVhEwIosWqRiqUTu5CDu7S38+K/n5NmTl33yFAs+kx/z6e6QvIBMG+xVO3/RWFhJzDKviAKw", + "up1BHLqe7diRem4ycB0+cCUp/uDTeXrwZAS8PH6vfMB6v7dgNEaJ5I/ez9J+LAAk/fJnOECwf+7NYtt7", + "G4UPsOGr1eDxzDC13u2py9kSBQyKu6Q9qqAT3PDD+EHtwfTKb1rZwKKdHB70AuAnH77W8bj5QuG347HD", + "77rSjpDN96XWCMaDWDDHEAusXtdFJYy2jGXHyzYqGPjJk6efRKm4US/1Jt5VKxfw73q6cKZfrKMpq+0T", + "EHFRQWerp9cnKi8swrV9jUQ0oxE3qz6hSeLuKHcTFNExg0L8nSpGL8FuPwQUMvdln1wMvtq+d9SSmOtL", + "24PzxQ4JhMfpfFoMjuBBs15jXHNI0TWSRDSJ8oQaRthsxiLMC8aSPLrFl1sMpXeDZ6f8SBAWphJhn39x", + "ZQvDNIG7V5JFk+L27FbvKRYllKftePJOUMPgRww1mEKnUkDsQmIpjkZKak1cVwOW8DmHshoYKQGVNMDM", + "RVM2FllChWCK5NpG6MPQB5liWueK+Q7g7nIU1SclVmSmpHGhCYmUSo+Fp/A3p0Qblm0gs5e251Oc8w3J", + "trZz96U7MlI3xtBuCnFNCGyIpRS74EBHYBq6g7B4O6C7lhK/lIP/SvE5xL5mSlLLZG1ckj3Wfjntoa9l", + "T7eWMD0vWnUrYVr0WsmQrGQPbsT6m5Tw6XHvahGIgY9f8lY4SPfoahnNP8FLHb9dz5wND8I9+shZhqqx", + "/jsWPj2vJCx2NWCVFP6lmZMqI68d1VrS73aIr85ZvjeZddsZy+vOILy+ZOQuWkvlbVN4Pz9CGN0u4sRt", + "1837smmrhrxV001b4Ae2F0j4LCjwZioj3DHiyjUqI3xWGABAO3eIxRI8qHeV01/zPUv1tbjBjaby2woH", + "CA3XlspvuZ4LXt2oKL1xbbqpSa7HP5MEbxfoKvK7X/avWn8HlaGyWNtc0EDwLM3Myr3kfZVl0Jnmv7Nh", + "iyO4iFu9OVfwNUI6Px15eDptDei8Q/SyO3Tk3EnMqKsGyTWE+ZVV93xi9xeGd1g9c7WLZQ9unQFV0YIv", + "WbvRvX6C3RJlig0ymaFzJbYL5tbD32WGquH8d+K6d/iv7l9YThQLGbCYxFyxyEC0iDASOYL9xjeaKCmN", + "ey7Vqj1KxB6RH5RMH7vZbLkP3ZlyxrAyzjBdDWJq6GDpuc0GE9pHRHf6eEpgeIQL8vx7ssPeG2WLlpAZ", + "aD6Qbe6XlL2PGIs10uRudcD7o/AQoePJfNpllBvKz7zAP2hColwbmfq9P3lKdrCc3ZwJ2AsQ9WcoyWZK", + "LnnM4toYIXjfrup+y4Je1e4KQkVRi9ArF3ZwdyLDdLmQ5r/zrM4WipCYKRcUB7e1wEv9TFlAAfge5cIH", + "4Lg98qP4eoU5zW/HKztAiVha1S2ikdLCTe9+vea+5Guumgzl77TabefDczYbr7vlR3VMW7qJIhRF7tzt", + "mq3ffD4pPVx/kdk8znS+LBTSNrP550WCo9u7H27bXP7mC04BBWP5srFstgO1DBPMzxjTHbMlS2SWYol7", + "bNvr93KV9I57C2Oy4709jP1eSG2Ojx49OOx9ePvh/x8AJoyqeVL+AQA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/otel/README.md b/lib/otel/README.md index ed4613b68..70762a5c9 100644 --- a/lib/otel/README.md +++ b/lib/otel/README.md @@ -99,7 +99,8 @@ This keeps pull and push views aligned because both are sourced from the same OT | `hypeman_disk_utilization_bytes` | gauge | component | Actual filesystem bytes allocated by disk component | | `hypeman_resources_image_storage_bytes` | gauge | kind | Current and maximum image storage bytes | | `hypeman_resources_gpu_slots` | gauge | kind | Total and used GPU slots | -| `hypeman_resources_gpu_profile_slots` | gauge | profile, kind | Available GPU slots by profile | +| `hypeman_resources_gpu_profile_slots` | gauge | profile, kind | How many more vGPUs of each profile fit if only that profile is added | +| `hypeman_resources_gpu_nvml_fallbacks_total` | counter | reason | vGPU capacity queries answered from the sysfs creatable lists for at least one GPU because NVML failed, a GPU was in equal-size mode, or NVML timed out or was stuck | ### Volumes | Metric | Type | Description | diff --git a/lib/resources/gpu.go b/lib/resources/gpu.go index 6dfd537ae..fd976a60e 100644 --- a/lib/resources/gpu.go +++ b/lib/resources/gpu.go @@ -27,8 +27,10 @@ type GPUResourceStatus struct { // GetGPUStatus returns the current GPU resource status and any error that // prevents determining allocatable vGPU capacity. The status is still // returned alongside such an error, with PlacementDisabledReason set. It -// returns nil if no GPU is available or the mode is "none". -func GetGPUStatus(ctx context.Context) (*GPUResourceStatus, error) { +// returns nil if no GPU is available or the mode is "none". Profiles are +// listed only if withProfiles is set, since admission uses only the slot +// counts. +func GetGPUStatus(ctx context.Context, withProfiles bool) (*GPUResourceStatus, error) { framework, vfs, err := devices.DiscoverVGPU() if err != nil { // Only report passthrough once vGPU discovery confirms no vGPU @@ -39,13 +41,13 @@ func GetGPUStatus(ctx context.Context) (*GPUResourceStatus, error) { return nil, nil } if framework != devices.VGPUFrameworkNone { - return getVGPUStatus(ctx, framework, vfs) + return getVGPUStatus(ctx, framework, vfs, withProfiles) } return getPassthroughStatus(), nil } // getVGPUStatus returns GPU status for vGPU mode (SR-IOV). -func getVGPUStatus(ctx context.Context, framework devices.VGPUFramework, vfs []devices.VirtualFunction) (*GPUResourceStatus, error) { +func getVGPUStatus(ctx context.Context, framework devices.VGPUFramework, vfs []devices.VirtualFunction, withProfiles bool) (*GPUResourceStatus, error) { usedSlots := 0 // Count used VFs (those with a vGPU assigned) for _, vf := range vfs { @@ -68,6 +70,9 @@ func getVGPUStatus(ctx context.Context, framework devices.VGPUFramework, vfs []d } status.AllocatableSlots = availability.AllocatableSlots status.QuarantinedSlots = availability.QuarantinedSlots + if !withProfiles { + return status, nil + } // Get available profiles (reuse VFs to avoid redundant discovery) profiles, err := devices.ListGPUProfilesWithVFs(framework, vfs, availability.Quarantined) diff --git a/lib/resources/gpu_test.go b/lib/resources/gpu_test.go index e15def07f..b579f5c6f 100644 --- a/lib/resources/gpu_test.go +++ b/lib/resources/gpu_test.go @@ -35,7 +35,7 @@ func initVFHealthForTest(t *testing.T, state []byte) { func TestGetVGPUStatusFailsClosedWhenVFHealthIsUnavailable(t *testing.T) { initVFHealthForTest(t, []byte("not json")) - status, err := getVGPUStatus(context.Background(), devices.VGPUFrameworkVendorVFIO, []devices.VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + status, err := getVGPUStatus(context.Background(), devices.VGPUFrameworkVendorVFIO, []devices.VirtualFunction{{PCIAddress: "0000:82:00.4"}}, false) assert.Zero(t, status.AllocatableSlots) assert.Zero(t, status.QuarantinedSlots) require.ErrorContains(t, err, "VF health state unavailable") @@ -53,12 +53,30 @@ func TestGetVGPUStatusReportsQuarantinedSlots(t *testing.T) { {PCIAddress: "0000:82:00.4"}, {PCIAddress: "0000:82:00.5", Allocated: true}, {PCIAddress: "0000:82:00.6"}, - }) + }, false) require.NoError(t, err) assert.Equal(t, 3, status.TotalSlots) assert.Equal(t, 1, status.UsedSlots) assert.Equal(t, 1, status.AllocatableSlots) assert.Equal(t, 1, status.QuarantinedSlots) + assert.Nil(t, status.Profiles) +} + +func TestOnlyFullStatusListsGPUProfiles(t *testing.T) { + var withProfiles []bool + setGPUStatusProvider(func(_ context.Context, profiles bool) (*GPUResourceStatus, error) { + withProfiles = append(withProfiles, profiles) + return &GPUResourceStatus{Mode: string(devices.GPUModeVGPU), TotalSlots: 1, AllocatableSlots: 1}, nil + }) + t.Cleanup(func() { setGPUStatusProvider(nil) }) + ctx := context.Background() + + mgr := NewManager(&config.Config{}, paths.New(t.TempDir())) + require.NoError(t, mgr.ValidateAllocation(ctx, 0, 0, 0, 0, 0, 0, true)) + statusMgr, _, _ := monitoringTestManager(t) + _, err := statusMgr.GetFullStatus(ctx) + require.NoError(t, err) + assert.Equal(t, []bool{false, true}, withProfiles) } func TestReserveAllocationUsesAllocatableGPUSlots(t *testing.T) { @@ -68,7 +86,7 @@ func TestReserveAllocationUsesAllocatableGPUSlots(t *testing.T) { UsedSlots: 1, AllocatableSlots: 0, } - setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { return status, nil }) + setGPUStatusProvider(func(context.Context, bool) (*GPUResourceStatus, error) { return status, nil }) t.Cleanup(func() { setGPUStatusProvider(nil) }) mgr := NewManager(&config.Config{}, paths.New(t.TempDir())) @@ -79,7 +97,7 @@ func TestReserveAllocationUsesAllocatableGPUSlots(t *testing.T) { disabled := *status disabled.PlacementDisabledReason = "VF health state unavailable: read failed" - setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { + setGPUStatusProvider(func(context.Context, bool) (*GPUResourceStatus, error) { return &disabled, errors.New(disabled.PlacementDisabledReason) }) err = mgr.ValidateAllocation(ctx, 0, 0, 0, 0, 0, 0, true) @@ -89,7 +107,7 @@ func TestReserveAllocationUsesAllocatableGPUSlots(t *testing.T) { require.NoError(t, err) require.NotNil(t, full.GPU) assert.Equal(t, "VF health state unavailable: read failed", full.GPU.PlacementDisabledReason) - setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { return status, nil }) + setGPUStatusProvider(func(context.Context, bool) (*GPUResourceStatus, error) { return status, nil }) full, err = statusMgr.GetFullStatus(ctx) require.NoError(t, err) assert.Empty(t, full.GPU.PlacementDisabledReason) diff --git a/lib/resources/monitoring.go b/lib/resources/monitoring.go index 59851658d..95aff3868 100644 --- a/lib/resources/monitoring.go +++ b/lib/resources/monitoring.go @@ -7,6 +7,7 @@ import ( "sync" "time" + "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/diskutilization" "github.com/kernel/hypeman/lib/logger" "go.opentelemetry.io/otel/attribute" @@ -199,7 +200,15 @@ func newMonitoringMetrics(meter metric.Meter, mgr *Manager) error { gpuProfileSlots, err := meter.Int64ObservableGauge( "hypeman_resources_gpu_profile_slots", - metric.WithDescription("Virtual functions able to create each vGPU profile (best-effort snapshot)"), + metric.WithDescription("How many more vGPUs of each profile fit if only that profile is added"), + ) + if err != nil { + return err + } + + gpuNVMLFallbacks, err := meter.Int64ObservableCounter( + "hypeman_resources_gpu_nvml_fallbacks_total", + metric.WithDescription("vGPU capacity queries answered from the sysfs creatable lists for at least one GPU because NVML failed (error), every GPU it failed for was in equal-size mode (equal_size), it timed out (timeout), or an earlier timed-out query had not returned (stuck)"), ) if err != nil { return err @@ -265,10 +274,13 @@ func newMonitoringMetrics(meter metric.Meter, mgr *Manager) error { ), ) } + for reason, count := range devices.NVMLFallbacks() { + o.ObserveInt64(gpuNVMLFallbacks, count, metric.WithAttributes(attribute.String("reason", reason))) + } } return nil - }, capacity, effectiveLimit, allocated, oversubRatio, diskBreakdown, diskUtilization, imageStorage, gpuSlots, gpuProfileSlots, gpuPlacementDisabled); err != nil { + }, capacity, effectiveLimit, allocated, oversubRatio, diskBreakdown, diskUtilization, imageStorage, gpuSlots, gpuProfileSlots, gpuPlacementDisabled, gpuNVMLFallbacks); err != nil { return err } diff --git a/lib/resources/monitoring_test.go b/lib/resources/monitoring_test.go index 357e1c79c..76aff19ea 100644 --- a/lib/resources/monitoring_test.go +++ b/lib/resources/monitoring_test.go @@ -199,7 +199,7 @@ func TestStartMonitoringPublishesGPUMetrics(t *testing.T) { mgr, _, _ := monitoringTestManager(t) originalProvider := currentGPUStatusProvider() - setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { + setGPUStatusProvider(func(context.Context, bool) (*GPUResourceStatus, error) { return &GPUResourceStatus{ Mode: "vgpu", TotalSlots: 8, @@ -233,13 +233,16 @@ func TestStartMonitoringPublishesGPUMetrics(t *testing.T) { require.Equal(t, int64(5), int64GaugeValue(t, rm, "hypeman_resources_gpu_profile_slots", map[string]string{"profile": "L40S-1Q", "kind": "available"})) require.Equal(t, int64(2), int64GaugeValue(t, rm, "hypeman_resources_gpu_profile_slots", map[string]string{"profile": "L40S-2Q", "kind": "available"})) require.Equal(t, int64(0), int64GaugeValue(t, rm, "hypeman_resources_gpu_placement_disabled", nil)) + for reason, count := range devices.NVMLFallbacks() { + require.Equal(t, count, int64SumValue(t, rm, "hypeman_resources_gpu_nvml_fallbacks_total", map[string]string{"reason": reason})) + } } func TestStartMonitoringPublishesGPUPlacementDisabled(t *testing.T) { mgr, _, _ := monitoringTestManager(t) originalProvider := currentGPUStatusProvider() - setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { + setGPUStatusProvider(func(context.Context, bool) (*GPUResourceStatus, error) { return &GPUResourceStatus{ Mode: "vgpu", TotalSlots: 8, @@ -375,6 +378,21 @@ func int64GaugeValue(t *testing.T, rm metricdata.ResourceMetrics, name string, w return 0 } +func int64SumValue(t *testing.T, rm metricdata.ResourceMetrics, name string, wantAttrs map[string]string) int64 { + t.Helper() + + metric := findMonitoringMetric(t, rm, name) + sum, ok := metric.Data.(metricdata.Sum[int64]) + require.True(t, ok, "expected int64 sum metric data for %s", name) + for _, point := range sum.DataPoints { + if metricAttrsMatch(point.Attributes, wantAttrs) { + return point.Value + } + } + t.Fatalf("metric %s with attrs %v not found", name, wantAttrs) + return 0 +} + func float64GaugeValue(t *testing.T, rm metricdata.ResourceMetrics, name string, wantAttrs map[string]string) float64 { t.Helper() diff --git a/lib/resources/resource.go b/lib/resources/resource.go index 51a92513e..8fd2efeab 100644 --- a/lib/resources/resource.go +++ b/lib/resources/resource.go @@ -37,13 +37,13 @@ var ( gpuStatusProvider = GetGPUStatus ) -func currentGPUStatusProvider() func(context.Context) (*GPUResourceStatus, error) { +func currentGPUStatusProvider() func(context.Context, bool) (*GPUResourceStatus, error) { gpuStatusProviderMu.RLock() defer gpuStatusProviderMu.RUnlock() return gpuStatusProvider } -func setGPUStatusProvider(fn func(context.Context) (*GPUResourceStatus, error)) { +func setGPUStatusProvider(fn func(context.Context, bool) (*GPUResourceStatus, error)) { if fn == nil { fn = GetGPUStatus } @@ -431,7 +431,7 @@ func (m *Manager) GetFullStatus(ctx context.Context) (*FullResourceStatus, error // A GPU status error only means vGPU placement is disabled. The status // carries the reason, so it is reported rather than failing the read. - gpuStatus, _ := currentGPUStatusProvider()(ctx) + gpuStatus, _ := currentGPUStatusProvider()(ctx, true) return &FullResourceStatus{ CPU: *cpuStatus, @@ -636,7 +636,7 @@ func (m *Manager) gpuAdmissionFor(ctx context.Context, req pendingAllocation) gp if req.GPUSlots == 0 { return gpuAdmission{} } - status, err := currentGPUStatusProvider()(ctx) + status, err := currentGPUStatusProvider()(ctx, false) return gpuAdmission{status: status, err: err} } diff --git a/openapi.yaml b/openapi.yaml index 14850b18f..73aa90cde 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1788,7 +1788,7 @@ components: example: 1024 available: type: integer - description: "Number of virtual functions currently able to create this profile. Best-effort: creating an instance may reduce availability on sibling functions sharing GPU framebuffer." + description: "Number of additional instances of this profile the host can take if only this profile is added. Counts for different profiles are not additive: an instance of one profile can reduce availability of others on the same GPU." example: 59 PassthroughDevice: