diff --git a/docs/admin/deployment.md b/docs/admin/deployment.md index 633a48a83c8..29381f4db12 100644 --- a/docs/admin/deployment.md +++ b/docs/admin/deployment.md @@ -298,6 +298,8 @@ Help Options: MD-on-SSD config -f, --fabric-ports= Allow custom fabric interface ports to be specified for each engine config section. Comma separated port numbers, one per engine + -n, --allow-numa-imbalance Distribute NVMe devices equally across engines regardless of NUMA + affinity (may result in suboptimal performance due to cross-NUMA access) --skip-prep Skip preparation of devices during scan. ``` @@ -344,6 +346,8 @@ Help Options: MD-on-SSD config -f, --fabric-ports= Allow custom fabric interface ports to be specified for each engine config section. Comma separated port numbers, one per engine + -n, --allow-numa-imbalance Distribute NVMe devices equally across engines regardless of NUMA + affinity (may result in suboptimal performance due to cross-NUMA access) ``` The `daos_server` service must be running on the remote storage servers and as such a minimal @@ -404,6 +408,17 @@ this option is set then a MD-on-SSD config will be generated. - `--fabric-ports` enables custom port numbers to be assigned to each engine's fabric settings. Comma separated list must contain enough numbers to cover all engines generated in config. +- `--allow-numa-imbalance` allows the generation of a server config with NVMe devices distributed +equally across engines regardless of NUMA affinity. By default, config generation respects NUMA +affinity and balances the number of SSDs assigned to each engine by limiting all engines to use +only the lowest common number of SSDs across NUMA nodes. When this flag is set, all available SSDs +are collected and distributed equally across all engines, ignoring NUMA node boundaries. If the total +number of SSDs is not evenly divisible by the number of engines, only the maximum divisible number +of SSDs are used (e.g., 7 SSDs / 2 engines = 3 per engine, 1 SSD unused). This maximizes device +utilization in heterogeneous environments but **may result in suboptimal performance** due to +cross-NUMA memory access. The generated config will include `allow_numa_imbalance: true` to allow +the server to start with this configuration. + The text generated by the command and output to stdout can be copied and used as the server config file on relevant hosts (normally by copying to `/etc/daos/daos_server.yml` and (re)starting service). diff --git a/src/control/common/cmdutil/auto.go b/src/control/common/cmdutil/auto.go index 26bd56eba49..aa3f82252df 100644 --- a/src/control/common/cmdutil/auto.go +++ b/src/control/common/cmdutil/auto.go @@ -8,14 +8,15 @@ type deprecatedParams struct { type ConfGenCmd struct { deprecatedParams - MgmtSvcReplicas string `default:"localhost" short:"r" long:"ms-replicas" description:"Comma separated list of MS replica addresses to host management service"` - NrEngines int `short:"e" long:"num-engines" description:"Set the number of DAOS Engine sections to be populated in the config file output. If unset then the value will be set to the number of NUMA nodes on storage hosts in the DAOS system."` - SCMOnly bool `short:"s" long:"scm-only" description:"Create a SCM-only config without NVMe SSDs."` - NetClass string `default:"infiniband" short:"c" long:"net-class" description:"Set the network device class to be used" choice:"ethernet" choice:"infiniband"` - NetProvider string `short:"p" long:"net-provider" description:"Set the network fabric provider to be used"` - UseTmpfsSCM bool `short:"t" long:"use-tmpfs-scm" description:"Use tmpfs for scm rather than PMem"` - ExtMetadataPath string `short:"m" long:"control-metadata-path" description:"External storage path to store control metadata. Set this to a persistent location and specify --use-tmpfs-scm to create an MD-on-SSD config"` - FabricPorts string `short:"f" long:"fabric-ports" description:"Allow custom fabric interface ports to be specified for each engine config section. Comma separated port numbers, one per engine"` + MgmtSvcReplicas string `default:"localhost" short:"r" long:"ms-replicas" description:"Comma separated list of MS replica addresses to host management service"` + NrEngines int `short:"e" long:"num-engines" description:"Set the number of DAOS Engine sections to be populated in the config file output. If unset then the value will be set to the number of NUMA nodes on storage hosts in the DAOS system."` + SCMOnly bool `short:"s" long:"scm-only" description:"Create a SCM-only config without NVMe SSDs."` + NetClass string `default:"infiniband" short:"c" long:"net-class" description:"Set the network device class to be used" choice:"ethernet" choice:"infiniband"` + NetProvider string `short:"p" long:"net-provider" description:"Set the network fabric provider to be used"` + UseTmpfsSCM bool `short:"t" long:"use-tmpfs-scm" description:"Use tmpfs for scm rather than PMem"` + ExtMetadataPath string `short:"m" long:"control-metadata-path" description:"External storage path to store control metadata. Set this to a persistent location and specify --use-tmpfs-scm to create an MD-on-SSD config"` + FabricPorts string `short:"f" long:"fabric-ports" description:"Allow custom fabric interface ports to be specified for each engine config section. Comma separated port numbers, one per engine"` + AllowNumaImbalance bool `short:"n" long:"allow-numa-imbalance" description:"Allow generation of config with imbalanced NVMe distribution. For non-VMD: redistribute devices equally across engines. For VMD: accept imbalanced VMD domains as-is"` } // CheckDeprecated will check for deprecated parameters and update as needed. diff --git a/src/control/lib/control/auto.go b/src/control/lib/control/auto.go index d07a0a3e9c7..eab28204232 100644 --- a/src/control/lib/control/auto.go +++ b/src/control/lib/control/auto.go @@ -1,6 +1,6 @@ // // (C) Copyright 2020-2024 Intel Corporation. -// (C) Copyright 2025 Hewlett Packard Enterprise Development LP +// (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -70,8 +70,10 @@ type ( // Generate config with a tmpfs RAM-disk SCM. UseTmpfsSCM bool `json:"UseTmpfsSCM"` // Location to persist control-plane metadata, will generate MD-on-SSD config. - ExtMetadataPath string `json:"ExtMetadataPath"` - Log logging.Logger `json:"-"` + ExtMetadataPath string `json:"ExtMetadataPath"` + // Allow NUMA imbalance when NVMe devices are not evenly distributed. + AllowNumaImbalance bool `json:"AllowNumaImbalance"` + Log logging.Logger `json:"-"` } // ConfGenerateResp contains the generated server config. @@ -476,8 +478,73 @@ func (nsm numaSSDsMap) keys() (keys []int) { return } -// mapSSDs maps NUMA node ID to NVMe SSD PCI address set. -func (nsm numaSSDsMap) fromNVMe(ssds storage.NvmeControllers) error { +// redistributeSsdsIgnNuma allocates all available SSDs equally across engines, ignoring NUMA affinity. +// Each engine receives the same number of SSDs. If the total number of SSDs is not evenly +// divisible by the number of engines, only the maximum divisible number of SSDs are used and +// remainder SSDs are not included in the generated configuration. +func redistributeSsdsIgnNuma(req *ConfGenerateReq, numaCount int, nsm numaSSDsMap) error { + if numaCount == 0 { + return errors.New("no numa nodes detected") + } + if len(nsm) == 0 { + return errors.New("no ssds detected") + } + + req.Log.Debug("allow-numa-imbalance enabled, distributing SSDs equally across engines") + + // Collect all SSDs from all NUMA nodes + var allSSDs []string + for _, ssdAddrs := range nsm { + addrs := ssdAddrs.Strings() + + if ssdAddrs.HasVMD() { + // If addresses are for VMD backing devices, convert to the logical VMD + // domain address as this is what is expected in the server config. + newAddrSet, err := ssdAddrs.BackingToVMDAddresses() + if err != nil { + return errors.Wrap(err, "converting backing addresses to vmd") + } + addrs = newAddrSet.Strings() + } + allSSDs = append(allSSDs, addrs...) + } + + // Divide SSDs across NUMA rather than requested number of engines to preserve how the + // selection algorithms work when deciding on NUMA-to-engine mappings. + totalSSDs := len(allSSDs) + ssdsPerNuma := totalSSDs / numaCount + remainder := totalSSDs % numaCount + ssdsToUse := ssdsPerNuma * numaCount + + if remainder > 0 { + req.Log.Noticef("total SSDs (%d) not evenly divisible by engines (%d); "+ + "using %d SSDs (%d per engine), %d SSDs will not be used", + totalSSDs, numaCount, ssdsToUse, ssdsPerNuma, remainder) + } + + req.Log.Debugf("distributing %d SSDs equally across %d NUMA nodes (%d per node)", + ssdsToUse, numaCount, ssdsPerNuma) + + // Clear existing map and repopulate with redistributed SSDs + for k := range nsm { + delete(nsm, k) + } + + // Distribute SSDs equally across engines, using only the divisible portion + idx := 0 + for numaID := 0; numaID < numaCount; numaID++ { + numaSSDs := allSSDs[idx : idx+ssdsPerNuma] + nsm[numaID] = hardware.MustNewPCIAddressSet(numaSSDs...) + req.Log.Debugf("assigned %d SSDs to NUMA-%d: %v", ssdsPerNuma, numaID, + numaSSDs) + idx += ssdsPerNuma + } + + return nil +} + +// fromNVMe maps NUMA node ID to NVMe SSD PCI address set. +func (nsm numaSSDsMap) fromNVMe(req *ConfGenerateReq, ssds storage.NvmeControllers, numaCount int) error { if nsm == nil { return errors.Errorf("%T receiver is nil", nsm) } @@ -498,6 +565,13 @@ func (nsm numaSSDsMap) fromNVMe(ssds storage.NvmeControllers) error { nsm[nn] = newAddrSet } + if req.AllowNumaImbalance { + // Pretend NVMe devices are distributed equally across NUMA nodes + if err := redistributeSsdsIgnNuma(req, numaCount, nsm); err != nil { + return errors.Wrap(err, "redistributing numa-imbalanced ssds") + } + } + return nil } @@ -530,7 +604,7 @@ func getStorageDetails(req ConfGenerateReq, numaCount int, hs *HostStorage) (*st return nil, errors.New("requires nonzero HugepageSizeKiB") } - if err := sd.NumaSSDs.fromNVMe(hs.NvmeDevices); err != nil { + if err := sd.NumaSSDs.fromNVMe(&req, hs.NvmeDevices, numaCount); err != nil { return nil, errors.Wrap(err, "mapping ssd addresses to numa node") } @@ -1231,7 +1305,8 @@ func genServerConfig(req ConfGenerateReq, ecs []*engine.Config, tc *threadCounts WithMgmtSvcReplicas(req.MgmtSvcReplicas...). WithFabricProvider(ecs[0].Fabric.Provider). WithEngines(ecs...). - WithControlLogFile(defaultControlLogFile) + WithControlLogFile(defaultControlLogFile). + WithAllowNumaImbalance(req.AllowNumaImbalance) for idx := range cfg.Engines { tiers := cfg.Engines[idx].Storage.Tiers diff --git a/src/control/lib/control/auto_test.go b/src/control/lib/control/auto_test.go index 596b0d3c5de..0f32b3c005d 100644 --- a/src/control/lib/control/auto_test.go +++ b/src/control/lib/control/auto_test.go @@ -1,6 +1,6 @@ // // (C) Copyright 2020-2024 Intel Corporation. -// (C) Copyright 2025 Hewlett Packard Enterprise Development LP +// (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -696,6 +696,103 @@ func TestControl_AutoConfig_getStorageDetails(t *testing.T) { } } +func TestControl_AutoConfig_fromNVMe(t *testing.T) { + for name, tc := range map[string]struct { + ssds storage.NvmeControllers + numaCount int + allowImbalance bool + expErr error + expNumaSSDs numaSSDsMap + }{ + "allow imbalance distributes equally": { + ssds: storage.NvmeControllers{ + &storage.NvmeController{PciAddr: test.MockPCIAddr(0), SocketID: 0}, + &storage.NvmeController{PciAddr: test.MockPCIAddr(1), SocketID: 0}, + &storage.NvmeController{PciAddr: test.MockPCIAddr(2), SocketID: 0}, + &storage.NvmeController{PciAddr: test.MockPCIAddr(3), SocketID: 0}, + &storage.NvmeController{PciAddr: test.MockPCIAddr(4), SocketID: 1}, + &storage.NvmeController{PciAddr: test.MockPCIAddr(5), SocketID: 1}, + }, + numaCount: 2, + allowImbalance: true, + expNumaSSDs: numaSSDsMap{ + 0: hardware.MustNewPCIAddressSet(test.MockPCIAddrs(0, 1, 2)...), + 1: hardware.MustNewPCIAddressSet(test.MockPCIAddrs(3, 4, 5)...), + }, + }, + "allow imbalance with remainder discards extras": { + ssds: storage.NvmeControllers{ + &storage.NvmeController{PciAddr: test.MockPCIAddr(0), SocketID: 0}, + &storage.NvmeController{PciAddr: test.MockPCIAddr(1), SocketID: 0}, + &storage.NvmeController{PciAddr: test.MockPCIAddr(2), SocketID: 0}, + &storage.NvmeController{PciAddr: test.MockPCIAddr(3), SocketID: 0}, + &storage.NvmeController{PciAddr: test.MockPCIAddr(4), SocketID: 0}, + &storage.NvmeController{PciAddr: test.MockPCIAddr(5), SocketID: 1}, + &storage.NvmeController{PciAddr: test.MockPCIAddr(6), SocketID: 1}, + }, + numaCount: 2, + allowImbalance: true, + expNumaSSDs: numaSSDsMap{ + 0: hardware.MustNewPCIAddressSet(test.MockPCIAddrs(0, 1, 2)...), + 1: hardware.MustNewPCIAddressSet(test.MockPCIAddrs(3, 4, 5)...), + }, + }, + "allow imbalance with 8 SSDs across 2 engines": { + ssds: storage.NvmeControllers{ + &storage.NvmeController{PciAddr: test.MockPCIAddr(0), SocketID: 0}, + &storage.NvmeController{PciAddr: test.MockPCIAddr(1), SocketID: 0}, + &storage.NvmeController{PciAddr: test.MockPCIAddr(2), SocketID: 0}, + &storage.NvmeController{PciAddr: test.MockPCIAddr(3), SocketID: 0}, + &storage.NvmeController{PciAddr: test.MockPCIAddr(4), SocketID: 0}, + &storage.NvmeController{PciAddr: test.MockPCIAddr(5), SocketID: 1}, + &storage.NvmeController{PciAddr: test.MockPCIAddr(6), SocketID: 1}, + &storage.NvmeController{PciAddr: test.MockPCIAddr(7), SocketID: 1}, + }, + numaCount: 2, + allowImbalance: true, + expNumaSSDs: numaSSDsMap{ + 0: hardware.MustNewPCIAddressSet(test.MockPCIAddrs(0, 1, 2, 3)...), + 1: hardware.MustNewPCIAddressSet(test.MockPCIAddrs(4, 5, 6, 7)...), + }, + }, + "no imbalance keeps original distribution": { + ssds: storage.NvmeControllers{ + &storage.NvmeController{PciAddr: test.MockPCIAddr(0), SocketID: 0}, + &storage.NvmeController{PciAddr: test.MockPCIAddr(1), SocketID: 0}, + &storage.NvmeController{PciAddr: test.MockPCIAddr(2), SocketID: 1}, + &storage.NvmeController{PciAddr: test.MockPCIAddr(3), SocketID: 1}, + }, + numaCount: 2, + allowImbalance: false, + expNumaSSDs: numaSSDsMap{ + 0: hardware.MustNewPCIAddressSet(test.MockPCIAddrs(0, 1)...), + 1: hardware.MustNewPCIAddressSet(test.MockPCIAddrs(2, 3)...), + }, + }, + } { + t.Run(name, func(t *testing.T) { + log, buf := logging.NewTestLogger(t.Name()) + defer test.ShowBufferOnFailure(t, buf) + + req := ConfGenerateReq{ + Log: log, + AllowNumaImbalance: tc.allowImbalance, + } + + nsm := make(numaSSDsMap) + gotErr := nsm.fromNVMe(&req, tc.ssds, tc.numaCount) + test.CmpErr(t, tc.expErr, gotErr) + if tc.expErr != nil { + return + } + + if diff := cmp.Diff(tc.expNumaSSDs, nsm, defStorCmpOpts...); diff != "" { + t.Fatalf("unexpected numa SSDs (-want, +got):\n%s\n", diff) + } + }) + } +} + func TestControl_AutoConfig_filterDevicesByAffinity(t *testing.T) { singlePMemMap := numaSCMsMap{0: []string{"/dev/pmem0"}} @@ -1075,7 +1172,7 @@ func TestControl_AutoConfig_correctSSDCounts(t *testing.T) { for name, tc := range map[string]struct { sd storageDetails expErr error - expSD storageDetails // expected details after updates + expSD storageDetails }{ "no ssds": { sd: storageDetails{ diff --git a/src/control/lib/hardware/pci.go b/src/control/lib/hardware/pci.go index e8109d9e799..2a5aca38e01 100644 --- a/src/control/lib/hardware/pci.go +++ b/src/control/lib/hardware/pci.go @@ -1,5 +1,6 @@ // // (C) Copyright 2021-2024 Intel Corporation. +// (C) Copyright 2026 Hewlett Packard Enterprise Development LP // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -180,7 +181,8 @@ func (pa *PCIAddress) BackingToVMDAddress() (*PCIAddress, error) { return nil, ErrNotVMDBackingAddress } - return pa.VMDAddr, nil + va := *pa.VMDAddr + return &va, nil } // NewPCIAddress creates a PCIAddress struct from input string. @@ -396,8 +398,11 @@ func (pas *PCIAddressSet) BackingToVMDAddresses() (*PCIAddressSet, error) { for _, inAddr := range pas.Addresses() { if !inAddr.IsVMDBackingAddress() { - if err := outAddrs.Add(inAddr); err != nil { - return nil, err + if inAddr != nil { + ia := *inAddr + if err := outAddrs.Add(&ia); err != nil { + return nil, err + } } continue }