diff --git a/man/fi_efa.7.md b/man/fi_efa.7.md index 712e3a5bab0..dc60d4e3771 100644 --- a/man/fi_efa.7.md +++ b/man/fi_efa.7.md @@ -205,6 +205,12 @@ These OFI runtime parameters apply only to the RDM endpoint. [`ptrace protection`](https://wiki.ubuntu.com/SecurityTeam/Roadmap/KernelHardening#ptrace_Protection) is turned on. You can turn it off to enable shm transfer. + FI_EFA_ENABLE_SHM_TRANSFER is related to the FI_OPT_SHARED_MEMORY_PERMITTED endpoint option. + If FI_EFA_ENABLE_SHM_TRANSFER is set to true, the FI_OPT_SHARED_MEMORY_PERMITTED endpoint + option overrides FI_EFA_ENABLE_SHM_TRANSFER. If FI_EFA_ENABLE_SHM_TRANSFER is set to false, + but the FI_OPT_SHARED_MEMORY_PERMITTED is set to true, the FI_OPT_SHARED_MEMORY_PERMITTED + setopt call will fail with -FI_EINVAL. + *FI_EFA_SHM_AV_SIZE* : Defines the maximum number of entries in SHM provider's address vector. diff --git a/prov/efa/src/rdm/efa_rdm_ep_fiops.c b/prov/efa/src/rdm/efa_rdm_ep_fiops.c index 0ae97d82913..68bedf4dd3c 100644 --- a/prov/efa/src/rdm/efa_rdm_ep_fiops.c +++ b/prov/efa/src/rdm/efa_rdm_ep_fiops.c @@ -1175,6 +1175,35 @@ static int efa_rdm_ep_set_cuda_api_permitted(struct efa_rdm_ep *ep, bool cuda_ap return 0; } +/** + * @brief act on shared_memory_permitted flag called by efa_rdm_ep_setopt + * @param[in,out] ep endpoint + * @param[in] shm_permitted whether shared memory is permitted + * @return 0 on success, + * -FI_EINVAL if shm is requested but the FI_EFA_ENABLE_SHM_TRANSFER environment variable is set to false + * @related efa_rdm_ep + */ +static int efa_rdm_ep_set_shared_memory_permitted(struct efa_rdm_ep *ep, bool shm_permitted) +{ + if (!shm_permitted) { + EFA_WARN(FI_LOG_EP_CTRL, + "FI_OPT_SHARED_MEMORY_PERMITTED set to false. " + "Tearing down shm provider resources."); + efa_rdm_ep_close_shm_resources(ep); + return FI_SUCCESS; + } + + if (!efa_env.enable_shm_transfer) { + EFA_WARN(FI_LOG_EP_CTRL, + "FI_OPT_SHARED_MEMORY_PERMITTED endpoint option set " + "to true but FI_EFA_ENABLE_SHM_TRANSFER environment " + "variable is set to false."); + return -FI_EINVAL; + } + + return 0; +} + /** * @brief set use_device_rdma flag in efa_rdm_ep. * @@ -1365,6 +1394,13 @@ static int efa_rdm_ep_setopt(fid_t fid, int level, int optname, if (ret) return ret; break; + case FI_OPT_SHARED_MEMORY_PERMITTED: + if (optlen != sizeof(bool)) + return -FI_EINVAL; + ret = efa_rdm_ep_set_shared_memory_permitted(efa_rdm_ep, *(bool *)optval); + if (ret) + return ret; + break; case FI_OPT_EFA_USE_DEVICE_RDMA: if (optlen != sizeof(bool)) return -FI_EINVAL;