prov/efa: Implement FI_OPT_SHARED_MEMORY_PERMITTED

This commit implements FI_OPT_SHARED_MEMORY_PERMITTED for the EFA
provider. All SHM provider resources are disabled when
FI_OPT_SHARED_MEMORY_PERMITTED is set to true.

Signed-off-by: Sai Sunku <sunkusa@amazon.com>

man/fi_efa.7.md

@@ -205,6 +205,12 @@ These OFI runtime parameters apply only to the RDM endpoint.
   [`ptrace protection`](https://wiki.ubuntu.com/SecurityTeam/Roadmap/KernelHardening#ptrace_Protection)
   is turned on. You can turn it off to enable shm transfer.
 
+  FI_EFA_ENABLE_SHM_TRANSFER is related to the FI_OPT_SHARED_MEMORY_PERMITTED endpoint option.
+  If FI_EFA_ENABLE_SHM_TRANSFER is set to true, the FI_OPT_SHARED_MEMORY_PERMITTED endpoint
+  option overrides FI_EFA_ENABLE_SHM_TRANSFER. If FI_EFA_ENABLE_SHM_TRANSFER is set to false,
+  but the FI_OPT_SHARED_MEMORY_PERMITTED is set to true, the FI_OPT_SHARED_MEMORY_PERMITTED
+  setopt call will fail with -FI_EINVAL.
+
 *FI_EFA_SHM_AV_SIZE*
 : Defines the maximum number of entries in SHM provider's address vector.
 

prov/efa/src/rdm/efa_rdm_ep_fiops.c

@@ -1175,6 +1175,35 @@ static int efa_rdm_ep_set_cuda_api_permitted(struct efa_rdm_ep *ep, bool cuda_ap
 	return 0;
 }
 
+/**
+ * @brief act on shared_memory_permitted flag called by efa_rdm_ep_setopt
+ * @param[in,out]	ep			endpoint
+ * @param[in]		shm_permitted	whether shared memory is permitted
+ * @return		0 on success,
+ *			-FI_EINVAL if shm is requested but the FI_EFA_ENABLE_SHM_TRANSFER environment variable is set to false
+ * @related efa_rdm_ep
+ */
+static int efa_rdm_ep_set_shared_memory_permitted(struct efa_rdm_ep *ep, bool shm_permitted)
+{
+	if (!shm_permitted) {
+		EFA_WARN(FI_LOG_EP_CTRL,
+			 "FI_OPT_SHARED_MEMORY_PERMITTED set to false. "
+			 "Tearing down shm provider resources.");
+		efa_rdm_ep_close_shm_resources(ep);
+		return FI_SUCCESS;
+	}
+
+	if (!efa_env.enable_shm_transfer) {
+		EFA_WARN(FI_LOG_EP_CTRL,
+			 "FI_OPT_SHARED_MEMORY_PERMITTED endpoint option set "
+			 "to true but FI_EFA_ENABLE_SHM_TRANSFER environment "
+			 "variable is set to false.");
+		return -FI_EINVAL;
+	}
+
+	return 0;
+}
+
 /**
  * @brief set use_device_rdma flag in efa_rdm_ep.
  *
@@ -1365,6 +1394,13 @@ static int efa_rdm_ep_setopt(fid_t fid, int level, int optname,
 		if (ret)
 			return ret;
 		break;
+	case FI_OPT_SHARED_MEMORY_PERMITTED:
+		if (optlen != sizeof(bool))
+			return -FI_EINVAL;
+		ret = efa_rdm_ep_set_shared_memory_permitted(efa_rdm_ep, *(bool *)optval);
+		if (ret)
+			return ret;
+		break;
 	case FI_OPT_EFA_USE_DEVICE_RDMA:
 		if (optlen != sizeof(bool))
 			return -FI_EINVAL;