mpi/finalized: revamp INITIALIZED/FINALIZED

Per MPI-3.1:8.7.1 p361:11-13, it's valid for MPI_FINALIZED to be
invoked during an attribute destruction callback (e.g., during the
destruction of keyvals on MPI_COMM_SELF during the very beginning of
MPI_FINALIZE).  In such cases, MPI_FINALIZED must return "false".

Prior to this commit, we hung in FINALIZED if it were invoked during
a COMM_SELF attribute destruction callback in FINALIZE.  See
#5084.

This commit converts the MPI_INITIALIZED / MPI_FINALIZED
infrastructure to use a single enum (ompi_mpi_state, set atomically)
to represent the state of MPI:

- not initialized
- init started
- init completed
- finalize started
- finalize past COMM_SELF destruction
- finalize completed

The "finalize past COMM_SELF destruction" state is what allows us to
return "false" from MPI_FINALIZED before COMM_SELF has been fully
destroyed / all attribute callbacks have been invoked.

Since this state is checked at nearly every MPI API call (to see if
we're outside of the INIT/FINALIZE epoch), care was taken to use
atomics to *set* the ompi_mpi_state value in ompi_mpi_init() and
ompi_mpi_finalize(), but performance-critical code paths can simply
read the variable without needing to use a slow call to an
opal_atomic_*() function.

Thanks to @AndrewGaspar for reporting the issue.

Signed-off-by: Jeff Squyres <jsquyres@cisco.com>

ompi/errhandler/errhandler.h

@@ -10,7 +10,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2008-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2008-2018 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2008-2009 Sun Microsystems, Inc.  All rights reserved.
  * Copyright (c) 2015-2016 Intel, Inc. All rights reserved.
  * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
@@ -193,11 +193,22 @@ struct ompi_request_t;
  * This macro directly invokes the ompi_mpi_errors_are_fatal_handler()
  * when an error occurs because MPI_COMM_WORLD does not exist (because
  * we're before MPI_Init() or after MPI_Finalize()).
+ *
+ * NOTE: The ompi_mpi_state variable is a volatile that is set
+ * atomically in ompi_mpi_init() and ompi_mpi_finalize().  The
+ * appropriate memory barriers are done in those 2 functions such that
+ * we do not need to do a read memory barrier here (in
+ * potentially-performance-critical code paths) before reading the
+ * variable.
  */
-#define OMPI_ERR_INIT_FINALIZE(name) \
-  if( OPAL_UNLIKELY(!ompi_mpi_initialized || ompi_mpi_finalized) ) { \
-    ompi_mpi_errors_are_fatal_comm_handler(NULL, NULL, name); \
-  }
+#define OMPI_ERR_INIT_FINALIZE(name)                                    \
+    {                                                                   \
+        int32_t state = ompi_mpi_state;                                 \
+        if (OPAL_UNLIKELY(state < OMPI_MPI_STATE_INIT_COMPLETED ||      \
+                          state > OMPI_MPI_STATE_FINALIZE_PAST_COMM_SELF_DESTRUCT)) { \
+            ompi_mpi_errors_are_fatal_comm_handler(NULL, NULL, name);   \
+        }                                                               \
+    }
 
 /**
  * This is the macro to invoke to directly invoke an MPI error

ompi/errhandler/errhandler_predefined.c

@@ -10,7 +10,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2006      University of Houston. All rights reserved.
- * Copyright (c) 2008-2013 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2008-2018 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2009      Sun Microsystems, Inc.  All rights reserved.
  * Copyright (c) 2010-2011 Oak Ridge National Labs.  All rights reserved.
  * Copyright (c) 2012      Los Alamos National Security, LLC.
@@ -149,7 +149,8 @@ void ompi_mpi_errors_return_win_handler(struct ompi_win_t **win,
 
 static void out(char *str, char *arg)
 {
-    if (ompi_rte_initialized && !ompi_mpi_finalized) {
+    if (ompi_rte_initialized &&
+        ompi_mpi_state < OMPI_MPI_STATE_FINALIZE_PAST_COMM_SELF_DESTRUCT) {
         if (NULL != arg) {
             opal_output(0, str, arg);
         } else {
@@ -280,7 +281,9 @@ static void backend_fatal_no_aggregate(char *type,
 {
     char *arg;
 
-    assert(!ompi_mpi_initialized || ompi_mpi_finalized);
+    int32_t state = ompi_mpi_state;
+    assert(state < OMPI_MPI_STATE_INIT_COMPLETED ||
+           state >= OMPI_MPI_STATE_FINALIZE_PAST_COMM_SELF_DESTRUCT);
 
     fflush(stdout);
     fflush(stderr);
@@ -289,7 +292,7 @@ static void backend_fatal_no_aggregate(char *type,
 
     /* Per #2152, print out in plain english if something was invoked
        before MPI_INIT* or after MPI_FINALIZE */
-    if (!ompi_mpi_init_started && !ompi_mpi_initialized) {
+    if (state < OMPI_MPI_STATE_INIT_STARTED) {
         if (NULL != arg) {
             out("*** The %s() function was called before MPI_INIT was invoked.\n"
                 "*** This is disallowed by the MPI standard.\n", arg);
@@ -300,7 +303,7 @@ static void backend_fatal_no_aggregate(char *type,
                 "*** function was invoked, sorry.  :-(\n", NULL);
         }
         out("*** Your MPI job will now abort.\n", NULL);
-    } else if (ompi_mpi_finalized) {
+    } else if (state >= OMPI_MPI_STATE_FINALIZE_PAST_COMM_SELF_DESTRUCT) {
         if (NULL != arg) {
             out("*** The %s() function was called after MPI_FINALIZE was invoked.\n"
                 "*** This is disallowed by the MPI standard.\n", arg);

ompi/mca/coll/fca/coll_fca_ops.c

@@ -2,6 +2,7 @@
  * Copyright (c) 2011      Mellanox Technologies. All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018      Cisco Systems, Inc.  All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -159,7 +160,7 @@ int mca_coll_fca_barrier(struct ompi_communicator_t *comm,
     int ret;
 
     FCA_VERBOSE(5,"Using FCA Barrier");
-    if (OPAL_UNLIKELY(ompi_mpi_finalize_started)) {
+    if (OPAL_UNLIKELY(ompi_mpi_state >= OMPI_MPI_STATE_FINALIZE_STARTED)) {
         FCA_VERBOSE(5, "In finalize, reverting to previous barrier");
         goto orig_barrier;
     }

ompi/mca/coll/hcoll/coll_hcoll_module.c

@@ -4,6 +4,7 @@
  * Copyright (c) 2017      The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
+ * Copyright (c) 2018      Cisco Systems, Inc.  All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -241,7 +242,7 @@ static int mca_coll_hcoll_module_enable(mca_coll_base_module_t *module,
 
 int mca_coll_hcoll_progress(void)
 {
-    if (ompi_mpi_finalized){
+    if (ompi_mpi_state >= OMPI_MPI_STATE_FINALIZE_STARTED) {
         hcoll_rte_p2p_disabled_notify();
     }
 

ompi/mca/coll/hcoll/coll_hcoll_ops.c

@@ -2,6 +2,7 @@
   Copyright (c) 2011      Mellanox Technologies. All rights reserved.
   Copyright (c) 2015      Research Organization for Information Science
                           and Technology (RIST). All rights reserved.
+  Copyright (c) 2018      Cisco Systems, Inc.  All rights reserved
   $COPYRIGHT$
 
   Additional copyrights may follow
@@ -21,7 +22,7 @@ int mca_coll_hcoll_barrier(struct ompi_communicator_t *comm,
     mca_coll_hcoll_module_t *hcoll_module = (mca_coll_hcoll_module_t*)module;
     HCOL_VERBOSE(20,"RUNNING HCOL BARRIER");
 
-    if (OPAL_UNLIKELY(ompi_mpi_finalize_started)) {
+    if (OPAL_UNLIKELY(ompi_mpi_state >= OMPI_MPI_STATE_FINALIZE_STARTED)) {
         HCOL_VERBOSE(5, "In finalize, reverting to previous barrier");
         goto orig_barrier;
     }

ompi/mca/io/romio314/src/io_romio314_file_open.c

@@ -12,6 +12,7 @@
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
+ * Copyright (c) 2018      Cisco Systems, Inc.  All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -69,7 +70,7 @@ mca_io_romio314_file_close (ompi_file_t *fh)
        which we obviously can't do if we've started to MPI_Finalize).
        The user didn't close the file, so they should expect
        unexpected behavior. */
-    if (ompi_mpi_finalized) {
+    if (ompi_mpi_state >= OMPI_MPI_STATE_FINALIZE_PAST_COMM_SELF_DESTRUCT) {
         return OMPI_SUCCESS;
     }
 

ompi/mca/pml/yalla/pml_yalla.c

@@ -2,6 +2,7 @@
  * Copyright (C) 2001-2011 Mellanox Technologies Ltd. ALL RIGHTS RESERVED.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018      Cisco Systems, Inc.  All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -265,7 +266,7 @@ int mca_pml_yalla_del_procs(struct ompi_proc_t **procs, size_t nprocs)
 {
     size_t i;
 
-    if (ompi_mpi_finalized) {
+    if (ompi_mpi_state >= OMPI_MPI_STATE_FINALIZE_STARTED) {
         PML_YALLA_VERBOSE(3, "%s", "using bulk powerdown");
         mxm_ep_powerdown(ompi_pml_yalla.mxm_ep);
     }

ompi/mpi/c/finalized.c

@@ -11,7 +11,7 @@
  *                         All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2015      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2018 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2015      Intel, Inc. All rights reserved
  * $COPYRIGHT$
  *
@@ -44,13 +44,7 @@ int MPI_Finalized(int *flag)
 
     ompi_hook_base_mpi_finalized_top(flag);
 
-    /* We must obtain the lock to guarnatee consistent values of
-       ompi_mpi_initialized and ompi_mpi_finalized.  Note, too, that
-       this lock is held for the bulk of the duration of
-       ompi_mpi_init() and ompi_mpi_finalize(), so when we get the
-       lock, we are guaranteed that some other thread is not part way
-       through initialization or finalization. */
-    opal_mutex_lock(&ompi_mpi_bootstrap_mutex);
+    int32_t state = ompi_mpi_state;
 
     if (MPI_PARAM_CHECK) {
         if (NULL == flag) {
@@ -59,12 +53,11 @@ int MPI_Finalized(int *flag)
                whether we're currently (after MPI_Init and before
                MPI_Finalize) or not */
 
-            if (ompi_mpi_initialized && !ompi_mpi_finalized) {
-                opal_mutex_unlock(&ompi_mpi_bootstrap_mutex);
+            if (state >= OMPI_MPI_STATE_INIT_COMPLETED &&
+                state < OMPI_MPI_STATE_FINALIZE_PAST_COMM_SELF_DESTRUCT) {
                 return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_ARG,
                                               FUNC_NAME);
             } else {
-                opal_mutex_unlock(&ompi_mpi_bootstrap_mutex);
                 /* We have no MPI object here so call ompi_errhandle_invoke
                  * directly */
                 return ompi_errhandler_invoke(NULL, NULL, -1,
@@ -74,8 +67,7 @@ int MPI_Finalized(int *flag)
         }
     }
 
-    *flag = ompi_mpi_finalized;
-    opal_mutex_unlock(&ompi_mpi_bootstrap_mutex);
+    *flag = (state >= OMPI_MPI_STATE_FINALIZE_PAST_COMM_SELF_DESTRUCT);
 
     ompi_hook_base_mpi_finalized_bottom(flag);
 

ompi/mpi/c/get_library_version.c

@@ -9,7 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2014-2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2015      Intel, Inc. All rights reserved
@@ -58,7 +58,9 @@ int MPI_Get_library_version(char *version, int *resultlen)
                (i.e., use a NULL communicator, which will end up at the
                default errhandler, which is abort). */
 
-            if (ompi_mpi_initialized && !ompi_mpi_finalized) {
+            int32_t state = ompi_mpi_state;
+            if (state >= OMPI_MPI_STATE_INIT_COMPLETED &&
+                state < OMPI_MPI_STATE_FINALIZE_PAST_COMM_SELF_DESTRUCT) {
                 return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_ARG,
                                               FUNC_NAME);
             } else {

ompi/mpi/c/get_version.c

@@ -12,6 +12,7 @@
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2015      Intel, Inc. All rights reserved
+ * Copyright (c) 2018      Cisco Systems, Inc.  All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -54,7 +55,9 @@ int MPI_Get_version(int *version, int *subversion)
                (i.e., use a NULL communicator, which will end up at the
                default errhandler, which is abort). */
 
-            if (ompi_mpi_initialized && !ompi_mpi_finalized) {
+            int32_t state = ompi_mpi_state;
+            if (state >= OMPI_MPI_STATE_INIT_COMPLETED &&
+                state < OMPI_MPI_STATE_FINALIZE_PAST_COMM_SELF_DESTRUCT) {
                 return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_ARG,
                                               FUNC_NAME);
             } else {

ompi/mpi/c/initialized.c

@@ -11,7 +11,7 @@
  *                         All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2015      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2018 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2015      Intel, Inc. All rights reserved
  * $COPYRIGHT$
  *
@@ -44,13 +44,7 @@ int MPI_Initialized(int *flag)
 
     ompi_hook_base_mpi_initialized_top(flag);
 
-    /* We must obtain the lock to guarnatee consistent values of
-       ompi_mpi_initialized and ompi_mpi_finalized.  Note, too, that
-       this lock is held for the bulk of the duration of
-       ompi_mpi_init() and ompi_mpi_finalize(), so when we get the
-       lock, we are guaranteed that some other thread is not part way
-       through initialization or finalization. */
-    opal_mutex_lock(&ompi_mpi_bootstrap_mutex);
+    int32_t state = ompi_mpi_state;
 
     if (MPI_PARAM_CHECK) {
         if (NULL == flag) {
@@ -59,12 +53,11 @@ int MPI_Initialized(int *flag)
                whether we're currently (after MPI_Init and before
                MPI_Finalize) or not */
 
-            if (ompi_mpi_initialized && !ompi_mpi_finalized) {
-                opal_mutex_unlock(&ompi_mpi_bootstrap_mutex);
+            if (state >= OMPI_MPI_STATE_INIT_COMPLETED &&
+                state < OMPI_MPI_STATE_FINALIZE_PAST_COMM_SELF_DESTRUCT) {
                 return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_ARG,
                                               FUNC_NAME);
             } else {
-                opal_mutex_unlock(&ompi_mpi_bootstrap_mutex);
                 /* We have no MPI object here so call ompi_errhandle_invoke
                  * directly */
                 return ompi_errhandler_invoke(NULL, NULL, -1,
@@ -74,8 +67,7 @@ int MPI_Initialized(int *flag)
         }
     }
 
-    *flag = ompi_mpi_initialized;
-    opal_mutex_unlock(&ompi_mpi_bootstrap_mutex);
+    *flag = (state >= OMPI_MPI_STATE_INIT_COMPLETED);
 
     ompi_hook_base_mpi_initialized_bottom(flag);
 

ompi/mpi/tool/finalize.c

@@ -2,7 +2,7 @@
 /*
  * Copyright (c) 2012-2015 Los Alamos National Security, LLC. All rights
  *                         reserved.
- * Copyright (c) 2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2017      IBM Corporation. All rights reserved.
  * $COPYRIGHT$
  *
@@ -39,7 +39,9 @@ int MPI_T_finalize (void)
     if (0 == --ompi_mpit_init_count) {
         (void) ompi_info_close_components ();
 
-        if ((!ompi_mpi_initialized || ompi_mpi_finalized) &&
+        int32_t state = ompi_mpi_state;
+        if ((state < OMPI_MPI_STATE_INIT_COMPLETED ||
+             state >= OMPI_MPI_STATE_FINALIZE_PAST_COMM_SELF_DESTRUCT) &&
             (NULL != ompi_mpi_main_thread)) {
             /* we are not between MPI_Init and MPI_Finalize so we
              * have to free the ompi_mpi_main_thread */

ompi/peruse/peruse.c

@@ -4,6 +4,7 @@
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
  *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2018      Cisco Systems, Inc.  All rights reserved
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -64,8 +65,11 @@ const int PERUSE_num_events = (sizeof(PERUSE_events) / sizeof(peruse_event_assoc
 int PERUSE_Init (void)
 {
     if (MPI_PARAM_CHECK) {
-        if (!ompi_mpi_initialized || ompi_mpi_finalized)
+        int32_t state = ompi_mpi_state;
+        if (state < OMPI_MPI_STATE_INIT_COMPLETED ||
+            state >= OMPI_MPI_STATE_FINALIZE_STARTED) {
             return PERUSE_ERR_INIT;
+        }
     }
     ompi_peruse_init ();
     return PERUSE_SUCCESS;