diff --git a/cime/config/xml_schemas/config_machines.xsd b/cime/config/xml_schemas/config_machines.xsd
index c4d189cb1602..f59f2a5f264c 100644
--- a/cime/config/xml_schemas/config_machines.xsd
+++ b/cime/config/xml_schemas/config_machines.xsd
@@ -56,6 +56,8 @@
   <xs:element name="executable" type="xs:string"/>
   <xs:element name="default_run_exe" type="xs:string"/>
   <xs:element name="default_run_misc_suffix" type="xs:string"/>
+  <xs:element name="run_exe" type="xs:string"/>
+  <xs:element name="run_misc_suffix" type="xs:string"/>
   <xs:element name="RUNDIR" type="xs:string"/>
   <xs:element name="EXEROOT" type="xs:string"/>
   <xs:element name="TEST_TPUT_TOLERANCE" type="xs:string"/>
@@ -179,6 +181,10 @@
       <xs:sequence>
         <xs:element ref="executable"/>
         <xs:element ref="arguments" minOccurs="0"/>
+        <!-- run_exe: The run executable to use; overrides default_run_exe -->
+        <xs:element ref="run_exe" minOccurs="0"/>
+        <!-- run_misc_suffix: the misc run suffix to use; overrides default_run_misc_suffix -->
+        <xs:element ref="run_misc_suffix" minOccurs="0"/>
       </xs:sequence>
       <xs:attribute ref="compiler"/>
       <xs:attribute ref="queue"/>
diff --git a/cime/config/xml_schemas/env_mach_specific.xsd b/cime/config/xml_schemas/env_mach_specific.xsd
index 251eae546ea8..fabd35c9314d 100644
--- a/cime/config/xml_schemas/env_mach_specific.xsd
+++ b/cime/config/xml_schemas/env_mach_specific.xsd
@@ -152,6 +152,8 @@
       <xs:sequence>
         <xs:element ref="executable"/>
         <xs:element minOccurs="0" maxOccurs="1" ref="arguments"/>
+        <xs:element minOccurs="0" maxOccurs="1" name="run_exe" type="xs:string"/>
+        <xs:element minOccurs="0" maxOccurs="1" name="run_misc_suffix" type="xs:string"/>
       </xs:sequence>
       <xs:attribute ref="compiler"/>
       <xs:attribute ref="mpilib"/>
diff --git a/cime/doc/source/users_guide/machine.rst b/cime/doc/source/users_guide/machine.rst
index 77d8ae8b510d..7be970b5e7ba 100644
--- a/cime/doc/source/users_guide/machine.rst
+++ b/cime/doc/source/users_guide/machine.rst
@@ -57,6 +57,8 @@ Each ``<machine>`` tag requires the following input:
   * May have optional attributes of ``compiler``, ``mpilib`` and/or ``threaded``
   * May have an optional ``<arguments>`` element which in turn contains one or more ``<arg>`` elements.
     These specify the arguments to the mpi executable and are dependent on your mpi library implementation.
+  * May have an option ``<run_exe>`` element which overrides the ``default_run_exe``
+  * May have an option ``<run_misc_suffix>`` element which overrides the ``default_run_misc_suffix``
 
 
 * ``module_system``: How and what modules to load on this system. Module systems allow you to easily load multiple compiler environments on a machine. CIME provides support for two types of module tools: `module <http://www.tacc.utexas.edu/tacc-projects/mclay/lmod>`_ and `soft  <http://www.mcs.anl.gov/hs/software/systems/softenv/softenv-intro.html>`_. If neither of these is available on your machine, simply set ``<module_system type="none"\>``.
diff --git a/cime/scripts/Tools/Makefile b/cime/scripts/Tools/Makefile
index c7ee0c95edfa..e44a27e5b8e8 100644
--- a/cime/scripts/Tools/Makefile
+++ b/cime/scripts/Tools/Makefile
@@ -290,10 +290,6 @@ ifeq ($(strip $(USE_KOKKOS)), TRUE)
   CXX_LDFLAGS += $(KOKKOS_LDFLAGS)
 endif
 
-ifeq ($(strip $(USE_FMS)), TRUE)
-  SLIBS += -lfms
-endif
-
 # Set MOAB info if it is being used
 ifeq ($(strip $(USE_MOAB)), TRUE)
   ifdef MOAB_PATH
diff --git a/cime/scripts/fortran_unit_testing/run_tests.py b/cime/scripts/fortran_unit_testing/run_tests.py
index b648b35d549e..1c7cb4bbbb5f 100755
--- a/cime/scripts/fortran_unit_testing/run_tests.py
+++ b/cime/scripts/fortran_unit_testing/run_tests.py
@@ -360,7 +360,7 @@ def _main():
         }
 
         # We can get away with specifying case=None since we're using exe_only=True
-        mpirun_command, _ = machspecific.get_mpirun(None, mpi_attribs, None, exe_only=True)
+        mpirun_command, _, _, _ = machspecific.get_mpirun(None, mpi_attribs, None, exe_only=True)
         mpirun_command = machspecific.get_resolved_value(mpirun_command)
         logger.info("mpirun command is '{}'".format(mpirun_command))
 
diff --git a/cime/scripts/lib/CIME/SystemTests/erp.py b/cime/scripts/lib/CIME/SystemTests/erp.py
index 876f08947d36..f80c82ce7ffc 100644
--- a/cime/scripts/lib/CIME/SystemTests/erp.py
+++ b/cime/scripts/lib/CIME/SystemTests/erp.py
@@ -3,7 +3,7 @@
 
 This is a pes counts hybrid (open-MP/MPI) restart bfb test from
 startup.  This is just like an ERS test but the pe-counts/threading
-count are modified on retart.
+count are modified on restart.
 (1) Do an initial run with pes set up out of the box (suffix base)
 (2) Do a restart test with half the number of tasks and threads (suffix rest)
 """
diff --git a/cime/scripts/lib/CIME/SystemTests/system_tests_common.py b/cime/scripts/lib/CIME/SystemTests/system_tests_common.py
index cb375ec7c43d..77c7c05ece86 100644
--- a/cime/scripts/lib/CIME/SystemTests/system_tests_common.py
+++ b/cime/scripts/lib/CIME/SystemTests/system_tests_common.py
@@ -93,7 +93,7 @@ def build(self, sharedlib_only=False, model_only=False):
                     else:
                         excmsg = "Exception during build:\n{}\n{}".format(str(e), traceback.format_exc())
 
-                    append_testlog(excmsg)
+                    append_testlog(excmsg, self._orig_caseroot)
                     raise
 
                 finally:
@@ -163,7 +163,7 @@ def run(self, skip_pnl=False):
             else:
                 excmsg = "Exception during run:\n{}\n{}".format(str(e), traceback.format_exc())
 
-            append_testlog(excmsg)
+            append_testlog(excmsg, self._orig_caseroot)
             raise
 
         finally:
@@ -261,7 +261,7 @@ def _coupler_log_indicates_run_complete(self):
 
     def _component_compare_copy(self, suffix):
         comments = copy(self._case, suffix)
-        append_testlog(comments)
+        append_testlog(comments, self._orig_caseroot)
 
     def _component_compare_test(self, suffix1, suffix2,
                                 success_change=False,
@@ -280,7 +280,7 @@ def _component_compare_test(self, suffix1, suffix2,
         if success_change:
             success = not success
 
-        append_testlog(comments)
+        append_testlog(comments, self._orig_caseroot)
         status = TEST_PASS_STATUS if success else TEST_FAIL_STATUS
         with self._test_status:
             self._test_status.set_status("{}_{}_{}".format(COMPARE_PHASE, suffix1, suffix2), status)
@@ -349,7 +349,7 @@ def _phase_modifying_call(self, phase, function):
             excmsg = "Exception during {}:\n{}\n{}".format(phase, msg, traceback.format_exc())
 
             logger.warning(excmsg)
-            append_testlog(excmsg)
+            append_testlog(excmsg, self._orig_caseroot)
 
             with self._test_status:
                 self._test_status.set_status(phase, TEST_FAIL_STATUS, comments="exception")
@@ -384,7 +384,7 @@ def _check_for_memleak(self):
                         self._test_status.set_status(MEMLEAK_PHASE, TEST_PASS_STATUS)
                     else:
                         comment = "memleak detected, memory went from {:f} to {:f} in {:d} days".format(originalmem, finalmem, finaldate-originaldate)
-                        append_testlog(comment)
+                        append_testlog(comment, self._orig_caseroot)
                         self._test_status.set_status(MEMLEAK_PHASE, TEST_FAIL_STATUS, comments=comment)
 
     def compare_env_run(self, expected=None):
@@ -449,7 +449,7 @@ def _compare_memory(self):
                     elif self._test_status.get_status(MEMCOMP_PHASE) != TEST_FAIL_STATUS:
                         comment = "Error: Memory usage increase > 10% from baseline"
                         self._test_status.set_status(MEMCOMP_PHASE, TEST_FAIL_STATUS, comments=comment)
-                        append_testlog(comment)
+                        append_testlog(comment, self._orig_caseroot)
 
     def _compare_throughput(self):
         with self._test_status:
@@ -481,7 +481,7 @@ def _compare_throughput(self):
                         elif self._test_status.get_status(THROUGHPUT_PHASE) != TEST_FAIL_STATUS:
                             comment = "Error: Computation time increase > {:d} pct from baseline".format(int(tolerance*100))
                             self._test_status.set_status(THROUGHPUT_PHASE, TEST_FAIL_STATUS, comments=comment)
-                            append_testlog(comment)
+                            append_testlog(comment, self._orig_caseroot)
 
     def _compare_baseline(self):
         """
@@ -490,7 +490,7 @@ def _compare_baseline(self):
         with self._test_status:
             # compare baseline
             success, comments = compare_baseline(self._case)
-            append_testlog(comments)
+            append_testlog(comments, self._orig_caseroot)
             status = TEST_PASS_STATUS if success else TEST_FAIL_STATUS
             baseline_name = self._case.get_value("BASECMP_CASE")
             ts_comments = os.path.dirname(baseline_name) + ": " + get_ts_synopsis(comments)
@@ -503,7 +503,7 @@ def _generate_baseline(self):
         with self._test_status:
             # generate baseline
             success, comments = generate_baseline(self._case)
-            append_testlog(comments)
+            append_testlog(comments, self._orig_caseroot)
             status = TEST_PASS_STATUS if success else TEST_FAIL_STATUS
             baseline_name = self._case.get_value("BASEGEN_CASE")
             self._test_status.set_status(GENERATE_PHASE, status, comments=os.path.dirname(baseline_name))
diff --git a/cime/scripts/lib/CIME/XML/env_mach_specific.py b/cime/scripts/lib/CIME/XML/env_mach_specific.py
index 124ecffc0466..1c5729c32e26 100644
--- a/cime/scripts/lib/CIME/XML/env_mach_specific.py
+++ b/cime/scripts/lib/CIME/XML/env_mach_specific.py
@@ -39,7 +39,7 @@ def populate(self, machobj):
                 if len(nodes) == 0:
                     value = self.text(default_run_exe_node) if item == "run_exe" else self.text(default_run_misc_suffix_node)
                 else:
-                    value = nodes[0].text
+                    value = self.text(nodes[0])
 
                 entity_node = self.make_child("entry", {"id":item, "value":value}, root=group_node)
 
@@ -452,7 +452,7 @@ def get_mpirun(self, case, attribs, job, exe_only=False):
 
         # if there are no special arguments required for mpi-serial it need not have an entry in config_machines.xml
         if "mpilib" in attribs and attribs["mpilib"] == "mpi-serial" and best_match is None:
-            return "",[]
+            return "",[],None,None
 
         expect(best_match is not None or default_match is not None,
                "Could not find a matching MPI for attributes: {}".format(attribs))
@@ -474,8 +474,18 @@ def get_mpirun(self, case, attribs, job, exe_only=False):
         exec_node = self.get_child("executable", root=the_match)
         expect(exec_node is not None,"No executable found")
         executable = self.text(exec_node)
+        run_exe = None
+        run_misc_suffix = None
 
-        return executable, args
+        run_exe_node = self.get_optional_child('run_exe', root=the_match)
+        if run_exe_node:
+            run_exe = self.text(run_exe_node)
+
+        run_misc_suffix_node = self.get_optional_child('run_misc_suffix', root=the_match)
+        if run_misc_suffix_node:
+            run_misc_suffix = self.text(run_misc_suffix_node)
+
+        return executable, args, run_exe, run_misc_suffix
 
     def get_type_info(self, vid):
         return "char"
diff --git a/cime/scripts/lib/CIME/case/case.py b/cime/scripts/lib/CIME/case/case.py
index ba7b942e4172..2305bc4ccc6f 100644
--- a/cime/scripts/lib/CIME/case/case.py
+++ b/cime/scripts/lib/CIME/case/case.py
@@ -1364,7 +1364,6 @@ def get_mpirun_cmd(self, job=None, allow_unresolved_envvars=True):
         run_exe = env_mach_specific.get_value("run_exe")
         run_misc_suffix = env_mach_specific.get_value("run_misc_suffix")
         run_misc_suffix = "" if run_misc_suffix is None else run_misc_suffix
-        run_suffix = run_exe + run_misc_suffix
 
         mpirun_cmd_override = self.get_value("MPI_RUN_COMMAND")
         if mpirun_cmd_override not in ["", None, "UNSET"]:
@@ -1379,7 +1378,14 @@ def get_mpirun_cmd(self, job=None, allow_unresolved_envvars=True):
             "unit_testing" : False
             }
 
-        executable, mpi_arg_list = env_mach_specific.get_mpirun(self, mpi_attribs, job)
+        executable, mpi_arg_list, custom_run_exe, custom_run_misc_suffix = env_mach_specific.get_mpirun(self, mpi_attribs, job)
+        if custom_run_exe:
+            logger.info('Using a custom run_exe {}'.format(custom_run_exe))
+            run_exe = custom_run_exe
+        if custom_run_misc_suffix:
+            logger.info('Using a custom run_misc_suffix {}'.format(custom_run_misc_suffix))
+            run_misc_suffix = custom_run_misc_suffix
+
 
         # special case for aprun
         if executable is not None and "aprun" in executable and not "theta" in self.get_value("MACH"):
@@ -1393,7 +1399,7 @@ def get_mpirun_cmd(self, job=None, allow_unresolved_envvars=True):
         if self.get_value("BATCH_SYSTEM") == "cobalt":
             mpi_arg_string += " : "
 
-        return self.get_resolved_value("{} {} {}".format(executable if executable is not None else "", mpi_arg_string, run_suffix), allow_unresolved_envvars=allow_unresolved_envvars)
+        return self.get_resolved_value("{} {} {} {}".format(executable if executable is not None else "", mpi_arg_string, run_exe, run_misc_suffix), allow_unresolved_envvars=allow_unresolved_envvars)
 
     def set_model_version(self, model):
         version = "unknown"
diff --git a/cime/scripts/lib/CIME/case/case_submit.py b/cime/scripts/lib/CIME/case/case_submit.py
index d31db5362363..aea7141b7285 100644
--- a/cime/scripts/lib/CIME/case/case_submit.py
+++ b/cime/scripts/lib/CIME/case/case_submit.py
@@ -177,7 +177,7 @@ def submit(self, job=None, no_batch=False, prereq=None, allow_fail=False, resubm
     # any submit options used on the original submit and use them again
     submit_options = os.path.join(caseroot, ".submit_options")
     if resubmit and os.path.exists(submit_options):
-        config = configparser.SafeConfigParser()
+        config = configparser.RawConfigParser()
         config.read(submit_options)
         if not skip_pnl and config.has_option('SubmitOptions','skip_pnl'):
             skip_pnl = config.getboolean('SubmitOptions', 'skip_pnl')
diff --git a/cime/scripts/lib/e3sm_cime_mgmt.py b/cime/scripts/lib/e3sm_cime_mgmt.py
index ed8b5007285f..b3325648d663 100644
--- a/cime/scripts/lib/e3sm_cime_mgmt.py
+++ b/cime/scripts/lib/e3sm_cime_mgmt.py
@@ -19,8 +19,11 @@ def setup():
     if ESMCI_REMOTE_NAME not in remotes:
         run_cmd_no_fail("git remote add {} {}".format(ESMCI_REMOTE_NAME, ESMCI_URL), verbose=True)
 
-    run_cmd_no_fail("git fetch --prune {}".format(ESMCI_REMOTE_NAME), verbose=True)
-    run_cmd_no_fail("git fetch --prune {} --tags".format(ESMCI_REMOTE_NAME), verbose=True)
+    for origin in ["origin", ESMCI_REMOTE_NAME]:
+        run_cmd_no_fail("git fetch --prune {}".format(origin), verbose=True)
+        run_cmd_no_fail("git fetch --prune {} --tags".format(origin), verbose=True)
+
+    run_cmd_no_fail("git clean -fd", verbose=True)
 
     run_cmd_no_fail("git clean -fd", verbose=True)