Merge pull request #2475 from ESMCI/jgfouca/safer_copy

Transition CIME to using a custom util for copying files.
The file-copy operation is very common in CIME and we were using
a smattering of shutil.copyfile, shutil.copy, and shutil.copy2.
This PR replaces these with safe_copy, a flexble and fault-tolerant
copying function. This should make it much easier for users to share
case directories and other common directories.

Test suite: scripts_regression_tests
Test baseline:
Test namelist changes:
Test status: bit for bit

Fixes [CIME Github issue #]

User interface changes?: N

Update gh-pages html (Y/N)?: n

Code review: @jedwards4b

scripts/lib/CIME/BuildTools/configure.py

@@ -15,9 +15,8 @@
 COMPILER, MPILIB, and DEBUG, respectively.
 """
 
-import shutil
 from CIME.XML.standard_module_setup import *
-from CIME.utils import expect
+from CIME.utils import expect, safe_copy
 from CIME.XML.compilers import Compilers
 from CIME.XML.env_mach_specific import EnvMachSpecific
 
@@ -58,13 +57,13 @@ def _copy_depends_files(machine_name, machines_dir, output_dir, compiler):
     outputdfile = os.path.join(output_dir, "Depends.{}.{}".format(machine_name,compiler))
     if os.path.isfile(dfile):
         if not os.path.isfile(outputdfile):
-            shutil.copyfile(dfile, outputdfile)
+            safe_copy(dfile, outputdfile)
     else:
         for dep in (machine_name, compiler):
             dfile = os.path.join(machines_dir, "Depends.{}".format(dep))
             outputdfile = os.path.join(output_dir, "Depends.{}".format(dep))
             if os.path.isfile(dfile) and not os.path.isfile(outputdfile):
-                shutil.copyfile(dfile, outputdfile)
+                safe_copy(dfile, outputdfile)
 
 class FakeCase(object):
 

scripts/lib/CIME/SystemTests/eri.py

@@ -2,6 +2,7 @@
 CIME ERI test  This class inherits from SystemTestsCommon
 """
 from CIME.XML.standard_module_setup import *
+from CIME.utils import safe_copy
 from CIME.SystemTests.system_tests_common import SystemTestsCommon
 import shutil, glob, os
 
@@ -17,7 +18,7 @@ def _helper(dout_sr, refdate, refsec, rundir):
         os.symlink(item, dst)
 
     for item in glob.glob("{}/*rpointer*".format(rest_path)):
-        shutil.copy(item, rundir)
+        safe_copy(item, rundir)
 
 class ERI(SystemTestsCommon):
 

scripts/lib/CIME/SystemTests/err.py

@@ -2,10 +2,10 @@
 CIME ERR test  This class inherits from ERS
 ERR tests short term archiving and restart capabilities
 """
-import glob, os, shutil
+import glob, os
 from CIME.XML.standard_module_setup import *
 from CIME.SystemTests.restart_tests import RestartTest
-from CIME.utils import ls_sorted_by_mtime
+from CIME.utils import ls_sorted_by_mtime, safe_copy
 
 logger = logging.getLogger(__name__)
 
@@ -46,4 +46,4 @@ def _case_two_custom_postrun_action(self):
                                                  "*.nc.{}".format(self._run_one_suffix))):
             orig_file = case_file[:-(1+len(self._run_one_suffix))]
             if not os.path.isfile(orig_file):
-                shutil.copyfile(case_file, orig_file)
+                safe_copy(case_file, orig_file)

scripts/lib/CIME/SystemTests/system_tests_common.py

@@ -3,14 +3,14 @@
 """
 from CIME.XML.standard_module_setup import *
 from CIME.XML.env_run import EnvRun
-from CIME.utils import append_testlog, get_model
+from CIME.utils import append_testlog, get_model, safe_copy
 from CIME.test_status import *
 from CIME.hist_utils import *
 from CIME.provenance import save_test_time
 from CIME.locked_files import LOCKED_DIR, lock_file, is_locked
 import CIME.build as build
 
-import shutil, glob, gzip, time, traceback, six
+import glob, gzip, time, traceback, six
 
 logger = logging.getLogger(__name__)
 
@@ -453,8 +453,8 @@ def _generate_baseline(self):
                 m = re.search(r"/(cpl.*.log).*.gz",cpllog)
                 if m is not None:
                     baselog = os.path.join(basegen_dir, m.group(1))+".gz"
-                    shutil.copyfile(cpllog,
-                                    os.path.join(basegen_dir,baselog))
+                    safe_copy(cpllog,
+                              os.path.join(basegen_dir,baselog))
 
 class FakeTest(SystemTestsCommon):
     """

scripts/lib/CIME/XML/generic_xml.py

@@ -3,10 +3,12 @@
 be used by other XML interface modules and not directly.
 """
 from CIME.XML.standard_module_setup import *
+from CIME.utils import safe_copy
+
 import xml.etree.ElementTree as ET
 #pylint: disable=import-error
 from distutils.spawn import find_executable
-import getpass, shutil
+import getpass
 import six
 from copy import deepcopy
 
@@ -121,7 +123,7 @@ def change_file(self, newfile, copy=False):
             new_case = os.path.dirname(newfile)
             if not os.path.exists(new_case):
                 os.makedirs(new_case)
-            shutil.copy(self.filename, newfile)
+            safe_copy(self.filename, newfile)
 
         self.tree = None
         self.filename = newfile

scripts/lib/CIME/build.py

@@ -3,7 +3,7 @@
 """
 import glob, shutil, time, threading, subprocess
 from CIME.XML.standard_module_setup  import *
-from CIME.utils                 import get_model, analyze_build_log, stringify_bool, run_and_log_case_status, get_timestamp, run_sub_or_cmd, run_cmd, get_batch_script_for_job, gzip_existing_file
+from CIME.utils                 import get_model, analyze_build_log, stringify_bool, run_and_log_case_status, get_timestamp, run_sub_or_cmd, run_cmd, get_batch_script_for_job, gzip_existing_file, safe_copy
 from CIME.provenance            import save_build_provenance
 from CIME.locked_files          import lock_file, unlock_file
 
@@ -94,7 +94,7 @@ def _build_model(build_threaded, exeroot, clm_config_opts, incroot, complist,
         expect(stat == 0, "BUILD FAIL: buildexe failed, cat {}".format(file_build))
 
         # Copy the just-built ${MODEL}.exe to ${MODEL}.exe.$LID
-        shutil.copy("{}/{}.exe".format(exeroot, cime_model), "{}/{}.exe.{}".format(exeroot, cime_model, lid))
+        safe_copy("{}/{}.exe".format(exeroot, cime_model), "{}/{}.exe.{}".format(exeroot, cime_model, lid))
 
         logs.append(file_build)
 
@@ -301,7 +301,7 @@ def _build_model_thread(config_dir, compclass, compname, caseroot, libroot, bldr
         thread_bad_results.append("BUILD FAIL: {}.buildlib failed, cat {}".format(compname, file_build))
 
     for mod_file in glob.glob(os.path.join(bldroot, "*_[Cc][Oo][Mm][Pp]_*.mod")):
-        shutil.copy(mod_file, incroot)
+        safe_copy(mod_file, incroot)
 
     t2 = time.time()
     logger.info("{} built in {:f} seconds".format(compname, (t2 - t1)))

scripts/lib/CIME/case/case.py

@@ -12,7 +12,7 @@
 from CIME.utils                     import expect, get_cime_root, append_status
 from CIME.utils                     import convert_to_type, get_model
 from CIME.utils                     import get_project, get_charge_account, check_name
-from CIME.utils                     import get_current_commit
+from CIME.utils                     import get_current_commit, safe_copy
 from CIME.locked_files         import LOCKED_DIR, lock_file
 from CIME.XML.machines              import Machines
 from CIME.XML.pes                   import Pes
@@ -1057,9 +1057,9 @@ def _create_caseroot_tools(self):
 
         if get_model() == "e3sm":
             if os.path.exists(os.path.join(machines_dir, "syslog.{}".format(machine))):
-                shutil.copy(os.path.join(machines_dir, "syslog.{}".format(machine)), os.path.join(casetools, "mach_syslog"))
+                safe_copy(os.path.join(machines_dir, "syslog.{}".format(machine)), os.path.join(casetools, "mach_syslog"))
             else:
-                shutil.copy(os.path.join(machines_dir, "syslog.noop"), os.path.join(casetools, "mach_syslog"))
+                safe_copy(os.path.join(machines_dir, "syslog.noop"), os.path.join(casetools, "mach_syslog"))
 
     def _create_caseroot_sourcemods(self):
         components = self.get_compset_components()

scripts/lib/CIME/case/case_clone.py

@@ -3,7 +3,7 @@
 """
 import os, glob, shutil
 from CIME.XML.standard_module_setup import *
-from CIME.utils import expect, check_name
+from CIME.utils import expect, check_name, safe_copy
 from CIME.user_mod_support import apply_user_mods
 from CIME.locked_files         import lock_file
 from CIME.simple_compare            import compare_files
@@ -108,7 +108,7 @@ def create_clone(self, newcase, keepexe=False, mach_dir=None, project=None,
     files = glob.glob(cloneroot + '/user_*')
 
     for item in files:
-        shutil.copy(item, newcaseroot)
+        safe_copy(item, newcaseroot)
 
     # copy SourceMod and Buildconf files
     # if symlinks exist, copy rather than follow links
@@ -126,8 +126,8 @@ def create_clone(self, newcase, keepexe=False, mach_dir=None, project=None,
         if keepexe:
             # If keepexe CANNOT change any env_build.xml variables - so make a temporary copy of
             # env_build.xml and verify that it has not been modified
-            shutil.copy(os.path.join(newcaseroot, "env_build.xml"),
-                        os.path.join(newcaseroot, "LockedFiles", "env_build.xml"))
+            safe_copy(os.path.join(newcaseroot, "env_build.xml"),
+                      os.path.join(newcaseroot, "LockedFiles", "env_build.xml"))
 
         # Now apply contents of user_mods directory
         apply_user_mods(newcase_root, user_mods_dir, keepexe=keepexe)

scripts/lib/CIME/case/case_cmpgen_namelists.py

@@ -7,7 +7,7 @@
 
 from CIME.compare_namelists import is_namelist_file, compare_namelist_files
 from CIME.simple_compare import compare_files
-from CIME.utils import append_status
+from CIME.utils import append_status, safe_copy
 from CIME.test_status import *
 
 import os, shutil, traceback, stat, glob
@@ -79,7 +79,7 @@ def _do_full_nl_gen(case, test, generate_name, baseline_root=None):
         if (os.path.exists(preexisting_baseline)):
             os.remove(preexisting_baseline)
 
-        shutil.copy2(item, baseline_dir)
+        safe_copy(item, baseline_dir)
         os.chmod(preexisting_baseline, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP)
 
 def case_cmpgen_namelists(self, compare=False, generate=False, compare_name=None, generate_name=None, baseline_root=None, logfile_name="TestStatus.log"):

scripts/lib/CIME/case/case_run.py

@@ -3,7 +3,7 @@
 '"""
 from CIME.XML.standard_module_setup import *
 from CIME.utils                     import gzip_existing_file, new_lid, run_and_log_case_status
-from CIME.utils                     import run_sub_or_cmd, append_status
+from CIME.utils                     import run_sub_or_cmd, append_status, safe_copy
 from CIME.get_timing                import get_timing
 from CIME.provenance                import save_prerun_provenance, save_postrun_provenance
 
@@ -28,7 +28,7 @@ def _pre_run_check(case, lid, skip_pnl=False, da_cycle=0):
 
     if case.get_value("TESTCASE") == "PFS":
         env_mach_pes = os.path.join(caseroot,"env_mach_pes.xml")
-        shutil.copy(env_mach_pes,"{}.{}".format(env_mach_pes, lid))
+        safe_copy(env_mach_pes,"{}.{}".format(env_mach_pes, lid))
 
     # check for locked files.
     case.check_lockedfiles()

scripts/lib/CIME/case/case_setup.py

@@ -7,12 +7,10 @@
 
 from CIME.XML.machines      import Machines
 from CIME.BuildTools.configure import configure
-from CIME.utils             import get_cime_root, run_and_log_case_status, get_model, get_batch_script_for_job
+from CIME.utils             import get_cime_root, run_and_log_case_status, get_model, get_batch_script_for_job, safe_copy
 from CIME.test_status       import *
 from CIME.locked_files      import unlock_file, lock_file
 
-import shutil
-
 logger = logging.getLogger(__name__)
 
 ###############################################################################
@@ -41,7 +39,7 @@ def _build_usernl_files(case, model, comp):
             expect(ninst_model==ninst_max,"MULTI_DRIVER mode, all components must have same NINST value.  NINST_{} != {}".format(model,ninst_max))
     if comp == "cpl":
         if not os.path.exists("user_nl_cpl"):
-            shutil.copy(os.path.join(model_dir, "user_nl_cpl"), ".")
+            safe_copy(os.path.join(model_dir, "user_nl_cpl"), ".")
     else:
         if ninst == 1:
             ninst = case.get_value("NINST_{}".format(model))
@@ -55,14 +53,14 @@ def _build_usernl_files(case, model, comp):
                     # to user_nl_foo_INST; otherwise, copy the original
                     # user_nl_foo from model_dir
                     if os.path.exists(nlfile):
-                        shutil.copy(nlfile, inst_nlfile)
+                        safe_copy(nlfile, inst_nlfile)
                     elif os.path.exists(model_nl):
-                        shutil.copy(model_nl, inst_nlfile)
+                        safe_copy(model_nl, inst_nlfile)
         else:
             # ninst = 1
             if not os.path.exists(nlfile):
                 if os.path.exists(model_nl):
-                    shutil.copy(model_nl, nlfile)
+                    safe_copy(model_nl, nlfile)
 
 ###############################################################################
 def _case_setup_impl(case, caseroot, clean=False, test_mode=False, reset=False):

scripts/lib/CIME/case/case_st_archive.py

@@ -7,7 +7,7 @@
 import shutil, glob, re, os
 
 from CIME.XML.standard_module_setup import *
-from CIME.utils                     import run_and_log_case_status, ls_sorted_by_mtime, symlink_force
+from CIME.utils                     import run_and_log_case_status, ls_sorted_by_mtime, symlink_force, safe_copy
 from CIME.date                      import get_file_date
 from os.path                        import isdir, join
 
@@ -19,7 +19,7 @@ def _get_archive_file_fn(copy_only):
     """
     Returns the function to use for archiving some files
     """
-    return shutil.copyfile if copy_only else shutil.move
+    return safe_copy if copy_only else shutil.move
 
 ###############################################################################
 def _get_datenames(case):
@@ -134,7 +134,7 @@ def _archive_rpointer_files(casename, ninst_strings, rundir, save_interim_restar
         # Copy of all rpointer files for latest restart date
         rpointers = glob.glob(os.path.join(rundir, 'rpointer.*'))
         for rpointer in rpointers:
-            shutil.copy(rpointer, os.path.join(archive_restdir, os.path.basename(rpointer)))
+            safe_copy(rpointer, os.path.join(archive_restdir, os.path.basename(rpointer)))
     else:
         # Generate rpointer file(s) for interim restarts for the one datename and each
         # possible value of ninst_strings
@@ -263,7 +263,7 @@ def _archive_history_files(case, archive, archive_entry,
                         destfile = join(archive_histdir, histfile)
                         if histfile in histfiles_savein_rundir:
                             logger.debug("histfiles_savein_rundir; copying \n  {} to \n  {} ".format(srcfile, destfile))
-                            shutil.copy(srcfile, destfile)
+                            safe_copy(srcfile, destfile)
                         else:
                             logger.debug("_archive_history_files; moving \n  {} to \n  {} ".format(srcfile, destfile))
                             archive_file_fn(srcfile, destfile)
@@ -379,7 +379,7 @@ def _archive_restarts_date_comp(case, archive, archive_entry,
         last_restart_file_fn = symlink_force
         last_restart_file_fn_msg = "linking"
     else:
-        last_restart_file_fn = shutil.copy
+        last_restart_file_fn = safe_copy
         last_restart_file_fn_msg = "copying"
 
     # the compname is drv but the files are named cpl
@@ -439,7 +439,7 @@ def _archive_restarts_date_comp(case, archive, archive_entry,
                         destfile = os.path.join(archive_restdir, histfile)
                         expect(os.path.isfile(srcfile),
                                "history restart file {} for last date does not exist ".format(srcfile))
-                        shutil.copy(srcfile, destfile)
+                        safe_copy(srcfile, destfile)
                         logger.debug("datename_is_last + histfiles_for_restart copying \n  {} to \n  {}".format(srcfile, destfile))
                 else:
                     # Only archive intermediate restarts if requested - otherwise remove them
@@ -458,7 +458,7 @@ def _archive_restarts_date_comp(case, archive, archive_entry,
                             destfile = os.path.join(archive_restdir, histfile)
                             expect(os.path.isfile(srcfile),
                                    "hist file {} does not exist ".format(srcfile))
-                            shutil.copy(srcfile, destfile)
+                            safe_copy(srcfile, destfile)
                             logger.debug("interim_restarts: copying \n  {} to \n  {}".format(srcfile, destfile))
                     else:
                         srcfile = os.path.join(rundir, restfile)
@@ -536,7 +536,7 @@ def restore_from_archive(self, rest_dir=None):
         if os.path.exists(dst):
             os.remove(dst)
 
-        shutil.copy(item, rundir)
+        safe_copy(item, rundir)
 
 ###############################################################################
 def archive_last_restarts(self, archive_restdir, last_date=None, link_to_restart_files=False):

scripts/lib/CIME/case/check_input_data.py

@@ -2,11 +2,11 @@
 API for checking input for testcase
 """
 from CIME.XML.standard_module_setup import *
-from CIME.utils import SharedArea, find_files
+from CIME.utils import SharedArea, find_files, safe_copy
 from CIME.XML.inputdata import Inputdata
 import CIME.Servers
 
-import glob, shutil
+import glob
 
 logger = logging.getLogger(__name__)
 
@@ -92,7 +92,7 @@ def stage_refcase(self):
         # copy the refcases' rpointer files to the run directory
         for rpointerfile in glob.iglob(os.path.join("{}","*rpointer*").format(refdir)):
             logger.info("Copy rpointer {}".format(rpointerfile))
-            shutil.copy(rpointerfile, rundir)
+            safe_copy(rpointerfile, rundir)
 
         # link everything else
 

scripts/lib/CIME/case/preview_namelists.py

@@ -4,8 +4,8 @@
 """
 
 from CIME.XML.standard_module_setup import *
-from CIME.utils import run_sub_or_cmd
-import glob, shutil
+from CIME.utils import run_sub_or_cmd, safe_copy
+import glob
 logger = logging.getLogger(__name__)
 
 def create_dirs(self):
@@ -102,9 +102,9 @@ def create_namelists(self, component=None):
                    "*streams*txt*", "*stxt", "*maps.rc", "*cism.config*"]:
         for file_to_copy in glob.glob(os.path.join(rundir, cpglob)):
             logger.debug("Copy file from '{}' to '{}'".format(file_to_copy, docdir))
-            shutil.copy2(file_to_copy, docdir)
+            safe_copy(file_to_copy, docdir)
 
     # Copy over chemistry mechanism docs if they exist
     if (os.path.isdir(os.path.join(casebuild, "camconf"))):
         for file_to_copy in glob.glob(os.path.join(casebuild, "camconf", "*chem_mech*")):
-            shutil.copy2(file_to_copy, docdir)
+            safe_copy(file_to_copy, docdir)