Merge branch 'jgfouca/cime/tar_archived_tests' into master (PR #3080)

Tar old test results

This is cleaner and will prevent us from hitting inode limits
on NERSC.

[BFB]

* jgfouca/cime/tar_archived_tests:
  Tar old test results

cime/scripts/lib/jenkins_generic_job.py

@@ -2,7 +2,7 @@
 from CIME.utils import expect, run_cmd_no_fail
 from CIME.case import Case
 
-import os, shutil, glob, signal, logging, threading, sys, re
+import os, shutil, glob, signal, logging, threading, sys, re, tarfile
 
 ##############################################################################
 def cleanup_queue(test_root, test_id):
@@ -79,7 +79,16 @@ def archive_old_test_data(machine, mach_comp, test_id_root, scratch_root, test_r
                     if not os.path.exists(os.path.join(old_test_archive, target_area)):
                         os.mkdir(os.path.join(old_test_archive, target_area))
 
-                    os.rename(the_dir, os.path.join(old_test_archive, target_area, os.path.basename(old_case)))
+                    old_case_name = os.path.basename(old_case)
+                    with tarfile.open(os.path.join(old_test_archive, target_area, "{}.tar.gz".format(old_case_name)), "w:gz") as tfd:
+                        tfd.add(the_dir, arcname=old_case_name)
+
+                    shutil.rmtree(the_dir)
+
+                    # Remove parent dir if it's empty
+                    parent_dir = os.path.dirname(the_dir)
+                    if not os.listdir(parent_dir) or os.listdir(parent_dir) == ["case2_output_root"]:
+                        shutil.rmtree(parent_dir)
 
     # Check size of archive
     bytes_of_old_test_data = int(run_cmd_no_fail("du -sb {}".format(old_test_archive)).split()[0])
@@ -92,14 +101,17 @@ def archive_old_test_data(machine, mach_comp, test_id_root, scratch_root, test_r
             for item in ["old_cases", "old_builds", "old_runs", "old_archives"]:
                 for dir_to_rm in glob.glob("{}/{}/*{}*{}*".format(old_test_archive, item, mach_comp, old_test_id)):
                     logging.info("TEST ARCHIVER:     Removing {}".format(dir_to_rm))
-                    shutil.rmtree(dir_to_rm)
+                    if (os.path.isdir(dir_to_rm)):
+                        shutil.rmtree(dir_to_rm)
+                    else:
+                        os.remove(dir_to_rm)
 
             bytes_of_old_test_data = int(run_cmd_no_fail("du -sb {}".format(old_test_archive)).split()[0])
             if bytes_of_old_test_data < bytes_allowed:
                 break
 
     else:
-        logging.info("TEST ARCHIVER: Test data is with accepted bounds, {}GB (actual) < {}GB (limit)".format(bytes_of_old_test_data / 1000000000, bytes_allowed / 1000000000))
+        logging.info("TEST ARCHIVER: Test data is within accepted bounds, {}GB (actual) < {}GB (limit)".format(bytes_of_old_test_data / 1000000000, bytes_allowed / 1000000000))
 
 ###############################################################################
 def handle_old_test_data(machine, compiler, test_id_root, scratch_root, test_root, avoid_test_id):