Merge pull request #1897 from simonrp84/ahi_grid_fix

Update AHI gridded reader to use HTTP instead of FTP

satpy/readers/ahi_l1b_gridded_bin.py

@@ -45,9 +45,7 @@
 from satpy.readers.utils import unzip_file
 
 # Hardcoded address of the reflectance and BT look-up tables
-AHI_REMOTE_LUTS = ['hmwr829gr.cr.chiba-u.ac.jp',
-                   '/gridded/FD/support/',
-                   'count2tbb_v101.tgz']
+AHI_REMOTE_LUTS = 'http://www.cr.chiba-u.jp/databases/GEO/H8_9/FD/count2tbb_v102.tgz'
 
 # Full disk image sizes for each spatial resolution
 AHI_FULLDISK_SIZES = {0.005: {'x_size': 24000,
@@ -110,7 +108,7 @@ def __init__(self, filename, filename_info, filetype_info):
             raise NotImplementedError("Only full disk data is supported.")
 
         # Set up directory path for the LUTs
-        app_dirs = AppDirs('ahi_gridded_luts', 'satpy', '1.0.1')
+        app_dirs = AppDirs('ahi_gridded_luts', 'satpy', '1.0.2')
         self.lut_dir = os.path.expanduser(app_dirs.user_data_dir) + '/'
         self.area = None
 
@@ -143,14 +141,13 @@ def _calibrate(self, data):
 
     @staticmethod
     def _download_luts(file_name):
-        """Download LUTs from remote FTP server."""
-        from ftplib import FTP
-        # Set up an FTP connection (anonymous) and download
-        ftp = FTP(AHI_REMOTE_LUTS[0])
-        ftp.login('anonymous', 'anonymous')
-        ftp.cwd(AHI_REMOTE_LUTS[1])
-        with open(file_name, 'wb') as _fp:
-            ftp.retrbinary("RETR " + AHI_REMOTE_LUTS[2], _fp.write)
+        """Download LUTs from remote server."""
+        import urllib
+        import shutil
+        # Set up an connection and download
+        with urllib.request.urlopen(AHI_REMOTE_LUTS) as response:  # nosec
+            with open(file_name, 'wb') as out_file:
+                shutil.copyfileobj(response, out_file)
 
     @staticmethod
     def _untar_luts(tarred_file, outdir):
@@ -180,7 +177,7 @@ def _get_luts(self):
         # The file is tarred, untar and remove the downloaded file
         self._untar_luts(fname, tempdir)
 
-        lut_dl_dir = os.path.join(tempdir, 'count2tbb/')
+        lut_dl_dir = os.path.join(tempdir, 'count2tbb_v102/')
 
         # Loop over the LUTs and copy to the correct location
         for lutfile in AHI_LUT_NAMES:
@@ -249,12 +246,9 @@ def calibrate(self, data, calib):
         """Calibrate the data."""
         if calib == 'counts':
             return data
-        elif calib == 'reflectance' or calib == 'brightness_temperature':
-            data = self._calibrate(data)
-        else:
-            raise NotImplementedError("ERROR: Unsupported calibration.",
-                                      "Only counts, reflectance and ",
-                                      "brightness_temperature calibration",
-                                      "are supported.")
-
-        return data
+        if calib == 'reflectance' or calib == 'brightness_temperature':
+            return self._calibrate(data)
+        raise NotImplementedError("ERROR: Unsupported calibration.",
+                                  "Only counts, reflectance and ",
+                                  "brightness_temperature calibration",
+                                  "are supported.")

satpy/tests/reader_tests/test_ahi_l1b_gridded_bin.py

@@ -155,8 +155,6 @@ def new_unzip(fname):
         """Fake unzipping."""
         if fname[-3:] == 'bz2':
             return fname[:-4]
-        else:
-            return fname
 
     @mock.patch('satpy.readers.ahi_l1b_gridded_bin.unzip_file',
                 mock.MagicMock(side_effect=new_unzip))
@@ -209,6 +207,13 @@ def test_get_dataset(self, mocked_read):
             self.assertEqual(res.attrs['name'], self.key['name'])
             self.assertEqual(res.attrs['wavelength'], self.info['wavelength'])
 
+    @mock.patch('os.path.exists', return_value=True)
+    @mock.patch('os.remove')
+    def test_destructor(self, exist_patch, remove_patch):
+        """Check that file handler deletes files if needed."""
+        del self.fh
+        remove_patch.assert_called()
+
 
 class TestAHIGriddedLUTs(unittest.TestCase):
     """Test case for the downloading and preparing LUTs."""
@@ -224,7 +229,7 @@ def mocked_ftp_dl(fname):
                 tmpf = os.path.join(tempfile.tempdir, namer)
                 with open(tmpf, 'w') as tmp_fid:
                     tmp_fid.write("TEST\n")
-                tar_handle.add(tmpf, arcname='count2tbb/'+namer)
+                tar_handle.add(tmpf, arcname='count2tbb_v102/'+namer)
                 os.remove(tmpf)
 
     def setUp(self):
@@ -262,14 +267,16 @@ def test_get_luts(self):
         tempdir = tempfile.gettempdir()
         print(self.fh.lut_dir)
         self.fh._get_luts()
-        self.assertFalse(os.path.exists(os.path.join(tempdir, 'count2tbb/')))
+        self.assertFalse(os.path.exists(os.path.join(tempdir, 'count2tbb_v102/')))
         for lut_name in AHI_LUT_NAMES:
             self.assertTrue(os.path.isfile(os.path.join(self.fh.lut_dir, lut_name)))
 
-    @mock.patch('ftplib.FTP')
-    def test_download_luts(self, mock_ftp):
+    @mock.patch('urllib.request.urlopen')
+    @mock.patch('shutil.copyfileobj')
+    def test_download_luts(self, mock_dl, mock_shutil):
         """Test that the FTP library is called for downloading LUTS."""
         m = mock.mock_open()
         with mock.patch('satpy.readers.ahi_l1b_gridded_bin.open', m, create=True):
             self.fh._download_luts('/test_file')
-            mock_ftp.assert_called()
+            mock_dl.assert_called()
+            mock_shutil.assert_called()