From 06c3dd938e69f473476377632b6c15e160b3c322 Mon Sep 17 00:00:00 2001 From: simonrp84 Date: Wed, 24 Nov 2021 14:47:22 +0000 Subject: [PATCH 1/5] Update AHI gridded reader to use HTTP instead of FTP --- satpy/readers/ahi_l1b_gridded_bin.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/satpy/readers/ahi_l1b_gridded_bin.py b/satpy/readers/ahi_l1b_gridded_bin.py index 8ccfb62526..1663bf7aaa 100644 --- a/satpy/readers/ahi_l1b_gridded_bin.py +++ b/satpy/readers/ahi_l1b_gridded_bin.py @@ -45,9 +45,7 @@ from satpy.readers.utils import unzip_file # Hardcoded address of the reflectance and BT look-up tables -AHI_REMOTE_LUTS = ['hmwr829gr.cr.chiba-u.ac.jp', - '/gridded/FD/support/', - 'count2tbb_v101.tgz'] +AHI_REMOTE_LUTS = 'http://www.cr.chiba-u.jp/databases/GEO/H8_9/FD/count2tbb_v102.tgz' # Full disk image sizes for each spatial resolution AHI_FULLDISK_SIZES = {0.005: {'x_size': 24000, @@ -110,7 +108,7 @@ def __init__(self, filename, filename_info, filetype_info): raise NotImplementedError("Only full disk data is supported.") # Set up directory path for the LUTs - app_dirs = AppDirs('ahi_gridded_luts', 'satpy', '1.0.1') + app_dirs = AppDirs('ahi_gridded_luts', 'satpy', '1.0.2') self.lut_dir = os.path.expanduser(app_dirs.user_data_dir) + '/' self.area = None @@ -143,14 +141,14 @@ def _calibrate(self, data): @staticmethod def _download_luts(file_name): - """Download LUTs from remote FTP server.""" - from ftplib import FTP - # Set up an FTP connection (anonymous) and download - ftp = FTP(AHI_REMOTE_LUTS[0]) - ftp.login('anonymous', 'anonymous') - ftp.cwd(AHI_REMOTE_LUTS[1]) - with open(file_name, 'wb') as _fp: - ftp.retrbinary("RETR " + AHI_REMOTE_LUTS[2], _fp.write) + """Download LUTs from remote server.""" + import requests + import shutil + # Set up an connection and download + response = requests.get(AHI_REMOTE_LUTS, stream=True) + with open(file_name, 'wb') as out_file: + shutil.copyfileobj(response.raw, out_file) + del response @staticmethod def _untar_luts(tarred_file, outdir): @@ -180,7 +178,7 @@ def _get_luts(self): # The file is tarred, untar and remove the downloaded file self._untar_luts(fname, tempdir) - lut_dl_dir = os.path.join(tempdir, 'count2tbb/') + lut_dl_dir = os.path.join(tempdir, 'count2tbb_v102/') # Loop over the LUTs and copy to the correct location for lutfile in AHI_LUT_NAMES: From 22700a955a2c787b4c9c479dd54b0bb0e0cd05d4 Mon Sep 17 00:00:00 2001 From: simonrp84 Date: Thu, 25 Nov 2021 11:23:32 +0000 Subject: [PATCH 2/5] Update AHI gridded reader and tests. --- satpy/readers/ahi_l1b_gridded_bin.py | 1 - .../reader_tests/test_ahi_l1b_gridded_bin.py | 21 ++++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/satpy/readers/ahi_l1b_gridded_bin.py b/satpy/readers/ahi_l1b_gridded_bin.py index 1663bf7aaa..d61b2450ed 100644 --- a/satpy/readers/ahi_l1b_gridded_bin.py +++ b/satpy/readers/ahi_l1b_gridded_bin.py @@ -148,7 +148,6 @@ def _download_luts(file_name): response = requests.get(AHI_REMOTE_LUTS, stream=True) with open(file_name, 'wb') as out_file: shutil.copyfileobj(response.raw, out_file) - del response @staticmethod def _untar_luts(tarred_file, outdir): diff --git a/satpy/tests/reader_tests/test_ahi_l1b_gridded_bin.py b/satpy/tests/reader_tests/test_ahi_l1b_gridded_bin.py index ff5f89a038..7a54a83f27 100644 --- a/satpy/tests/reader_tests/test_ahi_l1b_gridded_bin.py +++ b/satpy/tests/reader_tests/test_ahi_l1b_gridded_bin.py @@ -155,8 +155,6 @@ def new_unzip(fname): """Fake unzipping.""" if fname[-3:] == 'bz2': return fname[:-4] - else: - return fname @mock.patch('satpy.readers.ahi_l1b_gridded_bin.unzip_file', mock.MagicMock(side_effect=new_unzip)) @@ -209,6 +207,13 @@ def test_get_dataset(self, mocked_read): self.assertEqual(res.attrs['name'], self.key['name']) self.assertEqual(res.attrs['wavelength'], self.info['wavelength']) + @mock.patch('os.path.exists', return_value=True) + @mock.patch('os.remove') + def test_destructor(self, exist_patch, remove_patch): + """Check that file handler deletes files if needed.""" + del self.fh + remove_patch.assert_called() + class TestAHIGriddedLUTs(unittest.TestCase): """Test case for the downloading and preparing LUTs.""" @@ -224,7 +229,7 @@ def mocked_ftp_dl(fname): tmpf = os.path.join(tempfile.tempdir, namer) with open(tmpf, 'w') as tmp_fid: tmp_fid.write("TEST\n") - tar_handle.add(tmpf, arcname='count2tbb/'+namer) + tar_handle.add(tmpf, arcname='count2tbb_v102/'+namer) os.remove(tmpf) def setUp(self): @@ -262,14 +267,16 @@ def test_get_luts(self): tempdir = tempfile.gettempdir() print(self.fh.lut_dir) self.fh._get_luts() - self.assertFalse(os.path.exists(os.path.join(tempdir, 'count2tbb/'))) + self.assertFalse(os.path.exists(os.path.join(tempdir, 'count2tbb_v102/'))) for lut_name in AHI_LUT_NAMES: self.assertTrue(os.path.isfile(os.path.join(self.fh.lut_dir, lut_name))) - @mock.patch('ftplib.FTP') - def test_download_luts(self, mock_ftp): + @mock.patch('requests.get') + @mock.patch('shutil.copyfileobj') + def test_download_luts(self, mock_requests, mock_shutil): """Test that the FTP library is called for downloading LUTS.""" m = mock.mock_open() with mock.patch('satpy.readers.ahi_l1b_gridded_bin.open', m, create=True): self.fh._download_luts('/test_file') - mock_ftp.assert_called() + mock_requests.assert_called() + mock_shutil.assert_called() From e144f71f9e4a456b36456fd66dfa82a359c4ef87 Mon Sep 17 00:00:00 2001 From: simonrp84 Date: Thu, 25 Nov 2021 11:25:30 +0000 Subject: [PATCH 3/5] Remove some unnecessary code in the AHI gridded reader. --- satpy/readers/ahi_l1b_gridded_bin.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/satpy/readers/ahi_l1b_gridded_bin.py b/satpy/readers/ahi_l1b_gridded_bin.py index d61b2450ed..b1a19e63de 100644 --- a/satpy/readers/ahi_l1b_gridded_bin.py +++ b/satpy/readers/ahi_l1b_gridded_bin.py @@ -246,12 +246,9 @@ def calibrate(self, data, calib): """Calibrate the data.""" if calib == 'counts': return data - elif calib == 'reflectance' or calib == 'brightness_temperature': - data = self._calibrate(data) - else: - raise NotImplementedError("ERROR: Unsupported calibration.", - "Only counts, reflectance and ", - "brightness_temperature calibration", - "are supported.") - - return data + if calib == 'reflectance' or calib == 'brightness_temperature': + return self._calibrate(data) + raise NotImplementedError("ERROR: Unsupported calibration.", + "Only counts, reflectance and ", + "brightness_temperature calibration", + "are supported.") From 07b8fd7595c207bce71e21668b4c2808b9afe26b Mon Sep 17 00:00:00 2001 From: simonrp84 Date: Thu, 25 Nov 2021 12:07:20 +0000 Subject: [PATCH 4/5] Update AHI gridded reader to use `urllib` instead of `requests`. --- satpy/readers/ahi_l1b_gridded_bin.py | 8 ++++---- satpy/tests/reader_tests/test_ahi_l1b_gridded_bin.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/satpy/readers/ahi_l1b_gridded_bin.py b/satpy/readers/ahi_l1b_gridded_bin.py index b1a19e63de..7b4d225505 100644 --- a/satpy/readers/ahi_l1b_gridded_bin.py +++ b/satpy/readers/ahi_l1b_gridded_bin.py @@ -142,12 +142,12 @@ def _calibrate(self, data): @staticmethod def _download_luts(file_name): """Download LUTs from remote server.""" - import requests + import urllib import shutil # Set up an connection and download - response = requests.get(AHI_REMOTE_LUTS, stream=True) - with open(file_name, 'wb') as out_file: - shutil.copyfileobj(response.raw, out_file) + with urllib.request.urlopen(AHI_REMOTE_LUTS) as response: + with open(file_name, 'wb') as out_file: + shutil.copyfileobj(response, out_file) @staticmethod def _untar_luts(tarred_file, outdir): diff --git a/satpy/tests/reader_tests/test_ahi_l1b_gridded_bin.py b/satpy/tests/reader_tests/test_ahi_l1b_gridded_bin.py index 7a54a83f27..09fb065b06 100644 --- a/satpy/tests/reader_tests/test_ahi_l1b_gridded_bin.py +++ b/satpy/tests/reader_tests/test_ahi_l1b_gridded_bin.py @@ -271,12 +271,12 @@ def test_get_luts(self): for lut_name in AHI_LUT_NAMES: self.assertTrue(os.path.isfile(os.path.join(self.fh.lut_dir, lut_name))) - @mock.patch('requests.get') + @mock.patch('urllib.request.urlopen') @mock.patch('shutil.copyfileobj') - def test_download_luts(self, mock_requests, mock_shutil): + def test_download_luts(self, mock_dl, mock_shutil): """Test that the FTP library is called for downloading LUTS.""" m = mock.mock_open() with mock.patch('satpy.readers.ahi_l1b_gridded_bin.open', m, create=True): self.fh._download_luts('/test_file') - mock_requests.assert_called() + mock_dl.assert_called() mock_shutil.assert_called() From bb4b2feacecdce480c43e33678518c7ad0e2be55 Mon Sep 17 00:00:00 2001 From: Simon Proud Date: Thu, 25 Nov 2021 12:39:17 +0000 Subject: [PATCH 5/5] Skip security check for ahi gridded reader Co-authored-by: Martin Raspaud --- satpy/readers/ahi_l1b_gridded_bin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/readers/ahi_l1b_gridded_bin.py b/satpy/readers/ahi_l1b_gridded_bin.py index 7b4d225505..76a8134331 100644 --- a/satpy/readers/ahi_l1b_gridded_bin.py +++ b/satpy/readers/ahi_l1b_gridded_bin.py @@ -145,7 +145,7 @@ def _download_luts(file_name): import urllib import shutil # Set up an connection and download - with urllib.request.urlopen(AHI_REMOTE_LUTS) as response: + with urllib.request.urlopen(AHI_REMOTE_LUTS) as response: # nosec with open(file_name, 'wb') as out_file: shutil.copyfileobj(response, out_file)