Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add MiRS reader #1511

Merged
merged 33 commits into from
Mar 9, 2021
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
1bd35fb
Add files for MiRS reader takes over for #1285 PR and replaces #1486 PR
joleenf Jan 22, 2021
0b102c4
Remove unused lines
joleenf Jan 27, 2021
b5613fd
Merge branch 'master' of https://github.com/pytroll/satpy into mirs
joleenf Jan 27, 2021
2c7b733
Change name of mirs_l2_nc to mirs
joleenf Feb 3, 2021
59c32b3
Merge branch 'master' of https://github.com/pytroll/satpy into mirs
joleenf Feb 3, 2021
95852a9
Change name of mirs_l2_nc to mirs within code and yaml
joleenf Feb 3, 2021
a0386b6
Merge branch 'master' of https://github.com/pytroll/satpy into mirs
joleenf Feb 16, 2021
7064d39
Remove code that is not used
joleenf Feb 17, 2021
29576d9
Add code to support zenodo data download with pooch
joleenf Feb 17, 2021
188f609
Make counter names more unique and descriptive
joleenf Feb 17, 2021
9c6b0d1
Add known_hash to zenodo files.
joleenf Feb 19, 2021
5168e6f
Changes requested for retrieve of coeffs and application
joleenf Feb 19, 2021
aea1e9e
yield BT metadata in it's own generator.
joleenf Feb 19, 2021
935b518
Remove unused variables and fix some lines DeepCode doesn't like
joleenf Feb 19, 2021
45ce5f3
Move limb_correct_bt
joleenf Feb 20, 2021
7fa7143
BUG FIX (filename not file, but don't need it anyway)
joleenf Feb 21, 2021
346926c
Update map_blocks call without assigning to a numpy array
joleenf Feb 21, 2021
f526e99
Omit extra line and just return da.stack
joleenf Feb 21, 2021
40c0542
Remove else return statement
joleenf Feb 21, 2021
874dd7f
Merge upstream commits
joleenf Feb 21, 2021
0e871d4
BUG FIX: function that map_blocks calls should receive and return num…
joleenf Feb 23, 2021
ac4bccc
Merge branch 'master' of https://github.com/pytroll/satpy into mirs
joleenf Feb 23, 2021
f710028
Be more direct in assigning dims, and name to new corrected btemp arr…
joleenf Feb 24, 2021
0cb3280
Remove dask calls in function that map_blocks calls and reorder code
joleenf Feb 25, 2021
3e5bb3e
Remove xarray coord assignment for corrected bt_data
joleenf Feb 26, 2021
2c6fdad
Remove assignment of coordinates through xarray coord attributes
joleenf Feb 27, 2021
7f68d6c
Add Scanline/FOV dims to RR
joleenf Feb 27, 2021
186f53d
Fix computation of array being sent to through map_blocks
joleenf Mar 3, 2021
0de4ad2
Merge branch 'master' of https://github.com/pytroll/satpy into mirs
joleenf Mar 3, 2021
0b334af
Remove information that is not being processed
joleenf Mar 3, 2021
290fcdc
Fixes to lat/lon dims and data
joleenf Mar 3, 2021
fad34f7
Add mocking for download of data and reading of coefficients
joleenf Mar 5, 2021
e2bdd8b
Remove extra code that only adds one common line to the xarray.Datasets
joleenf Mar 6, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions satpy/etc/readers/mirs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,11 @@ reader:
long_name: MiRS Level 2 Swath Product Reader (NetCDF4)
reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader
sensors: [amsu, amsu-mhs, atms, ssmis, gmi]
data_files:
- url: "https://zenodo.org/record/4472664/files/limbcoef_atmsland_noaa20.txt"
djhoese marked this conversation as resolved.
Show resolved Hide resolved
- url: "https://zenodo.org/record/4472664/files/limbcoef_atmsland_snpp.txt"
- url: "https://zenodo.org/record/4472664/files/limbcoef_atmssea_noaa20.txt"
- url: "https://zenodo.org/record/4472664/files/limbcoef_atmssea_snpp.txt"

file_types:
mirs_atms:
Expand Down
70 changes: 41 additions & 29 deletions satpy/readers/mirs.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,16 +18,20 @@
"""Interface to MiRS product."""

import os
import re
import logging
import datetime
import appdirs
import numpy as np
import xarray as xr
import dask.array as da
from satpy import CHUNK_SIZE
from satpy.readers.file_handlers import BaseFileHandler
from satpy.aux_download import find_registerable_files, retrieve

LOG = logging.getLogger(__name__)
logging.basicConfig(level=logging.DEBUG)
SATPY_DATA_DIR = appdirs.user_data_dir("satpy")
joleenf marked this conversation as resolved.
Show resolved Hide resolved

try:
# try getting setuptools/distribute's version of resource retrieval first
Expand Down Expand Up @@ -189,9 +193,7 @@ def __init__(self, filename, filename_info, filetype_info):

self.platform_name = self._get_platform_name
self.sensor = self._get_sensor
self.lons = None
self.lats = None
self.coords = {}
self.coeffs = self.coeff_filenames

def new_coords(self):
"""Define coordinates when file does not use variable attributes."""
Expand Down Expand Up @@ -256,6 +258,28 @@ def end_time(self):
self.filename_info["end_time"] = end_time
return self.filename_info["end_time"]

@property
def coeff_filenames(self):
"""Retrieve necessary files for coefficients if needed."""
found_files = find_registerable_files(readers=['mirs'], writers=[],
composite_sensors=[])
if self.platform_name == "noaa-20":
platform_files = list(filter(lambda x: re.findall('noaa20', x),
found_files))
if self.platform_name == 'npp':
platform_files = list(filter(lambda x: re.findall('snpp', x),
found_files))
coeff_fn = {'sea': None, 'land': None}

for filename in platform_files:
if "atmssea" in filename:
coeff_fn['sea'] = retrieve(filename)
elif "atmsland" in filename:
coeff_fn['land'] = retrieve(filename)
else:
LOG.warning('Unknown coefficient filename {}', filename)
return coeff_fn
joleenf marked this conversation as resolved.
Show resolved Hide resolved

def force_date(self, key):
"""Force datetime.date for combine."""
if isinstance(self.filename_info[key], datetime.datetime):
Expand Down Expand Up @@ -285,10 +309,10 @@ def limb_correct_atms_bt(self, bt_data, ds_info):
surf_type_name = deps[1]
surf_type_mask = self[surf_type_name]

sea = read_atms_limb_correction_coeffs(LIMB_SEA_FILE)
sea = read_atms_limb_correction_coeffs(self.coeffs['sea'])
sea_bt = apply_atms_limb_correction(bt_data, idx, *sea)

land = read_atms_limb_correction_coeffs(LIMB_LAND_FILE)
land = read_atms_limb_correction_coeffs(self.coeffs['land'])
land_bt = apply_atms_limb_correction(bt_data, idx, *land)

LOG.info("Finishing limb correction")
Expand Down Expand Up @@ -359,13 +383,7 @@ def get_dataset(self, ds_id, ds_info):
LOG.debug('Calc {} {}'.format(idx, ds_id))
data = data.rename(new_name_or_name_dict=ds_info["name"])

# only correct for 'BT' data
if 'BT' not in ds_info['dependencies']:
do_not_apply = True
else:
do_not_apply = True # TODO: Change this to false when coeff loc is resolved.

if self.sensor.lower() != "atms" or do_not_apply:
if self.sensor.lower() != "atms":
LOG.info("Limb Correction will not be applied to non-ATMS BTs")
data = data[:, :, idx]
else:
Expand Down Expand Up @@ -402,19 +420,22 @@ def _available_new_datasets(self):
freq = self.nc.coords.get('Freq', self.nc.get('Freq'))
polo = self.nc['Polo']
from collections import Counter
c = Counter()
# count times a channel/polarization pair occur
chn_total = Counter()
normals = []
for idx, (f, p) in enumerate(zip(freq, polo)):
normal_f = str(int(f))
normal_p = 'v' if p == POLO_V else 'h'
c[normal_f + normal_p] += 1
chn_total[normal_f + normal_p] += 1
normals.append((idx, f, p, normal_f, normal_p))

c2 = Counter()
# keep track of current channel count for string description
chn_cnt = Counter()
for idx, _f, _p, normal_f, normal_p in normals:
c2[normal_f + normal_p] += 1
p_count = str(c2[normal_f + normal_p]
if c[normal_f + normal_p] > 1 else '')
chn_cnt[normal_f + normal_p] += 1
p_count = str(chn_cnt[normal_f + normal_p]
if chn_total[normal_f + normal_p] > 1
else '')

new_name = "btemp_{}{}{}".format(normal_f,
normal_p,
Expand Down Expand Up @@ -486,17 +507,8 @@ def __getitem__(self, item):
data, attrs = self._scale_data(data, attrs)
data, attrs = self._fill_data(data, attrs)

# handle coordinates (and recursive fun)
new_coords = {}
# 'time' dimension causes issues in other processing
# 'Freq' dimension causes issues in other processing
if 'Freq' in data.coords:
data = data.drop_vars('Freq')
if item in data.coords:
self.coords[item] = data
for coord_name in data.coords.keys():
if coord_name not in self.coords:
self.coords[coord_name] = self[coord_name]
new_coords[coord_name] = self.coords[coord_name]

data.coords.update(new_coords)

return data