diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 8a491ec2cb..ed84036d73 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.7.6 +current_version = 0.7.7 commit = True tag = True diff --git a/changelog.rst b/changelog.rst index 0ace200671..24a3eb19e1 100644 --- a/changelog.rst +++ b/changelog.rst @@ -2,6 +2,59 @@ Changelog ========= +v0.7.7 (2017-12-21) +------------------- +- Update changelog. [davidh-ssec] +- Bump version: 0.7.6 → 0.7.7. [davidh-ssec] +- Merge pull request #140 from pytroll/bugfix-scmi-signed. [David Hoese] + + Bugfix scmi signed integer data variables +- Add ipython tab completion for scene keys. [davidh-ssec] +- Fix SCMI writer because AWIPS doesn't like unsigned integers. [davidh- + ssec] + + Using the entire 16-bit unsigned integer space displays fine in AWIPS + but it doesn't handle them correctly when adding derived parameters. + Meaning once the data goes in to a python script and gets converted to + a signed interger...yeah. This change makes it so data is a signed + 16-bit integer that only uses the positive half of the bit space. + +- Merge pull request #138 from pytroll/bugfix-modis-reader. [David + Hoese] +- Add more file patterns to hdfeos_l1b reader. [davidh-ssec] +- Fix requesting a specific resolution from a reader. [davidh-ssec] +- Merge remote-tracking branch 'origin/fix-resolution' into bugfix- + modis-reader. [davidh-ssec] +- Allow providing resolution when loading a composite. [Martin Raspaud] +- Fix hdfeos_l1b reader not knowing what resolution of datasets it had. + [davidh-ssec] +- Fix interpolation problem at 250m resolution. [Martin Raspaud] +- Fix readers not returning the highest resolution dataset IDs. [davidh- + ssec] +- Merge pull request #139 from pytroll/bugfix-viirs-l1b. [David Hoese] + + Fix VIIRS L1B to work with JPSS-1 and new NASA filenames +- Fix VIIRS L1B to work with JPSS-1 and new NASA filenames. [davidh- + ssec] +- Clean up style. [Martin Raspaud] +- Fix lon/lat caching in hdfeos_l1b for different resolutions. [Martin + Raspaud] + + Fixes #132 +- Merge pull request #137 from pytroll/logging_corrupted_file. [Martin + Raspaud] + + When opening/reading a nc or hdf file fails, be verbose telling which file it is that fails +- When opening/reading a file fails, be verbose telling which file it is + that fails. [Adam.Dybbroe] +- Merge pull request #134 from howff/hdfeos_l1b_ipopp_filenames. [Martin + Raspaud] + + Added IPOPP-style MODIS-L1b filenames +- Update doc re. IMAPP and IPOPP. [Andrew Brooks] +- Added IPOPP-style MODIS-L1b filenames. [Andrew Brooks] + + v0.7.6 (2017-12-19) ------------------- - Update changelog. [Martin Raspaud] @@ -366,9 +419,6 @@ Other - Fix SCMI writer for lettered grids. [davidh-ssec] - Fix numbered tile counts for SCMI writer. [davidh-ssec] - Add initial SCMI writer. [davidh-ssec] - - WIP: Multiple tiles, lettered tiles, debug images - - Separate EnhancementDecisionTree in to base DecisionTree and subclass. [davidh-ssec] - Add 'goesr' as possible platform in geocat reader. [davidh-ssec] @@ -821,6 +871,7 @@ Other Conflicts: satpy/readers/__init__.py satpy/readers/hrit_msg.py + - Fix IR and VIS calibration. [Adam.Dybbroe] - Pep8 and editorial (header) updates. [Adam.Dybbroe] - Adding the native msg header record definitions. [Adam.Dybbroe] @@ -1159,12 +1210,11 @@ v0.3.1 (2017-01-16) and make the `safe_sar_c.py` reader compute coordinate arrays from a collection of GCPs provided in the measurement files. - NB: each polarization has it's set of longitudes and latitudes. - - Restore reducers to their original values. [Martin Raspaud] - Add alternatives for true color on ahi. [Martin Raspaud] Thanks balt + - Add name to the dataset attributes when writing nc files. [Martin Raspaud] - Improve documentation. [Martin Raspaud] @@ -1192,11 +1242,13 @@ v0.3.1 (2017-01-16) - Implement a mipp-free HRIT reader. [Martin Raspaud] WIP, supports only MSG, no calibration yet. + - Concatenate area_def through making new AreaDefinition. [Martin Raspaud] This makes the concatenation independent of the AreaDefinition implementation. + - Allow stacking area_def from bottom-up. [Martin Raspaud] - Fix yaml_reader testing. [Martin Raspaud] - Add support for filetype requirements. [Martin Raspaud] @@ -1263,10 +1315,12 @@ v0.3.0 (2016-12-13) Conflicts: satpy/readers/yaml_reader.py + - Merge branch 'develop' into feature-lonlat-datasets. [Martin Raspaud] Conflicts: satpy/readers/yaml_reader.py + - Pass down the calibration, polarization and resolution from main load. [Martin Raspaud] - Fix typo in sunzenith correction description. Default is 88 deg, not @@ -1285,6 +1339,7 @@ v0.3.0 (2016-12-13) the dataset at hand. + v0.2.1 (2016-12-08) ------------------- - Update changelog. [Martin Raspaud] @@ -1385,6 +1440,7 @@ Fix Prior to h5py 3.0, the h5 files open with h5py are not closed upon deletion, so we have to do it ourselves... + - Bugfix: area.id doesn't exist, use area.area_id. [Martin Raspaud] - Bugfix: return when each file has been loaded independently. [Martin Raspaud] @@ -1620,10 +1676,12 @@ Other This was triggering a `Too many open files` error since the memmap was called for every scanline. + - Fix loading for datasets with no navigation. [Martin Raspaud] - Read start and end time from filename for eps_l1b. [Martin Raspaud] This avoids opening every file just for time checks. + - Rename file handler's get_area to get_lonlats. [davidh-ssec] There is now a get_area_def and get_lonlats method on individual file handlers @@ -1682,6 +1740,7 @@ Other - Add mipp config file for MSG3. [Martin Raspaud] This is needed by mipp when the mipp_hrit reader is used. + - Remove `if True` from viirs sharp true color. [davidh-ssec] - Fix small bug in scene when dataset isn't found in a reader. [davidh- ssec] @@ -1696,9 +1755,11 @@ Other - Put AHI HSD reflectances in % [Martin Raspaud] They were between 0 and 1 by default + - Fix AHI HSD nav dtype. [Martin Raspaud] lon ssp and lat ssp where swaped + - Adjust correct standard names for seviri calibration. [Martin Raspaud] - Fix Seviri CO2 correction buggy yaml def. [Martin Raspaud] - Fix sunz corrector with different resolutions. [davidh-ssec] @@ -1756,6 +1817,7 @@ Other etc/composites/visir.yaml satpy/composites/__init__.py satpy/scene.py + - Add support for new prerequisite syntax. [Martin Raspaud] - Got VIIRS L1B True color working. [davidh-ssec] @@ -1769,6 +1831,7 @@ Other Conflicts: etc/composites/viirs.yaml satpy/readers/yaml_reader.py + - Add viirs composites. [Martin Raspaud] - Fix the area_def concatenation. [Martin Raspaud] - Mask nan in ir calibration for ahi hsd. [Martin Raspaud] @@ -1779,9 +1842,11 @@ Other This is enabled by implementing the `get_area_def` method in the file handler. + - Optimize AHI reading using inplace loading. [Martin Raspaud] Navigation is switched off for now. + - Allow area loading for the data file handlers. [Martin Raspaud] - Use a named tuple to pass both data, mask and info dict for inplace loading. [Martin Raspaud] @@ -1827,6 +1892,7 @@ Other - Avoid raising an error when no files are found. [Martin Raspaud] Instead, a warning is logged. + - Remove unused code from readers/__init__.py. [Martin Raspaud] - Cleanup style. [Martin Raspaud] - Fix unittests. [Martin Raspaud] @@ -1848,6 +1914,7 @@ Other - Add modifiers feature. [Martin Raspaud] Now modifiers can be added to the prerequisites as dictionnaries. + - Add standard_names to channels in mipp_xrit. [Martin Raspaud] - Add a NC4/CF writer. [Martin Raspaud] - Use YAML instead of CFG for composites. [Martin Raspaud] @@ -1888,6 +1955,7 @@ Other not covered by the YAMLBasedReader. Some parts of the class being still valid in this situation, we split the class to avoid code duplication, using subclassing instead. + - Add hrpt reader. [Martin Raspaud] - Change AMSR2 L1B reader config to be 2 spaces instead of 4. [davidh- ssec] @@ -1978,6 +2046,7 @@ Other The multiscene class adds the possibility to blend different datasets together, given a blend function. + - Add a test yaml-based reader for aapp1b. [Martin Raspaud] - Fix manually added datasets not being resampled. [davidh-ssec] - Merge pull request #8 from davidh-ssec/feature-ewa-resampling. [David @@ -2017,15 +2086,18 @@ Other No matching of file was done, resulting in assigning all found files to all readers. + - Fix reader_info reference in yaml base reader. [davidh-ssec] - Keep channels in the wishlist when necessary. [Martin Raspaud] Due to the creation of a DatasetID for each dataset key, the wishlist wasn't matching the actual ids of the datasets. + - Adapt reading to yaml reader way. [Martin Raspaud] Since there is more delegating of tasks to the reader, the reading has to be adapted. + - Cleanup using pep8. [Martin Raspaud] - Allow yaml files as config files. [Martin Raspaud] - Add the dependency tree based reading. [Martin Raspaud] @@ -2036,6 +2108,7 @@ Other The multiscene class adds the possibility to blend different datasets together, given a blend function. + - Add a test yaml-based reader for aapp1b. [Martin Raspaud] - Fix netcdf dimension use to work with older versions of netcdf-python library. [davidh-ssec] @@ -2134,6 +2207,7 @@ Other In the case of true color with crefl corrected channels for example, the true color needs to depend on 3 corrected channels, which in turn can now be composites. + - Add Scene import to __init__ for convience. [davidh-ssec] - Add composites to 'available_datasets' [davidh-ssec] @@ -2180,6 +2254,7 @@ Other In order to merge or keep metadata for Dataset during arithmetic operations we need to implement the numeric type methods. + - Cleanup unused arguments in base reader. [davidh-ssec] Also makes _load_navigation by renaming it to load_navigation to resolve some quantifiedcode code checks. @@ -2224,10 +2299,12 @@ Other 3d array masks were not precomputed correctly, so we now make a workaround. A better solution is yet to be found. + - Fix kd3 precomputation for AreaDefinitions. [Martin Raspaud] The lons and lats attributes aren't defined by default in AreaDefs, so we now make sure to call the get_lonlats method. + - Set default format for dataset saving to geotiff. [Martin Raspaud] - Move `save_datasets` logic from Scene to base Writer. [davidh-ssec] - Fix bug in resample when geolocation is 2D. [davidh-ssec] @@ -2262,10 +2339,12 @@ Other the dict `keys` method return views in py3. We now convert to list for consistency. + - Add a test case for resample caching. [Martin Raspaud] - Revert resample cache changes. [Martin Raspaud] They didn't seem necessary in the way resampling is called. + - Rename to satpy. [Martin Raspaud] - Remove the world_map.ascii file. [Martin Raspaud] - Allow compressed files to be checked by hrit reader. [Martin Raspaud] @@ -2292,29 +2371,35 @@ Other We add the .bumpversion.cfg and .gitchangelog.rc for easy version bumping and changelog updates. + - Remove v from version string. [Martin Raspaud] - Add str and repr methods for composites. [Martin Raspaud] This add simple repl and str methods for compositors. + - Restructure the documentation for mpop2. [Martin Raspaud] This is an attempt to reorganize the documentation to prepare for mpop2. Old stuff has been take away, and a fresh quickstart and api are now provided. + - Improve the ReaderFinder ImportError message to include original error. [Martin Raspaud] To make the ImportError more useful in ReaderFinder, the original error string is now provided. + - Fix save_dataset to allow both empty filename and writer. [Martin Raspaud] When saving a dataset without a filename and writer, save_dataset would crash. Instead, we are now putting writer to "simple_image" in that case. + - Rename projectable when assigning it through setitem. [Martin Raspaud] When a new dataset is added to a scene, it's name should match the string key provided by the user. + - Remove references to deprecated mpop.projector. [Martin Raspaud] - Allow resample to receive strings as area identifiers. [Martin Raspaud] @@ -2324,11 +2409,13 @@ Other area by name, than to get the area definition object from the file. This patch allows the `resample` projectable method to work with string ids also. + - Add a dataset to whishlish when added with setitem. [Martin Raspaud] When adding a dataset to a scene via the datasetdict.__setitem__ method, it is likely that the user case about this dataset. As such, it should be added to the wishlist in order not to get removed accidently. + - Move composite loading out of Scene to mpop.composites. [Martin Raspaud] @@ -2337,6 +2424,7 @@ Other place to have it is the mpop.composites modules. As a conterpart, we now provide the `available_composites` method to the Scene to be able to figure out what we have possibility to generate. + - Fix the travis file to allow python 2.6 to fail. [Martin Raspaud] - Allow travis to fail on python 2.6. [Martin Raspaud] - Install importlib for travis tests on python 2.6. [Martin Raspaud] @@ -2346,28 +2434,34 @@ Other Three scenarios were added, testing showing a dataset, saving a dataset, and bulk saving datasets (`save_datasets`). + - Fix loading behaviour tests. [Martin Raspaud] A little cleanup, and using builtin functions for getting the dataset_names + - Fix DatasetDict's setitem to allow empty md in value. [Martin Raspaud] Sometimes a dataset/projectable doesn't have any info attached to it, eg because the dataset is synthetic. In these cases, setitem would crash. This is now fixed, and if a string is provided as a key in setitem it is used as a name if no better name is already there. + - Simplify dataset saving to disk. [Martin Raspaud] saving datasets can now be done one by one. If a writer is not provided, it is guessed from the filename extension. + - Add a show method to the Scene class. [Martin Raspaud] That allows the user to interactively vizualize the data + - Add a default areas.def file. [Martin Raspaud] - Fix the manifest file to include the config files. [Martin Raspaud] - Add missing config files to setup.py. [Martin Raspaud] - Fix setup.py to add cfg files. [Martin Raspaud] This is in order to make mpop work out of the box after a pip install. + - Add a behaviour test to find out the available dataset. [Martin Raspaud] - Prevent crashing when a load requirement is not available. [Martin @@ -2375,9 +2469,11 @@ Other When requiring a band which isn't available, mpop would crash. This is now fixed and replaced by a warning in the log. + - Use behave to do higher level tests. [Martin Raspaud] Two small scenarios for testing the loading of the data are implemented now. + - Fix import error in scene. [davidh-ssec] A small refactor was done and then undone to move DatasetDict and DatasetID. This little import change wasn't properly cleaned up. @@ -2399,6 +2495,7 @@ Other config changing on the fly, but also more resilience for multiple sensor cases, like one sensor is loaded after another, and the composites wouldn't get updated. + - Fix the name issue in sensor-specific composite requests. [Martin Raspaud] @@ -2406,6 +2503,7 @@ Other should be empty or None, making it not read the sensor config file at all. In turn that meant that generic composites were used instead of sensor- specific ones. + - Got metadata requests working for composites. [davidh-ssec] - Use DatasetID in composite requirements instead of names and wavelengths only. [davidh-ssec] @@ -2489,6 +2587,7 @@ Other Raspaud] The class ReaderFinder was created for this purpose. + - Cleanup. [Martin Raspaud] - Fix overview and natural composites. [Martin Raspaud] - Make read and load argument lists consistent. [Martin Raspaud] @@ -2508,6 +2607,7 @@ Other "natural", "true_color"]) BandIDs are now used internally as key for the scene's projectables dict. + - Add file keys to metop's getitem. [Martin Raspaud] - Rename metop calibration functions. [Martin Raspaud] - Add file keys for start and end times for metop. [Martin Raspaud] @@ -2581,6 +2681,7 @@ Other - we can now provide "nearest" or "kdtree" instead of a resampler class. - The precompute/dump kwarg is now a directory where to save the proj info, defaulting to '.' if precompute=True. + - Switch to containers in travis. [Martin Raspaud] - Fix repo in .travis. [Martin Raspaud] - Add OrderedDict for python < 2.7. [Martin Raspaud] @@ -2591,6 +2692,7 @@ Other - OO architecture allowing other resampling methods to be implemented. - resampling is divided between pre- and actual computation. - hashing of areas is implemented, resampler-specific. + - Fixed bad patch on new scene test. [davidh-ssec] - First try at more scene tests. [davidh-ssec] - Move image generation methods to Dataset and move enh. application to @@ -2620,6 +2722,7 @@ Other mpop/satellites/__init__.py mpop/satin/helper_functions.py mpop/satin/mipp_xrit.py + - Add algorithm version in output cloud products. [Martin Raspaud] - Minor PEP8 tweaks. [Panu Lahtinen] - Script to generate external calibration files for AVHRR instruments. @@ -2714,6 +2817,7 @@ Other * Product dependencies * loading from viirs * generating images + - WIP: successfully loaded the first viirs granule with newscene! [Martin Raspaud] - Rewriting scene. [Martin Raspaud] @@ -2747,6 +2851,7 @@ Other - Fallback to pre-launch if not available. - load(..., pre_launch_coeffs=True) to force using pre-launch coeffs) + - Correct npp name in h5 files. [Martin Raspaud] - Add the pps v2014 h5 reader. [Martin Raspaud] - Use h5py for lonlat reading also. [Martin Raspaud] @@ -2827,6 +2932,7 @@ Other - Fix name matching in hdfeos_l1b. [Martin Raspaud] The full name didn't work with fnmatch, take basename instead. + - Allows hdfeos_l1b to read a batch of files. [Martin Raspaud] - Add delitem, and code cleanup. [Martin Raspaud] - Merge branch 'pre-master' of github.com:mraspaud/mpop into pre-master. @@ -2857,6 +2963,7 @@ Other - Allow loading a file directly for aapp1b and eps_l1b. [Martin Raspaud] Just run global_data.load(..., filename="/path/to/myfile.1b") + - Merge branch 'pre-master' of github.com:mraspaud/mpop into pre-master. [Martin Raspaud] - Viirs_sdr can now load depending on an area. [Martin Raspaud] @@ -2910,8 +3017,8 @@ Other - Merge pull request #10 from pnuu/pre-master. [Martin Raspaud] Fixed failed merging. Thanks Pnuu. -- Fixed failed merging (removed "<<<<<<< HEAD" and ">>>>>>> upstream - /pre-master" lines) [Panu Lahtinen] +- Fixed failed merging (removed "<<<<<<< HEAD" and ">>>>>>> + upstream/pre-master" lines) [Panu Lahtinen] - Merge branch 'pre-master' of https://github.com/mraspaud/mpop into pre-master. [Adam Dybbroe] - Merge branch 'pre-master' of https://github.com/mraspaud/mpop into @@ -3078,6 +3185,7 @@ Other Conflicts: mpop/imageo/geo_image.py + - Night_color (should had beed called night_overview) is the same as cloudtop. [Lars Orum Rasmussen] - Bug fix from Bocheng. [Lars Orum Rasmussen] @@ -3120,6 +3228,7 @@ Other channels 3a and 3b each time, one of them being entirely masked. This of course created some problem further down. Fixed by setting the not loadable channel to None. + - Merge branch 'unstable' of /data/proj/SAF/GIT/mpop into unstable. [Martin Raspaud] - Bugfix in npp.cfg template. [Adam Dybbroe] @@ -3142,6 +3251,7 @@ Other an allclose. This was inefficient, and the programming team decided that it was the user's task to know before projection if the source and target area were the same. In other words, the user should be at least a little smart. + - Remove dummy test to boost projection performance. [Martin Raspaud] Mpop was checking in 2 different places if the source and target areas were @@ -3150,6 +3260,7 @@ Other an allclose. This was inefficient, and the programming team decided that it was the user's task to know before projection if the source and target area were the same. In other words, the user should be at least a little smart. + - Update channel list for modis lvl2. [Martin Raspaud] - Bump up version number: 1.0.0. [Martin Raspaud] - Merge branch 'pre-master' into unstable. [Martin Raspaud] @@ -3330,6 +3441,7 @@ Other This fixes a bug in windows that prevents running strftime on string that contain mapping keys conversion specifiers. + - Catch the error if there is no file to load from. [Martin Raspaud] - Add a proper logger in hdfeos reader. [Martin Raspaud] - Get resolution from filename for eos data. [Martin Raspaud] @@ -3343,6 +3455,7 @@ Other - Fix the loading of BT for VIIRS M13 channel. [Martin Raspaud] Has no scale and offset + - Merge branch 'pre-master' of github.com:mraspaud/mpop into pre-master. [Lars Orum Rasmussen] - Refactor the unsigned netcdf packing code. [Martin Raspaud] @@ -3352,6 +3465,7 @@ Other - Replace auto mask and scale from netcdf4. [Martin Raspaud] Eats up too much memory. + - Merge branch 'pre-master' of github.com:mraspaud/mpop into pre-master. [Lars Orum Rasmussen] - Feature: Added template for electro-l satellite. [Martin Raspaud] @@ -4170,4 +4284,3 @@ Other - New rebase. [Martin Raspaud] - diff --git a/satpy/etc/readers/hdfeos_l1b.yaml b/satpy/etc/readers/hdfeos_l1b.yaml index 85bae431e0..2ceaa30695 100644 --- a/satpy/etc/readers/hdfeos_l1b.yaml +++ b/satpy/etc/readers/hdfeos_l1b.yaml @@ -16,9 +16,11 @@ navigations: datasets: '1': - file_type: hdf_eos_data name: '1' - resolution: [250, 500, 1000] + resolution: + 250: {file_type: hdf_eos_data_250m} + 500: {file_type: hdf_eos_data_500m} + 1000: {file_type: hdf_eos_data_1000m} calibration: reflectance coordinates: [longitude, latitude] wavelength: @@ -26,9 +28,11 @@ datasets: - 0.645 - 0.67 '2': - file_type: hdf_eos_data name: '2' - resolution: [250, 500, 1000] + resolution: + 250: {file_type: hdf_eos_data_250m} + 500: {file_type: hdf_eos_data_500m} + 1000: {file_type: hdf_eos_data_1000m} calibration: reflectance coordinates: [longitude, latitude] wavelength: @@ -36,52 +40,57 @@ datasets: - 0.8585 - 0.876 '3': - file_type: hdf_eos_data name: '3' - resolution: [500, 1000] + resolution: + 500: {file_type: hdf_eos_data_500m} + 1000: {file_type: hdf_eos_data_1000m} coordinates: [longitude, latitude] wavelength: - 0.459 - 0.469 - 0.479 '4': - file_type: hdf_eos_data name: '4' - resolution: [500, 1000] + resolution: + 500: {file_type: hdf_eos_data_500m} + 1000: {file_type: hdf_eos_data_1000m} coordinates: [longitude, latitude] wavelength: - 0.545 - 0.555 - 0.565 '5': - file_type: hdf_eos_data name: '5' - resolution: [500, 1000] + resolution: + 500: {file_type: hdf_eos_data_500m} + 1000: {file_type: hdf_eos_data_1000m} coordinates: [longitude, latitude] wavelength: - 1.23 - 1.24 - 1.25 '6': - file_type: hdf_eos_data name: '6' - resolution: [500, 1000] + resolution: + 500: {file_type: hdf_eos_data_500m} + 1000: {file_type: hdf_eos_data_1000m} coordinates: [longitude, latitude] wavelength: - 1.628 - 1.64 - 1.652 '7': - file_type: hdf_eos_data name: '7' - resolution: [500, 1000] + resolution: + 500: {file_type: hdf_eos_data_500m} + 1000: {file_type: hdf_eos_data_1000m} coordinates: [longitude, latitude] wavelength: - 2.105 - 2.13 - 2.155 '8': - file_type: hdf_eos_data + file_type: hdf_eos_data_1000m name: '8' resolution: 1000 coordinates: [longitude, latitude] @@ -90,7 +99,7 @@ datasets: - 0.4125 - 0.42 '9': - file_type: hdf_eos_data + file_type: hdf_eos_data_1000m name: '9' resolution: 1000 coordinates: [longitude, latitude] @@ -99,7 +108,7 @@ datasets: - 0.443 - 0.448 '10': - file_type: hdf_eos_data + file_type: hdf_eos_data_1000m name: '10' resolution: 1000 coordinates: [longitude, latitude] @@ -108,7 +117,7 @@ datasets: - 0.488 - 0.493 '11': - file_type: hdf_eos_data + file_type: hdf_eos_data_1000m name: '11' resolution: 1000 coordinates: [longitude, latitude] @@ -117,7 +126,7 @@ datasets: - 0.531 - 0.536 '12': - file_type: hdf_eos_data + file_type: hdf_eos_data_1000m name: '12' resolution: 1000 coordinates: [longitude, latitude] @@ -126,7 +135,7 @@ datasets: - 0.551 - 0.556 13hi: - file_type: hdf_eos_data + file_type: hdf_eos_data_1000m name: '13hi' resolution: 1000 coordinates: [longitude, latitude] @@ -135,7 +144,7 @@ datasets: - 0.667 - 0.672 13lo: - file_type: hdf_eos_data + file_type: hdf_eos_data_1000m name: '13lo' resolution: 1000 coordinates: [longitude, latitude] @@ -144,7 +153,7 @@ datasets: - 0.667 - 0.672 14hi: - file_type: hdf_eos_data + file_type: hdf_eos_data_1000m name: '14hi' resolution: 1000 coordinates: [longitude, latitude] @@ -153,7 +162,7 @@ datasets: - 0.678 - 0.683 14lo: - file_type: hdf_eos_data + file_type: hdf_eos_data_1000m name: '14lo' resolution: 1000 coordinates: [longitude, latitude] @@ -162,7 +171,7 @@ datasets: - 0.678 - 0.683 '15': - file_type: hdf_eos_data + file_type: hdf_eos_data_1000m name: '15' resolution: 1000 coordinates: [longitude, latitude] @@ -171,7 +180,7 @@ datasets: - 0.748 - 0.753 '16': - file_type: hdf_eos_data + file_type: hdf_eos_data_1000m name: '16' resolution: 1000 coordinates: [longitude, latitude] @@ -180,7 +189,7 @@ datasets: - 0.8695 - 0.877 '17': - file_type: hdf_eos_data + file_type: hdf_eos_data_1000m name: '17' resolution: 1000 coordinates: [longitude, latitude] @@ -189,7 +198,7 @@ datasets: - 0.905 - 0.92 '18': - file_type: hdf_eos_data + file_type: hdf_eos_data_1000m name: '18' resolution: 1000 coordinates: [longitude, latitude] @@ -198,7 +207,7 @@ datasets: - 0.936 - 0.941 '19': - file_type: hdf_eos_data + file_type: hdf_eos_data_1000m name: '19' resolution: 1000 coordinates: [longitude, latitude] @@ -207,7 +216,7 @@ datasets: - 0.94 - 0.965 '20': - file_type: hdf_eos_data + file_type: hdf_eos_data_1000m name: '20' resolution: 1000 coordinates: [longitude, latitude] @@ -216,7 +225,7 @@ datasets: - 3.75 - 3.84 '21': - file_type: hdf_eos_data + file_type: hdf_eos_data_1000m name: '21' resolution: 1000 coordinates: [longitude, latitude] @@ -225,7 +234,7 @@ datasets: - 3.959 - 3.989 '22': - file_type: hdf_eos_data + file_type: hdf_eos_data_1000m name: '22' resolution: 1000 coordinates: [longitude, latitude] @@ -234,7 +243,7 @@ datasets: - 3.959 - 3.989 '23': - file_type: hdf_eos_data + file_type: hdf_eos_data_1000m name: '23' resolution: 1000 coordinates: [longitude, latitude] @@ -243,7 +252,7 @@ datasets: - 4.05 - 4.08 '24': - file_type: hdf_eos_data + file_type: hdf_eos_data_1000m name: '24' resolution: 1000 coordinates: [longitude, latitude] @@ -252,7 +261,7 @@ datasets: - 4.4655 - 4.498 '25': - file_type: hdf_eos_data + file_type: hdf_eos_data_1000m name: '25' resolution: 1000 coordinates: [longitude, latitude] @@ -261,7 +270,7 @@ datasets: - 4.5155 - 4.549 '26': - file_type: hdf_eos_data + file_type: hdf_eos_data_1000m name: '26' resolution: 1000 coordinates: [longitude, latitude] @@ -270,7 +279,7 @@ datasets: - 1.375 - 1.39 '27': - file_type: hdf_eos_data + file_type: hdf_eos_data_1000m name: '27' resolution: 1000 coordinates: [longitude, latitude] @@ -279,7 +288,7 @@ datasets: - 6.715 - 6.895 '28': - file_type: hdf_eos_data + file_type: hdf_eos_data_1000m name: '28' resolution: 1000 coordinates: [longitude, latitude] @@ -288,7 +297,7 @@ datasets: - 7.325 - 7.475 '29': - file_type: hdf_eos_data + file_type: hdf_eos_data_1000m name: '29' resolution: 1000 coordinates: [longitude, latitude] @@ -297,7 +306,7 @@ datasets: - 8.55 - 8.7 '30': - file_type: hdf_eos_data + file_type: hdf_eos_data_1000m name: '30' resolution: 1000 coordinates: [longitude, latitude] @@ -306,7 +315,7 @@ datasets: - 9.73 - 9.88 '31': - file_type: hdf_eos_data + file_type: hdf_eos_data_1000m name: '31' resolution: 1000 coordinates: [longitude, latitude] @@ -315,7 +324,7 @@ datasets: - 11.03 - 11.28 '32': - file_type: hdf_eos_data + file_type: hdf_eos_data_1000m name: '32' resolution: 1000 coordinates: [longitude, latitude] @@ -324,7 +333,7 @@ datasets: - 12.02 - 12.27 '33': - file_type: hdf_eos_data + file_type: hdf_eos_data_1000m name: '33' resolution: 1000 coordinates: [longitude, latitude] @@ -333,7 +342,7 @@ datasets: - 13.335 - 13.485 '34': - file_type: hdf_eos_data + file_type: hdf_eos_data_1000m name: '34' resolution: 1000 coordinates: [longitude, latitude] @@ -342,7 +351,7 @@ datasets: - 13.635 - 13.785 '35': - file_type: hdf_eos_data + file_type: hdf_eos_data_1000m name: '35' resolution: 1000 coordinates: [longitude, latitude] @@ -351,7 +360,7 @@ datasets: - 13.935 - 14.085 '36': - file_type: hdf_eos_data + file_type: hdf_eos_data_1000m name: '36' resolution: 1000 coordinates: [longitude, latitude] @@ -374,15 +383,16 @@ datasets: standard_name: latitude units: degree + # For EUM reduced (thinned) files longitude5k: - file_type: hdf_eos_data + file_type: hdf_eos_data_1000m name: longitude resolution: 5000 standard_name: longitude units: degree latitude5k: - file_type: hdf_eos_data + file_type: hdf_eos_data_1000m name: latitude resolution: 5000 standard_name: latitude @@ -418,18 +428,36 @@ datasets: file_types: - hdf_eos_data: + hdf_eos_data_250m: file_patterns: - - 'M{platform_indicator:1s}D02{resolution:1s}km_A{start_time:%y%j_%H%M%S}_{processing_time:%Y%j%H%M%S}.hdf' - - 'M{platform_indicator:1s}D02{resolution:1s}km.A{start_time:%Y%j.%H%M}.{collection:03d}.{processing_time:%Y%j%H%M%S}.h5' - - 'M{platform_indicator:1s}D02{resolution:1s}KM.A{start_time:%Y%j.%H%M}.{collection:03d}.{processing_time:%Y%j%H%M%S}.hdf' - - 'M{platform_indicator:1s}D02{resolution:1s}KM.{start_time:%y%j%H%M%S}.hdf' - - '{platform_indicator:1s}1.{start_time:%y%j.%H%M}.{resolution:s}.hdf' - file_reader: !!python/name:satpy.readers.hdfeos_l1b.HDFEOSBandReader '' + - 'M{platform_indicator:1s}D02Qkm_A{start_time:%y%j_%H%M%S}_{processing_time:%Y%j%H%M%S}.hdf' + - 'M{platform_indicator:1s}D02QKM.A{start_time:%Y%j.%H%M}.{collection:03d}.{processing_time:%Y%j%H%M%S}.hdf' + - 'M{platform_indicator:1s}D02QKM.A{start_time:%Y%j.%H%M}.{collection:03d}{suffix}.hdf' + - 'M{platform_indicator:1s}D02QKM.{start_time:%y%j%H%M%S}.hdf' + - '{platform_indicator:1s}1.{start_time:%y%j.%H%M}.250m.hdf' + file_reader: !!python/name:satpy.readers.hdfeos_l1b.HDFEOSBandReader + hdf_eos_data_500m: + file_patterns: + - 'M{platform_indicator:1s}D02Hkm_A{start_time:%y%j_%H%M%S}_{processing_time:%Y%j%H%M%S}.hdf' + - 'M{platform_indicator:1s}D02HKM.A{start_time:%Y%j.%H%M}.{collection:03d}.{processing_time:%Y%j%H%M%S}.hdf' + - 'M{platform_indicator:1s}D02HKM.A{start_time:%Y%j.%H%M}.{collection:03d}{suffix}.hdf' + - 'M{platform_indicator:1s}D02HKM.{start_time:%y%j%H%M%S}.hdf' + - '{platform_indicator:1s}1.{start_time:%y%j.%H%M}.500m.hdf' + file_reader: !!python/name:satpy.readers.hdfeos_l1b.HDFEOSBandReader + hdf_eos_data_1000m: + file_patterns: + - 'M{platform_indicator:1s}D021km_A{start_time:%y%j_%H%M%S}_{processing_time:%Y%j%H%M%S}.hdf' + - 'M{platform_indicator:1s}D021KM.A{start_time:%Y%j.%H%M}.{collection:03d}.{processing_time:%Y%j%H%M%S}.hdf' + - 'M{platform_indicator:1s}D021KM.A{start_time:%Y%j.%H%M}.{collection:03d}{suffix}.hdf' + - 'thin_M{platform_indicator:1s}D021KM.A{start_time:%Y%j.%H%M}.{collection:03d}{suffix}.hdf' + - 'M{platform_indicator:1s}D021KM.{start_time:%y%j%H%M%S}.hdf' + - '{platform_indicator:1s}1.{start_time:%y%j.%H%M}.1000m.hdf' + file_reader: !!python/name:satpy.readers.hdfeos_l1b.HDFEOSBandReader hdf_eos_geo: file_patterns: - 'M{platform_indicator:1s}D03_A{start_time:%y%j_%H%M%S}_{processing_time:%Y%j%H%M%S}.hdf' - 'M{platform_indicator:1s}D03.A{start_time:%Y%j.%H%M}.{collection:03d}.{processing_time:%Y%j%H%M%S}.hdf' + - 'M{platform_indicator:1s}D03.A{start_time:%Y%j.%H%M}.{collection:03d}{suffix}.hdf' - 'M{platform_indicator:1s}D03.{start_time:%y%j%H%M%S}.hdf' - '{platform_indicator:1s}1.{start_time:%y%j.%H%M}.geo.hdf' file_reader: !!python/name:satpy.readers.hdfeos_l1b.HDFEOSGeoReader '' diff --git a/satpy/etc/readers/viirs_l1b.yaml b/satpy/etc/readers/viirs_l1b.yaml index a053de16c0..56ccc1b349 100644 --- a/satpy/etc/readers/viirs_l1b.yaml +++ b/satpy/etc/readers/viirs_l1b.yaml @@ -31,22 +31,34 @@ navigations: file_types: vgeoi: file_reader: !!python/name:satpy.readers.viirs_l1b.VIIRSL1BFileHandler - file_patterns: ['VGEOI_{platform_shortname}_d{start_time:%Y%m%d_t%H%M%S}_c{creation_time:%Y%m%d%H%M%S}.nc'] + file_patterns: + - 'VGEOI_{platform_shortname}_d{start_time:%Y%m%d_t%H%M%S}_c{creation_time:%Y%m%d%H%M%S}.nc' + - 'V{platform_shortname:2s}03IMG.A{start_time:%Y%j.%H%M}.{collection_number:3d}.{creation_time:%Y%j%H%M%S}{creator}.nc' vgeom: file_reader: !!python/name:satpy.readers.viirs_l1b.VIIRSL1BFileHandler - file_patterns: ['VGEOM_{platform_shortname}_d{start_time:%Y%m%d_t%H%M%S}_c{creation_time:%Y%m%d%H%M%S}.nc'] + file_patterns: + - 'VGEOM_{platform_shortname}_d{start_time:%Y%m%d_t%H%M%S}_c{creation_time:%Y%m%d%H%M%S}.nc' + - 'V{platform_shortname:2s}03MOD.A{start_time:%Y%j.%H%M}.{collection_number:3d}.{creation_time:%Y%j%H%M%S}{creator}.nc' vgeod: file_reader: !!python/name:satpy.readers.viirs_l1b.VIIRSL1BFileHandler - file_patterns: ['VGEOD_{platform_shortname}_d{start_time:%Y%m%d_t%H%M%S}_c{creation_time:%Y%m%d%H%M%S}.nc'] + file_patterns: + - 'VGEOD_{platform_shortname}_d{start_time:%Y%m%d_t%H%M%S}_c{creation_time:%Y%m%d%H%M%S}.nc' + - 'V{platform_shortname:2s}03DNB.A{start_time:%Y%j.%H%M}.{collection_number:3d}.{creation_time:%Y%j%H%M%S}{creator}.nc' vl1bi: file_reader: !!python/name:satpy.readers.viirs_l1b.VIIRSL1BFileHandler - file_patterns: ['VL1BI_{platform_shortname}_d{start_time:%Y%m%d_t%H%M%S}_c{creation_time:%Y%m%d%H%M%S}.nc'] + file_patterns: + - 'VL1BI_{platform_shortname}_d{start_time:%Y%m%d_t%H%M%S}_c{creation_time:%Y%m%d%H%M%S}.nc' + - 'V{platform_shortname:2s}02IMG.A{start_time:%Y%j.%H%M}.{collection_number:3d}.{creation_time:%Y%j%H%M%S}{creator}.nc' vl1bm: file_reader: !!python/name:satpy.readers.viirs_l1b.VIIRSL1BFileHandler - file_patterns: ['VL1BM_{platform_shortname}_d{start_time:%Y%m%d_t%H%M%S}_c{creation_time:%Y%m%d%H%M%S}.nc'] + file_patterns: + - 'VL1BM_{platform_shortname}_d{start_time:%Y%m%d_t%H%M%S}_c{creation_time:%Y%m%d%H%M%S}.nc' + - 'V{platform_shortname:2s}02MOD.A{start_time:%Y%j.%H%M}.{collection_number:3d}.{creation_time:%Y%j%H%M%S}{creator}.nc' vl1bd: file_reader: !!python/name:satpy.readers.viirs_l1b.VIIRSL1BFileHandler - file_patterns: ['VL1BD_{platform_shortname}_d{start_time:%Y%m%d_t%H%M%S}_c{creation_time:%Y%m%d%H%M%S}.nc'] + file_patterns: + - 'VL1BD_{platform_shortname}_d{start_time:%Y%m%d_t%H%M%S}_c{creation_time:%Y%m%d%H%M%S}.nc' + - 'V{platform_shortname:2s}02DNB.A{start_time:%Y%j.%H%M}.{collection_number:3d}.{creation_time:%Y%j%H%M%S}{creator}.nc' datasets: i_lon: diff --git a/satpy/node.py b/satpy/node.py index 1bfbbd7df3..7861b0d328 100644 --- a/satpy/node.py +++ b/satpy/node.py @@ -279,7 +279,8 @@ def _find_reader_dataset(self, # LOG.debug("Found {} in reader {}".format(str(ds_id), reader_name)) return Node(ds_id, {'reader_name': reader_name}) - def _get_compositor_prereqs(self, prereq_names, skip=False): + def _get_compositor_prereqs(self, prereq_names, skip=False, + calibration=None, polarization=None, resolution=None): """Determine prerequisite Nodes for a composite. Args: @@ -296,7 +297,7 @@ def _get_compositor_prereqs(self, prereq_names, skip=False): prereq_ids = [] unknowns = set() for prereq in prereq_names: - n, u = self._find_dependencies(prereq) + n, u = self._find_dependencies(prereq, calibration, polarization, resolution) if u: unknowns.update(u) if skip: @@ -307,7 +308,7 @@ def _get_compositor_prereqs(self, prereq_names, skip=False): prereq_ids.append(n) return prereq_ids, unknowns - def _find_compositor(self, dataset_key): + def _find_compositor(self, dataset_key, calibration=None, polarization=None, resolution=None): """Find the compositor object for the given dataset_key.""" # NOTE: This function can not find a modifier that performs one or more modifications # if it has modifiers see if we can find the unmodified version first @@ -315,7 +316,7 @@ def _find_compositor(self, dataset_key): if isinstance(dataset_key, DatasetID) and dataset_key.modifiers: new_prereq = DatasetID( *dataset_key[:-1] + (dataset_key.modifiers[:-1],)) - src_node, u = self._find_dependencies(new_prereq) + src_node, u = self._find_dependencies(new_prereq, calibration, polarization, resolution) if u: return None, u @@ -324,17 +325,22 @@ def _find_compositor(self, dataset_key): except KeyError: raise KeyError("Can't find anything called {}".format( str(dataset_key))) - + if resolution: + compositor.info['resolution'] = resolution + if calibration: + compositor.info['calibration'] = calibration + if polarization: + compositor.info['polarization'] = polarization dataset_key = compositor.id # 2.1 get the prerequisites prereqs, unknowns = self._get_compositor_prereqs( - compositor.info['prerequisites']) + compositor.info['prerequisites'], calibration=calibration, polarization=polarization, resolution=resolution) if unknowns: return None, unknowns optional_prereqs, _ = self._get_compositor_prereqs( compositor.info['optional_prerequisites'], - skip=True) + skip=True, calibration=calibration, polarization=polarization, resolution=resolution) # Is this the right place for that? if src_node is not None: @@ -390,7 +396,10 @@ def _find_dependencies(self, # 2 try to find a composite that matches try: - node, unknowns = self._find_compositor(dataset_key) + node, unknowns = self._find_compositor(dataset_key, + calibration=calibration, + polarization=polarization, + resolution=resolution) except KeyError: node = None unknowns = set([dataset_key]) diff --git a/satpy/readers/hdfeos_l1b.py b/satpy/readers/hdfeos_l1b.py index 05b5e64f0e..cfd971b66e 100644 --- a/satpy/readers/hdfeos_l1b.py +++ b/satpy/readers/hdfeos_l1b.py @@ -241,7 +241,7 @@ def _interpolate(data, coarse_resolution, resolution, interpolator=None): chunk_size = 20 elif resolution == 250: fine_cols = np.arange(1354 * 4) / 4.0 - fine_rows = (np.arange(lines * 4) + 1.5) / 4.0 + fine_rows = (np.arange(lines * 4) - 1.5) / 4.0 chunk_size = 40 along_track_order = 1 diff --git a/satpy/readers/viirs_l1b.py b/satpy/readers/viirs_l1b.py index 80c1cb102f..3c2f2b552d 100644 --- a/satpy/readers/viirs_l1b.py +++ b/satpy/readers/viirs_l1b.py @@ -59,11 +59,16 @@ def end_orbit_number(self): @property def platform_name(self): try: - res = self['/attr/platform'] + res = self.get('/attr/platform', + self.filename_info['platform_shortname']) except KeyError: res = 'Suomi-NPP' + return { 'Suomi-NPP': 'NPP', + 'JPSS-1': 'J01', + 'NP': 'NPP', + 'J1': 'J01', }.get(res, res) @property diff --git a/satpy/readers/yaml_reader.py b/satpy/readers/yaml_reader.py index f0736b97fd..2ebc12a54e 100644 --- a/satpy/readers/yaml_reader.py +++ b/satpy/readers/yaml_reader.py @@ -290,6 +290,9 @@ def _ds_ids_from_any_key(self, key): def filter_ds_ids(self, dataset_ids, dfilter): """Filter *dataset_ids* based on *dfilter*.""" + # sort by resolution, highest resolution first (lowest number) + dataset_ids = sorted(dataset_ids, key=lambda x: x.resolution or -1) + for attr in ['resolution', 'polarization']: if dfilter.get(attr) is not None: dataset_ids = [ds_id for ds_id in dataset_ids diff --git a/satpy/scene.py b/satpy/scene.py index d48c0d1039..0ac2b83b42 100644 --- a/satpy/scene.py +++ b/satpy/scene.py @@ -108,6 +108,9 @@ def __init__(self, self.wishlist = set() self.dep_tree = DependencyTree(self.readers, comps, mods) + def _ipython_key_completions_(self): + return [x.name for x in self.datasets.keys()] + def _compute_metadata_from_readers(self): mda = {} mda['sensor'] = self._get_sensor_names() diff --git a/satpy/version.py b/satpy/version.py index d30d9e2535..ec06771d66 100644 --- a/satpy/version.py +++ b/satpy/version.py @@ -23,4 +23,4 @@ """Version file. """ -__version__ = "0.7.6" +__version__ = "0.7.7" diff --git a/satpy/writers/scmi.py b/satpy/writers/scmi.py index 626277d9ad..67b4ba6566 100644 --- a/satpy/writers/scmi.py +++ b/satpy/writers/scmi.py @@ -83,6 +83,7 @@ LOG = logging.getLogger(__name__) # AWIPS 2 seems to not like data values under 0 AWIPS_USES_NEGATIVES = False +AWIPS_DATA_DTYPE = np.int16 DEFAULT_OUTPUT_PATTERN = '{source_name}_AII_{platform}_{sensor}_{name}_{sector_id}_{tile_id}_{start_time:%Y%m%d_%H%M}.nc' # misc. global attributes @@ -193,7 +194,9 @@ def _get_xy_arrays(self): if y.shape[0] > 2**15: # awips uses 0, 1, 2, 3 so we can't use the negative end of the variable space raise ValueError("Y variable too large for AWIPS-version of 16-bit integer space") - return x, y + # NetCDF library doesn't handle numpy arrays nicely anymore for some + # reason and has been masking values that shouldn't be + return np.ma.masked_array(x), np.ma.masked_array(y) def _get_xy_scaling_parameters(self): """Get the X/Y coordinate limits for the full resulting image""" @@ -524,7 +527,11 @@ def create_variables(self, bitdepth, fill_value, scale_factor=None, add_offset=N valid_min=None, valid_max=None): fgf_coords = "%s %s" % (self.y_var_name, self.x_var_name) - self.image_data = self._nc.createVariable(self.image_var_name, 'u2', dimensions=(self.row_dim_name, self.col_dim_name), fill_value=fill_value, zlib=self._compress) + self.image_data = self._nc.createVariable(self.image_var_name, + AWIPS_DATA_DTYPE, + dimensions=(self.row_dim_name, self.col_dim_name), + fill_value=fill_value, + zlib=self._compress) self.image_data.coordinates = fgf_coords self.apply_data_attributes(bitdepth, scale_factor, add_offset, valid_min=valid_min, valid_max=valid_max) @@ -825,7 +832,7 @@ def _area_id(area_def): ds_list.append(x) output_filenames = [] - dtype = np.dtype(np.uint16) + dtype = AWIPS_DATA_DTYPE fill_value = np.nan for area_id, (area_def, ds_list) in area_datasets.items(): tile_gen = self._get_tile_generator(area_def, lettered_grid, sector_id, num_subtiles, tile_size, tile_count)