diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index dd9b686334..b1b723a99f 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -199,9 +199,9 @@ jobs: if: matrix.use_dap == 'dap_off' - run: echo "ENABLE_DAP=ON" >> $GITHUB_ENV if: matrix.use_dap == 'dap_on' - - run: echo "ENABLE_NCZARR=--disable-nczarr" >> $GITHUB_ENV + - run: echo "ENABLE_NCZARR=OFF" >> $GITHUB_ENV if: matrix.use_nczarr == 'nczarr_off' - - run: echo "ENABLE_NCZARR=--enable-nczarr" >> $GITHUB_ENV + - run: echo "ENABLE_NCZARR=ON" >> $GITHUB_ENV if: matrix.use_nczarr == 'nczarr_on' - run: echo "CTEST_OUTPUT_ON_FAILURE=1" >> $GITHUB_ENV diff --git a/CMakeLists.txt b/CMakeLists.txt index 5926f471de..1ada1d0759 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,7 +26,7 @@ set(PACKAGE "netCDF" CACHE STRING "") SET(NC_VERSION_MAJOR 4) SET(NC_VERSION_MINOR 8) -SET(NC_VERSION_PATCH 2) +SET(NC_VERSION_PATCH 1) SET(NC_VERSION_NOTE "-development") SET(netCDF_VERSION ${NC_VERSION_MAJOR}.${NC_VERSION_MINOR}.${NC_VERSION_PATCH}${NC_VERSION_NOTE}) SET(VERSION ${netCDF_VERSION}) @@ -118,6 +118,11 @@ MACRO(CHECK_C_LINKER_FLAG M_FLAG M_RESULT) SET(CMAKE_REQUIRED_FLAGS "${T_REQ_FLAG}") ENDMACRO() +# Enable 'dist and distcheck'. +# File adapted from http://ensc.de/cmake/FindMakeDist.cmake +FIND_PACKAGE(MakeDist) +# End 'enable dist and distcheck' + # Set the build type. IF(NOT CMAKE_BUILD_TYPE) SET(CMAKE_BUILD_TYPE DEBUG CACHE STRING "Choose the type of build, options are: None, Debug, Release." @@ -194,15 +199,6 @@ ENDIF() OPTION(NC_FIND_SHARED_LIBS "Find dynamically-built versions of dependent libraries" ${BUILD_SHARED_LIBS}) -## -# Check to see if C compiler supports -fno-strict-aliasing, in support of -# https://github.com/Unidata/netcdf-c/issues/1983 -## -CHECK_C_COMPILER_FLAG(-fno-strict-aliasing CC_SUPPORTS_NO_STRICT_ALIASING) -IF(CC_SUPPORTS_NO_STRICT_ALIASING) - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-strict-aliasing") -ENDIF(CC_SUPPORTS_NO_STRICT_ALIASING) - ## # We've had a request to allow for non-versioned shared libraries. # This seems reasonable enough to accommodate. See @@ -773,10 +769,6 @@ IF(USE_HDF5) # are a hot mess between versions. #### - IF(HDF5_hdf5_LIBRARY AND NOT HDF5_C_LIBRARY) - SET(HDF5_C_LIBRARY ${HDF5_hdf5_LIBRARY}) - ENDIF() - # Depending on the install, either HDF5_hdf_library or # HDF5_C_LIBRARIES may be defined. We must check for either. IF(HDF5_C_LIBRARIES AND NOT HDF5_hdf5_LIBRARY) @@ -933,7 +925,7 @@ int main() { }" HAVE_LIBCURL_766) IF (HAVE_LIBCURL_766) - # If libcurl version is >= 7.66, then can skip tests + # If libcurl version is >= 7.66, then can skip tests # for these symbols which were added in an earlier version set(HAVE_CURLOPT_USERNAME TRUE) set(HAVE_CURLOPT_PASSWORD TRUE) @@ -1016,10 +1008,48 @@ OPTION(ENABLE_DAP_LONG_TESTS "Enable DAP long tests." OFF) OPTION(ENABLE_DAP_REMOTE_TESTS "Enable DAP remote tests." ON) SET(REMOTETESTSERVERS "remotetest.unidata.ucar.edu" CACHE STRING "test servers to use for remote test") +# See if we have zlib +FIND_PACKAGE(ZLIB) + +# Define a test flag for have zlib library +IF(ZLIB_FOUND) + INCLUDE_DIRECTORIES(${ZLIB_INCLUDE_DIRS}) + SET(ENABLE_ZLIB TRUE) +ELSE() + SET(ENABLE_ZLIB FALSE) +ENDIF() + +# See if we have libblosc +IF(!MSVC) +FIND_PACKAGE(Blosc) +ENDIF() +# Define a test flag for have blosc library +IF(Blosc_FOUND) + INCLUDE_DIRECTORIES(${Blosc_INCLUDE_DIRS}) + SET(ENABLE_BLOSC TRUE) +ELSE() + SET(ENABLE_BLOSC FALSE) +ENDIF() + +# See if we have libszip +IF(!MSVC) +#FIND_PACKAGE(SZIP) +#FIND_LIBRARY(SZIP PATH NAMES szip sz) +SET(SZIP_LIBRARY ${SZIP}) +ENDIF() +message("xxx: ${SZIP_FOUND} ; ${SZIP}") +# Define a test flag for have szip library +IF(SZIP_FOUND) + INCLUDE_DIRECTORIES(${SZIP_INCLUDE_DIRS}) + SET(ENABLE_SZIP TRUE) +ELSE() + SET(ENABLE_SZIP FALSE) +ENDIF() + # See if we have libzip FIND_PACKAGE(Zip) -# Define a test flag for have curl library +# Define a test flag for have zip library IF(Zip_FOUND) INCLUDE_DIRECTORIES(${Zip_INCLUDE_DIRS}) SET(ENABLE_NCZARR_ZIP TRUE) @@ -1027,6 +1057,14 @@ ELSE() SET(ENABLE_NCZARR_ZIP FALSE) ENDIF() +# libdl is always available; built-in in Windows and OSX +SET(ENABLE_PLUGINS TRUE) +IF(NOT WIN32) + IF(HAVE_DLFCN_H) + INCLUDE_DIRECTORIES("dlfcn.h") + ENDIF() +ENDIF() + # Enable some developer-only tests OPTION(ENABLE_EXTRA_TESTS "Enable Extra tests. Some may not work because of known issues. Developers only." OFF) IF(ENABLE_EXTRA_TESTS) @@ -1049,9 +1087,9 @@ IF(MSVC) FILE(COPY ${netCDF_SOURCE_DIR}/libsrc/XGetopt.c DESTINATION ${netCDF_BINARY_DIR}/ncdump/) FILE(COPY ${netCDF_SOURCE_DIR}/libsrc/XGetopt.c - DESTINATION ${netCDF_BINARY_DIR}/nczarr_test/) + DESTINATION ${netCDF_BINARY_DIR}/nczarr_test/) FILE(COPY ${netCDF_SOURCE_DIR}/libsrc/XGetopt.c - DESTINATION ${netCDF_BINARY_DIR}/ncdap_test/) + DESTINATION ${netCDF_BINARY_DIR}/ncdap_test/) ENDIF() ENDIF() @@ -1386,20 +1424,21 @@ IF (ENABLE_PARALLEL_TESTS AND NOT USE_PARALLEL) ENDIF() # Enable special filter test; experimental when using cmake. -OPTION(ENABLE_FILTER_TESTING "Enable filter testing. Ignored if shared libraries or netCDF4 are not enabled" ${ENABLE_HDF5}) -IF(ENABLE_FILTER_TESTING AND NOT ENABLE_HDF5) - MESSAGE(WARNING "ENABLE_FILTER_TESTING requires HDF5. Disabling.") - SET(ENABLE_FILTER_TESTING OFF) +OPTION(ENABLE_FILTER_TESTING "Enable filter testing. Ignored if shared libraries or netCDF4 are not enabled" yes) + +IF(ENABLE_FILTER_TESTING) + if(NOT ENABLE_HDF5 AND NOT ENABLE_NCZARR) + MESSAGE(WARNING "ENABLE_FILTER_TESTING requires HDF5 and/or NCZarr. Disabling.") + SET(ENABLE_FILTER_TESTING OFF CACHE BOOL "Enable Filter Testing" FORCE) + ENDIF() ENDIF() + IF(NOT BUILD_SHARED_LIBS) MESSAGE(WARNING "ENABLE_FILTER_TESTING requires shared libraries. Disabling.") SET(ENABLE_FILTER_TESTING OFF) ENDIF() -OPTION(ENABLE_CLIENTSIDE_FILTERS "Enable client-side filter registration." OFF) -IF(NOT ENABLE_FILTER_TESTING) -SET(ENABLE_CLIENTSIDE_FILTERS OFF) -ENDIF() +SET(ENABLE_CLIENTSIDE_FILTERS OFF) # Determine whether or not to generate documentation. OPTION(ENABLE_DOXYGEN "Enable generation of doxygen-based documentation." OFF) @@ -1533,6 +1572,7 @@ CHECK_INCLUDE_FILE("libgen.h" HAVE_LIBGEN_H) CHECK_INCLUDE_FILE("execinfo.h" HAVE_EXECINFO_H) CHECK_INCLUDE_FILE("dirent.h" HAVE_DIRENT_H) CHECK_INCLUDE_FILE("time.h" HAVE_TIME_H) +CHECK_INCLUDE_FILE("dlfcn.h" HAVE_DLFCN_H) # Symbol Exists CHECK_SYMBOL_EXISTS(isfinite "math.h" HAVE_DECL_ISFINITE) @@ -1931,7 +1971,7 @@ ENDIF(ENABLE_BASH_SCRIPT_TESTING) MACRO(add_sh_test prefix F) IF(HAVE_BASH) - ADD_TEST(${prefix}_${F} bash "-c" "export srcdir=${CMAKE_CURRENT_SOURCE_DIR};export TOPSRCDIR=${CMAKE_SOURCE_DIR};${CMAKE_CURRENT_BINARY_DIR}/${F}.sh ${ARGN}") + ADD_TEST(${prefix}_${F} bash "-c" "export srcdir=${CMAKE_CURRENT_SOURCE_DIR};export TOPSRCDIR=${CMAKE_SOURCE_DIR};${CMAKE_CURRENT_BINARY_DIR}/${F}.sh ${F}.sh ${ARGN}") ENDIF() ENDMACRO() @@ -2018,17 +2058,31 @@ IF(ENABLE_DAP4) ADD_SUBDIRECTORY(libdap4) ENDIF() +IF(ENABLE_PLUGINS) + ADD_SUBDIRECTORY(libncpoco) +ENDIF() + IF(ENABLE_NCZARR) ADD_SUBDIRECTORY(libnczarr) FILE(COPY ${netCDF_SOURCE_DIR}/unit_test/timer_utils.h DESTINATION ${netCDF_BINARY_DIR}/nczarr_test/) FILE(COPY ${netCDF_SOURCE_DIR}/unit_test/timer_utils.c DESTINATION ${netCDF_BINARY_DIR}/nczarr_test/) + FILE(COPY ${netCDF_SOURCE_DIR}/nc_test4/test_filter.c + DESTINATION ${netCDF_BINARY_DIR}/nczarr_test/) + FILE(COPY ${netCDF_SOURCE_DIR}/nc_test4/test_filter_misc.c + DESTINATION ${netCDF_BINARY_DIR}/nczarr_test/) + FILE(COPY ${netCDF_SOURCE_DIR}/nc_test4/test_filter_repeat.c + DESTINATION ${netCDF_BINARY_DIR}/nczarr_test/) + FILE(COPY ${netCDF_SOURCE_DIR}/nc_test4/test_filter_order.c + DESTINATION ${netCDF_BINARY_DIR}/nczarr_test/) + FILE(COPY ${netCDF_SOURCE_DIR}/nc_test4/tst_multifilter.c + DESTINATION ${netCDF_BINARY_DIR}/nczarr_test/) ENDIF() add_subdirectory(liblib) -IF(ENABLE_FILTER_TESTING) +IF(ENABLE_PLUGINS) add_subdirectory(plugins) ENDIF() @@ -2194,6 +2248,9 @@ INSTALL(PROGRAMS ${netCDF_BINARY_DIR}/nc-config ## print_conf_summary() +# Enable Makedist files. +ADD_MAKEDIST() +ENABLE_MAKEDIST(README.md COPYRIGHT RELEASE_NOTES.md INSTALL INSTALL.cmake test_prog.c lib_flags.am cmake CMakeLists.txt COMPILE.cmake.txt config.h.cmake.in cmake_uninstall.cmake.in netcdf-config-version.cmake.in netcdf-config.cmake.in FixBundle.cmake.in nc-config.cmake.in configure configure.ac install-sh config.h.in config.sub CTestConfig.cmake.in) ##### # Configure and print the libnetcdf.settings file. @@ -2245,6 +2302,8 @@ is_enabled(ENABLE_NCZARR_S3_TESTS DO_NCZARR_S3_TESTS) is_enabled(ENABLE_MULTIFILTERS HAS_MULTIFILTERS) is_enabled(ENABLE_NCZARR_ZIP DO_NCZARR_ZIP_TESTS) is_enabled(ENABLE_LOGGING HAS_LOGGING) +is_enabled(ENABLE_FILTER_TESTING DO_FILTER_TESTS) +is_enabled(ENABLE_BLOSC HAS_BLOSC) # Generate file from template. CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/libnetcdf.settings.in" @@ -2302,6 +2361,10 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test_common.in ${CMAKE_CURRENT_BINARY ##### configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/nc_test4/findplugin.sh @ONLY NEWLINE_STYLE LF) +IF(ENABLE_NCZARR) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/nczarr_test/findplugin.sh @ONLY NEWLINE_STYLE LF) +ENDIF() + IF(ENABLE_EXAMPLES) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/examples/C/findplugin.sh @ONLY NEWLINE_STYLE LF) ENDIF() diff --git a/Makefile.am b/Makefile.am index 0008d1e754..7e24f44c39 100644 --- a/Makefile.am +++ b/Makefile.am @@ -81,13 +81,18 @@ HDF4_TEST_DIR = hdf4_test LIBHDF4 = libhdf4 endif +# Build Plugin support +if ENABLE_PLUGINS +NCPOCO = libncpoco +endif + # Build Cloud Storage if desired. if ENABLE_NCZARR ZARR_TEST_DIR = nczarr_test -ZARR = libnczarr +ZARR = libnczarr endif -# Optionally build plugins +# Optionally build test plugins if ENABLE_FILTER_TESTING PLUGIN_DIR = plugins endif @@ -108,7 +113,7 @@ endif # and run. ncgen must come before ncdump, because their tests # depend on it. SUBDIRS = include $(H5_TEST_DIR) libdispatch libsrc $(LIBSRC4_DIR) \ -$(LIBSRCP) $(LIBHDF4) $(LIBHDF5) $(OCLIB) $(DAP2) ${DAP4} ${ZARR} liblib \ +$(LIBSRCP) $(LIBHDF4) $(LIBHDF5) $(OCLIB) $(DAP2) ${DAP4} ${NCPOCO} ${ZARR} liblib \ $(NCGEN3) $(NCGEN) $(NCDUMP) ${PLUGIN_DIR} $(TESTDIRS) docs \ $(EXAMPLES) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index cc711778e8..2ec1cae038 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.8.2 - TBD +* [Enhancement] Support filters for NCZarr. See [Github #2101](https://github.com/Unidata/netcdf-c/pull/2101). * [Bug Fix] Make PR 2075 long file name be idempotent. See [Github #2094](https://github.com/Unidata/netcdf-c/pull/2094). ## 4.8.1 - August 18, 2021 diff --git a/cmake/modules/FindBlosc.cmake b/cmake/modules/FindBlosc.cmake new file mode 100755 index 0000000000..da280a8101 --- /dev/null +++ b/cmake/modules/FindBlosc.cmake @@ -0,0 +1,64 @@ +# Searches for an installation of the blosc library. On success, it sets the following variables: +# +# Blosc_FOUND Set to true to indicate the blosc library was found +# Blosc_INCLUDE_DIRS The directory containing the header file blosc/blosc.h +# Blosc_LIBRARIES The libraries needed to use the blosc library +# +# To specify an additional directory to search, set Blosc_ROOT. +# +# Author: Siddhartha Chaudhuri, 2009 +# + +# Look for the header, first in the user-specified location and then in the system locations +SET(Blosc_INCLUDE_DOC "The directory containing the header file blosc.h") +FIND_PATH(Blosc_INCLUDE_DIRS NAMES blosc.h blosc/blosc.h PATHS ${Blosc_ROOT} ${Blosc_ROOT}/include DOC ${Blosc_INCLUDE_DOC} NO_DEFAULT_PATH) +IF(NOT Blosc_INCLUDE_DIRS) # now look in system locations + FIND_PATH(Blosc_INCLUDE_DIRS NAMES blosc.h blosc/blosc.h DOC ${Blosc_INCLUDE_DOC}) +ENDIF(NOT Blosc_INCLUDE_DIRS) + +SET(Blosc_FOUND FALSE) + +IF(Blosc_INCLUDE_DIRS) + SET(Blosc_LIBRARY_DIRS ${Blosc_INCLUDE_DIRS}) + + IF("${Blosc_LIBRARY_DIRS}" MATCHES "/include$") + # Strip off the trailing "/include" in the path. + GET_FILENAME_COMPONENT(Blosc_LIBRARY_DIRS ${Blosc_LIBRARY_DIRS} PATH) + ENDIF("${Blosc_LIBRARY_DIRS}" MATCHES "/include$") + + IF(EXISTS "${Blosc_LIBRARY_DIRS}/lib") + SET(Blosc_LIBRARY_DIRS ${Blosc_LIBRARY_DIRS}/lib) + ENDIF(EXISTS "${Blosc_LIBRARY_DIRS}/lib") + + # Find Blosc libraries + FIND_LIBRARY(Blosc_DEBUG_LIBRARY NAMES bloscd blosc_d libbloscd libblosc_d libblosc + PATH_SUFFIXES Debug ${CMAKE_LIBRARY_ARCHITECTURE} ${CMAKE_LIBRARY_ARCHITECTURE}/Debug + PATHS ${Blosc_LIBRARY_DIRS} NO_DEFAULT_PATH) + FIND_LIBRARY(Blosc_RELEASE_LIBRARY NAMES blosc libblosc + PATH_SUFFIXES Release ${CMAKE_LIBRARY_ARCHITECTURE} ${CMAKE_LIBRARY_ARCHITECTURE}/Release + PATHS ${Blosc_LIBRARY_DIRS} NO_DEFAULT_PATH) + + SET(Blosc_LIBRARIES ) + IF(Blosc_DEBUG_LIBRARY AND Blosc_RELEASE_LIBRARY) + SET(Blosc_LIBRARIES debug ${Blosc_DEBUG_LIBRARY} optimized ${Blosc_RELEASE_LIBRARY}) + ELSEIF(Blosc_DEBUG_LIBRARY) + SET(Blosc_LIBRARIES ${Blosc_DEBUG_LIBRARY}) + ELSEIF(Blosc_RELEASE_LIBRARY) + SET(Blosc_LIBRARIES ${Blosc_RELEASE_LIBRARY}) + ENDIF(Blosc_DEBUG_LIBRARY AND Blosc_RELEASE_LIBRARY) + + IF(Blosc_LIBRARIES) + SET(Blosc_FOUND TRUE) + ENDIF(Blosc_LIBRARIES) +ENDIF(Blosc_INCLUDE_DIRS) + +IF(Blosc_FOUND) +# IF(NOT Blosc_FIND_QUIETLY) + MESSAGE(STATUS "Found Blosc: headers at ${Blosc_INCLUDE_DIRS}, libraries at ${Blosc_LIBRARY_DIRS}") + MESSAGE(STATUS " library is ${Blosc_LIBRARIES}") +# ENDIF(NOT Blosc_FIND_QUIETLY) +ELSE(Blosc_FOUND) + IF(Blosc_FIND_REQUIRED) + MESSAGE(FATAL_ERROR "Blosc library not found") + ENDIF(Blosc_FIND_REQUIRED) +ENDIF(Blosc_FOUND) diff --git a/config.h.cmake.in b/config.h.cmake.in index b7a4c0bfe8..fa042b3833 100644 --- a/config.h.cmake.in +++ b/config.h.cmake.in @@ -153,6 +153,9 @@ are set when opening a binary file on Windows. */ /* if true, enable NCZARR */ #cmakedefine ENABLE_NCZARR 1 +/* if true, enable nczarr filter support */ +#cmakedefine ENABLE_NCZARR_FILTERS 1 + /* if true, enable S3 support */ #cmakedefine ENABLE_NCZARR_S3 1 @@ -165,6 +168,9 @@ are set when opening a binary file on Windows. */ /* if true, S3 SDK is available */ #cmakedefine ENABLE_S3_SDK 1 +/* if true, Allow dynamically loaded plugins */ +#cmakedefine ENABLE_PLUGINS 1 + /* define the possible sources for remote test servers */ #cmakedefine REMOTETESTSERVERS "${REMOTETESTSERVERS}" @@ -223,6 +229,9 @@ are set when opening a binary file on Windows. */ /* Define to 1 if you have the header file. */ #cmakedefine HAVE_DIRENT_H 1 +/* Define to 1 if you have the header file. */ +#cmakedefine HAVE_DLFCN_H 1 + /* Define to 1 if you have the header file. */ #cmakedefine HAVE_FCNTL_H 1 @@ -611,11 +620,14 @@ with zip */ /* if true, use stdio instead of posixio */ #cmakedefine USE_STDIO 1 -/* if true, compile in szip compression in netCDF-4 variables */ -#cmakedefine USE_SZIP 1 - /* if true, multi-filters enabled*/ -#cmakedefine HAVE_MULTIFILTERS 1 +#cmakedefine ENABLE_MULTIFILTERS 1 + +/* if true, enable nczarr blosc support */ +#cmakedefine ENABLE_BLOSC 1 + +/* if true, enable nczarr szip support */ +#cmakedefine USE_SZIP 1 /* Version number of package */ #cmakedefine VERSION "${netCDF_VERSION}" diff --git a/configure.ac b/configure.ac index 2206b92256..16401fafd1 100644 --- a/configure.ac +++ b/configure.ac @@ -596,6 +596,15 @@ if test "x$enable_nczarr" = xyes; then fi AM_CONDITIONAL(ENABLE_NCZARR, [test x$enable_nczarr = xyes]) +# See if we have libblosc +AC_CHECK_LIB([blosc],[blosc_init],[enable_blosc=yes],[enable_blosc=no]) +if test "x$enable_blosc" = "xyes" ; then + AC_SEARCH_LIBS([blosc_init],[blosc blosc.dll cygblosc.dll], [], []) + AC_DEFINE([ENABLE_BLOSC], [1], [if true, blosc library is available]) +fi +AC_MSG_CHECKING([whether libblosc library is available]) +AC_MSG_RESULT([${enable_blosc}]) + # See if we have libzip AC_CHECK_LIB([zip],[zip_open],[have_zip=yes],[have_zip=no]) if test "x$have_zip" = "xyes" ; then @@ -610,6 +619,7 @@ if test "x$enable_nczarr" = xno ; then enable_nczarr_zip=no fi + AC_MSG_CHECKING([whether nczarr zip support is enabled]) AC_MSG_RESULT([${enable_nczarr_zip}]) @@ -1101,6 +1111,25 @@ if test "x$enable_byterange" = xyes; then AC_DEFINE([ENABLE_BYTERANGE], [1], [if true, support byte-range read of remote datasets.]) fi +# Need libdl(d) for plugins +AC_CHECK_LIB([dl],[dlopen],[have_libdld=yes],[have_libdld=no]) +if test "x$have_libdld" = "xyes" ; then + AC_SEARCH_LIBS([dlopen],[dl dld], [],[]) +fi + +# Does the user want plugins? +AC_MSG_CHECKING([whether dynamically loaded plugins is enabled]) +AC_ARG_ENABLE([plugins], + [AS_HELP_STRING([--disable-plugins], + [allow dynamically loaded plugins])]) +if test "x$have_libdld" = xno ; then enable_plugins=no; fi +test "x$enable_plugins" = xno || enable_plugins=yes +AC_MSG_RESULT($enable_plugins) +if test "x$enable_plugins" = xyes; then + AC_DEFINE([ENABLE_PLUGINS], [1], [if true, support dynamically loaded plugins]) +fi +AM_CONDITIONAL(ENABLE_PLUGINS, [test "x$enable_plugins" = xyes]) + AC_FUNC_ALLOCA AC_CHECK_DECLS([isnan, isinf, isfinite],,,[#include ]) AC_STRUCT_ST_BLKSIZE @@ -1209,11 +1238,10 @@ AC_CHECK_SIZEOF(ssize_t) $SLEEPCMD AC_CHECK_SIZEOF([void*]) -if test "x$enable_hdf5" = xyes || test "x$enable_dap" = xyes; then +if test "x$enable_hdf5" = xyes || test "x$enable_dap" = xyes ; then AC_SEARCH_LIBS([deflate], [zlibwapi zlibstat zlib zlib1 z], [], [ AC_MSG_ERROR([Can't find or link to the z library. Turn off netCDF-4 and \ DAP clients with --disable-hdf5 --disable-dap, or see config.log for errors.])]) - AC_SEARCH_LIBS([dlopen], [dl dld], [], []) fi # We need the math library @@ -1315,6 +1343,16 @@ if test "x$enable_hdf5" = xyes; then if test "x$hdf5_version_1106" = xyes; then AC_DEFINE([HDF5_UTF8_PATHS], [1], [if true, HDF5 paths can be utf-8]) fi + +else # !enable_hdf5 + # See if we have libsz + AC_CHECK_LIB([sz],[SZ_BufftoBuffCompress],[enable_szlib=yes],[enable_szlib=no]) + if test "x$enable_szlib" = "xyes" ; then + AC_SEARCH_LIBS([SZ_BufftoBuffCompress],[sz sz.dll cygsz.dll], [], []) + AC_DEFINE([USE_SZIP], [1], [if true, szip library is available]) + fi + AC_MSG_CHECKING([whether szip library is available]) + AC_MSG_RESULT([${enable_szlib}]) fi AM_CONDITIONAL(ENABLE_NCDUMPCHUNKS, [test "x$has_readchunks" = xyes ]) @@ -1487,6 +1525,47 @@ if test "x$enable_logging" = xyes; then AC_DEFINE([LOGGING], 1, [If true, turn on logging.]) fi +# Control NCZarr filters +# Does the user want to use NCZarr filters? +AC_MSG_CHECKING([whether we should enable NCZarr filters]) +AC_ARG_ENABLE([nczarr-filters], [AS_HELP_STRING([--disable-nczarr-filters], + [disable NCZarr filters])]) +test "x$enable_nczarr_filters" = xno || enable_nczarr_filters=yes +AC_MSG_RESULT([$enable_nczarr_filters]) +if test "x$enable_nczarr_filters" = xyes; then + AC_DEFINE([ENABLE_NCZARR_FILTERS], [1], [if true, enable NCZarr filters]) +fi + +# Control filter test/example +AC_MSG_CHECKING([whether filter testing should be run]) +AC_ARG_ENABLE([filter-testing], + [AS_HELP_STRING([--disable-filter-testing], + [Do not run filter test and example; requires shared libraries and HDF5|NCZarr])]) +test "x$enable_filter_testing" = xno || enable_filter_testing=yes +AC_MSG_RESULT($enable_filter_testing) + +if test "x$enable_nczarr" = xno ; then +enable_nczarr_filters=no +fi + +if test "x$enable_hdf5" = xno && test "x$enable_nczarr" = xno ; then +AC_MSG_WARN([HDF5 and NCZarr are disabled => --disable-filter-testing]) +enable_filter_testing=no +fi + +if test "x$enable_shared" = xno ; then +AC_MSG_WARN([Shared libraries are disabled => --disable-filter-testing]) +enable_filter_testing=no +enable_nczarr_filters=no +fi + +# Client side filter registration is permanently disabled +enable_clientside_filters=no + +AM_CONDITIONAL(ENABLE_CLIENTSIDE_FILTERS, [test x$enable_clientside_filters = xyes]) +AM_CONDITIONAL(ENABLE_FILTER_TESTING, [test x$enable_filter_testing = xyes]) +AM_CONDITIONAL(ENABLE_NCZARR_FILTERS, [test x$enable_nczarr_filters = xyes]) + # Automake conditionals need to be called, whether the answer is yes # or no. AM_CONDITIONAL(BUILD_PARALLEL, [test x$enable_parallel = xyes]) @@ -1523,6 +1602,8 @@ AM_CONDITIONAL(HAS_PAR_FILTERS, [test x$hdf5_supports_par_filters = xyes ]) AM_CONDITIONAL(ENABLE_NCZARR, [test "x$enable_nczarr" = xyes]) AM_CONDITIONAL(HAVE_AWS, [test "x$have_aws" = xyes]) AM_CONDITIONAL(HAS_MULTIFILTERS, [test "x$has_multifilters" = xyes]) +AM_CONDITIONAL(ENABLE_BLOSC, [test "x$enable_blosc" = xyes]) +AM_CONDITIONAL(ENABLE_SZIP, [test "x$enable_szip" = xyes]) # If the machine doesn't have a long long, and we want netCDF-4, then # we've got problems! @@ -1598,30 +1679,6 @@ esac NC_FLIBS="-lnetcdff $NC_LIBS" -# Control filter test/example -AC_MSG_CHECKING([whether filter testing should be run]) -AC_ARG_ENABLE([filter-testing], - [AS_HELP_STRING([--disable-filter-testing], - [Do not run filter test and example; requires shared libraries and netCDF-4])]) -test "x$enable_filter_testing" = xno || enable_filter_testing=yes -AC_MSG_RESULT($enable_filter_testing) - -# Client side filter registration is permanently disabled -enable_clientside_filters=no - -if test "x$enable_hdf5" = xno ; then -AC_MSG_WARN([HDF5 disabled => --disable-filter-testing && --disable-client-filters]) -enable_filter_testing=no -fi - -if test "x$enable_shared" = xno ; then -AC_MSG_WARN([Shared libraries are disabled => --disable-filter-testing]) -enable_filter_testing=no -fi - -AM_CONDITIONAL(ENABLE_CLIENTSIDE_FILTERS, [test x$enable_clientside_filters = xyes]) -AM_CONDITIONAL(ENABLE_FILTER_TESTING, [test x$enable_filter_testing = xyes]) - AC_SUBST(NC_LIBS,[$NC_LIBS]) AC_SUBST(HAS_DAP,[$enable_dap]) @@ -1650,6 +1707,9 @@ AC_SUBST(DO_NCZARR_S3_TESTS,[$enable_nczarr_s3_tests]) AC_SUBST(HAS_MULTIFILTERS,[$has_multifilters]) AC_SUBST(DO_NCZARR_ZIP_TESTS,[$enable_nczarr_zip]) AC_SUBST(HAS_LOGGING,[$enable_logging]) +AC_SUBST(DO_FILTER_TESTS,[$enable_filter_testing]) +AC_SUBST(HAVE_BLOSC,[$enable_blosc]) +AC_SUBST(HAVE_SZIP,[$enable_szip]) # Include some specifics for netcdf on windows. #AH_VERBATIM([_WIN32_STRICMP], @@ -1745,9 +1805,8 @@ AM_CONDITIONAL([AX_IGNORE], [test xno = xyes]) AC_MSG_NOTICE([generating header files and makefiles]) AC_CONFIG_FILES(test_common.sh:test_common.in) -AC_CONFIG_FILES(nczarr_test/timer_utils.h:unit_test/timer_utils.h) -AC_CONFIG_FILES(nczarr_test/timer_utils.c:unit_test/timer_utils.c) AC_CONFIG_FILES(nc_test4/findplugin.sh:nc_test4/findplugin.in) +AC_CONFIG_FILES(nczarr_test/findplugin.sh:nc_test4/findplugin.in) AC_CONFIG_FILES(examples/C/findplugin.sh:nc_test4/findplugin.in) AC_CONFIG_FILES(ncdap_test/findtestserver.c:ncdap_test/findtestserver.c.in) AC_CONFIG_FILES(dap4_test/findtestserver4.c:ncdap_test/findtestserver.c.in) @@ -1756,6 +1815,13 @@ AC_CONFIG_FILES([h5_test/run_par_tests.sh], [chmod ugo+x h5_test/run_par_tests.s AC_CONFIG_FILES([nc_test4/run_par_test.sh], [chmod ugo+x nc_test4/run_par_test.sh]) AC_CONFIG_FILES([nc_perf/run_par_bm_test.sh], [chmod ugo+x nc_perf/run_par_bm_test.sh]) AC_CONFIG_FILES([nc_perf/run_gfs_test.sh], [chmod ugo+x nc_perf/run_gfs_test.sh]) +AC_CONFIG_FILES(nczarr_test/timer_utils.h:unit_test/timer_utils.h) +AC_CONFIG_FILES(nczarr_test/timer_utils.c:unit_test/timer_utils.c) +AC_CONFIG_FILES(nczarr_test/test_filter.c:nc_test4/test_filter.c) +AC_CONFIG_FILES(nczarr_test/test_filter_misc.c:nc_test4/test_filter_misc.c) +AC_CONFIG_FILES(nczarr_test/tst_multifilter.c:nc_test4/tst_multifilter.c) +AC_CONFIG_FILES(nczarr_test/test_filter_repeat.c:nc_test4/test_filter_repeat.c) +AC_CONFIG_FILES(nczarr_test/test_filter_order.c:nc_test4/test_filter_order.c) AC_CONFIG_FILES([examples/C/run_par_test.sh], [chmod ugo+x examples/C/run_par_test.sh]) AC_CONFIG_FILES([nc-config], [chmod 755 nc-config]) AC_CONFIG_FILES([Makefile @@ -1782,6 +1848,7 @@ AC_CONFIG_FILES([Makefile libdap4/Makefile libhdf4/Makefile libnczarr/Makefile + libncpoco/Makefile libdispatch/Makefile liblib/Makefile ncdump/cdl/Makefile diff --git a/dap4_test/baselinethredds/CMakeLists.txt b/dap4_test/baselinethredds/CMakeLists.txt index 50bc61111c..92e44b5e54 100644 --- a/dap4_test/baselinethredds/CMakeLists.txt +++ b/dap4_test/baselinethredds/CMakeLists.txt @@ -13,7 +13,7 @@ FILE(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/ FILE_PERMISSION # Rename file in support of https://github.com/Unidata/netcdf-c/issues/2077 ## IF(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/GOES16_CONUS_20170821_020218_0.47_1km_33.3N_91.4W.nc4.thredds) -FILE(RENAME ${CMAKE_CURRENT_BINARY_DIR}/GOES16_TEST1.nc4.thredds ${CMAKE_CURRENT_BINARY_DIR}/GOES16_CONUS_20170821_020218_0.47_1km_33.3N_91.4W.nc4.thredds) +FILE(COPY ${CMAKE_CURRENT_SOURCE_DIR}/GOES16_TEST1.nc4.thredds DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/GOES16_CONUS_20170821_020218_0.47_1km_33.3N_91.4W.nc4.thredds) ENDIF() FILE(GLOB CUR_EXTRA_DIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/*) diff --git a/docs/filters.md b/docs/filters.md new file mode 100644 index 0000000000..2a30b7fb76 --- /dev/null +++ b/docs/filters.md @@ -0,0 +1,952 @@ +NetCDF-4 Filter Support +============================ + + +# NetCDF-4 Filter Support {#filters} + +\tableofcontents +[TOC] + +The netCDF library supports a general filter mechanism to apply various +kinds of filters to datasets before reading or writing. + +The netCDF enhanced (aka netCDF-4) library inherits this capability since it depends on the HDF5 library. +The HDF5 library (1.8.11 and later) supports filters, and netCDF is based closely on that underlying HDF5 mechanism. + +Filters assume that a variable has chunking defined and each chunk is filtered before writing and "unfiltered" after reading and before passing the data to the user. + +In the event that multiple filters are defined on a variable, they are applied in first-defined order on writing and on the reverse order when reading. + +The most common kind of filter is a compression-decompression filter, and that is the focus of this document. + +For now, this document is strongly influenced by the HDF5 mechanism. +When other implementations (e.g. Zarr) support filters, this document will have multiple sections: one for each mechanism. + +# A Warning on Backward Compatibility {#filters_compatibility} + +The API defined in this document should accurately reflect +the current state of filters in the netCDF-c library. +Be aware that there was a short period in which the filter code was undergoing some revision and extension. +Those extensions have largely been reverted. +Unfortunately, some users may experience some compilation problems for previously working code because of these reversions. +In that case, please revise your code to adhere to this document. Apologies are extended for any inconvenience. + +A user may encounter an incompatibility if any of the following appears in user code. + +* The function _nc_inq_var_filter_ was returning the error value _NC_ENOFILTER_ if a variable had no associated filters. + It has been reverted to the previous case where it returned _NC_NOERR_ and the returned filter id was set to zero if the variable had no filters. +* The function _nc_inq_var_filterids_ was renamed to _nc_inq_var_filter_ids_. +* Some auxilliary functions for parsing textual filter specifications have been moved to __netcdf_aux.h__. + See Appendix A. +* All of the "filterx" functions have been removed. This is unlikely to cause problems because they had limited visibility. +* The undocumented function "nc_filter_remove" no longer exists. + +For additional information, see Appendix B. + +# Enabling A HDF5 Compression Filter {#filters_enable} + +HDF5 supports dynamic loading of compression filters using the following process for reading of compressed data. + +1. Assume that we have a dataset with one or more variables that were compressed using some algorithm. + How the dataset was compressed will be discussed subsequently. +2. Shared libraries or DLLs exist that implement the compress/decompress algorithm. + These libraries have a specific API so that the HDF5 library can locate, load, and utilize the compressor. + These libraries are expected to installed in a specific directory. + +In order to compress a variable with an HDF5 compliant filter, the netcdf-c library must be given three pieces of information: + +1. some unique identifier for the filter to be used, +2. a vector of parameters for controlling the action of the compression filter, and +3. a shared library implementation of the filter. + +The meaning of the parameters is, of course, completely filter dependent and the filter description [3] needs to be consulted. +For bzip2, for example, a single parameter is provided representing the compression level. +It is legal to provide a zero-length set of parameters. +Defaults are not provided, so this assumes that the filter can operate with zero parameters. + +Filter ids are assigned by the HDF group. +See [4] for a current list of assigned filter ids. +Note that ids above 32767 can be used for testing without registration. + +The first two pieces of information can be provided in one of three ways: using __ncgen__, via an API call, or via command line parameters to __nccopy__. +In any case, remember that filtering also requires setting chunking, so the variable must also be marked with chunking information. +If compression is set for a non-chunked variable, the variable will forcibly be +converted to chunked using a default chunking algorithm. + +## Using The API {#filters_API} +The necessary API methods are included in _netcdf_filter.h_ by default. +These functions implicitly use the HDF5 mechanisms and may produce an error if applied to a file format that is not compatible with the HDF5 mechanism. + +1. Add a filter to the set of filters to be used when writing a variable. + + This must be invoked after the variable has been created and before __nc_enddef__ is invoked. +```` +int nc_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, const unsigned int* params); + +Arguments: +* ncid -- File and group ID. +* varid -- Variable ID. +* id -- Filter ID. +* nparams -- Number of filter parameters. +* params -- Filter parameters. + +Return codes: +* NC_NOERR -- No error. +* NC_ENOTNC4 -- Not a netCDF-4 file. +* NC_EBADID -- Bad ncid or bad filter id +* NC_ENOTVAR -- Invalid variable ID. +* NC_EINDEFINE -- called when not in define mode +* NC_ELATEDEF -- called after variable was created +* NC_EINVAL -- Scalar variable, or parallel enabled and parallel filters not supported or nparams or params invalid. +```` + +2. Query a variable to obtain a list of all filters associated with that variable. + + The number of filters associated with the variable is stored in __nfiltersp__ (it may be zero). + The set of filter ids will be returned in __filterids__. + As is usual with the netcdf API, one is expected to call this function twice. + The first time to set __nfiltersp__ and the second to get the filter ids in client-allocated memory. + Any of these arguments can be NULL, in which case no value is returned. +```` +int nc_inq_var_filter_ids(int ncid, int varid, size_t* nfiltersp, unsigned int* filterids); + +Arguments: +* ncid -- File and group ID. +* varid -- Variable ID. +* nfiltersp -- Stores number of filters found; may be zero. +* filterids -- Stores set of filter ids. + +Return codes: +* NC_NOERR -- No error. +* NC_ENOTNC4 -- Not a netCDF-4 file. +* NC_EBADID -- Bad ncid +* NC_ENOTVAR -- Invalid variable ID. +```` + +3. Query a variable to obtain information about a specific filter associated with the variable. + + The __id__ indicates the filter of interest. + The actual parameters are stored in __params__. + The number of parameters is returned in __nparamsp__. + As is usual with the netcdf API, one is expected to call this function twice. + The first time to set __nparamsp__ and the second to get the parameters in client-allocated memory. + Any of these arguments can be NULL, in which case no value is returned. + If the specified id is not attached to the variable, then NC_ENOFILTER is returned. +```` +int nc_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparamsp, unsigned int* params); + +Arguments: +* ncid -- File and group ID. +* varid -- Variable ID. +* id -- The filter id of interest. +* nparamsp -- Stores number of parameters. +* params -- Stores set of filter parameters. + +Return codes: +* NC_NOERR -- No error. +* NC_ENOTNC4 -- Not a netCDF-4 file. +* NC_EBADID -- Bad ncid +* NC_ENOTVAR -- Invalid variable ID. +* NC_ENOFILTER -- Filter not defined for the variable. +```` + +4. Query a variable to obtain information about the first filter associated with the variable. + + When netcdf-c was modified to support multiple filters per variable, the utility of this function became redundant since it returns info only about the first defined filter for the variable. + Internally, it is implemented using the functions __nc_inq_var_filter_ids__ and __nc_inq_filter_info__. + + In any case, the filter id will be returned in the __idp__ argument. + If there are not filters, then zero is stored in this argument. + Otherwise, the number of parameters is stored in __nparamsp__ and the actual parameters in __params__. + As is usual with the netcdf API, one is expected to call this function twice. + The first time to get __nparamsp__ and the second to get the parameters in client-allocated memory. + Any of these arguments can be NULL, in which case no value is returned. +```` +int nc_inq_var_filter(int ncid, int varid, unsigned int* idp, size_t* nparamsp, unsigned int* params); + +Arguments: +* ncid -- File and group ID. +* varid -- Variable ID. +* idp -- Stores the id of the first found filter, set to zero if variable has no filters. +* nparamsp -- Stores number of parameters. +* params -- Stores set of filter parameters. + +Return codes: +* NC_NOERR -- No error. +* NC_ENOTNC4 -- Not a netCDF-4 file. +* NC_EBADID -- Bad ncid +* NC_ENOTVAR -- Invalid variable ID. +```` + +## Using ncgen {#filters_NCGEN} + +In a CDL file, compression of a variable can be specified by annotating it with the following attribute: + +* ''_Filter'' — a string containing a comma separated list of constants specifying (1) the filter id to apply, and (2) a vector of constants representing the parameters for controlling the operation of the specified filter. +See the section on the parameter encoding syntax for the details on the allowable kinds of constants. + +This is a "special" attribute, which means that it will normally be invisible when using __ncdump__ unless the -s flag is specified. + +This attribute may be repeated to specify multiple filters. +For backward compatibility it is probably better to use the ''_Deflate'' attribute instead of ''_Filter''. But using ''_Filter'' to specify deflation will work. + +Note that the lexical order of declaration is important when more than one filter is specified for a variable because it determines the order in which the filters are applied. + +### Example CDL File (Data elided) + +```` +netcdf bzip2szip { +dimensions: + dim0 = 4 ; dim1 = 4 ; dim2 = 4 ; dim3 = 4 ; +variables: + float var(dim0, dim1, dim2, dim3) ; + var:_Filter = "307,9|4,32,32" ; // bzip2 then szip + var:_Storage = "chunked" ; + var:_ChunkSizes = 4, 4, 4, 4 ; +data: +... +} +```` + +Note that the assigned filter id for bzip2 is 307 and for szip it is 4. + +## Using nccopy {#filters_NCCOPY} + +When copying a netcdf file using __nccopy__ it is possible to specify filter information for any output variable by using the "-F" option on the command line; for example: +```` +nccopy -F "var,307,9" unfiltered.nc filtered.nc +```` +Assume that _unfiltered.nc_ has a chunked but not bzip2 compressed variable named "var". +This command will copy that variable to the _filtered.nc_ output file but using filter with id 307 (i.e. bzip2) and with parameter(s) 9 indicating the compression level. +See the section on the parameter encoding syntax for the details on the allowable kinds of constants. + +The "-F" option can be used repeatedly, as long as a different variable is specified for each occurrence. + +It can be convenient to specify that the same compression is to be applied to more than one variable. To support this, two additional *-F* cases are defined. + +1. ````-F *,...```` means apply the filter to all variables in the dataset. +2. ````-F v1&v2&..,...```` means apply the filter to multiple variables. + +Multiple filters can be specified using the pipeline notions '|'. +For example + +1. ````-F v1&v2,307,9|4,32,32```` means apply filter 307 (bzip2) then filter 4 (szip) to the multiple variables. + +Note that the characters '*', '&', and '|' are shell reserved characters, so you will probably need to escape or quote the filter spec in that environment. + +As a rule, any input filter on an input variable will be applied to the equivalent output variable — assuming the output file type is netcdf-4. +It is, however, sometimes convenient to suppress output compression either totally or on a per-variable basis. +Total suppression of output filters can be accomplished by specifying a special case of "-F", namely this. +```` +nccopy -F none input.nc output.nc +```` +The expression ````-F *,none```` is equivalent to ````-F none````. + +Suppression of output filtering for a specific set of variables can be accomplished using these formats. +```` +nccopy -F "var,none" input.nc output.nc +nccopy -F "v1&v2&...,none" input.nc output.nc +```` +where "var" and the "vi" are the fully qualified name of a variable. + +The rules for all possible cases of the "-F none" flag are defined by this table. + + +
-F none-Fvar,...Input FilterApplied Output Filter +
trueundefinedNAunfiltered +
truenoneNAunfiltered +
truedefinedNAuse output filter(s) +
falseundefineddefineduse input filter(s) +
falsenoneNAunfiltered +
falsedefinedNAuse output filter(s) +
falseundefinedundefinedunfiltered +
falsedefineddefineduse output filter(s) +
+ +# Filter Specification Syntax {#filters_syntax} + +The utilities ncgen and nccopy, and also the output of __ncdump__, support the specification of filter ids, formats, and parameters in text format. +The BNF specification is defined in Appendix C. +Basically, These specifications consist of a filter id, a comma, and then a sequence of +comma separated constants representing the parameters. +The constants are converted within the utility to a proper set of unsigned int constants (see the parameter encoding section). + +To simplify things, various kinds of constants can be specified rather than just simple unsigned integers. +The __ncgen__ and __nccopy__ programs will encode them properly using the rules specified in the section on parameter encode/decode. +Since the original types are lost after encoding, __ncdump__ will always show a simple list of unsigned integer constants. + +The currently supported constants are as follows. + +
ExampleTypeFormat TagNotes +
-17bsigned 8-bit byteb|BTruncated to 8 bits and zero extended to 32 bits +
23ubunsigned 8-bit byteu|U b|BTruncated to 8 bits and zero extended to 32 bits +
-25Ssigned 16-bit shorts|STruncated to 16 bits and zero extended to 32 bits +
27USunsigned 16-bit shortu|U s|STruncated to 16 bits and zero extended to 32 bits +
-77implicit signed 32-bit integerLeading minus sign and no tag +
77implicit unsigned 32-bit integerNo tag +
93Uexplicit unsigned 32-bit integeru|U +
789f32-bit floatf|F +
12345678.12345678d64-bit doubled|DLE encoding +
-9223372036854775807L64-bit signed long longl|LLE encoding +
18446744073709551615UL64-bit unsigned long longu|U l|LLE encoding +
+Some things to note. + +1. In all cases, except for an untagged positive integer, the format tag is required and determines how the constant is converted to one or two unsigned int values. +2. For an untagged positive integer, the constant is treated as of the smallest type into which it fits (i.e. 8,16,32, or 64 bit). +3. For signed byte and short, the value is sign extended to 32 bits and then treated as an unsigned int value, but maintaining the bit-pattern. +4. For double, and signed|unsigned long long, they are converted as specified in the section on parameter encode/decode. +5. In order to support mutiple filters, the argument to ''_Filter'' may be a pipeline separated (using '|') to specify a list of filters specs. + +# Dynamic Loading Process {#filters_Process} + +Each filter is assumed to be compiled into a separate dynamically loaded library. +For HDF5 conformant filters, these filter libraries are assumed to be in some specific location. +The details for writing such a filter are defined in the HDF5 documentation[1,2]. + +## Plugin directory {#filters_Plugindir} + +The HDF5 loader expects plugins to be in a specified plugin directory. +The default directory is: +* "/usr/local/hdf5/lib/plugin” for linux/unix operating systems (including Cygwin) +* “%ALLUSERSPROFILE%\\hdf5\\lib\\plugin” for Windows systems, although the code does not appear to explicitly use this path. + +The default may be overridden using the environment variable __HDF5_PLUGIN_PATH__. + +## Plugin Library Naming {#filters_Pluginlib} + +Given a plugin directory, HDF5 examines every file in that directory that conforms to a specified name pattern as determined by the platform on which the library is being executed. + +
PlatformBasenameExtension +
Linuxlib*.so* +
OSXlib*.so* +
Cygwincyg*.dll* +
Windows*.dll +
+ +## Plugin Verification {#filters_Pluginverify} + +For each dynamic library located using the previous patterns, HDF5 attempts to load the library and attempts to obtain information from it. +Specifically, It looks for two functions with the following signatures. + +1. __H5PL_type_t H5PLget_plugin_type(void)__ — This function is expected to return the constant value __H5PL_TYPE_FILTER__ to indicate that this is a filter library. +2. __const void* H5PLget_plugin_info(void)__ — This function returns a pointer to a table of type __H5Z_class2_t__. + This table contains the necessary information needed to utilize the filter both for reading and for writing. + In particular, it specifies the filter id implemented by the library and it must match that id specified for the variable in __nc_def_var_filter__ in order to be used. + +If plugin verification fails, then that plugin is ignored and the search continues for another, matching plugin. + +# NCZarr Filter Support {#filters_nczarr} + +The inclusion of Zarr support in the netcdf-c library creates the need to provide a new representation consistent with the way that Zarr files store filter information. +For Zarr, filters are represented using the JSON notation. +Each filter is defined by a JSON dictionary, and each such filter dictionary +is guaranteed to have a key named "id" whose value is a unique string defining the filter algorithm: "lz4" or "bzip2", for example. + +The parameters of the filter are defined by additional -- algorithm specific -- keys in the filter dictionary. +One commonly used filter is "blosc", which has a JSON dictionary of this form. +```` + { + "id": "blosc", + "cname": "lz4", + "clevel": 5, + "shuffle": 1 + } +```` +So it has three parameters: + +1. "cname" -- the sub-algorithm used by the blosc compressor, LZ4 in this case. +2. "clevel" -- the compression level, 5 in this case. +3. "shuffle" -- is the input shuffled before compression, yes (1) in this case. + +NCZarr has three constraints that must be met. + +1. It must store its filter information in its metadata in the above JSON dictionary format. +2. It is required to re-use the HDF5 filter implementations. +This is to avoid having to rewrite the filter implementations +This means that some mechanism is needed to translate between the HDF5 id+parameter model and the Zarr JSON dictionary model. +3. It must be possible to modify the set of visible parameters in response to environment information such as the type of the associated variable; this is required to mimic the corresponding HDF5 capability. + +Note that the term "visible parameters" is used here to refer to the parameters provided by "nc_def_var_filter" or those stored in the dataset's metadata as provided by the JSON codec. The term "working parameters" refers to the parameters given to the compressor itself and derived from the visible parameters. + +The standard authority for defining Zarr filters is the list supported by the NumCodecs project [7]. +Comparing the set of standard filters (aka codecs) defined by NumCodecs to the set of standard filters defined by HDF5 [3], it can be seen that the two sets overlap, but each has filters not defined by the other. + +Note also that it is undesirable that a specific set of filters/codecs be built into the NCZarr implementation. +Rather, it is preferable for there be some extensible way to associate the JSON with the code implementing the codec. This mirrors the plugin model used by HDF5. + +The mechanism provided to address these issues is similar to that taken by HDF5. +A shared library must exist that has certain well-defined entry points that allow the NCZarr code to determine information about a Codec. +The shared library exports a well-known function name to access Codec information and relate it to a corresponding HDF5 implementation, + +## Processing Overview + +There are several paths by which the NCZarr filter API is invoked. + +1. The nc_def_var_filter function is invoked on a variable or +(1a) the metadata for a variable is read when opening an existing variable that has associated Codecs. +2. The visible parameters are converted to a set of working parameters. +3. The filter is invoked with the working parameters. +4. The dataset is closed using the final set of visible parameters. + +### Step 1: Invoking nc_def_var_filter + +In this case, the filter plugin is located and the set of visible parameters (from nc_def_var_filter) are provided. + +### Step 1a: Reading metadata + +In this case, the codec is read from the metadata and must be converted to a visible set of HDF5 style parameters. +It is possible that this set of visible parameters differs from the set that was provided by nc_def_var_filter. +If this is important, then the filter implementation is responsible for marking this difference using, for example, different number of parameters or some differing value. + +### Step 2: Convert visible parameters to working parameters + +Given environmental information such as the associated variables base type, the visible parameters +are converted to a potentially larger set of working parameters; additionally provide the opportunity +to modify the visible parameters. + +### Step 3: Invoking the filter + +As chunks are read or written, the filter is repeatedly invoked using the working parameters. + +### Step 4: Closing the dataset + +The visible parameters from step 2 are stored in the dataset's metadata. +It is desirable to determine if the set of visible parameters changes. +If no change is detected, then re-writing the compressor metadata may be avoided. + +## Client API + +Currently, there is no way to specify use of a filter via Codec through +the netcdf-c API. Rather, one must know the HDF5 id and parameters of +the filter of interest and use the functions ''nc_def_var_filter'' and ''nc_inq_var_filter''. +Internally, the NCZarr code will use information about known Codecs to convert the HDF5 filter reference to the corresponding Codec. +This restriction also holds for the specification of filters in ''ncgen'' and ''nccopy''. + +## Special Codecs Attribute + +A new special attribute is defined called ''_Codecs'' in parallel to the current ''_Filters'' special attribute. Its value is a string containing the JSON representation of the Codecs associated with a given variable. +This can be especially useful when a file is unreadable because it uses a filter not available to the netcdf-c library. +That is, no implementation was found in the e.g. ''HDF5_PLUGIN_PATH'' directory. +In this case ''ncdump -hs'' will display the raw Codec information so that it may be possible to see what filter is missing. + +## Pre-Processing Filter Libraries + +The process for using filters for NCZarr is defined to operate in several steps. +First, as with HDF5, all shared libraries in a specified directory +(e.g. ''HDF5_PLUGIN_PATH'') are scanned. +They are interrogated to see what kind of library they implement, if any. +This interrogation operates by seeing if certain well-known (function) names are defined in this library. + +There will be two library types: + +1. HDF5 -- exports a specific API: "H5Z\_plugin\_type" and "H5Z\_get\_plugin\_info". +2. Codec -- exports a specific API: "NCZ\_get\_codec\_info" + +Note that a given library can export either or both of these APIs. +This means that we can have three types of libraries: + +1. HDF5 only +2. Codec only +3. HDF5 + Codec + +Suppose that our ''HDF5_PLUGIN_PATH'' location has an HDF5-only library. +Then by adding a corresponding, separate, Codec-only library to that same location, it is possible to make an HDF5 library usable by NCZarr. +It is possible to do this without having to modify the HDF5-only library. +Over time, it is possible to merge an HDF5-only library with a Codec-only library to produce a single, combined library. + +## Using Plugin Libraries + +The netcdf-c library processes all of the shared libraries by interrogating each one for the well-known APIs and recording the result. +Any libraries that do not export one or both of the well-known APIs is ignored. + +Internally, the netcdf-c library pairs up each HDF5 library API with a corresponding Codec API by invoking the relevant well-known functions +(See Appendix E). +This results in this table for associated codec and hdf5 libraries. + +
HDF5 APICodec APIAction +
Not definedNot definedIgnore +
DefinedNot definedIgnore +
DefinedDefinedNCZarr usable +
+ +## Using the Codec API + +Given a set of filters for which the HDF5 API and the Codec API +are defined, it is then possible to use the APIs to invoke the +filters and to process the meta-data in Codec JSON format. + +### Writing an NCZarr Container + +When writing, the user program will invoke the NetCDF API function *nc_def_var_filter*. +This function is currently defined to operate using HDF5-style id and parameters (unsigned ints). +The netcdf-c library examines its list of known filters to find one matching the HDF5 id provided by *nc_def_var_filter*. +The set of parameters provided is stored internally. +Then during writing of data, the corresponding HDF5 filter is invoked to encode the data. + +When it comes time to write out the meta-data, the stored HDF5-style parameters are passed to a specific Codec function to obtain the corresponding JSON representation. Again see Appendix E. +This resulting JSON is then written in the NCZarr metadata. + +### Reading an NCZarr Container + +When reading, the netcdf-c library will read the metadata for a given variable and will see that some set of filters are applied to this variable. +The metadata is encoded as Codec-style JSON. + +Given a JSON Codec, it is parsed to provide a JSON dictionary containing the string "id" and the set of parameters as various keys. +The netcdf-c library examines its list of known filters to find one matching the Codec "id" string. +The JSON is passed to a Codec function to obtain the corresponding HDF5-style *unsigned int* parameter vector. +These parameters are stored for later use. + +## Supporting Filter Chains + +HDF5 supports *filter chains*, which is a sequence of filters where the output of one filter is provided as input to the next filter in the sequence. +When encoding, the filters are executed in the "forward" direction, +while when decoding the filters are executed in the "reverse" direction. + +In the Zarr meta-data, a filter chain is divided into two parts: +the "compressor" and the "filters". The former is a single JSON codec +as described above. The latter is an ordered JSON array of codecs. +So if compressor is something like + "compressor": {"id": "c"...} +and the filters array is like this: + "filters": [ {"id": "f1"...}, {"id": "f2"...}...{"id": "fn"...}] +then the filter chain is (f1,f2,...fn,c) with f1 being applied first and c being applied last when encoding. On decode, the filter chain is executed in the order (c,fn...f2,f1). + +So, an HDF5 filter chain is divided into two parts, where the last filter in the chain is assigned as the "compressor" and the remaining +filters are assigned as the "filters". +But independent of this, each codec, whether a compressor or a filter, +is stored in the JSON dictionary form described earlier. + +## Extensions + +The Codec style, using JSON, has the ability to provide very complex parameters that may be hard to encode as a vector of unsigned integers. +It might be desirable to consider exporting a JSON-base API out of the netcdf-c API to support user access to this complexity. +This would mean providing some alternate version of "nc_def_var_filter" that takes a string-valued argument instead of a vector of unsigned ints. +This extension is unlikely to be implemented until a compelling use-case is encountered. + +One bad side-effect of this is that we then may have two classes of plugins. +One class can be used by both HDF5 and NCZarr, and a second class that is usable only with NCZarr. + +## Using The NetCDF-C Plugins + +As part of its testing, the NetCDF build process creates a number of shared libraries in the ''netcdf-c/plugins'' (or sometimes ''netcdf-c/plugins/.libs'') directory. +If you need a filter from that set, you may be able to set ''HDF5_PLUGIN_PATH'' +to point to that directory or you may be able to copy the shared libraries out of that directory to your own location. + +# Debugging {#filters_debug} + +Depending on the debugger one uses, debugging plugins can be very difficult. +It may be necessary to use the old printf approach for debugging the filter itself. + +One case worth mentioning is when there is a dataset that is using an unknown filter. +For this situation, you need to identify what filter(s) are used in the dataset. +This can be accomplished using this command. +```` +ncdump -s -h +```` +Since ncdump is not being asked to access the data (the -h flag), it can obtain the filter information without failures. +Then it can print out the filter id and the parameters as well as the Codecs (via the -s flag). + +## Test Cases {#filters_TestCase} + +Within the netcdf-c source tree, the directory two directories contain test cases for testing dynamic filter operation. + +* __netcdf-c/nc_test4__ provides tests for testing HDF5 filters. +* __netcdf-c/nczarr_test__ provides tests for testing NCZarr filters. + +These tests are disabled if __--disable-shared__ or if __--disable-filter-tests__ is specified. + +## HDF5 Example {#filters_Example} + +A slightly simplified version of one of the HDF5 filter test cases is also available as an example within the netcdf-c source tree directory __netcdf-c/examples/C__. +The test is called __filter_example.c__ and it is executed as part of the __run_examples4.sh__ shell script. +The test case demonstrates dynamic filter writing and reading. + +The files __example/C/hdf5plugins/Makefile.am__ and __example/C/hdf5plugins/CMakeLists.txt__ demonstrate how to build the hdf5 plugin for bzip2. + +# Notes + +## Order of Invocation for Multiple Filters + +When multiple filters are defined on a variable, the order of application, when writing data to the file, is same as the order in which _nc_def_var_filter_ is called. +When reading a file the order of application is of necessity the reverse. + +There are some special cases. + +1. The fletcher32 filter is always applied first, if enabled. +2. If _nc_def_var_filter_ or _nc_def_var_deflate_ or _nc_def_var_szip_ is called multiple times with the same filter id, but possibly with different sets of parameters, then the position of that filter in the sequence of applictions does not change. + However the last set of parameters specified is used when actually writing the dataset. +3. Deflate and shuffle — these two are inextricably linked in the current API, but have quite different semantics. + If you call _nc_def_var_deflate_ multiple times, then the previous rule applies with respect to deflate. + However, the shuffle filter, if enabled, is ''always'' applied before applying any other filters, except fletcher32. +4. Once a filter is defined for a variable, it cannot be removed nor can its position in the filter order be changed. + +## Memory Allocation Issues + +Starting with HDF5 version 1.10.*, the plugin code MUST be careful when using the standard *malloc()*, *realloc()*, and *free()* function. + +In the event that the code is allocating, reallocating, for +free'ing memory that either came from or will be exported to the +calling HDF5 library, then one MUST use the corresponding HDF5 +functions *H5allocate_memory()*, *H5resize_memory()*, +*H5free_memory()* [5] to avoid memory failures. + +Additionally, if your filter code leaks memory, then the HDF5 library generates a failure something like this. +```` +H5MM.c:232: H5MM_final_sanity_check: Assertion `0 == H5MM_curr_alloc_bytes_s' failed. +```` + +One can look at the the code in plugins/H5Zbzip2.c and H5Zmisc.c as illustrations. + +## SZIP Issues + +The current szip plugin code in the HDF5 library has some behaviors that can catch the unwary. +These are handled internally to (mostly) hide them so that they should not affect users. +Specifically, this filter may do two things. + +1. Add extra parameters to the filter parameters: going from the two parameters provided by the user to four parameters for internal use. + It turns out that the two parameters provided when calling nc_def_var_filter correspond to the first two parameters of the four parameters returned by nc_inq_var_filter. +2. Change the values of some parameters: the value of the __options_mask__ argument is known to add additional flag bits, and the __pixels_per_block__ parameter may be modified. + +The reason for these changes is has to do with the fact that the szip API provided by the underlying H5Pset_szip function is actually a subset of the capabilities of the real szip implementation. +Presumably this is for historical reasons. + +In any case, if the caller uses the __nc_inq_var_szip__ or the __nc_inq_var_filter__ functions, then the parameter values returned may differ from those originally specified. + +## Supported Systems + +The current matrix of OS X build systems known to work is as follows. + +
Build SystemSupported OS +
AutomakeLinux, Cygwin, OSX +
CmakeLinux, Cygwin, OSX, Visual Studio +
+ +## Generic Plugin Build +If you do not want to use Automake or Cmake, the following has been known to work. +```` +gcc -g -O0 -shared -o libbzip2.so -L${HDF5LIBDIR} -lhdf5_hl -lhdf5 -L${ZLIBDIR} -lz +```` + +# References {#filters_References} + +1. https://support.hdfgroup.org/HDF5/doc/Advanced/DynamicallyLoadedFilters/HDF5DynamicallyLoadedFilters.pdf +2. https://support.hdfgroup.org/HDF5/doc/TechNotes/TechNote-HDF5-CompressionTroubleshooting.pdf +3. https://portal.hdfgroup.org/display/support/Registered+Filter+Plugins +4. https://support.hdfgroup.org/services/contributions.html#filters +5. https://support.hdfgroup.org/HDF5/doc/RM/RM_H5.html +6. https://confluence.hdfgroup.org/display/HDF5/Filters +7. https://numcodecs.readthedocs.io/en/stable/ + +# Appendix A. HDF5 Parameter Encode/Decode {#filters_appendixa} + +The filter id for an HDF5 format filter is an unsigned integer. +Further, the parameters passed to an HDF5 format filter are encoded internally as a vector of 32-bit unsigned integers. +It may be that the parameters required by a filter can naturally be encoded as unsigned integers. +The bzip2 compression filter, for example, expects a single integer value from zero thru nine. +This encodes naturally as a single unsigned integer. + +Note that signed integers and single-precision (32-bit) float values also can easily be represented as 32 bit unsigned integers by proper casting to an unsigned integer so that the bit pattern is preserved. +Simple integer values of type short or char (or the unsigned versions) can also be mapped to an unsigned integer by truncating to 16 or 8 bits respectively and then zero extending. + +Machine byte order (aka endian-ness) is an issue for passing some kinds of parameters. +You might define the parameters when compressing on a little endian machine, but later do the decompression on a big endian machine. + +When using HDF5 format filters, byte order is not an issue for 32-bit values because HDF5 takes care of converting them between the local machine byte order and network byte order. + +Parameters whose size is larger than 32-bits present a byte order problem. +This specifically includes double precision floats and (signed or unsigned) 64-bit integers. +For these cases, the machine byte order issue must be handled, in part, by the compression code. +This is because HDF5 will treat, for example, an unsigned long long as two 32-bit unsigned integers and will convert each to network order separately. +This means that on a machine whose byte order is different than the machine in which the parameters were initially created, the two integers will be separately +endian converted. +But this will be incorrect for 64-bit values. + +So, we have this situation (for HDF5 only): + +1. the 8 bytes come in as native machine order for the machine doing the call to *nc_def_var_filter*. +2. HDF5 divides the 8 bytes into 2 four byte pieces and ensures that each piece is in network (big) endian order. +3. When the filter is called, the two pieces are returned in the same order but with the bytes in each piece consistent with the native machine order for the machine executing the filter. + +## Encoding Algorithms for HDF5 + +In order to properly extract the correct 8-byte value, we need to ensure that the values stored in the HDF5 file have a known format independent of the native format of the creating machine. + +The idea is to do sufficient manipulation so that HDF5 will store the 8-byte value as a little endian value divided into two 4-byte integers. +Note that little-endian is used as the standard because it is the most common machine format. +When read, the filter code needs to be aware of this convention and do the appropriate conversions. + +This leads to the following set of rules. + +### Encoding + +1. Encode on little endian (LE) machine: no special action is required. + The 8-byte value is passed to HDF5 as two 4-byte integers. + HDF5 byte swaps each integer and stores it in the file. +2. Encode on a big endian (BE) machine: several steps are required: + + 1. Do an 8-byte byte swap to convert the original value to little-endian format. + 2. Since the encoding machine is BE, HDF5 will just store the value. + So it is necessary to simulate little endian encoding by byte-swapping each 4-byte integer separately. + 3. This doubly swapped pair of integers is then passed to HDF5 and is stored unchanged. + +### Decoding + +1. Decode on LE machine: no special action is required. + HDF5 will get the two 4-bytes values from the file and byte-swap each separately. + The concatenation of those two integers will be the expected LE value. +2. Decode on a big endian (BE) machine: the inverse of the encode case must be implemented. + + 1. HDF5 sends the two 4-byte values to the filter. + 2. The filter must then byte-swap each 4-byte value independently. + 3. The filter then must concatenate the two 4-byte values into a single 8-byte value. + Because of the encoding rules, this 8-byte value will be in LE format. + 4. The filter must finally do an 8-byte byte-swap on that 8-byte value to convert it to desired BE format. + +To support these rules, some utility programs exist and are discussed in Appendix B. + +# Appendix B. Support Utilities {#filters_appendixb} + +Several functions are exported from the netcdf-c library for use by client programs and by filter implementations. +They are defined in the header file __netcdf_aux.h__. +The h5 tag indicates that they assume that the result of the parse is a set of unsigned integers — the format used by HDF5. + +1. ````int ncaux_h5filterspec_parse(const char* txt, unsigned int* idp. size_t* nparamsp, unsigned int** paramsp);```` +* txt contains the text of a sequence of comma separated constants +* idp will contain the first constant — the filter id +* nparamsp will contain the number of params +* paramsp will contain a vector of params — the caller must free +This function can parse single filter spec strings as defined in the section on Filter Specification Syntax. + +2. ````int ncaux_h5filterspec_parselist(const char* txt, int* formatp, size_t* nspecsp, struct NC_H5_Filterspec*** vectorp);```` +* txt contains the text of a sequence '|' separated filter specs. +* formatp currently always returns 0. +* nspecsp will return the number of filter specifications. +* vectorp will return a pointer to a vector of pointers to filter specification instances — the caller must free. +This function parses a sequence of filter specifications each separated by a '|' character. +The text between '|' separators must be parsable by __ncaux_h5filterspec_parse__. + +3. ````void ncaux_h5filterspec_free(struct NC_H5_Filterspec* f);```` +* f is a pointer to an instance of ````struct NC_H5_Filterspec```` + Typically this was returned as an element of the vector returned + by __ncaux_h5filterspec_parselist__. +This reclaims the parameters of the filter spec object as well as the object itself. + +4. ````int ncaux_h5filterspec_fix8(unsigned char* mem8, int decode);```` +* mem8 is a pointer to the 8-byte value either to fix. +* decode is 1 if the function should apply the 8-byte decoding algorithm + else apply the encoding algorithm. +This function implements the 8-byte conversion algorithms for HDF5. +Before calling *nc_def_var_filter* (unless *NC_parsefilterspec* was used), the client must call this function with the decode argument set to 0. +Inside the filter code, this function should be called with the decode argument set to 1. + +Examples of the use of these functions can be seen in the test program ''nc_test4/tst_filterparser.c''. + +Some of the above functions use a C struct defined in _netcdf_filter.h_. +The definition of that struct is as follows. +```` +typedef struct NC_H5_Filterspec { + unsigned int filterid; /* ID for arbitrary filter. */ + size_t nparams; /* nparams for arbitrary filter. */ + unsigned int* params; /* Params for arbitrary filter. */ +} NC_H5_Filterspec; +```` +This struct in effect encapsulates all of the information about and HDF5 formatted filter — the id, the number of parameters, and the parameters themselves. + +# Appendix C. Build Flags for Detecting the Filter Mechanism {#filters_appendixc} + +The include file _netcdf_meta.h contains the following definition. +```` +#define NC_HAS_MULTIFILTERS 1 +```` + +This, in conjunction with the error code _NC_ENOFILTER_ in _netcdf.h_ can be used to see what filter mechanism is in place as described in the section on incompatibities. + +1. !defined(NC_ENOFILTER) && !defined(NC_HAS_MULTIFILTERS) — indicates that the old pre-4.7.4 mechanism is in place. + It does not support multiple filters. +2. defined(NC_ENOFILTER) && !defined(NC_HAS_MULTIFILTERS) — indicates that the 4.7.4 mechanism is in place. + It does support multiple filters, but the error return codes for _nc_inq_var_filter_ are different and the filter spec parser functions are in a different location with different names. +3. defined(NC_ENOFILTER) && defined(NC_HAS_MULTIFILTERS) — indicates that the multiple filters are supported, and that _nc_inq_var_filter_ returns a filterid of zero to indicate that a variable has no filters. + Also, the filter spec parsers have the names and signatures described in this document and are define in _netcdf_aux.h_. + +# Appendix D. BNF for Specifying Filters in Utilities {#filters_appendixd} + +```` +speclist: spec + | speclist '|' spec + ; +spec: filterid + | filterid ',' parameterlist + ; +filterid: unsigned32 + ; +parameterlist: parameter + | parameterlist ',' parameter + ; +parameter: unsigned32 + +where +unsigned32: <32 bit unsigned integer> +```` +# Appendix E. Codec API {#filters_appendixe} + +The Codec API mirrors the HDF5 API closely. It has one well-known function that can be invoked to obtain information about the Codec as well as pointers to special functions to perform conversions. + +## The Codec Plugin API + +### NCZ_get_codec_info + +This function returns a pointer to a C struct that provides detailed information about the codec plugin. + +#### Signature +```` +void* NCZ_get_codec_info(void); +```` +The value returned is actually of type ''struct NCZ_codec_t'', +but is of type ''void*'' to allow for extensions. + +### NCZ_codec_t +```` +typedef struct NCZ_codec_t { + int version; /* Version number of the struct */ + int sort; /* Format of remainder of the struct; + Currently always NCZ_CODEC_HDF5 */ + const char* codecid; /* The name/id of the codec */ + unsigned int hdf5id; /* corresponding hdf5 id */ + void (*NCZ_codec_initialize)(void); + void (*NCZ_codec_finalize)(void); + int (*NCZ_codec_to_hdf5)(const char* codec, int* nparamsp, unsigned** paramsp); + int (*NCZ_hdf5_to_codec)(size_t nparams, const unsigned* params, char** codecp); + int (*NCZ_modify_parameters)(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* nparamsp, unsigned** paramsp); +} NCZ_codec_t; +```` + +The semantics of the non-function fields is as follows: + +1. ''version'' -- Version number of the struct. +2. ''sort'' -- Format of remainder of the struct; currently always NCZ_CODEC_HDF5. +3. ''codecid'' -- The name/id of the codec. +4. ''hdf5id'' -- The corresponding hdf5 id. + +### NCZ_codec_to_hdf5 + +Given a JSON Codec representation, it will return a corresponding vector of unsigned integers representing the +visible parameters. + +#### Signature + int (*NCZ_codec_to_hdf)(const char* codec, int* nparamsp, unsigned** paramsp); + +#### Arguments +1. codec -- (in) ptr to JSON string representing the codec. +2. nparamsp -- (out) store the length of the converted HDF5 unsigned vector +3. paramsp -- (out) store a pointer to the converted HDF5 unsigned vector; caller must free the returned vector. Note the double indirection. + +Return Value: a netcdf-c error code. + +### NCZ_hdf5_to_codec + +Given an HDF5 visible parameters vector of unsigned integers and its length, +return a corresponding JSON codec representation of those visible parameters. + +#### Signature + int (*NCZ_hdf5_to_codec)(int ncid, int varid, size_t nparams, const unsigned* params, char** codecp); + +#### Arguments +1. ncid -- the variables' containing group +2. varid -- the containing variable +3. nparams -- (in) the length of the HDF5 visible parameters vector +4. params -- (in) pointer to the HDF5 visible parameters vector. +5. codecp -- (out) store the string representation of the codec; caller must free. + +Return Value: a netcdf-c error code. + +### NCZ_modify_parameters + +Extract environment information from the (ncid,varid) and use it to convert a set of visible parameters +to a set of working parameters; also provide option to modify visible parameters. + +#### Signature + int (*NCZ_modify_parameters)(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp); + +#### Arguments +1. ncid -- (in) group id containing the variable. +2. varid -- (in) the id of the variable to which this filter is being attached. +3. vnparamsp -- (in/out) the count of visible parameters +4. vparamsp -- (in/out) the set of visible parameters +5. wnparamsp -- (out) the count of working parameters +4. wparamsp -- (out) the set of working parameters + +Return Value: a netcdf-c error code. + +### NCZ_codec_initialize + +Some compressors may require library initialization. +This function is called as soon as a shared library is loaded and matched with an HDF5 filter. + +#### Signature + int (*NCZ_codec_initialize)(void); + +Return Value: a netcdf-c error code. + +### NCZ_codec_finalize + +Some compressors (like blosc) require invoking a finalize function in order to avoid memory loss. +This function is called during a call to ''nc_finalize'' to do any finalization. +If the client code does not invoke ''nc_finalize'' then memory checkers may complain about lost memory. + +#### Signature + int (*NCZ_codec_finalize)(void); + +Return Value: a netcdf-c error code. + +## Multi-Codec API + +As an aid to clients, it is convenient if a single shared library can provide multiple ''NCZ_code_t'' instances at one time. +This API is not intended to be used by plugin developers. +A shared library must only export this function. + +### NCZ_codec_info_defaults + +Return a NULL terminated vector of pointers to instances of ''NCZ_codec_t''. + +#### Signature + void* NCZ_codec_info_defaults(void); + +The value returned is actually of type ''NCZ_codec_t**'', +but is of type ''void*'' to allow for extensions. +The list of returned items are used to try to provide defaults +for any HDF5 filters that have no corresponding Codec. +This is for internal use only. + +# Appendix F. Pre-built Filters + +As part of the overall build process, a number of filters are built as shared libraries in the "plugins" directory. +They may be in that directory or the "plugins/.libs" subdirectory. +It may be possible for users to utilize some of those libraries to provide filter support for general use. +One simple way to reuse these libraries is to make the environment variable "HDF5_PLUGIN_PATH" refer to the plugin directory or the subdirectory, whichever contains the shared libraries. +This may not be possible if the user already has other filter shared libraries in use. + +The second way is to copy the necessary shared libraries from the plugins directory to the user's HDF5_PLUGIN_PATH directory. +If the user is using HDF5, then the following filters are probably usable: + +* ''libh5bzip2.so'' -- an HDF5 filter for bzip2 compression +* ''libh5blosc.so'' -- an HDF5 filter for blosc compression + +If the user is using NCZarr filters, then you can install the following additional shared libraries: + +* libh5shuffle.so -- shuffle filter +* libh5fletcher32.so -- fletcher32 checksum +* libh5deflate.so -- deflate compression +* libnczdefaults.so -- provide NCZarr support for shuffle, fletcher32, and deflate. + +The shuffle, fletcher32, and deflate filters in this case will be ignored by HDF5 and only used by the NCZarr code. +But in order to use them, it needs additional Codec capabilities provided by the libnczdefauts.so shared library. +Note also that if you disable HDF5 support, but leave NCZarr support enabled, then all of the above filters +should continue to work. + +# Point of Contact {#filters_poc} + +__Author__: Dennis Heimbigner
+__Email__: dmh at ucar dot edu
+__Initial Version__: 1/10/2018
+__Last Revised__: 7/17/2021 diff --git a/docs/nczarr.md b/docs/nczarr.md index 6a5f411ad0..d8ac3ae004 100644 --- a/docs/nczarr.md +++ b/docs/nczarr.md @@ -17,6 +17,10 @@ A note on terminology in this document. 1. The term "dataset" is used to refer to all of the Zarr objects constituting the meta-data and data. +There are some important "caveats" of which to be aware when using this software. + +1. NCZarr currently is not thread-safe. So any attempt to use it with parallelism, including MPIO, is likely to fail. + # The NCZarr Data Model {#nczarr_data_model} NCZarr uses a data model [4] that, by design, extends the Zarr Version 2 Specification [6] to add support for the NetCDF-4 data model. @@ -456,6 +460,8 @@ collections — High-performance dataset datatypes](https://docs.python.org/2/li [8] [Dynamic Filter Loading](https://support.hdfgroup.org/HDF5/doc/Advanced/DynamicallyLoadedFilters/HDF5DynamicallyLoadedFilters.pdf)
[9] [Officially Registered Custom HDF5 Filters](https://portal.hdfgroup.org/display/support/Registered+Filter+Plugins)
[10] [C-Blosc Compressor Implementation](https://github.com/Blosc/c-blosc) +[11] [Conda-forge / packages / aws-sdk-cpp] +(https://anaconda.org/conda-forge/aws-sdk-cpp)
# Appendix A. Building NCZarr Support {#nczarr_build} @@ -548,17 +554,18 @@ it any difficulties were reported to Unidata as a Github issue. In order to use the S3 storage driver, it is necessary to install the Amazon [aws-sdk-cpp library](https://github.com/aws/aws-sdk-cpp.git). -As a starting point, here are the CMake options used by Unidata to build that library. -It assumes that it is being executed in a build directory, `build` say, and that `build/../CMakeLists.txt exists`. -``` -cmake -DBUILD_ONLY=s3 -``` -The expected set of installed libraries are as follows: -* aws-cpp-sdk-s3 -* aws-cpp-sdk-core +Building this package from scratch has proven to be a formidable task. +This appears to be due to dependencies on very specific versions of, +for example, openssl. + +It is recommended that you use conda to install this package on linux. +See [11] for the relevant conda package information. + +For Windows we do not yet have solution. If you successfully install +on Windows, please let us know how you did it. -This library depends on libcurl, so you may need to install that -before building the sdk library. +This library depends on libcurl and openssl, so you may need to install those +before installing the sdk library. # Appendix C. Amazon S3 Imposed Limits {#nczarr_s3limits} diff --git a/include/Makefile.am b/include/Makefile.am index e43748d164..4664d68e90 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -7,7 +7,7 @@ # Ed Hartnett, Dennis Heimbigner, Ward Fisher include_HEADERS = netcdf.h netcdf_meta.h netcdf_mem.h netcdf_aux.h \ -netcdf_filter.h netcdf_filter_build.h netcdf_dispatch.h +netcdf_filter.h netcdf_filter_build.h netcdf_filter_hdf5_build.h netcdf_dispatch.h netcdf_json.h if BUILD_PARALLEL include_HEADERS += netcdf_par.h @@ -31,3 +31,9 @@ noinst_HEADERS += nchttp.h endif EXTRA_DIST = CMakeLists.txt XGetopt.h netcdf_meta.h.in netcdf_dispatch.h.in + +BUILT_SOURCES = netcdf_json.h +netcdf_json.h: Makefile ncjson.h ${srcdir}/../libdispatch/ncjson.c + sed -e 's/NCJSON_H/NETCDF_JSON_H/' -e '/ncjson.h/d' $@ + sed -e '/ncjson.h/d' < ${srcdir}/../libdispatch/ncjson.c >> $@ + diff --git a/include/hdf5internal.h b/include/hdf5internal.h index cbb8526356..c0ad560877 100644 --- a/include/hdf5internal.h +++ b/include/hdf5internal.h @@ -194,6 +194,8 @@ struct NC_HDF5_Filter { unsigned int* params; /**< Params for arbitrary filter. */ }; +int NC4_hdf5_filter_initialize(void); +int NC4_hdf5_filter_finalize(void); int NC4_hdf5_filter_remove(NC_VAR_INFO_T* var, unsigned int id); int NC4_hdf5_filter_lookup(NC_VAR_INFO_T* var, unsigned int id, struct NC_HDF5_Filter** fi); int NC4_hdf5_addfilter(NC_VAR_INFO_T* var, unsigned int id, size_t nparams, const unsigned int* params, int flags); diff --git a/include/nc4internal.h b/include/nc4internal.h index 49bda44f1c..2b772729b0 100644 --- a/include/nc4internal.h +++ b/include/nc4internal.h @@ -100,8 +100,7 @@ typedef enum {NCNAT, NCVAR, NCDIM, NCATT, NCTYP, NCFLD, NCGRP, NCFIL} NC_SORT; /** Hidden attributes; immutable and unreadable thru API. */ #define HIDDENATTRFLAG 1 -/** Readonly global attributes; readable, but immutable thru the - * API. */ +/** Readonly attributes; readable, but immutable thru the API. */ #define READONLYFLAG 2 /** Subset of readonly flags; readable by name only thru the API. */ @@ -110,6 +109,9 @@ typedef enum {NCNAT, NCVAR, NCDIM, NCATT, NCTYP, NCFLD, NCGRP, NCFIL} NC_SORT; /** Subset of readonly flags; Value is actually in file. */ #define MATERIALIZEDFLAG 8 +/** Per-variable attribute, as opposed to global */ +#define VARFLAG 16 + /** Boolean type, to make the code easier to read. */ typedef enum {NC_FALSE = 0, NC_TRUE = 1} nc_bool_t; @@ -201,9 +203,9 @@ typedef struct NC_VAR_INFO int parallel_access; /**< Type of parallel access for I/O on variable (collective or independent). */ nc_bool_t shuffle; /**< True if var has shuffle filter applied. */ nc_bool_t fletcher32; /**< True if var has fletcher32 filter applied. */ - size_t chunk_cache_size; /**< Size in bytes of the var chunk chache. */ + size_t chunk_cache_size; /**< Size in bytes of the var chunk cache. */ size_t chunk_cache_nelems; /**< Number of slots in var chunk cache. */ - float chunk_cache_preemption; /**< Chunk cache preemtion policy. */ + float chunk_cache_preemption; /**< Chunk cache preemption policy. */ void *format_var_info; /**< Pointer to any binary format info. */ void* filters; /**< Record of the list of filters to be applied to var data; format dependent */ } NC_VAR_INFO_T; @@ -461,6 +463,7 @@ extern const NC_reservedatt* NC_findreserved(const char* name); #define NC_ATT_DIMID_NAME "_Netcdf4Dimid" #define NC_ATT_NC3_STRICT_NAME "_nc3_strict" #define NC_XARRAY_DIMS "_ARRAY_DIMENSIONS" +#define NC_ATT_CODECS "_Codecs" #define NC_NCZARR_ATTR "_NCZARR_ATTR" #endif /* _NC4INTERNAL_ */ diff --git a/include/ncconfigure.h b/include/ncconfigure.h index 2b4375ea58..4a9321f07a 100644 --- a/include/ncconfigure.h +++ b/include/ncconfigure.h @@ -71,6 +71,9 @@ extern int fileno(FILE*); #ifndef HAVE_STRLCAT #define strlcat(d,s,n) strcat_s((d),(n),(s)) #endif +#ifndef HAVE_STRCASECMP +#define strcasecmp _stricmp +#endif #endif /* handle null arguments */ @@ -136,5 +139,13 @@ typedef long long fileoffset_t; #define NC_UNUSED(var) (void)var #endif +/* Protect old HDF5 code (pre 1.8.12) */ +#ifndef HAVE_H5ALLOCATE_MEMORY +#ifndef H5allocate_memory +#define H5allocate_memory(size,clear) ((clear)?calloc(1,(size)):malloc(size)) +#define H5free_memory(buf) free(buf) +#define H5resize_memory(buf,size) realloc(buf,size) +#endif +#endif #endif /* NCCONFIGURE_H */ diff --git a/include/ncjson.h b/include/ncjson.h index 18b4208633..86a91ef172 100644 --- a/include/ncjson.h +++ b/include/ncjson.h @@ -1,20 +1,19 @@ /* Copyright 2018, UCAR/Unidata. -Copyright 2018 Unidata - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + See the COPYRIGHT file for more information. */ #ifndef NCJSON_H #define NCJSON_H 1 +#ifndef DLLEXPORT +#ifdef _WIN32 +#define DLLEXPORT __declspec(dllexport) +#else +#define DLLEXPORT +#endif +#endif + +/**************************************************/ /* Json object sorts (note use of term sort rather than e.g. type or discriminant) */ #define NCJ_UNDEF 0 #define NCJ_STRING 1 @@ -29,12 +28,9 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND /* No flags are currently defined, but the argument is a placeholder */ - -/* Define a struct to store primitive values - as unquoted strings. The sort will - provide more info. - Do not bother with a union since - the amount of saved space is minimal. +/* Define a struct to store primitive values as unquoted + strings. The sort will provide more info. Do not bother with + a union since the amount of saved space is minimal. */ typedef struct NCjson { @@ -46,62 +42,65 @@ typedef struct NCjson { } list; /* sort == DICT|ARRAY */ } NCjson; -/* Support Windows declspec */ -#ifndef EXTERNL -# ifdef _WIN32 -# ifdef NCJSON_INTERNAL /* define when compiling code */ -# define EXTERNL __declspec(dllexport) extern -# else -# define EXTERNL __declspec(dllimport) extern -# endif -# else /* !_WIN32 */ -# define EXTERNL extern -# endif -#endif /* !defined EXTERNL */ +/* Structure to hold result of convertinf one json sort to value of another type; + don't use union so we can know when to reclaim sval +*/ +struct NCJconst {int bval; long long ival; double dval; char* sval;}; + +/**************************************************/ +/* Extended API */ + +/* Return 0 if ok else -1 */ #if defined(__cplusplus) extern "C" { #endif -/* int return value is either 1 (ok) or 0 (failure) */ +/* Parse a JSON string */ +DLLEXPORT int NCJparse(const char* text, unsigned flags, NCjson** jsonp); + +/* Reclaim a JSON tree */ +DLLEXPORT extern void NCJreclaim(NCjson* json); + +/* Create a new JSON node of a given sort */ +DLLEXPORT extern int NCJnew(int sort, NCjson** objectp); -/* Parse */ -EXTERNL int NCJparse(const char* text, unsigned flags, NCjson** jsonp); +/* Create new json object with given string content */ +DLLEXPORT extern int NCJnewstring(int sort, const char* value, NCjson** jsonp); -/* Build */ -EXTERNL int NCJnew(int sort, NCjson** object); +/* Create new json object with given counted string content */ +DLLEXPORT extern int NCJnewstringn(int sort, size_t len, const char* value, NCjson** jsonp); -/* Recursively free NCjson instance */ -EXTERNL void NCJreclaim(NCjson*); +/* Get dict key value by name */ +DLLEXPORT extern int NCJdictget(const NCjson* dict, const char* key, NCjson** valuep); -/* Assign a nul terminated string value to an NCjson object as its contents */ -EXTERNL int NCJnewstring(int sort, const char* value, NCjson** jsonp); +/* Convert one json sort to value of another type; don't use union so we can know when to reclaim sval */ +DLLEXPORT extern int NCJcvt(const NCjson* value, int outsort, struct NCJconst* output); + +#ifndef NETCDF_JSON_H -/* Assign a counted string value to an NCjson object as its contents */ -EXTERNL int NCJnewstringn(int sort, size_t len, const char* value, NCjson** jsonp); +/* Insert an atomic value to an array or dict object. */ +DLLEXPORT int NCJaddstring(NCjson* json, int sort, const char* s); /* Append value to an array or dict object. */ -EXTERNL int NCJappend(NCjson* object, NCjson* value); +DLLEXPORT extern int NCJappend(NCjson* object, NCjson* value); /* Insert key-value pair into a dict object. key will be copied */ -EXTERNL int NCJinsert(NCjson* object, char* key, NCjson* value); +DLLEXPORT extern int NCJinsert(NCjson* object, char* key, NCjson* value); /* Unparser to convert NCjson object to text in buffer */ -EXTERNL int NCJunparse(const NCjson* json, unsigned flags, char** textp); +DLLEXPORT extern int NCJunparse(const NCjson* json, unsigned flags, char** textp); -/* Utilities */ -EXTERNL int NCJaddstring(NCjson*, int sort, const char* s); -EXTERNL int NCJdictget(const NCjson* dict, const char* key, NCjson** valuep); +/* Deep clone a json object */ +DLLEXPORT extern int NCJclone(const NCjson* json, NCjson** clonep); /* dump NCjson* object to output file */ -EXTERNL void NCJdump(const NCjson* json, unsigned flags, FILE*); - -/* Convert one json sort to value of another type; don't use union so we can know when to reclaim sval */ -struct NCJconst {int bval; long long ival; double dval; char* sval;}; -EXTERNL int NCJcvt(const NCjson* value, int outsort, struct NCJconst* output); +DLLEXPORT extern void NCJdump(const NCjson* json, unsigned flags, FILE*); +#endif -/* Deep clone a json object */ -EXTERNL int NCJclone(const NCjson* json, NCjson** clonep); +#if defined(__cplusplus) +} +#endif /* Getters */ #define NCJsort(x) ((x)->sort) @@ -119,8 +118,7 @@ EXTERNL int NCJclone(const NCjson* json, NCjson** clonep); /* Misc */ #define NCJisatomic(j) ((j)->sort != NCJ_ARRAY && (j)->sort != NCJ_DICT && (j)->sort != NCJ_NULL && (j)->sort != NCJ_UNDEF) -#if defined(__cplusplus) -} -#endif +/**************************************************/ #endif /*NCJSON_H*/ + diff --git a/include/nclist.h b/include/nclist.h index ad9ab49b4e..f91d6816b2 100644 --- a/include/nclist.h +++ b/include/nclist.h @@ -22,6 +22,7 @@ typedef struct NClist { EXTERNL NClist* nclistnew(void); EXTERNL int nclistfree(NClist*); EXTERNL int nclistfreeall(NClist*); +EXTERNL int nclistclearall(NClist*); EXTERNL int nclistsetalloc(NClist*,size_t); EXTERNL int nclistsetlength(NClist*,size_t); diff --git a/include/netcdf_aux.h b/include/netcdf_aux.h index 1d54a860eb..c1b532d027 100644 --- a/include/netcdf_aux.h +++ b/include/netcdf_aux.h @@ -66,6 +66,7 @@ typedef struct NC_H5_Filterspec { EXTERNL int ncaux_h5filterspec_parse(const char* txt, unsigned int* idp, size_t* nparamsp, unsigned int** paramsp); EXTERNL int ncaux_h5filterspec_parselist(const char* txt0, int* formatp, size_t* nspecsp, struct NC_H5_Filterspec*** vectorp); +EXTERNL int ncaux_h5filterspec_parse_parameter(const char* txt, size_t* nuiparamsp, unsigned int* uiparams); EXTERNL void ncaux_h5filterspec_free(struct NC_H5_Filterspec* f); EXTERNL void ncaux_h5filterspec_fix8(unsigned char* mem, int decode); diff --git a/include/netcdf_filter.h b/include/netcdf_filter.h index 600baec076..b68b9c2ac1 100644 --- a/include/netcdf_filter.h +++ b/include/netcdf_filter.h @@ -58,12 +58,11 @@ EXTERNL int nc_inq_var_filter_ids(int ncid, int varid, size_t* nfilters, unsigne /* Learn about the filter with specified id wrt a variable */ EXTERNL int nc_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparams, unsigned int* params); - /* End HDF5 Format Declarations */ -/**************************************************/ #if defined(__cplusplus) } #endif +/**************************************************/ #endif /* NETCDF_FILTER_H */ diff --git a/include/netcdf_filter_build.h b/include/netcdf_filter_build.h index 0ed08bcaa0..6c6f8e9481 100644 --- a/include/netcdf_filter_build.h +++ b/include/netcdf_filter_build.h @@ -20,107 +20,7 @@ #ifndef NETCDF_FILTER_BUILD_H #define NETCDF_FILTER_BUILD_H 1 -/**************************************************/ -/* Build To the HDF5 C-API for Filters */ - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -/* Support headers */ -#include -#include - -#ifdef USE_HDF5 -#include -/* Older versions of the hdf library may define H5PL_type_t here */ -#include - -#else /*!USE_HDF5*/ /* Provide replacement definitions */ - -/* WARNING: In order make NCZARR independent of HDF5, - while still using HDF5-style filters, some HDF5 - declarations need to be duplicated here with - different names. Watch out for changes in - the underlying HDF5 declarations. - - See the file H5Zpublic.h for more detailed descriptions. - - Note that these declarations are always enabled because - HDF5-style filters may have been created with these definitions - but for use by HDF5. - - Note also that certain filters in the plugins directory will not build if HDF5 is not installed: - notably blosc. -*/ - -/* H5Z_FILTER_RESERVED => H5Z_FILTER_RESERVED */ -#define H5Z_FILTER_RESERVED 256 /*filter ids below this value are reserved for library use */ - -/* H5Z_FILTER_MAX => H5Z_FILTER_MAX */ -#define H5Z_FILTER_MAX 65535 /*maximum filter id */ - -/* Only a limited set of definition and invocation flags are allowed */ -#define H5Z_FLAG_MANDATORY 0x0000 /*filter is mandatory */ -#define H5Z_FLAG_OPTIONAL 0x0001 /*filter is optional */ -#define H5Z_FLAG_REVERSE 0x0100 /*reverse direction; read */ - -typedef int htri_t; -typedef int herr_t; -typedef size_t hsize_t; -typedef long long hid_t; - -#define H5allocate_memory(size,n) malloc(size) -#define H5free_memory(buf) free(buf) - -/* htri_t (*H5Z_can_apply_func_t)(hid_t dcpl_id, hid_t type_id, hid_t space_id) => currently not supported; must be NULL. */ -typedef htri_t (*H5Z_can_apply_func_t)(long long, long long, long long); - -/* herr_t (*H5Z_set_local_func_t)(hid_t dcpl_id, hid_t type_id, hid_t space_id); => currently not supported; must be NULL. */ -typedef herr_t (*H5Z_set_local_func_t)(long long, long long, long long); - -/* H5Z_funct_t => H5Z_filter_func_t */ -typedef size_t (*H5Z_func_t)(unsigned int flags, size_t cd_nelmts, - const unsigned int cd_values[], size_t nbytes, - size_t *buf_size, void **buf); - -typedef int H5Z_filter_t; - -#define H5Z_CLASS_T_VERS 1 - -/* - * The filter table maps filter identification numbers to structs that - * contain a pointers to the filter function and timing statistics. - */ -typedef struct H5Z_class2_t { - int version; /* Version number of the struct; should be H5Z_FILTER_CLASS_VER */ - H5Z_filter_t id; /* Filter ID number */ - unsigned encoder_present; /* Does this filter have an encoder? */ - unsigned decoder_present; /* Does this filter have a decoder? */ - const char *name; /* Comment for debugging */ - H5Z_can_apply_func_t can_apply; /* The "can apply" callback for a filter */ - H5Z_set_local_func_t set_local; /* The "set local" callback for a filter */ - H5Z_func_t filter; /* The actual filter function */ -} H5Z_class2_t; - -/* The HDF5/H5Zarr dynamic loader looks for the following:*/ - -/* Plugin type used by the plugin library */ -typedef enum H5PL_type_t { - H5PL_TYPE_ERROR = -1, /* Error */ - H5PL_TYPE_FILTER = 0, /* Filter */ - H5PL_TYPE_NONE = 1 /* This must be last! */ -} H5PL_type_t; - -#endif /*HAVE_HDF5_H*/ - -/* Following External Discovery Functions should be present for the dynamic loading of filters */ - -/* returns specific constant H5ZP_TYPE_FILTER */ -typedef H5PL_type_t (*H5PL_get_plugin_type_proto)(void); - -/* return */ -typedef const void* (*H5PL_get_plugin_info_proto)(void); +#include "netcdf_filter_hdf5_build.h" /**************************************************/ /* Build To a NumCodecs-style C-API for Filters */ @@ -139,62 +39,65 @@ typedef const void* (*H5PL_get_plugin_info_proto)(void); /* Obtain a pointer to an instance of NCZ_codec_class_t. -NCZ_get_plugin_info(void) -- returns pointer to instance of NCZ_codec_class_t. +NCZ_get_codec_info(void) -- returns pointer to instance of NCZ_codec_class_t. Instance an be recast based on version+sort to the plugin type specific info. So the void* return value is typically actually of type NCZ_codec_class_t*. */ -typedef const void* (*NCZ_get_plugin_info_proto)(void); +typedef const void* (*NCZ_get_codec_info_proto)(void); /* The current object returned by NCZ_get_plugin_info is a pointer to an instance of NCZ_codec_t. -The key to this struct are the four function pointers that do setup/reset/finalize +The key to this struct are the several function pointers that do initialize/finalize and conversion between codec JSON and HDF5 parameters. -Setup context state for the codec converter -int (*NCZ_codec_setup)(int ncid, int varid, void** contextp); +The function pointers defined in NCZ_codec_t manipulate HDF5 parameters and NumCodec JSON. -@param ncid -- (in) ncid of the variable's group -@param varid -- (in) varid of the variable -@params contextp -- (out) context for this (var,codec) combination. -@return -- a netcdf-c error code. +* Initialize use of the filter. This is invoked when a filter is loaded. -Reclaim any codec resources from setup. Not same as finalize. -int (*NCZ_codec_reset)(void* context); +void (*NCZ_codec_initialize)(void); -@param context -- (in) context state +* Finalize use of the filter. Since HDF5 does not provide this functionality, the codec may need to do it. +See H5Zblosc.c for an example. This function is invoked when a filter is unloaded. -Finalize use of the plugin. Since HDF5 does not provide this functionality, -the codec may need to do it. See H5Zblosc.c for an example. void (*NCZ_codec_finalize)(void); -@param context -- (in) context state +* Convert a JSON representation to an HDF5 representation. Invoked when a NumCodec JSON Codec is extracted +from Zarr metadata. -Convert a JSON representation to an HDF5 representation: -int (*NCZ_codec_to_hdf5)(void* context, const char* codec, int* nparamsp, unsigned** paramsp); +int (*NCZ_codec_to_hdf5)(const char* codec, int* nparamsp, unsigned** paramsp); -@param context -- (in) context state from setup. @param codec -- (in) ptr to JSON string representing the codec. @param nparamsp -- (out) store the length of the converted HDF5 unsigned vector @param paramsp -- (out) store a pointer to the converted HDF5 unsigned vector; caller frees. Note the double indirection. @return -- a netcdf-c error code. -Convert an HDF5 representation to a JSON representation -int (*NCZ_hdf5_to_codec)(void* context, int nparams, const unsigned* params, char** codecp); -@param context -- (in) context state from setup. +* Convert an HDF5 vector of visible parameters to a JSON representation. + +int (*NCZ_hdf5_to_codec)(size_t nparams, const unsigned* params, char** codecp); + @param nparams -- (in) the length of the HDF5 unsigned vector @param params -- (in) pointer to the HDF5 unsigned vector. @param codecp -- (out) store the string representation of the codec; caller must free. @return -- a netcdf-c error code. -*/ -/* QUESTION? do we want to provide a netcdf-specific - alternative to H5Z_set_local since NCZarr may not have HDF5 access? - HDF5: herr_t set_local(hid_t dcpl, hid_t type, hid_t space); - Proposed netcdf equivalent: int NCZ_set_local(int ncid, int varid, int* nparamsp, unsigned** paramsp); - where ncid+varid is equivalent to the space. +* Convert a set of visible parameters to a set of working parameters using extra environmental information. +Also allows for changes to the visible parameters. Invoked before filter is actually used. + +int (*NCZ_build_parameters)(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp); + +@param ncid -- (in) ncid of the variable's group +@param varid -- (in) varid of the variable +@params vnparamsp -- (in/out) number of visible parameters +@params vparamsp -- (in/out) vector of visible parameters +@params wnparamsp -- (out) number of working parameters +@params wparamsp -- (out) vector of working parameters +@return -- a netcdf-c error code. + +* Convert a set of working parameters to a set of visible parameters using extra environmental information, if needed. +Invoked before filter metadata is written. */ /* @@ -208,11 +111,11 @@ typedef struct NCZ_codec_t { Currently always NCZ_CODEC_HDF5 */ const char* codecid; /* The name/id of the codec */ unsigned int hdf5id; /* corresponding hdf5 id */ - int (*NCZ_codec_to_hdf5)(void* context, const char* codec, int* nparamsp, unsigned** paramsp); - int (*NCZ_hdf5_to_codec)(void* context, int nparams, const unsigned* params, char** codecp); - int (*NCZ_codec_setup)(int ncid, int varid, void** contextp); - int (*NCZ_codec_reset)(void* context); + void (*NCZ_codec_initialize)(void); void (*NCZ_codec_finalize)(void); + int (*NCZ_codec_to_hdf5)(const char* codec, size_t* nparamsp, unsigned** paramsp); + int (*NCZ_hdf5_to_codec)(size_t nparams, const unsigned* params, char** codecp); + int (*NCZ_modify_parameters)(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp); } NCZ_codec_t; #ifndef NC_UNUSED diff --git a/include/netcdf_filter_hdf5_build.h b/include/netcdf_filter_hdf5_build.h new file mode 100644 index 0000000000..8db64f72bf --- /dev/null +++ b/include/netcdf_filter_hdf5_build.h @@ -0,0 +1,207 @@ +/* Copyright 2018, UCAR/Unidata and OPeNDAP, Inc. + See the COPYRIGHT file for more information. */ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/hdf5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* This include file is used if one wished to build a filter plugin + independent of HDF5. See examples in the plugins directory +*/ + +#ifndef NETCDF_FILTER_HDF5_BUILD_H +#define NETCDF_FILTER_HDF5_BUILD_H 1 + +/**************************************************/ +/* Build To the HDF5 C-API for Filters */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +/* Support headers */ +#include +#include + +#ifdef USE_HDF5 +#include +/* Older versions of the hdf library may define H5PL_type_t here */ +#include + +#else /*!USE_HDF5*/ /* Provide replacement definitions */ + +/* WARNING: In order make NCZARR independent of HDF5, + while still using HDF5-style filters, some HDF5 + declarations need to be duplicated here with + different names. Watch out for changes in + the underlying HDF5 declarations. + + See the file H5Zpublic.h for more detailed descriptions. + + Note that these declarations are always enabled because + HDF5-style filters may have been created with these definitions + but for use by HDF5. + + Note also that certain filters in the plugins directory will not build if HDF5 is not installed: + notably blosc. +*/ + +/* H5Z_FILTER_RESERVED => H5Z_FILTER_RESERVED */ +#define H5Z_FILTER_RESERVED 256 /*filter ids below this value are reserved for library use */ + +/* H5Z_FILTER_MAX => H5Z_FILTER_MAX */ +#define H5Z_FILTER_MAX 65535 /*maximum filter id */ + +/* Only a limited set of definition and invocation flags are allowed */ +#define H5Z_FLAG_MANDATORY 0x0000 /*filter is mandatory */ +#define H5Z_FLAG_OPTIONAL 0x0001 /*filter is optional */ +#define H5Z_FLAG_REVERSE 0x0100 /*reverse direction; read */ +#define H5Z_FLAG_SKIP_EDC 0x0200 /*skip EDC filters for read */ + +typedef int htri_t; +typedef int herr_t; +typedef int hbool_t; +typedef size_t hsize_t; +typedef long long hid_t; + +/* htri_t (*H5Z_can_apply_func_t)(hid_t dcpl_id, hid_t type_id, hid_t space_id) => currently not supported; must be NULL. */ +typedef htri_t (*H5Z_can_apply_func_t)(long long, long long, long long); + +/* herr_t (*H5Z_set_local_func_t)(hid_t dcpl_id, hid_t type_id, hid_t space_id); => currently not supported; must be NULL. */ +typedef herr_t (*H5Z_set_local_func_t)(long long, long long, long long); + +/* H5Z_funct_t => H5Z_filter_func_t */ +typedef size_t (*H5Z_func_t)(unsigned int flags, size_t cd_nelmts, + const unsigned int cd_values[], size_t nbytes, + size_t *buf_size, void **buf); + +typedef int H5Z_filter_t; + +#define H5Z_CLASS_T_VERS 1 + +/* + * The filter table maps filter identification numbers to structs that + * contain a pointers to the filter function and timing statistics. + */ +typedef struct H5Z_class2_t { + int version; /* Version number of the struct; should be H5Z_FILTER_CLASS_VER */ + H5Z_filter_t id; /* Filter ID number */ + unsigned encoder_present; /* Does this filter have an encoder? */ + unsigned decoder_present; /* Does this filter have a decoder? */ + const char *name; /* Comment for debugging */ + H5Z_can_apply_func_t can_apply; /* The "can apply" callback for a filter */ + H5Z_set_local_func_t set_local; /* The "set local" callback for a filter */ + H5Z_func_t filter; /* The actual filter function */ +} H5Z_class2_t; + +/* The HDF5/H5Zarr dynamic loader looks for the following:*/ + +/* Plugin type used by the plugin library */ +typedef enum H5PL_type_t { + H5PL_TYPE_ERROR = -1, /* Error */ + H5PL_TYPE_FILTER = 0, /* Filter */ + H5PL_TYPE_NONE = 1 /* This must be last! */ +} H5PL_type_t; + +#endif /*USE_HDF5*/ + +/* Following External Discovery Functions should be present for the dynamic loading of filters */ + +/* returns specific constant H5ZP_TYPE_FILTER */ +typedef H5PL_type_t (*H5PL_get_plugin_type_proto)(void); + +/* return */ +typedef const void* (*H5PL_get_plugin_info_proto)(void); + +/*************************/ +/* Following are always defined */ + +/* Misc Macros */ + +#ifndef HGOTO_ERROR +#define HGOTO_ERROR(pline, err, action, msg) {fprintf(stderr,"%s\n",msg); ret_value = -1; goto done;} +#endif +#ifndef H5_ATTR_FALLTHROUGH +#define H5_ATTR_FALLTHROUGH /*FALLTHROUGH*/ +#endif +#ifndef H5MM_memcpy +#define H5MM_memcpy memcpy +#endif +#ifndef H5MM_malloc +#define H5MM_malloc malloc +#endif +#ifndef H5MM_realloc +#define H5MM_realloc realloc +#endif +#ifndef H5MM_xfree +#define H5MM_xfree nullfree +#endif +#ifndef H5_ATTR_UNUSED +#define H5_ATTR_UNUSED +#endif +#ifndef FUNC_ENTER_STATIC +#define FUNC_ENTER_STATIC +#endif +#ifndef FUNC_LEAVE_NOAPI +#define FUNC_LEAVE_NOAPI(ret_value) return ret_value; +#endif +#ifndef FALSE +#define FALSE 0 +#define TRUE 1 +#endif +#ifndef SUCCEED +#define SUCCEED 0 +#define FAIL -1 +#endif +#ifndef FUNC_ENTER_NOAPI_NOINIT_NOERR +#define FUNC_ENTER_NOAPI_NOINIT_NOERR +#endif +#ifndef FUNC_ENTER_STATIC_NOERR +#define FUNC_ENTER_STATIC_NOERR +#endif +#ifndef FUNC_LEAVE_NOAPI +#define FUNC_LEAVE_NOAPI(x) return x; +#endif +#ifndef FUNC_LEAVE_NOAPI_VOID +#define FUNC_LEAVE_NOAPI_VOID return; +#endif +#ifndef H5_CHECKED_ASSIGN +#define H5_CHECKED_ASSIGN(dst,dtype,src,stype) (dst) = (dtype)(src) +#endif +#ifndef H5_CHECK_OVERFLOW +#define H5_CHECK_OVERFLOW(dst,dtype,stype) +#endif +#ifndef HDceil +#define HDceil(x) ceil(x) +#endif +#ifndef HDassert +#define HDassert(x) +#endif +#ifndef HDmemset +#define HDmemset memset +#endif + +#ifndef UINT32ENCODE +# define UINT32ENCODE(p, i) { \ + *(p) = (uint8_t)( (i) & 0xff); (p)++; \ + *(p) = (uint8_t)(((i) >> 8) & 0xff); (p)++; \ + *(p) = (uint8_t)(((i) >> 16) & 0xff); (p)++; \ + *(p) = (uint8_t)(((i) >> 24) & 0xff); (p)++; \ +} +# define UINT32DECODE(p, i) { \ + (i) = (uint32_t)(*(p) & 0xff); (p)++; \ + (i) |= ((uint32_t)(*(p) & 0xff) << 8); (p)++; \ + (i) |= ((uint32_t)(*(p) & 0xff) << 16); (p)++; \ + (i) |= ((uint32_t)(*(p) & 0xff) << 24); (p)++; \ +} +#endif + +#endif /*NETCDF_FILTER_HDF5_BUILD_H*/ diff --git a/include/netcdf_json.h b/include/netcdf_json.h new file mode 100644 index 0000000000..9c6860c925 --- /dev/null +++ b/include/netcdf_json.h @@ -0,0 +1,1143 @@ +/* Copyright 2018, UCAR/Unidata. + See the COPYRIGHT file for more information. +*/ + +#ifndef NETCDF_JSON_H +#define NETCDF_JSON_H 1 + +#ifndef DLLEXPORT +#ifdef _WIN32 +#define DLLEXPORT __declspec(dllexport) +#else +#define DLLEXPORT +#endif +#endif + +/**************************************************/ +/* Json object sorts (note use of term sort rather than e.g. type or discriminant) */ +#define NCJ_UNDEF 0 +#define NCJ_STRING 1 +#define NCJ_INT 2 +#define NCJ_DOUBLE 3 +#define NCJ_BOOLEAN 4 +#define NCJ_DICT 5 +#define NCJ_ARRAY 6 +#define NCJ_NULL 7 + +#define NCJ_NSORTS 8 + +/* No flags are currently defined, but the argument is a placeholder */ + +/* Define a struct to store primitive values as unquoted + strings. The sort will provide more info. Do not bother with + a union since the amount of saved space is minimal. +*/ + +typedef struct NCjson { + int sort; /* of this object */ + char* string; /* sort != DICT|ARRAY */ + struct NCjlist { + int len; + struct NCjson** contents; + } list; /* sort == DICT|ARRAY */ +} NCjson; + +/* Structure to hold result of convertinf one json sort to value of another type; + don't use union so we can know when to reclaim sval +*/ +struct NCJconst {int bval; long long ival; double dval; char* sval;}; + +/**************************************************/ +/* Extended API */ + +/* Return 0 if ok else -1 */ + +#if defined(__cplusplus) +extern "C" { +#endif + +/* Parse a JSON string */ +DLLEXPORT int NCJparse(const char* text, unsigned flags, NCjson** jsonp); + +/* Reclaim a JSON tree */ +DLLEXPORT extern void NCJreclaim(NCjson* json); + +/* Create a new JSON node of a given sort */ +DLLEXPORT extern int NCJnew(int sort, NCjson** objectp); + +/* Create new json object with given string content */ +DLLEXPORT extern int NCJnewstring(int sort, const char* value, NCjson** jsonp); + +/* Create new json object with given counted string content */ +DLLEXPORT extern int NCJnewstringn(int sort, size_t len, const char* value, NCjson** jsonp); + +/* Get dict key value by name */ +DLLEXPORT extern int NCJdictget(const NCjson* dict, const char* key, NCjson** valuep); + +/* Convert one json sort to value of another type; don't use union so we can know when to reclaim sval */ +DLLEXPORT extern int NCJcvt(const NCjson* value, int outsort, struct NCJconst* output); + +#ifndef NETCDF_JSON_H + +/* Insert an atomic value to an array or dict object. */ +DLLEXPORT int NCJaddstring(NCjson* json, int sort, const char* s); + +/* Append value to an array or dict object. */ +DLLEXPORT extern int NCJappend(NCjson* object, NCjson* value); + +/* Insert key-value pair into a dict object. key will be copied */ +DLLEXPORT extern int NCJinsert(NCjson* object, char* key, NCjson* value); + +/* Unparser to convert NCjson object to text in buffer */ +DLLEXPORT extern int NCJunparse(const NCjson* json, unsigned flags, char** textp); + +/* Deep clone a json object */ +DLLEXPORT extern int NCJclone(const NCjson* json, NCjson** clonep); + +/* dump NCjson* object to output file */ +DLLEXPORT extern void NCJdump(const NCjson* json, unsigned flags, FILE*); +#endif + +#if defined(__cplusplus) +} +#endif + +/* Getters */ +#define NCJsort(x) ((x)->sort) +#define NCJstring(x) ((x)->string) +#define NCJlength(x) ((x)==NULL ? 0 : (x)->list.len) +#define NCJcontents(x) ((x)->list.contents) +#define NCJith(x,i) ((x)->list.contents[i]) + +/* Setters */ +#define NCJsetsort(x,s) (x)->sort=(s) +#define NCJsetstring(x,y) (x)->string=(y) +#define NCJsetcontents(x,c) (x)->list.contents=(c) +#define NCJsetlength(x,l) (x)->list.len=(l) + +/* Misc */ +#define NCJisatomic(j) ((j)->sort != NCJ_ARRAY && (j)->sort != NCJ_DICT && (j)->sort != NCJ_NULL && (j)->sort != NCJ_UNDEF) + +/**************************************************/ + +#endif /*NETCDF_JSON_H*/ + +/* Copyright 2018, UCAR/Unidata. + See the COPYRIGHT file for more information. +*/ + +/* +TODO: make utf8 safe +*/ + +#define NCJSON_INTERNAL + +#include +#include +#include +#include + + +#undef NCJDEBUG +#ifdef NCJDEBUG +static int ncjbreakpoint(int err) {return err;} +#define NCJTHROW(err) ((err)==NCJ_ERR?ncjbreakpoint(err):(err)) +#else +#define NCJTHROW(err) (err) +#endif + +/**************************************************/ +#define NCJ_OK 0 +#define NCJ_ERR (-1) + +#define NCJ_EOF -2 + +#define NCJ_LBRACKET '[' +#define NCJ_RBRACKET ']' +#define NCJ_LBRACE '{' +#define NCJ_RBRACE '}' +#define NCJ_COLON ':' +#define NCJ_COMMA ',' +#define NCJ_QUOTE '"' +#define NCJ_ESCAPE '\\' +#define NCJ_TAG_TRUE "true" +#define NCJ_TAG_FALSE "false" +#define NCJ_TAG_NULL "null" + +/* JSON_WORD Subsumes Number also */ +#define JSON_WORD "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_$+-." + +/**************************************************/ +typedef struct NCJparser { + char* text; + char* pos; + size_t yylen; /* |yytext| */ + char* yytext; /* string or word */ + long long num; + int tf; + int status; /* NCJ_ERR|NCJ_OK */ +} NCJparser; + +typedef struct NCJbuf { + int len; /* |text|; does not include nul terminator */ + char* text; /* NULL || nul terminated */ +} NCJbuf; + +/**************************************************/ + +#ifdef _WIN32 +#define strdup _strdup +#define strcasecmp _stricmp +#else +#include +#endif + +#ifndef nullfree +#define nullfree(x) {if(x)free(x);} +#endif +#ifndef nulldup +#define nulldup(x) ((x)?strdup(x):(x)) +#endif + +#ifdef NCJDEBUG +static char* tokenname(int token); +#endif + +/**************************************************/ +/* Forward */ +static int NCJparseR(NCJparser* parser, NCjson**); +static int NCJparseArray(NCJparser* parser, struct NCjlist* array); +static int NCJparseDict(NCJparser* parser, struct NCjlist* dict); +static int testbool(const char* word); +static int testint(const char* word); +static int testdouble(const char* word); +static int testnull(const char* word); +static int NCJlex(NCJparser* parser); +static int NCJyytext(NCJparser*, char* start, size_t pdlen); +static void NCJreclaimArray(struct NCjlist*); +static void NCJreclaimDict(struct NCjlist*); +static int NCJunescape(NCJparser* parser); +static int listappend(struct NCjlist* list, NCjson* element); + +#ifndef NETCDF_JSON_H +static int NCJcloneArray(const NCjson* array, NCjson** clonep); +static int NCJcloneDict(const NCjson* dict, NCjson** clonep); +static int NCJunparseR(const NCjson* json, NCJbuf* buf, unsigned flags); +static int bytesappendquoted(NCJbuf* buf, const char* s); +static int bytesappend(NCJbuf* buf, const char* s); +static int bytesappendc(NCJbuf* bufp, const char c); +#endif + +/**************************************************/ + +int +NCJparse(const char* text, unsigned flags, NCjson** jsonp) +{ + int stat = NCJ_OK; + size_t len; + NCJparser* parser = NULL; + NCjson* json = NULL; + + /* Need at least 1 character of input */ + if(text == NULL || text[0] == '\0') + {stat = NCJTHROW(NCJ_ERR); goto done;} + if(jsonp == NULL) goto done; + parser = calloc(1,sizeof(NCJparser)); + if(parser == NULL) + {stat = NCJTHROW(NCJ_ERR); goto done;} + len = strlen(text); + parser->text = (char*)malloc(len+1+1); + if(parser->text == NULL) + {stat = NCJTHROW(NCJ_ERR); goto done;} + strcpy(parser->text,text); + parser->text[len] = '\0'; + parser->text[len+1] = '\0'; + parser->pos = &parser->text[0]; + parser->status = NCJ_OK; +#ifdef NCJDEBUG +fprintf(stderr,"json: |%s|\n",parser->text); +#endif + if((stat=NCJparseR(parser,&json))==NCJ_ERR) goto done; + *jsonp = json; + json = NULL; + +done: + if(parser != NULL) { + nullfree(parser->text); + nullfree(parser->yytext); + free(parser); + } + (void)NCJreclaim(json); + return NCJTHROW(stat); +} + +/* +Simple recursive descent +intertwined with dict and list parsers. + +Invariants: +1. The json argument is provided by caller and filled in by NCJparseR. +2. Each call pushed back last unconsumed token +*/ + +static int +NCJparseR(NCJparser* parser, NCjson** jsonp) +{ + int stat = NCJ_OK; + int token = NCJ_UNDEF; + NCjson* json = NULL; + + if(jsonp == NULL) + {stat = NCJTHROW(NCJ_ERR); goto done;} + if((token = NCJlex(parser)) == NCJ_UNDEF) + {stat = NCJTHROW(NCJ_ERR); goto done;} + switch (token) { + case NCJ_EOF: + break; + case NCJ_NULL: + if((stat = NCJnew(NCJ_NULL,&json))==NCJ_ERR) goto done; + break; + case NCJ_BOOLEAN: + if((stat = NCJnew(NCJ_BOOLEAN,&json))==NCJ_ERR) goto done; + json->string = strdup(parser->yytext); + break; + case NCJ_INT: + if((stat = NCJnew(NCJ_INT,&json))==NCJ_ERR) goto done; + json->string = strdup(parser->yytext); + break; + case NCJ_DOUBLE: + if((stat = NCJnew(NCJ_DOUBLE,&json))==NCJ_ERR) goto done; + json->string = strdup(parser->yytext); + break; + case NCJ_STRING: + if((stat = NCJnew(NCJ_STRING,&json))==NCJ_ERR) goto done; + json->string = strdup(parser->yytext); + break; + case NCJ_LBRACE: + if((stat = NCJnew(NCJ_DICT,&json))==NCJ_ERR) goto done; + if((stat = NCJparseDict(parser, &json->list))==NCJ_ERR) goto done; + break; + case NCJ_LBRACKET: + if((stat = NCJnew(NCJ_ARRAY,&json))==NCJ_ERR) goto done; + if((stat = NCJparseArray(parser, &json->list))==NCJ_ERR) goto done; + break; + case NCJ_RBRACE: /* We hit end of the dict we are parsing */ + parser->pos--; /* pushback so NCJparseArray will catch */ + json = NULL; + break; + case NCJ_RBRACKET: + parser->pos--; /* pushback so NCJparseDict will catch */ + json = NULL; + break; + default: + stat = NCJTHROW(NCJ_ERR); + break; + } + if(jsonp && json) {*jsonp = json; json = NULL;} + +done: + NCJreclaim(json); + return NCJTHROW(stat); +} + +static int +NCJparseArray(NCJparser* parser, struct NCjlist* arrayp) +{ + int stat = NCJ_OK; + int token = NCJ_UNDEF; + NCjson* element = NULL; + int stop = 0; + + /* [ ^e1,e2, ...en] */ + + while(!stop) { + /* Recurse to get the value ei (might be null) */ + if((stat = NCJparseR(parser,&element))==NCJ_ERR) goto done; + token = NCJlex(parser); /* Get next token */ + /* Next token should be comma or rbracket */ + switch(token) { + case NCJ_RBRACKET: + if(element != NULL) listappend(arrayp,element); + element = NULL; + stop = 1; + break; + case NCJ_COMMA: + /* Append the ei to the list */ + if(element == NULL) {stat = NCJTHROW(NCJ_ERR); goto done;} /* error */ + listappend(arrayp,element); + element = NULL; + break; + case NCJ_EOF: + case NCJ_UNDEF: + default: + stat = NCJTHROW(NCJ_ERR); + goto done; + } + } + +done: + if(element != NULL) + NCJreclaim(element); + return NCJTHROW(stat); +} + +static int +NCJparseDict(NCJparser* parser, struct NCjlist* dictp) +{ + int stat = NCJ_OK; + int token = NCJ_UNDEF; + NCjson* value = NULL; + NCjson* key = NULL; + int stop = 0; + + /* { ^k1:v1,k2:v2, ...kn:vn] */ + + while(!stop) { + /* Get the key, which must be a word of some sort */ + token = NCJlex(parser); + switch(token) { + case NCJ_STRING: + case NCJ_BOOLEAN: + case NCJ_INT: case NCJ_DOUBLE: { + if((stat=NCJnewstring(token,parser->yytext,&key))==NCJ_ERR) goto done; + } break; + case NCJ_RBRACE: /* End of containing Dict */ + stop = 1; + continue; /* leave loop */ + case NCJ_EOF: case NCJ_UNDEF: + default: + stat = NCJTHROW(NCJ_ERR); + goto done; + } + /* Next token must be colon*/ + switch((token = NCJlex(parser))) { + case NCJ_COLON: break; + case NCJ_UNDEF: case NCJ_EOF: + default: stat = NCJTHROW(NCJ_ERR); goto done; + } + /* Get the value */ + if((stat = NCJparseR(parser,&value))==NCJ_ERR) goto done; + /* Next token must be comma or RBRACE */ + switch((token = NCJlex(parser))) { + case NCJ_RBRACE: + stop = 1; + /* fall thru */ + case NCJ_COMMA: + /* Insert key value into dict: key first, then value */ + listappend(dictp,key); + key = NULL; + listappend(dictp,value); + value = NULL; + break; + case NCJ_EOF: + case NCJ_UNDEF: + default: + stat = NCJTHROW(NCJ_ERR); + goto done; + } + } + +done: + if(key != NULL) + NCJreclaim(key); + if(value != NULL) + NCJreclaim(value); + return NCJTHROW(stat); +} + +static int +NCJlex(NCJparser* parser) +{ + int c; + int token = NCJ_UNDEF; + char* start; + size_t count; + + while(token == 0) { /* avoid need to goto when retrying */ + c = *parser->pos; + if(c == '\0') { + token = NCJ_EOF; + } else if(c <= ' ' || c == '\177') { + parser->pos++; + continue; /* ignore whitespace */ + } else if(strchr(JSON_WORD, c) != NULL) { + start = parser->pos; + for(;;) { + c = *parser->pos++; + if(c == '\0' || strchr(JSON_WORD,c) == NULL) break; /* end of word */ + } + /* Pushback c if not whitespace */ + parser->pos--; + count = ((parser->pos) - start); + if(NCJyytext(parser,start,count)) goto done; + /* Discriminate the word string to get the proper sort */ + if(testbool(parser->yytext) == NCJ_OK) + token = NCJ_BOOLEAN; + /* do int test first since double subsumes int */ + else if(testint(parser->yytext) == NCJ_OK) + token = NCJ_INT; + else if(testdouble(parser->yytext) == NCJ_OK) + token = NCJ_DOUBLE; + else if(testnull(parser->yytext) == NCJ_OK) + token = NCJ_NULL; + else + token = NCJ_STRING; + } else if(c == NCJ_QUOTE) { + parser->pos++; + start = parser->pos; + for(;;) { + c = *parser->pos++; + if(c == NCJ_ESCAPE) parser->pos++; + else if(c == NCJ_QUOTE || c == '\0') break; + } + if(c == '\0') { + parser->status = NCJ_ERR; + token = NCJ_UNDEF; + goto done; + } + count = ((parser->pos) - start) - 1; /* -1 for trailing quote */ + if(NCJyytext(parser,start,count)==NCJ_ERR) goto done; + if(NCJunescape(parser)==NCJ_ERR) goto done; + token = NCJ_STRING; + } else { /* single char token */ + if(NCJyytext(parser,parser->pos,1)==NCJ_ERR) goto done; + token = *parser->pos++; + } +#ifdef NCJDEBUG +fprintf(stderr,"%s(%d): |%s|\n",tokenname(token),token,parser->yytext); +#endif + } /*for(;;)*/ +done: + if(parser->status == NCJ_ERR) + token = NCJ_UNDEF; + return token; +} + +static int +testnull(const char* word) +{ + if(strcasecmp(word,NCJ_TAG_NULL)==0) + return NCJTHROW(NCJ_OK); + return NCJTHROW(NCJ_ERR); +} + +static int +testbool(const char* word) +{ + if(strcasecmp(word,NCJ_TAG_TRUE)==0 + || strcasecmp(word,NCJ_TAG_FALSE)==0) + return NCJTHROW(NCJ_OK); + return NCJTHROW(NCJ_ERR); +} + +static int +testint(const char* word) +{ + int ncvt; + long long i; + int count = 0; + /* Try to convert to number */ + ncvt = sscanf(word,"%lld%n",&i,&count); + return NCJTHROW((ncvt == 1 && strlen(word)==count ? NCJ_OK : NCJ_ERR)); +} + +static int +testdouble(const char* word) +{ + int ncvt; + double d; + int count = 0; + /* Check for Nan and Infinity */ + if(strcasecmp("nan",word)==0) return NCJTHROW(NCJ_OK); + if(strcasecmp("infinity",word)==0) return NCJTHROW(NCJ_OK); + if(strcasecmp("-infinity",word)==0) return NCJTHROW(NCJ_OK); + /* Allow the XXXf versions as well */ + if(strcasecmp("nanf",word)==0) return NCJTHROW(NCJ_OK); + if(strcasecmp("infinityf",word)==0) return NCJTHROW(NCJ_OK); + if(strcasecmp("-infinityf",word)==0) return NCJTHROW(NCJ_OK); + /* Try to convert to number */ + ncvt = sscanf(word,"%lg%n",&d,&count); + return NCJTHROW((ncvt == 1 && strlen(word)==count ? NCJ_OK : NCJ_ERR)); +} + +static int +NCJyytext(NCJparser* parser, char* start, size_t pdlen) +{ + size_t len = (size_t)pdlen; + if(parser->yytext == NULL) { + parser->yytext = (char*)malloc(len+1); + parser->yylen = len; + } else if(parser->yylen <= len) { + parser->yytext = (char*) realloc(parser->yytext,len+1); + parser->yylen = len; + } + if(parser->yytext == NULL) return NCJTHROW(NCJ_ERR); + memcpy(parser->yytext,start,len); + parser->yytext[len] = '\0'; + return NCJTHROW(NCJ_OK); +} + +/**************************************************/ + +void +NCJreclaim(NCjson* json) +{ + if(json == NULL) return; + switch(json->sort) { + case NCJ_INT: + case NCJ_DOUBLE: + case NCJ_BOOLEAN: + case NCJ_STRING: + nullfree(json->string); + break; + case NCJ_DICT: + NCJreclaimDict(&json->list); + break; + case NCJ_ARRAY: + NCJreclaimArray(&json->list); + break; + default: break; /* nothing to reclaim */ + } + free(json); +} + +static void +NCJreclaimArray(struct NCjlist* array) +{ + int i; + for(i=0;ilen;i++) { + NCJreclaim(array->contents[i]); + } + nullfree(array->contents); + array->contents = NULL; +} + +static void +NCJreclaimDict(struct NCjlist* dict) +{ + NCJreclaimArray(dict); +} + +/**************************************************/ +/* Build Functions */ + +int +NCJnew(int sort, NCjson** objectp) +{ + int stat = NCJ_OK; + NCjson* object = NULL; + + if((object = (NCjson*)calloc(1,sizeof(NCjson))) == NULL) + {stat = NCJTHROW(NCJ_ERR); goto done;} + NCJsetsort(object,sort); + switch (sort) { + case NCJ_INT: + case NCJ_DOUBLE: + case NCJ_BOOLEAN: + case NCJ_STRING: + case NCJ_NULL: + break; + case NCJ_DICT: + case NCJ_ARRAY: + break; + default: + stat = NCJTHROW(NCJ_ERR); + goto done; + } + if(objectp) {*objectp = object; object = NULL;} + +done: + if(stat) NCJreclaim(object); + return NCJTHROW(stat); +} + +int +NCJnewstring(int sort, const char* value, NCjson** jsonp) +{ + return NCJTHROW(NCJnewstringn(sort,strlen(value),value,jsonp)); +} + +int +NCJnewstringn(int sort, size_t len, const char* value, NCjson** jsonp) +{ + int stat = NCJ_OK; + NCjson* json = NULL; + + if(jsonp) *jsonp = NULL; + if(value == NULL) + {stat = NCJTHROW(NCJ_ERR); goto done;} + if((stat = NCJnew(sort,&json))==NCJ_ERR) + goto done; + if((json->string = (char*)malloc(len+1))==NULL) + {stat = NCJTHROW(NCJ_ERR); goto done;} + memcpy(json->string,value,len); + json->string[len] = '\0'; + if(jsonp) *jsonp = json; + json = NULL; /* avoid memory errors */ +done: + NCJreclaim(json); + return NCJTHROW(stat); +} + +int +NCJdictget(const NCjson* dict, const char* key, NCjson** valuep) +{ + int i,stat = NCJ_OK; + + if(dict == NULL || dict->sort != NCJ_DICT) + {stat = NCJTHROW(NCJ_ERR); goto done;} + if(valuep) {*valuep = NULL;} + for(i=0;istring != NULL && strcmp(jkey->string,key)==0) { + if(valuep) {*valuep = NCJith(dict,i+1); break;} + } + } + +done: + return NCJTHROW(stat); +} + +/* Unescape the text in parser->yytext; can + do in place because unescaped string will + always be shorter */ +static int +NCJunescape(NCJparser* parser) +{ + char* p = parser->yytext; + char* q = p; + int c; + for(;(c=*p++);) { + if(c == NCJ_ESCAPE) { + c = *p++; + switch (c) { + case 'b': c = '\b'; break; + case 'f': c = '\f'; break; + case 'n': c = '\n'; break; + case 'r': c = '\r'; break; + case 't': c = '\t'; break; + case NCJ_QUOTE: c = c; break; + case NCJ_ESCAPE: c = c; break; + default: c = c; break;/* technically not Json conformant */ + } + } + *q++ = c; + } + *q = '\0'; + return NCJTHROW(NCJ_OK); +} + +#ifdef NCJDEBUG +static char* +tokenname(int token) +{ + switch (token) { + case NCJ_STRING: return ("NCJ_STRING"); + case NCJ_INT: return ("NCJ_INT"); + case NCJ_DOUBLE: return ("NCJ_DOUBLE"); + case NCJ_BOOLEAN: return ("NCJ_BOOLEAN"); + case NCJ_DICT: return ("NCJ_DICT"); + case NCJ_ARRAY: return ("NCJ_ARRAY"); + case NCJ_NULL: return ("NCJ_NULL"); + default: + if(token > ' ' && token <= 127) { + static char s[4]; + s[0] = '\''; + s[1] = (char)token; + s[2] = '\''; + s[3] = '\0'; + return (s); + } else + break; + } + return ("NCJ_UNDEF"); +} +#endif + +/* Convert a JSON value to an equivalent value of a specified sort */ +int +NCJcvt(const NCjson* jvalue, int outsort, struct NCJconst* output) +{ + int stat = NCJ_OK; + + if(output == NULL) goto done; + +#undef CASE +#define CASE(t1,t2) ((t1)<<4 | (t2)) /* the shift constant must be larger than log2(NCJ_NSORTS) */ + switch (CASE(jvalue->sort,outsort)) { + + case CASE(NCJ_BOOLEAN,NCJ_BOOLEAN): + if(strcasecmp(jvalue->string,NCJ_TAG_FALSE)==0) output->bval = 0; else output->bval = 1; + break; + case CASE(NCJ_BOOLEAN,NCJ_INT): + if(strcasecmp(jvalue->string,NCJ_TAG_FALSE)==0) output->ival = 0; else output->ival = 1; + break; + case CASE(NCJ_BOOLEAN,NCJ_DOUBLE): + if(strcasecmp(jvalue->string,NCJ_TAG_FALSE)==0) output->dval = 0.0; else output->dval = 1.0; + break; + case CASE(NCJ_BOOLEAN,NCJ_STRING): + output->sval = nulldup(jvalue->string); + break; + + case CASE(NCJ_INT,NCJ_BOOLEAN): + sscanf(jvalue->string,"%lldd",&output->ival); + output->bval = (output->ival?1:0); + break; + case CASE(NCJ_INT,NCJ_INT): + sscanf(jvalue->string,"%lld",&output->ival); + break; + case CASE(NCJ_INT,NCJ_DOUBLE): + sscanf(jvalue->string,"%lld",&output->ival); + output->dval = (double)output->ival; + break; + case CASE(NCJ_INT,NCJ_STRING): + output->sval = nulldup(jvalue->string); + break; + + case CASE(NCJ_DOUBLE,NCJ_BOOLEAN): + sscanf(jvalue->string,"%lf",&output->dval); + output->bval = (output->dval == 0?0:1); + break; + case CASE(NCJ_DOUBLE,NCJ_INT): + sscanf(jvalue->string,"%lf",&output->dval); + output->ival = (long long)output->dval; + break; + case CASE(NCJ_DOUBLE,NCJ_DOUBLE): + sscanf(jvalue->string,"%lf",&output->dval); + break; + case CASE(NCJ_DOUBLE,NCJ_STRING): + output->sval = nulldup(jvalue->string); + break; + + case CASE(NCJ_STRING,NCJ_BOOLEAN): + if(strcasecmp(jvalue->string,NCJ_TAG_FALSE)==0) output->bval = 0; else output->bval = 1; + break; + case CASE(NCJ_STRING,NCJ_INT): + sscanf(jvalue->string,"%lld",&output->ival); + break; + case CASE(NCJ_STRING,NCJ_DOUBLE): + sscanf(jvalue->string,"%lf",&output->dval); + break; + case CASE(NCJ_STRING,NCJ_STRING): + output->sval = nulldup(jvalue->string); + break; + + default: + stat = NCJTHROW(NCJ_ERR); + break; + } + +done: + return NCJTHROW(stat); +} + +static int +listappend(struct NCjlist* list, NCjson* json) +{ + int stat = NCJ_OK; + NCjson** newcontents = NULL; + + assert(list->len == 0 || list->contents != NULL); + if(json == NULL) + {stat = NCJTHROW(NCJ_ERR); goto done;} + if(list->len == 0) { + nullfree(list->contents); + list->contents = (NCjson**)calloc(2,sizeof(NCjson*)); + if(list->contents == NULL) + {stat = NCJTHROW(NCJ_ERR); goto done;} + list->contents[0] = json; + list->len++; + } else { + if((newcontents = (NCjson**)calloc((2*list->len)+1,sizeof(NCjson*)))==NULL) + {stat = NCJTHROW(NCJ_ERR); goto done;} + memcpy(newcontents,list->contents,list->len*sizeof(NCjson*)); + newcontents[list->len] = json; + list->len++; + free(list->contents); + list->contents = newcontents; newcontents = NULL; + } + +done: + nullfree(newcontents); + return NCJTHROW(stat); +} + +/**************************************************/ + +#ifndef NETCDF_JSON_H + +int +NCJclone(const NCjson* json, NCjson** clonep) +{ + int stat = NCJ_OK; + NCjson* clone = NULL; + if(json == NULL) goto done; + switch(NCJsort(json)) { + case NCJ_INT: + case NCJ_DOUBLE: + case NCJ_BOOLEAN: + case NCJ_STRING: + if((stat=NCJnew(NCJsort(json),&clone))==NCJ_ERR) goto done; + if((NCJstring(clone) = strdup(NCJstring(json))) == NULL) + {stat = NCJTHROW(NCJ_ERR); goto done;} + break; + case NCJ_NULL: + if((stat=NCJnew(NCJsort(json),&clone))==NCJ_ERR) goto done; + break; + case NCJ_DICT: + if((stat=NCJcloneDict(json,&clone))==NCJ_ERR) goto done; + break; + case NCJ_ARRAY: + if((stat=NCJcloneArray(json,&clone))==NCJ_ERR) goto done; + break; + default: break; /* nothing to clone */ + } +done: + if(stat == NCJ_OK && clonep) {*clonep = clone; clone = NULL;} + NCJreclaim(clone); + return NCJTHROW(stat); +} + +static int +NCJcloneArray(const NCjson* array, NCjson** clonep) +{ + int i, stat=NCJ_OK; + NCjson* clone = NULL; + if((stat=NCJnew(NCJ_ARRAY,&clone))==NCJ_ERR) goto done; + for(i=0;isort != NCJ_DICT || key == NULL || jvalue == NULL) + {stat = NCJTHROW(NCJ_ERR); goto done;} + if((stat = NCJnewstring(NCJ_STRING,key,&jkey))==NCJ_ERR) goto done; + if((stat = NCJappend(object,jkey))==NCJ_ERR) goto done; + if((stat = NCJappend(object,jvalue))==NCJ_ERR) goto done; +done: + return NCJTHROW(stat); +} + +/* Append value to an array or dict object. */ +int +NCJappend(NCjson* object, NCjson* value) +{ + if(object == NULL || value == NULL) + return NCJTHROW(NCJ_ERR); + switch (object->sort) { + case NCJ_ARRAY: + case NCJ_DICT: + listappend(&object->list,value); + break; + default: + return NCJTHROW(NCJ_ERR); + } + return NCJTHROW(NCJ_OK); +} + +/**************************************************/ +/* Unparser to convert NCjson object to text in buffer */ + +int +NCJunparse(const NCjson* json, unsigned flags, char** textp) +{ + int stat = NCJ_OK; + NCJbuf buf = {0,NULL}; + if((stat = NCJunparseR(json,&buf,flags))==NCJ_ERR) + goto done; + if(textp) {*textp = buf.text; buf.text = NULL; buf.len = 0;} +done: + nullfree(buf.text); + return NCJTHROW(stat); +} + +static int +NCJunparseR(const NCjson* json, NCJbuf* buf, unsigned flags) +{ + int stat = NCJ_OK; + int i; + + switch (NCJsort(json)) { + case NCJ_STRING: + bytesappendquoted(buf,json->string); + break; + case NCJ_INT: + case NCJ_DOUBLE: + case NCJ_BOOLEAN: + bytesappend(buf,json->string); + break; + case NCJ_DICT: + bytesappendc(buf,NCJ_LBRACE); + if(json->list.len > 0 && json->list.contents != NULL) { + int shortlist = 0; + for(i=0;!shortlist && i < json->list.len;i+=2) { + if(i > 0) bytesappendc(buf,NCJ_COMMA); + NCJunparseR(json->list.contents[i],buf,flags); /* key */ + bytesappendc(buf,NCJ_COLON); + bytesappendc(buf,' '); + /* Allow for the possibility of a short dict entry */ + if(json->list.contents[i+1] == NULL) { /* short */ + bytesappendc(buf,'?'); + shortlist = 1; + } else { + NCJunparseR(json->list.contents[i+1],buf,flags); + } + } + } + bytesappendc(buf,NCJ_RBRACE); + break; + case NCJ_ARRAY: + bytesappendc(buf,NCJ_LBRACKET); + if(json->list.len > 0 && json->list.contents != NULL) { + for(i=0;i < json->list.len;i++) { + if(i > 0) bytesappendc(buf,NCJ_COMMA); + NCJunparseR(json->list.contents[i],buf,flags); + } + } + bytesappendc(buf,NCJ_RBRACKET); + break; + case NCJ_NULL: + bytesappend(buf,"null"); + break; + default: + stat = NCJTHROW(NCJ_ERR); goto done; + } +done: + return NCJTHROW(stat); +} + +/* Escape a string and append to buf */ +static int +escape(const char* text, NCJbuf* buf) +{ + const char* p = text; + int c; + for(;(c=*p++);) { + char replace = 0; + switch (c) { + case '\b': replace = 'b'; break; + case '\f': replace = 'f'; break; + case '\n': replace = 'n'; break; + case '\r': replace = 'r'; break; + case '\t': replace = 't'; break; + case NCJ_QUOTE: replace = '\''; break; + case NCJ_ESCAPE: replace = '\\'; break; + default: break; + } + if(replace) { + bytesappendc(buf,NCJ_ESCAPE); + bytesappendc(buf,replace); + } else + bytesappendc(buf,c); + } + return NCJTHROW(NCJ_OK); +} + +static int +bytesappendquoted(NCJbuf* buf, const char* s) +{ + bytesappend(buf,"\""); + escape(s,buf); + bytesappend(buf,"\""); + return NCJTHROW(NCJ_OK); +} + +void +NCJdump(const NCjson* json, unsigned flags, FILE* out) +{ + char* text = NULL; + (void)NCJunparse(json,0,&text); + if(out == NULL) out = stderr; + fprintf(out,"%s\n",text); + fflush(out); + nullfree(text); +} + +static int +bytesappend(NCJbuf* buf, const char* s) +{ + int stat = NCJ_OK; + char* newtext = NULL; + if(buf == NULL) + {stat = NCJTHROW(NCJ_ERR); goto done;} + if(s == NULL) s = ""; + if(buf->len == 0) { + assert(buf->text == NULL); + buf->text = strdup(s); + if(buf->text == NULL) + {stat = NCJTHROW(NCJ_ERR); goto done;} + buf->len = strlen(s); + } else { + size_t slen = strlen(s); + size_t newlen = buf->len + slen + 1; + if((newtext = (char*)malloc(newlen))==NULL) + {stat = NCJTHROW(NCJ_ERR); goto done;} + strcpy(newtext,buf->text); + strcat(newtext,s); + free(buf->text); buf->text = NULL; + buf->text = newtext; newtext = NULL; + buf->len = newlen; + } + +done: + nullfree(newtext); + return NCJTHROW(stat); +} + +static int +bytesappendc(NCJbuf* bufp, const char c) +{ + char s[2]; + s[0] = c; + s[1] = '\0'; + return bytesappend(bufp,s); +} +#endif /*!NETCDF_JSON_H*/ diff --git a/libdap2/dce.y b/libdap2/dce.y index 21b7b14069..fe37639d66 100644 --- a/libdap2/dce.y +++ b/libdap2/dce.y @@ -5,7 +5,7 @@ They will be commennted out when building a java parser. */ -%pure-parser +%define api.pure %lex-param {DCEparsestate* parsestate} %parse-param {DCEparsestate* parsestate} %{ diff --git a/libdap2/dcetab.c b/libdap2/dcetab.c index 60a602449d..909d13a65d 100644 --- a/libdap2/dcetab.c +++ b/libdap2/dcetab.c @@ -1,8 +1,9 @@ -/* A Bison parser, made by GNU Bison 3.0.4. */ +/* A Bison parser, made by GNU Bison 3.7.6. */ /* Bison implementation for Yacc-like parsers in C - Copyright (C) 1984, 1989-1990, 2000-2015 Free Software Foundation, Inc. + Copyright (C) 1984, 1989-1990, 2000-2015, 2018-2021 Free Software Foundation, + Inc. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -15,7 +16,7 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License - along with this program. If not, see . */ + along with this program. If not, see . */ /* As a special exception, you may create a larger work that contains part or all of the Bison parser skeleton and distribute that work @@ -33,6 +34,10 @@ /* C LALR(1) parser skeleton written by Richard Stallman, by simplifying the original so-called "semantic" parser. */ +/* DO NOT RELY ON FEATURES THAT ARE NOT DOCUMENTED in the manual, + especially those whose name start with YY_ or yy_. They are + private implementation details that can be changed or removed. */ + /* All symbols defined below should begin with yy or YY, to avoid infringing on user name space. This should be done even for local variables, as they might otherwise be expanded by user macros. @@ -40,11 +45,11 @@ define necessary library symbols; they are noted "INFRINGES ON USER NAME SPACE" below. */ -/* Identify Bison output. */ -#define YYBISON 1 +/* Identify Bison output, and Bison version. */ +#define YYBISON 30706 -/* Bison version. */ -#define YYBISON_VERSION "3.0.4" +/* Bison version string. */ +#define YYBISON_VERSION "3.7.6" /* Skeleton name. */ #define YYSKELETON_NAME "yacc.c" @@ -66,9 +71,8 @@ #define yydebug dcedebug #define yynerrs dcenerrs - -/* Copy the first part of user declarations. */ -#line 11 "dce.y" /* yacc.c:339 */ +/* First part of user prologue. */ +#line 11 "dce.y" #ifdef HAVE_CONFIG_H #include "config.h" @@ -80,90 +84,176 @@ #include "dceconstraints.h" #include "dceparselex.h" -#line 84 "dcetab.c" /* yacc.c:339 */ +#line 88 "dcetab.c" +# ifndef YY_CAST +# ifdef __cplusplus +# define YY_CAST(Type, Val) static_cast (Val) +# define YY_REINTERPRET_CAST(Type, Val) reinterpret_cast (Val) +# else +# define YY_CAST(Type, Val) ((Type) (Val)) +# define YY_REINTERPRET_CAST(Type, Val) ((Type) (Val)) +# endif +# endif # ifndef YY_NULLPTR -# if defined __cplusplus && 201103L <= __cplusplus -# define YY_NULLPTR nullptr +# if defined __cplusplus +# if 201103L <= __cplusplus +# define YY_NULLPTR nullptr +# else +# define YY_NULLPTR 0 +# endif # else -# define YY_NULLPTR 0 +# define YY_NULLPTR ((void*)0) # endif # endif -/* Enabling verbose error messages. */ -#ifdef YYERROR_VERBOSE -# undef YYERROR_VERBOSE -# define YYERROR_VERBOSE 1 -#else -# define YYERROR_VERBOSE 0 -#endif - -/* In a future release of Bison, this section will be replaced - by #include "dcetab.h". */ -#ifndef YY_DCE_DCE_TAB_H_INCLUDED -# define YY_DCE_DCE_TAB_H_INCLUDED -/* Debug traces. */ -#ifndef YYDEBUG -# define YYDEBUG 1 -#endif -#if YYDEBUG -extern int dcedebug; -#endif - -/* Token type. */ -#ifndef YYTOKENTYPE -# define YYTOKENTYPE - enum yytokentype - { - SCAN_WORD = 258, - SCAN_STRINGCONST = 259, - SCAN_NUMBERCONST = 260 - }; -#endif +#include "dcetab.h" +/* Symbol kind. */ +enum yysymbol_kind_t +{ + YYSYMBOL_YYEMPTY = -2, + YYSYMBOL_YYEOF = 0, /* "end of file" */ + YYSYMBOL_YYerror = 1, /* error */ + YYSYMBOL_YYUNDEF = 2, /* "invalid token" */ + YYSYMBOL_SCAN_WORD = 3, /* SCAN_WORD */ + YYSYMBOL_SCAN_STRINGCONST = 4, /* SCAN_STRINGCONST */ + YYSYMBOL_SCAN_NUMBERCONST = 5, /* SCAN_NUMBERCONST */ + YYSYMBOL_6_ = 6, /* '?' */ + YYSYMBOL_7_ = 7, /* ',' */ + YYSYMBOL_8_ = 8, /* '(' */ + YYSYMBOL_9_ = 9, /* ')' */ + YYSYMBOL_10_ = 10, /* '.' */ + YYSYMBOL_11_ = 11, /* '[' */ + YYSYMBOL_12_ = 12, /* ']' */ + YYSYMBOL_13_ = 13, /* ':' */ + YYSYMBOL_14_ = 14, /* '&' */ + YYSYMBOL_15_ = 15, /* '{' */ + YYSYMBOL_16_ = 16, /* '}' */ + YYSYMBOL_17_ = 17, /* '=' */ + YYSYMBOL_18_ = 18, /* '>' */ + YYSYMBOL_19_ = 19, /* '<' */ + YYSYMBOL_20_ = 20, /* '!' */ + YYSYMBOL_21_ = 21, /* '~' */ + YYSYMBOL_YYACCEPT = 22, /* $accept */ + YYSYMBOL_constraints = 23, /* constraints */ + YYSYMBOL_optquestionmark = 24, /* optquestionmark */ + YYSYMBOL_projections = 25, /* projections */ + YYSYMBOL_selections = 26, /* selections */ + YYSYMBOL_projectionlist = 27, /* projectionlist */ + YYSYMBOL_projection = 28, /* projection */ + YYSYMBOL_function = 29, /* function */ + YYSYMBOL_segmentlist = 30, /* segmentlist */ + YYSYMBOL_segment = 31, /* segment */ + YYSYMBOL_rangelist = 32, /* rangelist */ + YYSYMBOL_range = 33, /* range */ + YYSYMBOL_range1 = 34, /* range1 */ + YYSYMBOL_clauselist = 35, /* clauselist */ + YYSYMBOL_sel_clause = 36, /* sel_clause */ + YYSYMBOL_value_list = 37, /* value_list */ + YYSYMBOL_value = 38, /* value */ + YYSYMBOL_constant = 39, /* constant */ + YYSYMBOL_var = 40, /* var */ + YYSYMBOL_indexpath = 41, /* indexpath */ + YYSYMBOL_index = 42, /* index */ + YYSYMBOL_array_indices = 43, /* array_indices */ + YYSYMBOL_boolfunction = 44, /* boolfunction */ + YYSYMBOL_arg_list = 45, /* arg_list */ + YYSYMBOL_rel_op = 46, /* rel_op */ + YYSYMBOL_ident = 47, /* ident */ + YYSYMBOL_word = 48, /* word */ + YYSYMBOL_number = 49, /* number */ + YYSYMBOL_string = 50 /* string */ +}; +typedef enum yysymbol_kind_t yysymbol_kind_t; -/* Value type. */ -#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED -typedef int YYSTYPE; -# define YYSTYPE_IS_TRIVIAL 1 -# define YYSTYPE_IS_DECLARED 1 -#endif -int dceparse (DCEparsestate* parsestate); +#ifdef short +# undef short +#endif -#endif /* !YY_DCE_DCE_TAB_H_INCLUDED */ +/* On compilers that do not define __PTRDIFF_MAX__ etc., make sure + and (if available) are included + so that the code can choose integer types of a good width. */ -/* Copy the second part of user declarations. */ +#ifndef __PTRDIFF_MAX__ +# include /* INFRINGES ON USER NAME SPACE */ +# if defined __STDC_VERSION__ && 199901 <= __STDC_VERSION__ +# include /* INFRINGES ON USER NAME SPACE */ +# define YY_STDINT_H +# endif +#endif -#line 140 "dcetab.c" /* yacc.c:358 */ +/* Narrow types that promote to a signed type and that can represent a + signed or unsigned integer of at least N bits. In tables they can + save space and decrease cache pressure. Promoting to a signed type + helps avoid bugs in integer arithmetic. */ -#ifdef short -# undef short +#ifdef __INT_LEAST8_MAX__ +typedef __INT_LEAST8_TYPE__ yytype_int8; +#elif defined YY_STDINT_H +typedef int_least8_t yytype_int8; +#else +typedef signed char yytype_int8; #endif -#ifdef YYTYPE_UINT8 -typedef YYTYPE_UINT8 yytype_uint8; +#ifdef __INT_LEAST16_MAX__ +typedef __INT_LEAST16_TYPE__ yytype_int16; +#elif defined YY_STDINT_H +typedef int_least16_t yytype_int16; #else -typedef unsigned char yytype_uint8; +typedef short yytype_int16; #endif -#ifdef YYTYPE_INT8 -typedef YYTYPE_INT8 yytype_int8; -#else -typedef signed char yytype_int8; +/* Work around bug in HP-UX 11.23, which defines these macros + incorrectly for preprocessor constants. This workaround can likely + be removed in 2023, as HPE has promised support for HP-UX 11.23 + (aka HP-UX 11i v2) only through the end of 2022; see Table 2 of + . */ +#ifdef __hpux +# undef UINT_LEAST8_MAX +# undef UINT_LEAST16_MAX +# define UINT_LEAST8_MAX 255 +# define UINT_LEAST16_MAX 65535 #endif -#ifdef YYTYPE_UINT16 -typedef YYTYPE_UINT16 yytype_uint16; +#if defined __UINT_LEAST8_MAX__ && __UINT_LEAST8_MAX__ <= __INT_MAX__ +typedef __UINT_LEAST8_TYPE__ yytype_uint8; +#elif (!defined __UINT_LEAST8_MAX__ && defined YY_STDINT_H \ + && UINT_LEAST8_MAX <= INT_MAX) +typedef uint_least8_t yytype_uint8; +#elif !defined __UINT_LEAST8_MAX__ && UCHAR_MAX <= INT_MAX +typedef unsigned char yytype_uint8; #else -typedef unsigned short int yytype_uint16; +typedef short yytype_uint8; #endif -#ifdef YYTYPE_INT16 -typedef YYTYPE_INT16 yytype_int16; +#if defined __UINT_LEAST16_MAX__ && __UINT_LEAST16_MAX__ <= __INT_MAX__ +typedef __UINT_LEAST16_TYPE__ yytype_uint16; +#elif (!defined __UINT_LEAST16_MAX__ && defined YY_STDINT_H \ + && UINT_LEAST16_MAX <= INT_MAX) +typedef uint_least16_t yytype_uint16; +#elif !defined __UINT_LEAST16_MAX__ && USHRT_MAX <= INT_MAX +typedef unsigned short yytype_uint16; #else -typedef short int yytype_int16; +typedef int yytype_uint16; +#endif + +#ifndef YYPTRDIFF_T +# if defined __PTRDIFF_TYPE__ && defined __PTRDIFF_MAX__ +# define YYPTRDIFF_T __PTRDIFF_TYPE__ +# define YYPTRDIFF_MAXIMUM __PTRDIFF_MAX__ +# elif defined PTRDIFF_MAX +# ifndef ptrdiff_t +# include /* INFRINGES ON USER NAME SPACE */ +# endif +# define YYPTRDIFF_T ptrdiff_t +# define YYPTRDIFF_MAXIMUM PTRDIFF_MAX +# else +# define YYPTRDIFF_T long +# define YYPTRDIFF_MAXIMUM LONG_MAX +# endif #endif #ifndef YYSIZE_T @@ -171,15 +261,28 @@ typedef short int yytype_int16; # define YYSIZE_T __SIZE_TYPE__ # elif defined size_t # define YYSIZE_T size_t -# elif ! defined YYSIZE_T +# elif defined __STDC_VERSION__ && 199901 <= __STDC_VERSION__ # include /* INFRINGES ON USER NAME SPACE */ # define YYSIZE_T size_t # else -# define YYSIZE_T unsigned int +# define YYSIZE_T unsigned # endif #endif -#define YYSIZE_MAXIMUM ((YYSIZE_T) -1) +#define YYSIZE_MAXIMUM \ + YY_CAST (YYPTRDIFF_T, \ + (YYPTRDIFF_MAXIMUM < YY_CAST (YYSIZE_T, -1) \ + ? YYPTRDIFF_MAXIMUM \ + : YY_CAST (YYSIZE_T, -1))) + +#define YYSIZEOF(X) YY_CAST (YYPTRDIFF_T, sizeof (X)) + + +/* Stored state numbers (used for stacks). */ +typedef yytype_int8 yy_state_t; + +/* State numbers in computations. */ +typedef int yy_state_fast_t; #ifndef YY_ # if defined YYENABLE_NLS && YYENABLE_NLS @@ -193,47 +296,37 @@ typedef short int yytype_int16; # endif #endif -#ifndef YY_ATTRIBUTE -# if (defined __GNUC__ \ - && (2 < __GNUC__ || (__GNUC__ == 2 && 96 <= __GNUC_MINOR__))) \ - || defined __SUNPRO_C && 0x5110 <= __SUNPRO_C -# define YY_ATTRIBUTE(Spec) __attribute__(Spec) -# else -# define YY_ATTRIBUTE(Spec) /* empty */ -# endif -#endif #ifndef YY_ATTRIBUTE_PURE -# define YY_ATTRIBUTE_PURE YY_ATTRIBUTE ((__pure__)) +# if defined __GNUC__ && 2 < __GNUC__ + (96 <= __GNUC_MINOR__) +# define YY_ATTRIBUTE_PURE __attribute__ ((__pure__)) +# else +# define YY_ATTRIBUTE_PURE +# endif #endif #ifndef YY_ATTRIBUTE_UNUSED -# define YY_ATTRIBUTE_UNUSED YY_ATTRIBUTE ((__unused__)) -#endif - -#if !defined _Noreturn \ - && (!defined __STDC_VERSION__ || __STDC_VERSION__ < 201112) -# if defined _MSC_VER && 1200 <= _MSC_VER -# define _Noreturn __declspec (noreturn) +# if defined __GNUC__ && 2 < __GNUC__ + (7 <= __GNUC_MINOR__) +# define YY_ATTRIBUTE_UNUSED __attribute__ ((__unused__)) # else -# define _Noreturn YY_ATTRIBUTE ((__noreturn__)) +# define YY_ATTRIBUTE_UNUSED # endif #endif /* Suppress unused-variable warnings by "using" E. */ #if ! defined lint || defined __GNUC__ -# define YYUSE(E) ((void) (E)) +# define YY_USE(E) ((void) (E)) #else -# define YYUSE(E) /* empty */ +# define YY_USE(E) /* empty */ #endif -#if defined __GNUC__ && 407 <= __GNUC__ * 100 + __GNUC_MINOR__ +#if defined __GNUC__ && ! defined __ICC && 407 <= __GNUC__ * 100 + __GNUC_MINOR__ /* Suppress an incorrect diagnostic about yylval being uninitialized. */ -# define YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN \ - _Pragma ("GCC diagnostic push") \ - _Pragma ("GCC diagnostic ignored \"-Wuninitialized\"")\ +# define YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN \ + _Pragma ("GCC diagnostic push") \ + _Pragma ("GCC diagnostic ignored \"-Wuninitialized\"") \ _Pragma ("GCC diagnostic ignored \"-Wmaybe-uninitialized\"") -# define YY_IGNORE_MAYBE_UNINITIALIZED_END \ +# define YY_IGNORE_MAYBE_UNINITIALIZED_END \ _Pragma ("GCC diagnostic pop") #else # define YY_INITIAL_VALUE(Value) Value @@ -246,8 +339,22 @@ typedef short int yytype_int16; # define YY_INITIAL_VALUE(Value) /* Nothing. */ #endif +#if defined __cplusplus && defined __GNUC__ && ! defined __ICC && 6 <= __GNUC__ +# define YY_IGNORE_USELESS_CAST_BEGIN \ + _Pragma ("GCC diagnostic push") \ + _Pragma ("GCC diagnostic ignored \"-Wuseless-cast\"") +# define YY_IGNORE_USELESS_CAST_END \ + _Pragma ("GCC diagnostic pop") +#endif +#ifndef YY_IGNORE_USELESS_CAST_BEGIN +# define YY_IGNORE_USELESS_CAST_BEGIN +# define YY_IGNORE_USELESS_CAST_END +#endif + + +#define YY_ASSERT(E) ((void) (0 && (E))) -#if ! defined yyoverflow || YYERROR_VERBOSE +#if !defined yyoverflow /* The parser invokes alloca or malloc; define the necessary symbols. */ @@ -312,8 +419,7 @@ void free (void *); /* INFRINGES ON USER NAME SPACE */ # endif # endif # endif -#endif /* ! defined yyoverflow || YYERROR_VERBOSE */ - +#endif /* !defined yyoverflow */ #if (! defined yyoverflow \ && (! defined __cplusplus \ @@ -322,17 +428,17 @@ void free (void *); /* INFRINGES ON USER NAME SPACE */ /* A type that is properly aligned for any stack member. */ union yyalloc { - yytype_int16 yyss_alloc; + yy_state_t yyss_alloc; YYSTYPE yyvs_alloc; }; /* The size of the maximum gap between one aligned stack and the next. */ -# define YYSTACK_GAP_MAXIMUM (sizeof (union yyalloc) - 1) +# define YYSTACK_GAP_MAXIMUM (YYSIZEOF (union yyalloc) - 1) /* The size of an array large to enough to hold all stacks, each with N elements. */ # define YYSTACK_BYTES(N) \ - ((N) * (sizeof (yytype_int16) + sizeof (YYSTYPE)) \ + ((N) * (YYSIZEOF (yy_state_t) + YYSIZEOF (YYSTYPE)) \ + YYSTACK_GAP_MAXIMUM) # define YYCOPY_NEEDED 1 @@ -345,11 +451,11 @@ union yyalloc # define YYSTACK_RELOCATE(Stack_alloc, Stack) \ do \ { \ - YYSIZE_T yynewbytes; \ + YYPTRDIFF_T yynewbytes; \ YYCOPY (&yyptr->Stack_alloc, Stack, yysize); \ Stack = &yyptr->Stack_alloc; \ - yynewbytes = yystacksize * sizeof (*Stack) + YYSTACK_GAP_MAXIMUM; \ - yyptr += yynewbytes / sizeof (*yyptr); \ + yynewbytes = yystacksize * YYSIZEOF (*Stack) + YYSTACK_GAP_MAXIMUM; \ + yyptr += yynewbytes / YYSIZEOF (*yyptr); \ } \ while (0) @@ -361,12 +467,12 @@ union yyalloc # ifndef YYCOPY # if defined __GNUC__ && 1 < __GNUC__ # define YYCOPY(Dst, Src, Count) \ - __builtin_memcpy (Dst, Src, (Count) * sizeof (*(Src))) + __builtin_memcpy (Dst, Src, YY_CAST (YYSIZE_T, (Count)) * sizeof (*(Src))) # else # define YYCOPY(Dst, Src, Count) \ do \ { \ - YYSIZE_T yyi; \ + YYPTRDIFF_T yyi; \ for (yyi = 0; yyi < (Count); yyi++) \ (Dst)[yyi] = (Src)[yyi]; \ } \ @@ -389,17 +495,20 @@ union yyalloc /* YYNSTATES -- Number of states. */ #define YYNSTATES 87 -/* YYTRANSLATE[YYX] -- Symbol number corresponding to YYX as returned - by yylex, with out-of-bounds checking. */ -#define YYUNDEFTOK 2 +/* YYMAXUTOK -- Last valid token kind. */ #define YYMAXUTOK 260 -#define YYTRANSLATE(YYX) \ - ((unsigned int) (YYX) <= YYMAXUTOK ? yytranslate[YYX] : YYUNDEFTOK) + +/* YYTRANSLATE(TOKEN-NUM) -- Symbol number corresponding to TOKEN-NUM + as returned by yylex, with out-of-bounds checking. */ +#define YYTRANSLATE(YYX) \ + (0 <= (YYX) && (YYX) <= YYMAXUTOK \ + ? YY_CAST (yysymbol_kind_t, yytranslate[YYX]) \ + : YYSYMBOL_YYUNDEF) /* YYTRANSLATE[TOKEN-NUM] -- Symbol number corresponding to TOKEN-NUM - as returned by yylex, without out-of-bounds checking. */ -static const yytype_uint8 yytranslate[] = + as returned by yylex. */ +static const yytype_int8 yytranslate[] = { 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, @@ -443,42 +552,55 @@ static const yytype_uint8 yyrline[] = }; #endif -#if YYDEBUG || YYERROR_VERBOSE || 0 +/** Accessing symbol of state STATE. */ +#define YY_ACCESSING_SYMBOL(State) YY_CAST (yysymbol_kind_t, yystos[State]) + +#if YYDEBUG || 0 +/* The user-facing name of the symbol whose (internal) number is + YYSYMBOL. No bounds checking. */ +static const char *yysymbol_name (yysymbol_kind_t yysymbol) YY_ATTRIBUTE_UNUSED; + /* YYTNAME[SYMBOL-NUM] -- String name of the symbol SYMBOL-NUM. First, the terminals, then, starting at YYNTOKENS, nonterminals. */ static const char *const yytname[] = { - "$end", "error", "$undefined", "SCAN_WORD", "SCAN_STRINGCONST", - "SCAN_NUMBERCONST", "'?'", "','", "'('", "')'", "'.'", "'['", "']'", - "':'", "'&'", "'{'", "'}'", "'='", "'>'", "'<'", "'!'", "'~'", "$accept", - "constraints", "optquestionmark", "projections", "selections", - "projectionlist", "projection", "function", "segmentlist", "segment", - "rangelist", "range", "range1", "clauselist", "sel_clause", "value_list", - "value", "constant", "var", "indexpath", "index", "array_indices", - "boolfunction", "arg_list", "rel_op", "ident", "word", "number", - "string", YY_NULLPTR + "\"end of file\"", "error", "\"invalid token\"", "SCAN_WORD", + "SCAN_STRINGCONST", "SCAN_NUMBERCONST", "'?'", "','", "'('", "')'", + "'.'", "'['", "']'", "':'", "'&'", "'{'", "'}'", "'='", "'>'", "'<'", + "'!'", "'~'", "$accept", "constraints", "optquestionmark", "projections", + "selections", "projectionlist", "projection", "function", "segmentlist", + "segment", "rangelist", "range", "range1", "clauselist", "sel_clause", + "value_list", "value", "constant", "var", "indexpath", "index", + "array_indices", "boolfunction", "arg_list", "rel_op", "ident", "word", + "number", "string", YY_NULLPTR }; + +static const char * +yysymbol_name (yysymbol_kind_t yysymbol) +{ + return yytname[yysymbol]; +} #endif -# ifdef YYPRINT +#ifdef YYPRINT /* YYTOKNUM[NUM] -- (External) token number corresponding to the (internal) symbol number NUM (which must be that of a token). */ -static const yytype_uint16 yytoknum[] = +static const yytype_int16 yytoknum[] = { 0, 256, 257, 258, 259, 260, 63, 44, 40, 41, 46, 91, 93, 58, 38, 123, 125, 61, 62, 60, 33, 126 }; -# endif +#endif -#define YYPACT_NINF -36 +#define YYPACT_NINF (-36) -#define yypact_value_is_default(Yystate) \ - (!!((Yystate) == (-36))) +#define yypact_value_is_default(Yyn) \ + ((Yyn) == YYPACT_NINF) -#define YYTABLE_NINF -57 +#define YYTABLE_NINF (-57) -#define yytable_value_is_error(Yytable_value) \ +#define yytable_value_is_error(Yyn) \ 0 /* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing @@ -499,7 +621,7 @@ static const yytype_int8 yypact[] = /* YYDEFACT[STATE-NUM] -- Default reduction number in state STATE-NUM. Performed when YYTABLE does not specify something else to do. Zero means the default is an error. */ -static const yytype_uint8 yydefact[] = +static const yytype_int8 yydefact[] = { 7, 6, 0, 0, 1, 57, 0, 2, 3, 8, 10, 13, 12, 16, 9, 26, 0, 18, 59, 58, @@ -523,7 +645,7 @@ static const yytype_int8 yypgoto[] = /* YYDEFGOTO[NTERM-NUM]. */ static const yytype_int8 yydefgoto[] = { - -1, 2, 3, 7, 8, 9, 10, 20, 12, 13, + 0, 2, 3, 7, 8, 9, 10, 20, 12, 13, 37, 38, 47, 14, 15, 53, 54, 22, 23, 24, 25, 48, 26, 55, 43, 16, 28, 29, 30 }; @@ -544,7 +666,7 @@ static const yytype_int8 yytable[] = 69, 34, 67, 27 }; -static const yytype_uint8 yycheck[] = +static const yytype_int8 yycheck[] = { 6, 36, 3, 0, 3, 0, 8, 3, 0, 11, 0, 46, 3, 4, 5, 8, 6, 14, 14, 14, @@ -559,7 +681,7 @@ static const yytype_uint8 yycheck[] = /* YYSTOS[STATE-NUM] -- The (internal number of the) accessing symbol of state STATE-NUM. */ -static const yytype_uint8 yystos[] = +static const yytype_int8 yystos[] = { 0, 6, 23, 24, 0, 3, 14, 25, 26, 27, 28, 29, 30, 31, 35, 36, 47, 48, 4, 5, @@ -573,7 +695,7 @@ static const yytype_uint8 yystos[] = }; /* YYR1[YYN] -- Symbol number of symbol that rule YYN derives. */ -static const yytype_uint8 yyr1[] = +static const yytype_int8 yyr1[] = { 0, 22, 23, 23, 23, 23, 24, 24, 25, 26, 27, 27, 28, 28, 29, 29, 30, 30, 31, 31, @@ -584,7 +706,7 @@ static const yytype_uint8 yyr1[] = }; /* YYR2[YYN] -- Number of symbols on the right hand side of rule YYN. */ -static const yytype_uint8 yyr2[] = +static const yytype_int8 yyr2[] = { 0, 2, 2, 2, 3, 0, 1, 0, 1, 1, 1, 3, 1, 1, 3, 4, 1, 3, 1, 2, @@ -595,10 +717,10 @@ static const yytype_uint8 yyr2[] = }; +enum { YYENOMEM = -2 }; + #define yyerrok (yyerrstatus = 0) #define yyclearin (yychar = YYEMPTY) -#define YYEMPTY (-2) -#define YYEOF 0 #define YYACCEPT goto yyacceptlab #define YYABORT goto yyabortlab @@ -607,27 +729,26 @@ static const yytype_uint8 yyr2[] = #define YYRECOVERING() (!!yyerrstatus) -#define YYBACKUP(Token, Value) \ -do \ - if (yychar == YYEMPTY) \ - { \ - yychar = (Token); \ - yylval = (Value); \ - YYPOPSTACK (yylen); \ - yystate = *yyssp; \ - goto yybackup; \ - } \ - else \ - { \ - yyerror (parsestate, YY_("syntax error: cannot back up")); \ - YYERROR; \ - } \ -while (0) - -/* Error token number */ -#define YYTERROR 1 -#define YYERRCODE 256 - +#define YYBACKUP(Token, Value) \ + do \ + if (yychar == YYEMPTY) \ + { \ + yychar = (Token); \ + yylval = (Value); \ + YYPOPSTACK (yylen); \ + yystate = *yyssp; \ + goto yybackup; \ + } \ + else \ + { \ + yyerror (parsestate, YY_("syntax error: cannot back up")); \ + YYERROR; \ + } \ + while (0) + +/* Backward compatibility with an undocumented macro. + Use YYerror or YYUNDEF. */ +#define YYERRCODE YYUNDEF /* Enable debugging if requested. */ @@ -645,55 +766,59 @@ do { \ } while (0) /* This macro is provided for backward compatibility. */ -#ifndef YY_LOCATION_PRINT -# define YY_LOCATION_PRINT(File, Loc) ((void) 0) -#endif +# ifndef YY_LOCATION_PRINT +# define YY_LOCATION_PRINT(File, Loc) ((void) 0) +# endif -# define YY_SYMBOL_PRINT(Title, Type, Value, Location) \ +# define YY_SYMBOL_PRINT(Title, Kind, Value, Location) \ do { \ if (yydebug) \ { \ YYFPRINTF (stderr, "%s ", Title); \ yy_symbol_print (stderr, \ - Type, Value, parsestate); \ + Kind, Value, parsestate); \ YYFPRINTF (stderr, "\n"); \ } \ } while (0) -/*----------------------------------------. -| Print this symbol's value on YYOUTPUT. | -`----------------------------------------*/ +/*-----------------------------------. +| Print this symbol's value on YYO. | +`-----------------------------------*/ static void -yy_symbol_value_print (FILE *yyoutput, int yytype, YYSTYPE const * const yyvaluep, DCEparsestate* parsestate) +yy_symbol_value_print (FILE *yyo, + yysymbol_kind_t yykind, YYSTYPE const * const yyvaluep, DCEparsestate* parsestate) { - FILE *yyo = yyoutput; - YYUSE (yyo); - YYUSE (parsestate); + FILE *yyoutput = yyo; + YY_USE (yyoutput); + YY_USE (parsestate); if (!yyvaluep) return; # ifdef YYPRINT - if (yytype < YYNTOKENS) - YYPRINT (yyoutput, yytoknum[yytype], *yyvaluep); + if (yykind < YYNTOKENS) + YYPRINT (yyo, yytoknum[yykind], *yyvaluep); # endif - YYUSE (yytype); + YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN + YY_USE (yykind); + YY_IGNORE_MAYBE_UNINITIALIZED_END } -/*--------------------------------. -| Print this symbol on YYOUTPUT. | -`--------------------------------*/ +/*---------------------------. +| Print this symbol on YYO. | +`---------------------------*/ static void -yy_symbol_print (FILE *yyoutput, int yytype, YYSTYPE const * const yyvaluep, DCEparsestate* parsestate) +yy_symbol_print (FILE *yyo, + yysymbol_kind_t yykind, YYSTYPE const * const yyvaluep, DCEparsestate* parsestate) { - YYFPRINTF (yyoutput, "%s %s (", - yytype < YYNTOKENS ? "token" : "nterm", yytname[yytype]); + YYFPRINTF (yyo, "%s %s (", + yykind < YYNTOKENS ? "token" : "nterm", yysymbol_name (yykind)); - yy_symbol_value_print (yyoutput, yytype, yyvaluep, parsestate); - YYFPRINTF (yyoutput, ")"); + yy_symbol_value_print (yyo, yykind, yyvaluep, parsestate); + YYFPRINTF (yyo, ")"); } /*------------------------------------------------------------------. @@ -702,7 +827,7 @@ yy_symbol_print (FILE *yyoutput, int yytype, YYSTYPE const * const yyvaluep, DCE `------------------------------------------------------------------*/ static void -yy_stack_print (yytype_int16 *yybottom, yytype_int16 *yytop) +yy_stack_print (yy_state_t *yybottom, yy_state_t *yytop) { YYFPRINTF (stderr, "Stack now"); for (; yybottom <= yytop; yybottom++) @@ -725,21 +850,21 @@ do { \ `------------------------------------------------*/ static void -yy_reduce_print (yytype_int16 *yyssp, YYSTYPE *yyvsp, int yyrule, DCEparsestate* parsestate) +yy_reduce_print (yy_state_t *yyssp, YYSTYPE *yyvsp, + int yyrule, DCEparsestate* parsestate) { - unsigned long int yylno = yyrline[yyrule]; + int yylno = yyrline[yyrule]; int yynrhs = yyr2[yyrule]; int yyi; - YYFPRINTF (stderr, "Reducing stack by rule %d (line %lu):\n", + YYFPRINTF (stderr, "Reducing stack by rule %d (line %d):\n", yyrule - 1, yylno); /* The symbols being reduced. */ for (yyi = 0; yyi < yynrhs; yyi++) { YYFPRINTF (stderr, " $%d = ", yyi + 1); yy_symbol_print (stderr, - yystos[yyssp[yyi + 1 - yynrhs]], - &(yyvsp[(yyi + 1) - (yynrhs)]) - , parsestate); + YY_ACCESSING_SYMBOL (+yyssp[yyi + 1 - yynrhs]), + &yyvsp[(yyi + 1) - (yynrhs)], parsestate); YYFPRINTF (stderr, "\n"); } } @@ -754,8 +879,8 @@ do { \ multiple parsers can coexist. */ int yydebug; #else /* !YYDEBUG */ -# define YYDPRINTF(Args) -# define YY_SYMBOL_PRINT(Title, Type, Value, Location) +# define YYDPRINTF(Args) ((void) 0) +# define YY_SYMBOL_PRINT(Title, Kind, Value, Location) # define YY_STACK_PRINT(Bottom, Top) # define YY_REDUCE_PRINT(Rule) #endif /* !YYDEBUG */ @@ -778,249 +903,34 @@ int yydebug; #endif -#if YYERROR_VERBOSE - -# ifndef yystrlen -# if defined __GLIBC__ && defined _STRING_H -# define yystrlen strlen -# else -/* Return the length of YYSTR. */ -static YYSIZE_T -yystrlen (const char *yystr) -{ - YYSIZE_T yylen; - for (yylen = 0; yystr[yylen]; yylen++) - continue; - return yylen; -} -# endif -# endif -# ifndef yystpcpy -# if defined __GLIBC__ && defined _STRING_H && defined _GNU_SOURCE -# define yystpcpy stpcpy -# else -/* Copy YYSRC to YYDEST, returning the address of the terminating '\0' in - YYDEST. */ -static char * -yystpcpy (char *yydest, const char *yysrc) -{ - char *yyd = yydest; - const char *yys = yysrc; - while ((*yyd++ = *yys++) != '\0') - continue; - return yyd - 1; -} -# endif -# endif - -# ifndef yytnamerr -/* Copy to YYRES the contents of YYSTR after stripping away unnecessary - quotes and backslashes, so that it's suitable for yyerror. The - heuristic is that double-quoting is unnecessary unless the string - contains an apostrophe, a comma, or backslash (other than - backslash-backslash). YYSTR is taken from yytname. If YYRES is - null, do not copy; instead, return the length of what the result - would have been. */ -static YYSIZE_T -yytnamerr (char *yyres, const char *yystr) -{ - if (*yystr == '"') - { - YYSIZE_T yyn = 0; - char const *yyp = yystr; - - for (;;) - switch (*++yyp) - { - case '\'': - case ',': - goto do_not_strip_quotes; - - case '\\': - if (*++yyp != '\\') - goto do_not_strip_quotes; - /* Fall through. */ - default: - if (yyres) - yyres[yyn] = *yyp; - yyn++; - break; - - case '"': - if (yyres) - yyres[yyn] = '\0'; - return yyn; - } - do_not_strip_quotes: ; - } - - if (! yyres) - return yystrlen (yystr); - - return yystpcpy (yyres, yystr) - yyres; -} -# endif - -/* Copy into *YYMSG, which is of size *YYMSG_ALLOC, an error message - about the unexpected token YYTOKEN for the state stack whose top is - YYSSP. - - Return 0 if *YYMSG was successfully written. Return 1 if *YYMSG is - not large enough to hold the message. In that case, also set - *YYMSG_ALLOC to the required number of bytes. Return 2 if the - required number of bytes is too large to store. */ -static int -yysyntax_error (YYSIZE_T *yymsg_alloc, char **yymsg, - yytype_int16 *yyssp, int yytoken) -{ - YYSIZE_T yysize0 = yytnamerr (YY_NULLPTR, yytname[yytoken]); - YYSIZE_T yysize = yysize0; - enum { YYERROR_VERBOSE_ARGS_MAXIMUM = 5 }; - /* Internationalized format string. */ - const char *yyformat = YY_NULLPTR; - /* Arguments of yyformat. */ - char const *yyarg[YYERROR_VERBOSE_ARGS_MAXIMUM]; - /* Number of reported tokens (one for the "unexpected", one per - "expected"). */ - int yycount = 0; - - /* There are many possibilities here to consider: - - If this state is a consistent state with a default action, then - the only way this function was invoked is if the default action - is an error action. In that case, don't check for expected - tokens because there are none. - - The only way there can be no lookahead present (in yychar) is if - this state is a consistent state with a default action. Thus, - detecting the absence of a lookahead is sufficient to determine - that there is no unexpected or expected token to report. In that - case, just report a simple "syntax error". - - Don't assume there isn't a lookahead just because this state is a - consistent state with a default action. There might have been a - previous inconsistent state, consistent state with a non-default - action, or user semantic action that manipulated yychar. - - Of course, the expected token list depends on states to have - correct lookahead information, and it depends on the parser not - to perform extra reductions after fetching a lookahead from the - scanner and before detecting a syntax error. Thus, state merging - (from LALR or IELR) and default reductions corrupt the expected - token list. However, the list is correct for canonical LR with - one exception: it will still contain any token that will not be - accepted due to an error action in a later state. - */ - if (yytoken != YYEMPTY) - { - int yyn = yypact[*yyssp]; - yyarg[yycount++] = yytname[yytoken]; - if (!yypact_value_is_default (yyn)) - { - /* Start YYX at -YYN if negative to avoid negative indexes in - YYCHECK. In other words, skip the first -YYN actions for - this state because they are default actions. */ - int yyxbegin = yyn < 0 ? -yyn : 0; - /* Stay within bounds of both yycheck and yytname. */ - int yychecklim = YYLAST - yyn + 1; - int yyxend = yychecklim < YYNTOKENS ? yychecklim : YYNTOKENS; - int yyx; - - for (yyx = yyxbegin; yyx < yyxend; ++yyx) - if (yycheck[yyx + yyn] == yyx && yyx != YYTERROR - && !yytable_value_is_error (yytable[yyx + yyn])) - { - if (yycount == YYERROR_VERBOSE_ARGS_MAXIMUM) - { - yycount = 1; - yysize = yysize0; - break; - } - yyarg[yycount++] = yytname[yyx]; - { - YYSIZE_T yysize1 = yysize + yytnamerr (YY_NULLPTR, yytname[yyx]); - if (! (yysize <= yysize1 - && yysize1 <= YYSTACK_ALLOC_MAXIMUM)) - return 2; - yysize = yysize1; - } - } - } - } - - switch (yycount) - { -# define YYCASE_(N, S) \ - case N: \ - yyformat = S; \ - break - YYCASE_(0, YY_("syntax error")); - YYCASE_(1, YY_("syntax error, unexpected %s")); - YYCASE_(2, YY_("syntax error, unexpected %s, expecting %s")); - YYCASE_(3, YY_("syntax error, unexpected %s, expecting %s or %s")); - YYCASE_(4, YY_("syntax error, unexpected %s, expecting %s or %s or %s")); - YYCASE_(5, YY_("syntax error, unexpected %s, expecting %s or %s or %s or %s")); -# undef YYCASE_ - } - - { - YYSIZE_T yysize1 = yysize + yystrlen (yyformat); - if (! (yysize <= yysize1 && yysize1 <= YYSTACK_ALLOC_MAXIMUM)) - return 2; - yysize = yysize1; - } - - if (*yymsg_alloc < yysize) - { - *yymsg_alloc = 2 * yysize; - if (! (yysize <= *yymsg_alloc - && *yymsg_alloc <= YYSTACK_ALLOC_MAXIMUM)) - *yymsg_alloc = YYSTACK_ALLOC_MAXIMUM; - return 1; - } - - /* Avoid sprintf, as that infringes on the user's name space. - Don't have undefined behavior even if the translation - produced a string with the wrong number of "%s"s. */ - { - char *yyp = *yymsg; - int yyi = 0; - while ((*yyp = *yyformat) != '\0') - if (*yyp == '%' && yyformat[1] == 's' && yyi < yycount) - { - yyp += yytnamerr (yyp, yyarg[yyi++]); - yyformat += 2; - } - else - { - yyp++; - yyformat++; - } - } - return 0; -} -#endif /* YYERROR_VERBOSE */ /*-----------------------------------------------. | Release the memory associated to this symbol. | `-----------------------------------------------*/ static void -yydestruct (const char *yymsg, int yytype, YYSTYPE *yyvaluep, DCEparsestate* parsestate) +yydestruct (const char *yymsg, + yysymbol_kind_t yykind, YYSTYPE *yyvaluep, DCEparsestate* parsestate) { - YYUSE (yyvaluep); - YYUSE (parsestate); + YY_USE (yyvaluep); + YY_USE (parsestate); if (!yymsg) yymsg = "Deleting"; - YY_SYMBOL_PRINT (yymsg, yytype, yyvaluep, yylocationp); + YY_SYMBOL_PRINT (yymsg, yykind, yyvaluep, yylocationp); YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN - YYUSE (yytype); + YY_USE (yykind); YY_IGNORE_MAYBE_UNINITIALIZED_END } + + /*----------. | yyparse. | `----------*/ @@ -1028,7 +938,7 @@ yydestruct (const char *yymsg, int yytype, YYSTYPE *yyvaluep, DCEparsestate* par int yyparse (DCEparsestate* parsestate) { -/* The lookahead symbol. */ +/* Lookahead token kind. */ int yychar; @@ -1039,45 +949,38 @@ YY_INITIAL_VALUE (static YYSTYPE yyval_default;) YYSTYPE yylval YY_INITIAL_VALUE (= yyval_default); /* Number of syntax errors so far. */ - int yynerrs; + int yynerrs = 0; - int yystate; + yy_state_fast_t yystate = 0; /* Number of tokens to shift before error messages enabled. */ - int yyerrstatus; + int yyerrstatus = 0; - /* The stacks and their tools: - 'yyss': related to states. - 'yyvs': related to semantic values. - - Refer to the stacks through separate pointers, to allow yyoverflow + /* Refer to the stacks through separate pointers, to allow yyoverflow to reallocate them elsewhere. */ - /* The state stack. */ - yytype_int16 yyssa[YYINITDEPTH]; - yytype_int16 *yyss; - yytype_int16 *yyssp; + /* Their size. */ + YYPTRDIFF_T yystacksize = YYINITDEPTH; - /* The semantic value stack. */ - YYSTYPE yyvsa[YYINITDEPTH]; - YYSTYPE *yyvs; - YYSTYPE *yyvsp; + /* The state stack: array, bottom, top. */ + yy_state_t yyssa[YYINITDEPTH]; + yy_state_t *yyss = yyssa; + yy_state_t *yyssp = yyss; - YYSIZE_T yystacksize; + /* The semantic value stack: array, bottom, top. */ + YYSTYPE yyvsa[YYINITDEPTH]; + YYSTYPE *yyvs = yyvsa; + YYSTYPE *yyvsp = yyvs; int yyn; + /* The return value of yyparse. */ int yyresult; - /* Lookahead token as an internal (translated) token number. */ - int yytoken = 0; + /* Lookahead symbol kind. */ + yysymbol_kind_t yytoken = YYSYMBOL_YYEMPTY; /* The variables used to return semantic value and location from the action routines. */ YYSTYPE yyval; -#if YYERROR_VERBOSE - /* Buffer for error messages, and its allocated size. */ - char yymsgbuf[128]; - char *yymsg = yymsgbuf; - YYSIZE_T yymsg_alloc = sizeof yymsgbuf; -#endif + #define YYPOPSTACK(N) (yyvsp -= (N), yyssp -= (N)) @@ -1085,58 +988,60 @@ YYSTYPE yylval YY_INITIAL_VALUE (= yyval_default); Keep to zero when no symbol should be popped. */ int yylen = 0; - yyssp = yyss = yyssa; - yyvsp = yyvs = yyvsa; - yystacksize = YYINITDEPTH; - YYDPRINTF ((stderr, "Starting parse\n")); - yystate = 0; - yyerrstatus = 0; - yynerrs = 0; yychar = YYEMPTY; /* Cause a token to be read. */ goto yysetstate; + /*------------------------------------------------------------. -| yynewstate -- Push a new state, which is found in yystate. | +| yynewstate -- push a new state, which is found in yystate. | `------------------------------------------------------------*/ - yynewstate: +yynewstate: /* In all cases, when you get here, the value and location stacks have just been pushed. So pushing a state here evens the stacks. */ yyssp++; - yysetstate: - *yyssp = yystate; + +/*--------------------------------------------------------------------. +| yysetstate -- set current state (the top of the stack) to yystate. | +`--------------------------------------------------------------------*/ +yysetstate: + YYDPRINTF ((stderr, "Entering state %d\n", yystate)); + YY_ASSERT (0 <= yystate && yystate < YYNSTATES); + YY_IGNORE_USELESS_CAST_BEGIN + *yyssp = YY_CAST (yy_state_t, yystate); + YY_IGNORE_USELESS_CAST_END + YY_STACK_PRINT (yyss, yyssp); if (yyss + yystacksize - 1 <= yyssp) +#if !defined yyoverflow && !defined YYSTACK_RELOCATE + goto yyexhaustedlab; +#else { /* Get the current used size of the three stacks, in elements. */ - YYSIZE_T yysize = yyssp - yyss + 1; + YYPTRDIFF_T yysize = yyssp - yyss + 1; -#ifdef yyoverflow +# if defined yyoverflow { /* Give user a chance to reallocate the stack. Use copies of these so that the &'s don't force the real ones into memory. */ + yy_state_t *yyss1 = yyss; YYSTYPE *yyvs1 = yyvs; - yytype_int16 *yyss1 = yyss; /* Each stack pointer address is followed by the size of the data in use in that stack, in bytes. This used to be a conditional around just the two extra args, but that might be undefined if yyoverflow is a macro. */ yyoverflow (YY_("memory exhausted"), - &yyss1, yysize * sizeof (*yyssp), - &yyvs1, yysize * sizeof (*yyvsp), + &yyss1, yysize * YYSIZEOF (*yyssp), + &yyvs1, yysize * YYSIZEOF (*yyvsp), &yystacksize); - yyss = yyss1; yyvs = yyvs1; } -#else /* no yyoverflow */ -# ifndef YYSTACK_RELOCATE - goto yyexhaustedlab; -# else +# else /* defined YYSTACK_RELOCATE */ /* Extend the stack our own way. */ if (YYMAXDEPTH <= yystacksize) goto yyexhaustedlab; @@ -1145,9 +1050,10 @@ YYSTYPE yylval YY_INITIAL_VALUE (= yyval_default); yystacksize = YYMAXDEPTH; { - yytype_int16 *yyss1 = yyss; + yy_state_t *yyss1 = yyss; union yyalloc *yyptr = - (union yyalloc *) YYSTACK_ALLOC (YYSTACK_BYTES (yystacksize)); + YY_CAST (union yyalloc *, + YYSTACK_ALLOC (YY_CAST (YYSIZE_T, YYSTACK_BYTES (yystacksize)))); if (! yyptr) goto yyexhaustedlab; YYSTACK_RELOCATE (yyss_alloc, yyss); @@ -1157,30 +1063,30 @@ YYSTYPE yylval YY_INITIAL_VALUE (= yyval_default); YYSTACK_FREE (yyss1); } # endif -#endif /* no yyoverflow */ yyssp = yyss + yysize - 1; yyvsp = yyvs + yysize - 1; - YYDPRINTF ((stderr, "Stack size increased to %lu\n", - (unsigned long int) yystacksize)); + YY_IGNORE_USELESS_CAST_BEGIN + YYDPRINTF ((stderr, "Stack size increased to %ld\n", + YY_CAST (long, yystacksize))); + YY_IGNORE_USELESS_CAST_END if (yyss + yystacksize - 1 <= yyssp) YYABORT; } - - YYDPRINTF ((stderr, "Entering state %d\n", yystate)); +#endif /* !defined yyoverflow && !defined YYSTACK_RELOCATE */ if (yystate == YYFINAL) YYACCEPT; goto yybackup; + /*-----------. | yybackup. | `-----------*/ yybackup: - /* Do appropriate processing given the current state. Read a lookahead token if we need one and don't already have one. */ @@ -1191,18 +1097,29 @@ YYSTYPE yylval YY_INITIAL_VALUE (= yyval_default); /* Not known => get a lookahead token if don't already have one. */ - /* YYCHAR is either YYEMPTY or YYEOF or a valid lookahead symbol. */ + /* YYCHAR is either empty, or end-of-input, or a valid lookahead. */ if (yychar == YYEMPTY) { - YYDPRINTF ((stderr, "Reading a token: ")); + YYDPRINTF ((stderr, "Reading a token\n")); yychar = yylex (&yylval, parsestate); } if (yychar <= YYEOF) { - yychar = yytoken = YYEOF; + yychar = YYEOF; + yytoken = YYSYMBOL_YYEOF; YYDPRINTF ((stderr, "Now at end of input.\n")); } + else if (yychar == YYerror) + { + /* The scanner already issued an error message, process directly + to error recovery. But do not keep the error token as + lookahead, it is too special and may lead us to an endless + loop in error recovery. */ + yychar = YYUNDEF; + yytoken = YYSYMBOL_YYerror; + goto yyerrlab1; + } else { yytoken = YYTRANSLATE (yychar); @@ -1230,15 +1147,13 @@ YYSTYPE yylval YY_INITIAL_VALUE (= yyval_default); /* Shift the lookahead token. */ YY_SYMBOL_PRINT ("Shifting", yytoken, &yylval, &yylloc); - - /* Discard the shifted token. */ - yychar = YYEMPTY; - yystate = yyn; YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN *++yyvsp = yylval; YY_IGNORE_MAYBE_UNINITIALIZED_END + /* Discard the shifted token. */ + yychar = YYEMPTY; goto yynewstate; @@ -1253,7 +1168,7 @@ YYSTYPE yylval YY_INITIAL_VALUE (= yyval_default); /*-----------------------------. -| yyreduce -- Do a reduction. | +| yyreduce -- do a reduction. | `-----------------------------*/ yyreduce: /* yyn is the number of a rule to reduce with. */ @@ -1273,320 +1188,321 @@ YYSTYPE yylval YY_INITIAL_VALUE (= yyval_default); YY_REDUCE_PRINT (yyn); switch (yyn) { - case 8: -#line 43 "dce.y" /* yacc.c:1646 */ - {projections(parsestate,(yyvsp[0]));} -#line 1280 "dcetab.c" /* yacc.c:1646 */ + case 8: /* projections: projectionlist */ +#line 43 "dce.y" + {projections(parsestate,yyvsp[0]);} +#line 1195 "dcetab.c" break; - case 9: -#line 47 "dce.y" /* yacc.c:1646 */ - {selections(parsestate,(yyvsp[0]));} -#line 1286 "dcetab.c" /* yacc.c:1646 */ + case 9: /* selections: clauselist */ +#line 47 "dce.y" + {selections(parsestate,yyvsp[0]);} +#line 1201 "dcetab.c" break; - case 10: -#line 52 "dce.y" /* yacc.c:1646 */ - {(yyval)=projectionlist(parsestate,(Object)null,(yyvsp[0]));} -#line 1292 "dcetab.c" /* yacc.c:1646 */ + case 10: /* projectionlist: projection */ +#line 52 "dce.y" + {yyval=projectionlist(parsestate,(Object)null,yyvsp[0]);} +#line 1207 "dcetab.c" break; - case 11: -#line 54 "dce.y" /* yacc.c:1646 */ - {(yyval)=projectionlist(parsestate,(yyvsp[-2]),(yyvsp[0]));} -#line 1298 "dcetab.c" /* yacc.c:1646 */ + case 11: /* projectionlist: projectionlist ',' projection */ +#line 54 "dce.y" + {yyval=projectionlist(parsestate,yyvsp[-2],yyvsp[0]);} +#line 1213 "dcetab.c" break; - case 12: -#line 59 "dce.y" /* yacc.c:1646 */ - {(yyval)=projection(parsestate,(yyvsp[0]));} -#line 1304 "dcetab.c" /* yacc.c:1646 */ + case 12: /* projection: segmentlist */ +#line 59 "dce.y" + {yyval=projection(parsestate,yyvsp[0]);} +#line 1219 "dcetab.c" break; - case 13: -#line 61 "dce.y" /* yacc.c:1646 */ - {(yyval)=projection(parsestate,(yyvsp[0]));} -#line 1310 "dcetab.c" /* yacc.c:1646 */ + case 13: /* projection: function */ +#line 61 "dce.y" + {yyval=projection(parsestate,yyvsp[0]);} +#line 1225 "dcetab.c" break; - case 14: -#line 66 "dce.y" /* yacc.c:1646 */ - {(yyval)=function(parsestate,(yyvsp[-2]),null);} -#line 1316 "dcetab.c" /* yacc.c:1646 */ + case 14: /* function: ident '(' ')' */ +#line 66 "dce.y" + {yyval=function(parsestate,yyvsp[-2],null);} +#line 1231 "dcetab.c" break; - case 15: -#line 68 "dce.y" /* yacc.c:1646 */ - {(yyval)=function(parsestate,(yyvsp[-3]),(yyvsp[-1]));} -#line 1322 "dcetab.c" /* yacc.c:1646 */ + case 15: /* function: ident '(' arg_list ')' */ +#line 68 "dce.y" + {yyval=function(parsestate,yyvsp[-3],yyvsp[-1]);} +#line 1237 "dcetab.c" break; - case 16: -#line 73 "dce.y" /* yacc.c:1646 */ - {(yyval)=segmentlist(parsestate,null,(yyvsp[0]));} -#line 1328 "dcetab.c" /* yacc.c:1646 */ + case 16: /* segmentlist: segment */ +#line 73 "dce.y" + {yyval=segmentlist(parsestate,null,yyvsp[0]);} +#line 1243 "dcetab.c" break; - case 17: -#line 75 "dce.y" /* yacc.c:1646 */ - {(yyval)=segmentlist(parsestate,(yyvsp[-2]),(yyvsp[0]));} -#line 1334 "dcetab.c" /* yacc.c:1646 */ + case 17: /* segmentlist: segmentlist '.' segment */ +#line 75 "dce.y" + {yyval=segmentlist(parsestate,yyvsp[-2],yyvsp[0]);} +#line 1249 "dcetab.c" break; - case 18: -#line 80 "dce.y" /* yacc.c:1646 */ - {(yyval)=segment(parsestate,(yyvsp[0]),null);} -#line 1340 "dcetab.c" /* yacc.c:1646 */ + case 18: /* segment: word */ +#line 80 "dce.y" + {yyval=segment(parsestate,yyvsp[0],null);} +#line 1255 "dcetab.c" break; - case 19: -#line 82 "dce.y" /* yacc.c:1646 */ - {(yyval)=segment(parsestate,(yyvsp[-1]),(yyvsp[0]));} -#line 1346 "dcetab.c" /* yacc.c:1646 */ + case 19: /* segment: word rangelist */ +#line 82 "dce.y" + {yyval=segment(parsestate,yyvsp[-1],yyvsp[0]);} +#line 1261 "dcetab.c" break; - case 20: -#line 87 "dce.y" /* yacc.c:1646 */ - {(yyval)=rangelist(parsestate,null,(yyvsp[0]));} -#line 1352 "dcetab.c" /* yacc.c:1646 */ + case 20: /* rangelist: range */ +#line 87 "dce.y" + {yyval=rangelist(parsestate,null,yyvsp[0]);} +#line 1267 "dcetab.c" break; - case 21: -#line 89 "dce.y" /* yacc.c:1646 */ - {(yyval)=rangelist(parsestate,(yyvsp[-1]),(yyvsp[0]));} -#line 1358 "dcetab.c" /* yacc.c:1646 */ + case 21: /* rangelist: rangelist range */ +#line 89 "dce.y" + {yyval=rangelist(parsestate,yyvsp[-1],yyvsp[0]);} +#line 1273 "dcetab.c" break; - case 22: -#line 94 "dce.y" /* yacc.c:1646 */ - {(yyval)=range(parsestate,(yyvsp[-1]),null,null);} -#line 1364 "dcetab.c" /* yacc.c:1646 */ + case 22: /* range: '[' number ']' */ +#line 94 "dce.y" + {yyval=range(parsestate,yyvsp[-1],null,null);} +#line 1279 "dcetab.c" break; - case 23: -#line 96 "dce.y" /* yacc.c:1646 */ - {(yyval)=range(parsestate,(yyvsp[-3]),null,(yyvsp[-1]));} -#line 1370 "dcetab.c" /* yacc.c:1646 */ + case 23: /* range: '[' number ':' number ']' */ +#line 96 "dce.y" + {yyval=range(parsestate,yyvsp[-3],null,yyvsp[-1]);} +#line 1285 "dcetab.c" break; - case 24: -#line 98 "dce.y" /* yacc.c:1646 */ - {(yyval)=range(parsestate,(yyvsp[-5]),(yyvsp[-3]),(yyvsp[-1]));} -#line 1376 "dcetab.c" /* yacc.c:1646 */ + case 24: /* range: '[' number ':' number ':' number ']' */ +#line 98 "dce.y" + {yyval=range(parsestate,yyvsp[-5],yyvsp[-3],yyvsp[-1]);} +#line 1291 "dcetab.c" break; - case 25: -#line 102 "dce.y" /* yacc.c:1646 */ - {(yyval) = range1(parsestate,(yyvsp[-1]));} -#line 1382 "dcetab.c" /* yacc.c:1646 */ + case 25: /* range1: '[' number ']' */ +#line 102 "dce.y" + {yyval = range1(parsestate,yyvsp[-1]);} +#line 1297 "dcetab.c" break; - case 26: -#line 108 "dce.y" /* yacc.c:1646 */ - {(yyval)=clauselist(parsestate,null,(yyvsp[0]));} -#line 1388 "dcetab.c" /* yacc.c:1646 */ + case 26: /* clauselist: sel_clause */ +#line 108 "dce.y" + {yyval=clauselist(parsestate,null,yyvsp[0]);} +#line 1303 "dcetab.c" break; - case 27: -#line 110 "dce.y" /* yacc.c:1646 */ - {(yyval)=clauselist(parsestate,(yyvsp[-1]),(yyvsp[0]));} -#line 1394 "dcetab.c" /* yacc.c:1646 */ + case 27: /* clauselist: clauselist sel_clause */ +#line 110 "dce.y" + {yyval=clauselist(parsestate,yyvsp[-1],yyvsp[0]);} +#line 1309 "dcetab.c" break; - case 28: -#line 115 "dce.y" /* yacc.c:1646 */ - {(yyval)=sel_clause(parsestate,1,(yyvsp[-4]),(yyvsp[-3]),(yyvsp[-1]));} -#line 1400 "dcetab.c" /* yacc.c:1646 */ + case 28: /* sel_clause: '&' value rel_op '{' value_list '}' */ +#line 115 "dce.y" + {yyval=sel_clause(parsestate,1,yyvsp[-4],yyvsp[-3],yyvsp[-1]);} +#line 1315 "dcetab.c" break; - case 29: -#line 117 "dce.y" /* yacc.c:1646 */ - {(yyval)=sel_clause(parsestate,2,(yyvsp[-2]),(yyvsp[-1]),(yyvsp[0]));} -#line 1406 "dcetab.c" /* yacc.c:1646 */ + case 29: /* sel_clause: '&' value rel_op value */ +#line 117 "dce.y" + {yyval=sel_clause(parsestate,2,yyvsp[-2],yyvsp[-1],yyvsp[0]);} +#line 1321 "dcetab.c" break; - case 30: -#line 119 "dce.y" /* yacc.c:1646 */ - {(yyval)=(yyvsp[-1]);} -#line 1412 "dcetab.c" /* yacc.c:1646 */ + case 30: /* sel_clause: '&' boolfunction */ +#line 119 "dce.y" + {yyval=yyvsp[-1];} +#line 1327 "dcetab.c" break; - case 31: -#line 124 "dce.y" /* yacc.c:1646 */ - {(yyval)=value_list(parsestate,null,(yyvsp[0]));} -#line 1418 "dcetab.c" /* yacc.c:1646 */ + case 31: /* value_list: value */ +#line 124 "dce.y" + {yyval=value_list(parsestate,null,yyvsp[0]);} +#line 1333 "dcetab.c" break; - case 32: -#line 126 "dce.y" /* yacc.c:1646 */ - {(yyval)=value_list(parsestate,(yyvsp[-2]),(yyvsp[0]));} -#line 1424 "dcetab.c" /* yacc.c:1646 */ + case 32: /* value_list: value_list ',' value */ +#line 126 "dce.y" + {yyval=value_list(parsestate,yyvsp[-2],yyvsp[0]);} +#line 1339 "dcetab.c" break; - case 33: -#line 131 "dce.y" /* yacc.c:1646 */ - {(yyval)=value(parsestate,(yyvsp[0]));} -#line 1430 "dcetab.c" /* yacc.c:1646 */ + case 33: /* value: var */ +#line 131 "dce.y" + {yyval=value(parsestate,yyvsp[0]);} +#line 1345 "dcetab.c" break; - case 34: -#line 133 "dce.y" /* yacc.c:1646 */ - {(yyval)=value(parsestate,(yyvsp[0]));} -#line 1436 "dcetab.c" /* yacc.c:1646 */ + case 34: /* value: function */ +#line 133 "dce.y" + {yyval=value(parsestate,yyvsp[0]);} +#line 1351 "dcetab.c" break; - case 35: -#line 135 "dce.y" /* yacc.c:1646 */ - {(yyval)=value(parsestate,(yyvsp[0]));} -#line 1442 "dcetab.c" /* yacc.c:1646 */ + case 35: /* value: constant */ +#line 135 "dce.y" + {yyval=value(parsestate,yyvsp[0]);} +#line 1357 "dcetab.c" break; - case 36: -#line 140 "dce.y" /* yacc.c:1646 */ - {(yyval)=constant(parsestate,(yyvsp[0]),SCAN_NUMBERCONST);} -#line 1448 "dcetab.c" /* yacc.c:1646 */ + case 36: /* constant: number */ +#line 140 "dce.y" + {yyval=constant(parsestate,yyvsp[0],SCAN_NUMBERCONST);} +#line 1363 "dcetab.c" break; - case 37: -#line 142 "dce.y" /* yacc.c:1646 */ - {(yyval)=constant(parsestate,(yyvsp[0]),SCAN_STRINGCONST);} -#line 1454 "dcetab.c" /* yacc.c:1646 */ + case 37: /* constant: string */ +#line 142 "dce.y" + {yyval=constant(parsestate,yyvsp[0],SCAN_STRINGCONST);} +#line 1369 "dcetab.c" break; - case 38: -#line 147 "dce.y" /* yacc.c:1646 */ - {(yyval)=var(parsestate,(yyvsp[0]));} -#line 1460 "dcetab.c" /* yacc.c:1646 */ + case 38: /* var: indexpath */ +#line 147 "dce.y" + {yyval=var(parsestate,yyvsp[0]);} +#line 1375 "dcetab.c" break; - case 39: -#line 152 "dce.y" /* yacc.c:1646 */ - {(yyval)=indexpath(parsestate,null,(yyvsp[0]));} -#line 1466 "dcetab.c" /* yacc.c:1646 */ + case 39: /* indexpath: index */ +#line 152 "dce.y" + {yyval=indexpath(parsestate,null,yyvsp[0]);} +#line 1381 "dcetab.c" break; - case 40: -#line 154 "dce.y" /* yacc.c:1646 */ - {(yyval)=indexpath(parsestate,(yyvsp[-2]),(yyvsp[0]));} -#line 1472 "dcetab.c" /* yacc.c:1646 */ + case 40: /* indexpath: indexpath '.' index */ +#line 154 "dce.y" + {yyval=indexpath(parsestate,yyvsp[-2],yyvsp[0]);} +#line 1387 "dcetab.c" break; - case 41: -#line 159 "dce.y" /* yacc.c:1646 */ - {(yyval)=indexer(parsestate,(yyvsp[0]),null);} -#line 1478 "dcetab.c" /* yacc.c:1646 */ + case 41: /* index: word */ +#line 159 "dce.y" + {yyval=indexer(parsestate,yyvsp[0],null);} +#line 1393 "dcetab.c" break; - case 42: -#line 161 "dce.y" /* yacc.c:1646 */ - {(yyval)=indexer(parsestate,(yyvsp[-1]),(yyvsp[0]));} -#line 1484 "dcetab.c" /* yacc.c:1646 */ + case 42: /* index: word array_indices */ +#line 161 "dce.y" + {yyval=indexer(parsestate,yyvsp[-1],yyvsp[0]);} +#line 1399 "dcetab.c" break; - case 43: -#line 166 "dce.y" /* yacc.c:1646 */ - {(yyval)=array_indices(parsestate,null,(yyvsp[0]));} -#line 1490 "dcetab.c" /* yacc.c:1646 */ + case 43: /* array_indices: range1 */ +#line 166 "dce.y" + {yyval=array_indices(parsestate,null,yyvsp[0]);} +#line 1405 "dcetab.c" break; - case 44: -#line 168 "dce.y" /* yacc.c:1646 */ - {(yyval)=array_indices(parsestate,(yyvsp[-1]),(yyvsp[0]));} -#line 1496 "dcetab.c" /* yacc.c:1646 */ + case 44: /* array_indices: array_indices range1 */ +#line 168 "dce.y" + {yyval=array_indices(parsestate,yyvsp[-1],yyvsp[0]);} +#line 1411 "dcetab.c" break; - case 45: -#line 173 "dce.y" /* yacc.c:1646 */ - {(yyval)=function(parsestate,(yyvsp[-2]),null);} -#line 1502 "dcetab.c" /* yacc.c:1646 */ + case 45: /* boolfunction: ident '(' ')' */ +#line 173 "dce.y" + {yyval=function(parsestate,yyvsp[-2],null);} +#line 1417 "dcetab.c" break; - case 46: -#line 175 "dce.y" /* yacc.c:1646 */ - {(yyval)=function(parsestate,(yyvsp[-3]),(yyvsp[-1]));} -#line 1508 "dcetab.c" /* yacc.c:1646 */ + case 46: /* boolfunction: ident '(' arg_list ')' */ +#line 175 "dce.y" + {yyval=function(parsestate,yyvsp[-3],yyvsp[-1]);} +#line 1423 "dcetab.c" break; - case 47: -#line 180 "dce.y" /* yacc.c:1646 */ - {(yyval)=arg_list(parsestate,null,(yyvsp[0]));} -#line 1514 "dcetab.c" /* yacc.c:1646 */ + case 47: /* arg_list: value */ +#line 180 "dce.y" + {yyval=arg_list(parsestate,null,yyvsp[0]);} +#line 1429 "dcetab.c" break; - case 48: -#line 182 "dce.y" /* yacc.c:1646 */ - {(yyval)=arg_list(parsestate,(yyvsp[-2]),(yyvsp[0]));} -#line 1520 "dcetab.c" /* yacc.c:1646 */ + case 48: /* arg_list: value_list ',' value */ +#line 182 "dce.y" + {yyval=arg_list(parsestate,yyvsp[-2],yyvsp[0]);} +#line 1435 "dcetab.c" break; - case 49: -#line 186 "dce.y" /* yacc.c:1646 */ - {(yyval)=makeselectiontag(CEO_EQ);} -#line 1526 "dcetab.c" /* yacc.c:1646 */ + case 49: /* rel_op: '=' */ +#line 186 "dce.y" + {yyval=makeselectiontag(CEO_EQ);} +#line 1441 "dcetab.c" break; - case 50: -#line 187 "dce.y" /* yacc.c:1646 */ - {(yyval)=makeselectiontag(CEO_GT);} -#line 1532 "dcetab.c" /* yacc.c:1646 */ + case 50: /* rel_op: '>' */ +#line 187 "dce.y" + {yyval=makeselectiontag(CEO_GT);} +#line 1447 "dcetab.c" break; - case 51: -#line 188 "dce.y" /* yacc.c:1646 */ - {(yyval)=makeselectiontag(CEO_LT);} -#line 1538 "dcetab.c" /* yacc.c:1646 */ + case 51: /* rel_op: '<' */ +#line 188 "dce.y" + {yyval=makeselectiontag(CEO_LT);} +#line 1453 "dcetab.c" break; - case 52: -#line 189 "dce.y" /* yacc.c:1646 */ - {(yyval)=makeselectiontag(CEO_NEQ);} -#line 1544 "dcetab.c" /* yacc.c:1646 */ + case 52: /* rel_op: '!' '=' */ +#line 189 "dce.y" + {yyval=makeselectiontag(CEO_NEQ);} +#line 1459 "dcetab.c" break; - case 53: -#line 190 "dce.y" /* yacc.c:1646 */ - {(yyval)=makeselectiontag(CEO_GE);} -#line 1550 "dcetab.c" /* yacc.c:1646 */ + case 53: /* rel_op: '>' '=' */ +#line 190 "dce.y" + {yyval=makeselectiontag(CEO_GE);} +#line 1465 "dcetab.c" break; - case 54: -#line 191 "dce.y" /* yacc.c:1646 */ - {(yyval)=makeselectiontag(CEO_LE);} -#line 1556 "dcetab.c" /* yacc.c:1646 */ + case 54: /* rel_op: '<' '=' */ +#line 191 "dce.y" + {yyval=makeselectiontag(CEO_LE);} +#line 1471 "dcetab.c" break; - case 55: -#line 192 "dce.y" /* yacc.c:1646 */ - {(yyval)=makeselectiontag(CEO_RE);} -#line 1562 "dcetab.c" /* yacc.c:1646 */ + case 55: /* rel_op: '=' '~' */ +#line 192 "dce.y" + {yyval=makeselectiontag(CEO_RE);} +#line 1477 "dcetab.c" break; - case 56: -#line 196 "dce.y" /* yacc.c:1646 */ - {(yyval) = (yyvsp[0]);} -#line 1568 "dcetab.c" /* yacc.c:1646 */ + case 56: /* ident: word */ +#line 196 "dce.y" + {yyval = yyvsp[0];} +#line 1483 "dcetab.c" break; - case 57: -#line 200 "dce.y" /* yacc.c:1646 */ - {(yyval) = checkobject((yyvsp[0]));} -#line 1574 "dcetab.c" /* yacc.c:1646 */ + case 57: /* word: SCAN_WORD */ +#line 200 "dce.y" + {yyval = checkobject(yyvsp[0]);} +#line 1489 "dcetab.c" break; - case 58: -#line 204 "dce.y" /* yacc.c:1646 */ - {(yyval) = checkobject((yyvsp[0]));} -#line 1580 "dcetab.c" /* yacc.c:1646 */ + case 58: /* number: SCAN_NUMBERCONST */ +#line 204 "dce.y" + {yyval = checkobject(yyvsp[0]);} +#line 1495 "dcetab.c" break; - case 59: -#line 208 "dce.y" /* yacc.c:1646 */ - {(yyval) = checkobject((yyvsp[0]));} -#line 1586 "dcetab.c" /* yacc.c:1646 */ + case 59: /* string: SCAN_STRINGCONST */ +#line 208 "dce.y" + {yyval = checkobject(yyvsp[0]);} +#line 1501 "dcetab.c" break; -#line 1590 "dcetab.c" /* yacc.c:1646 */ +#line 1505 "dcetab.c" + default: break; } /* User semantic actions sometimes alter yychar, and that requires @@ -1600,25 +1516,23 @@ YYSTYPE yylval YY_INITIAL_VALUE (= yyval_default); case of YYERROR or YYBACKUP, subsequent parser actions might lead to an incorrect destructor call or verbose syntax error message before the lookahead is translated. */ - YY_SYMBOL_PRINT ("-> $$ =", yyr1[yyn], &yyval, &yyloc); + YY_SYMBOL_PRINT ("-> $$ =", YY_CAST (yysymbol_kind_t, yyr1[yyn]), &yyval, &yyloc); YYPOPSTACK (yylen); yylen = 0; - YY_STACK_PRINT (yyss, yyssp); *++yyvsp = yyval; /* Now 'shift' the result of the reduction. Determine what state that goes to, based on the state we popped back to and the rule number reduced by. */ - - yyn = yyr1[yyn]; - - yystate = yypgoto[yyn - YYNTOKENS] + *yyssp; - if (0 <= yystate && yystate <= YYLAST && yycheck[yystate] == *yyssp) - yystate = yytable[yystate]; - else - yystate = yydefgoto[yyn - YYNTOKENS]; + { + const int yylhs = yyr1[yyn] - YYNTOKENS; + const int yyi = yypgoto[yylhs] + *yyssp; + yystate = (0 <= yyi && yyi <= YYLAST && yycheck[yyi] == *yyssp + ? yytable[yyi] + : yydefgoto[yylhs]); + } goto yynewstate; @@ -1629,50 +1543,14 @@ YYSTYPE yylval YY_INITIAL_VALUE (= yyval_default); yyerrlab: /* Make sure we have latest lookahead translation. See comments at user semantic actions for why this is necessary. */ - yytoken = yychar == YYEMPTY ? YYEMPTY : YYTRANSLATE (yychar); - + yytoken = yychar == YYEMPTY ? YYSYMBOL_YYEMPTY : YYTRANSLATE (yychar); /* If not already recovering from an error, report this error. */ if (!yyerrstatus) { ++yynerrs; -#if ! YYERROR_VERBOSE yyerror (parsestate, YY_("syntax error")); -#else -# define YYSYNTAX_ERROR yysyntax_error (&yymsg_alloc, &yymsg, \ - yyssp, yytoken) - { - char const *yymsgp = YY_("syntax error"); - int yysyntax_error_status; - yysyntax_error_status = YYSYNTAX_ERROR; - if (yysyntax_error_status == 0) - yymsgp = yymsg; - else if (yysyntax_error_status == 1) - { - if (yymsg != yymsgbuf) - YYSTACK_FREE (yymsg); - yymsg = (char *) YYSTACK_ALLOC (yymsg_alloc); - if (!yymsg) - { - yymsg = yymsgbuf; - yymsg_alloc = sizeof yymsgbuf; - yysyntax_error_status = 2; - } - else - { - yysyntax_error_status = YYSYNTAX_ERROR; - yymsgp = yymsg; - } - } - yyerror (parsestate, yymsgp); - if (yysyntax_error_status == 2) - goto yyexhaustedlab; - } -# undef YYSYNTAX_ERROR -#endif } - - if (yyerrstatus == 3) { /* If just tried and failed to reuse lookahead token after an @@ -1701,12 +1579,10 @@ YYSTYPE yylval YY_INITIAL_VALUE (= yyval_default); | yyerrorlab -- error raised explicitly by YYERROR. | `---------------------------------------------------*/ yyerrorlab: - - /* Pacify compilers like GCC when the user code never invokes - YYERROR and the label yyerrorlab therefore never appears in user - code. */ - if (/*CONSTCOND*/ 0) - goto yyerrorlab; + /* Pacify compilers when the user code never invokes YYERROR and the + label yyerrorlab therefore never appears in user code. */ + if (0) + YYERROR; /* Do not reclaim the symbols of the rule whose action triggered this YYERROR. */ @@ -1723,13 +1599,14 @@ YYSTYPE yylval YY_INITIAL_VALUE (= yyval_default); yyerrlab1: yyerrstatus = 3; /* Each real token shifted decrements this. */ + /* Pop stack until we find a state that shifts the error token. */ for (;;) { yyn = yypact[yystate]; if (!yypact_value_is_default (yyn)) { - yyn += YYTERROR; - if (0 <= yyn && yyn <= YYLAST && yycheck[yyn] == YYTERROR) + yyn += YYSYMBOL_YYerror; + if (0 <= yyn && yyn <= YYLAST && yycheck[yyn] == YYSYMBOL_YYerror) { yyn = yytable[yyn]; if (0 < yyn) @@ -1743,7 +1620,7 @@ YYSTYPE yylval YY_INITIAL_VALUE (= yyval_default); yydestruct ("Error: popping", - yystos[yystate], yyvsp, parsestate); + YY_ACCESSING_SYMBOL (yystate), yyvsp, parsestate); YYPOPSTACK (1); yystate = *yyssp; YY_STACK_PRINT (yyss, yyssp); @@ -1755,7 +1632,7 @@ YYSTYPE yylval YY_INITIAL_VALUE (= yyval_default); /* Shift the error token. */ - YY_SYMBOL_PRINT ("Shifting", yystos[yyn], yyvsp, yylsp); + YY_SYMBOL_PRINT ("Shifting", YY_ACCESSING_SYMBOL (yyn), yyvsp, yylsp); yystate = yyn; goto yynewstate; @@ -1768,6 +1645,7 @@ YYSTYPE yylval YY_INITIAL_VALUE (= yyval_default); yyresult = 0; goto yyreturn; + /*-----------------------------------. | yyabortlab -- YYABORT comes here. | `-----------------------------------*/ @@ -1775,16 +1653,21 @@ YYSTYPE yylval YY_INITIAL_VALUE (= yyval_default); yyresult = 1; goto yyreturn; -#if !defined yyoverflow || YYERROR_VERBOSE + +#if !defined yyoverflow /*-------------------------------------------------. | yyexhaustedlab -- memory exhaustion comes here. | `-------------------------------------------------*/ yyexhaustedlab: yyerror (parsestate, YY_("memory exhausted")); yyresult = 2; - /* Fall through. */ + goto yyreturn; #endif + +/*-------------------------------------------------------. +| yyreturn -- parsing is finished, clean up and return. | +`-------------------------------------------------------*/ yyreturn: if (yychar != YYEMPTY) { @@ -1801,18 +1684,16 @@ YYSTYPE yylval YY_INITIAL_VALUE (= yyval_default); while (yyssp != yyss) { yydestruct ("Cleanup: popping", - yystos[*yyssp], yyvsp, parsestate); + YY_ACCESSING_SYMBOL (+*yyssp), yyvsp, parsestate); YYPOPSTACK (1); } #ifndef yyoverflow if (yyss != yyssa) YYSTACK_FREE (yyss); #endif -#if YYERROR_VERBOSE - if (yymsg != yymsgbuf) - YYSTACK_FREE (yymsg); -#endif + return yyresult; } -#line 211 "dce.y" /* yacc.c:1906 */ + +#line 211 "dce.y" diff --git a/libdap2/dcetab.h b/libdap2/dcetab.h index b399b6446d..a2f91777b7 100644 --- a/libdap2/dcetab.h +++ b/libdap2/dcetab.h @@ -1,8 +1,9 @@ -/* A Bison parser, made by GNU Bison 3.0.4. */ +/* A Bison parser, made by GNU Bison 3.7.6. */ /* Bison interface for Yacc-like parsers in C - Copyright (C) 1984, 1989-1990, 2000-2015 Free Software Foundation, Inc. + Copyright (C) 1984, 1989-1990, 2000-2015, 2018-2021 Free Software Foundation, + Inc. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -15,7 +16,7 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License - along with this program. If not, see . */ + along with this program. If not, see . */ /* As a special exception, you may create a larger work that contains part or all of the Bison parser skeleton and distribute that work @@ -30,6 +31,10 @@ This special exception was added by the Free Software Foundation in version 2.2 of Bison. */ +/* DO NOT RELY ON FEATURES THAT ARE NOT DOCUMENTED in the manual, + especially those whose name start with YY_ or yy_. They are + private implementation details that can be changed or removed. */ + #ifndef YY_DCE_DCE_TAB_H_INCLUDED # define YY_DCE_DCE_TAB_H_INCLUDED /* Debug traces. */ @@ -40,15 +45,20 @@ extern int dcedebug; #endif -/* Token type. */ +/* Token kinds. */ #ifndef YYTOKENTYPE # define YYTOKENTYPE enum yytokentype { - SCAN_WORD = 258, - SCAN_STRINGCONST = 259, - SCAN_NUMBERCONST = 260 + YYEMPTY = -2, + YYEOF = 0, /* "end of file" */ + YYerror = 256, /* error */ + YYUNDEF = 257, /* "invalid token" */ + SCAN_WORD = 258, /* SCAN_WORD */ + SCAN_STRINGCONST = 259, /* SCAN_STRINGCONST */ + SCAN_NUMBERCONST = 260 /* SCAN_NUMBERCONST */ }; + typedef enum yytokentype yytoken_kind_t; #endif /* Value type. */ diff --git a/libdispatch/daux.c b/libdispatch/daux.c index aa9eb0d827..b30cff7cea 100644 --- a/libdispatch/daux.c +++ b/libdispatch/daux.c @@ -735,7 +735,52 @@ ncaux_h5filterspec_parse(const char* txt, unsigned int* idp, size_t* nparamsp, u if(paramsp) {*paramsp = params; params = NULL;} done: nullfree(params); - nullfree(sdata); + nullfree(sdata0); + return stat; +} + +/* +Parse a filter parameter string into a sequence of unsigned ints. + +@param txt - a string containing the parameter string. +@param nuiparamsp - store the number of unsigned ints here +@param uiparamsp - store the vector of unsigned ints here; caller frees. +@return NC_NOERR if parse succeeded +@return NC_EINVAL otherwise +*/ + +EXTERNL int +ncaux_h5filterspec_parse_parameter(const char* txt, size_t* nuiparamsp, unsigned int* uiparams) +{ + int stat = NC_NOERR; + char* p; + char* sdata0 = NULL; /* what to free */ + char* sdata = NULL; /* sdata0 with leading prefix skipped */ + size_t nuiparams = 0; + size_t len; + + if(txt == NULL) + {stat = NC_EINVAL; goto done;} + len = strlen(txt); + if(len == 0) + {stat = NC_EINVAL; goto done;} + + if((sdata0 = (char*)calloc(1,len+1+1))==NULL) + {stat = NC_ENOMEM; goto done;} + memcpy(sdata0,txt,len); + sdata = sdata0; + + p = sdata; + + nuiparams = 0; + len = strlen(p); + /* skip leading white space */ + while(strchr(" ",*p) != NULL) {p++; len--;} + if((stat = filterspec_cvt(p,&nuiparams,uiparams))) goto done; + /* Now return results */ + if(nuiparamsp) *nuiparamsp = nuiparams; +done: + nullfree(sdata0); return stat; } diff --git a/libdispatch/dfilter.c b/libdispatch/dfilter.c index 454cb9c7e8..e5ea120a9d 100644 --- a/libdispatch/dfilter.c +++ b/libdispatch/dfilter.c @@ -26,9 +26,6 @@ Unified filter related code /**************************************************/ /* Per-variable filters */ -/* The original HDF5-based functions are left - but are now wrappers around the filterx functions. -*/ /** Find the set of filters (if any) associated with a variable. diff --git a/libdispatch/dfilterx.c b/libdispatch/dfilterx.c index 299fbd6d10..bc90a02c12 100644 --- a/libdispatch/dfilterx.c +++ b/libdispatch/dfilterx.c @@ -15,124 +15,50 @@ #include "netcdf_filter.h" #include "ncdispatch.h" #include "nc4internal.h" -#include "ncfilter.h" -#ifdef USE_HDF5 -#include "hdf5internal.h" -#endif +#include "ncjson.h" /* -Unified filter related code +NCZarr filter API */ -#ifndef H5Z_FILTER_SZIP -/** ID of HDF SZIP filter. */ -#define H5Z_FILTER_SZIP 4 -#endif - -/*Mnemonic*/ -#define USENAME 1 - -#define LPAREN '(' -#define RPAREN ')' -#define LBRACK '[' -#define RBRACK ']' - -#define NUMCHAR "0123456789" -#define NAMECHAR1 "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" -#define NAMECHARN (NAMECHAR1 NUMCHAR "_-") - -const struct FilterName { - const char* name; /* name or alias as assigned by HDF group*/ - unsigned int id; /* id as assigned by HDF group*/ -} known_filters[] = { -{"zip", 2}, /* Standard zlib compression */ -{"zlib", 2}, /* alias */ -{"deflate", 2}, /* alias */ -{"szip", 4}, /* Standard szip compression */ -{"bzip2", 307}, /* BZIP2 lossless compression used by PyTables */ -{"lzf", 32000}, /* LZF lossless compression used by H5Py project */ -{"blosc", 32001}, /* Blosc lossless compression used by PyTables */ -{"mafisc", 32002}, /* Modified LZMA compression filter, MAFISC (Multidimensional Adaptive Filtering Improved Scientific data Compression) */ -{"snappy", 32003}, /* Snappy lossless compression. */ -{"lz4", 32004}, /* LZ4 fast lossless compression algorithm */ -{"apax", 32005}, /* Samplify's APAX Numerical Encoding Technology */ -{"cbf", 32006}, /* All imgCIF/CBF compressions and decompressions, including Canonical, Packed, Packed Vesrsion 2, Byte Offset and Nibble Offset. */ -{"jpeg-xr", 32007}, /* Enables images to be compressed/decompressed with JPEG-XR compression */ -{"bitshuffle", 32008}, /* Extreme version of shuffle filter that shuffles data at bit level instead of byte level. */ -{"spdp", 32009}, /* SPDP fast lossless compression algorithm for single- and double-precision floating-point data. */ -{"lpc-rice", 32010}, /* LPC-Rice multi-threaded lossless compression */ -{"ccsds-123", 32011}, /* ESA CCSDS-123 multi-threaded compression filter */ -{"jpeg-ls", 32012}, /* CharLS JPEG-LS multi-threaded compression filter */ -{"zfp", 32013}, /* Rate, accuracy or precision bounded compression for floating-point arrays */ -{"fpzip", 32014}, /* Fast and Efficient Lossy or Lossless Compressor for Floating-Point Data */ -{"zstandard", 32015}, /* Real-time compression algorithm with wide range of compression / speed trade-off and fast decoder */ -{"b3d", 32016}, /* GPU based image compression method developed for light-microscopy applications */ -{"sz", 32017}, /* An error-bounded lossy compressor for scientific floating-point data */ -{"fcidecomp", 32018}, /* EUMETSAT CharLS compression filter for use with netCDF */ -{"user-defined", 32768}, /* First user-defined filter */ -{NULL,0} -}; - -/**************************************************/ -/*Forward*/ -static unsigned int NC_filterx_lookup(const char* filtername); -static const char* NC_filterx_toname(unsigned int id); -static int NC_filterx_transferstringvec(size_t n, char** vec, char** copy); - /**************************************************/ -/* Per-variable filters extended string-based */ +/* Per-variable filters */ /** Find the set of filters (if any) associated with a variable. +Assumes NCZarr format using json \param ncid NetCDF or group ID, from a previous call to nc_open(), -nc_create(), nc_def_grp(), or associated inquiry functions such as -nc_inq_ncid(). - \param varid Variable ID -\param nfilters return no. of filters -\param ids return the filter ids (caller allocates) +\param jsonp a JSON formatted string is returned in this argument \returns ::NC_NOERR No error. \returns ::NC_ENOTNC4 Not a netCDF-4 file. -\returns ::NC_EBADID Bad ncid. +\returns ::NC_EBADID Bad ncid \returns ::NC_ENOTVAR Invalid variable ID. \returns ::NC_EINVAL Invalid arguments \ingroup variables \author Dennis Heimbigner */ EXTERNL int -nc_inq_var_filterx_ids(int ncid, int varid, size_t* nfiltersp, char** ids) +nc_inq_var_filterx_ids(int ncid, int varid, char** textp) { - NC* ncp; - int stat = NC_NOERR; - NC_FILTERX_OBJ ncids; + NC* ncp; + int stat = NC_NOERR; - if((stat = NC_check_id(ncid,&ncp))) goto done; - TRACE(ncx_inq_var_filterids); - - memset(&ncids,0,sizeof(ncids)); - ncids.usort = NC_FILTER_UNION_IDS; - ncids.u.ids.nfilters = 0; - ncids.u.ids.filterids = NULL; - - if((stat = ncp->dispatch->filter_actions(ncid,varid, NCFILTER_FILTERIDS, &ncids))) goto done; - if(nfiltersp) *nfiltersp = ncids.u.ids.nfilters; - if(ids) { - if((stat = NC_filterx_transferstringvec(ncids.u.ids.nfilters, ncids.u.ids.filterids, ids))) goto done; - } + TRACE(nc_inq_var_filterx_ids); + if((stat = NC_check_id(ncid,&ncp))) return stat; + if((stat = ncp->dispatch->inq_var_filterx_ids(ncid,varid,textp))) goto done; done: - NC_filterx_freestringvec(ncids.u.ids.nfilters,ncids.u.ids.filterids); - return stat; - } + return stat; +} /** Find the the param info about filter (if any) associated with a variable and with specified id. - -This is a wrapper for nc_inq_var_all(). +Assumes HDF5 format using unsigned ints. \param ncid NetCDF or group ID, from a previous call to nc_open(), nc_create(), nc_def_grp(), or associated inquiry functions such as @@ -140,7 +66,6 @@ nc_inq_ncid(). \param varid Variable ID \param id The filter id of interest -\param formatp (Out) Storage for the filter format \param nparamsp (Out) Storage which will get the number of parameters to the filter \param params (Out) Storage which will get associated parameters. Note: the caller must allocate and free. @@ -149,38 +74,27 @@ Note: the caller must allocate and free. \returns ::NC_ENOTNC4 Not a netCDF-4 file. \returns ::NC_EBADID Bad ncid. \returns ::NC_ENOTVAR Invalid variable ID. -\returns ::NC_ENOFILTER No filter defined. +\returns ::NC_ENOFILTER Specified filter not defined for this variable. \ingroup variables \author Dennis Heimbigner */ EXTERNL int -nc_inq_var_filterx_info(int ncid, int varid, const char* id, size_t* nparamsp, char** params) +nc_inq_var_filterx_info(int ncid, int varid, const char* id, char** textp) { - NC* ncp; - int stat = NC_NOERR; - NC_FILTERX_OBJ spec; - - if((stat = NC_check_id(ncid,&ncp))) goto done; - TRACE(ncx_inq_var_filter_info); - - memset(&spec,0,sizeof(spec)); - spec.usort = NC_FILTER_UNION_SPEC; - spec.u.spec.filterid = (char*)id; - spec.u.spec.params = NULL; - - if((stat = ncp->dispatch->filter_actions(ncid,varid,NCFILTER_INFO,&spec))) goto done; - if(nparamsp) *nparamsp = spec.u.spec.nparams; - if(params) { - if((stat = NC_filterx_transferstringvec(spec.u.spec.nparams, spec.u.spec.params, params))) goto done; - } + NC* ncp; + int stat = NC_check_id(ncid,&ncp); + + TRACE(nc_inq_var_filterx_info); + if(stat != NC_NOERR) return stat; + if((stat = ncp->dispatch->inq_var_filterx_info(ncid,varid,id,textp))) goto done; + done: - NC_filterx_freestringvec(spec.u.spec.nparams,spec.u.spec.params); - return stat; + return stat; } /** - Define a new variable filter. - + Define a new variable filter + Assumes HDF5 format using unsigned ints. Only variables with chunked storage can use filters. @param ncid File and group ID. @@ -196,210 +110,134 @@ nc_inq_var_filterx_info(int ncid, int varid, const char* id, size_t* nparamsp, c */ EXTERNL int -nc_def_var_filterx(int ncid, int varid, const char* id, size_t nparams, const char** params) +nc_def_var_filterx(int ncid, int varid, const char* json) { - NC* ncp; - NC_FILTERX_OBJ spec; - int stat = NC_NOERR; - - if((stat = NC_check_id(ncid,&ncp))) goto done; - TRACE(ncx_def_var_filter); - - memset(&spec,0,sizeof(spec)); - spec.usort = NC_FILTER_UNION_SPEC; - spec.u.spec.filterid = (char*)id; - spec.u.spec.nparams = nparams; - spec.u.spec.params = (char**)params; /* discard const */ - if((stat=ncp->dispatch->filter_actions(ncid,varid,NCFILTER_DEF,&spec))) goto done; + NC* ncp; + int stat = NC_check_id(ncid,&ncp); + + TRACE(nc_def_var_filterx); + if(stat != NC_NOERR) return stat; + if((stat = ncp->dispatch->def_var_filterx(ncid,varid,json))) goto done; + done: return stat; } /** - Remove all filters with specified id from a variable +Find the first filter (if any) associated with a variable. + +\param ncid NetCDF or group ID, from a previous call to nc_open(), +nc_create(), nc_def_grp(), or associated inquiry functions such as +nc_inq_ncid(). - @param ncid File and group ID. - @param varid Variable ID. - @param id filter to remove +\param varid Variable ID - @return ::NC_NOERR No error. - @return ::NC_EBADID Bad ID. - @author Dennis Heimbigner +\param textp Storage which will get the filter info (id + parameters) in json format + +This is redundant over the multi-filter API, so +it can be implemented in terms of those functions. + +\returns ::NC_NOERR No error. +\returns ::NC_ENOTNC4 Not a netCDF-4 file. +\returns ::NC_EBADID Bad ncid. +\returns ::NC_ENOTVAR Invalid variable ID. + +\ingroup variables +\author Dennis Heimbigner */ EXTERNL int -nc_var_filterx_remove(int ncid, int varid, const char* id) +nc_inq_var_filterx(int ncid, int varid, char** textp) { NC* ncp; int stat = NC_NOERR; - NC_FILTERX_OBJ spec; + char* text = NULL; + NCjson* json = NULL; + NCjson* jid = NULL; - if((stat = NC_check_id(ncid,&ncp))) goto done; - TRACE(ncx_var_filter_remove); - - memset(&spec,0,sizeof(spec)); - spec.usort = NC_FILTER_UNION_SPEC; - spec.u.spec.filterid = (char*)id; - if((stat=ncp->dispatch->filter_actions(ncid,varid,NCFILTER_REMOVE,&spec))) goto done; -done: + TRACE(nc_inq_var_filterx); + if((stat = NC_check_id(ncid,&ncp))) goto done; + + /* Get the filters on this variable */ + if((stat = nc_inq_var_filterx_ids(ncid,varid,&text))) goto done; + /* Parse it */ + if((stat = NCJparse(text,0,&json))) goto done; + if(json->sort != NCJ_ARRAY) + {stat = NC_EFILTER; goto done;} + if(NCJlength(json) == 0 || NCJcontents(json) == NULL) + {stat = NC_ENOFILTER; goto done;} + jid = NCJith(json,0); + if(jid->sort == NCJ_DICT || jid->sort == NCJ_ARRAY) + {stat = NC_EFILTER; goto done;} + /* Get info about the first filter */ + if((stat = nc_inq_var_filterx_info(ncid,varid,NCJstring(jid),textp))) + {stat = NC_ENOFILTER; goto done;} + done: + NCJreclaim(json); return stat; } /**************************************************/ -/* Utilities */ +/* Support direct user defined filters */ -int -NC_cvtX2I_idlist(size_t n, const char** xidlist, unsigned int* ids) -{ - int i,stat = NC_NOERR; +#ifdef ENABLE_CLIENTSIDE_FILTERS - for(i=0;i 0 ? NC_NOERR : NC_EINVAL); -} - -/* Convert an int id to a string; optional name conversion */ -int -NC_cvtI2X_id(unsigned int id, char** xidp, int usename) -{ - char xid[NC_MAX_NAME]; - - snprintf(xid,sizeof(xid),"%u",id); - if(usename) {/* See if this id has a name */ - const char* name = NC_filterx_toname(id); - if(name != NULL) {xid[0] = '\0'; strlcat(xid,name,sizeof(xid));} - } - if(xidp) { - if((*xidp = strdup(xid))==NULL) return NC_ENOMEM; - } - return NC_NOERR; -} - -static unsigned int -NC_filterx_lookup(const char* filtername) -{ - const struct FilterName* p = known_filters; - /* Try a name lookup */ - for(;p->name;p++) { - if(strcasecmp(p->name,filtername)==0) - return p->id; - } - return 0; /* no match */ -} - -static const char* -NC_filterx_toname(unsigned int id) -{ - const struct FilterName* p = known_filters; - for(;p->name;p++) { - if(p->id == id) - return p->name; - } - return NULL; /* no match */ -} - -void -NC_filterx_freestringvec(size_t n, char** vec) -{ - int i; - if(vec != NULL) { - for(i=0;i 2) + nparams = 2; /* for compatibility, only return 2 params */ break; case NC_ENOFILTER: case NC_ENOTNC4: diff --git a/libdispatch/ncjson.c b/libdispatch/ncjson.c index 1cca2132ef..0759931db5 100644 --- a/libdispatch/ncjson.c +++ b/libdispatch/ncjson.c @@ -12,20 +12,22 @@ TODO: make utf8 safe #include #include #include + #include "ncjson.h" -#ifdef _WIN32 -#define strcasecmp _stricmp +#undef NCJDEBUG +#ifdef NCJDEBUG +static int ncjbreakpoint(int err) {return err;} +#define NCJTHROW(err) ((err)==NCJ_ERR?ncjbreakpoint(err):(err)) #else -#include +#define NCJTHROW(err) (err) #endif -#undef DEBUG - +/**************************************************/ #define NCJ_OK 0 -#define NCJ_ERR 1 +#define NCJ_ERR (-1) -#define NCJ_EOF -1 +#define NCJ_EOF -2 #define NCJ_LBRACKET '[' #define NCJ_RBRACKET ']' @@ -39,11 +41,10 @@ TODO: make utf8 safe #define NCJ_TAG_FALSE "false" #define NCJ_TAG_NULL "null" -/* WORD Subsumes Number also */ -#define WORD "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_$+-." - -/*//////////////////////////////////////////////////*/ +/* JSON_WORD Subsumes Number also */ +#define JSON_WORD "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_$+-." +/**************************************************/ typedef struct NCJparser { char* text; char* pos; @@ -51,7 +52,7 @@ typedef struct NCJparser { char* yytext; /* string or word */ long long num; int tf; - int err; + int status; /* NCJ_ERR|NCJ_OK */ } NCJparser; typedef struct NCJbuf { @@ -59,6 +60,26 @@ typedef struct NCJbuf { char* text; /* NULL || nul terminated */ } NCJbuf; +/**************************************************/ + +#ifdef _WIN32 +#define strdup _strdup +#define strcasecmp _stricmp +#else +#include +#endif + +#ifndef nullfree +#define nullfree(x) {if(x)free(x);} +#endif +#ifndef nulldup +#define nulldup(x) ((x)?strdup(x):(x)) +#endif + +#ifdef NCJDEBUG +static char* tokenname(int token); +#endif + /**************************************************/ /* Forward */ static int NCJparseR(NCJparser* parser, NCjson**); @@ -72,27 +93,20 @@ static int NCJlex(NCJparser* parser); static int NCJyytext(NCJparser*, char* start, size_t pdlen); static void NCJreclaimArray(struct NCjlist*); static void NCJreclaimDict(struct NCjlist*); -static int NCJunparseR(const NCjson* json, NCJbuf* buf, unsigned flags); static int NCJunescape(NCJparser* parser); static int listappend(struct NCjlist* list, NCjson* element); -static int bytesappendquoted(NCJbuf* buf, const char*); -static int bytesappend(NCJbuf* buf, const char* s); -static int bytesappendc(NCJbuf* buf, const char c); -static int escape(const char* text, NCJbuf* buf); + +#ifndef NETCDF_JSON_H static int NCJcloneArray(const NCjson* array, NCjson** clonep); static int NCJcloneDict(const NCjson* dict, NCjson** clonep); - -#ifndef nullfree -#define nullfree(x) {if(x)free(x);} -#endif -#ifndef nulldup -#define nulldup(x) ((x)?strdup(x):(x)) +static int NCJunparseR(const NCjson* json, NCJbuf* buf, unsigned flags); +static int bytesappendquoted(NCJbuf* buf, const char* s); +static int bytesappend(NCJbuf* buf, const char* s); +static int bytesappendc(NCJbuf* bufp, const char c); #endif -#ifdef DEBUG -static char* tokenname(int token); -#endif /**************************************************/ + int NCJparse(const char* text, unsigned flags, NCjson** jsonp) { @@ -103,23 +117,24 @@ NCJparse(const char* text, unsigned flags, NCjson** jsonp) /* Need at least 1 character of input */ if(text == NULL || text[0] == '\0') - {stat = NCJ_ERR; goto done;} + {stat = NCJTHROW(NCJ_ERR); goto done;} if(jsonp == NULL) goto done; parser = calloc(1,sizeof(NCJparser)); if(parser == NULL) - {stat = NCJ_ERR; goto done;} + {stat = NCJTHROW(NCJ_ERR); goto done;} len = strlen(text); parser->text = (char*)malloc(len+1+1); if(parser->text == NULL) - {stat = NCJ_ERR; goto done;} + {stat = NCJTHROW(NCJ_ERR); goto done;} strcpy(parser->text,text); parser->text[len] = '\0'; parser->text[len+1] = '\0'; parser->pos = &parser->text[0]; -#ifdef DEBUG + parser->status = NCJ_OK; +#ifdef NCJDEBUG fprintf(stderr,"json: |%s|\n",parser->text); #endif - if((stat=NCJparseR(parser,&json))) goto done; + if((stat=NCJparseR(parser,&json))==NCJ_ERR) goto done; *jsonp = json; json = NULL; @@ -130,7 +145,7 @@ fprintf(stderr,"json: |%s|\n",parser->text); free(parser); } (void)NCJreclaim(json); - return (stat); + return NCJTHROW(stat); } /* @@ -150,38 +165,38 @@ NCJparseR(NCJparser* parser, NCjson** jsonp) NCjson* json = NULL; if(jsonp == NULL) - {stat = NCJ_ERR; goto done;} + {stat = NCJTHROW(NCJ_ERR); goto done;} if((token = NCJlex(parser)) == NCJ_UNDEF) - {stat = NCJ_ERR; goto done;} + {stat = NCJTHROW(NCJ_ERR); goto done;} switch (token) { case NCJ_EOF: break; case NCJ_NULL: - if((stat = NCJnew(NCJ_NULL,&json))) goto done; + if((stat = NCJnew(NCJ_NULL,&json))==NCJ_ERR) goto done; break; case NCJ_BOOLEAN: - if((stat = NCJnew(NCJ_BOOLEAN,&json))) goto done; + if((stat = NCJnew(NCJ_BOOLEAN,&json))==NCJ_ERR) goto done; json->string = strdup(parser->yytext); break; case NCJ_INT: - if((stat = NCJnew(NCJ_INT,&json))) goto done; + if((stat = NCJnew(NCJ_INT,&json))==NCJ_ERR) goto done; json->string = strdup(parser->yytext); break; case NCJ_DOUBLE: - if((stat = NCJnew(NCJ_DOUBLE,&json))) goto done; + if((stat = NCJnew(NCJ_DOUBLE,&json))==NCJ_ERR) goto done; json->string = strdup(parser->yytext); break; case NCJ_STRING: - if((stat = NCJnew(NCJ_STRING,&json))) goto done; + if((stat = NCJnew(NCJ_STRING,&json))==NCJ_ERR) goto done; json->string = strdup(parser->yytext); break; case NCJ_LBRACE: - if((stat = NCJnew(NCJ_DICT,&json))) goto done; - if((stat = NCJparseDict(parser, &json->list))) goto done; + if((stat = NCJnew(NCJ_DICT,&json))==NCJ_ERR) goto done; + if((stat = NCJparseDict(parser, &json->list))==NCJ_ERR) goto done; break; case NCJ_LBRACKET: - if((stat = NCJnew(NCJ_ARRAY,&json))) goto done; - if((stat = NCJparseArray(parser, &json->list))) goto done; + if((stat = NCJnew(NCJ_ARRAY,&json))==NCJ_ERR) goto done; + if((stat = NCJparseArray(parser, &json->list))==NCJ_ERR) goto done; break; case NCJ_RBRACE: /* We hit end of the dict we are parsing */ parser->pos--; /* pushback so NCJparseArray will catch */ @@ -192,14 +207,14 @@ NCJparseR(NCJparser* parser, NCjson** jsonp) json = NULL; break; default: - stat = NCJ_ERR; + stat = NCJTHROW(NCJ_ERR); break; } if(jsonp && json) {*jsonp = json; json = NULL;} done: NCJreclaim(json); - return (stat); + return NCJTHROW(stat); } static int @@ -214,7 +229,7 @@ NCJparseArray(NCJparser* parser, struct NCjlist* arrayp) while(!stop) { /* Recurse to get the value ei (might be null) */ - if((stat = NCJparseR(parser,&element))) goto done; + if((stat = NCJparseR(parser,&element))==NCJ_ERR) goto done; token = NCJlex(parser); /* Get next token */ /* Next token should be comma or rbracket */ switch(token) { @@ -225,14 +240,14 @@ NCJparseArray(NCJparser* parser, struct NCjlist* arrayp) break; case NCJ_COMMA: /* Append the ei to the list */ - if(element == NULL) {stat = NCJ_ERR; goto done;} /* error */ + if(element == NULL) {stat = NCJTHROW(NCJ_ERR); goto done;} /* error */ listappend(arrayp,element); element = NULL; break; case NCJ_EOF: case NCJ_UNDEF: default: - stat = NCJ_ERR; + stat = NCJTHROW(NCJ_ERR); goto done; } } @@ -240,7 +255,7 @@ NCJparseArray(NCJparser* parser, struct NCjlist* arrayp) done: if(element != NULL) NCJreclaim(element); - return (stat); + return NCJTHROW(stat); } static int @@ -261,24 +276,24 @@ NCJparseDict(NCJparser* parser, struct NCjlist* dictp) case NCJ_STRING: case NCJ_BOOLEAN: case NCJ_INT: case NCJ_DOUBLE: { - if((stat=NCJnewstring(token,parser->yytext,&key))) goto done; + if((stat=NCJnewstring(token,parser->yytext,&key))==NCJ_ERR) goto done; } break; case NCJ_RBRACE: /* End of containing Dict */ stop = 1; continue; /* leave loop */ case NCJ_EOF: case NCJ_UNDEF: default: - stat = NCJ_ERR; + stat = NCJTHROW(NCJ_ERR); goto done; } /* Next token must be colon*/ switch((token = NCJlex(parser))) { case NCJ_COLON: break; case NCJ_UNDEF: case NCJ_EOF: - default: stat = NCJ_ERR; goto done; + default: stat = NCJTHROW(NCJ_ERR); goto done; } /* Get the value */ - if((stat = NCJparseR(parser,&value))) goto done; + if((stat = NCJparseR(parser,&value))==NCJ_ERR) goto done; /* Next token must be comma or RBRACE */ switch((token = NCJlex(parser))) { case NCJ_RBRACE: @@ -294,7 +309,7 @@ NCJparseDict(NCJparser* parser, struct NCjlist* dictp) case NCJ_EOF: case NCJ_UNDEF: default: - stat = NCJ_ERR; + stat = NCJTHROW(NCJ_ERR); goto done; } } @@ -304,14 +319,14 @@ NCJparseDict(NCJparser* parser, struct NCjlist* dictp) NCJreclaim(key); if(value != NULL) NCJreclaim(value); - return (stat); + return NCJTHROW(stat); } static int NCJlex(NCJparser* parser) { int c; - int token = 0; + int token = NCJ_UNDEF; char* start; size_t count; @@ -322,11 +337,11 @@ NCJlex(NCJparser* parser) } else if(c <= ' ' || c == '\177') { parser->pos++; continue; /* ignore whitespace */ - } else if(strchr(WORD, c) != NULL) { + } else if(strchr(JSON_WORD, c) != NULL) { start = parser->pos; for(;;) { c = *parser->pos++; - if(c == '\0' || strchr(WORD,c) == NULL) break; /* end of word */ + if(c == '\0' || strchr(JSON_WORD,c) == NULL) break; /* end of word */ } /* Pushback c if not whitespace */ parser->pos--; @@ -353,24 +368,25 @@ NCJlex(NCJparser* parser) else if(c == NCJ_QUOTE || c == '\0') break; } if(c == '\0') { - parser->err = NCJ_ERR; + parser->status = NCJ_ERR; token = NCJ_UNDEF; goto done; } count = ((parser->pos) - start) - 1; /* -1 for trailing quote */ - if(NCJyytext(parser,start,count)) goto done; - if(NCJunescape(parser)) goto done; + if(NCJyytext(parser,start,count)==NCJ_ERR) goto done; + if(NCJunescape(parser)==NCJ_ERR) goto done; token = NCJ_STRING; } else { /* single char token */ - if(NCJyytext(parser,parser->pos,1)) goto done; + if(NCJyytext(parser,parser->pos,1)==NCJ_ERR) goto done; token = *parser->pos++; } -#ifdef DEBUG +#ifdef NCJDEBUG fprintf(stderr,"%s(%d): |%s|\n",tokenname(token),token,parser->yytext); #endif } /*for(;;)*/ done: - if(parser->err) token = NCJ_UNDEF; + if(parser->status == NCJ_ERR) + token = NCJ_UNDEF; return token; } @@ -378,8 +394,8 @@ static int testnull(const char* word) { if(strcasecmp(word,NCJ_TAG_NULL)==0) - return NCJ_OK; - return NCJ_ERR; + return NCJTHROW(NCJ_OK); + return NCJTHROW(NCJ_ERR); } static int @@ -387,8 +403,8 @@ testbool(const char* word) { if(strcasecmp(word,NCJ_TAG_TRUE)==0 || strcasecmp(word,NCJ_TAG_FALSE)==0) - return NCJ_OK; - return NCJ_ERR; + return NCJTHROW(NCJ_OK); + return NCJTHROW(NCJ_ERR); } static int @@ -399,7 +415,7 @@ testint(const char* word) int count = 0; /* Try to convert to number */ ncvt = sscanf(word,"%lld%n",&i,&count); - return (ncvt == 1 && strlen(word)==count ? NCJ_OK : NCJ_ERR); + return NCJTHROW((ncvt == 1 && strlen(word)==count ? NCJ_OK : NCJ_ERR)); } static int @@ -409,16 +425,16 @@ testdouble(const char* word) double d; int count = 0; /* Check for Nan and Infinity */ - if(strcasecmp("nan",word)==0) return NCJ_OK; - if(strcasecmp("infinity",word)==0) return NCJ_OK; - if(strcasecmp("-infinity",word)==0) return NCJ_OK; + if(strcasecmp("nan",word)==0) return NCJTHROW(NCJ_OK); + if(strcasecmp("infinity",word)==0) return NCJTHROW(NCJ_OK); + if(strcasecmp("-infinity",word)==0) return NCJTHROW(NCJ_OK); /* Allow the XXXf versions as well */ - if(strcasecmp("nanf",word)==0) return NCJ_OK; - if(strcasecmp("infinityf",word)==0) return NCJ_OK; - if(strcasecmp("-infinityf",word)==0) return NCJ_OK; + if(strcasecmp("nanf",word)==0) return NCJTHROW(NCJ_OK); + if(strcasecmp("infinityf",word)==0) return NCJTHROW(NCJ_OK); + if(strcasecmp("-infinityf",word)==0) return NCJTHROW(NCJ_OK); /* Try to convert to number */ ncvt = sscanf(word,"%lg%n",&d,&count); - return (ncvt == 1 && strlen(word)==count ? NCJ_OK : NCJ_ERR); + return NCJTHROW((ncvt == 1 && strlen(word)==count ? NCJ_OK : NCJ_ERR)); } static int @@ -432,10 +448,10 @@ NCJyytext(NCJparser* parser, char* start, size_t pdlen) parser->yytext = (char*) realloc(parser->yytext,len+1); parser->yylen = len; } - if(parser->yytext == NULL) return NCJ_ERR; + if(parser->yytext == NULL) return NCJTHROW(NCJ_ERR); memcpy(parser->yytext,start,len); parser->yytext[len] = '\0'; - return NCJ_OK; + return NCJTHROW(NCJ_OK); } /**************************************************/ @@ -476,75 +492,7 @@ NCJreclaimArray(struct NCjlist* array) static void NCJreclaimDict(struct NCjlist* dict) { - return NCJreclaimArray(dict); -} - -int -NCJclone(const NCjson* json, NCjson** clonep) -{ - int stat = NCJ_OK; - NCjson* clone = NULL; - if(json == NULL) goto done; - switch(NCJsort(json)) { - case NCJ_INT: - case NCJ_DOUBLE: - case NCJ_BOOLEAN: - case NCJ_STRING: - if((stat=NCJnew(NCJsort(json),&clone))) goto done; - if((NCJstring(clone) = strdup(NCJstring(json))) == NULL) - {stat = NCJ_ERR; goto done;} - break; - case NCJ_NULL: - if((stat=NCJnew(NCJsort(json),&clone))) goto done; - break; - case NCJ_DICT: - if((stat=NCJcloneDict(json,&clone))) goto done; - break; - case NCJ_ARRAY: - if((stat=NCJcloneArray(json,&clone))) goto done; - break; - default: break; /* nothing to clone */ - } -done: - if(stat == NCJ_OK && clonep) {*clonep = clone; clone = NULL;} - NCJreclaim(clone); - return stat; -} - -static int -NCJcloneArray(const NCjson* array, NCjson** clonep) -{ - int i, stat=NCJ_OK; - NCjson* clone = NULL; - if((stat=NCJnew(NCJ_ARRAY,&clone))) goto done; - for(i=0;istring = (char*)malloc(len+1))==NULL) - {stat = NCJ_ERR; goto done;} + {stat = NCJTHROW(NCJ_ERR); goto done;} memcpy(json->string,value,len); json->string[len] = '\0'; if(jsonp) *jsonp = json; json = NULL; /* avoid memory errors */ done: NCJreclaim(json); - return (stat); -} - -int -NCJaddstring(NCjson* json, int sort, const char* s) -{ - int stat = NCJ_OK; - NCjson* jtmp = NULL; - - if(NCJsort(json) != NCJ_DICT && NCJsort(json) != NCJ_ARRAY) - {stat = NCJ_ERR; goto done;} - if((stat = NCJnewstring(sort, s, &jtmp))) goto done; - if((stat = NCJappend(json,jtmp))) goto done; - jtmp = NULL; - -done: - NCJreclaim(jtmp); - return stat; + return NCJTHROW(stat); } int @@ -631,7 +562,7 @@ NCJdictget(const NCjson* dict, const char* key, NCjson** valuep) int i,stat = NCJ_OK; if(dict == NULL || dict->sort != NCJ_DICT) - {stat = NCJ_ERR; goto done;} + {stat = NCJTHROW(NCJ_ERR); goto done;} if(valuep) {*valuep = NULL;} for(i=0;isort != NCJ_DICT || key == NULL || jvalue == NULL) - {stat = NCJ_ERR; goto done;} - if((stat = NCJnewstring(NCJ_STRING,key,&jkey))) goto done; - if((stat = NCJappend(object,jkey))) goto done; - if((stat = NCJappend(object,jvalue))) goto done; -done: - return stat; -} - -/* Append value to an array or dict object. */ -int -NCJappend(NCjson* object, NCjson* value) -{ - if(object == NULL || value == NULL) - return NCJ_ERR; - switch (object->sort) { - case NCJ_ARRAY: - case NCJ_DICT: - listappend(&object->list,value); - break; - default: - return NCJ_ERR; - } - return NCJ_OK; + return NCJTHROW(stat); } /* Unescape the text in parser->yytext; can @@ -702,139 +601,21 @@ NCJunescape(NCJparser* parser) *q++ = c; } *q = '\0'; - return NCJ_OK; -} - -/**************************************************/ -/* Unparser to convert NCjson object to text in buffer */ - -int -NCJunparse(const NCjson* json, unsigned flags, char** textp) -{ - int stat = NCJ_OK; - NCJbuf buf = {0,NULL}; - if((stat = NCJunparseR(json,&buf,flags))) - goto done; - if(textp) {*textp = buf.text; buf.text = NULL; buf.len = 0;} -done: - nullfree(buf.text); - return (stat); -} - -static int -NCJunparseR(const NCjson* json, NCJbuf* buf, unsigned flags) -{ - int stat = NCJ_OK; - int i; - - switch (NCJsort(json)) { - case NCJ_STRING: - bytesappendquoted(buf,json->string); - break; - case NCJ_INT: - case NCJ_DOUBLE: - case NCJ_BOOLEAN: - bytesappend(buf,json->string); - break; - case NCJ_DICT: - bytesappendc(buf,NCJ_LBRACE); - if(json->list.len > 0 && json->list.contents != NULL) { - int shortlist = 0; - for(i=0;!shortlist && i < json->list.len;i+=2) { - if(i > 0) bytesappendc(buf,NCJ_COMMA); - NCJunparseR(json->list.contents[i],buf,flags); /* key */ - bytesappendc(buf,NCJ_COLON); - bytesappendc(buf,' '); - /* Allow for the possibility of a short dict entry */ - if(json->list.contents[i+1] == NULL) { /* short */ - bytesappendc(buf,'?'); - shortlist = 1; - } else { - NCJunparseR(json->list.contents[i+1],buf,flags); - } - } - } - bytesappendc(buf,NCJ_RBRACE); - break; - case NCJ_ARRAY: - bytesappendc(buf,NCJ_LBRACKET); - if(json->list.len > 0 && json->list.contents != NULL) { - for(i=0;i < json->list.len;i++) { - if(i > 0) bytesappendc(buf,NCJ_COMMA); - NCJunparseR(json->list.contents[i],buf,flags); - } - } - bytesappendc(buf,NCJ_RBRACKET); - break; - case NCJ_NULL: - bytesappend(buf,"null"); - break; - default: - stat = NCJ_ERR; goto done; - } -done: - return (stat); -} - -/* Escape a string and append to buf */ -static int -escape(const char* text, NCJbuf* buf) -{ - const char* p = text; - int c; - for(;(c=*p++);) { - char replace = 0; - switch (c) { - case '\b': replace = 'b'; break; - case '\f': replace = 'f'; break; - case '\n': replace = 'n'; break; - case '\r': replace = 'r'; break; - case '\t': replace = 't'; break; - case NCJ_QUOTE: replace = '\''; break; - case NCJ_ESCAPE: replace = '\\'; break; - default: break; - } - if(replace) { - bytesappendc(buf,NCJ_ESCAPE); - bytesappendc(buf,replace); - } else - bytesappendc(buf,c); - } - return NCJ_OK; + return NCJTHROW(NCJ_OK); } -static int -bytesappendquoted(NCJbuf* buf, const char* s) -{ - bytesappend(buf,"\""); - escape(s,buf); - bytesappend(buf,"\""); - return NCJ_OK; -} - -void -NCJdump(const NCjson* json, unsigned flags, FILE* out) -{ - char* text = NULL; - (void)NCJunparse(json,0,&text); - if(out == NULL) out = stderr; - fprintf(out,"%s\n",text); - fflush(out); - nullfree(text); -} - -#ifdef DEBUG +#ifdef NCJDEBUG static char* tokenname(int token) { switch (token) { - case NCJ_STRING: return "NCJ_STRING"; - case NCJ_INT: return "NCJ_INT"; - case NCJ_DOUBLE: return "NCJ_DOUBLE"; - case NCJ_BOOLEAN: return "NCJ_BOOLEAN"; - case NCJ_DICT: return "NCJ_DICT"; - case NCJ_ARRAY: return "NCJ_ARRAY"; - case NCJ_NULL: return "NCJ_NULL"; + case NCJ_STRING: return ("NCJ_STRING"); + case NCJ_INT: return ("NCJ_INT"); + case NCJ_DOUBLE: return ("NCJ_DOUBLE"); + case NCJ_BOOLEAN: return ("NCJ_BOOLEAN"); + case NCJ_DICT: return ("NCJ_DICT"); + case NCJ_ARRAY: return ("NCJ_ARRAY"); + case NCJ_NULL: return ("NCJ_NULL"); default: if(token > ' ' && token <= 127) { static char s[4]; @@ -842,15 +623,14 @@ tokenname(int token) s[1] = (char)token; s[2] = '\''; s[3] = '\0'; - return s; + return (s); } else break; } - return "NCJ_UNDEF"; + return ("NCJ_UNDEF"); } #endif - /* Convert a JSON value to an equivalent value of a specified sort */ int NCJcvt(const NCjson* jvalue, int outsort, struct NCJconst* output) @@ -920,12 +700,12 @@ NCJcvt(const NCjson* jvalue, int outsort, struct NCJconst* output) break; default: - stat = NCJ_ERR; + stat = NCJTHROW(NCJ_ERR); break; } done: - return stat; + return NCJTHROW(stat); } static int @@ -936,17 +716,17 @@ listappend(struct NCjlist* list, NCjson* json) assert(list->len == 0 || list->contents != NULL); if(json == NULL) - {stat = NCJ_ERR; goto done;} + {stat = NCJTHROW(NCJ_ERR); goto done;} if(list->len == 0) { nullfree(list->contents); list->contents = (NCjson**)calloc(2,sizeof(NCjson*)); if(list->contents == NULL) - {stat = NCJ_ERR; goto done;} + {stat = NCJTHROW(NCJ_ERR); goto done;} list->contents[0] = json; list->len++; } else { if((newcontents = (NCjson**)calloc((2*list->len)+1,sizeof(NCjson*)))==NULL) - {stat = NCJ_ERR; goto done;} + {stat = NCJTHROW(NCJ_ERR); goto done;} memcpy(newcontents,list->contents,list->len*sizeof(NCjson*)); newcontents[list->len] = json; list->len++; @@ -956,28 +736,267 @@ listappend(struct NCjlist* list, NCjson* json) done: nullfree(newcontents); + return NCJTHROW(stat); +} + +/**************************************************/ + +#ifndef NETCDF_JSON_H + +int +NCJclone(const NCjson* json, NCjson** clonep) +{ + int stat = NCJ_OK; + NCjson* clone = NULL; + if(json == NULL) goto done; + switch(NCJsort(json)) { + case NCJ_INT: + case NCJ_DOUBLE: + case NCJ_BOOLEAN: + case NCJ_STRING: + if((stat=NCJnew(NCJsort(json),&clone))==NCJ_ERR) goto done; + if((NCJstring(clone) = strdup(NCJstring(json))) == NULL) + {stat = NCJTHROW(NCJ_ERR); goto done;} + break; + case NCJ_NULL: + if((stat=NCJnew(NCJsort(json),&clone))==NCJ_ERR) goto done; + break; + case NCJ_DICT: + if((stat=NCJcloneDict(json,&clone))==NCJ_ERR) goto done; + break; + case NCJ_ARRAY: + if((stat=NCJcloneArray(json,&clone))==NCJ_ERR) goto done; + break; + default: break; /* nothing to clone */ + } +done: + if(stat == NCJ_OK && clonep) {*clonep = clone; clone = NULL;} + NCJreclaim(clone); + return NCJTHROW(stat); +} + +static int +NCJcloneArray(const NCjson* array, NCjson** clonep) +{ + int i, stat=NCJ_OK; + NCjson* clone = NULL; + if((stat=NCJnew(NCJ_ARRAY,&clone))==NCJ_ERR) goto done; + for(i=0;isort != NCJ_DICT || key == NULL || jvalue == NULL) + {stat = NCJTHROW(NCJ_ERR); goto done;} + if((stat = NCJnewstring(NCJ_STRING,key,&jkey))==NCJ_ERR) goto done; + if((stat = NCJappend(object,jkey))==NCJ_ERR) goto done; + if((stat = NCJappend(object,jvalue))==NCJ_ERR) goto done; +done: + return NCJTHROW(stat); +} + +/* Append value to an array or dict object. */ +int +NCJappend(NCjson* object, NCjson* value) +{ + if(object == NULL || value == NULL) + return NCJTHROW(NCJ_ERR); + switch (object->sort) { + case NCJ_ARRAY: + case NCJ_DICT: + listappend(&object->list,value); + break; + default: + return NCJTHROW(NCJ_ERR); + } + return NCJTHROW(NCJ_OK); +} + +/**************************************************/ +/* Unparser to convert NCjson object to text in buffer */ + +int +NCJunparse(const NCjson* json, unsigned flags, char** textp) +{ + int stat = NCJ_OK; + NCJbuf buf = {0,NULL}; + if((stat = NCJunparseR(json,&buf,flags))==NCJ_ERR) + goto done; + if(textp) {*textp = buf.text; buf.text = NULL; buf.len = 0;} +done: + nullfree(buf.text); + return NCJTHROW(stat); +} + +static int +NCJunparseR(const NCjson* json, NCJbuf* buf, unsigned flags) +{ + int stat = NCJ_OK; + int i; + + switch (NCJsort(json)) { + case NCJ_STRING: + bytesappendquoted(buf,json->string); + break; + case NCJ_INT: + case NCJ_DOUBLE: + case NCJ_BOOLEAN: + bytesappend(buf,json->string); + break; + case NCJ_DICT: + bytesappendc(buf,NCJ_LBRACE); + if(json->list.len > 0 && json->list.contents != NULL) { + int shortlist = 0; + for(i=0;!shortlist && i < json->list.len;i+=2) { + if(i > 0) bytesappendc(buf,NCJ_COMMA); + NCJunparseR(json->list.contents[i],buf,flags); /* key */ + bytesappendc(buf,NCJ_COLON); + bytesappendc(buf,' '); + /* Allow for the possibility of a short dict entry */ + if(json->list.contents[i+1] == NULL) { /* short */ + bytesappendc(buf,'?'); + shortlist = 1; + } else { + NCJunparseR(json->list.contents[i+1],buf,flags); + } + } + } + bytesappendc(buf,NCJ_RBRACE); + break; + case NCJ_ARRAY: + bytesappendc(buf,NCJ_LBRACKET); + if(json->list.len > 0 && json->list.contents != NULL) { + for(i=0;i < json->list.len;i++) { + if(i > 0) bytesappendc(buf,NCJ_COMMA); + NCJunparseR(json->list.contents[i],buf,flags); + } + } + bytesappendc(buf,NCJ_RBRACKET); + break; + case NCJ_NULL: + bytesappend(buf,"null"); + break; + default: + stat = NCJTHROW(NCJ_ERR); goto done; + } +done: + return NCJTHROW(stat); +} + +/* Escape a string and append to buf */ +static int +escape(const char* text, NCJbuf* buf) +{ + const char* p = text; + int c; + for(;(c=*p++);) { + char replace = 0; + switch (c) { + case '\b': replace = 'b'; break; + case '\f': replace = 'f'; break; + case '\n': replace = 'n'; break; + case '\r': replace = 'r'; break; + case '\t': replace = 't'; break; + case NCJ_QUOTE: replace = '\''; break; + case NCJ_ESCAPE: replace = '\\'; break; + default: break; + } + if(replace) { + bytesappendc(buf,NCJ_ESCAPE); + bytesappendc(buf,replace); + } else + bytesappendc(buf,c); + } + return NCJTHROW(NCJ_OK); +} + +static int +bytesappendquoted(NCJbuf* buf, const char* s) +{ + bytesappend(buf,"\""); + escape(s,buf); + bytesappend(buf,"\""); + return NCJTHROW(NCJ_OK); +} + +void +NCJdump(const NCjson* json, unsigned flags, FILE* out) +{ + char* text = NULL; + (void)NCJunparse(json,0,&text); + if(out == NULL) out = stderr; + fprintf(out,"%s\n",text); + fflush(out); + nullfree(text); +} + static int bytesappend(NCJbuf* buf, const char* s) { int stat = NCJ_OK; char* newtext = NULL; if(buf == NULL) - {stat = NCJ_ERR; goto done;} + {stat = NCJTHROW(NCJ_ERR); goto done;} if(s == NULL) s = ""; if(buf->len == 0) { assert(buf->text == NULL); buf->text = strdup(s); if(buf->text == NULL) - {stat = NCJ_ERR; goto done;} + {stat = NCJTHROW(NCJ_ERR); goto done;} buf->len = strlen(s); } else { size_t slen = strlen(s); size_t newlen = buf->len + slen + 1; if((newtext = (char*)malloc(newlen))==NULL) - {stat = NCJ_ERR; goto done;} + {stat = NCJTHROW(NCJ_ERR); goto done;} strcpy(newtext,buf->text); strcat(newtext,s); free(buf->text); buf->text = NULL; @@ -987,7 +1006,7 @@ bytesappend(NCJbuf* buf, const char* s) done: nullfree(newtext); - return stat; + return NCJTHROW(stat); } static int @@ -998,3 +1017,4 @@ bytesappendc(NCJbuf* bufp, const char c) s[1] = '\0'; return bytesappend(bufp,s); } +#endif /*!NETCDF_JSON_H*/ diff --git a/libdispatch/nclist.c b/libdispatch/nclist.c index 3d45d92a9d..eddea55631 100644 --- a/libdispatch/nclist.c +++ b/libdispatch/nclist.c @@ -56,18 +56,26 @@ Free a list and its contents */ int nclistfreeall(NClist* l) +{ + nclistclearall(l); + return nclistfree(l); +} + +/* +Free the contents of a list +*/ +int +nclistclearall(NClist* l) { size_t i,len; - void** content = NULL; if(l == NULL) return TRUE; len = l->length; - content = nclistextract(l); for(i=0;icontent[i]; if(value != NULL) free(value); } - if(content != NULL) free(content); - return nclistfree(l); + nclistsetlength(l,0); + return TRUE; } int diff --git a/libhdf5/hdf5attr.c b/libhdf5/hdf5attr.c index 56b5edb373..c68fa83eca 100644 --- a/libhdf5/hdf5attr.c +++ b/libhdf5/hdf5attr.c @@ -460,7 +460,7 @@ nc4_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type, && (ra->flags & READONLYFLAG)) return NC_ENAMEINUSE; /* case 2: grp=NA, varid!=NC_GLOBAL, flags & HIDDENATTRFLAG */ - if (varid != NC_GLOBAL && (ra->flags & HIDDENATTRFLAG)) + if (varid != NC_GLOBAL && (ra->flags & (HIDDENATTRFLAG|READONLYFLAG))) return NC_ENAMEINUSE; } diff --git a/libhdf5/hdf5debug.c b/libhdf5/hdf5debug.c index fc8e206ca1..3a497d2f9c 100644 --- a/libhdf5/hdf5debug.c +++ b/libhdf5/hdf5debug.c @@ -15,14 +15,17 @@ #define STSIZE 1000 -#if !defined _WIN32 && !defined __CYGWIN__ +#ifdef H5BACKTRACE +# if !defined _WIN32 && !defined __CYGWIN__ static void* stacktrace[STSIZE]; +# endif #endif int nch5breakpoint(int err) { -#if !defined _WIN32 && !defined __CYGWIN__ +#ifdef H5BACKTRACE +# if !defined _WIN32 && !defined __CYGWIN__ int count = 0; char** trace = NULL; int i; @@ -32,9 +35,10 @@ nch5breakpoint(int err) fprintf(stderr,"backtrace:\n"); for(i=0;i +#endif + #undef TFILTERS +/* Forward */ static int NC4_hdf5_filter_free(struct NC_HDF5_Filter* spec); /**************************************************/ @@ -571,3 +576,15 @@ NC4_hdf5_find_missing_filter(NC_VAR_INFO_T* var, unsigned int* idp) if(idp) *idp = id; return stat; } + +int +NC4_hdf5_filter_initialize(void) +{ + return NC_NOERR; +} + +int +NC4_hdf5_filter_finalize(void) +{ + return NC_NOERR; +} diff --git a/libhdf5/hdf5internal.c b/libhdf5/hdf5internal.c index c2ffb6da99..843a919aff 100644 --- a/libhdf5/hdf5internal.c +++ b/libhdf5/hdf5internal.c @@ -82,6 +82,7 @@ nc4_hdf5_initialize(void) if (set_auto(NULL, NULL) < 0) LOG((0, "Couldn't turn off HDF5 error messages!")); LOG((1, "HDF5 error messages have been turned off.")); + NC4_hdf5_filter_initialize(); nc4_hdf5_initialized = 1; } @@ -94,6 +95,7 @@ nc4_hdf5_finalize(void) { /* Reclaim global resources */ NC4_provenance_finalize(); + NC4_hdf5_filter_finalize(); nc4_hdf5_initialized = 0; } diff --git a/liblib/CMakeLists.txt b/liblib/CMakeLists.txt index be72612c4e..f23898da94 100644 --- a/liblib/CMakeLists.txt +++ b/liblib/CMakeLists.txt @@ -37,6 +37,10 @@ IF(ENABLE_NCZARR) SET(liblib_LIBS ${liblib_LIBS} nczarr) ENDIF() +IF(ENABLE_PLUGINS) + SET(liblib_LIBS ${liblib_LIBS} ncpoco) +ENDIF() + FOREACH(LIBS ${liblib_LIBS}) SET(LARGS ${LARGS} $) ENDFOREACH() @@ -72,6 +76,10 @@ SET(TLL_LIBS ${TLL_LIBS} ${HAVE_LIBM} ${ZLIB_LIBRARY}) # Add extra dependencies specified via NC_EXTRA_DEPS SET(TLL_LIBS ${TLL_LIBS} ${EXTRA_DEPS}) +IF(ENABLE_BLOSC) +SET(TLL_LIBS ${TLL_LIBS} ${Blosc_LIBRARIES}) +ENDIF() + IF(HAVE_LIBDL) SET(TLL_LIBS ${LIBDL} ${TLL_LIBS}) ENDIF() @@ -117,6 +125,14 @@ IF(ENABLE_S3_SDK) TARGET_LINK_LIBRARIES(netcdf ${AWSSDK_LINK_LIBRARIES}) ENDIF() +IF(NOT WIN32) + IF(NOT APPLE) + IF(CMAKE_DL_LIBS) + TARGET_LINK_LIBRARIES(netcdf ${CMAKE_DL_LIBS}) + ENDIF() + ENDIF() +ENDIF() + IF(TLL_LIBS) LIST(REMOVE_DUPLICATES TLL_LIBS) ENDIF() diff --git a/liblib/Makefile.am b/liblib/Makefile.am index 4e693dbb06..55f6a82f6e 100644 --- a/liblib/Makefile.am +++ b/liblib/Makefile.am @@ -78,6 +78,11 @@ libnetcdf_la_LIBADD += ${aws_cpp_sdk_core_LIBS} ${aws_cpp_sdk_s3_LIBS} endif endif #ENABLE_NCZARR +if ENABLE_PLUGINS +AM_CPPFLAGS += -I${top_srcdir}/libncpoco +libnetcdf_la_LIBADD += ${top_builddir}/libncpoco/libncpoco.la +endif #ENABLE_PLUGINS + if ISCYGWIN # Force binary mode for file read/write AM_LDFLAGS += -lbinmode diff --git a/libncpoco/CMakeLists.txt b/libncpoco/CMakeLists.txt new file mode 100755 index 0000000000..a15ab5cffe --- /dev/null +++ b/libncpoco/CMakeLists.txt @@ -0,0 +1,8 @@ +SET(libncpoco_SOURCES ncpoco.c ncpoco.h) +IF(MSVC) + SET(libncpoco_SOURCES ${libncpoco_SOURCES} cp_win32.c) +ELSE() + SET(libncpoco_SOURCES ${libncpoco_SOURCES} cp_unix.c) +ENDIF() +add_library(ncpoco OBJECT ${libncpoco_SOURCES}) +ADD_EXTRA_DIST(${libncpoco_SOURCES} CMakeLists.txt) diff --git a/libncpoco/COPYRIGHT b/libncpoco/COPYRIGHT new file mode 100755 index 0000000000..0994357664 --- /dev/null +++ b/libncpoco/COPYRIGHT @@ -0,0 +1,39 @@ +The NetCDF Copyright. + +Copyright 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, +2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, +2015, 2016 +University Corporation for Atmospheric Research/Unidata. + +Portions of this software were developed by the Unidata Program at the +University Corporation for Atmospheric Research. + +Access and use of this software shall impose the following obligations +and understandings on the user. The user is granted the right, without +any fee or cost, to use, copy, modify, alter, enhance and distribute +this software, and any derivative works thereof, and its supporting +documentation for any purpose whatsoever, provided that this entire +notice appears in all copies of the software, derivative works and +supporting documentation. Further, UCAR requests that the user credit +UCAR/Unidata in any publications that result from the use of this +software or in any product that includes this software, although this +is not an obligation. The names UCAR and/or Unidata, however, may not +be used in any advertising or publicity to endorse or promote any +products or commercial entity unless specific written permission is +obtained from UCAR/Unidata. The user also understands that +UCAR/Unidata is not obligated to provide the user with any support, +consulting, training or assistance of any kind with regard to the use, +operation and performance of this software nor to provide the user +with any updates, revisions, new versions or "bug fixes." + +THIS SOFTWARE IS PROVIDED BY UCAR/UNIDATA "AS IS" AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL UCAR/UNIDATA BE LIABLE FOR ANY SPECIAL, +INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING +FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, +NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION +WITH THE ACCESS, USE OR PERFORMANCE OF THIS SOFTWARE. + + + diff --git a/libncpoco/Makefile.am b/libncpoco/Makefile.am new file mode 100755 index 0000000000..0a553cbd43 --- /dev/null +++ b/libncpoco/Makefile.am @@ -0,0 +1,40 @@ +# Copyright 2009, UCAR/Unidata +# See the COPYRIGHT file for more information. + +# Use automake or CMake for building under nix +# Use CMake for building under windows + +# Get AM_CPPFLAGS and AM_LDFLAGS +include $(top_srcdir)/lib_flags.am +#AM_CPPFLAGS += -D_LARGEFILE_SOURCE +#AM_CPPFLAGS += -I$(top_srcdir)/libsrc4 -I$(top_srcdir)/libdap4 +#libnczarr_la_CPPFLAGS = ${AM_CPPFLAGS} +AM_CXXFLAGS = + +# This is our output. The ZARR convenience library. +noinst_LTLIBRARIES = libncpoco.la +libncpoco_la_LIBADD = +libncpoco_la_LDFLAGS = + +libncpoco_la_SOURCES = ncpoco.c ncpoco.h +if ISMSVC +libncpoco_la_SOURCES += cp_win32.c +else +libncpoco_la_SOURCES += cp_unix.c +endif + +EXTRA_DIST = CMakeLists.txt README.md COPYRIGHT + +################################################## +# Testing +# check_LTLIBRARIES = libcpt.la +# libcpt_la_SOURCES = cptestlib.c +# +# # Normally check libraries are created only as archives, +# # but we need a shared lib. This appears to do the trick +# libcpt_la_LDFLAGS = -module -avoid-version -shared -export-dynamic -no-undefined -rpath ${abs_builddir} +# +# AM_LDFLAGS += libcpoco.la +# check_PROGRAMS = cp_test +# TESTS = cp_test +# diff --git a/libncpoco/README.md b/libncpoco/README.md new file mode 100644 index 0000000000..0c74977d81 --- /dev/null +++ b/libncpoco/README.md @@ -0,0 +1,128 @@ +# The cpoco Multi-Platform Dynamic-Loading Library + +## Description + +The primary goal of the cpoco project is to provide a C language +version of the poco multi-platform dynamic loading library. The +poco libraries are written in C++. + +The secondary goal of cpoco is to support dynamic loading of +HDF5 filters by the [netCDF C library](http://www.unidata.ucar.edu/netcdf/). + +# Mutual Exclusion Support + +Internally, cpoco (like poco) supports serialized access to the dynamic +loading functions using mutual exclusion locks. For *nix* systems, this usually requires pthreads support. + +In any case, it is possible to disable the use of mutual exclusion if you know +you are operating in a single threaded environment: see the [installation](#installation) section below. + +# Implementataion Restrictions + +Currently support is provided for the following systems. + +* libdl supporting operating systems: e.g. linux, os-x, cygwin. +* Windows-32 api + +# Installation + +## Automake + +Automake-based configuration is provided using ./configure is provided for systems supporting autoconf/automake. + +Use this command to see the available options. +```bash +./configure --help +``` +The most important options are these. + +* --disable-mutex -- disable using mutual exclusion (default is enabled) +* --disable-pthread -- disable using pthreads (default is enabled if operating system provides it) +* --enable-shared -- build a shared library (default is enabled) +* --enable-static -- build a static library (default is enabled) +* --prefix= -- defaults to /usr/local + +Note that is --enable-shared is disabled, then the test program +will not run because the test shared library (libcpt) cannot be built. + +Use these commands to build, test, and install using autoconf. +```bash +# Invoke configure +./configure +make all +# If testing is desired +make check +# Optional +make install +``` +The are those shown by `./configure --help`. + +## CMake +Cmake-based configuration is provided provided for systems supporting it. If building for windows, then this is the only option provided. + +Use these commands to build using cmake. +```bash +# Create a build directory +rm -fr build +mkdir build +cd build +# Invoke cmake +cmake .. +cmake --build . +# If testing is desired +CTEST_OUTPUT_ON_FAILURE=1 cmake --build . --target test +# Optional +cmake --build . --target install +``` + +The are these: + +* -DENABLE_MUTEX -- use mutual exclusion (default is true) +* -DENABLE_PTHREAD -- use pthreads (if available) (default is true) +* -DCMAKE_INSTALL_PREFIX= -- installation directory (defaults to /usr/local or c:/Program Files) + +# Support + +__Author__: Dennis Heimbigner +__Organization__: University Corporation for Atmospheric Research +__Initial Release__: 2015-9-28 +__Last Modified__: 2018-03-27 + +__Copyright__: Copyright 2018, UCAR/Unidata; see COPYRIGHT file for copying and redistribution conditions. This code is Derived from the poco library (See [Poco Information](#poco-information) below). + +# Change Log + +1. (2016-09-28) Initial release. +2. (2018-03-27) Updated to make fix some automake problems. + +# Poco Information + +* Base poco version: 1.7.5 (2016-08-29) +* Poco web page: http://pocoproject.org + + +## Poco License + +> Boost Software License - Version 1.0 - August 17th, 2003 +> +> Permission is hereby granted, free of charge, to any person or organization +> obtaining a copy of the software and accompanying documentation covered by +> this license (the "Software") to use, reproduce, display, distribute, +> execute, and transmit the Software, and to prepare derivative works of the +> Software, and to permit third-parties to whom the Software is furnished to +> do so, all subject to the following: +> +> The copyright notices in the Software and this entire statement, including +> the above license grant, this restriction and the following disclaimer, +> must be included in all copies of the Software, in whole or in part, and +> all derivative works of the Software, unless such copies or derivative +> works are solely in the form of machine-executable object code generated by +> a source language processor. +> +> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +> IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +> FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +> SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +> FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +> ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +> DEALINGS IN THE SOFTWARE. diff --git a/libncpoco/SourceLicence b/libncpoco/SourceLicence new file mode 100644 index 0000000000..5bd0a3865c --- /dev/null +++ b/libncpoco/SourceLicence @@ -0,0 +1,25 @@ +## Poco License + +> Boost Software License - Version 1.0 - August 17th, 2003 +> +> Permission is hereby granted, free of charge, to any person or organization +> obtaining a copy of the software and accompanying documentation covered by +> this license (the "Software") to use, reproduce, display, distribute, +> execute, and transmit the Software, and to prepare derivative works of the +> Software, and to permit third-parties to whom the Software is furnished to +> do so, all subject to the following: +> +> The copyright notices in the Software and this entire statement, including +> the above license grant, this restriction and the following disclaimer, +> must be included in all copies of the Software, in whole or in part, and +> all derivative works of the Software, unless such copies or derivative +> works are solely in the form of machine-executable object code generated by +> a source language processor. +> +> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +> IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +> FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +> SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +> FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +> ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +> DEALINGS IN THE SOFTWARE. diff --git a/libncpoco/cp_test.c b/libncpoco/cp_test.c new file mode 100755 index 0000000000..2f51b7465f --- /dev/null +++ b/libncpoco/cp_test.c @@ -0,0 +1,247 @@ +/* +Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. +and Contributors. +*/ + +#include "config.h" +#include +#include +#include +#include +#include +#ifdef HAVE_UNISTD_H +#include +#endif + +#include "cpoco.h" + +/* Possible names of the test shared library */ +static const char* SHAREDLIBS[] = { + "./libcpt.so", + "./cygcpt.dll", + "./cpt.dll", + "./.libs/libcpt.so", + "./.libs/cygcpt.dll", + "./.libs/cpt.dll", + "./.libs/cygcpt-0.dll", + NULL}; + +typedef int (*GimmeFiveFunc)(); + +#define CHECK(err) check(CE_ERR,0,__LINE__,__FILE__,(err)) +#define CHECKNO(err) check(CE_ERR,1,__LINE__,__FILE__,(err)) +#define CHECKBOOL(b) check(CE_BOOL,0,__LINE__,__FILE__,(b)) +#define CHECKNOBOOL(b) check(CE_BOOL,1,__LINE__,__FILE__,(b)) +#define CHECKSTR(s1,s2) check(CE_STR,0,__LINE__,__FILE__,(s1),(s2)) +#define CHECKSYM(sym) check(CE_SYM,0,__LINE__,__FILE__,(sym)) +#define CHECKNOSYM(sym) check(CE_SYM,1,__LINE__,__FILE__,(sym)) + +enum CE { + CE_ERR, + CE_BOOL, + CE_STR, + CE_SYM, +}; + +static void +check(enum CE ce, int invert, int lineno, const char* file, ...) +{ + va_list args; + va_start (args,file); + int err; + char* s1; + char* s2; + void* sym; + + switch (ce) { + case CE_ERR: + err = va_arg(args,int); + if((!invert && err != CP_OK) || (invert && err == CP_OK)) { + fprintf(stderr,"(%s:%d): %s\n",file,lineno,cperrstr(err)); + goto fail; + } + break; + case CE_BOOL: + err = va_arg(args,int); + if((err && invert) || (!err && !invert)) { + fprintf(stderr,"(%s:%d): boolean fail\n",file,lineno); + goto fail; + } + break; + case CE_STR: + s1 = va_arg(args,char*); + s2 = va_arg (args,char*); + if(s1 == NULL) s1 = "null"; + if(s2 == NULL) s2 = "null"; + if((!invert && strcmp(s1,s2) != 0) || (invert && strcmp(s1,s2) == 0)) { + fprintf(stderr,"(%s:%d): mismatch: %s::%s \n",file,lineno,s1,s2); + goto fail; + } + break; + case CE_SYM: + sym = va_arg(args,void*); + if((!invert && sym == NULL) || (invert && sym != NULL)) { + fprintf(stderr,"(%s:%d): get symbol failure\n",file,lineno); + goto fail; + } + break; + default: abort(); + } + va_end (args); // clean up the system stack + return; +fail: + exit(1); +} + +static void +test1(const char* path) +{ + SharedLib* sl = NULL; + void* p1; + + CHECK(cpsharedlibnew(&sl)); + CHECK(cpload(sl,path,0)); + CHECKBOOL(cpisloaded(sl)); + CHECKNO(cpload(sl,path,0)); + CHECKSTR(cpgetpath(sl),path); + CHECKBOOL(cpisloaded(sl)); + CHECKSYM(cpgetsymbol(sl,"gimmeFive")); + CHECKNOSYM(cpgetsymbol(sl,"fooBar123")); + + p1 = cpgetsymbol(sl,"pocoBuildManifest"); + CHECK((p1 != NULL)); + + p1 = cpgetsymbol(sl,"fooBar123"); + cpunload(sl); + CHECKNOBOOL(cpisloaded(sl)); +} + +static void +test2(const char* path) +{ + SharedLib* sl = NULL; + GimmeFiveFunc gimmeFive; + + CHECK(cpsharedlibnew(&sl)); + CHECK(cpload(sl,path,0)); + CHECKSTR(cpgetpath(sl),path); + CHECKBOOL(cpisloaded(sl)); + + gimmeFive = (GimmeFiveFunc) cpgetsymbol(sl,"gimmeFive"); + CHECKSYM(gimmeFive); + CHECKBOOL((gimmeFive() == 5)); + CHECK(cpunload(sl)); + CHECKNOBOOL(cpisloaded(sl)); +} + +static void +test3(const char* path) +{ + SharedLib* sl = NULL; + + CHECK(cpsharedlibnew(&sl)); + CHECK(cpload(sl,path,0)); + CHECKSTR(cpgetpath(sl),path); + CHECKBOOL(cpisloaded(sl)); + CHECKNO(cpload(sl,"NoSuchLibrary",0)); + CHECKBOOL(cpisloaded(sl)); + CHECKNO(cpload(sl,path,0)); + CHECKBOOL(cpisloaded(sl)); + CHECKNO(cpload(sl,path,0)); + CHECKBOOL(cpisloaded(sl)); + CHECK(cpunload(sl)); + CHECKNOBOOL(cpisloaded(sl)); +} + +static char* +abspath(const char* prefixpath, const char* relpath) +{ + char* full = NULL; + if(prefixpath == NULL) { +#ifdef _MSC_VER + full = _fullpath(NULL,relpath,8192); +#else + full = realpath(relpath, NULL); +#endif + } else { /* concatenate */ + full = malloc(strlen(prefixpath)+strlen(relpath)+2); + strcpy(full,prefixpath); +#ifdef _MSC_VER + strcat(full,"\\"); +#else + strcat(full,"/"); +#endif + strcat(full,relpath); + } + fprintf(stderr,"abspath=%s\n",full); + return full; +} + +static const char* +findlib(const char* prefixpath) +{ + char* path; + const char** p; + for(p=SHAREDLIBS;*p;p++) { + FILE* f; + path = abspath(prefixpath,*p); + if(path == NULL) continue; + f = fopen(path,"r"); +fprintf(stderr,"findlib: trying %s\n",path); + if(f != NULL) { + fclose(f); + return *p; + } + } +fprintf(stderr,"findlib: failed\n"); + return NULL; +} + + +int +main(int argc, char** argv) +{ + const char* prefixpath = NULL; + const char* relpath = NULL; + const char* path = NULL; + + switch (argc) { + default: + case 3: + relpath = argv[2]; + /* fall thru */ + case 2: + prefixpath = argv[1]; + break; + case 1: + case 0: + break; + } + + if(prefixpath == NULL) { + fprintf(stderr,"No prefix path specified\n"); + return 1; + } + + if(relpath == NULL) + relpath = findlib(prefixpath); + + if(relpath == NULL) { + fprintf(stderr,"No dynamic test library specified\n"); + return 1; + } + + path = abspath(prefixpath,relpath); + + if(path == NULL) { + fprintf(stderr,"Cannot locate dynamic test library: %s\n",relpath); + exit(1); + } + + printf("Using shared test library: %s\n",path); + + test1(path); + test2(path); + test3(path); + return 0; +} diff --git a/libncpoco/cp_unix.c b/libncpoco/cp_unix.c new file mode 100755 index 0000000000..1af2622fe8 --- /dev/null +++ b/libncpoco/cp_unix.c @@ -0,0 +1,173 @@ +/********************************************************************* + * Copyright 1993, UCAR/Unidata + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + * Derived from poco library from Boost Software: see nccpoco/LICENSE file. + *********************************************************************/ + +#include "config.h" +#include +#include +#include +#ifdef HAVE_DLFCN_H +#include +#endif +#ifdef USE_MUTEX +#include +#endif + +#include "netcdf.h" +#include "ncpoco.h" +#include "ncpathmgr.h" + +/* Note: cygwin is missing RTLD_LOCAL, set it to 0 */ +#if !defined(RTLD_LOCAL) +#define RTLD_LOCAL 0 +#endif + +#if !defined(RTLD_GLOBAL) +#define RTLD_GLOBAL 0 +#endif + +#if !defined(RTLD_LAZY) +#define RTLD_LAZY 0 +#endif + +#ifdef USE_MUTEX +static pthread_mutex_t mutex; +#endif + +int +ncp_unix_initialize() +{ + int ret = 1; +#ifdef USE_MUTEX + pthread_mutexattr_t attr; + pthread_mutexattr_init(&attr); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_NORMAL); + if(pthread_mutex_init(&mutex, &attr)) { + fprintf(stderr,"cp_unix: Cannot create mutext\n"); + ret = 1; + } + pthread_mutexattr_destroy(&attr); +#endif + return ret; +} + +int +ncp_unix_finalize() +{ +#ifdef USE_MUTEX + pthread_mutex_destroy(&mutex); +#endif + return 1; +} + +/**************************************************/ + +#ifdef USE_MUTEX +static void lock(void) {pthread_mutex_lock(&mutex);} +#else +#define lock() +#endif + +#ifdef USE_MUTEX +static void unlock(void) {pthread_mutex_unlock(&mutex);} +#else +#define unlock() +#endif + +/**************************************************/ + +static int +init(NCPSharedLib* lib) +{ + int ret = NC_NOERR; + return ret; +} + +static int +reclaim(NCPSharedLib* lib) +{ + int ret = NC_NOERR; + return ret; +} + +static int +load(NCPSharedLib* lib , const char* path0, int flags) +{ + int ret = NC_NOERR; + int realflags = RTLD_LAZY; + char* path = NULL; + + if((path = NCpathcvt(path0))==NULL) {ret = NC_ENOMEM; goto done;} + + lock(); + if(lib->state.handle != NULL) + {ret = NC_EEXIST; goto ldone;} + lib->path = nulldup(path); + lib->flags = flags; + if(flags & NCP_LOCAL) + realflags |= RTLD_LOCAL; + else + realflags |= RTLD_GLOBAL; + lib->state.flags = realflags; + lib->state.handle = dlopen(lib->path, lib->state.flags); + if(lib->state.handle == NULL) { + const char* msg = dlerror(); + if(msg == NULL) msg = ""; + strncpy(lib->err.msg,msg,sizeof(lib->err.msg)); + ret = NC_ENOTFOUND; + goto ldone; + } +ldone: + unlock(); +done: + nullfree(path); + return ret; +} + +static int +unload(NCPSharedLib* lib) +{ + int ret = NC_NOERR; + lock(); + if(lib->state.handle != NULL) { + dlclose(lib->state.handle); + lib->state.handle = NULL; + } + unlock(); + return ret; +} + +static int +isLoaded(NCPSharedLib* lib) +{ + return lib->state.handle != NULL; +} + +static void* +getsymbol(NCPSharedLib* lib, const char* name) +{ + void* result = NULL; + lock(); + if(lib->state.handle != NULL) + result = dlsym(lib->state.handle, name); + unlock(); + return result; +} + +static const char* +getpath(NCPSharedLib* lib) +{ + return lib->path; +} + +struct NCPAPI ncp_unix_api = { + init, + reclaim, + load, + unload, + isLoaded, + getsymbol, + getpath, +}; diff --git a/libncpoco/cp_win32.c b/libncpoco/cp_win32.c new file mode 100755 index 0000000000..cb696e0c0e --- /dev/null +++ b/libncpoco/cp_win32.c @@ -0,0 +1,186 @@ +/********************************************************************* + * Copyright 1993, UCAR/Unidata + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + * Derived from poco library from Boost Software: see nccpoco/LICENSE file. + *********************************************************************/ + +#ifdef _WIN32 + +#include "config.h" +#include +#include +#include +#include + +#include +#include + +#include "netcdf.h" +#include "ncpoco.h" +#include "ncpathmgr.h" + +#ifdef USE_MUTEX +static CRITICAL_SECTION mutex; +#endif + +static const char* driveletters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; + +/* Forward */ +static int isAbsolutePath(const char* path); + +/**************************************************/ + +int +ncp_win32_initialize() +{ + int ret = 1; +#ifdef USE_MUTEX + InitializeCriticalSectionAndSpinCount(&mutex, 4000); +#endif + (void)SetErrorMode(SEM_FAILCRITICALERRORS|SEM_NOGPFAULTERRORBOX|SEM_NOOPENFILEERRORBOX); + return ret; +} + +int +ncp_win32_finalize() +{ +#ifdef USE_MUTEX + DeleteCriticalSection(&mutex); +#endif + return 1; +} + +/**************************************************/ + + +#ifdef USE_MUTEX +static void lock(void) {EnterCriticalSection(&mutex);} +#else +#define lock() +#endif + +#ifdef USE_MUTEX +static void unlock(void) {LeaveCriticalSection(&mutex);} +#else +#define unlock() +#endif + +/**************************************************/ + +static int +init(NCPSharedLib* lib) +{ + int ret = NC_NOERR; + return ret; +} + +static int +reclaim(NCPSharedLib* lib) +{ + int ret = NC_NOERR; + return ret; +} + +static int +load(NCPSharedLib* lib , const char* path0, int flags) +{ + int ret = NC_NOERR; + DWORD realflags = 0; + char* path = NULL; + + path = NCpathcvt(path0); + if(path == NULL) {ret = NC_EINVAL; goto done;} + + lock(); + if(lib->state.handle != NULL) + {ret = NC_EEXIST; goto ldone;} + lib->path = nulldup(path); + lib->flags = flags; + if(isAbsolutePath(path)) realflags |= LOAD_WITH_ALTERED_SEARCH_PATH; + lib->state.flags = realflags; + lib->state.handle = LoadLibraryExA(path, 0, realflags); + if(lib->state.handle == NULL) { + int errcode = GetLastError(); + char* msg = NULL; + FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, errcode, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), &msg, 0, NULL); + if(msg) { + strncpy(lib->err.msg,msg,sizeof(lib->err.msg)); + } else + lib->err.msg[0] = '\0'; + ret = NC_ENOTFOUND; + goto ldone; + } + +ldone: + unlock(); +done: + nullfree(path); + return ret; +} + +static int +unload(NCPSharedLib* lib) +{ + int ret = NC_NOERR; + + lock(); + if(lib->state.handle != NULL) { + FreeLibrary((HMODULE)lib->state.handle); + lib->state.handle = NULL; + goto done; + } +done: + unlock(); + return ret; +} + +static int +isLoaded(NCPSharedLib* lib) +{ + return lib->state.handle != NULL; +} + +static void* +getsymbol(NCPSharedLib* lib, const char* name) +{ + void* result = NULL; + lock(); + if(lib->state.handle != NULL) { + result = (void*)GetProcAddress((HMODULE)lib->state.handle,name); + } + unlock(); + return result; +} + +static const char* +getpath(NCPSharedLib* lib) +{ + return lib->path; +} + +static int +isAbsolutePath(const char* path) +{ + if(path == NULL || path[0] == '\0') + return 0; + if(strchr(driveletters,path[0]) != NULL + && path[1] == ':' + && (path[2] == '/' || path[2] == '\\')) + return 1; + if(path[0] == '/' || path[2] == '\\') + return 1; + return 0; +} + +struct NCPAPI ncp_win32_api = { + init, + reclaim, + load, + unload, + isLoaded, + getsymbol, + getpath, +}; + +#endif /*_WIN32*/ diff --git a/libncpoco/cptestlib.c b/libncpoco/cptestlib.c new file mode 100755 index 0000000000..5614bf6bc3 --- /dev/null +++ b/libncpoco/cptestlib.c @@ -0,0 +1,10 @@ +#include "assert.h" + +#define DLL_EXPORT + +#include "cpoco.h" + +EXTERNL int gimmeFive() +{ + return 5; +} diff --git a/libncpoco/ncpoco.c b/libncpoco/ncpoco.c new file mode 100755 index 0000000000..12d4b41eec --- /dev/null +++ b/libncpoco/ncpoco.c @@ -0,0 +1,115 @@ +/********************************************************************* + * Copyright 1993, UCAR/Unidata + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + * Derived from poco library from Boost Software: see nccpoco/SourceLicense file. + *********************************************************************/ + +#include "config.h" +#include +#include +#ifdef HAVE_UNISTD_H +#include +#endif + +#include "netcdf.h" +#include "ncpoco.h" + +/**************************************************/ +#ifdef _WIN32 +extern struct NCPAPI ncp_win32_api; +#else +extern struct NCPAPI ncp_unix_api; +#endif + +/**************************************************/ + +/* Creates a SharedLib object. */ +EXTERNL int +ncpsharedlibnew(NCPSharedLib** libp) +{ + int ret = NC_NOERR; + NCPSharedLib* lib; + lib = (NCPSharedLib*)calloc(1,sizeof(NCPSharedLib)); + if(lib == 0) + {ret = NC_ENOMEM; goto done;} + /* fill in the api */ +#ifdef _WIN32 + lib->api = ncp_win32_api; +#else + lib->api = ncp_unix_api; +#endif + ret = lib->api.init(lib); + if(ret == NC_NOERR && libp) + *libp = lib; + +done: + return ret; +} + +/* free this shared library */ +EXTERNL int +ncpsharedlibfree(NCPSharedLib* lib) +{ + int ret = NC_NOERR; + if(lib == NULL) return NC_NOERR; + ret = lib->api.unload(lib); + ret = lib->api.reclaim(lib); + /* reclaim common stuff */ + nullfree(lib->path); + free(lib); + return ret; +} + +/* +Loads a shared library from the given path using specified flags. +Returns error if a library has already been loaded or cannot be loaded. +*/ +EXTERNL int +ncpload(NCPSharedLib* lib, const char* path, int flags) +{ + if(lib == NULL || path == NULL) return NC_EINVAL; + return lib->api.load(lib,path,flags); +} + +EXTERNL int +ncpunload(NCPSharedLib* lib) /* Unloads a shared library. */ +{ + if(lib == NULL) return NC_EINVAL; + return lib->api.unload(lib); +} + +EXTERNL int +ncpisloaded(NCPSharedLib* lib) /* Returns 1 iff a library has been loaded. */ +{ + if(lib == NULL) return 0; + return lib->api.isloaded(lib); +} + +/* Returns the address of the symbol with the given name. + For functions, this is the entry point of the function. + Return error if the symbol does not exist +*/ +EXTERNL void* +ncpgetsymbol(NCPSharedLib* lib,const char* name) +{ + if(lib == NULL) return NULL; + return lib->api.getsymbol(lib,name); +} + +/* Returns the path of the library, as specified in + a call to load() +*/ +EXTERNL const char* +ncpgetpath(NCPSharedLib* lib) +{ + if(lib == NULL) return NULL; + return lib->api.getpath(lib); +} + +/* Returns the last err msg. */ +EXTERNL const char* +ncpgeterrmsg(NCPSharedLib* lib) +{ + if(lib == NULL) return NULL; + return (lib->err.msg[0] == '\0' ? NULL : lib->err.msg); +} diff --git a/libncpoco/ncpoco.h b/libncpoco/ncpoco.h new file mode 100755 index 0000000000..43fb077bab --- /dev/null +++ b/libncpoco/ncpoco.h @@ -0,0 +1,76 @@ +/********************************************************************* + * Copyright 1993, UCAR/Unidata + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + * Derived from poco library from Boost Software: see nccpoco/SourceLicence file. + *********************************************************************/ + +#ifndef NCPOCO_H +#define NCPOCO_H + +/* Use NCP or nccp or ncpoco as our "namespace */ + +/* Forward */ +typedef struct NCPSharedLib NCPSharedLib; +typedef enum NCP_Flags NCP_Flags; + +enum NCP_Flags { + NCP_GLOBAL = 1, /* (default) use RTLD_GLOBAL; ignored on platforms that do not use dlopen(). */ + NCP_LOCAL = 2 /* use RTLD_LOCAL; RTTI will not work with this flag. */ +}; + +struct NCPSharedLib { + char* path; + int flags; + struct NCPstate { + void* handle; + int flags; + } state; + struct { + char msg[4096]; + } err; + struct NCPAPI { + int (*init)(NCPSharedLib*); + int (*reclaim)(NCPSharedLib*); /* reclaim resources; do not free lib */ + int (*load)(NCPSharedLib*,const char*,int); + int (*unload)(NCPSharedLib*); + int (*isloaded)(NCPSharedLib*); + void* (*getsymbol)(NCPSharedLib*,const char*); + const char* (*getpath)(NCPSharedLib*); + } api; +}; + + +/**************************************************/ +/* SharedLib API */ + +EXTERNL int ncpsharedlibnew(NCPSharedLib**); /* Creates a NCPSharedLib object. */ + +EXTERNL int ncpsharedlibfree(NCPSharedLib*); /* free this shared library */ + +/* +Loads a shared library from the given path using specified flags. +Returns error + message if a library has already been loaded or cannot be loaded. +*/ +EXTERNL int ncpload(NCPSharedLib*,const char* path, int flags); + +EXTERNL int ncpunload(NCPSharedLib*); /* Unloads a shared library. */ + +EXTERNL int ncpisloaded(NCPSharedLib*); /* Returns 1 iff a library has been loaded.*/ + +/* Returns the address of the symbol with the given name. + For functions, this is the entry point of the function. + Return NULL if the symbol does not exist +*/ +EXTERNL void* ncpgetsymbol(NCPSharedLib*,const char* name); + +/* Returns the path of the library, as specified in + a call to load(NCPSharedLib*) +*/ +EXTERNL const char* ncpgetpath(NCPSharedLib*); + +/* Return last err msg */ +EXTERNL const char* ncpgeterrmsg(NCPSharedLib* lib); + +EXTERNL const char* intstr(int err1); + +#endif /*NCPOCO_H*/ diff --git a/libnczarr/CMakeLists.txt b/libnczarr/CMakeLists.txt index 86e093b20d..b0d2b68e38 100644 --- a/libnczarr/CMakeLists.txt +++ b/libnczarr/CMakeLists.txt @@ -42,6 +42,7 @@ zinternal.h zmap.h zodom.h zprovenance.h +zfilter.h zdebug.h ) @@ -58,6 +59,8 @@ ENDIF() add_library(nczarr OBJECT ${libnczarr_SOURCES}) +TARGET_INCLUDE_DIRECTORIES(nczarr PUBLIC ../libncpoco) + IF(MPI_C_INCLUDE_PATH) target_include_directories(nczarr PUBLIC ${MPI_C_INCLUDE_PATH}) ENDIF(MPI_C_INCLUDE_PATH) diff --git a/libnczarr/Makefile.am b/libnczarr/Makefile.am index 22a0631ce5..e1f8ef5d43 100644 --- a/libnczarr/Makefile.am +++ b/libnczarr/Makefile.am @@ -8,7 +8,7 @@ # Get AM_CPPFLAGS and AM_LDFLAGS include $(top_srcdir)/lib_flags.am AM_CPPFLAGS += -D_LARGEFILE_SOURCE -AM_CPPFLAGS += -I$(top_srcdir)/libsrc4 -I$(top_srcdir)/libdap4 +AM_CPPFLAGS += -I$(top_srcdir)/libsrc4 -I$(top_srcdir)/libncpoco libnczarr_la_CPPFLAGS = ${AM_CPPFLAGS} AM_CXXFLAGS = libnczarr_la_LIBADD = @@ -38,7 +38,6 @@ zcvt.c \ zdim.c \ zdispatch.c \ zfile.c \ -zfilter.c \ zgrp.c \ zinternal.c \ zmap.c \ @@ -51,6 +50,7 @@ ztype.c \ zutil.c \ zvar.c \ zwalk.c \ +zfilter.c \ zdebug.c \ zarr.h \ zcache.h \ @@ -62,6 +62,7 @@ zinternal.h \ zmap.h \ zodom.h \ zprovenance.h \ +zfilter.h \ zdebug.h if ENABLE_NCZARR_ZIP diff --git a/libnczarr/awsincludes.h b/libnczarr/awsincludes.h index cf1b2bc510..f45c349e2f 100755 --- a/libnczarr/awsincludes.h +++ b/libnczarr/awsincludes.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include diff --git a/libnczarr/obsolete/zjson.c b/libnczarr/obsolete/zjson.c deleted file mode 100644 index a55fb81f4d..0000000000 --- a/libnczarr/obsolete/zjson.c +++ /dev/null @@ -1,865 +0,0 @@ -/* Copyright 2018, UCAR/Unidata. - See the COPYRIGHT file for more information. -*/ - -#include "zincludes.h" -#include "zjson.h" - -#undef DEBUG - -#define NCJ_LBRACKET '[' -#define NCJ_RBRACKET ']' -#define NCJ_LBRACE '{' -#define NCJ_RBRACE '}' -#define NCJ_COLON ':' -#define NCJ_COMMA ',' -#define NCJ_QUOTE '"' -#define NCJ_ESCAPE '\\' -#define NCJ_TAG_TRUE "true" -#define NCJ_TAG_FALSE "false" -#define NCJ_TAG_NULL "null" - -#define NCJ_EOF -1 -#define NCJ_ERR -2 - -/* WORD Subsumes Number also */ -#define WORD "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_$+-." - -/*//////////////////////////////////////////////////*/ - -typedef struct NCJparser { - char* text; - char* pos; - size_t yylen; /* |yytext| */ - char* yytext; /* string or word */ - long long num; - int tf; - int err; -} NCJparser; - -/**************************************************/ -/* Forward */ -static int NCJparseR(NCJparser* parser, NCjson**); -static int NCJparseArray(NCJparser* parser, NClist* array); -static int NCJparseDict(NCJparser* parser, NClist* dict); -static int testbool(const char* word); -static int testint(const char* word); -static int testdouble(const char* word); -static int testnull(const char* word); -static int NCJlex(NCJparser* parser); -static int NCJyytext(NCJparser*, char* start, ptrdiff_t pdlen); -static void NCJreclaimArray(NClist*); -static void NCJreclaimDict(NClist*); -static int NCJunparseR(const NCjson* json, NCbytes* buf, int flags); -static int NCJunescape(NCJparser* parser); -static int NCJescape(const char* text, NCbytes* buf); -static int NCJappendquoted(const char* value, NCbytes* buf); -static int NCJcloneArray(NClist* array, NCjson** clonep); -static int NCJcloneDict(NClist* dict, NCjson** clonep); - -#ifdef DEBUG -static char* tokenname(int token); -#endif -/**************************************************/ -int -NCJparse(const char* text, unsigned flags, NCjson** jsonp) -{ - int stat = NC_NOERR; - size_t len; - NCJparser* parser = NULL; - NCjson* json = NULL; - - /* Need at least 1 character of input */ - if(text == NULL || text[0] == '\0') - {stat = NC_EINVAL; goto done;} - if(jsonp == NULL) goto done; - parser = calloc(1,sizeof(NCJparser)); - if(parser == NULL) - {stat = NC_ENOMEM; goto done;} - len = strlen(text); - parser->text = (char*)malloc(len+1+1); - if(parser->text == NULL) - {stat = NC_ENOMEM; goto done;} - parser->text[0] = '\0'; - strlcat(parser->text,text,len+1); - parser->text[len] = '\0'; - parser->text[len+1] = '\0'; - parser->pos = &parser->text[0]; -#ifdef DEBUG -fprintf(stderr,"json: |%s|\n",parser->text); -#endif - if((stat=NCJparseR(parser,&json))) goto done; - *jsonp = json; - json = NULL; - -done: - if(parser != NULL) { - nullfree(parser->text); - nullfree(parser->yytext); - free(parser); - } - (void)NCJreclaim(json); - return THROW(stat); -} - -/* -Simple recursive descent -intertwined with dict and list parsers. - -Invariants: -1. The json argument is provided by caller and filled in by NCJparseR. -2. Each call pushed back last unconsumed token -*/ - -static int -NCJparseR(NCJparser* parser, NCjson** jsonp) -{ - int stat = NC_NOERR; - int token = NCJ_ERR; - NCjson* json = NULL; - - if(jsonp == NULL) - {stat = NC_EINTERNAL; goto done;} - if((token = NCJlex(parser)) == NCJ_ERR) - {stat = NC_EINVAL; goto done;} - switch (token) { - case NCJ_EOF: - break; - case NCJ_NULL: - if((stat = NCJnew(NCJ_NULL,&json))) goto done; - break; - case NCJ_BOOLEAN: - if((stat = NCJnew(NCJ_BOOLEAN,&json))) goto done; - json->value = strdup(parser->yytext); - break; - case NCJ_INT: - if((stat = NCJnew(NCJ_INT,&json))) goto done; - json->value = strdup(parser->yytext); - break; - case NCJ_DOUBLE: - if((stat = NCJnew(NCJ_DOUBLE,&json))) goto done; - json->value = strdup(parser->yytext); - break; - case NCJ_STRING: - if((stat = NCJnew(NCJ_STRING,&json))) goto done; - json->value = strdup(parser->yytext); - break; - case NCJ_LBRACE: - if((stat = NCJnew(NCJ_DICT,&json))) goto done; - if((stat = NCJparseDict(parser, json->contents))) goto done; - break; - case NCJ_LBRACKET: - if((stat = NCJnew(NCJ_ARRAY,&json))) goto done; - if((stat = NCJparseArray(parser, json->contents))) goto done; - break; - case NCJ_RBRACE: /* We hit end of the dict we are parsing */ - parser->pos--; /* pushback so NCJparseArray will catch */ - json = NULL; - break; - case NCJ_RBRACKET: - parser->pos--; /* pushback so NCJparseDict will catch */ - json = NULL; - break; - default: - stat = NC_EINVAL; - break; - } - if(jsonp && json) {*jsonp = json; json = NULL;} - -done: - NCJreclaim(json); - return THROW(stat); -} - -static int -NCJparseArray(NCJparser* parser, NClist* array) -{ - int stat = NC_NOERR; - int token = NCJ_ERR; - NCjson* element = NULL; - int stop = 0; - - /* [ ^e1,e2, ...en] */ - - while(!stop) { - /* Recurse to get the value ei (might be null) */ - if((stat = NCJparseR(parser,&element))) goto done; - token = NCJlex(parser); /* Get next token */ - /* Next token should be comma or rbracket */ - switch(token) { - case NCJ_RBRACKET: - if(element != NULL) nclistpush(array,element); - element = NULL; - stop = 1; - break; - case NCJ_COMMA: - /* Append the ei to the list */ - if(element == NULL) {stat = NC_EINVAL; goto done;} /* error */ - nclistpush(array,element); - element = NULL; - break; - case NCJ_EOF: - case NCJ_ERR: - default: - stat = NC_EINVAL; - goto done; - } - } - -done: - if(element != NULL) - NCJreclaim(element); - return THROW(stat); -} - -static int -NCJparseDict(NCJparser* parser, NClist* dict) -{ - int stat = NC_NOERR; - int token = NCJ_ERR; - NCjson* value = NULL; - NCjson* key = NULL; - int stop = 0; - - /* { ^k1:v1,k2:v2, ...kn:vn] */ - - while(!stop) { - /* Get the key, which must be a word of some sort */ - token = NCJlex(parser); - switch(token) { - case NCJ_STRING: - case NCJ_BOOLEAN: - case NCJ_INT: case NCJ_DOUBLE: { - if((stat=NCJnewstring(token,parser->yytext,&key))) goto done; - } break; - case NCJ_RBRACE: /* End of containing Dict */ - stop = 1; - continue; /* leave loop */ - case NCJ_EOF: case NCJ_ERR: - default: - stat = NC_EINVAL; - goto done; - } - /* Next token must be colon*/ - switch((token = NCJlex(parser))) { - case NCJ_COLON: break; - case NCJ_ERR: case NCJ_EOF: - default: stat = NC_EINVAL; goto done; - } - /* Get the value */ - if((stat = NCJparseR(parser,&value))) goto done; - /* Next token must be comma or RBRACE */ - switch((token = NCJlex(parser))) { - case NCJ_RBRACE: - stop = 1; - /* fall thru */ - case NCJ_COMMA: - /* Insert key value into dict: key first, then value */ - nclistpush(dict,key); - key = NULL; - nclistpush(dict,value); - value = NULL; - break; - case NCJ_EOF: - case NCJ_ERR: - default: - stat = NC_EINVAL; - goto done; - } - } - -done: - if(key != NULL) - NCJreclaim(key); - if(value != NULL) - NCJreclaim(value); - return THROW(stat); -} - -static int -NCJlex(NCJparser* parser) -{ - int c; - int token = 0; - char* start; - ptrdiff_t count; - - while(token == 0) { /* avoid need to goto when retrying */ - c = *parser->pos; - if(c == '\0') { - token = NCJ_EOF; - } else if(c <= ' ' || c == '\177') { - parser->pos++; - continue; /* ignore whitespace */ - } else if(strchr(WORD, c) != NULL) { - start = parser->pos; - for(;;) { - c = *parser->pos++; - if(c == '\0' || strchr(WORD,c) == NULL) break; /* end of word */ - } - /* Pushback c if not whitespace */ - parser->pos--; - count = ((parser->pos) - start); - if(NCJyytext(parser,start,count)) goto done; - /* Discriminate the word string to get the proper sort */ - if(testbool(parser->yytext) == NC_NOERR) - token = NCJ_BOOLEAN; - /* do int test first since double subsumes int */ - else if(testint(parser->yytext) == NC_NOERR) - token = NCJ_INT; - else if(testdouble(parser->yytext) == NC_NOERR) - token = NCJ_DOUBLE; - else if(testnull(parser->yytext) == NC_NOERR) - token = NCJ_NULL; - else - token = NCJ_STRING; - } else if(c == NCJ_QUOTE) { - parser->pos++; - start = parser->pos; - for(;;) { - c = *parser->pos++; - if(c == NCJ_ESCAPE) parser->pos++; - else if(c == NCJ_QUOTE || c == '\0') break; - } - if(c == '\0') { - parser->err = NC_EINVAL; - token = NCJ_ERR; - goto done; - } - count = ((parser->pos) - start) - 1; /* -1 for trailing quote */ - if(NCJyytext(parser,start,count)) goto done; - if(NCJunescape(parser)) goto done; - token = NCJ_STRING; - } else { /* single char token */ - if(NCJyytext(parser,parser->pos,1)) goto done; - token = *parser->pos++; - } -#ifdef DEBUG -fprintf(stderr,"%s(%d): |%s|\n",tokenname(token),token,parser->yytext); -#endif - } /*for(;;)*/ -done: - if(parser->err) token = NCJ_ERR; - return token; -} - -static int -testnull(const char* word) -{ - if(strcasecmp(word,NCJ_TAG_NULL)==0) - return NC_NOERR; - return NC_EINVAL; -} - -static int -testbool(const char* word) -{ - if(strcasecmp(word,NCJ_TAG_TRUE)==0 - || strcasecmp(word,NCJ_TAG_FALSE)==0) - return NC_NOERR; - return NC_EINVAL; -} - -static int -testint(const char* word) -{ - int ncvt; - long long i; - int count = 0; - /* Try to convert to number */ - ncvt = sscanf(word,"%lld%n",&i,&count); - return (ncvt == 1 && strlen(word)==count ? NC_NOERR : NC_EINVAL); -} - -static int -testdouble(const char* word) -{ - int ncvt; - double d; - int count = 0; - /* Check for Nan and Infinity */ - if(strcasecmp("nan",word)==0) return NC_NOERR; - if(strcasecmp("infinity",word)==0) return NC_NOERR; - if(strcasecmp("-infinity",word)==0) return NC_NOERR; - /* Allow the XXXf versions as well */ - if(strcasecmp("nanf",word)==0) return NC_NOERR; - if(strcasecmp("infinityf",word)==0) return NC_NOERR; - if(strcasecmp("-infinityf",word)==0) return NC_NOERR; - /* Try to convert to number */ - ncvt = sscanf(word,"%lg%n",&d,&count); - return (ncvt == 1 && strlen(word)==count ? NC_NOERR : NC_EINVAL); -} - -static int -NCJyytext(NCJparser* parser, char* start, ptrdiff_t pdlen) -{ - size_t len = (size_t)pdlen; - if(parser->yytext == NULL) { - parser->yytext = (char*)malloc(len+1); - parser->yylen = len; - } else if(parser->yylen <= len) { - parser->yytext = (char*) realloc(parser->yytext,len+1); - parser->yylen = len; - } - if(parser->yytext == NULL) return NC_ENOMEM; - memcpy(parser->yytext,start,len); - parser->yytext[len] = '\0'; - return NC_NOERR; -} - -/**************************************************/ - -void -NCJreclaim(NCjson* json) -{ - if(json == NULL) return; - switch(json->sort) { - case NCJ_INT: - case NCJ_DOUBLE: - case NCJ_BOOLEAN: - case NCJ_STRING: - nullfree(json->value); - break; - case NCJ_DICT: - NCJreclaimDict(json->contents); - nclistfree(json->contents); - break; - case NCJ_ARRAY: - NCJreclaimArray(json->contents); - nclistfree(json->contents); - break; - default: break; /* nothing to reclaim */ - } - free(json); -} - -static void -NCJreclaimArray(NClist* array) -{ - int i; - for(i=0;isort) { - case NCJ_INT: - case NCJ_DOUBLE: - case NCJ_BOOLEAN: - case NCJ_STRING: - if((stat=NCJnew(json->sort,&clone))) goto done; - if((clone->value = strdup(json->value)) == NULL) - {stat = NC_ENOMEM; goto done;} - break; - case NCJ_NULL: - if((stat=NCJnew(json->sort,&clone))) goto done; - break; - case NCJ_DICT: - if((stat=NCJcloneDict(json->contents,&clone))) goto done; - break; - case NCJ_ARRAY: - if((stat=NCJcloneArray(json->contents,&clone))) goto done; - break; - default: break; /* nothing to clone */ - } -done: - if(stat == NC_NOERR && clonep) {*clonep = clone; clone = NULL;} - NCJreclaim(clone); - return stat; -} - -static int -NCJcloneArray(NClist* array, NCjson** clonep) -{ - int i, stat=NC_NOERR; - NCjson* clone = NULL; - if((stat=NCJnew(NCJ_ARRAY,&clone))) goto done; - for(i=0;icontents,elemclone); - } -done: - if(stat == NC_NOERR && clonep) {*clonep = clone; clone = NULL;} - NCJreclaim(clone); - return stat; -} - -static int -NCJcloneDict(NClist* dict, NCjson** clonep) -{ - int stat = NC_NOERR; - NCjson* clone = NULL; - if((stat=NCJcloneArray(dict,&clone))) goto done; - /* Convert from array to dict */ - clone->sort = NCJ_DICT; -done: - if(stat == NC_NOERR && clonep) {*clonep = clone; clone = NULL;} - NCJreclaim(clone); - return stat; -} - -/**************************************************/ -/* Build Functions */ - -int -NCJnew(int sort, NCjson** objectp) -{ - int stat = NC_NOERR; - NCjson* object = NULL; - - if((object = calloc(1,sizeof(NCjson))) == NULL) - {stat = NC_ENOMEM; goto done;} - object->sort = sort; - switch (sort) { - case NCJ_INT: - case NCJ_DOUBLE: - case NCJ_BOOLEAN: - case NCJ_STRING: - case NCJ_NULL: - break; - case NCJ_DICT: - object->contents = nclistnew(); - break; - case NCJ_ARRAY: - object->contents = nclistnew(); - break; - default: - stat = NC_EINVAL; - goto done; - } - if(objectp) {*objectp = object; object = NULL;} - -done: - if(stat) NCJreclaim(object); - return THROW(stat); -} - -int -NCJnewstring(int sort, const char* value, NCjson** jsonp) -{ - return NCJnewstringn(sort,strlen(value),value,jsonp); -} - -int -NCJnewstringn(int sort, size_t len, const char* value, NCjson** jsonp) -{ - int stat = NC_NOERR; - NCjson* json = NULL; - - if(jsonp) *jsonp = NULL; - if((stat = NCJnew(sort,&json))) - goto done; - if((json->value = malloc(len+1))==NULL) - {stat = NC_ENOMEM; goto done;} - memcpy(json->value,value,len); - json->value[len] = '\0'; - if(jsonp) *jsonp = json; - json = NULL; /* avoid memory errors */ -done: - if(stat) NCJreclaim(json); - return THROW(stat); -} - -/* Insert key-value pair into a dict object. - key will be strdup'd -*/ -int -NCJinsert(NCjson* object, char* key, NCjson* value) -{ - if(object == NULL || object->sort != NCJ_DICT) - return NC_EINTERNAL; - NCJaddstring(object,NCJ_STRING,key); - nclistpush(object->contents,value); - return NC_NOERR; -} - -/* Remove a key-value pair from a dict object. -*/ -int -NCJremove(NCjson* dict, char* key, NCjson** keyp, NCjson** valuep) -{ - int i; - if(dict == NULL || dict->sort != NCJ_DICT) - return NC_ENCZARR; - for(i=0;icontents);i+=2) { - NCjson* tmp = (NCjson*)nclistget(dict->contents,i); - if(strcmp(tmp->value,key)==0) { - if(keyp) *keyp = tmp; - if(valuep) *valuep = (NCjson*)nclistget(dict->contents,i+1); - nclistremove(dict->contents,i+1); - nclistremove(dict->contents,i); - break; - } - } - return NC_NOERR; -} - -int -NCJaddstring(NCjson* dictarray, int sort, const char* value) -{ - int stat = NC_NOERR; - NCjson* jvalue = NULL; - switch (dictarray->sort) { - case NCJ_DICT: - if((stat = NCJnewstring(sort,value,&jvalue))) goto done; - nclistpush(dictarray->contents,jvalue); - break; - case NCJ_ARRAY: - if((stat = NCJnewstring(sort,value,&jvalue))) goto done; - nclistpush(dictarray->contents,jvalue); - break; - default: stat = NC_EINVAL; goto done; - } - -done: - return THROW(stat); -} - -int -NCJdictith(NCjson* object, size_t i, NCjson** keyp, NCjson** valuep) -{ - if(object == NULL || object->sort != NCJ_DICT) - return NC_EINTERNAL; - if(i >= (nclistlength(object->contents)/2)) - return NC_EINVAL; - if(keyp) *keyp = nclistget(object->contents,2*i); - if(valuep) *valuep = nclistget(object->contents,(2*i)+1); - return NC_NOERR; -} - -int -NCJdictget(NCjson* object, const char* key, NCjson** valuep) -{ - int i; - if(object == NULL || object->sort != NCJ_DICT) - return NC_EINTERNAL; - if(valuep) *valuep = NULL; - for(i=0;icontents);i+=2) { - const NCjson* k = nclistget(object->contents,i); - assert(k != NULL && k->sort == NCJ_STRING); - if(strcmp(k->value,key)==0) { - if(valuep) *valuep = nclistget(object->contents,i+1); - break; - } - } - return NC_NOERR; -} - -/* Append value to an array or dict object. -*/ -int -NCJappend(NCjson* object, NCjson* value) -{ - if(object == NULL) - return NC_EINTERNAL; - switch (object->sort) { - case NCJ_ARRAY: - nclistpush(object->contents,value); - break; - case NCJ_DICT: - nclistpush(object->contents,value); - break; - default: - return NC_EINTERNAL; - } - return NC_NOERR; -} - -int -NCJarrayith(NCjson* object, size_t i, NCjson** valuep) -{ - if(object == NULL || object->sort != NCJ_ARRAY) - return NC_EINTERNAL; - if(valuep) *valuep = nclistget(object->contents,i); - return NC_NOERR; -} - -/* Unescape the text in parser->yytext; can - do in place because unescaped string will - always be shorter */ -static int -NCJunescape(NCJparser* parser) -{ - char* p = parser->yytext; - char* q = p; - int c; - for(;(c=*p++);) { - if(c == NCJ_ESCAPE) { - c = *p++; - switch (c) { - case 'b': c = '\b'; break; - case 'f': c = '\f'; break; - case 'n': c = '\n'; break; - case 'r': c = '\r'; break; - case 't': c = '\t'; break; - case NCJ_QUOTE: c = c; break; - case NCJ_ESCAPE: c = c; break; - default: c = c; break;/* technically not Json conformant */ - } - } - *q++ = c; - } - *q = '\0'; - return NC_NOERR; -} - -/**************************************************/ -/* Unparser to convert NCjson object to text in buffer */ - -int -NCJunparse(const NCjson* json, int flags, char** textp) -{ - int stat = NC_NOERR; - NCbytes* buf = ncbytesnew(); - if((stat = NCJunparseR(json,buf,flags))) - goto done; - if(textp) { - ncbytesnull(buf); - *textp = ncbytesextract(buf); - } -done: - ncbytesfree(buf); - return THROW(stat); -} - -static int -NCJunparseR(const NCjson* json, NCbytes* buf, int flags) -{ - int stat = NC_NOERR; - int i; - switch (json->sort) { - case NCJ_STRING: - NCJappendquoted(json->value,buf); - break; - case NCJ_INT: - case NCJ_DOUBLE: - case NCJ_BOOLEAN: - ncbytescat(buf,json->value); - break; - case NCJ_DICT: - ncbytesappend(buf,NCJ_LBRACE); - for(i=0;icontents);) { - const NCjson* key = NULL; - const NCjson* value = NULL; - if(i > 0) ncbytesappend(buf,NCJ_COMMA); - key = nclistget(json->contents,i); - NCJunparseR(key,buf,flags); - ncbytesappend(buf,NCJ_COLON); - ncbytesappend(buf,' '); - /* Allow for the possibility of a short dict entry */ - i++; - if(i >= nclistlength(json->contents)) { /*short*/ - ncbytescat(buf,"?"); - } else { - value = nclistget(json->contents,i); - NCJunparseR(value,buf,flags); - i++; - } - } - ncbytesappend(buf,NCJ_RBRACE); - break; - case NCJ_ARRAY: - ncbytesappend(buf,NCJ_LBRACKET); - for(i=0;icontents);i++) { - const NCjson* value = nclistget(json->contents,i); - if(i > 0) ncbytesappend(buf,NCJ_COMMA); - NCJunparseR(value,buf,flags); - } - ncbytesappend(buf,NCJ_RBRACKET); - break; - case NCJ_NULL: - ncbytescat(buf,"null"); - break; - default: - stat = NC_EINVAL; goto done; - } -done: - return THROW(stat); -} - -/* Escape a string and append to buf */ -static int -NCJescape(const char* text, NCbytes* buf) -{ - const char* p = text; - int c; - for(;(c=*p++);) { - char replace = 0; - switch (c) { - case '\b': replace = 'b'; break; - case '\f': replace = 'f'; break; - case '\n': replace = 'n'; break; - case '\r': replace = 'r'; break; - case '\t': replace = 't'; break; - case NCJ_QUOTE: replace = '\''; break; - case NCJ_ESCAPE: replace = '\\'; break; - default: break; - } - if(replace) { - ncbytesappend(buf,NCJ_ESCAPE); - ncbytesappend(buf,replace); - } else - ncbytesappend(buf,c); - } - return NC_NOERR; -} - -static int -NCJappendquoted(const char* value, NCbytes* buf) -{ - ncbytesappend(buf,'"'); - NCJescape(value,buf); - ncbytesappend(buf,'"'); - return NC_NOERR; -} - -void -NCJdump(const NCjson* json, int flags) -{ - char* text = NULL; - (void)NCJunparse(json,0,&text); - fprintf(stderr,"%s\n",text); - fflush(stderr); -} - -#ifdef DEBUG -static char* -tokenname(int token) -{ - switch (token) { - case NCJ_STRING: return "NCJ_STRING"; - case NCJ_INT: return "NCJ_INT"; - case NCJ_DOUBLE: return "NCJ_DOUBLE"; - case NCJ_BOOLEAN: return "NCJ_BOOLEAN"; - case NCJ_DICT: return "NCJ_DICT"; - case NCJ_ARRAY: return "NCJ_ARRAY"; - case NCJ_NULL: return "NCJ_NULL"; - default: - if(token > ' ' && token <= 127) { - static char s[4]; - s[0] = '\''; - s[1] = (char)token; - s[2] = '\''; - s[3] = '\0'; - return s; - } else - break; - } - return "NCJ_UNDEF"; -} -#endif diff --git a/libnczarr/obsolete/zjson.h b/libnczarr/obsolete/zjson.h deleted file mode 100644 index 93d8439479..0000000000 --- a/libnczarr/obsolete/zjson.h +++ /dev/null @@ -1,93 +0,0 @@ -/* Copyright 2018, UCAR/Unidata. - See the COPYRIGHT file for more information. -*/ - -#ifndef NCJSON_H -#define NCJSON_H 1 - -#include "ncexternl.h" - -/* Json object sorts */ -#define NCJ_UNDEF 0 -#define NCJ_STRING 1 -#define NCJ_INT 2 -#define NCJ_DOUBLE 3 -#define NCJ_BOOLEAN 4 -#define NCJ_DICT 5 -#define NCJ_ARRAY 6 -#define NCJ_NULL 7 - -/* Don't bother with unions: define - a struct to store primitive values - as unquoted strings. Sort will - provide more info. - - Also, this does not use a true hashmap - but rather an envv style list where name - and value alternate. This works under - the assumption that we are generally - iterating over the Dict rather than - probing it. - -*/ -typedef struct NCjson { - int sort; - char* value; - NClist* contents; /* For array|dict */ -} NCjson; - -#define NCJF_MULTILINE 1 - -/* Parse */ -EXTERNL int NCJparse(const char* text, unsigned flags, NCjson** jsonp); - -/* Build */ -EXTERNL int NCJnew(int sort, NCjson** object); - -/* Convert a nul terminated string value to an NCjson object */ -EXTERNL int NCJnewstring(int sort, const char* value, NCjson** jsonp); - -/* Convert a counted string value to an NCjson object (+ nul term)*/ -EXTERNL int NCJnewstringn(int sort, size_t len, const char* value, NCjson** jsonp); - -/* Insert key-value pair into a dict object. - key will be strdup'd. -*/ -EXTERNL int NCJinsert(NCjson* object, char* key, NCjson* value); - -/* Remove a key-value pair from a dict object. -*/ -EXTERNL int NCJremove(NCjson* object, char* key, NCjson** keyp, NCjson** valuep); - -/* Insert a string value into a json Dict|Array */ -EXTERNL int NCJaddstring(NCjson* dictarray, int sort, const char* value); - -/* Get ith pair from dict */ -EXTERNL int NCJdictith(NCjson* object, size_t i, NCjson** keyp, NCjson** valuep); - -/* Get value for key from dict */ -EXTERNL int NCJdictget(NCjson* object, const char* key, NCjson** valuep); - -/* Append value to an array or dict object. */ -EXTERNL int NCJappend(NCjson* object, NCjson* value); - -/* Get ith element from array */ -EXTERNL int NCJarrayith(NCjson* object, size_t i, NCjson** valuep); - -/* Unparser to convert NCjson object to text in buffer */ -EXTERNL int NCJunparse(const NCjson* json, int flags, char** textp); - -/* Utilities */ -EXTERNL void NCJreclaim(NCjson*); -EXTERNL int NCJclone(NCjson* json, NCjson** clonep); /* deep clone */ - -/* dump NCjson* object */ -EXTERNL void NCJdump(const NCjson* json, int flags); - -/* Macro defined functions */ -#define NCJlength(json) \ -((json)->sort == NCJ_DICT ? (nclistlength((json)->contents)/2) \ - : ((json)->sort == NCJ_ARRAY ? (nclistlength((json)->contents)) \ - : 1)) - -#endif /*NCJSON_H*/ diff --git a/libnczarr/zarr.c b/libnczarr/zarr.c index c874eb709a..afc06ddd16 100644 --- a/libnczarr/zarr.c +++ b/libnczarr/zarr.c @@ -307,8 +307,8 @@ ncz_unload_jatts(NCZ_FILE_INFO_T* zinfo, NC_OBJ* container, NCjson* jattrs, NCjs char* tkey = NULL; NCZMAP* map = zinfo->map; - assert((jattrs->sort = NCJ_DICT)); - assert((jtypes->sort = NCJ_DICT)); + assert((NCJsort(jattrs) == NCJ_DICT)); + assert((NCJsort(jtypes) == NCJ_DICT)); if(container->sort == NCGRP) { NC_GRP_INFO_T* grp = (NC_GRP_INFO_T*)container; @@ -332,7 +332,7 @@ ncz_unload_jatts(NCZ_FILE_INFO_T* zinfo, NC_OBJ* container, NCjson* jattrs, NCjs NCjson* k = NULL; NCjson* v = NULL; /* remove any previous version */ - if(NCJremove(jattrs,NCZ_V2_ATTRS,&k,&v) == NC_NOERR) { + if(!NCJremove(jattrs,NCZ_V2_ATTRS,&k,&v)) { NCJreclaim(k); NCJreclaim(v); } } diff --git a/libnczarr/zarr.h b/libnczarr/zarr.h index 0922e3c012..20c1953b1b 100644 --- a/libnczarr/zarr.h +++ b/libnczarr/zarr.h @@ -31,13 +31,13 @@ extern int ncz_close_file(NC_FILE_INFO_T* file, int abort); /* zcvt.c */ extern int NCZ_convert1(NCjson* jsrc, nc_type, unsigned char* memory0); -extern int NCZ_stringconvert1(nc_type typid, unsigned char* src, char** strp); +extern int NCZ_stringconvert1(nc_type typid, size_t len, char* src, NCjson* jvalue); extern int NCZ_stringconvert(nc_type typid, size_t len, void* data0, NCjson** jdatap); /* zsync.c */ extern int ncz_sync_file(NC_FILE_INFO_T* file, int isclose); -extern int ncz_sync_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp); -extern int ncz_sync_atts(NC_FILE_INFO_T*, NC_OBJ* container, NCindex* attlist); +extern int ncz_sync_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, int isclose); +extern int ncz_sync_atts(NC_FILE_INFO_T*, NC_OBJ* container, NCindex* attlist, int isclose); extern int ncz_read_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp); extern int ncz_read_atts(NC_FILE_INFO_T* file, NC_OBJ* container); extern int ncz_read_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp); @@ -67,10 +67,10 @@ extern int NCZ_comma_parse(const char* s, NClist* list); extern int NCZ_swapatomicdata(size_t datalen, void* data, int typesize); extern char** NCZ_clonestringvec(size_t len, const char** vec); extern void NCZ_freestringvec(size_t len, char** vec); -extern int NCZ_create_fill_chunk(size64_t chunksize, size_t typesize, void* fill, void** fillchunkp); +extern int NCZ_create_fill_chunk(size64_t chunksize, size_t typesize, const void* fill, void** fillchunkp); extern int NCZ_s3clear(ZS3INFO* s3); extern int NCZ_ischunkname(const char* name,char dimsep); -extern char* NCZ_chunkpath(struct ChunkKey key,char dimsep); +extern char* NCZ_chunkpath(struct ChunkKey key); /* Export */ EXTERNL int NCZ_s3urlprocess(NCURI* url, ZS3INFO* s3); diff --git a/libnczarr/zattr.c b/libnczarr/zattr.c index 679a3f8603..38542b9b4b 100644 --- a/libnczarr/zattr.c +++ b/libnczarr/zattr.c @@ -10,10 +10,10 @@ */ #include "zincludes.h" +#include "zfilter.h" #undef ADEBUG - /** * @internal Get the attribute list for either a varid or NC_GLOBAL * @@ -73,6 +73,7 @@ ncz_getattlist(NC_GRP_INFO_T *grp, int varid, NC_VAR_INFO_T **varp, NCindex **at * the file, they are constructed on the fly. * * @param h5 Pointer to ZARR file info struct. + * @param var Pointer to var info struct; NULL signals global. * @param name Name of attribute. * @param filetypep Pointer that gets type of the attribute data in * file. @@ -87,21 +88,40 @@ ncz_getattlist(NC_GRP_INFO_T *grp, int varid, NC_VAR_INFO_T **varp, NCindex **at * @author Dennis Heimbigner */ int -ncz_get_att_special(NC_FILE_INFO_T* h5, const char* name, +ncz_get_att_special(NC_FILE_INFO_T* h5, NC_VAR_INFO_T* var, const char* name, nc_type* filetypep, nc_type mem_type, size_t* lenp, int* attnump, void* data) { + int stat = NC_NOERR; + /* Fail if asking for att id */ if(attnump) - return NC_EATTMETA; + {stat = NC_EATTMETA; goto done;} + + /* Handle the per-var case(s) first */ + if(var != NULL) { + if(strcmp(name,NC_ATT_CODECS)==0) { + NClist* filters = (NClist*)var->filters; + + if(mem_type == NC_NAT) mem_type = NC_CHAR; + if(mem_type != NC_CHAR) + {stat = NC_ECHAR; goto done;} + if(filetypep) *filetypep = NC_CHAR; + if(lenp) *lenp = 0; + if(filters == NULL) goto done; + if((stat = NCZ_codec_attr(var,lenp,data))) goto done; + } + goto done; + } + /* The global reserved attributes */ if(strcmp(name,NCPROPS)==0) { int len; if(h5->provenance.ncproperties == NULL) - return NC_ENOTATT; + {stat = NC_ENOTATT; goto done;} if(mem_type == NC_NAT) mem_type = NC_CHAR; if(mem_type != NC_CHAR) - return NC_ECHAR; + {stat = NC_ECHAR; goto done;} if(filetypep) *filetypep = NC_CHAR; len = strlen(h5->provenance.ncproperties); if(lenp) *lenp = len; @@ -127,10 +147,12 @@ ncz_get_att_special(NC_FILE_INFO_T* h5, const char* name, case NC_INT64: *((long long*)data) = (long long)iv; break; case NC_UINT64: *((unsigned long long*)data) = (unsigned long long)iv; break; default: - return NC_ERANGE; + {stat = NC_ERANGE; goto done;} } } - return NC_NOERR; +done: + return stat; + } /** @@ -571,7 +593,7 @@ ncz_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type, if (*(char **)var->fill_value) free(*(char **)var->fill_value); } - free(var->fill_value); + free(var->fill_value); var->fill_value = NULL; } #ifdef LOOK @@ -805,11 +827,10 @@ NCZ_inq_att(int ncid, int varid, const char *name, nc_type *xtypep, return retval; /* If this is one of the reserved atts, use nc_get_att_special. */ - if (!var) { const NC_reservedatt *ra = NC_findreserved(norm_name); if (ra && ra->flags & NAMEONLYFLAG) - return ncz_get_att_special(h5, norm_name, xtypep, NC_NAT, lenp, NULL, + return ncz_get_att_special(h5, var, norm_name, xtypep, NC_NAT, lenp, NULL, NULL); } @@ -846,11 +867,10 @@ NCZ_inq_attid(int ncid, int varid, const char *name, int *attnump) return retval; /* If this is one of the reserved atts, use nc_get_att_special. */ - if (!var) { const NC_reservedatt *ra = NC_findreserved(norm_name); if (ra && ra->flags & NAMEONLYFLAG) - return ncz_get_att_special(h5, norm_name, NULL, NC_NAT, NULL, attnump, + return ncz_get_att_special(h5, var, norm_name, NULL, NC_NAT, NULL, attnump, NULL); } @@ -924,12 +944,11 @@ NCZ_get_att(int ncid, int varid, const char *name, void *value, &h5, &grp, &var, NULL))) return retval; - /* If this is one of the reserved atts, use nc_get_att_special. */ - if (!var) + /* If this is one of the reserved global atts, use nc_get_att_special. */ { const NC_reservedatt *ra = NC_findreserved(norm_name); if (ra && ra->flags & NAMEONLYFLAG) - return ncz_get_att_special(h5, norm_name, NULL, NC_NAT, NULL, NULL, + return ncz_get_att_special(h5, var, norm_name, NULL, NC_NAT, NULL, NULL, value); } @@ -1014,6 +1033,16 @@ ncz_makeattr(NC_OBJ* container, NCindex* attlist, const char* name, nc_type type int stat = NC_NOERR; NC_ATT_INFO_T* att = NULL; NCZ_ATT_INFO_T* zatt = NULL; + void* clone = NULL; + size_t typesize, clonesize; + NC_GRP_INFO_T* grp = (container->sort == NCGRP ? (NC_GRP_INFO_T*)container + : ((NC_VAR_INFO_T*)container)->container); + + /* Duplicate the values */ + if ((stat = nc4_get_typelen_mem(grp->nc4_info, typeid, &typesize))) goto done; + clonesize = len*typesize; + if((clone = malloc(clonesize))==NULL) {stat = NC_ENOMEM; goto done;} + memcpy(clone,values,clonesize); if((stat=nc4_att_list_add(attlist,name,&att))) goto done; @@ -1030,11 +1059,12 @@ ncz_makeattr(NC_OBJ* container, NCindex* attlist, const char* name, nc_type type /* Fill in the attribute's type and value */ att->nc_typeid = typeid; att->len = len; - att->data = values; + att->data = clone; clone = NULL; att->dirty = NC_TRUE; if(attp) {*attp = att; att = NULL;} done: + nullfree(clone); if(stat) { if(att) nc4_att_list_del(attlist,att); nullfree(zatt); diff --git a/libnczarr/zcache.h b/libnczarr/zcache.h index f54ae04a53..c5065d1667 100644 --- a/libnczarr/zcache.h +++ b/libnczarr/zcache.h @@ -12,6 +12,16 @@ struct NCxcache; +/* Note in the following: the term "real" + refers to the unfiltered/uncompressed data + The term filtered refers to the result of running + the real data through the filter chain. Note that the + sizeof the filtered data might be larger than the size of + the real data. + The term "raw" is used to refer to the data on disk and it may either + be real or filtered. +*/ + typedef struct NCZCacheEntry { struct List {void* next; void* prev; void* unused;} list; int modified; @@ -21,24 +31,30 @@ typedef struct NCZCacheEntry { char* chunkkey; /* name of the chunk */ } key; size64_t hashkey; - void* data; + int isfiltered; /* 1=>data contains filtered data else real data */ + size64_t size; /* |data| */ + void* data; /* contains either filtered or real data */ } NCZCacheEntry; typedef struct NCZChunkCache { - const NC_VAR_INFO_T* var; /* backlink */ + NC_VAR_INFO_T* var; /* backlink */ size64_t ndims; /* true ndims == var->ndims + scalar */ - size64_t chunksize; - void* fillchunk; /* enough fillvalues to fill a chunk */ - size_t maxentries; /* Max number of entries allowed */ - NClist* mru; /* all cache entries in mru order */ + size64_t chunksize; /* for real data */ + void* fillchunk; /* enough fillvalues to fill a real chunk */ + size_t maxentries; /* Max number of entries allowed; maxsize can override */ + size_t maxsize; /* Maximum space used by cache; 0 => nolimit */ + size_t used; /* How much total space is being used */ + NClist* mru; /* NClist all cache entries in mru order */ struct NCxcache* xcache; char dimension_separator; } NCZChunkCache; /**************************************************/ +#define FILTERED(cache) (nclistlength((NClist*)(cache)->var->filters) || (cache)->var->shuffle || (cache)->var->fletcher32); + extern int NCZ_set_var_chunk_cache(int ncid, int varid, size_t size, size_t nelems, float preemption); -extern int NCZ_adjust_var_cache(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var); +extern int NCZ_adjust_var_cache(NC_VAR_INFO_T *var); extern int NCZ_create_chunk_cache(NC_VAR_INFO_T* var, size64_t, char dimsep, NCZChunkCache** cachep); extern void NCZ_free_chunk_cache(NCZChunkCache* cache); extern int NCZ_read_cache_chunk(NCZChunkCache* cache, const size64_t* indices, void** datap); diff --git a/libnczarr/zchunking.h b/libnczarr/zchunking.h index 829497fb40..78afde0654 100644 --- a/libnczarr/zchunking.h +++ b/libnczarr/zchunking.h @@ -67,7 +67,6 @@ struct Common { size64_t* memshape; void* memory; size_t typesize; - void* fillvalue; size64_t chunkcount; /* computed product of chunklens; warning indices, not bytes */ int swap; /* var->format_info_file->native_endianness == var->endianness */ size64_t shape[NC_MAX_VAR_DIMS]; /* shape of the output hyperslab */ diff --git a/libnczarr/zclose.c b/libnczarr/zclose.c index 7291a9bbd5..fd8234fd69 100644 --- a/libnczarr/zclose.c +++ b/libnczarr/zclose.c @@ -4,6 +4,7 @@ *********************************************************************/ #include "zincludes.h" +#include "zfilter.h" /* Forward */ static int zclose_group(NC_GRP_INFO_T*); diff --git a/libnczarr/zcreate.c b/libnczarr/zcreate.c index 4577ff8a11..896a91cb66 100644 --- a/libnczarr/zcreate.c +++ b/libnczarr/zcreate.c @@ -66,10 +66,8 @@ ncz_create_file(const char *path, int cmode, size_t initialsz, const char** cont return NC_NOERR; exit: /*failure exit*/ - if(!h5) return retval; -#ifdef LOOK - ncz_close_ncz_file(h5, 1, NULL); /* treat like abort */ -#endif + if(retval && h5) + ncz_closeorabort(h5, NULL, 1); /* treat like abort */ return retval; } diff --git a/libnczarr/zcvt.c b/libnczarr/zcvt.c index 374eda5b00..8cba760006 100644 --- a/libnczarr/zcvt.c +++ b/libnczarr/zcvt.c @@ -22,6 +22,10 @@ struct ZCVT { double float64v; }; +/* Forward */ +static int typeid2jtype(nc_type typeid); + +/* Convert an NCJ_STRING to a memory equivalent value of specified dsttype */ int NCZ_convert1(NCjson* jsrc, nc_type dsttype, unsigned char* memory) { @@ -31,7 +35,7 @@ NCZ_convert1(NCjson* jsrc, nc_type dsttype, unsigned char* memory) int outofrange = 0; /* Convert the incoming jsrc string to a restricted set of values */ - switch (jsrc->sort) { + switch (NCJsort(jsrc)) { case NCJ_INT: /* convert to (u)int64 */ if(NCJstring(jsrc)[0] == '-') { if(sscanf(NCJstring(jsrc),"%lld",&zcvt.int64v) != 1) @@ -70,10 +74,10 @@ NCZ_convert1(NCjson* jsrc, nc_type dsttype, unsigned char* memory) default: stat = NC_EINTERNAL; goto done; } - /* Now, do the down conversion into memory */ + /* Now, do the down conversion */ switch (dsttype) { case NC_BYTE: { - signed char* p = (signed char*)memory; + signed char* p = (signed char*)memory; switch (srctype) { case NC_DOUBLE: zcvt.int64v = (long long)zcvt.float64v; /* Convert to int64 */ @@ -234,8 +238,9 @@ NCZ_convert1(NCjson* jsrc, nc_type dsttype, unsigned char* memory) return stat; } +/* Convert a memory value to a JSON string value */ int -NCZ_stringconvert1(nc_type srctype, unsigned char* src, char** strp) +NCZ_stringconvert1(nc_type srctype, size_t len, char* src, NCjson* jvalue) { int stat = NC_NOERR; struct ZCVT zcvt; @@ -282,8 +287,8 @@ NCZ_stringconvert1(nc_type srctype, unsigned char* src, char** strp) dsttype = NC_DOUBLE; } break; case NC_DOUBLE: { - zcvt.float64v= (double)(*((double*)src)); dsttype = NC_DOUBLE; + zcvt.float64v= (double)(*((double*)src)); } break; default: stat = NC_EINTERNAL; goto done; } @@ -297,11 +302,33 @@ NCZ_stringconvert1(nc_type srctype, unsigned char* src, char** strp) snprintf(s,sizeof(s),"%llu",zcvt.uint64v); } break; case NC_DOUBLE: { - snprintf(s,sizeof(s),"%lg",zcvt.float64v); /* handles NAN? */ +#ifdef _WIN32 + switch (_fpclass(zcvt.float64v)) { + case _FPCLASS_SNAN: case _FPCLASS_QNAN: + strcpy(s,"Nan"); break; + case _FPCLASS_NINF: + strcpy(s,"-Infinity"); break; + case _FPCLASS_PINF: + strcpy(s,"Infinity"); break; + default: + snprintf(s,sizeof(s),"%lg",zcvt.float64v); /* handles NAN? */ + break; + } +#else + if(isnan(zcvt.float64v)) + strcpy(s,"NaN"); + else if(isinf(zcvt.float64v) && zcvt.float64v < 0) + strcpy(s,"-Infinity"); + else if(isinf(zcvt.float64v) && zcvt.float64v > 0) + strcpy(s,"Infinity"); + else { + snprintf(s,sizeof(s),"%lg",zcvt.float64v); /* handles NAN? */ + } +#endif } break; default: stat = NC_EINTERNAL; goto done; } - if(strp) *strp = strdup(s); + NCJsetstring(jvalue,strdup(s)); done: return stat; } @@ -316,6 +343,9 @@ NCZ_stringconvert(nc_type typeid, size_t len, void* data0, NCjson** jdatap) char* str = NULL; NCjson* jvalue = NULL; NCjson* jdata = NULL; + int jtype = NCJ_UNDEF; + + jtype = typeid2jtype(typeid); if((stat = NC4_inq_atomic_type(typeid, NULL, &typelen))) goto done; @@ -325,61 +355,15 @@ NCZ_stringconvert(nc_type typeid, size_t len, void* data0, NCjson** jdatap) /* Create a string valued json object */ if((stat = NCJnewstringn(NCJ_STRING,len,src,&jdata))) goto done; - } else { /* all other cases */ - if(len == 0) {stat = NC_EINVAL; goto done;} - if(len > 1) { - if((stat = NCJnew(NCJ_ARRAY,&jdata))) goto done; - } else /* return a singletone */ - jdata = NULL; + } else if(len == 1) { /* create singleton */ + if((stat = NCJnew(jtype,&jdata))) goto done; + if((stat = NCZ_stringconvert1(typeid, len, src, jdata))) goto done; + } else { /* len > 1 create array of values */ + if((stat = NCJnew(NCJ_ARRAY,&jdata))) goto done; for(i=0;i 0) - special = "Infinity"; - else {} -#endif - if((stat=NCJnew(NCJ_DOUBLE,&jvalue))) goto done; - } break; - case NC_CHAR: - if((stat=NCJnew(NCJ_STRING,&jvalue))) goto done; - break; - default: stat = NC_EINTERNAL; goto done; - } - if(special) {nullfree(str); str = strdup(special);} - NCJstring(jvalue) = str; - str = NULL; - if(len == 1) - jdata = jvalue; - else - NCJappend(jdata,jvalue); + if((stat = NCJnew(jtype,&jvalue))) goto done; + if((stat = NCZ_stringconvert1(typeid, len, src, jvalue))) goto done; + NCJappend(jdata,jvalue); jvalue = NULL; src += typelen; } @@ -392,3 +376,20 @@ NCZ_stringconvert(nc_type typeid, size_t len, void* data0, NCjson** jdatap) NCJreclaim(jdata); return stat; } + +static int +typeid2jtype(nc_type typeid) +{ + switch (typeid) { + case NC_BYTE: case NC_SHORT: case NC_INT: case NC_INT64: + case NC_UBYTE: case NC_USHORT: case NC_UINT: case NC_UINT64: + return NCJ_INT; + case NC_FLOAT: + case NC_DOUBLE: + return NCJ_DOUBLE; + case NC_CHAR: + return NCJ_STRING; + default: break; + } + return NCJ_UNDEF; +} diff --git a/libnczarr/zdebug.c b/libnczarr/zdebug.c index 2f360d18cb..19fdb1f6a9 100644 --- a/libnczarr/zdebug.c +++ b/libnczarr/zdebug.c @@ -288,6 +288,15 @@ nczprint_idvector(size_t len, const int* ids) return nczprint_vector(len,v); } +char* +nczprint_paramvector(size_t len, const unsigned* params) +{ + size64_t v[4096]; + size_t i; + for(i=0;iallgroups); i++) { NC_GRP_INFO_T* g = nclistget(h5->allgroups,i); for (j = 0; j < ncindexsize(g->vars); j++) { var = (NC_VAR_INFO_T *)ncindexith(g->vars, j); assert(var); - var->written_to = NC_TRUE; + /* set the fill value and _FillValue attribute */ + if((stat = ncz_get_fill_value(h5,var,NULL))) goto done; /* ensure var->fill_value is set */ + assert(var->fill_value != NULL); + var->written_to = NC_TRUE; /* mark it written */ + /* rebuild the fill chunk */ + if((stat = NCZ_adjust_var_cache(var))) goto done; + /* Build the filter working parameters for any filters */ + if((stat = NCZ_filter_setup(var))) goto done; } } stat = ncz_enddef_netcdf4_file(h5); @@ -236,13 +242,13 @@ ncz_closeorabort(NC_FILE_INFO_T* h5, void* params, int abort) NC_UNUSED(params); - ZTRACE(1,"file=%s abort=%d",h5->hdr.name,abort); + ZTRACE(3,"file=%s abort=%d",h5->hdr.name,abort); LOG((2, "%s: file: %p", __func__, h5)); /* If we're in define mode, but not redefing the file, delete it. */ if(!abort) { - /* Invoke enddef if needed, which mean sync */ + /* Invoke enddef if needed, which mean sync first */ if(h5->flags & NC_INDEF) h5->flags ^= NC_INDEF; /* Sync the file unless this is a read-only file. */ if(!h5->no_write) { diff --git a/libnczarr/zfilter.c b/libnczarr/zfilter.c index 3134951718..7319daf3f8 100644 --- a/libnczarr/zfilter.c +++ b/libnczarr/zfilter.c @@ -2,6 +2,20 @@ * Research. See the COPYRIGHT file for copying and redistribution * conditions. */ + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/hdf5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + /** * @file @internal Internal functions for filters * @@ -10,33 +24,244 @@ * functions all relate to the manipulation of netcdf-4 filters * * @author Dennis Heimbigner + * + * This file is very similar to libhdf5/hdf5filters.c, so changes + * should be propagated if needed. + * */ #include "config.h" #include + +#ifdef HAVE_SYS_STAT_H +#include +#endif +#ifdef HAVE_DIRENT_H +#include +#endif + +#ifdef _WIN32 +#include +#endif + #include "zincludes.h" +#include "zfilter.h" +#include "ncpathmgr.h" +#include "ncpoco.h" +#include "netcdf_filter.h" +#include "netcdf_filter_build.h" +#include "netcdf_aux.h" + +#undef DEBUG +#define DEBUGF +#undef DEBUGL + +#define NULLIFY(x) ((x)?(x):"NULL") + +/* Hold the loaded filter plugin information */ +typedef struct NCZ_Plugin { + struct HDF5API { + const H5Z_class2_t* filter; + NCPSharedLib* hdf5lib; /* source of the filter */ + } hdf5; + struct CodecAPI { + const NCZ_codec_t* codec; + NCPSharedLib* codeclib; /* of the source codec; null if same as hdf5 */ + } codec; +} NCZ_Plugin; + +/* The NC_VAR_INFO_T->filters field is an NClist of this struct */ +/* +Each filter can have two parts: HDF5 and Codec. +The NC_VAR_INFO_T.filters list only holds entries where both the HDF5 info +and the codec info are defined. +The NCZ_VAR_INFO_T.codecs list holds the codec info when reading a Zarr file. +Note that it is not possible to have an entry on the filters list that does not +have both HDF5 and codec. This is because nc_def_var_filter will fail if the codec +part is not available. If a codec is read from a file and there is no available +corresponding HDF5 implementation, then that codec will not appear in the filters list. +It is possible that some subset of the codecs do have a corresponding HDF5, but we +enforce the rule that no entries go into the filters list unless all are defined. +It is still desirable for a user to be able to see what filters and codecs are defined +for a variable. This is accommodated by providing two special attributes: +1, "_Filters" attribute shows the HDF5 filters defined on the variable, if any. +2, "_Codecs" attribute shows the codecs defined on the variable; for zarr, this list + should always be defined. +*/ + +/* Codec Info */ +typedef struct NCZ_Codec { + char* id; /**< The NumCodecs ID */ + char* codec; /**< The Codec from the file; NULL if creating */ +} NCZ_Codec; + +static NCZ_Codec codec_empty = {NULL, NULL}; + +static void +ncz_codec_clear(NCZ_Codec* codec) { + nullfree(codec->id); nullfree(codec->codec); + *codec = codec_empty; +} + +typedef struct NCZ_Params {size_t nparams; unsigned* params;} NCZ_Params; + +/* HDF5 Info */ +typedef struct NCZ_HDF5 { + unsigned id; /**< HDF5 id corresponding to filterid. */ + NCZ_Params visible; + NCZ_Params working; +} NCZ_HDF5; + +static NCZ_HDF5 hdf5_empty = {0, {0,NULL}, {0,NULL}}; + +static void +ncz_hdf5_clear(NCZ_HDF5* h) { + nullfree(h->visible.params); + nullfree(h->working.params); + *h = hdf5_empty; +} + +typedef struct NCZ_Filter { + int flags; /**< Flags describing state of this filter. */ +# define FLAG_VISIBLE 1 /* If set, then visible parameters are defined */ +# define FLAG_WORKING 2 /* If set, then WORKING parameters are defined */ +# define FLAG_CODEC 4 /* If set, then visbile parameters come from an existing codec string */ +# define FLAG_HDF5 8 /* If set, => visible parameters came from nc_def_var_filter */ +# define FLAG_NEWVISIBLE 16 /* If set, => visible parameters were modified */ + NCZ_HDF5 hdf5; + NCZ_Codec codec; + struct NCZ_Plugin* plugin; /**< Implementation of this filter. */ +} NCZ_Filter; -#undef TFILTERS +/* WARNING: GLOBAL DATA */ +/* All possible HDF5 filter plugins */ +/* Convert to linked list or hash table or equivalent since very sparse */ +NCZ_Plugin* loaded_plugins[H5Z_FILTER_MAX]; +int loaded_plugins_max = -1; + +static NCZ_codec_t** codec_defaults; /* NULL terminated */ +static NCPSharedLib* default_lib; /* source of the defaults */ + +static int NCZ_filter_initialized = 0; /**************************************************/ -#ifdef TFILTERS -static void printfilter1(struct NCZ_Filter* nfs); -static void printfilter(struct NCZ_Filter* nfs, const char* tag, int line); -static void printfilterlist(NC_VAR_INFO_T* var, const char* tag, int line); -#define PRINTFILTER(nfs, tag) printfilter(nfs,tag,__LINE__) -#define PRINTFILTERLIST(var,tag) printfilterlist(var,tag,__LINE__) -#else -#define PRINTFILTER(nfs, tag) -#define PRINTFILTERLIST(var,tag) -#endif /*TFILTERS*/ + +#ifdef ZTRACING +static const char* +NCJtrace(const NCjson* j) +{ + static char jstat[4096]; + char* js = NULL; + jstat[0] = '\0'; + if(j) { + (void)NCJunparse(j,0,&js); + if(js) strlcat(jstat,js,sizeof(jstat)); + nullfree(js); + } + return jstat; +} + +#define IEXISTS(x,p) (((x) && *(x)? (*(x))-> p : 0xffffffff)) +#define SEXISTS(x,p) (((x) && *(x)? (*(x))-> p : "null")) +#endif + + +#ifdef DEBUGF + +const char* +printplugin(const NCZ_Plugin* plugin) +{ + static char plbuf[4096]; + char plbuf2[4096]; + char plbuf1[4096]; + + if(plugin == NULL) return "plugin=NULL"; + plbuf2[0] = '\0'; plbuf1[0] = '\0'; + if(plugin->hdf5.filter) + snprintf(plbuf1,sizeof(plbuf1),"hdf5={id=%u name=%s}",plugin->hdf5.filter->id,plugin->hdf5.filter->name); + if(plugin->codec.codec) + snprintf(plbuf2,sizeof(plbuf2),"codec={codecid=%s hdf5id=%u}",plugin->codec.codec->codecid,plugin->codec.codec->hdf5id); + snprintf(plbuf,sizeof(plbuf),"plugin={%s %s}",plbuf1,plbuf2); + return plbuf; +} + +static const char* +printparams(size_t nparams, const unsigned* params) +{ + static char ppbuf[4096]; + + if(nparams == 0) + snprintf(ppbuf,sizeof(ppbuf),"{0,%p}",params); + else + snprintf(ppbuf,sizeof(ppbuf),"{%u %s}",(unsigned)nparams,nczprint_paramvector(nparams,params)); + return ppbuf; +} + +static const char* +printnczparams(const NCZ_Params p) +{ + return printparams(p.nparams,p.params); +} + +static const char* +printcodec(const NCZ_Codec c) +{ + static char pcbuf[4096]; + snprintf(pcbuf,sizeof(pcbuf),"{id=%s codec=%s}", + c.id,NULLIFY(c.codec)); + return pcbuf; +} + +static const char* +printhdf5(const NCZ_HDF5 h) +{ + static char phbuf[4096]; + snprintf(phbuf,sizeof(phbuf),"{id=%u visible=%s working=%s}", + h.id, printnczparams(h.visible), printnczparams(h.working)); + return phbuf; +} + +static const char* +printfilter(const NCZ_Filter* f) +{ + static char pfbuf[4096]; + + if(f == NULL) return "NULL"; + snprintf(pfbuf,sizeof(pfbuf),"{flags=%d hdf5=%s codec=%s plugin=%p}", + f->flags, printhdf5(f->hdf5),printcodec(f->codec),f->plugin); + return pfbuf; +} + +#endif + + +/* Forward */ +static int NCZ_load_all_plugins(void); +static int NCZ_load_plugin_dir(const char* path); +static int NCZ_load_plugin(const char* path, NCZ_Plugin** plugp); +static int NCZ_unload_plugin(NCZ_Plugin* plugin); +static int NCZ_plugin_loaded(int filterid, NCZ_Plugin** pp); +static int NCZ_plugin_save(int filterid, NCZ_Plugin* p); +static int NCZ_filter_free(NCZ_Filter* spec); +static int NCZ_filter_hdf5_clear(NCZ_HDF5* spec); +static int NCZ_filter_codec_clear(NCZ_Codec* spec); +static int NCZ_filter_lookup(NC_VAR_INFO_T* var, unsigned int id, struct NCZ_Filter** specp); + +static int getentries(const char* path, NClist* contents); +static int NCZ_split_plugin_path(const char* path0, NClist* list); + +static int ensure_working(const NC_VAR_INFO_T* var, NCZ_Filter* filter); + +static int paramnczclone(NCZ_Params* dst, const NCZ_Params* src); +static int paramclone(size_t nparams, unsigned** dstp, const unsigned* src); /**************************************************/ /** * @file * @internal - * Internal netcdf hdf5 filter functions. + * Internal netcdf nczarr filter functions. * - * This file contains functions internal to the libhdf5 library. + * This file contains functions internal to the libnczarr library. * None of the functions in this file are exposed in the exernal API. These * functions all relate to the manipulation of netcdf-4's var->filters list. * @@ -49,122 +274,184 @@ NCZ_filter_freelist(NC_VAR_INFO_T* var) int i, stat=NC_NOERR; NClist* filters = (NClist*)var->filters; + ZTRACE(6,"var=%s",var->hdr.name); if(filters == NULL) goto done; -PRINTFILTERLIST(var,"free: before"); - /* Free the filter list backward */ - for(i=nclistlength(filters)-1;i>=0;i--) { - struct NCZ_Filter* spec = nclistremove(filters,i); - if(spec->nparams > 0) nullfree(spec->params); - nullfree(spec); - } -PRINTFILTERLIST(var,"free: after"); + /* Free the filter list elements */ + for(i=0;ifilters = NULL; done: - return stat; + return ZUNTRACE(stat); } static int -NCZ_filter_free(struct NCZ_Filter* spec) +NCZ_filter_free(NCZ_Filter* spec) { - if(spec == NULL) goto done; -PRINTFILTER(spec,"free"); - if(spec->nparams > 0) nullfree(spec->params) + if(spec == NULL) return NC_NOERR; + NCZ_filter_hdf5_clear(&spec->hdf5); + NCZ_filter_codec_clear(&spec->codec); free(spec); -done: return NC_NOERR; } +static int +NCZ_filter_hdf5_clear(NCZ_HDF5* spec) +{ + ZTRACE(6,"spec=%d",spec->id); + if(spec == NULL) goto done; + nullfree(spec->visible.params); + nullfree(spec->working.params); +done: + return ZUNTRACE(NC_NOERR); +} + +static int +NCZ_filter_codec_clear(NCZ_Codec* spec) +{ + ZTRACE(6,"spec=%d",(spec?spec->id:"null")); + if(spec == NULL) goto done; + nullfree(spec->id); + nullfree(spec->codec); +done: + return ZUNTRACE(NC_NOERR); +} + +/* From NCZ_def_var_filter */ int -NCZ_addfilter(NC_VAR_INFO_T* var, unsigned int id, size_t nparams, const unsigned int* params) +NCZ_addfilter(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, unsigned int id, size_t nparams, const unsigned int* params) { int stat = NC_NOERR; struct NCZ_Filter* fi = NULL; - int olddef = 0; /* 1=>already defined */ - NClist* flist = (NClist*)var->filters; + NCZ_Plugin* plugin = NULL; + NCZ_HDF5 hdf5 = hdf5_empty; + + ZTRACE(6,"file=%s var=%s id=%u nparams=%u params=%p",file->hdr.name,var->hdr.name,id,nparams,params); if(nparams > 0 && params == NULL) {stat = NC_EINVAL; goto done;} - if((stat=NCZ_filter_lookup(var,id,&fi))==NC_NOERR) { - assert(fi != NULL); - /* already exists */ - olddef = 1; + if(var->filters == NULL) var->filters = (void*)nclistnew(); + + /* Before anything else, find the matching plugin */ + if((stat = NCZ_plugin_loaded(id,&plugin))) goto done; + if(plugin == NULL || plugin->codec.codec == NULL) { /* fail */ + ZLOG(NCLOGERR,"no such plugin: %u",(unsigned)id); + stat = NC_ENOFILTER; + goto done; + } + + /* Fill in the hdf5 */ + hdf5 = hdf5_empty; + hdf5.id = id; + /* Capture the visible parameters */ + hdf5.visible.nparams = nparams; + if(nparams > 0) { + if((stat = paramclone(nparams,&hdf5.visible.params,params))) goto done; + } + + /* Find the NCZ_Filter */ + if((stat=NCZ_filter_lookup(var,id,&fi))) goto done; + if(fi != NULL) { + if(fi->plugin != plugin) + {stat = NC_EINTERNAL; goto done;} } else { stat = NC_NOERR; if((fi = calloc(1,sizeof(struct NCZ_Filter))) == NULL) {stat = NC_ENOMEM; goto done;} - fi->filterid = id; - olddef = 0; + fi->plugin = plugin; + nclistpush((NClist*)var->filters, fi); } - fi->nparams = nparams; - if(fi->params != NULL) { - nullfree(fi->params); - fi->params = NULL; - } - assert(fi->params == NULL); - if(fi->nparams > 0) { - if((fi->params = (unsigned int*)malloc(sizeof(unsigned int)*fi->nparams)) == NULL) - {stat = NC_ENOMEM; goto done;} - memcpy(fi->params,params,sizeof(unsigned int)*fi->nparams); - } - if(!olddef) { - nclistpush(flist,fi); -PRINTFILTERLIST(var,"add"); - } + + /* (over)write the HDF5 parameters */ + nullfree(fi->hdf5.visible.params); + nullfree(fi->hdf5.working.params); + fi->hdf5.working.nparams = 0; + fi->hdf5.working.params = NULL; + fi->hdf5 = hdf5; + hdf5 = hdf5_empty; + fi->flags |= FLAG_VISIBLE; + fi = NULL; /* either way,its in the var->filters list */ done: if(fi) NCZ_filter_free(fi); - return THROW(stat); + return ZUNTRACE(stat); } int NCZ_filter_remove(NC_VAR_INFO_T* var, unsigned int id) { - int k; + int k, stat = NC_NOERR; NClist* flist = (NClist*)var->filters; + ZTRACE(6,"var=%s id=%u",var->hdr.name,id); /* Walk backwards */ for(k=nclistlength(flist)-1;k>=0;k--) { struct NCZ_Filter* f = (struct NCZ_Filter*)nclistget(flist,k); - if(f->filterid == id) { + if(f->hdf5.id == id) { /* Remove from variable */ nclistremove(flist,k); -#ifdef TFILTERS -PRINTFILTERLIST(var,"remove"); -fprintf(stderr,"\tid=%s\n",id); -#endif /* Reclaim */ NCZ_filter_free(f); - return NC_NOERR; + goto done; } } - return NC_ENOFILTER; + ZLOG(NCLOGERR,"no such filter: %u",(unsigned)id); + stat = NC_ENOFILTER; +done: + return ZUNTRACE(stat); } -int +static int NCZ_filter_lookup(NC_VAR_INFO_T* var, unsigned int id, struct NCZ_Filter** specp) { int i; NClist* flist = (NClist*)var->filters; + + ZTRACE(6,"var=%s id=%u",var->hdr.name,id); + + if(specp) *specp = NULL; if(flist == NULL) { if((flist = nclistnew())==NULL) return NC_ENOMEM; var->filters = (void*)flist; } for(i=0;ifilterid) { + NCZ_Filter* spec = nclistget(flist,i); + assert(spec != NULL); + if(spec->hdf5.id == id) { if(specp) *specp = spec; - return NC_NOERR; + break; } } - return NC_ENOFILTER; + return ZUNTRACEX(NC_NOERR,"spec=%d",IEXISTS(specp,hdf5.id)); } #if 0 +static int +NCZ_codec_lookup(NClist* codecs, const char* id, NCZ_Codec** codecp) +{ + int i; + + ZTRACE(6,"|codecs|=%u id=%u", (unsigned)nclistlength(codecs), id); + if(codecp) *codecp = NULL; + + if(codecs == NULL) return NC_NOERR; + for(i=0;iid,id)==0) { + if(codecp) *codecp = spec; + break; + } + } + return ZUNTRACEX(NC_NOERR,"codec=%s",SEXISTS(codecp,id)); +} + /** * @internal Remove a filter from filter list for a variable * @@ -176,7 +463,7 @@ NCZ_filter_lookup(NC_VAR_INFO_T* var, unsigned int id, struct NCZ_Filter** specp * @returns ::NC_EBADID Bad ncid. * @returns ::NC_ENOTVAR Invalid variable ID. * @returns ::NC_ENOTNC4 Attempting netcdf-4 operation on file that is - * not netCDF-4/HDF5. + * not netCDF-4/NCZARR. * @returns ::NC_ELATEDEF Too late to change settings for this variable. * @returns ::NC_ENOTINDEFINE Not in define mode. * @returns ::NC_EINVAL Invalid input @@ -193,7 +480,7 @@ nc_var_filter_remove(int ncid, int varid, unsigned int filterid) return stat; assert(var); - stat = NC4_hdf5_filter_remove(var,filterid); + stat = NC4_nczarr_filter_remove(var,filterid); return stat; } @@ -208,14 +495,15 @@ NCZ_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, NC_FILE_INFO_T* h5 = NULL; NC_GRP_INFO_T* grp = NULL; NC_VAR_INFO_T* var = NULL; - struct NCZ_Filter* oldspec = NULL; -#ifdef HAVE_H5Z_SZIP + NCZ_Filter* oldspec = NULL; + NCZ_Filter* tmp = NULL; int havedeflate = 0; int haveszip = 0; -#endif - LOG((2, "%s: ncid 0x%x varid %d", __func__, ncid, varid)); + ZTRACE(1,"ncid=%d varid=%d id=%u nparams=%u params=%s",ncid,varid,id,(unsigned)nparams,nczprint_paramvector(nparams,params)); + if((stat = NCZ_filter_initialize())) goto done; + if((stat = NC_check_id(ncid,&nc))) return stat; assert(nc); @@ -225,7 +513,7 @@ NCZ_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, assert(h5 && var && var->hdr.id == varid); - /* If the HDF5 dataset has already been created, then it is too + /* If the NCZARR dataset has already been created, then it is too * late to set all the extra stuff. */ if (!(h5->flags & NC_INDEF)) {stat = THROW(NC_EINDEFINE); goto done;} @@ -233,50 +521,30 @@ NCZ_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, {stat = NC_EINVAL; goto done;} /* For scalars, complain */ if (var->created) {stat = THROW(NC_ELATEDEF); goto done;} - /* Can't turn on parallel and szip before HDF5 1.10.2. */ -#ifdef USE_PARALLEL -#ifndef HDF5_SUPPORTS_PAR_FILTERS - if (h5->parallel == NC_TRUE) - {stat = THROW(NC_EINVAL); goto done;} -#endif /* HDF5_SUPPORTS_PAR_FILTERS */ -#endif /* USE_PARALLEL */ - - /* Lookup incoming id to see if already defined */ - switch((stat=NCZ_filter_lookup(var,id,&oldspec))) { - case NC_NOERR: break; /* already defined */ - case NC_ENOFILTER: break; /*not defined*/ - default: goto done; - } -#ifdef HAVE_H5Z_SZIP - /* See if deflate &/or szip is defined */ - switch ((stat = NCZ_filter_lookup(var,H5Z_FILTER_DEFLATE,NULL))) { - case NC_NOERR: havedeflate = 1; break; - case NC_ENOFILTER: havedeflate = 0; break; - default: goto done; - } - switch ((stat = NCZ_filter_lookup(var,H5Z_FILTER_SZIP,NULL))) { - case NC_NOERR: haveszip = 1; break; - case NC_ENOFILTER: haveszip = 0; break; - default: goto done; - } -#endif /* HAVE_H5Z_SZIP */ - /* If incoming filter not already defined, then check for conflicts */ - if(oldspec == NULL) { - if(id == H5Z_FILTER_DEFLATE) { + /* Lookup incoming id to see if already defined */ + if((stat=NCZ_filter_lookup(var,id,&oldspec))) goto done; + + /* See if deflate &/or szip is defined */ + if((stat = NCZ_filter_lookup(var,H5Z_FILTER_DEFLATE,&tmp))) goto done; + havedeflate = (tmp == NULL ? 0 : 1); + + if((stat = NCZ_filter_lookup(var,H5Z_FILTER_SZIP,&tmp))) goto done; + haveszip = (tmp == NULL ? 0 : 1); + + /* If incoming filter not already defined, then check for conflicts */ + if(oldspec == NULL) { + if(id == H5Z_FILTER_DEFLATE) { int level; if(nparams != 1) {stat = THROW(NC_EFILTER); goto done;}/* incorrect no. of parameters */ level = (int)params[0]; if (level < NC_MIN_DEFLATE_LEVEL || level > NC_MAX_DEFLATE_LEVEL) {stat = THROW(NC_EINVAL); goto done;} -#ifdef HAVE_H5Z_SZIP /* If szip compression is already applied, return error. */ if(haveszip) {stat = THROW(NC_EINVAL); goto done;} -#endif - } -#ifdef HAVE_H5Z_SZIP - if(id == H5Z_FILTER_SZIP) { /* Do error checking */ + } + if(id == H5Z_FILTER_SZIP) { /* Do error checking */ if(nparams != 2) {stat = THROW(NC_EFILTER); goto done;}/* incorrect no. of parameters */ /* Pixels per block must be an even number, < 32. */ @@ -284,26 +552,18 @@ NCZ_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, {stat = THROW(NC_EINVAL); goto done;} /* If zlib compression is already applied, return error. */ if(havedeflate) {stat = THROW(NC_EINVAL); goto done;} - } -#else /*!HAVE_H5Z_SZIP*/ - if(id == H5Z_FILTER_SZIP) - {stat = THROW(NC_EFILTER); goto done;} /* Not allowed */ -#endif - /* Filter => chunking */ - var->storage = NC_CHUNKED; - /* Determine default chunksizes for this variable unless already specified */ - if(var->chunksizes && !var->chunksizes[0]) { + } + /* Filter => chunking */ + var->storage = NC_CHUNKED; + /* Determine default chunksizes for this variable unless already specified */ + if(var->chunksizes && !var->chunksizes[0]) { /* Should this throw error? */ if((stat = nc4_find_default_chunksizes2(grp, var))) goto done; - /* Adjust the cache. */ - if ((stat = NCZ_adjust_var_cache(grp, var))) - goto done; - } - } -#ifdef HAVE_H5Z_SZIP - /* More error checking */ - if(id == H5Z_FILTER_SZIP) { /* szip X chunking error checking */ + } + } + /* More error checking */ + if(id == H5Z_FILTER_SZIP) { /* szip X chunking error checking */ /* For szip, the pixels_per_block parameter must not be greater * than the number of elements in a chunk of data. */ size_t num_elem = 1; @@ -314,25 +574,15 @@ NCZ_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, /* Pixels per block must be <= number of elements. */ if (params[1] > num_elem) {stat = THROW(NC_EINVAL); goto done;} - } -#endif - /* addfilter can handle case where filter is already defined, and will just replace parameters */ - if((stat = NCZ_addfilter(var,id,nparams,params))) - goto done; -#ifdef USE_PARALLEL -#ifdef HDF5_SUPPORTS_PAR_FILTERS - /* Switch to collective access. HDF5 requires collevtive access - * for filter use with parallel I/O. */ - if (h5->parallel) - var->parallel_access = NC_COLLECTIVE; -#else - if (h5->parallel) - {stat = THROW(NC_EINVAL); goto done;} -#endif /* HDF5_SUPPORTS_PAR_FILTERS */ -#endif /* USE_PARALLEL */ + } + /* addfilter can handle case where filter is already defined, and will just replace parameters */ + if((stat = NCZ_addfilter(h5,var,id,nparams,params))) + goto done; + if (h5->parallel) + {stat = THROW(NC_EINVAL); goto done;} done: - return stat; + return ZUNTRACE(stat); } int @@ -344,11 +594,11 @@ NCZ_inq_var_filter_ids(int ncid, int varid, size_t* nfiltersp, unsigned int* ids NC_GRP_INFO_T* grp = NULL; NC_VAR_INFO_T* var = NULL; NClist* flist = NULL; - size_t nfilters; + size_t nfilters = 0; - LOG((2, "%s: ncid 0x%x varid %d", __func__, ncid, varid)); + ZTRACE(1,"ncid=%d varid=%d",ncid,varid); - if((stat = NC_check_id(ncid,&nc))) return stat; + if((stat = NC_check_id(ncid,&nc))) goto done; assert(nc); /* Find info for this file and group and var, and set pointer to each. */ @@ -357,6 +607,9 @@ NCZ_inq_var_filter_ids(int ncid, int varid, size_t* nfiltersp, unsigned int* ids assert(h5 && var && var->hdr.id == varid); + /* Make sure all the filters are defined */ + if((stat = NCZ_filter_initialize())) goto done; + flist = var->filters; nfilters = nclistlength(flist); @@ -364,14 +617,13 @@ NCZ_inq_var_filter_ids(int ncid, int varid, size_t* nfiltersp, unsigned int* ids int k; for(k=0;kfilterid; + ids[k] = f->hdf5.id; } } if(nfiltersp) *nfiltersp = nfilters; done: - return stat; - + return ZUNTRACEX(stat, "nfilters=%u", nfilters); } int @@ -384,9 +636,9 @@ NCZ_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparamsp, NC_VAR_INFO_T* var = NULL; struct NCZ_Filter* spec = NULL; - LOG((2, "%s: ncid 0x%x varid %d", __func__, ncid, varid)); - - if((stat = NC_check_id(ncid,&nc))) return stat; + ZTRACE(1,"ncid=%d varid=%d id=%u",ncid,varid,id); + + if((stat = NC_check_id(ncid,&nc))) goto done; assert(nc); /* Find info for this file and group and var, and set pointer to each. */ @@ -395,61 +647,951 @@ NCZ_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparamsp, assert(h5 && var && var->hdr.id == varid); + /* Make sure all the plugins are defined */ + if((stat = NCZ_filter_initialize())) goto done; + if((stat = NCZ_filter_lookup(var,id,&spec))) goto done; - if(nparamsp) *nparamsp = spec->nparams; - if(params && spec->nparams > 0) { - memcpy(params,spec->params,sizeof(unsigned int)*spec->nparams); + if(spec != NULL) { +#if 0 + if(spec->flags & FLAG_WORKING) {/* working params are available */ + if(spec->plugin->codec.codec->NCZ_visible_parameters) { + stat = spec->plugin->codec.codec->NCZ_visible_parameters(ncid,varid, + spec->hdf5.working.nparams,spec->hdf5.working.params, + &spec->hdf5.visible.nparams,&spec->hdf5.visible.params); +#ifdef DEBUGF + fprintf(stderr,"DEBUGF: NCZ_visible_parameters: ncid=%d varid=%d working=%s visible=%s\n",ncid,varid, + printnczparams(spec->hdf5.visible),printnczparams(spec->hdf5.working)); +#endif + if(stat) goto done; + } + spec->flags |= FLAG_VISIBLE; + } +#endif + /* return the current visible parameters */ + if(nparamsp) *nparamsp = spec->hdf5.visible.nparams; + if(params && spec->hdf5.visible.nparams > 0) + memcpy(params,spec->hdf5.visible.params,sizeof(unsigned int)*spec->hdf5.visible.nparams); + } else { + ZLOG(NCLOGERR,"no such filter: %u",(unsigned)id); + stat = NC_ENOFILTER; + } +done: + return ZUNTRACEX(stat,"nparams=%u",(unsigned)(nparamsp?*nparamsp:0)); +} + + +/**************************************************/ +/* Filter application functions */ + +int +NCZ_filter_initialize(void) +{ + int stat = NC_NOERR; + ZTRACE(6,""); + if(NCZ_filter_initialized) goto done; + { + NCZ_filter_initialized = 1; + memset(loaded_plugins,0,sizeof(loaded_plugins)); + if((stat = NCZ_load_all_plugins())) goto done; + } +done: + return ZUNTRACE(stat); +} + +int +NCZ_filter_finalize(void) +{ + int stat = NC_NOERR; + int i; + ZTRACE(6,""); + /* Reclaim all loaded filters */ + for(i=0;i<=loaded_plugins_max;i++) { + NCZ_unload_plugin(loaded_plugins[i]); + loaded_plugins[i] = NULL; + } + /* Reclaim the defaults library; Must occur as last act */ + if(default_lib != NULL) {(void)ncpsharedlibfree(default_lib); default_lib = NULL; codec_defaults = NULL;} + return ZUNTRACE(stat); +} + +static int +NCZ_plugin_save(int filterid, NCZ_Plugin* p) +{ + int stat = NC_NOERR; + ZTRACE(6,"filterid=%d p=%p",filterid,p); + if(filterid <= 0 || filterid >= H5Z_FILTER_MAX) + {stat = NC_EINVAL; goto done;} + if(filterid > loaded_plugins_max) loaded_plugins_max = filterid; + loaded_plugins[filterid] = p; +done: + return ZUNTRACE(stat); +} + +static int +NCZ_plugin_loaded(int filterid, NCZ_Plugin** pp) +{ + int stat = NC_NOERR; + struct NCZ_Plugin* plug = NULL; + ZTRACE(6,"filterid=%d",filterid); + if(filterid <= 0 || filterid >= H5Z_FILTER_MAX) + {stat = NC_EINVAL; goto done;} + if(filterid <= loaded_plugins_max) + plug = loaded_plugins[filterid]; + if(pp) *pp = plug; +done: + return ZUNTRACEX(stat,"plugin=%p",*pp); +} + +int +NCZ_applyfilterchain(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, NClist* chain, size_t inlen, void* indata, size_t* outlenp, void** outdatap, int encode) +{ + int i, stat = NC_NOERR; + void* lastbuffer = NULL; /* if not null, then last allocated buffer */ + + ZTRACE(6,"|chain|=%u inlen=%u indata=%p encode=%d", (unsigned)nclistlength(chain), (unsigned)inlen, indata, encode); + + /* Make sure all the filters are loaded && setup */ + for(i=0;ihdf5.id > 0 && f->plugin != NULL); + if(!(f->flags & FLAG_WORKING)) {/* working not yet available */ + if((stat = ensure_working(var,f))) goto done; + } + } + + { + struct NCZ_Filter* f = NULL; + const H5Z_class2_t* ff = NULL; + size_t current_alloc = inlen; + void* current_buf = indata; + size_t current_used = inlen; + size_t next_alloc = 0; + void* next_buf = NULL; + size_t next_used = 0; + + +#ifdef DEBUG +fprintf(stderr,"current: alloc=%u used=%u buf=%p\n",(unsigned)current_alloc,(unsigned)current_used,current_buf); +#endif + /* Apply in proper order */ + if(encode) { + for(i=0;iplugin->hdf5.filter; + /* code can be simplified */ + next_alloc = current_alloc; + next_buf = current_buf; + next_used = 0; + next_used = ff->filter(0,f->hdf5.working.nparams,f->hdf5.working.params,current_used,&next_alloc,&next_buf); +#ifdef DEBUG +fprintf(stderr,"next: alloc=%u used=%u buf=%p\n",(unsigned)next_alloc,(unsigned)next_used,next_buf); +#endif + if(next_used == 0) {stat = NC_EFILTER; lastbuffer = next_buf; goto done; } + current_buf = next_buf; + current_alloc = next_alloc; + current_used = next_used; + } + } else { + /* Apply in reverse order */ + for(i=nclistlength(chain)-1;i>=0;i--) { + f = (struct NCZ_Filter*)nclistget(chain,i); + ff = f->plugin->hdf5.filter; + /* code can be simplified */ + next_alloc = current_alloc; + next_buf = current_buf; + next_used = 0; + next_used = ff->filter(H5Z_FLAG_REVERSE,f->hdf5.working.nparams,f->hdf5.working.params,current_used,&next_alloc,&next_buf); +#ifdef DEBUG +fprintf(stderr,"next: alloc=%u used=%u buf=%p\n",(unsigned)next_alloc,(unsigned)next_used,next_buf); +#endif + if(next_used == 0) {stat = NC_EFILTER; lastbuffer = next_buf; goto done;} + current_buf = next_buf; + current_alloc = next_alloc; + current_used = next_used; + } + } +#ifdef DEBUG +fprintf(stderr,"current: alloc=%u used=%u buf=%p\n",(unsigned)current_alloc,(unsigned)current_used,current_buf); +#endif + /* return results */ + if(outlenp) {*outlenp = current_used;} /* or should it be current_alloc? */ + if(outdatap) {*outdatap = current_buf;} + } + +done: + if(lastbuffer != NULL && lastbuffer != indata) nullfree(lastbuffer); /* cleanup */ + return ZUNTRACEX(stat,"outlen=%u outdata=%p",(unsigned)*outlenp,*outdatap); +} + +/**************************************************/ +/* JSON Parse/unparse of filters */ +int +NCZ_filter_jsonize(const NC_FILE_INFO_T* file, const NC_VAR_INFO_T* var, NCZ_Filter* filter, NCjson** jfilterp) +{ + int stat = NC_NOERR; + NCjson* jfilter = NULL; + + ZTRACE(6,"var=%s filter=%s",var->hdr.name,(filter != NULL && filter->codec.id != NULL?filter->codec.id:"null")); + + /* assumptions */ + assert(filter->flags & FLAG_WORKING); + + /* Convert the HDF5 id + parameters to the codec form */ + + /* We need to ensure the the current visible parameters are defined and had the opportunity to come + from the working parameters */ + assert((filter->flags & (FLAG_VISIBLE | FLAG_WORKING)) == (FLAG_VISIBLE | FLAG_WORKING)); +#if 0 + if((stat = rebuild_visible(var,filter))) goto done; +#endif + + /* Convert the visible parameters back to codec */ + /* Clear any previous codec */ + nullfree(filter->codec.id); filter->codec.id = NULL; + nullfree(filter->codec.codec); filter->codec.codec = NULL; + filter->codec.id = strdup(filter->plugin->codec.codec->codecid); + if(filter->plugin->codec.codec->NCZ_hdf5_to_codec) { + stat = filter->plugin->codec.codec->NCZ_hdf5_to_codec(filter->hdf5.visible.nparams,filter->hdf5.visible.params,&filter->codec.codec); +#ifdef DEBUGF + fprintf(stderr,"DEBUGF: NCZ_hdf5_to_codec: visible=%s codec=%s\n",printnczparams(filter->hdf5.visible),filter->codec.codec); +#endif + if(stat) goto done; + } else + {stat = NC_EFILTER; goto done;} + + /* Parse the codec as the return */ + if(NCJparse(filter->codec.codec,0,&jfilter) < 0) {stat = NC_EFILTER; goto done;} + if(jfilterp) {*jfilterp = jfilter; jfilter = NULL;} + +done: + NCJreclaim(jfilter); + return ZUNTRACEX(stat,"codec=%s",NULLIFY(filter->codec.codec)); +} + + +/* Build filter from parsed Zarr metadata */ +int +NCZ_filter_build(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, const NCjson* jfilter) +{ + int i,stat = NC_NOERR; + NCZ_Filter* filter = NULL; + NCjson* jvalue = NULL; + NCZ_Plugin* plugin = NULL; + NCZ_Codec codec = codec_empty; + NCZ_HDF5 hdf5 = hdf5_empty; + + ZTRACE(6,"file=%s var=%s jfilter=%s",file->hdr.name,var->hdr.name,NCJtrace(jfilter)); + + if(var->filters == NULL) var->filters = nclistnew(); + + /* Get the id of this codec filter */ + if(NCJdictget(jfilter,"id",&jvalue)<0) {stat = NC_EFILTER; goto done;} + if(NCJsort(jvalue) != NCJ_STRING) { + ZLOG(NCLOGERR,"no such filter: %s",NCJstring(jvalue)); + stat = NC_ENOFILTER; goto done; + } + + /* Build the codec */ + if((codec.id = strdup(NCJstring(jvalue)))==NULL) + {stat = NC_ENOMEM; goto done;} + if(NCJunparse(jfilter,0,&codec.codec)<0) {stat = NC_EFILTER; goto done;} + + /* Find the plugin for this filter */ + for(i=0;i<=loaded_plugins_max;i++) { + if (!loaded_plugins[i]) continue; + if(strcmp(NCJstring(jvalue), loaded_plugins[i]->codec.codec->codecid) == 0) + {plugin = loaded_plugins[i]; break;} + } + + if(plugin != NULL) { + /* Save the hdf5 id */ + hdf5.id = plugin->codec.codec->hdf5id; + /* Convert the codec to hdf5 form visible parameters */ + if(plugin->codec.codec->NCZ_codec_to_hdf5) { + stat = plugin->codec.codec->NCZ_codec_to_hdf5(codec.codec,&hdf5.visible.nparams,&hdf5.visible.params); +#ifdef DEBUGF + fprintf(stderr,"DEBUGF: NCZ_codec_to_hdf5: codec=%s, hdf5=%s\n",printcodec(codec),printhdf5(hdf5)); +#endif + if(stat) goto done; + } + + if((filter = calloc(1,sizeof(NCZ_Filter)))==NULL) {stat = NC_ENOMEM; goto done;} + filter->flags |= FLAG_VISIBLE; + filter->hdf5 = hdf5; hdf5 = hdf5_empty; + filter->codec = codec; codec = codec_empty; + filter->flags |= FLAG_CODEC; + filter->plugin = plugin; plugin = NULL; + } + + if(filter != NULL) { + NClist* filterlist = (NClist*)var->filters; + nclistpush(filterlist,filter); + filter = NULL; + } + +done: + ncz_hdf5_clear(&hdf5); + ncz_codec_clear(&codec); + NCZ_filter_free(filter); + return ZUNTRACE(stat); +} + +/**************************************************/ +/* Filter loading */ + +/* +Get entries in a path that is assumed to be a directory. +*/ + +#ifdef _WIN32 + +static int +getentries(const char* path, NClist* contents) +{ + /* Iterate over the entries in the directory */ + int ret = NC_NOERR; + errno = 0; + WIN32_FIND_DATA FindFileData; + HANDLE dir = NULL; + char* ffpath = NULL; + char* lpath = NULL; + size_t len; + char* d = NULL; + + ZTRACE(6,"path=%s",path); + + /* We need to process the path to make it work with FindFirstFile */ + len = strlen(path); + /* Need to terminate path with '/''*' */ + ffpath = (char*)malloc(len+2+1); + memcpy(ffpath,path,len); + if(path[len-1] != '/') { + ffpath[len] = '/'; + len++; + } + ffpath[len] = '*'; len++; + ffpath[len] = '\0'; + + /* localize it */ + if((ret = nczm_localize(ffpath,&lpath,LOCALIZE))) goto done; + dir = FindFirstFile(lpath, &FindFileData); + if(dir == INVALID_HANDLE_VALUE) { + /* Distinquish not-a-directory from no-matching-file */ + switch (GetLastError()) { + case ERROR_FILE_NOT_FOUND: /* No matching files */ /* fall thru */ + ret = NC_NOERR; + goto done; + case ERROR_DIRECTORY: /* not a directory */ + default: + ret = NC_EEMPTY; + goto done; + } + } + do { + char* p = NULL; + const char* name = NULL; + name = FindFileData.cFileName; + if(strcmp(name,".")==0 || strcmp(name,"..")==0) + continue; + nclistpush(contents,strdup(name)); + } while(FindNextFile(dir, &FindFileData)); + +done: + if(dir) FindClose(dir); + nullfree(lpath); + nullfree(ffpath); + nullfree(d); + errno = 0; + return ZUNTRACEX(ret,"|contents|=%d",(int)nclistlength(contents)); +} + +#else /* !_WIN32 */ + +int +getentries(const char* path, NClist* contents) +{ + int ret = NC_NOERR; + errno = 0; + DIR* dir = NULL; + + ZTRACE(6,"path=%s",path); + + dir = NCopendir(path); + if(dir == NULL) + {ret = (errno); goto done;} + for(;;) { + const char* name = NULL; + struct dirent* de = NULL; + errno = 0; + de = readdir(dir); + if(de == NULL) + {ret = (errno); goto done;} + if(strcmp(de->d_name,".")==0 || strcmp(de->d_name,"..")==0) + continue; + name = de->d_name; + nclistpush(contents,strdup(name)); } - done: + if(dir) NCclosedir(dir); + errno = 0; + return ZUNTRACEX(ret,"|contents|=%d",(int)nclistlength(contents)); +} +#endif /*_WIN32*/ + +static int +NCZ_load_all_plugins(void) +{ + int i,ret = NC_NOERR; + const char* pluginroot = NULL; + struct stat buf; + NClist* dirs = nclistnew(); +#ifdef _WIN32 + char pluginpath32[4096]; +#endif + + ZTRACE(6,""); + +#ifdef DEBUGL + fprintf(stderr,"DEBUGL: NCZ_load_all_plugins\n"); +#endif + + /* Find the plugin directory root(s) */ + pluginroot = getenv(plugin_env); + if(pluginroot == NULL || strlen(pluginroot) == 0) { +#ifdef _WIN32 + const char* win32_root; + win32_root = getenv(win32_root_env); + if(win32_root != NULL && strlen(win32_root) > 0) { + snprintf(pluginpath32,sizeof(pluginpath32),plugin_dir_win,win32_root); + pluginroot = pluginpath32; + } else + pluginroot = NULL; +#else /*!_WIN32*/ + pluginroot = plugin_dir_unix; +#endif + } + + ZTRACEMORE(6,"pluginroot=%s",(pluginroot?pluginroot:"null")); + if(pluginroot == NULL) { + ZLOG(NCLOGERR,"no pluginroot: %s",plugin_env); + ret = NC_ENOFILTER; goto done; + } + + if((ret = NCZ_split_plugin_path(pluginroot,dirs))) goto done; + + for(i=0;ihdf5id; + NCZ_Plugin* p = loaded_plugins[hdf5id]; /* get candidate */ + if(p != NULL && p->hdf5.filter != NULL && p->codec.codec == NULL) { +#ifdef DEBUGL + fprintf(stderr,"DEBUGL: plugin defaulted: id=%u, codec=%s\n",hdf5id,codec->codecid); +#endif + p->codec.codec = codec; codec = NULL; + p->codec.codeclib = NULL; + } + } + } + + /* Expunge all plugins for which we do not have both HDF5 and codec */ + { + int i; + NCZ_Plugin* p; + for(i=0;ihdf5.filter == NULL || p->codec.codec == NULL) { +#ifdef DEBUGL + fprintf(stderr,"DEBUGL: plugin expunged: id=%u\n",p->hdf5.filter->id); +#endif + /* expunge this entry */ + (void)NCZ_unload_plugin(p); + loaded_plugins[i] = NULL; + } + } + } + } + /* Iniitalize all remaining plugins */ + { + int i; + NCZ_Plugin* p; + for(i=0;ihdf5.filter != NULL && p->codec.codec != NULL) { + if(p->codec.codec && p->codec.codec->NCZ_codec_initialize) + p->codec.codec->NCZ_codec_initialize(); +#ifdef DEBUGL + fprintf(stderr,"DEBUGL: plugin initialized: id=%u\n",p->hdf5.filter->id); +#endif + } + } + } + } + +done: + nclistfreeall(dirs); + errno = 0; + return ZUNTRACE(ret); +} + +static int +NCZ_split_plugin_path(const char* path0, NClist* list) +{ + int i,stat = NC_NOERR; + char* path = NULL; + char* p; + int count; + size_t plen; +#ifdef _WIN32 + const char* seps = ";"; +#else + const char* seps = ";:"; +#endif + + if(path0 == NULL || path0[0] == '\0') goto done; + plen = strlen(path0); + if((path = malloc(plen+1+1))==NULL) {stat = NC_ENOMEM; goto done;} + memcpy(path,path0,plen); + path[plen] = '\0'; path[plen+1] = '\0'; /* double null term */ + for(count=0,p=path;*p;p++) { + if(strchr(seps,*p) != NULL) {*p = '\0'; count++;} + } + count++; /* for last piece */ + for(p=path,i=0;i 0) + nclistpush(list,strdup(p)); + p = p+len+1; /* point to next piece */ + } + +done: + nullfree(path); return stat; +} + + +/* Load all the filters within a specified directory */ +static int +NCZ_load_plugin_dir(const char* path) +{ + int i,stat = NC_NOERR; + size_t pathlen; + NClist* contents = nclistnew(); + char* file = NULL; + + ZTRACE(7,"path=%s",path); + +#ifdef DEBUGL + fprintf(stderr,"DEBUGL: NCZ_load_plugin_dir: path=%s\n",path); +#endif + if(path == NULL) {stat = NC_EINVAL; goto done;} + pathlen = strlen(path); + if(pathlen == 0) {stat = NC_EINVAL; goto done;} + + if((stat = getentries(path,contents))) goto done; + for(i=0;i 0); + nullfree(file); file = NULL; + if((file = (char*)malloc(flen))==NULL) {stat = NC_ENOMEM; goto done;} + file[0] = '\0'; + strlcat(file,path,flen); + strlcat(file,"/",flen); + strlcat(file,name,flen); + /* See if can load the file */ + switch ((stat = NCZ_load_plugin(file,&plugin))) { + case NC_NOERR: break; + case NC_ENOFILTER: case NC_ENOTFOUND: stat = NC_NOERR; break; /* will cause it to be ignored */ + default: goto done; + } + if(plugin != NULL) { + id = plugin->hdf5.filter->id; + if(loaded_plugins[id] == NULL) { + loaded_plugins[id] = plugin; + if(id > loaded_plugins_max) loaded_plugins_max = id; +#ifdef DEBUGL + fprintf(stderr,"DEBUGL: plugin loaded: %s\n",printplugin(plugin)); +#endif + } else { +#ifdef DEBUGL + fprintf(stderr,"DEBUGL: plugin duplicate: %s\n",printplugin(plugin)); +#endif + NCZ_unload_plugin(plugin); /* its a duplicate */ + } + } else + stat = NC_NOERR; /*ignore failure */ + } + +done: + nullfree(file); + nclistfreeall(contents); + return ZUNTRACE(stat); } +static int +NCZ_load_plugin(const char* path, struct NCZ_Plugin** plugp) +{ + int stat = NC_NOERR; + NCZ_Plugin* plugin = NULL; + const H5Z_class2_t* h5class = NULL; + const NCZ_codec_t* codec = NULL; + NCPSharedLib* lib = NULL; + int flags = NCP_GLOBAL; + int h5id = -1; + + assert(path != NULL && strlen(path) > 0 && plugp != NULL); + + ZTRACE(8,"path=%s",path); + +#ifdef DEBUGL + fprintf(stderr,"DEBUGL: NCZ_load_plugin: path=%s\n",path); +#endif + + if(plugp) *plugp = NULL; + +#ifdef _WIN32 + /*triage because visual studio does a popup if the file will not load*/ + if(memcmp(path+(strlen(path)-4),".dll",4) != 0) { + stat = NC_ENOFILTER; goto done; + } +#endif + + /* load the shared library */ + if((stat = ncpsharedlibnew(&lib))) goto done; + if((stat = ncpload(lib,path,flags))) goto done; + + /* See what we have */ + { + H5PL_get_plugin_type_proto gpt = (H5PL_get_plugin_type_proto)ncpgetsymbol(lib,"H5PLget_plugin_type"); + H5PL_get_plugin_info_proto gpi = (H5PL_get_plugin_info_proto)ncpgetsymbol(lib,"H5PLget_plugin_info"); + NCZ_get_codec_info_proto npi = (NCZ_get_codec_info_proto)ncpgetsymbol(lib,"NCZ_get_codec_info"); + NCZ_codec_info_defaults_proto cpd = (NCZ_codec_info_defaults_proto)ncpgetsymbol(lib,"NCZ_codec_info_defaults"); + + /* Deal with defaults first */ + if(cpd != NULL) { + if(codec_defaults == NULL) { + codec_defaults = (NCZ_codec_t**)cpd(); + default_lib = lib; lib = NULL; + } + goto done; + } + + if(gpt != NULL && gpi != NULL) { + /* get HDF5 info */ + H5PL_type_t h5type = gpt(); + h5class = gpi(); + /* Verify */ + if(h5type != H5PL_TYPE_FILTER) {stat = NC_EPLUGIN; goto done;} + if(h5class->version != H5Z_CLASS_T_VERS) {stat = NC_EFILTER; goto done;} + } + + if(npi != NULL) { + /* get Codec info */ + codec = npi(); + /* Verify */ + if(codec->version != NCZ_CODEC_CLASS_VER) {stat = NC_EPLUGIN; goto done;} + if(codec->sort != NCZ_CODEC_HDF5) {stat = NC_EPLUGIN; goto done;} + } + } + + /* Ignore this library if neither h5class nor codec are defined */ + if(h5class == NULL && codec == NULL) {stat = NC_ENOFILTER; goto done;} + +#ifdef DEBUGL +fprintf(stderr,"DEBUGL: load: %s:",path); +if(h5class) fprintf(stderr," %u",(unsigned)h5class->id); +if(codec) fprintf(stderr," %u/%s",codec->hdf5id,codec->codecid); +fprintf(stderr,"\n"); +#endif + + if(h5class != NULL && codec != NULL) { + /* Verify consistency of the HDF5 and the Codec */ + if(h5class->id != codec->hdf5id) goto done; /* ignore */ + } + + /* There are several cases to consider: + 1. This library has both HDF5 API and Codec API => merge + 2. This library has HDF5 API only and Codec API was already found in another library => merge + 3. This library has Codec API only and HDF5 API was already found in another library => merge + */ + + /* Get any previous plugin entry for this id; may be NULL */ + if(h5class != NULL) { + h5id = h5class->id; + if((stat = NCZ_plugin_loaded(h5class->id,&plugin))) goto done; + } else if(codec != NULL) { + h5id = codec->hdf5id; + if((stat = NCZ_plugin_loaded(codec->hdf5id,&plugin))) goto done; + } + if(plugin == NULL) { + /* create new entry */ + if((plugin = (NCZ_Plugin*)calloc(1,sizeof(NCZ_Plugin)))==NULL) {stat = NC_ENOMEM; goto done;} + } + + /* Fill in the plugin */ + if(plugin->hdf5.filter == NULL) { + plugin->hdf5.filter = h5class; + plugin->hdf5.hdf5lib = lib; + lib = NULL; + } + if(plugin->codec.codec == NULL) { + plugin->codec.codec = codec; + plugin->codec.codeclib = lib; + lib = NULL; + } + +#ifdef DEBUGL + if(plugin) + fprintf(stderr,"DEBUGL: load_plugin: %s\n",printplugin(plugin)); +#endif + + + /* Cleanup */ + if(plugin->hdf5.hdf5lib == plugin->codec.codeclib) + plugin->codec.codeclib = NULL; + if((stat=NCZ_plugin_save(h5id,plugin))) goto done; + plugin = NULL; + +done: + if(lib) { + (void)ncpsharedlibfree(lib); + } + if(plugin) NCZ_unload_plugin(plugin); + return ZUNTRACEX(stat,"plug=%p",*plugp); +} + +static int +NCZ_unload_plugin(NCZ_Plugin* plugin) +{ + ZTRACE(9,"plugin=%p",plugin); + + if(plugin) { +#ifdef DEBUGL + fprintf(stderr,"DEBUGL: unload: %s\n",printplugin(plugin)); +#endif + if(plugin->codec.codec && plugin->codec.codec->NCZ_codec_finalize) + plugin->codec.codec->NCZ_codec_finalize(); + if(plugin->hdf5.filter != NULL) loaded_plugins[plugin->hdf5.filter->id] = NULL; + if(plugin->hdf5.hdf5lib != NULL) (void)ncpsharedlibfree(plugin->hdf5.hdf5lib); + if(plugin->codec.codeclib != NULL) (void)ncpsharedlibfree(plugin->codec.codeclib); + memset(plugin,0,sizeof(NCZ_Plugin)); + free(plugin); + } + return ZUNTRACE(NC_NOERR); +} /**************************************************/ -/* Debug functions */ +/* _Codecs attribute */ -#ifdef TFILTERS -static void -printfilter1(struct NCZ_Filter* nfs) +int +NCZ_codec_attr(const NC_VAR_INFO_T* var, size_t* lenp, void* data) { - int i; - if(nfs == NULL) { - fprintf(stderr,"{null}"); - return; + int i,stat = NC_NOERR; + size_t len; + char* contents = NULL; + NCbytes* buf = NULL; + NClist* filters = (NClist*)var->filters; + + ZTRACE(6,"var=%s",var->hdr.name); + if(nclistlength(filters) == 0) {stat = NC_ENOTATT; goto done;} + buf = ncbytesnew(); ncbytessetalloc(buf,1024); + ncbytescat(buf,"["); + for(i=0;i 0) ncbytescat(buf,","); + ncbytescat(buf,spec->codec.codec); } - fprintf(stderr,"{%u,(%u)",nfs->filterid,(int)nfs->nparams); - for(i=0;inparams;i++) { - fprintf(stderr," %s",nfs->params[i]); + ncbytescat(buf,"]"); + len = ncbyteslength(buf); + contents = nclistcontents(buf); + if(lenp) *lenp = len; + if(data) strncpy((char*)data,contents,len+1); +done: + ncbytesfree(buf); + return ZUNTRACEX(stat,"len=%u data=%p",(unsigned)len,data); +} + +static int +ensure_working(const NC_VAR_INFO_T* var, NCZ_Filter* filter) +{ + int stat = NC_NOERR; + if(!(filter->flags & FLAG_WORKING)) { + size_t vnparams = filter->hdf5.visible.nparams; + unsigned* vparams = filter->hdf5.visible.params; + + assert(filter->flags & FLAG_VISIBLE); + /* Convert the visible parameters to working parameters (and back) */ + if(filter->plugin->codec.codec->NCZ_modify_parameters) { + stat = filter->plugin->codec.codec->NCZ_modify_parameters(ncidfor(var),var->hdr.id, + &filter->hdf5.visible.nparams, &filter->hdf5.visible.params, + &filter->hdf5.working.nparams, &filter->hdf5.working.params); + if(stat) goto done; + } else { + /* Just use the visible parameters */ + nullfree(filter->hdf5.working.params); + if((stat = paramnczclone(&filter->hdf5.working,&filter->hdf5.visible))) goto done; + } + filter->flags |= FLAG_WORKING; + /* See if the visible parameters were changed */ + if(vnparams != filter->hdf5.visible.nparams || vparams != filter->hdf5.visible.params) + filter->flags |= FLAG_NEWVISIBLE; } - fprintf(stderr,"}"); +#ifdef DEBUGF + fprintf(stderr,"DEBUGF: ensure_working_parameters: ncid=%lu varid=%u filter=%s\n", ncidfor(var), (unsigned)var->hdr.id,printfilter(filter)); +#endif +done: + return THROW(stat); } -static void -printfilter(struct NCZ_Filter* nfs, const char* tag, int line) +#if 0 +static int +rebuild_visible(const NC_VAR_INFO_T* var, NCZ_Filter* filter) { - fprintf(stderr,"%s: line=%d: ",tag,line); - printfilter1(nfs); - fprintf(stderr,"\n"); + int stat = NC_NOERR; + int nvisible0; + unsigned* visible0 = NULL; + + assert(filter->flags & FLAG_WORKING); + /* If the visible parameters are previously defined, save them */ + if(filter->flags & FLAG_VISIBLE) { + nvisible0 = filter->hdf5.visible.nparams; + visible0 = filter->hdf5.visible.params; + filter->hdf5.visible.nparams = 0; + filter->hdf5.visible.params = NULL; /* temporary */ + } + /* Cases to consider: + 1. visible already defined && NCZ_visible_parameters defined => apply + 2. visible not defined && NCZ_visible_parameters defined defined => apply + 3. visible already defined && NCZ_visible_parameters not defined => keep originals + 4. visible not defined && NCZ_visible_parameters not defined => use working parameters + */ + + /* Cases 1 and 2 */ + /* Convert the working parameters to visibleparameters, overwriting any existing visibles */ + if(filter->plugin->codec.codec->NCZ_visible_parameters) { + stat = filter->plugin->codec.codec->NCZ_visible_parameters(ncidfor(var),var->hdr.id, + filter->hdf5.working.nparams, filter->hdf5.working.params, + &filter->hdf5.visible.nparams, &filter->hdf5.visible.params); + if(stat) goto done; + } else if(filter->flags & FLAG_CODEC) {/* Case 3 */ + filter->hdf5.visible.nparams = nvisible0; + filter->hdf5.visible.params = visible0; visible0 = NULL; + } else {/* Case 4 */ + /* Use the working parameters as the visible parameters */ + filter->hdf5.visible.nparams = filter->hdf5.working.nparams; + if(filter->hdf5.working.nparams > 0) { + if((stat = paramnczclone(&filter->hdf5.visible,&filter->hdf5.working))) goto done; + } + } + filter->flags |= FLAG_VISIBLE; +#ifdef DEBUGF + fprintf(stderr,"DEBUGF: rebuild_visible_parameters: ncid=%lu varid=%u filter=%s\n", ncidfor(var), (unsigned)var->hdr.id,printfilter(filter)); +#endif +done: + nullfree(visible0); + return THROW(stat); } +#endif -static void -printfilterlist(NC_VAR_INFO_T* var, const char* tag, int line) +/* Called by NCZ_enddef to ensure that the working parameters are defined */ +int +NCZ_filter_setup(NC_VAR_INFO_T* var) { - int i; - const char* name; - if(var == NULL) name = "null"; - else if(var->hdr.name == NULL) name = "?"; - else name = var->hdr.name; - fprintf(stderr,"%s: line=%d: var=%s filters=",tag,line,name); - if(var != NULL) { - for(i=0;ifilters);i++) { - struct NCZ_Filter* nfs = nclistget((NClist*)var->filters,i); - fprintf(stderr,"[%d]",i); - printfilter1(nfs); + int i,stat = NC_NOERR; + NClist* filters = NULL; + + ZTRACE(6,"var=%s",var->hdr.name); + + filters = (NClist*)var->filters; + for(i=0;iplugin != NULL); + assert((filter->flags & FLAG_VISIBLE)); /* Assume visible params are defined */ + /* verify */ + assert(filter->hdf5.id > 0 && (filter->hdf5.visible.nparams == 0 || filter->hdf5.visible.params != NULL)); + assert((filter->flags & FLAG_WORKING)==0); /* Assume working params are not defined */ + assert(filter->hdf5.working.nparams == 0 && filter->hdf5.working.params == NULL); + vnparams = filter->hdf5.visible.nparams; + vparams = filter->hdf5.visible.params; + /* Initialize the working parameters */ + if(filter->plugin && filter->plugin->codec.codec->NCZ_modify_parameters) { + stat = filter->plugin->codec.codec->NCZ_modify_parameters(ncidfor(var),var->hdr.id, + &filter->hdf5.visible.nparams,&filter->hdf5.visible.params, + &filter->hdf5.working.nparams,&filter->hdf5.working.params); +#ifdef DEBUGF + fprintf(stderr,"DEBUGF: NCZ_modify_parameters: ncid=%d varid=%d filter=%s\n", (int)ncidfor(var),(int)var->hdr.id, + printfilter(filter)); +#endif + if(stat) goto done; + } else {/* Just copy over the visible params */ + filter->hdf5.working.nparams = vnparams; + if(vnparams > 0) { + if((stat=paramclone(vnparams,&filter->hdf5.working.params,vparams))) goto done; + } } +#ifdef DEBUGF + fprintf(stderr,"DEBUGF: NCZ_filter_setup: ncid=%d varid=%d filter=%s\n", (int)ncidfor(var),(int)var->hdr.id, + printfilter(filter)); +#endif + filter->flags |= FLAG_WORKING; + /* See if the visible parameters were changed */ + if(vnparams != filter->hdf5.visible.nparams || vparams != filter->hdf5.visible.params) + filter->flags |= FLAG_NEWVISIBLE; } - fprintf(stderr,"\n"); + +done: + return ZUNTRACE(stat); +} + +/**************************************************/ + +/* Clone an hdf5 parameter set */ +static int +paramclone(size_t nparams, unsigned** dstp, const unsigned* src) +{ + unsigned* dst = NULL; + if(nparams > 0) { + if(src == NULL) return NC_EINVAL; + if((dst = (unsigned*)malloc(sizeof(unsigned) * nparams))==NULL) + return NC_ENOMEM; + memcpy(dst,src,sizeof(unsigned) * nparams); + } + if(dstp) *dstp = dst; + return NC_NOERR; +} + +static int +paramnczclone(NCZ_Params* dst, const NCZ_Params* src) +{ + assert(src != NULL && dst != NULL && dst->params == NULL); + *dst = *src; + return paramclone(src->nparams,&dst->params,src->params); } -#endif /*TFILTERS*/ diff --git a/libnczarr/zfilter.h b/libnczarr/zfilter.h new file mode 100644 index 0000000000..7a4e2e517a --- /dev/null +++ b/libnczarr/zfilter.h @@ -0,0 +1,52 @@ +/* Copyright 2018-2018 University Corporation for Atmospheric + Research/Unidata. */ + +/** + * @file This header file containsfilter related macros, types, and prototypes for + * the filter code in libnczarr. This header should not be included in + * code outside libnczarr. + * + * @author Dennis Heimbigner + */ + +#ifndef ZFILTER_H +#define ZFILTER_H + +/* zfilter.c */ +/* Dispatch functions are also in zfilter.c */ +/* Filterlist management */ + +/*Mnemonic*/ +#define ENCODING 1 + +/* list of environment variables to check for plugin roots */ +#define plugin_env "HDF5_PLUGIN_PATH" +#define plugin_dir_unix "/usr/local/hdf5/plugin" +#define plugin_dir_win "%s/hdf5/lib/plugin" +#define win32_root_env "ALLUSERSPROFILE" + +/* +Return a NULL terminated vector of pointers to instances of ''NCZ_codec_t''. +The value returned is actually of type ''NCZ_codec_t**'', +but is of type ''void*'' to allow for extensions. +The list of returned items are used to try to provide defaults +for any HDF5 filters that have no corresponding Codec. +This is for internal use only. +*/ +typedef void* (*NCZ_codec_info_defaults_proto)(void); + +/* Opaque */ +struct NCZ_Filter; + +int NCZ_filter_initialize(void); +int NCZ_filter_finalize(void); +int NCZ_addfilter(NC_FILE_INFO_T*, NC_VAR_INFO_T* var, unsigned int id, size_t nparams, const unsigned int* params); +int NCZ_filter_setup(NC_VAR_INFO_T* var); +int NCZ_filter_freelist(NC_VAR_INFO_T* var); +int NCZ_codec_freelist(NCZ_VAR_INFO_T* zvar); +int NCZ_applyfilterchain(const NC_FILE_INFO_T*, NC_VAR_INFO_T*, NClist* chain, size_t insize, void* indata, size_t* outlen, void** outdata, int encode); +int NCZ_filter_jsonize(const NC_FILE_INFO_T*, const NC_VAR_INFO_T*, struct NCZ_Filter* filter, struct NCjson**); +int NCZ_filter_build(const NC_FILE_INFO_T*, NC_VAR_INFO_T* var, const NCjson* jfilter); +int NCZ_codec_attr(const NC_VAR_INFO_T* var, size_t* lenp, void* data); + +#endif /*ZFILTER_H*/ diff --git a/libnczarr/zinternal.c b/libnczarr/zinternal.c index 48b006ca5d..499e561af3 100644 --- a/libnczarr/zinternal.c +++ b/libnczarr/zinternal.c @@ -16,6 +16,7 @@ */ #include "zincludes.h" +#include "zfilter.h" /* These are the default chunk cache sizes for ZARR files created or * opened with netCDF-4. */ @@ -76,6 +77,7 @@ NCZ_initialize_internal(void) ngs->zarr.dimension_separator = dimsep[0]; } } + done: return stat; } @@ -89,6 +91,7 @@ NCZ_finalize_internal(void) { /* Reclaim global resources */ ncz_initialized = 0; + NCZ_filter_finalize(); return NC_NOERR; } @@ -631,6 +634,7 @@ ncz_find_grp_var_att(int ncid, int varid, const char *name, int attnum, /** * @internal What fill value should be used for a variable? + * Side effects: set as default if necessary and build _FillValue attribute. * * @param h5 Pointer to file info struct. * @param var Pointer to variable info struct. @@ -644,32 +648,45 @@ int ncz_get_fill_value(NC_FILE_INFO_T *h5, NC_VAR_INFO_T *var, void **fillp) { size_t size; - int retval; + int retval = NC_NOERR; +#if 0 /*LOOK*/ /* Find out how much space we need for this type's fill value. */ if (var->type_info->nc_type_class == NC_VLEN) size = sizeof(nc_vlen_t); else if (var->type_info->nc_type_class == NC_STRING) size = sizeof(char *); else +#endif { - if ((retval = nc4_get_typelen_mem(h5, var->type_info->hdr.id, &size))) - return retval; + if ((retval = nc4_get_typelen_mem(h5, var->type_info->hdr.id, &size))) goto done; } assert(size); - /* Allocate the space. */ - if (!((*fillp) = calloc(1, size))) - return NC_ENOMEM; + /* If the user has set a fill_value for this var, use, otherwise find the default fill value. */ - /* If the user has set a fill_value for this var, use, otherwise - * find the default fill value. */ - if (var->fill_value) - { - LOG((4, "Found a fill value for var %s", var->hdr.name)); + if (var->fill_value == NULL) { + /* initialize the fill_value to the default */ + /* Allocate the fill_value space. */ + if((var->fill_value = calloc(1, size))==NULL) + {retval = NC_ENOMEM; goto done;} + if((retval = nc4_get_default_fill_value(var->type_info->hdr.id, var->fill_value))) { + /* Note: release memory, but don't return error on failure */ + free(var->fill_value); + var->fill_value = NULL; + retval = NC_NOERR; + goto done; + } + } + assert(var->fill_value != NULL); + + LOG((4, "Found a fill value for var %s", var->hdr.name)); +#if 0 /*LOOK*/ + /* Need to copy both vlen and a single basetype */ if (var->type_info->nc_type_class == NC_VLEN) { - nc_vlen_t *in_vlen = (nc_vlen_t *)(var->fill_value), *fv_vlen = (nc_vlen_t *)(*fillp); + nc_vlen_t *in_vlen = (nc_vlen_t *)(var->fill_value); + nc_vlen-t *fv_vlen = (nc_vlen_t *)fill; size_t basetypesize = 0; if((retval=nc4_get_typelen_mem(h5, var->type_info->u.v.base_nc_typeid, &basetypesize))) @@ -694,20 +711,21 @@ ncz_get_fill_value(NC_FILE_INFO_T *h5, NC_VAR_INFO_T *var, void **fillp) return NC_ENOMEM; } } - else - memcpy((*fillp), var->fill_value, size); - } - else - { - if (nc4_get_default_fill_value(var->type_info->hdr.id, *fillp)) - { - /* Note: release memory, but don't return error on failure */ - free(*fillp); - *fillp = NULL; - } +#endif /*0*/ + /* Create _FillValue Attribute */ + if((retval = ncz_create_fillvalue(var))) goto done; + if(fillp) { + void* fill = NULL; + /* Allocate the return space. */ + if((fill = calloc(1, size))==NULL) + {retval = NC_ENOMEM; goto done;} + memcpy(fill, var->fill_value, size); + *fillp = fill; + fill = NULL; } - return NC_NOERR; +done: + return retval; } #ifdef LOGGING diff --git a/libnczarr/zinternal.h b/libnczarr/zinternal.h index a666181874..471bf2f1b0 100644 --- a/libnczarr/zinternal.h +++ b/libnczarr/zinternal.h @@ -56,6 +56,9 @@ #define ZATTRS ".zattrs" #define ZARRAY ".zarray" +/* Pure Zarr pseudo names */ +#define ZDIMANON "_zdim" + /* V2 Reserved Attributes */ /* Inserted into /.zgroup @@ -91,13 +94,16 @@ Inserted into any .zattrs ? or should it go into the container? #define islegaldimsep(c) ((c) != '\0' && strchr(LEGAL_DIM_SEPARATORS,(c)) != NULL) - /* Mnemonics */ #define ZCLOSE 1 /* this is closeorabort as opposed to enddef */ /* Mnemonics */ #define ZCLOSE 1 /* this is closeorabort as opposed to enddef */ +/* Useful macro */ +#define ncidforx(file,grpid) ((file)->controller->ext_ncid | (grpid)) +#define ncidfor(var) ncidforx((var)->container->nc4_info,(var)->container->hdr.id) + /**************************************************/ /* Forward */ @@ -246,23 +252,6 @@ int ncz_makeattr(NC_OBJ*, NCindex* attlist, const char* name, nc_type typid, siz int ncz_gettype(NC_FILE_INFO_T*, NC_GRP_INFO_T*, int xtype, NC_TYPE_INFO_T** typep); int ncz_find_default_chunksizes2(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var); -/* zfilter.c */ -/* Dispatch functions are also in zfilter.c */ -/* Filterlist management */ - -/* The NC_VAR_INFO_T->filters field is an NClist of this struct */ -struct NCZ_Filter { - int flags; /**< Flags describing state of this filter. */ -#define NCZ_FILTER_MISSING 1 /* Signal filter implementation is not available */ - unsigned int filterid; /**< ID for arbitrary filter. */ - size_t nparams; /**< nparams for arbitrary filter. */ - unsigned int* params; /**< Params for arbitrary filter. */ -}; - -int NCZ_filter_lookup(NC_VAR_INFO_T* var, unsigned int id, struct NCZ_Filter** specp); -int NCZ_addfilter(NC_VAR_INFO_T* var, unsigned int id, size_t nparams, const unsigned int* params); -int NCZ_filter_freelist(NC_VAR_INFO_T* var); - /* Undefined */ /* Find var, doing lazy var metadata read if needed. */ int ncz_find_grp_file_var(int ncid, int varid, NC_FILE_INFO_T** file, diff --git a/libnczarr/zmap.c b/libnczarr/zmap.c index ed162dead4..674ffe13c1 100644 --- a/libnczarr/zmap.c +++ b/libnczarr/zmap.c @@ -409,20 +409,20 @@ nczm_canonicalpath(const char* path, char** cpathp) { int ret = NC_NOERR; char* cpath = NULL; - char* tmp = NULL; + char* tmp1 = NULL; if(path == NULL) {cpath = NULL; goto done;} - /* Process path to make it be windows compatible */ - if((tmp = NCpathcvt(path))==NULL) {ret = NC_ENOMEM; goto done;} + /* Process path to make it be absolute*/ + if((tmp1 = NCpathabsolute(path))==NULL) {ret = NC_ENOMEM; goto done;} /* Fix slashes to be forward for now */ - if((ret = nczm_localize(tmp,&cpath,!LOCALIZE))) goto done; + if((ret = nczm_localize(tmp1,&cpath,!LOCALIZE))) goto done; if(cpathp) {*cpathp = cpath; cpath = NULL;} done: - nullfree(tmp); + nullfree(tmp1); nullfree(cpath); return THROW(ret); } diff --git a/libnczarr/zmap.h b/libnczarr/zmap.h index 438fd0597c..53564ea799 100644 --- a/libnczarr/zmap.h +++ b/libnczarr/zmap.h @@ -110,9 +110,10 @@ model, only content-bearing objects actually exist. Note that this different than, say, a direvtory tree where a key will always lead to something: a directory or a file. -In any case, the zmap API returns two distinguished error code: +In any case, the zmap API returns three distinguished error code: 1. NC_NOERR if a operation succeeded -2. NC_EEMPTY is returned when accessing a key that has no content. +2. NC_EEMPTY is returned when accessing a key that has no content or does not exist. + This does not preclude other errors being returned such NC_EACCESS or NC_EPERM or NC_EINVAL if there are permission errors or illegal function arguments, for example. It also does not preclude the use of other error codes internal to the zmap diff --git a/libnczarr/zmap_file.c b/libnczarr/zmap_file.c index 5ae1ca3551..f0370a41ff 100755 --- a/libnczarr/zmap_file.c +++ b/libnczarr/zmap_file.c @@ -740,9 +740,7 @@ platformopenfile(ZFMAP* zfmap, const char* canonpath, FD* fd) /* Try to open file (will localize) */ fd->fd = NCopen3(canonpath, ioflags, permissions); if(fd->fd < 0) - { -fprintf(stderr,"xxx: canonpath=%s\n",canonpath); -stat = platformerr(errno); goto done;} /* could not open */ + {stat = platformerr(errno); goto done;} /* could not open */ done: errno = 0; return ZUNTRACEX(stat,"fd=%d",(fd?fd->fd:-1)); diff --git a/libnczarr/zmap_zip.c b/libnczarr/zmap_zip.c index 4cb375be3a..93b9876920 100755 --- a/libnczarr/zmap_zip.c +++ b/libnczarr/zmap_zip.c @@ -101,8 +101,6 @@ static int zipcreate(const char *path, int mode, size64_t flags, void* parameters, NCZMAP** mapp) { int stat = NC_NOERR; - char* truepath = NULL; - char* dataset = NULL; ZZMAP* zzmap = NULL; NCURI* url = NULL; zip_flags_t zipflags = 0; @@ -124,12 +122,6 @@ zipcreate(const char *path, int mode, size64_t flags, void* parameters, NCZMAP** if(strcasecmp(url->protocol,"file") != 0) {stat = NC_EURL; goto done;} - /* Canonicalize the root path */ - if((stat = NCpathcanonical(url->path,&truepath))) goto done; - - /* Extract the dataset name */ - if((stat = nczm_basename(truepath,&dataset))) goto done; - /* Build the zz state */ if((zzmap = calloc(1,sizeof(ZZMAP))) == NULL) {stat = NC_ENOMEM; goto done;} @@ -140,10 +132,11 @@ zipcreate(const char *path, int mode, size64_t flags, void* parameters, NCZMAP** /* create => NC_WRITE */ zzmap->map.mode = mode; zzmap->map.api = &zapi; - zzmap->root = truepath; - truepath = NULL; - zzmap->dataset = dataset; - dataset = NULL; + /* Since root is in canonical form, we need to convert to local form */ + if((zzmap->root = NCpathcvt(url->path))==NULL) + {stat = NC_ENOMEM; goto done;} + /* Extract the dataset name */ + if((stat = nczm_basename(url->path,&zzmap->dataset))) goto done; /* Set zip openflags */ zipflags |= ZIP_CREATE; @@ -168,8 +161,6 @@ zipcreate(const char *path, int mode, size64_t flags, void* parameters, NCZMAP** done: ncurifree(url); - nullfree(truepath); - nullfree(dataset); if(zzmap) zipclose((NCZMAP*)zzmap,1); return ZUNTRACE(stat); } @@ -187,8 +178,6 @@ static int zipopen(const char *path, int mode, size64_t flags, void* parameters, NCZMAP** mapp) { int stat = NC_NOERR; - char* truepath = NULL; - char* dataset = NULL; ZZMAP* zzmap = NULL; NCURI*url = NULL; zip_flags_t zipflags = 0; @@ -209,9 +198,6 @@ zipopen(const char *path, int mode, size64_t flags, void* parameters, NCZMAP** m if(strcasecmp(url->protocol,"file") != 0) {stat = NC_EURL; goto done;} - /* Canonicalize the root path */ - if((stat = NCpathcanonical(url->path,&truepath))) goto done; - /* Build the zz state */ if((zzmap = calloc(1,sizeof(ZZMAP))) == NULL) {stat = NC_ENOMEM; goto done;} @@ -221,8 +207,9 @@ zipopen(const char *path, int mode, size64_t flags, void* parameters, NCZMAP** m zzmap->map.flags = flags; zzmap->map.mode = mode; zzmap->map.api = (NCZMAP_API*)&zapi; - zzmap->root = truepath; - truepath = NULL; + /* Since root is in canonical form, we need to convert to local form */ + if((zzmap->root = NCpathcvt(url->path))==NULL) + {stat = NC_ENOMEM; goto done;} /* Set zip open flags */ zipflags |= ZIP_CHECKCONS; @@ -258,8 +245,6 @@ zipopen(const char *path, int mode, size64_t flags, void* parameters, NCZMAP** m done: ncurifree(url); - nullfree(truepath); - nullfree(dataset); if(zzmap) zipclose((NCZMAP*)zzmap,0); return ZUNTRACE(stat); diff --git a/libnczarr/zopen.c b/libnczarr/zopen.c index df894d645c..4c850c1ada 100644 --- a/libnczarr/zopen.c +++ b/libnczarr/zopen.c @@ -124,7 +124,7 @@ ncz_open_file(const char *path, int mode, const char** controls, int ncid) exit: if (stat && h5) - ncz_close_file(h5, 1); /* treat like abort*/ + ncz_closeorabort(h5, NULL, 1); /* treat like abort*/ return ZUNTRACE(stat); } diff --git a/libnczarr/zsync.c b/libnczarr/zsync.c index 3b583aaddd..03f0fa4c1d 100644 --- a/libnczarr/zsync.c +++ b/libnczarr/zsync.c @@ -4,12 +4,13 @@ *********************************************************************/ #include "zincludes.h" +#include "zfilter.h" #undef FILLONCLOSE /* Forward */ static int ncz_collect_dims(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NCjson** jdimsp); -static int ncz_sync_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var); +static int ncz_sync_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, int isclose); static int ncz_jsonize_atts(NCindex* attlist, NCjson** jattrsp); static int load_jatts(NCZMAP* map, NC_OBJ* container, int nczarrv1, NCjson** jattrsp, NClist** atypes); @@ -63,7 +64,7 @@ ncz_sync_file(NC_FILE_INFO_T* file, int isclose) ZTRACE(3,"file=%s isclose=%d",file->controller->path,isclose); /* Write out root group recursively */ - if((stat = ncz_sync_grp(file, file->root_grp))) + if((stat = ncz_sync_grp(file, file->root_grp, isclose))) goto done; done: @@ -111,7 +112,7 @@ ncz_collect_dims(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NCjson** jdimsp) * @author Dennis Heimbigner */ int -ncz_sync_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp) +ncz_sync_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, int isclose) { int i,stat = NC_NOERR; NCZ_FILE_INFO_T* zinfo = NULL; @@ -205,19 +206,19 @@ ncz_sync_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp) /* Build the .zattrs object */ assert(grp->att); - if((stat = ncz_sync_atts(file,(NC_OBJ*)grp, grp->att))) + if((stat = ncz_sync_atts(file,(NC_OBJ*)grp, grp->att, isclose))) goto done; /* Now synchronize all the variables */ for(i=0; ivars); i++) { NC_VAR_INFO_T* var = (NC_VAR_INFO_T*)ncindexith(grp->vars,i); - if((stat = ncz_sync_var(file,var))) goto done; + if((stat = ncz_sync_var(file,var,isclose))) goto done; } /* Now recurse to synchronize all the subgrps */ for(i=0; ichildren); i++) { NC_GRP_INFO_T* g = (NC_GRP_INFO_T*)ncindexith(grp->children,i); - if((stat = ncz_sync_grp(file,g))) goto done; + if((stat = ncz_sync_grp(file,g,isclose))) goto done; } done: @@ -234,15 +235,17 @@ ncz_sync_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp) } /** - * @internal Synchronize variable data from memory to map. + * @internal Synchronize variable meta data from memory to map. * + * @param file Pointer to file struct * @param var Pointer to var struct + * @param isclose If this called as part of nc_close() as opposed to nc_enddef(). * * @return ::NC_NOERR No error. * @author Dennis Heimbigner */ static int -ncz_sync_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var) +ncz_sync_var_meta(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, int isclose) { int i,stat = NC_NOERR; NCZ_FILE_INFO_T* zinfo = NULL; @@ -259,9 +262,9 @@ ncz_sync_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var) NCjson* jfill = NULL; size64_t shape[NC_MAX_VAR_DIMS]; NCZ_VAR_INFO_T* zvar = var->format_var_info; + NClist* filterchain = NULL; + NCjson* jfilter = NULL; - LOG((3, "%s: dims: %s", __func__, key)); - zinfo = file->format_file_info; map = zinfo->map; @@ -338,30 +341,11 @@ ncz_sync_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var) if(var->no_fill) { if((stat=NCJnew(NCJ_NULL,&jfill))) goto done; } else {/*!var->no_fill*/ - int fillsort; int atomictype = var->type_info->hdr.id; - /* A scalar value providing the default value to use for uninitialized - portions of the array, or ``null`` if no fill_value is to be used. */ - /* Use the defaults defined in netdf.h */ - assert(atomictype > 0 && atomictype <= NC_MAX_ATOMIC_TYPE && atomictype != NC_STRING); - if((stat = ncz_fill_value_sort(atomictype,&fillsort))) goto done; - if(var->fill_value == NULL) { /* use default */ - size_t typelen; - if((stat = NC4_inq_atomic_type(atomictype, NULL, &typelen))) goto done; - var->fill_value = (atomictype == NC_CHAR ? malloc(typelen+1) : malloc(typelen)); - if(var->fill_value == NULL) {stat = NC_ENOMEM; goto done;} - if((stat = nc4_get_default_fill_value(atomictype,var->fill_value))) goto done; - } + assert(var->fill_value != NULL); /* Convert var->fill_value to a string */ if((stat = NCZ_stringconvert(atomictype,1,var->fill_value,&jfill))) goto done; - if(NCJsort(jfill) == NCJ_ARRAY) { /* stringconvert should prevent this from happening */ - assert(NCJlength(jfill) > 0); - jtmp = NCJith(jfill,0); - if((stat = NCJclone(jtmp,&jtmp))) goto done; /* clone so we can free it later */ - NCJreclaim(jfill); - jfill = jtmp; - jtmp = NULL; - } + assert(jfill->sort != NCJ_ARRAY); } if((stat = NCJinsert(jvar,"fill_value",jfill))) goto done; jfill = NULL; @@ -373,20 +357,44 @@ ncz_sync_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var) /* Default to C for now */ if((stat = NCJaddstring(jvar,NCJ_STRING,"C"))) goto done; + /* Compressor and Filters */ + filterchain = (NClist*)var->filters; + /* compressor key */ - /* A JSON object identifying the primary compression codec and providing + /* From V2 Spec: A JSON object identifying the primary compression codec and providing configuration parameters, or ``null`` if no compressor is to be used. */ if((stat = NCJaddstring(jvar,NCJ_STRING,"compressor"))) goto done; - /* Default to null for now */ - if((stat = NCJnew(NCJ_NULL,&jtmp))) goto done; - if((stat = NCJappend(jvar,jtmp))) goto done; + if(nclistlength(filterchain) > 0) { + struct NCZ_Filter* filter = (struct NCZ_Filter*)nclistget(filterchain,nclistlength(filterchain)-1); + /* encode up the compressor */ + if((stat = NCZ_filter_jsonize(file,var,filter,&jtmp))) goto done; + } else { /* no filters at all */ + /* Default to null */ + if((stat = NCJnew(NCJ_NULL,&jtmp))) goto done; + } + if(jtmp && (stat = NCJappend(jvar,jtmp))) goto done; jtmp = NULL; /* filters key */ - if((stat = NCJaddstring(jvar,NCJ_STRING,"filters"))) goto done; + /* From V2 Spec: A list of JSON objects providing codec configurations, + or null if no filters are to be applied. Each codec configuration + object MUST contain a "id" key identifying the codec to be used. */ /* A list of JSON objects providing codec configurations, or ``null`` if no filters are to be applied. */ - if((stat = NCJnew(NCJ_NULL,&jtmp))) goto done; + if((stat = NCJaddstring(jvar,NCJ_STRING,"filters"))) goto done; + if(nclistlength(filterchain) > 1) { + int k; + /* jtmp holds the array of filters */ + if((stat = NCJnew(NCJ_ARRAY,&jtmp))) goto done; + for(k=0;katt); - if((stat = ncz_sync_atts(file,(NC_OBJ*)var, var->att))) + if((stat = ncz_sync_atts(file,(NC_OBJ*)var, var->att, isclose))) goto done; - /* flush only chunks that have been written */ - if(zvar->cache) { - if((stat = NCZ_flush_chunk_cache(zvar->cache))) - goto done; - } - done: nclistfreeall(dimrefs); nullfree(fullpath); @@ -481,6 +483,37 @@ ncz_sync_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var) return THROW(stat); } +/** + * @internal Synchronize variable meta data and data from memory to map. + * + * @param file Pointer to file struct + * @param var Pointer to var struct + * @param isclose If this called as part of nc_close() as opposed to nc_enddef(). + * + * @return ::NC_NOERR No error. + * @author Dennis Heimbigner + */ +static int +ncz_sync_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, int isclose) +{ + int stat = NC_NOERR; + NCZ_VAR_INFO_T* zvar = var->format_var_info; + + if(!isclose) { + if((stat = ncz_sync_var_meta(file,var,isclose))) goto done; + } + + /* flush only chunks that have been written */ + if(zvar->cache) { + if((stat = NCZ_flush_chunk_cache(zvar->cache))) + goto done; + } + +done: + return THROW(stat); +} + + /* Flush all chunks to disk. Create any that are missing and fill as needed. @@ -535,17 +568,7 @@ ncz_write_var(NC_VAR_INFO_T* var) default: goto done; /* some other error */ } /* If we reach here, then chunk does not exist, create it with fill */ - /* ensure fillchunk exists */ - if(zvar->cache->fillchunk == NULL) { - nc_type typecode; - size_t typesize; - void* fillvalue = NULL; - typecode = var->type_info->hdr.id; - if((stat = NC4_inq_atomic_type(typecode, NULL, &typesize))) goto done; - if((stat = ncz_get_fill_value(file, var, &fillvalue))) goto done; - if((stat = NCZ_create_fill_chunk(zvar->cache->chunksize, typesize, fillvalue, &zvar->cache->fillchunk))) - goto done; - } + assert(zvar->cache->fillchunk != NULL); if((stat=nczmap_write(map,key,0,zvar->cache->chunksize,zvar->cache->fillchunk))) goto done; next: nullfree(key); @@ -570,7 +593,7 @@ ncz_write_var(NC_VAR_INFO_T* var) * @author Dennis Heimbigner */ int -ncz_sync_atts(NC_FILE_INFO_T* file, NC_OBJ* container, NCindex* attlist) +ncz_sync_atts(NC_FILE_INFO_T* file, NC_OBJ* container, NCindex* attlist, int isclose) { int i,stat = NC_NOERR; NCZ_FILE_INFO_T* zinfo = NULL; @@ -883,7 +906,7 @@ computeattrinfo(const char* name, NClist* atypes, NCjson* values, int stat = NC_NOERR; int i; size_t len, typelen; - void* data; + void* data = NULL; nc_type typeid; /* Get type info for the given att */ @@ -927,7 +950,7 @@ computeattrdata(nc_type* typeidp, NCjson* values, size_t* typelenp, size_t* lenp /* Get assumed type */ if(typeidp) typeid = *typeidp; - if(typeid == NC_NAT) inferattrtype(values,&typeid); + if(typeid == NC_NAT) if((stat = inferattrtype(values,&typeid))) goto done; if(typeid == NC_NAT) {stat = NC_EBADTYPE; goto done;} /* Collect the length of the attribute; might be a singleton */ @@ -936,10 +959,11 @@ computeattrdata(nc_type* typeidp, NCjson* values, size_t* typelenp, size_t* lenp case NCJ_ARRAY: count = NCJlength(values); break; - case NCJ_STRING: /* requires special handling as an array of characters */ - if(typeid == NC_CHAR) + case NCJ_STRING: /* requires special handling as an array of characters; also look out for empty string */ + if(typeid == NC_CHAR) { count = strlen(NCJstring(values)); - else + if(count == 0) count = 1; /* Actually a single nul char, probably default fill value ugh!*/ + } else count = 1; break; default: @@ -973,20 +997,27 @@ computeattrdata(nc_type* typeidp, NCjson* values, size_t* typelenp, size_t* lenp } static int -inferattrtype(NCjson* values, nc_type* typeidp) +inferattrtype(NCjson* value, nc_type* typeidp) { + int stat = NC_NOERR; nc_type typeid; NCjson* j = NULL; unsigned long long u64; long long i64; int negative = 0; + if(NCJsort(value) == NCJ_ARRAY && NCJlength(value) == 0) + {typeid = NC_NAT; goto done;} - switch (NCJsort(values)) { - case NCJ_ARRAY: - if(NCJlength(values) == 0) return NC_EINVAL; - j = NCJith(values,0); + if(NCJsort(value) == NCJ_NULL) + {typeid = NC_NAT; goto done;} + + if(value->sort == NCJ_ARRAY) { + j=NCJith(value,0); return inferattrtype(j,typeidp); + } + + switch (NCJsort(value)) { case NCJ_NULL: typeid = NC_CHAR; return NC_NOERR; @@ -996,15 +1027,15 @@ inferattrtype(NCjson* values, nc_type* typeidp) default: /* atomic */ break; } - if(NCJstring(values)) - negative = (NCJstring(values)[0] == '-'); - switch (NCJsort(values)) { + if(NCJstring(value) != NULL) + negative = (NCJstring(value)[0] == '-'); + switch (value->sort) { case NCJ_INT: if(negative) { - sscanf(NCJstring(values),"%lld",&i64); + sscanf(NCJstring(value),"%lld",&i64); u64 = (unsigned long long)i64; } else - sscanf(NCJstring(values),"%llu",&u64); + sscanf(NCJstring(value),"%llu",&u64); typeid = mininttype(u64,negative); break; case NCJ_DOUBLE: @@ -1017,10 +1048,11 @@ inferattrtype(NCjson* values, nc_type* typeidp) typeid = NC_CHAR; break; default: - return NC_ENCZARR; + stat = NC_ENCZARR; } +done: if(typeidp) *typeidp = typeid; - return NC_NOERR; + return stat; } static int @@ -1192,7 +1224,7 @@ ncz_read_atts(NC_FILE_INFO_T* file, NC_OBJ* container) NCjson* jattrs = NULL; NClist* atypes = NULL; nc_type typeid; - size_t len; + size_t len, typelen; void* data = NULL; NC_ATT_INFO_T* fillvalueatt = NULL; @@ -1255,10 +1287,11 @@ ncz_read_atts(NC_FILE_INFO_T* file, NC_OBJ* container) /* Create the attribute */ /* Collect the attribute's type and value */ if((stat = computeattrinfo(NCJstring(key),atypes,value, - &typeid,NULL,&len,&data))) + &typeid,&typelen,&len,&data))) goto done; if((stat = ncz_makeattr(container,attlist,NCJstring(key),typeid,len,data,&att))) goto done; + nullfree(data); data = NULL; /* passed to the attribute */ /* Is this _FillValue ? */ if(strcmp(att->hdr.name,_FillValue)==0) fillvalueatt = att; } @@ -1279,6 +1312,7 @@ ncz_read_atts(NC_FILE_INFO_T* file, NC_OBJ* container) NCJreclaim(jattrs); nclistfreeall(atypes); nullfree(fullpath); + nullfree(data); nullfree(key); return THROW(stat); } @@ -1345,6 +1379,7 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) NCjson* jncvar = NULL; NCjson* jdimrefs = NULL; NCjson* jvalue = NULL; + NCjson* jfilter = NULL; int purezarr = 0; int xarray = 0; int formatv1 = 0; @@ -1376,6 +1411,7 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) zvar->common.file = file; /* Set filter list */ + assert(var->filters == NULL); var->filters = (void*)nclistnew(); /* Construct var path */ @@ -1389,7 +1425,7 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) if((stat=NCZ_readdict(map,key,&jvar))) goto done; nullfree(key); key = NULL; - assert((NCJsort(jvar) == NCJ_DICT)); + assert(NCJsort(jvar) == NCJ_DICT); /* Extract the .zarray info from jvar */ @@ -1450,6 +1486,23 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) zvar->dimension_separator = ngs->zarr.dimension_separator; /* use global value */ assert(islegaldimsep(zvar->dimension_separator)); /* we are hosed */ } + /* fill_value; must precede calls to adjust cache */ + { + if((stat = NCJdictget(jvar,"fill_value",&jvalue))) goto done; + if(jvalue == NULL || NCJsort(jvalue) == NCJ_NULL) + var->no_fill = 1; + else { + size_t fvlen; + typeid = var->type_info->hdr.id; + var->no_fill = 0; + if((stat = computeattrdata(&typeid, jvalue, NULL, &fvlen, &var->fill_value))) + goto done; + assert(typeid == var->type_info->hdr.id); + /* Note that we do not create the _FillValue + attribute here to avoid having to read all + the attributes and thus foiling lazy read.*/ + } + } /* chunks */ { int rank; @@ -1478,25 +1531,9 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) /* Create the cache */ if((stat = NCZ_create_chunk_cache(var,var->type_info->size*zvar->chunkproduct,zvar->dimension_separator,&zvar->cache))) goto done; + if((stat = NCZ_adjust_var_cache(var))) goto done; } } - /* fill_value */ - { - if((stat = NCJdictget(jvar,"fill_value",&jvalue))) goto done; - if(jvalue == NULL || NCJsort(jvalue) == NCJ_NULL) - var->no_fill = 1; - else { - size_t fvlen; - typeid = var->type_info->hdr.id; - var->no_fill = 0; - if((stat = computeattrdata(&typeid, jvalue, NULL, &fvlen, &var->fill_value))) - goto done; - assert(typeid == var->type_info->hdr.id); - /* Note that we do not create the _FillValue - attribute here to avoid having to read all - the attributes and thus foiling lazy read.*/ - } - } /* Capture row vs column major; currently, column major not used*/ { if((stat = NCJdictget(jvar,"order",&jvalue))) goto done; @@ -1504,15 +1541,39 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) ((NCZ_VAR_INFO_T*)var->format_var_info)->order = 1; else ((NCZ_VAR_INFO_T*)var->format_var_info)->order = 0; } - /* compressor ignored */ + /* filters key */ + /* From V2 Spec: A list of JSON objects providing codec configurations, + or null if no filters are to be applied. Each codec configuration + object MUST contain a "id" key identifying the codec to be used. */ + /* Do filters key before compressor key so final filter chain is in correct order */ { - if((stat = NCJdictget(jvar,"compressor",&jvalue))) goto done; - /* ignore */ + int k; + if(var->filters == NULL) var->filters = (void*)nclistnew(); + if((stat = NCZ_filter_initialize())) goto done; + if((stat = NCJdictget(jvar,"filters",&jvalue))) goto done; + if(jvalue != NULL && NCJsort(jvalue) != NCJ_NULL) { + if(NCJsort(jvalue) != NCJ_ARRAY) {stat = NC_EFILTER; goto done;} + for(k=0;;k++) { + jfilter = NULL; + jfilter = NCJith(jvalue,k); + if(jfilter == NULL) break; /* done */ + if(NCJsort(jfilter) != NCJ_DICT) {stat = NC_EFILTER; goto done;} + if((stat = NCZ_filter_build(file,var,jfilter))) goto done; + } + } } - /* filters ignored */ + + /* compressor key */ + /* From V2 Spec: A JSON object identifying the primary compression codec and providing + configuration parameters, or ``null`` if no compressor is to be used. */ { - if((stat = NCJdictget(jvar,"filters",&jvalue))) goto done; - /* ignore */ + if(var->filters == NULL) var->filters = (void*)nclistnew(); + if((stat = NCZ_filter_initialize())) goto done; + if((stat = NCJdictget(jvar,"compressor",&jfilter))) goto done; + if(jfilter != NULL && NCJsort(jfilter) != NCJ_NULL) { + if(NCJsort(jfilter) != NCJ_DICT) {stat = NC_EFILTER; goto done;} + if((stat = NCZ_filter_build(file,var,jfilter))) goto done; + } } if(!purezarr) { @@ -1572,11 +1633,15 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) for(j=0;jdimids[j] = var->dim[j]->hdr.id; + /* At this point, we can finalize the filters */ + if((stat = NCZ_filter_setup(var))) goto done; + /* Clean up from last cycle */ nclistfreeall(dimnames); dimnames = nclistnew(); nullfree(varpath); varpath = NULL; nullfree(shapes); shapes = NULL; - NCJreclaim(jvar); jvar = NULL; + if(formatv1) {NCJreclaim(jncvar); jncvar = NULL;} + NCJreclaim(jvar); jvar = NULL; } done: @@ -1882,7 +1947,7 @@ decodeints(NCjson* jshape, size64_t* shapes) for(i=0;icontainer, var))) + if ((retval = nc4_adjust_var_cache(var))) BAIL(retval); if (var->coords_read && !var->dimscale) @@ -2134,7 +2199,7 @@ computedimrefs(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, int purezarr, int xarra for(i=0;isort != NCJ_DICT) {stat = NC_ENCZARR; goto done;} + if(NCJsort(json) != NCJ_DICT) {stat = NC_ENCZARR; goto done;} if(jsonp) {*jsonp = json; json = NULL;} done: NCJreclaim(json); @@ -364,7 +364,7 @@ NCZ_readarray(NCZMAP* zmap, const char* key, NCjson** jsonp) if((stat = NCZ_downloadjson(zmap,key,&json))) goto done; - if(json->sort != NCJ_ARRAY) {stat = NC_ENCZARR; goto done;} + if(NCJsort(json) != NCJ_ARRAY) {stat = NC_ENCZARR; goto done;} if(jsonp) {*jsonp = json; json = NULL;} done: NCJreclaim(json); @@ -414,9 +414,9 @@ ncz_default_fill_value(nc_type nctype, const char** dfaltp) /** @internal Given an nc_type, produce the corresponding -fill value sort +fill value JSON type @param nctype - [in] nc_type -@param sortp - [out] pointer to hold pointer to the sort +@param sortp - [out] pointer to hold pointer to the JSON type @return NC_NOERR @author Dennis Heimbigner */ @@ -732,12 +732,17 @@ NCZ_freestringvec(size_t len, char** vec) /* create a fill chunk */ int -NCZ_create_fill_chunk(size64_t chunksize, size_t typesize, void* fill, void** fillchunkp) +NCZ_create_fill_chunk(size64_t chunksize, size_t typesize, const void* fill, void** fillchunkp) { int i; void* fillchunk = NULL; if((fillchunk = malloc(chunksize))==NULL) return NC_ENOMEM; + if(fill == NULL) { + /* use zeros */ + memset(fillchunk,0,chunksize); + goto done; + } switch (typesize) { case 1: { unsigned char c = *((unsigned char*)fill); @@ -764,6 +769,7 @@ NCZ_create_fill_chunk(size64_t chunksize, size_t typesize, void* fill, void** fi memcpy(p,fill,typesize); } break; } +done: if(fillchunkp) {*fillchunkp = fillchunk; fillchunk = NULL;} nullfree(fillchunk); return NC_NOERR; @@ -901,7 +907,7 @@ NCZ_ischunkname(const char* name,char dimsep) } char* -NCZ_chunkpath(struct ChunkKey key,char dimsep) +NCZ_chunkpath(struct ChunkKey key) { size_t plen = nulllen(key.varkey)+1+nulllen(key.chunkkey); char* path = (char*)malloc(plen+1); diff --git a/libnczarr/zvar.c b/libnczarr/zvar.c index 7ba84f9683..6813dd5e72 100644 --- a/libnczarr/zvar.c +++ b/libnczarr/zvar.c @@ -390,6 +390,7 @@ NCZ_def_var(int ncid, const char *name, nc_type xtype, int ndims, var->atts_read = NC_TRUE; /* Set the filter list */ + assert(var->filters == NULL); var->filters = (void*)nclistnew(); /* Point to the type, and increment its ref. count */ @@ -461,10 +462,6 @@ var->type_info->rc++; if((retval=NCZ_create_chunk_cache(var,zvar->chunkproduct*var->type_info->size,zvar->dimension_separator,&zvar->cache))) BAIL(retval); - /* Is this a variable with a chunksize greater than the current cache size? */ - if ((retval = NCZ_adjust_var_cache(grp, var))) - BAIL(retval); - /* Return the varid. */ if (varidp) *varidp = var->hdr.id; @@ -599,6 +596,9 @@ ncz_def_var_extra(int ncid, int varid, int *shuffle, int *unused1, { if(*shuffle) var->shuffle = *shuffle; var->storage = NC_CHUNKED; + if(var->shuffle) { + if((retval = NCZ_def_var_filter(ncid,varid,2,0,NULL))) goto done; + } } /* Fletcher32 checksum error protection? */ @@ -606,6 +606,9 @@ ncz_def_var_extra(int ncid, int varid, int *shuffle, int *unused1, { if(*fletcher32) var->fletcher32 = *fletcher32; var->storage = NC_CHUNKED; + if(var->fletcher32) { + if((retval = NCZ_def_var_filter(ncid,varid,3,0,NULL))) goto done; + } } /* Handle storage settings. */ @@ -675,9 +678,6 @@ ncz_def_var_extra(int ncid, int varid, int *shuffle, int *unused1, for (d = 0; d < var->ndims; d++) zvar->chunkproduct *= var->chunksizes[d]; zvar->chunksize = zvar->chunkproduct * var->type_info->size; - /* Adjust the cache. */ - if ((retval = NCZ_adjust_var_cache(grp, var))) - goto done; } #ifdef LOGGING @@ -2002,10 +2002,22 @@ NCZ_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep, goto done; assert(grp && h5); + /* Short-circuit the filter-related inquiries */ + if(shufflep) { + *shufflep = 0; + if((retval = NCZ_inq_var_filter_info(ncid,varid,2,NULL,NULL))==NC_NOERR) + *shufflep = 1; + } + if(fletcher32p) { + *fletcher32p = 0; + if((retval = NCZ_inq_var_filter_info(ncid,varid,3,NULL,NULL))==NC_NOERR) + *fletcher32p = 1; + } + /* Now that lazy atts have been read, use the libsrc4 function to * get the answers. */ retval = NC4_inq_var_all(ncid, varid, name, xtypep, ndimsp, dimidsp, nattsp, - shufflep, unused4, unused5, fletcher32p, + NULL, unused4, unused5, NULL, storagep, chunksizesp, no_fill, fill_valuep, endiannessp, unused1, unused2, unused3); done: diff --git a/libnczarr/zwalk.c b/libnczarr/zwalk.c index 3374b2509c..635925ab36 100644 --- a/libnczarr/zwalk.c +++ b/libnczarr/zwalk.c @@ -25,7 +25,6 @@ static unsigned int wdebug = 0; static int NCZ_walk(NCZProjection** projv, NCZOdometer* chunkodom, NCZOdometer* slpodom, NCZOdometer* memodom, const struct Common* common, void* chunkdata); static int rangecount(NCZChunkRange range); static int readfromcache(void* source, size64_t* chunkindices, void** chunkdata); -static int NCZ_fillchunk(void* chunkdata, struct Common* common); static int iswholechunk(struct Common* common,NCZSlice*); static int wholechunk_indices(struct Common* common, NCZSlice* slices, size64_t* chunkindices); @@ -123,8 +122,6 @@ NCZ_transferslice(NC_VAR_INFO_T* var, int reading, common.typesize = typesize; common.cache = zvar->cache; - if((stat = ncz_get_fill_value(common.file, common.var, &common.fillvalue))) goto done; - /* We need to talk scalar into account */ common.rank = var->ndims + zvar->scalar; common.scalar = zvar->scalar; @@ -160,6 +157,9 @@ NCZ_transferslice(NC_VAR_INFO_T* var, int reading, common.reader.source = ((NCZ_VAR_INFO_T*)(var->format_var_info))->cache; common.reader.read = readfromcache; + /* verify */ + assert(var->no_fill || var->fill_value != NULL); + if(common.scalar) { if((stat = NCZ_transferscalar(&common))) goto done; } @@ -227,7 +227,6 @@ NCZ_transfer(struct Common* common, NCZSlice* slices) /* Read the chunk */ switch ((stat = common->reader.read(common->reader.source, chunkindices, &chunkdata))) { case NC_EEMPTY: /* cache created the chunk */ - if((stat = NCZ_fillchunk(chunkdata,common))) goto done; break; case NC_NOERR: break; default: goto done; @@ -299,7 +298,6 @@ NCZ_transfer(struct Common* common, NCZSlice* slices) stat = common->reader.read(common->reader.source, chunkindices, &chunkdata); switch (stat) { case NC_EEMPTY: /* cache created the chunk */ - if((stat = NCZ_fillchunk(chunkdata,common))) goto done; break; case NC_NOERR: break; default: goto done; @@ -503,6 +501,7 @@ unsigned srcidx = srcoff / sizeof(unsigned); (void)srcidx; } #endif +#if 0 /* This function may not be necessary if code in zvar does it instead */ static int NCZ_fillchunk(void* chunkdata, struct Common* common) @@ -523,6 +522,7 @@ NCZ_fillchunk(void* chunkdata, struct Common* common) done: return stat; } +#endif /* Break out this piece so we can use it for unit testing */ int @@ -681,7 +681,6 @@ NCZ_clearcommon(struct Common* common) { NCZ_clearsliceprojections(common->rank,common->allprojections); nullfree(common->allprojections); - nullfree(common->fillvalue); } /* Does the User want all of one and only chunk? */ @@ -730,7 +729,6 @@ NCZ_transferscalar(struct Common* common) chunkindices[0] = 0; switch ((stat = common->reader.read(common->reader.source, chunkindices, &chunkdata))) { case NC_EEMPTY: /* cache created the chunk */ - if((stat = NCZ_fillchunk(chunkdata,common))) goto done; break; case NC_NOERR: break; default: goto done; diff --git a/libnczarr/zxcache.c b/libnczarr/zxcache.c index 740c73f53a..e8602af477 100644 --- a/libnczarr/zxcache.c +++ b/libnczarr/zxcache.c @@ -13,20 +13,20 @@ #include "zincludes.h" #include "zcache.h" #include "ncxcache.h" +#include "zfilter.h" #undef DEBUG -#undef FILLONREAD - #undef FLUSH #define LEAFLEN 32 - /* Forward */ static int get_chunk(NCZChunkCache* cache, NCZCacheEntry* entry); -static int put_chunk(NCZChunkCache* cache, const NCZCacheEntry*); +static int put_chunk(NCZChunkCache* cache, NCZCacheEntry*); static int makeroom(NCZChunkCache* cache); +static int flushcache(NCZChunkCache* cache); +static int constraincache(NCZChunkCache* cache); /**************************************************/ /* Dispatch table per-var cache functions */ @@ -57,20 +57,20 @@ NCZ_set_var_chunk_cache(int ncid, int varid, size_t cachesize, size_t nelems, fl NC_FILE_INFO_T *h5; NC_VAR_INFO_T *var; NCZ_VAR_INFO_T *zvar; - int retval; + int retval = NC_NOERR; /* Check input for validity. */ if (preemption < 0 || preemption > 1) - return NC_EINVAL; + {retval = NC_EINVAL; goto done;} /* Find info for this file and group, and set pointer to each. */ if ((retval = nc4_find_nc_grp_h5(ncid, NULL, &grp, &h5))) - return retval; + goto done; assert(grp && h5); /* Find the var. */ if (!(var = (NC_VAR_INFO_T *)ncindexith(grp->vars, varid))) - return NC_ENOTVAR; + {retval = NC_ENOTVAR; goto done;} assert(var && var->hdr.id == varid); zvar = (NCZ_VAR_INFO_T*)var->format_var_info; @@ -81,12 +81,10 @@ NCZ_set_var_chunk_cache(int ncid, int varid, size_t cachesize, size_t nelems, fl var->chunk_cache_nelems = nelems; var->chunk_cache_preemption = preemption; -#ifdef LOOK - /* Reopen the dataset to bring new settings into effect. */ - if ((retval = nc4_reopen_dataset(grp, var))) - return retval; -#endif - return NC_NOERR; + /* Fix up cache */ + if((retval = NCZ_adjust_var_cache(var))) goto done; +done: + return retval; } /** @@ -103,31 +101,36 @@ NCZ_set_var_chunk_cache(int ncid, int varid, size_t cachesize, size_t nelems, fl * @author Ed Hartnett */ int -NCZ_adjust_var_cache(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var) +NCZ_adjust_var_cache(NC_VAR_INFO_T *var) { - size64_t cachesize,nelems; - + int stat = NC_NOERR; NCZ_VAR_INFO_T* zvar = (NCZ_VAR_INFO_T*)var->format_var_info; - /* empty the cache */ - zvar->cache->maxentries = 0; - makeroom(zvar->cache); + /* completely empty the cache */ + flushcache(zvar->cache); + +#ifdef DEBUG +fprintf(stderr,"xxx: adjusting cache for: %s\n",var->hdr.name); +#endif /* Reset the parameters */ - /* The total cache size is considered fixed here, so modify nelems */ - cachesize = var->chunk_cache_size; - nelems = floordiv(cachesize , zvar->chunksize); - if(nelems == 0) nelems = 1; - zvar->cache->maxentries = nelems; + zvar->cache->maxsize = var->chunk_cache_size; + zvar->cache->maxentries = var->chunk_cache_nelems; #ifdef DEBUG fprintf(stderr,"%s.cache.adjust: size=%ld nelems=%ld\n", - var->hdr.name,(unsigned long)cachesize,(unsigned long)zvar->cache->maxentries); + var->hdr.name,(unsigned long)zvar->cache->maxsize,(unsigned long)zvar->cache->maxentries); #endif /* One more thing, adjust the chunksize */ zvar->cache->chunksize = zvar->chunksize; - /* and also free the fillchunk */ + /* and also rebuild the fillchunk */ nullfree(zvar->cache->fillchunk); zvar->cache->fillchunk = NULL; - return NC_NOERR; + if(var->no_fill) + stat = NCZ_create_fill_chunk(zvar->cache->chunksize,var->type_info->size,NULL,&zvar->cache->fillchunk); + else { + assert(var->fill_value != NULL); + stat = NCZ_create_fill_chunk(zvar->cache->chunksize,var->type_info->size,var->fill_value,&zvar->cache->fillchunk); + } + return stat; } /**************************************************/ @@ -149,7 +152,6 @@ NCZ_create_chunk_cache(NC_VAR_INFO_T* var, size64_t chunksize, char dimsep, NCZC int stat = NC_NOERR; NCZChunkCache* cache = NULL; void* fill = NULL; - size_t nelems, cachesize; NCZ_VAR_INFO_T* zvar = NULL; if(chunksize == 0) return NC_EINVAL; @@ -164,21 +166,15 @@ NCZ_create_chunk_cache(NC_VAR_INFO_T* var, size64_t chunksize, char dimsep, NCZC cache->fillchunk = NULL; cache->chunksize = chunksize; cache->dimension_separator = dimsep; + zvar->cache = cache; - /* Figure out the actual cache size */ - cachesize = var->chunk_cache_size; - nelems = (cachesize / chunksize); - if(nelems == 0) nelems = 1; - /* Make consistent */ - cachesize = nelems * chunksize; - cache->maxentries = nelems; #ifdef FLUSH cache->maxentries = 1; #endif #ifdef DEBUG fprintf(stderr,"%s.cache: nelems=%ld size=%ld\n", - var->hdr.name,(unsigned long)cache->maxentries,(unsigned long)(cache->maxentries*cache->chunksize)); + var->hdr.name,(unsigned long)cache->maxentries,(unsigned long)cache->maxsize); #endif if((stat = ncxcachenew(LEAFLEN,&cache->xcache))) goto done; if((cache->mru = nclistnew()) == NULL) @@ -191,6 +187,17 @@ NCZ_create_chunk_cache(NC_VAR_INFO_T* var, size64_t chunksize, char dimsep, NCZC return THROW(stat); } +static void +free_cache_entry(NCZCacheEntry* entry) +{ + if(entry) { + nullfree(entry->data); + nullfree(entry->key.varkey); + nullfree(entry->key.chunkkey); + nullfree(entry); + } +} + void NCZ_free_chunk_cache(NCZChunkCache* cache) { @@ -204,7 +211,7 @@ NCZ_free_chunk_cache(NCZChunkCache* cache) NCZCacheEntry* entry = nclistremove(cache->mru,0); (void)ncxcacheremove(cache->xcache,entry->hashkey,&ptr); assert(ptr == entry); - nullfree(entry->data); nullfree(entry->key.varkey); nullfree(entry->key.chunkkey); nullfree(entry); + free_cache_entry(entry); } #ifdef DEBUG fprintf(stderr,"|cache.free|=%ld\n",nclistlength(cache->mru)); @@ -237,11 +244,10 @@ NCZ_read_cache_chunk(NCZChunkCache* cache, const size64_t* indices, void** datap { int stat = NC_NOERR; int rank = cache->ndims; - NC_FILE_INFO_T* file = cache->var->container->nc4_info; NCZCacheEntry* entry = NULL; ncexhashkey_t hkey = 0; int created = 0; - + /* the hash key */ hkey = ncxcachekey(indices,sizeof(size64_t)*cache->ndims); /* See if already in cache */ @@ -258,38 +264,22 @@ NCZ_read_cache_chunk(NCZChunkCache* cache, const size64_t* indices, void** datap } if(entry == NULL) { /*!found*/ - /* Make room in the cache */ - if((stat=makeroom(cache))) goto done; /* Create a new entry */ if((entry = calloc(1,sizeof(NCZCacheEntry)))==NULL) {stat = NC_ENOMEM; goto done;} memcpy(entry->indices,indices,rank*sizeof(size64_t)); - /* Create the local copy space */ - if((entry->data = calloc(1,cache->chunksize)) == NULL) - {stat = NC_ENOMEM; goto done;} /* Create the key for this cache */ if((stat = NCZ_buildchunkpath(cache,indices,&entry->key))) goto done; entry->hashkey = hkey; - /* Try to read the object in toto */ - stat=get_chunk(cache,entry); - switch (stat) { - case NC_NOERR: break; - case NC_EEMPTY: - /* If the file is read-only, then fake the chunk */ - entry->modified = (file->no_write?0:1); -#ifdef FILLONREAD - /* apply fill value */ - memcpy(entry->data,cache->fillchunk,cache->chunksize); -#else - memset(entry->data,0,cache->chunksize); -#endif - created = 1; - break; - default: goto done; - } + /* Try to read the object from "disk" */ + if((stat=get_chunk(cache,entry))) goto done; nclistpush(cache->mru,entry); + cache->used += entry->size; if((stat = ncxcacheinsert(cache->xcache,entry->hashkey,entry))) goto done; + /* Ensure cache constraints not violated */ + if((stat=makeroom(cache))) goto done; } + #ifdef DEBUG fprintf(stderr,"|cache.read.lru|=%ld\n",nclistlength(cache->mru)); #endif @@ -298,33 +288,34 @@ fprintf(stderr,"|cache.read.lru|=%ld\n",nclistlength(cache->mru)); done: if(created && stat == NC_NOERR) stat = NC_EEMPTY; /* tell upper layers */ - if(entry) {nullfree(entry->data); nullfree(entry->key.varkey); nullfree(entry->key.chunkkey);} - nullfree(entry); + if(entry) free_cache_entry(entry); return THROW(stat); } +#if 0 int -NCZ_write_cache_chunk(NCZChunkCache* cache, const size64_t* indices, void** datap) +NCZ_write_cache_chunk(NCZChunkCache* cache, const size64_t* indices, void* content) { int stat = NC_NOERR; int rank = cache->ndims; NCZCacheEntry* entry = NULL; ncexhashkey_t hkey; - /* and the hash key */ + /* create the hash key */ hkey = ncxcachekey(indices,sizeof(size64_t)*cache->ndims); if(entry == NULL) { /*!found*/ - if((stat=makeroom(cache))) goto done; /* Create a new entry */ if((entry = calloc(1,sizeof(NCZCacheEntry)))==NULL) {stat = NC_ENOMEM; goto done;} memcpy(entry->indices,indices,rank*sizeof(size64_t)); + if((stat = NCZ_buildchunkpath(cache,indices,&entry->key))) goto done; + entry->hashkey = hkey; /* Create the local copy space */ + entry->size = cache->chunksize; if((entry->data = calloc(1,cache->chunksize)) == NULL) {stat = NC_ENOMEM; goto done;} - if((stat = NCZ_buildchunkpath(cache,indices,&entry->key))) goto done; - entry->hashkey = hkey; + memcpy(entry->data,content,cache->chunksize); } entry->modified = 1; nclistpush(cache->mru,entry); /* MRU order */ @@ -333,18 +324,51 @@ fprintf(stderr,"|cache.write|=%ld\n",nclistlength(cache->mru)); #endif entry = NULL; + /* Ensure cache constraints not violated */ + if((stat=makeroom(cache))) goto done; + done: - if(entry) {nullfree(entry->data); nullfree(entry->key.varkey); nullfree(entry->key.chunkkey);} - nullfree(entry); + if(entry) free_cache_entry(entry); return THROW(stat); } +#endif +/* Constrain cache, but allow at least one entry */ static int makeroom(NCZChunkCache* cache) { int stat = NC_NOERR; + + /* Sanity check; make sure at least one entry is always allowed */ + if(nclistlength(cache->mru) == 1) + goto done; + stat = constraincache(cache); +done: + return stat; +} + +/* Completely flush cache */ + +static int +flushcache(NCZChunkCache* cache) +{ + cache->maxentries = 0; + return constraincache(cache); +} + + +/* Remove entries to ensure cache is not + violating any of its constraints. + On entry, constraints might be violated. +*/ + +static int +constraincache(NCZChunkCache* cache) +{ + int stat = NC_NOERR; + /* Flush from LRU end if we are at capacity */ - while(nclistlength(cache->mru) > cache->maxentries) { + while(nclistlength(cache->mru) > cache->maxentries || cache->used > cache->maxsize) { int i; void* ptr; NCZCacheEntry* e = ncxcachelast(cache->xcache); /* last entry is the least recently used */ @@ -359,6 +383,9 @@ makeroom(NCZChunkCache* cache) nclistremove(cache->mru,i); if(e->modified) /* flush to file */ stat=put_chunk(cache,e); + /* Decrement space used */ + assert(cache->used >= e->size); + cache->used -= e->size; /* reclaim */ nullfree(e->data); nullfree(e->key.varkey); nullfree(e->key.chunkkey); nullfree(e); } @@ -478,23 +505,43 @@ NCZ_buildchunkkey(size_t R, const size64_t* chunkindices, char dimsep, char** ke * @author Dennis Heimbigner */ static int -put_chunk(NCZChunkCache* cache, const NCZCacheEntry* entry) +put_chunk(NCZChunkCache* cache, NCZCacheEntry* entry) { int stat = NC_NOERR; + NC_FILE_INFO_T* file = NULL; NCZ_FILE_INFO_T* zfile = NULL; NCZMAP* map = NULL; + char* path = NULL; ZTRACE(5,"cache.var=%s entry.key=%s",cache->var->hdr.name,entry->key); LOG((3, "%s: var: %p", __func__, cache->var)); - zfile = ((cache->var->container)->nc4_info)->format_file_info; + file = (cache->var->container)->nc4_info; + zfile = file->format_file_info; map = zfile->map; - { - char* path = NCZ_chunkpath(entry->key,cache->dimension_separator); - stat = nczmap_write(map,path,0,cache->chunksize,entry->data); - nullfree(path); + /* Make sure the entry is in filtered state */ + if(!entry->isfiltered) { + NC_VAR_INFO_T* var = cache->var; + void* filtered = NULL; /* pointer to the filtered data */ + size_t flen; /* length of filtered data */ + /* Get the filter chain to apply */ + NClist* filterchain = (NClist*)var->filters; + if(nclistlength(filterchain) > 0) { + /* Apply the filter chain to get the filtered data */ + if((stat = NCZ_applyfilterchain(file,var,filterchain,entry->size,entry->data,&flen,&filtered,ENCODING))) goto done; + /* Fix up the cache entry */ + /* Note that if filtered is different from entry->data, then entry->data will have been freed */ + entry->data = filtered; + entry->size = flen; + entry->isfiltered = 1; + } } + + path = NCZ_chunkpath(entry->key); + stat = nczmap_write(map,path,0,entry->size,entry->data); + nullfree(path); path = NULL; + switch(stat) { case NC_NOERR: break; @@ -502,6 +549,7 @@ put_chunk(NCZChunkCache* cache, const NCZCacheEntry* entry) default: goto done; } done: + nullfree(path); return ZUNTRACE(stat); } @@ -522,6 +570,9 @@ get_chunk(NCZChunkCache* cache, NCZCacheEntry* entry) NCZMAP* map = NULL; NC_FILE_INFO_T* file = NULL; NCZ_FILE_INFO_T* zfile = NULL; + size64_t size; + int empty = 0; + char* path = NULL; ZTRACE(5,"cache.var=%s entry.key=%s sep=%d",cache->var->hdr.name,entry->key,cache->dimension_separator); @@ -530,14 +581,67 @@ get_chunk(NCZChunkCache* cache, NCZCacheEntry* entry) file = (cache->var->container)->nc4_info; zfile = file->format_file_info; map = zfile->map; - assert(map && entry->data); + assert(map); - { - char* path = NCZ_chunkpath(entry->key,cache->dimension_separator); - stat = nczmap_read(map,path,0,cache->chunksize,(char*)entry->data); - nullfree(path); + /* get size of the "raw" data on "disk" */ + path = NCZ_chunkpath(entry->key); + stat = nczmap_len(map,path,&size); + nullfree(path); path = NULL; + switch(stat) { + case NC_NOERR: break; + case NC_EEMPTY: empty = 1; stat = NC_NOERR; break; + default: goto done; } + if(!empty) { + /* Make sure we have a place to read it */ + entry->size = size; + entry->isfiltered = FILTERED(cache); /* Is the data being read filtered? */ + if((entry->data = (void*)malloc(entry->size)) == NULL) + {stat = NC_ENOMEM; goto done;} + /* Read the raw data */ + path = NCZ_chunkpath(entry->key); + stat = nczmap_read(map,path,0,entry->size,(char*)entry->data); + nullfree(path); path = NULL; + switch (stat) { + case NC_NOERR: break; + case NC_EEMPTY: empty = 1; stat = NC_NOERR;break; + default: goto done; + } + } + if(empty) { + /* fake the chunk */ + entry->modified = (file->no_write?0:1); + entry->size = cache->chunksize; + if((entry->data = (void*)malloc(entry->size)) == NULL) + {stat = NC_ENOMEM; goto done;} + /* apply fill value */ + assert(cache->fillchunk); + memcpy(entry->data,cache->fillchunk,entry->size); + entry->isfiltered = 0; + stat = NC_NOERR; + } + /* Make sure the entry is in unfiltered state */ + if(entry->isfiltered) { + NC_VAR_INFO_T* var = cache->var; + void* unfiltered = NULL; /* pointer to the unfiltered data */ + void* filtered = NULL; /* pointer to the filtered data */ + size_t unflen; /* length of unfiltered data */ + /* Get the filter chain to apply */ + NClist* filterchain = (NClist*)var->filters; + if(nclistlength(filterchain) == 0) {stat = NC_EFILTER; goto done;} + /* Apply the filter chain to get the unfiltered data */ + filtered = entry->data; + entry->data = NULL; + if((stat = NCZ_applyfilterchain(file,var,filterchain,entry->size,filtered,&unflen,&unfiltered,!ENCODING))) goto done; + /* Fix up the cache entry */ + entry->data = unfiltered; + entry->size = unflen; + entry->isfiltered = 0; + } + +done: + nullfree(path); return ZUNTRACE(stat); } @@ -561,4 +665,3 @@ NCZ_buildchunkpath(NCZChunkCache* cache, const size64_t* chunkindices, struct Ch nullfree(varkey); return THROW(stat); } - diff --git a/libsrc4/nc4internal.c b/libsrc4/nc4internal.c index cd5351d5f3..c7d85a637b 100644 --- a/libsrc4/nc4internal.c +++ b/libsrc4/nc4internal.c @@ -40,6 +40,7 @@ static const NC_reservedatt NC_reserved[] = { {NC_ATT_NAME, READONLYFLAG|HIDDENATTRFLAG}, /*NAME*/ {NC_ATT_REFERENCE_LIST, READONLYFLAG|HIDDENATTRFLAG}, /*REFERENCE_LIST*/ {NC_XARRAY_DIMS, READONLYFLAG|HIDDENATTRFLAG}, /*_ARRAY_DIMENSIONS*/ + {NC_ATT_CODECS, VARFLAG|READONLYFLAG|NAMEONLYFLAG|HIDDENATTRFLAG}, /*_Codecs*/ {NC_ATT_FORMAT, READONLYFLAG}, /*_Format*/ {ISNETCDF4ATT, READONLYFLAG|NAMEONLYFLAG}, /*_IsNetcdf4*/ {NCPROPS, READONLYFLAG|NAMEONLYFLAG|MATERIALIZEDFLAG}, /*_NCProperties*/ @@ -352,6 +353,7 @@ nc4_find_nc_grp_h5(int ncid, NC **nc, NC_GRP_INFO_T **grp, NC_FILE_INFO_T **h5) NC_FILE_INFO_T *my_h5 = NULL; NC *my_nc; int retval; + size_t index; /* Look up file metadata. */ if ((retval = NC_check_id(ncid, &my_nc))) @@ -360,7 +362,8 @@ nc4_find_nc_grp_h5(int ncid, NC **nc, NC_GRP_INFO_T **grp, NC_FILE_INFO_T **h5) assert(my_h5 && my_h5->root_grp); /* If we can't find it, the grp id part of ncid is bad. */ - if (!(my_grp = nclistget(my_h5->allgroups, (ncid & GRP_ID_MASK)))) + index = (ncid & GRP_ID_MASK); + if (!(my_grp = nclistget(my_h5->allgroups,index))) return NC_EBADID; /* Return pointers to caller, if desired. */ @@ -1336,7 +1339,7 @@ var_free(NC_VAR_INFO_T *var) /* Delete any fill value allocation. */ if (var->fill_value) - free(var->fill_value); + {free(var->fill_value); var->fill_value = NULL;} /* Release type information */ if (var->type_info) @@ -1837,12 +1840,13 @@ NC_findreserved(const char* name) int n = NRESERVED; int L = 0; int R = (n - 1); + for(;;) { if(L > R) break; int m = (L + R) / 2; const NC_reservedatt* p = &NC_reserved[m]; int cmp = strcmp(p->name,name); - if(cmp == 0) return p; + if(cmp == 0) return p; if(cmp < 0) L = (m + 1); else /*cmp > 0*/ diff --git a/nc_test4/CMakeLists.txt b/nc_test4/CMakeLists.txt index c5c345005b..3388c1ff41 100644 --- a/nc_test4/CMakeLists.txt +++ b/nc_test4/CMakeLists.txt @@ -8,6 +8,8 @@ # This is the cmake build file for the nc_test4 directory. # Ward Fisher, Ed Hartnett +SET(srcdir ${CMAKE_CURRENT_SOURCE_DIR}) + # Some extra tests SET(NC4_TESTS tst_dims tst_dims2 tst_dims3 tst_files tst_files4 tst_vars tst_varms tst_unlim_vars tst_converts tst_converts2 @@ -40,6 +42,9 @@ IF(ENABLE_FILTER_TESTING) build_bin_test(test_filter_order) build_bin_test(test_filter_repeat) ADD_SH_TEST(nc_test4 tst_filter) +IF(ENABLE_BLOSC) + ADD_SH_TEST(nc_test4 tst_specific_filters) +ENDIF() IF(ENABLE_CLIENTSIDE_FILTERS) add_bin_test(nc_test4 test_filter_reg) ENDIF(ENABLE_CLIENTSIDE_FILTERS) @@ -80,6 +85,7 @@ IF(USE_SZIP) ENDIF() ENDIF() + # Copy some test files from current source dir to out-of-tree build dir. FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.nc ${CMAKE_CURRENT_SOURCE_DIR}/ref_bzip2.c ${CMAKE_CURRENT_SOURCE_DIR}/*.sh ${CMAKE_CURRENT_SOURCE_DIR}/*.h5 ${CMAKE_CURRENT_SOURCE_DIR}/*.cdl) FILE(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) @@ -89,6 +95,7 @@ ENDIF() FOREACH(CTEST ${NC4_TESTS}) add_bin_test(nc_test4 ${CTEST}) + SET_TESTS_PROPERTIES(nc_test4_${CTEST} PROPERTIES ENVIRONMENT srcdir=${CMAKE_CURRENT_SOURCE_DIR}) ENDFOREACH() IF(TEST_PARALLEL4) diff --git a/nc_test4/Makefile.am b/nc_test4/Makefile.am index c356d50f08..e271254d93 100644 --- a/nc_test4/Makefile.am +++ b/nc_test4/Makefile.am @@ -78,6 +78,9 @@ extradir = check_PROGRAMS += test_filter test_filter_misc test_filter_order test_filter_repeat check_PROGRAMS += tst_multifilter TESTS += tst_filter.sh +if ENABLE_BLOSC +TESTS += tst_specific_filters.sh +endif endif endif # BUILD_UTILITIES @@ -100,10 +103,11 @@ ref_unfiltered.cdl ref_bzip2.c findplugin.in ref_unfilteredvv.cdl \ ref_filteredvv.cdl ref_multi.cdl \ ref_ncgenF.cdl ref_nccopyF.cdl \ ref_filter_repeat.txt ref_fillonly.cdl test_fillonly.sh \ -ref_filter_order_create.txt ref_filter_order_read.txt +ref_filter_order_create.txt ref_filter_order_read.txt \ +ref_any.cdl tst_specific_filters.sh CLEANFILES = tst_mpi_parallel.bin cdm_sea_soundings.nc bm_chunking.nc \ -tst_floats_1D.cdl floats_1D_3.nc floats_1D.cdl tst_*.nc \ +tst_floats_1D.cdl floats_1D_3.nc floats_1D.cdl tst_*.nc tmp_*.txt \ tst_floats2_*.cdl tst_ints2_*.cdl tst_shorts2_*.cdl tst_elena_*.cdl \ tst_simple*.cdl tst_chunks.cdl pr_A1.* tauu_A1.* usi_01.* thetau_01.* \ tst_*.h5 tst_grp_rename.cdl tst_grp_rename.dmp ref_grp_rename.cdl \ @@ -111,7 +115,8 @@ foo1.nc tst_*.h4 test.nc testszip.nc test.h5 szip_dump.cdl \ perftest.txt bigmeta.nc bigvars.nc *.gz MSGCPP_*.nc \ floats*.nc floats*.cdl shorts*.nc shorts*.cdl ints*.nc ints*.cdl \ testfilter_reg.nc filterrepeat.txt tmp_fillonly.nc \ -testfilter_order.nc crfilterorder.txt rdfilterorder.txt 1 +testfilter_order.nc crfilterorder.txt rdfilterorder.txt 1 \ +tmp_*.txt tmp_*.nc DISTCLEANFILES = findplugin.sh run_par_test.sh diff --git a/nc_test4/findplugin.in b/nc_test4/findplugin.in index 9819659686..3166b42c86 100644 --- a/nc_test4/findplugin.in +++ b/nc_test4/findplugin.in @@ -3,12 +3,14 @@ # Define a function that attempts to locate a # plugin with a given canonical name. # Assumptions: -# 1. plugins is a top-level directory +# 1. plugins is a top-level directory, possibly in "build" # Inputs: # $1 is the canonical name # $2 is 1 if we are running under cmake # $3 is 1 if we are running using Visual Studio, blank otherwise # $4 is the build type; only used if $3 is 1 +# Optional Input: +# HDF5_PLUGIN_PATH environment variable # Outputs: # return code is 0 is success, 1 if failed # Variable HDF5_PLUGIN_LIB is set to the library file name @@ -30,8 +32,6 @@ FP_ISMSVC=@ISMSVC@ FP_ISCYGWIN=@ISCYGWIN@ FP_ISOSX=@ISOSX@ -FP_PLUGINS="$TOPBUILDDIR/plugins" - FP_PLUGIN_LIB= FP_PLUGIN_PATH= @@ -47,16 +47,30 @@ else # Presumably some form on *nix" FP_PLUGIN_LIB="lib${FP_NAME}.so" fi +# If HDF5_PLUGIN_PATH is defined, then it overrides everything else. +if test "x$HDF5_PLUGIN_PATH" != x ; then + HDF5_PLUGIN_LIB="$FP_PLUGIN_LIB" + # HDF5_PLUGIN_PATH already set + HDF5_PLUGIN_PATH=`${NCPATHCVT} $HDF5_PLUGIN_PATH` + return 0; +fi + # Figure out the path to where the lib is stored # This can probably be simplified + +CURWD=`pwd` +cd ${TOPBUILDDIR}/plugins +FP_PLUGINS=`pwd` +cd ${CURWD} + # Case 1: Cmake with Visual Studio # Do not know where to look for a dylib # Case 1: Cmake with Visual Studio if test "x$FP_ISCMAKE" != x -a "x${FP_ISMSVC}" != x ; then - # Case 1a: ignore the build type directory - if test -f "${FP_PLUGINS}/${FP_PLUGIN_LIB}" ; then - FP_PLUGIN_PATH="${FP_PLUGINS}" - fi + # Case 1a: ignore the build type directory + if test -f "${FP_PLUGINS}/${FP_PLUGIN_LIB}" ; then + FP_PLUGIN_PATH="${FP_PLUGINS}" + fi else # Case 2: automake # Case 2a: look in .libs if test -f "${FP_PLUGINS}/.libs/${FP_PLUGIN_LIB}" ; then @@ -78,11 +92,7 @@ if ! test -f "$FP_PLUGIN_PATH/$FP_PLUGIN_LIB" ; then return 1 fi -# If we are operating using both Visual Studio and Cygwin, -# then we need to convert the FP_PLUGIN_PATH to windows format -if test "x$FP_ISMSVC" != x -a "x$FP_ISCYGWIN" != x ; then -FP_PLUGIN_PATH=`cygpath -wl $FP_PLUGIN_PATH` -fi +FP_PLUGIN_PATH=`${NCPATHCVT} $FP_PLUGIN_PATH` # Set the final output variables HDF5_PLUGIN_LIB="$FP_PLUGIN_LIB" diff --git a/nc_test4/ref_any.cdl b/nc_test4/ref_any.cdl new file mode 100644 index 0000000000..06457fc6d2 --- /dev/null +++ b/nc_test4/ref_any.cdl @@ -0,0 +1,53 @@ +netcdf ref_any { +dimensions: + dim0 = 4 ; + dim1 = 4 ; + dim2 = 4 ; +variables: + int ivar(dim0, dim1, dim2) ; + ivar:_Storage = @chunked@ ; + ivar:_ChunkSizes = 4, 4, 4 ; + ivar:_Filter = @IH5@ ; + float fvar(dim0, dim1, dim2) ; + fvar:_Storage = @chunked@ ; + fvar:_ChunkSizes = 4, 4, 4 ; + fvar:_Filter = @FH5@ ; + +data: + + ivar = + 0, 1, 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, + 12, 13, 14, 15, + 16, 17, 18, 19, + 20, 21, 22, 23, + 24, 25, 26, 27, + 28, 29, 30, 31, + 32, 33, 34, 35, + 36, 37, 38, 39, + 40, 41, 42, 43, + 44, 45, 46, 47, + 48, 49, 50, 51, + 52, 53, 54, 55, + 56, 57, 58, 59, + 60, 61, 62, 63 ; + + fvar = + 0.5, 1.5, 2.5, 3.5, + 4.5, 5.5, 6.5, 7.5, + 8.5, 9.5, 10.5, 11.5, + 12.5, 13.5, 14.5, 15.5, + 16.5, 17.5, 18.5, 19.5, + 20.5, 21.5, 22.5, 23.5, + 24.5, 25.5, 26.5, 27.5, + 28.5, 29.5, 30.5, 31.5, + 32.5, 33.5, 34.5, 35.5, + 36.5, 37.5, 38.5, 39.5, + 40.5, 41.5, 42.5, 43.5, + 44.5, 45.5, 46.5, 47.5, + 48.5, 49.5, 50.5, 51.5, + 52.5, 53.5, 54.5, 55.5, + 56.5, 57.5, 58.5, 59.5, + 60.5, 61.5, 62.5, 63.5 ; +} diff --git a/nc_test4/ref_filtered.cdl b/nc_test4/ref_filtered.cdl index e23cf4b655..dc2d2eaa98 100644 --- a/nc_test4/ref_filtered.cdl +++ b/nc_test4/ref_filtered.cdl @@ -13,7 +13,7 @@ group: g { float var(dim0, dim1, dim2, dim3) ; var:_Storage = "chunked" ; var:_ChunkSizes = 4, 4, 4, 4 ; - var:_Filter = "307,9,4"; + var:_Filter = "307,9"; var:_NoFill = "true" ; // group attributes: diff --git a/nc_test4/ref_filteredvv.cdl b/nc_test4/ref_filteredvv.cdl index 0e86392273..e91759e93d 100644 --- a/nc_test4/ref_filteredvv.cdl +++ b/nc_test4/ref_filteredvv.cdl @@ -6,7 +6,7 @@ variables: float var1(dim0, dim1) ; var1:_Storage = "chunked" ; var1:_ChunkSizes = 2, 2 ; - var1:_Filter = "307,9,4" ; + var1:_Filter = "307,9" ; var1:_NoFill = "true" ; // global attributes: @@ -24,7 +24,7 @@ group: g { float var2(dim0, dim1) ; var2:_Storage = "chunked" ; var2:_ChunkSizes = 2, 2 ; - var2:_Filter = "307,9,4" ; + var2:_Filter = "307,9" ; var2:_NoFill = "true" ; // group attributes: diff --git a/nc_test4/ref_filterx_hdf5.txt b/nc_test4/ref_filterx_hdf5.txt new file mode 100644 index 0000000000..3462eb23dc --- /dev/null +++ b/nc_test4/ref_filterx_hdf5.txt @@ -0,0 +1,8 @@ +test1: def filter repeat . +test1: compression. +direction=compress id=40000 cd_nelmts=2 cd_values= 0 18 +test1: decompression. +filter(40000): params=[0,18] +direction=decompress id=40000 cd_nelmts=2 cd_values= 0 18 +data comparison: |array|=256 +no data errors diff --git a/nc_test4/test_filter.c b/nc_test4/test_filter.c index 3fd5cf7363..27fb2fd361 100644 --- a/nc_test4/test_filter.c +++ b/nc_test4/test_filter.c @@ -51,7 +51,7 @@ netcdf bzip2 { /* The compression level used in this example */ #define BZIP2_LEVEL 9 -#define TESTFILE "bzip2.nc" +#define DFALT_TESTFILE "tmp_bzip2.nc" /* Point at which we give up */ #define MAXERRS 8 @@ -72,6 +72,8 @@ static size_t chunks[NDIMS]; static int nerrs = 0; +static const char* testfile = NULL; + static int ncid, varid; static int dimids[NDIMS]; static float* array = NULL; @@ -175,7 +177,7 @@ test_bzip2(void) memset(array,0,sizeof(float)*actualproduct); /* Create a file */ - CHECK(nc_create(TESTFILE, NC_NETCDF4|NC_CLOBBER, &ncid)); + CHECK(nc_create(testfile, NC_NETCDF4|NC_CLOBBER, &ncid)); /* Do not use fill for this file */ CHECK(nc_set_fill(ncid, NC_NOFILL, NULL)); @@ -234,7 +236,7 @@ test_bzip2(void) memset(array,0,sizeof(float)*actualproduct); /* Open the file */ - CHECK(nc_open(TESTFILE, NC_NOWRITE, &ncid)); + CHECK(nc_open(testfile, NC_NOWRITE, &ncid)); /* Get the variable id */ CHECK(nc_inq_varid(ncid, "var", &varid)); @@ -278,6 +280,13 @@ static void init(int argc, char** argv) { int i; + + /* get the testfile path */ + if(argc > 1) + testfile = argv[1]; + else + testfile = DFALT_TESTFILE; + /* Setup various variables */ actualproduct = 1; chunkproduct = 1; @@ -298,7 +307,6 @@ init(int argc, char** argv) int main(int argc, char **argv) { - H5Eprint1(stderr); init(argc,argv); if(test_bzip2() != NC_NOERR) ERRR; exit(nerrs > 0?1:0); diff --git a/nc_test4/test_filter_misc.c b/nc_test4/test_filter_misc.c index f188bfec9d..a77c719415 100644 --- a/nc_test4/test_filter_misc.c +++ b/nc_test4/test_filter_misc.c @@ -32,9 +32,11 @@ static unsigned int baseline[NPARAMS]; +static const char* testfile = NULL; + #define MAXDIMS 8 -#define TESTFILE "testmisc.nc" +#define DFALT_TESTFILE "tmp_misc.nc" #define spec "32768, -17b, 23ub, -25S, 27US, 77, 93U, 789f, 12345678.12345678d, -9223372036854775807L, 18446744073709551615UL" @@ -132,7 +134,7 @@ create(void) int i; /* Create a file with one big variable, but whose dimensions arte not a multiple of chunksize (to see what happens) */ - CHECK(nc_create(TESTFILE, NC_NETCDF4|NC_CLOBBER, &ncid)); + CHECK(nc_create(testfile, NC_NETCDF4|NC_CLOBBER, &ncid)); CHECK(nc_set_fill(ncid, NC_NOFILL, NULL)); for(i=0;i 1) + testfile = argv[1]; + else + testfile = DFALT_TESTFILE; + /* Setup various variables */ totalproduct = 1; actualproduct = 1; diff --git a/nc_test4/test_filter_order.c b/nc_test4/test_filter_order.c index 822d7d722d..2e572aeeea 100644 --- a/nc_test4/test_filter_order.c +++ b/nc_test4/test_filter_order.c @@ -25,7 +25,7 @@ #define MAXDIMS 8 -#define TESTFILE "testfilter_order.nc" +#define DFALT_TESTFILE "tmp_filter_order.nc" #define NPARAMS 1 #define PARAMVAL 17 @@ -43,6 +43,8 @@ static size_t chunkproduct = 1; /* x-product over actual chunks */ static int nerrs = 0; +static const char* testfile = NULL; + static int ncid, varid; static int dimids[MAXDIMS]; static size_t odom[MAXDIMS]; @@ -112,7 +114,7 @@ create(void) int i; /* Create a file with one big variable */ - CHECK(nc_create(TESTFILE, NC_NETCDF4|NC_CLOBBER, &ncid)); + CHECK(nc_create(testfile, NC_NETCDF4|NC_CLOBBER, &ncid)); CHECK(nc_set_fill(ncid, NC_NOFILL, NULL)); for(i=0;i 1 && strcmp(argv[1],"read")==0) - creating = 0; + if(argc == 1) + usage(); + + if(strcmp(argv[1],"read")==0) creating = 0; + else if(strcmp(argv[1],"create")==0) creating = 1; + else usage(); + + /* get the testfile path */ + if(argc > 2) + testfile = argv[2]; + else + testfile = DFALT_TESTFILE; /* Setup various variables */ totalproduct = 1; diff --git a/nc_test4/test_filter_reg.c b/nc_test4/test_filter_reg.c index 12257944ea..bad0e728ff 100644 --- a/nc_test4/test_filter_reg.c +++ b/nc_test4/test_filter_reg.c @@ -15,7 +15,7 @@ #include #include "netcdf.h" -#include "netcdf_filter.h" +#include "netcdf_filter_hdf5_build_.h" #undef TESTODDSIZE @@ -553,40 +553,24 @@ H5Z_filter_reg(unsigned int flags, size_t cd_nelmts, if (flags & H5Z_FLAG_REVERSE) { /* Replace buffer */ -#ifdef HAVE_H5ALLOCATE_MEMORY newbuf = H5allocate_memory(*buf_size,0); -#else - newbuf = malloc(*buf_size); -#endif if(newbuf == NULL) abort(); if(*buf != NULL) { memcpy(newbuf,*buf,*buf_size); /* reclaim old buffer */ -#ifdef HAVE_H5FREE_MEMORY H5free_memory(*buf); -#else - free(*buf); -#endif } *buf = newbuf; } else { /* Replace buffer */ -#ifdef HAVE_H5ALLOCATE_MEMORY newbuf = H5allocate_memory(*buf_size,0); -#else - newbuf = malloc(*buf_size); -#endif if(newbuf == NULL) abort(); if(*buf != NULL) { memcpy(newbuf,*buf,*buf_size); /* reclaim old buffer */ -#ifdef HAVE_H5FREE_MEMORY H5free_memory(*buf); -#else - free(*buf); -#endif } *buf = newbuf; diff --git a/nc_test4/test_filter_repeat.c b/nc_test4/test_filter_repeat.c index d3fe51fe00..b0539544ac 100644 --- a/nc_test4/test_filter_repeat.c +++ b/nc_test4/test_filter_repeat.c @@ -25,7 +25,7 @@ #define MAXDIMS 8 -#define TESTFILE "testfilter_reg.nc" +#define DFALT_TESTFILE "testfilter_reg.nc" #define NPARAMS 1 #define PARAMVAL 17 @@ -42,6 +42,9 @@ static size_t chunkproduct = 1; /* x-product over actual chunks */ static int nerrs = 0; +static char* testfile = NULL; + + static int ncid, varid; static int dimids[MAXDIMS]; static size_t odom[MAXDIMS]; @@ -111,7 +114,7 @@ create(void) int i; /* Create a file with one big variable */ - CHECK(nc_create(TESTFILE, NC_NETCDF4|NC_CLOBBER, &ncid)); + CHECK(nc_create(testfile, NC_NETCDF4|NC_CLOBBER, &ncid)); CHECK(nc_set_fill(ncid, NC_NOFILL, NULL)); for(i=0;i 1) + testfile = argv[1]; + else + testfile = DFALT_TESTFILE; + /* Setup various variables */ totalproduct = 1; actualproduct = 1; diff --git a/nc_test4/test_filterx_hdf5.c b/nc_test4/test_filterx_hdf5.c new file mode 100644 index 0000000000..5224bcff96 --- /dev/null +++ b/nc_test4/test_filterx_hdf5.c @@ -0,0 +1,457 @@ +/* + Copyright 2018, UCAR/Unidata + See COPYRIGHT file for copying and redistribution conditions. +*/ + +#include "config.h" +#include +#include +#include +#include +#include + +/* Test using the filterx interface + for HDF5 filters +*/ + +#include "netcdf.h" +#include "netcdf_filter.h" +#include "ncbytes.h" +#include "ncjson.h" + +#undef TESTODDSIZE + +#undef DEBUG + +#define FILTER_ID 40000 + +#define MAXERRS 8 + +#define MAXPARAMS 32 + +#define MAXDIMS 8 + +#define DFALT_TESTFILE "tmp_filterx_hdf5.nc" + +#define NPARAMS 1 +#define PARAMVAL 17 + +#define NDIMS 4 +static size_t dimsize[NDIMS] = {4,4,4,4}; +static size_t chunksize[NDIMS] = {4,4,4,4}; + +static size_t ndims = NDIMS; + +static size_t totalproduct = 1; /* x-product over max dims */ +static size_t actualproduct = 1; /* x-product over actualdims */ +static size_t chunkproduct = 1; /* x-product over actual chunks */ + +static int nerrs = 0; + +static char* testfile = NULL; + + +static int ncid, varid; +static int dimids[MAXDIMS]; +static size_t odom[MAXDIMS]; +static float* array = NULL; +static float* expected = NULL; + +/* Forward */ +static int filter_test1(void); +static void init(int argc, char** argv); +static void reset(void); +static void odom_reset(void); +static int odom_more(void); +static int odom_next(void); +static int odom_offset(void); +static float expectedvalue(void); +static int vector2json(size_t n, unsigned* values, char** textp); +static int json2vector(const NCjson* jarray, size_t* np, unsigned** valuesp); + +#define ERRR do { \ +fflush(stdout); /* Make sure our stdout is synced with stderr. */ \ +fprintf(stderr, "Sorry! Unexpected result, %s, line: %d\n", \ + __FILE__, __LINE__); \ +nerrs++;\ +} while (0) + +static int +check(int err,int line) +{ + if(err != NC_NOERR) { + fprintf(stderr,"fail (%d): %s\n",line,nc_strerror(err)); + } + return NC_NOERR; +} + +#if 0 +static void +report(const char* msg, int lineno) +{ + fprintf(stderr,"fail: line=%d %s\n",lineno,msg); + exit(1); +} +#endif + +#define CHECK(x) check(x,__LINE__) +#define REPORT(x) report(x,__LINE__) + +static int +verifychunks(void) +{ + int i; + int store = -1; + size_t localchunks[MAXDIMS]; + memset(localchunks,0,sizeof(localchunks)); + CHECK(nc_inq_var_chunking(ncid, varid, &store, localchunks)); + if(store != NC_CHUNKED) { + fprintf(stderr,"bad chunk store\n"); + return 0; + } + for(i=0;i= MAXERRS) + break; + } + } + } else + { + odom_reset(); + while(odom_more()) { + int offset = odom_offset(); + float expect = expectedvalue(); + if(array[offset] != expect) { + fprintf(stderr,"data mismatch: array[%d]=%f expected=%f\n", + offset,array[offset],expect); + errs++; + if(errs >= MAXERRS) + break; + } + odom_next(); + } + } + + if(errs == 0) + printf("no data errors\n"); + return (errs == 0); +} + +static int +filter_test1(void) +{ + int ok = 1; + unsigned int params[MAXPARAMS]; + + reset(); + + printf("test1: def filter repeat .\n"); + create(); + setchunking(); + + params[0] = 1; + params[1] = 17; + deffilter(FILTER_ID,2,params); + + params[0] = 0; + params[1] = 18; + deffilter(FILTER_ID,2,params); + + CHECK(nc_enddef(ncid)); + + /* Fill in the array */ + fill(); + + printf("test1: compression.\n"); + /* write array */ + CHECK(nc_put_var(ncid,varid,expected)); + CHECK(nc_close(ncid)); + + printf("test1: decompression.\n"); + reset(); + openfile(); + CHECK(nc_get_var_float(ncid, varid, array)); + ok = compare(); + + CHECK(nc_close(ncid)); + return ok; +} + +/**************************************************/ +/* Utilities */ + +static void +reset() +{ + memset(array,0,sizeof(float)*actualproduct); +} + +static void +odom_reset(void) +{ + memset(odom,0,sizeof(odom)); +} + +static int +odom_more(void) +{ + return (odom[0] < dimsize[0]); +} + +static int +odom_next(void) +{ + int i; /* do not make unsigned */ + for(i=ndims-1;i>=0;i--) { + odom[i] += 1; + if(odom[i] < dimsize[i]) break; + if(i == 0) return 0; /* leave the 0th entry if it overflows*/ + odom[i] = 0; /* reset this position*/ + } + return 1; +} + +static int +odom_offset(void) +{ + int i; + int offset = 0; + for(i=0;i 1) + testfile = argv[1]; + else + testfile = DFALT_TESTFILE; + + /* Setup various variables */ + totalproduct = 1; + actualproduct = 1; + chunkproduct = 1; + for(i=0;i 0 && NCJcontents(jarray) != NULL) { + if((values = (unsigned*)malloc(sizeof(unsigned)*NCJlength(jarray)))==NULL) + {stat = NC_ENOMEM; goto done;} + for(i=0;i 0?1:0); +} diff --git a/nc_test4/tst_dims.c b/nc_test4/tst_dims.c index 0e617d15df..d821e20f73 100644 --- a/nc_test4/tst_dims.c +++ b/nc_test4/tst_dims.c @@ -1332,7 +1332,8 @@ main(int argc, char **argv) strcat(file_in, REF_FILE_NAME); /* Reopen and check it out again. */ - if (nc_open(file_in, NC_NOWRITE, &ncid)) ERR; + if ((ret = nc_open(file_in, NC_NOWRITE, &ncid))) + ERR; /* Check it out. */ ret = nc_inq_dim(ncid, dimid, name_in, &len_in); diff --git a/nc_test4/tst_filter.sh b/nc_test4/tst_filter.sh index f21fcc914c..c98bf21b7b 100755 --- a/nc_test4/tst_filter.sh +++ b/nc_test4/tst_filter.sh @@ -59,7 +59,7 @@ findplugin h5bzip2 BZIP2LIB="${HDF5_PLUGIN_LIB}" BZIP2PATH="${HDF5_PLUGIN_PATH}/${BZIP2LIB}" # Find misc and capture -findplugin misc +findplugin h5misc MISCPATH="${HDF5_PLUGIN_PATH}/${HDF5_PLUGIN_LIB}" echo "final HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}" @@ -81,99 +81,99 @@ fi if test "x$API" = x1 ; then echo "*** Testing dynamic filters using API" -rm -f ./bzip2.nc ./bzip2.dump ./tst_filter.txt +rm -f ./tmp_bzip2.nc ./tmp_bzip2.dump ./tmp_filter.txt ${execdir}/test_filter -${NCDUMP} -s bzip2.nc > ./tst_filter.txt +${NCDUMP} -s -n bzip2 tmp_bzip2.nc > ./tmp_filter.txt # Remove irrelevant -s output -sclean ./tst_filter.txt ./bzip2.dump -diff -b -w ${srcdir}/bzip2.cdl ./bzip2.dump +sclean ./tmp_filter.txt ./tmp_bzip2.dump +diff -b -w ${srcdir}/bzip2.cdl ./tmp_bzip2.dump echo "*** Pass: API dynamic filter" fi if test "x$MISC" = x1 ; then echo echo "*** Testing dynamic filters parameter passing" -rm -f ./testmisc.nc tst_filter.txt tst_filter2.txt +rm -f ./tmp_misc.nc tmp_filter.txt tmp_filter2.txt ${execdir}/test_filter_misc # Verify the parameters via ncdump -${NCDUMP} -s testmisc.nc > ./tst_filter.txt +${NCDUMP} -s tmp_misc.nc > ./tmp_misc.txt # Extract the parameters -getfilterattr ./tst_filter.txt ./tst_filter2.txt -rm -f ./tst_filter.txt -trimleft ./tst_filter2.txt ./tst_filter.txt -rm -f ./tst_filter2.txt -cat >./tst_filter2.txt <./tmp_filter2.txt < ./tst_filter.txt +rm -f ./tmp_bzip2.nc ./tmp_bzip2.dump ./tmp_filter.txt +${NCGEN} -lb -4 -o tmp_bzip2.nc ${srcdir}/bzip2.cdl +${NCDUMP} -s -n bzip2 tmp_bzip2.nc > ./tmp_filter.txt # Remove irrelevant -s output -sclean ./tst_filter.txt ./bzip2.dump -diff -b -w ${srcdir}/bzip2.cdl ./bzip2.dump +sclean ./tmp_filter.txt ./tmp_bzip2.dump +diff -b -w ${srcdir}/bzip2.cdl ./tmp_bzip2.dump echo "*** Pass: ncgen dynamic filter" fi if test "x$NCP" = x1 ; then echo "*** Testing dynamic filters using nccopy" -rm -f ./unfiltered.nc ./filtered.nc ./tmp.nc ./filtered.dump ./tst_filter.txt +rm -f ./tmp_unfiltered.nc ./tmp_filtered.nc ./tmp.nc ./tmp_filtered.dump ./tmp_filter.txt # Create our input test files -${NCGEN} -4 -lb -o unfiltered.nc ${srcdir}/ref_unfiltered.cdl -${NCGEN} -4 -lb -o unfilteredvv.nc ${srcdir}/ref_unfilteredvv.cdl +${NCGEN} -4 -lb -o tmp_unfiltered.nc ${srcdir}/ref_unfiltered.cdl +${NCGEN} -4 -lb -o tmp_unfilteredvv.nc ${srcdir}/ref_unfilteredvv.cdl echo " *** Testing simple filter application" -${NCCOPY} -M0 -F "/g/var,307,9,4" unfiltered.nc filtered.nc -${NCDUMP} -s filtered.nc > ./tst_filter.txt +${NCCOPY} -M0 -F "/g/var,307,9" tmp_unfiltered.nc tmp_filtered.nc +${NCDUMP} -s -n filtered tmp_filtered.nc > ./tmp_filter.txt # Remove irrelevant -s output -sclean ./tst_filter.txt ./filtered.dump -diff -b -w ${srcdir}/ref_filtered.cdl ./filtered.dump +sclean ./tmp_filter.txt ./tmp_filtered.dump +diff -b -w ${srcdir}/ref_filtered.cdl ./tmp_filtered.dump echo " *** Pass: nccopy simple filter" echo " *** Testing '*' filter application" -${NCCOPY} -M0 -F "*,307,9,4" unfilteredvv.nc filteredvv.nc -${NCDUMP} -s filteredvv.nc > ./tst_filtervv.txt +${NCCOPY} -M0 -F "*,307,9" tmp_unfilteredvv.nc tmp_filteredvv.nc +${NCDUMP} -s -n filteredvv tmp_filteredvv.nc > ./tmp_filtervv.txt # Remove irrelevant -s output -sclean ./tst_filtervv.txt ./filteredvv.dump -diff -b -w ${srcdir}/ref_filteredvv.cdl ./filteredvv.dump +sclean ./tmp_filtervv.txt ./tmp_filteredvv.dump +diff -b -w ${srcdir}/ref_filteredvv.cdl ./tmp_filteredvv.dump echo " *** Pass: nccopy '*' filter" echo " *** Testing 'v&v' filter application" -${NCCOPY} -M0 -F "var1&/g/var2,307,9,4" unfilteredvv.nc filteredvbar.nc -${NCDUMP} -n filteredvv -s filteredvbar.nc > ./tst_filtervbar.txt +${NCCOPY} -M0 -F "var1&/g/var2,307,9" tmp_unfilteredvv.nc tmp_filteredvbar.nc +${NCDUMP} -n filteredvv -s tmp_filteredvbar.nc > ./tmp_filtervbar.txt # Remove irrelevant -s output -sclean ./tst_filtervbar.txt ./filteredvbar.dump -diff -b -w ${srcdir}/ref_filteredvv.cdl ./filteredvbar.dump +sclean ./tmp_filtervbar.txt ./tmp_filteredvbar.dump +diff -b -w ${srcdir}/ref_filteredvv.cdl ./tmp_filteredvbar.dump echo " *** Pass: nccopy 'v|v' filter" echo " *** Testing pass-thru of filters" -rm -f ./tst_filter.txt tst_filter2.txt ./tst_filter2.nc +rm -f ./tmp_filter.txt tmp_filter2.txt ./tmp_filter2.nc # Prevent failure by allowing any chunk size -${NCCOPY} -M0 ./filtered.nc ./tst_filter2.nc -${NCDUMP} -s tst_filter2.nc > ./tst_filter.txt -sed -e '/_Filter/p' -e d < ./tst_filter.txt >tst_filter2.txt -test -s tst_filter2.txt +${NCCOPY} -M0 ./tmp_filtered.nc ./tmp_filter2.nc +${NCDUMP} -s tmp_filter2.nc > ./tmp_filter.txt +sed -e '/_Filter/p' -e d < ./tmp_filter.txt >tmp_filter2.txt +test -s tmp_filter2.txt echo " *** Pass: pass-thru of filters" echo " *** Testing -F none" -rm -f ./tst_none.txt ./tst_none2.txt ./tst_none.nc -${NCCOPY} -M0 -F none ./filtered.nc ./tst_none.nc -${NCDUMP} -hs tst_none.nc > ./tst_none.txt -sed -e '/_Filter/p' -e d < ./tst_none.txt >./tst_none2.txt -test ! -s tst_none2.txt +rm -f ./tmp_none.txt ./tmp_none2.txt ./tmp_none.nc +${NCCOPY} -M0 -F none ./tmp_filtered.nc ./tmp_none.nc +${NCDUMP} -hs tmp_none.nc > ./tmp_none.txt +sed -e '/_Filter/p' -e d < ./tmp_none.txt >./tmp_none2.txt +test ! -s tmp_none2.txt echo " *** Pass: -F none" echo " *** Testing -F var,none " -rm -f ./tst_vnone.txt ./tst_vnone.nc tst_vnone2.txt -${NCCOPY} -M0 -F "/g/var,none" ./filtered.nc ./tst_vnone.nc -${NCDUMP} -s tst_vnone.nc > ./tst_vnone.txt -sed -e '/_Filter/p' -e d < ./tst_vnone.txt >tst_vnone2.txt -test ! -s tst_vnone2.txt +rm -f ./tmp_vnone.txt ./tmp_vnone.nc tmp_vnone2.txt +${NCCOPY} -M0 -F "/g/var,none" ./tmp_filtered.nc ./tmp_vnone.nc +${NCDUMP} -s tmp_vnone.nc > ./tmp_vnone.txt +sed -e '/_Filter/p' -e d < ./tmp_vnone.txt >tmp_vnone2.txt +test ! -s tmp_vnone2.txt echo " *** Pass: -F var,none" echo "*** Pass: all nccopy filter tests" @@ -210,75 +210,60 @@ echo "*** Pass: unknown filter" fi if test "x$NGC" = x1 ; then -rm -f ./test_bzip2.c +rm -f ./tmp_bzip2.c echo "*** Testing dynamic filters using ncgen with -lc" -${NCGEN} -lc -4 ${srcdir}/bzip2.cdl > test_bzip2.c -diff -b -w ${srcdir}/ref_bzip2.c ./test_bzip2.c +${NCGEN} -lc -4 ${srcdir}/bzip2.cdl > tmp_bzip2.c +diff -b -w ${srcdir}/ref_bzip2.c ./tmp_bzip2.c echo "*** Pass: ncgen dynamic filter" fi if test "x$MULTI" = x1 ; then echo "*** Testing multiple filters" -rm -f ./multifilter.nc ./multi.txt ./smulti.cdl -rm -f nccopyF.cdl nccopyF.nc ncgenF.cdl ncgenF.nc +rm -f ./tmp_multifilter.nc ./tmp_multi.txt ./tmp_smulti.cdl +rm -f tmp_nccopyF.cdl tmp_nccopyF.nc tmp_ncgenF.cdl tmp_ncgenF.nc ${execdir}/tst_multifilter -${NCDUMP} -hs multifilter.nc >./multi.cdl +${NCDUMP} -hs -n multifilter tmp_multifilter.nc >./tmp_multi.cdl # Remove irrelevant -s output -sclean ./multi.cdl ./smulti.cdl -diff -b -w ${srcdir}/ref_multi.cdl ./smulti.cdl +sclean ./tmp_multi.cdl ./tmp_smulti.cdl +diff -b -w ${srcdir}/ref_multi.cdl ./tmp_smulti.cdl echo "*** nccopy -F with multiple filters" -if ! test -f unfiltered.nc ; then - ${NCGEN} -4 -lb -o unfiltered.nc ${srcdir}/ref_unfiltered.cdl +if ! test -f tmp_unfiltered.nc ; then + ${NCGEN} -4 -lb -o tmp_unfiltered.nc ${srcdir}/ref_unfiltered.cdl fi -${NCCOPY} "-F/g/var,307,4|40000" unfiltered.nc nccopyF.nc -${NCDUMP} -hs nccopyF.nc > ./nccopyF.cdl -sclean nccopyF.cdl nccopyFs.cdl -diff -b -w ${srcdir}/ref_nccopyF.cdl ./nccopyFs.cdl +${NCCOPY} "-F/g/var,307,4|40000" tmp_unfiltered.nc tmp_nccopyF.nc +${NCDUMP} -hs -n nccopyF tmp_nccopyF.nc > ./tmp_nccopyF.cdl +sclean tmp_nccopyF.cdl tmp_nccopyFs.cdl +diff -b -w ${srcdir}/ref_nccopyF.cdl ./tmp_nccopyFs.cdl echo "*** ncgen with multiple filters" -${NCGEN} -4 -lb -o ncgenF.nc ${srcdir}/ref_nccopyF.cdl +${NCGEN} -4 -lb -o tmp_ncgenF.nc ${srcdir}/ref_nccopyF.cdl # Need to fix name using -n -${NCDUMP} -hs -n nccopyF ncgenF.nc > ./ncgenF.cdl -sclean ncgenF.cdl ncgenFs.cdl -diff -b -w ${srcdir}/ref_nccopyF.cdl ./ncgenFs.cdl +${NCDUMP} -hs -n nccopyF tmp_ncgenF.nc > ./tmp_ncgenF.cdl +sclean tmp_ncgenF.cdl tmp_ncgenFs.cdl +diff -b -w ${srcdir}/ref_nccopyF.cdl ./tmp_ncgenFs.cdl echo "*** Pass: multiple filters" fi if test "x$REP" = x1 ; then echo "*** Testing filter re-definition invocation" -rm -f filterrepeat.txt -${execdir}/test_filter_repeat >filterrepeat.txt -diff -b -w ${srcdir}/ref_filter_repeat.txt filterrepeat.txt +rm -f tmp_filterrepeat.txt +${execdir}/test_filter_repeat >tmp_filterrepeat.txt +diff -b -w ${srcdir}/ref_filter_repeat.txt tmp_filterrepeat.txt fi if test "x$ORDER" = x1 ; then echo "*** Testing multiple filter order of invocation on create" -rm -f crfilterorder.txt -${execdir}/test_filter_order create >crfilterorder.txt -diff -b -w ${srcdir}/ref_filter_order_create.txt crfilterorder.txt +rm -f tmp_crfilterorder.txt +${execdir}/test_filter_order create >tmp_crfilterorder.txt +diff -b -w ${srcdir}/ref_filter_order_create.txt tmp_crfilterorder.txt echo "*** Testing multiple filter order of invocation on read" -rm -f rdfilterorder.txt -${execdir}/test_filter_order read >rdfilterorder.txt -diff -b -w ${srcdir}/ref_filter_order_read.txt rdfilterorder.txt +rm -f tmp_rdfilterorder.txt +${execdir}/test_filter_order read >tmp_rdfilterorder.txt +diff -b -w ${srcdir}/ref_filter_order_read.txt tmp_rdfilterorder.txt fi echo "*** Pass: all selected tests passed" -#cleanup -rm -f testmisc.nc -rm -f unfiltered.nc unfilteredvv.nc filtered.nc filtered.dump -rm -f filteredvv.nc tst_filtervv.txt filteredvv.dump -rm -f filteredvbar.nc tst_filtervbar.txt filteredvbar.dump -rm -f tst_filter2.nc tst_filter2.txt -rm -f tst_none.nc tst_none.txt tst_none2.txt testfilter_reg.nc -rm -f tst_vnone.nc tst_vnone.txt tst_vnone2.txt -rm -f bzip2.nc bzip2.dump tst_filter.txt bzip2x.dump -rm -f test_bzip2.c -rm -f multifilter.nc multi.cdl smulti.cdl -rm -f nccopyF.nc nccopyF.cdl ncgenF.nc ncgenF.cdl -rm -f ncgenFs.cdl nccopyFs.cdl -rm -f crfilterorder.txt rdfilterorder.txt - exit 0 diff --git a/nc_test4/tst_interops.c b/nc_test4/tst_interops.c index 880623377f..e9215a0b6c 100644 --- a/nc_test4/tst_interops.c +++ b/nc_test4/tst_interops.c @@ -36,7 +36,9 @@ printRes(const char *msg, int ires) if (ires < 0) { printf("bad ires: %d\n", ires); - /*H5Eprint2(ires, stdout);*/ +#ifdef DEBUG + H5Eprint2(ires, stdout); +#endif exit(1); } } diff --git a/nc_test4/tst_multifilter.c b/nc_test4/tst_multifilter.c index 68fa66b593..79423fbf35 100644 --- a/nc_test4/tst_multifilter.c +++ b/nc_test4/tst_multifilter.c @@ -27,7 +27,7 @@ Test support for multiple filters per variable #define NFILTERS 3 -#define TESTFILE "multifilter.nc" +#define DFALT_TESTFILE "tmp_multifilter.nc" /* Point at which we give up */ #define MAXERRS 8 @@ -36,6 +36,8 @@ Test support for multiple filters per variable #define DIMSIZE 4 #define CHUNKSIZE 4 /* Note: not the total size of the chunk, but size wrt a dim*/ +static const char* testfile = NULL; + static size_t dimsize = DIMSIZE; static size_t chunksize = CHUNKSIZE; static size_t actualdims = NDIMS; @@ -199,7 +201,7 @@ test_multi(void) memset(array,0,sizeof(float)*actualproduct); /* Create a file */ - CHECK(nc_create(TESTFILE, NC_NETCDF4|NC_CLOBBER, &ncid)); + CHECK(nc_create(testfile, NC_NETCDF4|NC_CLOBBER, &ncid)); /* Do not use fill for this file */ CHECK(nc_set_fill(ncid, NC_NOFILL, NULL)); @@ -262,7 +264,7 @@ test_multi(void) memset(array,0,sizeof(float)*actualproduct); /* Open the file */ - CHECK(nc_open(TESTFILE, NC_NOWRITE, &ncid)); + CHECK(nc_open(testfile, NC_NOWRITE, &ncid)); /* Get the variable id */ CHECK(nc_inq_varid(ncid, "var", &varid)); @@ -289,6 +291,13 @@ static void init(int argc, char** argv) { int i; + + /* get the testfile path */ + if(argc > 1) + testfile = argv[1]; + else + testfile = DFALT_TESTFILE; + /* Setup various variables */ actualproduct = 1; chunkproduct = 1; @@ -309,7 +318,9 @@ init(int argc, char** argv) int main(int argc, char **argv) { - H5Eprint1(stderr); +#ifdef DEBUG + H5Eprint(stderr); +#endif init(argc,argv); if(test_multi() != NC_NOERR) ERRR; exit(nerrs > 0?1:0); diff --git a/nc_test4/tst_specific_filters.sh b/nc_test4/tst_specific_filters.sh new file mode 100755 index 0000000000..36ad8eaf10 --- /dev/null +++ b/nc_test4/tst_specific_filters.sh @@ -0,0 +1,131 @@ +#!/bin/bash + +# Test the implementations of specific filters + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +# Load the findplugins function +. ${builddir}/findplugin.sh +echo "findplugin.sh loaded" + +# Locate the plugin path and the library names; argument order is critical +# Find bzip2 and capture +# Assume all test filters are in same plugin dir +findplugin h5bzip2 + +echo "final HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}" +export HDF5_PLUGIN_PATH + +set -e + +# Function to remove selected -s attributes from file; +# These attributes might be platform dependent +sclean() { + cat $1 \ + | sed -e '/:_IsNetcdf4/d' \ + | sed -e '/:_Endianness/d' \ + | sed -e '/_NCProperties/d' \ + | sed -e '/_SuperblockVersion/d' \ + | sed -e '/_Format/d' \ + | sed -e '/global attributes:/d' \ + | cat > $2 +} + +# Function to extract _Filter attribute from a file +# These attributes might be platform dependent +getfilterattr() { +V="$1" +sed -e '/${V}.*:_Filter/p' -ed <$2 >$3 +} + +# Function to extract _Codecs attribute from a file +# These attributes might be platform dependent +getcodecsattr() { +V="$1" +sed -e '/${V}.*:_Codecs/p' -ed <$2 >$3 +} + +trimleft() { +sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2 +} + +setfilter() { + FF="$1" + FSRC="$2" + FDST="$3" + FIH5="$4" + FICX="$5" + FFH5="$6" + FFCX="$7" + if test "x$FFH5" = x ; then FFH5="$FIH5" ; fi + if test "x$FFCX" = x ; then FFCX="$FICX" ; fi + rm -f $FDST + cat ${srcdir}/$FSRC \ + | sed -e "s/ref_any/${FF}/" \ + | sed -e "s/IH5/${FIH5}/" -e "s/FH5/${FFH5}/" \ + | sed -e "s/ICX/${FICX}/" -e "s/FCX/${FFCX}/" \ + | sed -e 's/"/\\"/g' -e 's/@/"/g' \ + | cat > $FDST +} + +# Execute the specified tests + +runfilter() { +zext=$1 +zfilt="$2" +zparams="$3" +zcodec="$4" +echo "*** Testing processing of filter $zfilt" +file="tmp_${zfilt}.nc" +rm -f "tmp_${zfilt},nc" +setfilter $zfilt ref_any.cdl "tmp_${zfilt}.cdl" "$zparams" "$zcodec" +if ${NCGEN} -4 -lb -o $file "tmp_${zfilt}.cdl" ; then + ${NCDUMP} -n $zfilt -s $file > "tmp_${zfilt}.tmp" + sclean "tmp_${zfilt}.tmp" "tmp_${zfilt}.dump" +fi +unset NCTRACING +} + +testdeflate() { + zext=$1 + runfilter $zext deflate '1,9' '[{\"id\": \"zlib\",\"level\": \"9\"}]' + if test -f "tmp_deflate.dump" ; then + # need to replace _DeflateLevel + sed -e 's/_DeflateLevel = 9/_Filter = "1,9"/' < tmp_deflate.dump > tmp_deflatex.dump + diff -b -w "tmp_deflate.cdl" "tmp_deflatex.dump" + else + echo "XFAIL: filter=deflate" + fi +} + +testbzip2() { + zext=$1 + runfilter $zext bzip2 '307,9' '[{\"id\": \"bz2\",\"level\": \"9\"}]' + if test -f "tmp_bzip2.dump" ; then + diff -b -w "tmp_bzip2.cdl" "tmp_bzip2.dump" + else + echo "XFAIL: filter=bzip2" + fi +} + +testblosc() { + zext=$1 + runfilter $zext blosc '32001,2,2,4,256,5,1,1' '[{\"id\": \"blosc\",\"clevel\": 5,\"blocksize\": 0,\"cname\": \"lz4\",\"shuffle\": 1}]' + if test -f "tmp_blosc.dump" ; then + diff -b -w "tmp_blosc.cdl" "tmp_blosc.dump" + else + echo "XFAIL: filter=blosc" + fi +} + +testset() { +# Which test cases to exercise +# testdeflate $1 + testbzip2 $1 +# testblosc $1 +} + +testset nc + +exit 0 diff --git a/nc_test4/tst_vars3.c b/nc_test4/tst_vars3.c index 3b846158c7..37eca905aa 100644 --- a/nc_test4/tst_vars3.c +++ b/nc_test4/tst_vars3.c @@ -749,8 +749,10 @@ main(int argc, char **argv) params[0] = NC_SZIP_NN; /* options_mask */ params[1] = NC_SZIP_EC_BPP_IN; /* pixels_per_block */ if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR; - if (nc_def_var_filter(ncid, varid, H5_FILTER_SZIP, NUM_PARAMS_IN, - params) != NC_EFILTER) ERR; + { int stat; if ((stat = nc_def_var_filter(ncid, varid, H5_FILTER_SZIP, NUM_PARAMS_IN, + params)) != NC_EFILTER) + ERR; + } if (nc_def_var_szip(ncid, varid, NC_SZIP_NN, NC_SZIP_EC_BPP_IN) != NC_EFILTER) ERR; if (nc_close(ncid)) ERR; diff --git a/ncdap_test/CMakeLists.txt b/ncdap_test/CMakeLists.txt index 007e9d0ae8..8c4f1af31e 100644 --- a/ncdap_test/CMakeLists.txt +++ b/ncdap_test/CMakeLists.txt @@ -10,8 +10,7 @@ add_definitions(-D"TOPBINDIR=${CMAKE_CURRENT_BINARY_DIR}/../") remove_definitions(-DDLL_EXPORT) # add_definitions(-D"TOPSRCDIR=${CMAKE_CURRENT_SOURCE_DIR}") -FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.sh ${CMAKE_CURRENT_SOURCE_DIR}/ref_pathcvt.txt) - +FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.sh) FILE(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/ FILE_PERMISSIONS OWNER_WRITE OWNER_READ OWNER_EXECUTE) IF(ENABLE_TESTS) @@ -21,11 +20,6 @@ IF(ENABLE_TESTS) BUILD_BIN_TEST(pingurl) ENDIF() - IF(USE_X_GETOPT) - SET(XGSRC XGetopt.c) - ENDIF() - BUILD_BIN_TEST(pathcvt ${XGSRC}) - # Base tests # The tests are set up as a combination of shell scripts and executables that # must be run in a particular order. It is painful but will use macros to help @@ -34,13 +28,9 @@ IF(ENABLE_TESTS) IF(BUILD_UTILITIES) add_sh_test(ncdap tst_ncdap3) - add_sh_test(ncdap testpathcvt) IF(HAVE_BASH) SET_TESTS_PROPERTIES(ncdap_tst_ncdap3 PROPERTIES RUN_SERIAL TRUE) - SET_TESTS_PROPERTIES(ncdap_testpathcvt PROPERTIES RUN_SERIAL TRUE) ENDIF(HAVE_BASH) - - ENDIF() IF(NOT MSVC) diff --git a/ncdap_test/Makefile.am b/ncdap_test/Makefile.am index fe0dd6a0be..875c9fa5e2 100644 --- a/ncdap_test/Makefile.am +++ b/ncdap_test/Makefile.am @@ -32,22 +32,16 @@ check_PROGRAMS += t_dap3a test_cvt3 test_vara TESTS += t_dap3a test_cvt3 test_vara if BUILD_UTILITIES TESTS += tst_ncdap3.sh -if ENABLE_DAP_REMOTE_TESTS -TESTS += testpathcvt.sh -endif endif # remote tests are optional # because the server may be down or inaccessible if ENABLE_DAP_REMOTE_TESTS -noinst_PROGRAMS = findtestserver pingurl pathcvt +noinst_PROGRAMS = findtestserver pingurl findtestserver_SOURCES = findtestserver.c pingurl_SOURCES = pingurl.c -pathcvt_SOURCES = pathcvt.c -endif -if ENABLE_DAP_REMOTE_TESTS if BUILD_UTILITIES TESTS += tst_ber.sh tst_remote3.sh tst_formatx.sh testurl.sh tst_fillmismatch.sh tst_zero_len_var.sh tst_encode.sh endif @@ -97,8 +91,8 @@ EXTRA_DIST = tst_ncdap3.sh \ tst_zero_len_var.sh \ tst_filelists.sh tst_urls.sh tst_utils.sh \ t_dap.c CMakeLists.txt tst_formatx.sh testauth.sh testurl.sh \ - t_ncf330.c tst_ber.sh tst_fillmismatch.sh tst_encode.sh testpathcvt.sh \ - findtestserver.c.in ref_pathcvt.txt + t_ncf330.c tst_ber.sh tst_fillmismatch.sh tst_encode.sh \ + findtestserver.c.in CLEANFILES = test_varm3 test_cvt3 file_results/* remote_results/* datadds* t_dap3a test_nstride_cached *.exe tmp*.txt # This should only be left behind if using parallel io diff --git a/ncdap_test/testauth.sh b/ncdap_test/testauth.sh index 811b6227a9..3743b225c0 100755 --- a/ncdap_test/testauth.sh +++ b/ncdap_test/testauth.sh @@ -42,10 +42,10 @@ HOMERCFILES="$HOME/.dodsrc $HOME/.daprc $HOME/.ncrc $HOME/$NETRC $HOME/$NETRCIMP NETRCFILE=$WD/$NETRC DAPRCFILE=$WD/$RC if test "x$FP_ISMSVC" != x ; then - LOCALRCFILES=`${execdir}/pathcvt "$LOCALRCFILES"` - HOMERCFILES=`${execdir}/pathcvt "$HOMERCFILES"` - NETRCFILE=`${execdir}/pathcvt "$NETRCFILE"` - DAPRCFILE=`${execdir}/pathcvt "$DAPRCFILE"` + LOCALRCFILES=`${NCPATHCVT} "$LOCALRCFILES"` + HOMERCFILES=`${NCPATHCVT} "$HOMERCFILES"` + NETRCFILE=`${NCPATHCVT} "$NETRCFILE"` + DAPRCFILE=`${NCPATHCVT} "$DAPRCFILE"` fi HOMENETRCFILE=$HOME/$NETRC diff --git a/ncdump/CMakeLists.txt b/ncdump/CMakeLists.txt index 4ef660d1a4..f2aaf0668b 100644 --- a/ncdump/CMakeLists.txt +++ b/ncdump/CMakeLists.txt @@ -14,6 +14,7 @@ SET(nccopy_FILES nccopy.c nciter.c chunkspec.c utils.c dimmap.c list.c) SET(ocprint_FILES ocprint.c) SET(ncvalidator_FILES ncvalidator.c) SET(printfqn_FILES printfqn.c) +SET(ncpathcvt_FILES ncpathcvt.c) IF(USE_X_GETOPT) SET(ncdump_FILES ${ncdump_FILES} XGetopt.c) @@ -21,11 +22,13 @@ IF(USE_X_GETOPT) SET(ocprint_FILES ${ocprint_FILES} XGetopt.c) SET(ncvalidator_FILES ${ncvalidator_FILES} XGetopt.c) SET(printfqn_FILES ${printfqn_FILES} XGetopt.c) + SET(ncpathcvt_FILES ${ncpathcvt_FILES} XGetopt.c) ENDIF(USE_X_GETOPT) ADD_EXECUTABLE(ncdump ${ncdump_FILES}) ADD_EXECUTABLE(nccopy ${nccopy_FILES}) ADD_EXECUTABLE(ncvalidator ${ncvalidator_FILES}) +ADD_EXECUTABLE(ncpathcvt ${ncpathcvt_FILES}) IF(USE_HDF5) ADD_EXECUTABLE(nc4print nc4print.c nc4printer.c) @@ -39,6 +42,7 @@ ENDIF(ENABLE_DAP) TARGET_LINK_LIBRARIES(ncdump netcdf ${ALL_TLL_LIBS}) TARGET_LINK_LIBRARIES(nccopy netcdf ${ALL_TLL_LIBS}) TARGET_LINK_LIBRARIES(ncvalidator netcdf ${ALL_TLL_LIBS}) +TARGET_LINK_LIBRARIES(ncpathcvt netcdf ${ALL_TLL_LIBS}) IF(USE_HDF5) TARGET_LINK_LIBRARIES(nc4print netcdf ${ALL_TLL_LIBS}) @@ -77,23 +81,20 @@ IF(MSVC) SET_TARGET_PROPERTIES(ncvalidator PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_BINARY_DIR}) - IF(USE_HDF5) - - SET_TARGET_PROPERTIES(ncvalidator PROPERTIES RUNTIME_OUTPUT_DIRECTORY - ${CMAKE_CURRENT_BINARY_DIR}) - SET_TARGET_PROPERTIES(ncvalidator PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG - ${CMAKE_CURRENT_BINARY_DIR}) - SET_TARGET_PROPERTIES(ncvalidator PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE - ${CMAKE_CURRENT_BINARY_DIR}) + SET_TARGET_PROPERTIES(ncpathcvt PROPERTIES RUNTIME_OUTPUT_DIRECTORY + ${CMAKE_CURRENT_BINARY_DIR}) + SET_TARGET_PROPERTIES(ncpathcvt PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG + ${CMAKE_CURRENT_BINARY_DIR}) + SET_TARGET_PROPERTIES(ncpathcvt PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE + ${CMAKE_CURRENT_BINARY_DIR}) + IF(USE_HDF5) SET_TARGET_PROPERTIES(printfqn PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) SET_TARGET_PROPERTIES(printfqn PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_CURRENT_BINARY_DIR}) SET_TARGET_PROPERTIES(printfqn PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_BINARY_DIR}) - - ENDIF(USE_HDF5) IF(ENABLE_DAP) @@ -157,8 +158,7 @@ IF(ENABLE_TESTS) ${CMAKE_CURRENT_BINARY_DIR}) SET_TARGET_PROPERTIES(tst_rcmerge PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_CURRENT_BINARY_DIR}) - SET_TARGET_PROPERTIES(tst_rcmerge PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE - ${CMAKE_CURRENT_BINARY_DIR}) + SET_TARGET_PROPERTIES(tst_rcmerge PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_BINARY_DIR}) IF(USE_HDF5) SET_TARGET_PROPERTIES(tst_fileinfo PROPERTIES RUNTIME_OUTPUT_DIRECTORY @@ -339,6 +339,8 @@ IF(ENABLE_TESTS) add_sh_test(ncdump test_rcmerge) + add_sh_test(ncdump testpathcvt) + ENDIF(ENABLE_TESTS) #IF(MSVC) @@ -366,7 +368,7 @@ SET(MAN_FILES nccopy.1 ncdump.1) # Note, the L512.bin file is file containing exactly 512 bytes each of value 0. # It is used for creating hdf5 files with varying offsets for testing. -FILE(GLOB COPY_FILES ${CMAKE_BINARY_DIR}/ncgen/*.nc ${CMAKE_BINARY_DIR}/nc_test4/*.nc ${CMAKE_CURRENT_SOURCE_DIR}/*.ncml ${CMAKE_CURRENT_SOURCE_DIR}/*.nc ${CMAKE_CURRENT_SOURCE_DIR}/*.cdl ${CMAKE_CURRENT_SOURCE_DIR}/*.sh ${CMAKE_CURRENT_SOURCE_DIR}/*.1 ${CMAKE_CURRENT_SOURCE_DIR}/L512.bin ${CMAKE_CURRENT_SOURCE_DIR}/ref_ctest*.c) +FILE(GLOB COPY_FILES ${CMAKE_BINARY_DIR}/ncgen/*.nc ${CMAKE_BINARY_DIR}/nc_test4/*.nc ${CMAKE_CURRENT_SOURCE_DIR}/*.ncml ${CMAKE_CURRENT_SOURCE_DIR}/*.nc ${CMAKE_CURRENT_SOURCE_DIR}/*.cdl ${CMAKE_CURRENT_SOURCE_DIR}/*.sh ${CMAKE_CURRENT_SOURCE_DIR}/*.1 ${CMAKE_CURRENT_SOURCE_DIR}/L512.bin ${CMAKE_CURRENT_SOURCE_DIR}/ref_ctest*.c ) FILE(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/ FILE_PERMISSIONS OWNER_WRITE OWNER_READ OWNER_EXECUTE) ADD_SUBDIRECTORY(cdl) diff --git a/ncdump/Makefile.am b/ncdump/Makefile.am index 8b527aa690..9276b3ee7c 100644 --- a/ncdump/Makefile.am +++ b/ncdump/Makefile.am @@ -41,6 +41,10 @@ utils.h utils.c dimmap.h dimmap.c list.c list.h noinst_PROGRAMS += ncvalidator ncvalidator_SOURCES = ncvalidator.c +# A non-installed utility program to convert paths; similar to cygpath +noinst_PROGRAMS += ncpathcvt +ncpathcvt_SOURCES = ncpathcvt.c + # A simple netcdf-4 metadata -> xml printer. Do not install. if USE_HDF5 bin_PROGRAMS += nc4print @@ -95,6 +99,8 @@ if LARGE_FILE_TESTS TESTS += tst_iter.sh endif +TESTS += testpathcvt.sh + if USE_HDF5 # HDF5 has some extra C programs to build. These will be run by # the shell script tests. @@ -185,6 +191,8 @@ EXTRA_DIST += L512.bin EXTRA_DIST += tst_ctests.sh ref_ctest_small_3.c ref_ctest_small_4.c \ ref_ctest_special_atts_4.c +EXTRA_DIST += testpathcvt.sh ref_pathcvt.txt + # CDL files and Expected results SUBDIRS = cdl expected diff --git a/ncdump/nccopy.c b/ncdump/nccopy.c index 12f7592347..7568266f31 100644 --- a/ncdump/nccopy.c +++ b/ncdump/nccopy.c @@ -2057,16 +2057,16 @@ copy(char* infile, char* outfile) /* Check if any vars in -v don't exist */ if(missing_vars(igrp, option_nlvars, option_lvars)) - exit(EXIT_FAILURE); + goto fail; if(option_nlgrps > 0) { if(inkind != NC_FORMAT_NETCDF4) { error("Group list (-g ...) only permitted for netCDF-4 file"); - exit(EXIT_FAILURE); + goto fail; } /* Check if any grps in -g don't exist */ if(grp_matches(igrp, option_nlgrps, option_lgrps, option_grpids) == 0) - exit(EXIT_FAILURE); + goto fail; } if(option_write_diskless) @@ -2146,6 +2146,9 @@ copy(char* infile, char* outfile) NC_CHECK(nc_close(igrp)); NC_CHECK(nc_close(ogrp)); return stat; +fail: + nc_finalize(); + exit(EXIT_FAILURE); } /* @@ -2448,6 +2451,8 @@ main(int argc, char**argv) filteroptions = NULL; #endif /*USE_NETCDF4*/ + nc_finalize(); + exit(exitcode); } diff --git a/ncdump/ncdump.c b/ncdump/ncdump.c index f0cf351696..9920a72ade 100644 --- a/ncdump/ncdump.c +++ b/ncdump/ncdump.c @@ -1068,6 +1068,28 @@ pr_att_specials( printf(" = \"true\" ;\n"); } } + /* _Codecs*/ + { + int stat; + size_t len; + nc_type typeid; + stat = nc_inq_att(ncid,varid,NC_ATT_CODECS,&typeid,&len); + if(stat == NC_NOERR && typeid == NC_CHAR && len > 0) { + char* json = (char*)malloc(len+1); + if(json != NULL) { + stat = nc_get_att_text(ncid,varid,NC_ATT_CODECS,json); + if(stat == NC_NOERR) { + char* escapedjson = NULL; + pr_att_name(ncid, varp->name, NC_ATT_CODECS); + /* Escape the json */ + escapedjson = escaped_string(json); + printf(" = \"%s\" ;\n",escapedjson); + free(escapedjson); + } + free(json); + } + } + } /* _Checksum */ { int fletcher32 = 0; @@ -2459,6 +2481,7 @@ main(int argc, char *argv[]) NC_CHECK( nc_close(ncid) ); } nullfree(path) path = NULL; + nc_finalize(); exit(EXIT_SUCCESS); fail: /* ncstat failures */ @@ -2468,6 +2491,7 @@ main(int argc, char *argv[]) nullfree(path); path = NULL; if(strlen(errmsg) > 0) error("%s", errmsg); + nc_finalize(); exit(EXIT_FAILURE); } diff --git a/ncdap_test/pathcvt.c b/ncdump/ncpathcvt.c similarity index 89% rename from ncdap_test/pathcvt.c rename to ncdump/ncpathcvt.c index 58f854b17d..54638dcbca 100755 --- a/ncdap_test/pathcvt.c +++ b/ncdump/ncpathcvt.c @@ -14,14 +14,14 @@ #ifdef HAVE_UNISTD_H #include #endif -#ifdef HAVE_GETOPT_H -#include -#endif #if defined(_WIN32) && !defined(__MINGW32__) #include "XGetopt.h" +#else +#include #endif +#include "netcdf.h" #include "ncpathmgr.h" /* @@ -42,7 +42,7 @@ Default is to convert to the format used by the platform. */ -#define DEBUG +#undef DEBUG struct Options { int target; @@ -54,12 +54,42 @@ struct Options { static char* escape(const char* path); static void usage(const char* msg); +static void +usage(const char* msg) +{ + if(msg != NULL) fprintf(stderr,"%s\n",msg); + fprintf(stderr,"pathcvt [-u|-w|-m|-c] PATH\n"); + if(msg == NULL) exit(0); else exit(1); +} + +static char* +escape(const char* path) +{ + size_t slen = strlen(path); + const char* p; + char* q; + char* epath = NULL; + const char* escapes = " \\"; + + epath = (char*)malloc((2*slen) + 1); + if(epath == NULL) usage("out of memtory"); + p = path; + q = epath; + for(;*p;p++) { + if(strchr(escapes,*p) != NULL) + *q++ = '\\'; + *q++ = *p; + } + *q = '\0'; + return epath; +} + int main(int argc, char** argv) { int c; char* cvtpath = NULL; - char* inpath; + char* inpath, *canon; memset((void*)&cvtoptions,0,sizeof(cvtoptions)); cvtoptions.drive = 'c'; @@ -91,47 +121,23 @@ main(int argc, char** argv) if (argc > 1) usage("more than one path specified"); inpath = argv[0]; + + /* Canonicalize */ + if(NCpathcanonical(inpath,&canon)) + usage("Could not convert to canonical form"); + if(cvtoptions.target == NCPD_UNKNOWN) - cvtpath = NCpathcvt(inpath); + cvtpath = NCpathcvt(canon); else - cvtpath = NCpathcvt_test(inpath,cvtoptions.target,(char)cvtoptions.drive); + cvtpath = NCpathcvt_test(canon,cvtoptions.target,(char)cvtoptions.drive); if(cvtpath && cvtoptions.escapes) { char* path = cvtpath; cvtpath = NULL; cvtpath = escape(path); free(path); } printf("%s",cvtpath); + if(canon) free(canon); if(cvtpath) free(cvtpath); return 0; } -static void -usage(const char* msg) -{ - if(msg != NULL) fprintf(stderr,"%s\n",msg); - fprintf(stderr,"pathcvt [-u|-w|-m|-c] PATH\n"); - if(msg == NULL) exit(0); else exit(1); -} - -static char* -escape(const char* path) -{ - size_t slen = strlen(path); - const char* p; - char* q; - char* epath = NULL; - const char* escapes = " \\"; - - epath = (char*)malloc((2*slen) + 1); - if(epath == NULL) usage("out of memtory"); - p = path; - q = epath; - for(;*p;p++) { - if(strchr(escapes,*p) != NULL) - *q++ = '\\'; - *q++ = *p; - } - *q = '\0'; - return epath; -} - diff --git a/ncdump/ncvalidator.c b/ncdump/ncvalidator.c index dead0d0368..4395723858 100644 --- a/ncdump/ncvalidator.c +++ b/ncdump/ncvalidator.c @@ -2274,7 +2274,8 @@ usage(char *argv0) fprintf(stderr, help, argv0); } -int main(int argc, char **argv) +int +main(int argc, char **argv) { char filename[512], *path; int i, omode, fd, status=NC_NOERR; diff --git a/ncdap_test/ref_pathcvt.txt b/ncdump/ref_pathcvt.txt similarity index 100% rename from ncdap_test/ref_pathcvt.txt rename to ncdump/ref_pathcvt.txt diff --git a/ncdap_test/testpathcvt.sh b/ncdump/testpathcvt.sh similarity index 91% rename from ncdap_test/testpathcvt.sh rename to ncdump/testpathcvt.sh index 9c1e2aedcb..88f952c5e5 100755 --- a/ncdap_test/testpathcvt.sh +++ b/ncdump/testpathcvt.sh @@ -9,7 +9,8 @@ testcase1() { T="$1" P="$2" echo -n "path: $T: |$P| => |" >>tmp_pathcvt.txt -${execdir}/pathcvt "$T" -e "$P" >>tmp_pathcvt.txt + +${NCPATHCVT} "$T" -e "$P" >>tmp_pathcvt.txt echo "|" >> tmp_pathcvt.txt } @@ -23,8 +24,6 @@ testcase() { rm -f tmp_pathcvt.txt -set -x - testcase "/xxx/x/y" testcase "d:/x/y" testcase "/cygdrive/d/x/y" diff --git a/ncdump/utils.c b/ncdump/utils.c index 1dd24819b8..deb22e23e3 100644 --- a/ncdump/utils.c +++ b/ncdump/utils.c @@ -174,6 +174,41 @@ print_name(const char* name) { free(ename); } +/* + * Returns malloced string with selected chars escaped. + * Caller should free result when done with it. + */ +char* +escaped_string(const char* cp) { + char *ret; /* string returned */ + char *sp; + assert(cp != NULL); + + /* For some reason, and on some machines (e.g. tweety) + utf8 characters such as \343 are considered control character. */ + + ret = emalloc(4*strlen(cp) + 1); /* max if every char escaped */ + sp = ret; + *sp = 0; /* empty name OK */ + for (; *cp; cp++) { + if (isascii((int)*cp)) { + if(iscntrl((int)*cp)) { /* render control chars as two hex digits, \%xx */ + snprintf(sp, 4+1,"\\%%%.2x", *cp); + sp += 4; + } else if(*cp == '"') { + *sp++ = '\\'; + *sp++ = '"'; + } else + *sp++ = *cp; + } else { /* not ascii, assume just UTF-8 byte */ + *sp++ = *cp; + } + } + *sp = 0; + return ret; +} + + /* Convert a full path name to a group to the specific groupid. */ int nc_inq_grpid2(int ncid, const char *grpname0, int *grpidp) @@ -923,3 +958,4 @@ parseFQN(int ncid, const char* fqn0, VarID* idp) return stat; } #endif + diff --git a/ncdump/utils.h b/ncdump/utils.h index 4d187920ca..d27821c65b 100644 --- a/ncdump/utils.h +++ b/ncdump/utils.h @@ -89,6 +89,9 @@ extern void check(int err, const char* file, const int line); /* Return malloced name with chars special to CDL escaped. */ char* escaped_name(const char* cp); +/* Return malloced string with selected chars escaped. */ +char* escaped_string(const char* cp); + /* Print name of netCDF var, dim, att, group, type, member, or enum * symbol with escaped special chars */ void print_name(const char *name); diff --git a/ncgen/ncgen.h b/ncgen/ncgen.h index e93f4627b4..855d39ce84 100644 --- a/ncgen/ncgen.h +++ b/ncgen/ncgen.h @@ -79,6 +79,7 @@ various C global variables #define _SUPERBLOCK_FLAG 0x400 #define _FORMAT_FLAG 0x800 #define _FILTER_FLAG 0x1000 +#define _CODECS_FLAG 0x2000 extern struct Specialtoken { char* name; @@ -123,6 +124,7 @@ typedef struct Specialdata { int _Fill ; /* 0 => false, 1 => true WATCHOUT: this is inverse of NOFILL*/ NC_H5_Filterspec** _Filters; size_t nfilters; /* |filters| */ + char* _Codecs; /* in JSON form */ } Specialdata; typedef struct GlobalSpecialdata { diff --git a/ncgen/ncgen.l b/ncgen/ncgen.l index 821b0e38fa..c97df69d40 100644 --- a/ncgen/ncgen.l +++ b/ncgen/ncgen.l @@ -139,6 +139,7 @@ struct Specialtoken specials[] = { {"_IsNetcdf4",_ISNETCDF4,_ISNETCDF4_FLAG}, {"_SuperblockVersion",_SUPERBLOCK,_SUPERBLOCK_FLAG}, {"_Filter",_FILTER,_FILTER_FLAG}, +{"_Codecs",_CODECS,_CODECS_FLAG}, {NULL,0} /* null terminate */ }; @@ -217,7 +218,7 @@ NUMBER [+-]?[0-9][0-9]*[Uu]?([BbSs]|[Ll]|[Ll][Ll])? DBLNUMBER [+-]?[0-9]*\.[0-9]*{exp}?[LlDd]?|[+-]?[0-9]*{exp}[LlDd]? FLTNUMBER [+-]?[0-9]*\.[0-9]*{exp}?[Ff]|[+-]?[0-9]*{exp}[Ff] -SPECIAL "_FillValue"|"_Format"|"_Storage"|"_ChunkSizes"|"_Fletcher32"|"_DeflateLevel"|"_Shuffle"|"_Endianness"|"_NoFill"|"_NCProperties"|"_IsNetcdf4"|"_SuperblockVersion"|"_Filter" +SPECIAL "_FillValue"|"_Format"|"_Storage"|"_ChunkSizes"|"_Fletcher32"|"_DeflateLevel"|"_Shuffle"|"_Endianness"|"_NoFill"|"_NCProperties"|"_IsNetcdf4"|"_SuperblockVersion"|"_Filter"|"_Codecs" USASCII [\x01-\x7F] diff --git a/ncgen/ncgen.y b/ncgen/ncgen.y index c53b9822d8..feba80248e 100644 --- a/ncgen/ncgen.y +++ b/ncgen/ncgen.y @@ -6,7 +6,7 @@ /* yacc source for "ncgen", a netCDL parser and netCDF generator */ -%error-verbose +%define parse.error verbose %{ /* @@ -130,6 +130,7 @@ static void vercheck(int ncid); static long long extractint(NCConstant* con); #ifdef USE_NETCDF4 static int parsefilterflag(const char* sdata0, Specialdata* special); +static int parsecodecsflag(const char* sdata0, Specialdata* special); #ifdef GENDEBUG1 static void printfilters(int nfilters, NC_ParsedFilterSpec** filters); #endif @@ -212,6 +213,7 @@ NCConstant* constant; _ISNETCDF4 _SUPERBLOCK _FILTER + _CODECS DATASETID %type ident typename primtype dimd varspec @@ -765,6 +767,8 @@ attrdecl: {$$ = makespecial(_ENDIAN_FLAG,$1,NULL,(void*)$5,ISCONST);} | ambiguous_ref ':' _FILTER '=' conststring {$$ = makespecial(_FILTER_FLAG,$1,NULL,(void*)$5,ISCONST);} + | ambiguous_ref ':' _CODECS '=' conststring + {$$ = makespecial(_CODECS_FLAG,$1,NULL,(void*)$5,ISCONST);} | ambiguous_ref ':' _NOFILL '=' constbool {$$ = makespecial(_NOFILL_FLAG,$1,NULL,(void*)$5,ISCONST);} | ':' _FORMAT '=' conststring @@ -1212,6 +1216,7 @@ makespecial(int tag, Symbol* vsym, Symbol* tsym, void* data, int isconst) case _NCPROPS_FLAG: case _ENDIAN_FLAG: case _FILTER_FLAG: + case _CODECS_FLAG: tmp = nullconst(); tmp->nctype = NC_STRING; convert1(con,tmp); @@ -1370,6 +1375,18 @@ makespecial(int tag, Symbol* vsym, Symbol* tsym, void* data, int isconst) } #else derror("%s: the filter attribute requires netcdf-4 to be enabled",specialname(tag)); +#endif + break; + case _CODECS_FLAG: +#ifdef USE_NETCDF4 + /* Parse the codec spec */ + if(parsecodecsflag(sdata,special) == NC_NOERR) + special->flags |= _CODECS_FLAG; + else { + derror("_Codecs: unparsable codec spec: %s",sdata); + } +#else + derror("%s: the _Codecs attribute requires netcdf-4 to be enabled",specialname(tag)); #endif break; default: PANIC1("makespecial: illegal token: %d",tag); @@ -1498,6 +1515,21 @@ printfilters(special->nfilters,special->_Filters); #endif return stat; } + +/* +Store a Codecs spec string in special +*/ +static int +parsecodecsflag(const char* sdata, Specialdata* special) +{ + int stat = NC_NOERR; + + if(sdata == NULL || strlen(sdata) == 0) return NC_EINVAL; + + if((special->_Codecs = strdup(sdata))==NULL) + return NC_ENOMEM; + return stat; +} #endif /* diff --git a/ncgen/ncgenl.c b/ncgen/ncgenl.c index 4e4649ce24..a52ae3f071 100644 --- a/ncgen/ncgenl.c +++ b/ncgen/ncgenl.c @@ -619,7 +619,7 @@ struct yy_trans_info flex_int32_t yy_verify; flex_int32_t yy_nxt; }; -static const flex_int16_t yy_accept[425] = +static const flex_int16_t yy_accept[429] = { 0, 0, 0, 51, 51, 0, 0, 55, 53, 1, 49, 53, 53, 53, 53, 43, 37, 41, 41, 40, 40, @@ -640,34 +640,34 @@ static const flex_int16_t yy_accept[425] = 40, 40, 40, 40, 40, 40, 36, 33, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, - 13, 40, 33, 40, 40, 40, 40, 40, 40, 40, - 40, 40, 40, 40, 40, 0, 48, 0, 0, 43, + 40, 13, 40, 33, 40, 40, 40, 40, 40, 40, + 40, 40, 40, 40, 40, 40, 0, 48, 0, 0, - 0, 37, 0, 0, 0, 0, 0, 0, 0, 4, - 42, 42, 0, 40, 40, 34, 40, 40, 35, 40, + 43, 0, 37, 0, 0, 0, 0, 0, 0, 0, + 4, 42, 42, 0, 40, 40, 34, 40, 40, 35, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, - 40, 40, 11, 10, 40, 40, 40, 40, 6, 40, - 40, 40, 40, 21, 40, 40, 40, 20, 40, 40, - 40, 40, 40, 16, 40, 40, 40, 40, 0, 0, - 34, 0, 37, 0, 0, 0, 0, 0, 0, 0, + 40, 40, 40, 40, 11, 10, 40, 40, 40, 40, + 6, 40, 40, 40, 40, 21, 40, 40, 40, 20, + 40, 40, 40, 40, 40, 16, 40, 40, 40, 40, + 0, 0, 34, 0, 37, 0, 0, 0, 0, 0, + 0, 0, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, - 40, 40, 40, 40, 40, 40, 40, 40, 40, 29, - 40, 40, 8, 40, 17, 40, 40, 40, 40, 12, + 40, 40, 29, 40, 40, 8, 40, 17, 40, 40, - 40, 40, 40, 14, 40, 40, 23, 40, 40, 40, - 46, 47, 0, 0, 0, 0, 40, 40, 40, 31, + 40, 40, 12, 40, 40, 40, 14, 40, 40, 23, + 40, 40, 40, 46, 47, 0, 0, 0, 0, 40, + 40, 40, 31, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, - 40, 40, 40, 40, 40, 40, 9, 30, 40, 7, - 19, 5, 26, 18, 40, 40, 15, 40, 0, 0, - 40, 40, 40, 40, 40, 40, 40, 40, 38, 40, - 40, 40, 40, 40, 40, 40, 40, 22, 40, 40, - 40, 40, 0, 40, 32, 40, 40, 40, 40, 40, - 40, 40, 40, 40, 5, 40, 40, 24, 40, 40, - 32, 32, 25, 40, 40, 40, 40, 40, 40, 40, + 9, 30, 40, 7, 19, 5, 26, 18, 40, 40, + 15, 40, 0, 0, 40, 40, 40, 40, 40, 38, + 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, + 40, 22, 40, 40, 40, 40, 0, 40, 32, 40, + 40, 40, 40, 40, 40, 40, 40, 40, 5, 40, + 40, 24, 40, 40, 32, 32, 25, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, - 28, 40, 40, 40, 27, 40, 40, 40, 40, 40, - 40, 40, 40, 0 + 40, 40, 40, 40, 28, 40, 40, 40, 27, 40, + 40, 40, 40, 40, 40, 40, 40, 0 } ; static const YY_CHAR yy_ec[256] = @@ -713,113 +713,113 @@ static const YY_CHAR yy_meta[69] = 11, 11, 11, 14, 1, 11, 11, 11 } ; -static const flex_int16_t yy_base[443] = +static const flex_int16_t yy_base[447] = { 0, - 0, 0, 325, 321, 264, 255, 318, 2461, 67, 2461, + 0, 0, 325, 321, 264, 255, 318, 2458, 67, 2458, 64, 269, 61, 62, 95, 77, 136, 259, 51, 61, 188, 97, 118, 150, 65, 195, 233, 153, 183, 222, 241, 202, 207, 213, 238, 246, 243, 257, 268, 264, - 298, 276, 221, 219, 218, 270, 236, 0, 2461, 79, - 87, 2461, 244, 238, 344, 0, 206, 358, 190, 0, - 2461, 370, 2461, 2461, 0, 342, 377, 177, 175, 174, - 200, 2461, 54, 377, 0, 254, 397, 171, 170, 169, + 298, 276, 221, 219, 218, 270, 236, 0, 2458, 79, + 87, 2458, 244, 238, 344, 0, 206, 358, 190, 0, + 2458, 370, 2458, 2458, 0, 342, 377, 177, 175, 174, + 200, 2458, 54, 377, 0, 254, 397, 171, 170, 169, 358, 85, 404, 373, 376, 398, 391, 406, 412, 417, - 421, 428, 432, 451, 454, 443, 473, 476, 486, 489, - - 494, 499, 511, 508, 520, 530, 546, 542, 550, 553, - 556, 560, 563, 566, 586, 598, 602, 605, 608, 623, - 612, 644, 647, 168, 167, 222, 217, 2461, 0, 2461, - 221, 680, 219, 703, 710, 179, 728, 749, 0, 691, - 666, 765, 159, 158, 135, 130, 128, 651, 125, 123, - 739, 716, 721, 751, 764, 770, 757, 775, 783, 760, - 794, 800, 806, 809, 813, 824, 820, 844, 839, 830, - 854, 850, 862, 869, 874, 886, 904, 893, 908, 917, - 947, 934, 912, 964, 943, 954, 900, 957, 973, 969, - 987, 990, 995, 1006, 999, 118, 2461, 735, 0, 2461, - - 50, 1030, 1062, 117, 115, 112, 110, 108, 104, 904, - 72, 2461, 103, 1020, 1025, 1043, 1046, 1064, 1051, 1069, - 1076, 1055, 1085, 1088, 1099, 1094, 1118, 1102, 1125, 1132, - 1137, 1142, 1149, 1156, 1167, 1162, 1172, 1175, 1179, 1192, - 1197, 1210, 1214, 1205, 1223, 1230, 1227, 1217, 1240, 1247, - 1262, 1253, 1265, 1298, 1278, 1283, 1295, 1301, 160, 154, - 2461, 107, 1314, 1350, 93, 91, 77, 73, 72, 70, - 1332, 1344, 1335, 1320, 1356, 1340, 1352, 1374, 1388, 1391, - 1366, 1396, 1406, 1412, 1409, 1422, 1428, 1442, 1445, 2461, - 1448, 1452, 1459, 1462, 1465, 1478, 1495, 1501, 1498, 1482, - - 1512, 1515, 1518, 1521, 1536, 1551, 1533, 1556, 1567, 1570, - 2461, 2461, 85, 65, 59, 36, 1581, 1574, 1586, 1589, - 1604, 1612, 1622, 1629, 1619, 1626, 1636, 1652, 1662, 1659, - 1670, 1673, 1676, 1692, 1696, 1706, 1709, 2461, 1712, 1715, - 1718, 1727, 2461, 1722, 1752, 1757, 1732, 1762, 39, 27, - 1768, 1765, 1775, 1783, 1778, 1808, 1799, 1817, 1820, 1824, - 1833, 1850, 1857, 1864, 1854, 1867, 1873, 1876, 1880, 1897, - 1906, 1910, 24, 1913, 1921, 1931, 1927, 1945, 1957, 1961, - 1952, 1964, 1969, 1994, 1975, 1999, 2005, 2008, 2012, 2019, - 36, 2015, 2024, 2049, 2054, 2045, 2057, 2079, 2087, 2068, - - 2062, 2075, 2099, 2092, 2109, 2105, 2117, 2112, 2142, 2129, - 2461, 2148, 2155, 2151, 2461, 2135, 2166, 2159, 2172, 2189, - 2192, 2196, 2212, 2461, 2280, 2294, 2308, 2322, 2331, 2340, - 2349, 2362, 2376, 2389, 2403, 2413, 2419, 2427, 2429, 2435, - 2441, 2447 + 421, 428, 432, 451, 454, 458, 488, 484, 447, 468, + + 494, 471, 501, 506, 510, 525, 532, 536, 541, 544, + 557, 562, 576, 579, 567, 583, 593, 600, 588, 618, + 613, 626, 630, 168, 167, 222, 217, 2458, 0, 2458, + 221, 688, 219, 694, 701, 179, 719, 740, 0, 709, + 683, 774, 159, 158, 135, 130, 128, 710, 125, 123, + 721, 730, 733, 744, 756, 768, 751, 775, 764, 786, + 789, 781, 800, 811, 806, 819, 831, 826, 842, 822, + 852, 845, 856, 864, 878, 875, 889, 896, 900, 908, + 911, 941, 922, 926, 957, 931, 944, 962, 966, 975, + 978, 988, 982, 1012, 999, 1019, 118, 2458, 1056, 0, + + 2458, 50, 1024, 1080, 117, 115, 112, 110, 108, 104, + 1030, 72, 2458, 103, 1038, 1043, 1061, 1064, 1076, 1068, + 1081, 1084, 1094, 1087, 1107, 1118, 1124, 1127, 1131, 1139, + 1161, 1165, 1169, 1173, 1148, 1178, 1184, 1181, 1203, 1214, + 1191, 1195, 1221, 1238, 1225, 1228, 1258, 1234, 1247, 1251, + 1264, 1269, 1273, 1288, 1299, 1303, 1306, 1295, 1311, 1318, + 160, 154, 2458, 107, 1321, 1386, 93, 91, 77, 73, + 72, 70, 1336, 1366, 1360, 1369, 1372, 1379, 1376, 1410, + 1413, 1416, 1419, 1422, 1427, 1432, 1453, 1457, 1463, 1471, + 1475, 1467, 2458, 1478, 1483, 1487, 1520, 1508, 1513, 1529, + + 1523, 1526, 1533, 1538, 1543, 1564, 1559, 1590, 1569, 1573, + 1579, 1583, 1613, 2458, 2458, 85, 65, 59, 36, 1594, + 1620, 1604, 1599, 1609, 1630, 1635, 1645, 1639, 1650, 1669, + 1661, 1655, 1676, 1685, 1681, 1694, 1691, 1724, 1702, 1707, + 1711, 2458, 1715, 1728, 1732, 1737, 2458, 1741, 1748, 1762, + 1745, 1767, 39, 27, 1771, 1780, 1783, 1797, 1792, 1786, + 1816, 1803, 1823, 1827, 1834, 1840, 1847, 1858, 1865, 1873, + 1880, 1870, 1888, 1896, 1883, 1913, 24, 1920, 1926, 1932, + 1938, 1943, 1929, 1952, 1964, 1968, 1976, 1983, 1986, 2002, + 2017, 1999, 2020, 2024, 36, 2006, 2009, 2050, 2054, 2039, + + 2057, 2042, 2062, 2072, 2076, 2087, 2093, 2098, 2102, 2109, + 2112, 2123, 2132, 2153, 2458, 2142, 2149, 2146, 2458, 2135, + 2165, 2179, 2168, 2172, 2188, 2203, 2209, 2458, 2277, 2291, + 2305, 2319, 2328, 2337, 2346, 2359, 2373, 2386, 2400, 2410, + 2416, 2424, 2426, 2432, 2438, 2444 } ; -static const flex_int16_t yy_def[443] = +static const flex_int16_t yy_def[447] = { 0, - 424, 1, 425, 425, 426, 426, 424, 424, 424, 424, - 427, 428, 424, 429, 424, 430, 424, 17, 431, 431, - 431, 431, 431, 431, 431, 424, 431, 431, 431, 431, - 21, 431, 431, 431, 431, 431, 431, 431, 431, 431, - 431, 431, 424, 424, 424, 432, 432, 433, 424, 424, - 427, 424, 427, 424, 434, 15, 17, 424, 424, 15, - 424, 424, 424, 424, 435, 436, 424, 424, 424, 424, - 17, 424, 424, 424, 437, 431, 424, 424, 424, 424, - 431, 21, 21, 431, 431, 431, 431, 431, 431, 431, - 431, 431, 431, 431, 431, 431, 431, 431, 431, 431, - - 431, 431, 431, 431, 431, 431, 431, 431, 431, 431, - 431, 431, 431, 431, 431, 431, 431, 431, 431, 431, - 431, 431, 431, 424, 424, 432, 432, 424, 433, 424, - 424, 424, 438, 424, 424, 424, 424, 424, 435, 436, - 439, 424, 424, 424, 424, 424, 424, 440, 424, 424, - 431, 431, 431, 431, 431, 431, 431, 431, 431, 431, - 431, 431, 431, 431, 431, 431, 431, 431, 431, 431, - 431, 431, 431, 431, 431, 431, 431, 431, 431, 431, - 431, 431, 431, 431, 431, 431, 431, 431, 431, 431, - 431, 431, 431, 431, 431, 424, 424, 424, 441, 424, - - 424, 442, 424, 424, 424, 424, 424, 424, 424, 440, - 424, 424, 424, 431, 431, 431, 431, 431, 431, 431, - 431, 431, 431, 431, 431, 431, 431, 431, 431, 431, - 431, 431, 431, 431, 431, 431, 431, 431, 431, 431, - 431, 431, 431, 431, 431, 431, 431, 431, 431, 431, - 431, 431, 431, 431, 431, 431, 431, 431, 424, 424, - 424, 424, 442, 424, 424, 424, 424, 424, 424, 424, - 431, 431, 431, 431, 431, 431, 431, 431, 431, 431, - 431, 431, 431, 431, 431, 431, 431, 431, 431, 424, - 431, 431, 431, 431, 431, 431, 431, 431, 431, 431, - - 431, 431, 431, 431, 431, 431, 431, 431, 431, 431, - 424, 424, 424, 424, 424, 424, 431, 431, 431, 431, - 431, 431, 431, 431, 431, 431, 431, 431, 431, 431, - 431, 431, 431, 431, 431, 431, 431, 424, 431, 431, - 431, 431, 424, 431, 431, 431, 431, 431, 424, 424, - 431, 431, 431, 431, 431, 431, 431, 431, 431, 431, - 431, 431, 431, 431, 431, 431, 431, 431, 431, 431, - 431, 431, 424, 431, 431, 431, 431, 431, 431, 431, - 431, 431, 431, 431, 431, 431, 431, 431, 431, 431, - 424, 431, 431, 431, 431, 431, 431, 431, 431, 431, - - 431, 431, 431, 431, 431, 431, 431, 431, 431, 431, - 424, 431, 431, 431, 424, 431, 431, 431, 431, 431, - 431, 431, 431, 0, 424, 424, 424, 424, 424, 424, - 424, 424, 424, 424, 424, 424, 424, 424, 424, 424, - 424, 424 + 428, 1, 429, 429, 430, 430, 428, 428, 428, 428, + 431, 432, 428, 433, 428, 434, 428, 17, 435, 435, + 435, 435, 435, 435, 435, 428, 435, 435, 435, 435, + 21, 435, 435, 435, 435, 435, 435, 435, 435, 435, + 435, 435, 428, 428, 428, 436, 436, 437, 428, 428, + 431, 428, 431, 428, 438, 15, 17, 428, 428, 15, + 428, 428, 428, 428, 439, 440, 428, 428, 428, 428, + 17, 428, 428, 428, 441, 435, 428, 428, 428, 428, + 435, 21, 21, 435, 435, 435, 435, 435, 435, 435, + 435, 435, 435, 435, 435, 435, 435, 435, 435, 435, + + 435, 435, 435, 435, 435, 435, 435, 435, 435, 435, + 435, 435, 435, 435, 435, 435, 435, 435, 435, 435, + 435, 435, 435, 428, 428, 436, 436, 428, 437, 428, + 428, 428, 442, 428, 428, 428, 428, 428, 439, 440, + 443, 428, 428, 428, 428, 428, 428, 444, 428, 428, + 435, 435, 435, 435, 435, 435, 435, 435, 435, 435, + 435, 435, 435, 435, 435, 435, 435, 435, 435, 435, + 435, 435, 435, 435, 435, 435, 435, 435, 435, 435, + 435, 435, 435, 435, 435, 435, 435, 435, 435, 435, + 435, 435, 435, 435, 435, 435, 428, 428, 428, 445, + + 428, 428, 446, 428, 428, 428, 428, 428, 428, 428, + 444, 428, 428, 428, 435, 435, 435, 435, 435, 435, + 435, 435, 435, 435, 435, 435, 435, 435, 435, 435, + 435, 435, 435, 435, 435, 435, 435, 435, 435, 435, + 435, 435, 435, 435, 435, 435, 435, 435, 435, 435, + 435, 435, 435, 435, 435, 435, 435, 435, 435, 435, + 428, 428, 428, 428, 446, 428, 428, 428, 428, 428, + 428, 428, 435, 435, 435, 435, 435, 435, 435, 435, + 435, 435, 435, 435, 435, 435, 435, 435, 435, 435, + 435, 435, 428, 435, 435, 435, 435, 435, 435, 435, + + 435, 435, 435, 435, 435, 435, 435, 435, 435, 435, + 435, 435, 435, 428, 428, 428, 428, 428, 428, 435, + 435, 435, 435, 435, 435, 435, 435, 435, 435, 435, + 435, 435, 435, 435, 435, 435, 435, 435, 435, 435, + 435, 428, 435, 435, 435, 435, 428, 435, 435, 435, + 435, 435, 428, 428, 435, 435, 435, 435, 435, 435, + 435, 435, 435, 435, 435, 435, 435, 435, 435, 435, + 435, 435, 435, 435, 435, 435, 428, 435, 435, 435, + 435, 435, 435, 435, 435, 435, 435, 435, 435, 435, + 435, 435, 435, 435, 428, 435, 435, 435, 435, 435, + + 435, 435, 435, 435, 435, 435, 435, 435, 435, 435, + 435, 435, 435, 435, 428, 435, 435, 435, 428, 435, + 435, 435, 435, 435, 435, 435, 435, 0, 428, 428, + 428, 428, 428, 428, 428, 428, 428, 428, 428, 428, + 428, 428, 428, 428, 428, 428 } ; -static const flex_int16_t yy_nxt[2530] = +static const flex_int16_t yy_nxt[2527] = { 0, 8, 9, 10, 9, 8, 11, 12, 8, 13, 14, 15, 16, 17, 18, 18, 18, 18, 18, 18, 8, @@ -829,279 +829,279 @@ static const flex_int16_t yy_nxt[2530] = 35, 19, 36, 37, 19, 19, 38, 39, 40, 41, 42, 19, 19, 8, 8, 43, 44, 45, 50, 52, 50, 56, 56, 57, 57, 57, 57, 57, 57, 57, - 50, 261, 50, 72, 64, 391, 58, 58, 65, 77, - 59, 263, 52, 76, 76, 261, 90, 373, 262, 77, + 50, 263, 50, 72, 64, 395, 58, 58, 65, 77, + 59, 265, 52, 76, 76, 263, 90, 377, 264, 77, - 202, 212, 53, 77, 72, 58, 58, 60, 60, 60, + 203, 213, 53, 77, 72, 58, 58, 60, 60, 60, 60, 60, 60, 60, 81, 67, 78, 79, 80, 61, - 62, 63, 212, 350, 61, 53, 78, 79, 80, 263, - 78, 79, 80, 349, 140, 77, 316, 202, 61, 62, - 63, 315, 68, 69, 70, 61, 56, 84, 71, 71, - 71, 71, 71, 71, 71, 314, 77, 263, 72, 313, - 312, 58, 78, 79, 80, 73, 311, 76, 66, 72, - 85, 74, 270, 75, 140, 86, 269, 72, 87, 268, - 58, 202, 91, 78, 79, 80, 73, 213, 77, 76, - 88, 77, 209, 72, 66, 74, 82, 82, 89, 208, + 62, 63, 213, 354, 61, 53, 78, 79, 80, 265, + 78, 79, 80, 353, 140, 77, 319, 203, 61, 62, + 63, 318, 68, 69, 70, 61, 56, 84, 71, 71, + 71, 71, 71, 71, 71, 317, 77, 265, 72, 316, + 315, 58, 78, 79, 80, 73, 314, 76, 66, 72, + 85, 74, 272, 75, 140, 86, 271, 72, 87, 270, + 58, 203, 91, 78, 79, 80, 73, 214, 77, 76, + 88, 77, 210, 72, 66, 74, 82, 82, 89, 209, 83, 83, 83, 83, 83, 83, 83, 91, 91, 91, 91, 91, 91, 91, 99, 78, 79, 80, 78, 79, - 80, 77, 207, 140, 201, 197, 77, 197, 424, 127, - 100, 196, 91, 150, 149, 76, 101, 424, 147, 146, - 77, 66, 136, 424, 130, 77, 424, 128, 78, 79, + 80, 77, 208, 140, 202, 198, 77, 198, 428, 127, + 100, 197, 91, 150, 149, 76, 101, 428, 147, 146, + 77, 66, 136, 428, 130, 77, 428, 128, 78, 79, 80, 77, 106, 78, 79, 80, 92, 93, 94, 95, 77, 96, 102, 107, 97, 108, 98, 78, 79, 80, 103, 77, 78, 79, 80, 104, 77, 127, 78, 79, 80, 77, 125, 124, 77, 91, 110, 78, 79, 80, - 111, 109, 77, 105, 112, 77, 424, 113, 78, 79, + 111, 109, 77, 105, 112, 77, 428, 113, 78, 79, 80, 114, 77, 78, 79, 80, 77, 55, 78, 79, - 80, 78, 79, 80, 77, 115, 123, 424, 49, 78, + 80, 78, 79, 80, 77, 115, 123, 428, 49, 78, 79, 80, 78, 79, 80, 117, 116, 49, 47, 78, - 79, 80, 47, 78, 79, 80, 77, 424, 424, 118, - 424, 78, 79, 80, 424, 424, 119, 424, 120, 424, - 121, 424, 424, 141, 424, 122, 132, 132, 132, 132, - 132, 132, 424, 78, 79, 80, 134, 134, 424, 424, - 135, 135, 135, 135, 135, 135, 135, 424, 137, 137, + 79, 80, 47, 78, 79, 80, 77, 428, 428, 118, + 428, 78, 79, 80, 428, 428, 119, 428, 120, 428, + 121, 428, 428, 141, 428, 122, 132, 132, 132, 132, + 132, 132, 428, 78, 79, 80, 134, 134, 428, 428, + 135, 135, 135, 135, 135, 135, 135, 428, 137, 137, 142, 133, 138, 138, 138, 138, 138, 138, 138, 66, - 66, 66, 66, 66, 66, 66, 77, 424, 424, 72, - - 76, 76, 76, 76, 76, 424, 73, 143, 144, 145, - 72, 77, 76, 76, 77, 424, 76, 151, 72, 424, - 157, 155, 424, 78, 79, 80, 154, 73, 152, 77, - 153, 424, 156, 152, 72, 76, 77, 158, 78, 79, - 80, 78, 79, 80, 77, 424, 159, 152, 424, 153, - 77, 424, 424, 424, 152, 77, 78, 79, 80, 77, - 76, 424, 157, 78, 79, 80, 77, 424, 424, 424, - 77, 78, 79, 80, 424, 160, 161, 78, 79, 80, - 424, 77, 78, 79, 80, 424, 78, 79, 80, 77, - 424, 424, 77, 78, 79, 80, 167, 78, 79, 80, - - 166, 424, 163, 162, 164, 424, 424, 165, 78, 79, - 80, 77, 424, 424, 77, 424, 78, 79, 80, 78, - 79, 80, 424, 169, 77, 424, 168, 77, 424, 173, - 424, 424, 77, 424, 170, 171, 424, 77, 78, 79, - 80, 78, 79, 80, 172, 174, 77, 424, 424, 77, - 424, 78, 79, 80, 78, 79, 80, 175, 77, 78, - 79, 80, 176, 424, 78, 79, 80, 177, 77, 424, - 424, 424, 424, 78, 79, 80, 78, 79, 80, 178, - 77, 424, 424, 179, 77, 78, 79, 80, 77, 424, - 424, 77, 424, 424, 77, 78, 79, 80, 77, 180, - - 181, 77, 182, 185, 77, 183, 186, 78, 79, 80, - 157, 78, 79, 80, 184, 78, 79, 80, 78, 79, - 80, 78, 79, 80, 77, 78, 79, 80, 78, 79, - 80, 78, 79, 80, 424, 424, 77, 424, 424, 187, - 77, 424, 424, 77, 424, 424, 77, 424, 424, 424, - 77, 78, 79, 80, 188, 424, 189, 424, 424, 424, - 191, 77, 193, 78, 79, 80, 190, 78, 79, 80, - 78, 79, 80, 78, 79, 80, 192, 78, 79, 80, - 211, 424, 77, 424, 212, 77, 197, 424, 78, 79, - 80, 194, 198, 198, 198, 198, 198, 198, 424, 424, - - 424, 211, 141, 195, 203, 424, 424, 424, 212, 78, - 79, 80, 78, 79, 80, 135, 135, 135, 135, 135, - 135, 135, 135, 135, 135, 135, 135, 135, 135, 142, - 424, 204, 205, 206, 200, 424, 63, 424, 424, 200, - 138, 138, 138, 138, 138, 138, 138, 259, 259, 259, - 259, 259, 259, 200, 77, 63, 143, 144, 145, 77, - 200, 138, 138, 138, 138, 138, 138, 138, 140, 140, - 140, 140, 140, 61, 424, 63, 424, 77, 61, 424, - 214, 78, 79, 80, 140, 424, 78, 79, 80, 77, - 424, 215, 61, 218, 63, 77, 424, 424, 77, 61, - - 424, 424, 77, 140, 78, 79, 80, 424, 77, 216, - 424, 220, 217, 77, 424, 424, 78, 79, 80, 221, - 219, 77, 78, 79, 80, 78, 79, 80, 140, 78, - 79, 80, 77, 424, 424, 78, 79, 80, 77, 222, - 78, 79, 80, 223, 77, 424, 424, 77, 78, 79, - 80, 77, 228, 225, 424, 227, 224, 424, 77, 78, - 79, 80, 77, 424, 424, 78, 79, 80, 77, 226, - 229, 78, 79, 80, 78, 79, 80, 77, 78, 79, - 80, 424, 77, 231, 424, 78, 79, 80, 77, 78, - 79, 80, 77, 424, 233, 78, 79, 80, 230, 424, - - 77, 424, 424, 424, 78, 79, 80, 77, 232, 78, - 79, 80, 77, 424, 236, 78, 79, 80, 234, 78, - 79, 80, 424, 235, 77, 424, 424, 78, 79, 80, - 237, 77, 424, 211, 78, 79, 80, 212, 77, 78, - 79, 80, 77, 424, 239, 238, 77, 424, 240, 424, - 77, 78, 79, 80, 211, 77, 249, 219, 78, 79, - 80, 212, 424, 424, 242, 78, 79, 80, 424, 78, - 79, 80, 77, 78, 79, 80, 241, 78, 79, 80, - 244, 77, 78, 79, 80, 77, 424, 245, 424, 424, - 424, 243, 77, 424, 424, 77, 424, 424, 247, 78, - - 79, 80, 77, 424, 248, 250, 246, 77, 78, 79, - 80, 77, 78, 79, 80, 424, 251, 252, 424, 78, - 79, 80, 78, 79, 80, 77, 424, 253, 77, 78, - 79, 80, 424, 77, 78, 79, 80, 77, 78, 79, - 80, 141, 255, 256, 77, 254, 424, 258, 424, 424, - 424, 424, 78, 79, 80, 78, 79, 80, 77, 257, - 78, 79, 80, 77, 78, 79, 80, 424, 264, 424, - 271, 78, 79, 80, 202, 202, 202, 202, 202, 202, - 202, 77, 424, 272, 77, 78, 79, 80, 274, 77, - 78, 79, 80, 77, 424, 265, 266, 267, 273, 275, - - 424, 424, 77, 424, 424, 277, 424, 77, 78, 79, - 80, 78, 79, 80, 77, 424, 78, 79, 80, 424, - 78, 79, 80, 77, 424, 424, 77, 424, 276, 78, - 79, 80, 77, 278, 78, 79, 80, 77, 279, 424, - 77, 78, 79, 80, 424, 282, 280, 424, 424, 424, - 78, 79, 80, 78, 79, 80, 77, 281, 284, 78, - 79, 80, 283, 77, 78, 79, 80, 78, 79, 80, - 77, 424, 424, 285, 424, 77, 424, 286, 424, 424, - 77, 290, 424, 78, 79, 80, 288, 77, 424, 424, - 78, 79, 80, 287, 77, 424, 424, 78, 79, 80, - - 77, 424, 78, 79, 80, 77, 424, 78, 79, 80, - 77, 424, 424, 77, 78, 79, 80, 77, 424, 424, - 289, 78, 79, 80, 291, 292, 295, 78, 79, 80, - 77, 424, 78, 79, 80, 77, 424, 78, 79, 80, - 78, 79, 80, 77, 78, 79, 80, 297, 77, 424, - 293, 294, 77, 424, 424, 77, 424, 78, 79, 80, - 296, 77, 78, 79, 80, 77, 424, 424, 77, 424, - 78, 79, 80, 298, 424, 78, 79, 80, 77, 78, - 79, 80, 78, 79, 80, 77, 299, 424, 78, 79, - 80, 77, 78, 79, 80, 78, 79, 80, 300, 301, - - 77, 424, 424, 77, 302, 78, 79, 80, 424, 304, - 303, 424, 78, 79, 80, 305, 77, 424, 78, 79, - 80, 77, 424, 424, 307, 141, 424, 78, 79, 80, - 78, 79, 80, 77, 308, 424, 77, 424, 424, 77, - 424, 310, 306, 78, 79, 80, 320, 424, 78, 79, - 80, 309, 264, 263, 263, 263, 263, 263, 77, 424, - 78, 79, 80, 78, 79, 80, 78, 79, 80, 263, - 77, 424, 318, 77, 424, 424, 317, 424, 77, 265, - 266, 267, 77, 319, 321, 78, 79, 80, 263, 322, - 77, 424, 323, 424, 77, 424, 424, 78, 79, 80, - - 78, 79, 80, 424, 77, 78, 79, 80, 327, 78, - 79, 80, 77, 263, 324, 424, 424, 78, 79, 80, - 424, 78, 79, 80, 325, 424, 77, 424, 424, 77, - 424, 78, 79, 80, 77, 326, 328, 424, 424, 78, - 79, 80, 424, 424, 77, 424, 424, 77, 424, 424, - 77, 424, 424, 78, 79, 80, 78, 79, 80, 331, - 77, 78, 79, 80, 329, 330, 77, 332, 333, 424, - 424, 78, 79, 80, 78, 79, 80, 78, 79, 80, - 77, 338, 424, 77, 424, 424, 77, 78, 79, 80, - 77, 424, 424, 78, 79, 80, 337, 77, 334, 424, - - 77, 424, 424, 77, 335, 336, 424, 78, 79, 80, - 78, 79, 80, 78, 79, 80, 77, 78, 79, 80, - 77, 320, 339, 424, 78, 79, 80, 78, 79, 80, - 78, 79, 80, 77, 424, 424, 77, 343, 424, 77, - 424, 424, 340, 78, 79, 80, 320, 78, 79, 80, - 77, 424, 344, 77, 424, 424, 77, 424, 341, 77, - 78, 79, 80, 78, 79, 80, 78, 79, 80, 424, - 424, 77, 424, 342, 77, 424, 424, 78, 79, 80, - 78, 79, 80, 78, 79, 80, 78, 79, 80, 77, - 424, 424, 424, 424, 77, 424, 424, 345, 78, 79, - - 80, 78, 79, 80, 346, 77, 424, 424, 77, 351, - 424, 348, 77, 424, 424, 424, 78, 79, 80, 77, - 424, 78, 79, 80, 77, 347, 352, 77, 424, 424, - 424, 424, 78, 79, 80, 78, 79, 80, 354, 78, - 79, 80, 77, 424, 353, 355, 78, 79, 80, 424, - 77, 78, 79, 80, 78, 79, 80, 77, 424, 358, - 77, 424, 424, 424, 77, 424, 424, 77, 424, 78, - 79, 80, 424, 424, 77, 424, 424, 78, 79, 80, - 356, 357, 359, 360, 78, 79, 80, 78, 79, 80, - 77, 78, 79, 80, 78, 79, 80, 77, 424, 424, - - 77, 78, 79, 80, 361, 424, 424, 424, 77, 424, - 359, 77, 424, 362, 77, 424, 424, 78, 79, 80, - 359, 424, 364, 363, 78, 79, 80, 78, 79, 80, - 77, 424, 424, 365, 77, 78, 79, 80, 78, 79, - 80, 78, 79, 80, 77, 424, 424, 77, 366, 424, - 77, 424, 424, 77, 367, 424, 77, 78, 79, 80, - 77, 78, 79, 80, 424, 77, 424, 424, 368, 424, + 66, 66, 66, 66, 66, 66, 77, 428, 428, 72, + + 76, 76, 76, 76, 76, 428, 73, 143, 144, 145, + 72, 77, 76, 76, 77, 428, 76, 151, 72, 428, + 157, 155, 428, 78, 79, 80, 154, 73, 152, 77, + 153, 428, 156, 152, 72, 76, 77, 158, 78, 79, + 80, 78, 79, 80, 77, 428, 159, 152, 428, 153, + 77, 428, 428, 428, 152, 77, 78, 79, 80, 77, + 76, 428, 157, 78, 79, 80, 77, 428, 428, 428, + 77, 78, 79, 80, 428, 160, 162, 78, 79, 80, + 428, 161, 78, 79, 80, 77, 78, 79, 80, 77, + 428, 428, 77, 78, 79, 80, 77, 78, 79, 80, + + 428, 428, 164, 163, 165, 173, 77, 166, 174, 77, + 428, 168, 78, 79, 80, 167, 78, 79, 80, 78, + 79, 80, 77, 78, 79, 80, 77, 428, 428, 176, + 428, 170, 77, 78, 79, 80, 78, 79, 80, 77, + 428, 169, 171, 172, 77, 175, 428, 428, 77, 78, + 79, 80, 177, 78, 79, 80, 428, 428, 428, 78, + 79, 80, 428, 77, 428, 178, 78, 79, 80, 179, + 77, 78, 79, 80, 77, 78, 79, 80, 180, 77, + 428, 428, 77, 428, 428, 181, 428, 428, 428, 428, + 78, 79, 80, 183, 182, 77, 184, 78, 79, 80, + + 77, 78, 79, 80, 428, 77, 78, 79, 80, 78, + 79, 80, 157, 428, 77, 185, 186, 77, 428, 187, + 188, 77, 78, 79, 80, 428, 77, 78, 79, 80, + 428, 77, 78, 79, 80, 428, 428, 428, 77, 189, + 192, 78, 79, 80, 78, 79, 80, 190, 78, 79, + 80, 77, 428, 78, 79, 80, 77, 428, 78, 79, + 80, 191, 428, 194, 77, 78, 79, 80, 77, 428, + 428, 193, 428, 195, 428, 428, 428, 428, 78, 79, + 80, 428, 428, 78, 79, 80, 196, 428, 428, 428, + 428, 78, 79, 80, 198, 78, 79, 80, 428, 428, + + 199, 199, 199, 199, 199, 199, 135, 135, 135, 135, + 135, 135, 135, 135, 135, 135, 135, 135, 135, 135, + 141, 204, 428, 428, 428, 201, 428, 63, 428, 428, + 201, 138, 138, 138, 138, 138, 138, 138, 428, 212, + 428, 428, 428, 213, 201, 428, 63, 142, 205, 206, + 207, 201, 138, 138, 138, 138, 138, 138, 138, 77, + 212, 428, 215, 428, 61, 428, 63, 213, 77, 61, + 428, 77, 428, 428, 143, 144, 145, 140, 140, 140, + 140, 140, 77, 61, 216, 63, 78, 79, 80, 77, + 61, 219, 221, 140, 77, 78, 79, 80, 78, 79, + + 80, 217, 77, 428, 218, 428, 77, 428, 428, 78, + 79, 80, 140, 77, 428, 428, 78, 79, 80, 77, + 220, 78, 79, 80, 77, 428, 224, 77, 428, 78, + 79, 80, 223, 78, 79, 80, 428, 140, 77, 428, + 78, 79, 80, 225, 77, 222, 78, 79, 80, 77, + 227, 78, 79, 80, 78, 79, 80, 77, 230, 428, + 77, 226, 229, 428, 77, 78, 79, 80, 231, 77, + 428, 78, 79, 80, 428, 228, 78, 79, 80, 428, + 77, 232, 428, 77, 78, 79, 80, 78, 79, 80, + 77, 78, 79, 80, 77, 428, 78, 79, 80, 234, + + 235, 428, 77, 428, 428, 233, 428, 78, 79, 80, + 78, 79, 80, 77, 428, 238, 77, 78, 79, 80, + 236, 78, 79, 80, 428, 428, 428, 77, 428, 78, + 79, 80, 237, 239, 77, 428, 428, 240, 77, 428, + 78, 79, 80, 78, 79, 80, 77, 428, 242, 77, + 428, 241, 428, 428, 78, 79, 80, 428, 244, 428, + 77, 78, 79, 80, 77, 78, 79, 80, 246, 77, + 243, 220, 428, 78, 79, 80, 78, 79, 80, 77, + 247, 428, 77, 428, 428, 245, 249, 78, 79, 80, + 428, 78, 79, 80, 250, 77, 78, 79, 80, 248, + + 77, 428, 428, 428, 77, 428, 78, 79, 80, 78, + 79, 80, 428, 77, 252, 428, 77, 428, 251, 254, + 77, 428, 78, 79, 80, 253, 77, 78, 79, 80, + 428, 78, 79, 80, 257, 141, 255, 77, 428, 428, + 78, 79, 80, 78, 79, 80, 256, 78, 79, 80, + 77, 428, 259, 78, 79, 80, 428, 77, 428, 212, + 258, 428, 266, 213, 78, 79, 80, 260, 261, 261, + 261, 261, 261, 261, 428, 428, 77, 78, 79, 80, + 212, 77, 428, 428, 78, 79, 80, 213, 273, 267, + 268, 269, 203, 203, 203, 203, 203, 203, 203, 77, + + 276, 274, 77, 78, 79, 80, 77, 428, 78, 79, + 80, 277, 428, 428, 77, 428, 275, 428, 428, 77, + 428, 428, 77, 428, 428, 77, 78, 79, 80, 78, + 79, 80, 77, 78, 79, 80, 278, 280, 279, 428, + 428, 78, 79, 80, 428, 77, 78, 79, 80, 78, + 79, 80, 78, 79, 80, 281, 77, 428, 428, 78, + 79, 80, 77, 428, 428, 77, 428, 428, 282, 77, + 428, 428, 78, 79, 80, 286, 283, 77, 285, 428, + 428, 428, 284, 78, 79, 80, 77, 428, 428, 78, + 79, 80, 78, 79, 80, 287, 78, 79, 80, 77, + + 293, 428, 428, 77, 78, 79, 80, 77, 428, 288, + 289, 77, 428, 78, 79, 80, 77, 291, 428, 77, + 428, 428, 77, 428, 428, 290, 78, 79, 80, 77, + 78, 79, 80, 77, 78, 79, 80, 292, 78, 79, + 80, 77, 428, 78, 79, 80, 78, 79, 80, 78, + 79, 80, 77, 296, 298, 294, 78, 79, 80, 77, + 78, 79, 80, 77, 295, 428, 77, 428, 78, 79, + 80, 299, 77, 428, 428, 297, 77, 301, 428, 78, + 79, 80, 300, 428, 428, 77, 78, 79, 80, 77, + 78, 79, 80, 78, 79, 80, 77, 428, 428, 78, + + 79, 80, 77, 78, 79, 80, 302, 77, 428, 428, + 428, 77, 78, 79, 80, 305, 78, 79, 80, 428, + 308, 304, 303, 78, 79, 80, 77, 428, 428, 78, + 79, 80, 141, 77, 78, 79, 80, 77, 78, 79, + 80, 77, 428, 307, 77, 306, 311, 309, 428, 77, + 428, 428, 310, 78, 79, 80, 77, 428, 313, 266, + 78, 79, 80, 428, 78, 79, 80, 312, 78, 79, + 80, 78, 79, 80, 77, 428, 78, 79, 80, 428, + 320, 428, 428, 78, 79, 80, 267, 268, 269, 265, + 265, 265, 265, 265, 321, 323, 428, 428, 77, 428, + + 324, 78, 79, 80, 77, 265, 428, 77, 322, 428, + 77, 428, 428, 428, 77, 428, 428, 77, 326, 428, + 428, 428, 428, 428, 265, 78, 79, 80, 325, 428, + 428, 78, 79, 80, 78, 79, 80, 78, 79, 80, + 428, 78, 79, 80, 78, 79, 80, 428, 77, 265, + 327, 77, 329, 328, 77, 428, 428, 77, 428, 428, + 77, 428, 428, 330, 331, 77, 428, 332, 428, 428, + 77, 428, 428, 428, 428, 78, 79, 80, 78, 79, + 80, 78, 79, 80, 78, 79, 80, 78, 79, 80, + 333, 77, 78, 79, 80, 77, 428, 78, 79, 80, + + 428, 77, 428, 428, 428, 77, 334, 335, 336, 77, + 428, 337, 428, 77, 428, 428, 77, 428, 78, 79, + 80, 77, 78, 79, 80, 77, 339, 341, 78, 79, + 80, 338, 78, 79, 80, 340, 78, 79, 80, 342, + 78, 79, 80, 78, 79, 80, 77, 428, 78, 79, + 80, 77, 78, 79, 80, 323, 428, 343, 77, 428, + 428, 77, 428, 428, 77, 428, 428, 77, 323, 428, + 344, 77, 428, 78, 79, 80, 77, 428, 78, 79, + 80, 77, 428, 347, 345, 78, 79, 80, 78, 79, + 80, 78, 79, 80, 78, 79, 80, 77, 78, 79, + + 80, 346, 77, 78, 79, 80, 348, 77, 78, 79, + 80, 77, 428, 428, 428, 349, 428, 77, 428, 428, + 428, 77, 355, 428, 78, 79, 80, 350, 77, 78, + 79, 80, 77, 428, 78, 79, 80, 77, 78, 79, + 80, 351, 77, 358, 78, 79, 80, 77, 78, 79, + 80, 77, 428, 428, 352, 78, 79, 80, 77, 78, + 79, 80, 357, 359, 78, 79, 80, 428, 77, 78, + 79, 80, 356, 77, 78, 79, 80, 77, 78, 79, + 80, 428, 428, 77, 428, 78, 79, 80, 77, 428, + 363, 362, 360, 77, 428, 78, 79, 80, 428, 77, + + 78, 79, 80, 361, 78, 79, 80, 77, 364, 428, + 78, 79, 80, 360, 77, 78, 79, 80, 365, 77, + 78, 79, 80, 77, 428, 360, 78, 79, 80, 77, + 428, 360, 77, 428, 78, 79, 80, 368, 428, 366, + 77, 78, 79, 80, 367, 77, 78, 79, 80, 77, + 78, 79, 80, 77, 370, 371, 78, 79, 80, 78, + 79, 80, 77, 428, 428, 369, 77, 78, 79, 80, + 77, 372, 78, 79, 80, 77, 78, 79, 80, 77, + 78, 79, 80, 77, 428, 428, 77, 428, 428, 78, + 79, 80, 374, 78, 79, 80, 373, 78, 79, 80, + + 77, 428, 78, 79, 80, 77, 78, 79, 80, 77, + 78, 79, 80, 78, 79, 80, 428, 376, 77, 428, + 375, 77, 380, 378, 77, 217, 428, 78, 79, 80, + 77, 428, 78, 79, 80, 77, 78, 79, 80, 428, + 381, 77, 428, 428, 379, 78, 79, 80, 78, 79, + 80, 78, 79, 80, 77, 383, 428, 78, 79, 80, + 382, 77, 78, 79, 80, 77, 428, 428, 78, 79, + 80, 385, 77, 384, 428, 428, 428, 386, 77, 428, + 428, 78, 79, 80, 387, 77, 428, 428, 78, 79, + 80, 360, 78, 79, 80, 428, 77, 428, 428, 78, + + 79, 80, 360, 77, 428, 78, 79, 80, 77, 428, + 428, 77, 78, 79, 80, 388, 389, 428, 77, 428, + 428, 77, 428, 78, 79, 80, 77, 393, 428, 428, + 78, 79, 80, 390, 77, 78, 79, 80, 78, 79, + 80, 428, 428, 428, 391, 78, 79, 80, 78, 79, + 80, 77, 392, 78, 79, 80, 397, 394, 77, 428, + 428, 78, 79, 80, 77, 396, 428, 77, 428, 428, + 77, 217, 399, 400, 428, 428, 77, 428, 78, 79, + 80, 77, 428, 428, 428, 78, 79, 80, 428, 428, 77, 78, 79, 80, 78, 79, 80, 78, 79, 80, - 78, 79, 80, 78, 79, 80, 369, 78, 79, 80, - 77, 424, 78, 79, 80, 77, 370, 78, 79, 80, - - 77, 424, 424, 77, 424, 424, 77, 424, 376, 424, - 216, 424, 372, 77, 424, 371, 77, 78, 79, 80, - 374, 77, 78, 79, 80, 424, 377, 78, 79, 80, - 78, 79, 80, 78, 79, 80, 375, 77, 424, 424, - 78, 79, 80, 78, 79, 80, 77, 424, 78, 79, - 80, 379, 378, 424, 424, 77, 424, 424, 77, 424, - 424, 424, 77, 424, 78, 79, 80, 380, 381, 424, - 424, 77, 424, 78, 79, 80, 382, 424, 424, 424, - 424, 424, 78, 79, 80, 78, 79, 80, 77, 78, - 79, 80, 77, 424, 383, 77, 424, 424, 78, 79, - - 80, 359, 77, 424, 384, 77, 424, 424, 359, 424, - 385, 77, 424, 424, 77, 78, 79, 80, 77, 78, - 79, 80, 78, 79, 80, 424, 386, 424, 424, 78, - 79, 80, 78, 79, 80, 77, 387, 424, 78, 79, - 80, 78, 79, 80, 77, 78, 79, 80, 77, 424, - 389, 77, 424, 388, 390, 393, 424, 424, 392, 77, - 424, 424, 78, 79, 80, 77, 216, 424, 424, 77, - 424, 78, 79, 80, 395, 78, 79, 80, 78, 79, - 80, 424, 424, 77, 424, 424, 78, 79, 80, 394, - 77, 424, 78, 79, 80, 77, 78, 79, 80, 77, - - 424, 396, 77, 424, 424, 424, 424, 77, 398, 399, - 78, 79, 80, 77, 424, 424, 424, 78, 79, 80, - 397, 424, 78, 79, 80, 400, 78, 79, 80, 78, - 79, 80, 77, 424, 78, 79, 80, 77, 424, 424, - 78, 79, 80, 77, 424, 424, 77, 401, 424, 385, - 77, 402, 424, 77, 424, 393, 424, 77, 424, 78, - 79, 80, 77, 424, 78, 79, 80, 424, 424, 424, - 78, 79, 80, 78, 79, 80, 403, 78, 79, 80, - 78, 79, 80, 77, 78, 79, 80, 77, 424, 78, - 79, 80, 77, 404, 407, 77, 424, 424, 405, 424, - - 77, 359, 406, 359, 409, 424, 77, 424, 424, 424, - 78, 79, 80, 77, 78, 79, 80, 77, 411, 78, - 79, 80, 78, 79, 80, 77, 408, 78, 79, 80, - 77, 359, 410, 78, 79, 80, 424, 77, 424, 424, - 78, 79, 80, 77, 78, 79, 80, 77, 415, 359, - 77, 424, 78, 79, 80, 77, 424, 78, 79, 80, - 413, 424, 359, 424, 78, 79, 80, 77, 424, 412, - 78, 79, 80, 77, 78, 79, 80, 78, 79, 80, - 77, 424, 78, 79, 80, 359, 77, 418, 424, 77, - 424, 414, 416, 77, 78, 79, 80, 77, 424, 417, - - 78, 79, 80, 419, 77, 424, 424, 78, 79, 80, - 77, 424, 424, 78, 79, 80, 78, 79, 80, 424, - 78, 79, 80, 359, 78, 79, 80, 77, 420, 424, - 77, 78, 79, 80, 77, 424, 424, 78, 79, 80, - 422, 424, 424, 424, 424, 424, 421, 424, 424, 423, - 77, 424, 424, 424, 78, 79, 80, 78, 79, 80, - 424, 78, 79, 80, 359, 424, 424, 424, 424, 424, - 424, 424, 424, 424, 424, 424, 424, 78, 79, 80, + + 398, 428, 77, 78, 79, 80, 77, 428, 78, 79, + 80, 401, 428, 403, 77, 428, 428, 78, 79, 80, + 402, 77, 428, 428, 77, 428, 428, 428, 428, 78, + 79, 80, 404, 78, 79, 80, 405, 77, 428, 428, + 77, 78, 79, 80, 77, 428, 428, 77, 78, 79, + 80, 78, 79, 80, 406, 77, 428, 411, 77, 428, + 428, 389, 77, 397, 78, 79, 80, 78, 79, 80, + 428, 78, 79, 80, 78, 79, 80, 77, 360, 428, + 77, 407, 78, 79, 80, 78, 79, 80, 77, 78, + 79, 80, 77, 428, 408, 77, 410, 428, 409, 428, + + 77, 360, 428, 428, 78, 79, 80, 78, 79, 80, + 77, 428, 415, 428, 77, 78, 79, 80, 413, 78, + 79, 80, 78, 79, 80, 77, 360, 78, 79, 80, + 412, 77, 428, 428, 428, 428, 77, 78, 79, 80, + 77, 78, 79, 80, 414, 428, 428, 77, 428, 428, + 77, 428, 78, 79, 80, 360, 428, 428, 78, 79, + 80, 77, 416, 78, 79, 80, 360, 78, 79, 80, + 77, 417, 419, 77, 78, 79, 80, 78, 79, 80, + 77, 418, 422, 428, 77, 360, 420, 77, 78, 79, + 80, 77, 428, 421, 428, 428, 428, 78, 79, 80, + + 78, 79, 80, 77, 428, 428, 77, 78, 79, 80, + 77, 78, 79, 80, 78, 79, 80, 77, 78, 79, + 80, 428, 360, 423, 424, 428, 77, 428, 428, 425, + 78, 79, 80, 78, 79, 80, 426, 78, 79, 80, + 428, 77, 428, 428, 78, 79, 80, 77, 428, 428, + 428, 428, 428, 78, 79, 80, 427, 428, 428, 428, + 428, 360, 428, 428, 428, 428, 428, 428, 78, 79, + 80, 428, 428, 428, 78, 79, 80, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, - 46, 46, 46, 46, 48, 48, 48, 48, 48, 48, - - 48, 48, 48, 48, 48, 48, 48, 48, 51, 51, - 51, 51, 51, 51, 51, 51, 51, 51, 51, 51, - 51, 51, 54, 54, 54, 54, 54, 54, 54, 54, - 54, 54, 54, 54, 54, 54, 57, 424, 57, 424, - 57, 424, 57, 66, 424, 424, 66, 424, 66, 66, - 66, 66, 66, 76, 76, 424, 76, 76, 76, 76, - 76, 76, 126, 126, 126, 126, 126, 126, 126, 126, - 126, 126, 126, 126, 126, 126, 129, 129, 129, 129, - 129, 129, 129, 129, 129, 129, 129, 129, 129, 131, - 424, 131, 131, 131, 131, 131, 131, 131, 131, 131, - - 131, 131, 131, 139, 424, 139, 139, 139, 139, 139, - 139, 139, 139, 139, 139, 139, 139, 140, 140, 140, - 140, 140, 140, 140, 140, 140, 148, 148, 148, 199, - 424, 424, 424, 424, 199, 199, 199, 202, 202, 202, - 202, 202, 210, 210, 210, 424, 424, 210, 260, 260, - 260, 263, 263, 263, 263, 263, 263, 263, 263, 263, - 7, 424, 424, 424, 424, 424, 424, 424, 424, 424, - 424, 424, 424, 424, 424, 424, 424, 424, 424, 424, - 424, 424, 424, 424, 424, 424, 424, 424, 424, 424, - 424, 424, 424, 424, 424, 424, 424, 424, 424, 424, - - 424, 424, 424, 424, 424, 424, 424, 424, 424, 424, - 424, 424, 424, 424, 424, 424, 424, 424, 424, 424, - 424, 424, 424, 424, 424, 424, 424, 424, 424 + 46, 48, 48, 48, 48, 48, 48, 48, 48, 48, + + 48, 48, 48, 48, 48, 51, 51, 51, 51, 51, + 51, 51, 51, 51, 51, 51, 51, 51, 51, 54, + 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, + 54, 54, 54, 57, 428, 57, 428, 57, 428, 57, + 66, 428, 428, 66, 428, 66, 66, 66, 66, 66, + 76, 76, 428, 76, 76, 76, 76, 76, 76, 126, + 126, 126, 126, 126, 126, 126, 126, 126, 126, 126, + 126, 126, 126, 129, 129, 129, 129, 129, 129, 129, + 129, 129, 129, 129, 129, 129, 131, 428, 131, 131, + 131, 131, 131, 131, 131, 131, 131, 131, 131, 131, + + 139, 428, 139, 139, 139, 139, 139, 139, 139, 139, + 139, 139, 139, 139, 140, 140, 140, 140, 140, 140, + 140, 140, 140, 148, 148, 148, 200, 428, 428, 428, + 428, 200, 200, 200, 203, 203, 203, 203, 203, 211, + 211, 211, 428, 428, 211, 262, 262, 262, 265, 265, + 265, 265, 265, 265, 265, 265, 265, 7, 428, 428, + 428, 428, 428, 428, 428, 428, 428, 428, 428, 428, + 428, 428, 428, 428, 428, 428, 428, 428, 428, 428, + 428, 428, 428, 428, 428, 428, 428, 428, 428, 428, + 428, 428, 428, 428, 428, 428, 428, 428, 428, 428, + + 428, 428, 428, 428, 428, 428, 428, 428, 428, 428, + 428, 428, 428, 428, 428, 428, 428, 428, 428, 428, + 428, 428, 428, 428, 428, 428 } ; -static const flex_int16_t yy_chk[2530] = +static const flex_int16_t yy_chk[2527] = { 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -1111,18 +1111,18 @@ static const flex_int16_t yy_chk[2530] = 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 11, 9, 13, 14, 13, 13, 13, 13, 13, 13, 13, - 50, 391, 50, 73, 16, 373, 13, 14, 16, 19, - 14, 350, 51, 82, 82, 201, 25, 349, 201, 20, + 50, 395, 50, 73, 16, 377, 13, 14, 16, 19, + 14, 354, 51, 82, 82, 202, 25, 353, 202, 20, - 316, 211, 11, 25, 73, 13, 14, 15, 15, 15, + 319, 212, 11, 25, 73, 13, 14, 15, 15, 15, 15, 15, 15, 15, 20, 16, 19, 19, 19, 15, - 15, 15, 211, 315, 15, 51, 20, 20, 20, 314, - 25, 25, 25, 313, 270, 22, 269, 268, 15, 15, - 15, 267, 16, 16, 16, 15, 17, 22, 17, 17, - 17, 17, 17, 17, 17, 266, 23, 265, 17, 262, - 260, 17, 22, 22, 22, 17, 259, 213, 209, 17, - 23, 17, 208, 17, 207, 24, 206, 17, 24, 205, - 17, 204, 196, 23, 23, 23, 17, 150, 24, 149, + 15, 15, 212, 318, 15, 51, 20, 20, 20, 317, + 25, 25, 25, 316, 272, 22, 271, 270, 15, 15, + 15, 269, 16, 16, 16, 15, 17, 22, 17, 17, + 17, 17, 17, 17, 17, 268, 23, 267, 17, 264, + 262, 17, 22, 22, 22, 17, 261, 214, 210, 17, + 23, 17, 209, 17, 208, 24, 207, 17, 24, 206, + 17, 205, 197, 23, 23, 23, 17, 150, 24, 149, 24, 28, 147, 17, 146, 17, 21, 21, 24, 145, 21, 21, 21, 21, 21, 21, 21, 26, 26, 26, @@ -1155,232 +1155,232 @@ static const flex_int16_t yy_chk[2530] = 89, 0, 0, 0, 83, 90, 87, 87, 87, 91, 77, 0, 89, 86, 86, 86, 92, 0, 0, 0, 93, 88, 88, 88, 0, 92, 93, 89, 89, 89, - 0, 96, 90, 90, 90, 0, 91, 91, 91, 94, - 0, 0, 95, 92, 92, 92, 97, 93, 93, 93, - - 96, 0, 95, 94, 95, 0, 0, 95, 96, 96, - 96, 97, 0, 0, 98, 0, 94, 94, 94, 95, - 95, 95, 0, 98, 99, 0, 97, 100, 0, 100, - 0, 0, 101, 0, 98, 98, 0, 102, 97, 97, - 97, 98, 98, 98, 99, 101, 104, 0, 0, 103, - 0, 99, 99, 99, 100, 100, 100, 102, 105, 101, - 101, 101, 103, 0, 102, 102, 102, 104, 106, 0, - 0, 0, 0, 104, 104, 104, 103, 103, 103, 105, - 108, 0, 0, 106, 107, 105, 105, 105, 109, 0, - 0, 110, 0, 0, 111, 106, 106, 106, 112, 107, - - 108, 113, 109, 113, 114, 110, 114, 108, 108, 108, - 112, 107, 107, 107, 111, 109, 109, 109, 110, 110, - 110, 111, 111, 111, 115, 112, 112, 112, 113, 113, - 113, 114, 114, 114, 0, 0, 116, 0, 0, 115, - 117, 0, 0, 118, 0, 0, 119, 0, 0, 0, - 121, 115, 115, 115, 116, 0, 117, 0, 0, 0, - 119, 120, 121, 116, 116, 116, 118, 117, 117, 117, - 118, 118, 118, 119, 119, 119, 120, 121, 121, 121, - 148, 0, 122, 0, 148, 123, 132, 0, 120, 120, - 120, 122, 132, 132, 132, 132, 132, 132, 0, 0, - - 0, 148, 140, 123, 141, 0, 0, 0, 148, 122, - 122, 122, 123, 123, 123, 134, 134, 134, 134, 134, - 134, 134, 135, 135, 135, 135, 135, 135, 135, 140, - 0, 141, 141, 141, 135, 0, 135, 0, 0, 135, - 137, 137, 137, 137, 137, 137, 137, 198, 198, 198, - 198, 198, 198, 135, 152, 135, 140, 140, 140, 153, - 135, 138, 138, 138, 138, 138, 138, 138, 142, 142, - 142, 142, 142, 138, 0, 138, 0, 151, 138, 0, - 151, 152, 152, 152, 142, 0, 153, 153, 153, 154, - 0, 154, 138, 156, 138, 157, 0, 0, 160, 138, - - 0, 0, 155, 142, 151, 151, 151, 0, 156, 155, - 0, 159, 155, 158, 0, 0, 154, 154, 154, 160, - 158, 159, 157, 157, 157, 160, 160, 160, 142, 155, - 155, 155, 161, 0, 0, 156, 156, 156, 162, 161, - 158, 158, 158, 162, 163, 0, 0, 164, 159, 159, - 159, 165, 167, 164, 0, 166, 163, 0, 167, 161, - 161, 161, 166, 0, 0, 162, 162, 162, 170, 165, - 168, 163, 163, 163, 164, 164, 164, 169, 165, 165, - 165, 0, 168, 170, 0, 167, 167, 167, 172, 166, - 166, 166, 171, 0, 172, 170, 170, 170, 169, 0, - - 173, 0, 0, 0, 169, 169, 169, 174, 171, 168, - 168, 168, 175, 0, 175, 172, 172, 172, 173, 171, - 171, 171, 0, 174, 176, 0, 0, 173, 173, 173, - 176, 178, 0, 210, 174, 174, 174, 210, 187, 175, - 175, 175, 177, 0, 178, 177, 179, 0, 179, 0, - 183, 176, 176, 176, 210, 180, 187, 183, 178, 178, - 178, 210, 0, 0, 181, 187, 187, 187, 0, 177, - 177, 177, 182, 179, 179, 179, 180, 183, 183, 183, - 182, 185, 180, 180, 180, 181, 0, 184, 0, 0, - 0, 181, 186, 0, 0, 188, 0, 0, 185, 182, - - 182, 182, 184, 0, 186, 188, 184, 190, 185, 185, - 185, 189, 181, 181, 181, 0, 188, 189, 0, 186, - 186, 186, 188, 188, 188, 191, 0, 190, 192, 184, - 184, 184, 0, 193, 190, 190, 190, 195, 189, 189, - 189, 202, 192, 193, 194, 191, 0, 195, 0, 0, - 0, 0, 191, 191, 191, 192, 192, 192, 214, 194, - 193, 193, 193, 215, 195, 195, 195, 0, 202, 0, - 214, 194, 194, 194, 203, 203, 203, 203, 203, 203, - 203, 216, 0, 215, 217, 214, 214, 214, 218, 219, - 215, 215, 215, 222, 0, 202, 202, 202, 217, 220, - - 0, 0, 218, 0, 0, 222, 0, 220, 216, 216, - 216, 217, 217, 217, 221, 0, 219, 219, 219, 0, - 222, 222, 222, 223, 0, 0, 224, 0, 221, 218, - 218, 218, 226, 223, 220, 220, 220, 225, 224, 0, - 228, 221, 221, 221, 0, 226, 224, 0, 0, 0, - 223, 223, 223, 224, 224, 224, 227, 225, 228, 226, - 226, 226, 227, 229, 225, 225, 225, 228, 228, 228, - 230, 0, 0, 229, 0, 231, 0, 230, 0, 0, - 232, 236, 0, 227, 227, 227, 232, 233, 0, 0, - 229, 229, 229, 231, 234, 0, 0, 230, 230, 230, - - 236, 0, 231, 231, 231, 235, 0, 232, 232, 232, - 237, 0, 0, 238, 233, 233, 233, 239, 0, 0, - 235, 234, 234, 234, 237, 238, 242, 236, 236, 236, - 240, 0, 235, 235, 235, 241, 0, 237, 237, 237, - 238, 238, 238, 244, 239, 239, 239, 245, 242, 0, - 240, 241, 243, 0, 0, 248, 0, 240, 240, 240, - 243, 245, 241, 241, 241, 247, 0, 0, 246, 0, - 244, 244, 244, 246, 0, 242, 242, 242, 249, 243, - 243, 243, 248, 248, 248, 250, 247, 0, 245, 245, - 245, 252, 247, 247, 247, 246, 246, 246, 249, 250, - - 251, 0, 0, 253, 251, 249, 249, 249, 0, 253, - 252, 0, 250, 250, 250, 254, 255, 0, 252, 252, - 252, 256, 0, 0, 255, 263, 0, 251, 251, 251, - 253, 253, 253, 257, 256, 0, 254, 0, 0, 258, - 0, 258, 254, 255, 255, 255, 274, 0, 256, 256, - 256, 257, 263, 264, 264, 264, 264, 264, 274, 0, - 257, 257, 257, 254, 254, 254, 258, 258, 258, 264, - 271, 0, 272, 273, 0, 0, 271, 0, 276, 263, - 263, 263, 272, 273, 275, 274, 274, 274, 264, 276, - 277, 0, 277, 0, 275, 0, 0, 271, 271, 271, - - 273, 273, 273, 0, 281, 276, 276, 276, 281, 272, - 272, 272, 278, 264, 278, 0, 0, 277, 277, 277, - 0, 275, 275, 275, 279, 0, 279, 0, 0, 280, - 0, 281, 281, 281, 282, 280, 282, 0, 0, 278, - 278, 278, 0, 0, 283, 0, 0, 285, 0, 0, - 284, 0, 0, 279, 279, 279, 280, 280, 280, 285, - 286, 282, 282, 282, 283, 284, 287, 286, 287, 0, - 0, 283, 283, 283, 285, 285, 285, 284, 284, 284, - 288, 294, 0, 289, 0, 0, 291, 286, 286, 286, - 292, 0, 0, 287, 287, 287, 292, 293, 288, 0, - - 294, 0, 0, 295, 289, 291, 0, 288, 288, 288, - 289, 289, 289, 291, 291, 291, 296, 292, 292, 292, - 300, 297, 296, 0, 293, 293, 293, 294, 294, 294, - 295, 295, 295, 297, 0, 0, 299, 303, 0, 298, - 0, 0, 299, 296, 296, 296, 298, 300, 300, 300, - 301, 0, 305, 302, 0, 0, 303, 0, 301, 304, - 297, 297, 297, 299, 299, 299, 298, 298, 298, 0, - 0, 307, 0, 302, 305, 0, 0, 301, 301, 301, - 302, 302, 302, 303, 303, 303, 304, 304, 304, 306, - 0, 0, 0, 0, 308, 0, 0, 306, 307, 307, - - 307, 305, 305, 305, 308, 309, 0, 0, 310, 317, - 0, 310, 318, 0, 0, 0, 306, 306, 306, 317, - 0, 308, 308, 308, 319, 309, 318, 320, 0, 0, - 0, 0, 309, 309, 309, 310, 310, 310, 321, 318, - 318, 318, 321, 0, 319, 322, 317, 317, 317, 0, - 322, 319, 319, 319, 320, 320, 320, 325, 0, 325, - 323, 0, 0, 0, 326, 0, 0, 324, 0, 321, - 321, 321, 0, 0, 327, 0, 0, 322, 322, 322, - 323, 324, 326, 327, 325, 325, 325, 323, 323, 323, - 328, 326, 326, 326, 324, 324, 324, 330, 0, 0, - - 329, 327, 327, 327, 329, 0, 0, 0, 331, 0, - 328, 332, 0, 330, 333, 0, 0, 328, 328, 328, - 331, 0, 333, 332, 330, 330, 330, 329, 329, 329, - 334, 0, 0, 334, 335, 331, 331, 331, 332, 332, - 332, 333, 333, 333, 336, 0, 0, 337, 335, 0, - 339, 0, 0, 340, 336, 0, 341, 334, 334, 334, - 344, 335, 335, 335, 0, 342, 0, 0, 339, 0, - 347, 336, 336, 336, 337, 337, 337, 339, 339, 339, - 340, 340, 340, 341, 341, 341, 342, 344, 344, 344, - 345, 0, 342, 342, 342, 346, 345, 347, 347, 347, - - 348, 0, 0, 352, 0, 0, 351, 0, 354, 0, - 352, 0, 348, 353, 0, 346, 355, 345, 345, 345, - 351, 354, 346, 346, 346, 0, 355, 348, 348, 348, - 352, 352, 352, 351, 351, 351, 353, 357, 0, 0, - 353, 353, 353, 355, 355, 355, 356, 0, 354, 354, - 354, 357, 356, 0, 0, 358, 0, 0, 359, 0, - 0, 0, 360, 0, 357, 357, 357, 358, 360, 0, - 0, 361, 0, 356, 356, 356, 361, 0, 0, 0, - 0, 0, 358, 358, 358, 359, 359, 359, 362, 360, - 360, 360, 365, 0, 362, 363, 0, 0, 361, 361, - - 361, 363, 364, 0, 365, 366, 0, 0, 364, 0, - 366, 367, 0, 0, 368, 362, 362, 362, 369, 365, - 365, 365, 363, 363, 363, 0, 367, 0, 0, 364, - 364, 364, 366, 366, 366, 370, 369, 0, 367, 367, - 367, 368, 368, 368, 371, 369, 369, 369, 372, 0, - 371, 374, 0, 370, 372, 376, 0, 0, 374, 375, - 0, 0, 370, 370, 370, 377, 375, 0, 0, 376, - 0, 371, 371, 371, 378, 372, 372, 372, 374, 374, - 374, 0, 0, 378, 0, 0, 375, 375, 375, 377, - 381, 0, 377, 377, 377, 379, 376, 376, 376, 380, - - 0, 379, 382, 0, 0, 0, 0, 383, 381, 382, - 378, 378, 378, 385, 0, 0, 0, 381, 381, 381, - 380, 0, 379, 379, 379, 383, 380, 380, 380, 382, - 382, 382, 384, 0, 383, 383, 383, 386, 0, 0, - 385, 385, 385, 387, 0, 0, 388, 384, 0, 387, - 389, 386, 0, 392, 0, 389, 0, 390, 0, 384, - 384, 384, 393, 0, 386, 386, 386, 0, 0, 0, - 387, 387, 387, 388, 388, 388, 390, 389, 389, 389, - 392, 392, 392, 396, 390, 390, 390, 394, 0, 393, - 393, 393, 395, 394, 398, 397, 0, 0, 395, 0, - - 401, 397, 396, 399, 401, 0, 400, 0, 0, 0, - 396, 396, 396, 402, 394, 394, 394, 398, 403, 395, - 395, 395, 397, 397, 397, 399, 400, 401, 401, 401, - 404, 407, 402, 400, 400, 400, 0, 403, 0, 0, - 402, 402, 402, 406, 398, 398, 398, 405, 410, 404, - 408, 0, 399, 399, 399, 407, 0, 404, 404, 404, - 408, 0, 406, 0, 403, 403, 403, 410, 0, 405, - 406, 406, 406, 416, 405, 405, 405, 408, 408, 408, - 409, 0, 407, 407, 407, 416, 412, 414, 0, 414, - 0, 409, 412, 413, 410, 410, 410, 418, 0, 413, - - 416, 416, 416, 418, 417, 0, 0, 409, 409, 409, - 419, 0, 0, 412, 412, 412, 414, 414, 414, 0, - 413, 413, 413, 417, 418, 418, 418, 420, 419, 0, - 421, 417, 417, 417, 422, 0, 0, 419, 419, 419, - 421, 0, 0, 0, 0, 0, 420, 0, 0, 422, - 423, 0, 0, 0, 420, 420, 420, 421, 421, 421, - 0, 422, 422, 422, 423, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 423, 423, 423, - 425, 425, 425, 425, 425, 425, 425, 425, 425, 425, - 425, 425, 425, 425, 426, 426, 426, 426, 426, 426, - - 426, 426, 426, 426, 426, 426, 426, 426, 427, 427, - 427, 427, 427, 427, 427, 427, 427, 427, 427, 427, - 427, 427, 428, 428, 428, 428, 428, 428, 428, 428, - 428, 428, 428, 428, 428, 428, 429, 0, 429, 0, - 429, 0, 429, 430, 0, 0, 430, 0, 430, 430, - 430, 430, 430, 431, 431, 0, 431, 431, 431, 431, - 431, 431, 432, 432, 432, 432, 432, 432, 432, 432, - 432, 432, 432, 432, 432, 432, 433, 433, 433, 433, - 433, 433, 433, 433, 433, 433, 433, 433, 433, 434, - 0, 434, 434, 434, 434, 434, 434, 434, 434, 434, - - 434, 434, 434, 435, 0, 435, 435, 435, 435, 435, - 435, 435, 435, 435, 435, 435, 435, 436, 436, 436, - 436, 436, 436, 436, 436, 436, 437, 437, 437, 438, - 0, 0, 0, 0, 438, 438, 438, 439, 439, 439, - 439, 439, 440, 440, 440, 0, 0, 440, 441, 441, - 441, 442, 442, 442, 442, 442, 442, 442, 442, 442, - 424, 424, 424, 424, 424, 424, 424, 424, 424, 424, - 424, 424, 424, 424, 424, 424, 424, 424, 424, 424, - 424, 424, 424, 424, 424, 424, 424, 424, 424, 424, - 424, 424, 424, 424, 424, 424, 424, 424, 424, 424, - - 424, 424, 424, 424, 424, 424, 424, 424, 424, 424, - 424, 424, 424, 424, 424, 424, 424, 424, 424, 424, - 424, 424, 424, 424, 424, 424, 424, 424, 424 + 0, 92, 90, 90, 90, 99, 91, 91, 91, 94, + 0, 0, 95, 92, 92, 92, 96, 93, 93, 93, + + 0, 0, 95, 94, 95, 99, 100, 95, 100, 102, + 0, 97, 99, 99, 99, 96, 94, 94, 94, 95, + 95, 95, 98, 96, 96, 96, 97, 0, 0, 102, + 0, 98, 101, 100, 100, 100, 102, 102, 102, 103, + 0, 97, 98, 98, 104, 101, 0, 0, 105, 98, + 98, 98, 103, 97, 97, 97, 0, 0, 0, 101, + 101, 101, 0, 106, 0, 104, 103, 103, 103, 105, + 107, 104, 104, 104, 108, 105, 105, 105, 106, 109, + 0, 0, 110, 0, 0, 107, 0, 0, 0, 0, + 106, 106, 106, 109, 108, 111, 110, 107, 107, 107, + + 112, 108, 108, 108, 0, 115, 109, 109, 109, 110, + 110, 110, 112, 0, 113, 111, 113, 114, 0, 114, + 115, 116, 111, 111, 111, 0, 119, 112, 112, 112, + 0, 117, 115, 115, 115, 0, 0, 0, 118, 116, + 119, 113, 113, 113, 114, 114, 114, 117, 116, 116, + 116, 121, 0, 119, 119, 119, 120, 0, 117, 117, + 117, 118, 0, 121, 122, 118, 118, 118, 123, 0, + 0, 120, 0, 122, 0, 0, 0, 0, 121, 121, + 121, 0, 0, 120, 120, 120, 123, 0, 0, 0, + 0, 122, 122, 122, 132, 123, 123, 123, 0, 0, + + 132, 132, 132, 132, 132, 132, 134, 134, 134, 134, + 134, 134, 134, 135, 135, 135, 135, 135, 135, 135, + 140, 141, 0, 0, 0, 135, 0, 135, 0, 0, + 135, 137, 137, 137, 137, 137, 137, 137, 0, 148, + 0, 0, 0, 148, 135, 0, 135, 140, 141, 141, + 141, 135, 138, 138, 138, 138, 138, 138, 138, 151, + 148, 0, 151, 0, 138, 0, 138, 148, 152, 138, + 0, 153, 0, 0, 140, 140, 140, 142, 142, 142, + 142, 142, 154, 138, 154, 138, 151, 151, 151, 157, + 138, 156, 159, 142, 155, 152, 152, 152, 153, 153, + + 153, 155, 159, 0, 155, 0, 156, 0, 0, 154, + 154, 154, 142, 158, 0, 0, 157, 157, 157, 162, + 158, 155, 155, 155, 160, 0, 162, 161, 0, 159, + 159, 159, 161, 156, 156, 156, 0, 142, 163, 0, + 158, 158, 158, 163, 165, 160, 162, 162, 162, 164, + 165, 160, 160, 160, 161, 161, 161, 166, 168, 0, + 170, 164, 167, 0, 168, 163, 163, 163, 169, 167, + 0, 165, 165, 165, 0, 166, 164, 164, 164, 0, + 169, 170, 0, 172, 166, 166, 166, 170, 170, 170, + 171, 168, 168, 168, 173, 0, 167, 167, 167, 172, + + 173, 0, 174, 0, 0, 171, 0, 169, 169, 169, + 172, 172, 172, 176, 0, 176, 175, 171, 171, 171, + 174, 173, 173, 173, 0, 0, 0, 177, 0, 174, + 174, 174, 175, 177, 178, 0, 0, 178, 179, 0, + 176, 176, 176, 175, 175, 175, 180, 0, 180, 181, + 0, 179, 0, 0, 177, 177, 177, 0, 182, 0, + 183, 178, 178, 178, 184, 179, 179, 179, 183, 186, + 181, 184, 0, 180, 180, 180, 181, 181, 181, 182, + 185, 0, 187, 0, 0, 182, 186, 183, 183, 183, + 0, 184, 184, 184, 187, 185, 186, 186, 186, 185, + + 188, 0, 0, 0, 189, 0, 182, 182, 182, 187, + 187, 187, 0, 190, 189, 0, 191, 0, 188, 190, + 193, 0, 185, 185, 185, 189, 192, 188, 188, 188, + 0, 189, 189, 189, 193, 203, 191, 195, 0, 0, + 190, 190, 190, 191, 191, 191, 192, 193, 193, 193, + 194, 0, 195, 192, 192, 192, 0, 196, 0, 211, + 194, 0, 203, 211, 195, 195, 195, 196, 199, 199, + 199, 199, 199, 199, 0, 0, 215, 194, 194, 194, + 211, 216, 0, 0, 196, 196, 196, 211, 215, 203, + 203, 203, 204, 204, 204, 204, 204, 204, 204, 217, + + 219, 216, 218, 215, 215, 215, 220, 0, 216, 216, + 216, 221, 0, 0, 219, 0, 218, 0, 0, 221, + 0, 0, 222, 0, 0, 224, 217, 217, 217, 218, + 218, 218, 223, 220, 220, 220, 222, 224, 223, 0, + 0, 219, 219, 219, 0, 225, 221, 221, 221, 222, + 222, 222, 224, 224, 224, 225, 226, 0, 0, 223, + 223, 223, 227, 0, 0, 228, 0, 0, 226, 229, + 0, 0, 225, 225, 225, 229, 226, 230, 228, 0, + 0, 0, 227, 226, 226, 226, 235, 0, 0, 227, + 227, 227, 228, 228, 228, 230, 229, 229, 229, 231, + + 238, 0, 0, 232, 230, 230, 230, 233, 0, 231, + 232, 234, 0, 235, 235, 235, 236, 234, 0, 238, + 0, 0, 237, 0, 0, 233, 231, 231, 231, 241, + 232, 232, 232, 242, 233, 233, 233, 237, 234, 234, + 234, 239, 0, 236, 236, 236, 238, 238, 238, 237, + 237, 237, 240, 242, 244, 239, 241, 241, 241, 243, + 242, 242, 242, 245, 240, 0, 246, 0, 239, 239, + 239, 245, 248, 0, 0, 243, 244, 248, 0, 240, + 240, 240, 247, 0, 0, 249, 243, 243, 243, 250, + 245, 245, 245, 246, 246, 246, 247, 0, 0, 248, + + 248, 248, 251, 244, 244, 244, 249, 252, 0, 0, + 0, 253, 249, 249, 249, 253, 250, 250, 250, 0, + 256, 252, 251, 247, 247, 247, 254, 0, 0, 251, + 251, 251, 265, 258, 252, 252, 252, 255, 253, 253, + 253, 256, 0, 255, 257, 254, 258, 256, 0, 259, + 0, 0, 257, 254, 254, 254, 260, 0, 260, 265, + 258, 258, 258, 0, 255, 255, 255, 259, 256, 256, + 256, 257, 257, 257, 273, 0, 259, 259, 259, 0, + 273, 0, 0, 260, 260, 260, 265, 265, 265, 266, + 266, 266, 266, 266, 274, 276, 0, 0, 275, 0, + + 277, 273, 273, 273, 274, 266, 0, 276, 275, 0, + 277, 0, 0, 0, 279, 0, 0, 278, 279, 0, + 0, 0, 0, 0, 266, 275, 275, 275, 278, 0, + 0, 274, 274, 274, 276, 276, 276, 277, 277, 277, + 0, 279, 279, 279, 278, 278, 278, 0, 280, 266, + 280, 281, 282, 281, 282, 0, 0, 283, 0, 0, + 284, 0, 0, 283, 284, 285, 0, 285, 0, 0, + 286, 0, 0, 0, 0, 280, 280, 280, 281, 281, + 281, 282, 282, 282, 283, 283, 283, 284, 284, 284, + 286, 287, 285, 285, 285, 288, 0, 286, 286, 286, + + 0, 289, 0, 0, 0, 292, 287, 288, 289, 290, + 0, 290, 0, 291, 0, 0, 294, 0, 287, 287, + 287, 295, 288, 288, 288, 296, 292, 295, 289, 289, + 289, 291, 292, 292, 292, 294, 290, 290, 290, 297, + 291, 291, 291, 294, 294, 294, 298, 0, 295, 295, + 295, 299, 296, 296, 296, 300, 0, 299, 297, 0, + 0, 301, 0, 0, 302, 0, 0, 300, 301, 0, + 302, 303, 0, 298, 298, 298, 304, 0, 299, 299, + 299, 305, 0, 306, 304, 297, 297, 297, 301, 301, + 301, 302, 302, 302, 300, 300, 300, 307, 303, 303, + + 303, 305, 306, 304, 304, 304, 308, 309, 305, 305, + 305, 310, 0, 0, 0, 309, 0, 311, 0, 0, + 0, 312, 320, 0, 307, 307, 307, 311, 308, 306, + 306, 306, 320, 0, 309, 309, 309, 323, 310, 310, + 310, 312, 322, 324, 311, 311, 311, 324, 312, 312, + 312, 313, 0, 0, 313, 308, 308, 308, 321, 320, + 320, 320, 322, 325, 323, 323, 323, 0, 325, 322, + 322, 322, 321, 326, 324, 324, 324, 328, 313, 313, + 313, 0, 0, 327, 0, 321, 321, 321, 329, 0, + 329, 328, 326, 332, 0, 325, 325, 325, 0, 331, + + 326, 326, 326, 327, 328, 328, 328, 330, 331, 0, + 327, 327, 327, 332, 333, 329, 329, 329, 333, 335, + 332, 332, 332, 334, 0, 330, 331, 331, 331, 337, + 0, 335, 336, 0, 330, 330, 330, 337, 0, 334, + 339, 333, 333, 333, 336, 340, 335, 335, 335, 341, + 334, 334, 334, 343, 339, 340, 337, 337, 337, 336, + 336, 336, 338, 0, 0, 338, 344, 339, 339, 339, + 345, 343, 340, 340, 340, 346, 341, 341, 341, 348, + 343, 343, 343, 351, 0, 0, 349, 0, 0, 338, + 338, 338, 349, 344, 344, 344, 346, 345, 345, 345, + + 350, 0, 346, 346, 346, 352, 348, 348, 348, 355, + 351, 351, 351, 349, 349, 349, 0, 352, 356, 0, + 350, 357, 358, 355, 360, 356, 0, 350, 350, 350, + 359, 0, 352, 352, 352, 358, 355, 355, 355, 0, + 359, 362, 0, 0, 357, 356, 356, 356, 357, 357, + 357, 360, 360, 360, 361, 362, 0, 359, 359, 359, + 361, 363, 358, 358, 358, 364, 0, 0, 362, 362, + 362, 364, 365, 363, 0, 0, 0, 365, 366, 0, + 0, 361, 361, 361, 366, 367, 0, 0, 363, 363, + 363, 367, 364, 364, 364, 0, 368, 0, 0, 365, + + 365, 365, 368, 369, 0, 366, 366, 366, 372, 0, + 0, 370, 367, 367, 367, 369, 370, 0, 371, 0, + 0, 375, 0, 368, 368, 368, 373, 375, 0, 0, + 369, 369, 369, 371, 374, 372, 372, 372, 370, 370, + 370, 0, 0, 0, 373, 371, 371, 371, 375, 375, + 375, 376, 374, 373, 373, 373, 380, 376, 378, 0, + 0, 374, 374, 374, 379, 378, 0, 383, 0, 0, + 380, 379, 382, 383, 0, 0, 381, 0, 376, 376, + 376, 382, 0, 0, 0, 378, 378, 378, 0, 0, + 384, 379, 379, 379, 383, 383, 383, 380, 380, 380, + + 381, 0, 385, 381, 381, 381, 386, 0, 382, 382, + 382, 384, 0, 386, 387, 0, 0, 384, 384, 384, + 385, 388, 0, 0, 389, 0, 0, 0, 0, 385, + 385, 385, 387, 386, 386, 386, 388, 392, 0, 0, + 390, 387, 387, 387, 396, 0, 0, 397, 388, 388, + 388, 389, 389, 389, 390, 391, 0, 402, 393, 0, + 0, 391, 394, 393, 392, 392, 392, 390, 390, 390, + 0, 396, 396, 396, 397, 397, 397, 400, 403, 0, + 402, 394, 391, 391, 391, 393, 393, 393, 398, 394, + 394, 394, 399, 0, 398, 401, 400, 0, 399, 0, + + 403, 401, 0, 0, 400, 400, 400, 402, 402, 402, + 404, 0, 407, 0, 405, 398, 398, 398, 405, 399, + 399, 399, 401, 401, 401, 406, 411, 403, 403, 403, + 404, 407, 0, 0, 0, 0, 408, 404, 404, 404, + 409, 405, 405, 405, 406, 0, 0, 410, 0, 0, + 411, 0, 406, 406, 406, 408, 0, 0, 407, 407, + 407, 412, 409, 408, 408, 408, 410, 409, 409, 409, + 413, 412, 414, 420, 410, 410, 410, 411, 411, 411, + 416, 413, 418, 0, 418, 420, 416, 417, 412, 412, + 412, 414, 0, 417, 0, 0, 0, 413, 413, 413, + + 420, 420, 420, 421, 0, 0, 423, 416, 416, 416, + 424, 418, 418, 418, 417, 417, 417, 422, 414, 414, + 414, 0, 421, 422, 423, 0, 425, 0, 0, 424, + 421, 421, 421, 423, 423, 423, 425, 424, 424, 424, + 0, 426, 0, 0, 422, 422, 422, 427, 0, 0, + 0, 0, 0, 425, 425, 425, 426, 0, 0, 0, + 0, 427, 0, 0, 0, 0, 0, 0, 426, 426, + 426, 0, 0, 0, 427, 427, 427, 429, 429, 429, + 429, 429, 429, 429, 429, 429, 429, 429, 429, 429, + 429, 430, 430, 430, 430, 430, 430, 430, 430, 430, + + 430, 430, 430, 430, 430, 431, 431, 431, 431, 431, + 431, 431, 431, 431, 431, 431, 431, 431, 431, 432, + 432, 432, 432, 432, 432, 432, 432, 432, 432, 432, + 432, 432, 432, 433, 0, 433, 0, 433, 0, 433, + 434, 0, 0, 434, 0, 434, 434, 434, 434, 434, + 435, 435, 0, 435, 435, 435, 435, 435, 435, 436, + 436, 436, 436, 436, 436, 436, 436, 436, 436, 436, + 436, 436, 436, 437, 437, 437, 437, 437, 437, 437, + 437, 437, 437, 437, 437, 437, 438, 0, 438, 438, + 438, 438, 438, 438, 438, 438, 438, 438, 438, 438, + + 439, 0, 439, 439, 439, 439, 439, 439, 439, 439, + 439, 439, 439, 439, 440, 440, 440, 440, 440, 440, + 440, 440, 440, 441, 441, 441, 442, 0, 0, 0, + 0, 442, 442, 442, 443, 443, 443, 443, 443, 444, + 444, 444, 0, 0, 444, 445, 445, 445, 446, 446, + 446, 446, 446, 446, 446, 446, 446, 428, 428, 428, + 428, 428, 428, 428, 428, 428, 428, 428, 428, 428, + 428, 428, 428, 428, 428, 428, 428, 428, 428, 428, + 428, 428, 428, 428, 428, 428, 428, 428, 428, 428, + 428, 428, 428, 428, 428, 428, 428, 428, 428, 428, + + 428, 428, 428, 428, 428, 428, 428, 428, 428, 428, + 428, 428, 428, 428, 428, 428, 428, 428, 428, 428, + 428, 428, 428, 428, 428, 428 } ; static yy_state_type yy_last_accepting_state; @@ -1539,10 +1539,11 @@ struct Specialtoken specials[] = { {"_IsNetcdf4",_ISNETCDF4,_ISNETCDF4_FLAG}, {"_SuperblockVersion",_SUPERBLOCK,_SUPERBLOCK_FLAG}, {"_Filter",_FILTER,_FILTER_FLAG}, +{"_Codecs",_CODECS,_CODECS_FLAG}, {NULL,0} /* null terminate */ }; -#line 1545 "ncgenl.c" +#line 1546 "ncgenl.c" /* The most correct (validating) version of UTF8 character set (Taken from: http://www.w3.org/2005/03/23-lex-U) @@ -1585,7 +1586,7 @@ ID ([A-Za-z_]|{UTF8})([A-Z.@#\[\]a-z_0-9+-]|{UTF8})* /* Note: this definition of string will work for utf8 as well, although it is a very relaxed definition */ -#line 1588 "ncgenl.c" +#line 1589 "ncgenl.c" #define INITIAL 0 #define ST_C_COMMENT 1 @@ -1804,9 +1805,9 @@ YY_DECL } { -#line 224 "ncgen.l" +#line 225 "ncgen.l" -#line 1809 "ncgenl.c" +#line 1810 "ncgenl.c" while ( /*CONSTCOND*/1 ) /* loops until end-of-file is reached */ { @@ -1833,13 +1834,13 @@ YY_DECL while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 425 ) + if ( yy_current_state >= 429 ) yy_c = yy_meta[yy_c]; } yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; ++yy_cp; } - while ( yy_base[yy_current_state] != 2461 ); + while ( yy_base[yy_current_state] != 2458 ); yy_find_action: yy_act = yy_accept[yy_current_state]; @@ -1865,14 +1866,14 @@ YY_DECL case 1: YY_RULE_SETUP -#line 225 "ncgen.l" +#line 226 "ncgen.l" { /* whitespace */ break; } YY_BREAK case 2: YY_RULE_SETUP -#line 229 "ncgen.l" +#line 230 "ncgen.l" { /* comment */ break; } @@ -1880,7 +1881,7 @@ YY_RULE_SETUP case 3: /* rule 3 can match eol */ YY_RULE_SETUP -#line 233 "ncgen.l" +#line 234 "ncgen.l" {int len; char* s = NULL; /* In netcdf4, this will be used in a variety of places, so only remove escapes */ @@ -1904,7 +1905,7 @@ yytext[MAXTRST-1] = '\0'; YY_BREAK case 4: YY_RULE_SETUP -#line 254 "ncgen.l" +#line 255 "ncgen.l" { /* drop leading 0x; pad to even number of chars */ char* p = yytext+2; int len = yyleng - 2; @@ -1919,143 +1920,143 @@ YY_RULE_SETUP YY_BREAK case 5: YY_RULE_SETUP -#line 266 "ncgen.l" +#line 267 "ncgen.l" {return lexdebug(COMPOUND);} YY_BREAK case 6: YY_RULE_SETUP -#line 267 "ncgen.l" +#line 268 "ncgen.l" {return lexdebug(ENUM);} YY_BREAK case 7: YY_RULE_SETUP -#line 268 "ncgen.l" +#line 269 "ncgen.l" {return lexdebug(OPAQUE_);} YY_BREAK case 8: YY_RULE_SETUP -#line 270 "ncgen.l" +#line 271 "ncgen.l" {return lexdebug(FLOAT_K);} YY_BREAK case 9: YY_RULE_SETUP -#line 271 "ncgen.l" +#line 272 "ncgen.l" {return lexdebug(DOUBLE_K);} YY_BREAK case 10: YY_RULE_SETUP -#line 272 "ncgen.l" +#line 273 "ncgen.l" {return lexdebug(CHAR_K);} YY_BREAK case 11: YY_RULE_SETUP -#line 273 "ncgen.l" +#line 274 "ncgen.l" {return lexdebug(BYTE_K);} YY_BREAK case 12: YY_RULE_SETUP -#line 274 "ncgen.l" +#line 275 "ncgen.l" {return lexdebug(SHORT_K);} YY_BREAK case 13: YY_RULE_SETUP -#line 275 "ncgen.l" +#line 276 "ncgen.l" {return lexdebug(INT_K);} YY_BREAK case 14: YY_RULE_SETUP -#line 276 "ncgen.l" +#line 277 "ncgen.l" {return lexdebug(identcheck(UBYTE_K));} YY_BREAK case 15: YY_RULE_SETUP -#line 277 "ncgen.l" +#line 278 "ncgen.l" {return lexdebug(identcheck(USHORT_K));} YY_BREAK case 16: YY_RULE_SETUP -#line 278 "ncgen.l" +#line 279 "ncgen.l" {return lexdebug(identcheck(UINT_K));} YY_BREAK case 17: YY_RULE_SETUP -#line 279 "ncgen.l" +#line 280 "ncgen.l" {return lexdebug(identcheck(INT64_K));} YY_BREAK case 18: YY_RULE_SETUP -#line 280 "ncgen.l" +#line 281 "ncgen.l" {return lexdebug(identcheck(UINT64_K));} YY_BREAK case 19: YY_RULE_SETUP -#line 281 "ncgen.l" +#line 282 "ncgen.l" {return lexdebug(identcheck(STRING_K));} YY_BREAK case 20: YY_RULE_SETUP -#line 283 "ncgen.l" +#line 284 "ncgen.l" {return lexdebug(FLOAT_K);} YY_BREAK case 21: YY_RULE_SETUP -#line 284 "ncgen.l" +#line 285 "ncgen.l" {return lexdebug(INT_K);} YY_BREAK case 22: YY_RULE_SETUP -#line 285 "ncgen.l" +#line 286 "ncgen.l" {return lexdebug(INT_K);} YY_BREAK case 23: YY_RULE_SETUP -#line 286 "ncgen.l" +#line 287 "ncgen.l" {return lexdebug(identcheck(UINT_K));} YY_BREAK case 24: YY_RULE_SETUP -#line 287 "ncgen.l" +#line 288 "ncgen.l" {return lexdebug(identcheck(UINT_K));} YY_BREAK case 25: YY_RULE_SETUP -#line 290 "ncgen.l" +#line 291 "ncgen.l" {int32_val = -1; return lexdebug(NC_UNLIMITED_K);} YY_BREAK case 26: YY_RULE_SETUP -#line 293 "ncgen.l" +#line 294 "ncgen.l" {return lexdebug(TYPES);} YY_BREAK case 27: YY_RULE_SETUP -#line 294 "ncgen.l" +#line 295 "ncgen.l" {return lexdebug(DIMENSIONS);} YY_BREAK case 28: YY_RULE_SETUP -#line 295 "ncgen.l" +#line 296 "ncgen.l" {return lexdebug(VARIABLES);} YY_BREAK case 29: YY_RULE_SETUP -#line 296 "ncgen.l" +#line 297 "ncgen.l" {return lexdebug(DATA);} YY_BREAK case 30: YY_RULE_SETUP -#line 297 "ncgen.l" +#line 298 "ncgen.l" {return lexdebug(GROUP);} YY_BREAK case 31: YY_RULE_SETUP -#line 299 "ncgen.l" +#line 300 "ncgen.l" {BEGIN(TEXT);return lexdebug(NETCDF);} YY_BREAK case 32: YY_RULE_SETUP -#line 301 "ncgen.l" +#line 302 "ncgen.l" { /* missing value (pre-2.4 backward compatibility) */ if (yytext[0] == '-') { double_val = -INFINITY; @@ -2068,7 +2069,7 @@ YY_RULE_SETUP YY_BREAK case 33: YY_RULE_SETUP -#line 310 "ncgen.l" +#line 311 "ncgen.l" { /* missing value (pre-2.4 backward compatibility) */ double_val = NAN; specialconstants = 1; @@ -2077,7 +2078,7 @@ YY_RULE_SETUP YY_BREAK case 34: YY_RULE_SETUP -#line 316 "ncgen.l" +#line 317 "ncgen.l" {/* missing value (pre-2.4 backward compatibility)*/ if (yytext[0] == '-') { float_val = -INFINITYF; @@ -2090,7 +2091,7 @@ YY_RULE_SETUP YY_BREAK case 35: YY_RULE_SETUP -#line 325 "ncgen.l" +#line 326 "ncgen.l" { /* missing value (pre-2.4 backward compatibility) */ float_val = NANF; specialconstants = 1; @@ -2099,7 +2100,7 @@ YY_RULE_SETUP YY_BREAK case 36: YY_RULE_SETUP -#line 331 "ncgen.l" +#line 332 "ncgen.l" { #ifdef USE_NETCDF4 if(l_flag == L_C || l_flag == L_BINARY) @@ -2112,7 +2113,7 @@ YY_RULE_SETUP YY_BREAK case 37: YY_RULE_SETUP -#line 341 "ncgen.l" +#line 342 "ncgen.l" { bbClear(lextext); bbAppendn(lextext,(char*)yytext,yyleng+1); /* include null */ @@ -2123,7 +2124,7 @@ YY_RULE_SETUP YY_BREAK case 38: YY_RULE_SETUP -#line 350 "ncgen.l" +#line 351 "ncgen.l" {struct Specialtoken* st; bbClear(lextext); bbAppendn(lextext,(char*)yytext,yyleng+1); /* include null */ @@ -2137,7 +2138,7 @@ YY_RULE_SETUP case 39: /* rule 39 can match eol */ YY_RULE_SETUP -#line 360 "ncgen.l" +#line 361 "ncgen.l" { int c; char* p; char* q; @@ -2157,7 +2158,7 @@ YY_RULE_SETUP YY_BREAK case 40: YY_RULE_SETUP -#line 377 "ncgen.l" +#line 378 "ncgen.l" { char* id = NULL; int len; len = strlen(yytext); len = unescape(yytext,len,ISIDENT,&id); @@ -2172,7 +2173,7 @@ YY_RULE_SETUP YY_BREAK case 41: YY_RULE_SETUP -#line 389 "ncgen.l" +#line 390 "ncgen.l" { /* We need to try to see what size of integer ((u)int). @@ -2253,7 +2254,7 @@ done: return 0; YY_BREAK case 42: YY_RULE_SETUP -#line 467 "ncgen.l" +#line 468 "ncgen.l" { int c; int token = 0; @@ -2304,7 +2305,7 @@ YY_RULE_SETUP YY_BREAK case 43: YY_RULE_SETUP -#line 514 "ncgen.l" +#line 515 "ncgen.l" { if (sscanf((char*)yytext, "%le", &double_val) != 1) { sprintf(errstr,"bad long or double constant: %s",(char*)yytext); @@ -2315,7 +2316,7 @@ YY_RULE_SETUP YY_BREAK case 44: YY_RULE_SETUP -#line 521 "ncgen.l" +#line 522 "ncgen.l" { if (sscanf((char*)yytext, "%e", &float_val) != 1) { sprintf(errstr,"bad float constant: %s",(char*)yytext); @@ -2327,7 +2328,7 @@ YY_RULE_SETUP case 45: /* rule 45 can match eol */ YY_RULE_SETUP -#line 528 "ncgen.l" +#line 529 "ncgen.l" { (void) sscanf((char*)&yytext[1],"%c",&byte_val); return lexdebug(BYTE_CONST); @@ -2335,7 +2336,7 @@ YY_RULE_SETUP YY_BREAK case 46: YY_RULE_SETUP -#line 532 "ncgen.l" +#line 533 "ncgen.l" { int oct = unescapeoct(&yytext[2]); if(oct < 0) { @@ -2348,7 +2349,7 @@ YY_RULE_SETUP YY_BREAK case 47: YY_RULE_SETUP -#line 541 "ncgen.l" +#line 542 "ncgen.l" { int hex = unescapehex(&yytext[3]); if(byte_val < 0) { @@ -2361,7 +2362,7 @@ YY_RULE_SETUP YY_BREAK case 48: YY_RULE_SETUP -#line 550 "ncgen.l" +#line 551 "ncgen.l" { switch ((char)yytext[2]) { case 'a': byte_val = '\007'; break; /* not everyone under- @@ -2383,7 +2384,7 @@ YY_RULE_SETUP case 49: /* rule 49 can match eol */ YY_RULE_SETUP -#line 568 "ncgen.l" +#line 569 "ncgen.l" { lineno++ ; break; @@ -2391,7 +2392,7 @@ YY_RULE_SETUP YY_BREAK case 50: YY_RULE_SETUP -#line 573 "ncgen.l" +#line 574 "ncgen.l" {/*initial*/ BEGIN(ST_C_COMMENT); break; @@ -2400,21 +2401,21 @@ YY_RULE_SETUP case 51: /* rule 51 can match eol */ YY_RULE_SETUP -#line 578 "ncgen.l" +#line 579 "ncgen.l" {/* continuation */ break; } YY_BREAK case 52: YY_RULE_SETUP -#line 582 "ncgen.l" +#line 583 "ncgen.l" {/* final */ BEGIN(INITIAL); break; } YY_BREAK case YY_STATE_EOF(ST_C_COMMENT): -#line 587 "ncgen.l" +#line 588 "ncgen.l" {/* final, error */ fprintf(stderr,"unterminated /**/ comment"); BEGIN(INITIAL); @@ -2423,17 +2424,17 @@ case YY_STATE_EOF(ST_C_COMMENT): YY_BREAK case 53: YY_RULE_SETUP -#line 593 "ncgen.l" +#line 594 "ncgen.l" {/* Note: this next rule will not work for UTF8 characters */ return lexdebug(yytext[0]) ; } YY_BREAK case 54: YY_RULE_SETUP -#line 596 "ncgen.l" +#line 597 "ncgen.l" ECHO; YY_BREAK -#line 2436 "ncgenl.c" +#line 2437 "ncgenl.c" case YY_STATE_EOF(INITIAL): case YY_STATE_EOF(TEXT): yyterminate(); @@ -2731,7 +2732,7 @@ static int yy_get_next_buffer (void) while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 425 ) + if ( yy_current_state >= 429 ) yy_c = yy_meta[yy_c]; } yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; @@ -2759,11 +2760,11 @@ static int yy_get_next_buffer (void) while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 425 ) + if ( yy_current_state >= 429 ) yy_c = yy_meta[yy_c]; } yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; - yy_is_jam = (yy_current_state == 424); + yy_is_jam = (yy_current_state == 428); return yy_is_jam ? 0 : yy_current_state; } @@ -3439,7 +3440,7 @@ void yyfree (void * ptr ) #define YYTABLES_NAME "yytables" -#line 596 "ncgen.l" +#line 597 "ncgen.l" static int lexdebug(int token) diff --git a/ncgen/ncgeny.c b/ncgen/ncgeny.c index a5de1501a4..9a9684f5d4 100644 --- a/ncgen/ncgeny.c +++ b/ncgen/ncgeny.c @@ -1,8 +1,9 @@ -/* A Bison parser, made by GNU Bison 3.0.4. */ +/* A Bison parser, made by GNU Bison 3.7.6. */ /* Bison implementation for Yacc-like parsers in C - Copyright (C) 1984, 1989-1990, 2000-2015 Free Software Foundation, Inc. + Copyright (C) 1984, 1989-1990, 2000-2015, 2018-2021 Free Software Foundation, + Inc. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -15,7 +16,7 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License - along with this program. If not, see . */ + along with this program. If not, see . */ /* As a special exception, you may create a larger work that contains part or all of the Bison parser skeleton and distribute that work @@ -33,6 +34,10 @@ /* C LALR(1) parser skeleton written by Richard Stallman, by simplifying the original so-called "semantic" parser. */ +/* DO NOT RELY ON FEATURES THAT ARE NOT DOCUMENTED in the manual, + especially those whose name start with YY_ or yy_. They are + private implementation details that can be changed or removed. */ + /* All symbols defined below should begin with yy or YY, to avoid infringing on user name space. This should be done even for local variables, as they might otherwise be expanded by user macros. @@ -40,11 +45,11 @@ define necessary library symbols; they are noted "INFRINGES ON USER NAME SPACE" below. */ -/* Identify Bison output. */ -#define YYBISON 1 +/* Identify Bison output, and Bison version. */ +#define YYBISON 30706 -/* Bison version. */ -#define YYBISON_VERSION "3.0.4" +/* Bison version string. */ +#define YYBISON_VERSION "3.7.6" /* Skeleton name. */ #define YYSKELETON_NAME "yacc.c" @@ -65,12 +70,11 @@ #define yyerror ncgerror #define yydebug ncgdebug #define yynerrs ncgnerrs - #define yylval ncglval #define yychar ncgchar -/* Copy the first part of user declarations. */ -#line 11 "ncgen.y" /* yacc.c:339 */ +/* First part of user prologue. */ +#line 11 "ncgen.y" /* static char SccsId[] = "$Id: ncgen.y,v 1.42 2010/05/18 21:32:46 dmh Exp $"; @@ -193,6 +197,7 @@ static void vercheck(int ncid); static long long extractint(NCConstant* con); #ifdef USE_NETCDF4 static int parsefilterflag(const char* sdata0, Specialdata* special); +static int parsecodecsflag(const char* sdata0, Specialdata* special); #ifdef GENDEBUG1 static void printfilters(int nfilters, NC_ParsedFilterSpec** filters); #endif @@ -210,156 +215,258 @@ static void yyerror(fmt,va_alist) const char* fmt; va_dcl; extern int lex_init(void); -#line 214 "ncgeny.c" /* yacc.c:339 */ +#line 219 "ncgeny.c" +# ifndef YY_CAST +# ifdef __cplusplus +# define YY_CAST(Type, Val) static_cast (Val) +# define YY_REINTERPRET_CAST(Type, Val) reinterpret_cast (Val) +# else +# define YY_CAST(Type, Val) ((Type) (Val)) +# define YY_REINTERPRET_CAST(Type, Val) ((Type) (Val)) +# endif +# endif # ifndef YY_NULLPTR -# if defined __cplusplus && 201103L <= __cplusplus -# define YY_NULLPTR nullptr +# if defined __cplusplus +# if 201103L <= __cplusplus +# define YY_NULLPTR nullptr +# else +# define YY_NULLPTR 0 +# endif # else -# define YY_NULLPTR 0 +# define YY_NULLPTR ((void*)0) # endif # endif -/* Enabling verbose error messages. */ -#ifdef YYERROR_VERBOSE -# undef YYERROR_VERBOSE -# define YYERROR_VERBOSE 1 -#else -# define YYERROR_VERBOSE 1 -#endif - -/* In a future release of Bison, this section will be replaced - by #include "ncgeny.h". */ -#ifndef YY_NCG_NCGEN_TAB_H_INCLUDED -# define YY_NCG_NCGEN_TAB_H_INCLUDED -/* Debug traces. */ -#ifndef YYDEBUG -# define YYDEBUG 1 -#endif -#if YYDEBUG -extern int ncgdebug; -#endif - -/* Token type. */ -#ifndef YYTOKENTYPE -# define YYTOKENTYPE - enum yytokentype - { - NC_UNLIMITED_K = 258, - CHAR_K = 259, - BYTE_K = 260, - SHORT_K = 261, - INT_K = 262, - FLOAT_K = 263, - DOUBLE_K = 264, - UBYTE_K = 265, - USHORT_K = 266, - UINT_K = 267, - INT64_K = 268, - UINT64_K = 269, - STRING_K = 270, - IDENT = 271, - TERMSTRING = 272, - CHAR_CONST = 273, - BYTE_CONST = 274, - SHORT_CONST = 275, - INT_CONST = 276, - INT64_CONST = 277, - UBYTE_CONST = 278, - USHORT_CONST = 279, - UINT_CONST = 280, - UINT64_CONST = 281, - FLOAT_CONST = 282, - DOUBLE_CONST = 283, - DIMENSIONS = 284, - VARIABLES = 285, - NETCDF = 286, - DATA = 287, - TYPES = 288, - COMPOUND = 289, - ENUM = 290, - OPAQUE_ = 291, - OPAQUESTRING = 292, - GROUP = 293, - PATH = 294, - FILLMARKER = 295, - NIL = 296, - _FILLVALUE = 297, - _FORMAT = 298, - _STORAGE = 299, - _CHUNKSIZES = 300, - _DEFLATELEVEL = 301, - _SHUFFLE = 302, - _ENDIANNESS = 303, - _NOFILL = 304, - _FLETCHER32 = 305, - _NCPROPS = 306, - _ISNETCDF4 = 307, - _SUPERBLOCK = 308, - _FILTER = 309, - DATASETID = 310 - }; -#endif - -/* Value type. */ -#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED - -union YYSTYPE +#include "ncgeny.h" +/* Symbol kind. */ +enum yysymbol_kind_t { -#line 153 "ncgen.y" /* yacc.c:355 */ - -Symbol* sym; -unsigned long size; /* allow for zero size to indicate e.g. UNLIMITED*/ -long mark; /* track indices into the sequence*/ -int nctype; /* for tracking attribute list type*/ -Datalist* datalist; -NCConstant* constant; - -#line 319 "ncgeny.c" /* yacc.c:355 */ + YYSYMBOL_YYEMPTY = -2, + YYSYMBOL_YYEOF = 0, /* "end of file" */ + YYSYMBOL_YYerror = 1, /* error */ + YYSYMBOL_YYUNDEF = 2, /* "invalid token" */ + YYSYMBOL_NC_UNLIMITED_K = 3, /* NC_UNLIMITED_K */ + YYSYMBOL_CHAR_K = 4, /* CHAR_K */ + YYSYMBOL_BYTE_K = 5, /* BYTE_K */ + YYSYMBOL_SHORT_K = 6, /* SHORT_K */ + YYSYMBOL_INT_K = 7, /* INT_K */ + YYSYMBOL_FLOAT_K = 8, /* FLOAT_K */ + YYSYMBOL_DOUBLE_K = 9, /* DOUBLE_K */ + YYSYMBOL_UBYTE_K = 10, /* UBYTE_K */ + YYSYMBOL_USHORT_K = 11, /* USHORT_K */ + YYSYMBOL_UINT_K = 12, /* UINT_K */ + YYSYMBOL_INT64_K = 13, /* INT64_K */ + YYSYMBOL_UINT64_K = 14, /* UINT64_K */ + YYSYMBOL_STRING_K = 15, /* STRING_K */ + YYSYMBOL_IDENT = 16, /* IDENT */ + YYSYMBOL_TERMSTRING = 17, /* TERMSTRING */ + YYSYMBOL_CHAR_CONST = 18, /* CHAR_CONST */ + YYSYMBOL_BYTE_CONST = 19, /* BYTE_CONST */ + YYSYMBOL_SHORT_CONST = 20, /* SHORT_CONST */ + YYSYMBOL_INT_CONST = 21, /* INT_CONST */ + YYSYMBOL_INT64_CONST = 22, /* INT64_CONST */ + YYSYMBOL_UBYTE_CONST = 23, /* UBYTE_CONST */ + YYSYMBOL_USHORT_CONST = 24, /* USHORT_CONST */ + YYSYMBOL_UINT_CONST = 25, /* UINT_CONST */ + YYSYMBOL_UINT64_CONST = 26, /* UINT64_CONST */ + YYSYMBOL_FLOAT_CONST = 27, /* FLOAT_CONST */ + YYSYMBOL_DOUBLE_CONST = 28, /* DOUBLE_CONST */ + YYSYMBOL_DIMENSIONS = 29, /* DIMENSIONS */ + YYSYMBOL_VARIABLES = 30, /* VARIABLES */ + YYSYMBOL_NETCDF = 31, /* NETCDF */ + YYSYMBOL_DATA = 32, /* DATA */ + YYSYMBOL_TYPES = 33, /* TYPES */ + YYSYMBOL_COMPOUND = 34, /* COMPOUND */ + YYSYMBOL_ENUM = 35, /* ENUM */ + YYSYMBOL_OPAQUE_ = 36, /* OPAQUE_ */ + YYSYMBOL_OPAQUESTRING = 37, /* OPAQUESTRING */ + YYSYMBOL_GROUP = 38, /* GROUP */ + YYSYMBOL_PATH = 39, /* PATH */ + YYSYMBOL_FILLMARKER = 40, /* FILLMARKER */ + YYSYMBOL_NIL = 41, /* NIL */ + YYSYMBOL__FILLVALUE = 42, /* _FILLVALUE */ + YYSYMBOL__FORMAT = 43, /* _FORMAT */ + YYSYMBOL__STORAGE = 44, /* _STORAGE */ + YYSYMBOL__CHUNKSIZES = 45, /* _CHUNKSIZES */ + YYSYMBOL__DEFLATELEVEL = 46, /* _DEFLATELEVEL */ + YYSYMBOL__SHUFFLE = 47, /* _SHUFFLE */ + YYSYMBOL__ENDIANNESS = 48, /* _ENDIANNESS */ + YYSYMBOL__NOFILL = 49, /* _NOFILL */ + YYSYMBOL__FLETCHER32 = 50, /* _FLETCHER32 */ + YYSYMBOL__NCPROPS = 51, /* _NCPROPS */ + YYSYMBOL__ISNETCDF4 = 52, /* _ISNETCDF4 */ + YYSYMBOL__SUPERBLOCK = 53, /* _SUPERBLOCK */ + YYSYMBOL__FILTER = 54, /* _FILTER */ + YYSYMBOL__CODECS = 55, /* _CODECS */ + YYSYMBOL_DATASETID = 56, /* DATASETID */ + YYSYMBOL_57_ = 57, /* '{' */ + YYSYMBOL_58_ = 58, /* '}' */ + YYSYMBOL_59_ = 59, /* ';' */ + YYSYMBOL_60_ = 60, /* ',' */ + YYSYMBOL_61_ = 61, /* '=' */ + YYSYMBOL_62_ = 62, /* '(' */ + YYSYMBOL_63_ = 63, /* ')' */ + YYSYMBOL_64_ = 64, /* '*' */ + YYSYMBOL_65_ = 65, /* ':' */ + YYSYMBOL_YYACCEPT = 66, /* $accept */ + YYSYMBOL_ncdesc = 67, /* ncdesc */ + YYSYMBOL_datasetid = 68, /* datasetid */ + YYSYMBOL_rootgroup = 69, /* rootgroup */ + YYSYMBOL_groupbody = 70, /* groupbody */ + YYSYMBOL_subgrouplist = 71, /* subgrouplist */ + YYSYMBOL_namedgroup = 72, /* namedgroup */ + YYSYMBOL_73_1 = 73, /* $@1 */ + YYSYMBOL_74_2 = 74, /* $@2 */ + YYSYMBOL_typesection = 75, /* typesection */ + YYSYMBOL_typedecls = 76, /* typedecls */ + YYSYMBOL_typename = 77, /* typename */ + YYSYMBOL_type_or_attr_decl = 78, /* type_or_attr_decl */ + YYSYMBOL_typedecl = 79, /* typedecl */ + YYSYMBOL_optsemicolon = 80, /* optsemicolon */ + YYSYMBOL_enumdecl = 81, /* enumdecl */ + YYSYMBOL_enumidlist = 82, /* enumidlist */ + YYSYMBOL_enumid = 83, /* enumid */ + YYSYMBOL_opaquedecl = 84, /* opaquedecl */ + YYSYMBOL_vlendecl = 85, /* vlendecl */ + YYSYMBOL_compounddecl = 86, /* compounddecl */ + YYSYMBOL_fields = 87, /* fields */ + YYSYMBOL_field = 88, /* field */ + YYSYMBOL_primtype = 89, /* primtype */ + YYSYMBOL_dimsection = 90, /* dimsection */ + YYSYMBOL_dimdecls = 91, /* dimdecls */ + YYSYMBOL_dim_or_attr_decl = 92, /* dim_or_attr_decl */ + YYSYMBOL_dimdeclist = 93, /* dimdeclist */ + YYSYMBOL_dimdecl = 94, /* dimdecl */ + YYSYMBOL_dimd = 95, /* dimd */ + YYSYMBOL_vasection = 96, /* vasection */ + YYSYMBOL_vadecls = 97, /* vadecls */ + YYSYMBOL_vadecl_or_attr = 98, /* vadecl_or_attr */ + YYSYMBOL_vardecl = 99, /* vardecl */ + YYSYMBOL_varlist = 100, /* varlist */ + YYSYMBOL_varspec = 101, /* varspec */ + YYSYMBOL_dimspec = 102, /* dimspec */ + YYSYMBOL_dimlist = 103, /* dimlist */ + YYSYMBOL_dimref = 104, /* dimref */ + YYSYMBOL_fieldlist = 105, /* fieldlist */ + YYSYMBOL_fieldspec = 106, /* fieldspec */ + YYSYMBOL_fielddimspec = 107, /* fielddimspec */ + YYSYMBOL_fielddimlist = 108, /* fielddimlist */ + YYSYMBOL_fielddim = 109, /* fielddim */ + YYSYMBOL_varref = 110, /* varref */ + YYSYMBOL_typeref = 111, /* typeref */ + YYSYMBOL_ambiguous_ref = 112, /* ambiguous_ref */ + YYSYMBOL_attrdecllist = 113, /* attrdecllist */ + YYSYMBOL_attrdecl = 114, /* attrdecl */ + YYSYMBOL_path = 115, /* path */ + YYSYMBOL_datasection = 116, /* datasection */ + YYSYMBOL_datadecls = 117, /* datadecls */ + YYSYMBOL_datadecl = 118, /* datadecl */ + YYSYMBOL_datalist = 119, /* datalist */ + YYSYMBOL_datalist0 = 120, /* datalist0 */ + YYSYMBOL_datalist1 = 121, /* datalist1 */ + YYSYMBOL_dataitem = 122, /* dataitem */ + YYSYMBOL_constdata = 123, /* constdata */ + YYSYMBOL_econstref = 124, /* econstref */ + YYSYMBOL_function = 125, /* function */ + YYSYMBOL_arglist = 126, /* arglist */ + YYSYMBOL_simpleconstant = 127, /* simpleconstant */ + YYSYMBOL_intlist = 128, /* intlist */ + YYSYMBOL_constint = 129, /* constint */ + YYSYMBOL_conststring = 130, /* conststring */ + YYSYMBOL_constbool = 131, /* constbool */ + YYSYMBOL_ident = 132 /* ident */ }; +typedef enum yysymbol_kind_t yysymbol_kind_t; -typedef union YYSTYPE YYSTYPE; -# define YYSTYPE_IS_TRIVIAL 1 -# define YYSTYPE_IS_DECLARED 1 -#endif -extern YYSTYPE ncglval; -int ncgparse (void); +#ifdef short +# undef short +#endif -#endif /* !YY_NCG_NCGEN_TAB_H_INCLUDED */ +/* On compilers that do not define __PTRDIFF_MAX__ etc., make sure + and (if available) are included + so that the code can choose integer types of a good width. */ -/* Copy the second part of user declarations. */ +#ifndef __PTRDIFF_MAX__ +# include /* INFRINGES ON USER NAME SPACE */ +# if defined __STDC_VERSION__ && 199901 <= __STDC_VERSION__ +# include /* INFRINGES ON USER NAME SPACE */ +# define YY_STDINT_H +# endif +#endif -#line 336 "ncgeny.c" /* yacc.c:358 */ +/* Narrow types that promote to a signed type and that can represent a + signed or unsigned integer of at least N bits. In tables they can + save space and decrease cache pressure. Promoting to a signed type + helps avoid bugs in integer arithmetic. */ -#ifdef short -# undef short +#ifdef __INT_LEAST8_MAX__ +typedef __INT_LEAST8_TYPE__ yytype_int8; +#elif defined YY_STDINT_H +typedef int_least8_t yytype_int8; +#else +typedef signed char yytype_int8; #endif -#ifdef YYTYPE_UINT8 -typedef YYTYPE_UINT8 yytype_uint8; +#ifdef __INT_LEAST16_MAX__ +typedef __INT_LEAST16_TYPE__ yytype_int16; +#elif defined YY_STDINT_H +typedef int_least16_t yytype_int16; #else -typedef unsigned char yytype_uint8; +typedef short yytype_int16; #endif -#ifdef YYTYPE_INT8 -typedef YYTYPE_INT8 yytype_int8; -#else -typedef signed char yytype_int8; +/* Work around bug in HP-UX 11.23, which defines these macros + incorrectly for preprocessor constants. This workaround can likely + be removed in 2023, as HPE has promised support for HP-UX 11.23 + (aka HP-UX 11i v2) only through the end of 2022; see Table 2 of + . */ +#ifdef __hpux +# undef UINT_LEAST8_MAX +# undef UINT_LEAST16_MAX +# define UINT_LEAST8_MAX 255 +# define UINT_LEAST16_MAX 65535 #endif -#ifdef YYTYPE_UINT16 -typedef YYTYPE_UINT16 yytype_uint16; +#if defined __UINT_LEAST8_MAX__ && __UINT_LEAST8_MAX__ <= __INT_MAX__ +typedef __UINT_LEAST8_TYPE__ yytype_uint8; +#elif (!defined __UINT_LEAST8_MAX__ && defined YY_STDINT_H \ + && UINT_LEAST8_MAX <= INT_MAX) +typedef uint_least8_t yytype_uint8; +#elif !defined __UINT_LEAST8_MAX__ && UCHAR_MAX <= INT_MAX +typedef unsigned char yytype_uint8; #else -typedef unsigned short int yytype_uint16; +typedef short yytype_uint8; #endif -#ifdef YYTYPE_INT16 -typedef YYTYPE_INT16 yytype_int16; +#if defined __UINT_LEAST16_MAX__ && __UINT_LEAST16_MAX__ <= __INT_MAX__ +typedef __UINT_LEAST16_TYPE__ yytype_uint16; +#elif (!defined __UINT_LEAST16_MAX__ && defined YY_STDINT_H \ + && UINT_LEAST16_MAX <= INT_MAX) +typedef uint_least16_t yytype_uint16; +#elif !defined __UINT_LEAST16_MAX__ && USHRT_MAX <= INT_MAX +typedef unsigned short yytype_uint16; #else -typedef short int yytype_int16; +typedef int yytype_uint16; +#endif + +#ifndef YYPTRDIFF_T +# if defined __PTRDIFF_TYPE__ && defined __PTRDIFF_MAX__ +# define YYPTRDIFF_T __PTRDIFF_TYPE__ +# define YYPTRDIFF_MAXIMUM __PTRDIFF_MAX__ +# elif defined PTRDIFF_MAX +# ifndef ptrdiff_t +# include /* INFRINGES ON USER NAME SPACE */ +# endif +# define YYPTRDIFF_T ptrdiff_t +# define YYPTRDIFF_MAXIMUM PTRDIFF_MAX +# else +# define YYPTRDIFF_T long +# define YYPTRDIFF_MAXIMUM LONG_MAX +# endif #endif #ifndef YYSIZE_T @@ -367,15 +474,28 @@ typedef short int yytype_int16; # define YYSIZE_T __SIZE_TYPE__ # elif defined size_t # define YYSIZE_T size_t -# elif ! defined YYSIZE_T +# elif defined __STDC_VERSION__ && 199901 <= __STDC_VERSION__ # include /* INFRINGES ON USER NAME SPACE */ # define YYSIZE_T size_t # else -# define YYSIZE_T unsigned int +# define YYSIZE_T unsigned # endif #endif -#define YYSIZE_MAXIMUM ((YYSIZE_T) -1) +#define YYSIZE_MAXIMUM \ + YY_CAST (YYPTRDIFF_T, \ + (YYPTRDIFF_MAXIMUM < YY_CAST (YYSIZE_T, -1) \ + ? YYPTRDIFF_MAXIMUM \ + : YY_CAST (YYSIZE_T, -1))) + +#define YYSIZEOF(X) YY_CAST (YYPTRDIFF_T, sizeof (X)) + + +/* Stored state numbers (used for stacks). */ +typedef yytype_int16 yy_state_t; + +/* State numbers in computations. */ +typedef int yy_state_fast_t; #ifndef YY_ # if defined YYENABLE_NLS && YYENABLE_NLS @@ -389,47 +509,37 @@ typedef short int yytype_int16; # endif #endif -#ifndef YY_ATTRIBUTE -# if (defined __GNUC__ \ - && (2 < __GNUC__ || (__GNUC__ == 2 && 96 <= __GNUC_MINOR__))) \ - || defined __SUNPRO_C && 0x5110 <= __SUNPRO_C -# define YY_ATTRIBUTE(Spec) __attribute__(Spec) -# else -# define YY_ATTRIBUTE(Spec) /* empty */ -# endif -#endif #ifndef YY_ATTRIBUTE_PURE -# define YY_ATTRIBUTE_PURE YY_ATTRIBUTE ((__pure__)) +# if defined __GNUC__ && 2 < __GNUC__ + (96 <= __GNUC_MINOR__) +# define YY_ATTRIBUTE_PURE __attribute__ ((__pure__)) +# else +# define YY_ATTRIBUTE_PURE +# endif #endif #ifndef YY_ATTRIBUTE_UNUSED -# define YY_ATTRIBUTE_UNUSED YY_ATTRIBUTE ((__unused__)) -#endif - -#if !defined _Noreturn \ - && (!defined __STDC_VERSION__ || __STDC_VERSION__ < 201112) -# if defined _MSC_VER && 1200 <= _MSC_VER -# define _Noreturn __declspec (noreturn) +# if defined __GNUC__ && 2 < __GNUC__ + (7 <= __GNUC_MINOR__) +# define YY_ATTRIBUTE_UNUSED __attribute__ ((__unused__)) # else -# define _Noreturn YY_ATTRIBUTE ((__noreturn__)) +# define YY_ATTRIBUTE_UNUSED # endif #endif /* Suppress unused-variable warnings by "using" E. */ #if ! defined lint || defined __GNUC__ -# define YYUSE(E) ((void) (E)) +# define YY_USE(E) ((void) (E)) #else -# define YYUSE(E) /* empty */ +# define YY_USE(E) /* empty */ #endif -#if defined __GNUC__ && 407 <= __GNUC__ * 100 + __GNUC_MINOR__ +#if defined __GNUC__ && ! defined __ICC && 407 <= __GNUC__ * 100 + __GNUC_MINOR__ /* Suppress an incorrect diagnostic about yylval being uninitialized. */ -# define YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN \ - _Pragma ("GCC diagnostic push") \ - _Pragma ("GCC diagnostic ignored \"-Wuninitialized\"")\ +# define YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN \ + _Pragma ("GCC diagnostic push") \ + _Pragma ("GCC diagnostic ignored \"-Wuninitialized\"") \ _Pragma ("GCC diagnostic ignored \"-Wmaybe-uninitialized\"") -# define YY_IGNORE_MAYBE_UNINITIALIZED_END \ +# define YY_IGNORE_MAYBE_UNINITIALIZED_END \ _Pragma ("GCC diagnostic pop") #else # define YY_INITIAL_VALUE(Value) Value @@ -442,8 +552,22 @@ typedef short int yytype_int16; # define YY_INITIAL_VALUE(Value) /* Nothing. */ #endif +#if defined __cplusplus && defined __GNUC__ && ! defined __ICC && 6 <= __GNUC__ +# define YY_IGNORE_USELESS_CAST_BEGIN \ + _Pragma ("GCC diagnostic push") \ + _Pragma ("GCC diagnostic ignored \"-Wuseless-cast\"") +# define YY_IGNORE_USELESS_CAST_END \ + _Pragma ("GCC diagnostic pop") +#endif +#ifndef YY_IGNORE_USELESS_CAST_BEGIN +# define YY_IGNORE_USELESS_CAST_BEGIN +# define YY_IGNORE_USELESS_CAST_END +#endif + -#if ! defined yyoverflow || YYERROR_VERBOSE +#define YY_ASSERT(E) ((void) (0 && (E))) + +#if 1 /* The parser invokes alloca or malloc; define the necessary symbols. */ @@ -508,8 +632,7 @@ void free (void *); /* INFRINGES ON USER NAME SPACE */ # endif # endif # endif -#endif /* ! defined yyoverflow || YYERROR_VERBOSE */ - +#endif /* 1 */ #if (! defined yyoverflow \ && (! defined __cplusplus \ @@ -518,17 +641,17 @@ void free (void *); /* INFRINGES ON USER NAME SPACE */ /* A type that is properly aligned for any stack member. */ union yyalloc { - yytype_int16 yyss_alloc; + yy_state_t yyss_alloc; YYSTYPE yyvs_alloc; }; /* The size of the maximum gap between one aligned stack and the next. */ -# define YYSTACK_GAP_MAXIMUM (sizeof (union yyalloc) - 1) +# define YYSTACK_GAP_MAXIMUM (YYSIZEOF (union yyalloc) - 1) /* The size of an array large to enough to hold all stacks, each with N elements. */ # define YYSTACK_BYTES(N) \ - ((N) * (sizeof (yytype_int16) + sizeof (YYSTYPE)) \ + ((N) * (YYSIZEOF (yy_state_t) + YYSIZEOF (YYSTYPE)) \ + YYSTACK_GAP_MAXIMUM) # define YYCOPY_NEEDED 1 @@ -541,11 +664,11 @@ union yyalloc # define YYSTACK_RELOCATE(Stack_alloc, Stack) \ do \ { \ - YYSIZE_T yynewbytes; \ + YYPTRDIFF_T yynewbytes; \ YYCOPY (&yyptr->Stack_alloc, Stack, yysize); \ Stack = &yyptr->Stack_alloc; \ - yynewbytes = yystacksize * sizeof (*Stack) + YYSTACK_GAP_MAXIMUM; \ - yyptr += yynewbytes / sizeof (*yyptr); \ + yynewbytes = yystacksize * YYSIZEOF (*Stack) + YYSTACK_GAP_MAXIMUM; \ + yyptr += yynewbytes / YYSIZEOF (*yyptr); \ } \ while (0) @@ -557,12 +680,12 @@ union yyalloc # ifndef YYCOPY # if defined __GNUC__ && 1 < __GNUC__ # define YYCOPY(Dst, Src, Count) \ - __builtin_memcpy (Dst, Src, (Count) * sizeof (*(Src))) + __builtin_memcpy (Dst, Src, YY_CAST (YYSIZE_T, (Count)) * sizeof (*(Src))) # else # define YYCOPY(Dst, Src, Count) \ do \ { \ - YYSIZE_T yyi; \ + YYPTRDIFF_T yyi; \ for (yyi = 0; yyi < (Count); yyi++) \ (Dst)[yyi] = (Src)[yyi]; \ } \ @@ -574,42 +697,45 @@ union yyalloc /* YYFINAL -- State number of the termination state. */ #define YYFINAL 5 /* YYLAST -- Last index in YYTABLE. */ -#define YYLAST 393 +#define YYLAST 424 /* YYNTOKENS -- Number of terminals. */ -#define YYNTOKENS 65 +#define YYNTOKENS 66 /* YYNNTS -- Number of nonterminals. */ #define YYNNTS 67 /* YYNRULES -- Number of rules. */ -#define YYNRULES 153 +#define YYNRULES 154 /* YYNSTATES -- Number of states. */ -#define YYNSTATES 262 +#define YYNSTATES 265 -/* YYTRANSLATE[YYX] -- Symbol number corresponding to YYX as returned - by yylex, with out-of-bounds checking. */ -#define YYUNDEFTOK 2 -#define YYMAXUTOK 310 +/* YYMAXUTOK -- Last valid token kind. */ +#define YYMAXUTOK 311 -#define YYTRANSLATE(YYX) \ - ((unsigned int) (YYX) <= YYMAXUTOK ? yytranslate[YYX] : YYUNDEFTOK) + +/* YYTRANSLATE(TOKEN-NUM) -- Symbol number corresponding to TOKEN-NUM + as returned by yylex, with out-of-bounds checking. */ +#define YYTRANSLATE(YYX) \ + (0 <= (YYX) && (YYX) <= YYMAXUTOK \ + ? YY_CAST (yysymbol_kind_t, yytranslate[YYX]) \ + : YYSYMBOL_YYUNDEF) /* YYTRANSLATE[TOKEN-NUM] -- Symbol number corresponding to TOKEN-NUM - as returned by yylex, without out-of-bounds checking. */ -static const yytype_uint8 yytranslate[] = + as returned by yylex. */ +static const yytype_int8 yytranslate[] = { 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 61, 62, 63, 2, 59, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 64, 58, - 2, 60, 2, 2, 2, 2, 2, 2, 2, 2, + 62, 63, 64, 2, 60, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 65, 59, + 2, 61, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 56, 2, 57, 2, 2, 2, 2, + 2, 2, 2, 57, 2, 58, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, @@ -628,120 +754,134 @@ static const yytype_uint8 yytranslate[] = 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, - 55 + 55, 56 }; #if YYDEBUG /* YYRLINE[YYN] -- Source line where rule number YYN was defined. */ -static const yytype_uint16 yyrline[] = +static const yytype_int16 yyrline[] = { - 0, 235, 235, 241, 243, 250, 257, 257, 260, 269, - 259, 274, 275, 276, 280, 280, 282, 292, 292, 295, - 296, 297, 298, 301, 301, 304, 334, 336, 353, 362, - 374, 388, 421, 422, 425, 439, 440, 441, 442, 443, - 444, 445, 446, 447, 448, 449, 450, 453, 454, 455, - 458, 459, 462, 462, 464, 465, 469, 477, 487, 499, - 500, 501, 504, 505, 508, 508, 510, 532, 536, 540, - 569, 570, 573, 574, 578, 592, 596, 601, 630, 631, - 635, 636, 641, 651, 671, 682, 693, 712, 719, 719, - 722, 724, 726, 728, 730, 739, 750, 752, 754, 756, - 758, 760, 762, 764, 766, 768, 770, 775, 782, 791, - 792, 793, 796, 797, 800, 804, 805, 809, 813, 814, - 819, 820, 824, 825, 826, 827, 828, 829, 833, 837, - 841, 843, 848, 849, 850, 851, 852, 853, 854, 855, - 856, 857, 858, 859, 863, 864, 868, 870, 872, 874, - 879, 883, 884, 890 + 0, 237, 237, 243, 245, 252, 259, 259, 262, 271, + 261, 276, 277, 278, 282, 282, 284, 294, 294, 297, + 298, 299, 300, 303, 303, 306, 336, 338, 355, 364, + 376, 390, 423, 424, 427, 441, 442, 443, 444, 445, + 446, 447, 448, 449, 450, 451, 452, 455, 456, 457, + 460, 461, 464, 464, 466, 467, 471, 479, 489, 501, + 502, 503, 506, 507, 510, 510, 512, 534, 538, 542, + 571, 572, 575, 576, 580, 594, 598, 603, 632, 633, + 637, 638, 643, 653, 673, 684, 695, 714, 721, 721, + 724, 726, 728, 730, 732, 741, 752, 754, 756, 758, + 760, 762, 764, 766, 768, 770, 772, 774, 779, 786, + 795, 796, 797, 800, 801, 804, 808, 809, 813, 817, + 818, 823, 824, 828, 829, 830, 831, 832, 833, 837, + 841, 845, 847, 852, 853, 854, 855, 856, 857, 858, + 859, 860, 861, 862, 863, 867, 868, 872, 874, 876, + 878, 883, 887, 888, 894 }; #endif -#if YYDEBUG || YYERROR_VERBOSE || 1 +/** Accessing symbol of state STATE. */ +#define YY_ACCESSING_SYMBOL(State) YY_CAST (yysymbol_kind_t, yystos[State]) + +#if 1 +/* The user-facing name of the symbol whose (internal) number is + YYSYMBOL. No bounds checking. */ +static const char *yysymbol_name (yysymbol_kind_t yysymbol) YY_ATTRIBUTE_UNUSED; + /* YYTNAME[SYMBOL-NUM] -- String name of the symbol SYMBOL-NUM. First, the terminals, then, starting at YYNTOKENS, nonterminals. */ static const char *const yytname[] = { - "$end", "error", "$undefined", "NC_UNLIMITED_K", "CHAR_K", "BYTE_K", - "SHORT_K", "INT_K", "FLOAT_K", "DOUBLE_K", "UBYTE_K", "USHORT_K", - "UINT_K", "INT64_K", "UINT64_K", "STRING_K", "IDENT", "TERMSTRING", - "CHAR_CONST", "BYTE_CONST", "SHORT_CONST", "INT_CONST", "INT64_CONST", - "UBYTE_CONST", "USHORT_CONST", "UINT_CONST", "UINT64_CONST", - "FLOAT_CONST", "DOUBLE_CONST", "DIMENSIONS", "VARIABLES", "NETCDF", - "DATA", "TYPES", "COMPOUND", "ENUM", "OPAQUE_", "OPAQUESTRING", "GROUP", - "PATH", "FILLMARKER", "NIL", "_FILLVALUE", "_FORMAT", "_STORAGE", - "_CHUNKSIZES", "_DEFLATELEVEL", "_SHUFFLE", "_ENDIANNESS", "_NOFILL", - "_FLETCHER32", "_NCPROPS", "_ISNETCDF4", "_SUPERBLOCK", "_FILTER", - "DATASETID", "'{'", "'}'", "';'", "','", "'='", "'('", "')'", "'*'", - "':'", "$accept", "ncdesc", "datasetid", "rootgroup", "groupbody", - "subgrouplist", "namedgroup", "$@1", "$@2", "typesection", "typedecls", - "typename", "type_or_attr_decl", "typedecl", "optsemicolon", "enumdecl", - "enumidlist", "enumid", "opaquedecl", "vlendecl", "compounddecl", - "fields", "field", "primtype", "dimsection", "dimdecls", - "dim_or_attr_decl", "dimdeclist", "dimdecl", "dimd", "vasection", - "vadecls", "vadecl_or_attr", "vardecl", "varlist", "varspec", "dimspec", - "dimlist", "dimref", "fieldlist", "fieldspec", "fielddimspec", - "fielddimlist", "fielddim", "varref", "typeref", "ambiguous_ref", - "attrdecllist", "attrdecl", "path", "datasection", "datadecls", - "datadecl", "datalist", "datalist0", "datalist1", "dataitem", - "constdata", "econstref", "function", "arglist", "simpleconstant", - "intlist", "constint", "conststring", "constbool", "ident", YY_NULLPTR + "\"end of file\"", "error", "\"invalid token\"", "NC_UNLIMITED_K", + "CHAR_K", "BYTE_K", "SHORT_K", "INT_K", "FLOAT_K", "DOUBLE_K", "UBYTE_K", + "USHORT_K", "UINT_K", "INT64_K", "UINT64_K", "STRING_K", "IDENT", + "TERMSTRING", "CHAR_CONST", "BYTE_CONST", "SHORT_CONST", "INT_CONST", + "INT64_CONST", "UBYTE_CONST", "USHORT_CONST", "UINT_CONST", + "UINT64_CONST", "FLOAT_CONST", "DOUBLE_CONST", "DIMENSIONS", "VARIABLES", + "NETCDF", "DATA", "TYPES", "COMPOUND", "ENUM", "OPAQUE_", "OPAQUESTRING", + "GROUP", "PATH", "FILLMARKER", "NIL", "_FILLVALUE", "_FORMAT", + "_STORAGE", "_CHUNKSIZES", "_DEFLATELEVEL", "_SHUFFLE", "_ENDIANNESS", + "_NOFILL", "_FLETCHER32", "_NCPROPS", "_ISNETCDF4", "_SUPERBLOCK", + "_FILTER", "_CODECS", "DATASETID", "'{'", "'}'", "';'", "','", "'='", + "'('", "')'", "'*'", "':'", "$accept", "ncdesc", "datasetid", + "rootgroup", "groupbody", "subgrouplist", "namedgroup", "$@1", "$@2", + "typesection", "typedecls", "typename", "type_or_attr_decl", "typedecl", + "optsemicolon", "enumdecl", "enumidlist", "enumid", "opaquedecl", + "vlendecl", "compounddecl", "fields", "field", "primtype", "dimsection", + "dimdecls", "dim_or_attr_decl", "dimdeclist", "dimdecl", "dimd", + "vasection", "vadecls", "vadecl_or_attr", "vardecl", "varlist", + "varspec", "dimspec", "dimlist", "dimref", "fieldlist", "fieldspec", + "fielddimspec", "fielddimlist", "fielddim", "varref", "typeref", + "ambiguous_ref", "attrdecllist", "attrdecl", "path", "datasection", + "datadecls", "datadecl", "datalist", "datalist0", "datalist1", + "dataitem", "constdata", "econstref", "function", "arglist", + "simpleconstant", "intlist", "constint", "conststring", "constbool", + "ident", YY_NULLPTR }; + +static const char * +yysymbol_name (yysymbol_kind_t yysymbol) +{ + return yytname[yysymbol]; +} #endif -# ifdef YYPRINT +#ifdef YYPRINT /* YYTOKNUM[NUM] -- (External) token number corresponding to the (internal) symbol number NUM (which must be that of a token). */ -static const yytype_uint16 yytoknum[] = +static const yytype_int16 yytoknum[] = { 0, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, - 305, 306, 307, 308, 309, 310, 123, 125, 59, 44, - 61, 40, 41, 42, 58 + 305, 306, 307, 308, 309, 310, 311, 123, 125, 59, + 44, 61, 40, 41, 42, 58 }; -# endif +#endif -#define YYPACT_NINF -145 +#define YYPACT_NINF (-147) -#define yypact_value_is_default(Yystate) \ - (!!((Yystate) == (-145))) +#define yypact_value_is_default(Yyn) \ + ((Yyn) == YYPACT_NINF) -#define YYTABLE_NINF -108 +#define YYTABLE_NINF (-109) -#define yytable_value_is_error(Yytable_value) \ +#define yytable_value_is_error(Yyn) \ 0 /* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing STATE-NUM. */ static const yytype_int16 yypact[] = { - -10, -19, 54, -145, 12, -145, 219, -145, -145, -145, - -145, -145, -145, -145, -145, -145, -145, -145, -145, -145, - -145, -145, -8, -145, -145, 354, -9, 26, 11, -145, - -145, 15, 28, 31, 32, 36, -11, 34, 130, 183, - 70, 219, 83, 83, 45, 52, 301, 85, -145, -145, - -1, 42, 46, 49, 50, 51, 57, 58, 59, 60, - 61, 85, 53, 183, -145, -145, 64, 64, 64, 64, - 88, 255, 66, 219, 95, -145, -145, -145, -145, -145, - -145, -145, -145, -145, -145, -145, -145, -145, -145, -145, - -145, -145, -145, -145, -145, -145, -145, -145, -145, -145, - -145, -145, 301, -145, 68, -145, -145, -145, -145, -145, - -145, -145, 71, 75, 73, 77, 301, 83, 52, 52, - 45, 83, 45, 45, 83, 301, 79, -145, 117, -145, - -145, -145, -145, -145, -145, 85, 78, -145, 219, 82, - 86, -145, 84, -145, 87, 219, 118, 30, 301, 355, - -145, 301, 301, 68, -145, 90, -145, -145, -145, -145, - -145, -145, -145, 68, 354, 93, 96, 97, 100, -145, - 85, 35, 219, 102, -145, 354, -145, 354, -145, -145, - -145, -28, -145, 219, 68, 68, 52, 291, 103, 85, - -145, 85, 85, 85, -145, -145, -145, -145, -145, 104, - -145, 105, -145, -32, 108, -145, 354, 107, 355, -145, - -145, -145, -145, 115, -145, 123, -145, 125, -145, 38, - -145, 140, -145, -145, 85, 3, -145, 301, 143, -145, - -145, 166, -145, 85, -7, -145, -145, 85, 52, -145, - 145, 17, -145, -145, 68, -145, 106, -145, -145, -145, - 23, -145, -145, -145, 3, -145, 219, -7, -145, -145, - -145, -145 + -12, -6, 56, -147, 1, -147, 229, -147, -147, -147, + -147, -147, -147, -147, -147, -147, -147, -147, -147, -147, + -147, -147, -4, -147, -147, 385, -3, 28, 12, -147, + -147, 14, 19, 21, 26, 33, -16, 31, 143, 216, + 48, 229, 82, 82, 76, 8, 347, 84, -147, -147, + -1, 42, 43, 46, 47, 50, 52, 55, 59, 60, + 61, 62, 84, 63, 216, -147, -147, 58, 58, 58, + 58, 89, 291, 67, 229, 97, -147, -147, -147, -147, + -147, -147, -147, -147, -147, -147, -147, -147, -147, -147, + -147, -147, -147, -147, -147, -147, -147, -147, -147, -147, + -147, -147, -147, 347, -147, 69, -147, -147, -147, -147, + -147, -147, -147, 73, 79, 77, 78, 347, 82, 8, + 8, 76, 82, 76, 76, 82, 82, 347, 80, -147, + 119, -147, -147, -147, -147, -147, -147, 84, 81, -147, + 229, 85, 83, -147, 88, -147, 92, 229, 109, 10, + 347, 239, -147, 347, 347, 69, -147, 86, -147, -147, + -147, -147, -147, -147, -147, -147, 69, 385, 87, 98, + 91, 99, -147, 84, 64, 229, 101, -147, 385, -147, + 385, -147, -147, -147, -37, -147, 229, 69, 69, 8, + 304, 102, 84, -147, 84, 84, 84, -147, -147, -147, + -147, -147, 103, -147, 104, -147, -30, 105, -147, 385, + 108, 239, -147, -147, -147, -147, 110, -147, 111, -147, + 106, -147, 16, -147, 115, -147, -147, 84, -2, -147, + 347, 118, -147, -147, 140, -147, 84, 30, -147, -147, + 84, 8, -147, 117, 0, -147, -147, 69, -147, 122, + -147, -147, -147, 9, -147, -147, -147, -2, -147, 229, + 30, -147, -147, -147, -147 }; /* YYDEFACT[STATE-NUM] -- Default reduction number in state STATE-NUM. @@ -751,55 +891,55 @@ static const yytype_uint8 yydefact[] = { 0, 0, 0, 3, 0, 1, 88, 2, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, - 153, 108, 0, 6, 87, 0, 85, 11, 0, 86, - 107, 0, 0, 0, 0, 0, 0, 0, 0, 12, - 47, 88, 0, 0, 0, 0, 117, 0, 4, 7, + 154, 109, 0, 6, 87, 0, 85, 11, 0, 86, + 108, 0, 0, 0, 0, 0, 0, 0, 0, 12, + 47, 88, 0, 0, 0, 0, 118, 0, 4, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 13, 14, 17, 23, 23, 23, 23, - 87, 0, 0, 48, 59, 89, 150, 106, 90, 146, - 148, 147, 149, 152, 151, 91, 92, 143, 132, 133, - 134, 135, 136, 137, 138, 139, 140, 141, 142, 123, - 124, 125, 117, 128, 93, 115, 116, 118, 120, 126, - 127, 122, 107, 0, 0, 0, 117, 0, 0, 0, - 0, 0, 0, 0, 0, 117, 0, 16, 0, 15, - 24, 19, 22, 21, 20, 0, 0, 18, 49, 0, - 52, 54, 0, 53, 107, 60, 109, 0, 0, 0, - 8, 117, 117, 96, 98, 99, 144, 101, 102, 103, - 105, 100, 104, 95, 0, 0, 0, 0, 0, 50, - 0, 0, 61, 0, 64, 0, 65, 110, 5, 121, - 119, 0, 130, 88, 97, 94, 0, 0, 0, 0, - 85, 0, 0, 0, 51, 55, 58, 57, 56, 0, - 62, 66, 67, 70, 0, 84, 111, 0, 0, 129, - 6, 145, 31, 0, 32, 34, 75, 78, 29, 0, - 26, 0, 30, 63, 0, 0, 69, 117, 0, 112, - 131, 9, 33, 0, 0, 77, 25, 0, 0, 68, - 70, 0, 72, 74, 114, 113, 0, 76, 83, 82, - 0, 80, 27, 28, 0, 71, 88, 0, 79, 73, - 10, 81 + 0, 0, 0, 0, 13, 14, 17, 23, 23, 23, + 23, 87, 0, 0, 48, 59, 89, 151, 107, 90, + 147, 149, 148, 150, 153, 152, 91, 92, 144, 133, + 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, + 124, 125, 126, 118, 129, 93, 116, 117, 119, 121, + 127, 128, 123, 108, 0, 0, 0, 118, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 118, 0, 16, + 0, 15, 24, 19, 22, 21, 20, 0, 0, 18, + 49, 0, 52, 54, 0, 53, 108, 60, 110, 0, + 0, 0, 8, 118, 118, 96, 98, 99, 145, 101, + 102, 103, 106, 100, 104, 105, 95, 0, 0, 0, + 0, 0, 50, 0, 0, 61, 0, 64, 0, 65, + 111, 5, 122, 120, 0, 131, 88, 97, 94, 0, + 0, 0, 0, 85, 0, 0, 0, 51, 55, 58, + 57, 56, 0, 62, 66, 67, 70, 0, 84, 112, + 0, 0, 130, 6, 146, 31, 0, 32, 34, 75, + 78, 29, 0, 26, 0, 30, 63, 0, 0, 69, + 118, 0, 113, 132, 9, 33, 0, 0, 77, 25, + 0, 0, 68, 70, 0, 72, 74, 115, 114, 0, + 76, 83, 82, 0, 80, 27, 28, 0, 71, 88, + 0, 79, 73, 10, 81 }; /* YYPGOTO[NTERM-NUM]. */ static const yytype_int16 yypgoto[] = { - -145, -145, -145, -145, 24, -2, -145, -145, -145, -145, - -145, -128, 146, -145, -20, -145, -145, -25, -145, -145, - -145, -145, 27, -26, -145, -145, 80, -145, 43, -145, - -145, -145, 48, -145, -145, -3, -145, -145, -18, -145, - 4, -145, -145, -17, -145, -30, -21, -40, -33, -44, - -145, -145, 33, -99, -145, -145, 94, -145, -145, -145, - -145, -144, -145, -35, -31, -100, -22 + -147, -147, -147, -147, -5, -31, -147, -147, -147, -147, + -147, -130, 131, -147, -25, -147, -147, -57, -147, -147, + -147, -147, 6, -26, -147, -147, 66, -147, 29, -147, + -147, -147, 24, -147, -147, -24, -147, -147, -56, -147, + -32, -147, -147, -53, -147, -33, -15, -40, -28, -44, + -147, -147, 2, -100, -147, -147, 65, -147, -147, -147, + -147, -146, -147, -41, -34, -103, -22 }; /* YYDEFGOTO[NTERM-NUM]. */ -static const yytype_int16 yydefgoto[] = +static const yytype_uint8 yydefgoto[] = { - -1, 2, 4, 7, 23, 36, 49, 183, 246, 40, - 63, 126, 64, 65, 131, 66, 219, 220, 67, 68, - 69, 187, 188, 24, 74, 138, 139, 140, 141, 142, - 146, 172, 173, 174, 201, 202, 226, 241, 242, 215, - 216, 235, 250, 251, 204, 25, 26, 27, 28, 29, - 178, 206, 207, 104, 105, 106, 107, 108, 109, 110, - 181, 111, 155, 83, 84, 85, 30 + 0, 2, 4, 7, 23, 36, 49, 186, 249, 40, + 64, 128, 65, 66, 133, 67, 222, 223, 68, 69, + 70, 190, 191, 24, 75, 140, 141, 142, 143, 144, + 148, 175, 176, 177, 204, 205, 229, 244, 245, 218, + 219, 238, 253, 254, 207, 25, 26, 27, 28, 29, + 181, 209, 210, 105, 106, 107, 108, 109, 110, 111, + 184, 112, 157, 84, 85, 86, 30 }; /* YYTABLE[YYPACT[STATE-NUM]] -- What to do in state STATE-NUM. If @@ -807,148 +947,154 @@ static const yytype_int16 yydefgoto[] = number is the opposite. If YYTABLE_NINF, syntax error. */ static const yytype_int16 yytable[] = { - 35, 75, 103, 147, 37, 182, 72, 166, 20, 71, - 86, 77, 78, 70, 248, 20, 60, 153, 249, 20, - 158, 1, 160, 161, 112, 113, 163, 47, 115, 225, - 72, 208, -107, 71, 209, 31, 3, 70, 197, 127, - 143, 114, 21, 32, 33, 34, 48, 132, 133, 134, - 37, 144, 184, 185, 5, 38, 79, 80, 103, 39, - 81, 82, 76, 218, 230, 222, 79, 80, 6, 41, - 81, 82, 103, 79, 80, 42, 254, 81, 82, 255, - 112, 103, 257, 156, 157, 258, 154, 179, 43, 148, - 159, 44, 45, 162, 112, 236, 46, 237, 50, 73, - 76, 20, 116, 112, 103, 143, 117, 103, 103, 118, - 119, 120, 176, 127, 128, 175, 144, 121, 122, 123, - 124, 125, 130, 135, 137, 145, 112, 148, 244, 112, - 112, 150, 149, 151, 189, 164, 198, 152, 165, 176, - 169, 167, 175, 190, 171, 170, 20, -58, 196, 186, - 177, 211, 192, 203, 37, 191, 205, 189, 194, 193, - 200, 214, 223, 256, 224, 229, 190, 217, 227, 127, - 221, 127, 51, 232, 52, 53, 54, 55, 56, 57, - 58, 243, 233, 103, 59, 205, 234, 8, 9, 10, - 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, - 238, 245, 240, 253, 47, 112, 225, 210, 231, 129, - 243, 217, 252, 195, 213, 221, 260, 61, 168, 62, - 199, 239, 21, 8, 9, 10, 11, 12, 13, 14, - 15, 16, 17, 18, 19, 20, 259, 247, 0, 228, - 261, 0, 180, 0, 0, 0, 0, 22, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 21, 8, - 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, - 19, 20, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 21, 8, 9, 10, 11, 12, - 13, 14, 15, 16, 17, 18, 19, 20, 0, 0, - 0, 0, 0, 0, 0, 0, 136, 20, 87, 88, - 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, - 21, 0, 0, 0, 0, 0, 0, 0, 99, 0, - 21, 100, 101, 0, 0, 0, 0, 0, 212, 0, - 0, 0, 0, 0, 0, 0, 0, 102, 8, 9, + 35, 76, 104, 149, 87, 185, 72, 169, 78, 79, + 37, 73, 20, 71, 20, 20, 61, 155, 160, 1, + 162, 163, 47, 211, 113, 114, 212, 166, 116, 80, + 81, 72, 228, 82, 83, -108, 73, 21, 71, 31, + 129, 115, 48, 134, 135, 136, 145, 32, 33, 34, + 3, 251, 146, 187, 188, 252, 5, 37, 6, 104, + 257, 39, 38, 258, 221, 233, 225, 200, 182, 260, + 150, 41, 261, 104, 239, 42, 240, 74, 158, 159, + 43, 113, 44, 104, 156, 80, 81, 45, 161, 82, + 83, 164, 165, 77, 46, 113, 50, 80, 81, 77, + 20, 82, 83, 117, 118, 113, 104, 119, 120, 104, + 104, 121, 145, 122, 178, 129, 123, 132, 146, 179, + 124, 125, 126, 127, 137, 130, 139, 147, 113, 150, + 247, 113, 113, 201, 192, 151, 152, 167, 153, 154, + 168, 180, 178, 173, 172, 170, 189, 179, 214, 174, + 194, 199, 193, -58, 196, 195, 206, 192, 197, 20, + 203, 217, 226, 37, 227, 208, 230, 232, 237, 235, + 220, 236, 129, 224, 129, 193, 241, 248, 47, 228, + 259, 213, 234, 255, 246, 51, 104, 52, 53, 54, + 55, 56, 57, 58, 208, 131, 216, 59, 60, 202, + 256, 262, 198, 242, 250, 243, 171, 264, 113, 0, + 0, 231, 0, 246, 220, 183, 0, 0, 224, 263, + 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, + 18, 19, 20, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 0, 0, 0, 0, + 62, 0, 63, 0, 0, 21, 88, 89, 90, 91, + 92, 93, 94, 95, 96, 97, 98, 99, 21, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 22, 8, 9, 10, 11, 12, + 13, 14, 15, 16, 17, 18, 19, 20, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, - 20, 0, 87, 88, 89, 90, 91, 92, 93, 94, - 95, 96, 97, 98, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 21 + 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 138, 0, 0, 0, 0, 0, 0, + 0, 0, 215, 20, 88, 89, 90, 91, 92, 93, + 94, 95, 96, 97, 98, 99, 0, 0, 0, 0, + 0, 0, 0, 0, 100, 0, 21, 101, 102, 8, + 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 0, 0, 103, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 21 }; static const yytype_int16 yycheck[] = { - 22, 41, 46, 102, 25, 149, 39, 135, 16, 39, - 45, 42, 43, 39, 21, 16, 38, 116, 25, 16, - 120, 31, 122, 123, 46, 47, 125, 38, 50, 61, - 63, 59, 64, 63, 62, 43, 55, 63, 3, 61, - 73, 42, 39, 51, 52, 53, 57, 67, 68, 69, - 71, 73, 151, 152, 0, 64, 21, 22, 102, 33, - 25, 26, 17, 191, 208, 193, 21, 22, 56, 58, - 25, 26, 116, 21, 22, 60, 59, 25, 26, 62, - 102, 125, 59, 118, 119, 62, 117, 57, 60, 59, - 121, 60, 60, 124, 116, 57, 60, 59, 64, 29, - 17, 16, 60, 125, 148, 138, 60, 151, 152, 60, - 60, 60, 145, 135, 61, 145, 138, 60, 60, 60, - 60, 60, 58, 35, 58, 30, 148, 59, 227, 151, - 152, 56, 61, 60, 164, 56, 171, 60, 21, 172, - 58, 63, 172, 164, 60, 59, 16, 60, 170, 59, - 32, 186, 56, 175, 175, 62, 177, 187, 58, 62, - 58, 58, 58, 57, 59, 58, 187, 189, 60, 191, - 192, 193, 42, 58, 44, 45, 46, 47, 48, 49, - 50, 225, 59, 227, 54, 206, 61, 4, 5, 6, - 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, - 60, 58, 224, 238, 38, 227, 61, 183, 210, 63, - 254, 233, 237, 170, 187, 237, 256, 34, 138, 36, - 172, 224, 39, 4, 5, 6, 7, 8, 9, 10, - 11, 12, 13, 14, 15, 16, 254, 233, -1, 206, - 257, -1, 148, -1, -1, -1, -1, 64, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, 39, 4, - 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 15, 16, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 64, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 39, 4, 5, 6, 7, 8, - 9, 10, 11, 12, 13, 14, 15, 16, -1, -1, - -1, -1, -1, -1, -1, -1, 61, 16, 17, 18, - 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, - 39, -1, -1, -1, -1, -1, -1, -1, 37, -1, - 39, 40, 41, -1, -1, -1, -1, -1, 57, -1, - -1, -1, -1, -1, -1, -1, -1, 56, 4, 5, + 22, 41, 46, 103, 45, 151, 39, 137, 42, 43, + 25, 39, 16, 39, 16, 16, 38, 117, 121, 31, + 123, 124, 38, 60, 46, 47, 63, 127, 50, 21, + 22, 64, 62, 25, 26, 65, 64, 39, 64, 43, + 62, 42, 58, 68, 69, 70, 74, 51, 52, 53, + 56, 21, 74, 153, 154, 25, 0, 72, 57, 103, + 60, 33, 65, 63, 194, 211, 196, 3, 58, 60, + 60, 59, 63, 117, 58, 61, 60, 29, 119, 120, + 61, 103, 61, 127, 118, 21, 22, 61, 122, 25, + 26, 125, 126, 17, 61, 117, 65, 21, 22, 17, + 16, 25, 26, 61, 61, 127, 150, 61, 61, 153, + 154, 61, 140, 61, 147, 137, 61, 59, 140, 147, + 61, 61, 61, 61, 35, 62, 59, 30, 150, 60, + 230, 153, 154, 174, 167, 62, 57, 57, 61, 61, + 21, 32, 175, 60, 59, 64, 60, 175, 189, 61, + 63, 173, 167, 61, 63, 57, 178, 190, 59, 16, + 59, 59, 59, 178, 60, 180, 61, 59, 62, 59, + 192, 60, 194, 195, 196, 190, 61, 59, 38, 62, + 58, 186, 213, 240, 228, 42, 230, 44, 45, 46, + 47, 48, 49, 50, 209, 64, 190, 54, 55, 175, + 241, 257, 173, 227, 236, 227, 140, 260, 230, -1, + -1, 209, -1, 257, 236, 150, -1, -1, 240, 259, + 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, + 14, 15, 16, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, -1, -1, -1, -1, + 34, -1, 36, -1, -1, 39, 17, 18, 19, 20, + 21, 22, 23, 24, 25, 26, 27, 28, 39, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 65, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 65, 4, 5, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 16, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 16, -1, 17, 18, 19, 20, 21, 22, 23, 24, - 25, 26, 27, 28, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 39 + 16, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 39, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 39, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 62, -1, -1, -1, -1, -1, -1, + -1, -1, 58, 16, 17, 18, 19, 20, 21, 22, + 23, 24, 25, 26, 27, 28, -1, -1, -1, -1, + -1, -1, -1, -1, 37, -1, 39, 40, 41, 4, + 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, -1, -1, 57, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 39 }; /* YYSTOS[STATE-NUM] -- The (internal number of the) accessing symbol of state STATE-NUM. */ static const yytype_uint8 yystos[] = { - 0, 31, 66, 55, 67, 0, 56, 68, 4, 5, + 0, 31, 67, 56, 68, 0, 57, 69, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 16, 39, 64, 69, 88, 110, 111, 112, 113, 114, - 131, 43, 51, 52, 53, 131, 70, 111, 64, 33, - 74, 58, 60, 60, 60, 60, 60, 38, 57, 71, - 64, 42, 44, 45, 46, 47, 48, 49, 50, 54, - 131, 34, 36, 75, 77, 78, 80, 83, 84, 85, - 88, 110, 113, 29, 89, 112, 17, 129, 129, 21, - 22, 25, 26, 128, 129, 130, 128, 17, 18, 19, - 20, 21, 22, 23, 24, 25, 26, 27, 28, 37, - 40, 41, 56, 114, 118, 119, 120, 121, 122, 123, - 124, 126, 131, 131, 42, 131, 60, 60, 60, 60, - 60, 60, 60, 60, 60, 60, 76, 131, 61, 77, - 58, 79, 79, 79, 79, 35, 61, 58, 90, 91, - 92, 93, 94, 113, 131, 30, 95, 118, 59, 61, - 56, 60, 60, 118, 129, 127, 128, 128, 130, 129, - 130, 130, 129, 118, 56, 21, 76, 63, 91, 58, - 59, 60, 96, 97, 98, 110, 113, 32, 115, 57, - 121, 125, 126, 72, 118, 118, 59, 86, 87, 110, - 111, 62, 56, 62, 58, 93, 131, 3, 128, 97, - 58, 99, 100, 131, 109, 111, 116, 117, 59, 62, - 69, 128, 57, 87, 58, 104, 105, 131, 76, 81, - 82, 131, 76, 58, 59, 61, 101, 60, 117, 58, - 126, 70, 58, 59, 61, 106, 57, 59, 60, 100, - 131, 102, 103, 114, 118, 58, 73, 105, 21, 25, - 107, 108, 82, 128, 59, 62, 57, 59, 62, 103, - 112, 108 + 16, 39, 65, 70, 89, 111, 112, 113, 114, 115, + 132, 43, 51, 52, 53, 132, 71, 112, 65, 33, + 75, 59, 61, 61, 61, 61, 61, 38, 58, 72, + 65, 42, 44, 45, 46, 47, 48, 49, 50, 54, + 55, 132, 34, 36, 76, 78, 79, 81, 84, 85, + 86, 89, 111, 114, 29, 90, 113, 17, 130, 130, + 21, 22, 25, 26, 129, 130, 131, 129, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, + 37, 40, 41, 57, 115, 119, 120, 121, 122, 123, + 124, 125, 127, 132, 132, 42, 132, 61, 61, 61, + 61, 61, 61, 61, 61, 61, 61, 61, 77, 132, + 62, 78, 59, 80, 80, 80, 80, 35, 62, 59, + 91, 92, 93, 94, 95, 114, 132, 30, 96, 119, + 60, 62, 57, 61, 61, 119, 130, 128, 129, 129, + 131, 130, 131, 131, 130, 130, 119, 57, 21, 77, + 64, 92, 59, 60, 61, 97, 98, 99, 111, 114, + 32, 116, 58, 122, 126, 127, 73, 119, 119, 60, + 87, 88, 111, 112, 63, 57, 63, 59, 94, 132, + 3, 129, 98, 59, 100, 101, 132, 110, 112, 117, + 118, 60, 63, 70, 129, 58, 88, 59, 105, 106, + 132, 77, 82, 83, 132, 77, 59, 60, 62, 102, + 61, 118, 59, 127, 71, 59, 60, 62, 107, 58, + 60, 61, 101, 132, 103, 104, 115, 119, 59, 74, + 106, 21, 25, 108, 109, 83, 129, 60, 63, 58, + 60, 63, 104, 113, 109 }; /* YYR1[YYN] -- Symbol number of symbol that rule YYN derives. */ static const yytype_uint8 yyr1[] = { - 0, 65, 66, 67, 68, 69, 70, 70, 72, 73, - 71, 74, 74, 74, 75, 75, 76, 77, 77, 78, - 78, 78, 78, 79, 79, 80, 81, 81, 82, 83, - 84, 85, 86, 86, 87, 88, 88, 88, 88, 88, - 88, 88, 88, 88, 88, 88, 88, 89, 89, 89, - 90, 90, 91, 91, 92, 92, 93, 93, 94, 95, - 95, 95, 96, 96, 97, 97, 98, 99, 99, 100, - 101, 101, 102, 102, 103, 104, 104, 105, 106, 106, - 107, 107, 108, 108, 109, 110, 111, 111, 112, 112, - 113, 113, 113, 113, 113, 113, 113, 113, 113, 113, - 113, 113, 113, 113, 113, 113, 113, 114, 114, 115, - 115, 115, 116, 116, 117, 118, 118, 119, 120, 120, - 121, 121, 122, 122, 122, 122, 122, 122, 123, 124, - 125, 125, 126, 126, 126, 126, 126, 126, 126, 126, - 126, 126, 126, 126, 127, 127, 128, 128, 128, 128, - 129, 130, 130, 131 + 0, 66, 67, 68, 69, 70, 71, 71, 73, 74, + 72, 75, 75, 75, 76, 76, 77, 78, 78, 79, + 79, 79, 79, 80, 80, 81, 82, 82, 83, 84, + 85, 86, 87, 87, 88, 89, 89, 89, 89, 89, + 89, 89, 89, 89, 89, 89, 89, 90, 90, 90, + 91, 91, 92, 92, 93, 93, 94, 94, 95, 96, + 96, 96, 97, 97, 98, 98, 99, 100, 100, 101, + 102, 102, 103, 103, 104, 105, 105, 106, 107, 107, + 108, 108, 109, 109, 110, 111, 112, 112, 113, 113, + 114, 114, 114, 114, 114, 114, 114, 114, 114, 114, + 114, 114, 114, 114, 114, 114, 114, 114, 115, 115, + 116, 116, 116, 117, 117, 118, 119, 119, 120, 121, + 121, 122, 122, 123, 123, 123, 123, 123, 123, 124, + 125, 126, 126, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 128, 128, 129, 129, 129, + 129, 130, 131, 131, 132 }; /* YYR2[YYN] -- Number of symbols on the right hand side of rule YYN. */ -static const yytype_uint8 yyr2[] = +static const yytype_int8 yyr2[] = { 0, 2, 3, 1, 4, 5, 0, 2, 0, 0, 9, 0, 1, 2, 1, 2, 1, 1, 2, 2, @@ -960,19 +1106,19 @@ static const yytype_uint8 yyr2[] = 0, 3, 1, 3, 1, 1, 3, 2, 0, 3, 1, 3, 1, 1, 1, 1, 1, 1, 0, 3, 4, 4, 4, 4, 6, 5, 5, 6, 5, 5, - 5, 5, 5, 5, 5, 5, 4, 1, 1, 0, - 1, 2, 2, 3, 3, 1, 1, 0, 1, 3, - 1, 3, 1, 1, 1, 1, 1, 1, 1, 4, - 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, - 1, 1, 1, 1 + 5, 5, 5, 5, 5, 5, 5, 4, 1, 1, + 0, 1, 2, 2, 3, 3, 1, 1, 0, 1, + 3, 1, 3, 1, 1, 1, 1, 1, 1, 1, + 4, 1, 3, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, + 1, 1, 1, 1, 1 }; +enum { YYENOMEM = -2 }; + #define yyerrok (yyerrstatus = 0) #define yyclearin (yychar = YYEMPTY) -#define YYEMPTY (-2) -#define YYEOF 0 #define YYACCEPT goto yyacceptlab #define YYABORT goto yyabortlab @@ -981,27 +1127,26 @@ static const yytype_uint8 yyr2[] = #define YYRECOVERING() (!!yyerrstatus) -#define YYBACKUP(Token, Value) \ -do \ - if (yychar == YYEMPTY) \ - { \ - yychar = (Token); \ - yylval = (Value); \ - YYPOPSTACK (yylen); \ - yystate = *yyssp; \ - goto yybackup; \ - } \ - else \ - { \ - yyerror (YY_("syntax error: cannot back up")); \ - YYERROR; \ - } \ -while (0) - -/* Error token number */ -#define YYTERROR 1 -#define YYERRCODE 256 - +#define YYBACKUP(Token, Value) \ + do \ + if (yychar == YYEMPTY) \ + { \ + yychar = (Token); \ + yylval = (Value); \ + YYPOPSTACK (yylen); \ + yystate = *yyssp; \ + goto yybackup; \ + } \ + else \ + { \ + yyerror (YY_("syntax error: cannot back up")); \ + YYERROR; \ + } \ + while (0) + +/* Backward compatibility with an undocumented macro. + Use YYerror or YYUNDEF. */ +#define YYERRCODE YYUNDEF /* Enable debugging if requested. */ @@ -1019,54 +1164,58 @@ do { \ } while (0) /* This macro is provided for backward compatibility. */ -#ifndef YY_LOCATION_PRINT -# define YY_LOCATION_PRINT(File, Loc) ((void) 0) -#endif +# ifndef YY_LOCATION_PRINT +# define YY_LOCATION_PRINT(File, Loc) ((void) 0) +# endif -# define YY_SYMBOL_PRINT(Title, Type, Value, Location) \ +# define YY_SYMBOL_PRINT(Title, Kind, Value, Location) \ do { \ if (yydebug) \ { \ YYFPRINTF (stderr, "%s ", Title); \ yy_symbol_print (stderr, \ - Type, Value); \ + Kind, Value); \ YYFPRINTF (stderr, "\n"); \ } \ } while (0) -/*----------------------------------------. -| Print this symbol's value on YYOUTPUT. | -`----------------------------------------*/ +/*-----------------------------------. +| Print this symbol's value on YYO. | +`-----------------------------------*/ static void -yy_symbol_value_print (FILE *yyoutput, int yytype, YYSTYPE const * const yyvaluep) +yy_symbol_value_print (FILE *yyo, + yysymbol_kind_t yykind, YYSTYPE const * const yyvaluep) { - FILE *yyo = yyoutput; - YYUSE (yyo); + FILE *yyoutput = yyo; + YY_USE (yyoutput); if (!yyvaluep) return; # ifdef YYPRINT - if (yytype < YYNTOKENS) - YYPRINT (yyoutput, yytoknum[yytype], *yyvaluep); + if (yykind < YYNTOKENS) + YYPRINT (yyo, yytoknum[yykind], *yyvaluep); # endif - YYUSE (yytype); + YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN + YY_USE (yykind); + YY_IGNORE_MAYBE_UNINITIALIZED_END } -/*--------------------------------. -| Print this symbol on YYOUTPUT. | -`--------------------------------*/ +/*---------------------------. +| Print this symbol on YYO. | +`---------------------------*/ static void -yy_symbol_print (FILE *yyoutput, int yytype, YYSTYPE const * const yyvaluep) +yy_symbol_print (FILE *yyo, + yysymbol_kind_t yykind, YYSTYPE const * const yyvaluep) { - YYFPRINTF (yyoutput, "%s %s (", - yytype < YYNTOKENS ? "token" : "nterm", yytname[yytype]); + YYFPRINTF (yyo, "%s %s (", + yykind < YYNTOKENS ? "token" : "nterm", yysymbol_name (yykind)); - yy_symbol_value_print (yyoutput, yytype, yyvaluep); - YYFPRINTF (yyoutput, ")"); + yy_symbol_value_print (yyo, yykind, yyvaluep); + YYFPRINTF (yyo, ")"); } /*------------------------------------------------------------------. @@ -1075,7 +1224,7 @@ yy_symbol_print (FILE *yyoutput, int yytype, YYSTYPE const * const yyvaluep) `------------------------------------------------------------------*/ static void -yy_stack_print (yytype_int16 *yybottom, yytype_int16 *yytop) +yy_stack_print (yy_state_t *yybottom, yy_state_t *yytop) { YYFPRINTF (stderr, "Stack now"); for (; yybottom <= yytop; yybottom++) @@ -1098,21 +1247,21 @@ do { \ `------------------------------------------------*/ static void -yy_reduce_print (yytype_int16 *yyssp, YYSTYPE *yyvsp, int yyrule) +yy_reduce_print (yy_state_t *yyssp, YYSTYPE *yyvsp, + int yyrule) { - unsigned long int yylno = yyrline[yyrule]; + int yylno = yyrline[yyrule]; int yynrhs = yyr2[yyrule]; int yyi; - YYFPRINTF (stderr, "Reducing stack by rule %d (line %lu):\n", + YYFPRINTF (stderr, "Reducing stack by rule %d (line %d):\n", yyrule - 1, yylno); /* The symbols being reduced. */ for (yyi = 0; yyi < yynrhs; yyi++) { YYFPRINTF (stderr, " $%d = ", yyi + 1); yy_symbol_print (stderr, - yystos[yyssp[yyi + 1 - yynrhs]], - &(yyvsp[(yyi + 1) - (yynrhs)]) - ); + YY_ACCESSING_SYMBOL (+yyssp[yyi + 1 - yynrhs]), + &yyvsp[(yyi + 1) - (yynrhs)]); YYFPRINTF (stderr, "\n"); } } @@ -1127,8 +1276,8 @@ do { \ multiple parsers can coexist. */ int yydebug; #else /* !YYDEBUG */ -# define YYDPRINTF(Args) -# define YY_SYMBOL_PRINT(Title, Type, Value, Location) +# define YYDPRINTF(Args) ((void) 0) +# define YY_SYMBOL_PRINT(Title, Kind, Value, Location) # define YY_STACK_PRINT(Bottom, Top) # define YY_REDUCE_PRINT(Rule) #endif /* !YYDEBUG */ @@ -1151,28 +1300,76 @@ int yydebug; #endif -#if YYERROR_VERBOSE +/* Context of a parse error. */ +typedef struct +{ + yy_state_t *yyssp; + yysymbol_kind_t yytoken; +} yypcontext_t; + +/* Put in YYARG at most YYARGN of the expected tokens given the + current YYCTX, and return the number of tokens stored in YYARG. If + YYARG is null, return the number of expected tokens (guaranteed to + be less than YYNTOKENS). Return YYENOMEM on memory exhaustion. + Return 0 if there are more than YYARGN expected tokens, yet fill + YYARG up to YYARGN. */ +static int +yypcontext_expected_tokens (const yypcontext_t *yyctx, + yysymbol_kind_t yyarg[], int yyargn) +{ + /* Actual size of YYARG. */ + int yycount = 0; + int yyn = yypact[+*yyctx->yyssp]; + if (!yypact_value_is_default (yyn)) + { + /* Start YYX at -YYN if negative to avoid negative indexes in + YYCHECK. In other words, skip the first -YYN actions for + this state because they are default actions. */ + int yyxbegin = yyn < 0 ? -yyn : 0; + /* Stay within bounds of both yycheck and yytname. */ + int yychecklim = YYLAST - yyn + 1; + int yyxend = yychecklim < YYNTOKENS ? yychecklim : YYNTOKENS; + int yyx; + for (yyx = yyxbegin; yyx < yyxend; ++yyx) + if (yycheck[yyx + yyn] == yyx && yyx != YYSYMBOL_YYerror + && !yytable_value_is_error (yytable[yyx + yyn])) + { + if (!yyarg) + ++yycount; + else if (yycount == yyargn) + return 0; + else + yyarg[yycount++] = YY_CAST (yysymbol_kind_t, yyx); + } + } + if (yyarg && yycount == 0 && 0 < yyargn) + yyarg[0] = YYSYMBOL_YYEMPTY; + return yycount; +} + + -# ifndef yystrlen -# if defined __GLIBC__ && defined _STRING_H -# define yystrlen strlen -# else + +#ifndef yystrlen +# if defined __GLIBC__ && defined _STRING_H +# define yystrlen(S) (YY_CAST (YYPTRDIFF_T, strlen (S))) +# else /* Return the length of YYSTR. */ -static YYSIZE_T +static YYPTRDIFF_T yystrlen (const char *yystr) { - YYSIZE_T yylen; + YYPTRDIFF_T yylen; for (yylen = 0; yystr[yylen]; yylen++) continue; return yylen; } -# endif # endif +#endif -# ifndef yystpcpy -# if defined __GLIBC__ && defined _STRING_H && defined _GNU_SOURCE -# define yystpcpy stpcpy -# else +#ifndef yystpcpy +# if defined __GLIBC__ && defined _STRING_H && defined _GNU_SOURCE +# define yystpcpy stpcpy +# else /* Copy YYSRC to YYDEST, returning the address of the terminating '\0' in YYDEST. */ static char * @@ -1186,10 +1383,10 @@ yystpcpy (char *yydest, const char *yysrc) return yyd - 1; } -# endif # endif +#endif -# ifndef yytnamerr +#ifndef yytnamerr /* Copy to YYRES the contents of YYSTR after stripping away unnecessary quotes and backslashes, so that it's suitable for yyerror. The heuristic is that double-quoting is unnecessary unless the string @@ -1197,14 +1394,13 @@ yystpcpy (char *yydest, const char *yysrc) backslash-backslash). YYSTR is taken from yytname. If YYRES is null, do not copy; instead, return the length of what the result would have been. */ -static YYSIZE_T +static YYPTRDIFF_T yytnamerr (char *yyres, const char *yystr) { if (*yystr == '"') { - YYSIZE_T yyn = 0; + YYPTRDIFF_T yyn = 0; char const *yyp = yystr; - for (;;) switch (*++yyp) { @@ -1215,7 +1411,10 @@ yytnamerr (char *yyres, const char *yystr) case '\\': if (*++yyp != '\\') goto do_not_strip_quotes; - /* Fall through. */ + else + goto append; + + append: default: if (yyres) yyres[yyn] = *yyp; @@ -1230,36 +1429,20 @@ yytnamerr (char *yyres, const char *yystr) do_not_strip_quotes: ; } - if (! yyres) + if (yyres) + return yystpcpy (yyres, yystr) - yyres; + else return yystrlen (yystr); - - return yystpcpy (yyres, yystr) - yyres; } -# endif +#endif -/* Copy into *YYMSG, which is of size *YYMSG_ALLOC, an error message - about the unexpected token YYTOKEN for the state stack whose top is - YYSSP. - Return 0 if *YYMSG was successfully written. Return 1 if *YYMSG is - not large enough to hold the message. In that case, also set - *YYMSG_ALLOC to the required number of bytes. Return 2 if the - required number of bytes is too large to store. */ static int -yysyntax_error (YYSIZE_T *yymsg_alloc, char **yymsg, - yytype_int16 *yyssp, int yytoken) +yy_syntax_error_arguments (const yypcontext_t *yyctx, + yysymbol_kind_t yyarg[], int yyargn) { - YYSIZE_T yysize0 = yytnamerr (YY_NULLPTR, yytname[yytoken]); - YYSIZE_T yysize = yysize0; - enum { YYERROR_VERBOSE_ARGS_MAXIMUM = 5 }; - /* Internationalized format string. */ - const char *yyformat = YY_NULLPTR; - /* Arguments of yyformat. */ - char const *yyarg[YYERROR_VERBOSE_ARGS_MAXIMUM]; - /* Number of reported tokens (one for the "unexpected", one per - "expected"). */ + /* Actual size of YYARG. */ int yycount = 0; - /* There are many possibilities here to consider: - If this state is a consistent state with a default action, then the only way this function was invoked is if the default action @@ -1283,63 +1466,78 @@ yysyntax_error (YYSIZE_T *yymsg_alloc, char **yymsg, one exception: it will still contain any token that will not be accepted due to an error action in a later state. */ - if (yytoken != YYEMPTY) + if (yyctx->yytoken != YYSYMBOL_YYEMPTY) { - int yyn = yypact[*yyssp]; - yyarg[yycount++] = yytname[yytoken]; - if (!yypact_value_is_default (yyn)) - { - /* Start YYX at -YYN if negative to avoid negative indexes in - YYCHECK. In other words, skip the first -YYN actions for - this state because they are default actions. */ - int yyxbegin = yyn < 0 ? -yyn : 0; - /* Stay within bounds of both yycheck and yytname. */ - int yychecklim = YYLAST - yyn + 1; - int yyxend = yychecklim < YYNTOKENS ? yychecklim : YYNTOKENS; - int yyx; - - for (yyx = yyxbegin; yyx < yyxend; ++yyx) - if (yycheck[yyx + yyn] == yyx && yyx != YYTERROR - && !yytable_value_is_error (yytable[yyx + yyn])) - { - if (yycount == YYERROR_VERBOSE_ARGS_MAXIMUM) - { - yycount = 1; - yysize = yysize0; - break; - } - yyarg[yycount++] = yytname[yyx]; - { - YYSIZE_T yysize1 = yysize + yytnamerr (YY_NULLPTR, yytname[yyx]); - if (! (yysize <= yysize1 - && yysize1 <= YYSTACK_ALLOC_MAXIMUM)) - return 2; - yysize = yysize1; - } - } - } + int yyn; + if (yyarg) + yyarg[yycount] = yyctx->yytoken; + ++yycount; + yyn = yypcontext_expected_tokens (yyctx, + yyarg ? yyarg + 1 : yyarg, yyargn - 1); + if (yyn == YYENOMEM) + return YYENOMEM; + else + yycount += yyn; } + return yycount; +} + +/* Copy into *YYMSG, which is of size *YYMSG_ALLOC, an error message + about the unexpected token YYTOKEN for the state stack whose top is + YYSSP. + + Return 0 if *YYMSG was successfully written. Return -1 if *YYMSG is + not large enough to hold the message. In that case, also set + *YYMSG_ALLOC to the required number of bytes. Return YYENOMEM if the + required number of bytes is too large to store. */ +static int +yysyntax_error (YYPTRDIFF_T *yymsg_alloc, char **yymsg, + const yypcontext_t *yyctx) +{ + enum { YYARGS_MAX = 5 }; + /* Internationalized format string. */ + const char *yyformat = YY_NULLPTR; + /* Arguments of yyformat: reported tokens (one for the "unexpected", + one per "expected"). */ + yysymbol_kind_t yyarg[YYARGS_MAX]; + /* Cumulated lengths of YYARG. */ + YYPTRDIFF_T yysize = 0; + + /* Actual size of YYARG. */ + int yycount = yy_syntax_error_arguments (yyctx, yyarg, YYARGS_MAX); + if (yycount == YYENOMEM) + return YYENOMEM; switch (yycount) { -# define YYCASE_(N, S) \ +#define YYCASE_(N, S) \ case N: \ yyformat = S; \ - break + break + default: /* Avoid compiler warnings. */ YYCASE_(0, YY_("syntax error")); YYCASE_(1, YY_("syntax error, unexpected %s")); YYCASE_(2, YY_("syntax error, unexpected %s, expecting %s")); YYCASE_(3, YY_("syntax error, unexpected %s, expecting %s or %s")); YYCASE_(4, YY_("syntax error, unexpected %s, expecting %s or %s or %s")); YYCASE_(5, YY_("syntax error, unexpected %s, expecting %s or %s or %s or %s")); -# undef YYCASE_ +#undef YYCASE_ } + /* Compute error message size. Don't count the "%s"s, but reserve + room for the terminator. */ + yysize = yystrlen (yyformat) - 2 * yycount + 1; { - YYSIZE_T yysize1 = yysize + yystrlen (yyformat); - if (! (yysize <= yysize1 && yysize1 <= YYSTACK_ALLOC_MAXIMUM)) - return 2; - yysize = yysize1; + int yyi; + for (yyi = 0; yyi < yycount; ++yyi) + { + YYPTRDIFF_T yysize1 + = yysize + yytnamerr (YY_NULLPTR, yytname[yyarg[yyi]]); + if (yysize <= yysize1 && yysize1 <= YYSTACK_ALLOC_MAXIMUM) + yysize = yysize1; + else + return YYENOMEM; + } } if (*yymsg_alloc < yysize) @@ -1348,7 +1546,7 @@ yysyntax_error (YYSIZE_T *yymsg_alloc, char **yymsg, if (! (yysize <= *yymsg_alloc && *yymsg_alloc <= YYSTACK_ALLOC_MAXIMUM)) *yymsg_alloc = YYSTACK_ALLOC_MAXIMUM; - return 1; + return -1; } /* Avoid sprintf, as that infringes on the user's name space. @@ -1360,40 +1558,39 @@ yysyntax_error (YYSIZE_T *yymsg_alloc, char **yymsg, while ((*yyp = *yyformat) != '\0') if (*yyp == '%' && yyformat[1] == 's' && yyi < yycount) { - yyp += yytnamerr (yyp, yyarg[yyi++]); + yyp += yytnamerr (yyp, yytname[yyarg[yyi++]]); yyformat += 2; } else { - yyp++; - yyformat++; + ++yyp; + ++yyformat; } } return 0; } -#endif /* YYERROR_VERBOSE */ + /*-----------------------------------------------. | Release the memory associated to this symbol. | `-----------------------------------------------*/ static void -yydestruct (const char *yymsg, int yytype, YYSTYPE *yyvaluep) +yydestruct (const char *yymsg, + yysymbol_kind_t yykind, YYSTYPE *yyvaluep) { - YYUSE (yyvaluep); + YY_USE (yyvaluep); if (!yymsg) yymsg = "Deleting"; - YY_SYMBOL_PRINT (yymsg, yytype, yyvaluep, yylocationp); + YY_SYMBOL_PRINT (yymsg, yykind, yyvaluep, yylocationp); YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN - YYUSE (yytype); + YY_USE (yykind); YY_IGNORE_MAYBE_UNINITIALIZED_END } - - -/* The lookahead symbol. */ +/* Lookahead token kind. */ int yychar; /* The semantic value of the lookahead symbol. */ @@ -1402,6 +1599,8 @@ YYSTYPE yylval; int yynerrs; + + /*----------. | yyparse. | `----------*/ @@ -1409,43 +1608,39 @@ int yynerrs; int yyparse (void) { - int yystate; + yy_state_fast_t yystate = 0; /* Number of tokens to shift before error messages enabled. */ - int yyerrstatus; + int yyerrstatus = 0; - /* The stacks and their tools: - 'yyss': related to states. - 'yyvs': related to semantic values. - - Refer to the stacks through separate pointers, to allow yyoverflow + /* Refer to the stacks through separate pointers, to allow yyoverflow to reallocate them elsewhere. */ - /* The state stack. */ - yytype_int16 yyssa[YYINITDEPTH]; - yytype_int16 *yyss; - yytype_int16 *yyssp; + /* Their size. */ + YYPTRDIFF_T yystacksize = YYINITDEPTH; - /* The semantic value stack. */ - YYSTYPE yyvsa[YYINITDEPTH]; - YYSTYPE *yyvs; - YYSTYPE *yyvsp; + /* The state stack: array, bottom, top. */ + yy_state_t yyssa[YYINITDEPTH]; + yy_state_t *yyss = yyssa; + yy_state_t *yyssp = yyss; - YYSIZE_T yystacksize; + /* The semantic value stack: array, bottom, top. */ + YYSTYPE yyvsa[YYINITDEPTH]; + YYSTYPE *yyvs = yyvsa; + YYSTYPE *yyvsp = yyvs; int yyn; + /* The return value of yyparse. */ int yyresult; - /* Lookahead token as an internal (translated) token number. */ - int yytoken = 0; + /* Lookahead symbol kind. */ + yysymbol_kind_t yytoken = YYSYMBOL_YYEMPTY; /* The variables used to return semantic value and location from the action routines. */ YYSTYPE yyval; -#if YYERROR_VERBOSE /* Buffer for error messages, and its allocated size. */ char yymsgbuf[128]; char *yymsg = yymsgbuf; - YYSIZE_T yymsg_alloc = sizeof yymsgbuf; -#endif + YYPTRDIFF_T yymsg_alloc = sizeof yymsgbuf; #define YYPOPSTACK(N) (yyvsp -= (N), yyssp -= (N)) @@ -1453,58 +1648,60 @@ yyparse (void) Keep to zero when no symbol should be popped. */ int yylen = 0; - yyssp = yyss = yyssa; - yyvsp = yyvs = yyvsa; - yystacksize = YYINITDEPTH; - YYDPRINTF ((stderr, "Starting parse\n")); - yystate = 0; - yyerrstatus = 0; - yynerrs = 0; yychar = YYEMPTY; /* Cause a token to be read. */ goto yysetstate; + /*------------------------------------------------------------. -| yynewstate -- Push a new state, which is found in yystate. | +| yynewstate -- push a new state, which is found in yystate. | `------------------------------------------------------------*/ - yynewstate: +yynewstate: /* In all cases, when you get here, the value and location stacks have just been pushed. So pushing a state here evens the stacks. */ yyssp++; - yysetstate: - *yyssp = yystate; + +/*--------------------------------------------------------------------. +| yysetstate -- set current state (the top of the stack) to yystate. | +`--------------------------------------------------------------------*/ +yysetstate: + YYDPRINTF ((stderr, "Entering state %d\n", yystate)); + YY_ASSERT (0 <= yystate && yystate < YYNSTATES); + YY_IGNORE_USELESS_CAST_BEGIN + *yyssp = YY_CAST (yy_state_t, yystate); + YY_IGNORE_USELESS_CAST_END + YY_STACK_PRINT (yyss, yyssp); if (yyss + yystacksize - 1 <= yyssp) +#if !defined yyoverflow && !defined YYSTACK_RELOCATE + goto yyexhaustedlab; +#else { /* Get the current used size of the three stacks, in elements. */ - YYSIZE_T yysize = yyssp - yyss + 1; + YYPTRDIFF_T yysize = yyssp - yyss + 1; -#ifdef yyoverflow +# if defined yyoverflow { /* Give user a chance to reallocate the stack. Use copies of these so that the &'s don't force the real ones into memory. */ + yy_state_t *yyss1 = yyss; YYSTYPE *yyvs1 = yyvs; - yytype_int16 *yyss1 = yyss; /* Each stack pointer address is followed by the size of the data in use in that stack, in bytes. This used to be a conditional around just the two extra args, but that might be undefined if yyoverflow is a macro. */ yyoverflow (YY_("memory exhausted"), - &yyss1, yysize * sizeof (*yyssp), - &yyvs1, yysize * sizeof (*yyvsp), + &yyss1, yysize * YYSIZEOF (*yyssp), + &yyvs1, yysize * YYSIZEOF (*yyvsp), &yystacksize); - yyss = yyss1; yyvs = yyvs1; } -#else /* no yyoverflow */ -# ifndef YYSTACK_RELOCATE - goto yyexhaustedlab; -# else +# else /* defined YYSTACK_RELOCATE */ /* Extend the stack our own way. */ if (YYMAXDEPTH <= yystacksize) goto yyexhaustedlab; @@ -1513,9 +1710,10 @@ yyparse (void) yystacksize = YYMAXDEPTH; { - yytype_int16 *yyss1 = yyss; + yy_state_t *yyss1 = yyss; union yyalloc *yyptr = - (union yyalloc *) YYSTACK_ALLOC (YYSTACK_BYTES (yystacksize)); + YY_CAST (union yyalloc *, + YYSTACK_ALLOC (YY_CAST (YYSIZE_T, YYSTACK_BYTES (yystacksize)))); if (! yyptr) goto yyexhaustedlab; YYSTACK_RELOCATE (yyss_alloc, yyss); @@ -1525,30 +1723,30 @@ yyparse (void) YYSTACK_FREE (yyss1); } # endif -#endif /* no yyoverflow */ yyssp = yyss + yysize - 1; yyvsp = yyvs + yysize - 1; - YYDPRINTF ((stderr, "Stack size increased to %lu\n", - (unsigned long int) yystacksize)); + YY_IGNORE_USELESS_CAST_BEGIN + YYDPRINTF ((stderr, "Stack size increased to %ld\n", + YY_CAST (long, yystacksize))); + YY_IGNORE_USELESS_CAST_END if (yyss + yystacksize - 1 <= yyssp) YYABORT; } - - YYDPRINTF ((stderr, "Entering state %d\n", yystate)); +#endif /* !defined yyoverflow && !defined YYSTACK_RELOCATE */ if (yystate == YYFINAL) YYACCEPT; goto yybackup; + /*-----------. | yybackup. | `-----------*/ yybackup: - /* Do appropriate processing given the current state. Read a lookahead token if we need one and don't already have one. */ @@ -1559,18 +1757,29 @@ yyparse (void) /* Not known => get a lookahead token if don't already have one. */ - /* YYCHAR is either YYEMPTY or YYEOF or a valid lookahead symbol. */ + /* YYCHAR is either empty, or end-of-input, or a valid lookahead. */ if (yychar == YYEMPTY) { - YYDPRINTF ((stderr, "Reading a token: ")); + YYDPRINTF ((stderr, "Reading a token\n")); yychar = yylex (); } if (yychar <= YYEOF) { - yychar = yytoken = YYEOF; + yychar = YYEOF; + yytoken = YYSYMBOL_YYEOF; YYDPRINTF ((stderr, "Now at end of input.\n")); } + else if (yychar == YYerror) + { + /* The scanner already issued an error message, process directly + to error recovery. But do not keep the error token as + lookahead, it is too special and may lead us to an endless + loop in error recovery. */ + yychar = YYUNDEF; + yytoken = YYSYMBOL_YYerror; + goto yyerrlab1; + } else { yytoken = YYTRANSLATE (yychar); @@ -1598,15 +1807,13 @@ yyparse (void) /* Shift the lookahead token. */ YY_SYMBOL_PRINT ("Shifting", yytoken, &yylval, &yylloc); - - /* Discard the shifted token. */ - yychar = YYEMPTY; - yystate = yyn; YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN *++yyvsp = yylval; YY_IGNORE_MAYBE_UNINITIALIZED_END + /* Discard the shifted token. */ + yychar = YYEMPTY; goto yynewstate; @@ -1621,7 +1828,7 @@ yyparse (void) /*-----------------------------. -| yyreduce -- Do a reduction. | +| yyreduce -- do a reduction. | `-----------------------------*/ yyreduce: /* yyn is the number of a rule to reduce with. */ @@ -1641,75 +1848,75 @@ yyparse (void) YY_REDUCE_PRINT (yyn); switch (yyn) { - case 2: -#line 238 "ncgen.y" /* yacc.c:1646 */ - {if (error_count > 0) YYABORT;} -#line 1648 "ncgeny.c" /* yacc.c:1646 */ + case 2: /* ncdesc: NETCDF datasetid rootgroup */ +#line 240 "ncgen.y" + {if (error_count > 0) YYABORT;} +#line 1855 "ncgeny.c" break; - case 3: -#line 241 "ncgen.y" /* yacc.c:1646 */ - {createrootgroup(datasetname);} -#line 1654 "ncgeny.c" /* yacc.c:1646 */ + case 3: /* datasetid: DATASETID */ +#line 243 "ncgen.y" + {createrootgroup(datasetname);} +#line 1861 "ncgeny.c" break; - case 8: -#line 260 "ncgen.y" /* yacc.c:1646 */ - { + case 8: /* $@1: %empty */ +#line 262 "ncgen.y" + { Symbol* id = (yyvsp[-1].sym); markcdf4("Group specification"); if(creategroup(id) == NULL) yyerror("duplicate group declaration within parent group for %s", id->name); } -#line 1666 "ncgeny.c" /* yacc.c:1646 */ +#line 1873 "ncgeny.c" break; - case 9: -#line 269 "ncgen.y" /* yacc.c:1646 */ - {listpop(groupstack);} -#line 1672 "ncgeny.c" /* yacc.c:1646 */ + case 9: /* $@2: %empty */ +#line 271 "ncgen.y" + {listpop(groupstack);} +#line 1879 "ncgeny.c" break; - case 12: -#line 275 "ncgen.y" /* yacc.c:1646 */ - {} -#line 1678 "ncgeny.c" /* yacc.c:1646 */ + case 12: /* typesection: TYPES */ +#line 277 "ncgen.y" + {} +#line 1885 "ncgeny.c" break; - case 13: -#line 277 "ncgen.y" /* yacc.c:1646 */ - {markcdf4("Type specification");} -#line 1684 "ncgeny.c" /* yacc.c:1646 */ + case 13: /* typesection: TYPES typedecls */ +#line 279 "ncgen.y" + {markcdf4("Type specification");} +#line 1891 "ncgeny.c" break; - case 16: -#line 283 "ncgen.y" /* yacc.c:1646 */ - { /* Use when defining a type */ + case 16: /* typename: ident */ +#line 285 "ncgen.y" + { /* Use when defining a type */ (yyvsp[0].sym)->objectclass = NC_TYPE; if(dupobjectcheck(NC_TYPE,(yyvsp[0].sym))) yyerror("duplicate type declaration for %s", (yyvsp[0].sym)->name); listpush(typdefs,(void*)(yyvsp[0].sym)); } -#line 1696 "ncgeny.c" /* yacc.c:1646 */ +#line 1903 "ncgeny.c" break; - case 17: -#line 292 "ncgen.y" /* yacc.c:1646 */ - {} -#line 1702 "ncgeny.c" /* yacc.c:1646 */ + case 17: /* type_or_attr_decl: typedecl */ +#line 294 "ncgen.y" + {} +#line 1909 "ncgeny.c" break; - case 18: -#line 292 "ncgen.y" /* yacc.c:1646 */ - {} -#line 1708 "ncgeny.c" /* yacc.c:1646 */ + case 18: /* type_or_attr_decl: attrdecl ';' */ +#line 294 "ncgen.y" + {} +#line 1915 "ncgeny.c" break; - case 25: -#line 306 "ncgen.y" /* yacc.c:1646 */ - { + case 25: /* enumdecl: primtype ENUM typename '{' enumidlist '}' */ +#line 308 "ncgen.y" + { int i; addtogroup((yyvsp[-3].sym)); /* sets prefix*/ (yyvsp[-3].sym)->objectclass=NC_TYPE; @@ -1735,18 +1942,18 @@ yyparse (void) } listsetlength(stack,stackbase);/* remove stack nodes*/ } -#line 1739 "ncgeny.c" /* yacc.c:1646 */ +#line 1946 "ncgeny.c" break; - case 26: -#line 335 "ncgen.y" /* yacc.c:1646 */ - {(yyval.mark)=listlength(stack); listpush(stack,(void*)(yyvsp[0].sym));} -#line 1745 "ncgeny.c" /* yacc.c:1646 */ + case 26: /* enumidlist: enumid */ +#line 337 "ncgen.y" + {(yyval.mark)=listlength(stack); listpush(stack,(void*)(yyvsp[0].sym));} +#line 1952 "ncgeny.c" break; - case 27: -#line 337 "ncgen.y" /* yacc.c:1646 */ - { + case 27: /* enumidlist: enumidlist ',' enumid */ +#line 339 "ncgen.y" + { int i; (yyval.mark)=(yyvsp[-2].mark); /* check for duplicates*/ @@ -1760,23 +1967,23 @@ yyparse (void) } listpush(stack,(void*)(yyvsp[0].sym)); } -#line 1764 "ncgeny.c" /* yacc.c:1646 */ +#line 1971 "ncgeny.c" break; - case 28: -#line 354 "ncgen.y" /* yacc.c:1646 */ - { + case 28: /* enumid: ident '=' constint */ +#line 356 "ncgen.y" + { (yyvsp[-2].sym)->objectclass=NC_TYPE; (yyvsp[-2].sym)->subclass=NC_ECONST; (yyvsp[-2].sym)->typ.econst=(yyvsp[0].constant); (yyval.sym)=(yyvsp[-2].sym); } -#line 1775 "ncgeny.c" /* yacc.c:1646 */ +#line 1982 "ncgeny.c" break; - case 29: -#line 363 "ncgen.y" /* yacc.c:1646 */ - { + case 29: /* opaquedecl: OPAQUE_ '(' INT_CONST ')' typename */ +#line 365 "ncgen.y" + { vercheck(NC_OPAQUE); addtogroup((yyvsp[0].sym)); /*sets prefix*/ (yyvsp[0].sym)->objectclass=NC_TYPE; @@ -1785,12 +1992,12 @@ yyparse (void) (yyvsp[0].sym)->typ.size=int32_val; (yyvsp[0].sym)->typ.alignment=ncaux_class_alignment(NC_OPAQUE); } -#line 1789 "ncgeny.c" /* yacc.c:1646 */ +#line 1996 "ncgeny.c" break; - case 30: -#line 375 "ncgen.y" /* yacc.c:1646 */ - { + case 30: /* vlendecl: typeref '(' '*' ')' typename */ +#line 377 "ncgen.y" + { Symbol* basetype = (yyvsp[-4].sym); vercheck(NC_VLEN); addtogroup((yyvsp[0].sym)); /*sets prefix*/ @@ -1801,12 +2008,12 @@ yyparse (void) (yyvsp[0].sym)->typ.size=VLENSIZE; (yyvsp[0].sym)->typ.alignment=ncaux_class_alignment(NC_VLEN); } -#line 1805 "ncgeny.c" /* yacc.c:1646 */ +#line 2012 "ncgeny.c" break; - case 31: -#line 389 "ncgen.y" /* yacc.c:1646 */ - { + case 31: /* compounddecl: COMPOUND typename '{' fields '}' */ +#line 391 "ncgen.y" + { int i,j; vercheck(NC_COMPOUND); addtogroup((yyvsp[-3].sym)); @@ -1835,24 +2042,24 @@ yyparse (void) } listsetlength(stack,stackbase);/* remove stack nodes*/ } -#line 1839 "ncgeny.c" /* yacc.c:1646 */ +#line 2046 "ncgeny.c" break; - case 32: -#line 421 "ncgen.y" /* yacc.c:1646 */ - {(yyval.mark)=(yyvsp[-1].mark);} -#line 1845 "ncgeny.c" /* yacc.c:1646 */ + case 32: /* fields: field ';' */ +#line 423 "ncgen.y" + {(yyval.mark)=(yyvsp[-1].mark);} +#line 2052 "ncgeny.c" break; - case 33: -#line 422 "ncgen.y" /* yacc.c:1646 */ - {(yyval.mark)=(yyvsp[-2].mark);} -#line 1851 "ncgeny.c" /* yacc.c:1646 */ + case 33: /* fields: fields field ';' */ +#line 424 "ncgen.y" + {(yyval.mark)=(yyvsp[-2].mark);} +#line 2058 "ncgeny.c" break; - case 34: -#line 426 "ncgen.y" /* yacc.c:1646 */ - { + case 34: /* field: typeref fieldlist */ +#line 428 "ncgen.y" + { int i; (yyval.mark)=(yyvsp[0].mark); stackbase=(yyvsp[0].mark); @@ -1863,132 +2070,132 @@ yyparse (void) f->typ.basetype = (yyvsp[-1].sym); } } -#line 1867 "ncgeny.c" /* yacc.c:1646 */ +#line 2074 "ncgeny.c" break; - case 35: -#line 439 "ncgen.y" /* yacc.c:1646 */ - { (yyval.sym) = primsymbols[NC_CHAR]; } -#line 1873 "ncgeny.c" /* yacc.c:1646 */ + case 35: /* primtype: CHAR_K */ +#line 441 "ncgen.y" + { (yyval.sym) = primsymbols[NC_CHAR]; } +#line 2080 "ncgeny.c" break; - case 36: -#line 440 "ncgen.y" /* yacc.c:1646 */ - { (yyval.sym) = primsymbols[NC_BYTE]; } -#line 1879 "ncgeny.c" /* yacc.c:1646 */ + case 36: /* primtype: BYTE_K */ +#line 442 "ncgen.y" + { (yyval.sym) = primsymbols[NC_BYTE]; } +#line 2086 "ncgeny.c" break; - case 37: -#line 441 "ncgen.y" /* yacc.c:1646 */ - { (yyval.sym) = primsymbols[NC_SHORT]; } -#line 1885 "ncgeny.c" /* yacc.c:1646 */ + case 37: /* primtype: SHORT_K */ +#line 443 "ncgen.y" + { (yyval.sym) = primsymbols[NC_SHORT]; } +#line 2092 "ncgeny.c" break; - case 38: -#line 442 "ncgen.y" /* yacc.c:1646 */ - { (yyval.sym) = primsymbols[NC_INT]; } -#line 1891 "ncgeny.c" /* yacc.c:1646 */ + case 38: /* primtype: INT_K */ +#line 444 "ncgen.y" + { (yyval.sym) = primsymbols[NC_INT]; } +#line 2098 "ncgeny.c" break; - case 39: -#line 443 "ncgen.y" /* yacc.c:1646 */ - { (yyval.sym) = primsymbols[NC_FLOAT]; } -#line 1897 "ncgeny.c" /* yacc.c:1646 */ + case 39: /* primtype: FLOAT_K */ +#line 445 "ncgen.y" + { (yyval.sym) = primsymbols[NC_FLOAT]; } +#line 2104 "ncgeny.c" break; - case 40: -#line 444 "ncgen.y" /* yacc.c:1646 */ - { (yyval.sym) = primsymbols[NC_DOUBLE]; } -#line 1903 "ncgeny.c" /* yacc.c:1646 */ + case 40: /* primtype: DOUBLE_K */ +#line 446 "ncgen.y" + { (yyval.sym) = primsymbols[NC_DOUBLE]; } +#line 2110 "ncgeny.c" break; - case 41: -#line 445 "ncgen.y" /* yacc.c:1646 */ - { vercheck(NC_UBYTE); (yyval.sym) = primsymbols[NC_UBYTE]; } -#line 1909 "ncgeny.c" /* yacc.c:1646 */ + case 41: /* primtype: UBYTE_K */ +#line 447 "ncgen.y" + { vercheck(NC_UBYTE); (yyval.sym) = primsymbols[NC_UBYTE]; } +#line 2116 "ncgeny.c" break; - case 42: -#line 446 "ncgen.y" /* yacc.c:1646 */ - { vercheck(NC_USHORT); (yyval.sym) = primsymbols[NC_USHORT]; } -#line 1915 "ncgeny.c" /* yacc.c:1646 */ + case 42: /* primtype: USHORT_K */ +#line 448 "ncgen.y" + { vercheck(NC_USHORT); (yyval.sym) = primsymbols[NC_USHORT]; } +#line 2122 "ncgeny.c" break; - case 43: -#line 447 "ncgen.y" /* yacc.c:1646 */ - { vercheck(NC_UINT); (yyval.sym) = primsymbols[NC_UINT]; } -#line 1921 "ncgeny.c" /* yacc.c:1646 */ + case 43: /* primtype: UINT_K */ +#line 449 "ncgen.y" + { vercheck(NC_UINT); (yyval.sym) = primsymbols[NC_UINT]; } +#line 2128 "ncgeny.c" break; - case 44: -#line 448 "ncgen.y" /* yacc.c:1646 */ - { vercheck(NC_INT64); (yyval.sym) = primsymbols[NC_INT64]; } -#line 1927 "ncgeny.c" /* yacc.c:1646 */ + case 44: /* primtype: INT64_K */ +#line 450 "ncgen.y" + { vercheck(NC_INT64); (yyval.sym) = primsymbols[NC_INT64]; } +#line 2134 "ncgeny.c" break; - case 45: -#line 449 "ncgen.y" /* yacc.c:1646 */ - { vercheck(NC_UINT64); (yyval.sym) = primsymbols[NC_UINT64]; } -#line 1933 "ncgeny.c" /* yacc.c:1646 */ + case 45: /* primtype: UINT64_K */ +#line 451 "ncgen.y" + { vercheck(NC_UINT64); (yyval.sym) = primsymbols[NC_UINT64]; } +#line 2140 "ncgeny.c" break; - case 46: -#line 450 "ncgen.y" /* yacc.c:1646 */ - { vercheck(NC_STRING); (yyval.sym) = primsymbols[NC_STRING]; } -#line 1939 "ncgeny.c" /* yacc.c:1646 */ + case 46: /* primtype: STRING_K */ +#line 452 "ncgen.y" + { vercheck(NC_STRING); (yyval.sym) = primsymbols[NC_STRING]; } +#line 2146 "ncgeny.c" break; - case 48: -#line 454 "ncgen.y" /* yacc.c:1646 */ - {} -#line 1945 "ncgeny.c" /* yacc.c:1646 */ + case 48: /* dimsection: DIMENSIONS */ +#line 456 "ncgen.y" + {} +#line 2152 "ncgeny.c" break; - case 49: -#line 455 "ncgen.y" /* yacc.c:1646 */ - {} -#line 1951 "ncgeny.c" /* yacc.c:1646 */ + case 49: /* dimsection: DIMENSIONS dimdecls */ +#line 457 "ncgen.y" + {} +#line 2158 "ncgeny.c" break; - case 52: -#line 462 "ncgen.y" /* yacc.c:1646 */ - {} -#line 1957 "ncgeny.c" /* yacc.c:1646 */ + case 52: /* dim_or_attr_decl: dimdeclist */ +#line 464 "ncgen.y" + {} +#line 2164 "ncgeny.c" break; - case 53: -#line 462 "ncgen.y" /* yacc.c:1646 */ - {} -#line 1963 "ncgeny.c" /* yacc.c:1646 */ + case 53: /* dim_or_attr_decl: attrdecl */ +#line 464 "ncgen.y" + {} +#line 2170 "ncgeny.c" break; - case 56: -#line 470 "ncgen.y" /* yacc.c:1646 */ - { + case 56: /* dimdecl: dimd '=' constint */ +#line 472 "ncgen.y" + { (yyvsp[-2].sym)->dim.declsize = (size_t)extractint((yyvsp[0].constant)); #ifdef GENDEBUG1 fprintf(stderr,"dimension: %s = %llu\n",(yyvsp[-2].sym)->name,(unsigned long long)(yyvsp[-2].sym)->dim.declsize); #endif reclaimconstant((yyvsp[0].constant)); } -#line 1975 "ncgeny.c" /* yacc.c:1646 */ +#line 2182 "ncgeny.c" break; - case 57: -#line 478 "ncgen.y" /* yacc.c:1646 */ - { + case 57: /* dimdecl: dimd '=' NC_UNLIMITED_K */ +#line 480 "ncgen.y" + { (yyvsp[-2].sym)->dim.declsize = NC_UNLIMITED; (yyvsp[-2].sym)->dim.isunlimited = 1; #ifdef GENDEBUG1 fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); #endif } -#line 1987 "ncgeny.c" /* yacc.c:1646 */ +#line 2194 "ncgeny.c" break; - case 58: -#line 488 "ncgen.y" /* yacc.c:1646 */ - { + case 58: /* dimd: ident */ +#line 490 "ncgen.y" + { (yyvsp[0].sym)->objectclass=NC_DIM; if(dupobjectcheck(NC_DIM,(yyvsp[0].sym))) yyerror( "Duplicate dimension declaration for %s", @@ -1997,36 +2204,36 @@ fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); (yyval.sym)=(yyvsp[0].sym); listpush(dimdefs,(void*)(yyvsp[0].sym)); } -#line 2001 "ncgeny.c" /* yacc.c:1646 */ +#line 2208 "ncgeny.c" break; - case 60: -#line 500 "ncgen.y" /* yacc.c:1646 */ - {} -#line 2007 "ncgeny.c" /* yacc.c:1646 */ + case 60: /* vasection: VARIABLES */ +#line 502 "ncgen.y" + {} +#line 2214 "ncgeny.c" break; - case 61: -#line 501 "ncgen.y" /* yacc.c:1646 */ - {} -#line 2013 "ncgeny.c" /* yacc.c:1646 */ + case 61: /* vasection: VARIABLES vadecls */ +#line 503 "ncgen.y" + {} +#line 2220 "ncgeny.c" break; - case 64: -#line 508 "ncgen.y" /* yacc.c:1646 */ - {} -#line 2019 "ncgeny.c" /* yacc.c:1646 */ + case 64: /* vadecl_or_attr: vardecl */ +#line 510 "ncgen.y" + {} +#line 2226 "ncgeny.c" break; - case 65: -#line 508 "ncgen.y" /* yacc.c:1646 */ - {} -#line 2025 "ncgeny.c" /* yacc.c:1646 */ + case 65: /* vadecl_or_attr: attrdecl */ +#line 510 "ncgen.y" + {} +#line 2232 "ncgeny.c" break; - case 66: -#line 511 "ncgen.y" /* yacc.c:1646 */ - { + case 66: /* vardecl: typeref varlist */ +#line 513 "ncgen.y" + { int i; stackbase=(yyvsp[0].mark); stacklen=listlength(stack); @@ -2045,26 +2252,26 @@ fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); } listsetlength(stack,stackbase);/* remove stack nodes*/ } -#line 2049 "ncgeny.c" /* yacc.c:1646 */ +#line 2256 "ncgeny.c" break; - case 67: -#line 533 "ncgen.y" /* yacc.c:1646 */ - {(yyval.mark)=listlength(stack); + case 67: /* varlist: varspec */ +#line 535 "ncgen.y" + {(yyval.mark)=listlength(stack); listpush(stack,(void*)(yyvsp[0].sym)); } -#line 2057 "ncgeny.c" /* yacc.c:1646 */ +#line 2264 "ncgeny.c" break; - case 68: -#line 537 "ncgen.y" /* yacc.c:1646 */ - {(yyval.mark)=(yyvsp[-2].mark); listpush(stack,(void*)(yyvsp[0].sym));} -#line 2063 "ncgeny.c" /* yacc.c:1646 */ + case 68: /* varlist: varlist ',' varspec */ +#line 539 "ncgen.y" + {(yyval.mark)=(yyvsp[-2].mark); listpush(stack,(void*)(yyvsp[0].sym));} +#line 2270 "ncgeny.c" break; - case 69: -#line 541 "ncgen.y" /* yacc.c:1646 */ - { + case 69: /* varspec: ident dimspec */ +#line 543 "ncgen.y" + { int i; Dimset dimset; Symbol* var = (yyvsp[-1].sym); /* for debugging */ @@ -2090,36 +2297,36 @@ fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); listsetlength(stack,stackbase);/* remove stack nodes*/ (yyval.sym) = var; } -#line 2094 "ncgeny.c" /* yacc.c:1646 */ +#line 2301 "ncgeny.c" break; - case 70: -#line 569 "ncgen.y" /* yacc.c:1646 */ - {(yyval.mark)=listlength(stack);} -#line 2100 "ncgeny.c" /* yacc.c:1646 */ + case 70: /* dimspec: %empty */ +#line 571 "ncgen.y" + {(yyval.mark)=listlength(stack);} +#line 2307 "ncgeny.c" break; - case 71: -#line 570 "ncgen.y" /* yacc.c:1646 */ - {(yyval.mark)=(yyvsp[-1].mark);} -#line 2106 "ncgeny.c" /* yacc.c:1646 */ + case 71: /* dimspec: '(' dimlist ')' */ +#line 572 "ncgen.y" + {(yyval.mark)=(yyvsp[-1].mark);} +#line 2313 "ncgeny.c" break; - case 72: -#line 573 "ncgen.y" /* yacc.c:1646 */ - {(yyval.mark)=listlength(stack); listpush(stack,(void*)(yyvsp[0].sym));} -#line 2112 "ncgeny.c" /* yacc.c:1646 */ + case 72: /* dimlist: dimref */ +#line 575 "ncgen.y" + {(yyval.mark)=listlength(stack); listpush(stack,(void*)(yyvsp[0].sym));} +#line 2319 "ncgeny.c" break; - case 73: -#line 575 "ncgen.y" /* yacc.c:1646 */ - {(yyval.mark)=(yyvsp[-2].mark); listpush(stack,(void*)(yyvsp[0].sym));} -#line 2118 "ncgeny.c" /* yacc.c:1646 */ + case 73: /* dimlist: dimlist ',' dimref */ +#line 577 "ncgen.y" + {(yyval.mark)=(yyvsp[-2].mark); listpush(stack,(void*)(yyvsp[0].sym));} +#line 2325 "ncgeny.c" break; - case 74: -#line 579 "ncgen.y" /* yacc.c:1646 */ - {Symbol* dimsym = (yyvsp[0].sym); + case 74: /* dimref: path */ +#line 581 "ncgen.y" + {Symbol* dimsym = (yyvsp[0].sym); dimsym->objectclass = NC_DIM; /* Find the actual dimension*/ dimsym = locate(dimsym); @@ -2129,26 +2336,26 @@ fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); } (yyval.sym)=dimsym; } -#line 2133 "ncgeny.c" /* yacc.c:1646 */ +#line 2340 "ncgeny.c" break; - case 75: -#line 593 "ncgen.y" /* yacc.c:1646 */ - {(yyval.mark)=listlength(stack); + case 75: /* fieldlist: fieldspec */ +#line 595 "ncgen.y" + {(yyval.mark)=listlength(stack); listpush(stack,(void*)(yyvsp[0].sym)); } -#line 2141 "ncgeny.c" /* yacc.c:1646 */ +#line 2348 "ncgeny.c" break; - case 76: -#line 597 "ncgen.y" /* yacc.c:1646 */ - {(yyval.mark)=(yyvsp[-2].mark); listpush(stack,(void*)(yyvsp[0].sym));} -#line 2147 "ncgeny.c" /* yacc.c:1646 */ + case 76: /* fieldlist: fieldlist ',' fieldspec */ +#line 599 "ncgen.y" + {(yyval.mark)=(yyvsp[-2].mark); listpush(stack,(void*)(yyvsp[0].sym));} +#line 2354 "ncgeny.c" break; - case 77: -#line 602 "ncgen.y" /* yacc.c:1646 */ - { + case 77: /* fieldspec: ident fielddimspec */ +#line 604 "ncgen.y" + { int i; Dimset dimset; stackbase=(yyvsp[0].mark); @@ -2174,36 +2381,36 @@ fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); listsetlength(stack,stackbase);/* remove stack nodes*/ (yyval.sym) = (yyvsp[-1].sym); } -#line 2178 "ncgeny.c" /* yacc.c:1646 */ +#line 2385 "ncgeny.c" break; - case 78: -#line 630 "ncgen.y" /* yacc.c:1646 */ - {(yyval.mark)=listlength(stack);} -#line 2184 "ncgeny.c" /* yacc.c:1646 */ + case 78: /* fielddimspec: %empty */ +#line 632 "ncgen.y" + {(yyval.mark)=listlength(stack);} +#line 2391 "ncgeny.c" break; - case 79: -#line 631 "ncgen.y" /* yacc.c:1646 */ - {(yyval.mark)=(yyvsp[-1].mark);} -#line 2190 "ncgeny.c" /* yacc.c:1646 */ + case 79: /* fielddimspec: '(' fielddimlist ')' */ +#line 633 "ncgen.y" + {(yyval.mark)=(yyvsp[-1].mark);} +#line 2397 "ncgeny.c" break; - case 80: -#line 635 "ncgen.y" /* yacc.c:1646 */ - {(yyval.mark)=listlength(stack); listpush(stack,(void*)(yyvsp[0].sym));} -#line 2196 "ncgeny.c" /* yacc.c:1646 */ + case 80: /* fielddimlist: fielddim */ +#line 637 "ncgen.y" + {(yyval.mark)=listlength(stack); listpush(stack,(void*)(yyvsp[0].sym));} +#line 2403 "ncgeny.c" break; - case 81: -#line 637 "ncgen.y" /* yacc.c:1646 */ - {(yyval.mark)=(yyvsp[-2].mark); listpush(stack,(void*)(yyvsp[0].sym));} -#line 2202 "ncgeny.c" /* yacc.c:1646 */ + case 81: /* fielddimlist: fielddimlist ',' fielddim */ +#line 639 "ncgen.y" + {(yyval.mark)=(yyvsp[-2].mark); listpush(stack,(void*)(yyvsp[0].sym));} +#line 2409 "ncgeny.c" break; - case 82: -#line 642 "ncgen.y" /* yacc.c:1646 */ - { /* Anonymous integer dimension. + case 82: /* fielddim: UINT_CONST */ +#line 644 "ncgen.y" + { /* Anonymous integer dimension. Can only occur in type definitions*/ char anon[32]; sprintf(anon,"const%u",uint32_val); @@ -2212,12 +2419,12 @@ fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); (yyval.sym)->dim.isconstant = 1; (yyval.sym)->dim.declsize = uint32_val; } -#line 2216 "ncgeny.c" /* yacc.c:1646 */ +#line 2423 "ncgeny.c" break; - case 83: -#line 652 "ncgen.y" /* yacc.c:1646 */ - { /* Anonymous integer dimension. + case 83: /* fielddim: INT_CONST */ +#line 654 "ncgen.y" + { /* Anonymous integer dimension. Can only occur in type definitions*/ char anon[32]; if(int32_val <= 0) { @@ -2230,36 +2437,36 @@ fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); (yyval.sym)->dim.isconstant = 1; (yyval.sym)->dim.declsize = int32_val; } -#line 2234 "ncgeny.c" /* yacc.c:1646 */ +#line 2441 "ncgeny.c" break; - case 84: -#line 672 "ncgen.y" /* yacc.c:1646 */ - {Symbol* vsym = (yyvsp[0].sym); + case 84: /* varref: ambiguous_ref */ +#line 674 "ncgen.y" + {Symbol* vsym = (yyvsp[0].sym); if(vsym->objectclass != NC_VAR) { derror("Undefined or forward referenced variable: %s",vsym->name); YYABORT; } (yyval.sym)=vsym; } -#line 2246 "ncgeny.c" /* yacc.c:1646 */ +#line 2453 "ncgeny.c" break; - case 85: -#line 683 "ncgen.y" /* yacc.c:1646 */ - {Symbol* tsym = (yyvsp[0].sym); + case 85: /* typeref: ambiguous_ref */ +#line 685 "ncgen.y" + {Symbol* tsym = (yyvsp[0].sym); if(tsym->objectclass != NC_TYPE) { derror("Undefined or forward referenced type: %s",tsym->name); YYABORT; } (yyval.sym)=tsym; } -#line 2258 "ncgeny.c" /* yacc.c:1646 */ +#line 2465 "ncgeny.c" break; - case 86: -#line 694 "ncgen.y" /* yacc.c:1646 */ - {Symbol* tvsym = (yyvsp[0].sym); Symbol* sym; + case 86: /* ambiguous_ref: path */ +#line 696 "ncgen.y" + {Symbol* tvsym = (yyvsp[0].sym); Symbol* sym; /* disambiguate*/ tvsym->objectclass = NC_VAR; sym = locate(tvsym); @@ -2277,54 +2484,54 @@ fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); } (yyval.sym)=tvsym; } -#line 2281 "ncgeny.c" /* yacc.c:1646 */ +#line 2488 "ncgeny.c" break; - case 87: -#line 712 "ncgen.y" /* yacc.c:1646 */ - {(yyval.sym)=(yyvsp[0].sym);} -#line 2287 "ncgeny.c" /* yacc.c:1646 */ + case 87: /* ambiguous_ref: primtype */ +#line 714 "ncgen.y" + {(yyval.sym)=(yyvsp[0].sym);} +#line 2494 "ncgeny.c" break; - case 88: -#line 719 "ncgen.y" /* yacc.c:1646 */ - {} -#line 2293 "ncgeny.c" /* yacc.c:1646 */ + case 88: /* attrdecllist: %empty */ +#line 721 "ncgen.y" + {} +#line 2500 "ncgeny.c" break; - case 89: -#line 719 "ncgen.y" /* yacc.c:1646 */ - {} -#line 2299 "ncgeny.c" /* yacc.c:1646 */ + case 89: /* attrdecllist: attrdecl ';' attrdecllist */ +#line 721 "ncgen.y" + {} +#line 2506 "ncgeny.c" break; - case 90: -#line 723 "ncgen.y" /* yacc.c:1646 */ - {(yyval.sym) = makespecial(_NCPROPS_FLAG,NULL,NULL,(void*)(yyvsp[0].constant),ISCONST);} -#line 2305 "ncgeny.c" /* yacc.c:1646 */ + case 90: /* attrdecl: ':' _NCPROPS '=' conststring */ +#line 725 "ncgen.y" + {(yyval.sym) = makespecial(_NCPROPS_FLAG,NULL,NULL,(void*)(yyvsp[0].constant),ISCONST);} +#line 2512 "ncgeny.c" break; - case 91: -#line 725 "ncgen.y" /* yacc.c:1646 */ - {(yyval.sym) = makespecial(_ISNETCDF4_FLAG,NULL,NULL,(void*)(yyvsp[0].constant),ISCONST);} -#line 2311 "ncgeny.c" /* yacc.c:1646 */ + case 91: /* attrdecl: ':' _ISNETCDF4 '=' constbool */ +#line 727 "ncgen.y" + {(yyval.sym) = makespecial(_ISNETCDF4_FLAG,NULL,NULL,(void*)(yyvsp[0].constant),ISCONST);} +#line 2518 "ncgeny.c" break; - case 92: -#line 727 "ncgen.y" /* yacc.c:1646 */ - {(yyval.sym) = makespecial(_SUPERBLOCK_FLAG,NULL,NULL,(void*)(yyvsp[0].constant),ISCONST);} -#line 2317 "ncgeny.c" /* yacc.c:1646 */ + case 92: /* attrdecl: ':' _SUPERBLOCK '=' constint */ +#line 729 "ncgen.y" + {(yyval.sym) = makespecial(_SUPERBLOCK_FLAG,NULL,NULL,(void*)(yyvsp[0].constant),ISCONST);} +#line 2524 "ncgeny.c" break; - case 93: -#line 729 "ncgen.y" /* yacc.c:1646 */ - { (yyval.sym)=makeattribute((yyvsp[-2].sym),NULL,NULL,(yyvsp[0].datalist),ATTRGLOBAL);} -#line 2323 "ncgeny.c" /* yacc.c:1646 */ + case 93: /* attrdecl: ':' ident '=' datalist */ +#line 731 "ncgen.y" + { (yyval.sym)=makeattribute((yyvsp[-2].sym),NULL,NULL,(yyvsp[0].datalist),ATTRGLOBAL);} +#line 2530 "ncgeny.c" break; - case 94: -#line 731 "ncgen.y" /* yacc.c:1646 */ - {Symbol* tsym = (yyvsp[-5].sym); Symbol* vsym = (yyvsp[-4].sym); Symbol* asym = (yyvsp[-2].sym); + case 94: /* attrdecl: typeref ambiguous_ref ':' ident '=' datalist */ +#line 733 "ncgen.y" + {Symbol* tsym = (yyvsp[-5].sym); Symbol* vsym = (yyvsp[-4].sym); Symbol* asym = (yyvsp[-2].sym); if(vsym->objectclass == NC_VAR) { (yyval.sym)=makeattribute(asym,vsym,tsym,(yyvsp[0].datalist),ATTRVAR); } else { @@ -2332,12 +2539,12 @@ fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); YYABORT; } } -#line 2336 "ncgeny.c" /* yacc.c:1646 */ +#line 2543 "ncgeny.c" break; - case 95: -#line 740 "ncgen.y" /* yacc.c:1646 */ - {Symbol* sym = (yyvsp[-4].sym); Symbol* asym = (yyvsp[-2].sym); + case 95: /* attrdecl: ambiguous_ref ':' ident '=' datalist */ +#line 742 "ncgen.y" + {Symbol* sym = (yyvsp[-4].sym); Symbol* asym = (yyvsp[-2].sym); if(sym->objectclass == NC_VAR) { (yyval.sym)=makeattribute(asym,sym,NULL,(yyvsp[0].datalist),ATTRVAR); } else if(sym->objectclass == NC_TYPE) { @@ -2347,345 +2554,352 @@ fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); YYABORT; } } -#line 2351 "ncgeny.c" /* yacc.c:1646 */ +#line 2558 "ncgeny.c" break; - case 96: -#line 751 "ncgen.y" /* yacc.c:1646 */ - {(yyval.sym) = makespecial(_FILLVALUE_FLAG,(yyvsp[-4].sym),NULL,(void*)(yyvsp[0].datalist),ISLIST);} -#line 2357 "ncgeny.c" /* yacc.c:1646 */ + case 96: /* attrdecl: ambiguous_ref ':' _FILLVALUE '=' datalist */ +#line 753 "ncgen.y" + {(yyval.sym) = makespecial(_FILLVALUE_FLAG,(yyvsp[-4].sym),NULL,(void*)(yyvsp[0].datalist),ISLIST);} +#line 2564 "ncgeny.c" break; - case 97: -#line 753 "ncgen.y" /* yacc.c:1646 */ - {(yyval.sym) = makespecial(_FILLVALUE_FLAG,(yyvsp[-4].sym),(yyvsp[-5].sym),(void*)(yyvsp[0].datalist),ISLIST);} -#line 2363 "ncgeny.c" /* yacc.c:1646 */ + case 97: /* attrdecl: typeref ambiguous_ref ':' _FILLVALUE '=' datalist */ +#line 755 "ncgen.y" + {(yyval.sym) = makespecial(_FILLVALUE_FLAG,(yyvsp[-4].sym),(yyvsp[-5].sym),(void*)(yyvsp[0].datalist),ISLIST);} +#line 2570 "ncgeny.c" break; - case 98: -#line 755 "ncgen.y" /* yacc.c:1646 */ - {(yyval.sym) = makespecial(_STORAGE_FLAG,(yyvsp[-4].sym),NULL,(void*)(yyvsp[0].constant),ISCONST);} -#line 2369 "ncgeny.c" /* yacc.c:1646 */ + case 98: /* attrdecl: ambiguous_ref ':' _STORAGE '=' conststring */ +#line 757 "ncgen.y" + {(yyval.sym) = makespecial(_STORAGE_FLAG,(yyvsp[-4].sym),NULL,(void*)(yyvsp[0].constant),ISCONST);} +#line 2576 "ncgeny.c" break; - case 99: -#line 757 "ncgen.y" /* yacc.c:1646 */ - {(yyval.sym) = makespecial(_CHUNKSIZES_FLAG,(yyvsp[-4].sym),NULL,(void*)(yyvsp[0].datalist),ISLIST);} -#line 2375 "ncgeny.c" /* yacc.c:1646 */ + case 99: /* attrdecl: ambiguous_ref ':' _CHUNKSIZES '=' intlist */ +#line 759 "ncgen.y" + {(yyval.sym) = makespecial(_CHUNKSIZES_FLAG,(yyvsp[-4].sym),NULL,(void*)(yyvsp[0].datalist),ISLIST);} +#line 2582 "ncgeny.c" break; - case 100: -#line 759 "ncgen.y" /* yacc.c:1646 */ - {(yyval.sym) = makespecial(_FLETCHER32_FLAG,(yyvsp[-4].sym),NULL,(void*)(yyvsp[0].constant),ISCONST);} -#line 2381 "ncgeny.c" /* yacc.c:1646 */ + case 100: /* attrdecl: ambiguous_ref ':' _FLETCHER32 '=' constbool */ +#line 761 "ncgen.y" + {(yyval.sym) = makespecial(_FLETCHER32_FLAG,(yyvsp[-4].sym),NULL,(void*)(yyvsp[0].constant),ISCONST);} +#line 2588 "ncgeny.c" break; - case 101: -#line 761 "ncgen.y" /* yacc.c:1646 */ - {(yyval.sym) = makespecial(_DEFLATE_FLAG,(yyvsp[-4].sym),NULL,(void*)(yyvsp[0].constant),ISCONST);} -#line 2387 "ncgeny.c" /* yacc.c:1646 */ + case 101: /* attrdecl: ambiguous_ref ':' _DEFLATELEVEL '=' constint */ +#line 763 "ncgen.y" + {(yyval.sym) = makespecial(_DEFLATE_FLAG,(yyvsp[-4].sym),NULL,(void*)(yyvsp[0].constant),ISCONST);} +#line 2594 "ncgeny.c" break; - case 102: -#line 763 "ncgen.y" /* yacc.c:1646 */ - {(yyval.sym) = makespecial(_SHUFFLE_FLAG,(yyvsp[-4].sym),NULL,(void*)(yyvsp[0].constant),ISCONST);} -#line 2393 "ncgeny.c" /* yacc.c:1646 */ + case 102: /* attrdecl: ambiguous_ref ':' _SHUFFLE '=' constbool */ +#line 765 "ncgen.y" + {(yyval.sym) = makespecial(_SHUFFLE_FLAG,(yyvsp[-4].sym),NULL,(void*)(yyvsp[0].constant),ISCONST);} +#line 2600 "ncgeny.c" break; - case 103: -#line 765 "ncgen.y" /* yacc.c:1646 */ - {(yyval.sym) = makespecial(_ENDIAN_FLAG,(yyvsp[-4].sym),NULL,(void*)(yyvsp[0].constant),ISCONST);} -#line 2399 "ncgeny.c" /* yacc.c:1646 */ + case 103: /* attrdecl: ambiguous_ref ':' _ENDIANNESS '=' conststring */ +#line 767 "ncgen.y" + {(yyval.sym) = makespecial(_ENDIAN_FLAG,(yyvsp[-4].sym),NULL,(void*)(yyvsp[0].constant),ISCONST);} +#line 2606 "ncgeny.c" break; - case 104: -#line 767 "ncgen.y" /* yacc.c:1646 */ - {(yyval.sym) = makespecial(_FILTER_FLAG,(yyvsp[-4].sym),NULL,(void*)(yyvsp[0].constant),ISCONST);} -#line 2405 "ncgeny.c" /* yacc.c:1646 */ + case 104: /* attrdecl: ambiguous_ref ':' _FILTER '=' conststring */ +#line 769 "ncgen.y" + {(yyval.sym) = makespecial(_FILTER_FLAG,(yyvsp[-4].sym),NULL,(void*)(yyvsp[0].constant),ISCONST);} +#line 2612 "ncgeny.c" break; - case 105: -#line 769 "ncgen.y" /* yacc.c:1646 */ - {(yyval.sym) = makespecial(_NOFILL_FLAG,(yyvsp[-4].sym),NULL,(void*)(yyvsp[0].constant),ISCONST);} -#line 2411 "ncgeny.c" /* yacc.c:1646 */ + case 105: /* attrdecl: ambiguous_ref ':' _CODECS '=' conststring */ +#line 771 "ncgen.y" + {(yyval.sym) = makespecial(_CODECS_FLAG,(yyvsp[-4].sym),NULL,(void*)(yyvsp[0].constant),ISCONST);} +#line 2618 "ncgeny.c" break; - case 106: -#line 771 "ncgen.y" /* yacc.c:1646 */ - {(yyval.sym) = makespecial(_FORMAT_FLAG,NULL,NULL,(void*)(yyvsp[0].constant),ISCONST);} -#line 2417 "ncgeny.c" /* yacc.c:1646 */ + case 106: /* attrdecl: ambiguous_ref ':' _NOFILL '=' constbool */ +#line 773 "ncgen.y" + {(yyval.sym) = makespecial(_NOFILL_FLAG,(yyvsp[-4].sym),NULL,(void*)(yyvsp[0].constant),ISCONST);} +#line 2624 "ncgeny.c" break; - case 107: -#line 776 "ncgen.y" /* yacc.c:1646 */ - { + case 107: /* attrdecl: ':' _FORMAT '=' conststring */ +#line 775 "ncgen.y" + {(yyval.sym) = makespecial(_FORMAT_FLAG,NULL,NULL,(void*)(yyvsp[0].constant),ISCONST);} +#line 2630 "ncgeny.c" + break; + + case 108: /* path: ident */ +#line 780 "ncgen.y" + { (yyval.sym)=(yyvsp[0].sym); (yyvsp[0].sym)->ref.is_ref=1; (yyvsp[0].sym)->is_prefixed=0; setpathcurrent((yyvsp[0].sym)); } -#line 2428 "ncgeny.c" /* yacc.c:1646 */ +#line 2641 "ncgeny.c" break; - case 108: -#line 783 "ncgen.y" /* yacc.c:1646 */ - { + case 109: /* path: PATH */ +#line 787 "ncgen.y" + { (yyval.sym)=(yyvsp[0].sym); (yyvsp[0].sym)->ref.is_ref=1; (yyvsp[0].sym)->is_prefixed=1; /* path is set in ncgen.l*/ } -#line 2439 "ncgeny.c" /* yacc.c:1646 */ +#line 2652 "ncgeny.c" break; - case 110: -#line 792 "ncgen.y" /* yacc.c:1646 */ - {} -#line 2445 "ncgeny.c" /* yacc.c:1646 */ + case 111: /* datasection: DATA */ +#line 796 "ncgen.y" + {} +#line 2658 "ncgeny.c" break; - case 111: -#line 793 "ncgen.y" /* yacc.c:1646 */ - {} -#line 2451 "ncgeny.c" /* yacc.c:1646 */ + case 112: /* datasection: DATA datadecls */ +#line 797 "ncgen.y" + {} +#line 2664 "ncgeny.c" break; - case 114: -#line 801 "ncgen.y" /* yacc.c:1646 */ - {(yyvsp[-2].sym)->data = (yyvsp[0].datalist);} -#line 2457 "ncgeny.c" /* yacc.c:1646 */ + case 115: /* datadecl: varref '=' datalist */ +#line 805 "ncgen.y" + {(yyvsp[-2].sym)->data = (yyvsp[0].datalist);} +#line 2670 "ncgeny.c" break; - case 115: -#line 804 "ncgen.y" /* yacc.c:1646 */ - {(yyval.datalist) = (yyvsp[0].datalist);} -#line 2463 "ncgeny.c" /* yacc.c:1646 */ + case 116: /* datalist: datalist0 */ +#line 808 "ncgen.y" + {(yyval.datalist) = (yyvsp[0].datalist);} +#line 2676 "ncgeny.c" break; - case 116: -#line 805 "ncgen.y" /* yacc.c:1646 */ - {(yyval.datalist) = (yyvsp[0].datalist);} -#line 2469 "ncgeny.c" /* yacc.c:1646 */ + case 117: /* datalist: datalist1 */ +#line 809 "ncgen.y" + {(yyval.datalist) = (yyvsp[0].datalist);} +#line 2682 "ncgeny.c" break; - case 117: -#line 809 "ncgen.y" /* yacc.c:1646 */ - {(yyval.datalist) = builddatalist(0);} -#line 2475 "ncgeny.c" /* yacc.c:1646 */ + case 118: /* datalist0: %empty */ +#line 813 "ncgen.y" + {(yyval.datalist) = builddatalist(0);} +#line 2688 "ncgeny.c" break; - case 118: -#line 813 "ncgen.y" /* yacc.c:1646 */ - {(yyval.datalist) = const2list((yyvsp[0].constant));} -#line 2481 "ncgeny.c" /* yacc.c:1646 */ + case 119: /* datalist1: dataitem */ +#line 817 "ncgen.y" + {(yyval.datalist) = const2list((yyvsp[0].constant));} +#line 2694 "ncgeny.c" break; - case 119: -#line 815 "ncgen.y" /* yacc.c:1646 */ - {dlappend((yyvsp[-2].datalist),((yyvsp[0].constant))); (yyval.datalist)=(yyvsp[-2].datalist); } -#line 2487 "ncgeny.c" /* yacc.c:1646 */ + case 120: /* datalist1: datalist ',' dataitem */ +#line 819 "ncgen.y" + {dlappend((yyvsp[-2].datalist),((yyvsp[0].constant))); (yyval.datalist)=(yyvsp[-2].datalist); } +#line 2700 "ncgeny.c" break; - case 120: -#line 819 "ncgen.y" /* yacc.c:1646 */ - {(yyval.constant)=(yyvsp[0].constant);} -#line 2493 "ncgeny.c" /* yacc.c:1646 */ + case 121: /* dataitem: constdata */ +#line 823 "ncgen.y" + {(yyval.constant)=(yyvsp[0].constant);} +#line 2706 "ncgeny.c" break; - case 121: -#line 820 "ncgen.y" /* yacc.c:1646 */ - {(yyval.constant)=builddatasublist((yyvsp[-1].datalist));} -#line 2499 "ncgeny.c" /* yacc.c:1646 */ + case 122: /* dataitem: '{' datalist '}' */ +#line 824 "ncgen.y" + {(yyval.constant)=builddatasublist((yyvsp[-1].datalist));} +#line 2712 "ncgeny.c" break; - case 122: -#line 824 "ncgen.y" /* yacc.c:1646 */ - {(yyval.constant)=(yyvsp[0].constant);} -#line 2505 "ncgeny.c" /* yacc.c:1646 */ + case 123: /* constdata: simpleconstant */ +#line 828 "ncgen.y" + {(yyval.constant)=(yyvsp[0].constant);} +#line 2718 "ncgeny.c" break; - case 123: -#line 825 "ncgen.y" /* yacc.c:1646 */ - {(yyval.constant)=makeconstdata(NC_OPAQUE);} -#line 2511 "ncgeny.c" /* yacc.c:1646 */ + case 124: /* constdata: OPAQUESTRING */ +#line 829 "ncgen.y" + {(yyval.constant)=makeconstdata(NC_OPAQUE);} +#line 2724 "ncgeny.c" break; - case 124: -#line 826 "ncgen.y" /* yacc.c:1646 */ - {(yyval.constant)=makeconstdata(NC_FILLVALUE);} -#line 2517 "ncgeny.c" /* yacc.c:1646 */ + case 125: /* constdata: FILLMARKER */ +#line 830 "ncgen.y" + {(yyval.constant)=makeconstdata(NC_FILLVALUE);} +#line 2730 "ncgeny.c" break; - case 125: -#line 827 "ncgen.y" /* yacc.c:1646 */ - {(yyval.constant)=makeconstdata(NC_NIL);} -#line 2523 "ncgeny.c" /* yacc.c:1646 */ + case 126: /* constdata: NIL */ +#line 831 "ncgen.y" + {(yyval.constant)=makeconstdata(NC_NIL);} +#line 2736 "ncgeny.c" break; - case 126: -#line 828 "ncgen.y" /* yacc.c:1646 */ - {(yyval.constant)=(yyvsp[0].constant);} -#line 2529 "ncgeny.c" /* yacc.c:1646 */ + case 127: /* constdata: econstref */ +#line 832 "ncgen.y" + {(yyval.constant)=(yyvsp[0].constant);} +#line 2742 "ncgeny.c" break; - case 128: -#line 833 "ncgen.y" /* yacc.c:1646 */ - {(yyval.constant) = makeenumconstref((yyvsp[0].sym));} -#line 2535 "ncgeny.c" /* yacc.c:1646 */ + case 129: /* econstref: path */ +#line 837 "ncgen.y" + {(yyval.constant) = makeenumconstref((yyvsp[0].sym));} +#line 2748 "ncgeny.c" break; - case 129: -#line 837 "ncgen.y" /* yacc.c:1646 */ - {(yyval.constant)=evaluate((yyvsp[-3].sym),(yyvsp[-1].datalist));} -#line 2541 "ncgeny.c" /* yacc.c:1646 */ + case 130: /* function: ident '(' arglist ')' */ +#line 841 "ncgen.y" + {(yyval.constant)=evaluate((yyvsp[-3].sym),(yyvsp[-1].datalist));} +#line 2754 "ncgeny.c" break; - case 130: -#line 842 "ncgen.y" /* yacc.c:1646 */ - {(yyval.datalist) = const2list((yyvsp[0].constant));} -#line 2547 "ncgeny.c" /* yacc.c:1646 */ + case 131: /* arglist: simpleconstant */ +#line 846 "ncgen.y" + {(yyval.datalist) = const2list((yyvsp[0].constant));} +#line 2760 "ncgeny.c" break; - case 131: -#line 844 "ncgen.y" /* yacc.c:1646 */ - {dlappend((yyvsp[-2].datalist),((yyvsp[0].constant))); (yyval.datalist)=(yyvsp[-2].datalist);} -#line 2553 "ncgeny.c" /* yacc.c:1646 */ + case 132: /* arglist: arglist ',' simpleconstant */ +#line 848 "ncgen.y" + {dlappend((yyvsp[-2].datalist),((yyvsp[0].constant))); (yyval.datalist)=(yyvsp[-2].datalist);} +#line 2766 "ncgeny.c" break; - case 132: -#line 848 "ncgen.y" /* yacc.c:1646 */ - {(yyval.constant)=makeconstdata(NC_CHAR);} -#line 2559 "ncgeny.c" /* yacc.c:1646 */ + case 133: /* simpleconstant: CHAR_CONST */ +#line 852 "ncgen.y" + {(yyval.constant)=makeconstdata(NC_CHAR);} +#line 2772 "ncgeny.c" break; - case 133: -#line 849 "ncgen.y" /* yacc.c:1646 */ - {(yyval.constant)=makeconstdata(NC_BYTE);} -#line 2565 "ncgeny.c" /* yacc.c:1646 */ + case 134: /* simpleconstant: BYTE_CONST */ +#line 853 "ncgen.y" + {(yyval.constant)=makeconstdata(NC_BYTE);} +#line 2778 "ncgeny.c" break; - case 134: -#line 850 "ncgen.y" /* yacc.c:1646 */ - {(yyval.constant)=makeconstdata(NC_SHORT);} -#line 2571 "ncgeny.c" /* yacc.c:1646 */ + case 135: /* simpleconstant: SHORT_CONST */ +#line 854 "ncgen.y" + {(yyval.constant)=makeconstdata(NC_SHORT);} +#line 2784 "ncgeny.c" break; - case 135: -#line 851 "ncgen.y" /* yacc.c:1646 */ - {(yyval.constant)=makeconstdata(NC_INT);} -#line 2577 "ncgeny.c" /* yacc.c:1646 */ + case 136: /* simpleconstant: INT_CONST */ +#line 855 "ncgen.y" + {(yyval.constant)=makeconstdata(NC_INT);} +#line 2790 "ncgeny.c" break; - case 136: -#line 852 "ncgen.y" /* yacc.c:1646 */ - {(yyval.constant)=makeconstdata(NC_INT64);} -#line 2583 "ncgeny.c" /* yacc.c:1646 */ + case 137: /* simpleconstant: INT64_CONST */ +#line 856 "ncgen.y" + {(yyval.constant)=makeconstdata(NC_INT64);} +#line 2796 "ncgeny.c" break; - case 137: -#line 853 "ncgen.y" /* yacc.c:1646 */ - {(yyval.constant)=makeconstdata(NC_UBYTE);} -#line 2589 "ncgeny.c" /* yacc.c:1646 */ + case 138: /* simpleconstant: UBYTE_CONST */ +#line 857 "ncgen.y" + {(yyval.constant)=makeconstdata(NC_UBYTE);} +#line 2802 "ncgeny.c" break; - case 138: -#line 854 "ncgen.y" /* yacc.c:1646 */ - {(yyval.constant)=makeconstdata(NC_USHORT);} -#line 2595 "ncgeny.c" /* yacc.c:1646 */ + case 139: /* simpleconstant: USHORT_CONST */ +#line 858 "ncgen.y" + {(yyval.constant)=makeconstdata(NC_USHORT);} +#line 2808 "ncgeny.c" break; - case 139: -#line 855 "ncgen.y" /* yacc.c:1646 */ - {(yyval.constant)=makeconstdata(NC_UINT);} -#line 2601 "ncgeny.c" /* yacc.c:1646 */ + case 140: /* simpleconstant: UINT_CONST */ +#line 859 "ncgen.y" + {(yyval.constant)=makeconstdata(NC_UINT);} +#line 2814 "ncgeny.c" break; - case 140: -#line 856 "ncgen.y" /* yacc.c:1646 */ - {(yyval.constant)=makeconstdata(NC_UINT64);} -#line 2607 "ncgeny.c" /* yacc.c:1646 */ + case 141: /* simpleconstant: UINT64_CONST */ +#line 860 "ncgen.y" + {(yyval.constant)=makeconstdata(NC_UINT64);} +#line 2820 "ncgeny.c" break; - case 141: -#line 857 "ncgen.y" /* yacc.c:1646 */ - {(yyval.constant)=makeconstdata(NC_FLOAT);} -#line 2613 "ncgeny.c" /* yacc.c:1646 */ + case 142: /* simpleconstant: FLOAT_CONST */ +#line 861 "ncgen.y" + {(yyval.constant)=makeconstdata(NC_FLOAT);} +#line 2826 "ncgeny.c" break; - case 142: -#line 858 "ncgen.y" /* yacc.c:1646 */ - {(yyval.constant)=makeconstdata(NC_DOUBLE);} -#line 2619 "ncgeny.c" /* yacc.c:1646 */ + case 143: /* simpleconstant: DOUBLE_CONST */ +#line 862 "ncgen.y" + {(yyval.constant)=makeconstdata(NC_DOUBLE);} +#line 2832 "ncgeny.c" break; - case 143: -#line 859 "ncgen.y" /* yacc.c:1646 */ - {(yyval.constant)=makeconstdata(NC_STRING);} -#line 2625 "ncgeny.c" /* yacc.c:1646 */ + case 144: /* simpleconstant: TERMSTRING */ +#line 863 "ncgen.y" + {(yyval.constant)=makeconstdata(NC_STRING);} +#line 2838 "ncgeny.c" break; - case 144: -#line 863 "ncgen.y" /* yacc.c:1646 */ - {(yyval.datalist) = const2list((yyvsp[0].constant));} -#line 2631 "ncgeny.c" /* yacc.c:1646 */ + case 145: /* intlist: constint */ +#line 867 "ncgen.y" + {(yyval.datalist) = const2list((yyvsp[0].constant));} +#line 2844 "ncgeny.c" break; - case 145: -#line 864 "ncgen.y" /* yacc.c:1646 */ - {(yyval.datalist)=(yyvsp[-2].datalist); dlappend((yyvsp[-2].datalist),((yyvsp[0].constant)));} -#line 2637 "ncgeny.c" /* yacc.c:1646 */ + case 146: /* intlist: intlist ',' constint */ +#line 868 "ncgen.y" + {(yyval.datalist)=(yyvsp[-2].datalist); dlappend((yyvsp[-2].datalist),((yyvsp[0].constant)));} +#line 2850 "ncgeny.c" break; - case 146: -#line 869 "ncgen.y" /* yacc.c:1646 */ - {(yyval.constant)=makeconstdata(NC_INT);} -#line 2643 "ncgeny.c" /* yacc.c:1646 */ + case 147: /* constint: INT_CONST */ +#line 873 "ncgen.y" + {(yyval.constant)=makeconstdata(NC_INT);} +#line 2856 "ncgeny.c" break; - case 147: -#line 871 "ncgen.y" /* yacc.c:1646 */ - {(yyval.constant)=makeconstdata(NC_UINT);} -#line 2649 "ncgeny.c" /* yacc.c:1646 */ + case 148: /* constint: UINT_CONST */ +#line 875 "ncgen.y" + {(yyval.constant)=makeconstdata(NC_UINT);} +#line 2862 "ncgeny.c" break; - case 148: -#line 873 "ncgen.y" /* yacc.c:1646 */ - {(yyval.constant)=makeconstdata(NC_INT64);} -#line 2655 "ncgeny.c" /* yacc.c:1646 */ + case 149: /* constint: INT64_CONST */ +#line 877 "ncgen.y" + {(yyval.constant)=makeconstdata(NC_INT64);} +#line 2868 "ncgeny.c" break; - case 149: -#line 875 "ncgen.y" /* yacc.c:1646 */ - {(yyval.constant)=makeconstdata(NC_UINT64);} -#line 2661 "ncgeny.c" /* yacc.c:1646 */ + case 150: /* constint: UINT64_CONST */ +#line 879 "ncgen.y" + {(yyval.constant)=makeconstdata(NC_UINT64);} +#line 2874 "ncgeny.c" break; - case 150: -#line 879 "ncgen.y" /* yacc.c:1646 */ - {(yyval.constant)=makeconstdata(NC_STRING);} -#line 2667 "ncgeny.c" /* yacc.c:1646 */ + case 151: /* conststring: TERMSTRING */ +#line 883 "ncgen.y" + {(yyval.constant)=makeconstdata(NC_STRING);} +#line 2880 "ncgeny.c" break; - case 151: -#line 883 "ncgen.y" /* yacc.c:1646 */ - {(yyval.constant)=(yyvsp[0].constant);} -#line 2673 "ncgeny.c" /* yacc.c:1646 */ + case 152: /* constbool: conststring */ +#line 887 "ncgen.y" + {(yyval.constant)=(yyvsp[0].constant);} +#line 2886 "ncgeny.c" break; - case 152: -#line 884 "ncgen.y" /* yacc.c:1646 */ - {(yyval.constant)=(yyvsp[0].constant);} -#line 2679 "ncgeny.c" /* yacc.c:1646 */ + case 153: /* constbool: constint */ +#line 888 "ncgen.y" + {(yyval.constant)=(yyvsp[0].constant);} +#line 2892 "ncgeny.c" break; - case 153: -#line 890 "ncgen.y" /* yacc.c:1646 */ - {(yyval.sym)=(yyvsp[0].sym);} -#line 2685 "ncgeny.c" /* yacc.c:1646 */ + case 154: /* ident: IDENT */ +#line 894 "ncgen.y" + {(yyval.sym)=(yyvsp[0].sym);} +#line 2898 "ncgeny.c" break; -#line 2689 "ncgeny.c" /* yacc.c:1646 */ +#line 2902 "ncgeny.c" + default: break; } /* User semantic actions sometimes alter yychar, and that requires @@ -2699,25 +2913,23 @@ fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); case of YYERROR or YYBACKUP, subsequent parser actions might lead to an incorrect destructor call or verbose syntax error message before the lookahead is translated. */ - YY_SYMBOL_PRINT ("-> $$ =", yyr1[yyn], &yyval, &yyloc); + YY_SYMBOL_PRINT ("-> $$ =", YY_CAST (yysymbol_kind_t, yyr1[yyn]), &yyval, &yyloc); YYPOPSTACK (yylen); yylen = 0; - YY_STACK_PRINT (yyss, yyssp); *++yyvsp = yyval; /* Now 'shift' the result of the reduction. Determine what state that goes to, based on the state we popped back to and the rule number reduced by. */ - - yyn = yyr1[yyn]; - - yystate = yypgoto[yyn - YYNTOKENS] + *yyssp; - if (0 <= yystate && yystate <= YYLAST && yycheck[yystate] == *yyssp) - yystate = yytable[yystate]; - else - yystate = yydefgoto[yyn - YYNTOKENS]; + { + const int yylhs = yyr1[yyn] - YYNTOKENS; + const int yyi = yypgoto[yylhs] + *yyssp; + yystate = (0 <= yyi && yyi <= YYLAST && yycheck[yyi] == *yyssp + ? yytable[yyi] + : yydefgoto[yylhs]); + } goto yynewstate; @@ -2728,50 +2940,44 @@ fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); yyerrlab: /* Make sure we have latest lookahead translation. See comments at user semantic actions for why this is necessary. */ - yytoken = yychar == YYEMPTY ? YYEMPTY : YYTRANSLATE (yychar); - + yytoken = yychar == YYEMPTY ? YYSYMBOL_YYEMPTY : YYTRANSLATE (yychar); /* If not already recovering from an error, report this error. */ if (!yyerrstatus) { ++yynerrs; -#if ! YYERROR_VERBOSE - yyerror (YY_("syntax error")); -#else -# define YYSYNTAX_ERROR yysyntax_error (&yymsg_alloc, &yymsg, \ - yyssp, yytoken) { + yypcontext_t yyctx + = {yyssp, yytoken}; char const *yymsgp = YY_("syntax error"); int yysyntax_error_status; - yysyntax_error_status = YYSYNTAX_ERROR; + yysyntax_error_status = yysyntax_error (&yymsg_alloc, &yymsg, &yyctx); if (yysyntax_error_status == 0) yymsgp = yymsg; - else if (yysyntax_error_status == 1) + else if (yysyntax_error_status == -1) { if (yymsg != yymsgbuf) YYSTACK_FREE (yymsg); - yymsg = (char *) YYSTACK_ALLOC (yymsg_alloc); - if (!yymsg) + yymsg = YY_CAST (char *, + YYSTACK_ALLOC (YY_CAST (YYSIZE_T, yymsg_alloc))); + if (yymsg) { - yymsg = yymsgbuf; - yymsg_alloc = sizeof yymsgbuf; - yysyntax_error_status = 2; + yysyntax_error_status + = yysyntax_error (&yymsg_alloc, &yymsg, &yyctx); + yymsgp = yymsg; } else { - yysyntax_error_status = YYSYNTAX_ERROR; - yymsgp = yymsg; + yymsg = yymsgbuf; + yymsg_alloc = sizeof yymsgbuf; + yysyntax_error_status = YYENOMEM; } } yyerror (yymsgp); - if (yysyntax_error_status == 2) + if (yysyntax_error_status == YYENOMEM) goto yyexhaustedlab; } -# undef YYSYNTAX_ERROR -#endif } - - if (yyerrstatus == 3) { /* If just tried and failed to reuse lookahead token after an @@ -2800,12 +3006,10 @@ fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); | yyerrorlab -- error raised explicitly by YYERROR. | `---------------------------------------------------*/ yyerrorlab: - - /* Pacify compilers like GCC when the user code never invokes - YYERROR and the label yyerrorlab therefore never appears in user - code. */ - if (/*CONSTCOND*/ 0) - goto yyerrorlab; + /* Pacify compilers when the user code never invokes YYERROR and the + label yyerrorlab therefore never appears in user code. */ + if (0) + YYERROR; /* Do not reclaim the symbols of the rule whose action triggered this YYERROR. */ @@ -2822,13 +3026,14 @@ fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); yyerrlab1: yyerrstatus = 3; /* Each real token shifted decrements this. */ + /* Pop stack until we find a state that shifts the error token. */ for (;;) { yyn = yypact[yystate]; if (!yypact_value_is_default (yyn)) { - yyn += YYTERROR; - if (0 <= yyn && yyn <= YYLAST && yycheck[yyn] == YYTERROR) + yyn += YYSYMBOL_YYerror; + if (0 <= yyn && yyn <= YYLAST && yycheck[yyn] == YYSYMBOL_YYerror) { yyn = yytable[yyn]; if (0 < yyn) @@ -2842,7 +3047,7 @@ fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); yydestruct ("Error: popping", - yystos[yystate], yyvsp); + YY_ACCESSING_SYMBOL (yystate), yyvsp); YYPOPSTACK (1); yystate = *yyssp; YY_STACK_PRINT (yyss, yyssp); @@ -2854,7 +3059,7 @@ fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); /* Shift the error token. */ - YY_SYMBOL_PRINT ("Shifting", yystos[yyn], yyvsp, yylsp); + YY_SYMBOL_PRINT ("Shifting", YY_ACCESSING_SYMBOL (yyn), yyvsp, yylsp); yystate = yyn; goto yynewstate; @@ -2867,6 +3072,7 @@ fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); yyresult = 0; goto yyreturn; + /*-----------------------------------. | yyabortlab -- YYABORT comes here. | `-----------------------------------*/ @@ -2874,16 +3080,21 @@ fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); yyresult = 1; goto yyreturn; -#if !defined yyoverflow || YYERROR_VERBOSE + +#if 1 /*-------------------------------------------------. | yyexhaustedlab -- memory exhaustion comes here. | `-------------------------------------------------*/ yyexhaustedlab: yyerror (YY_("memory exhausted")); yyresult = 2; - /* Fall through. */ + goto yyreturn; #endif + +/*-------------------------------------------------------. +| yyreturn -- parsing is finished, clean up and return. | +`-------------------------------------------------------*/ yyreturn: if (yychar != YYEMPTY) { @@ -2900,20 +3111,19 @@ fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); while (yyssp != yyss) { yydestruct ("Cleanup: popping", - yystos[*yyssp], yyvsp); + YY_ACCESSING_SYMBOL (+*yyssp), yyvsp); YYPOPSTACK (1); } #ifndef yyoverflow if (yyss != yyssa) YYSTACK_FREE (yyss); #endif -#if YYERROR_VERBOSE if (yymsg != yymsgbuf) YYSTACK_FREE (yymsg); -#endif return yyresult; } -#line 893 "ncgen.y" /* yacc.c:1906 */ + +#line 897 "ncgen.y" #ifndef NO_STDARG @@ -3236,6 +3446,7 @@ makespecial(int tag, Symbol* vsym, Symbol* tsym, void* data, int isconst) case _NCPROPS_FLAG: case _ENDIAN_FLAG: case _FILTER_FLAG: + case _CODECS_FLAG: tmp = nullconst(); tmp->nctype = NC_STRING; convert1(con,tmp); @@ -3394,6 +3605,18 @@ makespecial(int tag, Symbol* vsym, Symbol* tsym, void* data, int isconst) } #else derror("%s: the filter attribute requires netcdf-4 to be enabled",specialname(tag)); +#endif + break; + case _CODECS_FLAG: +#ifdef USE_NETCDF4 + /* Parse the codec spec */ + if(parsecodecsflag(sdata,special) == NC_NOERR) + special->flags |= _CODECS_FLAG; + else { + derror("_Codecs: unparsable codec spec: %s",sdata); + } +#else + derror("%s: the _Codecs attribute requires netcdf-4 to be enabled",specialname(tag)); #endif break; default: PANIC1("makespecial: illegal token: %d",tag); @@ -3522,6 +3745,21 @@ printfilters(special->nfilters,special->_Filters); #endif return stat; } + +/* +Store a Codecs spec string in special +*/ +static int +parsecodecsflag(const char* sdata, Specialdata* special) +{ + int stat = NC_NOERR; + + if(sdata == NULL || strlen(sdata) == 0) return NC_EINVAL; + + if((special->_Codecs = strdup(sdata))==NULL) + return NC_ENOMEM; + return stat; +} #endif /* diff --git a/ncgen/ncgeny.h b/ncgen/ncgeny.h index 380bcf910e..307f64223d 100644 --- a/ncgen/ncgeny.h +++ b/ncgen/ncgeny.h @@ -1,8 +1,9 @@ -/* A Bison parser, made by GNU Bison 3.0.4. */ +/* A Bison parser, made by GNU Bison 3.7.6. */ /* Bison interface for Yacc-like parsers in C - Copyright (C) 1984, 1989-1990, 2000-2015 Free Software Foundation, Inc. + Copyright (C) 1984, 1989-1990, 2000-2015, 2018-2021 Free Software Foundation, + Inc. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -15,7 +16,7 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License - along with this program. If not, see . */ + along with this program. If not, see . */ /* As a special exception, you may create a larger work that contains part or all of the Bison parser skeleton and distribute that work @@ -30,6 +31,10 @@ This special exception was added by the Free Software Foundation in version 2.2 of Bison. */ +/* DO NOT RELY ON FEATURES THAT ARE NOT DOCUMENTED in the manual, + especially those whose name start with YY_ or yy_. They are + private implementation details that can be changed or removed. */ + #ifndef YY_NCG_NCGEN_TAB_H_INCLUDED # define YY_NCG_NCGEN_TAB_H_INCLUDED /* Debug traces. */ @@ -40,73 +45,78 @@ extern int ncgdebug; #endif -/* Token type. */ +/* Token kinds. */ #ifndef YYTOKENTYPE # define YYTOKENTYPE enum yytokentype { - NC_UNLIMITED_K = 258, - CHAR_K = 259, - BYTE_K = 260, - SHORT_K = 261, - INT_K = 262, - FLOAT_K = 263, - DOUBLE_K = 264, - UBYTE_K = 265, - USHORT_K = 266, - UINT_K = 267, - INT64_K = 268, - UINT64_K = 269, - STRING_K = 270, - IDENT = 271, - TERMSTRING = 272, - CHAR_CONST = 273, - BYTE_CONST = 274, - SHORT_CONST = 275, - INT_CONST = 276, - INT64_CONST = 277, - UBYTE_CONST = 278, - USHORT_CONST = 279, - UINT_CONST = 280, - UINT64_CONST = 281, - FLOAT_CONST = 282, - DOUBLE_CONST = 283, - DIMENSIONS = 284, - VARIABLES = 285, - NETCDF = 286, - DATA = 287, - TYPES = 288, - COMPOUND = 289, - ENUM = 290, - OPAQUE_ = 291, - OPAQUESTRING = 292, - GROUP = 293, - PATH = 294, - FILLMARKER = 295, - NIL = 296, - _FILLVALUE = 297, - _FORMAT = 298, - _STORAGE = 299, - _CHUNKSIZES = 300, - _DEFLATELEVEL = 301, - _SHUFFLE = 302, - _ENDIANNESS = 303, - _NOFILL = 304, - _FLETCHER32 = 305, - _NCPROPS = 306, - _ISNETCDF4 = 307, - _SUPERBLOCK = 308, - _FILTER = 309, - DATASETID = 310 + YYEMPTY = -2, + YYEOF = 0, /* "end of file" */ + YYerror = 256, /* error */ + YYUNDEF = 257, /* "invalid token" */ + NC_UNLIMITED_K = 258, /* NC_UNLIMITED_K */ + CHAR_K = 259, /* CHAR_K */ + BYTE_K = 260, /* BYTE_K */ + SHORT_K = 261, /* SHORT_K */ + INT_K = 262, /* INT_K */ + FLOAT_K = 263, /* FLOAT_K */ + DOUBLE_K = 264, /* DOUBLE_K */ + UBYTE_K = 265, /* UBYTE_K */ + USHORT_K = 266, /* USHORT_K */ + UINT_K = 267, /* UINT_K */ + INT64_K = 268, /* INT64_K */ + UINT64_K = 269, /* UINT64_K */ + STRING_K = 270, /* STRING_K */ + IDENT = 271, /* IDENT */ + TERMSTRING = 272, /* TERMSTRING */ + CHAR_CONST = 273, /* CHAR_CONST */ + BYTE_CONST = 274, /* BYTE_CONST */ + SHORT_CONST = 275, /* SHORT_CONST */ + INT_CONST = 276, /* INT_CONST */ + INT64_CONST = 277, /* INT64_CONST */ + UBYTE_CONST = 278, /* UBYTE_CONST */ + USHORT_CONST = 279, /* USHORT_CONST */ + UINT_CONST = 280, /* UINT_CONST */ + UINT64_CONST = 281, /* UINT64_CONST */ + FLOAT_CONST = 282, /* FLOAT_CONST */ + DOUBLE_CONST = 283, /* DOUBLE_CONST */ + DIMENSIONS = 284, /* DIMENSIONS */ + VARIABLES = 285, /* VARIABLES */ + NETCDF = 286, /* NETCDF */ + DATA = 287, /* DATA */ + TYPES = 288, /* TYPES */ + COMPOUND = 289, /* COMPOUND */ + ENUM = 290, /* ENUM */ + OPAQUE_ = 291, /* OPAQUE_ */ + OPAQUESTRING = 292, /* OPAQUESTRING */ + GROUP = 293, /* GROUP */ + PATH = 294, /* PATH */ + FILLMARKER = 295, /* FILLMARKER */ + NIL = 296, /* NIL */ + _FILLVALUE = 297, /* _FILLVALUE */ + _FORMAT = 298, /* _FORMAT */ + _STORAGE = 299, /* _STORAGE */ + _CHUNKSIZES = 300, /* _CHUNKSIZES */ + _DEFLATELEVEL = 301, /* _DEFLATELEVEL */ + _SHUFFLE = 302, /* _SHUFFLE */ + _ENDIANNESS = 303, /* _ENDIANNESS */ + _NOFILL = 304, /* _NOFILL */ + _FLETCHER32 = 305, /* _FLETCHER32 */ + _NCPROPS = 306, /* _NCPROPS */ + _ISNETCDF4 = 307, /* _ISNETCDF4 */ + _SUPERBLOCK = 308, /* _SUPERBLOCK */ + _FILTER = 309, /* _FILTER */ + _CODECS = 310, /* _CODECS */ + DATASETID = 311 /* DATASETID */ }; + typedef enum yytokentype yytoken_kind_t; #endif /* Value type. */ #if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED - union YYSTYPE { -#line 153 "ncgen.y" /* yacc.c:1909 */ +#line 154 "ncgen.y" Symbol* sym; unsigned long size; /* allow for zero size to indicate e.g. UNLIMITED*/ @@ -115,9 +125,9 @@ int nctype; /* for tracking attribute list type*/ Datalist* datalist; NCConstant* constant; -#line 119 "ncgeny.h" /* yacc.c:1909 */ -}; +#line 129 "ncgeny.h" +}; typedef union YYSTYPE YYSTYPE; # define YYSTYPE_IS_TRIVIAL 1 # define YYSTYPE_IS_DECLARED 1 diff --git a/ncgen/util.c b/ncgen/util.c index 2a6e3184ea..108fff0440 100644 --- a/ncgen/util.c +++ b/ncgen/util.c @@ -114,6 +114,8 @@ clearSpecialdata(Specialdata* data) } efree(data->_Filters); } + if(data->_Codecs) + efree(data->_Codecs); } void diff --git a/ncgen3/main.c b/ncgen3/main.c index 6f9c5f5df9..6c9a898fc1 100644 --- a/ncgen3/main.c +++ b/ncgen3/main.c @@ -169,7 +169,7 @@ main( strcmp(kind_name, "64-bit offset") == 0) { cmode_modifier |= NC_64BIT_OFFSET; } -#ifdef USE_NETCDF4 +#ifdef USE_HDF5 /* NetCDF-4 HDF5 format*/ else if (strcmp(kind_name, "3") == 0 || strcmp(kind_name, "hdf5") == 0 || diff --git a/ncgen3/ncgeny.c b/ncgen3/ncgeny.c index 1aa5f4e2cd..21e018f130 100644 --- a/ncgen3/ncgeny.c +++ b/ncgen3/ncgeny.c @@ -1,8 +1,9 @@ -/* A Bison parser, made by GNU Bison 3.0.4. */ +/* A Bison parser, made by GNU Bison 3.7.6. */ /* Bison implementation for Yacc-like parsers in C - Copyright (C) 1984, 1989-1990, 2000-2015 Free Software Foundation, Inc. + Copyright (C) 1984, 1989-1990, 2000-2015, 2018-2021 Free Software Foundation, + Inc. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -15,7 +16,7 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License - along with this program. If not, see . */ + along with this program. If not, see . */ /* As a special exception, you may create a larger work that contains part or all of the Bison parser skeleton and distribute that work @@ -33,6 +34,10 @@ /* C LALR(1) parser skeleton written by Richard Stallman, by simplifying the original so-called "semantic" parser. */ +/* DO NOT RELY ON FEATURES THAT ARE NOT DOCUMENTED in the manual, + especially those whose name start with YY_ or yy_. They are + private implementation details that can be changed or removed. */ + /* All symbols defined below should begin with yy or YY, to avoid infringing on user name space. This should be done even for local variables, as they might otherwise be expanded by user macros. @@ -40,11 +45,11 @@ define necessary library symbols; they are noted "INFRINGES ON USER NAME SPACE" below. */ -/* Identify Bison output. */ -#define YYBISON 1 +/* Identify Bison output, and Bison version. */ +#define YYBISON 30706 -/* Bison version. */ -#define YYBISON_VERSION "3.0.4" +/* Bison version string. */ +#define YYBISON_VERSION "3.7.6" /* Skeleton name. */ #define YYSKELETON_NAME "yacc.c" @@ -65,12 +70,11 @@ #define yyerror ncgerror #define yydebug ncgdebug #define yynerrs ncgnerrs - #define yylval ncglval #define yychar ncgchar -/* Copy the first part of user declarations. */ -#line 9 "ncgen.y" /* yacc.c:339 */ +/* First part of user prologue. */ +#line 9 "ncgen.y" #ifdef sccs static char SccsId[] = "$Id: ncgen.y,v 1.34 2010/03/31 18:18:41 dmh Exp $"; @@ -148,108 +152,198 @@ void yyerror(char*); int yyerror(char*); #endif -#line 152 "ncgeny.c" /* yacc.c:339 */ +#line 156 "ncgeny.c" +# ifndef YY_CAST +# ifdef __cplusplus +# define YY_CAST(Type, Val) static_cast (Val) +# define YY_REINTERPRET_CAST(Type, Val) reinterpret_cast (Val) +# else +# define YY_CAST(Type, Val) ((Type) (Val)) +# define YY_REINTERPRET_CAST(Type, Val) ((Type) (Val)) +# endif +# endif # ifndef YY_NULLPTR -# if defined __cplusplus && 201103L <= __cplusplus -# define YY_NULLPTR nullptr +# if defined __cplusplus +# if 201103L <= __cplusplus +# define YY_NULLPTR nullptr +# else +# define YY_NULLPTR 0 +# endif # else -# define YY_NULLPTR 0 +# define YY_NULLPTR ((void*)0) # endif # endif -/* Enabling verbose error messages. */ -#ifdef YYERROR_VERBOSE -# undef YYERROR_VERBOSE -# define YYERROR_VERBOSE 1 -#else -# define YYERROR_VERBOSE 0 -#endif - -/* In a future release of Bison, this section will be replaced - by #include "ncgeny.h". */ -#ifndef YY_NCG_NCGEN_TAB_H_INCLUDED -# define YY_NCG_NCGEN_TAB_H_INCLUDED -/* Debug traces. */ -#ifndef YYDEBUG -# define YYDEBUG 1 -#endif -#if YYDEBUG -extern int ncgdebug; -#endif - -/* Token type. */ -#ifndef YYTOKENTYPE -# define YYTOKENTYPE - enum yytokentype - { - NC_UNLIMITED_K = 258, - BYTE_K = 259, - CHAR_K = 260, - SHORT_K = 261, - INT_K = 262, - FLOAT_K = 263, - DOUBLE_K = 264, - IDENT = 265, - TERMSTRING = 266, - BYTE_CONST = 267, - CHAR_CONST = 268, - SHORT_CONST = 269, - INT_CONST = 270, - FLOAT_CONST = 271, - DOUBLE_CONST = 272, - DIMENSIONS = 273, - VARIABLES = 274, - NETCDF = 275, - DATA = 276, - FILLVALUE = 277 - }; -#endif +#include "ncgeny.h" +/* Symbol kind. */ +enum yysymbol_kind_t +{ + YYSYMBOL_YYEMPTY = -2, + YYSYMBOL_YYEOF = 0, /* "end of file" */ + YYSYMBOL_YYerror = 1, /* error */ + YYSYMBOL_YYUNDEF = 2, /* "invalid token" */ + YYSYMBOL_NC_UNLIMITED_K = 3, /* NC_UNLIMITED_K */ + YYSYMBOL_BYTE_K = 4, /* BYTE_K */ + YYSYMBOL_CHAR_K = 5, /* CHAR_K */ + YYSYMBOL_SHORT_K = 6, /* SHORT_K */ + YYSYMBOL_INT_K = 7, /* INT_K */ + YYSYMBOL_FLOAT_K = 8, /* FLOAT_K */ + YYSYMBOL_DOUBLE_K = 9, /* DOUBLE_K */ + YYSYMBOL_IDENT = 10, /* IDENT */ + YYSYMBOL_TERMSTRING = 11, /* TERMSTRING */ + YYSYMBOL_BYTE_CONST = 12, /* BYTE_CONST */ + YYSYMBOL_CHAR_CONST = 13, /* CHAR_CONST */ + YYSYMBOL_SHORT_CONST = 14, /* SHORT_CONST */ + YYSYMBOL_INT_CONST = 15, /* INT_CONST */ + YYSYMBOL_FLOAT_CONST = 16, /* FLOAT_CONST */ + YYSYMBOL_DOUBLE_CONST = 17, /* DOUBLE_CONST */ + YYSYMBOL_DIMENSIONS = 18, /* DIMENSIONS */ + YYSYMBOL_VARIABLES = 19, /* VARIABLES */ + YYSYMBOL_NETCDF = 20, /* NETCDF */ + YYSYMBOL_DATA = 21, /* DATA */ + YYSYMBOL_FILLVALUE = 22, /* FILLVALUE */ + YYSYMBOL_23_ = 23, /* '{' */ + YYSYMBOL_24_ = 24, /* '}' */ + YYSYMBOL_25_ = 25, /* ';' */ + YYSYMBOL_26_ = 26, /* ',' */ + YYSYMBOL_27_ = 27, /* '=' */ + YYSYMBOL_28_ = 28, /* '(' */ + YYSYMBOL_29_ = 29, /* ')' */ + YYSYMBOL_30_ = 30, /* ':' */ + YYSYMBOL_YYACCEPT = 31, /* $accept */ + YYSYMBOL_ncdesc = 32, /* ncdesc */ + YYSYMBOL_33_1 = 33, /* $@1 */ + YYSYMBOL_34_2 = 34, /* $@2 */ + YYSYMBOL_dimsection = 35, /* dimsection */ + YYSYMBOL_dimdecls = 36, /* dimdecls */ + YYSYMBOL_dimdecline = 37, /* dimdecline */ + YYSYMBOL_dimdecl = 38, /* dimdecl */ + YYSYMBOL_dimd = 39, /* dimd */ + YYSYMBOL_dim = 40, /* dim */ + YYSYMBOL_vasection = 41, /* vasection */ + YYSYMBOL_vadecls = 42, /* vadecls */ + YYSYMBOL_vadecl = 43, /* vadecl */ + YYSYMBOL_gattdecls = 44, /* gattdecls */ + YYSYMBOL_vardecl = 45, /* vardecl */ + YYSYMBOL_type = 46, /* type */ + YYSYMBOL_varlist = 47, /* varlist */ + YYSYMBOL_varspec = 48, /* varspec */ + YYSYMBOL_49_3 = 49, /* $@3 */ + YYSYMBOL_var = 50, /* var */ + YYSYMBOL_dimspec = 51, /* dimspec */ + YYSYMBOL_dimlist = 52, /* dimlist */ + YYSYMBOL_vdim = 53, /* vdim */ + YYSYMBOL_attdecl = 54, /* attdecl */ + YYSYMBOL_55_4 = 55, /* $@4 */ + YYSYMBOL_gattdecl = 56, /* gattdecl */ + YYSYMBOL_57_5 = 57, /* $@5 */ + YYSYMBOL_att = 58, /* att */ + YYSYMBOL_gatt = 59, /* gatt */ + YYSYMBOL_avar = 60, /* avar */ + YYSYMBOL_attr = 61, /* attr */ + YYSYMBOL_attvallist = 62, /* attvallist */ + YYSYMBOL_aconst = 63, /* aconst */ + YYSYMBOL_attconst = 64, /* attconst */ + YYSYMBOL_datasection = 65, /* datasection */ + YYSYMBOL_datadecls = 66, /* datadecls */ + YYSYMBOL_datadecl = 67, /* datadecl */ + YYSYMBOL_68_6 = 68, /* $@6 */ + YYSYMBOL_constlist = 69, /* constlist */ + YYSYMBOL_dconst = 70, /* dconst */ + YYSYMBOL_71_7 = 71, /* $@7 */ + YYSYMBOL_const = 72 /* const */ +}; +typedef enum yysymbol_kind_t yysymbol_kind_t; -/* Value type. */ -#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED -typedef int YYSTYPE; -# define YYSTYPE_IS_TRIVIAL 1 -# define YYSTYPE_IS_DECLARED 1 -#endif -extern YYSTYPE ncglval; -int ncgparse (void); +#ifdef short +# undef short +#endif -#endif /* !YY_NCG_NCGEN_TAB_H_INCLUDED */ +/* On compilers that do not define __PTRDIFF_MAX__ etc., make sure + and (if available) are included + so that the code can choose integer types of a good width. */ -/* Copy the second part of user declarations. */ +#ifndef __PTRDIFF_MAX__ +# include /* INFRINGES ON USER NAME SPACE */ +# if defined __STDC_VERSION__ && 199901 <= __STDC_VERSION__ +# include /* INFRINGES ON USER NAME SPACE */ +# define YY_STDINT_H +# endif +#endif -#line 226 "ncgeny.c" /* yacc.c:358 */ +/* Narrow types that promote to a signed type and that can represent a + signed or unsigned integer of at least N bits. In tables they can + save space and decrease cache pressure. Promoting to a signed type + helps avoid bugs in integer arithmetic. */ -#ifdef short -# undef short +#ifdef __INT_LEAST8_MAX__ +typedef __INT_LEAST8_TYPE__ yytype_int8; +#elif defined YY_STDINT_H +typedef int_least8_t yytype_int8; +#else +typedef signed char yytype_int8; #endif -#ifdef YYTYPE_UINT8 -typedef YYTYPE_UINT8 yytype_uint8; +#ifdef __INT_LEAST16_MAX__ +typedef __INT_LEAST16_TYPE__ yytype_int16; +#elif defined YY_STDINT_H +typedef int_least16_t yytype_int16; #else -typedef unsigned char yytype_uint8; +typedef short yytype_int16; #endif -#ifdef YYTYPE_INT8 -typedef YYTYPE_INT8 yytype_int8; -#else -typedef signed char yytype_int8; +/* Work around bug in HP-UX 11.23, which defines these macros + incorrectly for preprocessor constants. This workaround can likely + be removed in 2023, as HPE has promised support for HP-UX 11.23 + (aka HP-UX 11i v2) only through the end of 2022; see Table 2 of + . */ +#ifdef __hpux +# undef UINT_LEAST8_MAX +# undef UINT_LEAST16_MAX +# define UINT_LEAST8_MAX 255 +# define UINT_LEAST16_MAX 65535 #endif -#ifdef YYTYPE_UINT16 -typedef YYTYPE_UINT16 yytype_uint16; +#if defined __UINT_LEAST8_MAX__ && __UINT_LEAST8_MAX__ <= __INT_MAX__ +typedef __UINT_LEAST8_TYPE__ yytype_uint8; +#elif (!defined __UINT_LEAST8_MAX__ && defined YY_STDINT_H \ + && UINT_LEAST8_MAX <= INT_MAX) +typedef uint_least8_t yytype_uint8; +#elif !defined __UINT_LEAST8_MAX__ && UCHAR_MAX <= INT_MAX +typedef unsigned char yytype_uint8; #else -typedef unsigned short int yytype_uint16; +typedef short yytype_uint8; #endif -#ifdef YYTYPE_INT16 -typedef YYTYPE_INT16 yytype_int16; +#if defined __UINT_LEAST16_MAX__ && __UINT_LEAST16_MAX__ <= __INT_MAX__ +typedef __UINT_LEAST16_TYPE__ yytype_uint16; +#elif (!defined __UINT_LEAST16_MAX__ && defined YY_STDINT_H \ + && UINT_LEAST16_MAX <= INT_MAX) +typedef uint_least16_t yytype_uint16; +#elif !defined __UINT_LEAST16_MAX__ && USHRT_MAX <= INT_MAX +typedef unsigned short yytype_uint16; #else -typedef short int yytype_int16; +typedef int yytype_uint16; +#endif + +#ifndef YYPTRDIFF_T +# if defined __PTRDIFF_TYPE__ && defined __PTRDIFF_MAX__ +# define YYPTRDIFF_T __PTRDIFF_TYPE__ +# define YYPTRDIFF_MAXIMUM __PTRDIFF_MAX__ +# elif defined PTRDIFF_MAX +# ifndef ptrdiff_t +# include /* INFRINGES ON USER NAME SPACE */ +# endif +# define YYPTRDIFF_T ptrdiff_t +# define YYPTRDIFF_MAXIMUM PTRDIFF_MAX +# else +# define YYPTRDIFF_T long +# define YYPTRDIFF_MAXIMUM LONG_MAX +# endif #endif #ifndef YYSIZE_T @@ -257,15 +351,28 @@ typedef short int yytype_int16; # define YYSIZE_T __SIZE_TYPE__ # elif defined size_t # define YYSIZE_T size_t -# elif ! defined YYSIZE_T +# elif defined __STDC_VERSION__ && 199901 <= __STDC_VERSION__ # include /* INFRINGES ON USER NAME SPACE */ # define YYSIZE_T size_t # else -# define YYSIZE_T unsigned int +# define YYSIZE_T unsigned # endif #endif -#define YYSIZE_MAXIMUM ((YYSIZE_T) -1) +#define YYSIZE_MAXIMUM \ + YY_CAST (YYPTRDIFF_T, \ + (YYPTRDIFF_MAXIMUM < YY_CAST (YYSIZE_T, -1) \ + ? YYPTRDIFF_MAXIMUM \ + : YY_CAST (YYSIZE_T, -1))) + +#define YYSIZEOF(X) YY_CAST (YYPTRDIFF_T, sizeof (X)) + + +/* Stored state numbers (used for stacks). */ +typedef yytype_int8 yy_state_t; + +/* State numbers in computations. */ +typedef int yy_state_fast_t; #ifndef YY_ # if defined YYENABLE_NLS && YYENABLE_NLS @@ -279,47 +386,37 @@ typedef short int yytype_int16; # endif #endif -#ifndef YY_ATTRIBUTE -# if (defined __GNUC__ \ - && (2 < __GNUC__ || (__GNUC__ == 2 && 96 <= __GNUC_MINOR__))) \ - || defined __SUNPRO_C && 0x5110 <= __SUNPRO_C -# define YY_ATTRIBUTE(Spec) __attribute__(Spec) -# else -# define YY_ATTRIBUTE(Spec) /* empty */ -# endif -#endif #ifndef YY_ATTRIBUTE_PURE -# define YY_ATTRIBUTE_PURE YY_ATTRIBUTE ((__pure__)) +# if defined __GNUC__ && 2 < __GNUC__ + (96 <= __GNUC_MINOR__) +# define YY_ATTRIBUTE_PURE __attribute__ ((__pure__)) +# else +# define YY_ATTRIBUTE_PURE +# endif #endif #ifndef YY_ATTRIBUTE_UNUSED -# define YY_ATTRIBUTE_UNUSED YY_ATTRIBUTE ((__unused__)) -#endif - -#if !defined _Noreturn \ - && (!defined __STDC_VERSION__ || __STDC_VERSION__ < 201112) -# if defined _MSC_VER && 1200 <= _MSC_VER -# define _Noreturn __declspec (noreturn) +# if defined __GNUC__ && 2 < __GNUC__ + (7 <= __GNUC_MINOR__) +# define YY_ATTRIBUTE_UNUSED __attribute__ ((__unused__)) # else -# define _Noreturn YY_ATTRIBUTE ((__noreturn__)) +# define YY_ATTRIBUTE_UNUSED # endif #endif /* Suppress unused-variable warnings by "using" E. */ #if ! defined lint || defined __GNUC__ -# define YYUSE(E) ((void) (E)) +# define YY_USE(E) ((void) (E)) #else -# define YYUSE(E) /* empty */ +# define YY_USE(E) /* empty */ #endif -#if defined __GNUC__ && 407 <= __GNUC__ * 100 + __GNUC_MINOR__ +#if defined __GNUC__ && ! defined __ICC && 407 <= __GNUC__ * 100 + __GNUC_MINOR__ /* Suppress an incorrect diagnostic about yylval being uninitialized. */ -# define YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN \ - _Pragma ("GCC diagnostic push") \ - _Pragma ("GCC diagnostic ignored \"-Wuninitialized\"")\ +# define YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN \ + _Pragma ("GCC diagnostic push") \ + _Pragma ("GCC diagnostic ignored \"-Wuninitialized\"") \ _Pragma ("GCC diagnostic ignored \"-Wmaybe-uninitialized\"") -# define YY_IGNORE_MAYBE_UNINITIALIZED_END \ +# define YY_IGNORE_MAYBE_UNINITIALIZED_END \ _Pragma ("GCC diagnostic pop") #else # define YY_INITIAL_VALUE(Value) Value @@ -332,8 +429,22 @@ typedef short int yytype_int16; # define YY_INITIAL_VALUE(Value) /* Nothing. */ #endif +#if defined __cplusplus && defined __GNUC__ && ! defined __ICC && 6 <= __GNUC__ +# define YY_IGNORE_USELESS_CAST_BEGIN \ + _Pragma ("GCC diagnostic push") \ + _Pragma ("GCC diagnostic ignored \"-Wuseless-cast\"") +# define YY_IGNORE_USELESS_CAST_END \ + _Pragma ("GCC diagnostic pop") +#endif +#ifndef YY_IGNORE_USELESS_CAST_BEGIN +# define YY_IGNORE_USELESS_CAST_BEGIN +# define YY_IGNORE_USELESS_CAST_END +#endif + -#if ! defined yyoverflow || YYERROR_VERBOSE +#define YY_ASSERT(E) ((void) (0 && (E))) + +#if !defined yyoverflow /* The parser invokes alloca or malloc; define the necessary symbols. */ @@ -398,8 +509,7 @@ void free (void *); /* INFRINGES ON USER NAME SPACE */ # endif # endif # endif -#endif /* ! defined yyoverflow || YYERROR_VERBOSE */ - +#endif /* !defined yyoverflow */ #if (! defined yyoverflow \ && (! defined __cplusplus \ @@ -408,17 +518,17 @@ void free (void *); /* INFRINGES ON USER NAME SPACE */ /* A type that is properly aligned for any stack member. */ union yyalloc { - yytype_int16 yyss_alloc; + yy_state_t yyss_alloc; YYSTYPE yyvs_alloc; }; /* The size of the maximum gap between one aligned stack and the next. */ -# define YYSTACK_GAP_MAXIMUM (sizeof (union yyalloc) - 1) +# define YYSTACK_GAP_MAXIMUM (YYSIZEOF (union yyalloc) - 1) /* The size of an array large to enough to hold all stacks, each with N elements. */ # define YYSTACK_BYTES(N) \ - ((N) * (sizeof (yytype_int16) + sizeof (YYSTYPE)) \ + ((N) * (YYSIZEOF (yy_state_t) + YYSIZEOF (YYSTYPE)) \ + YYSTACK_GAP_MAXIMUM) # define YYCOPY_NEEDED 1 @@ -431,11 +541,11 @@ union yyalloc # define YYSTACK_RELOCATE(Stack_alloc, Stack) \ do \ { \ - YYSIZE_T yynewbytes; \ + YYPTRDIFF_T yynewbytes; \ YYCOPY (&yyptr->Stack_alloc, Stack, yysize); \ Stack = &yyptr->Stack_alloc; \ - yynewbytes = yystacksize * sizeof (*Stack) + YYSTACK_GAP_MAXIMUM; \ - yyptr += yynewbytes / sizeof (*yyptr); \ + yynewbytes = yystacksize * YYSIZEOF (*Stack) + YYSTACK_GAP_MAXIMUM; \ + yyptr += yynewbytes / YYSIZEOF (*yyptr); \ } \ while (0) @@ -447,12 +557,12 @@ union yyalloc # ifndef YYCOPY # if defined __GNUC__ && 1 < __GNUC__ # define YYCOPY(Dst, Src, Count) \ - __builtin_memcpy (Dst, Src, (Count) * sizeof (*(Src))) + __builtin_memcpy (Dst, Src, YY_CAST (YYSIZE_T, (Count)) * sizeof (*(Src))) # else # define YYCOPY(Dst, Src, Count) \ do \ { \ - YYSIZE_T yyi; \ + YYPTRDIFF_T yyi; \ for (yyi = 0; yyi < (Count); yyi++) \ (Dst)[yyi] = (Src)[yyi]; \ } \ @@ -475,17 +585,20 @@ union yyalloc /* YYNSTATES -- Number of states. */ #define YYNSTATES 112 -/* YYTRANSLATE[YYX] -- Symbol number corresponding to YYX as returned - by yylex, with out-of-bounds checking. */ -#define YYUNDEFTOK 2 +/* YYMAXUTOK -- Last valid token kind. */ #define YYMAXUTOK 277 -#define YYTRANSLATE(YYX) \ - ((unsigned int) (YYX) <= YYMAXUTOK ? yytranslate[YYX] : YYUNDEFTOK) + +/* YYTRANSLATE(TOKEN-NUM) -- Symbol number corresponding to TOKEN-NUM + as returned by yylex, with out-of-bounds checking. */ +#define YYTRANSLATE(YYX) \ + (0 <= (YYX) && (YYX) <= YYMAXUTOK \ + ? YY_CAST (yysymbol_kind_t, yytranslate[YYX]) \ + : YYSYMBOL_YYUNDEF) /* YYTRANSLATE[TOKEN-NUM] -- Symbol number corresponding to TOKEN-NUM - as returned by yylex, without out-of-bounds checking. */ -static const yytype_uint8 yytranslate[] = + as returned by yylex. */ +static const yytype_int8 yytranslate[] = { 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, @@ -519,7 +632,7 @@ static const yytype_uint8 yytranslate[] = #if YYDEBUG /* YYRLINE[YYN] -- Source line where rule number YYN was defined. */ -static const yytype_uint16 yyrline[] = +static const yytype_int16 yyrline[] = { 0, 119, 119, 122, 117, 135, 136, 138, 139, 141, 142, 144, 150, 161, 169, 186, 188, 189, 190, 192, @@ -532,45 +645,59 @@ static const yytype_uint16 yyrline[] = }; #endif -#if YYDEBUG || YYERROR_VERBOSE || 0 +/** Accessing symbol of state STATE. */ +#define YY_ACCESSING_SYMBOL(State) YY_CAST (yysymbol_kind_t, yystos[State]) + +#if YYDEBUG || 0 +/* The user-facing name of the symbol whose (internal) number is + YYSYMBOL. No bounds checking. */ +static const char *yysymbol_name (yysymbol_kind_t yysymbol) YY_ATTRIBUTE_UNUSED; + /* YYTNAME[SYMBOL-NUM] -- String name of the symbol SYMBOL-NUM. First, the terminals, then, starting at YYNTOKENS, nonterminals. */ static const char *const yytname[] = { - "$end", "error", "$undefined", "NC_UNLIMITED_K", "BYTE_K", "CHAR_K", - "SHORT_K", "INT_K", "FLOAT_K", "DOUBLE_K", "IDENT", "TERMSTRING", - "BYTE_CONST", "CHAR_CONST", "SHORT_CONST", "INT_CONST", "FLOAT_CONST", - "DOUBLE_CONST", "DIMENSIONS", "VARIABLES", "NETCDF", "DATA", "FILLVALUE", - "'{'", "'}'", "';'", "','", "'='", "'('", "')'", "':'", "$accept", - "ncdesc", "$@1", "$@2", "dimsection", "dimdecls", "dimdecline", - "dimdecl", "dimd", "dim", "vasection", "vadecls", "vadecl", "gattdecls", - "vardecl", "type", "varlist", "varspec", "$@3", "var", "dimspec", - "dimlist", "vdim", "attdecl", "$@4", "gattdecl", "$@5", "att", "gatt", - "avar", "attr", "attvallist", "aconst", "attconst", "datasection", - "datadecls", "datadecl", "$@6", "constlist", "dconst", "$@7", "const", YY_NULLPTR + "\"end of file\"", "error", "\"invalid token\"", "NC_UNLIMITED_K", + "BYTE_K", "CHAR_K", "SHORT_K", "INT_K", "FLOAT_K", "DOUBLE_K", "IDENT", + "TERMSTRING", "BYTE_CONST", "CHAR_CONST", "SHORT_CONST", "INT_CONST", + "FLOAT_CONST", "DOUBLE_CONST", "DIMENSIONS", "VARIABLES", "NETCDF", + "DATA", "FILLVALUE", "'{'", "'}'", "';'", "','", "'='", "'('", "')'", + "':'", "$accept", "ncdesc", "$@1", "$@2", "dimsection", "dimdecls", + "dimdecline", "dimdecl", "dimd", "dim", "vasection", "vadecls", "vadecl", + "gattdecls", "vardecl", "type", "varlist", "varspec", "$@3", "var", + "dimspec", "dimlist", "vdim", "attdecl", "$@4", "gattdecl", "$@5", "att", + "gatt", "avar", "attr", "attvallist", "aconst", "attconst", + "datasection", "datadecls", "datadecl", "$@6", "constlist", "dconst", + "$@7", "const", YY_NULLPTR }; + +static const char * +yysymbol_name (yysymbol_kind_t yysymbol) +{ + return yytname[yysymbol]; +} #endif -# ifdef YYPRINT +#ifdef YYPRINT /* YYTOKNUM[NUM] -- (External) token number corresponding to the (internal) symbol number NUM (which must be that of a token). */ -static const yytype_uint16 yytoknum[] = +static const yytype_int16 yytoknum[] = { 0, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 123, 125, 59, 44, 61, 40, 41, 58 }; -# endif +#endif -#define YYPACT_NINF -73 +#define YYPACT_NINF (-73) -#define yypact_value_is_default(Yystate) \ - (!!((Yystate) == (-73))) +#define yypact_value_is_default(Yyn) \ + ((Yyn) == YYPACT_NINF) -#define YYTABLE_NINF -1 +#define YYTABLE_NINF (-1) -#define yytable_value_is_error(Yytable_value) \ +#define yytable_value_is_error(Yyn) \ 0 /* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing @@ -594,7 +721,7 @@ static const yytype_int8 yypact[] = /* YYDEFACT[STATE-NUM] -- Default reduction number in state STATE-NUM. Performed when YYTABLE does not specify something else to do. Zero means the default is an error. */ -static const yytype_uint8 yydefact[] = +static const yytype_int8 yydefact[] = { 0, 0, 0, 2, 1, 5, 0, 16, 15, 6, 0, 9, 0, 14, 0, 0, 3, 18, 0, 45, @@ -623,7 +750,7 @@ static const yytype_int8 yypgoto[] = /* YYDEFGOTO[NTERM-NUM]. */ static const yytype_int8 yydefgoto[] = { - -1, 2, 5, 42, 7, 9, 10, 11, 12, 13, + 0, 2, 5, 42, 7, 9, 10, 11, 12, 13, 16, 31, 32, 17, 33, 34, 53, 54, 64, 35, 83, 90, 91, 36, 56, 37, 45, 38, 19, 39, 41, 78, 79, 80, 59, 68, 69, 85, 97, 98, @@ -633,7 +760,7 @@ static const yytype_int8 yydefgoto[] = /* YYTABLE[YYPACT[STATE-NUM]] -- What to do in state STATE-NUM. If positive, shift that token. If negative, reduce the rule whose number is the opposite. If YYTABLE_NINF, syntax error. */ -static const yytype_uint8 yytable[] = +static const yytype_int8 yytable[] = { 55, 24, 25, 26, 27, 28, 29, 30, 48, 95, 89, 18, 96, 102, 103, 104, 105, 106, 107, 108, @@ -659,7 +786,7 @@ static const yytype_int8 yycheck[] = /* YYSTOS[STATE-NUM] -- The (internal number of the) accessing symbol of state STATE-NUM. */ -static const yytype_uint8 yystos[] = +static const yytype_int8 yystos[] = { 0, 20, 32, 23, 0, 33, 18, 35, 10, 36, 37, 38, 39, 40, 19, 30, 41, 44, 56, 59, @@ -676,7 +803,7 @@ static const yytype_uint8 yystos[] = }; /* YYR1[YYN] -- Symbol number of symbol that rule YYN derives. */ -static const yytype_uint8 yyr1[] = +static const yytype_int8 yyr1[] = { 0, 31, 33, 34, 32, 35, 35, 36, 36, 37, 37, 38, 38, 38, 39, 40, 41, 41, 41, 42, @@ -689,7 +816,7 @@ static const yytype_uint8 yyr1[] = }; /* YYR2[YYN] -- Number of symbols on the right hand side of rule YYN. */ -static const yytype_uint8 yyr2[] = +static const yytype_int8 yyr2[] = { 0, 2, 0, 0, 8, 0, 2, 2, 3, 1, 3, 3, 3, 3, 1, 1, 0, 2, 1, 2, @@ -702,10 +829,10 @@ static const yytype_uint8 yyr2[] = }; +enum { YYENOMEM = -2 }; + #define yyerrok (yyerrstatus = 0) #define yyclearin (yychar = YYEMPTY) -#define YYEMPTY (-2) -#define YYEOF 0 #define YYACCEPT goto yyacceptlab #define YYABORT goto yyabortlab @@ -714,27 +841,26 @@ static const yytype_uint8 yyr2[] = #define YYRECOVERING() (!!yyerrstatus) -#define YYBACKUP(Token, Value) \ -do \ - if (yychar == YYEMPTY) \ - { \ - yychar = (Token); \ - yylval = (Value); \ - YYPOPSTACK (yylen); \ - yystate = *yyssp; \ - goto yybackup; \ - } \ - else \ - { \ - yyerror (YY_("syntax error: cannot back up")); \ - YYERROR; \ - } \ -while (0) - -/* Error token number */ -#define YYTERROR 1 -#define YYERRCODE 256 - +#define YYBACKUP(Token, Value) \ + do \ + if (yychar == YYEMPTY) \ + { \ + yychar = (Token); \ + yylval = (Value); \ + YYPOPSTACK (yylen); \ + yystate = *yyssp; \ + goto yybackup; \ + } \ + else \ + { \ + yyerror (YY_("syntax error: cannot back up")); \ + YYERROR; \ + } \ + while (0) + +/* Backward compatibility with an undocumented macro. + Use YYerror or YYUNDEF. */ +#define YYERRCODE YYUNDEF /* Enable debugging if requested. */ @@ -752,54 +878,58 @@ do { \ } while (0) /* This macro is provided for backward compatibility. */ -#ifndef YY_LOCATION_PRINT -# define YY_LOCATION_PRINT(File, Loc) ((void) 0) -#endif +# ifndef YY_LOCATION_PRINT +# define YY_LOCATION_PRINT(File, Loc) ((void) 0) +# endif -# define YY_SYMBOL_PRINT(Title, Type, Value, Location) \ +# define YY_SYMBOL_PRINT(Title, Kind, Value, Location) \ do { \ if (yydebug) \ { \ YYFPRINTF (stderr, "%s ", Title); \ yy_symbol_print (stderr, \ - Type, Value); \ + Kind, Value); \ YYFPRINTF (stderr, "\n"); \ } \ } while (0) -/*----------------------------------------. -| Print this symbol's value on YYOUTPUT. | -`----------------------------------------*/ +/*-----------------------------------. +| Print this symbol's value on YYO. | +`-----------------------------------*/ static void -yy_symbol_value_print (FILE *yyoutput, int yytype, YYSTYPE const * const yyvaluep) +yy_symbol_value_print (FILE *yyo, + yysymbol_kind_t yykind, YYSTYPE const * const yyvaluep) { - FILE *yyo = yyoutput; - YYUSE (yyo); + FILE *yyoutput = yyo; + YY_USE (yyoutput); if (!yyvaluep) return; # ifdef YYPRINT - if (yytype < YYNTOKENS) - YYPRINT (yyoutput, yytoknum[yytype], *yyvaluep); + if (yykind < YYNTOKENS) + YYPRINT (yyo, yytoknum[yykind], *yyvaluep); # endif - YYUSE (yytype); + YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN + YY_USE (yykind); + YY_IGNORE_MAYBE_UNINITIALIZED_END } -/*--------------------------------. -| Print this symbol on YYOUTPUT. | -`--------------------------------*/ +/*---------------------------. +| Print this symbol on YYO. | +`---------------------------*/ static void -yy_symbol_print (FILE *yyoutput, int yytype, YYSTYPE const * const yyvaluep) +yy_symbol_print (FILE *yyo, + yysymbol_kind_t yykind, YYSTYPE const * const yyvaluep) { - YYFPRINTF (yyoutput, "%s %s (", - yytype < YYNTOKENS ? "token" : "nterm", yytname[yytype]); + YYFPRINTF (yyo, "%s %s (", + yykind < YYNTOKENS ? "token" : "nterm", yysymbol_name (yykind)); - yy_symbol_value_print (yyoutput, yytype, yyvaluep); - YYFPRINTF (yyoutput, ")"); + yy_symbol_value_print (yyo, yykind, yyvaluep); + YYFPRINTF (yyo, ")"); } /*------------------------------------------------------------------. @@ -808,7 +938,7 @@ yy_symbol_print (FILE *yyoutput, int yytype, YYSTYPE const * const yyvaluep) `------------------------------------------------------------------*/ static void -yy_stack_print (yytype_int16 *yybottom, yytype_int16 *yytop) +yy_stack_print (yy_state_t *yybottom, yy_state_t *yytop) { YYFPRINTF (stderr, "Stack now"); for (; yybottom <= yytop; yybottom++) @@ -831,21 +961,21 @@ do { \ `------------------------------------------------*/ static void -yy_reduce_print (yytype_int16 *yyssp, YYSTYPE *yyvsp, int yyrule) +yy_reduce_print (yy_state_t *yyssp, YYSTYPE *yyvsp, + int yyrule) { - unsigned long int yylno = yyrline[yyrule]; + int yylno = yyrline[yyrule]; int yynrhs = yyr2[yyrule]; int yyi; - YYFPRINTF (stderr, "Reducing stack by rule %d (line %lu):\n", + YYFPRINTF (stderr, "Reducing stack by rule %d (line %d):\n", yyrule - 1, yylno); /* The symbols being reduced. */ for (yyi = 0; yyi < yynrhs; yyi++) { YYFPRINTF (stderr, " $%d = ", yyi + 1); yy_symbol_print (stderr, - yystos[yyssp[yyi + 1 - yynrhs]], - &(yyvsp[(yyi + 1) - (yynrhs)]) - ); + YY_ACCESSING_SYMBOL (+yyssp[yyi + 1 - yynrhs]), + &yyvsp[(yyi + 1) - (yynrhs)]); YYFPRINTF (stderr, "\n"); } } @@ -860,8 +990,8 @@ do { \ multiple parsers can coexist. */ int yydebug; #else /* !YYDEBUG */ -# define YYDPRINTF(Args) -# define YY_SYMBOL_PRINT(Title, Type, Value, Location) +# define YYDPRINTF(Args) ((void) 0) +# define YY_SYMBOL_PRINT(Title, Kind, Value, Location) # define YY_STACK_PRINT(Bottom, Top) # define YY_REDUCE_PRINT(Rule) #endif /* !YYDEBUG */ @@ -884,249 +1014,30 @@ int yydebug; #endif -#if YYERROR_VERBOSE - -# ifndef yystrlen -# if defined __GLIBC__ && defined _STRING_H -# define yystrlen strlen -# else -/* Return the length of YYSTR. */ -static YYSIZE_T -yystrlen (const char *yystr) -{ - YYSIZE_T yylen; - for (yylen = 0; yystr[yylen]; yylen++) - continue; - return yylen; -} -# endif -# endif - -# ifndef yystpcpy -# if defined __GLIBC__ && defined _STRING_H && defined _GNU_SOURCE -# define yystpcpy stpcpy -# else -/* Copy YYSRC to YYDEST, returning the address of the terminating '\0' in - YYDEST. */ -static char * -yystpcpy (char *yydest, const char *yysrc) -{ - char *yyd = yydest; - const char *yys = yysrc; - - while ((*yyd++ = *yys++) != '\0') - continue; - return yyd - 1; -} -# endif -# endif - -# ifndef yytnamerr -/* Copy to YYRES the contents of YYSTR after stripping away unnecessary - quotes and backslashes, so that it's suitable for yyerror. The - heuristic is that double-quoting is unnecessary unless the string - contains an apostrophe, a comma, or backslash (other than - backslash-backslash). YYSTR is taken from yytname. If YYRES is - null, do not copy; instead, return the length of what the result - would have been. */ -static YYSIZE_T -yytnamerr (char *yyres, const char *yystr) -{ - if (*yystr == '"') - { - YYSIZE_T yyn = 0; - char const *yyp = yystr; - - for (;;) - switch (*++yyp) - { - case '\'': - case ',': - goto do_not_strip_quotes; - - case '\\': - if (*++yyp != '\\') - goto do_not_strip_quotes; - /* Fall through. */ - default: - if (yyres) - yyres[yyn] = *yyp; - yyn++; - break; - - case '"': - if (yyres) - yyres[yyn] = '\0'; - return yyn; - } - do_not_strip_quotes: ; - } - if (! yyres) - return yystrlen (yystr); - return yystpcpy (yyres, yystr) - yyres; -} -# endif - -/* Copy into *YYMSG, which is of size *YYMSG_ALLOC, an error message - about the unexpected token YYTOKEN for the state stack whose top is - YYSSP. - - Return 0 if *YYMSG was successfully written. Return 1 if *YYMSG is - not large enough to hold the message. In that case, also set - *YYMSG_ALLOC to the required number of bytes. Return 2 if the - required number of bytes is too large to store. */ -static int -yysyntax_error (YYSIZE_T *yymsg_alloc, char **yymsg, - yytype_int16 *yyssp, int yytoken) -{ - YYSIZE_T yysize0 = yytnamerr (YY_NULLPTR, yytname[yytoken]); - YYSIZE_T yysize = yysize0; - enum { YYERROR_VERBOSE_ARGS_MAXIMUM = 5 }; - /* Internationalized format string. */ - const char *yyformat = YY_NULLPTR; - /* Arguments of yyformat. */ - char const *yyarg[YYERROR_VERBOSE_ARGS_MAXIMUM]; - /* Number of reported tokens (one for the "unexpected", one per - "expected"). */ - int yycount = 0; - - /* There are many possibilities here to consider: - - If this state is a consistent state with a default action, then - the only way this function was invoked is if the default action - is an error action. In that case, don't check for expected - tokens because there are none. - - The only way there can be no lookahead present (in yychar) is if - this state is a consistent state with a default action. Thus, - detecting the absence of a lookahead is sufficient to determine - that there is no unexpected or expected token to report. In that - case, just report a simple "syntax error". - - Don't assume there isn't a lookahead just because this state is a - consistent state with a default action. There might have been a - previous inconsistent state, consistent state with a non-default - action, or user semantic action that manipulated yychar. - - Of course, the expected token list depends on states to have - correct lookahead information, and it depends on the parser not - to perform extra reductions after fetching a lookahead from the - scanner and before detecting a syntax error. Thus, state merging - (from LALR or IELR) and default reductions corrupt the expected - token list. However, the list is correct for canonical LR with - one exception: it will still contain any token that will not be - accepted due to an error action in a later state. - */ - if (yytoken != YYEMPTY) - { - int yyn = yypact[*yyssp]; - yyarg[yycount++] = yytname[yytoken]; - if (!yypact_value_is_default (yyn)) - { - /* Start YYX at -YYN if negative to avoid negative indexes in - YYCHECK. In other words, skip the first -YYN actions for - this state because they are default actions. */ - int yyxbegin = yyn < 0 ? -yyn : 0; - /* Stay within bounds of both yycheck and yytname. */ - int yychecklim = YYLAST - yyn + 1; - int yyxend = yychecklim < YYNTOKENS ? yychecklim : YYNTOKENS; - int yyx; - - for (yyx = yyxbegin; yyx < yyxend; ++yyx) - if (yycheck[yyx + yyn] == yyx && yyx != YYTERROR - && !yytable_value_is_error (yytable[yyx + yyn])) - { - if (yycount == YYERROR_VERBOSE_ARGS_MAXIMUM) - { - yycount = 1; - yysize = yysize0; - break; - } - yyarg[yycount++] = yytname[yyx]; - { - YYSIZE_T yysize1 = yysize + yytnamerr (YY_NULLPTR, yytname[yyx]); - if (! (yysize <= yysize1 - && yysize1 <= YYSTACK_ALLOC_MAXIMUM)) - return 2; - yysize = yysize1; - } - } - } - } - - switch (yycount) - { -# define YYCASE_(N, S) \ - case N: \ - yyformat = S; \ - break - YYCASE_(0, YY_("syntax error")); - YYCASE_(1, YY_("syntax error, unexpected %s")); - YYCASE_(2, YY_("syntax error, unexpected %s, expecting %s")); - YYCASE_(3, YY_("syntax error, unexpected %s, expecting %s or %s")); - YYCASE_(4, YY_("syntax error, unexpected %s, expecting %s or %s or %s")); - YYCASE_(5, YY_("syntax error, unexpected %s, expecting %s or %s or %s or %s")); -# undef YYCASE_ - } - - { - YYSIZE_T yysize1 = yysize + yystrlen (yyformat); - if (! (yysize <= yysize1 && yysize1 <= YYSTACK_ALLOC_MAXIMUM)) - return 2; - yysize = yysize1; - } - - if (*yymsg_alloc < yysize) - { - *yymsg_alloc = 2 * yysize; - if (! (yysize <= *yymsg_alloc - && *yymsg_alloc <= YYSTACK_ALLOC_MAXIMUM)) - *yymsg_alloc = YYSTACK_ALLOC_MAXIMUM; - return 1; - } - - /* Avoid sprintf, as that infringes on the user's name space. - Don't have undefined behavior even if the translation - produced a string with the wrong number of "%s"s. */ - { - char *yyp = *yymsg; - int yyi = 0; - while ((*yyp = *yyformat) != '\0') - if (*yyp == '%' && yyformat[1] == 's' && yyi < yycount) - { - yyp += yytnamerr (yyp, yyarg[yyi++]); - yyformat += 2; - } - else - { - yyp++; - yyformat++; - } - } - return 0; -} -#endif /* YYERROR_VERBOSE */ /*-----------------------------------------------. | Release the memory associated to this symbol. | `-----------------------------------------------*/ static void -yydestruct (const char *yymsg, int yytype, YYSTYPE *yyvaluep) +yydestruct (const char *yymsg, + yysymbol_kind_t yykind, YYSTYPE *yyvaluep) { - YYUSE (yyvaluep); + YY_USE (yyvaluep); if (!yymsg) yymsg = "Deleting"; - YY_SYMBOL_PRINT (yymsg, yytype, yyvaluep, yylocationp); + YY_SYMBOL_PRINT (yymsg, yykind, yyvaluep, yylocationp); YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN - YYUSE (yytype); + YY_USE (yykind); YY_IGNORE_MAYBE_UNINITIALIZED_END } - - -/* The lookahead symbol. */ +/* Lookahead token kind. */ int yychar; /* The semantic value of the lookahead symbol. */ @@ -1135,6 +1046,8 @@ YYSTYPE yylval; int yynerrs; + + /*----------. | yyparse. | `----------*/ @@ -1142,43 +1055,36 @@ int yynerrs; int yyparse (void) { - int yystate; + yy_state_fast_t yystate = 0; /* Number of tokens to shift before error messages enabled. */ - int yyerrstatus; + int yyerrstatus = 0; - /* The stacks and their tools: - 'yyss': related to states. - 'yyvs': related to semantic values. - - Refer to the stacks through separate pointers, to allow yyoverflow + /* Refer to the stacks through separate pointers, to allow yyoverflow to reallocate them elsewhere. */ - /* The state stack. */ - yytype_int16 yyssa[YYINITDEPTH]; - yytype_int16 *yyss; - yytype_int16 *yyssp; + /* Their size. */ + YYPTRDIFF_T yystacksize = YYINITDEPTH; - /* The semantic value stack. */ - YYSTYPE yyvsa[YYINITDEPTH]; - YYSTYPE *yyvs; - YYSTYPE *yyvsp; + /* The state stack: array, bottom, top. */ + yy_state_t yyssa[YYINITDEPTH]; + yy_state_t *yyss = yyssa; + yy_state_t *yyssp = yyss; - YYSIZE_T yystacksize; + /* The semantic value stack: array, bottom, top. */ + YYSTYPE yyvsa[YYINITDEPTH]; + YYSTYPE *yyvs = yyvsa; + YYSTYPE *yyvsp = yyvs; int yyn; + /* The return value of yyparse. */ int yyresult; - /* Lookahead token as an internal (translated) token number. */ - int yytoken = 0; + /* Lookahead symbol kind. */ + yysymbol_kind_t yytoken = YYSYMBOL_YYEMPTY; /* The variables used to return semantic value and location from the action routines. */ YYSTYPE yyval; -#if YYERROR_VERBOSE - /* Buffer for error messages, and its allocated size. */ - char yymsgbuf[128]; - char *yymsg = yymsgbuf; - YYSIZE_T yymsg_alloc = sizeof yymsgbuf; -#endif + #define YYPOPSTACK(N) (yyvsp -= (N), yyssp -= (N)) @@ -1186,58 +1092,60 @@ yyparse (void) Keep to zero when no symbol should be popped. */ int yylen = 0; - yyssp = yyss = yyssa; - yyvsp = yyvs = yyvsa; - yystacksize = YYINITDEPTH; - YYDPRINTF ((stderr, "Starting parse\n")); - yystate = 0; - yyerrstatus = 0; - yynerrs = 0; yychar = YYEMPTY; /* Cause a token to be read. */ goto yysetstate; + /*------------------------------------------------------------. -| yynewstate -- Push a new state, which is found in yystate. | +| yynewstate -- push a new state, which is found in yystate. | `------------------------------------------------------------*/ - yynewstate: +yynewstate: /* In all cases, when you get here, the value and location stacks have just been pushed. So pushing a state here evens the stacks. */ yyssp++; - yysetstate: - *yyssp = yystate; + +/*--------------------------------------------------------------------. +| yysetstate -- set current state (the top of the stack) to yystate. | +`--------------------------------------------------------------------*/ +yysetstate: + YYDPRINTF ((stderr, "Entering state %d\n", yystate)); + YY_ASSERT (0 <= yystate && yystate < YYNSTATES); + YY_IGNORE_USELESS_CAST_BEGIN + *yyssp = YY_CAST (yy_state_t, yystate); + YY_IGNORE_USELESS_CAST_END + YY_STACK_PRINT (yyss, yyssp); if (yyss + yystacksize - 1 <= yyssp) +#if !defined yyoverflow && !defined YYSTACK_RELOCATE + goto yyexhaustedlab; +#else { /* Get the current used size of the three stacks, in elements. */ - YYSIZE_T yysize = yyssp - yyss + 1; + YYPTRDIFF_T yysize = yyssp - yyss + 1; -#ifdef yyoverflow +# if defined yyoverflow { /* Give user a chance to reallocate the stack. Use copies of these so that the &'s don't force the real ones into memory. */ + yy_state_t *yyss1 = yyss; YYSTYPE *yyvs1 = yyvs; - yytype_int16 *yyss1 = yyss; /* Each stack pointer address is followed by the size of the data in use in that stack, in bytes. This used to be a conditional around just the two extra args, but that might be undefined if yyoverflow is a macro. */ yyoverflow (YY_("memory exhausted"), - &yyss1, yysize * sizeof (*yyssp), - &yyvs1, yysize * sizeof (*yyvsp), + &yyss1, yysize * YYSIZEOF (*yyssp), + &yyvs1, yysize * YYSIZEOF (*yyvsp), &yystacksize); - yyss = yyss1; yyvs = yyvs1; } -#else /* no yyoverflow */ -# ifndef YYSTACK_RELOCATE - goto yyexhaustedlab; -# else +# else /* defined YYSTACK_RELOCATE */ /* Extend the stack our own way. */ if (YYMAXDEPTH <= yystacksize) goto yyexhaustedlab; @@ -1246,9 +1154,10 @@ yyparse (void) yystacksize = YYMAXDEPTH; { - yytype_int16 *yyss1 = yyss; + yy_state_t *yyss1 = yyss; union yyalloc *yyptr = - (union yyalloc *) YYSTACK_ALLOC (YYSTACK_BYTES (yystacksize)); + YY_CAST (union yyalloc *, + YYSTACK_ALLOC (YY_CAST (YYSIZE_T, YYSTACK_BYTES (yystacksize)))); if (! yyptr) goto yyexhaustedlab; YYSTACK_RELOCATE (yyss_alloc, yyss); @@ -1258,30 +1167,30 @@ yyparse (void) YYSTACK_FREE (yyss1); } # endif -#endif /* no yyoverflow */ yyssp = yyss + yysize - 1; yyvsp = yyvs + yysize - 1; - YYDPRINTF ((stderr, "Stack size increased to %lu\n", - (unsigned long int) yystacksize)); + YY_IGNORE_USELESS_CAST_BEGIN + YYDPRINTF ((stderr, "Stack size increased to %ld\n", + YY_CAST (long, yystacksize))); + YY_IGNORE_USELESS_CAST_END if (yyss + yystacksize - 1 <= yyssp) YYABORT; } - - YYDPRINTF ((stderr, "Entering state %d\n", yystate)); +#endif /* !defined yyoverflow && !defined YYSTACK_RELOCATE */ if (yystate == YYFINAL) YYACCEPT; goto yybackup; + /*-----------. | yybackup. | `-----------*/ yybackup: - /* Do appropriate processing given the current state. Read a lookahead token if we need one and don't already have one. */ @@ -1292,18 +1201,29 @@ yyparse (void) /* Not known => get a lookahead token if don't already have one. */ - /* YYCHAR is either YYEMPTY or YYEOF or a valid lookahead symbol. */ + /* YYCHAR is either empty, or end-of-input, or a valid lookahead. */ if (yychar == YYEMPTY) { - YYDPRINTF ((stderr, "Reading a token: ")); + YYDPRINTF ((stderr, "Reading a token\n")); yychar = yylex (); } if (yychar <= YYEOF) { - yychar = yytoken = YYEOF; + yychar = YYEOF; + yytoken = YYSYMBOL_YYEOF; YYDPRINTF ((stderr, "Now at end of input.\n")); } + else if (yychar == YYerror) + { + /* The scanner already issued an error message, process directly + to error recovery. But do not keep the error token as + lookahead, it is too special and may lead us to an endless + loop in error recovery. */ + yychar = YYUNDEF; + yytoken = YYSYMBOL_YYerror; + goto yyerrlab1; + } else { yytoken = YYTRANSLATE (yychar); @@ -1331,15 +1251,13 @@ yyparse (void) /* Shift the lookahead token. */ YY_SYMBOL_PRINT ("Shifting", yytoken, &yylval, &yylloc); - - /* Discard the shifted token. */ - yychar = YYEMPTY; - yystate = yyn; YY_IGNORE_MAYBE_UNINITIALIZED_BEGIN *++yyvsp = yylval; YY_IGNORE_MAYBE_UNINITIALIZED_END + /* Discard the shifted token. */ + yychar = YYEMPTY; goto yynewstate; @@ -1354,7 +1272,7 @@ yyparse (void) /*-----------------------------. -| yyreduce -- Do a reduction. | +| yyreduce -- do a reduction. | `-----------------------------*/ yyreduce: /* yyn is the number of a rule to reduce with. */ @@ -1374,45 +1292,45 @@ yyparse (void) YY_REDUCE_PRINT (yyn); switch (yyn) { - case 2: -#line 119 "ncgen.y" /* yacc.c:1646 */ - { init_netcdf(); } -#line 1381 "ncgeny.c" /* yacc.c:1646 */ + case 2: /* $@1: %empty */ +#line 119 "ncgen.y" + { init_netcdf(); } +#line 1299 "ncgeny.c" break; - case 3: -#line 122 "ncgen.y" /* yacc.c:1646 */ - { + case 3: /* $@2: %empty */ +#line 122 "ncgen.y" + { if (derror_count == 0) define_netcdf(netcdfname); if (derror_count > 0) exit(6); } -#line 1392 "ncgeny.c" /* yacc.c:1646 */ +#line 1310 "ncgeny.c" break; - case 4: -#line 130 "ncgen.y" /* yacc.c:1646 */ - { + case 4: /* ncdesc: NETCDF '{' $@1 dimsection vasection $@2 datasection '}' */ +#line 130 "ncgen.y" + { if (derror_count == 0) close_netcdf(); } -#line 1401 "ncgeny.c" /* yacc.c:1646 */ +#line 1319 "ncgeny.c" break; - case 11: -#line 145 "ncgen.y" /* yacc.c:1646 */ - { if (int_val <= 0) + case 11: /* dimdecl: dimd '=' INT_CONST */ +#line 145 "ncgen.y" + { if (int_val <= 0) derror("dimension length must be positive"); dims[ndims].size = int_val; ndims++; } -#line 1411 "ncgeny.c" /* yacc.c:1646 */ +#line 1329 "ncgeny.c" break; - case 12: -#line 151 "ncgen.y" /* yacc.c:1646 */ - { /* for rare case where 2^31 < dimsize < 2^32 */ + case 12: /* dimdecl: dimd '=' DOUBLE_CONST */ +#line 151 "ncgen.y" + { /* for rare case where 2^31 < dimsize < 2^32 */ if (double_val <= 0) derror("dimension length must be positive"); if (double_val > 4294967295.0) @@ -1422,79 +1340,79 @@ yyparse (void) dims[ndims].size = (size_t) double_val; ndims++; } -#line 1426 "ncgeny.c" /* yacc.c:1646 */ +#line 1344 "ncgeny.c" break; - case 13: -#line 162 "ncgen.y" /* yacc.c:1646 */ - { if (rec_dim != -1) + case 13: /* dimdecl: dimd '=' NC_UNLIMITED_K */ +#line 162 "ncgen.y" + { if (rec_dim != -1) derror("only one NC_UNLIMITED dimension allowed"); rec_dim = ndims; /* the unlimited (record) dimension */ dims[ndims].size = NC_UNLIMITED; ndims++; } -#line 1437 "ncgeny.c" /* yacc.c:1646 */ +#line 1355 "ncgeny.c" break; - case 14: -#line 170 "ncgen.y" /* yacc.c:1646 */ - { - if ((yyvsp[0])->is_dim == 1) { + case 14: /* dimd: dim */ +#line 170 "ncgen.y" + { + if (yyvsp[0]->is_dim == 1) { derror( "duplicate dimension declaration for %s", - (yyvsp[0])->name); + yyvsp[0]->name); } - (yyvsp[0])->is_dim = 1; - (yyvsp[0])->dnum = ndims; + yyvsp[0]->is_dim = 1; + yyvsp[0]->dnum = ndims; /* make sure dims array will hold dimensions */ grow_darray(ndims, /* must hold ndims+1 dims */ &dims); /* grow as needed */ - dims[ndims].name = (char *) emalloc(strlen((yyvsp[0])->name)+1); - (void) strcpy(dims[ndims].name, (yyvsp[0])->name); + dims[ndims].name = (char *) emalloc(strlen(yyvsp[0]->name)+1); + (void) strcpy(dims[ndims].name, yyvsp[0]->name); /* name for use in generated Fortran and C variables */ - dims[ndims].lname = decodify((yyvsp[0])->name); + dims[ndims].lname = decodify(yyvsp[0]->name); } -#line 1457 "ncgeny.c" /* yacc.c:1646 */ +#line 1375 "ncgeny.c" break; - case 27: -#line 202 "ncgen.y" /* yacc.c:1646 */ - { type_code = NC_BYTE; } -#line 1463 "ncgeny.c" /* yacc.c:1646 */ + case 27: /* type: BYTE_K */ +#line 202 "ncgen.y" + { type_code = NC_BYTE; } +#line 1381 "ncgeny.c" break; - case 28: -#line 203 "ncgen.y" /* yacc.c:1646 */ - { type_code = NC_CHAR; } -#line 1469 "ncgeny.c" /* yacc.c:1646 */ + case 28: /* type: CHAR_K */ +#line 203 "ncgen.y" + { type_code = NC_CHAR; } +#line 1387 "ncgeny.c" break; - case 29: -#line 204 "ncgen.y" /* yacc.c:1646 */ - { type_code = NC_SHORT; } -#line 1475 "ncgeny.c" /* yacc.c:1646 */ + case 29: /* type: SHORT_K */ +#line 204 "ncgen.y" + { type_code = NC_SHORT; } +#line 1393 "ncgeny.c" break; - case 30: -#line 205 "ncgen.y" /* yacc.c:1646 */ - { type_code = NC_INT; } -#line 1481 "ncgeny.c" /* yacc.c:1646 */ + case 30: /* type: INT_K */ +#line 205 "ncgen.y" + { type_code = NC_INT; } +#line 1399 "ncgeny.c" break; - case 31: -#line 206 "ncgen.y" /* yacc.c:1646 */ - { type_code = NC_FLOAT; } -#line 1487 "ncgeny.c" /* yacc.c:1646 */ + case 31: /* type: FLOAT_K */ +#line 206 "ncgen.y" + { type_code = NC_FLOAT; } +#line 1405 "ncgeny.c" break; - case 32: -#line 207 "ncgen.y" /* yacc.c:1646 */ - { type_code = NC_DOUBLE; } -#line 1493 "ncgeny.c" /* yacc.c:1646 */ + case 32: /* type: DOUBLE_K */ +#line 207 "ncgen.y" + { type_code = NC_DOUBLE; } +#line 1411 "ncgeny.c" break; - case 35: -#line 213 "ncgen.y" /* yacc.c:1646 */ - { + case 35: /* $@3: %empty */ +#line 213 "ncgen.y" + { static struct vars dummyvar; dummyvar.name = "dummy"; @@ -1506,49 +1424,49 @@ yyparse (void) nvdims = 0; /* make sure variable not re-declared */ - if ((yyvsp[0])->is_var == 1) { + if (yyvsp[0]->is_var == 1) { derror( "duplicate variable declaration for %s", - (yyvsp[0])->name); + yyvsp[0]->name); } - (yyvsp[0])->is_var = 1; - (yyvsp[0])->vnum = nvars; + yyvsp[0]->is_var = 1; + yyvsp[0]->vnum = nvars; /* make sure vars array will hold variables */ grow_varray(nvars, /* must hold nvars+1 vars */ &vars); /* grow as needed */ vars[nvars] = dummyvar; /* to make Purify happy */ - vars[nvars].name = (char *) emalloc(strlen((yyvsp[0])->name)+1); - (void) strcpy(vars[nvars].name, (yyvsp[0])->name); + vars[nvars].name = (char *) emalloc(strlen(yyvsp[0]->name)+1); + (void) strcpy(vars[nvars].name, yyvsp[0]->name); /* name for use in generated Fortran and C variables */ - vars[nvars].lname = decodify((yyvsp[0])->name); + vars[nvars].lname = decodify(yyvsp[0]->name); vars[nvars].type = type_code; /* set default fill value. You can override this with * the variable attribute "_FillValue". */ nc_getfill(type_code, &vars[nvars].fill_value); vars[nvars].has_data = 0; /* has no data (yet) */ } -#line 1530 "ncgeny.c" /* yacc.c:1646 */ +#line 1448 "ncgeny.c" break; - case 36: -#line 246 "ncgen.y" /* yacc.c:1646 */ - { + case 36: /* varspec: var $@3 dimspec */ +#line 246 "ncgen.y" + { vars[nvars].ndims = nvdims; nvars++; } -#line 1539 "ncgeny.c" /* yacc.c:1646 */ +#line 1457 "ncgeny.c" break; - case 42: -#line 260 "ncgen.y" /* yacc.c:1646 */ - { + case 42: /* vdim: dim */ +#line 260 "ncgen.y" + { if (nvdims >= NC_MAX_VAR_DIMS) { derror("%s has too many dimensions",vars[nvars].name); } - if ((yyvsp[0])->is_dim == 1) - dimnum = (yyvsp[0])->dnum; + if (yyvsp[0]->is_dim == 1) + dimnum = yyvsp[0]->dnum; else { derror( "%s is not declared as a dimension", - (yyvsp[0])->name); + yyvsp[0]->name); dimnum = ndims; } if (rec_dim != -1 && dimnum == rec_dim && nvdims != 0) { @@ -1559,100 +1477,100 @@ yyparse (void) vars[nvars].dims[nvdims] = dimnum; nvdims++; } -#line 1563 "ncgeny.c" /* yacc.c:1646 */ +#line 1481 "ncgeny.c" break; - case 43: -#line 281 "ncgen.y" /* yacc.c:1646 */ - { + case 43: /* $@4: %empty */ +#line 281 "ncgen.y" + { defatt(); } -#line 1571 "ncgeny.c" /* yacc.c:1646 */ +#line 1489 "ncgeny.c" break; - case 44: -#line 285 "ncgen.y" /* yacc.c:1646 */ - { + case 44: /* attdecl: att $@4 '=' attvallist */ +#line 285 "ncgen.y" + { equalatt(); } -#line 1579 "ncgeny.c" /* yacc.c:1646 */ +#line 1497 "ncgeny.c" break; - case 45: -#line 290 "ncgen.y" /* yacc.c:1646 */ - { + case 45: /* $@5: %empty */ +#line 290 "ncgen.y" + { defatt(); } -#line 1587 "ncgeny.c" /* yacc.c:1646 */ +#line 1505 "ncgeny.c" break; - case 46: -#line 294 "ncgen.y" /* yacc.c:1646 */ - { + case 46: /* gattdecl: gatt $@5 '=' attvallist */ +#line 294 "ncgen.y" + { equalatt(); } -#line 1595 "ncgeny.c" /* yacc.c:1646 */ +#line 1513 "ncgeny.c" break; - case 48: -#line 302 "ncgen.y" /* yacc.c:1646 */ - { + case 48: /* gatt: ':' attr */ +#line 302 "ncgen.y" + { varnum = NC_GLOBAL; /* handle of "global" attribute */ } -#line 1603 "ncgeny.c" /* yacc.c:1646 */ +#line 1521 "ncgeny.c" break; - case 49: -#line 308 "ncgen.y" /* yacc.c:1646 */ - { if ((yyvsp[0])->is_var == 1) - varnum = (yyvsp[0])->vnum; + case 49: /* avar: var */ +#line 308 "ncgen.y" + { if (yyvsp[0]->is_var == 1) + varnum = yyvsp[0]->vnum; else { derror("%s not declared as a variable, fatal error", - (yyvsp[0])->name); + yyvsp[0]->name); YYABORT; } } -#line 1616 "ncgeny.c" /* yacc.c:1646 */ +#line 1534 "ncgeny.c" break; - case 50: -#line 318 "ncgen.y" /* yacc.c:1646 */ - { + case 50: /* attr: IDENT */ +#line 318 "ncgen.y" + { /* make sure atts array will hold attributes */ grow_aarray(natts, /* must hold natts+1 atts */ &atts); /* grow as needed */ - atts[natts].name = (char *) emalloc(strlen((yyvsp[0])->name)+1); - (void) strcpy(atts[natts].name,(yyvsp[0])->name); + atts[natts].name = (char *) emalloc(strlen(yyvsp[0]->name)+1); + (void) strcpy(atts[natts].name,yyvsp[0]->name); /* name for use in generated Fortran and C variables */ - atts[natts].lname = decodify((yyvsp[0])->name); + atts[natts].lname = decodify(yyvsp[0]->name); } -#line 1630 "ncgeny.c" /* yacc.c:1646 */ +#line 1548 "ncgeny.c" break; - case 53: -#line 332 "ncgen.y" /* yacc.c:1646 */ - { + case 53: /* aconst: attconst */ +#line 332 "ncgen.y" + { if (valtype == NC_UNSPECIFIED) valtype = atype_code; if (valtype != atype_code) derror("values for attribute must be all of same type"); } -#line 1641 "ncgeny.c" /* yacc.c:1646 */ +#line 1559 "ncgeny.c" break; - case 54: -#line 341 "ncgen.y" /* yacc.c:1646 */ - { + case 54: /* attconst: CHAR_CONST */ +#line 341 "ncgen.y" + { atype_code = NC_CHAR; *char_valp++ = char_val; valnum++; } -#line 1651 "ncgeny.c" /* yacc.c:1646 */ +#line 1569 "ncgeny.c" break; - case 55: -#line 347 "ncgen.y" /* yacc.c:1646 */ - { + case 55: /* attconst: TERMSTRING */ +#line 347 "ncgen.y" + { atype_code = NC_CHAR; { /* don't null-terminate attribute strings */ @@ -1664,62 +1582,62 @@ yyparse (void) char_valp += len; } } -#line 1668 "ncgeny.c" /* yacc.c:1646 */ +#line 1586 "ncgeny.c" break; - case 56: -#line 360 "ncgen.y" /* yacc.c:1646 */ - { + case 56: /* attconst: BYTE_CONST */ +#line 360 "ncgen.y" + { atype_code = NC_BYTE; *byte_valp++ = byte_val; valnum++; } -#line 1678 "ncgeny.c" /* yacc.c:1646 */ +#line 1596 "ncgeny.c" break; - case 57: -#line 366 "ncgen.y" /* yacc.c:1646 */ - { + case 57: /* attconst: SHORT_CONST */ +#line 366 "ncgen.y" + { atype_code = NC_SHORT; *short_valp++ = short_val; valnum++; } -#line 1688 "ncgeny.c" /* yacc.c:1646 */ +#line 1606 "ncgeny.c" break; - case 58: -#line 372 "ncgen.y" /* yacc.c:1646 */ - { + case 58: /* attconst: INT_CONST */ +#line 372 "ncgen.y" + { atype_code = NC_INT; *int_valp++ = int_val; valnum++; } -#line 1698 "ncgeny.c" /* yacc.c:1646 */ +#line 1616 "ncgeny.c" break; - case 59: -#line 378 "ncgen.y" /* yacc.c:1646 */ - { + case 59: /* attconst: FLOAT_CONST */ +#line 378 "ncgen.y" + { atype_code = NC_FLOAT; *float_valp++ = float_val; valnum++; } -#line 1708 "ncgeny.c" /* yacc.c:1646 */ +#line 1626 "ncgeny.c" break; - case 60: -#line 384 "ncgen.y" /* yacc.c:1646 */ - { + case 60: /* attconst: DOUBLE_CONST */ +#line 384 "ncgen.y" + { atype_code = NC_DOUBLE; *double_valp++ = double_val; valnum++; } -#line 1718 "ncgeny.c" /* yacc.c:1646 */ +#line 1636 "ncgeny.c" break; - case 66: -#line 400 "ncgen.y" /* yacc.c:1646 */ - { + case 66: /* $@6: %empty */ +#line 400 "ncgen.y" + { valtype = vars[varnum].type; /* variable type */ valnum = 0; /* values accumulated for variable */ vars[varnum].has_data = 1; @@ -1769,12 +1687,12 @@ yyparse (void) default: break; } } -#line 1773 "ncgeny.c" /* yacc.c:1646 */ +#line 1691 "ncgeny.c" break; - case 67: -#line 451 "ncgen.y" /* yacc.c:1646 */ - { + case 67: /* datadecl: avar $@6 '=' constlist */ +#line 451 "ncgen.y" + { if (valnum < var_len) { /* leftovers */ nc_fill(valtype, var_len - valnum, @@ -1788,12 +1706,12 @@ yyparse (void) put_variable(rec_start); free ((char *) rec_start); } -#line 1792 "ncgeny.c" /* yacc.c:1646 */ +#line 1710 "ncgeny.c" break; - case 70: -#line 470 "ncgen.y" /* yacc.c:1646 */ - { + case 70: /* $@7: %empty */ +#line 470 "ncgen.y" + { if(valnum >= var_len) { if (vars[varnum].dims[0] != rec_dim) { /* not recvar */ derror("too many values for this variable, %d >= %d", @@ -1817,12 +1735,12 @@ yyparse (void) } not_a_string = 1; } -#line 1821 "ncgeny.c" /* yacc.c:1646 */ +#line 1739 "ncgeny.c" break; - case 71: -#line 495 "ncgen.y" /* yacc.c:1646 */ - { + case 71: /* dconst: $@7 const */ +#line 495 "ncgen.y" + { if (not_a_string) { switch (valtype) { case NC_CHAR: @@ -1847,12 +1765,12 @@ yyparse (void) } } } -#line 1851 "ncgeny.c" /* yacc.c:1646 */ +#line 1769 "ncgeny.c" break; - case 72: -#line 523 "ncgen.y" /* yacc.c:1646 */ - { + case 72: /* const: CHAR_CONST */ +#line 523 "ncgen.y" + { atype_code = NC_CHAR; switch (valtype) { case NC_CHAR: @@ -1877,12 +1795,12 @@ yyparse (void) } valnum++; } -#line 1881 "ncgeny.c" /* yacc.c:1646 */ +#line 1799 "ncgeny.c" break; - case 73: -#line 549 "ncgen.y" /* yacc.c:1646 */ - { + case 73: /* const: TERMSTRING */ +#line 549 "ncgen.y" + { not_a_string = 0; atype_code = NC_CHAR; { @@ -1936,12 +1854,12 @@ yyparse (void) } } } -#line 1940 "ncgeny.c" /* yacc.c:1646 */ +#line 1858 "ncgeny.c" break; - case 74: -#line 604 "ncgen.y" /* yacc.c:1646 */ - { + case 74: /* const: BYTE_CONST */ +#line 604 "ncgen.y" + { atype_code = NC_BYTE; switch (valtype) { case NC_CHAR: @@ -1966,12 +1884,12 @@ yyparse (void) } valnum++; } -#line 1970 "ncgeny.c" /* yacc.c:1646 */ +#line 1888 "ncgeny.c" break; - case 75: -#line 630 "ncgen.y" /* yacc.c:1646 */ - { + case 75: /* const: SHORT_CONST */ +#line 630 "ncgen.y" + { atype_code = NC_SHORT; switch (valtype) { case NC_CHAR: @@ -1996,12 +1914,12 @@ yyparse (void) } valnum++; } -#line 2000 "ncgeny.c" /* yacc.c:1646 */ +#line 1918 "ncgeny.c" break; - case 76: -#line 656 "ncgen.y" /* yacc.c:1646 */ - { + case 76: /* const: INT_CONST */ +#line 656 "ncgen.y" + { atype_code = NC_INT; switch (valtype) { case NC_CHAR: @@ -2026,12 +1944,12 @@ yyparse (void) } valnum++; } -#line 2030 "ncgeny.c" /* yacc.c:1646 */ +#line 1948 "ncgeny.c" break; - case 77: -#line 682 "ncgen.y" /* yacc.c:1646 */ - { + case 77: /* const: FLOAT_CONST */ +#line 682 "ncgen.y" + { atype_code = NC_FLOAT; switch (valtype) { case NC_CHAR: @@ -2056,12 +1974,12 @@ yyparse (void) } valnum++; } -#line 2060 "ncgeny.c" /* yacc.c:1646 */ +#line 1978 "ncgeny.c" break; - case 78: -#line 708 "ncgen.y" /* yacc.c:1646 */ - { + case 78: /* const: DOUBLE_CONST */ +#line 708 "ncgen.y" + { atype_code = NC_DOUBLE; switch (valtype) { case NC_CHAR: @@ -2089,12 +2007,12 @@ yyparse (void) } valnum++; } -#line 2093 "ncgeny.c" /* yacc.c:1646 */ +#line 2011 "ncgeny.c" break; - case 79: -#line 737 "ncgen.y" /* yacc.c:1646 */ - { + case 79: /* const: FILLVALUE */ +#line 737 "ncgen.y" + { /* store fill_value */ switch (valtype) { case NC_CHAR: @@ -2125,11 +2043,12 @@ yyparse (void) } valnum++; } -#line 2129 "ncgeny.c" /* yacc.c:1646 */ +#line 2047 "ncgeny.c" break; -#line 2133 "ncgeny.c" /* yacc.c:1646 */ +#line 2051 "ncgeny.c" + default: break; } /* User semantic actions sometimes alter yychar, and that requires @@ -2143,25 +2062,23 @@ yyparse (void) case of YYERROR or YYBACKUP, subsequent parser actions might lead to an incorrect destructor call or verbose syntax error message before the lookahead is translated. */ - YY_SYMBOL_PRINT ("-> $$ =", yyr1[yyn], &yyval, &yyloc); + YY_SYMBOL_PRINT ("-> $$ =", YY_CAST (yysymbol_kind_t, yyr1[yyn]), &yyval, &yyloc); YYPOPSTACK (yylen); yylen = 0; - YY_STACK_PRINT (yyss, yyssp); *++yyvsp = yyval; /* Now 'shift' the result of the reduction. Determine what state that goes to, based on the state we popped back to and the rule number reduced by. */ - - yyn = yyr1[yyn]; - - yystate = yypgoto[yyn - YYNTOKENS] + *yyssp; - if (0 <= yystate && yystate <= YYLAST && yycheck[yystate] == *yyssp) - yystate = yytable[yystate]; - else - yystate = yydefgoto[yyn - YYNTOKENS]; + { + const int yylhs = yyr1[yyn] - YYNTOKENS; + const int yyi = yypgoto[yylhs] + *yyssp; + yystate = (0 <= yyi && yyi <= YYLAST && yycheck[yyi] == *yyssp + ? yytable[yyi] + : yydefgoto[yylhs]); + } goto yynewstate; @@ -2172,50 +2089,14 @@ yyparse (void) yyerrlab: /* Make sure we have latest lookahead translation. See comments at user semantic actions for why this is necessary. */ - yytoken = yychar == YYEMPTY ? YYEMPTY : YYTRANSLATE (yychar); - + yytoken = yychar == YYEMPTY ? YYSYMBOL_YYEMPTY : YYTRANSLATE (yychar); /* If not already recovering from an error, report this error. */ if (!yyerrstatus) { ++yynerrs; -#if ! YYERROR_VERBOSE yyerror (YY_("syntax error")); -#else -# define YYSYNTAX_ERROR yysyntax_error (&yymsg_alloc, &yymsg, \ - yyssp, yytoken) - { - char const *yymsgp = YY_("syntax error"); - int yysyntax_error_status; - yysyntax_error_status = YYSYNTAX_ERROR; - if (yysyntax_error_status == 0) - yymsgp = yymsg; - else if (yysyntax_error_status == 1) - { - if (yymsg != yymsgbuf) - YYSTACK_FREE (yymsg); - yymsg = (char *) YYSTACK_ALLOC (yymsg_alloc); - if (!yymsg) - { - yymsg = yymsgbuf; - yymsg_alloc = sizeof yymsgbuf; - yysyntax_error_status = 2; - } - else - { - yysyntax_error_status = YYSYNTAX_ERROR; - yymsgp = yymsg; - } - } - yyerror (yymsgp); - if (yysyntax_error_status == 2) - goto yyexhaustedlab; - } -# undef YYSYNTAX_ERROR -#endif } - - if (yyerrstatus == 3) { /* If just tried and failed to reuse lookahead token after an @@ -2244,12 +2125,10 @@ yyparse (void) | yyerrorlab -- error raised explicitly by YYERROR. | `---------------------------------------------------*/ yyerrorlab: - - /* Pacify compilers like GCC when the user code never invokes - YYERROR and the label yyerrorlab therefore never appears in user - code. */ - if (/*CONSTCOND*/ 0) - goto yyerrorlab; + /* Pacify compilers when the user code never invokes YYERROR and the + label yyerrorlab therefore never appears in user code. */ + if (0) + YYERROR; /* Do not reclaim the symbols of the rule whose action triggered this YYERROR. */ @@ -2266,13 +2145,14 @@ yyparse (void) yyerrlab1: yyerrstatus = 3; /* Each real token shifted decrements this. */ + /* Pop stack until we find a state that shifts the error token. */ for (;;) { yyn = yypact[yystate]; if (!yypact_value_is_default (yyn)) { - yyn += YYTERROR; - if (0 <= yyn && yyn <= YYLAST && yycheck[yyn] == YYTERROR) + yyn += YYSYMBOL_YYerror; + if (0 <= yyn && yyn <= YYLAST && yycheck[yyn] == YYSYMBOL_YYerror) { yyn = yytable[yyn]; if (0 < yyn) @@ -2286,7 +2166,7 @@ yyparse (void) yydestruct ("Error: popping", - yystos[yystate], yyvsp); + YY_ACCESSING_SYMBOL (yystate), yyvsp); YYPOPSTACK (1); yystate = *yyssp; YY_STACK_PRINT (yyss, yyssp); @@ -2298,7 +2178,7 @@ yyparse (void) /* Shift the error token. */ - YY_SYMBOL_PRINT ("Shifting", yystos[yyn], yyvsp, yylsp); + YY_SYMBOL_PRINT ("Shifting", YY_ACCESSING_SYMBOL (yyn), yyvsp, yylsp); yystate = yyn; goto yynewstate; @@ -2311,6 +2191,7 @@ yyparse (void) yyresult = 0; goto yyreturn; + /*-----------------------------------. | yyabortlab -- YYABORT comes here. | `-----------------------------------*/ @@ -2318,16 +2199,21 @@ yyparse (void) yyresult = 1; goto yyreturn; -#if !defined yyoverflow || YYERROR_VERBOSE + +#if !defined yyoverflow /*-------------------------------------------------. | yyexhaustedlab -- memory exhaustion comes here. | `-------------------------------------------------*/ yyexhaustedlab: yyerror (YY_("memory exhausted")); yyresult = 2; - /* Fall through. */ + goto yyreturn; #endif + +/*-------------------------------------------------------. +| yyreturn -- parsing is finished, clean up and return. | +`-------------------------------------------------------*/ yyreturn: if (yychar != YYEMPTY) { @@ -2344,20 +2230,18 @@ yyparse (void) while (yyssp != yyss) { yydestruct ("Cleanup: popping", - yystos[*yyssp], yyvsp); + YY_ACCESSING_SYMBOL (+*yyssp), yyvsp); YYPOPSTACK (1); } #ifndef yyoverflow if (yyss != yyssa) YYSTACK_FREE (yyss); #endif -#if YYERROR_VERBOSE - if (yymsg != yymsgbuf) - YYSTACK_FREE (yymsg); -#endif + return yyresult; } -#line 772 "ncgen.y" /* yacc.c:1906 */ + +#line 772 "ncgen.y" /* HELPER PROGRAMS */ diff --git a/ncgen3/ncgeny.h b/ncgen3/ncgeny.h index b35939ff5a..4bdf4bfc34 100644 --- a/ncgen3/ncgeny.h +++ b/ncgen3/ncgeny.h @@ -1,8 +1,9 @@ -/* A Bison parser, made by GNU Bison 3.0.4. */ +/* A Bison parser, made by GNU Bison 3.7.6. */ /* Bison interface for Yacc-like parsers in C - Copyright (C) 1984, 1989-1990, 2000-2015 Free Software Foundation, Inc. + Copyright (C) 1984, 1989-1990, 2000-2015, 2018-2021 Free Software Foundation, + Inc. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -15,7 +16,7 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License - along with this program. If not, see . */ + along with this program. If not, see . */ /* As a special exception, you may create a larger work that contains part or all of the Bison parser skeleton and distribute that work @@ -30,6 +31,10 @@ This special exception was added by the Free Software Foundation in version 2.2 of Bison. */ +/* DO NOT RELY ON FEATURES THAT ARE NOT DOCUMENTED in the manual, + especially those whose name start with YY_ or yy_. They are + private implementation details that can be changed or removed. */ + #ifndef YY_NCG_NCGEN_TAB_H_INCLUDED # define YY_NCG_NCGEN_TAB_H_INCLUDED /* Debug traces. */ @@ -40,32 +45,37 @@ extern int ncgdebug; #endif -/* Token type. */ +/* Token kinds. */ #ifndef YYTOKENTYPE # define YYTOKENTYPE enum yytokentype { - NC_UNLIMITED_K = 258, - BYTE_K = 259, - CHAR_K = 260, - SHORT_K = 261, - INT_K = 262, - FLOAT_K = 263, - DOUBLE_K = 264, - IDENT = 265, - TERMSTRING = 266, - BYTE_CONST = 267, - CHAR_CONST = 268, - SHORT_CONST = 269, - INT_CONST = 270, - FLOAT_CONST = 271, - DOUBLE_CONST = 272, - DIMENSIONS = 273, - VARIABLES = 274, - NETCDF = 275, - DATA = 276, - FILLVALUE = 277 + YYEMPTY = -2, + YYEOF = 0, /* "end of file" */ + YYerror = 256, /* error */ + YYUNDEF = 257, /* "invalid token" */ + NC_UNLIMITED_K = 258, /* NC_UNLIMITED_K */ + BYTE_K = 259, /* BYTE_K */ + CHAR_K = 260, /* CHAR_K */ + SHORT_K = 261, /* SHORT_K */ + INT_K = 262, /* INT_K */ + FLOAT_K = 263, /* FLOAT_K */ + DOUBLE_K = 264, /* DOUBLE_K */ + IDENT = 265, /* IDENT */ + TERMSTRING = 266, /* TERMSTRING */ + BYTE_CONST = 267, /* BYTE_CONST */ + CHAR_CONST = 268, /* CHAR_CONST */ + SHORT_CONST = 269, /* SHORT_CONST */ + INT_CONST = 270, /* INT_CONST */ + FLOAT_CONST = 271, /* FLOAT_CONST */ + DOUBLE_CONST = 272, /* DOUBLE_CONST */ + DIMENSIONS = 273, /* DIMENSIONS */ + VARIABLES = 274, /* VARIABLES */ + NETCDF = 275, /* NETCDF */ + DATA = 276, /* DATA */ + FILLVALUE = 277 /* FILLVALUE */ }; + typedef enum yytokentype yytoken_kind_t; #endif /* Value type. */ diff --git a/nczarr_test/CMakeLists.txt b/nczarr_test/CMakeLists.txt index a96cdec548..3d8662d7fa 100644 --- a/nczarr_test/CMakeLists.txt +++ b/nczarr_test/CMakeLists.txt @@ -96,6 +96,17 @@ IF(ENABLE_TESTS) add_sh_test(nczarr_test run_s3_cleanup) ENDIF() + IF(ENABLE_FILTER_TESTING) + build_bin_test(testfilter) + build_bin_test(testfilter_misc) + build_bin_test(testfilter_multi) + build_bin_test(testfilter_order) + build_bin_test(testfilter_repeat) + IF(ENABLE_BLOSC) + ADD_SH_TEST(nczarr_test run_specific_filters) + ENDIF() + ADD_SH_TEST(nczarr_test run_filter) + ENDIF(ENABLE_FILTER_TESTING) if(ENABLE_NCZARR_ZIP) add_sh_test(nczarr_test run_newformat) endif() diff --git a/nczarr_test/Makefile.am b/nczarr_test/Makefile.am index aa28221516..95cbfeccfe 100644 --- a/nczarr_test/Makefile.am +++ b/nczarr_test/Makefile.am @@ -62,12 +62,13 @@ TESTS += run_nczarr_fill.sh endif +if BUILD_UTILITIES + if ENABLE_NCZARR_ZIP TESTS += run_newformat.sh endif if BUILD_BENCHMARKS -if BUILD_UTILITIES UTILSRC = bm_utils.c timer_utils.c tst_utils.c bm_utils.h bm_timer.h @@ -82,6 +83,20 @@ endif endif # BUILD_BENCHMARKS +if ENABLE_FILTER_TESTING +# Do simple filter functionality tests for nczarr +check_PROGRAMS += tst_nczfilter +TESTS += tst_nczfilter + +# Echo filter tests from nc_test4 +check_PROGRAMS += testfilter testfilter_misc testfilter_order testfilter_repeat testfilter_multi +TESTS += run_filter.sh +if ENABLE_BLOSC +TESTS += run_specific_filters.sh +endif + +endif #ENABLE_FILTER_TESTING + endif #BUILD_UTILITIES # These programs are used by the test cases @@ -109,7 +124,9 @@ ncdumpchunks_SOURCES = ncdumpchunks.c EXTRA_DIST = CMakeLists.txt \ run_ut_map.sh run_ut_mapapi.sh run_ut_misc.sh run_ut_chunk.sh run_ncgen4.sh \ run_nccopyz.sh run_fillonlyz.sh run_chunkcases.sh test_nczarr.sh run_perf_chunks1.sh run_s3_cleanup.sh \ -run_purezarr.sh run_interop.sh run_misc.sh run_newformat.sh run_nczarr_fill.sh +run_purezarr.sh run_interop.sh run_misc.sh \ +run_filter.sh run_specific_filters.sh \ +run_newformat.sh run_nczarr_fill.sh EXTRA_DIST += \ ref_ut_map_create.cdl ref_ut_map_writedata.cdl ref_ut_map_writemeta2.cdl ref_ut_map_writemeta.cdl \ @@ -125,11 +142,15 @@ ref_rem.cdl ref_rem.dmp ref_ndims.cdl ref_ndims.dmp \ ref_misc1.cdl ref_misc1.dmp ref_misc2.cdl \ ref_avail1.cdl ref_avail1.dmp ref_avail1.txt \ ref_xarray.cdl ref_purezarr.cdl ref_purezarr_base.cdl ref_nczarr2zarr.cdl \ -ref_power_901_constants.zip ref_power_901_constants.cdl ref_quotes.zip ref_quotes.cdl \ -ref_oldformat.cdl ref_oldformat.zip ref_newformatpure.cdl \ +ref_bzip2.cdl ref_filtered.cdl ref_multi.cdl \ +ref_any.cdl ref_oldformat.cdl ref_oldformat.zip ref_newformatpure.cdl \ +ref_quotes.zip ref_quotes.cdl \ ref_groups.h5 ref_byte.zarr.zip ref_byte_fill_value_null.zarr.zip \ ref_groups_regular.cdl ref_byte.cdl ref_byte_fill_value_null.cdl +# Interoperability files +EXTRA_DIST += ref_power_901_constants.zip ref_power_901_constants.cdl ref_quotes.zip ref_quotes.cdl + CLEANFILES = ut_*.txt ut*.cdl tmp*.nc tmp*.cdl tmp*.txt tmp*.dmp tmp*.zip tmp*.nc # Remove directories @@ -137,4 +158,8 @@ clean-local: rm -fr tmp*.file results.file results.s3 results.zip rm -fr power_901_constants.file rm -fr rcmiscdir - rm -rf ref_power_901_constants.file + +DISTCLEANFILES = findplugin.sh + +# If valgrind is present, add valgrind targets. +@VALGRIND_CHECK_RULES@ diff --git a/nczarr_test/ref_any.cdl b/nczarr_test/ref_any.cdl new file mode 100644 index 0000000000..bbbc30e860 --- /dev/null +++ b/nczarr_test/ref_any.cdl @@ -0,0 +1,57 @@ +netcdf ref_any { +dimensions: + dim0 = 4 ; + dim1 = 4 ; + dim2 = 4 ; +variables: + int ivar(dim0, dim1, dim2) ; + ivar:_FillValue = -2147483647 ; + ivar:_Storage = @chunked@ ; + ivar:_ChunkSizes = 4, 4, 4 ; + ivar:_Filter = @IH5@ ; + ivar:_Codecs = @ICX@ ; + float fvar(dim0, dim1, dim2) ; + fvar:_FillValue = 9.96921e+36f ; + fvar:_Storage = @chunked@ ; + fvar:_ChunkSizes = 4, 4, 4 ; + fvar:_Filter = @FH5@ ; + fvar:_Codecs = @FCX@ ; + +data: + + ivar = + 0, 1, 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, + 12, 13, 14, 15, + 16, 17, 18, 19, + 20, 21, 22, 23, + 24, 25, 26, 27, + 28, 29, 30, 31, + 32, 33, 34, 35, + 36, 37, 38, 39, + 40, 41, 42, 43, + 44, 45, 46, 47, + 48, 49, 50, 51, + 52, 53, 54, 55, + 56, 57, 58, 59, + 60, 61, 62, 63 ; + + fvar = + 0.5, 1.5, 2.5, 3.5, + 4.5, 5.5, 6.5, 7.5, + 8.5, 9.5, 10.5, 11.5, + 12.5, 13.5, 14.5, 15.5, + 16.5, 17.5, 18.5, 19.5, + 20.5, 21.5, 22.5, 23.5, + 24.5, 25.5, 26.5, 27.5, + 28.5, 29.5, 30.5, 31.5, + 32.5, 33.5, 34.5, 35.5, + 36.5, 37.5, 38.5, 39.5, + 40.5, 41.5, 42.5, 43.5, + 44.5, 45.5, 46.5, 47.5, + 48.5, 49.5, 50.5, 51.5, + 52.5, 53.5, 54.5, 55.5, + 56.5, 57.5, 58.5, 59.5, + 60.5, 61.5, 62.5, 63.5 ; +} diff --git a/nczarr_test/ref_blosc_zmap.txt b/nczarr_test/ref_blosc_zmap.txt new file mode 100644 index 0000000000..fd48ac290b --- /dev/null +++ b/nczarr_test/ref_blosc_zmap.txt @@ -0,0 +1,7 @@ +[0] /.nczarr : (44) |{"zarr_format": 2,"nczarr_version": "1.0.0"}| +[1] /.nczattrs : (33) |{"types": {"_NCProperties": "2"}}| +[2] /.nczgroup : (80) |{"dims": {"dim0": 4,"dim1": 4,"dim2": 4,"dim3": 4},"vars": ["var"],"groups": []}| +[3] /.zattrs : (68) |{"_NCProperties": "version=2,netcdf=4.8.1-development,nczarr=1.0.0"}| +[4] /.zgroup : (18) |{"zarr_format": 2}| +[6] /var/.nczarray : (67) |{"dimrefs": ["/dim0","/dim1","/dim2","/dim3"],"storage": "chunked"}| +[7] /var/.zarray : (172) |{"zarr_format": 2,"shape": [4,4,4,4],"dtype": " $2 +} + +# Function to extract _Filter attribute from a file +# These attributes might be platform dependent +getfilterattr() { +sed -e '/var.*:_Filter/p' -ed <$1 >$2 +} + +# Function to extract _Codecs attribute from a file +# These attributes might be platform dependent +getcodecsattr() { +sed -e '/var.*:_Codecs/p' -ed <$1 >$2 +} + +trimleft() { +sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2 +} + +# Locate the plugin path and the library names; argument order is critical +# Find bzip2 and capture +findplugin h5bzip2 +BZIP2LIB="${HDF5_PLUGIN_LIB}" +BZIP2PATH="${HDF5_PLUGIN_PATH}/${BZIP2LIB}" +# Find misc and capture +findplugin h5misc +MISCPATH="${HDF5_PLUGIN_PATH}/${HDF5_PLUGIN_LIB}" + +# Verify +if ! test -f ${BZIP2PATH} ; then echo "Unable to locate ${BZIP2PATH}"; exit 1; fi +if ! test -f ${MISCPATH} ; then echo "Unable to locate ${MISCPATH}"; exit 1; fi + +# Execute the specified tests + +testapi() { +zext=$1 +echo "*** Testing dynamic filters using API for map=$zext" +deletemap $zext tmp_api +fileargs tmp_api +${execdir}/testfilter $fileurl +${NCDUMP} -s -n bzip2 $fileurl > ./tmp_api_$zext.txt +# Remove irrelevant -s output +sclean ./tmp_api_$zext.txt ./tmp_api_$zext.dump +diff -b -w ${srcdir}/ref_bzip2.cdl ./tmp_api_$zext.dump +echo "*** Pass: API dynamic filter for map=$zext" +} + +testmisc() { +zext=$1 +echo "*** Testing dynamic filters parameter passing for map $zext" +deletemap $zext tmp_misc +fileargs tmp_misc +${execdir}/testfilter_misc $fileurl +# Verify the parameters via ncdump +${NCDUMP} -s $fileurl > ./tmp_misc_$zext.txt +# Extract the parameters +getfilterattr ./tmp_misc_$zext.txt ./tmp_misc2_$zext.txt +rm -f ./tmp_misc_$zext.txt +trimleft ./tmp_misc2_$zext.txt ./tmp_misc_$zext.txt +rm -f ./tmp_misc2_$zext.txt +cat >./tmp_misc2_$zext.txt < ./tmp_ng_$zext.txt +# Remove irrelevant -s output +sclean ./tmp_ng_$zext.txt ./tmp_ng2_$zext.txt +diff -b -w ${srcdir}/ref_bzip2.cdl ./tmp_ng2_$zext.txt +echo "*** Pass: ncgen dynamic filter for map $zext" +} + +testncp() { +zext=$1 +echo "*** Testing dynamic filters using nccopy for map $zext" +deletemap $zext tmp_misc +fileargs tmp_unfiltered +# Create our input test files +${NCGEN} -4 -lb -o $fileurl ${srcdir}/../nc_test4/ref_unfiltered.cdl +fileurl0=$fileurl +fileargs tmp_filtered +${NCCOPY} -M0 -F "/g/var,307,9,4" $fileurl0 $fileurl +${NCDUMP} -s -n filtered $fileurl > ./tmp_ncp_$zext.txt +# Remove irrelevant -s output +sclean ./tmp_ncp_$zext.txt ./tmp_ncp_$zext.dump +diff -b -w ${srcdir}/ref_filtered.cdl ./tmp_ncp_$zext.dump +echo " *** Pass: nccopy simple filter for map $zext" +} + +testunk() { +zext=$1 +echo "*** Testing access to filter info when filter implementation is not available for map $zext" +deletemap $zext tmp_known +fileargs tmp_known +# build bzip2.nc +${NCGEN} -lb -4 -o $fileurl ${srcdir}/../nc_test4/bzip2.cdl +# dump and clean bzip2.nc header when filter is avail +${NCDUMP} -hs $fileurl > ./tmp_known_$zext.txt +# Remove irrelevant -s output +sclean ./tmp_known_$zext.txt tmp_known_$zext.dump +# Now hide the filter code +mv ${BZIP2PATH} ./${BZIP2LIB}.save +# dump and clean bzip2.nc header when filter is not avail +${NCDUMP} -hs $fileurl > ./tmp_unk_$zext.txt +# Restore the filter code +mv ./${BZIP2LIB}.save ${BZIP2PATH} +# Verify that the filter is no longer defined +UNK=`sed -e '/var:_Filter/p' -e d ./tmp_unk_$zext.txt` +test "x$UNK" = x +echo "*** Pass: ncgen dynamic filter for map $zext" +} + +testngc() { +zext=$1 +echo "*** Testing dynamic filters using ncgen with -lc for map $zext" +deletemap $zext tmp_ngc +fileargs tmp_ngc +${NCGEN} -lc -4 ${srcdir}/../nc_test4/bzip2.cdl > tmp_ngc.c +diff -b -w ${srcdir}/../nc_test4/../nc_test4/ref_bzip2.c ./tmp_ngc.c +echo "*** Pass: ncgen dynamic filter for map $zext" +} + +testmulti() { +zext=$1 +echo "*** Testing multiple filters for map $zext" +deletemap $zext tmp_multi +fileargs tmp_multi +${execdir}/testfilter_multi $fileurl +${NCDUMP} -hs -n multifilter $fileurl >./tmp_multi_$zext.cdl +# Remove irrelevant -s output +sclean ./tmp_multi_$zext.cdl ./tmp_smulti_$zext.cdl +diff -b -w ${srcdir}/ref_multi.cdl ./tmp_smulti_$zext.cdl +echo "*** Pass: multiple filters for map $zext" +} + +testrep() { +zext=$1 +echo "*** Testing filter re-definition invocation for map $zext" +deletemap $zext tmp_rep +fileargs tmp_rep +${execdir}/testfilter_repeat $fileurl >tmp_rep_$zext.txt +diff -b -w ${srcdir}/../nc_test4/ref_filter_repeat.txt tmp_rep_$zext.txt +} + +testorder() { +zext=$1 +echo "*** Testing multiple filter order of invocation on create for map $zext" +deletemap $zext tmp_order +fileargs tmp_order +${execdir}/testfilter_order create $fileurl >tmp_order_$zext.txt +diff -b -w ${srcdir}/../nc_test4/ref_filter_order_create.txt tmp_order_$zext.txt +echo "*** Testing multiple filter order of invocation on read for map $zext" +${execdir}/testfilter_order read $fileurl >tmp_order_rd_$zext.txt +diff -b -w ${srcdir}/../nc_test4/ref_filter_order_read.txt tmp_order_rd_$zext.txt +} + +testset file +if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testset zip ; fi +if test "x$FEATURE_S3TESTS" = xyes ; then testset s3 ; fi + +exit 0 diff --git a/nczarr_test/run_ncgen4.sh b/nczarr_test/run_ncgen4.sh index b03e04944a..cbc9a9039d 100755 --- a/nczarr_test/run_ncgen4.sh +++ b/nczarr_test/run_ncgen4.sh @@ -12,8 +12,6 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi set -e -#SHOWXFAILS=1 - # To add a new test, # 1. put the .cdl file in the 'ncdump/cdl' directory @@ -27,14 +25,13 @@ TESTS="\ dimscope \ tst_group_data \ tst_solar_1 \ -tst_nans \ tst_nul4 \ " -HEADTESTS="" -SPECTESTS="" +# These tests need to leave _FillValue +FVTESTS="tst_nans" -XFAILTESTS="" +ALLTESTS="$TESTS $FVTESTS" # Location constants cdl="$srcdir/../ncdump/cdl" @@ -43,14 +40,55 @@ RESULTSDIR="./results" # Functions +# See if this is an FVTEST +testiffv() { + ok=0 + for FV in $FVTESTS ; do + if test "x$FV" = "x$1" ; then ok=1; fi + done +} + +# Remove fillvalue attribute since zarr generates it when hdf5 does not +fvclean() { + cat $1 \ + | sed -e '/:_FillValue/d' \ + | cat > $2 +} + +difftest() { +echo ""; echo "*** Test zext=$zext" +for t in ${ALLTESTS} ; do + echo "*** Testing: ${t}" + # determine if we need the specflag set + # determine properties + checkprops ${t} + ref="ref_${t}" + rm -fr ${t}.$zext + rm -f tmp_${t}.dmp + fileargs $t + ${NCGEN} -4 -lb -o ${fileurl} ${cdl}/${ref}.cdl + ${NCDUMP} ${headflag} ${specflag} -n ${ref} ${fileurl} > tmp_${t}.dmp + testiffv $t + if test "x$ok" = x0 ; then + fvclean tmp_${t}.dmp tmp_${t}.dmpx + else + cp tmp_${t}.dmp tmp_${t}.dmpx + fi + # compare against expected + diff -b -w ${expected}/${ref}.dmp ./tmp_${t}.dmpx + echo "*** SUCCEED: ${t}" +done +} + runtestset() { extfor $1 echo "*** Testing nczarr X ncgen with zmap=${zext}" rm -fr ${RESULTSDIR}.$zext mkdir ${RESULTSDIR}.${zext} +WD=`pwd ` cd ${RESULTSDIR}.${zext} difftest -cd .. +cd $WD echo "*** PASSED: zext=${zext}" } diff --git a/nczarr_test/run_specific_filters.sh b/nczarr_test/run_specific_filters.sh new file mode 100755 index 0000000000..78bc581f1d --- /dev/null +++ b/nczarr_test/run_specific_filters.sh @@ -0,0 +1,161 @@ +#!/bin/bash + +# Test the implementations of specific filters + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +. "$srcdir/test_nczarr.sh" + +set -e + +# Function to remove selected -s attributes from file; +# These attributes might be platform dependent +sclean() { + cat $1 \ + | sed -e '/:_IsNetcdf4/d' \ + | sed -e '/:_Endianness/d' \ + | sed -e '/_NCProperties/d' \ + | sed -e '/_SuperblockVersion/d' \ + | sed -e '/_Format/d' \ + | sed -e '/global attributes:/d' \ + | cat > $2 +} + +# Function to extract _Filter attribute from a file +# These attributes might be platform dependent +getfilterattr() { +V="$1" +sed -e '/${V}.*:_Filter/p' -ed <$2 >$3 +} + +# Function to extract _Codecs attribute from a file +# These attributes might be platform dependent +getcodecsattr() { +V="$1" +sed -e '/${V}.*:_Codecs/p' -ed <$2 >$3 +} + +trimleft() { +sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2 +} + + +setfilter() { + FF="$1" + FSRC="$2" + FDST="$3" + FIH5="$4" + FICX="$5" + FFH5="$6" + FFCX="$7" + if test "x$FFH5" = x ; then FFH5="$FIH5" ; fi + if test "x$FFCX" = x ; then FFCX="$FICX" ; fi + rm -f $FDST + cat ${srcdir}/$FSRC \ + | sed -e "s/ref_any/${FF}/" \ + | sed -e "s/IH5/${FIH5}/" -e "s/FH5/${FFH5}/" \ + | sed -e "s/ICX/${FICX}/" -e "s/FCX/${FFCX}/" \ + | sed -e 's/"/\\"/g' -e 's/@/"/g' \ + | cat > $FDST +} + +# Execute the specified tests + +runfilter() { +zext="$1" +zfilt="$2" +zparams="$3" +zcodec="$4" +echo "*** Testing processing of filter $zfilt for map $zext" +deletemap $zext "tmp_${zfilt}" +fileargs "tmp_${zfilt}" +setfilter $zfilt ref_any.cdl "tmp_${zfilt}.cdl" "$zparams" "$zcodec" +if ${NCGEN} -4 -lb -o $fileurl "tmp_${zfilt}.cdl" ; then + ${NCDUMP} -n $zfilt -s $fileurl > "tmp_${zfilt}.tmp" + sclean "tmp_${zfilt}.tmp" "tmp_${zfilt}.dump" +fi +} + +testfletcher32() { + zext=$1 + runfilter $zext fletcher32 '3' '[{\"id\": \"fletcher32\"}]' + if test -f "tmp_fletcher32.dump" ; then + # need to remove _Filter + sed -e '/_Fletcher32 = "true"/d' < tmp_fletcher32.dump > tmp_fletcher32x.dump + diff -b -w "tmp_fletcher32.cdl" "tmp_fletcher32x.dump" + else + echo "XFAIL: filter=fletcher32 zext=$zext" + fi +} + +testshuffle() { + zext=$1 + runfilter $zext shuffle '2' '[{\"id\": \"shuffle\",\"elementsize\": \"0\"}]' + if test -f "tmp_shuffle.dump" ; then + # need to replace _Filter + sed -e 's/_Filter = "2,4"/_Filter = "2"/' -e '/_Shuffle = "true"/d' < tmp_shuffle.dump > tmp_shufflex.dump + diff -b -w "tmp_shuffle.cdl" "tmp_shufflex.dump" + else + echo "XFAIL: filter=shuffle zext=$zext" + fi +} + +testdeflate() { + zext=$1 + runfilter $zext deflate '1,9' '[{\"id\": \"zlib\",\"level\": \"9\"}]' + if test -f "tmp_deflate.dump" ; then + # need to replace _DeflateLevel + sed -e 's/_DeflateLevel = 9/_Filter = "1,9"/' < tmp_deflate.dump > tmp_deflatex.dump + diff -b -w "tmp_deflate.cdl" "tmp_deflatex.dump" + else + echo "XFAIL: filter=deflate zext=$zext" + fi +} + +testbzip2() { + zext=$1 + runfilter $zext bzip2 '307,9' '[{\"id\": \"bz2\",\"level\": \"9\"}]' + if test -f "tmp_bzip2.dump" ; then + diff -b -w "tmp_bzip2.cdl" "tmp_bzip2.dump" + else + echo "XFAIL: filter=bzip2 zext=$zext" + fi +} + +testszip() { + zext=$1 +# H5_SZIP_NN_OPTION_MASK=32; H5_SZIP_MAX_PIXELS_PER_BLOCK_IN=32 + runfilter $zext szip '4,32,32' '[{\"id\": \"szip\",\"mask\": 32,\"pixels-per-block\": 32}]' + if test -f "tmp_szip.dump" ; then + diff -b -w "tmp_szip.cdl" "tmp_szip.dump" + else + echo "XFAIL: filter=szip zext=$zext" + fi +} + +testblosc() { + zext=$1 + runfilter $zext blosc '32001,0,0,0,0,5,1,1' '[{\"id\": \"blosc\",\"clevel\": 5,\"blocksize\": 0,\"cname\": \"lz4\",\"shuffle\": 1}]' + if test -f "tmp_blosc.dump" ; then + diff -b -w "tmp_blosc.cdl" "tmp_blosc.dump" + else + echo "XFAIL: filter=blosc zext=$zext" + fi +} + +testset() { +# Which test cases to exercise + testfletcher32 $1 + testshuffle $1 + testdeflate $1 + testszip $1 + testbzip2 $1 + testblosc $1 +} + +testset file +#if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testset zip ; fi +#if test "x$FEATURE_S3TESTS" = xyes ; then testset s3 ; fi + +exit 0 diff --git a/nczarr_test/run_ut_mapapi.sh b/nczarr_test/run_ut_mapapi.sh index 2948f1ce65..5a049acb89 100755 --- a/nczarr_test/run_ut_mapapi.sh +++ b/nczarr_test/run_ut_mapapi.sh @@ -3,7 +3,6 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh - . "$srcdir/test_nczarr.sh" #TR="-T10" diff --git a/nczarr_test/test_nczarr.sh b/nczarr_test/test_nczarr.sh index 726fa19a87..3f9d6753c8 100755 --- a/nczarr_test/test_nczarr.sh +++ b/nczarr_test/test_nczarr.sh @@ -100,32 +100,6 @@ dumpmap() { ${execdir}/zmapio -t int -x objdump $fileurl > $3 } -difftest() { -echo ""; echo "*** Test zext=$zext" -for t in ${TESTS} ; do - echo "*** Testing: ${t}" - # determine if we need the specflag set - # determine properties - checkprops ${t} - ref="ref_${t}" - rm -fr ${t}.$zext - rm -f tmp_${t}.dmp - fileargs $t - ${NCGEN} -4 -lb -o ${fileurl} ${cdl}/${ref}.cdl - ${NCDUMP} ${headflag} ${specflag} -n ${ref} ${fileurl} > tmp_${t}.dmp - # compare the expected (silently if XFAIL) - if diff -b -w ${expected}/${ref}.dmp tmp_${t}.dmp > ${t}.diff ; then ok=1; else ok=0; fi - if test "x$ok" = "x1" ; then - echo "*** SUCCEED: ${t}" - elif test "x${isxfail}" = "x1" ; then - echo "*** XFAIL : ${t}" - else - echo "*** FAIL: ${t}" - exit 1 - fi -done -} - # Function to remove selected -s attributes from file; # These attributes might be platform dependent sclean() { @@ -146,6 +120,22 @@ cd $execdir ; abs_execdir=`pwd` ; cd $WD WD=`pwd` if test "x$NCAUTH_HOMETEST" != x ; then RCHOME=1; fi +# Set plugin path + +#cd ../plugins; make clean all >/dev/null; cd ../nczarr_test + +# Load the findplugins function +. ${builddir}/findplugin.sh +echo "findplugin.sh loaded" + +# Locate the plugin path and the library names; argument order is critical +# Find bzip2 and capture +# Assume all test filters are in same plugin dir +findplugin h5misc + +echo "final HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}" +export HDF5_PLUGIN_PATH + resetrc() { if test "x$RCHOME" = x1 ; then rm -f ${HOME}/.dodsrc ${HOME}/.daprc ${HOME}/.ncrc diff --git a/nczarr_test/test_nczarr_utils.h b/nczarr_test/test_nczarr_utils.h index 912a39f539..98e754f75f 100644 --- a/nczarr_test/test_nczarr_utils.h +++ b/nczarr_test/test_nczarr_utils.h @@ -30,7 +30,6 @@ #include "ncbytes.h" #include "zincludes.h" - static char* progname = NULL; struct ITOptions { diff --git a/nczarr_test/testfilter.c b/nczarr_test/testfilter.c new file mode 100644 index 0000000000..e459f28738 --- /dev/null +++ b/nczarr_test/testfilter.c @@ -0,0 +1,317 @@ +/* + Copyright 2018, UCAR/Unidata + See COPYRIGHT file for copying and redistribution conditions. +*/ + +/*! \file +Example program for write then read of a variable using bzip2 compression. + +\ingroup tutorial + +This is an example which +creates a file with a variable that is compressed using bzip2. +Then it reads that file and verifies that it returned the correct +uncompressed data. + +The meta-data (.cdl) for the created file is as follows: +\code +netcdf bzip2 { +dimensions: + dim0 = 4 ; + dim1 = 4 ; + dim2 = 4 ; + dim3 = 4 ; +variables: + float var(dim0, dim1, dim2, dim3) ; + var:_Storage = "chunked" ; + var:_ChunkSizes = 4, 4, 4, 4 ; + var:_Filter = "307,9" ; + var:_NoFill = "true" ; +data: + + var = + 0, 1, 2, 3, + 4, 5, 6, 7, + ... + 252, 253, 254, 255 ; +} +\endcode +*/ + +#include "config.h" +#include +#include +#include + +#ifdef HAVE_HDF5_H +#include +#endif + +#include "netcdf.h" + +/* The HDF assigned id for bzip compression */ +#define BZIP2_ID 307 +/* The compression level used in this example */ +#define BZIP2_LEVEL 9 + +#define DFALT_TESTFILE "tmp_bzip2.nc" + +/* Point at which we give up */ +#define MAXERRS 8 + +#define NDIMS 4 +#define DIMSIZE 4 +#define CHUNKSIZE 4 /* Note: not the total size of the chunk, but size wrt a dim*/ + +static size_t dimsize = DIMSIZE; +static size_t chunksize = CHUNKSIZE; +static size_t actualdims = NDIMS; + +static size_t actualproduct = 1; /* x-product over dim sizes */ +static size_t chunkproduct = 1; /* x-product over chunksizes */ + +static size_t dims[NDIMS]; +static size_t chunks[NDIMS]; + +static int nerrs = 0; + +static const char* testfile = NULL; + +static int ncid, varid; +static int dimids[NDIMS]; +static float* array = NULL; +static float* expected = NULL; +static unsigned int filterid = 0; +static unsigned int* params = NULL; + +/* Forward */ +static void init(int argc, char** argv); +static int test_bzip2(void); +static int verifychunks(void); + +#define ERRR do { \ +fflush(stdout); /* Make sure our stdout is synced with stderr. */ \ +fprintf(stderr, "Sorry! Unexpected result, %s, line: %d\n", \ + __FILE__, __LINE__); \ +nerrs++;\ +} while (0) + +static int +check(int err,int line) +{ + if(err != NC_NOERR) { + fprintf(stderr,"fail (%d): %s\n",line,nc_strerror(err)); + fflush(stderr); + exit(1); + } + return NC_NOERR; +} + +#define CHECK(x) check(x,__LINE__) + +/* +Read the chunking information about the variable +and verify that it is as expected. +*/ + +static int +verifychunks(void) +{ + int i; + int store = -1; + size_t chunksizes[NDIMS]; + memset(chunksizes,0,sizeof(chunksizes)); + CHECK(nc_inq_var_chunking(ncid, varid, &store, chunksizes)); + /* Storate must be chunked, not contiguous */ + if(store != NC_CHUNKED) { + fprintf(stderr,"bad chunk store\n"); + return NC_ESTORAGE; + } + /* Chunk sizes must match our predefined set */ + for(i=0;i= MAXERRS) + break; + } + } + if(errs == 0) + printf("no data errors\n"); + if(actualproduct <= 1) + return NC_EBADDIM; + return (errs == 0 ? NC_NOERR: NC_EINVAL); +} + +/* +Create the file, write it, then re-read for comparison. +*/ +static int +test_bzip2(void) +{ + int i; + unsigned int level = BZIP2_LEVEL; + unsigned int id=0; + size_t nparams = 0; + + printf("\n*** Testing API: bzip2 compression.\n"); + + /* Clear the data array */ + memset(array,0,sizeof(float)*actualproduct); + + /* Create a file */ + CHECK(nc_create(testfile, NC_NETCDF4|NC_CLOBBER, &ncid)); + + /* Do not use fill for this file */ + CHECK(nc_set_fill(ncid, NC_NOFILL, NULL)); + + /* Define the dimensions */ + for(i=0;i 0) { + params = (unsigned int*)malloc(sizeof(unsigned int)*nparams); + if(params == NULL) + return NC_ENOMEM; + CHECK(nc_inq_var_filter(ncid,varid,&filterid,&nparams,params)); + } + if(filterid != BZIP2_ID) { + printf("Bzip2 id mismatch: %d\n",filterid); + return NC_EFILTER; + } + if(nparams != 1 && params != NULL && params[0] != BZIP2_LEVEL) { + printf("Compression parameter mismatch\n"); + return NC_EFILTER; + } + + /* Verify chunking */ + if(!verifychunks()) + return 0; + + /* Read the data */ + CHECK(nc_get_var_float(ncid, varid, array)); + + /* Close the file */ + CHECK(nc_close(ncid)); + return (compare() == NC_NOERR ? 0 : 1); +} + +/**************************************************/ +/* Utilities */ + +static void +init(int argc, char** argv) +{ + int i; + + /* get the testfile path */ + if(argc > 1) + testfile = argv[1]; + else + testfile = DFALT_TESTFILE; + + /* Setup various variables */ + actualproduct = 1; + chunkproduct = 1; + for(i=0;i 0?1:0); +} + diff --git a/nczarr_test/testfilter_misc.c b/nczarr_test/testfilter_misc.c new file mode 100644 index 0000000000..3fd12257cf --- /dev/null +++ b/nczarr_test/testfilter_misc.c @@ -0,0 +1,563 @@ +/* + Copyright 2018, UCAR/Unidata + See COPYRIGHT file for copying and redistribution conditions. +*/ + +#include "config.h" +#include +#include +#include + +#ifdef HAVE_HDF5_H +#include +#endif + +#include "netcdf.h" +#include "netcdf_aux.h" +#include "netcdf_filter.h" + +#undef TESTODDSIZE + +#undef DEBUG + +/* The C standard apparently defines all floating point constants as double; + we rely on that in this code. +*/ +#define DBLVAL 12345678.12345678 + +#define TEST_ID 32768 + +#define MAXERRS 8 + +#define MAXPARAMS 32 + +#define NPARAMS 14 + +static unsigned int baseline[NPARAMS]; + +static const char* testfile = NULL; + +#define MAXDIMS 8 + +#define DFALT_TESTFILE "tmp_misc.nc" + +#define spec "32768, -17b, 23ub, -25S, 27US, 77, 93U, 789f, 12345678.12345678d, -9223372036854775807L, 18446744073709551615UL" + +#ifdef TESTODDSIZE +#define NDIMS 1 +static size_t dimsize[NDIMS] = {4}; +static size_t chunksize[NDIMS] = {3}; +#else +#define NDIMS 4 +static size_t dimsize[NDIMS] = {4,4,4,4}; +static size_t chunksize[NDIMS] = {4,4,4,4}; +#endif + +static size_t ndims = NDIMS; + +static size_t totalproduct = 1; /* x-product over max dims */ +static size_t actualproduct = 1; /* x-product over actualdims */ +static size_t chunkproduct = 1; /* x-product over actual chunks */ + +static size_t pattern[MAXDIMS]; + +static int nerrs = 0; + +static int ncid, varid; +static int dimids[MAXDIMS]; +static size_t odom[MAXDIMS]; +static float* array = NULL; +static float* expected = NULL; + +static unsigned int filterid = 0; +static size_t nparams = 0; +static unsigned int params[MAXPARAMS]; + +/* Forward */ +static int test_test1(void); +static void init(int argc, char** argv); +static void reset(void); +static void odom_reset(void); +static int odom_more(void); +static int odom_next(void); +static int odom_offset(void); +static float expectedvalue(void); +static void verifyparams(void); + +#define ERRR do { \ +fflush(stdout); /* Make sure our stdout is synced with stderr. */ \ +fprintf(stderr, "Sorry! Unexpected result, %s, line: %d\n", \ + __FILE__, __LINE__); \ +nerrs++;\ +} while (0) + +static int +check(int err,int line) +{ + if(err != NC_NOERR) { + fprintf(stderr,"fail (%d): %s\n",line,nc_strerror(err)); + } + return NC_NOERR; +} + +static void +report(const char* msg, int lineno) +{ + fprintf(stderr,"fail: line=%d %s\n",lineno,msg); + exit(1); +} + +#define CHECK(x) check(x,__LINE__) +#define REPORT(x) report(x,__LINE__) + +static int +verifychunks(void) +{ + int i; + int store = -1; + size_t localchunks[MAXDIMS]; + memset(localchunks,0,sizeof(localchunks)); + CHECK(nc_inq_var_chunking(ncid, varid, &store, localchunks)); + if(store != NC_CHUNKED) { + fprintf(stderr,"bad chunk store\n"); + return 0; + } + for(i=0;i 0) { + params = (unsigned int*)malloc(sizeof(unsigned int)*nparams); + if(params == NULL) + return NC_ENOMEM; + CHECK(nc_inq_var_filter(ncid,varid,&filterid,&nparams,params)); + } + if(filterid != TEST_ID) { + fprintf(stderr,"open: test id mismatch: %d\n",filterid); + return NC_EFILTER; + } + if(nparams != NPARAMS) { + size_t i; + unsigned int inqparams[MAXPARAMS]; + fprintf(stderr,"nparams mismatch\n"); + for(nerrs=0,i=0;i 0) return NC_EFILTER; + + if(params) free(params); + + /* Verify chunking */ + if(!verifychunks()) + return 0; + fflush(stderr); + return 1; +} + +static int +setchunking(void) +{ + int store; + + store = NC_CHUNKED; + CHECK(nc_def_var_chunking(ncid,varid,store,chunksize)); + if(!verifychunks()) + return NC_EINVAL; + return NC_NOERR; +} + +static void +fill(void) +{ + odom_reset(); + if(1) { + int i; + if(actualproduct <= 1) abort(); + for(i=0;i= MAXERRS) + break; + } + } + } else + { + odom_reset(); + while(odom_more()) { + int offset = odom_offset(); + float expect = expectedvalue(); + if(array[offset] != expect) { + fprintf(stderr,"data mismatch: array[%d]=%f expected=%f\n", + offset,array[offset],expect); + errs++; + if(errs >= MAXERRS) + break; + } + odom_next(); + } + } + + if(errs == 0) + fprintf(stderr,"no data errors\n"); + return (errs == 0); +} + +static void +showparameters(void) +{ + int i; + fprintf(stderr,"test: nparams=%ld: params=",(unsigned long)nparams); + for(i=0;i=0;i--) { + odom[i] += 1; + if(odom[i] < dimsize[i]) break; + if(i == 0) return 0; /* leave the 0th entry if it overflows*/ + odom[i] = 0; /* reset this position*/ + } + return 1; +} + +static int +odom_offset(void) +{ + int i; + int offset = 0; + for(i=0;i 1) + testfile = argv[1]; + else + testfile = DFALT_TESTFILE; + + /* Setup various variables */ + totalproduct = 1; + actualproduct = 1; + chunkproduct = 1; + for(i=0;i 0?1:0); +} diff --git a/nczarr_test/testfilter_multi.c b/nczarr_test/testfilter_multi.c new file mode 100644 index 0000000000..2ceb33fef6 --- /dev/null +++ b/nczarr_test/testfilter_multi.c @@ -0,0 +1,331 @@ +/* + Copyright 2018, UCAR/Unidata + See COPYRIGHT file for copying and redistribution conditions. +*/ + +/*! \file +Test support for multiple filters per variable +*/ + +#include "config.h" +#include +#include +#include + +#ifdef HAVE_HDF5_H +#include +#endif + +#include "netcdf.h" +#include "netcdf_filter.h" + +/* The HDF assigned id for bzip compression */ +#define BZIP2_ID 307 +/* The compression level used in this example */ +#define BZIP2_LEVEL 9 + +#define DEFLATE_LEVEL 2 + +#define NOOP_ID 40000 + +#define NFILTERS 3 + +#define DFALT_TESTFILE "tmp_multifilter.nc" + +/* Point at which we give up */ +#define MAXERRS 8 + +#define NDIMS 4 +#define DIMSIZE 4 +#define CHUNKSIZE 4 /* Note: not the total size of the chunk, but size wrt a dim*/ + +static const char* testfile = NULL; + +static size_t dimsize = DIMSIZE; +static size_t chunksize = CHUNKSIZE; +static size_t actualdims = NDIMS; + +static size_t actualproduct = 1; /* x-product over dim sizes */ +static size_t chunkproduct = 1; /* x-product over chunksizes */ + +static size_t dims[NDIMS]; +static size_t chunks[NDIMS]; + +static int nerrs = 0; + +static int ncid, varid; +static int dimids[NDIMS]; +static float* array = NULL; +static float* expected = NULL; + +/* Forward */ +static void init(int argc, char** argv); +static int test_multi(void); +static int verifychunks(void); + +#define ERRR do { \ +fflush(stdout); /* Make sure our stdout is synced with stderr. */ \ +fprintf(stderr, "Sorry! Unexpected result, %s, line: %d\n", \ + __FILE__, __LINE__); \ +nerrs++;\ +} while (0) + +static int +check(int err,int line) +{ + if(err != NC_NOERR) { + fprintf(stderr,"fail (%d): %s\n",line,nc_strerror(err)); + fflush(stderr); + exit(1); + } + return NC_NOERR; +} + +#define CHECK(x) check(x,__LINE__) + +/* +Read the chunking information about the variable +and verify that it is as expected. +*/ + +static int +verifychunks(void) +{ + int i; + int store = -1; + size_t chunksizes[NDIMS]; + memset(chunksizes,0,sizeof(chunksizes)); + CHECK(nc_inq_var_chunking(ncid, varid, &store, chunksizes)); + /* Storate must be chunked, not contiguous */ + if(store != NC_CHUNKED) { + fprintf(stderr,"bad chunk store\n"); + return NC_ESTORAGE; + } + /* Chunk sizes must match our predefined set */ + for(i=0;i= MAXERRS) + break; + } + } + if(errs == 0) + printf("no data errors\n"); + if(actualproduct <= 1) + return NC_EBADDIM; + return (errs == 0 ? NC_NOERR: NC_EINVAL); +} + +int +verifyfilters(int ncid, int varid) +{ + size_t nparams; + size_t nfilters; + unsigned int filterids[NFILTERS]; + unsigned int params[2]; + + /* Read back the compression info and verify it */ + CHECK(nc_inq_var_filter_ids(ncid,varid,&nfilters,filterids)); + if(nfilters != NFILTERS) { + fprintf(stderr,"Fail: nfilters mismatch: expected=%d actual=%u\n",NFILTERS,(unsigned)nfilters); + return NC_EINVAL; + } + if(filterids[0] != BZIP2_ID + || filterids[1] != H5Z_FILTER_DEFLATE + || filterids[2] != NOOP_ID + ) { + fprintf(stderr,"Fail: filter id mismatch: actual/expected={%u/%u,%u/%u,%u/%u}\n", + filterids[0],BZIP2_ID, + filterids[1],H5Z_FILTER_DEFLATE, + filterids[2],NOOP_ID); + return NC_EINVAL; + } + /* Get level for each filter */ + CHECK(nc_inq_var_filter_info(ncid,varid,BZIP2_ID,&nparams,params)); + if(nparams != 1) { + fprintf(stderr,"Fail: nparams mismatch: id=%u expected=1 actual=%u\n",filterids[0],(unsigned)nparams); + return NC_EINVAL; + } + if(params[0] != BZIP2_LEVEL) { + fprintf(stderr,"Fail: parameter mismatch: expected=%u actual=%u\n",BZIP2_LEVEL,params[0]); + return NC_EINVAL; + } + CHECK(nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_DEFLATE,&nparams,params)); + if(nparams != 1) { + fprintf(stderr,"Fail: nparams mismatch: id=%u expected=1 actual=%u\n",filterids[1],(unsigned)nparams); + return NC_EINVAL; + } + if(params[0] != DEFLATE_LEVEL) { + fprintf(stderr,"Fail: parameter mismatch: expected=%u actual=%u\n",BZIP2_LEVEL,params[0]); + return NC_EINVAL; + } + CHECK(nc_inq_var_filter_info(ncid,varid,NOOP_ID,&nparams,params)); + if(nparams != 0) { + fprintf(stderr,"Fail: parameter mismatch: id=%u nparams: expected=0 actual=%u\n",NOOP_ID,(unsigned)nparams); + return NC_EINVAL; + } + return NC_NOERR; +} + +/* +Create the file, write it, then re-read for comparison. +*/ +static int +test_multi(void) +{ + int i; + unsigned int params[2]; + + printf("\n*** Testing Multi-filter application: filter set = bzip2 deflate noop"); + printf("\n"); + + /* Clear the data array */ + memset(array,0,sizeof(float)*actualproduct); + + /* Create a file */ + CHECK(nc_create(testfile, NC_NETCDF4|NC_CLOBBER, &ncid)); + + /* Do not use fill for this file */ + CHECK(nc_set_fill(ncid, NC_NOFILL, NULL)); + + /* Define the dimensions */ + for(i=0;i 1) + testfile = argv[1]; + else + testfile = DFALT_TESTFILE; + + /* Setup various variables */ + actualproduct = 1; + chunkproduct = 1; + for(i=0;i 0?1:0); +} + diff --git a/nczarr_test/testfilter_order.c b/nczarr_test/testfilter_order.c new file mode 100644 index 0000000000..2e572aeeea --- /dev/null +++ b/nczarr_test/testfilter_order.c @@ -0,0 +1,441 @@ +/* + Copyright 2018, UCAR/Unidata + See COPYRIGHT file for copying and redistribution conditions. +*/ + +#include "config.h" +#include +#include +#include +#include +#include + +#include "netcdf.h" +#include "netcdf_filter.h" + +#undef TESTODDSIZE + +#undef DEBUG + +#define FILTER_ID 40000 + +#define MAXERRS 8 + +#define MAXPARAMS 32 + +#define MAXDIMS 8 + +#define DFALT_TESTFILE "tmp_filter_order.nc" + +#define NPARAMS 1 +#define PARAMVAL 17 + +#define NDIMS 4 +static size_t dimsize[NDIMS] = {4,4,4,4}; +static size_t chunksize[NDIMS] = {4,4,4,4}; + +static size_t ndims = NDIMS; + +static int creating = 1; /* Default is to do filter test 1 */ +static size_t totalproduct = 1; /* x-product over max dims */ +static size_t actualproduct = 1; /* x-product over actualdims */ +static size_t chunkproduct = 1; /* x-product over actual chunks */ + +static int nerrs = 0; + +static const char* testfile = NULL; + +static int ncid, varid; +static int dimids[MAXDIMS]; +static size_t odom[MAXDIMS]; +static float* array = NULL; +static float* expected = NULL; + +/* Forward */ +static int filter_test1(void); +static void init(int argc, char** argv); +static void reset(void); +static void odom_reset(void); +static int odom_more(void); +static int odom_next(void); +static int odom_offset(void); +static float expectedvalue(void); + +#define ERRR do { \ +fflush(stdout); /* Make sure our stdout is synced with stderr. */ \ +fprintf(stderr, "Sorry! Unexpected result, %s, line: %d\n", \ + __FILE__, __LINE__); \ +nerrs++;\ +} while (0) + +static int +check(int err,int line) +{ + if(err != NC_NOERR) { + fprintf(stderr,"fail (%d): %s\n",line,nc_strerror(err)); + } + return NC_NOERR; +} + +static void +report(const char* msg, int lineno) +{ + fprintf(stderr,"fail: line=%d %s\n",lineno,msg); + exit(1); +} + +#define CHECK(x) check(x,__LINE__) +#define REPORT(x) report(x,__LINE__) + +static int +verifychunks(void) +{ + int i; + int store = -1; + size_t localchunks[MAXDIMS]; + memset(localchunks,0,sizeof(localchunks)); + CHECK(nc_inq_var_chunking(ncid, varid, &store, localchunks)); + if(store != NC_CHUNKED) { + fprintf(stderr,"bad chunk store\n"); + return 0; + } + for(i=0;i= MAXERRS) + break; + } + } + } else + { + odom_reset(); + while(odom_more()) { + int offset = odom_offset(); + float expect = expectedvalue(); + if(array[offset] != expect) { + fprintf(stderr,"data mismatch: array[%d]=%f expected=%f\n", + offset,array[offset],expect); + errs++; + if(errs >= MAXERRS) + break; + } + odom_next(); + } + } + + if(errs == 0) + {printf("no data errors\n"); fflush(stdout);} + return (errs == 0); +} + +/* Test filter order on creation */ +static int +filter_test1(void) +{ + int ok = 1; + + reset(); + + printf("test1: filter order: create\n"); fflush(stdout); + create(); + setchunking(); + deffilters(); + inqfilters(); + CHECK(nc_enddef(ncid)); + + /* Fill in the array */ + fill(); + + nc_sync(ncid); + + printf("test1: compression.\n"); fflush(stdout); + /* write array */ + CHECK(nc_put_var(ncid,varid,expected)); + + printf("test1: decompression.\n"); fflush(stdout); + CHECK(nc_get_var_float(ncid, varid, array)); + ok = compare(); + + CHECK(nc_close(ncid)); + + return ok; +} + +/* Test filter order on read */ +static int +filter_test2(void) +{ + int ok = 1; + + reset(); + + printf("test2: filter order: read\n"); fflush(stdout); + + /* Fill in the array */ + fill(); + + printf("test2: decompression.\n"); fflush(stdout); + reset(); + openfile(); + inqfilters(); + + printf("test2: decompression.\n"); fflush(stdout); + CHECK(nc_get_var_float(ncid, varid, array)); + ok = compare(); + + CHECK(nc_close(ncid)); + return ok; +} + +/**************************************************/ +/* Utilities */ + +static void +reset() +{ + memset(array,0,sizeof(float)*actualproduct); +} + +static void +odom_reset(void) +{ + memset(odom,0,sizeof(odom)); +} + +static int +odom_more(void) +{ + return (odom[0] < dimsize[0]); +} + +static int +odom_next(void) +{ + int i; /* do not make unsigned */ + for(i=ndims-1;i>=0;i--) { + odom[i] += 1; + if(odom[i] < dimsize[i]) break; + if(i == 0) return 0; /* leave the 0th entry if it overflows*/ + odom[i] = 0; /* reset this position*/ + } + return 1; +} + +static int +odom_offset(void) +{ + int i; + int offset = 0; + for(i=0;i 2) + testfile = argv[2]; + else + testfile = DFALT_TESTFILE; + + /* Setup various variables */ + totalproduct = 1; + actualproduct = 1; + chunkproduct = 1; + for(i=0;i 0?1:0); +} diff --git a/nczarr_test/testfilter_repeat.c b/nczarr_test/testfilter_repeat.c new file mode 100644 index 0000000000..b0539544ac --- /dev/null +++ b/nczarr_test/testfilter_repeat.c @@ -0,0 +1,378 @@ +/* + Copyright 2018, UCAR/Unidata + See COPYRIGHT file for copying and redistribution conditions. +*/ + +#include "config.h" +#include +#include +#include +#include +#include + +#include "netcdf.h" +#include "netcdf_filter.h" + +#undef TESTODDSIZE + +#undef DEBUG + +#define FILTER_ID 40000 + +#define MAXERRS 8 + +#define MAXPARAMS 32 + +#define MAXDIMS 8 + +#define DFALT_TESTFILE "testfilter_reg.nc" + +#define NPARAMS 1 +#define PARAMVAL 17 + +#define NDIMS 4 +static size_t dimsize[NDIMS] = {4,4,4,4}; +static size_t chunksize[NDIMS] = {4,4,4,4}; + +static size_t ndims = NDIMS; + +static size_t totalproduct = 1; /* x-product over max dims */ +static size_t actualproduct = 1; /* x-product over actualdims */ +static size_t chunkproduct = 1; /* x-product over actual chunks */ + +static int nerrs = 0; + +static char* testfile = NULL; + + +static int ncid, varid; +static int dimids[MAXDIMS]; +static size_t odom[MAXDIMS]; +static float* array = NULL; +static float* expected = NULL; + +/* Forward */ +static int filter_test1(void); +static void init(int argc, char** argv); +static void reset(void); +static void odom_reset(void); +static int odom_more(void); +static int odom_next(void); +static int odom_offset(void); +static float expectedvalue(void); + +#define ERRR do { \ +fflush(stdout); /* Make sure our stdout is synced with stderr. */ \ +fprintf(stderr, "Sorry! Unexpected result, %s, line: %d\n", \ + __FILE__, __LINE__); \ +nerrs++;\ +} while (0) + +static int +check(int err,int line) +{ + if(err != NC_NOERR) { + fprintf(stderr,"fail (%d): %s\n",line,nc_strerror(err)); + } + return NC_NOERR; +} + +static void +report(const char* msg, int lineno) +{ + fprintf(stderr,"fail: line=%d %s\n",lineno,msg); + exit(1); +} + +#define CHECK(x) check(x,__LINE__) +#define REPORT(x) report(x,__LINE__) + +static int +verifychunks(void) +{ + int i; + int store = -1; + size_t localchunks[MAXDIMS]; + memset(localchunks,0,sizeof(localchunks)); + CHECK(nc_inq_var_chunking(ncid, varid, &store, localchunks)); + if(store != NC_CHUNKED) { + fprintf(stderr,"bad chunk store\n"); + return 0; + } + for(i=0;i= MAXERRS) + break; + } + } + } else + { + odom_reset(); + while(odom_more()) { + int offset = odom_offset(); + float expect = expectedvalue(); + if(array[offset] != expect) { + fprintf(stderr,"data mismatch: array[%d]=%f expected=%f\n", + offset,array[offset],expect); + errs++; + if(errs >= MAXERRS) + break; + } + odom_next(); + } + } + + if(errs == 0) + printf("no data errors\n"); + return (errs == 0); +} + +static int +filter_test1(void) +{ + int ok = 1; + unsigned int params[MAXPARAMS]; + + reset(); + + printf("test1: def filter repeat .\n"); + create(); + setchunking(); + + params[0] = 1; + params[1] = 17; + deffilter(FILTER_ID,2,params); + + params[0] = 0; + params[1] = 18; + deffilter(FILTER_ID,2,params); + + CHECK(nc_enddef(ncid)); + + /* Fill in the array */ + fill(); + + printf("test1: compression.\n"); + /* write array */ + CHECK(nc_put_var(ncid,varid,expected)); + CHECK(nc_close(ncid)); + + printf("test1: decompression.\n"); + reset(); + openfile(); + CHECK(nc_get_var_float(ncid, varid, array)); + ok = compare(); + + CHECK(nc_close(ncid)); + return ok; +} + +/**************************************************/ +/* Utilities */ + +static void +reset() +{ + memset(array,0,sizeof(float)*actualproduct); +} + +static void +odom_reset(void) +{ + memset(odom,0,sizeof(odom)); +} + +static int +odom_more(void) +{ + return (odom[0] < dimsize[0]); +} + +static int +odom_next(void) +{ + int i; /* do not make unsigned */ + for(i=ndims-1;i>=0;i--) { + odom[i] += 1; + if(odom[i] < dimsize[i]) break; + if(i == 0) return 0; /* leave the 0th entry if it overflows*/ + odom[i] = 0; /* reset this position*/ + } + return 1; +} + +static int +odom_offset(void) +{ + int i; + int offset = 0; + for(i=0;i 1) + testfile = argv[1]; + else + testfile = DFALT_TESTFILE; + + /* Setup various variables */ + totalproduct = 1; + actualproduct = 1; + chunkproduct = 1; + for(i=0;i 0?1:0); +} diff --git a/nczarr_test/tst_nczfilter.c b/nczarr_test/tst_nczfilter.c new file mode 100644 index 0000000000..84c1ea5f84 --- /dev/null +++ b/nczarr_test/tst_nczfilter.c @@ -0,0 +1,62 @@ +/* This is part of the netCDF package. + Copyright 2018 University Corporation for Atmospheric Research/Unidata + See COPYRIGHT file for conditions of use. + + Test nczarr filter loading + Author: Dennis Heimbigner +*/ + +#include +#include +#include + +#include "netcdf.h" +#include "netcdf_filter.h" + +#define ERR(r) {fprintf(stderr,"fail: line %d: (%d) %s\n",__LINE__,(r),nc_strerror((r)));} + +#define FILTERID 1 + +int +main(int argc, char **argv) +{ + int ret = NC_NOERR; + int ncid; + int dimid; + int varid; + size_t chunksizes[1] = {4}; + unsigned params[1] = {9}; + char* furl = NULL; + int data[4] = {17,18,19,20}; + size_t nfilters; + unsigned int filterids[8]; + size_t nparams; + unsigned inqparams[8]; + + furl = argv[1]; + + if ((ret=nc_create(furl, NC_NETCDF4, &ncid))) ERR(ret); + if ((ret=nc_def_dim(ncid, "d", 4, &dimid))) ERR(ret); + if ((ret=nc_def_var(ncid, "v", NC_INT, 1, &dimid, &varid))) ERR(ret); + if((ret=nc_def_var_chunking(ncid,varid,NC_CHUNKED,chunksizes))) ERR(ret); + if((ret=nc_def_var_filter(ncid,varid,FILTERID,1,params))) ERR(ret); + if((ret=nc_put_var(ncid,varid,data))) ERR(ret); + if ((ret=nc_close(ncid))) ERR(ret); + + if ((ret=nc_open(furl, 0, &ncid))) ERR(ret); + if ((ret=nc_inq_varid(ncid, "v", &varid))) ERR(ret); + + if((ret=nc_inq_var_filter_ids(ncid, varid, &nfilters, filterids))) ERR(ret); + if(nfilters != 1) ERR(NC_EFILTER); + if(filterids[0] != FILTERID) ERR(NC_EFILTER); + printf("nfilters=%u filterids[0]=%u\n",(unsigned)nfilters,(unsigned)filterids[0]); + if((ret=nc_inq_var_filter_info(ncid, varid, filterids[0], &nparams, inqparams))) ERR(ret); + if(nparams != 1) ERR(NC_EFILTER); + if(inqparams[0] != params[0]) ERR(NC_EFILTER); + printf("nparams=%u params[0]=%u\n",(unsigned)nparams,(unsigned)inqparams[0]); + + if ((ret=nc_close(ncid))) ERR(ret); + + nc_finalize(); + return 0; +} diff --git a/nczarr_test/ut_json.c b/nczarr_test/ut_json.c index 98dd1a292b..c08c121442 100644 --- a/nczarr_test/ut_json.c +++ b/nczarr_test/ut_json.c @@ -21,6 +21,10 @@ static void dump(NCjson* json); static void dumpR(NCjson* json, int depth); static char* sortname(int sort); +static int jclone(NCjson* json, NCjson** clonep); +static int cloneArray(NCjson* array, NCjson** clonep); +static int cloneDict(NCjson* dict, NCjson** clonep); + struct Test tests[] = { {"build", testbuild}, {"parse", testparse}, @@ -57,19 +61,19 @@ static int build(NCJ* ncj) { int stat = NC_NOERR; - NCjson* clone; + NCjson* clone = NULL; memset(ncj,0,sizeof(NCJ)); /* Build instances of primitives */ if((stat = NCJnew(NCJ_STRING,&ncj->ncj_string))) goto done; - ncj->ncj_string->string = strdup("string"); + NCJsetstring(ncj->ncj_string, strdup("string")); if((stat = NCJnew(NCJ_INT,&ncj->ncj_int))) goto done; - ncj->ncj_int->string = strdup("117"); + NCJsetstring(ncj->ncj_int, strdup("117")); if((stat = NCJnew(NCJ_DOUBLE,&ncj->ncj_double))) goto done; - ncj->ncj_double->string = strdup("3.1415926"); + NCJsetstring(ncj->ncj_double, strdup("3.1415926")); if((stat = NCJnew(NCJ_BOOLEAN,&ncj->ncj_boolean))) goto done; - ncj->ncj_boolean->string = strdup("true"); + NCJsetstring(ncj->ncj_boolean, strdup("true")); if((stat = NCJnew(NCJ_NULL,&ncj->ncj_null))) goto done; /* Create an empty array */ @@ -77,17 +81,17 @@ build(NCJ* ncj) /* Create a filled array */ if((stat = NCJnew(NCJ_ARRAY,&ncj->ncj_array2))) goto done; - if((stat = NCJclone(ncj->ncj_string,&clone))) goto done; + if((stat = jclone(ncj->ncj_string,&clone))) goto done; if((stat = NCJappend(ncj->ncj_array2,clone))) goto done; - if((stat = NCJclone(ncj->ncj_int,&clone))) goto done; + if((stat = jclone(ncj->ncj_int,&clone))) goto done; if((stat = NCJappend(ncj->ncj_array2,clone))) goto done; - if((stat = NCJclone(ncj->ncj_double,&clone))) goto done; + if((stat = jclone(ncj->ncj_double,&clone))) goto done; if((stat = NCJappend(ncj->ncj_array2,clone))) goto done; - if((stat = NCJclone(ncj->ncj_boolean,&clone))) goto done; + if((stat = jclone(ncj->ncj_boolean,&clone))) goto done; if((stat = NCJappend(ncj->ncj_array2,clone))) goto done; - if((stat = NCJclone(ncj->ncj_null,&clone))) goto done; + if((stat = jclone(ncj->ncj_null,&clone))) goto done; if((stat = NCJappend(ncj->ncj_array2,clone))) goto done; - if((stat = NCJclone(ncj->ncj_array1,&clone))) goto done; + if((stat = jclone(ncj->ncj_array1,&clone))) goto done; if((stat = NCJappend(ncj->ncj_array2,clone))) goto done; /* Create an empty dict */ @@ -95,27 +99,94 @@ build(NCJ* ncj) /* Create a filled dict */ if((stat = NCJnew(NCJ_DICT,&ncj->ncj_dict2))) goto done; - if((stat = NCJclone(ncj->ncj_string,&clone))) goto done; + if((stat = jclone(ncj->ncj_string,&clone))) goto done; if((stat = NCJinsert(ncj->ncj_dict2,"string",clone))) goto done; - if((stat = NCJclone(ncj->ncj_int,&clone))) goto done; + if((stat = jclone(ncj->ncj_int,&clone))) goto done; if((stat = NCJinsert(ncj->ncj_dict2,"int",clone))) goto done; - if((stat = NCJclone(ncj->ncj_double,&clone))) goto done; + if((stat = jclone(ncj->ncj_double,&clone))) goto done; if((stat = NCJinsert(ncj->ncj_dict2,"double",clone))) goto done; - if((stat = NCJclone(ncj->ncj_boolean,&clone))) goto done; + if((stat = jclone(ncj->ncj_boolean,&clone))) goto done; if((stat = NCJinsert(ncj->ncj_dict2,"boolean",clone))) goto done; - if((stat = NCJclone(ncj->ncj_null,&clone))) goto done; + if((stat = jclone(ncj->ncj_null,&clone))) goto done; if((stat = NCJinsert(ncj->ncj_dict2,"null",clone))) goto done; - if((stat = NCJclone(ncj->ncj_array1,&clone))) goto done; + if((stat = jclone(ncj->ncj_array1,&clone))) goto done; if((stat = NCJinsert(ncj->ncj_dict2,"array1",clone))) goto done; - if((stat = NCJclone(ncj->ncj_array2,&clone))) goto done; + if((stat = jclone(ncj->ncj_array2,&clone))) goto done; if((stat = NCJinsert(ncj->ncj_dict2,"array2",clone))) goto done; - if((stat = NCJclone(ncj->ncj_dict1,&clone))) goto done; + if((stat = jclone(ncj->ncj_dict1,&clone))) goto done; if((stat = NCJinsert(ncj->ncj_dict2,"dict1",clone))) goto done; done: return THROW(stat); } +static int +jclone(NCjson* json, NCjson** clonep) +{ + int stat = NC_NOERR; + NCjson* clone = NULL; + + if(json == NULL) goto done; + + switch(json->sort) { + case NCJ_INT: + case NCJ_DOUBLE: + case NCJ_BOOLEAN: + case NCJ_STRING: + if((stat=NCJnew(json->sort,&clone))) goto done; + NCJsetstring(clone,strdup(NCJstring(json))); + if(NCJstring(clone) == NULL) + {stat = NC_ENOMEM; goto done;} + break; + case NCJ_NULL: + if((stat=NCJnew(json->sort,&clone))) goto done; + break; + case NCJ_DICT: + if((stat=cloneDict(json,&clone))) goto done; + break; + case NCJ_ARRAY: + if((stat=cloneArray(json,&clone))) goto done; + break; + default: break; /* nothing to clone */ + } +done: + if(stat == NC_NOERR && clonep) {*clonep = clone; clone = NULL;} + NCJreclaim(clone); + return stat; +} + +static int +cloneArray(NCjson* array, NCjson** clonep) +{ + int i, stat=NC_NOERR; + NCjson* clone = NULL; + if((stat=NCJnew(NCJ_ARRAY,&clone))) goto done; + for(i=0;isort)); - switch(json->sort) { - case NCJ_STRING: printf("\"%s\"",json->string); break; + printf("/%s/ ",sortname(NCJsort(json))); + switch(NCJsort(json)) { + case NCJ_STRING: printf("\"%s\"",NCJstring(json)); break; case NCJ_INT: - ok = sscanf(json->string,"%lld%n",&int64v,&count); - if(ok != 1 || count != strlen(json->string)) goto fail; + ok = sscanf(NCJstring(json),"%lld%n",&int64v,&count); + if(ok != 1 || count != strlen(NCJstring(json))) goto fail; printf("%lld",int64v); break; case NCJ_DOUBLE: - ok = sscanf(json->string,"%lg%n",&float64v,&count); - if(ok != 1 || count != strlen(json->string)) goto fail; + ok = sscanf(NCJstring(json),"%lg%n",&float64v,&count); + if(ok != 1 || count != strlen(NCJstring(json))) goto fail; printf("%lg",float64v); break; case NCJ_BOOLEAN: - if(strcasecmp(json->string,"true") != 0 - && strcasecmp(json->string,"false") != 0) goto fail; - printf("%s",json->string); + if(strcasecmp(NCJstring(json),"true") != 0 + && strcasecmp(NCJstring(json),"false") != 0) goto fail; + printf("%s",NCJstring(json)); break; case NCJ_NULL: printf("null"); @@ -237,9 +308,9 @@ dumpR(NCjson* json, int depth) for(i=0;isort == NCJ_STRING); + assert(NCJsort(j) == NCJ_STRING); printf("{%d} ",depth+1); - printf("\"%s\" => ",j->string); + printf("\"%s\" => ",NCJstring(j)); if(i+1 >= NCJlength(json)) {/* malformed */ printf(""); } else @@ -263,7 +334,7 @@ dumpR(NCjson* json, int depth) printf("\n"); return; fail: - printf("????\n"); + printf("clone?\n"); } static char* diff --git a/oc2/Makefile.am b/oc2/Makefile.am index 96184efbed..18d466b48e 100644 --- a/oc2/Makefile.am +++ b/oc2/Makefile.am @@ -42,6 +42,8 @@ liboc_la_CPPFLAGS = $(AM_CPPFLAGS) .PHONEY: bison bison:: dap.y - rm -f dap.tab.c dap.tab.h + rm -f dap.tab.c dap.tab.h dapy.c dapy.h bison --debug -d -p dap dap.y - mv dap.tab.c dapy.c; mv dap.tab.h dapy.h + sed -e 's/dap[.]tab[.]c/dapy.c/g' -e 's/dap[.]tab[.]h/dapy.h/g' dapy.c + mv dap.tab.h dapy.h + rm -f dap.tab.c diff --git a/oc2/dap.y b/oc2/dap.y index b08b2a0671..0305015429 100644 --- a/oc2/dap.y +++ b/oc2/dap.y @@ -4,8 +4,8 @@ /*The lines down to DO NOT DELETE ... comment are specific to the C Parser. They will be commennted out when building a java parser. */ -%error-verbose -%pure-parser +%define parse.error verbose +%define api.pure %lex-param {DAPparsestate* parsestate} %parse-param {DAPparsestate* parsestate} %{ diff --git a/oc2/dapy.c b/oc2/dapy.c index c4031eb3a8..fc7c45d1f2 100644 --- a/oc2/dapy.c +++ b/oc2/dapy.c @@ -75,7 +75,7 @@ #include "dapy.h" int dapdebug = 0; -#line 79 "dap.tab.c" /* yacc.c:339 */ +#line 79 "dapy.c" /* yacc.c:339 */ # ifndef YY_NULLPTR # if defined __cplusplus && 201103L <= __cplusplus @@ -94,7 +94,7 @@ int dapdebug = 0; #endif /* In a future release of Bison, this section will be replaced - by #include "dap.tab.h". */ + by #include "dapy.h". */ #ifndef YY_DAP_DAP_TAB_H_INCLUDED # define YY_DAP_DAP_TAB_H_INCLUDED /* Debug traces. */ @@ -153,7 +153,7 @@ int dapparse (DAPparsestate* parsestate); /* Copy the second part of user declarations. */ -#line 157 "dap.tab.c" /* yacc.c:358 */ +#line 157 "dapy.c" /* yacc.c:358 */ #ifdef short # undef short @@ -1408,611 +1408,611 @@ YYSTYPE yylval YY_INITIAL_VALUE (= yyval_default); case 6: #line 58 "dap.y" /* yacc.c:1646 */ {dap_unrecognizedresponse(parsestate); YYABORT;} -#line 1412 "dap.tab.c" /* yacc.c:1646 */ +#line 1412 "dapy.c" /* yacc.c:1646 */ break; case 7: #line 63 "dap.y" /* yacc.c:1646 */ {dap_tagparse(parsestate,SCAN_DATASET);} -#line 1418 "dap.tab.c" /* yacc.c:1646 */ +#line 1418 "dapy.c" /* yacc.c:1646 */ break; case 8: #line 67 "dap.y" /* yacc.c:1646 */ {dap_tagparse(parsestate,SCAN_ATTR);} -#line 1424 "dap.tab.c" /* yacc.c:1646 */ +#line 1424 "dapy.c" /* yacc.c:1646 */ break; case 9: #line 71 "dap.y" /* yacc.c:1646 */ {dap_tagparse(parsestate,SCAN_ERROR);} -#line 1430 "dap.tab.c" /* yacc.c:1646 */ +#line 1430 "dapy.c" /* yacc.c:1646 */ break; case 10: #line 76 "dap.y" /* yacc.c:1646 */ {dap_datasetbody(parsestate,(yyvsp[-1]),(yyvsp[-3]));} -#line 1436 "dap.tab.c" /* yacc.c:1646 */ +#line 1436 "dapy.c" /* yacc.c:1646 */ break; case 11: #line 81 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_declarations(parsestate,null,null);} -#line 1442 "dap.tab.c" /* yacc.c:1646 */ +#line 1442 "dapy.c" /* yacc.c:1646 */ break; case 12: #line 82 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_declarations(parsestate,(yyvsp[-1]),(yyvsp[0]));} -#line 1448 "dap.tab.c" /* yacc.c:1646 */ +#line 1448 "dapy.c" /* yacc.c:1646 */ break; case 13: #line 89 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_makebase(parsestate,(yyvsp[-2]),(yyvsp[-3]),(yyvsp[-1]));} -#line 1454 "dap.tab.c" /* yacc.c:1646 */ +#line 1454 "dapy.c" /* yacc.c:1646 */ break; case 14: #line 91 "dap.y" /* yacc.c:1646 */ {if(((yyval)=dap_makestructure(parsestate,(yyvsp[-2]),(yyvsp[-1]),(yyvsp[-4])))==null) {YYABORT;}} -#line 1460 "dap.tab.c" /* yacc.c:1646 */ +#line 1460 "dapy.c" /* yacc.c:1646 */ break; case 15: #line 93 "dap.y" /* yacc.c:1646 */ {if(((yyval)=dap_makesequence(parsestate,(yyvsp[-1]),(yyvsp[-3])))==null) {YYABORT;}} -#line 1466 "dap.tab.c" /* yacc.c:1646 */ +#line 1466 "dapy.c" /* yacc.c:1646 */ break; case 16: #line 96 "dap.y" /* yacc.c:1646 */ {if(((yyval)=dap_makegrid(parsestate,(yyvsp[-1]),(yyvsp[-6]),(yyvsp[-3])))==null) {YYABORT;}} -#line 1472 "dap.tab.c" /* yacc.c:1646 */ +#line 1472 "dapy.c" /* yacc.c:1646 */ break; case 17: #line 98 "dap.y" /* yacc.c:1646 */ {dapsemanticerror(parsestate,OC_EBADTYPE,"Unrecognized type"); YYABORT;} -#line 1478 "dap.tab.c" /* yacc.c:1646 */ +#line 1478 "dapy.c" /* yacc.c:1646 */ break; case 18: #line 103 "dap.y" /* yacc.c:1646 */ {(yyval)=(Object)SCAN_BYTE;} -#line 1484 "dap.tab.c" /* yacc.c:1646 */ +#line 1484 "dapy.c" /* yacc.c:1646 */ break; case 19: #line 104 "dap.y" /* yacc.c:1646 */ {(yyval)=(Object)SCAN_INT16;} -#line 1490 "dap.tab.c" /* yacc.c:1646 */ +#line 1490 "dapy.c" /* yacc.c:1646 */ break; case 20: #line 105 "dap.y" /* yacc.c:1646 */ {(yyval)=(Object)SCAN_UINT16;} -#line 1496 "dap.tab.c" /* yacc.c:1646 */ +#line 1496 "dapy.c" /* yacc.c:1646 */ break; case 21: #line 106 "dap.y" /* yacc.c:1646 */ {(yyval)=(Object)SCAN_INT32;} -#line 1502 "dap.tab.c" /* yacc.c:1646 */ +#line 1502 "dapy.c" /* yacc.c:1646 */ break; case 22: #line 107 "dap.y" /* yacc.c:1646 */ {(yyval)=(Object)SCAN_UINT32;} -#line 1508 "dap.tab.c" /* yacc.c:1646 */ +#line 1508 "dapy.c" /* yacc.c:1646 */ break; case 23: #line 108 "dap.y" /* yacc.c:1646 */ {(yyval)=(Object)SCAN_FLOAT32;} -#line 1514 "dap.tab.c" /* yacc.c:1646 */ +#line 1514 "dapy.c" /* yacc.c:1646 */ break; case 24: #line 109 "dap.y" /* yacc.c:1646 */ {(yyval)=(Object)SCAN_FLOAT64;} -#line 1520 "dap.tab.c" /* yacc.c:1646 */ +#line 1520 "dapy.c" /* yacc.c:1646 */ break; case 25: #line 110 "dap.y" /* yacc.c:1646 */ {(yyval)=(Object)SCAN_URL;} -#line 1526 "dap.tab.c" /* yacc.c:1646 */ +#line 1526 "dapy.c" /* yacc.c:1646 */ break; case 26: #line 111 "dap.y" /* yacc.c:1646 */ {(yyval)=(Object)SCAN_STRING;} -#line 1532 "dap.tab.c" /* yacc.c:1646 */ +#line 1532 "dapy.c" /* yacc.c:1646 */ break; case 27: #line 115 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_arraydecls(parsestate,null,null);} -#line 1538 "dap.tab.c" /* yacc.c:1646 */ +#line 1538 "dapy.c" /* yacc.c:1646 */ break; case 28: #line 116 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_arraydecls(parsestate,(yyvsp[-1]),(yyvsp[0]));} -#line 1544 "dap.tab.c" /* yacc.c:1646 */ +#line 1544 "dapy.c" /* yacc.c:1646 */ break; case 29: #line 120 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_arraydecl(parsestate,null,(yyvsp[-1]));} -#line 1550 "dap.tab.c" /* yacc.c:1646 */ +#line 1550 "dapy.c" /* yacc.c:1646 */ break; case 30: #line 121 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_arraydecl(parsestate,null,(yyvsp[-1]));} -#line 1556 "dap.tab.c" /* yacc.c:1646 */ +#line 1556 "dapy.c" /* yacc.c:1646 */ break; case 31: #line 122 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_arraydecl(parsestate,(yyvsp[-3]),(yyvsp[-1]));} -#line 1562 "dap.tab.c" /* yacc.c:1646 */ +#line 1562 "dapy.c" /* yacc.c:1646 */ break; case 32: #line 124 "dap.y" /* yacc.c:1646 */ {dapsemanticerror(parsestate,OC_EDIMSIZE,"Illegal dimension declaration"); YYABORT;} -#line 1568 "dap.tab.c" /* yacc.c:1646 */ +#line 1568 "dapy.c" /* yacc.c:1646 */ break; case 33: #line 128 "dap.y" /* yacc.c:1646 */ {(yyval)=(yyvsp[0]);} -#line 1574 "dap.tab.c" /* yacc.c:1646 */ +#line 1574 "dapy.c" /* yacc.c:1646 */ break; case 34: #line 130 "dap.y" /* yacc.c:1646 */ {dapsemanticerror(parsestate,OC_EDDS,"Illegal dataset declaration"); YYABORT;} -#line 1580 "dap.tab.c" /* yacc.c:1646 */ +#line 1580 "dapy.c" /* yacc.c:1646 */ break; case 35: #line 133 "dap.y" /* yacc.c:1646 */ {(yyval)=(yyvsp[0]);} -#line 1586 "dap.tab.c" /* yacc.c:1646 */ +#line 1586 "dapy.c" /* yacc.c:1646 */ break; case 36: #line 136 "dap.y" /* yacc.c:1646 */ {dap_attributebody(parsestate,(yyvsp[-1]));} -#line 1592 "dap.tab.c" /* yacc.c:1646 */ +#line 1592 "dapy.c" /* yacc.c:1646 */ break; case 37: #line 138 "dap.y" /* yacc.c:1646 */ {dapsemanticerror(parsestate,OC_EDAS,"Illegal DAS body"); YYABORT;} -#line 1598 "dap.tab.c" /* yacc.c:1646 */ +#line 1598 "dapy.c" /* yacc.c:1646 */ break; case 38: #line 142 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_attrlist(parsestate,null,null);} -#line 1604 "dap.tab.c" /* yacc.c:1646 */ +#line 1604 "dapy.c" /* yacc.c:1646 */ break; case 39: #line 143 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_attrlist(parsestate,(yyvsp[-1]),(yyvsp[0]));} -#line 1610 "dap.tab.c" /* yacc.c:1646 */ +#line 1610 "dapy.c" /* yacc.c:1646 */ break; case 40: #line 147 "dap.y" /* yacc.c:1646 */ {(yyval)=null;} -#line 1616 "dap.tab.c" /* yacc.c:1646 */ +#line 1616 "dapy.c" /* yacc.c:1646 */ break; case 41: #line 149 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_attribute(parsestate,(yyvsp[-2]),(yyvsp[-1]),(Object)SCAN_BYTE);} -#line 1622 "dap.tab.c" /* yacc.c:1646 */ +#line 1622 "dapy.c" /* yacc.c:1646 */ break; case 42: #line 151 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_attribute(parsestate,(yyvsp[-2]),(yyvsp[-1]),(Object)SCAN_INT16);} -#line 1628 "dap.tab.c" /* yacc.c:1646 */ +#line 1628 "dapy.c" /* yacc.c:1646 */ break; case 43: #line 153 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_attribute(parsestate,(yyvsp[-2]),(yyvsp[-1]),(Object)SCAN_UINT16);} -#line 1634 "dap.tab.c" /* yacc.c:1646 */ +#line 1634 "dapy.c" /* yacc.c:1646 */ break; case 44: #line 155 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_attribute(parsestate,(yyvsp[-2]),(yyvsp[-1]),(Object)SCAN_INT32);} -#line 1640 "dap.tab.c" /* yacc.c:1646 */ +#line 1640 "dapy.c" /* yacc.c:1646 */ break; case 45: #line 157 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_attribute(parsestate,(yyvsp[-2]),(yyvsp[-1]),(Object)SCAN_UINT32);} -#line 1646 "dap.tab.c" /* yacc.c:1646 */ +#line 1646 "dapy.c" /* yacc.c:1646 */ break; case 46: #line 159 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_attribute(parsestate,(yyvsp[-2]),(yyvsp[-1]),(Object)SCAN_FLOAT32);} -#line 1652 "dap.tab.c" /* yacc.c:1646 */ +#line 1652 "dapy.c" /* yacc.c:1646 */ break; case 47: #line 161 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_attribute(parsestate,(yyvsp[-2]),(yyvsp[-1]),(Object)SCAN_FLOAT64);} -#line 1658 "dap.tab.c" /* yacc.c:1646 */ +#line 1658 "dapy.c" /* yacc.c:1646 */ break; case 48: #line 163 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_attribute(parsestate,(yyvsp[-2]),(yyvsp[-1]),(Object)SCAN_STRING);} -#line 1664 "dap.tab.c" /* yacc.c:1646 */ +#line 1664 "dapy.c" /* yacc.c:1646 */ break; case 49: #line 165 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_attribute(parsestate,(yyvsp[-2]),(yyvsp[-1]),(Object)SCAN_URL);} -#line 1670 "dap.tab.c" /* yacc.c:1646 */ +#line 1670 "dapy.c" /* yacc.c:1646 */ break; case 50: #line 166 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_attrset(parsestate,(yyvsp[-3]),(yyvsp[-1]));} -#line 1676 "dap.tab.c" /* yacc.c:1646 */ +#line 1676 "dapy.c" /* yacc.c:1646 */ break; case 51: #line 168 "dap.y" /* yacc.c:1646 */ {dapsemanticerror(parsestate,OC_EDAS,"Illegal attribute"); YYABORT;} -#line 1682 "dap.tab.c" /* yacc.c:1646 */ +#line 1682 "dapy.c" /* yacc.c:1646 */ break; case 52: #line 172 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_attrvalue(parsestate,null,(yyvsp[0]),(Object)SCAN_BYTE);} -#line 1688 "dap.tab.c" /* yacc.c:1646 */ +#line 1688 "dapy.c" /* yacc.c:1646 */ break; case 53: #line 174 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_attrvalue(parsestate,(yyvsp[-2]),(yyvsp[0]),(Object)SCAN_BYTE);} -#line 1694 "dap.tab.c" /* yacc.c:1646 */ +#line 1694 "dapy.c" /* yacc.c:1646 */ break; case 54: #line 177 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_attrvalue(parsestate,null,(yyvsp[0]),(Object)SCAN_INT16);} -#line 1700 "dap.tab.c" /* yacc.c:1646 */ +#line 1700 "dapy.c" /* yacc.c:1646 */ break; case 55: #line 179 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_attrvalue(parsestate,(yyvsp[-2]),(yyvsp[0]),(Object)SCAN_INT16);} -#line 1706 "dap.tab.c" /* yacc.c:1646 */ +#line 1706 "dapy.c" /* yacc.c:1646 */ break; case 56: #line 182 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_attrvalue(parsestate,null,(yyvsp[0]),(Object)SCAN_UINT16);} -#line 1712 "dap.tab.c" /* yacc.c:1646 */ +#line 1712 "dapy.c" /* yacc.c:1646 */ break; case 57: #line 184 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_attrvalue(parsestate,(yyvsp[-2]),(yyvsp[0]),(Object)SCAN_UINT16);} -#line 1718 "dap.tab.c" /* yacc.c:1646 */ +#line 1718 "dapy.c" /* yacc.c:1646 */ break; case 58: #line 187 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_attrvalue(parsestate,null,(yyvsp[0]),(Object)SCAN_INT32);} -#line 1724 "dap.tab.c" /* yacc.c:1646 */ +#line 1724 "dapy.c" /* yacc.c:1646 */ break; case 59: #line 189 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_attrvalue(parsestate,(yyvsp[-2]),(yyvsp[0]),(Object)SCAN_INT32);} -#line 1730 "dap.tab.c" /* yacc.c:1646 */ +#line 1730 "dapy.c" /* yacc.c:1646 */ break; case 60: #line 192 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_attrvalue(parsestate,null,(yyvsp[0]),(Object)SCAN_UINT32);} -#line 1736 "dap.tab.c" /* yacc.c:1646 */ +#line 1736 "dapy.c" /* yacc.c:1646 */ break; case 61: #line 193 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_attrvalue(parsestate,(yyvsp[-2]),(yyvsp[0]),(Object)SCAN_UINT32);} -#line 1742 "dap.tab.c" /* yacc.c:1646 */ +#line 1742 "dapy.c" /* yacc.c:1646 */ break; case 62: #line 196 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_attrvalue(parsestate,null,(yyvsp[0]),(Object)SCAN_FLOAT32);} -#line 1748 "dap.tab.c" /* yacc.c:1646 */ +#line 1748 "dapy.c" /* yacc.c:1646 */ break; case 63: #line 197 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_attrvalue(parsestate,(yyvsp[-2]),(yyvsp[0]),(Object)SCAN_FLOAT32);} -#line 1754 "dap.tab.c" /* yacc.c:1646 */ +#line 1754 "dapy.c" /* yacc.c:1646 */ break; case 64: #line 200 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_attrvalue(parsestate,null,(yyvsp[0]),(Object)SCAN_FLOAT64);} -#line 1760 "dap.tab.c" /* yacc.c:1646 */ +#line 1760 "dapy.c" /* yacc.c:1646 */ break; case 65: #line 201 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_attrvalue(parsestate,(yyvsp[-2]),(yyvsp[0]),(Object)SCAN_FLOAT64);} -#line 1766 "dap.tab.c" /* yacc.c:1646 */ +#line 1766 "dapy.c" /* yacc.c:1646 */ break; case 66: #line 204 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_attrvalue(parsestate,null,(yyvsp[0]),(Object)SCAN_STRING);} -#line 1772 "dap.tab.c" /* yacc.c:1646 */ +#line 1772 "dapy.c" /* yacc.c:1646 */ break; case 67: #line 205 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_attrvalue(parsestate,(yyvsp[-2]),(yyvsp[0]),(Object)SCAN_STRING);} -#line 1778 "dap.tab.c" /* yacc.c:1646 */ +#line 1778 "dapy.c" /* yacc.c:1646 */ break; case 68: #line 209 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_attrvalue(parsestate,null,(yyvsp[0]),(Object)SCAN_URL);} -#line 1784 "dap.tab.c" /* yacc.c:1646 */ +#line 1784 "dapy.c" /* yacc.c:1646 */ break; case 69: #line 210 "dap.y" /* yacc.c:1646 */ {(yyval)=dap_attrvalue(parsestate,(yyvsp[-2]),(yyvsp[0]),(Object)SCAN_URL);} -#line 1790 "dap.tab.c" /* yacc.c:1646 */ +#line 1790 "dapy.c" /* yacc.c:1646 */ break; case 70: #line 214 "dap.y" /* yacc.c:1646 */ {(yyval)=(yyvsp[0]);} -#line 1796 "dap.tab.c" /* yacc.c:1646 */ +#line 1796 "dapy.c" /* yacc.c:1646 */ break; case 71: #line 218 "dap.y" /* yacc.c:1646 */ {(yyval)=(yyvsp[0]);} -#line 1802 "dap.tab.c" /* yacc.c:1646 */ +#line 1802 "dapy.c" /* yacc.c:1646 */ break; case 72: #line 219 "dap.y" /* yacc.c:1646 */ {(yyval)=(yyvsp[0]);} -#line 1808 "dap.tab.c" /* yacc.c:1646 */ +#line 1808 "dapy.c" /* yacc.c:1646 */ break; case 73: #line 230 "dap.y" /* yacc.c:1646 */ {(yyval)=(yyvsp[-1]); (yyval)=(yyvsp[0]); (yyval)=null;} -#line 1814 "dap.tab.c" /* yacc.c:1646 */ +#line 1814 "dapy.c" /* yacc.c:1646 */ break; case 74: #line 235 "dap.y" /* yacc.c:1646 */ {dap_errorbody(parsestate,(yyvsp[-5]),(yyvsp[-4]),(yyvsp[-3]),(yyvsp[-2]));} -#line 1820 "dap.tab.c" /* yacc.c:1646 */ +#line 1820 "dapy.c" /* yacc.c:1646 */ break; case 75: #line 238 "dap.y" /* yacc.c:1646 */ {(yyval)=null;} -#line 1826 "dap.tab.c" /* yacc.c:1646 */ +#line 1826 "dapy.c" /* yacc.c:1646 */ break; case 76: #line 238 "dap.y" /* yacc.c:1646 */ {(yyval)=(yyvsp[-1]);} -#line 1832 "dap.tab.c" /* yacc.c:1646 */ +#line 1832 "dapy.c" /* yacc.c:1646 */ break; case 77: #line 239 "dap.y" /* yacc.c:1646 */ {(yyval)=null;} -#line 1838 "dap.tab.c" /* yacc.c:1646 */ +#line 1838 "dapy.c" /* yacc.c:1646 */ break; case 78: #line 239 "dap.y" /* yacc.c:1646 */ {(yyval)=(yyvsp[-1]);} -#line 1844 "dap.tab.c" /* yacc.c:1646 */ +#line 1844 "dapy.c" /* yacc.c:1646 */ break; case 79: #line 240 "dap.y" /* yacc.c:1646 */ {(yyval)=null;} -#line 1850 "dap.tab.c" /* yacc.c:1646 */ +#line 1850 "dapy.c" /* yacc.c:1646 */ break; case 80: #line 240 "dap.y" /* yacc.c:1646 */ {(yyval)=(yyvsp[-1]);} -#line 1856 "dap.tab.c" /* yacc.c:1646 */ +#line 1856 "dapy.c" /* yacc.c:1646 */ break; case 81: #line 241 "dap.y" /* yacc.c:1646 */ {(yyval)=null;} -#line 1862 "dap.tab.c" /* yacc.c:1646 */ +#line 1862 "dapy.c" /* yacc.c:1646 */ break; case 82: #line 241 "dap.y" /* yacc.c:1646 */ {(yyval)=(yyvsp[-1]);} -#line 1868 "dap.tab.c" /* yacc.c:1646 */ +#line 1868 "dapy.c" /* yacc.c:1646 */ break; case 83: #line 247 "dap.y" /* yacc.c:1646 */ {(yyval)=dapdecode(parsestate->lexstate,(yyvsp[0]));} -#line 1874 "dap.tab.c" /* yacc.c:1646 */ +#line 1874 "dapy.c" /* yacc.c:1646 */ break; case 84: #line 248 "dap.y" /* yacc.c:1646 */ {(yyval)=strdup((yyvsp[0]));} -#line 1880 "dap.tab.c" /* yacc.c:1646 */ +#line 1880 "dapy.c" /* yacc.c:1646 */ break; case 85: #line 249 "dap.y" /* yacc.c:1646 */ {(yyval)=strdup((yyvsp[0]));} -#line 1886 "dap.tab.c" /* yacc.c:1646 */ +#line 1886 "dapy.c" /* yacc.c:1646 */ break; case 86: #line 250 "dap.y" /* yacc.c:1646 */ {(yyval)=strdup((yyvsp[0]));} -#line 1892 "dap.tab.c" /* yacc.c:1646 */ +#line 1892 "dapy.c" /* yacc.c:1646 */ break; case 87: #line 251 "dap.y" /* yacc.c:1646 */ {(yyval)=strdup((yyvsp[0]));} -#line 1898 "dap.tab.c" /* yacc.c:1646 */ +#line 1898 "dapy.c" /* yacc.c:1646 */ break; case 88: #line 252 "dap.y" /* yacc.c:1646 */ {(yyval)=strdup((yyvsp[0]));} -#line 1904 "dap.tab.c" /* yacc.c:1646 */ +#line 1904 "dapy.c" /* yacc.c:1646 */ break; case 89: #line 253 "dap.y" /* yacc.c:1646 */ {(yyval)=strdup((yyvsp[0]));} -#line 1910 "dap.tab.c" /* yacc.c:1646 */ +#line 1910 "dapy.c" /* yacc.c:1646 */ break; case 90: #line 254 "dap.y" /* yacc.c:1646 */ {(yyval)=strdup((yyvsp[0]));} -#line 1916 "dap.tab.c" /* yacc.c:1646 */ +#line 1916 "dapy.c" /* yacc.c:1646 */ break; case 91: #line 255 "dap.y" /* yacc.c:1646 */ {(yyval)=strdup((yyvsp[0]));} -#line 1922 "dap.tab.c" /* yacc.c:1646 */ +#line 1922 "dapy.c" /* yacc.c:1646 */ break; case 92: #line 256 "dap.y" /* yacc.c:1646 */ {(yyval)=strdup((yyvsp[0]));} -#line 1928 "dap.tab.c" /* yacc.c:1646 */ +#line 1928 "dapy.c" /* yacc.c:1646 */ break; case 93: #line 257 "dap.y" /* yacc.c:1646 */ {(yyval)=strdup((yyvsp[0]));} -#line 1934 "dap.tab.c" /* yacc.c:1646 */ +#line 1934 "dapy.c" /* yacc.c:1646 */ break; case 94: #line 258 "dap.y" /* yacc.c:1646 */ {(yyval)=strdup((yyvsp[0]));} -#line 1940 "dap.tab.c" /* yacc.c:1646 */ +#line 1940 "dapy.c" /* yacc.c:1646 */ break; case 95: #line 259 "dap.y" /* yacc.c:1646 */ {(yyval)=strdup((yyvsp[0]));} -#line 1946 "dap.tab.c" /* yacc.c:1646 */ +#line 1946 "dapy.c" /* yacc.c:1646 */ break; case 96: #line 260 "dap.y" /* yacc.c:1646 */ {(yyval)=strdup((yyvsp[0]));} -#line 1952 "dap.tab.c" /* yacc.c:1646 */ +#line 1952 "dapy.c" /* yacc.c:1646 */ break; case 97: #line 261 "dap.y" /* yacc.c:1646 */ {(yyval)=strdup((yyvsp[0]));} -#line 1958 "dap.tab.c" /* yacc.c:1646 */ +#line 1958 "dapy.c" /* yacc.c:1646 */ break; case 98: #line 262 "dap.y" /* yacc.c:1646 */ {(yyval)=strdup((yyvsp[0]));} -#line 1964 "dap.tab.c" /* yacc.c:1646 */ +#line 1964 "dapy.c" /* yacc.c:1646 */ break; case 99: #line 263 "dap.y" /* yacc.c:1646 */ {(yyval)=strdup((yyvsp[0]));} -#line 1970 "dap.tab.c" /* yacc.c:1646 */ +#line 1970 "dapy.c" /* yacc.c:1646 */ break; case 100: #line 264 "dap.y" /* yacc.c:1646 */ {(yyval)=strdup((yyvsp[0]));} -#line 1976 "dap.tab.c" /* yacc.c:1646 */ +#line 1976 "dapy.c" /* yacc.c:1646 */ break; case 101: #line 265 "dap.y" /* yacc.c:1646 */ {(yyval)=strdup((yyvsp[0]));} -#line 1982 "dap.tab.c" /* yacc.c:1646 */ +#line 1982 "dapy.c" /* yacc.c:1646 */ break; case 102: #line 266 "dap.y" /* yacc.c:1646 */ {(yyval)=strdup((yyvsp[0]));} -#line 1988 "dap.tab.c" /* yacc.c:1646 */ +#line 1988 "dapy.c" /* yacc.c:1646 */ break; case 103: #line 267 "dap.y" /* yacc.c:1646 */ {(yyval)=strdup((yyvsp[0]));} -#line 1994 "dap.tab.c" /* yacc.c:1646 */ +#line 1994 "dapy.c" /* yacc.c:1646 */ break; case 104: #line 268 "dap.y" /* yacc.c:1646 */ {(yyval)=strdup((yyvsp[0]));} -#line 2000 "dap.tab.c" /* yacc.c:1646 */ +#line 2000 "dapy.c" /* yacc.c:1646 */ break; case 105: #line 269 "dap.y" /* yacc.c:1646 */ {(yyval)=strdup((yyvsp[0]));} -#line 2006 "dap.tab.c" /* yacc.c:1646 */ +#line 2006 "dapy.c" /* yacc.c:1646 */ break; case 106: #line 270 "dap.y" /* yacc.c:1646 */ {(yyval)=strdup((yyvsp[0]));} -#line 2012 "dap.tab.c" /* yacc.c:1646 */ +#line 2012 "dapy.c" /* yacc.c:1646 */ break; -#line 2016 "dap.tab.c" /* yacc.c:1646 */ +#line 2016 "dapy.c" /* yacc.c:1646 */ default: break; } /* User semantic actions sometimes alter yychar, and that requires diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index d3e453ab24..7024bf1195 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -6,54 +6,76 @@ # See netcdf-c/COPYRIGHT file for more info. SET(CMAKE_BUILD_TYPE "") -CONFIGURE_FILE(H5Znoop.c ${CMAKE_CURRENT_BINARY_DIR}/H5Znoop1.c COPYONLY) +FILE(READ H5Znoop.c NOOP_SOURCE) +FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/H5Znoop1.c "#define NOOP_INSTANCE 1\n") +FILE(APPEND ${CMAKE_CURRENT_BINARY_DIR}/H5Znoop1.c "${NOOP_SOURCE}") -SET(libh5bzip2_SOURCES blocksort.c huffman.c crctable.c randtable.c compress.c decompress.c bzlib.c H5Zbzip2.c) +SET(h5bzip2_SOURCES blocksort.c huffman.c crctable.c randtable.c compress.c decompress.c bzlib.c H5Zbzip2.c ${NCJ}) -SET(libmisc_SOURCES H5Zmisc.c H5Zutil.c h5misc.h) +SET(h5misc_SOURCES H5Zmisc.c H5Zutil.c h5misc.h) -SET(libnoop_SOURCES H5Znoop.c H5Zutil.c h5noop.h) -SET(libnoop1_SOURCES ${CMAKE_CURRENT_BINARY_DIR}/H5Znoop1.c H5Zutil.c h5noop.h) +SET(h5noop_SOURCES H5Znoop.c H5Zutil.c h5noop.h) +SET_SOURCE_FILES_PROPERTIES(H5Znoop.c PROPERTIES COMPILE_OPTIONS -DNOOP_INSTANCE=0) + +SET(h5noop1_SOURCES ${CMAKE_CURRENT_BINARY_DIR}/H5Znoop1.c H5Zutil.c h5noop.h) +SET_SOURCE_FILES_PROPERTIES(${CMAKE_CURRENT_BINARY_DIR}/H5Znoop1.c PROPERTIES COMPILE_OPTIONS -DNOOP_INSTANCE=1) + +SET(h5shuffle_SOURCES H5Zshuffle.c) +SET(h5fletcher32_SOURCES H5Zfletcher32.c H5checksum.c) +SET(h5deflate_SOURCES H5Zdeflate.c) + +SET(nczmisc_SOURCES NCZmisc.c) +SET(nczdefaults_SOURCES NCZdefaults.c) IF(ENABLE_FILTER_TESTING) IF(BUILD_UTILITIES) # LDFLAGS = -module -avoid-version -shared -export-dynamic -no-undefined -IF(MSVC) -SET( CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") -SET( CMAKE_LIBRARY_OUTPUT_DIRECTORY_DEBUG "${CMAKE_CURRENT_BINARY_DIR}") -SET( CMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE "${CMAKE_CURRENT_BINARY_DIR}") -ENDIF() +SET(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") +SET(CMAKE_LIBRARY_OUTPUT_DIRECTORY_DEBUG "${CMAKE_CURRENT_BINARY_DIR}") +SET(CMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE "${CMAKE_CURRENT_BINARY_DIR}") + +MACRO(buildplugin TARGET TARGETLIB) + ADD_LIBRARY(${TARGET} MODULE ${${TARGET}_SOURCES}) + SET_TARGET_PROPERTIES(${TARGET} PROPERTIES LIBRARY_OUTPUT_NAME ${TARGETLIB}) + SET_TARGET_PROPERTIES(${TARGET} PROPERTIES ARCHIVE_OUTPUT_NAME ${TARGETLIB}) + SET_TARGET_PROPERTIES(${TARGET} PROPERTIES RUNTIME_OUTPUT_NAME ${TARGETLIB}) + TARGET_LINK_LIBRARIES(${TARGET} ${ALL_TLL_LIBS};${ARGN}) + IF(MSVC) + target_compile_options(${TARGET} PRIVATE /Zi) + # Tell linker to include symbol data + set_target_properties(${TARGET} PROPERTIES LINK_FLAGS "/INCREMENTAL:NO /DEBUG /OPT:REF /OPT:ICF") + # Set file name & location + set_target_properties(${TARGET} PROPERTIES COMPILE_PDB_NAME ${TARGET} COMPILE_PDB_OUTPUT_DIR ${CMAKE_BINARY_DIR}) + ENDIF() +ENDMACRO() -# Note we use name test_bzip2 instead of bzip2 to avoid logical +# Note we use name h5bzip2 instead of bzip2 to avoid logical # target name clash with examples/C/hdf5plugins -ADD_LIBRARY(test_bzip2 MODULE ${libh5bzip2_SOURCES}) -SET_TARGET_PROPERTIES(test_bzip2 PROPERTIES OUTPUT_NAME "bzip2") -SET_TARGET_PROPERTIES(test_bzip2 PROPERTIES LIBRARY_OUTPUT_NAME "h5bzip2") -SET_TARGET_PROPERTIES(test_bzip2 PROPERTIES ARCHIVE_OUTPUT_NAME "h5bzip2") -SET_TARGET_PROPERTIES(test_bzip2 PROPERTIES RUNTIME_OUTPUT_NAME "h5bzip2") -TARGET_LINK_LIBRARIES(test_bzip2 ${ALL_TLL_LIBS}) - -ADD_LIBRARY(misc MODULE ${libmisc_SOURCES}) -SET_TARGET_PROPERTIES(misc PROPERTIES LIBRARY_OUTPUT_NAME "misc") -SET_TARGET_PROPERTIES(misc PROPERTIES ARCHIVE_OUTPUT_NAME "misc") -SET_TARGET_PROPERTIES(misc PROPERTIES RUNTIME_OUTPUT_NAME "misc") -TARGET_LINK_LIBRARIES(misc ${ALL_TLL_LIBS}) +buildplugin(h5bzip2 "h5bzip2") +SET_TARGET_PROPERTIES(h5bzip2 PROPERTIES OUTPUT_NAME "bzip2") -SET_SOURCE_FILES_PROPERTIES(H5Znoop.c PROPERTIES COMPILE_OPTIONS -DNOOP_INSTANCE=0) -ADD_LIBRARY(noop MODULE ${libnoop_SOURCES}) -SET_TARGET_PROPERTIES(noop PROPERTIES LIBRARY_OUTPUT_NAME "noop") -SET_TARGET_PROPERTIES(noop PROPERTIES ARCHIVE_OUTPUT_NAME "noop") -SET_TARGET_PROPERTIES(noop PROPERTIES RUNTIME_OUTPUT_NAME "noop") -TARGET_LINK_LIBRARIES(noop ${ALL_TLL_LIBS}) +buildplugin(h5misc "h5misc") +buildplugin(h5noop "h5noop") +buildplugin(h5noop1 "h5noop1") -SET_SOURCE_FILES_PROPERTIES(${CMAKE_CURRENT_BINARY_DIR}/H5Znoop1.c PROPERTIES COMPILE_OPTIONS -DNOOP_INSTANCE=1) -ADD_LIBRARY(noop1 MODULE ${libnoop1_SOURCES}) -SET_TARGET_PROPERTIES(noop1 PROPERTIES LIBRARY_OUTPUT_NAME "noop1") -SET_TARGET_PROPERTIES(noop1 PROPERTIES ARCHIVE_OUTPUT_NAME "noop1") -SET_TARGET_PROPERTIES(noop1 PROPERTIES RUNTIME_OUTPUT_NAME "noop1") -TARGET_LINK_LIBRARIES(noop1 ${ALL_TLL_LIBS}) +buildplugin(h5shuffle "h5shuffle") +buildplugin(h5fletcher32 "h5fletcher32") +buildplugin(h5deflate "h5deflate") + +buildplugin(nczmisc "nczmisc") +buildplugin(nczdefaults "nczdefaults" netcdf) + +IF(ENABLE_BLOSC) +SET(h5blosc_SOURCES H5Zblosc.c) +buildplugin(h5blosc "h5blosc" netcdf;${Blosc_LIBRARIES}) +ENDIF() + +IF(ENABLE_SZIP) +SET(h5szip_SOURCES H5Zszip.c H5Zszip.h) +buildplugin(h5szip "h5szip" ${Szip_LIBRARIES}) +ENDIF() ENDIF(BUILD_UTILITIES) ENDIF(ENABLE_FILTER_TESTING) diff --git a/plugins/H5Zblosc.c b/plugins/H5Zblosc.c new file mode 100755 index 0000000000..de2a97edc8 --- /dev/null +++ b/plugins/H5Zblosc.c @@ -0,0 +1,613 @@ +/* +Blosc - A blocking, shuffling and lossless compression library + +Copyright (C) 2009-2015 Francesc Alted + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +Blosc for HDF5 - An HDF5 filter that uses the Blosc compressor. + +Copyright (C) 2009-2015 Francesc Alted + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +Copyright Notice and Statement for the h5py Project + +Copyright (c) 2008 Andrew Collette +http://h5py.alfven.org +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +a. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +b. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the + distribution. + +c. Neither the name of the author nor the names of contributors may + be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +/* + Copyright (C) 2010-2016 Francesc Alted + http://blosc.org + License: MIT (see LICENSE.txt) + + Filter program that allows the use of the Blosc filter in HDF5. + + This is based on the LZF filter interface (http://h5py.alfven.org) + by Andrew Collette. + +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#include "netcdf_filter_build.h" +#include + +#include "H5Zblosc.h" + +#ifdef USE_HDF5 +#include +#endif + +#define BLOSCCTX + +#ifdef USE_HDF5 +#if defined(__GNUC__) +#define PUSH_ERR(func, minor, str, ...) H5Epush1(__FILE__, func, __LINE__, H5_VERS_MAJOR, minor, str) +#elif defined(_MSC_VER) +#define PUSH_ERR(func, minor, str, ...) H5Epush1(__FILE__, func, __LINE__, H5_VERS_MAJOR, minor, str) +#else +/* This version is portable but it's better to use compiler-supported + approaches for handling the trailing comma issue when possible. */ +#define PUSH_ERR(func, minor, str, ...) H5Epush1(__FILE__, func, __LINE__, H5_VERS_MAJOR, minor, str +) +#endif /* defined(__GNUC__) */ + +#define GET_FILTER(a, b, c, d, e, f, g) H5Pget_filter_by_id1(a,b,c,d,e,f,g) + +#else /*!USE_HDF5*/ +#define PUSH_ERR(f,m,s,...) fprintf(stderr,"%s\n",s) +#endif /*USE_HDF5*/ + +static int h5z_blosc_initialized = 0; + +static size_t blosc_filter(unsigned flags, size_t cd_nelmts, + const unsigned cd_values[], size_t nbytes, + size_t* buf_size, void** buf); + +#ifndef USE_HDF5 +#define blosc_set_local NULL +#else +static herr_t blosc_set_local(hid_t dcpl, hid_t type, hid_t space); + +/* Filter setup. Records the following inside the DCPL: + + 1. If version information is not present, set slots 0 and 1 to the filter + revision and Blosc version, respectively. + + 2. Compute the type size in bytes and store it in slot 2. + + 3. Compute the chunk size in bytes and store it in slot 3. +*/ + +static +herr_t blosc_set_local(hid_t dcpl, hid_t type, hid_t space) +{ + int ndims; + int i; + herr_t r; + + unsigned int typesize, basetypesize; + unsigned int bufsize; + hsize_t chunkdims[32]; + unsigned int flags; + size_t nelements = 8; + unsigned int values[] = {0, 0, 0, 0, 0, 0, 0, 0}; + hid_t super_type; + H5T_class_t classt; + + r = GET_FILTER(dcpl, FILTER_BLOSC, &flags, &nelements, values, 0, NULL); + if (r < 0) return -1; + + if (nelements < 4) nelements = 4; /* First 4 slots reserved. */ + + /* Set Blosc info in first two slots */ + values[0] = FILTER_BLOSC_VERSION; + values[1] = BLOSC_VERSION_FORMAT; + + ndims = H5Pget_chunk(dcpl, 32, chunkdims); + if (ndims < 0) return -1; + if (ndims > 32) { + PUSH_ERR("blosc_set_local", H5E_CALLBACK, "Chunk rank exceeds limit"); + return -1; + } + + typesize = H5Tget_size(type); + if (typesize == 0) return -1; + /* Get the size of the base type, even for ARRAY types */ + classt = H5Tget_class(type); + if (classt == H5T_ARRAY) { + /* Get the array base component */ + super_type = H5Tget_super(type); + basetypesize = H5Tget_size(super_type); + /* Release resources */ + H5Tclose(super_type); + } else { + basetypesize = typesize; + } + + /* Limit large typesizes (they are pretty expensive to shuffle + and, in addition, Blosc does not handle typesizes larger than + 256 bytes). */ + if (basetypesize > BLOSC_MAX_TYPESIZE) basetypesize = 1; + values[2] = basetypesize; + + /* Get the size of the chunk */ + bufsize = typesize; + for (i = 0; i < ndims; i++) { + bufsize *= chunkdims[i]; + } + values[3] = bufsize; + +#ifdef BLOSC_DEBUG + fprintf(stderr, "Blosc: Computed buffer size %d\n", bufsize); +#endif + + r = H5Pmodify_filter(dcpl, FILTER_BLOSC, flags, nelements, values); + if (r < 0) return -1; + + return 1; +} + +#endif /*USE_HDF5*/ + +/* The filter function */ +static +size_t blosc_filter(unsigned flags, size_t cd_nelmts, + const unsigned cd_values[], size_t nbytes, + size_t* buf_size, void** buf) +{ + void* outbuf = NULL; + int status = 0; /* Return code from Blosc routines */ + size_t typesize; + size_t outbuf_size; + int clevel = 5; /* Compression level default */ + int doshuffle = 1; /* Shuffle default */ + int compcode; /* Blosc compressor */ + int code; + char* compname = "blosclz"; /* The compressor by default */ + char* complist = NULL; + + /* Filter params that are always set */ + typesize = cd_values[2]; /* The datatype size */ + outbuf_size = cd_values[3]; /* Precomputed buffer guess */ + /* Optional params */ + if (cd_nelmts >= 5) { + clevel = cd_values[4]; /* The compression level */ + } + if (cd_nelmts >= 6) { + doshuffle = cd_values[5]; /* BLOSC_SHUFFLE, BLOSC_BITSHUFFLE */ + /* bitshuffle is only meant for production in >= 1.8.0 */ +#if ((BLOSC_VERSION_MAJOR <= 1) && (BLOSC_VERSION_MINOR < 8)) + if (doshuffle == BLOSC_BITSHUFFLE) { + PUSH_ERR("blosc_filter", H5E_CALLBACK, + "this Blosc library version is not supported. Please update to >= 1.8"); + goto failed; + } +#endif + } + if (cd_nelmts >= 7) { + compcode = cd_values[6]; /* The Blosc compressor used */ + /* Check that we actually have support for the compressor code */ + complist = blosc_list_compressors(); + code = blosc_compcode_to_compname(compcode, &compname); + if (code == -1) { + char s[4096]; + snprintf(s,sizeof(s),"this Blosc library does not have support for " + "the '%s' compressor, but only for: %s", + compname, complist); + PUSH_ERR("blosc_filter", H5E_CALLBACK,s); + goto failed; + } + } + + /* We're compressing */ + if (!(flags & H5Z_FLAG_REVERSE)) { + + /* Allocate an output buffer exactly as long as the input data; if + the result is larger, we simply return 0. The filter is flagged + as optional, so HDF5 marks the chunk as uncompressed and + proceeds. + */ + + outbuf_size = (*buf_size); + +#ifdef BLOSC_DEBUG + fprintf(stderr, "Blosc: Compress %zd chunk w/buffer %zd\n", + nbytes, outbuf_size); +#endif + + outbuf = malloc(outbuf_size); + + if (outbuf == NULL) { + PUSH_ERR("blosc_filter", H5E_CALLBACK, + "Can't allocate compression buffer"); + goto failed; + } + +#ifdef BLOSCCTX + blosc_set_compressor(compname); + status = blosc_compress(clevel, doshuffle, typesize, nbytes, *buf, outbuf, nbytes); +#else + status = blosc_compress_ctx(clevel, doshuffle, typesize, nbytes, *buf, outbut, nbytes, + compname, /*blocksize*/0, /*no. thredds*/0); +#endif + + if (status < 0) { + PUSH_ERR("blosc_filter", H5E_CALLBACK, "Blosc compression error"); + goto failed; + } + + /* We're decompressing */ + } else { + /* declare dummy variables */ + size_t cbytes, blocksize; + + if(outbuf) { + free(outbuf); + outbuf = NULL; + } + + /* Extract the exact outbuf_size from the buffer header. + * + * NOTE: the guess value got from "cd_values" corresponds to the + * uncompressed chunk size but it should not be used in a general + * cases since other filters in the pipeline can modify the buffere + * size. + */ + blosc_cbuffer_sizes(*buf, &outbuf_size, &cbytes, &blocksize); + +#ifdef BLOSC_DEBUG + fprintf(stderr, "Blosc: Decompress %zd chunk w/buffer %zd\n", nbytes, outbuf_size); +#endif + + outbuf = malloc(outbuf_size); + + if (outbuf == NULL) { + PUSH_ERR("blosc_filter", H5E_CALLBACK, "Can't allocate decompression buffer"); + goto failed; + } + +#ifdef BLOSCCTX + status = blosc_decompress(*buf, outbuf, outbuf_size); +#else + status = blosc_decompress_ctx(*buf, outbuf, outbuf_size, /*no. thredds*/0); +#endif + + if (status <= 0) { /* decompression failed */ + PUSH_ERR("blosc_filter", H5E_CALLBACK, "Blosc decompression error"); + goto failed; + } /* if !status */ + + } /* compressing vs decompressing */ + + if (status != 0) { + free(*buf); + *buf = outbuf; + *buf_size = outbuf_size; + return status; /* Size of compressed/decompressed data */ + } + + failed: + free(outbuf); + return 0; + +} /* End filter function */ + + +/* HDF5 Plugin Interface */ +const H5Z_class2_t blosc_H5Filter[1] = { + { + H5Z_CLASS_T_VERS, + (H5Z_filter_t)(FILTER_BLOSC), + 1, /* encoder_present flag (set to true) */ + 1, /* decoder_present flag (set to true) */ + "blosc", + /* Filter info */ + NULL, /* The "can apply" callback */ + (H5Z_set_local_func_t)(blosc_set_local), /* The "set local" callback */ + (H5Z_func_t)(blosc_filter), /* The filter function */ + } +}; + + +H5PL_type_t H5PLget_plugin_type(void) { return H5PL_TYPE_FILTER; } +const void* H5PLget_plugin_info(void) { return blosc_H5Filter; } + + +/* Provide the codec support for the HDF5 blosc library */ + +/* NCZarr Filter Objects */ + +#define DEFAULT_LEVEL 9 +#define DEFAULT_BLOCKSIZE 1 +#define DEFAULT_TYPESIZE 1 +#define DEFAULT_COMPCODE BLOSC_LZ4 + +/* Forward */ +static void NCZ_blosc_codec_finalize(void); +static int NCZ_blosc_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp); +static int NCZ_blosc_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp); +static int NCZ_blosc_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp); + +/* Structure for NCZ_PLUGIN_CODEC */ +static NCZ_codec_t NCZ_blosc_codec = {/* NCZ_codec_t codec fields */ + NCZ_CODEC_CLASS_VER, /* Struct version number */ + NCZ_CODEC_HDF5, /* Struct sort */ + "blosc", /* Standard name/id of the codec */ + FILTER_BLOSC, /* HDF5 alias for blosc */ + NULL, /*NCZ_blosc_codec_initialize*/ + NCZ_blosc_codec_finalize, + NCZ_blosc_codec_to_hdf5, + NCZ_blosc_hdf5_to_codec, + NCZ_blosc_modify_parameters, +}; + +/* External Export API */ +const void* +NCZ_get_codec_info(void) +{ + if(!h5z_blosc_initialized) { + h5z_blosc_initialized = 1; + blosc_init(); + } + return (void*)&NCZ_blosc_codec; +} + +/* NCZarr Interface Functions */ + +/* Create the true parameter set: +Visible parameters: +param[0] -- reserved +param[1] -- reserved +param[2] -- reserved +param[3] -- variable chunksize in bytes | 0 (=>default) +param[4] -- compression level +param[5] -- BLOSC_SHUFFLE|BLOSC_BITSHUFFLE +param[6] -- compressor index + +Working parameters: +param[0] -- filter revision +param[1] -- blosc version +param[2] -- variable type size in bytes +param[3] -- variable chunksize in bytes +param[4] -- compression level +param[5] -- BLOSC_SHUFFLE|BLOSC_BITSHUFFLE +param[6] -- compressor index +*/ + +static void +NCZ_blosc_codec_finalize(void) +{ + if(h5z_blosc_initialized) { + blosc_destroy(); + h5z_blosc_initialized = 0; + } +} + +static int +NCZ_blosc_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp) +{ + int i,stat = NC_NOERR; + nc_type vtype; + int storage, ndims; + size_t* chunklens = NULL; + size_t typesize, chunksize; + char vname[NC_MAX_NAME+1]; + unsigned* params = NULL; + size_t nparams; + size_t vnparams = *vnparamsp; + unsigned* vparams = *vparamsp; + + if(vnparams < 7) + {stat = NC_EFILTER; goto done;} + nparams = 7; + + if(vnparams > 0 && vparams == NULL) + {stat = NC_EFILTER; goto done;} + + if(wnparamsp == NULL || wparamsp == NULL) + {stat = NC_EFILTER; goto done;} + + vnparams = *vnparamsp; + vparams = *vparamsp; + + /* Get variable info */ + if((stat = nc_inq_var(ncid,varid,vname,&vtype,&ndims,NULL,NULL))) goto done; + + if(ndims == 0) {stat = NC_EFILTER; goto done;} + + /* Get the typesize */ + if((stat = nc_inq_type(ncid,vtype,NULL,&typesize))) goto done; + + /* Compute chunksize */ + if((chunklens = (size_t*)calloc(ndims,sizeof(size_t)))==NULL) goto done; + if((stat = nc_inq_var_chunking(ncid,varid,&storage,chunklens))) goto done; + if(storage != NC_CHUNKED) {stat = NC_EFILTER; goto done;} + chunksize = typesize; + for(i=0;i NC_MAX_UINT) {stat = NC_EFILTER; goto done;} + params[4] = (unsigned)jc.ival; + + /* Get blocksize */ + if(NCJdictget(jcodec,"blocksize",&jtmp)) {stat = NC_EFILTER; goto done;} + if(jtmp) { + if(NCJcvt(jtmp,NCJ_INT,&jc)) {stat = NC_EFILTER; goto done;} + } else + jc.ival = DEFAULT_BLOCKSIZE; + if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EFILTER; goto done;} + params[3] = (unsigned)jc.ival; + + /* Get shuffle */ + if(NCJdictget(jcodec,"shuffle",&jtmp)) {stat = NC_EFILTER; goto done;} + if(jtmp) { + if(NCJcvt(jtmp,NCJ_INT,&jc)) {stat = NC_EFILTER; goto done;} + } else + jc.ival = BLOSC_NOSHUFFLE; + params[5] = (unsigned)jc.ival; + + /* Get compname */ + if(NCJdictget(jcodec,"cname",&jtmp)) {stat = NC_EFILTER; goto done;} + if(jtmp) { + if(NCJcvt(jtmp,NCJ_STRING,&jc)) {stat = NC_EFILTER; goto done;} + if(jc.sval == NULL || strlen(jc.sval) == 0) {stat = NC_EFILTER; goto done;} + if((compcode = blosc_compname_to_compcode(jc.sval)) < 0) {stat = NC_EFILTER; goto done;} + } else + compcode = DEFAULT_COMPCODE; + params[6] = (unsigned)compcode; + + if(nparamsp) *nparamsp = 7; + if(paramsp) {*paramsp = params; params = NULL;} + +done: + if(jc.sval) { + free(jc.sval); + } + if(params) { + free(params); + } + NCJreclaim(jcodec); + return stat; +} + +static int +NCZ_blosc_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp) +{ + int stat = NC_NOERR; + char json[1024]; + char* compname = NULL; + + if(nparams == 0 || params == NULL) + {stat = NC_EINVAL; goto done;} + + /* Get the sub-compressor name */ + if(blosc_compcode_to_compname((int)params[6],&compname) < 0) {stat = NC_EFILTER; goto done;} + + snprintf(json,sizeof(json), + "{\"id\": \"blosc\",\"clevel\": %u,\"blocksize\": %u,\"cname\": \"%s\",\"shuffle\": %d}", + params[4],params[3],compname,params[5]); + + if(codecp) { + if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;} + } + +done: + return stat; +} diff --git a/plugins/H5Zblosc.h b/plugins/H5Zblosc.h new file mode 100755 index 0000000000..6e834a2931 --- /dev/null +++ b/plugins/H5Zblosc.h @@ -0,0 +1,60 @@ +/* + * Dynamically loaded filter plugin for HDF5 blosc filter. + * + * Author: Kiyoshi Masui + * Created: 2014 + * + * + * Header file + * ----------- + * + * This provides dynamically loaded HDF5 filter functionality (introduced + * in HDF5-1.8.11, May 2013) to the blosc HDF5 filter. + * + * Usage: compile as a shared library and install either to the default + * search location for HDF5 filter plugins (on Linux + * /usr/local/hdf5/lib/plugin) or to a location pointed to by the + * HDF5_PLUGIN_PATH environment variable. + * + */ + + +#ifndef H5ZBLOSC_H +#define H5ZBLOSC_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "netcdf_filter_build.h" +#include "blosc.h" + +/* Filter revision number, starting at 1 */ +/* #define FILTER_BLOSC_VERSION 1 */ +#define FILTER_BLOSC_VERSION 2 /* multiple compressors since Blosc 1.3 */ + +/* Filter ID registered with the HDF Group */ +#define FILTER_BLOSC 32001 + +#ifdef _MSC_VER + #ifdef DLL_EXPORT /* define when building the library */ + #define DECLSPEC __declspec(dllexport) + #else + #define DECLSPEC __declspec(dllimport) + #endif +#else + #define DECLSPEC extern +#endif + +/* HDF5 Plugin API */ +DECLSPEC H5PL_type_t H5PLget_plugin_type(void); +DECLSPEC const void* H5PLget_plugin_info(void); + +/* NCZarr API */ +DECLSPEC const void* NCZ_get_plugin_info(void); + +#ifdef __cplusplus +} +#endif + +#endif /*H5ZBLOSC_H*/ diff --git a/plugins/H5Zbzip2.c b/plugins/H5Zbzip2.c index 6c166d25e5..fa9ed8a6c0 100644 --- a/plugins/H5Zbzip2.c +++ b/plugins/H5Zbzip2.c @@ -4,22 +4,15 @@ #include #include #include -#include -/* Older versions of the hdf library may define H5PL_type_t here */ -#include - - -#ifndef DLL_EXPORT -#define DLL_EXPORT -#endif +#include "netcdf_filter_build.h" /* WARNING: Starting with HDF5 version 1.10.x, the plugin code MUST be careful when using the standard *malloc()*, *realloc()*, and *free()* function. -In the event that the code is allocating, reallocating, for +In the event that the code is allocating, reallocating, or free'ing memory that either came from or will be exported to the calling HDF5 library, then one MUST use the corresponding HDF5 functions *H5allocate_memory()*, *H5resize_memory()*, @@ -30,8 +23,25 @@ will generate an error. */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#include "netcdf_filter_build.h" +#include + #include "h5bzip2.h" +/* Forward */ +static htri_t H5Z_bzip2_can_apply(hid_t dcpl_id, hid_t type_id, hid_t space_id); +static size_t H5Z_filter_bzip2(unsigned flags,size_t cd_nelmts,const unsigned cd_values[], + size_t nbytes,size_t *buf_size,void**buf); + const H5Z_class2_t H5Z_BZIP2[1] = {{ H5Z_CLASS_T_VERS, /* H5Z_class_t version */ (H5Z_filter_t)H5Z_FILTER_BZIP2, /* Filter id number */ @@ -44,12 +54,14 @@ const H5Z_class2_t H5Z_BZIP2[1] = {{ }}; /* External Discovery Functions */ +DLLEXPORT H5PL_type_t H5PLget_plugin_type(void) { return H5PL_TYPE_FILTER; } +DLLEXPORT const void* H5PLget_plugin_info(void) { @@ -61,13 +73,14 @@ H5PLget_plugin_info(void) * The "can_apply" callback returns positive a valid combination, zero for an * invalid combination and negative for an error. */ -htri_t +static htri_t H5Z_bzip2_can_apply(hid_t dcpl_id, hid_t type_id, hid_t space_id) { return 1; /* Assume it can always apply */ } -size_t H5Z_filter_bzip2(unsigned int flags, size_t cd_nelmts, +static size_t +H5Z_filter_bzip2(unsigned int flags, size_t cd_nelmts, const unsigned int cd_values[], size_t nbytes, size_t *buf_size, void **buf) { @@ -93,11 +106,7 @@ size_t H5Z_filter_bzip2(unsigned int flags, size_t cd_nelmts, /* Prepare the output buffer. */ outbuflen = nbytes * 3 + 1; /* average bzip2 compression ratio is 3:1 */ -#ifdef HAVE_H5ALLOCATE_MEMORY outbuf = H5allocate_memory(outbuflen,0); -#else - outbuf = (char*)malloc(outbuflen * sizeof(char)); -#endif if (outbuf == NULL) { fprintf(stderr, "memory allocation failed for bzip2 decompression\n"); goto cleanupAndFail; @@ -130,11 +139,7 @@ size_t H5Z_filter_bzip2(unsigned int flags, size_t cd_nelmts, if (ret != BZ_STREAM_END && stream.avail_out == 0) { /* Grow the output buffer. */ newbuflen = outbuflen * 2; -#ifdef HAVE_H5RESIZE_MEMORY newbuf = H5resize_memory(outbuf, newbuflen); -#else - newbuf = realloc(outbuf,newbuflen); -#endif if (newbuf == NULL) { fprintf(stderr, "memory allocation failed for bzip2 decompression\n"); goto cleanupAndFail; @@ -178,11 +183,7 @@ size_t H5Z_filter_bzip2(unsigned int flags, size_t cd_nelmts, /* Prepare the output buffer. */ outbuflen = nbytes + nbytes / 100 + 600; /* worst case (bzip2 docs) */ -#ifdef HAVE_H5ALLOCATE_MEMORY outbuf = H5allocate_memory(outbuflen,0); -#else - outbuf = (char*)malloc(outbuflen * sizeof(char)); -#endif if (outbuf == NULL) { fprintf(stderr, "memory allocation failed for bzip2 compression\n"); @@ -201,11 +202,7 @@ size_t H5Z_filter_bzip2(unsigned int flags, size_t cd_nelmts, } /* Always replace the input buffer with the output buffer. */ -#ifdef HAVE_H5FREE_MEMORY H5free_memory(*buf); -#else - free(*buf); -#endif *buf = outbuf; *buf_size = outbuflen; @@ -213,11 +210,96 @@ size_t H5Z_filter_bzip2(unsigned int flags, size_t cd_nelmts, cleanupAndFail: if (outbuf) -#ifdef HAVE_H5FREE_MEMORY H5free_memory(outbuf); -#else - free(outbuf); -#endif - return 0; } + +/**************************************************/ +/* NCZarr Filter Objects */ + +/* Provide the codec support for the HDF5 bzip library */ + +static int NCZ_bzip2_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp); +static int NCZ_bzip2_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp); + +static NCZ_codec_t NCZ_bzip2_codec = {/* NCZ_codec_t codec fields */ + NCZ_CODEC_CLASS_VER, /* Struct version number */ + NCZ_CODEC_HDF5, /* Struct sort */ + "bz2", /* Standard name/id of the codec */ + H5Z_FILTER_BZIP2, /* HDF5 alias for bzip2 */ + NULL, /*NCZ_bzip2_codec_initialize*/ + NULL, /*NCZ_bzip2_codec_finalize*/ + NCZ_bzip2_codec_to_hdf5, + NCZ_bzip2_hdf5_to_codec, + NULL, /*NCZ_bzip2_modify_parameters*/ +}; + +/* External Export API */ +DLLEXPORT +const void* +NCZ_get_codec_info(void) +{ + return (void*)&NCZ_bzip2_codec; +} + +static int +NCZ_bzip2_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp) +{ + int stat = NC_NOERR; + NCjson* jcodec = NULL; + NCjson* jtmp = NULL; + unsigned* params = NULL; + struct NCJconst jc; + + if(nparamsp == NULL || paramsp == NULL) + {stat = NC_EINTERNAL; goto done;} + + if((params = (unsigned*)calloc(1,sizeof(unsigned)))== NULL) + {stat = NC_ENOMEM; goto done;} + + /* parse the JSON */ + if(NCJparse(codec_json,0,&jcodec)) + {stat = NC_EFILTER; goto done;} + if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;} + /* Verify the codec ID */ + if(NCJdictget(jcodec,"id",&jtmp)) + {stat = NC_EFILTER; goto done;} + if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EFILTER; goto done;} + if(strcmp(NCJstring(jtmp),NCZ_bzip2_codec.codecid)!=0) {stat = NC_EINVAL; goto done;} + + /* Get Level */ + if(NCJdictget(jcodec,"level",&jtmp)) + {stat = NC_EFILTER; goto done;} + if(NCJcvt(jtmp,NCJ_INT,&jc)) + {stat = NC_EFILTER; goto done;} + if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EINVAL; goto done;} + params[0] = (unsigned)jc.ival; + *nparamsp = 1; + *paramsp = params; params = NULL; + +done: + if(params) free(params); + NCJreclaim(jcodec); + return stat; +} + +static int +NCZ_bzip2_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp) +{ + int stat = NC_NOERR; + unsigned level = 0; + char json[1024]; + + if(nparams == 0 || params == NULL) + {stat = NC_EFILTER; goto done;} + + level = params[0]; + snprintf(json,sizeof(json),"{\"id\": \"%s\", \"level\": \"%u\"}",NCZ_bzip2_codec.codecid,level); + if(codecp) { + if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;} + } + +done: + return stat; +} + diff --git a/plugins/H5Zdeflate.c b/plugins/H5Zdeflate.c new file mode 100644 index 0000000000..93a637ef10 --- /dev/null +++ b/plugins/H5Zdeflate.c @@ -0,0 +1,208 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Programmer: Robb Matzke + * Friday, August 27, 1999 + */ + +#include +#include +#include +#include "netcdf_filter_build.h" +#include + +/* Local function prototypes */ +static size_t H5Z__filter_deflate (unsigned flags, size_t cd_nelmts, + const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); + +/* This message derives from H5Z */ +const H5Z_class2_t H5Z_DEFLATE[1] = {{ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + H5Z_FILTER_DEFLATE, /* Filter id number */ + 1, /* encoder_present flag (set to true) */ + 1, /* decoder_present flag (set to true) */ + "deflate", /* Filter name for debugging */ + NULL, /* The "can apply" callback */ + NULL, /* The "set local" callback */ + H5Z__filter_deflate, /* The actual filter function */ +}}; + +/* External Discovery Functions */ +DLLEXPORT +H5PL_type_t +H5PLget_plugin_type(void) +{ + return H5PL_TYPE_FILTER; +} + +DLLEXPORT +const void* +H5PLget_plugin_info(void) +{ + return H5Z_DEFLATE; +} + + +#define H5Z_DEFLATE_SIZE_ADJUST(s) (HDceil(((double)(s)) * (double)1.001f) + 12) + + +/*------------------------------------------------------------------------- + * Function: H5Z__filter_deflate + * + * Purpose: Implement an I/O filter around the 'deflate' algorithm in + * libz + * + * Return: Success: Size of buffer filtered + * Failure: 0 + * + * Programmer: Robb Matzke + * Thursday, April 16, 1998 + * + *------------------------------------------------------------------------- + */ +static size_t +H5Z__filter_deflate(unsigned flags, size_t cd_nelmts, + const unsigned cd_values[], size_t nbytes, + size_t *buf_size, void **buf) +{ + void *outbuf = NULL; /* Pointer to new buffer */ + int status; /* Status from zlib operation */ + size_t ret_value = 0; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity check */ + HDassert(*buf_size > 0); + HDassert(buf); + HDassert(*buf); + + /* Check arguments */ + if (cd_nelmts!=1 || cd_values[0]>9) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid deflate aggression level") + + if (flags & H5Z_FLAG_REVERSE) { + /* Input; uncompress */ + z_stream z_strm; /* zlib parameters */ + size_t nalloc = *buf_size; /* Number of bytes for output (compressed) buffer */ + + /* Allocate space for the compressed buffer */ + if (NULL==(outbuf = H5MM_malloc(nalloc))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "memory allocation failed for deflate uncompression") + + /* Set the uncompression parameters */ + HDmemset(&z_strm, 0, sizeof(z_strm)); + z_strm.next_in = (Bytef *)*buf; + H5_CHECKED_ASSIGN(z_strm.avail_in, unsigned, nbytes, size_t); + z_strm.next_out = (Bytef *)outbuf; + H5_CHECKED_ASSIGN(z_strm.avail_out, unsigned, nalloc, size_t); + + /* Initialize the uncompression routines */ + if (Z_OK!=inflateInit(&z_strm)) + HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "inflateInit() failed") + + /* Loop to uncompress the buffer */ + do { + /* Uncompress some data */ + status = inflate(&z_strm, Z_SYNC_FLUSH); + + /* Check if we are done uncompressing data */ + if (Z_STREAM_END==status) + break; /*done*/ + + /* Check for error */ + if (Z_OK!=status) { + (void)inflateEnd(&z_strm); + HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "inflate() failed") + } + else { + /* If we're not done and just ran out of buffer space, get more */ + if(0 == z_strm.avail_out) { + void *new_outbuf; /* Pointer to new output buffer */ + + /* Allocate a buffer twice as big */ + nalloc *= 2; + if(NULL == (new_outbuf = H5MM_realloc(outbuf, nalloc))) { + (void)inflateEnd(&z_strm); + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "memory allocation failed for deflate uncompression") + } /* end if */ + outbuf = new_outbuf; + + /* Update pointers to buffer for next set of uncompressed data */ + z_strm.next_out = (unsigned char*)outbuf + z_strm.total_out; + z_strm.avail_out = (uInt)(nalloc - z_strm.total_out); + } /* end if */ + } /* end else */ + } while(status==Z_OK); + + /* Free the input buffer */ + H5MM_xfree(*buf); + + /* Set return values */ + *buf = outbuf; + outbuf = NULL; + *buf_size = nalloc; + ret_value = z_strm.total_out; + + /* Finish uncompressing the stream */ + (void)inflateEnd(&z_strm); + } /* end if */ + else { + /* + * Output; compress but fail if the result would be larger than the + * input. The library doesn't provide in-place compression, so we + * must allocate a separate buffer for the result. + */ + const Bytef *z_src = (const Bytef*)(*buf); + Bytef *z_dst; /*destination buffer */ + uLongf z_dst_nbytes = (uLongf)H5Z_DEFLATE_SIZE_ADJUST(nbytes); + uLong z_src_nbytes = (uLong)nbytes; + int aggression; /* Compression aggression setting */ + + /* Set the compression aggression level */ + H5_CHECKED_ASSIGN(aggression, int, cd_values[0], unsigned); + + /* Allocate output (compressed) buffer */ + if(NULL == (outbuf = H5MM_malloc(z_dst_nbytes))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "unable to allocate deflate destination buffer") + z_dst = (Bytef *)outbuf; + + /* Perform compression from the source to the destination buffer */ + status = compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression); + + /* Check for various zlib errors */ + if(Z_BUF_ERROR == status) + HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "overflow") + else if(Z_MEM_ERROR == status) + HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "deflate memory error") + else if(Z_OK != status) + HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "other deflate error") + /* Successfully uncompressed the buffer */ + else { + /* Free the input buffer */ + H5MM_xfree(*buf); + + /* Set return values */ + *buf = outbuf; + outbuf = NULL; + *buf_size = nbytes; + ret_value = z_dst_nbytes; + } /* end else */ + } /* end else */ + +done: + if(outbuf) + H5MM_xfree(outbuf); + FUNC_LEAVE_NOAPI(ret_value) +} + diff --git a/plugins/H5Zfletcher32.c b/plugins/H5Zfletcher32.c new file mode 100644 index 0000000000..aeec795a30 --- /dev/null +++ b/plugins/H5Zfletcher32.c @@ -0,0 +1,175 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Programmer: Raymond Lu + * Jan 3, 2003 + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#include "netcdf_filter_hdf5_build.h" + +#ifndef H5Z_FILTER_FLETCHER32 +#define H5Z_FILTER_FLETCHER32 3 +#endif + + +extern unsigned int H5_checksum_fletcher32(const void *data, size_t len); + +/* Local function prototypes */ +static size_t H5Z_filter_fletcher32(unsigned flags, size_t cd_nelmts, + const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); + +/* This message derives from H5Z */ +const H5Z_class2_t H5Z_FLETCHER32[1] = {{ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + H5Z_FILTER_FLETCHER32, /* Filter id number */ + 1, /* encoder_present flag (set to true) */ + 1, /* decoder_present flag (set to true) */ + "fletcher32", /* Filter name for debugging */ + NULL, /* The "can apply" callback */ + NULL, /* The "set local" callback */ + H5Z_filter_fletcher32, /* The actual filter function */ +}}; + +/* External Discovery Functions */ +H5PL_type_t +H5PLget_plugin_type(void) +{ + return H5PL_TYPE_FILTER; +} + +const void* +H5PLget_plugin_info(void) +{ + return H5Z_FLETCHER32; +} + +#define FLETCHER_LEN 4 + + +/*------------------------------------------------------------------------- + * Function: H5Z__filter_fletcher32 + * + * Purpose: Implement an I/O filter of Fletcher32 Checksum + * + * Return: Success: Size of buffer filtered + * Failure: 0 + * + * Programmer: Raymond Lu + * Jan 3, 2003 + * + *------------------------------------------------------------------------- + */ +static size_t +H5Z_filter_fletcher32(unsigned flags, size_t H5_ATTR_UNUSED cd_nelmts, const unsigned H5_ATTR_UNUSED cd_values[], + size_t nbytes, size_t *buf_size, void **buf) +{ + void *outbuf = NULL; /* Pointer to new buffer */ + unsigned char *src = (unsigned char*)(*buf); + uint32_t fletcher; /* Checksum value */ + uint32_t reversed_fletcher; /* Possible wrong checksum value */ + uint8_t c[4]; + uint8_t tmp; + size_t ret_value = 0; /* Return value */ + + FUNC_ENTER_STATIC + + HDassert(sizeof(uint32_t)>=4); + + if (flags & H5Z_FLAG_REVERSE) { /* Read */ + /* Do checksum if it's enabled for read; otherwise skip it + * to save performance. */ + if (!(flags & H5Z_FLAG_SKIP_EDC)) { + unsigned char *tmp_src; /* Pointer to checksum in buffer */ + size_t src_nbytes = nbytes; /* Original number of bytes */ + uint32_t stored_fletcher; /* Stored checksum value */ + + /* Get the stored checksum */ + src_nbytes -= FLETCHER_LEN; + tmp_src=src+src_nbytes; + UINT32DECODE(tmp_src, stored_fletcher); + + /* Compute checksum (can't fail) */ + fletcher = H5_checksum_fletcher32(src, src_nbytes); + + /* The reversed checksum. There was a bug in the calculating code of + * the Fletcher32 checksum in the library before v1.6.3. The checksum + * value wasn't consistent between big-endian and little-endian systems. + * This bug was fixed in Release 1.6.3. However, after fixing the bug, + * the checksum value is no longer the same as before on little-endian + * system. We'll check both the correct checksum and the wrong + * checksum to be consistent with Release 1.6.2 and before. + */ + H5MM_memcpy(c, &fletcher, (size_t)4); + + tmp = c[1]; + c[1] = c[0]; + c[0] = tmp; + + tmp = c[3]; + c[3] = c[2]; + c[2] = tmp; + + H5MM_memcpy(&reversed_fletcher, c, (size_t)4); + + /* Verify computed checksum matches stored checksum */ + if(stored_fletcher != fletcher && stored_fletcher != reversed_fletcher) + HGOTO_ERROR(H5E_STORAGE, H5E_READERROR, 0, "data error detected by Fletcher32 checksum") + } + + /* Set return values */ + /* (Re-use the input buffer, just note that the size is smaller by the size of the checksum) */ + ret_value = nbytes-FLETCHER_LEN; + } else { /* Write */ + unsigned char *dst; /* Temporary pointer to destination buffer */ + + /* Compute checksum (can't fail) */ + fletcher = H5_checksum_fletcher32(src, nbytes); + + if (NULL == (outbuf = H5MM_malloc(nbytes + FLETCHER_LEN))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "unable to allocate Fletcher32 checksum destination buffer") + + dst = (unsigned char *) outbuf; + + /* Copy raw data */ + H5MM_memcpy((void*)dst, (void*)(*buf), nbytes); + + /* Append checksum to raw data for storage */ + dst += nbytes; + UINT32ENCODE(dst, fletcher); + + /* Free input buffer */ + H5MM_xfree(*buf); + + /* Set return values */ + *buf_size = nbytes + FLETCHER_LEN; + *buf = outbuf; + outbuf = NULL; + ret_value = *buf_size; + } + +done: + if(outbuf) + H5MM_xfree(outbuf); + FUNC_LEAVE_NOAPI(ret_value) +} + diff --git a/plugins/H5Zmisc.c b/plugins/H5Zmisc.c index beb84aced2..f652992cee 100644 --- a/plugins/H5Zmisc.c +++ b/plugins/H5Zmisc.c @@ -4,15 +4,9 @@ #include #include #include -#include -/* Older versions of the hdf library may define H5PL_type_t here */ -#include +#include "netcdf_filter_build.h" #include "h5misc.h" -#ifndef DLL_EXPORT -#define DLL_EXPORT -#endif - /* WARNING: Starting with HDF5 version 1.10.x, the plugin code MUST be careful when using the standard *malloc()*, *realloc()*, and @@ -36,6 +30,11 @@ will generate an error. */ #define DBLVAL 12345678.12345678 +static htri_t H5Z_test_can_apply(hid_t dcpl_id, hid_t type_id, hid_t space_id); +static size_t H5Z_filter_test(unsigned int flags, size_t cd_nelmts, + const unsigned int cd_values[], size_t nbytes, + size_t *buf_size, void **buf); + static int paramcheck(size_t nparams, const unsigned int* params); static void mismatch(size_t i, const char* which); @@ -51,12 +50,14 @@ const H5Z_class2_t H5Z_TEST[1] = {{ }}; /* External Discovery Functions */ +DLLEXPORT H5PL_type_t H5PLget_plugin_type(void) { return H5PL_TYPE_FILTER; } +DLLEXPORT const void* H5PLget_plugin_info(void) { @@ -68,7 +69,7 @@ H5PLget_plugin_info(void) * The "can_apply" callback returns positive a valid combination, zero for an * invalid combination and negative for an error. */ -htri_t +static htri_t H5Z_test_can_apply(hid_t dcpl_id, hid_t type_id, hid_t space_id) { return 1; /* Assume it can always apply */ @@ -88,7 +89,7 @@ Test cases format: */ -size_t +static size_t H5Z_filter_test(unsigned int flags, size_t cd_nelmts, const unsigned int cd_values[], size_t nbytes, size_t *buf_size, void **buf) @@ -126,31 +127,19 @@ fprintf(stderr,"TC_EXPANDED: decompress: nbytes=%u buf_size=%u xdata[0..8]=|",(u } fprintf(stderr,"|\n"); /* Replace buffer */ -#ifdef HAVE_H5ALLOCATE_MEMORY newbuf = H5allocate_memory(*buf_size,0); -#else - newbuf = malloc(*buf_size); -#endif if(newbuf == NULL) abort(); memcpy(newbuf,*buf,*buf_size); } else { /* Replace buffer */ -#ifdef HAVE_H5ALLOCATE_MEMORY newbuf = H5allocate_memory(*buf_size,0); -#else - newbuf = malloc(*buf_size); -#endif if(newbuf == NULL) abort(); memcpy(newbuf,*buf,*buf_size); } /* reclaim old buffer */ -#ifdef HAVE_H5FREE_MEMORY H5free_memory(*buf); -#else - free(*buf); -#endif *buf = newbuf; } else { /* Compress */ @@ -161,11 +150,7 @@ fprintf(stderr,"TC_EXPANDED: decompress: nbytes=%u buf_size=%u xdata[0..8]=|",(u fprintf(stderr,"TC_EXPANDED: compress: nbytes=%u buf_size=%u size=%u\n",(unsigned)nbytes,(unsigned)*buf_size,(unsigned)size); #endif /* Replace buffer with one that is bigger than the chunk size */ -#ifdef HAVE_H5ALLOCATE_MEMORY newbuf = H5allocate_memory(size,0); -#else - newbuf = malloc(size); -#endif if(newbuf == NULL) abort(); b = (float*)newbuf; for(i=0;i<1024*2;i++) { @@ -175,21 +160,13 @@ fprintf(stderr,"TC_EXPANDED: compress: nbytes=%u buf_size=%u size=%u\n",(unsigne *buf_size = size; } else { /* Replace buffer */ -#ifdef HAVE_H5ALLOCATE_MEMORY newbuf = H5allocate_memory(*buf_size,0); -#else - newbuf = malloc(*buf_size); -#endif if(newbuf == NULL) abort(); memcpy(newbuf,*buf,*buf_size); } /* reclaim old buffer */ -#ifdef HAVE_H5FREE_MEMORY H5free_memory(*buf); -#else - free(*buf); -#endif *buf = newbuf; } diff --git a/plugins/H5Znoop.c b/plugins/H5Znoop.c index 86987a603d..03fa6d6bbe 100644 --- a/plugins/H5Znoop.c +++ b/plugins/H5Znoop.c @@ -4,13 +4,13 @@ #include #include #include -#include -/* Older versions of the hdf library may define H5PL_type_t here */ -#include +#include -#ifndef DLL_EXPORT -#define DLL_EXPORT +#include "netcdf_filter_build.h" + +#ifndef NOOP_INSTANCE +#define NOOP_INSTANCE 0 #endif #if NOOP_INSTANCE == 1 @@ -66,12 +66,14 @@ static H5Z_class2_t H5Z_NOOP[1] = {{ }}; /* External Discovery Functions */ +DLLEXPORT H5PL_type_t H5PLget_plugin_type(void) { return H5PL_TYPE_FILTER; } +DLLEXPORT const void* H5PLget_plugin_info(void) { @@ -108,39 +110,140 @@ H5Z_filter_noop(unsigned int flags, size_t cd_nelmts, if (flags & H5Z_FLAG_REVERSE) { /* Replace buffer */ -#ifdef HAVE_H5ALLOCATE_MEMORY newbuf = H5allocate_memory(*buf_size,0); -#else - newbuf = malloc(*buf_size); -#endif if(newbuf == NULL) abort(); memcpy(newbuf,*buf,*buf_size); /* reclaim old buffer */ -#ifdef HAVE_H5FREE_MEMORY H5free_memory(*buf); -#else - free(*buf); -#endif *buf = newbuf; } else { /* Replace buffer */ -#ifdef HAVE_H5ALLOCATE_MEMORY newbuf = H5allocate_memory(*buf_size,0); -#else - newbuf = malloc(*buf_size); -#endif if(newbuf == NULL) abort(); memcpy(newbuf,*buf,*buf_size); /* reclaim old buffer */ -#ifdef HAVE_H5FREE_MEMORY H5free_memory(*buf); -#else - free(*buf); -#endif *buf = newbuf; } return *buf_size; } + +/**************************************************/ +/* NCZarr Codec API */ + +/* Codec Format +{ +"id": "test", +"p0": "", +"p1": "", +"p2": "", +... +"pn": "", +} +*/ + +/* Forward */ +static int NCZ_noop_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp); +static int NCZ_noop_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp); + +/* Structure for NCZ_PLUGIN_CODEC */ +static NCZ_codec_t NCZ_noop_codec = {/* NCZ_codec_t codec fields */ + NCZ_CODEC_CLASS_VER, /* Struct version number */ + NCZ_CODEC_HDF5, /* Struct sort */ +#if NOOP_INSTANCE == 0 + "noop0", /* Standard name/id of the codec */ + H5Z_FILTER_NOOP, /* HDF5 alias for noop */ +#else + "noop1", /* Standard name/id of the codec */ + H5Z_FILTER_NOOP+1, /* HDF5 alias for noop */ +#endif + NULL, /*NCZ_noop_codec_initialize*/ + NULL, /*NCZ_noop_codec_finalize*/ + NCZ_noop_codec_to_hdf5, + NCZ_noop_hdf5_to_codec, + NULL, /*NCZ_noop_modify_parameters*/ +}; + +/* External Export API */ +DLLEXPORT +const void* +NCZ_get_codec_info(void) +{ + return (void*)&NCZ_noop_codec; +} + +/* NCZarr Interface Functions */ + +static int +NCZ_noop_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp) +{ + int stat = NC_NOERR; + NCjson* jcodec = NULL; + NCjson* jtmp = NULL; + int i,nparams = 0; + unsigned* params = NULL; + char field[1024]; + + /* parse the JSON */ + if(NCJparse(codec_json,0,&jcodec)) + {stat = NC_EFILTER; goto done;} + if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;} + + /* Verify the codec ID */ + if(NCJdictget(jcodec,"id",&jtmp)) + {stat = NC_EFILTER; goto done;} + if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EINVAL; goto done;} + if(strcmp(NCJstring(jtmp),NCZ_noop_codec.codecid)!=0) {stat = NC_EINVAL; goto done;} + + nparams = (NCJlength(jcodec) - 1) / 2; /* -1 for id each param is key+value */ + + if((params = (unsigned*)calloc(nparams,sizeof(unsigned)))== NULL) + {stat = NC_ENOMEM; goto done;} + + /* This filter has an arbitrary number of parameters named p0...pn */ + + for(i=0;i NC_MAX_UINT) {stat = NC_EINVAL; goto done;} + params[i] = (unsigned)jc.ival; + } + if(nparamsp) *nparamsp = nparams; + if(paramsp) {*paramsp = params; params = NULL;} + +done: + if(params) free(params); + NCJreclaim(jcodec); + return stat; +} + +static int +NCZ_noop_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp) +{ + int i,stat = NC_NOERR; + char json[8192]; + char value[1024]; + + if(nparams != 0 && params == NULL) + {stat = NC_EINVAL; goto done;} + + snprintf(json,sizeof(json),"{\"id\": \"%s\"",NCZ_noop_codec.codecid); + for(i=0;i +#include +#include +#include + +#include "netcdf_filter_build.h" + +#ifndef H5Z_FILTER_SHUFFLE +#define H5Z_FILTER_SHUFFLE 2 +#endif + +#ifndef H5Z_SHUFFLE_TOTAL_NPARMS +#define H5Z_SHUFFLE_TOTAL_NPARMS 1 +#endif +#ifndef H5Z_SHUFFLE_USER_NPARMS +#define H5Z_SHUFFLE_USER_NPARMS 0 +#endif + +/* Local function prototypes */ +#ifdef USE_HDF5 +static herr_t H5Z__set_local_shuffle(hid_t dcpl_id, hid_t type_id, hid_t space_id); +#else +#define H5Z__set_local_shuffle NULL +#endif + +static size_t H5Z__filter_shuffle(unsigned flags, size_t cd_nelmts, + const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); + +/* This message derives from H5Z */ +const H5Z_class2_t H5Z_SHUFFLE[1] = {{ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + H5Z_FILTER_SHUFFLE, /* Filter id number */ + 1, /* encoder_present flag (set to true) */ + 1, /* decoder_present flag (set to true) */ + "shuffle", /* Filter name for debugging */ + NULL, /* The "can apply" callback */ + H5Z__set_local_shuffle, /* The "set local" callback */ + H5Z__filter_shuffle, /* The actual filter function */ +}}; + +/* External Discovery Functions */ +DLLEXPORT +H5PL_type_t +H5PLget_plugin_type(void) +{ + return H5PL_TYPE_FILTER; +} + +DLLEXPORT +const void* +H5PLget_plugin_info(void) +{ + return H5Z_SHUFFLE; +} + +/* Local macros */ +#define H5Z_SHUFFLE_PARM_SIZE 0 /* "Local" parameter for shuffling size */ + +#ifdef USE_HDF5 +/*------------------------------------------------------------------------- + * Function: H5Z__set_local_shuffle + * + * Purpose: Set the "local" dataset parameter for data shuffling to be + * the size of the datatype. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Quincey Koziol + * Monday, April 7, 2003 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5Z__set_local_shuffle(hid_t dcpl_id, hid_t type_id, hid_t space_id) +{ + unsigned flags = 0; /* Filter flags */ + unsigned cd_values[H5Z_SHUFFLE_TOTAL_NPARMS]; /* Filter parameters */ + herr_t ret_value = 1; /* Return value */ + + /* Set "local" parameter for this dataset */ + if((cd_values[0] = (unsigned)H5Tget_size(type_id)) == 0) + HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype size") + + /* Modify the filter's parameters for this dataset */ + if(H5Pmodify_filter(dcpl_id, H5Z_FILTER_SHUFFLE, flags, (size_t)H5Z_SHUFFLE_TOTAL_NPARMS, cd_values) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTSET, FAIL, "can't set local shuffle parameters") + +done: + return ret_value; +} /* end H5Z__set_local_shuffle() */ +#endif + + +/*------------------------------------------------------------------------- + * Function: H5Z__filter_shuffle + * + * Purpose: Implement an I/O filter which "de-interlaces" a block of data + * by putting all the bytes in a byte-position for each element + * together in the block. For example, for 4-byte elements stored + * as: 012301230123, shuffling will store them as: 000111222333 + * Usually, the bytes in each byte position are more related to + * each other and putting them together will increase compression. + * + * Return: Success: Size of buffer filtered + * Failure: 0 + * + * Programmer: Kent Yang + * Wednesday, November 13, 2002 + * + *------------------------------------------------------------------------- + */ +static size_t +H5Z__filter_shuffle(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], + size_t nbytes, size_t *buf_size, void **buf) +{ + void *dest = NULL; /* Buffer to deposit [un]shuffled bytes into */ + unsigned char *_src=NULL; /* Alias for source buffer */ + unsigned char *_dest=NULL; /* Alias for destination buffer */ + unsigned bytesoftype; /* Number of bytes per element */ + size_t numofelements; /* Number of elements in buffer */ + size_t i; /* Local index variables */ +#ifdef NO_DUFFS_DEVICE + size_t j; /* Local index variable */ +#endif /* NO_DUFFS_DEVICE */ + size_t leftover; /* Extra bytes at end of buffer */ + size_t ret_value = 0; /* Return value */ + + /* Check arguments */ + if (cd_nelmts!=H5Z_SHUFFLE_TOTAL_NPARMS || cd_values[H5Z_SHUFFLE_PARM_SIZE]==0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid shuffle parameters") + + /* Get the number of bytes per element from the parameter block */ + bytesoftype=cd_values[H5Z_SHUFFLE_PARM_SIZE]; + + /* Compute the number of elements in buffer */ + numofelements=nbytes/bytesoftype; + + /* Don't do anything for 1-byte elements, or "fractional" elements */ + if(bytesoftype > 1 && numofelements > 1) { + /* Compute the leftover bytes if there are any */ + leftover = nbytes%bytesoftype; + + /* Allocate the destination buffer */ + if (NULL==(dest = H5MM_malloc(nbytes))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "memory allocation failed for shuffle buffer") + + if(flags & H5Z_FLAG_REVERSE) { + /* Get the pointer to the source buffer */ + _src =(unsigned char *)(*buf); + + /* Input; unshuffle */ + for(i=0; i 0) { + DUFF_GUTS; + + j--; + } /* end for */ +#else /* NO_DUFFS_DEVICE */ + { + size_t duffs_index; /* Counting index for Duff's device */ + + duffs_index = (numofelements + 7) / 8; + switch (numofelements % 8) { + default: + HDassert(0 && "This Should never be executed!"); + break; + case 0: + do + { + DUFF_GUTS + H5_ATTR_FALLTHROUGH + case 7: + DUFF_GUTS + H5_ATTR_FALLTHROUGH + case 6: + DUFF_GUTS + H5_ATTR_FALLTHROUGH + case 5: + DUFF_GUTS + H5_ATTR_FALLTHROUGH + case 4: + DUFF_GUTS + H5_ATTR_FALLTHROUGH + case 3: + DUFF_GUTS + H5_ATTR_FALLTHROUGH + case 2: + DUFF_GUTS + H5_ATTR_FALLTHROUGH + case 1: + DUFF_GUTS + } while (--duffs_index > 0); + } /* end switch */ + } +#endif /* NO_DUFFS_DEVICE */ +#undef DUFF_GUTS + } /* end for */ + + /* Add leftover to the end of data */ + if(leftover>0) { + /* Adjust back to end of shuffled bytes */ + _dest -= (bytesoftype - 1); /*lint !e794 _dest is initialized */ + H5MM_memcpy((void*)_dest, (void*)_src, leftover); + } + } /* end if */ + else { + /* Get the pointer to the destination buffer */ + _dest =(unsigned char *)dest; + + /* Output; shuffle */ + for(i=0; i 0) { + DUFF_GUTS; + + j--; + } /* end for */ +#else /* NO_DUFFS_DEVICE */ + { + size_t duffs_index; /* Counting index for Duff's device */ + + duffs_index = (numofelements + 7) / 8; + switch (numofelements % 8) { + default: + HDassert(0 && "This Should never be executed!"); + break; + case 0: + do + { + DUFF_GUTS + H5_ATTR_FALLTHROUGH + case 7: + DUFF_GUTS + H5_ATTR_FALLTHROUGH + case 6: + DUFF_GUTS + H5_ATTR_FALLTHROUGH + case 5: + DUFF_GUTS + H5_ATTR_FALLTHROUGH + case 4: + DUFF_GUTS + H5_ATTR_FALLTHROUGH + case 3: + DUFF_GUTS + H5_ATTR_FALLTHROUGH + case 2: + DUFF_GUTS + H5_ATTR_FALLTHROUGH + case 1: + DUFF_GUTS + } while (--duffs_index > 0); + } /* end switch */ + } +#endif /* NO_DUFFS_DEVICE */ +#undef DUFF_GUTS + } /* end for */ + + /* Add leftover to the end of data */ + if(leftover>0) { + /* Adjust back to end of shuffled bytes */ + _src -= (bytesoftype - 1); /*lint !e794 _src is initialized */ + H5MM_memcpy((void*)_dest, (void*)_src, leftover); + } + } /* end else */ + + /* Free the input buffer */ + H5MM_xfree(*buf); + + /* Set the buffer information to return */ + *buf = dest; + *buf_size=nbytes; + } /* end else */ + + /* Set the return value */ + ret_value = nbytes; + +done: + return ret_value; +} + diff --git a/plugins/H5Zszip.c b/plugins/H5Zszip.c new file mode 100644 index 0000000000..5c7c0071d0 --- /dev/null +++ b/plugins/H5Zszip.c @@ -0,0 +1,347 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include "netcdf_filter_build.h" + +#ifdef USE_SZIP + +#include +#include "H5Zszip.h" + +/* Local function prototypes */ +static htri_t H5Z__can_apply_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id); +static herr_t H5Z__set_local_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id); +static size_t H5Z__filter_szip(unsigned flags, size_t cd_nelmts, + const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); + +/* This message derives from H5Z */ +H5Z_class2_t H5Z_SZIP[1] = {{ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + H5Z_FILTER_SZIP, /* Filter id number */ + 1, /* Assume encoder present: check before registering */ + 1, /* decoder_present flag (set to true) */ + "szip", /* Filter name for debugging */ + H5Z__can_apply_szip, /* The "can apply" callback */ + H5Z__set_local_szip, /* The "set local" callback */ + H5Z__filter_szip, /* The actual filter function */ +}}; + + + +/*------------------------------------------------------------------------- + * Function: H5Z__can_apply_szip + * + * Purpose: Check the parameters for szip compression for validity and + * whether they fit a particular dataset. + * + * Note: This function currently range-checks for datatypes with + * 8-bit boundaries (8, 16, 24, etc.). It appears that the szip + * library can actually handle 1-24, 32 & 64 bit samples. If + * this becomes important, we should make the checks below more + * sophisticated and have them check for n-bit datatypes of the + * correct size, etc. - QAK + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Quincey Koziol + * Monday, April 7, 2003 + * + *------------------------------------------------------------------------- + */ +static htri_t +H5Z__can_apply_szip(hid_t H5_ATTR_UNUSED dcpl_id, hid_t type_id, hid_t H5_ATTR_UNUSED space_id) +{ +#ifdef USE_HDF5 + unsigned dtype_size; /* Datatype's size (in bits) */ + H5T_order_t dtype_order; /* Datatype's endianness order */ + htri_t ret_value = TRUE; /* Return value */ + + FUNC_ENTER_STATIC + + /* Get datatype's size, for checking the "bits-per-pixel" */ + if((dtype_size = (8 * H5Tget_size(type_id))) == 0) + HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype size") + + /* Range check datatype's size */ + if(dtype_size > 32 && dtype_size != 64) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FALSE, "invalid datatype size") + + /* Get datatype's endianness order */ + if((dtype_order = H5Tget_order(type_id)) == H5T_ORDER_ERROR) + HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "can't retrieve datatype endianness order") + + /* Range check datatype's endianness order */ + /* (Note: this may not handle non-atomic datatypes well) */ + if(dtype_order != H5T_ORDER_LE && dtype_order != H5T_ORDER_BE) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FALSE, "invalid datatype endianness order") + +done: + FUNC_LEAVE_NOAPI(ret_value) +#else + return SUCCEED; +#endif +} /* end H5Z__can_apply_szip() */ + + +/*------------------------------------------------------------------------- + * Function: H5Z__set_local_szip + * + * Purpose: Set the "local" dataset parameters for szip compression. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Quincey Koziol + * Monday, April 7, 2003 + * + *------------------------------------------------------------------------- + */ + +#ifdef USE_HDF5 + +static herr_t +H5Z__set_local_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id) +{ +#ifdef USE_HDF5 + unsigned flags; /* Filter flags */ + size_t cd_nelmts = H5Z_SZIP_USER_NPARMS; /* Number of filter parameters */ + unsigned cd_values[H5Z_SZIP_TOTAL_NPARMS]; /* Filter parameters */ + hsize_t dims[H5S_MAX_RANK+1]; /* Dataspace (i.e. chunk) dimensions */ + int ndims; /* Number of (chunk) dimensions */ + H5T_order_t dtype_order; /* Datatype's endianness order */ + size_t dtype_size; /* Datatype's size (in bits) */ + size_t dtype_precision; /* Datatype's precision (in bits) */ + size_t dtype_offset; /* Datatype's offset (in bits) */ + hsize_t scanline; /* Size of dataspace's fastest changing dimension */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Get the filter's current parameters */ + if(H5Pget_filter_by_id1(dcpl_id, H5Z_FILTER_SZIP, &flags, &cd_nelmts, cd_values, 0, NULL) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "can't get szip parameters") + + /* Get datatype's size, for checking the "bits-per-pixel" */ + if((dtype_size = (8 * H5Tget_size(type_id))) == 0) + HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype size"); + + /* Get datatype's precision, in case is less than full bits */ + if((dtype_precision = H5Tget_precision(type_id)) == 0) + HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype precision"); + + if(dtype_precision < dtype_size) { + dtype_offset = H5Tget_offset(type_id); + if(dtype_offset != 0) + dtype_precision = dtype_size; + } /* end if */ + if(dtype_precision > 24) { + if(dtype_precision <= 32) + dtype_precision = 32; + else if(dtype_precision <= 64) + dtype_precision = 64; + } /* end if */ + + /* Set "local" parameter for this dataset's "bits-per-pixel" */ + cd_values[H5Z_SZIP_PARM_BPP] = dtype_precision; + + /* Get dimensions for dataspace */ + if((ndims = H5Sget_simple_extent_dims(space_id, dims, NULL)) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "unable to get dataspace dimensions") + + /* Set "local" parameter for this dataset's "pixels-per-scanline" */ + /* (Use the chunk's fastest changing dimension size) */ + HDassert(ndims > 0); + scanline = dims[ndims - 1]; + + /* Adjust scanline if it is smaller than number of pixels per block or + if it is bigger than maximum pixels per scanline, or there are more than + SZ_MAX_BLOCKS_PER_SCANLINE blocks per scanline */ + + /* Check the pixels per block against the 'scanline' size */ + if(scanline < cd_values[H5Z_SZIP_PARM_PPB]) { + hssize_t npoints; /* Number of points in the dataspace */ + + /* Get number of elements for the dataspace; use + total number of elements in the chunk to define the new 'scanline' size */ + if((npoints = H5Sget_simple_extent_npoints(space_id)) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "unable to get number of points in the dataspace") + if(npoints < cd_values[H5Z_SZIP_PARM_PPB]) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "pixels per block greater than total number of elements in the chunk") + scanline = MIN((cd_values[H5Z_SZIP_PARM_PPB] * SZ_MAX_BLOCKS_PER_SCANLINE), npoints); + } + else { + if(scanline <= SZ_MAX_PIXELS_PER_SCANLINE) + scanline = MIN((cd_values[H5Z_SZIP_PARM_PPB] * SZ_MAX_BLOCKS_PER_SCANLINE), scanline); + else + scanline = cd_values[H5Z_SZIP_PARM_PPB] * SZ_MAX_BLOCKS_PER_SCANLINE; + } /* end else */ + + /* Assign the final value to the scanline */ + H5_CHECKED_ASSIGN(cd_values[H5Z_SZIP_PARM_PPS], unsigned, scanline, hsize_t); + + /* Get datatype's endianness order */ + if((dtype_order = H5Tget_order(type_id)) == H5T_ORDER_ERROR) + HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype endianness order") + + /* Set the correct endianness flag for szip */ + /* (Note: this may not handle non-atomic datatypes well) */ + cd_values[H5Z_SZIP_PARM_MASK] &= ~(SZ_LSB_OPTION_MASK|SZ_MSB_OPTION_MASK); + switch(dtype_order) { + case H5T_ORDER_LE: /* Little-endian byte order */ + cd_values[H5Z_SZIP_PARM_MASK] |= SZ_LSB_OPTION_MASK; + break; + + case H5T_ORDER_BE: /* Big-endian byte order */ + cd_values[H5Z_SZIP_PARM_MASK] |= SZ_MSB_OPTION_MASK; + break; + + case H5T_ORDER_ERROR: + case H5T_ORDER_VAX: + case H5T_ORDER_MIXED: + case H5T_ORDER_NONE: + default: + HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "bad datatype endianness order") + } /* end switch */ + + /* Modify the filter's parameters for this dataset */ + if(H5Pmodify_filter(dcpl_id, H5Z_FILTER_SZIP, flags, H5Z_SZIP_TOTAL_NPARMS, cd_values) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTSET, FAIL, "can't set local szip parameters") + +done: + FUNC_LEAVE_NOAPI(ret_value) +#else + return SUCCEED; +#endif +} /* end H5Z__set_local_szip() */ + +#endif /*USE_HDF5*/ + + +/*------------------------------------------------------------------------- + * Function: H5Z__filter_szip + * + * Purpose: Implement an I/O filter around the 'rice' algorithm in + * libsz + * + * Return: Success: Size of buffer filtered + * Failure: 0 + * + * Programmer: Kent Yang + * Tuesday, April 1, 2003 + * + *------------------------------------------------------------------------- + */ +static size_t +H5Z__filter_szip(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], + size_t nbytes, size_t *buf_size, void **buf) +{ + size_t ret_value = 0; /* Return value */ + size_t size_out = 0; /* Size of output buffer */ + unsigned char *outbuf = NULL; /* Pointer to new output buffer */ + unsigned char *newbuf = NULL; /* Pointer to input buffer */ + SZ_com_t sz_param; /* szip parameter block */ + + FUNC_ENTER_STATIC + + /* Sanity check to make certain that we haven't drifted out of date with + * the mask options from the szlib.h header */ + HDassert(H5_SZIP_ALLOW_K13_OPTION_MASK==SZ_ALLOW_K13_OPTION_MASK); + HDassert(H5_SZIP_CHIP_OPTION_MASK==SZ_CHIP_OPTION_MASK); + HDassert(H5_SZIP_EC_OPTION_MASK==SZ_EC_OPTION_MASK); + HDassert(H5_SZIP_LSB_OPTION_MASK==SZ_LSB_OPTION_MASK); + HDassert(H5_SZIP_MSB_OPTION_MASK==SZ_MSB_OPTION_MASK); + HDassert(H5_SZIP_NN_OPTION_MASK==SZ_NN_OPTION_MASK); + HDassert(H5_SZIP_RAW_OPTION_MASK==SZ_RAW_OPTION_MASK); + + /* Check arguments */ + if (cd_nelmts!=4) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid number of filter parameters") + + /* Copy the filter parameters into the szip parameter block */ + H5_CHECKED_ASSIGN(sz_param.options_mask, int, cd_values[H5Z_SZIP_PARM_MASK], unsigned); + H5_CHECKED_ASSIGN(sz_param.bits_per_pixel, int, cd_values[H5Z_SZIP_PARM_BPP], unsigned); + H5_CHECKED_ASSIGN(sz_param.pixels_per_block, int, cd_values[H5Z_SZIP_PARM_PPB], unsigned); + H5_CHECKED_ASSIGN(sz_param.pixels_per_scanline, int, cd_values[H5Z_SZIP_PARM_PPS], unsigned); + + /* Input; uncompress */ + if (flags & H5Z_FLAG_REVERSE) { + uint32_t stored_nalloc; /* Number of bytes the compressed block will expand into */ + size_t nalloc; /* Number of bytes the compressed block will expand into */ + + /* Get the size of the uncompressed buffer */ + newbuf = (unsigned char *)(*buf); + UINT32DECODE(newbuf,stored_nalloc); + H5_CHECKED_ASSIGN(nalloc, size_t, stored_nalloc, uint32_t); + + /* Allocate space for the uncompressed buffer */ + if(NULL == (outbuf = (unsigned char *)H5MM_malloc(nalloc))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "memory allocation failed for szip decompression") + + /* Decompress the buffer */ + size_out=nalloc; + if(SZ_BufftoBuffDecompress(outbuf, &size_out, newbuf, nbytes-4, &sz_param) != SZ_OK) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "szip_filter: decompression failed") + HDassert(size_out==nalloc); + + /* Free the input buffer */ + H5MM_xfree(*buf); + + /* Set return values */ + *buf = outbuf; + outbuf = NULL; + *buf_size = nalloc; + ret_value = size_out; + } + /* Output; compress */ + else { + unsigned char *dst = NULL; /* Temporary pointer to new output buffer */ + + /* Allocate space for the compressed buffer & header (assume data won't get bigger) */ + if(NULL == (dst=outbuf = (unsigned char *)H5MM_malloc(nbytes+4))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "unable to allocate szip destination buffer") + + /* Encode the uncompressed length */ + H5_CHECK_OVERFLOW(nbytes,size_t,uint32_t); + UINT32ENCODE(dst,nbytes); + + /* Compress the buffer */ + size_out = nbytes; + if(SZ_OK!= SZ_BufftoBuffCompress(dst, &size_out, *buf, nbytes, &sz_param)) + HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "overflow") + HDassert(size_out<=nbytes); + + /* Free the input buffer */ + H5MM_xfree(*buf); + + /* Set return values */ + *buf = outbuf; + outbuf = NULL; + *buf_size = nbytes+4; + ret_value = size_out+4; + } + +done: + if(outbuf) + H5MM_xfree(outbuf); + FUNC_LEAVE_NOAPI(ret_value) +} + +#endif /* USE_SZIP */ diff --git a/plugins/H5Zszip.h b/plugins/H5Zszip.h new file mode 100755 index 0000000000..1027a624f0 --- /dev/null +++ b/plugins/H5Zszip.h @@ -0,0 +1,14 @@ +#ifndef H5ZSZIP_H +#define H5ZSZIP_H + +/* Macros for the szip filter */ +#define H5Z_SZIP_USER_NPARMS 2 /* Number of parameters that users can set */ +#define H5Z_SZIP_TOTAL_NPARMS 4 /* Total number of parameters for filter */ +#define H5Z_SZIP_PARM_MASK 0 /* "User" parameter for option mask */ +#define H5Z_SZIP_PARM_PPB 1 /* "User" parameter for pixels-per-block */ +#define H5Z_SZIP_PARM_BPP 2 /* "Local" parameter for bits-per-pixel */ +#define H5Z_SZIP_PARM_PPS 3 /* "Local" parameter for pixels-per-scanline */ + +#define MIN(x,y) ((x) < (y) ? (x) : (y)) + +#endif /*H5ZSZIP_H*/ diff --git a/plugins/H5Ztemplate.c b/plugins/H5Ztemplate.c index a1ba9866d2..b57c8a320d 100644 --- a/plugins/H5Ztemplate.c +++ b/plugins/H5Ztemplate.c @@ -3,27 +3,30 @@ #include #include #include -#include -/* Older versions of the hdf library may define H5PL_type_t here */ -#include +#include #include "xxxx.h" /* Provide a textual template (not a C++ template) from which one can construct a new filter. The filter "name is marked with "XXXX" or "xxxx" - */ +static htri_t H5Z_can_apply_xxxx(hid_t dcpl_id, hid_t type_id, hid_t space_id); +static herr_t H5Z_set_local_xxxx)(long long, long long, long long); +static size_t H5Z_filter_xxxx(unsigned int flags, size_t cd_nelmts, + const unsigned int cd_values[], size_t nbytes, + size_t *buf_size, void **buf); + const H5Z_class2_t H5Z_XXXX[1] = {{ - H5Z_CLASS_T_VERS, /* H5Z_class_t version */ - (H5Z_filter_t)H5Z_FILTER_XXXX, /* Filter id number */ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + (H5Z_filter_t)H5Z_FILTER_XXXX, /* Filter id number */ 1, /* encoder_present flag (set to true) */ 1, /* decoder_present flag (set to true) */ "xxxx", /* Filter name for debugging */ - NULL, /* The "can apply" callback */ - NULL, /* The "set local" callback */ - (H5Z_func_t)H5Z_filter_xxxx, /* The actual filter function */ + (H5Z_can_apply_func_t)H5Z_can_apply_xxxx, /* The "can apply" callback | NULL */ + (H5Z_set_local_func_t)H5Z_set_local_xxxx, /* The "set local" callback | NULL */ + (H5Z_func_t)H5Z_filter_xxxx, /* The actual filter function */ }}; /* External Discovery Functions */ @@ -36,15 +39,23 @@ H5PLget_plugin_type(void) const void* H5PLget_plugin_info(void) { - return H5Z_XXXX; + return H5Z_FILTER_XXXX; } -size_t H5Z_filter_xxxx(unsigned int flags, size_t cd_nelmts, - const unsigned int cd_values[], size_t nbytes, - size_t *buf_size, void **buf); +static htri_t +H5Z_can_apply_xxxx(hid_t dcpl_id, hid_t type_id, hid_t space_id) +{ + return 1; /* always applicable */ +} +static herr_t +H5Z_set_local_xxxx)(long long, long long, long long) +{ + return 0; /* always succeed */ +} -size_t H5Z_filter_xxxx(unsigned int flags, size_t cd_nelmts, +static size_t +H5Z_filter_xxxx(unsigned int flags, size_t cd_nelmts, const unsigned int cd_values[], size_t nbytes, size_t *buf_size, void **buf) { @@ -102,3 +113,100 @@ size_t H5Z_filter_xxxx(unsigned int flags, size_t cd_nelmts, H5free_memory(outbuf); return 0; } + + +/**************************************************/ +/* The NCZarr API */ + +/* Forward */ +static int NCZ_xxxx_codec_setup(int ncid, int varid, int* nparamsp, unsigned** paramsp); +static int NCZ_xxxx_codec_shutdown(int nparams, unsigned* params); +static int NCZ_xxxx_codec_to_hdf5(const char* codec, int* nparamsp, unsigned** paramsp); +static int NCZ_xxxx_hdf5_to_codec(int nparams, unsigned* params, char** codecp); + +/* Structure for NCZ_PLUGIN_CODEC */ +static NCZ_codec_t NCZ_xxxx_codec = {/* NCZ_codec_t codec fields */ + NCZ_CODEC_CLASS_VER, /* Struct version number */ + NCZ_CODEC_HDF5, /* Struct sort */ + "xxxx", /* Standard name/id of the codec */ + H5Z_FILTER_XXXX, /* HDF5 alias for xxxx */ + NCZ_xxxx_codec_to_hdf5, + NCZ_xxxx_hdf5_to_codec + NCZ_xxxx_codec_setup, + NCZ_xxxx_codec_shutdown, +}; + +/* External Export API */ +const void* +NCZ_get_plugin_info(void) +{ + return (void*)&NCZ_xxxx_codec; +} + +/* NCZarr Interface Functions */ + +static int +NCZ_xxxx_codec_setup(int ncid, int varid, int* nparamsp, unsigned** paramsp) +{ + return NC_NOERR; +} + +static int +NCZ_xxxx_codec_shutdown(int nparams, unsigned* params) +{ + return NC_NOERR; +} + +static int +NCZ_xxxx_codec_to_hdf5(const char* codec_json, int* nparamsp, unsigned** paramsp) +{ + int stat = NC_NOERR; + NCjson* jcodec = NULL; + NCjson* jtmp = NULL; + int nparams = 0; + unsigned* params = NULL; + struct NCJconst jc; + + if((params = (unsigned*)malloc(sizeof(unsigned)))== NULL) + {stat = NC_ENOMEM; goto done;} + + /* parse the JSON */ + if((stat = NCJparse(codec_json,0,&jcodec))) goto done; + if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;} + /* Verify the codec ID */ + if((stat = NCJdictget(jcodec,"id",&jtmp))) goto done; + if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EINVAL; goto done;} + if(strcmp(NCJstring(jtmp),NCZ_xxxx_codec.codecid)!=0) {stat = NC_EINVAL; goto done;} + + /* Get unsigned integer param */ + if((stat = NCJdictget(jcodec,"param0",&jtmp))) goto done; + if((stat = NCJcvt(jtmp,NCJ_INT,&jc))) goto done; + if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EINVAL; goto done;} + params[i] = (unsigned)jc.ival; + if(nparamsp) *nparamsp = 1; + if(paramsp) {*paramsp = params; params = NULL;} + +done: + if(params) free(params); + NCJreclaim(jcodec); + return stat; +} + +static int +NCZ_xxxx_hdf5_to_codec(int nparams, unsigned* params, char** codecp) +{ + int stat = NC_NOERR; + char json[1024]; + + if(nparams == 0 || params == NULL) + {stat = NC_EINVAL; goto done;} + + snprintf(json,sizeof(json),"{\"id\": \"%s\", \"param0\": \"%u\"}",NCZ_xxxx_codec.codecid,param[0]); + if(codecp) { + if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;} + } + +done: + return stat; +} + diff --git a/plugins/H5Zutil.c b/plugins/H5Zutil.c index ce8ebacd65..16f255035c 100644 --- a/plugins/H5Zutil.c +++ b/plugins/H5Zutil.c @@ -5,7 +5,7 @@ #include "config.h" -#include +#include "netcdf_filter_build.h" /* Common utilities related to filters. diff --git a/plugins/H5checksum.c b/plugins/H5checksum.c new file mode 100644 index 0000000000..c5598735d3 --- /dev/null +++ b/plugins/H5checksum.c @@ -0,0 +1,504 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: H5checksum.c + * Aug 21 2006 + * Quincey Koziol + * + * Purpose: Internal code for computing fletcher32 checksums + * + *------------------------------------------------------------------------- + */ + +/****************/ +/* Module Setup */ +/****************/ + + +/***********/ +/* Headers */ +/***********/ +#include "netcdf_filter_hdf5_build.h" + +/****************/ +/* Local Macros */ +/****************/ + +/* Polynomial quotient */ +/* (same as the IEEE 802.3 (Ethernet) quotient) */ +#define H5_CRC_QUOTIENT 0x04C11DB7 + + +/******************/ +/* Local Typedefs */ +/******************/ + + +/********************/ +/* Package Typedefs */ +/********************/ + + +/********************/ +/* Local Prototypes */ +/********************/ + + +/*********************/ +/* Package Variables */ +/*********************/ + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ + +/* Table of CRCs of all 8-bit messages. */ +static uint32_t H5_crc_table[256]; + +/* Flag: has the table been computed? */ +static hbool_t H5_crc_table_computed = FALSE; + + + +/*------------------------------------------------------------------------- + * Function: H5_checksum_fletcher32 + * + * Purpose: This routine provides a generic, fast checksum algorithm for + * use in the library. + * + * Note: See the Wikipedia page for Fletcher's checksum: + * http://en.wikipedia.org/wiki/Fletcher%27s_checksum + * for more details, etc. + * + * Note #2: Per the information in RFC 3309: + * (http://tools.ietf.org/html/rfc3309) + * Fletcher's checksum is not reliable for small buffers. + * + * Note #3: The algorithm below differs from that given in the Wikipedia + * page by copying the data into 'sum1' in a more portable way + * and also by initializing 'sum1' and 'sum2' to 0 instead of + * 0xffff (for backward compatibility reasons with earlier + * HDF5 fletcher32 I/O filter routine, mostly). + * + * Return: 32-bit fletcher checksum of input buffer (can't fail) + * + * Programmer: Quincey Koziol + * Monday, August 21, 2006 + * + *------------------------------------------------------------------------- + */ +uint32_t +H5_checksum_fletcher32(const void *_data, size_t _len) +{ + const uint8_t *data = (const uint8_t *)_data; /* Pointer to the data to be summed */ + size_t len = _len / 2; /* Length in 16-bit words */ + uint32_t sum1 = 0, sum2 = 0; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Sanity check */ + HDassert(_data); + HDassert(_len > 0); + + /* Compute checksum for pairs of bytes */ + /* (the magic "360" value is is the largest number of sums that can be + * performed without numeric overflow) + */ + while (len) { + size_t tlen = len > 360 ? 360 : len; + len -= tlen; + do { + sum1 += (uint32_t)(((uint16_t)data[0]) << 8) | ((uint16_t)data[1]); + data += 2; + sum2 += sum1; + } while (--tlen); + sum1 = (sum1 & 0xffff) + (sum1 >> 16); + sum2 = (sum2 & 0xffff) + (sum2 >> 16); + } + + /* Check for odd # of bytes */ + if(_len % 2) { + sum1 += (uint32_t)(((uint16_t)*data) << 8); + sum2 += sum1; + sum1 = (sum1 & 0xffff) + (sum1 >> 16); + sum2 = (sum2 & 0xffff) + (sum2 >> 16); + } /* end if */ + + /* Second reduction step to reduce sums to 16 bits */ + sum1 = (sum1 & 0xffff) + (sum1 >> 16); + sum2 = (sum2 & 0xffff) + (sum2 >> 16); + + FUNC_LEAVE_NOAPI((sum2 << 16) | sum1) +} /* end H5_checksum_fletcher32() */ + + +/*------------------------------------------------------------------------- + * Function: H5__checksum_crc_make_table + * + * Purpose: Compute the CRC table for the CRC checksum algorithm + * + * Return: none + * + * Programmer: Quincey Koziol + * Tuesday, September 5, 2006 + * + *------------------------------------------------------------------------- + */ +static void +H5__checksum_crc_make_table(void) +{ + uint32_t c; /* Checksum for each byte value */ + unsigned n, k; /* Local index variables */ + + FUNC_ENTER_STATIC_NOERR + + /* Compute the checksum for each possible byte value */ + for(n = 0; n < 256; n++) { + c = (uint32_t) n; + for(k = 0; k < 8; k++) + if(c & 1) + c = H5_CRC_QUOTIENT ^ (c >> 1); + else + c = c >> 1; + H5_crc_table[n] = c; + } + H5_crc_table_computed = TRUE; + + FUNC_LEAVE_NOAPI_VOID +} /* end H5__checksum_crc_make_table() */ + + +/*------------------------------------------------------------------------- + * Function: H5__checksum_crc_update + * + * Purpose: Update a running CRC with the bytes buf[0..len-1]--the CRC + * should be initialized to all 1's, and the transmitted value + * is the 1's complement of the final running CRC (see the + * H5_checksum_crc() routine below)). + * + * Return: 32-bit CRC checksum of input buffer (can't fail) + * + * Programmer: Quincey Koziol + * Tuesday, September 5, 2006 + * + *------------------------------------------------------------------------- + */ +static uint32_t +H5__checksum_crc_update(uint32_t crc, const uint8_t *buf, size_t len) +{ + size_t n; /* Local index variable */ + + FUNC_ENTER_STATIC_NOERR + + /* Initialize the CRC table if necessary */ + if(!H5_crc_table_computed) + H5__checksum_crc_make_table(); + + /* Update the CRC with the results from this buffer */ + for(n = 0; n < len; n++) + crc = H5_crc_table[(crc ^ buf[n]) & 0xff] ^ (crc >> 8); + + FUNC_LEAVE_NOAPI(crc) +} /* end H5__checksum_crc_update() */ + + +/*------------------------------------------------------------------------- + * Function: H5_checksum_crc + * + * Purpose: This routine provides a generic checksum algorithm for + * use in the library. + * + * Note: This algorithm was based on the implementation described + * in the document describing the PNG image format: + * http://www.w3.org/TR/PNG/#D-CRCAppendix + * + * Return: 32-bit CRC checksum of input buffer (can't fail) + * + * Programmer: Quincey Koziol + * Tuesday, September 5, 2006 + * + *------------------------------------------------------------------------- + */ +uint32_t +H5_checksum_crc(const void *_data, size_t len) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Sanity check */ + HDassert(_data); + HDassert(len > 0); + + FUNC_LEAVE_NOAPI(H5__checksum_crc_update((uint32_t)0xffffffffL, (const uint8_t *)_data, len) ^ 0xffffffffL) +} /* end H5_checksum_crc() */ + +/* +------------------------------------------------------------------------------- +H5_lookup3_mix -- mix 3 32-bit values reversibly. + +This is reversible, so any information in (a,b,c) before mix() is +still in (a,b,c) after mix(). + +If four pairs of (a,b,c) inputs are run through mix(), or through +mix() in reverse, there are at least 32 bits of the output that +are sometimes the same for one pair and different for another pair. +This was tested for: +* pairs that differed by one bit, by two bits, in any combination + of top bits of (a,b,c), or in any combination of bottom bits of + (a,b,c). +* "differ" is defined as +, -, ^, or ~^. For + and -, I transformed + the output delta to a Gray code (a^(a>>1)) so a string of 1's (as + is commonly produced by subtraction) look like a single 1-bit + difference. +* the base values were pseudorandom, all zero but one bit set, or + all zero plus a counter that starts at zero. + +Some k values for my "a-=c; a^=rot(c,k); c+=b;" arrangement that +satisfy this are + 4 6 8 16 19 4 + 9 15 3 18 27 15 + 14 9 3 7 17 3 +Well, "9 15 3 18 27 15" didn't quite get 32 bits diffing +for "differ" defined as + with a one-bit base and a two-bit delta. I +used http://burtleburtle.net/bob/hash/avalanche.html to choose +the operations, constants, and arrangements of the variables. + +This does not achieve avalanche. There are input bits of (a,b,c) +that fail to affect some output bits of (a,b,c), especially of a. The +most thoroughly mixed value is c, but it doesn't really even achieve +avalanche in c. + +This allows some parallelism. Read-after-writes are good at doubling +the number of bits affected, so the goal of mixing pulls in the opposite +direction as the goal of parallelism. I did what I could. Rotates +seem to cost as much as shifts on every machine I could lay my hands +on, and rotates are much kinder to the top and bottom bits, so I used +rotates. +------------------------------------------------------------------------------- +*/ +#define H5_lookup3_rot(x,k) (((x)<<(k)) ^ ((x)>>(32-(k)))) +#define H5_lookup3_mix(a,b,c) \ +{ \ + a -= c; a ^= H5_lookup3_rot(c, 4); c += b; \ + b -= a; b ^= H5_lookup3_rot(a, 6); a += c; \ + c -= b; c ^= H5_lookup3_rot(b, 8); b += a; \ + a -= c; a ^= H5_lookup3_rot(c,16); c += b; \ + b -= a; b ^= H5_lookup3_rot(a,19); a += c; \ + c -= b; c ^= H5_lookup3_rot(b, 4); b += a; \ +} + +/* +------------------------------------------------------------------------------- +H5_lookup3_final -- final mixing of 3 32-bit values (a,b,c) into c + +Pairs of (a,b,c) values differing in only a few bits will usually +produce values of c that look totally different. This was tested for +* pairs that differed by one bit, by two bits, in any combination + of top bits of (a,b,c), or in any combination of bottom bits of + (a,b,c). +* "differ" is defined as +, -, ^, or ~^. For + and -, I transformed + the output delta to a Gray code (a^(a>>1)) so a string of 1's (as + is commonly produced by subtraction) look like a single 1-bit + difference. +* the base values were pseudorandom, all zero but one bit set, or + all zero plus a counter that starts at zero. + +These constants passed: + 14 11 25 16 4 14 24 + 12 14 25 16 4 14 24 +and these came close: + 4 8 15 26 3 22 24 + 10 8 15 26 3 22 24 + 11 8 15 26 3 22 24 +------------------------------------------------------------------------------- +*/ +#define H5_lookup3_final(a,b,c) \ +{ \ + c ^= b; c -= H5_lookup3_rot(b,14); \ + a ^= c; a -= H5_lookup3_rot(c,11); \ + b ^= a; b -= H5_lookup3_rot(a,25); \ + c ^= b; c -= H5_lookup3_rot(b,16); \ + a ^= c; a -= H5_lookup3_rot(c,4); \ + b ^= a; b -= H5_lookup3_rot(a,14); \ + c ^= b; c -= H5_lookup3_rot(b,24); \ +} + +/* +------------------------------------------------------------------------------- +H5_checksum_lookup3() -- hash a variable-length key into a 32-bit value + k : the key (the unaligned variable-length array of bytes) + length : the length of the key, counting by bytes + initval : can be any 4-byte value +Returns a 32-bit value. Every bit of the key affects every bit of +the return value. Two keys differing by one or two bits will have +totally different hash values. + +The best hash table sizes are powers of 2. There is no need to do +mod a prime (mod is sooo slow!). If you need less than 32 bits, +use a bitmask. For example, if you need only 10 bits, do + h = (h & hashmask(10)); +In which case, the hash table should have hashsize(10) elements. + +If you are hashing n strings (uint8_t **)k, do it like this: + for (i=0, h=0; i 0); + + /* Set up the internal state */ + a = b = c = 0xdeadbeef + ((uint32_t)length) + initval; + + /*--------------- all but the last block: affect some 32 bits of (a,b,c) */ + while (length > 12) + { + a += k[0]; + a += ((uint32_t)k[1])<<8; + a += ((uint32_t)k[2])<<16; + a += ((uint32_t)k[3])<<24; + b += k[4]; + b += ((uint32_t)k[5])<<8; + b += ((uint32_t)k[6])<<16; + b += ((uint32_t)k[7])<<24; + c += k[8]; + c += ((uint32_t)k[9])<<8; + c += ((uint32_t)k[10])<<16; + c += ((uint32_t)k[11])<<24; + H5_lookup3_mix(a, b, c); + length -= 12; + k += 12; + } + + /*-------------------------------- last block: affect all 32 bits of (c) */ + switch(length) /* all the case statements fall through */ + { + case 12: c+=((uint32_t)k[11])<<24; + H5_ATTR_FALLTHROUGH + case 11: c+=((uint32_t)k[10])<<16; + H5_ATTR_FALLTHROUGH + case 10: c+=((uint32_t)k[9])<<8; + H5_ATTR_FALLTHROUGH + case 9 : c+=k[8]; + H5_ATTR_FALLTHROUGH + case 8 : b+=((uint32_t)k[7])<<24; + H5_ATTR_FALLTHROUGH + case 7 : b+=((uint32_t)k[6])<<16; + H5_ATTR_FALLTHROUGH + case 6 : b+=((uint32_t)k[5])<<8; + H5_ATTR_FALLTHROUGH + case 5 : b+=k[4]; + H5_ATTR_FALLTHROUGH + case 4 : a+=((uint32_t)k[3])<<24; + H5_ATTR_FALLTHROUGH + case 3 : a+=((uint32_t)k[2])<<16; + H5_ATTR_FALLTHROUGH + case 2 : a+=((uint32_t)k[1])<<8; + H5_ATTR_FALLTHROUGH + case 1 : a+=k[0]; + break; + case 0 : goto done; + default: + HDassert(0 && "This Should never be executed!"); + } + + H5_lookup3_final(a, b, c); + +done: + FUNC_LEAVE_NOAPI(c) +} /* end H5_checksum_lookup3() */ + + +/*------------------------------------------------------------------------- + * Function: H5_checksum_metadata + * + * Purpose: Provide a more abstract routine for checksumming metadata + * in a file, where the policy of which algorithm to choose + * is centralized. + * + * Return: checksum of input buffer (can't fail) + * + * Programmer: Quincey Koziol + * Tuesday, August 22, 2006 + * + *------------------------------------------------------------------------- + */ +uint32_t +H5_checksum_metadata(const void *data, size_t len, uint32_t initval) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Sanity check */ + HDassert(data); + HDassert(len > 0); + + /* Choose the appropriate checksum routine */ + /* (use Bob Jenkin's "lookup3" algorithm for all buffer sizes) */ + FUNC_LEAVE_NOAPI(H5_checksum_lookup3(data, len, initval)) +} /* end H5_checksum_metadata() */ + + +/*------------------------------------------------------------------------- + * Function: H5_hash_string + * + * Purpose: Provide a simple & fast routine for hashing strings + * + * Note: This algorithm is the 'djb2' algorithm described on this page: + * http://www.cse.yorku.ca/~oz/hash.html + * + * Return: hash of input string (can't fail) + * + * Programmer: Quincey Koziol + * Tuesday, December 11, 2007 + * + *------------------------------------------------------------------------- + */ +uint32_t +H5_hash_string(const char *str) +{ + uint32_t hash = 5381; + int c; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Sanity check */ + HDassert(str); + + while((c = *str++)) + hash = ((hash << 5) + hash) + (uint32_t)c; /* hash * 33 + c */ + + FUNC_LEAVE_NOAPI(hash) +} /* end H5_hash_string() */ + diff --git a/plugins/Makefile.am b/plugins/Makefile.am index 9000e6b4b7..81892e581a 100644 --- a/plugins/Makefile.am +++ b/plugins/Makefile.am @@ -8,37 +8,51 @@ AM_LDFLAGS += -module -avoid-version -shared -export-dynamic \ -rpath ${abs_builddir} ${NOUNDEFINED} # BZIP2 version 1.0.8 (https://sourceware.org/bzip2/) -BZIP2HDRS = bzlib.h bzlib_private.h -BZIP2SRC = blocksort.c huffman.c crctable.c randtable.c compress.c decompress.c bzlib.c +BZIP2SRC = blocksort.c huffman.c crctable.c randtable.c compress.c decompress.c bzlib.c bzlib.h bzlib_private.h -PLUGINSRC = H5Zbzip2.c -PLUGINHDRS = h5bzip2.h - -EXTRA_DIST = ${PLUGINSRC} ${BZIP2SRC} ${PLUGINHDRS} ${BZIP2HDRS} \ -H5Ztemplate.c H5Zmisc.c H5Zutil.c H5Znoop.c h5noop.h CMakeLists.txt - -# WARNING: This list must be kept consistent with the corresponding -# AC_CONFIG_LINK commands near the end of configure.ac. -HDF5PLUGINSRC = ${PLUGINSRC} ${BZIP2SRC} ${PLUGINHDRS} ${BZIP2HDRS} +EXTRA_DIST = CMakeLists.txt \ + H5Ztemplate.c H5Zmisc.c H5Zutil.c H5Znoop.c h5noop.h NCZmisc.c \ + NCZdefault.c \ + H5Zbzip2.c H5Zbzip2.h H5Zblosc.c H5Zblosc.h \ + H5Zshuffle.c H5Zfletcher32.h H5Zdeflate.c H5Zszip.c H5Zszip.h \ + ${BZIP2SRC} H5checksum.c if ENABLE_FILTER_TESTING -noinst_LTLIBRARIES = libmisc.la libnoop.la libnoop1.la lib_LTLIBRARIES = libh5bzip2.la +libh5bzip2_la_SOURCES = H5Zbzip2.c H5Zbzip2.h ${BZIP2SRC} + +noinst_LTLIBRARIES = libh5misc.la libh5noop.la libh5noop1.la libnczmisc.la libnczdefaults.la \ + libh5shuffle.la libh5fletcher32.la libh5deflate.la + +libh5shuffle_la_SOURCES = H5Zshuffle.c +libh5fletcher32_la_SOURCES = H5Zfletcher32.c H5checksum.c +libh5deflate_la_SOURCES = H5Zdeflate.c + +if ENABLE_BLOSC +noinst_LTLIBRARIES += libh5blosc.la +libh5blosc_la_SOURCES = H5Zblosc.c H5Zblosc.h +endif -libh5bzip2_la_SOURCES = ${HDF5PLUGINSRC} +if ENABLE_SZIP +noinst_LTLIBRARIES += libh5szip.la +libh5szip_la_SOURCES = H5Zszip.c H5Zszip.h +endif -libmisc_la_SOURCES = H5Zmisc.c H5Zutil.c h5misc.h +libh5misc_la_SOURCES = H5Zmisc.c H5Zutil.c h5misc.h + +libnczmisc_la_SOURCES = NCZmisc.c + +libnczdefaults_la_SOURCES = NCZdefaults.c # The noop filter is to allow testing of multifilters and filter order # Need two distinct instances -libnoop_la_SOURCES = H5Znoop.c H5Zutil.c h5noop.h -libnoop1_la_SOURCES = H5Znoop1.c H5Zutil.c h5noop.h - +libh5noop_la_SOURCES = H5Znoop.c H5Zutil.c h5noop.h +libh5noop1_la_SOURCES = H5Znoop1.c H5Zutil.c h5noop.h endif #ENABLE_FILTER_TESTING BUILT_SOURCES = H5Znoop1.c -DISTCLEANFILES = H5Znoop1.c +DISTCLEANFILES = H5Znoop1.c ncjson.h H5Znoop1.c: Makefile H5Znoop.c echo '#define NOOP_INSTANCE 1' > $@ cat ${srcdir}/H5Znoop.c >> $@ @@ -47,4 +61,3 @@ H5Znoop1.c: Makefile H5Znoop.c BZIP2DIR=/cygdrive/d/bzip2-1.0.8 updatebzip2: for b in ${BZIP2HDRS} ${BZIP2SRC} ; do cp -f ${BZIP2DIR}/$${b} . ; done - diff --git a/plugins/NCZdefaults.c b/plugins/NCZdefaults.c new file mode 100644 index 0000000000..f0b6f52e7b --- /dev/null +++ b/plugins/NCZdefaults.c @@ -0,0 +1,534 @@ +/* Copyright 2003-2018, University Corporation for Atmospheric + * Research. See the COPYRIGHT file for copying and redistribution + * conditions. + */ + +/* +Author: Dennis Heimbigner +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#ifdef USE_SZIP +#include +#include "H5Zszip.h" +#endif + +#include "netcdf.h" +#include "netcdf_filter.h" +#include "netcdf_filter_build.h" +#include "netcdf_json.h" + +#define H5Z_FILTER_DEFLATE 1 /*deflation like gzip */ +#define H5Z_FILTER_SHUFFLE 2 /*shuffle the data */ +#define H5Z_FILTER_FLETCHER32 3 /*fletcher32 checksum of EDC */ +#define H5Z_FILTER_SZIP 4 /*szip compression */ + +/**************************************************/ +/* NCZarr Filter Objects */ + +/* Forward */ +static int NCZ_shuffle_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp); +static int NCZ_shuffle_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp); +static int NCZ_shuffle_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp); + +static int NCZ_fletcher32_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp); +static int NCZ_fletcher32_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp); +static int NCZ_fletcher32_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp); + +static int NCZ_deflate_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp); +static int NCZ_deflate_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp); + +#ifdef USE_SZIP +static int NCZ_szip_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp); +static int NCZ_szip_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp); +static int NCZ_szip_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp); +#endif + +/**************************************************/ + +static NCZ_codec_t NCZ_shuffle_codec = { + NCZ_CODEC_CLASS_VER, /* Struct version number */ + NCZ_CODEC_HDF5, /* Struct sort */ + "shuffle", /* Standard name/id of the codec */ + H5Z_FILTER_SHUFFLE, /* HDF5 alias for shuffle */ + NULL, /*NCZ_shuffle_codec_initialize*/ + NULL, /*NCZ_shuffle_codec_finalize*/ + NCZ_shuffle_codec_to_hdf5, + NCZ_shuffle_hdf5_to_codec, + NCZ_shuffle_modify_parameters, +}; + +static int +NCZ_shuffle_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp) +{ + int stat = NC_NOERR; + + /* Ignore any incoming elementsize */ + + if(nparamsp) *nparamsp = 0; + + return stat; +} + +static int +NCZ_shuffle_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp) +{ + int stat = NC_NOERR; + char json[1024]; + unsigned typesize = 0; + + if(nparams > 0) + typesize = params[0]; + snprintf(json,sizeof(json),"{\"id\": \"%s\", \"elementsize\": \"%u\"}",NCZ_shuffle_codec.codecid,typesize); + if(codecp) { + if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;} + } + +done: + return stat; +} + +static int +NCZ_shuffle_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp) +{ + int stat = NC_NOERR; + nc_type vtype; + size_t typesize; + char vname[NC_MAX_NAME+1]; + unsigned int* params = NULL; + + /* Ignore the visible parameters */ + + if(!wnparamsp || !wparamsp) {stat = NC_EINTERNAL; goto done;} + + /* Get variable info */ + if((stat = nc_inq_var(ncid,varid,vname,&vtype,NULL,NULL,NULL))) goto done; + + /* Get the typesize */ + if((stat = nc_inq_type(ncid,vtype,NULL,&typesize))) goto done; + + if((params=(unsigned*)malloc(sizeof(unsigned)))==NULL) + {stat = NC_ENOMEM; goto done;} + + *wnparamsp = 1; + nullfree(*wparamsp); + *wparamsp = params; params = NULL; + +done: + nullfree(params); + return stat; +} + +#if 0 +static int +NCZ_shuffle_visible_parameters(int ncid, int varid, size_t nparamsin, const unsigned int* paramsin, size_t* nparamsp, unsigned** paramsp) +{ + int stat = NC_NOERR; + unsigned* params = NULL; + size_t typesize = 0; + + if(!nparamsp || !paramsp) {stat = NC_EINTERNAL; goto done;} + + if(nparamsin > 0) + typesize = paramsin[0]; + + if((params=(unsigned*)malloc(sizeof(unsigned)))==NULL) + {stat = NC_ENOMEM; goto done;} + + params[0] = (unsigned)typesize; + + /* add the typesize as a new parameter */ + nullfree(*paramsp); + *paramsp = params; params = NULL; + *nparamsp = 1; + +done: + nullfree(params); + return stat; +} +#endif + +/**************************************************/ + +static NCZ_codec_t NCZ_fletcher32_codec = {/* NCZ_codec_t codec fields */ + NCZ_CODEC_CLASS_VER, /* Struct version number */ + NCZ_CODEC_HDF5, /* Struct sort */ + "fletcher32", /* Standard name/id of the codec */ + H5Z_FILTER_FLETCHER32, /* HDF5 alias for zlib */ + NULL, /*NCZ_fletcher32_codec_initialize*/ + NULL, /*NCZ_fletcher32_codec_finalize*/ + NCZ_fletcher32_codec_to_hdf5, + NCZ_fletcher32_hdf5_to_codec, + NCZ_fletcher32_modify_parameters, +}; + +static int +NCZ_fletcher32_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp) +{ + int stat = NC_NOERR; + + NC_UNUSED(codec); + if(!nparamsp || !paramsp) {stat = NC_EINTERNAL; goto done;} + + *nparamsp = 0; + nullfree(*paramsp); + *paramsp = NULL; + +done: + return stat; +} + +static int +NCZ_fletcher32_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp) +{ + int stat = NC_NOERR; + char json[1024]; + + snprintf(json,sizeof(json),"{\"id\": \"%s\"}",NCZ_fletcher32_codec.codecid); + if(codecp) { + if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;} + } + +done: + return stat; +} + +static int +NCZ_fletcher32_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp) +{ + int stat = NC_NOERR; + + /* Ignore the visible parameters */ + + if(!wnparamsp || !wparamsp) {stat = NC_EINTERNAL; goto done;} + + *wnparamsp = 0; + nullfree(*wparamsp); + *wparamsp = NULL; + +done: + return stat; +} + +#if 0 +static int +NCZ_fletcher32_visible_parameters(int ncid, int varid, size_t nparamsin, const unsigned int* paramsin, size_t* nparamsp, unsigned** paramsp) +{ + int stat = NC_NOERR; + + if(!nparamsp || !paramsp) {stat = NC_EINTERNAL; goto done;} + + nullfree(*paramsp); *paramsp = NULL; + *nparamsp = 0; + +done: + return stat; +} +#endif + +/**************************************************/ + +static NCZ_codec_t NCZ_zlib_codec = {/* NCZ_codec_t codec fields */ + NCZ_CODEC_CLASS_VER, /* Struct version number */ + NCZ_CODEC_HDF5, /* Struct sort */ + "zlib", /* Standard name/id of the codec */ + H5Z_FILTER_DEFLATE, /* HDF5 alias for zlib */ + NULL, /*NCZ_deflate_codec_initialize*/ + NULL, /*NCZ_deflate_codec_finalize*/ + NCZ_deflate_codec_to_hdf5, + NCZ_deflate_hdf5_to_codec, + NULL, /*NCZ_deflate_modify_parameters*/ +}; + +static int +NCZ_deflate_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp) +{ + int stat = NC_NOERR; + NCjson* jcodec = NULL; + NCjson* jtmp = NULL; + unsigned* params = NULL; + struct NCJconst jc; + + if(nparamsp == NULL || paramsp == NULL) + {stat = NC_EINTERNAL; goto done;} + + if((params = (unsigned*)calloc(1,sizeof(unsigned)))== NULL) + {stat = NC_ENOMEM; goto done;} + + /* parse the JSON */ + if(NCJparse(codec_json,0,&jcodec)) + {stat = NC_EFILTER; goto done;} + if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;} + /* Verify the codec ID */ + if(NCJdictget(jcodec,"id",&jtmp)) + {stat = NC_EFILTER; goto done;} + if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EFILTER; goto done;} + if(strcmp(NCJstring(jtmp),NCZ_zlib_codec.codecid)!=0) {stat = NC_EINVAL; goto done;} + + /* Get Level */ + if(NCJdictget(jcodec,"level",&jtmp)) + {stat = NC_EFILTER; goto done;} + if(NCJcvt(jtmp,NCJ_INT,&jc)) + {stat = NC_EFILTER; goto done;} + if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EINVAL; goto done;} + params[0] = (unsigned)jc.ival; + *nparamsp = 1; + *paramsp = params; params = NULL; + +done: + if(params) free(params); + NCJreclaim(jcodec); + return stat; +} + +static int +NCZ_deflate_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp) +{ + int stat = NC_NOERR; + unsigned level = 0; + char json[1024]; + + if(nparams == 0 || params == NULL) + {stat = NC_EFILTER; goto done;} + + level = params[0]; + snprintf(json,sizeof(json),"{\"id\": \"%s\", \"level\": \"%u\"}",NCZ_zlib_codec.codecid,level); + if(codecp) { + if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;} + } + +done: + return stat; +} + +/**************************************************/ + +#ifdef USE_SZIP + +static NCZ_codec_t NCZ_szip_codec = { + NCZ_CODEC_CLASS_VER, /* Struct version number */ + NCZ_CODEC_HDF5, /* Struct sort */ + "szip", /* Standard name/id of the codec */ + H5Z_FILTER_SZIP, /* HDF5 alias for szip */ + NULL, /*NCZ_szip_codec_initialize*/ + NULL, /*NCZ_szip_codec_finalize*/ + NCZ_szip_codec_to_hdf5, + NCZ_szip_hdf5_to_codec, + NCZ_szip_modify_parameters, +}; + +static int +NCZ_szip_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp) +{ + int stat = NC_NOERR; + unsigned* params = NULL; + size_t nparams = 2; /* No. of visible parameters */ + NCjson* json = NULL; + NCjson* jtmp = NULL; + struct NCJconst jc = {0,0,0,NULL}; + + if(nparamsp == NULL || paramsp == NULL) + {stat = NC_EINTERNAL; goto done;} + + if((params = (unsigned*)calloc(nparams,sizeof(unsigned)))== NULL) + {stat = NC_ENOMEM; goto done;} + + if(NCJparse(codec_json,0,&json)) + {stat = NC_EFILTER; goto done;} + + if(NCJdictget(json,"mask",&jtmp) || jtmp == NULL) + {stat = NC_EFILTER; goto done;} + if(NCJcvt(jtmp,NCJ_INT,&jc)) + {stat = NC_EFILTER; goto done;} + params[H5Z_SZIP_PARM_MASK] = (unsigned)jc.ival; + + jtmp = NULL; + if(NCJdictget(json,"pixels-per-block",&jtmp) || jtmp == NULL) + {stat = NC_EFILTER; goto done;} + if(NCJcvt(jtmp,NCJ_INT,&jc)) + {stat = NC_EFILTER; goto done;} + params[H5Z_SZIP_PARM_PPB] = (unsigned)jc.ival; + + *nparamsp = nparams; + *paramsp = params; params = NULL; + +done: + NCJreclaim(json); + nullfree(params); + return stat; +} + +static int +NCZ_szip_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp) +{ + int stat = NC_NOERR; + char json[2048]; + + snprintf(json,sizeof(json),"{\"id\": \"%s\", \"mask\": \"%u\", \"pixels-per-block\": \"%u\"}", + NCZ_szip_codec.codecid, + params[H5Z_SZIP_PARM_MASK], + params[H5Z_SZIP_PARM_PPB]); + if(codecp) { + if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;} + } + +done: + return stat; +} + +static int +NCZ_szip_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp) +{ + int i,ret_value = NC_NOERR; + nc_type vtype; + size_t typesize, scanline, dtype_precision, npoints; + int ndims, storage, dtype_order; + int dimids[NC_MAX_VAR_DIMS]; + char vname[NC_MAX_NAME+1]; + size_t chunklens[NC_MAX_VAR_DIMS]; + unsigned* params = NULL; + unsigned* vparams = NULL; + size_t wnparams = 4; + + if(wnparamsp == NULL || wparamsp == NULL) + {ret_value = NC_EFILTER; goto done;} + if(vnparamsp == NULL || vparamsp == NULL) + {ret_value = NC_EFILTER; goto done;} + if(*vnparamsp > 0 && *vparamsp == NULL) + {ret_value = NC_EFILTER; goto done;} + + vparams = *vparamsp; + + /* Get variable info */ + if((ret_value = nc_inq_var(ncid,varid,vname,&vtype,&ndims,dimids,NULL))) goto done; + + /* Get the typesize */ + if((ret_value = nc_inq_type(ncid,vtype,NULL,&typesize))) goto done; + + /* Get datatype's precision, in case is less than full bits */ + dtype_precision = typesize; + + if(dtype_precision > 24) { + if(dtype_precision <= 32) + dtype_precision = 32; + else if(dtype_precision <= 64) + dtype_precision = 64; + } /* end if */ + + if(ndims == 0) {ret_value = NC_EFILTER; goto done;} + /* Set "local" parameter for this dataset's "pixels-per-scanline" */ + if((ret_value = nc_inq_dimlen(ncid,dimids[ndims-1],&scanline))) goto done; + + /* Get number of elements for the dataspace; use + total number of elements in the chunk to define the new 'scanline' size */ + /* Compute chunksize */ + if((ret_value = nc_inq_var_chunking(ncid,varid,&storage,chunklens))) goto done; + if(storage != NC_CHUNKED) {ret_value = NC_EFILTER; goto done;} + npoints = 1; + for(i=0;i +#include +#include +#include +#include + +#include "netcdf_json.h" + +#include "netcdf.h" +#include "netcdf_filter.h" +#include "netcdf_filter_build.h" + +#include "h5misc.h" + +/**************************************************/ +/* NCZarr Filter Objects */ +/* Codec Format +{ +"id": "test", +"testcase": "n", +"byte": "", +"ubyte": "", +"short": "", +"ushort": "", +"int": "", +"uint": "", +"float": "", +"double0": "", +"double1": "", +"int640": "", +"int641": "", +"uint640": "", +"uint641": "", +} +*/ + +/* Give unique dict key names for parameters */ +static const char* fields[14] = { +"testcase", +"byte", +"ubyte", +"short", +"ushort", +"int", +"uint", +"float", +"double_0", +"double_1", +"int64_0", +"int64_1", +"uint64_0", +"uint64_1" +}; + +/* Forward */ +static int NCZ_misc_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp); +static int NCZ_misc_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp); + +/* Structure for NCZ_PLUGIN_CODEC */ +static NCZ_codec_t NCZ_misc_codec = {/* NCZ_codec_t codec fields */ + NCZ_CODEC_CLASS_VER, /* Struct version number */ + NCZ_CODEC_HDF5, /* Struct sort */ + "test", /* Standard name/id of the codec */ + H5Z_FILTER_TEST, /* HDF5 alias for misc */ + NULL, /*NCZ_misc_codec_initialize*/ + NULL, /*NCZ_misc_codec_finalize*/ + NCZ_misc_codec_to_hdf5, + NCZ_misc_hdf5_to_codec, + NULL, /*NCZ_misc_modify_parameters*/ +}; + +/* External Export API */ +DLLEXPORT +const void* +NCZ_get_codec_info(void) +{ + return (void*)&NCZ_misc_codec; +} + +/* NCZarr Interface Functions */ + +static int +NCZ_misc_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp) +{ + int stat = NC_NOERR; + NCjson* jcodec = NULL; + NCjson* jtmp = NULL; + size_t i,nparams = 0; + unsigned* params = NULL; + + /* parse the JSON */ + if(NCJparse(codec_json,0,&jcodec)) + {stat = NC_EFILTER; goto done;} + if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;} + + /* Verify the codec ID */ + if(NCJdictget(jcodec,"id",&jtmp)) + {stat = NC_EFILTER; goto done;} + if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EINVAL; goto done;} + if(strcmp(NCJstring(jtmp),NCZ_misc_codec.codecid)!=0) {stat = NC_EINVAL; goto done;} + + /* The codec will have (2*14 + 1) +1 = 29 dict entries + id*/ + nparams = (2*14 + 1) + 1; + if(NCJlength(jcodec) != nparams) { + fprintf(stderr,"Incorrect no. of codec parameters: need=29 sent=%ld\n",(unsigned long)(nparams-1)); + stat = NC_EINVAL; + goto done; + } + + /* Actual # of parameters is 14 (ignoring the testcase number) */ + nparams = 14; + if((params = (unsigned*)calloc(nparams,sizeof(unsigned)))== NULL) + {stat = NC_ENOMEM; goto done;} + + for(i=0;i NC_MAX_UINT) {stat = NC_EINVAL; goto done;} + params[i] = (unsigned)jc.ival; + } + if(nparamsp) *nparamsp = nparams; + if(paramsp) {*paramsp = params; params = NULL;} + +done: + if(params) free(params); + NCJreclaim(jcodec); + return stat; +} + +static int +NCZ_misc_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp) +{ + int i,stat = NC_NOERR; + char json[4096]; + char value[1024]; + + if(nparams == 0 || params == NULL) + {stat = NC_EINVAL; goto done;} + if(nparams != 14) { + fprintf(stderr,"Incorrect no. of parameters: need=14 sent=%ld\n",(unsigned long)nparams); + stat = NC_EINVAL; + goto done; + } + snprintf(json,sizeof(json),"{\"id\": \"%s\"",NCZ_misc_codec.codecid); + for(i=0;i<14;i++) { + snprintf(value,sizeof(value),", \"%s\": \"%u\"",fields[i],params[i]); + strlcat(json,value,sizeof(json)); + } + strlcat(json,"}",sizeof(json)); + if(codecp) { + if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;} + } + +done: + return stat; +} diff --git a/plugins/h5bzip2.h b/plugins/h5bzip2.h index 8a9a06c02d..380ff11ed7 100644 --- a/plugins/h5bzip2.h +++ b/plugins/h5bzip2.h @@ -21,10 +21,5 @@ DECLSPEC H5PL_type_t H5PLget_plugin_type(void); DECLSPEC const void* H5PLget_plugin_info(void); DECLSPEC const H5Z_class2_t H5Z_BZIP2[1]; -/* Declare filter specific functions */ -DECLSPEC htri_t H5Z_bzip2_can_apply(hid_t dcpl_id, hid_t type_id, hid_t space_id); -DECLSPEC size_t H5Z_filter_bzip2(unsigned flags,size_t cd_nelmts,const unsigned cd_values[], - size_t nbytes,size_t *buf_size,void**buf); - #endif /*H5BZIP2_H*/ diff --git a/plugins/h5misc.h b/plugins/h5misc.h index 61f3178732..c1b17ec640 100644 --- a/plugins/h5misc.h +++ b/plugins/h5misc.h @@ -28,10 +28,12 @@ DECLSPEC H5PL_type_t H5PLget_plugin_type(void); DECLSPEC const void* H5PLget_plugin_info(void); DECLSPEC const H5Z_class2_t H5Z_TEST[1]; +#if 0 /* Declare filter specific functions */ DECLSPEC htri_t H5Z_test_can_apply(hid_t dcpl_id, hid_t type_id, hid_t space_id); DECLSPEC size_t H5Z_filter_test(unsigned flags,size_t cd_nelmts,const unsigned cd_values[], size_t nbytes,size_t *buf_size,void**buf); +#endif DECLSPEC void NC_h5filterspec_fix8(void* mem0, int decode); diff --git a/test_common.in b/test_common.in index 5e36b301b1..d3fa388166 100644 --- a/test_common.in +++ b/test_common.in @@ -19,6 +19,7 @@ FEATURE_HDF5=@HAS_HDF5@ FEATURE_HDF5=@HAS_HDF5@ FEATURE_S3TESTS=@DO_NCZARR_S3_TESTS@ FEATURE_NCZARR_ZIP=@DO_NCZARR_ZIP_TESTS@ +FEATURE_FILTERTESTS=@DO_FILTER_TESTS@ set -e @@ -69,6 +70,7 @@ set -e # 7. NCCOPY - absolute path to the nccopy.exe executable # 8. NCGEN - absolute path to ncgen.exe # 9. NCGEN3 - absolute path to ncgen3.exe +#10. NCPATHCVT - absolute path to ncpathcvt.exe # Allow global set -x mechanism for debugging. if test "x$SETX" = x1 ; then set -x ; fi @@ -126,6 +128,7 @@ export NCDUMP="${top_builddir}/ncdump${VS}/ncdump${ext}" export NCCOPY="${top_builddir}/ncdump${VS}/nccopy${ext}" export NCGEN="${top_builddir}/ncgen${VS}/ncgen${ext}" export NCGEN3="${top_builddir}/ncgen3${VS}/ncgen3${ext}" +export NCPATHCVT="${top_builddir}/ncdump${VS}/ncpathcvt${ext}" # Temporary hacks (until we have a test_utils directory) # to locate certain specific test files