From ef98cca1d6a3ba7d061b434e48fa0382df061d45 Mon Sep 17 00:00:00 2001 From: edwardhartnett Date: Thu, 22 Aug 2019 13:41:14 -0600 Subject: [PATCH 1/7] moving to async netcdf integration --- src/clib/pioc_support.c | 42 ++++++++++++++++++++++++------------- tests/ncint/Makefile.am | 5 ----- tests/ncint/run_tests.sh | 2 +- tests/ncint/tst_pio_async.c | 4 ++-- 4 files changed, 30 insertions(+), 23 deletions(-) diff --git a/src/clib/pioc_support.c b/src/clib/pioc_support.c index 3b098a8b7ce..54200d0ecd9 100644 --- a/src/clib/pioc_support.c +++ b/src/clib/pioc_support.c @@ -1922,6 +1922,22 @@ PIOc_writemap_from_f90(const char *file, int ndims, const int *gdims, MPI_Comm_f2c(f90_comm)); } +/** + * Change the ncid of a file in the netCDF C library. This requires a + * knowledge of netCDF internals. + * + * @param ncidp Pointer to current ncid. + * @param pio_ncid The new ncid we want. + * + * @returns 0 for success, error code otherwise. + * @author Ed Hartnett + */ +int +change_ncid(int *ncidp, int pio_ncid) +{ + return PIO_NOERR; +} + /** * Create a new file using pio. This is an internal function that is * called by both PIOc_create() and PIOc_createfile(). Input @@ -2087,28 +2103,24 @@ PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename, PLOG((3, "createfile bcast pio_next_ncid %d", pio_next_ncid)); } + /* Assign the PIO ncid. */ + file->pio_ncid = pio_next_ncid++; + /* With the netCDF integration layer, the ncid is assigned for PIO * by the netCDF dispatch layer code. So it is passed in. In * normal PIO operation, the ncid is generated here. */ if (use_ext_ncid) { - /* Use the ncid passed in from the netCDF dispatch code. */ - file->pio_ncid = *ncidp; - - /* To prevent PIO from reusing the same ncid, if someone - * starting mingling netcdf integration PIO and regular PIO - * code. */ - pio_next_ncid = file->pio_ncid + 1; + /* The ncid was assigned on the computational + * processors. Change the ncid to one that I/O and + * computational components can agree on. */ + if ((ierr = change_ncid(*ncidp, file->pio_ncid))) + return pio_err(NULL, file, ierr, __FILE__, __LINE__); } - else - { - /* Assign the PIO ncid. */ - file->pio_ncid = pio_next_ncid++; - PLOG((2, "file->fh = %d file->pio_ncid = %d", file->fh, file->pio_ncid)); + PLOG((2, "file->fh = %d file->pio_ncid = %d", file->fh, file->pio_ncid)); - /* Return the ncid to the caller. */ - *ncidp = file->pio_ncid; - } + /* Return the ncid to the caller. */ + *ncidp = file->pio_ncid; /* Add the struct with this files info to the global list of * open files. */ diff --git a/tests/ncint/Makefile.am b/tests/ncint/Makefile.am index 07dc555c6ac..4de93c50c49 100644 --- a/tests/ncint/Makefile.am +++ b/tests/ncint/Makefile.am @@ -18,11 +18,6 @@ if RUN_TESTS TESTS = run_tests.sh endif # RUN_TESTS -# if RUN_TESTS -# # Tests will run from a bash script. -# TESTS = run_tests.sh -# endif # RUN_TESTS - # Distribute the test script. EXTRA_DIST = run_tests.sh diff --git a/tests/ncint/run_tests.sh b/tests/ncint/run_tests.sh index 442128fa443..1cbfebc6633 100755 --- a/tests/ncint/run_tests.sh +++ b/tests/ncint/run_tests.sh @@ -10,7 +10,7 @@ trap exit INT TERM printf 'running PIO tests...\n' -PIO_TESTS='tst_pio_udf' +PIO_TESTS='tst_pio_udf tst_pio_async' success1=true success2=true diff --git a/tests/ncint/tst_pio_async.c b/tests/ncint/tst_pio_async.c index e042fd737dc..894b30f5275 100644 --- a/tests/ncint/tst_pio_async.c +++ b/tests/ncint/tst_pio_async.c @@ -63,7 +63,7 @@ main(int argc, char **argv) if (my_rank) { /* Create a file with a 3D record var. */ - if (nc_create(FILE_NAME, NC_UDF0, &ncid)) PERR; + /* if (nc_create(FILE_NAME, NC_UDF0, &ncid)) PERR; */ /* if (nc_def_dim(ncid, DIM_NAME_UNLIMITED, dimlen[0], &dimid[0])) PERR; */ /* if (nc_def_dim(ncid, DIM_NAME_X, dimlen[1], &dimid[1])) PERR; */ /* if (nc_def_dim(ncid, DIM_NAME_Y, dimlen[2], &dimid[2])) PERR; */ @@ -106,7 +106,7 @@ main(int argc, char **argv) /* if (data_in[i] != my_data[i]) PERR; */ /* Close file. */ - if (nc_close(ncid)) PERR; + /* if (nc_close(ncid)) PERR; */ /* /\* Free resources. *\/ */ /* free(data_in); */ From 6e393976f42e9da72466080e726db18537af62e1 Mon Sep 17 00:00:00 2001 From: edwardhartnett Date: Thu, 22 Aug 2019 13:56:06 -0600 Subject: [PATCH 2/7] commented out ncint tests --- src/clib/pioc_support.c | 4 +++- src/ncint/ncintdispatch.c | 2 -- tests/fncint/Makefile.am | 8 ++++---- tests/ncint/Makefile.am | 8 ++++---- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/clib/pioc_support.c b/src/clib/pioc_support.c index 54200d0ecd9..17c0ae32ea6 100644 --- a/src/clib/pioc_support.c +++ b/src/clib/pioc_support.c @@ -39,6 +39,8 @@ extern int pio_next_ncid; /** The default error handler used when iosystem cannot be located. */ extern int default_error_handler; +int nc4_file_change_ncid(int ncid, unsigned short new_ncid_index); + /** * Start the PIO timer. * @@ -2114,7 +2116,7 @@ PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename, /* The ncid was assigned on the computational * processors. Change the ncid to one that I/O and * computational components can agree on. */ - if ((ierr = change_ncid(*ncidp, file->pio_ncid))) + if ((ierr = nc4_file_change_ncid(*ncidp, file->pio_ncid))) return pio_err(NULL, file, ierr, __FILE__, __LINE__); } PLOG((2, "file->fh = %d file->pio_ncid = %d", file->fh, file->pio_ncid)); diff --git a/src/ncint/ncintdispatch.c b/src/ncint/ncintdispatch.c index 2260137544d..6a54436d6fc 100644 --- a/src/ncint/ncintdispatch.c +++ b/src/ncint/ncintdispatch.c @@ -39,8 +39,6 @@ NC_Dispatch NCINT_dispatcher = { PIO_NCINT_abort, PIO_NCINT_close, PIO_NCINT_set_fill, - NC_NOTNC3_inq_base_pe, - NC_NOTNC3_set_base_pe, PIO_NCINT_inq_format, PIO_NCINT_inq_format_extended, diff --git a/tests/fncint/Makefile.am b/tests/fncint/Makefile.am index f45c1ca5810..5039e4a8098 100644 --- a/tests/fncint/Makefile.am +++ b/tests/fncint/Makefile.am @@ -23,10 +23,10 @@ check_PROGRAMS = ftst_pio ftst_pio_orig tst_c_pio ftst_pio_SOURCES = ftst_pio.f90 ftst_pio_orig_SOURCES = ftst_pio_orig.f90 -if RUN_TESTS -# Tests will run from a bash script. -TESTS = run_tests.sh -endif # RUN_TESTS +# if RUN_TESTS +# # Tests will run from a bash script. +# TESTS = run_tests.sh +# endif # RUN_TESTS # Distribute the test script. EXTRA_DIST = run_tests.sh diff --git a/tests/ncint/Makefile.am b/tests/ncint/Makefile.am index 4de93c50c49..b88b78593cd 100644 --- a/tests/ncint/Makefile.am +++ b/tests/ncint/Makefile.am @@ -13,10 +13,10 @@ check_PROGRAMS = tst_pio_udf tst_pio_async tst_pio_udf_SOURCES = tst_pio_udf.c pio_err_macros.h tst_pio_async_SOURCES = tst_pio_async.c pio_err_macros.h -if RUN_TESTS -# Tests will run from a bash script. -TESTS = run_tests.sh -endif # RUN_TESTS +# if RUN_TESTS +# # Tests will run from a bash script. +# TESTS = run_tests.sh +# endif # RUN_TESTS # Distribute the test script. EXTRA_DIST = run_tests.sh From b18bd1251081794ba66074d6bcc11f7865529a5f Mon Sep 17 00:00:00 2001 From: edwardhartnett Date: Fri, 23 Aug 2019 13:18:11 -0600 Subject: [PATCH 3/7] trying to get working --- src/clib/pio_msg.c | 17 +++++++++++----- src/clib/pioc_support.c | 40 ++++++++++++++++--------------------- tests/ncint/Makefile.am | 8 ++++---- tests/ncint/tst_pio_async.c | 25 ++++++++++++----------- 4 files changed, 46 insertions(+), 44 deletions(-) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index 9d9c25d0c90..35a15b4853c 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -168,10 +168,12 @@ int set_fill_handler(iosystem_desc_t *ios) */ int create_file_handler(iosystem_desc_t *ios) { - int ncid; int len; int iotype; int mode; + int use_ext_ncid; + char ncidp_present; + int *ncidp = NULL; int mpierr; PLOG((1, "create_file_handler comproot = %d", ios->comproot)); @@ -191,12 +193,17 @@ int create_file_handler(iosystem_desc_t *ios) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Bcast(&mode, 1, MPI_INT, 0, ios->intercomm))) return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); - PLOG((1, "create_file_handler got parameters len = %d filename = %s iotype = %d mode = %d", - len, filename, iotype, mode)); + if ((mpierr = MPI_Bcast(&use_ext_ncid, 1, MPI_INT, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(&ncidp_present, 1, MPI_CHAR, 0, ios->intercomm))) + return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__); + PLOG((1, "create_file_handler len %d filename %s iotype %d mode %d " + "use_ext_ncid %d ncidp_present %d", len, filename, iotype, mode, + use_ext_ncid, ncidp_present)); /* Call the create file function. */ - PIOc_createfile(ios->iosysid, &ncid, &iotype, filename, mode); - + PIOc_createfile_int(ios->iosysid, ncidp, &iotype, filename, mode, + use_ext_ncid); PLOG((1, "create_file_handler succeeded!")); return PIO_NOERR; diff --git a/src/clib/pioc_support.c b/src/clib/pioc_support.c index 17c0ae32ea6..01f8a064190 100644 --- a/src/clib/pioc_support.c +++ b/src/clib/pioc_support.c @@ -14,6 +14,9 @@ /** This is used with text decomposition files. */ #define VERSNO 2001 +/** Used to shift file index to first two bytes of ncid. */ +#define ID_SHIFT 16 + /** In decomposition files, backtraces are included. This is the max * number of trace levels that will be used. */ #define MAX_BACKTRACE 10 @@ -1924,22 +1927,6 @@ PIOc_writemap_from_f90(const char *file, int ndims, const int *gdims, MPI_Comm_f2c(f90_comm)); } -/** - * Change the ncid of a file in the netCDF C library. This requires a - * knowledge of netCDF internals. - * - * @param ncidp Pointer to current ncid. - * @param pio_ncid The new ncid we want. - * - * @returns 0 for success, error code otherwise. - * @author Ed Hartnett - */ -int -change_ncid(int *ncidp, int pio_ncid) -{ - return PIO_NOERR; -} - /** * Create a new file using pio. This is an internal function that is * called by both PIOc_create() and PIOc_createfile(). Input @@ -1987,8 +1974,8 @@ PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename, if (!iotype_is_valid(*iotype)) return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__); - PLOG((1, "PIOc_createfile_int iosysid = %d iotype = %d filename = %s mode = %d", - iosysid, *iotype, filename, mode)); + PLOG((1, "PIOc_createfile_int iosysid %d iotype %d filename %s mode %d " + "use_ext_ncid %d", iosysid, *iotype, filename, mode, use_ext_ncid)); /* Allocate space for the file info. */ if (!(file = calloc(sizeof(file_desc_t), 1))) @@ -2017,6 +2004,7 @@ PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename, { int msg = PIO_MSG_CREATE_FILE; size_t len = strlen(filename); + char ncidp_present = ncidp ? 1 : 0; /* Send the message to the message handler. */ PLOG((3, "msg %d ios->union_comm %d MPI_COMM_NULL %d", msg, ios->union_comm, MPI_COMM_NULL)); @@ -2032,8 +2020,12 @@ PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename, mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast(&mode, 1, MPI_INT, ios->compmaster, ios->intercomm); - PLOG((2, "len = %d filename = %s iotype = %d mode = %d", len, filename, - file->iotype, mode)); + if (!mpierr) + mpierr = MPI_Bcast(&use_ext_ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&ncidp_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + PLOG((2, "len %d filename %s iotype %d mode %d use_ext_ncid %d ncidp_present %d", + len, filename, file->iotype, mode, use_ext_ncid, ncidp_present)); } /* Handle MPI errors. */ @@ -2108,9 +2100,10 @@ PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename, /* Assign the PIO ncid. */ file->pio_ncid = pio_next_ncid++; - /* With the netCDF integration layer, the ncid is assigned for PIO - * by the netCDF dispatch layer code. So it is passed in. In - * normal PIO operation, the ncid is generated here. */ + /* With the netCDF integration layer, we must override the ncid + * generated on the computation processors, with the ncid + * generated by the I/O processors (which know about all open + * files). In normal PIO operation, the ncid is generated here. */ if (use_ext_ncid) { /* The ncid was assigned on the computational @@ -2118,6 +2111,7 @@ PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename, * computational components can agree on. */ if ((ierr = nc4_file_change_ncid(*ncidp, file->pio_ncid))) return pio_err(NULL, file, ierr, __FILE__, __LINE__); + file->pio_ncid = file->pio_ncid << ID_SHIFT; } PLOG((2, "file->fh = %d file->pio_ncid = %d", file->fh, file->pio_ncid)); diff --git a/tests/ncint/Makefile.am b/tests/ncint/Makefile.am index b88b78593cd..4de93c50c49 100644 --- a/tests/ncint/Makefile.am +++ b/tests/ncint/Makefile.am @@ -13,10 +13,10 @@ check_PROGRAMS = tst_pio_udf tst_pio_async tst_pio_udf_SOURCES = tst_pio_udf.c pio_err_macros.h tst_pio_async_SOURCES = tst_pio_async.c pio_err_macros.h -# if RUN_TESTS -# # Tests will run from a bash script. -# TESTS = run_tests.sh -# endif # RUN_TESTS +if RUN_TESTS +# Tests will run from a bash script. +TESTS = run_tests.sh +endif # RUN_TESTS # Distribute the test script. EXTRA_DIST = run_tests.sh diff --git a/tests/ncint/tst_pio_async.c b/tests/ncint/tst_pio_async.c index 894b30f5275..af2e662a597 100644 --- a/tests/ncint/tst_pio_async.c +++ b/tests/ncint/tst_pio_async.c @@ -63,23 +63,23 @@ main(int argc, char **argv) if (my_rank) { /* Create a file with a 3D record var. */ - /* if (nc_create(FILE_NAME, NC_UDF0, &ncid)) PERR; */ + if (nc_create(FILE_NAME, NC_UDF0, &ncid)) PERR; /* if (nc_def_dim(ncid, DIM_NAME_UNLIMITED, dimlen[0], &dimid[0])) PERR; */ /* if (nc_def_dim(ncid, DIM_NAME_X, dimlen[1], &dimid[1])) PERR; */ /* if (nc_def_dim(ncid, DIM_NAME_Y, dimlen[2], &dimid[2])) PERR; */ /* if (nc_def_var(ncid, VAR_NAME, NC_INT, NDIM3, dimid, &varid)) PERR; */ /* Calculate a decomposition for distributed arrays. */ - elements_per_pe = DIM_LEN_X * DIM_LEN_Y / ntasks; - if (!(compdof = malloc(elements_per_pe * sizeof(size_t)))) - PERR; - for (i = 0; i < elements_per_pe; i++) - compdof[i] = my_rank * elements_per_pe + i; + /* elements_per_pe = DIM_LEN_X * DIM_LEN_Y / ntasks; */ + /* if (!(compdof = malloc(elements_per_pe * sizeof(size_t)))) */ + /* PERR; */ + /* for (i = 0; i < elements_per_pe; i++) */ + /* compdof[i] = my_rank * elements_per_pe + i; */ - /* Create the PIO decomposition for this test. */ - if (nc_def_decomp(iosysid, PIO_INT, NDIM2, &dimlen[1], elements_per_pe, - compdof, &ioid, 1, NULL, NULL)) PERR; - free(compdof); + /* /\* Create the PIO decomposition for this test. *\/ */ + /* if (nc_def_decomp(iosysid, PIO_INT, NDIM2, &dimlen[1], elements_per_pe, */ + /* compdof, &ioid, 1, NULL, NULL)) PERR; */ + /* free(compdof); */ /* /\* Create some data on this processor. *\/ */ /* if (!(my_data = malloc(elements_per_pe * sizeof(int)))) PERR; */ @@ -88,7 +88,8 @@ main(int argc, char **argv) /* /\* Write some data with distributed arrays. *\/ */ /* if (nc_put_vard_int(ncid, varid, ioid, 0, my_data)) PERR; */ - /* if (nc_close(ncid)) PERR; */ + if (nc_enddef(ncid)) PERR; + if (nc_close(ncid)) PERR; /* /\* Check that our user-defined format has been added. *\/ */ /* if (nc_inq_user_format(NC_UDF0, &disp_in, NULL)) PERR; */ @@ -111,7 +112,7 @@ main(int argc, char **argv) /* /\* Free resources. *\/ */ /* free(data_in); */ /* free(my_data); */ - if (nc_free_decomp(ioid)) PERR; + /* if (nc_free_decomp(ioid)) PERR; */ if (nc_free_iosystem(iosysid)) PERR; } } From 1d4cc195ff2defb30fd07ad15ccaaae986acb0b3 Mon Sep 17 00:00:00 2001 From: edwardhartnett Date: Fri, 30 Aug 2019 08:38:46 -0600 Subject: [PATCH 4/7] got async working with netcdf integration with one computational component --- src/clib/pio_msg.c | 25 ++++++++++++++++--------- src/clib/pioc.c | 16 +++++++++++++++- src/clib/pioc_support.c | 11 ++++++++--- tests/ncint/tst_pio_async.c | 4 +++- tests/ncint/tst_pio_udf.c | 2 +- 5 files changed, 43 insertions(+), 15 deletions(-) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index 06fecf8a9ed..7b7cd909bf2 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -28,6 +28,10 @@ extern int pio_log_level; extern int event_num[2][NUM_EVENTS]; #endif /* USE_MPE */ +/* Prototypes from nc4internal.h. */ +int nc4_file_list_add(int ncid, const char *path, int mode, + void **dispatchdata); + /** * This function is run on the IO tasks to handle nc_inq_type*() * functions. @@ -168,20 +172,14 @@ int set_fill_handler(iosystem_desc_t *ios) */ int create_file_handler(iosystem_desc_t *ios) { -<<<<<<< HEAD -======= int ncid = 0; ->>>>>>> master int len; int iotype; int mode; int use_ext_ncid; char ncidp_present; -<<<<<<< HEAD - int *ncidp = NULL; -======= ->>>>>>> master int mpierr; + int ret; PLOG((1, "create_file_handler comproot = %d", ios->comproot)); assert(ios); @@ -212,8 +210,17 @@ int create_file_handler(iosystem_desc_t *ios) use_ext_ncid, ncidp_present, ncid)); /* Call the create file function. */ - PIOc_createfile_int(ios->iosysid, &ncid, &iotype, filename, mode, - use_ext_ncid); + if (use_ext_ncid) + { + PLOG((2, "about to call nc_create()")); + nc_create(filename, mode|NC_UDF0, &ncid); + } + else + { + PLOG((2, "about to call PIOc_createfile_int()")); + PIOc_createfile_int(ios->iosysid, &ncid, &iotype, filename, mode, + use_ext_ncid); + } PLOG((1, "create_file_handler succeeded!")); return PIO_NOERR; diff --git a/src/clib/pioc.c b/src/clib/pioc.c index 4535dfd0207..f944d92c4ff 100644 --- a/src/clib/pioc.c +++ b/src/clib/pioc.c @@ -19,8 +19,12 @@ extern int event_num[2][NUM_EVENTS]; #endif /* USE_MPE */ #ifdef NETCDF_INTEGRATION -/* Have we initialized? */ +/* Have we initialized the netcdf integration code? */ extern int ncint_initialized; + +/* This is used as the default iosysid for the netcdf integration + * code. */ +extern int diosysid; #endif /* NETCDF_INTEGRATION */ /** @@ -1736,6 +1740,16 @@ PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list, /* Add this id to the list of PIO iosystem ids. */ iosysidp[cmp] = pio_add_to_iosystem_list(my_iosys); PLOG((2, "new iosys ID added to iosystem_list iosysidp[%d] = %d", cmp, iosysidp[cmp])); + +#ifdef NETCDF_INTEGRATION + if (in_io || in_cmp) + { + /* Remember the io system id. */ + diosysid = iosysidp[cmp]; + PLOG((3, "diosysid = %d", iosysidp[cmp])); + } +#endif /* NETCDF_INTEGRATION */ + } /* next computational component */ /* Now call the function from which the IO tasks will not return diff --git a/src/clib/pioc_support.c b/src/clib/pioc_support.c index f23ef6a79c6..22ac350d311 100644 --- a/src/clib/pioc_support.c +++ b/src/clib/pioc_support.c @@ -1935,8 +1935,12 @@ PIOc_writemap_from_f90(const char *file, int ndims, const int *gdims, * @param iosysid A defined pio system ID, obtained from * PIOc_Init_Intracomm() or PIOc_InitAsync(). * @param ncidp A pointer that gets the ncid of the newly created - * file. For NetCDF integration, this contains the ncid assigned by - * the netCDF layer, which is used instead of a PIO-generated ncid. + * file. This is the PIO ncid. Within PIO, the file will have a + * different ID, the file->fh. When netCDF integration is used, the + * PIO ncid is also stored in the netcdf-c internal file list, and the + * PIO code is called by the netcdf-c dispatch code. In this case, + * there are two ncids for the file, the PIO ncid, and the file->fh + * ncid. * @param iotype A pointer to a pio output format. Must be one of * PIO_IOTYPE_PNETCDF, PIO_IOTYPE_NETCDF, PIO_IOTYPE_NETCDF4C, or * PIO_IOTYPE_NETCDF4P. @@ -2003,7 +2007,6 @@ PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename, if (!ios->ioproc) { int msg = PIO_MSG_CREATE_FILE; - char ncidp_present = ncidp ? 1 : 0; size_t len = strlen(filename); char ncidp_present = ncidp ? 1 : 0; @@ -2073,6 +2076,7 @@ PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename, break; #endif } + PLOG((3, "create call complete file->fh %d", file->fh)); } /* Broadcast and check the return code. */ @@ -2117,6 +2121,7 @@ PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename, if ((ierr = nc4_file_change_ncid(*ncidp, file->pio_ncid))) return pio_err(NULL, file, ierr, __FILE__, __LINE__); file->pio_ncid = file->pio_ncid << ID_SHIFT; + PLOG((2, "changed ncid to file->pio_ncid = %d", file->pio_ncid)); } PLOG((2, "file->fh = %d file->pio_ncid = %d", file->fh, file->pio_ncid)); diff --git a/tests/ncint/tst_pio_async.c b/tests/ncint/tst_pio_async.c index af2e662a597..80991326010 100644 --- a/tests/ncint/tst_pio_async.c +++ b/tests/ncint/tst_pio_async.c @@ -54,6 +54,8 @@ main(int argc, char **argv) /* Turn on logging for PIO library. */ PIOc_set_log_level(3); + if (!my_rank) + nc_set_log_level(3); /* Initialize the intracomm. */ if (nc_def_async(MPI_COMM_WORLD, num_io_procs, NULL, COMPONENT_COUNT, @@ -63,7 +65,7 @@ main(int argc, char **argv) if (my_rank) { /* Create a file with a 3D record var. */ - if (nc_create(FILE_NAME, NC_UDF0, &ncid)) PERR; + if (nc_create(FILE_NAME, NC_UDF0|NC_NETCDF4, &ncid)) PERR; /* if (nc_def_dim(ncid, DIM_NAME_UNLIMITED, dimlen[0], &dimid[0])) PERR; */ /* if (nc_def_dim(ncid, DIM_NAME_X, dimlen[1], &dimid[1])) PERR; */ /* if (nc_def_dim(ncid, DIM_NAME_Y, dimlen[2], &dimid[2])) PERR; */ diff --git a/tests/ncint/tst_pio_udf.c b/tests/ncint/tst_pio_udf.c index dedcd536ee2..1a25df9dacf 100644 --- a/tests/ncint/tst_pio_udf.c +++ b/tests/ncint/tst_pio_udf.c @@ -61,7 +61,7 @@ main(int argc, char **argv) int i; /* Turn on logging for PIO library. */ - /* PIOc_set_log_level(3); */ + PIOc_set_log_level(3); /* Initialize the intracomm. */ if (nc_def_iosystemm(MPI_COMM_WORLD, 1, 1, 0, 0, &iosysid)) PERR; From 541d0df86811cd7cde5ae1b918d3746e4b5f1b0e Mon Sep 17 00:00:00 2001 From: edwardhartnett Date: Fri, 30 Aug 2019 08:42:51 -0600 Subject: [PATCH 5/7] clean up --- src/clib/pio_msg.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index 7b7cd909bf2..83990b62500 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -28,10 +28,6 @@ extern int pio_log_level; extern int event_num[2][NUM_EVENTS]; #endif /* USE_MPE */ -/* Prototypes from nc4internal.h. */ -int nc4_file_list_add(int ncid, const char *path, int mode, - void **dispatchdata); - /** * This function is run on the IO tasks to handle nc_inq_type*() * functions. From 0fccc80ab363fc231f7cd6b667224c28bb7c196f Mon Sep 17 00:00:00 2001 From: edwardhartnett Date: Fri, 30 Aug 2019 11:51:52 -0600 Subject: [PATCH 6/7] protected use of NC_UDF0 with ifdef --- src/clib/pio_msg.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index 83990b62500..06fafd6a47b 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -208,8 +208,10 @@ int create_file_handler(iosystem_desc_t *ios) /* Call the create file function. */ if (use_ext_ncid) { +#ifdef NETCDF_INTEGRATION PLOG((2, "about to call nc_create()")); nc_create(filename, mode|NC_UDF0, &ncid); +#endif /* NETCDF_INTEGRATION */ } else { From 10962543ba78614d2ff92c47477158cdb348f9ef Mon Sep 17 00:00:00 2001 From: edwardhartnett Date: Fri, 30 Aug 2019 13:34:25 -0600 Subject: [PATCH 7/7] protected use of nc4_file_change_ncid() with ifdef --- src/clib/pioc_support.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/clib/pioc_support.c b/src/clib/pioc_support.c index 22ac350d311..202b7b51996 100644 --- a/src/clib/pioc_support.c +++ b/src/clib/pioc_support.c @@ -2113,6 +2113,7 @@ PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename, * generated on the computation processors, with the ncid * generated by the I/O processors (which know about all open * files). In normal PIO operation, the ncid is generated here. */ +#ifdef NETCDF_INTEGRATION if (use_ext_ncid) { /* The ncid was assigned on the computational @@ -2123,6 +2124,7 @@ PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename, file->pio_ncid = file->pio_ncid << ID_SHIFT; PLOG((2, "changed ncid to file->pio_ncid = %d", file->pio_ncid)); } +#endif /* NETCDF_INTEGRATION */ PLOG((2, "file->fh = %d file->pio_ncid = %d", file->fh, file->pio_ncid)); /* Return the ncid to the caller. */