Skip to content

Commit

Permalink
now passing diosysid when doing a create file in async mode
Browse files Browse the repository at this point in the history
  • Loading branch information
@edhartnett
edhartnett committed Sep 9, 2019
1 parent 85ec39c commit f195435
Show file tree
Hide file tree
Showing 3 changed files with 39 additions and 117 deletions.
13 changes: 10 additions & 3 deletions src/clib/pio_msg.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,7 @@ int create_file_handler(iosystem_desc_t *ios)
int mode;
int use_ext_ncid;
char ncidp_present;
int iosysid;
int mpierr;

PLOG((1, "create_file_handler comproot = %d", ios->comproot));
Expand All @@ -200,15 +201,21 @@ int create_file_handler(iosystem_desc_t *ios)
if (ncidp_present)
if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
if ((mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, 0, ios->intercomm)))
return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
PLOG((1, "create_file_handler len %d filename %s iotype %d mode %d "
"use_ext_ncid %d ncidp_present %d ncid %d", len, filename, iotype, mode,
use_ext_ncid, ncidp_present, ncid));
"use_ext_ncid %d ncidp_present %d ncid %d iosysid %d", len,
filename, iotype, mode, use_ext_ncid, ncidp_present, ncid,
iosysid));

/* Call the create file function. */
if (use_ext_ncid)
{
#ifdef NETCDF_INTEGRATION
PLOG((2, "about to call nc_create()"));
/* Set the IO system ID. */
nc_set_iosystem(iosysid);

PLOG((2, "about to call nc_create() have set iosysid to %d", iosysid));
nc_create(filename, mode|NC_UDF0, &ncid);
#endif /* NETCDF_INTEGRATION */
}
Expand Down
12 changes: 10 additions & 2 deletions src/clib/pioc_support.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,15 @@ extern int pio_next_ncid;
/** The default error handler used when iosystem cannot be located. */
extern int default_error_handler;

#ifdef NETCDF_INTEGRATION
/* This is used as the default iosysid for the netcdf integration
* code. */
extern int diosysid;

/** This prototype from netCDF is required for netCDF integration to
* work. */
int nc4_file_change_ncid(int ncid, unsigned short new_ncid_index);
#endif /* NETCDF_INTEGRATION */

/**
* Start the PIO timer.
Expand Down Expand Up @@ -2033,9 +2039,11 @@ PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename,
if (ncidp_present)
if (!mpierr)
mpierr = MPI_Bcast(ncidp, 1, MPI_INT, ios->compmaster, ios->intercomm);
if (!mpierr)
mpierr = MPI_Bcast(&diosysid, 1, MPI_INT, ios->compmaster, ios->intercomm);
PLOG((2, "len %d filename %s iotype %d mode %d use_ext_ncid %d "
"ncidp_present %d", len, filename, file->iotype, mode,
use_ext_ncid, ncidp_present));
"ncidp_present %d diosysid %d", len, filename, file->iotype, mode,
use_ext_ncid, ncidp_present, diosysid));
}

/* Handle MPI errors. */
Expand Down
131 changes: 19 additions & 112 deletions tests/ncint/tst_async_multi.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,12 @@
extern NC_Dispatch NCINT_dispatcher;

/* Number of computational components to create. */
#define COMPONENT_COUNT 1
#define COMPONENT_COUNT 2

/* Create a file with one 3D var. */
int
create_file_0(int my_rank, int ntasks, int num_io_procs, int iosysid)
create_file(int file_num, int my_rank, int ntasks, int num_io_procs,
int iosysid)
{
int ncid, ioid;
int dimid[NDIM3], varid;
Expand All @@ -40,106 +41,7 @@ create_file_0(int my_rank, int ntasks, int num_io_procs, int iosysid)
int i;

/* Create a file with a 3D record var. */
sprintf(file_name, "%s_component_%d.nc", TEST_NAME, 0);
if (nc_create(file_name, NC_PIO|NC_NETCDF4, &ncid)) PERR;
if (nc_def_dim(ncid, DIM_NAME_UNLIMITED, dimlen[0], &dimid[0])) PERR;
if (nc_def_dim(ncid, DIM_NAME_X, dimlen[1], &dimid[1])) PERR;
if (nc_def_dim(ncid, DIM_NAME_Y, dimlen[2], &dimid[2])) PERR;
if (nc_def_var(ncid, VAR_NAME, NC_INT, NDIM3, dimid, &varid)) PERR;
if (nc_enddef(ncid)) PERR;

/* Calculate a decomposition for distributed arrays. */
elements_per_pe = DIM_LEN_X * DIM_LEN_Y / (ntasks - num_io_procs);
/* printf("my_rank %d elements_per_pe %ld\n", my_rank, elements_per_pe); */
if (!(compdof = malloc(elements_per_pe * sizeof(size_t))))
PERR;
for (i = 0; i < elements_per_pe; i++)
{
compdof[i] = (my_rank - num_io_procs) * elements_per_pe + i;
/* printf("my_rank %d compdof[%d]=%ld\n", my_rank, i, compdof[i]); */
}

/* Create the PIO decomposition for this test. */
if (nc_def_decomp(iosysid, PIO_INT, NDIM2, &dimlen[1], elements_per_pe,
compdof, &ioid, 1, NULL, NULL)) PERR;
free(compdof);

/* Create some data on this processor. */
if (!(my_data = malloc(elements_per_pe * sizeof(int)))) PERR;
for (i = 0; i < elements_per_pe; i++)
my_data[i] = my_rank * 10 + i;

/* Write some data with distributed arrays. */
if (nc_put_vard_int(ncid, varid, ioid, 0, my_data)) PERR;
if (nc_close(ncid)) PERR;

/* Reopen the file using netCDF integration. */
{
int ndims, nvars, ngatts, unlimdimid;
nc_type xtype_in;
char var_name_in[NC_MAX_NAME + 1];
char dim_name_in[NC_MAX_NAME + 1];
int natts_in;
int dimids_in[NDIM3];
size_t dim_len_in;

/* Open the file. */
if (nc_open(file_name, NC_PIO, &ncid)) PERR;

/* Check the file. */
if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) PERR;
if (ndims != 3 || nvars != 1 || ngatts != 0 ||
unlimdimid != 0) PERR;
if (nc_inq_var(ncid, 0, var_name_in, &xtype_in, &ndims,
dimids_in, &natts_in)) PERR;
if (strcmp(var_name_in, VAR_NAME) || xtype_in != NC_INT || ndims != NDIM3
|| dimids_in[0] != 0 || dimids_in[1] != 1 || dimids_in[2] != 2 ||
natts_in != 0) PERR;
if (nc_inq_dim(ncid, 0, dim_name_in, &dim_len_in)) PERR;
if (strcmp(dim_name_in, DIM_NAME_UNLIMITED) || dim_len_in != 1) PERR;
if (nc_inq_dim(ncid, 1, dim_name_in, &dim_len_in)) PERR;
if (strcmp(dim_name_in, DIM_NAME_X) || dim_len_in != DIM_LEN_X) PERR;
if (nc_inq_dim(ncid, 2, dim_name_in, &dim_len_in)) PERR;
if (strcmp(dim_name_in, DIM_NAME_Y) || dim_len_in != DIM_LEN_Y) PERR;

/* Read distributed arrays. */
if (!(data_in = malloc(elements_per_pe * sizeof(int)))) PERR;
if (nc_get_vard_int(ncid, varid, ioid, 0, data_in)) PERR;

/* Check results. */
for (i = 0; i < elements_per_pe; i++)
if (data_in[i] != my_data[i]) PERR;

/* Close file. */
if (nc_close(ncid)) PERR;

/* Free resources. */
free(data_in);
}

/* Release resources. */
free(my_data);
if (nc_free_decomp(ioid)) PERR;

return 0;
}

/* Create a file with one 3D var. */
int
create_file_1(int my_rank, int ntasks, int num_io_procs, int iosysid)
{
int ncid, ioid;
int dimid[NDIM3], varid;
int dimlen[NDIM3] = {NC_UNLIMITED, DIM_LEN_X, DIM_LEN_Y};
size_t elements_per_pe;
size_t *compdof; /* The decomposition mapping. */
int *my_data;
int *data_in;
char file_name[NC_MAX_NAME + 1];
int i;

/* Create a file with a 3D record var. */
sprintf(file_name, "%s_component_%d.nc", TEST_NAME, 1);
sprintf(file_name, "%s_file_%d.nc", TEST_NAME, file_num);
if (nc_create(file_name, NC_PIO|NC_NETCDF4, &ncid)) PERR;
if (nc_def_dim(ncid, DIM_NAME_UNLIMITED, dimlen[0], &dimid[0])) PERR;
if (nc_def_dim(ncid, DIM_NAME_X, dimlen[1], &dimid[1])) PERR;
Expand Down Expand Up @@ -241,30 +143,35 @@ main(int argc, char **argv)
if (!my_rank)
printf("*** testing simple async use of netCDF integration layer...");
{
int iosysid;
int num_procs2[COMPONENT_COUNT] = {3};
int iosysid[COMPONENT_COUNT];
int num_procs2[COMPONENT_COUNT] = {1, 2};
int num_io_procs = 1;

/* Turn on logging for PIO library. */
/* PIOc_set_log_level(4); */
/* if (!my_rank) */
/* nc_set_log_level(3); */
PIOc_set_log_level(4);
if (!my_rank)
nc_set_log_level(3);

/* Initialize the intracomm. The IO task will not return from
* this call until the PIOc_finalize() is called by the
* compute tasks. */
if (nc_def_async(MPI_COMM_WORLD, num_io_procs, NULL, COMPONENT_COUNT,
num_procs2, NULL, NULL, NULL, PIO_REARR_BOX, &iosysid))
num_procs2, NULL, NULL, NULL, PIO_REARR_BOX, iosysid))
PERR;

if (my_rank)
if (my_rank == 1)
{
/* Create a file, write some data, and check it. */
if (create_file_0(my_rank, ntasks, num_io_procs, iosysid)) PERR;
if (create_file_1(my_rank, ntasks, num_io_procs, iosysid)) PERR;
if (create_file(0, my_rank, ntasks, num_io_procs, iosysid[0])) PERR;
if (create_file(1, my_rank, ntasks, num_io_procs, iosysid[0])) PERR;

if (nc_free_iosystem(iosysid)) PERR;
if (nc_free_iosystem(iosysid[0])) PERR;
}
else if (my_rank > 1)
{
if (nc_free_iosystem(iosysid[1])) PERR;
}

}
if (!my_rank)
PSUMMARIZE_ERR;
Expand Down

0 comments on commit f195435

Please sign in to comment.