From 7b1d45f6c624825304a8314db6cf98d4a6e669b2 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Tue, 12 Dec 2023 19:27:03 +0100 Subject: [PATCH 01/11] collective-io: Buffer per process for coll. write --- .../clawpatch/fclaw2d_clawpatch_output_vtk.c | 43 +++++++++++++++---- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c b/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c index 5d07be6f0..2de1f7f5a 100644 --- a/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c +++ b/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c @@ -59,6 +59,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include +#include typedef struct fclaw2d_vtk_state { @@ -91,6 +92,7 @@ typedef struct fclaw2d_vtk_state MPI_Offset mpibegin; #endif char *buf; + sc_io_sink_t *sink; } fclaw2d_vtk_state_t; @@ -226,7 +228,7 @@ write_position_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, fclaw2d_vtk_state_t *s = (fclaw2d_vtk_state_t *) g->user; s->coordinate_cb (g->glob, patch, blockno, patchno, s->buf); - write_buffer (s, s->psize_position); + sc_io_sink_write (s->sink, s->buf, s->psize_position); } static void @@ -322,7 +324,7 @@ write_connectivity_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, } #endif } - write_buffer (s, s->psize_connectivity); + sc_io_sink_write (s->sink, s->buf, s->psize_connectivity); } static void @@ -354,7 +356,7 @@ write_offsets_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, *idata++ = k; } } - write_buffer (s, s->psize_offsets); + sc_io_sink_write (s->sink, s->buf, s->psize_offsets); } static void @@ -374,7 +376,7 @@ write_types_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, *cdata++ = 12; #endif } - write_buffer (s, s->psize_types); + sc_io_sink_write (s->sink, s->buf, s->psize_types); } static void @@ -390,7 +392,7 @@ write_mpirank_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, { *idata++ = domain->mpirank; } - write_buffer (s, s->psize_mpirank); + sc_io_sink_write (s->sink, s->buf, s->psize_mpirank); } static void @@ -406,7 +408,7 @@ write_blockno_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, { *idata++ = blockno; } - write_buffer (s, s->psize_blockno); + sc_io_sink_write (s->sink, s->buf, s->psize_blockno); } static void @@ -437,7 +439,7 @@ write_patchno_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, *idata++ = gpno; } } - write_buffer (s, s->psize_patchno); + sc_io_sink_write (s->sink, s->buf, s->psize_patchno); } static void @@ -449,7 +451,7 @@ write_meqn_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, fclaw2d_vtk_state_t *s = (fclaw2d_vtk_state_t *) g->user; s->value_cb (g->glob, patch, blockno, patchno, s->buf); - write_buffer (s, s->psize_meqn); + sc_io_sink_write (s->sink, s->buf, s->psize_meqn); } static void @@ -460,6 +462,7 @@ fclaw2d_vtk_write_field (fclaw2d_global_t * glob, fclaw2d_vtk_state_t * s, fclaw2d_domain_t *domain = glob->domain; int64_t bcount; + sc_array_t buffer; #ifndef P4EST_ENABLE_MPIIO size_t retvalz; #else @@ -472,6 +475,10 @@ fclaw2d_vtk_write_field (fclaw2d_global_t * glob, fclaw2d_vtk_state_t * s, #endif s->buf = P4EST_ALLOC (char, psize_field); + /* The buffer array will be resized when required. */ + sc_array_init_size (&buffer, 1, psize_field); + s->sink = sc_io_sink_new (SC_IO_TYPE_BUFFER, SC_IO_MODE_APPEND, + SC_IO_ENCODE_NONE, &buffer); #ifdef P4EST_ENABLE_MPIIO mpipos = s->mpibegin + offset_field; if (domain->mpirank > 0) @@ -495,11 +502,26 @@ fclaw2d_vtk_write_field (fclaw2d_global_t * glob, fclaw2d_vtk_state_t * s, retvalz = fwrite (&bcount, s->ndsize, 1, s->file); SC_CHECK_ABORT (retvalz == 1, "VTK file write failed"); #else - mpiret = MPI_File_write (s->mpifile, &bcount, 1, MPI_LONG, &mpistatus); + mpiret = + MPI_File_write (s->mpifile, &bcount, 1, MPI_LONG, &mpistatus); SC_CHECK_MPI (mpiret); #endif } fclaw2d_global_iterate_patches (glob, cb, s); + +#ifdef P4EST_ENABLE_MPIIO + mpiret = + MPI_File_write_all (s->mpifile, buffer.array, s->sink->buffer_bytes, + MPI_BYTE, &mpistatus); + SC_CHECK_MPI (mpiret); +#else + retvalz = fwrite (buffer.array, s->sink->buffer_bytes, 1, s->file); + SC_CHECK_ABORT (retvalz == 1, "VTK file write failed"); +#endif + + /* free buffers */ + sc_array_reset (&buffer); + sc_io_sink_destroy (s->sink); P4EST_FREE (s->buf); #ifdef P4EST_ENABLE_MPIIO @@ -662,6 +684,9 @@ fclaw2d_vtk_write_file (fclaw2d_global_t * glob, const char *basename, s->offset_end = s->ndsize + s->offset_meqn + s->psize_meqn * domain->global_num_patches; + s->buf = NULL; + s->sink = NULL; + /* write header meta data and check for error */ retval = 0; if (domain->mpirank == 0) From d1eb3f4ac13cab118a577820a92fd0843efb412a Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Tue, 12 Dec 2023 19:55:12 +0100 Subject: [PATCH 02/11] collective-io: Remove unused static func. --- .../clawpatch/fclaw2d_clawpatch_output_vtk.c | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c b/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c index 2de1f7f5a..130cc0332 100644 --- a/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c +++ b/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c @@ -196,30 +196,6 @@ fclaw2d_vtk_write_header (fclaw2d_domain_t * domain, fclaw2d_vtk_state_t * s) return retval ? -1 : 0; } -/** - * @brief Write the buffer to file - * - * @param s the vtk state - * @param psize_field the size of the buffer - */ -static void -write_buffer (fclaw2d_vtk_state_t * s, int64_t psize_field) -{ -#ifndef P4EST_ENABLE_MPIIO - size_t retvalz; - - retvalz = fwrite (s->buf, psize_field, 1, s->file); - SC_CHECK_ABORT (retvalz == 1, "VTK file write failed"); -#else - int mpiret; - MPI_Status mpistatus; - - mpiret = MPI_File_write (s->mpifile, s->buf, psize_field, MPI_BYTE, - &mpistatus); - SC_CHECK_MPI (mpiret); -#endif -} - static void write_position_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, int blockno, int patchno, void *user) From 5e6b225d1cccc3a65370da2430064b8ff6599df7 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Mon, 18 Dec 2023 15:46:33 +0100 Subject: [PATCH 03/11] collective-io: Check return values of sink --- .../clawpatch/fclaw2d_clawpatch_output_vtk.c | 32 ++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c b/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c index 130cc0332..d60928200 100644 --- a/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c +++ b/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c @@ -200,17 +200,20 @@ static void write_position_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, int blockno, int patchno, void *user) { + int retval; fclaw2d_global_iterate_t *g = (fclaw2d_global_iterate_t*) user; fclaw2d_vtk_state_t *s = (fclaw2d_vtk_state_t *) g->user; s->coordinate_cb (g->glob, patch, blockno, patchno, s->buf); - sc_io_sink_write (s->sink, s->buf, s->psize_position); + retval = sc_io_sink_write (s->sink, s->buf, s->psize_position); + SC_CHECK_ABORT (retval == 0, "VTK buffer write failed"); } static void write_connectivity_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, int blockno, int patchno, void *user) { + int retval; fclaw2d_global_iterate_t *g = (fclaw2d_global_iterate_t*) user; fclaw2d_vtk_state_t *s = (fclaw2d_vtk_state_t *) g->user; int i, j; @@ -300,13 +303,15 @@ write_connectivity_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, } #endif } - sc_io_sink_write (s->sink, s->buf, s->psize_connectivity); + retval = sc_io_sink_write (s->sink, s->buf, s->psize_connectivity); + SC_CHECK_ABORT (retval == 0, "VTK buffer write failed"); } static void write_offsets_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, int blockno, int patchno, void *user) { + int retval; fclaw2d_global_iterate_t *g = (fclaw2d_global_iterate_t*) user; fclaw2d_vtk_state_t *s = (fclaw2d_vtk_state_t *) g->user; int c; @@ -332,13 +337,15 @@ write_offsets_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, *idata++ = k; } } - sc_io_sink_write (s->sink, s->buf, s->psize_offsets); + retval = sc_io_sink_write (s->sink, s->buf, s->psize_offsets); + SC_CHECK_ABORT (retval == 0, "VTK buffer write failed"); } static void write_types_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, int blockno, int patchno, void *user) { + int retval; fclaw2d_global_iterate_t *g = (fclaw2d_global_iterate_t*) user; fclaw2d_vtk_state_t *s = (fclaw2d_vtk_state_t *) g->user; int c; @@ -352,13 +359,15 @@ write_types_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, *cdata++ = 12; #endif } - sc_io_sink_write (s->sink, s->buf, s->psize_types); + retval = sc_io_sink_write (s->sink, s->buf, s->psize_types); + SC_CHECK_ABORT (retval == 0, "VTK buffer write failed"); } static void write_mpirank_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, int blockno, int patchno, void *user) { + int retval; fclaw2d_global_iterate_t *g = (fclaw2d_global_iterate_t*) user; fclaw2d_vtk_state_t *s = (fclaw2d_vtk_state_t *) g->user; int c; @@ -368,13 +377,15 @@ write_mpirank_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, { *idata++ = domain->mpirank; } - sc_io_sink_write (s->sink, s->buf, s->psize_mpirank); + retval = sc_io_sink_write (s->sink, s->buf, s->psize_mpirank); + SC_CHECK_ABORT (retval == 0, "VTK buffer write failed"); } static void write_blockno_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, int blockno, int patchno, void *user) { + int retval; fclaw2d_global_iterate_t *g = (fclaw2d_global_iterate_t*) user; fclaw2d_vtk_state_t *s = (fclaw2d_vtk_state_t *) g->user; int c; @@ -384,13 +395,15 @@ write_blockno_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, { *idata++ = blockno; } - sc_io_sink_write (s->sink, s->buf, s->psize_blockno); + retval = sc_io_sink_write (s->sink, s->buf, s->psize_blockno); + SC_CHECK_ABORT (retval == 0, "VTK buffer write failed"); } static void write_patchno_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, int blockno, int patchno, void *user) { + int retval; fclaw2d_global_iterate_t *g = (fclaw2d_global_iterate_t*) user; fclaw2d_vtk_state_t *s = (fclaw2d_vtk_state_t *) g->user; int c; @@ -415,19 +428,22 @@ write_patchno_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, *idata++ = gpno; } } - sc_io_sink_write (s->sink, s->buf, s->psize_patchno); + retval = sc_io_sink_write (s->sink, s->buf, s->psize_patchno); + SC_CHECK_ABORT (retval == 0, "VTK buffer write failed"); } static void write_meqn_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, int blockno, int patchno, void *user) { + int retval; fclaw2d_global_iterate_t* g = (fclaw2d_global_iterate_t*) user; fclaw2d_vtk_state_t *s = (fclaw2d_vtk_state_t *) g->user; s->value_cb (g->glob, patch, blockno, patchno, s->buf); - sc_io_sink_write (s->sink, s->buf, s->psize_meqn); + retval = sc_io_sink_write (s->sink, s->buf, s->psize_meqn); + SC_CHECK_ABORT (retval == 0, "VTK buffer write failed"); } static void From ce1fd0a671b1b0aa5ac9fe69c3ed9e126778db6f Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Mon, 18 Dec 2023 16:52:15 +0100 Subject: [PATCH 04/11] collective-io: Introduce buffer function --- .../clawpatch/fclaw2d_clawpatch_output_vtk.c | 50 ++++++++++--------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c b/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c index d60928200..f7e050257 100644 --- a/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c +++ b/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c @@ -196,24 +196,39 @@ fclaw2d_vtk_write_header (fclaw2d_domain_t * domain, fclaw2d_vtk_state_t * s) return retval ? -1 : 0; } +/** This function add to a buffer and writes to file if a threshold is exceeded. + * The writing is not yet implemented. + * + * \param [in,out] s The VTK state. + * \param [in] psize_field The number of bytes, which are intended to + * add to the buffer. + */ +static void +add_to_buffer (fclaw2d_vtk_state_t * s, int64_t psize_field) +{ + FCLAW_ASSERT (s != NULL); + + int retval; + + retval = sc_io_sink_write (s->sink, s->buf, (size_t) psize_field); + SC_CHECK_ABORT (retval == 0, "VTK buffer write failed"); +} + static void write_position_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, int blockno, int patchno, void *user) { - int retval; fclaw2d_global_iterate_t *g = (fclaw2d_global_iterate_t*) user; fclaw2d_vtk_state_t *s = (fclaw2d_vtk_state_t *) g->user; s->coordinate_cb (g->glob, patch, blockno, patchno, s->buf); - retval = sc_io_sink_write (s->sink, s->buf, s->psize_position); - SC_CHECK_ABORT (retval == 0, "VTK buffer write failed"); + add_to_buffer (s, s->psize_position); } static void write_connectivity_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, int blockno, int patchno, void *user) { - int retval; fclaw2d_global_iterate_t *g = (fclaw2d_global_iterate_t*) user; fclaw2d_vtk_state_t *s = (fclaw2d_vtk_state_t *) g->user; int i, j; @@ -303,15 +318,13 @@ write_connectivity_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, } #endif } - retval = sc_io_sink_write (s->sink, s->buf, s->psize_connectivity); - SC_CHECK_ABORT (retval == 0, "VTK buffer write failed"); + add_to_buffer (s, s->psize_connectivity); } static void write_offsets_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, int blockno, int patchno, void *user) { - int retval; fclaw2d_global_iterate_t *g = (fclaw2d_global_iterate_t*) user; fclaw2d_vtk_state_t *s = (fclaw2d_vtk_state_t *) g->user; int c; @@ -337,15 +350,13 @@ write_offsets_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, *idata++ = k; } } - retval = sc_io_sink_write (s->sink, s->buf, s->psize_offsets); - SC_CHECK_ABORT (retval == 0, "VTK buffer write failed"); + add_to_buffer (s, s->psize_offsets); } static void write_types_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, int blockno, int patchno, void *user) { - int retval; fclaw2d_global_iterate_t *g = (fclaw2d_global_iterate_t*) user; fclaw2d_vtk_state_t *s = (fclaw2d_vtk_state_t *) g->user; int c; @@ -359,15 +370,13 @@ write_types_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, *cdata++ = 12; #endif } - retval = sc_io_sink_write (s->sink, s->buf, s->psize_types); - SC_CHECK_ABORT (retval == 0, "VTK buffer write failed"); + add_to_buffer (s, s->psize_types); } static void write_mpirank_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, int blockno, int patchno, void *user) { - int retval; fclaw2d_global_iterate_t *g = (fclaw2d_global_iterate_t*) user; fclaw2d_vtk_state_t *s = (fclaw2d_vtk_state_t *) g->user; int c; @@ -377,15 +386,13 @@ write_mpirank_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, { *idata++ = domain->mpirank; } - retval = sc_io_sink_write (s->sink, s->buf, s->psize_mpirank); - SC_CHECK_ABORT (retval == 0, "VTK buffer write failed"); + add_to_buffer (s, s->psize_mpirank); } static void write_blockno_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, int blockno, int patchno, void *user) { - int retval; fclaw2d_global_iterate_t *g = (fclaw2d_global_iterate_t*) user; fclaw2d_vtk_state_t *s = (fclaw2d_vtk_state_t *) g->user; int c; @@ -395,15 +402,13 @@ write_blockno_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, { *idata++ = blockno; } - retval = sc_io_sink_write (s->sink, s->buf, s->psize_blockno); - SC_CHECK_ABORT (retval == 0, "VTK buffer write failed"); + add_to_buffer (s, s->psize_blockno); } static void write_patchno_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, int blockno, int patchno, void *user) { - int retval; fclaw2d_global_iterate_t *g = (fclaw2d_global_iterate_t*) user; fclaw2d_vtk_state_t *s = (fclaw2d_vtk_state_t *) g->user; int c; @@ -428,22 +433,19 @@ write_patchno_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, *idata++ = gpno; } } - retval = sc_io_sink_write (s->sink, s->buf, s->psize_patchno); - SC_CHECK_ABORT (retval == 0, "VTK buffer write failed"); + add_to_buffer (s, s->psize_patchno); } static void write_meqn_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, int blockno, int patchno, void *user) { - int retval; fclaw2d_global_iterate_t* g = (fclaw2d_global_iterate_t*) user; fclaw2d_vtk_state_t *s = (fclaw2d_vtk_state_t *) g->user; s->value_cb (g->glob, patch, blockno, patchno, s->buf); - retval = sc_io_sink_write (s->sink, s->buf, s->psize_meqn); - SC_CHECK_ABORT (retval == 0, "VTK buffer write failed"); + add_to_buffer (s, s->psize_meqn); } static void From 9d3c8359b525630d2f1ebcacaec500827332610d Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Mon, 18 Dec 2023 19:41:15 +0100 Subject: [PATCH 05/11] collective-io: Start balance collective calls --- .../clawpatch/fclaw2d_clawpatch_output_vtk.c | 103 +++++++++++++++--- 1 file changed, 87 insertions(+), 16 deletions(-) diff --git a/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c b/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c index f7e050257..fce1237fb 100644 --- a/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c +++ b/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c @@ -31,6 +31,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define PATCH_DIM 2 #endif +#define FCLAW_VTK_CEIL(x, y) (((x) + (y) - 1) / (y)) /**< round x / y up */ + #if REFINE_DIM == 2 && PATCH_DIM == 2 #include @@ -39,6 +41,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #define PATCH_CHILDREN 4 +#define FCLAW_VTK_BUF_THRESHOLD 6000 #elif REFINE_DIM == 2 && PATCH_DIM == 3 @@ -50,6 +53,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <_fclaw2d_to_fclaw3dx.h> #define PATCH_CHILDREN 8 +#define FCLAW_VTK_BUF_THRESHOLD 6000 #else #error "This combination of REFINE_DIM and PATCH_DIM is unsupported" @@ -202,16 +206,54 @@ fclaw2d_vtk_write_header (fclaw2d_domain_t * domain, fclaw2d_vtk_state_t * s) * \param [in,out] s The VTK state. * \param [in] psize_field The number of bytes, which are intended to * add to the buffer. + * \param [in] threshold If the sum of the bytes, which are already + * stored in the buffer and \b psize_field + * is greater than \b threshold, the buffer + * is flushed to disk. The \b psize_field + * many new bytes are added to the buffer. + * -1 means that there is no threshold and + * the data just added to \b s->sink. */ static void -add_to_buffer (fclaw2d_vtk_state_t * s, int64_t psize_field) +add_to_buffer (fclaw2d_vtk_state_t * s, int64_t psize_field, + int64_t threshold) { FCLAW_ASSERT (s != NULL); - int retval; + int mpiret; + size_t bytes_in, bytes_out; +#ifdef P4EST_ENABLE_MPIIO + MPI_Status mpistatus; +#else + size_t retvalz; +#endif - retval = sc_io_sink_write (s->sink, s->buf, (size_t) psize_field); - SC_CHECK_ABORT (retval == 0, "VTK buffer write failed"); + if (threshold != -1 + && (s->sink->buffer_bytes + (size_t) psize_field > threshold)) + { + /* buffer threshold exceeded */ + /* write the current buffer to disk */ +#ifdef P4EST_ENABLE_MPIIO + mpiret = + MPI_File_write_all (s->mpifile, s->sink->buffer->array, + s->sink->buffer_bytes, MPI_BYTE, &mpistatus); + SC_CHECK_MPI (mpiret); +#else + retvalz = + fwrite (s->sink->buffer->array, s->sink->buffer_bytes, 1, + s->file); + SC_CHECK_ABORT (retvalz == 1, "VTK file write failed"); +#endif + /* reset the sink */ + mpiret = sc_io_sink_complete (s->sink, &bytes_in, &bytes_out); + SC_CHECK_ABORT (mpiret == 0, "VTK buffer completion failed"); + /* reset the buffer */ + sc_array_reset (s->sink->buffer); + s->sink->buffer_bytes = 0; + } + + mpiret = sc_io_sink_write (s->sink, s->buf, (size_t) psize_field); + SC_CHECK_ABORT (mpiret == 0, "VTK buffer write failed"); } static void @@ -222,7 +264,7 @@ write_position_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, fclaw2d_vtk_state_t *s = (fclaw2d_vtk_state_t *) g->user; s->coordinate_cb (g->glob, patch, blockno, patchno, s->buf); - add_to_buffer (s, s->psize_position); + add_to_buffer (s, s->psize_position, FCLAW_VTK_BUF_THRESHOLD); } static void @@ -318,7 +360,7 @@ write_connectivity_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, } #endif } - add_to_buffer (s, s->psize_connectivity); + add_to_buffer (s, s->psize_connectivity, FCLAW_VTK_BUF_THRESHOLD); } static void @@ -350,7 +392,7 @@ write_offsets_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, *idata++ = k; } } - add_to_buffer (s, s->psize_offsets); + add_to_buffer (s, s->psize_offsets, FCLAW_VTK_BUF_THRESHOLD); } static void @@ -370,7 +412,7 @@ write_types_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, *cdata++ = 12; #endif } - add_to_buffer (s, s->psize_types); + add_to_buffer (s, s->psize_types, FCLAW_VTK_BUF_THRESHOLD); } static void @@ -386,7 +428,7 @@ write_mpirank_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, { *idata++ = domain->mpirank; } - add_to_buffer (s, s->psize_mpirank); + add_to_buffer (s, s->psize_mpirank, FCLAW_VTK_BUF_THRESHOLD); } static void @@ -402,7 +444,7 @@ write_blockno_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, { *idata++ = blockno; } - add_to_buffer (s, s->psize_blockno); + add_to_buffer (s, s->psize_blockno, FCLAW_VTK_BUF_THRESHOLD); } static void @@ -433,7 +475,7 @@ write_patchno_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, *idata++ = gpno; } } - add_to_buffer (s, s->psize_patchno); + add_to_buffer (s, s->psize_patchno, FCLAW_VTK_BUF_THRESHOLD); } static void @@ -445,7 +487,7 @@ write_meqn_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, fclaw2d_vtk_state_t *s = (fclaw2d_vtk_state_t *) g->user; s->value_cb (g->glob, patch, blockno, patchno, s->buf); - add_to_buffer (s, s->psize_meqn); + add_to_buffer (s, s->psize_meqn, FCLAW_VTK_BUF_THRESHOLD); } static void @@ -460,6 +502,8 @@ fclaw2d_vtk_write_field (fclaw2d_global_t * glob, fclaw2d_vtk_state_t * s, #ifndef P4EST_ENABLE_MPIIO size_t retvalz; #else + int i; + int max_num_writes, local_num_writes; int mpiret; MPI_Offset mpipos; #ifdef P4EST_ENABLE_DEBUG @@ -504,10 +548,37 @@ fclaw2d_vtk_write_field (fclaw2d_global_t * glob, fclaw2d_vtk_state_t * s, fclaw2d_global_iterate_patches (glob, cb, s); #ifdef P4EST_ENABLE_MPIIO - mpiret = - MPI_File_write_all (s->mpifile, buffer.array, s->sink->buffer_bytes, - MPI_BYTE, &mpistatus); - SC_CHECK_MPI (mpiret); + if (s->sink->buffer_bytes > 0) + { + /* write the remaining buffered bytes */ + mpiret = + MPI_File_write_all (s->mpifile, buffer.array, + s->sink->buffer_bytes, MPI_BYTE, &mpistatus); + SC_CHECK_MPI (mpiret); + } + + /* Ensure that the collective function MPI_File_write_all is called + * equally often on each rank. + */ + max_num_writes = + FCLAW_VTK_CEIL (glob->domain->local_max_patches * psize_field, + FCLAW_VTK_BUF_THRESHOLD); + local_num_writes = + FCLAW_VTK_CEIL (glob->domain->local_num_patches * psize_field, + FCLAW_VTK_BUF_THRESHOLD); + FCLAW_ASSERT (max_num_writes - local_num_writes >= 0); + + for (i = 0; i < max_num_writes - local_num_writes; ++i) + { + /* This rank has less patches than the rank that holds the maximal + * number of patches. We compensate this by empty write calls to avoid + * a deadlock. + */ + mpiret = + MPI_File_write_all (s->mpifile, buffer.array, 0, MPI_BYTE, + &mpistatus); + SC_CHECK_MPI (mpiret); + } #else retvalz = fwrite (buffer.array, s->sink->buffer_bytes, 1, s->file); SC_CHECK_ABORT (retvalz == 1, "VTK file write failed"); From 1644b0560537e2eafef603ee48dd487ab559538d Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Tue, 19 Dec 2023 15:11:03 +0100 Subject: [PATCH 06/11] collective-io: Use patch threshold --- .../clawpatch/fclaw2d_clawpatch_output_vtk.c | 78 ++++++++++--------- 1 file changed, 43 insertions(+), 35 deletions(-) diff --git a/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c b/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c index fce1237fb..b1b5484d7 100644 --- a/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c +++ b/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c @@ -41,7 +41,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #define PATCH_CHILDREN 4 -#define FCLAW_VTK_BUF_THRESHOLD 6000 +#define FCLAW_VTK_BUF_THRESHOLD -1 #elif REFINE_DIM == 2 && PATCH_DIM == 3 @@ -53,7 +53,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <_fclaw2d_to_fclaw3dx.h> #define PATCH_CHILDREN 8 -#define FCLAW_VTK_BUF_THRESHOLD 6000 +#define FCLAW_VTK_BUF_THRESHOLD -1 #else #error "This combination of REFINE_DIM and PATCH_DIM is unsupported" @@ -201,22 +201,19 @@ fclaw2d_vtk_write_header (fclaw2d_domain_t * domain, fclaw2d_vtk_state_t * s) } /** This function add to a buffer and writes to file if a threshold is exceeded. - * The writing is not yet implemented. * * \param [in,out] s The VTK state. * \param [in] psize_field The number of bytes, which are intended to * add to the buffer. - * \param [in] threshold If the sum of the bytes, which are already - * stored in the buffer and \b psize_field - * is greater than \b threshold, the buffer - * is flushed to disk. The \b psize_field - * many new bytes are added to the buffer. + * \param [in] threshold If the number of patches, which are already + * stored in the buffer + 1 is greater than + * \b threshold, the buffer is flushed to disk. + * Then the new patch is added to the buffer. * -1 means that there is no threshold and * the data just added to \b s->sink. */ static void -add_to_buffer (fclaw2d_vtk_state_t * s, int64_t psize_field, - int64_t threshold) +add_to_buffer (fclaw2d_vtk_state_t * s, int64_t psize_field, int threshold) { FCLAW_ASSERT (s != NULL); @@ -227,17 +224,24 @@ add_to_buffer (fclaw2d_vtk_state_t * s, int64_t psize_field, #else size_t retvalz; #endif + FCLAW_ASSERT (threshold == -1 || threshold > 0); + FCLAW_ASSERT (s->sink->buffer_bytes % psize_field == 0); if (threshold != -1 - && (s->sink->buffer_bytes + (size_t) psize_field > threshold)) + && ((s->sink->buffer_bytes / psize_field) + 1 > (size_t) threshold)) { + FCLAW_ASSERT ((int) s->sink->buffer_bytes >= psize_field); /* buffer threshold exceeded */ /* write the current buffer to disk */ #ifdef P4EST_ENABLE_MPIIO - mpiret = - MPI_File_write_all (s->mpifile, s->sink->buffer->array, - s->sink->buffer_bytes, MPI_BYTE, &mpistatus); - SC_CHECK_MPI (mpiret); + if (s->sink->buffer_bytes > 0) + { + mpiret = + MPI_File_write_all (s->mpifile, s->sink->buffer->array, + (int) s->sink->buffer_bytes, MPI_BYTE, + &mpistatus); + SC_CHECK_MPI (mpiret); + } #else retvalz = fwrite (s->sink->buffer->array, s->sink->buffer_bytes, 1, @@ -505,6 +509,7 @@ fclaw2d_vtk_write_field (fclaw2d_global_t * glob, fclaw2d_vtk_state_t * s, int i; int max_num_writes, local_num_writes; int mpiret; + int threshold; MPI_Offset mpipos; #ifdef P4EST_ENABLE_DEBUG MPI_Offset mpinew; @@ -548,7 +553,7 @@ fclaw2d_vtk_write_field (fclaw2d_global_t * glob, fclaw2d_vtk_state_t * s, fclaw2d_global_iterate_patches (glob, cb, s); #ifdef P4EST_ENABLE_MPIIO - if (s->sink->buffer_bytes > 0) + if (s->sink->buffer_bytes > 0 || FCLAW_VTK_BUF_THRESHOLD == -1) { /* write the remaining buffered bytes */ mpiret = @@ -557,27 +562,30 @@ fclaw2d_vtk_write_field (fclaw2d_global_t * glob, fclaw2d_vtk_state_t * s, SC_CHECK_MPI (mpiret); } - /* Ensure that the collective function MPI_File_write_all is called - * equally often on each rank. - */ - max_num_writes = - FCLAW_VTK_CEIL (glob->domain->local_max_patches * psize_field, - FCLAW_VTK_BUF_THRESHOLD); - local_num_writes = - FCLAW_VTK_CEIL (glob->domain->local_num_patches * psize_field, - FCLAW_VTK_BUF_THRESHOLD); - FCLAW_ASSERT (max_num_writes - local_num_writes >= 0); - - for (i = 0; i < max_num_writes - local_num_writes; ++i) + if (FCLAW_VTK_BUF_THRESHOLD != -1) { - /* This rank has less patches than the rank that holds the maximal - * number of patches. We compensate this by empty write calls to avoid - * a deadlock. + /* Ensure that the collective function MPI_File_write_all is called + * equally often on each rank. */ - mpiret = - MPI_File_write_all (s->mpifile, buffer.array, 0, MPI_BYTE, - &mpistatus); - SC_CHECK_MPI (mpiret); + max_num_writes = + FCLAW_VTK_CEIL (glob->domain->local_max_patches, + FCLAW_VTK_BUF_THRESHOLD); + local_num_writes = + FCLAW_VTK_CEIL (glob->domain->local_num_patches, + FCLAW_VTK_BUF_THRESHOLD); + FCLAW_ASSERT (max_num_writes - local_num_writes >= 0); + + for (i = 0; i < max_num_writes - local_num_writes; ++i) + { + /* This rank has less patches than the rank that holds the maximal + * number of patches. We compensate this by empty write calls to avoid + * a deadlock. + */ + mpiret = + MPI_File_write_all (s->mpifile, buffer.array, 0, MPI_BYTE, + &mpistatus); + SC_CHECK_MPI (mpiret); + } } #else retvalz = fwrite (buffer.array, s->sink->buffer_bytes, 1, s->file); From 1e156e5570538cbbaaf0ebd9fc8270587c124923 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Tue, 19 Dec 2023 15:21:55 +0100 Subject: [PATCH 07/11] collective-io: Polishing --- .../clawpatch/fclaw2d_clawpatch_output_vtk.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c b/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c index b1b5484d7..cdf13fc8c 100644 --- a/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c +++ b/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c @@ -41,7 +41,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #define PATCH_CHILDREN 4 -#define FCLAW_VTK_BUF_THRESHOLD -1 +#define FCLAW_VTK_BUF_THRESHOLD -1 /**< maximal patch count in the local buffer; + -1 means no threshold */ #elif REFINE_DIM == 2 && PATCH_DIM == 3 @@ -53,7 +54,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <_fclaw2d_to_fclaw3dx.h> #define PATCH_CHILDREN 8 -#define FCLAW_VTK_BUF_THRESHOLD -1 +#define FCLAW_VTK_BUF_THRESHOLD -1 /**< maximal patch count in the local buffer; + -1 means no threshold */ #else #error "This combination of REFINE_DIM and PATCH_DIM is unsupported" @@ -218,17 +220,16 @@ add_to_buffer (fclaw2d_vtk_state_t * s, int64_t psize_field, int threshold) FCLAW_ASSERT (s != NULL); int mpiret; - size_t bytes_in, bytes_out; #ifdef P4EST_ENABLE_MPIIO MPI_Status mpistatus; #else size_t retvalz; #endif FCLAW_ASSERT (threshold == -1 || threshold > 0); - FCLAW_ASSERT (s->sink->buffer_bytes % psize_field == 0); + FCLAW_ASSERT (((int) s->sink->buffer_bytes) % psize_field == 0); if (threshold != -1 - && ((s->sink->buffer_bytes / psize_field) + 1 > (size_t) threshold)) + && ((((int) s->sink->buffer_bytes) / psize_field) + 1 > threshold)) { FCLAW_ASSERT ((int) s->sink->buffer_bytes >= psize_field); /* buffer threshold exceeded */ @@ -249,7 +250,7 @@ add_to_buffer (fclaw2d_vtk_state_t * s, int64_t psize_field, int threshold) SC_CHECK_ABORT (retvalz == 1, "VTK file write failed"); #endif /* reset the sink */ - mpiret = sc_io_sink_complete (s->sink, &bytes_in, &bytes_out); + mpiret = sc_io_sink_complete (s->sink, NULL, NULL); SC_CHECK_ABORT (mpiret == 0, "VTK buffer completion failed"); /* reset the buffer */ sc_array_reset (s->sink->buffer); @@ -509,7 +510,6 @@ fclaw2d_vtk_write_field (fclaw2d_global_t * glob, fclaw2d_vtk_state_t * s, int i; int max_num_writes, local_num_writes; int mpiret; - int threshold; MPI_Offset mpipos; #ifdef P4EST_ENABLE_DEBUG MPI_Offset mpinew; @@ -558,7 +558,8 @@ fclaw2d_vtk_write_field (fclaw2d_global_t * glob, fclaw2d_vtk_state_t * s, /* write the remaining buffered bytes */ mpiret = MPI_File_write_all (s->mpifile, buffer.array, - s->sink->buffer_bytes, MPI_BYTE, &mpistatus); + (int) s->sink->buffer_bytes, MPI_BYTE, + &mpistatus); SC_CHECK_MPI (mpiret); } From 382a995c59dd0144fad3ce1b1a0cad55ca4c5096 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Tue, 19 Dec 2023 15:36:53 +0100 Subject: [PATCH 08/11] collective-io: Extend documentation --- src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c b/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c index cdf13fc8c..a39171322 100644 --- a/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c +++ b/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c @@ -202,7 +202,11 @@ fclaw2d_vtk_write_header (fclaw2d_domain_t * domain, fclaw2d_vtk_state_t * s) return retval ? -1 : 0; } -/** This function add to a buffer and writes to file if a threshold is exceeded. +/** This function adds to a buffer and writes to file if a threshold is exceeded. + * + * This function assumes that the buffer consists of patch data of the size + * \b psize_field. This means it must hold + * \b s->sink->buffer_bytes % \b s->sink->buffer_bytes == 0. * * \param [in,out] s The VTK state. * \param [in] psize_field The number of bytes, which are intended to From cffa6f5bae509057e43c25d4653845851d3db1ca Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Tue, 19 Dec 2023 17:41:29 +0100 Subject: [PATCH 09/11] collective-io: Adjust vtk state for buffering --- .../clawpatch/fclaw2d_clawpatch_output_vtk.c | 68 +++++++++++++------ 1 file changed, 46 insertions(+), 22 deletions(-) diff --git a/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c b/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c index a39171322..dae6f7cb0 100644 --- a/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c +++ b/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c @@ -41,8 +41,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #define PATCH_CHILDREN 4 -#define FCLAW_VTK_BUF_THRESHOLD -1 /**< maximal patch count in the local buffer; - -1 means no threshold */ #elif REFINE_DIM == 2 && PATCH_DIM == 3 @@ -54,8 +52,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <_fclaw2d_to_fclaw3dx.h> #define PATCH_CHILDREN 8 -#define FCLAW_VTK_BUF_THRESHOLD -1 /**< maximal patch count in the local buffer; - -1 means no threshold */ #else #error "This combination of REFINE_DIM and PATCH_DIM is unsupported" @@ -98,7 +94,23 @@ typedef struct fclaw2d_vtk_state MPI_Offset mpibegin; #endif char *buf; - sc_io_sink_t *sink; + + /* The following three struct elements are dedicated to manage buffering + * patches. + * We track the number of buffered patches in \b num_buffered_patches and + * if the buffer contains more than \b patch_threshold many patches, the + * patches are written to disk and buffer is flushed. + * The variable \b num_buffered_patches relates to the number of buffered + * patches during one fclaw2d_vtk_write_field call. + * If \b patch_threshold is -1, there are no intermediate writes but all + * patches during a fclaw2d_vtk_write_field are buffered and then written + * to the disk. + */ + sc_io_sink_t *sink; /**< the sink to manage the patches buffer */ + int num_buffered_patches; /**< the number of patches buffered during the + current fclaw2d_vtk_write_field call */ + int patch_threshold; /**< maximal number of patches that are buffered during + a call of fclaw2d_vtk_write_field */ } fclaw2d_vtk_state_t; @@ -219,7 +231,7 @@ fclaw2d_vtk_write_header (fclaw2d_domain_t * domain, fclaw2d_vtk_state_t * s) * the data just added to \b s->sink. */ static void -add_to_buffer (fclaw2d_vtk_state_t * s, int64_t psize_field, int threshold) +add_to_buffer (fclaw2d_vtk_state_t * s, int64_t psize_field) { FCLAW_ASSERT (s != NULL); @@ -229,17 +241,19 @@ add_to_buffer (fclaw2d_vtk_state_t * s, int64_t psize_field, int threshold) #else size_t retvalz; #endif - FCLAW_ASSERT (threshold == -1 || threshold > 0); + FCLAW_ASSERT (s->patch_threshold == -1 || s->patch_threshold > 0); FCLAW_ASSERT (((int) s->sink->buffer_bytes) % psize_field == 0); - if (threshold != -1 - && ((((int) s->sink->buffer_bytes) / psize_field) + 1 > threshold)) + if (s->patch_threshold != -1 + && (s->num_buffered_patches + 1 > s->patch_threshold)) { FCLAW_ASSERT ((int) s->sink->buffer_bytes >= psize_field); + FCLAW_ASSERT (((int) s->sink->buffer_bytes) / psize_field == + s->num_buffered_patches); /* buffer threshold exceeded */ /* write the current buffer to disk */ #ifdef P4EST_ENABLE_MPIIO - if (s->sink->buffer_bytes > 0) + if (s->num_buffered_patches > 0) { mpiret = MPI_File_write_all (s->mpifile, s->sink->buffer->array, @@ -259,10 +273,12 @@ add_to_buffer (fclaw2d_vtk_state_t * s, int64_t psize_field, int threshold) /* reset the buffer */ sc_array_reset (s->sink->buffer); s->sink->buffer_bytes = 0; + s->num_buffered_patches = 0; } mpiret = sc_io_sink_write (s->sink, s->buf, (size_t) psize_field); SC_CHECK_ABORT (mpiret == 0, "VTK buffer write failed"); + ++s->num_buffered_patches; } static void @@ -273,7 +289,7 @@ write_position_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, fclaw2d_vtk_state_t *s = (fclaw2d_vtk_state_t *) g->user; s->coordinate_cb (g->glob, patch, blockno, patchno, s->buf); - add_to_buffer (s, s->psize_position, FCLAW_VTK_BUF_THRESHOLD); + add_to_buffer (s, s->psize_position); } static void @@ -369,7 +385,7 @@ write_connectivity_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, } #endif } - add_to_buffer (s, s->psize_connectivity, FCLAW_VTK_BUF_THRESHOLD); + add_to_buffer (s, s->psize_connectivity); } static void @@ -401,7 +417,7 @@ write_offsets_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, *idata++ = k; } } - add_to_buffer (s, s->psize_offsets, FCLAW_VTK_BUF_THRESHOLD); + add_to_buffer (s, s->psize_offsets); } static void @@ -421,7 +437,7 @@ write_types_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, *cdata++ = 12; #endif } - add_to_buffer (s, s->psize_types, FCLAW_VTK_BUF_THRESHOLD); + add_to_buffer (s, s->psize_types); } static void @@ -437,7 +453,7 @@ write_mpirank_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, { *idata++ = domain->mpirank; } - add_to_buffer (s, s->psize_mpirank, FCLAW_VTK_BUF_THRESHOLD); + add_to_buffer (s, s->psize_mpirank); } static void @@ -453,7 +469,7 @@ write_blockno_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, { *idata++ = blockno; } - add_to_buffer (s, s->psize_blockno, FCLAW_VTK_BUF_THRESHOLD); + add_to_buffer (s, s->psize_blockno); } static void @@ -484,7 +500,7 @@ write_patchno_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, *idata++ = gpno; } } - add_to_buffer (s, s->psize_patchno, FCLAW_VTK_BUF_THRESHOLD); + add_to_buffer (s, s->psize_patchno); } static void @@ -496,7 +512,7 @@ write_meqn_cb (fclaw2d_domain_t * domain, fclaw2d_patch_t * patch, fclaw2d_vtk_state_t *s = (fclaw2d_vtk_state_t *) g->user; s->value_cb (g->glob, patch, blockno, patchno, s->buf); - add_to_buffer (s, s->psize_meqn, FCLAW_VTK_BUF_THRESHOLD); + add_to_buffer (s, s->psize_meqn); } static void @@ -526,6 +542,8 @@ fclaw2d_vtk_write_field (fclaw2d_global_t * glob, fclaw2d_vtk_state_t * s, sc_array_init_size (&buffer, 1, psize_field); s->sink = sc_io_sink_new (SC_IO_TYPE_BUFFER, SC_IO_MODE_APPEND, SC_IO_ENCODE_NONE, &buffer); + FCLAW_ASSERT (s->num_buffered_patches == 0); + s->num_buffered_patches = 0; #ifdef P4EST_ENABLE_MPIIO mpipos = s->mpibegin + offset_field; if (domain->mpirank > 0) @@ -557,7 +575,7 @@ fclaw2d_vtk_write_field (fclaw2d_global_t * glob, fclaw2d_vtk_state_t * s, fclaw2d_global_iterate_patches (glob, cb, s); #ifdef P4EST_ENABLE_MPIIO - if (s->sink->buffer_bytes > 0 || FCLAW_VTK_BUF_THRESHOLD == -1) + if (s->num_buffered_patches > 0 || s->patch_threshold == -1) { /* write the remaining buffered bytes */ mpiret = @@ -567,17 +585,17 @@ fclaw2d_vtk_write_field (fclaw2d_global_t * glob, fclaw2d_vtk_state_t * s, SC_CHECK_MPI (mpiret); } - if (FCLAW_VTK_BUF_THRESHOLD != -1) + if (s->patch_threshold != -1) { /* Ensure that the collective function MPI_File_write_all is called * equally often on each rank. */ max_num_writes = FCLAW_VTK_CEIL (glob->domain->local_max_patches, - FCLAW_VTK_BUF_THRESHOLD); + s->patch_threshold); local_num_writes = FCLAW_VTK_CEIL (glob->domain->local_num_patches, - FCLAW_VTK_BUF_THRESHOLD); + s->patch_threshold); FCLAW_ASSERT (max_num_writes - local_num_writes >= 0); for (i = 0; i < max_num_writes - local_num_writes; ++i) @@ -601,6 +619,7 @@ fclaw2d_vtk_write_field (fclaw2d_global_t * glob, fclaw2d_vtk_state_t * s, sc_array_reset (&buffer); sc_io_sink_destroy (s->sink); P4EST_FREE (s->buf); + s->num_buffered_patches = 0; #ifdef P4EST_ENABLE_MPIIO #ifdef P4EST_ENABLE_DEBUG @@ -764,6 +783,11 @@ fclaw2d_vtk_write_file (fclaw2d_global_t * glob, const char *basename, s->buf = NULL; s->sink = NULL; + /* The threshold may be adjusted; see the documentation of + * fclaw2d_vtk_state_t for further information. + */ + s->patch_threshold = -1; + s->num_buffered_patches = 0; /* write header meta data and check for error */ retval = 0; From 01f6906725aa1c8c56b38d34647b191fbc6cc0fc Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Tue, 19 Dec 2023 17:53:18 +0100 Subject: [PATCH 10/11] collective-io: Extend macro documentation --- src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c b/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c index dae6f7cb0..a7445d7b3 100644 --- a/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c +++ b/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c @@ -31,7 +31,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define PATCH_DIM 2 #endif -#define FCLAW_VTK_CEIL(x, y) (((x) + (y) - 1) / (y)) /**< round x / y up */ +/** Round up quotient x / y. */ +#define FCLAW_VTK_CEIL(x, y) (((x) + (y) - 1) / (y)) /**< round up the quotient + x / y; this is a local + macro used for + computations regarding + the VTK patch buffering */ #if REFINE_DIM == 2 && PATCH_DIM == 2 From 4307acc5ec3b42da032ec3f98c2b657ed7122ce5 Mon Sep 17 00:00:00 2001 From: Tim Griesbach Date: Tue, 19 Dec 2023 17:56:16 +0100 Subject: [PATCH 11/11] collective-io: Minor doc. adjustment --- src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c b/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c index a7445d7b3..9ef5e74dc 100644 --- a/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c +++ b/src/patches/clawpatch/fclaw2d_clawpatch_output_vtk.c @@ -104,7 +104,7 @@ typedef struct fclaw2d_vtk_state * patches. * We track the number of buffered patches in \b num_buffered_patches and * if the buffer contains more than \b patch_threshold many patches, the - * patches are written to disk and buffer is flushed. + * patches are written to disk and the buffer is flushed. * The variable \b num_buffered_patches relates to the number of buffered * patches during one fclaw2d_vtk_write_field call. * If \b patch_threshold is -1, there are no intermediate writes but all