diff --git a/src/fclaw2d_to_3d.h b/src/fclaw2d_to_3d.h index 8caaa4c4f..40af559eb 100644 --- a/src/fclaw2d_to_3d.h +++ b/src/fclaw2d_to_3d.h @@ -216,6 +216,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define fclaw2d_patch_wrap_cb fclaw3d_patch_wrap_cb #define fclaw2d_transfer_wrap_cb fclaw3d_transfer_wrap_cb #define fclaw2d_match_wrap_cb fclaw3d_match_wrap_cb +#define fclaw2d_pack_wrap_cb fclaw3d_pack_wrap_cb +#define fclaw2d_unpack_wrap_cb fclaw3d_unpack_wrap_cb #define fclaw2d_intersect_wrap_cb fclaw3d_intersect_wrap_cb #define fclaw2d_interpolate_point_wrap_cb fclaw3d_interpolate_point_wrap_cb #define fclaw2d_file_error_wrap fclaw3d_file_error_wrap diff --git a/src/fclaw2d_wrap.c b/src/fclaw2d_wrap.c index f01d3f23a..6e0562a56 100644 --- a/src/fclaw2d_wrap.c +++ b/src/fclaw2d_wrap.c @@ -216,6 +216,38 @@ fclaw2d_match_wrap_cb(fclaw2d_domain_t * old_domain_2d, wrap->user); } +void +fclaw2d_pack_wrap_cb (fclaw2d_domain_t * domain_2d, + fclaw2d_patch_t * patch_2d, + int blockno, int patchno, + void *pack_data_here, + void *user) +{ + fclaw_pack_wrap_user_t* wrap = + (fclaw_pack_wrap_user_t*) user; + + fclaw_domain_t* domain = get_domain(domain_2d); + fclaw_patch_t* patch = get_patch(patch_2d); + + wrap->pcb(domain, patch, blockno, patchno, pack_data_here, wrap->user); +} + +void +fclaw2d_unpack_wrap_cb (fclaw2d_domain_t * domain_2d, + fclaw2d_patch_t * patch_2d, + int blockno, int patchno, + void *unpack_data_from_here, + void *user) +{ + fclaw_unpack_wrap_user_t* wrap = + (fclaw_unpack_wrap_user_t*) user; + + fclaw_domain_t* domain = get_domain(domain_2d); + fclaw_patch_t* patch = get_patch(patch_2d); + + wrap->ucb(domain, patch, blockno, patchno, unpack_data_from_here, wrap->user); +} + int fclaw2d_intersect_wrap_cb (fclaw2d_domain_t * domain_2d, fclaw2d_patch_t * patch_2d, diff --git a/src/fclaw2d_wrap.h b/src/fclaw2d_wrap.h index 0bc40e712..5fe212a30 100644 --- a/src/fclaw2d_wrap.h +++ b/src/fclaw2d_wrap.h @@ -91,6 +91,31 @@ fclaw2d_match_wrap_cb(fclaw2d_domain_t * old_domain, int blockno, int old_patchno, int new_patchno, void *user); + +/** + * @brief Wraps a partition pack callback for 2d domains. + * This is used by passing this callback to a function that takes a @ref fclaw2d_pack_callback_t. + * The user pointer should be a pointer to a @ref fclaw_pack_wrap_user_t. + */ +void +fclaw2d_pack_wrap_cb(fclaw2d_domain_t * domain, + fclaw2d_patch_t * patch, + int blockno, int patchno, + void *pack_data_here, + void *user); + +/** + * @brief Wraps a partition unpack callback for 2d domains. + * This is used by passing this callback to a function that takes a @ref fclaw2d_unpack_callback_t. + * The user pointer should be a pointer to a @ref fclaw_unpack_wrap_user_t. + */ +void +fclaw2d_unpack_wrap_cb(fclaw2d_domain_t * domain, + fclaw2d_patch_t * patch, + int blockno, int patchno, + void *unpack_data_from_here, + void *user); + /** * @brief Wraps a intersect callback for 2d domains. * This is used by passing this callback to a function that takes a @ref fclaw2d_intersect_callback_t. diff --git a/src/fclaw3d_wrap.h b/src/fclaw3d_wrap.h index 3f9b42ba1..0f25332a7 100644 --- a/src/fclaw3d_wrap.h +++ b/src/fclaw3d_wrap.h @@ -92,6 +92,31 @@ fclaw3d_match_wrap_cb(fclaw3d_domain_t * old_domain, int old_patchno, int new_patchno, void *user); +/** + * @brief Wraps a partition pack callback for 3d domains. + * This is used by passing this callback to a function that takes a @ref fclaw3d_pack_callback_t. + * The user pointer should be a pointer to a @ref fclaw_pack_wrap_user_t. + */ +void +fclaw3d_pack_wrap_cb(fclaw3d_domain_t * domain, + fclaw3d_patch_t * patch, + int blockno, int patchno, + void *pack_data_here, + void *user); + +/** + * @brief Wraps a partition unpack callback for 3d domains. + * This is used by passing this callback to a function that takes a @ref fclaw3d_unpack_callback_t. + * The user pointer should be a pointer to a @ref fclaw_unpack_wrap_user_t. + */ +void +fclaw3d_unpack_wrap_cb(fclaw3d_domain_t * domain, + fclaw3d_patch_t * patch, + int blockno, int patchno, + void *unpack_data_from_here, + void *user); + + /** * @brief Wraps a intersect callback for 2d domains. * This is used by passing this callback to a function that takes a @ref fclaw2d_intersect_callback_t. diff --git a/src/fclaw_domain.c b/src/fclaw_domain.c index e4cf4e031..e41717479 100644 --- a/src/fclaw_domain.c +++ b/src/fclaw_domain.c @@ -78,7 +78,7 @@ void fclaw_domain_reset(fclaw_global_t* glob) /* This is here to delete any patches created during initialization, and not through regridding */ fclaw_patch_t *patch = block->patches + j; - fclaw_patch_data_delete(glob,patch); + fclaw_patch_data_delete(glob,*domain,patch); } block->user = NULL; } diff --git a/src/fclaw_initialize.c b/src/fclaw_initialize.c index c23959d1b..3fd76096a 100644 --- a/src/fclaw_initialize.c +++ b/src/fclaw_initialize.c @@ -64,10 +64,12 @@ void cb_initialize (fclaw_domain_t *domain, fclaw_build_mode_t build_mode = FCLAW_BUILD_FOR_UPDATE; - fclaw_patch_build(g->glob,this_patch, - this_block_idx, - this_patch_idx, - &build_mode); + fclaw_patch_build(g->glob, + domain, + this_patch, + this_block_idx, + this_patch_idx, + &build_mode); fclaw_patch_initialize(g->glob,this_patch,this_block_idx,this_patch_idx); } @@ -168,30 +170,11 @@ void build_initial_domain(fclaw_global_t *glob) { fclaw_global_infof(" -- Have new initial refinement\n"); - /* Re-initialize new grids. Ghost cell values needed for - interpolation have already been set by initialization */ - fclaw_timer_start (&glob->timers[FCLAW_TIMER_REGRID_BUILD]); - fclaw_global_iterate_adapted(glob, new_domain, - cb_fclaw_regrid_repopulate, - (void *) &domain_init); - - fclaw_timer_stop (&glob->timers[FCLAW_TIMER_REGRID_BUILD]); - - /* free all memory associated with old domain */ - fclaw_domain_reset(glob); - *domain = new_domain; - new_domain = NULL; - - /* Repartition domain to new processors. */ - fclaw_partition_domain(glob,FCLAW_TIMER_INIT); - - /* Set up ghost patches. This probably doesn't need to be done - each time we add a new level. */ - fclaw_exchange_setup(glob,FCLAW_TIMER_INIT); - - /* This is normally called from regrid, once the initial domain - has been set up */ - fclaw_regrid_set_neighbor_types(glob); + fclaw_regrid_process_new_refinement(glob, + domain, + new_domain, + domain_init, + FCLAW_TIMER_INIT); } else { @@ -219,9 +202,16 @@ void fclaw_initialize_domain_flags(fclaw_global_t *glob) (glob->domain, fclaw_opt->smooth_refine, fclaw_opt->smooth_level, fclaw_opt->coarsen_delay); + int skip_local = fclaw_opt->partition_mode == FCLAW_PARTITION_MODE_SKIP_LOCAL || + fclaw_opt->partition_mode == FCLAW_PARTITION_MODE_REFINE_AFTER; + + int skip_refined = fclaw_opt->partition_mode == FCLAW_PARTITION_MODE_REFINE_AFTER; + /* set partitioning */ fclaw_domain_set_partitioning (glob->domain, - fclaw_opt->partition_for_coarsening, 1, 0); + fclaw_opt->partition_for_coarsening, + skip_local, + skip_refined); } static void diff --git a/src/fclaw_options.c b/src/fclaw_options.c index fdc963156..519c289fb 100644 --- a/src/fclaw_options.c +++ b/src/fclaw_options.c @@ -382,6 +382,15 @@ fclaw_register (fclaw_options_t* fclaw_opt, sc_options_t * opt) &fclaw_opt->max_refinement_ratio, 1.0, "Ratio of patches to refine before paritioning and continuing refinement. [1.0]"); + kv = fclaw_opt->kv_partition_mode = sc_keyvalue_new (); + sc_keyvalue_set_int (kv, "legacy", FCLAW_PARTITION_MODE_LEGACY); + sc_keyvalue_set_int (kv, "pack-all", FCLAW_PARTITION_MODE_PACK_ALL); + sc_keyvalue_set_int (kv, "skip-local", FCLAW_PARTITION_MODE_SKIP_LOCAL); + sc_keyvalue_set_int (kv, "refine-after-partition", FCLAW_PARTITION_MODE_REFINE_AFTER); + sc_options_add_keyvalue (opt, 0, "partition-mode", + &fclaw_opt->partition_mode, + "legacy", kv, "Parition mode to use. One of (legacy, pack-all, skip-local, refine-after-partition) [legacy]"); + fclaw_opt->is_registered = 1; fclaw_opt->is_unpacked = 0; @@ -460,6 +469,7 @@ fclaw_options_destroy(fclaw_options_t* fclaw_opt) FCLAW_ASSERT (fclaw_opt->kv_timing_verbosity != NULL); sc_keyvalue_destroy (fclaw_opt->kv_timing_verbosity); + sc_keyvalue_destroy (fclaw_opt->kv_partition_mode); // Strings need to be freed if this was unpacked form buffer if(fclaw_opt->is_unpacked){ diff --git a/src/fclaw_options.h b/src/fclaw_options.h index 15a281f1e..109154da7 100644 --- a/src/fclaw_options.h +++ b/src/fclaw_options.h @@ -112,8 +112,21 @@ void fclaw_options_convert_double_array (const char *array_string, void fclaw_options_destroy_array(void* array); -/* Plan is to replace fclaw_options_t with fclaw_options_t. - Maybe use a macro as an intermediate step? */ +/** + * @brief Enum for partition mode + */ +enum fclaw_parition_mode +{ + /** Use legacy paritioning interface */ + FCLAW_PARTITION_MODE_LEGACY, + /** Pack/unpack all patches */ + FCLAW_PARTITION_MODE_PACK_ALL, + /** Skip packing of local patches */ + FCLAW_PARTITION_MODE_SKIP_LOCAL, + /** Skip packing of local patches and refine after partitioning */ + FCLAW_PARTITION_MODE_REFINE_AFTER +}; + struct fclaw_options { @@ -254,6 +267,9 @@ struct fclaw_options const char * regression_check; /**< filename of regression check values */ double max_refinement_ratio; /**< Maximum refinment ratio before partitioning and continuing refinement. */ + + int partition_mode; /**< Partition mode */ + sc_keyvalue_t *kv_partition_mode; /**< sc_keyvalue needed for enum option type */ }; struct fclaw_global; diff --git a/src/fclaw_partition.c b/src/fclaw_partition.c index 547a7b4b6..2cbf6f706 100644 --- a/src/fclaw_partition.c +++ b/src/fclaw_partition.c @@ -54,31 +54,46 @@ void cb_partition_pack(fclaw_domain_t *domain, } + +static +void cb_transfer(fclaw_domain_t * old_domain, + fclaw_patch_t * old_patch, + fclaw_domain_t * new_domain, + fclaw_patch_t * new_patch, + int blockno, + int old_patchno, int new_patchno, + void *user) +{ + fclaw_global_t *glob = (fclaw_global_t *) user; + FCLAW_ASSERT(old_patch->xlower == new_patch->xlower); + FCLAW_ASSERT(old_patch->ylower == new_patch->ylower); + FCLAW_ASSERT(old_patch->zlower == new_patch->zlower); + FCLAW_ASSERT(old_patch->xupper == new_patch->xupper); + FCLAW_ASSERT(old_patch->yupper == new_patch->yupper); + FCLAW_ASSERT(old_patch->zupper == new_patch->zupper); + + fclaw_patch_shallow_copy(glob,old_domain,old_patch,new_domain,new_patch, + blockno,old_patchno,new_patchno); + fclaw_patch_data_delete(glob,old_domain,old_patch); +} + static -void cb_partition_transfer(fclaw_domain_t * old_domain, +void cb_transfer_and_unpack(fclaw_domain_t * old_domain, fclaw_patch_t * old_patch, fclaw_domain_t * new_domain, fclaw_patch_t * new_patch, int blockno, int old_patchno, int new_patchno, void *user) + { /* Transfer data to new domain */ fclaw_global_iterate_t *g = (fclaw_global_iterate_t *) user; if (old_patch != NULL) { - FCLAW_ASSERT(old_patch->xlower == new_patch->xlower); - FCLAW_ASSERT(old_patch->ylower == new_patch->ylower); - FCLAW_ASSERT(old_patch->zlower == new_patch->zlower); - FCLAW_ASSERT(old_patch->xupper == new_patch->xupper); - FCLAW_ASSERT(old_patch->yupper == new_patch->yupper); - FCLAW_ASSERT(old_patch->zupper == new_patch->zupper); - - new_patch->user = old_patch->user; - old_patch->user = NULL; - ++old_domain->count_delete_patch; - ++new_domain->count_set_patch; + cb_transfer(old_domain,old_patch,new_domain,new_patch, + blockno,old_patchno,new_patchno,g->glob); } else { @@ -94,24 +109,13 @@ void cb_partition_transfer(fclaw_domain_t * old_domain, and the both domains are needed to increment/decrement patches */ fclaw_patch_partition_unpack(g->glob,new_domain,new_patch, blockno,new_patchno,unpack_data_from_here); - - /* Reason for the following two lines: the glob contains the old domain - which is incremented in ddata_old but we really want to increment the - new domain. */ - --old_domain->count_set_patch; - ++new_domain->count_set_patch; - - } } - -/* -------------------------------------------------------------------------- - Public interface - -------------------------------------------------------------------------- */ -/* Question : Do all patches on this processor get packed? */ -void fclaw_partition_domain(fclaw_global_t* glob, - fclaw_timer_names_t running) +/* old partitioning interface, kept for comparison */ +static +void partition_domain_legacy(fclaw_global_t* glob, + fclaw_timer_names_t running) { fclaw_domain_t** domain = &glob->domain; fclaw_timer_start (&glob->timers[FCLAW_TIMER_PARTITION]); @@ -173,7 +177,7 @@ void fclaw_partition_domain(fclaw_global_t* glob, /* New version? */ fclaw_global_iterate_partitioned(glob,domain_partitioned, - cb_partition_transfer, + cb_transfer_and_unpack, (void*) patch_data); /* then the old domain is no longer necessary */ @@ -191,3 +195,85 @@ void fclaw_partition_domain(fclaw_global_t* glob, fclaw_timer_stop (&glob->timers[FCLAW_TIMER_PARTITION]); } + +static +void patch_pack_cb(fclaw_domain_t * domain, + fclaw_patch_t * patch, int blockno, + int patchno, void *pack_data_here, + void *user) +{ + fclaw_global_t *glob = (fclaw_global_t *) user; + fclaw_patch_partition_pack(glob,patch, + blockno,patchno, + pack_data_here); + fclaw_patch_data_delete(glob,domain,patch); +} + +static +void patch_unpack_cb(fclaw_domain_t * domain, + fclaw_patch_t * patch, + int blockno, int patchno, + void *unpack_data_from_here, + void *user) +{ + fclaw_global_t *glob = (fclaw_global_t *) user; + fclaw_patch_partition_unpack(glob,domain,patch, + blockno,patchno, + unpack_data_from_here); +} + + +static +void partition_domain(fclaw_global_t* glob, + fclaw_timer_names_t running) +{ + fclaw_domain_t** domain = &glob->domain; + fclaw_options_t *fclaw_opt = fclaw_get_options(glob); + fclaw_timer_start (&glob->timers[FCLAW_TIMER_PARTITION]); + + int exponent = fclaw_opt->subcycle && fclaw_opt->weighted_partition ? 1 : 0; + fclaw_domain_t *new_domain = + fclaw_domain_partition (*domain, exponent); + + if(new_domain != NULL) + { + size_t psize = fclaw_patch_partition_packsize(glob); + fclaw_domain_partition_t *p = fclaw_domain_iterate_pack(*domain, + psize, + patch_pack_cb, + (void *) glob); + + fclaw_domain_iterate_transfer(*domain, new_domain, + cb_transfer, + glob); + + fclaw_domain_iterate_unpack(new_domain, p, patch_unpack_cb, (void *) glob); + fclaw_domain_partition_free(p); + + /* then the old domain is no longer necessary */ + fclaw_domain_reset(glob); + *domain = new_domain; + new_domain = NULL; + + /* internal clean up */ + fclaw_domain_complete(*domain); + } + + fclaw_timer_stop (&glob->timers[FCLAW_TIMER_PARTITION]); +} + +/* -------------------------------------------------------------------------- + Public interface + -------------------------------------------------------------------------- */ +void fclaw_partition_domain(fclaw_global_t* glob, + fclaw_timer_names_t running) +{ + if(fclaw_get_options(glob)->partition_mode == FCLAW_PARTITION_MODE_LEGACY) + { + partition_domain_legacy(glob,running); + } + else + { + partition_domain(glob,running); + } +} diff --git a/src/fclaw_patch.c b/src/fclaw_patch.c index 3943b164f..3067e4756 100644 --- a/src/fclaw_patch.c +++ b/src/fclaw_patch.c @@ -53,20 +53,39 @@ void* get_user_patch(fclaw_patch_t* patch) return pdata->user_patch; } - +static +void set_unpacking_flag(fclaw_patch_t* patch) +{ + fclaw_patch_data_t *pdata = get_patch_data(patch); + pdata->flags |= FCLAW_PATCH_DATA_UNPACKING; +} +static +int is_unpacking(fclaw_patch_t* patch) +{ + fclaw_patch_data_t *pdata = get_patch_data(patch); + return pdata->flags & FCLAW_PATCH_DATA_UNPACKING; +} +static +void clear_unpacking_flag(fclaw_patch_t* patch) +{ + fclaw_patch_data_t *pdata = get_patch_data(patch); + pdata->flags &= ~FCLAW_PATCH_DATA_UNPACKING; +} /* ----------------------------- Creating/deleting patches ---------------------------- */ static void patch_data_new(fclaw_global_t* glob, + fclaw_domain_t* domain, fclaw_patch_t* this_patch, int this_block_idx, int this_patch_idx) { - fclaw_patch_vtable_t *patch_vt = fclaw_patch_vt(glob); - - /* Initialize user data */ - fclaw_patch_data_t *pdata = FCLAW_ALLOC(fclaw_patch_data_t, 1); - this_patch->user = (void *) pdata; + if(this_patch->user == NULL) + { + fclaw_patch_vtable_t *patch_vt = fclaw_patch_vt(glob); + /* Initialize user data */ + fclaw_patch_data_t *pdata = FCLAW_ALLOC(fclaw_patch_data_t, 1); + this_patch->user = (void *) pdata; #if 0 /* This check is dubious, since glob->domain is the old domain */ @@ -80,22 +99,28 @@ void patch_data_new(fclaw_global_t* glob, } #endif - pdata->patch_idx = this_patch_idx; - pdata->block_idx = this_block_idx; + pdata->patch_idx = this_patch_idx; + pdata->block_idx = this_block_idx; - /* create new user data */ - FCLAW_ASSERT(patch_vt->patch_new != NULL); - pdata->user_patch = patch_vt->patch_new(); + /* create new user data */ + FCLAW_ASSERT(patch_vt->patch_new != NULL); + pdata->user_patch = patch_vt->patch_new(); - ++glob->domain->count_set_patch; //this is now in cb_fclaw2d_regrid_repopulate - pdata->neighbors_set = 0; - pdata->considered_for_refinement = 0; + ++domain->count_set_patch; + pdata->neighbors_set = 0; + pdata->flags = 0; + pdata->num_owners = 1; + } + else + { + FCLAW_ASSERT(is_unpacking(this_patch)); + } } static size_t patch_data_packsize() { - return sizeof(int); + return sizeof(fclaw_patch_data_flags_t); } static @@ -103,17 +128,21 @@ void patch_data_pack(fclaw_global_t* glob, fclaw_patch_t* this_patch, void* pack_data_here) { + FCLAW_ASSERT(!is_unpacking(this_patch)); fclaw_patch_data_t *pdata = get_patch_data(this_patch); - fclaw_pack_int(pdata->considered_for_refinement, (char*) pack_data_here); + FCLAW_PACK(pdata->flags, (char*) pack_data_here); } static void patch_data_unpack(fclaw_global_t* glob, + fclaw_domain_t* domain, fclaw_patch_t* this_patch, + int blockno, int patchno, void* unpack_data_from_here) { + patch_data_new(glob,domain,this_patch,blockno,patchno); fclaw_patch_data_t *pdata = get_patch_data(this_patch); - fclaw_unpack_int((char*) unpack_data_from_here, &pdata->considered_for_refinement); + FCLAW_UNPACK((char*) unpack_data_from_here, &pdata->flags); } void fclaw_patch_reset_data(fclaw_global_t* glob, @@ -137,7 +166,8 @@ void fclaw_patch_reset_data(fclaw_global_t* glob, } void fclaw_patch_data_delete(fclaw_global_t *glob, - fclaw_patch_t *this_patch) + fclaw_domain_t *domain, + fclaw_patch_t *this_patch) { fclaw_patch_vtable_t *patch_vt = fclaw_patch_vt(glob); FCLAW_ASSERT(patch_vt->patch_delete != NULL); @@ -145,29 +175,38 @@ void fclaw_patch_data_delete(fclaw_global_t *glob, if (pdata != NULL) { - if (patch_vt->destroy_user_data) - { - patch_vt->destroy_user_data(glob,this_patch); - } + if(pdata->num_owners > 1) + { + --pdata->num_owners; + } + else + { + if (patch_vt->destroy_user_data) + { + patch_vt->destroy_user_data(glob,this_patch); + } - patch_vt->patch_delete(pdata->user_patch); - ++glob->domain->count_delete_patch; + patch_vt->patch_delete(pdata->user_patch); - FCLAW_FREE(pdata); + FCLAW_FREE(pdata); + } + + ++domain->count_delete_patch; this_patch->user = NULL; } } void fclaw_patch_build(fclaw_global_t *glob, - fclaw_patch_t *this_patch, - int blockno, - int patchno, - void *user) + fclaw_domain_t *domain, + fclaw_patch_t *this_patch, + int blockno, + int patchno, + void *user) { fclaw_patch_vtable_t *patch_vt = fclaw_patch_vt(glob); /* This is where we store 'user' (from the point of view of a p4est user) data */ - patch_data_new(glob,this_patch,blockno, patchno); + patch_data_new(glob,domain,this_patch,blockno, patchno); FCLAW_ASSERT(patch_vt->build != NULL); patch_vt->build(glob, @@ -198,17 +237,18 @@ void fclaw_patch_build(fclaw_global_t *glob, void fclaw_patch_build_from_fine(fclaw_global_t *glob, - fclaw_patch_t *fine_patches, - fclaw_patch_t *coarse_patch, - int blockno, - int coarse_patchno, - int fine0_patchno, - fclaw_build_mode_t build_mode) + fclaw_domain_t *domain, + fclaw_patch_t *fine_patches, + fclaw_patch_t *coarse_patch, + int blockno, + int coarse_patchno, + int fine0_patchno, + fclaw_build_mode_t build_mode) { fclaw_patch_vtable_t *patch_vt = fclaw_patch_vt(glob); FCLAW_ASSERT(patch_vt->build_from_fine != NULL); - patch_data_new(glob,coarse_patch,blockno, coarse_patchno); + patch_data_new(glob,domain,coarse_patch,blockno, coarse_patchno); fclaw_patch_data_t *pdata = get_patch_data(coarse_patch); pdata->block_idx = blockno; @@ -232,6 +272,22 @@ void fclaw_patch_build_from_fine(fclaw_global_t *glob, } } +void fclaw_patch_shallow_copy(struct fclaw_global *glob, + struct fclaw_domain *src_domain, + struct fclaw_patch *src_patch, + struct fclaw_domain *dst_domain, + struct fclaw_patch *dst_patch, + int blockno, + int old_patchno, + int new_patchno) +{ + fclaw_patch_data_t *pdata = get_patch_data(src_patch); + dst_patch->user = pdata; + ++pdata->num_owners; + ++dst_domain->count_set_patch; +} + + #if 0 void fclaw2d_patch_create_user_data(fclaw2d_global_t* glob, fclaw2d_patch_t* patch) @@ -641,16 +697,16 @@ void fclaw_patch_average2coarse(fclaw_global_t *glob, void fclaw_patch_interpolate2fine(fclaw_global_t* glob, fclaw_patch_t* coarse_patch, - fclaw_patch_t* fine_patches, + fclaw_patch_t* fine_patch, int this_blockno, int coarse_patchno, - int fine0_patchno) + int fine_patchno, int igrid) { fclaw_patch_vtable_t *patch_vt = fclaw_patch_vt(glob); FCLAW_ASSERT(patch_vt->interpolate2fine != NULL); - patch_vt->interpolate2fine(glob,coarse_patch,fine_patches, + patch_vt->interpolate2fine(glob,coarse_patch,fine_patch, this_blockno,coarse_patchno, - fine0_patchno); + fine_patchno, igrid); } /* ---------------------------- Ghost patches (local and remote) ---------------------- */ @@ -717,7 +773,7 @@ void fclaw_patch_remote_ghost_build(fclaw_global_t *glob, FCLAW_ASSERT(patch_vt->remote_ghost_build != NULL); - patch_data_new(glob,this_patch,blockno,patchno); + patch_data_new(glob,glob->domain,this_patch,blockno,patchno); patch_vt->remote_ghost_build(glob,this_patch,blockno, patchno,build_mode); @@ -796,31 +852,154 @@ void fclaw_patch_partition_pack(fclaw_global_t *glob, void fclaw_patch_partition_unpack(fclaw_global_t *glob, fclaw_domain_t *new_domain, - fclaw_patch_t *this_patch, - int this_block_idx, - int this_patch_idx, + fclaw_patch_t *patch, + int blockno, + int patchno, void *unpack_data_from_here) { fclaw_patch_vtable_t *patch_vt = fclaw_patch_vt(glob); + /* unpack patch data struct first */ + FCLAW_ASSERT(patch_vt->partition_packsize != NULL); + void* pdata_unpack_data_from_here = (void*) ((char*) unpack_data_from_here + patch_vt->partition_packsize(glob)); + patch_data_unpack(glob,new_domain,patch, + blockno,patchno,pdata_unpack_data_from_here); + fclaw_build_mode_t build_mode = FCLAW_BUILD_FOR_UPDATE; - fclaw_patch_build(glob,this_patch,this_block_idx, - this_patch_idx,(void*) &build_mode); + set_unpacking_flag(patch); + if(fclaw_patch_has_coarse_data(glob, patch)) + { + int igrid = fclaw_patch_childid(patch); + double width = patch->xupper - patch->xlower; + double height = patch->yupper - patch->ylower; + double depth = patch->zupper - patch->zlower; + fclaw2d_patch_t artificial_patch_2d; + fclaw3d_patch_t artificial_patch_3d; + fclaw_patch_t artificial_patch; + if(patch->refine_dim == 2) + { + artificial_patch_2d.level = patch->level-1; + artificial_patch_2d.xlower = patch->xlower; + artificial_patch_2d.ylower = patch->ylower; + artificial_patch_2d.xupper = patch->xupper; + artificial_patch_2d.yupper = patch->yupper; + artificial_patch_2d.flags = patch->d2->flags; + int lower_x_axis = !(igrid & 0x1); + if(lower_x_axis) + { + artificial_patch_2d.xupper += width; + } + else + { + artificial_patch_2d.xlower -= width; + } + int lower_y_axis = !(igrid & 0x2); + if(lower_y_axis) + { + artificial_patch_2d.yupper += height; + } + else + { + artificial_patch_2d.ylower -= height; + } + artificial_patch.level = artificial_patch_2d.level; + artificial_patch.xlower = artificial_patch_2d.xlower; + artificial_patch.ylower = artificial_patch_2d.ylower; + artificial_patch.xupper = artificial_patch_2d.xupper; + artificial_patch.yupper = artificial_patch_2d.yupper; + artificial_patch.d2 = &artificial_patch_2d; + artificial_patch.refine_dim = 2; + artificial_patch_2d.user = &artificial_patch; + } + else + { + artificial_patch_3d.level = patch->level-1; + artificial_patch_3d.xlower = patch->xlower; + artificial_patch_3d.ylower = patch->ylower; + artificial_patch_3d.zlower = patch->zlower; + artificial_patch_3d.xupper = patch->xupper; + artificial_patch_3d.yupper = patch->yupper; + artificial_patch_3d.zupper = patch->zupper; + artificial_patch_3d.flags = patch->d3->flags; + int lower_x_axis = !(igrid & 0x1); + if(lower_x_axis) + { + artificial_patch_3d.xupper += width; + } + else + { + artificial_patch_3d.xlower -= width; + } + int lower_y_axis = !(igrid & 0x2); + if(lower_y_axis) + { + artificial_patch_3d.yupper += height; + } + else + { + artificial_patch_3d.ylower -= height; + } + int lower_z_axis = !(igrid & 0x4); + if(lower_z_axis) + { + artificial_patch_3d.zupper += depth; + } + else + { + artificial_patch_3d.zlower -= depth; + } + artificial_patch.level = artificial_patch_3d.level; + artificial_patch.xlower = artificial_patch_3d.xlower; + artificial_patch.ylower = artificial_patch_3d.ylower; + artificial_patch.zlower = artificial_patch_3d.zlower; + artificial_patch.xupper = artificial_patch_3d.xupper; + artificial_patch.yupper = artificial_patch_3d.yupper; + artificial_patch.zupper = artificial_patch_3d.zupper; + artificial_patch.d3 = &artificial_patch_3d; + artificial_patch.refine_dim = 3; + artificial_patch_3d.user = &artificial_patch; + } - /* This copied q data from memory */ - FCLAW_ASSERT(patch_vt->partition_unpack != NULL); + //shallow pach data + fclaw_patch_shallow_copy(glob, + new_domain,patch, + new_domain,&artificial_patch, + blockno,patchno,-1); - patch_vt->partition_unpack(glob, /* contains old domain */ - new_domain, - this_patch, - this_block_idx, - this_patch_idx, - unpack_data_from_here); - FCLAW_ASSERT(patch_vt->partition_packsize != NULL); - void* pdata_unpack_data_from_here = (void*) ((char*) unpack_data_from_here + patch_vt->partition_packsize(glob)); - patch_data_unpack(glob,this_patch,pdata_unpack_data_from_here); + fclaw_patch_build(glob,new_domain,&artificial_patch,blockno, + -1,(void*) &build_mode); + + /* This copied q data from memory */ + FCLAW_ASSERT(patch_vt->partition_unpack != NULL); + + patch_vt->partition_unpack(glob, /* contains old domain */ + new_domain, + &artificial_patch, + blockno, + patchno, + unpack_data_from_here); + + fclaw_patch_data_delete(glob,new_domain,&artificial_patch); + } + else + { + fclaw_patch_build(glob,new_domain,patch,blockno, + patchno,(void*) &build_mode); + + /* This copied q data from memory */ + FCLAW_ASSERT(patch_vt->partition_unpack != NULL); + + patch_vt->partition_unpack(glob, /* contains old domain */ + new_domain, + patch, + blockno, + patchno, + unpack_data_from_here); + } + + clear_unpacking_flag(patch); } /* ----------------------------------- Restart --------------------------------------- */ @@ -1242,22 +1421,21 @@ void fclaw_patch_considered_for_refinement_set(struct fclaw_global *glob, struct fclaw_patch* patch) { fclaw_patch_data_t *pdata = get_patch_data(patch); - //FCLAW_ASSERT(pdata->considered_for_refinement == 0); - pdata->considered_for_refinement = 1; + pdata->flags |= FCLAW_PATCH_DATA_CONSIDERED_FOR_REFINEMENT; } void fclaw_patch_considered_for_refinement_clear(struct fclaw_global *glob, struct fclaw_patch* patch) { fclaw_patch_data_t *pdata = get_patch_data(patch); - pdata->considered_for_refinement = 0; + pdata->flags &= ~FCLAW_PATCH_DATA_CONSIDERED_FOR_REFINEMENT; } int fclaw_patch_considered_for_refinement(struct fclaw_global *glob, struct fclaw_patch* patch) { fclaw_patch_data_t *pdata = get_patch_data(patch); - return pdata->considered_for_refinement; + return pdata->flags & FCLAW_PATCH_DATA_CONSIDERED_FOR_REFINEMENT; } static @@ -1266,9 +1444,9 @@ void considred_for_refinement_cb(fclaw_domain_t *domain, int blockno, int patchno, void *user) { - fclaw_patch_data_t *pdata = get_patch_data(patch); - int *all_patches_considered = (int*) user; - if (!pdata->considered_for_refinement) + fclaw_global_iterate_t *s = (fclaw_global_iterate_t*) user; + int *all_patches_considered = (int*) s->user; + if (!fclaw_patch_considered_for_refinement(s->glob,patch)) { *all_patches_considered = 0; } @@ -1277,7 +1455,7 @@ void considred_for_refinement_cb(fclaw_domain_t *domain, int fclaw_patch_all_considered_for_refinement(struct fclaw_global *glob) { int local_all_patches_considered = 1; - fclaw_domain_iterate_patches(glob->domain, considred_for_refinement_cb, &local_all_patches_considered); + fclaw_global_iterate_patches(glob, considred_for_refinement_cb, &local_all_patches_considered); int all_patches_considered; sc_MPI_Allreduce(&local_all_patches_considered, &all_patches_considered, 1, sc_MPI_INT, sc_MPI_LAND, glob->mpicomm); return all_patches_considered; @@ -1289,11 +1467,32 @@ void clear_considred_for_refinement_cb(fclaw_domain_t *domain, int blockno, int patchno, void *user) { - fclaw_patch_data_t *pdata = get_patch_data(patch); - pdata->considered_for_refinement = 0; + fclaw_global_t *glob = ((fclaw_global_iterate_t*) user)->glob; + fclaw_patch_considered_for_refinement_clear(glob, patch); } void fclaw_patch_clear_all_considered_for_refinement(struct fclaw_global *glob) { - fclaw_domain_iterate_patches(glob->domain, clear_considred_for_refinement_cb, NULL); + fclaw_global_iterate_patches(glob, clear_considred_for_refinement_cb, NULL); +} + +void fclaw_patch_has_coarse_data_set(struct fclaw_global *glob, + struct fclaw_patch* patch) +{ + fclaw_patch_data_t *pdata = get_patch_data(patch); + pdata->flags |= FCLAW_PATCH_DATA_HAS_COARSE_DATA; +} + +void fclaw_patch_has_coarse_data_clear(struct fclaw_global *glob, + struct fclaw_patch* patch) +{ + fclaw_patch_data_t *pdata = get_patch_data(patch); + pdata->flags &= ~FCLAW_PATCH_DATA_HAS_COARSE_DATA; +} + +int fclaw_patch_has_coarse_data(struct fclaw_global *glob, + struct fclaw_patch* patch) +{ + fclaw_patch_data_t *pdata = get_patch_data(patch); + return pdata->flags & FCLAW_PATCH_DATA_HAS_COARSE_DATA; } \ No newline at end of file diff --git a/src/fclaw_patch.h b/src/fclaw_patch.h index 72fbbc32c..f84fd5ae9 100644 --- a/src/fclaw_patch.h +++ b/src/fclaw_patch.h @@ -65,6 +65,16 @@ typedef enum } fclaw_build_mode_t; +typedef enum +{ + /** Patch considered for refinement */ + FCLAW_PATCH_DATA_CONSIDERED_FOR_REFINEMENT = 0x1, + /** Patch has coarse data and it's family needs to be refined */ + FCLAW_PATCH_DATA_HAS_COARSE_DATA = 0x2, + /** Patch is currently being unpacked during partitioning */ + FCLAW_PATCH_DATA_UNPACKING = 0x4, + +} fclaw_patch_data_flags_t; /** * @brief Structure for user patch data @@ -100,8 +110,10 @@ struct fclaw_patch_data /** Block index */ int block_idx; - /** True if patch has been considered for refinement */ - int considered_for_refinement; + /** Boolean flags */ + int flags; + /** Number of fclaw_patch_t objects that have this patch set as patch->user */ + int num_owners; /** User defined patch structure */ void *user_patch; @@ -212,30 +224,35 @@ struct fclaw_region; * @brief Deallocate the user data pointer for a patch * * @param[in] glob the global context + * @param[in] domain the domain * @param[in,out] patch the patch context, user data pointer is set to NULL on return */ void fclaw_patch_data_delete(struct fclaw_global *glob, - struct fclaw_patch *patch); + struct fclaw_domain *domain, + struct fclaw_patch *patch); /** * @brief Construct a new patch object * * @param[in] glob the global context + * @param[in] domain the domain * @param[in,out] this_patch the patch context * @param[in] blockno the block number * @param[in] patchno the patch number * @param[in,out] user user data pointer */ void fclaw_patch_build(struct fclaw_global *glob, - struct fclaw_patch *this_patch, - int blockno, - int patchno, - void *user); + struct fclaw_domain *domain, + struct fclaw_patch *this_patch, + int blockno, + int patchno, + void *user); /** * @brief Construct a new patch object from a set of fine patches * * @param[in] glob the global context + * @param[in] domain the domain * @param[in] fine_patches the fine patch contexts * @param[in,out] coarse_patches the coarse patch context * @param[in] blockno the block number @@ -244,13 +261,34 @@ void fclaw_patch_build(struct fclaw_global *glob, * @param[in] build_mode the build mode */ void fclaw_patch_build_from_fine(struct fclaw_global *glob, - struct fclaw_patch *fine_patches, - struct fclaw_patch *coarse_patch, - int blockno, - int coarse_patchno, - int fine0_patchno, - fclaw_build_mode_t build_mode); + struct fclaw_domain *domain, + struct fclaw_patch *fine_patches, + struct fclaw_patch *coarse_patch, + int blockno, + int coarse_patchno, + int fine0_patchno, + fclaw_build_mode_t build_mode); +/** + * @brief Create a shallow copy of a patch + * + * @param[in] glob the global context + * @param[in] src_domain the source domain + * @param[in] src_patch the source patch + * @param[in] dst_domain the destination domain + * @param[in,out] dst_patch the destination patch + * @param[in] blockno the block number + * @param[in] src_patchno the source patch number + * @param[in] dst_patchno the destination patch number + */ +void fclaw_patch_shallow_copy(struct fclaw_global *glob, + struct fclaw_domain *src_domain, + struct fclaw_patch *src_patch, + struct fclaw_domain *dst_domain, + struct fclaw_patch *dst_patch, + int blockno, + int old_patchno, + int new_patchno); ///@} /* ------------------------------------------------------------------------------------ */ @@ -697,13 +735,14 @@ int fclaw_patch_intersects_region(struct fclaw_global *glob, * @param[in,out] fine_patches the fine patch contexts * @param[in] blockno the block number * @param[in] coarse_patchno the patch number of the coarse patch - * @param[in] fine0_patchno the patch number of the first fine patch + * @param[in] fine_patchno the patch number of the fine patch + * @param[in] igrid the index of the fine patch in the child array */ void fclaw_patch_interpolate2fine(struct fclaw_global *glob, struct fclaw_patch* coarse_patch, - struct fclaw_patch* fine_patches, + struct fclaw_patch* fine_patch, int this_blockno, int coarse_patchno, - int fine0_patchno); + int fine_patchno, int igrid); /** * @brief Averages from a set of fine patches to a coarse patch @@ -1462,13 +1501,14 @@ typedef int (*fclaw_patch_intersects_region_t)(struct fclaw_global *glob, * @param[in,out] fine_patches the fine patch contexts * @param[in] blockno the block number * @param[in] coarse_patchno the patch number of the coarse patch - * @param[in] fine_patchno the patch number of the first fine patch + * @param[in] fine_patchno the patch number of the fine patch + * @param[in] igrid the index of the fine patch in the child array */ typedef void (*fclaw_patch_interpolate2fine_t)(struct fclaw_global *glob, struct fclaw_patch *coarse_patch, - struct fclaw_patch* fine_patches, + struct fclaw_patch* fine_patch, int blockno, int coarse_patchno, - int fine_patchno); + int fine_patchno, int igrid); /** * @brief Averages from a set of fine patches to a coarse patch * @@ -2162,6 +2202,12 @@ void fclaw_patch_set_block_corner_count(struct fclaw_global *glob, struct fclaw_patch* this_patch, int icorner, int block_corner_count); +///@} +/* ------------------------------------------------------------------------------------ */ +/// @name Patch Flags +/* ------------------------------------------------------------------------------------ */ +///@{ + /** * @brief Set that this patch has been considered for refinement * @@ -2204,6 +2250,33 @@ int fclaw_patch_all_considered_for_refinement(struct fclaw_global *glob); */ void fclaw_patch_clear_all_considered_for_refinement(struct fclaw_global *glob); +/** + * @brief Set that this patch has coarse data + * + * @param glob the global context + * @param patch the patch context + */ +void fclaw_patch_has_coarse_data_set(struct fclaw_global *glob, + struct fclaw_patch* patch); +/** + * @brief Clear the flag that this patch has coarse data + * + * @param glob the global context + * @param patch the patch context + */ +void fclaw_patch_has_coarse_data_clear(struct fclaw_global *glob, + struct fclaw_patch* patch); + +/** + * @brief Returns true if a patch has coarse data + * + * @param glob the global context + * @param patch the patch context + * @return int true if a patch has coarse data + */ +int fclaw_patch_has_coarse_data(struct fclaw_global *glob, + struct fclaw_patch* patch); + ///@} #ifdef __cplusplus diff --git a/src/fclaw_regrid.c b/src/fclaw_regrid.c index 6a008c489..676604790 100644 --- a/src/fclaw_regrid.c +++ b/src/fclaw_regrid.c @@ -40,6 +40,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include +#include +#include + /* This is also called from fclaw2d_initialize, so is not made static */ void cb_fclaw_regrid_tag4refinement(fclaw_domain_t *domain, fclaw_patch_t *this_patch, @@ -119,6 +122,201 @@ void cb_regrid_tag4coarsening(fclaw_domain_t *domain, } } +static +void refine_patch(fclaw_global_t *glob, + fclaw_domain_t *old_domain, + fclaw_domain_t *new_domain, + fclaw_patch_t *coarse_patch, + fclaw_patch_t *fine_siblings, + int blockno, + int coarse_patchno, + int fine0_patchno, + int domain_init) +{ + fclaw_options_t *fclaw_opt = fclaw_get_options(glob); + + for (int i = 0; i < fclaw_domain_num_siblings(old_domain); i++) + { + fclaw_patch_t *fine_patch = &fine_siblings[i]; + int fine_patchno = fine0_patchno + i; + + /* set the data */ + if (fclaw_opt->partition_mode != FCLAW_PARTITION_MODE_REFINE_AFTER) + { + fclaw_build_mode_t build_mode = FCLAW_BUILD_FOR_UPDATE; + fclaw_patch_build(glob,new_domain,fine_patch,blockno, + fine_patchno,(void*) &build_mode); + + if(domain_init) + { + fclaw_patch_initialize(glob,fine_patch,blockno,fine_patchno); + } + else + { + fclaw_patch_interpolate2fine(glob,coarse_patch,fine_patch, + blockno,coarse_patchno,fine_patchno,i); + } + } + else + { + fclaw_patch_shallow_copy(glob,old_domain,coarse_patch,new_domain, + fine_patch,blockno,coarse_patchno,fine_patchno); + fclaw_patch_has_coarse_data_set(glob, fine_patch); + if(!domain_init) + { + } + } + + /* don't try to refine this patch in the next round of refinement */ + fclaw_patch_considered_for_refinement_set(glob, fine_patch); + + if(fclaw_opt->partition_mode == FCLAW_PARTITION_MODE_REFINE_AFTER) + { + fclaw_patch_has_coarse_data_set(glob, fine_patch); + } + } + + /* used to pass in old_domain */ + fclaw_patch_data_delete(glob,old_domain,coarse_patch); + +} + +static +void cb_refine_after_partition(fclaw_domain_t *domain, + fclaw_patch_t *patch, + int blockno, + int patchno, + void *user) +{ + fclaw_global_iterate_t *g = (fclaw_global_iterate_t*) user; + int domain_init = *((int*) g->user); + + if(fclaw_patch_has_coarse_data(g->glob, patch)) + { + if(domain_init) + { + fclaw_patch_data_delete(g->glob,domain,patch); + fclaw_build_mode_t build_mode = FCLAW_BUILD_FOR_UPDATE; + fclaw_patch_build(g->glob,domain,patch,blockno, + patchno,(void*) &build_mode); + fclaw_patch_initialize(g->glob,patch,blockno,patchno); + } + else + { + int igrid = fclaw_patch_childid(patch); + double width = patch->xupper - patch->xlower; + double height = patch->yupper - patch->ylower; + double depth = patch->zupper - patch->zlower; + fclaw2d_patch_t artificial_patch_2d; + fclaw3d_patch_t artificial_patch_3d; + fclaw_patch_t artificial_patch; + if(patch->refine_dim == 2) + { + artificial_patch_2d.level = patch->level-1; + artificial_patch_2d.xlower = patch->xlower; + artificial_patch_2d.ylower = patch->ylower; + artificial_patch_2d.xupper = patch->xupper; + artificial_patch_2d.yupper = patch->yupper; + artificial_patch_2d.flags = patch->d2->flags; + int lower_x_axis = !(igrid & 0x1); + if(lower_x_axis) + { + artificial_patch_2d.xupper += width; + } + else + { + artificial_patch_2d.xlower -= width; + } + int lower_y_axis = !(igrid & 0x2); + if(lower_y_axis) + { + artificial_patch_2d.yupper += height; + } + else + { + artificial_patch_2d.ylower -= height; + } + artificial_patch.level = artificial_patch_2d.level; + artificial_patch.xlower = artificial_patch_2d.xlower; + artificial_patch.ylower = artificial_patch_2d.ylower; + artificial_patch.xupper = artificial_patch_2d.xupper; + artificial_patch.yupper = artificial_patch_2d.yupper; + artificial_patch.d2 = &artificial_patch_2d; + artificial_patch.refine_dim = 2; + artificial_patch_2d.user = &artificial_patch; + } + else + { + artificial_patch_3d.level = patch->level-1; + artificial_patch_3d.xlower = patch->xlower; + artificial_patch_3d.ylower = patch->ylower; + artificial_patch_3d.zlower = patch->zlower; + artificial_patch_3d.xupper = patch->xupper; + artificial_patch_3d.yupper = patch->yupper; + artificial_patch_3d.zupper = patch->zupper; + artificial_patch_3d.flags = patch->d3->flags; + int lower_x_axis = !(igrid & 0x1); + if(lower_x_axis) + { + artificial_patch_3d.xupper += width; + } + else + { + artificial_patch_3d.xlower -= width; + } + int lower_y_axis = !(igrid & 0x2); + if(lower_y_axis) + { + artificial_patch_3d.yupper += height; + } + else + { + artificial_patch_3d.ylower -= height; + } + int lower_z_axis = !(igrid & 0x4); + if(lower_z_axis) + { + artificial_patch_3d.zupper += depth; + } + else + { + artificial_patch_3d.zlower -= depth; + } + artificial_patch.level = artificial_patch_3d.level; + artificial_patch.xlower = artificial_patch_3d.xlower; + artificial_patch.ylower = artificial_patch_3d.ylower; + artificial_patch.zlower = artificial_patch_3d.zlower; + artificial_patch.xupper = artificial_patch_3d.xupper; + artificial_patch.yupper = artificial_patch_3d.yupper; + artificial_patch.zupper = artificial_patch_3d.zupper; + artificial_patch.d3 = &artificial_patch_3d; + artificial_patch.refine_dim = 3; + artificial_patch_3d.user = &artificial_patch; + } + + + + fclaw_patch_shallow_copy(g->glob,domain,patch,domain,&artificial_patch, + blockno,patchno,-1); + fclaw_patch_data_delete(g->glob,domain,patch); + + fclaw_build_mode_t build_mode = FCLAW_BUILD_FOR_UPDATE; + fclaw_patch_build(g->glob,domain,patch,blockno, + patchno,(void*) &build_mode); + + fclaw_patch_interpolate2fine(g->glob,&artificial_patch,patch, + blockno,-1,patchno,igrid); + + fclaw_patch_data_delete(g->glob,domain, &artificial_patch); + + fclaw_patch_considered_for_refinement_set(g->glob, patch); + } + + fclaw_patch_has_coarse_data_clear(g->glob, patch); + } + + +} /* ---------------------------------------------------------------- Public interface @@ -143,46 +341,13 @@ void cb_fclaw_regrid_repopulate(fclaw_domain_t * old_domain, { FCLAW_ASSERT(0 <= blockno && blockno < new_domain->num_blocks); FCLAW_ASSERT(0 <= new_patchno && new_patchno < new_domain->local_num_patches); - new_patch->user = old_patch->user; - old_patch->user = NULL; - ++old_domain->count_delete_patch; - ++new_domain->count_set_patch; + fclaw_patch_shallow_copy(g->glob, old_domain, old_patch, new_domain, new_patch, + blockno, old_patchno, new_patchno); } else if (newsize == FCLAW_PATCH_HALFSIZE) { - fclaw_patch_t *fine_siblings = new_patch; - fclaw_patch_t *coarse_patch = old_patch; - - int i; - for (i = 0; i < fclaw_domain_num_siblings(old_domain); i++) - { - fclaw_patch_t *fine_patch = &fine_siblings[i]; - int fine_patchno = new_patchno + i; - /* Reason for the following two lines: the glob contains the old domain which is incremented in ddata_old - but we really want to increment the new domain. This will be fixed! */ - --old_domain->count_set_patch; - ++new_domain->count_set_patch; - - fclaw_patch_build(g->glob,fine_patch,blockno, - fine_patchno,(void*) &build_mode); - if (domain_init) - { - fclaw_patch_initialize(g->glob,fine_patch,blockno,fine_patchno);//new_domain - } - /* don't try to refine this patch in the next round of refinement */ - fclaw_patch_considered_for_refinement_set(g->glob, fine_patch); - } - - if (!domain_init) - { - int coarse_patchno = old_patchno; - int fine_patchno = new_patchno; - - fclaw_patch_interpolate2fine(g->glob,coarse_patch,fine_siblings, - blockno,coarse_patchno,fine_patchno);//new_domain - } - /* used to pass in old_domain */ - fclaw_patch_data_delete(g->glob,coarse_patch); + refine_patch(g->glob, old_domain, new_domain, old_patch, new_patch, + blockno, old_patchno, new_patchno, domain_init); } else if (newsize == FCLAW_PATCH_DOUBLESIZE) { @@ -203,21 +368,16 @@ void cb_fclaw_regrid_repopulate(fclaw_domain_t * old_domain, fclaw_patch_t *coarse_patch = new_patch; int coarse_patchno = new_patchno; - /* Reason for the following two lines: the glob contains the old domain which is incremented in ddata_old - but we really want to increment the new domain. This will be fixed! */ - --old_domain->count_set_patch; - ++new_domain->count_set_patch; - if (domain_init) { - fclaw_patch_build(g->glob,coarse_patch,blockno, + fclaw_patch_build(g->glob,new_domain,coarse_patch,blockno, coarse_patchno,(void*) &build_mode); fclaw_patch_initialize(g->glob,coarse_patch,blockno,coarse_patchno); } else { /* Area (and possibly other things) should be averaged to coarse grid. */ - fclaw_patch_build_from_fine(g->glob,fine_siblings,coarse_patch, + fclaw_patch_build_from_fine(g->glob,new_domain,fine_siblings,coarse_patch, blockno,coarse_patchno,fine_patchno, build_mode); /* Average the solution. Does this need to be customizable? */ @@ -230,7 +390,7 @@ void cb_fclaw_regrid_repopulate(fclaw_domain_t * old_domain, { fclaw_patch_t* fine_patch = &fine_siblings[i]; /* used to pass in old_domain */ - fclaw_patch_data_delete(g->glob,fine_patch); + fclaw_patch_data_delete(g->glob,old_domain,fine_patch); } /* don't try to refine this patch in the next round of refinement */ fclaw_patch_considered_for_refinement_set(g->glob, coarse_patch); @@ -243,6 +403,47 @@ void cb_fclaw_regrid_repopulate(fclaw_domain_t * old_domain, fclaw_patch_neighbors_reset(new_patch); } +void fclaw_regrid_process_new_refinement(fclaw_global_t *glob, + fclaw_domain_t **domain, + fclaw_domain_t *new_domain, + int domain_init, + fclaw_timer_names_t timer) +{ + fclaw_options_t *fclaw_opt = fclaw_get_options(glob); + + /* Average to new coarse grids and interpolate to new fine grids */ + fclaw_timer_start (&glob->timers[FCLAW_TIMER_REGRID_BUILD]); + fclaw_global_iterate_adapted(glob, new_domain, + cb_fclaw_regrid_repopulate, + (void *) &domain_init); + fclaw_timer_stop (&glob->timers[FCLAW_TIMER_REGRID_BUILD]); + + /* free memory associated with old domain */ + fclaw_domain_reset(glob); + *domain = new_domain; + new_domain = NULL; + + /* Repartition for load balancing. Second arg (mode) for vtk output */ + fclaw_partition_domain(glob,timer); + + /* refine after partition (if_needed) */ + if(fclaw_opt->partition_mode == FCLAW_PARTITION_MODE_REFINE_AFTER) + { + fclaw_global_iterate_patches(glob, cb_refine_after_partition, + (void *) &domain_init); + } + + /* Set up ghost patches. Communication happens for indirect ghost exchanges. */ + + + /* This includes timers for building patches and (exclusive) communication */ + fclaw_exchange_setup(glob,timer); + + /* Get new neighbor information. This is used to short circuit + ghost filling procedures in some cases */ + fclaw_regrid_set_neighbor_types(glob); +} + /* ---------------------------------------------------------------- Public interface -------------------------------------------------------------- */ @@ -316,34 +517,15 @@ void fclaw_regrid(fclaw_global_t *glob) { fclaw_global_infof(" -- Have new refinement\n"); - /* Average to new coarse grids and interpolate to new fine grids */ - fclaw_timer_start (&glob->timers[FCLAW_TIMER_REGRID_BUILD]); - fclaw_global_iterate_adapted(glob, new_domain, - cb_fclaw_regrid_repopulate, - (void *) &domain_init); - fclaw_timer_stop (&glob->timers[FCLAW_TIMER_REGRID_BUILD]); - - /* free memory associated with old domain */ - fclaw_domain_reset(glob); - *domain = new_domain; - new_domain = NULL; - - /* Repartition for load balancing. Second arg (mode) for vtk output */ - fclaw_partition_domain(glob,FCLAW_TIMER_REGRID); - - /* Set up ghost patches. Communication happens for indirect ghost exchanges. */ - - - /* This includes timers for building patches and (exclusive) communication */ - fclaw_exchange_setup(glob,FCLAW_TIMER_REGRID); - - /* Get new neighbor information. This is used to short circuit - ghost filling procedures in some cases */ - fclaw_regrid_set_neighbor_types(glob); + fclaw_regrid_process_new_refinement(glob, + domain, + new_domain, + domain_init, + FCLAW_TIMER_REGRID); /* Update ghost cells. This is needed because we have new coarse or fine - patches without valid ghost cells. Time_interp = 0, since we only - only regrid when all levels are time synchronized. */ + patches without valid ghost cells. Time_interp = 0, since we only + only regrid when all levels are time synchronized. */ int minlevel = (*domain)->global_minlevel; int maxlevel = (*domain)->global_maxlevel; int time_interp = 0; diff --git a/src/fclaw_regrid.h b/src/fclaw_regrid.h index 2474865f1..1a217d817 100644 --- a/src/fclaw_regrid.h +++ b/src/fclaw_regrid.h @@ -27,6 +27,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define FCLAW_REGRID_H #include /* Needed to define fclaw2d_patch_relation_t */ +#include #ifdef __cplusplus extern "C" @@ -92,8 +93,23 @@ void fclaw_regrid(struct fclaw_global *glob); void fclaw_regrid_set_neighbor_types(struct fclaw_global *glob); - - +/** + * @brief Process new refinement. + * + * This is to be called with the new domain that fclaw_domain_adapt returns. + * THis will perform any coarsening and refinment and will repartition the domain. + * + * @param glob the global context + * @param domain the old domain + * @param new_domain the new domain returned by fclaw_domain_adapt + * @param domain_init true if this is the first time the domain is being initialized + * @param timer the timer to use + */ +void fclaw_regrid_process_new_refinement(struct fclaw_global *glob, + fclaw_domain_t **domain, + fclaw_domain_t *new_domain, + int domain_init, + fclaw_timer_names_t timer); #ifdef __cplusplus #if 0 diff --git a/src/fclaw_restart.c b/src/fclaw_restart.c index f933a334c..d30e73bd0 100644 --- a/src/fclaw_restart.c +++ b/src/fclaw_restart.c @@ -398,7 +398,7 @@ set_patches(fclaw_domain_t * domain, fclaw_patch_t * patch, int blockno, int pat if(user_data->pointerno == 0) { - fclaw_patch_build(user_data->glob, patch, blockno, patchno,(void*) &build_mode); + fclaw_patch_build(user_data->glob, domain, patch, blockno, patchno,(void*) &build_mode); } void* data = fclaw_patch_checkpoint_get_pointer(user_data->glob, patch, blockno, patchno, user_data->pointerno); diff --git a/src/fclaw_wrap.h b/src/fclaw_wrap.h index 9186dd5d2..850bfdee8 100644 --- a/src/fclaw_wrap.h +++ b/src/fclaw_wrap.h @@ -112,6 +112,52 @@ typedef struct fclaw_match_wrap_user void *user; } fclaw_match_wrap_user_t; +/** + * @brief Callback wrapper struct for pack callbacks. + * + * This allows callbacks with dimension independent types to be + * called from dimension dependent code. + * + * This type should be passed in as the user pointer alongside + * @ref fclaw2d_pack_wrap_cb or @ref fclaw3d_pack_wrap_cb + * to the function that takes a dimensioned callback. + * + * @ref fclaw2d_pack_wrap_cb or @ref fclaw3d_pack_wrap_cb + * will then call the dimension independent callback specified in the struct, + * passing the user pointer specified in the struct. + * + */ +typedef struct fclaw_pack_wrap_user +{ + /** Dimension independent pack callback to call */ + fclaw_pack_callback_t pcb; + /** User pointer to pass to dimension independent callback */ + void *user; +} fclaw_pack_wrap_user_t; + +/** + * @brief Callback wrapper struct for unpack callbacks. + * + * This allows callbacks with dimension independent types to be + * called from dimension dependent code. + * + * This type should be passed in as the user pointer alongside + * @ref fclaw2d_unpack_wrap_cb or @ref fclaw3d_unpack_wrap_cb + * to the function that takes a dimensioned callback. + * + * @ref fclaw2d_unpack_wrap_cb or @ref fclaw3d_unpack_wrap_cb + * will then call the dimension independent callback specified in the struct, + * passing the user pointer specified in the struct. + * + */ +typedef struct fclaw_unpack_wrap_user +{ + /** Dimension independent unpack callback to call */ + fclaw_unpack_callback_t ucb; + /** User pointer to pass to dimension independent callback */ + void *user; +} fclaw_unpack_wrap_user_t; + /** * @brief Callback wrapper struct for match callbacks. * diff --git a/src/forestclaw.c b/src/forestclaw.c index a42f42129..c1faa1bcc 100644 --- a/src/forestclaw.c +++ b/src/forestclaw.c @@ -837,6 +837,130 @@ void fclaw_domain_iterate_partitioned(fclaw_domain_t *old_domain, fclaw_domain_t } } +struct fclaw_domain_partition +{ + + int refine_dim; + fclaw2d_domain_partition_t *d2; + fclaw3d_domain_partition_t *d3; + fclaw_pack_wrap_user_t wrap; +}; + +fclaw_domain_partition_t + * fclaw_domain_iterate_pack (fclaw_domain_t * domain, + size_t data_size, + fclaw_pack_callback_t patch_pack, + void *user) +{ + fclaw_domain_partition_t *partition = FCLAW_ALLOC(fclaw_domain_partition_t, 1); + partition->wrap.pcb = patch_pack; + partition->wrap.user = user; + if(domain->refine_dim == 2) + { + partition->refine_dim = 2; + partition->d2 = fclaw2d_domain_iterate_pack(domain->d2, + data_size, + fclaw2d_pack_wrap_cb, + &partition->wrap); + partition->d3 = NULL; + } + else if (domain->refine_dim == 3) + { + partition->refine_dim = 3; + partition->d2 = NULL; + partition->d3 = fclaw3d_domain_iterate_pack(domain->d3, + data_size, + fclaw3d_pack_wrap_cb, + &partition->wrap); + } + else + { + SC_ABORT_NOT_REACHED(); + } + + return partition; +} + +void fclaw_domain_iterate_transfer (fclaw_domain_t * old_domain, + fclaw_domain_t * new_domain, + fclaw_transfer_callback_t + patch_transfer, void *user) +{ + FCLAW_ASSERT(old_domain->refine_dim == new_domain->refine_dim); + + fclaw_transfer_wrap_user_t wrap; + wrap.tcb = patch_transfer; + wrap.user = user; + + if(old_domain->refine_dim == 2) + { + fclaw2d_domain_iterate_transfer(old_domain->d2, + new_domain->d2, + fclaw2d_transfer_wrap_cb, + &wrap); + } + else if(old_domain->refine_dim == 3) + { + fclaw3d_domain_iterate_transfer(old_domain->d3, + new_domain->d3, + fclaw3d_transfer_wrap_cb, + &wrap); + } + else + { + SC_ABORT_NOT_REACHED(); + } + +} + +void fclaw_domain_iterate_unpack (fclaw_domain_t * domain, + fclaw_domain_partition_t * partition, + fclaw_unpack_callback_t patch_unpack, + void *user) +{ + FCLAW_ASSERT(domain->refine_dim == partition->refine_dim); + + fclaw_unpack_wrap_user_t wrap; + wrap.ucb = patch_unpack; + wrap.user = user; + + if(domain->refine_dim == 2) + { + fclaw2d_domain_iterate_unpack(domain->d2, + partition->d2, + fclaw2d_unpack_wrap_cb, + &wrap); + } + else if (domain->refine_dim == 3) + { + fclaw3d_domain_iterate_unpack(domain->d3, + partition->d3, + fclaw3d_unpack_wrap_cb, + &wrap); + } + else + { + SC_ABORT_NOT_REACHED(); + } +} + +void fclaw_domain_partition_free (fclaw_domain_partition_t * partition) +{ + if(partition->refine_dim == 2) + { + fclaw2d_domain_partition_free(partition->d2); + } + else if (partition->refine_dim == 3) + { + fclaw3d_domain_partition_free(partition->d3); + } + else + { + SC_ABORT_NOT_REACHED(); + } + FCLAW_FREE(partition); +} + void fclaw_domain_free_after_partition(fclaw_domain_t *domain, void ***patch_data) { if(domain->refine_dim == 2) diff --git a/src/forestclaw.h b/src/forestclaw.h index 56464519e..cd63bc0f8 100644 --- a/src/forestclaw.h +++ b/src/forestclaw.h @@ -1001,6 +1001,112 @@ void fclaw_domain_iterate_partitioned (fclaw_domain_t * old_domain, fclaw_transfer_callback_t tcb, void *user); +/** Data structure for storing allocated data for partition transfer. */ +typedef struct fclaw_domain_partition fclaw_domain_partition_t; + +/** Callback to pack patch data after partitioning. + * The function \ref fclaw_domain_iterate_pack traverses every local patch in + * the old partition. If that patch has to be packed for the new partition, it + * invokes this callback. + * \param [in,out] domain Domain before partition. + * \param [in,out] patch Patch from the domain. Its user data should be + packed into \b pack_data_here. + * \param [in] blockno Number of the current block. + * \param [in] patchno Number of the patch. + * \param [in] pack_data_here Address of storage space for the patch's user data. + * \param [in,out] user Pointer passed to \ref fclaw_domain_iterate_pack. + */ +typedef void (*fclaw_pack_callback_t) (fclaw_domain_t * domain, + fclaw_patch_t * patch, int blockno, + int patchno, void *pack_data_here, + void *user); + +/** Start asynchronous transfer of patch data after partition. + * This function iterates over the local patches of old partition and determines + * patches that have to be packed by use of \b patch_pack based on the options + * skip_local, skip_refined and partition_for_coarsening (see + * \ref fclaw_domain_set_partitioning). Then, the patches are packed and sent. + * This function must be followed by a call to \ref fclaw_domain_iterate_unpack. + * \param [in,out] domain The domain before partitioning. + * \param [in] data_size The number of bytes of user data that has to be + * packed and send per patch. + * \param [in] patch_pack Callback function to pack patch user data into + * a provided storage address. + * \param [in] user This pointer is passed to the callback. + * \return Allocated and initialized structure containing + * internal information for patch data transfer. + */ +fclaw_domain_partition_t + * fclaw_domain_iterate_pack (fclaw_domain_t * domain, + size_t data_size, + fclaw_pack_callback_t patch_pack, + void *user); + +/** Transfer data of patches still local after partition. + * This function iterates over all pairs of local patches in the domain before + * and after partitioning and invokes the transfer of their user-data via + * \b patch_transfer. + * It may be called before, between or after \ref fclaw_domain_iterate_pack + * and \ref fclaw_domain_iterate_unpack. It is recommended to call it between + * packing and unpacking, to overlap communication with computation. + * \param [in,out] old_domain The domain before partitioning. + * \param [in,out] new_domain The domain after partitioning. + * \param [in] patch_transfer Callback function to transfer user data between + * local old and new patch, e.g. by a copy + * or reassignment. + * \param [in] user This pointer is passed to the callback. + */ +void fclaw_domain_iterate_transfer (fclaw_domain_t * old_domain, + fclaw_domain_t * new_domain, + fclaw_transfer_callback_t + patch_transfer, void *user); + +/** Callback to unpack patch data after partitioning. + * The function \ref fclaw_domain_iterate_unpack traverses every local patch, + * that was not local before partitioning and invokes this callback. + * \param [in,out] domain Domain after partition. + * \param [in,out] patch Patch from the domain. Its user data should be + unpacked from \b unpack_data_from_here. + * \param [in] blockno Number of the current block. + * \param [in] patchno Number of the patch. + * \param [in] unpack_data_from_here Address of storage space that contains the + packed user data of the patch. + * \param [in,out] user Pointer passed to \ref fclaw_domain_iterate_unpack. + */ +typedef void (*fclaw_unpack_callback_t) (fclaw_domain_t * domain, + fclaw_patch_t * patch, + int blockno, int patchno, + void *unpack_data_from_here, + void *user); + +/** Complete asynchronous transfer of patch data after partition. + * This function iterates over all local patches of the new partition and + * determines patches that have to be unpacked by use of \b patch_unpack based + * on the options skip_local, skip_refined and partition_for_coarsening (see + * \ref fclaw_domain_set_partitioning). + * It must be preceded by a call to \ref fclaw_domain_iterate_unpack and + * provided with the corresponding \ref fclaw_domain_partition_t \b p. + * \param [in,out] domain The domain after partitioning. + * \param [in,out] p Structure containing internal information for + * the patch data transfer. + * \param [in] patch_pack Callback function to unpack patch user data from + * a provided storage address. + * \param [in] user This pointer is passed to the callback. + * \return Allocated and initialized structure containing + * internal information for patch data transfer. + */ +void fclaw_domain_iterate_unpack (fclaw_domain_t * domain, + fclaw_domain_partition_t * p, + fclaw_unpack_callback_t patch_unpack, + void *user); + +/** Free buffers used in transfering patch data during partition. + * \param [in] p Structure containing internal information for + * patch data transfer. Will be deallocated during + * this function call. + */ +void fclaw_domain_partition_free (fclaw_domain_partition_t * p); + /** Free buffers that were used in transfering data during partition. * \param [in,out] domain The memory lives inside this domain. * \param [in,out] patch_data Address of an array of void pointers to free. diff --git a/src/patches/clawpatch/fclaw_clawpatch.cpp b/src/patches/clawpatch/fclaw_clawpatch.cpp index bbddfa007..984390009 100644 --- a/src/patches/clawpatch/fclaw_clawpatch.cpp +++ b/src/patches/clawpatch/fclaw_clawpatch.cpp @@ -1154,55 +1154,50 @@ int clawpatch_tag4coarsening(fclaw_global_t *glob, static void clawpatch_interpolate2fine(fclaw_global_t* glob, fclaw_patch_t *coarse_patch, - fclaw_patch_t* fine_patches, + fclaw_patch_t* fine_patch, int this_blockno, int coarse_patchno, - int fine0_patchno) + int fine_patchno, int igrid) { const fclaw_clawpatch_options_t *clawpatch_opt = fclaw_clawpatch_get_options(glob); int mbc = clawpatch_opt->mbc; int meqn = clawpatch_opt->meqn; - /* Loop over four siblings (z-ordering) */ - for (int igrid = 0; igrid < fclaw_domain_num_siblings(glob->domain); igrid++) - { - fclaw_patch_t *fine_patch = &fine_patches[igrid]; - double *qfine = fclaw_clawpatch_get_q(glob,fine_patch); + double *qfine = fclaw_clawpatch_get_q(glob,fine_patch); - const fclaw_options_t* fclaw_opt = fclaw_get_options(glob); + const fclaw_options_t* fclaw_opt = fclaw_get_options(glob); - fclaw_clawpatch_vtable_t* clawpatch_vt = fclaw_clawpatch_vt(glob); + fclaw_clawpatch_vtable_t* clawpatch_vt = fclaw_clawpatch_vt(glob); - double *qcoarse = fclaw_clawpatch_get_q(glob,coarse_patch); + double *qcoarse = fclaw_clawpatch_get_q(glob,coarse_patch); - if(clawpatch_opt->patch_dim == 2) - { - double *areacoarse = clawpatch_get_area(glob, coarse_patch); - double *areafine = NULL; - if (fclaw_opt->manifold) - areafine = clawpatch_get_area(glob, fine_patch); + if(clawpatch_opt->patch_dim == 2) + { + double *areacoarse = clawpatch_get_area(glob, coarse_patch); + double *areafine = NULL; + if (fclaw_opt->manifold) + areafine = clawpatch_get_area(glob, fine_patch); - int mx = clawpatch_opt->mx; - int my = clawpatch_opt->my; - clawpatch_vt->d2->fort_interpolate2fine(&mx,&my,&mbc,&meqn,qcoarse,qfine, - areacoarse, areafine, &igrid, - &fclaw_opt->manifold); - } - else - { - double *volcoarse = clawpatch_get_volume(glob, coarse_patch); - double *volfine = NULL; - if (fclaw_opt->manifold) - volfine = clawpatch_get_volume(glob, fine_patch); + int mx = clawpatch_opt->mx; + int my = clawpatch_opt->my; + clawpatch_vt->d2->fort_interpolate2fine(&mx,&my,&mbc,&meqn,qcoarse,qfine, + areacoarse, areafine, &igrid, + &fclaw_opt->manifold); + } + else + { + double *volcoarse = clawpatch_get_volume(glob, coarse_patch); + double *volfine = NULL; + if (fclaw_opt->manifold) + volfine = clawpatch_get_volume(glob, fine_patch); - int mx = clawpatch_opt->mx; - int my = clawpatch_opt->my; - int mz = clawpatch_opt->mz; - clawpatch_vt->d3->fort_interpolate2fine(&mx,&my,&mz, &mbc,&meqn,qcoarse,qfine, - volcoarse, volfine, &igrid, - &fclaw_opt->manifold); - } + int mx = clawpatch_opt->mx; + int my = clawpatch_opt->my; + int mz = clawpatch_opt->mz; + clawpatch_vt->d3->fort_interpolate2fine(&mx,&my,&mz, &mbc,&meqn,qcoarse,qfine, + volcoarse, volfine, &igrid, + &fclaw_opt->manifold); } } diff --git a/src/patches/clawpatch/fclaw_clawpatch.h.3d.TEST.cpp b/src/patches/clawpatch/fclaw_clawpatch.h.3d.TEST.cpp index 6afc84542..6e6c190f9 100644 --- a/src/patches/clawpatch/fclaw_clawpatch.h.3d.TEST.cpp +++ b/src/patches/clawpatch/fclaw_clawpatch.h.3d.TEST.cpp @@ -285,7 +285,7 @@ TEST_CASE("3d fclaw_clawpatch patch_build") fclaw_clawpatch_vtable_initialize(glob, 4); CHECK(domain->blocks[0].patches[0].user == nullptr); - fclaw_patch_build(glob, &domain->blocks[0].patches[0], 0, 0, &build_mode); + fclaw_patch_build(glob,domain, &domain->blocks[0].patches[0], 0, 0, &build_mode); CHECK(domain->blocks[0].patches[0].user != nullptr); fclaw_clawpatch_t* cp = fclaw_clawpatch_get_clawpatch(&domain->blocks[0].patches[0]); @@ -343,7 +343,7 @@ TEST_CASE("3d fclaw_clawpatch patch_build") CHECK_BOX_DIMENSIONS(cp->elliptic_soln, opts->mbc, opts->mx, opts->my, opts->mz, opts->rhs_fields); } - fclaw_patch_data_delete(glob, &domain->blocks[0].patches[0]); + fclaw_patch_data_delete(glob,domain, &domain->blocks[0].patches[0]); fclaw_clawpatch_options_destroy(opts); fclaw_domain_destroy(domain); fclaw_map_destroy(map); @@ -960,7 +960,7 @@ namespace{ fclaw_clawpatch_t* i2f_cp1; fclaw_clawpatch_t* i2f_cp2; fclaw_clawpatch_t* i2f_cp3; - std::bitset<8> i2f_igrids; + int i2f_igrid; int i2f_manifold; } TEST_CASE("3d fclaw_clawpatch interpolate2fine") @@ -1015,15 +1015,16 @@ TEST_CASE("3d fclaw_clawpatch interpolate2fine") } #endif - CHECK(i2f_igrids[*igrid] == false); - i2f_igrids[*igrid] = true; + CHECK_EQ(*igrid, i2f_igrid); CHECK(*manifold == i2f_manifold); }; - fclaw_patch_interpolate2fine(coarse_test_data.glob, - &coarse_test_data.domain->blocks[0].patches[0], - &fine_test_data.domain->blocks[0].patches[0], - 0, 0, 0); - - CHECK(i2f_igrids.all()); + for(int i = 0; i < 8; i++) + { + i2f_igrid = i; + fclaw_patch_interpolate2fine(coarse_test_data.glob, + &coarse_test_data.domain->blocks[0].patches[0], + &fine_test_data.domain->blocks[0].patches[i], + 0, 0, 0, i); + } } \ No newline at end of file diff --git a/src/patches/clawpatch/fclaw_clawpatch.h.3dx.TEST.cpp b/src/patches/clawpatch/fclaw_clawpatch.h.3dx.TEST.cpp index 29ca0a869..2bd7dd8ca 100644 --- a/src/patches/clawpatch/fclaw_clawpatch.h.3dx.TEST.cpp +++ b/src/patches/clawpatch/fclaw_clawpatch.h.3dx.TEST.cpp @@ -282,7 +282,7 @@ TEST_CASE("3dx fclaw_clawpatch patch_build") fclaw_clawpatch_vtable_initialize(glob, 4); CHECK(domain->blocks[0].patches[0].user == nullptr); - fclaw_patch_build(glob, &domain->blocks[0].patches[0], 0, 0, &build_mode); + fclaw_patch_build(glob,domain, &domain->blocks[0].patches[0], 0, 0, &build_mode); CHECK(domain->blocks[0].patches[0].user != nullptr); fclaw_clawpatch_t* cp = fclaw_clawpatch_get_clawpatch(&domain->blocks[0].patches[0]); @@ -340,7 +340,7 @@ TEST_CASE("3dx fclaw_clawpatch patch_build") CHECK_BOX_DIMENSIONS(cp->elliptic_soln, opts->mbc, opts->mx, opts->my, opts->mz, opts->rhs_fields); } - fclaw_patch_data_delete(glob, &domain->blocks[0].patches[0]); + fclaw_patch_data_delete(glob,domain, &domain->blocks[0].patches[0]); fclaw_clawpatch_options_destroy(opts); fclaw_domain_destroy(domain); fclaw_map_destroy(map); @@ -944,7 +944,7 @@ namespace{ fclaw_clawpatch_t* i2f_cp1; fclaw_clawpatch_t* i2f_cp2; fclaw_clawpatch_t* i2f_cp3; - std::bitset<4> i2f_igrids; + int i2f_igrid; int i2f_manifold; } TEST_CASE("3dx fclaw_clawpatch interpolate2fine") @@ -999,15 +999,17 @@ TEST_CASE("3dx fclaw_clawpatch interpolate2fine") } #endif - CHECK(i2f_igrids[*igrid] == false); - i2f_igrids[*igrid] = true; + CHECK_EQ(*igrid, i2f_igrid); + CHECK(*manifold == i2f_manifold); }; - fclaw_patch_interpolate2fine(coarse_test_data.glob, - &coarse_test_data.domain->blocks[0].patches[0], - &fine_test_data.domain->blocks[0].patches[0], - 0, 0, 0); - - CHECK(i2f_igrids.all()); + for(int i = 0; i < 4; i++) + { + i2f_igrid = i; + fclaw_patch_interpolate2fine(coarse_test_data.glob, + &coarse_test_data.domain->blocks[0].patches[0], + &fine_test_data.domain->blocks[0].patches[i], + 0, 0, 0, i); + } } \ No newline at end of file diff --git a/src/solvers/fc2d_geoclaw/fc2d_geoclaw.cpp b/src/solvers/fc2d_geoclaw/fc2d_geoclaw.cpp index 870635355..430121dd0 100644 --- a/src/solvers/fc2d_geoclaw/fc2d_geoclaw.cpp +++ b/src/solvers/fc2d_geoclaw/fc2d_geoclaw.cpp @@ -516,10 +516,11 @@ int geoclaw_patch_tag4coarsening(fclaw_global_t *glob, static void geoclaw_interpolate2fine(fclaw_global_t *glob, fclaw_patch_t *coarse_patch, - fclaw_patch_t *fine_patches, + fclaw_patch_t *fine_patch, int blockno, int coarse_patchno, - int fine0_patchno) + int fine_patchno, + int igrid) { int mx,my,mbc; @@ -535,20 +536,14 @@ void geoclaw_interpolate2fine(fclaw_global_t *glob, double *auxcoarse; fclaw_clawpatch_aux_data(glob,coarse_patch,&auxcoarse,&maux); - /* Loop over four siblings (z-ordering) */ - for (int igrid = 0; igrid < 4; igrid++) - { - fclaw_patch_t* fine_patch = &fine_patches[igrid]; - - double *qfine; - fclaw_clawpatch_soln_data(glob,fine_patch,&qfine,&meqn); + double *qfine; + fclaw_clawpatch_soln_data(glob,fine_patch,&qfine,&meqn); - double *auxfine; - fclaw_clawpatch_aux_data(glob,fine_patch,&auxfine,&maux); + double *auxfine; + fclaw_clawpatch_aux_data(glob,fine_patch,&auxfine,&maux); - FC2D_GEOCLAW_FORT_INTERPOLATE2FINE(&mx,&my,&mbc,&meqn,qcoarse,qfine, - &maux,auxcoarse,auxfine, &igrid); - } + FC2D_GEOCLAW_FORT_INTERPOLATE2FINE(&mx,&my,&mbc,&meqn,qcoarse,qfine, + &maux,auxcoarse,auxfine, &igrid); } static diff --git a/src/solvers/fc2d_thunderegg/fc2d_thunderegg_vector_TEST.cpp b/src/solvers/fc2d_thunderegg/fc2d_thunderegg_vector_TEST.cpp index 98d8ed1f3..34940e0dc 100644 --- a/src/solvers/fc2d_thunderegg/fc2d_thunderegg_vector_TEST.cpp +++ b/src/solvers/fc2d_thunderegg/fc2d_thunderegg_vector_TEST.cpp @@ -93,16 +93,16 @@ struct QuadDomain { } void setup(){ fclaw_build_mode_t build_mode = FCLAW_BUILD_FOR_UPDATE; - fclaw_patch_build(glob, &domain->blocks[0].patches[0], 0, 0, &build_mode); - fclaw_patch_build(glob, &domain->blocks[0].patches[1], 0, 1, &build_mode); - fclaw_patch_build(glob, &domain->blocks[0].patches[2], 0, 2, &build_mode); - fclaw_patch_build(glob, &domain->blocks[0].patches[3], 0, 3, &build_mode); + fclaw_patch_build(glob,domain, &domain->blocks[0].patches[0], 0, 0, &build_mode); + fclaw_patch_build(glob,domain, &domain->blocks[0].patches[1], 0, 1, &build_mode); + fclaw_patch_build(glob,domain, &domain->blocks[0].patches[2], 0, 2, &build_mode); + fclaw_patch_build(glob,domain, &domain->blocks[0].patches[3], 0, 3, &build_mode); } ~QuadDomain(){ - fclaw_patch_data_delete(glob, &domain->blocks[0].patches[0]); - fclaw_patch_data_delete(glob, &domain->blocks[0].patches[1]); - fclaw_patch_data_delete(glob, &domain->blocks[0].patches[2]); - fclaw_patch_data_delete(glob, &domain->blocks[0].patches[3]); + fclaw_patch_data_delete(glob,domain, &domain->blocks[0].patches[0]); + fclaw_patch_data_delete(glob,domain, &domain->blocks[0].patches[1]); + fclaw_patch_data_delete(glob,domain, &domain->blocks[0].patches[2]); + fclaw_patch_data_delete(glob,domain, &domain->blocks[0].patches[3]); fclaw_clawpatch_options_destroy(opts); fclaw_map_destroy(map); fclaw_domain_destroy(domain); @@ -157,16 +157,16 @@ struct QuadDomainBrick { } void setup(){ fclaw_build_mode_t build_mode = FCLAW_BUILD_FOR_UPDATE; - fclaw_patch_build(glob, &domain->blocks[0].patches[0], 0, 0, &build_mode); - fclaw_patch_build(glob, &domain->blocks[1].patches[0], 1, 0, &build_mode); - fclaw_patch_build(glob, &domain->blocks[2].patches[0], 2, 0, &build_mode); - fclaw_patch_build(glob, &domain->blocks[3].patches[0], 3, 0, &build_mode); + fclaw_patch_build(glob,domain, &domain->blocks[0].patches[0], 0, 0, &build_mode); + fclaw_patch_build(glob,domain, &domain->blocks[1].patches[0], 1, 0, &build_mode); + fclaw_patch_build(glob,domain, &domain->blocks[2].patches[0], 2, 0, &build_mode); + fclaw_patch_build(glob,domain, &domain->blocks[3].patches[0], 3, 0, &build_mode); } ~QuadDomainBrick(){ - fclaw_patch_data_delete(glob, &domain->blocks[0].patches[0]); - fclaw_patch_data_delete(glob, &domain->blocks[1].patches[0]); - fclaw_patch_data_delete(glob, &domain->blocks[2].patches[0]); - fclaw_patch_data_delete(glob, &domain->blocks[3].patches[0]); + fclaw_patch_data_delete(glob,domain, &domain->blocks[0].patches[0]); + fclaw_patch_data_delete(glob,domain, &domain->blocks[1].patches[0]); + fclaw_patch_data_delete(glob,domain, &domain->blocks[2].patches[0]); + fclaw_patch_data_delete(glob,domain, &domain->blocks[3].patches[0]); fclaw_clawpatch_options_destroy(opts); fclaw_map_destroy(map); fclaw_domain_destroy(domain); diff --git a/test/test.cpp b/test/test.cpp index 387e2110b..a176d9af1 100644 --- a/test/test.cpp +++ b/test/test.cpp @@ -27,6 +27,9 @@ void throw_exception() expect_abort=false; std::longjmp(jump_buffer, 1); } + else{ + FAIL("An abort was called"); + } } void fclaw_test_expect_abort()