Merge pull request #368 from scottaiton/feature_refine_after_partition

Feature Refine After Partition

src/fclaw2d_to_3d.h

@@ -216,6 +216,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define fclaw2d_patch_wrap_cb              fclaw3d_patch_wrap_cb
 #define fclaw2d_transfer_wrap_cb           fclaw3d_transfer_wrap_cb
 #define fclaw2d_match_wrap_cb              fclaw3d_match_wrap_cb
+#define fclaw2d_pack_wrap_cb               fclaw3d_pack_wrap_cb
+#define fclaw2d_unpack_wrap_cb             fclaw3d_unpack_wrap_cb
 #define fclaw2d_intersect_wrap_cb          fclaw3d_intersect_wrap_cb
 #define fclaw2d_interpolate_point_wrap_cb  fclaw3d_interpolate_point_wrap_cb
 #define fclaw2d_file_error_wrap            fclaw3d_file_error_wrap

src/fclaw2d_wrap.c

@@ -216,6 +216,38 @@ fclaw2d_match_wrap_cb(fclaw2d_domain_t * old_domain_2d,
               wrap->user);
 }
 
+void
+fclaw2d_pack_wrap_cb (fclaw2d_domain_t * domain_2d,
+                      fclaw2d_patch_t * patch_2d,
+                      int blockno, int patchno,
+                      void *pack_data_here,
+                      void *user)
+{
+    fclaw_pack_wrap_user_t* wrap = 
+        (fclaw_pack_wrap_user_t*) user;
+
+    fclaw_domain_t* domain = get_domain(domain_2d);
+    fclaw_patch_t* patch = get_patch(patch_2d);
+
+    wrap->pcb(domain, patch, blockno, patchno, pack_data_here, wrap->user);
+}
+
+void
+fclaw2d_unpack_wrap_cb (fclaw2d_domain_t * domain_2d,
+                        fclaw2d_patch_t * patch_2d,
+                        int blockno, int patchno,
+                        void *unpack_data_from_here,
+                        void *user)
+{
+    fclaw_unpack_wrap_user_t* wrap = 
+        (fclaw_unpack_wrap_user_t*) user;
+
+    fclaw_domain_t* domain = get_domain(domain_2d);
+    fclaw_patch_t* patch = get_patch(patch_2d);
+
+    wrap->ucb(domain, patch, blockno, patchno, unpack_data_from_here, wrap->user);
+}
+
 int 
 fclaw2d_intersect_wrap_cb (fclaw2d_domain_t * domain_2d,
                         fclaw2d_patch_t * patch_2d,

src/fclaw2d_wrap.h

@@ -91,6 +91,31 @@ fclaw2d_match_wrap_cb(fclaw2d_domain_t * old_domain,
                       int blockno,
                       int old_patchno, int new_patchno,
                       void *user);
+
+/** 
+ * @brief Wraps a partition pack callback for 2d domains.
+ * This is used by passing this callback to a function that takes a @ref fclaw2d_pack_callback_t.
+ * The user pointer should be a pointer to a @ref fclaw_pack_wrap_user_t.
+ */
+void
+fclaw2d_pack_wrap_cb(fclaw2d_domain_t * domain,
+                     fclaw2d_patch_t * patch,
+                     int blockno, int patchno,
+                     void *pack_data_here,
+                     void *user);
+
+/** 
+ * @brief Wraps a partition unpack callback for 2d domains.
+ * This is used by passing this callback to a function that takes a @ref fclaw2d_unpack_callback_t.
+ * The user pointer should be a pointer to a @ref fclaw_unpack_wrap_user_t.
+ */
+void
+fclaw2d_unpack_wrap_cb(fclaw2d_domain_t * domain,
+                       fclaw2d_patch_t * patch,
+                       int blockno, int patchno,
+                       void *unpack_data_from_here,
+                       void *user);
+
 /**
  * @brief Wraps a intersect callback for 2d domains.
  * This is used by passing this callback to a function that takes a @ref fclaw2d_intersect_callback_t.

src/fclaw3d_wrap.h

@@ -92,6 +92,31 @@ fclaw3d_match_wrap_cb(fclaw3d_domain_t * old_domain,
                       int old_patchno, int new_patchno,
                       void *user);
 
+/** 
+ * @brief Wraps a partition pack callback for 3d domains.
+ * This is used by passing this callback to a function that takes a @ref fclaw3d_pack_callback_t.
+ * The user pointer should be a pointer to a @ref fclaw_pack_wrap_user_t.
+ */
+void
+fclaw3d_pack_wrap_cb(fclaw3d_domain_t * domain,
+                     fclaw3d_patch_t * patch,
+                     int blockno, int patchno,
+                     void *pack_data_here,
+                     void *user);
+
+/** 
+ * @brief Wraps a partition unpack callback for 3d domains.
+ * This is used by passing this callback to a function that takes a @ref fclaw3d_unpack_callback_t.
+ * The user pointer should be a pointer to a @ref fclaw_unpack_wrap_user_t.
+ */
+void
+fclaw3d_unpack_wrap_cb(fclaw3d_domain_t * domain,
+                       fclaw3d_patch_t * patch,
+                       int blockno, int patchno,
+                       void *unpack_data_from_here,
+                       void *user);
+
+
 /**
  * @brief Wraps a intersect callback for 2d domains.
  * This is used by passing this callback to a function that takes a @ref fclaw2d_intersect_callback_t.

src/fclaw_domain.c

@@ -78,7 +78,7 @@ void fclaw_domain_reset(fclaw_global_t* glob)
             /* This is here to delete any patches created during
                initialization, and not through regridding */
             fclaw_patch_t *patch = block->patches + j;
-            fclaw_patch_data_delete(glob,patch);
+            fclaw_patch_data_delete(glob,*domain,patch);
         }
         block->user = NULL;
     }

src/fclaw_initialize.c

@@ -64,10 +64,12 @@ void cb_initialize (fclaw_domain_t *domain,
 
 	fclaw_build_mode_t build_mode = FCLAW_BUILD_FOR_UPDATE;
 
-	fclaw_patch_build(g->glob,this_patch,
-						this_block_idx,
-						this_patch_idx,
-						&build_mode);
+	fclaw_patch_build(g->glob,
+                      domain,
+                      this_patch,
+                      this_block_idx,
+                      this_patch_idx,
+                      &build_mode);
 	fclaw_patch_initialize(g->glob,this_patch,this_block_idx,this_patch_idx);
 }
 
@@ -168,30 +170,11 @@ void build_initial_domain(fclaw_global_t *glob)
             {
                 fclaw_global_infof(" -- Have new initial refinement\n");
 
-                /* Re-initialize new grids.   Ghost cell values needed for
-                   interpolation have already been set by initialization */
-                fclaw_timer_start (&glob->timers[FCLAW_TIMER_REGRID_BUILD]);
-                fclaw_global_iterate_adapted(glob, new_domain,
-                                               cb_fclaw_regrid_repopulate,
-                                               (void *) &domain_init);
-
-                fclaw_timer_stop (&glob->timers[FCLAW_TIMER_REGRID_BUILD]);
-
-                /* free all memory associated with old domain */
-                fclaw_domain_reset(glob);
-                *domain = new_domain;
-                new_domain = NULL;
-
-                /* Repartition domain to new processors.    */
-                fclaw_partition_domain(glob,FCLAW_TIMER_INIT);
-
-                /* Set up ghost patches.  This probably doesn't need to be done
-                   each time we add a new level. */
-                fclaw_exchange_setup(glob,FCLAW_TIMER_INIT);
-
-                /* This is normally called from regrid, once the initial domain
-                   has been set up */
-                fclaw_regrid_set_neighbor_types(glob);
+                fclaw_regrid_process_new_refinement(glob, 
+                                                    domain, 
+                                                    new_domain, 
+                                                    domain_init, 
+                                                    FCLAW_TIMER_INIT);
             }
             else
             {
@@ -219,9 +202,16 @@ void fclaw_initialize_domain_flags(fclaw_global_t *glob)
         (glob->domain, fclaw_opt->smooth_refine, fclaw_opt->smooth_level,
          fclaw_opt->coarsen_delay);
 
+    int skip_local = fclaw_opt->partition_mode == FCLAW_PARTITION_MODE_SKIP_LOCAL || 
+                     fclaw_opt->partition_mode == FCLAW_PARTITION_MODE_REFINE_AFTER;
+
+    int skip_refined = fclaw_opt->partition_mode == FCLAW_PARTITION_MODE_REFINE_AFTER;
+
     /* set partitioning */
     fclaw_domain_set_partitioning (glob->domain,
-                                   fclaw_opt->partition_for_coarsening, 1, 0);
+                                   fclaw_opt->partition_for_coarsening, 
+                                   skip_local,
+                                   skip_refined);
 }
 
 static void

src/fclaw_options.c

@@ -382,6 +382,15 @@ fclaw_register (fclaw_options_t* fclaw_opt, sc_options_t * opt)
                           &fclaw_opt->max_refinement_ratio, 1.0,
                           "Ratio of patches to refine before paritioning and continuing refinement. [1.0]");
 
+    kv = fclaw_opt->kv_partition_mode = sc_keyvalue_new ();
+    sc_keyvalue_set_int (kv, "legacy",                 FCLAW_PARTITION_MODE_LEGACY);
+    sc_keyvalue_set_int (kv, "pack-all",               FCLAW_PARTITION_MODE_PACK_ALL); 
+    sc_keyvalue_set_int (kv, "skip-local",             FCLAW_PARTITION_MODE_SKIP_LOCAL);
+    sc_keyvalue_set_int (kv, "refine-after-partition", FCLAW_PARTITION_MODE_REFINE_AFTER);
+    sc_options_add_keyvalue (opt, 0, "partition-mode", 
+                             &fclaw_opt->partition_mode,
+                             "legacy", kv, "Parition mode to use. One of (legacy, pack-all, skip-local, refine-after-partition) [legacy]");
+
     fclaw_opt->is_registered = 1;
     fclaw_opt->is_unpacked = 0;
 
@@ -460,6 +469,7 @@ fclaw_options_destroy(fclaw_options_t* fclaw_opt)
 
     FCLAW_ASSERT (fclaw_opt->kv_timing_verbosity != NULL);
     sc_keyvalue_destroy (fclaw_opt->kv_timing_verbosity);
+    sc_keyvalue_destroy (fclaw_opt->kv_partition_mode);
 
     // Strings need to be freed if this was unpacked form buffer
     if(fclaw_opt->is_unpacked){

src/fclaw_options.h

@@ -112,8 +112,21 @@ void fclaw_options_convert_double_array (const char *array_string,
 void fclaw_options_destroy_array(void* array);
 
 
-/* Plan is to replace fclaw_options_t with fclaw_options_t.
-   Maybe use a macro as an intermediate step? */
+/**
+ * @brief Enum for partition mode
+ */
+enum fclaw_parition_mode
+{
+    /** Use legacy paritioning interface */
+    FCLAW_PARTITION_MODE_LEGACY,
+    /** Pack/unpack all patches */
+    FCLAW_PARTITION_MODE_PACK_ALL,
+    /** Skip packing of local patches */
+    FCLAW_PARTITION_MODE_SKIP_LOCAL,
+    /** Skip packing of local patches and refine after partitioning */
+    FCLAW_PARTITION_MODE_REFINE_AFTER
+};
+
 
 struct fclaw_options
 {
@@ -254,6 +267,9 @@ struct fclaw_options
     const char * regression_check; /**< filename of regression check values */
 
     double max_refinement_ratio; /**< Maximum refinment ratio before partitioning and continuing refinement. */
+
+    int partition_mode; /**< Partition mode */
+    sc_keyvalue_t *kv_partition_mode; /**< sc_keyvalue needed for enum option type */
 };
 
 struct fclaw_global;