From b9fe5970730205d9d1fb9b03b9b9e163c97ccac5 Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Fri, 12 Jan 2024 10:51:04 -0700 Subject: [PATCH 1/8] Panzer: add timers to BasisValues2 --- .../disc-fe/src/Panzer_BasisValues2_impl.hpp | 38 +++++++++++++++++-- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/packages/panzer/disc-fe/src/Panzer_BasisValues2_impl.hpp b/packages/panzer/disc-fe/src/Panzer_BasisValues2_impl.hpp index d3fb88a62d1d..af61007a9e46 100644 --- a/packages/panzer/disc-fe/src/Panzer_BasisValues2_impl.hpp +++ b/packages/panzer/disc-fe/src/Panzer_BasisValues2_impl.hpp @@ -160,6 +160,7 @@ evaluateValues(const PHX::MDField & cub_points, const PHX::MDField & jac_inv, const int in_num_cells) { + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::basisValues2::evaluateValues(5 arg)",bv_ev_5); build_weighted = false; @@ -225,6 +226,7 @@ evaluateValues(const PHX::MDField & cub_points, bool use_node_coordinates, const int in_num_cells) { + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::basisValues2::evaluateValues(8 arg, uniform cub pts)",bv_ev_8u); // This is the base call that will add all the non-weighted versions evaluateValues(cub_points, jac, jac_det, jac_inv, in_num_cells); @@ -278,6 +280,7 @@ evaluateValues(const PHX::MDField & cub_points, bool use_node_coordinates, const int in_num_cells) { + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::basisValues2::evaluateValues(8 arg,nonuniform cub pts)",bv_ev_8nu); cell_node_coordinates_ = node_coordinates; @@ -335,11 +338,11 @@ evaluateValuesCV(const PHX::MDField & cub_points, const PHX::MDField & jac_det, const PHX::MDField & jac_inv) { + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::basisValues2::evaluateValuesCV(5 arg)",bv_ev_cv_5); PHX::MDField weighted_measure; PHX::MDField node_coordinates; evaluateValues(cub_points,jac,jac_det,jac_inv,weighted_measure,node_coordinates,false,jac.extent(0)); - } template @@ -352,6 +355,7 @@ evaluateValuesCV(const PHX::MDField & cell_cub_points, bool use_node_coordinates, const int in_num_cells) { + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::basisValues2::evaluateValuesCV(7 arg)",bv_ev_cv_7); PHX::MDField weighted_measure; evaluateValues(cell_cub_points,jac,jac_det,jac_inv,weighted_measure,node_coordinates,use_node_coordinates,in_num_cells); } @@ -376,6 +380,8 @@ applyOrientations(const std::vector & orientations, if (!intrepid_basis->requireOrientation()) return; + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::basisValues2::applyOrientations()",bv_ev_app_orts); + // We only allow the orientations to be applied once TEUCHOS_ASSERT(not orientations_applied_); @@ -622,7 +628,6 @@ setup(const Teuchos::RCP & basis, // Reset internal data resetArrays(); - } template @@ -650,7 +655,6 @@ setupUniform(const Teuchos::RCP & basis, // Reset internal data resetArrays(); - } template @@ -771,6 +775,8 @@ getBasisCoordinatesRef(const bool cache, if(basis_coordinates_ref_evaluated_ and not force) return basis_coordinates_ref; + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::basisValues2::getBasisCoordinatesRef()",bv_get_bc_ref); + MDFieldArrayFactory af(prefix,getExtendedDimensions(),true); const int num_card = basis_layout->cardinality(); @@ -797,6 +803,8 @@ getBasisValuesRef(const bool cache, if(basis_ref_scalar_evaluated_ and not force) return basis_ref_scalar; + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::basisValues2::getBasisValuesRef()",bv_get_bV_ref); + // Reference quantities only exist for a uniform reference space TEUCHOS_ASSERT(hasUniformReferenceSpace()); @@ -831,6 +839,8 @@ getVectorBasisValuesRef(const bool cache, if(basis_ref_vector_evaluated_ and not force) return basis_ref_vector; + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::basisValues2::getVectorBasisValuesRef()",bv_get_vec_bv_ref); + // Reference quantities only exist for a uniform reference space TEUCHOS_ASSERT(hasUniformReferenceSpace()); @@ -866,6 +876,8 @@ getGradBasisValuesRef(const bool cache, if(grad_basis_ref_evaluated_ and not force) return grad_basis_ref; + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::basisValues2::getGradBasisValuesRef()",bv_get_grad_bv_ref); + // Reference quantities only exist for a uniform reference space TEUCHOS_ASSERT(hasUniformReferenceSpace()); @@ -901,6 +913,8 @@ getCurl2DVectorBasisRef(const bool cache, if(curl_basis_ref_scalar_evaluated_ and not force) return curl_basis_ref_scalar; + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::basisValues2::getCurl2DVectorBasisRef()",bv_get_curl2_bv_ref); + // Reference quantities only exist for a uniform reference space TEUCHOS_ASSERT(hasUniformReferenceSpace()); TEUCHOS_ASSERT(basis_layout->dimension() == 2); @@ -936,6 +950,8 @@ getCurlVectorBasisRef(const bool cache, if(curl_basis_ref_vector_evaluated_ and not force) return curl_basis_ref_vector; + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::basisValues2::getCurlVectorBasisRef()",bv_get_curl_vec_bv_ref); + // Reference quantities only exist for a uniform reference space TEUCHOS_ASSERT(hasUniformReferenceSpace()); TEUCHOS_ASSERT(basis_layout->dimension() == 3); @@ -972,6 +988,8 @@ getDivVectorBasisRef(const bool cache, if(div_basis_ref_evaluated_ and not force) return div_basis_ref; + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::basisValues2::getDivVectorBasisRef()",bv_get_div_vec_bv_ref); + // Reference quantities only exist for a uniform reference space TEUCHOS_ASSERT(hasUniformReferenceSpace()); @@ -1002,6 +1020,8 @@ BasisValues2:: getBasisCoordinates(const bool cache, const bool force) const { + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::basisValues2::getBasisCoordinates()",bv_get_bc); + // Check if array already exists if(basis_coordinates_evaluated_ and not force) return basis_coordinates; @@ -1047,6 +1067,8 @@ getBasisValues(const bool weighted, if(basis_scalar_evaluated_ and not force) return basis_scalar; + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::basisValues2::getBasisValues()",bv_get_bv); + MDFieldArrayFactory af(prefix,getExtendedDimensions(),true); const int num_cells = num_cells_; @@ -1206,6 +1228,8 @@ getVectorBasisValues(const bool weighted, if(basis_vector_evaluated_ and not force) return basis_vector; + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::basisValues2::getVectorBasisValues()",bv_get_vec_bv); + MDFieldArrayFactory af(prefix,getExtendedDimensions(),true); const int num_cells = num_cells_; @@ -1389,6 +1413,8 @@ getGradBasisValues(const bool weighted, if(grad_basis_evaluated_ and not force) return grad_basis; + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::basisValues2::getGradBasisValues()",bv_get_grad_bv); + MDFieldArrayFactory af(prefix,getExtendedDimensions(),true); const int num_cells = num_cells_; @@ -1536,6 +1562,8 @@ getCurl2DVectorBasis(const bool weighted, if(curl_basis_scalar_evaluated_ and not force) return curl_basis_scalar; + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::basisValues2::getCurl2DVectorBasis()",bv_get_curl2d_vec_bv); + MDFieldArrayFactory af(prefix,getExtendedDimensions(),true); const int num_cells = num_cells_; @@ -1681,6 +1709,8 @@ getCurlVectorBasis(const bool weighted, if(curl_basis_vector_evaluated_ and not force) return curl_basis_vector; + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::basisValues2::getCurlVectorBasis()",bv_get_curl_vec_bv); + MDFieldArrayFactory af(prefix,getExtendedDimensions(),true); const int num_cells = num_cells_; @@ -1829,6 +1859,8 @@ getDivVectorBasis(const bool weighted, if(div_basis_evaluated_ and not force) return div_basis; + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::basisValues2::getDevVectorBasis()",bv_get_div_vec_bv); + MDFieldArrayFactory af(prefix,getExtendedDimensions(),true); const int num_cells = num_cells_; From b24ad8c75e617aacca0d64a22e01917ee4d43f40 Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Fri, 12 Jan 2024 10:52:07 -0700 Subject: [PATCH 2/8] Panzer: add timers to IntegrationValues2 --- .../src/Panzer_STK_LocalMeshUtilities.cpp | 4 +- .../disc-fe/src/Panzer_IntegrationValues2.cpp | 95 +++++++++++++------ 2 files changed, 67 insertions(+), 32 deletions(-) diff --git a/packages/panzer/adapters-stk/src/Panzer_STK_LocalMeshUtilities.cpp b/packages/panzer/adapters-stk/src/Panzer_STK_LocalMeshUtilities.cpp index e2c4af0ea6f5..ba7a210efa0f 100644 --- a/packages/panzer/adapters-stk/src/Panzer_STK_LocalMeshUtilities.cpp +++ b/packages/panzer/adapters-stk/src/Panzer_STK_LocalMeshUtilities.cpp @@ -454,6 +454,8 @@ setupLocalMeshSidesetInfo(const panzer_stk::STK_Interface & mesh, Teuchos::RCP generateLocalMeshInfo(const panzer_stk::STK_Interface & mesh) { + TEUCHOS_FUNC_TIME_MONITOR_DIFF("panzer_stk::generateLocalMeshInfo",GenerateLocalMeshInfo); + using Teuchos::RCP; using Teuchos::rcp; @@ -474,8 +476,6 @@ generateLocalMeshInfo(const panzer_stk::STK_Interface & mesh) Teuchos::RCP > comm = mesh.getComm(); - TEUCHOS_FUNC_TIME_MONITOR_DIFF("panzer_stk::generateLocalMeshInfo",GenerateLocalMeshInfo); - // This horrible line of code is required since the connection manager only takes rcps of a mesh RCP mesh_rcp = Teuchos::rcpFromRef(mesh); // We're allowed to do this since the connection manager only exists in this scope... even though it is also an RCP... diff --git a/packages/panzer/disc-fe/src/Panzer_IntegrationValues2.cpp b/packages/panzer/disc-fe/src/Panzer_IntegrationValues2.cpp index bffb4f5c3d3f..08f80e9c3be3 100644 --- a/packages/panzer/disc-fe/src/Panzer_IntegrationValues2.cpp +++ b/packages/panzer/disc-fe/src/Panzer_IntegrationValues2.cpp @@ -102,6 +102,7 @@ correctVirtualNormals(PHX::MDField normals, const shards::CellTopology & cell_topology, const SubcellConnectivity & face_connectivity) { + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::integrationValues2::correctVirtualNormals()",corr_virt_norms); // What we want is for the adjoining face of the virtual cell to have normals that are the negated real cell's normals. // we correct the normals here: @@ -169,6 +170,7 @@ correctVirtualRotationMatrices(PHX::MDField rotation_mat const shards::CellTopology & cell_topology, const SubcellConnectivity & face_connectivity) { + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::integrationValues2::correctVirtualRotationMatrices()",corr_virt_rotmat); // What we want is for the adjoining face of the virtual cell to have normals that are the negated real cell's normals. // we correct the normals here: @@ -216,6 +218,7 @@ void applyBasePermutation(PHX::MDField field, PHX::MDField permutations) { + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::integrationValues2::applyBasePermutation(rank 1)",app_base_perm_r1); MDFieldArrayFactory af("",true); const int num_ip = field.extent(0); @@ -235,6 +238,7 @@ void applyBasePermutation(PHX::MDField field, PHX::MDField permutations) { + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::integrationValues2::applyBasePermutation(rank 2)",app_base_perm_r2); MDFieldArrayFactory af("",true); const int num_ip = field.extent(0); @@ -256,6 +260,7 @@ void applyPermutation(PHX::MDField field, PHX::MDField permutations) { + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::integrationValues2::applyPermutation(rank 2)",app_perm_r2); MDFieldArrayFactory af("",true); const int num_cells = field.extent(0); @@ -276,6 +281,7 @@ void applyPermutation(PHX::MDField field, PHX::MDField permutations) { + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::integrationValues2::applyPermutation(rank 3)",app_perm_r3); MDFieldArrayFactory af("",true); const int num_cells = field.extent(0); @@ -298,6 +304,7 @@ void applyPermutation(PHX::MDField field, PHX::MDField permutations) { + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::integrationValues2::applyPermutation(rank 4)",app_perm_r4); MDFieldArrayFactory af("",true); const int num_cells = field.extent(0); @@ -327,6 +334,8 @@ generatePermutations(const int num_cells, PHX::MDField coords, PHX::MDField other_coords) { + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::integrationValues2::generatePermutations()",gen_perms); + const int num_ip = coords.extent(1); const int num_dim = coords.extent(2); @@ -380,6 +389,7 @@ generateSurfacePermutations(const int num_cells, PHX::MDField surface_rotation_matrices) { + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::integrationValues2::generateSurfacePermutations()",gen_surf_perms); // The challenge for this call is handling wedge-based periodic boundaries // We need to make sure that we can align points along faces that are rotated with respect to one another. @@ -565,10 +575,9 @@ generateSurfacePermutations(const int num_cells, #undef PANZER_CROSS return permutation; - } -} +} // end anonymous namespace //template //using UnmanagedDynRankView = Kokkos::DynRankView::type,PHX::Device,Kokkos::MemoryTraits>; @@ -617,6 +626,8 @@ template void IntegrationValues2:: setupArrays(const Teuchos::RCP& ir) { + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::integrationValues2::setupArrays()",setup_arrays); + MDFieldArrayFactory af(prefix_,alloc_arrays_); typedef panzer::IntegrationDescriptor ID; @@ -668,7 +679,6 @@ setupArrays(const Teuchos::RCP& ir) surface_normals = af.template buildStaticArray("surface_normals",num_cells, num_ip,num_space_dim); surface_rotation_matrices = af.template buildStaticArray("surface_rotation_matrices",num_cells, num_ip,3,3); - } @@ -682,6 +692,7 @@ evaluateValues(const PHX::MDField & in_node_coordinates, const Teuchos::RCP & face_connectivity, const int num_virtual_cells) { + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::integrationValues2::evaluateValues(with virtual cells)",eval_vals_with_virts); setup(int_rule, in_node_coordinates, in_num_cells); @@ -692,7 +703,6 @@ evaluateValues(const PHX::MDField & in_node_coordinates, // Evaluate everything once permutations are generated evaluateEverything(); - } template @@ -702,6 +712,7 @@ evaluateValues(const PHX::MDField& in_node_coordinates, const PHX::MDField& other_ip_coordinates, const int in_num_cells) { + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::integrationValues2::evaluateValues(no virtual cells)",eval_vals_no_virts); setup(int_rule, in_node_coordinates, in_num_cells); @@ -710,7 +721,6 @@ evaluateValues(const PHX::MDField& in_node_coordinates, // Evaluate everything once permutations are generated evaluateEverything(); - } template @@ -719,6 +729,8 @@ IntegrationValues2:: setupPermutations(const Teuchos::RCP & face_connectivity, const int num_virtual_cells) { + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::integrationValues2::setupPermutations(connectivity)",setup_perms_conn); + TEUCHOS_ASSERT(not int_rule->isSide()); TEUCHOS_ASSERT(face_connectivity != Teuchos::null); TEUCHOS_TEST_FOR_EXCEPT_MSG(int_rule->getType() != panzer::IntegrationDescriptor::SURFACE, @@ -738,6 +750,7 @@ void IntegrationValues2:: setupPermutations(const PHX::MDField & other_ip_coordinates) { + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::integrationValues2::setupPermutations(other_coords)",setup_perms_coords); resetArrays(); requires_permutation_ = false; permutations_ = generatePermutations(num_evaluate_cells_, getCubaturePoints(false,true), other_ip_coordinates); @@ -752,6 +765,7 @@ setup(const Teuchos::RCP& ir, const PHX::MDField & cell_node_coordinates, const int num_cells) { + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::integrationValues2::setup()",setup); // Clear arrays just in case we are rebuilding this object resetArrays(); @@ -789,10 +803,12 @@ getUniformCubaturePointsRef(const bool cache, const bool force, const bool apply_permutation) const { - if(cub_points_evaluated_ and not force) return cub_points; + // Only log time if values computed (i.e. don't log if values are already cached) + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::integrationValues2::getUniformCubaturePointsRef()",get_uniform_cub_pts_ref); + Intrepid2::CellTools cell_tools; MDFieldArrayFactory af(prefix_,true); @@ -845,10 +861,12 @@ getUniformSideCubaturePointsRef(const bool cache, const bool force, const bool apply_permutation) const { - if(side_cub_points_evaluated_ and not force) return side_cub_points; + // Only log time if values computed (i.e. don't log if values are already cached) + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::integrationValues2::getUniformSideCubaturePointsRef()",get_uniform_side_cub_pts_ref); + MDFieldArrayFactory af(prefix_,true); int num_space_dim = int_rule->topology->getDimension(); @@ -895,10 +913,12 @@ getUniformCubatureWeightsRef(const bool cache, const bool force, const bool apply_permutation) const { - if(cub_weights_evaluated_ and not force) return cub_weights; + // Only log time if values computed (i.e. don't log if values are already cached) + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::integrationValues2::getUniformCubatureWeightRef()",get_uniform_cub_weights_ref); + MDFieldArrayFactory af(prefix_,true); int num_space_dim = int_rule->topology->getDimension(); @@ -953,6 +973,9 @@ getJacobian(const bool cache, if(jac_evaluated_ and not force) return jac; + // Only log time if values computed (i.e. don't log if values are already cached) + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::integrationValues2::getJacobian()",get_jacobian); + Intrepid2::CellTools cell_tools; MDFieldArrayFactory af(prefix_,true); @@ -980,7 +1003,6 @@ getJacobian(const bool cache, } return aux; - } template @@ -989,10 +1011,12 @@ IntegrationValues2:: getJacobianInverse(const bool cache, const bool force) const { - if(jac_inv_evaluated_ and not force) return jac_inv; + // Only log time if values computed (i.e. don't log if values are already cached) + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::integrationValues2::getJacobianInverse()",get_jacobian_inv); + Intrepid2::CellTools cell_tools; MDFieldArrayFactory af(prefix_,true); @@ -1016,7 +1040,6 @@ getJacobianInverse(const bool cache, } return aux; - } template @@ -1025,10 +1048,12 @@ IntegrationValues2:: getJacobianDeterminant(const bool cache, const bool force) const { - if(jac_det_evaluated_ and not force) return jac_det; + // Only log time if values computed (i.e. don't log if values are already cached) + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::integrationValues2::getJacobianDeterminant()",get_jacobian_det); + Intrepid2::CellTools cell_tools; MDFieldArrayFactory af(prefix_,true); @@ -1051,7 +1076,6 @@ getJacobianDeterminant(const bool cache, } return aux; - } template @@ -1060,10 +1084,12 @@ IntegrationValues2:: getWeightedMeasure(const bool cache, const bool force) const { - if(weighted_measure_evaluated_ and not force) return weighted_measure; + // Only log time if values computed (i.e. don't log if values are already cached) + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::integrationValues2::getWeightedMeasure()",get_wt_meas); + MDFieldArrayFactory af(prefix_,true); const int num_space_dim = int_rule->topology->getDimension(); @@ -1218,7 +1244,6 @@ getWeightedMeasure(const bool cache, } return aux; - } template @@ -1227,10 +1252,12 @@ IntegrationValues2:: getWeightedNormals(const bool cache, const bool force) const { - if(weighted_normals_evaluated_ and not force) return weighted_normals; + // Only log time if values computed (i.e. don't log if values are already cached) + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::integrationValues2::getWeightedNormals()",get_wt_normals); + MDFieldArrayFactory af(prefix_,true); const int num_space_dim = int_rule->topology->getDimension(); @@ -1264,7 +1291,6 @@ getWeightedNormals(const bool cache, } return aux; - } template @@ -1273,10 +1299,12 @@ IntegrationValues2:: getSurfaceNormals(const bool cache, const bool force) const { - if(surface_normals_evaluated_ and not force) return surface_normals; + // Only log time if values computed (i.e. don't log if values are already cached) + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::integrationValues2::getSurfaceNormals()",get_surf_normals); + TEUCHOS_TEST_FOR_EXCEPT_MSG(int_rule->isSide(), "IntegrationValues2::getSurfaceNormals : This call doesn't work with sides (only surfaces)."); @@ -1389,10 +1417,12 @@ IntegrationValues2:: getSurfaceRotationMatrices(const bool cache, const bool force) const { - if(surface_rotation_matrices_evaluated_ and not force) return surface_rotation_matrices; + // Only log time if values computed (i.e. don't log if values are already cached) + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::integrationValues2::getSurfaceRotationMatrices()",get_surf_rot_mat); + MDFieldArrayFactory af(prefix_,true); const int num_ip = int_rule->num_points; @@ -1448,10 +1478,12 @@ IntegrationValues2:: getCovarientMatrix(const bool cache, const bool force) const { - if(covarient_evaluated_ and not force) return covarient; + // Only log time if values computed (i.e. don't log if values are already cached) + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::integrationValues2::getCovariantMatrix()",get_cov_mat); + MDFieldArrayFactory af(prefix_,true); const int num_space_dim = int_rule->topology->getDimension(); @@ -1480,7 +1512,6 @@ getCovarientMatrix(const bool cache, } return aux; - } template @@ -1489,10 +1520,12 @@ IntegrationValues2:: getContravarientMatrix(const bool cache, const bool force) const { - if(contravarient_evaluated_ and not force) return contravarient; + // Only log time if values computed (i.e. don't log if values are already cached) + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::integrationValues2::getContravarientMatrix()",get_contra_mat); + MDFieldArrayFactory af(prefix_,true); const int num_space_dim = int_rule->topology->getDimension(); @@ -1514,7 +1547,6 @@ getContravarientMatrix(const bool cache, } return aux; - } template @@ -1523,10 +1555,12 @@ IntegrationValues2:: getNormContravarientMatrix(const bool cache, const bool force) const { - if(norm_contravarient_evaluated_ and not force) return norm_contravarient; + // Only log time if values computed (i.e. don't log if values are already cached) + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::integrationValues2::getNormContravarientMatrix()",get_norm_contr_mat); + MDFieldArrayFactory af(prefix_,true); const int num_space_dim = int_rule->topology->getDimension(); @@ -1554,7 +1588,6 @@ getNormContravarientMatrix(const bool cache, } return aux; - } template @@ -1563,10 +1596,12 @@ IntegrationValues2:: getCubaturePoints(const bool cache, const bool force) const { - if(ip_coordinates_evaluated_ and not force) return ip_coordinates; + // Only log time if values computed (i.e. don't log if values are already cached) + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::integrationValues2::getCubaturePoints()",get_cub_pts); + MDFieldArrayFactory af(prefix_,true); const int num_space_dim = int_rule->topology->getDimension(); @@ -1621,7 +1656,6 @@ getCubaturePoints(const bool cache, } return aux; - } @@ -1631,10 +1665,12 @@ IntegrationValues2:: getCubaturePointsRef(const bool cache, const bool force) const { - if(ref_ip_coordinates_evaluated_) return ref_ip_coordinates; + // Only log time if values computed (i.e. don't log if values are already cached) + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::integrationValues2::getCubaturePointsRef()",get_cub_pts_ref); + using ID=panzer::IntegrationDescriptor; const bool is_surface = int_rule->getType() == ID::SURFACE; const bool is_cv = (int_rule->getType() == ID::CV_VOLUME) or (int_rule->getType() == ID::CV_SIDE) or (int_rule->getType() == ID::CV_BOUNDARY); @@ -1743,7 +1779,6 @@ getCubaturePointsRef(const bool cache, } return aux; - } template @@ -1751,6 +1786,7 @@ void IntegrationValues2:: evaluateEverything() { + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::integrationValues2::evaluateEverything()",eval_everything); using ID=panzer::IntegrationDescriptor; const bool is_surface = int_rule->getType() == ID::SURFACE; @@ -1795,7 +1831,6 @@ evaluateEverything() getCovarientMatrix(true,true); getNormContravarientMatrix(true,true); } - } #define INTEGRATION_VALUES2_INSTANTIATION(SCALAR) \ From 8d4eea0003929ff132a84b1054194ef8b69cf91a Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Fri, 12 Jan 2024 10:54:39 -0700 Subject: [PATCH 3/8] Panzer: add timers to WorksetContainer --- .../panzer/disc-fe/src/Panzer_WorksetContainer.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/panzer/disc-fe/src/Panzer_WorksetContainer.cpp b/packages/panzer/disc-fe/src/Panzer_WorksetContainer.cpp index 9a691810df33..49d7988443b2 100644 --- a/packages/panzer/disc-fe/src/Panzer_WorksetContainer.cpp +++ b/packages/panzer/disc-fe/src/Panzer_WorksetContainer.cpp @@ -113,6 +113,8 @@ const WorksetNeeds & WorksetContainer::lookupNeeds(const std::string & eBlock) c Teuchos::RCP > WorksetContainer::getWorksets(const WorksetDescriptor & wd) { + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::WorksetContainer::getWorksets()",pwkst_con_get_worksets); + Teuchos::RCP > worksetVector; WorksetMap::iterator itr = worksets_.find(wd); if(itr==worksets_.end()) { @@ -143,6 +145,8 @@ WorksetContainer::getWorksets(const WorksetDescriptor & wd) Teuchos::RCP > WorksetContainer::getSideWorksets(const WorksetDescriptor & desc) { + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::WorksetContainer::getSideWorksets()",pwkst_con_get_side_worksets); + Teuchos::RCP > worksetMap; // this is the key for the workset map @@ -205,6 +209,8 @@ addBasis(const std::string & type,int order,const std::string & rep_field) void WorksetContainer:: applyOrientations(const Teuchos::RCP & ugi) { + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::WorksetContainer::applyOrientations(ugi)",pwkst_con_apply_orts); + // this gurantees orientations won't accidently be applied twice. TEUCHOS_ASSERT(globalIndexer_==Teuchos::null); @@ -257,6 +263,8 @@ applyOrientations(const std::string & eBlock, std::vector & worksets) c { using Teuchos::RCP; + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::WorksetContainer::applyOrientations(eBlock,worksets)",pwkst_con_apply_orts_eb_w); + ///////////////////////////////// // this is for volume worksets // ///////////////////////////////// @@ -377,6 +385,8 @@ applyOrientations(const WorksetDescriptor & desc,std::map & wo { using Teuchos::RCP; + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::WorksetContainer::applyOrientations(wd,worksets)",pwkst_con_apply_orts_wd_wksts); + ///////////////////////////////// // this is for side worksets // ///////////////////////////////// From 68f0691dd82f94b8172e931ca6b4448ac4a9e2cb Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Fri, 12 Jan 2024 11:12:02 -0700 Subject: [PATCH 4/8] Panzer: update documentation --- .../disc-fe/src/Panzer_BasisDescriptor.hpp | 24 +++++++------------ 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/packages/panzer/disc-fe/src/Panzer_BasisDescriptor.hpp b/packages/panzer/disc-fe/src/Panzer_BasisDescriptor.hpp index a6627bd75b7e..c125d6d1a09b 100644 --- a/packages/panzer/disc-fe/src/Panzer_BasisDescriptor.hpp +++ b/packages/panzer/disc-fe/src/Panzer_BasisDescriptor.hpp @@ -66,8 +66,8 @@ class BasisDescriptor /** \brief Constructor for basis description * - * \param[in] basis_order Basis order (e.g. 1 could be piecewise linear) - * \param[in] basis_type Basis type (e.g. HGrad, HDiv, HCurl, ...) + * \param[in] basis_order Basis order as defined by Intrepid2 (e.g. 1 could be piecewise linear) + * \param[in] basis_type Basis type (a string: "HGrad", "HDiv", "HCurl", or "HVol") */ BasisDescriptor(const int basis_order, const std::string & basis_type); @@ -90,7 +90,6 @@ class BasisDescriptor */ std::size_t getKey() const {return _key;} - /** \brief Build a point descriptor that builds reference points for * the DOF locations. This method throws if no points exist for this * basis. @@ -98,9 +97,8 @@ class BasisDescriptor PointDescriptor getPointDescriptor() const; protected: - - /// Basis type (HGrad, HDiv, HCurl,...) + /// Basis type (HGrad, HDiv, HCurl, HVol) std::string _basis_type; // Basis order (>0) @@ -108,20 +106,16 @@ class BasisDescriptor // Unique key associated with basis. std::size_t _key; - }; -} - +} // namespace panzer namespace std { - -template <> -struct hash -{ - std::size_t operator()(const panzer::BasisDescriptor& desc) const; -}; - + template <> + struct hash + { + std::size_t operator()(const panzer::BasisDescriptor& desc) const; + }; } From 82bb4891f67fdec39cd464db3bb6b3848d4e7996 Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Thu, 25 Jan 2024 12:51:47 -0700 Subject: [PATCH 5/8] Panzer: update doxygen --- packages/panzer/disc-fe/src/Panzer_PureBasis.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/panzer/disc-fe/src/Panzer_PureBasis.hpp b/packages/panzer/disc-fe/src/Panzer_PureBasis.hpp index a7cd7229981e..136f812b5e2d 100644 --- a/packages/panzer/disc-fe/src/Panzer_PureBasis.hpp +++ b/packages/panzer/disc-fe/src/Panzer_PureBasis.hpp @@ -64,14 +64,14 @@ namespace panzer { typedef enum { HGRAD=0, HCURL=1, HDIV=2, HVOL=3, CONST=4 } EElementSpace; /** Build a basis given a type, order and CellData object - \param[in] basis_type String name that describes the type of basis + \param[in] basis_type String name that describes the type of basis ("HGrad", "HDiv", "HCurl", or "HVol") \param[in] basis_order Order of the basis \param[in] cell_data Description of the basis */ PureBasis(const std::string & basis_type,const int basis_order,const CellData & cell_data); /** Build a basis given a type, order, number of cells (for data layouts) and shards topology - \param[in] basis_type String name that describes the type of basis + \param[in] basis_type String name that describes the type of basis ("HGrad", "HDiv", "HCurl", or "HVol") \param[in] basis_order Order of the basis \param[in] num_cells Number of cells used in the data layouts for this basis \param[in] cell_topo A shards topology description From 3d3fd4e07e38e5d7ebc85d3e3b68d518a8417c43 Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Fri, 9 Feb 2024 13:26:21 -0700 Subject: [PATCH 6/8] Panzer: remove unused includes from mini-em --- packages/panzer/mini-em/src/solvers/MiniEM_Interpolation.hpp | 1 - .../mini-em/src/solvers/MiniEM_MatrixFreeInterpolationOp.cpp | 1 - 2 files changed, 2 deletions(-) diff --git a/packages/panzer/mini-em/src/solvers/MiniEM_Interpolation.hpp b/packages/panzer/mini-em/src/solvers/MiniEM_Interpolation.hpp index 36a10758d532..a9f966c67da1 100644 --- a/packages/panzer/mini-em/src/solvers/MiniEM_Interpolation.hpp +++ b/packages/panzer/mini-em/src/solvers/MiniEM_Interpolation.hpp @@ -11,7 +11,6 @@ #endif #include "Panzer_LOCPair_GlobalEvaluationData.hpp" -#include "Panzer_IntrepidOrientation.hpp" #include "Panzer_IntrepidBasisFactory.hpp" #include "Intrepid2_OrientationTools.hpp" #include "Intrepid2_LagrangianInterpolation.hpp" diff --git a/packages/panzer/mini-em/src/solvers/MiniEM_MatrixFreeInterpolationOp.cpp b/packages/panzer/mini-em/src/solvers/MiniEM_MatrixFreeInterpolationOp.cpp index a4cb991718d9..8302820fcc1d 100644 --- a/packages/panzer/mini-em/src/solvers/MiniEM_MatrixFreeInterpolationOp.cpp +++ b/packages/panzer/mini-em/src/solvers/MiniEM_MatrixFreeInterpolationOp.cpp @@ -1,7 +1,6 @@ #include "MiniEM_MatrixFreeInterpolationOp.hpp" #include #include "Panzer_LOCPair_GlobalEvaluationData.hpp" -#include "Panzer_IntrepidOrientation.hpp" #include "Panzer_IntrepidBasisFactory.hpp" #include "Panzer_IntrepidFieldPattern.hpp" #include "Panzer_BlockedTpetraLinearObjFactory.hpp" From bba9fc0146767480a839dd820db5917b1abdfd20 Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Fri, 9 Feb 2024 15:14:00 -0700 Subject: [PATCH 7/8] Panzer: improve performance of mass matrix construction - Allow users to set the BasisValues2 object for element blocks on L2 projection, avoiding WorksetContainer use. - BasisValues2 retursn a BasisDescriptor for error checking - Orientations support can build orientations per element block --- .../test/projection/projection.cpp | 63 ++++++++++++++++- .../disc-fe/src/Panzer_BasisDescriptor.cpp | 7 ++ .../disc-fe/src/Panzer_BasisDescriptor.hpp | 3 + .../disc-fe/src/Panzer_BasisValues2.hpp | 3 + .../disc-fe/src/Panzer_BasisValues2_impl.hpp | 9 ++- .../src/Panzer_IntrepidOrientation.cpp | 51 ++++++++++++++ .../src/Panzer_IntrepidOrientation.hpp | 23 +++++++ .../disc-fe/src/Panzer_L2Projection.cpp | 67 ++++++++++++++++--- .../disc-fe/src/Panzer_L2Projection.hpp | 13 ++++ 9 files changed, 228 insertions(+), 11 deletions(-) diff --git a/packages/panzer/adapters-stk/test/projection/projection.cpp b/packages/panzer/adapters-stk/test/projection/projection.cpp index 5a9c33cf4d33..6639cd6639b2 100644 --- a/packages/panzer/adapters-stk/test/projection/projection.cpp +++ b/packages/panzer/adapters-stk/test/projection/projection.cpp @@ -61,6 +61,7 @@ #include "Panzer_Traits.hpp" #include "Panzer_Evaluator_WithBaseImpl.hpp" #include "Panzer_IntrepidBasisFactory.hpp" +#include "Panzer_IntrepidOrientation.hpp" #include "Panzer_L2Projection.hpp" #include "Panzer_DOFManager.hpp" #include "Panzer_BlockedDOFManager.hpp" @@ -152,7 +153,7 @@ TEUCHOS_UNIT_TEST(L2Projection, ToNodal) using LO = int; using GO = panzer::GlobalOrdinal; timer->start("ConnManager ctor"); - const RCP connManager = rcp(new panzer_stk::STKConnManager(mesh)); + const RCP connManager = rcp(new panzer_stk::STKConnManager(mesh)); timer->stop("ConnManager ctor"); // Set up bases for projections @@ -244,6 +245,66 @@ TEUCHOS_UNIT_TEST(L2Projection, ToNodal) projectionFactory.setup(hgradBD,integrationDescriptor,comm,connManager,eBlockNames,worksetContainer); timer->stop("projectionFactory.setup()"); + + + + + // Ignore the workset factory and set the basis directly + bool ignoreWorksetFactory = true; + std::map>> map_basis_values; + if (ignoreWorksetFactory) { + + // Get the orientations for all element blocks + std::map> orientations; + panzer::buildIntrepidOrientations(eBlockNames,*connManager,orientations); + + std::vector topologies; + connManager->getElementBlockTopologies(topologies); + + for (size_t i=0; i < eBlockNames.size(); ++i) { + + const auto& eblock = eBlockNames[i]; + const auto& topo = topologies[i]; + + // Get nodal coordinates for integration rules + std::vector lids; + panzer::Intrepid2FieldPattern coord_fp(hgradBasis); + Kokkos::DynRankView nodal_coords_kokkos; + connManager->getDofCoords(eblock,coord_fp,lids,nodal_coords_kokkos); + + // This is ugly. Need to fix connManager interface for layouts (fad vs non-fad). + PHX::MDField nodal_coords("nodal_coords","", + nodal_coords_kokkos.extent(0), + nodal_coords_kokkos.extent(1), + nodal_coords_kokkos.extent(2)); + Kokkos::deep_copy(nodal_coords.get_view(),nodal_coords_kokkos); + + // Build IV + Teuchos::RCP rcp_topo = Teuchos::rcp(new shards::CellTopology(topo.getCellTopologyData())); + Teuchos::RCP ir = Teuchos::rcp(new panzer::IntegrationRule(integrationDescriptor,rcp_topo,static_cast(nodal_coords_kokkos.extent(0)))); + Teuchos::RCP> iv = Teuchos::rcp(new panzer::IntegrationValues2); + iv->setup(ir,nodal_coords); + + // Build BV + Teuchos::RCP> bv = Teuchos::rcp(new panzer::BasisValues2); + Teuchos::RCP birl = Teuchos::rcp(new panzer::BasisIRLayout(hgradBD.getType(),hgradBD.getOrder(),*ir)); + bv->setupUniform(birl, + iv->getUniformCubaturePointsRef(true), + iv->getJacobian(true), + iv->getJacobianDeterminant(true), + iv->getJacobianInverse(true)); + bv->setOrientations(orientations[eblock]); + bv->setWeightedMeasure(iv->getWeightedMeasure(true)); + bv->setCellNodeCoordinates(nodal_coords); + map_basis_values[eblock] = bv; + } + projectionFactory.useBasisValues(map_basis_values); + } + + + + + TEST_ASSERT(nonnull(projectionFactory.getTargetGlobalIndexer())); // Build mass matrix diff --git a/packages/panzer/disc-fe/src/Panzer_BasisDescriptor.cpp b/packages/panzer/disc-fe/src/Panzer_BasisDescriptor.cpp index ead8704e5b2b..4184dc7ff5ab 100644 --- a/packages/panzer/disc-fe/src/Panzer_BasisDescriptor.cpp +++ b/packages/panzer/disc-fe/src/Panzer_BasisDescriptor.cpp @@ -134,6 +134,13 @@ getPointDescriptor() const return pd; } +bool operator==(const panzer::BasisDescriptor& left, + const panzer::BasisDescriptor& right) +{ + return ( (left.getType() == right.getType()) && + (left.getOrder() == right.getOrder()) ); +} + } // end namespace panzer std::size_t diff --git a/packages/panzer/disc-fe/src/Panzer_BasisDescriptor.hpp b/packages/panzer/disc-fe/src/Panzer_BasisDescriptor.hpp index c125d6d1a09b..911307a9c5f8 100644 --- a/packages/panzer/disc-fe/src/Panzer_BasisDescriptor.hpp +++ b/packages/panzer/disc-fe/src/Panzer_BasisDescriptor.hpp @@ -108,6 +108,9 @@ class BasisDescriptor std::size_t _key; }; +bool operator==(const panzer::BasisDescriptor& left, + const panzer::BasisDescriptor& right); + } // namespace panzer namespace std { diff --git a/packages/panzer/disc-fe/src/Panzer_BasisValues2.hpp b/packages/panzer/disc-fe/src/Panzer_BasisValues2.hpp index 6f010661a648..26bd32642495 100644 --- a/packages/panzer/disc-fe/src/Panzer_BasisValues2.hpp +++ b/packages/panzer/disc-fe/src/Panzer_BasisValues2.hpp @@ -384,6 +384,9 @@ namespace panzer { hasUniformReferenceSpace() const {return is_uniform_;} + /// Return the basis descriptor + panzer::BasisDescriptor getBasisDescriptor() const; + /// Get the extended dimensions used by sacado AD allocations const std::vector & getExtendedDimensions() const diff --git a/packages/panzer/disc-fe/src/Panzer_BasisValues2_impl.hpp b/packages/panzer/disc-fe/src/Panzer_BasisValues2_impl.hpp index af61007a9e46..b1f775e5ac47 100644 --- a/packages/panzer/disc-fe/src/Panzer_BasisValues2_impl.hpp +++ b/packages/panzer/disc-fe/src/Panzer_BasisValues2_impl.hpp @@ -127,6 +127,7 @@ BasisValues2(const std::string & pre, , num_cells_(0) , num_evaluate_cells_(0) , is_uniform_(false) + , num_orientations_cells_(0) { // Default all lazy evaluated components to not-evaluated @@ -686,6 +687,13 @@ setCellNodeCoordinates(PHX::MDField node_coordinates) cell_node_coordinates_ = node_coordinates; } +template +panzer::BasisDescriptor BasisValues2::getBasisDescriptor() const +{ + auto pure_basis = basis_layout->getBasis(); + return {pure_basis->order(),pure_basis->type()}; +} + template void BasisValues2:: @@ -1201,7 +1209,6 @@ getBasisValues(const bool weighted, // should only be reached if non-weighted. This is a by-product of // the two construction paths in panzer. Unification should help // fix the logic. - if(orientations_.size() > 0) applyOrientationsImpl(num_orientations_cells_, tmp_basis_scalar.get_view(), orientations_, *intrepid_basis); diff --git a/packages/panzer/disc-fe/src/Panzer_IntrepidOrientation.cpp b/packages/panzer/disc-fe/src/Panzer_IntrepidOrientation.cpp index 9c2d0956b963..238e22300ec3 100644 --- a/packages/panzer/disc-fe/src/Panzer_IntrepidOrientation.cpp +++ b/packages/panzer/disc-fe/src/Panzer_IntrepidOrientation.cpp @@ -130,4 +130,55 @@ namespace panzer { "panzer::buildIntrepidOrientation: Could not cast GlobalIndexer"); } + void buildIntrepidOrientations(const std::vector& eBlockNames, + const panzer::ConnManager & connMgrInput, + std::map> & orientations) + { + using Teuchos::rcp_dynamic_cast; + using Teuchos::RCP; + using Teuchos::rcp; + + auto connMgrPtr = connMgrInput.noConnectivityClone(); + auto& connMgr = *connMgrPtr; + + // Map element block name to topology + std::map eb_name_to_topo; + { + std::vector elementBlockIds; + connMgr.getElementBlockIds(elementBlockIds); + + std::vector elementBlockTopologies; + connMgr.getElementBlockTopologies(elementBlockTopologies); + + for (size_t i=0; i < elementBlockIds.size(); ++i) { + eb_name_to_topo[elementBlockIds[i]] = elementBlockTopologies[i]; + } + } + + // Currently panzer supports only one element topology for whole mesh (use the first cell topology) + const auto cellTopo = eb_name_to_topo[eBlockNames[0]]; + const int numVerticesPerCell = cellTopo.getVertexCount(); + + const auto fp = NodalFieldPattern(cellTopo); + connMgr.buildConnectivity(fp); + + // Loop over requested element blocks + for (size_t i=0;i + vertices(connMgr.getConnectivity(lid),numVerticesPerCell); + + // This function call expects a view for the vertices, not the nodes + orts[lid] = Intrepid2::Orientation::getOrientation(cellTopo, vertices); + } + } + } + } // end namespace panzer diff --git a/packages/panzer/disc-fe/src/Panzer_IntrepidOrientation.hpp b/packages/panzer/disc-fe/src/Panzer_IntrepidOrientation.hpp index 6b8cbb5ca542..105129424ddd 100644 --- a/packages/panzer/disc-fe/src/Panzer_IntrepidOrientation.hpp +++ b/packages/panzer/disc-fe/src/Panzer_IntrepidOrientation.hpp @@ -52,10 +52,33 @@ namespace panzer { + /** \brief Builds the element orientations for all element blocks. + + @param orientation [out] A vector that will be resized to the number of elements in the mesh and contain the orientaitons. + @param connMgr [in] Element connectivity interface. A no-connectivity clone will be created internally to generate vertex information. + */ void buildIntrepidOrientation(std::vector & orientation, panzer::ConnManager & connMgr); + /** \brief Builds the element orientations for the specified element blocks. + + @param eBlockNames [in] element block names to create orientations for. + @param connMgr [in] Element connectivity interface. A no-connectivity clone will be created internally to generate vertex information. + @param orientation [out] A map of Kokkos Views containing the orientations. The key is the element block name from the connMgr. Any previous views will be reallocated internally. + + @note: This is for L2 projections that operate on an element block basis. + */ + // template + // void + // buildIntrepidOrientations(const std::vector& eBlockNames, + // const panzer::ConnManager & connMgr, + // std::map> & orientations); + void + buildIntrepidOrientations(const std::vector& eBlockNames, + const panzer::ConnManager & connMgr, + std::map> & orientations); + /** Build an orientation container from a global indexer and a field. * Underneath this does several dynamic casts to determine the type of GlobalIndexer * object that has been passed in. diff --git a/packages/panzer/disc-fe/src/Panzer_L2Projection.cpp b/packages/panzer/disc-fe/src/Panzer_L2Projection.cpp index d145bd2b47a7..bf6bf838100d 100644 --- a/packages/panzer/disc-fe/src/Panzer_L2Projection.cpp +++ b/packages/panzer/disc-fe/src/Panzer_L2Projection.cpp @@ -37,6 +37,7 @@ namespace panzer { elementBlockNames_ = elementBlockNames; worksetContainer_ = worksetContainer; setupCalled_ = true; + useUserSuppliedBasisValues_ = false; // Build target DOF Manager targetGlobalIndexer_ = @@ -66,6 +67,18 @@ namespace panzer { // Check workset needs are correct } + void panzer::L2Projection::useBasisValues(const std::map>>& bv) + { + useUserSuppliedBasisValues_ = true; + basisValues_ = bv; + + // Check that the basis and integration descriptor match what was + // supplied in setup. + for (const auto& eblock : elementBlockNames_) { + TEUCHOS_ASSERT(basisValues_[eblock]->getBasisDescriptor()==targetBasisDescriptor_); + } + } + Teuchos::RCP panzer::L2Projection::getTargetGlobalIndexer() const {return targetGlobalIndexer_;} @@ -107,13 +120,49 @@ namespace panzer { if (elementBlockMultipliers != nullptr) ebMultiplier = elementBlockMultipliers->find(block)->second; + // Get the cell local ids and set the BasisValues object (BV + // can come from a user defined set or from WorksetContainer). + const panzer::BasisValues2* bv_ptr; + int num_cells_owned_ghosted_virtual = 0; + int num_cells_owned = 0; + Kokkos::View cell_local_ids; + if (useUserSuppliedBasisValues_) { + bv_ptr = basisValues_[block].get(); + auto tmp = connManager_->getElementBlock(block); // no ghosting or virtual in this case + Kokkos::View::HostMirror cell_local_ids_host(tmp.data(),tmp.size()); + Kokkos::View cell_local_ids_nonconst("cell_local_ids",tmp.size()); + Kokkos::deep_copy(cell_local_ids_nonconst,cell_local_ids_host); + cell_local_ids = cell_local_ids_nonconst; + num_cells_owned_ghosted_virtual = cell_local_ids.extent(0); + num_cells_owned = cell_local_ids.extent(0); + } + else { + // Based on descriptor, currently assumes there should only + // be one workset (partitioned path assumes a single + // workset). + panzer::WorksetDescriptor wd(block,panzer::WorksetSizeType::ALL_ELEMENTS,true,true); + const auto worksets = worksetContainer_->getWorksets(wd); + TEUCHOS_ASSERT(worksets->size() == 1); + const auto& workset = (*worksets)[0]; + bv_ptr = &(workset.getBasisValues(targetBasisDescriptor_,integrationDescriptor_)); + num_cells_owned_ghosted_virtual = workset.numOwnedCells()+workset.numGhostCells()+workset.numVirtualCells(); + num_cells_owned = workset.numOwnedCells(); + cell_local_ids = workset.getLocalCellIDs(); + } + const auto& basisValues = *bv_ptr; + + // Leaving the commented out workset loop in teh code for now + // in case we want to support the original workset + // construction path in the workset container that is used + // here. + // Based on descriptor, currently assumes there should only be one workset - panzer::WorksetDescriptor wd(block,panzer::WorksetSizeType::ALL_ELEMENTS,true,true); - const auto worksets = worksetContainer_->getWorksets(wd); + // panzer::WorksetDescriptor wd(block,panzer::WorksetSizeType::ALL_ELEMENTS,true,true); + // const auto worksets = worksetContainer_->getWorksets(wd); - for (const auto& workset : *worksets) { + // for (const auto& workset : *worksets) { - const auto basisValues = workset.getBasisValues(targetBasisDescriptor_,integrationDescriptor_); + // const auto basisValues = workset.getBasisValues(targetBasisDescriptor_,integrationDescriptor_); const auto unweightedBasis = basisValues.getBasisValues(false).get_static_view(); const auto weightedBasis = basisValues.getBasisValues(true).get_static_view(); @@ -128,17 +177,17 @@ namespace panzer { Kokkos::deep_copy(kOffsets, kOffsets_h); // Local Ids - PHX::View localIds("MassMatrix: LocalIds", workset.numOwnedCells()+workset.numGhostCells()+workset.numVirtualCells(), + PHX::View localIds("MassMatrix: LocalIds", num_cells_owned_ghosted_virtual, targetGlobalIndexer_->getElementBlockGIDCount(block)); // Remove the ghosted cell ids or the call to getElementLocalIds will spill array bounds - const auto cellLocalIdsNoGhost = Kokkos::subview(workset.getLocalCellIDs(),std::make_pair(0,workset.numOwnedCells())); + const auto cellLocalIdsNoGhost = Kokkos::subview(cell_local_ids,std::make_pair(0,num_cells_owned)); targetGlobalIndexer_->getElementLIDs(cellLocalIdsNoGhost,localIds); const int numBasisPoints = static_cast(weightedBasis.extent(1)); if ( use_lumping ) { - Kokkos::parallel_for(workset.numOwnedCells(),KOKKOS_LAMBDA (const int& cell) { + Kokkos::parallel_for(num_cells_owned,KOKKOS_LAMBDA (const int& cell) { double total_mass = 0.0, trace = 0.0; panzer::LocalOrdinal cLIDs[256]; @@ -176,7 +225,7 @@ namespace panzer { }); } else { - Kokkos::parallel_for(workset.numOwnedCells(),KOKKOS_LAMBDA (const int& cell) { + Kokkos::parallel_for(num_cells_owned,KOKKOS_LAMBDA (const int& cell) { panzer::LocalOrdinal cLIDs[256]; const int numIds = static_cast(localIds.extent(1)); @@ -201,7 +250,7 @@ namespace panzer { }); } - } + //} loop over worksets } } else { auto M = ghostedMatrix->getLocalMatrixDevice(); diff --git a/packages/panzer/disc-fe/src/Panzer_L2Projection.hpp b/packages/panzer/disc-fe/src/Panzer_L2Projection.hpp index a32042656f21..9a70d84877a5 100644 --- a/packages/panzer/disc-fe/src/Panzer_L2Projection.hpp +++ b/packages/panzer/disc-fe/src/Panzer_L2Projection.hpp @@ -28,6 +28,7 @@ namespace panzer { class DOFManager; class GlobalIndexer; class WorksetContainer; + template class BasisValues2; /** \brief Unified set of tools for building objects for lumped and consistent L2 projects between bases. @@ -42,6 +43,8 @@ namespace panzer { mutable Teuchos::RCP worksetContainer_; bool setupCalled_; Teuchos::RCP targetGlobalIndexer_; + bool useUserSuppliedBasisValues_; + std::map>> basisValues_; public: @@ -64,6 +67,16 @@ namespace panzer { const std::vector& elementBlockNames, const Teuchos::RCP worksetContainer = Teuchos::null); + /** \brief Override using the panzer::WorksetContainer and instead use the registered BasisValues object. + + \param[in] bv BasisValues object setup in lazy evalaution mode with registered orientations if required. + + The intention of this function is to minimize memory use and + redundant evaluations by avoiding workset construction and + enforcing lazy evaluation in basis values. + */ + void useBasisValues(const std::map>>& map_eblock_to_bv); + /// Returns the target global indexer. Will be null if setup() has not been called. Teuchos::RCP getTargetGlobalIndexer() const; From 89adc3ba3e86a4aef05f0ea2da795b0c6d3b61c6 Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Mon, 12 Feb 2024 09:38:23 -0700 Subject: [PATCH 8/8] Panzer: fix warning --- packages/panzer/disc-fe/src/Panzer_IntrepidOrientation.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/panzer/disc-fe/src/Panzer_IntrepidOrientation.cpp b/packages/panzer/disc-fe/src/Panzer_IntrepidOrientation.cpp index 238e22300ec3..e23b52888355 100644 --- a/packages/panzer/disc-fe/src/Panzer_IntrepidOrientation.cpp +++ b/packages/panzer/disc-fe/src/Panzer_IntrepidOrientation.cpp @@ -170,7 +170,7 @@ namespace panzer { auto& orts = orientations[eBlockNames[i]]; orts.resize(num_elems); - for (int c=0;c vertices(connMgr.getConnectivity(lid),numVerticesPerCell);