Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Porting fortran files to cpp in Src/Extern/HYPRE #1049

Merged
merged 22 commits into from
Jul 6, 2020
Merged

Porting fortran files to cpp in Src/Extern/HYPRE #1049

Show file tree
Hide file tree
Changes from 12 commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
95e41a8
Porting fortran files to cpp in Src/Extern/HYPRE
OscarAntepara Jun 24, 2020
5422f55
Adding files .H and .cpp
OscarAntepara Jun 24, 2020
c0a3440
Merge branch 'development' of https://github.com/AMReX-Codes/amrex in…
OscarAntepara Jun 24, 2020
1e3b70d
Fixing long integer problem
OscarAntepara Jun 25, 2020
d1c0ca1
Merge branch 'development' of https://github.com/AMReX-Codes/amrex in…
OscarAntepara Jun 25, 2020
dc66436
Merge branch 'development' of https://github.com/AMReX-Codes/amrex in…
OscarAntepara Jun 27, 2020
52f67b7
Fixing petsc issue
OscarAntepara Jun 27, 2020
e1f8edf
Merge branch 'development' of https://github.com/AMReX-Codes/amrex in…
OscarAntepara Jun 27, 2020
97f5ea7
Including templates for HYPRE_Int or PetscInt
OscarAntepara Jun 27, 2020
169f9c8
Merge branch 'development' of https://github.com/AMReX-Codes/amrex in…
OscarAntepara Jun 29, 2020
8980384
Forgot to modify this file
OscarAntepara Jun 29, 2020
4d90d38
Update Src/Extern/HYPRE/AMReX_HypreABecLap.cpp
OscarAntepara Jun 30, 2020
edb13dc
Fixing issues and adding AMReX_Habec_K.H
OscarAntepara Jun 30, 2020
59e2f59
Fixing issues to compile on gpus
OscarAntepara Jun 30, 2020
524c359
Merge branch 'development' of https://github.com/AMReX-Codes/amrex in…
OscarAntepara Jul 1, 2020
04fc02b
Fixing issues of repeated functions
OscarAntepara Jul 1, 2020
5907804
Merge branch 'development' of https://github.com/AMReX-Codes/amrex in…
OscarAntepara Jul 1, 2020
092051a
Putting SafeGuards for gpu-runs
OscarAntepara Jul 1, 2020
5988409
Merge branch 'development' of https://github.com/AMReX-Codes/amrex in…
OscarAntepara Jul 4, 2020
e47b0ea
Fixing iterator
OscarAntepara Jul 4, 2020
25d6aca
Update Src/Extern/HYPRE/AMReX_Habec_2D_K.H
OscarAntepara Jul 6, 2020
dabfae0
Looping over slices
OscarAntepara Jul 6, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
713 changes: 0 additions & 713 deletions Src/Extern/HYPRE/AMReX_HABEC_2D.F90
View file
Open in desktop

This file was deleted.

1,184 changes: 0 additions & 1,184 deletions Src/Extern/HYPRE/AMReX_HABEC_3D.F90
View file
Open in desktop

This file was deleted.

778 changes: 778 additions & 0 deletions Src/Extern/HYPRE/AMReX_Habec_2D_K.H
View file
Open in desktop

Large diffs are not rendered by default.

1,202 changes: 1,202 additions & 0 deletions Src/Extern/HYPRE/AMReX_Habec_3D_K.H
View file
Open in desktop

Large diffs are not rendered by default.

38 changes: 15 additions & 23 deletions Src/Extern/HYPRE/AMReX_HypreABecLap.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,14 @@

#include <AMReX_HypreABecLap.H>
#include <AMReX_HypreABec_F.H>
#include <string>
#include <algorithm>

#if (AMREX_SPACEDIM == 2)
#include <AMReX_Habec_2D_K.H>
#elif (AMREX_SPACEDIM == 3)
#include <AMReX_Habec_3D_K.H>
#endif

#include <_hypre_struct_mv.h>

namespace amrex {
Expand Down Expand Up @@ -195,24 +200,16 @@ HypreABecLap::prepareSolver ()
std::iota(stencil_indices.begin(), stencil_indices.end(), 0);
const Real* dx = geom.CellSize();
const int bho = (m_maxorder > 2) ? 1 : 0;
FArrayBox rfab;
BaseFab<GpuArray<Real,regular_stencil_size>> rfab;
for (MFIter mfi(acoefs); mfi.isValid(); ++mfi)
{
const Box &reg = mfi.validbox();

rfab.resize(reg,regular_stencil_size);
Real* mat = rfab.dataPtr();

amrex_hpacoef(BL_TO_FORTRAN_BOX(reg),
mat,
BL_TO_FORTRAN_ANYD(acoefs[mfi]),
&scalar_a);
rfab.resize(reg);
amrex_hpacoef(reg, rfab, acoefs[mfi], scalar_a);

for (int idim = 0; idim < AMREX_SPACEDIM; idim++) {
amrex_hpbcoef(BL_TO_FORTRAN_BOX(reg),
mat,
BL_TO_FORTRAN_ANYD(bcoefs[idim][mfi]),
&scalar_b, dx, &idim);
amrex_hpbcoef(reg, rfab, bcoefs[idim][mfi], scalar_b, dx, idim);
}

const Vector< Vector<BoundCond> > & bcs_i = m_bndry->bndryConds(mfi);
Expand All @@ -227,17 +224,12 @@ HypreABecLap::prepareSolver ()
const int bctype = bcs_i[cdir][0];
const Real &bcl = bcl_i[cdir];
const Mask &msk = m_bndry->bndryMasks(ori)[mfi];

amrex_hpmat(BL_TO_FORTRAN_BOX(reg),
mat,
BL_TO_FORTRAN_ANYD(bcoefs[idim][mfi]),
BL_TO_FORTRAN_ANYD(msk),
&scalar_b, dx, &cdir, &bctype, &bcl, &bho);

amrex_hpmat(reg, rfab, bcoefs[idim][mfi], msk, scalar_b, dx, cdir, bctype, bcl, bho);
}

amrex_hpdiag(BL_TO_FORTRAN_BOX(reg),
mat,
BL_TO_FORTRAN_ANYD(diaginv[mfi]));

amrex_hpdiag(reg, rfab, diaginv[mfi]);
Real* mat = (Real*) rfab.dataPtr();

auto reglo = Hypre::loV(reg);
auto reghi = Hypre::hiV(reg);
Expand Down
32 changes: 12 additions & 20 deletions Src/Extern/HYPRE/AMReX_HypreABecLap2.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
#include <AMReX_HypreABecLap2.H>
#include <AMReX_HypreABec_F.H>

#if (AMREX_SPACEDIM == 2)
#include <AMReX_Habec_2D_K.H>
#elif (AMREX_SPACEDIM == 3)
#include <AMReX_Habec_3D_K.H>
#endif

#include <cmath>
#include <numeric>
Expand Down Expand Up @@ -219,24 +224,16 @@ HypreABecLap2::prepareSolver ()
const HYPRE_Int part = 0;
const Real* dx = geom.CellSize();
const int bho = (m_maxorder > 2) ? 1 : 0;
FArrayBox rfab;
BaseFab<GpuArray<Real, regular_stencil_size>> rfab;
for (MFIter mfi(acoefs); mfi.isValid(); ++mfi)
{
const Box &reg = mfi.validbox();

rfab.resize(reg,regular_stencil_size);
Real* mat = rfab.dataPtr();

amrex_hpacoef(BL_TO_FORTRAN_BOX(reg),
mat,
BL_TO_FORTRAN_ANYD(acoefs[mfi]),
&scalar_a);
amrex_hpacoef(reg, rfab, acoefs[mfi], scalar_a);

for (int idim = 0; idim < AMREX_SPACEDIM; idim++) {
amrex_hpbcoef(BL_TO_FORTRAN_BOX(reg),
mat,
BL_TO_FORTRAN_ANYD(bcoefs[idim][mfi]),
&scalar_b, dx, &idim);
amrex_hpbcoef(reg, rfab, bcoefs[idim][mfi], scalar_b, dx, idim);
}

const Vector< Vector<BoundCond> > & bcs_i = m_bndry->bndryConds(mfi);
Expand All @@ -251,16 +248,11 @@ HypreABecLap2::prepareSolver ()
const Real &bcl = bcl_i[cdir];
const Mask &msk = m_bndry->bndryMasks(ori)[mfi];

amrex_hpmat(BL_TO_FORTRAN_BOX(reg),
mat,
BL_TO_FORTRAN_ANYD(bcoefs[idim][mfi]),
BL_TO_FORTRAN_ANYD(msk),
&scalar_b, dx, &cdir, &bctype, &bcl, &bho);
amrex_hpmat(reg, rfab, bcoefs[idim][mfi], msk, scalar_b, dx, cdir, bctype, bcl, bho);
}

amrex_hpdiag(BL_TO_FORTRAN_BOX(reg),
mat,
BL_TO_FORTRAN_ANYD(diaginv[mfi]));
amrex_hpdiag(reg, rfab, diaginv[mfi]);
Real* mat = (Real*) rfab.dataPtr();

// initialize matrix
auto reglo = Hypre::loV(reg);
Expand Down
2 changes: 1 addition & 1 deletion Src/Extern/HYPRE/AMReX_HypreABecLap3.H
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ private :
HYPRE_Solver solver = NULL;

LayoutData<HYPRE_Int> ncells_grid;
LayoutData<Vector<HYPRE_Int> > cell_id_vec;
LayoutData<Gpu::ManagedDeviceVector<HYPRE_Int> > cell_id_vec;
FabArray<BaseFab<HYPRE_Int> > cell_id;

MultiFab const* m_eb_b_coeffs = nullptr;
Expand Down
135 changes: 76 additions & 59 deletions Src/Extern/HYPRE/AMReX_HypreABecLap3.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,12 @@
#include <AMReX_HypreABecLap3.H>
#include <AMReX_HypreABec_F.H>
#include <AMReX_VisMF.H>

#if (AMREX_SPACEDIM == 2)
#include <AMReX_Habec_2D_K.H>
#elif (AMREX_SPACEDIM == 3)
#include <AMReX_Habec_3D_K.H>
#endif

#ifdef AMREX_USE_EB
#include <AMReX_EBMultiFabUtil.H>
#include <AMReX_MultiCutFab.H>
Expand Down Expand Up @@ -39,7 +44,7 @@ HypreABecLap3::solve (MultiFab& soln, const MultiFab& rhs, Real rel_tol, Real ab
int max_iter, const BndryData& bndry, int max_bndry_order)
{
BL_PROFILE("HypreABecLap3::solve()");

if (solver == NULL || m_bndry != &bndry || m_maxorder != max_bndry_order)
{
m_bndry = &bndry;
Expand Down Expand Up @@ -149,10 +154,10 @@ HypreABecLap3::getSolution (MultiFab& soln)
#ifdef AMREX_USE_EB
else if (fabtyp != FabType::regular)
{
amrex_hpeb_copy_from_vec(BL_TO_FORTRAN_BOX(bx),
BL_TO_FORTRAN_ANYD(soln[mfi]),
xfab->dataPtr(), &nrows,
BL_TO_FORTRAN_ANYD((*flags)[mfi]));
amrex_hpeb_copy_from_vec(bx,
soln[mfi],
xfab->dataPtr(),
(*flags)[mfi]);
}
#endif
}
Expand Down Expand Up @@ -214,10 +219,11 @@ HypreABecLap3::prepareSolver ()
}
else if (fabtyp == FabType::singlevalued)
{
amrex_hpeb_fill_cellid(BL_TO_FORTRAN_BOX(bx),
&ncells_grid[mfi],
BL_TO_FORTRAN_ANYD(cid_fab),
BL_TO_FORTRAN_ANYD((*flags)[mfi]));
amrex_hpeb_fill_cellid(bx,
ncells_grid[mfi],
cid_fab,
(*flags)[mfi]);

ncells_proc += ncells_grid[mfi];
}
else
Expand Down Expand Up @@ -291,8 +297,6 @@ HypreABecLap3::prepareSolver ()

const Real* dx = geom.CellSize();
const int bho = (m_maxorder > 2) ? 1 : 0;
FArrayBox rfab;
BaseFab<HYPRE_Int> ifab;
FArrayBox foo(Box::TheUnitBox());
const int is_eb_dirichlet = m_eb_b_coeffs != nullptr;

Expand All @@ -310,18 +314,15 @@ HypreABecLap3::prepareSolver ()
const HYPRE_Int max_stencil_size = (fabtyp == FabType::regular) ?
regular_stencil_size : eb_stencil_size;

ifab.resize(bx,(max_stencil_size+1));
rfab.resize(bx,max_stencil_size);

const HYPRE_Int nrows = ncells_grid[mfi];
cell_id_vec[mfi].resize(nrows);
HYPRE_Int* rows = cell_id_vec[mfi].data();
HYPRE_Int* ncols = ifab.dataPtr(0);
HYPRE_Int* cols = ifab.dataPtr(1);
Real* mat = rfab.dataPtr();

Array<int,AMREX_SPACEDIM*2> bctype;
Array<Real,AMREX_SPACEDIM*2> bcl;
Gpu::ManagedDeviceVector<HYPRE_Int> ncolsg(nrows,0);
Gpu::ManagedDeviceVector<HYPRE_Int> colsg(nrows*(AMREX_SPACEDIM*2+1),0);
Gpu::ManagedDeviceVector<Real> matg(nrows*(AMREX_SPACEDIM*2+1),0.0);

GpuArray<int,AMREX_SPACEDIM*2> bctype;
GpuArray<Real,AMREX_SPACEDIM*2> bcl;
const Vector< Vector<BoundCond> > & bcs_i = m_bndry->bndryConds(mfi);
const BndryData::RealTuple & bcl_i = m_bndry->bndryLocs(mfi);
for (OrientationIter oit; oit; oit++) {
Expand All @@ -332,48 +333,65 @@ HypreABecLap3::prepareSolver ()

if (fabtyp == FabType::regular)
{
amrex_hpijmatrix(BL_TO_FORTRAN_BOX(bx),
&nrows, ncols, rows, cols, mat,
BL_TO_FORTRAN_ANYD(cell_id[mfi]),
&(offset[mfi]),
BL_TO_FORTRAN_ANYD(diaginv[mfi]),
BL_TO_FORTRAN_ANYD(acoefs[mfi]),
AMREX_D_DECL(BL_TO_FORTRAN_ANYD(bcoefs[0][mfi]),
BL_TO_FORTRAN_ANYD(bcoefs[1][mfi]),
BL_TO_FORTRAN_ANYD(bcoefs[2][mfi])),
&scalar_a, &scalar_b, dx,
bctype.data(), bcl.data(), &bho);
amrex_hpijmatrix(bx,
nrows, ncolsg.dataPtr(),
cell_id_vec[mfi].dataPtr(),
colsg.dataPtr(), matg.dataPtr(),
cell_id[mfi],
offset[mfi],
diaginv[mfi],
acoefs[mfi],
bcoefs[0][mfi],
bcoefs[1][mfi],
#if (AMREX_SPACEDIM == 3)
bcoefs[2][mfi],
#endif
scalar_a, scalar_b, dx,
bctype, bcl, bho);

}
#ifdef AMREX_USE_EB
else
{
FArrayBox const& beb = (is_eb_dirichlet) ? (*m_eb_b_coeffs)[mfi] : foo;
int size_vec = std::pow(3,AMREX_SPACEDIM);
colsg.resize(nrows*size_vec);
matg.resize(nrows*size_vec);
amrex_hpeb_ijmatrix(bx,
nrows, ncolsg.dataPtr(),
cell_id_vec[mfi].dataPtr(),
colsg.dataPtr(), matg.dataPtr(),
cell_id[mfi],
offset[mfi], diaginv[mfi],
acoefs[mfi], bcoefs[0][mfi],
bcoefs[1][mfi],
#if (AMREX_SPACEDIM == 3)
bcoefs[2][mfi],
#endif
(*flags)[mfi],
(*vfrac)[mfi],
(*area[0])[mfi], (*area[1])[mfi],
#if (AMREX_SPACEDIM == 3)
(*area[2])[mfi],
#endif
(*fcent[0])[mfi], (*fcent[1])[mfi],
#if (AMREX_SPACEDIM == 3)
(*fcent[2])[mfi],
#endif
(*barea)[mfi],
(*bcent)[mfi],
beb, is_eb_dirichlet,
scalar_a, scalar_b, dx,
bctype, bcl, bho);

amrex_hpeb_ijmatrix(BL_TO_FORTRAN_BOX(bx),
&nrows, ncols, rows, cols, mat,
BL_TO_FORTRAN_ANYD(cell_id[mfi]),
&(offset[mfi]),
BL_TO_FORTRAN_ANYD(diaginv[mfi]),
BL_TO_FORTRAN_ANYD(acoefs[mfi]),
AMREX_D_DECL(BL_TO_FORTRAN_ANYD(bcoefs[0][mfi]),
BL_TO_FORTRAN_ANYD(bcoefs[1][mfi]),
BL_TO_FORTRAN_ANYD(bcoefs[2][mfi])),
BL_TO_FORTRAN_ANYD((*flags)[mfi]),
BL_TO_FORTRAN_ANYD((*vfrac)[mfi]),
AMREX_D_DECL(BL_TO_FORTRAN_ANYD((*area[0])[mfi]),
BL_TO_FORTRAN_ANYD((*area[1])[mfi]),
BL_TO_FORTRAN_ANYD((*area[2])[mfi])),
AMREX_D_DECL(BL_TO_FORTRAN_ANYD((*fcent[0])[mfi]),
BL_TO_FORTRAN_ANYD((*fcent[1])[mfi]),
BL_TO_FORTRAN_ANYD((*fcent[2])[mfi])),
BL_TO_FORTRAN_ANYD((*barea)[mfi]),
BL_TO_FORTRAN_ANYD((*bcent)[mfi]),
BL_TO_FORTRAN_ANYD(beb), &is_eb_dirichlet,
&scalar_a, &scalar_b, dx,
bctype.data(), bcl.data(), &bho);
}
#endif

HYPRE_Int* rows = cell_id_vec[mfi].data();
HYPRE_Int* ncols = (HYPRE_Int*) ncolsg.dataPtr();
HYPRE_Int* cols = (HYPRE_Int*) colsg.dataPtr();
Real* mat = (Real*) matg.dataPtr();

#ifdef AMREX_DEBUG
HYPRE_Int nvalues = 0;
for (HYPRE_Int i = 0; i < nrows; ++i) {
Expand All @@ -384,7 +402,6 @@ HypreABecLap3::prepareSolver ()
}
#endif


HYPRE_IJMatrixSetValues(A,nrows,ncols,rows,cols,mat);
}
}
Expand Down Expand Up @@ -447,10 +464,10 @@ HypreABecLap3::loadVectors (MultiFab& soln, const MultiFab& rhs)
{
bfab = &vecfab;
bfab->resize(bx);
amrex_hpeb_copy_to_vec(BL_TO_FORTRAN_BOX(bx),
BL_TO_FORTRAN_ANYD(rhsfab),
bfab->dataPtr(), &nrows,
BL_TO_FORTRAN_ANYD((*flags)[mfi]));
amrex_hpeb_copy_to_vec(bx,
rhsfab,
bfab->dataPtr(),
(*flags)[mfi]);
}
else
#endif
Expand Down
Loading