Skip to content

Commit

Permalink
Turn on reflux 2 (erf-model#1289)
Browse files Browse the repository at this point in the history 
* updates associated with multilevel coupling options and some clean-up of advection

* formatting

* for CUDA compile

* fix

* white space

* fix for CUDA

* fix CUDA issue

* fix oops
  • Loading branch information
@asalmgren
asalmgren authored Nov 13, 2023
1 parent e79c8e1 commit 52c09fd
Show file tree
Hide file tree
Showing 20 changed files with 540 additions and 700 deletions.
44 changes: 20 additions & 24 deletions Source/Advection/Advection.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,48 +4,44 @@
#include <AMReX.H>
#include <AMReX_MultiFab.H>
#include <AMReX_BCRec.H>
#include <AMReX_InterpFaceRegister.H>
#include <AMReX_YAFluxRegister.H>
#include <DataStruct.H>
#include <IndexDefines.H>
#include <ABLMost.H>


/** Compute advection tendency for density and potential temperature */
void AdvectionSrcForRhoAndTheta (const amrex::Box& bx, const amrex::Box& valid_bx,
const amrex::Array4<amrex::Real>& src,
const amrex::Array4<const amrex::Real>& rho_u, // These are being used
const amrex::Array4<const amrex::Real>& rho_v, // to define the fluxes
const amrex::Array4<const amrex::Real>& omega,
amrex::Real fac,
const amrex::Array4< amrex::Real>& avg_xmom, // These are being defined
const amrex::Array4< amrex::Real>& avg_ymom, // from the rho fluxes
const amrex::Array4< amrex::Real>& avg_zmom,
const amrex::Array4<const amrex::Real>& cell_prim,
const amrex::Array4<const amrex::Real>& z_nd,
const amrex::Array4<const amrex::Real>& detJ,
const amrex::GpuArray<amrex::Real, AMREX_SPACEDIM>& cellSize,
const amrex::Array4<const amrex::Real>& mf_m,
const amrex::Array4<const amrex::Real>& mf_u,
const amrex::Array4<const amrex::Real>& mf_v,
const AdvType horiz_adv_type, const AdvType vert_adv_type,
const bool use_terrain);
void AdvectionSrcForRho (const amrex::Box& bx, const amrex::Box& valid_bx,
const amrex::Array4<amrex::Real>& src,
const amrex::Array4<const amrex::Real>& rho_u, // These are being used
const amrex::Array4<const amrex::Real>& rho_v, // to define the fluxes
const amrex::Array4<const amrex::Real>& omega,
const amrex::Array4< amrex::Real>& avg_xmom, // These are being defined
const amrex::Array4< amrex::Real>& avg_ymom, // from the rho fluxes
const amrex::Array4< amrex::Real>& avg_zmom,
const amrex::Array4<const amrex::Real>& z_nd,
const amrex::Array4<const amrex::Real>& detJ,
const amrex::GpuArray<amrex::Real, AMREX_SPACEDIM>& cellSize,
const amrex::Array4<const amrex::Real>& mf_m,
const amrex::Array4<const amrex::Real>& mf_u,
const amrex::Array4<const amrex::Real>& mf_v,
const bool use_terrain,
const amrex::GpuArray<const amrex::Array4<amrex::Real>, AMREX_SPACEDIM>& flx_arr);

/** Compute advection tendency for all scalars other than density and potential temperature */
void AdvectionSrcForScalars (int level, int finest_level,
const amrex::MFIter& mfi,
const amrex::Box& bx,
void AdvectionSrcForScalars (const amrex::Box& bx,
const int icomp, const int ncomp,
const amrex::Array4<const amrex::Real>& avg_xmom,
const amrex::Array4<const amrex::Real>& avg_ymom,
const amrex::Array4<const amrex::Real>& avg_zmom,
const amrex::Array4<const amrex::Real>& cell_prim,
const amrex::Array4<amrex::Real>& src,
const amrex::Array4<const amrex::Real>& detJ,
const amrex::Real dt,
const amrex::GpuArray<amrex::Real, AMREX_SPACEDIM>& cellSize,
const amrex::Array4<const amrex::Real>& mf_m,
const AdvType horiz_adv_type, const AdvType vert_adv_type,
const bool use_terrain, const bool is_two_way_coupling);
const bool use_terrain,
const amrex::GpuArray<const amrex::Array4<amrex::Real>, AMREX_SPACEDIM>& flx_arr);

/** Compute advection tendencies for all components of momentum */
void AdvectionSrcForMom (const amrex::Box& bxx, const amrex::Box& bxy, const amrex::Box& bxz,
Expand Down
32 changes: 16 additions & 16 deletions Source/Advection/AdvectionSrcForScalars.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,13 @@
*/
template<typename InterpType_H, typename InterpType_V>
void
AdvectionSrcForScalarsWrapper_N(const amrex::Box& bx,
const int& ncomp, const int& icomp,
const amrex::GpuArray<const amrex::Array4<amrex::Real>, AMREX_SPACEDIM> flx_arr,
const amrex::Array4<const amrex::Real>& cell_prim,
const amrex::Array4<const amrex::Real>& avg_xmom,
const amrex::Array4<const amrex::Real>& avg_ymom,
const amrex::Array4<const amrex::Real>& avg_zmom)
AdvectionSrcForScalarsWrapper(const amrex::Box& bx,
const int& ncomp, const int& icomp,
const amrex::GpuArray<const amrex::Array4<amrex::Real>, AMREX_SPACEDIM> flx_arr,
const amrex::Array4<const amrex::Real>& cell_prim,
const amrex::Array4<const amrex::Real>& avg_xmom,
const amrex::Array4<const amrex::Real>& avg_ymom,
const amrex::Array4<const amrex::Real>& avg_zmom)
{
// Instantiate structs for vert/horiz interp
InterpType_H interp_prim_h(cell_prim);
Expand All @@ -30,7 +30,7 @@ AdvectionSrcForScalarsWrapper_N(const amrex::Box& bx,
amrex::Real interpx(0.);

interp_prim_h.InterpolateInX(i,j,k,prim_index,interpx,avg_xmom(i ,j ,k ));
(flx_arr[0])(i,j,k,n) = avg_xmom(i,j,k) * interpx;
(flx_arr[0])(i,j,k,cons_index) = avg_xmom(i,j,k) * interpx;
});
amrex::ParallelFor(ybx, ncomp,[=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
{
Expand All @@ -40,7 +40,7 @@ AdvectionSrcForScalarsWrapper_N(const amrex::Box& bx,
amrex::Real interpy(0.);
interp_prim_h.InterpolateInY(i,j,k,prim_index,interpy,avg_ymom(i ,j ,k ));

(flx_arr[1])(i,j,k,n) = avg_ymom(i,j,k) * interpy;
(flx_arr[1])(i,j,k,cons_index) = avg_ymom(i,j,k) * interpy;
});
amrex::ParallelFor(zbx, ncomp,[=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
{
Expand All @@ -51,7 +51,7 @@ AdvectionSrcForScalarsWrapper_N(const amrex::Box& bx,

interp_prim_v.InterpolateInZ(i,j,k,prim_index,interpz,avg_zmom(i ,j ,k ));

(flx_arr[2])(i,j,k,n) = avg_zmom(i,j,k) * interpz;
(flx_arr[2])(i,j,k,cons_index) = avg_zmom(i,j,k) * interpz;
});
}

Expand All @@ -60,7 +60,7 @@ AdvectionSrcForScalarsWrapper_N(const amrex::Box& bx,
*/
template<typename InterpType_H>
void
AdvectionSrcForScalarsVert_N(const amrex::Box& bx,
AdvectionSrcForScalarsVert(const amrex::Box& bx,
const int& ncomp, const int& icomp,
const amrex::GpuArray<const amrex::Array4<amrex::Real>, AMREX_SPACEDIM> flx_arr,
const amrex::Array4<const amrex::Real>& cell_prim,
Expand All @@ -71,27 +71,27 @@ AdvectionSrcForScalarsVert_N(const amrex::Box& bx,
{
switch(vert_adv_type) {
case AdvType::Centered_2nd:
AdvectionSrcForScalarsWrapper_N<InterpType_H,CENTERED2>(bx, ncomp, icomp,
AdvectionSrcForScalarsWrapper<InterpType_H,CENTERED2>(bx, ncomp, icomp,
flx_arr, cell_prim,
avg_xmom, avg_ymom, avg_zmom);
break;
case AdvType::Upwind_3rd:
AdvectionSrcForScalarsWrapper_N<InterpType_H,UPWIND3>(bx, ncomp, icomp,
AdvectionSrcForScalarsWrapper<InterpType_H,UPWIND3>(bx, ncomp, icomp,
flx_arr, cell_prim,
avg_xmom, avg_ymom, avg_zmom);
break;
case AdvType::Centered_4th:
AdvectionSrcForScalarsWrapper_N<InterpType_H,CENTERED4>(bx, ncomp, icomp,
AdvectionSrcForScalarsWrapper<InterpType_H,CENTERED4>(bx, ncomp, icomp,
flx_arr, cell_prim,
avg_xmom, avg_ymom, avg_zmom);
break;
case AdvType::Upwind_5th:
AdvectionSrcForScalarsWrapper_N<InterpType_H,UPWIND5>(bx, ncomp, icomp,
AdvectionSrcForScalarsWrapper<InterpType_H,UPWIND5>(bx, ncomp, icomp,
flx_arr, cell_prim,
avg_xmom, avg_ymom, avg_zmom);
break;
case AdvType::Centered_6th:
AdvectionSrcForScalarsWrapper_N<InterpType_H,CENTERED6>(bx, ncomp, icomp,
AdvectionSrcForScalarsWrapper<InterpType_H,CENTERED6>(bx, ncomp, icomp,
flx_arr, cell_prim,
avg_xmom, avg_ymom, avg_zmom);
break;
Expand Down
Loading

0 comments on commit 52c09fd

Please sign in to comment.