Skip to content

Commit

Permalink
Hypre: Use Gpu::hypreSynchronize (AMReX-Codes#4022)
Browse files Browse the repository at this point in the history 
Gpu::hypreSynchronize performs synchronization in Hypre's GPU stream.
Thus it should be more efficient than a device wide synchronization with
Gpu::synchronize().
  • Loading branch information
@WeiqunZhang
WeiqunZhang authored Jul 6, 2024
1 parent 3cd4640 commit 85ae84e
Show file tree
Hide file tree
Showing 5 changed files with 19 additions and 19 deletions.
6 changes: 3 additions & 3 deletions Src/Extern/HYPRE/AMReX_HypreABecLap.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ HypreABecLap::getSolution (MultiFab& a_soln)
auto reghi = Hypre::hiV(reg);
HYPRE_StructVectorGetBoxValues(x, reglo.data(), reghi.data(), (*soln)[mfi].dataPtr());
}
Gpu::synchronize();
Gpu::hypreSynchronize();

if (a_soln.nGrowVect() != 0) {
MultiFab::Copy(a_soln, tmp, 0, 0, 1, 0);
Expand Down Expand Up @@ -235,7 +235,7 @@ HypreABecLap::prepareSolver ()
HYPRE_StructMatrixSetBoxValues(A, reglo.data(), reghi.data(),
regular_stencil_size, stencil_indices.data(),
mat);
Gpu::synchronize();
Gpu::hypreSynchronize();
}
HYPRE_StructMatrixAssemble(A);

Expand Down Expand Up @@ -299,7 +299,7 @@ HypreABecLap::loadVectors (MultiFab& soln, const MultiFab& rhs)
HYPRE_StructVectorSetBoxValues(x, reglo.data(), reghi.data(), soln[mfi].dataPtr());
HYPRE_StructVectorSetBoxValues(b, reglo.data(), reghi.data(), rhs_diag[mfi].dataPtr());
}
Gpu::synchronize();
Gpu::hypreSynchronize();
}

}
6 changes: 3 additions & 3 deletions Src/Extern/HYPRE/AMReX_HypreABecLap2.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ HypreABecLap2::getSolution (MultiFab& a_soln)
HYPRE_SStructVectorGetBoxValues(x, part, reglo.data(), reghi.data(),
0, (*soln)[mfi].dataPtr());
}
Gpu::synchronize();
Gpu::hypreSynchronize();

if (a_soln.nGrowVect() != 0) {
MultiFab::Copy(a_soln, tmp, 0, 0, 1, 0);
Expand Down Expand Up @@ -262,7 +262,7 @@ HypreABecLap2::prepareSolver ()
HYPRE_SStructMatrixSetBoxValues(A, part, reglo.data(), reghi.data(),
0, regular_stencil_size, stencil_indices.data(),
mat);
Gpu::synchronize();
Gpu::hypreSynchronize();
}
HYPRE_SStructMatrixAssemble(A);

Expand Down Expand Up @@ -335,7 +335,7 @@ HypreABecLap2::loadVectors (MultiFab& soln, const MultiFab& rhs)
HYPRE_SStructVectorSetBoxValues(b, part, reglo.data(), reghi.data(),
0, rhs_diag[mfi].dataPtr());
}
Gpu::synchronize();
Gpu::hypreSynchronize();
}

}
8 changes: 4 additions & 4 deletions Src/Extern/HYPRE/AMReX_HypreABecLap3.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ HypreABecLap3::getSolution (MultiFab& a_soln)
(*l_soln)[mfi].setVal<RunOn::Device>(0.0);
}
}
Gpu::synchronize();
Gpu::hypreSynchronize();

if (use_tmp_mf) {
MultiFab::Copy(a_soln, tmp, 0, 0, 1, 0);
Expand Down Expand Up @@ -499,9 +499,9 @@ HypreABecLap3::prepareSolver ()
});
}

Gpu::synchronize();
Gpu::streamSynchronize();
HYPRE_IJMatrixSetValues(A,nrows,ncols,rows,cols,mat);
Gpu::synchronize();
Gpu::hypreSynchronize();
}
}
HYPRE_IJMatrixAssemble(A);
Expand Down Expand Up @@ -681,7 +681,7 @@ HypreABecLap3::loadVectors (MultiFab& soln, const MultiFab& rhs)
HYPRE_IJVectorSetValues(b, nrows, cell_id_vec[mfi].dataPtr(), rhs_diag[mfi].dataPtr());
}
}
Gpu::synchronize();
Gpu::hypreSynchronize();
}

} // namespace amrex
12 changes: 6 additions & 6 deletions Src/Extern/HYPRE/AMReX_HypreNodeLap.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,9 +123,9 @@ HypreNodeLap::HypreNodeLap (const BoxArray& grids_, const DistributionMapping& d
adjust_singular_matrix(ncols, cols, rows, mat);
}

Gpu::synchronize();
Gpu::streamSynchronize();
HYPRE_IJMatrixSetValues(A, nrows, ncols, rows, cols, mat);
Gpu::synchronize();
Gpu::hypreSynchronize();
}
}
HYPRE_IJMatrixAssemble(A);
Expand Down Expand Up @@ -324,9 +324,9 @@ HypreNodeLap::loadVectors (MultiFab& soln, const MultiFab& rhs)
});
}

Gpu::synchronize();
Gpu::streamSynchronize();
HYPRE_IJVectorSetValues(b, nrows, rows_vec.data(), bvec.data());
Gpu::synchronize();
Gpu::hypreSynchronize();
}
}
}
Expand All @@ -347,7 +347,7 @@ HypreNodeLap::getSolution (MultiFab& soln)
xvec.resize(nrows);
Real* xp = xvec.data();
HYPRE_IJVectorGetValues(x, nrows, rows_vec.data(), xp);
Gpu::synchronize();
Gpu::hypreSynchronize();

const Box& bx = mfi.validbox();
const auto& xfab = tmpsoln.array(mfi);
Expand All @@ -359,7 +359,7 @@ HypreNodeLap::getSolution (MultiFab& soln)
}
});

Gpu::synchronize();
Gpu::streamSynchronize();
}
}

Expand Down
6 changes: 3 additions & 3 deletions Src/Extern/HYPRE/AMReX_HypreSolver.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -569,7 +569,7 @@ HypreSolver<MSS>::fill_matrix (Filler const& filler)
Gpu::streamSynchronize();
HYPRE_IJMatrixSetValues(m_A, nrows, ncols_vec.data(), rows,
cols_vec.data(), mat_vec.data());
Gpu::synchronize();
Gpu::hypreSynchronize();
}
}
HYPRE_IJMatrixAssemble(m_A);
Expand Down Expand Up @@ -653,7 +653,7 @@ HypreSolver<MSS>::load_vectors (Vector<MF *> const& a_soln,
Gpu::streamSynchronize();
HYPRE_IJVectorSetValues(m_x, nrows, rows, xp);
HYPRE_IJVectorSetValues(m_b, nrows, rows, bp);
Gpu::synchronize();
Gpu::hypreSynchronize();
}
}
}
Expand Down Expand Up @@ -683,7 +683,7 @@ HypreSolver<MSS>::get_solution (Vector<MF*> const& a_soln)
HYPRE_Int const* rows = m_global_id_vec[mfi].data();

HYPRE_IJVectorGetValues(m_x, nrows, rows, xp);
Gpu::synchronize();
Gpu::hypreSynchronize();

HYPRE_Int offset = 0;
for (int ivar = 0; ivar < m_nvars; ++ivar) {
Expand Down

0 comments on commit 85ae84e

Please sign in to comment.