From 2f70432761485bc6a4c65a1833e7299dd2c340e2 Mon Sep 17 00:00:00 2001 From: Luc Berger-Vergiat Date: Thu, 9 Nov 2023 12:15:10 -0700 Subject: [PATCH] More fixes for GPUs only in tests this time. --- ode/impl/KokkosODE_BDF_impl.hpp | 2 +- ode/src/KokkosODE_BDF.hpp | 6 ++- ode/unit_test/Test_ODE_BDF.hpp | 79 +++++++++++++++++++++++++-------- 3 files changed, 66 insertions(+), 21 deletions(-) diff --git a/ode/impl/KokkosODE_BDF_impl.hpp b/ode/impl/KokkosODE_BDF_impl.hpp index d8216ca03a..6c27dd37ea 100644 --- a/ode/impl/KokkosODE_BDF_impl.hpp +++ b/ode/impl/KokkosODE_BDF_impl.hpp @@ -336,7 +336,7 @@ KOKKOS_FUNCTION void BDFStep(ode_type& ode, scalar_type& t, scalar_type& dt, scalar_type max_step = Kokkos::ArithTraits::max(); scalar_type min_step = Kokkos::ArithTraits::min(); - scalar_type safety, error_norm; + scalar_type safety = 0.675, error_norm; if(dt > max_step) { update_D(order, max_step / dt, coeffs, tempD, D); dt = max_step; diff --git a/ode/src/KokkosODE_BDF.hpp b/ode/src/KokkosODE_BDF.hpp index f9004d94f0..1bf6e80df3 100644 --- a/ode/src/KokkosODE_BDF.hpp +++ b/ode/src/KokkosODE_BDF.hpp @@ -172,6 +172,8 @@ KOKKOS_FUNCTION void BDFSolve(const ode_type& ode, const scalar_type t_start, co mat_type& temp, mat_type& temp2) { using KAT = Kokkos::ArithTraits; + Kokkos::printf("y0 = {%f, %f, %f}\n", y0(0), y0(1), y0(2)); + // This needs to go away and be pulled out of temp instead... auto rhs = Kokkos::subview(temp, Kokkos::ALL(), 0); auto update = Kokkos::subview(temp, Kokkos::ALL(), 1); @@ -213,8 +215,8 @@ KOKKOS_FUNCTION void BDFSolve(const ode_type& ode, const scalar_type t_start, co for(int eqIdx = 0; eqIdx < ode.neqs; ++eqIdx) { y0(eqIdx) = y_new(eqIdx); } - // std::cout << "At t=" << t << ", y={" << y_new(0) << ", " << y_new(1) << ", " << y_new(2) - // << "}, next dt will be " << dt << ", order will be " << order << std::endl; + Kokkos::printf("At t=%f, y={%f, %f, %f}, next dt will be %f, order will be %d\n", + t, y_new(0), y_new(1), y_new(2), dt, order); } } // BDFSolve diff --git a/ode/unit_test/Test_ODE_BDF.hpp b/ode/unit_test/Test_ODE_BDF.hpp index ba4ce1fbcb..70e7cad5ec 100644 --- a/ode/unit_test/Test_ODE_BDF.hpp +++ b/ode/unit_test/Test_ODE_BDF.hpp @@ -174,6 +174,29 @@ struct BDFSolve_wrapper { } }; +template +struct BDF_Solve_wrapper { + + const ode_type my_ode; + const scalar_type t_start, t_end, dt, max_step; + const vec_type y0, y_new; + const mat_type temp, temp2; + + BDF_Solve_wrapper(const ode_type& my_ode_, const scalar_type& t_start_, + const scalar_type& t_end_, const scalar_type& dt_, + const scalar_type& max_step_, const vec_type& y0_, + const vec_type& y_new_, const mat_type& temp_, + const mat_type& temp2_) + : my_ode(my_ode_), t_start(t_start_), t_end(t_end_), dt(dt_), + max_step(max_step_), y0(y0_), y_new(y_new_), temp(temp_), + temp2(temp2_) {} + + KOKKOS_FUNCTION void operator()(const int) const { + KokkosODE::Experimental::BDFSolve(my_ode, t_start, t_end, dt, + max_step, y0, y_new, temp, temp2); + } +}; + template void test_BDF_Logistic() { using execution_space = typename device_type::execution_space; @@ -514,6 +537,8 @@ void test_BDF_parallel() { template void compute_coeffs(const int order, const scalar_type factor, const mat_type& coeffs) { + std::cout << "compute_coeffs" << std::endl; + coeffs(0, 0) = 1.0; for(int colIdx = 0; colIdx < order; ++colIdx) { coeffs(0, colIdx + 1) = 1.0; @@ -530,12 +555,14 @@ void update_D(const int order, const scalar_type factor, const mat_type& coeffs, compute_coeffs(order, factor, coeffs); auto R = Kokkos::subview(coeffs, Kokkos::pair(0, order + 1), Kokkos::pair(0, order + 1)); + std::cout << "SerialGemm" << std::endl; KokkosBatched::SerialGemm::invoke(1.0, R, subD, 0.0, subTempD); compute_coeffs(order, 1.0, coeffs); auto U = Kokkos::subview(coeffs, Kokkos::pair(0, order + 1), Kokkos::pair(0, order + 1)); + std::cout << "SerialGemm" << std::endl; KokkosBatched::SerialGemm::invoke(1.0, U, subTempD, 0.0, subD); @@ -543,7 +570,7 @@ void update_D(const int order, const scalar_type factor, const mat_type& coeffs, template void test_Nordsieck() { - using execution_space = typename device_type::execution_space; + using execution_space = Kokkos::HostSpace; StiffChemistry mySys{}; Kokkos::View R("coeffs", 6, 6), U("coeffs", 6, 6); @@ -558,6 +585,7 @@ void test_Nordsieck() { auto y0 = Kokkos::subview(D, 0, Kokkos::ALL()); auto f = Kokkos::subview(D, 1, Kokkos::ALL()); y0(0) = 1.0; + mySys.evaluate_function(0, 0, y0, f); for(int eqIdx = 0; eqIdx < mySys.neqs; ++eqIdx) { f(eqIdx) *= dt; @@ -566,13 +594,15 @@ void test_Nordsieck() { compute_coeffs(order, factor, R); compute_coeffs(order, 1.0, U); - std::cout << "R: " << std::endl; - for(int i = 0; i < order + 1; ++i) { - std::cout << "{ "; - for(int j = 0; j < order + 1; ++j) { - std::cout << R(i,j) << ", "; + { + std::cout << "R: " << std::endl; + for(int i = 0; i < order + 1; ++i) { + std::cout << "{ "; + for(int j = 0; j < order + 1; ++j) { + std::cout << R(i,j) << ", "; + } + std::cout << "}" << std::endl; } - std::cout << "}" << std::endl; } std::cout << "D before update:" << std::endl; @@ -696,8 +726,9 @@ void test_adaptive_BDF_v2() { y0, y_new, temp, temp2); } -template +template void test_BDF_adaptive_stiff() { + using execution_space = typename Device::execution_space; using vec_type = Kokkos::View; using mat_type = Kokkos::View; using KAT = Kokkos::ArithTraits; @@ -707,6 +738,8 @@ void test_BDF_adaptive_stiff() { const scalar_type t_start = KAT::zero(), t_end = 500*KAT::one(); scalar_type dt = KAT::zero(); vec_type y0("initial conditions", mySys.neqs), y_new("solution", mySys.neqs); + + // Set initial conditions auto y0_h = Kokkos::create_mirror_view(y0); y0_h(0) = KAT::one(); y0_h(1) = KAT::zero(); @@ -716,17 +749,27 @@ void test_BDF_adaptive_stiff() { mat_type temp("buffer1", mySys.neqs, 23 + 2*mySys.neqs + 4), temp2("buffer2", 6, 7); { + auto temp_h = Kokkos::create_mirror_view(temp); + Kokkos::deep_copy(temp_h, temp); vec_type f0("initial value f", mySys.neqs); - mySys.evaluate_function(t_start, KAT::zero(), y0, f0); - KokkosODE::Impl::initial_step_size(mySys, 1, t_start, 1e-6, 1e-4, y0, f0, temp, dt); - Kokkos::deep_copy(temp, KAT::zero()); // zeroing out temp to avoid surprises down the road... + auto f0_h = Kokkos::create_mirror_view(f0); + + mySys.evaluate_function(t_start, KAT::zero(), y0_h, f0_h); + KokkosODE::Impl::initial_step_size(mySys, 1, t_start, 1e-6, 1e-4, y0_h, f0_h, temp_h, dt); } std::cout << "Initial Step Size: dt=" << dt << std::endl; - KokkosODE::Experimental::BDFSolve(mySys, t_start, t_end, dt, - (t_end - t_start) / 10, - y0, y_new, temp, temp2); + Kokkos::RangePolicy policy(0, 1); + BDF_Solve_wrapper bdf_wrapper(mySys, t_start, t_end, dt, + (t_end - t_start) / 10, + y0, y_new, temp, temp2); + Kokkos::parallel_for(policy, bdf_wrapper); + + auto y_new_h = Kokkos::create_mirror_view(y_new); + Kokkos::deep_copy(y_new_h, y_new); + std::cout << "Stiff Chemistry solution at t=500: {" << y_new_h(0) + << ", " << y_new_h(1) << ", " << y_new_h(2) << "}" << std::endl; } } // namespace Test @@ -746,10 +789,10 @@ TEST_F(TestCategory, BDF_parallel_serial) { TEST_F(TestCategory, BDF_Nordsieck) { ::Test::test_Nordsieck(); } -TEST_F(TestCategory, BDF_adaptive) { - ::Test::test_adaptive_BDF(); - ::Test::test_adaptive_BDF_v2(); -} +// TEST_F(TestCategory, BDF_adaptive) { +// ::Test::test_adaptive_BDF(); +// ::Test::test_adaptive_BDF_v2(); +// } TEST_F(TestCategory, BDF_StiffChemistry_adaptive) { ::Test::test_BDF_adaptive_stiff(); }