Merge pull request ORNL#14 from DavidJoy8/grid-output

Use new Cabana grid output

.github/workflows/CI.yml

@@ -20,7 +20,7 @@ jobs:
         cxx: ['g++', 'clang++']
         backend: ['SERIAL', 'OPENMP']
         cmake_build_type: ['Debug', 'Release']
-        kokkos_ver: ['3.7.02']
+        kokkos_ver: ['4.1.00']
     runs-on: ubuntu-20.04
     container: ghcr.io/ecp-copa/ci-containers/${{ matrix.distro }}
     steps:
@@ -62,7 +62,7 @@ jobs:
         uses: actions/checkout@v3
         with:
           repository: ECP-CoPA/Cabana
-          ref: 0.6.1
+          ref: cd7075fd66bc346c8f28831cfb5b0ea650cf9892
           path: cabana
       - name: Build Cabana
         working-directory: cabana
@@ -102,7 +102,7 @@ jobs:
       matrix:
         cxx: ['nvcc']
         cmake_build_type: ['Release']
-        kokkos_ver: ['3.7.02']
+        kokkos_ver: ['4.1.00']
     runs-on: ubuntu-20.04
     container: ghcr.io/ecp-copa/ci-containers/cuda:12.2.0
     steps:
@@ -144,7 +144,7 @@ jobs:
         uses: actions/checkout@v3
         with:
           repository: ECP-CoPA/Cabana
-          ref: 0.6.1
+          ref: cd7075fd66bc346c8f28831cfb5b0ea650cf9892
           path: cabana
       - name: Build Cabana
         working-directory: cabana

CMakeLists.txt

@@ -4,8 +4,8 @@ project(CabanaPF CXX)
 include(GNUInstallDirs)
 set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
 
-if(DEFINED RESULTS_PATH)
-  add_compile_definitions(RESULTS_PATH="${RESULTS_PATH}")
+if(DEFINED CABANAPF_RESULTS_PATH)
+  add_compile_definitions(CABANAPF_RESULTS_PATH="${CABANAPF_RESULTS_PATH}")
 endif()
 
 find_package(Cabana)

examples/PFHub1a.cpp

@@ -24,7 +24,7 @@ int main(int argc, char* argv[]) {
                 throw std::invalid_argument("");
 
             simulation->run_until_time(end_time);
-            simulation->output();
+            simulation->output_c();
         } catch (std::invalid_argument const&) {
             std::cout << "Usage: ./PFHub1a <benchmark|periodic> <grid points> <dt> <end time>" << std::endl;
         }

src/PFHub.hpp

@@ -109,7 +109,15 @@ class PFHub1aBase : public CabanaPFRunner<2> {
         vars.fft_inverse(0);
     }
 
-    virtual void output() {}
+    // versions of PFHub1a must override this to give themselves a name:
+    virtual std::string subproblem_name() const = 0;
+    // save a copy of the c grid to a file
+    void output_c() {
+        std::stringstream s;
+        s << subproblem_name() << "_N" << grid_points << "_DT" << std::fixed << std::setprecision(3) << std::scientific
+          << dt;
+        vars.save(0, s.str(), 0);
+    }
 
     // needed to allow polymorphism:
     virtual ~PFHub1aBase() {}
@@ -133,10 +141,8 @@ class PFHub1aBenchmark : public PFHub1aBase {
             });
     }
 
-    void output() override {
-        std::stringstream s;
-        s << "1aBenchmark_N" << grid_points << "_DT" << std::fixed << std::setprecision(3) << std::scientific << dt;
-        vars.save(0, s.str());
+    std::string subproblem_name() const override {
+        return "1aBenchmark";
     }
 
     PFHub1aBenchmark(int grid_points, double dt) : PFHub1aBase{grid_points, dt} {}
@@ -161,10 +167,8 @@ class PFHub1aPeriodic : public PFHub1aBase {
             });
     }
 
-    void output() override {
-        std::stringstream s;
-        s << "1aPeriodic_N" << grid_points << "_DT" << std::fixed << std::setprecision(3) << std::scientific << dt;
-        vars.save(0, s.str());
+    std::string subproblem_name() const override {
+        return "1aPeriodic";
     }
 
     PFHub1aPeriodic(int grid_points, double dt) : PFHub1aBase{grid_points, dt} {}

src/PFVariables.hpp

@@ -4,10 +4,6 @@
 #include <Cabana_Grid.hpp>
 #include <fstream>
 
-#ifdef RESULTS_PATH
-#include <filesystem>
-#endif
-
 namespace CabanaPF {
 
 template <std::size_t NumSpaceDim, std::size_t NumVariables>
@@ -26,6 +22,16 @@ class PFVariables {
         Cabana::Grid::Experimental::Impl::FFTBackendDefault>>
         fft_calculator;
 
+    // string to be used for Cabana's BOVWriter's prefix argument
+    std::string save_prefix(const int index, std::string run_name) {
+        std::stringstream name;
+#ifdef CABANAPF_RESULTS_PATH
+        name << CABANAPF_RESULTS_PATH;
+#endif
+        name << run_name << "_" << arrays[index]->label();
+        return name.str();
+    }
+
   public:
     std::array<GridArray, NumVariables> arrays;
 
@@ -61,48 +67,31 @@ class PFVariables {
         return Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), arrays[index]->view());
     }
 
-    std::string save_name(std::string run_name, [[maybe_unused]] const int index,
-                          [[maybe_unused]] const int timesteps_done = -1) {
-        std::stringstream name;
-#ifdef RESULTS_PATH
-        name << RESULTS_PATH;
-#endif
-        name << run_name << "_" << arrays[index]->label() << ".dat";
-        return name.str();
+    // Saves an array to file.
+    void save(const int index, std::string run_name, const int timesteps_done, const double time = 0) {
+        const std::string prefix = save_prefix(index, run_name);
+        Cabana::Grid::Experimental::BovWriter::writeTimeStep(prefix, timesteps_done, time, *arrays[index]);
     }
 
-    // If unfinished and just saving progress, pass in timesteps_done
-    void save(const int index, [[maybe_unused]] std::string run_name, [[maybe_unused]] const int timesteps_done = -1) {
-        Cabana::Grid::Experimental::BovWriter::writeTimeStep(999999, 0,
-                                                             *arrays[index]); // use 999999 to mark it for move
-#ifdef RESULTS_PATH // Comes from the CMake build; if not defined, won't have file I/O
-        try {
-            std::string old_name = "grid_" + arrays[index]->label() + "_999999.dat";
-            std::filesystem::rename(old_name, save_name(run_name, index, timesteps_done));
-            // TODO: Deal with the .bov file?
-        } catch (std::filesystem::filesystem_error& e) {
-            std::cerr << "Error when saving: " << e.what() << std::endl;
-        }
-#endif
-    }
-
-    void load(std::string run_name, const int timesteps_done = -1) {
-        for (std::size_t index = 0; index < NumVariables; index++) {
-            assert(NumSpaceDim == 2); // currently not supported for 3D
-            // open the file:
-            std::fstream infile(save_name(run_name, index, timesteps_done), std::fstream::in | std::fstream::binary);
-            // read it:
-            const auto view = arrays[index]->view();
-            double buffer[2];
-            for (int j = 0; j < array_size[1]; j++) {
-                for (int i = 0; i < array_size[0]; i++) {
-                    infile.read((char*)buffer, 2 * sizeof(double));
-                    view(i, j, 0) = buffer[0];
-                    view(i, j, 1) = buffer[1];
-                }
-                // since the grid is periodic, it writes the first value in the row again, so "burn it off":
+    // loads the file created by the same arguments to save
+    void load(const int index, std::string run_name, const int timesteps_done) {
+        assert(NumSpaceDim == 2); // currently not supported for 3D
+        // recreate Cabana's naming scheme:
+        std::stringstream name;
+        name << save_prefix(index, run_name) << "_" << std::setfill('0') << std::setw(6) << timesteps_done << ".dat";
+        // open the file:
+        std::fstream infile(name.str(), std::fstream::in | std::fstream::binary);
+        // read it:
+        const auto view = arrays[index]->view();
+        double buffer[2];
+        for (int j = 0; j < array_size[1]; j++) {
+            for (int i = 0; i < array_size[0]; i++) {
                 infile.read((char*)buffer, 2 * sizeof(double));
+                view(i, j, 0) = buffer[0];
+                view(i, j, 1) = buffer[1];
             }
+            // since the grid is periodic, it writes the first value in the row again, so "burn it off":
+            infile.read((char*)buffer, 2 * sizeof(double));
         }
     }
 };

unit_test/test.cpp

@@ -75,7 +75,6 @@ TEST(PFHub1a, AllTimestep) {
     EXPECT_NEAR(0.6482746495041702, results(67, 7, 0), 1e-9);
 }
 
-#ifdef RESULTS_PATH
 TEST(PFVariables, saveload) {
     auto global_mesh = Cabana::Grid::createUniformGlobalMesh(std::array<double, 2>{0, 0}, std::array<double, 2>{6, 6},
                                                              std::array<int, 2>{3, 3});
@@ -94,15 +93,14 @@ TEST(PFVariables, saveload) {
     vars.save(0, "Test", 0);
 
     PFVariables from_file(layout, std::array<std::string, 1>{"a"});
-    from_file.load("Test", 0);
+    from_file.load(0, "Test", 0);
     for (int i = 0; i < 3; i++) {
         for (int j = 0; j < 3; j++) {
             EXPECT_EQ(vars[0](i, j, 0), from_file[0](i, j, 0));
             EXPECT_EQ(vars[0](i, j, 1), from_file[0](i, j, 1));
         }
     }
 }
-#endif
 
 // Similar to above, the python implementation was modified to use the periodic initial conditions
 TEST(PFHub1aPeriodic, periodic) {