HYPRE memory fixup (#733)

Addresses memory issue due to repeated calls to vector or matrix initialize in HYPRE. Leak still present in call to setupImpl
ECP-copa · Jan 17, 2024 · 93deaba · 93deaba
1 parent 9e099a6
commit 93deaba
Show file tree

Hide file tree

Showing 5 changed files with 10 additions and 30 deletions.
diff --git a/...al/11_semi_structured_solver_multi_variate/hypre_semi_structured_solver_multi_example.cpp b/...al/11_semi_structured_solver_multi_variate/hypre_semi_structured_solver_multi_example.cpp
@@ -127,8 +127,6 @@ void hypreSemiStructuredSolverExample()
             entry_view( i, j, k, 6 ) = -1.0;
         } );
 
-    solver->initializeHypreMatrix();
-
     for ( int v_h = 0; v_h < 3; ++v_h )
     {
         solver->setMatrixValues( *matrix_entries, v_h, v_h );

diff --git a/grid/src/Cabana_Grid_HypreSemiStructuredSolver.hpp b/grid/src/Cabana_Grid_HypreSemiStructuredSolver.hpp
@@ -328,22 +328,16 @@ class HypreSemiStructuredSolver
         error = HYPRE_SStructGraphAssemble( _graph );
         checkHypreError( error );
 
-        // Create the matrix.
+        // Create the matrix. must be done after graph is assembled
         error = HYPRE_SStructMatrixCreate( _comm, _graph, &_A );
         checkHypreError( error );
 
         // Set the SStruct matrix object type
         error = HYPRE_SStructMatrixSetObjectType( _A, object_type );
         checkHypreError( error );
-    }
 
-    /*!
-        \brief Prepare the hypre matrix to have it's values set
-    */
-    void initializeHypreMatrix()
-    {
-        // Initialize the matrix.
-        auto error = HYPRE_SStructMatrixInitialize( _A );
+        // Prepare the matrix for setting values
+        error = HYPRE_SStructMatrixInitialize( _A );
         checkHypreError( error );
     }
 
@@ -524,10 +518,6 @@ class HypreSemiStructuredSolver
 
         int part = 0;
 
-        // Initialize the RHS.
-        auto error = HYPRE_SStructVectorInitialize( _b );
-        checkHypreError( error );
-
         // Copy the RHS into HYPRE. The HYPRE layout is fixed as layout-right.
         auto owned_space = b.layout()->indexSpace( Own(), Local() );
         std::array<long, num_space_dim + 1> reorder_min;
@@ -542,6 +532,7 @@ class HypreSemiStructuredSolver
         // The process of creating the view and then deep copying each
         // variable is functional, but we should avoid this process
         // for performance if possible
+        int error;
         for ( int var = 0; var < n_vars; ++var )
         {
             reorder_min.back() = var;

diff --git a/grid/src/Cabana_Grid_HypreStructuredSolver.hpp b/grid/src/Cabana_Grid_HypreStructuredSolver.hpp
@@ -209,11 +209,13 @@ class HypreStructuredSolver
             checkHypreError( error );
         }
 
-        // Create the matrix.
+        // Create the matrix object. Must be done after the stencil is setup
         error = HYPRE_StructMatrixCreate( _comm, _grid, _stencil, &_A );
         checkHypreError( error );
         error = HYPRE_StructMatrixSetSymmetric( _A, is_symmetric );
         checkHypreError( error );
+        error = HYPRE_StructMatrixInitialize( _A );
+        checkHypreError( error );
     }
 
     /*!
@@ -252,10 +254,6 @@ class HypreStructuredSolver
         // Spatial dimension.
         const std::size_t num_space_dim = Array_t::num_space_dim;
 
-        // Intialize the matrix for setting values.
-        auto error = HYPRE_StructMatrixInitialize( _A );
-        checkHypreError( error );
-
         // Copy the matrix entries into HYPRE. The HYPRE layout is fixed as
         // layout-right.
         auto owned_space = values.layout()->indexSpace( Own(), Local() );
@@ -275,7 +273,7 @@ class HypreStructuredSolver
         // Insert values into the HYPRE matrix.
         std::vector<HYPRE_Int> indices( _stencil_size );
         std::iota( indices.begin(), indices.end(), 0 );
-        error = HYPRE_StructMatrixSetBoxValues(
+        auto error = HYPRE_StructMatrixSetBoxValues(
             _A, _lower.data(), _upper.data(), indices.size(), indices.data(),
             a_values.data() );
         checkHypreError( error );
@@ -347,6 +345,7 @@ class HypreStructuredSolver
         if ( _is_preconditioner )
             throw std::logic_error( "Cannot call setup() on preconditioners" );
 
+        // FIXME: appears to be a memory issue in the call to this function
         this->setupImpl( _A, _b, _x );
     }
 
@@ -385,10 +384,6 @@ class HypreStructuredSolver
         // Spatial dimension.
         const std::size_t num_space_dim = Array_t::num_space_dim;
 
-        // Initialize the RHS.
-        auto error = HYPRE_StructVectorInitialize( _b );
-        checkHypreError( error );
-
         // Copy the RHS into HYPRE. The HYPRE layout is fixed as layout-right.
         auto owned_space = b.layout()->indexSpace( Own(), Local() );
         std::array<long, num_space_dim + 1> reorder_size;
@@ -405,7 +400,7 @@ class HypreStructuredSolver
         Kokkos::deep_copy( vector_values, b_subv );
 
         // Insert b values into the HYPRE vector.
-        error = HYPRE_StructVectorSetBoxValues(
+        auto error = HYPRE_StructVectorSetBoxValues(
             _b, _lower.data(), _upper.data(), vector_values.data() );
         checkHypreError( error );
         error = HYPRE_StructVectorAssemble( _b );

diff --git a/grid/unit_test/tstHypreSemiStructuredSolver.hpp b/grid/unit_test/tstHypreSemiStructuredSolver.hpp
@@ -105,8 +105,6 @@ poissonTest( const std::string& solver_type, const std::string& precond_type,
             entry_view( i, j, k, 6 ) = -1.0;
         } );
 
-    solver->initializeHypreMatrix();
-
     solver->setMatrixValues( *matrix_entries, 0, 0 );
 
     // Set the tolerance.

diff --git a/grid/unit_test/tstHypreSemiStructuredSolverMulti.hpp b/grid/unit_test/tstHypreSemiStructuredSolverMulti.hpp
@@ -115,8 +115,6 @@ poissonTest( const std::string& solver_type, const std::string& precond_type,
             entry_view( i, j, k, 6 ) = -1.0;
         } );
 
-    solver->initializeHypreMatrix();
-
     for ( int v_h = 0; v_h < 3; ++v_h )
     {
         solver->setMatrixValues( *matrix_entries, v_h, v_h );