Skip to content

Commit

Permalink
Merge pull request #212 from Adrian-Diaz/Parallel-Explicit-Solver
Browse files Browse the repository at this point in the history
MPI solvers update
  • Loading branch information
Adrian-Diaz authored Aug 6, 2024
2 parents fe0ecde + 25be3de commit 1dc8ac3
Show file tree
Hide file tree
Showing 25 changed files with 738 additions and 413 deletions.
18 changes: 16 additions & 2 deletions scripts/build-fierro.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ show_help() {
echo " --build_action=<full-app|set-env|install-trilinos|install-hdf5|install-heffte|fierro>. Default is 'full-app'"
echo " --machine=<darwin|chicoma|linux|mac>. Default is 'linux'"
echo " --build_cores=<Integers greater than 0>. Default is set 1"
echo " --intel_mkl=<enabled|disabled>. Default is 'disabled'"
echo " --heffte_build_type=<fftw|cufft|rocfft>. Default is set 'fftw'"
echo " --help: Display this help message"
echo " "
Expand Down Expand Up @@ -63,13 +64,15 @@ machine="linux"
kokkos_build_type="serial"
heffte_build_type="fftw"
build_cores="1"
intel_mkl="disabled"

# Define arrays of valid options
valid_build_action=("full-app" "set-env" "install-trilinos" "install-hdf5" "install-heffte" "install-uncrustify" "fierro")
valid_solver=("all" "explicit" "explicit-evpfft" "explicit-ls-evpfft" "implicit")
valid_kokkos_build_types=("serial" "openmp" "pthreads" "cuda" "hip")
valid_heffte_build_types=("fftw" "cufft" "rocfft")
valid_machines=("darwin" "chicoma" "linux" "mac" "msu")
valid_intel_mkl=("disabled" "enabled")

# Parse command line arguments
for arg in "$@"; do
Expand Down Expand Up @@ -114,6 +117,16 @@ for arg in "$@"; do
return 1
fi
;;
--intel_mkl=*)
option="${arg#*=}"
if [[ " ${valid_intel_mkl[*]} " == *" $option "* ]]; then
intel_mkl="$option"
else
echo "Error: Invalid --intel_mkl specified."
show_help
return 1
fi
;;
--heffte_build_type=*)
option="${arg#*=}"
if [[ " ${valid_heffte_build_types[*]} " == *" $option "* ]]; then
Expand Down Expand Up @@ -170,6 +183,7 @@ echo "Building based on these argument options:"
echo "Build action - ${build_action}"
echo "Solver - ${solver}"
echo "Kokkos backend - ${kokkos_build_type}"
echo "Intel MKL library - ${intel_mkl}"
echo "Machine - ${machine}"
if [ "${solver}" = "explicit-evpfft" ] || [ "${solver}" = "explicit-ls-evpfft" ]; then
echo "HEFFTE - ${heffte_build_type}"
Expand All @@ -184,14 +198,14 @@ source setup-env.sh ${machine} ${kokkos_build_type} ${build_cores}
# Next, do action based on args
if [ "$build_action" = "full-app" ]; then
source uncrustify-install.sh
source trilinos-install.sh ${kokkos_build_type} ${machine}
source trilinos-install.sh ${kokkos_build_type} ${machine} ${intel_mkl}
if [ "$solver" = "explicit-evpfft" ] || [ "${solver}" = "explicit-ls-evpfft" ]; then
source hdf5-install.sh
source heffte-install.sh ${heffte_build_type} ${machine}
fi
source cmake_build.sh ${solver} ${heffte_build_type} ${kokkos_build_type}
elif [ "$build_action" = "install-trilinos" ]; then
source trilinos-install.sh ${kokkos_build_type} ${machine}
source trilinos-install.sh ${kokkos_build_type} ${machine} ${intel_mkl}
elif [ "$build_action" = "install-hdf5" ]; then
source hdf5-install.sh
elif [ "$build_action" = "install-heffte" ]; then
Expand Down
2 changes: 1 addition & 1 deletion scripts/cmake_build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ kokkos_build_type="${3}"
if { [ ! -d "${ELEMENTS_SOURCE_DIR}/elements" ] || [ ! -d "${ELEMENTS_SOURCE_DIR}/matar/src" ] ;}
then
echo "Missing submodules, downloading them...."
git submodule update --recursive "${ELEMENTS_SOURCE_DIR}"
git submodule update --init --recursive "${ELEMENTS_SOURCE_DIR}"
fi

if [ ! -d "${TRILINOS_INSTALL_DIR}/lib" ]; then
Expand Down
25 changes: 24 additions & 1 deletion scripts/trilinos-install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

kokkos_build_type="${1}"
machine="${2}"
intel_mkl="${3}"

# If all arguments are valid, you can use them in your script as needed
echo "Trilinos Kokkos Build Type: $kokkos_build_type"
Expand All @@ -28,7 +29,9 @@ fi
#check if Trilinos library files were installed, install them otherwise.
[ -d "${TRILINOS_BUILD_DIR}/lib" ] && echo "Directory ${TRILINOS_BUILD_DIR}/lib exists, assuming successful installation; delete build folder and run build script again if there was an environment error that has been corrected."

if [ ! -d "${TRILINOS_BUILD_DIR}/lib" ]
[ -d "${TRILINOS_BUILD_DIR}/lib64" ] && echo "Directory ${TRILINOS_BUILD_DIR}/lib64 exists, assuming successful installation; delete build folder and run build script again if there was an environment error that has been corrected."

if [ ! -d "${TRILINOS_BUILD_DIR}/lib" ] && [ ! -d "${TRILINOS_BUILD_DIR}/lib64" ]
then
echo "Directory Trilinos/build/lib does not exist, compiling Trilinos (this might take a while)...."

Expand Down Expand Up @@ -113,6 +116,26 @@ ${ADDITIONS[@]}
-D CMAKE_INSTALL_PREFIX=${TRILINOS_INSTALL_DIR}
)

# Flags for building with Intel MKL library
INTEL_MKL_ADDITIONS=(
-D TPL_ENABLE_MKL=ON
-D BLAS_LIBRARY_NAMES="libmkl_rt.so"
-D BLAS_LIBRARY_DIRS="$MKLROOT/lib/intel64"
-D LAPACK_LIBRARY_NAMES="libmkl_rt.so"
-D LAPACK_LIBRARY_DIRS="$MKLROOT/lib/intel64"
-D MKL_LIBRARY_DIRS="$MKLROOT/lib/intel64"
-D MKL_LIBRARY_NAMES="mkl_rt"
-D MKL_INCLUDE_DIRS="$MKLROOT/include"
)

echo "**** Intel MKL = ${intel_mkl} ****"
if [ "$intel_mkl" = "enabled" ]; then
echo "**** assuming MKL installation at $MKLROOT ****"
cmake_options+=(
${INTEL_MKL_ADDITIONS[@]}
)
fi

if [ "$kokkos_build_type" = "openmp" ]; then
cmake_options+=(
${OPENMP_ADDITIONS[@]}
Expand Down
4 changes: 2 additions & 2 deletions scripts/uncrustify-install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@

# Check if the uncrustify build directory exists and is not empty in the parent directory; if not, clone it

if [ ! -d "${UNCRUSTIFY_SOURCE_DIR}" ];
if [ ! -d "${UNCRUSTIFY_SOURCE_DIR}/src" ];
then
echo "Missing Uncrustify submodule, downloading...."
git submodule update ${UNCRUSTIFY_SOURCE_DIR}
git submodule update --init ${UNCRUSTIFY_SOURCE_DIR}
fi

if [ ! -f "${UNCRUSTIFY_BUILD_DIR}/uncrustify" ]; then
Expand Down
5 changes: 3 additions & 2 deletions src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -186,8 +186,9 @@ void Implicit_Solver::run(){
all_initial_node_coords_distributed = all_node_coords_distributed;

//print element imbalance stats
int rnum_global_sum = 0;
MPI_Allreduce(&rnum_elem, &rnum_global_sum, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
long long int rnum_global_sum = 0;
long long int temp_rnum_elem = rnum_elem;
MPI_Allreduce(&temp_rnum_elem, &rnum_global_sum, 1, MPI_LONG_LONG_INT, MPI_SUM, MPI_COMM_WORLD);
double local_imbalance, max_imbalance, avg_elem;
max_imbalance = 0;
avg_elem = rnum_global_sum/((double) nranks);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
// @HEADER
// ************************************************************************
//
// Rapid Optimization Library (ROL) Package
// Copyright (2014) Sandia Corporation
//
// Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
// license for use of this work by or on behalf of the U.S. Government.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// 1. Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
//
// 3. Neither the name of the Corporation nor the names of the
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// Questions? Contact lead developers:
// Drew Kouri ([email protected]) and
// Denis Ridzal ([email protected])
//
// ************************************************************************
// @HEADER

#ifndef FIERRO_OPTIMIZATION_OBJECTIVE_H
#define FIERRO_OPTIMIZATION_OBJECTIVE_H

#include <string>
#include <Teuchos_ScalarTraits.hpp>
#include <Teuchos_RCP.hpp>
#include <Teuchos_oblackholestream.hpp>
#include <Teuchos_Tuple.hpp>
#include <Teuchos_VerboseObject.hpp>

#include <Tpetra_Core.hpp>
#include <Tpetra_Map.hpp>
#include <Tpetra_MultiVector.hpp>
#include "Tpetra_Details_DefaultTypes.hpp"

#include "ROL_Types.hpp"
#include <ROL_TpetraMultiVector.hpp>
#include "ROL_Elementwise_Reduce.hpp"
#include "ROL_Objective.hpp"
#include "ROL_BoundConstraint.hpp"

/** @ingroup func_group
\class ROL::ObjectiveMMA
\brief Provides the interface to to Method of Moving Asymptotes
Objective function
---
*/

class FierroOptimizationObjective : public ROL::Objective<real_t> {

typedef Tpetra::Map<>::local_ordinal_type LO;
typedef Tpetra::Map<>::global_ordinal_type GO;
typedef Tpetra::Map<>::node_type Node;
typedef Tpetra::Map<LO, GO, Node> Map;
typedef Tpetra::MultiVector<real_t, LO, GO, Node> MV;
typedef ROL::Vector<real_t> V;
typedef const ROL::Vector<real_t> const_V;
typedef ROL::TpetraMultiVector<real_t,LO,GO,Node> ROL_MV;

using traits = Kokkos::ViewTraits<LO*, Kokkos::LayoutLeft, void, void>;
using array_layout = typename traits::array_layout;
using execution_space = typename traits::execution_space;
using device_type = typename traits::device_type;
using memory_traits = typename traits::memory_traits;
using global_size_t = Tpetra::global_size_t;

typedef Kokkos::View<real_t*, Kokkos::LayoutRight, device_type, memory_traits> values_array;
typedef Kokkos::View<GO*, array_layout, device_type, memory_traits> global_indices_array;
typedef Kokkos::View<LO*, array_layout, device_type, memory_traits> indices_array;

//typedef Kokkos::DualView<real_t**, Kokkos::LayoutLeft, device_type>::t_dev vec_array;
typedef MV::dual_view_type::t_dev vec_array;
typedef MV::dual_view_type::t_host host_vec_array;
typedef Kokkos::View<const real_t**, array_layout, HostSpace, memory_traits> const_host_vec_array;
typedef MV::dual_view_type dual_vec_array;
typedef ROL::Objective<real_t> OBJ;
typedef ROL::BoundConstraint<real_t> BND;
int update_count = 0;

private:

real_t fval_; // Original objective value

real_t tol_;

ROL::Ptr<const MV> getVector( const V& x ) {
return dynamic_cast<const ROL_MV&>(x).getVector();
}

ROL::Ptr<MV> getVector( V& x ) {
return dynamic_cast<ROL_MV&>(x).getVector();
}

public:

bool time_accumulation;
real_t objective_accumulation;

FierroOptimizationObjective(){
objective_accumulation = 0;
time_accumulation = false;
}


}; // class ObjectiveMMA


#endif // FIERRO_MMA_OBJECTIVE

Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
num_dims: 3
dynamic_options:
time_final: 20.0
dt_min: 1.e-8
dt_max: 1.e-2
dt_start: 1.e-5
cycle_stop: 2000000

mesh_generation_options:
type: Box
origin: [0, 0, 0]
length: [1.2, 1.2, 1.2]
num_elems: [32, 32, 32]

output_options:
timer_output_level: thorough
output_file_format: vtk
graphics_step: 0.25

fea_module_parameters:
- type: SGH
material_id: 0
boundary_conditions:
# Tag X plane
- surface:
type: x_plane
plane_position: 0.0
type: reflected

# Tag Y plane
- surface:
type: y_plane
plane_position: 0.0
type: reflected

# Tag Z plane
- surface:
type: z_plane
plane_position: 0.0
type: reflected

loading_conditions:
# Load radially around 0 0 0 corner
- volume:
type: sphere
radius1: 1.0
radius2: 2.5
type: body_force
component_x: -0.00000005
component_y: -0.00000005
component_z: -0.00000005

materials:
- id: 0
eos_model: ideal_gas
strength_model: none
elastic_modulus: 10
poisson_ratio: 0.3
maximum_limiter: 1
q1: 2.0
q2: 4.0
q1ex: 2.0
q2ex: 4.0
eos_global_vars:
- 1.666666666666667
- 1.0E-14
- 1.0

regions:
- volume:
type: global
material_id: 0
den: 1.0
sie: 1.e-10

velocity: cartesian
u: 0.0
v: 0.0
w: 0.0

velocity: cartesian
u: 0.0
v: 0.0
w: 0.0
Original file line number Diff line number Diff line change
Expand Up @@ -105,14 +105,17 @@ regions:
optimization_options:
optimization_process: topology_optimization
optimization_objective: minimize_kinetic_energy
objective_regions:
- type: box
x1: 0.4
x2: 0.8
y1: 0.4
y2: 0.8
z1: 0.4
z2: 0.8
use_solve_checkpoints: true
num_solve_checkpoints : 10
disable_forward_solve_output: true
optimization_output_freq: 1
rol_params:
subproblem_algorithm: line_search
initial_constraint_penalty: 1.e-2
step_tolerance: 1.e-3
gradient_tolerance: 1.e-3
constraint_tolerance: 1.e-3
iteration_limit: 20
density_epsilon: 0.1
variable_outer_shell: true
constraints:
Expand Down
Loading

0 comments on commit 1dc8ac3

Please sign in to comment.