diff --git a/testing/adios2/derived/CMakeLists.txt b/testing/adios2/derived/CMakeLists.txt index 097e7cc7d7..60d47bf676 100644 --- a/testing/adios2/derived/CMakeLists.txt +++ b/testing/adios2/derived/CMakeLists.txt @@ -4,3 +4,4 @@ #------------------------------------------------------------------------------# gtest_add_tests_helper(DerivedCorrectness MPI_NONE BP Derived. "") +gtest_add_tests_helper(DerivedCorrectnessMPI MPI_ALLOW BP Derived. "") diff --git a/testing/adios2/derived/TestBPDerivedCorrectness.cpp b/testing/adios2/derived/TestBPDerivedCorrectness.cpp index 4c3fadbb3b..671158fa0e 100644 --- a/testing/adios2/derived/TestBPDerivedCorrectness.cpp +++ b/testing/adios2/derived/TestBPDerivedCorrectness.cpp @@ -25,301 +25,6 @@ class DerivedCorrectnessP : public DerivedCorrectness, adios2::DerivedVarType GetThreads() { return GetParam(); }; }; -TEST_P(DerivedCorrectnessP, ScalarFunctionsCorrectnessTest) -{ - const size_t Nx = 10, Ny = 3, Nz = 6; - const size_t steps = 2; - // Application variable - std::default_random_engine generator; - std::uniform_real_distribution distribution(0.0, 10.0); - - adios2::DerivedVarType mode = GetParam(); - std::cout << "Mode is " << mode << std::endl; - - std::vector simArray1(Nx * Ny * Nz); - std::vector simArray2(Nx * Ny * Nz); - std::vector simArray3(Nx * Ny * Nz); - for (size_t i = 0; i < Nx * Ny * Nz; ++i) - { - simArray1[i] = distribution(generator); - simArray2[i] = distribution(generator); - simArray3[i] = distribution(generator); - } - - adios2::ADIOS adios; - - adios2::IO bpOut = adios.DeclareIO("BPWriteAddExpression"); - - std::vector varname = {"sim1/Ux", "sim1/Uy", "sim1/Uz"}; - const std::string derAgrAdd = "derived/agradd"; - const std::string derAddName = "derived/add"; - const std::string derSubtrName = "derived/subtr"; - const std::string derMultName = "derived/mult"; - const std::string derDivName = "derived/div"; - const std::string derPowName = "derived/pow"; - const std::string derSqrtName = "derived/sqrt"; - - auto Ux = bpOut.DefineVariable(varname[0], {Nx, Ny, Nz}, {0, 0, 0}, {Nx, Ny, Nz}); - auto Uy = bpOut.DefineVariable(varname[1], {Nx, Ny, Nz}, {0, 0, 0}, {Nx, Ny, Nz}); - auto Uz = bpOut.DefineVariable(varname[2], {Nx, Ny, Nz}, {0, 0, 0}, {Nx, Ny, Nz}); - // clang-format off - bpOut.DefineDerivedVariable(derAgrAdd, - "x=" + varname[0] + "\n" - "add(x)", - mode); - bpOut.DefineDerivedVariable(derAddName, - "x =" + varname[0] + " \n" - "y =" + varname[1] + " \n" - "z =" + varname[2] + " \n" - "x+y+z", - mode); - bpOut.DefineDerivedVariable(derSubtrName, - "x =" + varname[0] + " \n" - "y =" + varname[1] + " \n" - "z =" + varname[2] + " \n" - "x-y-z", - mode); - bpOut.DefineDerivedVariable(derMultName, - "x =" + varname[0] + " \n" - "y =" + varname[1] + " \n" - "z =" + varname[2] + " \n" - "x*y*z", - mode); - bpOut.DefineDerivedVariable(derDivName, - "x =" + varname[0] + " \n" - "y =" + varname[1] + " \n" - "z =" + varname[2] + " \n" - "divide(x,y,z)", - mode); - bpOut.DefineDerivedVariable(derPowName, - "x =" + varname[0] + " \n" - "pow(x)", - mode); - bpOut.DefineDerivedVariable(derSqrtName, - "x =" + varname[0] + " \n" - "sqrt(x)", - mode); - // clang-format on - std::string filename = "derivedScalar.bp"; - adios2::Engine bpFileWriter = bpOut.Open(filename, adios2::Mode::Write); - - for (size_t i = 0; i < steps; i++) - { - bpFileWriter.BeginStep(); - bpFileWriter.Put(Ux, simArray1.data()); - bpFileWriter.Put(Uy, simArray2.data()); - bpFileWriter.Put(Uz, simArray3.data()); - bpFileWriter.EndStep(); - } - bpFileWriter.Close(); - - adios2::IO bpIn = adios.DeclareIO("BPReadExpression"); - adios2::Engine bpFileReader = bpIn.Open(filename, adios2::Mode::Read); - - std::vector readUx; - std::vector readUy; - std::vector readUz; - std::vector readAdd; - std::vector readAgrAdd; - std::vector readSubtr; - std::vector readMult; - std::vector readDiv; - std::vector readPow; - std::vector readSqrt; - - float calcFloat; - double calcDouble; - float epsilon = (float)0.01; - for (size_t i = 0; i < steps; i++) - { - bpFileReader.BeginStep(); - bpFileReader.Get(varname[0], readUx); - bpFileReader.Get(varname[1], readUy); - bpFileReader.Get(varname[2], readUz); - bpFileReader.Get(derAddName, readAdd); - bpFileReader.Get(derAgrAdd, readAgrAdd); - bpFileReader.Get(derSubtrName, readSubtr); - bpFileReader.Get(derMultName, readMult); - bpFileReader.Get(derDivName, readDiv); - bpFileReader.Get(derPowName, readPow); - bpFileReader.Get(derSqrtName, readSqrt); - bpFileReader.EndStep(); - - for (size_t ind = 0; ind < Nx * Ny * Nz; ++ind) - { - calcFloat = readUx[ind] + readUy[ind] + readUz[ind]; - EXPECT_TRUE(fabs(calcFloat - readAdd[ind]) < epsilon); - - calcFloat = readUx[ind] - readUy[ind] - readUz[ind]; - EXPECT_TRUE(fabs(calcFloat - readSubtr[ind]) < epsilon); - - calcFloat = readUx[ind] * readUy[ind] * readUz[ind]; - EXPECT_TRUE(fabs(calcFloat - readMult[ind]) < epsilon); - - calcFloat = readUx[ind] / readUy[ind] / readUz[ind]; - EXPECT_TRUE(fabs(calcFloat - readDiv[ind]) < epsilon); - - calcDouble = std::pow(readUx[ind], 2); - EXPECT_TRUE(fabs(calcDouble - readPow[ind]) < epsilon); - - calcDouble = std::sqrt(readUx[ind]); - EXPECT_TRUE(fabs(calcDouble - readSqrt[ind]) < epsilon); - } - - for (size_t ind = 0; ind < Nx * Ny; ++ind) - { - size_t start = ind * Nz; - float calcA = 0; - for (size_t z = 0; z < Nz; ++z) - { - calcA += readUx[z + start]; - } - EXPECT_TRUE(fabs(calcA - readAgrAdd[ind]) < epsilon); - } - } - bpFileReader.Close(); -} - -TEST_P(DerivedCorrectnessP, TrigCorrectnessTest) -{ - const size_t Nx = 10, Ny = 3, Nz = 6; - const size_t steps = 2; - // Application variable - std::default_random_engine generator; - std::uniform_real_distribution randomDist(0.0, 100.0); - std::uniform_real_distribution sinDist(-1.0, 1.0); - - adios2::DerivedVarType mode = GetParam(); - std::cout << "Mode is " << mode << std::endl; - - std::vector simArray1(Nx * Ny * Nz); - std::vector simArray2(Nx * Ny * Nz); - for (size_t i = 0; i < Nx * Ny * Nz; ++i) - { - simArray1[i] = randomDist(generator); - simArray2[i] = sinDist(generator); - } - - adios2::ADIOS adios; - - adios2::IO bpOut = adios.DeclareIO("BPWriteSinExpression"); - - std::string randDistVar = "sim/randDist"; - std::string sinDistVar = "sim/sinDist"; - std::string derSinName = "derived/sin"; - std::string derCosName = "derived/cos"; - std::string derTanName = "derived/tan"; - std::string derAsinName = "derived/asin"; - std::string derAcosName = "derived/acos"; - std::string derAtanName = "derived/atan"; - - auto Ux = bpOut.DefineVariable(randDistVar, {Nx, Ny, Nz}, {0, 0, 0}, {Nx, Ny, Nz}); - auto Uy = bpOut.DefineVariable(sinDistVar, {Nx, Ny, Nz}, {0, 0, 0}, {Nx, Ny, Nz}); - // clang-format off - bpOut.DefineDerivedVariable(derSinName, - "x =" + randDistVar + " \n" - "sin(x)", - mode); - bpOut.DefineDerivedVariable(derCosName, - "x =" + randDistVar + " \n" - "cos(x)", - mode); - bpOut.DefineDerivedVariable(derTanName, - "x =" + randDistVar + " \n" - "tan(x)", - mode); - bpOut.DefineDerivedVariable(derAsinName, - "x =" + sinDistVar + " \n" - "asin(x)", - mode); - bpOut.DefineDerivedVariable(derAcosName, - "x =" + sinDistVar + " \n" - "acos(x)", - mode); - bpOut.DefineDerivedVariable(derAtanName, - "x =" + randDistVar + " \n" - "atan(x)", - mode); - // clang-format on - std::string filename = "expTrig.bp"; - adios2::Engine bpFileWriter = bpOut.Open(filename, adios2::Mode::Write); - - for (size_t i = 0; i < steps; i++) - { - bpFileWriter.BeginStep(); - bpFileWriter.Put(Ux, simArray1.data()); - bpFileWriter.Put(Uy, simArray2.data()); - bpFileWriter.EndStep(); - } - bpFileWriter.Close(); - - adios2::IO bpIn = adios.DeclareIO("BPReadExpression"); - adios2::Engine bpFileReader = bpIn.Open(filename, adios2::Mode::Read); - - std::vector readRandDist; - std::vector readSinDist; - std::vector readSin; - std::vector readCos; - std::vector readTan; - std::vector readAsin; - std::vector readAcos; - std::vector readAtan; - - double calcTrig; - double epsilon = (double)0.01; - for (size_t i = 0; i < steps; i++) - { - bpFileReader.BeginStep(); - bpFileReader.Get(randDistVar, readRandDist); - bpFileReader.Get(sinDistVar, readSinDist); - - bpFileReader.Get(derSinName, readSin); - bpFileReader.Get(derCosName, readCos); - bpFileReader.Get(derTanName, readTan); - bpFileReader.Get(derAsinName, readAsin); - bpFileReader.Get(derAcosName, readAcos); - bpFileReader.Get(derAtanName, readAtan); - bpFileReader.EndStep(); - - for (size_t ind = 0; ind < Nx * Ny * Nz; ++ind) - { - calcTrig = std::sin(readRandDist[ind]); - EXPECT_TRUE(fabs(calcTrig - readSin[ind]) < epsilon); - } - - for (size_t ind = 0; ind < Nx * Ny * Nz; ++ind) - { - calcTrig = std::cos(readRandDist[ind]); - EXPECT_TRUE(fabs(calcTrig - readCos[ind]) < epsilon); - } - - for (size_t ind = 0; ind < Nx * Ny * Nz; ++ind) - { - calcTrig = std::tan(readRandDist[ind]); - EXPECT_TRUE(fabs(calcTrig - readTan[ind]) < epsilon); - } - - for (size_t ind = 0; ind < Nx * Ny * Nz; ++ind) - { - calcTrig = std::asin(readSinDist[ind]); - EXPECT_TRUE(fabs(calcTrig - readAsin[ind]) < epsilon); - } - - for (size_t ind = 0; ind < Nx * Ny * Nz; ++ind) - { - calcTrig = std::acos(readSinDist[ind]); - EXPECT_TRUE(fabs(calcTrig - readAcos[ind]) < epsilon); - } - - for (size_t ind = 0; ind < Nx * Ny * Nz; ++ind) - { - calcTrig = std::atan(readRandDist[ind]); - EXPECT_TRUE(fabs(calcTrig - readAtan[ind]) < epsilon); - } - } - bpFileReader.Close(); -} - TEST_P(DerivedCorrectnessP, VectorCorrectnessTest) { const size_t Nx = 2, Ny = 3, Nz = 10; @@ -327,9 +32,7 @@ TEST_P(DerivedCorrectnessP, VectorCorrectnessTest) // Application variable std::default_random_engine generator; std::uniform_real_distribution distribution(0.0, 10.0); - adios2::DerivedVarType mode = GetParam(); - std::cout << "Mode is " << mode << std::endl; std::vector simArray1(Nx * Ny * Nz); std::vector simArray2(Nx * Ny * Nz); @@ -377,7 +80,7 @@ TEST_P(DerivedCorrectnessP, VectorCorrectnessTest) "cross(Ux, Uy, Uz, Vx, Vy, Vz)", mode); // clang-format on - std::string filename = "expVector.bp"; + std::string filename = "ADIOS2BPWriteDerivedVector.bp"; adios2::Engine bpFileWriter = bpOut.Open(filename, adios2::Mode::Write); for (size_t i = 0; i < steps; i++) @@ -506,7 +209,7 @@ TEST_P(DerivedCorrectnessP, CurlCorrectnessTest) "curl(Vx,Vy,Vz)", mode); // clang-format on - std::string filename = "expCurl.bp"; + std::string filename = "ADIOS2BPWriteDerivedCurl.bp"; adios2::Engine bpFileWriter = bpOut.Open(filename, adios2::Mode::Write); bpFileWriter.BeginStep(); @@ -646,7 +349,7 @@ TEST_P(DerivedCorrectnessP, MagCurlCorrectnessTest) "magnitude(curl(x,y,z))", mode); // clang-format on - std::string filename = "expMagCurl.bp"; + std::string filename = "ADIOS2BPWriteDerivedMagCurl.bp"; adios2::Engine bpFileWriter = bpOut.Open(filename, adios2::Mode::Write); bpFileWriter.BeginStep(); diff --git a/testing/adios2/derived/TestBPDerivedCorrectnessMPI.cpp b/testing/adios2/derived/TestBPDerivedCorrectnessMPI.cpp new file mode 100644 index 0000000000..3acdb020cf --- /dev/null +++ b/testing/adios2/derived/TestBPDerivedCorrectnessMPI.cpp @@ -0,0 +1,388 @@ +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +#include + +class DerivedCorrectnessMPI : public ::testing::Test +{ +public: + DerivedCorrectnessMPI() = default; +}; + +class DerivedCorrectnessMPIP : public DerivedCorrectnessMPI, + public ::testing::WithParamInterface +{ +protected: + adios2::DerivedVarType GetThreads() { return GetParam(); }; +}; + +TEST_P(DerivedCorrectnessMPIP, ScalarFunctionsCorrectnessTest) +{ + int mpiRank = 0, mpiSize = 1; + const size_t Nx = 2, Ny = 3, Nz = 4; +#if ADIOS2_USE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string filename("ADIOS2BPWriteDerivedScalar_MPI.bp"); +#else + const std::string filename("ADIOS2BPWriteDerivedScalar.bp"); +#endif + + std::vector varname = {"sim/Ux", "sim/Uy", "sim/Uz"}; + const std::string derAgrAdd = "derived/agradd"; + const std::string derAddName = "derived/add"; + const std::string derSubtrName = "derived/subtr"; + const std::string derMultName = "derived/mult"; + const std::string derDivName = "derived/div"; + const std::string derPowName = "derived/pow"; + const std::string derSqrtName = "derived/sqrt"; + + { // write distributed over mpiSize processes + std::default_random_engine generator; + std::uniform_real_distribution distribution(0.0, 10.0); + adios2::DerivedVarType mode = GetParam(); + + std::vector simArray1(Nx * Ny * Nz); + std::vector simArray2(Nx * Ny * Nz); + std::vector simArray3(Nx * Ny * Nz); + for (size_t i = 0; i < Nx * Ny * Nz; ++i) + { + simArray1[i] = distribution(generator); + simArray2[i] = distribution(generator); + simArray3[i] = distribution(generator); + } + +#if ADIOS2_USE_MPI + adios2::ADIOS adios(MPI_COMM_WORLD); +#else + adios2::ADIOS adios; +#endif + + adios2::IO bpOut = adios.DeclareIO("WriteScalarExpression"); + + auto Ux = bpOut.DefineVariable(varname[0], {Nx * mpiSize, Ny, Nz}, + {Nx * mpiRank, 0, 0}, {Nx, Ny, Nz}); + auto Uy = bpOut.DefineVariable(varname[1], {Nx * mpiSize, Ny, Nz}, + {Nx * mpiRank, 0, 0}, {Nx, Ny, Nz}); + auto Uz = bpOut.DefineVariable(varname[2], {Nx * mpiSize, Ny, Nz}, + {Nx * mpiRank, 0, 0}, {Nx, Ny, Nz}); + // clang-format off + bpOut.DefineDerivedVariable(derAgrAdd, + "x=" + varname[0] + "\n" + "add(x)", + mode); + bpOut.DefineDerivedVariable(derAddName, + "x =" + varname[0] + " \n" + "y =" + varname[1] + " \n" + "z =" + varname[2] + " \n" + "x+y+z", + mode); + bpOut.DefineDerivedVariable(derSubtrName, + "x =" + varname[0] + " \n" + "y =" + varname[1] + " \n" + "z =" + varname[2] + " \n" + "x-y-z", + mode); + bpOut.DefineDerivedVariable(derMultName, + "x =" + varname[0] + " \n" + "y =" + varname[1] + " \n" + "z =" + varname[2] + " \n" + "x*y*z", + mode); + bpOut.DefineDerivedVariable(derDivName, + "x =" + varname[0] + " \n" + "y =" + varname[1] + " \n" + "z =" + varname[2] + " \n" + "divide(x,y,z)", + mode); + bpOut.DefineDerivedVariable(derPowName, + "x =" + varname[0] + " \n" + "pow(x)", + mode); + bpOut.DefineDerivedVariable(derSqrtName, + "x =" + varname[0] + " \n" + "sqrt(x)", + mode); + // clang-format on + adios2::Engine bpFileWriter = bpOut.Open(filename, adios2::Mode::Write); + + bpFileWriter.BeginStep(); + bpFileWriter.Put(Ux, simArray1.data()); + bpFileWriter.Put(Uy, simArray2.data()); + bpFileWriter.Put(Uz, simArray3.data()); + bpFileWriter.EndStep(); + bpFileWriter.Close(); + } + + if (mpiRank == 0) + { // read on one process + adios2::ADIOS adios; + adios2::IO bpIn = adios.DeclareIO("ReadScalarExpression"); + adios2::Engine bpFileReader = bpIn.Open(filename, adios2::Mode::Read); + + std::vector readUx(mpiSize * Nx * Ny * Nz); + std::vector readUy(mpiSize * Nx * Ny * Nz); + std::vector readUz(mpiSize * Nx * Ny * Nz); + std::vector readAdd(mpiSize * Nx * Ny * Nz); + std::vector readAgrAdd(mpiSize * Nx * Ny * Nz); + std::vector readSubtr(mpiSize * Nx * Ny * Nz); + std::vector readMult(mpiSize * Nx * Ny * Nz); + std::vector readDiv(mpiSize * Nx * Ny * Nz); + std::vector readPow(mpiSize * Nx * Ny * Nz); + std::vector readSqrt(mpiSize * Nx * Ny * Nz); + + float calcFloat; + double calcDouble; + float epsilon = (float)0.01; + bpFileReader.BeginStep(); + auto varUx = bpIn.InquireVariable(varname[0]); + auto varUy = bpIn.InquireVariable(varname[1]); + auto varUz = bpIn.InquireVariable(varname[2]); + auto varAdd = bpIn.InquireVariable(derAddName); + auto varAgrAdd = bpIn.InquireVariable(derAgrAdd); + auto varSubtr = bpIn.InquireVariable(derSubtrName); + auto varMult = bpIn.InquireVariable(derMultName); + auto varDiv = bpIn.InquireVariable(derDivName); + auto varPow = bpIn.InquireVariable(derPowName); + auto varSqrt = bpIn.InquireVariable(derSqrtName); + + bpFileReader.Get(varUx, readUx); + bpFileReader.Get(varUy, readUy); + bpFileReader.Get(varUz, readUz); + bpFileReader.Get(varAdd, readAdd); + bpFileReader.Get(varAgrAdd, readAgrAdd); + + bpFileReader.Get(varSubtr, readSubtr); + bpFileReader.Get(varMult, readMult); + bpFileReader.Get(varDiv, readDiv); + bpFileReader.Get(varPow, readPow); + bpFileReader.Get(varSqrt, readSqrt); + bpFileReader.EndStep(); + + for (size_t ind = 0; ind < Nx * Ny * Nz; ++ind) + { + calcFloat = readUx[ind] + readUy[ind] + readUz[ind]; + EXPECT_TRUE(fabs(calcFloat - readAdd[ind]) < epsilon); + + calcFloat = readUx[ind] - readUy[ind] - readUz[ind]; + EXPECT_TRUE(fabs(calcFloat - readSubtr[ind]) < epsilon); + + calcFloat = readUx[ind] * readUy[ind] * readUz[ind]; + EXPECT_TRUE(fabs(calcFloat - readMult[ind]) < epsilon); + + calcFloat = readUx[ind] / readUy[ind] / readUz[ind]; + EXPECT_TRUE(fabs(calcFloat - readDiv[ind]) < epsilon); + + calcDouble = std::pow(readUx[ind], 2); + EXPECT_TRUE(fabs(calcDouble - readPow[ind]) < epsilon); + + calcDouble = std::sqrt(readUx[ind]); + EXPECT_TRUE(fabs(calcDouble - readSqrt[ind]) < epsilon); + } + + for (size_t ind = 0; ind < Nx * Ny; ++ind) + { + size_t start = ind * Nz; + float calcA = 0; + for (size_t z = 0; z < Nz; ++z) + { + calcA += readUx[z + start]; + } + EXPECT_TRUE(fabs(calcA - readAgrAdd[ind]) < epsilon); + } + bpFileReader.Close(); + } +} + +TEST_P(DerivedCorrectnessMPIP, TrigCorrectnessTest) +{ + int mpiRank = 0, mpiSize = 1; + const size_t Nx = 2, Ny = 3, Nz = 4; + adios2::DerivedVarType mode = GetParam(); +#if ADIOS2_USE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + const std::string filename("ADIOS2BPWriteDerivedTrig_MPI.bp"); +#else + const std::string filename("ADIOS2BPWriteDerivedTrig.bp"); +#endif + + std::string randDistVar = "sim/randDist"; + std::string sinDistVar = "sim/sinDist"; + std::string derSinName = "derived/sin"; + std::string derCosName = "derived/cos"; + std::string derTanName = "derived/tan"; + std::string derAsinName = "derived/asin"; + std::string derAcosName = "derived/acos"; + std::string derAtanName = "derived/atan"; + + { // write distributed over mpiSize processes + std::default_random_engine generator; + std::uniform_real_distribution randomDist(0.0, 100.0); + std::uniform_real_distribution sinDist(-1.0, 1.0); + + std::vector simArray1(Nx * Ny * Nz); + std::vector simArray2(Nx * Ny * Nz); + for (size_t i = 0; i < Nx * Ny * Nz; ++i) + { + simArray1[i] = randomDist(generator); + simArray2[i] = sinDist(generator); + } +#if ADIOS2_USE_MPI + adios2::ADIOS adios(MPI_COMM_WORLD); +#else + adios2::ADIOS adios; +#endif + + adios2::IO bpOut = adios.DeclareIO("BPWriteSinExpression"); + + auto Ux = bpOut.DefineVariable(randDistVar, {Nx * mpiSize, Ny, Nz}, + {Nx * mpiRank, 0, 0}, {Nx, Ny, Nz}); + auto Uy = bpOut.DefineVariable(sinDistVar, {Nx * mpiSize, Ny, Nz}, + {Nx * mpiRank, 0, 0}, {Nx, Ny, Nz}); + // clang-format off + bpOut.DefineDerivedVariable(derSinName, + "x =" + randDistVar + " \n" + "sin(x)", + mode); + bpOut.DefineDerivedVariable(derCosName, + "x =" + randDistVar + " \n" + "cos(x)", + mode); + bpOut.DefineDerivedVariable(derTanName, + "x =" + randDistVar + " \n" + "tan(x)", + mode); + bpOut.DefineDerivedVariable(derAsinName, + "x =" + sinDistVar + " \n" + "asin(x)", + mode); + bpOut.DefineDerivedVariable(derAcosName, + "x =" + sinDistVar + " \n" + "acos(x)", + mode); + bpOut.DefineDerivedVariable(derAtanName, + "x =" + randDistVar + " \n" + "atan(x)", + mode); + // clang-format on + adios2::Engine bpFileWriter = bpOut.Open(filename, adios2::Mode::Write); + + bpFileWriter.BeginStep(); + bpFileWriter.Put(Ux, simArray1.data()); + bpFileWriter.Put(Uy, simArray2.data()); + bpFileWriter.EndStep(); + bpFileWriter.Close(); + } + + if (mpiRank == 0) + { // read on one process + adios2::ADIOS adios; + adios2::IO bpIn = adios.DeclareIO("BPReadExpression"); + adios2::Engine bpFileReader = bpIn.Open(filename, adios2::Mode::Read); + + std::vector readRandDist(mpiSize * Nx * Ny * Nz); + std::vector readSinDist(mpiSize * Nx * Ny * Nz); + std::vector readSin(mpiSize * Nx * Ny * Nz); + std::vector readCos(mpiSize * Nx * Ny * Nz); + std::vector readTan(mpiSize * Nx * Ny * Nz); + std::vector readAsin(mpiSize * Nx * Ny * Nz); + std::vector readAcos(mpiSize * Nx * Ny * Nz); + std::vector readAtan(mpiSize * Nx * Ny * Nz); + + double calcTrig; + double epsilon = (double)0.01; + bpFileReader.BeginStep(); + + auto varUx = bpIn.InquireVariable(randDistVar); + auto varUy = bpIn.InquireVariable(sinDistVar); + auto varSin = bpIn.InquireVariable(derSinName); + auto varCos = bpIn.InquireVariable(derCosName); + auto varTan = bpIn.InquireVariable(derTanName); + auto varAsin = bpIn.InquireVariable(derAsinName); + auto varAcos = bpIn.InquireVariable(derAcosName); + auto varAtan = bpIn.InquireVariable(derAtanName); + + bpFileReader.Get(varUx, readRandDist); + bpFileReader.Get(varUy, readSinDist); + bpFileReader.Get(varSin, readSin); + bpFileReader.Get(varCos, readCos); + bpFileReader.Get(varTan, readTan); + bpFileReader.Get(varAsin, readAsin); + bpFileReader.Get(varAcos, readAcos); + bpFileReader.Get(varAtan, readAtan); + bpFileReader.EndStep(); + + for (size_t ind = 0; ind < Nx * Ny * Nz; ++ind) + { + calcTrig = std::sin(readRandDist[ind]); + EXPECT_TRUE(fabs(calcTrig - readSin[ind]) < epsilon); + } + + for (size_t ind = 0; ind < Nx * Ny * Nz; ++ind) + { + calcTrig = std::cos(readRandDist[ind]); + EXPECT_TRUE(fabs(calcTrig - readCos[ind]) < epsilon); + } + + for (size_t ind = 0; ind < Nx * Ny * Nz; ++ind) + { + calcTrig = std::tan(readRandDist[ind]); + EXPECT_TRUE(fabs(calcTrig - readTan[ind]) < epsilon); + } + + for (size_t ind = 0; ind < Nx * Ny * Nz; ++ind) + { + calcTrig = std::asin(readSinDist[ind]); + EXPECT_TRUE(fabs(calcTrig - readAsin[ind]) < epsilon); + } + + for (size_t ind = 0; ind < Nx * Ny * Nz; ++ind) + { + calcTrig = std::acos(readSinDist[ind]); + EXPECT_TRUE(fabs(calcTrig - readAcos[ind]) < epsilon); + } + + for (size_t ind = 0; ind < Nx * Ny * Nz; ++ind) + { + calcTrig = std::atan(readRandDist[ind]); + EXPECT_TRUE(fabs(calcTrig - readAtan[ind]) < epsilon); + } + bpFileReader.Close(); + } +} + +INSTANTIATE_TEST_SUITE_P(DerivedCorrectnessMPI, DerivedCorrectnessMPIP, + ::testing::Values(adios2::DerivedVarType::StatsOnly, + adios2::DerivedVarType::ExpressionString, + adios2::DerivedVarType::StoreData)); + +int main(int argc, char **argv) +{ +#if ADIOS2_USE_MPI + int provided; + + // MPI_THREAD_MULTIPLE is only required if you enable the SST MPI_DP + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); +#endif + + int result; + ::testing::InitGoogleTest(&argc, argv); + + result = RUN_ALL_TESTS(); + +#if ADIOS2_USE_MPI + MPI_Finalize(); +#endif + + return result; +}