Skip to content

Commit

Permalink
Merge pull request #107 from ECP-WarpX/runtime_boosted_frame_particles
Browse files Browse the repository at this point in the history 
Runtime boosted frame particles
  • Loading branch information
@RemiLehe
RemiLehe authored May 1, 2019
2 parents e80b825 + deea0e1 commit 0f7796b
Show file tree
Hide file tree
Showing 11 changed files with 182 additions and 95 deletions.
2 changes: 0 additions & 2 deletions GNUmakefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@ TINY_PROFILE = TRUE
#COMM_PROFILE = TRUE
#TRACE_PROFILE = TRUE

STORE_OLD_PARTICLE_ATTRIBS = FALSE

USE_OMP = TRUE
USE_GPU = FALSE

Expand Down
19 changes: 10 additions & 9 deletions Source/Diagnostics/WarpXIO.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -631,15 +631,16 @@ WarpX::WritePlotFile () const
particle_varnames.push_back("theta");
#endif

#ifdef WARPX_STORE_OLD_PARTICLE_ATTRIBS
particle_varnames.push_back("xold");
particle_varnames.push_back("yold");
particle_varnames.push_back("zold");

particle_varnames.push_back("uxold");
particle_varnames.push_back("uyold");
particle_varnames.push_back("uzold");
#endif
if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles)
{
particle_varnames.push_back("xold");
particle_varnames.push_back("yold");
particle_varnames.push_back("zold");

particle_varnames.push_back("uxold");
particle_varnames.push_back("uyold");
particle_varnames.push_back("uzold");
}

mypc->WritePlotFile(plotfilename, particle_plot_flags, particle_varnames);

Expand Down
11 changes: 5 additions & 6 deletions Source/FortranInterface/WarpX_f.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,17 +81,16 @@ extern "C"
{
#endif

#ifdef WARPX_STORE_OLD_PARTICLE_ATTRIBS
void warpx_copy_attribs(const long* np,
const amrex_real* xp, const amrex_real* yp, const amrex_real* zp,
const amrex_real* uxp, const amrex_real* uyp, const amrex_real* uzp,
amrex_real* xpold, amrex_real* ypold, amrex_real* zpold,
amrex_real* uxpold, amrex_real* uypold, amrex_real* uzpold);
#endif
void WRPX_COPY_SLICE(const int* lo, const int* hi,
const amrex_real* tmp, const int* tlo, const int* thi,
amrex_real* buf, const int* blo, const int* bhi,
const int* ncomp, const int* i_boost, const int* i_lab);
void WRPX_COPY_SLICE(const int* lo, const int* hi,
const amrex_real* tmp, const int* tlo, const int* thi,
amrex_real* buf, const int* blo, const int* bhi,
const int* ncomp, const int* i_boost, const int* i_lab);

// Charge deposition
void warpx_charge_deposition(amrex::Real* rho,
Expand Down
4 changes: 0 additions & 4 deletions Source/Make.WarpX
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,10 +123,6 @@ ifeq ($(USE_RZ),TRUE)
DEFINES += -DWARPX_RZ
endif

ifeq ($(STORE_OLD_PARTICLE_ATTRIBS),TRUE)
DEFINES += -DWARPX_STORE_OLD_PARTICLE_ATTRIBS
endif

ifeq ($(DO_ELECTROSTATIC),TRUE)
include $(AMREX_HOME)/Src/LinearSolvers/C_to_F_MG/Make.package
include $(AMREX_HOME)/Src/LinearSolvers/F_MG/FParallelMG.mak
Expand Down
21 changes: 20 additions & 1 deletion Source/Particles/MultiParticleContainer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,26 @@ MultiParticleContainer::MultiParticleContainer (AmrCore* amr_core)
allcontainers[i].reset(new LaserParticleContainer(amr_core,i, lasers_names[i-nspecies]));
}

pc_tmp.reset(new PhysicalParticleContainer(amr_core));
pc_tmp.reset(new PhysicalParticleContainer(amr_core));

if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles)
{
for (int i = 0; i < nspecies + nlasers; ++i)
{
allcontainers[i]->AddRealComp("xold");
allcontainers[i]->AddRealComp("yold");
allcontainers[i]->AddRealComp("zold");
allcontainers[i]->AddRealComp("uxold");
allcontainers[i]->AddRealComp("uyold");
allcontainers[i]->AddRealComp("uzold");
}
pc_tmp->AddRealComp("xold");
pc_tmp->AddRealComp("yold");
pc_tmp->AddRealComp("zold");
pc_tmp->AddRealComp("uxold");
pc_tmp->AddRealComp("uyold");
pc_tmp->AddRealComp("uzold");
}
}

void
Expand Down
100 changes: 56 additions & 44 deletions Source/Particles/PhysicalParticleContainer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,18 @@ PhysicalParticleContainer::AddGaussianBeam(Real x_m, Real y_m, Real z_m,
attribs[PIdx::uz] = u[2];
attribs[PIdx::w ] = weight;

if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles)
{
auto& particle_tile = DefineAndReturnParticleTile(0, 0, 0);
particle_tile.push_back_real(particle_comps["xold"], x);
particle_tile.push_back_real(particle_comps["yold"], y);
particle_tile.push_back_real(particle_comps["zold"], z);

particle_tile.push_back_real(particle_comps["uxold"], u[0]);
particle_tile.push_back_real(particle_comps["uyold"], u[1]);
particle_tile.push_back_real(particle_comps["uzold"], u[2]);
}

AddOneParticle(0, 0, 0, x, y, z, attribs);
}
}
Expand Down Expand Up @@ -455,16 +467,18 @@ PhysicalParticleContainer::AddPlasmaCPU (int lev, RealBox part_realbox)
attribs[PIdx::ux] = u[0];
attribs[PIdx::uy] = u[1];
attribs[PIdx::uz] = u[2];

#ifdef WARPX_STORE_OLD_PARTICLE_ATTRIBS
attribs[PIdx::xold] = x;
attribs[PIdx::yold] = y;
attribs[PIdx::zold] = z;

attribs[PIdx::uxold] = u[0];
attribs[PIdx::uyold] = u[1];
attribs[PIdx::uzold] = u[2];
#endif

if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles)
{
auto& particle_tile = DefineAndReturnParticleTile(lev, grid_id, tile_id);
particle_tile.push_back_real(particle_comps["xold"], x);
particle_tile.push_back_real(particle_comps["yold"], y);
particle_tile.push_back_real(particle_comps["zold"], z);

particle_tile.push_back_real(particle_comps["uxold"], u[0]);
particle_tile.push_back_real(particle_comps["uyold"], u[1]);
particle_tile.push_back_real(particle_comps["uzold"], u[2]);
}

AddOneParticle(lev, grid_id, tile_id, x, y, z, attribs);
}
Expand Down Expand Up @@ -695,15 +709,18 @@ PhysicalParticleContainer::AddPlasmaGPU (int lev, RealBox part_realbox)
attribs[PIdx::uy] = u[1];
attribs[PIdx::uz] = u[2];

#ifdef WARPX_STORE_OLD_PARTICLE_ATTRIBS
attribs[PIdx::xold] = x;
attribs[PIdx::yold] = y;
attribs[PIdx::zold] = z;

attribs[PIdx::uxold] = u[0];
attribs[PIdx::uyold] = u[1];
attribs[PIdx::uzold] = u[2];
#endif
// note - this will be slow on the GPU, need to revisit
if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles)
{
auto& particle_tile = DefineAndReturnParticleTile(lev, grid_id, tile_id);
particle_tile.push_back_real(particle_comps["xold"], x);
particle_tile.push_back_real(particle_comps["yold"], y);
particle_tile.push_back_real(particle_comps["zold"], z);

particle_tile.push_back_real(particle_comps["uxold"], u[0]);
particle_tile.push_back_real(particle_comps["uyold"], u[1]);
particle_tile.push_back_real(particle_comps["uzold"], u[2]);
}

ParticleType p;
p.id() = ParticleType::NextID();
Expand Down Expand Up @@ -1666,20 +1683,20 @@ PhysicalParticleContainer::PushPX(WarpXParIter& pti,
auto& Bzp = attribs[PIdx::Bz];
const long np = pti.numParticles();

#ifdef WARPX_STORE_OLD_PARTICLE_ATTRIBS
auto& xpold = attribs[PIdx::xold];
auto& ypold = attribs[PIdx::yold];
auto& zpold = attribs[PIdx::zold];
auto& uxpold = attribs[PIdx::uxold];
auto& uypold = attribs[PIdx::uyold];
auto& uzpold = attribs[PIdx::uzold];

warpx_copy_attribs(&np, xp.dataPtr(), yp.dataPtr(), zp.dataPtr(),
uxp.dataPtr(), uyp.dataPtr(), uzp.dataPtr(),
xpold.dataPtr(), ypold.dataPtr(), zpold.dataPtr(),
uxpold.dataPtr(), uypold.dataPtr(), uzpold.dataPtr());

#endif
if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles)
{
auto& xpold = pti.GetAttribs(particle_comps["xold"]);
auto& ypold = pti.GetAttribs(particle_comps["yold"]);
auto& zpold = pti.GetAttribs(particle_comps["zold"]);
auto& uxpold = pti.GetAttribs(particle_comps["uxold"]);
auto& uypold = pti.GetAttribs(particle_comps["uyold"]);
auto& uzpold = pti.GetAttribs(particle_comps["uzold"]);

warpx_copy_attribs(&np, xp.dataPtr(), yp.dataPtr(), zp.dataPtr(),
uxp.dataPtr(), uyp.dataPtr(), uzp.dataPtr(),
xpold.dataPtr(), ypold.dataPtr(), zpold.dataPtr(),
uxpold.dataPtr(), uypold.dataPtr(), uzpold.dataPtr());
}

warpx_particle_pusher(&np,
xp.dataPtr(),
Expand Down Expand Up @@ -1801,7 +1818,6 @@ void PhysicalParticleContainer::GetParticleSlice(const int direction, const Real
{
BL_PROFILE("PhysicalParticleContainer::GetParticleSlice");

#ifdef WARPX_STORE_OLD_PARTICLE_ATTRIBS
// Assume that the boost in the positive z direction.
#if (AMREX_SPACEDIM == 2)
AMREX_ALWAYS_ASSERT(direction == 1);
Expand Down Expand Up @@ -1864,12 +1880,12 @@ void PhysicalParticleContainer::GetParticleSlice(const int direction, const Real
auto& uyp_new = attribs[PIdx::uy ];
auto& uzp_new = attribs[PIdx::uz ];

auto& xp_old = attribs[PIdx::xold ];
auto& yp_old = attribs[PIdx::yold ];
auto& zp_old = attribs[PIdx::zold ];
auto& uxp_old = attribs[PIdx::uxold];
auto& uyp_old = attribs[PIdx::uyold];
auto& uzp_old = attribs[PIdx::uzold];
auto& xp_old = pti.GetAttribs(particle_comps["xold"]);
auto& yp_old = pti.GetAttribs(particle_comps["yold"]);
auto& zp_old = pti.GetAttribs(particle_comps["zold"]);
auto& uxp_old = pti.GetAttribs(particle_comps["uxold"]);
auto& uyp_old = pti.GetAttribs(particle_comps["uyold"]);
auto& uzp_old = pti.GetAttribs(particle_comps["uzold"]);

const long np = pti.numParticles();

Expand Down Expand Up @@ -1919,10 +1935,6 @@ void PhysicalParticleContainer::GetParticleSlice(const int direction, const Real
}
}
}
#else
AMREX_ALWAYS_ASSERT_WITH_MESSAGE( false ,
"ERROR: WarpX must be compiled with STORE_OLD_PARTICLE_ATTRIBS=TRUE to use the back-transformed diagnostics");
#endif
}

int PhysicalParticleContainer::GetRefineFac(const Real x, const Real y, const Real z)
Expand Down
28 changes: 14 additions & 14 deletions Source/Particles/RigidInjectedParticleContainer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -225,20 +225,20 @@ RigidInjectedParticleContainer::PushPX(WarpXParIter& pti,
auto& Bzp = attribs[PIdx::Bz];
const long np = pti.numParticles();

#ifdef WARPX_STORE_OLD_PARTICLE_ATTRIBS
auto& xpold = attribs[PIdx::xold];
auto& ypold = attribs[PIdx::yold];
auto& zpold = attribs[PIdx::zold];
auto& uxpold = attribs[PIdx::uxold];
auto& uypold = attribs[PIdx::uyold];
auto& uzpold = attribs[PIdx::uzold];

warpx_copy_attribs(&np, xp.dataPtr(), yp.dataPtr(), zp.dataPtr(),
uxp.dataPtr(), uyp.dataPtr(), uzp.dataPtr(),
xpold.dataPtr(), ypold.dataPtr(), zpold.dataPtr(),
uxpold.dataPtr(), uypold.dataPtr(), uzpold.dataPtr());

#endif
if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles)
{
auto& xpold = pti.GetAttribs(particle_comps["xold"]);
auto& ypold = pti.GetAttribs(particle_comps["yold"]);
auto& zpold = pti.GetAttribs(particle_comps["zold"]);
auto& uxpold = pti.GetAttribs(particle_comps["uxold"]);
auto& uypold = pti.GetAttribs(particle_comps["uyold"]);
auto& uzpold = pti.GetAttribs(particle_comps["uzold"]);

warpx_copy_attribs(&np, xp.dataPtr(), yp.dataPtr(), zp.dataPtr(),
uxp.dataPtr(), uyp.dataPtr(), uzp.dataPtr(),
xpold.dataPtr(), ypold.dataPtr(), zpold.dataPtr(),
uxpold.dataPtr(), uypold.dataPtr(), uzpold.dataPtr());
}

// Save the position and momenta, making copies
Cuda::ManagedDeviceVector<Real> xp_save, yp_save, zp_save;
Expand Down
30 changes: 18 additions & 12 deletions Source/Particles/WarpXParticleContainer.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,6 @@ struct PIdx
#ifdef WARPX_RZ
theta, // RZ needs all three position components
#endif
#ifdef WARPX_STORE_OLD_PARTICLE_ATTRIBS
xold, yold, zold, uxold, uyold, uzold,
#endif
nattribs
};
};
Expand All @@ -42,15 +39,7 @@ namespace ParticleStringNames
{"Ez", PIdx::Ez },
{"Bx", PIdx::Bx },
{"By", PIdx::By },
{"Bz", PIdx::Bz },
#ifdef WARPX_STORE_OLD_PARTICLE_ATTRIBS
{"xold", PIdx::xold },
{"yold", PIdx::yold },
{"zold", PIdx::zold },
{"uxold", PIdx::uxold},
{"uyold", PIdx::uyold},
{"uzold", PIdx::uzold},
#endif
{"Bz", PIdx::Bz }
};
}

Expand Down Expand Up @@ -231,8 +220,25 @@ public:
// split along axes (0) or diagonals (1)
int split_type = 0;

using amrex::ParticleContainer<0, 0, PIdx::nattribs>::AddRealComp;
using amrex::ParticleContainer<0, 0, PIdx::nattribs>::AddIntComp;

void AddRealComp (const std::string& name, bool comm=true)
{
particle_comps[name] = NumRealComps();
AddRealComp(comm);
}

void AddIntComp (const std::string& name, bool comm=true)
{
particle_comps[name] = NumIntComps();
AddIntComp(comm);
}

protected:

std::map<std::string, int> particle_comps;

int species_id;

amrex::Real charge;
Expand Down
43 changes: 43 additions & 0 deletions Source/Particles/WarpXParticleContainer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,32 @@ WarpXParticleContainer::WarpXParticleContainer (AmrCore* amr_core, int ispecies)
SetParticleSize();
ReadParameters();

// build up the map of string names to particle component numbers
particle_comps["w"] = PIdx::w;
particle_comps["ux"] = PIdx::ux;
particle_comps["uy"] = PIdx::uy;
particle_comps["uz"] = PIdx::uz;
particle_comps["Ex"] = PIdx::Ex;
particle_comps["Ey"] = PIdx::Ey;
particle_comps["Ez"] = PIdx::Ez;
particle_comps["Bx"] = PIdx::Bx;
particle_comps["By"] = PIdx::By;
particle_comps["Bz"] = PIdx::Bz;
#ifdef WARPX_RZ
particle_comps["theta"] = PIdx::theta;
#endif

if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles)
{
particle_comps["xold"] = PIdx::nattribs;
particle_comps["yold"] = PIdx::nattribs+1;
particle_comps["zold"] = PIdx::nattribs+2;
particle_comps["uxold"] = PIdx::nattribs+3;
particle_comps["uyold"] = PIdx::nattribs+4;
particle_comps["uzold"] = PIdx::nattribs+5;

}

// Initialize temporary local arrays for charge/current deposition
int num_threads = 1;
#ifdef _OPENMP
Expand Down Expand Up @@ -204,6 +230,15 @@ WarpXParticleContainer::AddNParticles (int lev,
#endif
p.pos(1) = z[i];
#endif

if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles)
{
auto& particle_tile = DefineAndReturnParticleTile(0, 0, 0);
particle_tile.push_back_real(particle_comps["xold"], x[i]);
particle_tile.push_back_real(particle_comps["yold"], y[i]);
particle_tile.push_back_real(particle_comps["zold"], z[i]);
}

particle_tile.push_back(p);
}

Expand All @@ -214,6 +249,14 @@ WarpXParticleContainer::AddNParticles (int lev,
particle_tile.push_back_real(PIdx::uy, vy + ibegin, vy + iend);
particle_tile.push_back_real(PIdx::uz, vz + ibegin, vz + iend);

if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles)
{
auto& particle_tile = DefineAndReturnParticleTile(0, 0, 0);
particle_tile.push_back_real(particle_comps["uxold"], vx + ibegin, vx + iend);
particle_tile.push_back_real(particle_comps["uyold"], vy + ibegin, vy + iend);
particle_tile.push_back_real(particle_comps["uzold"], vz + ibegin, vz + iend);
}

for (int comp = PIdx::uz+1; comp < PIdx::nattribs; ++comp)
{
#ifdef WARPX_RZ
Expand Down
Loading

0 comments on commit 0f7796b

Please sign in to comment.