Skip to content

refactor: use the Buffer and BoundingBox types to perform the communication packing / unpacking in Axis class #385

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
svalat wants to merge 3 commits into
base: develop
Choose a base branch
from
Open

refactor: use the Buffer and BoundingBox types to perform the communication packing / unpacking in Axis class #385

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
74aba9e
refactor: axis communication by using Buffer (step 1 - add communicat…
svalat Jun 19, 2026
da1faf9
refactor: axis communication by using Buffer (step 2 - remove the old…
svalat Jun 19, 2026
b8ea3ff
refactor: use better comments + give uniq name to the idefix_for new …
svalat Jun 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
168 changes: 83 additions & 85 deletions src/fluid/boundary/axis.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -394,11 +394,14 @@ void Axis::ExchangeMPI(int side) {
// Load the buffers with data
int ibeg,iend,jbeg,jend,kbeg,kend,offset;
int nx,ny,nz;
auto bufferSend = this->bufferSend;
Buffer bufferSend = this->bufferSend;
IdefixArray1D<int> map = this->mapVars;
IdefixArray4D<real> Vc = this->Vc;
IdefixArray4D<real> Vs = this->Vs;

//reset pointer
bufferSend.ResetPointer();

// If MPI Persistent, start receiving even before the buffers are filled

MPI_Status sendStatus;
Expand All @@ -419,54 +422,56 @@ void Axis::ExchangeMPI(int side) {
kbeg = data->beg[KDIR];
kend = data->end[KDIR];
nz = kend - kbeg;

// Create the base box to be patched by the ops
BoundingBox baseBox;
baseBox[IDIR][0] = ibeg;
baseBox[IDIR][1] = iend;
baseBox[JDIR][0] = jbeg;
baseBox[JDIR][1] = jend;
baseBox[KDIR][0] = kbeg;
baseBox[KDIR][1] = kend;

if(side==left) {
idefix_for("LoadBufferX2Vc",0,mapNVars,kbeg,kend,jbeg,jend,ibeg,iend,
KOKKOS_LAMBDA (int n, int k, int j, int i) {
bufferSend(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + n*nx*ny*nz ) =
Vc(map(n),k,j+ny,i);
}
);
//shift by NY
BoundingBox sendBoxVc = baseBox;
sendBoxVc[JDIR][0] += ny;
sendBoxVc[JDIR][1] += ny;
bufferSend.Pack(Vc, map, sendBoxVc);

if (haveMHD) {
int VsIndex = mapNVars*nx*ny*nz;
idefix_for("LoadBufferX2IDIR",kbeg,kend,jbeg,jend,ibeg,iend+1,
KOKKOS_LAMBDA (int k, int j, int i) {
bufferSend(i + (j-jbeg)*(nx+1) + (k-kbeg)*(nx+1)*ny + VsIndex ) =
Vs(IDIR,k,j+ny,i);
}
);
VsIndex = mapNVars*nx*ny*nz + (nx+1)*ny*nz;
idefix_for("LoadBufferX2KDIR",kbeg,kend+1,jbeg,jend,ibeg,iend,
KOKKOS_LAMBDA (int k, int j, int i) {
bufferSend(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + VsIndex ) =
Vs(KDIR,k,j+ny,i);
}
);
//extend by one the end on idir
BoundingBox sendBoxVsIdir = sendBoxVc;
sendBoxVsIdir[IDIR][1] += 1;
bufferSend.Pack(Vs, IDIR, sendBoxVsIdir);

//extend by one the end on kdir
BoundingBox sendBoxVsKdir = sendBoxVc;
sendBoxVsKdir[KDIR][1] += 1;
bufferSend.Pack(Vs, KDIR, sendBoxVsKdir);
} // MHD
} else if(side==right) {
idefix_for("LoadBufferX2Vc",0,mapNVars,kbeg,kend,jbeg,jend,ibeg,iend,
KOKKOS_LAMBDA (int n, int k, int j, int i) {
bufferSend(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + n*nx*ny*nz ) =
Vc(map(n),k,j+offset-ny,i);
}
);
//shift by offset - NY (to select the right part)
BoundingBox sendBoxVc = baseBox;
sendBoxVc[JDIR][0] += offset - ny;
sendBoxVc[JDIR][1] += offset - ny;
bufferSend.Pack(Vc, map, sendBoxVc);

// Load face-centered field in the buffer
if (haveMHD) {
int VsIndex = mapNVars*nx*ny*nz;
idefix_for("LoadBufferX2IDIR",kbeg,kend,jbeg,jend,ibeg,iend+1,
KOKKOS_LAMBDA (int k, int j, int i) {
bufferSend(i + (j-jbeg)*(nx+1) + (k-kbeg)*(nx+1)*ny + VsIndex ) =
Vs(IDIR,k,j+offset-ny,i);
}
);
VsIndex = mapNVars*nx*ny*nz + (nx+1)*ny*nz;

idefix_for("LoadBufferX2KDIR",kbeg,kend+1,jbeg,jend,ibeg,iend,
KOKKOS_LAMBDA (int k, int j, int i) {
bufferSend(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + VsIndex ) =
Vs(KDIR,k,j+offset-ny,i);
}
);
//extend by one the end on idir + take the right part or the mesh on jdir
BoundingBox sendBoxVsIdir = baseBox;
sendBoxVsIdir[IDIR][1] += 1;
sendBoxVsIdir[JDIR][0] += offset - ny;
sendBoxVsIdir[JDIR][1] += offset - ny;
bufferSend.Pack(Vs, IDIR, sendBoxVsIdir);

//extend by one the end on kdir + take the right part or the mesh on jdir
BoundingBox sendBoxVsKdir = baseBox;
sendBoxVsKdir[KDIR][1] += 1;
sendBoxVsKdir[JDIR][0] += offset - ny;
sendBoxVsKdir[JDIR][1] += offset - ny;
bufferSend.Pack(Vs, KDIR, sendBoxVsKdir);
} // MHD
} // side==right

Expand All @@ -478,63 +483,55 @@ void Axis::ExchangeMPI(int side) {
idfx::mpiCallsTimer += MPI_Wtime() - tStart;

// Unpack
auto bufferRecv=this->bufferRecv;
Buffer bufferRecv=this->bufferRecv;
bufferRecv.ResetPointer();
auto sVc = this->symmetryVc;

if(side==left) {
idefix_for("StoreBufferAxis",0,mapNVars,kbeg,kend,jbeg,jend,ibeg,iend,
KOKKOS_LAMBDA (int n, int k, int j, int i) {
Vc(map(n),k,jend-(j-jbeg)-1,i) =
sVc(map(n))*bufferRecv(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + n*nx*ny*nz );
}
);
//unpack Vc
BoundingBox recvBoxVc = baseBox;
bufferRecv.UnpackJDirSymetric(Vc, map, sVc, recvBoxVc);

// Load face-centered field in the buffer
if (haveMHD) {
int VsIndex = mapNVars*nx*ny*nz;
auto sVs = this->symmetryVs;
idefix_for("StoreBufferX2IDIR",kbeg,kend,jbeg,jend,ibeg,iend+1,
KOKKOS_LAMBDA (int k, int j, int i) {
Vs(IDIR,k,jend-(j-jbeg)-1,i) =
sVs(IDIR)*bufferRecv(i + (j-jbeg)*(nx+1) + (k-kbeg)*(nx+1)*ny + VsIndex );
}
);

VsIndex = mapNVars*nx*ny*nz + (nx+1)*ny*nz;
//unpack Vs face-centered
BoundingBox recvBoxVsIdir = baseBox;
recvBoxVsIdir[IDIR][1] += 1;
bufferRecv.UnpackJDirSymetric(Vs, IDIR, sVs(IDIR), recvBoxVsIdir);

idefix_for("StoreBufferX2KDIR",kbeg,kend+1,jbeg,jend,ibeg,iend,
KOKKOS_LAMBDA ( int k, int j, int i) {
Vs(KDIR,k,jend-(j-jbeg)-1,i) =
sVs(KDIR)*bufferRecv(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + VsIndex );
}
);
//unpack Vs face-centered
BoundingBox recvBoxVsKdir = baseBox;
recvBoxVsKdir[KDIR][1] += 1;
bufferRecv.UnpackJDirSymetric(Vs, KDIR, sVs(KDIR), recvBoxVsKdir);
}
} else if(side==right) {
idefix_for("StoreBufferAxis",0,mapNVars,kbeg,kend,jbeg,jend,ibeg,iend,
KOKKOS_LAMBDA (int n, int k, int j, int i) {
Vc(map(n),k,jend-(j-jbeg)-1+offset,i) =
sVc(map(n))*bufferRecv(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + n*nx*ny*nz );
}
);
//unpack Vc on right part
BoundingBox recvBoxVc = baseBox;
recvBoxVc[JDIR][0] += offset;
recvBoxVc[JDIR][1] += offset;
bufferRecv.UnpackJDirSymetric(Vc, map, sVc, recvBoxVc);

// Load face-centered field in the buffer
if (haveMHD) {
int VsIndex = mapNVars*nx*ny*nz;
auto sVs = this->symmetryVs;
idefix_for("StoreBufferX2IDIR",kbeg,kend,jbeg,jend,ibeg,iend+1,
KOKKOS_LAMBDA (int k, int j, int i) {
Vs(IDIR,k,jend-(j-jbeg)-1+offset,i) =
sVs(IDIR)*bufferRecv(i + (j-jbeg)*(nx+1) + (k-kbeg)*(nx+1)*ny + VsIndex );
}
);
VsIndex = mapNVars*nx*ny*nz + (nx+1)*ny*nz;

idefix_for("StoreBufferX2KDIR",kbeg,kend+1,jbeg,jend,ibeg,iend,
KOKKOS_LAMBDA ( int k, int j, int i) {
Vs(KDIR,k,jend-(j-jbeg)-1+offset,i) =
sVs(KDIR)*bufferRecv(i + (j-jbeg)*nx + (k-kbeg)*nx*ny + VsIndex );
}
);
//unpack Vs face-centered on right part
BoundingBox recvBoxVsIdir = baseBox;
recvBoxVsIdir[IDIR][1] += 1;
recvBoxVsIdir[JDIR][0] += offset;
recvBoxVsIdir[JDIR][1] += offset;
bufferRecv.UnpackJDirSymetric(Vs, IDIR, sVs(IDIR), recvBoxVsIdir);

//unpack Vs face-centered on right part
BoundingBox recvBoxVsKdir = baseBox;
recvBoxVsKdir[KDIR][1] += 1;
recvBoxVsKdir[JDIR][0] += offset;
recvBoxVsKdir[JDIR][1] += offset;
bufferRecv.UnpackJDirSymetric(Vs, KDIR, sVs(KDIR), recvBoxVsKdir);
} // MHD
}

Expand Down Expand Up @@ -594,8 +591,9 @@ void Axis::InitMPI() {
#endif // DIMENSIONS
}

this->bufferRecv = IdefixArray1D<real>("bufferRecvAxis", bufferSize);
this->bufferSend = IdefixArray1D<real>("bufferSendAxis", bufferSize);
//build buffers
this->bufferRecv = Buffer(bufferSize);
this->bufferSend = Buffer(bufferSize);

// init persistent communications
// We receive from procRecv, and we send to procSend
Expand All @@ -605,10 +603,10 @@ void Axis::InitMPI() {
MPI_SAFE_CALL(MPI_Cart_shift(data->mygrid->AxisComm,0,data->mygrid->nproc[KDIR]/2,
&procRecv,&procSend ));

MPI_SAFE_CALL(MPI_Send_init(bufferSend.data(), bufferSize, realMPI, procSend,
MPI_SAFE_CALL(MPI_Send_init(bufferSend.data(), bufferSend.Size(), realMPI, procSend,
650, data->mygrid->AxisComm, &sendRequest));

MPI_SAFE_CALL(MPI_Recv_init(bufferRecv.data(), bufferSize, realMPI, procRecv,
MPI_SAFE_CALL(MPI_Recv_init(bufferRecv.data(), bufferRecv.Size(), realMPI, procRecv,
650, data->mygrid->AxisComm, &recvRequest));

#endif
Expand Down
5 changes: 3 additions & 2 deletions src/fluid/boundary/axis.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
#include <vector>
#include "idefix.hpp"
#include "grid.hpp"
#include "buffer.hpp"

// Forward class hydro declaration
#include "physics.hpp"
Expand Down Expand Up @@ -59,8 +60,8 @@ class Axis {
MPI_Request sendRequest;
MPI_Request recvRequest;

IdefixArray1D<real> bufferSend;
IdefixArray1D<real> bufferRecv;
Buffer bufferSend;
Buffer bufferRecv;

int bufferSize;

Expand Down
61 changes: 61 additions & 0 deletions src/mpi/buffer.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,10 @@ class Buffer {
return(array.data());
}

const IdefixArray1D<real> & getArray(void) const {
return this->array;
}

int Size() {
return(array.size());
}
Expand Down Expand Up @@ -159,6 +163,33 @@ class Buffer {
this->pointer += ninjnk;
}

void UnpackJDirSymetric(IdefixArray4D<real>& out,
const int var,
const int symMultiplier,
BoundingBox box) {
const int ni = box[IDIR][1]-box[IDIR][0];
const int ninj = (box[JDIR][1]-box[JDIR][0])*ni;
const int ninjnk = (box[KDIR][1]-box[KDIR][0])*ninj;
const int ibeg = box[IDIR][0];
const int jbeg = box[JDIR][0];
const int kbeg = box[KDIR][0];
const int iend = box[IDIR][1];
const int jend = box[JDIR][1];
const int kend = box[KDIR][1];
const int offset = this->pointer;

auto arr = this->array;
idefix_for("UnLoadBuffer4D_var_sym",kbeg,kend,jbeg,jend,ibeg,iend,
KOKKOS_LAMBDA (int k, int j, int i) {
const int jinverted = jend-(j-jbeg)-1;
const int arrIndex = i-ibeg + (j-jbeg)*ni + (k-kbeg)*ninj + offset;
out(var,k,jinverted,i) = symMultiplier * arr(arrIndex);
});

// Update pointer
this->pointer += ninjnk;
}

void Unpack(IdefixArray4D<real>& out,
IdefixArray1D<int>& map,
BoundingBox box) {
Expand Down Expand Up @@ -186,6 +217,36 @@ class Buffer {
this->pointer += ninjnk*map.size();
}

void UnpackJDirSymetric(IdefixArray4D<real>& out,
IdefixArray1D<int>& map,
IdefixArray1D<int>& SymMap,
BoundingBox box) {
const int ni = box[IDIR][1]-box[IDIR][0];
const int ninj = (box[JDIR][1]-box[JDIR][0])*ni;
const int ninjnk = (box[KDIR][1]-box[KDIR][0])*ninj;
const int ibeg = box[IDIR][0];
const int jbeg = box[JDIR][0];
const int kbeg = box[KDIR][0];
const int iend = box[IDIR][1];
const int jend = box[JDIR][1];
const int kend = box[KDIR][1];
const int offset = this->pointer;

auto arr = this->array;
idefix_for("UnLoadBuffer4D_map_sym",0,map.size(),
kbeg,kend,
jbeg,jend,
ibeg,iend,
KOKKOS_LAMBDA (int n, int k, int j, int i) {
const int jinverted = jend-(j-jbeg)-1;
const int arrIndex = i-ibeg + (j-jbeg)*ni + (k-kbeg)*ninj + n*ninjnk + offset;
out(map(n),k,jinverted,i) = SymMap(map(n)) * arr(arrIndex);
});

// Update pointer
this->pointer += ninjnk*map.size();
}


private:
size_t pointer;
Expand Down
Loading