Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Removing unused communications and use ialltoallv #20

Merged
merged 6 commits into from
Feb 29, 2024
Merged

Removing unused communications and use ialltoallv #20

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
e60e8fb
Removed a manual isend/irecv and replaced with alltoallv
marchdf Feb 27, 2024
870d4f6
Swap alltoallv to ialltoallv
marchdf Feb 28, 2024
8c4b0ec
Free sndPack if there is no data to be communicated
itopcuoglu Feb 28, 2024
1e43d8f
Clean up
marchdf Feb 28, 2024
a007ca2
more cleaning
marchdf Feb 28, 2024
f147042
remove fixme comment
marchdf Feb 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 12 additions & 3 deletions src/exchangeAMRDonors.C
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,6 @@ void tioga::exchangeAMRDonors(void)
//
// setup communicator for all to all now
// since the receiver side is unknown
// FIXME:
// add sophisticated code later to fix the all_to_all
// using MPI-2 standard
//
pc_cart->getMap(&nsend_sav,&nrecv_sav,&sndMap,&rcvMap);
sndMapAll=(int *)malloc(sizeof(int)*pc_cart->numprocs);
Expand Down Expand Up @@ -158,6 +155,18 @@ void tioga::exchangeAMRDonors(void)
}
}
}

// Remove sends that won't be used. The first 2 ints are obdonors
// and obreceptors. If nothing else is being sent, then these won't
// be used so we can safely remove them and avoid unnecessary
// communication
for(int i=0;i<nsend;i++){
if((sndPack[i].nints==2) && (sndPack[i].nreals==0)){
if (sndPack[i].intData) TIOGA_FREE(sndPack[i].intData);
if (sndPack[i].realData) TIOGA_FREE(sndPack[i].realData);
sndPack[i].nints=sndPack[i].nreals=0;
}
}
//if (myid==0) {
//for(i=0;i<nsend;i++)
// printf("intcount/intcount=%d %d\n",sndPack[i].nints,intcount[i]);
Expand Down
199 changes: 159 additions & 40 deletions src/parallelComm.C
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,23 +19,20 @@
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "codetypes.h"
#include "mpi.h"
#include <algorithm>
#include <numeric>
#include "parallelComm.h"
#define REAL double

void parallelComm::sendRecvPacketsAll(PACKET *sndPack, PACKET *rcvPack)
{
int i;
int *sint,*sreal,*rint,*rreal;
int tag,irnum;
MPI_Request *request;
MPI_Status *status;
//
sint=(int *)malloc(sizeof(int)*numprocs);
sreal=(int *) malloc(sizeof(int)*numprocs);
rint=(int *)malloc(sizeof(int)*numprocs);
rreal=(int *) malloc(sizeof(int)*numprocs);
request=(MPI_Request *) malloc(sizeof(MPI_Request)*4*numprocs);
status=(MPI_Status *) malloc(sizeof(MPI_Status)*4*numprocs);
//
for(i=0;i<numprocs;i++){
sint[i]=sndPack[i].nints;
Expand All @@ -50,49 +47,171 @@ void parallelComm::sendRecvPacketsAll(PACKET *sndPack, PACKET *rcvPack)
rcvPack[i].nreals=rreal[i];
}
//
irnum=0;
for(i=0;i<numprocs;i++)
{
if (rcvPack[i].nints > 0) {
tag=1;
rcvPack[i].intData=(int *) malloc(sizeof(int)*rcvPack[i].nints);
MPI_Irecv(rcvPack[i].intData,rcvPack[i].nints,
MPI_INT,i,
tag,scomm,&request[irnum++]);
}
if (rcvPack[i].nreals > 0) {
tag=2;
rcvPack[i].realData=(REAL *) malloc(sizeof(REAL)*rcvPack[i].nreals);
MPI_Irecv(rcvPack[i].realData,rcvPack[i].nreals,
MPI_DOUBLE,i,
tag,scomm,&request[irnum++]);
}

int all_snd_nints = std::accumulate(sint, sint + numprocs, 0);
int all_rcv_nints = std::accumulate(rint, rint + numprocs, 0);
int *all_snd_intData, *all_rcv_intData;
all_snd_intData=(int *) malloc(sizeof(int)*all_snd_nints);
all_rcv_intData=(int *) malloc(sizeof(int)*all_rcv_nints);
std::vector<int> snd_int_displs(numprocs+1, 0);
std::vector<int> rcv_int_displs(numprocs+1, 0);
for (int i=1; i <= numprocs; i++) {
snd_int_displs[i] = snd_int_displs[i-1] + sint[i-1];
rcv_int_displs[i] = rcv_int_displs[i-1] + rint[i-1];
}
for (int i=0; i < numprocs; i++) {
int displ = snd_int_displs[i];
for(int j=0; j < sint[i]; j++){
all_snd_intData[displ+j] = sndPack[i].intData[j];
}
for(i=0;i<numprocs;i++)
{
if (sndPack[i].nints > 0){
tag=1;
MPI_Isend(sndPack[i].intData,sndPack[i].nints,
MPI_INT,i,
tag,scomm,&request[irnum++]);
}
if (sndPack[i].nreals > 0){
tag=2;
MPI_Isend(sndPack[i].realData,sndPack[i].nreals,
MPI_DOUBLE,i,
tag,scomm,&request[irnum++]);
}
}
MPI_Request int_request;
MPI_Ialltoallv(all_snd_intData,
sint,
snd_int_displs.data(),
MPI_INT,
all_rcv_intData,
rint,
rcv_int_displs.data(),
MPI_INT,
scomm,
&int_request);

int all_snd_nreals = std::accumulate(sreal, sreal + numprocs, 0);
int all_rcv_nreals = std::accumulate(rreal, rreal + numprocs, 0);
REAL *all_snd_realData, *all_rcv_realData;
all_snd_realData=(REAL *) malloc(sizeof(REAL)*all_snd_nreals);
all_rcv_realData=(REAL *) malloc(sizeof(REAL)*all_rcv_nreals);
std::vector<int> snd_real_displs(numprocs+1, 0);
std::vector<int> rcv_real_displs(numprocs+1, 0);
for (int i=1; i <= numprocs; i++) {
snd_real_displs[i] = snd_real_displs[i-1] + sreal[i-1];
rcv_real_displs[i] = rcv_real_displs[i-1] + rreal[i-1];
}
for (int i=0; i < numprocs; i++) {
int displ = snd_real_displs[i];
for(int j=0; j < sreal[i]; j++){
all_snd_realData[displ+j] = sndPack[i].realData[j];
}
MPI_Waitall(irnum,request,status);

}
MPI_Request real_request;
MPI_Ialltoallv(all_snd_realData,
sreal,
snd_real_displs.data(),
MPI_DOUBLE,
all_rcv_realData,
rreal,
rcv_real_displs.data(),
MPI_DOUBLE,
scomm,
&real_request);

MPI_Wait(&int_request, MPI_STATUS_IGNORE);
for(i=0;i<numprocs;i++){
if (rcvPack[i].nints > 0) {
rcvPack[i].intData=(int *) malloc(sizeof(int)*rcvPack[i].nints);
}
if (rcvPack[i].nreals > 0) {
rcvPack[i].realData=(REAL *) malloc(sizeof(REAL)*rcvPack[i].nreals);
}
}

MPI_Wait(&real_request, MPI_STATUS_IGNORE);
for (int i=0; i < numprocs; i++) {
int displ = rcv_int_displs[i];
for(int j=0; j < rint[i]; j++){
rcvPack[i].intData[j] = all_rcv_intData[displ+j];
}
}
for (int i=0; i < numprocs; i++) {
int displ = rcv_real_displs[i];
for(int j=0; j < rreal[i]; j++){
rcvPack[i].realData[j] = all_rcv_realData[displ+j];
}
}

TIOGA_FREE(all_snd_intData);
TIOGA_FREE(all_rcv_intData);
TIOGA_FREE(all_snd_realData);
TIOGA_FREE(all_rcv_realData);
TIOGA_FREE(sint);
TIOGA_FREE(sreal);
TIOGA_FREE(rint);
TIOGA_FREE(rreal);
TIOGA_FREE(request);
TIOGA_FREE(status);
}

// Old version that does not use ialltoallv
// void parallelComm::sendRecvPacketsAll(PACKET *sndPack, PACKET *rcvPack)
// {
// int i;
// int *sint,*sreal,*rint,*rreal;
// int tag,irnum;
// MPI_Request *request;
// MPI_Status *status;
// //
// sint=(int *)malloc(sizeof(int)*numprocs);
// sreal=(int *) malloc(sizeof(int)*numprocs);
// rint=(int *)malloc(sizeof(int)*numprocs);
// rreal=(int *) malloc(sizeof(int)*numprocs);
// request=(MPI_Request *) malloc(sizeof(MPI_Request)*4*numprocs);
// status=(MPI_Status *) malloc(sizeof(MPI_Status)*4*numprocs);
// //
// for(i=0;i<numprocs;i++){
// sint[i]=sndPack[i].nints;
// sreal[i]=sndPack[i].nreals;
// }
// //
// MPI_Alltoall(sint,1,MPI_INT,rint,1,MPI_INT,scomm);
// MPI_Alltoall(sreal,1,MPI_INT,rreal,1,MPI_INT,scomm);
// //
// for(i=0;i<numprocs;i++) {
// rcvPack[i].nints=rint[i];
// rcvPack[i].nreals=rreal[i];
// }
// //
// irnum=0;
// for(i=0;i<numprocs;i++)
// {
// if (rcvPack[i].nints > 0) {
// tag=1;
// rcvPack[i].intData=(int *) malloc(sizeof(int)*rcvPack[i].nints);
// MPI_Irecv(rcvPack[i].intData,rcvPack[i].nints,
// MPI_INT,i,
// tag,scomm,&request[irnum++]);
// }
// if (rcvPack[i].nreals > 0) {
// tag=2;
// rcvPack[i].realData=(REAL *) malloc(sizeof(REAL)*rcvPack[i].nreals);
// MPI_Irecv(rcvPack[i].realData,rcvPack[i].nreals,
// MPI_DOUBLE,i,
// tag,scomm,&request[irnum++]);
// }
// }
// for(i=0;i<numprocs;i++)
// {
// if (sndPack[i].nints > 0){
// tag=1;
// MPI_Isend(sndPack[i].intData,sndPack[i].nints,
// MPI_INT,i,
// tag,scomm,&request[irnum++]);
// }
// if (sndPack[i].nreals > 0){
// tag=2;
// MPI_Isend(sndPack[i].realData,sndPack[i].nreals,
// MPI_DOUBLE,i,
// tag,scomm,&request[irnum++]);
// }
// }
// MPI_Waitall(irnum,request,status);

// TIOGA_FREE(sint);
// TIOGA_FREE(sreal);
// TIOGA_FREE(rint);
// TIOGA_FREE(rreal);
// TIOGA_FREE(request);
// TIOGA_FREE(status);
// }

void parallelComm::sendRecvPackets(PACKET *sndPack,PACKET *rcvPack)
{
int i;
Expand Down
Loading