From e60e8fbced30127ce7bce4aa6a7cf792134f919a Mon Sep 17 00:00:00 2001 From: Marc Henry de Frahan Date: Tue, 27 Feb 2024 15:11:52 -0700 Subject: [PATCH 1/6] Removed a manual isend/irecv and replaced with alltoallv --- src/exchangeAMRDonors.C | 24 ++++++++- src/parallelComm.C | 112 ++++++++++++++++++++++++++++++++++++++++ src/parallelComm.h | 4 +- 3 files changed, 138 insertions(+), 2 deletions(-) diff --git a/src/exchangeAMRDonors.C b/src/exchangeAMRDonors.C index e6a6af5..5c99d2c 100644 --- a/src/exchangeAMRDonors.C +++ b/src/exchangeAMRDonors.C @@ -20,6 +20,7 @@ #include "codetypes.h" #include "tioga.h" #include +#include using namespace TIOGA; @@ -170,7 +171,28 @@ void tioga::exchangeAMRDonors(void) // // send to All because there is no gain for now // - pc_cart->sendRecvPacketsAll(sndPack,rcvPack); + pc_cart->sendRecvPacketsAll2(sndPack,rcvPack); + //pc_cart->sendRecvPacketsAll(sndPack,rcvPack); + // for (i = 0; i < numprocs; i++) { + // for(int j = 0; j < rcvPack[i].nints; j++){ + // if (rcvPack2[i].intData[j] != rcvPack[i].intData[j]) { + // std::cout << "int recv: " << rcvPack2[i].intData[j] << " " + // << rcvPack[i].intData[j] << std::endl; + // } + // else{ + // std::cout << "these ints are the same" << std::endl; + // } + // } + // for(int j = 0; j < rcvPack[i].nreals; j++){ + // if (rcvPack2[i].realData[j] != rcvPack[i].realData[j]) { + // std::cout << "real recv: " << rcvPack2[i].realData[j] << " " + // << rcvPack[i].realData[j] << std::endl; + // } + // else{ + // std::cout << "these reals are the same" << std::endl; + // } + // } + // } // // decode the data now // diff --git a/src/parallelComm.C b/src/parallelComm.C index 9a4684c..0f27f6d 100644 --- a/src/parallelComm.C +++ b/src/parallelComm.C @@ -19,9 +19,121 @@ // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include "codetypes.h" #include "mpi.h" +#include +#include #include "parallelComm.h" #define REAL double +void parallelComm::sendRecvPacketsAll2(PACKET *sndPack, PACKET *rcvPack) +{ + int i; + int *sint,*sreal,*rint,*rreal; + // + sint=(int *)malloc(sizeof(int)*numprocs); + sreal=(int *) malloc(sizeof(int)*numprocs); + rint=(int *)malloc(sizeof(int)*numprocs); + rreal=(int *) malloc(sizeof(int)*numprocs); + // + for(i=0;i snd_int_displs(numprocs+1, 0); + std::vector rcv_int_displs(numprocs+1, 0); + for (int i=1; i <= numprocs; i++) { + snd_int_displs[i] = snd_int_displs[i-1] + sint[i-1]; + rcv_int_displs[i] = rcv_int_displs[i-1] + rint[i-1]; + } + for (int i=0; i < numprocs; i++) { + int displ = snd_int_displs[i]; + for(int j=0; j < sint[i]; j++){ + all_snd_intData[displ+j] = sndPack[i].intData[j]; + } + } + + int all_snd_nreals = std::accumulate(sreal, sreal + numprocs, 0); + int all_rcv_nreals = std::accumulate(rreal, rreal + numprocs, 0); + REAL *all_snd_realData, *all_rcv_realData; + all_snd_realData=(REAL *) malloc(sizeof(REAL)*all_snd_nreals); + all_rcv_realData=(REAL *) malloc(sizeof(REAL)*all_rcv_nreals); + std::vector snd_real_displs(numprocs+1, 0); + std::vector rcv_real_displs(numprocs+1, 0); + for (int i=1; i <= numprocs; i++) { + snd_real_displs[i] = snd_real_displs[i-1] + sreal[i-1]; + rcv_real_displs[i] = rcv_real_displs[i-1] + rreal[i-1]; + } + for (int i=0; i < numprocs; i++) { + int displ = snd_real_displs[i]; + for(int j=0; j < sreal[i]; j++){ + all_snd_realData[displ+j] = sndPack[i].realData[j]; + } + } + + MPI_Alltoallv(all_snd_intData, + sint, + snd_int_displs.data(), + MPI_INT, + all_rcv_intData, + rint, + rcv_int_displs.data(), + MPI_INT, + scomm); + MPI_Alltoallv(all_snd_realData, + sreal, + snd_real_displs.data(), + MPI_DOUBLE, + all_rcv_realData, + rreal, + rcv_real_displs.data(), + MPI_DOUBLE, + scomm); + + for(i=0;i 0) { + rcvPack[i].intData=(int *) malloc(sizeof(int)*rcvPack[i].nints); + } + if (rcvPack[i].nreals > 0) { + rcvPack[i].realData=(REAL *) malloc(sizeof(REAL)*rcvPack[i].nreals); + } + } + for (int i=0; i < numprocs; i++) { + int displ = rcv_int_displs[i]; + for(int j=0; j < rint[i]; j++){ + rcvPack[i].intData[j] = all_rcv_intData[displ+j]; + } + } + for (int i=0; i < numprocs; i++) { + int displ = rcv_real_displs[i]; + for(int j=0; j < rreal[i]; j++){ + rcvPack[i].realData[j] = all_rcv_realData[displ+j]; + } + } + + TIOGA_FREE(all_snd_intData); + TIOGA_FREE(all_rcv_intData); + TIOGA_FREE(all_snd_realData); + TIOGA_FREE(all_rcv_realData); + TIOGA_FREE(sint); + TIOGA_FREE(sreal); + TIOGA_FREE(rint); + TIOGA_FREE(rreal); +} + void parallelComm::sendRecvPacketsAll(PACKET *sndPack, PACKET *rcvPack) { int i; diff --git a/src/parallelComm.h b/src/parallelComm.h index 217673a..40ee9dd 100644 --- a/src/parallelComm.h +++ b/src/parallelComm.h @@ -50,8 +50,10 @@ class parallelComm ~parallelComm() { if (sndMap) free(sndMap); if (rcvMap) free(rcvMap);} - void sendRecvPacketsAll(PACKET *sndPack,PACKET *rcvPack); + void sendRecvPacketsAll2(PACKET *sndPack,PACKET *rcvPack); + void sendRecvPacketsAll(PACKET *sndPack,PACKET *rcvPack); + void sendRecvPackets(PACKET *sndPack,PACKET *rcvPack); void sendRecvPacketsCheck(PACKET *sndPack,PACKET *rcvPack); From 870d4f62b220a2066224a8020e7afd27331c1508 Mon Sep 17 00:00:00 2001 From: Marc Henry de Frahan Date: Wed, 28 Feb 2024 09:30:24 -0700 Subject: [PATCH 2/6] Swap alltoallv to ialltoallv --- src/parallelComm.C | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/src/parallelComm.C b/src/parallelComm.C index 0f27f6d..2e8da22 100644 --- a/src/parallelComm.C +++ b/src/parallelComm.C @@ -65,6 +65,17 @@ void parallelComm::sendRecvPacketsAll2(PACKET *sndPack, PACKET *rcvPack) all_snd_intData[displ+j] = sndPack[i].intData[j]; } } + MPI_Request int_request; + MPI_Ialltoallv(all_snd_intData, + sint, + snd_int_displs.data(), + MPI_INT, + all_rcv_intData, + rint, + rcv_int_displs.data(), + MPI_INT, + scomm, + &int_request); int all_snd_nreals = std::accumulate(sreal, sreal + numprocs, 0); int all_rcv_nreals = std::accumulate(rreal, rreal + numprocs, 0); @@ -83,17 +94,8 @@ void parallelComm::sendRecvPacketsAll2(PACKET *sndPack, PACKET *rcvPack) all_snd_realData[displ+j] = sndPack[i].realData[j]; } } - - MPI_Alltoallv(all_snd_intData, - sint, - snd_int_displs.data(), - MPI_INT, - all_rcv_intData, - rint, - rcv_int_displs.data(), - MPI_INT, - scomm); - MPI_Alltoallv(all_snd_realData, + MPI_Request real_request; + MPI_Ialltoallv(all_snd_realData, sreal, snd_real_displs.data(), MPI_DOUBLE, @@ -101,8 +103,10 @@ void parallelComm::sendRecvPacketsAll2(PACKET *sndPack, PACKET *rcvPack) rreal, rcv_real_displs.data(), MPI_DOUBLE, - scomm); + scomm, + &real_request); + MPI_Wait(&int_request, MPI_STATUS_IGNORE); for(i=0;i 0) { rcvPack[i].intData=(int *) malloc(sizeof(int)*rcvPack[i].nints); @@ -111,6 +115,8 @@ void parallelComm::sendRecvPacketsAll2(PACKET *sndPack, PACKET *rcvPack) rcvPack[i].realData=(REAL *) malloc(sizeof(REAL)*rcvPack[i].nreals); } } + + MPI_Wait(&real_request, MPI_STATUS_IGNORE); for (int i=0; i < numprocs; i++) { int displ = rcv_int_displs[i]; for(int j=0; j < rint[i]; j++){ From 8c4b0ece495dd81a29d6a85e25d80472016ff9ae Mon Sep 17 00:00:00 2001 From: Ilker Topcuoglu Date: Wed, 28 Feb 2024 10:50:45 -0700 Subject: [PATCH 3/6] Free sndPack if there is no data to be communicated --- src/exchangeAMRDonors.C | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/exchangeAMRDonors.C b/src/exchangeAMRDonors.C index 5c99d2c..17cfaf2 100644 --- a/src/exchangeAMRDonors.C +++ b/src/exchangeAMRDonors.C @@ -171,6 +171,13 @@ void tioga::exchangeAMRDonors(void) // // send to All because there is no gain for now // + for(int i=0;isendRecvPacketsAll2(sndPack,rcvPack); //pc_cart->sendRecvPacketsAll(sndPack,rcvPack); // for (i = 0; i < numprocs; i++) { From 1e43d8f18656eae708ef79f1e3d635b1815c4fe9 Mon Sep 17 00:00:00 2001 From: Marc Henry de Frahan Date: Wed, 28 Feb 2024 13:42:33 -0700 Subject: [PATCH 4/6] Clean up --- src/exchangeAMRDonors.C | 42 ++++-------- src/parallelComm.C | 141 ++++++++++++++++++++-------------------- src/parallelComm.h | 2 - 3 files changed, 84 insertions(+), 101 deletions(-) diff --git a/src/exchangeAMRDonors.C b/src/exchangeAMRDonors.C index 17cfaf2..d641123 100644 --- a/src/exchangeAMRDonors.C +++ b/src/exchangeAMRDonors.C @@ -159,6 +159,18 @@ void tioga::exchangeAMRDonors(void) } } } + + // Remove sends that won't be used. The first 2 ints are obdonors + // and obreceptors. If nothing else is being sent, then these won't + // be used so we can safely remove them and avoid unnecessary + // communication + for(int i=0;isendRecvPacketsAll2(sndPack,rcvPack); - //pc_cart->sendRecvPacketsAll(sndPack,rcvPack); - // for (i = 0; i < numprocs; i++) { - // for(int j = 0; j < rcvPack[i].nints; j++){ - // if (rcvPack2[i].intData[j] != rcvPack[i].intData[j]) { - // std::cout << "int recv: " << rcvPack2[i].intData[j] << " " - // << rcvPack[i].intData[j] << std::endl; - // } - // else{ - // std::cout << "these ints are the same" << std::endl; - // } - // } - // for(int j = 0; j < rcvPack[i].nreals; j++){ - // if (rcvPack2[i].realData[j] != rcvPack[i].realData[j]) { - // std::cout << "real recv: " << rcvPack2[i].realData[j] << " " - // << rcvPack[i].realData[j] << std::endl; - // } - // else{ - // std::cout << "these reals are the same" << std::endl; - // } - // } - // } + pc_cart->sendRecvPacketsAll(sndPack,rcvPack); // // decode the data now // diff --git a/src/parallelComm.C b/src/parallelComm.C index 2e8da22..b00f246 100644 --- a/src/parallelComm.C +++ b/src/parallelComm.C @@ -24,7 +24,7 @@ #include "parallelComm.h" #define REAL double -void parallelComm::sendRecvPacketsAll2(PACKET *sndPack, PACKET *rcvPack) +void parallelComm::sendRecvPacketsAll(PACKET *sndPack, PACKET *rcvPack) { int i; int *sint,*sreal,*rint,*rreal; @@ -140,76 +140,77 @@ void parallelComm::sendRecvPacketsAll2(PACKET *sndPack, PACKET *rcvPack) TIOGA_FREE(rreal); } -void parallelComm::sendRecvPacketsAll(PACKET *sndPack, PACKET *rcvPack) -{ - int i; - int *sint,*sreal,*rint,*rreal; - int tag,irnum; - MPI_Request *request; - MPI_Status *status; - // - sint=(int *)malloc(sizeof(int)*numprocs); - sreal=(int *) malloc(sizeof(int)*numprocs); - rint=(int *)malloc(sizeof(int)*numprocs); - rreal=(int *) malloc(sizeof(int)*numprocs); - request=(MPI_Request *) malloc(sizeof(MPI_Request)*4*numprocs); - status=(MPI_Status *) malloc(sizeof(MPI_Status)*4*numprocs); - // - for(i=0;i 0) { - tag=1; - rcvPack[i].intData=(int *) malloc(sizeof(int)*rcvPack[i].nints); - MPI_Irecv(rcvPack[i].intData,rcvPack[i].nints, - MPI_INT,i, - tag,scomm,&request[irnum++]); - } - if (rcvPack[i].nreals > 0) { - tag=2; - rcvPack[i].realData=(REAL *) malloc(sizeof(REAL)*rcvPack[i].nreals); - MPI_Irecv(rcvPack[i].realData,rcvPack[i].nreals, - MPI_DOUBLE,i, - tag,scomm,&request[irnum++]); - } - } - for(i=0;i 0){ - tag=1; - MPI_Isend(sndPack[i].intData,sndPack[i].nints, - MPI_INT,i, - tag,scomm,&request[irnum++]); - } - if (sndPack[i].nreals > 0){ - tag=2; - MPI_Isend(sndPack[i].realData,sndPack[i].nreals, - MPI_DOUBLE,i, - tag,scomm,&request[irnum++]); - } - } - MPI_Waitall(irnum,request,status); +// Old version that does not use ialltoallv +// void parallelComm::sendRecvPacketsAll(PACKET *sndPack, PACKET *rcvPack) +// { +// int i; +// int *sint,*sreal,*rint,*rreal; +// int tag,irnum; +// MPI_Request *request; +// MPI_Status *status; +// // +// sint=(int *)malloc(sizeof(int)*numprocs); +// sreal=(int *) malloc(sizeof(int)*numprocs); +// rint=(int *)malloc(sizeof(int)*numprocs); +// rreal=(int *) malloc(sizeof(int)*numprocs); +// request=(MPI_Request *) malloc(sizeof(MPI_Request)*4*numprocs); +// status=(MPI_Status *) malloc(sizeof(MPI_Status)*4*numprocs); +// // +// for(i=0;i 0) { +// tag=1; +// rcvPack[i].intData=(int *) malloc(sizeof(int)*rcvPack[i].nints); +// MPI_Irecv(rcvPack[i].intData,rcvPack[i].nints, +// MPI_INT,i, +// tag,scomm,&request[irnum++]); +// } +// if (rcvPack[i].nreals > 0) { +// tag=2; +// rcvPack[i].realData=(REAL *) malloc(sizeof(REAL)*rcvPack[i].nreals); +// MPI_Irecv(rcvPack[i].realData,rcvPack[i].nreals, +// MPI_DOUBLE,i, +// tag,scomm,&request[irnum++]); +// } +// } +// for(i=0;i 0){ +// tag=1; +// MPI_Isend(sndPack[i].intData,sndPack[i].nints, +// MPI_INT,i, +// tag,scomm,&request[irnum++]); +// } +// if (sndPack[i].nreals > 0){ +// tag=2; +// MPI_Isend(sndPack[i].realData,sndPack[i].nreals, +// MPI_DOUBLE,i, +// tag,scomm,&request[irnum++]); +// } +// } +// MPI_Waitall(irnum,request,status); - TIOGA_FREE(sint); - TIOGA_FREE(sreal); - TIOGA_FREE(rint); - TIOGA_FREE(rreal); - TIOGA_FREE(request); - TIOGA_FREE(status); -} +// TIOGA_FREE(sint); +// TIOGA_FREE(sreal); +// TIOGA_FREE(rint); +// TIOGA_FREE(rreal); +// TIOGA_FREE(request); +// TIOGA_FREE(status); +// } void parallelComm::sendRecvPackets(PACKET *sndPack,PACKET *rcvPack) { diff --git a/src/parallelComm.h b/src/parallelComm.h index 40ee9dd..6742ef8 100644 --- a/src/parallelComm.h +++ b/src/parallelComm.h @@ -50,8 +50,6 @@ class parallelComm ~parallelComm() { if (sndMap) free(sndMap); if (rcvMap) free(rcvMap);} - void sendRecvPacketsAll2(PACKET *sndPack,PACKET *rcvPack); - void sendRecvPacketsAll(PACKET *sndPack,PACKET *rcvPack); void sendRecvPackets(PACKET *sndPack,PACKET *rcvPack); From a007ca217124aead785ddcfcd72bd1b9f12eb0ef Mon Sep 17 00:00:00 2001 From: Marc Henry de Frahan Date: Wed, 28 Feb 2024 13:44:30 -0700 Subject: [PATCH 5/6] more cleaning --- src/exchangeAMRDonors.C | 1 - src/parallelComm.h | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/exchangeAMRDonors.C b/src/exchangeAMRDonors.C index d641123..feb8e3c 100644 --- a/src/exchangeAMRDonors.C +++ b/src/exchangeAMRDonors.C @@ -20,7 +20,6 @@ #include "codetypes.h" #include "tioga.h" #include -#include using namespace TIOGA; diff --git a/src/parallelComm.h b/src/parallelComm.h index 6742ef8..217673a 100644 --- a/src/parallelComm.h +++ b/src/parallelComm.h @@ -51,7 +51,7 @@ class parallelComm if (rcvMap) free(rcvMap);} void sendRecvPacketsAll(PACKET *sndPack,PACKET *rcvPack); - + void sendRecvPackets(PACKET *sndPack,PACKET *rcvPack); void sendRecvPacketsCheck(PACKET *sndPack,PACKET *rcvPack); From f147042796ae0dccb210c4f49c27a2e5b4beb30e Mon Sep 17 00:00:00 2001 From: Marc Henry de Frahan Date: Thu, 29 Feb 2024 08:48:02 -0700 Subject: [PATCH 6/6] remove fixme comment --- src/exchangeAMRDonors.C | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/exchangeAMRDonors.C b/src/exchangeAMRDonors.C index feb8e3c..eba12c0 100644 --- a/src/exchangeAMRDonors.C +++ b/src/exchangeAMRDonors.C @@ -48,9 +48,6 @@ void tioga::exchangeAMRDonors(void) // // setup communicator for all to all now // since the receiver side is unknown - // FIXME: - // add sophisticated code later to fix the all_to_all - // using MPI-2 standard // pc_cart->getMap(&nsend_sav,&nrecv_sav,&sndMap,&rcvMap); sndMapAll=(int *)malloc(sizeof(int)*pc_cart->numprocs);