From 06657626efb3799e8f33e7e4de3f2b7ae7b9eed6 Mon Sep 17 00:00:00 2001 From: Boris Martin <boris.martin@uliege.be> Date: Thu, 17 Apr 2025 16:27:33 +0200 Subject: [PATCH] Topology aware distbuted formulation --- src/field/DistributedField.cpp | 35 +++++++++++++++++++++++++++++++++- src/problem/Formulation.cpp | 2 +- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/src/field/DistributedField.cpp b/src/field/DistributedField.cpp index ff56b830..5f1ea0d5 100644 --- a/src/field/DistributedField.cpp +++ b/src/field/DistributedField.cpp @@ -14,6 +14,7 @@ #include <CSVio.h> #include <Options.h> #include <gmsh.h> +#include <numeric> #ifdef HAVE_MPI #include <mpi.h> @@ -178,7 +179,39 @@ namespace gmshfem::field MPI_COMM_WORLD); } else { - // TODO + const auto &ranks = *neighboringRanks; + std::vector< unsigned long long > recvSizes(ranks.size()); + unsigned long long toSend = local.size(); + + // 1) Send my local size to all my neighbors + std::vector<MPI_Request> sendRequests(ranks.size()), receiveRequests(ranks.size()); + for (size_t k = 0; k < ranks.size(); ++k) { + MPI_Isend(&toSend, 1, MPI_UNSIGNED_LONG_LONG, ranks[k], rank, MPI_COMM_WORLD, &sendRequests[k]); + MPI_Irecv(&recvSizes[k], 1, MPI_UNSIGNED_LONG_LONG, ranks[k], ranks[k], MPI_COMM_WORLD, &receiveRequests[k]); + } + MPI_Waitall(sendRequests.size(), sendRequests.data(), MPI_STATUSES_IGNORE); + MPI_Waitall(receiveRequests.size(), receiveRequests.data(), MPI_STATUSES_IGNORE); + + size_t total_size = std::reduce(recvSizes.begin(), recvSizes.end()); + + // Does global need to contain local ? + global.resize(total_size); + sendRequests.clear(); sendRequests.resize(ranks.size()); + receiveRequests.clear(); receiveRequests.resize(ranks.size()); + + size_t currentOffset = 0; + for (size_t k = 0; k < ranks.size(); ++k) { + MPI_Isend(local.data(), local.size(), mpi_struct_type, ranks[k], rank, MPI_COMM_WORLD, &sendRequests[k]); + MPI_Irecv(global.data() + currentOffset, recvSizes[k], mpi_struct_type, ranks[k], ranks[k], MPI_COMM_WORLD, &receiveRequests[k]); + currentOffset += recvSizes[k]; + } + + MPI_Waitall(sendRequests.size(), sendRequests.data(), MPI_STATUSES_IGNORE); + MPI_Waitall(receiveRequests.size(), receiveRequests.data(), MPI_STATUSES_IGNORE); + for (auto entry: local) { + global.push_back(entry); + } + } // Free the type diff --git a/src/problem/Formulation.cpp b/src/problem/Formulation.cpp index ad21949a..b7c52f04 100644 --- a/src/problem/Formulation.cpp +++ b/src/problem/Formulation.cpp @@ -1151,7 +1151,7 @@ namespace gmshfem::problem // Make all fields synchronize ownerships for(auto [tag, fieldInterfacePtr] : _unknownFields) { msg::debug << '[' << rank << "] does MPI prepro of field with tag " << tag << '.' << msg::endl; - fieldInterfacePtr->preProMPI(); + fieldInterfacePtr->preProMPI(neighboringRanks); localOwnedNumDof += fieldInterfacePtr->getAllOwnedDofs().size(); localNonOwnedNumDof += fieldInterfacePtr->getNonOwnedDofs().size(); } -- GitLab