From 3109d1266fb607356d63a7e42190deb7606bb17f Mon Sep 17 00:00:00 2001
From: Christophe Geuzaine <cgeuzaine@uliege.be>
Date: Thu, 20 Jan 2022 14:36:12 +0100
Subject: [PATCH] print system info sooner + cleanup messages
---
src/problem/Formulation.cpp | 112 +++++++++++++++++++-----------------
1 file changed, 58 insertions(+), 54 deletions(-)
diff --git a/src/problem/Formulation.cpp b/src/problem/Formulation.cpp
index 4bcabcb1..669b5986 100644
--- a/src/problem/Formulation.cpp
+++ b/src/problem/Formulation.cpp
@@ -54,7 +54,7 @@ namespace gmshddm
MPI_Barrier(PETSC_COMM_WORLD);
}
else {
- gmshfem::msg::info << "* Resources @ " << step << " Wall = " << time << "s, "
+ gmshfem::msg::info << "* Resources @ " << step << ": Wall = " << time << "s, "
<< "CPU = " << cpuTime << "s, "
<< "PeakRSS = " << peakRSS << "Mb, "
<< "CurrentRSS = " << currentRSS << "Mb"
@@ -307,25 +307,24 @@ namespace gmshddm
template< class T_Scalar >
gmshfem::common::Timer Formulation< T_Scalar >::pre()
{
- MPI_Barrier(PETSC_COMM_WORLD);
- printResources("DDM Pre-Processing Begin", true);
-
- gmshfem::common::Timer time;
- time.tick();
-
const int outerVerbosity = gmshfem::common::Options::instance()->verbose;
const int innerVerbosity = (outerVerbosity != 0 ? outerVerbosity - 1 : 0);
const unsigned int MPI_Rank = mpi::getMPIRank();
const unsigned int MPI_Size = mpi::getMPISize();
+ MPI_Barrier(PETSC_COMM_WORLD);
+
+ gmshfem::common::Timer time;
+ time.tick();
+
if(MPI_Rank == 0) {
gmshfem::msg::info << "Pre-processing " << _name << "..." << gmshfem::msg::endl;
}
-
if(_interfaceFields.size() == 0) {
gmshfem::msg::warning << "There is no interface field defined. Perhaps you have forgotten to declare them with 'addInterfaceField' function." << gmshfem::msg::endl;
}
+ printResources("DDM Pre-Processing Begin", true);
// Build interface field mapping
if(MPI_Size != 1) {
@@ -361,19 +360,21 @@ namespace gmshddm
_physicalCommutator = true;
_artificialCommutator = false;
- gmshfem::common::Options::instance()->verbose = innerVerbosity;
-
// Pre-process the problem for all subdomains to get the dofs from the neighbours
for(auto idom = 0ULL; idom < _volume.size(); ++idom) {
if(mpi::isItMySubdomain(idom)) {
+ gmshfem::common::Options::instance()->verbose = innerVerbosity;
_volume[idom]->pre();
+ gmshfem::common::Options::instance()->verbose = outerVerbosity;
}
}
if(MPI_Size == 1) {
for(auto idom = 0ULL; idom < _volume.size(); ++idom) {
for(auto it = _surface[idom].begin(); it != _surface[idom].end(); ++it) {
+ gmshfem::common::Options::instance()->verbose = innerVerbosity;
it->second->pre();
+ gmshfem::common::Options::instance()->verbose = outerVerbosity;
}
}
}
@@ -425,7 +426,9 @@ namespace gmshddm
}
for(auto it = _surface[idom].begin(); it != _surface[idom].end(); ++it) {
if(recv) {
+ gmshfem::common::Options::instance()->verbose = innerVerbosity;
it->second->pre();
+ gmshfem::common::Options::instance()->verbose = outerVerbosity;
}
if(mpi::isItMySubdomain(it->first)) continue;
if(recv) {
@@ -749,54 +752,63 @@ namespace gmshddm
MPI_Barrier(PETSC_COMM_WORLD);
+
+ for(auto idom = 0ULL; idom < _volume.size(); ++idom) {
+ if(mpi::isItMySubdomain(idom)) {
+ gmshfem::msg::info << "Subdomain " << idom << ":" << gmshfem::msg::endl;
+ gmshfem::msg::info << " - Subdomain size: " << _volume[idom]->getNumberOfUnknownDof() << " dofs" << gmshfem::msg::endl;
+ unsigned long long totalInterfaceSize = 0;
+ for(auto it = _surface[idom].begin(); it != _surface[idom].end(); ++it) {
+ totalInterfaceSize += it->second->getNumberOfUnknownDof();
+ }
+ gmshfem::msg::info << " - Interface size: " << totalInterfaceSize << " dofs:" << gmshfem::msg::endl;
+ for(auto it = _surface[idom].begin(); it != _surface[idom].end(); ++it) {
+ gmshfem::msg::info << " * Interface 'g(" << idom << ", " << it->first << ")' size: " << it->second->getNumberOfUnknownDof() << " dofs" << gmshfem::msg::endl;
+ }
+ }
+ }
printResources("DDM Pre-Processing Before Volume Assembly");
// Assemble locally on each processor
for(auto idom = 0ULL; idom < _volume.size(); ++idom) {
if(mpi::isItMySubdomain(idom)) {
- gmshfem::common::Options::instance()->verbose = outerVerbosity;
gmshfem::msg::info << " - assembling " << _volume[idom]->name() << gmshfem::msg::endl;
- gmshfem::common::Options::instance()->verbose = innerVerbosity;
_volume[idom]->setAttribute("ddm::physicalCommutator", _physicalCommutator);
_volume[idom]->setAttribute("ddm::artificialCommutator", _artificialCommutator);
+ gmshfem::common::Options::instance()->verbose = innerVerbosity;
_volume[idom]->assemble();
+ gmshfem::common::Options::instance()->verbose = outerVerbosity;
}
}
MPI_Barrier(PETSC_COMM_WORLD);
- gmshfem::common::Options::instance()->verbose = outerVerbosity;
printResources("DDM Pre-Processing Before Volume Solve");
- gmshfem::common::Options::instance()->verbose = innerVerbosity;
// solve locally on each processor
for(auto idom = 0ULL; idom < _volume.size(); ++idom) {
if(mpi::isItMySubdomain(idom)) {
- gmshfem::common::Options::instance()->verbose = outerVerbosity;
gmshfem::msg::info << " - solving " << _volume[idom]->name() << gmshfem::msg::endl;
gmshfem::common::Options::instance()->verbose = innerVerbosity;
_volume[idom]->solve();
+ gmshfem::common::Options::instance()->verbose = outerVerbosity;
}
}
MPI_Barrier(PETSC_COMM_WORLD);
- gmshfem::common::Options::instance()->verbose = outerVerbosity;
printResources("DDM Pre-Processing Before Surface Assembly and Solve");
- gmshfem::common::Options::instance()->verbose = innerVerbosity;
for(auto idom = 0ULL; idom < _surface.size(); ++idom) {
if(mpi::isItMySubdomain(idom)) {
for(auto it = _surface[idom].begin(); it != _surface[idom].end(); ++it) {
- gmshfem::common::Options::instance()->verbose = outerVerbosity;
gmshfem::msg::info << " - assembling and solving " << it->second->name() << gmshfem::msg::endl;
gmshfem::common::Options::instance()->verbose = innerVerbosity;
it->second->assemble();
it->second->solve();
+ gmshfem::common::Options::instance()->verbose = outerVerbosity;
}
}
}
- gmshfem::common::Options::instance()->verbose = outerVerbosity;
-
MPI_Barrier(PETSC_COMM_WORLD);
// get the local interface unknowns for each processor
@@ -815,22 +827,8 @@ namespace gmshddm
}
}
- for(auto idom = 0ULL; idom < _surface.size(); ++idom) {
- if(mpi::isItMySubdomain(idom)) {
- gmshfem::msg::info << "Subdomain " << idom << ":" << gmshfem::msg::endl;
- gmshfem::msg::info << " - Subdomain size: " << _volume[idom]->getNumberOfUnknownDof() << " dofs" << gmshfem::msg::endl;
- unsigned long long totalInterfaceSize = 0;
- for(auto it = _surface[idom].begin(); it != _surface[idom].end(); ++it) {
- totalInterfaceSize += it->second->getNumberOfUnknownDof();
- }
- gmshfem::msg::info << " - Interface size: " << totalInterfaceSize << " dofs:" << gmshfem::msg::endl;
- for(auto it = _surface[idom].begin(); it != _surface[idom].end(); ++it) {
- gmshfem::msg::info << " * Interface 'g(" << idom << ", " << it->first << ")' size: " << it->second->getNumberOfUnknownDof() << " dofs" << gmshfem::msg::endl;
- }
- }
- }
-
MPI_Barrier(PETSC_COMM_WORLD);
+
printResources("DDM Pre-Processing End");
time.tock();
@@ -849,20 +847,20 @@ namespace gmshddm
// physical/artifical communicator and if there are no non-homogeneous
// Dirichlet BCs
- MPI_Barrier(PETSC_COMM_WORLD);
- printResources("DDM Iterative Solver Begin");
-
- gmshfem::common::Timer time;
- time.tick();
-
const int outerVerbosity = gmshfem::common::Options::instance()->verbose;
const int innerVerbosity = (outerVerbosity != 0 ? outerVerbosity - 1 : 0);
const unsigned int MPI_Rank = mpi::getMPIRank();
+ MPI_Barrier(PETSC_COMM_WORLD);
+
+ gmshfem::common::Timer time;
+ time.tick();
+
if(MPI_Rank == 0) {
gmshfem::msg::info << "Solving " << _name << " using " << solver << "..." << gmshfem::msg::endl;
}
+ printResources("DDM Iterative Solver Begin");
// ******************************************************
// Iterative solver
@@ -890,8 +888,6 @@ namespace gmshddm
_physicalCommutator = false;
_artificialCommutator = true;
- gmshfem::common::Options::instance()->verbose = innerVerbosity;
-
auto deactivateBilinear = [=](const unsigned long long idom) {
for(auto it = _volume[idom]->begin(); it != _volume[idom]->end(); ++it) {
if((*it)->isBilinear()) {
@@ -927,6 +923,7 @@ namespace gmshddm
if(mpi::isItMySubdomain(idom)) {
_volume[idom]->setAttribute("ddm::physicalCommutator", _physicalCommutator);
_volume[idom]->setAttribute("ddm::artificialCommutator", _artificialCommutator);
+ gmshfem::common::Options::instance()->verbose = innerVerbosity;
if(sameMatrixWithArtificialAndPhysicalSources) {
deactivateBilinear(idom);
_volume[idom]->setRHSToZero();
@@ -939,11 +936,10 @@ namespace gmshddm
_volume[idom]->assemble();
deactivateBilinear(idom);
}
+ gmshfem::common::Options::instance()->verbose = outerVerbosity;
}
}
- gmshfem::common::Options::instance()->verbose = outerVerbosity;
-
if(solver == "jacobi") {
_IA = false;
Vec W; // residual
@@ -988,8 +984,6 @@ namespace gmshddm
// Final solution
// ******************************************************
- gmshfem::common::Options::instance()->verbose = innerVerbosity;
-
_physicalCommutator = true;
_artificialCommutator = true;
@@ -997,6 +991,7 @@ namespace gmshddm
if(mpi::isItMySubdomain(idom)) {
_volume[idom]->setAttribute("ddm::physicalCommutator", _physicalCommutator);
_volume[idom]->setAttribute("ddm::artificialCommutator", _artificialCommutator);
+ gmshfem::common::Options::instance()->verbose = innerVerbosity;
if(sameMatrixWithArtificialAndPhysicalSources) {
_volume[idom]->setRHSToZero();
}
@@ -1005,6 +1000,7 @@ namespace gmshddm
_volume[idom]->initSystem();
_volume[idom]->pre();
}
+ gmshfem::common::Options::instance()->verbose = outerVerbosity;
}
}
@@ -1020,6 +1016,7 @@ namespace gmshddm
// SolveVolumePDE:
for(auto idom = 0ULL; idom < _volume.size(); ++idom) {
if(mpi::isItMySubdomain(idom)) {
+ gmshfem::common::Options::instance()->verbose = innerVerbosity;
if(!sameMatrixWithArtificialAndPhysicalSources) {
activateBilinear(idom);
}
@@ -1028,11 +1025,10 @@ namespace gmshddm
if(sameMatrixWithArtificialAndPhysicalSources) {
activateBilinear(idom);
}
+ gmshfem::common::Options::instance()->verbose = outerVerbosity;
}
}
- gmshfem::common::Options::instance()->verbose = outerVerbosity;
-
MatDestroy(&A);
VecDestroy(&X);
VecDestroy(&Y);
@@ -1053,6 +1049,9 @@ namespace gmshddm
template< class T_Scalar >
gmshfem::algebra::MatrixCCS< T_Scalar > Formulation< T_Scalar >::computeMatrix()
{
+ const int outerVerbosity = gmshfem::common::Options::instance()->verbose;
+ const int innerVerbosity = (outerVerbosity != 0 ? outerVerbosity - 1 : 0);
+
if(mpi::getMPISize() != 1) {
throw gmshfem::common::Exception("Formulation< T_Scalar >::computeMatrix() does not work on multiple processes");
}
@@ -1076,7 +1075,9 @@ namespace gmshddm
for(auto idom = 0ULL; idom < _volume.size(); ++idom) {
_volume[idom]->setAttribute("ddm::physicalCommutator", _physicalCommutator);
_volume[idom]->setAttribute("ddm::artificialCommutator", _artificialCommutator);
+ gmshfem::common::Options::instance()->verbose = innerVerbosity;
_volume[idom]->pre();
+ gmshfem::common::Options::instance()->verbose = outerVerbosity;
}
_IA = true;
@@ -1152,19 +1153,21 @@ namespace gmshddm
Formulation< T_Scalar >::template PetscInterface< T_Scalar, PetscScalar, PetscInt >::freeArray(array);
VecRestoreArrayRead(X, &petscArray);
- gmshfem::common::Options::instance()->verbose = innerVerbosity;
-
// SolveVolumePDE
for(auto idom = 0ULL; idom < formulation->_volume.size(); ++idom) {
if(mpi::isItMySubdomain(idom)) {
+ gmshfem::common::Options::instance()->verbose = innerVerbosity;
formulation->_volume[idom]->setRHSToZero();
formulation->_volume[idom]->assemble();
+ gmshfem::common::Options::instance()->verbose = outerVerbosity;
}
}
for(auto idom = 0ULL; idom < formulation->_volume.size(); ++idom) {
if(mpi::isItMySubdomain(idom)) {
+ gmshfem::common::Options::instance()->verbose = innerVerbosity;
formulation->_volume[idom]->solve(true);
+ gmshfem::common::Options::instance()->verbose = outerVerbosity;
}
}
@@ -1172,8 +1175,10 @@ namespace gmshddm
for(auto idom = 0ULL; idom < formulation->_surface.size(); ++idom) {
if(mpi::isItMySubdomain(idom)) {
for(auto it = formulation->_surface[idom].begin(); it != formulation->_surface[idom].end(); ++it) {
+ gmshfem::common::Options::instance()->verbose = innerVerbosity;
it->second->setRHSToZero();
it->second->assemble();
+ gmshfem::common::Options::instance()->verbose = outerVerbosity;
}
}
}
@@ -1181,15 +1186,14 @@ namespace gmshddm
for(auto idom = 0ULL; idom < formulation->_surface.size(); ++idom) {
if(mpi::isItMySubdomain(idom)) {
for(auto it = formulation->_surface[idom].begin(); it != formulation->_surface[idom].end(); ++it) {
+ gmshfem::common::Options::instance()->verbose = innerVerbosity;
it->second->solve(true);
+ gmshfem::common::Options::instance()->verbose = outerVerbosity;
}
}
}
MPI_Barrier(PETSC_COMM_WORLD);
- printResources("DDM Iteration");
-
- gmshfem::common::Options::instance()->verbose = outerVerbosity;
gmshfem::algebra::Vector< T_Scalar > g_local;
for(auto idom = 0ULL; idom < formulation->_surface.size(); ++idom) {
--
GitLab