Skip to content
Snippets Groups Projects
Commit 3109d126 authored by Christophe Geuzaine's avatar Christophe Geuzaine
Browse files

print system info sooner + cleanup messages

parent e750e721
No related branches found
No related tags found
No related merge requests found
Pipeline #9270 passed
......@@ -54,7 +54,7 @@ namespace gmshddm
MPI_Barrier(PETSC_COMM_WORLD);
}
else {
gmshfem::msg::info << "* Resources @ " << step << " Wall = " << time << "s, "
gmshfem::msg::info << "* Resources @ " << step << ": Wall = " << time << "s, "
<< "CPU = " << cpuTime << "s, "
<< "PeakRSS = " << peakRSS << "Mb, "
<< "CurrentRSS = " << currentRSS << "Mb"
......@@ -307,25 +307,24 @@ namespace gmshddm
template< class T_Scalar >
gmshfem::common::Timer Formulation< T_Scalar >::pre()
{
MPI_Barrier(PETSC_COMM_WORLD);
printResources("DDM Pre-Processing Begin", true);
gmshfem::common::Timer time;
time.tick();
const int outerVerbosity = gmshfem::common::Options::instance()->verbose;
const int innerVerbosity = (outerVerbosity != 0 ? outerVerbosity - 1 : 0);
const unsigned int MPI_Rank = mpi::getMPIRank();
const unsigned int MPI_Size = mpi::getMPISize();
MPI_Barrier(PETSC_COMM_WORLD);
gmshfem::common::Timer time;
time.tick();
if(MPI_Rank == 0) {
gmshfem::msg::info << "Pre-processing " << _name << "..." << gmshfem::msg::endl;
}
if(_interfaceFields.size() == 0) {
gmshfem::msg::warning << "There is no interface field defined. Perhaps you have forgotten to declare them with 'addInterfaceField' function." << gmshfem::msg::endl;
}
printResources("DDM Pre-Processing Begin", true);
// Build interface field mapping
if(MPI_Size != 1) {
......@@ -361,19 +360,21 @@ namespace gmshddm
_physicalCommutator = true;
_artificialCommutator = false;
gmshfem::common::Options::instance()->verbose = innerVerbosity;
// Pre-process the problem for all subdomains to get the dofs from the neighbours
for(auto idom = 0ULL; idom < _volume.size(); ++idom) {
if(mpi::isItMySubdomain(idom)) {
gmshfem::common::Options::instance()->verbose = innerVerbosity;
_volume[idom]->pre();
gmshfem::common::Options::instance()->verbose = outerVerbosity;
}
}
if(MPI_Size == 1) {
for(auto idom = 0ULL; idom < _volume.size(); ++idom) {
for(auto it = _surface[idom].begin(); it != _surface[idom].end(); ++it) {
gmshfem::common::Options::instance()->verbose = innerVerbosity;
it->second->pre();
gmshfem::common::Options::instance()->verbose = outerVerbosity;
}
}
}
......@@ -425,7 +426,9 @@ namespace gmshddm
}
for(auto it = _surface[idom].begin(); it != _surface[idom].end(); ++it) {
if(recv) {
gmshfem::common::Options::instance()->verbose = innerVerbosity;
it->second->pre();
gmshfem::common::Options::instance()->verbose = outerVerbosity;
}
if(mpi::isItMySubdomain(it->first)) continue;
if(recv) {
......@@ -749,54 +752,63 @@ namespace gmshddm
MPI_Barrier(PETSC_COMM_WORLD);
for(auto idom = 0ULL; idom < _volume.size(); ++idom) {
if(mpi::isItMySubdomain(idom)) {
gmshfem::msg::info << "Subdomain " << idom << ":" << gmshfem::msg::endl;
gmshfem::msg::info << " - Subdomain size: " << _volume[idom]->getNumberOfUnknownDof() << " dofs" << gmshfem::msg::endl;
unsigned long long totalInterfaceSize = 0;
for(auto it = _surface[idom].begin(); it != _surface[idom].end(); ++it) {
totalInterfaceSize += it->second->getNumberOfUnknownDof();
}
gmshfem::msg::info << " - Interface size: " << totalInterfaceSize << " dofs:" << gmshfem::msg::endl;
for(auto it = _surface[idom].begin(); it != _surface[idom].end(); ++it) {
gmshfem::msg::info << " * Interface 'g(" << idom << ", " << it->first << ")' size: " << it->second->getNumberOfUnknownDof() << " dofs" << gmshfem::msg::endl;
}
}
}
printResources("DDM Pre-Processing Before Volume Assembly");
// Assemble locally on each processor
for(auto idom = 0ULL; idom < _volume.size(); ++idom) {
if(mpi::isItMySubdomain(idom)) {
gmshfem::common::Options::instance()->verbose = outerVerbosity;
gmshfem::msg::info << " - assembling " << _volume[idom]->name() << gmshfem::msg::endl;
gmshfem::common::Options::instance()->verbose = innerVerbosity;
_volume[idom]->setAttribute("ddm::physicalCommutator", _physicalCommutator);
_volume[idom]->setAttribute("ddm::artificialCommutator", _artificialCommutator);
gmshfem::common::Options::instance()->verbose = innerVerbosity;
_volume[idom]->assemble();
gmshfem::common::Options::instance()->verbose = outerVerbosity;
}
}
MPI_Barrier(PETSC_COMM_WORLD);
gmshfem::common::Options::instance()->verbose = outerVerbosity;
printResources("DDM Pre-Processing Before Volume Solve");
gmshfem::common::Options::instance()->verbose = innerVerbosity;
// solve locally on each processor
for(auto idom = 0ULL; idom < _volume.size(); ++idom) {
if(mpi::isItMySubdomain(idom)) {
gmshfem::common::Options::instance()->verbose = outerVerbosity;
gmshfem::msg::info << " - solving " << _volume[idom]->name() << gmshfem::msg::endl;
gmshfem::common::Options::instance()->verbose = innerVerbosity;
_volume[idom]->solve();
gmshfem::common::Options::instance()->verbose = outerVerbosity;
}
}
MPI_Barrier(PETSC_COMM_WORLD);
gmshfem::common::Options::instance()->verbose = outerVerbosity;
printResources("DDM Pre-Processing Before Surface Assembly and Solve");
gmshfem::common::Options::instance()->verbose = innerVerbosity;
for(auto idom = 0ULL; idom < _surface.size(); ++idom) {
if(mpi::isItMySubdomain(idom)) {
for(auto it = _surface[idom].begin(); it != _surface[idom].end(); ++it) {
gmshfem::common::Options::instance()->verbose = outerVerbosity;
gmshfem::msg::info << " - assembling and solving " << it->second->name() << gmshfem::msg::endl;
gmshfem::common::Options::instance()->verbose = innerVerbosity;
it->second->assemble();
it->second->solve();
gmshfem::common::Options::instance()->verbose = outerVerbosity;
}
}
}
gmshfem::common::Options::instance()->verbose = outerVerbosity;
MPI_Barrier(PETSC_COMM_WORLD);
// get the local interface unknowns for each processor
......@@ -815,22 +827,8 @@ namespace gmshddm
}
}
for(auto idom = 0ULL; idom < _surface.size(); ++idom) {
if(mpi::isItMySubdomain(idom)) {
gmshfem::msg::info << "Subdomain " << idom << ":" << gmshfem::msg::endl;
gmshfem::msg::info << " - Subdomain size: " << _volume[idom]->getNumberOfUnknownDof() << " dofs" << gmshfem::msg::endl;
unsigned long long totalInterfaceSize = 0;
for(auto it = _surface[idom].begin(); it != _surface[idom].end(); ++it) {
totalInterfaceSize += it->second->getNumberOfUnknownDof();
}
gmshfem::msg::info << " - Interface size: " << totalInterfaceSize << " dofs:" << gmshfem::msg::endl;
for(auto it = _surface[idom].begin(); it != _surface[idom].end(); ++it) {
gmshfem::msg::info << " * Interface 'g(" << idom << ", " << it->first << ")' size: " << it->second->getNumberOfUnknownDof() << " dofs" << gmshfem::msg::endl;
}
}
}
MPI_Barrier(PETSC_COMM_WORLD);
printResources("DDM Pre-Processing End");
time.tock();
......@@ -849,20 +847,20 @@ namespace gmshddm
// physical/artifical communicator and if there are no non-homogeneous
// Dirichlet BCs
MPI_Barrier(PETSC_COMM_WORLD);
printResources("DDM Iterative Solver Begin");
gmshfem::common::Timer time;
time.tick();
const int outerVerbosity = gmshfem::common::Options::instance()->verbose;
const int innerVerbosity = (outerVerbosity != 0 ? outerVerbosity - 1 : 0);
const unsigned int MPI_Rank = mpi::getMPIRank();
MPI_Barrier(PETSC_COMM_WORLD);
gmshfem::common::Timer time;
time.tick();
if(MPI_Rank == 0) {
gmshfem::msg::info << "Solving " << _name << " using " << solver << "..." << gmshfem::msg::endl;
}
printResources("DDM Iterative Solver Begin");
// ******************************************************
// Iterative solver
......@@ -890,8 +888,6 @@ namespace gmshddm
_physicalCommutator = false;
_artificialCommutator = true;
gmshfem::common::Options::instance()->verbose = innerVerbosity;
auto deactivateBilinear = [=](const unsigned long long idom) {
for(auto it = _volume[idom]->begin(); it != _volume[idom]->end(); ++it) {
if((*it)->isBilinear()) {
......@@ -927,6 +923,7 @@ namespace gmshddm
if(mpi::isItMySubdomain(idom)) {
_volume[idom]->setAttribute("ddm::physicalCommutator", _physicalCommutator);
_volume[idom]->setAttribute("ddm::artificialCommutator", _artificialCommutator);
gmshfem::common::Options::instance()->verbose = innerVerbosity;
if(sameMatrixWithArtificialAndPhysicalSources) {
deactivateBilinear(idom);
_volume[idom]->setRHSToZero();
......@@ -939,11 +936,10 @@ namespace gmshddm
_volume[idom]->assemble();
deactivateBilinear(idom);
}
gmshfem::common::Options::instance()->verbose = outerVerbosity;
}
}
gmshfem::common::Options::instance()->verbose = outerVerbosity;
if(solver == "jacobi") {
_IA = false;
Vec W; // residual
......@@ -988,8 +984,6 @@ namespace gmshddm
// Final solution
// ******************************************************
gmshfem::common::Options::instance()->verbose = innerVerbosity;
_physicalCommutator = true;
_artificialCommutator = true;
......@@ -997,6 +991,7 @@ namespace gmshddm
if(mpi::isItMySubdomain(idom)) {
_volume[idom]->setAttribute("ddm::physicalCommutator", _physicalCommutator);
_volume[idom]->setAttribute("ddm::artificialCommutator", _artificialCommutator);
gmshfem::common::Options::instance()->verbose = innerVerbosity;
if(sameMatrixWithArtificialAndPhysicalSources) {
_volume[idom]->setRHSToZero();
}
......@@ -1005,6 +1000,7 @@ namespace gmshddm
_volume[idom]->initSystem();
_volume[idom]->pre();
}
gmshfem::common::Options::instance()->verbose = outerVerbosity;
}
}
......@@ -1020,6 +1016,7 @@ namespace gmshddm
// SolveVolumePDE:
for(auto idom = 0ULL; idom < _volume.size(); ++idom) {
if(mpi::isItMySubdomain(idom)) {
gmshfem::common::Options::instance()->verbose = innerVerbosity;
if(!sameMatrixWithArtificialAndPhysicalSources) {
activateBilinear(idom);
}
......@@ -1028,11 +1025,10 @@ namespace gmshddm
if(sameMatrixWithArtificialAndPhysicalSources) {
activateBilinear(idom);
}
gmshfem::common::Options::instance()->verbose = outerVerbosity;
}
}
gmshfem::common::Options::instance()->verbose = outerVerbosity;
MatDestroy(&A);
VecDestroy(&X);
VecDestroy(&Y);
......@@ -1053,6 +1049,9 @@ namespace gmshddm
template< class T_Scalar >
gmshfem::algebra::MatrixCCS< T_Scalar > Formulation< T_Scalar >::computeMatrix()
{
const int outerVerbosity = gmshfem::common::Options::instance()->verbose;
const int innerVerbosity = (outerVerbosity != 0 ? outerVerbosity - 1 : 0);
if(mpi::getMPISize() != 1) {
throw gmshfem::common::Exception("Formulation< T_Scalar >::computeMatrix() does not work on multiple processes");
}
......@@ -1076,7 +1075,9 @@ namespace gmshddm
for(auto idom = 0ULL; idom < _volume.size(); ++idom) {
_volume[idom]->setAttribute("ddm::physicalCommutator", _physicalCommutator);
_volume[idom]->setAttribute("ddm::artificialCommutator", _artificialCommutator);
gmshfem::common::Options::instance()->verbose = innerVerbosity;
_volume[idom]->pre();
gmshfem::common::Options::instance()->verbose = outerVerbosity;
}
_IA = true;
......@@ -1152,19 +1153,21 @@ namespace gmshddm
Formulation< T_Scalar >::template PetscInterface< T_Scalar, PetscScalar, PetscInt >::freeArray(array);
VecRestoreArrayRead(X, &petscArray);
gmshfem::common::Options::instance()->verbose = innerVerbosity;
// SolveVolumePDE
for(auto idom = 0ULL; idom < formulation->_volume.size(); ++idom) {
if(mpi::isItMySubdomain(idom)) {
gmshfem::common::Options::instance()->verbose = innerVerbosity;
formulation->_volume[idom]->setRHSToZero();
formulation->_volume[idom]->assemble();
gmshfem::common::Options::instance()->verbose = outerVerbosity;
}
}
for(auto idom = 0ULL; idom < formulation->_volume.size(); ++idom) {
if(mpi::isItMySubdomain(idom)) {
gmshfem::common::Options::instance()->verbose = innerVerbosity;
formulation->_volume[idom]->solve(true);
gmshfem::common::Options::instance()->verbose = outerVerbosity;
}
}
......@@ -1172,8 +1175,10 @@ namespace gmshddm
for(auto idom = 0ULL; idom < formulation->_surface.size(); ++idom) {
if(mpi::isItMySubdomain(idom)) {
for(auto it = formulation->_surface[idom].begin(); it != formulation->_surface[idom].end(); ++it) {
gmshfem::common::Options::instance()->verbose = innerVerbosity;
it->second->setRHSToZero();
it->second->assemble();
gmshfem::common::Options::instance()->verbose = outerVerbosity;
}
}
}
......@@ -1181,15 +1186,14 @@ namespace gmshddm
for(auto idom = 0ULL; idom < formulation->_surface.size(); ++idom) {
if(mpi::isItMySubdomain(idom)) {
for(auto it = formulation->_surface[idom].begin(); it != formulation->_surface[idom].end(); ++it) {
gmshfem::common::Options::instance()->verbose = innerVerbosity;
it->second->solve(true);
gmshfem::common::Options::instance()->verbose = outerVerbosity;
}
}
}
MPI_Barrier(PETSC_COMM_WORLD);
printResources("DDM Iteration");
gmshfem::common::Options::instance()->verbose = outerVerbosity;
gmshfem::algebra::Vector< T_Scalar > g_local;
for(auto idom = 0ULL; idom < formulation->_surface.size(); ++idom) {
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment