diff --git a/dG3D/src/dG3DDomain.cpp b/dG3D/src/dG3DDomain.cpp index bcb0fa64a6d0ded4c8978604c34e25d4b45f8c88..032c5941e097ca3e223155d19f7da74654545181 100644 --- a/dG3D/src/dG3DDomain.cpp +++ b/dG3D/src/dG3DDomain.cpp @@ -2111,11 +2111,254 @@ void dG3DDomain::computeStrain(MElement *e, const int npts_bulk, IntPt *GP, } } } + +void dG3DDomain::prepareMPIExchange(AllIPState *aips, IPStateBase::whichState ws) +{ + MPI_Status status; + if (Msg::GetCommRank() == _rootRank){ + // compute all IP strain on root rank + // this->computeAllIPStrain(aips,ufield,ws,false); + // send strain to other procs + // for bulk elements + for (std::set<int>::iterator it = _otherRanks.begin(); it!= _otherRanks.end(); it++){ + int otherRank = *it; + std::set<int>& IPBulkonRank = _mapIPBulk[otherRank]; + for (std::set<int>::iterator its = IPBulkonRank.begin(); its!= IPBulkonRank.end(); its++){ + int num = *its; + int elnum,gnum; + numericalMaterialBase::getTwoIntsFromType(num,elnum,gnum); + AllIPState::ipstateElementContainer *vips = aips->getIPstate(elnum); + IPStateBase* ips = (*vips)[gnum]; + IPVariable* ipv = ips->getState(ws); + int bufferSize = ipv->getMacroNumberElementDataSendToMicroProblem(); // already known + double *buffer = new double[bufferSize]; + ipv->getMacroDataSendToMicroProblem(buffer); + MPI_Send(buffer,bufferSize,MPI_DOUBLE,otherRank,num,MPI_COMM_WORLD); + delete[] buffer; + }; + + std::set<int>& IPInterfaceMinusonRank = _mapIPInterfaceMinus[otherRank]; + for (std::set<int>::iterator its = IPInterfaceMinusonRank.begin(); its!= IPInterfaceMinusonRank.end(); its++){ + int num = *its; + int elnum,gnum; + numericalMaterialBase::getTwoIntsFromType(num,elnum,gnum); + AllIPState::ipstateElementContainer *vips = aips->getIPstate(elnum); + IPStateBase* ips = (*vips)[gnum]; + IPVariable* ipv = ips->getState(ws); + int bufferSize = ipv->getMacroNumberElementDataSendToMicroProblem(); // already known + double *buffer = new double[bufferSize]; + ipv->getMacroDataSendToMicroProblem(buffer); + MPI_Send(buffer,bufferSize,MPI_DOUBLE,otherRank,num,MPI_COMM_WORLD); + delete[] buffer; + }; + + std::set<int>& IPInterfacePlusonRank = _mapIPInterfacePlus[otherRank]; + for (std::set<int>::iterator its = IPInterfacePlusonRank.begin(); its!= IPInterfacePlusonRank.end(); its++){ + int num = *its; + int elnum,gnum; + numericalMaterialBase::getTwoIntsFromType(num,elnum,gnum); + AllIPState::ipstateElementContainer *vips = aips->getIPstate(elnum); + IPStateBase* ips = (*vips)[gnum]; + IPVariable* ipv = ips->getState(ws); + int bufferSize = ipv->getMacroNumberElementDataSendToMicroProblem(); // already known + double *buffer = new double[bufferSize]; + ipv->getMacroDataSendToMicroProblem(buffer); + MPI_Send(buffer,bufferSize,MPI_DOUBLE,otherRank,num,MPI_COMM_WORLD); + delete[] buffer; + }; + } + } + else if (_otherRanks.find(Msg::GetCommRank()) != _otherRanks.end()){ + for (std::set<int>::iterator it = _domainIPBulk.begin(); it != _domainIPBulk.end(); it++){ + int num = *it; + int elnum,gnum; + numericalMaterialBase::getTwoIntsFromType(num,elnum,gnum); + AllIPState::ipstateElementContainer *vips = aips->getIPstate(num); + IPStateBase* ips = (*vips)[0]; + IPVariable* ipv = ips->getState(ws); + int bufferSize = ipv->getMacroNumberElementDataSendToMicroProblem(); + double *buffer = new double[bufferSize]; + MPI_Recv(buffer,bufferSize,MPI_DOUBLE,_rootRank,num,MPI_COMM_WORLD,&status); + ipv->setReceivedMacroDataToMicroProblem(buffer); + delete [] buffer; + }; + + for (std::set<int>::iterator it = _domainIPInterfaceMinus.begin(); it != _domainIPInterfaceMinus.end(); it++){ + int num = *it; + int elnum, phys, gnum; + numericalMaterialBase::getTwoIntsFromType(num,elnum,gnum); + AllIPState::ipstateElementContainer *vips = aips->getIPstate(num); + IPStateBase* ips = (*vips)[0]; + IPVariable* ipv = ips->getState(ws); + int bufferSize = ipv->getMacroNumberElementDataSendToMicroProblem(); + double *buffer = new double[bufferSize]; + MPI_Recv(buffer,bufferSize,MPI_DOUBLE,_rootRank,num,MPI_COMM_WORLD,&status); + ipv->setReceivedMacroDataToMicroProblem(buffer); + delete [] buffer; + }; + + for (std::set<int>::iterator it = _domainIPInterfacePlus.begin(); it != _domainIPInterfacePlus.end(); it++){ + int num = *it; + int elnum,gnum; + numericalMaterialBase::getTwoIntsFromType(num,elnum,gnum); + AllIPState::ipstateElementContainer *vips = aips->getIPstate(num); + IPStateBase* ips = (*vips)[0]; + IPVariable* ipv = ips->getState(ws); + int bufferSize = ipv->getMacroNumberElementDataSendToMicroProblem(); + double *buffer = new double[bufferSize]; + MPI_Recv(buffer,bufferSize,MPI_DOUBLE,_rootRank,num,MPI_COMM_WORLD,&status); + ipv->setReceivedMacroDataToMicroProblem(buffer); + delete [] buffer; + }; + }; +} +void dG3DDomain::endMPIExchange(AllIPState *aips, IPStateBase::whichState ws) +{ + MPI_Status status; + IntPt *GP; + if (_otherRanks.find(Msg::GetCommRank()) != _otherRanks.end()){ + for (std::set<int>::iterator it = _domainIPBulk.begin(); it != _domainIPBulk.end(); it++){ + int num = *it; + int elnum,gnum; + numericalMaterialBase::getTwoIntsFromType(num,elnum,gnum); + AllIPState::ipstateElementContainer *vips = aips->getIPstate(num); + IPStateBase* ips = (*vips)[0]; + IPVariable* ipv = ips->getState(ws); + int bufferSize = ipv->getMicroNumberElementDataSendToMacroProblem(); + double *buffer = new double[bufferSize]; + ipv->getMicroDataToMacroProblem(buffer); + MPI_Send(buffer,bufferSize,MPI_DOUBLE,_rootRank,num,MPI_COMM_WORLD); + delete [] buffer; + }; + for (std::set<int>::iterator it = _domainIPInterfaceMinus.begin(); it != _domainIPInterfaceMinus.end(); it++){ + int num = *it; + int elnum,gnum; + numericalMaterialBase::getTwoIntsFromType(num,elnum,gnum); + AllIPState::ipstateElementContainer *vips = aips->getIPstate(num); + IPStateBase* ips = (*vips)[0]; + IPVariable* ipv = ips->getState(ws); + int bufferSize = ipv->getMicroNumberElementDataSendToMacroProblem(); + double *buffer = new double[bufferSize]; + ipv->getMicroDataToMacroProblem(buffer); + MPI_Send(buffer,bufferSize,MPI_DOUBLE,_rootRank,num,MPI_COMM_WORLD); + delete [] buffer; + }; + for (std::set<int>::iterator it = _domainIPInterfacePlus.begin(); it != _domainIPInterfacePlus.end(); it++){ + int num = *it; + int elnum,gnum; + numericalMaterialBase::getTwoIntsFromType(num,elnum,gnum); + AllIPState::ipstateElementContainer *vips = aips->getIPstate(num); + IPStateBase* ips = (*vips)[0]; + IPVariable* ipv = ips->getState(ws); + int bufferSize = ipv->getMicroNumberElementDataSendToMacroProblem(); + double *buffer = new double[bufferSize]; + ipv->getMicroDataToMacroProblem(buffer); + MPI_Send(buffer,bufferSize,MPI_DOUBLE,_rootRank,num,MPI_COMM_WORLD); + delete [] buffer; + }; + } + else if (Msg::GetCommRank() == _rootRank){ + + for (std::set<int>::iterator it = _otherRanks.begin(); it!= _otherRanks.end(); it++){ + int otherRank = *it; + std::set<int>& IPBulkonRank = _mapIPBulk[otherRank]; + for (std::set<int>::iterator its = IPBulkonRank.begin(); its!= IPBulkonRank.end(); its++){ + int num = *its; + int elnum,gnum; + numericalMaterialBase::getTwoIntsFromType(num,elnum,gnum); + AllIPState::ipstateElementContainer *vips = aips->getIPstate(elnum); + IPStateBase* ips = (*vips)[gnum]; + IPVariable* ipv = ips->getState(ws); + int bufferSize = ipv->getMicroNumberElementDataSendToMacroProblem(); + double *buffer = new double[bufferSize]; + MPI_Recv(buffer,bufferSize,MPI_DOUBLE,otherRank,num,MPI_COMM_WORLD,&status); + ipv->setReceivedMicroDataToMacroProblem(buffer); + dG3DMaterialLaw* mlaw = dynamic_cast<dG3DMaterialLaw*>(this->getMaterialLaw()); + mlaw->setElasticStiffness(ipv); + delete[] buffer; + }; + std::set<int>& IPInterfaceMinusonRank = _mapIPInterfaceMinus[otherRank]; + for (std::set<int>::iterator its = IPInterfaceMinusonRank.begin(); its!= IPInterfaceMinusonRank.end(); its++){ + int num = *its; + int elnum,gnum; + numericalMaterialBase::getTwoIntsFromType(num,elnum,gnum); + AllIPState::ipstateElementContainer *vips = aips->getIPstate(elnum); + IPStateBase* ips = (*vips)[gnum]; + IPVariable* ipv = ips->getState(ws); + int bufferSize = ipv->getMicroNumberElementDataSendToMacroProblem(); + double *buffer = new double[bufferSize]; + MPI_Recv(buffer,bufferSize,MPI_DOUBLE,otherRank,num,MPI_COMM_WORLD,&status); + ipv->setReceivedMicroDataToMacroProblem(buffer); + dG3DMaterialLaw* mlawMinus = dynamic_cast<dG3DMaterialLaw*>(this->getMaterialLawMinus()); + mlawMinus->setElasticStiffness(ipv); + delete[] buffer; + }; + + std::set<int>& IPInterfacePlusonRank = _mapIPInterfacePlus[otherRank]; + for (std::set<int>::iterator its = IPInterfacePlusonRank.begin(); its!= IPInterfacePlusonRank.end(); its++){ + int num = *its; + int elnum,gnum; + numericalMaterialBase::getTwoIntsFromType(num,elnum,gnum); + AllIPState::ipstateElementContainer *vips = aips->getIPstate(elnum); + IPStateBase* ips = (*vips)[gnum]; + IPVariable* ipv = ips->getState(ws); + int bufferSize = ipv->getMicroNumberElementDataSendToMacroProblem(); + double *buffer = new double[bufferSize]; + MPI_Recv(buffer,bufferSize,MPI_DOUBLE,otherRank,num,MPI_COMM_WORLD,&status); + ipv->setReceivedMicroDataToMacroProblem(buffer); + dG3DMaterialLaw* mlawPlus = dynamic_cast<dG3DMaterialLaw*>(this->getMaterialLawPlus()); + mlawPlus->setElasticStiffness(ipv); + delete[] buffer; + }; + } + + if (_averageStrainBased and (this->getMaterialLawMinus()->getNum() == this->getMaterialLawPlus()->getNum())){ + for (elementGroup::elementContainer::const_iterator ite = gi->begin(); ite!= gi->end(); ite++){ + MElement* ele = ite->second; + int npts = this->getInterfaceGaussIntegrationRule()->getIntPoints(ele,&GP); + AllIPState::ipstateElementContainer *vips = aips->getIPstate(ele->getNum()); + for(int j=0;j<npts;j++){ + IPStateBase* ipsm = (*vips)[j]; + IPStateBase* ipsp = (*vips)[j+npts]; + + IPVariable* ipvm = ipsm->getState(IPStateBase::current); + IPVariable* ipvp = ipsp->getState(IPStateBase::current); + + ipvp->operator=(*ipvm); + } + + } + } + } +} + + +void dG3DDomain::computeIpvMPI(AllIPState *aips,MElement *e, IPStateBase::whichState ws, + materialLaw *mlaw__,fullVector<double> &disp, bool stiff) +{ + prepareMPIExchange(aips,IPStateBase::current); + computeIpv(aips, e, ws, mlaw__, disp, stiff); + endMPIExchange(aips,IPStateBase::current); +} + + + void dG3DDomain::computeIpv(AllIPState *aips,MElement *e, IPStateBase::whichState ws, materialLaw *mlaw__,fullVector<double> &disp, bool stiff) { if (!getElementErosionFilter()(e)) return; + + //#if defined(HAVE_MPI) + //if (mlaw__->isNumeric() and this->_otherRanks.size()>0 and stiff==false) + //{ + // this->computeIpvMPI(aips,e,ws,mlaw__,disp,stiff); + //} + //else + //#endif //HAVE_MPI + { + + dG3DMaterialLaw *mlaw; AllIPState::ipstateElementContainer *vips = aips->getIPstate(e->getNum()); IntPt *GP; @@ -2228,6 +2471,7 @@ void dG3DDomain::computeIpv(AllIPState *aips,MElement *e, IPStateBase::whichStat Msg::Error("This substepping method %d has not been implemented",_subSteppingMethod); } } + } } void dG3DDomain::setValuesForBodyForce(AllIPState *aips, const IPStateBase::whichState ws){ @@ -4942,11 +5186,18 @@ MElement* dG3DDomain::createInterface(IElement *ie1, IElement *ie2) const #if defined(HAVE_MPI) void dG3DDomain::computeIPVariableMPI(AllIPState *aips,const unknownField *ufield,const IPStateBase::whichState ws, bool stiff){ - IntPt *GP; + + + //IntPt *GP; MPI_Status status; if (Msg::GetCommRank() == _rootRank){ // compute all IP strain on root rank this->computeAllIPStrain(aips,ufield,ws,false); + + } + this->prepareMPIExchange(aips, ws); + + /*if (Msg::GetCommRank() == _rootRank){ // send strain to other procs // for bulk elements for (std::set<int>::iterator it = _otherRanks.begin(); it!= _otherRanks.end(); it++){ @@ -5039,7 +5290,7 @@ void dG3DDomain::computeIPVariableMPI(AllIPState *aips,const unknownField *ufiel ipv->setReceivedMacroDataToMicroProblem(buffer); delete [] buffer; }; - }; + };*/ bool needDTangent=false; bool usePreviousModuli = false; @@ -5095,8 +5346,8 @@ void dG3DDomain::computeIPVariableMPI(AllIPState *aips,const unknownField *ufiel mlawMinus->stress(ipv,ipvprev,stiff,true,needDTangent); }; - - for (std::set<int>::iterator it = _domainIPInterfacePlus.begin(); it != _domainIPInterfacePlus.end(); it++){ + this->endMPIExchange(aips, ws); + /*for (std::set<int>::iterator it = _domainIPInterfacePlus.begin(); it != _domainIPInterfacePlus.end(); it++){ int num = *it; int elnum,gnum; numericalMaterialBase::getTwoIntsFromType(num,elnum,gnum); @@ -5232,7 +5483,7 @@ void dG3DDomain::computeIPVariableMPI(AllIPState *aips,const unknownField *ufiel } } - }; + };*/ }; #endif diff --git a/dG3D/src/dG3DDomain.h b/dG3D/src/dG3DDomain.h index aa4936cf2ea744d80ea4f9d13aa18f8ed3618be0..80bd80cc746ee15f51be70c68bdc18b2e9ba0652 100644 --- a/dG3D/src/dG3DDomain.h +++ b/dG3D/src/dG3DDomain.h @@ -257,6 +257,9 @@ class dG3DDomain : public dgPartDomain{ virtual double getNonLocalStabilityParameter() const { return _nonLocalBeta;} virtual bool getNonLocalContinuity() const { return _nonLocalContinuity;} virtual double getNonLocalEqRatio() const { return _nonLocalEqRatio;} + + virtual void computeIpvMPI(AllIPState *aips,MElement *e, IPStateBase::whichState ws, + materialLaw *mlaw__,fullVector<double> &disp, bool stiff); // virtual bool getConstitutiveExtraDofDiffusionUseEnergyConjugatedField() const { return _constitutiveExtraDofDiffusionUseEnergyConjugatedField;} @@ -277,6 +280,9 @@ class dG3DDomain : public dgPartDomain{ } }; + virtual void prepareMPIExchange(AllIPState *aips, IPStateBase::whichState ws); + virtual void endMPIExchange(AllIPState *aips, IPStateBase::whichState ws); + #endif };