DistributedField.h

// GmshFEM - Copyright (C) 2023, B. Martin, C. Geuzaine, Université de Liège
//
// See the LICENSE.txt file for license information. Please report all
// issues on https://gitlab.onelab.info/gmsh/fem/issues

#ifndef H_GMSHFEM_DISTRIBUTED_FIELD
#define H_GMSHFEM_DISTRIBUTED_FIELD

#include "Field.h"

#include <set>
#include <unordered_set>
#include <map>
#include <unordered_map>
#include <optional>

namespace gmshfem::field
{


  template< class T_Scalar, field::Form T_Form >
  class DistributedField : public Field< T_Scalar, T_Form >
  {

    private:
    using DofGlobalIndex = std::pair<int, std::size_t>; // Key without tag information (modulo FIELD_OFFSET)
    using DofIndex = std::pair<unsigned long long, unsigned long long>; // Includes tag information


    struct HashBySecond {
      size_t operator()(const DofGlobalIndex &p) const
      {
        return std::hash<  std::size_t >()(p.second); // Hash based on the entity ID
      }
      size_t operator()(const DofIndex &p) const
      {
        return std::hash<  std::size_t >()(p.second); // Hash based on the entity ID
      }
    };

   public:
        domain::Domain _innerDomain, _overlapDomain;


    std::unordered_set< dofs::Dof * > _sharedDofs; // DOFs having non-zero support on the overlap
    std::unordered_set< dofs::Dof * > _subdomainInterfaceDofs; // Intersection of _sharedDofs and _innerDofs
    std::unordered_set<dofs::Dof *> _ownedDofs; // Strict interior + interface dofs owned

    std::unordered_set< dofs::Dof * > _toSend; // List of DOFs objects whose ID should be broadcasted by
    std::unordered_set< dofs::Dof * > _toRead;

    std::unordered_map<DofIndex, dofs::Dof*, HashBySecond> _indexToDof;
    std::unordered_map<unsigned long long, unsigned long long> _localToGlobal; //numDof()

   private:
    void _computeDofsSubsets(); // Computes the sets above, assuming this->_values is filled.    
    void _synchronizeInterfaceOwnership(const std::optional<std::vector<int>>&  neighboringRanks);
    void _computeToSend(const std::optional<std::vector<int>>&  neighboringRanks);

  public:
    virtual void preProMPI(const std::optional<std::vector<int>>&  neighboringRanks = std::nullopt) override;
    virtual std::unordered_set<dofs::Dof*> getAllOwnedDofs() override;
    virtual std::unordered_set<dofs::Dof*> getNonOwnedDofs() override;
    virtual void syncGlobalDofs(std::vector<unsigned long long>& localToGlobal, std::vector<unsigned long long>& readIDs) override;
    void dumpOwnedKeys();

   public:
    DistributedField();
    DistributedField(const std::string &name, const domain::Domain &innerdomain, const domain::Domain &outerdomain, const field::FunctionSpaceOfForm< T_Form > &type, const unsigned int order = 0, const std::string &model = "");
    DistributedField(const std::string &name, const domain::Domain &innerdomain, const domain::Domain &outerdomain, const FunctionSpace< scalar::Precision< T_Scalar >, T_Form > &functionSpace, const std::string &model = "");
    DistributedField(const DistributedField< T_Scalar, T_Form > &other);
    virtual ~DistributedField();
  };


} // namespace gmshfem::field


#endif // H_GMSHFEM_DISTRIBUTED_FIELD