Skip to content
Snippets Groups Projects
Commit a6f2ae82 authored by Roland Greffe's avatar Roland Greffe
Browse files

fix race condition (all ?) when using gmshddm

parent 3b0f9533
No related branches found
No related tags found
No related merge requests found
...@@ -433,6 +433,31 @@ if(ENABLE_MPI) ...@@ -433,6 +433,31 @@ if(ENABLE_MPI)
endif() endif()
endif() endif()
# Thrust
set(Thrust_NVIDIA FALSE)
set(Thrust_ROCm FALSE)
if(ENABLE_DDMGPU)
if(NOT ENABLE_OPENMP)
message(WARNING "DDMGPU requires OPENMP ! Disabling DDMGPU.")
set(ENABLE_DDMGPU OFF)
else()
find_package(Thrust) #Find it the NVIDIA way
if(THRUST_FOUND)
set(Thrust_NVIDIA TRUE)
thrust_create_target(Thrust HOST CPP DEVICE CPP)
set_config_option("DDMGPU" DDMGPU) # -> GPU DDM USED
else() # Try to find rocThrust
find_package(rocprim CONFIG PATHS "/opt/rocm/rocprim")
find_package(rocthrust CONFIG PATHS "/opt/rocm/rocthrust")
if(rocprim_FOUND AND rocthrust_FOUND)
set(Thrust_ROC TRUE)
set_config_option("DDMGPU" DDMGPU) # -> GPU DDM USED
set(Thrust_ROCm TRUE)
endif()
endif()
endif()
endif()
# OpenMP # OpenMP
if(ENABLE_OPENMP) if(ENABLE_OPENMP)
...@@ -454,7 +479,7 @@ if(ENABLE_OPENMP) ...@@ -454,7 +479,7 @@ if(ENABLE_OPENMP)
list(APPEND EXTRA_LIBS "-L/opt/homebrew/opt/libomp/lib -lomp") list(APPEND EXTRA_LIBS "-L/opt/homebrew/opt/libomp/lib -lomp")
set_config_option("OpenMP[Homebrew]" HAVE_OPENMP) set_config_option("OpenMP[Homebrew]" HAVE_OPENMP)
else() else()
if(OPENMP_FOUND) if(OPENMP_FOUND AND NOT ENABLE_DDMGPU)
set_config_option("OpenMP" HAVE_OPENMP) set_config_option("OpenMP" HAVE_OPENMP)
endif() endif()
endif() endif()
...@@ -477,30 +502,6 @@ if(ENABLE_ROBINHOOD) ...@@ -477,30 +502,6 @@ if(ENABLE_ROBINHOOD)
endif() endif()
endif() endif()
# Thrust
set(Thrust_NVIDIA FALSE)
set(Thrust_ROCm FALSE)
if(ENABLE_DDMGPU)
if(ENABLE_OPENMP)
message(WARNING "DDMGPU and OpenMP are not compatible. Disabling DDMGPU.")
set(ENABLE_DDMGPU OFF)
else()
find_package(Thrust) #Find it the NVIDIA way
if(THRUST_FOUND)
set(Thrust_NVIDIA TRUE)
thrust_create_target(Thrust HOST CPP DEVICE CPP)
set_config_option("DDMGPU" DDMGPU) # -> GPU DDM USED
else() # Try to find rocThrust
find_package(rocprim CONFIG PATHS "/opt/rocm/rocprim")
find_package(rocthrust CONFIG PATHS "/opt/rocm/rocthrust")
if(rocprim_FOUND AND rocthrust_FOUND)
set(Thrust_ROC TRUE)
set_config_option("DDMGPU" DDMGPU) # -> GPU DDM USED
set(Thrust_ROCm TRUE)
endif()
endif()
endif()
endif()
# Coverage # Coverage
......
...@@ -106,9 +106,15 @@ namespace gmshfem::function ...@@ -106,9 +106,15 @@ namespace gmshfem::function
} }
T *p = nullptr; T *p = nullptr;
if(n * sizeof(value_type) <= GMSHFEM_FUNCTION_MEMORY_ALIGNMENT) { if(n * sizeof(value_type) <= GMSHFEM_FUNCTION_MEMORY_ALIGNMENT) {
#ifdef DDMGPU
#pragma omp critical
#endif
p = static_cast< T * >(MemoryPoolAllocator::instance()->allocateSmall(n * sizeof(value_type))); p = static_cast< T * >(MemoryPoolAllocator::instance()->allocateSmall(n * sizeof(value_type)));
} }
else { else {
#ifdef DDMGPU
#pragma omp critical
#endif
p = static_cast< T * >(MemoryPoolAllocator::instance()->allocate(n * sizeof(value_type))); p = static_cast< T * >(MemoryPoolAllocator::instance()->allocate(n * sizeof(value_type)));
} }
...@@ -122,9 +128,15 @@ namespace gmshfem::function ...@@ -122,9 +128,15 @@ namespace gmshfem::function
} }
if(n * sizeof(value_type) <= GMSHFEM_FUNCTION_MEMORY_ALIGNMENT) { if(n * sizeof(value_type) <= GMSHFEM_FUNCTION_MEMORY_ALIGNMENT) {
#ifdef DDMGPU
#pragma omp critical
#endif
MemoryPoolAllocator::instance()->deallocateSmall(p, n * sizeof(value_type)); MemoryPoolAllocator::instance()->deallocateSmall(p, n * sizeof(value_type));
} }
else { else {
#ifdef DDMGPU
#pragma omp critical
#endif
MemoryPoolAllocator::instance()->deallocate(p, n * sizeof(value_type)); MemoryPoolAllocator::instance()->deallocate(p, n * sizeof(value_type));
} }
} }
...@@ -142,6 +154,9 @@ namespace gmshfem::function ...@@ -142,6 +154,9 @@ namespace gmshfem::function
static T *s_p = nullptr; static T *s_p = nullptr;
#ifdef HAVE_OPENMP #ifdef HAVE_OPENMP
#pragma omp master #pragma omp master
#endif
#ifdef DDMGPU
#pragma omp critical
#endif #endif
s_p = static_cast< T * >(MemoryPoolAllocator::instance()->allocate(n * sizeof(value_type))); s_p = static_cast< T * >(MemoryPoolAllocator::instance()->allocate(n * sizeof(value_type)));
#ifdef HAVE_OPENMP #ifdef HAVE_OPENMP
...@@ -157,6 +172,9 @@ namespace gmshfem::function ...@@ -157,6 +172,9 @@ namespace gmshfem::function
#pragma omp barrier #pragma omp barrier
#pragma omp master #pragma omp master
#endif
#ifdef DDMGPU
#pragma omp critical
#endif #endif
MemoryPoolAllocator::instance()->deallocate(s_p, n * sizeof(value_type)); MemoryPoolAllocator::instance()->deallocate(s_p, n * sizeof(value_type));
#ifdef HAVE_OPENMP #ifdef HAVE_OPENMP
......
...@@ -1108,12 +1108,12 @@ namespace gmshfem::problem ...@@ -1108,12 +1108,12 @@ namespace gmshfem::problem
} }
time.tock(); time.tock();
msg::info << "Done pre-processing in " << time << "s" << msg::endl; //msg::info << "Done pre-processing in " << time << "s" << msg::endl;
if(common::Options::instance()->memory) { if(common::Options::instance()->memory) {
auto memory = _A->memory(); auto memory = _A->memory();
for (auto& bi: _multiB) for (auto& bi: _multiB)
memory += bi.memory(); memory += bi.memory();
msg::info << "Memory footprint of system: " << memory << msg::endl; //msg::info << "Memory footprint of system: " << memory << msg::endl;
} }
return time; return time;
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment