Newer
Older
cmake_minimum_required(VERSION 3.18)
project(gmsh_gpu_dg)
include(CheckLanguage)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CUDA_ARCHITECTURES 35 70)
# Additional modules path for cmake
set (CMAKE_MODULE_PATH
${CMAKE_MODULE_PATH}
${CMAKE_CURRENT_SOURCE_DIR}/cmake)
if(NOT DEFINED HIP_PATH)
if(NOT DEFINED ENV{HIP_PATH})
set(HIP_PATH "/opt/rocm/hip" CACHE PATH "Path to which HIP has been installed")
else()
set(HIP_PATH $ENV{HIP_PATH} CACHE PATH "Path to which HIP has been installed")
endif()
endif()
set(CMAKE_MODULE_PATH "${HIP_PATH}/cmake" ${CMAKE_MODULE_PATH})
if (CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
cmake_host_system_information(RESULT OSVER QUERY OS_RELEASE)
if (OSVER VERSION_LESS "10.15")
add_definitions(-DDONT_USE_STD_FILESYSTEM)
endif()
endif()
######################################################################
## Find required libraries
find_package(Eigen3 REQUIRED)
set(LINK_LIBS ${LINK_LIBS} Eigen3::Eigen)
Christophe Geuzaine
committed
find_library(GMSH_LIB gmsh)
find_path(GMSH_INC gmsh.h)
if(GMSH_LIB AND GMSH_INC)
set(LINK_LIBS ${LINK_LIBS} ${GMSH_LIB})
include_directories(${GMSH_INC})
else()
set(GMSH_DIR "/opt/uliege/gmsh" CACHE PATH "GMSH install path")
if (DEFINED ENV{GMSH_DIR})
Christophe Geuzaine
committed
endif()
if (GMSH_DIR STREQUAL "" OR NOT EXISTS ${GMSH_DIR})
message(FATAL_ERROR "GMSH not found")
else()
set(LINK_LIBS ${LINK_LIBS} gmsh)
include_directories(${GMSH_DIR}/include)
link_directories(${GMSH_DIR}/lib)
endif()
if (SILO_FOUND)
add_definitions(-DHAVE_SILO)
include_directories("${SILO_INCLUDE_DIRS}")
set(LINK_LIBS ${LINK_LIBS} ${SILO_LIBRARIES})
option(OPT_SILO_USE_HDF5 "Use HDF5 driver for silo files" ON)
if (OPT_SILO_USE_HDF5)
add_definitions(-DSILO_USE_HDF5)
endif()
endif()
find_package(Lua REQUIRED)
include_directories("${LUA_INCLUDE_DIR}")
set(LINK_LIBS ${LINK_LIBS} ${LUA_LIBRARIES})
set(SOL2_BUILD_LUA OFF CACHE BOOL "Override sol2 build lua option")
add_subdirectory(contrib/sol2)
include_directories(contrib/sol2/include)
option(OPT_USE_MPI "Use MPI if available" OFF)
find_package(MPI)
if (OPT_USE_MPI AND MPI_FOUND)
add_definitions(-DUSE_MPI)
include_directories("${MPI_CXX_INCLUDE_DIRS}")
set(LINK_LIBS ${LINK_LIBS} ${MPI_CXX_LIBRARIES})
endif()
######################################################################
## Use or not use GPU solvers
option(OPT_ENABLE_GPU_SOLVER "Enable GPU solvers" OFF)
if (OPT_ENABLE_GPU_SOLVER)
add_definitions(-DENABLE_GPU_SOLVER)
######################################################################
## CUDA stuff, find toolkit, compiler & NVML
find_package(CUDAToolkit)
if (CUDAToolkit_FOUND)
set(CMAKE_CUDA_COMPILER ${CUDAToolkit_NVCC_EXECUTABLE})
enable_language(CUDA)
set(CMAKE_CUDA_STANDARD 17)
set(CMAKE_CUDA_STANDARD_REQUIRED TRUE)
#set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -lineinfo --resource-usage -Xptxas -dlcm=ca")
set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -lineinfo -prec-div=true \
-Wno-cpp -Xptxas -warn-spills -Xptxas -lineinfo -Xptxas -warn-lmem-usage")
option(OPT_PTXAS_SHOW_RUSAGE "Enable PTXAS resource usage reporting")
if (OPT_PTXAS_SHOW_RUSAGE)
set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --resource-usage")
endif()
if (TARGET CUDA::cudart)
set(HAVE_CUDA TRUE)
set(LINK_LIBS ${LINK_LIBS} CUDA::cudart)
add_definitions(-DHAVE_CUDA)
endif()
option(OPT_USE_CUDA_NVML "Enable NVML" ON)
if (OPT_USE_CUDA_NVML AND TARGET CUDA::nvml)
set(HAVE_CUDA_NVML TRUE)
set(LINK_LIBS ${LINK_LIBS} CUDA::nvml)
add_definitions(-DHAVE_CUDA_NVML)
endif()
link_directories("/usr/local/cuda/lib64")
endif()
######################################################################
## Find HIP stuff
find_package(HIP QUIET MODULE)
if (HIP_FOUND)
add_definitions(-DHAVE_HIP)
include_directories("/opt/rocm/include")
set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
Christophe Geuzaine
committed
if (DEFINED ENV{HIP_PLATFORM})
if ($ENV{HIP_PLATFORM} STREQUAL "nvcc")
#set(HIP_NVCC_FLAGS "-ccbin g++-7" ${HIP_NVCC_FLAGS})
set(CMAKE_HIP_LINK_EXECUTABLE "${CUDAToolkit_NVCC_EXECUTABLE} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
endif()
endif()
set(HIP_NVCC_FLAGS "-ccbin g++-8" ${HIP_NVCC_FLAGS})
option(OPT_USE_HIP "Use HIP GPU code path" OFF)
if (OPT_USE_HIP)
Matteo Cicuttin
committed
execute_process(COMMAND sh -c "/opt/rocm/hip/bin/hipify-perl ${CMAKE_SOURCE_DIR}/src/kernels_cuda.cu > ${CMAKE_SOURCE_DIR}/src/hipified_kernels_cuda.cpp")
execute_process(COMMAND sh -c "/opt/rocm/hip/bin/hipify-perl ${CMAKE_SOURCE_DIR}/src/maxwell_kernels_cuda.cu > ${CMAKE_SOURCE_DIR}/src/hipified_maxwell_kernels_cuda.cpp")
add_definitions(-DGPU_USE_HIP)
add_definitions(-D__HIP_PLATFORM_HCC__)
set(LINK_LIBS ${LINK_LIBS})
option(OPT_BLOCKED_GPU_KERNELS "Use blocked kernels on GPU (DO NOT USE)" OFF)
if (OPT_BLOCKED_GPU_KERNELS)
add_definitions(-DUSE_BLOCKED_GPU_KERNELS)
endif()
if (OPT_ENABLE_GPU_SOLVER)
option(OPT_DEBUG_GPU "Enable GPU debugging" OFF)
if (OPT_DEBUG_GPU)
add_definitions(-DGPU_DEBUG)
endif()
endif()
######################################################################
## Compiler identification stuff
if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
set(COMPILER_IS_CLANG TRUE)
elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
set(COMPILER_IS_GNU TRUE)
elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
set(COMPILER_IS_INTEL TRUE)
endif ()
######################################################################
## Compiler optimization options
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Weverything -Wno-c++98-compat \
-Wno-c++98-compat-pedantic -Wno-zero-as-null-pointer-constant -Wno-global-constructors \
-Wno-padded -Wno-shorten-64-to-32 -Wno-sign-conversion -Wno-old-style-cast \
-Wno-sign-compare -Wno-c99-extensions -Wno-extra-semi-stmt -Wno-source-uses-openmp")
Matteo Cicuttin
committed
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Wno-sign-compare -Wshadow \
-Wno-unknown-pragmas -Wno-unused-parameter")
endif()
set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g -fpermissive")
set(CMAKE_CXX_FLAGS_RELEASE "-O3 -g -DNDEBUG -fpermissive")
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
set(CMAKE_CXX_FLAGS_RELEASEASSERT "-O3 -g -fpermissive")
option(OPT_ENABLE_OPENMP "Enable OpenMP parallelization" OFF)
if (OPT_ENABLE_OPENMP)
find_package(OpenMP)
if (OpenMP_FOUND)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
endif()
endif()
if (COMPILER_IS_CLANG OR COMPILER_IS_GNU OR COMPILER_IS_INTEL)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
endif()
######################################################################
## Optimization stuff
option(OPT_PREFER_512bit "Prefer 512 bit vectors with AVX512 (only Clang > 10 & GCC)" OFF)
if (OPT_PREFER_512bit)
# https://reviews.llvm.org/D67259
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mprefer-vector-width=512")
endif()
option(OPT_AGGRESSIVE_FP "Enable DAZ, FTZ and -ffast-math" ON)
if (OPT_AGGRESSIVE_FP)
add_definitions(-DDISALLOW_DENORMALS)
if (COMPILER_IS_CLANG OR COMPILER_IS_GNU)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ffast-math")
endif()
endif()
option(OPT_VECTORIZER_REMARKS "Enable vectorizer remarks" OFF)
if (OPT_VECTORIZER_REMARKS)
if (COMPILER_IS_CLANG)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Rpass=loop-vectorize \
-Rpass-missed=loop-vectorize -Rpass-analysis=loop-vectorize")
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
endif()
if (COMPILER_IS_INTEL)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -qopt-report-phase=vec -qopt-report=2")
endif()
if (COMPILER_IS_GNU)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopt-info-vec-optimized")
endif()
endif()
option(OPT_MEASURE_PERFORMANCE "Enable performance measurement (produces a ton of output)" OFF)
if (OPT_MEASURE_PERFORMANCE)
add_definitions(-DMEASURE_PERFORMANCE)
endif()
option(OPT_CRASH_ON_NAN "Crash code if NaNs are generated" OFF)
if (OPT_CRASH_ON_NAN)
add_definitions(-DCRASH_ON_NAN)
endif()
######################################################################
## Development & debug stuff
if (COMPILER_IS_CLANG OR COMPILER_IS_GNU)
option(OPT_ASAN "Enable Address Sanitizer" OFF)
if (OPT_ASAN)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address")
endif()
endif()
option(OPT_WRITE_TEST_OUTPUTS "Produce SILO files in tests" OFF)
if (OPT_WRITE_TEST_OUTPUTS)
add_definitions(-DWRITE_TEST_OUTPUTS)
option(OPT_EXPENSIVE_ASSERTS "Enable expensive assert()s" OFF)
if (OPT_EXPENSIVE_ASSERTS)
add_definitions(-DEXPENSIVE_ASSERTS)
endif()
######################################################################
## Enable/disable solver modules
option(OPT_DEBUG_PRINT "Print debug information")
if (OPT_DEBUG_PRINT)
add_definitions(-DENABLE_DEBUG_PRINT)
endif()
######################################################################
## Handle 4.8.4 API breakage introduced in commit 91312e67
option(OPT_GMSH_484_BEFORE_SPLIT "Use old version of getKeysForElements() (see 91312e67)" OFF)
if (OPT_GMSH_484_BEFORE_SPLIT)
add_definitions(-DUSE_INITIAL_4_8_4_API)
endif()
option(OPT_DISABLE_TEXTURE_CACHE "On ROCm 4.1 texture stuff is broken, disable it" OFF)
if (OPT_DISABLE_TEXTURE_CACHE)
add_definitions(-DDISABLE_TEXTURE_CACHE)
endif()
######################################################################
Christophe Geuzaine
committed
## Configure targets
include_directories(contrib/sgr)
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include)
########## libgmshdg
set(LIBGMSHDG_SOURCES src/gmsh_io.cpp
src/silo_io.cpp
src/entity.cpp
src/reference_element.cpp
src/physical_element.cpp
src/connectivity.cpp
src/entity_data.cpp
src/kernels_cpu.cpp
src/param_loader.cpp
if (OPT_USE_MPI)
set(LIBGMSHDG_SOURCES ${LIBGMSHDG_SOURCES} src/gmsh_mpi.cpp)
endif()
set(LIBGMSHDG_SOURCES ${LIBGMSHDG_SOURCES} src/kernels_gpu_mi.cpp)
if (OPT_USE_HIP)
set_source_files_properties(src/hipified_kernels_cuda.cpp PROPERTIES HIP_SOURCE_PROPERTY_FORMAT 1)
set(LIBGMSHDG_SOURCES ${LIBGMSHDG_SOURCES} src/hipified_kernels_cuda.cpp)
hip_add_library(gmshdg SHARED ${LIBGMSHDG_SOURCES})
target_link_libraries(gmshdg ${LINK_LIBS})
else()
set(LIBGMSHDG_SOURCES ${LIBGMSHDG_SOURCES} src/kernels_cuda.cu)
add_library(gmshdg SHARED ${LIBGMSHDG_SOURCES})
target_link_libraries(gmshdg ${LINK_LIBS})
endif()
else()
add_library(gmshdg SHARED ${LIBGMSHDG_SOURCES})
target_link_libraries(gmshdg ${LINK_LIBS})
########## mpi testing stuff
add_executable(testmpi src/testmpi.cpp)
target_link_libraries(testmpi gmshdg)
add_executable(testpart src/testpart.cpp)
target_link_libraries(testpart gmshdg)
add_executable(testmod src/testmod.cpp)
target_link_libraries(testmod gmshdg)
########## maxwell solver
set(MAXWELL_SOURCES src/maxwell_interface.cpp
src/maxwell_cpu.cpp
src/maxwell_common.cpp
src/maxwell_solver.cpp
)
set(MAXWELL_SOURCES ${MAXWELL_SOURCES} src/maxwell_gpu.cpp)
if (OPT_USE_HIP)
set_source_files_properties(src/hipified_maxwell_kernels_cuda.cpp PROPERTIES HIP_SOURCE_PROPERTY_FORMAT 1)
set(MAXWELL_SOURCES ${MAXWELL_SOURCES} src/hipified_maxwell_kernels_cuda.cpp)
hip_add_executable(maxwell_solver ${MAXWELL_SOURCES} HIPCC_OPTIONS "-std=c++17")
target_link_libraries(maxwell_solver gmshdg)
else()
set(MAXWELL_SOURCES ${MAXWELL_SOURCES} src/maxwell_kernels_cuda.cu)
add_executable(maxwell_solver ${MAXWELL_SOURCES})
target_link_libraries(maxwell_solver gmshdg)
endif()
else()
add_executable(maxwell_solver ${MAXWELL_SOURCES})
target_link_libraries(maxwell_solver gmshdg)
########## tests
option(OPT_BUILD_TESTS "Build tests" OFF)
if (OPT_BUILD_TESTS)
add_executable(test_basics tests/test_basics.cpp)
target_link_libraries(test_basics gmshdg ${LINK_LIBS})
add_executable(test_mass tests/test_mass.cpp)
target_link_libraries(test_mass gmshdg ${LINK_LIBS})
add_executable(test_differentiation tests/test_differentiation.cpp)
target_link_libraries(test_differentiation gmshdg ${LINK_LIBS})
add_executable(test_lifting tests/test_lifting.cpp)
target_link_libraries(test_lifting gmshdg ${LINK_LIBS})
if (OPT_ENABLE_GPU_SOLVER)
add_executable(test_differentiation_gpu tests/test_differentiation_gpu.cpp)
target_link_libraries(test_differentiation_gpu gmshdg ${LINK_LIBS})
add_executable(profile_differentiation_gpu tests/profile_differentiation_gpu.cpp)
target_link_libraries(profile_differentiation_gpu gmshdg ${LINK_LIBS})
add_executable(test_lifting_gpu tests/test_lifting_gpu.cpp)
target_link_libraries(test_lifting_gpu gmshdg ${LINK_LIBS})