CMakeLists.txt

cmake_minimum_required(VERSION 3.18)
project(gmsh_gpu_dg)
include(CheckLanguage)


set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CUDA_ARCHITECTURES 35 70)

# Additional modules path for cmake
set (CMAKE_MODULE_PATH
     ${CMAKE_MODULE_PATH}
     ${CMAKE_CURRENT_SOURCE_DIR}/cmake)


if(NOT DEFINED HIP_PATH)
    if(NOT DEFINED ENV{HIP_PATH})
        set(HIP_PATH "/opt/rocm/hip" CACHE PATH "Path to which HIP has been installed")
    else()
        set(HIP_PATH $ENV{HIP_PATH} CACHE PATH "Path to which HIP has been installed")
    endif()
endif()
set(CMAKE_MODULE_PATH "${HIP_PATH}/cmake" ${CMAKE_MODULE_PATH})

if (CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
    cmake_host_system_information(RESULT OSVER QUERY OS_RELEASE)
    if (OSVER VERSION_LESS "10.15")
        add_definitions(-DDONT_USE_STD_FILESYSTEM)
    endif()
endif()

######################################################################
## Find required libraries
find_package(Eigen3 REQUIRED)
set(LINK_LIBS ${LINK_LIBS} Eigen3::Eigen)

find_library(GMSH_LIB gmsh)
find_path(GMSH_INC gmsh.h)
if(GMSH_LIB AND GMSH_INC)
    set(LINK_LIBS ${LINK_LIBS} ${GMSH_LIB})
    include_directories(${GMSH_INC})
else()
  set(GMSH_DIR "/opt/uliege/gmsh" CACHE PATH "GMSH install path")
  if (DEFINED ENV{GMSH_DIR})
    set (GMSH_DIR $ENV{GMSH_DIR})
  endif()
  if (GMSH_DIR STREQUAL "" OR NOT EXISTS ${GMSH_DIR})
      message(FATAL_ERROR "GMSH not found")
  else()
      set(LINK_LIBS ${LINK_LIBS} gmsh)
      include_directories(${GMSH_DIR}/include)
      link_directories(${GMSH_DIR}/lib)
  endif()
endif()

find_package(SILO REQUIRED)
if (SILO_FOUND)
    add_definitions(-DHAVE_SILO)
    include_directories("${SILO_INCLUDE_DIRS}")
    set(LINK_LIBS ${LINK_LIBS} ${SILO_LIBRARIES})
    option(OPT_SILO_USE_HDF5 "Use HDF5 driver for silo files" ON)
    if (OPT_SILO_USE_HDF5)
        add_definitions(-DSILO_USE_HDF5)
    endif()
endif()

find_package(Lua REQUIRED)
include_directories("${LUA_INCLUDE_DIR}")
set(LINK_LIBS ${LINK_LIBS} ${LUA_LIBRARIES})
set(SOL2_BUILD_LUA OFF CACHE BOOL "Override sol2 build lua option")
add_subdirectory(contrib/sol2)
include_directories(contrib/sol2/include)

option(OPT_USE_MPI "Use MPI if available" OFF)
find_package(MPI)
if (OPT_USE_MPI AND MPI_FOUND)
    add_definitions(-DUSE_MPI)
    include_directories("${MPI_CXX_INCLUDE_DIRS}")
    set(LINK_LIBS ${LINK_LIBS} ${MPI_CXX_LIBRARIES})
endif()
######################################################################
## Use or not use GPU solvers
option(OPT_ENABLE_GPU_SOLVER "Enable GPU solvers" OFF)
if (OPT_ENABLE_GPU_SOLVER)
    add_definitions(-DENABLE_GPU_SOLVER)

    ######################################################################
    ## CUDA stuff, find toolkit, compiler & NVML
    find_package(CUDAToolkit)
    if (CUDAToolkit_FOUND)
        set(CMAKE_CUDA_COMPILER ${CUDAToolkit_NVCC_EXECUTABLE})
        enable_language(CUDA)
        set(CMAKE_CUDA_STANDARD 17)
        set(CMAKE_CUDA_STANDARD_REQUIRED TRUE)
        #set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -lineinfo --resource-usage -Xptxas -dlcm=ca")
        set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -lineinfo -prec-div=true \
            -Wno-cpp -Xptxas -warn-spills -Xptxas -lineinfo -Xptxas -warn-lmem-usage")

        option(OPT_PTXAS_SHOW_RUSAGE "Enable PTXAS resource usage reporting")
        if (OPT_PTXAS_SHOW_RUSAGE)
            set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --resource-usage")
        endif()

        if (TARGET CUDA::cudart)
            set(HAVE_CUDA TRUE)
            set(LINK_LIBS ${LINK_LIBS} CUDA::cudart)
            add_definitions(-DHAVE_CUDA)
        endif()

        option(OPT_USE_CUDA_NVML "Enable NVML" ON)
        if (OPT_USE_CUDA_NVML AND TARGET CUDA::nvml)
            set(HAVE_CUDA_NVML TRUE)
            set(LINK_LIBS ${LINK_LIBS} CUDA::nvml)
            add_definitions(-DHAVE_CUDA_NVML)
        endif()
    link_directories("/usr/local/cuda/lib64")
    endif()

    ######################################################################
    ## Find HIP stuff
    find_package(HIP QUIET MODULE)
    if (HIP_FOUND)
        add_definitions(-DHAVE_HIP)
        include_directories("/opt/rocm/include")
        set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
        if (DEFINED ENV{HIP_PLATFORM})
            if ($ENV{HIP_PLATFORM} STREQUAL "nvcc")
                #set(HIP_NVCC_FLAGS "-ccbin g++-7" ${HIP_NVCC_FLAGS})
                set(CMAKE_HIP_LINK_EXECUTABLE "${CUDAToolkit_NVCC_EXECUTABLE} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
            endif()
        endif()

        set(HIP_NVCC_FLAGS "-ccbin g++-8" ${HIP_NVCC_FLAGS})
        option(OPT_USE_HIP "Use HIP GPU code path" OFF)
        if (OPT_USE_HIP)
            execute_process(COMMAND sh -c "/opt/rocm/hip/bin/hipify-perl ${CMAKE_SOURCE_DIR}/src/kernels_cuda.cu > ${CMAKE_SOURCE_DIR}/src/hipified_kernels_cuda.cpp")
            execute_process(COMMAND sh -c "/opt/rocm/hip/bin/hipify-perl ${CMAKE_SOURCE_DIR}/src/maxwell_kernels_cuda.cu > ${CMAKE_SOURCE_DIR}/src/hipified_maxwell_kernels_cuda.cpp")
            add_definitions(-DGPU_USE_HIP)
            add_definitions(-D__HIP_PLATFORM_HCC__)
            set(LINK_LIBS ${LINK_LIBS})
        endif()

    endif()

    option(OPT_BLOCKED_GPU_KERNELS "Use blocked kernels on GPU (DO NOT USE)" OFF)
    if (OPT_BLOCKED_GPU_KERNELS)
        add_definitions(-DUSE_BLOCKED_GPU_KERNELS)
    endif()
endif()

if (OPT_ENABLE_GPU_SOLVER)
    option(OPT_DEBUG_GPU "Enable GPU debugging" OFF)
    if (OPT_DEBUG_GPU)
        add_definitions(-DCUDA_DEBUG)
        add_definitions(-DGPU_DEBUG)
    endif()
endif()

######################################################################
## Compiler identification stuff
if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
    set(COMPILER_IS_CLANG TRUE)
elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
    set(COMPILER_IS_GNU TRUE)
elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
    set(COMPILER_IS_INTEL TRUE)
endif ()

######################################################################
## Compiler optimization options

if (COMPILER_IS_CLANG)
    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Weverything -Wno-c++98-compat \
        -Wno-c++98-compat-pedantic -Wno-zero-as-null-pointer-constant -Wno-global-constructors \
        -Wno-padded -Wno-shorten-64-to-32 -Wno-sign-conversion -Wno-old-style-cast \
        -Wno-sign-compare -Wno-c99-extensions -Wno-extra-semi-stmt -Wno-source-uses-openmp")
else()
    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Wno-sign-compare -Wshadow \
        -Wno-unknown-pragmas -Wno-unused-parameter")
endif()
set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g -fpermissive -fsanitize=address")
set(CMAKE_CXX_FLAGS_RELEASE "-O3 -g -DNDEBUG -fpermissive")
set(CMAKE_CXX_FLAGS_RELEASEASSERT "-O3 -g -fpermissive")

option(OPT_ENABLE_OPENMP "Enable OpenMP parallelization" OFF)
if (OPT_ENABLE_OPENMP)
    find_package(OpenMP)
    if (OpenMP_FOUND)
        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
    endif()
endif()

if (COMPILER_IS_CLANG OR COMPILER_IS_GNU OR COMPILER_IS_INTEL)
    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
endif()

######################################################################
## Optimization stuff

option(OPT_PREFER_512bit "Prefer 512 bit vectors with AVX512 (only Clang > 10 & GCC)" OFF)
if (OPT_PREFER_512bit)
    # https://reviews.llvm.org/D67259
    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mprefer-vector-width=512")
endif()

option(OPT_AGGRESSIVE_FP "Enable DAZ, FTZ and -ffast-math" ON)
if (OPT_AGGRESSIVE_FP)
    add_definitions(-DDISALLOW_DENORMALS)
    if (COMPILER_IS_CLANG OR COMPILER_IS_GNU)
        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ffast-math")
    endif()
endif()

option(OPT_VECTORIZER_REMARKS "Enable vectorizer remarks" OFF)
if (OPT_VECTORIZER_REMARKS)
    if (COMPILER_IS_CLANG)
        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Rpass=loop-vectorize \
            -Rpass-missed=loop-vectorize -Rpass-analysis=loop-vectorize")
    endif()

    if (COMPILER_IS_INTEL)
        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -qopt-report-phase=vec -qopt-report=2")
    endif()

    if (COMPILER_IS_GNU)
        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopt-info-vec-optimized")
    endif()
endif()

option(OPT_MEASURE_PERFORMANCE "Enable performance measurement (produces a ton of output)" OFF)
if (OPT_MEASURE_PERFORMANCE)
    add_definitions(-DMEASURE_PERFORMANCE)
endif()

option(OPT_CRASH_ON_NAN "Crash code if NaNs are generated" OFF)
if (OPT_CRASH_ON_NAN)
    add_definitions(-DCRASH_ON_NAN)
endif()

######################################################################
## Development & debug stuff

if (COMPILER_IS_CLANG OR COMPILER_IS_GNU)
    option(OPT_ASAN "Enable Address Sanitizer" OFF)
    if (OPT_ASAN)
        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address")
    endif()
endif()

option(OPT_WRITE_TEST_OUTPUTS "Produce SILO files in tests" OFF)
if (OPT_WRITE_TEST_OUTPUTS)
    add_definitions(-DWRITE_TEST_OUTPUTS)
endif()

option(OPT_EXPENSIVE_ASSERTS "Enable expensive assert()s" OFF)
if (OPT_EXPENSIVE_ASSERTS)
    add_definitions(-DEXPENSIVE_ASSERTS)
endif()
######################################################################
## Enable/disable solver modules
option(OPT_DEBUG_PRINT "Print debug information")
if (OPT_DEBUG_PRINT)
    add_definitions(-DENABLE_DEBUG_PRINT)
endif()

######################################################################
## Handle 4.8.4 API breakage introduced in commit 91312e67
option(OPT_GMSH_484_BEFORE_SPLIT "Use old version of getKeysForElements() (see 91312e67)" OFF)
if (OPT_GMSH_484_BEFORE_SPLIT)
    add_definitions(-DUSE_INITIAL_4_8_4_API)
endif()

option(OPT_DISABLE_TEXTURE_CACHE "On ROCm 4.1 texture stuff is broken, disable it" OFF)
if (OPT_DISABLE_TEXTURE_CACHE)
    add_definitions(-DDISABLE_TEXTURE_CACHE)
endif()
######################################################################
## Configure targets
include_directories(contrib/sgr)
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include)

########## libgmshdg
set(LIBGMSHDG_SOURCES   src/gmsh_io.cpp
                        src/silo_io.cpp
                        src/entity.cpp
                        src/reference_element.cpp
                        src/physical_element.cpp
                        src/connectivity.cpp
                        src/entity_data.cpp
                        src/kernels_cpu.cpp
                        src/param_loader.cpp
                    )

if (OPT_ENABLE_GPU_SOLVER)
    set(LIBGMSHDG_SOURCES ${LIBGMSHDG_SOURCES} src/kernels_gpu_mi.cpp)

    if (OPT_USE_HIP)
        set_source_files_properties(src/hipified_kernels_cuda.cpp PROPERTIES HIP_SOURCE_PROPERTY_FORMAT 1)
        set(LIBGMSHDG_SOURCES ${LIBGMSHDG_SOURCES} src/hipified_kernels_cuda.cpp)
        hip_add_library(gmshdg SHARED ${LIBGMSHDG_SOURCES})
        target_link_libraries(gmshdg ${LINK_LIBS})
    else()
        set(LIBGMSHDG_SOURCES ${LIBGMSHDG_SOURCES} src/kernels_cuda.cu)
        add_library(gmshdg SHARED ${LIBGMSHDG_SOURCES})
        target_link_libraries(gmshdg ${LINK_LIBS})
    endif()
else()
    add_library(gmshdg SHARED ${LIBGMSHDG_SOURCES})
    target_link_libraries(gmshdg ${LINK_LIBS})
endif()

########## mpi testing stuff

add_executable(testpart src/testpart.cpp)
target_link_libraries(testpart gmshdg)

add_executable(testmod src/testmod.cpp)
target_link_libraries(testmod gmshdg)

########## maxwell solver
set(MAXWELL_SOURCES     src/maxwell_interface.cpp
                        src/maxwell_cpu.cpp
                        src/maxwell_common.cpp
                        src/maxwell_solver.cpp
                    )

if (OPT_ENABLE_GPU_SOLVER)
    set(MAXWELL_SOURCES ${MAXWELL_SOURCES} src/maxwell_gpu.cpp)

    if (OPT_USE_HIP)
        set_source_files_properties(src/hipified_maxwell_kernels_cuda.cpp PROPERTIES HIP_SOURCE_PROPERTY_FORMAT 1)
        set(MAXWELL_SOURCES ${MAXWELL_SOURCES} src/hipified_maxwell_kernels_cuda.cpp)
        hip_add_executable(maxwell_solver ${MAXWELL_SOURCES} HIPCC_OPTIONS "-std=c++17")
        target_link_libraries(maxwell_solver gmshdg)
    else()
        set(MAXWELL_SOURCES ${MAXWELL_SOURCES} src/maxwell_kernels_cuda.cu)
        add_executable(maxwell_solver ${MAXWELL_SOURCES})
        target_link_libraries(maxwell_solver gmshdg)
    endif()
else()
    add_executable(maxwell_solver ${MAXWELL_SOURCES})
    target_link_libraries(maxwell_solver gmshdg)
endif()

########## tests
option(OPT_BUILD_TESTS "Build tests" OFF)
if (OPT_BUILD_TESTS)
    add_executable(test_basics tests/test_basics.cpp)
    target_link_libraries(test_basics gmshdg ${LINK_LIBS})

    add_executable(test_mass tests/test_mass.cpp)
    target_link_libraries(test_mass gmshdg ${LINK_LIBS})

    add_executable(test_differentiation tests/test_differentiation.cpp)
    target_link_libraries(test_differentiation gmshdg ${LINK_LIBS})

    add_executable(test_lifting tests/test_lifting.cpp)
    target_link_libraries(test_lifting gmshdg ${LINK_LIBS})

    if (OPT_ENABLE_GPU_SOLVER)
        add_executable(test_differentiation_gpu tests/test_differentiation_gpu.cpp)
        target_link_libraries(test_differentiation_gpu gmshdg ${LINK_LIBS})

        add_executable(profile_differentiation_gpu tests/profile_differentiation_gpu.cpp)
        target_link_libraries(profile_differentiation_gpu gmshdg ${LINK_LIBS})

        add_executable(test_lifting_gpu tests/test_lifting_gpu.cpp)
        target_link_libraries(test_lifting_gpu gmshdg ${LINK_LIBS})
    endif()
endif()