cmake_minimum_required(VERSION 3.18)
project(gmsh_gpu_dg)
include(CheckLanguage)


set(CMAKE_CXX_STANDARD 20)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CUDA_ARCHITECTURES 35)

# Additional modules path for cmake
set (CMAKE_MODULE_PATH
     ${CMAKE_MODULE_PATH}
     ${CMAKE_CURRENT_SOURCE_DIR}/cmake)


if(NOT DEFINED HIP_PATH)
    if(NOT DEFINED ENV{HIP_PATH})
        set(HIP_PATH "/opt/rocm/hip" CACHE PATH "Path to which HIP has been installed")
    else()
        set(HIP_PATH $ENV{HIP_PATH} CACHE PATH "Path to which HIP has been installed")
    endif()
endif()
set(CMAKE_MODULE_PATH "${HIP_PATH}/cmake" ${CMAKE_MODULE_PATH})

######################################################################
## Find required libraries

find_package(Eigen3 REQUIRED)
set(LINK_LIBS ${LINK_LIBS} Eigen3::Eigen)

set(GMSH_DIR "/opt/uliege/gmsh" CACHE PATH "GMSH install path")
if (DEFINED ENV{GMSH_DIR})
    set (GMSH_DIR $ENV{GMSH_DIR})
endif()

find_package(SILO)
if (SILO_FOUND)
    add_definitions(-DHAVE_SILO)
    include_directories("${SILO_INCLUDE_DIRS}")
    set(LINK_LIBS ${LINK_LIBS} ${SILO_LIBRARIES})
endif()

find_package(Lua REQUIRED)
include_directories("${LUA_INCLUDE_DIR}")
set(LINK_LIBS ${LINK_LIBS} ${LUA_LIBRARIES})
set(SOL2_BUILD_LUA OFF CACHE BOOL "Override sol2 build lua option")
add_subdirectory(contrib/sol2)
include_directories(contrib/sol2/include)

######################################################################
## Set GMSH paths

if (GMSH_DIR STREQUAL "" OR NOT EXISTS ${GMSH_DIR})
    message(FATAL_ERROR "GMSH not found")
else()
    set(GMSH_INCLUDE ${GMSH_DIR}/include)
    set(GMSH_LIB ${GMSH_DIR}/lib)
    set(LINK_LIBS ${LINK_LIBS} gmsh)
endif()

include_directories(${GMSH_INCLUDE})
link_directories(${GMSH_LIB})




######################################################################
## Use or not use GPU solvers

option(OPT_ENABLE_GPU_SOLVER "Enable GPU solvers" OFF)
if (OPT_ENABLE_GPU_SOLVER)
    add_definitions(-DENABLE_GPU_SOLVER)

    ######################################################################
    ## CUDA stuff, find toolkit, compiler & NVML
    find_package(CUDAToolkit)
    if (CUDAToolkit_FOUND)
        set(CMAKE_CUDA_COMPILER ${CUDAToolkit_NVCC_EXECUTABLE})
        enable_language(CUDA)
        set(CMAKE_CUDA_STANDARD 14)
        set(CMAKE_CUDA_STANDARD_REQUIRED TRUE)
        #set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -lineinfo --resource-usage -Xptxas -dlcm=ca")
        set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -lineinfo --resource-usage -prec-div=true")

        if (TARGET CUDA::cudart)
            set(HAVE_CUDA TRUE)
            set(LINK_LIBS ${LINK_LIBS} CUDA::cudart)
            add_definitions(-DHAVE_CUDA)
        endif()

        if (TARGET CUDA::nvml)
            set(HAVE_CUDA_NVML TRUE)
            set(LINK_LIBS ${LINK_LIBS} CUDA::nvml)
            add_definitions(-DHAVE_CUDA_NVML)
        endif()
	link_directories("/usr/local/cuda/lib64")
    endif()

    ######################################################################
    ## Find HIP stuff
    find_package(HIP QUIET MODULE)
    if (HIP_FOUND)
        add_definitions(-DHAVE_HIP)
        include_directories("/opt/rocm/include")
        set(CMAKE_HIP_LINK_EXECUTABLE "${HIP_HIPCC_CMAKE_LINKER_HELPER} ${HCC_HOME} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
        if (DEFINED ENV{HIP_PLATFORM}) 
            if ($ENV{HIP_PLATFORM} STREQUAL "nvcc")
                #set(HIP_NVCC_FLAGS "-ccbin g++-7" ${HIP_NVCC_FLAGS})
                set(CMAKE_HIP_LINK_EXECUTABLE "${CUDAToolkit_NVCC_EXECUTABLE} <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
            endif()
        endif()

        set(HIP_NVCC_FLAGS "-ccbin g++-8" ${HIP_NVCC_FLAGS})
        option(OPT_USE_HIP "Use HIP GPU code path" OFF)
        if (OPT_USE_HIP)
            add_definitions(-DGPU_USE_HIP)
            add_definitions(-D__HIP_PLATFORM_HCC__)
        endif()
    endif()

endif()


######################################################################
## Compiler identification stuff
if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
    set(COMPILER_IS_CLANG TRUE)
elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
    set(COMPILER_IS_GNU TRUE)
elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
    set(COMPILER_IS_INTEL TRUE)
endif ()

######################################################################
## Compiler optimization options

if (COMPILER_IS_CLANG)
    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Weverything -Wno-c++98-compat -Wno-c++98-compat-pedantic -Wno-zero-as-null-pointer-constant -Wno-global-constructors -Wno-padded -Wno-shorten-64-to-32 -Wno-sign-conversion -Wno-old-style-cast -Wno-sign-compare -Wno-c99-extensions -Wno-extra-semi-stmt")
else()
    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall")
endif()
set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g -fpermissive")
set(CMAKE_CXX_FLAGS_RELEASE "-O3 -g -DNDEBUG -fpermissive")
set(CMAKE_CXX_FLAGS_RELEASEASSERT "-O3 -g -fpermissive")

option(OPT_ENABLE_OPENMP "Enable OpenMP parallelization" OFF)
if (OPT_ENABLE_OPENMP)
    find_package(OpenMP)
    if (OpenMP_FOUND)
        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
    endif()
endif()

if (COMPILER_IS_CLANG OR COMPILER_IS_GNU OR COMPILER_IS_INTEL)
    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
endif()

######################################################################
## Optimization stuff

option(OPT_PREFER_512bit "Prefer 512 bit vectors with AVX512 (only Clang > 10 & GCC)" OFF)
if (OPT_PREFER_512bit)
    # https://reviews.llvm.org/D67259
    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mprefer-vector-width=512")
endif()

option(OPT_AGGRESSIVE_FP "Enable DAZ, FTZ and -ffast-math" ON)
if (OPT_AGGRESSIVE_FP)
    add_definitions(-DDISALLOW_DENORMALS)
    if (COMPILER_IS_CLANG OR COMPILER_IS_GNU)
        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ffast-math")
    endif()
endif()

option(OPT_VECTORIZER_REMARKS "Enable vectorizer remarks" OFF)
if (OPT_VECTORIZER_REMARKS)
    if (COMPILER_IS_CLANG)
        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Rpass=loop-vectorize -Rpass-missed=loop-vectorize -Rpass-analysis=loop-vectorize")
    endif()

    if (COMPILER_IS_INTEL)
        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -qopt-report-phase=vec -qopt-report=2")
    endif()

    if (COMPILER_IS_GNU)
        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopt-info-vec-optimized")
    endif()
endif()

option(OPT_MEASURE_PERFORMANCE "Enable performance measurement (produces a ton of output)" OFF)
if (OPT_MEASURE_PERFORMANCE)
    add_definitions(-DMEASURE_PERFORMANCE)
endif()

option(OPT_CRASH_ON_NAN "Crash code if NaNs are generated" OFF)
if (OPT_CRASH_ON_NAN)
    add_definitions(-DCRASH_ON_NAN)
endif()

######################################################################
## Development & debug stuff

if (COMPILER_IS_CLANG OR COMPILER_IS_GNU)
    option(OPT_ASAN "Enable Address Sanitizer" OFF)
    if (OPT_ASAN)
        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address")
    endif()
endif()

option(OPT_ENABLE_CUDA_DEBUG_GUARDS "Enable CUDA debug guards")
if (OPT_ENABLE_CUDA_DEBUG_GUARDS AND HAVE_CUDA)
    add_definitions(-DCUDA_DEBUG)
endif()

######################################################################
## Enable/disable solver modules

option(OPT_ENABLE_CPU_SOLVER "Enable CPU DG solver")
if (OPT_ENABLE_CPU_SOLVER)
    add_definitions(-DENABLE_CPU_SOLVER)
endif()

option(OPT_DEBUG_PRINT "Print debug information")
if (OPT_DEBUG_PRINT)
    add_definitions(-DENABLE_DEBUG_PRINT)
endif()

######################################################################
## Configure targets 

include_directories(contrib/sgr)
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include)

set(COMMON_SOURCES gmsh_io.cpp reference_element.cpp physical_element.cpp entity.cpp kernels_cpu.cpp connectivity.cpp entity_data.cpp)

set(LIBGMSHDG_SOURCES src/gmsh_io.cpp
                      src/reference_element.cpp
                      src/physical_element.cpp
                      src/entity.cpp
                      src/kernels_cpu.cpp
                      src/connectivity.cpp
                      src/entity_data.cpp)

add_library(gmshdg SHARED ${LIBGMSHDG_SOURCES})
target_link_libraries(gmshdg ${LINK_LIBS})

add_executable(test_basics tests/test_basics.cpp)
target_link_libraries(test_basics gmshdg)

add_executable(test_mass tests/test_mass.cpp)
target_link_libraries(test_mass gmshdg)

add_executable(test_differentiation tests/test_differentiation.cpp)
target_link_libraries(test_differentiation gmshdg)

add_executable(test_differentiation_gpu tests/test_differentiation_gpu.cpp)
target_link_libraries(test_differentiation_gpu gmshdg)

#add_executable(test test.cpp test_basics.cpp test_mass.cpp test_differentiation.cpp test_differentiation_gpu.cpp test_lifting.cpp ${COMMON_SOURCES})
#target_link_libraries(test ${LINK_LIBS})

#add_executable(dump_orient dump_orient.cpp)
#target_link_libraries(dump_orient ${LINK_LIBS})

#add_executable(crash_gmsh doublefree.cpp)
#target_link_libraries(crash_gmsh ${LINK_LIBS})