diff --git a/CMakeLists.txt b/CMakeLists.txt index 7ddec8b..c4f00c7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,10 +1,9 @@ -cmake_minimum_required(VERSION 3.18) +cmake_minimum_required(VERSION 3.21) set(project remhos) project(${project} LANGUAGES CXX) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_CXX_COMPILER_LAUNCHER ccache) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) # CXX flags ******************************************************************* @@ -18,21 +17,35 @@ set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2" CACHE STRING "" FORCE) # Verbosity options *********************************************************** set(CMAKE_VERBOSE_MAKEFILE OFF CACHE BOOL "" FORCE) -# set(CUDA_VERBOSE_BUILD OFF CACHE BOOL "" FORCE) - -# Include paths *************************************************************** -include_directories(${CMAKE_CURRENT_SOURCE_DIR}) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../mfem) -if (${CMAKE_SYSTEM_NAME} MATCHES "Linux") -include_directories(/usr/include/hypre /usr/include) -elseif(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") -include_directories(/opt/homebrew/opt/fmt/include - /opt/homebrew/opt/openmpi/include - /opt/homebrew/opt/metis/include - /opt/homebrew/opt/hypre/include) -add_compile_definitions(MFEM_USE_CMAKE_TESTS) -else() - message(FATAL_ERROR "Unsupported system") + +# Dependencies **************************************************************** +find_package(HYPRE REQUIRED) +find_package(MFEM REQUIRED) +find_package(MPI REQUIRED) + +option(REMHOS_USE_CALIPER "Use Caliper" OFF) +if (REMHOS_USE_CALIPER) + find_package(caliper REQUIRED) + find_package(adiak REQUIRED) +endif() + +# HIP support ***************************************************************** +# Read MFEM_USE_HIP from MFEMConfig or from the cache (passed via -DMFEM_USE_HIP=ON) +if (NOT DEFINED MFEM_USE_HIP) + set(MFEM_USE_HIP OFF) +endif() + +if (MFEM_USE_HIP) + if(NOT DEFINED CMAKE_HIP_ARCHITECTURES) + set(CMAKE_HIP_ARCHITECTURES gfx942) + endif() + enable_language(HIP) + set(CMAKE_HIP_STANDARD ${CMAKE_CXX_STANDARD} CACHE STRING + "HIP standard to use.") + set(CMAKE_HIP_STANDARD_REQUIRED ON CACHE BOOL + "Force the use of the chosen HIP standard.") + set(CMAKE_HIP_EXTENSIONS OFF CACHE BOOL "Enable HIP standard extensions.") + message(STATUS "Using HIP architecture: ${CMAKE_HIP_ARCHITECTURES}") endif() # Copy mesh files ************************************************************* @@ -46,40 +59,73 @@ add_custom_command(OUTPUT data_is_copied add_custom_target(copy_mesh_files DEPENDS data_is_copied) # Library source files ******************************************************** -add_library(Remhos STATIC remhos.cpp remhos_fct.cpp - remhos_ho.cpp remhos_lo.cpp - remhos_mono.cpp remhos_sync.cpp - remhos_tools.cpp) +set(LIB_SOURCES + remhos.cpp + remhos_fct.cpp + remhos_ho.cpp + remhos_lo.cpp + remhos_mono.cpp + remhos_solvers.cpp + remhos_sync.cpp + remhos_tools.cpp +) + +if (MFEM_USE_HIP) + set_source_files_properties(${LIB_SOURCES} PROPERTIES LANGUAGE HIP) +endif() + +add_library(Remhos STATIC ${LIB_SOURCES}) + +target_compile_definitions(Remhos PUBLIC MFEM_USE_CMAKE_TESTS) -# Testing options ************************************************************* +target_include_directories(Remhos PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR} + ${MFEM_INCLUDE_DIRS} + ${MFEM_INCLUDE_DIRS}/mfem +) + +target_link_libraries(Remhos + PUBLIC mfem + PUBLIC MPI::MPI_CXX + PUBLIC HYPRE::HYPRE +) + +if (REMHOS_USE_CALIPER) + target_compile_definitions(Remhos PUBLIC REMHOS_USE_CALIPER) + target_link_libraries(Remhos PUBLIC caliper adiak::adiak) +endif() + +# Executables ***************************************************************** +if (MFEM_USE_HIP) + set_source_files_properties(remhos_main.cpp PROPERTIES LANGUAGE HIP) +endif() +add_executable(remhos_main remhos_main.cpp) +add_dependencies(remhos_main copy_mesh_files) +target_link_libraries(remhos_main PRIVATE Remhos) + +# Install ********************************************************************* +install(TARGETS remhos_main DESTINATION bin) +install(DIRECTORY data/ DESTINATION data) + +# Testing ********************************************************************* enable_testing() + function(add_remhos_test name) - set(file ${name}.cpp) - set(target ${name}) - list(LENGTH ARGN ARGN_COUNT) - if(ARGN_COUNT GREATER 0) - list(GET ARGN 0 extra) - string(APPEND name "_" ${extra}) - string(APPEND target "_" ${extra}) + add_executable(${name} ${name}.cpp) + add_dependencies(${name} copy_mesh_files) + target_link_libraries(${name} PRIVATE Remhos) + if (MFEM_USE_HIP) + set_source_files_properties(${name}.cpp PROPERTIES LANGUAGE HIP) endif() - message(STATUS "Adding target: ${target}") - add_executable(${target} ${file}) - add_dependencies(${target} copy_mesh_files) - target_link_libraries(${target} Remhos -lmpi -lmfem -lhypre -lmetis -lfmt) -if (${CMAKE_SYSTEM_NAME} MATCHES "Linux") - target_link_directories(${target} PUBLIC /usr/lib/x86_64-linux-gnu) -elseif(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") - target_link_directories(${target} PUBLIC /opt/homebrew/opt/openmpi/lib - /opt/homebrew/opt/metis/lib - /opt/homebrew/opt/hypre/lib - /opt/homebrew/opt/fmt/lib) -endif() - target_link_directories(${target} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../mfem) - message(STATUS "Adding test: ${target}_n1") - add_test(NAME ${target}_n1 COMMAND mpirun -n 1 ${target} ${extra}) - message(STATUS "Adding test: ${target}_n3") - add_test(NAME ${target}_n3 COMMAND mpirun -n 3 ${target}) + add_test(NAME ${name}_n1 + COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 1 $) + add_test(NAME ${name}_n3 + COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 3 $) endfunction() -add_remhos_test(remhos_main) -add_remhos_test(remhos_tests) \ No newline at end of file +add_test(NAME remhos_main_n1 + COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 1 $) +add_test(NAME remhos_main_n3 + COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 3 $) + +add_remhos_test(remhos_tests) diff --git a/remhos.cpp b/remhos.cpp index 693d826..b8b8958 100644 --- a/remhos.cpp +++ b/remhos.cpp @@ -216,9 +216,9 @@ int main(int argc, char *argv[]) MFEM_EXPORT int remhos(int argc, char *argv[], double &final_mass_u) { - // Initialize MPI. - mfem::MPI_Session mpi(argc, argv); - const int myid = mpi.WorldRank(); + // Initialize MPI (skip if already initialized, e.g. when called from a test harness). + if (!mfem::Mpi::IsInitialized()) { mfem::Mpi::Init(argc, argv); } + const int myid = mfem::Mpi::WorldRank(); const char *mesh_file = "default"; int dim = 3; @@ -351,10 +351,13 @@ MFEM_EXPORT int remhos(int argc, char *argv[], double &final_mass_u) const char * allocator_name = REMHOS_DEVICE_ALLOCATOR_NAME; size_t umpire_dev_pool_size = ((size_t) dev_pool_size) * 1024 * 1024 * 1024; size_t umpire_dev_block_size = 1024 * 1024; - rm.makeAllocator(allocator_name, - rm.getAllocator("DEVICE"), - umpire_dev_pool_size, - umpire_dev_block_size); + if (!rm.isAllocator(allocator_name)) + { + rm.makeAllocator(allocator_name, + rm.getAllocator("DEVICE"), + umpire_dev_pool_size, + umpire_dev_block_size); + } #ifdef HYPRE_USING_UMPIRE HYPRE_SetUmpireDevicePoolName(allocator_name); diff --git a/remhos_tests.cpp b/remhos_tests.cpp index 1aa1af5..2845260 100644 --- a/remhos_tests.cpp +++ b/remhos_tests.cpp @@ -1,15 +1,18 @@ #include +#include +#include #include #include #include +#include #include -// #define DBG_COLOR ::debug::kCyan -// #include "debug.hpp" +#include "general/communication.hpp" +using mfem::real_t; int remhos(int, char *[], double &); -//// /////////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// template std::enable_if_t::is_integer, bool> AlmostEq(T x, T y, T tolerance = 10.0*std::numeric_limits::epsilon()) @@ -25,83 +28,74 @@ AlmostEq(T x, T y, T tolerance = 10.0*std::numeric_limits::epsilon()) /////////////////////////////////////////////////////////////////////////////// struct Test { - static constexpr const char *binary = "remhos "; - static constexpr const char *common = "-no-vis -vs 1"; - std::string mesh, options; - const double result = 0.0; - Test(const char * msh, const char * extra, double result) - : mesh(msh), options(std::string(extra) + common), result(result) {} - std::string Command() const { return binary + mesh + options; } + static constexpr const char *binary = "remhos"; + static constexpr const char *common = + "-dt -1.0 -tf 0.5 -ho 3 -lo 5 -fct 2 -ms 5 -no-vis -vs 1"; + std::string name, mesh, options; + const real_t result = std::numeric_limits::signaling_NaN(); + Test(const char *name, const char *mesh, const char *extra, real_t result) + : name(name), mesh(mesh), options(std::string(extra) + " " + common), + result(result) {} + std::string Command() const + { + return std::string(binary) + " " + mesh + " " + options; + } }; /////////////////////////////////////////////////////////////////////////////// const Test runs[] = { { - // #0 - "-m ./data/inline-quad.mesh ", - "-p 14 -rs 1 -o 2 -dt -1.0 -tf 0.5 -ho 3 -lo 5 -fct 2 -ms 5 ", - 0.09711395400387984 + "quad-14-1-2", // #0 + "-m ./data/inline-quad.mesh", "-p 14 -rs 1 -o 2", + 0.0971135253228191 }, { - // #1, 65536 - "-m ./data/inline-quad.mesh ", - "-p 14 -rs 4 -o 3 -dt -1.0 -tf 0.5 -ho 3 -lo 5 -fct 2 -ms 5 ", - 0.0930984399257905 + "quad-14-4-3", // #1 + "-m ./data/inline-quad.mesh", "-p 14 -rs 4 -o 3", + 0.0930984398474033 }, { - // #2, 102400 - // RHS kernel time: 5.4988707 - // L2inv kernel time: 0.94180588 - // LO kernel time: 0.028575583 - // FCT kernel time: 0.0095909583 - // Total kernel time: 5.5370372 - "-m ./data/inline-quad.mesh ", - "-p 14 -rs 4 -o 4 -dt -1.0 -tf 0.5 -ho 3 -lo 5 -fct 2 -ms 5 ", - 0.09237630484178257 + "quad-14-4-4", // #2 + "-m ./data/inline-quad.mesh", "-p 14 -rs 4 -o 4", + 0.0923763049829040 }, { - // #3 - "-m ./data/cube01_hex.mesh ", - "-p 10 -rs 1 -o 2 -dt -1.0 -tf 0.5 -ho 3 -lo 5 -fct 2 -ms 5 ", - 0.11972857593296446 + "cube01-10-1-2", // #3 + "-m ./data/cube01_hex.mesh", "-p 10 -rs 1 -o 2", + 0.1197219672304615 }, { - // #4 Partial assembly - "-m ./data/inline-quad.mesh ", - "-pa -p 14 -rs 1 -o 2 -dt -1.0 -tf 0.5 -ho 3 -lo 5 -fct 2 -ms 5 ", - 0.09711395400387984 + "quad-pa-14-1-2", // #4 + "-m ./data/inline-quad.mesh", "-pa -p 14 -rs 1 -o 2", + 0.0971135252706320 }, { - // #5 Partial assembly - "-m ./data/inline-quad.mesh ", - "-pa -p 14 -rs 4 -o 2 -dt -1.0 -tf 0.5 -ho 3 -lo 5 -fct 2 -ms 5 ", - 0.09185717760402806 + "quad-pa-14-4-2", // #5 + "-m ./data/inline-quad.mesh", "-pa -p 14 -rs 4 -o 2", + 0.0918571775667415 }, { - // #7: 3D PA - "-m ./data/cube01_hex.mesh ", - "-pa -p 10 -rs 3 -o 3 -dt -1.0 -tf 0.5 -ho 3 -lo 5 -fct 2 -ms 1 ", - 0.11601536511552431 + "cube01-pa-10-2-3", // #6 + "-m ./data/cube01_hex.mesh", "-pa -p 10 -rs 2 -o 3", + 0.1162578642211454 }, { - // #8 - "-m ./data/star-q2.mesh ", - "-pa -p 14 -rs 1 -o 3 -dt -1.0 -tf 0.5 -ho 3 -lo 5 -fct 2 -ms 5 ", - 0.8069675186775516 + "star-q2-pa-14-1-3", // #7 + "-m ./data/star-q2.mesh", "-pa -p 14 -rs 1 -o 3", + 0.7975688080682961 }, { - // #9, 🔥 debug device 🔥 (last or !dup) - "-m ./data/inline-quad.mesh ", - "-d debug -pa -p 14 -rs 1 -o 2 -dt -1.0 -tf 0.5 -ho 3 -lo 5 -fct 2 -ms 5 ", - 0.09711395400387984 + "debug-quad-14-1-2", // #8 + "-m ./data/inline-quad.mesh", "-d debug -pa -p 14 -rs 1 -o 2", + 0.0971135252706320 }, #ifdef MFEM_USE_CUDA { - // #10 - "-m ./data/inline-quad.mesh ", - "-d cuda -pa -p 14 -rs 1 -o 2 -dt -1.0 -tf 0.5 -ho 3 -lo 5 -fct 2 -ms 5 ", - 0.09711395400387984 + "cuda-quad-14-1-2", + "-m ./data/inline-quad.mesh", + "-d cuda -pa -p 14 -rs 1 -o 2", + 0.0971135252706320 }, #endif }; @@ -133,17 +127,20 @@ int RemhosTest(const Test & test) double final_mass_u{}; remhos(args.size()-1, args.data(), final_mass_u); - // dbg("final_mass_u: {} vs. {}", final_mass_u, test.result); + if (AlmostEq(final_mass_u, test.result)) { return EXIT_SUCCESS; } - if (AlmostEq(final_mass_u, test.result)) { return std::cout << "✅" << std::endl, EXIT_SUCCESS; } - - return std::cout << "❌" << std::endl, EXIT_FAILURE; + mfem::err << "❌ " << test.name + << std::fixed << std::setprecision(16) + << ": final_mass_u: " << final_mass_u + << " vs. " << test.result + << std::endl; + return EXIT_FAILURE; } /////////////////////////////////////////////////////////////////////////////// int main(int argc, char* argv[]) try { - // dbgClearScreen(), dbg(); + mfem::Mpi::Init(argc, argv); int opt; int test = -1; @@ -178,4 +175,4 @@ catch (std::exception& e) std::cerr << "\033[31m..xxxXXX[ERROR]XXXxxx.." << std::endl; std::cerr << "\033[31m{}" << e.what() << std::endl; return EXIT_FAILURE; -} \ No newline at end of file +}