From 66ef1fca35a92c25ace750cb67c90738248b62d1 Mon Sep 17 00:00:00 2001 From: Peter Josef Scheibel Date: Mon, 27 Jul 2026 12:16:27 -0700 Subject: [PATCH 1/4] update cmake --- CMakeLists.txt | 129 +++++++++++++++++++++++++++++-------------------- 1 file changed, 76 insertions(+), 53 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7ddec8b..efb0f38 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,10 +1,9 @@ -cmake_minimum_required(VERSION 3.18) +cmake_minimum_required(VERSION 3.21) set(project remhos) project(${project} LANGUAGES CXX) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_CXX_COMPILER_LAUNCHER ccache) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) # CXX flags ******************************************************************* @@ -18,21 +17,35 @@ set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2" CACHE STRING "" FORCE) # Verbosity options *********************************************************** set(CMAKE_VERBOSE_MAKEFILE OFF CACHE BOOL "" FORCE) -# set(CUDA_VERBOSE_BUILD OFF CACHE BOOL "" FORCE) - -# Include paths *************************************************************** -include_directories(${CMAKE_CURRENT_SOURCE_DIR}) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../mfem) -if (${CMAKE_SYSTEM_NAME} MATCHES "Linux") -include_directories(/usr/include/hypre /usr/include) -elseif(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") -include_directories(/opt/homebrew/opt/fmt/include - /opt/homebrew/opt/openmpi/include - /opt/homebrew/opt/metis/include - /opt/homebrew/opt/hypre/include) -add_compile_definitions(MFEM_USE_CMAKE_TESTS) -else() - message(FATAL_ERROR "Unsupported system") + +# Dependencies **************************************************************** +find_package(HYPRE REQUIRED) +find_package(MFEM REQUIRED) +find_package(MPI REQUIRED) + +option(REMHOS_USE_CALIPER "Use Caliper" OFF) +if (REMHOS_USE_CALIPER) + find_package(caliper REQUIRED) + find_package(adiak REQUIRED) +endif() + +# HIP support ***************************************************************** +# Read MFEM_USE_HIP from MFEMConfig or from the cache (passed via -DMFEM_USE_HIP=ON) +if (NOT DEFINED MFEM_USE_HIP) + set(MFEM_USE_HIP OFF) +endif() + +if (MFEM_USE_HIP) + if(NOT DEFINED CMAKE_HIP_ARCHITECTURES) + set(CMAKE_HIP_ARCHITECTURES gfx942) + endif() + enable_language(HIP) + set(CMAKE_HIP_STANDARD ${CMAKE_CXX_STANDARD} CACHE STRING + "HIP standard to use.") + set(CMAKE_HIP_STANDARD_REQUIRED ON CACHE BOOL + "Force the use of the chosen HIP standard.") + set(CMAKE_HIP_EXTENSIONS OFF CACHE BOOL "Enable HIP standard extensions.") + message(STATUS "Using HIP architecture: ${CMAKE_HIP_ARCHITECTURES}") endif() # Copy mesh files ************************************************************* @@ -46,40 +59,50 @@ add_custom_command(OUTPUT data_is_copied add_custom_target(copy_mesh_files DEPENDS data_is_copied) # Library source files ******************************************************** -add_library(Remhos STATIC remhos.cpp remhos_fct.cpp - remhos_ho.cpp remhos_lo.cpp - remhos_mono.cpp remhos_sync.cpp - remhos_tools.cpp) - -# Testing options ************************************************************* -enable_testing() -function(add_remhos_test name) - set(file ${name}.cpp) - set(target ${name}) - list(LENGTH ARGN ARGN_COUNT) - if(ARGN_COUNT GREATER 0) - list(GET ARGN 0 extra) - string(APPEND name "_" ${extra}) - string(APPEND target "_" ${extra}) - endif() - message(STATUS "Adding target: ${target}") - add_executable(${target} ${file}) - add_dependencies(${target} copy_mesh_files) - target_link_libraries(${target} Remhos -lmpi -lmfem -lhypre -lmetis -lfmt) -if (${CMAKE_SYSTEM_NAME} MATCHES "Linux") - target_link_directories(${target} PUBLIC /usr/lib/x86_64-linux-gnu) -elseif(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") - target_link_directories(${target} PUBLIC /opt/homebrew/opt/openmpi/lib - /opt/homebrew/opt/metis/lib - /opt/homebrew/opt/hypre/lib - /opt/homebrew/opt/fmt/lib) +set(LIB_SOURCES + remhos.cpp + remhos_fct.cpp + remhos_ho.cpp + remhos_lo.cpp + remhos_mono.cpp + remhos_solvers.cpp + remhos_sync.cpp + remhos_tools.cpp +) + +if (MFEM_USE_HIP) + set_source_files_properties(${LIB_SOURCES} PROPERTIES LANGUAGE HIP) endif() - target_link_directories(${target} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../mfem) - message(STATUS "Adding test: ${target}_n1") - add_test(NAME ${target}_n1 COMMAND mpirun -n 1 ${target} ${extra}) - message(STATUS "Adding test: ${target}_n3") - add_test(NAME ${target}_n3 COMMAND mpirun -n 3 ${target}) -endfunction() - -add_remhos_test(remhos_main) -add_remhos_test(remhos_tests) \ No newline at end of file + +add_library(Remhos STATIC ${LIB_SOURCES}) + +target_compile_definitions(Remhos PUBLIC MFEM_USE_CMAKE_TESTS) + +target_include_directories(Remhos PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR} + ${MFEM_INCLUDE_DIRS} + ${MFEM_INCLUDE_DIRS}/mfem +) + +target_link_libraries(Remhos + PUBLIC mfem + PUBLIC MPI::MPI_CXX + PUBLIC HYPRE::HYPRE +) + +if (REMHOS_USE_CALIPER) + target_compile_definitions(Remhos PUBLIC REMHOS_USE_CALIPER) + target_link_libraries(Remhos PUBLIC caliper adiak::adiak) +endif() + +# Executables ***************************************************************** +if (MFEM_USE_HIP) + set_source_files_properties(remhos_main.cpp PROPERTIES LANGUAGE HIP) +endif() +add_executable(remhos_main remhos_main.cpp) +add_dependencies(remhos_main copy_mesh_files) +target_link_libraries(remhos_main PRIVATE Remhos) + +# Install ********************************************************************* +install(TARGETS remhos_main DESTINATION bin) +install(DIRECTORY data/ DESTINATION data) From fd764171606b9e7d3cd734c7955aaea76446bfd8 Mon Sep 17 00:00:00 2001 From: Peter Josef Scheibel Date: Mon, 27 Jul 2026 15:49:07 -0700 Subject: [PATCH 2/4] changes to enable tests --- CMakeLists.txt | 23 +++++++++++++++++++++++ remhos.cpp | 17 ++++++++++------- remhos_tests.cpp | 7 ++++++- 3 files changed, 39 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index efb0f38..c4f00c7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -106,3 +106,26 @@ target_link_libraries(remhos_main PRIVATE Remhos) # Install ********************************************************************* install(TARGETS remhos_main DESTINATION bin) install(DIRECTORY data/ DESTINATION data) + +# Testing ********************************************************************* +enable_testing() + +function(add_remhos_test name) + add_executable(${name} ${name}.cpp) + add_dependencies(${name} copy_mesh_files) + target_link_libraries(${name} PRIVATE Remhos) + if (MFEM_USE_HIP) + set_source_files_properties(${name}.cpp PROPERTIES LANGUAGE HIP) + endif() + add_test(NAME ${name}_n1 + COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 1 $) + add_test(NAME ${name}_n3 + COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 3 $) +endfunction() + +add_test(NAME remhos_main_n1 + COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 1 $) +add_test(NAME remhos_main_n3 + COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 3 $) + +add_remhos_test(remhos_tests) diff --git a/remhos.cpp b/remhos.cpp index e65781e..cb85fe9 100644 --- a/remhos.cpp +++ b/remhos.cpp @@ -209,9 +209,9 @@ int main(int argc, char *argv[]) MFEM_EXPORT int remhos(int argc, char *argv[], double &final_mass_u) { - // Initialize MPI. - mfem::MPI_Session mpi(argc, argv); - const int myid = mpi.WorldRank(); + // Initialize MPI (skip if already initialized, e.g. when called from a test harness). + if (!mfem::Mpi::IsInitialized()) { mfem::Mpi::Init(argc, argv); } + const int myid = mfem::Mpi::WorldRank(); const char *mesh_file = "default"; int dim = 3; @@ -344,10 +344,13 @@ MFEM_EXPORT int remhos(int argc, char *argv[], double &final_mass_u) const char * allocator_name = REMHOS_DEVICE_ALLOCATOR_NAME; size_t umpire_dev_pool_size = ((size_t) dev_pool_size) * 1024 * 1024 * 1024; size_t umpire_dev_block_size = 1024 * 1024; - rm.makeAllocator(allocator_name, - rm.getAllocator("DEVICE"), - umpire_dev_pool_size, - umpire_dev_block_size); + if (!rm.isAllocator(allocator_name)) + { + rm.makeAllocator(allocator_name, + rm.getAllocator("DEVICE"), + umpire_dev_pool_size, + umpire_dev_block_size); + } #ifdef HYPRE_USING_UMPIRE HYPRE_SetUmpireDevicePoolName(allocator_name); diff --git a/remhos_tests.cpp b/remhos_tests.cpp index 1aa1af5..89dd249 100644 --- a/remhos_tests.cpp +++ b/remhos_tests.cpp @@ -1,9 +1,13 @@ #include +#include #include #include #include +#include #include +#include "mfem/general/communication.hpp" + // #define DBG_COLOR ::debug::kCyan // #include "debug.hpp" @@ -12,7 +16,7 @@ int remhos(int, char *[], double &); //// /////////////////////////////////////////////////////////////////////////// template std::enable_if_t::is_integer, bool> -AlmostEq(T x, T y, T tolerance = 10.0*std::numeric_limits::epsilon()) +AlmostEq(T x, T y, T tolerance = 1e-8) { const T neg = std::abs(x - y); constexpr T min = std::numeric_limits::min(); @@ -143,6 +147,7 @@ int RemhosTest(const Test & test) /////////////////////////////////////////////////////////////////////////////// int main(int argc, char* argv[]) try { + mfem::Mpi::Init(argc, argv); // dbgClearScreen(), dbg(); int opt; From ea12983614d71aaf9bda714289fe8f83420c4221 Mon Sep 17 00:00:00 2001 From: Peter Josef Scheibel Date: Mon, 5 Oct 2026 10:41:22 -0700 Subject: [PATCH 3/4] suggested edits to tolerance --- remhos_tests.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/remhos_tests.cpp b/remhos_tests.cpp index 89dd249..6fad788 100644 --- a/remhos_tests.cpp +++ b/remhos_tests.cpp @@ -16,7 +16,7 @@ int remhos(int, char *[], double &); //// /////////////////////////////////////////////////////////////////////////// template std::enable_if_t::is_integer, bool> -AlmostEq(T x, T y, T tolerance = 1e-8) +AlmostEq(T x, T y, T tolerance = 10.0*std::numeric_limits::epsilon()) { const T neg = std::abs(x - y); constexpr T min = std::numeric_limits::min(); @@ -45,7 +45,7 @@ const Test runs[] = // #0 "-m ./data/inline-quad.mesh ", "-p 14 -rs 1 -o 2 -dt -1.0 -tf 0.5 -ho 3 -lo 5 -fct 2 -ms 5 ", - 0.09711395400387984 + 0.0971139540038799 }, { // #1, 65536 @@ -74,31 +74,31 @@ const Test runs[] = // #4 Partial assembly "-m ./data/inline-quad.mesh ", "-pa -p 14 -rs 1 -o 2 -dt -1.0 -tf 0.5 -ho 3 -lo 5 -fct 2 -ms 5 ", - 0.09711395400387984 + 0.0971139539524386 }, { // #5 Partial assembly "-m ./data/inline-quad.mesh ", "-pa -p 14 -rs 4 -o 2 -dt -1.0 -tf 0.5 -ho 3 -lo 5 -fct 2 -ms 5 ", - 0.09185717760402806 + 0.0918571775959331 }, { // #7: 3D PA "-m ./data/cube01_hex.mesh ", - "-pa -p 10 -rs 3 -o 3 -dt -1.0 -tf 0.5 -ho 3 -lo 5 -fct 2 -ms 1 ", - 0.11601536511552431 + "-pa -p 10 -rs 2 -o 3 -dt -1.0 -tf 0.5 -ho 3 -lo 5 -fct 2 -ms 5 ", + 0.1162581030309265 }, { // #8 "-m ./data/star-q2.mesh ", "-pa -p 14 -rs 1 -o 3 -dt -1.0 -tf 0.5 -ho 3 -lo 5 -fct 2 -ms 5 ", - 0.8069675186775516 + 0.8069675185906946 }, { // #9, 🔥 debug device 🔥 (last or !dup) "-m ./data/inline-quad.mesh ", "-d debug -pa -p 14 -rs 1 -o 2 -dt -1.0 -tf 0.5 -ho 3 -lo 5 -fct 2 -ms 5 ", - 0.09711395400387984 + 0.0971139539524386 }, #ifdef MFEM_USE_CUDA { @@ -183,4 +183,4 @@ catch (std::exception& e) std::cerr << "\033[31m..xxxXXX[ERROR]XXXxxx.." << std::endl; std::cerr << "\033[31m{}" << e.what() << std::endl; return EXIT_FAILURE; -} \ No newline at end of file +} From 9771146ceb52a3dfcd52d209ffa718509472bc65 Mon Sep 17 00:00:00 2001 From: camierjs Date: Wed, 7 Oct 2026 08:45:48 -0700 Subject: [PATCH 4/4] Refresh remhos test expected results --- remhos_tests.cpp | 116 ++++++++++++++++++++++------------------------- 1 file changed, 54 insertions(+), 62 deletions(-) diff --git a/remhos_tests.cpp b/remhos_tests.cpp index 6fad788..2845260 100644 --- a/remhos_tests.cpp +++ b/remhos_tests.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -6,14 +7,12 @@ #include #include -#include "mfem/general/communication.hpp" - -// #define DBG_COLOR ::debug::kCyan -// #include "debug.hpp" +#include "general/communication.hpp" +using mfem::real_t; int remhos(int, char *[], double &); -//// /////////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// template std::enable_if_t::is_integer, bool> AlmostEq(T x, T y, T tolerance = 10.0*std::numeric_limits::epsilon()) @@ -29,83 +28,74 @@ AlmostEq(T x, T y, T tolerance = 10.0*std::numeric_limits::epsilon()) /////////////////////////////////////////////////////////////////////////////// struct Test { - static constexpr const char *binary = "remhos "; - static constexpr const char *common = "-no-vis -vs 1"; - std::string mesh, options; - const double result = 0.0; - Test(const char * msh, const char * extra, double result) - : mesh(msh), options(std::string(extra) + common), result(result) {} - std::string Command() const { return binary + mesh + options; } + static constexpr const char *binary = "remhos"; + static constexpr const char *common = + "-dt -1.0 -tf 0.5 -ho 3 -lo 5 -fct 2 -ms 5 -no-vis -vs 1"; + std::string name, mesh, options; + const real_t result = std::numeric_limits::signaling_NaN(); + Test(const char *name, const char *mesh, const char *extra, real_t result) + : name(name), mesh(mesh), options(std::string(extra) + " " + common), + result(result) {} + std::string Command() const + { + return std::string(binary) + " " + mesh + " " + options; + } }; /////////////////////////////////////////////////////////////////////////////// const Test runs[] = { { - // #0 - "-m ./data/inline-quad.mesh ", - "-p 14 -rs 1 -o 2 -dt -1.0 -tf 0.5 -ho 3 -lo 5 -fct 2 -ms 5 ", - 0.0971139540038799 + "quad-14-1-2", // #0 + "-m ./data/inline-quad.mesh", "-p 14 -rs 1 -o 2", + 0.0971135253228191 }, { - // #1, 65536 - "-m ./data/inline-quad.mesh ", - "-p 14 -rs 4 -o 3 -dt -1.0 -tf 0.5 -ho 3 -lo 5 -fct 2 -ms 5 ", - 0.0930984399257905 + "quad-14-4-3", // #1 + "-m ./data/inline-quad.mesh", "-p 14 -rs 4 -o 3", + 0.0930984398474033 }, { - // #2, 102400 - // RHS kernel time: 5.4988707 - // L2inv kernel time: 0.94180588 - // LO kernel time: 0.028575583 - // FCT kernel time: 0.0095909583 - // Total kernel time: 5.5370372 - "-m ./data/inline-quad.mesh ", - "-p 14 -rs 4 -o 4 -dt -1.0 -tf 0.5 -ho 3 -lo 5 -fct 2 -ms 5 ", - 0.09237630484178257 + "quad-14-4-4", // #2 + "-m ./data/inline-quad.mesh", "-p 14 -rs 4 -o 4", + 0.0923763049829040 }, { - // #3 - "-m ./data/cube01_hex.mesh ", - "-p 10 -rs 1 -o 2 -dt -1.0 -tf 0.5 -ho 3 -lo 5 -fct 2 -ms 5 ", - 0.11972857593296446 + "cube01-10-1-2", // #3 + "-m ./data/cube01_hex.mesh", "-p 10 -rs 1 -o 2", + 0.1197219672304615 }, { - // #4 Partial assembly - "-m ./data/inline-quad.mesh ", - "-pa -p 14 -rs 1 -o 2 -dt -1.0 -tf 0.5 -ho 3 -lo 5 -fct 2 -ms 5 ", - 0.0971139539524386 + "quad-pa-14-1-2", // #4 + "-m ./data/inline-quad.mesh", "-pa -p 14 -rs 1 -o 2", + 0.0971135252706320 }, { - // #5 Partial assembly - "-m ./data/inline-quad.mesh ", - "-pa -p 14 -rs 4 -o 2 -dt -1.0 -tf 0.5 -ho 3 -lo 5 -fct 2 -ms 5 ", - 0.0918571775959331 + "quad-pa-14-4-2", // #5 + "-m ./data/inline-quad.mesh", "-pa -p 14 -rs 4 -o 2", + 0.0918571775667415 }, { - // #7: 3D PA - "-m ./data/cube01_hex.mesh ", - "-pa -p 10 -rs 2 -o 3 -dt -1.0 -tf 0.5 -ho 3 -lo 5 -fct 2 -ms 5 ", - 0.1162581030309265 + "cube01-pa-10-2-3", // #6 + "-m ./data/cube01_hex.mesh", "-pa -p 10 -rs 2 -o 3", + 0.1162578642211454 }, { - // #8 - "-m ./data/star-q2.mesh ", - "-pa -p 14 -rs 1 -o 3 -dt -1.0 -tf 0.5 -ho 3 -lo 5 -fct 2 -ms 5 ", - 0.8069675185906946 + "star-q2-pa-14-1-3", // #7 + "-m ./data/star-q2.mesh", "-pa -p 14 -rs 1 -o 3", + 0.7975688080682961 }, { - // #9, 🔥 debug device 🔥 (last or !dup) - "-m ./data/inline-quad.mesh ", - "-d debug -pa -p 14 -rs 1 -o 2 -dt -1.0 -tf 0.5 -ho 3 -lo 5 -fct 2 -ms 5 ", - 0.0971139539524386 + "debug-quad-14-1-2", // #8 + "-m ./data/inline-quad.mesh", "-d debug -pa -p 14 -rs 1 -o 2", + 0.0971135252706320 }, #ifdef MFEM_USE_CUDA { - // #10 - "-m ./data/inline-quad.mesh ", - "-d cuda -pa -p 14 -rs 1 -o 2 -dt -1.0 -tf 0.5 -ho 3 -lo 5 -fct 2 -ms 5 ", - 0.09711395400387984 + "cuda-quad-14-1-2", + "-m ./data/inline-quad.mesh", + "-d cuda -pa -p 14 -rs 1 -o 2", + 0.0971135252706320 }, #endif }; @@ -137,18 +127,20 @@ int RemhosTest(const Test & test) double final_mass_u{}; remhos(args.size()-1, args.data(), final_mass_u); - // dbg("final_mass_u: {} vs. {}", final_mass_u, test.result); - - if (AlmostEq(final_mass_u, test.result)) { return std::cout << "✅" << std::endl, EXIT_SUCCESS; } + if (AlmostEq(final_mass_u, test.result)) { return EXIT_SUCCESS; } - return std::cout << "❌" << std::endl, EXIT_FAILURE; + mfem::err << "❌ " << test.name + << std::fixed << std::setprecision(16) + << ": final_mass_u: " << final_mass_u + << " vs. " << test.result + << std::endl; + return EXIT_FAILURE; } /////////////////////////////////////////////////////////////////////////////// int main(int argc, char* argv[]) try { mfem::Mpi::Init(argc, argv); - // dbgClearScreen(), dbg(); int opt; int test = -1;