From b8bd28c662b8286bbd1edaaa80653827496e4bda Mon Sep 17 00:00:00 2001 From: Brad Whitlock Date: Thu, 4 Sep 2025 18:55:07 -0700 Subject: [PATCH 01/24] Initial port of read_stl_mesh_shared and quest::signed_distance functionality so it uses Umpire shared memory allocators. --- .../quest/interface/internal/QuestHelpers.cpp | 243 ++++++++---------- .../quest/interface/internal/QuestHelpers.hpp | 137 +--------- src/axom/quest/interface/signed_distance.cpp | 53 ++-- .../tests/quest_signed_distance_interface.cpp | 19 +- 4 files changed, 155 insertions(+), 297 deletions(-) diff --git a/src/axom/quest/interface/internal/QuestHelpers.cpp b/src/axom/quest/interface/internal/QuestHelpers.cpp index 4049749e55..ce50416f75 100644 --- a/src/axom/quest/interface/internal/QuestHelpers.cpp +++ b/src/axom/quest/interface/internal/QuestHelpers.cpp @@ -23,6 +23,11 @@ #endif #endif +#if defined(AXOM_USE_MPI) && defined(AXOM_USE_UMPIRE) && (defined(UMPIRE_ENABLE_IPC_SHARED_MEMORY) || defined(UMPIRE_ENABLE_MPI3_SHARED_MEMORY)) + #include "umpire/Umpire.hpp" + #define AXOM_USE_UMPIRE_SHARED_MEMORY +#endif + #include namespace axom @@ -31,40 +36,84 @@ namespace quest { namespace internal { -/// MPI Helper/Wrapper Methods - -#ifdef AXOM_USE_MPI +/// Mesh I/O methods -/* - * Deallocates the specified MPI window object. +#if defined(AXOM_USE_UMPIRE_SHARED_MEMORY) +/*! + * \brief Allocates a shared memory buffer for the mesh that is shared among + * all the ranks within the same compute node. + * + * \param [in] allocatorID A valid Umpire allocator id that can allocate shared memory. + * \param [in] mesh_metada tuple with the number of nodes/faces on the mesh + * \param [out] x pointer into the buffer where the x--coordinates are stored. + * \param [out] y pointer into the buffer where the y--coordinates are stored. + * \param [out] z pointer into the buffer where the z--coordinates are stored. + * \param [out] conn pointer into the buffer consisting the cell-connectivity. + * \param [out] mesh_buffer raw buffer consisting of all the mesh data. + * + * \return bytesize the number of bytes in the raw buffer. + * + * \pre allocatorID is a valid shared memory allocator. + * \pre mesh_metadata != nullptr + * \pre x == nullptr + * \pre y == nullptr + * \pre z == nullptr + * \pre conn == nullptr + * \pre mesh_buffer == nullptr + * + * \post x != nullptr + * \post y != nullptr + * \post z != nullptr + * \post coon != nullptr + * \post mesh_buffer != nullptr */ -void mpi_win_free(MPI_Win* window) +static size_t allocate_shared_buffer(int allocatorID, + const axom::IndexType mesh_metadata[2], + double*& x, + double*& y, + double*& z, + axom::IndexType*& conn, + unsigned char*& mesh_buffer) { - if(*window != MPI_WIN_NULL) - { - MPI_Win_free(window); - } -} + // Allocate the buffer. + const axom::IndexType nnodes = mesh_metadata[0]; + const axom::IndexType nfaces = mesh_metadata[1]; + const size_t bytesize = nnodes * 3 * sizeof(double) + nfaces * 3 * sizeof(axom::IndexType); + mesh_buffer = allocate(bytesize, allocatorID); -/* - * Deallocates the specified MPI communicator object. - */ -void mpi_comm_free(MPI_Comm* comm) -{ - if(*comm != MPI_COMM_NULL) - { - MPI_Comm_free(comm); - } + // calculate offset to the coordinates & cell connectivity in the buffer + int baseOffset = nnodes * sizeof(double); + int x_offset = 0; + int y_offset = baseOffset; + int z_offset = y_offset + baseOffset; + int conn_offset = z_offset + baseOffset; + + x = reinterpret_cast(&mesh_buffer[x_offset]); + y = reinterpret_cast(&mesh_buffer[y_offset]); + z = reinterpret_cast(&mesh_buffer[z_offset]); + conn = reinterpret_cast(&mesh_buffer[conn_offset]); + + return (bytesize); } -/* - * Reads the mesh on rank 0 and exchanges the mesh metadata, i.e., the - * number of nodes and faces with all other ranks. +/*! + * \brief Reads the mesh on rank 0 and exchanges the mesh metadata, i.e., the + * number of nodes and faces with all other ranks. + * + * \param [in] global_rank_id MPI rank w.r.t. the global communicator + * \param [in] global_comm handle to the global communicator + * \param [in,out] reader the corresponding STL reader + * \param [out] mesh_metadata an array consisting of the mesh metadata. + * + * \note This method calls read() on the reader on rank 0. + * + * \pre global_comm != MPI_COMM_NULL + * \pre mesh_metadata != nullptr */ -int read_and_exchange_mesh_metadata(int global_rank_id, - MPI_Comm global_comm, - quest::STLReader& reader, - axom::IndexType mesh_metadata[2]) +static int read_and_exchange_mesh_metadata(int global_rank_id, + MPI_Comm global_comm, + quest::STLReader& reader, + axom::IndexType mesh_metadata[2]) { constexpr int NUM_NODES = 0; constexpr int NUM_FACES = 1; @@ -94,108 +143,18 @@ int read_and_exchange_mesh_metadata(int global_rank_id, return rc; } -#endif /* AXOM_USE_MPI */ - -#ifdef AXOM_USE_MPI3 -/* - * Creates inter-node and intra-node communicators from the given global - * MPI communicator handle. - */ -void create_communicators(MPI_Comm global_comm, - MPI_Comm& intra_node_comm, - MPI_Comm& inter_node_comm, - int& global_rank_id, - int& local_rank_id, - int& intercom_rank_id) -{ - // Sanity checks - SLIC_ASSERT(global_comm != MPI_COMM_NULL); - SLIC_ASSERT(intra_node_comm == MPI_COMM_NULL); - SLIC_ASSERT(inter_node_comm == MPI_COMM_NULL); - - constexpr int IGNORE_KEY = 0; - - // STEP 0: get global rank, used to order ranks in the inter-node comm. - MPI_Comm_rank(global_comm, &global_rank_id); - - // STEP 1: create the intra-node communicator - MPI_Comm_split_type(global_comm, MPI_COMM_TYPE_SHARED, IGNORE_KEY, MPI_INFO_NULL, &intra_node_comm); - MPI_Comm_rank(intra_node_comm, &local_rank_id); - SLIC_ASSERT(local_rank_id >= 0); - - // STEP 2: create inter-node communicator - const int color = (local_rank_id == 0) ? 1 : MPI_UNDEFINED; - MPI_Comm_split(global_comm, color, global_rank_id, &inter_node_comm); - - if(color == 1) - { - MPI_Comm_rank(inter_node_comm, &intercom_rank_id); - } - - SLIC_ASSERT(intra_node_comm != MPI_COMM_NULL); -} -#endif - -#if defined(AXOM_USE_MPI) && defined(AXOM_USE_MPI3) -/* - * Allocates a shared memory buffer for the mesh that is shared among - * all the ranks within the same compute node. - */ -MPI_Aint allocate_shared_buffer(int local_rank_id, - MPI_Comm intra_node_comm, - const axom::IndexType mesh_metadata[2], - double*& x, - double*& y, - double*& z, - axom::IndexType*& conn, - unsigned char*& mesh_buffer, - MPI_Win& shared_window) -{ - constexpr int ROOT_RANK = 0; - - const int nnodes = mesh_metadata[0]; - const int nfaces = mesh_metadata[1]; - - int disp = sizeof(unsigned char); - MPI_Aint bytesize = nnodes * 3 * sizeof(double) + nfaces * 3 * sizeof(axom::IndexType); - MPI_Aint window_size = (local_rank_id != ROOT_RANK) ? 0 : bytesize; - - MPI_Win_allocate_shared(window_size, disp, MPI_INFO_NULL, intra_node_comm, &mesh_buffer, &shared_window); - MPI_Win_shared_query(shared_window, ROOT_RANK, &bytesize, &disp, &mesh_buffer); - - // calculate offset to the coordinates & cell connectivity in the buffer - int baseOffset = nnodes * sizeof(double); - int x_offset = 0; - int y_offset = baseOffset; - int z_offset = y_offset + baseOffset; - int conn_offset = z_offset + baseOffset; - - x = reinterpret_cast(&mesh_buffer[x_offset]); - y = reinterpret_cast(&mesh_buffer[y_offset]); - z = reinterpret_cast(&mesh_buffer[z_offset]); - conn = reinterpret_cast(&mesh_buffer[conn_offset]); - - return (bytesize); -} -#endif - -/// Mesh I/O methods - -#if defined(AXOM_USE_MPI) && defined(AXOM_USE_MPI3) /* * Reads in the surface mesh from the specified file into a shared * memory buffer that is attached to the given MPI shared window. */ int read_stl_mesh_shared(const std::string& file, MPI_Comm global_comm, + int allocatorID, unsigned char*& mesh_buffer, - mint::Mesh*& m, - MPI_Comm& intra_node_comm, - MPI_Win& shared_window) + mint::Mesh*& m) { SLIC_ASSERT(global_comm != MPI_COMM_NULL); - SLIC_ASSERT(intra_node_comm == MPI_COMM_NULL); - SLIC_ASSERT(shared_window == MPI_WIN_NULL); + SLIC_ASSERT(allocatorID != INVALID_ALLOCATOR_ID); // NOTE: STL meshes are always 3D mesh consisting of triangles. using TriangleMesh = mint::UnstructuredMesh; @@ -213,17 +172,23 @@ int read_stl_mesh_shared(const std::string& file, return READ_FAILED; } - // STEP 1: create intra-node and inter-node MPI communicators + // STEP 1: Get the communicator from the Umpire allocator. int global_rank_id = -1; - int local_rank_id = -1; int intercom_rank_id = -1; + MPI_Comm_rank(global_comm, &global_rank_id); MPI_Comm inter_node_comm = MPI_COMM_NULL; - create_communicators(global_comm, - intra_node_comm, - inter_node_comm, - global_rank_id, - local_rank_id, - intercom_rank_id); + umpire::ResourceManager& rm = umpire::ResourceManager::getInstance(); + if(rm.isAllocator(allocatorID)) + { + umpire::Allocator allocator = rm.getAllocator(allocatorID); + inter_node_comm = umpire::get_communicator_for_allocator(allocator, global_comm); + MPI_Comm_rank(inter_node_comm, &intercom_rank_id); +std::cout << "Rank " << global_rank_id << ": intercom_rank_id=" << intercom_rank_id << std::endl; + } + else + { + SLIC_ERROR(axom::fmt::format("An invalid allocatorID {} was provided.", allocatorID)); + } // STEP 2: Exchange mesh metadata constexpr int NUM_NODES = 0; @@ -243,15 +208,13 @@ int read_stl_mesh_shared(const std::string& file, double* y = nullptr; double* z = nullptr; axom::IndexType* conn = nullptr; - MPI_Aint numBytes = allocate_shared_buffer(local_rank_id, - intra_node_comm, - mesh_metadata, - x, - y, - z, - conn, - mesh_buffer, - shared_window); + const size_t numBytes = allocate_shared_buffer(allocatorID, + mesh_metadata, + x, + y, + z, + conn, + mesh_buffer); SLIC_ASSERT(x != nullptr); SLIC_ASSERT(y != nullptr); SLIC_ASSERT(z != nullptr); @@ -267,16 +230,14 @@ int read_stl_mesh_shared(const std::string& file, reader.getMesh(static_cast(m)); } - // STEP 5: inter-node communication + // STEP 5: inter-node communication (broadcast mesh_buffer to shared_memory on other nodes) if(intercom_rank_id >= 0) { MPI_Bcast(mesh_buffer, numBytes, MPI_UNSIGNED_CHAR, 0, inter_node_comm); } - // STEP 6 free communicators + // NOTE: Communicators are associated with Umpire allocators and Umpire will free them. - MPI_Barrier(global_comm); - mpi_comm_free(&inter_node_comm); return READ_SUCCESS; } #endif diff --git a/src/axom/quest/interface/internal/QuestHelpers.hpp b/src/axom/quest/interface/internal/QuestHelpers.hpp index 8fe5da0760..d350e875c1 100644 --- a/src/axom/quest/interface/internal/QuestHelpers.hpp +++ b/src/axom/quest/interface/internal/QuestHelpers.hpp @@ -60,137 +60,10 @@ class ScopedLogLevelChanger slic::message::Level m_previousLevel {slic::message::Level::Debug}; }; -/// \name MPI Helper/Wrapper Methods -/// @{ -#ifdef AXOM_USE_MPI - -/*! - * \brief Deallocates the specified MPI window object. - * \param [in] window handle to the MPI window. - * \note All buffers attached to the window are also deallocated. - */ -void mpi_win_free(MPI_Win* window); - -/*! - * \brief Deallocates the specified MPI communicator object. - * \param [in] comm handle to the MPI communicator object. - */ -void mpi_comm_free(MPI_Comm* comm); - -/*! - * \brief Reads the mesh on rank 0 and exchanges the mesh metadata, i.e., the - * number of nodes and faces with all other ranks. - * - * \param [in] global_rank_id MPI rank w.r.t. the global communicator - * \param [in] global_comm handle to the global communicator - * \param [in,out] reader the corresponding STL reader - * \param [out] mesh_metadata an array consisting of the mesh metadata. - * - * \note This method calls read() on the reader on rank 0. - * - * \pre global_comm != MPI_COMM_NULL - * \pre mesh_metadata != nullptr - */ -int read_and_exchange_mesh_metadata(int global_rank_id, - MPI_Comm global_comm, - quest::STLReader& reader, - axom::IndexType mesh_metadata[2]); - -#endif /* AXOM_USE_MPI */ - -#ifdef AXOM_USE_MPI3 -/*! - * \brief Creates inter-node and intra-node communicators from the given global - * MPI communicator handle. - * - * The intra-node communicator groups the ranks within the same compute node. - * Consequently, all ranks have a global rank ID, w.r.t. the global - * communicator, and a corresponding local rank ID, w.r.t. the intra-node - * communicator. The global rank ID can span and is unique across multiple - * compute nodes, while the local rank ID, is only uniquely defined within the - * same compute node and is generally different from the global rank ID. - * - * In contrast, the inter-node communicator groups only a subset of the ranks - * defined by the global communicator. Specifically, ranks that have a - * local rank ID of zero are included in the inter-node commuinicator and - * have a corresponding inter-comm rank ID, w.r.t., the inter-node - * communicator. For all other ranks, that are not included in the inter-node - * communicator, the inter-comm rank ID is set to "-1". - * - * \param [in] global_comm handle to the global MPI communicator. - * \param [out] intra_node_comm handle to the intra-node communicator object. - * \param [out] inter_node_comm handle to the inter-node communicator object. - * \param [out] global_rank_id rank ID w.r.t. the global communicator - * \param [out] local_rank_id rank ID within the a compute node - * \param [out] intercom_rank_id rank ID w.r.t. the inter-node communicator - - * \note The caller must call `MPI_Comm_free` on the corresponding communicator - * handles, namely, `intra_node_comm` and `inter_node_comm` which are created - * by this routine. - * - * \pre global_comm != MPI_COMM_NULL - * \pre intra_node_comm == MPI_COMM_NULL - * \pre inter_node_comm == MPI_COMM_NULL - * \post intra_node_comm != MPI_COMM_NULL - * \post inter_node_comm != MPI_COMM_NULL - */ -void create_communicators(MPI_Comm global_comm, - MPI_Comm& intra_node_comm, - MPI_Comm& inter_node_comm, - int& global_rank_id, - int& local_rank_id, - int& intercom_rank_id); -#endif - -#if defined(AXOM_USE_MPI) && defined(AXOM_USE_MPI3) -/*! - * \brief Allocates a shared memory buffer for the mesh that is shared among - * all the ranks within the same compute node. - * - * \param [in] intra_node_comm intra-node communicator within a node. - * \param [in] mesh_metada tuple with the number of nodes/faces on the mesh - * \param [out] x pointer into the buffer where the x--coordinates are stored. - * \param [out] y pointer into the buffer where the y--coordinates are stored. - * \param [out] z pointer into the buffer where the z--coordinates are stored. - * \param [out] conn pointer into the buffer consisting the cell-connectivity. - * \param [out] mesh_buffer raw buffer consisting of all the mesh data. - * \param [out] shared_window MPI window to which the shared buffer is attached. - * - * \return bytesize the number of bytes in the raw buffer. - * - * \pre intra_node_comm != MPI_COMM_NULL - * \pre mesh_metadata != nullptr - * \pre x == nullptr - * \pre y == nullptr - * \pre z == nullptr - * \pre conn == nullptr - * \pre mesh_buffer == nullptr - * \pre shared_window == MPI_WIN_NULL - * - * \post x != nullptr - * \post y != nullptr - * \post z != nullptr - * \post coon != nullptr - * \post mesh_buffer != nullptr - * \post shared_window != MPI_WIN_NULL - */ -MPI_Aint allocate_shared_buffer(int local_rank_id, - MPI_Comm intra_node_comm, - const axom::IndexType mesh_metadata[2], - double*& x, - double*& y, - double*& z, - axom::IndexType*& conn, - unsigned char*& mesh_buffer, - MPI_Win& shared_window); -#endif - -/// @} - /// \name Mesh I/O methods /// @{ -#if defined(AXOM_USE_MPI) && defined(AXOM_USE_MPI3) +#if defined(AXOM_USE_MPI) && defined(AXOM_USE_UMPIRE) && (defined(UMPIRE_ENABLE_IPC_SHARED_MEMORY) || defined(UMPIRE_ENABLE_MPI3_SHARED_MEMORY)) /*! * \brief Reads in the surface mesh from the specified file into a shared @@ -198,10 +71,9 @@ MPI_Aint allocate_shared_buffer(int local_rank_id, * * \param [in] file the file consisting of the surface mesh * \param [in] global_comm handle to the global MPI communicator + * \param [in] allocatorID An UMPIRE allocator ID that can allocate shared memory. * \param [out] mesh_buffer pointer to the raw mesh buffer * \param [out] m pointer to the mesh object - * \param [out] intra_node_comm handle to the shared MPI communicator. - * \param [out] shared_window handle to the MPI shared window. * * \return status set to READ_SUCCESS, or READ_FAILED on error. * @@ -224,10 +96,9 @@ MPI_Aint allocate_shared_buffer(int local_rank_id, */ int read_stl_mesh_shared(const std::string& file, MPI_Comm global_comm, + int allocatorID, unsigned char*& mesh_buffer, - mint::Mesh*& m, - MPI_Comm& intra_node_comm, - MPI_Win& shared_window); + mint::Mesh*& m); #endif /*! diff --git a/src/axom/quest/interface/signed_distance.cpp b/src/axom/quest/interface/signed_distance.cpp index ee7a18038c..3bacfe6224 100644 --- a/src/axom/quest/interface/signed_distance.cpp +++ b/src/axom/quest/interface/signed_distance.cpp @@ -13,10 +13,21 @@ #include "axom/slic/interface/slic.hpp" +#if defined(AXOM_USE_MPI) && defined(AXOM_USE_UMPIRE) + #include "umpire/Umpire.hpp" + #if defined(UMPIRE_ENABLE_IPC_SHARED_MEMORY) || defined(UMPIRE_ENABLE_MPI3_SHARED_MEMORY) + #include "umpire/strategy/NamedAllocationStrategy.hpp" + #include "umpire/util/MemoryResourceTraits.hpp" + #define AXOM_USE_UMPIRE_SHARED_MEMORY + #endif +#endif + #ifdef AXOM_USE_MPI #include #endif +#include + namespace axom { namespace quest @@ -60,7 +71,7 @@ static struct parameters_t bool verbose; /*!< logger verbosity */ bool is_closed_surface; /*!< indicates if the input is a closed surface */ - bool use_shared_memory; /*!< use MPI-3 shared memory for the surface mesh */ + bool use_shared_memory; /*!< use Umpire shared memory for the surface mesh */ bool compute_sign; /*!< indicates if sign should be computed */ int allocator_id; /*!< the allocator ID to create BVH with (-1 for default) */ SignedDistExec exec_space; /*!< indicates the execution space to run in */ @@ -93,10 +104,13 @@ static bool s_must_delete_mesh = false; static bool s_must_finalize_logger = false; static bool s_logger_is_initialized = false; -#if defined(AXOM_USE_MPI) && defined(AXOM_USE_MPI3) +#if defined(AXOM_USE_UMPIRE_SHARED_MEMORY) +static int s_allocator_id = INVALID_ALLOCATOR_ID; static unsigned char* s_shared_mesh_buffer = nullptr; -MPI_Comm s_intra_node_comm = MPI_COMM_NULL; -MPI_Win s_window = MPI_WIN_NULL; +#else +static std::string s_shared_memory_requirements( + "Shared memory requires MPI and an Umpire library built with " + "UMPIRE_ENABLE_IPC_SHARED_MEMORY set to ON"); #endif } // end anonymous namespace @@ -122,16 +136,26 @@ int signed_distance_init(const std::string& file, MPI_Comm comm) // STEP 0: read the STL mesh int rc = INIT_FAILED; -#if defined(AXOM_USE_MPI) && defined(AXOM_USE_MPI3) +#if defined(AXOM_USE_UMPIRE_SHARED_MEMORY) + if(s_allocator_id == INVALID_ALLOCATOR_ID) + { + // Make a shared memory allocator and get its id. + auto& rm = umpire::ResourceManager::getInstance(); + auto traits{umpire::get_default_resource_traits("SHARED")}; + traits.scope = umpire::MemoryResourceTraits::shared_scope::node; + auto node_allocator{rm.makeResource("SHARED::node_allocator", traits)}; + auto signed_distance_allocator{ + rm.makeAllocator("signed_distance_allocator", node_allocator)}; + s_allocator_id = signed_distance_allocator.getId(); + } if(Parameters.use_shared_memory) { rc = internal::read_stl_mesh_shared(file, comm, + s_allocator_id, s_shared_mesh_buffer, - s_surface_mesh, - s_intra_node_comm, - s_window); + s_surface_mesh); } else { @@ -141,8 +165,7 @@ int signed_distance_init(const std::string& file, MPI_Comm comm) #else SLIC_WARNING_IF(Parameters.use_shared_memory, - "Shared memory requires MPI3 and building Axom with " - "AXOM_USE_MPI3 set to ON"); + s_shared_memory_requirements_message); rc = internal::read_stl_mesh(file, s_surface_mesh, comm); #endif @@ -313,9 +336,9 @@ void signed_distance_use_shared_memory(bool status) Parameters.use_shared_memory = status; -#ifndef AXOM_USE_MPI3 +#if !defined(AXOM_USE_UMPIRE_SHARED_MEMORY) SLIC_WARNING_IF(Parameters.use_shared_memory, - "Enabling shared memory requires MPI-3. Option is ignored!"); + s_shared_memory_requirements + " Option is ignored!"); #endif } @@ -499,10 +522,8 @@ void signed_distance_finalize() SLIC_ASSERT(!signed_distance_initialized()); internal::logger_finalize(s_must_finalize_logger); -#if defined(AXOM_USE_MPI) && defined(AXOM_USE_MPI3) - internal::mpi_comm_free(&s_intra_node_comm); - internal::mpi_win_free(&s_window); - s_shared_mesh_buffer = nullptr; +#if defined(AXOM_USE_UMPIRE_SHARED_MEMORY) + deallocate(s_shared_mesh_buffer); #endif } diff --git a/src/axom/quest/tests/quest_signed_distance_interface.cpp b/src/axom/quest/tests/quest_signed_distance_interface.cpp index e9edffaed5..ebfc1acdfa 100644 --- a/src/axom/quest/tests/quest_signed_distance_interface.cpp +++ b/src/axom/quest/tests/quest_signed_distance_interface.cpp @@ -46,8 +46,11 @@ using UnstructuredMesh = mint::UnstructuredMesh; //------------------------------------------------------------------------------ namespace { -#if defined(AXOM_USE_MPI) && defined(AXOM_USE_MPI3) -constexpr bool USE_MPI3_SHARED_MEMORY = true; +#if defined(AXOM_USE_MPI) && defined(AXOM_USE_UMPIRE) && (defined(UMPIRE_ENABLE_IPC_SHARED_MEMORY) || defined(UMPIRE_ENABLE_MPI3_SHARED_MEMORY)) +#define AXOM_USE_UMPIRE_SHARED_MEMORY +constexpr bool USE_SHARED_MEMORY = true; +#else +constexpr bool USE_SHARED_MEMORY = false; #endif /*! @@ -410,17 +413,17 @@ TEST(quest_signed_distance_interface, analytic_plane) { check_analytic_plane(); -#if defined(AXOM_USE_MPI) && defined(AXOM_USE_MPI3) - check_analytic_plane(USE_MPI3_SHARED_MEMORY); +#if defined(AXOM_USE_UMPIRE_SHARED_MEMORY) + check_analytic_plane(USE_SHARED_MEMORY); #endif } //------------------------------------------------------------------------------ -#if defined(AXOM_USE_MPI) && defined(AXOM_USE_MPI3) +#if defined(AXOM_USE_UMPIRE_SHARED_MEMORY) TEST(quest_signed_distance_interface, call_twice_using_shared_memory) { - check_analytic_plane(USE_MPI3_SHARED_MEMORY); - check_analytic_plane(USE_MPI3_SHARED_MEMORY); + check_analytic_plane(USE_SHARED_MEMORY); + check_analytic_plane(USE_SHARED_MEMORY); } #endif @@ -654,6 +657,8 @@ int main(int argc, char* argv[]) ::testing::FLAGS_gtest_death_test_style = "threadsafe"; axom::slic::SimpleLogger logger; + SLIC_INFO(axom::fmt::format("USE_SHARED_MEMORY: {}", USE_SHARED_MEMORY)); + result = RUN_ALL_TESTS(); #ifdef AXOM_USE_MPI From 5fb886eda9e7f05472672eca9d63bda203d7aad0 Mon Sep 17 00:00:00 2001 From: Brad Whitlock Date: Fri, 5 Sep 2025 12:27:57 -0700 Subject: [PATCH 02/24] Fix some race conditions that prevented passage of quest_signed_distance_interface when run on multiple ranks. Remove build logic to limit the test to 1 rank. --- src/axom/quest/tests/CMakeLists.txt | 3 - .../tests/quest_signed_distance_interface.cpp | 193 +++++++++++------- 2 files changed, 120 insertions(+), 76 deletions(-) diff --git a/src/axom/quest/tests/CMakeLists.txt b/src/axom/quest/tests/CMakeLists.txt index 35da691a24..df8e61a954 100644 --- a/src/axom/quest/tests/CMakeLists.txt +++ b/src/axom/quest/tests/CMakeLists.txt @@ -142,9 +142,6 @@ foreach(test ${quest_mpi_tests}) ) set(_numMPITasks 4) - if (${test_name} STREQUAL "quest_signed_distance_interface") - set(_numMPITasks 1) - endif() if (${test_name} STREQUAL "quest_inout_interface") set(_numMPITasks 1) endif() diff --git a/src/axom/quest/tests/quest_signed_distance_interface.cpp b/src/axom/quest/tests/quest_signed_distance_interface.cpp index ebfc1acdfa..bdf73604dc 100644 --- a/src/axom/quest/tests/quest_signed_distance_interface.cpp +++ b/src/axom/quest/tests/quest_signed_distance_interface.cpp @@ -57,85 +57,120 @@ constexpr bool USE_SHARED_MEMORY = false; * \brief Generate a mesh of 4 triangles along the XY plane. * * \param [in] file the file to write to. + * \param [in] comm The MPI communicator to use. * * \pre file.empty() == false. * \note Used primarily for debugging. */ -void generate_planar_mesh_stl_file(const std::string& file) +void generate_planar_mesh_stl_file(const std::string& file, MPI_Comm comm = MPI_COMM_SELF) { - EXPECT_FALSE(file.empty()); - - std::ofstream ofs(file.c_str()); - EXPECT_TRUE(ofs.is_open()); - - ofs << "solid plane" << std::endl; - - // Triangle T1 - ofs << "\t facet normal 0.0 0.0 1.0" << std::endl; - ofs << "\t\t outer loop" << std::endl; - ofs << "\t\t\t vertex -5.0 -5.0 0.0" << std::endl; - ofs << "\t\t\t vertex 5.0 -5.0 0.0" << std::endl; - ofs << "\t\t\t vertex 0.0 0.0 0.0" << std::endl; - ofs << "\t\t endloop" << std::endl; - ofs << "\t endfacet" << std::endl; - - // Triangle T2 - ofs << "\t facet normal 0.0 0.0 1.0" << std::endl; - ofs << "\t\t outer loop" << std::endl; - ofs << "\t\t\t vertex 5.0 -5.0 0.0" << std::endl; - ofs << "\t\t\t vertex 5.0 5.0 0.0" << std::endl; - ofs << "\t\t\t vertex 0.0 0.0 0.0" << std::endl; - ofs << "\t\t endloop" << std::endl; - ofs << "\t endfacet" << std::endl; - - // Triangle T3 - ofs << "\t facet normal 0.0 0.0 1.0" << std::endl; - ofs << "\t\t outer loop" << std::endl; - ofs << "\t\t\t vertex 5.0 5.0 0.0" << std::endl; - ofs << "\t\t\t vertex -5.0 5.0 0.0" << std::endl; - ofs << "\t\t\t vertex 0.0 0.0 0.0" << std::endl; - ofs << "\t\t endloop" << std::endl; - ofs << "\t endfacet" << std::endl; - - // Triangle T4 - ofs << "\t facet normal 0.0 0.0 1.0" << std::endl; - ofs << "\t\t outer loop" << std::endl; - ofs << "\t\t\t vertex -5.0 5.0 0.0" << std::endl; - ofs << "\t\t\t vertex -5.0 -5.0 0.0" << std::endl; - ofs << "\t\t\t vertex 0.0 0.0 0.0" << std::endl; - ofs << "\t\t endloop" << std::endl; - ofs << "\t endfacet" << std::endl; - - ofs << "endsolid" << std::endl; - ofs.close(); + int rank = 0; + MPI_Comm_rank(comm, &rank); + if(rank == 0) + { + EXPECT_FALSE(file.empty()); + + std::ofstream ofs(file.c_str()); + EXPECT_TRUE(ofs.is_open()); + + ofs << "solid plane" << std::endl; + + // Triangle T1 + ofs << "\t facet normal 0.0 0.0 1.0" << std::endl; + ofs << "\t\t outer loop" << std::endl; + ofs << "\t\t\t vertex -5.0 -5.0 0.0" << std::endl; + ofs << "\t\t\t vertex 5.0 -5.0 0.0" << std::endl; + ofs << "\t\t\t vertex 0.0 0.0 0.0" << std::endl; + ofs << "\t\t endloop" << std::endl; + ofs << "\t endfacet" << std::endl; + + // Triangle T2 + ofs << "\t facet normal 0.0 0.0 1.0" << std::endl; + ofs << "\t\t outer loop" << std::endl; + ofs << "\t\t\t vertex 5.0 -5.0 0.0" << std::endl; + ofs << "\t\t\t vertex 5.0 5.0 0.0" << std::endl; + ofs << "\t\t\t vertex 0.0 0.0 0.0" << std::endl; + ofs << "\t\t endloop" << std::endl; + ofs << "\t endfacet" << std::endl; + + // Triangle T3 + ofs << "\t facet normal 0.0 0.0 1.0" << std::endl; + ofs << "\t\t outer loop" << std::endl; + ofs << "\t\t\t vertex 5.0 5.0 0.0" << std::endl; + ofs << "\t\t\t vertex -5.0 5.0 0.0" << std::endl; + ofs << "\t\t\t vertex 0.0 0.0 0.0" << std::endl; + ofs << "\t\t endloop" << std::endl; + ofs << "\t endfacet" << std::endl; + + // Triangle T4 + ofs << "\t facet normal 0.0 0.0 1.0" << std::endl; + ofs << "\t\t outer loop" << std::endl; + ofs << "\t\t\t vertex -5.0 5.0 0.0" << std::endl; + ofs << "\t\t\t vertex -5.0 -5.0 0.0" << std::endl; + ofs << "\t\t\t vertex 0.0 0.0 0.0" << std::endl; + ofs << "\t\t endloop" << std::endl; + ofs << "\t endfacet" << std::endl; + + ofs << "endsolid" << std::endl; + ofs.close(); + } + MPI_Barrier(comm); } /*! * \brief Generates a simple ASCII STL file consisting of a single triangle. * * \param [in] file the file to write to. + * \param [in] comm The MPI communicator to use. * \pre file.empty() == false. * * \note Used primarily for debugging. */ -void generate_stl_file(const std::string& file) +void generate_stl_file(const std::string& file, MPI_Comm comm = MPI_COMM_SELF) { - EXPECT_FALSE(file.empty()); - - std::ofstream ofs(file.c_str()); - EXPECT_TRUE(ofs.is_open()); - - ofs << "solid triangle" << std::endl; - ofs << "\t facet normal 0.0 0.0 1.0" << std::endl; - ofs << "\t\t outer loop" << std::endl; - ofs << "\t\t\t vertex 0.0 0.0 0.0" << std::endl; - ofs << "\t\t\t vertex 1.0 0.0 0.0" << std::endl; - ofs << "\t\t\t vertex 0.0 1.0 0.0" << std::endl; - ofs << "\t\t endloop" << std::endl; - ofs << "\t endfacet" << std::endl; - ofs << "endsolid triangle" << std::endl; - - ofs.close(); + int rank = 0; + MPI_Comm_rank(comm, &rank); + if(rank == 0) + { + EXPECT_FALSE(file.empty()); + + std::ofstream ofs(file.c_str()); + EXPECT_TRUE(ofs.is_open()); + + ofs << "solid triangle" << std::endl; + ofs << "\t facet normal 0.0 0.0 1.0" << std::endl; + ofs << "\t\t outer loop" << std::endl; + ofs << "\t\t\t vertex 0.0 0.0 0.0" << std::endl; + ofs << "\t\t\t vertex 1.0 0.0 0.0" << std::endl; + ofs << "\t\t\t vertex 0.0 1.0 0.0" << std::endl; + ofs << "\t\t endloop" << std::endl; + ofs << "\t endfacet" << std::endl; + ofs << "endsolid triangle" << std::endl; + + ofs.close(); + } + MPI_Barrier(comm); +} + +/*! + * \brief Remove a file using rank 0 of the supplied MPI communicator. + * + * \param fileName The name of the file to remove. + * \param comm The MPI communicator. + * + * \return 0 on success; non-zero otherwise. + */ +int removeFile(const std::string &fileName, MPI_Comm comm = MPI_COMM_WORLD) +{ + int rank = 0, retval = 0; + MPI_Comm_rank(comm, &rank); + MPI_Barrier(comm); + if(rank == 0) + { + retval = axom::utilities::filesystem::removeFile(fileName); + } + MPI_Bcast(&retval, 1, MPI_INT, 0, comm); + return retval; } /*! @@ -185,8 +220,9 @@ void getUniformMesh(const UnstructuredMesh* mesh, mint::UniformMesh*& umesh) /*! * \brief Tests the signed against an anlytic surface mesh input. * \param [in] use_shared indicates whether to use shared memory or not. + * \param [in] comm The MPI communicator to use. */ -void check_analytic_plane(bool use_shared = false) +void check_analytic_plane(bool use_shared = false, MPI_Comm comm = MPI_COMM_SELF) { // STEP 0: construct uniform box mesh constexpr int NDIMS = 3; @@ -199,7 +235,7 @@ void check_analytic_plane(bool use_shared = false) // STEP 1: generate planar STL mesh file const std::string file = "plane.stl"; - generate_planar_mesh_stl_file(file); + generate_planar_mesh_stl_file(file, comm); // STEP 2: define analytic plane corresponding to the planar mesh; const primal::Point origin {0.0, 0.0, 0.0}; @@ -209,7 +245,7 @@ void check_analytic_plane(bool use_shared = false) // STEP 2: initialize the signed distance quest::signed_distance_use_shared_memory(use_shared); quest::signed_distance_set_closed_surface(false); - quest::signed_distance_init(file); + quest::signed_distance_init(file, comm); EXPECT_TRUE(quest::signed_distance_initialized()); axom::IndexType nnodes = mesh.getNumberOfNodes(); @@ -236,7 +272,7 @@ void check_analytic_plane(bool use_shared = false) EXPECT_FALSE(quest::signed_distance_initialized()); #ifdef REMOVE_FILES - EXPECT_EQ(axom::utilities::filesystem::removeFile(file), 0); + EXPECT_EQ(removeFile(file, comm), 0); #endif } @@ -353,7 +389,7 @@ TEST(quest_signed_distance_interface, initialize) // generate a temp STL file const std::string fileName = "test_triangle.stl"; - generate_stl_file(fileName); + generate_stl_file(fileName, MPI_COMM_WORLD); // test valid initialization quest::signed_distance_set_dimension(3); @@ -367,7 +403,7 @@ TEST(quest_signed_distance_interface, initialize) // remove temp STL file #ifdef REMOVE_FILES - EXPECT_EQ(axom::utilities::filesystem::removeFile(fileName), 0); + EXPECT_EQ(removeFile(fileName, MPI_COMM_WORLD), 0); #endif } @@ -411,10 +447,19 @@ TEST(quest_signed_distance_interface, get_mesh_bounds) //------------------------------------------------------------------------------ TEST(quest_signed_distance_interface, analytic_plane) { - check_analytic_plane(); + // Serial test - Only call on 1 rank since the test creates/removes a file with + // a fixed name. + int rank = 0; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + if(rank == 0) + { + check_analytic_plane(); + } #if defined(AXOM_USE_UMPIRE_SHARED_MEMORY) - check_analytic_plane(USE_SHARED_MEMORY); + // We are using shared memory that we want to coordinate among ranks in + // MPI_COMM_WORLD so pass that communicator. + check_analytic_plane(USE_SHARED_MEMORY, MPI_COMM_WORLD); #endif } @@ -422,8 +467,10 @@ TEST(quest_signed_distance_interface, analytic_plane) #if defined(AXOM_USE_UMPIRE_SHARED_MEMORY) TEST(quest_signed_distance_interface, call_twice_using_shared_memory) { - check_analytic_plane(USE_SHARED_MEMORY); - check_analytic_plane(USE_SHARED_MEMORY); + // We are using shared memory that we want to coordinate among ranks in + // MPI_COMM_WORLD so pass that communicator. + check_analytic_plane(USE_SHARED_MEMORY, MPI_COMM_WORLD); + check_analytic_plane(USE_SHARED_MEMORY, MPI_COMM_WORLD); } #endif From 7d4245d9ec62b5f1111580c82ef9a196ea24a9dc Mon Sep 17 00:00:00 2001 From: Brad Whitlock Date: Fri, 5 Sep 2025 17:47:13 -0700 Subject: [PATCH 03/24] Fix issue in quest_signed_distance_interface test. Complete shared memory via Umpire, as far as I can tell. --- RELEASE-NOTES.md | 1 + .../quest/interface/internal/QuestHelpers.cpp | 79 +++++++++++++++---- src/axom/quest/interface/signed_distance.cpp | 28 +++---- .../tests/quest_signed_distance_interface.cpp | 20 +++-- 4 files changed, 88 insertions(+), 40 deletions(-) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 5d85f2f9f5..8e562410e1 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -77,6 +77,7 @@ The Axom project release numbers follow [Semantic Versioning](http://semver.org/ `NURBSCurve` and `NURBSPatch` classes and add overloads from `axom::ArrayView` - Core: Updates behavior of `FlatMap::reserve()` to only trigger a rehash if maximum load factor would be exceeded. +- Quest: The signed_distance functions were modified so they use Umpire's shared memory mechanisms instead of using MPI3 directly. ### Fixed - Core: prevent incorrect instantiations of `axom::Array` from a host-only compile, when Axom is compiled diff --git a/src/axom/quest/interface/internal/QuestHelpers.cpp b/src/axom/quest/interface/internal/QuestHelpers.cpp index ce50416f75..a1928181b7 100644 --- a/src/axom/quest/interface/internal/QuestHelpers.cpp +++ b/src/axom/quest/interface/internal/QuestHelpers.cpp @@ -39,6 +39,55 @@ namespace internal /// Mesh I/O methods #if defined(AXOM_USE_UMPIRE_SHARED_MEMORY) +/* + * Deallocates the specified MPI communicator object. + */ +static void mpi_comm_free(MPI_Comm* comm) +{ + if(*comm != MPI_COMM_NULL) + { + MPI_Comm_free(comm); + } +} + +/* + * Creates inter-node and intra-node communicators from the given global + * MPI communicator handle. + */ +static void create_communicators(MPI_Comm global_comm, + MPI_Comm& intra_node_comm, + MPI_Comm& inter_node_comm, + int& global_rank_id, + int& local_rank_id, + int& intercom_rank_id) +{ + // Sanity checks + SLIC_ASSERT(global_comm != MPI_COMM_NULL); + SLIC_ASSERT(intra_node_comm == MPI_COMM_NULL); + SLIC_ASSERT(inter_node_comm == MPI_COMM_NULL); + + constexpr int IGNORE_KEY = 0; + + // STEP 0: get global rank, used to order ranks in the inter-node comm. + MPI_Comm_rank(global_comm, &global_rank_id); + + // STEP 1: create the intra-node communicator + MPI_Comm_split_type(global_comm, MPI_COMM_TYPE_SHARED, IGNORE_KEY, MPI_INFO_NULL, &intra_node_comm); + MPI_Comm_rank(intra_node_comm, &local_rank_id); + SLIC_ASSERT(local_rank_id >= 0); + + // STEP 2: create inter-node communicator + const int color = (local_rank_id == 0) ? 1 : MPI_UNDEFINED; + MPI_Comm_split(global_comm, color, global_rank_id, &inter_node_comm); + + if(color == 1) + { + MPI_Comm_rank(inter_node_comm, &intercom_rank_id); + } + + SLIC_ASSERT(intra_node_comm != MPI_COMM_NULL); +} + /*! * \brief Allocates a shared memory buffer for the mesh that is shared among * all the ranks within the same compute node. @@ -172,23 +221,18 @@ int read_stl_mesh_shared(const std::string& file, return READ_FAILED; } - // STEP 1: Get the communicator from the Umpire allocator. + // STEP 1: Create communicators int global_rank_id = -1; + int local_rank_id = -1; int intercom_rank_id = -1; - MPI_Comm_rank(global_comm, &global_rank_id); + MPI_Comm intra_node_comm = MPI_COMM_NULL; MPI_Comm inter_node_comm = MPI_COMM_NULL; - umpire::ResourceManager& rm = umpire::ResourceManager::getInstance(); - if(rm.isAllocator(allocatorID)) - { - umpire::Allocator allocator = rm.getAllocator(allocatorID); - inter_node_comm = umpire::get_communicator_for_allocator(allocator, global_comm); - MPI_Comm_rank(inter_node_comm, &intercom_rank_id); -std::cout << "Rank " << global_rank_id << ": intercom_rank_id=" << intercom_rank_id << std::endl; - } - else - { - SLIC_ERROR(axom::fmt::format("An invalid allocatorID {} was provided.", allocatorID)); - } + create_communicators(global_comm, + intra_node_comm, + inter_node_comm, + global_rank_id, + local_rank_id, + intercom_rank_id); // STEP 2: Exchange mesh metadata constexpr int NUM_NODES = 0; @@ -200,6 +244,8 @@ std::cout << "Rank " << global_rank_id << ": intercom_rank_id=" << intercom_rank int rc = read_and_exchange_mesh_metadata(global_rank_id, global_comm, reader, mesh_metadata); if(rc != READ_SUCCESS) { + mpi_comm_free(&inter_node_comm); + mpi_comm_free(&intra_node_comm); return READ_FAILED; } @@ -236,7 +282,10 @@ std::cout << "Rank " << global_rank_id << ": intercom_rank_id=" << intercom_rank MPI_Bcast(mesh_buffer, numBytes, MPI_UNSIGNED_CHAR, 0, inter_node_comm); } - // NOTE: Communicators are associated with Umpire allocators and Umpire will free them. + // STEP 6 free communicators + MPI_Barrier(global_comm); + mpi_comm_free(&inter_node_comm); + mpi_comm_free(&intra_node_comm); return READ_SUCCESS; } diff --git a/src/axom/quest/interface/signed_distance.cpp b/src/axom/quest/interface/signed_distance.cpp index 3bacfe6224..d4336ccecc 100644 --- a/src/axom/quest/interface/signed_distance.cpp +++ b/src/axom/quest/interface/signed_distance.cpp @@ -136,21 +136,22 @@ int signed_distance_init(const std::string& file, MPI_Comm comm) // STEP 0: read the STL mesh int rc = INIT_FAILED; -#if defined(AXOM_USE_UMPIRE_SHARED_MEMORY) - if(s_allocator_id == INVALID_ALLOCATOR_ID) - { - // Make a shared memory allocator and get its id. - auto& rm = umpire::ResourceManager::getInstance(); - auto traits{umpire::get_default_resource_traits("SHARED")}; - traits.scope = umpire::MemoryResourceTraits::shared_scope::node; - auto node_allocator{rm.makeResource("SHARED::node_allocator", traits)}; - auto signed_distance_allocator{ - rm.makeAllocator("signed_distance_allocator", node_allocator)}; - s_allocator_id = signed_distance_allocator.getId(); - } - if(Parameters.use_shared_memory) { +#if defined(AXOM_USE_UMPIRE_SHARED_MEMORY) + if(s_allocator_id == INVALID_ALLOCATOR_ID) + { + // Make a shared memory allocator if we have not made it before. We'll reuse + // the allocator to allocate different buffers (1 at a time). + auto& rm = umpire::ResourceManager::getInstance(); + auto traits{umpire::get_default_resource_traits("SHARED")}; + traits.scope = umpire::MemoryResourceTraits::shared_scope::node; + auto node_allocator{rm.makeResource("SHARED::node_allocator", traits)}; + auto signed_distance_allocator{ + rm.makeAllocator("signed_distance_allocator", node_allocator)}; + s_allocator_id = signed_distance_allocator.getId(); + } + rc = internal::read_stl_mesh_shared(file, comm, s_allocator_id, @@ -161,7 +162,6 @@ int signed_distance_init(const std::string& file, MPI_Comm comm) { rc = internal::read_stl_mesh(file, s_surface_mesh, comm); } - #else SLIC_WARNING_IF(Parameters.use_shared_memory, diff --git a/src/axom/quest/tests/quest_signed_distance_interface.cpp b/src/axom/quest/tests/quest_signed_distance_interface.cpp index bdf73604dc..79773782cd 100644 --- a/src/axom/quest/tests/quest_signed_distance_interface.cpp +++ b/src/axom/quest/tests/quest_signed_distance_interface.cpp @@ -219,10 +219,11 @@ void getUniformMesh(const UnstructuredMesh* mesh, mint::UniformMesh*& umesh) /*! * \brief Tests the signed against an anlytic surface mesh input. + * \param [in] file The name of the STL file to write. * \param [in] use_shared indicates whether to use shared memory or not. * \param [in] comm The MPI communicator to use. */ -void check_analytic_plane(bool use_shared = false, MPI_Comm comm = MPI_COMM_SELF) +void check_analytic_plane(const std::string &file, bool use_shared = false, MPI_Comm comm = MPI_COMM_SELF) { // STEP 0: construct uniform box mesh constexpr int NDIMS = 3; @@ -234,7 +235,6 @@ void check_analytic_plane(bool use_shared = false, MPI_Comm comm = MPI_COMM_SELF double* err = mesh.createField("err", mint::NODE_CENTERED); // STEP 1: generate planar STL mesh file - const std::string file = "plane.stl"; generate_planar_mesh_stl_file(file, comm); // STEP 2: define analytic plane corresponding to the planar mesh; @@ -447,19 +447,17 @@ TEST(quest_signed_distance_interface, get_mesh_bounds) //------------------------------------------------------------------------------ TEST(quest_signed_distance_interface, analytic_plane) { - // Serial test - Only call on 1 rank since the test creates/removes a file with - // a fixed name. + // Serial test - called independently on each rank. In order to avoid problems + // writing/reading the STL file, we pass a unique name for each + // rank. int rank = 0; MPI_Comm_rank(MPI_COMM_WORLD, &rank); - if(rank == 0) - { - check_analytic_plane(); - } + check_analytic_plane(axom::fmt::format("plane{}.stl", rank)); #if defined(AXOM_USE_UMPIRE_SHARED_MEMORY) // We are using shared memory that we want to coordinate among ranks in // MPI_COMM_WORLD so pass that communicator. - check_analytic_plane(USE_SHARED_MEMORY, MPI_COMM_WORLD); + check_analytic_plane("plane.stl", USE_SHARED_MEMORY, MPI_COMM_WORLD); #endif } @@ -469,8 +467,8 @@ TEST(quest_signed_distance_interface, call_twice_using_shared_memory) { // We are using shared memory that we want to coordinate among ranks in // MPI_COMM_WORLD so pass that communicator. - check_analytic_plane(USE_SHARED_MEMORY, MPI_COMM_WORLD); - check_analytic_plane(USE_SHARED_MEMORY, MPI_COMM_WORLD); + check_analytic_plane("plane.stl", USE_SHARED_MEMORY, MPI_COMM_WORLD); + check_analytic_plane("plane.stl", USE_SHARED_MEMORY, MPI_COMM_WORLD); } #endif From 4c73b35dd8a82bbfabf2665df776499afa59e01d Mon Sep 17 00:00:00 2001 From: Brad Whitlock Date: Fri, 5 Sep 2025 17:47:49 -0700 Subject: [PATCH 04/24] Initial change to Axom spack package.py to select Umpire that has MPI3 shared memory support. --- scripts/spack/packages/axom/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/spack/packages/axom/package.py b/scripts/spack/packages/axom/package.py index fa5ac0469b..68e9f4cffa 100644 --- a/scripts/spack/packages/axom/package.py +++ b/scripts/spack/packages/axom/package.py @@ -146,6 +146,7 @@ class Axom(CachedCMakePackage, CudaPackage, ROCmPackage): depends_on("umpire@6.0.0", when="@0.6.0") depends_on("umpire@5:5.0.1", when="@:0.5.0") depends_on("umpire+openmp", when="+openmp") + depends_on("umpire+mpi3_shmem", when="+mpi") with when("+raja"): depends_on("raja") From 55d8f1c7549813074bbc430032ca0a9d8a915130 Mon Sep 17 00:00:00 2001 From: Brad Whitlock Date: Fri, 5 Sep 2025 18:13:59 -0700 Subject: [PATCH 05/24] make style --- .../quest/interface/internal/QuestHelpers.cpp | 34 ++++++++----------- .../quest/interface/internal/QuestHelpers.hpp | 3 +- src/axom/quest/interface/signed_distance.cpp | 25 ++++++-------- .../tests/quest_signed_distance_interface.cpp | 11 +++--- 4 files changed, 35 insertions(+), 38 deletions(-) diff --git a/src/axom/quest/interface/internal/QuestHelpers.cpp b/src/axom/quest/interface/internal/QuestHelpers.cpp index a1928181b7..270589defb 100644 --- a/src/axom/quest/interface/internal/QuestHelpers.cpp +++ b/src/axom/quest/interface/internal/QuestHelpers.cpp @@ -23,7 +23,8 @@ #endif #endif -#if defined(AXOM_USE_MPI) && defined(AXOM_USE_UMPIRE) && (defined(UMPIRE_ENABLE_IPC_SHARED_MEMORY) || defined(UMPIRE_ENABLE_MPI3_SHARED_MEMORY)) +#if defined(AXOM_USE_MPI) && defined(AXOM_USE_UMPIRE) && \ + (defined(UMPIRE_ENABLE_IPC_SHARED_MEMORY) || defined(UMPIRE_ENABLE_MPI3_SHARED_MEMORY)) #include "umpire/Umpire.hpp" #define AXOM_USE_UMPIRE_SHARED_MEMORY #endif @@ -55,11 +56,11 @@ static void mpi_comm_free(MPI_Comm* comm) * MPI communicator handle. */ static void create_communicators(MPI_Comm global_comm, - MPI_Comm& intra_node_comm, - MPI_Comm& inter_node_comm, - int& global_rank_id, - int& local_rank_id, - int& intercom_rank_id) + MPI_Comm& intra_node_comm, + MPI_Comm& inter_node_comm, + int& global_rank_id, + int& local_rank_id, + int& intercom_rank_id) { // Sanity checks SLIC_ASSERT(global_comm != MPI_COMM_NULL); @@ -117,12 +118,12 @@ static void create_communicators(MPI_Comm global_comm, * \post mesh_buffer != nullptr */ static size_t allocate_shared_buffer(int allocatorID, - const axom::IndexType mesh_metadata[2], - double*& x, - double*& y, - double*& z, - axom::IndexType*& conn, - unsigned char*& mesh_buffer) + const axom::IndexType mesh_metadata[2], + double*& x, + double*& y, + double*& z, + axom::IndexType*& conn, + unsigned char*& mesh_buffer) { // Allocate the buffer. const axom::IndexType nnodes = mesh_metadata[0]; @@ -254,13 +255,8 @@ int read_stl_mesh_shared(const std::string& file, double* y = nullptr; double* z = nullptr; axom::IndexType* conn = nullptr; - const size_t numBytes = allocate_shared_buffer(allocatorID, - mesh_metadata, - x, - y, - z, - conn, - mesh_buffer); + const size_t numBytes = + allocate_shared_buffer(allocatorID, mesh_metadata, x, y, z, conn, mesh_buffer); SLIC_ASSERT(x != nullptr); SLIC_ASSERT(y != nullptr); SLIC_ASSERT(z != nullptr); diff --git a/src/axom/quest/interface/internal/QuestHelpers.hpp b/src/axom/quest/interface/internal/QuestHelpers.hpp index d350e875c1..1dcc9d805f 100644 --- a/src/axom/quest/interface/internal/QuestHelpers.hpp +++ b/src/axom/quest/interface/internal/QuestHelpers.hpp @@ -63,7 +63,8 @@ class ScopedLogLevelChanger /// \name Mesh I/O methods /// @{ -#if defined(AXOM_USE_MPI) && defined(AXOM_USE_UMPIRE) && (defined(UMPIRE_ENABLE_IPC_SHARED_MEMORY) || defined(UMPIRE_ENABLE_MPI3_SHARED_MEMORY)) +#if defined(AXOM_USE_MPI) && defined(AXOM_USE_UMPIRE) && \ + (defined(UMPIRE_ENABLE_IPC_SHARED_MEMORY) || defined(UMPIRE_ENABLE_MPI3_SHARED_MEMORY)) /*! * \brief Reads in the surface mesh from the specified file into a shared diff --git a/src/axom/quest/interface/signed_distance.cpp b/src/axom/quest/interface/signed_distance.cpp index d4336ccecc..3dfe039ba0 100644 --- a/src/axom/quest/interface/signed_distance.cpp +++ b/src/axom/quest/interface/signed_distance.cpp @@ -109,8 +109,8 @@ static int s_allocator_id = INVALID_ALLOCATOR_ID; static unsigned char* s_shared_mesh_buffer = nullptr; #else static std::string s_shared_memory_requirements( - "Shared memory requires MPI and an Umpire library built with " - "UMPIRE_ENABLE_IPC_SHARED_MEMORY set to ON"); + "Shared memory requires MPI and an Umpire library built with " + "UMPIRE_ENABLE_IPC_SHARED_MEMORY set to ON"); #endif } // end anonymous namespace @@ -144,19 +144,17 @@ int signed_distance_init(const std::string& file, MPI_Comm comm) // Make a shared memory allocator if we have not made it before. We'll reuse // the allocator to allocate different buffers (1 at a time). auto& rm = umpire::ResourceManager::getInstance(); - auto traits{umpire::get_default_resource_traits("SHARED")}; + auto traits {umpire::get_default_resource_traits("SHARED")}; traits.scope = umpire::MemoryResourceTraits::shared_scope::node; - auto node_allocator{rm.makeResource("SHARED::node_allocator", traits)}; - auto signed_distance_allocator{ - rm.makeAllocator("signed_distance_allocator", node_allocator)}; + auto node_allocator {rm.makeResource("SHARED::node_allocator", traits)}; + auto signed_distance_allocator { + rm.makeAllocator("signed_distance_allocator", + node_allocator)}; s_allocator_id = signed_distance_allocator.getId(); } - rc = internal::read_stl_mesh_shared(file, - comm, - s_allocator_id, - s_shared_mesh_buffer, - s_surface_mesh); + rc = + internal::read_stl_mesh_shared(file, comm, s_allocator_id, s_shared_mesh_buffer, s_surface_mesh); } else { @@ -164,10 +162,9 @@ int signed_distance_init(const std::string& file, MPI_Comm comm) } #else - SLIC_WARNING_IF(Parameters.use_shared_memory, - s_shared_memory_requirements_message); + SLIC_WARNING_IF(Parameters.use_shared_memory, s_shared_memory_requirements_message); - rc = internal::read_stl_mesh(file, s_surface_mesh, comm); + rc = internal::read_stl_mesh(file, s_surface_mesh, comm); #endif if(rc != 0) diff --git a/src/axom/quest/tests/quest_signed_distance_interface.cpp b/src/axom/quest/tests/quest_signed_distance_interface.cpp index 79773782cd..0189144148 100644 --- a/src/axom/quest/tests/quest_signed_distance_interface.cpp +++ b/src/axom/quest/tests/quest_signed_distance_interface.cpp @@ -46,8 +46,9 @@ using UnstructuredMesh = mint::UnstructuredMesh; //------------------------------------------------------------------------------ namespace { -#if defined(AXOM_USE_MPI) && defined(AXOM_USE_UMPIRE) && (defined(UMPIRE_ENABLE_IPC_SHARED_MEMORY) || defined(UMPIRE_ENABLE_MPI3_SHARED_MEMORY)) -#define AXOM_USE_UMPIRE_SHARED_MEMORY +#if defined(AXOM_USE_MPI) && defined(AXOM_USE_UMPIRE) && \ + (defined(UMPIRE_ENABLE_IPC_SHARED_MEMORY) || defined(UMPIRE_ENABLE_MPI3_SHARED_MEMORY)) + #define AXOM_USE_UMPIRE_SHARED_MEMORY constexpr bool USE_SHARED_MEMORY = true; #else constexpr bool USE_SHARED_MEMORY = false; @@ -160,7 +161,7 @@ void generate_stl_file(const std::string& file, MPI_Comm comm = MPI_COMM_SELF) * * \return 0 on success; non-zero otherwise. */ -int removeFile(const std::string &fileName, MPI_Comm comm = MPI_COMM_WORLD) +int removeFile(const std::string& fileName, MPI_Comm comm = MPI_COMM_WORLD) { int rank = 0, retval = 0; MPI_Comm_rank(comm, &rank); @@ -223,7 +224,9 @@ void getUniformMesh(const UnstructuredMesh* mesh, mint::UniformMesh*& umesh) * \param [in] use_shared indicates whether to use shared memory or not. * \param [in] comm The MPI communicator to use. */ -void check_analytic_plane(const std::string &file, bool use_shared = false, MPI_Comm comm = MPI_COMM_SELF) +void check_analytic_plane(const std::string& file, + bool use_shared = false, + MPI_Comm comm = MPI_COMM_SELF) { // STEP 0: construct uniform box mesh constexpr int NDIMS = 3; From ebab4a6d9816f107d349e6756b5d847bb1dd256d Mon Sep 17 00:00:00 2001 From: Brad Whitlock Date: Fri, 5 Sep 2025 18:19:53 -0700 Subject: [PATCH 06/24] Fix variable name use. --- src/axom/quest/interface/signed_distance.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/axom/quest/interface/signed_distance.cpp b/src/axom/quest/interface/signed_distance.cpp index 3dfe039ba0..77e70bfaff 100644 --- a/src/axom/quest/interface/signed_distance.cpp +++ b/src/axom/quest/interface/signed_distance.cpp @@ -162,7 +162,7 @@ int signed_distance_init(const std::string& file, MPI_Comm comm) } #else - SLIC_WARNING_IF(Parameters.use_shared_memory, s_shared_memory_requirements_message); + SLIC_WARNING_IF(Parameters.use_shared_memory, s_shared_memory_requirements); rc = internal::read_stl_mesh(file, s_surface_mesh, comm); #endif From 6adb053123ac4c80a579c0185cb417e2c1f626d9 Mon Sep 17 00:00:00 2001 From: Brad Whitlock Date: Fri, 5 Sep 2025 18:20:45 -0700 Subject: [PATCH 07/24] Change message about Umpire. --- src/axom/quest/interface/signed_distance.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/axom/quest/interface/signed_distance.cpp b/src/axom/quest/interface/signed_distance.cpp index 77e70bfaff..ce8fa7291a 100644 --- a/src/axom/quest/interface/signed_distance.cpp +++ b/src/axom/quest/interface/signed_distance.cpp @@ -109,8 +109,7 @@ static int s_allocator_id = INVALID_ALLOCATOR_ID; static unsigned char* s_shared_mesh_buffer = nullptr; #else static std::string s_shared_memory_requirements( - "Shared memory requires MPI and an Umpire library built with " - "UMPIRE_ENABLE_IPC_SHARED_MEMORY set to ON"); + "Shared memory requires MPI and an Umpire library built with shared memory support"); #endif } // end anonymous namespace From 4684659513bcd728e0026bfff4e0db50e05999b7 Mon Sep 17 00:00:00 2001 From: Brad Whitlock Date: Mon, 8 Sep 2025 11:07:30 -0700 Subject: [PATCH 08/24] Fix old typo --- src/axom/quest/interface/internal/QuestHelpers.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/axom/quest/interface/internal/QuestHelpers.cpp b/src/axom/quest/interface/internal/QuestHelpers.cpp index 270589defb..bc7f0665dc 100644 --- a/src/axom/quest/interface/internal/QuestHelpers.cpp +++ b/src/axom/quest/interface/internal/QuestHelpers.cpp @@ -94,7 +94,7 @@ static void create_communicators(MPI_Comm global_comm, * all the ranks within the same compute node. * * \param [in] allocatorID A valid Umpire allocator id that can allocate shared memory. - * \param [in] mesh_metada tuple with the number of nodes/faces on the mesh + * \param [in] mesh_metadata tuple with the number of nodes/faces on the mesh * \param [out] x pointer into the buffer where the x--coordinates are stored. * \param [out] y pointer into the buffer where the y--coordinates are stored. * \param [out] z pointer into the buffer where the z--coordinates are stored. From cb19673e3cd2e13aee1fb9a5d0483c01a1a5bb6c Mon Sep 17 00:00:00 2001 From: Brad Whitlock Date: Mon, 8 Sep 2025 11:07:48 -0700 Subject: [PATCH 09/24] Fix conditional compilation block. --- src/axom/quest/interface/signed_distance.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/axom/quest/interface/signed_distance.cpp b/src/axom/quest/interface/signed_distance.cpp index ce8fa7291a..e9c06884e5 100644 --- a/src/axom/quest/interface/signed_distance.cpp +++ b/src/axom/quest/interface/signed_distance.cpp @@ -135,9 +135,9 @@ int signed_distance_init(const std::string& file, MPI_Comm comm) // STEP 0: read the STL mesh int rc = INIT_FAILED; +#if defined(AXOM_USE_UMPIRE_SHARED_MEMORY) if(Parameters.use_shared_memory) { -#if defined(AXOM_USE_UMPIRE_SHARED_MEMORY) if(s_allocator_id == INVALID_ALLOCATOR_ID) { // Make a shared memory allocator if we have not made it before. We'll reuse @@ -160,10 +160,9 @@ int signed_distance_init(const std::string& file, MPI_Comm comm) rc = internal::read_stl_mesh(file, s_surface_mesh, comm); } #else + SLIC_WARNING_IF(Parameters.use_shared_memory, s_shared_memory_requirements); - SLIC_WARNING_IF(Parameters.use_shared_memory, s_shared_memory_requirements); - - rc = internal::read_stl_mesh(file, s_surface_mesh, comm); + rc = internal::read_stl_mesh(file, s_surface_mesh, comm); #endif if(rc != 0) From 73ea86a262acc1adbdce424019d7feab71748e76 Mon Sep 17 00:00:00 2001 From: Brad Whitlock Date: Mon, 8 Sep 2025 11:09:08 -0700 Subject: [PATCH 10/24] Removed extra dashes. --- src/axom/quest/interface/internal/QuestHelpers.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/axom/quest/interface/internal/QuestHelpers.cpp b/src/axom/quest/interface/internal/QuestHelpers.cpp index bc7f0665dc..0851894e19 100644 --- a/src/axom/quest/interface/internal/QuestHelpers.cpp +++ b/src/axom/quest/interface/internal/QuestHelpers.cpp @@ -95,9 +95,9 @@ static void create_communicators(MPI_Comm global_comm, * * \param [in] allocatorID A valid Umpire allocator id that can allocate shared memory. * \param [in] mesh_metadata tuple with the number of nodes/faces on the mesh - * \param [out] x pointer into the buffer where the x--coordinates are stored. - * \param [out] y pointer into the buffer where the y--coordinates are stored. - * \param [out] z pointer into the buffer where the z--coordinates are stored. + * \param [out] x pointer into the buffer where the x-coordinates are stored. + * \param [out] y pointer into the buffer where the y-coordinates are stored. + * \param [out] z pointer into the buffer where the z-coordinates are stored. * \param [out] conn pointer into the buffer consisting the cell-connectivity. * \param [out] mesh_buffer raw buffer consisting of all the mesh data. * From f12ff686bae8f27cbb2b93f3483b53e05f3695a2 Mon Sep 17 00:00:00 2001 From: Brad Whitlock Date: Mon, 8 Sep 2025 13:00:41 -0700 Subject: [PATCH 11/24] Avoid direct MPI calls for case when this MPI test is actually NOT used with MPI. --- .../tests/quest_signed_distance_interface.cpp | 48 ++++++++++++++----- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/src/axom/quest/tests/quest_signed_distance_interface.cpp b/src/axom/quest/tests/quest_signed_distance_interface.cpp index 0189144148..734de56178 100644 --- a/src/axom/quest/tests/quest_signed_distance_interface.cpp +++ b/src/axom/quest/tests/quest_signed_distance_interface.cpp @@ -54,6 +54,33 @@ constexpr bool USE_SHARED_MEMORY = true; constexpr bool USE_SHARED_MEMORY = false; #endif +// NOTE: This test can actually be built in serial so we provide parallel and +// serial versions of these functions. +#ifdef AXOM_USE_UMPIRE_SHARED_MEMORY +// Parallel versions +int comm_rank(MPI_Comm comm) +{ + int rank = 0; + MPI_Comm_rank(comm, &rank); + return rank; +} +void barrier(MPI_Comm comm) { MPI_Barrier(comm); } +void bcast_int(int& value, MPI_Comm comm) { MPI_Bcast(&value, 1, MPI_INT, 0, comm); } +#else +// Serial versions +constexpr int MPI_COMM_WORLD = -1; + +int comm_rank(MPI_Comm comm) { return 0; } +void barrier(MPI_Comm) +{ + // no-op +} +void bcast_int(int&) +{ + // no-op +} +#endif + /*! * \brief Generate a mesh of 4 triangles along the XY plane. * @@ -65,8 +92,7 @@ constexpr bool USE_SHARED_MEMORY = false; */ void generate_planar_mesh_stl_file(const std::string& file, MPI_Comm comm = MPI_COMM_SELF) { - int rank = 0; - MPI_Comm_rank(comm, &rank); + int rank = comm_rank(comm); if(rank == 0) { EXPECT_FALSE(file.empty()); @@ -115,7 +141,7 @@ void generate_planar_mesh_stl_file(const std::string& file, MPI_Comm comm = MPI_ ofs << "endsolid" << std::endl; ofs.close(); } - MPI_Barrier(comm); + barrier(comm); } /*! @@ -129,8 +155,7 @@ void generate_planar_mesh_stl_file(const std::string& file, MPI_Comm comm = MPI_ */ void generate_stl_file(const std::string& file, MPI_Comm comm = MPI_COMM_SELF) { - int rank = 0; - MPI_Comm_rank(comm, &rank); + int rank = comm_rank(comm); if(rank == 0) { EXPECT_FALSE(file.empty()); @@ -150,7 +175,7 @@ void generate_stl_file(const std::string& file, MPI_Comm comm = MPI_COMM_SELF) ofs.close(); } - MPI_Barrier(comm); + barrier(comm); } /*! @@ -163,14 +188,14 @@ void generate_stl_file(const std::string& file, MPI_Comm comm = MPI_COMM_SELF) */ int removeFile(const std::string& fileName, MPI_Comm comm = MPI_COMM_WORLD) { - int rank = 0, retval = 0; - MPI_Comm_rank(comm, &rank); - MPI_Barrier(comm); + int retval = 0; + int rank = comm_rank(comm); + barrier(comm); if(rank == 0) { retval = axom::utilities::filesystem::removeFile(fileName); } - MPI_Bcast(&retval, 1, MPI_INT, 0, comm); + bcast_int(retval, comm); return retval; } @@ -453,8 +478,7 @@ TEST(quest_signed_distance_interface, analytic_plane) // Serial test - called independently on each rank. In order to avoid problems // writing/reading the STL file, we pass a unique name for each // rank. - int rank = 0; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); + int rank = comm_rank(MPI_COMM_WORLD); check_analytic_plane(axom::fmt::format("plane{}.stl", rank)); #if defined(AXOM_USE_UMPIRE_SHARED_MEMORY) From 88fa4504201492c2f9042068fed20591f61a4212 Mon Sep 17 00:00:00 2001 From: Brad Whitlock Date: Mon, 8 Sep 2025 13:03:33 -0700 Subject: [PATCH 12/24] Fix mistake --- src/axom/quest/tests/quest_signed_distance_interface.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/axom/quest/tests/quest_signed_distance_interface.cpp b/src/axom/quest/tests/quest_signed_distance_interface.cpp index 734de56178..68e902e7cd 100644 --- a/src/axom/quest/tests/quest_signed_distance_interface.cpp +++ b/src/axom/quest/tests/quest_signed_distance_interface.cpp @@ -75,7 +75,7 @@ void barrier(MPI_Comm) { // no-op } -void bcast_int(int&) +void bcast_int(int&, MPI_Comm) { // no-op } From 5aa1325f46829784c59ee6c0ca35db51d184d178 Mon Sep 17 00:00:00 2001 From: Brad Whitlock Date: Mon, 8 Sep 2025 14:19:29 -0700 Subject: [PATCH 13/24] Change macro used to select MPI abstraction functions. --- src/axom/quest/tests/quest_signed_distance_interface.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/axom/quest/tests/quest_signed_distance_interface.cpp b/src/axom/quest/tests/quest_signed_distance_interface.cpp index 68e902e7cd..f6d694071f 100644 --- a/src/axom/quest/tests/quest_signed_distance_interface.cpp +++ b/src/axom/quest/tests/quest_signed_distance_interface.cpp @@ -56,7 +56,7 @@ constexpr bool USE_SHARED_MEMORY = false; // NOTE: This test can actually be built in serial so we provide parallel and // serial versions of these functions. -#ifdef AXOM_USE_UMPIRE_SHARED_MEMORY +#if defined(AXOM_USE_MPI) // Parallel versions int comm_rank(MPI_Comm comm) { From f1149865ec60ddc83c3c1e54e96742e6f385b22a Mon Sep 17 00:00:00 2001 From: Brad Whitlock Date: Mon, 8 Sep 2025 16:47:12 -0700 Subject: [PATCH 14/24] Moved a function so it is more like the previous ordering. --- .../quest/interface/internal/QuestHelpers.cpp | 106 +++++++++--------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/src/axom/quest/interface/internal/QuestHelpers.cpp b/src/axom/quest/interface/internal/QuestHelpers.cpp index 0851894e19..aa7fef7def 100644 --- a/src/axom/quest/interface/internal/QuestHelpers.cpp +++ b/src/axom/quest/interface/internal/QuestHelpers.cpp @@ -40,8 +40,8 @@ namespace internal /// Mesh I/O methods #if defined(AXOM_USE_UMPIRE_SHARED_MEMORY) -/* - * Deallocates the specified MPI communicator object. +/*! + * \brief Deallocates the specified MPI communicator object. */ static void mpi_comm_free(MPI_Comm* comm) { @@ -51,9 +51,56 @@ static void mpi_comm_free(MPI_Comm* comm) } } -/* - * Creates inter-node and intra-node communicators from the given global - * MPI communicator handle. +/*! + * \brief Reads the mesh on rank 0 and exchanges the mesh metadata, i.e., the + * number of nodes and faces with all other ranks. + * + * \param [in] global_rank_id MPI rank w.r.t. the global communicator + * \param [in] global_comm handle to the global communicator + * \param [in,out] reader the corresponding STL reader + * \param [out] mesh_metadata an array consisting of the mesh metadata. + * + * \note This method calls read() on the reader on rank 0. + * + * \pre global_comm != MPI_COMM_NULL + * \pre mesh_metadata != nullptr + */ +static int read_and_exchange_mesh_metadata(int global_rank_id, + MPI_Comm global_comm, + quest::STLReader& reader, + axom::IndexType mesh_metadata[2]) +{ + constexpr int NUM_NODES = 0; + constexpr int NUM_FACES = 1; + constexpr int ROOT_RANK = 0; + + switch(global_rank_id) + { + case 0: + if(reader.read() == READ_SUCCESS) + { + mesh_metadata[NUM_NODES] = reader.getNumNodes(); + mesh_metadata[NUM_FACES] = reader.getNumFaces(); + } + else + { + SLIC_WARNING("reading STL file failed, setting mesh to NULL"); + mesh_metadata[NUM_NODES] = READ_FAILED; + mesh_metadata[NUM_FACES] = READ_FAILED; + } + MPI_Bcast(mesh_metadata, 2, axom::mpi_traits::type, ROOT_RANK, global_comm); + break; + default: + MPI_Bcast(mesh_metadata, 2, axom::mpi_traits::type, ROOT_RANK, global_comm); + } + + int rc = (mesh_metadata[NUM_NODES] == READ_FAILED) ? READ_FAILED : READ_SUCCESS; + return rc; +} + +/*! + * \brief Creates inter-node and intra-node communicators from the given global + * MPI communicator handle. */ static void create_communicators(MPI_Comm global_comm, MPI_Comm& intra_node_comm, @@ -146,53 +193,6 @@ static size_t allocate_shared_buffer(int allocatorID, return (bytesize); } -/*! - * \brief Reads the mesh on rank 0 and exchanges the mesh metadata, i.e., the - * number of nodes and faces with all other ranks. - * - * \param [in] global_rank_id MPI rank w.r.t. the global communicator - * \param [in] global_comm handle to the global communicator - * \param [in,out] reader the corresponding STL reader - * \param [out] mesh_metadata an array consisting of the mesh metadata. - * - * \note This method calls read() on the reader on rank 0. - * - * \pre global_comm != MPI_COMM_NULL - * \pre mesh_metadata != nullptr - */ -static int read_and_exchange_mesh_metadata(int global_rank_id, - MPI_Comm global_comm, - quest::STLReader& reader, - axom::IndexType mesh_metadata[2]) -{ - constexpr int NUM_NODES = 0; - constexpr int NUM_FACES = 1; - constexpr int ROOT_RANK = 0; - - switch(global_rank_id) - { - case 0: - if(reader.read() == READ_SUCCESS) - { - mesh_metadata[NUM_NODES] = reader.getNumNodes(); - mesh_metadata[NUM_FACES] = reader.getNumFaces(); - } - else - { - SLIC_WARNING("reading STL file failed, setting mesh to NULL"); - mesh_metadata[NUM_NODES] = READ_FAILED; - mesh_metadata[NUM_FACES] = READ_FAILED; - } - MPI_Bcast(mesh_metadata, 2, axom::mpi_traits::type, ROOT_RANK, global_comm); - break; - default: - MPI_Bcast(mesh_metadata, 2, axom::mpi_traits::type, ROOT_RANK, global_comm); - } - - int rc = (mesh_metadata[NUM_NODES] == READ_FAILED) ? READ_FAILED : READ_SUCCESS; - return rc; -} - /* * Reads in the surface mesh from the specified file into a shared * memory buffer that is attached to the given MPI shared window. @@ -222,7 +222,7 @@ int read_stl_mesh_shared(const std::string& file, return READ_FAILED; } - // STEP 1: Create communicators + // STEP 1: Create intra-node and inter-node MPI communicators int global_rank_id = -1; int local_rank_id = -1; int intercom_rank_id = -1; From 0ddce42421fb52199f703edc0928fa383d9258c0 Mon Sep 17 00:00:00 2001 From: Brad Whitlock Date: Tue, 9 Sep 2025 12:31:00 -0700 Subject: [PATCH 15/24] Make AXOM_USE_UMPIRE_SHARED_MEMORY be set in config.hpp --- RELEASE-NOTES.md | 2 ++ src/axom/config.hpp.in | 6 +++++- .../quest/interface/internal/QuestHelpers.cpp | 6 ------ .../quest/interface/internal/QuestHelpers.hpp | 5 ++--- src/axom/quest/interface/signed_distance.cpp | 9 +++------ .../tests/quest_signed_distance_interface.cpp | 4 +--- .../thirdparty/SetupAxomThirdParty.cmake | 19 +++++++++++++++++++ 7 files changed, 32 insertions(+), 19 deletions(-) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 8e562410e1..512a005928 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -78,6 +78,8 @@ The Axom project release numbers follow [Semantic Versioning](http://semver.org/ - Core: Updates behavior of `FlatMap::reserve()` to only trigger a rehash if maximum load factor would be exceeded. - Quest: The signed_distance functions were modified so they use Umpire's shared memory mechanisms instead of using MPI3 directly. +- Axom's `AXOM_USE_MPI3` CMake build option and corresponding macro definition were removed. +- When Umpire is present, Axom now detects whether it supports shared memory and defines the `AXOM_USE_UMPIRE_SHARED_MEMORY` macro if appropriate. This macro can be used to conditionally compile code involving shared memory via Umpire. ### Fixed - Core: prevent incorrect instantiations of `axom::Array` from a host-only compile, when Axom is compiled diff --git a/src/axom/config.hpp.in b/src/axom/config.hpp.in index 5ea85a6f2c..83449df43a 100644 --- a/src/axom/config.hpp.in +++ b/src/axom/config.hpp.in @@ -54,7 +54,6 @@ #cmakedefine AXOM_USE_CUDA #cmakedefine AXOM_USE_HIP #cmakedefine AXOM_USE_MPI -#cmakedefine AXOM_USE_MPI3 #cmakedefine AXOM_USE_MPIF_HEADER #cmakedefine AXOM_USE_OPENMP @@ -77,6 +76,11 @@ #cmakedefine AXOM_USE_SPARSEHASH #cmakedefine AXOM_USE_UMPIRE +/* + * Compiler defines for library capabilities + */ +#cmakedefine AXOM_USE_UMPIRE_SHARED_MEMORY + /* * Compiler defines for third-party executables */ diff --git a/src/axom/quest/interface/internal/QuestHelpers.cpp b/src/axom/quest/interface/internal/QuestHelpers.cpp index aa7fef7def..d49a3238be 100644 --- a/src/axom/quest/interface/internal/QuestHelpers.cpp +++ b/src/axom/quest/interface/internal/QuestHelpers.cpp @@ -23,12 +23,6 @@ #endif #endif -#if defined(AXOM_USE_MPI) && defined(AXOM_USE_UMPIRE) && \ - (defined(UMPIRE_ENABLE_IPC_SHARED_MEMORY) || defined(UMPIRE_ENABLE_MPI3_SHARED_MEMORY)) - #include "umpire/Umpire.hpp" - #define AXOM_USE_UMPIRE_SHARED_MEMORY -#endif - #include namespace axom diff --git a/src/axom/quest/interface/internal/QuestHelpers.hpp b/src/axom/quest/interface/internal/QuestHelpers.hpp index 1dcc9d805f..a318b2ca73 100644 --- a/src/axom/quest/interface/internal/QuestHelpers.hpp +++ b/src/axom/quest/interface/internal/QuestHelpers.hpp @@ -63,12 +63,11 @@ class ScopedLogLevelChanger /// \name Mesh I/O methods /// @{ -#if defined(AXOM_USE_MPI) && defined(AXOM_USE_UMPIRE) && \ - (defined(UMPIRE_ENABLE_IPC_SHARED_MEMORY) || defined(UMPIRE_ENABLE_MPI3_SHARED_MEMORY)) +#if defined(AXOM_USE_UMPIRE_SHARED_MEMORY) /*! * \brief Reads in the surface mesh from the specified file into a shared - * memory buffer that is attached to the given MPI shared window. + * memory buffer. * * \param [in] file the file consisting of the surface mesh * \param [in] global_comm handle to the global MPI communicator diff --git a/src/axom/quest/interface/signed_distance.cpp b/src/axom/quest/interface/signed_distance.cpp index e9c06884e5..e63e9dca03 100644 --- a/src/axom/quest/interface/signed_distance.cpp +++ b/src/axom/quest/interface/signed_distance.cpp @@ -13,13 +13,10 @@ #include "axom/slic/interface/slic.hpp" -#if defined(AXOM_USE_MPI) && defined(AXOM_USE_UMPIRE) +#if defined(AXOM_USE_UMPIRE_SHARED_MEMORY) #include "umpire/Umpire.hpp" - #if defined(UMPIRE_ENABLE_IPC_SHARED_MEMORY) || defined(UMPIRE_ENABLE_MPI3_SHARED_MEMORY) - #include "umpire/strategy/NamedAllocationStrategy.hpp" - #include "umpire/util/MemoryResourceTraits.hpp" - #define AXOM_USE_UMPIRE_SHARED_MEMORY - #endif + #include "umpire/strategy/NamedAllocationStrategy.hpp" + #include "umpire/util/MemoryResourceTraits.hpp" #endif #ifdef AXOM_USE_MPI diff --git a/src/axom/quest/tests/quest_signed_distance_interface.cpp b/src/axom/quest/tests/quest_signed_distance_interface.cpp index f6d694071f..afd4bc8e5a 100644 --- a/src/axom/quest/tests/quest_signed_distance_interface.cpp +++ b/src/axom/quest/tests/quest_signed_distance_interface.cpp @@ -46,9 +46,7 @@ using UnstructuredMesh = mint::UnstructuredMesh; //------------------------------------------------------------------------------ namespace { -#if defined(AXOM_USE_MPI) && defined(AXOM_USE_UMPIRE) && \ - (defined(UMPIRE_ENABLE_IPC_SHARED_MEMORY) || defined(UMPIRE_ENABLE_MPI3_SHARED_MEMORY)) - #define AXOM_USE_UMPIRE_SHARED_MEMORY +#if defined(AXOM_USE_UMPIRE_SHARED_MEMORY) constexpr bool USE_SHARED_MEMORY = true; #else constexpr bool USE_SHARED_MEMORY = false; diff --git a/src/cmake/thirdparty/SetupAxomThirdParty.cmake b/src/cmake/thirdparty/SetupAxomThirdParty.cmake index 97a13e77a9..78d177c519 100644 --- a/src/cmake/thirdparty/SetupAxomThirdParty.cmake +++ b/src/cmake/thirdparty/SetupAxomThirdParty.cmake @@ -57,6 +57,25 @@ if (UMPIRE_DIR) set(UMPIRE_FOUND TRUE) blt_convert_to_system_includes(TARGET umpire) + + # Check whether the Umpire defines symbols for shared memory + blt_check_code_compiles(CODE_COMPILES UMPIRE_SHARED_MEMORY + VERBOSE_OUTPUT OFF + DEPENDS_ON umpire + SOURCE_STRING " + #include + #if defined(UMPIRE_ENABLE_IPC_SHARED_MEMORY) || defined(UMPIRE_ENABLE_MPI3_SHARED_MEMORY) + int main() { return 0; } + #else + #error Macros not defined + #endif + ") + if (AXOM_ENABLE_MPI AND UMPIRE_SHARED_MEMORY) + set(AXOM_USE_UMPIRE_SHARED_MEMORY TRUE) + else() + set(AXOM_USE_UMPIRE_SHARED_MEMORY FALSE) + endif() + message(STATUS "Umpire supports shared memory: ${AXOM_USE_UMPIRE_SHARED_MEMORY}") else() message(STATUS "Umpire support is OFF") set(UMPIRE_FOUND FALSE) From 40c11f110776a2912994988f89acd8b4f15f1deb Mon Sep 17 00:00:00 2001 From: Brad Whitlock Date: Tue, 9 Sep 2025 12:33:32 -0700 Subject: [PATCH 16/24] Removed MPI build option. --- src/cmake/AxomOptions.cmake | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/cmake/AxomOptions.cmake b/src/cmake/AxomOptions.cmake index 754e473df3..12a0683595 100644 --- a/src/cmake/AxomOptions.cmake +++ b/src/cmake/AxomOptions.cmake @@ -44,9 +44,6 @@ cmake_dependent_option(AXOM_ENABLE_DOCS "Enables Axom Docs" ON "ENABLE_DOCS" OFF cmake_dependent_option(AXOM_ENABLE_EXAMPLES "Enables Axom Examples" ON "ENABLE_EXAMPLES" OFF) option(AXOM_ENABLE_TOOLS "Enables Axom Tools" ON) -cmake_dependent_option(AXOM_ENABLE_MPI3 "Enables use of MPI-3 features" OFF "ENABLE_MPI" OFF) -mark_as_advanced(AXOM_ENABLE_MPI3) - #-------------------------------------------------------------------------- # Option to control whether AXOM_DEFINE compiler define is enabled # From ee8de792a1f0fa2436e899cdb65cdbccaa3644f6 Mon Sep 17 00:00:00 2001 From: Brad Whitlock Date: Tue, 9 Sep 2025 15:04:36 -0700 Subject: [PATCH 17/24] Moved shared memory allocator to core. --- src/axom/core/CMakeLists.txt | 1 + src/axom/core/memory_management.cpp | 58 +++++++++++++++++++ src/axom/core/memory_management.hpp | 17 ++++++ src/axom/core/tests/CMakeLists.txt | 3 +- src/axom/core/tests/core_mpi_main.cpp | 1 + src/axom/core/tests/core_shared_memory.hpp | 52 +++++++++++++++++ .../quest/interface/internal/QuestHelpers.cpp | 11 +--- .../quest/interface/internal/QuestHelpers.hpp | 2 - src/axom/quest/interface/signed_distance.cpp | 17 +----- 9 files changed, 135 insertions(+), 27 deletions(-) create mode 100644 src/axom/core/memory_management.cpp create mode 100644 src/axom/core/tests/core_shared_memory.hpp diff --git a/src/axom/core/CMakeLists.txt b/src/axom/core/CMakeLists.txt index 89fa639774..77ec820ef2 100644 --- a/src/axom/core/CMakeLists.txt +++ b/src/axom/core/CMakeLists.txt @@ -107,6 +107,7 @@ set(core_sources numerics/polynomial_solvers.cpp + memory_management.cpp Path.cpp Types.cpp ) diff --git a/src/axom/core/memory_management.cpp b/src/axom/core/memory_management.cpp new file mode 100644 index 0000000000..29bd8975f6 --- /dev/null +++ b/src/axom/core/memory_management.cpp @@ -0,0 +1,58 @@ +// Copyright (c) 2017-2025, Lawrence Livermore National Security, LLC and +// other Axom Project Developers. See the top-level LICENSE file for details. +// +// SPDX-License-Identifier: (BSD-3-Clause) + +#include "axom/core/memory_management.hpp" + +#if defined(AXOM_USE_UMPIRE_SHARED_MEMORY) + #include "umpire/Umpire.hpp" + #include "umpire/strategy/NamedAllocationStrategy.hpp" + #include "umpire/util/MemoryResourceTraits.hpp" +#endif + +namespace axom +{ + +bool isSharedMemoryAllocator(int allocID) +{ + bool isShared = false; +#if defined(AXOM_USE_UMPIRE) + umpire::ResourceManager& rm = umpire::ResourceManager::getInstance(); + if(rm.isAllocator(allocID)) + { + umpire::Allocator allocator = rm.getAllocator(allocID); + + isShared = allocator.getAllocationStrategy()->getTraits().resource == + umpire::MemoryResourceTraits::resource_type::shared; + } +#endif + return isShared; +} + +int getSharedMemoryAllocatorID() +{ + int allocator_id = INVALID_ALLOCATOR_ID; +#if defined(AXOM_USE_UMPIRE_SHARED_MEMORY) + const std::string allocatorName("axom_named_allocator"); + auto& rm = umpire::ResourceManager::getInstance(); + if(!rm.isAllocator(allocatorName)) + { + // Create the allocator + auto traits {umpire::get_default_resource_traits("SHARED")}; + traits.scope = umpire::MemoryResourceTraits::shared_scope::node; + auto axom_node_allocator {rm.makeResource("SHARED::axom_node_allocator", traits)}; + auto axom_named_allocator { + rm.makeAllocator(allocatorName, + axom_node_allocator)}; + allocator_id = axom_named_allocator.getId(); + } + else + { + allocator_id = rm.getAllocator(allocatorName).getId(); + } +#endif + return allocator_id; +} + +} // end namespace axom diff --git a/src/axom/core/memory_management.hpp b/src/axom/core/memory_management.hpp index 9ee550ab81..59acc91238 100644 --- a/src/axom/core/memory_management.hpp +++ b/src/axom/core/memory_management.hpp @@ -542,6 +542,23 @@ inline bool isDeviceAllocator(int allocator_id) inline bool isDeviceAllocator(int AXOM_UNUSED_PARAM(allocator_id)) { return false; } #endif +/*! + * \brief Determines whether an allocator id is for shared memory. + * + * \param allocID An allocator id. + * + * \return True if the allocator id is for shared memory; false otherwise. + */ +bool isSharedMemoryAllocator(int allocID); + +/*! + * \brief Get the allocator ID for Axom's shared memory allocator. + * + * \return The allocator ID for Axom's shared memory allocator (if Axom is using Umpire), + * or INVALID_ALLOCATOR_ID otherwise. + */ +int getSharedMemoryAllocatorID(); + } // namespace axom #endif /* AXOM_MEMORYMANAGEMENT_HPP_ */ diff --git a/src/axom/core/tests/CMakeLists.txt b/src/axom/core/tests/CMakeLists.txt index 60bc0779d8..854ca85edc 100644 --- a/src/axom/core/tests/CMakeLists.txt +++ b/src/axom/core/tests/CMakeLists.txt @@ -104,6 +104,7 @@ axom_add_test( NAME core_collections if (AXOM_ENABLE_MPI) set( core_mpi_tests core_types.hpp + core_shared_memory.hpp ) axom_add_executable(NAME core_mpi_tests @@ -117,7 +118,7 @@ if (AXOM_ENABLE_MPI) get_filename_component( test_suite ${test_suite} NAME_WE ) axom_add_test( NAME ${test_suite}_mpi COMMAND core_mpi_tests --gtest_filter=${test_suite}* - NUM_MPI_TASKS 1 ) + NUM_MPI_TASKS 2 ) endforeach() endif() diff --git a/src/axom/core/tests/core_mpi_main.cpp b/src/axom/core/tests/core_mpi_main.cpp index 4eae819475..ffa2653362 100644 --- a/src/axom/core/tests/core_mpi_main.cpp +++ b/src/axom/core/tests/core_mpi_main.cpp @@ -10,6 +10,7 @@ #include "axom/config.hpp" #include "core_types.hpp" +#include "core_shared_memory.hpp" int main(int argc, char* argv[]) { diff --git a/src/axom/core/tests/core_shared_memory.hpp b/src/axom/core/tests/core_shared_memory.hpp new file mode 100644 index 0000000000..0819f48e29 --- /dev/null +++ b/src/axom/core/tests/core_shared_memory.hpp @@ -0,0 +1,52 @@ +// Copyright (c) 2017-2025, Lawrence Livermore National Security, LLC and +// other Axom Project Developers. See the top-level LICENSE file for details. +// +// SPDX-License-Identifier: (BSD-3-Clause) + +// Axom includes +#include "axom/config.hpp" +#include "axom/core/memory_management.hpp" + +// gtest includes +#include "gtest/gtest.h" + +#include + +//------------------------------------------------------------------------------ +#if defined(AXOM_USE_UMPIRE_SHARED_MEMORY) +TEST(core_shared_memory, shared_memory_allocator) +{ + EXPECT_FALSE(axom::isSharedMemoryAllocator(axom::getDefaultAllocatorID())); + EXPECT_TRUE(axom::isSharedMemoryAllocator(axom::getSharedMemoryAllocatorID())); + + int rank = 0; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + // Allocate shared memory + constexpr int sizes[] = {1024, 4096, 1000000}; + for(int si = 0; si < 3; si++) + { + const int N = sizes[si]; + int *buffer = axom::allocate(N, axom::getSharedMemoryAllocatorID()); + + // Populate the shared memory on rank 0 + if(rank == 0) + { + for(int i = 0; i < N; i++) + { + buffer[i] = i; + } + } + + MPI_Barrier(MPI_COMM_WORLD); + + // Test the shared memory on all ranks. + for(int i = 0; i < N; i++) + { + EXPECT_EQ(buffer[i], i); + } + + axom::deallocate(buffer); + } +} +#endif diff --git a/src/axom/quest/interface/internal/QuestHelpers.cpp b/src/axom/quest/interface/internal/QuestHelpers.cpp index d49a3238be..557b6a3ca4 100644 --- a/src/axom/quest/interface/internal/QuestHelpers.cpp +++ b/src/axom/quest/interface/internal/QuestHelpers.cpp @@ -134,7 +134,6 @@ static void create_communicators(MPI_Comm global_comm, * \brief Allocates a shared memory buffer for the mesh that is shared among * all the ranks within the same compute node. * - * \param [in] allocatorID A valid Umpire allocator id that can allocate shared memory. * \param [in] mesh_metadata tuple with the number of nodes/faces on the mesh * \param [out] x pointer into the buffer where the x-coordinates are stored. * \param [out] y pointer into the buffer where the y-coordinates are stored. @@ -144,7 +143,6 @@ static void create_communicators(MPI_Comm global_comm, * * \return bytesize the number of bytes in the raw buffer. * - * \pre allocatorID is a valid shared memory allocator. * \pre mesh_metadata != nullptr * \pre x == nullptr * \pre y == nullptr @@ -158,8 +156,7 @@ static void create_communicators(MPI_Comm global_comm, * \post coon != nullptr * \post mesh_buffer != nullptr */ -static size_t allocate_shared_buffer(int allocatorID, - const axom::IndexType mesh_metadata[2], +static size_t allocate_shared_buffer(const axom::IndexType mesh_metadata[2], double*& x, double*& y, double*& z, @@ -170,7 +167,7 @@ static size_t allocate_shared_buffer(int allocatorID, const axom::IndexType nnodes = mesh_metadata[0]; const axom::IndexType nfaces = mesh_metadata[1]; const size_t bytesize = nnodes * 3 * sizeof(double) + nfaces * 3 * sizeof(axom::IndexType); - mesh_buffer = allocate(bytesize, allocatorID); + mesh_buffer = allocate(bytesize, getSharedMemoryAllocatorID()); // calculate offset to the coordinates & cell connectivity in the buffer int baseOffset = nnodes * sizeof(double); @@ -193,12 +190,10 @@ static size_t allocate_shared_buffer(int allocatorID, */ int read_stl_mesh_shared(const std::string& file, MPI_Comm global_comm, - int allocatorID, unsigned char*& mesh_buffer, mint::Mesh*& m) { SLIC_ASSERT(global_comm != MPI_COMM_NULL); - SLIC_ASSERT(allocatorID != INVALID_ALLOCATOR_ID); // NOTE: STL meshes are always 3D mesh consisting of triangles. using TriangleMesh = mint::UnstructuredMesh; @@ -250,7 +245,7 @@ int read_stl_mesh_shared(const std::string& file, double* z = nullptr; axom::IndexType* conn = nullptr; const size_t numBytes = - allocate_shared_buffer(allocatorID, mesh_metadata, x, y, z, conn, mesh_buffer); + allocate_shared_buffer(mesh_metadata, x, y, z, conn, mesh_buffer); SLIC_ASSERT(x != nullptr); SLIC_ASSERT(y != nullptr); SLIC_ASSERT(z != nullptr); diff --git a/src/axom/quest/interface/internal/QuestHelpers.hpp b/src/axom/quest/interface/internal/QuestHelpers.hpp index a318b2ca73..dcec418d43 100644 --- a/src/axom/quest/interface/internal/QuestHelpers.hpp +++ b/src/axom/quest/interface/internal/QuestHelpers.hpp @@ -71,7 +71,6 @@ class ScopedLogLevelChanger * * \param [in] file the file consisting of the surface mesh * \param [in] global_comm handle to the global MPI communicator - * \param [in] allocatorID An UMPIRE allocator ID that can allocate shared memory. * \param [out] mesh_buffer pointer to the raw mesh buffer * \param [out] m pointer to the mesh object * @@ -96,7 +95,6 @@ class ScopedLogLevelChanger */ int read_stl_mesh_shared(const std::string& file, MPI_Comm global_comm, - int allocatorID, unsigned char*& mesh_buffer, mint::Mesh*& m); #endif diff --git a/src/axom/quest/interface/signed_distance.cpp b/src/axom/quest/interface/signed_distance.cpp index e63e9dca03..7e766a5d5d 100644 --- a/src/axom/quest/interface/signed_distance.cpp +++ b/src/axom/quest/interface/signed_distance.cpp @@ -102,7 +102,6 @@ static bool s_must_finalize_logger = false; static bool s_logger_is_initialized = false; #if defined(AXOM_USE_UMPIRE_SHARED_MEMORY) -static int s_allocator_id = INVALID_ALLOCATOR_ID; static unsigned char* s_shared_mesh_buffer = nullptr; #else static std::string s_shared_memory_requirements( @@ -135,22 +134,8 @@ int signed_distance_init(const std::string& file, MPI_Comm comm) #if defined(AXOM_USE_UMPIRE_SHARED_MEMORY) if(Parameters.use_shared_memory) { - if(s_allocator_id == INVALID_ALLOCATOR_ID) - { - // Make a shared memory allocator if we have not made it before. We'll reuse - // the allocator to allocate different buffers (1 at a time). - auto& rm = umpire::ResourceManager::getInstance(); - auto traits {umpire::get_default_resource_traits("SHARED")}; - traits.scope = umpire::MemoryResourceTraits::shared_scope::node; - auto node_allocator {rm.makeResource("SHARED::node_allocator", traits)}; - auto signed_distance_allocator { - rm.makeAllocator("signed_distance_allocator", - node_allocator)}; - s_allocator_id = signed_distance_allocator.getId(); - } - rc = - internal::read_stl_mesh_shared(file, comm, s_allocator_id, s_shared_mesh_buffer, s_surface_mesh); + internal::read_stl_mesh_shared(file, comm, s_shared_mesh_buffer, s_surface_mesh); } else { From ee8e9ba74f77908cb1ef6c126584f2b0c3de0ed7 Mon Sep 17 00:00:00 2001 From: Brad Whitlock Date: Tue, 9 Sep 2025 15:05:29 -0700 Subject: [PATCH 18/24] make style --- src/axom/core/memory_management.cpp | 7 +++---- src/axom/quest/interface/internal/QuestHelpers.cpp | 3 +-- src/axom/quest/interface/signed_distance.cpp | 3 +-- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/axom/core/memory_management.cpp b/src/axom/core/memory_management.cpp index 29bd8975f6..2ad06d6fab 100644 --- a/src/axom/core/memory_management.cpp +++ b/src/axom/core/memory_management.cpp @@ -24,7 +24,7 @@ bool isSharedMemoryAllocator(int allocID) umpire::Allocator allocator = rm.getAllocator(allocID); isShared = allocator.getAllocationStrategy()->getTraits().resource == - umpire::MemoryResourceTraits::resource_type::shared; + umpire::MemoryResourceTraits::resource_type::shared; } #endif return isShared; @@ -43,8 +43,7 @@ int getSharedMemoryAllocatorID() traits.scope = umpire::MemoryResourceTraits::shared_scope::node; auto axom_node_allocator {rm.makeResource("SHARED::axom_node_allocator", traits)}; auto axom_named_allocator { - rm.makeAllocator(allocatorName, - axom_node_allocator)}; + rm.makeAllocator(allocatorName, axom_node_allocator)}; allocator_id = axom_named_allocator.getId(); } else @@ -55,4 +54,4 @@ int getSharedMemoryAllocatorID() return allocator_id; } -} // end namespace axom +} // end namespace axom diff --git a/src/axom/quest/interface/internal/QuestHelpers.cpp b/src/axom/quest/interface/internal/QuestHelpers.cpp index 557b6a3ca4..9a370ad4e6 100644 --- a/src/axom/quest/interface/internal/QuestHelpers.cpp +++ b/src/axom/quest/interface/internal/QuestHelpers.cpp @@ -244,8 +244,7 @@ int read_stl_mesh_shared(const std::string& file, double* y = nullptr; double* z = nullptr; axom::IndexType* conn = nullptr; - const size_t numBytes = - allocate_shared_buffer(mesh_metadata, x, y, z, conn, mesh_buffer); + const size_t numBytes = allocate_shared_buffer(mesh_metadata, x, y, z, conn, mesh_buffer); SLIC_ASSERT(x != nullptr); SLIC_ASSERT(y != nullptr); SLIC_ASSERT(z != nullptr); diff --git a/src/axom/quest/interface/signed_distance.cpp b/src/axom/quest/interface/signed_distance.cpp index 7e766a5d5d..984806be59 100644 --- a/src/axom/quest/interface/signed_distance.cpp +++ b/src/axom/quest/interface/signed_distance.cpp @@ -134,8 +134,7 @@ int signed_distance_init(const std::string& file, MPI_Comm comm) #if defined(AXOM_USE_UMPIRE_SHARED_MEMORY) if(Parameters.use_shared_memory) { - rc = - internal::read_stl_mesh_shared(file, comm, s_shared_mesh_buffer, s_surface_mesh); + rc = internal::read_stl_mesh_shared(file, comm, s_shared_mesh_buffer, s_surface_mesh); } else { From d367dc793c0703a6abd1436b7e3837db52abf500 Mon Sep 17 00:00:00 2001 From: Brad Whitlock Date: Tue, 9 Sep 2025 15:14:00 -0700 Subject: [PATCH 19/24] Moved some function declarations. --- src/axom/core/memory_management.hpp | 34 ++++++++++++++--------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/axom/core/memory_management.hpp b/src/axom/core/memory_management.hpp index 59acc91238..8a9de5595b 100644 --- a/src/axom/core/memory_management.hpp +++ b/src/axom/core/memory_management.hpp @@ -144,6 +144,23 @@ inline int getAllocatorIDFromPointer(const void* ptr) return MALLOC_ALLOCATOR_ID; } +/*! + * \brief Determines whether an allocator id is for shared memory. + * + * \param allocID An allocator id. + * + * \return True if the allocator id is for shared memory; false otherwise. + */ +bool isSharedMemoryAllocator(int allocID); + +/*! + * \brief Get the allocator ID for Axom's shared memory allocator. + * + * \return The allocator ID for Axom's shared memory allocator (if Axom is using Umpire), + * or INVALID_ALLOCATOR_ID otherwise. + */ +int getSharedMemoryAllocatorID(); + /*! * \brief Allocates a chunk of memory of type T. * @@ -542,23 +559,6 @@ inline bool isDeviceAllocator(int allocator_id) inline bool isDeviceAllocator(int AXOM_UNUSED_PARAM(allocator_id)) { return false; } #endif -/*! - * \brief Determines whether an allocator id is for shared memory. - * - * \param allocID An allocator id. - * - * \return True if the allocator id is for shared memory; false otherwise. - */ -bool isSharedMemoryAllocator(int allocID); - -/*! - * \brief Get the allocator ID for Axom's shared memory allocator. - * - * \return The allocator ID for Axom's shared memory allocator (if Axom is using Umpire), - * or INVALID_ALLOCATOR_ID otherwise. - */ -int getSharedMemoryAllocatorID(); - } // namespace axom #endif /* AXOM_MEMORYMANAGEMENT_HPP_ */ From 4ce036a014266a71c77c908800b676267d288d9e Mon Sep 17 00:00:00 2001 From: Brad Whitlock Date: Tue, 9 Sep 2025 15:22:54 -0700 Subject: [PATCH 20/24] Change how we get some Umpire includes --- src/axom/core/memory_management.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/axom/core/memory_management.cpp b/src/axom/core/memory_management.cpp index 2ad06d6fab..d38177be5d 100644 --- a/src/axom/core/memory_management.cpp +++ b/src/axom/core/memory_management.cpp @@ -5,10 +5,12 @@ #include "axom/core/memory_management.hpp" -#if defined(AXOM_USE_UMPIRE_SHARED_MEMORY) +#if defined(AXOM_USE_UMPIRE) #include "umpire/Umpire.hpp" - #include "umpire/strategy/NamedAllocationStrategy.hpp" #include "umpire/util/MemoryResourceTraits.hpp" + #if defined(AXOM_USE_UMPIRE_SHARED_MEMORY) + #include "umpire/strategy/NamedAllocationStrategy.hpp" + #endif #endif namespace axom From cad2b2b2a6c6285c047728f7fc4cdfa0720c2d5e Mon Sep 17 00:00:00 2001 From: Brad Whitlock Date: Tue, 9 Sep 2025 15:26:16 -0700 Subject: [PATCH 21/24] Removed some comments related to MPI windows. --- src/axom/quest/interface/internal/QuestHelpers.hpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/axom/quest/interface/internal/QuestHelpers.hpp b/src/axom/quest/interface/internal/QuestHelpers.hpp index dcec418d43..a74190d295 100644 --- a/src/axom/quest/interface/internal/QuestHelpers.hpp +++ b/src/axom/quest/interface/internal/QuestHelpers.hpp @@ -84,14 +84,10 @@ class ScopedLogLevelChanger * \pre global_comm != MPI_COMM_NULL * \pre mesh_buffer == nullptr * \pre m == nullptr - * \pre intra_node_comm == MPI_COMM_NULL - * \pre shared_window == MPI_WIN_NULL * * \post m != nullptr * \post m->isExternal() == true * \post mesh_buffer != nullptr - * \post intra_node_comm != MPI_COMM_NULL - * \post shared_window != MPI_WIN_NULL */ int read_stl_mesh_shared(const std::string& file, MPI_Comm global_comm, From c69d950f6bdfb7dcc9c2cc29ac694f2f5e0a37b5 Mon Sep 17 00:00:00 2001 From: Brad Whitlock Date: Tue, 9 Sep 2025 15:29:15 -0700 Subject: [PATCH 22/24] Removed some Umpire includes from signed_distance. --- src/axom/quest/interface/signed_distance.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/axom/quest/interface/signed_distance.cpp b/src/axom/quest/interface/signed_distance.cpp index 984806be59..ac9a3fd31d 100644 --- a/src/axom/quest/interface/signed_distance.cpp +++ b/src/axom/quest/interface/signed_distance.cpp @@ -13,12 +13,6 @@ #include "axom/slic/interface/slic.hpp" -#if defined(AXOM_USE_UMPIRE_SHARED_MEMORY) - #include "umpire/Umpire.hpp" - #include "umpire/strategy/NamedAllocationStrategy.hpp" - #include "umpire/util/MemoryResourceTraits.hpp" -#endif - #ifdef AXOM_USE_MPI #include #endif From 7bef6e0b32965088718212bc262ec343ed128a3d Mon Sep 17 00:00:00 2001 From: Brad Whitlock Date: Tue, 9 Sep 2025 16:45:06 -0700 Subject: [PATCH 23/24] Renamed axom shared allocator. --- src/axom/core/memory_management.cpp | 6 +++--- src/axom/core/tests/core_shared_memory.hpp | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/axom/core/memory_management.cpp b/src/axom/core/memory_management.cpp index d38177be5d..8e22511ec3 100644 --- a/src/axom/core/memory_management.cpp +++ b/src/axom/core/memory_management.cpp @@ -36,7 +36,7 @@ int getSharedMemoryAllocatorID() { int allocator_id = INVALID_ALLOCATOR_ID; #if defined(AXOM_USE_UMPIRE_SHARED_MEMORY) - const std::string allocatorName("axom_named_allocator"); + const std::string allocatorName("axom_shared_allocator"); auto& rm = umpire::ResourceManager::getInstance(); if(!rm.isAllocator(allocatorName)) { @@ -44,9 +44,9 @@ int getSharedMemoryAllocatorID() auto traits {umpire::get_default_resource_traits("SHARED")}; traits.scope = umpire::MemoryResourceTraits::shared_scope::node; auto axom_node_allocator {rm.makeResource("SHARED::axom_node_allocator", traits)}; - auto axom_named_allocator { + auto axom_shared_allocator { rm.makeAllocator(allocatorName, axom_node_allocator)}; - allocator_id = axom_named_allocator.getId(); + allocator_id = axom_shared_allocator.getId(); } else { diff --git a/src/axom/core/tests/core_shared_memory.hpp b/src/axom/core/tests/core_shared_memory.hpp index 0819f48e29..532c19ad2e 100644 --- a/src/axom/core/tests/core_shared_memory.hpp +++ b/src/axom/core/tests/core_shared_memory.hpp @@ -22,10 +22,10 @@ TEST(core_shared_memory, shared_memory_allocator) int rank = 0; MPI_Comm_rank(MPI_COMM_WORLD, &rank); - // Allocate shared memory - constexpr int sizes[] = {1024, 4096, 1000000}; + const int sizes[] = {1024, 4096, 1000000}; for(int si = 0; si < 3; si++) { + // Allocate shared memory const int N = sizes[si]; int *buffer = axom::allocate(N, axom::getSharedMemoryAllocatorID()); From 9e8d791130205244ad4cbb61fcd4f5154345961e Mon Sep 17 00:00:00 2001 From: Brad Whitlock Date: Tue, 21 Oct 2025 11:23:17 -0700 Subject: [PATCH 24/24] Use UMPIRE_DEFAULT_SHARED_MEMORY_RESOURCE in the name of the requested shared memory allocator. Add cmake status message for value of UMPIRE_DEFAULT_SHARED_MEMORY_RESOURCE. --- src/axom/core/memory_management.cpp | 6 ++++-- src/axom/core/tests/core_shared_memory.hpp | 2 ++ .../thirdparty/SetupAxomThirdParty.cmake | 20 ++++++++++++++++++- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/axom/core/memory_management.cpp b/src/axom/core/memory_management.cpp index 8e22511ec3..5b8c592bae 100644 --- a/src/axom/core/memory_management.cpp +++ b/src/axom/core/memory_management.cpp @@ -6,6 +6,7 @@ #include "axom/core/memory_management.hpp" #if defined(AXOM_USE_UMPIRE) + #include "axom/fmt.hpp" #include "umpire/Umpire.hpp" #include "umpire/util/MemoryResourceTraits.hpp" #if defined(AXOM_USE_UMPIRE_SHARED_MEMORY) @@ -41,9 +42,10 @@ int getSharedMemoryAllocatorID() if(!rm.isAllocator(allocatorName)) { // Create the allocator - auto traits {umpire::get_default_resource_traits("SHARED")}; + const auto name = axom::fmt::format("SHARED::{}", UMPIRE_DEFAULT_SHARED_MEMORY_RESOURCE); + auto traits {umpire::get_default_resource_traits(name)}; traits.scope = umpire::MemoryResourceTraits::shared_scope::node; - auto axom_node_allocator {rm.makeResource("SHARED::axom_node_allocator", traits)}; + auto axom_node_allocator {rm.makeResource(axom::fmt::format("{}::axom_node_allocator", name), traits)}; auto axom_shared_allocator { rm.makeAllocator(allocatorName, axom_node_allocator)}; allocator_id = axom_shared_allocator.getId(); diff --git a/src/axom/core/tests/core_shared_memory.hpp b/src/axom/core/tests/core_shared_memory.hpp index 532c19ad2e..4666c6cc9f 100644 --- a/src/axom/core/tests/core_shared_memory.hpp +++ b/src/axom/core/tests/core_shared_memory.hpp @@ -46,6 +46,8 @@ TEST(core_shared_memory, shared_memory_allocator) EXPECT_EQ(buffer[i], i); } + MPI_Barrier(MPI_COMM_WORLD); + axom::deallocate(buffer); } } diff --git a/src/cmake/thirdparty/SetupAxomThirdParty.cmake b/src/cmake/thirdparty/SetupAxomThirdParty.cmake index 78d177c519..836d70d625 100644 --- a/src/cmake/thirdparty/SetupAxomThirdParty.cmake +++ b/src/cmake/thirdparty/SetupAxomThirdParty.cmake @@ -75,7 +75,25 @@ if (UMPIRE_DIR) else() set(AXOM_USE_UMPIRE_SHARED_MEMORY FALSE) endif() - message(STATUS "Umpire supports shared memory: ${AXOM_USE_UMPIRE_SHARED_MEMORY}") + message(STATUS " Umpire supports shared memory: ${AXOM_USE_UMPIRE_SHARED_MEMORY}") + + # If it looks like Umpire supports shared memory (and the header file exists) + # then print out the default type of shared memory. + set(UMPIRE_CONFIG_HPP "${UMPIRE_DIR}/include/umpire/config.hpp") + if(AXOM_USE_UMPIRE_SHARED_MEMORY AND EXISTS "${UMPIRE_CONFIG_HPP}") + file(READ "${UMPIRE_CONFIG_HPP}" UMPIRE_CONFIG_HPP_CONTENTS) + # Try to match: #define UMPIRE_DEFAULT_SHARED_MEMORY_RESOURCE + string(REGEX MATCH "#define[ \t]+UMPIRE_DEFAULT_SHARED_MEMORY_RESOURCE[ \t]+([^\n\r ]+)" UMPIRE_MACRO_LINE "${UMPIRE_CONFIG_HPP_CONTENTS}") + if(UMPIRE_MACRO_LINE) + # Extract just the value (the first capture group) + string(REGEX REPLACE ".*#define[ \t]+UMPIRE_DEFAULT_SHARED_MEMORY_RESOURCE[ \t]+\"([^\"]*)\".*" "\\1" UMPIRE_DEFAULT_SHARED_MEMORY_RESOURCE "${UMPIRE_MACRO_LINE}") + message(STATUS " UMPIRE_DEFAULT_SHARED_MEMORY_RESOURCE: ${UMPIRE_DEFAULT_SHARED_MEMORY_RESOURCE}") + else() + message(STATUS " UMPIRE_DEFAULT_SHARED_MEMORY_RESOURCE is not defined in ${UMPIRE_CONFIG_HPP}") + endif() + else() + message(WARNING " Could not find ${UMPIRE_CONFIG_HPP}") + endif() else() message(STATUS "Umpire support is OFF") set(UMPIRE_FOUND FALSE)