parthenon-hpc-lab · pgrete · Jun 12, 2026 · Jan 8, 2024 · Jan 11, 2024 · Jan 11, 2024
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -1,7 +1,7 @@
 // devcontainer.json
     {
       "name": "parthenon-dev",
-      "image": "ghcr.io/parthenon-hpc-lab/cuda12.8-mpi-hdf5-opmd",
+      "image": "ghcr.io/parthenon-hpc-lab/cuda12.9-mpi-hdf5-opmd",
       // Somehow required to make GPUs work inside docker on the CI machine.
       // Just passing the runtime and/or --gpus=all works initially but then
       // the GPU disappears in the active container.

diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml
@@ -28,10 +28,14 @@ jobs:
         cache-dependency-path: '**/requirements.txt'
     - run: pip install -r requirements.txt
 
-    - name: Install dependencies
+    - name: Install dependencies (Homebrew)
       run: |
         brew install openmpi hdf5-mpi adios2 || true
 
+    - name: Install OpenPMD
+      run: |
+        openPMD_USE_MPI=ON python3 -m pip install openpmd-api --no-binary openpmd-api
+
     - name: Install parthenon_tools
       run: |
         pip install scripts/python/packages/parthenon_tools

diff --git a/.github/workflows/ci-short.yml b/.github/workflows/ci-short.yml
@@ -32,8 +32,9 @@ jobs:
       - name: cpplint
         run: python3 ./tst/style/cpplint.py --counting=detailed --recursive src example tst
       - name: copyright
+        # using variant mpi because openpmd lib is only installed with mpi support (no serial)
         run: |
-          cmake -DCMAKE_CXX_FLAGS=-Werror -Bbuild-copyright-check
+          cmake -DCMAKE_CXX_FLAGS=-Werror -Bbuild-copyright-check -DMACHINE_VARIANT=mpi
           cmake --build build-copyright-check -t check-copyright
       - uses: actions/upload-artifact@v4
         with:
@@ -104,6 +105,7 @@ jobs:
           # make sure parthenon tools is in our python path
           export PYTHONPATH="$PYTHONPATH:$(python -m site --user-site)"
           ctest -R regression_mpi_test:output_hdf5
+          ctest -R regression_mpi_test:restart_opmd
       # Test example with swarms
       - name: particle-leapfrog
         run: |
@@ -178,6 +180,7 @@ jobs:
           # make sure parthenon tools is in our python path
           export PYTHONPATH="$PYTHONPATH:$(python -m site --user-site)"
           ctest -R regression_mpi_test:output_hdf5
+          ctest -R regression_mpi_test:restart_opmd
       # Test example with swarms
       - name: particle-leapfrog
         run: |

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -37,6 +37,7 @@ jobs:
             mkdir -p build && cd build
             cmake -DPARTHENON_DISABLE_HDF5=ON \
                   -DPARTHENON_DISABLE_MPI=ON \
+                  -DPARTHENON_DISABLE_OPENPMD=ON \
                   -DREGRESSION_GOLD_STANDARD_SYNC=OFF \
                   -DBUILD_TESTING=OFF \
                   ..

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,7 @@
 ## Current develop
 
 ### Added (new features/APIs/variables/...)
+- [[PR 1050]](https://github.com/parthenon-hpc-lab/parthenon/pull/1050) Add support for OpenPMD/ADIOS2 output (incl slices and coarsened dumps)
 - [[PR 1271]](https://github.com/parthenon-hpc-lab/parthenon/pull/1271) Add option to set a minimum number of teams for boundary communication kernels
 - [[PR 1382]](https://github.com/parthenon-hpc-lab/parthenon/pull/1382) Support Particle AMR
 - [[PR 1378]](https://github.com/parthenon-hpc-lab/parthenon/pull/1378) MeshData Swarm Tasks

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -36,6 +36,8 @@ option(PARTHENON_DISABLE_MPI "MPI is enabled by default if found, set this to Tr
 option(PARTHENON_ENABLE_HOST_COMM_BUFFERS "CUDA/HIP Only: Allocate communication buffers on host (may be slower)" OFF)
 option(PARTHENON_DISABLE_HDF5 "HDF5 is enabled by default if found, set this to True to disable HDF5" OFF)
 option(PARTHENON_DISABLE_HDF5_COMPRESSION "HDF5 compression is enabled by default, set this to True to disable compression in HDF5 output/restart files" OFF)
+option(PARTHENON_DISABLE_OPENPMD "OpenPMD is enabled by default if found, set this to ON to disable OpenPMD" ON)
+option(PARTHENON_USE_SYSTEM_OPENPMD "OpenPMD API is downloaded and installed automatically by default. Set this to ON to find and use a version installed on the system." OFF)
 option(PARTHENON_DISABLE_SPARSE "Sparse capability is enabled by default, set this to True to compile-time disable all sparse capability" OFF)
 option(PARTHENON_ENABLE_ASCENT "Enable Ascent for in situ visualization and analysis" OFF)
 option(PARTHENON_LINT_DEFAULT "Linting is turned off by default, use the \"lint\" target or set \
@@ -239,6 +241,51 @@ if (NOT PARTHENON_DISABLE_HDF5)
   install(TARGETS HDF5_C EXPORT parthenonTargets)
 endif()
 
+set(PARTHENON_ENABLE_OPENPMD OFF)
+if (NOT PARTHENON_DISABLE_OPENPMD)
+  if (PARTHENON_USE_SYSTEM_OPENPMD)
+    # we want to enforce the use of ADIOS2 with this backend
+    set(PARTHENON_OPENPMD_COMPONENTS "ADIOS2")
+    if (ENABLE_MPI)
+      list(APPEND PARTHENON_OPENPMD_COMPONENTS "MPI")
+    else()
+      list(APPEND PARTHENON_OPENPMD_COMPONENTS "NOMPI")
+    endif()
+    find_package(openPMD 0.16.1 CONFIG COMPONENTS ${PARTHENON_OPENPMD_COMPONENTS})
+    if (NOT openPMD_FOUND)
+      message(FATAL_ERROR "OpenPMD API requested to be used from the environment but it was "
+      "not found. Either append the path to the installed OpenPMD lib via the "
+      "openPMD_DIR or CMAKE_PREFIX_PATH, or use the version shipped with Parthenon by keeping "
+      "PARTHENON_USE_SYSTEM_OPENPMD=OFF")
+    endif()
+  else()
+    include(FetchContent)
+    set(CMAKE_POLICY_DEFAULT_CMP0077 NEW)
+    set(openPMD_BUILD_CLI_TOOLS OFF)
+    set(openPMD_BUILD_EXAMPLES OFF)
+    set(openPMD_BUILD_TESTING OFF)
+    set(openPMD_BUILD_SHARED_LIBS OFF)
+    set(openPMD_INSTALL OFF)
+    set(openPMD_USE_ADIOS2 ON)  # we definitely want ADIOS2
+    if( NOT Python3_Interpreter_FOUND)
+      find_package(Python3 REQUIRED COMPONENTS Interpreter)
+    endif()
+    # Why not build the python interface for easier testing when Python is available
+    if(Python3_Interpreter_FOUND)
+      set(openPMD_USE_PYTHON ON)
+    else()
+      set(openPMD_USE_PYTHON OFF)
+    endif()
+    FetchContent_Declare(openPMD
+      GIT_REPOSITORY "https://github.com/openPMD/openPMD-api.git"
+      GIT_TAG        "0.17.0") # Release 0.17.0
+    FetchContent_MakeAvailable(openPMD)
+    install(TARGETS openPMD EXPORT parthenonTargets)
+  endif()
+
+  set(PARTHENON_ENABLE_OPENPMD ON)
+endif()
+
 # Kokkos recommendatation resulting in not using default GNU extensions
 set(CMAKE_CXX_EXTENSIONS OFF)
 

diff --git a/README.md b/README.md
@@ -39,6 +39,7 @@ Parthenon -- a performance portable block-structured adaptive mesh refinement fr
 
 * MPI
 * OpenMP
+* OpenPMD and ADIOS2 (for outputs)
 * HDF5 (for outputs)
 * Ascent (for in situ visualization and analysis)
 

diff --git a/cmake/machinecfg/CI.cmake b/cmake/machinecfg/CI.cmake
@@ -1,6 +1,6 @@
 #========================================================================================
 # Parthenon performance portable AMR framework
-# Copyright(C) 2020 The Parthenon collaboration
+# Copyright(C) 2020-2026 The Parthenon collaboration
 # Licensed under the 3-clause BSD License, see LICENSE file for details
 #========================================================================================
 # (C) (or copyright) 2020. Triad National Security, LLC. All rights reserved.
@@ -25,7 +25,6 @@ set(NUM_MPI_PROC_TESTING "4" CACHE STRING "CI runs tests with 4 MPI ranks")
 if (${MACHINE_VARIANT} MATCHES "cuda")
   set(Kokkos_ARCH_AMPERE80 ON CACHE BOOL "GPU architecture")
   set(Kokkos_ENABLE_CUDA ON CACHE BOOL "Enable Cuda")
-  set(CMAKE_CXX_COMPILER ${CMAKE_CURRENT_SOURCE_DIR}/external/Kokkos/bin/nvcc_wrapper CACHE STRING "Use nvcc_wrapper")
 else()
   set(CMAKE_CXX_FLAGS "-fopenmp-simd" CACHE STRING "Default opt flags")
 endif()
@@ -34,7 +33,13 @@ if (${MACHINE_VARIANT} MATCHES "mpi")
   # not using the following as the default is determined correctly
   #set(TEST_MPIEXEC mpiexec CACHE STRING "Command to launch MPI applications")
   set(HDF5_ROOT /usr/local/hdf5/parallel CACHE STRING "HDF5 path")
+  set(PARTHENON_USE_SYSTEM_OPENPMD ON CACHE BOOL "Use API in container")
 else()
   set(HDF5_ROOT /usr/local/hdf5/serial CACHE STRING "HDF5 path")
   set(PARTHENON_DISABLE_MPI ON CACHE BOOL "Disable MPI")
+  # testing auto fetch and compile
+  set(PARTHENON_USE_SYSTEM_OPENPMD OFF CACHE BOOL "Use API in container")
 endif()
+
+set(PARTHENON_DISABLE_OPENPMD OFF CACHE BOOL "Always use OpenPMD build in CI env")
+
diff --git a/cmake/machinecfg/GitHubActions.cmake b/cmake/machinecfg/GitHubActions.cmake
@@ -1,6 +1,6 @@
 #========================================================================================
 # Parthenon performance portable AMR framework
-# Copyright(C) 2021 The Parthenon collaboration
+# Copyright(C) 2021-2026 The Parthenon collaboration
 # Licensed under the 3-clause BSD License, see LICENSE file for details
 #========================================================================================
 # (C) (or copyright) 2021. Triad National Security, LLC. All rights reserved.
@@ -45,9 +45,15 @@ if (${MACHINE_VARIANT} MATCHES "mpi")
   #set(TEST_MPIEXEC mpiexec CACHE STRING "Command to launch MPI applications")
   list(APPEND TEST_MPIOPTS "--allow-run-as-root")
   set(HDF5_ROOT /usr/local/hdf5/parallel CACHE STRING "HDF5 path")
+  set(PARTHENON_USE_SYSTEM_OPENPMD ON CACHE BOOL "Use API in container")
+
 else()
   set(HDF5_ROOT /usr/local/hdf5/serial CACHE STRING "HDF5 path")
   set(PARTHENON_DISABLE_MPI ON CACHE BOOL "Disable MPI")
+  # testing auto fetch and compile
+  set(PARTHENON_USE_SYSTEM_OPENPMD OFF CACHE BOOL "Use API in container")
 endif()
 
+set(PARTHENON_DISABLE_OPENPMD OFF CACHE BOOL "Always use OpenPMD build in CI env")
+
 set(CMAKE_CXX_FLAGS "${MACHINE_CXX_FLAGS}" CACHE STRING "Default flags for this config")
diff --git a/doc/sphinx/src/building.rst b/doc/sphinx/src/building.rst
@@ -18,6 +18,8 @@ General list of cmake options:
 || PARTHENON\_SINGLE\_PRECISION             || OFF                           || Option || Enable single precision mode if requested                                                                                                                   |
 || PARTHENON\_DISABLE\_HDF5                 || OFF                           || Option || HDF5 is enabled by default if found, set this to True to disable HDF5                                                                                       |
 || PARTHENON\_DISABLE_HDF5\_COMPRESSION     || OFF                           || Option || HDF5 compression is enabled by default, set this to True to disable compression in HDF5 output/restart files                                                |
+|| PARTHENON\_DISABLE\_OPENPMD              || ON                            || Option || OpenPMD is disabled by default (to be changed in the future). Set this to `OFF` to enable OpenPMD                                                           |
+|| PARTHENON\_USE\_SYSTEM\_OPENPMD          || OFF                           || Option || OpenPMD by default is built along Parthenon. To use a system version (e.g., available as `module` set this to `ON`.                                         |
 || PARTHENON\_ENABLE\_ASCENT                || OFF                           || Option || Enable Ascent for in situ visualization and analysis                                                                                                        |
 || PARTHENON\_DISABLE\_MPI                  || OFF                           || Option || MPI is enabled by default if found, set this to True to disable MPI                                                                                         |
 || PARTHENON\_ENABLE\_HOST\_COMM\_BUFFERS   || OFF                           || Option || MPI communication buffers are by default allocated on the execution device. This options forces allocation in memory accessible directly by the host.       |

diff --git a/doc/sphinx/src/interface/state.rst b/doc/sphinx/src/interface/state.rst
@@ -50,7 +50,7 @@ several useful features and functions.
   restart file and updated upon restart. In contrast, ``Mutable``
   params not marked ``Restart`` are updated only by user code, not
   automatically. Note that not all parameter types can be output to
-  HDF5 file. However, most common scalar, vector, and ``Kokkos`` view
+  restart files. However, most common scalar, vector, and ``Kokkos`` view
   types are supported. Note also that if the value of a ``Param`` is
   different on different MPI ranks, this will result in undefined
   behaviour.