Updates for [email protected] (#20)

.cmake-format.py

@@ -0,0 +1,20 @@
+# -*- Python -*-
+
+with section("format"):
+
+    # How wide to allow formatted cmake files
+    line_width = 100
+
+    # How many spaces to tab for indent
+    tab_size = 4
+
+    # If true, separate flow control names from their parentheses with a space
+    separate_ctrl_name_with_space = False
+
+    # If true, separate function names from parentheses with a space
+    separate_fn_name_with_space = False
+
+    # If a statement is wrapped to more than one line, than dangle the closing
+    # parenthesis on its own line.
+    dangle_parens = True
+

.gitignore

@@ -0,0 +1,2 @@
+/build
+/builddir

CMakeLists.txt

@@ -1,73 +1,170 @@
-cmake_minimum_required(VERSION 3.12)
-project(xsdk-examples
-        DESCRIPTION "xSDK Examples"
-        LANGUAGES CXX C)
+cmake_minimum_required(VERSION 3.21)
+project(
+    xsdk-examples
+    DESCRIPTION "xSDK Examples"
+    LANGUAGES CXX C
+)
 
 set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
+include(CTest)
 include(FindPackageHandleStandardArgs)
+include(XsdkAddTest)
 
 # build options
 option(ENABLE_CUDA "Enable CUDA" OFF)
-option(CMAKE_CUDA_ARCHITECTURES "CUDA architecture(s) to target" "70")
+option(ENABLE_HIP "Enable HIP" OFF)
+
+option(ENABLE_AMREX "Enable AMReX" ON)
+set(AMREX_DIR
+    "${AMREX_DIR}"
+    CACHE PATH "Path to AMReX installation directory"
+)
+
+option(ENABLE_GINKGO "Enable Ginkgo" ON)
+set(Ginkgo_DIR
+    "${Ginkgo_DIR}"
+    CACHE PATH "Path to Ginkgo installation directory"
+)
 
 option(ENABLE_HYPRE "Enable hypre" ON)
-set(HYPRE_DIR "${HYPRE_DIR}" CACHE PATH "Path to hypre installation directory")
+set(HYPRE_DIR
+    "${HYPRE_DIR}"
+    CACHE PATH "Path to hypre installation directory"
+)
 
 option(ENABLE_MFEM "Enable MFEM" ON)
-set(MFEM_DIR "${MFEM_DIR}" CACHE PATH "Path to MFEM installation directory")
+set(MFEM_DIR
+    "${MFEM_DIR}"
+    CACHE PATH "Path to MFEM installation directory"
+)
 
 option(ENABLE_MAGMA "Enable MAGMA" OFF)
-set(MAGMA_DIR "${MAGMA_DIR}" CACHE PATH "Path to MAGMA installation directory")
+set(MAGMA_DIR
+    "${MAGMA_DIR}"
+    CACHE PATH "Path to MAGMA installation directory"
+)
 
 option(ENABLE_PETSC "Enable PETSc" ON)
-set(PETSC_DIR "${PETSC_DIR}" CACHE PATH "Path to PETSc installation directory")
+set(PETSc_DIR
+    "${PETSc_DIR}"
+    CACHE PATH "Path to PETSc installation directory"
+)
+
+option(ENABLE_PLASMA "Enable PLASMA" ON)
+set(PLASMA_DIR
+    "${PLASMA_DIR}"
+    CACHE PATH "Path to PLASMA installation directory"
+)
 
 option(ENABLE_SUNDIALS "Enable SUNDIALS" ON)
-set(SUNDIALS_DIR "${SUNDIALS_DIR}" CACHE PATH "Path to SUNDIALS installation directory")
+set(SUNDIALS_DIR
+    "${SUNDIALS_DIR}"
+    CACHE PATH "Path to SUNDIALS installation directory"
+)
 
 option(ENABLE_SUPERLU "Enable SuperLU" ON)
-set(SUPERLUDIST_DIR "${SUPERLUDIST__DIR}" CACHE PATH "Path to SuperLU_DIST installation directory")
+set(SUPERLUDIST_DIR
+    "${SUPERLUDIST_DIR}"
+    CACHE PATH "Path to SuperLU_DIST installation directory"
+)
+
+option(ENABLE_STRUMPACK "Enable STRUMPACK" OFF)
+set(STRUMPACK_DIR
+    "${STRUMPACK_DIR}"
+    CACHE PATH "Path to STRUMPACK installation directory"
+)
+
+option(ENABLE_TRILINOS "Enable TRILINOS" OFF)
+set(TRILINOS_DIR
+    "${Trilinos_DIR}"
+    CACHE PATH "Path to Trilinos installation directory"
+)
+
+set(METIS_DIR
+    "${METIS_DIR}"
+    CACHE PATH "Path to Metis installation directory"
+)
 
-option(ENABLE_TRILINOS "Enable TRILINOS" ON)
-set(TRILINOS_DIR "${Trilinos_DIR}" CACHE PATH "Path to Trilinos installation directory")
+# check for MPI
+find_package(MPI REQUIRED)
 
-set(METIS_DIR "${METIS_DIR}" CACHE PATH "Path to Metis installation directory")
+# check for OpenMP
+find_package(OpenMP)
 
+# compiler options
+if(NOT DEFINED CMAKE_CXX_STANDARD)
+    set(CMAKE_CXX_STANDARD 14)
+    set(CMAKE_CXX_STANDARD_REQUIRED TRUE)
+    set(CMAKE_CXX_EXTENSIONS OFF)
+endif()
+
+# setup CUDA
 if(ENABLE_CUDA)
-  enable_language(CUDA)
-  set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER})
+    enable_language(CUDA)
+    set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER})
+    find_package(CUDAToolkit REQUIRED)
+endif()
+
+if(ENABLE_HIP)
+    enable_language(HIP)
+    find_package(hip REQUIRED)
+    find_package(hipsparse REQUIRED)
+    find_package(hiprand REQUIRED)
+    find_package(rocrand REQUIRED)
+    find_package(rocprim REQUIRED)
+    find_package(rocsparse REQUIRED)
+endif()
+
+# check for AMReX
+if(ENABLE_AMREX)
+    find_package(AMReX REQUIRED)
 endif()
 
 # check for hypre
 if(ENABLE_HYPRE)
-  find_package(HYPRE REQUIRED)
+    find_package(HYPRE REQUIRED)
 endif()
 
 # check for MFEM
 if(ENABLE_MFEM)
-  find_package(ZLIB REQUIRED)
-  find_package(MFEM REQUIRED)
-  find_package(Ginkgo REQUIRED)
+    find_package(ZLIB REQUIRED)
+    find_package(MFEM REQUIRED)
+    if(ENABLE_GINKGO)
+        find_package(Ginkgo REQUIRED)
+    endif()
 endif()
 
 # check for MAGMA
 if(ENABLE_MAGMA)
-  find_package(MAGMA REQUIRED)
+    find_package(MAGMA REQUIRED)
 endif()
 
 # check for PETSC
 if(ENABLE_PETSC)
-  find_package(PETSC REQUIRED)
+    find_package(PETSc REQUIRED)
+endif()
+
+# check for PLASMA
+if(ENABLE_PLASMA)
+    find_package(PLASMA REQUIRED)
+    find_package(BLASPP REQUIRED)
+    find_package(LAPACKPP REQUIRED)
+    find_package(SLATE REQUIRED)
 endif()
 
 # check for SUNDIALS
 if(ENABLE_SUNDIALS)
-  find_package(SUNDIALS REQUIRED)
+    find_package(SUNDIALS REQUIRED)
 endif()
 
 # check for SuperLU DIST
 if(ENABLE_SUPERLU)
-  find_package(SUPERLUDIST REQUIRED)
+    find_package(SUPERLUDIST REQUIRED)
+endif()
+
+# check for STRUMPACK
+if(ENABLE_STRUMPACK)
+    find_package(STRUMPACK REQUIRED)
 endif()
 
 # check for math
@@ -76,22 +173,28 @@ find_library(MATH_LIBRARY NAMES m)
 # check for metis
 find_package(METIS)
 
-# check for MPI
-find_package(MPI REQUIRED)
-
 # example subdirectories
+if(ENABLE_AMREX)
+    add_subdirectory(amrex)
+endif()
 if(ENABLE_HYPRE)
-  add_subdirectory(hypre)
+    add_subdirectory(hypre)
 endif()
 if(ENABLE_MFEM)
-  add_subdirectory(mfem)
+    add_subdirectory(mfem)
 endif()
 if(ENABLE_PETSC)
-  add_subdirectory(petsc)
+    add_subdirectory(petsc)
+endif()
+if(ENABLE_PLASMA)
+    add_subdirectory(plasma)
 endif()
 if(ENABLE_SUNDIALS)
-  add_subdirectory(sundials)
+    add_subdirectory(sundials)
 endif()
 if(ENABLE_TRILINOS)
-  add_subdirectory(trilinos)
+    add_subdirectory(trilinos)
+endif()
+if(ENABLE_STRUMPACK)
+    add_subdirectory(strumpack)
 endif()

README.md

@@ -1,4 +1,4 @@
-# xSDK Examples
+# xSDK Examples v0.3.0
 
 The example codes provided here demonstrate the use of of various xSDK libraries in tandem to solve problems of 
 interest.  Each of the library folders has one or more examples codes that are built of that library 
@@ -7,34 +7,77 @@ examining the output is a good way to better understand how these libraries can
 code samples are a good place to start for new projects.  More details about the examples can be found 
 in the README.md files in the library subfolders.  For more information on the xSDK see <https://xsdk.info/>.
 
-## Code Example Summary
-
-|   Example                                  | Libraries                   | Description                                       |
-|:-------------------------------------------|:----------------------------|:--------------------------------------------------|
-|  hypre/ij_laplacian.c                      | HYPRE+SuperLU_Dist          | 2D Laplacian problem                              |
-|  libensemble/test_persistent_aposmm_tao.py | libEnsemble+PETSc           | 2D constrained optimization problem               |
-|  mfem/hypre-superlu/convdiff.cpp           | MFEM+HYPRE+SuperLU_Dist     | 2D steady state convective diffusion              |
-|  mfem/ginkgo/mfem_ex1_gko.cpp              | MFEM+Ginkgo                 | 2D Poisson problem with Ginko solver              |
-|  mfem/petsc/obstacle.cpp                   | MFEM+PETSc                  | Membrane obstacle problem (min energy functional) |
-|  mfem/sundials/transient-heat.cpp          | MFEM+SUNDIALS               | 2D Transient nonlinear heat conduction            |
-|  petsc/ex19.c                              | PETSc+HYPRE+SuperLU_Dist    | 2D nonlinear driven cavity problem                |
-|  sundials/ark_brusselator1D_FEM_sludist.cpp| SUNDIALS+SuperLU_Dist       | Chemical kinetics brusselator problem             |
-|  sundials/cv_petsc_ex7.c                   | SUNDIALS+PETSc              | 2D nonlinear PDE solution                         |
-|  trilinos/SimpleSolve_WithParameters.cpp   | Trilinos+SuperLU_Dist       | Small linear system direct solution               |
+## Example Summary
+
+These examples were tested and verified against [email protected]. 
+
+| Example                                               | Libraries                | Description                                       | GPUs           |
+|:------------------------------------------------------|:-------------------------|:--------------------------------------------------|:---------------|
+| hypre/ij_laplacian.c                                  | HYPRE+SuperLU_Dist       | 2D Laplacian problem                              |                |
+| libensemble/test_persistent_aposmm_tao.py             | libEnsemble+PETSc        | 2D constrained optimization problem               |                |
+| mfem/hypre-superlu/convdiff.cpp                       | MFEM+HYPRE+SuperLU_Dist  | 2D steady state convective diffusion              |                |
+| mfem/ginkgo/mfem_ex1_gko.cpp                          | MFEM+Ginkgo              | 2D Poisson problem with Ginko solver              | ![cuda]        |
+| mfem/petsc/obstacle.cpp                               | MFEM+PETSc               | Membrane obstacle problem (min energy functional) |                |
+| mfem/strumpack/diffusion-eigen.cpp                    | MFEM+STRUMPACK+HYPRE     | Diffusion eigenvalue problem                      |                |
+| mfem/sundials/transient-heat.cpp                      | MFEM+SUNDIALS            | 2D Transient nonlinear heat conduction            |                |
+| mfem/hypre/magnetic-diffusion.cpp                     | MFEM+HYPRE               | Steady state magnetic diffusion problem           | ![cuda]        |
+| mfem/sundials/advection.cpp                           | MFEM+SUNDIALS            | 2D Time-dependent advection                       | ![cuda]        |
+| petsc/ex19.c                                          | PETSc+HYPRE+SuperLU_Dist | 2D nonlinear driven cavity problem                | ![cuda]        |
+| plasma/ex1solve.c                                     | PLASMA+SLATE+BLASPP      | Linear system direct solution                     | ![cuda]        |
+| sundials/ark_brusselator1D_FEM_sludist.cpp            | SUNDIALS+SuperLU_Dist    | Chemical kinetics brusselator problem             |                |
+| sundials/cv_petsc_ex7.c                               | SUNDIALS+PETSc           | 2D nonlinear PDE solution                         |                |
+| sundials/cvRoberts_blockdiag_magma.cpp                | SUNDIALS+MAGMA           | Solves a group of chemical kinetics ODEs          | ![cuda] ![hip] | 
+| trilinos/SimpleSolve_WithParameters.cpp               | Trilinos+SuperLU_Dist    | Small linear system direct solution               |                |
+| strumpack/sparse.cpp                                  | STRUMPACK+ButterflyPACK  | 3D Poisson problem with STRUMPACK preconditioner  |                |
 
 These examples are currently in the repo but will not be enabled in the xsdk-examples spack package until we release a new version of the xSDK.
 They can still be built using CMake directly.
 
-|   Example                                  | Libraries                   | Description                                       |
-|:-------------------------------------------|:----------------------------|:--------------------------------------------------|
-|  mfem/sundials/advection.cpp               | MFEM+SUNDIALS (CUDA)        | 2D Time-dependent advection                       |
-| sundials/cvRoberts_blockdiag_magma.cpp     | SUNDIALS+MAGMA (CUDA)       | Solves a group of chemical kinetics ODEs          |
+| Example                                               | Libraries                | Description                                       | GPUs           |
+|:------------------------------------------------------|:-------------------------|:--------------------------------------------------|:---------------|
+| amrex/sundials/amrex_sundials_advection_diffusion.cpp | AMReX+SUNDIALS           | 2D Advection-diffusion problem                    | ![cuda] ![hip] |
+| mfem/hypre/magnetic-diffusion.cpp                     | MFEM+HYPRE               | Steady state magnetic diffusion problem           | ![hip]         |
+
+## Installing the Examples
 
-## Install the code samples
+The examples can be installed along with the xSDK utilizing the Spack package.
 
-The examples can be installed along with the xSDK utilizing the spack package.
 ```
 spack install xsdk-examples
 ```
 
-Further details on how to run each example code can be found in each example folder's README.md file.
+To install with CUDA support,
+
+```
+spack install xsdk-examples+cuda cuda_arch=<arch>
+```
+
+Since `xsdk-examples` depends on the `xsdk` Spack package, Spack will also install `xsdk`. In some cases, it may be easier to install the `xsdk` package (separately) following https://xsdk.info/download/ prior to the `xsdk-examples` package. 
+
+Alternatively the examples can be built and installed with CMake directly:
+
+```
+git clone https://github.com/xsdk-project/xsdk-examples
+cmake -DCMAKE_PREFIX_PATH=/path/to/libraries -DENABLE_CUDA=<TRUE|FALSE> -DENABLE_HIP=<TRUE|FALSE> -S xsdk-examples/ -B xsdk-examples/builddir
+cd xsdk-examples/builddir
+make
+make install
+```
+
+Note, that to build with HIP support CMake must be used directly.
+
+## Running and Testing
+
+xsdk-examples is setup to use `ctest`. Each example in the repository is tested with at least a set of default options. If CMake is used to build xsdk-examples, the tests can be run from the build directory (`builddir` above):
+```
+ctest .
+```
+or
+```
+make test
+```
+Details on how to run each example code manually (and with different options) can be found in each example folder's README.md file.
+
+
+[cuda]: https://img.shields.io/badge/-cuda-brightgreen?style=flat "CUDA"
+[hip]: https://img.shields.io/badge/-hip-red?style=flat "HIP"

amrex/CMakeLists.txt

@@ -0,0 +1,6 @@
+# AMReX+SUNDIALS requires [email protected]: and [email protected]:
+if(ENABLE_SUNDIALS AND ("${AMReX_RELEASE_NUMBER}" VERSION_GREATER_EQUAL "22.04") AND ("${SUNDIALS_PACKAGE_VERSION}" VERSION_GREATER_EQUAL 6.2.0))
+    add_subdirectory(sundials)
+else()
+    message(STATUS "SKIPPED AMReX+SUNDIALS example because AMReX version is too old or SUNDIALS version is too old (need 22.04: and 6.2.0:)")
+endif()

amrex/sundials/CMakeLists.txt

@@ -0,0 +1,17 @@
+add_executable(amrex_sundials_advection_diffusion amrex_sundials_advection_diffusion.cpp)
+
+if(ENABLE_CUDA)
+    set_source_files_properties(amrex_sundials_advection_diffusion.cpp PROPERTIES LANGUAGE CUDA)
+    set_target_properties(
+        amrex_sundials_advection_diffusion PROPERTIES CUDA_SEPARABLE_COMPILATION ON
+    ) # This adds -dc
+endif()
+
+target_link_libraries(amrex_sundials_advection_diffusion PRIVATE XSDK::AMReX XSDK::SUNDIALS MPI::MPI_CXX)
+
+xsdk_add_test(
+    NAME AMREX-amrex_sundials_advection_diffusion COMMAND
+    $<TARGET_FILE:amrex_sundials_advection_diffusion>
+)
+
+install(TARGETS amrex_sundials_advection_diffusion RUNTIME DESTINATION bin)