add nvhpc to test suite github runners

.github/workflows/test.yml

@@ -149,7 +149,7 @@ jobs:
         continue-on-error: true
 
   github:
-    name: Github
+    name: ${{ matrix.nvhpc && format('NVHPC {0} ({1})', matrix.nvhpc, matrix.target) || format('Github ({0}, {1}, {2}, intel={3})', matrix.os, matrix.mpi, matrix.debug, matrix.intel) }}
     needs: [lint-gate, file-changes, rebuild-cache]
     if: >-
       !cancelled() &&
@@ -164,6 +164,8 @@ jobs:
         precision: ['']
         debug: ['debug', 'no-debug']
         intel: [true, false]
+        nvhpc: ['']
+        target: ['']
         exclude:
           - os:    macos
             intel: true
@@ -175,11 +177,78 @@ jobs:
             debug: no-debug
             intel: false
 
+          # NVHPC compiler matrix (CPU build+test, GPU build-only)
+          - nvhpc: '23.11'
+            target: cpu
+          - nvhpc: '23.11'
+            target: gpu-acc
+          - nvhpc: '23.11'
+            target: gpu-omp
+          - nvhpc: '24.5'
+            target: cpu
+          - nvhpc: '24.5'
+            target: gpu-acc
+          - nvhpc: '24.5'
+            target: gpu-omp
+          - nvhpc: '24.11'
+            target: cpu
+          - nvhpc: '24.11'
+            target: gpu-acc
+          - nvhpc: '24.11'
+            target: gpu-omp
+          - nvhpc: '25.1'
+            target: cpu
+          - nvhpc: '25.1'
+            target: gpu-acc
+          - nvhpc: '25.1'
+            target: gpu-omp
+          - nvhpc: '25.3'
+            target: cpu
+          - nvhpc: '25.3'
+            target: gpu-acc
+          - nvhpc: '25.3'
+            target: gpu-omp
+          - nvhpc: '25.5'
+            target: cpu
+          - nvhpc: '25.5'
+            target: gpu-acc
+          - nvhpc: '25.5'
+            target: gpu-omp
+          - nvhpc: '25.7'
+            target: cpu
+          - nvhpc: '25.7'
+            target: gpu-acc
+          - nvhpc: '25.7'
+            target: gpu-omp
+          - nvhpc: '26.1'
+            target: cpu
+          - nvhpc: '26.1'
+            target: gpu-acc
+          - nvhpc: '26.1'
+            target: gpu-omp
+          - nvhpc: '26.3'
+            target: cpu
+          - nvhpc: '26.3'
+            target: gpu-acc
+          - nvhpc: '26.3'
+            target: gpu-omp
+
       fail-fast: false
     continue-on-error: true
-    runs-on: ${{ matrix.os }}-latest
+    runs-on: ${{ matrix.nvhpc && 'ubuntu-22.04' || format('{0}-latest', matrix.os) }}
+    container: ${{ matrix.nvhpc && format('nvcr.io/nvidia/nvhpc:{0}-devel-cuda_multi-ubuntu22.04', matrix.nvhpc) || '' }}
+    env:
+      CC:  ${{ matrix.nvhpc && 'nvc' || '' }}
+      CXX: ${{ matrix.nvhpc && 'nvc++' || '' }}
+      FC:  ${{ matrix.nvhpc && 'nvfortran' || '' }}
+      OMPI_ALLOW_RUN_AS_ROOT:         ${{ matrix.nvhpc && '1' || '' }}
+      OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: ${{ matrix.nvhpc && '1' || '' }}
 
     steps:
+      - name: Git safe directory
+        if:   matrix.nvhpc
+        run:  git config --global --add safe.directory /__w/MFC/MFC
+
       - name: Clone
         uses: actions/checkout@v4
 
@@ -222,7 +291,7 @@ jobs:
           fi
 
       - name: Setup MacOS
-        if:   matrix.os == 'macos'
+        if:   matrix.os == 'macos' && !matrix.nvhpc
         run:  |
           brew update
           brew upgrade || true
@@ -231,7 +300,7 @@ jobs:
           echo "BOOST_INCLUDE=/opt/homebrew/include/" >> $GITHUB_ENV
 
       - name: Setup Ubuntu
-        if:   matrix.os == 'ubuntu' && matrix.intel == false
+        if:   matrix.os == 'ubuntu' && matrix.intel == false && !matrix.nvhpc
         run: |
            sudo apt update -y
            sudo apt install -y cmake gcc g++ python3 python3-dev hdf5-tools \
@@ -254,21 +323,53 @@ jobs:
           printenv | sort > /tmp/env_after
           diff /tmp/env_before /tmp/env_after | grep '^>' | sed 's/^> //' >> $GITHUB_ENV
 
+      # --- NVHPC container setup ---
+      - name: Setup NVHPC
+        if:   matrix.nvhpc
+        run: |
+          apt-get update -y
+          apt-get install -y cmake python3 python3-venv python3-pip \
+                             libfftw3-dev libhdf5-dev hdf5-tools git
+          MPI_LIB=$(dirname "$(mpifort --showme:link | grep -oP '/\S+/libmpi\S+' | head -1)")
+          echo "LD_LIBRARY_PATH=${MPI_LIB}:${LD_LIBRARY_PATH}" >> $GITHUB_ENV
+
+      # --- Standard build + test ---
       - name: Build
+        if:   '!matrix.nvhpc'
         run:  |
           /bin/bash mfc.sh test -v --dry-run -j $(nproc) --${{ matrix.debug }} --${{ matrix.mpi }} $PRECISION $TEST_ALL
         env:
           TEST_ALL: ${{ matrix.mpi == 'mpi' && '--test-all' || '' }}
           PRECISION: ${{ matrix.precision != '' && format('--{0}', matrix.precision) || '' }}
 
       - name: Test
+        if:   '!matrix.nvhpc'
         run:  |
           /bin/bash mfc.sh test -v --max-attempts 3 -j $(nproc) $ONLY_CHANGES $TEST_ALL $TEST_PCT
         env:
           TEST_ALL: ${{ matrix.mpi == 'mpi' && '--test-all' || '' }}
           TEST_PCT: ${{ matrix.debug == 'debug' && '-% 20' || '' }}
           ONLY_CHANGES: ${{ github.event_name == 'pull_request' && '--only-changes' || '' }}
 
+      # --- NVHPC build + test ---
+      - name: Build (NVHPC)
+        if:   matrix.nvhpc
+        run:  /bin/bash mfc.sh test -v --dry-run -j ${{ matrix.target == 'cpu' && '$(nproc)' || '2' }} --test-all $GPU
+        env:
+          GPU: ${{ matrix.target == 'gpu-acc' && '--gpu acc' || matrix.target == 'gpu-omp' && '--gpu mp' || '' }}
+
+      - name: Test (NVHPC)
+        if:   matrix.nvhpc && matrix.target == 'cpu'
+        run:  |
+          echo "=== NVHPC CPU test env debug ==="
+          echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH"
+          echo "PATH=$PATH"
+          echo "which mpirun: $(which mpirun)"
+          echo "which nvfortran: $(which nvfortran)"
+          ldd $(find build/install -name simulation -type f | head -1) 2>/dev/null | head -20
+          echo "=== end debug ==="
+          /bin/bash mfc.sh test -v --max-attempts 3 -j $(nproc) --test-all
+
   self:
     name: "${{ matrix.cluster_name }} (${{ matrix.device }}${{ matrix.interface != 'none' && format('-{0}', matrix.interface) || '' }}${{ matrix.shard != '' && format(' [{0}]', matrix.shard) || '' }})"
     needs: [lint-gate, file-changes, rebuild-cache]

CMakeLists.txt

@@ -262,11 +262,6 @@ if (CMAKE_BUILD_TYPE STREQUAL "Release")
     elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC")
         if (MFC_Unified)
             message(STATUS "LTO/IPO is not available with NVHPC using Unified Memory")
-        elseif (CMAKE_Fortran_COMPILER_VERSION VERSION_GREATER "24.11" AND CMAKE_Fortran_COMPILER_VERSION VERSION_LESS "25.9")
-            message(STATUS "LTO/IPO is not supported in NVHPC Version 24.11 to 25.9. Use >=25.9 or (<=24.11 && > 23.11) Performance will be degraded.")
-            set(NVHPC_USE_TWO_PASS_IPO FALSE)
-        elseif(CMAKE_Fortran_COMPILER_VERSION VERSION_LESS "23.11")
-            message(STATUS "LTO/IPO is not supported in NVHPC Version < 23.11. Use a newer version of NVHPC for best performance.")
         else()
             message(STATUS "Performing IPO using -Mextract followed by -Minline")
             set(NVHPC_USE_TWO_PASS_IPO TRUE)
@@ -453,7 +448,20 @@ function(MFC_SETUP_TARGET)
 	        $<$<COMPILE_LANGUAGE:Fortran>:-Minline>
     	)
         add_dependencies(${ARGS_TARGET} ${ARGS_TARGET}_lib)
-        target_compile_options(${ARGS_TARGET} PRIVATE -Minline=lib:${ARGS_TARGET}_lib)
+        target_compile_options(${ARGS_TARGET} PRIVATE -Minline=lib:${ARGS_TARGET}_lib,except:f_is_default,except:s_compute_dt,except:my_inquire,except:s_mpi_abort,except:s_mpi_barrier,except:s_prohibit_abort,except:s_int_to_str,except:s_associate_cbc_coefficients_pointers)
+
+        # Exclude m_start_up and m_cbc from cross-file inlining: these are
+        # initialization/boundary code that trigger NVHPC 25.x fort2 ICE when
+        # too many functions are cross-inlined into them. GPU hot-path files
+        # (m_rhs, m_riemann_solvers, m_viscous, m_weno, etc.) keep full IPO.
+        foreach(_no_inline_file m_start_up m_cbc)
+            set_source_files_properties(
+                "${CMAKE_BINARY_DIR}/fypp/${ARGS_TARGET}/${_no_inline_file}.fpp.f90"
+                TARGET_DIRECTORY ${ARGS_TARGET}
+                PROPERTIES COMPILE_OPTIONS "-Mnoinline"
+            )
+        endforeach()
+
         list(PREPEND IPO_TARGETS ${ARGS_TARGET}_lib)
     endif()
 

src/simulation/m_riemann_solvers.fpp

@@ -2205,7 +2205,8 @@ contains
                                         & G_L, G_R, rho_avg, H_avg, c_avg, gamma_avg, ptilde_L, ptilde_R, vel_L_rms, vel_R_rms, &
                                         & vel_avg_rms, vel_L_tmp, vel_R_tmp, Ms_L, Ms_R, pres_SL, pres_SR, alpha_L_sum, &
                                         & alpha_R_sum, rho_Star, E_Star, p_Star, p_K_Star, vel_K_star, s_L, s_R, s_M, s_P, s_S, &
-                                        & xi_M, xi_P, xi_L, xi_R, xi_MP, xi_PP]')
+                                        & xi_M, xi_P, xi_L, xi_R, xi_MP, xi_PP, Ys_L, Ys_R, Cp_iL, Cp_iR, Xs_L, Xs_R, Gamma_iL, &
+                                        & Gamma_iR, Yi_avg, Phi_avg, h_iL, h_iR, h_avg_2]')
                     do l = is3%beg, is3%end
                         do k = is2%beg, is2%end
                             do j = is1%beg, is1%end
@@ -2423,7 +2424,8 @@ contains
                                         & qv_R, qv_avg, c_L, c_R, c_avg, vel_L_rms, vel_R_rms, vel_avg_rms, vel_L_tmp, vel_R_tmp, &
                                         & Ms_L, Ms_R, pres_SL, pres_SR, alpha_L_sum, alpha_R_sum, s_L, s_R, s_M, s_P, s_S, xi_M, &
                                         & xi_P, xi_L, xi_R, xi_MP, xi_PP, nbub_L, nbub_R, PbwR3Lbar, PbwR3Rbar, R3Lbar, R3Rbar, &
-                                        & R3V2Lbar, R3V2Rbar]')
+                                        & R3V2Lbar, R3V2Rbar, Ys_L, Ys_R, Cp_iL, Cp_iR, Xs_L, Xs_R, Gamma_iL, Gamma_iR, Yi_avg, &
+                                        & Phi_avg, h_iL, h_iR, h_avg_2]')
                     do l = is3%beg, is3%end
                         do k = is2%beg, is2%end
                             do j = is1%beg, is1%end

toolchain/dependencies/CMakeLists.txt

@@ -61,11 +61,13 @@ if (MFC_HDF5)
             GIT_PROGRESS   ON
             CMAKE_ARGS     "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}"
                             -DBUILD_SHARED_LIBS=OFF
-                            -DFORTRAN_LIBRARIES=ON
                             -DBUILD_TESTING=OFF
                             -DHDF5_BUILD_UTILS=OFF
                             -DHDF5_BUILD_TOOLS=ON
                             -DHDF5_BUILD_EXAMPLES=OFF
+                            -DHDF5_BUILD_CPP_LIB=OFF
+                            -DHDF5_BUILD_FORTRAN=OFF
+                            -DHDF5_ENABLE_PARALLEL=OFF
         )
     endif()
 endif()
@@ -82,13 +84,15 @@ if (MFC_SILO)
 
         ExternalProject_Add(silo
             GIT_REPOSITORY "https://github.com/LLNL/Silo"
-            GIT_TAG        0cddaa6865fb49ae7b2110fa1b6f2709592abe0b
+            GIT_TAG        4.12.0
+            GIT_SHALLOW    ON
             GIT_PROGRESS   ON
             PATCH_COMMAND  "${GIT_EXECUTABLE}" stash
                         && "${GIT_EXECUTABLE}" apply "${CMAKE_SOURCE_DIR}/Silo.patch"
             CMAKE_ARGS     -DSILO_ENABLE_SHARED=OFF
                            -DSILO_ENABLE_SILOCK=OFF
                            -DSILO_ENABLE_BROWSER=OFF
+                           -DSILO_ENABLE_ZFP=OFF
                            -DFIND_LIBRARY_USE_LIB64_PATHS=ON
                            "-DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}"
                            "-DCMAKE_FIND_ROOT_PATH=${CMAKE_FIND_ROOT_PATH}"