Merge branch 'development' into nse_bailout

.codespell-ignore-words

@@ -11,3 +11,4 @@ thi
 nd
 ue
 bion
+aas

.github/workflows/c-linter.yml

@@ -24,6 +24,16 @@ jobs:
           sudo apt-get update -y -qq
           sudo apt-get -qq -y install curl clang-tidy cmake jq clang cppcheck clang-format bear g++>=9.3.0 gfortran>=9.3.0
 
+      - name: Install hypre
+        run: |
+          wget -q https://github.com/hypre-space/hypre/archive/refs/tags/v2.28.0.tar.gz
+          tar xfz v2.28.0.tar.gz
+          cd hypre-2.28.0/src
+          ./configure --with-cxxstandard=17 --without-MPI
+          make -j 2
+          make install
+          cd ../../
+
       - name: Get cpp linter repo
         run: |
           cd external
@@ -38,6 +48,7 @@ jobs:
 
       - name: Run cpp linter
         run: |
+          export AMREX_HYPRE_HOME=${PWD}/hypre-2.28.0/src/hypre
           python3 external/cpp-linter-action/run_on_changed_files.py ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} \
           -ignore-files="amrex|Microphysics" \
           -config-file="${GITHUB_WORKSPACE}/.clang-tidy" \

.github/workflows/gpu_action.yml

@@ -31,7 +31,17 @@ jobs:
           sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub
           sudo add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /"
           sudo apt-get update
-          sudo apt-get -y install cuda
+          sudo apt-get -y install cuda-toolkit-11-8
+
+      - name: Install hypre
+        run: |
+          wget -q https://github.com/hypre-space/hypre/archive/refs/tags/v2.28.0.tar.gz
+          tar xfz v2.28.0.tar.gz
+          cd hypre-2.28.0/src
+          CUDA_HOME=/usr/local/cuda HYPRE_CUDA_SM=60 ./configure --with-cxxstandard=17 --with-cuda --enable-unified-memory --without-MPI
+          make -j 2
+          make install
+          cd ../../
 
       - name: Get cpp linter repo
         run: |
@@ -48,4 +58,5 @@ jobs:
       - name: Compile problems for GPU
         run: |
           export PATH=$PATH:/usr/local/cuda/bin
+          export AMREX_HYPRE_HOME=${PWD}/hypre-2.28.0/src/hypre
           python3 external/cpp-linter-action/run_on_changed_files.py ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} -header-filter=Castro -ignore-files="amrex|Microphysics" -gpu

.github/workflows/mhd-compare.yml

@@ -0,0 +1,45 @@
+name: OrszagTang
+
+on: [pull_request]
+jobs:
+  OrszagTang-3d:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Get submodules
+        run: |
+          git submodule update --init
+          cd external/Microphysics
+          git fetch; git checkout development
+          cd ../amrex
+          git fetch; git checkout development
+          cd ../..
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update -y -qq
+          sudo apt-get -qq -y install curl cmake jq clang g++>=9.3.0 libopenmpi-dev openmpi-bin
+
+      - name: Compile OrszagTang
+        run: |
+          cd Exec/mhd_tests/OrszagTang
+          make USE_MPI=TRUE -j 2
+
+      - name: Run OrszagTang-3d
+        run: |
+          cd Exec/mhd_tests/OrszagTang
+          mpirun -np 2 ./Castro3d.gnu.MPI.ex inputs.test amr.plot_files_output=1
+
+      - name: Build the fextrema tool
+        run: |
+          cd external/amrex/Tools/Plotfile
+          make programs=fextrema -j 2
+
+      - name: Check the extrema
+        run: |
+          cd Exec/mhd_tests/OrszagTang
+          ../../../external/amrex/Tools/Plotfile/fextrema.gnu.ex plt00010 > fextrema.out
+          diff fextrema.out ci-benchmarks/OrszagTang-3d.out

.github/workflows/reacting-convergence-true-sdc.yml

@@ -0,0 +1,48 @@
+name: reacting-convergence-true-sdc
+
+on: [pull_request]
+jobs:
+  reacting-convergence-true-sdc:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Get submodules
+        run: |
+          git submodule update --init
+          cd external/Microphysics
+          git fetch; git checkout development
+          cd ../amrex
+          git fetch; git checkout development
+          cd ../..
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update -y -qq
+          sudo apt-get -qq -y install curl g++>=9.3.0
+
+      - name: Compile reacting_convergence
+        run: |
+          cd Exec/reacting_tests/reacting_convergence
+          make DIM=2 USE_MPI=FALSE -j 2 USE_TRUE_SDC=TRUE
+
+      - name: Run reacting_convergence
+        run: |
+          cd Exec/reacting_tests/reacting_convergence
+          ./Castro2d.gnu.TRUESDC.ex inputs.128.testsuite max_step=10
+
+      - name: Build the fextrema tool
+        run: |
+          cd external/amrex/Tools/Plotfile
+          make programs=fextrema -j 2
+
+      - name: Check the extrema
+        run: |
+          cd Exec/reacting_tests/reacting_convergence
+          ../../../external/amrex/Tools/Plotfile/fextrema.gnu.ex react_converge_128_plt00010 > fextrema.out
+          # ignore angular momenta, since 0 and -0 don't compare
+          diff -I "^ angular_momentum" fextrema.out ci-benchmarks/react_converge_128_true_sdc.out
+
+

CHANGES.md

@@ -1,5 +1,12 @@
 # 23.07
 
+  * The parameter castro.state_nghost, which allowed State_Type to have ghost
+    zones, has been removed. (#2502)
+
+  * The additional ghost zone in State_Type, used when radiation is enabled,
+    has been removed. The checkpoint version number has been updated to avoid
+    restarting from a checkpoint with the wrong number of ghost zones. (#2495)
+
   * The parameter gravity.no_composite was removed (#2483)
 
   * The parameter spherical_star was removed (#2482)

Docs/source/FlowChart.rst

@@ -56,8 +56,7 @@ The time-integration method used is controlled by
     order integration implemented.  At the moment, this does not support
     multilevel domains.  Note: because of differences in the interfaces with the 
     default Strang method, you must compile with ``USE_TRUE_SDC = TRUE`` for this
-    method to work (in particular, this defines ``EXTRA_THERMO`` which enables some
-    additional EOS derivatives).
+    method to work.
 
   * ``time_integration_method = 3``: this is the simplified SDC method
     described above that uses the CTU hydro advection and an ODE

Docs/source/rotation.rst

@@ -224,7 +224,7 @@ a rotating frame
 
 
 The kinetic energy equation can be obtained from :eq:`eq:v-rot` by
-mulitplying by :math:`\rho\vbt`:
+multiplying by :math:`\rho\vbt`:
 
 .. math::
    \begin{align}

Docs/source/software.rst

@@ -256,10 +256,7 @@ The current ``StateData`` names Castro carries are:
       simply be advected, but we will allow rotation (in particular,
       the Coriolis force) to affect them.
 
-   ``State_Type`` ``MultiFab`` s have no ghost cells by default for
-   pure hydro and a single ghost cell by default when ``RADIATION``
-   is enabled. There is an option to force them to have ghost cells by
-   setting the parameter ``castro.state_nghost`` at runtime.
+   ``State_Type`` ``MultiFab`` s have no ghost cells.
 
    Note that the prediction of the hydrodynamic state to the interface
    will require 4 ghost cells. This accommodated by creating a separate

Exec/Make.Castro

@@ -186,18 +186,6 @@ endif
 ifeq ($(USE_REACT), TRUE)
   Bdirs += Source/reactions
   DEFINES += -DREACTIONS
-
-  ifeq ($(USE_TRUE_SDC), TRUE)
-    # we need the compositional derivatives for SDC
-    DEFINES += -DEXTRA_THERMO
-  endif
-
-  ifeq ($(USE_SIMPLIFIED_SDC), TRUE)
-    # we need the compositional derivatives for SDC
-    DEFINES += -DEXTRA_THERMO
-    # we only implement this for C++ reactions
-  endif
-
 endif
 
 ifeq ($(USE_REACT_SPARSE_JACOBIAN), TRUE)

Exec/mhd_tests/OrszagTang/ci-benchmarks/OrszagTang-3d.out

@@ -0,0 +1,19 @@
+ plotfile = plt00010
+ time = 0.00017983829222995601
+              variables          minimum value          maximum value
+ density                         0.22099460191          0.22100305489
+ xmom                            -0.2210626652           0.2210626652
+ ymom                            -0.2211744105           0.2211744105
+ zmom                        -1.3394504115e-20       1.2796480966e-20
+ rho_E                           0.19805685456          0.46260143162
+ rho_e                           0.19794567403          0.19795835472
+ Temp                         7.2178022365e-09       7.2179887773e-09
+ rho_X                           0.22099460191          0.22100305489
+ pressure                         0.1326236016          0.13263209766
+ x_velocity                      -1.0002839593           1.0002839593
+ y_velocity                      -1.0007881112           1.0007881112
+ z_velocity                  -6.0608661064e-20       5.7902777063e-20
+ B_x                            -0.28206060038          0.28206060038
+ B_y                            -0.28220293547          0.28220293547
+ B_z                         -1.5570588641e-18        1.630216695e-18
+

Exec/reacting_tests/reacting_bubble/inputs_2d_test

@@ -23,10 +23,6 @@ castro.yr_ext_bc_type = 1
 
 castro.hse_interp_temp = 1
 
-
-# this is here just for test suite coverage
-castro.state_nghost = 1
-
 # WHICH PHYSICS
 castro.do_hydro = 1
 castro.do_react = 1

Exec/reacting_tests/reacting_bubble/inputs_3d_test

@@ -27,9 +27,6 @@ castro.fill_ambient_bc = 1
 castro.ambient_fill_dir = 2
 castro.ambient_outflow_vel = 1
 
-# this is here just for test suite coverage
-castro.state_nghost = 1
-
 # WHICH PHYSICS
 castro.do_hydro = 1
 castro.do_react = 1

Exec/reacting_tests/reacting_convergence/ci-benchmarks/react_converge_128_true_sdc.out

@@ -0,0 +1,59 @@
+ plotfile = react_converge_128_plt00010
+ time = 0.000901
+              variables          minimum value          maximum value
+ density                          499999.99996           1003267.4037
+ xmom                        -1.6405409399e+12       1.6405409399e+12
+ ymom                        -1.6405409399e+12       1.6405409399e+12
+ zmom                                        0                      0
+ rho_E                         2.263117857e+22       7.0166374924e+22
+ rho_e                         2.263117857e+22       7.0166358971e+22
+ Temp                             300032898.05           454530914.94
+ rho_He4                          499997.75752           1002591.9747
+ rho_C12                          2.2423384461           675.42840306
+ rho_O16                      5.0003881018e-05       0.00047398074259
+ rho_Fe56                     4.9999999996e-05       0.00010032674037
+ rho_enuc                     1.4565744228e+21       4.5249976081e+23
+ density_sdc_source              -544925.79439           78513.247881
+ xmom_sdc_source             -1.8266340473e+15       1.8266340473e+15
+ ymom_sdc_source             -1.8266340473e+15       1.8266340473e+15
+ zmom_sdc_source                             0                      0
+ rho_E_sdc_source            -6.1992465072e+22       7.5781242372e+21
+ rho_e_sdc_source            -6.1984074245e+22       6.3082607661e+21
+ Temp_sdc_source                             0                      0
+ rho_He4_sdc_source              -543231.29031           78510.973912
+ rho_C12_sdc_source              -1694.4762124           10.073671104
+ rho_O16_sdc_source            -0.027819745447       5.9867619328e-06
+ rho_Fe56_sdc_source         -5.4492579439e-05       7.8513247881e-06
+ pressure                     1.4155320805e+22         4.26081309e+22
+ kineng                       1.0456708952e-13       1.6647276223e+18
+ soundspeed                       212069503.63           257404342.33
+ Gamma_1                          1.5601126451           1.5885713572
+ MachNumber                   3.0496449897e-18        0.0086982771588
+ entropy                           348439780.9           349209883.71
+ magvort                      2.2088105346e-29       0.00018541051833
+ divu                           -0.11633879119          0.55816306183
+ eint_E                       4.5262357143e+16       6.9937847751e+16
+ eint_e                       4.5262357143e+16       6.9937843801e+16
+ logden                           5.6989700043           6.0014167022
+ StateErr_0                       499999.99996           1003267.4037
+ StateErr_1                       300032898.05           454530914.94
+ StateErr_2                      0.99932677073          0.99999551512
+ X(He4)                          0.99932677073          0.99999551512
+ X(C12)                       4.4846768926e-06       0.00067322869314
+ X(O16)                       1.0000776204e-10       4.7243709987e-10
+ X(Fe56)                                 1e-10                  1e-10
+ abar                             4.0000119598           4.0017960844
+ Ye                                        0.5                    0.5
+ x_velocity                      -2064636.8119           2064636.8119
+ y_velocity                      -2064636.8119           2064636.8119
+ z_velocity                                  0                      0
+ t_sound_t_enuc               0.00023710314718         0.019573278445
+ enuc                         2.9131488458e+15       4.5102607654e+17
+ magvel                       6.4673669922e-10           2067446.1361
+ radvel                      -0.00067837157832            2067446.136
+ circvel                                     0           11.820185992
+ magmom                       0.00032336834958       1.6422547233e+12
+ angular_momentum_x                         -0                     -0
+ angular_momentum_y                         -0                     -0
+ angular_momentum_z          -1.2410862709e+14        1.241086269e+14
+

Exec/science/flame_wave/analysis/vis_3d/vol-xrb-abar.py

@@ -1,16 +1,19 @@
 #!/usr/bin/env python
 
-import matplotlib
-matplotlib.use('agg')
-
 import sys
 
+import matplotlib
+import numpy as np
+
 import yt
 from yt.frontends.boxlib.api import CastroDataset
-import numpy as np
-#from yt.visualization.volume_rendering.render_source import VolumeSource
-from yt.visualization.volume_rendering.api import create_volume_source, Scene
 from yt.units import cm
+#from yt.visualization.volume_rendering.render_source import VolumeSource
+from yt.visualization.volume_rendering.api import Scene, create_volume_source
+
+matplotlib.use('agg')
+
+resolution = (1920, 1080)
 
 # this is for the wdconvect problem
 
@@ -56,7 +59,7 @@ def doit(plotfile):
     sc.get_source(0).transfer_function = tf
 
     cam = sc.add_camera(ds, lens_type="perspective")
-    cam.resolution = (1920, 1280)
+    cam.resolution = resolution
 
     # view 1
 
@@ -77,14 +80,24 @@ def doit(plotfile):
     cam.zoom(3.0)
     sc.camera = cam
 
-    sc.save_annotated("{}_abar_annotated_side.png".format(plotfile),
-                      sigma_clip=3.0,
+    sc.save(f"{plotfile}_abar_noaxes_side.png", sigma_clip=3.0)
+
+    sc.annotate_axes(alpha=0.005, thickness=6)
+
+    sc.save(f"{plotfile}_abar_side.png", sigma_clip=3.0)
+
+    sc.save_annotated(f"{plotfile}_abar_annotated_side.png",
+                      sigma_clip=3.0, label_fmt="%.2f",
                       text_annotate=[[(0.05, 0.05),
                                       f"t = {ds.current_time.d:7.5f} s",
                                       dict(horizontalalignment="left")]])
 
     # view 2
 
+    # remove the annotation source for now
+    print(list(sc.sources.keys()))
+    sc.sources.pop("source_01")
+
     dx = ds.domain_right_edge[0] - ds.domain_left_edge[0]
     cam.position = [0.5*(ds.domain_left_edge[0] + ds.domain_right_edge[0]) + 0.0001 * dx,
                     0.5*(ds.domain_left_edge[1] + ds.domain_right_edge[1]),
@@ -103,8 +116,13 @@ def doit(plotfile):
     cam.zoom(0.6)
     sc.camera = cam
 
+    sc.save(f"{plotfile}_abar_noaxes_top.png", sigma_clip=3.0)
+
+    sc.annotate_axes(alpha=0.005, thickness=6)
+
+    sc.save(f"{plotfile}_abar_top.png", sigma_clip=3.0)
     sc.save_annotated("{}_abar_annotated_top.png".format(plotfile),
-                      sigma_clip=3.0,
+                      sigma_clip=3.0, label_fmt="%.2f",
                       text_annotate=[[(0.05, 0.05),
                                       f"t = {ds.current_time.d:7.5f} s",
                                       dict(horizontalalignment="left")]])

Exec/science/subchandra/GNUmakefile

@@ -22,7 +22,7 @@ USE_SIMPLIFIED_SDC=TRUE
 EOS_DIR     := helmholtz
 
 # This sets the network directory in $(MICROPHYSICS_HOME)/networks
-NETWORK_DIR := subch_approx
+NETWORK_DIR := subch_simple
 
 INTEGRATOR_DIR := VODE