diff --git a/.github/actions/check-skip-labels/README.md b/.github/actions/check-skip-labels/README.md deleted file mode 100644 index 42d1e3bc64e..00000000000 --- a/.github/actions/check-skip-labels/README.md +++ /dev/null @@ -1,24 +0,0 @@ -# Composite action to check if we can skip a job for a PR - -This action is meant to be used inside a PR testing workflow, as - -```yaml -jobs: - my-testing: - steps: - ... - - name: check-pr-labels - if: github.event_name == "pull_request" || github.event_name == "pull_request_review" - uses: ./.github/actions/check-skip-labels - with: - skip_labels: label1,label2,label3 -``` - -The input skip_label is a comma-separated list of labels that, if found -on the PR, will cause this job to terminate immediately with a PASS state. - -Ideally, we would like to run this check at the job level, so that we can -skip the job altogether (without using runner time). But while for the -pull_request event we DO have access to the labels from the gh context -(and therefore can check), for pull_request_review we don't, so we need -to ping github for some information diff --git a/.github/actions/check-skip-labels/action.yml b/.github/actions/check-skip-labels/action.yml deleted file mode 100644 index 55d35bfa2b0..00000000000 --- a/.github/actions/check-skip-labels/action.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: 'Check Skip Labels' -description: 'Check for specific skip labels in a pull request' -inputs: - skip_labels: - description: 'Comma-separated list of skip labels' - required: true - token: - description: 'GitHub token for authentication' - required: true - pr_number: - description: 'Pull request number' - required: true - -# Note: inputs are available as env vars in the shell run steps, convertet to uppercase - -runs: - using: "composite" - steps: - - name: Get Pull Request Labels - run: | - echo "Fetching pull request labels..." - if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then - LABELS="$SKIP_LABELS" - elif [[ "$GITHUB_EVENT_NAME" == "pull_request_review" ]]; then - response=$(curl -s -H "Authorization: token $TOKEN" \ - "https://api.github.com/repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER") - # TODO: reinstante jq once the SNL image is rebuilt! - # LABELS=$(echo "$response" | jq -r '.labels | map(.name) | join(",")') - LABELS=$(echo "$response" | grep -o '"name": *"[^"]*"' | sed 's/"name": *//;s/"//g' | tr '\n' ',' | sed 's/,$//') - fi - echo "labels=$LABELS" >> $GITHUB_ENV - shell: sh - - name: Check for Skip Labels - run: | - echo "Checking for skip labels..." - IFS=',' read -r -a SKIP_LABELS <<< "$SKIP_LABELS" - IFS=',' read -r -a LABEL_ARRAY <<< "$labels" - - for label in "${SKIP_LABELS[@]}"; do - for pr_label in "${LABEL_ARRAY[@]}"; do - if [[ "$pr_label" == "$label" ]]; then - echo "Found skip label '$label'. Skipping this job." - exit 0 # Exit with success status - fi - done - done - echo "No relevant skip labels found. Continuing with the job." - shell: sh diff --git a/.github/workflows/e3sm-gh-ci-cime-tests.yml b/.github/workflows/e3sm-gh-ci-cime-tests.yml index 04f7fcb4ffc..5c6ff081f73 100644 --- a/.github/workflows/e3sm-gh-ci-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-cime-tests.yml @@ -22,10 +22,14 @@ on: workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: ci: - if: false + if: ${{ github.repository == 'E3SM-Project/E3SM' }} runs-on: ubuntu-latest strategy: fail-fast: false @@ -36,7 +40,7 @@ jobs: - SMS_D_Ln5_P4.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.ghci-oci_gnu - ERS_Ld5_P4.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.ghci-oci_gnu.eamxx-prod container: - image: ghcr.io/e3sm-project/containers-ghci:ghci-0.1.0 + image: ghcr.io/e3sm-project/containers-ghci:ghci-0.2.0 steps: - diff --git a/.github/workflows/e3sm-gh-ci-w-cime-tests.yml b/.github/workflows/e3sm-gh-ci-w-cime-tests.yml index 48c367c8f62..f51aa88a34c 100644 --- a/.github/workflows/e3sm-gh-ci-w-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-w-cime-tests.yml @@ -1,4 +1,4 @@ -name: gh +name: gh-w on: pull_request: @@ -11,10 +11,14 @@ on: workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: - ci-w: - if: ${{ github.event.repository.name == 'e3sm' }} + ci: + if: ${{ github.repository == 'E3SM-Project/E3SM' }} runs-on: ubuntu-latest strategy: fail-fast: false @@ -23,7 +27,7 @@ jobs: - SMS_D_Ld1_P8.ne4pg2_oQU480.WCYCL2010NS.ghci-oci_gnu - ERS_Ld3_P8.ne4pg2_oQU480.WCYCL2010NS.ghci-oci_gnu.allactive-wcprod_1850 container: - image: ghcr.io/e3sm-project/containers-ghci:ghci-0.1.0 + image: ghcr.io/e3sm-project/containers-ghci:ghci-0.2.0 steps: - diff --git a/.github/workflows/e3sm-gh-md-linter.yml b/.github/workflows/e3sm-gh-md-linter.yml index 424a871637b..ad24487695e 100644 --- a/.github/workflows/e3sm-gh-md-linter.yml +++ b/.github/workflows/e3sm-gh-md-linter.yml @@ -10,8 +10,13 @@ on: # for now let's not lint files in eamxx - '!components/eamxx/**/*.md' +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: linter: + if: ${{ github.repository == 'E3SM-Project/E3SM' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -22,7 +27,7 @@ jobs: with: files: '**/*.md' separator: "," - - uses: DavidAnson/markdownlint-cli2-action@v17 + - uses: DavidAnson/markdownlint-cli2-action@v18 if: steps.changed-files.outputs.any_changed == 'true' with: config: 'docs/.markdownlint.json' diff --git a/.github/workflows/e3sm-gh-pages.yml b/.github/workflows/e3sm-gh-pages.yml index ebd2ac9c1e9..dec9bc696bf 100644 --- a/.github/workflows/e3sm-gh-pages.yml +++ b/.github/workflows/e3sm-gh-pages.yml @@ -15,7 +15,7 @@ concurrency: jobs: Build-and-Deploy-docs: - if: ${{ github.event.repository.name == 'e3sm' }} + if: ${{ github.repository == 'E3SM-Project/E3SM' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml b/.github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml index 8fe212886d9..cacb951b8a8 100644 --- a/.github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml +++ b/.github/workflows/e3sm-gh-tools-mkatmsrffile-test.yml @@ -11,8 +11,13 @@ on: - cron: '00 15 * * 2' workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: mkatmsrffile-test: + if: ${{ github.repository == 'E3SM-Project/E3SM' }} runs-on: ubuntu-latest defaults: run: @@ -31,10 +36,7 @@ jobs: uses: conda-incubator/setup-miniconda@v3 with: activate-environment: "envmkatmsrffile" - miniforge-variant: Mambaforge miniforge-version: latest - use-mamba: true - mamba-version: "*" channel-priority: strict auto-update-conda: true python-version: 3.11 @@ -42,7 +44,7 @@ jobs: name: Install dependencies run: | echo $CONDA_PREFIX - mamba install -y nco xarray numba numpy netcdf4 + conda install -y nco xarray numba numpy netcdf4 -c conda-forge - name: Run tests working-directory: components/eam/tools/mkatmsrffile diff --git a/.github/workflows/eamxx-gh-ci-standalone.yml b/.github/workflows/eamxx-gh-ci-standalone.yml new file mode 100644 index 00000000000..19a2ec9cd8e --- /dev/null +++ b/.github/workflows/eamxx-gh-ci-standalone.yml @@ -0,0 +1,60 @@ +name: gh-standalone + +on: + pull_request: + branches: [ master ] + paths: + # first, yes to these + - '.github/workflows/eamxx-gh-ci-standalone.yml' + - 'cime_config/machine/config_machines.xml' + - 'components/eamxx/**' + - 'components/homme/**' + # second, no to these + - '!components/eamxx/docs/**' + - '!components/eamxx/mkdocs.yml' + + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + +jobs: + + ci: + if: ${{ github.repository == 'E3SM-Project/E3SM' }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + test: + - sp + - opt + - dbg + - fpe + container: + image: ghcr.io/e3sm-project/containers-standalone-ghci:standalone-ghci-0.1.0 + + steps: + - + name: Checkout + uses: actions/checkout@v4 + with: + show-progress: false + submodules: recursive + - + name: standalone + env: + SHELL: sh + run: | + # TODO: get rid of this extra line if we can? + git config --global safe.directory '*' + ./components/eamxx/scripts/test-all-scream -m ghci-oci -t ${{ matrix.test }} -c BUILD_SHARED_LIBS=ON + - + name: Artifacts + uses: actions/upload-artifact@v4 + if: ${{ always() }} + with: + name: ${{ matrix.test }} + path: | + components/eamxx/ctest-build/*/Testing/Temporary/Last*.log diff --git a/.github/workflows/eamxx-gh-pages.yml b/.github/workflows/eamxx-gh-pages.yml index 28415b54dbd..2e763c544cd 100644 --- a/.github/workflows/eamxx-gh-pages.yml +++ b/.github/workflows/eamxx-gh-pages.yml @@ -34,7 +34,7 @@ concurrency: jobs: eamxx-docs: - if: ${{ github.event.repository.name == 'scream' }} + if: ${{ github.repository == 'E3SM-Project/scream' }} runs-on: ubuntu-latest steps: diff --git a/.github/workflows/eamxx-sa-coverage.yml b/.github/workflows/eamxx-sa-coverage.yml index aa69d6263e6..46e87f8b8b0 100644 --- a/.github/workflows/eamxx-sa-coverage.yml +++ b/.github/workflows/eamxx-sa-coverage.yml @@ -2,6 +2,11 @@ name: eamxx-sa-coverage on: workflow_dispatch: + inputs: + submit: + description: 'Force cdash submission' + required: true + type: boolean # Add schedule trigger for nightly runs at midnight MT (Standard Time) schedule: @@ -13,7 +18,8 @@ concurrency: cancel-in-progress: true env: - submit: ${{ github.event_name == 'schedule' && 'true' || 'false' }} # Submit to cdash only for nightlies + # Submit to cdash only for nightlies or if the user explicitly forced a submission via workflow dispatch + submit: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.submit) }} jobs: gcc-openmp: @@ -48,6 +54,35 @@ jobs: submodules: recursive - name: Show action trigger uses: ./.github/actions/show-workflow-trigger + - name: Get CUDA Arch + run: | + # Ensure nvidia-smi is available + if ! command -v nvidia-smi &> /dev/null; then + echo "nvidia-smi could not be found. Please ensure you have Nvidia drivers installed." + exit 1 + fi + + # Get the GPU model from nvidia-smi, and set env for next step + gpu_model=$(nvidia-smi --query-gpu=name --format=csv,noheader | head -n 1) + case "$gpu_model" in + *"H100"*) + echo "Hopper=ON" >> $GITHUB_ENV + echo "CUDA_ARCH=90" >> $GITHUB_ENV + ARCH=90 + ;; + *"A100"*) + echo "Ampere=ON" >> $GITHUB_ENV + echo "CUDA_ARCH=80" >> $GITHUB_ENV + ;; + *"V100"*) + echo "Volta=ON" >> $GITHUB_ENV + echo "CUDA_ARCH=70" >> $GITHUB_ENV + ;; + *) + echo "Unsupported GPU model: $gpu_model" + exit 1 + ;; + esac - name: Run tests uses: ./.github/actions/test-all-scream with: @@ -55,4 +90,4 @@ jobs: machine: ghci-snl-cuda generate: false submit: ${{ env.submit }} - cmake-configs: Kokkos_ARCH_VOLTA70=ON;CMAKE_CUDA_ARCHITECTURES=70 + cmake-configs: Kokkos_ARCH_HOPPER90=${{ env.Hopper }};Kokkos_ARCH_AMPERE80=${{ env.Ampere }};Kokkos_ARCH_VOLTA70=${{ env.Volta }};CMAKE_CUDA_ARCHITECTURES=${{ env.CUDA_ARCH }} diff --git a/.github/workflows/eamxx-sa-sanitizer.yml b/.github/workflows/eamxx-sa-sanitizer.yml index 7e3a1a49fcf..00f60f7da2e 100644 --- a/.github/workflows/eamxx-sa-sanitizer.yml +++ b/.github/workflows/eamxx-sa-sanitizer.yml @@ -2,6 +2,11 @@ name: eamxx-sa-sanitizer on: workflow_dispatch: + inputs: + submit: + description: 'Force cdash submission' + required: true + type: boolean # Add schedule trigger for nightly runs at midnight MT (Standard Time) schedule: @@ -13,12 +18,13 @@ concurrency: cancel-in-progress: true env: - submit: ${{ github.event_name == 'schedule' && 'true' || 'false' }} # Submit to cdash only for nightlies + # Submit to cdash only for nightlies or if the user explicitly forced a submission via workflow dispatch + submit: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.submit) }} jobs: gcc-openmp: runs-on: [self-hosted, ghci-snl-cpu, gcc] - name: gcc-openmp / cov + name: gcc-openmp / valg steps: - name: Check out the repository uses: actions/checkout@v4 @@ -52,6 +58,35 @@ jobs: submodules: recursive - name: Show action trigger uses: ./.github/actions/show-workflow-trigger + - name: Get CUDA Arch + run: | + # Ensure nvidia-smi is available + if ! command -v nvidia-smi &> /dev/null; then + echo "nvidia-smi could not be found. Please ensure you have Nvidia drivers installed." + exit 1 + fi + + # Get the GPU model from nvidia-smi, and set env for next step + gpu_model=$(nvidia-smi --query-gpu=name --format=csv,noheader | head -n 1) + case "$gpu_model" in + *"H100"*) + echo "Hopper=ON" >> $GITHUB_ENV + echo "CUDA_ARCH=90" >> $GITHUB_ENV + ARCH=90 + ;; + *"A100"*) + echo "Ampere=ON" >> $GITHUB_ENV + echo "CUDA_ARCH=80" >> $GITHUB_ENV + ;; + *"V100"*) + echo "Volta=ON" >> $GITHUB_ENV + echo "CUDA_ARCH=70" >> $GITHUB_ENV + ;; + *) + echo "Unsupported GPU model: $gpu_model" + exit 1 + ;; + esac - name: Run tests uses: ./.github/actions/test-all-scream with: @@ -59,4 +94,4 @@ jobs: machine: ghci-snl-cuda generate: false submit: ${{ env.submit }} - cmake-configs: Kokkos_ARCH_VOLTA70=ON;CMAKE_CUDA_ARCHITECTURES=70 + cmake-configs: Kokkos_ARCH_HOPPER90=${{ env.Hopper }};Kokkos_ARCH_AMPERE80=${{ env.Ampere }};Kokkos_ARCH_VOLTA70=${{ env.Volta }};CMAKE_CUDA_ARCHITECTURES=${{ env.CUDA_ARCH }} diff --git a/.github/workflows/eamxx-sa-testing.yml b/.github/workflows/eamxx-sa-testing.yml index 12e45514ad0..a4397d4fdba 100644 --- a/.github/workflows/eamxx-sa-testing.yml +++ b/.github/workflows/eamxx-sa-testing.yml @@ -5,17 +5,6 @@ on: pull_request: branches: [ master ] types: [opened, synchronize, ready_for_review, reopened] - paths: - - components/eamxx/** - - components/eam/src/physics/rrtmgp/** - - components/eam/src/physics/p3/scream/** - - components/eam/src/physics/cam/** - - .github/workflows/eamxx-standalone-testing.yml - - externals/ekat/** - - externals/scorpio/** - - externals/haero/** - - externals/YAKL/** - - components/eam/src/physics/rrtmgp/external/** # Manual run is used to bless workflow_dispatch: @@ -32,6 +21,10 @@ on: description: 'Generate baselines' required: true type: boolean + submit: + description: 'Force cdash submission' + required: true + type: boolean # Add schedule trigger for nightly runs at midnight MT (Standard Time) schedule: @@ -45,18 +38,90 @@ concurrency: cancel-in-progress: true env: - submit: ${{ github.event_name == 'schedule' && 'true' || 'false' }} # Submit to cdash only for nightlies + # Submit to cdash only for nightlies or if the user explicitly forced a submission via workflow dispatch + submit: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.submit) }} jobs: + pre_process_pr: + if: ${{ github.event_name == 'pull_request' }} + runs-on: ubuntu-latest # This job can run anywhere + outputs: + relevant_paths: ${{ steps.check_paths.outputs.value }} + labels: ${{ steps.get_labels.outputs.labels }} + steps: + - name: Check files modified by PR + id: check_paths + run: | + paths=( + components/eamxx + components/eam/src/physics/rrtmgp + components/eam/src/physics/p3/scream + components/eam/src/physics/cam + components/eam/src/physics/rrtmgp/external + externals/ekat + externals/scorpio + externals/haero + externals/YAKL + .github/workflows/eamxx-sa-testing.yml + ) + pattern=$(IFS=\|; echo "${paths[*]}") + + # Use the GitHub API to get the list of changed files + # There are page size limits, so do it in chunks + page=1 + while true; do + response=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + "https://api.github.com/repos/E3SM-Project/scream/pulls/${{ github.event.number }}/files?per_page=100&page=$page") + + # Check if the response is empty, and break if it is + [ -z "$response" ] && break + + changed_files+=$(echo "$response" | grep -o '"filename": *"[^"]*"' | sed 's/"filename": *//; s/"//g')$'\n' + + # Check if there are more pages, and quite if there aren't + [[ $(echo "$response" | jq '. | length') -lt 100 ]] && break + + page=$((page + 1)) + done + + # Check for matches and echo the matching files (or "" if none) + matching_files=$(echo "$changed_files" | grep -E "^($pattern)" || echo "") + if [[ -n "$matching_files" ]]; then + echo "Found relevant files: $matching_files" + echo "value=true" >> $GITHUB_OUTPUT + else + echo "No relevant files touched by this PR." + echo "value=false" >> $GITHUB_OUTPUT + fi + - name: Retrieve PR labels + id: get_labels + run: | + labels="${{ join(github.event.pull_request.labels.*.name, ',') }}" + echo "labels=${labels}" >> $GITHUB_OUTPUT gcc-openmp: + needs: [pre_process_pr] + if: | + !failure() && !cancelled() && + ( + github.event_name == 'schedule' || + ( + github.event_name == 'pull_request' && + needs.pre_process_pr.outputs.relevant_paths=='true' && + !contains(needs.pre_process_pr.outputs.labels,'CI: skip gcc') && + !contains(needs.pre_process_pr.outputs.labels,'CI: skip openmp') && + !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-sa') && + !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-all') + ) || ( + github.event_name == 'workflow_dispatch' && + github.event.inputs.job_to_run == 'gcc-openmp' || + github.event.inputs.job_to_run == 'all' + ) + ) runs-on: [self-hosted, ghci-snl-cpu, gcc] strategy: fail-fast: false matrix: build_type: [sp, dbg, fpe, opt] - if: ${{ github.event_name != 'workflow_dispatch' || - github.event.inputs.job_to_run == 'gcc-openmp' || - github.event.inputs.job_to_run == 'all' }} name: gcc-openmp / ${{ matrix.build_type }} steps: - name: Check out the repository @@ -67,13 +132,6 @@ jobs: submodules: recursive - name: Show action trigger uses: ./.github/actions/show-workflow-trigger - - name: Check for skip labels - if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_review' }} - uses: ./.github/actions/check-skip-labels - with: - skip_labels: 'AT: skip gcc,AT: skip openmp,AT: skip eamxx-sa,AT: skip eamxx-all' - token: ${{ secrets.GITHUB_TOKEN }} - pr_number: ${{ github.event.pull_request.number }} - name: Set test-all inputs based on event specs run: | echo "generate=false" >> $GITHUB_ENV @@ -91,14 +149,29 @@ jobs: submit: ${{ env.submit }} cmake-configs: Kokkos_ENABLE_OPENMP=ON gcc-cuda: + needs: [pre_process_pr] + if: | + !failure() && !cancelled() && + ( + github.event_name == 'schedule' || + ( + github.event_name == 'pull_request' && + needs.pre_process_pr.outputs.relevant_paths=='true' && + !contains(needs.pre_process_pr.outputs.labels,'CI: skip gcc') && + !contains(needs.pre_process_pr.outputs.labels,'CI: skip cuda') && + !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-sa') && + !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-all') + ) || ( + github.event_name == 'workflow_dispatch' && + github.event.inputs.job_to_run == 'gcc-cuda' || + github.event.inputs.job_to_run == 'all' + ) + ) runs-on: [self-hosted, ghci-snl-cuda, cuda, gcc] strategy: fail-fast: false matrix: build_type: [sp, dbg, opt] - if: ${{ github.event_name != 'workflow_dispatch' || - github.event.inputs.job_to_run == 'gcc-cuda' || - github.event.inputs.job_to_run == 'all' }} name: gcc-cuda / ${{ matrix.build_type }} steps: - name: Check out the repository @@ -109,13 +182,6 @@ jobs: submodules: recursive - name: Show action trigger uses: ./.github/actions/show-workflow-trigger - - name: Check for skip labels - if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_review' }} - uses: ./.github/actions/check-skip-labels - with: - skip_labels: 'AT: skip gcc,AT: skip cuda,AT: skip eamxx-sa,AT: skip eamxx-all' - token: ${{ secrets.GITHUB_TOKEN }} - pr_number: ${{ github.event.pull_request.number }} - name: Set test-all inputs based on event specs run: | echo "generate=false" >> $GITHUB_ENV @@ -124,6 +190,35 @@ jobs: echo "generate=true" >> $GITHUB_ENV fi fi + - name: Get CUDA Arch + run: | + # Ensure nvidia-smi is available + if ! command -v nvidia-smi &> /dev/null; then + echo "nvidia-smi could not be found. Please ensure you have Nvidia drivers installed." + exit 1 + fi + + # Get the GPU model from nvidia-smi, and set env for next step + gpu_model=$(nvidia-smi --query-gpu=name --format=csv,noheader | head -n 1) + case "$gpu_model" in + *"H100"*) + echo "Hopper=ON" >> $GITHUB_ENV + echo "CUDA_ARCH=90" >> $GITHUB_ENV + ARCH=90 + ;; + *"A100"*) + echo "Ampere=ON" >> $GITHUB_ENV + echo "CUDA_ARCH=80" >> $GITHUB_ENV + ;; + *"V100"*) + echo "Volta=ON" >> $GITHUB_ENV + echo "CUDA_ARCH=70" >> $GITHUB_ENV + ;; + *) + echo "Unsupported GPU model: $gpu_model" + exit 1 + ;; + esac - name: Run tests uses: ./.github/actions/test-all-scream with: @@ -131,4 +226,4 @@ jobs: machine: ghci-snl-cuda generate: ${{ env.generate }} submit: ${{ env.submit }} - cmake-configs: Kokkos_ARCH_VOLTA70=ON;CMAKE_CUDA_ARCHITECTURES=70 + cmake-configs: Kokkos_ARCH_HOPPER90=${{ env.Hopper }};Kokkos_ARCH_AMPERE80=${{ env.Ampere }};Kokkos_ARCH_VOLTA70=${{ env.Volta }};CMAKE_CUDA_ARCHITECTURES=${{ env.CUDA_ARCH }} diff --git a/.github/workflows/eamxx-scripts-tests.yml b/.github/workflows/eamxx-scripts-tests.yml index bd719539bab..a14cdc4f350 100644 --- a/.github/workflows/eamxx-scripts-tests.yml +++ b/.github/workflows/eamxx-scripts-tests.yml @@ -5,18 +5,14 @@ on: pull_request: branches: [ master ] types: [opened, synchronize, ready_for_review, reopened] - paths: - - components/eamxx/scripts/** - - components/eamxx/cime_config/*.py - - .github/workflows/eamxx-scripts-tests.yml - - externals/ekat/** - - externals/scorpio/** - - externals/haero/** - - externals/YAKL/** - - components/eam/src/physics/rrtmgp/external/** # Manual run for debug purposes only workflow_dispatch: + inputs: + submit: + description: 'Force cdash submission' + required: true + type: boolean # Add schedule trigger for nightly runs at midnight MT (Standard Time) schedule: @@ -29,8 +25,73 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + # Submit to cdash only for nightlies or if the user explicitly forced a submission via workflow dispatch + submit: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.submit) }} + jobs: + pre_process_pr: + if: ${{ github.event_name == 'pull_request' }} + runs-on: ubuntu-latest # This job can run anywhere + outputs: + relevant_paths: ${{ steps.check_paths.outputs.value}} + labels: ${{ steps.get_labels.outputs.labels }} + steps: + - name: Check files modified by PR + id: check_paths + run: | + paths=( + components/eamxx/scripts + components/eamxx/cime_config/eamxx + components/eamxx/cime_config/build + components/eamxx/cime_config/yaml_utils.py + .github/workflows/eamxx-scripts-tests.yml + ) + pattern=$(IFS=\|; echo "${paths[*]}") + + # Use the GitHub API to get the list of changed files + # There are page size limits, so do it in chunks + page=1 + while true; do + response=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + "https://api.github.com/repos/E3SM-Project/scream/pulls/${{ github.event.number }}/files?per_page=100&page=$page") + + # Check if the response is empty, and break if it is + [ -z "$response" ] && break + + changed_files+=$(echo "$response" | grep -o '"filename": *"[^"]*"' | sed 's/"filename": *//; s/"//g')$'\n' + + # Check if there are more pages, and quite if there aren't + [[ $(echo "$response" | jq '. | length') -lt 100 ]] && break + + page=$((page + 1)) + done + + # Check for matches and echo the matching files (or "" if none) + matching_files=$(echo "$changed_files" | grep -E "^($pattern)" || echo "") + if [[ -n "$matching_files" ]]; then + echo "Found relevant files: $matching_files" + echo "value=true" >> $GITHUB_OUTPUT + else + echo "No relevant files touched by this PR." + echo "value=false" >> $GITHUB_OUTPUT + fi + - name: Retrieve PR labels + id: get_labels + run: | + labels="${{ join(github.event.pull_request.labels.*.name, ',') }}" + echo "labels=${labels}" >> $GITHUB_OUTPUT cpu-gcc: + needs: [pre_process_pr] + if: | + !failure() && !cancelled() && + ( + github.event_name != 'pull_request' || + ( + needs.pre_process_pr.outputs.relevant_paths == 'true' && + !contains(needs.pre_process_pr.outputs.labels, 'CI: skip eamxx-all') + ) + ) runs-on: [self-hosted, gcc, ghci-snl-cpu] steps: - name: Check out the repository @@ -41,17 +102,10 @@ jobs: submodules: recursive - name: Show action trigger uses: ./.github/actions/show-workflow-trigger - - name: Check for skip labels - if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_review' }} - uses: ./.github/actions/check-skip-labels - with: - skip_labels: 'AT: skip eamxx-all' - token: ${{ secrets.GITHUB_TOKEN }} - pr_number: ${{ github.event.pull_request.number }} - name: Run test run: | cd components/eamxx - if [ ${{ github.event_name == 'schedule' }} ]; then + if [ "${{ env.submit }}" == "true" ]; then ./scripts/scripts-ctest-driver -s -m ghci-snl-cpu else ./scripts/scripts-tests -f -m ghci-snl-cpu diff --git a/.github/workflows/eamxx-v1-testing.yml b/.github/workflows/eamxx-v1-testing.yml index 47129159f82..d55ed8252a5 100644 --- a/.github/workflows/eamxx-v1-testing.yml +++ b/.github/workflows/eamxx-v1-testing.yml @@ -5,17 +5,6 @@ on: pull_request: branches: [ master ] types: [opened, synchronize, ready_for_review, reopened] - paths: - - components/eamxx/** - - components/eam/src/physics/rrtmgp/** - - components/eam/src/physics/p3/scream/** - - components/eam/src/physics/cam/** - - .github/workflows/eamxx-v1-testing.yml - - externals/ekat/** - - externals/scorpio/** - - externals/haero/** - - externals/YAKL/** - - components/eam/src/physics/rrtmgp/external/** # Manual run is used to bless workflow_dispatch: @@ -40,7 +29,80 @@ concurrency: cancel-in-progress: true jobs: + pre_process_pr: + if: ${{ github.event_name == 'pull_request' }} + runs-on: ubuntu-latest # This job can run anywhere + outputs: + relevant_paths: ${{ steps.check_paths.outputs.value }} + labels: ${{ steps.get_labels.outputs.labels }} + steps: + - name: Check files modified by PR + id: check_paths + run: | + paths=( + components/eamxx + components/eam/src/physics/rrtmgp + components/eam/src/physics/p3/scream + components/eam/src/physics/cam + components/eam/src/physics/rrtmgp/external + externals/ekat + externals/scorpio + externals/haero + externals/YAKL + .github/workflows/eamxx-v1-testing.yml + ) + pattern=$(IFS=\|; echo "${paths[*]}") + + # Use the GitHub API to get the list of changed files + # There are page size limits, so do it in chunks + page=1 + while true; do + response=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + "https://api.github.com/repos/E3SM-Project/scream/pulls/${{ github.event.number }}/files?per_page=100&page=$page") + + # Check if the response is empty, and break if it is + [ -z "$response" ] && break + + changed_files+=$(echo "$response" | grep -o '"filename": *"[^"]*"' | sed 's/"filename": *//; s/"//g')$'\n' + + # Check if there are more pages, and quite if there aren't + [[ $(echo "$response" | jq '. | length') -lt 100 ]] && break + + page=$((page + 1)) + done + + # Check for matches and echo the matching files (or "" if none) + matching_files=$(echo "$changed_files" | grep -E "^($pattern)" || echo "") + if [[ -n "$matching_files" ]]; then + echo "Found relevant files: $matching_files" + echo "value=true" >> $GITHUB_OUTPUT + else + echo "No relevant files touched by this PR." + echo "value=false" >> $GITHUB_OUTPUT + fi + - name: Retrieve PR labels + id: get_labels + run: | + labels="${{ join(github.event.pull_request.labels.*.name, ',') }}" + echo "labels=${labels}" >> $GITHUB_OUTPUT cpu-gcc: + needs: [pre_process_pr] + if: | + !failure() && !cancelled() && + ( + github.event_name == 'schedule' || + ( + github.event_name == 'pull_request' && + needs.pre_process_pr.outputs.relevant_paths=='true' && + !contains(needs.pre_process_pr.outputs.labels,'CI: skip gcc') && + !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-v1') && + !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-all') + ) || ( + github.event_name == 'workflow_dispatch' && + github.event.inputs.job_to_run == 'cpu-gcc' || + github.event.inputs.job_to_run == 'all' + ) + ) runs-on: [self-hosted, gcc, ghci-snl-cpu] strategy: matrix: @@ -55,9 +117,6 @@ jobs: short_name: SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-all_mam4xx_procs fail-fast: false name: cpu-gcc / ${{ matrix.test.short_name }} - if: ${{ github.event_name != 'workflow_dispatch' || - github.event.inputs.job_to_run == 'cpu-gcc' || - github.event.inputs.job_to_run == 'all' }} steps: - name: Check out the repository uses: actions/checkout@v4 @@ -67,13 +126,6 @@ jobs: submodules: recursive - name: Show action trigger uses: ./.github/actions/show-workflow-trigger - - name: Check for skip labels - if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_review' }} - uses: ./.github/actions/check-skip-labels - with: - skip_labels: 'AT: skip gcc,AT: skip openmp,AT: skip eamxx-sa,AT: skip eamxx-all' - token: ${{ secrets.GITHUB_TOKEN }} - pr_number: ${{ github.event.pull_request.number }} - name: Set CA certificates env var run: | # Ensure the operating system is Linux diff --git a/.github/workflows/eamxx_default_files.yml b/.github/workflows/eamxx_default_files.yml index d3971758991..38c528306c4 100644 --- a/.github/workflows/eamxx_default_files.yml +++ b/.github/workflows/eamxx_default_files.yml @@ -11,9 +11,13 @@ on: - cron: '00 00 * * *' workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: scream-defaults: - if: false + if: ${{ github.repository == 'E3SM-Project/E3SM' }} runs-on: ubuntu-latest outputs: event_name: ${{ github.event_name }} diff --git a/.mergify.yml b/.mergify.yml new file mode 100644 index 00000000000..89fcc821e57 --- /dev/null +++ b/.mergify.yml @@ -0,0 +1,53 @@ +merge_protections: + - name: Enforce checks passing + description: Make sure that checks are not failing on the PR, and reviewers approved + if: + - base = master + success_conditions: + - "#approved-reviews-by >= 1" # At least 1 approval + - "#changes-requested-reviews-by == 0" # No reviewer asked for changes + - or: + - and: + - check-success="gcc-openmp / dbg" + - check-success="gcc-openmp / sp" + - check-success="gcc-openmp / fpe" + - check-success="gcc-openmp / opt" + - check-skipped={% raw %}gcc-openmp / ${{ matrix.build_type }}{% endraw %} + - or: + - and: + - check-success="gcc-cuda / dbg" + - check-success="gcc-cuda / sp" + - check-success="gcc-cuda / opt" + - check-skipped={% raw %}gcc-cuda / ${{ matrix.build_type }}{% endraw %} + - or: + - and: + - check-success="cpu-gcc / ERS_Ln9.ne4_ne4.F2000-SCREAMv1-AQP1.scream-output-preset-2" + - check-success="cpu-gcc / ERS_P16_Ln22.ne30pg2_ne30pg2.FIOP-SCREAMv1-DP.scream-dpxx-arm97" + - check-success="cpu-gcc / ERS_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-small_kernels--scream-output-preset-5" + - check-success="cpu-gcc / SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-all_mam4xx_procs" + - check-skipped={% raw %}cpu-gcc / ${{ matrix.test.short_name }}{% endraw %} + - or: + - check-success=cpu-gcc + - check-skipped=cpu-gcc + +pull_request_rules: + - name: dismiss stale reviews + conditions: + - base=master + actions: + dismiss_reviews: + when: synchronize # Dismiss reviews when synchronize event happens + - name: Automatic merge when CI passes and approved + conditions: + - "label=CI: automerge" + - base=master + actions: + merge: + method: merge + commit_message_template: | + Merge pull request #{{number}} from {{head}} + + Automatically merged using mergify + PR title: {{title}} + PR author: {{author}} + PR labels: {{label}} diff --git a/CITATION.cff b/CITATION.cff index 9542a7d01ef..78099055fc3 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -9,7 +9,7 @@ authors: - given-names: E3SM family-names: Project version: 3.0.0 -doi: 10.11578/E3SM/dc.20240301.3 +doi: 10.11578/E3SM/dc.20240930.1 repository-code: 'https://github.com/E3SM-Project/E3SM' url: 'https://e3sm.org' license: BSD-3-Clause diff --git a/LICENSE b/LICENSE index d74a2aa127a..247287ab4c0 100644 --- a/LICENSE +++ b/LICENSE @@ -2,7 +2,7 @@ Except for the separable pieces descibed below, E3SM is released under the following 3-Clause BSD Open Source license. ******************************************************************************* -Copyright ©2023, UChicago Argonne, LLC All Rights Reserved +Copyright 2024, UChicago Argonne, LLC All Rights Reserved Software Name: Energy Exascale Earth System Model (E3SM) @@ -55,15 +55,29 @@ GPTL share/timing author non-commeric MCT externals/mct ANL BSD YAKL externals/YAKL author BSD cub externals/cub author, NVIDIA BSD -kokkos externals/kokkos SNL BSD +kokkos externals/ekat SNL BSD +haero externals/haero SNL/Battelle BSD +mam4xx externals/mam4xx SNL/Battelle BSD Ocean/Ice under components/ ----------- ----------------- CICE cice LANL BSD MPAS Framework mpas-framework LANL BSD -MPAS Ocean mpas-ocean LANL BSD -MPAS SeaIce mpas-seaice LANL BSD -MPAS-Albany LandIce mpas-albany-landice LANL, SNL BSD +FFTW mpas-ocean/src/FFTW author/MIT GPL +MARBL mpas-ocean/src/MARBL NCAR BSD +SHTNS mpas-ocean/src/SHTNS CeCILL GPL +cvmix mpas-ocean/src/cvmix NCAR LGPL +gotm mpas-ocean/src/gotm authors GPL +ppr mpas-ocean/src/ppr author custom +Icepack mpas-seaice/src/icepack LANL BSD + +Waves under components/ww3 +----------- -------------------- +WW3 src/WW3 NWS/NOAA custom + +Land-ice under components/mpas-albany-landice +----------- ------------------------------------ +SeaLevelModel src/SeaLevelModel author MIT Land under components/elm/src ----------- ------------------------ @@ -91,7 +105,7 @@ HOMMEXX components/homme/src/share/cxx SNL BSD Actual copyright holder for above Institutions: NCAR = University Corporation for Atmospheric Research -LANL = Los Alamos National Security, LLC +LANL = Los Alamos National Security, LLC, Triad National Security, LLC SNL = National Technology & Engineering Solutions of Sandia, LLC LBNL = The Regents of the University of California, through Lawrence Berkeley National Laboratory diff --git a/README.md b/README.md index 192c288d78e..13c927b8c7f 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ + + [![E3SM Logo](https://e3sm.org/wp-content/themes/e3sm/assets/images/e3sm-logo.png)](https://e3sm.org) Energy Exascale Earth System Model (E3SM) @@ -9,14 +11,14 @@ the most challenging and demanding climate-change research problems and Department of Energy mission needs while efficiently using DOE Leadership Computing Facilities. -DOI: [10.11578/E3SM/dc.20240301.3](http://dx.doi.org/10.11578/E3SM/dc.20240301.3) +DOI: [10.11578/E3SM/dc.20240930.1](http://dx.doi.org/10.11578/E3SM/dc.20240930.1) Please visit the [project website](https://e3sm.org) or our [Confluence site](https://acme-climate.atlassian.net/wiki/spaces/DOC/overview) for further details. For questions about the model, use [Github Discussions](https://github.com/E3SM-Project/E3SM/discussions). -See our Github-hosted documentation at [https://e3sm-project.github.io/E3SM/](https://e3sm-project.github.io/E3SM/). +See our Github-hosted documentation at [https://docs.e3sm.org/E3SM](https://docs.e3sm.org/E3SM/). Table of Contents -------------------------------------------------------------------------------- diff --git a/cime_config/allactive/config_compsets.xml b/cime_config/allactive/config_compsets.xml index e66f1e473d8..d5a52a8bac3 100755 --- a/cime_config/allactive/config_compsets.xml +++ b/cime_config/allactive/config_compsets.xml @@ -383,42 +383,52 @@ CRYO1850 - 1850SOI_EAM%CMIP6_ELM%SPBC_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV + 1850SOI_EAM%CMIP6_ELM%CNPRDCTCBCTOP_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV CRYO1850-4xCO2 - 1850SOI_EAM%CMIP6-4xCO2_ELM%SPBC_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV + 1850SOI_EAM%CMIP6-4xCO2_ELM%CNPRDCTCBCTOP_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV + + + + CRYO1850-1pctCO2 + 1850SOI_EAM%CMIP6-1pctCO2_ELM%CNPRDCTCBCTOP_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV CRYO1950 - 1950SOI_EAM%CMIP6_ELM%SPBC_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV + 1950SOI_EAM%CMIP6_ELM%CNPRDCTCBCTOP_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV CRYO1850-DISMF - 1850SOI_EAM%CMIP6_ELM%SPBC_MPASSI%DIB_MPASO%IBDISMF_MOSART_SGLC_SWAV + 1850SOI_EAM%CMIP6_ELM%CNPRDCTCBCTOP_MPASSI%DIB_MPASO%IBDISMF_MOSART_SGLC_SWAV CRYO1950-DISMF - 1950SOI_EAM%CMIP6_ELM%SPBC_MPASSI%DIB_MPASO%IBDISMF_MOSART_SGLC_SWAV + 1950SOI_EAM%CMIP6_ELM%CNPRDCTCBCTOP_MPASSI%DIB_MPASO%IBDISMF_MOSART_SGLC_SWAV CRYO20TR - 20TRSOI_EAM%CMIP6_ELM%SPBC_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV + 20TRSOI_EAM%CMIP6_ELM%CNPRDCTCBCTOP_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV + + + + CRYOSSP245 + SSP245SOI_EAM%CMIP6_ELM%CNPRDCTCBCTOP_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV CRYOSSP585 - SSP585SOI_EAM%CMIP6_ELM%SPBC_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV + SSP585SOI_EAM%CMIP6_ELM%CNPRDCTCBCTOP_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV CRYOSSP370 - SSP370SOI_EAM%CMIP6_ELM%SPBC_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV + SSP370SOI_EAM%CMIP6_ELM%CNPRDCTCBCTOP_MPASSI%DIB_MPASO%IBPISMF_MOSART_SGLC_SWAV diff --git a/cime_config/allactive/config_pesall.xml b/cime_config/allactive/config_pesall.xml index 3392ad71ca1..0d5f2456596 100644 --- a/cime_config/allactive/config_pesall.xml +++ b/cime_config/allactive/config_pesall.xml @@ -843,6 +843,29 @@ + + + + -compset WCYCL*/CRYO* -res SOwISC12to30E3r3* on 52 nodes pure-MPI, ~8.5 sypd + + 1408 + 384 + 384 + 1024 + 1920 + 1408 + + + 0 + 1024 + 1024 + 0 + 1408 + 0 + + + + @@ -1774,6 +1797,21 @@ + + + allactive+pm-cpu: default, 1 node, 96 tasks, 1 thread + + 96 + 96 + 96 + 96 + 96 + 96 + 96 + 96 + + + diff --git a/cime_config/config_grids.xml b/cime_config/config_grids.xml index 2d43f3390a3..b093285e59d 100755 --- a/cime_config/config_grids.xml +++ b/cime_config/config_grids.xml @@ -416,6 +416,16 @@ RRSwISC6to18E3r5 + + T62 + T62 + SOwISC12to30E3r3 + rx1 + null + null + SOwISC12to30E3r3 + + TL319 TL319 @@ -656,6 +666,16 @@ RRSwISC6to18E3r5 + + TL319 + TL319 + SOwISC12to30E3r3 + JRA025 + null + null + SOwISC12to30E3r3 + + TL319 TL319 @@ -1400,6 +1420,16 @@ RRSwISC6to18E3r5 + + ne30np4.pg2 + ne30np4.pg2 + SOwISC12to30E3r3 + r05 + null + null + SOwISC12to30E3r3 + + ne0np4_northamericax4v1 r0125 @@ -2009,6 +2039,26 @@ IcoswISC30E3r5 + + ERA5r025 + r05 + IcoswISC30E3r5 + r05 + mpas.gis1to10kmR2 + null + IcoswISC30E3r5 + + + + ERA5r025 + r025 + IcoswISC30E3r5 + r025 + mpas.gis1to10kmR2 + null + IcoswISC30E3r5 + + ne120np4.pg2 r0125 @@ -2365,6 +2415,16 @@ EC30to60E2r2 + + ne30np4.pg2 + r05 + IcoswISC30E3r5 + r05 + null + wQU225Icos30E3r5 + IcoswISC30E3r5 + + ne30np4.pg2 ne30np4.pg2 @@ -2457,6 +2517,16 @@ RRSwISC6to18E3r5 + + ne30np4.pg2 + r05 + SOwISC12to30E3r3 + r05 + null + null + SOwISC12to30E3r3 + + ne30np4.pg2 r05 @@ -2768,6 +2838,7 @@ $DIN_LOC_ROOT/share/domains/domain.lnd.T62_ECwISC30to60E2r1.201007.nc $DIN_LOC_ROOT/share/domains/domain.lnd.T62_IcoswISC30E3r5.231121.nc $DIN_LOC_ROOT/share/domains/domain.lnd.T62_RRSwISC6to18E3r5.240328.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.T62_SOwISC12to30E3r3.240808.nc T62 is Gaussian grid: @@ -2826,11 +2897,21 @@ $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_IcosXISC30E3r7.240326.nc $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_RRSwISC6to18E3r5.240328.nc $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_RRSwISC6to18E3r5.240328.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_SOwISC12to30E3r3.240808.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_SOwISC12to30E3r3.240808.nc $DIN_LOC_ROOT/share/domains/domain.lnd.TL319_oRRS18to6v3.220124.nc $DIN_LOC_ROOT/share/domains/domain.ocn.TL319_oRRS18to6v3.220124.nc TL319 is JRA lat/lon grid: + + 1440 + 721 + $DIN_LOC_ROOT/share/domains/domain.lnd.ERA5r025_IcoswISC30E3r5.240903.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.ERA5r025_IcoswISC30E3r5.240903.nc + ERA5r025 is the lat/lon cap grid used by ERA5 data: + + @@ -2939,6 +3020,8 @@ $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_IcosXISC30E3r7.240326.nc $DIN_LOC_ROOT/share/domains/domain.lnd.ne30pg2_RRSwISC6to18E3r5.240328.nc $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_RRSwISC6to18E3r5.240328.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.ne30pg2_SOwISC12to30E3r3.240808.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_SOwISC12to30E3r3.240808.nc $DIN_LOC_ROOT/share/domains/domain.lnd.ne30pg2_gx1v6.190806.nc $DIN_LOC_ROOT/share/domains/domain.ocn.ne30pg2_gx1v6.190806.nc ne30np4.pg2 is Spectral Elem 1-deg grid w/ 2x2 FV physics grid per element: @@ -3259,6 +3342,13 @@ RRSwISC6to18E3r5 is a MPAS ocean grid generated with the jigsaw/compass process using a mesh density function that is roughly proportional to the Rossby radius of deformation, with 18 km gridcells at low and 6 km gridcells at high latitudes. Additionally, it has ocean in ice-shelf cavities: + + 807630 + 1 + $DIN_LOC_ROOT/share/domains/domain.ocn.SOwISC12to30E3r3.240808.nc + SOwISC12to30E3r3 is a MPAS ocean grid generated with the jigsaw/compass process using XXXXX. Additionally, it has ocean in ice-shelf cavities: + + @@ -3295,6 +3385,8 @@ $DIN_LOC_ROOT/share/domains/domain.lnd.r05_IcosXISC30E3r7.240326.nc $DIN_LOC_ROOT/share/domains/domain.lnd.r05_RRSwISC6to18E3r5.240328.nc $DIN_LOC_ROOT/share/domains/domain.lnd.r05_RRSwISC6to18E3r5.240328.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.r05_SOwISC12to30E3r3.240808.nc + $DIN_LOC_ROOT/share/domains/domain.lnd.r05_SOwISC12to30E3r3.240808.nc $DIN_LOC_ROOT/share/domains/domain.lnd.r05_gx1v6.191014.nc r05 is 1/2 degree river routing grid: @@ -3404,6 +3496,13 @@ $DIN_LOC_ROOT/share/domains/domain.ocn.wQU225EC30to60E2r2.220224.nc WW3 unstructured QU 225km global grid with EC30to60E2r2 coastlines + + + 97988 + 1 + $DIN_LOC_ROOT/share/domains/domain.ocn.wQU225Icos30E3r5.240910.nc + WW3 unstructured QU 225km global grid with ICOS30 coastlines + @@ -3573,6 +3672,7 @@ ATM2ROF_SMAPNAME ATM2WAV_SMAPNAME OCN2WAV_SMAPNAME + WAV2OCN_SMAPNAME ICE2WAV_SMAPNAME ROF2OCN_LIQ_RMAPNAME @@ -3817,6 +3917,16 @@ cpl/gridmaps/ne30pg2/map_ne30pg2_to_RRSwISC6to18E3r5_trfvnp2.20240328.nc + + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_traave.20240808.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trbilin.20240808.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3-nomask_trbilin.20240808.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_ne30pg2_traave.20240808.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_ne30pg2_traave.20240808.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trfvnp2.20240808.nc + cpl/gridmaps/ne30pg2/map_ne30pg2_to_SOwISC12to30E3r3_trfvnp2.20240808.nc + + cpl/gridmaps/ne30pg3/map_ne30pg3_to_oEC60to30v3_mono.200331.nc cpl/gridmaps/ne30pg3/map_ne30pg3_to_oEC60to30v3_bilin.200331.nc @@ -4626,6 +4736,14 @@ cpl/gridmaps/RRSwISC6to18E3r5/map_RRSwISC6to18E3r5_to_T62_traave.20240328.nc + + cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3_traave.20240808.nc + cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3-nomask_trbilin.20240808.nc + cpl/gridmaps/T62/map_T62_to_SOwISC12to30E3r3_esmfpatch.20240808.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_T62_traave.20240808.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_T62_traave.20240808.nc + + cpl/gridmaps/TL319/map_TL319_to_oQU240wLI_traave.20240509.nc cpl/gridmaps/TL319/map_TL319_to_oQU240wLI-nomask_trbilin.20240509.nc @@ -4778,6 +4896,14 @@ cpl/gridmaps/RRSwISC6to18E3r5/map_RRSwISC6to18E3r5_to_TL319_traave.20240328.nc + + cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3_traave.20240808.nc + cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3-nomask_trbilin.20240808.nc + cpl/gridmaps/TL319/map_TL319_to_SOwISC12to30E3r3_esmfpatch.20240808.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_TL319_traave.20240808.nc + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_TL319_traave.20240808.nc + + cpl/gridmaps/TL319/map_TL319_to_oRRS18to6v3_aave.220124.nc cpl/gridmaps/TL319/map_TL319_to_oRRS18to6v3_bilin.220124.nc @@ -4793,6 +4919,38 @@ cpl/gridmaps/TL319/map_r05_to_TL319_traave.20240212.nc + + cpl/gridmaps/ERA5r025/map_ERA5r025_to_IcoswISC30E3r5_traave.20240903.nc + cpl/gridmaps/ERA5r025/map_ERA5r025_to_IcoswISC30E3r5-nomask_trbilin.20240903.nc + cpl/gridmaps/ERA5r025/map_ERA5r025_to_IcoswISC30E3r5_esmfpatch.20240903.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_ERA5r025_traave.20240903.nc + cpl/gridmaps/IcoswISC30E3r5/map_IcoswISC30E3r5_to_ERA5r025_traave.20240903.nc + + + + cpl/gridmaps/ERA5r025/map_ERA5r025_to_r05_traave.20240905.nc + cpl/gridmaps/ERA5r025/map_ERA5r025_to_r05_esmfbilin.20240907.nc + cpl/gridmaps/r05/map_r05_to_ERA5r025_traave.20240905.nc + cpl/gridmaps/r05/map_r05_to_ERA5r025_traave.20240905.nc + + + + cpl/gridmaps/ERA5r025/map_ERA5r025_to_r05_traave.20240905.nc + cpl/gridmaps/ERA5r025/map_ERA5r025_to_r05_esmfbilin.20240907.nc + + + + cpl/gridmaps/ERA5r025/map_ERA5r025_to_r025_traave.20240903.nc + cpl/gridmaps/ERA5r025/map_ERA5r025_to_r025_trbilin.20240903.nc + cpl/gridmaps/r025/map_r025_to_ERA5r025_traave.20240903.nc + cpl/gridmaps/r025/map_r025_to_ERA5r025_traave.20240903.nc + + + + cpl/gridmaps/ERA5r025/map_ERA5r025_to_r025_traave.20240903.nc + cpl/gridmaps/ERA5r025/map_ERA5r025_to_r025_trbilin.20240903.nc + + cpl/cpl6/map_T31_to_gx3v7_aave_da_090903.nc cpl/cpl6/map_T31_to_gx3v7_patch_090903.nc @@ -4927,6 +5085,10 @@ cpl/gridmaps/wQU225EC30to60E2r2/map_ne30pg2_TO_wQU225EC30to60E2r2_blin.20220222.nc + + cpl/gridmaps/wQU225Icos30E3r5/map_ne30pg2_to_wQU225Icos30E3r5_esmfbilin.20240910.nc + + cpl/gridmaps/wQU225EC30to60E2r2/map_TL319_TO_wQU225EC30to60E2r2_blin.20220602.nc @@ -4943,6 +5105,12 @@ cpl/gridmaps/wQU225EC30to60E2r2/map_EC30to60E2r2_TO_wQU225EC30to60E2r2_blin.20220222.nc + + cpl/gridmaps/wQU225Icos30E3r5/map_wQU225Icos30E3r5_to_IcoswISC30E3r5_esmfbilin.20240910.nc + cpl/gridmaps/wQU225Icos30E3r5/map_IcoswISC30E3r5_to_wQU225Icos30E3r5_esmfbilin.20240910.nc + cpl/gridmaps/wQU225Icos30E3r5/map_IcoswISC30E3r5_to_wQU225Icos30E3r5_esmfbilin.20240910.nc + + cpl/gridmaps/wQU225EC60to30/map_CFSv2_TO_wQU225EC60to30_blin.20210412.nc @@ -5172,6 +5340,10 @@ cpl/gridmaps/RRSwISC6to18E3r5/map_RRSwISC6to18E3r5_to_r05_traave.20240328.nc + + cpl/gridmaps/SOwISC12to30E3r3/map_SOwISC12to30E3r3_to_r05_traave.20240808.nc + + cpl/cpl6/map_EC30to60E2r2_to_r05_neareststod.220728.nc @@ -5287,6 +5459,11 @@ cpl/cpl6/map_rx1_to_RRSwISC6to18E3r5_cstmnn.r50e100.20240328.nc + + cpl/cpl6/map_rx1_to_SOwISC12to30E3r3_cstmnn.r150e300.20240808.nc + cpl/cpl6/map_rx1_to_SOwISC12to30E3r3_cstmnn.r150e300.20240808.nc + + cpl/cpl6/map_JRA025_to_oQU240wLI_cstmnn.r150e300.20240516.nc cpl/cpl6/map_JRA025_to_oQU240wLI_cstmnn.r150e300.20240516.nc @@ -5382,6 +5559,11 @@ cpl/cpl6/map_JRA025_to_RRSwISC6to18E3r5_cstmnn.r50e100.20240328.nc + + cpl/cpl6/map_JRA025_to_SOwISC12to30E3r3_cstmnn.r150e300.20240808.nc + cpl/cpl6/map_JRA025_to_SOwISC12to30E3r3_cstmnn.r150e300.20240808.nc + + cpl/cpl6/map_JRA025_to_oRRS18to6v3_smoothed.r50e100.220124.nc cpl/cpl6/map_JRA025_to_oRRS18to6v3_smoothed.r50e100.220124.nc @@ -5473,10 +5655,15 @@ - cpl/cpl6/map_r05_to_RRSwISC6to18E3r5.cstmnn.r250e1250_58NS.20240328.nc + cpl/cpl6/map_r05_to_RRSwISC6to18E3r5_cstmnn.r250e1250_58NS.20240328.nc cpl/cpl6/map_r05_to_RRSwISC6to18E3r5_cstmnn.r50e100.20240328.nc + + cpl/cpl6/map_r05_to_SOwISC12to30E3r3_cstmnn.r150e300.20240808.nc + cpl/cpl6/map_r05_to_SOwISC12to30E3r3_cstmnn.r150e300.20240808.nc + + cpl/cpl6/map_r025_to_IcoswISC30E3r5_cstmnn.r150e300.20240401.nc cpl/cpl6/map_r025_to_IcoswISC30E3r5_cstmnn.r150e300.20240401.nc @@ -5706,6 +5893,13 @@ cpl/gridmaps/mpas.gis1to10km/map_gis1to10kmR2_to_r05_traave.20240403.nc + + cpl/gridmaps/r025/map_r025_to_gis1to10kmR2_traave.20240903.nc + cpl/gridmaps/r025/map_r025_to_gis1to10kmR2_trbilin.20240903.nc + cpl/gridmaps/mpas.gis1to10km/map_gis1to10kmR2_to_r025_traave.20240903.nc + cpl/gridmaps/mpas.gis1to10km/map_gis1to10kmR2_to_r025_traave.20240903.nc + + cpl/gridmaps/ne30pg2/map_ne30pg2_to_gis1to10kmR2_traave.20240403.nc cpl/gridmaps/ne30pg2/map_ne30pg2_to_gis1to10kmR2_trbilin.20240403.nc diff --git a/cime_config/machines/Depends.muller-cpu.gnu.cmake b/cime_config/machines/Depends.muller-cpu.gnu.cmake index 5c7331f979a..53f8b536651 100644 --- a/cime_config/machines/Depends.muller-cpu.gnu.cmake +++ b/cime_config/machines/Depends.muller-cpu.gnu.cmake @@ -7,23 +7,3 @@ if (NOT DEBUG) e3sm_deoptimize_file("${ITEM}") endforeach() endif() - -# On pm-cpu (and muller-cpu), with gcc-native/12.3, we see hang with DEBUG runs of certain tests. -# https://github.com/E3SM-Project/E3SM/issues/6516 -# Currently, we have pm-cpu using gcc/12.2.0 which does not have this issue, but using muller-cpu to test 12.3 -# Turning off -O0 for these 2 files (by adding -O) at least avoids hang and will produce FPE in HOMME code -if (CMAKE_Fortran_COMPILER_VERSION VERSION_GREATER_EQUAL 12.3) - if (DEBUG) - - set(ADJUST - eam/src/dynamics/se/inidat.F90 - eam/src/dynamics/se/dyn_comp.F90 - ) - - foreach(ITEM IN LISTS ADJUST) - e3sm_add_flags("${ITEM}" "-O") - #e3sm_add_flags("${ITEM}" "-DNDEBUG -O") - endforeach() - - endif() -endif() diff --git a/cime_config/machines/cmake_macros/amdclanggpu_frontier.cmake b/cime_config/machines/cmake_macros/amdclanggpu_frontier.cmake index 4412ea0de7b..1deebdac85d 100644 --- a/cime_config/machines/cmake_macros/amdclanggpu_frontier.cmake +++ b/cime_config/machines/cmake_macros/amdclanggpu_frontier.cmake @@ -13,8 +13,6 @@ string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2") string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") -string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") - string(APPEND CMAKE_CXX_FLAGS " --offload-arch=gfx90a") string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa") string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/opt/cray/pe/gcc/12.2.0/snos/lib64 -lgfortran -lstdc++") diff --git a/cime_config/machines/cmake_macros/crayclanggpu_frontier.cmake b/cime_config/machines/cmake_macros/crayclanggpu_frontier.cmake index a37ccde439e..92567416c56 100644 --- a/cime_config/machines/cmake_macros/crayclanggpu_frontier.cmake +++ b/cime_config/machines/cmake_macros/crayclanggpu_frontier.cmake @@ -52,8 +52,6 @@ endif() # https://github.com/E3SM-Project/E3SM/pull/5208 string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero -em -ef -hnoacc") -string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") - string(APPEND CMAKE_CXX_FLAGS " --offload-arch=gfx90a") string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa") string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64") diff --git a/cime_config/machines/cmake_macros/gnu_chicoma-cpu.cmake b/cime_config/machines/cmake_macros/gnu_chicoma-cpu.cmake index a6148451eb7..a6c13942620 100644 --- a/cime_config/machines/cmake_macros/gnu_chicoma-cpu.cmake +++ b/cime_config/machines/cmake_macros/gnu_chicoma-cpu.cmake @@ -5,6 +5,7 @@ endif() set(PIO_FILESYSTEM_HINTS "lustre") string(APPEND CMAKE_C_FLAGS_RELEASE " -O2 -g") string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2 -g") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--enable-new-dtags") set(MPICC "cc") set(MPICXX "CC") set(MPIFC "ftn") diff --git a/cime_config/machines/cmake_macros/gnugpu_frontier.cmake b/cime_config/machines/cmake_macros/gnugpu_frontier.cmake index 6ca4b83d9c2..7a29a5ca154 100644 --- a/cime_config/machines/cmake_macros/gnugpu_frontier.cmake +++ b/cime_config/machines/cmake_macros/gnugpu_frontier.cmake @@ -14,7 +14,6 @@ string(APPEND CMAKE_Fortran_FLAGS " -Wno-implicit-interface") string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2") string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") -string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") string(APPEND CMAKE_CXX_FLAGS " --offload-arch=gfx90a") string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa") diff --git a/cime_config/machines/cmake_macros/intel_dane.cmake b/cime_config/machines/cmake_macros/intel_dane.cmake index 8091325c6ce..ef25a97b300 100644 --- a/cime_config/machines/cmake_macros/intel_dane.cmake +++ b/cime_config/machines/cmake_macros/intel_dane.cmake @@ -1,4 +1,10 @@ string(APPEND CPPDEFS " -DNO_SHR_VMATH -DCNL") string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -check all -ftrapuv") string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/usr/tce/packages/gcc/gcc-10.3.1-magic/lib/gcc/x86_64-redhat-linux/10/") + +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/hdf5-1.10.7-766kapalbrdntu2pcgdgbhg2ch26gsuv/lib") +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/netcdf-c-4.4.1.1-2uznnlwgiezxute6iyqzqjrpolokeaib/lib") +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/netcdf-fortran-4.4.4-itpstyordbern7vlulmlnt47eeeokzfp/lib") +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/parallel-netcdf-1.11.0-26sxm4mormsglmhi24poix7sugbigkck/lib") + set(KOKKOS_OPTIONS "--with-serial --ldflags='-L/usr/tce/packages/gcc/gcc-10.3.1-magic/lib/gcc/x86_64-redhat-linux/10/'") diff --git a/cime_config/machines/cmake_macros/intel_ruby.cmake b/cime_config/machines/cmake_macros/intel_ruby.cmake index 8091325c6ce..e874bfb7eaf 100644 --- a/cime_config/machines/cmake_macros/intel_ruby.cmake +++ b/cime_config/machines/cmake_macros/intel_ruby.cmake @@ -1,4 +1,10 @@ string(APPEND CPPDEFS " -DNO_SHR_VMATH -DCNL") string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -check all -ftrapuv") string(APPEND CMAKE_EXE_LINKER_FLAGS " -L/usr/tce/packages/gcc/gcc-10.3.1-magic/lib/gcc/x86_64-redhat-linux/10/") + +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/hdf5-1.10.7-ewjpbjdhjgjzrzjcvwyjyuulaesbsjhg/lib") +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/netcdf-c-4.4.1.1-vaxofekwvnvngh7wptmzkwdb7tkzvesn/lib") +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/netcdf-fortran-4.4.4-3pzbx2unddhladhubaahhhysjmprzqi2/lib") +list(APPEND CMAKE_BUILD_RPATH "/usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/parallel-netcdf-1.11.0-tzgdalakmem7tod6cruhqyeackeix5q5/lib") + set(KOKKOS_OPTIONS "--with-serial --ldflags='-L/usr/tce/packages/gcc/gcc-10.3.1-magic/lib/gcc/x86_64-redhat-linux/10/'") diff --git a/cime_config/machines/config_batch.xml b/cime_config/machines/config_batch.xml index 13cfdfb337b..ee271d1f9dc 100644 --- a/cime_config/machines/config_batch.xml +++ b/cime_config/machines/config_batch.xml @@ -237,33 +237,17 @@ - - squeue - sbatch - scancel - #SBATCH - (\d+)$ - --dependency=afterok:jobid - --dependency=afterany:jobid - : - %H:%M:%S - --mail-user - --mail-type - none, all, begin, end, fail - - --export=ALL - -p {{ job_queue }} - -J {{ job_id }} - -N {{ num_nodes }} - -n {{ total_tasks }} - -t {{ job_wallclock_time }} - -o {{ job_id }}.out - -e {{ job_id }}.err - -A {{ project }} - + - pbatch - pdebug + pbatch + pdebug + + + + + + pbatch + pdebug @@ -464,6 +448,7 @@ regular preempt shared + overrun debug @@ -518,10 +503,11 @@ regular preempt shared + overrun debug - + --constraint=cpu diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 5bed00ea5d5..fce20fca185 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -190,6 +190,7 @@ module + cpe cray-hdf5-parallel cray-netcdf-hdf5parallel cray-parallel-netcdf @@ -218,34 +219,34 @@ PrgEnv-gnu/8.5.0 - gcc/12.2.0 - cray-libsci/23.02.1.1 + gcc-native/12.3 + cray-libsci/23.12.5 - PrgEnv-intel/8.3.3 - intel/2023.1.0 + PrgEnv-intel/8.5.0 + intel/2023.2.0 PrgEnv-nvidia - nvidia/22.7 - cray-libsci/23.02.1.1 + nvidia/24.5 + cray-libsci/23.12.5 PrgEnv-aocc - aocc/4.0.0 - cray-libsci/23.02.1.1 + aocc/4.1.0 + cray-libsci/23.12.5 craype-accel-host - craype/2.7.20 - cray-mpich/8.1.25 - cray-hdf5-parallel/1.12.2.3 - cray-netcdf-hdf5parallel/4.9.0.3 - cray-parallel-netcdf/1.12.3.3 + craype/2.7.30 + cray-mpich/8.1.28 + cray-hdf5-parallel/1.12.2.9 + cray-netcdf-hdf5parallel/4.9.0.9 + cray-parallel-netcdf/1.12.3.9 cmake/3.24.3 evp-patch @@ -265,7 +266,7 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch - software + kdreg2 MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} @@ -355,6 +356,7 @@ module + cpe cray-hdf5-parallel cray-netcdf-hdf5parallel cray-parallel-netcdf @@ -365,13 +367,14 @@ PrgEnv-nvidia PrgEnv-cray PrgEnv-aocc + gcc-native intel intel-oneapi nvidia aocc cudatoolkit - cray-libsci climate-utils + cray-libsci matlab craype-accel-nvidia80 craype-accel-host @@ -423,6 +426,7 @@ $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 + 0.20 1 @@ -433,6 +437,7 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch + kdreg2 MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} @@ -578,12 +583,11 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch - software + kdreg2 MPI_Bcast - $SHELL{if [ -z "$Albany_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/albany-e3sm-serial-release-gcc; else echo "$Albany_ROOT"; fi} - $SHELL{if [ -z "$Trilinos_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/trilinos-e3sm-serial-release-gcc; else echo "$Trilinos_ROOT"; fi} $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} + 4000MB $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/intel-2023.1.0; else echo "$ADIOS2_ROOT"; fi} @@ -591,6 +595,8 @@ $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/gcc-11.2.0; else echo "$ADIOS2_ROOT"; fi} Generic + $SHELL{if [ -z "$Albany_ROOT" ]; then echo /global/common/software/e3sm/albany/2024.03.26/gcc/11.2.0; else echo "$Albany_ROOT"; fi} + $SHELL{if [ -z "$Trilinos_ROOT" ]; then echo /global/common/software/e3sm/trilinos/15.1.1/gcc/11.2.0; else echo "$Trilinos_ROOT"; fi} $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/nvidia-22.7; else echo "$ADIOS2_ROOT"; fi} @@ -606,6 +612,13 @@ $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/aocc-4.0.0; else echo "$ADIOS2_ROOT"; fi} + + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/intel; else echo "$MOAB_ROOT"; fi} + + + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/gnu; else echo "$MOAB_ROOT"; fi} + + -1 @@ -672,13 +685,14 @@ PrgEnv-nvidia PrgEnv-cray PrgEnv-aocc + gcc-native intel intel-oneapi nvidia aocc cudatoolkit - cray-libsci climate-utils + cray-libsci matlab craype-accel-nvidia80 craype-accel-host @@ -730,6 +744,7 @@ $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 + 0.20 1 @@ -740,6 +755,7 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch + kdreg2 MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} @@ -750,6 +766,9 @@ 1 + + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/gnugpu ; else echo "$MOAB_ROOT"; fi} + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/gcc-11.2.0; else echo "$ADIOS2_ROOT"; fi} @@ -807,6 +826,7 @@ module + cpe cray-hdf5-parallel cray-netcdf-hdf5parallel cray-parallel-netcdf @@ -834,36 +854,35 @@ - PrgEnv-gnu - gcc-native - cray-libsci + PrgEnv-gnu/8.5.0 + gcc-native/13.2 + cray-libsci/24.03.0 - PrgEnv-intel - intel + PrgEnv-intel/8.5.0 + intel/2024.1.0 PrgEnv-nvidia nvidia/24.5 - cray-libsci + cray-libsci/24.03.0 PrgEnv-aocc - aocc/4.0.1 - cray-libsci + aocc/4.1.0 + cray-libsci/24.03.0 craype-accel-host - cray-libsci - craype/2.7.30 - cray-mpich/8.1.28 - cray-hdf5-parallel/1.12.2.9 - cray-netcdf-hdf5parallel/4.9.0.9 - cray-parallel-netcdf/1.12.3.9 + craype/2.7.31.11 + cray-mpich/8.1.29 + cray-hdf5-parallel/1.12.2.11 + cray-netcdf-hdf5parallel/4.9.0.11 + cray-parallel-netcdf/1.12.3.11 cmake/3.24.3 @@ -871,6 +890,7 @@ $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 + 0.20 1 @@ -881,16 +901,48 @@ threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch - software + kdreg2 MPI_Bcast $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} + 4000MB + + + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/intel-2023.1.0; else echo "$ADIOS2_ROOT"; fi} + + + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/gcc-11.2.0; else echo "$ADIOS2_ROOT"; fi} + Generic + $SHELL{if [ -z "$Albany_ROOT" ]; then echo /global/common/software/e3sm/albany/2024.03.26/gcc/11.2.0; else echo "$Albany_ROOT"; fi} + $SHELL{if [ -z "$Trilinos_ROOT" ]; then echo /global/common/software/e3sm/trilinos/15.1.1/gcc/11.2.0; else echo "$Trilinos_ROOT"; fi} + + + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/nvidia-22.7; else echo "$ADIOS2_ROOT"; fi} + + + $SHELL{if [ -z "$BLAS_ROOT" ]; then echo $NVIDIA_PATH/compilers; else echo "$BLAS_ROOT"; fi} + $SHELL{if [ -z "$LAPACK_ROOT" ]; then echo $NVIDIA_PATH/compilers; else echo "$LAPACK_ROOT"; fi} + NVHPC + + + Intel10_64_dyn + + + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/aocc-4.0.0; else echo "$ADIOS2_ROOT"; fi} + + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/intel; else echo "$MOAB_ROOT"; fi} + + + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /global/cfs/cdirs/e3sm/software/moab/gnu; else echo "$MOAB_ROOT"; fi} + + -1 + Spock. NCCS moderate-security system that contains similar hardware and software as the upcoming Frontier system at ORNL. .*spock.* @@ -1017,6 +1069,7 @@ /usr/share/lmod/lmod/libexec/lmod python + Core Core/24.07 PrgEnv-cray PrgEnv-cray/8.3.3 cce cce/15.0.1 @@ -1029,6 +1082,7 @@ + Core Core/24.07 PrgEnv-cray PrgEnv-amd/8.3.3 amd amd/5.4.0 @@ -1037,6 +1091,7 @@ + Core Core/24.07 PrgEnv-cray PrgEnv-gnu/8.3.3 gcc gcc/12.2.0 @@ -1045,12 +1100,13 @@ rocm/5.4.0 - cray-python/3.9.13.1 + cray-python/3.11.5 cray-libsci - subversion/1.14.1 - git/2.36.1 - cmake/3.21.3 - zlib/1.2.11 + cmake/3.27.9 + subversion + git + zlib + libfabric/1.15.2.0 cray-hdf5-parallel/1.12.2.1 cray-netcdf-hdf5parallel/4.9.0.1 cray-parallel-netcdf/1.12.3.1 @@ -1084,9 +1140,6 @@ spread threads - @@ -1976,7 +2028,8 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss /nfs/gce/projects/climate/software/perl5/lib/perl5 - $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/adios2/2.9.1/mpich-4.1.2/gcc-12.1.0; else echo "$ADIOS2_ROOT"; fi} + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/adios2/2.10.1/mpich-4.1.2/gcc-12.1.0; else echo "$ADIOS2_ROOT"; fi} + $SHELL{if [ -z "$BLOSC2_ROOT" ]; then echo /nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/c-blosc2/2.15.1/gcc-12.1.0; else echo "$BLOSC2_ROOT"; fi} @@ -2398,17 +2451,15 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss openmpi/4.1.6-2mm63n2 hdf5/1.10.7-4cghwvq - netcdf-c/4.4.1-a4hji6e - netcdf-cxx/4.2-ldoxr43 - netcdf-fortran/4.4.4-husened + netcdf-c/4.7.4-4qjdadt + netcdf-fortran/4.5.3-qozrykr parallel-netcdf/1.11.0-icrpxty intel-mpi/2019.9.304-tkzvizk - hdf5/1.8.16-se4xyo7 - netcdf-c/4.4.1-qvxyzq2 - netcdf-cxx/4.2-binixgj - netcdf-fortran/4.4.4-rdxohvp + hdf5/1.10.7-wczt56s + netcdf-c/4.7.4-ba6agmb + netcdf-fortran/4.5.3-5lvy5p4 parallel-netcdf/1.11.0-b74wv4m @@ -2418,17 +2469,19 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss openmpi/4.1.6-ggebj5o hdf5/1.10.7-ol6xuae - netcdf-c/4.4.1-2njo6xx - netcdf-cxx/4.2-7pdzqua - netcdf-fortran/4.4.4-52c6oqi + netcdf-c/4.7.4-pfocec2 + netcdf-fortran/4.5.3-va3hoor parallel-netcdf/1.11.0-d7h4ysd + gcc/11.2.0-bgddrif + intel-oneapi-mkl/2022.1.0-w4kgsn4 + gcc/9.2.0-ugetvbp + intel-mkl/2020.4.304-n3b5fye intel-mpi/2019.9.304-jdih7h5 hdf5/1.8.16-dtbpce3 - netcdf-c/4.4.1-zcoa44z - netcdf-cxx/4.2-ayxg4c7 - netcdf-fortran/4.4.4-2lfr2lr + netcdf-c/4.7.4-seagl7g + netcdf-fortran/4.5.3-ova6t37 parallel-netcdf/1.11.0-ifdodru @@ -2798,6 +2851,7 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss /lcrc/group/e3sm/soft/improv/pnetcdf/1.12.3/gcc-12.3.0/openmpi-4.1.6 /lcrc/group/e3sm/soft/improv/pnetcdf/1.12.3/gcc-12.3.0/openmpi-4.1.6/bin:/lcrc/group/e3sm/soft/improv/netcdf-fortran/4.6.1b/gcc-12.3.0/openmpi-4.1.6/bin:/lcrc/group/e3sm/soft/improv/netcdf-c/4.9.2b/gcc-12.3.0/openmpi-4.1.6/bin:/lcrc/group/e3sm/soft/improv/openmpi/4.1.6/gcc-12.3.0/bin:/lcrc/group/e3sm/soft/perl/improv/bin:$ENV{PATH} $SHELL{lp=/lcrc/group/e3sm/soft/improv/netlib-lapack/3.12.0/gcc-12.3.0:/lcrc/group/e3sm/soft/improv/pnetcdf/1.12.3/gcc-12.3.0/openmpi-4.1.6/lib:/lcrc/group/e3sm/soft/improv/netcdf-fortran/4.6.1b/gcc-12.3.0/openmpi-4.1.6/lib:/lcrc/group/e3sm/soft/improv/netcdf-c/4.9.2b/gcc-12.3.0/openmpi-4.1.6/lib:/opt/pbs/lib:/lcrc/group/e3sm/soft/improv/openmpi/4.1.6/gcc-12.3.0/lib; if [ -z "$LD_LIBRARY_PATH" ]; then echo $lp; else echo "$lp:$LD_LIBRARY_PATH"; fi} + $SHELL{if [ -z "$MOAB_ROOT" ]; then echo /lcrc/soft/climate/moab/improv/gnu; else echo "$MOAB_ROOT"; fi} ^lockedfile @@ -2819,9 +2873,9 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss /usr/workspace/e3sm/ccsm3data/inputdata/atm/datm7 /p/lustre2/$USER/archive/$CASE /p/lustre2/$USER/ccsm_baselines/$COMPILER - /usr/workspace/e3sm/tools/cprnc + /usr/workspace/e3sm/apps/cprnc 8 - lc_slurm + slurm boutte3 -at- llnl.gov 56 56 @@ -2830,8 +2884,16 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss srun + + --mpi=pmi2 + --export=ALL + -n {{ total_tasks }} -N {{ num_nodes }} + -c 1 + --cpu_bind=cores + -m plane={{ tasks_per_node }} + - + /usr/share/lmod/lmod/init/env_modules_python.py /usr/share/lmod/lmod/init/perl /usr/share/lmod/lmod/init/sh @@ -2843,24 +2905,27 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss python/3.9.12 git + subversion + cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic - mvapich2/2.3.7 - cmake/3.19.2 - /usr/workspace/e3sm/install/quartz/modulefiles - hdf5/1.12.2 - netcdf-c/4.9.0 - netcdf-fortran/4.6.0 - parallel-netcdf/1.12.3 - screamML-venv/0.0.1 - subversion + /usr/workspace/e3sm/spack/modules/ruby/linux-rhel8-x86_64/Core + mvapich2/2.3.7-ll7cmqm + hdf5/1.10.7-ewjpbjd + netcdf-c/4.4.1.1-vaxofek + netcdf-fortran/4.4.4-3pzbx2u + parallel-netcdf/1.11.0-tzgdala $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld - /usr/workspace/e3sm/install/quartz/netcdf-fortran/ - /usr/tce/packages/parallel-netcdf/parallel-netcdf-1.12.3-mvapich2-2.3.7-intel-classic-2021.6.0 + 128M + FALSE + /usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/hdf5-1.10.7-ewjpbjdhjgjzrzjcvwyjyuulaesbsjhg + /usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/netcdf-c-4.4.1.1-vaxofekwvnvngh7wptmzkwdb7tkzvesn + /usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/netcdf-fortran-4.4.4-3pzbx2unddhladhubaahhhysjmprzqi2 + /usr/workspace/e3sm/spack/libs/linux-rhel8-cascadelake/intel-2021.6.0/parallel-netcdf-1.11.0-tzgdalakmem7tod6cruhqyeackeix5q5 @@ -2875,9 +2940,9 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss /usr/workspace/e3sm/ccsm3data/inputdata/atm/datm7 /p/lustre2/$USER/archive/$CASE /p/lustre2/$USER/ccsm_baselines/$COMPILER - /usr/workspace/e3sm/tools/cprnc + /usr/workspace/e3sm/apps/cprnc 8 - lc_slurm + slurm boutte3 -at- llnl.gov 224 112 @@ -2886,8 +2951,16 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss srun + + --mpi=pmi2 + --export=ALL + -n {{ total_tasks }} -N {{ num_nodes }} + -c 1 + --cpu_bind=cores + -m plane={{ tasks_per_node }} + - + /usr/share/lmod/lmod/init/env_modules_python.py /usr/share/lmod/lmod/init/perl /usr/share/lmod/lmod/init/sh @@ -2899,24 +2972,27 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss python/3.9.12 git + subversion mkl/2022.1.0 intel-classic/2021.6.0-magic - mvapich2/2.3.7 cmake/3.19.2 - /usr/workspace/e3sm/install/quartz/modulefiles - hdf5/1.12.2 - netcdf-c/4.9.0 - netcdf-fortran/4.6.0 - parallel-netcdf/1.12.3 - screamML-venv/0.0.1 - subversion + /usr/workspace/e3sm/spack/modules/dane/linux-rhel8-x86_64/Core + mvapich2/2.3.7-27jao34 + hdf5/1.10.7-766kapa + netcdf-c/4.4.1.1-2uznnlw + netcdf-fortran/4.4.4-itpstyo + parallel-netcdf/1.11.0-26sxm4m $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld - /usr/workspace/e3sm/install/quartz/netcdf-fortran/ - /usr/tce/packages/parallel-netcdf/parallel-netcdf-1.12.3-mvapich2-2.3.7-intel-classic-2021.6.0 + 128M + FALSE + /usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/hdf5-1.10.7-766kapalbrdntu2pcgdgbhg2ch26gsuv + /usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/netcdf-c-4.4.1.1-2uznnlwgiezxute6iyqzqjrpolokeaib + /usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/netcdf-fortran-4.4.4-itpstyordbern7vlulmlnt47eeeokzfp + /usr/workspace/e3sm/spack/libs/linux-rhel8-sapphirerapids/intel-2021.6.0/parallel-netcdf-1.11.0-26sxm4mormsglmhi24poix7sugbigkck @@ -3946,7 +4022,7 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss /usr/projects/e3sm/inputdata/atm/datm7 /lustre/scratch5/$ENV{USER}/E3SM/archive/$CASE /lustre/scratch5/$ENV{USER}/E3SM/input_data/ccsm_baselines/$COMPILER - /usr/projects/climate/SHARED_CLIMATE/software/badger/cprnc + /usr/projects/e3sm/software/chicoma-cpu/cprnc 10 e3sm_developer 4 @@ -3966,11 +4042,11 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss - /usr/share/lmod/8.3.1/init/perl + /usr/share/lmod/lmod/init/perl - /usr/share/lmod/8.3.1/init/python - /usr/share/lmod/8.3.1/init/sh - /usr/share/lmod/8.3.1/init/csh + /usr/share/lmod/lmod/init/python + /usr/share/lmod/lmod/init/sh + /usr/share/lmod/lmod/init/csh /usr/share/lmod/lmod/libexec/lmod perl /usr/share/lmod/lmod/libexec/lmod python module @@ -3982,39 +4058,42 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss cray-parallel-netcdf cray-netcdf cray-hdf5 - PrgEnv-gnu - PrgEnv-intel - PrgEnv-nvidia - PrgEnv-cray - PrgEnv-aocc + gcc + gcc-native intel intel-oneapi nvidia aocc cudatoolkit climate-utils + cray-libsci craype-accel-nvidia80 craype-accel-host perftools-base perftools darshan + PrgEnv-gnu + PrgEnv-intel + PrgEnv-nvidia + PrgEnv-cray + PrgEnv-aocc - PrgEnv-gnu/8.4.0 - gcc/12.2.0 - cray-libsci/23.05.1.4 + PrgEnv-gnu/8.5.0 + gcc-native/12.3 + cray-libsci/23.12.5 - PrgEnv-nvidia/8.4.0 - nvidia/22.7 - cray-libsci/23.05.1.4 + PrgEnv-nvidia/8.5.0 + nvidia/24.7 + cray-libsci/23.12.5 - PrgEnv-intel/8.4.0 - intel-classic/2023.2.0 + PrgEnv-intel/8.5.0 + intel/2023.2.0 @@ -4025,13 +4104,12 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss craype-accel-host - craype/2.7.21 - cray-mpich/8.1.26 - libfabric/1.15.2.0 - cray-hdf5-parallel/1.12.2.3 - cray-netcdf-hdf5parallel/4.9.0.3 - cray-parallel-netcdf/1.12.3.3 - cmake/3.25.1 + craype/2.7.30 + cray-mpich/8.1.28 + cray-hdf5-parallel/1.12.2.9 + cray-netcdf-hdf5parallel/4.9.0.9 + cray-parallel-netcdf/1.12.3.9 + cmake/3.27.7 @@ -4053,6 +4131,9 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} $ENV{CRAY_PARALLEL_NETCDF_PREFIX} + + /usr/lib64/gcc/x86_64-suse-linux/12:$ENV{LD_LIBRARY_PATH} + -1 diff --git a/cime_config/testmods_dirs/io/force_adiosc/shell_commands b/cime_config/testmods_dirs/io/force_adiosc/shell_commands new file mode 100644 index 00000000000..543d5dbfef0 --- /dev/null +++ b/cime_config/testmods_dirs/io/force_adiosc/shell_commands @@ -0,0 +1,2 @@ +#!/bin/bash +./xmlchange PIO_TYPENAME="adiosc" diff --git a/cime_config/tests.py b/cime_config/tests.py index 3c89e8e9099..46894e6ea96 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -49,6 +49,9 @@ "ERS.f19_g16.I20TRGSWCNPECACNTBC.elm-eca_f19_g16_I20TRGSWCNPECACNTBC", "ERS.f19_g16.I20TRGSWCNPRDCTCBC.elm-ctc_f19_g16_I20TRGSWCNPRDCTCBC", "ERS.r05_r05.ICNPRDCTCBC.elm-cbudget", + "ERS.ELM_USRDAT.I1850CNPRDCTCBC.elm-snowveg_arctic", + "ERS.ELM_USRDAT.I1850CNPRDCTCBC.elm-usrpft_default_I1850CNPRDCTCBC", + "ERS.ELM_USRDAT.I1850CNPRDCTCBC.elm-usrpft_codetest_I1850CNPRDCTCBC", ) }, @@ -94,8 +97,6 @@ "SMS.r05_r05.IELM.elm-topounit", "ERS.ELM_USRDAT.I1850ELM.elm-usrdat", "ERS.r05_r05.IELM.elm-lnd_rof_2way", - "ERS.ELM_USRDAT.I1850CNPRDCTCBC.elm-usrpft_default_I1850CNPRDCTCBC", - "ERS.ELM_USRDAT.I1850CNPRDCTCBC.elm-usrpft_codetest_I1850CNPRDCTCBC", "ERS.r05_r05.IELM.elm-V2_ELM_MOSART_features", "ERS.ELM_USRDAT.IELM.elm-surface_water_dynamics" ) @@ -207,6 +208,7 @@ "REP_Ln5.ne4pg2_oQU480.F2010", "SMS_Ld3.ne4pg2_oQU480.F2010.eam-thetahy_sl_pg2_mass", "ERP_Ld3.ne4pg2_ne4pg2.FIDEAL.allactive-pioroot1", + "ERS_Ld5.ne4pg2_oQU480.F2010.eam-sathist_F2010", ) }, @@ -266,6 +268,7 @@ "SMS_D_Ld1.T62_oQU240wLI.GMPAS-IAF-PISMF.mpaso-impl_top_drag", "SMS_D_Ld1.T62_oQU240.GMPAS-IAF.mpaso-harmonic_mean_drag", "SMS_D_Ld1.T62_oQU240.GMPAS-IAF.mpaso-upwind_advection", + "ERS_Ld5_D.T62_oQU240.GMPAS-IAF.mpaso-conservation_check", ) }, @@ -275,7 +278,7 @@ "ERS_P480_Ld5.TL319_IcoswISC30E3r5.GMPAS-JRA1p5-DIB-PISMF.mpaso-jra_1958", "PEM_P480_Ld5.TL319_IcoswISC30E3r5.GMPAS-JRA1p5-DIB-PISMF.mpaso-jra_1958", "SMS_P480_Ld5.TL319_IcoswISC30E3r5.GMPAS-JRA1p5-DIB-PISMF-TMIX.mpaso-jra_1958", - "SMS_P480_Ld5.TL319_IcoswISC30E3r5.GMPAS-JRA1p5-DIB-PISMF-DSGR.mpaso-jra_1958", + "PET_P480_Ld2.TL319_IcoswISC30E3r5.GMPAS-JRA1p5-DIB-PISMF-DSGR.mpaso-jra_1958", ) }, @@ -706,7 +709,8 @@ "PEM_Ln90.ne30pg2_ne30pg2.F2010-SCREAMv1.scream-spa_remap--scream-output-preset-4", "ERS_Ln90.ne30pg2_ne30pg2.F2010-SCREAMv1.scream-small_kernels--scream-output-preset-5", "ERP_Ln22.conusx4v1pg2_r05_oECv3.F2010-SCREAMv1-noAero.scream-bfbhash--scream-output-preset-6", - "ERS_Ln22.ne30pg2_ne30pg2.F2010-SCREAMv1.scream-L128--scream-output-preset-4" + "ERS_Ln22.ne30pg2_ne30pg2.F2010-SCREAMv1.scream-L128--scream-output-preset-4", + "REP_Ld5.ne30pg2_ne30pg2.F2010-SCREAMv1.scream-L128--scream-output-preset-6" ) }, @@ -757,6 +761,7 @@ "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-aci", "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-wetscav", "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-drydep", + "SMS_D_Ln5.ne30pg2_oECv3.F2010-SCREAMv1-MPASSI.scream-mam4xx-remap_emiss_ne4_ne30" ) }, @@ -819,9 +824,13 @@ }, "e3sm_wav_developer" : { - "time" : "0:45:00", + "time" : "1:00:00", "tests" : ( - "ERS.T62_oEC60to30v3_wQU225EC60to30.GMPAS-IAF-WW3", + "SMS_D_Ln3.TL319_EC30to60E2r2_wQU225EC30to60E2r2.GMPAS-JRA1p5-WW3.ww3-jra_1958", + "ERS.ne30pg2_IcoswISC30E3r5_wQU225Icos30E3r5.WCYCL1850-WW3", + "PEM_P480.ne30pg2_IcoswISC30E3r5_wQU225Icos30E3r5.WCYCL1850-WW3", + "PET.ne30pg2_IcoswISC30E3r5_wQU225Icos30E3r5.WCYCL1850-WW3", + "SMS_D_Ln3.ne30pg2_IcoswISC30E3r5_wQU225Icos30E3r5.WCYCL1850-WW3", ) }, diff --git a/components/cmake/modules/FindPIO.cmake b/components/cmake/modules/FindPIO.cmake index 0277918ac8e..5589dff0ed0 100644 --- a/components/cmake/modules/FindPIO.cmake +++ b/components/cmake/modules/FindPIO.cmake @@ -42,6 +42,9 @@ endif() # we can assume that an MPI case with ADIOS2_ROOT set is probably # using adios. if (NOT MPILIB STREQUAL "mpi-serial" AND DEFINED ENV{ADIOS2_ROOT}) + if(DEFINED ENV{BLOSC2_ROOT}) + set(ENV{Blosc2_DIR} "$ENV{BLOSC2_ROOT}") + endif() find_package(MPI REQUIRED COMPONENTS C) find_package(ADIOS2 REQUIRED COMPONENTS C) list(APPEND PIOLIBS adios2::adios2) diff --git a/components/data_comps/datm/cime_config/config_component.xml b/components/data_comps/datm/cime_config/config_component.xml index 9145bdf4b16..5b884ecd1ad 100644 --- a/components/data_comps/datm/cime_config/config_component.xml +++ b/components/data_comps/datm/cime_config/config_component.xml @@ -10,7 +10,7 @@ This file may have atm desc entries. --> - Data driven ATM + Data driven ATM QIAN data set QIAN with water isotopes CRUNCEP data set @@ -28,8 +28,9 @@ COREv2 normal year forcing COREv2 interannual forcing interannual JRA55 forcing - interannual JRA55 forcing, v1.5, through 2020 interannual JRA55 forcing, v1.4, through 2018 + interannual JRA55 forcing, v1.5, through 2020 + interannual JRA55 forcing, v1.5, through 2023 JRA55 Repeat Year Forcing v1.3 1984-1985 JRA55 Repeat Year Forcing v1.3 1990-1991 JRA55 Repeat Year Forcing v1.3 2003-2004 @@ -63,8 +64,9 @@ data (see cime issue #3653 -- https://github.com/ESMCI/cime/issues/3653). CORE2_NYF CORE2_IAF CORE_IAF_JRA - IAF_JRA_1p5 CORE_IAF_JRA_1p4_2018 + IAF_JRA_1p5 + IAF_JRA_1p5 CORE_RYF8485_JRA CORE_RYF9091_JRA CORE_RYF0304_JRA @@ -391,6 +393,7 @@ data (see cime issue #3653 -- https://github.com/ESMCI/cime/issues/3653). $DATM_CLMNCEP_YR_START 1 1 + 1 $DATM_CLMNCEP_YR_START $DATM_CLMNCEP_YR_START $DATM_CLMNCEP_YR_START @@ -488,6 +491,7 @@ data (see cime issue #3653 -- https://github.com/ESMCI/cime/issues/3653). 2003 2016 2020 + 2023 1979 1979 diff --git a/components/data_comps/dlnd/src/dlnd_comp_mod.F90 b/components/data_comps/dlnd/src/dlnd_comp_mod.F90 index bf723259be4..fbd6e35cf87 100644 --- a/components/data_comps/dlnd/src/dlnd_comp_mod.F90 +++ b/components/data_comps/dlnd/src/dlnd_comp_mod.F90 @@ -29,6 +29,10 @@ module dlnd_comp_mod use dlnd_shr_mod , only: domain_fracname ! namelist input use dlnd_shr_mod , only: nullstr +#ifdef HAVE_MOAB + use seq_comm_mct, only : mlnid ! id of moab lnd app + use iso_c_binding +#endif ! !PUBLIC TYPES: implicit none private ! except @@ -100,6 +104,15 @@ subroutine dlnd_comp_init(Eclock, x2l, l2x, & scmMode, scmlat, scmlon) ! !DESCRIPTION: initialize dlnd model +#ifdef HAVE_MOAB + use iMOAB, only: iMOAB_DefineTagStorage, & + iMOAB_SetIntTagStorage, iMOAB_SetDoubleTagStorage, & + iMOAB_ResolveSharedEntities, iMOAB_CreateVertices, & + iMOAB_UpdateMeshInfo +#ifdef MOABDEBUG + use iMOAB, only: iMOAB_WriteMesh +#endif +#endif implicit none ! !INPUT/OUTPUT PARAMETERS: @@ -135,6 +148,18 @@ subroutine dlnd_comp_init(Eclock, x2l, l2x, & character(nec_len) :: nec_str ! elevation class, as character string character(*), parameter :: domain_fracname_unset = 'null' +#ifdef HAVE_MOAB + character*400 tagname + real(R8) latv, lonv + integer iv, tagindex, ilat, ilon + real(R8), allocatable, target :: data(:) + integer(IN), pointer :: idata(:) ! temporary + real(R8), dimension(:), allocatable :: moab_vert_coords ! temporary +#ifdef MOABDEBUG + character*100 outfile, wopts +#endif +#endif + !--- formats --- character(*), parameter :: F00 = "('(dlnd_comp_init) ',8a)" character(*), parameter :: F0L = "('(dlnd_comp_init) ',a, l2)" @@ -256,6 +281,119 @@ subroutine dlnd_comp_init(Eclock, x2l, l2x, & call t_stopf('dlnd_initmctdom') +#ifdef HAVE_MOAB + ilat = mct_aVect_indexRA(ggrid%data,'lat') + ilon = mct_aVect_indexRA(ggrid%data,'lon') + allocate(moab_vert_coords(lsize*3)) + do iv = 1, lsize + lonv = ggrid%data%rAttr(ilon, iv) * SHR_CONST_PI/180. + latv = ggrid%data%rAttr(ilat, iv) * SHR_CONST_PI/180. + moab_vert_coords(3*iv-2)=COS(latv)*COS(lonv) + moab_vert_coords(3*iv-1)=COS(latv)*SIN(lonv) + moab_vert_coords(3*iv )=SIN(latv) + enddo + + ! create the vertices with coordinates from MCT domain + ierr = iMOAB_CreateVertices(mlnid, lsize*3, 3, moab_vert_coords) + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to create MOAB vertices in data lnd model') + + tagname='GLOBAL_ID'//C_NULL_CHAR + ierr = iMOAB_DefineTagStorage(mlnid, tagname, & + 0, & ! dense, integer + 1, & ! number of components + tagindex ) + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to retrieve GLOBAL_ID tag ') + + ! get list of global IDs for Dofs + call mct_gsMap_orderedPoints(gsMap, my_task, idata) + + ierr = iMOAB_SetIntTagStorage ( mlnid, tagname, lsize, & + 0, & ! vertex type + idata) + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to set GLOBAL_ID tag ') + + ierr = iMOAB_ResolveSharedEntities( mlnid, lsize, idata ); + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to resolve shared entities') + + deallocate(moab_vert_coords) + deallocate(idata) + + ierr = iMOAB_UpdateMeshInfo( mlnid ) + if (ierr .ne. 0) & + call shr_sys_abort('Error: fail to update mesh info ') + + allocate(data(lsize)) + ierr = iMOAB_DefineTagStorage( mlnid, "area:aream:frac:mask"//C_NULL_CHAR, & + 1, & ! dense, double + 1, & ! number of components + tagindex ) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to create tag: area:aream:frac:mask' ) + + data(:) = ggrid%data%rAttr(mct_aVect_indexRA(ggrid%data,'area'),:) + tagname='area'//C_NULL_CHAR + ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsize, & + 0, & ! set data on vertices + data) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to get area tag ') + + ! set the same data for aream (model area) as area + ! data(:) = ggrid%data%rAttr(mct_aVect_indexRA(ggrid%data,'aream'),:) + tagname='aream'//C_NULL_CHAR + ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsize, & + 0, & ! set data on vertices + data) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to set aream tag ') + + data(:) = ggrid%data%rAttr(mct_aVect_indexRA(ggrid%data,'mask'),:) + tagname='mask'//C_NULL_CHAR + ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsize, & + 0, & ! set data on vertices + data) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to set mask tag ') + + data(:) = ggrid%data%rAttr(mct_aVect_indexRA(ggrid%data,'frac'),:) + tagname='frac'//C_NULL_CHAR + ierr = iMOAB_SetDoubleTagStorage ( mlnid, tagname, lsize, & + 0, & ! set data on vertices + data) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to set frac tag ') + + deallocate(data) + + ! define tags + ierr = iMOAB_DefineTagStorage( mlnid, trim(seq_flds_x2l_fields)//C_NULL_CHAR, & + 1, & ! dense, double + 1, & ! number of components + tagindex ) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to create seq_flds_x2l_fields tags ') + + ierr = iMOAB_DefineTagStorage( mlnid, trim(seq_flds_l2x_fields)//C_NULL_CHAR, & + 1, & ! dense, double + 1, & ! number of components + tagindex ) + if (ierr > 0 ) & + call shr_sys_abort('Error: fail to create seq_flds_l2x_fields tags ') +#ifdef MOABDEBUG + ! debug test + outfile = 'LndDataMesh.h5m'//C_NULL_CHAR + wopts = ';PARALLEL=WRITE_PART'//C_NULL_CHAR ! + ! write out the mesh file to disk + ierr = iMOAB_WriteMesh(mlnid, trim(outfile), trim(wopts)) + if (ierr .ne. 0) then + call shr_sys_abort(subname//' ERROR in writing data mesh lnd ') + endif +#endif +#endif !---------------------------------------------------------------------------- ! Initialize MCT attribute vectors !---------------------------------------------------------------------------- @@ -339,8 +477,15 @@ subroutine dlnd_comp_run(EClock, x2l, l2x, & inst_suffix, logunit, case_name) ! !DESCRIPTION: run method for dlnd model - implicit none +#ifdef HAVE_MOAB +#ifdef MOABDEBUG + use iMOAB, only: iMOAB_WriteMesh +#endif + use seq_flds_mod , only: seq_flds_l2x_fields + use seq_flds_mod , only: moab_set_tag_from_av +#endif + implicit none ! !INPUT/OUTPUT PARAMETERS: type(ESMF_Clock) , intent(in) :: EClock type(mct_aVect) , intent(inout) :: x2l @@ -366,6 +511,17 @@ subroutine dlnd_comp_run(EClock, x2l, l2x, & integer(IN) :: nu ! unit number logical :: write_restart ! restart now character(len=18) :: date_str +#ifdef HAVE_MOAB + real(R8), allocatable, target :: datam(:) + type(mct_list) :: temp_list + integer :: size_list, index_list, lsize + type(mct_string) :: mctOStr ! + character*400 tagname, mct_field +#ifdef MOABDEBUG + integer :: cur_dlnd_stepno, ierr + character*100 outfile, wopts, lnum +#endif +#endif character(*), parameter :: F00 = "('(dlnd_comp_run) ',8a)" character(*), parameter :: F04 = "('(dlnd_comp_run) ',2a,2i8,'s')" @@ -464,6 +620,32 @@ subroutine dlnd_comp_run(EClock, x2l, l2x, & call t_stopf('DLND_RUN') +#ifdef HAVE_MOAB + lsize = mct_avect_lsize(l2x) ! is it the same as mct_avect_lsize(avstrm) ? + allocate(datam(lsize)) ! + call mct_list_init(temp_list ,seq_flds_l2x_fields) + size_list=mct_list_nitem (temp_list) + do index_list = 1, size_list + call mct_list_get(mctOStr,index_list,temp_list) + mct_field = mct_string_toChar(mctOStr) + tagname= trim(mct_field)//C_NULL_CHAR + call moab_set_tag_from_av(tagname, l2x, index_list, mlnid, datam, lsize) ! loop over all a2x fields, not just a few + enddo + call mct_list_clean(temp_list) + deallocate(datam) ! maybe we should keep it around, deallocate at the final only? + +#ifdef MOABDEBUG + call seq_timemgr_EClockGetData( EClock, stepno=cur_dlnd_stepno ) + write(lnum,"(I0.2)")cur_dlnd_stepno + outfile = 'dlnd_comp_run_'//trim(lnum)//'.h5m'//C_NULL_CHAR + wopts = 'PARALLEL=WRITE_PART'//C_NULL_CHAR + ierr = iMOAB_WriteMesh(mlnid, outfile, wopts) + if (ierr > 0 ) then + write(logunit,*) 'Failed to write data lnd component state ' + endif +#endif +#endif + end subroutine dlnd_comp_run !=============================================================================== diff --git a/components/data_comps/dlnd/src/lnd_comp_mct.F90 b/components/data_comps/dlnd/src/lnd_comp_mct.F90 index f5193ca8458..b699ec217f0 100644 --- a/components/data_comps/dlnd/src/lnd_comp_mct.F90 +++ b/components/data_comps/dlnd/src/lnd_comp_mct.F90 @@ -16,7 +16,11 @@ module lnd_comp_mct use dlnd_comp_mod , only: dlnd_comp_init, dlnd_comp_run, dlnd_comp_final use dlnd_shr_mod , only: dlnd_shr_read_namelists use seq_flds_mod , only: seq_flds_x2l_fields, seq_flds_l2x_fields - +#ifdef HAVE_MOAB + use seq_comm_mct, only : mlnid ! iMOAB app id for lnd + use iso_c_binding + use iMOAB , only: iMOAB_RegisterApplication +#endif ! !PUBLIC TYPES: implicit none private ! except @@ -52,7 +56,9 @@ module lnd_comp_mct !=============================================================================== subroutine lnd_init_mct( EClock, cdata, x2l, l2x, NLFilename ) - +#ifdef HAVE_MOAB + use shr_stream_mod, only: shr_stream_getDomainInfo, shr_stream_getFile +#endif ! !DESCRIPTION: initialize dlnd model implicit none @@ -146,13 +152,25 @@ subroutine lnd_init_mct( EClock, cdata, x2l, l2x, NLFilename ) !---------------------------------------------------------------------------- ! Initialize dlnd !---------------------------------------------------------------------------- - +#ifdef HAVE_MOAB + ierr = iMOAB_RegisterApplication(trim("DLND")//C_NULL_CHAR, mpicom, compid, mlnid) + if (ierr .ne. 0) then + write(logunit,*) subname,' error in registering data lnd comp' + call shr_sys_abort(subname//' ERROR in registering data lnd comp') + endif +#endif call dlnd_comp_init(Eclock, x2l, l2x, & seq_flds_x2l_fields, seq_flds_l2x_fields, & SDLND, gsmap, ggrid, mpicom, compid, my_task, master_task, & inst_suffix, inst_name, logunit, read_restart, & scmMode, scmlat, scmlon) - +#ifdef HAVE_MOAB + if (my_task == master_task) then + call seq_infodata_PutData( infodata, lnd_domain=SDLND%domainFile) ! we use the same one for regular case + ! in regular case, it is copied from fatmlndfrc ; so we don't know if it is data land or not + write(logunit,*), ' use this land domain file: ', SDLND%domainFile + endif +#endif !---------------------------------------------------------------------------- ! Fill infodata that needs to be returned from dlnd !---------------------------------------------------------------------------- diff --git a/components/data_comps/docn/cime_config/config_component.xml b/components/data_comps/docn/cime_config/config_component.xml index 5a30c69df6c..431d358f995 100644 --- a/components/data_comps/docn/cime_config/config_component.xml +++ b/components/data_comps/docn/cime_config/config_component.xml @@ -13,7 +13,7 @@ This file may have ocn desc entries. --> - DOCN + DOCN null mode prescribed ocean mode slab ocean mode @@ -45,7 +45,7 @@ char - prescribed,sst_aquap1,sst_aquap2,sst_aquap3,sst_aquap4,sst_aquap5,sst_aquap6,sst_aquap7,sst_aquap8,sst_aquap9,sst_aquap10,sst_aquapfile,som,som_aquap,sst_aquap_constant,interannual,null + prescribed,sst_aquap1,sst_aquap2,sst_aquap3,sst_aquap4,sst_aquap5,sst_aquap6,sst_aquap7,sst_aquap8,sst_aquap9,sst_aquap10,sst_aquap11,sst_aquap12,sst_aquap13,sst_aquap14,sst_aquap15,sst_aquapfile,som,som_aquap,sst_aquap_constant,interannual,null prescribed null @@ -63,6 +63,12 @@ sst_aquap8 sst_aquap9 sst_aquap10 + + sst_aquap11 + sst_aquap12 + sst_aquap13 + sst_aquap14 + sst_aquap15 sst_aquapfile sst_aquap_constant diff --git a/components/data_comps/docn/cime_config/namelist_definition_docn.xml b/components/data_comps/docn/cime_config/namelist_definition_docn.xml index 948902e3732..a191d088d7f 100644 --- a/components/data_comps/docn/cime_config/namelist_definition_docn.xml +++ b/components/data_comps/docn/cime_config/namelist_definition_docn.xml @@ -257,7 +257,7 @@ char streams shr_strdata_nml - SSTDATA,SST_AQUAP1,SST_AQUAP2,SST_AQUAP3,SST_AQUAP4,SST_AQUAP5,SST_AQUAP6,SST_AQUAP7,SST_AQUAP8,SST_AQUAP9,SST_AQUAP10,SST_AQUAPFILE,SST_AQUAP_CONSTANT,SOM,SOM_AQUAP,IAF,NULL,COPYALL + SSTDATA,SST_AQUAP1,SST_AQUAP2,SST_AQUAP3,SST_AQUAP4,SST_AQUAP5,SST_AQUAP6,SST_AQUAP7,SST_AQUAP8,SST_AQUAP9,SST_AQUAP10,SST_AQUAP11,SST_AQUAP12,SST_AQUAP13,SST_AQUAP14,SST_AQUAP15,SST_AQUAPFILE,SST_AQUAP_CONSTANT,SOM,SOM_AQUAP,IAF,NULL,COPYALL General method that operates on the data. This is generally implemented in the data models but is set in the strdata method for @@ -323,6 +323,11 @@ SST_AQUAP8 SST_AQUAP9 SST_AQUAP10 + SST_AQUAP11 + SST_AQUAP12 + SST_AQUAP13 + SST_AQUAP14 + SST_AQUAP15 SST_AQUAPFILE SST_AQUAP_CONSTANT SOM diff --git a/components/data_comps/docn/src/docn_comp_mod.F90 b/components/data_comps/docn/src/docn_comp_mod.F90 index e692882c9db..43bac32bff7 100644 --- a/components/data_comps/docn/src/docn_comp_mod.F90 +++ b/components/data_comps/docn/src/docn_comp_mod.F90 @@ -984,6 +984,7 @@ subroutine prescribed_sst(xc, yc, lsize, sst_option, sst) integer :: i real(r8) :: tmp, tmp1, pi real(r8) :: rlon(lsize), rlat(lsize) + real(r8) :: mean_SST, delta_SST real(r8), parameter :: pio180 = SHR_CONST_PI/180._r8 @@ -1013,8 +1014,8 @@ subroutine prescribed_sst(xc, yc, lsize, sst_option, sst) ! Control - if (sst_option < 1 .or. sst_option > 10) then - call shr_sys_abort ('prescribed_sst: ERROR: sst_option must be between 1 and 10') + if (sst_option < 1 .or. sst_option > 15) then + call shr_sys_abort ('prescribed_sst: ERROR: sst_option must be between 1 and 15') end if if (sst_option == 1 .or. sst_option == 6 .or. sst_option == 7 .or. sst_option == 8) then @@ -1174,6 +1175,20 @@ subroutine prescribed_sst(xc, yc, lsize, sst_option, sst) end do end if + !------------------------------------------------------------------------------- + ! RCEMIP phase 2 - Mock-Walker + if (sst_option>=11 .and. sst_option<=15) then + if (sst_option==11) then; mean_SST = 295 - TkFrz; delta_SST = 1.250; end if ! MW_295dT1p25 + if (sst_option==12) then; mean_SST = 300 - TkFrz; delta_SST = 0.625; end if ! MW_300dT0p625 + if (sst_option==13) then; mean_SST = 300 - TkFrz; delta_SST = 1.250; end if ! MW_300dT1p25 + if (sst_option==14) then; mean_SST = 300 - TkFrz; delta_SST = 2.500; end if ! MW_300dT2p5 + if (sst_option==15) then; mean_SST = 305 - TkFrz; delta_SST = 1.250; end if ! MW_305dT1p25 + do i = 1, lsize + sst(i) = mean_SST + (delta_SST/2) * cos( rlat(i) * 360/54 ) + end do + end if + !------------------------------------------------------------------------------- + end subroutine prescribed_sst end module docn_comp_mod diff --git a/components/data_comps/drof/cime_config/config_component.xml b/components/data_comps/drof/cime_config/config_component.xml index b5e0a8071fa..1c0052d9364 100644 --- a/components/data_comps/drof/cime_config/config_component.xml +++ b/components/data_comps/drof/cime_config/config_component.xml @@ -13,23 +13,14 @@ --> - Data runoff model + Data runoff model NULL mode COREv2 normal year forcing: - COREv2 normal year forcing: - COREv2 normal year forcing: - COREv2 normal year forcing: COREv2 interannual year forcing: - COREv2 interannual year forcing: - COREv2 interannual year forcing: - COREv2 interannual year forcing: CPLHIST mode: + JRA55 interannual forcing, v1.5, through 2023 JRA55 interannual forcing, v1.5, through 2020 - JRA55 interannual forcing, v1.5, through 2020, no rofi or rofl around AIS JRA55 interannual forcing, v1.4, through 2018 - JRA55 interannual forcing, v1.4, through 2018, no rofi around AIS - JRA55 interannual forcing, v1.4, through 2018, no rofl around AIS - JRA55 interannual forcing, v1.4, through 2018, no rofi or rofl around AIS JRA55 interannual forcing JRA55 Repeat Year Forcing v1.3 1984-1985 JRA55 Repeat Year Forcing v1.3 1990-1991 @@ -47,26 +38,17 @@ char - CPLHIST,DIATREN_ANN_RX1,DIATREN_ANN_AIS00_RX1,DIATREN_ANN_AIS45_RX1,DIATREN_ANN_AIS55_RX1,DIATREN_IAF_RX1,DIATREN_IAF_AIS00_RX1,DIATREN_IAF_AIS45_RX1,DIATREN_IAF_AIS55_RX1,IAF_JRA,IAF_JRA_1p5,IAF_JRA_1p5_AIS0ROF,IAF_JRA_1p4_2018,IAF_JRA_1p4_2018_AIS0ICE,IAF_JRA_1p4_2018_AIS0LIQ,IAF_JRA_1p4_2018_AIS0ROF,RYF8485_JRA,RYF9091_JRA,RYF0304_JRA,NULL + CPLHIST,DIATREN_ANN_RX1,DIATREN_IAF_RX1,IAF_JRA,IAF_JRA_1p5,IAF_JRA_1p4_2018,RYF8485_JRA,RYF9091_JRA,RYF0304_JRA,NULL DIATREN_ANN_RX1 NULL DIATREN_ANN_RX1 - DIATREN_ANN_AIS00_RX1 - DIATREN_ANN_AIS45_RX1 - DIATREN_ANN_AIS55_RX1 DIATREN_IAF_RX1 - DIATREN_IAF_AIS00_RX1 - DIATREN_IAF_AIS45_RX1 - DIATREN_IAF_AIS55_RX1 CPLHIST IAF_JRA + IAF_JRA_1p5 IAF_JRA_1p5 - IAF_JRA_1p5_AIS0ROF IAF_JRA_1p4_2018 - IAF_JRA_1p4_2018_AIS0ICE - IAF_JRA_1p4_2018_AIS0LIQ - IAF_JRA_1p4_2018_AIS0ROF RYF8485_JRA RYF9091_JRA RYF0304_JRA @@ -165,6 +147,7 @@ 1 1 1 + 1 run_component_drof env_run.xml @@ -179,6 +162,7 @@ 1958 1958 1958 + 1958 run_component_drof env_run.xml @@ -193,6 +177,7 @@ 2016 2018 2020 + 2023 run_component_drof env_run.xml diff --git a/components/data_comps/drof/cime_config/namelist_definition_drof.xml b/components/data_comps/drof/cime_config/namelist_definition_drof.xml index c4139552c70..d4d70074596 100644 --- a/components/data_comps/drof/cime_config/namelist_definition_drof.xml +++ b/components/data_comps/drof/cime_config/namelist_definition_drof.xml @@ -52,18 +52,8 @@ NULL rof.cplhist rof.diatren_ann_rx1 - rof.diatren_ann_ais00_rx1 - rof.diatren_ann_ais45_rx1 - rof.diatren_ann_ais55_rx1 rof.diatren_iaf_rx1 - rof.diatren_iaf_ais00_rx1 - rof.diatren_iaf_ais45_rx1 - rof.diatren_iaf_ais55_rx1 - rof.iaf_jra_1p4_2018_ais0ice - rof.iaf_jra_1p4_2018_ais0liq - rof.iaf_jra_1p4_2018_ais0rof rof.iaf_jra_1p4_2018 - rof.iaf_jra_1p5_ais0rof rof.iaf_jra_1p5 rof.iaf_jra rof.ryf8485_jra @@ -80,13 +70,7 @@ $DIN_LOC_ROOT/lnd/dlnd7/RX1 $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 $DIN_LOC_ROOT/lnd/dlnd7/JRA55 $DIN_LOC_ROOT/lnd/dlnd7/JRA55 $DIN_LOC_ROOT/lnd/dlnd7/JRA55 @@ -101,13 +85,7 @@ Stream domain file path(s). runoff.daitren.annual.20190226.nc - runoff.daitren.annual-AISx00.20190226.nc - runoff.daitren.annual-AISx45.20190226.nc - runoff.daitren.annual-AISx55.20190226.nc runoff.daitren.iaf.20120419.nc - runoff.daitren.iaf-AISx00.20120419.nc - runoff.daitren.iaf-AISx45.20120419.nc - runoff.daitren.iaf-AISx55.20120419.nc domain.roff.JRA025.170111.nc domain.roff.JRA025.170111.nc null @@ -134,27 +112,6 @@ arear area mask mask - - time time - xc lon - yc lat - arear area - mask mask - - - time time - xc lon - yc lat - arear area - mask mask - - - time time - xc lon - yc lat - arear area - mask mask - time time domrb_lon lon @@ -173,13 +130,7 @@ $DIN_LOC_ROOT/lnd/dlnd7/RX1 $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 - $DIN_LOC_ROOT/lnd/dlnd7/RX1 $DIN_LOC_ROOT/ocn/jra55/v1.5_noleap $DIN_LOC_ROOT/lnd/dlnd7/JRA55 $DIN_LOC_ROOT/lnd/dlnd7/JRA55 @@ -194,31 +145,13 @@ Stream data file path(s). runoff.daitren.annual.20190226.nc - runoff.daitren.annual.20190226.nc - runoff.daitren.annual.20190226.nc - runoff.daitren.annual.20190226.nc runoff.daitren.iaf.20120419.nc - runoff.daitren.iaf-AISx00.20120419.nc - runoff.daitren.iaf-AISx45.20120419.nc - runoff.daitren.iaf-AISx55.20120419.nc RAF_8485.JRA.v1.3.runoff.180404.nc RAF_9091.JRA.v1.3.runoff.180404.nc RAF_0304.JRA.v1.3.runoff.180404.nc - - JRA.v1.5.runoff.%y.no_rofi_no_rofl.240411.nc - JRA.v1.5.runoff.%y.240411.nc - - JRA.v1.4.runoff.%y.no_rofi.190214.nc - - - JRA.v1.4.runoff.%y.no_rofl.190214.nc - - - JRA.v1.4.runoff.%y.no_rofi_no_rofl.190214.nc - JRA.v1.4.runoff.%y.190214.nc @@ -244,9 +177,6 @@ runoff rofl - - runoff rofl - rofl rofl rofi rofi @@ -282,7 +212,6 @@ 1 1 1 - 1 $DROF_STRM_YR_ALIGN $DROF_STRM_YR_ALIGN $DROF_CPLHIST_YR_ALIGN @@ -296,13 +225,7 @@ First year of stream. 1 - 1 - 1 - 1 1948 - 1948 - 1948 - 1948 $DROF_STRM_YR_START 1984 1990 @@ -318,18 +241,9 @@ Last year of stream. 1 - 1 - 1 - 1 2009 - 2009 - 2009 - 2009 $DROF_STRM_YR_END $DROF_STRM_YR_END - $DROF_STRM_YR_END - $DROF_STRM_YR_END - $DROF_STRM_YR_END $DROF_STRM_YR_END 1984 1990 diff --git a/components/eam/bld/namelist_files/use_cases/RCEMIP_EAMv1.xml b/components/eam/bld/namelist_files/use_cases/RCEMIP_EAMv1.xml index c1c8e9e027a..146a9483b36 100644 --- a/components/eam/bld/namelist_files/use_cases/RCEMIP_EAMv1.xml +++ b/components/eam/bld/namelist_files/use_cases/RCEMIP_EAMv1.xml @@ -35,7 +35,7 @@ 0.0 - + diff --git a/components/eam/cime_config/config_compsets.xml b/components/eam/cime_config/config_compsets.xml index 7050bd0c8f9..6b3c41321ee 100644 --- a/components/eam/cime_config/config_compsets.xml +++ b/components/eam/cime_config/config_compsets.xml @@ -229,6 +229,50 @@ 2000_EAM%RCE-MMF2_SLND_SICE_DOCN%AQPCONST_SROF_SGLC_SWAV + + + FRCE-MW_295dT1p25 + 2000_EAM%RCE_SLND_SICE_DOCN%AQP11_SROF_SGLC_SWAV + + + FRCE-MW_300dT0p625 + 2000_EAM%RCE_SLND_SICE_DOCN%AQP12_SROF_SGLC_SWAV + + + FRCE-MW_300dT1p25 + 2000_EAM%RCE_SLND_SICE_DOCN%AQP13_SROF_SGLC_SWAV + + + FRCE-MW_300dT2p5 + 2000_EAM%RCE_SLND_SICE_DOCN%AQP14_SROF_SGLC_SWAV + + + FRCE-MW_305dT1p25 + 2000_EAM%RCE_SLND_SICE_DOCN%AQP15_SROF_SGLC_SWAV + + + + FRCE-MW-MMF1_295dT1p25 + 2000_EAM%RCE-MMF1_SLND_SICE_DOCN%AQP11_SROF_SGLC_SWAV + + + FRCE-MW-MMF1_300dT0p625 + 2000_EAM%RCE-MMF1_SLND_SICE_DOCN%AQP12_SROF_SGLC_SWAV + + + FRCE-MW-MMF1_300dT1p25 + 2000_EAM%RCE-MMF1_SLND_SICE_DOCN%AQP13_SROF_SGLC_SWAV + + + FRCE-MW-MMF1_300dT2p5 + 2000_EAM%RCE-MMF1_SLND_SICE_DOCN%AQP14_SROF_SGLC_SWAV + + + FRCE-MW-MMF1_305dT1p25 + 2000_EAM%RCE-MMF1_SLND_SICE_DOCN%AQP15_SROF_SGLC_SWAV + + + diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/readme b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/readme new file mode 100644 index 00000000000..35f64eb4b9a --- /dev/null +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/readme @@ -0,0 +1 @@ +test for sat hist capability components/eam/src/control/sat_hist.F90 diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/shell_commands b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/shell_commands new file mode 100644 index 00000000000..92cb057059a --- /dev/null +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/shell_commands @@ -0,0 +1,3 @@ +#!/bin/bash + +./xmlchange RUN_STARTDATE=2018-01-01 diff --git a/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/user_nl_eam b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/user_nl_eam new file mode 100644 index 00000000000..e9b723f13b8 --- /dev/null +++ b/components/eam/cime_config/testdefs/testmods_dirs/eam/sathist_F2010/user_nl_eam @@ -0,0 +1,7 @@ +&satellite_options_nl + sathist_mfilt = 10000, + sathist_track_infile = '$DIN_LOC_ROOT/atm/waccm/sat/satellite_profilelist_orcas_to_socrates_c190208.nc' + sathist_hfilename_spec = '%c.eam.h9.sathist.%y-%m-%d-%s.nc' + sathist_nclosest = 1 + sathist_ntimestep = 1 + sathist_fincl = 'T', 'PS' diff --git a/components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl b/components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl index 3ecd465f7a1..3a47a8bb2ff 100644 --- a/components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl +++ b/components/eam/cime_config/usermods_dirs/rcemip/user_nl_cpl @@ -31,4 +31,5 @@ seq_flux_mct_albdif = 0.07 seq_flux_mct_albdir = 0.07 seq_flux_atmocn_minwind = 1 -constant_zenith_deg = 42.05 \ No newline at end of file +constant_zenith_deg = 42.04 + diff --git a/components/eam/src/chemistry/modal_aero/modal_aero_amicphys.F90 b/components/eam/src/chemistry/modal_aero/modal_aero_amicphys.F90 index 6546677cac3..5f7f690cf62 100644 --- a/components/eam/src/chemistry/modal_aero/modal_aero_amicphys.F90 +++ b/components/eam/src/chemistry/modal_aero/modal_aero_amicphys.F90 @@ -3530,7 +3530,7 @@ subroutine mam_soaexch_vbs_1subarea( & ! convert sat vapor conc from ug/m^3 to mol/m^3 then to mol/liter tmpa = (c0_soa_298(ll)*1.0e-6_r8/mw_gas(igas)) * 1.0e-3_r8 ! calc sat vapor pressure (atm) from molar-conc and temp [ 0.082056 = gas constant in (atm/deg-K/(mol/liter)) ] - p0_soa_298(ll) = 0.082056_r8*tmpa*temp + p0_soa_298(ll) = 0.082056_r8*tmpa*298.0_r8 end do ! calc soa gas saturation molar-mixing-ratio at local temp and air-pressure diff --git a/components/eam/src/control/sat_hist.F90 b/components/eam/src/control/sat_hist.F90 index 64b978ac0f1..17d2a92bc7b 100644 --- a/components/eam/src/control/sat_hist.F90 +++ b/components/eam/src/control/sat_hist.F90 @@ -6,18 +6,20 @@ module sat_hist use perf_mod, only: t_startf, t_stopf - use shr_kind_mod, only: r8 => shr_kind_r8 + use shr_kind_mod, only: r4 => shr_kind_r4 + use shr_kind_mod, only: r8 => shr_kind_r8, cl=>shr_kind_cl use cam_logfile, only: iulog - use ppgrid, only: pcols, pver, begchunk, endchunk + use ppgrid, only: pcols, pver, pverp, begchunk, endchunk use cam_history_support, only: fieldname_lenp2, max_string_len, ptapes use spmd_utils, only: masterproc, iam use cam_abortutils, only: endrun - use pio, only: file_desc_t, iosystem_desc_t, iosystem_desc_t, var_desc_t, io_desc_t - use pio, only: pio_openfile, pio_redef, pio_enddef, pio_inq_dimid, pio_inq_varid, pio_seterrorhandling, pio_def_var + use pio, only: file_desc_t,iosystem_desc_t, var_desc_t, io_desc_t + use pio, only: pio_inq_dimid, pio_inq_varid + use pio, only: pio_seterrorhandling, pio_def_var use pio, only: pio_inq_dimlen, pio_get_att, pio_put_att, pio_get_var, pio_put_var, pio_write_darray - use pio, only: pio_real, pio_int, pio_double - use pio, only: PIO_WRITE,PIO_NOWRITE, PIO_NOERR, PIO_BCAST_ERROR, PIO_INTERNAL_ERROR, PIO_Rearr_box, PIO_GLOBAL + use pio, only: pio_real,pio_double + use pio, only: PIO_NOWRITE, PIO_NOERR, PIO_BCAST_ERROR, PIO_INTERNAL_ERROR, PIO_GLOBAL use spmd_utils, only: mpicom #ifdef SPMD use mpishorthand, only: mpichar, mpiint @@ -82,6 +84,7 @@ module sat_hist real(r8), parameter :: rad2deg = 180._r8/pi ! degrees per radian + contains !------------------------------------------------------------------------------- @@ -122,7 +125,7 @@ subroutine sat_hist_readnl(nlfile, hfilename_spec, mfilt, fincl, nhtfrq, avgflag ! set defaults sathist_track_infile = ' ' - sathist_hfilename_spec = '%c.cam' // trim(inst_suffix) // '.hs.%y-%m-%d-%s.nc' + sathist_hfilename_spec = '%c.eam.hs.' // trim(inst_suffix) // '.%y-%m-%d-%s.nc' sathist_fincl(:) = ' ' sathist_mfilt = 100000 sathist_nclosest = 1 @@ -189,14 +192,13 @@ end subroutine sat_hist_readnl subroutine sat_hist_init use cam_pio_utils, only: cam_pio_openfile use ioFileMod, only: getfil - use spmd_utils, only: npes use time_manager, only: get_step_size use string_utils, only: to_lower, GLC implicit none character(len=max_string_len) :: locfn ! Local filename - integer :: ierr, dimid, i + integer :: ierr, dimid character(len=128) :: date_format @@ -406,17 +408,15 @@ end subroutine sat_hist_define !------------------------------------------------------------------------------- subroutine sat_hist_write( tape , nflds, nfils) - use ppgrid, only : pcols, begchunk, endchunk use phys_grid, only: phys_decomp use dyn_grid, only: dyn_decomp use cam_history_support, only : active_entry - use pio, only : pio_file_is_open - implicit none + use pio, only : pio_file_is_open, pio_syncfile type(active_entry) :: tape integer, intent(in) :: nflds integer, intent(inout) :: nfils - integer :: t, f, i, ncols, nocols + integer :: ncols, nocols integer :: ierr integer, allocatable :: col_ndxs(:) @@ -430,9 +430,13 @@ subroutine sat_hist_write( tape , nflds, nfils) real(r8),allocatable :: phs_dists(:) integer :: coldim - - integer :: io_type - logical :: has_dyn_flds + logical :: has_dyn_flds = .false. + logical :: has_phys_srf_flds = .false. + logical :: has_phys_lev_flds = .false. + logical :: has_phys_ilev_flds = .false. + logical :: has_dyn_srf_flds = .false. + logical :: has_dyn_lev_flds = .false. + logical :: has_dyn_ilev_flds = .false. if (.not.has_sat_hist) return @@ -456,13 +460,11 @@ subroutine sat_hist_write( tape , nflds, nfils) allocate( mlons(nocols) ) allocate( phs_dists(nocols) ) - has_dyn_flds = .false. - dyn_flds_loop: do f=1,nflds - if ( tape%hlist(f)%field%decomp_type == dyn_decomp ) then - has_dyn_flds = .true. - exit dyn_flds_loop - endif - enddo dyn_flds_loop + call scan_flds( tape, nflds & + , has_phys_srf_flds, has_phys_lev_flds, has_phys_ilev_flds & + , has_dyn_srf_flds, has_dyn_lev_flds, has_dyn_ilev_flds ) + + has_dyn_flds = has_dyn_srf_flds .or. has_dyn_lev_flds .or. has_dyn_ilev_flds call get_indices( obs_lats, obs_lons, ncols, nocols, has_dyn_flds, col_ndxs, chk_ndxs, & fdyn_ndxs, ldyn_ndxs, phs_owners, dyn_owners, mlats, mlons, phs_dists ) @@ -479,16 +481,35 @@ subroutine sat_hist_write( tape , nflds, nfils) call write_record_coord( tape, mlats(:), mlons(:), phs_dists(:), ncols, nfils ) - do f=1,nflds + ! dump columns of 2D fields + if (has_phys_srf_flds) then + call dump_columns( tape%File, tape%hlist, nflds, nocols, 1, nfils, & + col_ndxs, chk_ndxs, phs_owners, phys_decomp ) + endif + if (has_dyn_srf_flds) then + call dump_columns( tape%File, tape%hlist, nflds, nocols, 1, nfils, & + fdyn_ndxs, ldyn_ndxs, dyn_owners, dyn_decomp ) + endif - select case (tape%hlist(f)%field%decomp_type) - case (phys_decomp) - call dump_columns(tape%File, tape%hlist(f), nocols, nfils, col_ndxs(:), chk_ndxs(:), phs_owners(:) ) - case (dyn_decomp) - call dump_columns(tape%File, tape%hlist(f), nocols, nfils, fdyn_ndxs(:), ldyn_ndxs(:), dyn_owners(:) ) - end select + ! dump columns of 3D fields defined on mid pres levels + if (has_phys_lev_flds) then + call dump_columns( tape%File, tape%hlist, nflds, nocols, pver, nfils, & + col_ndxs, chk_ndxs, phs_owners, phys_decomp ) + endif + if (has_dyn_lev_flds) then + call dump_columns( tape%File, tape%hlist, nflds, nocols, pver, nfils, & + fdyn_ndxs, ldyn_ndxs, dyn_owners, dyn_decomp ) + endif - enddo + ! dump columns of 3D fields defined on interface pres levels + if (has_phys_ilev_flds) then + call dump_columns( tape%File, tape%hlist, nflds, nocols, pverp, nfils, & + col_ndxs, chk_ndxs, phs_owners, phys_decomp ) + endif + if (has_dyn_ilev_flds) then + call dump_columns( tape%File, tape%hlist, nflds, nocols, pverp, nfils, & + fdyn_ndxs, ldyn_ndxs, dyn_owners, dyn_decomp ) + endif deallocate( col_ndxs, chk_ndxs, fdyn_ndxs, ldyn_ndxs, phs_owners, dyn_owners ) deallocate( mlons, mlats, phs_dists ) @@ -501,92 +522,167 @@ subroutine sat_hist_write( tape , nflds, nfils) end subroutine sat_hist_write !------------------------------------------------------------------------------- - subroutine dump_columns( File, hitem, ncols, nfils, fdims, ldims, owners ) - use cam_history_support, only: field_info, hentry, hist_coords, fillvalue - use pio, only: pio_initdecomp, pio_freedecomp, pio_setframe, pio_iam_iotask, pio_setdebuglevel, pio_offset_kind +! FIXME extra work > +! dump_columns routine is doing unnecessary extra work serially +! this happens because there is an unneeded mpi_allreduce call +! and then the gathered data is written in a serial manner; this +! could be improved by avoiding the mpi_allreduce call, and then +! writing local data out using pio_write_darray, which is parallel +! FIXME extra work < + subroutine dump_columns( File, hitems, nflds, ncols, nlevs, nfils, fdims, ldims, owners, decomp ) + use cam_history_support, only: field_info, hentry, fillvalue + use pio, only: pio_setframe, pio_offset_kind + use spmd_utils, only: mpi_real4, mpi_real8, mpicom, mpi_sum type(File_desc_t),intent(inout) :: File - type(hentry), intent(in), target :: hitem + type(hentry), intent(in), target :: hitems(:) + integer, intent(in) :: nflds integer, intent(in) :: ncols + integer, intent(in) :: nlevs integer, intent(in) :: nfils integer, intent(in) :: fdims(:) integer, intent(in) :: ldims(:) integer, intent(in) :: owners(:) + integer, intent(in) :: decomp + type(field_info), pointer :: field type(var_desc_t) :: vardesc - type(iosystem_desc_t), pointer :: sat_iosystem - type(io_desc_t) :: iodesc - integer :: t, ierr, ndims - integer, allocatable :: dimlens(:) + integer :: ierr - real(r8), allocatable :: buf(:) - integer, allocatable :: dof(:) - integer :: i,k, cnt + real(r8) :: sbuf1d(ncols),rbuf1d(ncols) + real(r4) :: buf1d(ncols) + real(r8) :: sbuf2d(nlevs,ncols), rbuf2d(nlevs,ncols) + real(r4) :: buf2d(nlevs,ncols) + integer :: i,k,f, cnt call t_startf ('sat_hist::dump_columns') - sat_iosystem => File%iosystem - field => hitem%field - vardesc = hitem%varid(1) - - - ndims=1 - if(associated(field%mdims)) then - ndims = size(field%mdims)+1 - else if(field%numlev>1) then - ndims=2 - end if - allocate(dimlens(ndims)) - dimlens(ndims)=ncols - if(ndims>2) then - do i=1,ndims-1 - dimlens(i)=hist_coords(field%mdims(i))%dimsize - enddo - else if(field%numlev>1) then - dimlens(1) = field%numlev - end if - - - allocate( buf( product(dimlens) ) ) - allocate( dof( product(dimlens) ) ) + do f = 1,nflds + field => hitems(f)%field + + if (field%numlev==nlevs .and. field%decomp_type==decomp) then + vardesc = hitems(f)%varid(1) + + if (nlevs==1) then + sbuf1d = 0.0_r8 + rbuf1d = 0.0_r8 + do i=1,ncols + if ( iam == owners(i) ) then + sbuf1d(i) = hitems(f)%hbuf( fdims(i), 1, ldims(i) ) + endif + enddo + ! FIXME extra work: unnecessary mpi call, then serial write + ! FIXME extra work: can use pio_write_darray on local data instead + call mpi_allreduce(sbuf1d,rbuf1d,ncols,mpi_real8, mpi_sum, mpicom, ierr) + buf1d(:) = real(rbuf1d(:),r4) + ierr = pio_put_var(File, vardesc, (/nfils/),(/ncols/), buf1d(:)) + if ( ierr /= PIO_NOERR ) then + call endrun('sat_hist::dump_columns: pio_put_var error') + endif + else + sbuf2d = 0.0_r8 + rbuf2d = 0.0_r8 + do i=1,ncols + if ( iam == owners(i) ) then + do k = 1,nlevs + sbuf2d(k,i) = hitems(f)%hbuf( fdims(i), k, ldims(i) ) + enddo + endif + enddo + ! FIXME extra work: unnecessary mpi call, then serial write + ! FIXME extra work: can use pio_write_darray on local data instead + call mpi_allreduce(sbuf2d,rbuf2d,ncols*nlevs,mpi_real8, mpi_sum, mpicom, ierr) + buf2d(:,:) = real(rbuf2d(:,:),r4) + ierr = pio_put_var(File, vardesc, (/1,nfils/),(/nlevs,ncols/), buf2d(:,:)) + if ( ierr /= PIO_NOERR ) then + call endrun('sat_hist::dump_columns: pio_put_var error') + endif + endif - cnt = 0 - buf = fillvalue - dof = 0 + endif - do i = 1,ncols - do k = 1,field%numlev - cnt = cnt+1 - if ( iam == owners(i) ) then - buf(cnt) = hitem%hbuf( fdims(i), k, ldims(i) ) - dof(cnt) = cnt - endif - enddo enddo - call pio_setframe(File, vardesc, int(-1,kind=PIO_OFFSET_KIND)) - - call pio_initdecomp(sat_iosystem, pio_double, dimlens, dof, iodesc ) + call t_stopf ('sat_hist::dump_columns') - call pio_setframe(File, vardesc, int(nfils,kind=PIO_OFFSET_KIND)) + end subroutine dump_columns - call pio_write_darray(File, vardesc, iodesc, buf, ierr, fillval=fillvalue) +!------------------------------------------------------------------------------- +! scan the fields for possible different decompositions +!------------------------------------------------------------------------------- + subroutine scan_flds( tape, nflds & + , has_phys_srf_flds, has_phys_lev_flds, has_phys_ilev_flds & + , has_dyn_srf_flds, has_dyn_lev_flds, has_dyn_ilev_flds ) + use cam_history_support, only : active_entry + use phys_grid, only: phys_decomp + use dyn_grid, only: dyn_decomp - call pio_freedecomp(sat_iosystem, iodesc) + type(active_entry), intent(in) :: tape + integer, intent(in) :: nflds + logical, save :: flds_scanned + logical, intent(out) :: has_phys_srf_flds + logical, intent(out) :: has_phys_lev_flds + logical, intent(out) :: has_phys_ilev_flds + logical, intent(out) :: has_dyn_srf_flds + logical, intent(out) :: has_dyn_lev_flds + logical, intent(out) :: has_dyn_ilev_flds + + integer :: f + character(len=cl) :: msg1, msg2 + + if (flds_scanned) return + + do f = 1,nflds + if ( tape%hlist(f)%field%decomp_type == phys_decomp ) then + if ( tape%hlist(f)%field%numlev == 1 ) then + has_phys_srf_flds = .true. + elseif ( tape%hlist(f)%field%numlev == pver ) then + has_phys_lev_flds = .true. + elseif ( tape%hlist(f)%field%numlev == pverp ) then + has_phys_ilev_flds = .true. + else + call endrun('sat_hist::scan_flds numlev error : '//tape%hlist(f)%field%name) + endif + elseif ( tape%hlist(f)%field%decomp_type == dyn_decomp ) then + if ( tape%hlist(f)%field%numlev == 1 ) then + has_dyn_srf_flds = .true. + elseif ( tape%hlist(f)%field%numlev == pver ) then + has_dyn_lev_flds = .true. + elseif ( tape%hlist(f)%field%numlev == pverp ) then + has_dyn_ilev_flds = .true. + else + call endrun('sat_hist::scan_flds numlev error : '//tape%hlist(f)%field%name) + endif + else + call endrun('sat_hist::scan_flds decomp_type error : '//tape%hlist(f)%field%name) + endif - deallocate( buf ) - deallocate( dof ) - deallocate( dimlens ) + ! Check that the only "mdim" is the vertical coordinate. + if (has_phys_srf_flds .or. has_phys_lev_flds .or. has_phys_ilev_flds .or. & + has_dyn_srf_flds .or. has_dyn_lev_flds .or. has_dyn_ilev_flds) then + ! The mdims pointer is unassociated on a restart. The restart initialization + ! should be fixed rather than requiring the check to make sure it is associated. + if (associated(tape%hlist(f)%field%mdims)) then + if (size(tape%hlist(f)%field%mdims) > 1) then + msg1 = 'sat_hist::scan_flds mdims error :'//tape%hlist(f)%field%name + msg2 = trim(msg1)//' has mdims in addition to the vertical coordinate.'//& + new_line('a')//' This is not currently supported.' + write(iulog,*) msg2 + call endrun(msg1) + end if + end if + end if - call t_stopf ('sat_hist::dump_columns') + enddo - end subroutine dump_columns + flds_scanned = .true. + end subroutine scan_flds !------------------------------------------------------------------------------- !------------------------------------------------------------------------------- subroutine read_next_position( ncols ) - use time_manager, only: get_curr_date, get_prev_date + use time_manager, only: get_curr_date use time_manager, only: set_time_float_from_date implicit none @@ -626,8 +722,14 @@ subroutine read_next_position( ncols ) call read_buffered_datetime( datetime, i ) - if ( datetime>begdatetime .and. beg_ndx<0 ) beg_ndx = i - if ( datetime>enddatetime ) exit bnds_loop + if (datetime > begdatetime .and. beg_ndx < 0) then + beg_ndx = i + end if + + if (datetime > enddatetime) then + exit bnds_loop + end if + end_ndx = i enddo bnds_loop @@ -660,7 +762,7 @@ end subroutine read_next_position !------------------------------------------------------------------------------- subroutine write_record_coord( tape, mod_lats, mod_lons, mod_dists, ncols, nfils ) - use time_manager, only: get_nstep, get_curr_date, get_curr_time + use time_manager, only: get_curr_date, get_curr_time use cam_history_support, only : active_entry implicit none type(active_entry), intent(inout) :: tape @@ -671,9 +773,8 @@ subroutine write_record_coord( tape, mod_lats, mod_lons, mod_dists, ncols, nfils real(r8), intent(in) :: mod_dists(ncols * sathist_nclosest) integer, intent(in) :: nfils - integer :: t, ierr, i + integer :: ierr, i integer :: yr, mon, day ! year, month, and day components of a date - integer :: nstep ! current timestep number integer :: ncdate ! current date in integer format [yyyymmdd] integer :: ncsec ! current time of day [seconds] integer :: ndcur ! day component of current time @@ -686,7 +787,6 @@ subroutine write_record_coord( tape, mod_lats, mod_lons, mod_dists, ncols, nfils call t_startf ('sat_hist::write_record_coord') - nstep = get_nstep() call get_curr_date(yr, mon, day, ncsec) ncdate = yr*10000 + mon*100 + day call get_curr_time(ndcur, nscur) diff --git a/components/eam/src/physics/cam/cam_diagnostics.F90 b/components/eam/src/physics/cam/cam_diagnostics.F90 index ead4f558f05..5dca258775d 100644 --- a/components/eam/src/physics/cam/cam_diagnostics.F90 +++ b/components/eam/src/physics/cam/cam_diagnostics.F90 @@ -306,6 +306,7 @@ subroutine diag_init() call addfld ('MQ',(/ 'lev' /), 'A','kg/m2','Water vapor mass in layer') call addfld ('TMQ',horiz_only, 'A','kg/m2','Total (vertically integrated) precipitable water', & standard_name='atmosphere_mass_content_of_water_vapor') + call addfld ('TMQS',horiz_only, 'A','kg/m2','Total (vertically integrated) saturated precipitable water') call addfld ('TTQ',horiz_only, 'A', 'kg/m/s','Total (vertically integrated) vapor transport') call addfld ('TUQ',horiz_only, 'A','kg/m/s','Total (vertically integrated) zonal water flux') call addfld ('TVQ',horiz_only, 'A','kg/m/s','Total (vertically integrated) meridional water flux') @@ -1368,6 +1369,14 @@ subroutine diag_phys_writeout(state, psl) if (moist_physics) then + ! Mass of saturated q vertically integrated + call qsat(state%t(:ncol,:), state%pmid(:ncol,:), tem2(:ncol,:), ftem(:ncol,:)) + ftem(:ncol,:) = ftem(:ncol,:) * state%pdel(:ncol,:) * rga + do k=2,pver + ftem(:ncol,1) = ftem(:ncol,1) + ftem(:ncol,k) + end do + call outfld ('TMQS ',ftem, pcols ,lchnk ) + ! Relative humidity call qsat(state%t(:ncol,:), state%pmid(:ncol,:), & tem2(:ncol,:), ftem(:ncol,:)) diff --git a/components/eam/src/physics/crm/pam/CMakeLists.txt b/components/eam/src/physics/crm/pam/CMakeLists.txt index 4a8f88f669b..ac629fa89fd 100644 --- a/components/eam/src/physics/crm/pam/CMakeLists.txt +++ b/components/eam/src/physics/crm/pam/CMakeLists.txt @@ -9,8 +9,9 @@ set(PAM_DRIVER_SRC params.F90) add_library(pam_driver - ${PAM_DRIVER_SRC}) + ${PAM_DRIVER_SRC}) +set(SCREAM_LIBS_ONLY TRUE) set(SCREAM_HOME ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../..) add_library(eamxx_physics INTERFACE ${SCREAM_HOME}/components/eamxx/src/physics/) diff --git a/components/eam/src/physics/crm/pam/external b/components/eam/src/physics/crm/pam/external index 3ea20ad38f2..1c37054d1ff 160000 --- a/components/eam/src/physics/crm/pam/external +++ b/components/eam/src/physics/crm/pam/external @@ -1 +1 @@ -Subproject commit 3ea20ad38f286730973429e0d491420a6f599f11 +Subproject commit 1c37054d1ff9b160290cc286dcbd3cdc6cd7e7f6 diff --git a/components/eam/src/physics/crm/pam/pam_statistics.h b/components/eam/src/physics/crm/pam/pam_statistics.h index bb277600dfa..8166b9912d2 100644 --- a/components/eam/src/physics/crm/pam/pam_statistics.h +++ b/components/eam/src/physics/crm/pam/pam_statistics.h @@ -458,29 +458,34 @@ inline void pam_statistics_compute_means( pam::PamCoupler &coupler ) { if (clear_rh_cnt(k,iens)>0) { clear_rh(k,iens) = clear_rh(k,iens) / clear_rh_cnt(k,iens); } - phys_tend_sgs_temp (k,iens) = phys_tend_sgs_temp (k,iens) / phys_tend_sgs_cnt (iens); - phys_tend_sgs_qv (k,iens) = phys_tend_sgs_qv (k,iens) / phys_tend_sgs_cnt (iens); - phys_tend_sgs_qc (k,iens) = phys_tend_sgs_qc (k,iens) / phys_tend_sgs_cnt (iens); - phys_tend_sgs_qi (k,iens) = phys_tend_sgs_qi (k,iens) / phys_tend_sgs_cnt (iens); - phys_tend_sgs_qr (k,iens) = phys_tend_sgs_qr (k,iens) / phys_tend_sgs_cnt (iens); - - phys_tend_micro_temp(k,iens) = phys_tend_micro_temp(k,iens) / phys_tend_micro_cnt(iens); - phys_tend_micro_qv (k,iens) = phys_tend_micro_qv (k,iens) / phys_tend_micro_cnt(iens); - phys_tend_micro_qc (k,iens) = phys_tend_micro_qc (k,iens) / phys_tend_micro_cnt(iens); - phys_tend_micro_qi (k,iens) = phys_tend_micro_qi (k,iens) / phys_tend_micro_cnt(iens); - phys_tend_micro_qr (k,iens) = phys_tend_micro_qr (k,iens) / phys_tend_micro_cnt(iens); - - phys_tend_dycor_temp (k,iens) = phys_tend_dycor_temp (k,iens) / phys_tend_dycor_cnt(iens); - phys_tend_dycor_qv (k,iens) = phys_tend_dycor_qv (k,iens) / phys_tend_dycor_cnt(iens); - phys_tend_dycor_qc (k,iens) = phys_tend_dycor_qc (k,iens) / phys_tend_dycor_cnt(iens); - phys_tend_dycor_qi (k,iens) = phys_tend_dycor_qi (k,iens) / phys_tend_dycor_cnt(iens); - phys_tend_dycor_qr (k,iens) = phys_tend_dycor_qr (k,iens) / phys_tend_dycor_cnt(iens); - - phys_tend_sponge_temp(k,iens) = phys_tend_sponge_temp(k,iens) / phys_tend_sponge_cnt(iens); - phys_tend_sponge_qv (k,iens) = phys_tend_sponge_qv (k,iens) / phys_tend_sponge_cnt(iens); - phys_tend_sponge_qc (k,iens) = phys_tend_sponge_qc (k,iens) / phys_tend_sponge_cnt(iens); - phys_tend_sponge_qi (k,iens) = phys_tend_sponge_qi (k,iens) / phys_tend_sponge_cnt(iens); - phys_tend_sponge_qr (k,iens) = phys_tend_sponge_qr (k,iens) / phys_tend_sponge_cnt(iens); + if (phys_tend_sgs_cnt(iens)>0) { + phys_tend_sgs_temp (k,iens) = phys_tend_sgs_temp (k,iens) / phys_tend_sgs_cnt (iens); + phys_tend_sgs_qv (k,iens) = phys_tend_sgs_qv (k,iens) / phys_tend_sgs_cnt (iens); + phys_tend_sgs_qc (k,iens) = phys_tend_sgs_qc (k,iens) / phys_tend_sgs_cnt (iens); + phys_tend_sgs_qi (k,iens) = phys_tend_sgs_qi (k,iens) / phys_tend_sgs_cnt (iens); + phys_tend_sgs_qr (k,iens) = phys_tend_sgs_qr (k,iens) / phys_tend_sgs_cnt (iens); + } + if (phys_tend_micro_cnt(iens)>0) { + phys_tend_micro_temp(k,iens) = phys_tend_micro_temp(k,iens) / phys_tend_micro_cnt(iens); + phys_tend_micro_qv (k,iens) = phys_tend_micro_qv (k,iens) / phys_tend_micro_cnt(iens); + phys_tend_micro_qc (k,iens) = phys_tend_micro_qc (k,iens) / phys_tend_micro_cnt(iens); + phys_tend_micro_qi (k,iens) = phys_tend_micro_qi (k,iens) / phys_tend_micro_cnt(iens); + phys_tend_micro_qr (k,iens) = phys_tend_micro_qr (k,iens) / phys_tend_micro_cnt(iens); + } + if (phys_tend_dycor_cnt(iens)>0) { + phys_tend_dycor_temp (k,iens) = phys_tend_dycor_temp (k,iens) / phys_tend_dycor_cnt(iens); + phys_tend_dycor_qv (k,iens) = phys_tend_dycor_qv (k,iens) / phys_tend_dycor_cnt(iens); + phys_tend_dycor_qc (k,iens) = phys_tend_dycor_qc (k,iens) / phys_tend_dycor_cnt(iens); + phys_tend_dycor_qi (k,iens) = phys_tend_dycor_qi (k,iens) / phys_tend_dycor_cnt(iens); + phys_tend_dycor_qr (k,iens) = phys_tend_dycor_qr (k,iens) / phys_tend_dycor_cnt(iens); + } + if (phys_tend_sponge_cnt(iens)>0) { + phys_tend_sponge_temp(k,iens) = phys_tend_sponge_temp(k,iens) / phys_tend_sponge_cnt(iens); + phys_tend_sponge_qv (k,iens) = phys_tend_sponge_qv (k,iens) / phys_tend_sponge_cnt(iens); + phys_tend_sponge_qc (k,iens) = phys_tend_sponge_qc (k,iens) / phys_tend_sponge_cnt(iens); + phys_tend_sponge_qi (k,iens) = phys_tend_sponge_qi (k,iens) / phys_tend_sponge_cnt(iens); + phys_tend_sponge_qr (k,iens) = phys_tend_sponge_qr (k,iens) / phys_tend_sponge_cnt(iens); + } }); //------------------------------------------------------------------------------------------------ } diff --git a/components/eam/src/physics/crm/rrtmgp/radiation.F90 b/components/eam/src/physics/crm/rrtmgp/radiation.F90 index b7b253d1b6c..b80ea0c2fe1 100644 --- a/components/eam/src/physics/crm/rrtmgp/radiation.F90 +++ b/components/eam/src/physics/crm/rrtmgp/radiation.F90 @@ -768,6 +768,18 @@ subroutine radiation_init(state) call addfld('FLNTC'//diag(icall), horiz_only, 'A', 'W/m2', & 'Clearsky net longwave flux at top of model', & sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLUTOA'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Upwelling longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLNTOA'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Net longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLUTOAC'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Clearsky upwelling longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLNTOAC'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Clearsky net longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) call addfld('LWCF'//diag(icall), horiz_only, 'A', 'W/m2', & 'Longwave cloud forcing', & sampling_seq='rad_lwsw', flag_xyfill=.true.) @@ -2494,6 +2506,7 @@ subroutine output_fluxes_lw(icall, state, flux_all, flux_clr, qrl, qrlc) ! Working arrays real(r8), dimension(pcols,pver+1) :: flux_up, flux_dn, flux_net integer :: ncol + integer :: ktop_rad = 1 ncol = state%ncol @@ -2531,6 +2544,12 @@ subroutine output_fluxes_lw(icall, state, flux_all, flux_clr, qrl, qrlc) call outfld('FLUTC'//diag(icall), flux_clr%flux_up(1:ncol,ktop), ncol, state%lchnk) call outfld('FLDSC'//diag(icall), flux_clr%flux_dn(1:ncol,kbot+1), ncol, state%lchnk) + ! TOA fluxes (above model top, use index to rad top) + call outfld('FLUTOA'//diag(icall), flux_all%flux_up(1:ncol,ktop_rad), ncol, state%lchnk) + call outfld('FLNTOA'//diag(icall), flux_all%flux_net(1:ncol,ktop_rad), ncol, state%lchnk) + call outfld('FLUTOAC'//diag(icall), flux_clr%flux_up(1:ncol,ktop_rad), ncol, state%lchnk) + call outfld('FLNTOAC'//diag(icall), flux_clr%flux_net(1:ncol,ktop_rad), ncol, state%lchnk) + ! Calculate and output the cloud radiative effect (LWCF in history) cloud_radiative_effect(1:ncol) = flux_all%flux_net(1:ncol,ktop) - flux_clr%flux_net(1:ncol,ktop) call outfld('LWCF'//diag(icall), cloud_radiative_effect, ncol, state%lchnk) diff --git a/components/eam/src/physics/rrtmgp/external b/components/eam/src/physics/rrtmgp/external index e64b99cce24..b24ca1f616e 160000 --- a/components/eam/src/physics/rrtmgp/external +++ b/components/eam/src/physics/rrtmgp/external @@ -1 +1 @@ -Subproject commit e64b99cce24eb31bb6f317bddb6f0ffbdfaf8bb7 +Subproject commit b24ca1f616e45659b334dbd7297017cb7927367e diff --git a/components/eam/src/physics/rrtmgp/radiation.F90 b/components/eam/src/physics/rrtmgp/radiation.F90 index 0c715000951..5c87c3376d9 100644 --- a/components/eam/src/physics/rrtmgp/radiation.F90 +++ b/components/eam/src/physics/rrtmgp/radiation.F90 @@ -767,6 +767,18 @@ subroutine radiation_init(state,pbuf) call addfld('FLNTC'//diag(icall), horiz_only, 'A', 'W/m2', & 'Clearsky net longwave flux at top of model', & sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLUTOA'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Upwelling longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLNTOA'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Net longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLUTOAC'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Clearsky upwelling longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) + call addfld('FLNTOAC'//diag(icall), horiz_only, 'A', 'W/m2', & + 'Clearsky net longwave flux at top of atmosphere', & + sampling_seq='rad_lwsw', flag_xyfill=.true.) call addfld('LWCF'//diag(icall), horiz_only, 'A', 'W/m2', & 'Longwave cloud forcing', & sampling_seq='rad_lwsw', flag_xyfill=.true.) @@ -2375,6 +2387,7 @@ subroutine output_fluxes_lw(icall, state, flux_all, flux_clr, qrl, qrlc) ! Working arrays real(r8), dimension(pcols,pver+1) :: flux_up, flux_dn, flux_net integer :: ncol + integer :: ktop_rad = 1 ncol = state%ncol @@ -2412,6 +2425,12 @@ subroutine output_fluxes_lw(icall, state, flux_all, flux_clr, qrl, qrlc) call outfld('FLUTC'//diag(icall), flux_clr%flux_up(1:ncol,ktop), ncol, state%lchnk) call outfld('FLDSC'//diag(icall), flux_clr%flux_dn(1:ncol,kbot+1), ncol, state%lchnk) + ! TOA fluxes (above model top, use index to rad top) + call outfld('FLUTOA'//diag(icall), flux_all%flux_up(1:ncol,ktop_rad), ncol, state%lchnk) + call outfld('FLNTOA'//diag(icall), flux_all%flux_net(1:ncol,ktop_rad), ncol, state%lchnk) + call outfld('FLUTOAC'//diag(icall), flux_clr%flux_up(1:ncol,ktop_rad), ncol, state%lchnk) + call outfld('FLNTOAC'//diag(icall), flux_clr%flux_net(1:ncol,ktop_rad), ncol, state%lchnk) + ! Calculate and output the cloud radiative effect (LWCF in history) cloud_radiative_effect(1:ncol) = flux_all%flux_net(1:ncol,ktop) - flux_clr%flux_net(1:ncol,ktop) call outfld('LWCF'//diag(icall), cloud_radiative_effect, ncol, state%lchnk) diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index a21a15cf6ef..5b50f703109 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -293,29 +293,31 @@ be lost if SCREAM_HACK_XML is not enabled. ${DIN_LOC_ROOT}/atm/scream/mam4xx/photolysis/RSF_GT200nm_v3.0_c080811.nc ${DIN_LOC_ROOT}/atm/scream/mam4xx/photolysis/temp_prs_GT200nm_JPL10_c130206.nc - - 20100101 + + 20100101 + + ${DIN_LOC_ROOT}/atm/scream/mam4xx/drydep/season_wes.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_so2_elev_1x1_2010_clim_ne30pg2_c20241008.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_so4_a1_elev_1x1_2010_clim_ne30pg2_c20241008.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_so4_a2_elev_1x1_2010_clim_ne30pg2_c20241008.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_pom_a4_elev_1x1_2010_clim_ne30pg2_c20241008.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_bc_a4_elev_1x1_2010_clim_ne30pg2_c20241008.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_num_a1_elev_1x1_2010_clim_ne30pg2_c20241008.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_num_a2_elev_1x1_2010_clim_ne30pg2_c20241008.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_num_a4_elev_1x1_2010_clim_ne30pg2_c20241008.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_soag_elev_1x1_2010_clim_ne30pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_so2_elev_1x1_2010_clim_ne30pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_so4_a1_elev_1x1_2010_clim_ne30pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_so4_a2_elev_1x1_2010_clim_ne30pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_pom_a4_elev_1x1_2010_clim_ne30pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_bc_a4_elev_1x1_2010_clim_ne30pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_num_a1_elev_1x1_2010_clim_ne30pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_num_a2_elev_1x1_2010_clim_ne30pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_num_a4_elev_1x1_2010_clim_ne30pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/elevated/cmip6_mam4_soag_elev_1x1_2010_clim_ne30pg2_c20241008.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so2_elev_1x1_2010_clim_ne4pg2_c20241008.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so4_a1_elev_1x1_2010_clim_ne4pg2_c20241008.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so4_a2_elev_1x1_2010_clim_ne4pg2_c20241008.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_pom_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_bc_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a1_elev_1x1_2010_clim_ne4pg2_c20241008.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a2_elev_1x1_2010_clim_ne4pg2_c20241008.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_soag_elev_1x1_2010_clim_ne4pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so2_elev_1x1_2010_clim_ne4pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so4_a1_elev_1x1_2010_clim_ne4pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so4_a2_elev_1x1_2010_clim_ne4pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_pom_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_bc_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a1_elev_1x1_2010_clim_ne4pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a2_elev_1x1_2010_clim_ne4pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_soag_elev_1x1_2010_clim_ne4pg2_c20241008.nc @@ -370,17 +372,11 @@ be lost if SCREAM_HACK_XML is not enabled. ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a1_surf_ne4pg2_2010_clim_c20240815.nc ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a2_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/dst_ne30pg2_c20241028.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/dst_ne4pg2_c20241028.nc - - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/DMSflux.2010.ne4pg2_conserv.POPmonthlyClimFromACES4BGC_c20240814.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so2_surf_ne4pg2_2010_clim_c20240815.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_bc_a4_surf_ne4pg2_2010_clim_c20240815.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a1_surf_ne4pg2_2010_clim_c20240815.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a2_surf_ne4pg2_2010_clim_c20240815.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_num_a4_surf_ne4pg2_2010_clim_c20240815.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_pom_a4_surf_ne4pg2_2010_clim_c20240815.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a1_surf_ne4pg2_2010_clim_c20240815.nc - ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/surface/cmip6_mam4_so4_a2_surf_ne4pg2_2010_clim_c20240815.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne30pg2/monthly_macromolecules_0.1deg_bilinear_year01_merge_ne30pg2_c20241030.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/monthly_macromolecules_0.1deg_bilinear_year01_merge_ne4pg2_c20241030.nc @@ -498,6 +494,11 @@ be lost if SCREAM_HACK_XML is not enabled. 0.0 -9999.0 0.0 + + + -9999.0 + 551.58 + 1 2 4 @@ -641,6 +642,7 @@ be lost if SCREAM_HACK_XML is not enabled. 1.37146e-07 ,3.45899e-08 ,1.00000e-06 ,9.99601e-08 1.37452e-07 ,3.46684e-08 ,1.00900e-06 ,9.99601e-08 5.08262e-12 ,1.54035e-13 ,3.09018e-13 ,9.14710e-22 + 0.0 0.0 0.0 0.0 diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/shell_commands index 435556401a2..2520f7b6b12 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/shell_commands @@ -3,6 +3,7 @@ cime_root=$(./xmlquery --value CIMEROOT) input_data_dir=$(./xmlquery --value DIN_LOC_ROOT) atmchange=$cime_root/../components/eamxx/scripts/atmchange +case_name=$(./xmlquery --value CASE) # Change run length ./xmlchange RUN_STARTDATE="1994-10-01" @@ -61,7 +62,7 @@ else fi # set the output yaml files -output_yaml_files=$(find ${cime_root}/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/v1prod/yaml_outs/ -maxdepth 1 -type f) +output_yaml_files=$(find ${cime_root}/../components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/ -maxdepth 1 -type f) for file in ${output_yaml_files[@]}; do # if the word "coarse" is in the file name, do nothing if [[ "${file}" == *"_coarse.yaml" && "${hmapfile}" == "not-supported-yet" ]]; then @@ -82,6 +83,8 @@ for file in ${output_yaml_files[@]}; do sed -i "s|horiz_remap_file:.*_to_ne30.*|horiz_remap_file: ${hmapfile}|" ./$(basename ${file}) sed -i "s|horiz_remap_file:.*_to_DecadalSites.*|horiz_remap_file: ${armmapfile}|" ./$(basename ${file}) fi + # replace all filename prefixes so that st_archive works... + sed -i "s|eamxx_output.decadal|${case_name}.scream|" ./$(basename ${file}) done # TODO: diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyAVG_native.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyAVG_native.yaml similarity index 81% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyAVG_native.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyAVG_native.yaml index 4e1239b5c33..181f841eeb8 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyAVG_native.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyAVG_native.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.1dailyAVG_native.h +filename_prefix: eamxx_output.decadal.1dailyAVG_native.h iotype: pnetcdf Averaging Type: Average Max Snapshots Per File: 1 diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyMAX_native.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyMAX_native.yaml similarity index 82% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyMAX_native.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyMAX_native.yaml index a7d10efe070..a8974c75b35 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyMAX_native.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyMAX_native.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.1dailyMAX_native.h +filename_prefix: eamxx_output.decadal.1dailyMAX_native.h iotype: pnetcdf Averaging Type: Max Max Snapshots Per File: 1 diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyMIN_native.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyMIN_native.yaml similarity index 80% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyMIN_native.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyMIN_native.yaml index 653e194d278..8d48a1bedf6 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1dailyMIN_native.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1dailyMIN_native.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.1dailyMIN_native.h +filename_prefix: eamxx_output.decadal.1dailyMIN_native.h iotype: pnetcdf Averaging Type: Min Max Snapshots Per File: 1 diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1hourlyINST_arm.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1hourlyINST_arm.yaml similarity index 92% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1hourlyINST_arm.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1hourlyINST_arm.yaml index 5bb07048aed..52fc391ca4d 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1hourlyINST_arm.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1hourlyINST_arm.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.1hourlyINST_arm.h +filename_prefix: eamxx_output.decadal.1hourlyINST_arm.h iotype: pnetcdf Averaging Type: Instant Max Snapshots Per File: 24 # one file per day diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1hourlyINST_native.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1hourlyINST_native.yaml similarity index 85% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1hourlyINST_native.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1hourlyINST_native.yaml index 7a221e89f1c..0aba4827ead 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.1hourlyINST_native.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.1hourlyINST_native.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.1hourlyINST_native.h +filename_prefix: eamxx_output.decadal.1hourlyINST_native.h iotype: pnetcdf Averaging Type: Instant Max Snapshots Per File: 24 diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.3hourlyAVG_coarse.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.3hourlyAVG_coarse.yaml similarity index 96% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.3hourlyAVG_coarse.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.3hourlyAVG_coarse.yaml index 665294c6227..d429b11ebd1 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.3hourlyAVG_coarse.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.3hourlyAVG_coarse.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.3hourlyAVG_coarse.h +filename_prefix: eamxx_output.decadal.3hourlyAVG_coarse.h iotype: pnetcdf Averaging Type: Average Max Snapshots Per File: 8 # one file per day diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.3hourlyINST_coarse.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.3hourlyINST_coarse.yaml similarity index 96% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.3hourlyINST_coarse.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.3hourlyINST_coarse.yaml index 42c64954508..a2faa1b971c 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.3hourlyINST_coarse.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.3hourlyINST_coarse.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.3hourlyINST_coarse.h +filename_prefix: eamxx_output.decadal.3hourlyINST_coarse.h iotype: pnetcdf Averaging Type: Instant Max Snapshots Per File: 8 # one file per day diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyAVG_coarse.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyAVG_coarse.yaml similarity index 94% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyAVG_coarse.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyAVG_coarse.yaml index 5e4aaed0738..437142ba559 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyAVG_coarse.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyAVG_coarse.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.6hourlyAVG_coarse.h +filename_prefix: eamxx_output.decadal.6hourlyAVG_coarse.h iotype: pnetcdf Averaging Type: Average Max Snapshots Per File: 4 # one file per day diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyINST_coarse.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyINST_coarse.yaml similarity index 91% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyINST_coarse.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyINST_coarse.yaml index e9e0f34d5e0..bb83718a8eb 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyINST_coarse.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyINST_coarse.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.6hourlyINST_coarse.h +filename_prefix: eamxx_output.decadal.6hourlyINST_coarse.h iotype: pnetcdf Averaging Type: Instant Max Snapshots Per File: 4 # one file per day diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyINST_native.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyINST_native.yaml similarity index 90% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyINST_native.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyINST_native.yaml index bb7fd275abf..c69dc4b2212 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.6hourlyINST_native.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.6hourlyINST_native.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.6hourlyINST_native.h +filename_prefix: eamxx_output.decadal.6hourlyINST_native.h iotype: pnetcdf Averaging Type: Instant Max Snapshots Per File: 4 # one file per day diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.dailyAVG_coarse.yaml b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.dailyAVG_coarse.yaml similarity index 97% rename from components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.dailyAVG_coarse.yaml rename to components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.dailyAVG_coarse.yaml index 7c1990a7b56..2d1e6e7221e 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/scream_output.decadal.dailyAVG_coarse.yaml +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/eamxx/prod/yaml_outs/eamxx_output.decadal.dailyAVG_coarse.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: scream_output.decadal.dailyAVG_coarse.h +filename_prefix: eamxx_output.decadal.dailyAVG_coarse.h iotype: pnetcdf Averaging Type: Average Max Snapshots Per File: 1 diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aero_microphysics/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aero_microphysics/shell_commands index 1d6757a5bd9..ac1709f7dca 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aero_microphysics/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aero_microphysics/shell_commands @@ -8,9 +8,8 @@ $CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh -b #------------------------------------------------------ -#Update IC file and add drydep process +# Add microphysics process #------------------------------------------------------ -$CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b $CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,rrtmgp,mam4_aero_microphys" -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/remap_emiss_ne4_ne30/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/remap_emiss_ne4_ne30/shell_commands new file mode 100644 index 00000000000..b2d0286b870 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/remap_emiss_ne4_ne30/shell_commands @@ -0,0 +1,28 @@ + +#!/bin/sh +#------------------------------------------------------ +# MAM4xx adds additionaltracers to the simulation +# Increase number of tracers for MAM4xx simulations +#------------------------------------------------------ + +$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/update_eamxx_num_tracers.sh -b + +#------------------------------------------------------ +# Add aerosol microphysics process, force ne4pg2 +# emission files and provide a ne4pg2->ne30pg2 mapping +# file +#------------------------------------------------------ +alias ATMCHANGE='$CIMEROOT/../components/eamxx/scripts/atmchange' + +ATMCHANGE physics::atm_procs_list="mac_aero_mic,rrtmgp,mam4_aero_microphys" -b + +ATMCHANGE mam4_aero_microphys::mam4_so2_elevated_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so2_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::mam4_so4_a1_elevated_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so4_a1_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::mam4_so4_a2_elevated_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_so4_a2_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::mam4_pom_a4_elevated_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_pom_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::mam4_bc_a4_elevated_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_bc_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::mam4_num_a1_elevated_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a1_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::mam4_num_a2_elevated_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a2_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::mam4_num_a4_elevated_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_num_a4_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::mam4_soag_elevated_emiss_file_name='${DIN_LOC_ROOT}/atm/scream/mam4xx/emissions/ne4pg2/elevated/cmip6_mam4_soag_elev_1x1_2010_clim_ne4pg2_c20241008.nc' -b +ATMCHANGE mam4_aero_microphys::aero_microphys_remap_file='${DIN_LOC_ROOT}/atm/scream/maps/map_ne4pg2_to_ne30pg2_nco_c20241108.nc' -b diff --git a/components/eamxx/cmake/BuildCprnc.cmake b/components/eamxx/cmake/BuildCprnc.cmake index 2f4f1f00a36..287956c5a9d 100644 --- a/components/eamxx/cmake/BuildCprnc.cmake +++ b/components/eamxx/cmake/BuildCprnc.cmake @@ -8,32 +8,44 @@ include (EkatUtils) macro(BuildCprnc) - # Make sure this is built only once - if (NOT TARGET cprnc) - if (SCREAM_CIME_BUILD) - string (CONCAT MSG - "WARNING! By default, scream should not build tests in a CIME build,\n" - "and cprnc should only be built by scream in case tests are enabled.\n" - "If you explicitly requested tests to be on in a CIME build,\n" - "then you can discard this warning. Otherwise, please, contact developers.\n") - message("${MSG}") - endif() - set(BLDROOT ${PROJECT_BINARY_DIR}/externals/cprnc) - file(WRITE ${BLDROOT}/Macros.cmake - " - set(SCC ${CMAKE_C_COMPILER}) - set(SFC ${CMAKE_Fortran_COMPILER}) - set(FFLAGS \"${CMAKE_Fortran_FLAGS}\") - set(NETCDF_PATH ${NetCDF_Fortran_PATH}) - " - ) - set(SRC_ROOT ${SCREAM_BASE_DIR}/../..) - add_subdirectory(${SRC_ROOT}/cime/CIME/non_py/cprnc ${BLDROOT}) - EkatDisableAllWarning(cprnc) - - set(CPRNC_BINARY ${BLDROOT}/cprnc CACHE INTERNAL "") - + # TODO: handle this more carefully and more gracefully in the future + # TODO: For now, it is just a hack to get going... + # find cprnc defined in machine entries + set(CCSM_CPRNC $ENV{CCSM_CPRNC}) + if(EXISTS "${CCSM_CPRNC}") + message(STATUS "Path ${CCSM_CPRNC} exists, so we will use it") + set(CPRNC_BINARY ${CCSM_CPRNC} CACHE INTERNAL "") configure_file (${SCREAM_BASE_DIR}/cmake/CprncTest.cmake.in ${CMAKE_BINARY_DIR}/bin/CprncTest.cmake @ONLY) + else() + message(WARNING "Path ${CCSM_CPRNC} does not exist, so we will try to build it") + # Make sure this is built only once + if (NOT TARGET cprnc) + if (SCREAM_CIME_BUILD) + string (CONCAT MSG + "WARNING! By default, scream should not build tests in a CIME build,\n" + "and cprnc should only be built by scream in case tests are enabled.\n" + "If you explicitly requested tests to be on in a CIME build,\n" + "then you can discard this warning. Otherwise, please, contact developers.\n") + message("${MSG}") + endif() + set(BLDROOT ${PROJECT_BINARY_DIR}/externals/cprnc) + file(WRITE ${BLDROOT}/Macros.cmake + " + set(SCC ${CMAKE_C_COMPILER}) + set(SFC ${CMAKE_Fortran_COMPILER}) + set(FFLAGS \"${CMAKE_Fortran_FLAGS}\") + set(NETCDF_PATH ${NetCDF_Fortran_PATH}) + " + ) + set(SRC_ROOT ${SCREAM_BASE_DIR}/../..) + add_subdirectory(${SRC_ROOT}/cime/CIME/non_py/cprnc ${BLDROOT}) + EkatDisableAllWarning(cprnc) + + set(CPRNC_BINARY ${BLDROOT}/cprnc CACHE INTERNAL "") + + configure_file (${SCREAM_BASE_DIR}/cmake/CprncTest.cmake.in + ${CMAKE_BINARY_DIR}/bin/CprncTest.cmake @ONLY) + endif() endif() endmacro() diff --git a/components/eamxx/cmake/machine-files/gcp.cmake b/components/eamxx/cmake/machine-files/gcp12.cmake similarity index 100% rename from components/eamxx/cmake/machine-files/gcp.cmake rename to components/eamxx/cmake/machine-files/gcp12.cmake diff --git a/components/eamxx/cmake/machine-files/ghci-oci.cmake b/components/eamxx/cmake/machine-files/ghci-oci.cmake index 86a2fb1d530..85eabbaa848 100644 --- a/components/eamxx/cmake/machine-files/ghci-oci.cmake +++ b/components/eamxx/cmake/machine-files/ghci-oci.cmake @@ -1,2 +1,15 @@ include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) common_setup() + +set(CMAKE_Fortran_FLAGS "-Wno-maybe-uninitialized -Wno-unused-dummy-argument -fallow-argument-mismatch" CACHE STRING "" FORCE) +set(CMAKE_CXX_FLAGS "-fvisibility-inlines-hidden -fmessage-length=0 -Wno-use-after-free -Wno-unused-variable -Wno-maybe-uninitialized" CACHE STRING "" FORCE) + +# TODO: figure out a better way to handle this, e.g., +# TODO: --map-by ppr:1:node:pe=1 doesn't work with mpich, +# TODO: but -map-by core:1:numa:hwthread=1 may work well? +# TODO: this will need to be handled in EKAT at some point +set(EKAT_MPI_NP_FLAG "-np" CACHE STRING "-np") + +# TODO: hack in place to get eamxx to recognize CPRNC +# TODO: See note in BuildCprnc.cmake... +set(ENV{CCSM_CPRNC} "/usr/local/packages/bin/cprnc") diff --git a/components/eamxx/docs/developer/ci_nightly.md b/components/eamxx/docs/developer/ci_nightly.md index b222139dd55..0716ce4c9f3 100644 --- a/components/eamxx/docs/developer/ci_nightly.md +++ b/components/eamxx/docs/developer/ci_nightly.md @@ -1,17 +1,17 @@ # Continuous Integration and Nightly Testing -## Autotester ## +## Autotester EAMxx using github actions and a Sandia product called Autotester 2 to run CI testing on a CPU and GPU machine for every github pull request. By default, we run the e3sm_scream_v1_at suite and the standalone eamxx tests (test-all-scream). -## Nightly overview, CDash ## +## Nightly overview, CDash Our nightly testing is much more extensive than the CI testing. You can see our dashboard here under the section "E3SM_SCREAM": -https://my.cdash.org/index.php?project=E3SM + We run a variety of CIME test suites and standalone testing on a number of platforms. We even do some performance testing on frontier. diff --git a/components/eamxx/docs/developer/cime_testing.md b/components/eamxx/docs/developer/cime_testing.md index 71233a245b4..667488960f6 100644 --- a/components/eamxx/docs/developer/cime_testing.md +++ b/components/eamxx/docs/developer/cime_testing.md @@ -4,33 +4,37 @@ Full model system testing of eamxx is done through CIME test cases (much like the rest of E3SM). We offer a number of test suites, including: + * e3sm_scream_v0: Test the full set of V0 (pre-C++) tests * e3sm_scream_v1: Test the full set of V1 (C++) tests * e3sm_scream_v1_at: A smaller and quicker set of tests for autotesting * e3sm_scream_hires: A small number of bigger, longer-running tests to measure performance Example for running a suite: -``` -% cd $repo/cime/scripts -% ./create_test e3sm_scream_v1_at --wait + +```shell +cd $repo/cime/scripts +./create_test e3sm_scream_v1_at --wait ``` Example for running a single test case: -``` -% cd $repo/cime/scripts -% ./create_test SMS.ne4_ne4.F2010-SCREAMv1 --wait + +```shell +cd $repo/cime/scripts +./create_test SMS.ne4_ne4.F2010-SCREAMv1 --wait ``` There are many behavioral tweaks you can make to a test case, like changing the run length, test type, etc. Most of this is not specific to eamxx and works for any CIME case. This generic stuff is well-documentated here: -http://esmci.github.io/cime/versions/master/html/users_guide/testing.html + When it comes to things specific to eamxx, you have grids, compsets, and testmods. Common EAMxx grids are: + * ne4_ne4 (low resolution) * ne4pg2_ne4pg2 (low resolution with phys grid) * ne30_ne30 (med resolution) @@ -38,9 +42,10 @@ Common EAMxx grids are: * ne1024pg2_ne1024pg2 (ultra high with phys grid) More grid info can be found here: -https://acme-climate.atlassian.net/wiki/spaces/DOC/pages/933986549/ATM+Grid+Resolution+Summary + Common EAMxx compsets are: + * F2010-SCREAM-LR: V0 low res compset with eamxx V0 atmosphere * F2010-SCREAMv1: V1 standard compset with eamxx V1 atmosphere * FIOP-SCREAMv1-DP: V1 with dpxx (doubly-periodic lateral boundary condition in C++) @@ -50,10 +55,14 @@ Full info on supported compsets can be found by looking at this file: `$scream_repo/components/eamxx/cime_config/config_compsets.xml` Common EAMxx testmods are: -* small_kernels: Enable smaller-granularity kernels, can improve performance on some systems -* scream-output-preset-[1-6]: Our 6 output presets. These turn some combination of our three output streams (phys_dyn, phys, and diags), + +* small_kernels: Enable smaller-granularity kernels, + can improve performance on some systems +* scream-output-preset-[1-6]: Our 6 output presets. + These turn some combination of our three output streams + (phys_dyn, phys, and diags), various remaps, etc. -* bfbhash: Turns on bit-for-bit hash output: https://acme-climate.atlassian.net/wiki/spaces/NGDNA/pages/3831923056/EAMxx+BFB+hashing +* bfbhash: Turns on bit-for-bit hash output: More info on running EAMxx can be found here: -https://acme-climate.atlassian.net/wiki/spaces/DOC/pages/3386015745/How+To+Run+EAMxx+SCREAMv1 + diff --git a/components/eamxx/docs/developer/field.md b/components/eamxx/docs/developer/field.md index 4170b28ac2b..8df83440a2f 100644 --- a/components/eamxx/docs/developer/field.md +++ b/components/eamxx/docs/developer/field.md @@ -1,45 +1,58 @@ -## Field +# Field -In EAMxx, a `Field` is a data structure holding two things: pointers to the data and pointers to metadata. -Both the data and metadata are stored in `std::shared_ptr` instances, to ensure consistency across all copies -of the field. This allows for fast shallow copy semantic for this class. +In EAMxx, a `Field` is a data structure holding two things: pointers to the +data and pointers to metadata. Both the data and metadata are stored in +`std::shared_ptr` instances, to ensure consistency across all copies of +the field. This allows for fast shallow copy semantic for this class. -The data is stored on both CPU and device memory (these may be the same, depending on the Kokkos -backend). In EAMxx, we always assume and guarantee that the device data is up to date. That implies that the data -must be explicitly synced to host before using it on host, and explicitly synced to device after host manipulation, -in order to ensure correctness. In order to access the data, users must use the `get_view`/'get_strided_view' methods, -which takes two template arguments: the data type, and an enum specifying whether CPU or device data is needed. -The data type is used to reinterpret the generic pointer stored inside to a view of the correct scalar type and layout. -It is a possibly const-qualified type, and if the field was marked as "read-only", the method ensures that the -provided data type is const. A read-only field can be created via the `getConst` method, which returns a shallow -copy of the field, but marked as read-only. The enum specifying host or device data is optional, with device being the default. +The data is stored on both CPU and device memory (these may be the same, +depending on the Kokkos backend). In EAMxx, we always assume and guarantee +that the device data is up to date. That implies that the data must be +explicitly synced to host before using it on host, and explicitly synced +to device after host manipulation, in order to ensure correctness. +In order to access the data, users must use the `get_view`/ +`get_strided_view` methods, which takes two template arguments: +the data type, and an enum specifying whether CPU or device data is needed. +The data type is used to reinterpret the generic pointer stored inside +to a view of the correct scalar type and layout. It is a possibly +const-qualified type, and if the field was marked as "read-only", +the method ensures that the provided data type is const. A read-only field +can be created via the `getConst` method, which returns a shallow copy of +the field, but marked as read-only. The enum specifying host or device data +is optional, with device being the default. -The metadata is a collection of information on the field, such as name, layout, units, allocation size, and more. -Part of the metadata is immutable after creation (e.g., name, units, or layout), while some metadata can be -partially or completely modified. The metadata is contained in the `FieldHeader` data structure, which contains -four parts: +The metadata is a collection of information on the field, such as name, layout, units, +allocation size, and more. Part of the metadata is immutable after creation (e.g., +name, units, or layout), while some metadata can be partially or completely modified. +The metadata is contained in the `FieldHeader` data structure, which contains four +parts: -* `FieldIdentifier`: stores the field's name, layout, units, data type, and name of the grid where it's defined. - These information are condensed in a single string, that can be used to uniquely identify a field, - allowing to distinguish between different version of the same field. The layout is stored in the `FieldLayout` - data structure, which includes: - * the field tags: stored as a `std::vector`, they give context to the field's extents. - * the field dims: stored both as a `std::vector`, as well as a 1d `Kokkos::View`. -* `FieldTracking`: stores information on the usage of the field, as well as its possible connections to other - fields. In particular, the tracked items are: - * the field time stamp: the time stamp when the field was last updated. - * the field accumulation start time: used for fields that are accumulated over several time steps - (or time step subcycles). For instance, it allows to reconstruct fluxes from raw accumulations. - * the providers/customers: lists of atmosphere processes (see below) that respectively require/compute - the field in their calculations. - * the field groups: a list of field groups that this field belongs too. Field groups are used to access - a group of fields without explicit prior knowledge about the number and/or names of the fields. -* `FieldAllocProp`: stores information about the allocation. While the field is not yet allocated, users can - request special allocations for the field, for instance to accommodate packing (for SIMD), which may - require padding. Upon allocation, this information is then used by the Field structure to extract the - actual data, wrapped in a properly shaped `Kokkos::View`. The alloc props are also responsible of tracking - additional information in case the field is a "slice" of a higher-dimensional one, a fact that can affect - how the data is accessed. -* Extra data: stored as a `std::map`, allows to catch any metadata that does not fit - in the above structures. This is a last resort structure, intended to accommodate the most peculiar - corner cases, and should be used sparingly. +* `FieldIdentifier`: stores the field's name, layout, units, data type, + and name of the grid where it's defined. These information are condensed + in a single string, that can be used to uniquely identify a field, allowing + to distinguish between different version of the same field. + The layout is stored in the `FieldLayout` data structure, which includes: + * the field tags: stored as a `std::vector`, they give context to the + field's extents. + * the field dims: stored both as a `std::vector`, as well as a 1d `Kokkos::View`. +* `FieldTracking`: stores information on the usage of the field, as well as its + possible connections to other fields. In particular, the tracked items are: + * the field time stamp: the time stamp when the field was last updated. + * the field accumulation start time: used for fields that are accumulated over + several time steps (or time step subcycles). For instance, it allows to + reconstruct fluxes from raw accumulations. + * the providers/customers: lists of atmosphere processes (see below) that + respectively require/compute the field in their calculations. + * the field groups: a list of field groups that this field belongs too. Field groups + are used to access a group of fields without explicit prior knowledge about the + number and/or names of the fields. +* `FieldAllocProp`: stores information about the allocation. While the field is not + yet allocated, users can request special allocations for the field, for instance + to accommodate packing (for SIMD), which may require padding. Upon allocation, + this information is then used by the Field structure to extract the actual data, + wrapped in a properly shaped `Kokkos::View`. The alloc props are also + responsible of tracking additional information in case the field is a "slice" of + a higher-dimensional one, a fact that can affect how the data is accessed. +* Extra data: stored as a `std::map`, allows to catch any + metadata that does not fit in the above structures. This is a last resort structure, + intended to accommodate the most peculiar corner cases, and should be used sparingly. diff --git a/components/eamxx/docs/developer/grid.md b/components/eamxx/docs/developer/grid.md index 8a61b97e079..b4e1a1c8c03 100644 --- a/components/eamxx/docs/developer/grid.md +++ b/components/eamxx/docs/developer/grid.md @@ -1,22 +1,29 @@ -## Grids and Remappers +# Grids and Remappers -In EAMxx, the `AbstractGrid` is an interface used to access information regarding the horizontal and vertical -discretization. The most important information that the grid stores is: +In EAMxx, the `AbstractGrid` is an interface used to access information regarding +the horizontal and vertical discretization. The most important information that +the grid stores is: -* the number of local/global DOFs: these are the degrees of freedom of the horizontal grid only. Here, - local/global refers to the MPI partitioning. -* the DOFs global IDs (GIDs): a list of GIDs of the DOFs on the current MPI rank, stored as a Field -* the local IDs (LIDs) to index list: this list maps the LID of a DOF (that is, the position of the DOF - in the GID list) to a "native" indexing system for that DOF. For instance, a `PointGrid` (a class derived from - `AbstractGrid`) is a simple collection of points, so the "native" indexing system coincides with the LIDs. - However, for a `SEGrid` (a derived class, for spectral element grids), the "native" indexing is a triplet - `(ielem,igp,jgp)`, specifying the element index, and the two indices of the Gauss point within the element. -* geometry data: stored as a `std::map`, this represent any data that is intrinsically - linked to the grid (either along the horizontal or vertical direction), such as lat/lon coordinates, - vertical coordinates, area associated with the DOF. +* the number of local/global DOFs: these are the degrees of freedom of the + horizontal grid only. Here, local/global refers to the MPI partitioning. +* the DOFs global IDs (GIDs): a list of GIDs of the DOFs on the current MPI rank, + stored as a Field +* the local IDs (LIDs) to index list: this list maps the LID of a DOF (that is, + the position of the DOF in the GID list) to a "native" indexing system for that + DOF. For instance, a `PointGrid` (a class derived from `AbstractGrid`) is a + simple collection of points, so the "native" indexing system coincides with the + LIDs. However, for a `SEGrid` (a derived class, for spectral element grids), + the "native" indexing is a triplet `(ielem,igp,jgp)`, specifying the element + index, and the two indices of the Gauss point within the element. +* geometry data: stored as a `std::map`, this represent any + data that is intrinsically linked to the grid (either along the horizontal or + vertical direction), such as lat/lon coordinates, vertical coordinates, area + associated with the DOF. -Grids can also be used to retrieve the layout of a 2d/3d scalar/vector field, which allows certain downstream -classes to perform certain operations without assuming anything on the horizontal grid. +Grids can also be used to retrieve the layout of a 2d/3d scalar/vector field, +which allows certain downstream classes to perform certain operations without +assuming anything on the horizontal grid. -In general, grid objects are passed around the different parts of EAMxx as const objects (read-only). -The internal data can only be modified during construction, which usually is handled by a `GridsManager` object. +In general, grid objects are passed around the different parts of EAMxx as const +objects (read-only). The internal data can only be modified during construction, +which usually is handled by a `GridsManager` object. diff --git a/components/eamxx/docs/developer/index.md b/components/eamxx/docs/developer/index.md index 2d47bab65fe..69673b12ebd 100644 --- a/components/eamxx/docs/developer/index.md +++ b/components/eamxx/docs/developer/index.md @@ -1,3 +1 @@ # SCREAM Developer Guide - - diff --git a/components/eamxx/docs/developer/io.md b/components/eamxx/docs/developer/io.md index caf237010a3..0a4c7b2d832 100644 --- a/components/eamxx/docs/developer/io.md +++ b/components/eamxx/docs/developer/io.md @@ -1,5 +1,5 @@ # Input-Output -In EAMxx, I/O is handled through the SCORPIO library, currently a submodule of E3SM. -The `scream_io` library within eamxx allows to interface the EAMxx infrastructure classes -with the SCORPIO library. +In EAMxx, I/O is handled through the SCORPIO library, currently a submodule of +E3SM. The `scream_io` library within eamxx allows to interface the EAMxx +infrastructure classes with the SCORPIO library. diff --git a/components/eamxx/docs/developer/kokkos_ekat.md b/components/eamxx/docs/developer/kokkos_ekat.md index 45827a11f83..a736384b2c6 100644 --- a/components/eamxx/docs/developer/kokkos_ekat.md +++ b/components/eamxx/docs/developer/kokkos_ekat.md @@ -2,99 +2,164 @@ ## Kokkos -EAMxx uses Kokkos for performance portable abstractions for parallel execution of code and data management to various HPC platforms, including OpenMP, Cuda, HIP, and SYCL. Here we give a brief overview of the important concepts for understanding Kokkos in EAMxx. For a more in depth description, see the [Kokkos wiki](https://kokkos.org/kokkos-core-wiki). +EAMxx uses Kokkos for performance portable abstractions for parallel execution +of code and data management to various HPC platforms, including OpenMP, Cuda, +HIP, and SYCL. Here we give a brief overview of the important concepts for +understanding Kokkos in EAMxx. For a more in depth description, see the +[Kokkos wiki](https://kokkos.org/kokkos-core-wiki). ### Kokkos::Device -`Kokkos::Device` is a struct which contain the type definitions for two main Kokkos concepts: execution space (`Kokkos::Device::execution_space`), the place on-node where parallel operations (like for-loops, reductions, etc.) are executed, and the memory space (`Kokkos::Device::memory_space`), the memory location on-node where data is stored. Given your machine architecture, Kokkos defines a default "device" space, given by -``` -Kokkos::Device -``` -where all performance critical code should be executed (e.g., on an NVIDIA machine, this device would be the GPU accelerators) and a default "host" space, given by +`Kokkos::Device` is a struct which contain the type definitions for two main +Kokkos concepts: execution space (`Kokkos::Device::execution_space`), the place +on-node where parallel operations (like for-loops, reductions, etc.) are +executed, and the memory space (`Kokkos::Device::memory_space`), the memory +location on-node where data is stored. Given your machine architecture, Kokkos +defines a default "device" space, given by + +```cpp +Kokkos::Device ``` -Kokkos::Device + +where all performance critical code should be executed (e.g., on an NVIDIA +machine, this device would be the GPU accelerators) and a default "host" space, +given by + +```c++ +Kokkos::Device ``` -where data can be accessed by the CPU cores and is necessary for I/O interfacing, for example. Currently, these default spaces are the ones used by EAMxx. On CPU-only machines, host and device represent the same space. + +where data can be accessed by the CPU cores and is necessary for I/O +interfacing, for example. Currently, these default spaces are the ones used by +EAMxx. On CPU-only machines, host and device represent the same space. ### Kokkos Views -The main data struct provided by Kokkos used in EAMxx in the `Kokkos::View`. This is a multi-dimensional data array that can live on either device or host memory space. These Views are necessary when running on GPU architectures as data structures like `std::vector` and `std::array` will be unavailable on device. +The main data struct provided by Kokkos used in EAMxx in the `Kokkos::View`. +This is a multi-dimensional data array that can live on either device or host +memory space. These Views are necessary when running on GPU architectures as +data structures like `std::vector` and `std::array` will be unavailable on +device. -Views are constructed in EAMxx most commonly with the following template and input arguments -``` -Kokkos::View(const std::string& label, int dim0, int dim1, ...) +Views are constructed in EAMxx most commonly with the following template and +input arguments + +```cpp +Kokkos::View(const std::string& label, + int dim0, int dim1, ...) ``` -where - - `DataType`: scalar type of the view, given as `ScalarType`+`*`(x's number of run-time dimensions). E.g., a 2D view of doubles will have `DataType = double**`. There is also an ability to define compile-time dimensions by using `[]`, see [Kokkos wiki section on views](https://kokkos.org/kokkos-core-wiki/API/core/view/view.html). - - `LayoutType`: mapping of indices into the underlying 1D memory storage. Types are: - - `LayoutRight` (used in EAMxx): strides increase from the right most to the left most dimension, right-most dimension is contiguous - - `LayoutLeft`: strides increase from the left most to the right most dimension, left-most dimension is contiguous - - `LayoutStride`: strides can be arbitrary for each dimension - - `DeviceType`: provides space where data live, defaults to the default device +where -The following example defines a view "temperature" which has dimensions columns and levels: -``` -Kokkos::View temperature("temperature", ncols, nlevs); +- `DataType`: scalar type of the view, given as `ScalarType`+`*`(x's number of + run-time dimensions). E.g., a 2D view of doubles will have `DataType = + double**`. There is also an ability to define compile-time dimensions by + using `[]`, see + [Kokkos wiki section on views] + (). +- `LayoutType`: mapping of indices into the underlying 1D memory storage. Types + are: + - `LayoutRight` (used in EAMxx): strides increase from the right most to the + left most dimension, right-most dimension is contiguous + - `LayoutLeft`: strides increase from the left most to the right most + dimension, left-most dimension is contiguous + - `LayoutStride`: strides can be arbitrary for each dimension +- `DeviceType`: provides space where data live, defaults to the default device + +The following example defines a view "temperature" which has dimensions columns +and levels: + +```cpp +Kokkos::View temperature( + "temperature", ncols, nlevs); ``` ### Deep Copy -Kokkos provides `Kokkos::deep_copy(dst, src)` which copies data between views of the same dimensions, or a scalar values into a view. Common uses -``` +Kokkos provides `Kokkos::deep_copy(dst, src)` which copies data between views +of the same dimensions, or a scalar values into a view. Common uses + +```cpp Kokkos::deep_copy(view0, view1); // Copy all data from view1 into view0 Kokkos::deep_copy(view0, 5); // Set all values of view0 to 5 ``` -As seen in the next section, we can use `deep_copy()` to copy data between host and device. + +As seen in the next section, we can use `deep_copy()` to copy data between host +and device. ### Mirror Views -We will often need to have memory allocation the resides on device (for computation), and then need that identical data on host (say, for output). Kokkos has a concept of mirror views, where data can be copied from host to device and vice versa. +We will often need to have memory allocation the resides on device (for +computation), and then need that identical data on host (say, for output). +Kokkos has a concept of mirror views, where data can be copied from host to +device and vice versa. Here is an example using the device view `temperature` from above -``` -// Allocate view on host that exactly mirrors the size of layout of the device view + +```cpp +// Allocate view on host that exactly mirrors the size of layout of the device +view auto host_temperature = Kokkos::create_mirror_view(temperature); // Copy all data from device to host Kokkos::deep_copy(host_temperature, temperature); ``` + Kokkos also offers an all-in-one option -``` + +```cpp // Note: must hand the host device instance as first argument -auto host_temperature = Kokkos::create_mirror_view_and_copy(Kokkos::DefaultHostDevice(), temperature); +auto host_temperature = Kokkos::create_mirror_view_and_copy( + Kokkos::DefaultHostDevice(), temperature); ``` ### Parallel Execution -The most basic parallel execution pattern used by EAMxx is the `Kokkos::parallel_for` which defines a for-loop with completely independent iterations. The `parallel_for` takes in an optional label for debugging, an execution policy, which defines a range and location (host or device) for the code to be run, and a lambda describing the body of code to be executed. The following are execution policies used in EAMxx - - - `int count`: 1D iteration range `[0, count)` - - `RangePolicy(int beg, int end)`: 1D iteration range for indices `[beg, end)` - - `MDRangePolicy>(int[N] beg, int[N] end)`: multi-dimensional iteration range `[beg, end)` - - `TeamPolicy(int league_size, int team_size, int vector_size)`: 1D iteration over `league_size`, assigned to thread teams of size `team_size`, each with `vector_size` vector lanes. Both `team_size` and `vector_size` can be given `Kokkos::AUTO` as input for Kokkos to automatically compute. +The most basic parallel execution pattern used by EAMxx is the +`Kokkos::parallel_for` which defines a for-loop with completely independent +iterations. The `parallel_for` takes in an optional label for debugging, an +execution policy, which defines a range and location (host or device) for the +code to be run, and a lambda describing the body of code to be executed. The +following are execution policies used in EAMxx + +- `int count`: 1D iteration range `[0, count)` +- `RangePolicy(int beg, int end)`: 1D iteration range for indices + `[beg, end)` +- `MDRangePolicy>(int[N] beg, int[N] end)`: multi- + dimensional iteration range `[beg, end)` +- `TeamPolicy(int league_size, int team_size, int vector_size)`: 1D + iteration over `league_size`, assigned to thread teams of size `team_size`, + each with `vector_size` vector lanes. Both `team_size` and `vector_size` can + be given `Kokkos::AUTO` as input for Kokkos to automatically compute. If no `ExecSpace` template is given, the default execution space is used. -For lambda capture, use `KOKKOS_LAMBDA` macro which sets capture automatically based on architecture. +For lambda capture, use `KOKKOS_LAMBDA` macro which sets capture automatically +based on architecture. Example using `RangePolicy` to initialize a view -``` -Kokkos::View temperature("temperature", ncols, nlevs); + +```cpp +Kokkos::View temperature("temperature", ncols, + nlevs); Kokkos::parallel_for("Init_temp", - Kokkos::RangePolicy(0, ncols*nlevs), - KOKKOS_LAMBDA (const int idx) { + Kokkos::RangePolicy(0, ncols*nlevs), + KOKKOS_LAMBDA (const int idx) { int icol = idx/nlevs; int ilev = idx%nlevs; temperature(icol, ilev) = 0; }); ``` + Same example with `TeamPolicy` -``` + +```cpp Kokkos::parallel_for("Init_temp", - Kokkos::TeamPolicy(ncols*nlevs, Kokkos::AUTO, Kokkos::AUTO), - KOKKOS_LAMBDA (const TeamPolicy::member_type& team) { + Kokkos::TeamPolicy(ncols*nlevs, Kokkos::AUTO, Kokkos::AUTO), + KOKKOS_LAMBDA (const TeamPolicy::member_type& team) { // league_rank() gives the index for this team int icol = team.league_rank()/nlevs; int ilev = team.league_rank()%nlevs; @@ -105,32 +170,39 @@ Kokkos::parallel_for("Init_temp", ### Hierarchical Parallelism -Using `TeamPolicy`, we can have up to three nested levels of parallelism: team parallelism, thread parallelism, vector parallelism. These nested policies can be called within the lambda body using the following execution policies +Using `TeamPolicy`, we can have up to three nested levels of parallelism: team +parallelism, thread parallelism, vector parallelism. These nested policies can +be called within the lambda body using the following execution policies - - `TeamThreadRange(team, begin, end)`: execute over threads of a team - - `TeamVectorRange(team, begin, end)`: execute over threads and vector lanes of a team - - `ThreadVectorRange(team, begin, end)`: execute over vector lanes of a thread +- `TeamThreadRange(team, begin, end)`: execute over threads of a team +- `TeamVectorRange(team, begin, end)`: execute over threads and vector lanes of + a team +- `ThreadVectorRange(team, begin, end)`: execute over vector lanes of a thread An example of using these policies -``` + +```cpp Kokkos::View Q("tracers", ncols, ntracers, nlevs); Kokkos::parallel_for(Kokkos::TeamPolicy(ncols, Kokkos::AUTO), - KOKKOS_LAMBDA (TeamPolicy::member_type& team) { + KOKKOS_LAMBDA (TeamPolicy::member_type& team) { int icol = team.league_rank(); Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlevs), [&](int ilev) { - temperature(icol, ilev) = 0; + temperature(icol, ilev) = 0; }); Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nlevs), [&](int ilev) { - Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, ntracers), [&](int iq) { - Q(icol, iq, ilev) = 0; - }); + Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, ntracers), [&](int iq) { + Q(icol, iq, ilev) = 0; + }); }); }); ``` -IMPORTANT! Nested policies cannot be used in arbitrary order. `ThreadVectorRange` must be used inside a `TeamThreadRange`, and `TeamVectorRange` must be the only level of nested parallelism. -``` +IMPORTANT! Nested policies cannot be used in arbitrary order. `ThreadVectorRange` +must be used inside a `TeamThreadRange`, and `TeamVectorRange` must be the only +level of nested parallelism. + +```cpp Kokkos::parallel_for(TeamPolicy(...), ... { // OK Kokkos::parallel_for(TeamThreadRange, ... { @@ -139,9 +211,9 @@ Kokkos::parallel_for(TeamPolicy(...), ... { // OK Kokkos::parallel_for(TeamThreadRange, ... { - Kokkos::parallel_for(ThreadVectorRange, ... { + Kokkos::parallel_for(ThreadVectorRange, ... { - }); + }); }); // OK @@ -156,13 +228,15 @@ Kokkos::parallel_for(TeamPolicy(...), ... { // WRONG, a TeamVectorRange must be the only nested level Kokkos::parallel_for(TeamVectorRange, ...{ - Kokkos::parallel_for(ThreadVectorRange, ... { + Kokkos::parallel_for(ThreadVectorRange, ... { - }); + }); }); }); ``` -Using these incorrectly can be very tricky to debug as the code almost certainly will _not_ error out, but race conditions will exist among threads. + +Using these incorrectly can be very tricky to debug as the code almost certainly +will _not_ error out, but race conditions will exist among threads. ## EKAT @@ -175,6 +249,3 @@ Using these incorrectly can be very tricky to debug as the code almost certainly ### Scratch Memory: WorspaceManager ### Algorithms - - - diff --git a/components/eamxx/docs/developer/managers.md b/components/eamxx/docs/developer/managers.md index 676449a2184..fa98c8b1d72 100644 --- a/components/eamxx/docs/developer/managers.md +++ b/components/eamxx/docs/developer/managers.md @@ -1 +1 @@ -## FieldManager and GridsManager +# FieldManager and GridsManager diff --git a/components/eamxx/docs/developer/processes.md b/components/eamxx/docs/developer/processes.md index 9ad556a3183..adb90e2dfbc 100644 --- a/components/eamxx/docs/developer/processes.md +++ b/components/eamxx/docs/developer/processes.md @@ -1,59 +1,77 @@ # Atmospheric Processes -In EAMxx, `AtmosphereProcess` (AP) is an abstract class representing a portion of the atmosphere timestep algorithm. -In simple terms, an AP is an object that given certain input fields performs some calculations to compute -some output fields. The concrete AP classes allow to create a buffer layer between particular packages (e.g., -dynamics dycore, physics parametrizations) and the atmosphere driver (AD), allowing separation of concerns, -so that the AD does not need to know details about the package, and the package does not need to know about -the EAMxx infrastructure. - -To enhance this separation of concerns, EAMxx implements two more classes for handling APs: - -- the concrete class `AtmosphereProcessGroup` (APG), which allows to group together a set of AP's, which can be seen from outside as a single process; -- the `AtmosphereProcessFactory` class, which allows an APG to create its internal processes without any knowledge of -what they are. - -This infrastructure allows the AD to view the whole atmosphere as a single APG, and to be completely agnostic to -what processes are run, and in which order. This design allows to have a code base that is cleaner, self-container, -and easy to test via a battery of targeted unit tests. - -In EAMxx, we already have a few concrete AP's, interfacing the AD to the Hommexx non-hydrostatic dycore as well as -some physics parametrizations (P3, SHOC, RRMTPG, etc). In the next section we describe the interfaces of an AP class, -and we show an example of how to write a new concrete AP class. +In EAMxx, `AtmosphereProcess` (AP) is an abstract class representing a portion +of the atmosphere timestep algorithm. In simple terms, an AP is an object that +given certain input fields performs some calculations to compute some output +fields. The concrete AP classes allow to create a buffer layer between +particular packages (e.g., dynamics dycore, physics parametrizations) and the +atmosphere driver (AD), allowing separation of concerns, so that the AD does +not need to know details about the package, and the package does not need to +know about the EAMxx infrastructure. + +To enhance this separation of concerns, EAMxx implements two more classes for +handling APs: + +- the concrete class `AtmosphereProcessGroup` (APG), which allows to group + together a set of AP's, which can be seen from outside as a single process; +- the `AtmosphereProcessFactory` class, which allows an APG to create its + internal processes without any knowledge of what they are. + +This infrastructure allows the AD to view the whole atmosphere as a single APG, +and to be completely agnostic to what processes are run, and in which order. +This design allows to have a code base that is cleaner, self-container, and +easy to test via a battery of targeted unit tests. + +In EAMxx, we already have a few concrete AP's, interfacing the AD to the +Hommexx non-hydrostatic dycore as well as some physics parametrizations (P3, +SHOC, RRMTPG, etc). In the next section we describe the interfaces of an AP +class, and we show an example of how to write a new concrete AP class. ## Atmosphere process interfaces An AP has several interfaces, which can be grouped into three categories: - - initialization: these interfaces are used to create the AP, as well as to initialize internal data structures; - - run: these interfaces are used to make the AP compute its output fields from its input fields; - - finalization: these interfaces are used to perform any clean up operation (e.g., release files) before the AP is - destroyed. - -Among the above, the initialization sequence is the most complex, and conists of several steps: - - - The AD creates the APG corresponding to the whole atmosphere. As mentioned above, this phase will make use of a factory, - which allows the AD to be agnostic to what is actually in the group. All AP's can start performing any initialization - work that they can, but at this point they are limited to use only an MPI communicator as well as a list of runtime - parameters (which were previously read from an input file). - - The AD passes a `GridsManager` to the AP's, so that they can get information about the grids they need. At this point, - all AP's have all the information they need to establish the layout of the input and output fields they need, - and can store a list of these "requests" - - After creating all fields (based on AP's requests), the AD passes a copy of each input and output field to - the AP's. These fields will be divided in "required" and "computed", which differ in that the former are only - passed to the AP's as 'read-only' fields (see the [field](field.md#Field) documentation for more details) - - The AP's are queried for how much scratch memory they may need at run time. After all AP's communicate their needs, - the AD will provide a pointer to scratch memory to the AP's. This is memory that can be used to initialize - temporary views/fields or other internal data structures. All AP's are given the same pointer, which means no - data persistence should be expected at run time between one timestep and the next. - - The AD calls the 'initialize' method on each AP. At this point, all fields are set, and AP's can complete any - remaining initialization task - -While the base AP class provides an (empty) implementation for some methods, in case derived classes do not need a -feature, some methods are purely virtual, and concrete classes will have to override them. Looking at existing -concrete AP implementations is a good way to have a first idea of what a new AP class needs to implement. Here, -we show go over the possible implementation of these methods in a hypothetical AP class. The header file may -look something like this +- initialization: these interfaces are used to create the AP, as well as to + initialize internal data structures; +- run: these interfaces are used to make the AP compute its output fields from + its input fields; +- finalization: these interfaces are used to perform any clean up operation + (e.g., release files) before the AP is destroyed. + +Among the above, the initialization sequence is the most complex, and consists +of several steps: + +- The AD creates the APG corresponding to the whole atmosphere. As mentioned + above, this phase will make use of a factory, which allows the AD to be + agnostic to what is actually in the group. All AP's can start performing any + initialization work that they can, but at this point they are limited to use + only an MPI communicator as well as a list of runtime parameters (which were + previously read from an input file). +- The AD passes a `GridsManager` to the AP's, so that they can get information + about the grids they need. At this point, all AP's have all the information + they need to establish the layout of the input and output fields they need, + and can store a list of these "requests" +- After creating all fields (based on AP's requests), the AD passes a copy of + each input and output field to the AP's. These fields will be divided in + "required" and "computed", which differ in that the former are only passed + to the AP's as 'read-only' fields (see the [field](field.md#Field) + documentation for more details) +- The AP's are queried for how much scratch memory they may need at run time. + After all AP's communicate their needs, the AD will provide a pointer to + scratch memory to the AP's. This is memory that can be used to initialize + temporary views/fields or other internal data structures. All AP's are given + the same pointer, which means no data persistence should be expected at run + time between one timestep and the next. +- The AD calls the 'initialize' method on each AP. At this point, all fields + are set, and AP's can complete any remaining initialization task + +While the base AP class provides an (empty) implementation for some methods, in +case derived classes do not need a feature, some methods are purely virtual, +and concrete classes will have to override them. Looking at existing concrete +AP implementations is a good way to have a first idea of what a new AP class +needs to implement. Here, we show go over the possible implementation of these +methods in a hypothetical AP class. The header file may look something like +this ```c++ #include @@ -86,21 +104,26 @@ protected: bool m_has_blah; }; ``` + A few comments: - - we added two views to the class, which are meant to be used to store intermediate results during calculations at -runtime; - - there are other methods that the class can override (such as additional operations when the AD sets a field in the - AP), but most AP's only need to override only these; - - we strongly encourage to add the keyword `override` when overriding a method; in case of small typos (e.g., missing - a `&` or a `const`, the compiler will be erroring out, since the signature will not match any virtual method in the - base class; - - `findalize_impl` is often empty; unless the AP is managing external resources, everything should be correctly released - during destruction; - - the two methods for buffers can be omitted if the AP does not need any scratch memory (and the default implementation - from the base class will be used). - -Here is a possible implementation of the methods, with some inline comments to explain +- we added two views to the class, which are meant to be used to store + intermediate results during calculations at runtime; +- there are other methods that the class can override (such as additional + operations when the AD sets a field in the AP), but most AP's only need to + override only these; +- we strongly encourage to add the keyword `override` when overriding a method; + in case of small typos (e.g., missing a `&` or a `const`, the compiler will + be erroring out, since the signature will not match any virtual method in the + base class; +- `finalize_impl` is often empty; unless the AP is managing external resources, + everything should be correctly released during destruction; +- the two methods for buffers can be omitted if the AP does not need any + scratch memory (and the default implementation from the base class will be + used). + +Here is a possible implementation of the methods, with some inline comments to +explain ```c++ MyProcess::MyProcess (const ekat::Comm& comm, const ekat::ParameterList& pl) diff --git a/components/eamxx/docs/developer/source_tree.md b/components/eamxx/docs/developer/source_tree.md index 15c018cc885..ed8270db635 100644 --- a/components/eamxx/docs/developer/source_tree.md +++ b/components/eamxx/docs/developer/source_tree.md @@ -56,4 +56,3 @@ You'll also see some other files in the `src/` directory itself, such as + `scream_config.h.in`: A template for generating a C++ header file with EAMxx configuration information. - diff --git a/components/eamxx/docs/developer/standalone_testing.md b/components/eamxx/docs/developer/standalone_testing.md index e2bb5d62556..633dcc34dc1 100644 --- a/components/eamxx/docs/developer/standalone_testing.md +++ b/components/eamxx/docs/developer/standalone_testing.md @@ -27,26 +27,30 @@ be made known to EAMxx by editing the eamxx/scripts/machines_specs.py files. There are some instructions on what to do at the top of this file. `test-all-scream` has a good help dump -``` -% cd $scream_repo/components/eamxx -% ./scripts/test-all-scream -h + +```shell +cd $scream_repo/components/eamxx +./scripts/test-all-scream -h ``` If you are unsure of the cmake configuration for you development cycle, one trick you can use is to run `test-all-scream` for the `dbg` test and just copy the cmake command it prints (then ctrl-C the process). -``` -% cd $scream_repo/components/eamxx -% ./scripts/test-all-scream -t dbg -m $machine -* wait for a few seconds* -* Ctrl-C * -* Copy the contents of DCMAKE_COMMAND that was passed to ctest * -* Add "cmake" to beginning of contents and path to eamxx at the end. * + +```shell +cd $scream_repo/components/eamxx +./scripts/test-all-scream -t dbg -m $machine +# wait for a few seconds* +# Ctrl-C * +# Copy the contents of DCMAKE_COMMAND that was passed to ctest * +# Add "cmake" to beginning of contents and path to eamxx at the end. * ``` Considerations for using `test-all-scream`: + * Your machine must be known to our scripts, see above. -* If you try to run commands by-hand (outside of test-all-scream; cmake, make, ctest, etc), you'll need to remember to +* If you try to run commands by-hand (outside of test-all-scream; + cmake, make, ctest, etc), you'll need to remember to load the scream-env into your shell, which can be done like this: `cd eamxx/scripts; eval $(./scripts/scream-env-cmd $machine)` * test-all-scream expects to be run from a compute node if you @@ -63,7 +67,7 @@ Considerations for using `test-all-scream`: Before running the tests, generate a baseline file: -``` +```shell cd $RUN_ROOT_DIR make baseline ``` @@ -75,7 +79,7 @@ path has been provided. To run all of SCREAM's tests, make sure you're in `$RUN_ROOT_DIR` and type -``` +```shell ctest -VV ``` @@ -84,7 +88,7 @@ This runs everything and reports results in an extra-verbose (`-VV`) manner. You can also run subsets of the SCREAM tests. For example, to run only the P3 regression tests (again, from the `$RUN_ROOT_DIR` directory), use -``` +```shell ctest -R p3_regression ``` @@ -94,13 +98,13 @@ We can create groupings of tests by using **labels**. For example, we have a `driver` label that runs tests for SCREAM's standalone driver. You can see a list of available labels by typing -``` +```shell ctest --print-labels ``` To see which tests are associated with a given label (e.g. `driver`), use -``` +```shell ctest -L driver -N ``` @@ -117,4 +121,3 @@ on the C++/Kokkos implementation, you can invoke any new tests to the function If the reference Fortran implementation changes enough that a new baseline file is required, make sure to let other SCREAM team members know, in order to minimize disruptions. - diff --git a/components/eamxx/docs/developer/style_guide.md b/components/eamxx/docs/developer/style_guide.md index f4367833009..4f6f340cb66 100644 --- a/components/eamxx/docs/developer/style_guide.md +++ b/components/eamxx/docs/developer/style_guide.md @@ -7,4 +7,3 @@ Here's our style guide. Let the holy wars begin! ## Functions and Methods ## Variables - diff --git a/components/eamxx/scripts/machines_specs.py b/components/eamxx/scripts/machines_specs.py index 41d09f4683e..c9c9f973022 100644 --- a/components/eamxx/scripts/machines_specs.py +++ b/components/eamxx/scripts/machines_specs.py @@ -221,9 +221,19 @@ class GHCISNLCuda(Machine): concrete = True @classmethod def setup(cls): - super().setup_base(name="ghci-snl-cuda",num_bld_res=16,num_run_res=1) + super().setup_base(name="ghci-snl-cuda") cls.baselines_dir = "/projects/e3sm/baselines/scream/ghci-snl-cuda" cls.gpu_arch = "cuda" + cls.num_run_res = int(run_cmd_no_fail("nvidia-smi --query-gpu=name --format=csv,noheader | wc -l")) + +############################################################################### +class GHCIOCI(Machine): +############################################################################### + concrete = True + @classmethod + def setup(cls): + super().setup_base(name="ghci-oci") + cls.env_setup = [f"eval $({CIMEROOT}/CIME/Tools/get_case_env -c SMS.ne4pg2_ne4pg2.F2010-SCREAMv1.ghci-oci_gnu)"] ############################################################################### class Lassen(Machine): diff --git a/components/eamxx/scripts/query_scream.py b/components/eamxx/scripts/query_scream.py index 4b26451c7cb..a6fe5096bfb 100644 --- a/components/eamxx/scripts/query_scream.py +++ b/components/eamxx/scripts/query_scream.py @@ -1,9 +1,5 @@ -from machines_specs import assert_machine_supported, \ - get_mach_cxx_compiler, get_mach_c_compiler, get_mach_f90_compiler, \ - get_mach_batch_command, get_mach_env_setup_command, \ - get_mach_baseline_root_dir, is_cuda_machine, \ - get_mach_compilation_resources, get_mach_testing_resources +from machines_specs import assert_machine_supported, get_machine, get_mach_env_setup_command from utils import expect CHOICES = ( @@ -24,23 +20,24 @@ def query_scream(machine, param): assert_machine_supported(machine) expect(param in CHOICES, f"Unknown param {param}") + mach = get_machine(machine) if param == "cxx_compiler": - return get_mach_cxx_compiler(machine) + return mach.cxx_compiler elif param == "c_compiler": - return get_mach_c_compiler(machine) + return mach.c_compiler elif param == "f90_compiler": - return get_mach_f90_compiler(machine) + return mach.ftn_compiler elif param == "batch": - return get_mach_batch_command(machine) + return mach.batch elif param == "env": return get_mach_env_setup_command(machine) elif param == "baseline_root": - return get_mach_baseline_root_dir(machine) + return mach.baselines_dir elif param == "cuda": - return str(is_cuda_machine(machine)) + return str(mach.gpu_arch == "cuda") elif param == "comp_j": - return get_mach_compilation_resources() + return num_bld_res elif param == "test_j": - return get_mach_testing_resources(machine) + return gnum_run_res else: expect(False, f"Unhandled param {param}") diff --git a/components/eamxx/src/control/atmosphere_driver.cpp b/components/eamxx/src/control/atmosphere_driver.cpp index 252a3fe53b7..e935cff5cfc 100644 --- a/components/eamxx/src/control/atmosphere_driver.cpp +++ b/components/eamxx/src/control/atmosphere_driver.cpp @@ -807,7 +807,8 @@ void AtmosphereDriver:: set_provenance_data (std::string caseid, std::string rest_caseid, std::string hostname, - std::string username) + std::string username, + std::string versionid) { #ifdef SCREAM_CIME_BUILD // Check the inputs are valid @@ -816,6 +817,7 @@ set_provenance_data (std::string caseid, "Error! Invalid restart case id: " + rest_caseid + "\n"); EKAT_REQUIRE_MSG (hostname!="", "Error! Invalid hostname: " + hostname + "\n"); EKAT_REQUIRE_MSG (username!="", "Error! Invalid username: " + username + "\n"); + EKAT_REQUIRE_MSG (versionid!="", "Error! Invalid version: " + versionid + "\n"); #else caseid = rest_caseid = m_casename; char* user = new char[32]; @@ -835,13 +837,14 @@ set_provenance_data (std::string caseid, } delete[] user; delete[] host; + versionid = EAMXX_GIT_VERSION; #endif auto& provenance = m_atm_params.sublist("provenance"); provenance.set("caseid",caseid); provenance.set("rest_caseid",rest_caseid); provenance.set("hostname",hostname); provenance.set("username",username); - provenance.set("version",std::string(EAMXX_GIT_VERSION)); + provenance.set("git_version",versionid); } void AtmosphereDriver:: diff --git a/components/eamxx/src/control/atmosphere_driver.hpp b/components/eamxx/src/control/atmosphere_driver.hpp index f53cd18302d..9b191371b36 100644 --- a/components/eamxx/src/control/atmosphere_driver.hpp +++ b/components/eamxx/src/control/atmosphere_driver.hpp @@ -116,7 +116,8 @@ class AtmosphereDriver void set_provenance_data (std::string caseid = "", std::string rest_caseid = "", std::string hostname = "", - std::string username = ""); + std::string username = "", + std::string versionid = ""); // Load initial conditions for atm inputs void initialize_fields (); diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp b/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp index 790b60c449c..385e57cae55 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp @@ -28,35 +28,41 @@ void SurfaceCouplingImporter::set_grids(const std::shared_ptr("sfc_alb_dir_vis", scalar2d_layout, nondim, grid_name); - add_field("sfc_alb_dir_nir", scalar2d_layout, nondim, grid_name); - add_field("sfc_alb_dif_vis", scalar2d_layout, nondim, grid_name); - add_field("sfc_alb_dif_nir", scalar2d_layout, nondim, grid_name); - add_field("surf_lw_flux_up", scalar2d_layout, W/m2, grid_name); - add_field("surf_sens_flux", scalar2d_layout, W/m2, grid_name); - add_field("surf_evap", scalar2d_layout, kg/m2/s, grid_name); - add_field("surf_mom_flux", vector2d_layout, N/m2, grid_name); - add_field("surf_radiative_T", scalar2d_layout, K, grid_name); - add_field("T_2m", scalar2d_layout, K, grid_name); - add_field("qv_2m", scalar2d_layout, kg/kg, grid_name); - add_field("wind_speed_10m", scalar2d_layout, m/s, grid_name); - add_field("snow_depth_land", scalar2d_layout, m, grid_name); - add_field("ocnfrac", scalar2d_layout, nondim, grid_name); - add_field("landfrac", scalar2d_layout, nondim, grid_name); - add_field("icefrac", scalar2d_layout, nondim, grid_name); + const FieldLayout scalar2d = m_grid->get_2d_scalar_layout(); + const FieldLayout vector2d = m_grid->get_2d_vector_layout(2); + const FieldLayout vector4d = m_grid->get_2d_vector_layout(4); + + add_field("sfc_alb_dir_vis", scalar2d, nondim, grid_name); + add_field("sfc_alb_dir_nir", scalar2d, nondim, grid_name); + add_field("sfc_alb_dif_vis", scalar2d, nondim, grid_name); + add_field("sfc_alb_dif_nir", scalar2d, nondim, grid_name); + add_field("surf_lw_flux_up", scalar2d, W/m2, grid_name); + add_field("surf_sens_flux", scalar2d, W/m2, grid_name); + add_field("surf_evap", scalar2d, kg/m2/s, grid_name); + add_field("surf_mom_flux", vector2d, N/m2, grid_name); + add_field("surf_radiative_T", scalar2d, K, grid_name); + add_field("T_2m", scalar2d, K, grid_name); + add_field("qv_2m", scalar2d, kg/kg, grid_name); + add_field("wind_speed_10m", scalar2d, m/s, grid_name); + add_field("snow_depth_land", scalar2d, m, grid_name); + add_field("ocnfrac", scalar2d, nondim, grid_name); + add_field("landfrac", scalar2d, nondim, grid_name); + add_field("icefrac", scalar2d, nondim, grid_name); // Friction velocity [m/s] - add_field("fv", scalar2d_layout, m/s, grid_name); + add_field("fv", scalar2d, m/s, grid_name); // Aerodynamical resistance - add_field("ram1", scalar2d_layout, s/m, grid_name); + add_field("ram1", scalar2d, s/m, grid_name); + // Sea surface temperature [K] + add_field("sst", scalar2d, K, grid_name); + //dust fluxes [kg/m^2/s]: Four flux values for eacch column + add_field("dstflx", vector4d, kg/m2/s, grid_name); + } // ========================================================================================= void SurfaceCouplingImporter::setup_surface_coupling_data(const SCDataManager &sc_data_manager) diff --git a/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp b/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp index 87009c7d074..df5de6827f6 100644 --- a/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp +++ b/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp @@ -271,6 +271,7 @@ build_physics_grid (const ci_string& type, const ci_string& rebalance) { auto hyam = phys_grid->create_geometry_data("hyam",layout_mid,nondim); auto hybm = phys_grid->create_geometry_data("hybm",layout_mid,nondim); auto lev = phys_grid->create_geometry_data("lev", layout_mid,mbar); + auto ilev = phys_grid->create_geometry_data("ilev",layout_int,mbar); for (auto f : {hyai, hybi, hyam, hybm}) { auto f_d = get_grid("Dynamics")->get_geometry_data(f.name()); @@ -281,13 +282,20 @@ build_physics_grid (const ci_string& type, const ci_string& rebalance) { // Build lev from hyam and hybm const Real ps0 = 100000.0; - auto hya_v = hyam.get_view(); - auto hyb_v = hybm.get_view(); - auto lev_v = lev.get_view(); - for (int ii=0;iiget_num_vertical_levels();ii++) { - lev_v(ii) = 0.01*ps0*(hya_v(ii)+hyb_v(ii)); + auto hyam_v = hyam.get_view(); + auto hybm_v = hybm.get_view(); + auto hyai_v = hyai.get_view(); + auto hybi_v = hybi.get_view(); + auto lev_v = lev.get_view(); + auto ilev_v = ilev.get_view(); + auto num_v_levs = phys_grid->get_num_vertical_levels(); + for (int ii=0;ii("sfc_alb_dir_vis", scalar2d, nondim, grid_name); + //----------- Variables from microphysics scheme ------------- + + // Evaporation from stratiform rain [kg/kg/s] + add_field("nevapr", scalar3d_mid, kg / kg / s, grid_name); + + // Stratiform rain production rate [kg/kg/s] + add_field("precip_total_tend", scalar3d_mid, kg / kg / s, + grid_name); + // --------------------------------------------------------------------- // These variables are "updated" or inputs/outputs for the process // --------------------------------------------------------------------- @@ -205,7 +215,7 @@ void MAMMicrophysics::set_grids( LinozHorizInterp_, linoz_file_name_); // linoz reader - const auto io_grid_linoz = LinozHorizInterp_->get_src_grid(); + const auto io_grid_linoz = LinozHorizInterp_->get_tgt_grid(); const int num_cols_io_linoz = io_grid_linoz->get_num_local_dofs(); // Number of columns on this rank const int num_levs_io_linoz = @@ -233,7 +243,7 @@ void MAMMicrophysics::set_grids( TracerHorizInterp_, oxid_file_name_); const int nvars = int(var_names.size()); - const auto io_grid = TracerHorizInterp_->get_src_grid(); + const auto io_grid = TracerHorizInterp_->get_tgt_grid(); const int num_cols_io = io_grid->get_num_local_dofs(); // Number of columns on this rank const int num_levs_io = @@ -258,78 +268,88 @@ void MAMMicrophysics::set_grids( "num_a1", "num_a2", "num_a4", "soag"}; for(const auto &var_name : extfrc_lst_) { - std::string item_name = "mam4_" + var_name + "_verti_emiss_file_name"; + std::string item_name = "mam4_" + var_name + "_elevated_emiss_file_name"; const auto file_name = m_params.get(item_name); - vert_emis_file_name_[var_name] = file_name; + elevated_emis_file_name_[var_name] = file_name; } - vert_emis_var_names_["so2"] = {"BB", "ENE_ELEV", "IND_ELEV", "contvolc"}; - vert_emis_var_names_["so4_a1"] = {"BB", "ENE_ELEV", "IND_ELEV", "contvolc"}; - vert_emis_var_names_["so4_a2"] = {"contvolc"}; - vert_emis_var_names_["pom_a4"] = {"BB"}; - vert_emis_var_names_["bc_a4"] = {"BB"}; - vert_emis_var_names_["num_a1"] = { + elevated_emis_var_names_["so2"] = {"BB", "ENE_ELEV", "IND_ELEV", "contvolc"}; + elevated_emis_var_names_["so4_a1"] = {"BB", "ENE_ELEV", "IND_ELEV", "contvolc"}; + elevated_emis_var_names_["so4_a2"] = {"contvolc"}; + elevated_emis_var_names_["pom_a4"] = {"BB"}; + elevated_emis_var_names_["bc_a4"] = {"BB"}; + elevated_emis_var_names_["num_a1"] = { "num_a1_SO4_ELEV_BB", "num_a1_SO4_ELEV_ENE", "num_a1_SO4_ELEV_IND", "num_a1_SO4_ELEV_contvolc"}; - vert_emis_var_names_["num_a2"] = {"num_a2_SO4_ELEV_contvolc"}; + elevated_emis_var_names_["num_a2"] = {"num_a2_SO4_ELEV_contvolc"}; // num_a4 // FIXME: why the sectors in this files are num_a1; // I guess this should be num_a4? Is this a bug in the orginal nc files? - vert_emis_var_names_["num_a4"] = {"num_a1_BC_ELEV_BB", + elevated_emis_var_names_["num_a4"] = {"num_a1_BC_ELEV_BB", "num_a1_POM_ELEV_BB"}; - vert_emis_var_names_["soag"] = {"SOAbb_src", "SOAbg_src", "SOAff_src"}; + elevated_emis_var_names_["soag"] = {"SOAbb_src", "SOAbg_src", "SOAff_src"}; - int verti_emiss_cyclical_ymd = m_params.get("verti_emiss_ymd"); + int elevated_emiss_cyclical_ymd = m_params.get("elevated_emiss_ymd"); for(const auto &var_name : extfrc_lst_) { - const auto file_name = vert_emis_file_name_[var_name]; - const auto var_names = vert_emis_var_names_[var_name]; + const auto file_name = elevated_emis_file_name_[var_name]; + const auto var_names = elevated_emis_var_names_[var_name]; scream::mam_coupling::TracerData data_tracer; scream::mam_coupling::setup_tracer_data(data_tracer, file_name, - verti_emiss_cyclical_ymd); + elevated_emiss_cyclical_ymd); auto hor_rem = scream::mam_coupling::create_horiz_remapper( grid_, file_name, extfrc_map_file, var_names, data_tracer); + auto file_reader = - scream::mam_coupling::create_tracer_data_reader(hor_rem, file_name); - VertEmissionsHorizInterp_.push_back(hor_rem); - VertEmissionsDataReader_.push_back(file_reader); - vert_emis_data_.push_back(data_tracer); - } // var_name vert emissions + scream::mam_coupling::create_tracer_data_reader(hor_rem, file_name, + data_tracer.file_type); + ElevatedEmissionsHorizInterp_.push_back(hor_rem); + ElevatedEmissionsDataReader_.push_back(file_reader); + elevated_emis_data_.push_back(data_tracer); + } // var_name elevated emissions int i = 0; int offset_emis_ver = 0; for(const auto &var_name : extfrc_lst_) { - const auto file_name = vert_emis_file_name_[var_name]; - const auto var_names = vert_emis_var_names_[var_name]; + const auto file_name = elevated_emis_file_name_[var_name]; + const auto var_names = elevated_emis_var_names_[var_name]; const int nvars = static_cast(var_names.size()); forcings_[i].nsectors = nvars; // I am assuming the order of species in extfrc_lst_. // Indexing in mam4xx is fortran. forcings_[i].frc_ndx = i + 1; - const auto io_grid_emis = VertEmissionsHorizInterp_[i]->get_src_grid(); + const auto io_grid_emis = ElevatedEmissionsHorizInterp_[i]->get_tgt_grid(); const int num_cols_io_emis = io_grid_emis->get_num_local_dofs(); // Number of columns on this rank const int num_levs_io_emis = io_grid_emis ->get_num_vertical_levels(); // Number of levels per column - vert_emis_data_[i].init(num_cols_io_emis, num_levs_io_emis, nvars); - vert_emis_data_[i].allocate_temporal_views(); - forcings_[i].file_alt_data = vert_emis_data_[i].has_altitude_; + elevated_emis_data_[i].init(num_cols_io_emis, num_levs_io_emis, nvars); + elevated_emis_data_[i].allocate_temporal_views(); + forcings_[i].file_alt_data = elevated_emis_data_[i].has_altitude_; for(int isp = 0; isp < nvars; ++isp) { forcings_[i].offset = offset_emis_ver; - vert_emis_output_[isp + offset_emis_ver] = - view_2d("vert_emis_output_", ncol_, nlev_); + elevated_emis_output_[isp + offset_emis_ver] = + view_2d("elevated_emis_output_", ncol_, nlev_); } offset_emis_ver += nvars; ++i; } // end i EKAT_REQUIRE_MSG( - offset_emis_ver <= int(mam_coupling::MAX_NUM_VERT_EMISSION_FIELDS), + offset_emis_ver <= int(mam_coupling::MAX_NUM_ELEVATED_EMISSIONS_FIELDS), "Error! Number of fields is bigger than " - "MAX_NUM_VERT_EMISSION_FIELDS. Increase the " - "MAX_NUM_VERT_EMISSION_FIELDS in tracer_reader_utils.hpp \n"); + "MAX_NUM_ELEVATED_EMISSIONS_FIELDS. Increase the " + "MAX_NUM_ELEVATED_EMISSIONS_FIELDS in tracer_reader_utils.hpp \n"); } // Tracer external forcing data + + { + const std::string season_wes_file = m_params.get("mam4_season_wes_file"); + const auto& clat = col_latitudes_; + mam_coupling::find_season_index_reader(season_wes_file, + clat, + index_season_lai_); + } } // set_grids // ================================================================ @@ -501,6 +521,9 @@ void MAMMicrophysics::initialize_impl(const RunType run_type) { const int photo_table_len = get_photo_table_work_len(photo_table_); work_photo_table_ = view_2d("work_photo_table", ncol_, photo_table_len); + const int sethet_work_len = mam4::mo_sethet::get_total_work_len_sethet(); + work_set_het_ = view_2d("work_set_het_array", ncol_, sethet_work_len); + cmfdqr_ = view_1d("cmfdqr_", nlev_); // here's where we store per-column photolysis rates photo_rates_ = view_3d("photo_rates", ncol_, nlev_, mam4::mo_photo::phtcnt); @@ -518,8 +541,8 @@ void MAMMicrophysics::initialize_impl(const RunType run_type) { for(int i = 0; i < static_cast(extfrc_lst_.size()); ++i) { scream::mam_coupling::update_tracer_data_from_file( - VertEmissionsDataReader_[i], curr_month, *VertEmissionsHorizInterp_[i], - vert_emis_data_[i]); + ElevatedEmissionsDataReader_[i], curr_month, *ElevatedEmissionsHorizInterp_[i], + elevated_emis_data_[i]); } invariants_ = view_3d("invarians", ncol_, nlev_, mam4::gas_chemistry::nfs); @@ -554,6 +577,14 @@ void MAMMicrophysics::run_impl(const double dt) { Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); + //----------- Variables from microphysics scheme ------------- + + // Evaporation from stratiform rain [kg/kg/s] + const auto& nevapr = get_field_in("nevapr").get_view(); + + // Stratiform rain production rate [kg/kg/s] + const auto& prain = get_field_in("precip_total_tend").get_view(); + const auto wet_geometric_mean_diameter_i = get_field_in("dgnumwet").get_view(); const auto dry_geometric_mean_diameter_i = @@ -616,20 +647,20 @@ void MAMMicrophysics::run_impl(const double dt) { linoz_output); // out Kokkos::fence(); - vert_emiss_time_state_.t_now = ts.frac_of_year_in_days(); + elevated_emiss_time_state_.t_now = ts.frac_of_year_in_days(); int i = 0; for(const auto &var_name : extfrc_lst_) { - const auto file_name = vert_emis_file_name_[var_name]; - const auto var_names = vert_emis_var_names_[var_name]; + const auto file_name = elevated_emis_file_name_[var_name]; + const auto var_names = elevated_emis_var_names_[var_name]; const int nsectors = int(var_names.size()); - view_2d vert_emis_output[nsectors]; + view_2d elevated_emis_output[nsectors]; for(int isp = 0; isp < nsectors; ++isp) { - vert_emis_output[isp] = vert_emis_output_[isp + forcings_[i].offset]; + elevated_emis_output[isp] = elevated_emis_output_[isp + forcings_[i].offset]; } scream::mam_coupling::advance_tracer_data( - VertEmissionsDataReader_[i], *VertEmissionsHorizInterp_[i], ts, - vert_emiss_time_state_, vert_emis_data_[i], dry_atm_.p_mid, - dry_atm_.z_iface, vert_emis_output); + ElevatedEmissionsDataReader_[i], *ElevatedEmissionsHorizInterp_[i], ts, + elevated_emiss_time_state_, elevated_emis_data_[i], dry_atm_.p_mid, + dry_atm_.z_iface, elevated_emis_output); i++; Kokkos::fence(); } @@ -704,7 +735,7 @@ void MAMMicrophysics::run_impl(const double dt) { const auto zenith_angle = acos_cosine_zenith_; constexpr int gas_pcnst = mam_coupling::gas_pcnst(); - const auto& vert_emis_output = vert_emis_output_; + const auto& elevated_emis_output = elevated_emis_output_; const auto& extfrc = extfrc_; const auto& forcings = forcings_; constexpr int extcnt = mam4::gas_chemistry::extcnt; @@ -722,6 +753,8 @@ void MAMMicrophysics::run_impl(const double dt) { clsmap_4[i] = mam4::gas_chemistry::clsmap_4[i]; permute_4[i] = mam4::gas_chemistry::permute_4[i]; } + const auto& cmfdqr = cmfdqr_; + const auto& work_set_het =work_set_het_; // loop over atmosphere columns and compute aerosol microphyscs Kokkos::parallel_for( policy, KOKKOS_LAMBDA(const ThreadTeam &team) { @@ -756,7 +789,7 @@ void MAMMicrophysics::run_impl(const double dt) { // We may need to move this line where we read files. forcings_in[i].file_alt_data = file_alt_data; for(int isec = 0; isec < forcings[i].nsectors; ++isec) { - const auto field = vert_emis_output[isec + forcings[i].offset]; + const auto field = elevated_emis_output[isec + forcings[i].offset]; forcings_in[i].fields_data[isec] = ekat::subview(field, icol); } } // extcnt for loop @@ -787,6 +820,9 @@ void MAMMicrophysics::run_impl(const double dt) { ekat::subview(linoz_dPmL_dO3col, icol); const auto linoz_cariolle_pscs_icol = ekat::subview(linoz_cariolle_pscs, icol); + const auto nevapr_icol = ekat::subview(nevapr, icol); + const auto prain_icol = ekat::subview(prain, icol); + const auto work_set_het_icol = ekat::subview(work_set_het, icol); // Note: All variables are inputs, except for progs, which is an // input/output variable. mam4::microphysics::perform_atmospheric_chemistry_and_microphysics( @@ -800,7 +836,12 @@ void MAMMicrophysics::run_impl(const double dt) { linoz_cariolle_pscs_icol, eccf, adv_mass_kg_per_moles, clsmap_4, permute_4, offset_aerosol, config.linoz.o3_sfc, config.linoz.o3_tau, config.linoz.o3_lbl, - dry_diameter_icol, wet_diameter_icol, wetdens_icol); + dry_diameter_icol, wet_diameter_icol, wetdens_icol, + dry_atm.phis(icol), + cmfdqr, + prain_icol, + nevapr_icol, + work_set_het_icol); }); // parallel_for for the column loop Kokkos::fence(); diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp index 6b1dd33dfaa..6ff846d0d0c 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp @@ -25,6 +25,8 @@ class MAMMicrophysics final : public scream::AtmosphereProcess { using view_1d_host = typename KT::view_1d::HostMirror; + using view_int_2d = typename KT::template view_2d; + // a thread team dispatched to a single vertical column using ThreadTeam = mam4::ThreadTeam; @@ -225,20 +227,25 @@ class MAMMicrophysics final : public scream::AtmosphereProcess { // Vertical emission uses 9 files, here I am using std::vector to stote // instance of each file. - mam_coupling::TracerTimeState vert_emiss_time_state_; - std::vector> VertEmissionsDataReader_; - std::vector> VertEmissionsHorizInterp_; + mam_coupling::TracerTimeState elevated_emiss_time_state_; + std::vector> ElevatedEmissionsDataReader_; + std::vector> ElevatedEmissionsHorizInterp_; std::vector extfrc_lst_; - std::vector vert_emis_data_; - std::map vert_emis_file_name_; - std::map> vert_emis_var_names_; - view_2d vert_emis_output_[mam_coupling::MAX_NUM_VERT_EMISSION_FIELDS]; + std::vector elevated_emis_data_; + std::map elevated_emis_file_name_; + std::map> elevated_emis_var_names_; + view_2d elevated_emis_output_[mam_coupling::MAX_NUM_ELEVATED_EMISSIONS_FIELDS]; view_3d extfrc_; mam_coupling::ForcingHelper forcings_[mam4::gas_chemistry::extcnt]; view_1d_host acos_cosine_zenith_host_; view_1d acos_cosine_zenith_; + view_int_2d index_season_lai_; + // // dq/dt for convection [kg/kg/s] + view_1d cmfdqr_; + view_2d work_set_het_; + }; // MAMMicrophysics } // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_functions.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_functions.hpp new file mode 100644 index 00000000000..9c01daf8223 --- /dev/null +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_functions.hpp @@ -0,0 +1,73 @@ +#ifndef EAMXX_MAM_SRF_AND_ONLINE_EMISSIONS_FUNCTIONS_HPP +#define EAMXX_MAM_SRF_AND_ONLINE_EMISSIONS_FUNCTIONS_HPP + +namespace scream { + +namespace { + +using KT = ekat::KokkosTypes; +using view_1d = typename KT::template view_1d; +using view_2d = typename KT::template view_2d; +using const_view_1d = typename KT::template view_1d; +using const_view_2d = typename KT::template view_2d; + +//-------- Inititlize gas and aerosol fluxes ------ +void init_fluxes(const int &ncol, + view_2d &constituent_fluxes) { // input-output + + constexpr int pcnst = mam4::aero_model::pcnst; + const int gas_start_ind = mam4::utils::gasses_start_ind(); + + const auto policy = + ekat::ExeSpaceUtils::get_default_team_policy( + ncol, pcnst - gas_start_ind); + + // Parallel loop over all the columns + Kokkos::parallel_for( + policy, KOKKOS_LAMBDA(const KT::MemberType &team) { + const int icol = team.league_rank(); + view_1d flux_col = ekat::subview(constituent_fluxes, icol); + + // Zero out constituent fluxes only for gasses and aerosols + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, gas_start_ind, pcnst), + [&](int icnst) { flux_col(icnst) = 0; }); + }); +} // init_fluxes ends + +//-------- compute online emissions for dust, sea salt and marine organics ----- +void compute_online_dust_nacl_emiss( + const int &ncol, const int &nlev, const const_view_1d &ocnfrac, + const const_view_1d &sst, const const_view_2d &u_wind, + const const_view_2d &v_wind, const const_view_2d &dstflx, + const const_view_1d &mpoly, const const_view_1d &mprot, + const const_view_1d &mlip, const const_view_1d &soil_erodibility, + const const_view_2d &z_mid, + // output + view_2d &constituent_fluxes) { + const int surf_lev = nlev - 1; // surface level + + Kokkos::parallel_for( + "online_emis_fluxes", ncol, KOKKOS_LAMBDA(int icol) { + // Input + const const_view_1d dstflx_icol = ekat::subview(dstflx, icol); + + // Output + view_1d fluxes_col = ekat::subview(constituent_fluxes, icol); + + // Compute online emissions + // NOTE: mam4::aero_model_emissions calculates mass and number emission + // fluxes in units of [kg/m2/s or #/m2/s] (MKS), so no need to convert + mam4::aero_model_emissions::aero_model_emissions( + sst(icol), ocnfrac(icol), u_wind(icol, surf_lev), + v_wind(icol, surf_lev), z_mid(icol, surf_lev), dstflx_icol, + soil_erodibility(icol), mpoly(icol), mprot(icol), mlip(icol), + // out + fluxes_col); + }); +} // compute_online_dust_nacl_emiss ends + +} // namespace +} // namespace scream + +#endif // EAMXX_MAM_SRF_AND_ONLINE_EMISSIONS_FUNCTIONS_HPP diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp index 850a82d0896..fd3ebf79700 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.cpp @@ -1,14 +1,24 @@ -//#include #include +// For surface and online emission functions +#include + +// For reading soil erodibility file +#include + namespace scream { +// For reading soil erodibility file +using soilErodibilityFunc = + soil_erodibility::soilErodibilityFunctions; + // ================================================================ // Constructor // ================================================================ MAMSrfOnlineEmiss::MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::ParameterList ¶ms) : AtmosphereProcess(comm, params) { + // FIXME: Do we want to read dust emiss factor from the namelist?? /* Anything that can be initialized without grid information can be * initialized here. Like universal constants. */ @@ -26,18 +36,98 @@ void MAMSrfOnlineEmiss::set_grids( nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column using namespace ekat::units; + constexpr auto m2 = pow(m, 2); + constexpr auto s2 = pow(s, 2); + constexpr auto q_unit = kg / kg; // units of mass mixing ratios of tracers + constexpr auto n_unit = 1 / kg; // units of number mixing ratios of tracers + constexpr auto nondim = ekat::units::Units::nondimensional(); - static constexpr int pcnst = mam4::aero_model::pcnst; - const FieldLayout scalar2d_pcnct = - grid_->get_2d_vector_layout(pcnst, "num_phys_constituents"); + const FieldLayout scalar2d = grid_->get_2d_scalar_layout(); + const FieldLayout scalar3d_m = grid_->get_3d_scalar_layout(true); // mid + const FieldLayout scalar3d_i = grid_->get_3d_scalar_layout(false); // int + + // For U and V components of wind + const FieldLayout vector3d = grid_->get_3d_vector_layout(true, 2); + + // For components of dust flux + const FieldLayout vector4d = grid_->get_2d_vector_layout(4); + + // -------------------------------------------------------------------------- + // These variables are "Required" or pure inputs for the process + // -------------------------------------------------------------------------- + + // ----------- Atmospheric quantities ------------- + + // -- Variables required for building DS to compute z_mid -- + // Specific humidity [kg/kg] + // FIXME: Comply with add_tracer calls + add_field("qv", scalar3d_m, q_unit, grid_name, "tracers"); + + // Cloud liquid mass mixing ratio [kg/kg] + add_field("qc", scalar3d_m, q_unit, grid_name, "tracers"); + + // Cloud ice mass mixing ratio [kg/kg] + add_field("qi", scalar3d_m, q_unit, grid_name, "tracers"); + + // Cloud liquid number mixing ratio [1/kg] + add_field("nc", scalar3d_m, n_unit, grid_name, "tracers"); + + // Cloud ice number mixing ratio [1/kg] + add_field("ni", scalar3d_m, n_unit, grid_name, "tracers"); + + // Temperature[K] at midpoints + add_field("T_mid", scalar3d_m, K, grid_name); + + // Vertical pressure velocity [Pa/s] at midpoints + add_field("omega", scalar3d_m, Pa / s, grid_name); + + // Total pressure [Pa] at midpoints + add_field("p_mid", scalar3d_m, Pa, grid_name); + + // Total pressure [Pa] at interfaces + add_field("p_int", scalar3d_i, Pa, grid_name); + + // Layer thickness(pdel) [Pa] at midpoints + add_field("pseudo_density", scalar3d_m, Pa, grid_name); + + // Planetary boundary layer height [m] + add_field("pbl_height", scalar2d, m, grid_name); + + // Surface geopotential [m2/s2] + add_field("phis", scalar2d, m2 / s2, grid_name); + + //----------- Variables from microphysics scheme ------------- + + // Total cloud fraction [fraction] (Require only for building DS) + add_field("cldfrac_tot", scalar3d_m, nondim, grid_name); + + // -- Variables required for online dust and sea salt emissions -- + + // U and V components of the wind[m/s] + add_field("horiz_winds", vector3d, m / s, grid_name); + + //----------- Variables from coupler (ocean component)--------- + // Ocean fraction [unitless] + add_field("ocnfrac", scalar2d, nondim, grid_name); + + // Sea surface temperature [K] + add_field("sst", scalar2d, K, grid_name); + + // dust fluxes [kg/m^2/s]: Four flux values for each column + add_field("dstflx", vector4d, kg / m2 / s, grid_name); // ------------------------------------------------------------- - // These variables are "Computed" or outputs for the process + // These variables are "Updated" or input-outputs for the process // ------------------------------------------------------------- - static constexpr Units m2(m * m, "m2"); + + constexpr int pcnst = mam4::aero_model::pcnst; + const FieldLayout vector2d_pcnst = + grid_->get_2d_vector_layout(pcnst, "num_phys_constituents"); + // Constituent fluxes of species in [kg/m2/s] - add_field("constituent_fluxes", scalar2d_pcnct, kg / m2 / s, - grid_name); + // FIXME: confirm if it is Updated or Computed + add_field("constituent_fluxes", vector2d_pcnst, kg / m2 / s, + grid_name); // Surface emissions remapping file auto srf_map_file = m_params.get("srf_remap_file", ""); @@ -63,6 +153,7 @@ void MAMSrfOnlineEmiss::set_grids( so2.species_name = "so2"; so2.sectors = {"AGR", "RCO", "SHP", "SLV", "TRA", "WST"}; srf_emiss_species_.push_back(so2); // add to the vector + //-------------------------------------------------------------------- // Init bc_a4 srf emiss data structures //-------------------------------------------------------------------- @@ -147,7 +238,48 @@ void MAMSrfOnlineEmiss::set_grids( // output ispec_srf.horizInterp_, ispec_srf.data_start_, ispec_srf.data_end_, ispec_srf.data_out_, ispec_srf.dataReader_); - } + } // srf emissions file read init + + // ------------------------------------------------------------- + // Setup to enable reading soil erodibility file + // ------------------------------------------------------------- + + const std::string soil_erodibility_data_file = + m_params.get("soil_erodibility_file"); + + // Field to be read from file + const std::string soil_erod_fld_name = "mbl_bsn_fct_geo"; + + // Dimensions of the field + const std::string soil_erod_dname = "ncol"; + + // initialize the file read + soilErodibilityFunc::init_soil_erodibility_file_read( + ncol_, soil_erod_fld_name, soil_erod_dname, grid_, + soil_erodibility_data_file, srf_map_file, serod_horizInterp_, + serod_dataReader_); // output + + // ------------------------------------------------------------- + // Setup to enable reading marine organics file + // ------------------------------------------------------------- + const std::string marine_organics_data_file = + m_params.get("marine_organics_file"); + + // Fields to be read from file (order matters as they are read in the same + // order) + const std::vector marine_org_fld_name = { + "TRUEPOLYC", "TRUEPROTC", "TRUELIPC"}; + + // Dimensions of the field + const std::string marine_org_dname = "ncol"; + + // initialize the file read + marineOrganicsFunc::init_marine_organics_file_read( + ncol_, marine_org_fld_name, marine_org_dname, grid_, + marine_organics_data_file, srf_map_file, + // output + morg_horizInterp_, morg_data_start_, morg_data_end_, morg_data_out_, + morg_dataReader_); } // set_grid ends @@ -182,6 +314,42 @@ void MAMSrfOnlineEmiss::init_buffers(const ATMBufferManager &buffer_manager) { // INITIALIZE_IMPL // ================================================================ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { + // --------------------------------------------------------------- + // Input fields read in from IC file, namelist or other processes + // --------------------------------------------------------------- + + // Populate the wet atmosphere state with views from fields + wet_atm_.qv = get_field_in("qv").get_view(); + + // Following wet_atm vars are required only for building DS + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + + // Populate the dry atmosphere state with views from fields + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.p_int = get_field_in("p_int").get_view(); + + // Following dry_atm vars are required only for building DS + dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); + dry_atm_.pblh = get_field_in("pbl_height").get_view(); + dry_atm_.omega = get_field_in("omega").get_view(); + + // store fields converted to dry mmr from wet mmr in dry_atm_ + dry_atm_.z_mid = buffer_.z_mid; + dry_atm_.z_iface = buffer_.z_iface; + dry_atm_.dz = buffer_.dz; + dry_atm_.qv = buffer_.qv_dry; + dry_atm_.qc = buffer_.qc_dry; + dry_atm_.nc = buffer_.nc_dry; + dry_atm_.qi = buffer_.qi_dry; + dry_atm_.ni = buffer_.ni_dry; + dry_atm_.w_updraft = buffer_.w_updraft; + dry_atm_.z_surf = 0.0; // FIXME: for now + // --------------------------------------------------------------- // Output fields // --------------------------------------------------------------- @@ -212,10 +380,27 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { ispec_srf.data_end_); // output } + //----------------------------------------------------------------- + // Read Soil erodibility data + //----------------------------------------------------------------- + // This data is time-independent, we read all data here for the + // entire simulation + soilErodibilityFunc::update_soil_erodibility_data_from_file( + serod_dataReader_, *serod_horizInterp_, + soil_erodibility_); // output + + //-------------------------------------------------------------------- + // Update marine orgaincs from file + //-------------------------------------------------------------------- + // Time dependent data + marineOrganicsFunc::update_marine_organics_data_from_file( + morg_dataReader_, timestamp(), curr_month, *morg_horizInterp_, + morg_data_end_); // output + //----------------------------------------------------------------- // Setup preprocessing and post processing //----------------------------------------------------------------- - preprocess_.initialize(constituent_fluxes_); + preprocess_.initialize(ncol_, nlev_, wet_atm_, dry_atm_); } // end initialize_impl() @@ -223,14 +408,79 @@ void MAMSrfOnlineEmiss::initialize_impl(const RunType run_type) { // RUN_IMPL // ================================================================ void MAMSrfOnlineEmiss::run_impl(const double dt) { - // Zero-out output - Kokkos::deep_copy(preprocess_.constituent_fluxes_pre_, 0); + const auto scan_policy = ekat::ExeSpaceUtils< + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + + // preprocess input -- needs a scan for the calculation of atm height + Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + Kokkos::fence(); + + // Constituent fluxes [kg/m^2/s] + auto constituent_fluxes = this->constituent_fluxes_; + // Zero out constituent fluxes only for gasses and aerosols + init_fluxes(ncol_, // in + constituent_fluxes); // in-out + Kokkos::fence(); // Gather time and state information for interpolation - auto ts = timestamp() + dt; + const auto ts = timestamp() + dt; //-------------------------------------------------------------------- - // Interpolate srf emiss data + // Online emissions from dust and sea salt + //-------------------------------------------------------------------- + + // --- Interpolate marine organics data -- + + // Update TimeState, note the addition of dt + morg_timeState_.t_now = ts.frac_of_year_in_days(); + + // Update time state and if the month has changed, update the data. + marineOrganicsFunc::update_marine_organics_timestate( + morg_dataReader_, ts, *morg_horizInterp_, + // output + morg_timeState_, morg_data_start_, morg_data_end_); + + // Call the main marine organics routine to get interpolated forcings. + marineOrganicsFunc::marineOrganics_main(morg_timeState_, morg_data_start_, + morg_data_end_, morg_data_out_); + + // Marine organics emission data read from the file (order is important here) + const const_view_1d mpoly = ekat::subview(morg_data_out_.emiss_sectors, 0); + const const_view_1d mprot = ekat::subview(morg_data_out_.emiss_sectors, 1); + const const_view_1d mlip = ekat::subview(morg_data_out_.emiss_sectors, 2); + + // Ocean fraction [unitless] + const const_view_1d ocnfrac = + get_field_in("ocnfrac").get_view(); + + // Sea surface temperature [K] + const const_view_1d sst = get_field_in("sst").get_view(); + + // U wind component [m/s] + const const_view_2d u_wind = + get_field_in("horiz_winds").get_component(0).get_view(); + + // V wind component [m/s] + const const_view_2d v_wind = + get_field_in("horiz_winds").get_component(1).get_view(); + + // Dust fluxes [kg/m^2/s]: Four flux values for each column + const const_view_2d dstflx = get_field_in("dstflx").get_view(); + + // Soil edodibility [fraction] + const const_view_1d soil_erodibility = this->soil_erodibility_; + + // Vertical layer height at midpoints + const const_view_2d z_mid = dry_atm_.z_mid; + + compute_online_dust_nacl_emiss(ncol_, nlev_, ocnfrac, sst, u_wind, v_wind, + dstflx, mpoly, mprot, mlip, soil_erodibility, + z_mid, + // output + constituent_fluxes); + Kokkos::fence(); + //-------------------------------------------------------------------- + // Interpolate srf emiss data read in from emissions files //-------------------------------------------------------------------- for(srf_emiss_ &ispec_srf : srf_emiss_species_) { @@ -256,20 +506,18 @@ void MAMSrfOnlineEmiss::run_impl(const double dt) { // modify units from molecules/cm2/s to kg/m2/s auto fluxes_in_mks_units = this->fluxes_in_mks_units_; - auto constituent_fluxes = this->constituent_fluxes_; const Real mfactor = amufac * mam4::gas_chemistry::adv_mass[species_index - offset_]; + const view_1d ispec_outdata0 = + ekat::subview(ispec_srf.data_out_.emiss_sectors, 0); // Parallel loop over all the columns to update units Kokkos::parallel_for( - "fluxes", ncol_, KOKKOS_LAMBDA(int icol) { - fluxes_in_mks_units(icol) = - ispec_srf.data_out_.emiss_sectors(0, icol) * mfactor; + "srf_emis_fluxes", ncol_, KOKKOS_LAMBDA(int icol) { + fluxes_in_mks_units(icol) = ispec_outdata0(icol) * mfactor; constituent_fluxes(icol, species_index) = fluxes_in_mks_units(icol); }); - } // for loop for species Kokkos::fence(); -} // run_imple ends - +} // run_impl ends // ============================================================================= } // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp index 031fb62d8b7..1a3bb4f36e3 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_srf_and_online_emissions_process_interface.hpp @@ -8,11 +8,12 @@ #include #include +// For reading marine organics file +#include + // For declaring surface and online emission class derived from atm process // class #include - -// #include #include namespace scream { @@ -20,19 +21,31 @@ namespace scream { // The process responsible for handling MAM4 surface and online emissions. The // AD stores exactly ONE instance of this class in its list of subcomponents. class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { - using KT = ekat::KokkosTypes; - using view_1d = typename KT::template view_1d; - using view_2d = typename KT::template view_2d; + using KT = ekat::KokkosTypes; + using view_1d = typename KT::template view_1d; + using view_2d = typename KT::template view_2d; + using const_view_1d = typename KT::template view_1d; + using const_view_2d = typename KT::template view_2d; // number of horizontal columns and vertical levels int ncol_, nlev_; + // Wet and dry states of atmosphere + mam_coupling::WetAtmosphere wet_atm_; + mam_coupling::DryAtmosphere dry_atm_; + // buffer for sotring temporary variables mam_coupling::Buffer buffer_; // physics grid for column information std::shared_ptr grid_; + // Sea surface temoerature [K] + const_view_1d sst_; + + // Dust fluxes (four values for each col) [kg/m2/s] + const_view_2d dust_fluxes_; + // Constituent fluxes of species in [kg/m2/s] view_2d constituent_fluxes_; @@ -42,8 +55,16 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // Unified atomic mass unit used for unit conversion (BAD constant) static constexpr Real amufac = 1.65979e-23; // 1.e4* kg / amu + // For reading soil erodibility file + std::shared_ptr serod_horizInterp_; + std::shared_ptr serod_dataReader_; + const_view_1d soil_erodibility_; + public: + // For reading surface emissions and marine organics file using srfEmissFunc = mam_coupling::srfEmissFunctions; + using marineOrganicsFunc = + marine_organics::marineOrganicsFunctions; // Constructor MAMSrfOnlineEmiss(const ekat::Comm &comm, const ekat::ParameterList ¶ms); @@ -80,13 +101,35 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { struct Preprocess { Preprocess() = default; // on host: initializes preprocess functor with necessary state data - void initialize(const view_2d &constituent_fluxes) { - constituent_fluxes_pre_ = constituent_fluxes; + void initialize(const int &ncol, const int &nlev, + const mam_coupling::WetAtmosphere &wet_atm, + const mam_coupling::DryAtmosphere &dry_atm) { + ncol_pre_ = ncol; + nlev_pre_ = nlev; + wet_atm_pre_ = wet_atm; + dry_atm_pre_ = dry_atm; } + KOKKOS_INLINE_FUNCTION + void operator()( + const Kokkos::TeamPolicy::member_type &team) const { + const int icol = team.league_rank(); // column index + + compute_dry_mixing_ratios(team, wet_atm_pre_, dry_atm_pre_, icol); + team.team_barrier(); + // vertical heights has to be computed after computing dry mixing ratios + // for atmosphere + compute_vertical_layer_heights(team, dry_atm_pre_, icol); + compute_updraft_velocities(team, wet_atm_pre_, dry_atm_pre_, icol); + } // Preprocess operator() + // local variables for preprocess struct - view_2d constituent_fluxes_pre_; - }; // MAMSrfOnlineEmiss::Preprocess + // number of horizontal columns and vertical levels + int ncol_pre_, nlev_pre_; + // local atmospheric and aerosol state data + mam_coupling::WetAtmosphere wet_atm_pre_; + mam_coupling::DryAtmosphere dry_atm_pre_; + }; // MAMSrfOnlineEmiss::Preprocess private: // preprocessing scratch pad Preprocess preprocess_; @@ -95,9 +138,11 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // FIXME: Remove the hardwired indices and use a function // to find them from an array. const std::map spcIndex_in_pcnst_ = { - {"so2", 12}, {"dms", 13}, {"so4_a1", 15}, - {"num_a1", 22}, {"so4_a2", 23}, {"num_a2", 27}, - {"pom_a4", 36}, {"bc_a4", 37}, {"num_a4", 39}}; + {"so2", 12}, {"dms", 13}, {"so4_a1", 15}, {"dst_a1", 19}, + {"ncl_a1", 20}, {"mom_a1", 21}, {"num_a1", 22}, {"so4_a2", 23}, + {"ncl_a2", 25}, {"mom_a2", 26}, {"num_a2", 27}, {"dst_a3", 28}, + {"ncl_a3", 29}, {"num_a3", 35}, {"pom_a4", 36}, {"bc_a4", 37}, + {"mom_a4", 38}, {"num_a4", 39}}; // A struct carrying all the fields needed to read // surface emissions of a species @@ -122,6 +167,13 @@ class MAMSrfOnlineEmiss final : public scream::AtmosphereProcess { // A vector for carrying emissions for all the species std::vector srf_emiss_species_; + // For reading marine organics file + std::shared_ptr morg_horizInterp_; + std::shared_ptr morg_dataReader_; + marineOrganicsFunc::marineOrganicsTimeState morg_timeState_; + marineOrganicsFunc::marineOrganicsInput morg_data_start_, morg_data_end_; + marineOrganicsFunc::marineOrganicsOutput morg_data_out_; + // offset for converting pcnst index to gas_pcnst index static constexpr int offset_ = mam4::aero_model::pcnst - mam4::gas_chemistry::gas_pcnst; diff --git a/components/eamxx/src/physics/mam/readfiles/find_season_index_utils.hpp b/components/eamxx/src/physics/mam/readfiles/find_season_index_utils.hpp new file mode 100644 index 00000000000..2e930cc65cb --- /dev/null +++ b/components/eamxx/src/physics/mam/readfiles/find_season_index_utils.hpp @@ -0,0 +1,75 @@ +#ifndef EAMXX_MAM_FIND_SEASON_INDEX_UTILS +#define EAMXX_MAM_FIND_SEASON_INDEX_UTILS + +#include +#include + +#include "share/io/scorpio_input.hpp" +#include "share/io/scream_scorpio_interface.hpp" + +namespace scream::mam_coupling { + +// views for single- and multi-column data + +using const_view_1d = typename KT::template view_1d; +using view_int_2d = typename KT::template view_2d; + +using view_1d_host = typename KT::view_1d::HostMirror; +using view_int_3d_host = typename KT::view_3d::HostMirror; +using view_int_2d_host = typename KT::view_2d::HostMirror; + +/** + * @brief Reads the season index from the given file and computes the season + * indices based on latitudes. + * + * @param[in] season_wes_file The path to the season_wes.nc file. + * @param[in] clat A 1D view of latitude values in degrees. + * @param[out] index_season_lai A 2D view to store the computed season indices. + * Note that indices are in C++ (starting from zero). + */ + +inline void find_season_index_reader(const std::string &season_wes_file, + const const_view_1d &clat, + view_int_2d &index_season_lai) { + const int plon = clat.extent(0); + scorpio::register_file(season_wes_file, scorpio::Read); + + const int nlat_lai = scorpio::get_dimlen(season_wes_file, "lat"); + const int npft_lai = scorpio::get_dimlen(season_wes_file, "pft"); + + view_1d_host lat_lai("lat_lai", nlat_lai); + view_int_2d_host wk_lai_temp("wk_lai", npft_lai, nlat_lai); + view_int_3d_host wk_lai("wk_lai", nlat_lai, npft_lai, 12); + + scorpio::read_var(season_wes_file, "lat", lat_lai.data()); + + Kokkos::MDRangePolicy> + policy_wk_lai({0, 0}, {nlat_lai, npft_lai}); + + // loop over time to get all 12 instantence of season_wes + for(int itime = 0; itime < 12; ++itime) { + scorpio::read_var(season_wes_file, "season_wes", wk_lai_temp.data(), itime); + // copy data from wk_lai_temp to wk_lai. + // NOTE: season_wes has different layout that wk_lai + Kokkos::parallel_for("copy_to_wk_lai", policy_wk_lai, + [&](const int j, const int k) { + wk_lai(j, k, itime) = wk_lai_temp(k, j); + }); + Kokkos::fence(); + } + scorpio::release_file(season_wes_file); + + index_season_lai = view_int_2d("index_season_lai", plon, 12); + const view_int_2d_host index_season_lai_host = + Kokkos::create_mirror_view(index_season_lai); + + const view_1d_host clat_host = Kokkos::create_mirror_view(clat); + Kokkos::deep_copy(clat_host, clat); + + // Computation is performed on the host + mam4::mo_drydep::find_season_index(clat_host, lat_lai, nlat_lai, wk_lai, + index_season_lai_host); + Kokkos::deep_copy(index_season_lai, index_season_lai_host); +} +} // namespace scream::mam_coupling +#endif // diff --git a/components/eamxx/src/physics/mam/readfiles/marine_organics.hpp b/components/eamxx/src/physics/mam/readfiles/marine_organics.hpp new file mode 100644 index 00000000000..a04dff129f4 --- /dev/null +++ b/components/eamxx/src/physics/mam/readfiles/marine_organics.hpp @@ -0,0 +1,133 @@ +#ifndef MARINE_ORGANICS_HPP +#define MARINE_ORGANICS_HPP + +// For AtmosphereInput +#include "share/io/scorpio_input.hpp" + +namespace scream { +namespace marine_organics { + +template +struct marineOrganicsFunctions { + using Device = DeviceType; + + using KT = KokkosTypes; + using MemberType = typename KT::MemberType; + using view_2d = typename KT::template view_2d; + + // ------------------------------------------------------------------------------------------- + struct marineOrganicsTimeState { + marineOrganicsTimeState() = default; + // Whether the timestate has been initialized. + // The current month + int current_month = -1; + // Julian Date for the beginning of the month, as defined in + // /src/share/util/scream_time_stamp.hpp + // See this file for definition of Julian Date. + Real t_beg_month; + // Current simulation Julian Date + Real t_now; + // Number of days in the current month, cast as a Real + Real days_this_month; + }; // marineOrganicsTimeState + + struct marineOrganicsData { + marineOrganicsData() = default; + marineOrganicsData(const int &ncol_, const int &nfields_) + : ncols(ncol_), nsectors(nfields_) { + init(ncols, nsectors, true); + } + + void init(const int &ncol, const int &nsector, const bool allocate) { + ncols = ncol; + nsectors = nsector; + if(allocate) emiss_sectors = view_2d("morgAllSectors", nsectors, ncols); + } // marineOrganicsData init + + // Basic spatial dimensions of the data + int ncols, nsectors; + view_2d emiss_sectors; + }; // marineOrganicsData + + // ------------------------------------------------------------------------------------------- + struct marineOrganicsInput { + marineOrganicsInput() = default; + marineOrganicsInput(const int &ncols_, const int &nfields_) { + init(ncols_, nfields_); + } + + void init(const int &ncols_, const int &nfields_) { + data.init(ncols_, nfields_, true); + } + marineOrganicsData data; // All marineOrganics fields + }; // marineOrganicsInput + + // The output is really just marineOrganicsData, but for clarity it might + // help to see a marineOrganicsOutput along a marineOrganicsInput in functions + // signatures + using marineOrganicsOutput = marineOrganicsData; + + // ------------------------------------------------------------------------------------------- + static std::shared_ptr create_horiz_remapper( + const std::shared_ptr &model_grid, + const std::string &marineOrganics_data_file, const std::string &map_file, + const std::vector &field_name, const std::string &dim_name1); + + // ------------------------------------------------------------------------------------------- + static std::shared_ptr create_data_reader( + const std::shared_ptr &horiz_remapper, + const std::string &data_file); + + // ------------------------------------------------------------------------------------------- + static void update_marine_organics_data_from_file( + std::shared_ptr &scorpio_reader, + const util::TimeStamp &ts, + const int &time_index, // zero-based + AbstractRemapper &horiz_interp, + marineOrganicsInput &marineOrganics_input); + + // ------------------------------------------------------------------------------------------- + static void update_marine_organics_timestate( + std::shared_ptr &scorpio_reader, + const util::TimeStamp &ts, AbstractRemapper &horiz_interp, + marineOrganicsTimeState &time_state, marineOrganicsInput &beg, + marineOrganicsInput &end); + + // ------------------------------------------------------------------------------------------- + static void marineOrganics_main(const marineOrganicsTimeState &time_state, + const marineOrganicsInput &data_beg, + const marineOrganicsInput &data_end, + const marineOrganicsOutput &data_out); + + // ------------------------------------------------------------------------------------------- + static void perform_time_interpolation( + const marineOrganicsTimeState &time_state, + const marineOrganicsInput &data_beg, const marineOrganicsInput &data_end, + const marineOrganicsOutput &data_out); + + // ------------------------------------------------------------------------------------------- + // Performs convex interpolation of x0 and x1 at point t + template + KOKKOS_INLINE_FUNCTION static ScalarX linear_interp(const ScalarX &x0, + const ScalarX &x1, + const ScalarT &t); + + // ------------------------------------------------------------------------------------------- + static void init_marine_organics_file_read( + const int &ncol, const std::vector &field_name, + const std::string &dim_name1, + const std::shared_ptr &grid, + const std::string &data_file, const std::string &mapping_file, + // output + std::shared_ptr &marineOrganicsHorizInterp, + marineOrganicsInput &morg_data_start_, + marineOrganicsInput &morg_data_end_, marineOrganicsData &morg_data_out_, + std::shared_ptr &marineOrganicsDataReader); + +}; // struct marineOrganicsFunctions + +} // namespace marine_organics +} // namespace scream +#endif // MARINE_ORGANICS_HPP + +#include "marine_organics_impl.hpp" \ No newline at end of file diff --git a/components/eamxx/src/physics/mam/readfiles/marine_organics_impl.hpp b/components/eamxx/src/physics/mam/readfiles/marine_organics_impl.hpp new file mode 100644 index 00000000000..389445fa024 --- /dev/null +++ b/components/eamxx/src/physics/mam/readfiles/marine_organics_impl.hpp @@ -0,0 +1,296 @@ +#ifndef MARINE_ORGANICS_IMPL_HPP +#define MARINE_ORGANICS_IMPL_HPP + +#include "share/grid/remap/identity_remapper.hpp" +#include "share/grid/remap/refining_remapper_p2p.hpp" +#include "share/io/scream_scorpio_interface.hpp" +#include "share/util/scream_timing.hpp" + +namespace scream { +namespace marine_organics { + +template +std::shared_ptr +marineOrganicsFunctions::create_horiz_remapper( + const std::shared_ptr &model_grid, + const std::string &data_file, const std::string &map_file, + const std::vector &field_name, const std::string &dim_name1) { + using namespace ShortFieldTagsNames; + + scorpio::register_file(data_file, scorpio::Read); + const int ncols_data = scorpio::get_dimlen(data_file, dim_name1); + + scorpio::release_file(data_file); + + // Since shallow clones are cheap, we may as well do it (less lines of + // code) + auto horiz_interp_tgt_grid = + model_grid->clone("marine_organics_horiz_interp_tgt_grid", true); + + const int ncols_model = model_grid->get_num_global_dofs(); + std::shared_ptr remapper; + if(ncols_data == ncols_model) { + remapper = std::make_shared( + horiz_interp_tgt_grid, IdentityRemapper::SrcAliasTgt); + } else { + EKAT_REQUIRE_MSG(ncols_data <= ncols_model, + "Error! We do not allow to coarsen marine organics " + "data to fit the model. We only allow\n" + " marine organics data to be at the same or " + "coarser resolution as the model.\n"); + // We must have a valid map file + EKAT_REQUIRE_MSG(map_file != "", + "ERROR: marine organics data is on a different grid " + "than the model one,\n" + " but remap file is missing from marine organics " + "parameter list."); + + remapper = + std::make_shared(horiz_interp_tgt_grid, map_file); + } + + remapper->registration_begins(); + + const auto tgt_grid = remapper->get_tgt_grid(); + + const auto layout_2d = tgt_grid->get_2d_scalar_layout(); + using namespace ekat::units; + using namespace ekat::prefixes; + Units umolC(micro * mol, "umol C"); + + std::vector fields_vector; + + const int field_size = field_name.size(); + for(int icomp = 0; icomp < field_size; ++icomp) { + auto comp_name = field_name[icomp]; + // set and allocate fields + Field f(FieldIdentifier(comp_name, layout_2d, umolC, tgt_grid->name())); + f.allocate_view(); + fields_vector.push_back(f); + remapper->register_field_from_tgt(f); + } + + remapper->registration_ends(); + + return remapper; + +} // create_horiz_remapper + +// ------------------------------------------------------------------------------------------- +template +std::shared_ptr +marineOrganicsFunctions::create_data_reader( + const std::shared_ptr &horiz_remapper, + const std::string &data_file) { + std::vector io_fields; + for(int ifld = 0; ifld < horiz_remapper->get_num_fields(); ++ifld) { + io_fields.push_back(horiz_remapper->get_src_field(ifld)); + } + const auto io_grid = horiz_remapper->get_src_grid(); + return std::make_shared(data_file, io_grid, io_fields, true); +} // create_data_reader + +// ------------------------------------------------------------------------------------------- +template +void marineOrganicsFunctions::update_marine_organics_data_from_file( + std::shared_ptr &scorpio_reader, const util::TimeStamp &ts, + const int &time_index, // zero-based + AbstractRemapper &horiz_interp, marineOrganicsInput &marineOrganics_input) { + start_timer("EAMxx::marineOrganics::update_marine_organics_data_from_file"); + + // 1. Read from file + start_timer( + "EAMxx::marineOrganics::update_marine_organics_data_from_file::read_" + "data"); + scorpio_reader->read_variables(); + stop_timer( + "EAMxx::marineOrganics::update_marine_organics_data_from_file::read_" + "data"); + + // 2. Run the horiz remapper (it is a do-nothing op if marineOrganics data is + // on same grid as model) + start_timer( + "EAMxx::marineOrganics::update_marine_organics_data_from_file::horiz_" + "remap"); + horiz_interp.remap(/*forward = */ true); + stop_timer( + "EAMxx::marineOrganics::update_marine_organics_data_from_file::horiz_" + "remap"); + + // 3. Get the tgt field of the remapper + start_timer( + "EAMxx::marineOrganics::update_marine_organics_data_from_file::get_" + "field"); + // Recall, the fields are registered in the order: + // Read the field from the file + + for(int ifld = 0; ifld < horiz_interp.get_num_fields(); ++ifld) { + auto sector = horiz_interp.get_tgt_field(ifld).get_view(); + const auto emiss = Kokkos::subview(marineOrganics_input.data.emiss_sectors, + ifld, Kokkos::ALL()); + Kokkos::deep_copy(emiss, sector); + } + + Kokkos::fence(); + + stop_timer( + "EAMxx::marineOrganics::update_marine_organics_data_from_file::get_" + "field"); + + stop_timer("EAMxx::marineOrganics::update_marine_organics_data_from_file"); + +} // END update_marine_organics_data_from_file + +// ------------------------------------------------------------------------------------------- +template +void marineOrganicsFunctions::update_marine_organics_timestate( + std::shared_ptr &scorpio_reader, const util::TimeStamp &ts, + AbstractRemapper &horiz_interp, marineOrganicsTimeState &time_state, + marineOrganicsInput &beg, marineOrganicsInput &end) { + // Now we check if we have to update the data that changes monthly + // NOTE: This means that marineOrganics assumes monthly data to update. Not + // any other frequency. + const auto month = ts.get_month() - 1; // Make it 0-based + if(month != time_state.current_month) { + // Update the marineOrganics time state information + time_state.current_month = month; + time_state.t_beg_month = + util::TimeStamp({ts.get_year(), month + 1, 1}, {0, 0, 0}) + .frac_of_year_in_days(); + time_state.days_this_month = util::days_in_month(ts.get_year(), month + 1); + + // Copy end'data into beg'data, and read in the new + // end + std::swap(beg, end); + + // Update the marineOrganics forcing data for this month and next month + // Start by copying next months data to this months data structure. + // NOTE: If the timestep is bigger than monthly this could cause the wrong + // values + // to be assigned. A timestep greater than a month is very unlikely + // so we will proceed. + int next_month = (time_state.current_month + 1) % 12; + update_marine_organics_data_from_file(scorpio_reader, ts, next_month, + horiz_interp, end); + } + +} // END updata_marine_organics_timestate + +// ------------------------------------------------------------------------------------------- +template +template +KOKKOS_INLINE_FUNCTION ScalarX marineOrganicsFunctions::linear_interp( + const ScalarX &x0, const ScalarX &x1, const ScalarT &t) { + return (1 - t) * x0 + t * x1; +} // linear_interp + +// ------------------------------------------------------------------------------------------- +template +void marineOrganicsFunctions::perform_time_interpolation( + const marineOrganicsTimeState &time_state, + const marineOrganicsInput &data_beg, const marineOrganicsInput &data_end, + const marineOrganicsOutput &data_out) { + using ExeSpace = typename KT::ExeSpace; + using ESU = ekat::ExeSpaceUtils; + + // Gather time stamp info + auto &t_now = time_state.t_now; + auto &t_beg = time_state.t_beg_month; + auto &delta_t = time_state.days_this_month; + + // At this stage, begin/end must have the same dimensions + EKAT_REQUIRE(data_end.data.ncols == data_beg.data.ncols); + + auto delta_t_fraction = (t_now - t_beg) / delta_t; + + EKAT_REQUIRE_MSG(delta_t_fraction >= 0 && delta_t_fraction <= 1, + "Error! Convex interpolation with coefficient out of " + "[0,1].\n t_now : " + + std::to_string(t_now) + + "\n" + " t_beg : " + + std::to_string(t_beg) + + "\n delta_t: " + std::to_string(delta_t) + "\n"); + + const int nsectors = data_beg.data.nsectors; + const int ncols = data_beg.data.ncols; + using ExeSpace = typename KT::ExeSpace; + using ESU = ekat::ExeSpaceUtils; + const auto policy = ESU::get_default_team_policy(ncols, nsectors); + + Kokkos::parallel_for( + policy, KOKKOS_LAMBDA(const MemberType &team) { + const int icol = team.league_rank(); // column index + Kokkos::parallel_for( + Kokkos::TeamVectorRange(team, 0u, nsectors), [&](int isec) { + const auto beg = data_beg.data.emiss_sectors(isec, icol); + const auto end = data_end.data.emiss_sectors(isec, icol); + data_out.emiss_sectors(isec, icol) = + linear_interp(beg, end, delta_t_fraction); + }); + }); + Kokkos::fence(); + +} // perform_time_interpolation + +// ------------------------------------------------------------------------------------------- +template +void marineOrganicsFunctions::marineOrganics_main( + const marineOrganicsTimeState &time_state, + const marineOrganicsInput &data_beg, const marineOrganicsInput &data_end, + const marineOrganicsOutput &data_out) { + // Beg/End/Tmp month must have all sizes matching + + EKAT_REQUIRE_MSG( + data_end.data.ncols == data_beg.data.ncols, + "Error! marineOrganicsInput data structs must have the same number of " + "columns.\n"); + + // Horiz interpolation can be expensive, and does not depend on the particular + // time of the month, so it can be done ONCE per month, *outside* + // marineOrganics_main (when updating the beg/end states, reading them from + // file). + EKAT_REQUIRE_MSG(data_end.data.ncols == data_out.ncols, + "Error! Horizontal interpolation is performed *before* " + "calling marineOrganics_main,\n" + " marineOrganicsInput and marineOrganicsOutput data " + "structs must have the " + "same number columns " + << data_end.data.ncols << " " << data_out.ncols + << ".\n"); + + // Step 1. Perform time interpolation + perform_time_interpolation(time_state, data_beg, data_end, data_out); +} // marineOrganics_main + +// ------------------------------------------------------------------------------------------- +template +void marineOrganicsFunctions::init_marine_organics_file_read( + const int &ncol, const std::vector &field_name, + const std::string &dim_name1, + const std::shared_ptr &grid, + const std::string &data_file, const std::string &mapping_file, + // output + std::shared_ptr &marineOrganicsHorizInterp, + marineOrganicsInput &data_start_, marineOrganicsInput &data_end_, + marineOrganicsData &data_out_, + std::shared_ptr &marineOrganicsDataReader) { + // Init horizontal remap + + marineOrganicsHorizInterp = create_horiz_remapper( + grid, data_file, mapping_file, field_name, dim_name1); + + // Initialize the size of start/end/out data structures + data_start_ = marineOrganicsInput(ncol, field_name.size()); + data_end_ = marineOrganicsInput(ncol, field_name.size()); + data_out_.init(ncol, field_name.size(), true); + + // Create reader (an AtmosphereInput object) + marineOrganicsDataReader = + create_data_reader(marineOrganicsHorizInterp, data_file); + +} // init_marine_organics_file_read +} // namespace marine_organics +} // namespace scream + +#endif // MARINE_ORGANICS_IMPL_HPP \ No newline at end of file diff --git a/components/eamxx/src/physics/mam/readfiles/soil_erodibility.hpp b/components/eamxx/src/physics/mam/readfiles/soil_erodibility.hpp new file mode 100644 index 00000000000..8b47c81d907 --- /dev/null +++ b/components/eamxx/src/physics/mam/readfiles/soil_erodibility.hpp @@ -0,0 +1,44 @@ +#ifndef SOIL_ERODIBILITY_HPP +#define SOIL_ERODIBILITY_HPP + +// For AtmosphereInput +#include "share/io/scorpio_input.hpp" + +namespace scream { +namespace soil_erodibility { + +template +struct soilErodibilityFunctions { + using Device = DeviceType; + + using KT = KokkosTypes; + using const_view_1d = typename KT::template view_1d; + + static std::shared_ptr create_horiz_remapper( + const std::shared_ptr &model_grid, + const std::string &soilErodibility_data_file, const std::string &map_file, + const std::string &field_name, const std::string &dim_name1); + + static std::shared_ptr create_data_reader( + const std::shared_ptr &horiz_remapper, + const std::string &data_file); + + static void update_soil_erodibility_data_from_file( + std::shared_ptr &scorpio_reader, + AbstractRemapper &horiz_interp, const_view_1d &input); + + static void init_soil_erodibility_file_read( + const int ncol, const std::string field_name, const std::string dim_name1, + const std::shared_ptr &grid, + const std::string &data_file, const std::string &mapping_file, + // output + std::shared_ptr &SoilErodibilityHorizInterp, + std::shared_ptr &SoilErodibilityDataReader); + +}; // struct soilErodilityFunctions + +} // namespace soil_erodibility +} // namespace scream +#endif // SOIL_ERODIBILITY_HPP + +#include "soil_erodibility_impl.hpp" \ No newline at end of file diff --git a/components/eamxx/src/physics/mam/readfiles/soil_erodibility_impl.hpp b/components/eamxx/src/physics/mam/readfiles/soil_erodibility_impl.hpp new file mode 100644 index 00000000000..af0c4d73c17 --- /dev/null +++ b/components/eamxx/src/physics/mam/readfiles/soil_erodibility_impl.hpp @@ -0,0 +1,147 @@ +#ifndef SOIL_ERODIBILITY_IMPL_HPP +#define SOIL_ERODIBILITY_IMPL_HPP + +#include "share/grid/remap/identity_remapper.hpp" +#include "share/grid/remap/refining_remapper_p2p.hpp" +#include "share/io/scream_scorpio_interface.hpp" +#include "share/util/scream_timing.hpp" + +namespace scream { +namespace soil_erodibility { + +template +std::shared_ptr +soilErodibilityFunctions::create_horiz_remapper( + const std::shared_ptr &model_grid, + const std::string &data_file, const std::string &map_file, + const std::string &field_name, const std::string &dim_name1) { + using namespace ShortFieldTagsNames; + + scorpio::register_file(data_file, scorpio::Read); + const int ncols_data = scorpio::get_dimlen(data_file, dim_name1); + + scorpio::release_file(data_file); + + // We could use model_grid directly if using same num levels, + // but since shallow clones are cheap, we may as well do it (less lines of + // code) + auto horiz_interp_tgt_grid = + model_grid->clone("soil_erodibility_horiz_interp_tgt_grid", true); + + const int ncols_model = model_grid->get_num_global_dofs(); + std::shared_ptr remapper; + if(ncols_data == ncols_model) { + remapper = std::make_shared( + horiz_interp_tgt_grid, IdentityRemapper::SrcAliasTgt); + } else { + EKAT_REQUIRE_MSG(ncols_data <= ncols_model, + "Error! We do not allow to coarsen soil erodibility " + "data to fit the model. We only allow\n" + " soil erodibility data to be at the same or " + "coarser resolution as the model.\n"); + // We must have a valid map file + EKAT_REQUIRE_MSG(map_file != "", + "ERROR: soil erodibility data is on a different grid " + "than the model one,\n" + " but remap file is missing from soil erodibility " + "parameter list."); + + remapper = + std::make_shared(horiz_interp_tgt_grid, map_file); + } + + remapper->registration_begins(); + + const auto tgt_grid = remapper->get_tgt_grid(); + + const auto layout_2d = tgt_grid->get_2d_scalar_layout(); + const auto nondim = ekat::units::Units::nondimensional(); + + Field soil_erodibility( + FieldIdentifier(field_name, layout_2d, nondim, tgt_grid->name())); + soil_erodibility.allocate_view(); + + remapper->register_field_from_tgt(soil_erodibility); + + remapper->registration_ends(); + + return remapper; + +} // create_horiz_remapper + +// ------------------------------------------------------------------------------------------- +template +std::shared_ptr +soilErodibilityFunctions::create_data_reader( + const std::shared_ptr &horiz_remapper, + const std::string &data_file) { + std::vector io_fields; + for(int i = 0; i < horiz_remapper->get_num_fields(); ++i) { + io_fields.push_back(horiz_remapper->get_src_field(i)); + } + const auto io_grid = horiz_remapper->get_src_grid(); + return std::make_shared(data_file, io_grid, io_fields, true); +} // create_data_reader + +// ------------------------------------------------------------------------------------------- +template +void soilErodibilityFunctions::update_soil_erodibility_data_from_file( + std::shared_ptr &scorpio_reader, + AbstractRemapper &horiz_interp, const_view_1d &input) { + start_timer("EAMxx::soilErodibility::update_soil_erodibility_data_from_file"); + + // 1. Read from file + start_timer( + "EAMxx::soilErodibility::update_soil_erodibility_data_from_file::read_" + "data"); + scorpio_reader->read_variables(); + stop_timer( + "EAMxx::soilErodibility::update_soil_erodibility_data_from_file::read_" + "data"); + + // 2. Run the horiz remapper (it is a do-nothing op if soilErodibility data is + // on same grid as model) + start_timer( + "EAMxx::soilErodibility::update_soil_erodibility_data_from_file::horiz_" + "remap"); + horiz_interp.remap(/*forward = */ true); + stop_timer( + "EAMxx::soilErodibility::update_soil_erodibility_data_from_file::horiz_" + "remap"); + + // 3. Get the tgt field of the remapper + start_timer( + "EAMxx::soilErodibility::update_soil_erodibility_data_from_file::get_" + "field"); + // Recall, the fields are registered in the order: + // Read the field from the file + input = horiz_interp.get_tgt_field(0).get_view(); + stop_timer( + "EAMxx::soilErodibility::update_soil_erodibility_data_from_file::get_" + "field"); + + stop_timer("EAMxx::soilErodibility::update_soil_erodibility_data_from_file"); + +} // END update_soil_erodibility_data_from_file + +// ------------------------------------------------------------------------------------------- +template +void soilErodibilityFunctions::init_soil_erodibility_file_read( + const int ncol, const std::string field_name, const std::string dim_name1, + const std::shared_ptr &grid, + const std::string &data_file, const std::string &mapping_file, + // output + std::shared_ptr &soilErodibilityHorizInterp, + std::shared_ptr &soilErodibilityDataReader) { + // Init horizontal remap + soilErodibilityHorizInterp = create_horiz_remapper( + grid, data_file, mapping_file, field_name, dim_name1); + + // Create reader (an AtmosphereInput object) + soilErodibilityDataReader = + create_data_reader(soilErodibilityHorizInterp, data_file); +} // init_soil_erodibility_file_read +} // namespace soil_erodibility +} // namespace scream + +#endif // SOIL_ERODIBILITY_IMPL_HPP diff --git a/components/eamxx/src/physics/mam/readfiles/tracer_reader_utils.hpp b/components/eamxx/src/physics/mam/readfiles/tracer_reader_utils.hpp index 2e34db2b496..c605d3a0b9f 100644 --- a/components/eamxx/src/physics/mam/readfiles/tracer_reader_utils.hpp +++ b/components/eamxx/src/physics/mam/readfiles/tracer_reader_utils.hpp @@ -66,12 +66,19 @@ struct ForcingHelper { }; enum TracerFileType { - // file with PS ncol, lev, and time + // file with ncol, lev, ilev, time and has P0 and PS as variables + // example: oxidants FORMULA_PS, - // nc zonal file from ncremap + // file with ncol, lev, ilev, time + // example: linoz ZONAL, - // vertical emission files - VERT_EMISSION, + // file with ncol, altitude, altitude_int, time + // example: elevated (at a height) emissions of aerosols and precursors + // NOTE: we must rename the default vertical tags when horiz remapping + // NOTE: we vert remap in a different routine in mam4xx + ELEVATED_EMISSIONS, + // Placeholder for cases where no file type is applicable + NONE }; enum TracerDataIndex { BEG = 0, END = 1, OUT = 2 }; @@ -81,7 +88,7 @@ enum TracerDataIndex { BEG = 0, END = 1, OUT = 2 }; Therefore, if a file contains more than this number, it is acceptable to increase this limit. Currently, Linoz files have 8 fields. */ constexpr int MAX_NVARS_TRACER = 10; -constexpr int MAX_NUM_VERT_EMISSION_FIELDS = 25; +constexpr int MAX_NUM_ELEVATED_EMISSIONS_FIELDS = 25; // Linoz structures to help manage all of the variables: struct TracerTimeState { @@ -300,7 +307,7 @@ inline void setup_tracer_data(TracerData &tracer_data, // out // This type of files use altitude (zi) for vertical interpolation if(has_altitude) { nlevs_data = scorpio::get_dimlen(trace_data_file, "altitude"); - tracer_file_type = VERT_EMISSION; + tracer_file_type = ELEVATED_EMISSIONS; } EKAT_REQUIRE_MSG( nlevs_data != -1, @@ -332,7 +339,7 @@ inline void setup_tracer_data(TracerData &tracer_data, // out tracer_data.zonal_levs_ = levs; } - if(tracer_file_type == VERT_EMISSION) { + if(tracer_file_type == ELEVATED_EMISSIONS) { const int nilevs_data = scorpio::get_dimlen(trace_data_file, "altitude_int"); view_1d_host altitude_int_host("altitude_int_host", nilevs_data); @@ -381,11 +388,6 @@ inline std::shared_ptr create_horiz_remapper( model_grid->clone("tracer_horiz_interp_tgt_grid", true); horiz_interp_tgt_grid->reset_num_vertical_lev(tracer_data.nlevs_data); - if(tracer_data.file_type == VERT_EMISSION) { - horiz_interp_tgt_grid->reset_field_tag_name(LEV, "altitude"); - horiz_interp_tgt_grid->reset_field_tag_name(ILEV, "altitude_int"); - } - const int ncols_model = model_grid->get_num_global_dofs(); std::shared_ptr remapper; if(tracer_data.ncols_data == ncols_model) { @@ -438,14 +440,31 @@ inline std::shared_ptr create_horiz_remapper( inline std::shared_ptr create_tracer_data_reader( const std::shared_ptr &horiz_remapper, - const std::string &tracer_data_file) { + const std::string &tracer_data_file, + const TracerFileType file_type = NONE) +{ std::vector io_fields; for(int i = 0; i < horiz_remapper->get_num_fields(); ++i) { io_fields.push_back(horiz_remapper->get_src_field(i)); } const auto io_grid = horiz_remapper->get_src_grid(); - return std::make_shared(tracer_data_file, io_grid, io_fields, + if(file_type == ELEVATED_EMISSIONS ){ + // NOTE: If we are using a vertical emission nc file with altitude instead of lev, + // we must rename this tag. + // We need to perform a shallow clone of io_grid because tags are const in this object. + auto horiz_interp_src_grid = + io_grid->clone("tracer_horiz_interp_src_grid", true); + horiz_interp_src_grid->reset_field_tag_name(LEV, "altitude"); + horiz_interp_src_grid->reset_field_tag_name(ILEV, "altitude_int"); + return std::make_shared(tracer_data_file, horiz_interp_src_grid, io_fields, + true); + } else{ + // We do not need to rename tags in or clone io_grid for other types of files. + return std::make_shared(tracer_data_file, io_grid, io_fields, true); + } + + } // create_tracer_data_reader inline void update_tracer_data_from_file( @@ -653,7 +672,7 @@ inline void perform_vertical_interpolation(const const_view_1d &altitude_int, const TracerData &input, const view_2d output[]) { EKAT_REQUIRE_MSG( - input.file_type == VERT_EMISSION, + input.file_type == ELEVATED_EMISSIONS, "Error! vertical interpolation only with altitude variable. \n"); const int ncols = input.ncol_; const int num_vars = input.nvars_; @@ -730,7 +749,7 @@ inline void advance_tracer_data( if(data_tracer.file_type == FORMULA_PS || data_tracer.file_type == ZONAL) { perform_vertical_interpolation(data_tracer.p_src_, p_tgt, data_tracer, output); - } else if(data_tracer.file_type == VERT_EMISSION) { + } else if(data_tracer.file_type == ELEVATED_EMISSIONS) { perform_vertical_interpolation(data_tracer.altitude_int_, zi_tgt, data_tracer, output); } diff --git a/components/eamxx/src/physics/mam/srf_emission.hpp b/components/eamxx/src/physics/mam/srf_emission.hpp index 29aaca421ea..8dc5be1d05a 100644 --- a/components/eamxx/src/physics/mam/srf_emission.hpp +++ b/components/eamxx/src/physics/mam/srf_emission.hpp @@ -4,8 +4,6 @@ #include "share/util/scream_timing.hpp" namespace scream::mam_coupling { -namespace { - template struct srfEmissFunctions { using Device = DeviceType; @@ -131,7 +129,6 @@ struct srfEmissFunctions { std::shared_ptr &SrfEmissDataReader); }; // struct srfEmissFunctions -} // namespace } // namespace scream::mam_coupling #endif // SRF_EMISSION_HPP diff --git a/components/eamxx/src/physics/mam/srf_emission_impl.hpp b/components/eamxx/src/physics/mam/srf_emission_impl.hpp index 48dc1fa7087..b8ebfdbe501 100644 --- a/components/eamxx/src/physics/mam/srf_emission_impl.hpp +++ b/components/eamxx/src/physics/mam/srf_emission_impl.hpp @@ -6,8 +6,6 @@ #include "share/io/scream_scorpio_interface.hpp" namespace scream::mam_coupling { -namespace { - template std::shared_ptr srfEmissFunctions::create_horiz_remapper( @@ -201,11 +199,6 @@ void srfEmissFunctions::update_srfEmiss_data_from_file( // Recall, the fields are registered in the order: ps, ccn3, g_sw, ssa_sw, // tau_sw, tau_lw - const auto &layout = srfEmiss_horiz_interp.get_tgt_field(0) - .get_header() - .get_identifier() - .get_layout(); - // Read fields from the file for(int i = 0; i < srfEmiss_horiz_interp.get_num_fields(); ++i) { auto sector = @@ -279,8 +272,6 @@ void srfEmissFunctions::init_srf_emiss_objects( SrfEmissDataReader = create_srfEmiss_data_reader(SrfEmissHorizInterp, data_file); } // init_srf_emiss_objects - -} // namespace } // namespace scream::mam_coupling #endif // SRF_EMISSION_IMPL_HPP diff --git a/components/eamxx/src/physics/p3/CMakeLists.txt b/components/eamxx/src/physics/p3/CMakeLists.txt index a1dfc946267..58a026e1601 100644 --- a/components/eamxx/src/physics/p3/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/CMakeLists.txt @@ -1,19 +1,10 @@ set(P3_SRCS - p3_f90.cpp - p3_ic_cases.cpp p3_iso_c.f90 ${SCREAM_BASE_DIR}/../eam/src/physics/p3/scream/micro_p3.F90 eamxx_p3_process_interface.cpp eamxx_p3_run.cpp ) -if (NOT SCREAM_LIB_ONLY) - list(APPEND P3_SRCS - p3_functions_f90.cpp - p3_main_wrap.cpp - ) # Add f90 bridges needed for testing -endif() - # Add ETI source files if not on CUDA/HIP if (NOT EAMXX_ENABLE_GPU OR Kokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE OR Kokkos_ENABLE_HIP_RELOCATABLE_DEVICE_CODE) list(APPEND P3_SRCS @@ -27,6 +18,7 @@ if (NOT EAMXX_ENABLE_GPU OR Kokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE OR Kokkos eti/p3_table_ice.cpp eti/p3_dsd2.cpp eti/p3_find.cpp + eti/p3_init.cpp eti/p3_update_prognostics.cpp eti/p3_get_time_space_phys_variables.cpp eti/p3_autoconversion.cpp @@ -77,6 +69,8 @@ if (SCREAM_P3_SMALL_KERNELS) add_library(p3 ${P3_SRCS} ${P3_SK_SRCS}) else() add_library(p3 ${P3_SRCS}) + # If small kernels are ON, we don't need a separate executable to test them. + # Also, we never want to generate baselines with this separate executable if (NOT SCREAM_LIBS_ONLY AND NOT SCREAM_ONLY_GENERATE_BASELINES) add_library(p3_sk ${P3_SRCS} ${P3_SK_SRCS}) # Always build p3_sk with SCREAM_P3_SMALL_KERNELS on @@ -123,11 +117,6 @@ endforeach() add_executable(p3_tables_setup EXCLUDE_FROM_ALL p3_tables_setup.cpp) target_link_libraries(p3_tables_setup p3) -#crusher change -if (Kokkos_ENABLE_HIP) -set_source_files_properties(p3_functions_f90.cpp PROPERTIES COMPILE_FLAGS -O0) -endif() - if (NOT SCREAM_LIB_ONLY) add_subdirectory(tests) endif() diff --git a/components/eamxx/src/physics/p3/disp/p3_check_values_impl_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_check_values_impl_disp.cpp index cc13e99bc91..5bac8306477 100644 --- a/components/eamxx/src/physics/p3/disp/p3_check_values_impl_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_check_values_impl_disp.cpp @@ -16,7 +16,7 @@ ::check_values_disp(const uview_2d& qv, const uview_2d using ExeSpace = typename KT::ExeSpace; const Int nk_pack = ekat::npack(nk); const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(nj, nk_pack); - + Kokkos::parallel_for( "p3_check_values", policy, KOKKOS_LAMBDA(const MemberType& team) { @@ -32,4 +32,3 @@ ::check_values_disp(const uview_2d& qv, const uview_2d } // namespace p3 } // namespace scream - diff --git a/components/eamxx/src/physics/p3/disp/p3_cloud_sed_impl_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_cloud_sed_impl_disp.cpp index 8b755be4857..4b528663d23 100644 --- a/components/eamxx/src/physics/p3/disp/p3_cloud_sed_impl_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_cloud_sed_impl_disp.cpp @@ -48,7 +48,7 @@ ::cloud_sedimentation_disp( } cloud_sedimentation( - ekat::subview(qc_incld, i), ekat::subview(rho, i), ekat::subview(inv_rho, i), ekat::subview(cld_frac_l, i), + ekat::subview(qc_incld, i), ekat::subview(rho, i), ekat::subview(inv_rho, i), ekat::subview(cld_frac_l, i), ekat::subview(acn, i), ekat::subview(inv_dz, i), dnu, team, workspace, nk, ktop, kbot, kdir, dt, inv_dt, do_predict_nc, ekat::subview(qc, i), ekat::subview(nc, i), ekat::subview(nc_incld, i), ekat::subview(mu_c, i), ekat::subview(lamc, i), ekat::subview(qc_tend, i), @@ -60,4 +60,3 @@ ::cloud_sedimentation_disp( } // namespace p3 } // namespace scream - diff --git a/components/eamxx/src/physics/p3/disp/p3_rain_sed_impl_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_rain_sed_impl_disp.cpp index d152313dc7f..a1b38afe8ed 100644 --- a/components/eamxx/src/physics/p3/disp/p3_rain_sed_impl_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_rain_sed_impl_disp.cpp @@ -45,11 +45,11 @@ ::rain_sedimentation_disp( // Rain sedimentation: (adaptive substepping) rain_sedimentation( - ekat::subview(rho, i), ekat::subview(inv_rho, i), ekat::subview(rhofacr, i), ekat::subview(cld_frac_r, i), - ekat::subview(inv_dz, i), ekat::subview(qr_incld, i), - team, workspace, vn_table_vals, vm_table_vals, nk, ktop, kbot, kdir, dt, inv_dt, - ekat::subview(qr, i), ekat::subview(nr, i), ekat::subview(nr_incld, i), ekat::subview(mu_r, i), - ekat::subview(lamr, i), ekat::subview(precip_liq_flux, i), + ekat::subview(rho, i), ekat::subview(inv_rho, i), ekat::subview(rhofacr, i), ekat::subview(cld_frac_r, i), + ekat::subview(inv_dz, i), ekat::subview(qr_incld, i), + team, workspace, vn_table_vals, vm_table_vals, nk, ktop, kbot, kdir, dt, inv_dt, + ekat::subview(qr, i), ekat::subview(nr, i), ekat::subview(nr_incld, i), ekat::subview(mu_r, i), + ekat::subview(lamr, i), ekat::subview(precip_liq_flux, i), ekat::subview(qr_tend, i), ekat::subview(nr_tend, i), precip_liq_surf(i), runtime_options); }); diff --git a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp index a2b8ccb2e29..f6771d6bf17 100644 --- a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp +++ b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp @@ -1,9 +1,7 @@ -#include "physics/p3/eamxx_p3_process_interface.hpp" #include "share/property_checks/field_within_interval_check.hpp" #include "share/property_checks/field_lower_bound_check.hpp" -// Needed for p3_init, the only F90 code still used. -#include "physics/p3/p3_functions.hpp" -#include "physics/p3/p3_f90.hpp" +#include "p3_functions.hpp" +#include "eamxx_p3_process_interface.hpp" #include "ekat/ekat_assert.hpp" #include "ekat/util/ekat_units.hpp" @@ -243,8 +241,8 @@ void P3Microphysics::initialize_impl (const RunType /* run_type */) add_postcondition_check(get_field_out("eff_radius_qr"),m_grid,0.0,5.0e3,false); // Initialize p3 - p3::p3_init(/* write_tables = */ false, - this->get_comm().am_i_root()); + P3F::p3_init(/* write_tables = */ false, + this->get_comm().am_i_root()); // Initialize all of the structures that are passed to p3_main in run_impl. // Note: Some variables in the structures are not stored in the field manager. For these diff --git a/components/eamxx/src/physics/p3/eti/p3_init.cpp b/components/eamxx/src/physics/p3/eti/p3_init.cpp new file mode 100644 index 00000000000..b1878af8984 --- /dev/null +++ b/components/eamxx/src/physics/p3/eti/p3_init.cpp @@ -0,0 +1,14 @@ +#include "p3_init_impl.hpp" + +namespace scream { +namespace p3 { + +/* + * Explicit instantiation for doing find functions on Reals using the + * default device. + */ + +template struct Functions; + +} // namespace p3 +} // namespace scream diff --git a/components/eamxx/src/physics/p3/impl/p3_init_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_init_impl.hpp new file mode 100644 index 00000000000..9b4b999bce0 --- /dev/null +++ b/components/eamxx/src/physics/p3/impl/p3_init_impl.hpp @@ -0,0 +1,40 @@ +#ifndef P3_INIT_IMPL_HPP +#define P3_INIT_IMPL_HPP + +#include "p3_functions.hpp" // for ETI only but harmless for GPU + +extern "C" { + void micro_p3_utils_init_c(scream::Real Cpair, scream::Real Rair, scream::Real RH2O, scream::Real RHO_H2O, + scream::Real MWH2O, scream::Real MWdry, scream::Real gravit, scream::Real LatVap, scream::Real LatIce, + scream::Real CpLiq, scream::Real Tmelt, scream::Real Pi, bool masterproc); + void p3_init_c(const char** lookup_file_dir, int* info, const bool& write_tables); +} + +namespace scream { +namespace p3 { + +/* + * Implementation of p3 init. Clients should NOT #include + * this file, #include p3_functions.hpp instead. + */ +template +void Functions +::p3_init (const bool write_tables, const bool masterproc) { + static bool is_init = false; + if (!is_init) { + using c = scream::physics::Constants; + micro_p3_utils_init_c(c::Cpair, c::Rair, c::RH2O, c::RHO_H2O, + c::MWH2O, c::MWdry, c::gravit, c::LatVap, c::LatIce, + c::CpLiq, c::Tmelt, c::Pi, masterproc); + static const char* dir = SCREAM_DATA_DIR "/tables"; + Int info; + p3_init_c(&dir, &info, write_tables); + EKAT_REQUIRE_MSG(info == 0, "p3_init_c returned info " << info); + is_init = true; + } +} + +} // namespace p3 +} // namespace scream + +#endif diff --git a/components/eamxx/src/physics/p3/p3_functions.hpp b/components/eamxx/src/physics/p3/p3_functions.hpp index 4ac88b4afc4..eef5a8ec73e 100644 --- a/components/eamxx/src/physics/p3/p3_functions.hpp +++ b/components/eamxx/src/physics/p3/p3_functions.hpp @@ -357,6 +357,9 @@ struct Functions static void init_kokkos_ice_lookup_tables( view_ice_table& ice_table_vals, view_collect_table& collect_table_vals); + static void p3_init(const bool write_tables = false, + const bool masterproc = false); + // Map (mu_r, lamr) to Table3 data. KOKKOS_FUNCTION static void lookup(const Spack& mu_r, const Spack& lamr, @@ -1468,5 +1471,6 @@ void init_tables_from_f90_c(Real* vn_table_vals_data, Real* vm_table_vals_data, # include "p3_nr_conservation_impl.hpp" # include "p3_ni_conservation_impl.hpp" # include "p3_prevent_liq_supersaturation_impl.hpp" +# include "p3_init_impl.hpp" #endif // GPU && !KOKKOS_ENABLE_*_RELOCATABLE_DEVICE_CODE #endif // P3_FUNCTIONS_HPP diff --git a/components/eamxx/src/physics/p3/p3_iso_c.f90 b/components/eamxx/src/physics/p3/p3_iso_c.f90 index ea0a18411c1..71c846b7167 100644 --- a/components/eamxx/src/physics/p3/p3_iso_c.f90 +++ b/components/eamxx/src/physics/p3/p3_iso_c.f90 @@ -14,15 +14,6 @@ module p3_iso_c ! contains - subroutine append_precision(string, prefix) - - character(kind=c_char, len=512), intent(out) :: string - character(*), intent(in) :: prefix - real(kind=c_real) :: s - - write (string, '(a,i1,a1)') prefix, sizeof(s), C_NULL_CHAR - end subroutine append_precision - subroutine init_tables_from_f90_c(vn_table_vals_c, vm_table_vals_c, revap_table_vals_c, mu_table_c) bind(C) use micro_p3, only: p3_get_tables @@ -129,59 +120,6 @@ subroutine p3_init_c(lookup_file_dir_c, info, write_tables) bind(c) end subroutine p3_init_c - subroutine p3_main_c(qc,nc,qr,nr,th_atm,qv,dt,qi,qm,ni,bm, & - pres,dz,nc_nuceat_tend,nccn_prescribed,ni_activated,inv_qc_relvar,it,precip_liq_surf,precip_ice_surf,its,ite,kts,kte,diag_eff_radius_qc, & - diag_eff_radius_qi,diag_eff_radius_qr,rho_qi,do_predict_nc,do_prescribed_CCN,dpres,inv_exner,qv2qi_depos_tend, & - precip_liq_flux,precip_ice_flux,cld_frac_r,cld_frac_l,cld_frac_i,liq_ice_exchange, & - vap_liq_exchange, vap_ice_exchange, qv_prev, t_prev, elapsed_s) bind(C) - use micro_p3, only : p3_main - - real(kind=c_real), intent(inout), dimension(its:ite,kts:kte) :: qc, nc, qr, nr, qv, th_atm - real(kind=c_real), intent(inout), dimension(its:ite,kts:kte) :: qi, qm, ni, bm - real(kind=c_real), intent(in), dimension(its:ite,kts:kte) :: pres, dz - real(kind=c_real), intent(in), dimension(its:ite,kts:kte) :: nc_nuceat_tend,nccn_prescribed,ni_activated - real(kind=c_real), intent(in), dimension(its:ite,kts:kte) :: inv_qc_relvar - real(kind=c_real), value, intent(in) :: dt - real(kind=c_real), intent(out), dimension(its:ite) :: precip_liq_surf, precip_ice_surf - real(kind=c_real), intent(out), dimension(its:ite,kts:kte) :: diag_eff_radius_qc - real(kind=c_real), intent(out), dimension(its:ite,kts:kte) :: diag_eff_radius_qi - real(kind=c_real), intent(out), dimension(its:ite,kts:kte) :: diag_eff_radius_qr - real(kind=c_real), intent(out), dimension(its:ite,kts:kte) :: rho_qi - integer(kind=c_int), value, intent(in) :: its,ite, kts,kte, it - logical(kind=c_bool), value, intent(in) :: do_predict_nc,do_prescribed_CCN - - real(kind=c_real), intent(in), dimension(its:ite,kts:kte) :: dpres - real(kind=c_real), intent(in), dimension(its:ite,kts:kte) :: inv_exner - real(kind=c_real), intent(out), dimension(its:ite,kts:kte) :: qv2qi_depos_tend - real(kind=c_real), intent(out), dimension(its:ite,kts:kte+1) :: precip_liq_flux - real(kind=c_real), intent(out), dimension(its:ite,kts:kte+1) :: precip_ice_flux - real(kind=c_real), intent(in), dimension(its:ite,kts:kte) :: cld_frac_i, cld_frac_l, cld_frac_r - real(kind=c_real), intent(out), dimension(its:ite,kts:kte) :: liq_ice_exchange - real(kind=c_real), intent(out), dimension(its:ite,kts:kte) :: vap_liq_exchange - real(kind=c_real), intent(out), dimension(its:ite,kts:kte) :: vap_ice_exchange - real(kind=c_real), intent(in), dimension(its:ite,kts:kte) :: qv_prev - real(kind=c_real), intent(in), dimension(its:ite,kts:kte) :: t_prev - - real(kind=c_real), intent(out) :: elapsed_s - - real(kind=c_real), dimension(its:ite,kts:kte,49) :: p3_tend_out - real(kind=c_real), dimension(its:ite,3) :: col_location - real(kind=c_real), dimension(its:ite,kts:kte) :: mu_c, lamc - real(kind=c_real), dimension(its:ite,kts:kte) :: precip_total_tend - real(kind=c_real), dimension(its:ite,kts:kte) :: nevapr - real(kind=c_real), dimension(its:ite,kts:kte) :: qr_evap_tend - integer :: i - do i = its,ite - col_location(i,:) = real(i) - end do - - call p3_main(qc,nc,qr,nr,th_atm,qv,dt,qi,qm,ni,bm, & - pres,dz,nc_nuceat_tend,nccn_prescribed,ni_activated,inv_qc_relvar,it,precip_liq_surf,precip_ice_surf,its,ite,kts,kte,diag_eff_radius_qc, & - diag_eff_radius_qi,diag_eff_radius_qr, rho_qi,do_predict_nc,do_prescribed_CCN,dpres,inv_exner,qv2qi_depos_tend,precip_total_tend,nevapr, & - qr_evap_tend,precip_liq_flux,precip_ice_flux,cld_frac_r,cld_frac_l,cld_frac_i,p3_tend_out,mu_c,lamc,liq_ice_exchange,& - vap_liq_exchange,vap_ice_exchange,qv_prev,t_prev,col_location,elapsed_s) - end subroutine p3_main_c - subroutine micro_p3_utils_init_c(Cpair, Rair, RH2O, RHO_H2O, & MWH2O, MWdry, gravit, LatVap, LatIce, & CpLiq, Tmelt, Pi, masterproc) bind(C) @@ -217,772 +155,4 @@ subroutine p3_init_a_c(ice_table_vals_c, collect_table_vals_c) bind(C) collect_table_vals_c(:,:,:,:,:) = collect_table_vals(:,:,:,:,:) end subroutine p3_init_a_c - subroutine find_lookuptable_indices_1a_c(dumi,dumjj,dumii,dumzz,dum1,dum4,dum5,dum6, & - qi,ni,qm,rhop) bind(C) - use micro_p3, only: find_lookupTable_indices_1a - use micro_p3_utils, only: densize,rimsize,isize - - ! arguments: - integer(kind=c_int), intent(out) :: dumi,dumjj,dumii,dumzz - real(kind=c_real), intent(out) :: dum1,dum4,dum5,dum6 - real(kind=c_real), value, intent(in) :: qi,ni,qm,rhop - - call find_lookupTable_indices_1a(dumi, dumjj, dumii, dumzz, dum1, dum4, dum5, dum6, & - isize, rimsize, densize, qi, ni, qm, rhop) - end subroutine find_lookuptable_indices_1a_c - - subroutine find_lookuptable_indices_1b_c(dumj,dum3,qr,nr) bind(C) - use micro_p3, only: find_lookupTable_indices_1b - use micro_p3_utils, only: rcollsize - - integer(kind=c_int), intent(out) :: dumj - real(kind=c_real), intent(out) :: dum3 - real(kind=c_real), value, intent(in) :: qr, nr - - call find_lookupTable_indices_1b(dumj, dum3, rcollsize, qr, nr) - end subroutine find_lookupTable_indices_1b_c - - subroutine access_lookup_table_c(dumjj,dumii,dumi,index,dum1,dum4,dum5,proc) bind(C) - use micro_p3, only: access_lookup_table - - integer(kind=c_int), value, intent(in) :: dumjj, dumii, dumi, index - real(kind=c_real), value, intent(in) :: dum1, dum4, dum5 - real(kind=c_real), intent(out) :: proc - - call access_lookup_table(dumjj,dumii,dumi,index,dum1,dum4,dum5,proc) - end subroutine access_lookup_table_c - - subroutine access_lookup_table_coll_c(dumjj,dumii,dumj,dumi,index,dum1,dum3,dum4,dum5,proc) bind(C) - use micro_p3, only: access_lookup_table_coll - - integer(kind=c_int), value, intent(in) :: dumjj,dumii,dumj,dumi,index - real(kind=c_real), value, intent(in) :: dum1,dum3,dum4,dum5 - real(kind=c_real), intent(out) :: proc - - call access_lookup_table_coll(dumjj,dumii,dumj,dumi,index,dum1,dum3,dum4,dum5,proc) - end subroutine access_lookup_table_coll_c - - subroutine back_to_cell_average_c(cld_frac_l,cld_frac_r,cld_frac_i, qc2qr_accret_tend,qr2qv_evap_tend,qc2qr_autoconv_tend,& - nc_accret_tend,nc_selfcollect_tend,nc2nr_autoconv_tend,nr_selfcollect_tend,nr_evap_tend,ncautr,qi2qv_sublim_tend,nr_ice_shed_tend,qc2qi_hetero_freeze_tend,& - qr2qi_collect_tend,qc2qr_ice_shed_tend,qi2qr_melt_tend,qc2qi_collect_tend,qr2qi_immers_freeze_tend,ni2nr_melt_tend,nc_collect_tend,ncshdc,nc2ni_immers_freeze_tend,nr_collect_tend,ni_selfcollect_tend,& - qv2qi_vapdep_tend,nr2ni_immers_freeze_tend,ni_sublim_tend,qv2qi_nucleat_tend,ni_nucleat_tend,qc2qi_berg_tend) bind(C) - - use micro_p3, only: back_to_cell_average - real(kind=c_real), value, intent(in) :: cld_frac_l, cld_frac_r, cld_frac_i - - real(kind=c_real), intent(inout) :: qc2qr_accret_tend, qr2qv_evap_tend, qc2qr_autoconv_tend, nc_accret_tend, nc_selfcollect_tend, nc2nr_autoconv_tend, & - nr_selfcollect_tend, nr_evap_tend, ncautr, qi2qv_sublim_tend, & - nr_ice_shed_tend, qc2qi_hetero_freeze_tend, qr2qi_collect_tend, qc2qr_ice_shed_tend, qi2qr_melt_tend, qc2qi_collect_tend, & - qr2qi_immers_freeze_tend, ni2nr_melt_tend, nc_collect_tend, ncshdc, nc2ni_immers_freeze_tend, nr_collect_tend,& - ni_selfcollect_tend, qv2qi_vapdep_tend, nr2ni_immers_freeze_tend, ni_sublim_tend, qv2qi_nucleat_tend, ni_nucleat_tend, & - qc2qi_berg_tend - - call back_to_cell_average(cld_frac_l, cld_frac_r, cld_frac_i, qc2qr_accret_tend, qr2qv_evap_tend, qc2qr_autoconv_tend,& - nc_accret_tend, nc_selfcollect_tend, nc2nr_autoconv_tend, nr_selfcollect_tend, nr_evap_tend, ncautr, qi2qv_sublim_tend, nr_ice_shed_tend, qc2qi_hetero_freeze_tend,& - qr2qi_collect_tend, qc2qr_ice_shed_tend, qi2qr_melt_tend, qc2qi_collect_tend, qr2qi_immers_freeze_tend, ni2nr_melt_tend, nc_collect_tend, ncshdc, nc2ni_immers_freeze_tend, nr_collect_tend, ni_selfcollect_tend,& - qv2qi_vapdep_tend, nr2ni_immers_freeze_tend, ni_sublim_tend, qv2qi_nucleat_tend, ni_nucleat_tend, qc2qi_berg_tend) - end subroutine back_to_cell_average_c - - subroutine cloud_water_conservation_c(qc,dt,qc2qr_autoconv_tend,qc2qr_accret_tend,qc2qi_collect_tend,qc2qi_hetero_freeze_tend,qc2qr_ice_shed_tend, & - qc2qi_berg_tend,qi2qv_sublim_tend,qv2qi_vapdep_tend) bind(C) - use micro_p3, only: cloud_water_conservation - - real(kind=c_real), value, intent(in) :: qc, dt - real(kind=c_real), intent(inout) :: qc2qr_autoconv_tend, qc2qr_accret_tend, qc2qi_collect_tend, qc2qi_hetero_freeze_tend, qc2qr_ice_shed_tend, qc2qi_berg_tend, qi2qv_sublim_tend, qv2qi_vapdep_tend - - call cloud_water_conservation(qc,dt,qc2qr_autoconv_tend,qc2qr_accret_tend,qc2qi_collect_tend,qc2qi_hetero_freeze_tend,qc2qr_ice_shed_tend,qc2qi_berg_tend,qi2qv_sublim_tend,qv2qi_vapdep_tend) - end subroutine cloud_water_conservation_c - - subroutine rain_water_conservation_c(qr,qc2qr_autoconv_tend,qc2qr_accret_tend,qi2qr_melt_tend,qc2qr_ice_shed_tend,dt, & - qr2qv_evap_tend,qr2qi_collect_tend,qr2qi_immers_freeze_tend) bind(C) - use micro_p3, only: rain_water_conservation - - real(kind=c_real), value, intent(in) :: qr, qc2qr_autoconv_tend, qc2qr_accret_tend, qi2qr_melt_tend, qc2qr_ice_shed_tend, dt - real(kind=c_real), intent(inout) :: qr2qv_evap_tend, qr2qi_collect_tend, qr2qi_immers_freeze_tend - - call rain_water_conservation(qr,qc2qr_autoconv_tend,qc2qr_accret_tend,qi2qr_melt_tend,qc2qr_ice_shed_tend,dt,qr2qv_evap_tend,qr2qi_collect_tend,qr2qi_immers_freeze_tend) - end subroutine rain_water_conservation_c - - subroutine rain_self_collection_c(rho, qr_incld, nr_incld, nr_selfcollect_tend) bind(C) - use micro_p3, only: rain_self_collection - - real(kind=c_real), value, intent(in) :: rho, qr_incld, nr_incld - real(kind=c_real), intent(out) :: nr_selfcollect_tend - - call rain_self_collection(rho, qr_incld, nr_incld, nr_selfcollect_tend) - end subroutine rain_self_collection_c - - subroutine ice_water_conservation_c(qi,qv2qi_vapdep_tend,qv2qi_nucleat_tend,qc2qi_berg_tend,qr2qi_collect_tend,qc2qi_collect_tend,qr2qi_immers_freeze_tend,qc2qi_hetero_freeze_tend,dt, & - qi2qv_sublim_tend,qi2qr_melt_tend) bind(C) - use micro_p3, only: ice_water_conservation - - real(kind=c_real), value, intent(in) :: qi, qv2qi_vapdep_tend, qv2qi_nucleat_tend, qr2qi_collect_tend, qc2qi_collect_tend, qr2qi_immers_freeze_tend, qc2qi_hetero_freeze_tend, qc2qi_berg_tend, dt - real(kind=c_real), intent(inout) :: qi2qv_sublim_tend, qi2qr_melt_tend - - call ice_water_conservation(qi,qv2qi_vapdep_tend,qv2qi_nucleat_tend,qr2qi_collect_tend,qc2qi_collect_tend,qr2qi_immers_freeze_tend,qc2qi_hetero_freeze_tend,qc2qi_berg_tend,dt,qi2qv_sublim_tend,qi2qr_melt_tend) - end subroutine ice_water_conservation_c - - subroutine get_cloud_dsd2_c(qc,nc,mu_c,rho,nu,lamc,cdist,cdist1) bind(C) - use micro_p3, only: get_cloud_dsd2 - use micro_p3_utils, only: dnu - - !arguments: - real(kind=c_real), value, intent(in) :: qc,rho - real(kind=c_real), intent(inout) :: nc - real(kind=c_real), intent(out) :: mu_c,nu,lamc,cdist,cdist1 - - call get_cloud_dsd2(qc,nc,mu_c,rho,nu,dnu,lamc,cdist,cdist1) - end subroutine get_cloud_dsd2_c - - subroutine get_rain_dsd2_c(qr,nr,mu_r,lamr,cdistr,logn0r) bind(C) - use micro_p3, only: get_rain_dsd2 - - !arguments: - real(kind=c_real), value, intent(in) :: qr - real(kind=c_real), intent(inout) :: nr - real(kind=c_real), intent(out) :: lamr,mu_r,cdistr,logn0r - - call get_rain_dsd2(qr,nr,mu_r,lamr,cdistr,logn0r) - end subroutine get_rain_dsd2_c - - subroutine calc_rime_density_c(T_atm,rhofaci,table_val_qi_fallspd,acn,lamc,mu_c,qc_incld,qc2qi_collect_tend, & - vtrmi1,rho_qm_cloud) bind(C) - - use micro_p3, only: calc_rime_density - real(kind=c_real), value, intent(in) :: T_atm, rhofaci, table_val_qi_fallspd, acn, lamc, mu_c, qc_incld, qc2qi_collect_tend - real(kind=c_real), intent(out) :: vtrmi1, rho_qm_cloud - - call calc_rime_density(T_atm, rhofaci, table_val_qi_fallspd, acn, lamc, mu_c, qc_incld, qc2qi_collect_tend, vtrmi1, rho_qm_cloud) - end subroutine calc_rime_density_c - - subroutine cldliq_immersion_freezing_c(T_atm,lamc,mu_c,cdist1,qc_incld,inv_qc_relvar,qc2qi_hetero_freeze_tend,nc2ni_immers_freeze_tend) bind(C) - - use micro_p3, only: cldliq_immersion_freezing - real(kind=c_real), value, intent(in) :: T_atm, lamc, mu_c, cdist1, qc_incld,inv_qc_relvar - real(kind=c_real), intent(out) :: qc2qi_hetero_freeze_tend, nc2ni_immers_freeze_tend - - call cldliq_immersion_freezing(T_atm, lamc, mu_c, cdist1, qc_incld, inv_qc_relvar, qc2qi_hetero_freeze_tend, nc2ni_immers_freeze_tend) - end subroutine cldliq_immersion_freezing_c - - subroutine rain_immersion_freezing_c(T_atm,lamr,mu_r,cdistr,qr_incld,qr2qi_immers_freeze_tend,nr2ni_immers_freeze_tend) bind(C) - - use micro_p3, only: rain_immersion_freezing - real(kind=c_real), value, intent(in) :: T_atm, lamr, mu_r, cdistr, qr_incld - real(kind=c_real), intent(out) :: qr2qi_immers_freeze_tend, nr2ni_immers_freeze_tend - - call rain_immersion_freezing(T_atm, lamr, mu_r, cdistr, qr_incld, qr2qi_immers_freeze_tend, nr2ni_immers_freeze_tend) - end subroutine rain_immersion_freezing_c - - subroutine droplet_self_collection_c(rho,inv_rho,qc_incld,mu_c,nu,nc2nr_autoconv_tend,nc_selfcollect_tend) bind(C) - - use micro_p3, only: droplet_self_collection - real(kind=c_real), value, intent(in) :: rho, inv_rho, qc_incld, mu_c, nu, nc2nr_autoconv_tend - real(kind=c_real), intent(out) :: nc_selfcollect_tend - - call droplet_self_collection(rho, inv_rho, qc_incld, mu_c, nu, nc2nr_autoconv_tend, nc_selfcollect_tend) - end subroutine droplet_self_collection_c - - subroutine cloud_rain_accretion_c(rho,inv_rho,qc_incld,nc_incld,qr_incld,inv_qc_relvar,qc2qr_accret_tend,nc_accret_tend) bind(C) - - use micro_p3, only: cloud_rain_accretion - real(kind=c_real), value, intent(in) :: rho, inv_rho, qc_incld, nc_incld, qr_incld,inv_qc_relvar - real(kind=c_real), intent(out) :: qc2qr_accret_tend, nc_accret_tend - - call cloud_rain_accretion(rho, inv_rho, qc_incld, nc_incld, qr_incld, inv_qc_relvar, qc2qr_accret_tend, nc_accret_tend) - end subroutine cloud_rain_accretion_c - - subroutine cloud_water_autoconversion_c(rho,qc_incld,nc_incld,inv_qc_relvar,qc2qr_autoconv_tend,nc2nr_autoconv_tend,ncautr) bind(C) - - use micro_p3, only: cloud_water_autoconversion - real(kind=c_real), value, intent(in) :: rho, qc_incld, nc_incld,inv_qc_relvar - real(kind=c_real), intent(inout) :: qc2qr_autoconv_tend, nc2nr_autoconv_tend, ncautr - - call cloud_water_autoconversion(rho, qc_incld, nc_incld, inv_qc_relvar, qc2qr_autoconv_tend, nc2nr_autoconv_tend, ncautr) - end subroutine cloud_water_autoconversion_c - - subroutine impose_max_total_ni_c(ni_local, max_total_ni, inv_rho_local) bind(C) - use micro_p3, only: impose_max_total_ni - - real(kind=c_real), intent(inout) :: ni_local - real(kind=c_real), value, intent(in) :: max_total_ni, inv_rho_local - - call impose_max_total_ni(ni_local, max_total_ni, inv_rho_local) - end subroutine impose_max_total_ni_c - - subroutine calc_first_order_upwind_step_c(kts, kte, kdir, kbot, k_qxtop, dt_sub, rho, inv_rho, inv_dz, num_arrays, fluxes, vs, qnx) bind(C) - use micro_p3, only: calc_first_order_upwind_step, realptr - - !arguments: - integer(kind=c_int), value, intent(in) :: kts, kte, kdir, kbot, k_qxtop, num_arrays - real(kind=c_real), value, intent(in) :: dt_sub - real(kind=c_real), dimension(kts:kte), intent(in) :: rho, inv_rho, inv_dz - type(c_ptr), intent(in), dimension(num_arrays) :: fluxes, vs, qnx - - type(realptr), dimension(num_arrays) :: fluxes_f, vs_f, qnx_f - integer :: i - - do i = 1, num_arrays - call c_f_pointer(fluxes(i), fluxes_f(i)%p, [(kte-kts)+1]) - call c_f_pointer(vs(i), vs_f(i)%p, [(kte-kts)+1]) - call c_f_pointer(qnx(i), qnx_f(i)%p , [(kte-kts)+1]) - end do - - call calc_first_order_upwind_step(kts, kte, kdir, kbot, k_qxtop, dt_sub, rho, inv_rho, inv_dz, num_arrays, fluxes_f, vs_f, qnx_f) - - end subroutine calc_first_order_upwind_step_c - - subroutine generalized_sedimentation_c(kts, kte, kdir, k_qxtop, k_qxbot, kbot, Co_max, dt_left, prt_accum, inv_dz, inv_rho, rho, num_arrays, vs, fluxes, qnx) bind(C) - use micro_p3, only: generalized_sedimentation, realptr - - ! arguments - integer(kind=c_int), value, intent(in) :: kts, kte, kdir, k_qxtop, kbot, num_arrays - integer(kind=c_int), intent(inout) :: k_qxbot - real(kind=c_real), value, intent(in) :: Co_max - real(kind=c_real), intent(inout) :: dt_left, prt_accum - real(kind=c_real), dimension(kts:kte), intent(in) :: inv_dz, inv_rho, rho - type(c_ptr), intent(in), dimension(num_arrays) :: vs, fluxes, qnx - - type(realptr), dimension(num_arrays) :: fluxes_f, vs_f, qnx_f - integer :: i - - do i = 1, num_arrays - call c_f_pointer(fluxes(i), fluxes_f(i)%p, [(kte-kts)+1]) - call c_f_pointer(vs(i), vs_f(i)%p, [(kte-kts)+1]) - call c_f_pointer(qnx(i), qnx_f(i)%p , [(kte-kts)+1]) - end do - - call generalized_sedimentation(kts, kte, kdir, k_qxtop, k_qxbot, kbot, Co_max, dt_left, prt_accum, inv_dz, inv_rho, rho, num_arrays, vs_f, fluxes_f, qnx_f) - - end subroutine generalized_sedimentation_c - - subroutine cloud_sedimentation_c(kts,kte,ktop,kbot,kdir, & - qc_incld,rho,inv_rho,cld_frac_l,acn,inv_dz,& - dt,inv_dt,do_predict_nc, & - qc, nc, nc_incld,mu_c,lamc,precip_liq_surf,qc_tend,nc_tend) bind(C) - use micro_p3, only: cloud_sedimentation, dnu - - ! arguments - integer(kind=c_int), value, intent(in) :: kts, kte, ktop, kbot, kdir - - real(kind=c_real), intent(in), dimension(kts:kte) :: rho - real(kind=c_real), intent(in), dimension(kts:kte) :: inv_rho - real(kind=c_real), intent(in), dimension(kts:kte) :: cld_frac_l - real(kind=c_real), intent(in), dimension(kts:kte) :: acn - real(kind=c_real), intent(in), dimension(kts:kte) :: inv_dz - - real(kind=c_real), value, intent(in) :: dt - real(kind=c_real), value, intent(in) :: inv_dt - logical(kind=c_bool), value, intent(in) :: do_predict_nc - - real(kind=c_real), intent(inout), dimension(kts:kte) :: qc - real(kind=c_real), intent(inout), dimension(kts:kte) :: nc - real(kind=c_real), intent(inout), dimension(kts:kte) :: qc_incld - real(kind=c_real), intent(inout), dimension(kts:kte) :: nc_incld - real(kind=c_real), intent(inout), dimension(kts:kte) :: mu_c - real(kind=c_real), intent(inout), dimension(kts:kte) :: lamc - real(kind=c_real), intent(inout) :: precip_liq_surf - real(kind=c_real), intent(inout), dimension(kts:kte) :: qc_tend - real(kind=c_real), intent(inout), dimension(kts:kte) :: nc_tend - - call cloud_sedimentation(kts,kte,ktop,kbot,kdir, & - qc_incld,rho,inv_rho,cld_frac_l,acn,inv_dz,& - dt,inv_dt,dnu,do_predict_nc, & - qc, nc, nc_incld,mu_c,lamc,precip_liq_surf,qc_tend,nc_tend) - - end subroutine cloud_sedimentation_c - - subroutine ice_sedimentation_c(kts,kte,ktop,kbot,kdir, & - rho,inv_rho,rhofaci,cld_frac_i,inv_dz,dt,inv_dt, & - qi,qi_incld,ni,qm,qm_incld,bm,bm_incld,ni_incld,precip_ice_surf,qi_tend,ni_tend) bind(C) - use micro_p3, only: ice_sedimentation - - ! arguments - integer(kind=c_int), value, intent(in) :: kts, kte, ktop, kbot, kdir - - real(kind=c_real), intent(in), dimension(kts:kte) :: rho - real(kind=c_real), intent(in), dimension(kts:kte) :: inv_rho - real(kind=c_real), intent(in), dimension(kts:kte) :: rhofaci - real(kind=c_real), intent(in), dimension(kts:kte) :: cld_frac_i - real(kind=c_real), intent(in), dimension(kts:kte) :: inv_dz - real(kind=c_real), value, intent(in) :: dt, inv_dt - - real(kind=c_real), intent(inout), dimension(kts:kte), target :: qi - real(kind=c_real), intent(inout), dimension(kts:kte) :: qi_incld - real(kind=c_real), intent(inout), dimension(kts:kte), target :: ni - real(kind=c_real), intent(inout), dimension(kts:kte) :: ni_incld - real(kind=c_real), intent(inout), dimension(kts:kte), target :: qm - real(kind=c_real), intent(inout), dimension(kts:kte) :: qm_incld - real(kind=c_real), intent(inout), dimension(kts:kte), target :: bm - real(kind=c_real), intent(inout), dimension(kts:kte) :: bm_incld - - real(kind=c_real), intent(inout) :: precip_ice_surf - real(kind=c_real), intent(inout), dimension(kts:kte) :: qi_tend - real(kind=c_real), intent(inout), dimension(kts:kte) :: ni_tend - - call ice_sedimentation(kts,kte,ktop,kbot,kdir, & - rho,inv_rho,rhofaci,cld_frac_i,inv_dz,dt,inv_dt, & - qi,qi_incld,ni,qm,qm_incld,bm,bm_incld,ni_incld,precip_ice_surf,qi_tend,ni_tend) - - end subroutine ice_sedimentation_c - - subroutine rain_sedimentation_c(kts,kte,ktop,kbot,kdir, & - qr_incld,rho,inv_rho,rhofacr,cld_frac_r,inv_dz,dt,inv_dt, & - qr,nr,nr_incld,mu_r,lamr,precip_liq_surf,precip_liq_flux,qr_tend,nr_tend) bind(C) - use micro_p3, only: rain_sedimentation - - integer(kind=c_int), value, intent(in) :: kts, kte, ktop, kbot, kdir - - real(kind=c_real), intent(in), dimension(kts:kte) :: rho - real(kind=c_real), intent(in), dimension(kts:kte) :: inv_rho - real(kind=c_real), intent(in), dimension(kts:kte) :: rhofacr - real(kind=c_real), intent(in), dimension(kts:kte) :: cld_frac_r - real(kind=c_real), intent(in), dimension(kts:kte) :: inv_dz - real(kind=c_real), value, intent(in) :: dt, inv_dt - - real(kind=c_real), intent(inout), target, dimension(kts:kte) :: qr - real(kind=c_real), intent(inout), target, dimension(kts:kte) :: nr - real(kind=c_real), intent(inout), dimension(kts:kte) :: qr_incld - real(kind=c_real), intent(inout), dimension(kts:kte) :: nr_incld - real(kind=c_real), intent(inout), dimension(kts:kte) :: mu_r - real(kind=c_real), intent(inout), dimension(kts:kte) :: lamr - real(kind=c_real), intent(inout) :: precip_liq_surf - real(kind=c_real), intent(inout), dimension(kts:kte+1) :: precip_liq_flux - real(kind=c_real), intent(inout), dimension(kts:kte) :: qr_tend - real(kind=c_real), intent(inout), dimension(kts:kte) :: nr_tend - - call rain_sedimentation(kts,kte,ktop,kbot,kdir, & - qr_incld,rho,inv_rho,rhofacr,cld_frac_r,inv_dz,dt,inv_dt, & - qr,nr,nr_incld,mu_r,lamr,precip_liq_surf,precip_liq_flux,qr_tend,nr_tend) - - end subroutine rain_sedimentation_c - - subroutine calc_bulk_rho_rime_c(qi_tot, qi_rim, bi_rim, rho_rime) bind(C) - use micro_p3, only: calc_bulkRhoRime - - ! arguments: - real(kind=c_real), value, intent(in) :: qi_tot - real(kind=c_real), intent(inout) :: qi_rim, bi_rim - real(kind=c_real), intent(out) :: rho_rime - - call calc_bulkRhoRime(qi_tot, qi_rim, bi_rim, rho_rime) - end subroutine calc_bulk_rho_rime_c - - subroutine homogeneous_freezing_c(kts,kte,ktop,kbot,kdir,T_atm,inv_exner,latent_heat_fusion, & - qc,nc,qr,nr,qi,ni,qm,bm,th_atm) bind(C) - use micro_p3, only: homogeneous_freezing - - ! arguments: - integer(kind=c_int), value, intent(in) :: kts, kte, ktop, kbot, kdir - real(kind=c_real), intent(in), dimension(kts:kte) :: T_atm - real(kind=c_real), intent(in), dimension(kts:kte) :: inv_exner - real(kind=c_real), intent(in), dimension(kts:kte) :: latent_heat_fusion - - real(kind=c_real), intent(inout), dimension(kts:kte) :: qc - real(kind=c_real), intent(inout), dimension(kts:kte) :: nc - real(kind=c_real), intent(inout), dimension(kts:kte) :: qr - real(kind=c_real), intent(inout), dimension(kts:kte) :: nr - - real(kind=c_real), intent(inout), dimension(kts:kte) :: qi - real(kind=c_real), intent(inout), dimension(kts:kte) :: ni - real(kind=c_real), intent(inout), dimension(kts:kte) :: qm - real(kind=c_real), intent(inout), dimension(kts:kte) :: bm - real(kind=c_real), intent(inout), dimension(kts:kte) :: th_atm - - call homogeneous_freezing(kts,kte,ktop,kbot,kdir,T_atm,inv_exner,latent_heat_fusion, & - qc,nc,qr,nr,qi,ni,qm,bm,th_atm) - end subroutine homogeneous_freezing_c - - subroutine compute_rain_fall_velocity_c(qr_incld, rhofacr, nr_incld, mu_r, lamr, V_qr, V_nr) bind(C) - use micro_p3, only: compute_rain_fall_velocity - - ! arguments: - real(kind=c_real), value, intent(in) :: qr_incld, rhofacr - real(kind=c_real), intent(inout) :: nr_incld - real(kind=c_real), intent(out) :: mu_r, lamr, V_qr, V_nr - - call compute_rain_fall_velocity(qr_incld, rhofacr, nr_incld, mu_r, lamr, V_qr, V_nr) - end subroutine compute_rain_fall_velocity_c - -subroutine update_prognostic_ice_c(qc2qi_hetero_freeze_tend,qc2qi_collect_tend,qc2qr_ice_shed_tend,nc_collect_tend,nc2ni_immers_freeze_tend,ncshdc,qr2qi_collect_tend,nr_collect_tend,qr2qi_immers_freeze_tend,nr2ni_immers_freeze_tend,nr_ice_shed_tend, & - qi2qr_melt_tend,ni2nr_melt_tend,qi2qv_sublim_tend,qv2qi_vapdep_tend,qv2qi_nucleat_tend,ni_nucleat_tend,ni_selfcollect_tend,ni_sublim_tend,qc2qi_berg_tend,inv_exner,latent_heat_sublim,latent_heat_fusion,do_predict_nc,log_wetgrowth, & - dt,nmltratio,rho_qm_cloud,th_atm,qv,qi,ni,qm,bm,qc,nc,qr,nr) bind(C) - use micro_p3, only: update_prognostic_ice - - ! arguments - real(kind=c_real), value, intent(in) :: qc2qi_hetero_freeze_tend, qc2qi_collect_tend, qc2qr_ice_shed_tend, nc_collect_tend, nc2ni_immers_freeze_tend, ncshdc, qr2qi_collect_tend, nr_collect_tend, & - qr2qi_immers_freeze_tend, nr2ni_immers_freeze_tend, nr_ice_shed_tend, qi2qr_melt_tend, ni2nr_melt_tend, qi2qv_sublim_tend, qv2qi_vapdep_tend, qv2qi_nucleat_tend, ni_nucleat_tend, ni_selfcollect_tend, ni_sublim_tend, qc2qi_berg_tend, inv_exner, & - latent_heat_fusion, latent_heat_sublim, dt, nmltratio, rho_qm_cloud - - logical(kind=c_bool), value, intent(in) :: do_predict_nc, log_wetgrowth - - real(kind=c_real), intent(inout) :: th_atm, qv, qc, nc, qr, nr, qi, ni, qm, bm - - call update_prognostic_ice(qc2qi_hetero_freeze_tend,qc2qi_collect_tend,qc2qr_ice_shed_tend,nc_collect_tend,nc2ni_immers_freeze_tend,ncshdc,qr2qi_collect_tend,nr_collect_tend,qr2qi_immers_freeze_tend,nr2ni_immers_freeze_tend,nr_ice_shed_tend, & - qi2qr_melt_tend,ni2nr_melt_tend,qi2qv_sublim_tend,qv2qi_vapdep_tend,qv2qi_nucleat_tend,ni_nucleat_tend,ni_selfcollect_tend,ni_sublim_tend,qc2qi_berg_tend,inv_exner,latent_heat_sublim,latent_heat_fusion,do_predict_nc,log_wetgrowth, & - dt,nmltratio,rho_qm_cloud,th_atm,qv,qi,ni,qm,bm,qc,nc,qr,nr) - - end subroutine update_prognostic_ice_c - - subroutine get_time_space_phys_variables_c(T_atm, pres, rho, latent_heat_vapor, latent_heat_sublim, qv_sat_l, qv_sat_i, mu, dv, sc, dqsdt, dqsidt, & - ab, abi, kap, eii) bind(C) - use micro_p3, only: get_time_space_phys_variables - - !arguments - real(kind=c_real), value, intent(in) :: T_atm, pres, rho, latent_heat_vapor, latent_heat_sublim, qv_sat_l, qv_sat_i - real(kind=c_real), intent(out) :: mu, dv, sc, dqsdt, dqsidt, ab, abi, kap, eii - - call get_time_space_phys_variables(T_atm, pres, rho, latent_heat_vapor, latent_heat_sublim, qv_sat_l, qv_sat_i, mu, dv, sc, dqsdt, dqsidt, & - ab, abi, kap, eii) - end subroutine get_time_space_phys_variables_c - - subroutine ice_cldliq_collection_c(rho, temp, rhofaci, table_val_qc2qi_collect, qi_incld, qc_incld, ni_incld, & - nc_incld, qc2qi_collect_tend, nc_collect_tend, qc2qr_ice_shed_tend, ncshdc) bind(C) - use micro_p3, only: ice_cldliq_collection - - ! arguments: - real(kind=c_real), value, intent(in) :: rho, temp, rhofaci, table_val_qc2qi_collect - real(kind=c_real), value, intent(in) :: qi_incld, qc_incld, ni_incld, nc_incld - real(kind=c_real), intent(out) :: qc2qi_collect_tend, nc_collect_tend, qc2qr_ice_shed_tend, ncshdc - - call ice_cldliq_collection(rho, temp, rhofaci, table_val_qc2qi_collect, qi_incld, qc_incld, ni_incld, & - nc_incld, qc2qi_collect_tend, nc_collect_tend, qc2qr_ice_shed_tend, ncshdc) - end subroutine ice_cldliq_collection_c - - subroutine ice_rain_collection_c(rho, temp, rhofaci, logn0r, table_val_nr_collect, table_val_qr2qi_collect, & - qi_incld, ni_incld, qr_incld, qr2qi_collect_tend, nr_collect_tend) bind(C) - use micro_p3, only: ice_rain_collection - - ! arguments: - real(kind=c_real), value, intent(in) :: rho, temp, rhofaci, logn0r, table_val_nr_collect, table_val_qr2qi_collect - real(kind=c_real), value, intent(in) :: qi_incld, ni_incld, qr_incld - real(kind=c_real), intent(out) :: qr2qi_collect_tend, nr_collect_tend - - call ice_rain_collection(rho, temp, rhofaci, logn0r, table_val_nr_collect, table_val_qr2qi_collect, & - qi_incld, ni_incld, qr_incld, qr2qi_collect_tend, nr_collect_tend) - end subroutine ice_rain_collection_c - - subroutine ice_self_collection_c(rho, rhofaci, table_val_ni_self_collect, eii, qm_incld, & - qi_incld, ni_incld, ni_selfcollect_tend) bind(C) - use micro_p3, only: ice_self_collection - - ! arguments: - real(kind=c_real), value, intent(in) :: rho, rhofaci, table_val_ni_self_collect, eii, qm_incld - real(kind=c_real), value, intent(in) :: qi_incld, ni_incld - real(kind=c_real), intent(out) :: ni_selfcollect_tend - - call ice_self_collection(rho, rhofaci, table_val_ni_self_collect, eii, qm_incld, & - qi_incld, ni_incld, ni_selfcollect_tend) - end subroutine ice_self_collection_c - - subroutine evaporate_rain_c(qr_incld,qc_incld,nr_incld,qi_incld, & - cld_frac_l,cld_frac_r,qv,qv_prev,qv_sat_l,qv_sat_i, & - ab,abi,epsr,epsi_tot,t,t_prev,latent_heat_sublim,dqsdt,dt,& - qr2qv_evap_tend,nr_evap_tend) bind(C) - use micro_p3, only: evaporate_rain - - ! arguments - real(kind=c_real), value, intent(in) :: qr_incld,qc_incld,nr_incld,qi_incld, & - cld_frac_l,cld_frac_r,qv,qv_prev,qv_sat_l,qv_sat_i, & - ab,abi,epsr,epsi_tot,t,t_prev,latent_heat_sublim,dqsdt,dt - - real(kind=c_real), intent(out) :: qr2qv_evap_tend, nr_evap_tend - - call evaporate_rain(qr_incld,qc_incld,nr_incld,qi_incld, & - cld_frac_l,cld_frac_r,qv,qv_prev,qv_sat_l,qv_sat_i, & - ab,abi,epsr,epsi_tot,t,t_prev,latent_heat_sublim,dqsdt,dt,& - qr2qv_evap_tend,nr_evap_tend) - end subroutine evaporate_rain_c - - subroutine update_prognostic_liquid_c(qc2qr_accret_tend, nc_accret_tend, qc2qr_autoconv_tend,nc2nr_autoconv_tend, ncautr, nc_selfcollect_tend, & - qr2qv_evap_tend, nr_evap_tend, nr_selfcollect_tend, do_predict_nc, do_prescribed_CCN, inv_rho, inv_exner, latent_heat_vapor, dt, th_atm, qv, qc, nc, qr, nr) bind(C) - use micro_p3, only: update_prognostic_liquid - - ! arguments - real(kind=c_real), value, intent(in) :: qc2qr_accret_tend, nc_accret_tend, qc2qr_autoconv_tend, nc2nr_autoconv_tend, ncautr, nc_selfcollect_tend, & - qr2qv_evap_tend, nr_evap_tend, nr_selfcollect_tend - - logical(kind=c_bool), value, intent(in) :: do_predict_nc - logical(kind=c_bool), value, intent(in) :: do_prescribed_CCN - - real(kind=c_real), value, intent(in) :: inv_rho, inv_exner, latent_heat_vapor, dt - - real(kind=c_real), intent(inout) :: th_atm, qv, qc, nc, qr, nr - - call update_prognostic_liquid(qc2qr_accret_tend, nc_accret_tend, qc2qr_autoconv_tend,nc2nr_autoconv_tend, ncautr, nc_selfcollect_tend, & - qr2qv_evap_tend, nr_evap_tend, nr_selfcollect_tend, do_predict_nc, do_prescribed_CCN, inv_rho, inv_exner, latent_heat_vapor, dt, th_atm, qv, qc, nc, qr, nr) - - end subroutine update_prognostic_liquid_c - - subroutine ice_deposition_sublimation_c(qi_incld, ni_incld, t_atm, qv_sat_l, qv_sat_i, epsi, abi, qv, inv_dt, qidep, qi2qv_sublim_tend, ni_sublim_tend, qiberg) bind(C) - use micro_p3, only : ice_deposition_sublimation - - real(kind=c_real) , value, intent(in) :: qi_incld, ni_incld, t_atm, qv_sat_l, qv_sat_i, epsi, abi, qv, inv_dt - real(kind=c_real) , intent(out) :: qidep, qi2qv_sublim_tend, ni_sublim_tend, qiberg - - call ice_deposition_sublimation(qi_incld, ni_incld, t_atm, qv_sat_l, qv_sat_i, epsi, abi, qv, inv_dt, qidep, qi2qv_sublim_tend, ni_sublim_tend, qiberg) - end subroutine ice_deposition_sublimation_c - - subroutine ice_relaxation_timescale_c(rho, temp, rhofaci, table_val_qi2qr_melting, table_val_qi2qr_vent_melt, & - dv, mu, sc, qi_incld, ni_incld, & - epsi, epsi_tot) bind(C) - use micro_p3, only: calc_ice_relaxation_timescale - - ! arguments - real(kind=c_real), value, intent(in) :: rho, temp, rhofaci, table_val_qi2qr_melting, table_val_qi2qr_vent_melt, & - dv, mu, sc, qi_incld, ni_incld - real(kind=c_real), intent(out) :: epsi - real(kind=c_real), intent(inout) :: epsi_tot - - call calc_ice_relaxation_timescale(rho, temp, rhofaci, table_val_qi2qr_melting, table_val_qi2qr_vent_melt, & - dv, mu, sc, qi_incld, ni_incld, & - epsi, epsi_tot) - end subroutine ice_relaxation_timescale_c - - subroutine calc_liq_relaxation_timescale_c(rho, f1r, f2r, dv, mu, sc, mu_r, & - lamr, cdistr, cdist, qr_incld, & - qc_incld, epsr, epsc) bind(C) - use micro_p3, only: calc_liq_relaxation_timescale - - ! arguments - real(kind=c_real), value, intent(in) :: rho,f1r,f2r,dv,mu,sc,mu_r,lamr, & - cdistr,cdist,qr_incld,qc_incld - real(kind=c_real), intent(out) :: epsr - real(kind=c_real), intent(out) :: epsc - - call calc_liq_relaxation_timescale(rho,f1r,f2r,dv,mu,sc,mu_r,lamr, & - cdistr,cdist,qr_incld,qc_incld,epsr, & - epsc) - end subroutine calc_liq_relaxation_timescale_c - - subroutine ice_nucleation_c(temp, inv_rho, ni, ni_activated, qv_supersat_i, inv_dt, & - do_predict_nc, do_prescribed_CCN, qv2qi_nucleat_tend, ni_nucleat_tend) bind(C) - use micro_p3, only: ice_nucleation - - ! arguments - real(kind=c_real), value, intent(in) :: temp, inv_rho, ni, ni_activated, qv_supersat_i, inv_dt - logical(c_bool), value, intent(in) :: do_predict_nc - logical(c_bool), value, intent(in) :: do_prescribed_CCN - - real(kind=c_real), intent(inout) :: qv2qi_nucleat_tend, ni_nucleat_tend - - call ice_nucleation(temp, inv_rho, ni, ni_activated, qv_supersat_i, inv_dt, & - do_predict_nc, do_prescribed_CCN, qv2qi_nucleat_tend, ni_nucleat_tend) - end subroutine ice_nucleation_c - - subroutine ice_melting_c(rho,T_atm,pres,rhofaci,table_val_qi2qr_melting,table_val_qi2qr_vent_melt,latent_heat_vapor,latent_heat_fusion, & - dv,sc,mu,kap,qv,qi_incld,ni_incld,qi2qr_melt_tend,ni2nr_melt_tend) bind(C) - use micro_p3, only: ice_melting - - ! arguments: - real(kind=c_real), value, intent(in) :: rho,T_atm,pres,rhofaci,table_val_qi2qr_melting,table_val_qi2qr_vent_melt,latent_heat_vapor,latent_heat_fusion,dv,sc,mu,kap,qv,qi_incld,ni_incld - real(kind=c_real), intent(out) :: qi2qr_melt_tend,ni2nr_melt_tend - - call ice_melting(rho,T_atm,pres,rhofaci,table_val_qi2qr_melting,table_val_qi2qr_vent_melt,latent_heat_vapor,latent_heat_fusion,dv,sc,mu,kap,qv,qi_incld,ni_incld,qi2qr_melt_tend,ni2nr_melt_tend) - - end subroutine ice_melting_c - - subroutine ice_cldliq_wet_growth_c(rho, temp, pres, rhofaci, table_val_qi2qr_melting, & - table_val_qi2qr_vent_melt, latent_heat_vapor, latent_heat_fusion, dv, kap, mu, sc, qv, qc_incld, & - qi_incld, ni_incld, qr_incld, & - log_wetgrowth, qr2qi_collect_tend, qc2qi_collect_tend, qc_growth_rate, nr_ice_shed_tend, qc2qr_ice_shed_tend) bind(C) - use micro_p3, only: ice_cldliq_wet_growth - - ! argmens - real(kind=c_real), value, intent(in) :: rho, temp ,pres, rhofaci, table_val_qi2qr_melting, table_val_qi2qr_vent_melt, latent_heat_vapor, latent_heat_fusion, dv, & - kap, mu, sc, qv, qc_incld, qi_incld, ni_incld,qr_incld - logical(kind=c_bool), intent(inout) :: log_wetgrowth - real(kind=c_real), intent(inout) :: qr2qi_collect_tend, qc2qi_collect_tend, qc_growth_rate, nr_ice_shed_tend, qc2qr_ice_shed_tend - - call ice_cldliq_wet_growth(rho, temp, pres, rhofaci, table_val_qi2qr_melting, & - table_val_qi2qr_vent_melt, latent_heat_vapor, latent_heat_fusion, dv, kap, mu, sc, qv, qc_incld, & - qi_incld, ni_incld, qr_incld, & - log_wetgrowth, qr2qi_collect_tend, qc2qi_collect_tend, qc_growth_rate, nr_ice_shed_tend, qc2qr_ice_shed_tend) - end subroutine ice_cldliq_wet_growth_c - - subroutine get_latent_heat_c(its,ite,kts,kte,v,s,f) bind(C) - use micro_p3, only: get_latent_heat - - ! arguments - integer(kind=c_int), intent(in), value :: its, ite, kts, kte - real(kind=c_real), dimension(its:ite, kts:kte), intent(out) :: v, s, f - - call get_latent_heat(its,ite,kts,kte,v,s,f) - end subroutine get_latent_heat_c - - function subgrid_variance_scaling_c(relvar,expon) result(res) bind(C) - use micro_p3, only: subgrid_variance_scaling - - ! arguments - real(kind=c_real), value, intent(in) :: relvar,expon - real(kind=c_real) :: res - - res = subgrid_variance_scaling(relvar,expon) - return - end function subgrid_variance_scaling_c - - subroutine check_values_c(qv, temp, kts, kte, timestepcount, & - force_abort, source_ind, col_loc) bind(C) - use micro_p3, only: check_values - - ! argmens - real(kind=c_real), intent(in) :: qv(kts:kte), temp(kts:kte), col_loc(3) - integer(kind=c_int), value, intent(in) :: kts, kte, timestepcount, source_ind - logical(kind=c_bool), value, intent(in) :: force_abort - - call check_values(qv,Temp,kts,kte,timestepcount,force_abort,source_ind,col_loc) - end subroutine check_values_c - - subroutine calculate_incloud_mixingratios_c(qc, qr, qi, qm, nc, nr, ni, bm, & - inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r, & - qc_incld, qr_incld, qi_incld, qm_incld, & - nc_incld, nr_incld, ni_incld, bm_incld) bind(C) - use micro_p3, only: calculate_incloud_mixingratios - - ! argumens - real(kind=c_real), value, intent(in) :: qc, qr, qi, qm, nc, nr, ni, bm, inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r - real(kind=c_real), intent(inout) :: qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, ni_incld, bm_incld - - call calculate_incloud_mixingratios(qc, qr, qi, qm, nc, nr, ni, bm, & - inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r, & - qc_incld, qr_incld, qi_incld, qm_incld, & - nc_incld, nr_incld, ni_incld, bm_incld) - end subroutine calculate_incloud_mixingratios_c - - subroutine p3_main_part1_c(kts, kte, kbot, ktop, kdir, do_predict_nc, do_prescribed_CCN, dt, & - pres, dpres, dz, nc_nuceat_tend, inv_exner, exner, inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion, nccn_prescribed, & - T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, acn, qv, th_atm, qc, nc, qr, nr, & - qi, ni, qm, bm, qc_incld, qr_incld, qi_incld, qm_incld, & - nc_incld, nr_incld, ni_incld, bm_incld, is_nucleat_possible, is_hydromet_present) bind(C) - - use micro_p3, only: p3_main_part1 - - ! arguments - integer(kind=c_int), value, intent(in) :: kts, kte, kbot, ktop, kdir - logical(kind=c_bool), value, intent(in) :: do_predict_nc, do_prescribed_CCN - real(kind=c_real), value, intent(in) :: dt - - real(kind=c_real), intent(in), dimension(kts:kte) :: pres, dpres, dz, nc_nuceat_tend, inv_exner, exner, inv_cld_frac_l, inv_cld_frac_i, & - inv_cld_frac_r, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion, nccn_prescribed - - real(kind=c_real), intent(inout), dimension(kts:kte) :: T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, & - acn, qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, qc_incld, qr_incld, qi_incld, & - qm_incld, nc_incld, nr_incld, ni_incld, bm_incld - - logical(kind=c_bool), intent(out) :: is_nucleat_possible, is_hydromet_present - - call p3_main_part1(kts, kte, kbot, ktop, kdir, do_predict_nc, do_prescribed_CCN, dt, & - pres, dpres, dz, nc_nuceat_tend, inv_exner, exner, inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion, nccn_prescribed, & - T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, acn, qv, th_atm, qc, nc, qr, nr, & - qi, ni, qm, bm, qc_incld, qr_incld, qi_incld, qm_incld, & - nc_incld, nr_incld, ni_incld, bm_incld, is_nucleat_possible, is_hydromet_present) - - end subroutine p3_main_part1_c - - subroutine p3_main_part2_c(kts, kte, kbot, ktop, kdir, do_predict_nc, do_prescribed_CCN, dt, inv_dt, & - pres, inv_exner, inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r, & - ni_activated, inv_qc_relvar, cld_frac_i, cld_frac_l, cld_frac_r, qv_prev, t_prev, & - T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofaci, acn, qv, th_atm, qc, nc, qr, nr, qi, ni, & - qm, bm, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, & - ni_incld, bm_incld, mu_c, nu, lamc, cdist, cdist1, cdistr, mu_r, lamr, logn0r, qv2qi_depos_tend, precip_total_tend, & - nevapr, qr_evap_tend, vap_liq_exchange, vap_ice_exchange, liq_ice_exchange, pratot, & - prctot, is_hydromet_present) bind(C) - - use micro_p3, only: p3_main_part2 - - !arguments - integer(kind=c_int), value, intent(in) :: kts, kte, kbot, ktop, kdir - logical(kind=c_bool), value, intent(in) :: do_predict_nc, do_prescribed_CCN - real(kind=c_real), value, intent(in) :: dt, inv_dt - - real(kind=c_real), intent(in), dimension(kts:kte) :: pres, inv_exner, inv_cld_frac_l, inv_cld_frac_i, & - inv_cld_frac_r, ni_activated, inv_qc_relvar, cld_frac_i, cld_frac_l, cld_frac_r, qv_prev, t_prev - - real(kind=c_real), intent(inout), dimension(kts:kte) :: T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofaci, acn, & - qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion, qc_incld, qr_incld, & - qi_incld, qm_incld, nc_incld, nr_incld, ni_incld, bm_incld, mu_c, nu, lamc, cdist, cdist1, & - cdistr, mu_r, lamr, logn0r, qv2qi_depos_tend, precip_total_tend, nevapr, qr_evap_tend, vap_liq_exchange, & - vap_ice_exchange, liq_ice_exchange, pratot, prctot - - logical(kind=c_bool), intent(out) :: is_hydromet_present - - ! throwaway - real(kind=c_real), dimension(kts:kte,49) :: p3_tend_out - - call p3_main_part2(kts, kte, kbot, ktop, kdir, do_predict_nc, do_prescribed_CCN, dt, inv_dt, & - pres, inv_exner, inv_cld_frac_l, inv_cld_frac_i, inv_cld_frac_r, ni_activated, inv_qc_relvar, cld_frac_i, cld_frac_l, cld_frac_r, qv_prev, t_prev, & - T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofaci, acn, qv, th_atm, qc, nc, qr, nr, qi, ni, & - qm, bm, latent_heat_vapor, latent_heat_sublim, latent_heat_fusion, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, & - ni_incld, bm_incld, mu_c, nu, lamc, cdist, cdist1, cdistr, mu_r, lamr, logn0r, qv2qi_depos_tend, precip_total_tend, & - nevapr, qr_evap_tend, vap_liq_exchange, vap_ice_exchange, liq_ice_exchange, pratot, & - prctot, p3_tend_out, is_hydromet_present) - - end subroutine p3_main_part2_c - - subroutine p3_main_part3_c(kts, kte, kbot, ktop, kdir, & - inv_exner, cld_frac_l, cld_frac_r, cld_frac_i, & - rho, inv_rho, rhofaci, qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, latent_heat_vapor, latent_heat_sublim, & - mu_c, nu, lamc, mu_r, lamr, vap_liq_exchange, & - ze_rain, ze_ice, diag_vm_qi, diag_eff_radius_qi, diag_diam_qi, rho_qi, diag_equiv_reflectivity, diag_eff_radius_qc, diag_eff_radius_qr) bind(C) - - use micro_p3, only: p3_main_part3 - - ! args - - integer(kind=c_int), value, intent(in) :: kts, kte, kbot, ktop, kdir - real(kind=c_real), intent(in), dimension(kts:kte) :: inv_exner, cld_frac_l, cld_frac_r, cld_frac_i - real(kind=c_real), intent(inout), dimension(kts:kte) :: rho, inv_rho, rhofaci, & - qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, latent_heat_vapor, latent_heat_sublim, & - mu_c, nu, lamc, mu_r, & - lamr, vap_liq_exchange, & - ze_rain, ze_ice, diag_vm_qi, diag_eff_radius_qi, diag_diam_qi, rho_qi, & - diag_equiv_reflectivity, diag_eff_radius_qc, diag_eff_radius_qr - - call p3_main_part3(kts, kte, kbot, ktop, kdir, & - inv_exner, cld_frac_l, cld_frac_r, cld_frac_i, & - rho, inv_rho, rhofaci, qv, th_atm, qc, nc, qr, nr, qi, ni, qm, bm, latent_heat_vapor, latent_heat_sublim, & - mu_c, nu, lamc, mu_r, lamr, vap_liq_exchange, & - ze_rain, ze_ice, diag_vm_qi, diag_eff_radius_qi, diag_diam_qi, rho_qi, diag_equiv_reflectivity, diag_eff_radius_qc, diag_eff_radius_qr) - - end subroutine p3_main_part3_c - - subroutine ice_supersat_conservation_c(qidep, qinuc, cld_frac_i, qv, qv_sat_i, latent_heat_sublim, t_atm, dt, qi2qv_sublim_tend, qr2qv_evap_tend) bind(C) - use micro_p3, only : ice_supersat_conservation - - real(kind=c_real) , intent(inout) :: qidep, qinuc - real(kind=c_real) , value, intent(in) :: cld_frac_i, qv, qv_sat_i, latent_heat_sublim, t_atm, dt, qi2qv_sublim_tend, qr2qv_evap_tend - - call ice_supersat_conservation(qidep, qinuc, cld_frac_i, qv, qv_sat_i, latent_heat_sublim, t_atm, dt, qi2qv_sublim_tend, qr2qv_evap_tend) - end subroutine ice_supersat_conservation_c - subroutine nc_conservation_c(nc, nc_selfcollect_tend, dt, nc_collect_tend, nc2ni_immers_freeze_tend, nc_accret_tend, nc2nr_autoconv_tend) bind(C) - use micro_p3, only : nc_conservation - - real(kind=c_real) , value, intent(in) :: nc, nc_selfcollect_tend, dt - real(kind=c_real) , intent(inout) :: nc_collect_tend, nc2ni_immers_freeze_tend, nc_accret_tend, nc2nr_autoconv_tend - - call nc_conservation(nc, nc_selfcollect_tend, dt, nc_collect_tend, nc2ni_immers_freeze_tend, nc_accret_tend, nc2nr_autoconv_tend) - end subroutine nc_conservation_c - subroutine nr_conservation_c(nr, ni2nr_melt_tend, nr_ice_shed_tend, ncshdc, nc2nr_autoconv_tend, dt, nmltratio, nr_collect_tend, nr2ni_immers_freeze_tend, nr_selfcollect_tend, nr_evap_tend) bind(C) - use micro_p3, only : nr_conservation - - real(kind=c_real) , value, intent(in) :: nr, ni2nr_melt_tend, nr_ice_shed_tend, ncshdc, nc2nr_autoconv_tend, dt, nmltratio - real(kind=c_real) , intent(inout) :: nr_collect_tend, nr2ni_immers_freeze_tend, nr_selfcollect_tend, nr_evap_tend - - call nr_conservation(nr, ni2nr_melt_tend, nr_ice_shed_tend, ncshdc, nc2nr_autoconv_tend, dt, nmltratio, nr_collect_tend, nr2ni_immers_freeze_tend, nr_selfcollect_tend, nr_evap_tend) - end subroutine nr_conservation_c - subroutine ni_conservation_c(ni, ni_nucleat_tend, nr2ni_immers_freeze_tend, nc2ni_immers_freeze_tend, dt, ni2nr_melt_tend, ni_sublim_tend, ni_selfcollect_tend) bind(C) - use micro_p3, only : ni_conservation - - real(kind=c_real) , value, intent(in) :: ni, ni_nucleat_tend, nr2ni_immers_freeze_tend, nc2ni_immers_freeze_tend, dt - real(kind=c_real) , intent(inout) :: ni2nr_melt_tend, ni_sublim_tend, ni_selfcollect_tend - - call ni_conservation(ni, ni_nucleat_tend, nr2ni_immers_freeze_tend, nc2ni_immers_freeze_tend, dt, ni2nr_melt_tend, ni_sublim_tend, ni_selfcollect_tend) - end subroutine ni_conservation_c - subroutine prevent_liq_supersaturation_c(pres, t_atm, qv, latent_heat_vapor, latent_heat_sublim, dt, qidep, qinuc, qi2qv_sublim_tend, qr2qv_evap_tend) bind(C) - use micro_p3, only : prevent_liq_supersaturation - - real(kind=c_real) , value, intent(in) :: pres, t_atm, qv, latent_heat_vapor, latent_heat_sublim, dt, qidep, qinuc - real(kind=c_real) , intent(inout) :: qi2qv_sublim_tend, qr2qv_evap_tend - - call prevent_liq_supersaturation(pres, t_atm, qv, latent_heat_vapor, latent_heat_sublim, dt, qidep, qinuc, qi2qv_sublim_tend, qr2qv_evap_tend) - end subroutine prevent_liq_supersaturation_c end module p3_iso_c diff --git a/components/eamxx/src/physics/p3/p3_main_wrap.cpp b/components/eamxx/src/physics/p3/p3_main_wrap.cpp deleted file mode 100644 index ed46e281a67..00000000000 --- a/components/eamxx/src/physics/p3/p3_main_wrap.cpp +++ /dev/null @@ -1,76 +0,0 @@ -#include "p3_main_wrap.hpp" -#include "p3_f90.hpp" -#include "p3_functions_f90.hpp" -#include "physics_constants.hpp" -#include "p3_ic_cases.hpp" - -#include "ekat/ekat_assert.hpp" - -using scream::Real; -using scream::Int; -extern "C" { - void p3_main_c(Real* qc, Real* nc, Real* qr, Real* nr, Real* th_atm, - Real* qv, Real dt, Real* qi, Real* qm, - Real* ni, Real* bm, Real* pres, - Real* dz, Real* nc_nuceat_tend, Real* nccn_prescribed, Real* ni_activated, Real* inv_qc_relvar, - Int it, Real* precip_liq_surf, Real* precip_ice_surf, Int its, - Int ite, Int kts, Int kte, Real* diag_eff_radius_qc, Real* diag_eff_radius_qi, Real* diag_eff_radius_qr, - Real* rho_qi, bool do_predict_nc, bool do_prescribed_CCN, Real* dpres, Real* inv_exner, - Real* qv2qi_depos_tend, - Real* precip_liq_flux, Real* precip_ice_flux, // 1 extra column size - Real* cld_frac_r, Real* cld_frac_l, Real* cld_frac_i, - Real* liq_ice_exchange, Real* vap_liq_exchange, - Real* vap_ice_exchange, Real* qv_prev, Real* t_prev, Real* elapsed_s); -} - -namespace scream { -namespace p3 { - -Int p3_main_wrap(const FortranData& d, bool use_fortran) { - EKAT_REQUIRE_MSG(d.dt > 0, "invalid dt"); - if (use_fortran) { - Real elapsed_s; - p3_main_c(d.qc.data(), d.nc.data(), d.qr.data(), d.nr.data(), - d.th_atm.data(), d.qv.data(), d.dt, d.qi.data(), - d.qm.data(), d.ni.data(), d.bm.data(), - d.pres.data(), d.dz.data(), d.nc_nuceat_tend.data(), d.nccn_prescribed.data(), d.ni_activated.data(), d.inv_qc_relvar.data(), - d.it, d.precip_liq_surf.data(), d.precip_ice_surf.data(), 1, d.ncol, 1, d.nlev, - d.diag_eff_radius_qc.data(), d.diag_eff_radius_qi.data(), d.diag_eff_radius_qr.data(), d.rho_qi.data(), - d.do_predict_nc, d.do_prescribed_CCN, d.dpres.data(), d.inv_exner.data(), d.qv2qi_depos_tend.data(), - d.precip_liq_flux.data(), d.precip_ice_flux.data(), d.cld_frac_r.data(), d.cld_frac_l.data(), d.cld_frac_i.data(), - d.liq_ice_exchange.data(), d.vap_liq_exchange.data(),d.vap_ice_exchange.data(),d.qv_prev.data(),d.t_prev.data(), &elapsed_s); - return static_cast(elapsed_s * 1000000); - } - else { - return p3_main_f(d.qc.data(), d.nc.data(), d.qr.data(), d.nr.data(), d.th_atm.data(), - d.qv.data(), d.dt, d.qi.data(), d.qm.data(), d.ni.data(), - d.bm.data(), d.pres.data(), d.dz.data(), d.nc_nuceat_tend.data(), d.nccn_prescribed.data(), - d.ni_activated.data(), d.inv_qc_relvar.data(), d.it, d.precip_liq_surf.data(), - d.precip_ice_surf.data(), 1, d.ncol, 1, d.nlev, d.diag_eff_radius_qc.data(), - d.diag_eff_radius_qi.data(), d.diag_eff_radius_qr.data(), d.rho_qi.data(), d.do_predict_nc, d.do_prescribed_CCN, - d.dpres.data(), d.inv_exner.data(), d.qv2qi_depos_tend.data(), - d.precip_liq_flux.data(), d.precip_ice_flux.data(), - d.cld_frac_r.data(), d.cld_frac_l.data(), d.cld_frac_i.data(), - d.liq_ice_exchange.data(), d.vap_liq_exchange.data(), - d.vap_ice_exchange.data(),d.qv_prev.data(),d.t_prev.data() ); - - } -} - -int test_p3_init () { - p3_init(); - P3GlobalForFortran::deinit(); - return 0; -} - -int test_p3_ic (bool use_fortran) { - const auto d = ic::Factory::create(ic::Factory::mixed); - d->dt = 300.0; - p3_init(); - p3_main_wrap(*d, use_fortran); - P3GlobalForFortran::deinit(); - return 0; -} - -} // namespace p3 -} // namespace scream diff --git a/components/eamxx/src/physics/p3/p3_tables_setup.cpp b/components/eamxx/src/physics/p3/p3_tables_setup.cpp index 8582674d1fe..ec3f5ccbb43 100644 --- a/components/eamxx/src/physics/p3/p3_tables_setup.cpp +++ b/components/eamxx/src/physics/p3/p3_tables_setup.cpp @@ -1,6 +1,6 @@ // This is a tiny program that calls p3_init() to generate tables used by p3 -#include "physics/p3/p3_f90.hpp" +#include "physics/p3/p3_data.hpp" int main(int /* argc */, char** /* argv */) { scream::p3::p3_init(/* write_tables = */ true); diff --git a/components/eamxx/src/physics/p3/tests/CMakeLists.txt b/components/eamxx/src/physics/p3/tests/CMakeLists.txt index 217c2945e48..d705b30cc4e 100644 --- a/components/eamxx/src/physics/p3/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/tests/CMakeLists.txt @@ -1,5 +1,7 @@ include(ScreamUtils) +add_subdirectory(infra) + set(P3_TESTS_SRCS p3_tests.cpp p3_unit_tests.cpp @@ -44,75 +46,69 @@ else () set (FORCE_RUN_DIFF_FAILS "") endif() -# NOTE: tests inside this if statement won't be built in a baselines-only build -if (NOT SCREAM_ONLY_GENERATE_BASELINES) - CreateUnitTest(p3_tests "${P3_TESTS_SRCS}" - LIBS p3 - THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} - LABELS "p3;physics") - - # Make sure that a diff in the two implementation triggers a failed test (in debug only) - CreateUnitTest (p3_tests_fail p3_rain_sed_unit_tests.cpp - LIBS p3 - COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF - THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} - LABELS "p3;physics;fail" - ${FORCE_RUN_DIFF_FAILS}) - - if (NOT SCREAM_P3_SMALL_KERNELS) - CreateUnitTest(p3_sk_tests "${P3_TESTS_SRCS}" - LIBS p3_sk - THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} - LABELS "p3_sk;physics") - - # Make sure that a diff in the two implementation triggers a failed test (in debug only) - CreateUnitTest (p3_sk_tests_fail p3_rain_sed_unit_tests.cpp - LIBS p3_sk - COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF - THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} - LABELS "p3_sk;physics;fail" - ${FORCE_RUN_DIFF_FAILS}) - endif() -endif() - +# All tests should understand the same baseline args if (SCREAM_ENABLE_BASELINE_TESTS) if (SCREAM_ONLY_GENERATE_BASELINES) - set(BASELINE_FILE_ARG "-g -b ${SCREAM_BASELINES_DIR}/data/p3_run_and_cmp.baseline") + set(BASELINE_FILE_ARG "-g -b ${SCREAM_BASELINES_DIR}/data") + # We don't want to do thread spreads when generating. That + # could cause race conditions in the file system. + set(P3_THREADS "${SCREAM_TEST_MAX_THREADS}") else() - set(BASELINE_FILE_ARG "-b ${SCREAM_BASELINES_DIR}/data/p3_run_and_cmp.baseline") + set(BASELINE_FILE_ARG "-c -b ${SCREAM_BASELINES_DIR}/data") + set(P3_THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC}) endif() +else() + set(BASELINE_FILE_ARG "-n") # no baselines + set(P3_THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC}) +endif() - CreateUnitTestExec(p3_run_and_cmp "p3_run_and_cmp.cpp" - LIBS p3 - EXCLUDE_MAIN_CPP) +CreateUnitTest(p3_tests "${P3_TESTS_SRCS}" + LIBS p3 p3_test_infra + EXE_ARGS "--args ${BASELINE_FILE_ARG}" + THREADS ${P3_THREADS} + LABELS "p3;physics;baseline_gen;baseline_cmp") - CreateUnitTestFromExec(p3_run_and_cmp_cxx p3_run_and_cmp - THREADS ${SCREAM_TEST_MAX_THREADS} - EXE_ARGS "${BASELINE_FILE_ARG}" - LABELS "p3;physics") +# Make sure that a diff in the two implementation triggers a failed test (in debug only) +# No need to run lots of different thread counts. +if (SCREAM_ENABLE_BASELINE_TESTS) + CreateUnitTest (p3_tests_fail p3_rain_sed_unit_tests.cpp + LIBS p3 p3_test_infra + EXE_ARGS "--args ${BASELINE_FILE_ARG}" + COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF + LABELS "p3;physics;fail" + ${FORCE_RUN_DIFF_FAILS}) +endif() - CreateUnitTestFromExec(p3_run_and_cmp_f90 p3_run_and_cmp - THREADS ${SCREAM_TEST_MAX_THREADS} - EXE_ARGS "-f ${BASELINE_FILE_ARG}" - LABELS "p3;physics") +# If small kernels are ON, we don't need a separate executable to test them. +# Also, we never want to generate baselines with this separate executable +if (NOT SCREAM_P3_SMALL_KERNELS AND NOT SCREAM_ONLY_GENERATE_BASELINES) + # Note: Only the p3_main test does something different when + # small kernels are on. The SK dispatch routines are mostly trivial + # and it's not worth adding tons of test infrastructure to support + # BFB unit tests for these. + CreateUnitTest(p3_sk_tests "p3_main_unit_tests.cpp" + LIBS p3_sk p3_test_infra + EXE_ARGS "--args ${BASELINE_FILE_ARG}" + THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} + LABELS "p3_sk;physics;baseline_cmp") +endif() - # Make sure that a diff from baselines triggers a failed test (in debug only) - CreateUnitTest(p3_run_and_cmp_cxx_fail "p3_run_and_cmp.cpp" - LIBS p3 - COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF - THREADS ${SCREAM_TEST_MAX_THREADS} - EXE_ARGS "${BASELINE_FILE_ARG}" - LABELS "p3;physics;fail" - EXCLUDE_MAIN_CPP - ${FORCE_RUN_DIFF_FAILS}) +# Note: the baseline_gen label label is really only used if SCREAM_ONLY_GENERATE_BASELINES=ON, but no harm adding it +CreateUnitTest(p3_run_and_cmp "p3_run_and_cmp.cpp" + LIBS p3 p3_test_infra + EXCLUDE_MAIN_CPP + THREADS ${SCREAM_TEST_MAX_THREADS} + EXE_ARGS "${BASELINE_FILE_ARG}" + LABELS "p3;physics;baseline_gen;baseline_cmp") - # By default, baselines should be created using all fortran (ctest -L baseline_gen). If the user wants - # to use CXX to generate their baselines, they should use "ctest -L baseline_gen_cxx". - # Note: the baseline_gen label label is really only used if SCREAM_ONLY_GENERATE_BASELINES=ON, but no harm adding it - if (SCREAM_TEST_MAX_THREADS GREATER 1) - # ECUT only adds _ompX if we have more than one value of X, or if X>1 - set (TEST_SUFFIX _omp${SCREAM_TEST_MAX_THREADS}) - endif() - set_tests_properties (p3_run_and_cmp_f90${TEST_SUFFIX} PROPERTIES LABELS "baseline_gen;baseline_cmp") - set_tests_properties (p3_run_and_cmp_cxx${TEST_SUFFIX} PROPERTIES LABELS "baseline_gen;cxx baseline_cmp") +# Make sure that a diff from baselines triggers a failed test (in debug only) +if (SCREAM_ENABLE_BASELINE_TESTS) + CreateUnitTest(p3_run_and_cmp_fail "p3_run_and_cmp.cpp" + LIBS p3 p3_test_infra + COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF + THREADS ${SCREAM_TEST_MAX_THREADS} + EXE_ARGS "${BASELINE_FILE_ARG}" + LABELS "p3;physics;fail" + EXCLUDE_MAIN_CPP + ${FORCE_RUN_DIFF_FAILS}) endif() diff --git a/components/eamxx/src/physics/p3/tests/infra/CMakeLists.txt b/components/eamxx/src/physics/p3/tests/infra/CMakeLists.txt new file mode 100644 index 00000000000..6f7093a33ec --- /dev/null +++ b/components/eamxx/src/physics/p3/tests/infra/CMakeLists.txt @@ -0,0 +1,15 @@ +set(INFRA_SRCS + p3_data.cpp + p3_ic_cases.cpp + p3_main_wrap.cpp + p3_test_data.cpp +) + +#crusher change +if (Kokkos_ENABLE_HIP) +set_source_files_properties(p3_test_data.cpp PROPERTIES COMPILE_FLAGS -O0) +endif() + +add_library(p3_test_infra ${INFRA_SRCS}) +target_link_libraries(p3_test_infra p3) +target_include_directories(p3_test_infra PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/components/eamxx/src/physics/p3/p3_f90.cpp b/components/eamxx/src/physics/p3/tests/infra/p3_data.cpp similarity index 75% rename from components/eamxx/src/physics/p3/p3_f90.cpp rename to components/eamxx/src/physics/p3/tests/infra/p3_data.cpp index 38bb84c416e..24fd6529f22 100644 --- a/components/eamxx/src/physics/p3/p3_f90.cpp +++ b/components/eamxx/src/physics/p3/tests/infra/p3_data.cpp @@ -1,4 +1,4 @@ -#include "p3_f90.hpp" +#include "p3_data.hpp" #include "physics_constants.hpp" #include "p3_ic_cases.hpp" @@ -6,17 +6,11 @@ using scream::Real; using scream::Int; -extern "C" { - void micro_p3_utils_init_c(Real Cpair, Real Rair, Real RH2O, Real RHO_H2O, - Real MWH2O, Real MWdry, Real gravit, Real LatVap, Real LatIce, - Real CpLiq, Real Tmelt, Real Pi, bool masterproc); - void p3_init_c(const char** lookup_file_dir, int* info, const bool& write_tables); -} namespace scream { namespace p3 { -FortranData::FortranData (Int ncol_, Int nlev_) +P3Data::P3Data (Int ncol_, Int nlev_) : ncol(ncol_), nlev(nlev_) { do_predict_nc = true; @@ -62,11 +56,11 @@ FortranData::FortranData (Int ncol_, Int nlev_) vap_ice_exchange = Array2("sum of vap-ice phase change tendenices", ncol, nlev); } -FortranDataIterator::FortranDataIterator (const FortranData::Ptr& d) { +P3DataIterator::P3DataIterator (const P3Data::Ptr& d) { init(d); } -void FortranDataIterator::init (const FortranData::Ptr& dp) { +void P3DataIterator::init (const P3Data::Ptr& dp) { d_ = dp; #define fdipb(name) \ fields_.push_back({#name, \ @@ -79,7 +73,7 @@ void FortranDataIterator::init (const FortranData::Ptr& dp) { fdipb(nc); fdipb(qr); fdipb(nr); fdipb(qi); fdipb(ni); fdipb(qm); fdipb(bm); fdipb(precip_liq_surf); fdipb(precip_ice_surf); fdipb(diag_eff_radius_qc); fdipb(diag_eff_radius_qi); fdipb(diag_eff_radius_qr); fdipb(rho_qi); - fdipb(dpres); fdipb(inv_exner); fdipb(qv2qi_depos_tend); + fdipb(dpres); fdipb(inv_exner); fdipb(qv2qi_depos_tend); fdipb(precip_liq_flux); fdipb(precip_ice_flux); fdipb(cld_frac_r); fdipb(cld_frac_l); fdipb(cld_frac_i); fdipb(liq_ice_exchange); fdipb(vap_liq_exchange); @@ -87,33 +81,14 @@ void FortranDataIterator::init (const FortranData::Ptr& dp) { #undef fdipb } -const FortranDataIterator::RawArray& -FortranDataIterator::getfield (Int i) const { +const P3DataIterator::RawArray& +P3DataIterator::getfield (Int i) const { EKAT_ASSERT(i >= 0 || i < nfield()); return fields_[i]; } -void micro_p3_utils_init (const bool masterproc) { - using c = scream::physics::Constants; - micro_p3_utils_init_c(c::Cpair, c::Rair, c::RH2O, c::RHO_H2O, - c::MWH2O, c::MWdry, c::gravit, c::LatVap, c::LatIce, - c::CpLiq, c::Tmelt, c::Pi, masterproc); -} - -void p3_init (const bool write_tables, const bool masterproc) { - static bool is_init = false; - if (!is_init) { - micro_p3_utils_init(masterproc); - static const char* dir = SCREAM_DATA_DIR "/tables"; - Int info; - p3_init_c(&dir, &info, write_tables); - EKAT_REQUIRE_MSG(info == 0, "p3_init_c returned info " << info); - is_init = true; - } -} - -int test_FortranData () { - FortranData d(11, 72); +int test_P3Data () { + P3Data d(11, 72); return 0; } diff --git a/components/eamxx/src/physics/p3/p3_f90.hpp b/components/eamxx/src/physics/p3/tests/infra/p3_data.hpp similarity index 68% rename from components/eamxx/src/physics/p3/p3_f90.hpp rename to components/eamxx/src/physics/p3/tests/infra/p3_data.hpp index d07524d9c7a..df5b25e311a 100644 --- a/components/eamxx/src/physics/p3/p3_f90.hpp +++ b/components/eamxx/src/physics/p3/tests/infra/p3_data.hpp @@ -1,5 +1,5 @@ -#ifndef SCREAM_P3_F90_HPP -#define SCREAM_P3_F90_HPP +#ifndef SCREAM_P3_DATA_HPP +#define SCREAM_P3_DATA_HPP #include "share/scream_types.hpp" @@ -9,9 +9,9 @@ namespace scream { namespace p3 { -// Data format we can use to communicate with Fortran version. -struct FortranData { - typedef std::shared_ptr Ptr; +// Data format we can use to store (and read/write) data for a full P3 run. +struct P3Data { + typedef std::shared_ptr Ptr; using KT = KokkosTypes; using Scalar = Real; @@ -36,41 +36,38 @@ struct FortranData { Array3 p3_tend_out; Array2 liq_ice_exchange,vap_liq_exchange,vap_ice_exchange; - FortranData(Int ncol, Int nlev); + P3Data(Int ncol, Int nlev); }; -// Iterate over a FortranData's arrays. For examples, see Baseline::write, read. -struct FortranDataIterator { +// Iterate over a P3Data's arrays. For examples, see Baseline::write, read. +struct P3DataIterator { struct RawArray { std::string name; Int dim; Int extent[3]; - FortranData::Scalar* data; - FortranData::Array1::size_type size; + P3Data::Scalar* data; + P3Data::Array1::size_type size; }; - explicit FortranDataIterator(const FortranData::Ptr& d); + explicit P3DataIterator(const P3Data::Ptr& d); Int nfield () const { return fields_.size(); } const RawArray& getfield(Int i) const; private: - FortranData::Ptr d_; + P3Data::Ptr d_; std::vector fields_; - void init(const FortranData::Ptr& d); + void init(const P3Data::Ptr& d); }; -void p3_init(const bool write_tables = false, - const bool masterproc = false); - // We will likely want to remove these checks in the future, as we're not tied // to the exact implementation or arithmetic in P3. For now, these checks are // here to establish that the initial regression-testing code gives results that // match the python f2py tester, without needing a data file. -Int check_against_python(const FortranData& d); +Int check_against_python(const P3Data& d); -int test_FortranData(); +int test_P3Data(); } // namespace p3 } // namespace scream diff --git a/components/eamxx/src/physics/p3/p3_ic_cases.cpp b/components/eamxx/src/physics/p3/tests/infra/p3_ic_cases.cpp similarity index 95% rename from components/eamxx/src/physics/p3/p3_ic_cases.cpp rename to components/eamxx/src/physics/p3/tests/infra/p3_ic_cases.cpp index 1560dc0afb2..52b70de6025 100644 --- a/components/eamxx/src/physics/p3/p3_ic_cases.cpp +++ b/components/eamxx/src/physics/p3/tests/infra/p3_ic_cases.cpp @@ -8,12 +8,12 @@ namespace p3 { namespace ic { // From mixed_case_data.py in scream-docs at commit 4bbea4. -FortranData::Ptr make_mixed (const Int ncol, const Int nlev) { +P3Data::Ptr make_mixed (const Int ncol, const Int nlev) { using consts = scream::physics::Constants; const Int nk = nlev; Int k; - const auto dp = std::make_shared(ncol, nk); + const auto dp = std::make_shared(ncol, nk); auto& d = *dp; for (Int i = 0; i < ncol; ++i) { @@ -66,7 +66,7 @@ FortranData::Ptr make_mixed (const Int ncol, const Int nlev) { // To get potential temperature, start by making absolute temperature vary // between 150K at top of atmos and 300k at surface, then convert to potential // temp. - FortranData::Array1 T_atm("T", nk); + P3Data::Array1 T_atm("T", nk); for (k = 0; k < nk; ++k) { T_atm(k) = 150 + 150/double(nk)*k; if (i > 0) T_atm(k) += ((i % 3) - 0.5)/double(nk)*k; @@ -119,7 +119,7 @@ FortranData::Ptr make_mixed (const Int ncol, const Int nlev) { return dp; } -FortranData::Ptr Factory::create (IC ic, Int ncol, Int nlev) { +P3Data::Ptr Factory::create (IC ic, Int ncol, Int nlev) { switch (ic) { case mixed: return make_mixed(ncol, nlev); default: diff --git a/components/eamxx/src/physics/p3/p3_ic_cases.hpp b/components/eamxx/src/physics/p3/tests/infra/p3_ic_cases.hpp similarity index 64% rename from components/eamxx/src/physics/p3/p3_ic_cases.hpp rename to components/eamxx/src/physics/p3/tests/infra/p3_ic_cases.hpp index a8c461b06da..bbf6f133e51 100644 --- a/components/eamxx/src/physics/p3/p3_ic_cases.hpp +++ b/components/eamxx/src/physics/p3/tests/infra/p3_ic_cases.hpp @@ -1,18 +1,18 @@ #ifndef INCLUDE_SCREAM_P3_IC_CASES_HPP #define INCLUDE_SCREAM_P3_IC_CASES_HPP -#include "p3_f90.hpp" +#include "p3_data.hpp" namespace scream { namespace p3 { namespace ic { -FortranData::Ptr make_mixed(Int ncol); +P3Data::Ptr make_mixed(Int ncol); struct Factory { enum IC { mixed }; - static FortranData::Ptr create(IC ic, Int ncol = 1, Int nlev = 72); + static P3Data::Ptr create(IC ic, Int ncol = 1, Int nlev = 72); }; } // namespace ic diff --git a/components/eamxx/src/physics/p3/tests/infra/p3_main_wrap.cpp b/components/eamxx/src/physics/p3/tests/infra/p3_main_wrap.cpp new file mode 100644 index 00000000000..2b758a513ec --- /dev/null +++ b/components/eamxx/src/physics/p3/tests/infra/p3_main_wrap.cpp @@ -0,0 +1,51 @@ +#include "p3_main_wrap.hpp" +#include "p3_data.hpp" +#include "p3_test_data.hpp" +#include "physics_constants.hpp" +#include "p3_ic_cases.hpp" + +#include "ekat/ekat_assert.hpp" + +using scream::Real; +using scream::Int; + +namespace scream { +namespace p3 { + +Int p3_main_wrap(const P3Data& d) { + EKAT_REQUIRE_MSG(d.dt > 0, "invalid dt"); + return p3_main_host(d.qc.data(), d.nc.data(), d.qr.data(), d.nr.data(), d.th_atm.data(), + d.qv.data(), d.dt, d.qi.data(), d.qm.data(), d.ni.data(), + d.bm.data(), d.pres.data(), d.dz.data(), d.nc_nuceat_tend.data(), d.nccn_prescribed.data(), + d.ni_activated.data(), d.inv_qc_relvar.data(), d.it, d.precip_liq_surf.data(), + d.precip_ice_surf.data(), 1, d.ncol, 1, d.nlev, d.diag_eff_radius_qc.data(), + d.diag_eff_radius_qi.data(), d.diag_eff_radius_qr.data(), d.rho_qi.data(), d.do_predict_nc, d.do_prescribed_CCN, + d.dpres.data(), d.inv_exner.data(), d.qv2qi_depos_tend.data(), + d.precip_liq_flux.data(), d.precip_ice_flux.data(), + d.cld_frac_r.data(), d.cld_frac_l.data(), d.cld_frac_i.data(), + d.liq_ice_exchange.data(), d.vap_liq_exchange.data(), + d.vap_ice_exchange.data(),d.qv_prev.data(),d.t_prev.data() ); +} + +int test_p3_init () { + using P3F = Functions; + + P3F::p3_init(); + P3GlobalForFortran::deinit(); + return 0; +} + +int test_p3_ic () { + using P3F = Functions; + + const auto d = ic::Factory::create(ic::Factory::mixed); + + d->dt = 300.0; + P3F::p3_init(); + p3_main_wrap(*d); + P3GlobalForFortran::deinit(); + return 0; +} + +} // namespace p3 +} // namespace scream diff --git a/components/eamxx/src/physics/p3/p3_main_wrap.hpp b/components/eamxx/src/physics/p3/tests/infra/p3_main_wrap.hpp similarity index 71% rename from components/eamxx/src/physics/p3/p3_main_wrap.hpp rename to components/eamxx/src/physics/p3/tests/infra/p3_main_wrap.hpp index 7c980fa8a5b..c55007427cd 100644 --- a/components/eamxx/src/physics/p3/p3_main_wrap.hpp +++ b/components/eamxx/src/physics/p3/tests/infra/p3_main_wrap.hpp @@ -8,15 +8,14 @@ namespace scream { namespace p3 { -struct FortranData; +struct P3Data; // Returns number of microseconds of p3_main execution -Int p3_main_wrap(const FortranData& d, bool use_fortran=false); +Int p3_main_wrap(const P3Data& d); int test_p3_init(); -int test_p3_ic(bool use_fortran); - +int test_p3_ic(); } // namespace p3 } // namespace scream diff --git a/components/eamxx/src/physics/p3/p3_functions_f90.cpp b/components/eamxx/src/physics/p3/tests/infra/p3_test_data.cpp similarity index 64% rename from components/eamxx/src/physics/p3/p3_functions_f90.cpp rename to components/eamxx/src/physics/p3/tests/infra/p3_test_data.cpp index 7ce3f8aaa32..23a9998f43c 100644 --- a/components/eamxx/src/physics/p3/p3_functions_f90.cpp +++ b/components/eamxx/src/physics/p3/tests/infra/p3_test_data.cpp @@ -1,6 +1,6 @@ -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "ekat/kokkos/ekat_kokkos_types.hpp" -#include "p3_f90.hpp" +#include "p3_data.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "ekat/ekat_pack_kokkos.hpp" @@ -11,275 +11,22 @@ using scream::Real; using scream::Int; -// -// A C++ interface to micro_p3 fortran calls and vice versa -// extern "C" { void p3_init_a_c(Real* ice_table_vals, Real* collect_table_vals); -void find_lookuptable_indices_1a_c(Int* dumi, Int* dumjj, Int* dumii, Int* dumzz, - Real* dum1, Real* dum4, Real* dum5, Real* dum6, - Real qi, Real ni, Real qm, Real rhop); - -void find_lookuptable_indices_1b_c(Int* dumj, Real* dum3, Real qr, Real nr); - -void access_lookup_table_c(Int dumjj, Int dumii, Int dumi, Int index, - Real dum1, Real dum4, Real dum5, Real* proc); - -void access_lookup_table_coll_c(Int dumjj, Int dumii, Int dumj, Int dumi, Int index, - Real dum1, Real dum3, Real dum4, Real dum5, Real* proc); - -void back_to_cell_average_c(Real cld_frac_l_, Real cld_frac_r_, Real cld_frac_i_, - Real* qc2qr_accret_tend_, Real* qr2qv_evap_tend_, Real* qc2qr_autoconv_tend_, - Real* nc_accret_tend_, Real* nc_selfcollect_tend_, Real* nc2nr_autoconv_tend_, - Real* nr_selfcollect_tend_, Real* nr_evap_tend_, Real* ncautr_, - Real* qi2qv_sublim_tend_, - Real* nr_ice_shed_tend_, Real* qc2qi_hetero_freeze_tend_, Real* qr2qi_collect_tend_, - Real* qc2qr_ice_shed_tend_, Real* qi2qr_melt_tend_, Real* qc2qi_collect_tend_, - Real* qr2qi_immers_freeze_tend_, Real* ni2nr_melt_tend_, Real* nc_collect_tend_, - Real* ncshdc_, Real* nc2ni_immers_freeze_tend_, Real* nr_collect_tend_, - Real* ni_selfcollect_tend_, Real* qv2qi_vapdep_tend_, Real* nr2ni_immers_freeze_tend_, - Real* ni_sublim_tend_, Real* qv2qi_nucleat_tend_, Real* ni_nucleat_tend_, - Real* qc2qi_berg_tend_); - -void cloud_water_conservation_c(Real qc, Real dt, Real* qc2qr_autoconv_tend, Real* qc2qr_accret_tend, Real* qc2qi_collect_tend, - Real* qc2qi_hetero_freeze_tend, Real* qc2qr_ice_shed_tend, Real* qc2qi_berg_tend, Real* qi2qv_sublim_tend, Real* qv2qi_vapdep_tend); - -void rain_water_conservation_c(Real qr, Real qc2qr_autoconv_tend, Real qc2qr_accret_tend, Real qi2qr_melt_tend, Real qc2qr_ice_shed_tend, - Real dt, Real* qr2qv_evap_tend, Real* qr2qi_collect_tend, Real* qr2qi_immers_freeze_tend); - -void ice_water_conservation_c(Real qi, Real qv2qi_vapdep_tend, Real qv2qi_nucleat_tend, Real qc2qi_berg_tend, Real qr2qi_collect_tend, Real qc2qi_collect_tend, - Real qr2qi_immers_freeze_tend, Real qc2qi_hetero_freeze_tend, Real dt, Real* qi2qv_sublim_tend, Real* qi2qr_melt_tend); - -void get_cloud_dsd2_c(Real qc, Real* nc, Real* mu_c, Real rho, Real* nu, Real* lamc, - Real* cdist, Real* cdist1); - -void get_rain_dsd2_c(Real qr, Real* nr, Real* mu_r, Real* lamr, Real* cdistr, Real* logn0r); - -void calc_rime_density_c(Real T_atm, Real rhofaci, Real table_val_qi_fallspd, Real acn, - Real lamc, Real mu_c, Real qc_incld, Real qc2qi_collect_tend, - Real* vtrmi1, Real* rho_qm_cloud); - -void cldliq_immersion_freezing_c(Real T_atm, Real lamc, Real mu_c, Real cdist1, - Real qc_incld, Real inv_qc_relvar, Real* qc2qi_hetero_freeze_tend, Real* nc2ni_immers_freeze_tend); - -void rain_immersion_freezing_c(Real T_atm, Real lamr, Real mu_r, Real cdistr, - Real qr_incld, Real* qr2qi_immers_freeze_tend, Real* nr2ni_immers_freeze_tend); - -void droplet_self_collection_c(Real rho, Real inv_rho, Real qc_incld, Real mu_c, - Real nu, Real nc2nr_autoconv_tend, Real* nc_accret_tend); - -void cloud_rain_accretion_c(Real rho, Real inv_rho, Real qc_incld, Real nc_incld, - Real qr_incld, Real inv_qc_relvar, Real* qc2qr_accret_tend, Real* nc_accret_tend); - -void cloud_water_autoconversion_c(Real rho, Real qc_incld, Real nc_incld, Real inv_qc_relvar, Real* qc2qr_autoconv_tend, Real* nc2nr_autoconv_tend, Real* ncautr); - -void rain_self_collection_c(Real rho, Real qr_incld, Real nr_incld, Real* nr_selfcollect_tend); - -void impose_max_total_ni_c(Real* ni_local, Real max_total_ni, Real inv_rho_local); - -void ice_melting_c(Real rho,Real T_atm,Real pres,Real rhofaci,Real table_val_qi2qr_melting,Real table_val_qi2qr_vent_melt, - Real latent_heat_vapor,Real latent_heat_fusion,Real dv,Real sc,Real mu,Real kap,Real qv,Real qi_incld, - Real ni_incld,Real* qi2qr_melt_tend,Real* ni2nr_melt_tend); - -void calc_first_order_upwind_step_c(Int kts, Int kte, Int kdir, Int kbot, Int k_qxtop, Real dt_sub, Real* rho, - Real* inv_rho, Real* inv_dz, Int num_arrays, Real** fluxes, Real** vs, Real** qnx); - -void generalized_sedimentation_c(Int kts, Int kte, Int kdir, Int k_qxtop, Int* k_qxbot, Int kbot, Real Co_max, - Real* dt_left, Real* prt_accum, Real* inv_dz, Real* inv_rho, Real* rho, - Int num_arrays, Real** vs, Real** fluxes, Real** qnx); -void cloud_sedimentation_c( - Int kts, Int kte, Int ktop, Int kbot, Int kdir, - Real* qc_incld, Real* rho, Real* inv_rho, Real* cld_frac_l, Real* acn, Real* inv_dz, - Real dt, Real inv_dt, bool do_predict_nc, - Real* qc, Real* nc, Real* nc_incld, Real* mu_c, Real* lamc, Real* precip_liq_surf, Real* qc_tend, Real* nc_tend); - -void ice_sedimentation_c( - Int kts, Int kte, Int ktop, Int kbot, Int kdir, - Real* rho, Real* inv_rho, Real* rhofaci, Real* cld_frac_i, Real* inv_dz, - Real dt, Real inv_dt, - Real* qi, Real* qi_incld, Real* ni, Real* qm, Real* qm_incld, Real* bm, Real* bm_incld, - Real* ni_incld, Real* precip_ice_surf, Real* qi_tend, Real* ni_tend); - -void rain_sedimentation_c( - Int kts, Int kte, Int ktop, Int kbot, Int kdir, - Real* qr_incld, Real* rho, Real* inv_rho, Real* rhofacr, Real* cld_frac_r, Real* inv_dz, - Real dt, Real inv_dt, - Real* qr, Real* nr, Real* nr_incld, Real* mu_r, Real* lamr, Real* precip_liq_surf, Real* precip_liq_flux, Real* qr_tend, Real* nr_tend); - -void calc_bulk_rho_rime_c(Real qi_tot, Real* qi_rim, Real* bi_rim, Real* rho_rime); - -void homogeneous_freezing_c( - Int kts, Int kte, Int ktop, Int kbot, Int kdir, - Real* T_atm, Real* inv_exner, Real* latent_heat_fusion, - Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, Real* bm, Real* th_atm); - -void get_time_space_phys_variables_c(Real T_atm, Real pres, Real rho, Real latent_heat_vapor, Real latent_heat_sublim, Real qv_sat_l, Real qv_sat_i, - Real* mu, Real* dv, Real* sc, Real* dqsdt, Real* dqsidt, Real* ab, Real* abi, Real* kap, Real* eii); - -void update_prognostic_ice_c( - Real qc2qi_hetero_freeze_tend, Real qc2qi_collect_tend, Real qc2qr_ice_shed_tend, Real nc_collect_tend, Real nc2ni_immers_freeze_tend, Real ncshdc, - Real qr2qi_collect_tend, Real nr_collect_tend, Real qr2qi_immers_freeze_tend, Real nr2ni_immers_freeze_tend, Real nr_ice_shed_tend, - Real qi2qr_melt_tend, Real ni2nr_melt_tend, Real qi2qv_sublim_tend, Real qv2qi_vapdep_tend, Real qv2qi_nucleat_tend, Real ni_nucleat_tend, - Real ni_selfcollect_tend, Real ni_sublim_tend, Real qc2qi_berg_tend, Real inv_exner, Real latent_heat_sublim, Real latent_heat_fusion, - bool do_predict_nc, bool log_wetgrowth, Real dt, Real nmltratio, - Real rho_qm_cloud, Real* th_atm, Real* qv, Real* qi, Real* ni, Real* qm, - Real* bm, Real* qc, Real* nc, Real* qr, Real* nr); - -void evaporate_rain_c( Real qr_incld, Real qc_incld, Real nr_incld, Real qi_incld, - Real cld_frac_l, Real cld_frac_r, Real qv, Real qv_prev, - Real qv_sat_l, Real qv_sat_i, Real ab, Real abi, - Real epsr, Real epsi_tot, Real t, Real t_prev, - Real latent_heat_sublim, Real dqsdt, Real dt, - Real* qr2qv_evap_tend, Real* nr_evap_tend); - -void update_prognostic_liquid_c( - Real qc2qr_accret_tend, Real nc_accret_tend, Real qc2qr_autoconv_tend, Real nc2nr_autoconv_tend, Real ncautr, - Real nc_selfcollect_tend, Real qr2qv_evap_tend, Real nr_evap_tend, Real nr_selfcollect_tend , bool do_predict_nc, bool do_prescribed_CCN, - Real inv_rho, Real inv_exner, Real latent_heat_vapor, Real dt, Real* th_atm, Real* qv, - Real* qc, Real* nc, Real* qr, Real* nr); - -void ice_deposition_sublimation_c(Real qi_incld, Real ni_incld, Real t_atm, Real qv_sat_l, Real qv_sat_i, Real epsi, Real abi, Real qv, Real inv_dt, Real* qidep, Real* qi2qv_sublim_tend, Real* ni_sublim_tend, Real* qiberg); - -void compute_rain_fall_velocity_c(Real qr_incld, Real rhofacr, - Real* nr_incld, Real* mu_r, Real* lamr, Real* V_qr, Real* V_nr); - -void ice_cldliq_collection_c(Real rho, Real temp, Real rhofaci, Real table_val_qc2qi_collect, - Real qi_incld,Real qc_incld, Real ni_incld, Real nc_incld, - Real* qc2qi_collect_tend, Real* nc_collect_tend, Real* qc2qr_ice_shed_tend, Real* ncshdc); - -void ice_rain_collection_c(Real rho, Real temp, Real rhofaci, Real logn0r, Real table_val_nr_collect, Real table_val_qr2qi_collect, - Real qi_incld, Real ni_incld, Real qr_incld, Real* qr2qi_collect_tend, Real* nr_collect_tend); - - -void ice_self_collection_c(Real rho, Real rhofaci, Real table_val_ni_self_collect, Real eii, - Real qm_incld, Real qi_incld, Real ni_incld, Real* ni_selfcollect_tend); - -void ice_relaxation_timescale_c(Real rho, Real temp, Real rhofaci, Real table_val_qi2qr_melting, Real table_val_qi2qr_vent_melt, - Real dv, Real mu, Real sc, Real qi_incld, Real ni_incld, - Real* epsi, Real* epsi_tot); - -void calc_liq_relaxation_timescale_c(Real rho, Real f1r, Real f2r, Real dv, - Real mu, Real sc, Real mu_r, Real lamr, - Real cdistr, Real cdist, Real qr_incld, - Real qc_incld, Real* epsr, Real* epsc); - -void ice_nucleation_c(Real temp, Real inv_rho, Real ni, Real ni_activated, - Real qv_supersat_i, Real inv_dt, bool do_predict_nc, bool do_prescribed_CCN, - Real* qv2qi_nucleat_tend, Real* ni_nucleat_tend); - -void ice_cldliq_wet_growth_c(Real rho, Real temp, Real pres, Real rhofaci, Real table_val_qi2qr_melting, - Real table_val_qi2qr_vent_melt, Real latent_heat_vapor, Real latent_heat_fusion, Real dv, - Real kap, Real mu, Real sc, Real qv, Real qc_incld, - Real qi_incld, Real ni_incld, Real qr_incld, bool* log_wetgrowth, - Real* qr2qi_collect_tend, Real* qc2qi_collect_tend, Real* qc_growth_rate, Real* nr_ice_shed_tend, Real* qc2qr_ice_shed_tend); - -void get_latent_heat_c(Int its, Int ite, Int kts, Int kte, Real* s, Real* v, Real* f); - -Real subgrid_variance_scaling_c(Real relvar, Real expon); - -void check_values_c(Real* qv, Real* temp, Int kts, Int kte, Int timestepcount, - Int force_abort, Int source_ind, Real* col_loc); - -void calculate_incloud_mixingratios_c(Real qc, Real qr, Real qi, Real qm, Real nc, Real nr, Real ni, Real bm, - Real inv_cld_frac_l, Real inv_cld_frac_i, Real inv_cld_frac_r, - Real* qc_incld, Real* qr_incld, Real* qi_incld, Real* qm_incld, - Real* nc_incld, Real* nr_incld, Real* ni_incld, Real* bm_incld); - -void p3_main_part1_c( - Int kts, Int kte, Int kbot, Int ktop, Int kdir, - bool do_predict_nc, bool do_prescribed_CCN, - Real dt, - Real* pres, Real* dpres, Real* dz, Real* nc_nuceat_tend, Real* nccn_prescribed, Real* inv_exner, Real* exner, Real* inv_cld_frac_l, Real* inv_cld_frac_i, - Real* inv_cld_frac_r, Real* latent_heat_vapor, Real* latent_heat_sublim, Real* latent_heat_fusion, - Real* T_atm, Real* rho, Real* inv_rho, Real* qv_sat_l, Real* qv_sat_i, Real* qv_supersat_i, Real* rhofacr, Real* rhofaci, - Real* acn, Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, Real* bm, Real* qc_incld, Real* qr_incld, Real* qi_incld, - Real* qm_incld, Real* nc_incld, Real* nr_incld, Real* ni_incld, Real* bm_incld, - bool* is_nucleat_possible, bool* is_hydromet_present); - -void p3_main_part2_c( - Int kts, Int kte, Int kbot, Int ktop, Int kdir, bool do_predict_nc, bool do_prescribed_CCN, Real dt, Real inv_dt, - Real* pres, Real* inv_exner, Real* inv_cld_frac_l, Real* inv_cld_frac_i, - Real* inv_cld_frac_r, Real* ni_activated, Real* inv_qc_relvar, Real* cld_frac_i, Real* cld_frac_l, Real* cld_frac_r, Real* qv_prev, Real* t_prev, - Real* T_atm, Real* rho, Real* inv_rho, Real* qv_sat_l, Real* qv_sat_i, Real* qv_supersat_i, Real* rhofaci, Real* acn, - Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, - Real* qm, Real* bm, Real* latent_heat_vapor, Real* latent_heat_sublim, Real* latent_heat_fusion, Real* qc_incld, - Real* qr_incld, Real* qi_incld, Real* qm_incld, Real* nc_incld, Real* nr_incld, - Real* ni_incld, Real* bm_incld, Real* mu_c, Real* nu, Real* lamc, Real* cdist, Real* cdist1, - Real* cdistr, Real* mu_r, Real* lamr, Real* logn0r, Real* qv2qi_depos_tend, Real* precip_total_tend, - Real* nevapr, Real* qr_evap_tend, Real* vap_liq_exchange, Real* vap_ice_exchange, Real* liq_ice_exchange, Real* pratot, - Real* prctot, bool* is_hydromet_present); - -void p3_main_part3_c( - Int kts, Int kte, Int kbot, Int ktop, Int kdir, - Real* inv_exner, Real* cld_frac_l, Real* cld_frac_r, Real* cld_frac_i, - Real* rho, Real* inv_rho, Real* rhofaci, Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, - Real* qi, Real* ni, Real* qm, Real* bm, Real* latent_heat_vapor, Real* latent_heat_sublim, - Real* mu_c, Real* nu, Real* lamc, Real* mu_r, Real* lamr, Real* vap_liq_exchange, - Real* ze_rain, Real* ze_ice, Real* diag_vm_qi, Real* diag_eff_radius_qi, Real* diag_diam_qi, Real* rho_qi, Real* diag_equiv_reflectivity, Real* diag_eff_radius_qc, Real* diag_eff_radius_qr); - -void p3_main_c( - Real* qc, Real* nc, Real* qr, Real* nr, Real* th_atm, Real* qv, Real dt, - Real* qi, Real* qm, Real* ni, Real* bm, Real* pres, Real* dz, - Real* nc_nuceat_tend, Real* nccn_prescribed, Real* ni_activated, Real* inv_qc_relvar, Int it, Real* precip_liq_surf, - Real* precip_ice_surf, Int its, Int ite, Int kts, Int kte, Real* diag_eff_radius_qc, - Real* diag_eff_radius_qi, Real* diag_eff_radius_qr, Real* rho_qi, bool do_predict_nc, bool do_prescribed, Real* dpres, Real* inv_exner, - Real* qv2qi_depos_tend, Real* precip_liq_flux, Real* precip_ice_flux, Real* cld_frac_r, Real* cld_frac_l, Real* cld_frac_i, - Real* liq_ice_exchange, Real* vap_liq_exchange, Real* vap_ice_exchange, Real* qv_prev, Real* t_prev, Real* elapsed_s); - -void ice_supersat_conservation_c(Real* qidep, Real* qinuc, Real cld_frac_i, Real qv, Real qv_sat_i, Real latent_heat_sublim, Real t_atm, Real dt, Real qi2qv_sublim_tend, Real qr2qv_evap_tend); -void nc_conservation_c(Real nc, Real nc_selfcollect_tend, Real dt, Real* nc_collect_tend, Real* nc2ni_immers_freeze_tend, Real* nc_accret_tend, Real* nc2nr_autoconv_tend); -void nr_conservation_c(Real nr, Real ni2nr_melt_tend, Real nr_ice_shed_tend, Real ncshdc, Real nc2nr_autoconv_tend, Real dt, Real nmltratio, Real* nr_collect_tend, Real* nr2ni_immers_freeze_tend, Real* nr_selfcollect_tend, Real* nr_evap_tend); -void ni_conservation_c(Real ni, Real ni_nucleat_tend, Real nr2ni_immers_freeze_tend, Real nc2ni_immers_freeze_tend, Real dt, Real* ni2nr_melt_tend, Real* ni_sublim_tend, Real* ni_selfcollect_tend); -void prevent_liq_supersaturation_c(Real pres, Real t_atm, Real qv, Real latent_heat_vapor, Real latent_heat_sublim, Real dt, Real qidep, Real qinuc, Real* qi2qv_sublim_tend, Real* qr2qv_evap_tend); } // extern "C" : end _c decls namespace scream { namespace p3 { -// -// In all C++ -> Fortran bridge functions you should see p3_init(). P3 needs -// to be initialized since most of its function depend on global tables to be -// populated. The 'true' argument is to set p3 to use its fortran implementations -// instead of calling back to C++. We want this behavior since it doesn't make much -// sense for C++ to bridge over to fortran only to have fortran bridge back to C++. -// If the client wanted the C++ implementation, they should just call it directly. -// - -void p3_init_a(P3InitAFortranData& d) -{ - p3_init(); // need to initialize p3 first so that tables are loaded - p3_init_a_c(d.ice_table_vals.data(), d.collect_table_vals.data()); -} - -void find_lookuptable_indices_1a(LookupIceData& d) -{ - p3_init(); // need to initialize p3 first so that tables are loaded - find_lookuptable_indices_1a_c(&d.dumi, &d.dumjj, &d.dumii, &d.dumzz, - &d.dum1, &d.dum4, &d.dum5, &d.dum6, - d.qi, d.ni, d.qm, d.rhop); -} - -void find_lookuptable_indices_1b(LookupIceDataB& d) -{ - p3_init(); - find_lookuptable_indices_1b_c(&d.dumj, &d.dum3, d.qr, d.nr); -} - -void access_lookup_table(AccessLookupTableData& d) +void p3_init_a(P3InitAP3Data& d) { - p3_init(); // need to initialize p3 first so that tables are loaded - access_lookup_table_c(d.lid.dumjj, d.lid.dumii, d.lid.dumi, d.index, - d.lid.dum1, d.lid.dum4, d.lid.dum5, &d.proc); -} + using P3F = Functions; -void access_lookup_table_coll(AccessLookupTableCollData& d) -{ - p3_init(); // need to initialize p3 first so that tables are loaded - access_lookup_table_coll_c(d.lid.dumjj, d.lid.dumii, d.lidb.dumj, d.lid.dumi, d.index, - d.lid.dum1, d.lidb.dum3, d.lid.dum4, d.lid.dum5, &d.proc); + P3F::p3_init(); // need to initialize p3 first so that tables are loaded + p3_init_a_c(d.ice_table_vals.data(), d.collect_table_vals.data()); } void BackToCellAverageData::randomize(std::mt19937_64& engine) @@ -322,155 +69,6 @@ void BackToCellAverageData::randomize(std::mt19937_64& engine) qc2qi_berg_tend = data_dist(engine); } -void back_to_cell_average(BackToCellAverageData& d) -{ - p3_init(); - back_to_cell_average_c(d.cld_frac_l, d.cld_frac_r, d.cld_frac_i, &d.qc2qr_accret_tend, &d.qr2qv_evap_tend, - &d.qc2qr_autoconv_tend, &d.nc_accret_tend, &d.nc_selfcollect_tend, &d.nc2nr_autoconv_tend, &d.nr_selfcollect_tend, &d.nr_evap_tend, &d.ncautr, - &d.qi2qv_sublim_tend, &d.nr_ice_shed_tend, &d.qc2qi_hetero_freeze_tend, &d.qr2qi_collect_tend, &d.qc2qr_ice_shed_tend, - &d.qi2qr_melt_tend, &d.qc2qi_collect_tend, &d.qr2qi_immers_freeze_tend, &d.ni2nr_melt_tend, &d.nc_collect_tend, &d.ncshdc, &d.nc2ni_immers_freeze_tend, - &d.nr_collect_tend, &d.ni_selfcollect_tend, &d.qv2qi_vapdep_tend, &d.nr2ni_immers_freeze_tend, &d.ni_sublim_tend, &d.qv2qi_nucleat_tend, &d.ni_nucleat_tend, - &d.qc2qi_berg_tend); -} - -void calc_rime_density(CalcRimeDensityData& d) -{ - p3_init(); - calc_rime_density_c(d.T_atm, d.rhofaci, d.table_val_qi_fallspd, d.acn, d.lamc, d.mu_c, - d.qc_incld, d.qc2qi_collect_tend, &d.vtrmi1, &d.rho_qm_cloud); -} - -void cldliq_immersion_freezing(CldliqImmersionFreezingData& d) -{ - p3_init(); - cldliq_immersion_freezing_c(d.T_atm, d.lamc, d.mu_c, d.cdist1, d.qc_incld, d.inv_qc_relvar, - &d.qc2qi_hetero_freeze_tend, &d.nc2ni_immers_freeze_tend); -} - -LatentHeatData::LatentHeatData(Int kts_, Int kte_, Int its_, Int ite_) : - PhysicsTestData( { {(ite_ - its_) + 1, (kte_ - kts_) + 1} }, - { {&v, &s, &f} }), - its(its_), ite(ite_), kts(kts_), kte(kte_) -{} - -void get_latent_heat(LatentHeatData& d) -{ - p3_init(); - d.transpose(); - get_latent_heat_c(d.its, d.ite, d.kts, d.kte, d.v, d.s, d.f); - d.transpose(); -} - -void droplet_self_collection(DropletSelfCollectionData& d) -{ - p3_init(); - droplet_self_collection_c(d.rho, d.inv_rho, d.qc_incld, d.mu_c, d.nu, d.nc2nr_autoconv_tend, - &d.nc_selfcollect_tend); -} - -void rain_immersion_freezing(RainImmersionFreezingData& d) -{ - p3_init(); - rain_immersion_freezing_c(d.T_atm, d.lamr, d.mu_r, d.cdistr, d.qr_incld, - &d.qr2qi_immers_freeze_tend, &d.nr2ni_immers_freeze_tend); -} - -void cloud_rain_accretion(CloudRainAccretionData& d) -{ - p3_init(); - cloud_rain_accretion_c(d.rho, d.inv_rho, d.qc_incld, d.nc_incld, d.qr_incld, d.inv_qc_relvar, - &d.qc2qr_accret_tend, &d.nc_accret_tend); -} - -void cloud_water_conservation(CloudWaterConservationData& d){ - p3_init(); - cloud_water_conservation_c(d.qc, d.dt, &d.qc2qr_autoconv_tend, &d.qc2qr_accret_tend, &d.qc2qi_collect_tend, &d.qc2qi_hetero_freeze_tend, - &d.qc2qr_ice_shed_tend, &d.qc2qi_berg_tend, &d.qi2qv_sublim_tend, &d.qv2qi_vapdep_tend); -} - -void rain_water_conservation(RainWaterConservationData& d){ - p3_init(); - rain_water_conservation_c(d.qr, d.qc2qr_autoconv_tend, d.qc2qr_accret_tend, d.qi2qr_melt_tend, d.qc2qr_ice_shed_tend, - d.dt, &d.qr2qv_evap_tend, &d.qr2qi_collect_tend, &d.qr2qi_immers_freeze_tend); -} - -void ice_water_conservation(IceWaterConservationData& d){ - p3_init(); - ice_water_conservation_c(d.qi, d.qv2qi_vapdep_tend, d.qv2qi_nucleat_tend, d.qc2qi_berg_tend, d.qr2qi_collect_tend, d.qc2qi_collect_tend, d.qr2qi_immers_freeze_tend, - d.qc2qi_hetero_freeze_tend, d.dt, &d.qi2qv_sublim_tend, &d.qi2qr_melt_tend); -} - -void cloud_water_autoconversion(CloudWaterAutoconversionData& d){ - p3_init(); - cloud_water_autoconversion_c(d.rho, d.qc_incld, d.nc_incld, d.inv_qc_relvar, - &d.qc2qr_autoconv_tend, &d.nc2nr_autoconv_tend, &d.ncautr); -} - -void rain_self_collection(RainSelfCollectionData& d){ - p3_init(); - rain_self_collection_c(d.rho, d.qr_incld, d.nr_incld, &d.nr_selfcollect_tend); -} - -void impose_max_total_ni(ImposeMaxTotalNiData& d){ - p3_init(); - impose_max_total_ni_c(&d.ni_local, d.max_total_ni, d.inv_rho_local); -} - -void get_cloud_dsd2(GetCloudDsd2Data& d) -{ - p3_init(); - Real nc_in = d.nc_in; - get_cloud_dsd2_c(d.qc, &nc_in, &d.mu_c, d.rho, &d.nu, &d.lamc, &d.cdist, &d.cdist1); - d.nc_out = nc_in; -} - -void get_rain_dsd2(GetRainDsd2Data& d) -{ - p3_init(); - Real nr_in = d.nr_in; - get_rain_dsd2_c(d.qr, &nr_in, &d.mu_r, &d.lamr, &d.cdistr, &d.logn0r); - d.nr_out = nr_in; -} - -void ice_cldliq_collection(IceCldliqCollectionData& d) -{ - p3_init(); - ice_cldliq_collection_c(d.rho, d.temp, d.rhofaci, d.table_val_qc2qi_collect, - d.qi_incld, d.qc_incld, d.ni_incld, d.nc_incld, - &d.qc2qi_collect_tend, &d.nc_collect_tend, &d.qc2qr_ice_shed_tend, &d.ncshdc); -} - -void ice_rain_collection(IceRainCollectionData& d) -{ - p3_init(); - ice_rain_collection_c(d.rho, d.temp, d.rhofaci, d.logn0r, d.table_val_nr_collect, d.table_val_qr2qi_collect, - d.qi_incld, d.ni_incld, d.qr_incld, - &d.qr2qi_collect_tend, &d.nr_collect_tend); -} - -void ice_self_collection(IceSelfCollectionData& d) -{ - p3_init(); - ice_self_collection_c(d.rho, d.rhofaci, d.table_val_ni_self_collect, d.eii, d.qm_incld, - d.qi_incld, d.ni_incld, - &d.ni_selfcollect_tend); -} - -void get_time_space_phys_variables(GetTimeSpacePhysVarsData& d) -{ - p3_init(); - get_time_space_phys_variables_c(d.T_atm, d.pres, d.rho, d.latent_heat_vapor, d.latent_heat_sublim, d.qv_sat_l, d.qv_sat_i, &d.mu, &d.dv, - &d.sc, &d.dqsdt, &d.dqsidt, &d.ab, &d.abi, &d.kap, &d.eii); -} - -void ice_relaxation_timescale(IceRelaxationData& d) -{ - p3_init(); - ice_relaxation_timescale_c(d.rho, d.temp, d.rhofaci, d.table_val_qi2qr_melting, d.table_val_qi2qr_vent_melt, - d.dv, d.mu, d.sc, d.qi_incld, d.ni_incld, - &d.epsi, &d.epsi_tot); -} - void CalcLiqRelaxationData::randomize(std::mt19937_64& engine) { // Populate the struct's input fields with numbers between 0 and 1. @@ -489,31 +87,6 @@ void CalcLiqRelaxationData::randomize(std::mt19937_64& engine) qc_incld = data_dist(engine); } -void calc_liq_relaxation_timescale(CalcLiqRelaxationData& d) -{ - p3_init(); - calc_liq_relaxation_timescale_c(d.rho, d.f1r, d.f2r, d.dv, d.mu, d.sc, d.mu_r, - d.lamr, d.cdistr, d.cdist, d.qr_incld, d.qc_incld, &d.epsr, &d.epsc); -} - -void ice_nucleation(IceNucleationData& d) -{ - p3_init(); - ice_nucleation_c(d.temp, d.inv_rho, d.ni, d.ni_activated, - d.qv_supersat_i, d.inv_dt, d.do_predict_nc, d.do_prescribed_CCN, &d.qv2qi_nucleat_tend, &d.ni_nucleat_tend); -} - -void ice_cldliq_wet_growth(IceWetGrowthData& d) -{ - p3_init(); - - ice_cldliq_wet_growth_c(d.rho, d.temp, d.pres, d.rhofaci, d.table_val_qi2qr_melting, - d.table_val_qi2qr_vent_melt, d.latent_heat_vapor, d.latent_heat_fusion, d.dv, - d.kap, d.mu, d.sc, d.qv, d.qc_incld, - d.qi_incld, d.ni_incld, d.qr_incld, &d.log_wetgrowth, - &d.qr2qi_collect_tend, &d.qc2qi_collect_tend, &d.qc_growth_rate, &d.nr_ice_shed_tend, &d.qc2qr_ice_shed_tend); -} - CheckValuesData::CheckValuesData( Int kts_, Int kte_, Int timestepcount_, Int source_ind_, bool force_abort_) : PhysicsTestData( { {(kte_-kts_)+1} }, @@ -523,57 +96,6 @@ CheckValuesData::CheckValuesData( EKAT_REQUIRE_MSG(nk() >= 3 || (kte == 1 && kts == 1), "nk too small to use for col_loc"); } -void check_values(CheckValuesData& d) -{ - p3_init(); - check_values_c(d.qv, d.temp, d.kts, d.kte, d.timestepcount, - d.force_abort, d.source_ind, d.col_loc); -} - -void calculate_incloud_mixingratios(IncloudMixingData& d) -{ - p3_init(); - - calculate_incloud_mixingratios_c(d.qc, d.qr, d.qi, d.qm, d.nc, d.nr, d.ni, d.bm, d.inv_cld_frac_l, d.inv_cld_frac_i, d.inv_cld_frac_r, - &d.qc_incld, &d.qr_incld, &d.qi_incld, &d.qm_incld, - &d.nc_incld, &d.nr_incld, &d.ni_incld, &d.bm_incld); - -} - -void update_prognostic_ice(P3UpdatePrognosticIceData& d){ - p3_init(); - update_prognostic_ice_c(d.qc2qi_hetero_freeze_tend, d.qc2qi_collect_tend, d.qc2qr_ice_shed_tend, d.nc_collect_tend, d.nc2ni_immers_freeze_tend, d.ncshdc, - d.qr2qi_collect_tend, d.nr_collect_tend, d.qr2qi_immers_freeze_tend, d.nr2ni_immers_freeze_tend, d.nr_ice_shed_tend, - d.qi2qr_melt_tend, d.ni2nr_melt_tend, d.qi2qv_sublim_tend, d.qv2qi_vapdep_tend, d.qv2qi_nucleat_tend, d.ni_nucleat_tend, - d.ni_selfcollect_tend, d.ni_sublim_tend, d.qc2qi_berg_tend, d.inv_exner, d.latent_heat_sublim, d.latent_heat_fusion, - d.do_predict_nc, d.log_wetgrowth, d.dt, d.nmltratio, - d.rho_qm_cloud, &d.th_atm, &d.qv, &d.qi, &d.ni, &d.qm, - &d.bm, &d.qc, &d.nc, &d.qr, &d.nr); -} - -void evaporate_rain(EvapRainData& d) -{ - p3_init(); - evaporate_rain_c(d.qr_incld,d.qc_incld,d.nr_incld,d.qi_incld, - d.cld_frac_l,d.cld_frac_r,d.qv,d.qv_prev,d.qv_sat_l,d.qv_sat_i, - d.ab,d.abi,d.epsr,d.epsi_tot,d.t,d.t_prev,d.latent_heat_sublim,d.dqsdt,d.dt, - &d.qr2qv_evap_tend,&d.nr_evap_tend); -} - -void update_prognostic_liquid(P3UpdatePrognosticLiqData& d){ - p3_init(); - update_prognostic_liquid_c(d.qc2qr_accret_tend, d.nc_accret_tend, d.qc2qr_autoconv_tend, d.nc2nr_autoconv_tend, d.ncautr, - d.nc_selfcollect_tend, d. qr2qv_evap_tend, d.nr_evap_tend, d.nr_selfcollect_tend , d.do_predict_nc, d.do_prescribed_CCN, - d.inv_rho, d.inv_exner, d.latent_heat_vapor, d.dt, &d.th_atm, &d.qv, - &d.qc, &d.nc, &d.qr, &d.nr); -} - -void ice_deposition_sublimation(IceDepositionSublimationData& d) -{ - p3_init(); - ice_deposition_sublimation_c(d.qi_incld, d.ni_incld, d.T_atm, d.qv_sat_l, d.qv_sat_i, d.epsi, d.abi, d.qv, d.inv_dt, &d.qv2qi_vapdep_tend, &d.qi2qv_sublim_tend, &d.ni_sublim_tend, &d.qc2qi_berg_tend); -} - CalcUpwindData::CalcUpwindData( Int kts_, Int kte_, Int kdir_, Int kbot_, Int k_qxtop_, Int num_arrays_, Real dt_sub_) : PhysicsTestData({ {(kte_ - kts_)+1, num_arrays_}, {(kte_ - kts_)+1} }, @@ -594,15 +116,6 @@ void CalcUpwindData::convert_to_ptr_arr(std::vector& mem_space, Real**& f qnx_ = mem_space.data() + num_arrays*2; } -void calc_first_order_upwind_step(CalcUpwindData& d) -{ - p3_init(); - std::vector tmp; - Real** fluxes, **vs, **qnx; - d.convert_to_ptr_arr(tmp, fluxes, vs, qnx); - calc_first_order_upwind_step_c(d.kts, d.kte, d.kdir, d.kbot, d.k_qxtop, d.dt_sub, d.rho, d.inv_rho, d.inv_dz, d.num_arrays, fluxes, vs, qnx); -} - GenSedData::GenSedData( Int kts_, Int kte_, Int kdir_, Int k_qxtop_, Int k_qxbot_, Int kbot_, Real Co_max_, Real dt_left_, Real prt_accum_, Int num_arrays_) : @@ -610,17 +123,6 @@ GenSedData::GenSedData( Co_max(Co_max_), k_qxbot(k_qxbot_), dt_left(dt_left_), prt_accum(prt_accum_) { } -void generalized_sedimentation(GenSedData& d) -{ - p3_init(); - std::vector tmp; - Real** fluxes, **vs, **qnx; - d.convert_to_ptr_arr(tmp, fluxes, vs, qnx); - generalized_sedimentation_c(d.kts, d.kte, d.kdir, d.k_qxtop, &d.k_qxbot, d.kbot, d.Co_max, - &d.dt_left, &d.prt_accum, d.inv_dz, d.inv_rho, d.rho, - d.num_arrays, fluxes, vs, qnx); -} - CloudSedData::CloudSedData( Int kts_, Int kte_, Int ktop_, Int kbot_, Int kdir_, Real dt_, Real inv_dt_, bool do_predict_nc_, Real precip_liq_surf_) : @@ -630,15 +132,6 @@ CloudSedData::CloudSedData( dt(dt_), inv_dt(inv_dt_), do_predict_nc(do_predict_nc_), precip_liq_surf(precip_liq_surf_) {} -void cloud_sedimentation(CloudSedData& d) -{ - p3_init(); - cloud_sedimentation_c(d.kts, d.kte, d.ktop, d.kbot, d.kdir, - d.qc_incld, d.rho, d.inv_rho, d.cld_frac_l, d.acn, d.inv_dz, - d.dt, d.inv_dt, d.do_predict_nc, - d.qc, d.nc, d.nc_incld, d.mu_c, d.lamc, &d.precip_liq_surf, d.qc_tend, d.nc_tend); -} - IceSedData::IceSedData( Int kts_, Int kte_, Int ktop_, Int kbot_, Int kdir_, Real dt_, Real inv_dt_, Real precip_ice_surf_) : @@ -648,15 +141,6 @@ IceSedData::IceSedData( dt(dt_), inv_dt(inv_dt_), precip_ice_surf(precip_ice_surf_) {} -void ice_sedimentation(IceSedData& d) -{ - p3_init(); - ice_sedimentation_c(d.kts, d.kte, d.ktop, d.kbot, d.kdir, - d.rho, d.inv_rho, d.rhofaci, d.cld_frac_i, d.inv_dz, d.dt, d.inv_dt, - d.qi, d.qi_incld, d.ni, d.qm, d.qm_incld, d.bm, d.bm_incld, d.ni_incld, - &d.precip_ice_surf, d.qi_tend, d.ni_tend); -} - RainSedData::RainSedData( Int kts_, Int kte_, Int ktop_, Int kbot_, Int kdir_, Real dt_, Real inv_dt_, Real precip_liq_surf_) : @@ -666,21 +150,6 @@ RainSedData::RainSedData( dt(dt_), inv_dt(inv_dt_), precip_liq_surf(precip_liq_surf_) {} -void rain_sedimentation(RainSedData& d) -{ - p3_init(); - rain_sedimentation_c(d.kts, d.kte, d.ktop, d.kbot, d.kdir, - d.qr_incld, d.rho, d.inv_rho, d.rhofacr, d.cld_frac_r, d.inv_dz, - d.dt, d.inv_dt, - d.qr, d.nr, d.nr_incld, d.mu_r, d.lamr, &d.precip_liq_surf, d.precip_liq_flux, d.qr_tend, d.nr_tend); -} - -void calc_bulk_rho_rime(CalcBulkRhoRimeData& d) -{ - p3_init(); - calc_bulk_rho_rime_c(d.qi_tot, &d.qi_rim, &d.bi_rim, &d.rho_rime); -} - HomogeneousFreezingData::HomogeneousFreezingData( Int kts_, Int kte_, Int ktop_, Int kbot_, Int kdir_) : PhysicsTestData( { {(kte_ - kts_) + 1} }, @@ -688,36 +157,9 @@ HomogeneousFreezingData::HomogeneousFreezingData( kts(kts_), kte(kte_), ktop(ktop_), kbot(kbot_), kdir(kdir_) {} -void homogeneous_freezing(HomogeneousFreezingData& d) -{ - p3_init(); - homogeneous_freezing_c(d.kts, d.kte, d.ktop, d.kbot, d.kdir, - d.T_atm, d.inv_exner, d.latent_heat_fusion, - d.qc, d.nc, d.qr, d.nr, d.qi, d.ni, d.qm, d.bm, d.th_atm); -} - -void ice_melting(IceMeltingData& d){ - p3_init(); - ice_melting_c(d.rho,d.T_atm,d.pres,d.rhofaci,d.table_val_qi2qr_melting,d.table_val_qi2qr_vent_melt, - d.latent_heat_vapor,d.latent_heat_fusion,d.dv,d.sc,d.mu,d.kap, - d.qv,d.qi_incld,d.ni_incld,&d.qi2qr_melt_tend,&d.ni2nr_melt_tend); -} - -Real subgrid_variance_scaling(SubgridVarianceScalingData& d){ - p3_init(); - return subgrid_variance_scaling_c(d.relvar,d.expon); -} - -void compute_rain_fall_velocity(ComputeRainFallVelocityData& d) -{ - p3_init(); - compute_rain_fall_velocity_c(d.qr_incld, d.rhofacr, - &d.nr_incld, &d.mu_r, &d.lamr, &d.V_qr, &d.V_nr); -} - P3MainPart1Data::P3MainPart1Data( Int kts_, Int kte_, Int kbot_, Int ktop_, Int kdir_, - bool do_predict_nc_, bool do_prescribed_CCN_, Real dt_) : + bool do_predict_nc_, bool do_prescribed_CCN_, Real dt_, bool, bool) : PhysicsTestData( { {(kte_ - kts_) + 1} }, { { &pres, &dpres, &dz, &nc_nuceat_tend, &inv_exner, &exner, &inv_cld_frac_l, &inv_cld_frac_i, &inv_cld_frac_r, &latent_heat_vapor, &latent_heat_sublim, &latent_heat_fusion, &nccn_prescribed, &T_atm, &rho, &inv_rho, &qv_sat_l, &qv_sat_i, &qv_supersat_i, &rhofacr, &rhofaci, @@ -727,26 +169,11 @@ P3MainPart1Data::P3MainPart1Data( do_predict_nc(do_predict_nc_), do_prescribed_CCN(do_prescribed_CCN_), dt(dt_) {} -void p3_main_part1(P3MainPart1Data& d) -{ - p3_init(); - p3_main_part1_c( - d.kts, d.kte, d.kbot, d.ktop, d.kdir, - d.do_predict_nc, d.do_prescribed_CCN, - d.dt, - d.pres, d.dpres, d.dz, d.nc_nuceat_tend, d.nccn_prescribed, d.inv_exner, d.exner, d.inv_cld_frac_l, d.inv_cld_frac_i, d.inv_cld_frac_r, d.latent_heat_vapor, - d.latent_heat_sublim, d.latent_heat_fusion, - d.T_atm, d.rho, d.inv_rho, d.qv_sat_l, d.qv_sat_i, d.qv_supersat_i, d.rhofacr, d.rhofaci, - d.acn, d.qv, d.th_atm, d.qc, d.nc, d.qr, d.nr, d.qi, d.ni, d.qm, d.bm, d.qc_incld, d.qr_incld, d.qi_incld, - d.qm_incld, d.nc_incld, d.nr_incld, d.ni_incld, d.bm_incld, - &d.is_nucleat_possible, &d.is_hydromet_present); -} - /////////////////////////////////////////////////////////////////////////////// P3MainPart2Data::P3MainPart2Data( Int kts_, Int kte_, Int kbot_, Int ktop_, Int kdir_, - bool do_predict_nc_, bool do_prescribed_CCN_, Real dt_) : + bool do_predict_nc_, bool do_prescribed_CCN_, Real dt_, Real, bool) : PhysicsTestData( { {(kte_ - kts_) + 1} }, { { &pres, &dpres, &dz, &nc_nuceat_tend, &inv_exner, &exner, &inv_cld_frac_l, &inv_cld_frac_i, &inv_cld_frac_r, &ni_activated, &inv_qc_relvar, &cld_frac_i, &cld_frac_l, &cld_frac_r, &qv_prev, &t_prev, &T_atm, &rho, &inv_rho, &qv_sat_l, &qv_sat_i, &qv_supersat_i, &rhofacr, &rhofaci, &acn, @@ -758,20 +185,6 @@ P3MainPart2Data::P3MainPart2Data( do_predict_nc(do_predict_nc_), do_prescribed_CCN(do_prescribed_CCN_), dt(dt_), inv_dt(1 / dt) {} -void p3_main_part2(P3MainPart2Data& d) -{ - p3_init(); - p3_main_part2_c( - d.kts, d.kte, d.kbot, d.ktop, d.kdir, d.do_predict_nc, d.do_prescribed_CCN, d.dt, d.inv_dt, - d.pres, d.inv_exner, d.inv_cld_frac_l, d.inv_cld_frac_i, d.inv_cld_frac_r, d.ni_activated, d.inv_qc_relvar, - d.cld_frac_i, d.cld_frac_l, d.cld_frac_r, d.qv_prev, d.t_prev, - d.T_atm, d.rho, d.inv_rho, d.qv_sat_l, d.qv_sat_i, d.qv_supersat_i, d.rhofaci, d.acn, d.qv, d.th_atm, d.qc, d.nc, d.qr, d.nr, d.qi, d.ni, - d.qm, d.bm, d.latent_heat_vapor, d.latent_heat_sublim, d.latent_heat_fusion, d.qc_incld, d.qr_incld, d.qi_incld, d.qm_incld, d.nc_incld, d.nr_incld, - d.ni_incld, d.bm_incld, d.mu_c, d.nu, d.lamc, d.cdist, d.cdist1, d.cdistr, d.mu_r, d.lamr, d.logn0r, d.qv2qi_depos_tend, d.precip_total_tend, - d.nevapr, d.qr_evap_tend, d.vap_liq_exchange, d.vap_ice_exchange, d.liq_ice_exchange, d.pratot, - d.prctot, &d.is_hydromet_present); -} - /////////////////////////////////////////////////////////////////////////////// P3MainPart3Data::P3MainPart3Data( @@ -787,21 +200,10 @@ P3MainPart3Data::P3MainPart3Data( kts(kts_), kte(kte_), kbot(kbot_), ktop(ktop_), kdir(kdir_) {} -void p3_main_part3(P3MainPart3Data& d) -{ - p3_init(); - p3_main_part3_c( - d.kts, d.kte, d.kbot, d.ktop, d.kdir, - d.inv_exner, d.cld_frac_l, d.cld_frac_r, d.cld_frac_i, - d.rho, d.inv_rho, d.rhofaci, d.qv, d.th_atm, d.qc, d.nc, d.qr, d.nr, d.qi, d.ni, d.qm, d.bm, d.latent_heat_vapor, d.latent_heat_sublim, - d.mu_c, d.nu, d.lamc, d.mu_r, d.lamr, d.vap_liq_exchange, - d. ze_rain, d.ze_ice, d.diag_vm_qi, d.diag_eff_radius_qi, d.diag_diam_qi, d.rho_qi, d.diag_equiv_reflectivity, d.diag_eff_radius_qc, d.diag_eff_radius_qr); -} - /////////////////////////////////////////////////////////////////////////////// P3MainData::P3MainData( - Int its_, Int ite_, Int kts_, Int kte_, Int it_, Real dt_, bool do_predict_nc_, bool do_prescribed_CCN_) : + Int its_, Int ite_, Int kts_, Int kte_, Int it_, Real dt_, bool do_predict_nc_, bool do_prescribed_CCN_, Real) : PhysicsTestData( { {(ite_ - its_) + 1, (kte_ - kts_) + 1}, {(ite_ - its_) + 1, (kte_ - kts_) + 2} }, { { &pres, &dz, &nc_nuceat_tend, &nccn_prescribed, &ni_activated, &dpres, &inv_exner, &cld_frac_i, &cld_frac_l, &cld_frac_r, &inv_qc_relvar, &qc, &nc, &qr, &nr, &qi, &qm, &ni, &bm, &qv, &th_atm, &qv_prev, &t_prev, @@ -812,51 +214,6 @@ P3MainData::P3MainData( its(its_), ite(ite_), kts(kts_), kte(kte_), it(it_), dt(dt_), do_predict_nc(do_predict_nc_), do_prescribed_CCN(do_prescribed_CCN_) {} -//This is the variable ordering from micro_p3.F90 -void p3_main(P3MainData& d) -{ - p3_init(); - d.transpose(); - p3_main_c( - d.qc, d.nc, d.qr, d.nr, d.th_atm, d.qv, d.dt, d.qi, d.qm, d.ni, - d.bm, d.pres, d.dz, d.nc_nuceat_tend, d.nccn_prescribed, d.ni_activated, d.inv_qc_relvar, d.it, d.precip_liq_surf, - d.precip_ice_surf, d.its, d.ite, d.kts, d.kte, d.diag_eff_radius_qc, d.diag_eff_radius_qi, d.diag_eff_radius_qr, - d.rho_qi, d.do_predict_nc, d.do_prescribed_CCN, d.dpres, d.inv_exner, d.qv2qi_depos_tend, - d.precip_liq_flux, d.precip_ice_flux, d.cld_frac_r, d.cld_frac_l, d.cld_frac_i, - d.liq_ice_exchange, d.vap_liq_exchange, d.vap_ice_exchange, d.qv_prev, d.t_prev, &d.elapsed_s); - d.transpose(); -} - -void ice_supersat_conservation(IceSupersatConservationData& d) -{ - p3_init(); - ice_supersat_conservation_c(&d.qidep, &d.qinuc, d.cld_frac_i, d.qv, d.qv_sat_i, d.latent_heat_sublim, d.t_atm, d.dt, d.qi2qv_sublim_tend, d.qr2qv_evap_tend); -} - -void nc_conservation(NcConservationData& d) -{ - p3_init(); - nc_conservation_c(d.nc, d.nc_selfcollect_tend, d.dt, &d.nc_collect_tend, &d.nc2ni_immers_freeze_tend, &d.nc_accret_tend, &d.nc2nr_autoconv_tend); -} - -void nr_conservation(NrConservationData& d) -{ - p3_init(); - nr_conservation_c(d.nr, d.ni2nr_melt_tend, d.nr_ice_shed_tend, d.ncshdc, d.nc2nr_autoconv_tend, d.dt, d.nmltratio, &d.nr_collect_tend, &d.nr2ni_immers_freeze_tend, &d.nr_selfcollect_tend, &d.nr_evap_tend); -} - -void ni_conservation(NiConservationData& d) -{ - p3_init(); - ni_conservation_c(d.ni, d.ni_nucleat_tend, d.nr2ni_immers_freeze_tend, d.nc2ni_immers_freeze_tend, d.dt, &d.ni2nr_melt_tend, &d.ni_sublim_tend, &d.ni_selfcollect_tend); -} - -void prevent_liq_supersaturation(PreventLiqSupersaturationData& d) -{ - p3_init(); - prevent_liq_supersaturation_c(d.pres, d.t_atm, d.qv, d.latent_heat_vapor, d.latent_heat_sublim, d.dt, d.qidep, d.qinuc, &d.qi2qv_sublim_tend, &d.qr2qv_evap_tend); -} - void IceSupersatConservationData::randomize(std::mt19937_64& engine) { std::uniform_real_distribution data_dist(0.0, 1.0); @@ -970,8 +327,6 @@ void PreventLiqSupersaturationData::randomize(std::mt19937_64& engine) */ } -// end _c impls - /////////////////////////////////////////////////////////////////////////////// std::shared_ptr P3GlobalForFortran::s_views; @@ -993,7 +348,7 @@ void P3GlobalForFortran::deinit() } // -// _f function definitions +// _host function definitions // template @@ -1006,7 +361,7 @@ std::vector ptr_to_arr(T** data, int n) } template -void calc_first_order_upwind_step_f_impl( +void calc_first_order_upwind_step_host_impl( Int kts, Int kte, Int kdir, Int kbot, Int k_qxtop, Real dt_sub, Real* rho, Real* inv_rho, Real* inv_dz, Real** fluxes, Real** vs, Real** qnx) @@ -1070,7 +425,7 @@ void calc_first_order_upwind_step_f_impl( } template -void generalized_sedimentation_f_impl( +void generalized_sedimentation_host_impl( Int kts, Int kte, Int kdir, Int k_qxtop, Int* k_qxbot, Int kbot, Real Co_max, Real* dt_left, Real* prt_accum, Real* inv_dz, Real* inv_rho, Real* rho, Real** vs, Real** fluxes, Real** qnx) @@ -1159,40 +514,40 @@ void generalized_sedimentation_f_impl( *k_qxbot = scalars[2] + 1; } -void calc_first_order_upwind_step_f( +void calc_first_order_upwind_step_host( Int kts, Int kte, Int kdir, Int kbot, Int k_qxtop, Real dt_sub, Real* rho, Real* inv_rho, Real* inv_dz, Int num_arrays, Real** fluxes, Real** vs, Real** qnx) { if (num_arrays == 1) { - calc_first_order_upwind_step_f_impl<1>(kts, kte, kdir, kbot, k_qxtop, dt_sub, rho, inv_rho, inv_dz, fluxes, vs, qnx); + calc_first_order_upwind_step_host_impl<1>(kts, kte, kdir, kbot, k_qxtop, dt_sub, rho, inv_rho, inv_dz, fluxes, vs, qnx); } else if (num_arrays == 2) { - calc_first_order_upwind_step_f_impl<2>(kts, kte, kdir, kbot, k_qxtop, dt_sub, rho, inv_rho, inv_dz, fluxes, vs, qnx); + calc_first_order_upwind_step_host_impl<2>(kts, kte, kdir, kbot, k_qxtop, dt_sub, rho, inv_rho, inv_dz, fluxes, vs, qnx); } else if (num_arrays == 4) { - calc_first_order_upwind_step_f_impl<4>(kts, kte, kdir, kbot, k_qxtop, dt_sub, rho, inv_rho, inv_dz, fluxes, vs, qnx); + calc_first_order_upwind_step_host_impl<4>(kts, kte, kdir, kbot, k_qxtop, dt_sub, rho, inv_rho, inv_dz, fluxes, vs, qnx); } else { EKAT_REQUIRE_MSG(false, "Unsupported num arrays in bridge calc_first_order_upwind_step_f: " << num_arrays); } } -void generalized_sedimentation_f( +void generalized_sedimentation_host( Int kts, Int kte, Int kdir, Int k_qxtop, Int* k_qxbot, Int kbot, Real Co_max, Real* dt_left, Real* prt_accum, Real* inv_dz, Real* inv_rho, Real* rho, Int num_arrays, Real** vs, Real** fluxes, Real** qnx) { if (num_arrays == 1) { - generalized_sedimentation_f_impl<1>(kts, kte, kdir, k_qxtop, k_qxbot, kbot, Co_max, dt_left, prt_accum, + generalized_sedimentation_host_impl<1>(kts, kte, kdir, k_qxtop, k_qxbot, kbot, Co_max, dt_left, prt_accum, inv_dz, inv_rho, rho, vs, fluxes, qnx); } else if (num_arrays == 2) { - generalized_sedimentation_f_impl<2>(kts, kte, kdir, k_qxtop, k_qxbot, kbot, Co_max, dt_left, prt_accum, + generalized_sedimentation_host_impl<2>(kts, kte, kdir, k_qxtop, k_qxbot, kbot, Co_max, dt_left, prt_accum, inv_dz, inv_rho, rho, vs, fluxes, qnx); } else if (num_arrays == 4) { - generalized_sedimentation_f_impl<4>(kts, kte, kdir, k_qxtop, k_qxbot, kbot, Co_max, dt_left, prt_accum, + generalized_sedimentation_host_impl<4>(kts, kte, kdir, k_qxtop, k_qxbot, kbot, Co_max, dt_left, prt_accum, inv_dz, inv_rho, rho, vs, fluxes, qnx); } else { @@ -1200,7 +555,7 @@ void generalized_sedimentation_f( } } -void cloud_sedimentation_f( +void cloud_sedimentation_host( Int kts, Int kte, Int ktop, Int kbot, Int kdir, Real* qc_incld, Real* rho, Real* inv_rho, Real* cld_frac_l, Real* acn, Real* inv_dz, Real dt, Real inv_dt, bool do_predict_nc, @@ -1267,7 +622,7 @@ void cloud_sedimentation_f( ekat::device_to_host({qc, nc, nc_incld, mu_c, lamc, qc_tend, nc_tend}, nk, inout_views); } -void ice_sedimentation_f( +void ice_sedimentation_host( Int kts, Int kte, Int ktop, Int kbot, Int kdir, Real* rho, Real* inv_rho, Real* rhofaci, Real* cld_frac_i, Real* inv_dz, Real dt, Real inv_dt, @@ -1340,7 +695,7 @@ void ice_sedimentation_f( ekat::device_to_host({qi, qi_incld, ni, ni_incld, qm, qm_incld, bm, bm_incld, qi_tend, ni_tend}, nk, inout_views); } -void rain_sedimentation_f( +void rain_sedimentation_host( Int kts, Int kte, Int ktop, Int kbot, Int kdir, Real* qr_incld, Real* rho, Real* inv_rho, Real* rhofacr, Real* cld_frac_r, Real* inv_dz, Real dt, Real inv_dt, @@ -1415,7 +770,7 @@ void rain_sedimentation_f( ekat::device_to_host({qr, nr, nr_incld, mu_r, lamr, qr_tend, nr_tend, precip_liq_flux}, sizes_out, inout_views); } -void homogeneous_freezing_f( +void homogeneous_freezing_host( Int kts, Int kte, Int ktop, Int kbot, Int kdir, Real* T_atm, Real* inv_exner, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, Real* bm, Real* th_atm) @@ -1476,7 +831,7 @@ void homogeneous_freezing_f( ekat::device_to_host({qc, nc, qr, nr, qi, ni, qm, bm, th_atm}, nk, inout_views); } -void check_values_f(Real* qv, Real* temp, Int kstart, Int kend, +void check_values_host(Real* qv, Real* temp, Int kstart, Int kend, Int timestepcount, bool force_abort, Int source_ind, Real* col_loc) { using P3F = Functions; @@ -1509,7 +864,7 @@ void check_values_f(Real* qv, Real* temp, Int kstart, Int kend, }); } -void p3_main_part1_f( +void p3_main_part1_host( Int kts, Int kte, Int kbot, Int ktop, Int kdir, bool do_predict_nc, bool do_prescribed_CCN, Real dt, @@ -1621,7 +976,7 @@ void p3_main_part1_f( *is_hydromet_present = bools_h(1); } -void p3_main_part2_f( +void p3_main_part2_host( Int kts, Int kte, Int kbot, Int ktop, Int kdir, bool do_predict_nc, bool do_prescribed_CCN, Real dt, Real inv_dt, Real* pres, Real* dpres, Real* dz, Real* nc_nuceat_tend, Real* inv_exner, Real* exner, Real* inv_cld_frac_l, Real* inv_cld_frac_i, Real* inv_cld_frac_r, Real* ni_activated, Real* inv_qc_relvar, Real* cld_frac_i, Real* cld_frac_l, Real* cld_frac_r, Real* qv_prev, Real* t_prev, @@ -1777,7 +1132,7 @@ void p3_main_part2_f( *is_hydromet_present = bools_h(0); } -void p3_main_part3_f( +void p3_main_part3_host( Int kts, Int kte, Int kbot, Int ktop, Int kdir, Real* inv_exner, Real* cld_frac_l, Real* cld_frac_r, Real* cld_frac_i, Real* rho, Real* inv_rho, Real* rhofaci, Real* qv, Real* th_atm, Real* qc, @@ -1885,7 +1240,7 @@ void p3_main_part3_f( nk, inout_views); } -Int p3_main_f( +Int p3_main_host( Real* qc, Real* nc, Real* qr, Real* nr, Real* th_atm, Real* qv, Real dt, Real* qi, Real* qm, Real* ni, Real* bm, Real* pres, Real* dz, Real* nc_nuceat_tend, Real* nccn_prescribed, Real* ni_activated, Real* inv_qc_relvar, Int it, Real* precip_liq_surf, diff --git a/components/eamxx/src/physics/p3/p3_functions_f90.hpp b/components/eamxx/src/physics/p3/tests/infra/p3_test_data.hpp similarity index 83% rename from components/eamxx/src/physics/p3/p3_functions_f90.hpp rename to components/eamxx/src/physics/p3/tests/infra/p3_test_data.hpp index 89e6ac56908..a1bbe864f4c 100644 --- a/components/eamxx/src/physics/p3/p3_functions_f90.hpp +++ b/components/eamxx/src/physics/p3/tests/infra/p3_test_data.hpp @@ -4,23 +4,40 @@ #include "physics/p3/p3_functions.hpp" #include "physics/share/physics_test_data.hpp" #include "share/scream_types.hpp" +#include "ekat/util/ekat_file_utils.hpp" #include #include #include // for shared_ptr -// -// Bridge functions to call fortran version of p3 functions from C++ -// - namespace scream { namespace p3 { -// +/////////////////////////////////////////////////////////////////////////////// + +struct P3InitAP3Data +{ + // Must use Host as device, f90 code might not be able to use Device memory + using P3F = Functions; + using P3C = typename P3F::P3C; + + using view_ice_table = typename P3F::KT::template lview; + using view_collect_table = typename P3F::KT::template lview; + + // Need to be LayoutLeft to be fortran compatible + view_ice_table ice_table_vals; + view_collect_table collect_table_vals; + + P3InitAP3Data() : + ice_table_vals("P3InitAP3Data::ice_table_vals"), + collect_table_vals("P3InitAP3Data::collect_table_vals") + {} +}; + +/////////////////////////////////////////////////////////////////////////////// + // Singleton for holding the same global data that are maintained in -// micro_p3, but for use in C++. This data is necessary to complete -// the "bridge" when calling C++ from micro_p3. -// +// micro_p3, but for use in C++. struct P3GlobalForFortran { using P3F = Functions; @@ -63,26 +80,10 @@ struct P3GlobalForFortran /////////////////////////////////////////////////////////////////////////////// -struct P3InitAFortranData -{ - // Must use Host as device, f90 code might not be able to use Device memory - using P3F = Functions; - using P3C = typename P3F::P3C; - - using view_ice_table = typename P3F::KT::template lview; - using view_collect_table = typename P3F::KT::template lview; - - // Need to be LayoutLeft to be fortran compatible - view_ice_table ice_table_vals; - view_collect_table collect_table_vals; - - P3InitAFortranData() : - ice_table_vals("P3InitAFortranData::ice_table_vals"), - collect_table_vals("P3InitAFortranData::collect_table_vals") - {} -}; - -/////////////////////////////////////////////////////////////////////////////// +/** + * Structs for holding data related to specific P3 calls; these are used for + * the BFB unit tests. + */ struct LookupIceData { @@ -92,6 +93,8 @@ struct LookupIceData // Outputs Int dumi, dumjj, dumii, dumzz; Real dum1, dum4, dum5, dum6; + + PTD_RW_SCALARS_ONLY(8, dumi, dumjj, dumii, dumzz, dum1, dum4, dum5, dum6); }; /////////////////////////////////////////////////////////////////////////////// @@ -104,6 +107,8 @@ struct LookupIceDataB // Outputs Int dumj; Real dum3; + + PTD_RW_SCALARS_ONLY(2, dumj, dum3); }; /////////////////////////////////////////////////////////////////////////////// @@ -116,6 +121,8 @@ struct AccessLookupTableData // Outputs Real proc; + + PTD_RW_SCALARS_ONLY(1, proc); }; /////////////////////////////////////////////////////////////////////////////// @@ -129,6 +136,8 @@ struct AccessLookupTableCollData // Outputs Real proc; + + PTD_RW_SCALARS_ONLY(1, proc); }; /////////////////////////////////////////////////////////////////////////////// @@ -146,6 +155,11 @@ struct BackToCellAverageData // This populates all fields with test data within [0,1]. void randomize(std::mt19937_64& engine); + + PTD_RW_SCALARS_ONLY(31, qc2qr_accret_tend, qr2qv_evap_tend, qc2qr_autoconv_tend, nc_accret_tend, nc_selfcollect_tend, nc2nr_autoconv_tend, nr_selfcollect_tend, nr_evap_tend, ncautr, qcnuc, + nc_nuceat_tend, qi2qv_sublim_tend, nr_ice_shed_tend, qc2qi_hetero_freeze_tend, qr2qi_collect_tend, qc2qr_ice_shed_tend, qi2qr_melt_tend, qc2qi_collect_tend, qr2qi_immers_freeze_tend, ni2nr_melt_tend, + nc_collect_tend, ncshdc, nc2ni_immers_freeze_tend, nr_collect_tend, ni_selfcollect_tend, qv2qi_vapdep_tend, nr2ni_immers_freeze_tend, ni_sublim_tend, qv2qi_nucleat_tend, ni_nucleat_tend, + qc2qi_berg_tend); }; /////////////////////////////////////////////////////////////////////////////// @@ -157,6 +171,8 @@ struct CloudWaterConservationData //output Real qc2qr_autoconv_tend, qc2qr_accret_tend, qc2qi_collect_tend, qc2qi_hetero_freeze_tend, qc2qr_ice_shed_tend, qc2qi_berg_tend, qi2qv_sublim_tend, qv2qi_vapdep_tend; + + PTD_RW_SCALARS_ONLY(8, qc2qr_autoconv_tend, qc2qr_accret_tend, qc2qi_collect_tend, qc2qi_hetero_freeze_tend, qc2qr_ice_shed_tend, qc2qi_berg_tend, qi2qv_sublim_tend, qv2qi_vapdep_tend); }; struct RainWaterConservationData @@ -166,6 +182,8 @@ struct RainWaterConservationData //output Real qr2qv_evap_tend, qr2qi_collect_tend, qr2qi_immers_freeze_tend; + + PTD_RW_SCALARS_ONLY(3, qr2qv_evap_tend, qr2qi_collect_tend, qr2qi_immers_freeze_tend); }; struct IceWaterConservationData @@ -175,6 +193,8 @@ struct IceWaterConservationData //output Real qi2qv_sublim_tend, qi2qr_melt_tend; + + PTD_RW_SCALARS_ONLY(2, qi2qv_sublim_tend, qi2qr_melt_tend); }; /////////////////////////////////////////////////////////////////////////////// @@ -186,6 +206,8 @@ struct CalcRimeDensityData // output Real vtrmi1, rho_qm_cloud; + + PTD_RW_SCALARS_ONLY(2, vtrmi1, rho_qm_cloud); }; /////////////////////////////////////////////////////////////////////////////// @@ -197,6 +219,8 @@ struct CldliqImmersionFreezingData // output Real qc2qi_hetero_freeze_tend, nc2ni_immers_freeze_tend; + + PTD_RW_SCALARS_ONLY(2, qc2qi_hetero_freeze_tend, nc2ni_immers_freeze_tend); }; /////////////////////////////////////////////////////////////////////////////// @@ -208,6 +232,8 @@ struct RainImmersionFreezingData // output Real qr2qi_immers_freeze_tend, nr2ni_immers_freeze_tend; + + PTD_RW_SCALARS_ONLY(2, qr2qi_immers_freeze_tend, nr2ni_immers_freeze_tend); }; /////////////////////////////////////////////////////////////////////////////// @@ -219,6 +245,8 @@ struct DropletSelfCollectionData // output Real nc_selfcollect_tend; + + PTD_RW_SCALARS_ONLY(1, nc_selfcollect_tend); }; /////////////////////////////////////////////////////////////////////////////// @@ -230,6 +258,8 @@ struct CloudRainAccretionData // output Real qc2qr_accret_tend, nc_accret_tend; + + PTD_RW_SCALARS_ONLY(2, qc2qr_accret_tend, nc_accret_tend); }; /////////////////////////////////////////////////////////////////////////////// @@ -237,15 +267,12 @@ struct CloudRainAccretionData struct CloudWaterAutoconversionData { // inputs - Real rho; - Real qc_incld; - Real nc_incld; - Real inv_qc_relvar; + Real rho, qc_incld, nc_incld, inv_qc_relvar; // output - Real qc2qr_autoconv_tend; - Real nc2nr_autoconv_tend; - Real ncautr; + Real qc2qr_autoconv_tend, nc2nr_autoconv_tend, ncautr; + + PTD_RW_SCALARS_ONLY(3, qc2qr_autoconv_tend, nc2nr_autoconv_tend, ncautr); }; /////////////////////////////////////////////////////////////////////////////// @@ -257,6 +284,8 @@ struct RainSelfCollectionData //output Real nr_selfcollect_tend; + + PTD_RW_SCALARS_ONLY(1, nr_selfcollect_tend); }; /////////////////////////////////////////////////////////////////////////////// @@ -267,6 +296,8 @@ struct ImposeMaxTotalNiData{ //input Real max_total_ni, inv_rho_local; + + PTD_RW_SCALARS_ONLY(2, ni_local, inv_rho_local); }; /////////////////////////////////////////////////////////////////////////////// @@ -278,6 +309,8 @@ struct IceMeltingData // output Real qi2qr_melt_tend,ni2nr_melt_tend; + + PTD_RW_SCALARS_ONLY(2, qi2qr_melt_tend, ni2nr_melt_tend); }; /////////////////////////////////////////////////////////////////////////////// @@ -298,6 +331,8 @@ struct GetCloudDsd2Data // Outputs Real nc_out, mu_c, nu, lamc, cdist, cdist1; + + PTD_RW_SCALARS_ONLY(6, nc_out, mu_c, nu, lamc, cdist, cdist1) }; ////////////////////////////////////////////////////////////////////////// @@ -309,6 +344,8 @@ struct GetRainDsd2Data // Outputs Real nr_out, lamr, mu_r, cdistr, logn0r; + + PTD_RW_SCALARS_ONLY(5, nr_out, lamr, mu_r, cdistr, logn0r); }; /////////////////////////////////////////////////////////////////////////////// @@ -352,6 +389,8 @@ struct GenSedData : public CalcUpwindData PTD_DATA_COPY_CTOR(GenSedData, 10); PTD_ASSIGN_OP(GenSedData, 11, kts, kte, kdir, kbot, k_qxtop, num_arrays, dt_sub, Co_max, k_qxbot, dt_left, prt_accum); + PTD_RW(); + PTD_RW_SCALARS(11, kts, kte, kdir, kbot, k_qxtop, num_arrays, dt_sub, Co_max, k_qxbot, dt_left, prt_accum); }; /////////////////////////////////////////////////////////////////////////////// @@ -437,6 +476,8 @@ struct CalcBulkRhoRimeData // Outputs Real rho_rime; + + PTD_RW_SCALARS_ONLY(3, qi_rim, bi_rim, rho_rime); }; /////////////////////////////////////////////////////////////////////////////// @@ -471,6 +512,8 @@ struct ComputeRainFallVelocityData // Outputs Real mu_r, lamr, V_qr, V_nr; + + PTD_RW_SCALARS_ONLY(5, nr_incld, mu_r, lamr, V_qr, V_nr); }; /////////////////////////////////////////////////////////////////////////////// @@ -482,6 +525,8 @@ struct GetTimeSpacePhysVarsData //Outs Real mu, dv, sc, dqsdt, dqsidt, ab, abi, kap, eii; + + PTD_RW_SCALARS_ONLY(9, mu, dv, sc, dqsdt, dqsidt, ab, abi, kap, eii); }; /////////////////////////////////////////////////////////////////////////////// @@ -496,6 +541,8 @@ struct P3UpdatePrognosticIceData // In/outs Real th_atm, qv, qi, ni, qm, bm, qc, nc, qr, nr; + + PTD_RW_SCALARS_ONLY(10, th_atm, qv, qi, ni, qm, bm, qc, nc, qr, nr); }; /////////////////////////////////////////////////////////////////////////////// @@ -508,6 +555,8 @@ struct EvapRainData //Outs Real qr2qv_evap_tend, nr_evap_tend; + + PTD_RW_SCALARS_ONLY(2, qr2qv_evap_tend, nr_evap_tend); }; /////////////////////////////////////////////////////////////////////////////// @@ -523,6 +572,8 @@ struct P3UpdatePrognosticLiqData // In/outs Real th_atm, qv, qc, nc, qr, nr; + + PTD_RW_SCALARS_ONLY(6, th_atm, qv, qc, nc, qr, nr); }; /////////////////////////////////////////////////////////////////////////////// @@ -538,6 +589,7 @@ struct IceDepositionSublimationData // This populates all input fields with test data within [0,1]. void randomize(std::mt19937_64& engine); + PTD_RW_SCALARS_ONLY(4, qv2qi_vapdep_tend, qi2qv_sublim_tend, ni_sublim_tend, qc2qi_berg_tend); }; struct IceCldliqCollectionData @@ -549,6 +601,7 @@ struct IceCldliqCollectionData // Outputs Real qc2qi_collect_tend, nc_collect_tend, qc2qr_ice_shed_tend, ncshdc; + PTD_RW_SCALARS_ONLY(4, qc2qi_collect_tend, nc_collect_tend, qc2qr_ice_shed_tend, ncshdc); }; struct IceRainCollectionData @@ -560,6 +613,7 @@ struct IceRainCollectionData // Outputs Real qr2qi_collect_tend, nr_collect_tend; + PTD_RW_SCALARS_ONLY(2, qr2qi_collect_tend, nr_collect_tend); }; struct IceSelfCollectionData @@ -571,6 +625,7 @@ struct IceSelfCollectionData // Outputs Real ni_selfcollect_tend; + PTD_RW_SCALARS_ONLY(1, ni_selfcollect_tend); }; struct IceRelaxationData @@ -580,6 +635,8 @@ struct IceRelaxationData // Outputs Real epsi, epsi_tot; + + PTD_RW_SCALARS_ONLY(2, epsi, epsi_tot); }; struct CalcLiqRelaxationData @@ -592,6 +649,8 @@ struct CalcLiqRelaxationData // This populates all input fields with test data within [0,1]. void randomize(std::mt19937_64& engine); + + PTD_RW_SCALARS_ONLY(2, epsr, epsc); }; struct IceNucleationData @@ -603,6 +662,8 @@ struct IceNucleationData // Outputs Real qv2qi_nucleat_tend, ni_nucleat_tend; + + PTD_RW_SCALARS_ONLY(2, qv2qi_nucleat_tend, ni_nucleat_tend); }; struct IceWetGrowthData @@ -615,21 +676,8 @@ struct IceWetGrowthData bool log_wetgrowth; Real qr2qi_collect_tend, qc2qi_collect_tend, qc_growth_rate, nr_ice_shed_tend, qc2qr_ice_shed_tend; -}; - -struct LatentHeatData : public PhysicsTestData -{ - static constexpr size_t NUM_ARRAYS = 3; - - // Inputs - Int its, ite, kts, kte; - - // Outputs - Real* v, *s, *f; - LatentHeatData(Int its_, Int ite_, Int kts_, Int kte_); - - PTD_STD_DEF(LatentHeatData, 4, its, ite, kts, kte); + PTD_RW_SCALARS_ONLY(6, log_wetgrowth, qr2qi_collect_tend, qc2qi_collect_tend, qc_growth_rate, nr_ice_shed_tend, qc2qr_ice_shed_tend); }; struct CheckValuesData : public PhysicsTestData @@ -659,6 +707,8 @@ struct IncloudMixingData // Outputs Real qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, ni_incld, bm_incld; + + PTD_RW_SCALARS_ONLY(8, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, ni_incld, bm_incld); }; /////////////////////////////////////////////////////////////////////////////// @@ -683,9 +733,9 @@ struct P3MainPart1Data : public PhysicsTestData bool is_nucleat_possible, is_hydromet_present; P3MainPart1Data(Int kts_, Int kte_, Int kbot_, Int ktop_, Int kdir_, - bool do_predict_nc_, bool do_prescribed_CCN_, Real dt_); + bool do_predict_nc_, bool do_prescribed_CCN_, Real dt_, bool=false, bool=false); - PTD_STD_DEF(P3MainPart1Data, 8, kts, kte, kbot, ktop, kdir, do_predict_nc, do_prescribed_CCN, dt); + PTD_STD_DEF(P3MainPart1Data, 10, kts, kte, kbot, ktop, kdir, do_predict_nc, do_prescribed_CCN, dt, is_nucleat_possible, is_hydromet_present); Int nk() const { return (kte - kts) + 1; } }; @@ -712,10 +762,9 @@ struct P3MainPart2Data : public PhysicsTestData bool is_hydromet_present; P3MainPart2Data(Int kts_, Int kte_, Int kbot_, Int ktop_, Int kdir_, - bool do_predict_nc_, bool do_prescribed_CCN, Real dt_); + bool do_predict_nc_, bool do_prescribed_CCN, Real dt_, Real=0., bool=false); - PTD_DATA_COPY_CTOR(P3MainPart2Data, 8); - PTD_ASSIGN_OP(P3MainPart2Data, 10, kts, kte, kbot, ktop, kdir, do_predict_nc, do_prescribed_CCN, dt, inv_dt, is_hydromet_present); + PTD_STD_DEF(P3MainPart2Data, 10, kts, kte, kbot, ktop, kdir, do_predict_nc, do_prescribed_CCN, dt, inv_dt, is_hydromet_present); Int nk() const { return (kte - kts) + 1; } }; @@ -767,9 +816,9 @@ struct P3MainData : public PhysicsTestData *precip_liq_flux, *precip_ice_flux, *precip_liq_surf, *precip_ice_surf; Real elapsed_s; - P3MainData(Int its_, Int ite_, Int kts_, Int kte_, Int it_, Real dt_, bool do_predict_nc_, bool do_prescribed_CCN_); + P3MainData(Int its_, Int ite_, Int kts_, Int kte_, Int it_, Real dt_, bool do_predict_nc_, bool do_prescribed_CCN_, Real=0.); - PTD_STD_DEF(P3MainData, 8, its, ite, kts, kte, it, dt, do_predict_nc, do_prescribed_CCN); + PTD_STD_DEF(P3MainData, 9, its, ite, kts, kte, it, dt, do_predict_nc, do_prescribed_CCN, elapsed_s); }; struct IceSupersatConservationData { @@ -780,6 +829,8 @@ struct IceSupersatConservationData { Real qidep, qinuc; void randomize(std::mt19937_64& engine); + + PTD_RW_SCALARS_ONLY(2, qidep, qinuc); }; struct NcConservationData { @@ -790,6 +841,8 @@ struct NcConservationData { Real nc_collect_tend, nc2ni_immers_freeze_tend, nc_accret_tend, nc2nr_autoconv_tend; void randomize(std::mt19937_64& engine); + + PTD_RW_SCALARS_ONLY(4, nc_collect_tend, nc2ni_immers_freeze_tend, nc_accret_tend, nc2nr_autoconv_tend); }; struct NrConservationData { @@ -800,6 +853,8 @@ struct NrConservationData { Real nr_collect_tend, nr2ni_immers_freeze_tend, nr_selfcollect_tend, nr_evap_tend; void randomize(std::mt19937_64& engine); + + PTD_RW_SCALARS_ONLY(4, nr_collect_tend, nr2ni_immers_freeze_tend, nr_selfcollect_tend, nr_evap_tend); }; struct NiConservationData { @@ -810,6 +865,8 @@ struct NiConservationData { Real ni2nr_melt_tend, ni_sublim_tend, ni_selfcollect_tend; void randomize(std::mt19937_64& engine); + + PTD_RW_SCALARS_ONLY(3, ni2nr_melt_tend, ni_sublim_tend, ni_selfcollect_tend); }; struct PreventLiqSupersaturationData { @@ -821,101 +878,54 @@ struct PreventLiqSupersaturationData { // This populates all fields with test data within [0,1]. void randomize(std::mt19937_64& engine); + + PTD_RW_SCALARS_ONLY(2, qi2qv_sublim_tend, qr2qv_evap_tend); }; -// Glue functions to call fortran from from C++ with the Data struct -void p3_init_a(P3InitAFortranData& d); -void find_lookuptable_indices_1a(LookupIceData& d); -void find_lookuptable_indices_1b(LookupIceDataB& d); -void access_lookup_table(AccessLookupTableData& d); -void access_lookup_table_coll(AccessLookupTableCollData& d); -void back_to_cell_average(BackToCellAverageData& d); -void cloud_water_conservation(CloudWaterConservationData& d); -void rain_water_conservation(RainWaterConservationData& d); -void ice_water_conservation(IceWaterConservationData& d); -void calc_rime_density(CalcRimeDensityData& d); -void cldliq_immersion_freezing(CldliqImmersionFreezingData& d); -void rain_immersion_freezing(RainImmersionFreezingData& d); -void droplet_self_collection(DropletSelfCollectionData& d); -void cloud_rain_accretion(CloudRainAccretionData& d); -void cloud_water_autoconversion(CloudWaterAutoconversionData& d); -void rain_self_collection(RainSelfCollectionData& d); -void impose_max_total_ni(ImposeMaxTotalNiData& d); -void ice_melting(IceMeltingData& d); -Real subgrid_variance_scaling(SubgridVarianceScalingData& d); -void get_cloud_dsd2(GetCloudDsd2Data& d); -void get_rain_dsd2(GetRainDsd2Data& d); -void calc_first_order_upwind_step(CalcUpwindData& d); -void generalized_sedimentation(GenSedData& d); -void cloud_sedimentation(CloudSedData& d); -void ice_sedimentation(IceSedData& d); -void rain_sedimentation(RainSedData& d); -void calc_bulk_rho_rime(CalcBulkRhoRimeData& d); -void homogeneous_freezing(HomogeneousFreezingData& d); -void compute_rain_fall_velocity(ComputeRainFallVelocityData& d); -void get_time_space_phys_variables(GetTimeSpacePhysVarsData& d); -void update_prognostic_ice(P3UpdatePrognosticIceData& d); -void evaporate_rain(EvapRainData& d); -void update_prognostic_liquid(P3UpdatePrognosticLiqData& d); -void ice_deposition_sublimation(IceDepositionSublimationData& d); -void ice_cldliq_collection(IceCldliqCollectionData& d); -void ice_rain_collection(IceRainCollectionData& d); -void ice_self_collection(IceSelfCollectionData& d); -void ice_relaxation_timescale(IceRelaxationData& d); -void calc_liq_relaxation_timescale(CalcLiqRelaxationData& d); -void ice_nucleation(IceNucleationData& d); -void ice_cldliq_wet_growth(IceWetGrowthData& d); -void get_latent_heat(LatentHeatData& d); -void check_values(CheckValuesData& d); -void calculate_incloud_mixingratios(IncloudMixingData& d); -void p3_main_part1(P3MainPart1Data& d); -void p3_main_part2(P3MainPart2Data& d); -void p3_main_part3(P3MainPart3Data& d); -void p3_main(P3MainData& d); - -void ice_supersat_conservation(IceSupersatConservationData& d); -void nc_conservation(NcConservationData& d); -void nr_conservation(NrConservationData& d); -void ni_conservation(NiConservationData& d); -void prevent_liq_supersaturation(PreventLiqSupersaturationData& d); -extern "C" { // _f function decls - -void calc_first_order_upwind_step_f( +void p3_init_a(P3InitAP3Data& d); + +/** + * Convenience functions for calling p3 routines from the host with scalar data. + * These function will pack your data, sync it to device, call the p3 function, + * then sync back to host and unpack. These are used by the BFB unit tests. + */ + +void calc_first_order_upwind_step_host( Int kts, Int kte, Int kdir, Int kbot, Int k_qxtop, Real dt_sub, Real* rho, Real* inv_rho, Real* inv_dz, Int num_arrays, Real** fluxes, Real** vs, Real** qnx); -void generalized_sedimentation_f(Int kts, Int kte, Int kdir, Int k_qxtop, Int *k_qxbot, Int kbot, Real Co_max, +void generalized_sedimentation_host(Int kts, Int kte, Int kdir, Int k_qxtop, Int *k_qxbot, Int kbot, Real Co_max, Real* dt_left, Real* prt_accum, Real* inv_dz, Real* inv_rho, Real* rho, Int num_arrays, Real** vs, Real** fluxes, Real** qnx); -void cloud_sedimentation_f( +void cloud_sedimentation_host( Int kts, Int kte, Int ktop, Int kbot, Int kdir, Real* qc_incld, Real* rho, Real* inv_rho, Real* cld_frac_l, Real* acn, Real* inv_dz, Real dt, Real inv_dt, bool do_predict_nc, Real* qc, Real* nc, Real* nc_incld, Real* mu_c, Real* lamc, Real* precip_liq_surf, Real* qc_tend, Real* nc_tend); -void ice_sedimentation_f( +void ice_sedimentation_host( Int kts, Int kte, Int ktop, Int kbot, Int kdir, Real* rho, Real* inv_rho, Real* rhofaci, Real* cld_frac_i, Real* inv_dz, Real dt, Real inv_dt, Real* qi, Real* qi_incld, Real* ni, Real* qm, Real* qm_incld, Real* bm, Real* bm_incld, Real* ni_incld, Real* precip_ice_surf, Real* qi_tend, Real* ni_tend); -void rain_sedimentation_f( +void rain_sedimentation_host( Int kts, Int kte, Int ktop, Int kbot, Int kdir, Real* qr_incld, Real* rho, Real* inv_rho, Real* rhofacr, Real* cld_frac_r, Real* inv_dz, Real dt, Real inv_dt, Real* qr, Real* nr, Real* nr_incld, Real* mu_r, Real* lamr, Real* precip_liq_surf, Real* precip_liq_flux, Real* qr_tend, Real* nr_tend); -void homogeneous_freezing_f( +void homogeneous_freezing_host( Int kts, Int kte, Int ktop, Int kbot, Int kdir, Real* T_atm, Real* inv_exner, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, Real* bm, Real* th_atm); -void check_values_f(Real* Qv, Real* temp, Int kstart, Int kend, - Int timestepcount, bool force_abort, Int source_ind, Real* col_loc); +void check_values_host(Real* Qv, Real* temp, Int kstart, Int kend, + Int timestepcount, bool force_abort, Int source_ind, Real* col_loc); -void p3_main_part1_f( +void p3_main_part1_host( Int kts, Int kte, Int kbot, Int ktop, Int kdir, bool do_predict_nc, bool do_prescribed_CCN, Real dt, @@ -926,7 +936,7 @@ void p3_main_part1_f( Real* qm_incld, Real* nc_incld, Real* nr_incld, Real* ni_incld, Real* bm_incld, bool* is_nucleat_possible, bool* is_hydromet_present); -void p3_main_part2_f( +void p3_main_part2_host( Int kts, Int kte, Int kbot, Int ktop, Int kdir, bool do_predict_nc, bool do_prescribed_CCN, Real dt, Real inv_dt, Real* pres, Real* dpres, Real* dz, Real* nc_nuceat_tend, Real* inv_exner, Real* exner, Real* inv_cld_frac_l, Real* inv_cld_frac_i, Real* inv_cld_frac_r, Real* ni_activated, Real* inv_qc_relvar, Real* cld_frac_i, Real* cld_frac_l, Real* cld_frac_r, Real* qv_prev, Real* t_prev, Real* T_atm, Real* rho, Real* inv_rho, Real* qv_sat_l, Real* qv_sat_i, Real* qv_supersat_i, Real* rhofacr, Real* rhofaci, Real* acn, Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, @@ -935,14 +945,14 @@ void p3_main_part2_f( Real* nevapr, Real* qr_evap_tend, Real* vap_liq_exchange, Real* vap_ice_exchange, Real* liq_ice_exchange, Real* pratot, Real* prctot, bool* is_hydromet_present); -void p3_main_part3_f( +void p3_main_part3_host( Int kts, Int kte, Int kbot, Int ktop, Int kdir, Real* inv_exner, Real* cld_frac_l, Real* cld_frac_r, Real* cld_frac_i, Real* rho, Real* inv_rho, Real* rhofaci, Real* qv, Real* th_atm, Real* qc, Real* nc, Real* qr, Real* nr, Real* qi, Real* ni, Real* qm, Real* bm, Real* mu_c, Real* nu, Real* lamc, Real* mu_r, Real* lamr, Real* vap_liq_exchange, Real* ze_rain, Real* ze_ice, Real* diag_vm_qi, Real* diag_eff_radius_qi, Real* diag_diam_qi, Real* rho_qi, Real* diag_equiv_reflectivity, Real* diag_eff_radius_qc, Real* diag_eff_radius_qr); -Int p3_main_f( +Int p3_main_host( Real* qc, Real* nc, Real* qr, Real* nr, Real* th_atm, Real* qv, Real dt, Real* qi, Real* qm, Real* ni, Real* bm, Real* pres, Real* dz, Real* nc_nuceat_tend, Real* nccn_prescribed, Real* ni_activated, Real* inv_qc_relvar, Int it, Real* precip_liq_surf, @@ -951,8 +961,6 @@ Int p3_main_f( Real* qv2qi_depos_tend, Real* precip_liq_flux, Real* precip_ice_flux, Real* cld_frac_r, Real* cld_frac_l, Real* cld_frac_i, Real* liq_ice_exchange, Real* vap_liq_exchange, Real* vap_ice_exchange, Real* qv_prev, Real* t_prev); -} // end _f function decls - } // namespace p3 } // namespace scream diff --git a/components/eamxx/src/physics/p3/tests/p3_unit_tests_common.hpp b/components/eamxx/src/physics/p3/tests/infra/p3_unit_tests_common.hpp similarity index 63% rename from components/eamxx/src/physics/p3/tests/p3_unit_tests_common.hpp rename to components/eamxx/src/physics/p3/tests/infra/p3_unit_tests_common.hpp index 68f4491f789..9dd7dee95b3 100644 --- a/components/eamxx/src/physics/p3/tests/p3_unit_tests_common.hpp +++ b/components/eamxx/src/physics/p3/tests/infra/p3_unit_tests_common.hpp @@ -2,7 +2,14 @@ #define P3_UNIT_TESTS_COMMON_HPP #include "share/scream_types.hpp" +#include "share/util/scream_setup_random_test.hpp" #include "p3_functions.hpp" +#include "p3_data.hpp" +#include "ekat/util/ekat_test_utils.hpp" +#include "p3_test_data.hpp" + +#include +#include namespace scream { namespace p3 { @@ -20,6 +27,12 @@ namespace unit_test { struct UnitWrap { + enum BASELINE_ACTION { + NONE, + COMPARE, + GENERATE + }; + template struct UnitTest : public KokkosTypes { @@ -58,6 +71,69 @@ struct UnitWrap { static constexpr Int max_pack_size = 16; static constexpr Int num_test_itrs = max_pack_size / Spack::n; + struct Base { + std::string m_baseline_path; + std::string m_test_name; + BASELINE_ACTION m_baseline_action; + ekat::FILEPtr m_fid; + + Base() : + m_baseline_path(""), + m_test_name(Catch::getResultCapture().getCurrentTestName()), + m_baseline_action(NONE), + m_fid() + { + Functions::p3_init(); // many tests will need fortran table data + auto& ts = ekat::TestSession::get(); + if (ts.flags["c"]) { + m_baseline_action = COMPARE; + } + else if (ts.flags["g"]) { + m_baseline_action = GENERATE; + } + else if (ts.flags["n"]) { + m_baseline_action = NONE; + } + m_baseline_path = ts.params["b"]; + + EKAT_REQUIRE_MSG( !(m_baseline_action != NONE && m_baseline_path == ""), + "P3 unit test flags problem: baseline actions were requested but no baseline path was provided"); + + std::string baseline_name = m_baseline_path + "/" + m_test_name; + if (m_baseline_action == COMPARE) { + m_fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "r")); + } + else if (m_baseline_action == GENERATE) { + m_fid = ekat::FILEPtr(fopen(baseline_name.c_str(), "w")); + } + } + + ~Base() + { + scream::p3::P3GlobalForFortran::deinit(); + } + + std::mt19937_64 get_engine() + { + if (m_baseline_action != COMPARE) { + // We can use any seed + int seed; + auto engine = setup_random_test(nullptr, &seed); + if (m_baseline_action == GENERATE) { + // Write the seed + ekat::write(&seed, 1, m_fid); + } + return engine; + } + else { + // Read the seed + int seed; + ekat::read(&seed, 1, m_fid); + return setup_random_test(seed); + } + } + }; + // Put struct decls here struct TestTableIce; struct TestTable3; @@ -102,7 +178,6 @@ struct UnitWrap { struct TestIceDepositionSublimation; struct TestPreventLiqSupersaturation; }; - }; } // namespace unit_test diff --git a/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp index 78f673121dd..be08e5b0d4f 100644 --- a/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -19,10 +19,10 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestP3CloudWaterAutoconversion +struct UnitWrap::UnitTest::TestP3CloudWaterAutoconversion : public UnitWrap::UnitTest::Base { -static void cloud_water_autoconversion_unit_bfb_tests(){ +void cloud_water_autoconversion_unit_bfb_tests() { CloudWaterAutoconversionData cwadc[max_pack_size] = { // rho, qc_incld, nc_incld, inv_qc_relvar @@ -59,12 +59,14 @@ static void cloud_water_autoconversion_unit_bfb_tests(){ std::copy(&cwadc[0], &cwadc[0] + max_pack_size, cwadc_host.data()); Kokkos::deep_copy(cwadc_device, cwadc_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - cloud_water_autoconversion(cwadc[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + cwadc[i].read(Base::m_fid); + } } - // Run the lookup from a kernel and copy results back to host + // Run the lookup from a kernel and copy results back to host Kokkos::parallel_for(num_test_itrs, KOKKOS_LAMBDA(const Int& i) { const Int offset = i * Spack::n; @@ -101,7 +103,7 @@ static void cloud_water_autoconversion_unit_bfb_tests(){ Kokkos::deep_copy(cwadc_host, cwadc_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(cwadc[s].rho == cwadc_host(s).rho); REQUIRE(cwadc[s].qc_incld == cwadc_host(s).qc_incld); @@ -112,13 +114,18 @@ static void cloud_water_autoconversion_unit_bfb_tests(){ REQUIRE(cwadc[s].ncautr == cwadc_host(s).ncautr); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + cwadc_host(s).write(Base::m_fid); + } + } } - static void run_bfb(){ + void run_bfb() { cloud_water_autoconversion_unit_bfb_tests(); } - KOKKOS_FUNCTION static void autoconversion_is_positive(const Int &i, Int &errors){ + KOKKOS_FUNCTION static void autoconversion_is_positive(const Int &i, Int &errors){ const Spack rho(1.0), inv_qc_relvar(1.0); Spack qc_incld, nc_incld(1e7), qc2qr_autoconv_tend(0.0), nc2nr_autoconv_tend(0.0), ncautr(0.0); @@ -134,7 +141,7 @@ static void cloud_water_autoconversion_unit_bfb_tests(){ } } - static void run_physics(){ + void run_physics(){ int nerr = 0; @@ -153,12 +160,14 @@ static void cloud_water_autoconversion_unit_bfb_tests(){ } // namespace p3 } // namespace scream -namespace{ +namespace { TEST_CASE("p3_cloud_water_autoconversion_test", "[p3_cloud_water_autoconversion_test]"){ - scream::p3::unit_test::UnitWrap::UnitTest::TestP3CloudWaterAutoconversion::run_physics(); - scream::p3::unit_test::UnitWrap::UnitTest::TestP3CloudWaterAutoconversion::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestP3CloudWaterAutoconversion; + + T t; + t.run_physics(); + t.run_bfb(); } } // namespace - diff --git a/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp index 0377b990d78..b108ce329d3 100644 --- a/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_back_to_cell_average_unit_tests.cpp @@ -4,8 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" -#include "share/util/scream_setup_random_test.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -19,45 +18,49 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestBackToCellAverage { +struct UnitWrap::UnitTest::TestBackToCellAverage : public UnitWrap::UnitTest::Base { -static void run_phys() +void run_phys() { // TODO } -static void run_bfb() +void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); // Generate n test structs, each populated with random data (values within // [0,1]) by the default constructor. - BackToCellAverageData back_to_cell_average_data[Spack::n]; - for (Int i = 0; i < Spack::n; ++i) { - back_to_cell_average_data[i].randomize(engine); + BackToCellAverageData back_to_cell_average_data[max_pack_size]; + for (auto& item : back_to_cell_average_data) { + item.randomize(engine); } // Sync to device. - view_1d device_data("back_to_cell_average", Spack::n); + view_1d device_data("back_to_cell_average", max_pack_size); const auto host_data = Kokkos::create_mirror_view(device_data); - std::copy(&back_to_cell_average_data[0], &back_to_cell_average_data[0] + Spack::n, + std::copy(&back_to_cell_average_data[0], &back_to_cell_average_data[0] + max_pack_size, host_data.data()); Kokkos::deep_copy(device_data, host_data); - // Run the Fortran subroutine. - for (Int i = 0; i < Spack::n; ++i) { - back_to_cell_average(back_to_cell_average_data[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + back_to_cell_average_data[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host - Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { + Kokkos::parallel_for(num_test_itrs, KOKKOS_LAMBDA(const Int& i) { + const Int offset = i * Spack::n; + // Init pack inputs Spack cld_frac_l, cld_frac_r, cld_frac_i, qc2qr_accret_tend, qr2qv_evap_tend, qc2qr_autoconv_tend, nc_accret_tend, nc_selfcollect_tend, nc2nr_autoconv_tend, nr_selfcollect_tend, nr_evap_tend, ncautr, qi2qv_sublim_tend, nr_ice_shed_tend, qc2qi_hetero_freeze_tend, qr2qi_collect_tend, qc2qr_ice_shed_tend, qi2qr_melt_tend, qc2qi_collect_tend, qr2qi_immers_freeze_tend, ni2nr_melt_tend, nc_collect_tend, ncshdc, nc2ni_immers_freeze_tend, nr_collect_tend, ni_selfcollect_tend, qv2qi_vapdep_tend, nr2ni_immers_freeze_tend, ni_sublim_tend, qv2qi_nucleat_tend, ni_nucleat_tend, qc2qi_berg_tend; - for (Int s = 0; s < Spack::n; ++s) { + for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { cld_frac_l[s] = device_data[s].cld_frac_l; cld_frac_r[s] = device_data[s].cld_frac_r; cld_frac_i[s] = device_data[s].cld_frac_i; @@ -99,7 +102,7 @@ static void run_bfb() nr_collect_tend, ni_selfcollect_tend, qv2qi_vapdep_tend, nr2ni_immers_freeze_tend, ni_sublim_tend, qv2qi_nucleat_tend, ni_nucleat_tend, qc2qi_berg_tend); // Copy results back into views - for (Int s = 0; s < Spack::n; ++s) { + for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { device_data(s).qc2qr_accret_tend = qc2qr_accret_tend[s]; device_data(s).qr2qv_evap_tend = qr2qv_evap_tend[s]; device_data(s).qc2qr_autoconv_tend = qc2qr_autoconv_tend[s]; @@ -136,8 +139,8 @@ static void run_bfb() Kokkos::deep_copy(host_data, device_data); // Validate results. - if (SCREAM_BFB_TESTING) { - for (Int s = 0; s < Spack::n; ++s) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(back_to_cell_average_data[s].qc2qr_accret_tend == host_data[s].qc2qr_accret_tend); REQUIRE(back_to_cell_average_data[s].qr2qv_evap_tend == host_data[s].qr2qv_evap_tend); REQUIRE(back_to_cell_average_data[s].qc2qr_autoconv_tend == host_data[s].qc2qr_autoconv_tend); @@ -169,6 +172,11 @@ static void run_bfb() REQUIRE(back_to_cell_average_data[s].qc2qi_berg_tend == host_data[s].qc2qi_berg_tend); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + host_data(s).write(Base::m_fid); + } + } } }; @@ -181,12 +189,11 @@ namespace { TEST_CASE("p3_back_to_cell_average", "[p3_functions]") { - using TRIF = scream::p3::unit_test::UnitWrap::UnitTest::TestBackToCellAverage; - - TRIF::run_phys(); - TRIF::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestBackToCellAverage; - scream::p3::P3GlobalForFortran::deinit(); + T t; + t.run_phys(); + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp index 97193fd06d3..2752746b792 100644 --- a/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_calc_liq_relaxation_timescale_unit_tests.cpp @@ -4,8 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" -#include "share/util/scream_setup_random_test.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -20,16 +19,16 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestCalcLiqRelaxationTimescale { +struct UnitWrap::UnitTest::TestCalcLiqRelaxationTimescale : public UnitWrap::UnitTest::Base { - static void run_phys() + void run_phys() { // TODO } - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); // Read in tables view_2d_table vn_table_vals, vm_table_vals, revap_table_vals; @@ -54,9 +53,11 @@ struct UnitWrap::UnitTest::TestCalcLiqRelaxationTimescale { self[i].f2r = C::f2r; } - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - calc_liq_relaxation_timescale(self[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + self[i].read(Base::m_fid); + } } // Sync to device @@ -97,12 +98,17 @@ struct UnitWrap::UnitTest::TestCalcLiqRelaxationTimescale { Kokkos::deep_copy(self_host, self_device); - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(self[s].epsr == self_host(s).epsr); REQUIRE(self[s].epsc == self_host(s).epsc); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + self_host(s).write(Base::m_fid); + } + } } }; @@ -115,10 +121,11 @@ namespace { TEST_CASE("p3_calc_liq_relaxation_timescale", "[p3_functions]") { - using TD = scream::p3::unit_test::UnitWrap::UnitTest::TestCalcLiqRelaxationTimescale; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestCalcLiqRelaxationTimescale; - TD::run_phys(); - TD::run_bfb(); + T t; + t.run_phys(); + t.run_bfb(); } } diff --git a/components/eamxx/src/physics/p3/tests/p3_calc_rime_density_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_calc_rime_density_unit_tests.cpp index 5d70c7302ae..9b7722895fa 100644 --- a/components/eamxx/src/physics/p3/tests/p3_calc_rime_density_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_calc_rime_density_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -18,14 +18,14 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestCalcRimeDensity { +struct UnitWrap::UnitTest::TestCalcRimeDensity : public UnitWrap::UnitTest::Base { -static void run_phys() +void run_phys() { // TODO } -static void run_bfb() +void run_bfb() { // This is the threshold for whether the qc cloud mixing ratio is large // enough to affect the rime density. @@ -87,9 +87,11 @@ static void run_bfb() host_data.data()); Kokkos::deep_copy(device_data, host_data); - // Run the Fortran subroutine. - for (Int i = 0; i < max_pack_size; ++i) { - calc_rime_density(calc_rime_density_data[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + calc_rime_density_data[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -126,12 +128,17 @@ static void run_bfb() Kokkos::deep_copy(host_data, device_data); // Validate results. - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(calc_rime_density_data[s].vtrmi1 == host_data[s].vtrmi1); REQUIRE(calc_rime_density_data[s].rho_qm_cloud == host_data[s].rho_qm_cloud); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + host_data(s).write(Base::m_fid); + } + } } }; @@ -144,12 +151,11 @@ namespace { TEST_CASE("p3_calc_rime_density", "[p3_functions]") { - using TRIF = scream::p3::unit_test::UnitWrap::UnitTest::TestCalcRimeDensity; - - TRIF::run_phys(); - TRIF::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestCalcRimeDensity; - scream::p3::P3GlobalForFortran::deinit(); + T t; + t.run_phys(); + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_check_values_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_check_values_unit_tests.cpp index f2f4a7f4404..90c8d28be93 100644 --- a/components/eamxx/src/physics/p3/tests/p3_check_values_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_check_values_unit_tests.cpp @@ -4,9 +4,8 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" -#include "p3_f90.hpp" -#include "share/util/scream_setup_random_test.hpp" +#include "p3_test_data.hpp" +#include "p3_data.hpp" #include "p3_unit_tests_common.hpp" @@ -20,13 +19,14 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestCheckValues { +struct UnitWrap::UnitTest::TestCheckValues : public UnitWrap::UnitTest::Base { -static void run_check_values_bfb() +void run_check_values_bfb() { + // This is not really a bfb test since no results are being checked. auto engine = setup_random_test(); - CheckValuesData cvd_fortran[] = { + CheckValuesData cvd_cxx[] = { // kts_, kte_, timestepcount_, source_ind_, force_abort_ CheckValuesData(1, 72, 2, 100, false), CheckValuesData(1, 72, 3, 100, false), @@ -34,33 +34,17 @@ static void run_check_values_bfb() CheckValuesData(1, 72, 5, 100, false), }; - static constexpr Int num_runs = sizeof(cvd_fortran) / sizeof(CheckValuesData); - - for (auto& d : cvd_fortran) { + for (auto& d : cvd_cxx) { d.randomize(engine, { {d.qv, {-4.056E-01, 1.153E+00}}, {d.temp, {1.000E+02, 5.000E+02}} }); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that - // inout data is in original state - CheckValuesData cvd_cxx[num_runs] = { - CheckValuesData(cvd_fortran[0]), - CheckValuesData(cvd_fortran[1]), - CheckValuesData(cvd_fortran[2]), - CheckValuesData(cvd_fortran[3]), - }; - - // Get data from fortran - for (auto& d : cvd_fortran) { - check_values(d); - } - // Get data from cxx for (auto& d : cvd_cxx) { - check_values_f(d.qv, d.temp, d.kts, d.kte, d.timestepcount, d.force_abort, d.source_ind, d.col_loc); + check_values_host(d.qv, d.temp, d.kts, d.kte, d.timestepcount, d.force_abort, d.source_ind, d.col_loc); } } -static void run_check_values_phys() +void run_check_values_phys() { // TODO } @@ -75,14 +59,11 @@ namespace { TEST_CASE("p3_check_values", "[p3_functions]") { - using TRS = scream::p3::unit_test::UnitWrap::UnitTest::TestCheckValues; - - scream::p3::p3_init(); // need fortran table data - - TRS::run_check_values_phys(); - TRS::run_check_values_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestCheckValues; - scream::p3::P3GlobalForFortran::deinit(); + T t; + t.run_check_values_phys(); + t.run_check_values_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp index eeccb70fd58..bbaa95e87a0 100644 --- a/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -18,14 +18,14 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestCldliqImmersionFreezing { +struct UnitWrap::UnitTest::TestCldliqImmersionFreezing : public UnitWrap::UnitTest::Base { -static void run_phys() +void run_phys() { // TODO } -static void run_bfb() +void run_bfb() { // This is the threshold for whether the qc and qr cloud mixing ratios are // large enough to affect the warm-phase process rates qc2qr_accret_tend and nc_accret_tend. @@ -70,9 +70,11 @@ static void run_bfb() host_data.data()); Kokkos::deep_copy(device_data, host_data); - // Run the Fortran subroutine. - for (Int i = 0; i < max_pack_size; ++i) { - cldliq_immersion_freezing(cldliq_imm_freezing_data[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + cldliq_imm_freezing_data[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -109,12 +111,17 @@ static void run_bfb() Kokkos::deep_copy(host_data, device_data); // Validate results. - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(cldliq_imm_freezing_data[s].qc2qi_hetero_freeze_tend == host_data[s].qc2qi_hetero_freeze_tend); REQUIRE(cldliq_imm_freezing_data[s].nc2ni_immers_freeze_tend == host_data[s].nc2ni_immers_freeze_tend); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + host_data(s).write(Base::m_fid); + } + } } }; @@ -127,12 +134,11 @@ namespace { TEST_CASE("p3_cldliq_immersion_freezing", "[p3_functions]") { - using TRIF = scream::p3::unit_test::UnitWrap::UnitTest::TestCldliqImmersionFreezing; - - TRIF::run_phys(); - TRIF::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestCldliqImmersionFreezing; - scream::p3::P3GlobalForFortran::deinit(); + T t; + t.run_phys(); + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp index 472f9de1987..271f7e27bd6 100644 --- a/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -18,14 +18,14 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestCloudRainAccretion { +struct UnitWrap::UnitTest::TestCloudRainAccretion : public UnitWrap::UnitTest::Base { -static void run_phys() +void run_phys() { // TODO } -static void run_bfb() +void run_bfb() { // This is the threshold for whether the qc and qr cloud mixing ratios are // large enough to affect the warm-phase process rates qc2qr_accret_tend and nc_accret_tend. @@ -73,9 +73,11 @@ static void run_bfb() host_data.data()); Kokkos::deep_copy(device_data, host_data); - // Run the Fortran subroutine. - for (Int i = 0; i < max_pack_size; ++i) { - cloud_rain_accretion(cloud_rain_acc_data[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + cloud_rain_acc_data[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -112,12 +114,17 @@ static void run_bfb() Kokkos::deep_copy(host_data, device_data); // Validate results. - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(cloud_rain_acc_data[s].qc2qr_accret_tend == host_data[s].qc2qr_accret_tend); REQUIRE(cloud_rain_acc_data[s].nc_accret_tend == host_data[s].nc_accret_tend); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + host_data(s).write(Base::m_fid); + } + } } }; @@ -130,12 +137,11 @@ namespace { TEST_CASE("p3_cloud_rain_accretion", "[p3_functions]") { - using TCRA = scream::p3::unit_test::UnitWrap::UnitTest::TestCloudRainAccretion; - - TCRA::run_phys(); - TCRA::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestCloudRainAccretion; - scream::p3::P3GlobalForFortran::deinit(); + T t; + t.run_phys(); + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp index 19fe5b3279e..b29bf11faa1 100644 --- a/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_cloud_sed_unit_tests.cpp @@ -4,8 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" -#include "share/util/scream_setup_random_test.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -19,18 +18,18 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestCloudSed { +struct UnitWrap::UnitTest::TestCloudSed : public UnitWrap::UnitTest::Base { -static void run_phys() +void run_phys() { // TODO } -static void run_bfb() +void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - CloudSedData csds_fortran[] = { + CloudSedData csds_baseline[] = { // kts, kte, ktop, kbot, kdir, dt, inv_dt, do_predict_nc, precip_liq_surf, CloudSedData(1, 72, 27, 72, -1, 1.800E+03, 5.556E-04, false, 0.0), CloudSedData(1, 72, 72, 27, 1, 1.800E+03, 5.556E-04, false, 0.0), @@ -39,51 +38,58 @@ static void run_bfb() CloudSedData(1, 72, 27, 27, -1, 1.800E+03, 5.556E-04, true, 0.0), }; - static constexpr Int num_runs = sizeof(csds_fortran) / sizeof(CloudSedData); + static constexpr Int num_runs = sizeof(csds_baseline) / sizeof(CloudSedData); // Set up random input data - for (auto& d : csds_fortran) { + for (auto& d : csds_baseline) { d.randomize(engine, { {d.qc_incld, {C::QSMALL/2, C::QSMALL*2}} }); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state CloudSedData csds_cxx[num_runs] = { - CloudSedData(csds_fortran[0]), - CloudSedData(csds_fortran[1]), - CloudSedData(csds_fortran[2]), - CloudSedData(csds_fortran[3]), - CloudSedData(csds_fortran[4]), + CloudSedData(csds_baseline[0]), + CloudSedData(csds_baseline[1]), + CloudSedData(csds_baseline[2]), + CloudSedData(csds_baseline[3]), + CloudSedData(csds_baseline[4]), }; - // Get data from fortran - for (auto& d : csds_fortran) { - cloud_sedimentation(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : csds_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : csds_cxx) { - cloud_sedimentation_f(d.kts, d.kte, d.ktop, d.kbot, d.kdir, + cloud_sedimentation_host(d.kts, d.kte, d.ktop, d.kbot, d.kdir, d.qc_incld, d.rho, d.inv_rho, d.cld_frac_l, d.acn, d.inv_dz, d.dt, d.inv_dt, d.do_predict_nc, d.qc, d.nc, d.nc_incld, d.mu_c, d.lamc, &d.precip_liq_surf, d.qc_tend, d.nc_tend); } - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { // Due to pack issues, we must restrict checks to the active k space - Int start = std::min(csds_fortran[i].kbot, csds_fortran[i].ktop) - 1; // 0-based indx - Int end = std::max(csds_fortran[i].kbot, csds_fortran[i].ktop); // 0-based indx + Int start = std::min(csds_baseline[i].kbot, csds_baseline[i].ktop) - 1; // 0-based indx + Int end = std::max(csds_baseline[i].kbot, csds_baseline[i].ktop); // 0-based indx for (Int k = start; k < end; ++k) { - REQUIRE(csds_fortran[i].qc[k] == csds_cxx[i].qc[k]); - REQUIRE(csds_fortran[i].nc[k] == csds_cxx[i].nc[k]); - REQUIRE(csds_fortran[i].nc_incld[k] == csds_cxx[i].nc_incld[k]); - REQUIRE(csds_fortran[i].mu_c[k] == csds_cxx[i].mu_c[k]); - REQUIRE(csds_fortran[i].lamc[k] == csds_cxx[i].lamc[k]); - REQUIRE(csds_fortran[i].qc_tend[k] == csds_cxx[i].qc_tend[k]); - REQUIRE(csds_fortran[i].nc_tend[k] == csds_cxx[i].nc_tend[k]); + REQUIRE(csds_baseline[i].qc[k] == csds_cxx[i].qc[k]); + REQUIRE(csds_baseline[i].nc[k] == csds_cxx[i].nc[k]); + REQUIRE(csds_baseline[i].nc_incld[k] == csds_cxx[i].nc_incld[k]); + REQUIRE(csds_baseline[i].mu_c[k] == csds_cxx[i].mu_c[k]); + REQUIRE(csds_baseline[i].lamc[k] == csds_cxx[i].lamc[k]); + REQUIRE(csds_baseline[i].qc_tend[k] == csds_cxx[i].qc_tend[k]); + REQUIRE(csds_baseline[i].nc_tend[k] == csds_cxx[i].nc_tend[k]); } - REQUIRE(csds_fortran[i].precip_liq_surf == csds_cxx[i].precip_liq_surf); + REQUIRE(csds_baseline[i].precip_liq_surf == csds_cxx[i].precip_liq_surf); + } + } + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + csds_cxx[i].write(Base::m_fid); } } } @@ -98,12 +104,11 @@ namespace { TEST_CASE("p3_cloud_sed", "[p3_functions]") { - using TCS = scream::p3::unit_test::UnitWrap::UnitTest::TestCloudSed; - - TCS::run_phys(); - TCS::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestCloudSed; - scream::p3::P3GlobalForFortran::deinit(); + T t; + t.run_phys(); + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_droplet_self_coll_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_droplet_self_coll_unit_tests.cpp index b81505816ea..f183bba65da 100644 --- a/components/eamxx/src/physics/p3/tests/p3_droplet_self_coll_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_droplet_self_coll_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -18,14 +18,14 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestDropletSelfCollection { +struct UnitWrap::UnitTest::TestDropletSelfCollection : public UnitWrap::UnitTest::Base { -static void run_phys() +void run_phys() { // TODO } -static void run_bfb() +void run_bfb() { // This is the threshold for whether the qc and qr cloud mixing ratios are // large enough to affect the warm-phase process rates qc2qr_accret_tend and nc_accret_tend. @@ -72,9 +72,11 @@ static void run_bfb() host_data.data()); Kokkos::deep_copy(device_data, host_data); - // Run the Fortran subroutine. - for (Int i = 0; i < max_pack_size; ++i) { - droplet_self_collection(droplet_self_coll_data[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + droplet_self_coll_data[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -107,11 +109,17 @@ static void run_bfb() Kokkos::deep_copy(host_data, device_data); // Validate results. - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(droplet_self_coll_data[s].nc_selfcollect_tend == host_data[s].nc_selfcollect_tend); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + host_data(s).write(Base::m_fid); + } + } + } }; @@ -124,12 +132,11 @@ namespace { TEST_CASE("p3_droplet_self_collection", "[p3_functions]") { - using TCRA = scream::p3::unit_test::UnitWrap::UnitTest::TestDropletSelfCollection; - - TCRA::run_phys(); - TCRA::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestDropletSelfCollection; - scream::p3::P3GlobalForFortran::deinit(); + T t; + t.run_phys(); + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp index 50894b03855..268607645f0 100644 --- a/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -22,9 +22,9 @@ namespace unit_test { */ template -struct UnitWrap::UnitTest::TestDsd2 { +struct UnitWrap::UnitTest::TestDsd2 : public UnitWrap::UnitTest::Base { - static void run_cloud_bfb() + void run_cloud_bfb() { // Read in tables view_2d_table vn_table_vals; view_2d_table vm_table_vals; view_2d_table revap_table_vals; @@ -60,9 +60,11 @@ struct UnitWrap::UnitTest::TestDsd2 { std::copy(&gcdd[0], &gcdd[0] + max_pack_size, gcdd_host.data()); Kokkos::deep_copy(gcdd_device, gcdd_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - get_cloud_dsd2(gcdd[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + gcdd[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -95,7 +97,7 @@ struct UnitWrap::UnitTest::TestDsd2 { Kokkos::deep_copy(gcdd_host, gcdd_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(gcdd[s].nc_out == gcdd_host(s).nc_out); REQUIRE(gcdd[s].mu_c == gcdd_host(s).mu_c); @@ -105,14 +107,19 @@ struct UnitWrap::UnitTest::TestDsd2 { REQUIRE(gcdd[s].cdist1 == gcdd_host(s).cdist1); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + gcdd_host(s).write(Base::m_fid); + } + } } - static void run_cloud_phys() + void run_cloud_phys() { // TODO } - static void run_rain_bfb() + void run_rain_bfb() { using KTH = KokkosTypes; @@ -144,9 +151,11 @@ struct UnitWrap::UnitTest::TestDsd2 { std::copy(&grdd[0], &grdd[0] + max_pack_size, grdd_host.data()); Kokkos::deep_copy(grdd_device, grdd_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - get_rain_dsd2(grdd[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + grdd[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -179,7 +188,7 @@ struct UnitWrap::UnitTest::TestDsd2 { Kokkos::deep_copy(grdd_host, grdd_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(grdd[s].nr_out == grdd_host(s).nr_out); REQUIRE(grdd[s].mu_r == grdd_host(s).mu_r); @@ -188,9 +197,14 @@ struct UnitWrap::UnitTest::TestDsd2 { REQUIRE(grdd[s].logn0r == grdd_host(s).logn0r); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + grdd_host(s).write(Base::m_fid); + } + } } - static void run_rain_phys() + void run_rain_phys() { // TODO } @@ -204,18 +218,20 @@ namespace { TEST_CASE("p3_cloud_dsd2", "[p3_functions]") { - using TD = scream::p3::unit_test::UnitWrap::UnitTest::TestDsd2; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestDsd2; - TD::run_cloud_phys(); - TD::run_cloud_bfb(); + T t; + t.run_cloud_phys(); + t.run_cloud_bfb(); } TEST_CASE("p3_rain_dsd2", "[p3_functions]") { - using TD = scream::p3::unit_test::UnitWrap::UnitTest::TestDsd2; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestDsd2; - TD::run_rain_phys(); - TD::run_rain_bfb(); + T t; + t.run_rain_phys(); + t.run_rain_bfb(); } } diff --git a/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp index f8398135a19..a84b1e0448a 100644 --- a/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_evaporate_rain_unit_tests.cpp @@ -4,25 +4,18 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" -//#include -//#include -//#include -//#include -//#include // std::setprecision - namespace scream { namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestEvapSublPrecip -{ +struct UnitWrap::UnitTest::TestEvapSublPrecip : public UnitWrap::UnitTest::Base { - static void run_property(){ + void run_property() { //TEST WEIGHTING TIMESCALE //======================== @@ -133,13 +126,13 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip REQUIRE( qrtend[0] <= qr_incld[0]/dt); REQUIRE( nrtend[0] <= nr_incld[0]/dt); //keep end-of-step nr positive. Should always be true. - }; //end run_property + } //end run_property - static void run_bfb(){ + void run_bfb() { constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; - //fortran generated data is input to the following + //baseline generated data is input to the following //This subroutine has 20 args, only 18 are supplied here for invoking it as last 2 are intent-outs //note that dt is the same val for each row - this is needed since dt is a scalar and all rows are executed simultaneously on CPU in C++. //row1: above freezing, should trigger @@ -180,9 +173,11 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip std::copy(&espd[0], &espd[0] + max_pack_size, espd_host.data()); Kokkos::deep_copy(espd_device, espd_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - evaporate_rain(espd[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + espd[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -248,12 +243,17 @@ struct UnitWrap::UnitTest::TestEvapSublPrecip Kokkos::deep_copy(espd_host, espd_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(espd[s].qr2qv_evap_tend == espd_host(s).qr2qv_evap_tend); REQUIRE(espd[s].nr_evap_tend == espd_host(s).nr_evap_tend); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + espd_host(s).write(Base::m_fid); + } + } } // end run_bfb @@ -267,14 +267,18 @@ namespace { TEST_CASE("p3_evaporate_rain_property", "p3_unit_tests") { - using TestStruct = scream::p3::unit_test::UnitWrap::UnitTest::TestEvapSublPrecip; - TestStruct::run_property(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestEvapSublPrecip; + + T t; + t.run_property(); } TEST_CASE("p3_evaporate_rain_test", "p3_unit_tests") { - using TestStruct = scream::p3::unit_test::UnitWrap::UnitTest::TestEvapSublPrecip; - TestStruct::run_bfb(); -} + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestEvapSublPrecip; + + T t; + t.run_bfb(); + } }// end anonymous namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_find_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_find_unit_tests.cpp index 4ba2dc6760e..6e8e09226b7 100644 --- a/components/eamxx/src/physics/p3/tests/p3_find_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_find_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -22,9 +22,9 @@ namespace unit_test { // template -struct UnitWrap::UnitTest::TestFind { +struct UnitWrap::UnitTest::TestFind : public UnitWrap::UnitTest::Base { -static void run() +void run() { const int max_threads = #ifdef KOKKOS_ENABLE_OPENMP @@ -112,7 +112,10 @@ namespace { TEST_CASE("p3_find", "[p3_functions]") { - scream::p3::unit_test::UnitWrap::UnitTest::TestFind::run(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestFind; + + T t; + t.run(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_get_latent_heat_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_get_latent_heat_unit_tests.cpp deleted file mode 100644 index 5d7b4de4ea7..00000000000 --- a/components/eamxx/src/physics/p3/tests/p3_get_latent_heat_unit_tests.cpp +++ /dev/null @@ -1,78 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/scream_types.hpp" -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "p3_functions.hpp" -#include "p3_functions_f90.hpp" -#include "p3_f90.hpp" - -#include "p3_unit_tests_common.hpp" - -#include -#include -#include -#include -#include // std::setprecision - -namespace scream { -namespace p3 { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestLatentHeat { - - static void run_latent_heat_bfb() - { - constexpr Scalar latvap = C::LatVap; - constexpr Scalar latice = C::LatIce; - - LatentHeatData latent_fortran[] = { - // its, ite, kts, kte - LatentHeatData(1, 7, 1, 10), - }; - - static constexpr Int num_runs = sizeof(latent_fortran) / sizeof(LatentHeatData); - - LatentHeatData latent_cxx[num_runs] = { - LatentHeatData(latent_fortran[0]), - LatentHeatData(latent_fortran[1]), - LatentHeatData(latent_fortran[2]), - LatentHeatData(latent_fortran[3]), - }; - - for (Int i = 0; i < num_runs; ++i) { - LatentHeatData& h = latent_fortran[i]; - get_latent_heat(h); - - if (SCREAM_BFB_TESTING) { - for (Int j = 0; j < h.total(h.v); ++j) { - REQUIRE(h.v[j] == latvap); - REQUIRE(h.s[j] == (latvap+latice)); - REQUIRE(h.f[j] == latice); - } - } - } - } - - static void run_latent_heat_phys() - { - // TODO - } -}; - -} -} -} - -namespace { - -TEST_CASE("p3_latent_heat", "[p3_functions]") -{ - using TD = scream::p3::unit_test::UnitWrap::UnitTest::TestLatentHeat; - - TD::run_latent_heat_phys(); - TD::run_latent_heat_bfb(); -} - -} diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp index caf6638d5cd..cca204b9b3a 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_cldliq_wet_growth_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -19,9 +19,9 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestIceCldliqWetGrowth { +struct UnitWrap::UnitTest::TestIceCldliqWetGrowth : public UnitWrap::UnitTest::Base { - static void run_ice_cldliq_wet_growth_bfb() + void run_ice_cldliq_wet_growth_bfb() { using KTH = KokkosTypes; @@ -57,9 +57,11 @@ struct UnitWrap::UnitTest::TestIceCldliqWetGrowth { std::copy(&self[0], &self[0] + max_pack_size, self_host.data()); Kokkos::deep_copy(self_device, self_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - ice_cldliq_wet_growth(self[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + self[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -114,7 +116,7 @@ struct UnitWrap::UnitTest::TestIceCldliqWetGrowth { Kokkos::deep_copy(self_host, self_device); - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(static_cast(self[s].log_wetgrowth) == static_cast(self_host(s).log_wetgrowth)); @@ -125,9 +127,14 @@ struct UnitWrap::UnitTest::TestIceCldliqWetGrowth { REQUIRE(self[s].qc2qr_ice_shed_tend == self_host(s).qc2qr_ice_shed_tend); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + self_host(s).write(Base::m_fid); + } + } } - static void run_ice_cldliq_wet_growth_phys() + void run_ice_cldliq_wet_growth_phys() { // TODO } @@ -141,10 +148,11 @@ namespace { TEST_CASE("p3_ice_cldliq_wet_growth", "[p3_functions]") { - using TD = scream::p3::unit_test::UnitWrap::UnitTest::TestIceCldliqWetGrowth; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestIceCldliqWetGrowth; - TD::run_ice_cldliq_wet_growth_phys(); - TD::run_ice_cldliq_wet_growth_bfb(); + T t; + t.run_ice_cldliq_wet_growth_phys(); + t.run_ice_cldliq_wet_growth_bfb(); } } diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp index 8fa1649b121..2561114b519 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -22,9 +22,9 @@ namespace unit_test { * Unit-tests for p3 ice collection functions. */ template -struct UnitWrap::UnitTest::TestIceCollection { +struct UnitWrap::UnitTest::TestIceCollection : public UnitWrap::UnitTest::Base { - static void run_ice_cldliq_bfb() + void run_ice_cldliq_bfb() { // Read in tables view_2d_table vn_table_vals; @@ -63,9 +63,11 @@ struct UnitWrap::UnitTest::TestIceCollection { std::copy(&cldliq[0], &cldliq[0] + max_pack_size, cldliq_host.data()); Kokkos::deep_copy(cldliq_device, cldliq_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - ice_cldliq_collection(cldliq[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + cldliq[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -109,7 +111,7 @@ struct UnitWrap::UnitTest::TestIceCollection { Kokkos::deep_copy(cldliq_host, cldliq_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(cldliq[s].qc2qi_collect_tend == cldliq_host(s).qc2qi_collect_tend); REQUIRE(cldliq[s].nc_collect_tend == cldliq_host(s).nc_collect_tend); @@ -117,14 +119,19 @@ struct UnitWrap::UnitTest::TestIceCollection { REQUIRE(cldliq[s].ncshdc == cldliq_host(s).ncshdc); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + cldliq_host(s).write(Base::m_fid); + } + } } - static void run_ice_cldliq_phys() + void run_ice_cldliq_phys() { // TODO } - static void run_ice_rain_bfb() + void run_ice_rain_bfb() { using KTH = KokkosTypes; @@ -157,9 +164,11 @@ struct UnitWrap::UnitTest::TestIceCollection { std::copy(&rain[0], &rain[0] + max_pack_size, rain_host.data()); Kokkos::deep_copy(rain_device, rain_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - ice_rain_collection(rain[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + rain[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -198,20 +207,25 @@ struct UnitWrap::UnitTest::TestIceCollection { Kokkos::deep_copy(rain_host, rain_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(rain[s].qr2qi_collect_tend == rain_host(s).qr2qi_collect_tend); REQUIRE(rain[s].nr_collect_tend == rain_host(s).nr_collect_tend); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + rain_host(s).write(Base::m_fid); + } + } } - static void run_ice_rain_phys() + void run_ice_rain_phys() { // TODO } - static void run_ice_self_bfb() + void run_ice_self_bfb() { using KTH = KokkosTypes; @@ -244,9 +258,11 @@ struct UnitWrap::UnitTest::TestIceCollection { std::copy(&self[0], &self[0] + max_pack_size, self_host.data()); Kokkos::deep_copy(self_device, self_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - ice_self_collection(self[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + self[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -276,15 +292,19 @@ struct UnitWrap::UnitTest::TestIceCollection { Kokkos::deep_copy(self_host, self_device); - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(self[s].ni_selfcollect_tend == self_host(s).ni_selfcollect_tend); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + self_host(s).write(Base::m_fid); + } + } } - - static void run_ice_self_phys() + void run_ice_self_phys() { // TODO } @@ -298,24 +318,29 @@ namespace { TEST_CASE("p3_ice_cldliq", "[p3_functions]") { - using TD = scream::p3::unit_test::UnitWrap::UnitTest::TestIceCollection; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestIceCollection; - TD::run_ice_cldliq_phys(); - TD::run_ice_cldliq_bfb(); + T t; + t.run_ice_cldliq_phys(); + t.run_ice_cldliq_bfb(); } TEST_CASE("p3_ice_rain", "[p3_functions]") { - using TD = scream::p3::unit_test::UnitWrap::UnitTest::TestIceCollection; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestIceCollection; - TD::run_ice_rain_phys(); - TD::run_ice_rain_bfb(); + T t; + t.run_ice_rain_phys(); + t.run_ice_rain_bfb(); } TEST_CASE("p3_ice_self", "[p3_functions]") { - using TD = scream::p3::unit_test::UnitWrap::UnitTest::TestIceCollection; - TD::run_ice_self_phys(); - TD::run_ice_self_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestIceCollection; + + T t; + t.run_ice_self_phys(); + t.run_ice_self_bfb(); } + } diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_deposition_sublimation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_deposition_sublimation_tests.cpp index bc1921441bb..9b261409ad0 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_deposition_sublimation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_deposition_sublimation_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "physics/share/physics_constants.hpp" #include "p3_unit_tests_common.hpp" @@ -13,9 +13,9 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestIceDepositionSublimation { +struct UnitWrap::UnitTest::TestIceDepositionSublimation : public UnitWrap::UnitTest::Base { - static void run_property(){ + void run_property() { //Note that a lot of property tests are included in run_bfb for simplicity //Choose default values (just grabbed a row from the array in run_bfb) @@ -30,13 +30,13 @@ struct UnitWrap::UnitTest::TestIceDepositionSublimation { //init output vars Spack qv2qi_vapdep_tend, qi2qv_sublim_tend, ni_sublim_tend, qc2qi_berg_tend; - + //CHECK THAT UNREASONABLY LARGE VAPOR DEPOSITION DOESN'T LEAVE QV SUBSATURATED WRT QI Spack epsi_tmp=1e6; //make 1/(sat removal timescale) huge so vapdep rate removes all supersat in 1 dt. Functions::ice_deposition_sublimation(qi_incld, ni_incld, T_atm, qv_sat_l, qv_sat_i, epsi_tmp, abi, qv, inv_dt, qv2qi_vapdep_tend, qi2qv_sublim_tend, ni_sublim_tend, qc2qi_berg_tend); REQUIRE( (qv2qi_vapdep_tend[0]==0 || std::abs( qv2qi_vapdep_tend[0] - (qv[0] - qv_sat_i[0])*inv_dt) <1e-8) ); - + //CHECK THAT HUGE SUBLIMATION DOESN'T LEAVE QV SUPERSATURATED WRT QI Spack qv_sat_i_tmp=1e-2; Functions::ice_deposition_sublimation(qi_incld, ni_incld, T_atm, qv_sat_l, qv_sat_i_tmp, @@ -46,10 +46,10 @@ struct UnitWrap::UnitTest::TestIceDepositionSublimation { //CHECK BEHAVIOR AS DT->0? } - - static void run_bfb() + + void run_bfb() { - IceDepositionSublimationData f90_data[max_pack_size] = { + IceDepositionSublimationData baseline_data[max_pack_size] = { {1.0000E-04,4.5010E+05,2.8750E+02,1.1279E-02,1.1279E-02,0.0000E+00,3.3648E+00,5.0000E-03,1.666667e-02}, {5.1000E-03,4.5370E+05,2.8542E+02,9.9759E-03,9.9759E-03,0.0000E+00,3.1223E+00,5.0000E-03,1.666667e-02}, {5.1000E-03,4.5742E+05,2.8334E+02,8.8076E-03,8.8076E-03,0.0000E+00,2.9014E+00,5.0000E-03,1.666667e-02}, @@ -67,25 +67,25 @@ struct UnitWrap::UnitTest::TestIceDepositionSublimation { {5.1000E-03,5.1317E+05,2.5834E+02,1.6757E-03,1.4491E-03,6.0620E-02,1.3763E+00,5.0000E-03,1.666667e-02}, {5.0000E-08,5.4479E+05,2.4793E+02,7.5430E-04,5.8895E-04,4.6769E-04,1.1661E+00,1.5278E-04,1.666667e-02}, }; - - static constexpr Int num_runs = sizeof(f90_data) / sizeof(IceDepositionSublimationData); // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - //for (auto& d : f90_data) { + // Alternatively, you can use the baseline_data construtors/initializer lists to hardcode data + //for (auto& d : baseline_data) { // d.randomize(); //} - // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that + // Create copies of data for use by cxx and sync it to device. Needs to happen before reads so that // inout data is in original state view_1d cxx_device("cxx_device", max_pack_size); const auto cxx_host = Kokkos::create_mirror_view(cxx_device); - std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); + std::copy(&baseline_data[0], &baseline_data[0] + max_pack_size, cxx_host.data()); Kokkos::deep_copy(cxx_device, cxx_host); - // Get data from fortran - for (auto& d : f90_data) { - ice_deposition_sublimation(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + baseline_data[i].read(Base::m_fid); + } } // Get data from cxx. Run ice_deposition_sublimation from a kernel and copy results back to host @@ -110,7 +110,6 @@ struct UnitWrap::UnitTest::TestIceDepositionSublimation { // Init outputs Spack ni_sublim_tend(0), qi2qv_sublim_tend(0), qc2qi_berg_tend(0), qv2qi_vapdep_tend(0); - Functions::ice_deposition_sublimation(qi_incld, ni_incld, T_atm, qv_sat_l, qv_sat_i, epsi, abi, qv, inv_dt, qv2qi_vapdep_tend, qi2qv_sublim_tend, ni_sublim_tend, qc2qi_berg_tend); // Copy spacks back into cxx_device view @@ -120,35 +119,40 @@ struct UnitWrap::UnitTest::TestIceDepositionSublimation { cxx_device(vs).qc2qi_berg_tend = qc2qi_berg_tend[s]; cxx_device(vs).qv2qi_vapdep_tend = qv2qi_vapdep_tend[s]; } - }); Kokkos::deep_copy(cxx_host, cxx_device); - for (Int i = 0; i < num_runs; ++i) { - // Verify BFB results - IceDepositionSublimationData& d_f90 = f90_data[i]; - IceDepositionSublimationData& d_cxx = cxx_host[i]; - REQUIRE(d_f90.qv2qi_vapdep_tend == d_cxx.qv2qi_vapdep_tend); - REQUIRE(d_f90.qi2qv_sublim_tend == d_cxx.qi2qv_sublim_tend); - REQUIRE(d_f90.ni_sublim_tend == d_cxx.ni_sublim_tend); - REQUIRE(d_f90.qc2qi_berg_tend == d_cxx.qc2qi_berg_tend); - - //MAKE SURE OUTPUT IS WITHIN EXPECTED BOUNDS: - REQUIRE(d_cxx.qv2qi_vapdep_tend >=0); - REQUIRE(d_cxx.qi2qv_sublim_tend >=0); - REQUIRE(d_cxx.ni_sublim_tend >=0); - REQUIRE(d_cxx.qc2qi_berg_tend >=0); - - //vapdep should only occur when qv>qv_sat_i - REQUIRE( (d_cxx.qv2qi_vapdep_tend==0 || d_cxx.qv + d_cxx.qv2qi_vapdep_tend*d_cxx.inv_dt >= d_cxx.qv_sat_i) ); - //sublim should only occur when qvfrz, berg and vapdep should be 0: - REQUIRE( (d_cxx.T_atmm_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + // Verify BFB results + IceDepositionSublimationData& d_f90 = baseline_data[i]; + IceDepositionSublimationData& d_cxx = cxx_host[i]; + REQUIRE(d_f90.qv2qi_vapdep_tend == d_cxx.qv2qi_vapdep_tend); + REQUIRE(d_f90.qi2qv_sublim_tend == d_cxx.qi2qv_sublim_tend); + REQUIRE(d_f90.ni_sublim_tend == d_cxx.ni_sublim_tend); + REQUIRE(d_f90.qc2qi_berg_tend == d_cxx.qc2qi_berg_tend); + + //MAKE SURE OUTPUT IS WITHIN EXPECTED BOUNDS: + REQUIRE(d_cxx.qv2qi_vapdep_tend >=0); + REQUIRE(d_cxx.qi2qv_sublim_tend >=0); + REQUIRE(d_cxx.ni_sublim_tend >=0); + REQUIRE(d_cxx.qc2qi_berg_tend >=0); + + //vapdep should only occur when qv>qv_sat_i + REQUIRE( (d_cxx.qv2qi_vapdep_tend==0 || d_cxx.qv + d_cxx.qv2qi_vapdep_tend*d_cxx.inv_dt >= d_cxx.qv_sat_i) ); + //sublim should only occur when qvfrz, berg and vapdep should be 0: + REQUIRE( (d_cxx.T_atmm_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + cxx_host(s).write(Base::m_fid); + } } } // run_bfb @@ -162,16 +166,18 @@ namespace { TEST_CASE("ice_deposition_sublimation_property", "[p3]") { - using TestStruct = scream::p3::unit_test::UnitWrap::UnitTest::TestIceDepositionSublimation; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestIceDepositionSublimation; - TestStruct::run_property(); + T t; + t.run_property(); } - + TEST_CASE("ice_deposition_sublimation_bfb", "[p3]") { - using TestStruct = scream::p3::unit_test::UnitWrap::UnitTest::TestIceDepositionSublimation; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestIceDepositionSublimation; - TestStruct::run_bfb(); + T t; + t.run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp index 28c6582f8a1..4e24cc8be0e 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_melting_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -19,10 +19,10 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestP3IceMelting +struct UnitWrap::UnitTest::TestP3IceMelting : public UnitWrap::UnitTest::Base { -static void ice_melting_bfb(){ +void ice_melting_bfb() { constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; @@ -57,9 +57,11 @@ static void ice_melting_bfb(){ std::copy(&IceMelt[0], &IceMelt[0] + max_pack_size, IceMelt_host.data()); Kokkos::deep_copy(IceMelt_device, IceMelt_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - ice_melting(IceMelt[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + IceMelt[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -99,13 +101,19 @@ static void ice_melting_bfb(){ Kokkos::deep_copy(IceMelt_host, IceMelt_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(IceMelt[s].qi2qr_melt_tend == IceMelt_host(s).qi2qr_melt_tend); REQUIRE(IceMelt[s].ni2nr_melt_tend == IceMelt_host(s).ni2nr_melt_tend); } } -}; // TestP3IceMelting + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + IceMelt_host(s).write(Base::m_fid); + } + } + +} }; // UnitWrap @@ -113,11 +121,13 @@ static void ice_melting_bfb(){ } // namespace p3 } // namespace scream -namespace{ +namespace { -TEST_CASE("p3_ice_melting_test", "[p3_ice_melting_test]"){ - scream::p3::unit_test::UnitWrap::UnitTest::TestP3IceMelting::ice_melting_bfb(); +TEST_CASE("p3_ice_melting_test", "[p3_ice_melting_test]") { + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestP3IceMelting; + + T t; + t.ice_melting_bfb(); } } // namespace - diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp index 4e54b8837e0..7eeb4fa875e 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -19,9 +19,9 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestIceNucleation { +struct UnitWrap::UnitTest::TestIceNucleation : public UnitWrap::UnitTest::Base { - static void run_ice_nucleation_bfb() + void run_ice_nucleation_bfb() { using KTH = KokkosTypes; @@ -54,10 +54,13 @@ struct UnitWrap::UnitTest::TestIceNucleation { {2.702E+02, 1.069E+00, 0.323E+03, 2.221E+01, 9.952E-01, inv_dt, do_predict_nc, do_prescribed_CCN } }; - // Run the fortran code - for (Int i = 0; i < max_pack_size; ++i) { - ice_nucleation(self[i]); - } + std::string root_name = "ice_nucleation"; + std::string file_name = root_name + (do_predict_nc ? "1" : "0") + (do_prescribed_CCN ? "1" : "0"); + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + self[i].read(Base::m_fid); + } + } // Sync to device KTH::view_1d self_host("self_host", max_pack_size); @@ -94,17 +97,22 @@ struct UnitWrap::UnitTest::TestIceNucleation { Kokkos::deep_copy(self_host, self_device); - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(self[s].qv2qi_nucleat_tend == self_host(s).qv2qi_nucleat_tend); REQUIRE(self[s].ni_nucleat_tend == self_host(s).ni_nucleat_tend); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + self_host(s).write(Base::m_fid); + } + } } //end for do_predict_nc } //end for do_prescribed_CCN } - static void run_ice_nucleation_phys() + void run_ice_nucleation_phys() { // TODO } @@ -118,10 +126,11 @@ namespace { TEST_CASE("p3_ice_nucleation", "[p3_functions]") { - using TD = scream::p3::unit_test::UnitWrap::UnitTest::TestIceNucleation; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestIceNucleation; - TD::run_ice_nucleation_phys(); - TD::run_ice_nucleation_bfb(); + T t; + t.run_ice_nucleation_phys(); + t.run_ice_nucleation_bfb(); } } diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_relaxation_timescale_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_relaxation_timescale_unit_tests.cpp index a75b55fdbec..59768bbf82c 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_relaxation_timescale_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_relaxation_timescale_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -19,9 +19,9 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestIceRelaxationTimescale { +struct UnitWrap::UnitTest::TestIceRelaxationTimescale : public UnitWrap::UnitTest::Base { - static void run_ice_relaxation_timescale_bfb() + void run_ice_relaxation_timescale_bfb() { using KTH = KokkosTypes; @@ -49,10 +49,12 @@ struct UnitWrap::UnitTest::TestIceRelaxationTimescale { {1.352E+01, 3.210E+03, 1.069E+00, 0.123E+00, 3.456E+00, 1.221E-02, 9.952E-07, 6.596E-05, 4.532E-01, 1.734E+04} }; - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - ice_relaxation_timescale(self[i]); - } + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + self[i].read(Base::m_fid); + } + } // Sync to device KTH::view_1d self_host("self_host", max_pack_size); @@ -93,15 +95,20 @@ struct UnitWrap::UnitTest::TestIceRelaxationTimescale { Kokkos::deep_copy(self_host, self_device); - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(self[s].epsi == self_host(s).epsi); REQUIRE(self[s].epsi_tot == self_host(s).epsi_tot); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + self_host(s).write(Base::m_fid); + } + } } - static void run_ice_relaxation_timescale_phys() + void run_ice_relaxation_timescale_phys() { // TODO } @@ -115,10 +122,11 @@ namespace { TEST_CASE("p3_ice_relaxation_timescale", "[p3_functions]") { - using TD = scream::p3::unit_test::UnitWrap::UnitTest::TestIceRelaxationTimescale; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestIceRelaxationTimescale; - TD::run_ice_relaxation_timescale_phys(); - TD::run_ice_relaxation_timescale_bfb(); + T t; + t.run_ice_relaxation_timescale_phys(); + t.run_ice_relaxation_timescale_bfb(); } } diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp index c04aa55be00..faa6da1596e 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp @@ -4,8 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" -#include "share/util/scream_setup_random_test.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -19,36 +18,36 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestIceSed { +struct UnitWrap::UnitTest::TestIceSed : public UnitWrap::UnitTest::Base { -static void run_phys_calc_bulk_rhime() +void run_phys_calc_bulk_rhime() { // TODO } -static void run_phys_ice_sed() +void run_phys_ice_sed() { // TODO } -static void run_phys_homogeneous_freezing() +void run_phys_homogeneous_freezing() { // TODO } -static void run_phys() +void run_phys() { run_phys_calc_bulk_rhime(); run_phys_ice_sed(); run_phys_homogeneous_freezing(); } -static void run_bfb_calc_bulk_rhime() +void run_bfb_calc_bulk_rhime() { constexpr Scalar qsmall = C::QSMALL; // Load some lookup inputs, need at least one per pack value - CalcBulkRhoRimeData cbrr_fortran[max_pack_size] = { + CalcBulkRhoRimeData cbrr_baseline[max_pack_size] = { // qi_tot, qi_rim, bi_rim {9.999978E-08, 9.999978E-03, 1.111108E-10}, {0.000000E+00, 8.571428E-05, 1.000000E-02}, @@ -71,17 +70,17 @@ static void run_bfb_calc_bulk_rhime() {5.164017E-10, 0.000000E+00, 0.000000E+00}, }; - // Sync to device, needs to happen before fortran calls so that + // Sync to device, needs to happen before reads so that // inout data is in original state view_1d cbrr_device("cbrr", max_pack_size); const auto cbrr_host = Kokkos::create_mirror_view(cbrr_device); - std::copy(&cbrr_fortran[0], &cbrr_fortran[0] + max_pack_size, cbrr_host.data()); + std::copy(&cbrr_baseline[0], &cbrr_baseline[0] + max_pack_size, cbrr_host.data()); Kokkos::deep_copy(cbrr_device, cbrr_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - if (cbrr_fortran[i].qi_tot > qsmall) { - calc_bulk_rho_rime(cbrr_fortran[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + cbrr_baseline[i].read(Base::m_fid); } } @@ -114,20 +113,25 @@ static void run_bfb_calc_bulk_rhime() Kokkos::deep_copy(cbrr_host, cbrr_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { - REQUIRE(cbrr_fortran[s].qi_rim == cbrr_host(s).qi_rim); - REQUIRE(cbrr_fortran[s].bi_rim == cbrr_host(s).bi_rim); - REQUIRE(cbrr_fortran[s].rho_rime == cbrr_host(s).rho_rime); + REQUIRE(cbrr_baseline[s].qi_rim == cbrr_host(s).qi_rim); + REQUIRE(cbrr_baseline[s].bi_rim == cbrr_host(s).bi_rim); + REQUIRE(cbrr_baseline[s].rho_rime == cbrr_host(s).rho_rime); + } + } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + cbrr_host(s).write(Base::m_fid); } } } -static void run_bfb_ice_sed() +void run_bfb_ice_sed() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - IceSedData isds_fortran[] = { + IceSedData isds_baseline[] = { // kts, kte, ktop, kbot, kdir, dt, inv_dt, precip_ice_surf IceSedData(1, 72, 27, 72, -1, 1.800E+03, 5.556E-04, 0.0), IceSedData(1, 72, 72, 27, 1, 1.800E+03, 5.556E-04, 1.0), @@ -135,65 +139,72 @@ static void run_bfb_ice_sed() IceSedData(1, 72, 27, 27, 1, 1.800E+03, 5.556E-04, 2.0), }; - static constexpr Int num_runs = sizeof(isds_fortran) / sizeof(IceSedData); + static constexpr Int num_runs = sizeof(isds_baseline) / sizeof(IceSedData); // Set up random input data - for (auto& d : isds_fortran) { + for (auto& d : isds_baseline) { d.randomize(engine, { {d.qi_incld, {C::QSMALL/2, C::QSMALL*2}} }); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state IceSedData isds_cxx[num_runs] = { - IceSedData(isds_fortran[0]), - IceSedData(isds_fortran[1]), - IceSedData(isds_fortran[2]), - IceSedData(isds_fortran[3]), + IceSedData(isds_baseline[0]), + IceSedData(isds_baseline[1]), + IceSedData(isds_baseline[2]), + IceSedData(isds_baseline[3]), }; - // Get data from fortran - for (auto& d : isds_fortran) { - ice_sedimentation(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < num_runs; ++i) { + isds_baseline[i].read(Base::m_fid); + } } // Get data from cxx for (auto& d : isds_cxx) { - ice_sedimentation_f(d.kts, d.kte, d.ktop, d.kbot, d.kdir, + ice_sedimentation_host(d.kts, d.kte, d.ktop, d.kbot, d.kdir, d.rho, d.inv_rho, d.rhofaci, d.cld_frac_i, d.inv_dz, d.dt, d.inv_dt, d.qi, d.qi_incld, d.ni, d.qm, d.qm_incld, d.bm, d.bm_incld, d.ni_incld, &d.precip_ice_surf, d.qi_tend, d.ni_tend); } - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { // Due to pack issues, we must restrict checks to the active k space - Int start = std::min(isds_fortran[i].kbot, isds_fortran[i].ktop) - 1; // 0-based indx - Int end = std::max(isds_fortran[i].kbot, isds_fortran[i].ktop); // 0-based indx + Int start = std::min(isds_baseline[i].kbot, isds_baseline[i].ktop) - 1; // 0-based indx + Int end = std::max(isds_baseline[i].kbot, isds_baseline[i].ktop); // 0-based indx for (Int k = start; k < end; ++k) { - REQUIRE(isds_fortran[i].qi[k] == isds_cxx[i].qi[k]); - REQUIRE(isds_fortran[i].qi_incld[k] == isds_cxx[i].qi_incld[k]); - REQUIRE(isds_fortran[i].ni[k] == isds_cxx[i].ni[k]); - REQUIRE(isds_fortran[i].ni_incld[k] == isds_cxx[i].ni_incld[k]); - REQUIRE(isds_fortran[i].qm[k] == isds_cxx[i].qm[k]); - REQUIRE(isds_fortran[i].qm_incld[k] == isds_cxx[i].qm_incld[k]); - REQUIRE(isds_fortran[i].bm[k] == isds_cxx[i].bm[k]); - REQUIRE(isds_fortran[i].bm_incld[k] == isds_cxx[i].bm_incld[k]); - REQUIRE(isds_fortran[i].qi_tend[k] == isds_cxx[i].qi_tend[k]); - REQUIRE(isds_fortran[i].ni_tend[k] == isds_cxx[i].ni_tend[k]); + REQUIRE(isds_baseline[i].qi[k] == isds_cxx[i].qi[k]); + REQUIRE(isds_baseline[i].qi_incld[k] == isds_cxx[i].qi_incld[k]); + REQUIRE(isds_baseline[i].ni[k] == isds_cxx[i].ni[k]); + REQUIRE(isds_baseline[i].ni_incld[k] == isds_cxx[i].ni_incld[k]); + REQUIRE(isds_baseline[i].qm[k] == isds_cxx[i].qm[k]); + REQUIRE(isds_baseline[i].qm_incld[k] == isds_cxx[i].qm_incld[k]); + REQUIRE(isds_baseline[i].bm[k] == isds_cxx[i].bm[k]); + REQUIRE(isds_baseline[i].bm_incld[k] == isds_cxx[i].bm_incld[k]); + REQUIRE(isds_baseline[i].qi_tend[k] == isds_cxx[i].qi_tend[k]); + REQUIRE(isds_baseline[i].ni_tend[k] == isds_cxx[i].ni_tend[k]); } - REQUIRE(isds_fortran[i].precip_ice_surf == isds_cxx[i].precip_ice_surf); + REQUIRE(isds_baseline[i].precip_ice_surf == isds_cxx[i].precip_ice_surf); + } + } + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + isds_cxx[i].write(Base::m_fid); } } } -static void run_bfb_homogeneous_freezing() +void run_bfb_homogeneous_freezing() { constexpr Scalar latice = C::LatIce; - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - HomogeneousFreezingData hfds_fortran[] = { + HomogeneousFreezingData hfds_baseline[] = { // kts, kte, ktop, kbot, kdir HomogeneousFreezingData(1, 72, 27, 72, -1), HomogeneousFreezingData(1, 72, 72, 27, 1), @@ -201,10 +212,10 @@ static void run_bfb_homogeneous_freezing() HomogeneousFreezingData(1, 72, 27, 27, 1), }; - static constexpr Int num_runs = sizeof(hfds_fortran) / sizeof(HomogeneousFreezingData); + static constexpr Int num_runs = sizeof(hfds_baseline) / sizeof(HomogeneousFreezingData); // Set up random input data - for (auto& d : hfds_fortran) { + for (auto& d : hfds_baseline) { const auto qsmall_r = std::make_pair(C::QSMALL/2, C::QSMALL*2); d.randomize(engine, { {d.T_atm, {C::T_homogfrz - 10, C::T_homogfrz + 10}}, {d.qc, qsmall_r}, {d.qr, qsmall_r} }); @@ -215,48 +226,56 @@ static void run_bfb_homogeneous_freezing() } } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state HomogeneousFreezingData hfds_cxx[num_runs] = { - HomogeneousFreezingData(hfds_fortran[0]), - HomogeneousFreezingData(hfds_fortran[1]), - HomogeneousFreezingData(hfds_fortran[2]), - HomogeneousFreezingData(hfds_fortran[3]), + HomogeneousFreezingData(hfds_baseline[0]), + HomogeneousFreezingData(hfds_baseline[1]), + HomogeneousFreezingData(hfds_baseline[2]), + HomogeneousFreezingData(hfds_baseline[3]), }; - // Get data from fortran - for (auto& d : hfds_fortran) { - homogeneous_freezing(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : hfds_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : hfds_cxx) { - homogeneous_freezing_f(d.kts, d.kte, d.ktop, d.kbot, d.kdir, + homogeneous_freezing_host(d.kts, d.kte, d.ktop, d.kbot, d.kdir, d.T_atm, d.inv_exner, d.qc, d.nc, d.qr, d.nr, d.qi, d.ni, d.qm, d.bm, d.th_atm); } - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { // Due to pack issues, we must restrict checks to the active k space - Int start = std::min(hfds_fortran[i].kbot, hfds_fortran[i].ktop) - 1; // 0-based indx - Int end = std::max(hfds_fortran[i].kbot, hfds_fortran[i].ktop); // 0-based indx + Int start = std::min(hfds_baseline[i].kbot, hfds_baseline[i].ktop) - 1; // 0-based indx + Int end = std::max(hfds_baseline[i].kbot, hfds_baseline[i].ktop); // 0-based indx for (Int k = start; k < end; ++k) { - REQUIRE(hfds_fortran[i].qc[k] == hfds_cxx[i].qc[k]); - REQUIRE(hfds_fortran[i].nc[k] == hfds_cxx[i].nc[k]); - REQUIRE(hfds_fortran[i].qr[k] == hfds_cxx[i].qr[k]); - REQUIRE(hfds_fortran[i].nr[k] == hfds_cxx[i].nr[k]); - REQUIRE(hfds_fortran[i].qi[k] == hfds_cxx[i].qi[k]); - REQUIRE(hfds_fortran[i].ni[k] == hfds_cxx[i].ni[k]); - REQUIRE(hfds_fortran[i].qm[k] == hfds_cxx[i].qm[k]); - REQUIRE(hfds_fortran[i].bm[k] == hfds_cxx[i].bm[k]); - REQUIRE(hfds_fortran[i].th_atm[k] == hfds_cxx[i].th_atm[k]); + REQUIRE(hfds_baseline[i].qc[k] == hfds_cxx[i].qc[k]); + REQUIRE(hfds_baseline[i].nc[k] == hfds_cxx[i].nc[k]); + REQUIRE(hfds_baseline[i].qr[k] == hfds_cxx[i].qr[k]); + REQUIRE(hfds_baseline[i].nr[k] == hfds_cxx[i].nr[k]); + REQUIRE(hfds_baseline[i].qi[k] == hfds_cxx[i].qi[k]); + REQUIRE(hfds_baseline[i].ni[k] == hfds_cxx[i].ni[k]); + REQUIRE(hfds_baseline[i].qm[k] == hfds_cxx[i].qm[k]); + REQUIRE(hfds_baseline[i].bm[k] == hfds_cxx[i].bm[k]); + REQUIRE(hfds_baseline[i].th_atm[k] == hfds_cxx[i].th_atm[k]); } } } + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + hfds_cxx[i].write(Base::m_fid); + } + } + } -static void run_bfb() +void run_bfb() { run_bfb_calc_bulk_rhime(); run_bfb_ice_sed(); @@ -273,12 +292,11 @@ namespace { TEST_CASE("p3_ice_sed", "[p3_functions]") { - using TCS = scream::p3::unit_test::UnitWrap::UnitTest::TestIceSed; - - TCS::run_phys(); - TCS::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestIceSed; - scream::p3::P3GlobalForFortran::deinit(); + T t; + t.run_phys(); + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp index 5873b8335c3..dcbdfc61826 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_supersat_conservation_tests.cpp @@ -4,8 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" -#include "share/util/scream_setup_random_test.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -14,38 +13,40 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestIceSupersatConservation { +struct UnitWrap::UnitTest::TestIceSupersatConservation : public UnitWrap::UnitTest::Base { - static void run_bfb() + void run_bfb() { constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - IceSupersatConservationData f90_data[max_pack_size]; + IceSupersatConservationData baseline_data[max_pack_size]; // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { + // Alternatively, you can use the baseline_data construtors/initializer lists to hardcode data + for (auto& d : baseline_data) { d.randomize(engine); - d.dt = f90_data[0].dt; // hold this fixed, it is not packed data + d.dt = baseline_data[0].dt; // hold this fixed, it is not packed data // C++ impl uses constants for latent_heat values. Manually set here // so F90 can match d.latent_heat_sublim = latvap+latice; } - // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that + // Create copies of data for use by cxx and sync it to device. Needs to happen before reads so that // inout data is in original state view_1d cxx_device("cxx_device", max_pack_size); const auto cxx_host = Kokkos::create_mirror_view(cxx_device); - std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); + std::copy(&baseline_data[0], &baseline_data[0] + max_pack_size, cxx_host.data()); Kokkos::deep_copy(cxx_device, cxx_host); - // Get data from fortran - for (auto& d : f90_data) { - ice_supersat_conservation(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + baseline_data[i].read(Base::m_fid); + } } // Get data from cxx. Run ice_supersat_conservation from a kernel and copy results back to host @@ -72,20 +73,24 @@ struct UnitWrap::UnitTest::TestIceSupersatConservation { cxx_device(vs).qidep = qidep[s]; cxx_device(vs).qinuc = qinuc[s]; } - }); Kokkos::deep_copy(cxx_host, cxx_device); // Verify BFB results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - IceSupersatConservationData& d_f90 = f90_data[i]; + IceSupersatConservationData& d_f90 = baseline_data[i]; IceSupersatConservationData& d_cxx = cxx_host[i]; REQUIRE(d_f90.qidep == d_cxx.qidep); REQUIRE(d_f90.qinuc == d_cxx.qinuc); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + cxx_host(s).write(Base::m_fid); + } + } } // run_bfb }; @@ -98,9 +103,10 @@ namespace { TEST_CASE("ice_supersat_conservation_bfb", "[p3]") { - using TestStruct = scream::p3::unit_test::UnitWrap::UnitTest::TestIceSupersatConservation; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestIceSupersatConservation; - TestStruct::run_bfb(); + T t; + t.run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp index 091d05895fc..ea3fdaccf7e 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_tables_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -22,9 +22,9 @@ namespace unit_test { */ template -struct UnitWrap::UnitTest::TestTableIce { +struct UnitWrap::UnitTest::TestTableIce : public UnitWrap::UnitTest::Base { - static void test_read_lookup_tables_bfb() + void test_read_lookup_tables_bfb() { // Read in ice tables view_ice_table ice_table_vals; @@ -32,7 +32,7 @@ struct UnitWrap::UnitTest::TestTableIce { Functions::init_kokkos_ice_lookup_tables(ice_table_vals, collect_table_vals); // Get data from fortran - P3InitAFortranData d; + P3InitAP3Data d; p3_init_a(d); // Copy device data to host @@ -62,7 +62,7 @@ struct UnitWrap::UnitTest::TestTableIce { } template - static void init_table_linear_dimension(View& table, int linear_dimension) + void init_table_linear_dimension(View& table, int linear_dimension) { // set up views using NonConstView = typename View::non_const_type; @@ -96,7 +96,7 @@ struct UnitWrap::UnitTest::TestTableIce { table = view_device; } - static void run_bfb() + void run_bfb() { using KTH = KokkosTypes; @@ -199,14 +199,6 @@ struct UnitWrap::UnitTest::TestTableIce { {lid[15], lidb[15], access_table_index} }; - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - find_lookuptable_indices_1a(lid[i]); - find_lookuptable_indices_1b(lidb[i]); - access_lookup_table(altd[i]); - access_lookup_table_coll(altcd[i]); - } - // Sync to device KTH::view_1d lid_host("lid_host", max_pack_size); KTH::view_1d lidb_host("lidb_host", max_pack_size); @@ -217,6 +209,16 @@ struct UnitWrap::UnitTest::TestTableIce { Kokkos::deep_copy(lid_device, lid_host); Kokkos::deep_copy(lidb_device, lidb_host); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + lid[i].read(Base::m_fid); + lidb[i].read(Base::m_fid); + altd[i].read(Base::m_fid); + altcd[i].read(Base::m_fid); + } + } + // Run the lookup from a kernel and copy results back to host view_2d int_results("int results", 5, max_pack_size); view_2d real_results("real results", 7, max_pack_size); @@ -270,15 +272,14 @@ struct UnitWrap::UnitTest::TestTableIce { Kokkos::deep_copy(real_results_mirror, real_results); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for(int s = 0; s < max_pack_size; ++s) { - // +1 for O vs 1-based indexing - REQUIRE(int_results_mirror(0, s)+1 == lid[s].dumi); - REQUIRE(int_results_mirror(1, s)+1 == lid[s].dumjj); - REQUIRE(int_results_mirror(2, s)+1 == lid[s].dumii); - REQUIRE(int_results_mirror(3, s)+1 == lid[s].dumzz); + REQUIRE(int_results_mirror(0, s) == lid[s].dumi); + REQUIRE(int_results_mirror(1, s) == lid[s].dumjj); + REQUIRE(int_results_mirror(2, s) == lid[s].dumii); + REQUIRE(int_results_mirror(3, s) == lid[s].dumzz); - REQUIRE(int_results_mirror(4, s)+1 == lidb[s].dumj); + REQUIRE(int_results_mirror(4, s) == lidb[s].dumj); REQUIRE(real_results_mirror(0, s) == lid[s].dum1); REQUIRE(real_results_mirror(1, s) == lid[s].dum4); @@ -292,9 +293,35 @@ struct UnitWrap::UnitTest::TestTableIce { REQUIRE(real_results_mirror(6, s) == altcd[s].proc); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + lid[s].dumi = int_results_mirror(0, s); + lid[s].dumjj = int_results_mirror(1, s); + lid[s].dumii = int_results_mirror(2, s); + lid[s].dumzz = int_results_mirror(3, s); + + lidb[s].dumj = int_results_mirror(4, s); + + lid[s].dum1 = real_results_mirror(0, s); + lid[s].dum4 = real_results_mirror(1, s); + lid[s].dum5 = real_results_mirror(2, s); + lid[s].dum6 = real_results_mirror(3, s); + + lidb[s].dum3 = real_results_mirror(4, s); + + altd[s].proc = real_results_mirror(5, s); + + altcd[s].proc = real_results_mirror(6, s); + + lid[s].write(Base::m_fid); + lidb[s].write(Base::m_fid); + altd[s].write(Base::m_fid); + altcd[s].write(Base::m_fid); + } + } } - static void run_phys() + void run_phys() { #if 0 view_ice_table ice_table_vals; @@ -343,11 +370,12 @@ namespace { TEST_CASE("p3_ice_tables", "[p3_functions]") { - using TTI = scream::p3::unit_test::UnitWrap::UnitTest::TestTableIce; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestTableIce; - TTI::test_read_lookup_tables_bfb(); - TTI::run_phys(); - TTI::run_bfb(); + T t; + t.test_read_lookup_tables_bfb(); + t.run_phys(); + t.run_bfb(); } } diff --git a/components/eamxx/src/physics/p3/tests/p3_incloud_mixingratios_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_incloud_mixingratios_unit_tests.cpp index 4a405fcc7c4..eae1454c36a 100644 --- a/components/eamxx/src/physics/p3/tests/p3_incloud_mixingratios_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_incloud_mixingratios_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -19,9 +19,9 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestIncloudMixing { +struct UnitWrap::UnitTest::TestIncloudMixing : public UnitWrap::UnitTest::Base { - static void run_incloud_mixing_bfb() + void run_incloud_mixing_bfb() { using KTH = KokkosTypes; @@ -69,9 +69,11 @@ struct UnitWrap::UnitTest::TestIncloudMixing { std::copy(&self[0], &self[0] + max_pack_size, self_host.data()); Kokkos::deep_copy(self_device, self_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - calculate_incloud_mixingratios(self[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + self[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -115,7 +117,7 @@ struct UnitWrap::UnitTest::TestIncloudMixing { Kokkos::deep_copy(self_host, self_device); - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(self[s].qc_incld == self_host(s).qc_incld); REQUIRE(self[s].qr_incld == self_host(s).qr_incld); @@ -127,9 +129,14 @@ struct UnitWrap::UnitTest::TestIncloudMixing { REQUIRE(self[s].bm_incld == self_host(s).bm_incld); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + self_host(s).write(Base::m_fid); + } + } } - static void run_incloud_mixing_phys() + void run_incloud_mixing_phys() { // TODO } @@ -143,10 +150,11 @@ namespace { TEST_CASE("p3_incloud_mixingratios", "[p3_functions]") { - using TD = scream::p3::unit_test::UnitWrap::UnitTest::TestIncloudMixing; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestIncloudMixing; - TD::run_incloud_mixing_phys(); - TD::run_incloud_mixing_bfb(); + T t; + t.run_incloud_mixing_phys(); + t.run_incloud_mixing_bfb(); } } diff --git a/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp index a804bd8756d..4fb471c3cf5 100644 --- a/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_main_unit_tests.cpp @@ -4,8 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" -#include "share/util/scream_setup_random_test.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -19,29 +18,29 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestP3Main { +struct UnitWrap::UnitTest::TestP3Main : public UnitWrap::UnitTest::Base { -static void run_phys_p3_main_part1() +void run_phys_p3_main_part1() { // TODO } -static void run_phys_p3_main_part2() +void run_phys_p3_main_part2() { // TODO } -static void run_phys_p3_main_part3() +void run_phys_p3_main_part3() { // TODO } -static void run_phys_p3_main() +void run_phys_p3_main() { // TODO } -static void run_phys() +void run_phys() { run_phys_p3_main_part1(); run_phys_p3_main_part2(); @@ -49,9 +48,9 @@ static void run_phys() run_phys_p3_main(); } -static void run_bfb_p3_main_part1() +void run_bfb_p3_main_part1() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); constexpr Scalar qsmall = C::QSMALL; //PMC wouldn't it make more sense to define qsmall at a higher level since used in part1, part2, and part3? constexpr Scalar T_zerodegc = C::T_zerodegc; @@ -60,7 +59,7 @@ static void run_bfb_p3_main_part1() constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; - P3MainPart1Data isds_fortran[] = { + P3MainPart1Data isds_baseline[] = { // kts, kte, ktop, kbot, kdir, do_predict_nc, do_prescribed_CCN, dt P3MainPart1Data(1, 72, 1, 72, 1, false, true, 1.800E+03), P3MainPart1Data(1, 72, 1, 72, 1, true, true, 1.800E+03), @@ -68,9 +67,9 @@ static void run_bfb_p3_main_part1() P3MainPart1Data(1, 72, 72, 1, -1, true, false, 1.800E+03), }; - static constexpr Int num_runs = sizeof(isds_fortran) / sizeof(P3MainPart1Data); + static constexpr Int num_runs = sizeof(isds_baseline) / sizeof(P3MainPart1Data); - for (auto& d : isds_fortran) { + for (auto& d : isds_baseline) { const auto qsmall_r = std::make_pair(0, qsmall*2); //PMC this range seems inappropriately small d.randomize(engine, { {d.T_atm, {T_zerodegc - 10, T_zerodegc + 10}}, @@ -86,23 +85,25 @@ static void run_bfb_p3_main_part1() } } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state P3MainPart1Data isds_cxx[num_runs] = { - P3MainPart1Data(isds_fortran[0]), - P3MainPart1Data(isds_fortran[1]), - P3MainPart1Data(isds_fortran[2]), - P3MainPart1Data(isds_fortran[3]), + P3MainPart1Data(isds_baseline[0]), + P3MainPart1Data(isds_baseline[1]), + P3MainPart1Data(isds_baseline[2]), + P3MainPart1Data(isds_baseline[3]), }; - // Get data from fortran - for (auto& d : isds_fortran) { - p3_main_part1(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : isds_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : isds_cxx) { - p3_main_part1_f(d.kts, d.kte, d.ktop, d.kbot, d.kdir, d.do_predict_nc, d.do_prescribed_CCN, d.dt, + p3_main_part1_host(d.kts, d.kte, d.ktop, d.kbot, d.kdir, d.do_predict_nc, d.do_prescribed_CCN, d.dt, d.pres, d.dpres, d.dz, d.nc_nuceat_tend, d.nccn_prescribed, d.inv_exner, d.exner, d.inv_cld_frac_l, d.inv_cld_frac_i, d.inv_cld_frac_r, d.T_atm, d.rho, d.inv_rho, d.qv_sat_l, d.qv_sat_i, d.qv_supersat_i, d.rhofacr, d.rhofaci, @@ -111,48 +112,53 @@ static void run_bfb_p3_main_part1() &d.is_nucleat_possible, &d.is_hydromet_present); } - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - Int start = std::min(isds_fortran[i].kbot, isds_fortran[i].ktop) - 1; // 0-based indx - Int end = std::max(isds_fortran[i].kbot, isds_fortran[i].ktop); // 0-based indx + Int start = std::min(isds_baseline[i].kbot, isds_baseline[i].ktop) - 1; // 0-based indx + Int end = std::max(isds_baseline[i].kbot, isds_baseline[i].ktop); // 0-based indx for (Int k = start; k < end; ++k) { - REQUIRE(isds_fortran[i].T_atm[k] == isds_cxx[i].T_atm[k]); - REQUIRE(isds_fortran[i].rho[k] == isds_cxx[i].rho[k]); - REQUIRE(isds_fortran[i].inv_rho[k] == isds_cxx[i].inv_rho[k]); - REQUIRE(isds_fortran[i].qv_sat_l[k] == isds_cxx[i].qv_sat_l[k]); - REQUIRE(isds_fortran[i].qv_sat_i[k] == isds_cxx[i].qv_sat_i[k]); - REQUIRE(isds_fortran[i].qv_supersat_i[k] == isds_cxx[i].qv_supersat_i[k]); - REQUIRE(isds_fortran[i].rhofacr[k] == isds_cxx[i].rhofacr[k]); - REQUIRE(isds_fortran[i].rhofaci[k] == isds_cxx[i].rhofaci[k]); - REQUIRE(isds_fortran[i].acn[k] == isds_cxx[i].acn[k]); - REQUIRE(isds_fortran[i].qv[k] == isds_cxx[i].qv[k]); - REQUIRE(isds_fortran[i].th_atm[k] == isds_cxx[i].th_atm[k]); - REQUIRE(isds_fortran[i].qc[k] == isds_cxx[i].qc[k]); - REQUIRE(isds_fortran[i].nc[k] == isds_cxx[i].nc[k]); - REQUIRE(isds_fortran[i].qr[k] == isds_cxx[i].qr[k]); - REQUIRE(isds_fortran[i].nr[k] == isds_cxx[i].nr[k]); - REQUIRE(isds_fortran[i].qi[k] == isds_cxx[i].qi[k]); - REQUIRE(isds_fortran[i].ni[k] == isds_cxx[i].ni[k]); - REQUIRE(isds_fortran[i].qm[k] == isds_cxx[i].qm[k]); - REQUIRE(isds_fortran[i].bm[k] == isds_cxx[i].bm[k]); - REQUIRE(isds_fortran[i].qc_incld[k] == isds_cxx[i].qc_incld[k]); - REQUIRE(isds_fortran[i].qr_incld[k] == isds_cxx[i].qr_incld[k]); - REQUIRE(isds_fortran[i].qi_incld[k] == isds_cxx[i].qi_incld[k]); - REQUIRE(isds_fortran[i].qm_incld[k] == isds_cxx[i].qm_incld[k]); - REQUIRE(isds_fortran[i].nc_incld[k] == isds_cxx[i].nc_incld[k]); - REQUIRE(isds_fortran[i].nr_incld[k] == isds_cxx[i].nr_incld[k]); - REQUIRE(isds_fortran[i].ni_incld[k] == isds_cxx[i].ni_incld[k]); - REQUIRE(isds_fortran[i].bm_incld[k] == isds_cxx[i].bm_incld[k]); + REQUIRE(isds_baseline[i].T_atm[k] == isds_cxx[i].T_atm[k]); + REQUIRE(isds_baseline[i].rho[k] == isds_cxx[i].rho[k]); + REQUIRE(isds_baseline[i].inv_rho[k] == isds_cxx[i].inv_rho[k]); + REQUIRE(isds_baseline[i].qv_sat_l[k] == isds_cxx[i].qv_sat_l[k]); + REQUIRE(isds_baseline[i].qv_sat_i[k] == isds_cxx[i].qv_sat_i[k]); + REQUIRE(isds_baseline[i].qv_supersat_i[k] == isds_cxx[i].qv_supersat_i[k]); + REQUIRE(isds_baseline[i].rhofacr[k] == isds_cxx[i].rhofacr[k]); + REQUIRE(isds_baseline[i].rhofaci[k] == isds_cxx[i].rhofaci[k]); + REQUIRE(isds_baseline[i].acn[k] == isds_cxx[i].acn[k]); + REQUIRE(isds_baseline[i].qv[k] == isds_cxx[i].qv[k]); + REQUIRE(isds_baseline[i].th_atm[k] == isds_cxx[i].th_atm[k]); + REQUIRE(isds_baseline[i].qc[k] == isds_cxx[i].qc[k]); + REQUIRE(isds_baseline[i].nc[k] == isds_cxx[i].nc[k]); + REQUIRE(isds_baseline[i].qr[k] == isds_cxx[i].qr[k]); + REQUIRE(isds_baseline[i].nr[k] == isds_cxx[i].nr[k]); + REQUIRE(isds_baseline[i].qi[k] == isds_cxx[i].qi[k]); + REQUIRE(isds_baseline[i].ni[k] == isds_cxx[i].ni[k]); + REQUIRE(isds_baseline[i].qm[k] == isds_cxx[i].qm[k]); + REQUIRE(isds_baseline[i].bm[k] == isds_cxx[i].bm[k]); + REQUIRE(isds_baseline[i].qc_incld[k] == isds_cxx[i].qc_incld[k]); + REQUIRE(isds_baseline[i].qr_incld[k] == isds_cxx[i].qr_incld[k]); + REQUIRE(isds_baseline[i].qi_incld[k] == isds_cxx[i].qi_incld[k]); + REQUIRE(isds_baseline[i].qm_incld[k] == isds_cxx[i].qm_incld[k]); + REQUIRE(isds_baseline[i].nc_incld[k] == isds_cxx[i].nc_incld[k]); + REQUIRE(isds_baseline[i].nr_incld[k] == isds_cxx[i].nr_incld[k]); + REQUIRE(isds_baseline[i].ni_incld[k] == isds_cxx[i].ni_incld[k]); + REQUIRE(isds_baseline[i].bm_incld[k] == isds_cxx[i].bm_incld[k]); } - REQUIRE( isds_fortran[i].is_hydromet_present == isds_cxx[i].is_hydromet_present ); - REQUIRE( isds_fortran[i].is_nucleat_possible == isds_cxx[i].is_nucleat_possible ); + REQUIRE( isds_baseline[i].is_hydromet_present == isds_cxx[i].is_hydromet_present ); + REQUIRE( isds_baseline[i].is_nucleat_possible == isds_cxx[i].is_nucleat_possible ); + } + } + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + isds_cxx[i].write(Base::m_fid); } } } -static void run_bfb_p3_main_part2() +void run_bfb_p3_main_part2() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); constexpr Scalar qsmall = C::QSMALL; constexpr Scalar T_zerodegc = C::T_zerodegc; @@ -161,7 +167,7 @@ static void run_bfb_p3_main_part2() constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; - P3MainPart2Data isds_fortran[] = { + P3MainPart2Data isds_baseline[] = { // kts, kte, ktop, kbot, kdir, do_predict_nc, do_prescribed_CCN, dt P3MainPart2Data(1, 72, 1, 72, 1, false, true, 1.800E+03), P3MainPart2Data(1, 72, 1, 72, 1, true, true, 1.800E+03), @@ -169,9 +175,9 @@ static void run_bfb_p3_main_part2() P3MainPart2Data(1, 72, 72, 1, -1, true, false, 1.800E+03), }; - static constexpr Int num_runs = sizeof(isds_fortran) / sizeof(P3MainPart2Data); + static constexpr Int num_runs = sizeof(isds_baseline) / sizeof(P3MainPart2Data); - for (auto& d : isds_fortran) { + for (auto& d : isds_baseline) { const auto qsmall_r = std::make_pair(0, qsmall*2); d.randomize(engine, { {d.T_atm, {T_zerodegc - 10, T_zerodegc + 10}}, @@ -188,23 +194,25 @@ static void run_bfb_p3_main_part2() } } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state P3MainPart2Data isds_cxx[num_runs] = { - P3MainPart2Data(isds_fortran[0]), - P3MainPart2Data(isds_fortran[1]), - P3MainPart2Data(isds_fortran[2]), - P3MainPart2Data(isds_fortran[3]), + P3MainPart2Data(isds_baseline[0]), + P3MainPart2Data(isds_baseline[1]), + P3MainPart2Data(isds_baseline[2]), + P3MainPart2Data(isds_baseline[3]), }; - // Get data from fortran - for (auto& d : isds_fortran) { - p3_main_part2(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : isds_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : isds_cxx) { - p3_main_part2_f( + p3_main_part2_host( d.kts, d.kte, d.kbot, d.ktop, d.kdir, d.do_predict_nc, d.do_prescribed_CCN, d.dt, d.inv_dt, d.pres, d.dpres, d.dz, d.nc_nuceat_tend, d.inv_exner, d.exner, d.inv_cld_frac_l, d.inv_cld_frac_i, d.inv_cld_frac_r, d.ni_activated, d.inv_qc_relvar, d.cld_frac_i, d.cld_frac_l, d.cld_frac_r, d.qv_prev, d.t_prev, @@ -215,75 +223,80 @@ static void run_bfb_p3_main_part2() d.prctot, &d.is_hydromet_present); } - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - Int start = std::min(isds_fortran[i].kbot, isds_fortran[i].ktop) - 1; // 0-based indx - Int end = std::max(isds_fortran[i].kbot, isds_fortran[i].ktop); // 0-based indx + Int start = std::min(isds_baseline[i].kbot, isds_baseline[i].ktop) - 1; // 0-based indx + Int end = std::max(isds_baseline[i].kbot, isds_baseline[i].ktop); // 0-based indx for (Int k = start; k < end; ++k) { - REQUIRE(isds_fortran[i].T_atm[k] == isds_cxx[i].T_atm[k]); - REQUIRE(isds_fortran[i].rho[k] == isds_cxx[i].rho[k]); - REQUIRE(isds_fortran[i].inv_rho[k] == isds_cxx[i].inv_rho[k]); - REQUIRE(isds_fortran[i].qv_sat_l[k] == isds_cxx[i].qv_sat_l[k]); - REQUIRE(isds_fortran[i].qv_sat_i[k] == isds_cxx[i].qv_sat_i[k]); - REQUIRE(isds_fortran[i].qv_supersat_i[k] == isds_cxx[i].qv_supersat_i[k]); - REQUIRE(isds_fortran[i].rhofacr[k] == isds_cxx[i].rhofacr[k]); - REQUIRE(isds_fortran[i].rhofaci[k] == isds_cxx[i].rhofaci[k]); - REQUIRE(isds_fortran[i].acn[k] == isds_cxx[i].acn[k]); - REQUIRE(isds_fortran[i].qv[k] == isds_cxx[i].qv[k]); - REQUIRE(isds_fortran[i].th_atm[k] == isds_cxx[i].th_atm[k]); - REQUIRE(isds_fortran[i].qc[k] == isds_cxx[i].qc[k]); - REQUIRE(isds_fortran[i].nc[k] == isds_cxx[i].nc[k]); - REQUIRE(isds_fortran[i].qr[k] == isds_cxx[i].qr[k]); - REQUIRE(isds_fortran[i].nr[k] == isds_cxx[i].nr[k]); - REQUIRE(isds_fortran[i].qi[k] == isds_cxx[i].qi[k]); - REQUIRE(isds_fortran[i].ni[k] == isds_cxx[i].ni[k]); - REQUIRE(isds_fortran[i].qm[k] == isds_cxx[i].qm[k]); - REQUIRE(isds_fortran[i].bm[k] == isds_cxx[i].bm[k]); - REQUIRE(isds_fortran[i].latent_heat_vapor[k] == latvap); - REQUIRE(isds_fortran[i].latent_heat_sublim[k] == (latvap+latice)); - REQUIRE(isds_fortran[i].latent_heat_fusion[k] == latice); - REQUIRE(isds_fortran[i].qc_incld[k] == isds_cxx[i].qc_incld[k]); - REQUIRE(isds_fortran[i].qr_incld[k] == isds_cxx[i].qr_incld[k]); - REQUIRE(isds_fortran[i].qi_incld[k] == isds_cxx[i].qi_incld[k]); - REQUIRE(isds_fortran[i].qm_incld[k] == isds_cxx[i].qm_incld[k]); - REQUIRE(isds_fortran[i].nc_incld[k] == isds_cxx[i].nc_incld[k]); - REQUIRE(isds_fortran[i].nr_incld[k] == isds_cxx[i].nr_incld[k]); - REQUIRE(isds_fortran[i].ni_incld[k] == isds_cxx[i].ni_incld[k]); - REQUIRE(isds_fortran[i].bm_incld[k] == isds_cxx[i].bm_incld[k]); - REQUIRE(isds_fortran[i].mu_c[k] == isds_cxx[i].mu_c[k]); - REQUIRE(isds_fortran[i].nu[k] == isds_cxx[i].nu[k]); - REQUIRE(isds_fortran[i].lamc[k] == isds_cxx[i].lamc[k]); - REQUIRE(isds_fortran[i].cdist[k] == isds_cxx[i].cdist[k]); - REQUIRE(isds_fortran[i].cdist1[k] == isds_cxx[i].cdist1[k]); - REQUIRE(isds_fortran[i].cdistr[k] == isds_cxx[i].cdistr[k]); - REQUIRE(isds_fortran[i].mu_r[k] == isds_cxx[i].mu_r[k]); - REQUIRE(isds_fortran[i].lamr[k] == isds_cxx[i].lamr[k]); - REQUIRE(isds_fortran[i].logn0r[k] == isds_cxx[i].logn0r[k]); - REQUIRE(isds_fortran[i].qv2qi_depos_tend[k] == isds_cxx[i].qv2qi_depos_tend[k]); - REQUIRE(isds_fortran[i].precip_total_tend[k] == isds_cxx[i].precip_total_tend[k]); - REQUIRE(isds_fortran[i].nevapr[k] == isds_cxx[i].nevapr[k]); - REQUIRE(isds_fortran[i].qr_evap_tend[k] == isds_cxx[i].qr_evap_tend[k]); - REQUIRE(isds_fortran[i].vap_liq_exchange[k] == isds_cxx[i].vap_liq_exchange[k]); - REQUIRE(isds_fortran[i].vap_ice_exchange[k] == isds_cxx[i].vap_ice_exchange[k]); - REQUIRE(isds_fortran[i].liq_ice_exchange[k] == isds_cxx[i].liq_ice_exchange[k]); - REQUIRE(isds_fortran[i].pratot[k] == isds_cxx[i].pratot[k]); - REQUIRE(isds_fortran[i].prctot[k] == isds_cxx[i].prctot[k]); + REQUIRE(isds_baseline[i].T_atm[k] == isds_cxx[i].T_atm[k]); + REQUIRE(isds_baseline[i].rho[k] == isds_cxx[i].rho[k]); + REQUIRE(isds_baseline[i].inv_rho[k] == isds_cxx[i].inv_rho[k]); + REQUIRE(isds_baseline[i].qv_sat_l[k] == isds_cxx[i].qv_sat_l[k]); + REQUIRE(isds_baseline[i].qv_sat_i[k] == isds_cxx[i].qv_sat_i[k]); + REQUIRE(isds_baseline[i].qv_supersat_i[k] == isds_cxx[i].qv_supersat_i[k]); + REQUIRE(isds_baseline[i].rhofacr[k] == isds_cxx[i].rhofacr[k]); + REQUIRE(isds_baseline[i].rhofaci[k] == isds_cxx[i].rhofaci[k]); + REQUIRE(isds_baseline[i].acn[k] == isds_cxx[i].acn[k]); + REQUIRE(isds_baseline[i].qv[k] == isds_cxx[i].qv[k]); + REQUIRE(isds_baseline[i].th_atm[k] == isds_cxx[i].th_atm[k]); + REQUIRE(isds_baseline[i].qc[k] == isds_cxx[i].qc[k]); + REQUIRE(isds_baseline[i].nc[k] == isds_cxx[i].nc[k]); + REQUIRE(isds_baseline[i].qr[k] == isds_cxx[i].qr[k]); + REQUIRE(isds_baseline[i].nr[k] == isds_cxx[i].nr[k]); + REQUIRE(isds_baseline[i].qi[k] == isds_cxx[i].qi[k]); + REQUIRE(isds_baseline[i].ni[k] == isds_cxx[i].ni[k]); + REQUIRE(isds_baseline[i].qm[k] == isds_cxx[i].qm[k]); + REQUIRE(isds_baseline[i].bm[k] == isds_cxx[i].bm[k]); + REQUIRE(isds_baseline[i].latent_heat_vapor[k] == latvap); + REQUIRE(isds_baseline[i].latent_heat_sublim[k] == (latvap+latice)); + REQUIRE(isds_baseline[i].latent_heat_fusion[k] == latice); + REQUIRE(isds_baseline[i].qc_incld[k] == isds_cxx[i].qc_incld[k]); + REQUIRE(isds_baseline[i].qr_incld[k] == isds_cxx[i].qr_incld[k]); + REQUIRE(isds_baseline[i].qi_incld[k] == isds_cxx[i].qi_incld[k]); + REQUIRE(isds_baseline[i].qm_incld[k] == isds_cxx[i].qm_incld[k]); + REQUIRE(isds_baseline[i].nc_incld[k] == isds_cxx[i].nc_incld[k]); + REQUIRE(isds_baseline[i].nr_incld[k] == isds_cxx[i].nr_incld[k]); + REQUIRE(isds_baseline[i].ni_incld[k] == isds_cxx[i].ni_incld[k]); + REQUIRE(isds_baseline[i].bm_incld[k] == isds_cxx[i].bm_incld[k]); + REQUIRE(isds_baseline[i].mu_c[k] == isds_cxx[i].mu_c[k]); + REQUIRE(isds_baseline[i].nu[k] == isds_cxx[i].nu[k]); + REQUIRE(isds_baseline[i].lamc[k] == isds_cxx[i].lamc[k]); + REQUIRE(isds_baseline[i].cdist[k] == isds_cxx[i].cdist[k]); + REQUIRE(isds_baseline[i].cdist1[k] == isds_cxx[i].cdist1[k]); + REQUIRE(isds_baseline[i].cdistr[k] == isds_cxx[i].cdistr[k]); + REQUIRE(isds_baseline[i].mu_r[k] == isds_cxx[i].mu_r[k]); + REQUIRE(isds_baseline[i].lamr[k] == isds_cxx[i].lamr[k]); + REQUIRE(isds_baseline[i].logn0r[k] == isds_cxx[i].logn0r[k]); + REQUIRE(isds_baseline[i].qv2qi_depos_tend[k] == isds_cxx[i].qv2qi_depos_tend[k]); + REQUIRE(isds_baseline[i].precip_total_tend[k] == isds_cxx[i].precip_total_tend[k]); + REQUIRE(isds_baseline[i].nevapr[k] == isds_cxx[i].nevapr[k]); + REQUIRE(isds_baseline[i].qr_evap_tend[k] == isds_cxx[i].qr_evap_tend[k]); + REQUIRE(isds_baseline[i].vap_liq_exchange[k] == isds_cxx[i].vap_liq_exchange[k]); + REQUIRE(isds_baseline[i].vap_ice_exchange[k] == isds_cxx[i].vap_ice_exchange[k]); + REQUIRE(isds_baseline[i].liq_ice_exchange[k] == isds_cxx[i].liq_ice_exchange[k]); + REQUIRE(isds_baseline[i].pratot[k] == isds_cxx[i].pratot[k]); + REQUIRE(isds_baseline[i].prctot[k] == isds_cxx[i].prctot[k]); } - REQUIRE( isds_fortran[i].is_hydromet_present == isds_cxx[i].is_hydromet_present ); + REQUIRE( isds_baseline[i].is_hydromet_present == isds_cxx[i].is_hydromet_present ); + } + } + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + isds_cxx[i].write(Base::m_fid); } } } -static void run_bfb_p3_main_part3() +void run_bfb_p3_main_part3() { constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; - auto engine = setup_random_test(); + auto engine = Base::get_engine(); constexpr Scalar qsmall = C::QSMALL; - P3MainPart3Data isds_fortran[] = { + P3MainPart3Data isds_baseline[] = { // kts, kte, ktop, kbot, kdir P3MainPart3Data(1, 72, 1, 72, 1), P3MainPart3Data(1, 72, 1, 72, 1), @@ -291,9 +304,9 @@ static void run_bfb_p3_main_part3() P3MainPart3Data(1, 72, 72, 1, -1), }; - static constexpr Int num_runs = sizeof(isds_fortran) / sizeof(P3MainPart3Data); + static constexpr Int num_runs = sizeof(isds_baseline) / sizeof(P3MainPart3Data); - for (auto& d : isds_fortran) { + for (auto& d : isds_baseline) { const auto qsmall_r = std::make_pair(0, qsmall*2); d.randomize(engine, { {d.qc, qsmall_r}, {d.qr, qsmall_r}, {d.qi, qsmall_r} }); @@ -305,23 +318,25 @@ static void run_bfb_p3_main_part3() } } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state P3MainPart3Data isds_cxx[num_runs] = { - P3MainPart3Data(isds_fortran[0]), - P3MainPart3Data(isds_fortran[1]), - P3MainPart3Data(isds_fortran[2]), - P3MainPart3Data(isds_fortran[3]), + P3MainPart3Data(isds_baseline[0]), + P3MainPart3Data(isds_baseline[1]), + P3MainPart3Data(isds_baseline[2]), + P3MainPart3Data(isds_baseline[3]), }; - // Get data from fortran - for (auto& d : isds_fortran) { - p3_main_part3(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : isds_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : isds_cxx) { - p3_main_part3_f( + p3_main_part3_host( d.kts, d.kte, d.kbot, d.ktop, d.kdir, d.inv_exner, d.cld_frac_l, d.cld_frac_r, d.cld_frac_i, d.rho, d.inv_rho, d.rhofaci, d.qv, d.th_atm, d.qc, d.nc, d.qr, d.nr, d.qi, d.ni, d.qm, d.bm, @@ -329,59 +344,64 @@ static void run_bfb_p3_main_part3() d. ze_rain, d.ze_ice, d.diag_vm_qi, d.diag_eff_radius_qi, d.diag_diam_qi, d.rho_qi, d.diag_equiv_reflectivity, d.diag_eff_radius_qc, d.diag_eff_radius_qr); } - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - Int start = std::min(isds_fortran[i].kbot, isds_fortran[i].ktop) - 1; // 0-based indx - Int end = std::max(isds_fortran[i].kbot, isds_fortran[i].ktop); // 0-based indx + Int start = std::min(isds_baseline[i].kbot, isds_baseline[i].ktop) - 1; // 0-based indx + Int end = std::max(isds_baseline[i].kbot, isds_baseline[i].ktop); // 0-based indx for (Int k = start; k < end; ++k) { - REQUIRE(isds_fortran[i].rho[k] == isds_cxx[i].rho[k]); - REQUIRE(isds_fortran[i].inv_rho[k] == isds_cxx[i].inv_rho[k]); - REQUIRE(isds_fortran[i].rhofaci[k] == isds_cxx[i].rhofaci[k]); - REQUIRE(isds_fortran[i].qv[k] == isds_cxx[i].qv[k]); - REQUIRE(isds_fortran[i].th_atm[k] == isds_cxx[i].th_atm[k]); - REQUIRE(isds_fortran[i].qc[k] == isds_cxx[i].qc[k]); - REQUIRE(isds_fortran[i].nc[k] == isds_cxx[i].nc[k]); - REQUIRE(isds_fortran[i].qr[k] == isds_cxx[i].qr[k]); - REQUIRE(isds_fortran[i].nr[k] == isds_cxx[i].nr[k]); - REQUIRE(isds_fortran[i].qi[k] == isds_cxx[i].qi[k]); - REQUIRE(isds_fortran[i].ni[k] == isds_cxx[i].ni[k]); - REQUIRE(isds_fortran[i].qm[k] == isds_cxx[i].qm[k]); - REQUIRE(isds_fortran[i].bm[k] == isds_cxx[i].bm[k]); - REQUIRE(isds_fortran[i].latent_heat_vapor[k] == latvap); - REQUIRE(isds_fortran[i].latent_heat_sublim[k] == latvap+latice); - REQUIRE(isds_fortran[i].mu_c[k] == isds_cxx[i].mu_c[k]); - REQUIRE(isds_fortran[i].nu[k] == isds_cxx[i].nu[k]); - REQUIRE(isds_fortran[i].lamc[k] == isds_cxx[i].lamc[k]); - REQUIRE(isds_fortran[i].mu_r[k] == isds_cxx[i].mu_r[k]); - REQUIRE(isds_fortran[i].lamr[k] == isds_cxx[i].lamr[k]); - REQUIRE(isds_fortran[i].vap_liq_exchange[k] == isds_cxx[i].vap_liq_exchange[k]); - REQUIRE(isds_fortran[i].ze_rain[k] == isds_cxx[i].ze_rain[k]); - REQUIRE(isds_fortran[i].ze_ice[k] == isds_cxx[i].ze_ice[k]); - REQUIRE(isds_fortran[i].diag_vm_qi[k] == isds_cxx[i].diag_vm_qi[k]); - REQUIRE(isds_fortran[i].diag_eff_radius_qi[k] == isds_cxx[i].diag_eff_radius_qi[k]); - REQUIRE(isds_fortran[i].diag_diam_qi[k] == isds_cxx[i].diag_diam_qi[k]); - REQUIRE(isds_fortran[i].rho_qi[k] == isds_cxx[i].rho_qi[k]); - REQUIRE(isds_fortran[i].diag_equiv_reflectivity[k] == isds_cxx[i].diag_equiv_reflectivity[k]); - REQUIRE(isds_fortran[i].diag_eff_radius_qc[k] == isds_cxx[i].diag_eff_radius_qc[k]); - REQUIRE(isds_fortran[i].diag_eff_radius_qr[k] == isds_cxx[i].diag_eff_radius_qr[k]); + REQUIRE(isds_baseline[i].rho[k] == isds_cxx[i].rho[k]); + REQUIRE(isds_baseline[i].inv_rho[k] == isds_cxx[i].inv_rho[k]); + REQUIRE(isds_baseline[i].rhofaci[k] == isds_cxx[i].rhofaci[k]); + REQUIRE(isds_baseline[i].qv[k] == isds_cxx[i].qv[k]); + REQUIRE(isds_baseline[i].th_atm[k] == isds_cxx[i].th_atm[k]); + REQUIRE(isds_baseline[i].qc[k] == isds_cxx[i].qc[k]); + REQUIRE(isds_baseline[i].nc[k] == isds_cxx[i].nc[k]); + REQUIRE(isds_baseline[i].qr[k] == isds_cxx[i].qr[k]); + REQUIRE(isds_baseline[i].nr[k] == isds_cxx[i].nr[k]); + REQUIRE(isds_baseline[i].qi[k] == isds_cxx[i].qi[k]); + REQUIRE(isds_baseline[i].ni[k] == isds_cxx[i].ni[k]); + REQUIRE(isds_baseline[i].qm[k] == isds_cxx[i].qm[k]); + REQUIRE(isds_baseline[i].bm[k] == isds_cxx[i].bm[k]); + REQUIRE(isds_baseline[i].latent_heat_vapor[k] == latvap); + REQUIRE(isds_baseline[i].latent_heat_sublim[k] == latvap+latice); + REQUIRE(isds_baseline[i].mu_c[k] == isds_cxx[i].mu_c[k]); + REQUIRE(isds_baseline[i].nu[k] == isds_cxx[i].nu[k]); + REQUIRE(isds_baseline[i].lamc[k] == isds_cxx[i].lamc[k]); + REQUIRE(isds_baseline[i].mu_r[k] == isds_cxx[i].mu_r[k]); + REQUIRE(isds_baseline[i].lamr[k] == isds_cxx[i].lamr[k]); + REQUIRE(isds_baseline[i].vap_liq_exchange[k] == isds_cxx[i].vap_liq_exchange[k]); + REQUIRE(isds_baseline[i].ze_rain[k] == isds_cxx[i].ze_rain[k]); + REQUIRE(isds_baseline[i].ze_ice[k] == isds_cxx[i].ze_ice[k]); + REQUIRE(isds_baseline[i].diag_vm_qi[k] == isds_cxx[i].diag_vm_qi[k]); + REQUIRE(isds_baseline[i].diag_eff_radius_qi[k] == isds_cxx[i].diag_eff_radius_qi[k]); + REQUIRE(isds_baseline[i].diag_diam_qi[k] == isds_cxx[i].diag_diam_qi[k]); + REQUIRE(isds_baseline[i].rho_qi[k] == isds_cxx[i].rho_qi[k]); + REQUIRE(isds_baseline[i].diag_equiv_reflectivity[k] == isds_cxx[i].diag_equiv_reflectivity[k]); + REQUIRE(isds_baseline[i].diag_eff_radius_qc[k] == isds_cxx[i].diag_eff_radius_qc[k]); + REQUIRE(isds_baseline[i].diag_eff_radius_qr[k] == isds_cxx[i].diag_eff_radius_qr[k]); } } } + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + isds_cxx[i].write(Base::m_fid); + } + } } -static void run_bfb_p3_main() +void run_bfb_p3_main() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - P3MainData isds_fortran[] = { + P3MainData isds_baseline[] = { // its, ite, kts, kte, it, dt, do_predict_nc, do_prescribed_CCN P3MainData(1, 10, 1, 72, 1, 1.800E+03, false, true), P3MainData(1, 10, 1, 72, 1, 1.800E+03, true, false), }; - static constexpr Int num_runs = sizeof(isds_fortran) / sizeof(P3MainData); + static constexpr Int num_runs = sizeof(isds_baseline) / sizeof(P3MainData); - for (auto& d : isds_fortran) { + for (auto& d : isds_baseline) { d.randomize(engine, { {d.pres , {1.00000000E+02 , 9.87111111E+04}}, {d.dz , {1.22776609E+02 , 3.49039167E+04}}, @@ -409,22 +429,24 @@ static void run_bfb_p3_main() }); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state P3MainData isds_cxx[num_runs] = { - P3MainData(isds_fortran[0]), - P3MainData(isds_fortran[1]), + P3MainData(isds_baseline[0]), + P3MainData(isds_baseline[1]), }; - // Get data from fortran - for (auto& d : isds_fortran) { - p3_main(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : isds_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx for (auto& d : isds_cxx) { d.template transpose(); - p3_main_f( + p3_main_host( d.qc, d.nc, d.qr, d.nr, d.th_atm, d.qv, d.dt, d.qi, d.qm, d.ni, d.bm, d.pres, d.dz, d.nc_nuceat_tend, d.nccn_prescribed, d.ni_activated, d.inv_qc_relvar, d.it, d.precip_liq_surf, d.precip_ice_surf, d.its, d.ite, d.kts, d.kte, d.diag_eff_radius_qc, d.diag_eff_radius_qi, d.diag_eff_radius_qr, @@ -434,11 +456,11 @@ static void run_bfb_p3_main() d.template transpose(); } - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { - const auto& df90 = isds_fortran[i]; - const auto& dcxx = isds_fortran[i]; - const auto tot = isds_fortran[i].total(df90.qc); + const auto& df90 = isds_baseline[i]; + const auto& dcxx = isds_baseline[i]; + const auto tot = isds_baseline[i].total(df90.qc); for (Int t = 0; t < tot; ++t) { REQUIRE(df90.qc[t] == dcxx.qc[t]); REQUIRE(df90.nc[t] == dcxx.nc[t]); @@ -474,9 +496,14 @@ static void run_bfb_p3_main() REQUIRE(df90.precip_ice_surf[tot] == dcxx.precip_ice_surf[tot]); } } + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + isds_cxx[i].write(Base::m_fid); + } + } } -static void run_bfb() +void run_bfb() { run_bfb_p3_main_part1(); run_bfb_p3_main_part2(); @@ -494,12 +521,11 @@ namespace { TEST_CASE("p3_main", "[p3_functions]") { - using TP3 = scream::p3::unit_test::UnitWrap::UnitTest::TestP3Main; - - TP3::run_phys(); - TP3::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestP3Main; - scream::p3::P3GlobalForFortran::deinit(); + T t; + t.run_phys(); + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp index ffe7ea504e1..175247bb501 100644 --- a/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_nc_conservation_tests.cpp @@ -4,8 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" -#include "share/util/scream_setup_random_test.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -14,31 +13,33 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestNcConservation { +struct UnitWrap::UnitTest::TestNcConservation : public UnitWrap::UnitTest::Base { - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - NcConservationData f90_data[max_pack_size]; + NcConservationData baseline_data[max_pack_size]; // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { + // Alternatively, you can use the baseline_data construtors/initializer lists to hardcode data + for (auto& d : baseline_data) { d.randomize(engine); - d.dt = f90_data[0].dt; // Hold this fixed, this is not packed data + d.dt = baseline_data[0].dt; // Hold this fixed, this is not packed data } - // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that + // Create copies of data for use by cxx and sync it to device. Needs to happen before reads so that // inout data is in original state view_1d cxx_device("cxx_device", max_pack_size); const auto cxx_host = Kokkos::create_mirror_view(cxx_device); - std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); + std::copy(&baseline_data[0], &baseline_data[0] + max_pack_size, cxx_host.data()); Kokkos::deep_copy(cxx_device, cxx_host); - // Get data from fortran - for (auto& d : f90_data) { - nc_conservation(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + baseline_data[i].read(Base::m_fid); + } } // Get data from cxx. Run nc_conservation from a kernel and copy results back to host @@ -71,14 +72,19 @@ struct UnitWrap::UnitTest::TestNcConservation { Kokkos::deep_copy(cxx_host, cxx_device); // Verify BFB results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - NcConservationData& d_f90 = f90_data[i]; + NcConservationData& d_baseline = baseline_data[i]; NcConservationData& d_cxx = cxx_host[i]; - REQUIRE(d_f90.nc_collect_tend == d_cxx.nc_collect_tend); - REQUIRE(d_f90.nc2ni_immers_freeze_tend == d_cxx.nc2ni_immers_freeze_tend); - REQUIRE(d_f90.nc_accret_tend == d_cxx.nc_accret_tend); - REQUIRE(d_f90.nc2nr_autoconv_tend == d_cxx.nc2nr_autoconv_tend); + REQUIRE(d_baseline.nc_collect_tend == d_cxx.nc_collect_tend); + REQUIRE(d_baseline.nc2ni_immers_freeze_tend == d_cxx.nc2ni_immers_freeze_tend); + REQUIRE(d_baseline.nc_accret_tend == d_cxx.nc_accret_tend); + REQUIRE(d_baseline.nc2nr_autoconv_tend == d_cxx.nc2nr_autoconv_tend); + } + } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + cxx_host(s).write(Base::m_fid); } } } // run_bfb @@ -93,9 +99,10 @@ namespace { TEST_CASE("nc_conservation_bfb", "[p3]") { - using TestStruct = scream::p3::unit_test::UnitWrap::UnitTest::TestNcConservation; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestNcConservation; - TestStruct::run_bfb(); + T t; + t.run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp index 3aa3428825c..b8cfdf4333a 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ni_conservation_tests.cpp @@ -4,8 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" -#include "share/util/scream_setup_random_test.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -14,31 +13,33 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestNiConservation { +struct UnitWrap::UnitTest::TestNiConservation : public UnitWrap::UnitTest::Base { - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - NiConservationData f90_data[max_pack_size]; + NiConservationData baseline_data[max_pack_size]; // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { + // Alternatively, you can use the baseline_data construtors/initializer lists to hardcode data + for (auto& d : baseline_data) { d.randomize(engine); - d.dt = f90_data[0].dt; // hold dt fixed, it is not packed data + d.dt = baseline_data[0].dt; // hold dt fixed, it is not packed data } - // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that + // Create copies of data for use by cxx and sync it to device. Needs to happen before reads so that // inout data is in original state view_1d cxx_device("cxx_device", max_pack_size); const auto cxx_host = Kokkos::create_mirror_view(cxx_device); - std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); + std::copy(&baseline_data[0], &baseline_data[0] + max_pack_size, cxx_host.data()); Kokkos::deep_copy(cxx_device, cxx_host); - // Get data from fortran - for (auto& d : f90_data) { - ni_conservation(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + baseline_data[i].read(Base::m_fid); + } } // Get data from cxx. Run ni_conservation from a kernel and copy results back to host @@ -71,17 +72,21 @@ struct UnitWrap::UnitTest::TestNiConservation { Kokkos::deep_copy(cxx_host, cxx_device); // Verify BFB results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - NiConservationData& d_f90 = f90_data[i]; + NiConservationData& d_baseline = baseline_data[i]; NiConservationData& d_cxx = cxx_host[i]; - REQUIRE(d_f90.ni2nr_melt_tend == d_cxx.ni2nr_melt_tend); - REQUIRE(d_f90.ni_sublim_tend == d_cxx.ni_sublim_tend); - REQUIRE(d_f90.ni_selfcollect_tend == d_cxx.ni_selfcollect_tend); + REQUIRE(d_baseline.ni2nr_melt_tend == d_cxx.ni2nr_melt_tend); + REQUIRE(d_baseline.ni_sublim_tend == d_cxx.ni_sublim_tend); + REQUIRE(d_baseline.ni_selfcollect_tend == d_cxx.ni_selfcollect_tend); + } + } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + cxx_host(s).write(Base::m_fid); } } } // run_bfb - }; } // namespace unit_test @@ -92,9 +97,10 @@ namespace { TEST_CASE("ni_conservation_bfb", "[p3]") { - using TestStruct = scream::p3::unit_test::UnitWrap::UnitTest::TestNiConservation; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestNiConservation; - TestStruct::run_bfb(); + T t; + t.run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp index fb9e1e758b4..dc9a44af88d 100644 --- a/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_nr_conservation_tests.cpp @@ -4,8 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" -#include "share/util/scream_setup_random_test.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -14,32 +13,34 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestNrConservation { +struct UnitWrap::UnitTest::TestNrConservation : public UnitWrap::UnitTest::Base { - static void run_bfb() + void run_bfb() { - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - NrConservationData f90_data[max_pack_size]; + NrConservationData baseline_data[max_pack_size]; // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { + // Alternatively, you can use the baseline_data construtors/initializer lists to hardcode data + for (auto& d : baseline_data) { d.randomize(engine); - d.dt = f90_data[0].dt; // hold dt fixed, it is not packed data - d.nmltratio = f90_data[0].nmltratio; // hold nmltratio fixed, it is not packed data + d.dt = baseline_data[0].dt; // hold dt fixed, it is not packed data + d.nmltratio = baseline_data[0].nmltratio; // hold nmltratio fixed, it is not packed data } - // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that + // Create copies of data for use by cxx and sync it to device. Needs to happen before reads so that // inout data is in original state view_1d cxx_device("cxx_device", max_pack_size); const auto cxx_host = Kokkos::create_mirror_view(cxx_device); - std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); + std::copy(&baseline_data[0], &baseline_data[0] + max_pack_size, cxx_host.data()); Kokkos::deep_copy(cxx_device, cxx_host); - // Get data from fortran - for (auto& d : f90_data) { - nr_conservation(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + baseline_data[i].read(Base::m_fid); + } } // Get data from cxx. Run nr_conservation from a kernel and copy results back to host @@ -75,14 +76,19 @@ struct UnitWrap::UnitTest::TestNrConservation { Kokkos::deep_copy(cxx_host, cxx_device); // Verify BFB results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { - NrConservationData& d_f90 = f90_data[i]; + NrConservationData& d_baseline = baseline_data[i]; NrConservationData& d_cxx = cxx_host[i]; - REQUIRE(d_f90.nr_collect_tend == d_cxx.nr_collect_tend); - REQUIRE(d_f90.nr2ni_immers_freeze_tend == d_cxx.nr2ni_immers_freeze_tend); - REQUIRE(d_f90.nr_selfcollect_tend == d_cxx.nr_selfcollect_tend); - REQUIRE(d_f90.nr_evap_tend == d_cxx.nr_evap_tend); + REQUIRE(d_baseline.nr_collect_tend == d_cxx.nr_collect_tend); + REQUIRE(d_baseline.nr2ni_immers_freeze_tend == d_cxx.nr2ni_immers_freeze_tend); + REQUIRE(d_baseline.nr_selfcollect_tend == d_cxx.nr_selfcollect_tend); + REQUIRE(d_baseline.nr_evap_tend == d_cxx.nr_evap_tend); + } + } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + cxx_host(s).write(Base::m_fid); } } } // run_bfb @@ -97,9 +103,10 @@ namespace { TEST_CASE("nr_conservation_bfb", "[p3]") { - using TestStruct = scream::p3::unit_test::UnitWrap::UnitTest::TestNrConservation; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestNrConservation; - TestStruct::run_bfb(); + T t; + t.run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp index a1b27056281..1b439aca7aa 100644 --- a/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_prevent_liq_supersaturation_tests.cpp @@ -3,8 +3,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" -#include "share/util/scream_setup_random_test.hpp" +#include "p3_test_data.hpp" #include "share/scream_types.hpp" #include "physics/share/physics_functions.hpp" @@ -15,9 +14,9 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { +struct UnitWrap::UnitTest::TestPreventLiqSupersaturation : public UnitWrap::UnitTest::Base { - static void run_property() + void run_property() //Conceptual tests for prevent_liq_supersaturation. Note many conceptual tests make sense to run on //random data, so are included in run_bfb rather than here. { @@ -88,32 +87,32 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { } //end run_property - static void run_bfb() + void run_bfb() { constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; - auto engine = setup_random_test(); + auto engine = Base::get_engine(); - PreventLiqSupersaturationData f90_data[max_pack_size]; + PreventLiqSupersaturationData baseline_data[max_pack_size]; // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { + // Alternatively, you can use the baseline_data construtors/initializer lists to hardcode data + for (auto& d : baseline_data) { d.randomize(engine); - d.dt = f90_data[0].dt; // Hold this fixed, this is not packed data + d.dt = baseline_data[0].dt; // Hold this fixed, this is not packed data // C++ impl uses constants for latent_heat values. Manually set here - // so F90 can match + // so BASELINE can match d.latent_heat_vapor = latvap; d.latent_heat_sublim = latvap+latice; } // Create copies of data for use by cxx and sync it to device. Needs to happen before - // fortran calls so that inout data is in original state + // reads so that inout data is in original state view_1d cxx_device("cxx_device", max_pack_size); const auto cxx_host = Kokkos::create_mirror_view(cxx_device); - std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); + std::copy(&baseline_data[0], &baseline_data[0] + max_pack_size, cxx_host.data()); Kokkos::deep_copy(cxx_device, cxx_host); // Save copy of inout vars to check that prevent_liq_supersaturation always makes them smaller @@ -123,9 +122,11 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { qr2qv_evap_tend_init[i] = cxx_host(i).qr2qv_evap_tend; } - // Get data from fortran - for (auto& d : f90_data) { - prevent_liq_supersaturation(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + baseline_data[i].read(Base::m_fid); + } } // Get data from cxx. Run prevent_liq_supersaturation from a kernel and copy results back to host @@ -158,13 +159,13 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { Kokkos::deep_copy(cxx_host, cxx_device); - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < max_pack_size; ++i) { // Verify BFB results - PreventLiqSupersaturationData& d_f90 = f90_data[i]; + PreventLiqSupersaturationData& d_baseline = baseline_data[i]; PreventLiqSupersaturationData& d_cxx = cxx_host[i]; - REQUIRE(d_f90.qi2qv_sublim_tend == d_cxx.qi2qv_sublim_tend); - REQUIRE(d_f90.qr2qv_evap_tend == d_cxx.qr2qv_evap_tend); + REQUIRE(d_baseline.qi2qv_sublim_tend == d_cxx.qi2qv_sublim_tend); + REQUIRE(d_baseline.qr2qv_evap_tend == d_cxx.qr2qv_evap_tend); //Verify tendencies are always >=0: REQUIRE(d_cxx.qi2qv_sublim_tend>=0); @@ -175,8 +176,12 @@ struct UnitWrap::UnitTest::TestPreventLiqSupersaturation { REQUIRE(d_cxx.qr2qv_evap_tend<=qr2qv_evap_tend_init[i]); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + cxx_host(s).write(Base::m_fid); + } + } } // run_bfb - }; } // namespace unit_test @@ -187,14 +192,18 @@ namespace { TEST_CASE("prevent_liq_supersaturation_property", "[p3]") { - using TestStruct = scream::p3::unit_test::UnitWrap::UnitTest::TestPreventLiqSupersaturation; - TestStruct::run_property(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestPreventLiqSupersaturation; + + T t; + t.run_property(); } TEST_CASE("prevent_liq_supersaturation_bfb", "[p3]") { - using TestStruct = scream::p3::unit_test::UnitWrap::UnitTest::TestPreventLiqSupersaturation; - TestStruct::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestPreventLiqSupersaturation; + + T t; + t.run_bfb(); } } // empty namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp index ae142ae77ac..935842cb6d9 100644 --- a/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -18,14 +18,14 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestRainImmersionFreezing { +struct UnitWrap::UnitTest::TestRainImmersionFreezing : public UnitWrap::UnitTest::Base { -static void run_phys() +void run_phys() { // TODO } -static void run_bfb() +void run_bfb() { // This is the threshold for whether the qc and qr cloud mixing ratios are // large enough to affect the warm-phase process rates qc2qr_accret_tend and nc_accret_tend. @@ -69,9 +69,11 @@ static void run_bfb() host_data.data()); Kokkos::deep_copy(device_data, host_data); - // Run the Fortran subroutine. - for (Int i = 0; i < max_pack_size; ++i) { - rain_immersion_freezing(rain_imm_freezing_data[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + rain_imm_freezing_data[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -106,12 +108,17 @@ static void run_bfb() Kokkos::deep_copy(host_data, device_data); // Validate results. - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(rain_imm_freezing_data[s].qr2qi_immers_freeze_tend == host_data[s].qr2qi_immers_freeze_tend); REQUIRE(rain_imm_freezing_data[s].nr2ni_immers_freeze_tend == host_data[s].nr2ni_immers_freeze_tend); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + host_data(s).write(Base::m_fid); + } + } } }; @@ -124,12 +131,11 @@ namespace { TEST_CASE("p3_rain_immersion_freezing", "[p3_functions]") { - using TRIF = scream::p3::unit_test::UnitWrap::UnitTest::TestRainImmersionFreezing; - - TRIF::run_phys(); - TRIF::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestRainImmersionFreezing; - scream::p3::P3GlobalForFortran::deinit(); + T t; + t.run_phys(); + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp index 443b3801404..cd4d1955aed 100644 --- a/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp @@ -4,9 +4,8 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" -#include "p3_f90.hpp" -#include "share/util/scream_setup_random_test.hpp" +#include "p3_test_data.hpp" +#include "p3_data.hpp" #include "p3_unit_tests_common.hpp" @@ -20,25 +19,25 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestRainSed { +struct UnitWrap::UnitTest::TestRainSed : public UnitWrap::UnitTest::Base { -static void run_phys_rain_vel() +void run_phys_rain_vel() { // TODO } -static void run_phys_rain_sed() +void run_phys_rain_sed() { // TODO } -static void run_phys() +void run_phys() { run_phys_rain_vel(); run_phys_rain_sed(); } -static void run_bfb_rain_vel() +void run_bfb_rain_vel() { // Read in tables view_2d_table vn_table_vals; view_2d_table vm_table_vals; view_2d_table revap_table_vals; @@ -46,7 +45,7 @@ static void run_bfb_rain_vel() Functions::init_kokkos_tables(vn_table_vals, vm_table_vals, revap_table_vals, mu_r_table_vals, dnu); // Load some lookup inputs, need at least one per pack value - ComputeRainFallVelocityData crfv_fortran[max_pack_size] = { + ComputeRainFallVelocityData crfv_baseline[max_pack_size] = { // qr_incld, rhofacr, nr_incld {1.1030E-04, 1.3221E+00, 6.2964E+05}, {2.1437E-13, 1.0918E+00, 6.5337E+07}, @@ -70,16 +69,18 @@ static void run_bfb_rain_vel() }; - // Sync to device, needs to happen before fortran calls so that + // Sync to device, needs to happen before reads so that // inout data is in original state view_1d crfv_device("crfv", max_pack_size); const auto crfv_host = Kokkos::create_mirror_view(crfv_device); - std::copy(&crfv_fortran[0], &crfv_fortran[0] + max_pack_size, crfv_host.data()); + std::copy(&crfv_baseline[0], &crfv_baseline[0] + max_pack_size, crfv_host.data()); Kokkos::deep_copy(crfv_device, crfv_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - compute_rain_fall_velocity(crfv_fortran[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + crfv_baseline[i].read(Base::m_fid); + } } // Calc bulk rime from a kernel and copy results back to host @@ -112,21 +113,27 @@ static void run_bfb_rain_vel() // Sync back to host Kokkos::deep_copy(crfv_host, crfv_device); - // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + // Validate results for (Int s = 0; s < max_pack_size; ++s) { - REQUIRE(crfv_fortran[s].nr_incld == crfv_host(s).nr_incld); - REQUIRE(crfv_fortran[s].mu_r == crfv_host(s).mu_r); - REQUIRE(crfv_fortran[s].lamr == crfv_host(s).lamr); - REQUIRE(crfv_fortran[s].V_qr == crfv_host(s).V_qr); - REQUIRE(crfv_fortran[s].V_nr == crfv_host(s).V_nr); + REQUIRE(crfv_baseline[s].nr_incld == crfv_host(s).nr_incld); + REQUIRE(crfv_baseline[s].mu_r == crfv_host(s).mu_r); + REQUIRE(crfv_baseline[s].lamr == crfv_host(s).lamr); + REQUIRE(crfv_baseline[s].V_qr == crfv_host(s).V_qr); + REQUIRE(crfv_baseline[s].V_nr == crfv_host(s).V_nr); + } + } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + crfv_host(s).write(Base::m_fid); } } } -static void run_bfb_rain_sed() +void run_bfb_rain_sed() { - auto engine = setup_random_test(); + // With stored baselines, we must use a fixed seed! + auto engine = Base::get_engine(); // F90 is quite slow on weaver, so we decrease dt to reduce // the number of steps in rain_sed. @@ -136,7 +143,7 @@ static void run_bfb_rain_sed() constexpr Scalar dt = 1.800E+03; #endif - RainSedData rsds_fortran[] = { + RainSedData rsds_baseline[] = { // kts, kte, ktop, kbot, kdir, dt, inv_dt, precip_liq_surf RainSedData(1, 72, 27, 72, -1, dt, 1/dt, 0.0), RainSedData(1, 72, 72, 27, 1, dt, 1/dt, 1.0), @@ -144,25 +151,27 @@ static void run_bfb_rain_sed() RainSedData(1, 72, 27, 27, 1, dt, 1/dt, 2.0), }; - static constexpr Int num_runs = sizeof(rsds_fortran) / sizeof(RainSedData); + static constexpr Int num_runs = sizeof(rsds_baseline) / sizeof(RainSedData); // Set up random input data - for (auto& d : rsds_fortran) { + for (auto& d : rsds_baseline) { d.randomize(engine, { {d.qr_incld, {C::QSMALL/2, C::QSMALL*2}} }); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state RainSedData rsds_cxx[num_runs] = { - RainSedData(rsds_fortran[0]), - RainSedData(rsds_fortran[1]), - RainSedData(rsds_fortran[2]), - RainSedData(rsds_fortran[3]), + RainSedData(rsds_baseline[0]), + RainSedData(rsds_baseline[1]), + RainSedData(rsds_baseline[2]), + RainSedData(rsds_baseline[3]), }; - // Get data from fortran - for (auto& d : rsds_fortran) { - rain_sedimentation(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : rsds_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx @@ -173,35 +182,40 @@ static void run_bfb_rain_sed() #if defined(SCREAM_FORCE_RUN_DIFF) inv_dt *= 2; #endif - rain_sedimentation_f(d.kts, d.kte, d.ktop, d.kbot, d.kdir, + rain_sedimentation_host(d.kts, d.kte, d.ktop, d.kbot, d.kdir, d.qr_incld, d.rho, d.inv_rho, d.rhofacr, d.cld_frac_r, d.inv_dz, d.dt, inv_dt, d.qr, d.nr, d.nr_incld, d.mu_r, d.lamr, &d.precip_liq_surf, d.precip_liq_flux, d.qr_tend, d.nr_tend); } - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { // Due to pack issues, we must restrict checks to the active k space - Int start = std::min(rsds_fortran[i].kbot, rsds_fortran[i].ktop) - 1; // 0-based indx - Int end = std::max(rsds_fortran[i].kbot, rsds_fortran[i].ktop); // 0-based indx + Int start = std::min(rsds_baseline[i].kbot, rsds_baseline[i].ktop) - 1; // 0-based indx + Int end = std::max(rsds_baseline[i].kbot, rsds_baseline[i].ktop); // 0-based indx for (Int k = start; k < end; ++k) { - REQUIRE(rsds_fortran[i].qr[k] == rsds_cxx[i].qr[k]); - REQUIRE(rsds_fortran[i].nr[k] == rsds_cxx[i].nr[k]); - REQUIRE(rsds_fortran[i].nr_incld[k] == rsds_cxx[i].nr_incld[k]); - REQUIRE(rsds_fortran[i].mu_r[k] == rsds_cxx[i].mu_r[k]); - REQUIRE(rsds_fortran[i].lamr[k] == rsds_cxx[i].lamr[k]); - REQUIRE(rsds_fortran[i].precip_liq_flux[k] == rsds_cxx[i].precip_liq_flux[k]); - REQUIRE(rsds_fortran[i].qr_tend[k] == rsds_cxx[i].qr_tend[k]); - REQUIRE(rsds_fortran[i].nr_tend[k] == rsds_cxx[i].nr_tend[k]); + REQUIRE(rsds_baseline[i].qr[k] == rsds_cxx[i].qr[k]); + REQUIRE(rsds_baseline[i].nr[k] == rsds_cxx[i].nr[k]); + REQUIRE(rsds_baseline[i].nr_incld[k] == rsds_cxx[i].nr_incld[k]); + REQUIRE(rsds_baseline[i].mu_r[k] == rsds_cxx[i].mu_r[k]); + REQUIRE(rsds_baseline[i].lamr[k] == rsds_cxx[i].lamr[k]); + REQUIRE(rsds_baseline[i].precip_liq_flux[k] == rsds_cxx[i].precip_liq_flux[k]); + REQUIRE(rsds_baseline[i].qr_tend[k] == rsds_cxx[i].qr_tend[k]); + REQUIRE(rsds_baseline[i].nr_tend[k] == rsds_cxx[i].nr_tend[k]); } - REQUIRE(rsds_fortran[i].precip_liq_flux[end] == rsds_cxx[i].precip_liq_flux[end]); - REQUIRE(rsds_fortran[i].precip_liq_surf == rsds_cxx[i].precip_liq_surf); + REQUIRE(rsds_baseline[i].precip_liq_flux[end] == rsds_cxx[i].precip_liq_flux[end]); + REQUIRE(rsds_baseline[i].precip_liq_surf == rsds_cxx[i].precip_liq_surf); + } + } + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + rsds_cxx[i].write(Base::m_fid); } } } -static void run_bfb() +void run_bfb() { run_bfb_rain_vel(); run_bfb_rain_sed(); @@ -217,14 +231,11 @@ namespace { TEST_CASE("p3_rain_sed", "[p3_functions]") { - using TRS = scream::p3::unit_test::UnitWrap::UnitTest::TestRainSed; - - scream::p3::p3_init(); // need fortran table data - - TRS::run_phys(); - TRS::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestRainSed; - scream::p3::P3GlobalForFortran::deinit(); + T t; + t.run_phys(); + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp index a8c0dbc07dd..26440b6680b 100644 --- a/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp @@ -4,7 +4,7 @@ #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -22,9 +22,9 @@ namespace unit_test { * Unit-tests for p3 ice collection functions. */ template -struct UnitWrap::UnitTest::TestRainSelfCollection { +struct UnitWrap::UnitTest::TestRainSelfCollection : public UnitWrap::UnitTest::Base { - static void run_rain_self_collection_bfb_tests(){ + void run_rain_self_collection_bfb_tests() { RainSelfCollectionData dc[max_pack_size] = { // rho, qr_incld, nr_incld, nr_selfcollect_tend @@ -56,9 +56,11 @@ struct UnitWrap::UnitTest::TestRainSelfCollection { std::copy(&dc[0], &dc[0] + max_pack_size, dc_host.data()); Kokkos::deep_copy(dc_device, dc_host); - //Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - rain_self_collection(dc[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + dc[i].read(Base::m_fid); + } } //Run function from a kernal and copy results back to the host @@ -91,7 +93,7 @@ struct UnitWrap::UnitTest::TestRainSelfCollection { Kokkos::deep_copy(dc_host, dc_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(dc[s].rho == dc_host(s).rho); REQUIRE(dc[s].qr_incld == dc_host(s).qr_incld); @@ -99,9 +101,14 @@ struct UnitWrap::UnitTest::TestRainSelfCollection { REQUIRE(dc[s].nr_selfcollect_tend == dc_host(s).nr_selfcollect_tend); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + dc_host(s).write(Base::m_fid); + } + } } - static void run_bfb(){ + void run_bfb() { run_rain_self_collection_bfb_tests(); } @@ -114,7 +121,10 @@ struct UnitWrap::UnitTest::TestRainSelfCollection { namespace { TEST_CASE("p3_rain_self_collection_test", "[p3_rain_self_collection_test"){ - scream::p3::unit_test::UnitWrap::UnitTest::TestRainSelfCollection::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestRainSelfCollection; + + T t; + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp b/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp index 5bc12b2464d..ff069032013 100644 --- a/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_run_and_cmp.cpp @@ -3,7 +3,7 @@ #include "share/util/scream_utils.hpp" #include "p3_main_wrap.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_ic_cases.hpp" #include "ekat/util/ekat_file_utils.hpp" @@ -16,6 +16,7 @@ namespace { using namespace scream; using namespace scream::p3; +using P3F = Functions; /* * p3_run_and_cmp can be run in 2 modes. First, generate_baseline @@ -35,10 +36,10 @@ using namespace scream::p3; * large discrepancies. */ Int compare (const double& tol, - const FortranData::Ptr& ref, const FortranData::Ptr& d) { + const P3Data::Ptr& ref, const P3Data::Ptr& d) { Int nerr = 0; - FortranDataIterator refi(ref), di(d); + P3DataIterator refi(ref), di(d); EKAT_ASSERT(refi.nfield() == di.nfield()); for (Int i = 0, n = refi.nfield(); i < n; ++i) { const auto& fr = refi.getfield(i); @@ -75,7 +76,7 @@ struct Baseline { } } - Int generate_baseline (const std::string& filename, bool use_fortran) { + Int generate_baseline (const std::string& filename) { auto fid = ekat::FILEPtr(fopen(filename.c_str(), "w")); EKAT_REQUIRE_MSG( fid, "generate_baseline can't write " << filename); Int nerr = 0; @@ -87,22 +88,19 @@ struct Baseline { for (Int r = -1; r < ps.repeat; ++r) { const auto d = ic::Factory::create(ps.ic, ps.ncol, ps.nlev); set_params(ps, *d); - p3_init(); + P3F::p3_init(); if (ps.repeat > 0 && r == -1) { std::cout << "Running P3 with ni=" << d->ncol << ", nk=" << d->nlev << ", dt=" << d->dt << ", ts=" << d->it << ", predict_nc=" << d->do_predict_nc - << ", prescribed_CCN=" << d->do_prescribed_CCN; - - if (!use_fortran) { - std::cout << ", small_packn=" << SCREAM_SMALL_PACK_SIZE; - } - std::cout << std::endl; + << ", prescribed_CCN=" << d->do_prescribed_CCN + << ", small_packn=" << SCREAM_SMALL_PACK_SIZE + << std::endl; } for (int it=0; it 0) { // do not count the "cold" run total_duration_microsec += current_microsec; @@ -123,29 +121,43 @@ struct Baseline { return nerr; } - Int run_and_cmp (const std::string& filename, const double& tol, bool use_fortran) { - auto fid = ekat::FILEPtr(fopen(filename.c_str(), "r")); - EKAT_REQUIRE_MSG( fid, "generate_baseline can't read " << filename); + Int run_and_cmp (const std::string& filename, const double& tol, bool no_baseline) { + ekat::FILEPtr fid; + if (!no_baseline) { + fid = ekat::FILEPtr(fopen(filename.c_str(), "r")); + EKAT_REQUIRE_MSG( fid, "generate_baseline can't read " << filename); + } Int nerr = 0, ne; int case_num = 0; for (auto ps : params_) { case_num++; - // Read the reference impl's data from the baseline file. - const auto d_ref = ic::Factory::create(ps.ic, ps.ncol, ps.nlev); - set_params(ps, *d_ref); - // Now run a sequence of other impls. This includes the reference - // implementation b/c it's likely we'll want to change it as we go. - { + if (no_baseline) { const auto d = ic::Factory::create(ps.ic, ps.ncol, ps.nlev); set_params(ps, *d); - p3_init(); + P3F::p3_init(); for (int it=0; it params_; - static void write (const ekat::FILEPtr& fid, const FortranData::Ptr& d) { - FortranDataIterator fdi(d); + static void write (const ekat::FILEPtr& fid, const P3Data::Ptr& d) { + P3DataIterator fdi(d); for (Int i = 0, n = fdi.nfield(); i < n; ++i) { const auto& f = fdi.getfield(i); ekat::write(&f.dim, 1, fid); @@ -182,8 +194,8 @@ struct Baseline { } } - static void read (const ekat::FILEPtr& fid, const FortranData::Ptr& d) { - FortranDataIterator fdi(d); + static void read (const ekat::FILEPtr& fid, const P3Data::Ptr& d) { + P3DataIterator fdi(d); for (Int i = 0, n = fdi.nfield(); i < n; ++i) { const auto& f = fdi.getfield(i); int dim, ds[3]; @@ -220,22 +232,24 @@ int main (int argc, char** argv) { if (argc == 1) { std::cout << - argv[0] << " [options] baseline-filename\n" + argv[0] << " [options] \n" "Options:\n" " -g Generate baseline file. Default False.\n" - " -f Use fortran impls instead of c++. Default False.\n" + " -c Compare baseline file. Default False.\n" + " -n Run without baseline actions. Default True.\n" + " -b Path to directory containing baselines.\n" " -t Tolerance for relative error. Default 0.\n" " -s Number of timesteps. Default=6.\n" " -dt Length of timestep. Default=300.\n" " -i Number of columns. Default=3.\n" " -k Number of vertical levels. Default=72.\n" " -r Number of repetitions, implies timing run (generate + no I/O). Default=0.\n" - " -p yes|no|both. Default=both.\n" - " -c yes|no|both. Default=both.\n"; + " --predict-nc yes|no|both. Default=both.\n" + " --prescribed-ccn yes|no|both. Default=both.\n"; return 1; } - bool generate = false, use_fortran = false; + bool generate = false, no_baseline = true; scream::Real tol = SCREAM_BFB_TESTING ? 0 : std::numeric_limits::infinity(); Int timesteps = 6; Int dt = 300; @@ -247,8 +261,8 @@ int main (int argc, char** argv) { std::string prescribed_ccn = "both"; std::string baseline_fn; for (int i = 1; i < argc-1; ++i) { - if (ekat::argv_matches(argv[i], "-g", "--generate")) generate = true; - if (ekat::argv_matches(argv[i], "-f", "--fortran")) use_fortran = true; + if (ekat::argv_matches(argv[i], "-g", "--generate")) { generate = true; no_baseline = false; } + if (ekat::argv_matches(argv[i], "-c", "--compare")) { no_baseline = false; } if (ekat::argv_matches(argv[i], "-t", "--tol")) { expect_another_arg(i, argc); ++i; @@ -279,10 +293,9 @@ int main (int argc, char** argv) { ++i; nlev = std::atoi(argv[i]); } - if (std::string(argv[i])=="--ekat-kokkos-device") { - expect_another_arg(i, argc); - ++i; - device = argv[i]; + if (std::string(argv[i])=="--kokkos-device-id=") { + auto tokens = ekat::split(argv[i],"="); + device = tokens[1]; } if (ekat::argv_matches(argv[i], "-r", "--repeat")) { expect_another_arg(i, argc); @@ -292,14 +305,14 @@ int main (int argc, char** argv) { generate = true; } } - if (ekat::argv_matches(argv[i], "-p", "--predict-nc")) { + if (ekat::argv_matches(argv[i], "-pn", "--predict-nc")) { expect_another_arg(i, argc); ++i; predict_nc = std::string(argv[i]); EKAT_REQUIRE_MSG(predict_nc == "yes" || predict_nc == "no" || predict_nc == "both", "Predict option value must be one of yes|no|both"); } - if (ekat::argv_matches(argv[i], "-c", "--prescribed-ccn")) { + if (ekat::argv_matches(argv[i], "-pc", "--prescribed-ccn")) { expect_another_arg(i, argc); ++i; prescribed_ccn = std::string(argv[i]); @@ -308,45 +321,26 @@ int main (int argc, char** argv) { } } - // Decorate baseline name with precision. - baseline_fn += std::to_string(sizeof(scream::Real)); + // Compute full baseline file name with precision. + baseline_fn += "/p3_run_and_cmp.baseline" + std::to_string(sizeof(scream::Real)); - std::vector args; - for (int i=0; i" was specified, add kokkos - // initialization flag to argv - // Create it outside the if, so its c_str pointer survives - std::string dev_arg; - if (device!="") { - auto is_int = [] (const std::string& s)->bool { - std::istringstream is(s); - int d; - is >> d; - return !is.fail() && is.eof(); - }; - - EKAT_REQUIRE_MSG (is_int(device), "Error! Invalid device specification.\n"); - - if (std::stoi(device) != -1) { - dev_arg = "--kokkos-device-id=" + device; - args.push_back(const_cast(dev_arg.c_str())); - } - } - - scream::initialize_scream_session(args.size(), args.data()); { + scream::initialize_scream_session(argc, argv); + { Baseline bln(timesteps, static_cast(dt), ncol, nlev, repeat, predict_nc, prescribed_ccn); if (generate) { std::cout << "Generating to " << baseline_fn << "\n"; - nerr += bln.generate_baseline(baseline_fn, use_fortran); - } else { + nerr += bln.generate_baseline(baseline_fn); + } else if (no_baseline) { + printf("Running with no baseline actions\n"); + nerr += bln.run_and_cmp(baseline_fn, tol, no_baseline); + } + else { printf("Comparing with %s at tol %1.1e\n", baseline_fn.c_str(), tol); - nerr += bln.run_and_cmp(baseline_fn, tol, use_fortran); + nerr += bln.run_and_cmp(baseline_fn, tol, no_baseline); } P3GlobalForFortran::deinit(); - } scream::finalize_scream_session(); + } + scream::finalize_scream_session(); return nerr != 0 ? 1 : 0; } diff --git a/components/eamxx/src/physics/p3/tests/p3_subgrid_variance_scaling_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_subgrid_variance_scaling_unit_tests.cpp index 9a728b6c57e..c58ac68b4a6 100644 --- a/components/eamxx/src/physics/p3/tests/p3_subgrid_variance_scaling_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_subgrid_variance_scaling_unit_tests.cpp @@ -3,7 +3,7 @@ #include "share/scream_types.hpp" #include "ekat/ekat_pack.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "p3_unit_tests_common.hpp" @@ -19,11 +19,11 @@ namespace p3 { namespace unit_test { template -struct UnitWrap::UnitTest::TestP3SubgridVarianceScaling +struct UnitWrap::UnitTest::TestP3SubgridVarianceScaling : public UnitWrap::UnitTest::Base { //----------------------------------------------------------------- - static void run_bfb_tests(){ + void run_bfb_tests() { //test that C++ and F90 implementations are BFB //Set of relvar values to loop over @@ -36,9 +36,7 @@ struct UnitWrap::UnitTest::TestP3SubgridVarianceScaling //Set of exponents to loop over Scalar expons[3] = {1.0,2.47,0.1}; - //initialize struct required for F90 call - SubgridVarianceScalingData f_data; - Scalar f_scaling; + Scalar baseline_scaling; //Make C++ output available on host and device view_1d scaling_device("c scaling",1); @@ -47,11 +45,11 @@ struct UnitWrap::UnitTest::TestP3SubgridVarianceScaling for (Int i = 0; i < 3; ++i) { // loop over exponents for (Int j = 0; j < 16; ++j) { // loop over relvars - // Get F90 solution + // Get baseline solution // ---------------------------------- - f_data.relvar=relvars[j]; - f_data.expon =expons[i]; - f_scaling = subgrid_variance_scaling(f_data); + if (this->m_baseline_action == COMPARE) { + ekat::read(&baseline_scaling, 1, Base::m_fid); + } // Get C++ solution // ---------------------------------- @@ -72,8 +70,11 @@ struct UnitWrap::UnitTest::TestP3SubgridVarianceScaling Kokkos::deep_copy(scaling_host, scaling_device); // Validate results - if (SCREAM_BFB_TESTING) { - REQUIRE(f_scaling == scaling_host(0) ); + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { + REQUIRE(baseline_scaling == scaling_host(0) ); + } + else if (this->m_baseline_action == GENERATE) { + ekat::write(&scaling_host(0), 1, Base::m_fid); } } //end loop over relvar[j] } //end loop over expons[i] @@ -81,7 +82,7 @@ struct UnitWrap::UnitTest::TestP3SubgridVarianceScaling //----------------------------------------------------------------- KOKKOS_FUNCTION static void subgrid_variance_scaling_linearity_test(const Scalar& relvar, - int& errors){ + int& errors) { //If expon=1, subgrid_variance_scaling should be 1 Scalar tol = C::macheps * 1e3; //1e3 is scale factor to make pass, essentially an estimate of numerical error @@ -97,7 +98,7 @@ struct UnitWrap::UnitTest::TestP3SubgridVarianceScaling } //----------------------------------------------------------------- - KOKKOS_FUNCTION static void subgrid_variance_scaling_relvar1_test(int& errors){ + KOKKOS_FUNCTION static void subgrid_variance_scaling_relvar1_test(int& errors) { //If relvar=1, subgrid_variance_scaling should be factorial(expon) Scalar tol = C::macheps * 1e3; //1e3 is scale factor to make pass, essentially an estimate of numerical error @@ -116,7 +117,7 @@ struct UnitWrap::UnitTest::TestP3SubgridVarianceScaling } //----------------------------------------------------------------- - KOKKOS_FUNCTION static void subgrid_variance_scaling_relvar3_test(int& errors){ + KOKKOS_FUNCTION static void subgrid_variance_scaling_relvar3_test(int& errors) { //If expon=3, subgrid variance scaling should be relvar^3+3*relvar^2+2*relvar/relvar^3 Scalar tol = C::macheps * 100; //100 is a fudge factor to make sure tests pass. 10 was too small for gnu on CPU. @@ -151,7 +152,7 @@ struct UnitWrap::UnitTest::TestP3SubgridVarianceScaling } //end relvar3_test //----------------------------------------------------------------- - static void run_property_tests(){ + void run_property_tests() { /*This function executes all the SGS variance scaling tests by looping *over a bunch of test and summing their return statuses. *If that sum is zero, no errors have occurred. Otherwise you have errors. @@ -189,12 +190,14 @@ struct UnitWrap::UnitTest::TestP3SubgridVarianceScaling } // namespace p3 } // namespace scream -namespace{ +namespace { TEST_CASE("p3_subgrid_variance_scaling_test", "[p3_subgrid_variance_scaling_test]"){ - scream::p3::unit_test::UnitWrap::UnitTest::TestP3SubgridVarianceScaling::run_bfb_tests(); - scream::p3::unit_test::UnitWrap::UnitTest::TestP3SubgridVarianceScaling::run_property_tests(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestP3SubgridVarianceScaling; + + T t; + t.run_bfb_tests(); + t.run_property_tests(); } } // namespace - diff --git a/components/eamxx/src/physics/p3/tests/p3_table3_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_table3_unit_tests.cpp index c99d156004e..57197fea6c8 100644 --- a/components/eamxx/src/physics/p3/tests/p3_table3_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_table3_unit_tests.cpp @@ -3,8 +3,8 @@ #include "p3_unit_tests_common.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" -#include "p3_f90.hpp" +#include "p3_test_data.hpp" +#include "p3_data.hpp" #include "share/scream_types.hpp" #include "ekat/ekat_pack.hpp" @@ -44,7 +44,7 @@ namespace unit_test { // refinement, where the mesh is a 1D mesh transecting the table domain. template -struct UnitWrap::UnitTest::TestTable3 { +struct UnitWrap::UnitTest::TestTable3 : public UnitWrap::UnitTest::Base { KOKKOS_FUNCTION static Scalar calc_lamr (const Scalar& mu_r, const Scalar& alpha) { // Parameters for lower and upper bounds, derived above, multiplied by @@ -68,7 +68,7 @@ struct UnitWrap::UnitTest::TestTable3 { return Functions::apply_table(table, t3); } - static void run () { + void run () { // This test doesn't use mu_r_table_vals, as that is not a table3 type. It // doesn't matter whether we use vm_table_vals or vn_table_vals, as the table values // don't matter in what we are testing; we are testing interpolation @@ -170,9 +170,10 @@ namespace { TEST_CASE("p3_tables", "[p3_functions]") { - scream::p3::p3_init(); // need fortran table data + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestTable3; - scream::p3::unit_test::UnitWrap::UnitTest::TestTable3::run(); + T t; + t.run(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_tests.cpp index d1a64f48520..858a231094d 100644 --- a/components/eamxx/src/physics/p3/tests/p3_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_tests.cpp @@ -1,19 +1,19 @@ #include "catch2/catch.hpp" -#include "p3_f90.hpp" +#include "p3_data.hpp" #include "p3_main_wrap.hpp" #include "p3_ic_cases.hpp" namespace { -TEST_CASE("FortranData", "p3") { - int val = scream::p3::test_FortranData(); +TEST_CASE("P3Data", "p3") { + int val = scream::p3::test_P3Data(); REQUIRE(val == 0); } -TEST_CASE("FortranDataIterator", "p3") { +TEST_CASE("P3DataIterator", "p3") { using scream::p3::ic::Factory; const auto d = Factory::create(Factory::mixed); - scream::p3::FortranDataIterator fdi(d); + scream::p3::P3DataIterator fdi(d); REQUIRE(fdi.nfield() == 35); const auto& f = fdi.getfield(0); REQUIRE(f.dim == 2); @@ -29,13 +29,8 @@ TEST_CASE("p3_init", "p3") { REQUIRE(nerr == 0); } -TEST_CASE("p3_ic_f", "p3") { - int nerr = scream::p3::test_p3_ic(true); - REQUIRE(nerr == 0); -} - TEST_CASE("p3_ic_c", "p3") { - int nerr = scream::p3::test_p3_ic(false); + int nerr = scream::p3::test_p3_ic(); REQUIRE(nerr == 0); } diff --git a/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp index 3b994cfdad5..91e3db2f297 100644 --- a/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_unit_tests.cpp @@ -5,7 +5,7 @@ #include "ekat/kokkos/ekat_kokkos_utils.hpp" #include "ekat/util/ekat_arch.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "p3_unit_tests_common.hpp" @@ -21,10 +21,10 @@ namespace unit_test { * Unit-tests for p3_functions. */ template -struct UnitWrap::UnitTest::TestP3Conservation +struct UnitWrap::UnitTest::TestP3Conservation : public UnitWrap::UnitTest::Base { - static void cloud_water_conservation_tests_device() { + void cloud_water_conservation_tests_device() { using KTH = KokkosTypes; @@ -78,7 +78,7 @@ struct UnitWrap::UnitTest::TestP3Conservation REQUIRE(cwdc_host[0].qc2qr_autoconv_tend * cwdc[0].dt <= cwdc_host[0].qc); } - static void rain_water_conservation_tests_device() { + void rain_water_conservation_tests_device() { using KTH = KokkosTypes; RainWaterConservationData rwdc[1] = {{sp(1e-5), 0.0, 0.0, 0.0, 0.0, sp(1.1), sp(1e-4), 0.0, 0.0 }}; @@ -131,7 +131,7 @@ struct UnitWrap::UnitTest::TestP3Conservation REQUIRE( rwdc_host(0).qr2qv_evap_tend * rwdc_host(0).dt <= rwdc_host(0).qr); } - static void ice_water_conservation_tests_device(){ + void ice_water_conservation_tests_device() { using KTH = KokkosTypes; IceWaterConservationData iwdc[1] = {{sp(1e-5), 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, sp(1.1), sp(1e-4), 0.0}}; @@ -173,7 +173,7 @@ struct UnitWrap::UnitTest::TestP3Conservation } - static void run() + void run() { cloud_water_conservation_tests_device(); @@ -182,7 +182,7 @@ struct UnitWrap::UnitTest::TestP3Conservation ice_water_conservation_tests_device(); } - static void cloud_water_conservation_unit_bfb_tests(){ + void cloud_water_conservation_unit_bfb_tests() { using KTH = KokkosTypes; @@ -222,9 +222,11 @@ struct UnitWrap::UnitTest::TestP3Conservation std::copy(&cwdc[0], &cwdc[0] + max_pack_size, cwdc_host.data()); Kokkos::deep_copy(cwdc_device, cwdc_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - cloud_water_conservation(cwdc[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + cwdc[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -263,7 +265,7 @@ struct UnitWrap::UnitTest::TestP3Conservation Kokkos::deep_copy(cwdc_host, cwdc_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(cwdc[s].qc == cwdc_host(s).qc); REQUIRE(cwdc[s].qc2qr_autoconv_tend == cwdc_host(s).qc2qr_autoconv_tend); @@ -275,9 +277,14 @@ struct UnitWrap::UnitTest::TestP3Conservation REQUIRE(cwdc[s].qv2qi_vapdep_tend == cwdc_host(s).qv2qi_vapdep_tend); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + cwdc_host(s).write(Base::m_fid); + } + } } - static void ice_water_conservation_unit_bfb_tests() + void ice_water_conservation_unit_bfb_tests() { using KTH = KokkosTypes; @@ -312,9 +319,11 @@ struct UnitWrap::UnitTest::TestP3Conservation std::copy(&iwdc[0], &iwdc[0] + max_pack_size, iwdc_host.data()); Kokkos::deep_copy(iwdc_device, iwdc_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - ice_water_conservation(iwdc[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + iwdc[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -356,7 +365,7 @@ struct UnitWrap::UnitTest::TestP3Conservation Kokkos::deep_copy(iwdc_host, iwdc_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(iwdc[s].qi == iwdc_host(s).qi); REQUIRE(iwdc[s].qv2qi_vapdep_tend == iwdc_host(s).qv2qi_vapdep_tend ); @@ -370,9 +379,14 @@ struct UnitWrap::UnitTest::TestP3Conservation REQUIRE(iwdc[s].qi2qr_melt_tend == iwdc_host(s).qi2qr_melt_tend); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + iwdc_host(s).write(Base::m_fid); + } + } } - static void rain_water_conservation_unit_bfb_tests(){ + void rain_water_conservation_unit_bfb_tests() { using KTH = KokkosTypes; @@ -407,9 +421,11 @@ struct UnitWrap::UnitTest::TestP3Conservation std::copy(&rwdc[0], &rwdc[0] + max_pack_size, rwdc_host.data()); Kokkos::deep_copy(rwdc_device, rwdc_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - rain_water_conservation(rwdc[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + rwdc[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -448,7 +464,7 @@ struct UnitWrap::UnitTest::TestP3Conservation Kokkos::deep_copy(rwdc_host, rwdc_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(rwdc[s].qr == rwdc_host(s).qr); REQUIRE(rwdc[s].qc2qr_autoconv_tend == rwdc_host(s).qc2qr_autoconv_tend); @@ -460,9 +476,14 @@ struct UnitWrap::UnitTest::TestP3Conservation REQUIRE(rwdc[s].qr2qi_immers_freeze_tend == rwdc_host(s).qr2qi_immers_freeze_tend); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + rwdc_host(s).write(Base::m_fid); + } + } } - static void run_bfb() { + void run_bfb() { cloud_water_conservation_unit_bfb_tests(); rain_water_conservation_unit_bfb_tests(); @@ -473,9 +494,9 @@ struct UnitWrap::UnitTest::TestP3Conservation }; template -struct UnitWrap::UnitTest::TestP3UpdatePrognosticIce +struct UnitWrap::UnitTest::TestP3UpdatePrognosticIce : public UnitWrap::UnitTest::Base { - static void update_prognostic_ice_unit_bfb_tests() { + void update_prognostic_ice_unit_bfb_tests() { constexpr Scalar nmltratio = C::nmltratio; constexpr Scalar dt = 1.8000E+03; @@ -483,7 +504,7 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticIce constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; - //fortran generated data is input to the following + //baseline generated data is input to the following P3UpdatePrognosticIceData pupidc[max_pack_size] = { {4.9078E-19, 1.5312E-09, 4.4387E-09, 3.7961E+06, 1.7737E-04, 0.0000E+00, 3.8085E-08, 5.1281E+04, 1.9251E-15, @@ -591,9 +612,11 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticIce std::copy(&pupidc[0], &pupidc[0] + max_pack_size, pupidc_host.data()); Kokkos::deep_copy(pupidc_device, pupidc_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - update_prognostic_ice(pupidc[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + pupidc[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -679,7 +702,7 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticIce Kokkos::deep_copy(pupidc_host, pupidc_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(pupidc[s].th_atm == pupidc_host(s).th_atm); REQUIRE(pupidc[s].qc == pupidc_host(s).qc); @@ -693,22 +716,27 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticIce REQUIRE(pupidc[s].bm == pupidc_host(s).bm ); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + pupidc_host(s).write(Base::m_fid); + } + } } - static void run_bfb(){ + void run_bfb() { update_prognostic_ice_unit_bfb_tests(); } }; //TestP3UpdatePrognosticIce template -struct UnitWrap::UnitTest::TestGetTimeSpacePhysVariables +struct UnitWrap::UnitTest::TestGetTimeSpacePhysVariables : public UnitWrap::UnitTest::Base { - static void get_time_space_phys_variables_unit_bfb_tests(){ + void get_time_space_phys_variables_unit_bfb_tests() { constexpr Scalar latvap = C::LatVap; constexpr Scalar latice = C::LatIce; - //fortran generated data is input to the following + //baseline generated data is input to the following GetTimeSpacePhysVarsData gtspvd[max_pack_size] = { // T_atm, pres, rho, latent_heat_vapor, latent_heat_sublim, qv_sat_l, qv_sat_i {2.9792E+02, 9.8711E+04, 1.1532E+00, latvap, latvap+latice, 2.0321E-02, 2.0321E-02}, @@ -737,9 +765,11 @@ struct UnitWrap::UnitTest::TestGetTimeSpacePhysVariables std::copy(>spvd[0], >spvd[0] + max_pack_size, gtspvd_host.data()); Kokkos::deep_copy(gtspvd_device, gtspvd_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - get_time_space_phys_variables(gtspvd[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + gtspvd[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -794,7 +824,7 @@ struct UnitWrap::UnitTest::TestGetTimeSpacePhysVariables Kokkos::deep_copy(gtspvd_host, gtspvd_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(gtspvd[s].mu == gtspvd_host(s).mu); REQUIRE(gtspvd[s].dv == gtspvd_host(s).dv); @@ -807,20 +837,25 @@ struct UnitWrap::UnitTest::TestGetTimeSpacePhysVariables REQUIRE(gtspvd[s].eii == gtspvd_host(s).eii); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + gtspvd_host(s).write(Base::m_fid); + } + } } - static void run_bfb(){ + void run_bfb() { get_time_space_phys_variables_unit_bfb_tests(); } }; //TestGetTimeSpacePhysVariables template -struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq +struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq : public UnitWrap::UnitTest::Base { - static void update_prognostic_liquid_unit_bfb_tests(){ + void update_prognostic_liquid_unit_bfb_tests() { constexpr Scalar latvap = C::LatVap; - //fortran generated data is input to the following + //baseline generated data is input to the following P3UpdatePrognosticLiqData pupldc[max_pack_size] = { {1.0631E-12, 1.0631E+00, 1.5833E-12, 1.5833E+00, 2.4190E-02, 0.0000E+00, 0.0000E+00, 0.0000E+00, 4.2517E+00, @@ -896,9 +931,11 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq std::copy(&pupldc[0], &pupldc[0] + max_pack_size, pupldc_host.data()); Kokkos::deep_copy(pupldc_device, pupldc_host); - // Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - update_prognostic_liquid(pupldc[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + pupldc[i].read(Base::m_fid); + } } // Run the lookup from a kernel and copy results back to host @@ -971,7 +1008,7 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq Kokkos::deep_copy(pupldc_host, pupldc_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(pupldc[s].th_atm == pupldc_host(s).th_atm); REQUIRE(pupldc[s].qv == pupldc_host(s).qv); @@ -981,18 +1018,23 @@ struct UnitWrap::UnitTest::TestP3UpdatePrognosticLiq REQUIRE(pupldc[s].nr == pupldc_host(s).nr); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + pupldc_host(s).write(Base::m_fid); + } + } } - static void run_bfb(){ + void run_bfb() { update_prognostic_liquid_unit_bfb_tests(); } }; //TestP3UpdatePrognosticLiq template -struct UnitWrap::UnitTest::TestP3FunctionsImposeMaxTotalNi +struct UnitWrap::UnitTest::TestP3FunctionsImposeMaxTotalNi : public UnitWrap::UnitTest::Base { - static void impose_max_total_ni_bfb_test(){ + void impose_max_total_ni_bfb_test() { constexpr Scalar max_total_ni = 740.0e3; ImposeMaxTotalNiData dc[max_pack_size]= { @@ -1026,9 +1068,11 @@ struct UnitWrap::UnitTest::TestP3FunctionsImposeMaxTotalNi std::copy(&dc[0], &dc[0] + max_pack_size, dc_host.data()); Kokkos::deep_copy(dc_device, dc_host); - //Get data from fortran - for (Int i = 0; i < max_pack_size; ++i) { - impose_max_total_ni(dc[i]); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (Int i = 0; i < max_pack_size; ++i) { + dc[i].read(Base::m_fid); + } } //Run function from a kernal and copy results back to the host @@ -1054,15 +1098,20 @@ struct UnitWrap::UnitTest::TestP3FunctionsImposeMaxTotalNi Kokkos::deep_copy(dc_host, dc_device); // Validate results - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int s = 0; s < max_pack_size; ++s) { REQUIRE(dc[s].ni_local == dc_host(s).ni_local); REQUIRE(dc[s].inv_rho_local == dc_host(s).inv_rho_local); } } + else if (this->m_baseline_action == GENERATE) { + for (Int s = 0; s < max_pack_size; ++s) { + dc_host(s).write(Base::m_fid); + } + } } - static void run_bfb(){ + void run_bfb() { impose_max_total_ni_bfb_test(); } @@ -1075,24 +1124,39 @@ struct UnitWrap::UnitTest::TestP3FunctionsImposeMaxTotalNi namespace { TEST_CASE("p3_conservation_test", "[p3_unit_tests]"){ - scream::p3::unit_test::UnitWrap::UnitTest::TestP3Conservation::run(); - scream::p3::unit_test::UnitWrap::UnitTest::TestP3Conservation::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestP3Conservation; + + T t; + t.run(); + t.run_bfb(); } TEST_CASE("p3_get_time_space_phys_variables_test", "[p3_unit_tests]"){ - scream::p3::unit_test::UnitWrap::UnitTest::TestGetTimeSpacePhysVariables::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestGetTimeSpacePhysVariables; + + T t; + t.run_bfb(); } TEST_CASE("p3_update_prognostic_ice_test", "[p3_unit_tests]"){ - scream::p3::unit_test::UnitWrap::UnitTest::TestP3UpdatePrognosticIce::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestP3UpdatePrognosticIce; + + T t; + t.run_bfb(); } TEST_CASE("p3_update_prognostic_liquid_test", "[p3_unit_tests]"){ - scream::p3::unit_test::UnitWrap::UnitTest::TestP3UpdatePrognosticLiq::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestP3UpdatePrognosticLiq; + + T t; + t.run_bfb(); } TEST_CASE("p3_impose_max_total_ni_test", "[p3_unit_tests]"){ - scream::p3::unit_test::UnitWrap::UnitTest::TestP3FunctionsImposeMaxTotalNi::run_bfb(); + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestP3FunctionsImposeMaxTotalNi; + + T t; + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/p3/tests/p3_upwind_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_upwind_unit_tests.cpp index fde2ca644cb..6e83a8ee4ea 100644 --- a/components/eamxx/src/physics/p3/tests/p3_upwind_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_upwind_unit_tests.cpp @@ -3,11 +3,9 @@ #include "p3_unit_tests_common.hpp" #include "p3_functions.hpp" -#include "p3_functions_f90.hpp" +#include "p3_test_data.hpp" #include "share/scream_types.hpp" -#include "share/util/scream_setup_random_test.hpp" -#include "share/util/scream_setup_random_test.hpp" #include "ekat/ekat_pack.hpp" #include "ekat/kokkos/ekat_kokkos_utils.hpp" @@ -36,9 +34,9 @@ namespace unit_test { // cells in the domain are 0. This lets us check the restricted-domain usage of // the upwind routine in the first time step. template -struct UnitWrap::UnitTest::TestUpwind { +struct UnitWrap::UnitTest::TestUpwind : public UnitWrap::UnitTest::Base { -static void run_phys() +void run_phys() { using ekat::repack; constexpr auto SPS = SCREAM_SMALL_PACK_SIZE; @@ -201,11 +199,12 @@ static void run_phys() } } -static void run_bfb() +void run_bfb() { - auto engine = setup_random_test(); + // With stored baselines, we must use a fixed seed! + auto engine = Base::get_engine(); - CalcUpwindData cuds_fortran[] = { + CalcUpwindData cuds_baseline[] = { // kts, kte, kdir, kbot, k_qxtop, na, dt_sub, CalcUpwindData( 1, 72, -1, 72, 36, 2, 1.833E+03), CalcUpwindData( 1, 72, 1, 36, 72, 2, 1.833E+03), @@ -216,28 +215,30 @@ static void run_bfb() CalcUpwindData( 1, 32, -1, 21, 7, 1, 1.833E+03), }; - static constexpr Int num_runs = sizeof(cuds_fortran) / sizeof(CalcUpwindData); + static constexpr Int num_runs = sizeof(cuds_baseline) / sizeof(CalcUpwindData); // Set up random input data - for (auto& d : cuds_fortran) { + for (auto& d : cuds_baseline) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state CalcUpwindData cuds_cxx[num_runs] = { - CalcUpwindData(cuds_fortran[0]), - CalcUpwindData(cuds_fortran[1]), - CalcUpwindData(cuds_fortran[2]), - CalcUpwindData(cuds_fortran[3]), - CalcUpwindData(cuds_fortran[4]), - CalcUpwindData(cuds_fortran[5]), - CalcUpwindData(cuds_fortran[6]), + CalcUpwindData(cuds_baseline[0]), + CalcUpwindData(cuds_baseline[1]), + CalcUpwindData(cuds_baseline[2]), + CalcUpwindData(cuds_baseline[3]), + CalcUpwindData(cuds_baseline[4]), + CalcUpwindData(cuds_baseline[5]), + CalcUpwindData(cuds_baseline[6]), }; - // Get data from fortran - for (auto& d : cuds_fortran) { - calc_first_order_upwind_step(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : cuds_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx @@ -245,23 +246,23 @@ static void run_bfb() for (auto& d : cuds_cxx) { Real** fluxes, **vs, **qnx; d.convert_to_ptr_arr(tmp1, fluxes, vs, qnx); - calc_first_order_upwind_step_f( + calc_first_order_upwind_step_host( d.kts, d.kte, d.kdir, d.kbot, d.k_qxtop, d.dt_sub, d.rho, d.inv_rho, d.inv_dz, d.num_arrays, fluxes, vs, qnx); } - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { // Due to pack issues, we must restrict checks to the active k space - Int start = std::min(cuds_fortran[i].kbot, cuds_fortran[i].k_qxtop) - 1; // 0-based indx - Int end = std::max(cuds_fortran[i].kbot, cuds_fortran[i].k_qxtop); // 0-based indx + Int start = std::min(cuds_baseline[i].kbot, cuds_baseline[i].k_qxtop) - 1; // 0-based indx + Int end = std::max(cuds_baseline[i].kbot, cuds_baseline[i].k_qxtop); // 0-based indx Real** fluxesf90, **vsf90, **qnxf90, **fluxescxx, **vscxx, **qnxcxx; - cuds_fortran[i].convert_to_ptr_arr(tmp1, fluxesf90, vsf90, qnxf90); + cuds_baseline[i].convert_to_ptr_arr(tmp1, fluxesf90, vsf90, qnxf90); cuds_cxx[i].convert_to_ptr_arr(tmp1, fluxescxx, vscxx, qnxcxx); - for (int n = 0; n < cuds_fortran[i].num_arrays; ++n) { + for (int n = 0; n < cuds_baseline[i].num_arrays; ++n) { for (Int k = start; k < end; ++k) { REQUIRE(fluxesf90[n][k] == fluxescxx[n][k]); REQUIRE(qnxf90[n][k] == qnxcxx[n][k]); @@ -269,23 +270,29 @@ static void run_bfb() } } } + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + cuds_cxx[i].write(Base::m_fid); + } + } } }; template -struct UnitWrap::UnitTest::TestGenSed { +struct UnitWrap::UnitTest::TestGenSed : public UnitWrap::UnitTest::Base { -static void run_phys() +void run_phys() { // TODO } -static void run_bfb() +void run_bfb() { - auto engine = setup_random_test(); + // With stored baselines, we must use a fixed seed! + auto engine = Base::get_engine(); - GenSedData gsds_fortran[] = { + GenSedData gsds_baseline[] = { // kts, kte, kdir, k_qxtop, k_qxbot, kbot, Co_max, dt_left, prt_accum, num_arrays GenSedData(1, 72, -1, 36, 72, 72, 9.196E-02, 1.818E+01, 4.959E-05, 2), GenSedData(1, 72, -1, 36, 57, 72, 4.196E-01, 1.418E+02, 4.959E-06, 1), @@ -293,25 +300,27 @@ static void run_bfb() GenSedData(1, 72, -1, 72, 72, 72, 4.196E-01, 1.418E+02, 4.959E-06, 1), }; - static constexpr Int num_runs = sizeof(gsds_fortran) / sizeof(GenSedData); + static constexpr Int num_runs = sizeof(gsds_baseline) / sizeof(GenSedData); // Set up random input data - for (auto& d : gsds_fortran) { + for (auto& d : gsds_baseline) { d.randomize(engine); } - // Create copies of data for use by cxx. Needs to happen before fortran calls so that + // Create copies of data for use by cxx. Needs to happen before reads so that // inout data is in original state GenSedData gsds_cxx[num_runs] = { - GenSedData(gsds_fortran[0]), - GenSedData(gsds_fortran[1]), - GenSedData(gsds_fortran[2]), - GenSedData(gsds_fortran[3]), + GenSedData(gsds_baseline[0]), + GenSedData(gsds_baseline[1]), + GenSedData(gsds_baseline[2]), + GenSedData(gsds_baseline[3]), }; - // Get data from fortran - for (auto& d : gsds_fortran) { - generalized_sedimentation(d); + // Read baseline data + if (this->m_baseline_action == COMPARE) { + for (auto& d : gsds_baseline) { + d.read(Base::m_fid); + } } // Get data from cxx @@ -319,31 +328,36 @@ static void run_bfb() for (auto& d : gsds_cxx) { Real** fluxes, **vs, **qnx; d.convert_to_ptr_arr(tmp1, fluxes, vs, qnx); - generalized_sedimentation_f(d.kts, d.kte, d.kdir, d.k_qxtop, + generalized_sedimentation_host(d.kts, d.kte, d.kdir, d.k_qxtop, &d.k_qxbot, d.kbot, d.Co_max, &d.dt_left, &d.prt_accum, d.inv_dz, d.inv_rho, d.rho, d.num_arrays, fluxes, vs, qnx); } - if (SCREAM_BFB_TESTING) { + if (SCREAM_BFB_TESTING && this->m_baseline_action == COMPARE) { for (Int i = 0; i < num_runs; ++i) { // Due to pack issues, we must restrict checks to the active k space - Int start = std::min(gsds_fortran[i].k_qxbot, gsds_fortran[i].k_qxtop) - 1; // 0-based indx - Int end = std::max(gsds_fortran[i].k_qxbot, gsds_fortran[i].k_qxtop); // 0-based indx + Int start = std::min(gsds_baseline[i].k_qxbot, gsds_baseline[i].k_qxtop) - 1; // 0-based indx + Int end = std::max(gsds_baseline[i].k_qxbot, gsds_baseline[i].k_qxtop); // 0-based indx Real** fluxesf90, **vsf90, **qnxf90, **fluxescxx, **vscxx, **qnxcxx; - gsds_fortran[i].convert_to_ptr_arr(tmp1, fluxesf90, vsf90, qnxf90); + gsds_baseline[i].convert_to_ptr_arr(tmp1, fluxesf90, vsf90, qnxf90); gsds_cxx[i].convert_to_ptr_arr(tmp1, fluxescxx, vscxx, qnxcxx); - for (int n = 0; n < gsds_fortran[i].num_arrays; ++n) { + for (int n = 0; n < gsds_baseline[i].num_arrays; ++n) { for (Int k = start; k < end; ++k) { REQUIRE(fluxesf90[n][k] == fluxescxx[n][k]); REQUIRE(qnxf90[n][k] == qnxcxx[n][k]); } } - REQUIRE(gsds_fortran[i].k_qxbot == gsds_cxx[i].k_qxbot); - REQUIRE(gsds_fortran[i].dt_left == gsds_cxx[i].dt_left); - REQUIRE(gsds_fortran[i].prt_accum == gsds_cxx[i].prt_accum); + REQUIRE(gsds_baseline[i].k_qxbot == gsds_cxx[i].k_qxbot); + REQUIRE(gsds_baseline[i].dt_left == gsds_cxx[i].dt_left); + REQUIRE(gsds_baseline[i].prt_accum == gsds_cxx[i].prt_accum); + } + } + else if (this->m_baseline_action == GENERATE) { + for (Int i = 0; i < num_runs; ++i) { + gsds_cxx[i].write(Base::m_fid); } } } @@ -358,18 +372,20 @@ namespace { TEST_CASE("p3_upwind", "[p3_functions]") { - using TU = scream::p3::unit_test::UnitWrap::UnitTest::TestUpwind; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestUpwind; - TU::run_phys(); - TU::run_bfb(); + T t; + t.run_phys(); + t.run_bfb(); } TEST_CASE("p3_gen_sed", "[p3_functions]") { - using TG = scream::p3::unit_test::UnitWrap::UnitTest::TestGenSed; + using T = scream::p3::unit_test::UnitWrap::UnitTest::TestGenSed; - TG::run_phys(); - TG::run_bfb(); + T t; + t.run_phys(); + t.run_bfb(); } } // namespace diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 4a1e1f040a0..0121741963d 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -620,6 +620,10 @@ void RRTMGPRadiation::initialize_impl(const RunType /* run_type */) { m_orbital_obliq = m_params.get("orbital_obliquity" ,-9999); m_orbital_mvelp = m_params.get("orbital_mvelp" ,-9999); + // Value for prescribing an invariant solar constant (i.e. total solar irradiance at + // TOA). Used for idealized experiments such as RCE. Disabled when value is less than 0. + m_fixed_total_solar_irradiance = m_params.get("fixed_total_solar_irradiance", -9999); + // Determine whether or not we are using a fixed solar zenith angle (positive value) m_fixed_solar_zenith_angle = m_params.get("Fixed Solar Zenith Angle", -9999); @@ -840,6 +844,12 @@ void RRTMGPRadiation::run_impl (const double dt) { shr_orb_decl_c2f(calday, eccen, mvelpp, lambm0, obliqr, &delta, &eccf); + // Overwrite eccf if using a fixed solar constant. + auto fixed_total_solar_irradiance = m_fixed_total_solar_irradiance; + if (fixed_total_solar_irradiance >= 0){ + eccf = fixed_total_solar_irradiance/1360.9; + } + // Precompute VMR for all gases, on all cols, before starting the chunks loop // // h2o is taken from qv diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp index 329c7eeaba8..02a047cce59 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.hpp @@ -95,6 +95,11 @@ class RRTMGPRadiation : public AtmosphereProcess { Real m_orbital_obliq; // Obliquity Real m_orbital_mvelp; // Vernal Equinox Mean Longitude of Perihelion + // Value for prescribing an invariant solar constant (i.e. total solar irradiance + // at TOA). Used for idealized experiments such as RCE. This is only used when a + // positive value is supplied. + Real m_fixed_total_solar_irradiance; + // Fixed solar zenith angle to use for shortwave calculations // This is only used if a positive value is supplied Real m_fixed_solar_zenith_angle; diff --git a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp index a420c0c1dd4..e807643b05a 100644 --- a/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp +++ b/components/eamxx/src/physics/rrtmgp/scream_rrtmgp_interface.hpp @@ -167,9 +167,6 @@ using MDRP = typename conv::MDRP; template using view_t = Kokkos::View; -template -using oview_t = Kokkos::Experimental::OffsetView; - template using hview_t = Kokkos::View; @@ -243,7 +240,7 @@ static void rrtmgp_initialize( load_cld_lutcoeff(cloud_optics_lw_k, cloud_optics_file_lw); // initialize kokkos rrtmgp pool allocator - const size_t base_ref = 18000; + const size_t base_ref = 80000; const size_t ncol = gas_concs.ncol; const size_t nlay = gas_concs.nlay; const size_t nlev = SCREAM_NUM_VERTICAL_LEV; @@ -561,47 +558,9 @@ static void rrtmgp_sw( const bool extra_clnclrsky_diag, const bool extra_clnsky_diag) { // Get problem sizes - int nbnd = k_dist.get_nband(); - int ngpt = k_dist.get_ngpt(); - int ngas = gas_concs.get_num_gases(); - - // Allocate temporaries from pool - const int size1 = nday; - const int size2 = nday*nlay; // 4 - const int size3 = nday*(nlay+1); // 5 - const int size4 = ncol*nlay; - const int size5 = nbnd*nday; //2 - const int size6 = nday*ngpt; - const int size7 = nday*(nlay+1)*nbnd; // 3 - const int size8 = ncol*nlay*(k_dist.get_ngas()+1); - - RealT* data = pool_t::template alloc_raw(size1 + size2*4 + size3*5 + size4 + size5*2 + size6 + size7*3 + size8), *dcurr = data; - - auto mu0_day = view_t (dcurr, nday); dcurr += size1; - - auto p_lay_day = view_t (dcurr, nday, nlay); dcurr += size2; - auto t_lay_day = view_t (dcurr, nday, nlay); dcurr += size2; - auto vmr_day = view_t (dcurr, nday, nlay); dcurr += size2; - auto t_lay_limited = view_t (dcurr, nday, nlay); dcurr += size2; - - auto p_lev_day = view_t (dcurr, nday, nlay+1); dcurr += size3; - auto t_lev_day = view_t (dcurr, nday, nlay+1); dcurr += size3; - auto flux_up_day = view_t (dcurr, nday, nlay+1); dcurr += size3; - auto flux_dn_day = view_t (dcurr, nday, nlay+1); dcurr += size3; - auto flux_dn_dir_day = view_t (dcurr, nday, nlay+1); dcurr += size3; - - auto vmr = view_t (dcurr, ncol, nlay); dcurr += size4; - - auto sfc_alb_dir_T = view_t (dcurr, nbnd, nday); dcurr += size5; - auto sfc_alb_dif_T = view_t (dcurr, nbnd, nday); dcurr += size5; - - auto toa_flux = view_t (dcurr, nday, ngpt); dcurr += size6; - - auto bnd_flux_up_day = view_t(dcurr, nday, nlay+1, nbnd); dcurr += size7; - auto bnd_flux_dn_day = view_t(dcurr, nday, nlay+1, nbnd); dcurr += size7; - auto bnd_flux_dn_dir_day = view_t(dcurr, nday, nlay+1, nbnd); dcurr += size7; - - auto col_gas = view_t(dcurr, ncol, nlay, k_dist.get_ngas()+1); dcurr += size8; + const int nbnd = k_dist.get_nband(); + const int ngpt = k_dist.get_ngpt(); + const int ngas = gas_concs.get_num_gases(); // Associate local pointers for fluxes auto &flux_up = fluxes.flux_up; @@ -642,7 +601,7 @@ static void rrtmgp_sw( }); // Get daytime indices - auto dayIndices = view_t("dayIndices", ncol); + auto dayIndices = pool_t::template alloc_and_init(ncol); Kokkos::deep_copy(dayIndices, -1); int nday = 0; @@ -657,9 +616,49 @@ static void rrtmgp_sw( if (nday == 0) { // No daytime columns in this chunk, skip the rest of this routine + pool_t::dealloc(dayIndices); return; } + // Allocate temporaries from pool + const int size1 = nday; + const int size2 = nday*nlay; // 4 + const int size3 = nday*(nlay+1); // 5 + const int size4 = ncol*nlay; + const int size5 = nbnd*nday; //2 + const int size6 = nday*ngpt; + const int size7 = nday*(nlay+1)*nbnd; // 3 + const int size8 = ncol*nlay*(k_dist.get_ngas()+1); + + const int total_size = size1 + size2*4 + size3*5 + size4 + size5*2 + size6 + size7*3 + size8; + auto data = pool_t::template alloc_and_init(total_size); RealT* dcurr = data.data(); + + auto mu0_day = view_t (dcurr, nday); dcurr += size1; + + auto p_lay_day = view_t (dcurr, nday, nlay); dcurr += size2; + auto t_lay_day = view_t (dcurr, nday, nlay); dcurr += size2; + auto vmr_day = view_t (dcurr, nday, nlay); dcurr += size2; + auto t_lay_limited = view_t (dcurr, nday, nlay); dcurr += size2; + + auto p_lev_day = view_t (dcurr, nday, nlay+1); dcurr += size3; + auto t_lev_day = view_t (dcurr, nday, nlay+1); dcurr += size3; + auto flux_up_day = view_t (dcurr, nday, nlay+1); dcurr += size3; + auto flux_dn_day = view_t (dcurr, nday, nlay+1); dcurr += size3; + auto flux_dn_dir_day = view_t (dcurr, nday, nlay+1); dcurr += size3; + + auto vmr = view_t (dcurr, ncol, nlay); dcurr += size4; + + auto sfc_alb_dir_T = view_t (dcurr, nbnd, nday); dcurr += size5; + auto sfc_alb_dif_T = view_t (dcurr, nbnd, nday); dcurr += size5; + + auto toa_flux = view_t (dcurr, nday, ngpt); dcurr += size6; + + auto bnd_flux_up_day = view_t(dcurr, nday, nlay+1, nbnd); dcurr += size7; + auto bnd_flux_dn_day = view_t(dcurr, nday, nlay+1, nbnd); dcurr += size7; + auto bnd_flux_dn_dir_day = view_t(dcurr, nday, nlay+1, nbnd); dcurr += size7; + + auto col_gas = view_t(dcurr, ncol, nlay, k_dist.get_ngas()+1); dcurr += size8; + // Subset mu0 Kokkos::parallel_for(nday, KOKKOS_LAMBDA(int iday) { mu0_day(iday) = mu0(dayIndices(iday)); @@ -823,7 +822,8 @@ static void rrtmgp_sw( }); } - pool_t::dealloc(data, dcurr - data); + pool_t::dealloc(data); + pool_t::dealloc(dayIndices); } /* @@ -849,7 +849,8 @@ static void rrtmgp_lw( const int size5 = ncol*(nlay+1); const int size6 = ncol*nlay*(k_dist.get_ngas()+1); - RealT* data = pool_t::template alloc_raw(size1 + size2 + size3*2 + size4 + size5 + size6), *dcurr = data; + const int total_size = size1 + size2 + size3*2 + size4 + size5 + size6; + auto data = pool_t::template alloc_and_init(total_size); RealT *dcurr = data.data(); view_t t_sfc (dcurr, ncol); dcurr += size1; view_t emis_sfc (dcurr, nbnd,ncol); dcurr += size2; @@ -857,7 +858,7 @@ static void rrtmgp_lw( view_t gauss_wts (dcurr, max_gauss_pts,max_gauss_pts); dcurr += size3; view_t t_lay_limited(dcurr, ncol, nlay); dcurr += size4; view_t t_lev_limited(dcurr, ncol, nlay+1); dcurr += size5; - view_t col_gas (dcurr, std::make_pair(0, ncol-1), std::make_pair(0, nlay-1), std::make_pair(-1, k_dist.get_ngas()-1)); dcurr += size6; + view_t col_gas (dcurr, ncol, nlay, k_dist.get_ngas()+1); dcurr += size6; // Associate local pointers for fluxes auto &flux_up = fluxes.flux_up; @@ -978,13 +979,14 @@ static void rrtmgp_lw( rte_lw(max_gauss_pts, gauss_Ds, gauss_wts, optics_no_aerosols, top_at_1, lw_sources, emis_sfc, clnsky_fluxes); } - pool_t::dealloc(data, dcurr - data); + pool_t::dealloc(data); } /* * Return a subcolumn mask consistent with a specified overlap assumption */ -static void get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, const real2dk &cldf, const int overlap_option, int1dk &seeds, int3dk& subcolumn_mask) +template +static void get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, const CldfT &cldf, const int overlap_option, const SeedsT &seeds, const SubcT& subcolumn_mask) { // Subcolumn generators are a means for producing a variable x(i,j,k), where // @@ -992,7 +994,7 @@ static void get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, c // c(i,j,k) = 0 for x(i,j,k) <= 1 - cldf(i,j) // // I am going to call this "cldx" to be just slightly less ambiguous - auto cldx = pool_t::template alloc(ncol, nlay, ngpt); + auto cldx = pool_t::template alloc_and_init(ncol, nlay, ngpt); // Apply overlap assumption to set cldx if (overlap_option == 0) { // Dummy mask, always cloudy @@ -1054,8 +1056,6 @@ static void get_subcolumn_mask(const int ncol, const int nlay, const int ngpt, c }); pool_t::dealloc(cldx); - - return subcolumn_mask; } /* @@ -1067,7 +1067,7 @@ static void compute_cloud_area( { // Subcolumn binary cld mask; if any layers with pressure between pmin and pmax are cloudy // then 2d subcol mask is 1, otherwise it is 0 - auto subcol_mask = pool_t::template alloc(ncol, ngpt); + auto subcol_mask = pool_t::template alloc_and_init(ncol, ngpt); Kokkos::parallel_for(MDRP::template get<3>({ngpt, nlay, ncol}), KOKKOS_LAMBDA(int igpt, int ilay, int icol) { // NOTE: using plev would need to assume level ordering (top to bottom or bottom to top), but // using play/pmid does not @@ -1118,7 +1118,7 @@ static void compute_aerocom_cloudtop( Kokkos::deep_copy(eff_radius_qi_at_cldtop, 0.0); // Initialize the 1D "clear fraction" as 1 (totally clear) - auto aerocom_clr = pool_t::template alloc(ncol); + auto aerocom_clr = pool_t::template alloc_and_init(ncol); Kokkos::deep_copy(aerocom_clr, 1.0); // Get gravity acceleration constant from constants @@ -1295,8 +1295,8 @@ static optical_props2_t get_cloud_optics_sw( cloud_optics.set_ice_roughness(2); // Limit effective radii to be within bounds of lookup table - auto rel_limited = pool_t::template alloc(ncol, nlay); - auto rei_limited = pool_t::template alloc(ncol, nlay); + auto rel_limited = pool_t::template alloc_and_init(ncol, nlay); + auto rei_limited = pool_t::template alloc_and_init(ncol, nlay); limit_to_bounds_k(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); limit_to_bounds_k(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); @@ -1324,8 +1324,8 @@ static optical_props1_t get_cloud_optics_lw( cloud_optics.set_ice_roughness(2); // Limit effective radii to be within bounds of lookup table - auto rel_limited = pool_t::alloc(ncol, nlay); - auto rei_limited = pool_t::alloc(ncol, nlay); + auto rel_limited = pool_t::template alloc_and_init(ncol, nlay); + auto rei_limited = pool_t::template alloc_and_init(ncol, nlay); limit_to_bounds_k(rel, cloud_optics.radliq_lwr, cloud_optics.radliq_upr, rel_limited); limit_to_bounds_k(rei, cloud_optics.radice_lwr, cloud_optics.radice_upr, rei_limited); @@ -1348,7 +1348,7 @@ static optical_props2_t get_subsampled_clouds( subsampled_optics.alloc_2str(ncol, nlay); // Subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud - auto cldmask = pool_t::alloc(ncol, nlay, ngpt); + auto cldmask = pool_t::template alloc_and_init(ncol, nlay, ngpt); // Check that we do not have clouds with no optical properties; this would get corrected // when we assign optical props, but we want to use a "radiative cloud fraction" @@ -1357,7 +1357,7 @@ static optical_props2_t get_subsampled_clouds( // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped // even when separated by layers with no cloud properties, when in fact those layers should be // randomly overlapped. - auto cldfrac_rad = pool_t::alloc(ncol, nlay); + auto cldfrac_rad = pool_t::template alloc_and_init(ncol, nlay); Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay,ncol}), KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { if (cloud_optics.tau(icol,ilay,ibnd) > 0) { cldfrac_rad(icol,ilay) = cld(icol,ilay); @@ -1371,7 +1371,7 @@ static optical_props2_t get_subsampled_clouds( int overlap = 1; // Get unique seeds for each column that are reproducible across different MPI rank layouts; // use decimal part of pressure for this, consistent with the implementation in EAM - auto seeds = pool_t::alloc(ncol); + auto seeds = pool_t::template alloc_and_init(ncol); Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { seeds(icol) = 1e9 * (p_lay(icol,nlay-1) - int(p_lay(icol,nlay-1))); }); @@ -1408,7 +1408,7 @@ static optical_props1_t get_subsampled_clouds( subsampled_optics.alloc_1scl(ncol, nlay); // Subcolumn mask with values of 0 indicating no cloud, 1 indicating cloud - auto cldmask = pool_t::alloc(ncol, nlay, ngpt); + auto cldmask = pool_t::template alloc_and_init(ncol, nlay, ngpt); // Check that we do not have clouds with no optical properties; this would get corrected // when we assign optical props, but we want to use a "radiative cloud fraction" @@ -1417,7 +1417,7 @@ static optical_props1_t get_subsampled_clouds( // the vertical correlation of cloudy layers. I.e., cloudy layers might look maximally overlapped // even when separated by layers with no cloud properties, when in fact those layers should be // randomly overlapped. - auto cldfrac_rad = pool_t::alloc(ncol, nlay); + auto cldfrac_rad = pool_t::template alloc_and_init(ncol, nlay); Kokkos::parallel_for(MDRP::template get<3>({nbnd,nlay,ncol}), KOKKOS_LAMBDA (int ibnd, int ilay, int icol) { if (cloud_optics.tau(icol,ilay,ibnd) > 0) { cldfrac_rad(icol,ilay) = cld(icol,ilay); @@ -1428,7 +1428,7 @@ static optical_props1_t get_subsampled_clouds( // Get unique seeds for each column that are reproducible across different MPI rank layouts; // use decimal part of pressure for this, consistent with the implementation in EAM; use different // seed values for longwave and shortwave - auto seeds = pool_t::alloc(ncol); + auto seeds = pool_t::template alloc_and_init(ncol); Kokkos::parallel_for(ncol, KOKKOS_LAMBDA(int icol) { seeds(icol) = 1e9 * (p_lay(icol,nlay-2) - int(p_lay(icol,nlay-2))); }); diff --git a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp index 0d3f18e7d84..fedb8c3d047 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp +++ b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_tests.cpp @@ -46,7 +46,7 @@ int run_yakl(int argc, char** argv) { logger->error(msg); return 1; } - std::string inputfile, baseline, device; + std::string inputfile, baseline; for (int i = 1; i < argc-1; ++i) { if (ekat::argv_matches(argv[i], "-b", "--baseline-file")) { @@ -60,10 +60,8 @@ int run_yakl(int argc, char** argv) { inputfile = argv[i]; } // RRTMGP baselines tests to not use kokoks. Swallow the arg, but ignore it - if (std::string(argv[i])=="--ekat-kokkos-device") { - expect_another_arg(i, argc); - ++i; - device = argv[i]; + if (std::string(argv[i])=="--kokkos-device-id=") { + continue; } } @@ -352,10 +350,8 @@ int run_kokkos(int argc, char** argv) { inputfile = argv[i]; } // RRTMGP baselines tests to not use kokoks. Swallow the arg, but ignore it - if (std::string(argv[i])=="--ekat-kokkos-device") { - expect_another_arg(i, argc); - ++i; - device = argv[i]; + if (std::string(argv[i])=="--kokkos-device-id=") { + continue; } } diff --git a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp index 99a58244561..06ca64af5f2 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp +++ b/components/eamxx/src/physics/rrtmgp/tests/rrtmgp_unit_tests.cpp @@ -850,6 +850,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop") { #ifdef RRTMGP_ENABLE_KOKKOS using interface_t = scream::rrtmgp::rrtmgp_interface<>; +using pool_t = interface_t::pool_t; using real1dk = interface_t::view_t; using real2dk = interface_t::view_t; using real3dk = interface_t::view_t; @@ -861,6 +862,7 @@ using MDRP = interface_t::MDRP; TEST_CASE("rrtmgp_test_heating_k") { // Initialize Kokkos scream::init_kls(); + pool_t::init(10000); // Test heating rate function by passing simple inputs auto dp = real2dk("dp", 1, 1); @@ -908,12 +910,14 @@ TEST_CASE("rrtmgp_test_heating_k") { REQUIRE(chc(heating)(0,0) == heating_ref); // Clean up + pool_t::finalize(); scream::finalize_kls(); } TEST_CASE("rrtmgp_test_mixing_ratio_to_cloud_mass_k") { // Initialize YAKL scream::init_kls(); + pool_t::init(10000); using physconst = scream::physics::Constants; @@ -966,12 +970,14 @@ TEST_CASE("rrtmgp_test_mixing_ratio_to_cloud_mass_k") { REQUIRE(chc(cloud_mass)(0,0) == cloud_mass_ref); // Clean up + pool_t::finalize(); scream::finalize_kls(); } TEST_CASE("rrtmgp_test_limit_to_bounds_k") { // Initialize YAKL scream::init_kls(); + pool_t::init(10000); // Test limiter function auto arr = real2dk("arr", 2, 2); @@ -998,6 +1004,8 @@ TEST_CASE("rrtmgp_test_limit_to_bounds_k") { REQUIRE(chc(arr_limited)(0,1) == 2.0); REQUIRE(chc(arr_limited)(1,0) == 3.0); REQUIRE(chc(arr_limited)(1,1) == 3.5); + + pool_t::finalize(); scream::finalize_kls(); } @@ -1070,6 +1078,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { // Initialize YAKL scream::init_kls(); + pool_t::init(10000); // Create arrays const int ncol = 1; @@ -1227,6 +1236,7 @@ TEST_CASE("rrtmgp_test_compute_broadband_surface_flux_k") { logger->info("Free memory...\n"); interface_t::rrtmgp_finalize(); gas_concs.reset(); + pool_t::finalize(); scream::finalize_kls(); } @@ -1255,6 +1265,7 @@ TEST_CASE("rrtmgp_test_radiation_do_k") { TEST_CASE("rrtmgp_test_check_range_k") { // Initialize YAKL scream::init_kls(); + pool_t::init(10000); // Create some dummy data and test with both values inside valid range and outside auto dummy = real2dk("dummy", 2, 1); // All values within range @@ -1266,12 +1277,14 @@ TEST_CASE("rrtmgp_test_check_range_k") { // At least one value above upper bound Kokkos::parallel_for(1, KOKKOS_LAMBDA (int i) {dummy(i, 0) = 1.1;}); REQUIRE(scream::rrtmgp::check_range_k(dummy, 0.0, 1.0, "dummy") == false); + pool_t::finalize(); scream::finalize_kls(); } TEST_CASE("rrtmgp_test_subcol_gen_k") { // Initialize YAKL scream::init_kls(); + pool_t::init(10000); // Create dummy data const int ncol = 1; const int nlay = 4; @@ -1291,7 +1304,7 @@ TEST_CASE("rrtmgp_test_subcol_gen_k") { for (unsigned seed = 0; seed < 10; seed++) { auto seeds = int1dk("seeds", ncol); Kokkos::deep_copy(seeds, seed); - cldmask = interface_t::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds); + interface_t::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds, cldmask); // Check answers by computing new cldfrac from mask Kokkos::deep_copy(cldfrac_from_mask, 0.0); Kokkos::parallel_for(MDRP::template get<2>({nlay,ncol}), KOKKOS_LAMBDA(int ilay, int icol) { @@ -1325,7 +1338,7 @@ TEST_CASE("rrtmgp_test_subcol_gen_k") { for (unsigned seed = 0; seed < 10; seed++) { auto seeds = int1dk("seeds", ncol); Kokkos::deep_copy(seeds, seed); - cldmask = interface_t::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds); + interface_t::get_subcolumn_mask(ncol, nlay, ngpt, cldfrac, 1, seeds, cldmask); auto cldmask_h = chc(cldmask); for (int igpt = 0; igpt < ngpt; igpt++) { if (cldmask_h(0,0,igpt) == 1) { @@ -1334,12 +1347,14 @@ TEST_CASE("rrtmgp_test_subcol_gen_k") { } } // Clean up after test + pool_t::finalize(); scream::finalize_kls(); } TEST_CASE("rrtmgp_cloud_area_k") { // Initialize YAKL scream::init_kls(); + pool_t::init(10000); // Create dummy data const int ncol = 1; const int nlay = 2; @@ -1429,12 +1444,14 @@ TEST_CASE("rrtmgp_cloud_area_k") { REQUIRE(chc(cldtot)(0) == 0.0); interface_t::compute_cloud_area(ncol, nlay, ngpt, 100, 300, pmid, cldtau, cldtot); REQUIRE(chc(cldtot)(0) == 2.0 / 3.0); + pool_t::finalize(); scream::finalize_kls(); } TEST_CASE("rrtmgp_aerocom_cloudtop_k") { // Initialize YAKL scream::init_kls(); + pool_t::init(10000); // Create dummy data const int ncol = 1; @@ -1623,6 +1640,7 @@ TEST_CASE("rrtmgp_aerocom_cloudtop_k") { REQUIRE(chc(cldfrac_ice_at_cldtop)(0) == 0.7); // max // cleanup + pool_t::finalize(); scream::finalize_kls(); } #endif diff --git a/components/eamxx/src/physics/share/physics_test_data.cpp b/components/eamxx/src/physics/share/physics_test_data.cpp index 4a13528e98b..f6c6d733fd0 100644 --- a/components/eamxx/src/physics/share/physics_test_data.cpp +++ b/components/eamxx/src/physics/share/physics_test_data.cpp @@ -26,4 +26,21 @@ PhysicsTestData& PhysicsTestData::assignment_impl(const PhysicsTestData& rhs) return *this; } +void PhysicsTestData::read(const ekat::FILEPtr& fid) +{ + EKAT_REQUIRE_MSG(fid, + "Tried to read from missing file. You may have forgotten to generate baselines for some BFB unit tests"); + m_reals.read(fid); + m_ints.read(fid); + m_bools.read(fid); +} + +void PhysicsTestData::write(const ekat::FILEPtr& fid) const +{ + m_reals.write(fid); + m_ints.write(fid); + m_bools.write(fid); +} + + } // namespace scream diff --git a/components/eamxx/src/physics/share/physics_test_data.hpp b/components/eamxx/src/physics/share/physics_test_data.hpp index 099480b4439..ddd7d77fa86 100644 --- a/components/eamxx/src/physics/share/physics_test_data.hpp +++ b/components/eamxx/src/physics/share/physics_test_data.hpp @@ -5,6 +5,7 @@ #include "ekat/util/ekat_math_utils.hpp" #include "ekat/ekat_assert.hpp" +#include "ekat/util/ekat_file_utils.hpp" #include #include @@ -68,34 +69,81 @@ struct SHOCGridData : public PhysicsTestData { #define PTD_DATA_COPY_CTOR(name, num_args) \ name(const name& rhs) : name(PTD_ONES(num_args)) { *this = rhs; } -#define PTD_ASS0( ) ((void) (0)) -#define PTD_ASS1(a ) a = rhs.a -#define PTD_ASS2(a, b ) PTD_ASS1(a) ; b = rhs.b -#define PTD_ASS3(a, b, c ) PTD_ASS2(a, b) ; c = rhs.c -#define PTD_ASS4(a, b, c, d ) PTD_ASS3(a, b, c) ; d = rhs.d -#define PTD_ASS5(a, b, c, d, e ) PTD_ASS4(a, b, c, d) ; e = rhs.e -#define PTD_ASS6(a, b, c, d, e, f ) PTD_ASS5(a, b, c, d, e) ; f = rhs.f -#define PTD_ASS7(a, b, c, d, e, f, g ) PTD_ASS6(a, b, c, d, e, f) ; g = rhs.g -#define PTD_ASS8(a, b, c, d, e, f, g, h ) PTD_ASS7(a, b, c, d, e, f, g) ; h = rhs.h -#define PTD_ASS9(a, b, c, d, e, f, g, h, i ) PTD_ASS8(a, b, c, d, e, f, g, h) ; i = rhs.i -#define PTD_ASS10(a, b, c, d, e, f, g, h, i, j ) PTD_ASS9(a, b, c, d, e, f, g, h, i) ; j = rhs.j -#define PTD_ASS11(a, b, c, d, e, f, g, h, i, j, k ) PTD_ASS10(a, b, c, d, e, f, g, h, i, j) ; k = rhs.k -#define PTD_ASS12(a, b, c, d, e, f, g, h, i, j, k, l ) PTD_ASS11(a, b, c, d, e, f, g, h, i, j, k) ; l = rhs.l -#define PTD_ASS13(a, b, c, d, e, f, g, h, i, j, k, l, m ) PTD_ASS12(a, b, c, d, e, f, g, h, i, j, k, l) ; m = rhs.m -#define PTD_ASS14(a, b, c, d, e, f, g, h, i, j, k, l, m, n ) PTD_ASS13(a, b, c, d, e, f, g, h, i, j, k, l, m) ; n = rhs.n -#define PTD_ASS15(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o ) PTD_ASS14(a, b, c, d, e, f, g, h, i, j, k, l, m, n) ; o = rhs.o -#define PTD_ASS16(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p ) PTD_ASS15(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o) ; p = rhs.p -#define PTD_ASS17(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q ) PTD_ASS16(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p) ; q = rhs.q -#define PTD_ASS18(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r ) PTD_ASS17(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q) ; r = rhs.r -#define PTD_ASS19(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s ) PTD_ASS18(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r) ; s = rhs.s -#define PTD_ASS20(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t ) PTD_ASS19(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s) ; t = rhs.t +#define PTD_ASS0() ((void) (0)) +#define PTD_ASS1(first) first = rhs.first; PTD_ASS0() +#define PTD_ASS2(first, ...) first = rhs.first; PTD_ASS1(__VA_ARGS__) +#define PTD_ASS3(first, ...) first = rhs.first; PTD_ASS2(__VA_ARGS__) +#define PTD_ASS4(first, ...) first = rhs.first; PTD_ASS3(__VA_ARGS__) +#define PTD_ASS5(first, ...) first = rhs.first; PTD_ASS4(__VA_ARGS__) +#define PTD_ASS6(first, ...) first = rhs.first; PTD_ASS5(__VA_ARGS__) +#define PTD_ASS7(first, ...) first = rhs.first; PTD_ASS6(__VA_ARGS__) +#define PTD_ASS8(first, ...) first = rhs.first; PTD_ASS7(__VA_ARGS__) +#define PTD_ASS9(first, ...) first = rhs.first; PTD_ASS8(__VA_ARGS__) +#define PTD_ASS10(first, ...) first = rhs.first; PTD_ASS9(__VA_ARGS__) +#define PTD_ASS11(first, ...) first = rhs.first; PTD_ASS10(__VA_ARGS__) +#define PTD_ASS12(first, ...) first = rhs.first; PTD_ASS11(__VA_ARGS__) +#define PTD_ASS13(first, ...) first = rhs.first; PTD_ASS12(__VA_ARGS__) +#define PTD_ASS14(first, ...) first = rhs.first; PTD_ASS13(__VA_ARGS__) +#define PTD_ASS15(first, ...) first = rhs.first; PTD_ASS14(__VA_ARGS__) +#define PTD_ASS16(first, ...) first = rhs.first; PTD_ASS15(__VA_ARGS__) +#define PTD_ASS17(first, ...) first = rhs.first; PTD_ASS16(__VA_ARGS__) +#define PTD_ASS18(first, ...) first = rhs.first; PTD_ASS17(__VA_ARGS__) +#define PTD_ASS19(first, ...) first = rhs.first; PTD_ASS18(__VA_ARGS__) +#define PTD_ASS20(first, ...) first = rhs.first; PTD_ASS19(__VA_ARGS__) + +#define PTD_RW0(action) ((void) (0)) +#define PTD_RW1(action, first) ekat::action(&first, 1, fid); PTD_RW0(action) +#define PTD_RW2(action, first, ...) ekat::action(&first, 1, fid); PTD_RW1(action, __VA_ARGS__) +#define PTD_RW3(action, first, ...) ekat::action(&first, 1, fid); PTD_RW2(action, __VA_ARGS__) +#define PTD_RW4(action, first, ...) ekat::action(&first, 1, fid); PTD_RW3(action, __VA_ARGS__) +#define PTD_RW5(action, first, ...) ekat::action(&first, 1, fid); PTD_RW4(action, __VA_ARGS__) +#define PTD_RW6(action, first, ...) ekat::action(&first, 1, fid); PTD_RW5(action, __VA_ARGS__) +#define PTD_RW7(action, first, ...) ekat::action(&first, 1, fid); PTD_RW6(action, __VA_ARGS__) +#define PTD_RW8(action, first, ...) ekat::action(&first, 1, fid); PTD_RW7(action, __VA_ARGS__) +#define PTD_RW9(action, first, ...) ekat::action(&first, 1, fid); PTD_RW8(action, __VA_ARGS__) +#define PTD_RW10(action, first, ...) ekat::action(&first, 1, fid); PTD_RW9(action, __VA_ARGS__) +#define PTD_RW11(action, first, ...) ekat::action(&first, 1, fid); PTD_RW10(action, __VA_ARGS__) +#define PTD_RW12(action, first, ...) ekat::action(&first, 1, fid); PTD_RW11(action, __VA_ARGS__) +#define PTD_RW13(action, first, ...) ekat::action(&first, 1, fid); PTD_RW12(action, __VA_ARGS__) +#define PTD_RW14(action, first, ...) ekat::action(&first, 1, fid); PTD_RW13(action, __VA_ARGS__) +#define PTD_RW15(action, first, ...) ekat::action(&first, 1, fid); PTD_RW14(action, __VA_ARGS__) +#define PTD_RW16(action, first, ...) ekat::action(&first, 1, fid); PTD_RW15(action, __VA_ARGS__) +#define PTD_RW17(action, first, ...) ekat::action(&first, 1, fid); PTD_RW16(action, __VA_ARGS__) +#define PTD_RW18(action, first, ...) ekat::action(&first, 1, fid); PTD_RW17(action, __VA_ARGS__) +#define PTD_RW19(action, first, ...) ekat::action(&first, 1, fid); PTD_RW18(action, __VA_ARGS__) +#define PTD_RW20(action, first, ...) ekat::action(&first, 1, fid); PTD_RW19(action, __VA_ARGS__) +#define PTD_RW21(action, first, ...) ekat::action(&first, 1, fid); PTD_RW20(action, __VA_ARGS__) +#define PTD_RW22(action, first, ...) ekat::action(&first, 1, fid); PTD_RW21(action, __VA_ARGS__) +#define PTD_RW23(action, first, ...) ekat::action(&first, 1, fid); PTD_RW22(action, __VA_ARGS__) +#define PTD_RW24(action, first, ...) ekat::action(&first, 1, fid); PTD_RW23(action, __VA_ARGS__) +#define PTD_RW25(action, first, ...) ekat::action(&first, 1, fid); PTD_RW24(action, __VA_ARGS__) +#define PTD_RW26(action, first, ...) ekat::action(&first, 1, fid); PTD_RW25(action, __VA_ARGS__) +#define PTD_RW27(action, first, ...) ekat::action(&first, 1, fid); PTD_RW26(action, __VA_ARGS__) +#define PTD_RW28(action, first, ...) ekat::action(&first, 1, fid); PTD_RW27(action, __VA_ARGS__) +#define PTD_RW29(action, first, ...) ekat::action(&first, 1, fid); PTD_RW28(action, __VA_ARGS__) +#define PTD_RW30(action, first, ...) ekat::action(&first, 1, fid); PTD_RW29(action, __VA_ARGS__) +#define PTD_RW31(action, first, ...) ekat::action(&first, 1, fid); PTD_RW30(action, __VA_ARGS__) #define PTD_ASSIGN_OP(name, num_scalars, ...) \ name& operator=(const name& rhs) { PTD_ASS##num_scalars(__VA_ARGS__); assignment_impl(rhs); return *this; } +#define PTD_RW_SCALARS(num_scalars, ...) \ + void read_scalars(const ekat::FILEPtr& fid) { EKAT_REQUIRE_MSG(fid, "Tried to read from missing file. You may have forgotten to generate baselines for some BFB unit tests"); PTD_RW##num_scalars(read, __VA_ARGS__); } \ + void write_scalars(const ekat::FILEPtr& fid) const { PTD_RW##num_scalars(write, __VA_ARGS__); } + +#define PTD_RW_SCALARS_ONLY(num_scalars, ...) \ + void read(const ekat::FILEPtr& fid) { EKAT_REQUIRE_MSG(fid, "Tried to read from missing file. You may have forgotten to generate baselines for some BFB unit tests"); PTD_RW##num_scalars(read, __VA_ARGS__); } \ + void write(const ekat::FILEPtr& fid) const { PTD_RW##num_scalars(write, __VA_ARGS__); } + +#define PTD_RW() \ + void read(const ekat::FILEPtr& fid) { read_scalars(fid); PhysicsTestData::read(fid); } \ + void write(const ekat::FILEPtr& fid) const { write_scalars(fid); PhysicsTestData::write(fid); } + #define PTD_STD_DEF(name, num_scalars, ...) \ PTD_DATA_COPY_CTOR(name, num_scalars); \ - PTD_ASSIGN_OP(name, num_scalars, __VA_ARGS__) + PTD_ASSIGN_OP(name, num_scalars, __VA_ARGS__) \ + PTD_RW() \ + PTD_RW_SCALARS(num_scalars, __VA_ARGS__) namespace scream { @@ -241,6 +289,16 @@ class PhysicsTestData m_data = new_data; } + void read(const ekat::FILEPtr& fid) + { + ekat::read(m_data.data(), m_data.size(), fid); + } + + void write(const ekat::FILEPtr& fid) const + { + ekat::write(m_data.data(), m_data.size(), fid); + } + std::vector > m_dims_list; // list of dims, one per unique set of dims std::vector > m_members_list; // list of member pointers, same outer index space as m_dims_list std::vector m_data; // the member data in a flat vector @@ -336,6 +394,10 @@ class PhysicsTestData } } + void read(const ekat::FILEPtr& fid); + + void write(const ekat::FILEPtr& fid) const; + protected: PhysicsTestData& assignment_impl(const PhysicsTestData& rhs); diff --git a/components/eamxx/src/physics/shoc/tests/shoc_run_and_cmp.cpp b/components/eamxx/src/physics/shoc/tests/shoc_run_and_cmp.cpp index a5198d95980..94a58b4de9c 100644 --- a/components/eamxx/src/physics/shoc/tests/shoc_run_and_cmp.cpp +++ b/components/eamxx/src/physics/shoc/tests/shoc_run_and_cmp.cpp @@ -271,42 +271,17 @@ int main (int argc, char** argv) { generate = true; } } - if (std::string(argv[i])=="--ekat-kokkos-device") { - expect_another_arg(i, argc); - ++i; - device = argv[i]; + if (std::string(argv[i])=="--kokkos-device-id=") { + auto tokens = ekat::split(argv[i],"="); + device = tokens[1]; } } // Decorate baseline name with precision. baseline_fn += std::to_string(sizeof(scream::Real)); - std::vector args; - for (int i=0; i" was specified, add kokkos - // initialization flag to argv - // Create it outside the if, so its c_str pointer survives - std::string dev_arg; - if (device!="") { - auto is_int = [] (const std::string& s)->bool { - std::istringstream is(s); - int d; - is >> d; - return !is.fail() && is.eof(); - }; - - EKAT_REQUIRE_MSG (is_int(device), "Error! Invalid device specification.\n"); - - if (std::stoi(device) != -1) { - dev_arg = "--kokkos-device-id=" + device; - args.push_back(const_cast(dev_arg.c_str())); - } - } - - scream::initialize_scream_session(args.size(), args.data()); { + scream::initialize_scream_session(argc, argv); + { Baseline bln(nsteps, static_cast(dt), ncol, nlev, num_qtracers, nadv, repeat); if (generate) { std::cout << "Generating to " << baseline_fn << "\n"; @@ -315,7 +290,8 @@ int main (int argc, char** argv) { printf("Comparing with %s at tol %1.1e\n", baseline_fn.c_str(), tol); nerr += bln.run_and_cmp(baseline_fn, tol, use_fortran); } - } scream::finalize_scream_session(); + } + scream::finalize_scream_session(); return nerr != 0 ? 1 : 0; } diff --git a/components/eamxx/src/share/atm_process/IOPDataManager.cpp b/components/eamxx/src/share/atm_process/IOPDataManager.cpp index 8f14b2bdb6a..2e2c3552d8e 100644 --- a/components/eamxx/src/share/atm_process/IOPDataManager.cpp +++ b/components/eamxx/src/share/atm_process/IOPDataManager.cpp @@ -270,7 +270,7 @@ initialize_iop_file(const util::TimeStamp& run_t0, scorpio::read_var(iop_file,"lon",&iop_file_lon); const Real rel_lat_err = std::fabs(iop_file_lat - m_params.get("target_latitude"))/ - std::max(m_params.get("target_latitude"),(Real)0.1); + std::max(std::fabs(m_params.get("target_latitude")),(Real)0.1); const Real rel_lon_err = std::fabs(std::fmod(iop_file_lon + 360.0, 360.0)-m_params.get("target_longitude"))/ std::max(m_params.get("target_longitude"),(Real)0.1); EKAT_REQUIRE_MSG(rel_lat_err < std::numeric_limits::epsilon(), diff --git a/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp b/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp index c930013f839..b083dfcfa48 100644 --- a/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp +++ b/components/eamxx/src/share/grid/mesh_free_grids_manager.cpp @@ -147,24 +147,30 @@ add_geo_data (const nonconstgrid_ptr_type& grid) const if (geo_data_source=="CREATE_EMPTY_DATA") { using namespace ShortFieldTagsNames; FieldLayout layout_mid ({LEV},{grid->get_num_vertical_levels()}); + FieldLayout layout_int ({ILEV},{grid->get_num_vertical_levels()+1}); const auto units = ekat::units::Units::nondimensional(); auto lat = grid->create_geometry_data("lat" , grid->get_2d_scalar_layout(), units); auto lon = grid->create_geometry_data("lon" , grid->get_2d_scalar_layout(), units); auto hyam = grid->create_geometry_data("hyam" , layout_mid, units); auto hybm = grid->create_geometry_data("hybm" , layout_mid, units); + auto hyai = grid->create_geometry_data("hyai" , layout_int, units); + auto hybi = grid->create_geometry_data("hybi" , layout_int, units); auto lev = grid->create_geometry_data("lev" , layout_mid, units); + auto ilev = grid->create_geometry_data("ilev" , layout_int, units); lat.deep_copy(ekat::ScalarTraits::invalid()); lon.deep_copy(ekat::ScalarTraits::invalid()); hyam.deep_copy(ekat::ScalarTraits::invalid()); hybm.deep_copy(ekat::ScalarTraits::invalid()); lev.deep_copy(ekat::ScalarTraits::invalid()); + ilev.deep_copy(ekat::ScalarTraits::invalid()); lat.sync_to_dev(); lon.sync_to_dev(); hyam.sync_to_dev(); hybm.sync_to_dev(); lev.sync_to_dev(); + ilev.sync_to_dev(); } else if (geo_data_source=="IC_FILE"){ const auto& filename = m_params.get("ic_filename"); if (scorpio::has_var(filename,"lat") && @@ -173,7 +179,9 @@ add_geo_data (const nonconstgrid_ptr_type& grid) const } if (scorpio::has_var(filename,"hyam") && - scorpio::has_var(filename,"hybm")) { + scorpio::has_var(filename,"hybm") && + scorpio::has_var(filename,"hyai") && + scorpio::has_var(filename,"hybi") ) { load_vertical_coordinates(grid,filename); } } @@ -234,53 +242,71 @@ load_vertical_coordinates (const nonconstgrid_ptr_type& grid, const std::string& using namespace ekat::units; FieldLayout layout_mid ({LEV},{grid->get_num_vertical_levels()}); + FieldLayout layout_int ({ILEV},{grid->get_num_vertical_levels()+1}); Units nondim = Units::nondimensional(); Units mbar (100*Pa,"mb"); auto hyam = grid->create_geometry_data("hyam", layout_mid, nondim); auto hybm = grid->create_geometry_data("hybm", layout_mid, nondim); + auto hyai = grid->create_geometry_data("hyai", layout_int, nondim); + auto hybi = grid->create_geometry_data("hybi", layout_int, nondim); auto lev = grid->create_geometry_data("lev", layout_mid, mbar); + auto ilev = grid->create_geometry_data("ilev", layout_int, mbar); // Create host mirrors for reading in data std::map host_views = { { "hyam", hyam.get_view() }, - { "hybm", hybm.get_view() } + { "hybm", hybm.get_view() }, + { "hyai", hyai.get_view() }, + { "hybi", hybi.get_view() } }; // Store view layouts using namespace ShortFieldTagsNames; std::map layouts = { { "hyam", hyam.get_header().get_identifier().get_layout() }, - { "hybm", hybm.get_header().get_identifier().get_layout() } + { "hybm", hybm.get_header().get_identifier().get_layout() }, + { "hyai", hyai.get_header().get_identifier().get_layout() }, + { "hybi", hybi.get_header().get_identifier().get_layout() } }; // Read hyam/hybm into host views ekat::ParameterList vcoord_reader_pl; vcoord_reader_pl.set("Filename",filename); - vcoord_reader_pl.set>("Field Names",{"hyam","hybm"}); + vcoord_reader_pl.set>("Field Names",{"hyam","hybm","hyai","hybi"}); AtmosphereInput vcoord_reader(vcoord_reader_pl,grid, host_views, layouts); vcoord_reader.read_variables(); vcoord_reader.finalize(); - // Build lev from hyam and hybm + // Build lev and ilev from hyam and hybm, and ilev from hyai and hybi using PC = scream::physics::Constants; const Real ps0 = PC::P0; - auto hya_v = hyam.get_view(); - auto hyb_v = hybm.get_view(); - auto lev_v = lev.get_view(); - for (int ii=0;iiget_num_vertical_levels();ii++) { - lev_v(ii) = 0.01*ps0*(hya_v(ii)+hyb_v(ii)); + auto hyam_v = hyam.get_view(); + auto hybm_v = hybm.get_view(); + auto lev_v = lev.get_view(); + auto hyai_v = hyai.get_view(); + auto hybi_v = hybi.get_view(); + auto ilev_v = ilev.get_view(); + auto num_lev = grid->get_num_vertical_levels(); + for (int ii=0;ii(f,grid)->check(); EKAT_REQUIRE_MSG (nan_check.result==CheckResult::Pass, "ERROR! NaN values detected in " + f.name() + " field.\n" + nan_check.msg); diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index 87fe9af7133..e8f322b85f3 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -1044,9 +1044,15 @@ register_variables(const std::string& filename, // Gather longname (if not already in the io: string attributes) if (str_atts.count("long_name")==0) { - auto longname = m_longnames.get_longname(name); + auto longname = m_default_metadata.get_longname(name); scorpio::set_attribute(filename, name, "long_name", longname); } + + // Gather standard name, CF-Compliant (if not already in the io: string attributes) + if (str_atts.count("standard_name")==0) { + auto standardname = m_default_metadata.get_standardname(name); + scorpio::set_attribute(filename, name, "standard_name", standardname); + } } } // Now register the average count variables diff --git a/components/eamxx/src/share/io/scorpio_output.hpp b/components/eamxx/src/share/io/scorpio_output.hpp index 2cb45f28a32..696e08c9982 100644 --- a/components/eamxx/src/share/io/scorpio_output.hpp +++ b/components/eamxx/src/share/io/scorpio_output.hpp @@ -206,7 +206,7 @@ class AtmosphereOutput std::map> m_diagnostics; std::map> m_diag_depends_on_diags; std::map m_diag_computed; - LongNames m_longnames; + DefaultMetadata m_default_metadata; // Use float, so that if output fp_precision=float, this is a representable value. // Otherwise, you would get an error from Netcdf, like diff --git a/components/eamxx/src/share/io/scream_io_utils.hpp b/components/eamxx/src/share/io/scream_io_utils.hpp index 01bc46e1e60..9eda82e2bbd 100644 --- a/components/eamxx/src/share/io/scream_io_utils.hpp +++ b/components/eamxx/src/share/io/scream_io_utils.hpp @@ -72,7 +72,7 @@ std::string find_filename_in_rpointer ( const OutputAvgType avg_type = OutputAvgType::Instant, const IOControl& control = {}); -struct LongNames { +struct DefaultMetadata { std::string get_longname (const std::string& name) { if (name_2_longname.count(name)>0) { @@ -83,14 +83,97 @@ struct LongNames { } } + std::string get_standardname (const std::string& name) { + if (name_2_standardname.count(name)>0) { + return name_2_standardname.at(name); + } else { + // TODO: Do we want to print a Warning message? I'm not sure if its needed. + return name; + } + } + // Create map of longnames, can be added to as developers see fit. std::map name_2_longname = { - {"lev","hybrid level at midpoints (1000*(A+B))"}, - {"hyai","hybrid A coefficient at layer interfaces"}, + {"lev","hybrid level at midpoints (1000*(A+B))"}, + {"ilev","hybrid level at interfaces (1000*(A+B))"}, + {"hyai","hybrid A coefficient at layer interfaces"}, {"hybi","hybrid B coefficient at layer interfaces"}, {"hyam","hybrid A coefficient at layer midpoints"}, {"hybm","hybrid B coefficient at layer midpoints"} }; + + // Create map of longnames, can be added to as developers see fit. + std::map name_2_standardname = { + {"p_mid" , "air_pressure"}, + {"p_mid_at_cldtop" , "air_pressure_at_cloud_top"}, + {"T_2m" , "air_temperature"}, + {"T_mid" , "air_temperature"}, + {"T_mid_at_cldtop" , "air_temperature_at_cloud_top"}, + {"aero_g_sw" , "asymmetry_factor_of_ambient_aerosol_particles"}, + {"pbl_height" , "atmosphere_boundary_layer_thickness"}, + {"precip_liq_surf_mass" , "atmosphere_mass_content_of_liquid_precipitation"}, + {"cldlow" , "low_type_cloud_area_fraction"}, + {"cldmed" , "medium_type_cloud_area_fraction"}, + {"cldhgh" , "high_type_cloud_area_fraction"}, + {"cldtot" , "cloud_area_fraction"}, + {"cldfrac_tot_at_cldtop" , "cloud_area_fraction"}, + {"cldfrac_tot" , "cloud_area_fraction_in_atmosphere_layer"}, + {"cldfrac_tot_for_analysis" , "cloud_area_fraction_in_atmosphere_layer"}, + {"cldfrac_rad" , "cloud_area_fraction_in_atmosphere_layer"}, + {"qi" , "cloud_ice_mixing_ratio"}, + {"qc" , "cloud_liquid_water_mixing_ratio"}, + {"U" , "eastward_wind"}, + {"eff_radius_qi" , "effective_radius_of_cloud_ice_particles"}, + {"eff_radius_qc" , "effective_radius_of_cloud_liquid_water_particles"}, + {"eff_radius_qc_at_cldtop" , "effective_radius_of_cloud_liquid_water_particles_at_liquid_water_cloud_top"}, + {"eff_radius_qr" , "effective_radius_of_cloud_rain_particles"}, + {"qv" , "humidity_mixing_ratio"}, + {"cldfrac_ice_at_cldtop" , "ice_cloud_area_fraction"}, + {"cldfrac_ice" , "ice_cloud_area_fraction_in_atmosphere_layer"}, + {"omega" , "lagrangian_tendency_of_air_pressure"}, + {"landfrac" , "land_area_fraction"}, + {"latitude" , "latitude"}, + {"cldfrac_liq_at_cldtop" , "liquid_water_cloud_area_fraction"}, + {"cldfrac_liq" , "liquid_water_cloud_area_fraction_in_atmosphere_layer"}, + {"longitude" , "longitude"}, + {"rainfrac" , "mass_fraction_of_liquid_precipitation_in_air"}, + {"V" , "northward_wind"}, + {"nc" , "number_concentration_of_cloud_liquid_water_particles_in_air"}, + {"cdnc_at_cldtop" , "number_concentration_of_cloud_liquid_water_particles_in_air_at_liquid_water_cloud_top"}, + {"ni" , "number_concentration_of_ice_crystals_in_air"}, + {"aero_tau_sw" , "optical_thickness_of_atmosphere_layer_due_to_ambient_aerosol_particles"}, + {"aero_tau_lw" , "optical_thickness_of_atmosphere_layer_due_to_ambient_aerosol_particles"}, + {"aero_ssa_sw" , "single_scattering_albedo_in_air_due_to_ambient_aerosol_particles"}, + {"sunlit" , "sunlit_binary_mask"}, + {"ps" , "surface_air_pressure"}, + {"LW_flux_dn_at_model_bot" , "surface_downwelling_longwave_flux_in_air"}, + {"SW_flux_dn_at_model_bot" , "surface_downwelling_shortwave_flux_in_air"}, + {"SW_clrsky_flux_dn_at_model_bot" , "surface_downwelling_shortwave_flux_in_air_assuming_clear_sky"}, + {"phis" , "surface_geopotential"}, + {"surf_radiative_T" , "surface_temperature"}, + {"surf_sens_flux" , "surface_upward_sensible_heat_flux"}, + {"SW_flux_dn_at_model_top" , "toa_incoming_shortwave_flux"}, + {"LW_flux_up_at_model_top" , "toa_outgoing_longwave_flux"}, + {"LW_clrsky_flux_up_at_model_top" , "toa_outgoing_longwave_flux_assuming_clear_sky"}, + {"surf_evap" , "water_evapotranspiration_flux"}, + {"AtmosphereDensity" , "air_density"}, + {"PotentialTemperature" , "air_potential_temperature"}, + {"SeaLevelPressure" , "air_pressure_at_mean_sea_level"}, + {"IceWaterPath" , "atmosphere_mass_content_of_cloud_ice"}, + {"LiqWaterPath" , "atmosphere_mass_content_of_cloud_liquid_water"}, + {"VapWaterPath" , "atmosphere_mass_content_of_water_vapor"}, + {"AerosolOpticalDepth550nm" , "atmosphere_optical_thickness_due_to_ambient_aerosol_particles"}, + {"Exner" , "dimensionless_exner_function"}, + {"z_mid" , "geopotential_height"}, + {"geopotential_mid" , "geopotential_height"}, + {"RelativeHumidity" , "relative_humidity"}, + {"surface_upward_latent_heat_flux" , "surface_upward_latent_heat_flux"}, + {"LongwaveCloudForcing" , "toa_longwave_cloud_radiative_effect"}, + {"ShortwaveCloudForcing" , "toa_shortwave_cloud_radiative_effect"}, + {"VirtualTemperature" , "virtual_temperature"}, + {"VaporFlux" , "water_evapotranspiration_flux"}, + {"wind_speed" , "wind_speed"} + }; }; diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index fd05ca193b9..050a55936b4 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -402,7 +402,7 @@ void OutputManager::run(const util::TimeStamp& timestamp) // Check if we need to open a new file if (not filespecs.is_open) { filespecs.filename = compute_filename (filespecs,timestamp); - // Register all dims/vars, write geometry data (e.g. lat/lon/hyam/hybm) + // Register all dims/vars, write geometry data (e.g. lat/lon/hyam/hybm/hyai/hybi) setup_file(filespecs,control); } diff --git a/components/eamxx/src/share/util/eamxx_ad_test.cpp b/components/eamxx/src/share/util/eamxx_ad_test.cpp index 0f64bd7c5ab..ba493478878 100644 --- a/components/eamxx/src/share/util/eamxx_ad_test.cpp +++ b/components/eamxx/src/share/util/eamxx_ad_test.cpp @@ -32,7 +32,7 @@ TEST_CASE("scream_ad_test") { // Create a comm ekat::Comm atm_comm (MPI_COMM_WORLD); - // User can prescribe input file name via --ekat-test-params ifile= + // User can prescribe input file name via --args --ifile auto& session = ekat::TestSession::get(); session.params.emplace("ifile","input.yaml"); std::string fname = session.params["ifile"]; diff --git a/components/eamxx/src/share/util/scream_setup_random_test.hpp b/components/eamxx/src/share/util/scream_setup_random_test.hpp index 419bd4ee64e..09f3f3099d8 100644 --- a/components/eamxx/src/share/util/scream_setup_random_test.hpp +++ b/components/eamxx/src/share/util/scream_setup_random_test.hpp @@ -41,9 +41,13 @@ inline int get_random_test_seed(const ekat::Comm* comm=nullptr) } template -Engine setup_random_test(const ekat::Comm* comm=nullptr) +Engine setup_random_test(const ekat::Comm* comm=nullptr, int* return_seed=nullptr) { - return Engine (get_random_test_seed(comm)); + int seed = get_random_test_seed(comm); + if (return_seed != nullptr) { + *return_seed = seed; + } + return Engine (seed); } template diff --git a/components/eamxx/tests/multi-process/dynamics_physics/model_restart/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/CMakeLists.txt index 5645a9383a7..a0288b454b2 100644 --- a/components/eamxx/tests/multi-process/dynamics_physics/model_restart/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/CMakeLists.txt @@ -24,20 +24,20 @@ set (CASE_TN 2023-01-01-00060) # Create the baseline (run all 6 timsteps in a single run) CreateUnitTestFromExec(model_baseline model_restart - EXE_ARGS "--use-colour no --ekat-test-params ifile=input_baseline.yaml" + EXE_ARGS "--args -ifile=input_baseline.yaml" MPI_RANKS ${SCREAM_TEST_MAX_RANKS} FIXTURES_SETUP baseline_run) # Start a simulation, but only run half of the time steps CreateUnitTestFromExec(model_initial model_restart - EXE_ARGS "--use-colour no --ekat-test-params ifile=input_initial.yaml" + EXE_ARGS "--args -ifile=input_initial.yaml" MPI_RANKS ${SCREAM_TEST_MAX_RANKS} FIXTURES_SETUP initial_run PROPERTIES RESOURCE_LOCK rpointer_file) # Restart the simulation, and run the second half of the time steps CreateUnitTestFromExec(model_restart model_restart - EXE_ARGS "--use-colour no --ekat-test-params ifile=input_restarted.yaml" + EXE_ARGS "--args -ifile=input_restarted.yaml" MPI_RANKS ${SCREAM_TEST_MAX_RANKS} FIXTURES_REQUIRED initial_run FIXTURES_SETUP restarted_run diff --git a/components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/CMakeLists.txt index 71fe2a5f43c..88c5f1022f6 100644 --- a/components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/CMakeLists.txt @@ -23,7 +23,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output_tend.yaml ${CMAKE_CURRENT_BINARY_DIR}/output_tend_subcycled.yaml) CreateUnitTestFromExec (shoc_p3_subcycled shoc_p3 - EXE_ARGS "--use-colour no --ekat-test-params ifile=input_subcycled.yaml" + EXE_ARGS "--args -ifile=input_subcycled.yaml" FIXTURES_SETUP shoc_p3_subcycled) # Run a test without subcycling and more steps @@ -38,7 +38,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output_tend.yaml ${CMAKE_CURRENT_BINARY_DIR}/output_tend_monolithic.yaml) CreateUnitTestFromExec (shoc_p3_monolithic shoc_p3 - EXE_ARGS "--use-colour no --ekat-test-params ifile=input_monolithic.yaml" + EXE_ARGS "--args -ifile=input_monolithic.yaml" FIXTURES_SETUP shoc_p3_monolithic) # Finally, compare output of the two tests diff --git a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml index 55c62d69aa2..48045295050 100644 --- a/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/mam/mam4_srf_online_emiss_mam4_constituent_fluxes/input.yaml @@ -23,7 +23,9 @@ atmosphere_processes: srf_emis_specifier_for_pom_a4: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc - + + soil_erodibility_file: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/dst_ne2np4_c20241028.nc + marine_organics_file: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/monthly_macromolecules_0.1deg_bilinear_year01_merge_ne2np4_c20241030.nc grids_manager: Type: Mesh Free geo_data_source: IC_FILE @@ -40,6 +42,13 @@ initial_conditions: topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} pbl_height: 0.0 + dstflx : 0.0 + ocnfrac: 0.10000000000000000E+001 + sst: 0.30178553874977507E+003 + cldfrac_tot: 0.138584624960092 + horiz_winds: [-0.24988988196194634E+000, -0.23959782871450760E+000] + constituent_fluxes: 0.0 + # The parameters for I/O control Scorpio: diff --git a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/CMakeLists.txt index 62c4b6d2266..16568dc55b3 100644 --- a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/CMakeLists.txt @@ -27,7 +27,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output_remapped.yaml ${CMAKE_CURRENT_BINARY_DIR}/output_source_data_remapped.yaml) CreateUnitTestFromExec (shoc_p3_source shoc_p3_nudging - EXE_ARGS "--use-colour no --ekat-test-params ifile=input_source_data.yaml" + EXE_ARGS "--args -ifile=input_source_data.yaml" FIXTURES_SETUP shoc_p3_source_data FIXTURES_REQUIRED shoc_p3_create_vertical_remap_and_weights_file) @@ -41,7 +41,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input_nudging.yaml configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml ${CMAKE_CURRENT_BINARY_DIR}/output_nudged.yaml) CreateUnitTestFromExec (shoc_p3_nudged shoc_p3_nudging - EXE_ARGS "--use-colour no --ekat-test-params ifile=input_nudging.yaml" + EXE_ARGS "--args -ifile=input_nudging.yaml" FIXTURES_REQUIRED shoc_p3_source_data) # Run a test with nudging turned on using remapped source data for nudging: @@ -54,7 +54,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input_nudging.yaml configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml ${CMAKE_CURRENT_BINARY_DIR}/output_nudged_remapped.yaml) CreateUnitTestFromExec (shoc_p3_nudged_remapped shoc_p3_nudging - EXE_ARGS "--use-colour no --ekat-test-params ifile=input_nudging_remapped.yaml" + EXE_ARGS "--args -ifile=input_nudging_remapped.yaml" FIXTURES_REQUIRED shoc_p3_source_data) # Run a test with nudging using data read in glob pattern and skip vertical interpolation: @@ -67,6 +67,6 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input_nudging_glob_novert.yaml configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml ${CMAKE_CURRENT_BINARY_DIR}/output_nudged_glob_novert.yaml) CreateUnitTestFromExec (shoc_p3_nudging_glob_novert shoc_p3_nudging - EXE_ARGS "--use-colour no --ekat-test-params ifile=input_nudging_glob_novert.yaml" + EXE_ARGS "--args -ifile=input_nudging_glob_novert.yaml" FIXTURES_REQUIRED shoc_p3_source_data) diff --git a/components/eamxx/tests/single-process/mam/aero_microphys/CMakeLists.txt b/components/eamxx/tests/single-process/mam/aero_microphys/CMakeLists.txt index e1ba85a685f..6f6f8b364b8 100644 --- a/components/eamxx/tests/single-process/mam/aero_microphys/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/aero_microphys/CMakeLists.txt @@ -39,6 +39,7 @@ set (TEST_INPUT_FILES scream/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_num_a2_elev_ne2np4_2010_clim_c20240823.nc scream/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_num_a4_elev_ne2np4_2010_clim_c20240823.nc scream/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_soag_elev_ne2np4_2010_clim_c20240823.nc + scream/mam4xx/drydep/season_wes.nc ) foreach (file IN ITEMS ${TEST_INPUT_FILES}) GetInputFile(${file}) diff --git a/components/eamxx/tests/single-process/mam/aero_microphys/input.yaml b/components/eamxx/tests/single-process/mam/aero_microphys/input.yaml index fa4538bb75a..7dd28a87f47 100644 --- a/components/eamxx/tests/single-process/mam/aero_microphys/input.yaml +++ b/components/eamxx/tests/single-process/mam/aero_microphys/input.yaml @@ -27,17 +27,17 @@ atmosphere_processes: mam4_chlorine_loading_ymd : 20100101 mam4_rsf_file : ${SCREAM_DATA_DIR}/mam4xx/photolysis/RSF_GT200nm_v3.0_c080811.nc mam4_xs_long_file : ${SCREAM_DATA_DIR}/mam4xx/photolysis/temp_prs_GT200nm_JPL10_c130206.nc - verti_emiss_ymd : 20100101 - mam4_so2_verti_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_so2_elev_ne2np4_2010_clim_c20240726.nc - mam4_so4_a1_verti_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated//cmip6_mam4_so4_a1_elev_ne2np4_2010_clim_c20240823.nc - mam4_so4_a2_verti_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated//cmip6_mam4_so4_a2_elev_ne2np4_2010_clim_c20240823.nc - mam4_pom_a4_verti_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated//cmip6_mam4_pom_a4_elev_ne2np4_2010_clim_c20240823.nc - mam4_bc_a4_verti_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_bc_a4_elev_ne2np4_2010_clim_c20240823.nc - mam4_num_a1_verti_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated//cmip6_mam4_num_a1_elev_ne2np4_2010_clim_c20240823.nc - mam4_num_a2_verti_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated//cmip6_mam4_num_a2_elev_ne2np4_2010_clim_c20240823.nc - mam4_num_a4_verti_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated//cmip6_mam4_num_a4_elev_ne2np4_2010_clim_c20240823.nc - mam4_soag_verti_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated//cmip6_mam4_soag_elev_ne2np4_2010_clim_c20240823.nc - + elevated_emiss_ymd : 20100101 + mam4_so2_elevated_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_so2_elev_ne2np4_2010_clim_c20240726.nc + mam4_so4_a1_elevated_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_so4_a1_elev_ne2np4_2010_clim_c20240823.nc + mam4_so4_a2_elevated_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_so4_a2_elev_ne2np4_2010_clim_c20240823.nc + mam4_pom_a4_elevated_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_pom_a4_elev_ne2np4_2010_clim_c20240823.nc + mam4_bc_a4_elevated_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_bc_a4_elev_ne2np4_2010_clim_c20240823.nc + mam4_num_a1_elevated_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_num_a1_elev_ne2np4_2010_clim_c20240823.nc + mam4_num_a2_elevated_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_num_a2_elev_ne2np4_2010_clim_c20240823.nc + mam4_num_a4_elevated_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_num_a4_elev_ne2np4_2010_clim_c20240823.nc + mam4_soag_elevated_emiss_file_name : ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/elevated/cmip6_mam4_soag_elev_ne2np4_2010_clim_c20240823.nc + mam4_season_wes_file : ${SCREAM_DATA_DIR}/mam4xx/drydep/season_wes.nc grids_manager: Type: Mesh Free geo_data_source: IC_FILE @@ -58,7 +58,8 @@ initial_conditions: dgnum: [1.246662106183775E-007, 4.081134799487888E-008, 1.103139143795796E-006, 1.000000011686097E-007] dgnumwet: [2.367209731605067E-007, 6.780643470563889E-008, 3.028011448344027E-006, 1.000000096285154E-007] wetdens: [1038.67760516297, 1046.20002003441, 1031.74623165457, 1086.79731859184] - + nevapr: 0.0 + precip_total_tend: 0.0 # The parameters for I/O control Scorpio: output_yaml_files: ["output.yaml"] diff --git a/components/eamxx/tests/single-process/mam/aero_microphys/output.yaml b/components/eamxx/tests/single-process/mam/aero_microphys/output.yaml index 388781fb8f0..5af90cac547 100644 --- a/components/eamxx/tests/single-process/mam/aero_microphys/output.yaml +++ b/components/eamxx/tests/single-process/mam/aero_microphys/output.yaml @@ -6,6 +6,62 @@ Fields: Physics: Field Names: - T_mid + - O3 + - H2O2 + - H2SO4 + - SO2 + - DMS + - SOAG + - bc_a1 + - bc_a3 + - bc_a4 + - dst_a1 + - dst_a3 + - so4_a1 + - so4_a2 + - so4_a3 + - pom_a1 + - pom_a3 + - pom_a4 + - soa_a1 + - soa_a2 + - soa_a3 + - nacl_a1 + - nacl_a2 + - nacl_a3 + - mom_a1 + - mom_a2 + - mom_a3 + - mom_a4 + - num_a1 + - num_a2 + - num_a3 + - num_a4 + - bc_c1 + - bc_c3 + - bc_c4 + - dst_c1 + - dst_c3 + - so4_c1 + - so4_c2 + - so4_c3 + - pom_c1 + - pom_c3 + - pom_c4 + - soa_c1 + - soa_c2 + - soa_c3 + - nacl_c1 + - nacl_c2 + - nacl_c3 + - mom_c1 + - mom_c2 + - mom_c3 + - mom_c4 + - num_c1 + - num_c2 + - num_c3 + - num_c4 # To save these fields make sure to turn on # OUTPUT_TRACER_FIELDS #- oxi_fields diff --git a/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt b/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt index c8e690b929f..65f377b4db1 100644 --- a/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt +++ b/components/eamxx/tests/single-process/mam/emissions/CMakeLists.txt @@ -36,6 +36,8 @@ set (TEST_INPUT_FILES scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_pom_a4_surf_ne2np4_2010_clim_c20240726.nc scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc scream/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc + scream/mam4xx/emissions/ne2np4/dst_ne2np4_c20241028.nc + scream/mam4xx/emissions/ne2np4/monthly_macromolecules_0.1deg_bilinear_year01_merge_ne2np4_c20241030.nc ) foreach (file IN ITEMS ${TEST_INPUT_FILES}) GetInputFile(${file}) diff --git a/components/eamxx/tests/single-process/mam/emissions/input.yaml b/components/eamxx/tests/single-process/mam/emissions/input.yaml index e7af45178aa..0819562d959 100644 --- a/components/eamxx/tests/single-process/mam/emissions/input.yaml +++ b/components/eamxx/tests/single-process/mam/emissions/input.yaml @@ -23,6 +23,8 @@ atmosphere_processes: srf_emis_specifier_for_so4_a1: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a1_surf_ne2np4_2010_clim_c20240726.nc srf_emis_specifier_for_so4_a2: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/surface/cmip6_mam4_so4_a2_surf_ne2np4_2010_clim_c20240726.nc + soil_erodibility_file: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/dst_ne2np4_c20241028.nc + marine_organics_file: ${SCREAM_DATA_DIR}/mam4xx/emissions/ne2np4/monthly_macromolecules_0.1deg_bilinear_year01_merge_ne2np4_c20241030.nc grids_manager: Type: Mesh Free geo_data_source: IC_FILE @@ -38,11 +40,15 @@ initial_conditions: Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} phis : 1.0 - #These should come from the input file - - #we should get the following variables from other processes + + # we should get the following variables from other processes pbl_height : 1.0 - + dstflx : 0.0 + ocnfrac: 0.10000000000000000E+001 + sst: 0.30178553874977507E+003 + cldfrac_tot: 0.138584624960092 + horiz_winds: [-0.24988988196194634E+000, -0.23959782871450760E+000] + constituent_fluxes: 0.0 # The parameters for I/O control Scorpio: output_yaml_files: ["output.yaml"] diff --git a/components/eamxx/tests/single-process/rrtmgp/CMakeLists.txt b/components/eamxx/tests/single-process/rrtmgp/CMakeLists.txt index 792d3946a4c..b16c2e73288 100644 --- a/components/eamxx/tests/single-process/rrtmgp/CMakeLists.txt +++ b/components/eamxx/tests/single-process/rrtmgp/CMakeLists.txt @@ -13,7 +13,7 @@ if (SCREAM_ENABLE_BASELINE_TESTS AND NOT SCREAM_ONLY_GENERATE_BASELINES) CreateUnitTest(${TEST_BASE_NAME}_unit rrtmgp_standalone_unit.cpp LABELS rrtmgp physics driver LIBS scream_rrtmgp rrtmgp scream_control yakl diagnostics rrtmgp_test_utils - EXE_ARGS "--ekat-test-params rrtmgp_inputfile=${SCREAM_DATA_DIR}/init/rrtmgp-allsky.nc,rrtmgp_baseline=${SCREAM_BASELINES_DIR}/data/rrtmgp-allsky-baseline.nc" + EXE_ARGS "--args --inputfile ${SCREAM_DATA_DIR}/init/rrtmgp-allsky.nc --baseline ${SCREAM_BASELINES_DIR}/data/rrtmgp-allsky-baseline.nc" ) endif() @@ -39,7 +39,7 @@ CreateUnitTestFromExec( ${TEST_BASE_NAME}_not_chunked ${TEST_BASE_NAME} LABELS rrtmgp physics driver MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} - EXE_ARGS "--ekat-test-params inputfile=input_not_chunked.yaml" + EXE_ARGS "--args -inputfile input_not_chunked.yaml" FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME}_not_chunked ) @@ -70,7 +70,7 @@ CreateUnitTestFromExec( ${TEST_BASE_NAME}_chunked ${TEST_BASE_NAME} LABELS rrtmgp physics driver MPI_RANKS ${TEST_RANK_END} - EXE_ARGS "--ekat-test-params inputfile=input_chunked.yaml" + EXE_ARGS "--args --inputfile=input_chunked.yaml" FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME}_chunked PROPERTIES PASS_REGULAR_EXPRESSION "(beg.end: 0, ${COL_CHUNK_SIZE})" ) diff --git a/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp b/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp index dc1ece59b28..5be27ce2b1e 100644 --- a/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp +++ b/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp @@ -49,8 +49,8 @@ using PC = scream::physics::Constants; #ifdef RRTMGP_ENABLE_YAKL TEST_CASE("rrtmgp_scream_standalone", "") { // Get baseline name (needs to be passed as an arg) - std::string inputfile = ekat::TestSession::get().params.at("rrtmgp_inputfile"); - std::string baseline = ekat::TestSession::get().params.at("rrtmgp_baseline"); + std::string inputfile = ekat::TestSession::get().params.at("inputfile"); + std::string baseline = ekat::TestSession::get().params.at("baseline"); // Check if files exists REQUIRE(rrtmgpTest::file_exists(inputfile.c_str())); diff --git a/components/elm/bld/namelist_files/namelist_defaults.xml b/components/elm/bld/namelist_files/namelist_defaults.xml index 80fb0aa5bda..3e3ce90223f 100644 --- a/components/elm/bld/namelist_files/namelist_defaults.xml +++ b/components/elm/bld/namelist_files/namelist_defaults.xml @@ -134,7 +134,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case). -lnd/clm2/paramdata/fates_params_api.36.0.0_12pft_c240517.nc +lnd/clm2/paramdata/fates_params_api.36.1.0_14pft_c241003.nc lnd/clm2/paramdata/CNP_parameters_c131108.nc @@ -232,6 +232,12 @@ ic_tod="0" sim_year="1850" glc_nec="0" use_crop=".false." >lnd/clm2/initdata_map ic_tod="0" sim_year="2000" glc_nec="0" use_crop=".false." >lnd/clm2/initdata_map/clmi.ICRUCLM45SP.2000-01-01.ne240np4_tx0.1v2.162cd32_simyr2000_c170910.nc + + +lnd/clm2/initdata/elmi.v3-SORRM.ne30pg2_r05_SOwISC12to30E3r3.1850-01-01-00000.c20240923.nc + + lnd/clm2/surfdata_map/surfdata_ne0_CAx32.pg2_simyr2010_c220621.nc @@ -335,7 +341,7 @@ lnd/clm2/surfdata_map/surfdata_64x128_simyr2000_c170111.nc lnd/clm2/surfdata_map/surfdata_nldas2_simyr2000_c181207.nc - + lnd/clm2/surfdata_map/surfdata_0.125x0.125_simyr2000_c190730.nc lnd/clm2/surfdata_map/surfdata_0.9x1.25_simyr2000_c180404.nc @@ -370,6 +376,8 @@ lnd/clm2/surfdata_map/surfdata_ne256np4_simyr2010_c220518.nc lnd/clm2/surfdata_map/surfdata_ne256np4_simyr2010_c220518.nc lnd/clm2/surfdata_map/surfdata_0.5x0.5_simyr2000_c200624.nc + +lnd/clm2/surfdata_map/surfdata_r025_simyr1980_c240920.nc lnd/clm2/surfdata_map/surfdata_0.125x0.125_simyr2000_c190730.nc @@ -406,7 +414,7 @@ lnd/clm2/surfdata_map/surfdata_360x720cru_simyr1850_c180216.nc lnd/clm2/surfdata_map/surfdata_48x96_simyr1850_c130927.nc - + lnd/clm2/surfdata_map/surfdata_0.125x0.125_simyr1850_c190730.nc lnd/clm2/surfdata_map/surfdata_0.9x1.25_simyr1850_c180306.nc @@ -623,6 +631,7 @@ this mask will have smb calculated over the entire global land surface lnd/clm2/griddata/glcmaskdata_0.9x1.25_60S.nc lnd/clm2/griddata/glcmaskdata_0.5x0.5_GIS_AIS.nc lnd/clm2/griddata/glcmaskdata_0.5x0.5_GIS_AIS.nc +lnd/clm2/griddata/glcmaskdata_r025_GIS_AIS.20240910.nc lnd/clm2/griddata/glcmaskdata_r0125_GIS_AIS.210407.nc lnd/clm2/griddata/glcmaskdata_ne30pg2_GIS_AIS.210407.nc @@ -633,6 +642,7 @@ this mask will have smb calculated over the entire global land surface lnd/clm2/griddata/topodata_48x96_USGS_070110.nc lnd/clm2/surfdata_map/surfdata_0.5x0.5_simyr1850_c200609.nc lnd/clm2/surfdata_map/surfdata_0.5x0.5_simyr1850_c211019.nc +lnd/clm2/surfdata_map/surfdata_0.25x0.25_simyr1850_c240125.nc lnd/clm2/surfdata_map/surfdata_0.125x0.125_simyr1850_c190730.nc lnd/clm2/surfdata_map/surfdata_ne30pg2_simyr1850_c210402.nc @@ -1696,13 +1706,6 @@ this mask will have smb calculated over the entire global land surface lnd/clm2/mappingdata/maps/2.5x3.33/map_1km-merge-10min_HYDRO1K-merge-nomask_to_2.5x3.33_nomask_aave_da_c130405.nc - - -lnd/clm2/mappingdata/maps/0.25x0.25/map_0.01x0.01_nomask_to_0.25x0.25_nomask_aave_da_c240501.nc - - - @@ -2066,76 +2069,83 @@ this mask will have smb calculated over the entire global land surface lnd/clm2/mappingdata/maps/northamericax4v1pg2/map_0.5x0.5_GSDTG2000_to_northamericax4v1pg2_nomask_aave_da_c210112.nc - + -lnd/clm2/mappingdata/maps/0.125x0.125/map_0.5x0.5_AVHRR_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_0.5x0.5_MODIS_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_0.9x1.25_GRDC_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_10x10min_IGBPmergeICESatGIS_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_10x10min_nomask_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_1km-merge-10min_HYDRO1K-merge-nomask_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_360x720cru_cruncep_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_3x3min_GLOBE-Gardner-mergeGIS_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_3x3min_GLOBE-Gardner_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_3x3min_LandScan2004_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_3x3min_MODIS_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_3x3min_USGS_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_5x5min_IGBP-GSDP_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_5x5min_ISRIC-WISE_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_5x5min_nomask_to_0.125x0.125_nomask_aave_da_c190725.nc -lnd/clm2/mappingdata/maps/0.125x0.125/map_0.5x0.5_GSDTG2000_to_0.125x0.125_nomask_aave_da_c190725.nc - + -lnd/clm2/mappingdata/maps/0.25x0.25/map_0.5x0.5_AVHRR_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_0.5x0.5_MODIS_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_0.9x1.25_GRDC_to_0.25x0.25_nomask_aave_da_c240124.nc lnd/clm2/mappingdata/maps/0.25x0.25/map_10x10min_IGBPmergeICESatGIS_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_10x10min_nomask_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_1km-merge-10min_HYDRO1K-merge-nomask_to_0.25x0.25_nomask_aave_da_c240123.nc lnd/clm2/mappingdata/maps/0.25x0.25/map_360x720cru_cruncep_to_0.25x0.25_nomask_aave_da_c240124.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_3x3min_GLOBE-Gardner-mergeGIS_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_3x3min_GLOBE-Gardner_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_3x3min_LandScan2004_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_3x3min_MODIS_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_3x3min_USGS_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_5x5min_IGBP-GSDP_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_5x5min_ISRIC-WISE_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_360x720_cruncep_to_0.25x0.25_nomask_aave_da_c240124.nc +lnd/clm2/mappingdata/maps/0.25x0.25/map_5x5min_nomask_to_0.25x0.25_nomask_aave_da_c240123.nc -lnd/clm2/mappingdata/maps/0.25x0.25/map_0.5x0.5_GSDTG2000_to_0.25x0.25_nomask_aave_da_c240123.nc +lnd/clm2/mappingdata/maps/0.25x0.25/map_0.01x0.01_nomask_to_0.25x0.25_nomask_aave_da_c240501.nc +lnd/clm2/mappingdata/maps/0.25x0.25/map_0.1x0.1_nomask_to_0.25x0.25_nomask_aave_da_c240308.nc + lnd/clm2/mappingdata/maps/antarcticax4v1/map_0.5x0.5_AVHRR_to_antarcticax4v1_nomask_aave_da_c210130.nc diff --git a/components/elm/bld/namelist_files/namelist_definition.xml b/components/elm/bld/namelist_files/namelist_definition.xml index 2524eae9db5..c76271b7baf 100644 --- a/components/elm/bld/namelist_files/namelist_definition.xml +++ b/components/elm/bld/namelist_files/namelist_definition.xml @@ -139,6 +139,12 @@ If TRUE consider priority of plant to get a fraction of symbiotic N fixation and P phosphatase + +BGC balance check tolerance +Default 1.0e-7 hardwired + + Set date for beginning of adding temperature to atmospheric forcing @@ -1491,7 +1497,7 @@ Representative concentration pathway for future scenarios [radiative forcing at + valid_values="USGS,gx3v7,gx1v6,navy,test,tx0.1v2,tx1v1,T62,TL319,cruncep,oEC60to30v3,oEC60to30v3wLI,ECwISC30to60E1r2,EC30to60E2r2,WC14to60E2r3,WCAtl12to45E2r4,SOwISC12to60E2r4,ECwISC30to60E2r1,oRRS18to6,oRRS18to6v3,oRRS15to5,oARRM60to10,oARRM60to6,ARRM10to60E2r1,oQU480,oQU240,oQU240wLI,oQU120,oRRS30to10v3,oRRS30to10v3wLI,360x720cru,NLDASww3a,NLDAS,tx0.1v2,ICOS10,IcoswISC30E3r5,IcosXISC30E3r7,RRSwISC6to18E3r5,SOwISC12to30E3r3"> Land mask description @@ -1502,7 +1508,7 @@ If TRUE, irrigation will be active (find surface datasets with active irrigation +"1000,850,1100,1350,1600,1850,1855,1865,1875,1885,1895,1905,1915,1925,1935,1945,1950,1955,1965,1975,1980,1985,1995,2000,2005,2010,2015,2025,2035,2045,2055,2065,2075,2085,2095,2100,2105"> Year to simulate and to provide datasets for (such as surface datasets, initial conditions, aerosol-deposition, Nitrogen deposition rates etc.) A sim_year of 1000 corresponds to data used for testing only, NOT corresponding to any real datasets. A sim_year greater than 2005 corresponds to rcp scenario data diff --git a/components/elm/cime_config/config_compsets.xml b/components/elm/cime_config/config_compsets.xml index 5eed24e49da..4afbdfbf3a3 100644 --- a/components/elm/cime_config/config_compsets.xml +++ b/components/elm/cime_config/config_compsets.xml @@ -972,6 +972,11 @@ 2000_DATM%QIA_ELM%SP_SICE_SOCN_MOSART_MALI_SWAV + + IGERA5ELM_MLI + 2000_DATM%ERA5_ELM%SP_SICE_SOCN_MOSART_MALI_SWAV + + I1PTELM 2000_DATM%1PT_ELM%SP_SICE_SOCN_MOSART_SGLC_SWAV diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/lnd_rof_2way/shell_commands b/components/elm/cime_config/testdefs/testmods_dirs/elm/lnd_rof_2way/shell_commands new file mode 100644 index 00000000000..6eeec0102e5 --- /dev/null +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/lnd_rof_2way/shell_commands @@ -0,0 +1,2 @@ +./xmlchange LND_NCPL=48 +./xmlchange ROF_NCPL=48 \ No newline at end of file diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/lnd_rof_2way/user_nl_mosart b/components/elm/cime_config/testdefs/testmods_dirs/elm/lnd_rof_2way/user_nl_mosart index c7a09f76ae4..fe460362de6 100644 --- a/components/elm/cime_config/testdefs/testmods_dirs/elm/lnd_rof_2way/user_nl_mosart +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/lnd_rof_2way/user_nl_mosart @@ -1 +1,2 @@ -inundflag = .true. \ No newline at end of file +inundflag = .true. +delt_mosart = 1800 \ No newline at end of file diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/snowveg_arctic/shell_commands b/components/elm/cime_config/testdefs/testmods_dirs/elm/snowveg_arctic/shell_commands new file mode 100644 index 00000000000..396d8cea8d6 --- /dev/null +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/snowveg_arctic/shell_commands @@ -0,0 +1,10 @@ +./xmlchange LND_DOMAIN_FILE=domain_42_FLUXNETSITES_simyr1850_c170912.nc +./xmlchange ATM_DOMAIN_FILE=domain_42_FLUXNETSITES_simyr1850_c170912.nc +./xmlchange LND_DOMAIN_PATH="\$DIN_LOC_ROOT/share/domains/domain.clm" +./xmlchange ATM_DOMAIN_PATH="\$DIN_LOC_ROOT/share/domains/domain.clm" +./xmlchange DATM_MODE=CLM1PT +./xmlchange DATM_CLMNCEP_YR_START=2000 +./xmlchange DATM_CLMNCEP_YR_END=2000 +./xmlchange ELM_USRDAT_NAME=42_FLUXNETSITES +./xmlchange NTASKS=1 +./xmlchange NTHRDS=1 diff --git a/components/elm/cime_config/testdefs/testmods_dirs/elm/snowveg_arctic/user_nl_elm b/components/elm/cime_config/testdefs/testmods_dirs/elm/snowveg_arctic/user_nl_elm new file mode 100644 index 00000000000..bd07678a134 --- /dev/null +++ b/components/elm/cime_config/testdefs/testmods_dirs/elm/snowveg_arctic/user_nl_elm @@ -0,0 +1,8 @@ +fsurdat = '$DIN_LOC_ROOT/lnd/clm2/surfdata_map/surfdata_42_FLUXNETSITES_simyr1850_c170912.nc' +! this test has a parameter file with four new parameters defined for NGEE Arctic IM3 +! meant to improve snow-vegetation interactions. +! bendresist -> varies between 0 and 1, represents how much vegetation bends under snow loading +! vegshape -> suggested value 1 (parabolic), but 2 also used previously (Liston and Heimstra, 2011) +! taper -> deadstem height/radius ratio, moved from hardcoded in VegStructUpdateMod to pftvarcon +! stocking -> individual density on landscape (plants/m2), moved from hardcoded in VegStructUpdateMod to pftvarcon +paramfile = '$DIN_LOC_ROOT/lnd/clm2/paramdata/clm_params_ngeea-im3_c240822.nc' diff --git a/components/elm/docs/user-guide/index.md b/components/elm/docs/user-guide/index.md index 7995e54acb3..43fe53d6655 100644 --- a/components/elm/docs/user-guide/index.md +++ b/components/elm/docs/user-guide/index.md @@ -76,3 +76,11 @@ Using the above-mentioned settings: [FATES](fates.md) can be run in various modes with ELM through the use of namelist settings. The [FATES User's Guide section on namelist options](https://fates-users-guide.readthedocs.io/en/latest/user/Namelist-Options-and-Run-Time-Modes.html) provides guidance on enabling these different FATES run modes. + +## Create land surface dataset + +A new surface dataset for ELM is generated using `mksurfdata_map` and the notes about it are available [here](surface_dataset.md) + +## Generate land initial condition + +Initial ELM condition can be generated using `interpinic` and the notes about it are available [here](interpinic.md). diff --git a/components/elm/docs/user-guide/interpinic.md b/components/elm/docs/user-guide/interpinic.md new file mode 100644 index 00000000000..f868ee7e8d2 --- /dev/null +++ b/components/elm/docs/user-guide/interpinic.md @@ -0,0 +1,79 @@ +# Creating an ELM initial condition file + +An ELM initial condition (IC) file can be created by remapping an existing IC file from +one resolution to another using the `interpinic`, located at +`components/elm/tools/interpinic`. An ELM IC file is in the same format as an ELM restart file. +The composet of the remapped IC file will be the same as that of the input IC file. +So, for a new ELM SP-mode IC file, use an ELM input file corresponding to the SP-mode. + +The steps involved in creating a new IC files are as follows: + +1. Identifying an input ELM IC or restart file that will be remapped. +2. Obtaining an ELM restart file at the new resolution. +3. Compiling `interpinic` on the machine of interest. +4. Running `interpinic` to perform the interpolation. + +The notes below provide an example of creating 1850 ELM IC file for the NARRM grid using E3SM v3 LR piControl from year = 0101. These notes are provided for Chrysalis. + +## 1. Identification of the input ELM IC file + +The identified input land condition file for this case is the following: + +```bash +/lcrc/group/e3sm2/ac.golaz/E3SMv3/v3.LR.piControl/archive/rest/0101-01-01-00000/v3.LR.piControl.elm.r.0101-01-01-00000.nc +``` + +## 2. Obtaining an ELM restart file + +Using an existing NARRM land IC and making a copy of it + +```bash +cd components/elm/tools/interpinic + +cp /lcrc/group/e3sm/data/inputdata/lnd/clm2/initdata_map/elmi.v3-NARRM.northamericax4v1pg2_r025_IcoswISC30E3r5.1870-01-01-00000.c20240704.nc \ +elmi.v3-NARRM.northamericax4v1pg2_r025_IcoswISC30E3r5.1850-01-01-00000.c`date "+%Y%m%d"`.nc +``` + +## 3. Compiling `interpinic` + +```bash +# Load relevant modules +cd +eval $(./cime/CIME/Tools/get_case_env) + +# change directory +cd components/elm/tools/interpinic/src + +export USER_LDFLAGS="-L$NETCDF_C_DIR/lib -lnetcdf -L$NETCDF_F_DIR/lib -lnetcdff -L$HDF5_DIR/lib -lhdf5" + +USER_FC=ifort LIB_NETCDF="`nc-config --flibs`" INC_NETCDF="`nf-config --includedir`" make VERBOSE=1 + +cd ../ +``` + +## 4. Run `interpinic` + +The `interpinic` can then be run via the following batch job (e.g., `remap.r025_RRSwISC6to18E3r4.1850.batch`) to generate the initial condition. + +```bash +>cat remap.r025_RRSwISC6to18E3r4.1850.batch + +#!/bin/sh +#SBATCH --job-name=remap +#SBATCH --nodes=1 +#SBATCH --exclusive +#SBATCH --time 24:00:00 +#SBATCH -p slurm +#SBATCH --account esmd + +# Load relevant modules. +cd +eval $(./cime/CIME/Tools/get_case_env) + +# Change dir to `interpinic` +cd components/elm/tools/interpinic/ + +srun -n 1 ./interpinic \ +-i /lcrc/group/e3sm2/ac.golaz/E3SMv3/v3.LR.piControl/archive/rest/0101-01-01-00000/v3.LR.piControl.elm.r.0101-01-01-00000.nc \ +-o elmi.v3-NARRM.northamericax4v1pg2_r025_IcoswISC30E3r5.1850-01-01-00000.c20240903.nc +``` diff --git a/components/elm/docs/user-guide/surface_dataset.md b/components/elm/docs/user-guide/surface_dataset.md new file mode 100644 index 00000000000..5adaad100d6 --- /dev/null +++ b/components/elm/docs/user-guide/surface_dataset.md @@ -0,0 +1,56 @@ +# Creating an ELM surface dataset + +The notes describe the steps in creating an ELM surface dataset at 0.5x0.5 resolution for 1950 on Perlmutter. + +## 1. Load the appropriate modules + +```bash +cd +eval $(./cime/CIME/Tools/get_case_env) +``` + +## 2. Compile `mksurfdata_map` + +```bash +cd components/elm/tools/mksurfdata_map/src/ + +make clean +export USER_LDFLAGS="-L$NETCDF_DIR/lib -lnetcdf -lnetcdff -lnetcdf_intel" +export USER_LDFLAGS=$USER_LDFLAGS" -L$HDF5_DIR/lib -lhdf5 -lhdf5_fortran -lhdf5_hl_intel -lhdf5hl_fortran_intel" + +USER_FC=ifort LIB_NETCDF="`nc-config --flibs`" INC_NETCDF="`nf-config --includedir`" make VERBOSE=1 +``` + +## Build the namelist + +This step assumes that the resolution for which the new surface dataset is being created is a supported resolution. +If the surface dataset is being created for an unsupported resolution, 16 mapping files will have to be created to map the raw datasets +onto this unsupported resolution. The `namelist` file with default number of glaciers (equal to zero) can be generated as: + +```bash +cd ../ + +RES=0.5x0.5 +YR=1950 +DIN_LOC_ROOT=/global/cfs/cdirs/e3sm/inputdata + +./mksurfdata.pl -res $RES -years $YR -d -dinlc $DIN_LOC_ROOT +``` + +An example of generating the namelist for 0.25 deg (`r025`) resolution for 1980 with 10 glacier layers is as follows: + +```bash +RES=r025 +YR=1980 +DIN_LOC_ROOT=/global/cfs/cdirs/e3sm/inputdata + +./mksurfdata.pl -res $RES -years $YR -d -dinlc $DIN_LOC_ROOT -glc_nec 10 +``` + +## Run `mksurfdata_map` via an interactive job + +```bash +salloc --nodes 1 --qos interactive --time 01:00:00 --constraint cpu --account e3sm + +srun -n 1 ./mksurfdata_map < namelist +``` diff --git a/components/elm/src/biogeochem/AllocationMod.F90 b/components/elm/src/biogeochem/AllocationMod.F90 index 141194397fb..ba14dbc352d 100644 --- a/components/elm/src/biogeochem/AllocationMod.F90 +++ b/components/elm/src/biogeochem/AllocationMod.F90 @@ -36,11 +36,11 @@ module AllocationMod use elm_varctl , only : NFIX_PTASE_plant use ELMFatesInterfaceMod , only : hlm_fates_interface_type use elm_varctl , only: iulog - use elm_varctl , only : carbon_only - use elm_varctl , only : carbonnitrogen_only + use elm_varctl , only : carbon_only + use elm_varctl , only : carbonnitrogen_only use elm_varctl , only : carbonphosphorus_only use shr_infnan_mod , only: nan => shr_infnan_nan, assignment(=) - + ! implicit none save @@ -114,23 +114,23 @@ module AllocationMod ! to toggle and update which processes are active. ! This will get set to false ! after ad_carbon_only is complete. - - + + logical :: crop_supln = .false. !Prognostic crop receives supplemental Nitrogen - + real(r8), allocatable,target :: veg_rootc_bigleaf(:,:) ! column-level fine-root biomas kgc/m3 integer, pointer :: ft_index_bigleaf(:) ! array holding the pft index of each competitor ! ECA parameters - ! scaling factor for plant fine root biomass to calculate nutrient carrier enzyme abundance - real(r8), parameter :: e_plant_scalar = 0.0000125_r8 - - ! scaling factor for plant fine root biomass to calculate nutrient carrier enzyme abundance - real(r8), parameter :: e_decomp_scalar = 0.05_r8 + ! scaling factor for plant fine root biomass to calculate nutrient carrier enzyme abundance + real(r8), parameter :: e_plant_scalar = 0.0000125_r8 + + ! scaling factor for plant fine root biomass to calculate nutrient carrier enzyme abundance + real(r8), parameter :: e_decomp_scalar = 0.05_r8 !$acc declare create(e_decomp_scalar) !$acc declare create(e_plant_scalar) - + !$acc declare copyin(crop_supln) !----------------------------------------------------------------------- @@ -213,8 +213,8 @@ subroutine AllocationInit ( bounds, elm_fates) use elm_time_manager, only: get_step_size use elm_varpar , only: crop_prog use elm_varctl , only: iulog - use elm_varctl , only : carbon_only - use elm_varctl , only : carbonnitrogen_only + use elm_varctl , only : carbon_only + use elm_varctl , only : carbonnitrogen_only use elm_varctl , only : carbonphosphorus_only @@ -234,9 +234,9 @@ subroutine AllocationInit ( bounds, elm_fates) integer :: max_comps ! maximum number of possible plant competitors ! elm big-leaf: number of pfts/patches ! fates: number of cohorts in the column - - + + !----------------------------------------------------------------------- if ( crop_prog )then @@ -253,7 +253,7 @@ subroutine AllocationInit ( bounds, elm_fates) end if end if - + ! set time steps dt = real( get_step_size(), r8 ) @@ -374,7 +374,7 @@ subroutine EvaluateSupplStatus() end subroutine EvaluateSupplStatus !------------------------------------------------------------------------------------------------- - + subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp, filter_soilp, & photosyns_vars, crop_vars, canopystate_vars, cnstate_vars, dt, yr) ! PHASE-1 of Allocation: loop over patches to assess the total plant N demand and P demand @@ -430,7 +430,7 @@ subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) froot_leaf => veg_vp%froot_leaf , & ! Input: [real(r8) (:) ] allocation parameter: new fine root C per new leaf C (gC/gC) croot_stem => veg_vp%croot_stem , & ! Input: [real(r8) (:) ] allocation parameter: new coarse root C per new stem C (gC/gC) stem_leaf => veg_vp%stem_leaf , & ! Input: [real(r8) (:) ] allocation parameter: new stem c per new leaf C (gC/gC) @@ -645,7 +645,7 @@ subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp if (stem_leaf(ivt(p)) < 0._r8) then if (stem_leaf(ivt(p)) == -1._r8) then - f3 = (2.7/(1.0+exp(-0.004*(annsum_npp(p) - 300.0)))) - 0.4 + f3 = max((2.7/(1.0+exp(-0.004*(annsum_npp(p) - 300.0)))) - 0.4_r8, 0.2_r8) else f3 = max((-1.0_r8*stem_leaf(ivt(p))*2.7_r8)/(1.0_r8+exp(-0.004_r8*(annsum_npp(p) - & 300.0_r8))) - 0.4_r8, 0.2_r8) @@ -971,7 +971,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & ! Fractional uptake profiles, that are proportional to root density real(r8):: nuptake_prof(bounds%begc:bounds%endc,1:nlevdecomp) real(r8):: puptake_prof(bounds%begc:bounds%endc,1:nlevdecomp) - integer, allocatable :: filter_pcomp(:) ! this is a plant competitor map for FATES/ELM-BL w/ ECA + integer, allocatable :: filter_pcomp(:) ! this is a plant competitor map for FATES/ELM-BL w/ ECA real(r8), allocatable,target :: plant_nh4demand_vr_fates(:,:) ! nh4 demand per competitor per soil layer real(r8), allocatable,target :: plant_no3demand_vr_fates(:,:) ! no3 demand per competitor per soil layer real(r8), allocatable,target :: plant_pdemand_vr_fates(:,:) ! p demand per competitor per soil layer @@ -990,7 +990,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & real(r8):: cp_stoich_var=0.4 ! variability of CP ratio - + !----------------------------------------------------------------------- associate( & @@ -1152,7 +1152,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & n_pcomp = elm_fates%fates(ci)%bc_out(s)%num_plant_comps pci = 1 pcf = n_pcomp - + if( nu_com.eq.'RD') then ! Overwrite the column level demands, since fates plants are all sharing @@ -1160,13 +1160,13 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & ! to scale up to column plant_ndemand_col(c) = 0._r8 plant_pdemand_col(c) = 0._r8 - + ! We fill the vertically resolved array to simplify some jointly used code do j = 1, nlevdecomp col_plant_ndemand_vr(c,j) = 0._r8 col_plant_pdemand_vr(c,j) = 0._r8 - + do f = 1,n_pcomp ft = elm_fates%fates(ci)%bc_out(s)%ft_index(f) @@ -1184,10 +1184,10 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & ! [gN/m2/s] plant_ndemand_col(c) = plant_ndemand_col(c) + col_plant_ndemand_vr(c,j)*dzsoi_decomp(j) plant_pdemand_col(c) = plant_pdemand_col(c) + col_plant_pdemand_vr(c,j)*dzsoi_decomp(j) - + end do - + else !(ECA) do f = 1,n_pcomp @@ -1195,7 +1195,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & end do veg_rootc_ptr => elm_fates%fates(ci)%bc_out(s)%veg_rootc - ft_index_ptr => elm_fates%fates(ci)%bc_out(s)%ft_index ! Should be + ft_index_ptr => elm_fates%fates(ci)%bc_out(s)%ft_index ! Should be decompmicc(:) = elm_fates%fates(ci)%bc_out(s)%decompmicc(:) ! Should be (nlevdecomp) cn_scalar_runmean_ptr => elm_fates%fates(ci)%bc_out(s)%cn_scalar ! This is 1.0 @@ -1216,7 +1216,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & pci = col_pp%pfti(c) ! Initial plant competitor index pcf = col_pp%pftf(c) ! Final plant competitor index - + if (nu_com .eq. 'RD') then do j = 1, nlevdecomp @@ -1225,7 +1225,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & end do else - + f = 0 decompmicc(:) = 0._r8 do p = col_pp%pfti(c), col_pp%pftf(c) @@ -1246,7 +1246,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & end if end do n_pcomp = f - + ft_index_ptr => ft_index_bigleaf veg_rootc_ptr => veg_rootc_bigleaf @@ -1263,7 +1263,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & (leafcn(ivt(p)) - leafcn(ivt(p))*(1- cn_stoich_var)),0.0_r8),1.0_r8) end do end if - + km_nh4_ptr => km_plant_nh4 vmax_nh4_ptr => vmax_plant_nh4 cn_scalar_runmean_ptr => cn_scalar_runmean @@ -1298,14 +1298,14 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & ! (1) add nitrogen and phosphorus competition ! (2) nitrogen and phosphorus uptake is based on root kinetics ! (3) no second pass nutrient uptake for plants - ! ============================================================= - + ! ============================================================= + if (nu_com .eq. 'RD') then ! Estimate actual allocation rates via Relative Demand ! approach (RD) - + call NAllocationRD(col_plant_ndemand_vr(c,:), & ! IN potential_immob_vr(c,:), & ! IN AllocParamsInst%compet_plant_nh4, & ! IN @@ -1332,19 +1332,19 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & ! Estimate actual allocation rates via Capacitance Aquisition ! approach (ECA/CA) - + call NAllocationECAMIC(pci,dt, & ! IN bd(c,:), & ! IN h2osoi_vol(c,:), & ! IN t_scalar(c,:), & ! IN - n_pcomp, & ! IN + n_pcomp, & ! IN filter_pcomp(1:n_pcomp), & ! IN veg_rootc_ptr(pci:pcf,:), & ! IN ft_index_ptr(pci:pcf), & ! IN cn_scalar_runmean_ptr(pci:pcf), & ! IN decompmicc, & ! IN smin_nh4_vr(c,:), & ! IN - nu_com, & ! IN + nu_com, & ! IN km_nh4_ptr, & ! IN vmax_nh4_ptr, & ! IN km_decomp_nh4, & ! IN @@ -1389,7 +1389,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & ! NO3 flux demands. supplement_to_sminn_vr(c,j) = 0._r8 if (carbon_only .or. carbonphosphorus_only) then - + if ( fpi_no3_vr(j) + fpi_nh4_vr(j) < 1._r8 ) then fpi_vr(c,j) = 1._r8 fpi_nh4_vr(j) = 1.0_r8 - fpi_no3_vr(j) @@ -1409,9 +1409,9 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & supplement_to_sminn_vr(c,j) = supplement_to_sminn_vr(c,j) + col_plant_ndemand_vr(c,j) smin_nh4_to_plant_vr(c,j) = col_plant_ndemand_vr(c,j) - smin_no3_to_plant_vr(c,j) end if - - + + end if ! sum up nitrogen limitation to decomposition @@ -1422,14 +1422,14 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & actual_immob_vr(c,j) = actual_immob_no3_vr(c,j) + actual_immob_nh4_vr(c,j) end do - + ! Starting resolving P limitation !!! ! ============================================================= if (nu_com .eq. 'RD') then ! Relative Demand (RD) - + call PAllocationRD(col_plant_pdemand_vr(c,:), & ! IN potential_immob_p_vr(c,:), & ! IN solutionp_vr(c,:), & ! IN @@ -1438,38 +1438,38 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & actual_immob_p_vr(c,:), & ! OUT sminp_to_plant_vr(c,:), & ! OUT supplement_to_sminp_vr(c,:)) ! OUT - + else call PAllocationECAMIC(pci,dt, & ! IN h2osoi_vol(c,:), & ! IN - t_scalar(c,:), & ! IN - gross_pmin_vr(c,:), & ! IN - potential_immob_p_vr(c,:), & ! IN - biochem_pmin_vr_col(c,:), & ! IN - primp_to_labilep_vr_col(c,:), & ! IN - pdep_to_sminp(c), & ! IN - pdep_prof(c,:), & ! IN - vmax_minsurf_p_vr(isoilorder(c),:), & ! IN - km_minsurf_p_vr(isoilorder(c),:), & ! IN - solutionp_vr(c,:), & ! IN - nu_com, & ! IN - n_pcomp, & ! IN + t_scalar(c,:), & ! IN + gross_pmin_vr(c,:), & ! IN + potential_immob_p_vr(c,:), & ! IN + biochem_pmin_vr_col(c,:), & ! IN + primp_to_labilep_vr_col(c,:), & ! IN + pdep_to_sminp(c), & ! IN + pdep_prof(c,:), & ! IN + vmax_minsurf_p_vr(isoilorder(c),:), & ! IN + km_minsurf_p_vr(isoilorder(c),:), & ! IN + solutionp_vr(c,:), & ! IN + nu_com, & ! IN + n_pcomp, & ! IN filter_pcomp(1:n_pcomp), & ! IN - veg_rootc_ptr(pci:pcf,:), & ! IN - ft_index_ptr(pci:pcf), & ! IN - decompmicc, & ! IN - cp_scalar_runmean_ptr(pci:pcf), & ! IN - km_p_ptr(:), & ! IN - vmax_p_ptr(:), & ! IN - km_decomp_p, & ! IN - labilep_vr(c,:), & ! IN + veg_rootc_ptr(pci:pcf,:), & ! IN + ft_index_ptr(pci:pcf), & ! IN + decompmicc, & ! IN + cp_scalar_runmean_ptr(pci:pcf), & ! IN + km_p_ptr(:), & ! IN + vmax_p_ptr(:), & ! IN + km_decomp_p, & ! IN + labilep_vr(c,:), & ! IN plant_pdemand_vr_ptr(pci:pcf,:), & ! INOUT - col_plant_pdemand_vr(c,:), & ! OUT + col_plant_pdemand_vr(c,:), & ! OUT adsorb_to_labilep_vr(c,:), & ! OUT fpi_p_vr(c,:), & ! OUT actual_immob_p_vr(c,:), & ! OUT - sminp_to_plant_vr(c,:), & ! OUT + sminp_to_plant_vr(c,:), & ! OUT desorb_to_solutionp_vr(c,:), & ! OUT supplement_to_sminp_vr(c,:)) ! OUT @@ -1482,13 +1482,13 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & supplement_to_sminp_vr(c,j) = col_plant_pdemand_vr(c,j) end do end if - + end if ! end of P competition ! resolving N limitation vs. P limitation for decomposition ! update (1) actual immobilization for N and P (2) sminn_to_plant and sminp_to_plant ! We only resolve co-limitations when are supplementing neither element - + np_bothactive: if ( .not.carbon_only .and. & .not.carbonphosphorus_only .and. & .not.carbonnitrogen_only ) then @@ -1593,7 +1593,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & actual_immob_vr(c,j) = potential_immob_vr(c,j) * fpi_p_vr(c,j) end do end if - + ! sum up plant N/P uptake at column level and patch level ! sum up N fluxes to plant after initial competition sminn_to_plant(c) = 0._r8 @@ -1602,7 +1602,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & sminn_to_plant(c) = sminn_to_plant(c) + sminn_to_plant_vr(c,j) * dzsoi_decomp(j) sminp_to_plant(c) = sminp_to_plant(c) + sminp_to_plant_vr(c,j) * dzsoi_decomp(j) end do - + ! update column plant N/P demand, pft level plant NP uptake for ECA and MIC mode eca_filter: if (nu_com .eq. 'ECA' .or. nu_com .eq. 'MIC') then @@ -1648,14 +1648,14 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & end if end do end if - + end do end if eca_filter end do col_loop - + if ((nu_com .eq. 'ECA' .or. nu_com .eq. 'MIC')) then deallocate(filter_pcomp) if(.not.use_fates)then @@ -1750,7 +1750,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & ! Set the FATES N and P uptake fluxes - + if(use_fates)then do fc=1,num_soilc c = filter_soilc(fc) @@ -1771,15 +1771,15 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & (elm_fates%fates(ci)%bc_pconst%vmax_nh4(ft)+elm_fates%fates(ci)%bc_pconst%vmax_no3(ft)) * & dzsoi_decomp(j) end do - + do j = 1,nlevdecomp - elm_fates%fates(ci)%bc_in(s)%plant_nh4_uptake_flux(f,1) = & + elm_fates%fates(ci)%bc_in(s)%plant_nh4_uptake_flux(f,1) = & elm_fates%fates(ci)%bc_in(s)%plant_nh4_uptake_flux(f,1) + & smin_nh4_to_plant_vr(c,j)*dt*dzsoi_decomp(j) * & (ndemand/plant_ndemand_col(c)) - elm_fates%fates(ci)%bc_in(s)%plant_no3_uptake_flux(f,1) = & + elm_fates%fates(ci)%bc_in(s)%plant_no3_uptake_flux(f,1) = & elm_fates%fates(ci)%bc_in(s)%plant_no3_uptake_flux(f,1) + & smin_no3_to_plant_vr(c,j)*dt*dzsoi_decomp(j) * & (ndemand/plant_ndemand_col(c)) @@ -1787,12 +1787,12 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & end do end do end if - + if( plant_pdemand_col(c)>tiny(plant_pdemand_col(c)) ) then do f = 1,n_pcomp ft = elm_fates%fates(ci)%bc_out(s)%ft_index(f) - + pdemand=0._r8 do j = 1,nlevdecomp ! [gP/m2/s] @@ -1800,14 +1800,14 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & elm_fates%fates(ci)%bc_pconst%vmax_p(ft) * & dzsoi_decomp(j) end do - + do j = 1,nlevdecomp ! [gP/m2/step] - elm_fates%fates(ci)%bc_in(s)%plant_p_uptake_flux(f,1) = & + elm_fates%fates(ci)%bc_in(s)%plant_p_uptake_flux(f,1) = & elm_fates%fates(ci)%bc_in(s)%plant_p_uptake_flux(f,1) + & sminp_to_plant_vr(c,j)*dt*dzsoi_decomp(j) * & (pdemand/plant_pdemand_col(c)) - + end do end do end if @@ -1817,21 +1817,21 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & do f = 1,n_pcomp do j = 1,nlevdecomp - elm_fates%fates(ci)%bc_in(s)%plant_nh4_uptake_flux(f,1) = & - elm_fates%fates(ci)%bc_in(s)%plant_nh4_uptake_flux(f,1) + & + elm_fates%fates(ci)%bc_in(s)%plant_nh4_uptake_flux(f,1) = & + elm_fates%fates(ci)%bc_in(s)%plant_nh4_uptake_flux(f,1) + & plant_nh4demand_vr_fates(f,j) * fpg_nh4_vr(c,j) * dzsoi_decomp(j) * dt - - elm_fates%fates(ci)%bc_in(s)%plant_no3_uptake_flux(f,1) = & - elm_fates%fates(ci)%bc_in(s)%plant_no3_uptake_flux(f,1) + & + + elm_fates%fates(ci)%bc_in(s)%plant_no3_uptake_flux(f,1) = & + elm_fates%fates(ci)%bc_in(s)%plant_no3_uptake_flux(f,1) + & plant_no3demand_vr_fates(f,j) * fpg_no3_vr(c,j) * dzsoi_decomp(j) * dt - - elm_fates%fates(ci)%bc_in(s)%plant_p_uptake_flux(f,1) = & - elm_fates%fates(ci)%bc_in(s)%plant_p_uptake_flux(f,1) + & + + elm_fates%fates(ci)%bc_in(s)%plant_p_uptake_flux(f,1) = & + elm_fates%fates(ci)%bc_in(s)%plant_p_uptake_flux(f,1) + & (plant_pdemand_vr_fates(f,j) * fpg_p_vr(c,j)) * dzsoi_decomp(j) * dt - + end do end do - + end if end do @@ -1840,7 +1840,7 @@ subroutine Allocation2_ResolveNPLimit (bounds, num_soilc, filter_soilc , & deallocate(plant_no3demand_vr_fates) deallocate(plant_pdemand_vr_fates) end if - + end if ! if(use_fates) end associate @@ -1907,7 +1907,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) froot_leaf => veg_vp%froot_leaf , & ! Input: [real(r8) (:) ] allocation parameter: new fine root C per new leaf C (gC/gC) croot_stem => veg_vp%croot_stem , & ! Input: [real(r8) (:) ] allocation parameter: new coarse root C per new stem C (gC/gC) stem_leaf => veg_vp%stem_leaf , & ! Input: [real(r8) (:) ] allocation parameter: new stem c per new leaf C (gC/gC) @@ -2118,7 +2118,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & if (stem_leaf(ivt(p)) < 0._r8) then if (stem_leaf(ivt(p)) == -1._r8) then - f3 = (2.7/(1.0+exp(-0.004*(annsum_npp(p) - 300.0)))) - 0.4 + f3 = max((2.7/(1.0+exp(-0.004*(annsum_npp(p) - 300.0)))) - 0.4_r8, 0.2_r8) else f3 = max((-1.0_r8*stem_leaf(ivt(p))*2.7_r8)/(1.0_r8+exp(-0.004_r8*(annsum_npp(p) - & 300.0_r8))) - 0.4_r8, 0.2_r8) @@ -2959,7 +2959,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & end associate end subroutine Allocation3_PlantCNPAlloc - + ! ====================================================================================== subroutine NAllocationECAMIC(pci,dt, & ! IN @@ -2976,14 +2976,14 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN nu_com, & ! IN km_nh4_plant, & ! IN (pft) vmax_nh4_plant, & ! IN (pft) - km_decomp_nh4, & ! IN + km_decomp_nh4, & ! IN potential_immob_vr, & ! IN (j) plant_nh4demand_vr, & ! INOUT (i,j) col_plant_nh4demand_vr, & ! OUT (j) fpi_nh4_vr, & ! OUT (j) actual_immob_nh4_vr, & ! OUT (j) smin_nh4_to_plant_vr, & ! OUT (j) - smin_no3_vr, & ! IN (j) + smin_no3_vr, & ! IN (j) km_no3_plant, & ! IN (pft) vmax_no3_plant, & ! IN (pft) km_decomp_no3, & ! IN (j) @@ -3004,12 +3004,12 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN ! kinetics following Zhu et al., 2016 DOI: 10.1002/2016JG003554 ! ------------------------------------------------------------------------------------ use elm_varpar , only: nlevdecomp - + integer, intent(in) :: pci ! First index of plant comp arrays real(r8), intent(in) :: dt ! Time step duration [s] real(r8), intent(in) :: bd(:) ! Bulk density of dry soil material [kg m-3] real(r8), intent(in) :: h2osoi_vol(:) ! Vol. Soil Water in each layer [m3] - real(r8), intent(in) :: t_scalar(:) ! fraction by which decomposition is limited by temperature + real(r8), intent(in) :: t_scalar(:) ! fraction by which decomposition is limited by temperature integer, intent(in) :: n_pcomp ! number of plant competitors integer, intent(in) :: filter_pcomp(:) ! plant competition filter real(r8), intent(in) :: veg_rootc(pci:,:) ! total fine-root biomass of each competitor [gC/m3] @@ -3065,7 +3065,7 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN integer :: i,ip ! loop index for competitors integer :: ft ! loop index for pfts - ! 2.76 consider soil adsorption effect on [NH4+] availability, + ! 2.76 consider soil adsorption effect on [NH4+] availability, ! based on Zhu et al., 2016 DOI: 10.1002/2016JG003554 real(r8), parameter :: adsorp_nh4_eff = 2.76_r8 @@ -3073,14 +3073,14 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN do j = 1, nlevdecomp - ! Plant, microbial decomposers compete for NH4. Thus loop over each + ! Plant, microbial decomposers compete for NH4. Thus loop over each ! plant competitor in this competitive space (column). - ! Calculate competition coefficients for N/P, first need to convert - ! concentration to per soil water based + ! Calculate competition coefficients for N/P, first need to convert + ! concentration to per soil water based ! concentration of mineralized nutrient, per soil water solution_conc = smin_nh4_vr(j) / (bd(j)*adsorp_nh4_eff*m3_per_liter + h2osoi_vol(j)) - + e_km = 0._r8 do i = 1, n_pcomp ip = filter_pcomp(i) @@ -3089,14 +3089,14 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN end do e_km = e_km + e_decomp_scalar*decompmicc(j)*(1._r8/km_decomp_nh4 + 1._r8/km_nit) - + do i = 1, n_pcomp ip = filter_pcomp(i) ft = ft_index(ip) - compet_plant(i) = solution_conc / & + compet_plant(i) = solution_conc / & ( km_nh4_plant(ft) * (1._r8 + solution_conc/km_nh4_plant(ft) + e_km)) end do - + compet_decomp = solution_conc / (km_decomp_nh4 * (1._r8 + solution_conc/km_decomp_nh4 + e_km)) compet_nit = solution_conc / (km_nit * (1._r8 + solution_conc/km_nit + e_km)) @@ -3110,7 +3110,7 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN ip = filter_pcomp(i) ft = ft_index(ip) - ! This is the demand per m3 of the column (not patch) + ! This is the demand per m3 of the column (not patch) ! (for native ELM divide through by the patch weight to get per m3 of patch) plant_nh4demand_vr(ip,j) = max(0._r8,vmax_nh4_plant(ft) * veg_rootc(ip,j) * & cn_scalar_runmean(ip) * t_scalar(j) * compet_plant(i)) @@ -3130,7 +3130,7 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN else ! 'MIC' mode - sum_nh4_demand_scaled = potential_immob_vr(j)*compet_decomp + & + sum_nh4_demand_scaled = potential_immob_vr(j)*compet_decomp + & pot_f_nit_vr(j)*compet_nit end if @@ -3199,10 +3199,10 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN do i = 1, n_pcomp ip = filter_pcomp(i) ft = ft_index(ip) - compet_plant(i) = solution_conc / & + compet_plant(i) = solution_conc / & ( km_no3_plant(ft) * (1._r8 + solution_conc/km_no3_plant(ft) + e_km)) end do - + compet_decomp = solution_conc / (km_decomp_no3 * (1._r8 + solution_conc/km_decomp_no3 + e_km)) compet_denit = solution_conc / (km_den * (1._r8 + solution_conc/km_den + e_km)) @@ -3215,7 +3215,7 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN ip = filter_pcomp(i) ft = ft_index(ip) - ! This is the demand per m3 of the column (not patch) + ! This is the demand per m3 of the column (not patch) ! (for native ELM divide through by the patch weight to get per m3 of patch) plant_no3demand_vr(ip,j) = max(0._r8,vmax_no3_plant(ft) * veg_rootc(ip,j) * & cn_scalar_runmean(ip) * t_scalar(j) * compet_plant(i)) @@ -3245,7 +3245,7 @@ subroutine NAllocationECAMIC(pci,dt, & ! IN smin_no3_to_plant_vr(j) = col_plant_no3demand_vr(j) f_denit_vr(j) = pot_f_denit_vr(j) - else + else ! NO3 availability can not satisfy the sum of immobilization, denitrification, and ! plant growth demands, so these three demands compete for available @@ -3284,34 +3284,34 @@ end subroutine NAllocationECAMIC subroutine PAllocationECAMIC(pci, & dt, & - h2osoi_vol, & - t_scalar, & - gross_pmin_vr, & - potential_immob_p_vr, & - biochem_pmin_vr_col, & - primp_to_labilep_vr_col, & - pdep_to_sminp, & - pdep_prof, & + h2osoi_vol, & + t_scalar, & + gross_pmin_vr, & + potential_immob_p_vr, & + biochem_pmin_vr_col, & + primp_to_labilep_vr_col, & + pdep_to_sminp, & + pdep_prof, & vmax_minsurf_p_vr, & km_minsurf_p_vr, & solutionp_vr, & nu_com, & n_pcomp, & - filter_pcomp, & - veg_rootc, & + filter_pcomp, & + veg_rootc, & ft_index, & decompmicc, & - cp_scalar_runmean, & + cp_scalar_runmean, & km_plant_p, & vmax_plant_p, & - km_decomp_p, & + km_decomp_p, & labilep_vr, & - plant_pdemand_vr_patch, & - col_plant_pdemand_vr, & + plant_pdemand_vr_patch, & + col_plant_pdemand_vr, & adsorb_to_labilep_vr, & fpi_p_vr, & actual_immob_p_vr, & - sminp_to_plant_vr, & + sminp_to_plant_vr, & desorb_to_solutionp_vr, & supplement_to_sminp_vr) @@ -3370,7 +3370,7 @@ subroutine PAllocationECAMIC(pci, & ! plant P uptake, microbial P uptake/release ! secondary P desorption is assumed to go into solution P pool - do j = 1, nlevdecomp + do j = 1, nlevdecomp ! plant, microbial decomposer, mineral surface compete for P ! loop over each pft within the same column @@ -3390,27 +3390,27 @@ subroutine PAllocationECAMIC(pci, & do i = 1,n_pcomp ip = filter_pcomp(i) ft = ft_index(ip) - compet_plant(i) = solution_pconc / & + compet_plant(i) = solution_pconc / & (km_plant_p(ft)*(1._r8 + solution_pconc/km_plant_p(ft) + e_km_p)) end do - + compet_decomp_p = solution_pconc / & (km_decomp_p * (1._r8 + solution_pconc/km_decomp_p + e_km_p)) - compet_minsurf_p = solution_pconc/ & + compet_minsurf_p = solution_pconc/ & (km_minsurf_p_vr(j) * (1._r8 + solution_pconc/km_minsurf_p_vr(j) + e_km_p)) col_plant_pdemand_vr(j) = 0._r8 do i = 1,n_pcomp ip = filter_pcomp(i) ft = ft_index(ip) - plant_pdemand_vr_patch(ip,j) = max(0._r8,vmax_plant_p(ft) * veg_rootc(ip,j) * & + plant_pdemand_vr_patch(ip,j) = max(0._r8,vmax_plant_p(ft) * veg_rootc(ip,j) * & cp_scalar_runmean(ip) * t_scalar(j) * compet_plant(i)) col_plant_pdemand_vr(j) = col_plant_pdemand_vr(j) + plant_pdemand_vr_patch(ip,j) end do ! potential adsorption rate without plant and microbial interaction - ! including weathering, deposition, phosphatase, mineralization, + ! including weathering, deposition, phosphatase, mineralization, ! immobilization, plant uptake dsolutionp_dt = gross_pmin_vr(j) -potential_immob_p_vr(j) - & col_plant_pdemand_vr(j) + biochem_pmin_vr_col(j) + & @@ -3477,7 +3477,7 @@ subroutine PAllocationECAMIC(pci, & if (nu_com .eq. 'MIC') sminp_to_plant_vr(j) = min(max( 0._r8, & (solutionp_vr(j)/dt) - actual_immob_p_vr(j) - adsorb_to_labilep_vr(j) ), & - col_plant_pdemand_vr(j)) + col_plant_pdemand_vr(j)) end if end do @@ -3488,7 +3488,7 @@ end subroutine PAllocationECAMIC subroutine NAllocationRD(col_plant_ndemand_vr, &! IN (j) potential_immob_vr, & ! IN (j) - compet_plants_nh4, & ! IN + compet_plants_nh4, & ! IN compet_decomp_nh4, & ! IN dt, & ! IN smin_nh4_vr, & ! IN (j) @@ -3641,7 +3641,7 @@ end subroutine NAllocationRD ! ====================================================================================== - subroutine PAllocationRD(col_plant_pdemand_vr, & ! IN + subroutine PAllocationRD(col_plant_pdemand_vr, & ! IN potential_immob_p_vr, & ! IN (j) solutionp_vr, & ! IN (j) dt, & ! IN @@ -3684,7 +3684,7 @@ subroutine PAllocationRD(col_plant_pdemand_vr, & ! IN actual_immob_p_vr(j) = potential_immob_p_vr(j) sminp_to_plant_vr(j) = col_plant_pdemand_vr(j) supplement_to_sminp_vr(j) = sum_pdemand - (solutionp_vr(j)/dt) - + else ! P availability can not satisfy the sum of immobilization and ! plant growth demands, so these two demands compete for @@ -3702,7 +3702,7 @@ subroutine PAllocationRD(col_plant_pdemand_vr, & ! IN fpi_p_vr(j) = 0.0_r8 end if - sminp_to_plant_vr(j) = max( 0._r8,(solutionp_vr(j)/dt) - actual_immob_p_vr(j) ) + sminp_to_plant_vr(j) = max( 0._r8,(solutionp_vr(j)/dt) - actual_immob_p_vr(j) ) end if end do @@ -3749,7 +3749,7 @@ subroutine calc_nuptake_prof(bounds, num_soilc, filter_soilc, cnstate_vars, nupt c = filter_soilc(fc) sminn_vr_loc(c,j) = smin_no3_vr(c,j) + smin_nh4_vr(c,j) - + if(use_pflotran .and. pf_cmode) then sminn_tot(c) = sminn_tot(c) + sminn_vr_loc(c,j) * dzsoi_decomp(j) & *(nfixation_prof(c,j)*dzsoi_decomp(j)) ! weighted by froot fractions in annual max. active layers diff --git a/components/elm/src/biogeochem/CH4Mod.F90 b/components/elm/src/biogeochem/CH4Mod.F90 index 3b3c1fa8061..a5cbd2f041c 100644 --- a/components/elm/src/biogeochem/CH4Mod.F90 +++ b/components/elm/src/biogeochem/CH4Mod.F90 @@ -223,7 +223,7 @@ subroutine InitAllocate(this, bounds) ! Allocate module variables and data structures ! ! !USES: - use shr_infnan_mod, only: spval => shr_infnan_nan, assignment(=) + use shr_infnan_mod, only: nan => shr_infnan_nan, assignment(=) use elm_varpar , only: nlevgrnd ! ! !ARGUMENTS: diff --git a/components/elm/src/biogeochem/CNAllocationBetrMod.F90 b/components/elm/src/biogeochem/CNAllocationBetrMod.F90 index 8cfc342bb40..32f9fc6eb23 100644 --- a/components/elm/src/biogeochem/CNAllocationBetrMod.F90 +++ b/components/elm/src/biogeochem/CNAllocationBetrMod.F90 @@ -26,13 +26,13 @@ module CNAllocationBeTRMod use PhotosynthesisType , only : photosyns_type use CropType , only : crop_type use VegetationPropertiesType, only : veg_vp - use LandunitType , only : lun_pp + use LandunitType , only : lun_pp use ColumnType , only : col_pp - use ColumnDataType , only : col_cf, col_ns, col_nf, col_ps, col_pf + use ColumnDataType , only : col_cf, col_ns, col_nf, col_ps, col_pf use VegetationType , only : veg_pp use VegetationDataType , only : veg_cs, veg_ns, veg_nf, veg_ps, veg_pf - use VegetationDataType , only : veg_cf, c13_veg_cf, c14_veg_cf - + use VegetationDataType , only : veg_cf, c13_veg_cf, c14_veg_cf + ! bgc interface & pflotran module switches use elm_varctl , only : nu_com use SoilStatetype , only : soilstate_type @@ -359,7 +359,7 @@ subroutine Allocation1_PlantNPDemand (bounds, num_soilc, filter_soilc, num_soilp associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) froot_leaf => veg_vp%froot_leaf , & ! Input: [real(r8) (:) ] allocation parameter: new fine root C per new leaf C (gC/gC) croot_stem => veg_vp%croot_stem , & ! Input: [real(r8) (:) ] allocation parameter: new coarse root C per new stem C (gC/gC) stem_leaf => veg_vp%stem_leaf , & ! Input: [real(r8) (:) ] allocation parameter: new stem c per new leaf C (gC/gC) @@ -1167,7 +1167,7 @@ subroutine Allocation3_PlantCNPAlloc (bounds , & associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type ! - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) froot_leaf => veg_vp%froot_leaf , & ! Input: [real(r8) (:) ] allocation parameter: new fine root C per new leaf C (gC/gC) croot_stem => veg_vp%croot_stem , & ! Input: [real(r8) (:) ] allocation parameter: new coarse root C per new stem C (gC/gC) stem_leaf => veg_vp%stem_leaf , & ! Input: [real(r8) (:) ] allocation parameter: new stem c per new leaf C (gC/gC) diff --git a/components/elm/src/biogeochem/CNGapMortalityBeTRMod.F90 b/components/elm/src/biogeochem/CNGapMortalityBeTRMod.F90 index 3bbbf1db1f6..ed8811f61f8 100644 --- a/components/elm/src/biogeochem/CNGapMortalityBeTRMod.F90 +++ b/components/elm/src/biogeochem/CNGapMortalityBeTRMod.F90 @@ -18,9 +18,9 @@ module CNGapMortalityBeTRMod use ColumnType , only : col_pp use ColumnDataType , only : col_cf, col_nf, col_pf use VegetationPropertiesType , only : veg_vp - use VegetationType , only : veg_pp - use VegetationDataType , only : veg_cs, veg_cf, veg_ns, veg_nf - use VegetationDataType , only : veg_ps, veg_pf + use VegetationType , only : veg_pp + use VegetationDataType , only : veg_cs, veg_cf, veg_ns, veg_nf + use VegetationDataType , only : veg_ps, veg_pf use PhosphorusFluxType , only : phosphorusflux_type use PhosphorusStateType , only : phosphorusstate_type @@ -121,7 +121,7 @@ subroutine CNGapMortality (& associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody & ! Input: [real(r8) (:) ] binary flag for woody lifeform + woody => veg_vp%woody & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) ) ! set the mortality rate based on annual rate diff --git a/components/elm/src/biogeochem/CNNStateUpdate1BeTRMod.F90 b/components/elm/src/biogeochem/CNNStateUpdate1BeTRMod.F90 index 465c92f82b4..b785f52549e 100644 --- a/components/elm/src/biogeochem/CNNStateUpdate1BeTRMod.F90 +++ b/components/elm/src/biogeochem/CNNStateUpdate1BeTRMod.F90 @@ -59,7 +59,7 @@ subroutine NStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp, & associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) cascade_donor_pool => decomp_cascade_con%cascade_donor_pool , & ! Input: [integer (:) ] which pool is C taken from for a given decomposition step cascade_receiver_pool => decomp_cascade_con%cascade_receiver_pool , & ! Input: [integer (:) ] which pool is C added to for a given decomposition step diff --git a/components/elm/src/biogeochem/CNPhenologyBeTRMod.F90 b/components/elm/src/biogeochem/CNPhenologyBeTRMod.F90 index ac3b58538e8..066dcfb62c8 100644 --- a/components/elm/src/biogeochem/CNPhenologyBeTRMod.F90 +++ b/components/elm/src/biogeochem/CNPhenologyBeTRMod.F90 @@ -525,7 +525,7 @@ subroutine CNSeasonDecidPhenology (num_soilp, filter_soilp , & prev_dayl => grc_pp%prev_dayl , & ! Input: [real(r8) (:) ] daylength from previous time step (s) season_decid => veg_vp%season_decid , & ! Input: [real(r8) (:) ] binary flag for seasonal-deciduous leaf habit (0 or 1) - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) t_soisno => col_es%t_soisno , & ! Input: [real(r8) (:,:) ] soil temperature (Kelvin) (-nlevsno+1:nlevgrnd) @@ -875,7 +875,7 @@ subroutine CNStressDecidPhenology (num_soilp, filter_soilp , & dayl => grc_pp%dayl , & ! Input: [real(r8) (:) ] daylength (s) leaf_long => veg_vp%leaf_long , & ! Input: [real(r8) (:) ] leaf longevity (yrs) - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) stress_decid => veg_vp%stress_decid , & ! Input: [real(r8) (:) ] binary flag for stress-deciduous leaf habit (0 or 1) soilpsi => soilstate_vars%soilpsi_col , & ! Input: [real(r8) (:,:) ] soil water potential in each soil layer (MPa) @@ -2295,7 +2295,7 @@ subroutine CNOnsetGrowth (num_soilp, filter_soilp, & associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) onset_flag => cnstate_vars%onset_flag_patch , & ! Input: [real(r8) (:) ] onset flag onset_counter => cnstate_vars%onset_counter_patch , & ! Input: [real(r8) (:) ] onset days counter @@ -2880,7 +2880,7 @@ subroutine CNLivewoodTurnover (num_soilp, filter_soilp, & associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) livewdcn => veg_vp%livewdcn , & ! Input: [real(r8) (:) ] live wood (phloem and ray parenchyma) C:N (gC/gN) deadwdcn => veg_vp%deadwdcn , & ! Input: [real(r8) (:) ] dead wood (xylem and heartwood) C:N (gC/gN) diff --git a/components/elm/src/biogeochem/CarbonStateUpdate1Mod.F90 b/components/elm/src/biogeochem/CarbonStateUpdate1Mod.F90 index 1eb7f64d025..5133575abd3 100644 --- a/components/elm/src/biogeochem/CarbonStateUpdate1Mod.F90 +++ b/components/elm/src/biogeochem/CarbonStateUpdate1Mod.F90 @@ -200,7 +200,7 @@ subroutine CarbonStateUpdate1(bounds, & associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) cascade_donor_pool => decomp_cascade_con%cascade_donor_pool , & ! Input: [integer (:) ] which pool is C taken from for a given decomposition step cascade_receiver_pool => decomp_cascade_con%cascade_receiver_pool , & ! Input: [integer (:) ] which pool is C added to for a given decomposition step harvdate => crop_vars%harvdate_patch & ! Input: [integer (:) ] harvest date diff --git a/components/elm/src/biogeochem/ComputeSeedMod.F90 b/components/elm/src/biogeochem/ComputeSeedMod.F90 index 9b7ebf8838d..2c2f781bdb5 100644 --- a/components/elm/src/biogeochem/ComputeSeedMod.F90 +++ b/components/elm/src/biogeochem/ComputeSeedMod.F90 @@ -183,16 +183,10 @@ subroutine LeafProportions(pft_type, ignore_current_state, & pstorage = 0._r8 pxfer = 0._r8 - if (tot_leaf == 0._r8 .or. ignore_current_state) then - if (veg_vp%evergreen(pft_type) == 1._r8) then - pleaf = 1._r8 - else - pstorage = 1._r8 - end if + if (veg_vp%evergreen(pft_type) == 1._r8) then + pleaf = 1._r8 else - pleaf = leaf /tot_leaf - pstorage = leaf_storage/tot_leaf - pxfer = leaf_xfer /tot_leaf + pstorage = 1._r8 end if end subroutine LeafProportions diff --git a/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 b/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 index ffe1be5d909..1a9b59f776e 100644 --- a/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 +++ b/components/elm/src/biogeochem/EcosystemBalanceCheckMod.F90 @@ -48,7 +48,10 @@ module EcosystemBalanceCheckMod implicit none save private - real(r8), parameter :: balance_check_tolerance = 1e-8_r8 + + ! This corersponds to namelist variable bgc_balance_check_tolerance + real(r8), public :: balance_check_tolerance = 1e-7_r8 + ! ! !PUBLIC MEMBER FUNCTIONS: public :: BeginColCBalance @@ -278,7 +281,7 @@ subroutine ColCBalanceCheck(bounds, & end if ! check for significant errors - if (abs(col_errcb(c)) > 1e-8_r8) then + if (abs(col_errcb(c)) > balance_check_tolerance) then err_found = .true. err_index = c end if @@ -498,7 +501,7 @@ subroutine ColNBalanceCheck(bounds, & ! here is '-' adjustment. It says that the adding to PF decomp n pools was less. end if - if (abs(col_errnb(c)) > 1e-8_r8) then + if (abs(col_errnb(c)) > balance_check_tolerance) then err_found = .true. err_index = c end if @@ -731,7 +734,7 @@ subroutine ColPBalanceCheck(bounds, & col_errpb(c) = (col_pinputs(c) - col_poutputs(c))*dt - & (col_endpb(c) - col_begpb(c)) - if (abs(col_errpb(c)) > 1e-8_r8) then + if (abs(col_errpb(c)) > balance_check_tolerance) then err_found = .true. err_index = c end if diff --git a/components/elm/src/biogeochem/GrowthRespMod.F90 b/components/elm/src/biogeochem/GrowthRespMod.F90 index cb7ad522265..cc6707d723b 100644 --- a/components/elm/src/biogeochem/GrowthRespMod.F90 +++ b/components/elm/src/biogeochem/GrowthRespMod.F90 @@ -51,7 +51,7 @@ subroutine GrowthResp(num_soilp, filter_soilp) associate( & ivt => veg_pp%itype , & ! Input: [integer (:)] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:)] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:)] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) cpool_to_leafc => veg_cf%cpool_to_leafc , & ! Input: [real(r8) (:)] cpool_to_leafc_storage => veg_cf%cpool_to_leafc_storage , & ! Input: [real(r8) (:)] @@ -98,7 +98,7 @@ subroutine GrowthResp(num_soilp, filter_soilp) ) ! Loop through patches - ! start pft loop + ! start pft loop do fp = 1,num_soilp p = filter_soilp(fp) diff --git a/components/elm/src/biogeochem/MaintenanceRespMod.F90 b/components/elm/src/biogeochem/MaintenanceRespMod.F90 index 54005275b29..829e5c742a0 100644 --- a/components/elm/src/biogeochem/MaintenanceRespMod.F90 +++ b/components/elm/src/biogeochem/MaintenanceRespMod.F90 @@ -70,7 +70,7 @@ subroutine readMaintenanceRespParams ( ncid ) call ncd_io(varname=trim(tString),data=tempr, flag='read', ncid=ncid, readvar=readv) if ( .not. readv ) call endrun(msg=trim(errCode)//trim(tString)//errMsg(__FILE__, __LINE__)) br_mr_Inst = tempr - + end subroutine readMaintenanceRespParams !----------------------------------------------------------------------- @@ -107,7 +107,7 @@ subroutine MaintenanceResp(bounds, & associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] patch vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) br_xr => veg_vp%br_xr , & ! Input: [real(r8) (:) ] base rate for excess respiration frac_veg_nosno => canopystate_vars%frac_veg_nosno_patch , & ! Input: [integer (:) ] fraction of vegetation not covered by snow (0 OR 1) [-] laisun => canopystate_vars%laisun_patch , & ! Input: [real(r8) (:) ] sunlit projected leaf area index @@ -160,7 +160,7 @@ subroutine MaintenanceResp(bounds, & ! calculate temperature corrections for each soil layer, for use in ! estimating fine root maintenance respiration with depth tcsoi(c,j) = Q10**((t_soisno(c,j)-SHR_CONST_TKFRZ - 20.0_r8)/10.0_r8) - + end do end do diff --git a/components/elm/src/biogeochem/NitrogenStateUpdate1Mod.F90 b/components/elm/src/biogeochem/NitrogenStateUpdate1Mod.F90 index c1b2bfc5d65..f95b9a258cb 100644 --- a/components/elm/src/biogeochem/NitrogenStateUpdate1Mod.F90 +++ b/components/elm/src/biogeochem/NitrogenStateUpdate1Mod.F90 @@ -130,7 +130,7 @@ subroutine NitrogenStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) cascade_donor_pool => decomp_cascade_con%cascade_donor_pool , & ! Input: [integer (:) ] which pool is C taken from for a given decomposition step cascade_receiver_pool => decomp_cascade_con%cascade_receiver_pool , & ! Input: [integer (:) ] which pool is C added to for a given decomposition step @@ -147,7 +147,7 @@ subroutine NitrogenStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp do j = 1, nlevdecomp do fc = 1,num_soilc c = filter_soilc(fc) - + ! N deposition and fixation (put all into NH4 pool) col_ns%smin_nh4_vr(c,j) = col_ns%smin_nh4_vr(c,j) + col_nf%ndep_to_sminn(c)*dt * ndep_prof(c,j) col_ns%smin_nh4_vr(c,j) = col_ns%smin_nh4_vr(c,j) + col_nf%nfix_to_sminn(c)*dt * nfixation_prof(c,j) @@ -219,7 +219,7 @@ subroutine NitrogenStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp end do end if end do - + do j = 1, nlevdecomp ! column loop do fc = 1,num_soilc @@ -254,8 +254,8 @@ subroutine NitrogenStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp end do ! end of column loop end do - - endif !end if is_active_betr_bgc + + endif !end if is_active_betr_bgc ! forest fertilization call get_curr_date(kyr, kmo, kda, mcsec) diff --git a/components/elm/src/biogeochem/PhenologyFluxLimitMod.F90 b/components/elm/src/biogeochem/PhenologyFluxLimitMod.F90 index 87b9637d551..d80b844e600 100644 --- a/components/elm/src/biogeochem/PhenologyFluxLimitMod.F90 +++ b/components/elm/src/biogeochem/PhenologyFluxLimitMod.F90 @@ -397,7 +397,7 @@ subroutine InitPhenoFluxLimiter() call spm_list_insert(spm_list, -1._r8, f_gresp_storage_to_xfer , s_gresp_storage,nelms) !turn the list into sparse matrix form call spm_list_to_mat(spm_list, spm_carbon_d, nelms, f_gresp_storage_to_xfer) - + !initialize stoichiometric relationship between carbon production flux and corresponding state varaibles call spm_list_init(spm_list, 1._r8, f_cpool_to_leafc , s_leafc, nelms) call spm_list_insert(spm_list, 1._r8, f_leafc_xfer_to_leafc , s_leafc, nelms) @@ -432,7 +432,7 @@ subroutine InitPhenoFluxLimiter() call spm_list_insert(spm_list, 1._r8, f_gresp_storage_to_xfer , s_gresp_xfer,nelms) call spm_list_insert(spm_list, 1._r8, f_cpool_to_gresp_storage , s_gresp_storage, nelms) - + !turn the list into sparse matrix form call spm_list_to_mat(spm_list, spm_carbon_p, nelms, f_gresp_storage_to_xfer) !initialize stoichiometry relationship between nutrient consumption and corresponding state variables @@ -474,7 +474,7 @@ subroutine InitPhenoFluxLimiter() call spm_list_insert(spm_list, -1._r8, f_grainn_to_food , s_grainn, nelms) call spm_list_insert(spm_list, -1._r8, f_grainn_xfer_to_grainn , s_grainn_xfer,nelms) call spm_list_insert(spm_list, -1._r8, f_retransn_to_npool , s_retransn, nelms) - !turn the list into sparse matrix form + !turn the list into sparse matrix form call spm_list_to_mat(spm_list, spm_nutrient_d, nelms, f_supplement_to_plantn) !initialize stoichiometry relationship between nutrient production and corresponding state variables call spm_list_init(spm_list, 1._r8, f_retransn_to_npool , s_npool, nelms) @@ -512,7 +512,7 @@ subroutine InitPhenoFluxLimiter() call spm_list_insert(spm_list, 1._r8, f_frootn_to_retransn , s_retransn, nelms) call spm_list_insert(spm_list, 1._r8, f_livestemn_to_retransn , s_retransn, nelms) call spm_list_insert(spm_list, 1._r8, f_livecrootn_to_retransn , s_retransn, nelms) - !turn the list into sparse matrix form + !turn the list into sparse matrix form call spm_list_to_mat(spm_list, spm_nutrient_p, nelms, f_supplement_to_plantn) end subroutine InitPhenoFluxLimiter !--------------------------------------------------------------------------- @@ -613,7 +613,7 @@ subroutine carbon_flux_limiter(bounds, num_soilc, filter_soilc,& real(r8) :: ar_p associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) harvdate => crop_vars%harvdate_patch & ! Input: [integer (:) ] harvest date ) ! set time steps @@ -847,7 +847,7 @@ subroutine nitrogen_flux_limiter(bounds, num_soilc, filter_soilc,& real(r8) :: dt associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) nf => veg_nf , & ns => veg_ns & ) @@ -1031,7 +1031,7 @@ subroutine phosphorus_flux_limiter(bounds, num_soilc, filter_soilc,& associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) pf => veg_pf , & ps => veg_ps & ) diff --git a/components/elm/src/biogeochem/PhenologyMod.F90 b/components/elm/src/biogeochem/PhenologyMod.F90 index 08e9b94da52..c46e251b60c 100644 --- a/components/elm/src/biogeochem/PhenologyMod.F90 +++ b/components/elm/src/biogeochem/PhenologyMod.F90 @@ -163,7 +163,7 @@ subroutine readPhenolParams ( ncid ) allocate(PhenolParamsInst%lwtop ) ! ! read in parameters - ! + ! tString='crit_dayl' call ncd_io(varname=trim(tString),data=tempr, flag='read', ncid=ncid, readvar=readv) if ( .not. readv ) call endrun( msg=trim(errCode)//trim(tString)//errMsg(__FILE__, __LINE__)) @@ -175,7 +175,7 @@ subroutine readPhenolParams ( ncid ) else tString='crit_dayl_stress' call ncd_io(varname=trim(tString),data=tempr, flag='read', ncid=ncid, readvar=readv) - if ( .not. readv ) then + if ( .not. readv ) then PhenolParamsInst%crit_dayl_stress = secspqtrday else PhenolParamsInst%crit_dayl_stress = tempr @@ -238,7 +238,7 @@ subroutine readPhenolParams ( ncid ) tString='lwtop_ann' call ncd_io(varname=trim(tString),data=tempr, flag='read', ncid=ncid, readvar=readv) if ( .not. readv ) call endrun( msg=trim(errCode)//trim(tString)//errMsg(__FILE__, __LINE__)) - PhenolParamsInst%lwtop=tempr + PhenolParamsInst%lwtop=tempr !!!!========== Update to device ========= !!! !$acc update device(PhenolParamsInst%crit_dayl, & @@ -601,7 +601,7 @@ subroutine CNSeasonDecidPhenology (num_soilp, filter_soilp, cnstate_vars) prev_dayl => grc_pp%prev_dayl , & ! Input: [real(r8) (:) ] daylength from previous time step (s) season_decid => veg_vp%season_decid , & ! Input: [real(r8) (:) ] binary flag for seasonal-deciduous leaf habit (0 or 1) - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) t_soisno => col_es%t_soisno , & ! Input: [real(r8) (:,:) ] soil temperature (Kelvin) (-nlevsno+1:nlevgrnd) @@ -946,7 +946,7 @@ subroutine CNStressDecidPhenology (num_soilp, filter_soilp , & leaf_long => veg_vp%leaf_long , & ! Input: [real(r8) (:) ] leaf longevity (yrs) froot_long => veg_vp%froot_long , & ! Input: [real(r8) (:) ] fine root longevity (yrs) - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) stress_decid => veg_vp%stress_decid , & ! Input: [real(r8) (:) ] binary flag for stress-deciduous leaf habit (0 or 1) soilpsi => soilstate_vars%soilpsi_col , & ! Input: [real(r8) (:,:) ] soil water potential in each soil layer (MPa) @@ -1445,7 +1445,7 @@ subroutine CropPhenology(num_pcropp, filter_pcropp, & froot_long => veg_vp%froot_long , & ! Input: [real(r8) (:) ] fine root longevity (yrs) leafcn => veg_vp%leafcn , & ! Input: [real(r8) (:) ] leaf C:N (gC/gN) - manunitro => veg_vp%manunitro , & ! Input: max manure to apply (kgN/m2) + manunitro => veg_vp%manunitro , & ! Input: max manure to apply (kgN/m2) t_ref2m_min => veg_es%t_ref2m_min , & ! Input: [real(r8) (:) ] daily minimum of average 2 m height surface air temperature (K) t10 => veg_es%t_a10 , & ! Input: [real(r8) (:) ] 10-day running mean of the 2 m temperature (K) a5tmin => veg_es%t_a5min , & ! Input: [real(r8) (:) ] 5-day running mean of min 2-m temperature @@ -1988,7 +1988,7 @@ subroutine PerennialCropPhenology(num_ppercropp, filter_ppercropp, & froot_long => veg_vp%froot_long , & ! Input: [real(r8) (:) ] fine root longevity (yrs) leafcn => veg_vp%leafcn , & ! Input: [real(r8) (:) ] leaf C:N (gC/gN) leafcp => veg_vp%leafcp , & ! Input: [real(r8) (:) ] leaf C:P (gC/gP) - manunitro => veg_vp%manunitro , & ! Input: max manure to apply (kgN/m2) + manunitro => veg_vp%manunitro , & ! Input: max manure to apply (kgN/m2) t10 => veg_es%t_a10 , & ! Input: [real(r8) (:) ] 10-day running mean of the 2 m temperature (K) a10tmin => veg_es%t_a10min , & ! Input: [real(r8) (:) ] 10-day running mean of min 2-m temperature fertnitro => crop_vars%fertnitro_patch , & ! Input: [real(r8) (:) ] max fertilizer to be applied in total (kgN/m2) @@ -2016,8 +2016,8 @@ subroutine PerennialCropPhenology(num_ppercropp, filter_ppercropp, & crop_seedc_to_leaf => veg_cf%crop_seedc_to_leaf , & ! Output: [real(r8) (:) ] (gC/m2/s) seed source to PFT-level - synthfert => veg_nf%synthfert , & ! Output: [real(r8) (:) ] (gN/m2/s) fertilizer applied each timestep - manure => veg_nf%manure , & ! Output: [real(r8) (:) ] (gN/m2/s) manure applied each timestep + synthfert => veg_nf%synthfert , & ! Output: [real(r8) (:) ] (gN/m2/s) fertilizer applied each timestep + manure => veg_nf%manure , & ! Output: [real(r8) (:) ] (gN/m2/s) manure applied each timestep fert_p => veg_pf%fert_p , & ! Output: [real(r8) (:) ] (gP/m2/s) phosphorus fertilizer applied each timestep fert_counter => veg_nf%fert_counter , & ! Output: [real(r8) (:) ] >0 fertilize; <=0 not (seconds) @@ -2550,22 +2550,22 @@ subroutine CropPlantDate (num_soilp, filter_soilp, num_pcropp, filter_pcropp, & xt(p,kmo) = xt(p,kmo) + t_ref2m(p) * fracday/ndaypm(kmo) ! monthly average temperature xp(p,kmo) = xp(p,kmo) + (forc_rain(t)+forc_snow(t))*dt ! monthly average precipitation - ! calculate the potential evapotranspiration + ! calculate the potential evapotranspiration netrad = fsa(p) + eflx_lwrad_net(p) ! moved this here because it is calculated too late call calculate_eto(t_ref2m(p), netrad, eflx_soil_grnd(p), forc_pbot(t), forc_rh(t), forc_wind(t), dt, ETout) ! monthly ETo ETo(p,kmo) = ETo(p,kmo) + ETout - + ! calculate the P:PET for each month - if ( abs(ETo(p,kmo)) > 0._r8) then + if ( abs(ETo(p,kmo)) > 0._r8) then p2ETo(p,kmo) = xp(p,kmo)/ETo(p,kmo) else ! P:PET is undefined. ! Setting to a fill value ( 'spval' ) would - ! require nested if statements due to + ! require nested if statements due to ! the weighting of previous years (i.e., p2ETo and prev_p2ETo_bar ) ! So, set to zero for simplicity. p2ETo(p,kmo) = 0._r8 - end if + end if if (nyrs_crop_active(p) == 0) then ! for the first year, use last years values prev_xt_bar(p,kmo) = xt(p,kmo) @@ -2617,7 +2617,7 @@ subroutine CNOnsetGrowth (num_soilp, filter_soilp, & associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) onset_flag => cnstate_vars%onset_flag_patch , & ! Input: [real(r8) (:) ] onset flag onset_counter => cnstate_vars%onset_counter_patch , & ! Input: [real(r8) (:) ] onset days counter @@ -3277,7 +3277,7 @@ subroutine CNLivewoodTurnover (num_soilp, filter_soilp) associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) livewdcn => veg_vp%livewdcn , & ! Input: [real(r8) (:) ] live wood (phloem and ray parenchyma) C:N (gC/gN) deadwdcn => veg_vp%deadwdcn , & ! Input: [real(r8) (:) ] dead wood (xylem and heartwood) C:N (gC/gN) diff --git a/components/elm/src/biogeochem/PhosphorusStateUpdate1Mod.F90 b/components/elm/src/biogeochem/PhosphorusStateUpdate1Mod.F90 index 6e111116922..9297ab41e1b 100644 --- a/components/elm/src/biogeochem/PhosphorusStateUpdate1Mod.F90 +++ b/components/elm/src/biogeochem/PhosphorusStateUpdate1Mod.F90 @@ -75,7 +75,7 @@ subroutine PhosphorusStateUpdateDynPatch(bounds, num_soilc_with_inactive,& c = filter_soilc_with_inactive(fc) col_ps%prod10p(c) = col_ps%prod10p(c) + col_pf%dwt_prod10p_gain(c)*dt col_ps%prod100p(c) = col_ps%prod100p(c) + col_pf%dwt_prod100p_gain(c)*dt - col_ps%prod1p(c) = col_ps%prod1p(c) + col_pf%dwt_crop_productp_gain(c)*dt + col_ps%prod1p(c) = col_ps%prod1p(c) + col_pf%dwt_crop_productp_gain(c)*dt do j = 1,nlevdecomp @@ -125,7 +125,7 @@ subroutine PhosphorusStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soi associate( & ivt => veg_pp%itype , & ! Input: [integer (:) ] pft vegetation type - woody => veg_vp%woody , & ! Input: [real(r8) (:) ] binary flag for woody lifeform (1=woody, 0=not woody) + woody => veg_vp%woody , & ! Input: [real(r8) (:) ] woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) cascade_donor_pool => decomp_cascade_con%cascade_donor_pool , & ! Input: [integer (:) ] which pool is C taken from for a given decomposition step cascade_receiver_pool => decomp_cascade_con%cascade_receiver_pool , & ! Input: [integer (:) ] which pool is C added to for a given decomposition step @@ -147,18 +147,18 @@ subroutine PhosphorusStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soi do j = 1, nlevdecomp do fc = 1,num_soilc c = filter_soilc(fc) - + ! plant to litter fluxes ! phenology and dynamic landcover fluxes col_pf%decomp_ppools_sourcesink(c,j,i_met_lit) = & col_pf%phenology_p_to_litr_met_p(c,j) * dt - + col_pf%decomp_ppools_sourcesink(c,j,i_cel_lit) = & col_pf%phenology_p_to_litr_cel_p(c,j) * dt - + col_pf%decomp_ppools_sourcesink(c,j,i_lig_lit) = & col_pf%phenology_p_to_litr_lig_p(c,j) * dt - + end do end do end if diff --git a/components/elm/src/biogeochem/VegStructUpdateMod.F90 b/components/elm/src/biogeochem/VegStructUpdateMod.F90 index 5943da355a0..74e135f6f0e 100644 --- a/components/elm/src/biogeochem/VegStructUpdateMod.F90 +++ b/components/elm/src/biogeochem/VegStructUpdateMod.F90 @@ -40,6 +40,7 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & use pftvarcon , only : noveg, woody, iscft, crop use pftvarcon , only : ncorn, ncornirrig, ztopmx, laimx use pftvarcon , only : nmiscanthus, nmiscanthusirrig, nswitchgrass, nswitchgrassirrig + use pftvarcon , only : bendresist, vegshape, stocking, taper use elm_time_manager , only : get_rad_step_size use elm_varctl , only : spinup_state, spinup_mortality_factor ! @@ -58,8 +59,6 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & ! !LOCAL VARIABLES: integer :: p,c,g ! indices integer :: fp ! lake filter indices - real(r8) :: taper ! ratio of height:radius_breast_height (tree allometry) - real(r8) :: stocking ! #stems / ha (stocking density) real(r8) :: ol ! thickness of canopy layer covered by snow (m) real(r8) :: fb ! fraction of canopy layer covered by snow real(r8) :: tlai_old ! for use in Zeng tsai formula @@ -88,6 +87,9 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & z0mr => veg_vp%z0mr , & ! Input: [real(r8) (:) ] ratio of momentum roughness length to canopy top height (-) displar => veg_vp%displar , & ! Input: [real(r8) (:) ] ratio of displacement height to canopy top height (-) dwood => veg_vp%dwood , & ! Input: [real(r8) (:) ] density of wood (gC/m^3) + bendresist => veg_vp%bendresist , & ! Input: [real(r8) (:) ] resistance to bending under snow loading Sturm et al. 2005 (-) [0,1] + vegshape => veg_vp%vegshape , & ! Input: [real(r8) (:) ] vegetation shape for calculating snow buried fraction only (-) (Liston and Heimstra, 2011) + stocking => veg_vp%stocking , & ! Input: [real(r8) (:) ] Stocking density [stems / hectare] snow_depth => col_ws%snow_depth , & ! Input: [real(r8) (:) ] snow height (m) @@ -113,13 +115,6 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & dt = real( get_rad_step_size(), r8 ) - ! constant allometric parameters - taper = 200._r8 - stocking = 1000._r8 - - ! convert from stems/ha -> stems/m^2 - stocking = stocking / 10000._r8 - ! patch loop do fp = 1,num_soilp p = filter_soilp(fp) @@ -160,24 +155,16 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & if (woody(ivt(p)) >= 1.0_r8) then - ! trees and shrubs - - ! if shrubs have a squat taper - if (woody(ivt(p)) == 2.0_r8) then - taper = 10._r8 - ! otherwise have a tall taper - else - taper = 200._r8 - end if - - ! trees and shrubs for now have a very simple allometry, with hard-wired - ! stem taper (height:radius) and hard-wired stocking density (#individuals/area) + ! trees and shrubs currently have a very simple allometry, based on + ! stem taper (height:radius) and stocking density (#individuals/area) + ! taper and stocking density can be set as input variables now to + ! change from default values set in pftvarcon.F90 if (spinup_state >= 1) then - htop(p) = ((3._r8 * deadstemc(p) * spinup_mortality_factor * taper * taper)/ & - (SHR_CONST_PI * stocking * dwood(ivt(p))))**(1._r8/3._r8) + htop(p) = ((3._r8 * deadstemc(p) * spinup_mortality_factor * taper(ivt(p)) * taper(ivt(p)))/ & + (SHR_CONST_PI * stocking(ivt(p)) * dwood(ivt(p))))**(1._r8/3._r8) else - htop(p) = ((3._r8 * deadstemc(p) * taper * taper)/ & - (SHR_CONST_PI * stocking * dwood(ivt(p))))**(1._r8/3._r8) + htop(p) = ((3._r8 * deadstemc(p) * taper(ivt(p)) * taper(ivt(p)))/ & + (SHR_CONST_PI * stocking(ivt(p)) * dwood(ivt(p))))**(1._r8/3._r8) end if ! Peter Thornton, 5/3/2004 @@ -246,10 +233,12 @@ subroutine VegStructUpdate(num_soilp, filter_soilp, & ! adjust lai and sai for burying by snow. ! snow burial fraction for short vegetation (e.g. grasses) as in - ! Wang and Zeng, 2007. - if (woody(ivt(p)) >= 1.0_r8 ) then + ! Wang and Zeng et al 2007. + ! Taller vegetation (trees and shrubs) have been updated to use formulation similar to + ! Sturm et al. 2005; Liston and Hiemstra, 2011; and Belke-Brea et al. 2020 + if ( woody(ivt(p)) >= 1.0_r8 ) then ol = min( max(snow_depth(c)-hbot(p), 0._r8), htop(p)-hbot(p)) - fb = 1._r8 - ol / max(1.e-06_r8, htop(p)-hbot(p)) + fb = 1._r8 - (ol / max(1.e-06_r8, bendresist(ivt(p)) * (htop(p)-hbot(p)))) ** vegshape(ivt(p)) else fb = 1._r8 - max(min(snow_depth(c),0.2_r8),0._r8)/0.2_r8 ! 0.2m is assumed !depth of snow required for complete burial of grasses diff --git a/components/elm/src/biogeochem/VerticalProfileMod.F90 b/components/elm/src/biogeochem/VerticalProfileMod.F90 index 3f5ced72f33..1fa3db093b4 100644 --- a/components/elm/src/biogeochem/VerticalProfileMod.F90 +++ b/components/elm/src/biogeochem/VerticalProfileMod.F90 @@ -25,9 +25,9 @@ module VerticalProfileMod logical , public :: exponential_rooting_profile = .true. logical , public :: pftspecific_rootingprofile = .true. ! how steep profile is for root C inputs (1/ e-folding depth) (1/m) - real(r8), public :: rootprof_exp = 3. + real(r8), public :: rootprof_exp = 3._r8 ! how steep profile is for surface components (1/ e_folding depth) (1/m) - real(r8), public :: surfprof_exp = 10. + real(r8), public :: surfprof_exp = 10._r8 !----------------------------------------------------------------------- contains @@ -83,7 +83,8 @@ subroutine decomp_vertprofiles(bounds, & real(r8) :: ndep_prof_sum real(r8) :: nfixation_prof_sum real(r8) :: pdep_prof_sum - real(r8) :: delta = 1.e-10 + real(r8) :: delta = 1.e-10_r8 + real(r8), parameter :: smallparameter = tiny(1._r8) character(len=32) :: subname = 'decomp_vertprofiles' !----------------------------------------------------------------------- @@ -191,7 +192,7 @@ subroutine decomp_vertprofiles(bounds, & surface_prof_tot = surface_prof_tot + surface_prof(j) * dzsoi_decomp(j) end if end do - if ( (altmax_lastyear_indx(c) > 0) .and. (rootfr_tot > 0._r8) .and. (surface_prof_tot > 0._r8) ) then + if ( (altmax_lastyear_indx(c) > 0) .and. (rootfr_tot > smallparameter) .and. (surface_prof_tot > smallparameter) ) then ! where there is not permafrost extending to the surface, integrate the profiles over the active layer ! this is equivalnet to integrating over all soil layers outside of permafrost regions do j = 1, min(max(altmax_lastyear_indx(c), 1), nlevdecomp) @@ -212,10 +213,10 @@ subroutine decomp_vertprofiles(bounds, & end do else ! if fully frozen, or no roots, put everything in the top layer - froot_prof(p,1) = 1./dzsoi_decomp(1) - croot_prof(p,1) = 1./dzsoi_decomp(1) - leaf_prof(p,1) = 1./dzsoi_decomp(1) - stem_prof(p,1) = 1./dzsoi_decomp(1) + froot_prof(p,1) = 1._r8/dzsoi_decomp(1) + croot_prof(p,1) = 1._r8/dzsoi_decomp(1) + leaf_prof(p,1) = 1._r8/dzsoi_decomp(1) + stem_prof(p,1) = 1._r8/dzsoi_decomp(1) endif end do @@ -250,19 +251,19 @@ subroutine decomp_vertprofiles(bounds, & surface_prof_tot = surface_prof_tot + surface_prof(j) * dzsoi_decomp(j) end do if(col_pp%is_fates(c))then - if ( (altmax_lastyear_indx(c) > 0) .and. (surface_prof_tot > 0._r8) ) then + if ( (altmax_lastyear_indx(c) > 0) .and. (surface_prof_tot > smallparameter) ) then do j = 1,min(alt_ind, nlevbed) nfixation_prof(c,j) = surface_prof(j)/ surface_prof_tot ndep_prof(c,j) = surface_prof(j)/ surface_prof_tot pdep_prof(c,j) = surface_prof(j)/ surface_prof_tot end do else - nfixation_prof(c,1) = 1./dzsoi_decomp(1) - ndep_prof(c,1) = 1./dzsoi_decomp(1) - pdep_prof(c,1) = 1./dzsoi_decomp(1) + nfixation_prof(c,1) = 1._r8/dzsoi_decomp(1) + ndep_prof(c,1) = 1._r8/dzsoi_decomp(1) + pdep_prof(c,1) = 1._r8/dzsoi_decomp(1) endif else - if ( (altmax_lastyear_indx(c) > 0) .and. (rootfr_tot > 0._r8) .and. (surface_prof_tot > 0._r8) ) then + if ( (altmax_lastyear_indx(c) > 0) .and. (rootfr_tot > smallparameter) .and. (surface_prof_tot > smallparameter) ) then do j = 1, min(max(altmax_lastyear_indx(c), 1), nlevdecomp) nfixation_prof(c,j) = col_cinput_rootfr(c,j) / rootfr_tot if (j <= nlevbed) then @@ -271,9 +272,9 @@ subroutine decomp_vertprofiles(bounds, & end if end do else - nfixation_prof(c,1) = 1./dzsoi_decomp(1) - ndep_prof(c,1) = 1./dzsoi_decomp(1) - pdep_prof(c,1) = 1./dzsoi_decomp(1) + nfixation_prof(c,1) = 1._r8/dzsoi_decomp(1) + ndep_prof(c,1) = 1._r8/dzsoi_decomp(1) + pdep_prof(c,1) = 1._r8/dzsoi_decomp(1) endif end if end do @@ -294,9 +295,9 @@ subroutine decomp_vertprofiles(bounds, & ! check to make sure integral of all profiles = 1. do fc = 1,num_soilc c = filter_soilc(fc) - ndep_prof_sum = 0. - nfixation_prof_sum = 0. - pdep_prof_sum = 0. + ndep_prof_sum = 0._r8 + nfixation_prof_sum = 0._r8 + pdep_prof_sum = 0._r8 do j = 1, nlevdecomp ndep_prof_sum = ndep_prof_sum + ndep_prof(c,j) * dzsoi_decomp(j) nfixation_prof_sum = nfixation_prof_sum + nfixation_prof(c,j) * dzsoi_decomp(j) @@ -324,10 +325,10 @@ subroutine decomp_vertprofiles(bounds, & do fp = 1,num_soilp p = filter_soilp(fp) - froot_prof_sum = 0. - croot_prof_sum = 0. - leaf_prof_sum = 0. - stem_prof_sum = 0. + froot_prof_sum = 0._r8 + croot_prof_sum = 0._r8 + leaf_prof_sum = 0._r8 + stem_prof_sum = 0._r8 do j = 1, nlevdecomp froot_prof_sum = froot_prof_sum + froot_prof(p,j) * dzsoi_decomp(j) croot_prof_sum = croot_prof_sum + croot_prof(p,j) * dzsoi_decomp(j) @@ -336,7 +337,19 @@ subroutine decomp_vertprofiles(bounds, & end do if ( ( abs(froot_prof_sum - 1._r8) > delta ) .or. ( abs(croot_prof_sum - 1._r8) > delta ) .or. & ( abs(stem_prof_sum - 1._r8) > delta ) .or. ( abs(leaf_prof_sum - 1._r8) > delta ) ) then + c = veg_pp%column(p) write(iulog, *) 'profile sums: ', froot_prof_sum, croot_prof_sum, leaf_prof_sum, stem_prof_sum + write(iulog, *) 'c: ',c + write(iulog, *) 'altmax_lastyear_indx: ', altmax_lastyear_indx(c) + write(iulog, *) 'cinput_rootfr: ', col_cinput_rootfr(c,:) + write(iulog, *) 'dzsoi_decomp: ', dzsoi_decomp(:) + write(iulog, *) 'surface_prof: ', surface_prof(:) + write(iulog, *) 'p, itype(p), wtcol(p): ', p, veg_pp%itype(p), veg_pp%wtcol(p) + write(iulog, *) 'cinput_rootfr(p,:): ', cinput_rootfr(p,:) + write(iulog,*) 'croot_prof(p,:): ',croot_prof(p,:) + write(iulog,*) 'froot_prof(p,:): ',froot_prof(p,:) + write(iulog,*) 'leaf_prof(p,:): ',leaf_prof(p,:) + write(iulog,*) 'stem_prof(p,:): ',stem_prof(p,:) call endrun(msg=' ERROR: sum-1 > delta'//errMsg(__FILE__, __LINE__)) endif end do diff --git a/components/elm/src/cpl/lnd_downscale_atm_forcing.F90 b/components/elm/src/cpl/lnd_downscale_atm_forcing.F90 index 99b03a2e25a..3b69158f49e 100644 --- a/components/elm/src/cpl/lnd_downscale_atm_forcing.F90 +++ b/components/elm/src/cpl/lnd_downscale_atm_forcing.F90 @@ -35,10 +35,13 @@ module lnd_downscale_atm_forcing ! ! !PUBLIC MEMBER FUNCTIONS: public :: downscale_atm_forcing_to_topounit ! Calls downscaling subroutines of forcing fields from gridcell to topounit + public :: downscale_atm_forcing_to_topounit_cpl_bypass ! Calls downscaling subroutines of forcing fields from gridcell to topounit, for use within the CPL_BYPASS code ! ! !PRIVATE MEMBER FUNCTIONS: private :: downscale_atm_state_to_topounit ! Downscale atmosperic state fields from gridcell to topounit + private :: downscale_atm_state_to_topounit_cpl_bypass ! Downscale atmosperic state fields from gridcell to topounit, for cpl_bypass code private :: downscale_longwave_to_topounit ! Downscale longwave radiation field from gridcell to topounit + private :: downscale_longwave_to_topounit_cpl_bypass ! Downscale longwave radiation field from gridcell to topounit, for cpl_bypass code private :: downscale_precip_to_topounit_FNM ! Downscale precipitation field from gridcell to topounit using Froude number method (FNM) private :: downscale_precip_to_topounit_ERMM ! Downscale precipitation field from gridcell to topounit using elevation ration with maximum elevation method (ERMM) private :: build_normalization ! Compute normalization factors so that downscaled fields are conservative @@ -787,4 +790,526 @@ subroutine build_normalization(orig_field, sum_field, sum_wts, norms) end subroutine build_normalization + subroutine downscale_atm_forcing_to_topounit_cpl_bypass(g, atm2lnd_vars, lnd2atm_vars) + ! + ! !DESCRIPTION: + ! Downscale fields from gridcell to topounit + ! + ! Downscaling is done over topounits if the number of topounits > 1. + ! + ! !USES: + use elm_time_manager, only : get_nstep + use elm_varcon , only : rair, cpair, grav, lapse_glcmec + use elm_varcon , only : glcmec_rain_snow_threshold, o2_molar_const + use shr_const_mod , only : SHR_CONST_TKFRZ + use landunit_varcon , only : istice_mec + use elm_varctl , only : glcmec_downscale_rain_snow_convert + use domainMod , only : ldomain + use QsatMod , only : Qsat + use FrictionVelocityMod, only: atm_gustiness + ! + ! !ARGUMENTS: + integer , intent(in) :: g + type(atm2lnd_type) , intent(in) :: atm2lnd_vars + type(lnd2atm_type) , intent(in) :: lnd2atm_vars + + ! + ! !LOCAL VARIABLES: + integer :: t, l, c, fc, t2 ! indices + integer :: clo, cc + integer :: numt_pg ! Number of topounits per grid + + ! temporaries for topo downscaling + real(r8) :: rain_g, snow_g + real(r8) :: mxElv ! Maximum elevation value per grid + real(r8) :: uovern_t ! Froude Number + real(r8) :: grdElv ! Grid elevation + real(r8) :: topoElv ! Topounit elevation + real(r8) :: max_tpuElv ! Maximum topounit elevation for calculating elevation range + real(r8) :: min_tpuElv ! Minimum topounit elevation for calculating elevation range + real(r8) :: crnt_temp_t ! Current downscaled topounit temperature + real(r8) :: temp_r ! Temporary topounit rainfall + real(r8) :: temp_s ! Temporary topounit snowfall + real(r8) :: t_th ! Temperature threshold for snowfall + real(r8) :: Ta_th1 ! Temperature at 0.5 Celsius in K 273.65 + real(r8) :: Ta_th2 ! Temperature at 2.0 Celsius in K 275.15 + real(r8) :: Ta_th3 ! Temperature at 2.5 Celsius in K 275.65 + real(r8) :: tmp_Snow_frc ! Current snow fraction + + real(r8) :: e + real(r8) :: qvsat + + real(r8) :: sum_qbot_g ! weighted sum of column-level lwrad + real(r8) :: sum_wtsq_g ! sum of weights that contribute to sum_lwrad_g + real(r8) :: qbot_norm_g ! normalization factors + real(r8) :: sum_lwrad_g ! weighted sum of column-level lwrad + real(r8) :: sum_wtslw_g ! sum of weights that contribute to sum_lwrad_g + real(r8) :: lwrad_norm_g ! normalization factors + real(r8) :: esatw ! saturation vapor pressure over water (Pa) + real(r8) :: esati ! saturation vapor pressure over ice (Pa) + real(r8) :: a0,a1,a2,a3,a4,a5,a6 ! coefficients for esat over water + real(r8) :: b0,b1,b2,b3,b4,b5,b6 ! coefficients for esat over ice + real(r8) :: tdc, temp ! Kelvins to Celcius function and its input + real(r8) :: vp ! water vapor pressure (Pa) + + real(r8), allocatable :: deltaRain(:) ! Deviation of subgrid rain from grid rain + real(r8), allocatable :: deltaSnow(:) ! Deviation of subgrid snow from grid snow + real(r8) :: deltaR ! Temporary deltaRain + real(r8) :: deltaS ! Temporary deltaSnow + real(r8) :: sum_of_hrise ! Sum of height rise of air parcel of all subgrids of a grid + real(r8) :: hrise ! Temporary height rise + real(r8) :: elvrnge ! Elevation range between lowest and highest tpu + real(r8) :: ave_elv + integer :: elv_flag ! Elevation flag to trac grids with +ve grid elevation and -ve tpu elevation + + integer :: uaflag = 0 + integer :: precip_dwn = 0 ! Used to turn on/off the downscaling of precipitation 0 = on; 1 = off + integer :: other_forcing_dwn = 0 ! Used to turn on/off the downscaling of other forcing 0 = on; 1 = off + + character(len=*), parameter :: subname = 'downscale_atm_forcing_to_topounit' + !---------------------------------------------------------------------------------------- + + ! Constants to compute vapor pressure + parameter (a0=6.107799961_r8 , a1=4.436518521e-01_r8, & + a2=1.428945805e-02_r8, a3=2.650648471e-04_r8, & + a4=3.031240396e-06_r8, a5=2.034080948e-08_r8, & + a6=6.136820929e-11_r8) + + parameter (b0=6.109177956_r8 , b1=5.034698970e-01_r8, & + b2=1.886013408e-02_r8, b3=4.176223716e-04_r8, & + b4=5.824720280e-06_r8, b5=4.838803174e-08_r8, & + b6=1.838826904e-10_r8) + + ! + ! function declarations + ! + tdc(temp) = min( 50._r8, max(-50._r8,(temp-SHR_CONST_TKFRZ)) ) ! Taken from lnd_import_export.F90 + esatw(temp) = 100._r8*(a0+temp*(a1+temp*(a2+temp*(a3+temp*(a4+temp*(a5+temp*a6)))))) ! Taken from lnd_import_export.F90 + esati(temp) = 100._r8*(b0+temp*(b1+temp*(b2+temp*(b3+temp*(b4+temp*(b5+temp*b6)))))) ! Taken from lnd_import_export.F90 + !----------------------------------------------------------------------- + ! Get required inputs + numt_pg = grc_pp%ntopounits(g) ! Number of topounits per grid + grdElv = grc_pp%elevation(g) ! Grid level sfc elevation + mxElv = grc_pp%MaxElevation(g) ! Maximum src elevation per grid obtained from the highest elevation topounit + uovern_t = atm2lnd_vars%forc_uovern(g) ! Froude Number + + snow_g = atm2lnd_vars%forc_snow_not_downscaled_grc(g) + rain_g = atm2lnd_vars%forc_rain_not_downscaled_grc(g) + + sum_qbot_g = 0._r8 + sum_wtsq_g = 0._r8 + sum_lwrad_g = 0._r8 + sum_wtslw_g = 0._r8 + + sum_of_hrise = 0._r8 + t_th = 273.15_r8 ! Freezing temperature in K + Ta_th1 = 273.65_r8 ! Lowest threshold for snow calculation + Ta_th2 = 275.15_r8 ! Middle threshold for rain/snow partitioning + Ta_th3 = 275.65_r8 ! Highest threshold for rain/snow partitioning + tmp_Snow_frc = 0._r8 ! Snow fraction + elv_flag = 0 + elvrnge = 0._r8 + ave_elv = 0._r8 + max_tpuElv = 0._r8 + min_tpuElv = 0._r8 + if (numt_pg > 1) then !downscaling is done only if a grid has more than 1 topounits + if (precip_downscaling_method == 'FNM') then + allocate(deltaRain(numt_pg)) + deltaRain(:) = 0._r8 + allocate(deltaSnow(numt_pg)) + deltaSnow(:) = 0._r8 + hrise = 0._r8 + end if + + ! calculate elevation range and track grids with +ve elevation but have -ve tpu elevation + max_tpuElv = -100000._r8 + min_tpuElv = 100000._r8 + do t = grc_pp%topi(g), grc_pp%topf(g) ! Check occurence of grid elevation is +ve while tpu elevation is -ve + topoElv = top_pp%elevation(t) + if (topoElv > max_tpuElv) then + max_tpuElv = topoElv + end if + if (topoElv < min_tpuElv) then + min_tpuElv = topoElv + end if + end do + elvrnge = max_tpuElv - min_tpuElv + + do t = grc_pp%topi(g), grc_pp%topf(g) + t2 = t - grc_pp%topi(g) + 1 + topoElv = top_pp%elevation(t) ! Topounit sfc elevation + + ! Downscale precipitation + if (mxElv == 0._r8 .or. precip_dwn == 1) then ! avoid dividing by 0 + top_af%rain(t) = rain_g + top_af%snow(t) = snow_g + else + if (precip_downscaling_method == 'FNM') then + call downscale_precip_to_topounit_FNM(mxElv,uovern_t,grdElv,topoElv,rain_g,snow_g,deltaR,deltaS,hrise) !Use FNM method + deltaRain(t2) = deltaR + deltaSnow(t2) = deltaS + sum_of_hrise = sum_of_hrise + hrise + else + call downscale_precip_to_topounit_ERMM(t,mxElv,grdElv,topoElv,rain_g, snow_g,elv_flag,elvrnge) ! Use ERMM method + end if + end if + + ! Downscale other fluxes + if (other_forcing_dwn == 1) then ! flag to turn on or off downscaling of other forcing + top_af%rain(t) = rain_g + top_af%snow(t) = snow_g + top_af%lwrad(t) = atm2lnd_vars%forc_lwrad_not_downscaled_grc(g) + ! Update top_as + top_as%tbot(t) = atm2lnd_vars%forc_t_not_downscaled_grc(g) ! forc_txy Atm state K + top_as%thbot(t) = atm2lnd_vars%forc_th_not_downscaled_grc(g) ! forc_thxy Atm state K + top_as%pbot(t) = atm2lnd_vars%forc_pbot_not_downscaled_grc(g) ! ptcmxy Atm state Pa + top_as%qbot(t) = atm2lnd_vars%forc_q_not_downscaled_grc(g) ! forc_qxy Atm state kg/kg + top_as%ubot(t) = atm2lnd_vars%forc_u_grc(g) ! forc_uxy Atm state m/s + top_as%vbot(t) = atm2lnd_vars%forc_v_grc(g) ! forc_vxy Atm state m/s + top_as%zbot(t) = atm2lnd_vars%forc_hgt_grc(g) ! zgcmxy Atm state m + + ! assign the state forcing fields derived from other inputs + ! Horizontal windspeed (m/s) + top_as%windbot(t) = sqrt(top_as%ubot(t)**2 + top_as%vbot(t)**2) + ! Relative humidity (percent) + if (top_as%tbot(t) > SHR_CONST_TKFRZ) then + e = esatw(tdc(top_as%tbot(t))) + else + e = esati(tdc(top_as%tbot(t))) + end if + qvsat = 0.622_r8*e / (top_as%pbot(t) - 0.378_r8*e) + top_as%rhbot(t) = 100.0_r8*(top_as%qbot(t) / qvsat) + ! partial pressure of oxygen (Pa) + top_as%po2bot(t) = o2_molar_const * top_as%pbot(t) + ! air density (kg/m**3) - uses a temporary calculation + ! of water vapor pressure (Pa) + vp = top_as%qbot(t) * top_as%pbot(t) / (0.622_r8 + 0.378_r8 * top_as%qbot(t)) + top_as%rhobot(t) = (top_as%pbot(t) - 0.378_r8 * vp) / (rair * top_as%tbot(t)) + + top_af%solad(t,2) = atm2lnd_vars%forc_solad_grc(g,2) + top_af%solad(t,1) = atm2lnd_vars%forc_solad_grc(g,1) + top_af%solai(t,2) = atm2lnd_vars%forc_solai_grc(g,2) + top_af%solai(t,1) = atm2lnd_vars%forc_solai_grc(g,1) + ! derived flux forcings + top_af%solar(t) = top_af%solad(t,2) + top_af%solad(t,1) + & + top_af%solai(t,2) + top_af%solai(t,1) + else + call downscale_atm_state_to_topounit_cpl_bypass(g, t, atm2lnd_vars) + call downscale_longwave_to_topounit_cpl_bypass(g, t, atm2lnd_vars) + top_as%ubot(t) = atm2lnd_vars%forc_u_grc(g) ! forc_uxy Atm state m/s + top_as%vbot(t) = atm2lnd_vars%forc_v_grc(g) ! forc_vxy Atm state m/s + top_as%zbot(t) = atm2lnd_vars%forc_hgt_grc(g) ! zgcmxy Atm state m + + sum_qbot_g = sum_qbot_g + top_pp%wtgcell(t)*top_as%qbot(t) + sum_wtsq_g = sum_wtsq_g + top_pp%wtgcell(t) + + ! assign the state forcing fields derived from other inputs + ! Horizontal windspeed (m/s) + top_as%windbot(t) = sqrt(top_as%ubot(t)**2 + top_as%vbot(t)**2) + ! partial pressure of oxygen (Pa) + top_as%po2bot(t) = o2_molar_const * top_as%pbot(t) + ! air density (kg/m**3) - uses a temporary calculation + ! of water vapor pressure (Pa) + vp = top_as%qbot(t) * top_as%pbot(t) / (0.622_r8 + 0.378_r8 * top_as%qbot(t)) + top_as%rhobot(t) = (top_as%pbot(t) - 0.378_r8 * vp) / (rair * top_as%tbot(t)) + top_af%solad(t,2) = atm2lnd_vars%forc_solad_grc(g,2) + top_af%solad(t,1) = atm2lnd_vars%forc_solad_grc(g,1) + top_af%solai(t,2) = atm2lnd_vars%forc_solai_grc(g,2) + top_af%solai(t,1) = atm2lnd_vars%forc_solai_grc(g,1) + ! derived flux forcings + top_af%solar(t) = top_af%solad(t,2) + top_af%solad(t,1) + & + top_af%solai(t,2) + top_af%solai(t,1) + + ! Keep track of the gridcell-level weighted sum for later normalization. + ! + ! This gridcell-level weighted sum just includes points for which we do the + ! downscaling (e.g., glc_mec points). Thus the contributing weights + ! generally do not add to 1. So to do the normalization properly, we also + ! need to keep track of the weights that have contributed to this sum. + sum_lwrad_g = sum_lwrad_g + top_pp%wtgcell(t)*top_af%lwrad(t) + sum_wtslw_g = sum_wtslw_g + top_pp%wtgcell(t) + end if + end do + if (precip_downscaling_method == 'FNM') then + do t = grc_pp%topi(g), grc_pp%topf(g) + t2 = t - grc_pp%topi(g) + 1 + if (mxElv == 0.) then ! avoid dividing by 0 + top_af%rain(t) = rain_g + top_af%snow(t) = snow_g + else + top_af%rain(t) = rain_g + (deltaRain(t2) - (rain_g/mxElv)*(sum_of_hrise/numt_pg)) + top_af%snow(t) = snow_g + (deltaSnow(t2) - (snow_g/mxElv)*(sum_of_hrise/numt_pg)) + end if + + end do + deallocate(deltaRain) + deallocate(deltaSnow) + end if + + ! Precipitation partitioning using simple method following Jordan (1991) + do t = grc_pp%topi(g), grc_pp%topf(g) + crnt_temp_t = top_as%tbot(t) + if (crnt_temp_t > Ta_th3) then ! No snow or all rain + temp_r = top_af%rain(t) + top_af%snow(t) + temp_s = 0._r8 + else if (crnt_temp_t >= Ta_th2 .and. crnt_temp_t <= Ta_th3) then + temp_s = (top_af%snow(t) + top_af%rain(t))*0.6_r8 ! 0.6 fraction of the total precip is snow + temp_r = (top_af%snow(t) + top_af%rain(t)) - temp_s + else if (crnt_temp_t < Ta_th2 .and. crnt_temp_t > Ta_th1) then + tmp_Snow_frc = (crnt_temp_t-Ta_th2)*(0.4_r8/(Ta_th1-Ta_th2))+0.6_r8 ! Snow fraction value 1-0.6 = 0.4 snowfrc is 1 at t = Ta_th1 + temp_s = (top_af%snow(t) + top_af%rain(t))*tmp_Snow_frc ! 0.6 is snow fraction at t = Ta_th2 + temp_r = (top_af%snow(t) + top_af%rain(t)) - temp_s + else ! crnt_temp_t <= Ta_th1 ==> all snow + temp_s = top_af%snow(t) + top_af%rain(t) + temp_r = 0._r8 + + end if + + top_af%rain(t) = temp_r + top_af%snow(t) = temp_s + + end do + + else !grid has a single topounit + ! update top_af using grid level values + t = grc_pp%topi(g) + top_af%rain(t) = rain_g + top_af%snow(t) = snow_g + top_af%lwrad(t) = atm2lnd_vars%forc_lwrad_not_downscaled_grc(g) + + ! Update top_as + top_as%tbot(t) = atm2lnd_vars%forc_t_not_downscaled_grc(g) ! forc_txy Atm state K + top_as%thbot(t) = atm2lnd_vars%forc_th_not_downscaled_grc(g) ! forc_thxy Atm state K + top_as%pbot(t) = atm2lnd_vars%forc_pbot_not_downscaled_grc(g) ! ptcmxy Atm state Pa + top_as%qbot(t) = atm2lnd_vars%forc_q_not_downscaled_grc(g) ! forc_qxy Atm state kg/kg + top_as%ubot(t) = atm2lnd_vars%forc_u_grc(g) ! forc_uxy Atm state m/s + top_as%vbot(t) = atm2lnd_vars%forc_v_grc(g) ! forc_vxy Atm state m/s + top_as%zbot(t) = atm2lnd_vars%forc_hgt_grc(g) ! zgcmxy Atm state m + + ! assign the state forcing fields derived from other inputs + ! Horizontal windspeed (m/s) + top_as%windbot(t) = sqrt(top_as%ubot(t)**2 + top_as%vbot(t)**2) + if (atm_gustiness) then + top_as%windbot(t) = sqrt(top_as%windbot(t)**2 + top_as%ugust(t)**2) + end if + ! Relative humidity (percent) + if (top_as%tbot(t) > SHR_CONST_TKFRZ) then + e = esatw(tdc(top_as%tbot(t))) + else + e = esati(tdc(top_as%tbot(t))) + end if + qvsat = 0.622_r8*e / (top_as%pbot(t) - 0.378_r8*e) + top_as%rhbot(t) = 100.0_r8*(top_as%qbot(t) / qvsat) + ! partial pressure of oxygen (Pa) + top_as%po2bot(t) = o2_molar_const * top_as%pbot(t) + ! air density (kg/m**3) - uses a temporary calculation + ! of water vapor pressure (Pa) + vp = top_as%qbot(t) * top_as%pbot(t) / (0.622_r8 + 0.378_r8 * top_as%qbot(t)) + top_as%rhobot(t) = (top_as%pbot(t) - 0.378_r8 * vp) / (rair * top_as%tbot(t)) + + top_af%solad(t,2) = atm2lnd_vars%forc_solad_grc(g,2) + top_af%solad(t,1) = atm2lnd_vars%forc_solad_grc(g,1) + top_af%solai(t,2) = atm2lnd_vars%forc_solai_grc(g,2) + top_af%solai(t,1) = atm2lnd_vars%forc_solai_grc(g,1) + ! derived flux forcings + top_af%solar(t) = top_af%solad(t,2) + top_af%solad(t,1) + & + top_af%solai(t,2) + top_af%solai(t,1) + + end if + + if (numt_pg > 1) then + + ! Normalize forc_qbot to conserve energy + + call build_normalization(orig_field=atm2lnd_vars%forc_q_not_downscaled_grc(g), & + sum_field=sum_qbot_g, sum_wts=sum_wtsq_g, norms=qbot_norm_g) + + do t = grc_pp%topi(g), grc_pp%topf(g) + top_as%qbot(t) = top_as%qbot(t) * qbot_norm_g + + ! Relative humidity (percent) + if (top_as%tbot(t) > SHR_CONST_TKFRZ) then + e = esatw(tdc(top_as%tbot(t))) + else + e = esati(tdc(top_as%tbot(t))) + end if + qvsat = 0.622_r8*e / (top_as%pbot(t) - 0.378_r8*e) + top_as%rhbot(t) = 100.0_r8*(top_as%qbot(t) / qvsat) + ! partial pressure of oxygen (Pa) + top_as%po2bot(t) = o2_molar_const * top_as%pbot(t) + ! air density (kg/m**3) - uses a temporary calculation + ! of water vapor pressure (Pa) + vp = top_as%qbot(t) * top_as%pbot(t) / (0.622_r8 + 0.378_r8 * top_as%qbot(t)) + top_as%rhobot(t) = (top_as%pbot(t) - 0.378_r8 * vp) / (rair * top_as%tbot(t)) + + end do + + + ! Normalize forc_lwrad_c(c) to conserve energy + + call build_normalization(orig_field=atm2lnd_vars%forc_lwrad_not_downscaled_grc(g), & + sum_field=sum_lwrad_g, sum_wts=sum_wtslw_g, norms=lwrad_norm_g) + + do t = grc_pp%topi(g), grc_pp%topf(g) + top_af%lwrad(t) = top_af%lwrad(t) * lwrad_norm_g + end do + + end if + + end subroutine downscale_atm_forcing_to_topounit_cpl_bypass + + !----------------------------------------------------------------------- + ! Downscale other atmospheric state variables + !----------------------------------------------------------------------- + subroutine downscale_atm_state_to_topounit_cpl_bypass(g, t, atm2lnd_vars) + ! + ! !DESCRIPTION: + ! Downscale atmospheric forcing fields from gridcell to topounit + ! + ! Downscaling is done over topounits. + ! + ! !USES: + use elm_time_manager, only : get_nstep + use elm_varcon , only : rair, cpair, grav, lapse_glcmec + use elm_varcon , only : glcmec_rain_snow_threshold + use landunit_varcon , only : istice_mec + use elm_varctl , only : glcmec_downscale_rain_snow_convert + use domainMod , only : ldomain + use QsatMod , only : Qsat + ! + ! !ARGUMENTS: + integer , intent(in) :: g + integer , intent(in) :: t + type(atm2lnd_type) , intent(in) :: atm2lnd_vars + ! + ! !LOCAL VARIABLES: + integer :: l, c, fc ! indices + integer :: clo, cc + integer :: nstep + + ! temporaries for topo downscaling + real(r8) :: hsurf_g,hsurf_t,Hbot + real(r8) :: zbot_g, tbot_g, pbot_g, thbot_g, qbot_g, qs_g, es_g + real(r8) :: zbot_t, tbot_t, pbot_t, thbot_t, qbot_t, qs_t, es_t + real(r8) :: egcm_t, rhos_t + real(r8) :: dum1, dum2 + + character(len=*), parameter :: subname = 'downscale_atm_state_to_topounit' + !----------------------------------------------------------------------- + + nstep = get_nstep() + + ! Downscale forc_t, forc_th, forc_q, forc_pbot, and forc_rho to columns. + ! For glacier_mec columns the downscaling is based on surface elevation. + ! For other columns the downscaling is a simple copy (above). + + ! This is a simple downscaling procedure + ! Note that forc_hgt, forc_u, and forc_v are not downscaled. + + hsurf_g = grc_pp%elevation(g) ! gridcell sfc elevation + hsurf_t = top_pp%elevation(t) ! topounit sfc elevation + tbot_g = atm2lnd_vars%forc_t_not_downscaled_grc(g) ! atm sfc temp + thbot_g = atm2lnd_vars%forc_th_not_downscaled_grc(g) ! atm sfc pot temp + qbot_g = atm2lnd_vars%forc_q_not_downscaled_grc(g) ! atm sfc spec humid + pbot_g = atm2lnd_vars%forc_pbot_not_downscaled_grc(g) ! atm sfc pressure + zbot_g = atm2lnd_vars%forc_hgt_grc(g) ! atm ref height + + zbot_t = zbot_g + tbot_t = tbot_g-lapse_glcmec*(hsurf_t-hsurf_g) ! sfc temp for column + + Hbot = rair*0.5_r8*(tbot_g+tbot_t)/grav ! scale ht at avg temp + pbot_t = pbot_g*exp(-(hsurf_t-hsurf_g)/Hbot) ! column sfc press + + ! Derivation of potential temperature calculation: + ! + ! The textbook definition would be: + ! thbot_c = tbot_c * (p0/pbot_c)^(rair/cpair) + ! + ! Note that pressure is related to scale height as: + ! pbot_c = p0 * exp(-zbot_c/H) + ! + ! Using Hbot in place of H, we get: + ! pbot_c = p0 * exp(-zbot_c/Hbot) + ! + ! Plugging this in to the textbook definition, then manipulating, we get: + ! thbot_c = tbot_c * (p0/(p0*exp(-zbot_c/Hbot)))^(rair/cpair) + ! = tbot_c * (1/exp(-zbot_c/Hbot))^(rair/cpair) + ! = tbot_c * (exp(zbot_c/Hbot))^(rair/cpair) + ! = tbot_c * exp((zbot_c/Hbot) * (rair/cpair)) + + thbot_t= tbot_t*exp((zbot_t/Hbot)*(rair/cpair)) ! pot temp calc + + call Qsat(tbot_g,pbot_g,es_g,dum1,qs_g,dum2) + call Qsat(tbot_t,pbot_t,es_t,dum1,qs_t,dum2) + + qbot_t = qbot_g*(qs_t/qs_g) + egcm_t = qbot_t*pbot_t/(0.622_r8+0.378_r8*qbot_t) + rhos_t = (pbot_t-0.378_r8*egcm_t) / (rair*tbot_t) + + top_as%tbot(t) = tbot_t + top_as%thbot(t) = thbot_t + top_as%qbot(t) = qbot_t + top_as%pbot(t) = pbot_t + +! call check_downscale_consistency(bounds, atm2lnd_vars) + + end subroutine downscale_atm_state_to_topounit_cpl_bypass + + !------------------------------------------------------- + ! Downscale longwave radiation place holder + subroutine downscale_longwave_to_topounit_cpl_bypass(g, t, atm2lnd_vars) + + ! !DESCRIPTION: + ! Downscale longwave radiation from gridcell to column + ! Must be done AFTER temperature downscaling + + ! !USES: + use elm_time_manager, only : get_nstep + use domainMod , only : ldomain + use landunit_varcon , only : istice_mec + use elm_varcon , only : lapse_glcmec + use elm_varctl , only : glcmec_downscale_longwave + + ! !ARGUMENTS: + integer , intent(in) :: g + integer , intent(in) :: t + type(atm2lnd_type) , intent(in) :: atm2lnd_vars + + ! !LOCAL VARIABLES: + integer :: c,l,fc ! indices + integer :: nstep + real(r8) :: hsurf_t ! column-level elevation (m) + real(r8) :: hsurf_g ! gridcell-level elevation (m) + + real(r8) :: tair_g ! original gridcell mean air temperature + real(r8) :: tair_t ! downscaled topounit air temperature + real(r8) :: tsfc_g ! original gridcell surface temperature + real(r8) :: tsfc_t ! downscaled topounit surface temperature + real(r8) :: lwrad_g ! original gridcell mean LW radiation + real(r8) :: lwrad_t ! downscaled topounit LW radiation + real(r8) :: newsum_lwrad_g ! weighted sum of column-level lwrad after normalization + + character(len=*), parameter :: subname = 'downscale_longwave_to_topounit' + !----------------------------------------------------------------------- + + nstep = get_nstep() + + ! Do the downscaling + hsurf_g = grc_pp%elevation(g) + hsurf_t = top_pp%elevation(t) + + ! Here we assume that deltaLW = (dLW/dT)*(dT/dz)*deltaz + ! We get dLW/dT = 4*eps*sigma*T^3 = 4*LW/T from the Stefan-Boltzmann law, + ! evaluated at the mean temp. + ! We assume the same temperature lapse rate as above. + + tair_g = atm2lnd_vars%forc_t_not_downscaled_grc(g) + tair_t = top_as%tbot(t) + lwrad_g = atm2lnd_vars%forc_lwrad_not_downscaled_grc(g) + top_af%lwrad(t) = lwrad_g - & + 4.0_r8 * lwrad_g/(0.5_r8*(tair_t+tair_g)) * & + lapse_glcmec * (hsurf_t - hsurf_g) + + end subroutine downscale_longwave_to_topounit_cpl_bypass + end module lnd_downscale_atm_forcing diff --git a/components/elm/src/cpl/lnd_import_export.F90 b/components/elm/src/cpl/lnd_import_export.F90 index ad81846196d..27c01e66818 100644 --- a/components/elm/src/cpl/lnd_import_export.F90 +++ b/components/elm/src/cpl/lnd_import_export.F90 @@ -1030,52 +1030,68 @@ subroutine lnd_import( bounds, x2l, atm2lnd_vars, glc2lnd_vars, lnd2atm_vars) end if end if - !set the topounit-level atmospheric state and flux forcings (bypass mode) + ! Adding topographic downscaling capability within CPL_BYPASS code block + ! PET, 7/12/2024 + if (use_atm_downscaling_to_topunit) then + atm2lnd_vars%forc_uovern = x2l(index_x2l_Sa_uovern,i) + atm2lnd_vars%forc_rain_not_downscaled_grc = forc_rainc + forc_rainl + atm2lnd_vars%forc_snow_not_downscaled_grc = forc_snowc + forc_snowl + + if(atm_gustiness) then + call endrun("Error: atm_gustiness not yet supported with multiple topounits (in CPL_BYPASS)") + end if + do topo = grc_pp%topi(g) , grc_pp%topf(g) + top_as%ugust(topo) = 0._r8 + end do + + call downscale_atm_forcing_to_topounit_cpl_bypass(g, atm2lnd_vars, lnd2atm_vars) + else + do topo = grc_pp%topi(g), grc_pp%topf(g) + top_as%tbot(topo) = atm2lnd_vars%forc_t_not_downscaled_grc(g) ! forc_txy Atm state K + top_as%thbot(topo) = atm2lnd_vars%forc_th_not_downscaled_grc(g) ! forc_thxy Atm state K + top_as%pbot(topo) = atm2lnd_vars%forc_pbot_not_downscaled_grc(g) ! ptcmxy Atm state Pa + top_as%qbot(topo) = atm2lnd_vars%forc_q_not_downscaled_grc(g) ! forc_qxy Atm state kg/kg + top_as%ubot(topo) = atm2lnd_vars%forc_u_grc(g) ! forc_uxy Atm state m/s + top_as%vbot(topo) = atm2lnd_vars%forc_v_grc(g) ! forc_vxy Atm state m/s + top_as%zbot(topo) = atm2lnd_vars%forc_hgt_grc(g) ! zgcmxy Atm state m + top_as%windbot(topo) = sqrt(top_as%ubot(topo)**2 + top_as%vbot(topo)**2) + ! Relative humidity (percent) + if (top_as%tbot(topo) > SHR_CONST_TKFRZ) then + e = esatw(tdc(top_as%tbot(topo))) + else + e = esati(tdc(top_as%tbot(topo))) + end if + qsat = 0.622_r8*e / (top_as%pbot(topo) - 0.378_r8*e) + top_as%rhbot(topo) = 100.0_r8*(top_as%qbot(topo) / qsat) + ! partial pressure of oxygen (Pa) + top_as%po2bot(topo) = o2_molar_const * top_as%pbot(topo) + ! air density (kg/m**3) - uses a temporary calculation of water vapor pressure (Pa) + vp = top_as%qbot(topo) * top_as%pbot(topo) / (0.622_r8 + 0.378_r8 * top_as%qbot(topo)) + top_as%rhobot(topo) = (top_as%pbot(topo) - 0.378_r8 * vp) / (rair * top_as%tbot(topo)) + top_af%rain(topo) = forc_rainc + forc_rainl ! sum of convective and large-scale rain + top_af%snow(topo) = forc_snowc + forc_snowl ! sum of convective and large-scale snow + top_af%solad(topo,2) = atm2lnd_vars%forc_solad_grc(g,2) ! forc_sollxy Atm flux W/m^2 + top_af%solad(topo,1) = atm2lnd_vars%forc_solad_grc(g,1) ! forc_solsxy Atm flux W/m^2 + top_af%solai(topo,2) = atm2lnd_vars%forc_solai_grc(g,2) ! forc_solldxy Atm flux W/m^2 + top_af%solai(topo,1) = atm2lnd_vars%forc_solai_grc(g,1) ! forc_solsdxy Atm flux W/m^2 + top_af%lwrad(topo) = atm2lnd_vars%forc_lwrad_not_downscaled_grc(g) ! flwdsxy Atm flux W/m^2 + ! derived flux forcings + top_af%solar(topo) = top_af%solad(topo,2) + top_af%solad(topo,1) + & + top_af%solai(topo,2) + top_af%solai(topo,1) + end do + end if + + !set the topounit-level atmospheric variables that are not handled in downscaling code do topo = grc_pp%topi(g), grc_pp%topf(g) ! first, all the state forcings - top_as%tbot(topo) = atm2lnd_vars%forc_t_not_downscaled_grc(g) ! forc_txy Atm state K - top_as%thbot(topo) = atm2lnd_vars%forc_th_not_downscaled_grc(g) ! forc_thxy Atm state K - top_as%pbot(topo) = atm2lnd_vars%forc_pbot_not_downscaled_grc(g) ! ptcmxy Atm state Pa - top_as%qbot(topo) = atm2lnd_vars%forc_q_not_downscaled_grc(g) ! forc_qxy Atm state kg/kg - top_as%ubot(topo) = atm2lnd_vars%forc_u_grc(g) ! forc_uxy Atm state m/s - top_as%vbot(topo) = atm2lnd_vars%forc_v_grc(g) ! forc_vxy Atm state m/s - if (implicit_stress) then + if (implicit_stress) then top_as%wsresp(topo) = 0._r8 ! Atm state m/s/Pa top_as%tau_est(topo) = 0._r8 ! Atm state Pa end if top_as%ugust(topo) = 0._r8 ! Atm state m/s - top_as%zbot(topo) = atm2lnd_vars%forc_hgt_grc(g) ! zgcmxy Atm state m - ! assign the state forcing fields derived from other inputs - ! Horizontal windspeed (m/s) - top_as%windbot(topo) = sqrt(top_as%ubot(topo)**2 + top_as%vbot(topo)**2) if (atm_gustiness) then top_as%windbot(topo) = sqrt(top_as%windbot(topo)**2 + top_as%ugust(topo)**2) end if - ! Relative humidity (percent) - if (top_as%tbot(topo) > SHR_CONST_TKFRZ) then - e = esatw(tdc(top_as%tbot(topo))) - else - e = esati(tdc(top_as%tbot(topo))) - end if - qsat = 0.622_r8*e / (top_as%pbot(topo) - 0.378_r8*e) - top_as%rhbot(topo) = 100.0_r8*(top_as%qbot(topo) / qsat) - ! partial pressure of oxygen (Pa) - top_as%po2bot(topo) = o2_molar_const * top_as%pbot(topo) - ! air density (kg/m**3) - uses a temporary calculation of water vapor pressure (Pa) - vp = top_as%qbot(topo) * top_as%pbot(topo) / (0.622_r8 + 0.378_r8 * top_as%qbot(topo)) - top_as%rhobot(topo) = (top_as%pbot(topo) - 0.378_r8 * vp) / (rair * top_as%tbot(topo)) - - ! second, all the flux forcings - top_af%rain(topo) = forc_rainc + forc_rainl ! sum of convective and large-scale rain - top_af%snow(topo) = forc_snowc + forc_snowl ! sum of convective and large-scale snow - top_af%solad(topo,2) = atm2lnd_vars%forc_solad_grc(g,2) ! forc_sollxy Atm flux W/m^2 - top_af%solad(topo,1) = atm2lnd_vars%forc_solad_grc(g,1) ! forc_solsxy Atm flux W/m^2 - top_af%solai(topo,2) = atm2lnd_vars%forc_solai_grc(g,2) ! forc_solldxy Atm flux W/m^2 - top_af%solai(topo,1) = atm2lnd_vars%forc_solai_grc(g,1) ! forc_solsdxy Atm flux W/m^2 - top_af%lwrad(topo) = atm2lnd_vars%forc_lwrad_not_downscaled_grc(g) ! flwdsxy Atm flux W/m^2 - ! derived flux forcings - top_af%solar(topo) = top_af%solad(topo,2) + top_af%solad(topo,1) + & - top_af%solai(topo,2) + top_af%solai(topo,1) end do !----------------------------------------------------------------------------------------------------- diff --git a/components/elm/src/data_types/VegetationPropertiesType.F90 b/components/elm/src/data_types/VegetationPropertiesType.F90 index 12f9a7b22ff..d62167301cd 100644 --- a/components/elm/src/data_types/VegetationPropertiesType.F90 +++ b/components/elm/src/data_types/VegetationPropertiesType.F90 @@ -43,7 +43,7 @@ module VegetationPropertiesType real(r8), pointer :: dsladlai (:) => null() ! dSLA/dLAI, projected area basis [m^2/gC] real(r8), pointer :: leafcn (:) => null() ! leaf C:N (gC/gN) real(r8), pointer :: flnr (:) => null() ! fraction of leaf N in the Rubisco enzyme (gN Rubisco / gN leaf) - real(r8), pointer :: woody (:) => null() ! binary flag for woody lifeform (1=woody, 0=not woody) + real(r8), pointer :: woody (:) => null() ! woody lifeform flag (0 = non-woody, 1 = tree, 2 = shrub) real(r8), pointer :: lflitcn (:) => null() ! leaf litter C:N (gC/gN) real(r8), pointer :: frootcn (:) => null() ! fine root C:N (gC/gN) real(r8), pointer :: livewdcn (:) => null() ! live wood (phloem and ray parenchyma) C:N (gC/gN) @@ -115,7 +115,7 @@ module VegetationPropertiesType real(r8), pointer :: lamda_ptase => null()! critical value that incur biochemical production real(r8), pointer :: i_vc(:) => null() ! intercept of photosynthesis vcmax ~ leaf n content regression model real(r8), pointer :: s_vc(:) => null() ! slope of photosynthesis vcmax ~ leaf n content regression model - real(r8), pointer :: nsc_rtime(:) => null() ! non-structural carbon residence time + real(r8), pointer :: nsc_rtime(:) => null() ! non-structural carbon residence time real(r8), pointer :: pinit_beta1(:) => null() ! shaping parameter for P initialization real(r8), pointer :: pinit_beta2(:) => null() ! shaping parameter for P initialization real(r8), pointer :: alpha_nfix(:) => null() ! fraction of fixed N goes directly to plant @@ -151,6 +151,11 @@ module VegetationPropertiesType real(r8), pointer :: needleleaf(:) => null() !needleleaf or broadleaf real(r8), pointer :: nfixer(:) => null() !cablity of nitrogen fixation from atm. N2 + ! NGEE Arctic snow-vegetation interactions + real(r8), pointer :: bendresist(:) ! vegetation resistance to bending under snow loading, 0 to 1 (e.g., Liston and Hiemstra 2011) + real(r8), pointer :: vegshape(:) ! shape parameter to modify shrub burial by snow (1 = parabolic, 2 = hemispheric) + real(r8), pointer :: stocking(:) ! stocking density for pft (stems / hectare) + real(r8), pointer :: taper(:) ! ratio of height:radius_breast_height (woody vegetation allometry) contains procedure, public :: Init => veg_vp_init @@ -188,8 +193,10 @@ subroutine veg_vp_init(this) use pftvarcon , only : fnr, act25, kcha, koha, cpha, vcmaxha, jmaxha, tpuha use pftvarcon , only : lmrha, vcmaxhd, jmaxhd, tpuhd, lmrse, qe, theta_cj use pftvarcon , only : bbbopt, mbbopt, nstor, br_xr, tc_stress, lmrhd - ! new properties for flexible PFT + ! new properties for flexible PFT (NGEE Arctic IM4) use pftvarcon , only : climatezone, nonvascular, graminoid, iscft,needleleaf, nfixer + ! snow/vegetation interactions (NGEE Arctic IM3) + use pftvarcon , only : bendresist, stocking, vegshape, taper ! class (vegetation_properties_type) :: this @@ -322,6 +329,11 @@ subroutine veg_vp_init(this) allocate( this%needleleaf(0:numpft)) ; this%needleleaf(:) =spval allocate( this%nfixer(0:numpft)) ; this%nfixer(:) =spval ! ----------------------------------------------------------------------------------------------------------- + ! NGEE Arctic snow-vegetation interactions + allocate(this%bendresist(0:numpft)) ; this%bendresist(:) =spval + allocate(this%vegshape(0:numpft)) ; this%vegshape(:) =spval + allocate(this%stocking(0:numpft)) ; this%stocking(:) =spval + allocate(this%taper(0:numpft)) ; this%taper(:) =spval do m = 0,numpft @@ -472,6 +484,13 @@ subroutine veg_vp_init(this) this%lamda_ptase = lamda_ptase this%tc_stress = tc_stress + ! NGEE Arctic - snow/vegetation interactions + do m = 0, numpft ! RPF - move up to earlier pft loops? + this%bendresist(m) = bendresist(m) + this%vegshape(m) = vegshape(m) + this%stocking(m) = stocking(m) + this%taper(m) = taper(m) + end do end subroutine veg_vp_init end module VegetationPropertiesType diff --git a/components/elm/src/external_models/fates b/components/elm/src/external_models/fates index 1982b0032c3..825579d0b40 160000 --- a/components/elm/src/external_models/fates +++ b/components/elm/src/external_models/fates @@ -1 +1 @@ -Subproject commit 1982b0032c3cab6278892eccb85f643114ffb1af +Subproject commit 825579d0b406fe99344591b5ed8356e5c7aeebec diff --git a/components/elm/src/main/atm2lndType.F90 b/components/elm/src/main/atm2lndType.F90 index a00140ab2dd..f2facc77622 100644 --- a/components/elm/src/main/atm2lndType.F90 +++ b/components/elm/src/main/atm2lndType.F90 @@ -146,6 +146,10 @@ module atm2lndType real(r8) , pointer :: t_mo_patch (:) => null() ! patch 30-day average temperature (Kelvin) real(r8) , pointer :: t_mo_min_patch (:) => null() ! patch annual min of t_mo (Kelvin) + ! Needed for FNM precip downscaling, when used within CPL_BYPASS + real(r8), pointer :: forc_uovern (:) => null() ! Froude number (dimensionless) + + contains procedure, public :: Init @@ -310,6 +314,7 @@ subroutine InitAllocate(this, bounds) allocate(this%forc_ndep_nitr_grc (begg:endg)) ; this%forc_ndep_nitr_grc (:) = ival allocate(this%forc_soilph_grc (begg:endg)) ; this%forc_soilph_grc (:) = ival end if + allocate(this%forc_uovern (begg:endg)) ; this%forc_uovern (:) = ival end subroutine InitAllocate diff --git a/components/elm/src/main/controlMod.F90 b/components/elm/src/main/controlMod.F90 index 1d8f48d7cc7..dc7fa975dbb 100755 --- a/components/elm/src/main/controlMod.F90 +++ b/components/elm/src/main/controlMod.F90 @@ -55,6 +55,7 @@ module controlMod use elm_varctl , only: const_climate_hist use elm_varctl , only: use_top_solar_rad use elm_varctl , only: snow_shape, snicar_atm_type, use_dust_snow_internal_mixing + use EcosystemBalanceCheckMod, only: bgc_balance_check_tolerance => balance_check_tolerance ! ! !PUBLIC TYPES: @@ -181,6 +182,10 @@ subroutine control_init( ) namelist /elm_inparm/ & NFIX_PTASE_plant + ! BGC balance check + namelist /elm_inparm/ & + bgc_balance_check_tolerance + ! For experimental manipulations namelist /elm_inparm/ & startdate_add_temperature @@ -325,7 +330,7 @@ subroutine control_init( ) namelist /elm_mosart/ & lnd_rof_coupling_nstep - + namelist /elm_inparm/ & snow_shape, snicar_atm_type, use_dust_snow_internal_mixing @@ -796,6 +801,7 @@ subroutine control_spmd() call mpi_bcast (forest_fert_exp, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (ECA_Pconst_RGspin, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (NFIX_PTASE_plant, 1, MPI_LOGICAL, 0, mpicom, ier) + call mpi_bcast (bgc_balance_check_tolerance, 1, MPI_REAL8, 0, mpicom, ier) call mpi_bcast (use_pheno_flux_limiter, 1, MPI_LOGICAL, 0, mpicom, ier) call mpi_bcast (startdate_add_temperature, 1, MPI_CHARACTER, 0, mpicom, ier) call mpi_bcast (startdate_add_co2, 1, MPI_CHARACTER, 0, mpicom, ier) diff --git a/components/elm/src/main/pftvarcon.F90 b/components/elm/src/main/pftvarcon.F90 index fc5bfd54e7d..66c2adda90f 100644 --- a/components/elm/src/main/pftvarcon.F90 +++ b/components/elm/src/main/pftvarcon.F90 @@ -29,7 +29,7 @@ module pftvarcon ! ! Vegetation type constants ! - integer :: noveg !value for not vegetated + integer :: noveg !value for not vegetated integer :: ndllf_evr_tmp_tree !value for Needleleaf evergreen temperate tree integer :: ndllf_evr_brl_tree !value for Needleleaf evergreen boreal tree integer :: ndllf_dcd_brl_tree !value for Needleleaf deciduous boreal tree @@ -76,7 +76,7 @@ module pftvarcon integer :: nrtubers !value for root tubers, rain fed (rf) integer :: nrtubersirrig !value for root tubers, irrigated (ir) integer :: nsugarcane !value for sugarcane, rain fed (rf) - integer :: nsugarcaneirrig !value for sugarcane, irrigated (ir) + integer :: nsugarcaneirrig !value for sugarcane, irrigated (ir) integer :: nmiscanthus !value for miscanthus, rain fed (rf) integer :: nmiscanthusirrig !value for miscanthus, irrigated (ir) integer :: nswitchgrass !value for switchgrass, rain fed (rf) @@ -84,8 +84,8 @@ module pftvarcon integer :: npoplar !value for poplar, rain fed (rf) integer :: npoplarirrig !value for poplar, irrigated (ir) integer :: nwillow !value for willow, rain fed (rf) - integer :: nwillowirrig !value for willow, irrigated (ir) - + integer :: nwillowirrig !value for willow, irrigated (ir) + ! Number of crop functional types actually used in the model. This includes each CFT for ! which is_pft_known_to_model is true. Note that this includes irrigated crops even if ! irrigation is turned off in this run: it just excludes crop types that aren't handled @@ -171,7 +171,7 @@ module pftvarcon real(r8), allocatable :: minplanttemp(:) !mininum planting temperature used in Phenology (K) real(r8), allocatable :: senestemp(:) !senescence temperature for perennial crops used in Phenology (K) real(r8), allocatable :: min_days_senes(:) !minimum leaf age to allow for leaf senescence - real(r8), allocatable :: froot_leaf(:) !allocation parameter: new fine root C per new leaf C (gC/gC) + real(r8), allocatable :: froot_leaf(:) !allocation parameter: new fine root C per new leaf C (gC/gC) real(r8), allocatable :: stem_leaf(:) !allocation parameter: new stem c per new leaf C (gC/gC) real(r8), allocatable :: croot_stem(:) !allocation parameter: new coarse root C per new stem C (gC/gC) real(r8), allocatable :: flivewd(:) !allocation parameter: fraction of new wood that is live (phloem and ray parenchyma) (no units) @@ -215,7 +215,7 @@ module pftvarcon real(r8), allocatable :: fyield(:) !fraction of grain that is actually harvested real(r8), allocatable :: root_dmx(:) !maximum root depth - integer, parameter :: pftname_len = 40 ! max length of pftname + integer, parameter :: pftname_len = 40 ! max length of pftname character(len=:), allocatable :: pftname(:) !PFT description real(r8), parameter :: reinickerp = 1.6_r8 !parameter in allometric equation @@ -276,7 +276,7 @@ module pftvarcon real(r8), allocatable :: deadwdcp_obs_flex(:,:) !upper and lower range of dead wood (xylem and heartwood) C:P (gC/gP) ! Photosynthesis parameters real(r8), allocatable :: fnr(:) !fraction of nitrogen in RuBisCO - real(r8), allocatable :: act25(:) + real(r8), allocatable :: act25(:) real(r8), allocatable :: kcha(:) !Activation energy for kc real(r8), allocatable :: koha(:) !Activation energy for ko real(r8), allocatable :: cpha(:) !Activation energy for cp @@ -293,7 +293,7 @@ module pftvarcon real(r8), allocatable :: theta_cj(:) ! real(r8), allocatable :: bbbopt(:) !Ball-Berry stomatal conductance intercept real(r8), allocatable :: mbbopt(:) !Ball-Berry stomatal conductance slope - real(r8), allocatable :: nstor(:) !Nitrogen storage pool timescale + real(r8), allocatable :: nstor(:) !Nitrogen storage pool timescale real(r8), allocatable :: br_xr(:) !Base rate for excess respiration real(r8) :: tc_stress !Critial temperature for moisture stress real(r8), allocatable :: vcmax_np1(:) !vcmax~np relationship coefficient @@ -312,6 +312,13 @@ module pftvarcon real(r8), allocatable :: gcbr_p(:) !effectiveness of roots in reducing rainfall-driven erosion real(r8), allocatable :: gcbr_q(:) !effectiveness of roots in reducing runoff-driven erosion + ! NGEE Arctic snow-vegetation interactions + real(r8), allocatable :: bendresist(:) ! vegetation resistance to bending under snow loading, 0 to 1 (e.g., Liston and Hiemstra 2011; Sturm et al. 2005) + real(r8), allocatable :: vegshape(:) ! shape parameter to modify shrub burial by snow (1 = parabolic, 2 = hemispheric) + real(r8), allocatable :: stocking(:) ! stocking density for pft (stems / hectare) + real(r8), allocatable :: taper(:) ! ratio of height:radius_breast_height (woody vegetation allometry) + logical :: taper_defaults ! set flag to use taper defaults if not on params file (necessary as import and set values are in different places) + ! new pft properties, together with woody, crop, percrop, evergreen, stress_decid, season_decid, defined above, ! are introduced to define vegetation properties. This will be well defineing a pft so that no indices needed for codes. real(r8), allocatable :: climatezone(:) ! distributed climate zone (0 = any zone, 1 = tropical, 2 = temperate, 3 = boreal, 4 = arctic) @@ -379,7 +386,7 @@ subroutine pftconrd ! and finally crops, ending with soybean ! DO NOT CHANGE THE ORDER -- WITHOUT MODIFYING OTHER PARTS OF THE CODE WHERE THE ORDER MATTERS! ! - character(len=pftname_len) :: expected_pftnames(0:mxpft) + character(len=pftname_len) :: expected_pftnames(0:mxpft) !----------------------------------------------------------------------- expected_pftnames( 0) = 'not_vegetated ' @@ -452,95 +459,95 @@ subroutine pftconrd ! but cannot be less, i.e. there cannot be more pfts in the surface file than there are set of pft parameters, if running the non-crop version of model. if (npft mxpft_nc) EXIT ! exit the do loop - + ! (FATES-INTERF) Later, depending on how the team plans to structure the crop model ! or other modules that co-exist while FATES is on, we may want to preserve these pft definitions ! on non-fates columns. For now, they are incompatible, and this check is warranted (rgk 04-2017) @@ -1507,6 +1542,17 @@ subroutine pftconrd end do end if + ! reassign taper values for shrubs - RPF + if (taper_defaults) then + do i = 0, npft-1 + if (woody(i) == 2._r8) then + taper(i) = 10._r8 ! shrubs + else if (woody(i) == 1._r8) then + taper(i) = 200._r8 + end if + end do + end if + if (masterproc) then write(iulog,*) 'Successfully read PFT physiological data' write(iulog,*) diff --git a/components/elm/tools/interpinic/src/fmain.F90 b/components/elm/tools/interpinic/src/fmain.F90 index 0f55c12c67c..a67e6e2627f 100644 --- a/components/elm/tools/interpinic/src/fmain.F90 +++ b/components/elm/tools/interpinic/src/fmain.F90 @@ -14,7 +14,7 @@ program fmain character(len= 256) :: arg integer :: n !index integer :: nargs !number of arguments - integer, external :: iargc !number of arguments function + integer :: iargc !number of arguments function character(len=256) :: finidati !input initial dataset to read character(len=256) :: finidato !output initial dataset to create character(len=256) :: cmdline !input command line diff --git a/components/elm/tools/interpinic/src/interpinic.F90 b/components/elm/tools/interpinic/src/interpinic.F90 index b75afd3a411..3d890a14b89 100644 --- a/components/elm/tools/interpinic/src/interpinic.F90 +++ b/components/elm/tools/interpinic/src/interpinic.F90 @@ -146,9 +146,19 @@ subroutine interp_filei (fin, fout, cmdline) call check_ret (nf90_open(fin, NF90_NOWRITE, ncidi )) call check_ret (nf90_open(fout, NF90_NOWRITE, ncido )) call check_ret (nf_inq_format( ncido, ncformat )) - if ( ncformat /= NF_FORMAT_64BIT )then - write (6,*) 'error: output file is NOT in NetCDF large-file format!' - stop + + ! Allow any format for output dataset + + if ( ncformat == NF_FORMAT_CLASSIC )then + write (6,*) 'info: output file is NF_FORMAT_CLASSIC' + else if ( ncformat == NF_FORMAT_64BIT_OFFSET )then + write (6,*) 'info: output file is NF_FORMAT_64BIT_OFFSET' + else if ( ncformat == NF_FORMAT_64BIT_DATA )then + write (6,*) 'info: output file is NF_FORMAT_64BIT_DATA' + else if ( ncformat == NF_FORMAT_NETCDF4 )then + write (6,*) 'info: output file is NF_FORMAT_NETCDF4' + else if ( ncformat == NF_FORMAT_NETCDF4_CLASSIC )then + write (6,*) 'info: output file is NF_FORMAT_NETCDF4_CLASSIC' end if call check_ret (nf90_inq_dimid(ncidi, "column", dimidcols )) @@ -214,12 +224,25 @@ subroutine interp_filei (fin, fout, cmdline) ret = nf90_inq_dimid(ncidi, "month", dimidmon) if (ret == NF90_NOERR) then call check_ret (nf90_inquire_dimension(ncidi, dimidmon, len=nlevmon)) - call check_ret (nf90_inq_dimid(ncido, "month", dimid )) - call check_ret (nf90_inquire_dimension(ncido, dimid, len=dimlen)) - if (dimlen/=nlevmon) then - write (6,*) 'error: input and output nlevmon values disagree' - write (6,*) 'input nlevmon = ',nlevmon,' output nlevmon = ',dimlen - stop + + ! Many restart files have "month" dimension in input dataset + ! It is only necessary that the output dataset contains "month" dimension + ! when a variable in the input dataset contains the "month" dimension + ! Otherwise, the "month" dimension will never be used + ! Warn rather than die when input has "month" and output does not + + ret = nf90_inq_dimid(ncido, "month", dimid ) + if ( ret == nf_ebaddim ) then + write (6,*) 'warning: input has "month" dimension and output does not' + write (6,*) 'warning: interpolation will fail if any input variable uses "month" dimension' + write (6,*) 'chill: many times the "month" dimension is superfluous so this might work...' + else + call check_ret (nf90_inquire_dimension(ncido, dimid, len=dimlen)) + if (dimlen/=nlevmon) then + write (6,*) 'error: input and output nlevmon values disagree' + write (6,*) 'input nlevmon = ',nlevmon,' output nlevmon = ',dimlen + stop + end if end if else write (6,*) 'month dimension does NOT exist on the input dataset' @@ -321,7 +344,9 @@ subroutine interp_filei (fin, fout, cmdline) ! OK now, open the output file for writing ! call check_ret(nf90_close( ncido)) - call check_ret (nf90_open(fout, ior(NF90_WRITE, NF_64BIT_OFFSET), ncido )) + + ! Allow any format for output dataset + call check_ret (nf90_open(fout, NF90_WRITE, ncido )) call addglobal (ncido, cmdline) @@ -1503,8 +1528,7 @@ subroutine addglobal (ncid, cmdline) character(len=10) :: time character(len= 5) :: zone character(len=18) :: datetime - character(len=256):: version = & - "$HeadURL: https://svn-ccsm-models.cgd.ucar.edu/clm2/trunk_tags/clm4_5_1_r085/models/lnd/clm/tools/clm4_5/interpinic/src/interpinic.F90 $" + character(len=256):: version = "" character(len=256) :: revision_id = "$Id: interpinic.F90 54953 2013-11-06 16:29:45Z sacks $" character(len=16) :: logname character(len=16) :: hostname diff --git a/components/elm/tools/interpinic/src/shr_infnan_mod.F90 b/components/elm/tools/interpinic/src/shr_infnan_mod.F90 index 638cad84d20..31ffb1dff32 100644 --- a/components/elm/tools/interpinic/src/shr_infnan_mod.F90 +++ b/components/elm/tools/interpinic/src/shr_infnan_mod.F90 @@ -2,11 +2,11 @@ module shr_infnan_mod -!! Inf_NaN_Detection module +!! Inf_NaN_Detection module !! Copyright(c) 2003, Lahey Computer Systems, Inc. -!! Copies of this source code, or standalone compiled files +!! Copies of this source code, or standalone compiled files !! derived from this source may not be sold without permission -!! from Lahey Computers Systems. All or part of this module may be +!! from Lahey Computers Systems. All or part of this module may be !! freely incorporated into executable programs which are offered !! for sale. Otherwise, distribution of all or part of this file is !! permitted, provided this copyright notice and header are included. @@ -22,12 +22,12 @@ module shr_infnan_mod !! isneginf(x) - test for a negative "infinite" value !! !! Each function accepts a single or double precision real argument, and -!! returns a true or false value to indicate the presence of the value +!! returns a true or false value to indicate the presence of the value !! being tested for. If the argument is array valued, the function returns !! a conformable logical array, suitable for use with the ANY function, or !! as a logical mask. !! -!! Each function operates by transferring the bit pattern from a real +!! Each function operates by transferring the bit pattern from a real !! variable to an integer container. Unless testing for + or - infinity, !! the sign bit is cleared to zero. The value is exclusive ORed with !! the value being tested for. The integer result of the IEOR function is @@ -48,14 +48,14 @@ module shr_infnan_mod integer, parameter :: Double = selected_int_kind(precision(1.0_r8)) ! Single precision IEEE values - integer(Single), parameter :: sNaN = Z"7FC00000" - integer(Single), parameter :: sPosInf = Z"7F800000" - integer(Single), parameter :: sNegInf = Z"FF800000" + integer(Single), parameter :: sNaN = int(Z"7FC00000") + integer(Single), parameter :: sPosInf = int(Z"7F800000") + integer(Single), parameter :: sNegInf = int(Z"FF800000") ! Double precision IEEE values - integer(Double), parameter :: dNaN = Z"7FF8000000000000" - integer(Double), parameter :: dPosInf = Z"7FF0000000000000" - integer(Double), parameter :: dNegInf = Z"FFF0000000000000" + integer(Double), parameter :: dNaN = int(Z"7FF8000000000000") + integer(Double), parameter :: dPosInf = int(Z"7FF0000000000000") + integer(Double), parameter :: dNegInf = int(Z"FFF0000000000000") ! Locatation of single and double precision sign bit (Intel) ! Subtract one because bit numbering starts at zero @@ -84,22 +84,22 @@ module shr_infnan_mod module procedure sisnan module procedure disnan #endif - end interface + end interface interface shr_infnan_isinf module procedure sisinf module procedure disinf - end interface - + end interface + interface shr_infnan_isposinf module procedure sisposinf module procedure disposinf - end interface - + end interface + interface shr_infnan_isneginf module procedure sisneginf module procedure disneginf - end interface + end interface integer :: shr_sisnan @@ -107,7 +107,7 @@ module shr_infnan_mod integer :: shr_disnan external :: shr_disnan -contains +contains ! ! If FORTRAN intrinsic's exist use them @@ -134,7 +134,7 @@ elemental function sisnan(x) result(res) res = isnan(x) #endif - end function + end function ! Double precision test for NaN elemental function disnan(d) result(res) @@ -156,7 +156,7 @@ elemental function disnan(d) result(res) res = isnan(d) #endif - end function + end function ! ! Otherwise link to a C function call that either uses the C90 isnan function or a x != x check @@ -176,13 +176,13 @@ function c_sisnan_1D(x) result(res) real(r4), intent(in) :: x(:) logical :: res(size(x)) - integer :: i + integer :: i do i = 1, size(x) res(i) = (shr_sisnan(x(i)) /= 0) end do end function c_sisnan_1D - + function c_sisnan_2D(x) result(res) real(r4), intent(in) :: x(:,:) logical :: res(size(x,1),size(x,2)) @@ -195,7 +195,7 @@ function c_sisnan_2D(x) result(res) end do end do end function c_sisnan_2D - + function c_sisnan_3D(x) result(res) real(r4), intent(in) :: x(:,:,:) logical :: res(size(x,1),size(x,2),size(x,3)) @@ -210,7 +210,7 @@ function c_sisnan_3D(x) result(res) end do end do end function c_sisnan_3D - + function c_sisnan_4D(x) result(res) real(r4), intent(in) :: x(:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4)) @@ -227,7 +227,7 @@ function c_sisnan_4D(x) result(res) end do end do end function c_sisnan_4D - + function c_sisnan_5D(x) result(res) real(r4), intent(in) :: x(:,:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4),size(x,5)) @@ -246,7 +246,7 @@ function c_sisnan_5D(x) result(res) end do end do end function c_sisnan_5D - + function c_sisnan_6D(x) result(res) real(r4), intent(in) :: x(:,:,:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4),size(x,5),size(x,6)) @@ -267,7 +267,7 @@ function c_sisnan_6D(x) result(res) end do end do end function c_sisnan_6D - + function c_sisnan_7D(x) result(res) real(r4), intent(in) :: x(:,:,:,:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4),size(x,5),size(x,6),size(x,7)) @@ -290,7 +290,7 @@ function c_sisnan_7D(x) result(res) end do end do end function c_sisnan_7D - + function c_disnan_scalar(x) result(res) real(r8), intent(in) :: x logical :: res @@ -302,13 +302,13 @@ function c_disnan_1D(x) result(res) real(r8), intent(in) :: x(:) logical :: res(size(x)) - integer :: i + integer :: i do i = 1, size(x) res(i) = (shr_disnan(x(i)) /= 0) end do end function c_disnan_1D - + function c_disnan_2D(x) result(res) real(r8), intent(in) :: x(:,:) logical :: res(size(x,1),size(x,2)) @@ -321,7 +321,7 @@ function c_disnan_2D(x) result(res) end do end do end function c_disnan_2D - + function c_disnan_3D(x) result(res) real(r8), intent(in) :: x(:,:,:) logical :: res(size(x,1),size(x,2),size(x,3)) @@ -336,7 +336,7 @@ function c_disnan_3D(x) result(res) end do end do end function c_disnan_3D - + function c_disnan_4D(x) result(res) real(r8), intent(in) :: x(:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4)) @@ -353,7 +353,7 @@ function c_disnan_4D(x) result(res) end do end do end function c_disnan_4D - + function c_disnan_5D(x) result(res) real(r8), intent(in) :: x(:,:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4),size(x,5)) @@ -372,7 +372,7 @@ function c_disnan_5D(x) result(res) end do end do end function c_disnan_5D - + function c_disnan_6D(x) result(res) real(r8), intent(in) :: x(:,:,:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4),size(x,5),size(x,6)) @@ -393,7 +393,7 @@ function c_disnan_6D(x) result(res) end do end do end function c_disnan_6D - + function c_disnan_7D(x) result(res) real(r8), intent(in) :: x(:,:,:,:,:,:,:) logical :: res(size(x,1),size(x,2),size(x,3),size(x,4),size(x,5),size(x,6),size(x,7)) @@ -418,48 +418,48 @@ function c_disnan_7D(x) result(res) end function c_disnan_7D #endif - + ! Single precision test for Inf elemental function sisinf(x) result(res) real(r4), intent(in) :: x logical :: res res = ieor(ibclr(transfer(x,sPosInf),SPSB), sPosInf) == 0 - end function + end function ! Double precision test for Inf elemental function disinf(d) result(res) real(r8), intent(in) :: d logical :: res res = ieor(ibclr(transfer(d,dPosInf),DPSB), dPosInf) == 0 - end function - + end function + ! Single precision test for +Inf elemental function sisposinf(x) result(res) real(r4), intent(in) :: x logical :: res res = ieor(transfer(x,sPosInf), sPosInf) == 0 - end function + end function ! Double precision test for +Inf elemental function disposinf(d) result(res) real(r8), intent(in) :: d logical :: res res = ieor(transfer(d,dPosInf), dPosInf) == 0 - end function - + end function + ! Single precision test for -Inf elemental function sisneginf(x) result(res) real(r4), intent(in) :: x logical :: res res = ieor(transfer(x,sNegInf), sNegInf) == 0 - end function + end function ! Double precision test for -Inf elemental function disneginf(d) result(res) real(r8), intent(in) :: d logical :: res res = ieor(transfer(d,dNegInf), dNegInf) == 0 - end function + end function end module shr_infnan_mod diff --git a/components/homme/CMakeLists.txt b/components/homme/CMakeLists.txt index 80a89a29691..ec291d8bd26 100644 --- a/components/homme/CMakeLists.txt +++ b/components/homme/CMakeLists.txt @@ -206,7 +206,9 @@ IF (HOMME_USE_KOKKOS) STRING (TOUPPER ${HOMMEXX_EXEC_SPACE} HOMMEXX_EXEC_SPACE_UPPER) - IF (HOMMEXX_EXEC_SPACE_UPPER STREQUAL "HIP") + IF (${HOMMEXX_EXEC_SPACE_UPPER} STREQUAL "SYCL") + SET (HOMMEXX_SYCL_SPACE ON) + ELSEIF (${HOMMEXX_EXEC_SPACE_UPPER} STREQUAL "HIP") SET (HOMMEXX_HIP_SPACE ON) ELSEIF (HOMMEXX_EXEC_SPACE_UPPER STREQUAL "CUDA") SET (HOMMEXX_CUDA_SPACE ON) @@ -254,9 +256,22 @@ elseif (NOT TARGET csm_share) message (FATAL_ERROR "Aborting.") endif() -OPTION(HOMME_USE_MKL "Whether to use Intel's MKL instead of blas/lapack" FALSE) +OPTION(HOMME_USE_MKL "Whether to use Intel's MKL/oneMKL instead of blas/lapack" FALSE) IF(HOMME_USE_MKL) - MESSAGE(STATUS "HOMME_USE_MKL is ON. The flag -mkl will be added to each executable/library.") + IF(DEFINED ENV{MKLROOT}) + SET(MKL_ROOT_PATH "$ENV{MKLROOT}") + IF(MKL_ROOT_PATH MATCHES "oneapi/mkl") + SET(MKL_TYPE "oneMKL") + MESSAGE(STATUS "Detected oneMKL based on MKLROOT: ${MKL_ROOT_PATH}") + FIND_PACKAGE(MKL REQUIRED $ENV{MKLROOT}/lib/cmake/mkl) + ELSE() + SET(MKL_TYPE "Intel MKL") + MESSAGE(STATUS "Detected standalone Intel MKL based on MKLROOT: ${MKL_ROOT_PATH}") + ENDIF() + ELSE() + MESSAGE(FATAL_ERROR "MKLROOT environment variable is not set. Please set it to your MKL installation path.") + ENDIF() + MESSAGE(STATUS "HOMME_USE_MKL is ON. The flag -mkl/-qmkl will be added to each executable/library.") ELSE() OPTION(HOMME_FIND_BLASLAPACK "Whether to use system blas/lapack" FALSE) MESSAGE(STATUS "HOMME_FIND_BLASLAPACK=${HOMME_FIND_BLASLAPACK}") @@ -303,7 +318,7 @@ SET (HOMMEXX_ENABLE_GPU_F90 FALSE) IF (HOMME_USE_KOKKOS) - IF (CUDA_BUILD OR HIP_BUILD) + IF (CUDA_BUILD OR HIP_BUILD OR SYCL_BUILD) SET (DEFAULT_VECTOR_SIZE 1) SET (HOMMEXX_ENABLE_GPU TRUE) SET (HOMMEXX_ENABLE_GPU_F90 TRUE) @@ -312,7 +327,7 @@ IF (HOMME_USE_KOKKOS) ENDIF() SET (HOMMEXX_VECTOR_SIZE ${DEFAULT_VECTOR_SIZE} CACHE STRING - "If AVX or Cuda or HIP don't take priority, use this software vector size.") + "If AVX or Cuda or HIP or SYCL don't take priority, use this software vector size.") IF (CMAKE_BUILD_TYPE_UPPER MATCHES "DEBUG" OR CMAKE_BUILD_TYPE_UPPER MATCHES "RELWITHDEBINFO") SET (HOMMEXX_DEBUG ON) diff --git a/components/homme/cmake/HommeMacros.cmake b/components/homme/cmake/HommeMacros.cmake index 6d073dbbe83..29a3c64bba9 100644 --- a/components/homme/cmake/HommeMacros.cmake +++ b/components/homme/cmake/HommeMacros.cmake @@ -112,7 +112,13 @@ macro(createTestExec execName execType macroNP macroNC ADD_DEFINITIONS(-DHAVE_CONFIG_H) ADD_EXECUTABLE(${execName} ${EXEC_SOURCES}) - SET_TARGET_PROPERTIES(${execName} PROPERTIES LINKER_LANGUAGE Fortran) + # For SYCL builds it is suggested to use CXX linker with `-fortlib` + # for mixed-language setups + IF(SYCL_BUILD) + SET_TARGET_PROPERTIES(${execName} PROPERTIES LINKER_LANGUAGE CXX) + ELSE() + SET_TARGET_PROPERTIES(${execName} PROPERTIES LINKER_LANGUAGE Fortran) + ENDIF() IF(BUILD_HOMME_WITHOUT_PIOLIBRARY) TARGET_COMPILE_DEFINITIONS(${execName} PUBLIC HOMME_WITHOUT_PIOLIBRARY) ENDIF() @@ -156,17 +162,20 @@ macro(createTestExec execName execType macroNP macroNC ENDIF () IF (HOMME_USE_KOKKOS) - target_link_libraries(${execName} Kokkos::kokkos) + TARGET_LINK_LIBRARIES(${execName} Kokkos::kokkos) ENDIF () # Move the module files out of the way so the parallel build # doesn't have a race condition SET_TARGET_PROPERTIES(${execName} - PROPERTIES Fortran_MODULE_DIRECTORY ${EXEC_MODULE_DIR}) + PROPERTIES Fortran_MODULE_DIRECTORY ${EXEC_MODULE_DIR}) IF (HOMME_USE_MKL) - TARGET_COMPILE_OPTIONS(${execName} PUBLIC -mkl) - TARGET_LINK_LIBRARIES(${execName} -mkl) + IF (MKL_TYPE STREQUAL "oneMKL") + TARGET_LINK_LIBRARIES(${execName} -qmkl) + ELSEIF (MKL_TYPE STREQUAL "Intel MKL") + TARGET_LINK_LIBRARIES(${execName} -mkl) + ENDIF () ELSE() IF (NOT HOMME_FIND_BLASLAPACK) TARGET_LINK_LIBRARIES(${execName} lapack blas) @@ -264,8 +273,11 @@ macro(createExecLib libName execType libSrcs inclDirs macroNP ENDIF () IF (HOMME_USE_MKL) - TARGET_COMPILE_OPTIONS(${libName} PUBLIC -mkl) - TARGET_LINK_LIBRARIES(${libName} -mkl) + IF (MKL_TYPE STREQUAL "oneMKL") + TARGET_LINK_LIBRARIES(${libName} -qmkl) + ELSEIF (MKL_TYPE STREQUAL "Intel MKL") + TARGET_LINK_LIBRARIES(${libName} -mkl) + ENDIF () ELSE() IF (NOT HOMME_FIND_BLASLAPACK) TARGET_LINK_LIBRARIES(${libName} lapack blas) diff --git a/components/homme/cmake/machineFiles/aurora-aot.cmake b/components/homme/cmake/machineFiles/aurora-aot.cmake new file mode 100644 index 00000000000..14e6d008da8 --- /dev/null +++ b/components/homme/cmake/machineFiles/aurora-aot.cmake @@ -0,0 +1,55 @@ +#module restore +#module load spack-pe-base cmake +#module list + + +SET (AURORA_MACHINE TRUE CACHE BOOL "") + +SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "") +SET(HOMMEXX_MPI_ON_DEVICE FALSE CACHE BOOL "") + +SET(HOMME_FIND_BLASLAPACK TRUE CACHE BOOL "") + +SET(WITH_PNETCDF FALSE CACHE FILEPATH "") + +SET(USE_QUEUING FALSE CACHE BOOL "") + +#temp hack +SET(HOMME_USE_KOKKOS TRUE CACHE BOOL "") +SET(HOMME_USE_MKL TRUE CACHE BOOL "") + +SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "") +SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "") + +set(Kokkos_ROOT $ENV{KOKKOS_HOME} CACHE STRING "") + +SET(USE_TRILINOS OFF CACHE BOOL "") + +SET(SYCL_BUILD TRUE CACHE BOOL "") +SET(HOMME_ENABLE_COMPOSE FALSE CACHE BOOL "") + +SET(CMAKE_CXX_STANDARD 17) + +SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") +SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") +SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "") + +#AOT flags +SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") +SET(SYCL_LINK_FLAGS "-fsycl-max-parallel-link-jobs=32 -fsycl -fsycl-device-code-split=per_kernel -fsycl-targets=intel_gpu_pvc") + +SET(ADD_Fortran_FLAGS "-fc=ifx -fpscomp logicals -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "") +SET(ADD_C_FLAGS "-O3 -DNDEBUG " CACHE STRING "") + +SET(ADD_CXX_FLAGS "-std=c++17 -fp-model=precise -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "") +SET(ADD_LINKER_FLAGS "-O3 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "") + +set (ENABLE_OPENMP OFF CACHE BOOL "") +set (ENABLE_COLUMN_OPENMP OFF CACHE BOOL "") +set (ENABLE_HORIZ_OPENMP OFF CACHE BOOL "") + +set (HOMME_TESTING_PROFILE "dev" CACHE STRING "") + +set (USE_NUM_PROCS 12 CACHE STRING "") + +SET (USE_MPI_OPTIONS "--pmi=pmix --cpu-bind list:0-7,104-111:8-15,112-119:16-23,120-127:24-31,128-135:32-39,136-143:40-47,144-151:52-59,156-163:60-67,164-171:68-75,172-179:76-83,180-187:84-91,188-195:92-99,196-203 gpu_tile_compact.sh" CACHE FILEPATH "") diff --git a/components/homme/cmake/machineFiles/aurora-jit.cmake b/components/homme/cmake/machineFiles/aurora-jit.cmake new file mode 100644 index 00000000000..8c35930fdf6 --- /dev/null +++ b/components/homme/cmake/machineFiles/aurora-jit.cmake @@ -0,0 +1,55 @@ +#module restore +#module load spack-pe-base cmake +#module list + + +SET (AURORA_MACHINE TRUE CACHE BOOL "") + +SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "") +SET(HOMMEXX_MPI_ON_DEVICE FALSE CACHE BOOL "") + +SET(HOMME_FIND_BLASLAPACK TRUE CACHE BOOL "") + +SET(WITH_PNETCDF FALSE CACHE FILEPATH "") + +SET(USE_QUEUING FALSE CACHE BOOL "") + +#temp hack +SET(HOMME_USE_KOKKOS TRUE CACHE BOOL "") +SET(HOMME_USE_MKL TRUE CACHE BOOL "") + +SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "") +SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "") + +set(Kokkos_ROOT $ENV{KOKKOS_HOME} CACHE STRING "") + +SET(USE_TRILINOS OFF CACHE BOOL "") + +SET(SYCL_BUILD TRUE CACHE BOOL "") +SET(HOMME_ENABLE_COMPOSE FALSE CACHE BOOL "") + +SET(CMAKE_CXX_STANDARD 17) + +SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") +SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") +SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "") + +#AOT flags +SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") +SET(SYCL_LINK_FLAGS "-fsycl-max-parallel-link-jobs=32 -fsycl") + +SET(ADD_Fortran_FLAGS "-fc=ifx -fpscomp logicals -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "") +SET(ADD_C_FLAGS "-O3 -DNDEBUG " CACHE STRING "") + +SET(ADD_CXX_FLAGS "-std=c++17 -fp-model=precise -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "") +SET(ADD_LINKER_FLAGS "-O3 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "") + +set (ENABLE_OPENMP OFF CACHE BOOL "") +set (ENABLE_COLUMN_OPENMP OFF CACHE BOOL "") +set (ENABLE_HORIZ_OPENMP OFF CACHE BOOL "") + +set (HOMME_TESTING_PROFILE "dev" CACHE STRING "") + +set (USE_NUM_PROCS 12 CACHE STRING "") + +SET (USE_MPI_OPTIONS "--pmi=pmix --cpu-bind list:0-7,104-111:8-15,112-119:16-23,120-127:24-31,128-135:32-39,136-143:40-47,144-151:52-59,156-163:60-67,164-171:68-75,172-179:76-83,180-187:84-91,188-195:92-99,196-203 gpu_tile_compact.sh" CACHE FILEPATH "") diff --git a/components/homme/cmake/machineFiles/polaris-a100.sh b/components/homme/cmake/machineFiles/polaris-a100.sh new file mode 100644 index 00000000000..2b63c61a55e --- /dev/null +++ b/components/homme/cmake/machineFiles/polaris-a100.sh @@ -0,0 +1,74 @@ +#Currently Loaded Modules: +# 1) craype-x86-rome 6) craype/2.7.15 11) cray-libpals/1.1.7 16) nvhpc-mixed/21.9 +# 2) libfabric/1.11.0.4.125 7) cray-dsmml/0.2.2 12) PrgEnv-gnu/8.3.3 17) cudatoolkit-standalone/11.6.2 +# 3) craype-network-ofi 8) cray-pmi/6.1.2 13) gnu-parallel/2021-09-22 18) cmake/3.23.2 +# 4) perftools-base/22.05.0 9) cray-pmi-lib/6.0.17 14) gcc/11.2.0 +# 5) craype-accel-nvidia80 10) cray-pals/1.1.7 15) cray-mpich/8.1.16 + + + +#SET(HOMMEXX_EXEC_SPACE CUDA CACHE STRING "") +#SET(HOMMEXX_MPI_ON_DEVICE FALSE CACHE BOOL "") +#SET(HOMMEXX_CUDA_MAX_WARP_PER_TEAM "16" CACHE STRING "") + +# cray-hdf5-parallel/1.12.0.6 cray-netcdf-hdf5parallel/4.7.4.6 cray-parallel-netcdf/1.12.1.6 +#SET(NETCDF_DIR $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} CACHE FILEPATH "") +#SET(PNETCDF_DIR $ENV{CRAY_PARALLEL_NETCDF_DIR} CACHE FILEPATH "") +#SET(HDF5_DIR $ENV{CRAY_HDF5_PARALLEL_PREFIX} CACHE FILEPATH "") + +#for scorpio +#SET (NetCDF_C_PATH $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} CACHE FILEPATH "") +#SET (NetCDF_Fortran_PATH $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} CACHE FILEPATH "") + +SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "") + +SET(HOMME_FIND_BLASLAPACK FALSE CACHE BOOL "") + +SET(WITH_PNETCDF FALSE CACHE FILEPATH "") + +SET(USE_QUEUING FALSE CACHE BOOL "") + +SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "") + +SET(CUDA_BUILD TRUE CACHE BOOL "") + +#SET(HOMMEXX_BFB_TESTING TRUE CACHE BOOL "") + +SET(USE_TRILINOS OFF CACHE BOOL "") + +SET(Kokkos_ENABLE_OPENMP OFF CACHE BOOL "") +SET(Kokkos_ENABLE_CUDA ON CACHE BOOL "") +SET(Kokkos_ENABLE_CUDA_LAMBDA ON CACHE BOOL "") +SET(Kokkos_ARCH_AMPERE80 ON CACHE BOOL "") +#SET(Kokkos_ARCH_ZEN2 ON CACHE BOOL "") # works, and perf same if both AMPERE80 and ZEN2 are on +#SET(Kokkos_ENABLE_CUDA_UVM ON CACHE BOOL "") +SET(Kokkos_ENABLE_EXPLICIT_INSTANTIATION OFF CACHE BOOL "") +#SET(Kokkos_ENABLE_CUDA_ARCH_LINKING OFF CACHE BOOL "") + +#SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") +#SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") +#SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "") +SET(CMAKE_C_COMPILER "cc" CACHE STRING "") +SET(CMAKE_Fortran_COMPILER "ftn" CACHE STRING "") +SET(CMAKE_CXX_COMPILER "CC" CACHE STRING "") + +#SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") +#SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") +#SET(CMAKE_CXX_COMPILER "${CMAKE_CURRENT_SOURCE_DIR}/../../externals/kokkos/bin/nvcc_wrapper" CACHE STRING "") + +# Note: need to set MPICH_CXX env variable and perhaps NVCC_WRAPPER_DEFAULT_COMPILER + +SET(CXXLIB_SUPPORTED_CACHE FALSE CACHE BOOL "") + +SET(ENABLE_OPENMP OFF CACHE BOOL "") +SET(ENABLE_COLUMN_OPENMP OFF CACHE BOOL "") +SET(ENABLE_HORIZ_OPENMP OFF CACHE BOOL "") + +SET(CMAKE_VERBOSE_MAKEFILE ON CACHE BOOL "") + +#SET(HOMME_TESTING_PROFILE "dev" CACHE STRING "") + +SET(USE_NUM_PROCS 4 CACHE STRING "") + +SET(USE_MPIEXEC "srun" CACHE STRING "") +#SET(CPRNC_DIR /global/cfs/cdirs/e3sm/tools/cprnc CACHE FILEPATH "") diff --git a/components/homme/cmake/machineFiles/spot-aot-AB2.cmake b/components/homme/cmake/machineFiles/spot-aot-AB2.cmake new file mode 100644 index 00000000000..23fad2361cc --- /dev/null +++ b/components/homme/cmake/machineFiles/spot-aot-AB2.cmake @@ -0,0 +1,63 @@ +#module restore +#module load oneapi/eng-compiler/2022.12.30.005 +#module load intel_compute_runtime/release/agama-devel-627 +#module load spack cmake +#module list + +SET (SUNSPOT_MACHINE TRUE CACHE BOOL "") + +SET (HOMMEXX_MPI_ON_DEVICE TRUE CACHE BOOL "") + +#SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "") + +SET(HOMME_FIND_BLASLAPACK TRUE CACHE BOOL "") + +SET(WITH_PNETCDF FALSE CACHE FILEPATH "") + +SET(USE_QUEUING FALSE CACHE BOOL "") + +#temp hack +SET(HOMME_USE_KOKKOS TRUE CACHE BOOL "") + +SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "") +SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "") + +#set(KOKKOS_HOME "/home/onguba/kokkos-build/june22-2024-aot/install" CACHE STRING "") +#set(E3SM_KOKKOS_PATH ${KOKKOS_HOME} CACHE STRING "") + +SET (NetCDF_Fortran_PATH "/lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf" CACHE STRING "") +SET (NetCDF_C_PATH "/lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf" CACHE STRING "") + +SET(USE_TRILINOS OFF CACHE BOOL "") + +SET(SYCL_BUILD TRUE CACHE BOOL "") +SET(HOMME_ENABLE_COMPOSE FALSE CACHE BOOL "") + +#SET(CMAKE_CXX_STANDARD 17) +SET(CMAKE_CXX_STANDARD 17 CACHE STRING "CXX Standard") + +SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") +SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") +SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "") + +SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") +SET(SYCL_LINK_FLAGS "-fsycl-max-parallel-link-jobs=32 -fsycl-link-huge-device-code -fsycl -fsycl-device-code-split=per_kernel -fsycl-targets=spir64_gen -Xsycl-target-backend \"-device 12.60.7\"") + +#-fpscomp does not actually solve the issue with bools in here,another suggestion was -fp-model=precise, not working either +SET(ADD_Fortran_FLAGS " -fc=ifx -fpscomp logicals -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "") +SET(ADD_C_FLAGS "-O3 -DNDEBUG " CACHE STRING "") + +SET(ADD_CXX_FLAGS " -std=c++17 -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "") +SET(ADD_LINKER_FLAGS "-O3 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "") + +set (ENABLE_OPENMP OFF CACHE BOOL "") +set (ENABLE_COLUMN_OPENMP OFF CACHE BOOL "") +set (ENABLE_HORIZ_OPENMP OFF CACHE BOOL "") + +set (HOMME_TESTING_PROFILE "dev" CACHE STRING "") + +set (USE_NUM_PROCS 4 CACHE STRING "") + +SET (USE_MPI_OPTIONS "--bind-to core" CACHE FILEPATH "") + + diff --git a/components/homme/src/preqx_kokkos/cxx/CamForcing.cpp b/components/homme/src/preqx_kokkos/cxx/CamForcing.cpp index 2b1e6514389..36ca5f4a95f 100644 --- a/components/homme/src/preqx_kokkos/cxx/CamForcing.cpp +++ b/components/homme/src/preqx_kokkos/cxx/CamForcing.cpp @@ -51,7 +51,7 @@ void state_forcing( void tracer_forcing( const ExecViewUnmanaged &f_q, const HybridVCoord &hvcoord, const TimeLevel &tl, const int &num_q, - const MoistDry &moisture, const double &dt, + const bool &use_moisture, const double &dt, const ExecViewManaged &ps_v, const ExecViewManaged< Scalar * [Q_NUM_TIME_LEVELS][QSIZE_D][NP][NP][NUM_LEV]> &qdp, @@ -61,7 +61,7 @@ void tracer_forcing( const int np1 = tl.n0; const int np1_qdp = tl.n0_qdp; - if (moisture == MoistDry::MOIST) { + if (use_moisture) { // Remove the m_fq_ps_v buffer since it's not actually needed. // Instead apply the forcing to m_ps_v directly // Bonus - one less parallel reduce in dry cases! @@ -161,7 +161,7 @@ void apply_cam_forcing(const Real &dt) { tracers.fq = decltype(tracers.fq)("fq", elems.num_elems(),tracers.num_tracers()); } tracer_forcing(tracers.fq, hvcoord, tl, tracers.num_tracers(), - sim_params.moisture, dt, elems.m_state.m_ps_v, tracers.qdp, tracers.Q); + sim_params.use_moisture, dt, elems.m_state.m_ps_v, tracers.qdp, tracers.Q); GPTLstop("ApplyCAMForcing"); } diff --git a/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp b/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp index c75143a9836..aa21f5c16f2 100644 --- a/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp +++ b/components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp @@ -37,7 +37,7 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, const int& time_step_type, const int& qsize, const int& state_frequency, const Real& nu, const Real& nu_p, const Real& nu_q, const Real& nu_s, const Real& nu_div, const Real& nu_top, const int& hypervis_order, const int& hypervis_subcycle, const double& hypervis_scaling, - const int& ftype, const bool& prescribed_wind, const bool& moisture, const bool& disable_diagnostics, + const int& ftype, const bool& prescribed_wind, const bool& use_moisture, const bool& disable_diagnostics, const bool& use_cpstar, const int& transport_alg, const int& dt_remap_factor, const int& dt_tracer_factor, const double& scale_factor, const double& laplacian_rigid_factor) @@ -90,7 +90,8 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option, params.hypervis_subcycle = hypervis_subcycle; params.hypervis_scaling = hypervis_scaling; params.disable_diagnostics = disable_diagnostics; - params.moisture = (moisture ? MoistDry::MOIST : MoistDry::DRY); + params.use_moisture = use_moisture; + params.moisture = params.use_moisture ? MoistDry::MOIST : MoistDry::DRY; //todo-repo-unification params.use_cpstar = use_cpstar; params.transport_alg = transport_alg; // SphereOperators parameters; preqx supports only the sphere. diff --git a/components/homme/src/preqx_kokkos/cxx/prim_advance_exp.cpp b/components/homme/src/preqx_kokkos/cxx/prim_advance_exp.cpp index f7c7600aab8..58e58f0160b 100644 --- a/components/homme/src/preqx_kokkos/cxx/prim_advance_exp.cpp +++ b/components/homme/src/preqx_kokkos/cxx/prim_advance_exp.cpp @@ -34,7 +34,7 @@ void prim_advance_exp (TimeLevel& tl, const Real dt, const bool compute_diagnost // Determine the tracers time level tl.n0_qdp= -1; - if (params.moisture == MoistDry::MOIST) { + if (params.use_moisture) { tl.update_tracers_levels(params.qsplit); } diff --git a/components/homme/src/prim_main.F90 b/components/homme/src/prim_main.F90 index bfbe57e8b31..d6901151d36 100644 --- a/components/homme/src/prim_main.F90 +++ b/components/homme/src/prim_main.F90 @@ -20,7 +20,7 @@ program prim_main use element_mod, only: element_t use common_io_mod, only: output_dir, infilenames use common_movie_mod, only: nextoutputstep - use perf_mod, only: t_initf, t_prf, t_finalizef, t_startf, t_stopf ! _EXTERNAL + use perf_mod, only: t_initf, t_prf, t_finalizef, t_startf, t_stopf, t_disablef, t_enablef ! _EXTERNAL use restart_io_mod , only: restartheader_t, writerestart use hybrid_mod, only: hybrid_create #if (defined MODEL_THETA_L && defined ARKODE) @@ -240,6 +240,11 @@ end subroutine finalize_kokkos_f90 nstep = nextoutputstep(tl) do while(tl%nstep= 2) call t_enablef() call t_startf('prim_run') call prim_run_subcycle(elem, hybrid,nets,nete, tstep, .false., tl, hvcoord,1) call t_stopf('prim_run') diff --git a/components/homme/src/share/cxx/Config.hpp b/components/homme/src/share/cxx/Config.hpp index 684f9143bea..b204b1dbd04 100644 --- a/components/homme/src/share/cxx/Config.hpp +++ b/components/homme/src/share/cxx/Config.hpp @@ -21,7 +21,7 @@ # endif #endif -#if ! defined HOMMEXX_CUDA_SPACE && ! defined HOMMEXX_OPENMP_SPACE && ! defined HOMMEXX_THREADS_SPACE && ! defined HOMMEXX_SERIAL_SPACE && ! defined HOMMEXX_HIP_SPACE +#if ! defined HOMMEXX_CUDA_SPACE && ! defined HOMMEXX_OPENMP_SPACE && ! defined HOMMEXX_THREADS_SPACE && ! defined HOMMEXX_SERIAL_SPACE && ! defined HOMMEXX_HIP_SPACE && ! defined HOMMEXX_SYCL_SPACE # define HOMMEXX_DEFAULT_SPACE #endif diff --git a/components/homme/src/share/cxx/EulerStepFunctorImpl.hpp b/components/homme/src/share/cxx/EulerStepFunctorImpl.hpp index f3029764dac..f87bb108beb 100644 --- a/components/homme/src/share/cxx/EulerStepFunctorImpl.hpp +++ b/components/homme/src/share/cxx/EulerStepFunctorImpl.hpp @@ -652,7 +652,10 @@ class EulerStepFunctorImpl { minmax_and_biharmonic(); } } + + GPTLstart("tl-at adv-n-limit"); advect_and_limit(); + GPTLstop("tl-at adv-n-limit"); exchange_qdp_dss_var(); } @@ -667,6 +670,7 @@ class EulerStepFunctorImpl { void run_tracer_phase (const KernelVariables& kv) const { compute_qtens(kv); kv.team_barrier(); + if (m_data.limiter_option == 8) { limiter_optim_iter_full(kv); kv.team_barrier(); @@ -674,6 +678,7 @@ class EulerStepFunctorImpl { limiter_clip_and_sum(kv); kv.team_barrier(); } + apply_spheremp(kv); } diff --git a/components/homme/src/share/cxx/ExecSpaceDefs.cpp b/components/homme/src/share/cxx/ExecSpaceDefs.cpp index 8d496bff5d1..0b7a3ab34f8 100644 --- a/components/homme/src/share/cxx/ExecSpaceDefs.cpp +++ b/components/homme/src/share/cxx/ExecSpaceDefs.cpp @@ -21,6 +21,10 @@ #include #endif +#ifdef KOKKOS_ENABLE_SYCL +#include +#endif + namespace Homme { // Since we're initializing from inside a Fortran code and don't have access to @@ -52,7 +56,16 @@ void initialize_kokkos () { // It isn't a big deal if we can't get the device count. nd = 1; } +#elif defined(KOKKOS_ENABLE_SYCL) + +//https://developer.codeplay.com/products/computecpp/ce/2.11.0/guides/sycl-for-cuda-developers/migrating-from-cuda-to-sycl + +//to make it build + int nd = 1; + #endif + + #ifdef HOMMEXX_ENABLE_GPU std::stringstream ss; ss << "--kokkos-num-devices=" << nd; @@ -117,6 +130,7 @@ team_num_threads_vectors_for_gpu ( assert(num_warps_total >= max_num_warps); assert(tp.max_threads_usable >= 1 && tp.max_vectors_usable >= 1); +#ifndef KOKKOS_ENABLE_SYCL int num_warps; if (tp.prefer_larger_team) { const int num_warps_usable = @@ -161,6 +175,9 @@ team_num_threads_vectors_for_gpu ( return std::make_pair( num_device_threads / num_vectors, num_vectors ); } +#else + return std::make_pair(16,8); +#endif } } // namespace Parallel diff --git a/components/homme/src/share/cxx/ExecSpaceDefs.hpp b/components/homme/src/share/cxx/ExecSpaceDefs.hpp index cd6649c7ab2..82f5e803801 100644 --- a/components/homme/src/share/cxx/ExecSpaceDefs.hpp +++ b/components/homme/src/share/cxx/ExecSpaceDefs.hpp @@ -34,6 +34,10 @@ using HommexxGPU = Kokkos::Cuda; using HommexxGPU = Kokkos::Experimental::HIP; #endif +#ifdef KOKKOS_ENABLE_SYCL +using HommexxGPU = Kokkos::Experimental::SYCL; +#endif + #else using HommexxGPU = void; #endif diff --git a/components/homme/src/share/cxx/GllFvRemap.cpp b/components/homme/src/share/cxx/GllFvRemap.cpp index e36dbc14d74..7b0400427f3 100644 --- a/components/homme/src/share/cxx/GllFvRemap.cpp +++ b/components/homme/src/share/cxx/GllFvRemap.cpp @@ -16,13 +16,13 @@ namespace Homme { void init_gllfvremap_c (int nelemd, int np, int nf, int nf_max, - bool theta_hydrostatic_mode, + int theta_hydrostatic_mode, CF90Ptr fv_metdet, CF90Ptr g2f_remapd, CF90Ptr f2g_remapd, CF90Ptr D_f, CF90Ptr Dinv_f) { auto& c = Context::singleton(); auto& g = c.get(); - g.init_data(nf, nf_max, theta_hydrostatic_mode, fv_metdet, g2f_remapd, - f2g_remapd, D_f, Dinv_f); + const bool thm = static_cast(theta_hydrostatic_mode); + g.init_data(nf, nf_max, thm, fv_metdet, g2f_remapd, f2g_remapd, D_f, Dinv_f); } GllFvRemap::GllFvRemap () { @@ -52,7 +52,7 @@ void GllFvRemap::init_boundary_exchanges () { } void GllFvRemap -::init_data (const int nf, const int nf_max, bool theta_hydrostatic_mode, +::init_data (const int nf, const int nf_max, const bool theta_hydrostatic_mode, const Real* fv_metdet, const Real* g2f_remapd, const Real* f2g_remapd, const Real* D_f, const Real* Dinv_f) { m_impl->init_data(nf, nf_max, theta_hydrostatic_mode, fv_metdet, diff --git a/components/homme/src/share/cxx/GllFvRemap.hpp b/components/homme/src/share/cxx/GllFvRemap.hpp index 07e4bf58a90..7ebf5a82b71 100644 --- a/components/homme/src/share/cxx/GllFvRemap.hpp +++ b/components/homme/src/share/cxx/GllFvRemap.hpp @@ -40,7 +40,7 @@ class GllFvRemap { typedef Phys2T::const_type CPhys2T; typedef Phys3T::const_type CPhys3T; - void init_data(const int nf, const int nf_max, bool theta_hydrostatic_mode, + void init_data(const int nf, const int nf_max, const bool theta_hydrostatic_mode, const Real* fv_metdet, const Real* g2f_remapd, const Real* f2g_remapd, const Real* D_f, const Real* Dinv_f); @@ -81,7 +81,7 @@ class GllFvRemap { extern "C" void init_gllfvremap_c(int nelemd, int np, int nf, int nf_max, - const bool theta_hydrostatic_mode, + const int theta_hydrostatic_mode, CF90Ptr fv_metdet, CF90Ptr g2f_remapd, CF90Ptr f2g_remapd, CF90Ptr D_f, CF90Ptr Dinv_f); diff --git a/components/homme/src/share/cxx/GllFvRemapImpl.cpp b/components/homme/src/share/cxx/GllFvRemapImpl.cpp index 6148f69cfa9..ea1a52f5efd 100644 --- a/components/homme/src/share/cxx/GllFvRemapImpl.cpp +++ b/components/homme/src/share/cxx/GllFvRemapImpl.cpp @@ -142,7 +142,7 @@ ::init_data (const int nf, const int nf_max, const bool theta_hydrostatic_mode, " nf must be > 1.", Errors::err_not_implemented); auto& sp = Context::singleton().get(); - m_data.use_moisture = sp.moisture == MoistDry::MOIST; + m_data.use_moisture = sp.use_moisture; // Only in the unit test gllfvremap_ut does theta_hydrostatic_mode not already // == sp.theta_hydrostatic_mode. m_data.theta_hydrostatic_mode = sp.theta_hydrostatic_mode = theta_hydrostatic_mode; diff --git a/components/homme/src/share/cxx/HommexxEnums.hpp b/components/homme/src/share/cxx/HommexxEnums.hpp index 59c8f3c9652..f41ed9d3651 100644 --- a/components/homme/src/share/cxx/HommexxEnums.hpp +++ b/components/homme/src/share/cxx/HommexxEnums.hpp @@ -40,6 +40,12 @@ enum class ComparisonOp { // =================== Run parameters enums ====================== // +//todo-repo-unification Remove this enum in favor of bool +// SimulationParams::use_moisture once we change EAMxx to use +// use_moisture. Search "todo-repo-unification" for other bits of code to +// remove. +enum class MoistDry { MOIST, DRY }; + enum class ForcingAlg : int { FORCING_OFF =-1, FORCING_0 = 0, @@ -47,11 +53,6 @@ enum class ForcingAlg : int { FORCING_2 = 2, // TODO: Rename FORCING_1 and FORCING_2 to something more descriptive }; -enum class MoistDry { - MOIST, - DRY -}; - enum class AdvectionForm { Conservative, NonConservative diff --git a/components/homme/src/share/cxx/SimulationParams.hpp b/components/homme/src/share/cxx/SimulationParams.hpp index b435911da2e..9247f54a352 100644 --- a/components/homme/src/share/cxx/SimulationParams.hpp +++ b/components/homme/src/share/cxx/SimulationParams.hpp @@ -23,7 +23,8 @@ struct SimulationParams void print(std::ostream& out = std::cout); TimeStepType time_step_type; - MoistDry moisture; + bool use_moisture; + MoistDry moisture; //todo-repo-unification RemapAlg remap_alg; TestCase test_case; ForcingAlg ftype = ForcingAlg::FORCING_OFF; @@ -77,7 +78,7 @@ inline void SimulationParams::print (std::ostream& out) { out << "\n************** CXX SimulationParams **********************\n\n"; out << " time_step_type: " << etoi(time_step_type) << "\n"; - out << " moisture: " << (moisture==MoistDry::DRY ? "dry" : "moist") << "\n"; + out << " use_moisture: " << (use_moisture ? "moist" : "dry") << "\n"; out << " remap_alg: " << etoi(remap_alg) << "\n"; out << " test case: " << etoi(test_case) << "\n"; out << " ftype: " << etoi(ftype) << "\n"; diff --git a/components/homme/src/share/cxx/utilities/BfbUtils.hpp b/components/homme/src/share/cxx/utilities/BfbUtils.hpp index e3570874e26..7fb4d042f7f 100644 --- a/components/homme/src/share/cxx/utilities/BfbUtils.hpp +++ b/components/homme/src/share/cxx/utilities/BfbUtils.hpp @@ -64,7 +64,7 @@ KOKKOS_INLINE_FUNCTION ScalarType int_pow (ScalarType val, int k) { constexpr int max_shift = 30; if (k<0) { - printf ("k = %d\n",k); + Kokkos::printf ("k = %d\n",k); Kokkos::abort("int_pow implemented only for k>=0.\n"); } diff --git a/components/homme/src/share/cxx/utilities/scream_tridiag.hpp b/components/homme/src/share/cxx/utilities/scream_tridiag.hpp index e18bbc4e7e2..f302432ef79 100644 --- a/components/homme/src/share/cxx/utilities/scream_tridiag.hpp +++ b/components/homme/src/share/cxx/utilities/scream_tridiag.hpp @@ -120,7 +120,7 @@ int get_team_nthr (const TeamMember& team) { return team.team_size(); } -// Impl details for Nvidia and AMD GPUs. +// Impl details for Nvidia, AMD and Intel GPUs. template KOKKOS_FORCEINLINE_FUNCTION int get_thread_id_within_team_gpu (const TeamMember& team) { @@ -128,6 +128,10 @@ int get_thread_id_within_team_gpu (const TeamMember& team) { // Can't use team.team_rank() here because vector direction also uses physical // threads but TeamMember types don't expose that information. return blockDim.x * threadIdx.y + threadIdx.x; +#elif defined(__SYCL_DEVICE_ONLY__) + auto item = team.item(); + return static_cast(item.get_local_range(1) * item.get_local_id(0) + + item.get_local_id(1)); #else assert(0); return -1; @@ -138,6 +142,9 @@ template KOKKOS_FORCEINLINE_FUNCTION int get_team_nthr_gpu (const TeamMember& team) { #if defined __CUDA_ARCH__ || defined __HIP_DEVICE_COMPILE__ return blockDim.x * blockDim.y; +#elif defined __SYCL_DEVICE_ONLY__ + auto item = team.item(); + return static_cast(item.get_local_range(0) * item.get_local_range(1)); #else assert(0); return -1; @@ -161,6 +168,16 @@ KOKKOS_FORCEINLINE_FUNCTION int get_team_nthr (const Kokkos::Impl::HIPTeamMember& team) { return get_team_nthr_gpu(team); } #endif // KOKKOS_ENABLE_HIP + +#ifdef KOKKOS_ENABLE_SYCL +KOKKOS_FORCEINLINE_FUNCTION +int get_thread_id_within_team (const Kokkos::Impl::SYCLTeamMember& team) +{ return get_thread_id_within_team_gpu(team); } +KOKKOS_FORCEINLINE_FUNCTION +int get_team_nthr (const Kokkos::Impl::SYCLTeamMember& team) +{ return get_team_nthr_gpu(team); } +#endif // KOKKOS_ENABLE_SYCL + template KOKKOS_INLINE_FUNCTION const T& min (const T& a, const T& b) { return a < b ? a : b; } @@ -634,7 +651,7 @@ void bfb (const TeamMember& team, const auto f = [&] (const int& j) { impl::bfb_thomas_solve(dl, d, du, Kokkos::subview(X , Kokkos::ALL(), j)); }; - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nrhs), f); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nrhs), f); } template @@ -664,7 +681,7 @@ void bfb (const TeamMember& team, subview(du, ALL(), j), subview(X , ALL(), j)); }; - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nrhs), f); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nrhs), f); } } // namespace tridiag diff --git a/components/homme/src/share/cxx/vector/vector_pragmas.hpp b/components/homme/src/share/cxx/vector/vector_pragmas.hpp index 3a0ae7f97ee..b23788a8ccf 100644 --- a/components/homme/src/share/cxx/vector/vector_pragmas.hpp +++ b/components/homme/src/share/cxx/vector/vector_pragmas.hpp @@ -7,7 +7,7 @@ #ifndef HOMMEXX_VECTOR_PRAGMAS_HPP #define HOMMEXX_VECTOR_PRAGMAS_HPP -#if defined(__INTEL_COMPILER) +#if defined(__INTEL_COMPILER) || defined(__INTEL_CLANG_COMPILER) || defined(__INTEL_LLVM_COMPILER) #define VECTOR_IVDEP_LOOP _Pragma("ivdep") #define ALWAYS_VECTORIZE_LOOP _Pragma("vector always") diff --git a/components/homme/src/share/gllfvremap_mod.F90 b/components/homme/src/share/gllfvremap_mod.F90 index e8013df217b..d9dd8e33365 100644 --- a/components/homme/src/share/gllfvremap_mod.F90 +++ b/components/homme/src/share/gllfvremap_mod.F90 @@ -266,21 +266,22 @@ end subroutine gfr_init subroutine gfr_init_hxx() bind(c) #if KOKKOS_TARGET use control_mod, only: theta_hydrostatic_mode - use iso_c_binding, only: c_bool + use iso_c_binding, only: c_int interface subroutine init_gllfvremap_c(nelemd, np, nf, nf_max, theta_hydrostatic_mode, & fv_metdet, g2f_remapd, f2g_remapd, D_f, Dinv_f) bind(c) - use iso_c_binding, only: c_bool, c_int, c_double + use iso_c_binding, only: c_int, c_double integer (c_int), value, intent(in) :: nelemd, np, nf, nf_max - logical (c_bool), value, intent(in) :: theta_hydrostatic_mode + integer (c_int), value, intent(in) :: theta_hydrostatic_mode real (c_double), dimension(nf*nf,nelemd), intent(in) :: fv_metdet real (c_double), dimension(np,np,nf_max*nf_max), intent(in) :: g2f_remapd real (c_double), dimension(nf_max*nf_max,np,np), intent(in) :: f2g_remapd real (c_double), dimension(nf*nf,2,2,nelemd), intent(in) :: D_f, Dinv_f end subroutine init_gllfvremap_c end interface - logical (c_bool) :: thm - thm = theta_hydrostatic_mode + integer (c_int) :: thm + thm = 0 + if (theta_hydrostatic_mode) thm = 1 call init_gllfvremap_c(nelemd, np, gfr%nphys, nphys_max, thm, & gfr%fv_metdet, gfr%g2f_remapd, gfr%f2g_remapd, gfr%D_f, gfr%Dinv_f) #endif diff --git a/components/homme/src/share/parallel_mod.F90 b/components/homme/src/share/parallel_mod.F90 index 69016a8a063..49ff71779a8 100644 --- a/components/homme/src/share/parallel_mod.F90 +++ b/components/homme/src/share/parallel_mod.F90 @@ -274,6 +274,7 @@ subroutine abortmp(string) #endif #endif character*(*) string + call flush(iulog) #ifdef CAM call endrun(string) #else diff --git a/components/homme/src/theta-l_kokkos/cxx/CamForcing.cpp b/components/homme/src/theta-l_kokkos/cxx/CamForcing.cpp index 02b999db16e..bd7cee3e7c0 100644 --- a/components/homme/src/theta-l_kokkos/cxx/CamForcing.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/CamForcing.cpp @@ -33,7 +33,7 @@ static void apply_cam_forcing_tracers(const Real dt, ForcingFunctor& ff, if ( p.ftype == ForcingAlg::FORCING_2) adjustment = true; #endif - ff.tracers_forcing(dt, tl.n0, tl.n0_qdp, adjustment, p.moisture); + ff.tracers_forcing(dt, tl.n0, tl.n0_qdp, adjustment, p.use_moisture); GPTLstop("ApplyCAMForcing_tracers"); } diff --git a/components/homme/src/theta-l_kokkos/cxx/DirkFunctorImpl.hpp b/components/homme/src/theta-l_kokkos/cxx/DirkFunctorImpl.hpp index ace1ba92014..d1676907972 100644 --- a/components/homme/src/theta-l_kokkos/cxx/DirkFunctorImpl.hpp +++ b/components/homme/src/theta-l_kokkos/cxx/DirkFunctorImpl.hpp @@ -382,8 +382,8 @@ struct DirkFunctorImpl { kv.team_barrier(); if (it >= maxiter) { - printf("[DIRK] WARNING! Newton reached max iteration count," - " with deltaerr = %3.17f\n", deltaerr); + Kokkos::printf("[DIRK] WARNING! Newton reached max iteration count," + " with deltaerr = %3.17f\n", deltaerr); nerr = 1; } diff --git a/components/homme/src/theta-l_kokkos/cxx/ForcingFunctor.hpp b/components/homme/src/theta-l_kokkos/cxx/ForcingFunctor.hpp index 28a702c1d27..00fa1deef66 100644 --- a/components/homme/src/theta-l_kokkos/cxx/ForcingFunctor.hpp +++ b/components/homme/src/theta-l_kokkos/cxx/ForcingFunctor.hpp @@ -236,7 +236,7 @@ class ForcingFunctor }); } - void tracers_forcing (const Real dt, const int np1, const int np1_qdp, const bool adjustment, const MoistDry moisture) { + void tracers_forcing (const Real dt, const int np1, const int np1_qdp, const bool adjustment, const bool use_moisture) { // The Functor needs to be fully setup to use this function assert (is_setup); @@ -245,7 +245,7 @@ class ForcingFunctor m_np1_qdp = np1_qdp; m_adjustment = adjustment; - m_moist = (moisture==MoistDry::MOIST); + m_moist = use_moisture; Kokkos::parallel_for("temperature, NH perturb press, FQps",m_policy_tracers_pre,*this); Kokkos::fence(); diff --git a/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp b/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp index 046e6f9956d..d160e114475 100644 --- a/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp +++ b/components/homme/src/theta-l_kokkos/cxx/HyperviscosityFunctorImpl.cpp @@ -118,9 +118,9 @@ void HyperviscosityFunctorImpl::init_params(const SimulationParams& params) m_eos.init(params.theta_hydrostatic_mode,m_hvcoord); #ifdef HOMMEXX_BFB_TESTING - m_process_nh_vars = true; + m_process_nh_vars = 1; #else - m_process_nh_vars = !params.theta_hydrostatic_mode; + m_process_nh_vars = not params.theta_hydrostatic_mode; #endif } diff --git a/components/homme/src/theta-l_kokkos/cxx/LimiterFunctor.hpp b/components/homme/src/theta-l_kokkos/cxx/LimiterFunctor.hpp index cd3bf7c3252..7914c0a60e3 100644 --- a/components/homme/src/theta-l_kokkos/cxx/LimiterFunctor.hpp +++ b/components/homme/src/theta-l_kokkos/cxx/LimiterFunctor.hpp @@ -141,8 +141,8 @@ struct LimiterFunctor { [&](const int k,Real& result) { #ifndef HOMMEXX_BFB_TESTING if(diff_as_real(k) < 0){ - printf("WARNING:CAAR: dp3d too small. k=%d, dp3d(k)=%f, dp0=%f \n", - k+1,dp_as_real(k),dp0_as_real(k)); + Kokkos::printf("WARNING:CAAR: dp3d too small. k=%d, dp3d(k)=%f, dp0=%f \n", + k+1,dp_as_real(k),dp0_as_real(k)); } #endif result = result<=diff_as_real(k) ? result : diff_as_real(k); @@ -202,8 +202,8 @@ struct LimiterFunctor { for (int ivec=0; ivec(e.m_forcing); } -void init_functors_c (const bool& allocate_buffer) +void init_functors_c (const int& allocate_buffer) { auto& c = Context::singleton(); diff --git a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 index 96b42314453..e7cc245a2bd 100644 --- a/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 +++ b/components/homme/src/theta-l_kokkos/prim_driver_mod.F90 @@ -64,7 +64,7 @@ subroutine prim_init2(elem, hybrid, nets, nete, tl, hvcoord) end subroutine prim_init2 subroutine prim_create_c_data_structures (tl, hvcoord, mp) - use iso_c_binding, only : c_loc, c_ptr, c_bool, C_NULL_CHAR + use iso_c_binding, only : c_loc, c_ptr, C_NULL_CHAR use theta_f2c_mod, only : init_reference_element_c, init_simulation_params_c, & init_time_level_c, init_hvcoord_c, init_elements_c use time_mod, only : TimeLevel_t, nsplit @@ -73,7 +73,7 @@ subroutine prim_create_c_data_structures (tl, hvcoord, mp) nu, nu_p, nu_q, nu_s, nu_div, nu_top, vert_remap_q_alg, & hypervis_order, hypervis_subcycle, hypervis_subcycle_tom,& hypervis_scaling, & - ftype, prescribed_wind, moisture, disable_diagnostics, & + ftype, prescribed_wind, use_moisture, disable_diagnostics, & use_cpstar, transport_alg, theta_hydrostatic_mode, & dcmip16_mu, theta_advect_form, test_case, & MAX_STRING_LEN, dt_remap_factor, dt_tracer_factor, & @@ -93,6 +93,8 @@ subroutine prim_create_c_data_structures (tl, hvcoord, mp) type (c_ptr) :: hybrid_am_ptr, hybrid_ai_ptr, hybrid_bm_ptr, hybrid_bi_ptr character(len=MAX_STRING_LEN), target :: test_name + integer :: disable_diagnostics_int, theta_hydrostatic_mode_int, use_moisture_int + ! Initialize the C++ reference element structure (i.e., pseudo-spectral deriv matrix and ref element mass matrix) dvv = deriv1%dvv elem_mp = mp @@ -100,22 +102,30 @@ subroutine prim_create_c_data_structures (tl, hvcoord, mp) ! Fill the simulation params structures in C++ test_name = TRIM(test_case) // C_NULL_CHAR + + disable_diagnostics_int = 0 + if (disable_diagnostics) disable_diagnostics_int = 1 + use_moisture_int = 0 + if (use_moisture) use_moisture_int = 1 + theta_hydrostatic_mode_int = 0 + if (theta_hydrostatic_mode) theta_hydrostatic_mode_int = 1 + call init_simulation_params_c (vert_remap_q_alg, limiter_option, rsplit, qsplit, tstep_type, & qsize, statefreq, nu, nu_p, nu_q, nu_s, nu_div, nu_top, & hypervis_order, hypervis_subcycle, hypervis_subcycle_tom, & hypervis_scaling, & dcmip16_mu, ftype, theta_advect_form, & - LOGICAL(prescribed_wind==1,c_bool), & - LOGICAL(moisture/="dry",c_bool), & - LOGICAL(disable_diagnostics,c_bool), & - LOGICAL(use_cpstar==1,c_bool), & + prescribed_wind, & + use_moisture_int, & + disable_diagnostics_int, & + use_cpstar, & transport_alg, & - LOGICAL(theta_hydrostatic_mode,c_bool), & + theta_hydrostatic_mode_int, & c_loc(test_name), & dt_remap_factor, dt_tracer_factor, & scale_factor, laplacian_rigid_factor, & nsplit, & - LOGICAL(pgrad_correction==1,c_bool), & + pgrad_correction, & dp3d_thresh, vtheta_thresh, internal_diagnostics_level) ! Initialize time level structure in C++ @@ -343,22 +353,23 @@ subroutine prim_init_elements_views (elem) end subroutine prim_init_elements_views subroutine prim_init_kokkos_functors (allocate_buffer) - use iso_c_binding, only : c_bool + !todo-repo-unification Remove the use of c_bool here. It's used in purely + ! F90 code. + use iso_c_binding, only : c_int, c_bool use theta_f2c_mod, only : init_functors_c, init_boundary_exchanges_c - ! ! Optional Input ! - logical(kind=c_bool), optional :: allocate_buffer ! Whether functor memory buffer should be allocated internally - + logical(kind=c_bool), intent(in), optional :: allocate_buffer ! Whether functor memory buffer should be allocated internally + integer(kind=c_int) :: ab ! Initialize the C++ functors in the C++ context ! If no argument allocate_buffer is present, ! let Homme internally allocate buffers + ab = 1 if (present(allocate_buffer)) then - call init_functors_c (logical(allocate_buffer,c_bool)) - else - call init_functors_c (logical(.true.,c_bool)) - endif + if (.not. allocate_buffer) ab = 0 + end if + call init_functors_c (ab) ! Initialize boundary exchange structure in C++ call init_boundary_exchanges_c () diff --git a/components/homme/src/theta-l_kokkos/theta_f2c_mod.F90 b/components/homme/src/theta-l_kokkos/theta_f2c_mod.F90 index 7a4c0424807..ba39bb03c22 100644 --- a/components/homme/src/theta-l_kokkos/theta_f2c_mod.F90 +++ b/components/homme/src/theta-l_kokkos/theta_f2c_mod.F90 @@ -11,14 +11,14 @@ subroutine init_simulation_params_c (remap_alg, limiter_option, rsplit, qsplit, qsize, state_frequency, nu, nu_p, nu_q, nu_s, nu_div, nu_top, & hypervis_order, hypervis_subcycle, hypervis_subcycle_tom, & hypervis_scaling, & - dcmip16_mu, ftype, theta_adv_form, prescribed_wind, moisture, & + dcmip16_mu, ftype, theta_adv_form, prescribed_wind, use_moisture, & disable_diagnostics, use_cpstar, transport_alg, & theta_hydrostatic_mode, test_case_name, dt_remap_factor, & dt_tracer_factor, scale_factor, laplacian_rigid_factor, & nsplit, pgrad_correction, dp3d_thresh, vtheta_thresh, & internal_diagnostics_level) bind(c) - use iso_c_binding, only: c_int, c_bool, c_double, c_ptr + use iso_c_binding, only: c_int, c_double, c_ptr ! ! Inputs ! @@ -29,8 +29,8 @@ subroutine init_simulation_params_c (remap_alg, limiter_option, rsplit, qsplit, scale_factor, laplacian_rigid_factor, dp3d_thresh, vtheta_thresh integer(kind=c_int), intent(in) :: hypervis_order, hypervis_subcycle, hypervis_subcycle_tom integer(kind=c_int), intent(in) :: ftype, theta_adv_form - logical(kind=c_bool), intent(in) :: prescribed_wind, moisture, disable_diagnostics, use_cpstar - logical(kind=c_bool), intent(in) :: theta_hydrostatic_mode, pgrad_correction + integer(kind=c_int), intent(in) :: prescribed_wind, use_moisture, disable_diagnostics, use_cpstar + integer(kind=c_int), intent(in) :: theta_hydrostatic_mode, pgrad_correction type(c_ptr), intent(in) :: test_case_name end subroutine init_simulation_params_c @@ -138,11 +138,11 @@ end subroutine init_reference_element_c ! Create C++ functors subroutine init_functors_c (allocate_buffer) bind(c) - use iso_c_binding, only: c_bool + use iso_c_binding, only: c_int ! ! Inputs ! - logical(kind=c_bool), intent(in) :: allocate_buffer + integer(kind=c_int), intent(in) :: allocate_buffer end subroutine init_functors_c ! Initialize C++ boundary exchange structures diff --git a/components/homme/test/unit_tests/CMakeLists.txt b/components/homme/test/unit_tests/CMakeLists.txt index 2b601ae172c..b029e73277e 100644 --- a/components/homme/test/unit_tests/CMakeLists.txt +++ b/components/homme/test/unit_tests/CMakeLists.txt @@ -22,14 +22,17 @@ macro(cxx_unit_test target_name target_f90_srcs target_cxx_srcs include_dirs con cxx_unit_test_add_test(${target_name}_test ${target_name} ${NUM_CPUS}) TARGET_LINK_LIBRARIES(${target_name} timing ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES}) - target_link_libraries(${target_name} Kokkos::kokkos) + TARGET_LINK_LIBRARIES(${target_name} Kokkos::kokkos) IF (HOMME_USE_MKL) - TARGET_COMPILE_OPTIONS (${target_name} PUBLIC -mkl) - TARGET_LINK_LIBRARIES (${target_name} -mkl) + IF (MKL_TYPE STREQUAL "oneMKL") + TARGET_LINK_LIBRARIES(${target_name} -qmkl) + ELSEIF (MKL_TYPE STREQUAL "Intel MKL") + TARGET_LINK_LIBRARIES(${target_name} -mkl) + ENDIF () ENDIF() # Link csm_share lib - target_link_libraries(${target_name} csm_share) + TARGET_LINK_LIBRARIES(${target_name} csm_share) STRING(TOUPPER "${PERFORMANCE_PROFILE}" PERF_PROF_UPPER) IF ("${PERF_PROF_UPPER}" STREQUAL "VTUNE") diff --git a/components/homme/test_execs/CMakeLists.txt b/components/homme/test_execs/CMakeLists.txt index a3113921b02..a007a5532b6 100644 --- a/components/homme/test_execs/CMakeLists.txt +++ b/components/homme/test_execs/CMakeLists.txt @@ -142,8 +142,11 @@ ADD_CUSTOM_TARGET(test-execs) ADD_CUSTOM_TARGET(check COMMAND ${CMAKE_CTEST_COMMAND} "--output-on-failure") +if(NOT BUILD_HOMME_WITHOUT_PIOLIBRARY) # Force cprnc to be built when make check is run ADD_DEPENDENCIES(check cprnc) +endif() + # Create a target for making the reference data ADD_CUSTOM_TARGET(baseline diff --git a/components/homme/test_execs/preqx_kokkos_ut/CMakeLists.txt b/components/homme/test_execs/preqx_kokkos_ut/CMakeLists.txt index 9dd30f58f63..3347e7b4894 100644 --- a/components/homme/test_execs/preqx_kokkos_ut/CMakeLists.txt +++ b/components/homme/test_execs/preqx_kokkos_ut/CMakeLists.txt @@ -36,10 +36,14 @@ ADD_LIBRARY(preqx_kokkos_ut_lib TARGET_INCLUDE_DIRECTORIES(preqx_kokkos_ut_lib PUBLIC ${EXEC_INCLUDE_DIRS}) TARGET_INCLUDE_DIRECTORIES(preqx_kokkos_ut_lib PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) TARGET_COMPILE_DEFINITIONS(preqx_kokkos_ut_lib PUBLIC "HAVE_CONFIG_H") -target_link_libraries(preqx_kokkos_ut_lib Kokkos::kokkos) +TARGET_LINK_LIBRARIES(preqx_kokkos_ut_lib Kokkos::kokkos) TARGET_LINK_LIBRARIES(preqx_kokkos_ut_lib timing csm_share ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES}) IF (HOMME_USE_MKL) - TARGET_LINK_LIBRARIES (preqx_kokkos_ut_lib -mkl) + IF (MKL_TYPE STREQUAL "oneMKL") + TARGET_LINK_LIBRARIES(preqx_kokkos_ut_lib -qmkl) + ELSEIF (MKL_TYPE STREQUAL "Intel MKL") + TARGET_LINK_LIBRARIES(preqx_kokkos_ut_lib -mkl) + ENDIF () ENDIF() IF(NOT BUILD_HOMME_WITHOUT_PIOLIBRARY) IF(HOMME_USE_SCORPIO) diff --git a/components/homme/test_execs/share_kokkos_ut/CMakeLists.txt b/components/homme/test_execs/share_kokkos_ut/CMakeLists.txt index 3fbeff9f6f2..bc788462ce6 100644 --- a/components/homme/test_execs/share_kokkos_ut/CMakeLists.txt +++ b/components/homme/test_execs/share_kokkos_ut/CMakeLists.txt @@ -10,7 +10,7 @@ SET(UTILS_TIMING_DIRS ${UTILS_TIMING_SRC_DIR} ${UTILS_TIMING_BIN_DIR}) # Note: need CUDA_BUILD and HOMMEXX_BFB_TESTING here, since the share # unit tests do not include a config.h file SET (COMMON_DEFINITIONS NP=4 NC=4) -IF (CUDA_BUILD OR HIP_BUILD) +IF (CUDA_BUILD OR HIP_BUILD OR SYCL_BUILD) SET(COMMON_DEFINITIONS ${COMMON_DEFINITIONS} HOMMEXX_ENABLE_GPU_F90) ENDIF() IF (HOMMEXX_BFB_TESTING) @@ -158,7 +158,7 @@ ELSE() SET (NUM_CPUS 1) ENDIF() cxx_unit_test (sphere_op_ut "${SPHERE_OP_UT_F90_SRCS}" "${SPHERE_OP_UT_CXX_SRCS}" "${SPHERE_OP_UT_INCLUDE_DIRS}" "${CONFIG_DEFINES}" ${NUM_CPUS}) -endif () +endif () #BFB ### Limiters unit test ### diff --git a/components/homme/test_execs/thetal_kokkos_ut/CMakeLists.txt b/components/homme/test_execs/thetal_kokkos_ut/CMakeLists.txt index 205635e918c..11030b15e87 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/CMakeLists.txt +++ b/components/homme/test_execs/thetal_kokkos_ut/CMakeLists.txt @@ -11,6 +11,8 @@ SET(UTILS_TIMING_BIN_DIR ${HOMME_BINARY_DIR}/utils/cime/CIME/non_py/src/timing) THETAL_KOKKOS_SETUP() # This is needed to compile the lib and test executables with the correct options +#these vars shared between all targets, so changing one var +#for one test only won't work, config is built once and for the last test SET(THIS_CONFIG_IN ${HOMME_SOURCE_DIR}/src/theta-l_kokkos/config.h.cmake.in) SET(THIS_CONFIG_HC ${CMAKE_CURRENT_BINARY_DIR}/config.h.c) SET(THIS_CONFIG_H ${CMAKE_CURRENT_BINARY_DIR}/config.h) @@ -18,6 +20,7 @@ SET (NUM_POINTS 4) SET (NUM_PLEV 12) SET (QSIZE_D 4) SET (PIO_INTERP TRUE) + HommeConfigFile (${THIS_CONFIG_IN} ${THIS_CONFIG_HC} ${THIS_CONFIG_H} ) ADD_LIBRARY(thetal_kokkos_ut_lib @@ -39,10 +42,14 @@ ADD_LIBRARY(thetal_kokkos_ut_lib TARGET_INCLUDE_DIRECTORIES(thetal_kokkos_ut_lib PUBLIC ${EXEC_INCLUDE_DIRS}) TARGET_INCLUDE_DIRECTORIES(thetal_kokkos_ut_lib PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) TARGET_COMPILE_DEFINITIONS(thetal_kokkos_ut_lib PUBLIC "HAVE_CONFIG_H") -target_link_libraries(thetal_kokkos_ut_lib Kokkos::kokkos) +TARGET_LINK_LIBRARIES(thetal_kokkos_ut_lib Kokkos::kokkos) TARGET_LINK_LIBRARIES(thetal_kokkos_ut_lib timing csm_share ${COMPOSE_LIBRARY_CPP} ${BLAS_LIBRARIES} ${LAPACK_LIBRARIES}) IF (HOMME_USE_MKL) - TARGET_LINK_LIBRARIES (thetal_kokkos_ut_lib -mkl) + IF (MKL_TYPE STREQUAL "oneMKL") + TARGET_LINK_LIBRARIES(thetal_kokkos_ut_lib -qmkl) + ELSEIF (MKL_TYPE STREQUAL "Intel MKL") + TARGET_LINK_LIBRARIES(thetal_kokkos_ut_lib -mkl) + ENDIF () ENDIF() IF(BUILD_HOMME_WITHOUT_PIOLIBRARY) TARGET_COMPILE_DEFINITIONS(thetal_kokkos_ut_lib PUBLIC HOMME_WITHOUT_PIOLIBRARY) diff --git a/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp b/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp index 5e4c51c7ca1..fb301166f42 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp +++ b/components/homme/test_execs/thetal_kokkos_ut/forcing_ut.cpp @@ -160,8 +160,8 @@ TEST_CASE("forcing", "forcing") { std::cout << "Testing tracers forcing.\n"; for (const bool hydrostatic : {true,false}) { std::cout << " -> hydrostatic mode: " << (hydrostatic ? "true" : "false") << "\n"; - for (const MoistDry moisture : {MoistDry::DRY,MoistDry::MOIST}) { - std::cout << " -> moisture: " << (moisture==MoistDry::MOIST ? "moist" : "dry") << "\n"; + for (const bool use_moisture: {false,true}) { + std::cout << " -> moisture: " << (use_moisture ? "moist" : "dry") << "\n"; for (const bool adjustment : {true,false}) { std::cout << " -> adjustment: " << (adjustment ? "true" : "false") << "\n"; @@ -200,8 +200,8 @@ TEST_CASE("forcing", "forcing") { ff.init_buffers(fbm); // Run tracers forcing (cxx and f90) - ff.tracers_forcing(dt,np1,np1_qdp,adjustment,moisture); - tracers_forcing_f90(dt,np1+1,np1_qdp+1,hydrostatic,moisture==MoistDry::MOIST,adjustment); + ff.tracers_forcing(dt,np1,np1_qdp,adjustment,use_moisture); + tracers_forcing_f90(dt,np1+1,np1_qdp+1,hydrostatic,use_moisture,adjustment); // Compare answers Kokkos::deep_copy(h_dp,state.m_dp3d); diff --git a/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_ut.cpp b/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_ut.cpp index 0f14b0c3e55..cf9db941ea1 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_ut.cpp +++ b/components/homme/test_execs/thetal_kokkos_ut/gllfvremap_ut.cpp @@ -183,7 +183,7 @@ struct Session { p.qsize = qsize; p.hypervis_scaling = 0; p.transport_alg = 0; - p.moisture = MoistDry::MOIST; + p.use_moisture = true; p.theta_hydrostatic_mode = false; p.scale_factor = is_sphere ? PhysicalConstants::rearth0 : 1; p.laplacian_rigid_factor = is_sphere ? 1/p.scale_factor : 0; @@ -725,7 +725,7 @@ static void test_get_temperature (Session& s) { const auto& sp = c.get(); EquationOfState eos; eos.init(theta_hydrostatic_mode, s.h); ElementOps ops; ops.init(s.h); - const bool use_moisture = sp.moisture == MoistDry::MOIST; + const bool use_moisture = sp.use_moisture; const auto state = c.get(); const auto tracers = c.get(); const auto dp3d = state.m_dp3d; diff --git a/components/mosart/src/riverroute/MOSART_physics_mod.F90 b/components/mosart/src/riverroute/MOSART_physics_mod.F90 index 912014904e9..8d3843abad5 100644 --- a/components/mosart/src/riverroute/MOSART_physics_mod.F90 +++ b/components/mosart/src/riverroute/MOSART_physics_mod.F90 @@ -680,10 +680,12 @@ subroutine Euler end do ! DLevelH2R ! subcycling within MOSART ends -! check for negative channel storage - if (negchan < -1.e-10) then - write(iulog,*) 'Warning: Negative channel storage found! ',negchan -! call shr_sys_abort('mosart: negative channel storage') + ! check for negative channel storage + if (negchan < -1.e-10 .and. negchan >= -1.e-8) then + write(iulog,*) 'Warning: Small negative channel storage found! ',negchan + elseif(negchan < -1.e-8) then + write(iulog,*) 'Error: Negative channel storage found! ',negchan + call shr_sys_abort('mosart: negative channel storage') endif TRunoff%flow = TRunoff%flow / Tctl%DLevelH2R TRunoff%erowm_regi(:,nt_nmud:nt_nsan) = TRunoff%erowm_regi(:,nt_nmud:nt_nsan) / Tctl%DLevelH2R @@ -876,11 +878,7 @@ subroutine Routing_KW(iunit, nt, theDeltaT) TRunoff%erout(iunit,nt) = -TRunoff%vr(iunit,nt) * TRunoff%mr(iunit,nt) if(-TRunoff%erout(iunit,nt) > TINYVALUE .and. TRunoff%wr(iunit,nt) + & (TRunoff%erlateral(iunit,nt) + TRunoff%erin(iunit,nt) + TRunoff%erout(iunit,nt)) * theDeltaT < TINYVALUE) then - if (sediflag) then - TRunoff%erout(iunit,nt) = -(TRunoff%erlateral(iunit,nt) + TRunoff%erin(iunit,nt) + TRunoff%wr(iunit,nt)*MaxStorageDepleted/ theDeltaT) - else - TRunoff%erout(iunit,nt) = -(TRunoff%erlateral(iunit,nt) + TRunoff%erin(iunit,nt) + TRunoff%wr(iunit,nt)/ theDeltaT) - end if + TRunoff%erout(iunit,nt) = -(TRunoff%erlateral(iunit,nt) + TRunoff%erin(iunit,nt) + TRunoff%wr(iunit,nt)*MaxStorageDepleted/ theDeltaT) if(TRunoff%mr(iunit,nt) > 0._r8) then TRunoff%vr(iunit,nt) = -TRunoff%erout(iunit,nt) / TRunoff%mr(iunit,nt) end if @@ -918,16 +916,6 @@ subroutine Routing_KW(iunit, nt, theDeltaT) TRunoff%dwr(iunit,nt) = TRunoff%erlateral(iunit,nt) + TRunoff%erin(iunit,nt) + TRunoff%erout(iunit,nt) + temp_gwl - !if(TRunoff%wr(iunit,nt) < TINYVALUE .and. abs(TRunoff%erout(iunit,nt))> TINYVALUE) then - ! write(unit=1111,fmt="(i10, 4(e20.11))") iunit, TRunoff%wr(iunit,nt), TRunoff%erout(iunit,nt), TRunoff%erlateral(iunit,nt) + TRunoff%erin(iunit,nt), TRunoff%dwr(iunit,nt) - ! write(unit=1112,fmt="(2(i10), 4(e20.11))") iunit, TUnit%mask(iunit), TRunoff%vr(iunit,nt), TUnit%rlen(iunit), TUnit%rwidth(iunit), TUnit%areaTotal2(iunit)/TUnit%rwidth(iunit)/TUnit%rlen(iunit) - !end if - -! if(iunit==490 .and. nt==1) then -! write(unit=1111,fmt="(3(e20.11), 5(f12.4))") TUnit%areaTotal2(iunit),TUnit%areaTotal(iunit),TUnit%area(iunit),TUnit%rdepth(iunit), TUnit%rwidth(iunit), TUnit%rslp(iunit), TUnit%nr(iunit), TUnit%nt(iunit) -! write(unit=1112,fmt="(6(e20.11))") TRunoff%wr(iunit,nt), TRunoff%dwr(iunit,nt), TRunoff%erlateral(iunit,nt), TRunoff%erin(iunit,nt), TRunoff%erout(iunit,nt), temp_gwl -! end if - ! check for stability ! if(TRunoff%vr(iunit,nt) < -TINYVALUE .or. TRunoff%vr(iunit,nt) > 30) then ! write(iulog,*) "Numerical error inRouting_KW, ", iunit,nt,TRunoff%vr(iunit,nt) diff --git a/components/mpas-albany-landice/bld/build-namelist b/components/mpas-albany-landice/bld/build-namelist index 15272f7a2da..f08d94a0e27 100755 --- a/components/mpas-albany-landice/bld/build-namelist +++ b/components/mpas-albany-landice/bld/build-namelist @@ -620,6 +620,7 @@ add_default($nl, 'config_pio_stride'); add_default($nl, 'config_year_digits'); add_default($nl, 'config_output_external_velocity_solver_data'); add_default($nl, 'config_write_albany_ascii_mesh'); +add_default($nl, 'config_create_all_logs_in_e3sm'); ################################# # Namelist group: decomposition # diff --git a/components/mpas-albany-landice/bld/build-namelist-section b/components/mpas-albany-landice/bld/build-namelist-section index 4b5e05ec3cd..735dca93c19 100644 --- a/components/mpas-albany-landice/bld/build-namelist-section +++ b/components/mpas-albany-landice/bld/build-namelist-section @@ -182,6 +182,7 @@ add_default($nl, 'config_pio_stride'); add_default($nl, 'config_year_digits'); add_default($nl, 'config_output_external_velocity_solver_data'); add_default($nl, 'config_write_albany_ascii_mesh'); +add_default($nl, 'config_create_all_logs_in_e3sm'); ################################# # Namelist group: decomposition # diff --git a/components/mpas-albany-landice/bld/namelist_files/namelist_defaults_mali.xml b/components/mpas-albany-landice/bld/namelist_files/namelist_defaults_mali.xml index 0450eb44f11..3e0b5625835 100644 --- a/components/mpas-albany-landice/bld/namelist_files/namelist_defaults_mali.xml +++ b/components/mpas-albany-landice/bld/namelist_files/namelist_defaults_mali.xml @@ -18,7 +18,7 @@ 'fo' -'none' +'fo' 3 0.25 .false. @@ -150,6 +150,7 @@ 4 .false. .false. +.false. 3 diff --git a/components/mpas-albany-landice/bld/namelist_files/namelist_definition_mali.xml b/components/mpas-albany-landice/bld/namelist_files/namelist_definition_mali.xml index e16e7042a2c..c75c1e163aa 100644 --- a/components/mpas-albany-landice/bld/namelist_files/namelist_definition_mali.xml +++ b/components/mpas-albany-landice/bld/namelist_files/namelist_definition_mali.xml @@ -1087,6 +1087,14 @@ Valid values: .true. or .false. Default: Defined in namelist_defaults.xml + +Logical flag determining if log files will be created for each processor in an E3SM configuration. If .true., the model initializes and writes to one files per processor. + +Valid values: .true. or .false. +Default: Defined in namelist_defaults.xml + + diff --git a/components/mpas-albany-landice/cime_config/buildnml b/components/mpas-albany-landice/cime_config/buildnml index 9489b6dfa8f..3b38ed39c43 100755 --- a/components/mpas-albany-landice/cime_config/buildnml +++ b/components/mpas-albany-landice/cime_config/buildnml @@ -280,7 +280,9 @@ def buildnml(case, caseroot, compname): lines.append(' ') lines.append(' ') lines.append(' ') - lines.append(' ') + if mali_use_albany: + lines.append(' ') + lines.append('') lines.append('') lines.append(' .false. .true. +.false. .false. .false. .false. @@ -626,6 +643,7 @@ .false. .false. .false. +.false. .false. .false. .false. @@ -1153,6 +1171,7 @@ .true. .true. .true. +.true. '0000-00-00_01:00:00' 'mocStreamfunctionOutput' .true. @@ -1232,37 +1251,10 @@ .false. -.true. -.true. -.true. -.true. -.true. -.true. -.true. -.true. -.true. 'dt' 'conservationCheckOutput' .false. -.true. -.true. -.true. -.true. -.true. -.true. -.true. -.true. -.true. .false. -.true. -.true. -.true. -.true. -.true. -.true. -.true. -.true. -.true. .true. 'conservationCheckRestart' diff --git a/components/mpas-ocean/cime_config/SystemTests/mvko.py b/components/mpas-ocean/cime_config/SystemTests/mvko.py index 71477a4dfc1..90fa4b7a553 100644 --- a/components/mpas-ocean/cime_config/SystemTests/mvko.py +++ b/components/mpas-ocean/cime_config/SystemTests/mvko.py @@ -18,6 +18,7 @@ import netCDF4 as nc import CIME.test_status +from CIME.status import append_testlog import CIME.utils from CIME.SystemTests.system_tests_common import SystemTestsCommon from CIME.case.case_setup import case_setup @@ -394,4 +395,4 @@ def _compare_baseline(self): f" {viewing}" ) - CIME.utils.append_testlog(comments, self._orig_caseroot) + append_testlog(comments, self._orig_caseroot) diff --git a/components/mpas-ocean/cime_config/buildnml b/components/mpas-ocean/cime_config/buildnml index 8f5193c334b..d45d94344fe 100755 --- a/components/mpas-ocean/cime_config/buildnml +++ b/components/mpas-ocean/cime_config/buildnml @@ -406,6 +406,20 @@ def buildnml(case, caseroot, compname): if ocn_ismf == 'data': data_ismf_file = 'prescribed_ismf_paolo2023.RRSwISC6to18E3r5.20240327.nc' + elif ocn_grid == 'SOwISC12to30E3r3': + decomp_date = '20240829' + decomp_prefix = 'partitions/mpas-o.graph.info.' + restoring_file = 'sss.PHC2_monthlyClimatology.SOwISC12to30E3r3.20240829.nc' + analysis_mask_file = 'SOwISC12to30E3r3_mocBasinsAndTransects20210623.nc' + ic_date = '20240829' + ic_prefix = 'mpaso.SOwISC12to30E3r3' + if ocn_ic_mode == 'spunup': + ic_date = '20240829' + ic_prefix = 'mpaso.SOwISC12to30E3r3.rstFromG-chrysalis' + if ocn_ismf == 'data': + data_ismf_file = 'prescribed_ismf_paolo2023.SOwISC12to30E3r3.20240829.nc' + + #-------------------------------------------------------------------- # Set OCN_FORCING = datm_forced_restoring if restoring file is available #-------------------------------------------------------------------- @@ -948,6 +962,7 @@ def buildnml(case, caseroot, compname): lines.append(' ') lines.append(' ') lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') @@ -1156,7 +1171,7 @@ def buildnml(case, caseroot, compname): lines.append(' filename_interval="00-01-00_00:00:00"') lines.append(' reference_time="01-01-01_00:00:00"') lines.append(' output_interval="00-00-01_00:00:00"') - lines.append(' clobber_mode="truncate"') + lines.append(' clobber_mode="append"') lines.append(' packages="conservationCheckAMPKG">') lines.append('') lines.append('') diff --git a/components/mpas-ocean/cime_config/config_compsets.xml b/components/mpas-ocean/cime_config/config_compsets.xml index be36f010347..62c87ddea3f 100644 --- a/components/mpas-ocean/cime_config/config_compsets.xml +++ b/components/mpas-ocean/cime_config/config_compsets.xml @@ -34,17 +34,17 @@ GMPAS-NYF-PISMF - 2000_DATM%NYF_SLND_MPASSI_MPASO%PISMFDATMFORCED_DROF%NYFAIS45_SGLC_SWAV + 2000_DATM%NYF_SLND_MPASSI_MPASO%PISMFDATMFORCED_DROF%NYF_SGLC_SWAV GMPAS-NYF-PISMF-DSGR - 2000_DATM%NYF_SLND_MPASSI_MPASO%PISMFDATMFORCEDDSGR_DROF%NYFAIS45_SGLC_SWAV + 2000_DATM%NYF_SLND_MPASSI_MPASO%PISMFDATMFORCEDDSGR_DROF%NYF_SGLC_SWAV GMPAS-NYF-DISMF - 2000_DATM%NYF_SLND_MPASSI_MPASO%DISMFDATMFORCED_DROF%NYFAIS45_SGLC_SWAV + 2000_DATM%NYF_SLND_MPASSI_MPASO%DISMFDATMFORCED_DROF%NYF_SGLC_SWAV @@ -67,6 +67,11 @@ 2000_DATM%JRA-1p5_SLND_MPASSI_MPASO%DATMFORCED_DROF%JRA-1p5_SGLC_SWAV + + GMPAS-JRA1p5-2023 + 2000_DATM%JRA-1p5-2023_SLND_MPASSI_MPASO%DATMFORCED_DROF%JRA-1p5-2023_SGLC_SWAV + + GMPAS-OECO-JRA1p4 2000_DATM%JRA-1p4-2018_SLND_MPASSI_MPASO%OECODATMFORCED_DROF%JRA-1p4-2018_SGLC_SWAV @@ -89,27 +94,27 @@ GMPAS-JRA1p5-DIB-PISMF - 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCED_DROF%JRA-1p5-AIS0ROF_SGLC_SWAV + 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCED_DROF%JRA-1p5_SGLC_SWAV GMPAS-JRA1p5-DIB-PISMF-DSGR - 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCEDDSGR_DROF%JRA-1p5-AIS0ROF_SGLC_SWAV + 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCEDDSGR_DROF%JRA-1p5_SGLC_SWAV GMPAS-JRA1p5-DIB-PISMF-DSGR-TMIX - 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCEDDSGRTMIX_DROF%JRA-1p5-AIS0ROF_SGLC_SWAV + 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCEDDSGRTMIX_DROF%JRA-1p5_SGLC_SWAV GMPAS-JRA1p5-DIB-DISMF - 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBDISMFDATMFORCED_DROF%JRA-1p5-AIS0ROF_SGLC_SWAV + 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBDISMFDATMFORCED_DROF%JRA-1p5_SGLC_SWAV GMPAS-JRA1p5-DIB-PISMF-TMIX - 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCEDTMIX_DROF%JRA-1p5-AIS0ROF_SGLC_SWAV + 2000_DATM%JRA-1p5_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCEDTMIX_DROF%JRA-1p5_SGLC_SWAV @@ -119,77 +124,77 @@ GMPAS-JRA1p4-PISMF - 2000_DATM%JRA-1p4-2018_SLND_MPASSI_MPASO%PISMFDATMFORCED_DROF%JRA-1p4-2018-AIS0LIQ_SGLC_SWAV + 2000_DATM%JRA-1p4-2018_SLND_MPASSI_MPASO%PISMFDATMFORCED_DROF%JRA-1p4-2018_SGLC_SWAV GMPAS-JRA1p4-DISMF - 2000_DATM%JRA-1p4-2018_SLND_MPASSI_MPASO%DISMFDATMFORCED_DROF%JRA-1p4-2018-AIS0LIQ_SGLC_SWAV + 2000_DATM%JRA-1p4-2018_SLND_MPASSI_MPASO%DISMFDATMFORCED_DROF%JRA-1p4-2018_SGLC_SWAV GMPAS-JRA1p4-DIB - 2000_DATM%JRA-1p4-2018_SLND_MPASSI%DIB_MPASO%IBDATMFORCED_DROF%JRA-1p4-2018-AIS0ICE_SGLC_SWAV + 2000_DATM%JRA-1p4-2018_SLND_MPASSI%DIB_MPASO%IBDATMFORCED_DROF%JRA-1p4-2018_SGLC_SWAV GMPAS-JRA1p4-DIB-PISMF - 2000_DATM%JRA-1p4-2018_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCED_DROF%JRA-1p4-2018-AIS0ROF_SGLC_SWAV + 2000_DATM%JRA-1p4-2018_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCED_DROF%JRA-1p4-2018_SGLC_SWAV GMPAS-JRA1p4-DIB-DISMF - 2000_DATM%JRA-1p4-2018_SLND_MPASSI%DIB_MPASO%IBDISMFDATMFORCED_DROF%JRA-1p4-2018-AIS0ROF_SGLC_SWAV + 2000_DATM%JRA-1p4-2018_SLND_MPASSI%DIB_MPASO%IBDISMFDATMFORCED_DROF%JRA-1p4-2018_SGLC_SWAV GMPAS-IAF-PISMF - 2000_DATM%IAF_SLND_MPASSI_MPASO%PISMFDATMFORCED_DROF%IAFAIS45_SGLC_SWAV + 2000_DATM%IAF_SLND_MPASSI_MPASO%PISMFDATMFORCED_DROF%IAF_SGLC_SWAV GMPAS-IAF-DISMF - 2000_DATM%IAF_SLND_MPASSI_MPASO%DISMFDATMFORCED_DROF%IAFAIS45_SGLC_SWAV + 2000_DATM%IAF_SLND_MPASSI_MPASO%DISMFDATMFORCED_DROF%IAF_SGLC_SWAV GMPAS-DIB-NYF - 2000_DATM%NYF_SLND_MPASSI%DIB_MPASO%IBDATMFORCED_DROF%NYFAIS55_SGLC_SWAV + 2000_DATM%NYF_SLND_MPASSI%DIB_MPASO%IBDATMFORCED_DROF%NYF_SGLC_SWAV GMPAS-DIB-NYF-PISMF - 2000_DATM%NYF_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCED_DROF%NYFAIS00_SGLC_SWAV + 2000_DATM%NYF_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCED_DROF%NYF_SGLC_SWAV GMPAS-DIB-NYF-DISMF - 2000_DATM%NYF_SLND_MPASSI%DIB_MPASO%IBDISMFDATMFORCED_DROF%NYFAIS00_SGLC_SWAV + 2000_DATM%NYF_SLND_MPASSI%DIB_MPASO%IBDISMFDATMFORCED_DROF%NYF_SGLC_SWAV GMPAS-DIB-IAF - 2000_DATM%IAF_SLND_MPASSI%DIB_MPASO%IBDATMFORCED_DROF%IAFAIS55_SGLC_SWAV + 2000_DATM%IAF_SLND_MPASSI%DIB_MPASO%IBDATMFORCED_DROF%IAF_SGLC_SWAV GMPAS-DIB-IAF-PISMF - 2000_DATM%IAF_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCED_DROF%IAFAIS00_SGLC_SWAV + 2000_DATM%IAF_SLND_MPASSI%DIB_MPASO%IBPISMFDATMFORCED_DROF%IAF_SGLC_SWAV GMPAS-DIB-IAF-DISMF - 2000_DATM%IAF_SLND_MPASSI%DIB_MPASO%IBDISMFDATMFORCED_DROF%IAFAIS00_SGLC_SWAV + 2000_DATM%IAF_SLND_MPASSI%DIB_MPASO%IBDISMFDATMFORCED_DROF%IAF_SGLC_SWAV GMPAS-MALI-DIB-IAF-DISMF - 2000_DATM%IAF_SLND_MPASSI%DIB_MPASO%IBDISMFCOREFORCED_DROF%IAFAIS00_MALI%SIASTATIC_SWAV + 2000_DATM%IAF_SLND_MPASSI%DIB_MPASO%IBDISMFCOREFORCED_DROF%IAF_MALI%SIASTATIC_SWAV GMPAS-MALI-DIB-IAF-DISMF - 2000_DATM%IAF_SLND_MPASSI%DIB_MPASO%IBDISMFCOREFORCED_DROF%IAFAIS00_MALI%SIASTATIC_SWAV + 2000_DATM%IAF_SLND_MPASSI%DIB_MPASO%IBDISMFCOREFORCED_DROF%IAF_MALI%SIASTATIC_SWAV diff --git a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/README b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/README new file mode 100644 index 00000000000..fb85b185fd4 --- /dev/null +++ b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/README @@ -0,0 +1,8 @@ +This testdef is used to test the conservation check analysis member, which +wasintroduced in MPAS-Ocean PR #4521 and has been made a stealth feature in +#6643. This test turns on the consevation check analysis member by setting: + + config_AM_conservationCheck_enable = .true. + +However, it should be noted that MPAS-Ocean history files are not currently +included in E3SM testing so non-BFB results will not be detected. diff --git a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands new file mode 100644 index 00000000000..764340d699f --- /dev/null +++ b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/shell_commands @@ -0,0 +1,5 @@ +# include mpas-ocean outputs in testing +sed -i 's/\(compname="mpaso" exclude_testing="true"\)/compname="mpaso"/' env_archive.xml +sed -i '/\(compname="mpaso"\)/,// s/hist/hist.am.conservationCheck\\..*\\.nc$/' env_archive.xml +sed -i 's/casename.mpaso.hist.am.globalStats.1976-01-01.nc/casename.mpaso.hist.am.conservationCheck.1976-01-01.nc/' env_archive.xml +sed -i '/casename.mpaso.hist.am.highFrequencyOutput.1976-01-01_00.00.00.nc/d' env_archive.xml diff --git a/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/user_nl_mpaso b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/user_nl_mpaso new file mode 100644 index 00000000000..c57241a9fcf --- /dev/null +++ b/components/mpas-ocean/cime_config/testdefs/testmods_dirs/mpaso/conservation_check/user_nl_mpaso @@ -0,0 +1 @@ + config_AM_conservationCheck_enable = .true. diff --git a/components/mpas-ocean/src/analysis_members/Makefile b/components/mpas-ocean/src/analysis_members/Makefile index c07186ad9ad..9afb438b77e 100644 --- a/components/mpas-ocean/src/analysis_members/Makefile +++ b/components/mpas-ocean/src/analysis_members/Makefile @@ -2,6 +2,9 @@ OBJS = mpas_ocn_analysis_driver.o +UTILS = shr_kind_mod.o \ + shr_const_mod.o + MEMBERS = mpas_ocn_global_stats.o \ mpas_ocn_okubo_weiss.o \ mpas_ocn_layer_volume_weighted_averages.o \ @@ -35,7 +38,7 @@ MEMBERS = mpas_ocn_global_stats.o \ all: $(OBJS) -mpas_ocn_analysis_driver.o: $(MEMBERS) +mpas_ocn_analysis_driver.o: $(UTILS) $(MEMBERS) mpas_ocn_okubo_weiss.o: mpas_ocn_okubo_weiss_eigenvalues.o diff --git a/components/mpas-ocean/src/analysis_members/Registry_high_frequency_output.xml b/components/mpas-ocean/src/analysis_members/Registry_high_frequency_output.xml index 5ab40af1923..67f0dcf60c4 100644 --- a/components/mpas-ocean/src/analysis_members/Registry_high_frequency_output.xml +++ b/components/mpas-ocean/src/analysis_members/Registry_high_frequency_output.xml @@ -27,6 +27,9 @@ + @@ -304,12 +307,12 @@ + - diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F index beed937d676..95a9bca8f70 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_conservation_check.F @@ -19,6 +19,9 @@ module ocn_conservation_check + use shr_kind_mod, only: SHR_KIND_R8 + use shr_const_mod + use mpas_derived_types use mpas_pool_routines use mpas_dmpar @@ -158,10 +161,9 @@ subroutine ocn_init_conservation_check(domain, err)!{{{ !----------------------------------------------------------------- ! taking PI from SHR_CONST_PI in share/util/shr_const_mod.F90 to match coupler - !real (kind=RKIND), parameter :: piE3SM = 3.14159265358979323846_RKIND ! pi - real (kind=RKIND), parameter :: piE3SM = 3.141592653589793_RKIND + real (kind=RKIND), parameter :: piE3SM = SHR_CONST_PI ! taking earth radius from SHR_CONST_REARTH in share/util/shr_const_mod.F90 to match coupler - real (kind=RKIND), parameter :: earthRadiusE3SM = 6.371229e6_RKIND ! radius of earth, m + real (kind=RKIND), parameter :: earthRadiusE3SM = SHR_CONST_REARTH ! radius of earth, m err = 0 diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_harmonic_analysis.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_harmonic_analysis.F index ee1495a3e59..c60ac179ccf 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_harmonic_analysis.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_harmonic_analysis.F @@ -681,7 +681,7 @@ SUBROUTINE harmonic_analysis_solve(MNP,nfreq,hmat,GLOELV,haff,haface,emagt,phase REAL(kind=RKIND),ALLOCATABLE :: hap(:),hax(:) REAL(kind=RKIND),ALLOCATABLE :: ha(:,:) - convrd=180.0_RKIND/pii + convrd=180.0_RKIND/pi mm = 2*nfreq ALLOCATE ( PHASEE(nfreq),EMAG(nfreq) ) diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_high_frequency_output.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_high_frequency_output.F index 60a5dececcb..8ed030392cb 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_high_frequency_output.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_high_frequency_output.F @@ -175,7 +175,7 @@ subroutine ocn_compute_high_frequency_output(domain, timeLevel, err)!{{{ type (mpas_pool_type), pointer :: highFrequencyOutputAMPool type (mpas_pool_type), pointer :: tracersPool - integer :: iLevel, iLevelTarget, iCell, iEdge, i, cell1, cell2, k, eoe + integer :: iLevel, iCell, iEdge, i, cell1, cell2, k, eoe integer :: iLevel0100, iLevel0250, iLevel0700, iLevel2000 real (kind=RKIND) :: sumLayerThickness integer, pointer :: nVertLevels, nCells, nEdges @@ -183,7 +183,7 @@ subroutine ocn_compute_high_frequency_output(domain, timeLevel, err)!{{{ integer, dimension(:,:), pointer :: edgesOnCell, cellsOnEdge, edgesOnEdge real (kind=RKIND) :: invAreaCell1, layerThicknessEdge1, coeff, weightedNormalVel, cellArea - real (kind=RKIND), dimension(:), pointer :: refBottomDepth, kineticEnergyAt250m, kineticEnergyAtSurface, relativeVorticityAt250m + real (kind=RKIND), dimension(:), pointer :: refBottomDepth, kineticEnergyAt250m, kineticEnergyAtSurface, relativeVorticityAt250m, relativeVorticityAtSurface real (kind=RKIND), dimension(:), pointer :: divergenceAt250m, relativeVorticityVertexAt250m real (kind=RKIND), dimension(:), pointer :: divergenceAtBottom,relativeVorticityAtBottom,kineticEnergyAtBottom real (kind=RKIND), dimension(:), pointer :: vertVelAt250m @@ -263,6 +263,7 @@ subroutine ocn_compute_high_frequency_output(domain, timeLevel, err)!{{{ call mpas_pool_get_array(highFrequencyOutputAMPool, 'kineticEnergyAt250m', kineticEnergyAt250m) call mpas_pool_get_array(highFrequencyOutputAMPool, 'kineticEnergyAtSurface', kineticEnergyAtSurface) call mpas_pool_get_array(highFrequencyOutputAMPool, 'relativeVorticityAt250m', relativeVorticityAt250m) + call mpas_pool_get_array(highFrequencyOutputAMPool, 'relativeVorticityAtSurface', relativeVorticityAtSurface) call mpas_pool_get_array(highFrequencyOutputAMPool, 'divergenceAt250m', divergenceAt250m) call mpas_pool_get_array(highFrequencyOutputAMPool, 'relativeVorticityAtBottom', relativeVorticityAtBottom) call mpas_pool_get_array(highFrequencyOutputAMPool, 'divergenceAtBottom', divergenceAtBottom) @@ -377,37 +378,41 @@ subroutine ocn_compute_high_frequency_output(domain, timeLevel, err)!{{{ call mpas_pool_get_array(highFrequencyOutputAMPool, 'columnIntegratedSpeed', columnIntegratedSpeed) ! find vertical level that is just above the 100 m reference level - iLevel0100 = 1 - do iLevel=2,nVertLevels + ! if even the bottom level isn't deep enough, we still default to the bottom level + iLevel0100 = nVertLevels + do iLevel=1,nVertLevels if(refBottomDepth(iLevel) > 100.0_RKIND) then - iLevel0100 = iLevel-1 + iLevel0100 = iLevel exit endif enddo ! find vertical level that is just above the 250 m reference level - iLevel0250 = 1 + ! if even the bottom level isn't deep enough, we still default to the bottom level + iLevel0250 = nVertLevels do iLevel=iLevel0100,nVertLevels if(refBottomDepth(iLevel) > 250.0_RKIND) then - iLevel0250 = iLevel-1 + iLevel0250 = iLevel exit endif enddo ! find vertical level that is just above the 700 m reference level - iLevel0700 = 1 + ! if even the bottom level isn't deep enough, we still default to the bottom level + iLevel0700 = nVertLevels do iLevel=iLevel0250,nVertLevels if(refBottomDepth(iLevel) > 700.0_RKIND) then - iLevel0700 = iLevel-1 + iLevel0700 = iLevel exit endif enddo ! find vertical level that is just above the 2000 m reference level - iLevel2000 = 1 + ! if even the bottom level isn't deep enough, we still default to the bottom level + iLevel2000 = nVertLevels do iLevel=iLevel0700,nVertLevels if(refBottomDepth(iLevel) > 2000.0_RKIND) then - iLevel2000 = iLevel-1 + iLevel2000 = iLevel exit endif enddo @@ -418,6 +423,7 @@ subroutine ocn_compute_high_frequency_output(domain, timeLevel, err)!{{{ divergenceAt250m(:) = divergence(iLevel0250,:) relativeVorticityVertexAt250m(:) = relativeVorticity(iLevel0250,:) kineticEnergyAtSurface(:) = kineticEnergyCell(1,:) + relativeVorticityAtSurface(:) = relativeVorticityCell(1,:) activeTracersAtSurface(1,:) = activeTracers(1,1,:) activeTracersAtSurface(2,:) = activeTracers(2,1,:) activeTracersAt250m(1,:) = activeTracers(1,iLevel0250,:) diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_lagrangian_particle_tracking.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_lagrangian_particle_tracking.F index ba02542eb45..8a8a2665bcb 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_lagrangian_particle_tracking.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_lagrangian_particle_tracking.F @@ -3833,7 +3833,7 @@ subroutine convert_latlon_from_xyz(lat, lon, x, y, z) !{{{ ! ensure range in 0, 2*pi if (lon < 0.0_RKIND) then - lon = 2*pii + lon + lon = 2*pi + lon end if end subroutine convert_latlon_from_xyz !}}} diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_okubo_weiss.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_okubo_weiss.F index 88c9960092b..64ab02e0573 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_okubo_weiss.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_okubo_weiss.F @@ -946,8 +946,8 @@ subroutine ocn_compute_eddy_stats(dminfo, block, nVertLevels, nCells, nCellsSolv ! for lat/lon coordinates, convert from radians to degrees for output if (config_AM_okuboWeiss_use_lat_lon_coords) then - wsPosX = wsPosX *180.0_RKIND/pii - wsPosY = wsPosY *180.0_RKIND/pii + wsPosX = wsPosX *180.0_RKIND/pi + wsPosY = wsPosY *180.0_RKIND/pi end if call mpas_timer_stop("stats per proc") diff --git a/components/mpas-ocean/src/analysis_members/mpas_ocn_time_filters.F b/components/mpas-ocean/src/analysis_members/mpas_ocn_time_filters.F index 78adb9a0583..498f3f09c7a 100644 --- a/components/mpas-ocean/src/analysis_members/mpas_ocn_time_filters.F +++ b/components/mpas-ocean/src/analysis_members/mpas_ocn_time_filters.F @@ -62,7 +62,7 @@ module ocn_time_filters !-------------------------------------------------------------------- #ifdef TIME_FILTERS_DEBUG integer :: iEdgeOutput = 0, iBlockOutput = 0, iklevel = 1 - real (kind=RKIND) :: lonEdgePoint = 10.0_RKIND*pii/180.0_RKIND, latEdgePoint = 30.0_RKIND*pii/180.0_RKIND + real (kind=RKIND) :: lonEdgePoint = 10.0_RKIND*pi/180.0_RKIND, latEdgePoint = 30.0_RKIND*pi/180.0_RKIND #endif !*********************************************************************** @@ -182,7 +182,7 @@ subroutine ocn_init_time_filters(domain, err)!{{{ call mpas_pool_get_subpool(block % structs, 'mesh', statePool) call mpas_pool_get_array(statePool, 'latEdge', latEdge) call mpas_pool_get_array(statePool, 'lonEdge', lonEdge) - print *, 'lon = ', 180.0_RKIND/pii*lonEdge(iEdgeOutput), ' lat = ', 180.0_RKIND/pii*latEdge(iEdgeOutput), & + print *, 'lon = ', 180.0_RKIND/pi*lonEdge(iEdgeOutput), ' lat = ', 180.0_RKIND/pi*latEdge(iEdgeOutput), & ' iklevel=',iklevel, ' iEdgeOutput=',iEdgeOutput, ' iBlockOutput = ', iBlockOutput #endif diff --git a/components/mpas-ocean/src/analysis_members/shr_const_mod.F b/components/mpas-ocean/src/analysis_members/shr_const_mod.F new file mode 120000 index 00000000000..c471e79113f --- /dev/null +++ b/components/mpas-ocean/src/analysis_members/shr_const_mod.F @@ -0,0 +1 @@ +../../../../share/util/shr_const_mod.F90 \ No newline at end of file diff --git a/components/mpas-ocean/src/analysis_members/shr_kind_mod.F b/components/mpas-ocean/src/analysis_members/shr_kind_mod.F new file mode 120000 index 00000000000..77a61f967b6 --- /dev/null +++ b/components/mpas-ocean/src/analysis_members/shr_kind_mod.F @@ -0,0 +1 @@ +../../../../share/util/shr_kind_mod.F90 \ No newline at end of file diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_si.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_si.F index 9529fbdc6ea..8e542d5ac61 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_si.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_si.F @@ -2587,7 +2587,7 @@ subroutine ocn_time_integrator_si(domain, dt)!{{{ ! Reset tracer2 to 2 in top n layers ! in zonal bands, and 1 outside - lat = latCell(iCell)*180./3.1415 + lat = latCell(iCell)*180.0_RKIND/pi if ( lat>-60.0.and.lat<-55.0 & .or.lat>-40.0.and.lat<-35.0 & .or.lat>- 2.5.and.lat< 2.5 & @@ -2604,7 +2604,7 @@ subroutine ocn_time_integrator_si(domain, dt)!{{{ ! Reset tracer3 to 2 in top n layers ! in zonal bands, and 1 outside - lat = latCell(iCell)*180./3.1415 + lat = latCell(iCell)*180.0_RKIND/pi if ( lat>-55.0.and.lat<-50.0 & .or.lat>-35.0.and.lat<-30.0 & .or.lat>-15.0.and.lat<-10.0 & diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F index 653d8819813..2b35a4efae4 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split.F @@ -2225,7 +2225,7 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ ! Reset tracer2 to 2 in top n layers ! in zonal bands, and 1 outside - lat = latCell(iCell)*180./3.1415 + lat = latCell(iCell)*180.0_RKIND/pi if ( lat>-60.0.and.lat<-55.0 & .or.lat>-40.0.and.lat<-35.0 & .or.lat>- 2.5.and.lat< 2.5 & @@ -2242,7 +2242,7 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ ! Reset tracer3 to 2 in top n layers ! in zonal bands, and 1 outside - lat = latCell(iCell)*180./3.1415 + lat = latCell(iCell)*180.0_RKIND/pi if ( lat>-55.0.and.lat<-50.0 & .or.lat>-35.0.and.lat<-30.0 & .or.lat>-15.0.and.lat<-10.0 & diff --git a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split_ab2.F b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split_ab2.F index c6040aeba1d..0693d00a72f 100644 --- a/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split_ab2.F +++ b/components/mpas-ocean/src/mode_forward/mpas_ocn_time_integration_split_ab2.F @@ -2405,7 +2405,7 @@ subroutine ocn_time_integrator_split_ab2(domain, dt)!{{{ ! Reset tracer2 to 2 in top n layers ! in zonal bands, and 1 outside - lat = latCell(iCell)*180./3.1415 + lat = latCell(iCell)*180.0_RKIND/pi if ( lat>-60.0.and.lat<-55.0 & .or.lat>-40.0.and.lat<-35.0 & .or.lat>- 2.5.and.lat< 2.5 & @@ -2422,7 +2422,7 @@ subroutine ocn_time_integrator_split_ab2(domain, dt)!{{{ ! Reset tracer3 to 2 in top n layers ! in zonal bands, and 1 outside - lat = latCell(iCell)*180./3.1415 + lat = latCell(iCell)*180.0_RKIND/pi if ( lat>-55.0.and.lat<-50.0 & .or.lat>-35.0.and.lat<-30.0 & .or.lat>-15.0.and.lat<-10.0 & diff --git a/components/mpas-ocean/src/shared/Makefile b/components/mpas-ocean/src/shared/Makefile index 35017551c75..d378b68624b 100644 --- a/components/mpas-ocean/src/shared/Makefile +++ b/components/mpas-ocean/src/shared/Makefile @@ -143,7 +143,7 @@ mpas_ocn_tracer_advection_std.o: mpas_ocn_config.o mpas_ocn_mesh.o mpas_ocn_trac mpas_ocn_tracer_advection_vert.o: mpas_ocn_mesh.o mpas_ocn_config.o -mpas_ocn_tracer_advection_shared.o: mpas_ocn_mesh.o mpas_ocn_config.o +mpas_ocn_tracer_advection_shared.o: mpas_ocn_constants.o mpas_ocn_mesh.o mpas_ocn_config.o mpas_ocn_tracer_hmix_redi.o: mpas_ocn_constants.o mpas_ocn_config.o diff --git a/components/mpas-ocean/src/shared/mpas_ocn_constants.F b/components/mpas-ocean/src/shared/mpas_ocn_constants.F index 35ada1306f8..dd3214d8d4f 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_constants.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_constants.F @@ -65,6 +65,7 @@ module ocn_constants real (kind=RKIND), public :: & T0_Kelvin ,&! zero point for Celsius + pi ,&! pi mpercm ,&! meters per m cmperm ,&! m per meter days_per_second ,&! days per second @@ -117,6 +118,7 @@ subroutine ocn_constants_init(configPool, packagePool)!{{{ !----------------------------------------------------------------------- T0_Kelvin = 273.16_RKIND ! zero point for Celsius + pi = 3.14159265358979323846_RKIND ! pi rho_air = 1.2_RKIND ! ambient air density (kg/m^3) rho_sw = config_density0 ! density of salt water (kg/m^3) rho_fw = 1.0e3_RKIND ! avg. water density (kg/m^3) @@ -159,6 +161,7 @@ subroutine ocn_constants_init(configPool, packagePool)!{{{ #ifdef MPAS_ESM_SHR_CONST T0_Kelvin = SHR_CONST_TKFRZ ! zero point for Celsius + pi = SHR_CONST_PI ! zero point for Celsius cp_sw = SHR_CONST_CPSW ! erg/g/K cp_air = SHR_CONST_CPDAIR ! J/kg/K rho_air = SHR_CONST_RHODAIR ! kg/m^3 diff --git a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F index a2d6586a7d6..0334667f2f1 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_diagnostics.F @@ -3540,6 +3540,7 @@ subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & !$omp parallel !$omp do schedule(runtime) & !$omp private(kmin, fracAbsorbed, fracAbsorbedRunoff, fracAbsorbedSubglacialRunoff, & + !$omp zTop, k, zBot, transmissionCoeffTop, transmissionCoeffBot, & !$omp sumSurfaceStressSquared, i, iEdge) do iCell = 1, nCells @@ -3573,8 +3574,8 @@ subroutine ocn_compute_KPP_input_fields(statePool, forcingPool, & do k = maxLevelCell(iCell), minLevelCell(iCell), -1 zBot = zTop - layerThickness(k,iCell) if (k == minLevelCell(iCell)) then - transmissionCoeffTop = exp( max(zTop / config_flux_attenuation_coefficient_runoff, -100.0_RKIND) ) - transmissionCoeffBot = exp( max(zBot / config_flux_attenuation_coefficient_runoff, -100.0_RKIND) ) + transmissionCoeffTop = exp( max(zTop / config_flux_attenuation_coefficient_subglacial_runoff, -100.0_RKIND) ) + transmissionCoeffBot = exp( max(zBot / config_flux_attenuation_coefficient_subglacial_runoff, -100.0_RKIND) ) fracAbsorbedSubglacialRunoff = transmissionCoeffTop - transmissionCoeffBot end if zTop = zBot diff --git a/components/mpas-ocean/src/shared/mpas_ocn_gm.F b/components/mpas-ocean/src/shared/mpas_ocn_gm.F index 6f4442103fa..3db04c4d3e8 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_gm.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_gm.F @@ -484,7 +484,7 @@ subroutine ocn_GM_compute_Bolus_velocity(statePool, & ! Compute the speed of the first baroclinic mode from the Brunt-Vaisala frequency. cGMphaseSpeed(iEdge) = max(c_min, & - sum_hN/(config_GM_spatially_variable_baroclinic_mode*3.141592_RKIND)) + sum_hN/(config_GM_spatially_variable_baroclinic_mode*pi)) end do !$omp end do @@ -615,7 +615,7 @@ subroutine ocn_GM_compute_Bolus_velocity(statePool, & ! for the first-mode (m=1) baroclinic gravity wave speed in m/s ! c_m = 1/pi integral(N dz ) - c_mode1 = max(1.0E-5_RKIND, sum_hN/3.141592_RKIND) + c_mode1 = max(1.0E-5_RKIND, sum_hN/pi) ! See Hallberg (2013) https://doi.org/10.1016/j.ocemod.2013.08.007 ! just after eqn 4 the defines the deformation diff --git a/components/mpas-ocean/src/shared/mpas_ocn_manufactured_solution.F b/components/mpas-ocean/src/shared/mpas_ocn_manufactured_solution.F index e9ad6786103..2631ba1737b 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_manufactured_solution.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_manufactured_solution.F @@ -217,8 +217,8 @@ subroutine ocn_manufactured_solution_init(domain, err)!{{{ if (.not. config_use_manufactured_solution) return - kx = 2.0_RKIND*pii / config_manufactured_solution_wavelength_x - ky = 2.0_RKIND*pii / config_manufactured_solution_wavelength_y + kx = 2.0_RKIND*pi / config_manufactured_solution_wavelength_x + ky = 2.0_RKIND*pi / config_manufactured_solution_wavelength_y eta0 = config_manufactured_solution_amplitude diff --git a/components/mpas-ocean/src/shared/mpas_ocn_mesh.F b/components/mpas-ocean/src/shared/mpas_ocn_mesh.F index 5e0690704e4..9eadb339b66 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_mesh.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_mesh.F @@ -33,6 +33,7 @@ module ocn_mesh use mpas_log use ocn_config + use ocn_constants implicit none private @@ -1792,7 +1793,7 @@ subroutine ocn_meshScaling() !{{{ ! neighboring cells are circles for this calculation. cellWidth = 2.0_RKIND* & sqrt((areaCell(cell1) + areaCell(cell2))/ & - 2.0_RKIND/pii) + 2.0_RKIND/pi) meshScalingDel2(iEdge) = cellWidth/ & config_hmix_ref_cell_width meshScalingDel4(iEdge) = (cellWidth/ & diff --git a/components/mpas-ocean/src/shared/mpas_ocn_surface_land_ice_fluxes.F b/components/mpas-ocean/src/shared/mpas_ocn_surface_land_ice_fluxes.F index c249f202757..fe489029a13 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_surface_land_ice_fluxes.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_surface_land_ice_fluxes.F @@ -492,168 +492,179 @@ subroutine ocn_surface_land_ice_fluxes_build_arrays(meshPool, & err = 0 - if (.not.landIceStandaloneOn) return + if (.not. (landIceStandaloneOn .or. landIceDataOn)) return call mpas_timer_start("land_ice_build_arrays") call mpas_pool_get_dimension(meshPool, 'nCellsArray', nCellsArray) - call mpas_pool_get_array(forcingPool, 'landIcePressure', landIcePressure) - - call mpas_pool_get_array(forcingPool, 'landIceFloatingFraction', landIceFloatingFraction) - call mpas_pool_get_array(forcingPool, 'landIceFloatingMask', landIceFloatingMask) - call mpas_pool_get_array(forcingPool, 'landIceFreshwaterFlux', landIceFreshwaterFlux) - call mpas_pool_get_array(forcingPool, 'landIceHeatFlux', landIceHeatFlux) - call mpas_pool_get_array(forcingPool, 'heatFluxToLandIce', heatFluxToLandIce) - call mpas_pool_get_array(forcingPool, 'frazilIceFreshwaterFlux', frazilIceFreshwaterFlux) call mpas_pool_get_array(forcingPool, 'landIceFreshwaterFluxTotal', landIceFreshwaterFluxTotal) - - call mpas_pool_get_array(forcingPool, 'landIceInterfaceTracers', landIceInterfaceTracers) - call mpas_pool_get_dimension(forcingPool, & - 'index_landIceInterfaceTemperature', & - indexITPtr) - call mpas_pool_get_dimension(forcingPool, & - 'index_landIceInterfaceSalinity', & - indexISPtr) - indexIT = indexITPtr - indexIS = indexISPtr - - if (useHollandJenkinsAdvDiff) then - call mpas_pool_get_array(forcingPool, 'landIceSurfaceTemperature', landIceSurfaceTemperature) - - allocate(freezeInterfaceSalinity(nCells), & - freezeInterfaceTemperature(nCells), & - freezeFreshwaterFlux(nCells), & - freezeHeatFlux(nCells), & - freezeIceHeatFlux(nCells)) - end if + call mpas_pool_get_array(forcingPool, 'landIceFloatingMask', landIceFloatingMask) nCells = nCellsArray( size(nCellsArray) ) - if (isomipOn) then !*** ISOMIP formulation + if (landIceStandaloneOn) then - !$omp parallel - !$omp do schedule(runtime) private(freshwaterFlux, heatFlux) - do iCell = 1, nCells - if (landIceFloatingMask(iCell) == 0) cycle + call mpas_pool_get_array(forcingPool, 'landIcePressure', landIcePressure) + call mpas_pool_get_array(forcingPool, 'landIceFloatingFraction', landIceFloatingFraction) + call mpas_pool_get_array(forcingPool, 'landIceHeatFlux', landIceHeatFlux) + call mpas_pool_get_array(forcingPool, 'heatFluxToLandIce', heatFluxToLandIce) + + + call mpas_pool_get_array(forcingPool, 'landIceInterfaceTracers', landIceInterfaceTracers) + call mpas_pool_get_dimension(forcingPool, & + 'index_landIceInterfaceTemperature', & + indexITPtr) + call mpas_pool_get_dimension(forcingPool, & + 'index_landIceInterfaceSalinity', & + indexISPtr) + indexIT = indexITPtr + indexIS = indexISPtr + + if (useHollandJenkinsAdvDiff) then + call mpas_pool_get_array(forcingPool, 'landIceSurfaceTemperature', landIceSurfaceTemperature) + + allocate(freezeInterfaceSalinity(nCells), & + freezeInterfaceTemperature(nCells), & + freezeFreshwaterFlux(nCells), & + freezeHeatFlux(nCells), & + freezeIceHeatFlux(nCells)) + end if - ! linearized equaiton for the S and p dependent potential freezing temperature - landIceInterfaceTracers(indexIT,iCell) = ocn_freezing_temperature( & - salinity=landIceBoundaryLayerTracers(indexBLT,iCell), & - pressure=landIcePressure(iCell), & - inLandIceCavity=.true.) + if (isomipOn) then !*** ISOMIP formulation - ! using (3) and (4) from Hunter (2006) - ! or (7) from Jenkins et al. (2001) if gamma constant - ! and no heat flux into ice - ! freshwater flux = density * melt rate is in kg/m^2/s - freshwaterFlux = -rho_sw * ISOMIPgammaT * (cp_sw/latent_heat_fusion_mks) & - * (landIceInterfaceTracers(indexIT,iCell)-landIceBoundaryLayerTracers(indexBLT,iCell)) + !$omp parallel + !$omp do schedule(runtime) private(freshwaterFlux, heatFlux) + do iCell = 1, nCells + if (landIceFloatingMask(iCell) == 0) cycle - landIceFreshwaterFlux(iCell) = landIceFloatingFraction(iCell)*freshwaterFlux + ! linearized equaiton for the S and p dependent potential freezing temperature + landIceInterfaceTracers(indexIT,iCell) = ocn_freezing_temperature( & + salinity=landIceBoundaryLayerTracers(indexBLT,iCell), & + pressure=landIcePressure(iCell), & + inLandIceCavity=.true.) - ! Using (13) from Jenkins et al. (2001) - ! heat flux is in W/s - heatFlux = cp_sw*(freshwaterFlux*landIceInterfaceTracers(indexIT,iCell) & - + rho_sw*ISOMIPgammaT & - * (landIceInterfaceTracers(indexIT,iCell)-landIceBoundaryLayerTracers(indexBLT,iCell))) - landIceHeatFlux(iCell) = landIceFloatingFraction(iCell)*heatFlux + ! using (3) and (4) from Hunter (2006) + ! or (7) from Jenkins et al. (2001) if gamma constant + ! and no heat flux into ice + ! freshwater flux = density * melt rate is in kg/m^2/s + freshwaterFlux = -rho_sw * ISOMIPgammaT * (cp_sw/latent_heat_fusion_mks) & + * (landIceInterfaceTracers(indexIT,iCell)-landIceBoundaryLayerTracers(indexBLT,iCell)) - heatFluxToLandIce(iCell) = 0.0_RKIND + landIceFreshwaterFlux(iCell) = landIceFloatingFraction(iCell)*freshwaterFlux - end do - !$omp end do - !$omp end parallel - endif ! isomipOn + ! Using (13) from Jenkins et al. (2001) + ! heat flux is in W/s + heatFlux = cp_sw*(freshwaterFlux*landIceInterfaceTracers(indexIT,iCell) & + + rho_sw*ISOMIPgammaT & + * (landIceInterfaceTracers(indexIT,iCell)-landIceBoundaryLayerTracers(indexBLT,iCell))) + landIceHeatFlux(iCell) = landIceFloatingFraction(iCell)*heatFlux - if (jenkinsOn .or. hollandJenkinsOn) then - if(useHollandJenkinsAdvDiff) then - ! melting solution - call compute_HJ99_melt_fluxes( & - landIceFloatingMask, & - landIceBoundaryLayerTracers(indexBLT,:), & - landIceBoundaryLayerTracers(indexBLS,:), & - landIceTracerTransferVelocities(indexHeatTrans,:), & - landIceTracerTransferVelocities(indexSaltTrans,:), & - landIceSurfaceTemperature, & - landIcePressure, & - landIceInterfaceTracers(indexIT,:), & - landIceInterfaceTracers(indexIS,:), & - landIceFreshwaterFlux, & - landIceHeatFlux, & - heatFluxToLandIce, & - nCells, & - err) - if(err .ne. 0) then - call mpas_log_write( & - 'compute_HJ99_melt_fluxes failed.', & - MPAS_LOG_CRIT) - end if + heatFluxToLandIce(iCell) = 0.0_RKIND - ! freezing solution - call compute_melt_fluxes( & - landIceFloatingMask, & - landIceBoundaryLayerTracers(indexBLT,:), & - landIceBoundaryLayerTracers(indexBLS,:), & - landIceTracerTransferVelocities(indexHeatTrans,:), & - landIceTracerTransferVelocities(indexSaltTrans,:), & - landIcePressure, & - freezeInterfaceTemperature, & - freezeInterfaceSalinity, & - freezeFreshwaterFlux, & - freezeHeatFlux, & - freezeIceHeatFlux, & - nCells, & - err) - if(err .ne. 0) then - call mpas_log_write( & - 'compute_melt_fluxes failed.', & - MPAS_LOG_CRIT) + end do + !$omp end do + !$omp end parallel + end if ! isomipOn + + if (jenkinsOn .or. hollandJenkinsOn) then + if(useHollandJenkinsAdvDiff) then + ! melting solution + call compute_HJ99_melt_fluxes( & + landIceFloatingMask, & + landIceBoundaryLayerTracers(indexBLT,:), & + landIceBoundaryLayerTracers(indexBLS,:), & + landIceTracerTransferVelocities(indexHeatTrans,:), & + landIceTracerTransferVelocities(indexSaltTrans,:), & + landIceSurfaceTemperature, & + landIcePressure, & + landIceInterfaceTracers(indexIT,:), & + landIceInterfaceTracers(indexIS,:), & + landIceFreshwaterFlux, & + landIceHeatFlux, & + heatFluxToLandIce, & + nCells, & + err) + if(err .ne. 0) then + call mpas_log_write( & + 'compute_HJ99_melt_fluxes failed.', & + MPAS_LOG_CRIT) + end if + + ! freezing solution + call compute_melt_fluxes( & + landIceFloatingMask, & + landIceBoundaryLayerTracers(indexBLT,:), & + landIceBoundaryLayerTracers(indexBLS,:), & + landIceTracerTransferVelocities(indexHeatTrans,:), & + landIceTracerTransferVelocities(indexSaltTrans,:), & + landIcePressure, & + freezeInterfaceTemperature, & + freezeInterfaceSalinity, & + freezeFreshwaterFlux, & + freezeHeatFlux, & + freezeIceHeatFlux, & + nCells, & + err) + if(err .ne. 0) then + call mpas_log_write( & + 'compute_melt_fluxes failed.', & + MPAS_LOG_CRIT) + end if + + do iCell = 1, nCells + if ((landIceFloatingMask(iCell) == 0) .or. (landIceFreshwaterFlux(iCell) >= 0.0_RKIND)) cycle + + landIceInterfaceTracers(indexIS,iCell) = freezeInterfaceSalinity(iCell) + landIceInterfaceTracers(indexIT,iCell) = freezeInterfaceTemperature(iCell) + landIceFreshwaterFlux(iCell) = freezeFreshwaterFlux(iCell) + landIceHeatFlux(iCell) = freezeHeatFlux(iCell) + heatFluxToLandIce(iCell) = freezeIceHeatFlux(iCell) + end do + else ! not using Holland and Jenkins advection/diffusion + call compute_melt_fluxes( & + landIceFloatingMask, & + landIceBoundaryLayerTracers(indexBLT,:), & + landIceBoundaryLayerTracers(indexBLS,:), & + landIceTracerTransferVelocities(indexHeatTrans,:), & + landIceTracerTransferVelocities(indexSaltTrans,:), & + landIcePressure, & + landIceInterfaceTracers(indexIT,:), & + landIceInterfaceTracers(indexIS,:), & + landIceFreshwaterFlux, & + landIceHeatFlux, & + heatFluxToLandIce, & + nCells, & + err) + if(err .ne. 0) then + call mpas_log_write( & + 'compute_melt_fluxes failed.', & + MPAS_LOG_CRIT) + end if end if + ! modulate the fluxes by the landIceFloatingFraction do iCell = 1, nCells - if ((landIceFloatingMask(iCell) == 0) .or. (landIceFreshwaterFlux(iCell) >= 0.0_RKIND)) cycle + if (landIceFloatingMask(iCell) == 0) cycle - landIceInterfaceTracers(indexIS,iCell) = freezeInterfaceSalinity(iCell) - landIceInterfaceTracers(indexIT,iCell) = freezeInterfaceTemperature(iCell) - landIceFreshwaterFlux(iCell) = freezeFreshwaterFlux(iCell) - landIceHeatFlux(iCell) = freezeHeatFlux(iCell) - heatFluxToLandIce(iCell) = freezeIceHeatFlux(iCell) + landIceFreshwaterFlux(iCell) = landIceFloatingFraction(iCell)*landIceFreshwaterFlux(iCell) + landIceHeatFlux(iCell) = landIceFloatingFraction(iCell)*landIceHeatFlux(iCell) + heatFluxToLandIce(iCell) = landIceFloatingFraction(iCell)*heatFluxToLandIce(iCell) end do - else ! not using Holland and Jenkins advection/diffusion - call compute_melt_fluxes( & - landIceFloatingMask, & - landIceBoundaryLayerTracers(indexBLT,:), & - landIceBoundaryLayerTracers(indexBLS,:), & - landIceTracerTransferVelocities(indexHeatTrans,:), & - landIceTracerTransferVelocities(indexSaltTrans,:), & - landIcePressure, & - landIceInterfaceTracers(indexIT,:), & - landIceInterfaceTracers(indexIS,:), & - landIceFreshwaterFlux, & - landIceHeatFlux, & - heatFluxToLandIce, & - nCells, & - err) - if(err .ne. 0) then - call mpas_log_write( & - 'compute_melt_fluxes failed.', & - MPAS_LOG_CRIT) - end if - end if - ! modulate the fluxes by the landIceFloatingFraction - do iCell = 1, nCells - if (landIceFloatingMask(iCell) == 0) cycle + end if ! jenkinsOn or hollandJenkinsOn - landIceFreshwaterFlux(iCell) = landIceFloatingFraction(iCell)*landIceFreshwaterFlux(iCell) - landIceHeatFlux(iCell) = landIceFloatingFraction(iCell)*landIceHeatFlux(iCell) - heatFluxToLandIce(iCell) = landIceFloatingFraction(iCell)*heatFluxToLandIce(iCell) - end do + if(useHollandJenkinsAdvDiff) then + deallocate(freezeInterfaceSalinity, & + freezeInterfaceTemperature, & + freezeFreshwaterFlux, & + freezeHeatFlux, & + freezeIceHeatFlux) + end if - endif ! jenkinsOn or hollandJenkinsOn + end if ! landIceStandaloneOn ! Add frazil and interface melt/freeze to get total fresh water flux if ( associated(frazilIceFreshwaterFlux) ) then @@ -666,14 +677,6 @@ subroutine ocn_surface_land_ice_fluxes_build_arrays(meshPool, & end do end if - if(useHollandJenkinsAdvDiff) then - deallocate(freezeInterfaceSalinity, & - freezeInterfaceTemperature, & - freezeFreshwaterFlux, & - freezeHeatFlux, & - freezeIceHeatFlux) - end if - call mpas_timer_stop("land_ice_build_arrays") !-------------------------------------------------------------------- diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F index f85427913b0..182fca7dbb9 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tendency.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tendency.F @@ -1527,7 +1527,7 @@ subroutine ocn_tend_freq_filtered_thickness(tendPool, statePool, & do k = 1, maxLevelCell(iCell) tend_lowFreqDivergence(k,iCell) = & - -2.0*pii/thickness_filter_timescale_sec & + -2.0*pi/thickness_filter_timescale_sec & *(lowFreqDivergence(k,iCell) - div_hu(k) & + div_hu_btr * layerThickness(k,iCell)/totalThickness) @@ -1535,7 +1535,7 @@ subroutine ocn_tend_freq_filtered_thickness(tendPool, statePool, & div_hu_btr*layerThickness(k,iCell)/totalThickness + & lowFreqDivergence(k,iCell) + & use_highFreqThick_restore* & - (-2.0*pii/highFreqThick_restore_time_sec* & + (-2.0*pi/highFreqThick_restore_time_sec* & highFreqThickness(k,iCell) ) end do ! vert (k) loop diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tidal_forcing.F b/components/mpas-ocean/src/shared/mpas_ocn_tidal_forcing.F index 2a9b400dd90..48901fbf5d5 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tidal_forcing.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tidal_forcing.F @@ -243,8 +243,8 @@ subroutine ocn_tidal_forcing_build_array(domain, meshPool, forcingPool, statePoo ! compute the tidalHeight if (trim(config_tidal_forcing_model) == 'monochromatic') then tidalHeight = config_tidal_forcing_monochromatic_amp * & - SIN(2.0_RKIND*pii/config_tidal_forcing_monochromatic_period * daysSinceStartOfSim - & - pii*config_tidal_forcing_monochromatic_phaseLag/180.0_RKIND) - & + SIN(2.0_RKIND*pi/config_tidal_forcing_monochromatic_period * daysSinceStartOfSim - & + pi*config_tidal_forcing_monochromatic_phaseLag/180.0_RKIND) - & config_tidal_forcing_monochromatic_baseline elseif (trim(config_tidal_forcing_model) == 'linear') then tidalHeight = max(config_tidal_forcing_linear_min, & diff --git a/components/mpas-ocean/src/shared/mpas_ocn_tracer_advection_shared.F b/components/mpas-ocean/src/shared/mpas_ocn_tracer_advection_shared.F index 2d14c961fba..3439776094d 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_tracer_advection_shared.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_tracer_advection_shared.F @@ -24,10 +24,12 @@ module ocn_tracer_advection_shared use mpas_derived_types use mpas_hash use mpas_sort + use mpas_constants use mpas_geometry_utils use mpas_log use ocn_config + use ocn_constants use ocn_mesh implicit none @@ -370,7 +372,6 @@ subroutine computeDerivTwo(derivTwo, err)!{{{ xec, yec, zec, &! arc bisection coords thetae_tmp, &! angle xv1, xv2, yv1, yv2, zv1, zv2, &! vertex cart coords - pii, &! pi math constant length_scale, &! length scale cos2t, costsint, sin2t ! trig function temps @@ -413,7 +414,6 @@ subroutine computeDerivTwo(derivTwo, err)!{{{ ! Initialize derivTwo and pi - pii = 2.*asin(1.0) derivTwo(:,:,:) = 0.0_RKIND do iCell = 1, nCellsAll @@ -461,9 +461,9 @@ subroutine computeDerivTwo(derivTwo, err)!{{{ end do if ( zc(1) == 1.0_RKIND) then - theta_abs(iCell) = pii/2.0_RKIND + theta_abs(iCell) = pi/2.0_RKIND else - theta_abs(iCell) = pii/2.0_RKIND & + theta_abs(iCell) = pi/2.0_RKIND & - mpas_sphere_angle( xc(1), yc(1), zc(1), & xc(2), yc(2), zc(2), & 0.0_RKIND, 0.0_RKIND, 1.0_RKIND) @@ -509,7 +509,7 @@ subroutine computeDerivTwo(derivTwo, err)!{{{ angle_2d(i) = angleEdge(edgesOnCell(i,iCell)) iEdge = edgesOnCell(i,iCell) if ( iCell .ne. cellsOnEdge(1,iEdge)) & - angle_2d(i) = angle_2d(i) - pii + angle_2d(i) = angle_2d(i) - pi xp(i) = dcEdge(edgesOnCell(i,iCell)) * cos(angle_2d(i)) yp(i) = dcEdge(edgesOnCell(i,iCell)) * sin(angle_2d(i)) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_vel_forcing_topographic_wave_drag.F b/components/mpas-ocean/src/shared/mpas_ocn_vel_forcing_topographic_wave_drag.F index 8f641ca4f31..a024ec97e51 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_vel_forcing_topographic_wave_drag.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_vel_forcing_topographic_wave_drag.F @@ -199,7 +199,7 @@ subroutine ocn_vel_forcing_topographic_wave_drag_init(err)!{{{ call mpas_log_write(" Topographic wave drag scheme is: Jayne and St. Laurent") call mpas_log_write("") tensorScheme = .false. - kappa = pii/10000.0_RKIND + kappa = pi/10000.0_RKIND do iEdge = 1, nEdgesAll kmax = maxLevelEdgeTop(iEdge) @@ -241,7 +241,7 @@ subroutine ocn_vel_forcing_topographic_wave_drag_init(err)!{{{ topographicWaveDrag(iEdge) = topographicWaveDragCoeff * gam & * bed_slope_edges(iEdge)**2 * Nbar * Nb & - / (8.0_RKIND * omegaM2 * pii**2) + / (8.0_RKIND * omegaM2 * pi**2) endif enddo else if (config_topographic_wave_drag_scheme.EQ."LGF") then @@ -256,7 +256,7 @@ subroutine ocn_vel_forcing_topographic_wave_drag_init(err)!{{{ topographicWaveDrag(iEdge) = topographicWaveDragCoeff & * (sqrt((topo_buoyancy_N1B(iEdge)**2 - omegaM2**2) & * (topo_buoyancy_N1V(iEdge)**2 - omegaM2**2))) & - / (4.0_RKIND*pii*omegaM2) + / (4.0_RKIND*pi*omegaM2) normalCoeffTWD(iEdge) = (lonGradEdge(iEdge)**2) & * (cos(angleEdge(iEdge))**2) + (latGradEdge(iEdge)**2)*(sin(angleEdge(iEdge))**2) & - 2.0_RKIND*latGradEdge(iEdge)*lonGradEdge(iEdge)*sin(angleEdge(iEdge))*(cos(angleEdge(iEdge))) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_vel_hmix_leith.F b/components/mpas-ocean/src/shared/mpas_ocn_vel_hmix_leith.F index a6f9ec5c073..db3e78e9b88 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_vel_hmix_leith.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_vel_hmix_leith.F @@ -159,7 +159,7 @@ subroutine ocn_vel_hmix_leith_tend(div, relVort, tend, err)!{{{ dcEdgeInv = 1.0_RKIND / dcEdge(iEdge) dvEdgeInv = 1.0_RKIND / dvEdge(iEdge) - visc2tmp = (leithParam*dxLeith*meshScalingDel2(iEdge)/pii)**3 + visc2tmp = (leithParam*dxLeith*meshScalingDel2(iEdge)/pi)**3 do k = minLevelEdgeBot(iEdge), maxLevelEdgeTop(iEdge) diff --git a/components/mpas-ocean/src/shared/mpas_ocn_vel_self_attraction_loading.F b/components/mpas-ocean/src/shared/mpas_ocn_vel_self_attraction_loading.F index 90d1a4e4630..a1c7a65452f 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_vel_self_attraction_loading.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_vel_self_attraction_loading.F @@ -863,8 +863,8 @@ subroutine ocn_vel_self_attraction_loading_init(domain,err)!{{{ ! Pre-compute sin and cos of latCell (co-latitude) values allocate(sinLatCell(nCellsAll), cosLatCell(nCellsAll)) do iCell = 1,nCellsAll - sinLatCell(iCell) = sin(0.5_RKIND*pii-latCell(iCell)) - cosLatCell(iCell) = cos(0.5_RKIND*pii-latCell(iCell)) + sinLatCell(iCell) = sin(0.5_RKIND*pi-latCell(iCell)) + cosLatCell(iCell) = cos(0.5_RKIND*pi-latCell(iCell)) enddo ! Calculate blocking indices @@ -2526,7 +2526,7 @@ subroutine associatedLegendrePolynomials(n, m, startIdx, endIdx, l, pmnm2, pmnm1 if (n == m) then do iCell = startIdx,endIdx - pmnm2(iCell) = sqrt(1.0_RKIND/(4.0_RKIND*pii))*sinLatCell(iCell)**m + pmnm2(iCell) = sqrt(1.0_RKIND/(4.0_RKIND*pi))*sinLatCell(iCell)**m do i = 1,m pmnm2(iCell) = pmnm2(iCell)*sqrt(real(2*i+1,RKIND)/real(2*i,RKIND)) enddo diff --git a/components/mpas-ocean/src/shared/mpas_ocn_vel_tidal_potential.F b/components/mpas-ocean/src/shared/mpas_ocn_vel_tidal_potential.F index 211538ac0a7..daafa816fab 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_vel_tidal_potential.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_vel_tidal_potential.F @@ -282,7 +282,7 @@ subroutine ocn_compute_tidal_potential_forcing(err)!{{{ !*** Compute eta by summing all constituent contributions do jCon = 1, nTidalConstituents - period = 2.0_RKIND*pii/tidalConstituentFrequency(jCon) + period = 2.0_RKIND*pi/tidalConstituentFrequency(jCon) nCycles = real(int(t/period),RKIND) targ = tidalConstituentFrequency(jCon)*(t - nCycles*period) + & tidalConstituentNodalPhase(jCon) + & @@ -583,7 +583,7 @@ subroutine tidal_constituent_factors(constituentList,nTidalConstituents,refTime, !h = 280.4661_RKIND + 0.98564736_RKIND*T; !p = 83.3535_RKIND + 0.11140353_RKIND*T; !N = 125.0445_RKIND - 0.05295377_RKIND*T; - !N = N*pii/180.0_RKIND + !N = N*pi/180.0_RKIND !! M2 !tidalConstituentAstronomical(j) = 2.0_RKIND*h - 2.0_RKIND*s @@ -611,7 +611,7 @@ subroutine tidal_constituent_factors(constituentList,nTidalConstituents,refTime, !tidalConstituentNodalAmplitude(j) = 1.009_RKIND + 0.187_RKIND*cos(N) - deg2rad = pii/180.0_RKIND + deg2rad = pi/180.0_RKIND T = adjust_angle(180.0_RKIND + real(refHour,RKIND)*(360.0_RKIND/24.0_RKIND)) do j = 1,nTidalConstituents @@ -742,8 +742,8 @@ subroutine orbit(year,julianDay,hour, & real (kind=RKIND) :: deg2rad,rad2deg real (kind=RKIND) :: NRad,pRad,IRad,nuRad,xiRad,nupRad,nup2Rad - deg2rad = pii/180.0_RKIND - rad2deg = 180.0_RKIND/pii + deg2rad = pi/180.0_RKIND + rad2deg = 180.0_RKIND/pi x = int((real(year,RKIND)-1901.0_RKIND)/4.0_RKIND) yr = real(year,RKIND) - 1900.0_RKIND diff --git a/components/mpas-ocean/src/shared/mpas_ocn_vmix_cvmix.F b/components/mpas-ocean/src/shared/mpas_ocn_vmix_cvmix.F index fcede65e420..c059fd5e9ca 100644 --- a/components/mpas-ocean/src/shared/mpas_ocn_vmix_cvmix.F +++ b/components/mpas-ocean/src/shared/mpas_ocn_vmix_cvmix.F @@ -141,6 +141,9 @@ subroutine ocn_vmix_coefs_cvmix_build(meshPool, statePool, forcingPool, err, tim integer, dimension(:), pointer :: & maxLevelCell, minLevelCell, nEdgesOnCell, maxLevelEdgeTop, minLevelEdgeBot + integer, dimension(:), pointer :: landIceMask + integer :: landIceMaskValue + real (kind=RKIND), dimension(:), pointer :: & latCell, lonCell, bottomDepth, fCell, & ssh, dcEdge, dvEdge, areaCell, iceFraction, & @@ -241,6 +244,7 @@ subroutine ocn_vmix_coefs_cvmix_build(meshPool, statePool, forcingPool, err, tim ! set pointers for fields related to ocean forcing state ! call mpas_pool_get_array(forcingPool, 'iceFraction', iceFraction) + call mpas_pool_get_array(forcingPool, 'landIceMask', landIceMask) call mpas_pool_get_array(forcingPool, 'windSpeed10m', windSpeed10m) call mpas_pool_get_array(forcingPool, 'windStressZonal', windStressZonal) call mpas_pool_get_array(forcingPool, 'windStressMeridional', windStressMeridional) @@ -345,8 +349,8 @@ subroutine ocn_vmix_coefs_cvmix_build(meshPool, statePool, forcingPool, err, tim ! specify geometry/location cvmix_variables % SeaSurfaceHeight = ssh(iCell) cvmix_variables % Coriolis = fCell(iCell) - cvmix_variables % lat = latCell(iCell) * 180.0_RKIND / 3.14_RKIND - cvmix_variables % lon = lonCell(iCell) * 180.0_RKIND / 3.14_RKIND + cvmix_variables % lat = latCell(iCell) * 180.0_RKIND / pi + cvmix_variables % lon = lonCell(iCell) * 180.0_RKIND / pi ! fill vertical position of column ! CVMix assume top of ocean is at z=0, so building all z-coordinate data based on layerThickness @@ -479,18 +483,25 @@ subroutine ocn_vmix_coefs_cvmix_build(meshPool, statePool, forcingPool, err, tim Nsqr_iface(k:maxLevelCell(iCell)+1) = Nsqr_iface(k-1) ! compute Langmuir number and Langmuir enhancement factor - if (config_cvmix_kpp_use_theory_wave .and. iceFraction(iCell) .lt. 0.05_RKIND) then - langmuirNumber(iCell) = sqrt(surfaceFrictionVelocity(iCell) / ( & + if (associated(landIceMask)) then + landIceMaskValue = landIceMask(iCell) + else + landIceMaskValue = 0 + endif + + if ( landIceMaskValue .eq. 0 .and. iceFraction(iCell) .lt. 0.05_RKIND) then + if (config_cvmix_kpp_use_theory_wave) then + langmuirNumber(iCell) = sqrt(surfaceFrictionVelocity(iCell) / ( & cvmix_kpp_ustokes_SL_model(windSpeed10m(iCell), & boundaryLayerDepth(iCell), & cvmix_global_params)+1e-15_RKIND) ) - langmuirEnhancementFactor = & + langmuirEnhancementFactor = & cvmix_kpp_EFactor_model(windSpeed10m(iCell), & surfaceFrictionVelocity(iCell), & boundaryLayerDepth(iCell), & cvmix_global_params) - else if (config_cvmix_kpp_use_active_wave .and. iceFraction(iCell) .lt. 0.05_RKIND) then - call ocn_stokes_drift_langmuir_number(windStressZonal(iCell), & + else if (config_cvmix_kpp_use_active_wave) then + call ocn_stokes_drift_langmuir_number(windStressZonal(iCell), & windStressMeridional(iCell), & surfaceFrictionVelocity(iCell), & significantWaveHeight(iCell), & @@ -499,9 +510,10 @@ subroutine ocn_vmix_coefs_cvmix_build(meshPool, statePool, forcingPool, err, tim stokesDriftMeridionalWavenumber(:,iCell), & alphaAngle, & langmuirNumber(iCell)) - call ocn_stokes_drift_kpp_enhancement_factor(alphaAngle, & + call ocn_stokes_drift_kpp_enhancement_factor(alphaAngle, & langmuirNumber(iCell), & langmuirEnhancementFactor) + endif else ! arbitrarily large Langmuir number langmuirNumber(iCell) = 1.e10_RKIND @@ -678,14 +690,21 @@ subroutine ocn_vmix_coefs_cvmix_build(meshPool, statePool, forcingPool, err, tim OBL_depth = boundaryLayerDepth(iCell) ) ! update Langmuir enhancement factor - if (config_cvmix_kpp_use_theory_wave .and. iceFraction(iCell) .lt. 0.05_RKIND) then - langmuirEnhancementFactor = & + if (associated(landIceMask)) then + landIceMaskValue = landIceMask(iCell) + else + landIceMaskValue = 0 + endif + + if (landIceMaskValue .eq. 0 .and. iceFraction(iCell) .lt. 0.05_RKIND) then + if (config_cvmix_kpp_use_theory_wave) then + langmuirEnhancementFactor = & cvmix_kpp_EFactor_model(windSpeed10m(iCell), & surfaceFrictionVelocity(iCell), & boundaryLayerDepth(iCell), & cvmix_global_params) - else if (config_cvmix_kpp_use_active_wave .and. iceFraction(iCell) .lt. 0.05_RKIND) then - call ocn_stokes_drift_langmuir_number(windStressZonal(iCell), & + else if (config_cvmix_kpp_use_active_wave) then + call ocn_stokes_drift_langmuir_number(windStressZonal(iCell), & windStressMeridional(iCell), & surfaceFrictionVelocity(iCell), & significantWaveHeight(iCell), & @@ -694,13 +713,13 @@ subroutine ocn_vmix_coefs_cvmix_build(meshPool, statePool, forcingPool, err, tim stokesDriftMeridionalWavenumber(:,iCell), & alphaAngle, & langmuirNumber(iCell)) - call ocn_stokes_drift_kpp_enhancement_factor(alphaAngle, & - langmuirNumber(iCell), & - langmuirEnhancementFactor) + call ocn_stokes_drift_kpp_enhancement_factor(alphaAngle, & + langmuirNumber(iCell), & + langmuirEnhancementFactor) + endif else langmuirEnhancementFactor = 1.0_RKIND - end if - + endif ! call mpas_timer_start('cvmix coeffs kpp', .false.) call cvmix_coeffs_kpp( & Mdiff_out = cvmix_variables % Mdiff_iface(minLevelCell(iCell):maxLevelCell(iCell)+1), & diff --git a/components/mpas-seaice/bld/namelist_files/namelist_defaults_mpassi.xml b/components/mpas-seaice/bld/namelist_files/namelist_defaults_mpassi.xml index d62797de451..69294640ff6 100644 --- a/components/mpas-seaice/bld/namelist_files/namelist_defaults_mpassi.xml +++ b/components/mpas-seaice/bld/namelist_files/namelist_defaults_mpassi.xml @@ -31,6 +31,7 @@ 120.0 60.0 900.0 +1800.0 'noleap' '2000-01-01_00:00:00' 'none' @@ -87,6 +88,7 @@ 85.0 85.0 85.0 +85.0 75.0 85.0 85.0 @@ -102,6 +104,7 @@ -85.0 -85.0 -85.0 +-85.0 -85.0 -85.0 -85.0 @@ -170,6 +173,7 @@ 1 1 2 +1 true true 120 diff --git a/components/mpas-seaice/bld/namelist_files/namelist_definition_mpassi.xml b/components/mpas-seaice/bld/namelist_files/namelist_definition_mpassi.xml index efbaef07cb9..ba6ae4be4bb 100644 --- a/components/mpas-seaice/bld/namelist_files/namelist_definition_mpassi.xml +++ b/components/mpas-seaice/bld/namelist_files/namelist_definition_mpassi.xml @@ -1077,7 +1077,7 @@ Default: Defined in namelist_defaults.xml -Use the humic matter tracer +Use the humic (refractory dissolved organic matter) tracer Valid values: true or false Default: Defined in namelist_defaults.xml @@ -1949,7 +1949,7 @@ Default: Defined in namelist_defaults.xml -Transport type of humics +Transport type of humics (refractory dissolved organic matter) Valid values: -1 = entirely in the mobile phase; 0 = retention dominated; 1 = release dominated; 0.5 = equal but rapid exchange; 2 = equal but slow exchange Default: Defined in namelist_defaults.xml diff --git a/components/mpas-seaice/cime_config/buildnml b/components/mpas-seaice/cime_config/buildnml index c38ca08a31d..29b017f6d8b 100755 --- a/components/mpas-seaice/cime_config/buildnml +++ b/components/mpas-seaice/cime_config/buildnml @@ -330,6 +330,16 @@ def buildnml(case, caseroot, compname): logger.warning("WARNING: The specified compset is requesting seaice ICs spunup from a G-case") logger.warning(" But no file available for this grid.") + elif ice_grid == 'SOwISC12to30E3r3': + grid_date = '20240829' + grid_prefix = 'mpassi.SOwISC12to30E3r3' + decomp_date = '20240829' + decomp_prefix = 'partitions/mpas-seaice.graph.info.' + data_iceberg_file = 'Iceberg_Climatology_Merino.SOwISC12to30E3r3.20240829.nc' + if ice_ic_mode == 'spunup': + grid_date = '20240829' + grid_prefix = 'mpassi.SOwISC12to30E3r3.rstFromG-chrysalis' + elif ice_grid == 'ICOS10': grid_date = '211015' grid_prefix = 'seaice.ICOS10' @@ -924,22 +934,41 @@ def buildnml(case, caseroot, compname): lines.append(' ') if ice_bgc == 'ice_bgc': - lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') - lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') - lines.append(' ') - lines.append(' ') - lines.append(' ') lines.append(' ') lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') @@ -999,15 +1028,35 @@ def buildnml(case, caseroot, compname): lines.append(' ') lines.append(' ') lines.append(' ') - lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') - lines.append(' ') - lines.append(' ') - lines.append(' ') lines.append(' ') lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(' ') lines.append(' ') lines.append(' ') lines.append(' ') diff --git a/components/mpas-seaice/src/Registry.xml b/components/mpas-seaice/src/Registry.xml index 7c842ca2f3c..78830161c62 100644 --- a/components/mpas-seaice/src/Registry.xml +++ b/components/mpas-seaice/src/Registry.xml @@ -866,7 +866,7 @@ icepack_name="tr_bgc_PON" /> @@ -948,77 +948,77 @@ possible_values="positive real number" icepack_name="frazil_scav" /> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -1473,32 +1474,32 @@ possible_values="-1 = entirely in the mobile phase; 0 = retention dominated; 1 = release dominated; 0.5 = equal but rapid exchange; 2 = equal but slow exchange" icepack_name="zaerotype_dust4" /> - - - - - - - - - - - - - - - - - - - - + + @@ -2701,17 +2705,17 @@ - - - - - - - - - - - + + + + + + + + + + + @@ -2885,11 +2889,13 @@ @@ -3422,241 +3474,325 @@ + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -5887,6 +6242,7 @@ diff --git a/components/mpas-seaice/src/seaice.cmake b/components/mpas-seaice/src/seaice.cmake index 8b72de8ec7d..5d314fd7594 100644 --- a/components/mpas-seaice/src/seaice.cmake +++ b/components/mpas-seaice/src/seaice.cmake @@ -144,6 +144,7 @@ set(SEAICE_MODEL_FORWARD ) list(APPEND RAW_SOURCES ${SEAICE_MODEL_FORWARD}) list(APPEND DISABLE_QSMP ${SEAICE_MODEL_FORWARD}) +list(APPEND NOOPT_FILES "core_seaice/icepack/columnphysics/icepack_shortwave_data.F90") # Generate core input handle_st_nl_gen("namelist.seaice" "streams.seaice stream_list.seaice. listed" ${CORE_INPUT_DIR} ${CORE_BLDDIR}) diff --git a/components/mpas-seaice/src/shared/mpas_seaice_icepack.F b/components/mpas-seaice/src/shared/mpas_seaice_icepack.F index 36899b4b10d..9dc6f06ec34 100644 --- a/components/mpas-seaice/src/shared/mpas_seaice_icepack.F +++ b/components/mpas-seaice/src/shared/mpas_seaice_icepack.F @@ -3923,7 +3923,11 @@ subroutine column_biogeochemistry(domain) abortFlag = icepack_warnings_aborted() call get_cice_tracer_array_category(block, ciceTracerObject, & - tracerArrayCategory, iCell, setGetPhysicsTracers, setGetBGCTracers) + tracerArrayCategory, iCell, setGetPhysicsTracers, setGetBGCTracers) + + if (config_use_vertical_tracers) & + call define_total_biogeochemistry_array_cell(block, ciceTracerObject, totalVerticalBiologyIce(:,iCell), & + totalVerticalBiologySnow(:,iCell), iCell) if (checkCarbon) then call seaice_total_carbon_content_category(block,totalCarbonCatFinal(:,iCell),iceAreaCategory(1,:,:),iceVolumeCategory(1,:,:),iCell) @@ -8648,10 +8652,13 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace nzAerosols type(MPAS_pool_type), pointer :: & - tracers_aggregate + tracers_aggregate, & + biogeochemistry real(kind=RKIND), dimension(:), pointer :: & - brineFractionCell + brineFractionCell, & + carbonToNitrogenRatioAlgae, & + carbonToNitrogenRatioDON real(kind=RKIND), dimension(:,:), pointer :: & skeletalAlgaeConcCell, & @@ -8699,14 +8706,21 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace verticalParticulateIronIceCell, & verticalDissolvedIronIceCell, & verticalAerosolsIceCell, & - verticalAerosolsSnowCell + verticalAerosolsSnowCell, & + verticalDOCLabileIceCell, & + verticalAlgaeTotalCarbonIceCell, & + verticalBCTotalIceCell, & + verticalDustTotalIceCell, & + verticalBCTotalSnowCell, & + verticalDustTotalSnowCell integer :: & iBioTracers, & iBioCount, & iLayers, & iIceCount, & - iSnowCount + iSnowCount, & + nAeroType call MPAS_pool_get_config(block % configs, "config_use_skeletal_biochemistry", config_use_skeletal_biochemistry) call MPAS_pool_get_config(block % configs, "config_use_vertical_biochemistry", config_use_vertical_biochemistry) @@ -8736,6 +8750,7 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace call MPAS_pool_get_dimension(block % dimensions, "nDissolvedIron", nDissolvedIron) call MPAS_pool_get_subpool(block % structs, "tracers_aggregate", tracers_aggregate) + call MPAS_pool_get_subpool(block % structs, "biogeochemistry", biogeochemistry) call MPAS_pool_get_array(tracers_aggregate, "skeletalAlgaeConcCell", skeletalAlgaeConcCell) call MPAS_pool_get_array(tracers_aggregate, "skeletalDOCConcCell", skeletalDOCConcCell) @@ -8784,6 +8799,14 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace call MPAS_pool_get_array(tracers_aggregate, "verticalAerosolsSnowCell", verticalAerosolsSnowCell) call MPAS_pool_get_array(tracers_aggregate, "verticalSalinityCell", verticalSalinityCell) call MPAS_pool_get_array(tracers_aggregate, "brineFractionCell", brineFractionCell) + call MPAS_pool_get_array(tracers_aggregate, "verticalAlgaeTotalCarbonIceCell", verticalAlgaeTotalCarbonIceCell) + call MPAS_pool_get_array(tracers_aggregate, "verticalDOCLabileIceCell", verticalDOCLabileIceCell) + call MPAS_pool_get_array(tracers_aggregate, "verticalBCTotalIceCell", verticalBCTotalIceCell) + call MPAS_pool_get_array(tracers_aggregate, "verticalDustTotalIceCell", verticalDustTotalIceCell) + call MPAS_pool_get_array(tracers_aggregate, "verticalDustTotalSnowCell", verticalDustTotalSnowCell) + call MPAS_pool_get_array(tracers_aggregate, "verticalBCTotalSnowCell", verticalBCTotalSnowCell) + call MPAS_pool_get_array(biogeochemistry, "carbonToNitrogenRatioAlgae", carbonToNitrogenRatioAlgae) + call MPAS_pool_get_array(biogeochemistry, "carbonToNitrogenRatioDON", carbonToNitrogenRatioDON) ! biogeochemistry ! brine height fraction @@ -8866,12 +8889,15 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace ! algal nitrogen do iBioTracers = 1, nAlgae iIceCount = (iBioTracers-1)*nBioLayersP1 + verticalAlgaeTotalCarbonIceCell(:,iCell) = 0.0_RKIND do iLayers = 1,nBioLayersP1 iBiocount = iBiocount + 1 verticalAlgaeConcCell(iBioCount,iCell) = & tracerArrayCell(tracerObject % index_algaeConc(iBioTracers)+iLayers-1) verticalAlgaeIceCell(iLayers+iIceCount,iCell) = verticalAlgaeConcCell(iBioCount,iCell) + verticalAlgaeTotalCarbonIceCell(iLayers,iCell) = verticalAlgaeTotalCarbonIceCell(iLayers,iCell) + & + carbonToNitrogenRatioAlgae(iBioTracers) * verticalAlgaeConcCell(iBioCount,iCell) enddo do iLayers = nBioLayersP1+1,nBioLayersP3 iBiocount = iBiocount + 1 @@ -8894,6 +8920,7 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace if (config_use_carbon) then iBioCount = 0 + verticalDOCLabileIceCell(:,iCell) = 0.0_RKIND ! DOC do iBioTracers = 1, nDOC @@ -8904,6 +8931,8 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace verticalDOCConcCell(iBioCount,iCell) = & tracerArrayCell(tracerObject % index_DOCConc(iBioTracers) + iLayers-1) verticalDOCIceCell(iLayers+iIceCount,iCell) = verticalDOCConcCell(iBioCount,iCell) + verticalDOCLabileIceCell(iLayers, iCell) = verticalDOCLabileIceCell(iLayers, iCell) + & + verticalDOCConcCell(iBioCount,iCell) enddo do iLayers = nBioLayersP1+1,nBioLayersP3 iBioCount = iBioCount + 1 @@ -8942,6 +8971,8 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace verticalDONConcCell(iBioCount,iCell) = & tracerArrayCell(tracerObject % index_DONConc(iBioTracers) + iLayers-1) verticalDONIceCell(iLayers+iIceCount,iCell) = verticalDONConcCell(iBioCount,iCell) + verticalDOCLabileIceCell(iLayers, iCell) = verticalDOCLabileIceCell(iLayers, iCell) + & + verticalDONConcCell(iBioCount,iCell) * carbonToNitrogenRatioDON(iBioTracers) enddo do iLayers = nBioLayersP1+1,nBioLayersP3 iBioCount = iBioCount + 1 @@ -9058,15 +9089,25 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace ! black carbon and dust aerosols if (config_use_zaerosols) then iBioCount = 0 + verticalDustTotalIceCell(:,iCell) = 0.0_RKIND + verticalBCTotalIceCell(:,iCell) = 0.0_RKIND + verticalDustTotalSnowCell(:,iCell) = 0.0_RKIND + verticalBCTotalSnowCell(:,iCell) = 0.0_RKIND + do iBioTracers = 1, nzAerosols iIceCount = (iBioTracers-1)*nBioLayersP1 iSnowCount = (iBioTracers-1)*2 - + nAeroType = 1 + if (iBioTracers .gt. 2) nAeroType = 0 do iLayers = 1,nBioLayersP1 iBioCount = iBioCount + 1 verticalAerosolsConcCell(iBioCount,iCell) = & tracerArrayCell(tracerObject % index_verticalAerosolsConc(iBioTracers)+iLayers-1) verticalAerosolsIceCell(iLayers+iIceCount,iCell) = verticalAerosolsConcCell(iBioCount,iCell) + verticalBCTotalIceCell(iLayers, iCell) = verticalBCTotalIceCell(iLayers, iCell) + nAeroType * & + verticalAerosolsConcCell(iBioCount,iCell) + verticalDustTotalIceCell(iLayers, iCell) = verticalDustTotalIceCell(iLayers, iCell) + (1-nAeroType) * & + verticalAerosolsConcCell(iBioCount,iCell) enddo do iLayers = nBioLayersP1+1,nBioLayersP3 iBioCount = iBioCount + 1 @@ -9074,6 +9115,10 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace tracerArrayCell(tracerObject % index_verticalAerosolsConc(iBioTracers)+iLayers-1) verticalAerosolsSnowCell(iLayers-nBioLayersP1+iSnowCount,iCell) = & verticalAerosolsConcCell(iBioCount,iCell) + verticalBCTotalSnowCell(iLayers-nBioLayersP1, iCell) = verticalBCTotalSnowCell(iLayers-nBioLayersP1, iCell) + & + nAeroType * verticalAerosolsConcCell(iBioCount,iCell) + verticalDustTotalSnowCell(iLayers-nBioLayersP1,iCell) = verticalDustTotalSnowCell(iLayers-nBioLayersP1,iCell) + & + (1-nAeroType) * verticalAerosolsConcCell(iBioCount,iCell) enddo enddo endif @@ -9088,6 +9133,274 @@ subroutine get_cice_biogeochemistry_tracer_array_cell(block, tracerObject, trace end subroutine get_cice_biogeochemistry_tracer_array_cell +!||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +! +! define_total_biogeochemistry_array_cell +! +!> \brief +!> \author Nicole Jeffery, LANL +!> \date 16rd September 2024 +!> \details +!> +! Defines total ice and snow arrays for biogeochemistry and aerosols diagnostics +! +!----------------------------------------------------------------------- + + subroutine define_total_biogeochemistry_array_cell(block, tracerObject, totalVerticalBioIceCell, & + totalVerticalBioSnowCell, iCell) + + type(block_type), intent(in) :: & + block + + type(ciceTracerObjectType), intent(in) :: & + tracerObject + + real(kind=RKIND), dimension(:), intent(in):: & + totalVerticalBioIceCell, & + totalVerticalBioSnowCell + + integer, intent(in) :: & + iCell + + logical, pointer :: & + config_use_vertical_biochemistry, & + config_use_nitrate, & + config_use_carbon, & + config_use_ammonium, & + config_use_silicate, & + config_use_DMS, & + config_use_nonreactive, & + config_use_humics, & + config_use_DON, & + config_use_iron, & + config_use_zaerosols + + integer, pointer :: & + nAlgae, & + nDOC, & + nDIC, & + nDON, & + nParticulateIron, & + nDissolvedIron, & + nzAerosols + + type(MPAS_pool_type), pointer :: & + biogeochemistry + + real(kind=RKIND), dimension(:), pointer :: & + totalVerticalDiatomIce, & + totalVerticalSmallPlanktonIce, & + totalVerticalPhaeocystisIce, & + totalVerticalNitrateIce, & + totalVerticalPolysaccharidsIce, & + totalVerticalLipidsIce, & + totalVerticalDICIce, & + totalVerticalAmmoniumIce, & + totalVerticalSilicateIce, & + totalVerticalDMSPpIce, & + totalVerticalDMSPdIce, & + totalVerticalDMSIce, & + totalVerticalNonreactiveIce, & + totalVerticalHumicsIce, & + totalVerticalProteinsIce, & + totalVerticalDissolvedIronIce, & + totalVerticalParticulateIronIce, & + totalVerticalDustIce, & + totalVerticalDustSnow, & + totalVerticalBC1Ice, & + totalVerticalBC1Snow, & + totalVerticalBC2Ice, & + totalVerticalBC2Snow, & + totalVerticalBCIce, & + totalVerticalBCSnow, & + totalVerticalDissolvedIronSnow, & + totalVerticalAlgaeCarbonIce, & + totalVerticalDOCLabileIce, & + carbonToNitrogenRatioAlgae, & + carbonToNitrogenRatioDON + + call MPAS_pool_get_config(block % configs, "config_use_vertical_biochemistry", config_use_vertical_biochemistry) + call MPAS_pool_get_config(block % configs, "config_use_nitrate", config_use_nitrate) + call MPAS_pool_get_config(block % configs, "config_use_carbon", config_use_carbon) + call MPAS_pool_get_config(block % configs, "config_use_ammonium",config_use_ammonium) + call MPAS_pool_get_config(block % configs, "config_use_silicate",config_use_silicate) + call MPAS_pool_get_config(block % configs, "config_use_DMS",config_use_DMS) + call MPAS_pool_get_config(block % configs, "config_use_nonreactive",config_use_nonreactive) + call MPAS_pool_get_config(block % configs, "config_use_humics",config_use_humics) + call MPAS_pool_get_config(block % configs, "config_use_DON",config_use_DON) + call MPAS_pool_get_config(block % configs, "config_use_iron",config_use_iron) + call MPAS_pool_get_config(block % configs, "config_use_zaerosols",config_use_zaerosols) + + call MPAS_pool_get_dimension(block % dimensions, "nzAerosols", nzAerosols) + call MPAS_pool_get_dimension(block % dimensions, "nAlgae", nAlgae) + call MPAS_pool_get_dimension(block % dimensions, "nDOC", nDOC) + call MPAS_pool_get_dimension(block % dimensions, "nDIC", nDIC) + call MPAS_pool_get_dimension(block % dimensions, "nDON", nDON) + call MPAS_pool_get_dimension(block % dimensions, "nParticulateIron", nParticulateIron) + call MPAS_pool_get_dimension(block % dimensions, "nDissolvedIron", nDissolvedIron) + + call MPAS_pool_get_subpool(block % structs, "biogeochemistry", biogeochemistry) + + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDiatomIce", totalVerticalDiatomIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalSmallPlanktonIce", totalVerticalSmallPlanktonIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalPhaeocystisIce", totalVerticalPhaeocystisIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalPolysaccharidsIce", totalVerticalPolysaccharidsIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalLipidsIce", totalVerticalLipidsIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDICIce", totalVerticalDICIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalProteinsIce", totalVerticalProteinsIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalNitrateIce", totalVerticalNitrateIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalSilicateIce", totalVerticalSilicateIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalAmmoniumIce", totalVerticalAmmoniumIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDMSIce", totalVerticalDMSIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDMSPpIce", totalVerticalDMSPpIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDMSPdIce", totalVerticalDMSPdIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalNonreactiveIce", totalVerticalNonreactiveIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalHumicsIce", totalVerticalHumicsIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalParticulateIronIce", totalVerticalParticulateIronIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDissolvedIronIce", totalVerticalDissolvedIronIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDissolvedIronSnow", totalVerticalDissolvedIronSnow) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDustIce", totalVerticalDustIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDustSnow", totalVerticalDustSnow) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalBC1Ice", totalVerticalBC1Ice) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalBC1Snow", totalVerticalBC1Snow) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalBC2Ice", totalVerticalBC2Ice) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalBC2Snow", totalVerticalBC2Snow) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalBCIce", totalVerticalBCIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalBCSnow", totalVerticalBCSnow) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalAlgaeCarbonIce", totalVerticalAlgaeCarbonIce) + call MPAS_pool_get_array(biogeochemistry, "totalVerticalDOCLabileIce", totalVerticalDOCLabileIce) + call MPAS_pool_get_array(biogeochemistry, "carbonToNitrogenRatioAlgae", carbonToNitrogenRatioAlgae) + call MPAS_pool_get_array(biogeochemistry, "carbonToNitrogenRatioDON", carbonToNitrogenRatioDON) + + ! biogeochemistry + + if (config_use_vertical_biochemistry) then + + ! algal nitrogen + totalVerticalDiatomIce(iCell) = totalVerticalBioIceCell(tracerObject % index_algaeConcLayer(1)) + totalVerticalAlgaeCarbonIce(iCell) = totalVerticalDiatomIce(iCell) * carbonToNitrogenRatioAlgae(1) + + SELECT CASE (nAlgae) + CASE (2) + totalVerticalSmallPlanktonIce(iCell) = totalVerticalBioIceCell(tracerObject % index_algaeConcLayer(2)) + totalVerticalAlgaeCarbonIce(iCell) = totalVerticalAlgaeCarbonIce(iCell) + & + totalVerticalSmallPlanktonIce(iCell) * carbonToNitrogenRatioAlgae(2) + CASE (3) + totalVerticalSmallPlanktonIce(iCell) = totalVerticalBioIceCell(tracerObject % index_algaeConcLayer(2)) + totalVerticalPhaeocystisIce(iCell) = totalVerticalBioIceCell(tracerObject % index_algaeConcLayer(3)) + totalVerticalAlgaeCarbonIce(iCell) = totalVerticalAlgaeCarbonIce(iCell) + & + totalVerticalSmallPlanktonIce(iCell) * carbonToNitrogenRatioAlgae(2) + & + totalVerticalPhaeocystisIce(iCell) * carbonToNitrogenRatioAlgae(3) + END SELECT + end if + + ! nitrate + if (config_use_nitrate) & + totalVerticalNitrateIce(iCell) = totalVerticalBioIceCell(tracerObject % index_nitrateConcLayer) + + if (config_use_carbon) then + + ! DOC + totalVerticalDOCLabileIce(iCell) = 0.0_RKIND + SELECT CASE (nDOC) + CASE (1) + totalVerticalPolysaccharidsIce(iCell) = totalVerticalBioIceCell(tracerObject % index_DOCConcLayer(1)) + totalVerticalDOCLabileIce(iCell) = totalVerticalPolysaccharidsIce(iCell) + CASE (2) + totalVerticalPolysaccharidsIce(iCell) = totalVerticalBioIceCell(tracerObject % index_DOCConcLayer(1)) + totalVerticalLipidsIce(iCell) = totalVerticalBioIceCell(tracerObject % index_DOCConcLayer(1)) + totalVerticalDOCLabileIce(iCell) = totalVerticalPolysaccharidsIce(iCell) + totalVerticalLipidsIce(iCell) + END SELECT + + ! DIC + SELECT CASE (nDIC) + CASE (1) + totalVerticalDICIce(iCell) = totalVerticalBioIceCell(tracerObject % index_DICConcLayer(1)) + END SELECT + end if + + if (config_use_DON) then + + ! DON + SELECT CASE (nDON) + CASE (1) + totalVerticalProteinsIce(iCell) = totalVerticalBioIceCell(tracerObject % index_DONConcLayer(1)) + totalVerticalDOCLabileIce(iCell) = totalVerticalDOCLabileIce(iCell) + & + totalVerticalProteinsIce(iCell)* carbonToNitrogenRatioDON(1) + END SELECT + end if + + ! ammonium + if (config_use_ammonium) & + totalVerticalAmmoniumIce(iCell) = totalVerticalBioIceCell(tracerObject % index_ammoniumConcLayer) + + ! silicate + if (config_use_silicate) & + totalVerticalSilicateIce(iCell) = totalVerticalBioIceCell(tracerObject % index_silicateConcLayer) + + ! DMS, DMSPp, DMSPd + if (config_use_DMS) then + totalVerticalDMSIce(iCell) = totalVerticalBioIceCell(tracerObject % index_DMSConcLayer) + totalVerticalDMSPpIce(iCell) = totalVerticalBioIceCell(tracerObject % index_DMSPpConcLayer) + totalVerticalDMSPdIce(iCell) = totalVerticalBioIceCell(tracerObject % index_DMSPdConcLayer) + end if + + ! nonreactive + if (config_use_nonreactive) & + totalVerticalNonreactiveIce(iCell) = totalVerticalBioIceCell(tracerObject % index_nonreactiveConcLayer) + + ! humic material (refractory DOC) + if (config_use_humics) then + totalVerticalHumicsIce(iCell) = totalVerticalBioIceCell(tracerObject % index_humicsConcLayer) + end if + + if (config_use_iron) then + + ! ParticulateIron + SELECT CASE (nParticulateIron) + CASE (1) + totalVerticalParticulateIronIce(iCell) = totalVerticalBioIceCell(tracerObject % index_particulateIronConcLayer(1)) + END SELECT + + ! DissolvedIron + SELECT CASE (nDissolvedIron) + CASE (1) + totalVerticalDissolvedIronIce(iCell) = totalVerticalBioIceCell(tracerObject % index_dissolvedIronConcLayer(1)) + totalVerticalDissolvedIronSnow(iCell) = totalVerticalBioSnowCell(tracerObject % index_dissolvedIronConcLayer(1)) + END SELECT + + end if + + ! black carbon and dust aerosols + if (config_use_zaerosols) then + + SELECT CASE (nzAerosols) + CASE (1) + totalVerticalBC1Ice(iCell) = totalVerticalBioIceCell(tracerObject % index_verticalAerosolsConcLayer(1)) + totalVerticalBC1Snow(iCell) = totalVerticalBioSnowCell(tracerObject % index_verticalAerosolsConcLayer(1)) + totalVerticalBCIce(iCell) = totalVerticalBC1Ice(iCell) + totalVerticalBCSnow(iCell) = totalVerticalBC1Snow(iCell) + CASE (2) + totalVerticalBC1Ice(iCell) = totalVerticalBioIceCell(tracerObject % index_verticalAerosolsConcLayer(1)) + totalVerticalBC1Snow(iCell) = totalVerticalBioSnowCell(tracerObject % index_verticalAerosolsConcLayer(1)) + totalVerticalBC2Ice(iCell) = totalVerticalBioIceCell(tracerObject % index_verticalAerosolsConcLayer(2)) + totalVerticalBC2Snow(iCell) = totalVerticalBioSnowCell(tracerObject % index_verticalAerosolsConcLayer(2)) + totalVerticalBCIce(iCell) = totalVerticalBC1Ice(iCell) + totalVerticalBC2Ice(iCell) + totalVerticalBCSnow(iCell) = totalVerticalBC1Snow(iCell) + totalVerticalBC2Snow(iCell) + CASE (3) + totalVerticalBC1Ice(iCell) = totalVerticalBioIceCell(tracerObject % index_verticalAerosolsConcLayer(1)) + totalVerticalBC1Snow(iCell) = totalVerticalBioSnowCell(tracerObject % index_verticalAerosolsConcLayer(1)) + totalVerticalBC2Ice(iCell) = totalVerticalBioIceCell(tracerObject % index_verticalAerosolsConcLayer(2)) + totalVerticalBC2Snow(iCell) = totalVerticalBioSnowCell(tracerObject % index_verticalAerosolsConcLayer(2)) + totalVerticalDustIce(iCell) = totalVerticalBioIceCell(tracerObject % index_verticalAerosolsConcLayer(3)) + totalVerticalDustSnow(iCell) = totalVerticalBioSnowCell(tracerObject % index_verticalAerosolsConcLayer(3)) + totalVerticalBCIce(iCell) = totalVerticalBC1Ice(iCell) + totalVerticalBC2Ice(iCell) + totalVerticalBCSnow(iCell) = totalVerticalBC1Snow(iCell) + totalVerticalBC2Snow(iCell) + END SELECT + end if + + end subroutine define_total_biogeochemistry_array_cell + !----------------------------------------------------------------------- ! Init CICE parameters !----------------------------------------------------------------------- @@ -14689,7 +15002,8 @@ subroutine seaice_icepack_reinitialize_diagnostics_bgc(domain) type(MPAS_pool_type), pointer :: & biogeochemistryPool, & - diagnostics_biogeochemistryPool + diagnostics_biogeochemistryPool, & + tracers_aggregatePool ! biogeochemistry real(kind=RKIND), dimension(:), pointer :: & @@ -14699,7 +15013,35 @@ subroutine seaice_icepack_reinitialize_diagnostics_bgc(domain) ! zSalinityFlux, & !echmod deprecate ! zSalinityGDFlux, & !echmod deprecate totalChlorophyll, & - totalCarbonContentCell + totalCarbonContentCell, & + totalVerticalDiatomIce, & + totalVerticalSmallPlanktonIce, & + totalVerticalPhaeocystisIce, & + totalVerticalNitrateIce, & + totalVerticalPolysaccharidsIce, & + totalVerticalLipidsIce, & + totalVerticalDICIce, & + totalVerticalAmmoniumIce, & + totalVerticalSilicateIce, & + totalVerticalDMSPpIce, & + totalVerticalDMSPdIce, & + totalVerticalDMSIce, & + totalVerticalNonreactiveIce, & + totalVerticalHumicsIce, & + totalVerticalProteinsIce, & + totalVerticalDissolvedIronIce, & + totalVerticalParticulateIronIce, & + totalVerticalDustIce, & + totalVerticalDustSnow, & + totalVerticalBC1Ice, & + totalVerticalBC1Snow, & + totalVerticalBC2Ice, & + totalVerticalBC2Snow, & + totalVerticalBCIce, & + totalVerticalBCSnow, & + totalVerticalAlgaeCarbonIce, & + totalVerticalDOCLabileIce, & + totalVerticalDissolvedIronSnow real(kind=RKIND), dimension(:,:), pointer :: & oceanBioFluxes, & @@ -14709,7 +15051,12 @@ subroutine seaice_icepack_reinitialize_diagnostics_bgc(domain) totalVerticalBiologySnow, & bgridPorosityIceCell, & bgridSalinityIceCell, & - bgridTemperatureIceCell + bgridTemperatureIceCell, & + verticalBCTotalIceCell, & + verticalDustTotalIceCell, & + verticalBCTotalSnowCell, & + verticalDustTotalSnowCell + real(kind=RKIND), dimension(:,:,:), pointer :: & bioTracerShortwave @@ -14739,6 +15086,7 @@ subroutine seaice_icepack_reinitialize_diagnostics_bgc(domain) call MPAS_pool_get_subpool(block % structs, "biogeochemistry", biogeochemistryPool) call MPAS_pool_get_subpool(block % structs, "diagnostics_biogeochemistry", diagnostics_biogeochemistryPool) + call MPAS_pool_get_subpool(block % structs, "tracers_aggregate", tracers_aggregatePool) if (config_use_vertical_tracers) then call MPAS_pool_get_array(biogeochemistryPool, "primaryProduction", primaryProduction) @@ -14747,13 +15095,12 @@ subroutine seaice_icepack_reinitialize_diagnostics_bgc(domain) call MPAS_pool_get_array(diagnostics_biogeochemistryPool, "bgridSalinityIceCell", bgridSalinityIceCell) call MPAS_pool_get_array(diagnostics_biogeochemistryPool, "bgridPorosityIceCell", bgridPorosityIceCell) call MPAS_pool_get_array(diagnostics_biogeochemistryPool, "bgridTemperatureIceCell", bgridTemperatureIceCell) - - primaryProduction = 0.0_RKIND - totalChlorophyll = 0.0_RKIND - netSpecificAlgalGrowthRate = 0.0_RKIND - bgridSalinityIceCell = 0.0_RKIND - bgridPorosityIceCell = 0.0_RKIND - bgridTemperatureIceCell = 0.0_RKIND + primaryProduction = 0.0_RKIND + totalChlorophyll = 0.0_RKIND + netSpecificAlgalGrowthRate = 0.0_RKIND + bgridSalinityIceCell = 0.0_RKIND + bgridPorosityIceCell = 0.0_RKIND + bgridTemperatureIceCell = 0.0_RKIND end if @@ -14764,6 +15111,38 @@ subroutine seaice_icepack_reinitialize_diagnostics_bgc(domain) call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalBiologyIce", totalVerticalBiologyIce) call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalBiologySnow", totalVerticalBiologySnow) call MPAS_pool_get_array(biogeochemistryPool, "totalCarbonContentCell", totalCarbonContentCell) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDiatomIce", totalVerticalDiatomIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalSmallPlanktonIce", totalVerticalSmallPlanktonIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalPhaeocystisIce", totalVerticalPhaeocystisIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalPolysaccharidsIce", totalVerticalPolysaccharidsIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalLipidsIce", totalVerticalLipidsIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDICIce", totalVerticalDICIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalProteinsIce", totalVerticalProteinsIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalNitrateIce", totalVerticalNitrateIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalSilicateIce", totalVerticalSilicateIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalAmmoniumIce", totalVerticalAmmoniumIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDMSIce", totalVerticalDMSIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDMSPpIce", totalVerticalDMSPpIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDMSPdIce", totalVerticalDMSPdIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalNonreactiveIce", totalVerticalNonreactiveIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalHumicsIce", totalVerticalHumicsIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalParticulateIronIce", totalVerticalParticulateIronIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDissolvedIronIce", totalVerticalDissolvedIronIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDissolvedIronSnow", totalVerticalDissolvedIronSnow) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDustIce", totalVerticalDustIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDustSnow", totalVerticalDustSnow) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalBC1Ice", totalVerticalBC1Ice) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalBC1Snow", totalVerticalBC1Snow) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalBC2Ice", totalVerticalBC2Ice) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalBC2Snow", totalVerticalBC2Snow) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalBCIce", totalVerticalBCIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalBCSnow", totalVerticalBCSnow) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalAlgaeCarbonIce", totalVerticalAlgaeCarbonIce) + call MPAS_pool_get_array(biogeochemistryPool, "totalVerticalDOCLabileIce", totalVerticalDOCLabileIce) + call MPAS_pool_get_array(tracers_aggregatePool, "verticalBCTotalIceCell", verticalBCTotalIceCell) + call MPAS_pool_get_array(tracers_aggregatePool, "verticalDustTotalIceCell", verticalDustTotalIceCell) + call MPAS_pool_get_array(tracers_aggregatePool, "verticalDustTotalSnowCell", verticalDustTotalSnowCell) + call MPAS_pool_get_array(tracers_aggregatePool, "verticalBCTotalSnowCell", verticalBCTotalSnowCell) netBrineHeight = 0.0_RKIND @@ -14773,6 +15152,38 @@ subroutine seaice_icepack_reinitialize_diagnostics_bgc(domain) totalVerticalBiologyIce = 0.0_RKIND totalVerticalBiologySnow = 0.0_RKIND totalCarbonContentCell = 0.0_RKIND + totalVerticalDiatomIce = 0.0_RKIND + totalVerticalSmallPlanktonIce = 0.0_RKIND + totalVerticalPhaeocystisIce = 0.0_RKIND + totalVerticalPolysaccharidsIce = 0.0_RKIND + totalVerticalLipidsIce = 0.0_RKIND + totalVerticalDICIce = 0.0_RKIND + totalVerticalProteinsIce = 0.0_RKIND + totalVerticalNitrateIce = 0.0_RKIND + totalVerticalSilicateIce = 0.0_RKIND + totalVerticalAmmoniumIce = 0.0_RKIND + totalVerticalDMSIce = 0.0_RKIND + totalVerticalDMSPpIce = 0.0_RKIND + totalVerticalDMSPdIce = 0.0_RKIND + totalVerticalNonreactiveIce = 0.0_RKIND + totalVerticalHumicsIce = 0.0_RKIND + totalVerticalParticulateIronIce = 0.0_RKIND + totalVerticalDissolvedIronIce = 0.0_RKIND + totalVerticalDissolvedIronSnow = 0.0_RKIND + totalVerticalBC1Ice = 0.0_RKIND + totalVerticalBC2Ice = 0.0_RKIND + totalVerticalDustIce = 0.0_RKIND + totalVerticalBC1Snow = 0.0_RKIND + totalVerticalBC2Snow = 0.0_RKIND + totalVerticalBCSnow = 0.0_RKIND + totalVerticalBCIce = 0.0_RKIND + totalVerticalDustSnow = 0.0_RKIND + totalVerticalAlgaeCarbonIce = 0.0_RKIND + totalVerticalDOCLabileIce = 0.0_RKIND + verticalBCTotalIceCell = 0.0_RKIND + verticalDustTotalIceCell = 0.0_RKIND + verticalBCTotalSnowCell = 0.0_RKIND + verticalDustTotalSnowCell = 0.0_RKIND endif diff --git a/components/ww3/bld/build-namelist b/components/ww3/bld/build-namelist index 314949671c4..092dab3ffba 100755 --- a/components/ww3/bld/build-namelist +++ b/components/ww3/bld/build-namelist @@ -363,7 +363,7 @@ if ($NML_TYPE eq "ww3_grid") { add_default($nl, 'grid%dmin'); add_default($nl, 'unst%sf'); - add_default($nl, 'unst%filename', 'val'=>"'${DIN_LOC_ROOT}/wav/ww3/${WAV_GRID}.msh'"); + add_default($nl, 'unst%filename', 'val'=>"'${DIN_LOC_ROOT}/wav/ww3/${WAV_GRID}_rtd.msh'"); add_default($nl, 'unst%idf'); add_default($nl, 'unst%idla'); add_default($nl, 'unst%idfm'); @@ -398,8 +398,8 @@ if ($NML_TYPE eq "ww3_grid_nml") { add_default($nl, 'icedisp'); add_default($nl, 'rwndc'); - add_default($nl, 'uostfilelocal', 'val'=>"'${DIN_LOC_ROOT}/wav/ww3/obstructions_local.${WAV_GRID}${WAV_SPEC}.in'"); - add_default($nl, 'uostfileshadow', 'val'=>"'${DIN_LOC_ROOT}/wav/ww3/obstructions_shadow.${WAV_GRID}${WAV_SPEC}.in'"); + add_default($nl, 'uostfilelocal', 'val'=>"'${DIN_LOC_ROOT}/wav/ww3/obstructions_local.${WAV_GRID}${WAV_SPEC}.rtd.in'"); + add_default($nl, 'uostfileshadow', 'val'=>"'${DIN_LOC_ROOT}/wav/ww3/obstructions_shadow.${WAV_GRID}${WAV_SPEC}.rtd.in'"); } diff --git a/components/ww3/bld/namelist_files/namelist_defaults_ww3_grid.xml b/components/ww3/bld/namelist_files/namelist_defaults_ww3_grid.xml index e8bb1d42b24..0e4d4674087 100644 --- a/components/ww3/bld/namelist_files/namelist_defaults_ww3_grid.xml +++ b/components/ww3/bld/namelist_files/namelist_defaults_ww3_grid.xml @@ -11,9 +11,9 @@ variables. Values that depend on the model configuration use attributes to express the dependency. --> -1.07 +1.1 0.035 -50 +36 36 0.5 @@ -30,7 +30,7 @@ attributes to express the dependency. 450.0 30.0 -wQU225EC30to60E2r2 +wQU225Icos30E3r5 ww3_grid_namelists.nml UNST SPHE diff --git a/components/ww3/bld/namelist_files/namelist_defaults_ww3_grid_nml.xml b/components/ww3/bld/namelist_files/namelist_defaults_ww3_grid_nml.xml index 9573b482a45..9ebe2b4e505 100644 --- a/components/ww3/bld/namelist_files/namelist_defaults_ww3_grid_nml.xml +++ b/components/ww3/bld/namelist_files/namelist_defaults_ww3_grid_nml.xml @@ -158,8 +158,8 @@ attributes to express the dependency. -obstructions_local.glo_unst.in -obstructions_shadow.glo_unst.in +obstructions_local.rtd.in +obstructions_shadow.rtd.in 1.00 1.00 diff --git a/components/ww3/cime_config/buildlib_cmake b/components/ww3/cime_config/buildlib_cmake index 4f15ea2d58e..42d92b2f0ad 100755 --- a/components/ww3/cime_config/buildlib_cmake +++ b/components/ww3/cime_config/buildlib_cmake @@ -43,14 +43,20 @@ def buildlib(bldroot, installpath, case): # Define WW3 repository directories repodir = "{}/components/ww3/src".format(srcroot) modeldir = "{}/WW3/model".format(repodir) - - # TODO: these work dirs will have to be changed to live in the binary/build area. - # Achieving this will probably require a significant refactor of the ww3 infrastructure. - # Doing this stuff in-source not only clutters the repo, but also introduces potential race - # conditions if we were to try to build multiple ww3 cases simultaneously. - bindir = "{}/bin".format(modeldir) - exedir = "{}/exe".format(modeldir) - tmpdir = "{}/tmp".format(modeldir) + builddir = "{}/wav".format(exeroot) + + # work dirs are placed in the binary/build area. + bindir_source = "{}/bin".format(modeldir) + bindir = "{}/bin".format(builddir) + shutil.copytree(bindir_source, bindir) + auxdir_source = "{}/aux".format(modeldir) + auxdir = "{}/aux".format(builddir) + shutil.copytree(auxdir_source, auxdir) + ftndir_source = "{}/ftn".format(modeldir) + ftndir = "{}/ftn".format(builddir) + shutil.copytree(ftndir_source, ftndir) + + tmpdir = "{}/tmp".format(builddir) # Run w3_setup to create wwatch3.env file env_file = os.path.join(bindir, "wwatch3.env") @@ -77,7 +83,7 @@ def buildlib(bldroot, installpath, case): y """.format(sf90, scc, tmpdir)) - run_bld_cmd_ensure_logging("./w3_setup {} -s E3SM < w3_setup.inp".format(modeldir), logger, from_dir=bindir) + run_bld_cmd_ensure_logging("./w3_setup {} -s E3SM < w3_setup.inp".format(builddir), logger, from_dir=bindir) os.remove(inp_file) # Generate pre-processed WW3 source code @@ -85,7 +91,7 @@ y for exe in ww3_exe: run_bld_cmd_ensure_logging("./w3_source {}".format(exe), logger, from_dir=bindir) for exe in ww3_exe: - tarfile = "{}/work/{}.tar.gz".format(modeldir,exe) + tarfile = "{}/work/{}.tar.gz".format(builddir,exe) shutil.move(tarfile, tmpdir) run_bld_cmd_ensure_logging("tar -xzvf {}.tar.gz".format(exe), logger, from_dir=tmpdir) run_bld_cmd_ensure_logging("rm ww3_shel.F90", logger, from_dir=tmpdir) diff --git a/components/ww3/cime_config/buildnml b/components/ww3/cime_config/buildnml index 8e7cd2ea84e..c43a521c29c 100755 --- a/components/ww3/cime_config/buildnml +++ b/components/ww3/cime_config/buildnml @@ -55,7 +55,8 @@ def buildnml(case, caseroot, compname): "wQU225EC60to30sp50x36", "wQU225EC30to60E2r2sp50x36", "wQU225EC30to60E2r2sp36x36", - "wQU225EC30to60E2r2sp25x36") + "wQU225EC30to60E2r2sp25x36", + "wQU225Icos30E3r5sp36x36") expect((wav_grid+wav_spec) in wav_grid_supported, "Combination of WAV_GRID {} and WAV_SPEC {} is not supported in ww3. Choose from: '{}'".format(wav_grid,wav_spec,wav_grid_supported) ) #-------------------------------------------------------------------- @@ -64,10 +65,10 @@ def buildnml(case, caseroot, compname): with open(os.path.join(casebuild, "ww3.input_data_list"), "w") as input_list: - input_list.write("mesh = {}/wav/ww3/{}.msh\n".format(din_loc_root,wav_grid)) + input_list.write("mesh = {}/wav/ww3/{}_rtd.msh\n".format(din_loc_root,wav_grid)) input_list.write("stations = {}/wav/ww3/stations.txt\n".format(din_loc_root)) - input_list.write("uostfilelocal = {}/wav/ww3/obstructions_local.{}{}.in\n".format(din_loc_root,wav_grid,wav_spec)) - input_list.write("uostfileshadow = {}/wav/ww3/obstructions_shadow.{}{}.in\n".format(din_loc_root,wav_grid,wav_spec)) + input_list.write("uostfilelocal = {}/wav/ww3/obstructions_local.{}{}.rtd.in\n".format(din_loc_root,wav_grid,wav_spec)) + input_list.write("uostfileshadow = {}/wav/ww3/obstructions_shadow.{}{}.rtd.in\n".format(din_loc_root,wav_grid,wav_spec)) #-------------------------------------------------------------------- # Invoke ww3 build-namelist - output will go in $CASEBUILD/ww3conf diff --git a/driver-mct/cime_config/config_component.xml b/driver-mct/cime_config/config_component.xml index 8cddad0abf8..cdfca40befa 100644 --- a/driver-mct/cime_config/config_component.xml +++ b/driver-mct/cime_config/config_component.xml @@ -2628,7 +2628,7 @@ char - netcdf,pnetcdf,netcdf4p,netcdf4c,adios,hdf5,default + netcdf,pnetcdf,netcdf4p,netcdf4c,adios,adiosc,hdf5,default run_pio env_run.xml pio io type diff --git a/driver-moab/cime_config/config_component.xml b/driver-moab/cime_config/config_component.xml index d3685126c61..ac9e46a476e 100644 --- a/driver-moab/cime_config/config_component.xml +++ b/driver-moab/cime_config/config_component.xml @@ -2620,7 +2620,7 @@ char - netcdf,pnetcdf,netcdf4p,netcdf4c,adios,hdf5,default + netcdf,pnetcdf,netcdf4p,netcdf4c,adios,adiosc,hdf5,default run_pio env_run.xml pio io type diff --git a/driver-moab/main/cime_comp_mod.F90 b/driver-moab/main/cime_comp_mod.F90 index 265f8e11338..59fa2daf408 100644 --- a/driver-moab/main/cime_comp_mod.F90 +++ b/driver-moab/main/cime_comp_mod.F90 @@ -2896,10 +2896,12 @@ subroutine cime_run() num_moab_exports = num_moab_exports + 1! this is moab clock used for debugging call seq_timemgr_clockAdvance( seq_SyncClock, force_stop, force_stop_ymd, force_stop_tod) call seq_timemgr_EClockGetData(EClock_d, stepno=cur_step_no) +#ifdef MOABDEBUG if (iamroot_CPLID) then write(logunit,*) ' num_moab_exports , cur_step_no ',num_moab_exports, cur_step_no call shr_sys_flush(logunit) endif +#endif call seq_timemgr_EClockGetData( EClock_d, curr_ymd=ymd, curr_tod=tod) call shr_cal_date2ymd(ymd,year,month,day) stop_alarm = seq_timemgr_alarmIsOn(EClock_d,seq_timemgr_alarm_stop) diff --git a/driver-moab/main/component_mod.F90 b/driver-moab/main/component_mod.F90 index 88d152a9568..3e8ba9042a5 100644 --- a/driver-moab/main/component_mod.F90 +++ b/driver-moab/main/component_mod.F90 @@ -748,6 +748,7 @@ subroutine component_init_areacor_moab (comp, mbccid, mbcxid, seq_flds_c2x_fluxe lsize = comp(1)%mblsize allocate(areas (lsize, 3)) ! lsize is along grid; read mask too allocate(factors (lsize, 2)) + factors = 1.0 ! initialize with 1.0 all factors; then maybe correct them ! get areas tagname='area:aream:mask'//C_NULL_CHAR arrsize = 3 * lsize diff --git a/driver-moab/main/prep_lnd_mod.F90 b/driver-moab/main/prep_lnd_mod.F90 index 44ae5490e2c..7bdb12376a3 100644 --- a/driver-moab/main/prep_lnd_mod.F90 +++ b/driver-moab/main/prep_lnd_mod.F90 @@ -1,6 +1,6 @@ module prep_lnd_mod - use shr_kind_mod , only: r8 => SHR_KIND_R8 + use shr_kind_mod , only: R8 => SHR_KIND_R8 use shr_kind_mod , only: cs => SHR_KIND_CS use shr_kind_mod , only: cl => SHR_KIND_CL use shr_kind_mod , only: cxx => SHR_KIND_CXX @@ -162,7 +162,7 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln integer nrflds ! number of rof fields projected on land integer arrsize ! for setting the r2x fields on land to 0 integer ent_type ! for setting tags - real (kind=r8) , allocatable :: tmparray (:) ! used to set the r2x fields to 0 + real (kind=R8) , allocatable :: tmparray (:) ! used to set the r2x fields to 0 #endif character(*), parameter :: subname = '(prep_lnd_init)' @@ -222,7 +222,6 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln call seq_map_init_rcfile(mapper_Fr2l, rof(1), lnd(1), & 'seq_maps.rc','rof2lnd_fmapname:','rof2lnd_fmaptype:',samegrid_lr, & string='mapper_Fr2l initialization',esmf_map=esmf_map_flag) - end if ! symmetric of l2r, from prep_rof #ifdef HAVE_MOAB ! Call moab intx only if land and river are init in moab @@ -370,7 +369,7 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln arrsize = nrflds*mlsize allocate (tmparray(arrsize)) ! mlsize is the size of local land ! do we need to zero out others or just river ? - tmparray = 0._r8 + tmparray = 0._R8 ierr = iMOAB_SetDoubleTagStorage(mblxid, tagname, arrsize , ent_type, tmparray) if (ierr .ne. 0) then write(logunit,*) subname,' cant zero out r2x tags on land' @@ -381,6 +380,7 @@ subroutine prep_lnd_init(infodata, atm_c2_lnd, rof_c2_lnd, glc_c2_lnd, iac_c2_ln end if ! if ((mbrxid .ge. 0) .and. (mblxid .ge. 0)) ! endif HAVE_MOAB #endif + end if call shr_sys_flush(logunit) if (atm_c2_lnd) then @@ -693,7 +693,7 @@ subroutine prep_lnd_mrg_moab (infodata) #endif #ifdef MOABCOMP character(CXX) :: tagname, mct_field - real(r8) :: difference + real(R8) :: difference type(mct_list) :: temp_list integer :: size_list, index_list, ent_type type(mct_string) :: mctOStr ! diff --git a/driver-moab/main/prep_ocn_mod.F90 b/driver-moab/main/prep_ocn_mod.F90 index 44ec0c6a3c7..b13cce62f95 100644 --- a/driver-moab/main/prep_ocn_mod.F90 +++ b/driver-moab/main/prep_ocn_mod.F90 @@ -1,6 +1,6 @@ module prep_ocn_mod - use shr_kind_mod, only: r8 => SHR_KIND_R8 + use shr_kind_mod, only: R8 => SHR_KIND_R8 use shr_kind_mod, only: cs => SHR_KIND_CS use shr_kind_mod, only: cl => SHR_KIND_CL use shr_kind_mod, only: CX => shr_kind_CX, CXX => shr_kind_CXX @@ -137,7 +137,7 @@ module prep_ocn_mod integer , target :: x2oacc_ox_cnt ! x2oacc_ox: number of time samples accumulated ! accumulation variables for moab data - real (kind=r8) , allocatable, private, target :: x2oacc_om (:,:) ! Ocn import, ocn grid, cpl pes, moab array + real (kind=R8) , allocatable, private, target :: x2oacc_om (:,:) ! Ocn import, ocn grid, cpl pes, moab array integer , target :: x2oacc_om_cnt ! x2oacc_ox: number of time samples accumulated, in moab array integer :: arrSize_x2o_om ! this will be a module variable, size moabLocal_size * nof @@ -154,20 +154,20 @@ module prep_ocn_mod !================================================================================================ - real (kind=r8) , allocatable, private :: fractions_om (:,:) ! will retrieve the fractions from ocean, and use them + real (kind=R8) , allocatable, private :: fractions_om (:,:) ! will retrieve the fractions from ocean, and use them ! they were init with ! character(*),parameter :: fraclist_o = 'afrac:ifrac:ofrac:ifrad:ofrad' in moab, on the fractions - real (kind=r8) , allocatable, private :: x2o_om (:,:) - real (kind=r8) , allocatable, private :: a2x_om (:,:) - real (kind=r8) , allocatable, private :: i2x_om (:,:) - real (kind=r8) , allocatable, private :: r2x_om (:,:) - real (kind=r8) , allocatable, private :: xao_om (:,:) + real (kind=R8) , allocatable, private :: x2o_om (:,:) + real (kind=R8) , allocatable, private :: a2x_om (:,:) + real (kind=R8) , allocatable, private :: i2x_om (:,:) + real (kind=R8) , allocatable, private :: r2x_om (:,:) + real (kind=R8) , allocatable, private :: xao_om (:,:) ! this will be constructed first time, and be used to copy fields for shared indices ! between xao and x2o character(CXX) :: shared_fields_xao_x2o ! will need some array to hold the data for copying - real(r8) , allocatable, save :: shared_values(:) ! will be the size of shared indices * lsize + real(R8) , allocatable, save :: shared_values(:) ! will be the size of shared indices * lsize integer :: size_of_shared_values logical :: iamin_CPLALLICEID ! pe associated with CPLALLICEID @@ -268,7 +268,7 @@ subroutine prep_ocn_init(infodata, atm_c2_ocn, atm_c2_ice, ice_c2_ocn, rof_c2_oc integer arrsize ! for setting the r2x fields on land to 0 integer ent_type ! for setting tags integer noflds ! used for number of fields in allocating moab accumulated array x2oacc_om - real (kind=r8) , allocatable :: tmparray (:) ! used to set the r2x fields to 0 + real (kind=R8) , allocatable :: tmparray (:) ! used to set the r2x fields to 0 !--------------------------------------------------------------- @@ -786,7 +786,7 @@ subroutine prep_ocn_init(infodata, atm_c2_ocn, atm_c2_ice, ice_c2_ocn, rof_c2_oc arrsize = nrflds*mlsize allocate (tmparray(arrsize)) ! mlsize is the size of local land ! do we need to zero out others or just river ? - tmparray = 0._r8 + tmparray = 0._R8 ierr = iMOAB_SetDoubleTagStorage(mboxid, tagname, arrsize , ent_type, tmparray) if (ierr .ne. 0) then write(logunit,*) subname,' cant zero out r2x tags on ocn' @@ -1095,7 +1095,7 @@ subroutine prep_ocn_mrg(infodata, fractions_ox, xao_ox, timer_mrg) ! ! Local Variables integer :: eii, ewi, egi, eoi, eai, eri, exi, efi, emi - real(r8) :: flux_epbalfact ! adjusted precip factor + real(R8) :: flux_epbalfact ! adjusted precip factor type(mct_avect), pointer :: x2o_ox integer :: cnt character(*), parameter :: subname = '(prep_ocn_mrg)' @@ -1177,7 +1177,7 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) !--------------------------------------------------------------- - real(r8) :: flux_epbalfact ! adjusted precip factor + real(R8) :: flux_epbalfact ! adjusted precip factor ! will build x2o_om , similar to x2o_ox ! no averages, just one ocn instance @@ -1187,11 +1187,11 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) integer :: kof,kif integer :: lsize, arrsize ! for double arrays integer , save :: noflds,naflds,niflds,nrflds,nxflds! ,ngflds,nwflds, no glacier or wave model - real(r8) :: ifrac,ifracr - real(r8) :: afrac,afracr - real(r8) :: frac_sum - real(r8) :: avsdr, anidr, avsdf, anidf ! albedos - real(r8) :: fswabsv, fswabsi ! sw + real(R8) :: ifrac,ifracr + real(R8) :: afrac,afracr + real(R8) :: frac_sum + real(R8) :: avsdr, anidr, avsdf, anidf ! albedos + real(R8) :: fswabsv, fswabsi ! sw character(CL),allocatable :: field_ocn(:) ! string converted to char character(CL),allocatable :: field_atm(:) ! string converted to char character(CL),allocatable :: field_ice(:) ! string converted to char @@ -1289,7 +1289,7 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) #endif #ifdef MOABCOMP character(CXX) :: mct_field - real(r8) :: difference + real(R8) :: difference type(mct_list) :: temp_list integer :: size_list, index_list type(mct_string) :: mctOStr ! @@ -1345,7 +1345,7 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) allocate(a2x_om (lsize, naflds)) allocate(i2x_om (lsize, niflds)) allocate(r2x_om (lsize, nrflds)) - r2x_om = 0._r8 ! should we zero out all of them ? + r2x_om = 0._R8 ! should we zero out all of them ? allocate(xao_om (lsize, nxflds)) ! allocate fractions too ! use the fraclist fraclist_o = 'afrac:ifrac:ofrac:ifrad:ofrad' @@ -1769,7 +1769,7 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) ifrac = fractions_om(n,kif) ! fo_kif_ifrac(n) ! fractions_o%rAttr(kif,n) afrac = fractions_om(n,kof) ! fo_kof_ofrac(n) ! fractions_o%rAttr(kof,n) frac_sum = ifrac + afrac - if ((frac_sum) /= 0._r8) then + if ((frac_sum) /= 0._R8) then ifrac = ifrac / (frac_sum) afrac = afrac / (frac_sum) endif @@ -1777,7 +1777,7 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) ifracr = fractions_om(n,kir) ! fo_kir_ifrad(n) ! fractions_o%rAttr(kir,n) afracr = fractions_om(n,kor) ! fo_kor_ofrad(n) ! fractions_o%rAttr(kor,n) frac_sum = ifracr + afracr - if ((frac_sum) /= 0._r8) then + if ((frac_sum) /= 0._R8) then ifracr = ifracr / (frac_sum) afracr = afracr / (frac_sum) endif @@ -1913,7 +1913,7 @@ subroutine prep_ocn_mrg_moab(infodata, xao_ox) ifrac = fractions_om(n,kif) !fo_kif_ifrac(n) ! fractions_o%rAttr(kif) afrac = fractions_om(n,kof) ! fo_kof_ofrac(n) ! fractions_o%rAttr(kof,n) frac_sum = ifrac + afrac - if ((frac_sum) /= 0._r8) then + if ((frac_sum) /= 0._R8) then ifrac = ifrac / (frac_sum) afrac = afrac / (frac_sum) endif @@ -2046,7 +2046,7 @@ subroutine prep_ocn_merge( flux_epbalfact, a2x_o, i2x_o, r2x_o, w2x_o, g2x_o, xa !----------------------------------------------------------------------- ! ! Arguments - real(r8) , intent(in) :: flux_epbalfact + real(R8) , intent(in) :: flux_epbalfact type(mct_aVect), intent(in) :: a2x_o type(mct_aVect), intent(in) :: i2x_o type(mct_aVect), intent(in) :: r2x_o @@ -2061,11 +2061,11 @@ subroutine prep_ocn_merge( flux_epbalfact, a2x_o, i2x_o, r2x_o, w2x_o, g2x_o, xa integer :: kof,kif integer :: lsize integer :: noflds,naflds,niflds,nrflds,nwflds,nxflds,ngflds - real(r8) :: ifrac,ifracr - real(r8) :: afrac,afracr - real(r8) :: frac_sum - real(r8) :: avsdr, anidr, avsdf, anidf ! albedos - real(r8) :: fswabsv, fswabsi ! sw + real(R8) :: ifrac,ifracr + real(R8) :: afrac,afracr + real(R8) :: frac_sum + real(R8) :: avsdr, anidr, avsdf, anidf ! albedos + real(R8) :: fswabsv, fswabsi ! sw character(CL),allocatable :: field_ocn(:) ! string converted to char character(CL),allocatable :: field_atm(:) ! string converted to char character(CL),allocatable :: field_ice(:) ! string converted to char @@ -2584,7 +2584,7 @@ subroutine prep_ocn_merge( flux_epbalfact, a2x_o, i2x_o, r2x_o, w2x_o, g2x_o, xa ifrac = fractions_o%rAttr(kif,n) afrac = fractions_o%rAttr(kof,n) frac_sum = ifrac + afrac - if ((frac_sum) /= 0._r8) then + if ((frac_sum) /= 0._R8) then ifrac = ifrac / (frac_sum) afrac = afrac / (frac_sum) endif @@ -2592,7 +2592,7 @@ subroutine prep_ocn_merge( flux_epbalfact, a2x_o, i2x_o, r2x_o, w2x_o, g2x_o, xa ifracr = fractions_o%rAttr(kir,n) afracr = fractions_o%rAttr(kor,n) frac_sum = ifracr + afracr - if ((frac_sum) /= 0._r8) then + if ((frac_sum) /= 0._R8) then ifracr = ifracr / (frac_sum) afracr = afracr / (frac_sum) endif @@ -2738,7 +2738,7 @@ subroutine prep_ocn_merge( flux_epbalfact, a2x_o, i2x_o, r2x_o, w2x_o, g2x_o, xa ifrac = fractions_o%rAttr(kif,n) afrac = fractions_o%rAttr(kof,n) frac_sum = ifrac + afrac - if ((frac_sum) /= 0._r8) then + if ((frac_sum) /= 0._R8) then ifrac = ifrac / (frac_sum) afrac = afrac / (frac_sum) endif @@ -3068,7 +3068,7 @@ function prep_ocn_get_mapper_Sw2o() prep_ocn_get_mapper_Sw2o => mapper_Sw2o end function prep_ocn_get_mapper_Sw2o function prep_ocn_get_x2oacc_om() - real(r8), DIMENSION(:, :), pointer :: prep_ocn_get_x2oacc_om + real(R8), DIMENSION(:, :), pointer :: prep_ocn_get_x2oacc_om prep_ocn_get_x2oacc_om => x2oacc_om end function prep_ocn_get_x2oacc_om function prep_ocn_get_x2oacc_om_cnt() diff --git a/driver-moab/main/prep_rof_mod.F90 b/driver-moab/main/prep_rof_mod.F90 index 59cc61c5889..83a8e331e91 100644 --- a/driver-moab/main/prep_rof_mod.F90 +++ b/driver-moab/main/prep_rof_mod.F90 @@ -1,7 +1,7 @@ module prep_rof_mod #include "shr_assert.h" - use shr_kind_mod, only: r8 => SHR_KIND_R8 + use shr_kind_mod, only: R8 => SHR_KIND_R8 use shr_kind_mod, only: cs => SHR_KIND_CS use shr_kind_mod, only: cl => SHR_KIND_CL use shr_kind_mod, only: cxx => SHR_KIND_CXX @@ -112,22 +112,22 @@ module prep_rof_mod ! accumulation variables over moab fields character(CXX) :: sharedFieldsLndRof ! used in moab to define l2racc_lm - real (kind=r8) , allocatable, private, target :: l2racc_lm(:,:) ! lnd export, lnd grid, cpl pes - real (kind=r8) , allocatable, private :: l2x_lm2(:,:) ! basically l2x_lm, but in another copy, on rof module + real (kind=R8) , allocatable, private, target :: l2racc_lm(:,:) ! lnd export, lnd grid, cpl pes + real (kind=R8) , allocatable, private :: l2x_lm2(:,:) ! basically l2x_lm, but in another copy, on rof module integer , target :: l2racc_lm_cnt ! l2racc_lm: number of time samples accumulated integer :: nfields_sh_lr ! number of fields in sharedFieldsLndRof integer :: lsize_lm ! size of land in moab, local character(CXX) :: sharedFieldsAtmRof ! used in moab to define a2racc_am - real (kind=r8) , allocatable, private :: a2racc_am(:,:) ! atm export, atm grid, cpl pes - real (kind=r8) , allocatable, private :: a2x_am2(:,:) ! basically a2x_am, but in another copy, on rof module + real (kind=R8) , allocatable, private :: a2racc_am(:,:) ! atm export, atm grid, cpl pes + real (kind=R8) , allocatable, private :: a2x_am2(:,:) ! basically a2x_am, but in another copy, on rof module integer , target :: a2racc_am_cnt ! a2racc_am: number of time samples accumulated integer :: nfields_sh_ar ! number of fields in sharedFieldsAtmRof integer :: lsize_am ! size of atm in moab, local character(CXX) :: sharedFieldsOcnRof ! used in moab to define o2racc_om - real (kind=r8) , allocatable, private, target :: o2racc_om(:,:) ! ocn export, ocn grid, cpl pes - real (kind=r8) , allocatable, private :: o2r_om2(:,:) ! basically o2x_om, but in another copy, on rof module, only shared with rof + real (kind=R8) , allocatable, private, target :: o2racc_om(:,:) ! ocn export, ocn grid, cpl pes + real (kind=R8) , allocatable, private :: o2r_om2(:,:) ! basically o2x_om, but in another copy, on rof module, only shared with rof integer , target :: o2racc_om_cnt ! o2racc_om: number of time samples accumulated integer :: nfields_sh_or ! number of fields in sharedFieldsOcnRof integer :: lsize_om ! size of ocn in moab, local @@ -145,12 +145,12 @@ module prep_rof_mod logical :: samegrid_al ! samegrid atm and lnd ! moab stuff - real (kind=r8) , allocatable, private :: fractions_rm (:,:) ! will retrieve the fractions from rof, and use them + real (kind=R8) , allocatable, private :: fractions_rm (:,:) ! will retrieve the fractions from rof, and use them ! they were init with ! character(*),parameter :: fraclist_r = 'lfrac:lfrin:rfrac' in moab, on the fractions - real (kind=r8) , allocatable, private :: x2r_rm (:,:) ! result of merge - real (kind=r8) , allocatable, private :: a2x_rm (:,:) - real (kind=r8) , allocatable, private :: l2x_rm (:,:) + real (kind=R8) , allocatable, private :: x2r_rm (:,:) ! result of merge + real (kind=R8) , allocatable, private :: a2x_rm (:,:) + real (kind=R8) , allocatable, private :: l2x_rm (:,:) !================================================================================================ @@ -327,6 +327,7 @@ subroutine prep_rof_init(infodata, lnd_c2_rof, atm_c2_rof, ocn_c2_rof) write(logunit,*) subname,' error in defining tags for seq_flds_a2x_fields on rof cpl' call shr_sys_abort(subname//' ERROR in defining tags for seq_flds_a2x_fields on rof cpl') endif + call seq_comm_getData(CPLID ,mpigrp=mpigrp_CPLID) if (samegrid_lr) then ! the same mesh , lnd and rof use the same dofs, but restricted ! we do not compute intersection, so we will have to just send data from lnd to rof and viceversa, by GLOBAL_ID matching @@ -363,7 +364,6 @@ subroutine prep_rof_init(infodata, lnd_c2_rof, atm_c2_rof, ocn_c2_rof) ! we also need to compute the comm graph for the second hop, from the lnd on coupler to the ! lnd for the intx lnd-rof context (coverage) ! - call seq_comm_getData(CPLID ,mpigrp=mpigrp_CPLID) type1 = 3 ! land is FV now on coupler side type2 = 3; @@ -1223,7 +1223,7 @@ subroutine prep_rof_merge(l2x_r, a2x_r, fractions_r, x2r_r, cime_model,o2x_r) integer, save :: index_x2r_coszen_str integer, save :: index_frac - real(r8) :: frac + real(R8) :: frac character(CL) :: fracstr logical, save :: first_time = .true. logical, save :: flds_wiso_rof = .false. @@ -1529,7 +1529,7 @@ subroutine prep_rof_mrg_moab (infodata, cime_model) integer, save :: index_x2r_coszen_str integer, save :: index_frac - real(r8) :: frac + real(R8) :: frac character(CL) :: fracstr logical, save :: first_time = .true. logical, save :: flds_wiso_rof = .false. @@ -1548,7 +1548,7 @@ subroutine prep_rof_mrg_moab (infodata, cime_model) character*32 :: outfile, wopts, lnum #endif #ifdef MOABCOMP - real(r8) :: difference + real(R8) :: difference type(mct_list) :: temp_list integer :: size_list, index_list type(mct_string) :: mctOStr ! @@ -1995,7 +1995,7 @@ function prep_rof_get_o2racc_om_cnt() end function prep_rof_get_o2racc_om_cnt function prep_rof_get_o2racc_om() - real(r8), DIMENSION(:, :), pointer :: prep_rof_get_o2racc_om + real(R8), DIMENSION(:, :), pointer :: prep_rof_get_o2racc_om prep_rof_get_o2racc_om => o2racc_om end function prep_rof_get_o2racc_om @@ -2011,7 +2011,7 @@ function prep_rof_get_l2racc_lm_cnt() end function prep_rof_get_l2racc_lm_cnt function prep_rof_get_l2racc_lm() - real(r8), DIMENSION(:, :), pointer :: prep_rof_get_l2racc_lm + real(R8), DIMENSION(:, :), pointer :: prep_rof_get_l2racc_lm prep_rof_get_l2racc_lm => l2racc_lm end function prep_rof_get_l2racc_lm diff --git a/driver-moab/main/seq_frac_mct.F90 b/driver-moab/main/seq_frac_mct.F90 index a89fc0b9aa4..031845a102e 100644 --- a/driver-moab/main/seq_frac_mct.F90 +++ b/driver-moab/main/seq_frac_mct.F90 @@ -180,7 +180,7 @@ module seq_frac_mct use iMOAB, only : iMOAB_DefineTagStorage, iMOAB_SetDoubleTagStorage, & iMOAB_GetMeshInfo, iMOAB_SetDoubleTagStorageWithGid, iMOAB_WriteMesh, & - iMOAB_ApplyScalarProjectionWeights, iMOAB_SendElementTag, iMOAB_ReceiveElementTag, & + iMOAB_SendElementTag, iMOAB_ReceiveElementTag, & iMOAB_FreeSenderBuffers, iMOAB_GetDoubleTagStorage #ifdef MOABDEBUG use seq_comm_mct, only : num_moab_exports diff --git a/externals/ekat b/externals/ekat index 4e36a487d9c..1d441b22df3 160000 --- a/externals/ekat +++ b/externals/ekat @@ -1 +1 @@ -Subproject commit 4e36a487d9ced3951907b36a1a16b13325d796d1 +Subproject commit 1d441b22df3e4f8f8b3ea96099b0e848eb74afd7 diff --git a/externals/mam4xx b/externals/mam4xx index aae46807bf5..fdbb0816c5c 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit aae46807bf58d6ffcbc6db620c27568450b2d040 +Subproject commit fdbb0816c5c0c541265ec17f544908da935d8af6 diff --git a/externals/scorpio b/externals/scorpio index 804f87c48fc..59601f625e3 160000 --- a/externals/scorpio +++ b/externals/scorpio @@ -1 +1 @@ -Subproject commit 804f87c48fc9013009307b806d1aff2afa3a0d7c +Subproject commit 59601f625e34019de0e1d3411d07bdc9e70d0054 diff --git a/share/build/buildlib.spio b/share/build/buildlib.spio index d17627c4323..064fa751e4a 100755 --- a/share/build/buildlib.spio +++ b/share/build/buildlib.spio @@ -78,6 +78,8 @@ def buildlib(bldroot, installpath, case): # ADIOS2_ROOT. This is a workaround if "ADIOS2_ROOT" in os.environ: os.environ["ADIOS2_DIR"] = os.environ["ADIOS2_ROOT"] + if "BLOSC2_ROOT" in os.environ: + os.environ["Blosc2_DIR"] = os.environ["BLOSC2_ROOT"] # If variable PIO_VERSION_MAJOR is defined in the environment then # we assume that PIO is installed on the system @@ -105,10 +107,14 @@ def buildlib(bldroot, installpath, case): else: cmake_opts += f"-DGENF90_PATH={scorpio_src_dir}/src/genf90 " + adiosc_found = False if "ADIOS2_ROOT" in os.environ: cmake_opts += "-DWITH_ADIOS2:BOOL=ON " if "FROM_CREATE_TEST" in os.environ and os.environ["FROM_CREATE_TEST"] == "True": cmake_opts += "-DADIOS_BP2NC_TEST:BOOL=ON " + if "BLOSC2_ROOT" in os.environ: + cmake_opts += "-DADIOS_USE_COMPRESSION:BOOL=ON " + adiosc_found = True if debug: cmake_opts += "-DPIO_ENABLE_LOGGING=ON " @@ -281,6 +287,8 @@ def buildlib(bldroot, installpath, case): valid_values += ",netcdf4p,netcdf4c" if adios_found: valid_values += ",adios" + if adiosc_found: + valid_values += ",adiosc" if hdf5_found: valid_values += ",hdf5" diff --git a/share/util/shr_pio_mod.F90 b/share/util/shr_pio_mod.F90 index d500cbc680a..11cefd725b5 100644 --- a/share/util/shr_pio_mod.F90 +++ b/share/util/shr_pio_mod.F90 @@ -679,6 +679,8 @@ subroutine shr_pio_getiotypefromname(typename, iotype, defaulttype) #ifndef PIO1 else if ( typename .eq. 'ADIOS') then iotype = pio_iotype_adios + else if ( typename .eq. 'ADIOSC') then + iotype = pio_iotype_adiosc else if ( typename .eq. 'HDF5') then iotype = pio_iotype_hdf5 #endif