From 6c5b9bb4d51d5189a5ca7e78d42b4a316957eb4f Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Mon, 9 Dec 2024 14:33:58 -0500 Subject: [PATCH] Update HPC Tier Definitions (#3138) Updates the tier definitions for supported platforms. This also updates and reformats the README. ~Note that the README badges will display errors until this PR is merged.~ @TerrenceMcGuinness-NOAA updated the gist, so the badges are now showing up correctly in the README. Resolves #3135 --- .../workflows/{orion.yaml => hercules.yaml} | 20 +- .github/workflows/wcoss2.yaml | 81 ++++++++ README.md | 44 +++-- docs/source/components.rst | 1 + docs/source/hpc.rst | 174 ++++++++++++++++++ docs/source/index.rst | 1 + 6 files changed, 296 insertions(+), 25 deletions(-) rename .github/workflows/{orion.yaml => hercules.yaml} (75%) create mode 100644 .github/workflows/wcoss2.yaml diff --git a/.github/workflows/orion.yaml b/.github/workflows/hercules.yaml similarity index 75% rename from .github/workflows/orion.yaml rename to .github/workflows/hercules.yaml index aaf1e28370..a08ec867b6 100644 --- a/.github/workflows/orion.yaml +++ b/.github/workflows/hercules.yaml @@ -1,4 +1,4 @@ -name: Orion +name: Hercules on: pull_request_target: @@ -26,7 +26,7 @@ jobs: echo "labels=$LABELS" >> $GITHUB_OUTPUT passed: - if: contains( needs.getlabels.outputs.labels, 'CI-Orion-Passed') && github.event.pull_request.merged + if: contains( needs.getlabels.outputs.labels, 'CI-Hercules-Passed') && github.event.pull_request.merged runs-on: ubuntu-22.04 needs: - getlabels @@ -38,13 +38,13 @@ jobs: forceUpdate: true auth: ${{ secrets.CLI_DYNAMIC_BADGES }} gistID: e35aa2904a54deae6bbb1fdc2d960c71 - filename: orion.json - label: orion + filename: hercules.json + label: hercules message: passing color: green failed: - if: contains( needs.getlabels.outputs.labels, 'CI-Orion-Failed') && github.event.pull_request.merged + if: contains( needs.getlabels.outputs.labels, 'CI-Hercules-Failed') && github.event.pull_request.merged runs-on: ubuntu-latest needs: - getlabels @@ -56,14 +56,14 @@ jobs: forceUpdate: true auth: ${{ secrets.CLI_DYNAMIC_BADGES }} gistID: e35aa2904a54deae6bbb1fdc2d960c71 - filename: orion.json - label: orion + filename: hercules.json + label: hercules message: failing color: red pending: - if: "!contains( needs.getlabels.outputs.labels, 'CI-Orion-Passed') && !contains( needs.getlabels.outputs.labels, 'CI-Orion-Failed')" + if: "!contains( needs.getlabels.outputs.labels, 'CI-Hercules-Passed') && !contains( needs.getlabels.outputs.labels, 'CI-Hercules-Failed')" runs-on: ubuntu-latest needs: - getlabels @@ -75,7 +75,7 @@ jobs: forceUpdate: true auth: ${{ secrets.CLI_DYNAMIC_BADGES }} gistID: e35aa2904a54deae6bbb1fdc2d960c71 - filename: orion.json - label: orion + filename: hercules.json + label: hercules message: pending color: orange diff --git a/.github/workflows/wcoss2.yaml b/.github/workflows/wcoss2.yaml new file mode 100644 index 0000000000..489ae58406 --- /dev/null +++ b/.github/workflows/wcoss2.yaml @@ -0,0 +1,81 @@ +name: WCOSS2 + +on: + pull_request_target: + branches: + - develop + types: [closed] + +jobs: + + getlabels: + runs-on: ubuntu-22.04 + outputs: + labels: ${{ steps.id.outputs.labels }} + steps: + - name: Get Label Steps + id: id + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OWNER: ${{ github.repository_owner }} + REPO_NAME: ${{ github.event.repository.name }} + PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} + run: | + LABELS1="$(gh api repos/$OWNER/$REPO_NAME/pulls/$PULL_REQUEST_NUMBER --jq '.labels.[].name')" + LABELS=$(echo "$LABELS1" | tr '\n' ' ') + echo "labels=$LABELS" >> $GITHUB_OUTPUT + + passed: + if: contains( needs.getlabels.outputs.labels, 'CI-Wcoss2-Passed') && github.event.pull_request.merged + runs-on: ubuntu-latest + needs: + - getlabels + + steps: + - name: Passed + uses: schneegans/dynamic-badges-action@v1.6.0 + with: + forceUpdate: true + auth: ${{ secrets.CLI_DYNAMIC_BADGES }} + gistID: e35aa2904a54deae6bbb1fdc2d960c71 + filename: wcoss2.json + label: wcoss2 + message: passing + color: green + + failed: + if: contains( needs.getlabels.outputs.labels, 'CI-Wcoss2-Failed') && github.event.pull_request.merged + runs-on: ubuntu-latest + needs: + - getlabels + + steps: + - name: Failed + uses: schneegans/dynamic-badges-action@v1.6.0 + with: + forceUpdate: true + auth: ${{ secrets.CLI_DYNAMIC_BADGES }} + gistID: e35aa2904a54deae6bbb1fdc2d960c71 + filename: wcoss2.json + label: wcoss2 + message: failing + color: red + + + pending: + if: "!contains( needs.getlabels.outputs.labels, 'CI-Wcoss2-Passed') && !contains( needs.getlabels.outputs.labels, 'CI-Wcoss2-Failed')" + runs-on: ubuntu-latest + needs: + - getlabels + + steps: + - name: Pending + uses: schneegans/dynamic-badges-action@v1.6.0 + with: + forceUpdate: true + auth: ${{ secrets.CLI_DYNAMIC_BADGES }} + gistID: e35aa2904a54deae6bbb1fdc2d960c71 + filename: wcoss2.json + label: wcoss2 + message: pending + color: orange diff --git a/README.md b/README.md index 289e74933b..3311f540a5 100644 --- a/README.md +++ b/README.md @@ -3,28 +3,42 @@ [![pynorms](https://github.com/NOAA-EMC/global-workflow/actions/workflows/pynorms.yaml/badge.svg)](https://github.com/NOAA-EMC/global-workflow/actions/workflows/pynorms.yaml) ![Custom badge](https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/emcbot/e35aa2904a54deae6bbb1fdc2d960c71/raw/hera.json) -![Custom badge](https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/emcbot/e35aa2904a54deae6bbb1fdc2d960c71/raw/orion.json) +![Custom badge](https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/emcbot/e35aa2904a54deae6bbb1fdc2d960c71/raw/hercules.json) +![Custom badge](https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/emcbot/e35aa2904a54deae6bbb1fdc2d960c71/raw/wcoss2.json) -# global-workflow -Global Workflow currently supporting the Global Forecast System (GFS) with the [UFS-weather-model](https://github.com/ufs-community/ufs-weather-model) and [GSI](https://github.com/NOAA-EMC/GSI)-based Data Assimilation System. +The Global Workflow supporting the Global Forecast System (GFS), the Global Ensemble Forecasting System (GEFS), and the Seasonal Forecast System (SFS) with the [UFS-weather-model](https://github.com/ufs-community/ufs-weather-model). Data assimilation, currently only available for the GFS, is provides by both the [GSI](https://github.com/NOAA-EMC/GSI)- and [GDASApp (JEDI)](https://github.com/NOAA-EMC/GDASApp)-based Data Assimilation systems. -The `global-workflow` depends on the following prerequisities to be available on the system: +In progress [documentation](https://global-workflow.readthedocs.io/en/latest/) is available. -* Workflow Engine - [Rocoto](https://github.com/christopherwharrop/rocoto) and [ecFlow](https://github.com/ecmwf/ecflow) (for NWS Operations) -* Compiler - Intel Compiler Suite -* Software - NCEPLIBS (various), ESMF, HDF5, NetCDF, and a host of other softwares (see module files under /modulefiles for additional details) +# Prerequisites -The `global-workflow` current supports the following tier-1 machines: +The Global Workflow depends on the following prerequisities to be available on the system: -* NOAA RDHPCS - Hera -* MSU HPC - Orion -* MSU HPC - Hercules -* NOAA's operational HPC - WCOSS2 +* Workflow Engines - [Rocoto](https://github.com/christopherwharrop/rocoto) and [ecFlow](https://github.com/ecmwf/ecflow) (for NWS Operations) +* Compiler - Intel classic compiler suite version 2021.x +* Software - NCEPLIBS (various), ESMF, HDF5, NetCDF, and a host of other softwares (see module files under /modulefiles for additional details). + - [Spack-stack](https://github.com/JCSDA/spack-stack) is recommended for the installation of dependent libraries. -Additionally, the following tier-2 machine is supported: -* SSEC at Univ. of Wisconsin - S4 (Note that S2S+ experiments are not fully supported) +## Supported platforms -Documentation (in progress) is available [here](https://global-workflow.readthedocs.io/en/latest/). +The Global Workflow currently supports the following machines at the indicated tier. + +| HPC | Tier | Notes | +| --------------------------------------- |:----:|:--------------------------------------------------------------------------:| +| WCOSS2
NCO | 1 | GEFS testing is not regularly performed. | +| Hera
NOAA RDHPCS | 1 | | +| Hercules
MSU | 1 | Currently does not support the TC Tracker. | +| Orion
MSU | 2 | The GSI runs very slowly on Orion and the TC tracker is not supported. | +| Gaea C5/C6
RDHPCS | 3 | Currently non-operational following an OS upgrade.
Supported by EPIC. | +| AWS, GCP, Azure
NOAA Parallel Works | 3 | Supported by EPIC. | +| Jet
RDHPCS | 3 | Supported by NESDIS. | +| S4
SSEC | 3 | Currently non-operational following an OS upgrade.
Supported by NESDIS. | + +**Tier Definitions** + +1. Fully supported by the EMC global workflow team. CI testing is regularly performed on these systems, the majority of the global workflow features are supported, and the team will address any platform-specific features, bugs, upgrades, and requests for data. +2. Supported by the global workflow team on an ad-hoc basis. CI tests are supported on these systems, but not regularly performed. +3. No official support by the global workflow team, but may be supported by other entities (e.g. EPIC). # Disclaimer diff --git a/docs/source/components.rst b/docs/source/components.rst index f5a60a96af..98803c96c7 100644 --- a/docs/source/components.rst +++ b/docs/source/components.rst @@ -12,6 +12,7 @@ The major components of the system are: * Forecast * Post-processing * Verification +* Archiving The Global Workflow repository contains the workflow and script layers. External components will be checked out as git submodules. All of the submodules of the system reside in their respective repositories on GitHub. diff --git a/docs/source/hpc.rst b/docs/source/hpc.rst index e83851b1a2..8337f61886 100644 --- a/docs/source/hpc.rst +++ b/docs/source/hpc.rst @@ -44,6 +44,180 @@ The following system software requirements are the minimum for any new or existi | | | WCOSS2 | +--------------+-------------+---------------------------------------+ +=========================== +Feature availability by HPC +=========================== + +The Global Workflow provides capabilities for deterministic and ensemble forecasts along with data assimilation on multiple platforms. However, not all features are currently supported on all platforms. The following table lists the features by platform and states their level of support. + +.. list-table:: Capabilities matrix by HPC + :header-rows: 1 + :align: center + + * - HPC + - Tier + - Coupled + GFS + - Coupled + GEFS + - GSI + DA + - GDASApp + DA + - Coupled + DA + - TC Tracker + - AWIPS + - MOS + - Fit2Obs + - METplus + Verification + - HPSS + Archiving + * - WCOSS2 + - 1 + - X + - X + - X + - X + - + - X + - X + - X + - X + - X + - X + * - Hera + - 1 + - X + - X + - X + - X + - X + - X + - X + - + - X + - X + - X + * - Hercules + - 1 + - X + - X + - X + - X + - X + - + - + - + - X + - X + - + * - Orion + - 2 + - X + - X + - X + - X + - + - X + - + - + - X + - X + - + * - Gaea C5 + - 3 + - X + - X + - X + - X + - + - + - + - + - + - + - X + * - Gaea C6 + - 3 + - X + - X + - X + - X + - + - + - + - + - + - + - X + * - AWS (PW) + - 3 + - X + - X + - X + - + - + - + - + - + - + - + - + * - GCP (PW) + - 3 + - X + - X + - + - + - + - + - + - + - + - + - + * - Azure + - 3 + - X + - X + - + - + - + - + - + - + - + - + - + * - Jet + - 3 + - X + - + - X + - + - + - X + - + - + - X + - X + - X + * - S4 + - 3 + - + - + - X + - + - + - X + - + - + - X + - X + - + ================================ Experiment troubleshooting help ================================ diff --git a/docs/source/index.rst b/docs/source/index.rst index e6513b743a..aefec10676 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -18,6 +18,7 @@ Code managers * Kate Friedman - @KateFriedman-NOAA / kate.friedman@noaa.gov * Walter Kolczynski - @WalterKolczynski-NOAA / walter.kolczynski@noaa.gov +* David Huber - @DavidHuber-NOAA / david.huber@noaa.gov ============= Announcements