Merge pull request #1006 from CliMA/gb/landsea

Use higher resolution land-sea mask

.buildkite/amip/pipeline.yml

@@ -24,9 +24,6 @@ steps:
       - "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.precompile()'"
       - "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.status()'"
 
-      - echo "--- Download artifacts"
-      - "julia --project=experiments/ClimaEarth/ artifacts/download_artifacts.jl"
-
     agents:
       slurm_gpus: 1
       slurm_cpus_per_task: 8

.buildkite/benchmarks/pipeline.yml

@@ -27,10 +27,6 @@ steps:
       - "julia --project=test/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
       - "julia --project=test/ -e 'using Pkg; Pkg.precompile()'"
       - "julia --project=test/ -e 'using Pkg; Pkg.status()'"
-
-      - echo "--- Download artifacts"
-      - "julia --project=experiments/ClimaEarth artifacts/download_artifacts.jl"
-
     agents:
       slurm_gpus: 1
       slurm_cpus_per_task: 8

.buildkite/hierarchies/pipeline.yml

@@ -13,7 +13,6 @@ env:
   SLURM_KILL_BAD_EXIT: 1
 
   CONFIG_PATH: "config/longrun_configs"
-  PERF_CONFIG_PATH: "config/perf_configs"
 
 timeout_in_minutes: 1440
 
@@ -29,9 +28,6 @@ steps:
       - "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.precompile()'"
       - "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.status()'"
 
-      - echo "--- Download artifacts"
-      - "julia --project=experiments/ClimaEarth/ artifacts/download_artifacts.jl"
-
     agents:
       queue: clima
       modules: climacommon/2024_10_09

.buildkite/longruns/pipeline.yml

@@ -27,12 +27,6 @@ steps:
       - "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.precompile()'"
       - "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.status()'"
 
-      - echo "--- Download artifacts"
-      - "julia --project=artifacts -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
-      - "julia --project=artifacts -e 'using Pkg; Pkg.precompile()'"
-      - "julia --project=artifacts -e 'using Pkg; Pkg.status()'"
-      - "julia --project=artifacts artifacts/download_artifacts.jl"
-
     agents:
       slurm_cpus_per_task: 8
     env:
@@ -50,12 +44,6 @@ steps:
       - "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.precompile()'"
       - "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.status()'"
 
-      - echo "--- Download artifacts"
-      - "julia --project=artifacts -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
-      - "julia --project=artifacts -e 'using Pkg; Pkg.precompile()'"
-      - "julia --project=artifacts -e 'using Pkg; Pkg.status()'"
-      - "julia --project=artifacts artifacts/download_artifacts.jl"
-
     agents:
       queue: clima
       modules: climacommon/2024_10_09

.buildkite/nightly/pipeline.yml

@@ -34,9 +34,6 @@ steps:
       - "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.precompile()'"
       - "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.status()'"
 
-      - echo "--- Download artifacts"
-      - "julia --project=experiments/ClimaEarth/ artifacts/download_artifacts.jl"
-
     agents:
       slurm_gpus: 1
       slurm_cpus_per_task: 8

.buildkite/pipeline.yml

@@ -14,7 +14,6 @@ env:
   SLURM_KILL_BAD_EXIT: 1
 
   CONFIG_PATH: "config/ci_configs"
-  PERF_CONFIG_PATH: "config/perf_configs"
 
 timeout_in_minutes: 240
 
@@ -33,12 +32,6 @@ steps:
       - "julia --project -e 'using Pkg; Pkg.precompile()'"
       - "julia --project -e 'using Pkg; Pkg.status()'"
 
-      - echo "--- Instantiate sea breeze env"
-      - "julia --project=experiments/ClimaCore/sea_breeze -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
-      - "julia --project=experiments/ClimaCore/sea_breeze -e 'using Pkg; Pkg.add(\"MPI\"); Pkg.add(\"CUDA\")'"
-      - "julia --project=experiments/ClimaCore/sea_breeze -e 'using Pkg; Pkg.precompile()'"
-      - "julia --project=experiments/ClimaCore/sea_breeze -e 'using Pkg; Pkg.status()'"
-
       - echo "--- Instantiate climacore exp env"
       - "julia --project=experiments/ClimaCore/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
       - "julia --project=experiments/ClimaCore/ -e 'using Pkg; Pkg.add(\"MPI\"); Pkg.add(\"CUDA\")'"
@@ -48,27 +41,17 @@ steps:
       - echo "--- Instantiate ClimaEarth env"
       - "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.develop(path=\".\")'"
       - "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
-      - "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.add(\"MPI\"); Pkg.add(\"CUDA\")'"
+      - "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.add(\"MPI\"); Pkg.add(\"CUDA\");'"
       - "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.precompile()'"
       - "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.status()'"
 
-      - echo "--- Instantiate perf env"
-      - "julia --project=perf/ -e 'using Pkg; Pkg.develop(path=\".\")'"
-      - "julia --project=perf/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
-      - "julia --project=perf/ -e 'using Pkg; Pkg.add(\"MPI\"); Pkg.add(\"CUDA\")'"
-      - "julia --project=perf/ -e 'using Pkg; Pkg.precompile()'"
-      - "julia --project=perf/ -e 'using Pkg; Pkg.status()'"
-
       - echo "--- Instantiate test env"
       - "julia --project=test/ -e 'using Pkg; Pkg.develop(path=\".\")'"
       - "julia --project=test/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
       - "julia --project=test/ -e 'using Pkg; Pkg.add(\"MPI\"); Pkg.add(\"CUDA\")'"
       - "julia --project=test/ -e 'using Pkg; Pkg.precompile()'"
       - "julia --project=test/ -e 'using Pkg; Pkg.status()'"
 
-      - echo "--- Download artifacts"
-      - "julia --project=experiments/ClimaEarth/ artifacts/download_artifacts.jl"
-
     concurrency: 1
     concurrency_group: 'depot/climacoupler-ci'
     agents:
@@ -113,12 +96,6 @@ steps:
           slurm_ntasks: 2
           slurm_mem: 16GB
 
-      - label: "Perf flame graph diff tests"
-        command: "julia --color=yes --project=perf/ perf/flame_test.jl --job_id flame_perf_target"
-        timeout_in_minutes: 5
-        agents:
-          slurm_mem: 16GB
-
   - group: "GPU: unit tests and global bucket"
     steps:
       - label: "GPU runtests"
@@ -217,39 +194,12 @@ steps:
 
       # ...
 
-
-      # PERFORMANCE
-
-      # slabplanet default: track unthreaded performance (alloc tests, flame graph, flame graph diff, build history)
-      - label: ":rocket: Slabplanet: default (unthreaded)"
-        key: "slabplanet_unthreaded"
-        command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/default_unthreaded.yml --job_id default_unthreaded"
-        artifact_paths: "experiments/ClimaEarth/output/slabplanet/default_unthreaded/artifacts/*"
-        env:
-          FLAME_PLOT: ""
-          BUILD_HISTORY_HANDLE: ""
-        agents:
-          slurm_ntasks: 1
-          slurm_mem: 20GB
-
-      - label: ":rocket: Slabplanet: default (unthreaded) - flame graph and allocation tests"
-        command: "julia --color=yes --project=perf perf/flame.jl --config_file $PERF_CONFIG_PATH/perf_default_unthreaded.yml --job_id perf_default_unthreaded"
-        artifact_paths: "perf/output/perf_default_unthreaded/*"
-        agents:
-          slurm_mem: 20GB
-
-      - label: ":rocket: Slabplanet: default (unthreaded) - flame graph diff"
-        command: "julia --color=yes --project=perf perf/flame_diff.jl --config_file $PERF_CONFIG_PATH/perf_diff_default_unthreaded.yml --job_id perf_diff_default_unthreaded"
-        artifact_paths: "perf/output/perf_diff_default_unthreaded/*"
-        agents:
-          slurm_mem: 20GB
-
       # < end Drivers for release
 
       # CLIMACORE EXPERIMENTS
 
       - label: "sea_breeze"
-        command: "julia --color=yes --project=experiments/ClimaCore/sea_breeze experiments/ClimaCore/sea_breeze/run.jl"
+        command: "julia --color=yes --project=experiments/ClimaCore experiments/ClimaCore/sea_breeze/run.jl"
         artifact_paths: "experiments/ClimaCore/sea_breeze/output/*"
         agents:
           slurm_mem: 20GB
@@ -355,20 +305,6 @@ steps:
         agents:
           slurm_mem: 20GB
 
-      # PERFORMANCE RUNS: flame graphs + allocation tests
-
-      - label: ":rocket: flame graph and allocation tests: perf_coarse_single_ft64"
-        command: "julia --color=yes --project=perf perf/flame.jl --config_file $PERF_CONFIG_PATH/perf_coarse_single_ft64.yml --job_id perf_coarse_single_ft64"
-        artifact_paths: "perf/output/perf_coarse_single_ft64/*"
-        agents:
-          slurm_mem: 20GB
-
-      - label: ":rocket: performance: flame graph diff: perf_diff_coarse_single_ft64"
-        command: "julia --color=yes --project=perf perf/flame_diff.jl --config_file $PERF_CONFIG_PATH/perf_diff_coarse_single_ft64.yml --job_id perf_diff_coarse_single_ft64"
-        artifact_paths: "perf/output/perf_diff_coarse_single_ft64/*"
-        agents:
-          slurm_mem: 20GB
-
   - group: "Hierarchy tests (1d)"
     steps:
       - label: ":construction: Dry Held Suarez"

.dev/up_deps.jl

@@ -6,10 +6,8 @@ files in all of our environments.
 root = dirname(@__DIR__)
 dirs = (
     root,
-    joinpath(root, "artifacts"),
     joinpath(root, "test"),
     joinpath(root, ".dev"),
-    joinpath(root, "perf"),
     joinpath(root, "docs"),
     joinpath(root, "experiments/ClimaEarth"),
 )

.gitignore

@@ -30,7 +30,6 @@ docs/src/generated/
 # Experiments
 !experiments/ClimaEarth/**/Manifest.toml
 !experiments/ClimaCore/**/Manifest.toml
-!perf/Manifest.toml
 
 # Output
 output/

NEWS.md

@@ -13,8 +13,19 @@ low-resolution version of such files is automatically downloaded when a
 higher-resolution version is not available. Please, refer to
 [ClimaArtifacts](https://github.com/CliMA/ClimaArtifacts) for more information.
 
+### A higher resolution land-sea mask is now used and automatically downloaded - PR [#1006](https://github.com/CliMA/ClimaCoupler.jl/pull/1006)
+
+A 60 arcsecond land-sea mask constructed from topographic data is now used.
+Topographic data is automatically downloaded and a land-sea mask is constructed
+by identifying where elevation is greater than 0. Note, this can lead to
+misidentification of ocean in some areas of the globe that are inland but below
+sea level (Dead Sea, Death Valley, ...).
+
+
+
 ### Code cleanup
 #### Remove ClimaCoupler.Diagnostics module - PR [#953](https://github.com/CliMA/ClimaCoupler.jl/pull/953)
+
 The ClimaCoupler Diagnostics module had become redundant with
 ClimaDiagnostics.jl, a package designed to provide robust
 diagnostics across the CliMA ecosystem.
@@ -37,5 +48,4 @@ This PR follows directly from the Diagnostics module removal.
 
 ### Maintenance
 - Update to JuliaFormatter v2. PR [#1024](https://github.com/CliMA/ClimaCoupler.jl/pull/1024)
-- Update CI to use Julia v1.11. Introduce Manifest files for
-Julia 1.11, in addition to the existing generic Manifests. PR [#1026](https://github.com/CliMA/ClimaCoupler.jl/pull/1026)
+- Update CI to use Julia v1.11. Introduce Manifest files for Julia 1.11, in addition to the existing generic Manifests. PR [#1026](https://github.com/CliMA/ClimaCoupler.jl/pull/1026)

docs/src/performance.md

@@ -1,5 +1,11 @@
 # Performance Analysis Tools
 
+Until commit
+[7a7e98](https://github.com/CliMA/ClimaCoupler.jl/tree/7a7e98db25fe740b6ff0e3eca1e028e876a8a09e),
+ClimaCoupler provided included performance jobs. You can find them by following the link above.
+
+Below is a short description.
+
 `ClimaCoupler.jl` provides basic tools for analyzing performance:
 1. **Flame graphs**: the `perf/flame.jl` script is run by Buildkite to produce flame graphs using [ProfileCanvas.jl](https://github.com/pfitzseb/ProfileCanvas.jl) in the `perf/output/` directory.
 2. **Job walltime and allocation history**: use Buildkite to trigger the [`build_history`](https://github.com/CliMA/slurm-buildkite/blob/master/bin/build_history) script to output an interactive plot with the history of memory usage and time elapsed for each tracked job (default: current build and past builds of the `staging` branch over the past year). Use `key` to select which jobs to track. More documentation can be found in the [SLURM-Buildkite Wiki](https://github.com/CliMA/slurm-buildkite/wiki/Memory#plotting-memory-usage-over-time).