From 72fccfa1807ccbd5fc8693d4f7414bea5e256c0d Mon Sep 17 00:00:00 2001 From: Gabriele Bozzola Date: Thu, 16 May 2024 12:21:34 -0700 Subject: [PATCH] Move precipitation to artifact --- Project.toml | 3 ++- artifacts/Artifacts.toml | 3 --- artifacts/artifact_funcs.jl | 12 ------------ artifacts/download_artifacts.jl | 1 - experiments/ClimaEarth/Artifacts.toml | 11 +++++++---- experiments/ClimaEarth/Project.toml | 2 ++ .../ClimaEarth/user_io/leaderboard/Leaderboard.jl | 4 ++-- .../ClimaEarth/user_io/leaderboard/cmip_rmse.jl | 2 +- .../user_io/leaderboard/compare_with_obs.jl | 6 ++++-- test/Project.toml | 2 ++ test/experiment_tests/Artifacts.toml | 6 ++++++ test/experiment_tests/leaderboard.jl | 3 ++- 12 files changed, 28 insertions(+), 27 deletions(-) create mode 100644 test/experiment_tests/Artifacts.toml diff --git a/Project.toml b/Project.toml index 263a0aee1a..d7accf624c 100644 --- a/Project.toml +++ b/Project.toml @@ -21,10 +21,11 @@ TempestRemap_jll = "8573a8c5-1df0-515e-a024-abad257ee284" Thermodynamics = "b60c26fb-14c3-4610-9d3e-2d17fe7ff00c" [compat] +CUDA = "5" ClimaComms = "0.5.6" ClimaCore = "0.14" ClimaCoreTempestRemap = "0.3" -CUDA = "5" +ClimaUtilities = "0.1" Dates = "1" DocStringExtensions = "0.8, 0.9" JLD2 = "0.4" diff --git a/artifacts/Artifacts.toml b/artifacts/Artifacts.toml index 818474a9c0..21dbee67cd 100644 --- a/artifacts/Artifacts.toml +++ b/artifacts/Artifacts.toml @@ -4,9 +4,6 @@ git-tree-sha1 = "0c6039bfcc39f6c6cb20fb331a71f6a381d93f24" [land_mask] git-tree-sha1 = "e41bc8c44124f867b64a9d70f1599515ae27f38a" -[pr_obs_data] -git-tree-sha1 = "7486bf32e9352493f69364aead26f01eaf90d2af" - [sic] git-tree-sha1 = "139d23f6de29900917d79b41a87a8ee6483c72d0" diff --git a/artifacts/artifact_funcs.jl b/artifacts/artifact_funcs.jl index d900b6ebfe..af82b0f623 100644 --- a/artifacts/artifact_funcs.jl +++ b/artifacts/artifact_funcs.jl @@ -48,18 +48,6 @@ function mask_dataset_path() return AW.get_data_folder(mask_dataset) end -function pr_obs_data_path() - pr_obs_data = AW.ArtifactWrapper( - @__DIR__, - "pr_obs_data", - AW.ArtifactFile[AW.ArtifactFile( - url = "https://caltech.box.com/shared/static/k1or5d0d9xdvyfiytl1os7z341monkzn.nc", - filename = "gpcp.precip.mon.mean.197901-202305.nc", - ),], - ) - return AW.get_data_folder(pr_obs_data) -end - """ artifact_data(datapath_full, filename) diff --git a/artifacts/download_artifacts.jl b/artifacts/download_artifacts.jl index bc95ed9aef..ee686a5e92 100644 --- a/artifacts/download_artifacts.jl +++ b/artifacts/download_artifacts.jl @@ -23,7 +23,6 @@ function trigger_download() @info "sic dataset path: `$(sic_dataset_path())`" @info "co2 dataset path: `$(co2_dataset_path())`" @info "mask dataset path: `$(mask_dataset_path())`" - @info "pr obs data path: `$(pr_obs_data_path())`" return nothing end trigger_download() diff --git a/experiments/ClimaEarth/Artifacts.toml b/experiments/ClimaEarth/Artifacts.toml index 8065620663..385fb2673c 100644 --- a/experiments/ClimaEarth/Artifacts.toml +++ b/experiments/ClimaEarth/Artifacts.toml @@ -1,9 +1,5 @@ [land_mask] git-tree-sha1 = "e41bc8c44124f867b64a9d70f1599515ae27f38a" - -[pr_obs_data] -git-tree-sha1 = "7486bf32e9352493f69364aead26f01eaf90d2af" - [sic] git-tree-sha1 = "139d23f6de29900917d79b41a87a8ee6483c72d0" @@ -19,3 +15,10 @@ git-tree-sha1 = "aa2d5178a656e9ce7a37698d978d8ed92aa46d99" [[cmip_model_rmse.download]] sha256 = "b019d40510cebc678899a74d2cf14e33aa60452805478b2bf7176b891012aa97" url = "https://caltech.box.com/shared/static/lmzvnjkxctlb0fanieeqxll0vdrbei9e.gz" + +[precipitation_obs] +git-tree-sha1 = "7486bf32e9352493f69364aead26f01eaf90d2af" + + [[precipitation_obs.download]] + sha256 = "463ebc1886c4e0be9656656da699f2cbb87d7c2568d5e81277c0c8a415dc8143" + url = "https://caltech.box.com/shared/static/j63uak557kw7tlzb3dgwifoaeczekzm4.gz" diff --git a/experiments/ClimaEarth/Project.toml b/experiments/ClimaEarth/Project.toml index f3c2cb90a2..46d23174b1 100644 --- a/experiments/ClimaEarth/Project.toml +++ b/experiments/ClimaEarth/Project.toml @@ -17,6 +17,7 @@ ClimaCoupler = "4ade58fe-a8da-486c-bd89-46df092ec0c7" ClimaLand = "08f4d4ce-cf43-44bb-ad95-9d2d5f413532" ClimaParams = "5c42b081-d73a-476f-9059-fd94b934656c" ClimaTimeSteppers = "595c0a79-7f3d-439a-bc5a-b232dc3bde79" +ClimaUtilities = "b3f4f4ca-9299-4f7f-bd9b-81e1242a7513" CloudMicrophysics = "6a9e3e04-43cd-43ba-94b9-e8782df3c71b" Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" CommonDataModel = "1fbeeb36-5f17-413c-809b-666fb144f157" @@ -61,6 +62,7 @@ ClimaCorePlots = "0.2" ClimaLand = "0.12" ClimaParams = "0.10" ClimaTimeSteppers = "0.7" +ClimaUtilities = "0.1" Colors = "0.12" Dierckx = "0.5" ForwardDiff = "0.10" diff --git a/experiments/ClimaEarth/user_io/leaderboard/Leaderboard.jl b/experiments/ClimaEarth/user_io/leaderboard/Leaderboard.jl index 1388a00c5c..c99fd103aa 100644 --- a/experiments/ClimaEarth/user_io/leaderboard/Leaderboard.jl +++ b/experiments/ClimaEarth/user_io/leaderboard/Leaderboard.jl @@ -1,5 +1,6 @@ module Leaderboard +import ClimaComms import ClimaAnalysis import Dates import NCDatasets @@ -9,11 +10,10 @@ import GeoMakie import ClimaCoupler import Statistics: mean import Artifacts +import ClimaUtilities: @clima_artifact -include(joinpath(pkgdir(ClimaCoupler), "artifacts", "artifact_funcs.jl")) include("data_sources.jl") include("utils.jl") include("compare_with_obs.jl") - end diff --git a/experiments/ClimaEarth/user_io/leaderboard/cmip_rmse.jl b/experiments/ClimaEarth/user_io/leaderboard/cmip_rmse.jl index 47b07e4967..3662098953 100644 --- a/experiments/ClimaEarth/user_io/leaderboard/cmip_rmse.jl +++ b/experiments/ClimaEarth/user_io/leaderboard/cmip_rmse.jl @@ -1,6 +1,6 @@ import Statistics: median -pr_file_path = joinpath(artifact"cmip_model_rmse", "pr_rmse_amippr_amip.csv") +pr_file_path = joinpath(@clima_artifact("cmip_model_rmse"), "pr_rmse_amip_pr_amip_5yr") open(pr_file_path, "r") do io # Skip the header diff --git a/experiments/ClimaEarth/user_io/leaderboard/compare_with_obs.jl b/experiments/ClimaEarth/user_io/leaderboard/compare_with_obs.jl index 9a2e8ad772..aaf2359a58 100644 --- a/experiments/ClimaEarth/user_io/leaderboard/compare_with_obs.jl +++ b/experiments/ClimaEarth/user_io/leaderboard/compare_with_obs.jl @@ -24,8 +24,10 @@ function Base.values(r::RMSEs) return val_or_rmse.([r.ANN, r.DJF, r.JJA, r.MAM, r.SON]) end -OBS_DS["pr"] = - ObsDataSource(; path = joinpath(pr_obs_data_path(), "gpcp.precip.mon.mean.197901-202305.nc"), var_name = "precip") +OBS_DS["pr"] = ObsDataSource(; + path = joinpath(@clima_artifact("precipitation_obs"), "gpcp.precip.mon.mean.197901-202305.nc"), + var_name = "precip", +) SIM_DS_KWARGS["pr"] = (; preprocess_data_fn = preprocess_pr_fn, new_units = "mm / day") diff --git a/test/Project.toml b/test/Project.toml index 710626fc3c..440a29048c 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -14,6 +14,7 @@ ClimaCoupler = "4ade58fe-a8da-486c-bd89-46df092ec0c7" ClimaLand = "08f4d4ce-cf43-44bb-ad95-9d2d5f413532" ClimaParams = "5c42b081-d73a-476f-9059-fd94b934656c" ClimaTimeSteppers = "595c0a79-7f3d-439a-bc5a-b232dc3bde79" +ClimaUtilities = "b3f4f4ca-9299-4f7f-bd9b-81e1242a7513" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" GeoMakie = "db073c08-6b98-4ee5-b6a4-5efafb3259c6" @@ -43,6 +44,7 @@ ArtifactWrappers = "0.2" ClimaAtmos = "0.24" ClimaLand = "0.12" ClimaParams = "0.10" +ClimaUtilities = "0.1" Dates = "1" Downloads = "1" IntervalSets = "0.5, 0.6, 0.7" diff --git a/test/experiment_tests/Artifacts.toml b/test/experiment_tests/Artifacts.toml new file mode 100644 index 0000000000..cad73fcc2c --- /dev/null +++ b/test/experiment_tests/Artifacts.toml @@ -0,0 +1,6 @@ +[precipitation_obs] +git-tree-sha1 = "7486bf32e9352493f69364aead26f01eaf90d2af" + + [[precipitation_obs.download]] + sha256 = "463ebc1886c4e0be9656656da699f2cbb87d7c2568d5e81277c0c8a415dc8143" + url = "https://caltech.box.com/shared/static/j63uak557kw7tlzb3dgwifoaeczekzm4.gz" diff --git a/test/experiment_tests/leaderboard.jl b/test/experiment_tests/leaderboard.jl index de0e28bca1..ff5f355e20 100644 --- a/test/experiment_tests/leaderboard.jl +++ b/test/experiment_tests/leaderboard.jl @@ -1,4 +1,5 @@ import Test: @test, @testset, @test_throws +import ClimaComms import ClimaCoupler import ClimaAnalysis import Dates @@ -67,7 +68,7 @@ end @test Leaderboard.data_at_date(sim_datasource, Dates.DateTime(1979, 1, 2)) == preprocess_fn(pr.data[1, :, :]) obs_datasource = Leaderboard.ObsDataSource(; - path = joinpath(pr_obs_data_path(), "gpcp.precip.mon.mean.197901-202305.nc"), + path = joinpath(@clima_artifact("precipitation_obs"), "gpcp.precip.mon.mean.197901-202305.nc"), var_name = "precip", preprocess_data_fn = preprocess_fn, )