From ee1b34ef46b3165ffce9efe4bfd3288c0666bb0d Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sun, 26 Nov 2023 09:28:09 -0500 Subject: [PATCH 001/567] full interior map --- src/ImmersedBoundaries/ImmersedBoundaries.jl | 16 ++- src/ImmersedBoundaries/active_cells_map.jl | 46 +++++--- .../hydrostatic_free_surface_ab2_step.jl | 21 +++- .../split_explicit_free_surface_kernels.jl | 104 ++++++++++++++++-- src/Solvers/batched_tridiagonal_solver.jl | 14 ++- src/TimeSteppers/quasi_adams_bashforth_2.jl | 16 ++- src/TimeSteppers/store_tendencies.jl | 12 +- .../vertically_implicit_diffusion_solver.jl | 4 +- .../inertial_particles.jl | 66 +++++++++++ 9 files changed, 255 insertions(+), 44 deletions(-) create mode 100644 validation/lagrangian_particles/inertial_particles.jl diff --git a/src/ImmersedBoundaries/ImmersedBoundaries.jl b/src/ImmersedBoundaries/ImmersedBoundaries.jl index 49d49d3934..f7e8d3f429 100644 --- a/src/ImmersedBoundaries/ImmersedBoundaries.jl +++ b/src/ImmersedBoundaries/ImmersedBoundaries.jl @@ -102,18 +102,19 @@ abstract type AbstractImmersedBoundary end ##### ImmersedBoundaryGrid ##### -struct ImmersedBoundaryGrid{FT, TX, TY, TZ, G, I, M, Arch} <: AbstractGrid{FT, TX, TY, TZ, Arch} +struct ImmersedBoundaryGrid{FT, TX, TY, TZ, G, I, M, S, Arch} <: AbstractGrid{FT, TX, TY, TZ, Arch} architecture :: Arch underlying_grid :: G immersed_boundary :: I interior_active_cells :: M - + surface_active_cells :: S + # Internal interface - function ImmersedBoundaryGrid{TX, TY, TZ}(grid::G, ib::I, mi::M) where {TX, TY, TZ, G <: AbstractUnderlyingGrid, I, M} + function ImmersedBoundaryGrid{TX, TY, TZ}(grid::G, ib::I, mi::M, ms::S) where {TX, TY, TZ, G <: AbstractUnderlyingGrid, I, M, S} FT = eltype(grid) arch = architecture(grid) Arch = typeof(arch) - return new{FT, TX, TY, TZ, G, I, M, Arch}(arch, grid, ib, mi) + return new{FT, TX, TY, TZ, G, I, M, S, Arch}(arch, grid, ib, mi, ms) end # Constructor with no active map @@ -121,7 +122,7 @@ struct ImmersedBoundaryGrid{FT, TX, TY, TZ, G, I, M, Arch} <: AbstractGrid{FT, T FT = eltype(grid) arch = architecture(grid) Arch = typeof(arch) - return new{FT, TX, TY, TZ, G, I, Nothing, Arch}(arch, grid, ib, nothing) + return new{FT, TX, TY, TZ, G, I, Nothing, Nothing, Arch}(arch, grid, ib, nothing, nothing) end end @@ -141,7 +142,10 @@ const IBG = ImmersedBoundaryGrid @inline z_domain(ibg::IBG) = z_domain(ibg.underlying_grid) Adapt.adapt_structure(to, ibg::IBG{FT, TX, TY, TZ}) where {FT, TX, TY, TZ} = - ImmersedBoundaryGrid{TX, TY, TZ}(adapt(to, ibg.underlying_grid), adapt(to, ibg.immersed_boundary), adapt(to, ibg.interior_active_cells)) + ImmersedBoundaryGrid{TX, TY, TZ}(adapt(to, ibg.underlying_grid), + adapt(to, ibg.immersed_boundary), + adapt(to, ibg.interior_active_cells), + adapt(to, ibg.surface_active_cells)) with_halo(halo, ibg::ImmersedBoundaryGrid) = ImmersedBoundaryGrid(with_halo(halo, ibg.underlying_grid), ibg.immersed_boundary) diff --git a/src/ImmersedBoundaries/active_cells_map.jl b/src/ImmersedBoundaries/active_cells_map.jl index f3ce915df7..0369198ebe 100644 --- a/src/ImmersedBoundaries/active_cells_map.jl +++ b/src/ImmersedBoundaries/active_cells_map.jl @@ -6,21 +6,24 @@ using KernelAbstractions: @kernel, @index import Oceananigans.Utils: active_cells_work_layout, use_only_active_interior_cells -const ActiveCellsIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractArray} +import Oceananigans.Solvers: solve_batched_tridiagonal_system_kernel! +using Oceananigans.Solvers: solve_batched_tridiagonal_system_z!, ZDirection + +const ActiveCellsIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractArray} +const ActiveSurfaceIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractArray} struct InteriorMap end struct SurfaceMap end -@inline use_only_active_interior_cells(grid::ActiveCellsIBG) = InteriorMap() - -@inline use_only_active_surface_cells(grid::AbstractGrid) = nothing -@inline use_only_active_surface_cells(grid::ActiveCellsIBG) = SurfaceMap() +@inline use_only_active_surface_cells(::AbstractGrid) = nothing +@inline use_only_active_interior_cells(::ActiveCellsIBG) = InteriorMap() +@inline use_only_active_surface_cells(::ActiveSurfaceIBG) = SurfaceMap() -@inline active_cells_work_layout(group, size, ::InteriorMap, grid::ActiveCellsIBG) = min(length(grid.interior_active_cells), 256), length(grid.interior_active_cells) -@inline active_cells_work_layout(group, size, ::SurfaceMap, grid::ActiveCellsIBG) = min(length(grid.surface_active_cells), 256), length(grid.surface_active_cells) +@inline active_cells_work_layout(group, size, ::InteriorMap, grid::ActiveCellsIBG) = min(length(grid.interior_active_cells), 256), length(grid.interior_active_cells) +@inline active_cells_work_layout(group, size, ::SurfaceMap, grid::ActiveSurfaceIBG) = min(length(grid.surface_active_cells), 256), length(grid.surface_active_cells) -@inline active_linear_index_to_interior_tuple(idx, grid::ActiveCellsIBG) = Base.map(Int, grid.interior_active_cells[idx]) -@inline active_linear_index_to_surface_tuple(idx, grid::ActiveCellsIBG) = Base.map(Int, grid.surface_active_cells[idx]) +@inline active_linear_index_to_interior_tuple(idx, grid::ActiveCellsIBG) = Base.map(Int, grid.interior_active_cells[idx]) +@inline active_linear_index_to_surface_tuple(idx, grid::ActiveSurfaceIBG) = Base.map(Int, grid.surface_active_cells[idx]) function ImmersedBoundaryGrid(grid, ib; active_cells_map::Bool = true) @@ -29,17 +32,18 @@ function ImmersedBoundaryGrid(grid, ib; active_cells_map::Bool = true) # Create the cells map on the CPU, then switch it to the GPU if active_cells_map - map_interior = active_cells_map_interior(ibg) - map_interior = arch_array(architecture(ibg), map_interior) - # map_surface = active_cells_map_surface(ibg) - # map_surface = arch_array(architecture(ibg), map_surface) + interior_map = active_cells_interior_map(ibg) + interior_map = arch_array(architecture(ibg), interior_map) + surface_map = active_cells_surface_map(ibg) + surface_map = arch_array(architecture(ibg), surface_map) else - map_interior = nothing + interior_map = nothing end return ImmersedBoundaryGrid{TX, TY, TZ}(ibg.underlying_grid, ibg.immersed_boundary, - map_interior) + interior_map, + surface_map) end @inline active_cell(i, j, k, ibg) = !immersed_cell(i, j, k, ibg) @@ -65,7 +69,7 @@ const MAXUInt8 = 2^8 - 1 const MAXUInt16 = 2^16 - 1 const MAXUInt32 = 2^32 - 1 -function active_cells_map_interior(ibg) +function active_cells_interior_map(ibg) active_cells_field = compute_interior_active_cells(ibg) N = maximum(size(ibg)) @@ -105,7 +109,7 @@ end # If we eventually want to perform also barotropic step, `w` computation and `p` # computation only on active `columns` -function active_cells_map_surface(ibg) +function active_cells_surface_map(ibg) active_cells_field = compute_surface_active_cells(ibg) interior_cells = arch_array(CPU(), interior(active_cells_field, :, :, 1)) @@ -119,3 +123,11 @@ function active_cells_map_surface(ibg) return smaller_indices end + +@kernel function solve_batched_tridiagonal_system_kernel!(ϕ, a, b, c, f, t, grid::ActiveSurfaceIBG, p, args, tridiagonal_direction::ZDirection) + idx = @index(Global, Linear) + i, j = active_linear_index_to_surface_tuple(idx, grid) + Nz = size(grid, 3) + + solve_batched_tridiagonal_system_z!(i, j, Nz, ϕ, a, b, c, f, t, grid, p, args, tridiagonal_direction) +end \ No newline at end of file diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl index 509b1fe0ea..24c58ff2b4 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl @@ -2,6 +2,8 @@ using Oceananigans.Fields: location using Oceananigans.TimeSteppers: ab2_step_field! using Oceananigans.TurbulenceClosures: implicit_step! +using Oceananigans.ImmersedBoundaries: use_only_active_interior_cells, use_only_active_surface_cells + import Oceananigans.TimeSteppers: ab2_step! ##### @@ -35,13 +37,18 @@ end function ab2_step_velocities!(velocities, model, Δt, χ) + only_active_interior_cells = use_only_active_interior_cells(model.grid) + only_active_surface_cells = use_only_active_surface_cells(model.grid) + for (i, name) in enumerate((:u, :v)) Gⁿ = model.timestepper.Gⁿ[name] G⁻ = model.timestepper.G⁻[name] velocity_field = model.velocities[name] + launch!(model.architecture, model.grid, :xyz, - ab2_step_field!, velocity_field, Δt, χ, Gⁿ, G⁻) + ab2_step_field!, velocity_field, Δt, χ, Gⁿ, G⁻; + only_active_cells = only_active_interior_cells) # TODO: let next implicit solve depend on previous solve + explicit velocity step # Need to distinguish between solver events and tendency calculation events. @@ -52,7 +59,8 @@ function ab2_step_velocities!(velocities, model, Δt, χ) model.diffusivity_fields, nothing, model.clock, - Δt) + Δt; + only_active_cells = only_active_surface_cells) end return nothing @@ -68,6 +76,9 @@ ab2_step_tracers!(::EmptyNamedTuple, model, Δt, χ) = nothing function ab2_step_tracers!(tracers, model, Δt, χ) + only_active_interior_cells = use_only_active_interior_cells(model.grid) + only_active_surface_cells = use_only_active_surface_cells(model.grid) + # Tracer update kernels for (tracer_index, tracer_name) in enumerate(propertynames(tracers)) Gⁿ = model.timestepper.Gⁿ[tracer_name] @@ -76,7 +87,8 @@ function ab2_step_tracers!(tracers, model, Δt, χ) closure = model.closure launch!(model.architecture, model.grid, :xyz, - ab2_step_field!, tracer_field, Δt, χ, Gⁿ, G⁻) + ab2_step_field!, tracer_field, Δt, χ, Gⁿ, G⁻; + only_active_cells = only_active_interior_cells) implicit_step!(tracer_field, model.timestepper.implicit_solver, @@ -84,7 +96,8 @@ function ab2_step_tracers!(tracers, model, Δt, χ) model.diffusivity_fields, Val(tracer_index), model.clock, - Δt) + Δt; + only_active_cells = only_active_surface_cells) end return nothing diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 1062cd43c9..754ffa362b 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -7,7 +7,8 @@ using Oceananigans.BoundaryConditions using Oceananigans.Operators using Oceananigans.ImmersedBoundaries: peripheral_node, immersed_inactive_node using Oceananigans.ImmersedBoundaries: inactive_node, IBG, c, f -using Oceananigans.ImmersedBoundaries: mask_immersed_field! +using Oceananigans.ImmersedBoundaries: mask_immersed_field!, use_only_active_surface_cells, use_only_active_interior_cells +using Oceananigans.ImmersedBoundaries: active_linear_index_to_surface_tuple, ActiveCellsIBG, ActiveSurfaceIBG # constants for AB3 time stepping scheme (from https://doi.org/10.1016/j.ocemod.2004.08.002) const β = 0.281105 @@ -141,8 +142,20 @@ using Printf Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, timestepper) i, j = @index(Global, NTuple) - k_top = grid.Nz+1 + free_surface_evolution_kernel!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) +end + +@kernel function split_explicit_free_surface_evolution_kernel!(grid::ActiveSurfaceIBG, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², + η̅, U̅, V̅, averaging_weight, + Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, + timestepper) + idx = @index(Global, Linear) + i, j = active_linear_index_to_surface_tuple(idx, grid) + free_surface_evolution_kernel!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) +end +@inline function free_surface_evolution_kernel!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) + k_top = grid.Nz+1 TX, TY, _ = topology(grid) @inbounds begin @@ -151,6 +164,8 @@ using Printf η[i, j, k_top] -= Δτ * (div_xᶜᶜᶠ_U(i, j, k_top-1, grid, TX, U★, timestepper, U, Uᵐ⁻¹, Uᵐ⁻²) + div_yᶜᶜᶠ_V(i, j, k_top-1, grid, TY, U★, timestepper, V, Vᵐ⁻¹, Vᵐ⁻²)) end + + return nothing end @kernel function split_explicit_barotropic_velocity_evolution_kernel!(grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², @@ -158,6 +173,31 @@ end Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, timestepper) i, j = @index(Global, NTuple) + + velocity_evolution_kernel!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², + η̅, U̅, V̅, averaging_weight, + Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, + timestepper ) +end + + +@kernel function split_explicit_barotropic_velocity_evolution_kernel!(grid::ActiveSurfaceIBG, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², + η̅, U̅, V̅, averaging_weight, + Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, + timestepper) + idx = @index(Global, Linear) + i, j = active_linear_index_to_surface_tuple(idx, grid) + + velocity_evolution_kernel!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², + η̅, U̅, V̅, averaging_weight, + Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, + timestepper ) +end + +@inline function velocity_evolution_kernel!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², + η̅, U̅, V̅, averaging_weight, + Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, + timestepper ) k_top = grid.Nz+1 TX, TY, _ = topology(grid) @@ -195,8 +235,10 @@ function split_explicit_free_surface_substep!(η, state, auxiliary, settings, we η̅, U̅, V̅, averaging_weight, Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, timestepper) - launch!(arch, grid, parameters, split_explicit_free_surface_evolution_kernel!, args...) - launch!(arch, grid, parameters, split_explicit_barotropic_velocity_evolution_kernel!, args...) + launch!(arch, grid, parameters, split_explicit_free_surface_evolution_kernel!, args...; + only_active_cells = use_only_active_surface_cells(grid)) + launch!(arch, grid, parameters, split_explicit_barotropic_velocity_evolution_kernel!, args...; + only_active_cells = use_only_active_surface_cells(grid)) return nothing end @@ -216,9 +258,26 @@ end end end +# Barotropic Model Kernels +# u_Δz = u * Δz +@kernel function _barotropic_mode_kernel!(U, V, grid::ActiveSurfaceIBG, u, v) + idx = @index(Global, Linear) + i, j = active_linear_index_to_surface_tuple(idx, grid) + + # hand unroll first loop + @inbounds U[i, j, 1] = Δzᶠᶜᶜ(i, j, 1, grid) * u[i, j, 1] + @inbounds V[i, j, 1] = Δzᶜᶠᶜ(i, j, 1, grid) * v[i, j, 1] + + @unroll for k in 2:grid.Nz + @inbounds U[i, j, 1] += Δzᶠᶜᶜ(i, j, k, grid) * u[i, j, k] + @inbounds V[i, j, 1] += Δzᶜᶠᶜ(i, j, k, grid) * v[i, j, k] + end +end + # may need to do Val(Nk) since it may not be known at compile compute_barotropic_mode!(U, V, grid, u, v) = - launch!(architecture(grid), grid, :xy, _barotropic_mode_kernel!, U, V, grid, u, v) + launch!(architecture(grid), grid, :xy, _barotropic_mode_kernel!, U, V, grid, u, v; + only_active_cells = use_only_active_surface_cells(grid)) function initialize_free_surface_state!(free_surface_state, η) state = free_surface_state @@ -243,7 +302,7 @@ function initialize_free_surface_state!(free_surface_state, η) return nothing end -@kernel function barotropic_split_explicit_corrector_kernel!(u, v, U̅, V̅, U, V, Hᶠᶜ, Hᶜᶠ) +@kernel function barotropic_split_explicit_corrector_kernel!(u, v, U̅, V̅, U, V, Hᶠᶜ, Hᶜᶠ, grid) i, j, k = @index(Global, NTuple) @inbounds begin u[i, j, k] = u[i, j, k] + (U̅[i, j] - U[i, j]) / Hᶠᶜ[i, j] @@ -251,6 +310,15 @@ end end end +@kernel function barotropic_split_explicit_corrector_kernel!(u, v, U̅, V̅, U, V, Hᶠᶜ, Hᶜᶠ, grid::ActiveCellsIBG) + idx = @index(Global, Linear) + i, j, k = active_linear_index_to_interior_tuple(idx, grid) + @inbounds begin + u[i, j, k] = u[i, j, k] + (U̅[i, j] - U[i, j]) / Hᶠᶜ[i, j] + v[i, j, k] = v[i, j, k] + (V̅[i, j] - V[i, j]) / Hᶜᶠ[i, j] + end +end + # may need to do Val(Nk) since it may not be known at compile. Also figure out where to put H function barotropic_split_explicit_corrector!(u, v, free_surface, grid) sefs = free_surface.state @@ -264,7 +332,8 @@ function barotropic_split_explicit_corrector!(u, v, free_surface, grid) # add in "good" barotropic mode launch!(arch, grid, :xyz, barotropic_split_explicit_corrector_kernel!, - u, v, U̅, V̅, U, V, Hᶠᶜ, Hᶜᶠ) + u, v, U̅, V̅, U, V, Hᶠᶜ, Hᶜᶠ, grid; + only_active_cells = use_only_active_interior_cells(grid)) return nothing end @@ -362,6 +431,21 @@ end end end +# Calculate RHS for the barotopic time step. +@kernel function _compute_integrated_ab2_tendencies!(Gᵁ, Gⱽ, grid::ActiveSurfaceIBG, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) + idx = @index(Global, Linear) + i, j = active_linear_index_to_surface_tuple(idx, grid) + + # hand unroll first loop + @inbounds Gᵁ[i, j, 1] = Δzᶠᶜᶜ(i, j, 1, grid) * ab2_step_Gu(i, j, 1, grid, Gu⁻, Guⁿ, χ) + @inbounds Gⱽ[i, j, 1] = Δzᶜᶠᶜ(i, j, 1, grid) * ab2_step_Gv(i, j, 1, grid, Gv⁻, Gvⁿ, χ) + + @unroll for k in 2:grid.Nz + @inbounds Gᵁ[i, j, 1] += Δzᶠᶜᶜ(i, j, k, grid) * ab2_step_Gu(i, j, k, grid, Gu⁻, Guⁿ, χ) + @inbounds Gⱽ[i, j, 1] += Δzᶜᶠᶜ(i, j, k, grid) * ab2_step_Gv(i, j, k, grid, Gv⁻, Gvⁿ, χ) + end +end + @inline ab2_step_Gu(i, j, k, grid, G⁻, Gⁿ, χ::FT) where FT = ifelse(peripheral_node(i, j, k, grid, f, c, c), zero(grid), (convert(FT, 1.5) + χ) * Gⁿ[i, j, k] - G⁻[i, j, k] * (convert(FT, 0.5) + χ)) @inline ab2_step_Gv(i, j, k, grid, G⁻, Gⁿ, χ::FT) where FT = ifelse(peripheral_node(i, j, k, grid, c, f, c), zero(grid), (convert(FT, 1.5) + χ) * Gⁿ[i, j, k] - G⁻[i, j, k] * (convert(FT, 0.5) + χ)) @@ -388,6 +472,8 @@ function setup_free_surface!(model, free_surface::SplitExplicitFreeSurface, χ) end setup_split_explicit_tendency!(auxiliary, grid, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) = - launch!(architecture(grid), grid, :xy, _compute_integrated_ab2_tendencies!, auxiliary.Gᵁ, auxiliary.Gⱽ, grid, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) - + launch!(architecture(grid), grid, :xy, _compute_integrated_ab2_tendencies!, auxiliary.Gᵁ, auxiliary.Gⱽ, grid, + Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ; only_active_cells = use_only_active_surface_cells(grid)) + wait_free_surface_communication!(free_surface, arch) = nothing + diff --git a/src/Solvers/batched_tridiagonal_solver.jl b/src/Solvers/batched_tridiagonal_solver.jl index a188d0fc89..d69ad5ecfc 100644 --- a/src/Solvers/batched_tridiagonal_solver.jl +++ b/src/Solvers/batched_tridiagonal_solver.jl @@ -88,7 +88,7 @@ Reference implementation per Numerical Recipes, Press et al. 1992 (§ 2.4). Note a slightly different notation from Press et al. is used for indexing the off-diagonal elements; see [`BatchedTridiagonalSolver`](@ref). """ -function solve!(ϕ, solver::BatchedTridiagonalSolver, rhs, args...) +function solve!(ϕ, solver::BatchedTridiagonalSolver, rhs, args...; only_active_cells = nothing) launch_config = if solver.tridiagonal_direction isa XDirection :yz @@ -108,7 +108,8 @@ function solve!(ϕ, solver::BatchedTridiagonalSolver, rhs, args...) solver.grid, solver.parameters, Tuple(args), - solver.tridiagonal_direction) + solver.tridiagonal_direction; + only_active_cells) return nothing end @@ -124,7 +125,10 @@ end @kernel function solve_batched_tridiagonal_system_kernel!(ϕ, a, b, c, f, t, grid, p, args, tridiagonal_direction::XDirection) Nx = size(grid, 1) j, k = @index(Global, NTuple) + solve_batched_tridiagonal_system_x!(j, k, Nx, ϕ, a, b, c, f, t, grid, p, args, tridiagonal_direction) +end +@inline function solve_batched_tridiagonal_system_x!(j, k, Nx, ϕ, a, b, c, f, t, grid, p, args, tridiagonal_direction) @inbounds begin β = get_coefficient(1, j, k, grid, b, p, tridiagonal_direction, args...) f₁ = get_coefficient(1, j, k, grid, f, p, tridiagonal_direction, args...) @@ -156,7 +160,10 @@ end @kernel function solve_batched_tridiagonal_system_kernel!(ϕ, a, b, c, f, t, grid, p, args, tridiagonal_direction::YDirection) Ny = size(grid, 2) i, k = @index(Global, NTuple) + solve_batched_tridiagonal_system_y!(i, k, Ny, ϕ, a, b, c, f, t, grid, p, args, tridiagonal_direction) +end +@inline function solve_batched_tridiagonal_system_y!(i, k, Ny, ϕ, a, b, c, f, t, grid, p, args, tridiagonal_direction) @inbounds begin β = get_coefficient(i, 1, k, grid, b, p, tridiagonal_direction, args...) f₁ = get_coefficient(i, 1, k, grid, f, p, tridiagonal_direction, args...) @@ -188,7 +195,10 @@ end @kernel function solve_batched_tridiagonal_system_kernel!(ϕ, a, b, c, f, t, grid, p, args, tridiagonal_direction::ZDirection) Nz = size(grid, 3) i, j = @index(Global, NTuple) + solve_batched_tridiagonal_system_z!(i, j, Nz, ϕ, a, b, c, f, t, grid, p, args, tridiagonal_direction) +end +@inline function solve_batched_tridiagonal_system_z!(i, j, Nz, ϕ, a, b, c, f, t, grid, p, args, tridiagonal_direction) @inbounds begin β = get_coefficient(i, j, 1, grid, b, p, tridiagonal_direction, args...) f₁ = get_coefficient(i, j, 1, grid, f, p, tridiagonal_direction, args...) diff --git a/src/TimeSteppers/quasi_adams_bashforth_2.jl b/src/TimeSteppers/quasi_adams_bashforth_2.jl index 6c3854e843..21f309148e 100644 --- a/src/TimeSteppers/quasi_adams_bashforth_2.jl +++ b/src/TimeSteppers/quasi_adams_bashforth_2.jl @@ -1,6 +1,7 @@ using Oceananigans.Fields: FunctionField, location using Oceananigans.TurbulenceClosures: implicit_step! using Oceananigans.Utils: @apply_regionally, apply_regionally! +using Oceananigans.ImmersedBoundaries: ActiveCellsIBG, active_linear_index_to_interior_tuple mutable struct QuasiAdamsBashforth2TimeStepper{FT, GT, IT} <: AbstractTimeStepper χ :: FT @@ -147,7 +148,7 @@ Time step velocity fields via the 2nd-order quasi Adams-Bashforth method `U^{n+1} = U^n + Δt ((3/2 + χ) * G^{n} - (1/2 + χ) G^{n-1})` """ -@kernel function ab2_step_field!(u, Δt, χ, Gⁿ, G⁻) +@kernel function ab2_step_field!(u, Δt, χ, Gⁿ, G⁻, grid) i, j, k = @index(Global, NTuple) FT = eltype(χ) @@ -157,4 +158,15 @@ Time step velocity fields via the 2nd-order quasi Adams-Bashforth method @inbounds u[i, j, k] += convert(FT, Δt) * ((one_point_five + χ) * Gⁿ[i, j, k] - (oh_point_five + χ) * G⁻[i, j, k]) end -@kernel ab2_step_field!(::FunctionField, Δt, χ, Gⁿ, G⁻) = nothing +@kernel function ab2_step_field!(u, Δt, χ, Gⁿ, G⁻, grid::ActiveCellsIBG) + idx = @index(Global, Linear) + i, j, k = active_linear_index_to_interior_tuple(idx, grid) + + FT = eltype(χ) + one_point_five = convert(FT, 1.5) + oh_point_five = convert(FT, 0.5) + + @inbounds u[i, j, k] += convert(FT, Δt) * ((one_point_five + χ) * Gⁿ[i, j, k] - (oh_point_five + χ) * G⁻[i, j, k]) +end + +@kernel ab2_step_field!(::FunctionField, Δt, χ, Gⁿ, G⁻, grid) = nothing diff --git a/src/TimeSteppers/store_tendencies.jl b/src/TimeSteppers/store_tendencies.jl index 06d179bd3a..6045494cee 100644 --- a/src/TimeSteppers/store_tendencies.jl +++ b/src/TimeSteppers/store_tendencies.jl @@ -9,15 +9,23 @@ using Oceananigans.Utils: launch! @inbounds G⁻[i, j, k] = G⁰[i, j, k] end +""" Store source terms for `u`, `v`, and `w`. """ +@kernel function store_field_tendencies!(G⁻, grid::ActiveCellsIBG, G⁰) + idx = @index(Global, Linear) + i, j, k = active_linear_index_to_interior_tuple(idx, grid) + @inbounds G⁻[i, j, k] = G⁰[i, j, k] +end + """ Store previous source terms before updating them. """ -function store_tendencies!(model) +function store_tendencies!(model; only_active_cells = only_active_interior_cells(model.grid)) model_fields = prognostic_fields(model) for field_name in keys(model_fields) launch!(model.architecture, model.grid, :xyz, store_field_tendencies!, model.timestepper.G⁻[field_name], model.grid, - model.timestepper.Gⁿ[field_name]) + model.timestepper.Gⁿ[field_name]; + only_active_cells) end return nothing diff --git a/src/TurbulenceClosures/vertically_implicit_diffusion_solver.jl b/src/TurbulenceClosures/vertically_implicit_diffusion_solver.jl index 71fa9bd8e4..f3b3e05c3a 100644 --- a/src/TurbulenceClosures/vertically_implicit_diffusion_solver.jl +++ b/src/TurbulenceClosures/vertically_implicit_diffusion_solver.jl @@ -178,7 +178,7 @@ function implicit_step!(field::Field, diffusivity_fields, tracer_index, clock, - Δt) + Δt; kwargs...) loc = location(field) @@ -208,6 +208,6 @@ function implicit_step!(field::Field, return solve!(field, implicit_solver, field, # ivd_*_diagonal gets called with these args after (i, j, k, grid): - vi_closure, vi_diffusivity_fields, tracer_index, map(ℓ -> ℓ(), loc)..., clock, Δt, κz) + vi_closure, vi_diffusivity_fields, tracer_index, map(ℓ -> ℓ(), loc)..., clock, Δt, κz; kwargs...) end diff --git a/validation/lagrangian_particles/inertial_particles.jl b/validation/lagrangian_particles/inertial_particles.jl new file mode 100644 index 0000000000..e6bbfbe9d2 --- /dev/null +++ b/validation/lagrangian_particles/inertial_particles.jl @@ -0,0 +1,66 @@ +using StructArrays +using Oceananigans +using Oceananigans: architecture +using Oceananigans.Models.LagrangianParticleTracking: AbstractParticle +using Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_w_from_continuity! +import Oceananigans.Models.LagrangianParticleTracking: particle_u_velocity, particle_v_velocity, particle_w_velocity + +struct InertialParticle{T} <: AbstractParticle + x :: T + y :: T + z :: T + u :: T + v :: T + w :: T + particle_respose_time :: T +end + +# 10 Particles with different inertia +x = ones(10) +y = ones(10) +z = ones(10) +u = zeros(10) +v = zeros(10) +w = zeros(10) + +particle_respose_time = range(0.1, 1.0, length = 10) + +properties = StructArray{InertialParticle}((x, y, z, u, v, w, particle_respose_time)) +particles = LagrangianParticles(properties) + +grid = RectilinearGrid(size = (50, 50, 50), x = (0, 2), y = (0, 2), z = (0, 2), topology = (Periodic, Periodic, Periodic)) + +u_fluid = XFaceField(grid) +v_fluid = YFaceField(grid) +w_fluid = ZFaceField(grid) + +@inline particles_u_velocity(u_fluid, particle, Δt) = particle.u + Δt / particles.particle_respose_time * (u_fluid - particle.u) +@inline particles_v_velocity(v_fluid, particle, Δt) = particle.v + Δt / particles.particle_respose_time * (v_fluid - particle.v) +@inline particles_w_velocity(w_fluid, particle, Δt) = particle.w + Δt / particles.particle_respose_time * (w_fluid - particle.w) + +set!(u_fluid, (x, y, z) -> rand()) +set!(v_fluid, (x, y, z) -> rand()) + +fill_halo_regions!((u_fluid, v_fluid)) + +compute_w_from_continuity!((; u = u_fluid, v = v_fluid, w = w_fluid), architecture(grid), grid) + +velocities = PrescribedVelocityFields(; u = u_fluid, v = v_fluid, w = w_fluid) + +model = HydrostaticFreeSurfaceModel(; grid, + tracers = (), + buoyancy = nothing, + particles, + velocities) + +simulation = Simulation(model, Δt = 1e-2, stop_time = 10) + +particles_save = [deepcopy(properties)] + +save_particles(sim) = + push!(particles_save, deepcopy(sim.model.particles.properties)) + +simulation.callbacks[:particles] = Callback(save_particles, IterationInterval(10)) + +run!(simulation) + From 31ba1d67be0d7ab735b0e551ca723cededbe7504 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sun, 26 Nov 2023 09:55:19 -0500 Subject: [PATCH 002/567] bugfix --- src/TimeSteppers/store_tendencies.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TimeSteppers/store_tendencies.jl b/src/TimeSteppers/store_tendencies.jl index 6045494cee..d5bae5d376 100644 --- a/src/TimeSteppers/store_tendencies.jl +++ b/src/TimeSteppers/store_tendencies.jl @@ -1,6 +1,6 @@ using Oceananigans: prognostic_fields using Oceananigans.Grids: AbstractGrid - +using Oceananigans.ImmersedBoundaries: ActiveCellsIBG using Oceananigans.Utils: launch! """ Store source terms for `u`, `v`, and `w`. """ From 14031619c31c5357d66422431a986071a8059957 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sun, 26 Nov 2023 10:19:55 -0500 Subject: [PATCH 003/567] bugfix --- src/ImmersedBoundaries/active_cells_map.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ImmersedBoundaries/active_cells_map.jl b/src/ImmersedBoundaries/active_cells_map.jl index 0369198ebe..d52e3db5a2 100644 --- a/src/ImmersedBoundaries/active_cells_map.jl +++ b/src/ImmersedBoundaries/active_cells_map.jl @@ -38,6 +38,7 @@ function ImmersedBoundaryGrid(grid, ib; active_cells_map::Bool = true) surface_map = arch_array(architecture(ibg), surface_map) else interior_map = nothing + surface_map = nothing end return ImmersedBoundaryGrid{TX, TY, TZ}(ibg.underlying_grid, From 7d7620362eb46322f2c6c0806dd3b0e574a025d1 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sun, 26 Nov 2023 10:21:24 -0500 Subject: [PATCH 004/567] bugfixes --- .../hydrostatic_free_surface_ab2_step.jl | 4 ++-- src/TimeSteppers/quasi_adams_bashforth_2.jl | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl index 24c58ff2b4..c2a88821af 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl @@ -47,7 +47,7 @@ function ab2_step_velocities!(velocities, model, Δt, χ) launch!(model.architecture, model.grid, :xyz, - ab2_step_field!, velocity_field, Δt, χ, Gⁿ, G⁻; + ab2_step_field!, velocity_field, Δt, χ, Gⁿ, G⁻, model.grid; only_active_cells = only_active_interior_cells) # TODO: let next implicit solve depend on previous solve + explicit velocity step @@ -87,7 +87,7 @@ function ab2_step_tracers!(tracers, model, Δt, χ) closure = model.closure launch!(model.architecture, model.grid, :xyz, - ab2_step_field!, tracer_field, Δt, χ, Gⁿ, G⁻; + ab2_step_field!, tracer_field, Δt, χ, Gⁿ, G⁻, model.grid; only_active_cells = only_active_interior_cells) implicit_step!(tracer_field, diff --git a/src/TimeSteppers/quasi_adams_bashforth_2.jl b/src/TimeSteppers/quasi_adams_bashforth_2.jl index 21f309148e..c589dfee73 100644 --- a/src/TimeSteppers/quasi_adams_bashforth_2.jl +++ b/src/TimeSteppers/quasi_adams_bashforth_2.jl @@ -125,7 +125,8 @@ function ab2_step!(model, Δt, χ) step_field_kernel!(field, Δt, χ, model.timestepper.Gⁿ[i], - model.timestepper.G⁻[i]) + model.timestepper.G⁻[i], + model.grid) # TODO: function tracer_index(model, field_index) = field_index - 3, etc... tracer_index = Val(i - 3) # assumption From b5d6a4294dabc181cf7bbd652962f41533f7a5e2 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sun, 26 Nov 2023 10:26:32 -0500 Subject: [PATCH 005/567] hmmm --- ext/OceananigansEnzymeCoreExt.jl | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/ext/OceananigansEnzymeCoreExt.jl b/ext/OceananigansEnzymeCoreExt.jl index 9d273f56f1..c43033a0e3 100644 --- a/ext/OceananigansEnzymeCoreExt.jl +++ b/ext/OceananigansEnzymeCoreExt.jl @@ -102,19 +102,19 @@ function EnzymeCore.EnzymeRules.augmented_primal(config, end function EnzymeCore.EnzymeRules.reverse(config::EnzymeCore.EnzymeRules.ConfigWidth{1}, - func::EnzymeCore.Const{typeof(Oceananigans.Utils.launch!)}, - ::Type{EnzymeCore.Const{Nothing}}, - tape, - arch, - grid, - workspec, - kernel!, - kernel_args...; - include_right_boundaries = false, - reduced_dimensions = (), - location = nothing, - only_active_cells = nothing, - kwargs...) + func::EnzymeCore.Const{typeof(Oceananigans.Utils.launch!)}, + ::Type{EnzymeCore.Const{Nothing}}, + tape, + arch, + grid, + workspec, + kernel!, + kernel_args...; + include_right_boundaries = false, + reduced_dimensions = (), + location = nothing, + only_active_cells = nothing, + kwargs...) subrets = if tape !== nothing duploop, subtape = tape From ee62becb940e8df6b29cf0037cd6907212d205af Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sun, 26 Nov 2023 10:34:43 -0500 Subject: [PATCH 006/567] disambiguate --- .../hydrostatic_free_surface_ab2_step.jl | 1 - src/TimeSteppers/quasi_adams_bashforth_2.jl | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl index c2a88821af..6cf8f349db 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl @@ -45,7 +45,6 @@ function ab2_step_velocities!(velocities, model, Δt, χ) G⁻ = model.timestepper.G⁻[name] velocity_field = model.velocities[name] - launch!(model.architecture, model.grid, :xyz, ab2_step_field!, velocity_field, Δt, χ, Gⁿ, G⁻, model.grid; only_active_cells = only_active_interior_cells) diff --git a/src/TimeSteppers/quasi_adams_bashforth_2.jl b/src/TimeSteppers/quasi_adams_bashforth_2.jl index c589dfee73..f8348777f7 100644 --- a/src/TimeSteppers/quasi_adams_bashforth_2.jl +++ b/src/TimeSteppers/quasi_adams_bashforth_2.jl @@ -170,4 +170,5 @@ end @inbounds u[i, j, k] += convert(FT, Δt) * ((one_point_five + χ) * Gⁿ[i, j, k] - (oh_point_five + χ) * G⁻[i, j, k]) end -@kernel ab2_step_field!(::FunctionField, Δt, χ, Gⁿ, G⁻, grid) = nothing +@kernel ab2_step_field!(::FunctionField, Δt, χ, Gⁿ, G⁻, grid) = nothing +@kernel ab2_step_field!(::FunctionField, Δt, χ, Gⁿ, G⁻, grid::ActiveCellsIBG) = nothing From e880b2c2853e59872f1009744792daca423ea8dd Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sun, 26 Nov 2023 11:45:28 -0500 Subject: [PATCH 007/567] some organizing --- src/Advection/vector_invariant_advection.jl | 31 ++++++++++----------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/src/Advection/vector_invariant_advection.jl b/src/Advection/vector_invariant_advection.jl index a16af6f950..c782a934a6 100644 --- a/src/Advection/vector_invariant_advection.jl +++ b/src/Advection/vector_invariant_advection.jl @@ -167,10 +167,7 @@ Base.show(io::IO, a::VectorInvariant{N, FT}) where {N, FT} = ##### Convenience for WENO Vector Invariant ##### -# VectorInvariant{N, FT, M, Z (vorticity scheme), ZS, V (vertical scheme), K (kinetic energy gradient scheme) -const WENOVectorInvariant = VectorInvariant{<:Any, <:Any, <:Any, <:WENO, <:Any, <:WENO, <:WENO} - -nothing_to_default(user_value, default) = isnothing(user_value) ? default : user_value +nothing_to_default(user_value; default) = isnothing(user_value) ? default : user_value """ function WENOVectorInvariant(; upwinding = nothing, @@ -189,23 +186,23 @@ function WENOVectorInvariant(; upwinding = nothing, weno_kw...) if isnothing(order) # apply global defaults - vorticity_order = nothing_to_default(vorticity_order, default=9) - vertical_order = nothing_to_default(vertical_order, default=5) - divergence_order = nothing_to_default(divergence_order, default=5) - kinetic_energy_gradient_order = nothing_to_default(kinetic_energy_gradient_order, default=5) + vorticity_order = nothing_to_default(vorticity_order, default = 9) + vertical_order = nothing_to_default(vertical_order, default = 5) + divergence_order = nothing_to_default(divergence_order, default = 5) + kinetic_energy_gradient_order = nothing_to_default(kinetic_energy_gradient_order, default = 5) else # apply user supplied `order` unless overridden by more specific value - vorticity_order = nothing_to_default(vorticity_order, default=order) - vertical_order = nothing_to_default(vertical_order, default=order) - divergence_order = nothing_to_default(divergence_order, default=order) - kinetic_energy_gradient_order = nothing_to_default(kinetic_energy_gradient_order, default=order) + vorticity_order = nothing_to_default(vorticity_order, default = order) + vertical_order = nothing_to_default(vertical_order, default = order) + divergence_order = nothing_to_default(divergence_order, default = order) + kinetic_energy_gradient_order = nothing_to_default(kinetic_energy_gradient_order, default = order) end - vorticity_scheme = WENO(; order=vorticity_order, weno_kw...) - vertical_scheme = WENO(; order=vertical_order, weno_kw...) - kinetic_energy_gradient_scheme = WENO(; order=kinetic_energy_gradient_order, weno_kw...) - divergence_scheme = WENO(; order=divergence_order, weno_kw...) + vorticity_scheme = WENO(; order = vorticity_order, weno_kw...) + vertical_scheme = WENO(; order = vertical_order, weno_kw...) + kinetic_energy_gradient_scheme = WENO(; order = kinetic_energy_gradient_order, weno_kw...) + divergence_scheme = WENO(; order = divergence_order, weno_kw...) - default_upwinding = OnlySelfUpwinding(cross_scheme=divergence_scheme) + default_upwinding = OnlySelfUpwinding(cross_scheme = divergence_scheme) upwinding = nothing_to_default(upwinding; default = default_upwinding) schemes = (vorticity_scheme, vertical_scheme, kinetic_energy_gradient_scheme, divergence_scheme) From 6f4aaad47a31e7036c41f02d83da9a4f967296f7 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sun, 26 Nov 2023 13:24:16 -0500 Subject: [PATCH 008/567] hmmm --- src/ImmersedBoundaries/active_cells_map.jl | 6 +++--- .../vertically_implicit_diffusion_solver.jl | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/ImmersedBoundaries/active_cells_map.jl b/src/ImmersedBoundaries/active_cells_map.jl index d52e3db5a2..c8256b9cf2 100644 --- a/src/ImmersedBoundaries/active_cells_map.jl +++ b/src/ImmersedBoundaries/active_cells_map.jl @@ -6,9 +6,10 @@ using KernelAbstractions: @kernel, @index import Oceananigans.Utils: active_cells_work_layout, use_only_active_interior_cells -import Oceananigans.Solvers: solve_batched_tridiagonal_system_kernel! using Oceananigans.Solvers: solve_batched_tridiagonal_system_z!, ZDirection +import Oceananigans.Solvers: solve_batched_tridiagonal_system_kernel! + const ActiveCellsIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractArray} const ActiveSurfaceIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractArray} @@ -126,9 +127,8 @@ function active_cells_surface_map(ibg) end @kernel function solve_batched_tridiagonal_system_kernel!(ϕ, a, b, c, f, t, grid::ActiveSurfaceIBG, p, args, tridiagonal_direction::ZDirection) + Nz = size(grid, 3) idx = @index(Global, Linear) i, j = active_linear_index_to_surface_tuple(idx, grid) - Nz = size(grid, 3) - solve_batched_tridiagonal_system_z!(i, j, Nz, ϕ, a, b, c, f, t, grid, p, args, tridiagonal_direction) end \ No newline at end of file diff --git a/src/TurbulenceClosures/vertically_implicit_diffusion_solver.jl b/src/TurbulenceClosures/vertically_implicit_diffusion_solver.jl index f3b3e05c3a..56933e77e9 100644 --- a/src/TurbulenceClosures/vertically_implicit_diffusion_solver.jl +++ b/src/TurbulenceClosures/vertically_implicit_diffusion_solver.jl @@ -178,7 +178,8 @@ function implicit_step!(field::Field, diffusivity_fields, tracer_index, clock, - Δt; kwargs...) + Δt; + kwargs...) loc = location(field) From f0b59a3d34442ec3d24a20e9bd20b901d02a32fd Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 27 Nov 2023 17:44:50 -0500 Subject: [PATCH 009/567] improve speed --- src/Advection/tracer_advection_operators.jl | 24 +++++++++++++++++++++ src/Advection/vector_invariant_advection.jl | 13 ++++++----- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/src/Advection/tracer_advection_operators.jl b/src/Advection/tracer_advection_operators.jl index b01f2471ff..f6bd834af5 100644 --- a/src/Advection/tracer_advection_operators.jl +++ b/src/Advection/tracer_advection_operators.jl @@ -1,6 +1,24 @@ using Oceananigans.Operators: Vᶜᶜᶜ using Oceananigans.Fields: ZeroField +struct ThreeDimensionalTracerAdvection{N, FT, A, B, C} <: AbstractAdvectionScheme{N, FT} + x :: A + y :: B + z :: C + + ThreeDimensionalTracerAdvection{N, FT}(x::A, y::B, z::C) where {N, FT, A, B, C} = new{N, FT, A, B, C}(x, y, z) +end + +function ThreeDimensionalTracerAdvection(; x, y, z) + Nx = required_halo_size(x) + Ny = required_halo_size(y) + Nz = required_halo_size(z) + + FT = eltype(x) + + return ThreeDimensionalTracerAdvection{max(Nx, Ny, Nz), FT}(x, y, z) +end + @inline _advective_tracer_flux_x(args...) = advective_tracer_flux_x(args...) @inline _advective_tracer_flux_y(args...) = advective_tracer_flux_y(args...) @inline _advective_tracer_flux_z(args...) = advective_tracer_flux_z(args...) @@ -32,3 +50,9 @@ which ends up at the location `ccc`. δyᵃᶜᵃ(i, j, k, grid, _advective_tracer_flux_y, advection, U.v, c) + δzᵃᵃᶜ(i, j, k, grid, _advective_tracer_flux_z, advection, U.w, c)) end + +@inline function div_Uc(i, j, k, grid, advection::ThreeDimensionalTracerAdvection, U, c) + return 1/Vᶜᶜᶜ(i, j, k, grid) * (δxᶜᵃᵃ(i, j, k, grid, _advective_tracer_flux_x, advection.x, U.u, c) + + δyᵃᶜᵃ(i, j, k, grid, _advective_tracer_flux_y, advection.y, U.v, c) + + δzᵃᵃᶜ(i, j, k, grid, _advective_tracer_flux_z, advection.z, U.w, c)) +end diff --git a/src/Advection/vector_invariant_advection.jl b/src/Advection/vector_invariant_advection.jl index c782a934a6..dde715f7fc 100644 --- a/src/Advection/vector_invariant_advection.jl +++ b/src/Advection/vector_invariant_advection.jl @@ -111,9 +111,9 @@ Vector Invariant, Dimension-by-dimension reconstruction function VectorInvariant(; vorticity_scheme = EnstrophyConserving(), vorticity_stencil = VelocityStencil(), vertical_scheme = EnergyConserving(), - kinetic_energy_gradient_scheme = vertical_scheme, divergence_scheme = vertical_scheme, - upwinding = OnlySelfUpwinding(; cross_scheme = vertical_scheme), + kinetic_energy_gradient_scheme = divergence_scheme, + upwinding = OnlySelfUpwinding(; cross_scheme = divergence_scheme), multi_dimensional_stencil = false) N = required_halo_size(vorticity_scheme) @@ -132,7 +132,6 @@ end const MultiDimensionalVectorInvariant = VectorInvariant{<:Any, <:Any, true} # VectorInvariant{N, FT, M, Z (vorticity scheme) -const MultiDimensionalVectorInvariant = VectorInvariant{<:Any, <:Any, true} const VectorInvariantEnergyConserving = VectorInvariant{<:Any, <:Any, <:Any, <:EnergyConserving} const VectorInvariantEnstrophyConserving = VectorInvariant{<:Any, <:Any, <:Any, <:EnstrophyConserving} const VectorInvariantUpwindVorticity = VectorInvariant{<:Any, <:Any, <:Any, <:AbstractUpwindBiasedAdvectionScheme} @@ -145,10 +144,10 @@ const VectorInvariantKEGradientEnergyConserving = VectorInvariant{<:Any, <:Any, const VectorInvariantKineticEnergyUpwinding = VectorInvariant{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractUpwindBiasedAdvectionScheme} -# VectorInvariant{N, FT, M, Z, ZS, V, K, D, U (upwinding) -const VectorInvariantCrossVerticalUpwinding = VectorInvariant{<:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractUpwindBiasedAdvectionScheme, <:Any, <:Any, <:CrossAndSelfUpwinding} -const VectorInvariantSelfVerticalUpwinding = VectorInvariant{<:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractUpwindBiasedAdvectionScheme, <:Any, <:Any, <:OnlySelfUpwinding} -const VectorInvariantVelocityVerticalUpwinding = VectorInvariant{<:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractUpwindBiasedAdvectionScheme, <:Any, <:Any, <:VelocityUpwinding} +# VectorInvariant{N, FT, M, Z, ZS, V, K, D, U (upwinding) +const VectorInvariantCrossVerticalUpwinding = VectorInvariant{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractUpwindBiasedAdvectionScheme, <:CrossAndSelfUpwinding} +const VectorInvariantSelfVerticalUpwinding = VectorInvariant{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractUpwindBiasedAdvectionScheme, <:OnlySelfUpwinding} +const VectorInvariantVelocityVerticalUpwinding = VectorInvariant{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractUpwindBiasedAdvectionScheme, <:VelocityUpwinding} Base.summary(a::VectorInvariant) = string("Vector Invariant, Dimension-by-dimension reconstruction") Base.summary(a::MultiDimensionalVectorInvariant) = string("Vector Invariant, Multidimensional reconstruction") From 5dddbb9e24a2e95ba881c0c7e7169cf3d71d23c1 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 27 Nov 2023 20:44:00 -0500 Subject: [PATCH 010/567] now we get going --- .../store_hydrostatic_free_surface_tendencies.jl | 6 ++++-- src/TimeSteppers/store_tendencies.jl | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/store_hydrostatic_free_surface_tendencies.jl b/src/Models/HydrostaticFreeSurfaceModels/store_hydrostatic_free_surface_tendencies.jl index 469fb62c33..24d26c213c 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/store_hydrostatic_free_surface_tendencies.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/store_hydrostatic_free_surface_tendencies.jl @@ -4,6 +4,7 @@ using Oceananigans.TimeSteppers: store_field_tendencies! using Oceananigans: prognostic_fields using Oceananigans.Grids: AbstractGrid +using Oceananigans.ImmersedBoundaries: use_only_active_interior_cells using Oceananigans.Utils: launch! @@ -27,7 +28,7 @@ function store_free_surface_tendency!(::ExplicitFreeSurface, model) end """ Store previous source terms before updating them. """ -function store_tendencies!(model::HydrostaticFreeSurfaceModel) +function store_tendencies!(model::HydrostaticFreeSurfaceModel; only_active_cells = use_only_active_interior_cells(model.grid)) prognostic_field_names = keys(prognostic_fields(model)) three_dimensional_prognostic_field_names = filter(name -> name != :η, prognostic_field_names) @@ -37,7 +38,8 @@ function store_tendencies!(model::HydrostaticFreeSurfaceModel) store_field_tendencies!, model.timestepper.G⁻[field_name], model.grid, - model.timestepper.Gⁿ[field_name]) + model.timestepper.Gⁿ[field_name]; + only_active_cells) end diff --git a/src/TimeSteppers/store_tendencies.jl b/src/TimeSteppers/store_tendencies.jl index d5bae5d376..ea700ef1e2 100644 --- a/src/TimeSteppers/store_tendencies.jl +++ b/src/TimeSteppers/store_tendencies.jl @@ -17,7 +17,7 @@ end end """ Store previous source terms before updating them. """ -function store_tendencies!(model; only_active_cells = only_active_interior_cells(model.grid)) +function store_tendencies!(model; only_active_cells = use_only_active_interior_cells(model.grid)) model_fields = prognostic_fields(model) for field_name in keys(model_fields) From 90b0f7a7b040e301b1c30d14ae50ce20161571e5 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 27 Nov 2023 21:36:44 -0500 Subject: [PATCH 011/567] check it out --- src/DistributedComputations/distributed_grids.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/DistributedComputations/distributed_grids.jl b/src/DistributedComputations/distributed_grids.jl index fb5567e91d..2663fc843b 100644 --- a/src/DistributedComputations/distributed_grids.jl +++ b/src/DistributedComputations/distributed_grids.jl @@ -41,7 +41,7 @@ end @inline local_sizes(N, R::Fractional) = Tuple(ceil(Int, N * r) for r in R.sizes) @inline function local_sizes(N, R::Sizes) if N != sum(R.sizes) - @warn "The domain size specified in the architecture $(R.sizes) is inconsistent + @warn "The domain size specified in the architecture $(sum(R.sizes)) is inconsistent with the grid size $N: using the architecture-specified size" end return R.sizes @@ -130,6 +130,8 @@ function LatitudeLongitudeGrid(arch::Distributed, φl = partition(latitude, nφ, arch, 2) zl = partition(z, nz, arch, 3) + @info arch.local_rank longitude latitude λl φl + # Calculate all direction (which might be stretched) # A direction is regular if the domain passed is a Tuple{<:Real, <:Real}, # it is stretched if being passed is a function or vector (as for the VerticallyStretchedRectilinearGrid) From 6ce5b45de5fba6475a32fbfb36092f023b45e6ef Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 27 Nov 2023 21:47:19 -0500 Subject: [PATCH 012/567] check bathymetry --- src/DistributedComputations/distributed_fields.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/DistributedComputations/distributed_fields.jl b/src/DistributedComputations/distributed_fields.jl index ebffdf4f76..514a7434c9 100644 --- a/src/DistributedComputations/distributed_fields.jl +++ b/src/DistributedComputations/distributed_fields.jl @@ -39,6 +39,7 @@ end function set!(u::DistributedField, v::Union{Array, CuArray}) gsize = global_size(architecture(u), size(u)) + @show gsize size(v) size(u) if size(v) == size(u) f = arch_array(architecture(u), v) u .= f From 2875984f6523fe3d81c9149a695b49b92c56548f Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 27 Nov 2023 21:49:49 -0500 Subject: [PATCH 013/567] fixit --- .../distributed_fields.jl | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/DistributedComputations/distributed_fields.jl b/src/DistributedComputations/distributed_fields.jl index 514a7434c9..878646662e 100644 --- a/src/DistributedComputations/distributed_fields.jl +++ b/src/DistributedComputations/distributed_fields.jl @@ -39,17 +39,19 @@ end function set!(u::DistributedField, v::Union{Array, CuArray}) gsize = global_size(architecture(u), size(u)) - @show gsize size(v) size(u) - if size(v) == size(u) - f = arch_array(architecture(u), v) - u .= f - return u - elseif size(v) == gsize + if size(v) == gsize f = partition_global_array(architecture(u), v, size(u)) u .= f return u else - throw(ArgumentError("ERROR: DimensionMismatch: array could not be set to match destination field")) + try + f = arch_array(architecture(u), v) + u .= f + return u + + catch + throw(ArgumentError("ERROR: DimensionMismatch: array could not be set to match destination field")) + end end end From 6cd7444160e37d918f9b8ad200f9189f77a62718 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 27 Nov 2023 21:51:36 -0500 Subject: [PATCH 014/567] rmove distributed --- src/DistributedComputations/distributed_grids.jl | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/DistributedComputations/distributed_grids.jl b/src/DistributedComputations/distributed_grids.jl index 2663fc843b..27064906bd 100644 --- a/src/DistributedComputations/distributed_grids.jl +++ b/src/DistributedComputations/distributed_grids.jl @@ -130,8 +130,6 @@ function LatitudeLongitudeGrid(arch::Distributed, φl = partition(latitude, nφ, arch, 2) zl = partition(z, nz, arch, 3) - @info arch.local_rank longitude latitude λl φl - # Calculate all direction (which might be stretched) # A direction is regular if the domain passed is a Tuple{<:Real, <:Real}, # it is stretched if being passed is a function or vector (as for the VerticallyStretchedRectilinearGrid) From 8319dbcdfd1f04fd3824f60d0e8224b38213c0e4 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Fri, 1 Dec 2023 14:27:48 -0500 Subject: [PATCH 015/567] test it like this --- src/ImmersedBoundaries/active_cells_map.jl | 12 ++++++------ .../hydrostatic_free_surface_ab2_step.jl | 6 ++---- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/ImmersedBoundaries/active_cells_map.jl b/src/ImmersedBoundaries/active_cells_map.jl index c8256b9cf2..e9c097fda7 100644 --- a/src/ImmersedBoundaries/active_cells_map.jl +++ b/src/ImmersedBoundaries/active_cells_map.jl @@ -126,9 +126,9 @@ function active_cells_surface_map(ibg) return smaller_indices end -@kernel function solve_batched_tridiagonal_system_kernel!(ϕ, a, b, c, f, t, grid::ActiveSurfaceIBG, p, args, tridiagonal_direction::ZDirection) - Nz = size(grid, 3) - idx = @index(Global, Linear) - i, j = active_linear_index_to_surface_tuple(idx, grid) - solve_batched_tridiagonal_system_z!(i, j, Nz, ϕ, a, b, c, f, t, grid, p, args, tridiagonal_direction) -end \ No newline at end of file +# @kernel function solve_batched_tridiagonal_system_kernel!(ϕ, a, b, c, f, t, grid::ActiveSurfaceIBG, p, args, tridiagonal_direction::ZDirection) +# Nz = size(grid, 3) +# idx = @index(Global, Linear) +# i, j = active_linear_index_to_surface_tuple(idx, grid) +# solve_batched_tridiagonal_system_z!(i, j, Nz, ϕ, a, b, c, f, t, grid, p, args, tridiagonal_direction) +# end \ No newline at end of file diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl index 6cf8f349db..a0709d9d72 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl @@ -58,8 +58,7 @@ function ab2_step_velocities!(velocities, model, Δt, χ) model.diffusivity_fields, nothing, model.clock, - Δt; - only_active_cells = only_active_surface_cells) + Δt) end return nothing @@ -95,8 +94,7 @@ function ab2_step_tracers!(tracers, model, Δt, χ) model.diffusivity_fields, Val(tracer_index), model.clock, - Δt; - only_active_cells = only_active_surface_cells) + Δt) end return nothing From b78b04263cac700a579eb1338ddaa1b03e5c8559 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sat, 2 Dec 2023 19:10:03 -0500 Subject: [PATCH 016/567] I hope it works! --- src/ImmersedBoundaries/active_cells_map.jl | 87 ++++++++++++++++--- ...static_free_surface_boundary_tendencies.jl | 20 ++++- ...ute_hydrostatic_free_surface_tendencies.jl | 18 ++-- .../hydrostatic_free_surface_ab2_step.jl | 12 +-- .../split_explicit_free_surface_kernels.jl | 12 +-- .../compute_nonhydrostatic_tendencies.jl | 18 ++-- src/TimeSteppers/quasi_adams_bashforth_2.jl | 14 +-- src/TimeSteppers/store_tendencies.jl | 12 +-- 8 files changed, 117 insertions(+), 76 deletions(-) diff --git a/src/ImmersedBoundaries/active_cells_map.jl b/src/ImmersedBoundaries/active_cells_map.jl index e9c097fda7..4538470d69 100644 --- a/src/ImmersedBoundaries/active_cells_map.jl +++ b/src/ImmersedBoundaries/active_cells_map.jl @@ -7,23 +7,51 @@ import Oceananigans.Utils: active_cells_work_layout, use_only_active_interior_cells using Oceananigans.Solvers: solve_batched_tridiagonal_system_z!, ZDirection +using Oceananigans.DistributedComputations: DistributedGrid import Oceananigans.Solvers: solve_batched_tridiagonal_system_kernel! -const ActiveCellsIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractArray} -const ActiveSurfaceIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractArray} +const ActiveCellsIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, Union{<:AbstractArray, <:NamedTuple}} +const ActiveSurfaceIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractArray} +const DistributedActiveCellsIBG = ImmersedBoundaryGrid{<:DistributedGrid, <:Any, <:Any, <:Any, <:Any, <:Any, <:NamedTuple} struct InteriorMap end struct SurfaceMap end +struct WestMap end +struct EastMap end +struct SouthMap end +struct NorthMap end + +active_map(::Val{:west}) = WestMap() +active_map(::Val{:east}) = EastMap() +active_map(::Val{:south}) = SouthMap() +active_map(::Val{:north}) = NorthMap() + @inline use_only_active_surface_cells(::AbstractGrid) = nothing @inline use_only_active_interior_cells(::ActiveCellsIBG) = InteriorMap() @inline use_only_active_surface_cells(::ActiveSurfaceIBG) = SurfaceMap() +@inline use_only_active_west_cells(::DistributedActiveCellsIBG) = WestMap() +@inline use_only_active_east_cells(::DistributedActiveCellsIBG) = EastMap() +@inline use_only_active_south_cells(::DistributedActiveCellsIBG) = SouthMap() +@inline use_only_active_nouth_cells(::DistributedActiveCellsIBG) = NorthMap() + +@inline active_cells_work_layout(group, size, ::InteriorMap, grid::ActiveCellsIBG) = min(length(grid.interior_active_cells), 256), length(grid.interior_active_cells) +@inline active_cells_work_layout(group, size, ::SurfaceMap, grid::ActiveSurfaceIBG) = min(length(grid.surface_active_cells), 256), length(grid.surface_active_cells) + +@inline active_cells_work_layout(group, size, ::InteriorMap, grid::DistributedActiveCellsIBG) = min(length(grid.interior_active_cells.interior), 256), length(grid.interior_active_cells.interior) +@inline active_cells_work_layout(group, size, ::WestMap, grid::DistributedActiveCellsIBG) = min(length(grid.interior_active_cells.west), 256), length(grid.interior_active_cells.west) +@inline active_cells_work_layout(group, size, ::EastMap, grid::DistributedActiveCellsIBG) = min(length(grid.interior_active_cells.east), 256), length(grid.interior_active_cells.east) +@inline active_cells_work_layout(group, size, ::SouthMap, grid::DistributedActiveCellsIBG) = min(length(grid.interior_active_cells.south), 256), length(grid.interior_active_cells.south) +@inline active_cells_work_layout(group, size, ::NorthMap, grid::DistributedActiveCellsIBG) = min(length(grid.interior_active_cells.north), 256), length(grid.interior_active_cells.north) + +@inline active_linear_index_to_tuple(idx, ::InteriorMap, grid::ActiveCellsIBG) = Base.map(Int, grid.interior_active_cells[idx]) +@inline active_linear_index_to_tuple(idx, ::InteriorMap, grid::DistributedActiveCellsIBG) = Base.map(Int, grid.interior_active_cells.interior[idx]) +@inline active_linear_index_to_tuple(idx, ::WestMap, grid::DistributedActiveCellsIBG) = Base.map(Int, grid.interior_active_cells.west[idx]) +@inline active_linear_index_to_tuple(idx, ::EastMap, grid::DistributedActiveCellsIBG) = Base.map(Int, grid.interior_active_cells.east[idx]) +@inline active_linear_index_to_tuple(idx, ::SouthMap, grid::DistributedActiveCellsIBG) = Base.map(Int, grid.interior_active_cells.south[idx]) +@inline active_linear_index_to_tuple(idx, ::NorthMap, grid::DistributedActiveCellsIBG) = Base.map(Int, grid.interior_active_cells.north[idx]) -@inline active_cells_work_layout(group, size, ::InteriorMap, grid::ActiveCellsIBG) = min(length(grid.interior_active_cells), 256), length(grid.interior_active_cells) -@inline active_cells_work_layout(group, size, ::SurfaceMap, grid::ActiveSurfaceIBG) = min(length(grid.surface_active_cells), 256), length(grid.surface_active_cells) - -@inline active_linear_index_to_interior_tuple(idx, grid::ActiveCellsIBG) = Base.map(Int, grid.interior_active_cells[idx]) @inline active_linear_index_to_surface_tuple(idx, grid::ActiveSurfaceIBG) = Base.map(Int, grid.surface_active_cells[idx]) function ImmersedBoundaryGrid(grid, ib; active_cells_map::Bool = true) @@ -34,7 +62,6 @@ function ImmersedBoundaryGrid(grid, ib; active_cells_map::Bool = true) # Create the cells map on the CPU, then switch it to the GPU if active_cells_map interior_map = active_cells_interior_map(ibg) - interior_map = arch_array(architecture(ibg), interior_map) surface_map = active_cells_surface_map(ibg) surface_map = arch_array(architecture(ibg), surface_map) else @@ -82,6 +109,7 @@ function active_cells_interior_map(ibg) # Cannot findall on the entire field because we incur on OOM errors active_indices = IndicesType[] active_indices = findall_active_indices!(active_indices, active_cells_field, ibg, IndicesType) + active_indices = arch_array(architecture(ibg), active_indices) return active_indices end @@ -126,9 +154,42 @@ function active_cells_surface_map(ibg) return smaller_indices end -# @kernel function solve_batched_tridiagonal_system_kernel!(ϕ, a, b, c, f, t, grid::ActiveSurfaceIBG, p, args, tridiagonal_direction::ZDirection) -# Nz = size(grid, 3) -# idx = @index(Global, Linear) -# i, j = active_linear_index_to_surface_tuple(idx, grid) -# solve_batched_tridiagonal_system_z!(i, j, Nz, ϕ, a, b, c, f, t, grid, p, args, tridiagonal_direction) -# end \ No newline at end of file +# In case of a `DistributedGrid` we want to have different maps depending on the +# partitioning of the domain +function active_cells_interior_map(ibg::ImmersedBoundaryGrid{<:DistributedGrid}) + active_cells_field = compute_interior_active_cells(ibg) + + N = maximum(size(ibg)) + IntType = N > MAXUInt8 ? (N > MAXUInt16 ? (N > MAXUInt32 ? UInt64 : UInt32) : UInt16) : UInt8 + + IndicesType = Tuple{IntType, IntType, IntType} + + # Cannot findall on the entire field because we incur on OOM errors + active_indices = IndicesType[] + active_indices = findall_active_indices!(active_indices, active_cells_field, ibg, IndicesType) + active_indices = separate_active_indices!(active_indices, ibg) + + return active_indices +end + +function separate_active_indices!(indices, ibg) + arch = architecture(ibg) + Hx, Hy, _ = halo_size(ibg) + Nx, Ny, _ = size(ibg) + Rx, Ry, _ = arch.ranks + west = Rx > 1 ? findall(idx -> idx[1] <= Hx, indices) : nothing + east = Rx > 1 ? findall(idx -> idx[1] >= Nx-Hx, indices) : nothing + south = Ry > 1 ? findall(idx -> idx[2] <= Hy, indices) : nothing + north = Ry > 1 ? findall(idx -> idx[2] <= Ny-Hy, indices) : nothing + + interior = findall(idx -> !(idx ∈ west) && !(idx ∈ east) && !(idx ∈ south) && !(idx ∈ north), indices) + + interior = arch_array(architecture(ibg), interior) + + west = west isa Nothing ? nothing : arch_array(architecture(ibg), west) + east = east isa Nothing ? nothing : arch_array(architecture(ibg), east) + south = south isa Nothing ? nothing : arch_array(architecture(ibg), south) + north = north isa Nothing ? nothing : arch_array(architecture(ibg), north) + + return (; interior, west, east, south, north) +end \ No newline at end of file diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl index ab510f7c76..b9279574c3 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl @@ -8,6 +8,8 @@ using Oceananigans.Models.NonhydrostaticModels: boundary_tendency_kernel_paramet import Oceananigans.Models: compute_boundary_tendencies! +using Oceananigans.ImmersedBoundaries: active_map, DistributedActiveCellsIBG + # We assume here that top/bottom BC are always synched (no partitioning in z) function compute_boundary_tendencies!(model::HydrostaticFreeSurfaceModel) grid = model.grid @@ -21,12 +23,28 @@ function compute_boundary_tendencies!(model::HydrostaticFreeSurfaceModel) compute_auxiliaries!(model; w_parameters, p_parameters, κ_parameters) # parameters for communicating North / South / East / West side + compute_boundary_tendency_contributions!(grid, arch, model) + + return nothing +end + +function compute_boundary_tendency_contributions!(grid, arch, model) kernel_parameters = boundary_tendency_kernel_parameters(grid, arch) - compute_hydrostatic_free_surface_tendency_contributions!(model, kernel_parameters) + compute_hydrostatic_free_surface_tendency_contributions!(grid, model, kernel_parameters) return nothing end +function compute_boundary_tendency_contributions!(grid::DistributedActiveCellsIBG, arch, model) + maps = grid.interior_active_cells + + for (name, map) in zip(keys(maps), maps) + if name != :interior && !isnothing(map) + compute_hydrostatic_free_surface_tendency_contributions!(model, :xyz; only_active_cells = active_map(Val(name))) + end + end +end + # w needs computing in the range - H + 1 : 0 and N - 1 : N + H - 1 function boundary_w_kernel_parameters(grid, arch) Nx, Ny, _ = size(grid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl index 996eaf1ad4..8ef6d8ea1e 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl @@ -14,7 +14,7 @@ import Oceananigans.Models: complete_communication_and_compute_boundary! import Oceananigans.Models: interior_tendency_kernel_parameters using Oceananigans.ImmersedBoundaries: use_only_active_interior_cells, ActiveCellsIBG, - InteriorMap, active_linear_index_to_interior_tuple + InteriorMap, active_linear_index_to_tuple """ compute_tendencies!(model::HydrostaticFreeSurfaceModel, callbacks) @@ -227,9 +227,9 @@ end @inbounds Gu[i, j, k] = hydrostatic_free_surface_u_velocity_tendency(i, j, k, grid, args...) end -@kernel function compute_hydrostatic_free_surface_Gu!(Gu, grid::ActiveCellsIBG, ::InteriorMap, args) +@kernel function compute_hydrostatic_free_surface_Gu!(Gu, grid::ActiveCellsIBG, map, args) idx = @index(Global, Linear) - i, j, k = active_linear_index_to_interior_tuple(idx, grid) + i, j, k = active_linear_index_to_tuple(idx, map, grid) @inbounds Gu[i, j, k] = hydrostatic_free_surface_u_velocity_tendency(i, j, k, grid, args...) end @@ -239,9 +239,9 @@ end @inbounds Gv[i, j, k] = hydrostatic_free_surface_v_velocity_tendency(i, j, k, grid, args...) end -@kernel function compute_hydrostatic_free_surface_Gv!(Gv, grid::ActiveCellsIBG, ::InteriorMap, args) +@kernel function compute_hydrostatic_free_surface_Gv!(Gv, grid::ActiveCellsIBG, map, args) idx = @index(Global, Linear) - i, j, k = active_linear_index_to_interior_tuple(idx, grid) + i, j, k = active_linear_index_to_tuple(idx, map, grid) @inbounds Gv[i, j, k] = hydrostatic_free_surface_v_velocity_tendency(i, j, k, grid, args...) end @@ -255,9 +255,9 @@ end @inbounds Gc[i, j, k] = hydrostatic_free_surface_tracer_tendency(i, j, k, grid, args...) end -@kernel function compute_hydrostatic_free_surface_Gc!(Gc, grid::ActiveCellsIBG, ::InteriorMap, args) +@kernel function compute_hydrostatic_free_surface_Gc!(Gc, grid::ActiveCellsIBG, map, args) idx = @index(Global, Linear) - i, j, k = active_linear_index_to_interior_tuple(idx, grid) + i, j, k = active_linear_index_to_tuple(idx, map, grid) @inbounds Gc[i, j, k] = hydrostatic_free_surface_tracer_tendency(i, j, k, grid, args...) end @@ -267,9 +267,9 @@ end @inbounds Ge[i, j, k] = hydrostatic_turbulent_kinetic_energy_tendency(i, j, k, grid, args...) end -@kernel function compute_hydrostatic_free_surface_Ge!(Ge, grid::ActiveCellsIBG, ::InteriorMap, args) +@kernel function compute_hydrostatic_free_surface_Ge!(Ge, grid::ActiveCellsIBG, map, args) idx = @index(Global, Linear) - i, j, k = active_linear_index_to_interior_tuple(idx, grid) + i, j, k = active_linear_index_to_tuple(idx, map, grid) @inbounds Ge[i, j, k] = hydrostatic_turbulent_kinetic_energy_tendency(i, j, k, grid, args...) end diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl index a0709d9d72..bc7dde1c31 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl @@ -37,17 +37,13 @@ end function ab2_step_velocities!(velocities, model, Δt, χ) - only_active_interior_cells = use_only_active_interior_cells(model.grid) - only_active_surface_cells = use_only_active_surface_cells(model.grid) - for (i, name) in enumerate((:u, :v)) Gⁿ = model.timestepper.Gⁿ[name] G⁻ = model.timestepper.G⁻[name] velocity_field = model.velocities[name] launch!(model.architecture, model.grid, :xyz, - ab2_step_field!, velocity_field, Δt, χ, Gⁿ, G⁻, model.grid; - only_active_cells = only_active_interior_cells) + ab2_step_field!, velocity_field, Δt, χ, Gⁿ, G⁻, model.grid) # TODO: let next implicit solve depend on previous solve + explicit velocity step # Need to distinguish between solver events and tendency calculation events. @@ -74,9 +70,6 @@ ab2_step_tracers!(::EmptyNamedTuple, model, Δt, χ) = nothing function ab2_step_tracers!(tracers, model, Δt, χ) - only_active_interior_cells = use_only_active_interior_cells(model.grid) - only_active_surface_cells = use_only_active_surface_cells(model.grid) - # Tracer update kernels for (tracer_index, tracer_name) in enumerate(propertynames(tracers)) Gⁿ = model.timestepper.Gⁿ[tracer_name] @@ -85,8 +78,7 @@ function ab2_step_tracers!(tracers, model, Δt, χ) closure = model.closure launch!(model.architecture, model.grid, :xyz, - ab2_step_field!, tracer_field, Δt, χ, Gⁿ, G⁻, model.grid; - only_active_cells = only_active_interior_cells) + ab2_step_field!, tracer_field, Δt, χ, Gⁿ, G⁻, model.grid) implicit_step!(tracer_field, model.timestepper.implicit_solver, diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 754ffa362b..e068a9874f 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -310,15 +310,6 @@ end end end -@kernel function barotropic_split_explicit_corrector_kernel!(u, v, U̅, V̅, U, V, Hᶠᶜ, Hᶜᶠ, grid::ActiveCellsIBG) - idx = @index(Global, Linear) - i, j, k = active_linear_index_to_interior_tuple(idx, grid) - @inbounds begin - u[i, j, k] = u[i, j, k] + (U̅[i, j] - U[i, j]) / Hᶠᶜ[i, j] - v[i, j, k] = v[i, j, k] + (V̅[i, j] - V[i, j]) / Hᶜᶠ[i, j] - end -end - # may need to do Val(Nk) since it may not be known at compile. Also figure out where to put H function barotropic_split_explicit_corrector!(u, v, free_surface, grid) sefs = free_surface.state @@ -332,8 +323,7 @@ function barotropic_split_explicit_corrector!(u, v, free_surface, grid) # add in "good" barotropic mode launch!(arch, grid, :xyz, barotropic_split_explicit_corrector_kernel!, - u, v, U̅, V̅, U, V, Hᶠᶜ, Hᶜᶠ, grid; - only_active_cells = use_only_active_interior_cells(grid)) + u, v, U̅, V̅, U, V, Hᶠᶜ, Hᶜᶠ, grid) return nothing end diff --git a/src/Models/NonhydrostaticModels/compute_nonhydrostatic_tendencies.jl b/src/Models/NonhydrostaticModels/compute_nonhydrostatic_tendencies.jl index 4e47a7b472..6f9380e619 100644 --- a/src/Models/NonhydrostaticModels/compute_nonhydrostatic_tendencies.jl +++ b/src/Models/NonhydrostaticModels/compute_nonhydrostatic_tendencies.jl @@ -4,7 +4,7 @@ using Oceananigans.Utils: work_layout using Oceananigans.Models: complete_communication_and_compute_boundary!, interior_tendency_kernel_parameters using Oceananigans.ImmersedBoundaries: use_only_active_interior_cells, ActiveCellsIBG, - InteriorMap, active_linear_index_to_interior_tuple + InteriorMap, active_linear_index_to_tuple import Oceananigans.TimeSteppers: compute_tendencies! @@ -138,9 +138,9 @@ end @inbounds Gu[i, j, k] = u_velocity_tendency(i, j, k, grid, args...) end -@kernel function compute_Gu!(Gu, grid::ActiveCellsIBG, ::InteriorMap, args) +@kernel function compute_Gu!(Gu, grid::ActiveCellsIBG, map::InteriorMap, args) idx = @index(Global, Linear) - i, j, k = active_linear_index_to_interior_tuple(idx, grid) + i, j, k = active_linear_index_to_tuple(idx, map, grid) @inbounds Gu[i, j, k] = u_velocity_tendency(i, j, k, grid, args...) end @@ -150,9 +150,9 @@ end @inbounds Gv[i, j, k] = v_velocity_tendency(i, j, k, grid, args...) end -@kernel function compute_Gv!(Gv, grid::ActiveCellsIBG, ::InteriorMap, args) +@kernel function compute_Gv!(Gv, grid::ActiveCellsIBG, map::InteriorMap, args) idx = @index(Global, Linear) - i, j, k = active_linear_index_to_interior_tuple(idx, grid) + i, j, k = active_linear_index_to_tuple(idx, map, grid) @inbounds Gv[i, j, k] = v_velocity_tendency(i, j, k, grid, args...) end @@ -162,9 +162,9 @@ end @inbounds Gw[i, j, k] = w_velocity_tendency(i, j, k, grid, args...) end -@kernel function compute_Gw!(Gw, grid::ActiveCellsIBG, ::InteriorMap, args) +@kernel function compute_Gw!(Gw, grid::ActiveCellsIBG, map, ::InteriorMap, args) idx = @index(Global, Linear) - i, j, k = active_linear_index_to_interior_tuple(idx, grid) + i, j, k = active_linear_index_to_tuple(idx, map, grid) @inbounds Gw[i, j, k] = w_velocity_tendency(i, j, k, grid, args...) end @@ -178,9 +178,9 @@ end @inbounds Gc[i, j, k] = tracer_tendency(i, j, k, grid, args...) end -@kernel function compute_Gc!(Gc, grid::ActiveCellsIBG, ::InteriorMap, args) +@kernel function compute_Gc!(Gc, grid::ActiveCellsIBG, map::InteriorMap, args) idx = @index(Global, Linear) - i, j, k = active_linear_index_to_interior_tuple(idx, grid) + i, j, k = active_linear_index_to_tuple(idx, map, grid) @inbounds Gc[i, j, k] = tracer_tendency(i, j, k, grid, args...) end diff --git a/src/TimeSteppers/quasi_adams_bashforth_2.jl b/src/TimeSteppers/quasi_adams_bashforth_2.jl index f8348777f7..f63dcf817f 100644 --- a/src/TimeSteppers/quasi_adams_bashforth_2.jl +++ b/src/TimeSteppers/quasi_adams_bashforth_2.jl @@ -1,7 +1,7 @@ using Oceananigans.Fields: FunctionField, location using Oceananigans.TurbulenceClosures: implicit_step! using Oceananigans.Utils: @apply_regionally, apply_regionally! -using Oceananigans.ImmersedBoundaries: ActiveCellsIBG, active_linear_index_to_interior_tuple +using Oceananigans.ImmersedBoundaries: ActiveCellsIBG, active_linear_index_to_tuple mutable struct QuasiAdamsBashforth2TimeStepper{FT, GT, IT} <: AbstractTimeStepper χ :: FT @@ -159,16 +159,4 @@ Time step velocity fields via the 2nd-order quasi Adams-Bashforth method @inbounds u[i, j, k] += convert(FT, Δt) * ((one_point_five + χ) * Gⁿ[i, j, k] - (oh_point_five + χ) * G⁻[i, j, k]) end -@kernel function ab2_step_field!(u, Δt, χ, Gⁿ, G⁻, grid::ActiveCellsIBG) - idx = @index(Global, Linear) - i, j, k = active_linear_index_to_interior_tuple(idx, grid) - - FT = eltype(χ) - one_point_five = convert(FT, 1.5) - oh_point_five = convert(FT, 0.5) - - @inbounds u[i, j, k] += convert(FT, Δt) * ((one_point_five + χ) * Gⁿ[i, j, k] - (oh_point_five + χ) * G⁻[i, j, k]) -end - @kernel ab2_step_field!(::FunctionField, Δt, χ, Gⁿ, G⁻, grid) = nothing -@kernel ab2_step_field!(::FunctionField, Δt, χ, Gⁿ, G⁻, grid::ActiveCellsIBG) = nothing diff --git a/src/TimeSteppers/store_tendencies.jl b/src/TimeSteppers/store_tendencies.jl index ea700ef1e2..b4a117e5b8 100644 --- a/src/TimeSteppers/store_tendencies.jl +++ b/src/TimeSteppers/store_tendencies.jl @@ -9,23 +9,15 @@ using Oceananigans.Utils: launch! @inbounds G⁻[i, j, k] = G⁰[i, j, k] end -""" Store source terms for `u`, `v`, and `w`. """ -@kernel function store_field_tendencies!(G⁻, grid::ActiveCellsIBG, G⁰) - idx = @index(Global, Linear) - i, j, k = active_linear_index_to_interior_tuple(idx, grid) - @inbounds G⁻[i, j, k] = G⁰[i, j, k] -end - """ Store previous source terms before updating them. """ -function store_tendencies!(model; only_active_cells = use_only_active_interior_cells(model.grid)) +function store_tendencies!(model) model_fields = prognostic_fields(model) for field_name in keys(model_fields) launch!(model.architecture, model.grid, :xyz, store_field_tendencies!, model.timestepper.G⁻[field_name], model.grid, - model.timestepper.Gⁿ[field_name]; - only_active_cells) + model.timestepper.Gⁿ[field_name]) end return nothing From 11e81431ff7fa8639c7a14de5568abf9c74e85dd Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sat, 2 Dec 2023 19:17:07 -0500 Subject: [PATCH 017/567] bugfix --- src/ImmersedBoundaries/active_cells_map.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ImmersedBoundaries/active_cells_map.jl b/src/ImmersedBoundaries/active_cells_map.jl index 4538470d69..65a0a48947 100644 --- a/src/ImmersedBoundaries/active_cells_map.jl +++ b/src/ImmersedBoundaries/active_cells_map.jl @@ -180,7 +180,7 @@ function separate_active_indices!(indices, ibg) west = Rx > 1 ? findall(idx -> idx[1] <= Hx, indices) : nothing east = Rx > 1 ? findall(idx -> idx[1] >= Nx-Hx, indices) : nothing south = Ry > 1 ? findall(idx -> idx[2] <= Hy, indices) : nothing - north = Ry > 1 ? findall(idx -> idx[2] <= Ny-Hy, indices) : nothing + north = Ry > 1 ? findall(idx -> idx[2] >= Ny-Hy, indices) : nothing interior = findall(idx -> !(idx ∈ west) && !(idx ∈ east) && !(idx ∈ south) && !(idx ∈ north), indices) From 2a2f77280a3132241224280e2343e48f1422257e Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sat, 2 Dec 2023 19:24:22 -0500 Subject: [PATCH 018/567] bugfix --- src/ImmersedBoundaries/active_cells_map.jl | 21 +++++++++++-------- ...static_free_surface_boundary_tendencies.jl | 4 ++-- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/ImmersedBoundaries/active_cells_map.jl b/src/ImmersedBoundaries/active_cells_map.jl index 65a0a48947..0fed88b291 100644 --- a/src/ImmersedBoundaries/active_cells_map.jl +++ b/src/ImmersedBoundaries/active_cells_map.jl @@ -177,19 +177,22 @@ function separate_active_indices!(indices, ibg) Hx, Hy, _ = halo_size(ibg) Nx, Ny, _ = size(ibg) Rx, Ry, _ = arch.ranks - west = Rx > 1 ? findall(idx -> idx[1] <= Hx, indices) : nothing - east = Rx > 1 ? findall(idx -> idx[1] >= Nx-Hx, indices) : nothing - south = Ry > 1 ? findall(idx -> idx[2] <= Hy, indices) : nothing - north = Ry > 1 ? findall(idx -> idx[2] >= Ny-Hy, indices) : nothing + west = Rx > 1 ? findall(idx -> idx[1] <= Hx, indices) : [] + east = Rx > 1 ? findall(idx -> idx[1] >= Nx-Hx, indices) : [] + south = Ry > 1 ? findall(idx -> idx[2] <= Hy, indices) : [] + north = Ry > 1 ? findall(idx -> idx[2] >= Ny-Hy, indices) : [] - interior = findall(idx -> !(idx ∈ west) && !(idx ∈ east) && !(idx ∈ south) && !(idx ∈ north), indices) + interior = findall(idx -> !(idx ∈ west) && + !(idx ∈ east) && + !(idx ∈ south) && + !(idx ∈ north), indices) interior = arch_array(architecture(ibg), interior) - west = west isa Nothing ? nothing : arch_array(architecture(ibg), west) - east = east isa Nothing ? nothing : arch_array(architecture(ibg), east) - south = south isa Nothing ? nothing : arch_array(architecture(ibg), south) - north = north isa Nothing ? nothing : arch_array(architecture(ibg), north) + west = arch_array(architecture(ibg), west) + east = arch_array(architecture(ibg), east) + south = arch_array(architecture(ibg), south) + north = arch_array(architecture(ibg), north) return (; interior, west, east, south, north) end \ No newline at end of file diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl index b9279574c3..3864607641 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl @@ -30,7 +30,7 @@ end function compute_boundary_tendency_contributions!(grid, arch, model) kernel_parameters = boundary_tendency_kernel_parameters(grid, arch) - compute_hydrostatic_free_surface_tendency_contributions!(grid, model, kernel_parameters) + compute_hydrostatic_free_surface_tendency_contributions!(model, kernel_parameters) return nothing end @@ -39,7 +39,7 @@ function compute_boundary_tendency_contributions!(grid::DistributedActiveCellsIB maps = grid.interior_active_cells for (name, map) in zip(keys(maps), maps) - if name != :interior && !isnothing(map) + if name != :interior && !isempy(map) compute_hydrostatic_free_surface_tendency_contributions!(model, :xyz; only_active_cells = active_map(Val(name))) end end From d91cd93d524393f07b5f0ce2426a2af6fcb76697 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sat, 2 Dec 2023 20:45:20 -0500 Subject: [PATCH 019/567] bugfix --- src/ImmersedBoundaries/active_cells_map.jl | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/ImmersedBoundaries/active_cells_map.jl b/src/ImmersedBoundaries/active_cells_map.jl index 0fed88b291..970b31d70b 100644 --- a/src/ImmersedBoundaries/active_cells_map.jl +++ b/src/ImmersedBoundaries/active_cells_map.jl @@ -13,7 +13,7 @@ import Oceananigans.Solvers: solve_batched_tridiagonal_system_kernel! const ActiveCellsIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, Union{<:AbstractArray, <:NamedTuple}} const ActiveSurfaceIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractArray} -const DistributedActiveCellsIBG = ImmersedBoundaryGrid{<:DistributedGrid, <:Any, <:Any, <:Any, <:Any, <:Any, <:NamedTuple} +const DistributedActiveCellsIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:DistributedGrid, <:Any, <:NamedTuple} struct InteriorMap end struct SurfaceMap end @@ -40,10 +40,10 @@ active_map(::Val{:north}) = NorthMap() @inline active_cells_work_layout(group, size, ::SurfaceMap, grid::ActiveSurfaceIBG) = min(length(grid.surface_active_cells), 256), length(grid.surface_active_cells) @inline active_cells_work_layout(group, size, ::InteriorMap, grid::DistributedActiveCellsIBG) = min(length(grid.interior_active_cells.interior), 256), length(grid.interior_active_cells.interior) -@inline active_cells_work_layout(group, size, ::WestMap, grid::DistributedActiveCellsIBG) = min(length(grid.interior_active_cells.west), 256), length(grid.interior_active_cells.west) -@inline active_cells_work_layout(group, size, ::EastMap, grid::DistributedActiveCellsIBG) = min(length(grid.interior_active_cells.east), 256), length(grid.interior_active_cells.east) -@inline active_cells_work_layout(group, size, ::SouthMap, grid::DistributedActiveCellsIBG) = min(length(grid.interior_active_cells.south), 256), length(grid.interior_active_cells.south) -@inline active_cells_work_layout(group, size, ::NorthMap, grid::DistributedActiveCellsIBG) = min(length(grid.interior_active_cells.north), 256), length(grid.interior_active_cells.north) +@inline active_cells_work_layout(group, size, ::WestMap, grid::DistributedActiveCellsIBG) = min(length(grid.interior_active_cells.west), 256), length(grid.interior_active_cells.west) +@inline active_cells_work_layout(group, size, ::EastMap, grid::DistributedActiveCellsIBG) = min(length(grid.interior_active_cells.east), 256), length(grid.interior_active_cells.east) +@inline active_cells_work_layout(group, size, ::SouthMap, grid::DistributedActiveCellsIBG) = min(length(grid.interior_active_cells.south), 256), length(grid.interior_active_cells.south) +@inline active_cells_work_layout(group, size, ::NorthMap, grid::DistributedActiveCellsIBG) = min(length(grid.interior_active_cells.north), 256), length(grid.interior_active_cells.north) @inline active_linear_index_to_tuple(idx, ::InteriorMap, grid::ActiveCellsIBG) = Base.map(Int, grid.interior_active_cells[idx]) @inline active_linear_index_to_tuple(idx, ::InteriorMap, grid::DistributedActiveCellsIBG) = Base.map(Int, grid.interior_active_cells.interior[idx]) @@ -75,6 +75,9 @@ function ImmersedBoundaryGrid(grid, ib; active_cells_map::Bool = true) surface_map) end +with_halo(halo, ibg::ActiveCellsIBG) = + ImmersedBoundaryGrid(with_halo(halo, ibg.underlying_grid), ibg.immersed_boundary; active_cells_map = true) + @inline active_cell(i, j, k, ibg) = !immersed_cell(i, j, k, ibg) @inline active_column(i, j, k, grid, column) = column[i, j, k] != 0 From 8eaf808bc7fe38b8db99b552111975de95f20740 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sat, 2 Dec 2023 21:18:20 -0500 Subject: [PATCH 020/567] bugfix --- src/ImmersedBoundaries/active_cells_map.jl | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/ImmersedBoundaries/active_cells_map.jl b/src/ImmersedBoundaries/active_cells_map.jl index 970b31d70b..56e3e1989d 100644 --- a/src/ImmersedBoundaries/active_cells_map.jl +++ b/src/ImmersedBoundaries/active_cells_map.jl @@ -28,13 +28,10 @@ active_map(::Val{:east}) = EastMap() active_map(::Val{:south}) = SouthMap() active_map(::Val{:north}) = NorthMap() -@inline use_only_active_surface_cells(::AbstractGrid) = nothing -@inline use_only_active_interior_cells(::ActiveCellsIBG) = InteriorMap() -@inline use_only_active_surface_cells(::ActiveSurfaceIBG) = SurfaceMap() -@inline use_only_active_west_cells(::DistributedActiveCellsIBG) = WestMap() -@inline use_only_active_east_cells(::DistributedActiveCellsIBG) = EastMap() -@inline use_only_active_south_cells(::DistributedActiveCellsIBG) = SouthMap() -@inline use_only_active_nouth_cells(::DistributedActiveCellsIBG) = NorthMap() +@inline use_only_active_surface_cells(::AbstractGrid) = nothing +@inline use_only_active_interior_cells(::ActiveCellsIBG) = InteriorMap() +@inline use_only_active_surface_cells(::ActiveSurfaceIBG) = SurfaceMap() +@inline use_only_active_interior_cells(::DistributedActiveCellsIBG) = InteriorMap() @inline active_cells_work_layout(group, size, ::InteriorMap, grid::ActiveCellsIBG) = min(length(grid.interior_active_cells), 256), length(grid.interior_active_cells) @inline active_cells_work_layout(group, size, ::SurfaceMap, grid::ActiveSurfaceIBG) = min(length(grid.surface_active_cells), 256), length(grid.surface_active_cells) @@ -159,7 +156,7 @@ end # In case of a `DistributedGrid` we want to have different maps depending on the # partitioning of the domain -function active_cells_interior_map(ibg::ImmersedBoundaryGrid{<:DistributedGrid}) +function active_cells_interior_map(ibg::ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:DistributedGrid}) active_cells_field = compute_interior_active_cells(ibg) N = maximum(size(ibg)) From 023e4045b854fc57935707ed58cb917297cb1635 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sun, 3 Dec 2023 09:49:11 -0500 Subject: [PATCH 021/567] couple of bugfixes --- .../distributed_architectures.jl | 2 +- src/ImmersedBoundaries/active_cells_map.jl | 20 +++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/DistributedComputations/distributed_architectures.jl b/src/DistributedComputations/distributed_architectures.jl index 0f1d5e7a52..33fcfc3da8 100644 --- a/src/DistributedComputations/distributed_architectures.jl +++ b/src/DistributedComputations/distributed_architectures.jl @@ -209,7 +209,7 @@ function Distributed(child_architecture = CPU(); partition = Partition(MPI.Comm_size(communicator))) if !(MPI.Initialized()) - @info "MPI has not been initialized, so we are calling MPI.Init()". + @info "MPI has not been initialized, so we are calling MPI.Init()" MPI.Init() end diff --git a/src/ImmersedBoundaries/active_cells_map.jl b/src/ImmersedBoundaries/active_cells_map.jl index 56e3e1989d..ccb03759b5 100644 --- a/src/ImmersedBoundaries/active_cells_map.jl +++ b/src/ImmersedBoundaries/active_cells_map.jl @@ -177,22 +177,22 @@ function separate_active_indices!(indices, ibg) Hx, Hy, _ = halo_size(ibg) Nx, Ny, _ = size(ibg) Rx, Ry, _ = arch.ranks - west = Rx > 1 ? findall(idx -> idx[1] <= Hx, indices) : [] - east = Rx > 1 ? findall(idx -> idx[1] >= Nx-Hx, indices) : [] - south = Ry > 1 ? findall(idx -> idx[2] <= Hy, indices) : [] - north = Ry > 1 ? findall(idx -> idx[2] >= Ny-Hy, indices) : [] + west = Rx > 1 ? findall(idx -> Int(idx[1]) <= Hx, indices) : Int[] + east = Rx > 1 ? findall(idx -> Int(idx[1]) >= Nx-Hx, indices) : Int[] + south = Ry > 1 ? findall(idx -> Int(idx[2]) <= Hy, indices) : Int[] + north = Ry > 1 ? findall(idx -> Int(idx[2]) >= Ny-Hy, indices) : Int[] + + west = arch_array(architecture(ibg), indices[west]) + east = arch_array(architecture(ibg), indices[east]) + south = arch_array(architecture(ibg), indices[south]) + north = arch_array(architecture(ibg), indices[north]) interior = findall(idx -> !(idx ∈ west) && !(idx ∈ east) && !(idx ∈ south) && !(idx ∈ north), indices) - interior = arch_array(architecture(ibg), interior) - - west = arch_array(architecture(ibg), west) - east = arch_array(architecture(ibg), east) - south = arch_array(architecture(ibg), south) - north = arch_array(architecture(ibg), north) + interior = arch_array(architecture(ibg), indices[interior]) return (; interior, west, east, south, north) end \ No newline at end of file From 81e70f699103a0b09d9bda24bb34991b6e5492c6 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sun, 3 Dec 2023 12:41:07 -0500 Subject: [PATCH 022/567] bugfix --- src/ImmersedBoundaries/active_cells_map.jl | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/ImmersedBoundaries/active_cells_map.jl b/src/ImmersedBoundaries/active_cells_map.jl index ccb03759b5..61a2b10c10 100644 --- a/src/ImmersedBoundaries/active_cells_map.jl +++ b/src/ImmersedBoundaries/active_cells_map.jl @@ -182,17 +182,21 @@ function separate_active_indices!(indices, ibg) south = Ry > 1 ? findall(idx -> Int(idx[2]) <= Hy, indices) : Int[] north = Ry > 1 ? findall(idx -> Int(idx[2]) >= Ny-Hy, indices) : Int[] - west = arch_array(architecture(ibg), indices[west]) - east = arch_array(architecture(ibg), indices[east]) - south = arch_array(architecture(ibg), indices[south]) - north = arch_array(architecture(ibg), indices[north]) + west = indices[west] + east = indices[east] + south = indices[south] + north = indices[north] interior = findall(idx -> !(idx ∈ west) && !(idx ∈ east) && !(idx ∈ south) && !(idx ∈ north), indices) - interior = arch_array(architecture(ibg), indices[interior]) - + interior = arch_array(architecture(ibg), indices[interior]) + west = arch_array(architecture(ibg), west ) + east = arch_array(architecture(ibg), east ) + south = arch_array(architecture(ibg), south) + north = arch_array(architecture(ibg), north) + return (; interior, west, east, south, north) end \ No newline at end of file From 7ab646ba31be6fc2f78d7a0430f1d543e66d42d7 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sun, 3 Dec 2023 16:29:58 -0500 Subject: [PATCH 023/567] bugfixes --- src/ImmersedBoundaries/active_cells_map.jl | 111 +++++++++--------- ...static_free_surface_boundary_tendencies.jl | 7 +- .../split_explicit_free_surface_kernels.jl | 12 +- .../distributed_hydrostatic_turbulence.jl | 23 ++-- 4 files changed, 79 insertions(+), 74 deletions(-) diff --git a/src/ImmersedBoundaries/active_cells_map.jl b/src/ImmersedBoundaries/active_cells_map.jl index 61a2b10c10..a10efa0a7e 100644 --- a/src/ImmersedBoundaries/active_cells_map.jl +++ b/src/ImmersedBoundaries/active_cells_map.jl @@ -1,4 +1,5 @@ using Oceananigans +using Oceananigans.Utils using Oceananigans.Grids: AbstractGrid using KernelAbstractions: @kernel, @index @@ -11,9 +12,10 @@ using Oceananigans.DistributedComputations: DistributedGrid import Oceananigans.Solvers: solve_batched_tridiagonal_system_kernel! -const ActiveCellsIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, Union{<:AbstractArray, <:NamedTuple}} const ActiveSurfaceIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractArray} const DistributedActiveCellsIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:DistributedGrid, <:Any, <:NamedTuple} +const SerialActiveCellsIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractArray} +const ActiveCellsIBG = Union{DistributedActiveCellsIBG, SerialActiveCellsIBG} struct InteriorMap end struct SurfaceMap end @@ -49,7 +51,7 @@ active_map(::Val{:north}) = NorthMap() @inline active_linear_index_to_tuple(idx, ::SouthMap, grid::DistributedActiveCellsIBG) = Base.map(Int, grid.interior_active_cells.south[idx]) @inline active_linear_index_to_tuple(idx, ::NorthMap, grid::DistributedActiveCellsIBG) = Base.map(Int, grid.interior_active_cells.north[idx]) -@inline active_linear_index_to_surface_tuple(idx, grid::ActiveSurfaceIBG) = Base.map(Int, grid.surface_active_cells[idx]) +@inline active_linear_index_to_tuple(idx, ::SurfaceMap, grid::ActiveSurfaceIBG) = Base.map(Int, grid.surface_active_cells[idx]) function ImmersedBoundaryGrid(grid, ib; active_cells_map::Bool = true) @@ -78,10 +80,15 @@ with_halo(halo, ibg::ActiveCellsIBG) = @inline active_cell(i, j, k, ibg) = !immersed_cell(i, j, k, ibg) @inline active_column(i, j, k, grid, column) = column[i, j, k] != 0 -function compute_interior_active_cells(ibg) - is_immersed_operation = KernelFunctionOperation{Center, Center, Center}(active_cell, ibg) +@kernel function _set_active_indices!(active_cells_field, grid) + i, j, k = @index(Global, NTuple) + @inbounds active_cells_field[i, j, k] = active_cell(i, j, k, grid) +end + +function compute_interior_active_cells(ibg; parameters = :xyz) active_cells_field = Field{Center, Center, Center}(ibg, Bool) - set!(active_cells_field, is_immersed_operation) + fill!(active_cells_field, false) + launch!(architecture(ibg), ibg, parameters, _set_active_indices!, active_cells_field, ibg) return active_cells_field end @@ -98,8 +105,8 @@ const MAXUInt8 = 2^8 - 1 const MAXUInt16 = 2^16 - 1 const MAXUInt32 = 2^32 - 1 -function active_cells_interior_map(ibg) - active_cells_field = compute_interior_active_cells(ibg) +function active_interior_indices(ibg; parameters = :xyz) + active_cells_field = compute_interior_active_cells(ibg; parameters) N = maximum(size(ibg)) IntType = N > MAXUInt8 ? (N > MAXUInt16 ? (N > MAXUInt32 ? UInt64 : UInt32) : UInt16) : UInt8 @@ -137,6 +144,49 @@ end @inline add_3rd_index(t::Tuple, k) = (t[1], t[2], k) +active_cells_interior_map(ibg) = active_interior_indices(ibg; parameters = :xyz) + +# In case of a `DistributedGrid` we want to have different maps depending on the +# partitioning of the domain +function active_cells_interior_map(ibg::ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:DistributedGrid}) + + arch = architecture(ibg) + Rx, Ry, _ = arch.ranks + Tx, Ty, _ = topology(ibg) + Nx, Ny, Nz = size(ibg) + Hx, Hy, _ = halo_size(ibg) + + Sx = (Hx, Ny, Nz) + Sy = (Nx, Hy, Nz) + + Oᴸ = (0, 0, 0) + Oxᴿ = (Nx-Hx, 0, 0) + Oyᴿ = (0, Ny-Hy, 0) + + sizes = (Sx, Sy, Sx, Sy) + offs = (Oᴸ, Oᴸ, Oxᴿ, Oyᴿ) + + include_west = !isa(ibg, XFlatGrid) && (Rx != 1) && !(Tx == RightConnected) + include_east = !isa(ibg, XFlatGrid) && (Rx != 1) && !(Tx == LeftConnected) + include_south = !isa(ibg, YFlatGrid) && (Ry != 1) && !(Ty == RightConnected) + include_north = !isa(ibg, YFlatGrid) && (Ry != 1) && !(Ty == LeftConnected) + + west = include_west ? active_interior_indices(ibg; parameters = KernelParameters(Sx, Oᴸ)) : nothing + east = include_east ? active_interior_indices(ibg; parameters = KernelParameters(Sx, Oxᴿ)) : nothing + south = include_south ? active_interior_indices(ibg; parameters = KernelParameters(Sy, Oᴸ)) : nothing + north = include_north ? active_interior_indices(ibg; parameters = KernelParameters(Sy, Oyᴿ)) : nothing + + nx = Rx == 1 ? Nx : (Tx == RightConnected || Tx == LeftConnected ? Nx - Hx : Nx - 2Hx) + ny = Ry == 1 ? Ny : (Ty == RightConnected || Ty == LeftConnected ? Ny - Hy : Ny - 2Hy) + + ox = Rx == 1 || Tx == RightConnected ? 0 : Hx + oy = Ry == 1 || Ty == RightConnected ? 0 : Hy + + interior = active_interior_indices(ibg; parameters = KernelParameters((nx, ny, Nz), (ox, oy, 0))) + + return (; interior, west, east, south, north) +end + # If we eventually want to perform also barotropic step, `w` computation and `p` # computation only on active `columns` function active_cells_surface_map(ibg) @@ -153,50 +203,3 @@ function active_cells_surface_map(ibg) return smaller_indices end - -# In case of a `DistributedGrid` we want to have different maps depending on the -# partitioning of the domain -function active_cells_interior_map(ibg::ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:DistributedGrid}) - active_cells_field = compute_interior_active_cells(ibg) - - N = maximum(size(ibg)) - IntType = N > MAXUInt8 ? (N > MAXUInt16 ? (N > MAXUInt32 ? UInt64 : UInt32) : UInt16) : UInt8 - - IndicesType = Tuple{IntType, IntType, IntType} - - # Cannot findall on the entire field because we incur on OOM errors - active_indices = IndicesType[] - active_indices = findall_active_indices!(active_indices, active_cells_field, ibg, IndicesType) - active_indices = separate_active_indices!(active_indices, ibg) - - return active_indices -end - -function separate_active_indices!(indices, ibg) - arch = architecture(ibg) - Hx, Hy, _ = halo_size(ibg) - Nx, Ny, _ = size(ibg) - Rx, Ry, _ = arch.ranks - west = Rx > 1 ? findall(idx -> Int(idx[1]) <= Hx, indices) : Int[] - east = Rx > 1 ? findall(idx -> Int(idx[1]) >= Nx-Hx, indices) : Int[] - south = Ry > 1 ? findall(idx -> Int(idx[2]) <= Hy, indices) : Int[] - north = Ry > 1 ? findall(idx -> Int(idx[2]) >= Ny-Hy, indices) : Int[] - - west = indices[west] - east = indices[east] - south = indices[south] - north = indices[north] - - interior = findall(idx -> !(idx ∈ west) && - !(idx ∈ east) && - !(idx ∈ south) && - !(idx ∈ north), indices) - - interior = arch_array(architecture(ibg), indices[interior]) - west = arch_array(architecture(ibg), west ) - east = arch_array(architecture(ibg), east ) - south = arch_array(architecture(ibg), south) - north = arch_array(architecture(ibg), north) - - return (; interior, west, east, south, north) -end \ No newline at end of file diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl index 3864607641..eca6381402 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl @@ -31,7 +31,6 @@ end function compute_boundary_tendency_contributions!(grid, arch, model) kernel_parameters = boundary_tendency_kernel_parameters(grid, arch) compute_hydrostatic_free_surface_tendency_contributions!(model, kernel_parameters) - return nothing end @@ -39,10 +38,12 @@ function compute_boundary_tendency_contributions!(grid::DistributedActiveCellsIB maps = grid.interior_active_cells for (name, map) in zip(keys(maps), maps) - if name != :interior && !isempy(map) - compute_hydrostatic_free_surface_tendency_contributions!(model, :xyz; only_active_cells = active_map(Val(name))) + if name != :interior && !isnothing(map) + compute_hydrostatic_free_surface_tendency_contributions!(model, tuple(:xyz); only_active_cells = active_map(Val(name))) end end + + return nothing end # w needs computing in the range - H + 1 : 0 and N - 1 : N + H - 1 diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index e068a9874f..db331408c7 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -6,9 +6,9 @@ using Oceananigans.AbstractOperations: Δz using Oceananigans.BoundaryConditions using Oceananigans.Operators using Oceananigans.ImmersedBoundaries: peripheral_node, immersed_inactive_node -using Oceananigans.ImmersedBoundaries: inactive_node, IBG, c, f +using Oceananigans.ImmersedBoundaries: inactive_node, IBG, c, f, SurfaceMap using Oceananigans.ImmersedBoundaries: mask_immersed_field!, use_only_active_surface_cells, use_only_active_interior_cells -using Oceananigans.ImmersedBoundaries: active_linear_index_to_surface_tuple, ActiveCellsIBG, ActiveSurfaceIBG +using Oceananigans.ImmersedBoundaries: active_linear_index_to_tuple, ActiveCellsIBG, ActiveSurfaceIBG # constants for AB3 time stepping scheme (from https://doi.org/10.1016/j.ocemod.2004.08.002) const β = 0.281105 @@ -150,7 +150,7 @@ end Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, timestepper) idx = @index(Global, Linear) - i, j = active_linear_index_to_surface_tuple(idx, grid) + i, j = active_linear_index_to_tuple(idx, SurfaceMap(), grid) free_surface_evolution_kernel!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) end @@ -186,7 +186,7 @@ end Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, timestepper) idx = @index(Global, Linear) - i, j = active_linear_index_to_surface_tuple(idx, grid) + i, j = active_linear_index_to_tuple(idx, SurfaceMap(), grid) velocity_evolution_kernel!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅, averaging_weight, @@ -262,7 +262,7 @@ end # u_Δz = u * Δz @kernel function _barotropic_mode_kernel!(U, V, grid::ActiveSurfaceIBG, u, v) idx = @index(Global, Linear) - i, j = active_linear_index_to_surface_tuple(idx, grid) + i, j = active_linear_index_to_tuple(idx, SurfaceMap(), grid) # hand unroll first loop @inbounds U[i, j, 1] = Δzᶠᶜᶜ(i, j, 1, grid) * u[i, j, 1] @@ -424,7 +424,7 @@ end # Calculate RHS for the barotopic time step. @kernel function _compute_integrated_ab2_tendencies!(Gᵁ, Gⱽ, grid::ActiveSurfaceIBG, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) idx = @index(Global, Linear) - i, j = active_linear_index_to_surface_tuple(idx, grid) + i, j = active_linear_index_to_tuple(idx, SurfaceMap(), grid) # hand unroll first loop @inbounds Gᵁ[i, j, 1] = Δzᶠᶜᶜ(i, j, 1, grid) * ab2_step_Gu(i, j, 1, grid, Gu⁻, Guⁿ, χ) diff --git a/validation/distributed_simulations/distributed_hydrostatic_turbulence.jl b/validation/distributed_simulations/distributed_hydrostatic_turbulence.jl index 5fbe1c766a..8ba70ba83f 100644 --- a/validation/distributed_simulations/distributed_hydrostatic_turbulence.jl +++ b/validation/distributed_simulations/distributed_hydrostatic_turbulence.jl @@ -6,6 +6,7 @@ using Statistics using Oceananigans.BoundaryConditions using Oceananigans.DistributedComputations using Random +using Oceananigans.ImmersedBoundaries: ActiveCellsIBG, use_only_active_interior_cells # Run with # @@ -13,10 +14,14 @@ using Random # mpiexec -n 4 julia --project distributed_hydrostatic_turbulence.jl # ``` -function run_simulation(nx, ny, arch, topo) - grid = RectilinearGrid(arch; topology=topo, size=(nx, ny, 1), extent=(4π, 4π, 0.5), halo=(7, 7, 7)) +function run_simulation(nx, ny, arch; topology = (Periodic, Periodic, Bounded)) + grid = RectilinearGrid(arch; topology, size = (Nx, Ny, 1), extent=(4π, 4π, 0.5), halo=(7, 7, 7)) + bottom(x, y) = (x > π && x < 3π/2 && y > π/2 && y < 3π/2) ? 1.0 : - grid.Lz - 1.0 - grid = ImmersedBoundaryGrid(grid, GridFittedBottom(bottom)) + grid = ImmersedBoundaryGrid(grid, GridFittedBottom(bottom); active_cells_map = true) + + @show grid isa ActiveCellsIBG + @show use_only_active_interior_cells(grid) model = HydrostaticFreeSurfaceModel(; grid, momentum_advection = VectorInvariant(vorticity_scheme=WENO(order=9)), @@ -60,17 +65,13 @@ function run_simulation(nx, ny, arch, topo) MPI.Barrier(arch.communicator) end -topo = (Periodic, Periodic, Bounded) +Nx = 128 +Ny = 128 -# Use non-uniform partitioning in x, y. -# TODO: Explain what local_index is. -nx = [90, 128-90][arch.local_index[1]] -ny = [56, 128-56][arch.local_index[2]] -@show arch.local_index -arch = Distributed(CPU(), topology = topo, ranks=(2, 2, 1)) +arch = Distributed(CPU(), partition = Partition(2, 2)) # Run the simulation -run_simulation(nx, ny, arch, topo) +run_simulation(Nx, Ny, arch) # Visualize the plane # Produce a video for variable `var` From a6ab8406dd076590d505d0fbe3cec35c1f6fda7c Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sun, 3 Dec 2023 18:38:51 -0500 Subject: [PATCH 024/567] changes --- .../compute_hydrostatic_free_surface_tendencies.jl | 8 ++++---- .../store_hydrostatic_free_surface_tendencies.jl | 6 ++---- src/TimeSteppers/store_tendencies.jl | 3 +-- .../distributed_hydrostatic_turbulence.jl | 5 ++--- 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl index 8ef6d8ea1e..8f69e3c773 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl @@ -222,7 +222,7 @@ end ##### """ Calculate the right-hand-side of the u-velocity equation. """ -@kernel function compute_hydrostatic_free_surface_Gu!(Gu, grid, interior_map, args) +@kernel function compute_hydrostatic_free_surface_Gu!(Gu, grid, map, args) i, j, k = @index(Global, NTuple) @inbounds Gu[i, j, k] = hydrostatic_free_surface_u_velocity_tendency(i, j, k, grid, args...) end @@ -234,7 +234,7 @@ end end """ Calculate the right-hand-side of the v-velocity equation. """ -@kernel function compute_hydrostatic_free_surface_Gv!(Gv, grid, interior_map, args) +@kernel function compute_hydrostatic_free_surface_Gv!(Gv, grid, map, args) i, j, k = @index(Global, NTuple) @inbounds Gv[i, j, k] = hydrostatic_free_surface_v_velocity_tendency(i, j, k, grid, args...) end @@ -250,7 +250,7 @@ end ##### """ Calculate the right-hand-side of the tracer advection-diffusion equation. """ -@kernel function compute_hydrostatic_free_surface_Gc!(Gc, grid, interior_map, args) +@kernel function compute_hydrostatic_free_surface_Gc!(Gc, grid, map, args) i, j, k = @index(Global, NTuple) @inbounds Gc[i, j, k] = hydrostatic_free_surface_tracer_tendency(i, j, k, grid, args...) end @@ -262,7 +262,7 @@ end end """ Calculate the right-hand-side of the subgrid scale energy equation. """ -@kernel function compute_hydrostatic_free_surface_Ge!(Ge, grid, interior_map, args) +@kernel function compute_hydrostatic_free_surface_Ge!(Ge, grid, map, args) i, j, k = @index(Global, NTuple) @inbounds Ge[i, j, k] = hydrostatic_turbulent_kinetic_energy_tendency(i, j, k, grid, args...) end diff --git a/src/Models/HydrostaticFreeSurfaceModels/store_hydrostatic_free_surface_tendencies.jl b/src/Models/HydrostaticFreeSurfaceModels/store_hydrostatic_free_surface_tendencies.jl index 24d26c213c..7ae1cda0b5 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/store_hydrostatic_free_surface_tendencies.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/store_hydrostatic_free_surface_tendencies.jl @@ -28,7 +28,7 @@ function store_free_surface_tendency!(::ExplicitFreeSurface, model) end """ Store previous source terms before updating them. """ -function store_tendencies!(model::HydrostaticFreeSurfaceModel; only_active_cells = use_only_active_interior_cells(model.grid)) +function store_tendencies!(model::HydrostaticFreeSurfaceModel) prognostic_field_names = keys(prognostic_fields(model)) three_dimensional_prognostic_field_names = filter(name -> name != :η, prognostic_field_names) @@ -37,9 +37,7 @@ function store_tendencies!(model::HydrostaticFreeSurfaceModel; only_active_cells launch!(model.architecture, model.grid, :xyz, store_field_tendencies!, model.timestepper.G⁻[field_name], - model.grid, - model.timestepper.Gⁿ[field_name]; - only_active_cells) + model.timestepper.Gⁿ[field_name]) end diff --git a/src/TimeSteppers/store_tendencies.jl b/src/TimeSteppers/store_tendencies.jl index b4a117e5b8..e2f1b07c7e 100644 --- a/src/TimeSteppers/store_tendencies.jl +++ b/src/TimeSteppers/store_tendencies.jl @@ -4,7 +4,7 @@ using Oceananigans.ImmersedBoundaries: ActiveCellsIBG using Oceananigans.Utils: launch! """ Store source terms for `u`, `v`, and `w`. """ -@kernel function store_field_tendencies!(G⁻, grid, G⁰) +@kernel function store_field_tendencies!(G⁻, G⁰) i, j, k = @index(Global, NTuple) @inbounds G⁻[i, j, k] = G⁰[i, j, k] end @@ -16,7 +16,6 @@ function store_tendencies!(model) for field_name in keys(model_fields) launch!(model.architecture, model.grid, :xyz, store_field_tendencies!, model.timestepper.G⁻[field_name], - model.grid, model.timestepper.Gⁿ[field_name]) end diff --git a/validation/distributed_simulations/distributed_hydrostatic_turbulence.jl b/validation/distributed_simulations/distributed_hydrostatic_turbulence.jl index 8ba70ba83f..fdec0fcc83 100644 --- a/validation/distributed_simulations/distributed_hydrostatic_turbulence.jl +++ b/validation/distributed_simulations/distributed_hydrostatic_turbulence.jl @@ -43,9 +43,9 @@ function run_simulation(nx, ny, arch; topology = (Periodic, Periodic, Bounded)) set!(c, mask) u, v, _ = model.velocities - ζ = VerticalVorticityField(model) + # ζ = VerticalVorticityField(model) η = model.free_surface.η - outputs = merge(model.velocities, model.tracers, (; ζ, η)) + outputs = merge(model.velocities, model.tracers) progress(sim) = @info "Iteration: $(sim.model.clock.iteration), time: $(sim.model.clock.time), Δt: $(sim.Δt)" simulation = Simulation(model, Δt=0.02, stop_time=100.0) @@ -110,7 +110,6 @@ try if MPI.Comm_rank(MPI.COMM_WORLD) == 0 visualize_simulation("u") visualize_simulation("v") - visualize_simulation("ζ") visualize_simulation("c") end catch err From 6c96131a9a2861a844a319de795542f6cddf5a29 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sun, 3 Dec 2023 19:22:03 -0500 Subject: [PATCH 025/567] try like this --- .../compute_hydrostatic_free_surface_boundary_tendencies.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl index eca6381402..767454abc1 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl @@ -56,8 +56,8 @@ function boundary_w_kernel_parameters(grid, arch) # Offsets in tangential direction are == -1 to # cover the required corners - Oxᴸ = (-Hx, -1) - Oyᴸ = (-1, -Hy) + Oxᴸ = (-Hx+1, -1) + Oyᴸ = (-1, -Hy+1) Oxᴿ = (Nx-1, -1) Oyᴿ = (-1, Ny-1) From 805790933abf4e93c9ad4b56dd7be71ce644b53b Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 4 Dec 2023 11:22:31 -0500 Subject: [PATCH 026/567] some tests... --- src/Grids/grid_generation.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Grids/grid_generation.jl b/src/Grids/grid_generation.jl index b1480aa1f3..f79a2829dc 100644 --- a/src/Grids/grid_generation.jl +++ b/src/Grids/grid_generation.jl @@ -13,6 +13,7 @@ get_face_node(coord::Function, i) = coord(i) get_face_node(coord::AbstractVector, i) = CUDA.@allowscalar coord[i] const AT = AbstractTopology + lower_exterior_Δcoordᶠ(::AT, Fi, Hcoord) = [Fi[end - Hcoord + i] - Fi[end - Hcoord + i - 1] for i = 1:Hcoord] lower_exterior_Δcoordᶠ(::BoundedTopology, Fi, Hcoord) = [Fi[2] - Fi[1] for _ = 1:Hcoord] @@ -34,6 +35,8 @@ function generate_coordinate(FT, topo::AT, N, H, node_generator, coordinate_name # Ensure correct type for F and derived quantities interior_face_nodes = zeros(FT, N+1) + @show typeof(node_generator), size(node_generator), coordinate_name + # Use the user-supplied "generator" to build the interior nodes for idx = 1:N+1 interior_face_nodes[idx] = get_face_node(node_generator, idx) From 183ea906347465d9053becdb17fb9ef335932c6d Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 4 Dec 2023 11:44:03 -0500 Subject: [PATCH 027/567] show the coordinate --- src/DistributedComputations/distributed_grids.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/DistributedComputations/distributed_grids.jl b/src/DistributedComputations/distributed_grids.jl index 27064906bd..7d94aa3896 100644 --- a/src/DistributedComputations/distributed_grids.jl +++ b/src/DistributedComputations/distributed_grids.jl @@ -130,6 +130,8 @@ function LatitudeLongitudeGrid(arch::Distributed, φl = partition(latitude, nφ, arch, 2) zl = partition(z, nz, arch, 3) + @show Base.size(zl), Base.size(z) + # Calculate all direction (which might be stretched) # A direction is regular if the domain passed is a Tuple{<:Real, <:Real}, # it is stretched if being passed is a function or vector (as for the VerticallyStretchedRectilinearGrid) From 05593e21aad1b2757424758511e2f1846561c835 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 4 Dec 2023 11:52:20 -0500 Subject: [PATCH 028/567] bugfix --- src/DistributedComputations/distributed_grids.jl | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/DistributedComputations/distributed_grids.jl b/src/DistributedComputations/distributed_grids.jl index 7d94aa3896..b15d61f07f 100644 --- a/src/DistributedComputations/distributed_grids.jl +++ b/src/DistributedComputations/distributed_grids.jl @@ -126,11 +126,9 @@ function LatitudeLongitudeGrid(arch::Distributed, TY = insert_connected_topology(topology[2], Ry, rj) TZ = insert_connected_topology(topology[3], Rz, rk) - λl = partition(longitude, nλ, arch, 1) - φl = partition(latitude, nφ, arch, 2) - zl = partition(z, nz, arch, 3) - - @show Base.size(zl), Base.size(z) + λl = partition(longitude, nλ, arch, 1) + φl = partition(latitude, nφ, arch, 2) + zl = partition(z, nz+1, arch, 3) # Calculate all direction (which might be stretched) # A direction is regular if the domain passed is a Tuple{<:Real, <:Real}, From 44dbee0062959acce7d90559f406e8df844c26c8 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 4 Dec 2023 11:59:14 -0500 Subject: [PATCH 029/567] bugfix --- src/DistributedComputations/distributed_grids.jl | 6 +++--- src/DistributedComputations/partition_assemble.jl | 7 ++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/DistributedComputations/distributed_grids.jl b/src/DistributedComputations/distributed_grids.jl index b15d61f07f..27064906bd 100644 --- a/src/DistributedComputations/distributed_grids.jl +++ b/src/DistributedComputations/distributed_grids.jl @@ -126,9 +126,9 @@ function LatitudeLongitudeGrid(arch::Distributed, TY = insert_connected_topology(topology[2], Ry, rj) TZ = insert_connected_topology(topology[3], Rz, rk) - λl = partition(longitude, nλ, arch, 1) - φl = partition(latitude, nφ, arch, 2) - zl = partition(z, nz+1, arch, 3) + λl = partition(longitude, nλ, arch, 1) + φl = partition(latitude, nφ, arch, 2) + zl = partition(z, nz, arch, 3) # Calculate all direction (which might be stretched) # A direction is regular if the domain passed is a Tuple{<:Real, <:Real}, diff --git a/src/DistributedComputations/partition_assemble.jl b/src/DistributedComputations/partition_assemble.jl index 1010c0f6bf..0e96374955 100644 --- a/src/DistributedComputations/partition_assemble.jl +++ b/src/DistributedComputations/partition_assemble.jl @@ -41,7 +41,12 @@ end function partition(c::AbstractVector, n, arch, idx) nl = concatenate_local_sizes(n, arch, idx) r = arch.local_index[idx] - return c[1 + sum(nl[1:r-1]) : sum(nl[1:r])] + # Allow for Face values + if r == arch.ranks[idx] + return c[1 + sum(nl[1:r-1]) : end] + else + return c[1 + sum(nl[1:r-1]) : sum(nl[1:r])] + end end function partition(c::Tuple, n, arch, idx) From 8c81e15ff18e74b35c1301d5165826fe7e5d200f Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 4 Dec 2023 21:03:24 -0500 Subject: [PATCH 030/567] test this hypothesis --- src/ImmersedBoundaries/active_cells_map.jl | 21 ++++++++----------- ...ute_hydrostatic_free_surface_tendencies.jl | 2 ++ 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/ImmersedBoundaries/active_cells_map.jl b/src/ImmersedBoundaries/active_cells_map.jl index a10efa0a7e..92a8ca6517 100644 --- a/src/ImmersedBoundaries/active_cells_map.jl +++ b/src/ImmersedBoundaries/active_cells_map.jl @@ -156,25 +156,22 @@ function active_cells_interior_map(ibg::ImmersedBoundaryGrid{<:Any, <:Any, <:Any Nx, Ny, Nz = size(ibg) Hx, Hy, _ = halo_size(ibg) - Sx = (Hx, Ny, Nz) - Sy = (Nx, Hy, Nz) + x_boundary = (Hx, Ny, Nz) + y_boundary = (Nx, Hy, Nz) - Oᴸ = (0, 0, 0) - Oxᴿ = (Nx-Hx, 0, 0) - Oyᴿ = (0, Ny-Hy, 0) - - sizes = (Sx, Sy, Sx, Sy) - offs = (Oᴸ, Oᴸ, Oxᴿ, Oyᴿ) + left_offsets = (0, 0, 0) + right_x_offsets = (Nx-Hx, 0, 0) + right_y_offsets = (0, Ny-Hy, 0) include_west = !isa(ibg, XFlatGrid) && (Rx != 1) && !(Tx == RightConnected) include_east = !isa(ibg, XFlatGrid) && (Rx != 1) && !(Tx == LeftConnected) include_south = !isa(ibg, YFlatGrid) && (Ry != 1) && !(Ty == RightConnected) include_north = !isa(ibg, YFlatGrid) && (Ry != 1) && !(Ty == LeftConnected) - west = include_west ? active_interior_indices(ibg; parameters = KernelParameters(Sx, Oᴸ)) : nothing - east = include_east ? active_interior_indices(ibg; parameters = KernelParameters(Sx, Oxᴿ)) : nothing - south = include_south ? active_interior_indices(ibg; parameters = KernelParameters(Sy, Oᴸ)) : nothing - north = include_north ? active_interior_indices(ibg; parameters = KernelParameters(Sy, Oyᴿ)) : nothing + west = include_west ? active_interior_indices(ibg; parameters = KernelParameters(x_boundary, left_offsets)) : nothing + east = include_east ? active_interior_indices(ibg; parameters = KernelParameters(x_boundary, right_x_offsets)) : nothing + south = include_south ? active_interior_indices(ibg; parameters = KernelParameters(y_boundary, left_offsets)) : nothing + north = include_north ? active_interior_indices(ibg; parameters = KernelParameters(y_boundary, right_y_offsets)) : nothing nx = Rx == 1 ? Nx : (Tx == RightConnected || Tx == LeftConnected ? Nx - Hx : Nx - 2Hx) ny = Ry == 1 ? Ny : (Ty == RightConnected || Ty == LeftConnected ? Ny - Hy : Ny - 2Hy) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl index 8f69e3c773..2403aaf5b6 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl @@ -223,11 +223,13 @@ end """ Calculate the right-hand-side of the u-velocity equation. """ @kernel function compute_hydrostatic_free_surface_Gu!(Gu, grid, map, args) + @show "I am inside the non-active kernel" i, j, k = @index(Global, NTuple) @inbounds Gu[i, j, k] = hydrostatic_free_surface_u_velocity_tendency(i, j, k, grid, args...) end @kernel function compute_hydrostatic_free_surface_Gu!(Gu, grid::ActiveCellsIBG, map, args) + @show "I am inside the active kernel" idx = @index(Global, Linear) i, j, k = active_linear_index_to_tuple(idx, map, grid) @inbounds Gu[i, j, k] = hydrostatic_free_surface_u_velocity_tendency(i, j, k, grid, args...) From 3e88fb4f5b5ddc6d1542bdb63030ee981ea45a50 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 4 Dec 2023 21:38:29 -0500 Subject: [PATCH 031/567] another test --- src/ImmersedBoundaries/active_cells_map.jl | 7 +++++-- .../compute_hydrostatic_free_surface_tendencies.jl | 5 +++-- src/Utils/kernel_launching.jl | 1 - 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/ImmersedBoundaries/active_cells_map.jl b/src/ImmersedBoundaries/active_cells_map.jl index 92a8ca6517..008c155b8f 100644 --- a/src/ImmersedBoundaries/active_cells_map.jl +++ b/src/ImmersedBoundaries/active_cells_map.jl @@ -5,7 +5,8 @@ using Oceananigans.Grids: AbstractGrid using KernelAbstractions: @kernel, @index import Oceananigans.Utils: active_cells_work_layout, - use_only_active_interior_cells + use_only_active_interior_cells, + use_only_active_surface_cells using Oceananigans.Solvers: solve_batched_tridiagonal_system_z!, ZDirection using Oceananigans.DistributedComputations: DistributedGrid @@ -31,8 +32,10 @@ active_map(::Val{:south}) = SouthMap() active_map(::Val{:north}) = NorthMap() @inline use_only_active_surface_cells(::AbstractGrid) = nothing -@inline use_only_active_interior_cells(::ActiveCellsIBG) = InteriorMap() @inline use_only_active_surface_cells(::ActiveSurfaceIBG) = SurfaceMap() + +@inline use_only_active_interior_cells(::AbstractGrid) = nothing +@inline use_only_active_interior_cells(::ActiveCellsIBG) = InteriorMap() @inline use_only_active_interior_cells(::DistributedActiveCellsIBG) = InteriorMap() @inline active_cells_work_layout(group, size, ::InteriorMap, grid::ActiveCellsIBG) = min(length(grid.interior_active_cells), 256), length(grid.interior_active_cells) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl index 2403aaf5b6..cd1726e7a7 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl @@ -181,6 +181,9 @@ function compute_hydrostatic_momentum_tendencies!(model, velocities, kernel_para u_kernel_args = tuple(start_momentum_kernel_args..., u_immersed_bc, end_momentum_kernel_args...) v_kernel_args = tuple(start_momentum_kernel_args..., v_immersed_bc, end_momentum_kernel_args...) + @show grid isa ActiveCellsIBG + @show only_active_cells + for parameters in kernel_parameters launch!(arch, grid, parameters, compute_hydrostatic_free_surface_Gu!, model.timestepper.Gⁿ.u, grid, @@ -223,13 +226,11 @@ end """ Calculate the right-hand-side of the u-velocity equation. """ @kernel function compute_hydrostatic_free_surface_Gu!(Gu, grid, map, args) - @show "I am inside the non-active kernel" i, j, k = @index(Global, NTuple) @inbounds Gu[i, j, k] = hydrostatic_free_surface_u_velocity_tendency(i, j, k, grid, args...) end @kernel function compute_hydrostatic_free_surface_Gu!(Gu, grid::ActiveCellsIBG, map, args) - @show "I am inside the active kernel" idx = @index(Global, Linear) i, j, k = active_linear_index_to_tuple(idx, map, grid) @inbounds Gu[i, j, k] = hydrostatic_free_surface_u_velocity_tendency(i, j, k, grid, args...) diff --git a/src/Utils/kernel_launching.jl b/src/Utils/kernel_launching.jl index dfdb741199..1d0b744937 100644 --- a/src/Utils/kernel_launching.jl +++ b/src/Utils/kernel_launching.jl @@ -106,7 +106,6 @@ function work_layout(grid, workdims::Symbol; include_right_boundaries=false, loc end @inline active_cells_work_layout(workgroup, worksize, only_active_cells, grid) = workgroup, worksize -@inline use_only_active_interior_cells(grid) = nothing """ launch!(arch, grid, layout, kernel!, args...; kwargs...) From d74f5f518ede95aa6d72aadc32cf0f57a8116998 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 4 Dec 2023 21:39:47 -0500 Subject: [PATCH 032/567] bugfix --- .../ri_based_vertical_diffusivity.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/TurbulenceClosures/turbulence_closure_implementations/ri_based_vertical_diffusivity.jl b/src/TurbulenceClosures/turbulence_closure_implementations/ri_based_vertical_diffusivity.jl index 16dc985252..acc96fd408 100644 --- a/src/TurbulenceClosures/turbulence_closure_implementations/ri_based_vertical_diffusivity.jl +++ b/src/TurbulenceClosures/turbulence_closure_implementations/ri_based_vertical_diffusivity.jl @@ -3,7 +3,6 @@ using Oceananigans.BuoyancyModels: ∂z_b using Oceananigans.Operators using Oceananigans.Grids: inactive_node using Oceananigans.Operators: ℑzᵃᵃᶜ -using Oceananigans.Utils: use_only_active_interior_cells struct RiBasedVerticalDiffusivity{TD, FT, R} <: AbstractScalarDiffusivity{TD, VerticalFormulation, 1} ν₀ :: FT From 25a1dbb193e571fa24e74b3f421e01b9742196a7 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 4 Dec 2023 21:40:42 -0500 Subject: [PATCH 033/567] other bugfix --- src/ImmersedBoundaries/active_cells_map.jl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/ImmersedBoundaries/active_cells_map.jl b/src/ImmersedBoundaries/active_cells_map.jl index 008c155b8f..2bfd2bc627 100644 --- a/src/ImmersedBoundaries/active_cells_map.jl +++ b/src/ImmersedBoundaries/active_cells_map.jl @@ -4,9 +4,7 @@ using Oceananigans.Grids: AbstractGrid using KernelAbstractions: @kernel, @index -import Oceananigans.Utils: active_cells_work_layout, - use_only_active_interior_cells, - use_only_active_surface_cells +import Oceananigans.Utils: active_cells_work_layout using Oceananigans.Solvers: solve_batched_tridiagonal_system_z!, ZDirection using Oceananigans.DistributedComputations: DistributedGrid From 188eedc667db35db1f6ed867b5ce60b438effc40 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 4 Dec 2023 21:50:52 -0500 Subject: [PATCH 034/567] now we'll see... --- ...ute_hydrostatic_free_surface_tendencies.jl | 41 ++++++++++--------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl index cd1726e7a7..e7fa777da1 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl @@ -183,6 +183,7 @@ function compute_hydrostatic_momentum_tendencies!(model, velocities, kernel_para @show grid isa ActiveCellsIBG @show only_active_cells + @show size(grid.interior_active_cells.interior) for parameters in kernel_parameters launch!(arch, grid, parameters, @@ -224,11 +225,11 @@ end ##### Tendency calculators for u, v ##### -""" Calculate the right-hand-side of the u-velocity equation. """ -@kernel function compute_hydrostatic_free_surface_Gu!(Gu, grid, map, args) - i, j, k = @index(Global, NTuple) - @inbounds Gu[i, j, k] = hydrostatic_free_surface_u_velocity_tendency(i, j, k, grid, args...) -end +# """ Calculate the right-hand-side of the u-velocity equation. """ +# @kernel function compute_hydrostatic_free_surface_Gu!(Gu, grid, map, args) +# i, j, k = @index(Global, NTuple) +# @inbounds Gu[i, j, k] = hydrostatic_free_surface_u_velocity_tendency(i, j, k, grid, args...) +# end @kernel function compute_hydrostatic_free_surface_Gu!(Gu, grid::ActiveCellsIBG, map, args) idx = @index(Global, Linear) @@ -236,11 +237,11 @@ end @inbounds Gu[i, j, k] = hydrostatic_free_surface_u_velocity_tendency(i, j, k, grid, args...) end -""" Calculate the right-hand-side of the v-velocity equation. """ -@kernel function compute_hydrostatic_free_surface_Gv!(Gv, grid, map, args) - i, j, k = @index(Global, NTuple) - @inbounds Gv[i, j, k] = hydrostatic_free_surface_v_velocity_tendency(i, j, k, grid, args...) -end +# """ Calculate the right-hand-side of the v-velocity equation. """ +# @kernel function compute_hydrostatic_free_surface_Gv!(Gv, grid, map, args) +# i, j, k = @index(Global, NTuple) +# @inbounds Gv[i, j, k] = hydrostatic_free_surface_v_velocity_tendency(i, j, k, grid, args...) +# end @kernel function compute_hydrostatic_free_surface_Gv!(Gv, grid::ActiveCellsIBG, map, args) idx = @index(Global, Linear) @@ -252,11 +253,11 @@ end ##### Tendency calculators for tracers ##### -""" Calculate the right-hand-side of the tracer advection-diffusion equation. """ -@kernel function compute_hydrostatic_free_surface_Gc!(Gc, grid, map, args) - i, j, k = @index(Global, NTuple) - @inbounds Gc[i, j, k] = hydrostatic_free_surface_tracer_tendency(i, j, k, grid, args...) -end +# """ Calculate the right-hand-side of the tracer advection-diffusion equation. """ +# @kernel function compute_hydrostatic_free_surface_Gc!(Gc, grid, map, args) +# i, j, k = @index(Global, NTuple) +# @inbounds Gc[i, j, k] = hydrostatic_free_surface_tracer_tendency(i, j, k, grid, args...) +# end @kernel function compute_hydrostatic_free_surface_Gc!(Gc, grid::ActiveCellsIBG, map, args) idx = @index(Global, Linear) @@ -264,11 +265,11 @@ end @inbounds Gc[i, j, k] = hydrostatic_free_surface_tracer_tendency(i, j, k, grid, args...) end -""" Calculate the right-hand-side of the subgrid scale energy equation. """ -@kernel function compute_hydrostatic_free_surface_Ge!(Ge, grid, map, args) - i, j, k = @index(Global, NTuple) - @inbounds Ge[i, j, k] = hydrostatic_turbulent_kinetic_energy_tendency(i, j, k, grid, args...) -end +# """ Calculate the right-hand-side of the subgrid scale energy equation. """ +# @kernel function compute_hydrostatic_free_surface_Ge!(Ge, grid, map, args) +# i, j, k = @index(Global, NTuple) +# @inbounds Ge[i, j, k] = hydrostatic_turbulent_kinetic_energy_tendency(i, j, k, grid, args...) +# end @kernel function compute_hydrostatic_free_surface_Ge!(Ge, grid::ActiveCellsIBG, map, args) idx = @index(Global, Linear) From 7d418e039f2dd9e9c00cd2fdd993f9dd744e3136 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 4 Dec 2023 22:03:33 -0500 Subject: [PATCH 035/567] now it will work hopefully --- src/ImmersedBoundaries/active_cells_map.jl | 5 ++- ...ute_hydrostatic_free_surface_tendencies.jl | 40 +++++++++---------- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/src/ImmersedBoundaries/active_cells_map.jl b/src/ImmersedBoundaries/active_cells_map.jl index 2bfd2bc627..16946cdceb 100644 --- a/src/ImmersedBoundaries/active_cells_map.jl +++ b/src/ImmersedBoundaries/active_cells_map.jl @@ -13,8 +13,9 @@ import Oceananigans.Solvers: solve_batched_tridiagonal_system_kernel! const ActiveSurfaceIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractArray} const DistributedActiveCellsIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:DistributedGrid, <:Any, <:NamedTuple} -const SerialActiveCellsIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractArray} -const ActiveCellsIBG = Union{DistributedActiveCellsIBG, SerialActiveCellsIBG} +const ArrayActiveCellsIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractArray} +const NamedTupleActiveCellsIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:NamedTuple} +const ActiveCellsIBG = Union{DistributedActiveCellsIBG, ArrayActiveCellsIBG, NamedTupleActiveCellsIBG} struct InteriorMap end struct SurfaceMap end diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl index e7fa777da1..94c9b868d6 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl @@ -225,11 +225,11 @@ end ##### Tendency calculators for u, v ##### -# """ Calculate the right-hand-side of the u-velocity equation. """ -# @kernel function compute_hydrostatic_free_surface_Gu!(Gu, grid, map, args) -# i, j, k = @index(Global, NTuple) -# @inbounds Gu[i, j, k] = hydrostatic_free_surface_u_velocity_tendency(i, j, k, grid, args...) -# end +""" Calculate the right-hand-side of the u-velocity equation. """ +@kernel function compute_hydrostatic_free_surface_Gu!(Gu, grid, map, args) + i, j, k = @index(Global, NTuple) + @inbounds Gu[i, j, k] = hydrostatic_free_surface_u_velocity_tendency(i, j, k, grid, args...) +end @kernel function compute_hydrostatic_free_surface_Gu!(Gu, grid::ActiveCellsIBG, map, args) idx = @index(Global, Linear) @@ -237,11 +237,11 @@ end @inbounds Gu[i, j, k] = hydrostatic_free_surface_u_velocity_tendency(i, j, k, grid, args...) end -# """ Calculate the right-hand-side of the v-velocity equation. """ -# @kernel function compute_hydrostatic_free_surface_Gv!(Gv, grid, map, args) -# i, j, k = @index(Global, NTuple) -# @inbounds Gv[i, j, k] = hydrostatic_free_surface_v_velocity_tendency(i, j, k, grid, args...) -# end +""" Calculate the right-hand-side of the v-velocity equation. """ +@kernel function compute_hydrostatic_free_surface_Gv!(Gv, grid, map, args) + i, j, k = @index(Global, NTuple) + @inbounds Gv[i, j, k] = hydrostatic_free_surface_v_velocity_tendency(i, j, k, grid, args...) +end @kernel function compute_hydrostatic_free_surface_Gv!(Gv, grid::ActiveCellsIBG, map, args) idx = @index(Global, Linear) @@ -253,11 +253,11 @@ end ##### Tendency calculators for tracers ##### -# """ Calculate the right-hand-side of the tracer advection-diffusion equation. """ -# @kernel function compute_hydrostatic_free_surface_Gc!(Gc, grid, map, args) -# i, j, k = @index(Global, NTuple) -# @inbounds Gc[i, j, k] = hydrostatic_free_surface_tracer_tendency(i, j, k, grid, args...) -# end +""" Calculate the right-hand-side of the tracer advection-diffusion equation. """ +@kernel function compute_hydrostatic_free_surface_Gc!(Gc, grid, map, args) + i, j, k = @index(Global, NTuple) + @inbounds Gc[i, j, k] = hydrostatic_free_surface_tracer_tendency(i, j, k, grid, args...) +end @kernel function compute_hydrostatic_free_surface_Gc!(Gc, grid::ActiveCellsIBG, map, args) idx = @index(Global, Linear) @@ -265,11 +265,11 @@ end @inbounds Gc[i, j, k] = hydrostatic_free_surface_tracer_tendency(i, j, k, grid, args...) end -# """ Calculate the right-hand-side of the subgrid scale energy equation. """ -# @kernel function compute_hydrostatic_free_surface_Ge!(Ge, grid, map, args) -# i, j, k = @index(Global, NTuple) -# @inbounds Ge[i, j, k] = hydrostatic_turbulent_kinetic_energy_tendency(i, j, k, grid, args...) -# end +""" Calculate the right-hand-side of the subgrid scale energy equation. """ +@kernel function compute_hydrostatic_free_surface_Ge!(Ge, grid, map, args) + i, j, k = @index(Global, NTuple) + @inbounds Ge[i, j, k] = hydrostatic_turbulent_kinetic_energy_tendency(i, j, k, grid, args...) +end @kernel function compute_hydrostatic_free_surface_Ge!(Ge, grid::ActiveCellsIBG, map, args) idx = @index(Global, Linear) From 83b4d5b1183174028b568a44cd49e81b35e5bd3c Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 4 Dec 2023 22:08:51 -0500 Subject: [PATCH 036/567] all bugs fixed? --- src/ImmersedBoundaries/active_cells_map.jl | 32 +++++++++++----------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/ImmersedBoundaries/active_cells_map.jl b/src/ImmersedBoundaries/active_cells_map.jl index 16946cdceb..546902792d 100644 --- a/src/ImmersedBoundaries/active_cells_map.jl +++ b/src/ImmersedBoundaries/active_cells_map.jl @@ -12,7 +12,7 @@ using Oceananigans.DistributedComputations: DistributedGrid import Oceananigans.Solvers: solve_batched_tridiagonal_system_kernel! const ActiveSurfaceIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractArray} -const DistributedActiveCellsIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:DistributedGrid, <:Any, <:NamedTuple} +const DistributedActiveCellsIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:DistributedGrid, <:Any, <:NamedTuple} # Cannot be used to dispatch in kernels!!! const ArrayActiveCellsIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractArray} const NamedTupleActiveCellsIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:NamedTuple} const ActiveCellsIBG = Union{DistributedActiveCellsIBG, ArrayActiveCellsIBG, NamedTupleActiveCellsIBG} @@ -37,21 +37,21 @@ active_map(::Val{:north}) = NorthMap() @inline use_only_active_interior_cells(::ActiveCellsIBG) = InteriorMap() @inline use_only_active_interior_cells(::DistributedActiveCellsIBG) = InteriorMap() -@inline active_cells_work_layout(group, size, ::InteriorMap, grid::ActiveCellsIBG) = min(length(grid.interior_active_cells), 256), length(grid.interior_active_cells) -@inline active_cells_work_layout(group, size, ::SurfaceMap, grid::ActiveSurfaceIBG) = min(length(grid.surface_active_cells), 256), length(grid.surface_active_cells) - -@inline active_cells_work_layout(group, size, ::InteriorMap, grid::DistributedActiveCellsIBG) = min(length(grid.interior_active_cells.interior), 256), length(grid.interior_active_cells.interior) -@inline active_cells_work_layout(group, size, ::WestMap, grid::DistributedActiveCellsIBG) = min(length(grid.interior_active_cells.west), 256), length(grid.interior_active_cells.west) -@inline active_cells_work_layout(group, size, ::EastMap, grid::DistributedActiveCellsIBG) = min(length(grid.interior_active_cells.east), 256), length(grid.interior_active_cells.east) -@inline active_cells_work_layout(group, size, ::SouthMap, grid::DistributedActiveCellsIBG) = min(length(grid.interior_active_cells.south), 256), length(grid.interior_active_cells.south) -@inline active_cells_work_layout(group, size, ::NorthMap, grid::DistributedActiveCellsIBG) = min(length(grid.interior_active_cells.north), 256), length(grid.interior_active_cells.north) - -@inline active_linear_index_to_tuple(idx, ::InteriorMap, grid::ActiveCellsIBG) = Base.map(Int, grid.interior_active_cells[idx]) -@inline active_linear_index_to_tuple(idx, ::InteriorMap, grid::DistributedActiveCellsIBG) = Base.map(Int, grid.interior_active_cells.interior[idx]) -@inline active_linear_index_to_tuple(idx, ::WestMap, grid::DistributedActiveCellsIBG) = Base.map(Int, grid.interior_active_cells.west[idx]) -@inline active_linear_index_to_tuple(idx, ::EastMap, grid::DistributedActiveCellsIBG) = Base.map(Int, grid.interior_active_cells.east[idx]) -@inline active_linear_index_to_tuple(idx, ::SouthMap, grid::DistributedActiveCellsIBG) = Base.map(Int, grid.interior_active_cells.south[idx]) -@inline active_linear_index_to_tuple(idx, ::NorthMap, grid::DistributedActiveCellsIBG) = Base.map(Int, grid.interior_active_cells.north[idx]) +@inline active_cells_work_layout(group, size, ::InteriorMap, grid::ArrayActiveCellsIBG) = min(length(grid.interior_active_cells), 256), length(grid.interior_active_cells) +@inline active_cells_work_layout(group, size, ::InteriorMap, grid::NamedTupleActiveCellsIBG) = min(length(grid.interior_active_cells.interior), 256), length(grid.interior_active_cells.interior) +@inline active_cells_work_layout(group, size, ::WestMap, grid::NamedTupleActiveCellsIBG) = min(length(grid.interior_active_cells.west), 256), length(grid.interior_active_cells.west) +@inline active_cells_work_layout(group, size, ::EastMap, grid::NamedTupleActiveCellsIBG) = min(length(grid.interior_active_cells.east), 256), length(grid.interior_active_cells.east) +@inline active_cells_work_layout(group, size, ::SouthMap, grid::NamedTupleActiveCellsIBG) = min(length(grid.interior_active_cells.south), 256), length(grid.interior_active_cells.south) +@inline active_cells_work_layout(group, size, ::NorthMap, grid::NamedTupleActiveCellsIBG) = min(length(grid.interior_active_cells.north), 256), length(grid.interior_active_cells.north) + +@inline active_linear_index_to_tuple(idx, ::InteriorMap, grid::ArrayActiveCellsIBG) = Base.map(Int, grid.interior_active_cells[idx]) +@inline active_linear_index_to_tuple(idx, ::InteriorMap, grid::NamedTupleActiveCellsIBG) = Base.map(Int, grid.interior_active_cells.interior[idx]) +@inline active_linear_index_to_tuple(idx, ::WestMap, grid::NamedTupleActiveCellsIBG) = Base.map(Int, grid.interior_active_cells.west[idx]) +@inline active_linear_index_to_tuple(idx, ::EastMap, grid::NamedTupleActiveCellsIBG) = Base.map(Int, grid.interior_active_cells.east[idx]) +@inline active_linear_index_to_tuple(idx, ::SouthMap, grid::NamedTupleActiveCellsIBG) = Base.map(Int, grid.interior_active_cells.south[idx]) +@inline active_linear_index_to_tuple(idx, ::NorthMap, grid::NamedTupleActiveCellsIBG) = Base.map(Int, grid.interior_active_cells.north[idx]) + +@inline active_cells_work_layout(group, size, ::SurfaceMap, grid::ActiveSurfaceIBG) = min(length(grid.surface_active_cells), 256), length(grid.surface_active_cells) @inline active_linear_index_to_tuple(idx, ::SurfaceMap, grid::ActiveSurfaceIBG) = Base.map(Int, grid.surface_active_cells[idx]) From 7a0df19fa9801ff9145c69d572d7477e3e464c28 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 5 Dec 2023 00:03:21 -0500 Subject: [PATCH 037/567] bugfix --- .../split_explicit_free_surface_kernels.jl | 29 ++----------------- 1 file changed, 2 insertions(+), 27 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index db331408c7..5761e7ca29 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -145,15 +145,6 @@ using Printf free_surface_evolution_kernel!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) end -@kernel function split_explicit_free_surface_evolution_kernel!(grid::ActiveSurfaceIBG, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, averaging_weight, - Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, - timestepper) - idx = @index(Global, Linear) - i, j = active_linear_index_to_tuple(idx, SurfaceMap(), grid) - free_surface_evolution_kernel!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) -end - @inline function free_surface_evolution_kernel!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) k_top = grid.Nz+1 TX, TY, _ = topology(grid) @@ -180,20 +171,6 @@ end timestepper ) end - -@kernel function split_explicit_barotropic_velocity_evolution_kernel!(grid::ActiveSurfaceIBG, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, averaging_weight, - Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, - timestepper) - idx = @index(Global, Linear) - i, j = active_linear_index_to_tuple(idx, SurfaceMap(), grid) - - velocity_evolution_kernel!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, averaging_weight, - Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, - timestepper ) -end - @inline function velocity_evolution_kernel!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅, averaging_weight, Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, @@ -235,10 +212,8 @@ function split_explicit_free_surface_substep!(η, state, auxiliary, settings, we η̅, U̅, V̅, averaging_weight, Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, timestepper) - launch!(arch, grid, parameters, split_explicit_free_surface_evolution_kernel!, args...; - only_active_cells = use_only_active_surface_cells(grid)) - launch!(arch, grid, parameters, split_explicit_barotropic_velocity_evolution_kernel!, args...; - only_active_cells = use_only_active_surface_cells(grid)) + launch!(arch, grid, parameters, split_explicit_free_surface_evolution_kernel!, args...;) + launch!(arch, grid, parameters, split_explicit_barotropic_velocity_evolution_kernel!, args...;) return nothing end From 26265794c9ce99d8100b92710506a4d1302d1d26 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 5 Dec 2023 08:59:52 -0500 Subject: [PATCH 038/567] remove the shows --- src/Grids/grid_generation.jl | 2 -- .../compute_hydrostatic_free_surface_tendencies.jl | 4 ---- 2 files changed, 6 deletions(-) diff --git a/src/Grids/grid_generation.jl b/src/Grids/grid_generation.jl index f79a2829dc..98bd421d43 100644 --- a/src/Grids/grid_generation.jl +++ b/src/Grids/grid_generation.jl @@ -35,8 +35,6 @@ function generate_coordinate(FT, topo::AT, N, H, node_generator, coordinate_name # Ensure correct type for F and derived quantities interior_face_nodes = zeros(FT, N+1) - @show typeof(node_generator), size(node_generator), coordinate_name - # Use the user-supplied "generator" to build the interior nodes for idx = 1:N+1 interior_face_nodes[idx] = get_face_node(node_generator, idx) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl index 94c9b868d6..c1168298e6 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl @@ -180,10 +180,6 @@ function compute_hydrostatic_momentum_tendencies!(model, velocities, kernel_para u_kernel_args = tuple(start_momentum_kernel_args..., u_immersed_bc, end_momentum_kernel_args...) v_kernel_args = tuple(start_momentum_kernel_args..., v_immersed_bc, end_momentum_kernel_args...) - - @show grid isa ActiveCellsIBG - @show only_active_cells - @show size(grid.interior_active_cells.interior) for parameters in kernel_parameters launch!(arch, grid, parameters, From 95e90e7226b24cd43fc5420c23c05bf13f3b2c3e Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 5 Dec 2023 15:18:38 -0500 Subject: [PATCH 039/567] unroll the loop --- .../split_explicit_free_surface_kernels.jl | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 5761e7ca29..6bdb90369b 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -138,8 +138,6 @@ using Oceananigans.DistributedComputations: Distributed using Printf @kernel function split_explicit_free_surface_evolution_kernel!(grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, averaging_weight, - Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, timestepper) i, j = @index(Global, NTuple) free_surface_evolution_kernel!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) @@ -208,12 +206,12 @@ function split_explicit_free_surface_substep!(η, state, auxiliary, settings, we parameters = auxiliary.kernel_parameters - args = (grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, averaging_weight, - Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, timestepper) - - launch!(arch, grid, parameters, split_explicit_free_surface_evolution_kernel!, args...;) - launch!(arch, grid, parameters, split_explicit_barotropic_velocity_evolution_kernel!, args...;) + launch!(arch, grid, parameters, split_explicit_free_surface_evolution_kernel!, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) + launch!(arch, grid, parameters, split_explicit_barotropic_velocity_evolution_kernel!, + grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², + η̅, U̅, V̅, averaging_weight, + Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, + timestepper) return nothing end @@ -359,6 +357,17 @@ const MINIMUM_SUBSTEPS = 5 @inline calculate_adaptive_settings(substepping::FTS, substeps) = weights_from_substeps(eltype(substepping.Δt_barotropic), substeps, substepping.averaging_kernel) +macro unroll_split_explicit_loop(exp) + lim2 = eval(exp.args[1].args[2].args[3]) + iterator = exp.args[1].args[1] + loop = quote + Base.Cartesian.@nexprs $lim2 $iterator -> $(exp.args[2]) + end + return quote + $(esc(loop)) + end +end + function iterate_split_explicit!(free_surface, grid, Δt) arch = architecture(grid) @@ -375,7 +384,7 @@ function iterate_split_explicit!(free_surface, grid, Δt) Δτᴮ = fractional_Δt * Δt # barotropic time step in seconds - for substep in 1:Nsubsteps + @unroll_split_explicit_loop for substep in 1:Nsubsteps split_explicit_free_surface_substep!(η, state, auxiliary, settings, weights, arch, grid, g, Δτᴮ, substep) end From 4b1f2cd6f1915bf96c7216fd2ce873d045e1e063 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 5 Dec 2023 15:23:35 -0500 Subject: [PATCH 040/567] fully unrolled --- .../split_explicit_free_surface_kernels.jl | 44 ++++++++----------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 6bdb90369b..9dcaa86fa0 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -192,30 +192,6 @@ end end end -function split_explicit_free_surface_substep!(η, state, auxiliary, settings, weights, arch, grid, g, Δτ, substep_index) - # unpack state quantities, parameters and forcing terms - U, V = state.U, state.V - Uᵐ⁻¹, Uᵐ⁻² = state.Uᵐ⁻¹, state.Uᵐ⁻² - Vᵐ⁻¹, Vᵐ⁻² = state.Vᵐ⁻¹, state.Vᵐ⁻² - ηᵐ, ηᵐ⁻¹, ηᵐ⁻² = state.ηᵐ, state.ηᵐ⁻¹, state.ηᵐ⁻² - η̅, U̅, V̅ = state.η̅, state.U̅, state.V̅ - Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ = auxiliary.Gᵁ, auxiliary.Gⱽ, auxiliary.Hᶠᶜ, auxiliary.Hᶜᶠ - - timestepper = settings.timestepper - averaging_weight = weights[substep_index] - - parameters = auxiliary.kernel_parameters - - launch!(arch, grid, parameters, split_explicit_free_surface_evolution_kernel!, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) - launch!(arch, grid, parameters, split_explicit_barotropic_velocity_evolution_kernel!, - grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, averaging_weight, - Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, - timestepper) - - return nothing -end - # Barotropic Model Kernels # u_Δz = u * Δz @kernel function _barotropic_mode_kernel!(U, V, grid, u, v) @@ -384,8 +360,26 @@ function iterate_split_explicit!(free_surface, grid, Δt) Δτᴮ = fractional_Δt * Δt # barotropic time step in seconds + # unpack state quantities, parameters and forcing terms + U, V = state.U, state.V + Uᵐ⁻¹, Uᵐ⁻² = state.Uᵐ⁻¹, state.Uᵐ⁻² + Vᵐ⁻¹, Vᵐ⁻² = state.Vᵐ⁻¹, state.Vᵐ⁻² + ηᵐ, ηᵐ⁻¹, ηᵐ⁻² = state.ηᵐ, state.ηᵐ⁻¹, state.ηᵐ⁻² + η̅, U̅, V̅ = state.η̅, state.U̅, state.V̅ + Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ = auxiliary.Gᵁ, auxiliary.Gⱽ, auxiliary.Hᶠᶜ, auxiliary.Hᶜᶠ + + timestepper = settings.timestepper + + parameters = auxiliary.kernel_parameters + @unroll_split_explicit_loop for substep in 1:Nsubsteps - split_explicit_free_surface_substep!(η, state, auxiliary, settings, weights, arch, grid, g, Δτᴮ, substep) + averaging_weight = weights[substep] + launch!(arch, grid, parameters, split_explicit_free_surface_evolution_kernel!, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) + launch!(arch, grid, parameters, split_explicit_barotropic_velocity_evolution_kernel!, + grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², + η̅, U̅, V̅, averaging_weight, + Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, + timestepper) end return nothing From 20a12d132b3feb710ed85a72acfc2528943546ee Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 5 Dec 2023 19:43:33 -0500 Subject: [PATCH 041/567] split explicit loop unrolling --- .../split_explicit_free_surface.jl | 16 +-- .../split_explicit_free_surface_kernels.jl | 98 ++++++++++++++----- 2 files changed, 76 insertions(+), 38 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index 8b78623673..fc4dbf33c8 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -85,20 +85,6 @@ function FreeSurface(free_surface::SplitExplicitFreeSurface, velocities, grid) free_surface.settings) end -function SplitExplicitFreeSurface(grid; gravitational_acceleration = g_Earth, - settings = SplitExplicitSettings(eltype(grid); gravitational_acceleration, substeps = 200)) - - if eltype(settings) != eltype(grid) - @warn "Using $(eltype(settings)) settings for the SplitExplicitFreeSurface on a $(eltype(grid)) grid" - end - - η = ZFaceField(grid, indices = (:, :, size(grid, 3)+1)) - gravitational_acceleration = convert(eltype(grid), gravitational_acceleration) - - return SplitExplicitFreeSurface(η, SplitExplicitState(grid), SplitExplicitAuxiliaryFields(grid), - gravitational_acceleration, settings) -end - """ struct SplitExplicitState @@ -287,7 +273,7 @@ end averaging_weights = averaging_weights[1:idx] averaging_weights ./= sum(averaging_weights) - return Δτ, averaging_weights + return Δτ, tuple(averaging_weights...) end function SplitExplicitSettings(FT::DataType=Float64; diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 9dcaa86fa0..aee662ea91 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -137,13 +137,12 @@ end using Oceananigans.DistributedComputations: Distributed using Printf -@kernel function split_explicit_free_surface_evolution_kernel!(grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², - timestepper) +@kernel function _split_explicit_free_surface!(grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) i, j = @index(Global, NTuple) - free_surface_evolution_kernel!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) + free_surface_evolution!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) end -@inline function free_surface_evolution_kernel!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) +@inline function free_surface_evolution!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) k_top = grid.Nz+1 TX, TY, _ = topology(grid) @@ -157,22 +156,22 @@ end return nothing end -@kernel function split_explicit_barotropic_velocity_evolution_kernel!(grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, averaging_weight, - Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, - timestepper) +@kernel function _split_explicit_barotropic_velocity!(grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², + η̅, U̅, V̅, averaging_weight, + Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, + timestepper) i, j = @index(Global, NTuple) - velocity_evolution_kernel!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, averaging_weight, - Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, - timestepper ) + velocity_evolution!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², + η̅, U̅, V̅, averaging_weight, + Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, + timestepper ) end -@inline function velocity_evolution_kernel!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, averaging_weight, - Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, - timestepper ) +@inline function velocity_evolution!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², + η̅, U̅, V̅, averaging_weight, + Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, + timestepper ) k_top = grid.Nz+1 TX, TY, _ = topology(grid) @@ -251,7 +250,7 @@ function initialize_free_surface_state!(free_surface_state, η) return nothing end -@kernel function barotropic_split_explicit_corrector_kernel!(u, v, U̅, V̅, U, V, Hᶠᶜ, Hᶜᶠ, grid) +@kernel function _barotropic_split_explicit_corrector!(u, v, U̅, V̅, U, V, Hᶠᶜ, Hᶜᶠ, grid) i, j, k = @index(Global, NTuple) @inbounds begin u[i, j, k] = u[i, j, k] + (U̅[i, j] - U[i, j]) / Hᶠᶜ[i, j] @@ -271,7 +270,7 @@ function barotropic_split_explicit_corrector!(u, v, free_surface, grid) compute_barotropic_mode!(U, V, grid, u, v) # add in "good" barotropic mode - launch!(arch, grid, :xyz, barotropic_split_explicit_corrector_kernel!, + launch!(arch, grid, :xyz, _barotropic_split_explicit_corrector!, u, v, U̅, V̅, U, V, Hᶠᶜ, Hᶜᶠ, grid) return nothing @@ -344,6 +343,54 @@ macro unroll_split_explicit_loop(exp) end end +const FixedSubstepsSetting{N} = SplitExplicitSettings{<:FixedSubstepNumber{<:Any, <:NTuple{N, <:Any}}} where N +const FixedSubstepsSplitExplicit{F} = SplitExplicitFreeSurface{<:Any, <:Any, <:Any, <:Any, <:FixedSubstepsSetting{N}} where N + +# For a fixed number of substeps it is possible to +function iterate_split_explicit!(free_surface::FixedSubstepsSplitExplicit{N}, grid, Δt) where N + arch = architecture(grid) + + η = free_surface.η + state = free_surface.state + auxiliary = free_surface.auxiliary + settings = free_surface.settings + g = free_surface.gravitational_acceleration + + weights = settings.substepping.averaging_weights + fractional_Δt = settings.substepping.fractional_step_size + + Δτᴮ = fractional_Δt * Δt # barotropic time step in seconds + + # unpack state quantities, parameters and forcing terms + U, V = state.U, state.V + Uᵐ⁻¹, Uᵐ⁻² = state.Uᵐ⁻¹, state.Uᵐ⁻² + Vᵐ⁻¹, Vᵐ⁻² = state.Vᵐ⁻¹, state.Vᵐ⁻² + ηᵐ, ηᵐ⁻¹, ηᵐ⁻² = state.ηᵐ, state.ηᵐ⁻¹, state.ηᵐ⁻² + η̅, U̅, V̅ = state.η̅, state.U̅, state.V̅ + Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ = auxiliary.Gᵁ, auxiliary.Gⱽ, auxiliary.Hᶠᶜ, auxiliary.Hᶜᶠ + + timestepper = settings.timestepper + parameters = auxiliary.kernel_parameters + + @unroll for substep in 1:N + Base.@_inline_meta + + averaging_weight = weights[substep] + + launch!(arch, grid, parameters, _split_explicit_free_surface!, + grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², + timestepper) + + launch!(arch, grid, parameters, _split_explicit_barotropic_velocity!, + grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², + η̅, U̅, V̅, averaging_weight, + Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, + timestepper) + end + + return nothing +end + function iterate_split_explicit!(free_surface, grid, Δt) arch = architecture(grid) @@ -359,7 +406,7 @@ function iterate_split_explicit!(free_surface, grid, Δt) Nsubsteps = length(weights) Δτᴮ = fractional_Δt * Δt # barotropic time step in seconds - + # unpack state quantities, parameters and forcing terms U, V = state.U, state.V Uᵐ⁻¹, Uᵐ⁻² = state.Uᵐ⁻¹, state.Uᵐ⁻² @@ -371,15 +418,20 @@ function iterate_split_explicit!(free_surface, grid, Δt) timestepper = settings.timestepper parameters = auxiliary.kernel_parameters + + @unroll for substep in 1:Nsubsteps - @unroll_split_explicit_loop for substep in 1:Nsubsteps averaging_weight = weights[substep] - launch!(arch, grid, parameters, split_explicit_free_surface_evolution_kernel!, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) + + launch!(arch, grid, parameters, split_explicit_free_surface_evolution_kernel!, + grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², + timestepper) + launch!(arch, grid, parameters, split_explicit_barotropic_velocity_evolution_kernel!, - grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², + grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅, averaging_weight, Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, - timestepper) + timestepper) end return nothing From 58e7acbbe3f4591e79b08efb195982a0551d9f17 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Wed, 6 Dec 2023 16:29:22 -0800 Subject: [PATCH 042/567] update --- .../distributed_grids.jl | 24 +++++++++---------- .../partition_assemble.jl | 12 +++++----- .../distributed_hydrostatic_turbulence.jl | 10 ++++---- 3 files changed, 22 insertions(+), 24 deletions(-) diff --git a/src/DistributedComputations/distributed_grids.jl b/src/DistributedComputations/distributed_grids.jl index 27064906bd..b79705cf90 100644 --- a/src/DistributedComputations/distributed_grids.jl +++ b/src/DistributedComputations/distributed_grids.jl @@ -80,9 +80,9 @@ function RectilinearGrid(arch::Distributed, TY = insert_connected_topology(TY, Ry, rj) TZ = insert_connected_topology(TZ, Rz, rk) - xl = partition(x, nx, arch, 1) - yl = partition(y, ny, arch, 2) - zl = partition(z, nz, arch, 3) + xl = Rx == 1 ? x : partition_coordinate(x, nx, arch, 1) + yl = Ry == 1 ? y : partition_coordinate(y, ny, arch, 2) + zl = Rz == 1 ? z : partition_coordinate(z, nz, arch, 3) Lx, xᶠᵃᵃ, xᶜᵃᵃ, Δxᶠᵃᵃ, Δxᶜᵃᵃ = generate_coordinate(FT, topology[1](), nx, Hx, xl, :x, child_architecture(arch)) Ly, yᵃᶠᵃ, yᵃᶜᵃ, Δyᵃᶠᵃ, Δyᵃᶜᵃ = generate_coordinate(FT, topology[2](), ny, Hy, yl, :y, child_architecture(arch)) @@ -126,9 +126,9 @@ function LatitudeLongitudeGrid(arch::Distributed, TY = insert_connected_topology(topology[2], Ry, rj) TZ = insert_connected_topology(topology[3], Rz, rk) - λl = partition(longitude, nλ, arch, 1) - φl = partition(latitude, nφ, arch, 2) - zl = partition(z, nz, arch, 3) + λl = Rx == 1 ? λ : partition_coordinate(longitude, nλ, arch, 1) + φl = Ry == 1 ? φ : partition_coordinate(latitude, nφ, arch, 2) + zl = Rz == 1 ? z : partition_coordinate(z, nz, arch, 3) # Calculate all direction (which might be stretched) # A direction is regular if the domain passed is a Tuple{<:Real, <:Real}, @@ -185,9 +185,9 @@ function reconstruct_global_grid(grid::DistributedRectilinearGrid) z = cpu_face_constructor_z(grid) ## This will not work with 3D parallelizations!! - xG = Rx == 1 ? x : assemble(x, nx, Rx, ri, rj, rk, arch.communicator) - yG = Ry == 1 ? y : assemble(y, ny, Ry, rj, ri, rk, arch.communicator) - zG = Rz == 1 ? z : assemble(z, nz, Rz, rk, ri, rj, arch.communicator) + xG = Rx == 1 ? x : assemble_coordinate(x, nx, Rx, ri, rj, rk, arch.communicator) + yG = Ry == 1 ? y : assemble_coordinate(y, ny, Ry, rj, ri, rk, arch.communicator) + zG = Rz == 1 ? z : assemble_coordinate(z, nz, Rz, rk, ri, rj, arch.communicator) child_arch = child_architecture(arch) @@ -228,9 +228,9 @@ function reconstruct_global_grid(grid::DistributedLatitudeLongitudeGrid) z = cpu_face_constructor_z(grid) ## This will not work with 3D parallelizations!! - λG = Rx == 1 ? λ : assemble(λ, nλ, Rx, ri, rj, rk, arch.communicator) - φG = Ry == 1 ? φ : assemble(φ, nφ, Ry, rj, ri, rk, arch.communicator) - zG = Rz == 1 ? z : assemble(z, nz, Rz, rk, ri, rj, arch.communicator) + λG = Rx == 1 ? λ : assemble_coordinate(λ, nλ, Rx, ri, rj, rk, arch.communicator) + φG = Ry == 1 ? φ : assemble_coordinate(φ, nφ, Ry, rj, ri, rk, arch.communicator) + zG = Rz == 1 ? z : assemble_coordinate(z, nz, Rz, rk, ri, rj, arch.communicator) child_arch = child_architecture(arch) diff --git a/src/DistributedComputations/partition_assemble.jl b/src/DistributedComputations/partition_assemble.jl index 0e96374955..c8b90e3cbb 100644 --- a/src/DistributedComputations/partition_assemble.jl +++ b/src/DistributedComputations/partition_assemble.jl @@ -37,8 +37,8 @@ end # Partitioning (localization of global objects) and assembly (global assembly of local objects) # Used for grid constructors (cpu_face_constructor_x, cpu_face_constructor_y, cpu_face_constructor_z) -# which means that we need to repeat the value at the right boundary -function partition(c::AbstractVector, n, arch, idx) +# We need to repeat the value at the right boundary +function partition_coordinate(c::AbstractVector, n, arch, idx) nl = concatenate_local_sizes(n, arch, idx) r = arch.local_index[idx] # Allow for Face values @@ -49,7 +49,7 @@ function partition(c::AbstractVector, n, arch, idx) end end -function partition(c::Tuple, n, arch, idx) +function partition_coordinate(c::Tuple, n, arch, idx) nl = concatenate_local_sizes(n, arch, idx) N = sum(nl) R = arch.ranks[idx] @@ -65,14 +65,14 @@ function partition(c::Tuple, n, arch, idx) end """ - assemble(c::AbstractVector, n, R, r, r1, r2, comm) + assemble_coordinate(c::AbstractVector, n, R, r, r1, r2, comm) Builds a linear global coordinate vector given a local coordinate vector `c_local` a local number of elements `Nc`, number of ranks `Nr`, rank `r`, and `arch`itecture. Since we use a global reduction, only ranks at positions 1 in the other two directions `r1 == 1` and `r2 == 1` fill the 1D array. """ -function assemble(c_local::AbstractVector, n, R, r, r1, r2, comm) +function assemble_coordinate(c_local::AbstractVector, n, R, r, r1, r2, comm) nl = concatenate_local_sizes(n, R, r) c_global = zeros(eltype(c_local), sum(nl)+1) @@ -88,7 +88,7 @@ function assemble(c_local::AbstractVector, n, R, r, r1, r2, comm) end # Simple case, just take the first and the last core -function assemble(c::Tuple, n, R, r, r1, r2, comm) +function assemble_coordinate(c::Tuple, n, R, r, r1, r2, comm) c_global = zeros(Float64, 2) if r == 1 && r1 == 1 && r2 == 1 diff --git a/validation/distributed_simulations/distributed_hydrostatic_turbulence.jl b/validation/distributed_simulations/distributed_hydrostatic_turbulence.jl index fdec0fcc83..01b899e87c 100644 --- a/validation/distributed_simulations/distributed_hydrostatic_turbulence.jl +++ b/validation/distributed_simulations/distributed_hydrostatic_turbulence.jl @@ -6,6 +6,7 @@ using Statistics using Oceananigans.BoundaryConditions using Oceananigans.DistributedComputations using Random +using JLD2 using Oceananigans.ImmersedBoundaries: ActiveCellsIBG, use_only_active_interior_cells # Run with @@ -15,14 +16,11 @@ using Oceananigans.ImmersedBoundaries: ActiveCellsIBG, use_only_active_interior_ # ``` function run_simulation(nx, ny, arch; topology = (Periodic, Periodic, Bounded)) - grid = RectilinearGrid(arch; topology, size = (Nx, Ny, 1), extent=(4π, 4π, 0.5), halo=(7, 7, 7)) + grid = RectilinearGrid(arch; topology, size = (Nx, Ny, 10), extent=(4π, 4π, 0.5), halo=(8, 8, 8)) bottom(x, y) = (x > π && x < 3π/2 && y > π/2 && y < 3π/2) ? 1.0 : - grid.Lz - 1.0 grid = ImmersedBoundaryGrid(grid, GridFittedBottom(bottom); active_cells_map = true) - @show grid isa ActiveCellsIBG - @show use_only_active_interior_cells(grid) - model = HydrostaticFreeSurfaceModel(; grid, momentum_advection = VectorInvariant(vorticity_scheme=WENO(order=9)), free_surface = SplitExplicitFreeSurface(substeps=10), @@ -65,8 +63,8 @@ function run_simulation(nx, ny, arch; topology = (Periodic, Periodic, Bounded)) MPI.Barrier(arch.communicator) end -Nx = 128 -Ny = 128 +Nx = 32 +Ny = 32 arch = Distributed(CPU(), partition = Partition(2, 2)) From 217e3af2fb81f81f1d6912560263e7692ba99b4f Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Wed, 6 Dec 2023 16:36:13 -0800 Subject: [PATCH 043/567] annotations --- src/Utils/kernel_launching.jl | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/src/Utils/kernel_launching.jl b/src/Utils/kernel_launching.jl index 1d0b744937..bdfffe6380 100644 --- a/src/Utils/kernel_launching.jl +++ b/src/Utils/kernel_launching.jl @@ -120,29 +120,39 @@ function launch!(arch, grid, workspec, kernel!, kernel_args...; only_active_cells = nothing, kwargs...) - workgroup, worksize = work_layout(grid, workspec; - include_right_boundaries, - reduced_dimensions, - location) + NVTX.@range "work layout" begin + workgroup, worksize = work_layout(grid, workspec; + include_right_boundaries, + reduced_dimensions, + location) + end - offset = offsets(workspec) + NVTX.@range "offsets" begin + offset = offsets(workspec) + end - if !isnothing(only_active_cells) - workgroup, worksize = active_cells_work_layout(workgroup, worksize, only_active_cells, grid) - offset = nothing + NVTX.@range "active cells layout" begin + if !isnothing(only_active_cells) + workgroup, worksize = active_cells_work_layout(workgroup, worksize, only_active_cells, grid) + offset = nothing + end end if worksize == 0 return nothing end - # We can only launch offset kernels with Static sizes!!!! - loop! = isnothing(offset) ? kernel!(Architectures.device(arch), workgroup, worksize) : - kernel!(Architectures.device(arch), StaticSize(workgroup), OffsetStaticSize(contiguousrange(worksize, offset))) + NVTX.@range "configuring kernel" begin + # We can only launch offset kernels with Static sizes!!!! + loop! = isnothing(offset) ? kernel!(Architectures.device(arch), workgroup, worksize) : + kernel!(Architectures.device(arch), StaticSize(workgroup), OffsetStaticSize(contiguousrange(worksize, offset))) + end @debug "Launching kernel $kernel! with worksize $worksize and offsets $offset from $workspec" - loop!(kernel_args...) + NVTX.@range "actual kernel" begin + loop!(kernel_args...) + end return nothing end From 8cf6453c04f83546528917f211954350c67156da Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Wed, 6 Dec 2023 16:40:42 -0800 Subject: [PATCH 044/567] using NVTX --- src/Utils/kernel_launching.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Utils/kernel_launching.jl b/src/Utils/kernel_launching.jl index bdfffe6380..ac5f7b5061 100644 --- a/src/Utils/kernel_launching.jl +++ b/src/Utils/kernel_launching.jl @@ -7,7 +7,7 @@ using Oceananigans.Utils using Oceananigans.Grids using Oceananigans.Grids: AbstractGrid - +using NVTX import Base struct KernelParameters{S, O} end From 3cc1468ff94dcca95eb6f84addc1f9db1841e17f Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Wed, 6 Dec 2023 17:29:59 -0800 Subject: [PATCH 045/567] add NVTX --- Manifest.toml | 12 ++++++------ Project.toml | 1 + 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Manifest.toml b/Manifest.toml index a596e1df16..827fabbfc6 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -1,8 +1,8 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.9.3" +julia_version = "1.9.4" manifest_format = "2.0" -project_hash = "72ed8b1b7715053c6d7b675f75dd867b9f153685" +project_hash = "21eb6b02d2870a916430d805acf3d926ca95d5b2" [[deps.AbstractFFTs]] deps = ["LinearAlgebra"] @@ -420,12 +420,12 @@ uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" [[deps.LibCURL]] deps = ["LibCURL_jll", "MozillaCACerts_jll"] uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" -version = "0.6.3" +version = "0.6.4" [[deps.LibCURL_jll]] deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" -version = "7.84.0+0" +version = "8.4.0+0" [[deps.LibGit2]] deps = ["Base64", "NetworkOptions", "Printf", "SHA"] @@ -434,7 +434,7 @@ uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" [[deps.LibSSH2_jll]] deps = ["Artifacts", "Libdl", "MbedTLS_jll"] uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" -version = "1.10.2+0" +version = "1.11.0+1" [[deps.Libdl]] uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" @@ -988,7 +988,7 @@ version = "5.8.0+0" [[deps.nghttp2_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" -version = "1.48.0+0" +version = "1.52.0+1" [[deps.p7zip_jll]] deps = ["Artifacts", "Libdl"] diff --git a/Project.toml b/Project.toml index 8e753f2e63..f2316435f4 100644 --- a/Project.toml +++ b/Project.toml @@ -22,6 +22,7 @@ LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195" NCDatasets = "85f8d34a-cbdd-5861-8df4-14fed0d494ab" +NVTX = "5da4648a-3479-48b8-97b9-01cb529c0a1f" OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" PencilArrays = "0e08944d-e94e-41b1-9406-dcf66b6a9d2e" From 1c1ff63d737582f4ec5b2881bf24ec5bdc14b140 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 7 Dec 2023 09:30:46 -0800 Subject: [PATCH 046/567] bugfix --- src/DistributedComputations/distributed_grids.jl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/DistributedComputations/distributed_grids.jl b/src/DistributedComputations/distributed_grids.jl index b79705cf90..e8f80a0320 100644 --- a/src/DistributedComputations/distributed_grids.jl +++ b/src/DistributedComputations/distributed_grids.jl @@ -126,9 +126,9 @@ function LatitudeLongitudeGrid(arch::Distributed, TY = insert_connected_topology(topology[2], Ry, rj) TZ = insert_connected_topology(topology[3], Rz, rk) - λl = Rx == 1 ? λ : partition_coordinate(longitude, nλ, arch, 1) - φl = Ry == 1 ? φ : partition_coordinate(latitude, nφ, arch, 2) - zl = Rz == 1 ? z : partition_coordinate(z, nz, arch, 3) + λl = Rx == 1 ? longitude : partition_coordinate(longitude, nλ, arch, 1) + φl = Ry == 1 ? latitude : partition_coordinate(latitude, nφ, arch, 2) + zl = Rz == 1 ? z : partition_coordinate(z, nz, arch, 3) # Calculate all direction (which might be stretched) # A direction is regular if the domain passed is a Tuple{<:Real, <:Real}, @@ -228,9 +228,9 @@ function reconstruct_global_grid(grid::DistributedLatitudeLongitudeGrid) z = cpu_face_constructor_z(grid) ## This will not work with 3D parallelizations!! - λG = Rx == 1 ? λ : assemble_coordinate(λ, nλ, Rx, ri, rj, rk, arch.communicator) - φG = Ry == 1 ? φ : assemble_coordinate(φ, nφ, Ry, rj, ri, rk, arch.communicator) - zG = Rz == 1 ? z : assemble_coordinate(z, nz, Rz, rk, ri, rj, arch.communicator) + λG = Rx == 1 ? longitude : assemble_coordinate(λ, nλ, Rx, ri, rj, rk, arch.communicator) + φG = Ry == 1 ? latitude : assemble_coordinate(φ, nφ, Ry, rj, ri, rk, arch.communicator) + zG = Rz == 1 ? z : assemble_coordinate(z, nz, Rz, rk, ri, rj, arch.communicator) child_arch = child_architecture(arch) From e0bedee953ef16d05d3375fd241b9cdffafa5c72 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 7 Dec 2023 09:57:09 -0800 Subject: [PATCH 047/567] bugfix --- src/DistributedComputations/distributed_grids.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/DistributedComputations/distributed_grids.jl b/src/DistributedComputations/distributed_grids.jl index e8f80a0320..b6fe50b5fb 100644 --- a/src/DistributedComputations/distributed_grids.jl +++ b/src/DistributedComputations/distributed_grids.jl @@ -228,9 +228,9 @@ function reconstruct_global_grid(grid::DistributedLatitudeLongitudeGrid) z = cpu_face_constructor_z(grid) ## This will not work with 3D parallelizations!! - λG = Rx == 1 ? longitude : assemble_coordinate(λ, nλ, Rx, ri, rj, rk, arch.communicator) - φG = Ry == 1 ? latitude : assemble_coordinate(φ, nφ, Ry, rj, ri, rk, arch.communicator) - zG = Rz == 1 ? z : assemble_coordinate(z, nz, Rz, rk, ri, rj, arch.communicator) + λG = Rx == 1 ? λ : assemble_coordinate(λ, nλ, Rx, ri, rj, rk, arch.communicator) + φG = Ry == 1 ? φ : assemble_coordinate(φ, nφ, Ry, rj, ri, rk, arch.communicator) + zG = Rz == 1 ? z : assemble_coordinate(z, nz, Rz, rk, ri, rj, arch.communicator) child_arch = child_architecture(arch) From b2f92ddd14be8b6837d9b5fa99d0c45184ef4803 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 7 Dec 2023 13:45:01 -0800 Subject: [PATCH 048/567] utils --- .../split_explicit_free_surface_kernels.jl | 39 ++++++------ src/Utils/Utils.jl | 2 +- src/Utils/kernel_launching.jl | 60 ++++++++++--------- 3 files changed, 55 insertions(+), 46 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index aee662ea91..fa8709d8c8 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -372,20 +372,21 @@ function iterate_split_explicit!(free_surface::FixedSubstepsSplitExplicit{N}, gr timestepper = settings.timestepper parameters = auxiliary.kernel_parameters + free_surface_kernel! = configured_kernel(arch, grid, parameters, _split_explicit_free_surface!) + barotropic_velocity_kernel! = configured_kernel(arch, grid, parameters, _split_explicit_barotropic_velocity!) + @unroll for substep in 1:N Base.@_inline_meta averaging_weight = weights[substep] - launch!(arch, grid, parameters, _split_explicit_free_surface!, - grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², - timestepper) + free_surface_kernel!(grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², + U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) - launch!(arch, grid, parameters, _split_explicit_barotropic_velocity!, - grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, averaging_weight, - Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, - timestepper) + barotropic_velocity_kernel!(grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², + η̅, U̅, V̅, averaging_weight, + Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, + timestepper) end return nothing @@ -419,19 +420,21 @@ function iterate_split_explicit!(free_surface, grid, Δt) parameters = auxiliary.kernel_parameters - @unroll for substep in 1:Nsubsteps + free_surface_kernel! = configured_kernel(arch, grid, parameters, _split_explicit_free_surface!) + barotropic_velocity_kernel! = configured_kernel(arch, grid, parameters, _split_explicit_barotropic_velocity!) - averaging_weight = weights[substep] + @unroll for substep in 1:N + Base.@_inline_meta - launch!(arch, grid, parameters, split_explicit_free_surface_evolution_kernel!, - grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², - timestepper) + averaging_weight = weights[substep] - launch!(arch, grid, parameters, split_explicit_barotropic_velocity_evolution_kernel!, - grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, averaging_weight, - Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, - timestepper) + free_surface_kernel!(grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², + U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) + + barotropic_velocity_kernel!(grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², + η̅, U̅, V̅, averaging_weight, + Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, + timestepper) end return nothing diff --git a/src/Utils/Utils.jl b/src/Utils/Utils.jl index 6c6543692c..761c8d23a6 100644 --- a/src/Utils/Utils.jl +++ b/src/Utils/Utils.jl @@ -1,6 +1,6 @@ module Utils -export launch_config, work_layout, launch!, KernelParameters +export configured_kernel, work_layout, launch!, KernelParameters export prettytime, pretty_filesize export tupleit, parenttuple, datatuple, datatuples export validate_intervals, time_to_run diff --git a/src/Utils/kernel_launching.jl b/src/Utils/kernel_launching.jl index ac5f7b5061..80a8307a99 100644 --- a/src/Utils/kernel_launching.jl +++ b/src/Utils/kernel_launching.jl @@ -120,43 +120,49 @@ function launch!(arch, grid, workspec, kernel!, kernel_args...; only_active_cells = nothing, kwargs...) - NVTX.@range "work layout" begin - workgroup, worksize = work_layout(grid, workspec; - include_right_boundaries, - reduced_dimensions, - location) - end - NVTX.@range "offsets" begin - offset = offsets(workspec) - end + loop! = configured_kernel(arch, grid, workspec, kernel!; + include_right_boundaries, + reduced_dimensions, + location, + only_active_cells, + kwargs...) + + loop!(kernel_args...) + + return nothing +end - NVTX.@range "active cells layout" begin - if !isnothing(only_active_cells) - workgroup, worksize = active_cells_work_layout(workgroup, worksize, only_active_cells, grid) - offset = nothing - end +function configured_kernel(arch, grid, workspec, kernel!; + include_right_boundaries = false, + reduced_dimensions = (), + location = nothing, + only_active_cells = nothing, + kwargs...) + + workgroup, worksize = work_layout(grid, workspec; + include_right_boundaries, + reduced_dimensions, + location) + + offset = offsets(workspec) + + if !isnothing(only_active_cells) + workgroup, worksize = active_cells_work_layout(workgroup, worksize, only_active_cells, grid) + offset = nothing end if worksize == 0 return nothing end - NVTX.@range "configuring kernel" begin - # We can only launch offset kernels with Static sizes!!!! - loop! = isnothing(offset) ? kernel!(Architectures.device(arch), workgroup, worksize) : - kernel!(Architectures.device(arch), StaticSize(workgroup), OffsetStaticSize(contiguousrange(worksize, offset))) - end + # We can only launch offset kernels with Static sizes!!!! + loop! = isnothing(offset) ? kernel!(Architectures.device(arch), workgroup, worksize) : + kernel!(Architectures.device(arch), StaticSize(workgroup), OffsetStaticSize(contiguousrange(worksize, offset))) - @debug "Launching kernel $kernel! with worksize $worksize and offsets $offset from $workspec" - - NVTX.@range "actual kernel" begin - loop!(kernel_args...) - end - - return nothing + return loop! end - + # When dims::Val @inline launch!(arch, grid, ::Val{workspec}, args...; kwargs...) where workspec = launch!(arch, grid, workspec, args...; kwargs...) From 00458ab1596f358370345a89471da89821e805ab Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 7 Dec 2023 20:34:21 -0800 Subject: [PATCH 049/567] try like this --- .../split_explicit_free_surface_kernels.jl | 4 ++-- src/Utils/kernel_launching.jl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index fa8709d8c8..50a40bba6e 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -420,8 +420,8 @@ function iterate_split_explicit!(free_surface, grid, Δt) parameters = auxiliary.kernel_parameters - free_surface_kernel! = configured_kernel(arch, grid, parameters, _split_explicit_free_surface!) - barotropic_velocity_kernel! = configured_kernel(arch, grid, parameters, _split_explicit_barotropic_velocity!) + free_surface_kernel! = configured_kernel(arch, grid, :xy, _split_explicit_free_surface!) + barotropic_velocity_kernel! = configured_kernel(arch, grid, :xy, _split_explicit_barotropic_velocity!) @unroll for substep in 1:N Base.@_inline_meta diff --git a/src/Utils/kernel_launching.jl b/src/Utils/kernel_launching.jl index 80a8307a99..543f977388 100644 --- a/src/Utils/kernel_launching.jl +++ b/src/Utils/kernel_launching.jl @@ -127,7 +127,7 @@ function launch!(arch, grid, workspec, kernel!, kernel_args...; location, only_active_cells, kwargs...) - + loop!(kernel_args...) return nothing From 08a86b52e0535c60857f26616554fc1cf2efc966 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 7 Dec 2023 21:11:04 -0800 Subject: [PATCH 050/567] text like this --- .../split_explicit_free_surface_kernels.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 50a40bba6e..996d6be7df 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -372,8 +372,8 @@ function iterate_split_explicit!(free_surface::FixedSubstepsSplitExplicit{N}, gr timestepper = settings.timestepper parameters = auxiliary.kernel_parameters - free_surface_kernel! = configured_kernel(arch, grid, parameters, _split_explicit_free_surface!) - barotropic_velocity_kernel! = configured_kernel(arch, grid, parameters, _split_explicit_barotropic_velocity!) + free_surface_kernel! = configured_kernel(arch, grid, :xy, _split_explicit_free_surface!) + barotropic_velocity_kernel! = configured_kernel(arch, grid, :xy, _split_explicit_barotropic_velocity!) @unroll for substep in 1:N Base.@_inline_meta From e402f5c79788e1d693519efdfd09bf0ff8fb0438 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Fri, 8 Dec 2023 06:28:06 -0800 Subject: [PATCH 051/567] remove reduced fields --- .../split_explicit_free_surface.jl | 32 ++++---- .../split_explicit_free_surface_kernels.jl | 76 +++++++++---------- 2 files changed, 51 insertions(+), 57 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index fc4dbf33c8..ac64f01521 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -135,17 +135,17 @@ function SplitExplicitState(grid::AbstractGrid) ηᵐ⁻¹ = ZFaceField(grid, indices = (:, :, size(grid, 3)+1)) ηᵐ⁻² = ZFaceField(grid, indices = (:, :, size(grid, 3)+1)) - U = Field((Face, Center, Nothing), grid) - V = Field((Center, Face, Nothing), grid) + U = ZFaceField(grid, indices = (:, :, size(grid, 3))) + V = ZFaceField(grid, indices = (:, :, size(grid, 3))) - Uᵐ⁻¹ = Field((Face, Center, Nothing), grid) - Vᵐ⁻¹ = Field((Center, Face, Nothing), grid) + Uᵐ⁻¹ = ZFaceField(grid, indices = (:, :, size(grid, 3))) + Vᵐ⁻¹ = ZFaceField(grid, indices = (:, :, size(grid, 3))) - Uᵐ⁻² = Field((Face, Center, Nothing), grid) - Vᵐ⁻² = Field((Center, Face, Nothing), grid) + Uᵐ⁻² = ZFaceField(grid, indices = (:, :, size(grid, 3))) + Vᵐ⁻² = ZFaceField(grid, indices = (:, :, size(grid, 3))) - U̅ = Field((Face, Center, Nothing), grid) - V̅ = Field((Center, Face, Nothing), grid) + U̅ = ZFaceField(grid, indices = (:, :, size(grid, 3))) + V̅ = ZFaceField(grid, indices = (:, :, size(grid, 3))) return SplitExplicitState(; ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅) end @@ -183,21 +183,21 @@ Return the `SplitExplicitAuxiliaryFields` for `grid`. """ function SplitExplicitAuxiliaryFields(grid::AbstractGrid) - Gᵁ = Field((Face, Center, Nothing), grid) - Gⱽ = Field((Center, Face, Nothing), grid) + Gᵁ = ZFaceField(grid, indices = (:, :, size(grid, 3))) + Gⱽ = ZFaceField(grid, indices = (:, :, size(grid, 3))) - Hᶠᶜ = Field((Face, Center, Nothing), grid) - Hᶜᶠ = Field((Center, Face, Nothing), grid) - Hᶜᶜ = Field((Center, Center, Nothing), grid) + Hᶠᶜ = ZFaceField(grid, indices = (:, :, size(grid, 3))) + Hᶜᶠ = ZFaceField(grid, indices = (:, :, size(grid, 3))) + Hᶜᶜ = ZFaceField(grid, indices = (:, :, size(grid, 3))) dz = GridMetricOperation((Face, Center, Center), Δz, grid) - sum!(Hᶠᶜ, dz) + Hᶠᶜ .= sum(dz; dims = 3) dz = GridMetricOperation((Center, Face, Center), Δz, grid) - sum!(Hᶜᶠ, dz) + Hᶜᶠ .= sum(dz; dims = 3) dz = GridMetricOperation((Center, Center, Center), Δz, grid) - sum!(Hᶜᶜ, dz) + Hᶜᶜ .= sum(dz; dims = 3) fill_halo_regions!((Hᶠᶜ, Hᶜᶠ, Hᶜᶜ)) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 996d6be7df..e3b93c7ae5 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -177,17 +177,17 @@ end TX, TY, _ = topology(grid) @inbounds begin - advance_previous_velocity!(i, j, 1, timestepper, U, Uᵐ⁻¹, Uᵐ⁻²) - advance_previous_velocity!(i, j, 1, timestepper, V, Vᵐ⁻¹, Vᵐ⁻²) + advance_previous_velocity!(i, j, k_top-1, timestepper, U, Uᵐ⁻¹, Uᵐ⁻²) + advance_previous_velocity!(i, j, k_top-1, timestepper, V, Vᵐ⁻¹, Vᵐ⁻²) # ∂τ(U) = - ∇η + G - U[i, j, 1] += Δτ * (- g * Hᶠᶜ[i, j] * ∂xᶠᶜᶠ_η(i, j, k_top, grid, TX, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gᵁ[i, j, 1]) - V[i, j, 1] += Δτ * (- g * Hᶜᶠ[i, j] * ∂yᶜᶠᶠ_η(i, j, k_top, grid, TY, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gⱽ[i, j, 1]) + U[i, j, k_top-1] += Δτ * (- g * Hᶠᶜ[i, j] * ∂xᶠᶜᶠ_η(i, j, k_top, grid, TX, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gᵁ[i, j, k_top-1]) + V[i, j, k_top-1] += Δτ * (- g * Hᶜᶠ[i, j] * ∂yᶜᶠᶠ_η(i, j, k_top, grid, TY, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gⱽ[i, j, k_top-1]) # time-averaging - η̅[i, j, k_top] += averaging_weight * η[i, j, k_top] - U̅[i, j, 1] += averaging_weight * U[i, j, 1] - V̅[i, j, 1] += averaging_weight * V[i, j, 1] + η̅[i, j, k_top] += averaging_weight * η[i, j, k_top] + U̅[i, j, k_top-1] += averaging_weight * U[i, j, k_top-1] + V̅[i, j, k_top-1] += averaging_weight * V[i, j, k_top-1] end end @@ -195,14 +195,15 @@ end # u_Δz = u * Δz @kernel function _barotropic_mode_kernel!(U, V, grid, u, v) i, j = @index(Global, NTuple) + k_top = grid.Nz+1 # hand unroll first loop - @inbounds U[i, j, 1] = Δzᶠᶜᶜ(i, j, 1, grid) * u[i, j, 1] - @inbounds V[i, j, 1] = Δzᶜᶠᶜ(i, j, 1, grid) * v[i, j, 1] + @inbounds U[i, j, k_top-1] = Δzᶠᶜᶜ(i, j, 1, grid) * u[i, j, 1] + @inbounds V[i, j, k_top-1] = Δzᶜᶠᶜ(i, j, 1, grid) * v[i, j, 1] @unroll for k in 2:grid.Nz - @inbounds U[i, j, 1] += Δzᶠᶜᶜ(i, j, k, grid) * u[i, j, k] - @inbounds V[i, j, 1] += Δzᶜᶠᶜ(i, j, k, grid) * v[i, j, k] + @inbounds U[i, j, k_top-1] += Δzᶠᶜᶜ(i, j, k, grid) * u[i, j, k] + @inbounds V[i, j, k_top-1] += Δzᶜᶠᶜ(i, j, k, grid) * v[i, j, k] end end @@ -211,18 +212,18 @@ end @kernel function _barotropic_mode_kernel!(U, V, grid::ActiveSurfaceIBG, u, v) idx = @index(Global, Linear) i, j = active_linear_index_to_tuple(idx, SurfaceMap(), grid) + k_top = grid.Nz+1 # hand unroll first loop - @inbounds U[i, j, 1] = Δzᶠᶜᶜ(i, j, 1, grid) * u[i, j, 1] - @inbounds V[i, j, 1] = Δzᶜᶠᶜ(i, j, 1, grid) * v[i, j, 1] + @inbounds U[i, j, k_top-1] = Δzᶠᶜᶜ(i, j, 1, grid) * u[i, j, 1] + @inbounds V[i, j, k_top-1] = Δzᶜᶠᶜ(i, j, 1, grid) * v[i, j, 1] @unroll for k in 2:grid.Nz - @inbounds U[i, j, 1] += Δzᶠᶜᶜ(i, j, k, grid) * u[i, j, k] - @inbounds V[i, j, 1] += Δzᶜᶠᶜ(i, j, k, grid) * v[i, j, k] + @inbounds U[i, j, k_top-1] += Δzᶠᶜᶜ(i, j, k, grid) * u[i, j, k] + @inbounds V[i, j, k_top-1] += Δzᶜᶠᶜ(i, j, k, grid) * v[i, j, k] end end -# may need to do Val(Nk) since it may not be known at compile compute_barotropic_mode!(U, V, grid, u, v) = launch!(architecture(grid), grid, :xy, _barotropic_mode_kernel!, U, V, grid, u, v; only_active_cells = use_only_active_surface_cells(grid)) @@ -252,9 +253,11 @@ end @kernel function _barotropic_split_explicit_corrector!(u, v, U̅, V̅, U, V, Hᶠᶜ, Hᶜᶠ, grid) i, j, k = @index(Global, NTuple) + k_top = grid.Nz+1 + @inbounds begin - u[i, j, k] = u[i, j, k] + (U̅[i, j] - U[i, j]) / Hᶠᶜ[i, j] - v[i, j, k] = v[i, j, k] + (V̅[i, j] - V[i, j]) / Hᶜᶠ[i, j] + u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / Hᶠᶜ[i, j, k_top-1] + v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / Hᶜᶠ[i, j, k_top-1] end end @@ -332,17 +335,6 @@ const MINIMUM_SUBSTEPS = 5 @inline calculate_adaptive_settings(substepping::FTS, substeps) = weights_from_substeps(eltype(substepping.Δt_barotropic), substeps, substepping.averaging_kernel) -macro unroll_split_explicit_loop(exp) - lim2 = eval(exp.args[1].args[2].args[3]) - iterator = exp.args[1].args[1] - loop = quote - Base.Cartesian.@nexprs $lim2 $iterator -> $(exp.args[2]) - end - return quote - $(esc(loop)) - end -end - const FixedSubstepsSetting{N} = SplitExplicitSettings{<:FixedSubstepNumber{<:Any, <:NTuple{N, <:Any}}} where N const FixedSubstepsSplitExplicit{F} = SplitExplicitFreeSurface{<:Any, <:Any, <:Any, <:Any, <:FixedSubstepsSetting{N}} where N @@ -372,8 +364,8 @@ function iterate_split_explicit!(free_surface::FixedSubstepsSplitExplicit{N}, gr timestepper = settings.timestepper parameters = auxiliary.kernel_parameters - free_surface_kernel! = configured_kernel(arch, grid, :xy, _split_explicit_free_surface!) - barotropic_velocity_kernel! = configured_kernel(arch, grid, :xy, _split_explicit_barotropic_velocity!) + free_surface_kernel! = configured_kernel(arch, grid, parameters, _split_explicit_free_surface!) + barotropic_velocity_kernel! = configured_kernel(arch, grid, parameters, _split_explicit_barotropic_velocity!) @unroll for substep in 1:N Base.@_inline_meta @@ -420,8 +412,8 @@ function iterate_split_explicit!(free_surface, grid, Δt) parameters = auxiliary.kernel_parameters - free_surface_kernel! = configured_kernel(arch, grid, :xy, _split_explicit_free_surface!) - barotropic_velocity_kernel! = configured_kernel(arch, grid, :xy, _split_explicit_barotropic_velocity!) + free_surface_kernel! = configured_kernel(arch, grid, parameters, _split_explicit_free_surface!) + barotropic_velocity_kernel! = configured_kernel(arch, grid, parameters, _split_explicit_barotropic_velocity!) @unroll for substep in 1:N Base.@_inline_meta @@ -443,14 +435,15 @@ end # Calculate RHS for the barotopic time step. @kernel function _compute_integrated_ab2_tendencies!(Gᵁ, Gⱽ, grid, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) i, j = @index(Global, NTuple) + k_top = grid.Nz+1 # hand unroll first loop - @inbounds Gᵁ[i, j, 1] = Δzᶠᶜᶜ(i, j, 1, grid) * ab2_step_Gu(i, j, 1, grid, Gu⁻, Guⁿ, χ) - @inbounds Gⱽ[i, j, 1] = Δzᶜᶠᶜ(i, j, 1, grid) * ab2_step_Gv(i, j, 1, grid, Gv⁻, Gvⁿ, χ) + @inbounds Gᵁ[i, j, k_top-1] = Δzᶠᶜᶜ(i, j, 1, grid) * ab2_step_Gu(i, j, 1, grid, Gu⁻, Guⁿ, χ) + @inbounds Gⱽ[i, j, k_top-1] = Δzᶜᶠᶜ(i, j, 1, grid) * ab2_step_Gv(i, j, 1, grid, Gv⁻, Gvⁿ, χ) @unroll for k in 2:grid.Nz - @inbounds Gᵁ[i, j, 1] += Δzᶠᶜᶜ(i, j, k, grid) * ab2_step_Gu(i, j, k, grid, Gu⁻, Guⁿ, χ) - @inbounds Gⱽ[i, j, 1] += Δzᶜᶠᶜ(i, j, k, grid) * ab2_step_Gv(i, j, k, grid, Gv⁻, Gvⁿ, χ) + @inbounds Gᵁ[i, j, k_top-1] += Δzᶠᶜᶜ(i, j, k, grid) * ab2_step_Gu(i, j, k, grid, Gu⁻, Guⁿ, χ) + @inbounds Gⱽ[i, j, k_top-1] += Δzᶜᶠᶜ(i, j, k, grid) * ab2_step_Gv(i, j, k, grid, Gv⁻, Gvⁿ, χ) end end @@ -458,14 +451,15 @@ end @kernel function _compute_integrated_ab2_tendencies!(Gᵁ, Gⱽ, grid::ActiveSurfaceIBG, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) idx = @index(Global, Linear) i, j = active_linear_index_to_tuple(idx, SurfaceMap(), grid) + k_top = grid.Nz+1 # hand unroll first loop - @inbounds Gᵁ[i, j, 1] = Δzᶠᶜᶜ(i, j, 1, grid) * ab2_step_Gu(i, j, 1, grid, Gu⁻, Guⁿ, χ) - @inbounds Gⱽ[i, j, 1] = Δzᶜᶠᶜ(i, j, 1, grid) * ab2_step_Gv(i, j, 1, grid, Gv⁻, Gvⁿ, χ) + @inbounds Gᵁ[i, j, k_top-1] = Δzᶠᶜᶜ(i, j, 1, grid) * ab2_step_Gu(i, j, 1, grid, Gu⁻, Guⁿ, χ) + @inbounds Gⱽ[i, j, k_top-1] = Δzᶜᶠᶜ(i, j, 1, grid) * ab2_step_Gv(i, j, 1, grid, Gv⁻, Gvⁿ, χ) @unroll for k in 2:grid.Nz - @inbounds Gᵁ[i, j, 1] += Δzᶠᶜᶜ(i, j, k, grid) * ab2_step_Gu(i, j, k, grid, Gu⁻, Guⁿ, χ) - @inbounds Gⱽ[i, j, 1] += Δzᶜᶠᶜ(i, j, k, grid) * ab2_step_Gv(i, j, k, grid, Gv⁻, Gvⁿ, χ) + @inbounds Gᵁ[i, j, k_top-1] += Δzᶠᶜᶜ(i, j, k, grid) * ab2_step_Gu(i, j, k, grid, Gu⁻, Guⁿ, χ) + @inbounds Gⱽ[i, j, k_top-1] += Δzᶜᶠᶜ(i, j, k, grid) * ab2_step_Gv(i, j, k, grid, Gv⁻, Gvⁿ, χ) end end From 6cf89bc50a0810e6de9e5b51252c739c1f0ad5d2 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Fri, 8 Dec 2023 07:37:49 -0800 Subject: [PATCH 052/567] small test --- .../split_explicit_free_surface_kernels.jl | 44 +++++++++---------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index e3b93c7ae5..b3e2eb8eac 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -101,9 +101,9 @@ end # Time stepping extrapolation U★, and η★ # AB3 step -@inline function U★(i, j, k, grid, ::AdamsBashforth3Scheme, ϕᵐ, ϕᵐ⁻¹, ϕᵐ⁻²) +@inline function U★(i, j, k, grid, ::AdamsBashforth3Scheme, Uᵐ, Uᵐ⁻¹, Uᵐ⁻²) FT = eltype(grid) - return @inbounds FT(α) * ϕᵐ[i, j, k] + FT(θ) * ϕᵐ⁻¹[i, j, k] + FT(β) * ϕᵐ⁻²[i, j, k] + return @inbounds FT(α) * Uᵐ[i, j, k] + FT(θ) * Uᵐ⁻¹[i, j, k] + FT(β) * Uᵐ⁻²[i, j, k] end @inline function η★(i, j, k, grid, ::AdamsBashforth3Scheme, ηᵐ⁺¹, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) @@ -112,7 +112,7 @@ end end # Forward Backward Step -@inline U★(i, j, k, grid, ::ForwardBackwardScheme, ϕ, args...) = @inbounds ϕ[i, j, k] +@inline U★(i, j, k, grid, ::ForwardBackwardScheme, U, args...) = @inbounds U[i, j, k] @inline η★(i, j, k, grid, ::ForwardBackwardScheme, η, args...) = @inbounds η[i, j, k] @inline advance_previous_velocity!(i, j, k, ::ForwardBackwardScheme, U, Uᵐ⁻¹, Uᵐ⁻²) = nothing @@ -137,38 +137,38 @@ end using Oceananigans.DistributedComputations: Distributed using Printf -@kernel function _split_explicit_free_surface!(grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) +@kernel function _split_explicit_free_surface!(grid, Δτ, η, U, V, timestepper) i, j = @index(Global, NTuple) - free_surface_evolution!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) + free_surface_evolution!(i, j, grid, Δτ, η, U, V, timestepper) end -@inline function free_surface_evolution!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) +@inline function free_surface_evolution!(i, j, grid, Δτ, η, U, V, timestepper) k_top = grid.Nz+1 TX, TY, _ = topology(grid) @inbounds begin advance_previous_free_surface!(i, j, k_top, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) - η[i, j, k_top] -= Δτ * (div_xᶜᶜᶠ_U(i, j, k_top-1, grid, TX, U★, timestepper, U, Uᵐ⁻¹, Uᵐ⁻²) + - div_yᶜᶜᶠ_V(i, j, k_top-1, grid, TY, U★, timestepper, V, Vᵐ⁻¹, Vᵐ⁻²)) + η[i, j, k_top] -= Δτ * (div_xᶜᶜᶠ_U(i, j, k_top-1, grid, TX, U★, timestepper, U, 0, 0) + + div_yᶜᶜᶠ_V(i, j, k_top-1, grid, TY, U★, timestepper, V, 0, 0)) end return nothing end -@kernel function _split_explicit_barotropic_velocity!(grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², +@kernel function _split_explicit_barotropic_velocity!(grid, Δτ, η, U, V, η̅, U̅, V̅, averaging_weight, Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, timestepper) i, j = @index(Global, NTuple) - velocity_evolution!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², + velocity_evolution!(i, j, grid, Δτ, η, U, V, η̅, U̅, V̅, averaging_weight, Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, - timestepper ) + timestepper) end -@inline function velocity_evolution!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², +@inline function velocity_evolution!(i, j, grid, Δτ, η, U, V, η̅, U̅, V̅, averaging_weight, Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, timestepper ) @@ -177,12 +177,12 @@ end TX, TY, _ = topology(grid) @inbounds begin - advance_previous_velocity!(i, j, k_top-1, timestepper, U, Uᵐ⁻¹, Uᵐ⁻²) - advance_previous_velocity!(i, j, k_top-1, timestepper, V, Vᵐ⁻¹, Vᵐ⁻²) + # advance_previous_velocity!(i, j, k_top-1, timestepper, U, Uᵐ⁻¹, Uᵐ⁻²) + # advance_previous_velocity!(i, j, k_top-1, timestepper, V, Vᵐ⁻¹, Vᵐ⁻²) # ∂τ(U) = - ∇η + G - U[i, j, k_top-1] += Δτ * (- g * Hᶠᶜ[i, j] * ∂xᶠᶜᶠ_η(i, j, k_top, grid, TX, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gᵁ[i, j, k_top-1]) - V[i, j, k_top-1] += Δτ * (- g * Hᶜᶠ[i, j] * ∂yᶜᶠᶠ_η(i, j, k_top, grid, TY, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gⱽ[i, j, k_top-1]) + U[i, j, k_top-1] += Δτ * (- g * Hᶠᶜ[i, j, k_top-1] * ∂xᶠᶜᶠ_η(i, j, k_top, grid, TX, η★, timestepper, η, 0, 0, 0) + Gᵁ[i, j, k_top-1]) + V[i, j, k_top-1] += Δτ * (- g * Hᶜᶠ[i, j, k_top-1] * ∂yᶜᶠᶠ_η(i, j, k_top, grid, TY, η★, timestepper, η, 0, 0, 0) + Gⱽ[i, j, k_top-1]) # time-averaging η̅[i, j, k_top] += averaging_weight * η[i, j, k_top] @@ -372,10 +372,9 @@ function iterate_split_explicit!(free_surface::FixedSubstepsSplitExplicit{N}, gr averaging_weight = weights[substep] - free_surface_kernel!(grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², - U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) - - barotropic_velocity_kernel!(grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², + free_surface_kernel!(grid, Δτᴮ, η, U, V, timestepper) + + barotropic_velocity_kernel!(grid, Δτᴮ, η, U, V, η̅, U̅, V̅, averaging_weight, Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, timestepper) @@ -420,10 +419,9 @@ function iterate_split_explicit!(free_surface, grid, Δt) averaging_weight = weights[substep] - free_surface_kernel!(grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², - U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) + free_surface_kernel!(grid, Δτᴮ, η, U, V, timestepper) - barotropic_velocity_kernel!(grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², + barotropic_velocity_kernel!(grid, Δτᴮ, η, U, V, η̅, U̅, V̅, averaging_weight, Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, timestepper) From 4c1c351ee8fb1cd2cc1c637ebf889af08c20350e Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 11 Dec 2023 10:06:14 -0500 Subject: [PATCH 053/567] first commit --- src/Grids/latitude_longitude_grid.jl | 12 +- src/Grids/rectilinear_grid.jl | 20 ++-- .../moving_vertical_coordinate.jl | 104 ++++++++++++++++++ ...te_hydrostatic_free_surface_model_state.jl | 13 ++- .../update_nonhydrostatic_model_state.jl | 2 +- .../update_shallow_water_state.jl | 2 +- src/TimeSteppers/quasi_adams_bashforth_2.jl | 4 +- src/TimeSteppers/runge_kutta_3.jl | 8 +- 8 files changed, 135 insertions(+), 30 deletions(-) create mode 100644 src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl diff --git a/src/Grids/latitude_longitude_grid.jl b/src/Grids/latitude_longitude_grid.jl index 51b01ce983..330f343b8d 100644 --- a/src/Grids/latitude_longitude_grid.jl +++ b/src/Grids/latitude_longitude_grid.jl @@ -1,6 +1,6 @@ using KernelAbstractions: @kernel, @index -struct LatitudeLongitudeGrid{FT, TX, TY, TZ, M, MY, FX, FY, FZ, VX, VY, VZ, Arch} <: AbstractHorizontallyCurvilinearGrid{FT, TX, TY, TZ, Arch} +struct LatitudeLongitudeGrid{FT, TX, TY, TZ, M, MY, FX, FY, FZF, FZC, VX, VY, VZ, Arch} <: AbstractHorizontallyCurvilinearGrid{FT, TX, TY, TZ, Arch} architecture :: Arch Nx :: Int Ny :: Int @@ -21,8 +21,8 @@ struct LatitudeLongitudeGrid{FT, TX, TY, TZ, M, MY, FX, FY, FZ, VX, VY, VZ, Arch Δφᵃᶜᵃ :: FY φᵃᶠᵃ :: VY φᵃᶜᵃ :: VY - Δzᵃᵃᶠ :: FZ - Δzᵃᵃᶜ :: FZ + Δzᵃᵃᶠ :: FZF + Δzᵃᵃᶜ :: FZC zᵃᵃᶠ :: VZ zᵃᵃᶜ :: VZ # Precomputed metrics M <: Nothing means metrics will be computed on the fly @@ -47,7 +47,7 @@ struct LatitudeLongitudeGrid{FT, TX, TY, TZ, M, MY, FX, FY, FZ, VX, VY, VZ, Arch λᶠᵃᵃ :: VX, λᶜᵃᵃ :: VX, Δφᵃᶠᵃ :: FY, Δφᵃᶜᵃ :: FY, φᵃᶠᵃ :: VY, φᵃᶜᵃ :: VY, - Δzᵃᵃᶠ :: FZ, Δzᵃᵃᶜ :: FZ, + Δzᵃᵃᶠ :: FZF, Δzᵃᵃᶜ :: FZC, zᵃᵃᶠ :: VZ, zᵃᵃᶜ :: VZ, Δxᶠᶜ::M, Δxᶜᶠ::M, Δxᶠᶠ::M, Δxᶜᶜ::M, @@ -55,11 +55,11 @@ struct LatitudeLongitudeGrid{FT, TX, TY, TZ, M, MY, FX, FY, FZ, VX, VY, VZ, Arch Azᶠᶜ::M, Azᶜᶠ::M, Azᶠᶠ::M, Azᶜᶜ::M, radius::FT) where {Arch, FT, TX, TY, TZ, - FX, FY, FZ, + FX, FY, FZF, FZC, VX, VY, VZ, M, MY} - return new{FT, TX, TY, TZ, M, MY, FX, FY, FZ, VX, VY, VZ, Arch}(architecture, + return new{FT, TX, TY, TZ, M, MY, FX, FY, FZF, FZC, VX, VY, VZ, Arch}(architecture, Nλ, Nφ, Nz, Hλ, Hφ, Hz, Lλ, Lφ, Lz, diff --git a/src/Grids/rectilinear_grid.jl b/src/Grids/rectilinear_grid.jl index 66c7728c63..4d3361b215 100644 --- a/src/Grids/rectilinear_grid.jl +++ b/src/Grids/rectilinear_grid.jl @@ -1,4 +1,4 @@ -struct RectilinearGrid{FT, TX, TY, TZ, FX, FY, FZ, VX, VY, VZ, Arch} <: AbstractRectilinearGrid{FT, TX, TY, TZ, Arch} +struct RectilinearGrid{FT, TX, TY, TZ, FX, FY, FZF, FZC, VX, VY, VZ, Arch} <: AbstractRectilinearGrid{FT, TX, TY, TZ, Arch} architecture :: Arch Nx :: Int Ny :: Int @@ -19,8 +19,8 @@ struct RectilinearGrid{FT, TX, TY, TZ, FX, FY, FZ, VX, VY, VZ, Arch} <: Abstract Δyᵃᶜᵃ :: FY yᵃᶠᵃ :: VY yᵃᶜᵃ :: VY - Δzᵃᵃᶠ :: FZ - Δzᵃᵃᶜ :: FZ + Δzᵃᵃᶠ :: FZF + Δzᵃᵃᶜ :: FZC zᵃᵃᶠ :: VZ zᵃᵃᶜ :: VZ @@ -32,17 +32,17 @@ struct RectilinearGrid{FT, TX, TY, TZ, FX, FY, FZ, VX, VY, VZ, Arch} <: Abstract xᶠᵃᵃ :: VX, xᶜᵃᵃ :: VX, Δyᵃᶠᵃ :: FY, Δyᵃᶜᵃ :: FY, yᵃᶠᵃ :: VY, yᵃᶜᵃ :: VY, - Δzᵃᵃᶠ :: FZ, Δzᵃᵃᶜ :: FZ, + Δzᵃᵃᶠ :: FZF, Δzᵃᵃᶜ :: FZC, zᵃᵃᶠ :: VZ, zᵃᵃᶜ :: VZ) where {Arch, FT, TX, TY, TZ, FX, VX, FY, - VY, FZ, VZ} + VY, FZF, FZC, VZ} - return new{FT, TX, TY, TZ, FX, FY, FZ, VX, VY, VZ, Arch}(arch, Nx, Ny, Nz, - Hx, Hy, Hz, Lx, Ly, Lz, - Δxᶠᵃᵃ, Δxᶜᵃᵃ, xᶠᵃᵃ, xᶜᵃᵃ, - Δyᵃᶠᵃ, Δyᵃᶜᵃ, yᵃᶠᵃ, yᵃᶜᵃ, - Δzᵃᵃᶠ, Δzᵃᵃᶜ, zᵃᵃᶠ, zᵃᵃᶜ) + return new{FT, TX, TY, TZ, FX, FY, FZF, FZC, VX, VY, VZ, Arch}(arch, Nx, Ny, Nz, + Hx, Hy, Hz, Lx, Ly, Lz, + Δxᶠᵃᵃ, Δxᶜᵃᵃ, xᶠᵃᵃ, xᶜᵃᵃ, + Δyᵃᶠᵃ, Δyᵃᶜᵃ, yᵃᶠᵃ, yᵃᶜᵃ, + Δzᵃᵃᶠ, Δzᵃᵃᶜ, zᵃᵃᶠ, zᵃᵃᶜ) end end diff --git a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl new file mode 100644 index 0000000000..2e3ac8cb14 --- /dev/null +++ b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl @@ -0,0 +1,104 @@ +using Oceananigans.Grids: AbstractGrid + +const RigidLidModel = HydrostaticFreeSurfaceModel{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Nothing} + +struct ZStarCoordinate{R, S, Z} + reference :: R + scaling :: S + star_value :: Z +end + +Adapt.adapt_structure(to, coord::ZStarCoordinate) = + ZStarCoordinate(Adapt.adapt(to, coord.reference), + Adapt.adapt(to, coord.scaling), + Adapt.adapt(to, coord.star_value)) + +const ZStarCoordinateRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:ZStarCoordinate} +const ZStarCoordinateLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:ZStarCoordinate} + +const ZStarCoordinateGrid = Union{ZStarCoordinateRG, ZStarCoordinateLLG} + +function MovingCoordinateGrid(grid::AbstractGrid{FT, TX, TY, TZ}, ::ZStarCoordinate) where {FT, TX, TY, TZ} + ΔzF = ZFaceField(grid) + ΔzC = CenterField(grid) + scaling = ZFaceField(grid, indices = (:, :, grid.Nz + 1)) + + Δzᵃᵃᶠ = ZStarCoordinate(grid.Δzᵃᵃᶠ, ΔzF, scaling) + Δzᵃᵃᶜ = ZStarCoordinate(grid.Δzᵃᵃᶜ, ΔzC, scaling) + + args = [] + for prop in propertynames(grid) + if prop == :Δzᵃᵃᶠ + push!(args, Δzᵃᵃᶠ) + elseif prop == :Δzᵃᵃᶜ + push!(args, Δzᵃᵃᶜ) + else + push!(args, getproperty(grid, prop)) + end + end + + GridType = getnamewrapper(grid) + + return GridType{TX, TY, TZ}(args...) +end + +# Fallback +update_vertical_coordinate!(model, grid) = nothing + +# Do not update if rigid lid!! +update_vertical_coordinate!(::RigidLidModel, ::ZStarCoordinateGrid) = nothing + +function update_vertical_coordinate!(model, grid::ZStarCoordinateGrid) + η = model.free_surface.η + + # Scaling + scaling = grid.Δzᵃᵃᶠ.scaling + + # Moving coordinates + Δzᵃᵃᶠ = grid.Δzᵃᵃᶠ.star_value + Δzᵃᵃᶜ = grid.Δzᵃᵃᶜ.star_value + + # Reference coordinates + Δz₀ᵃᵃᶠ = grid.Δzᵃᵃᶠ.reference + Δz₀ᵃᵃᶜ = grid.Δzᵃᵃᶜ.reference + + launch!(architecture(grid), grid, _update_scaling!, :xy, + scaling, η, grid.Lz, grid.Nz) + + # Update vertical coordinate + launch!(architecture(grid), grid, _update_z_star!, :xyz, + Δzᵃᵃᶠ, Δzᵃᵃᶜ, Δz₀ᵃᵃᶠ, Δz₀ᵃᵃᶜ, scaling) + + return nothing +end + +@kernel function update_scaling!(scaling, η, grid.Lz, grid.Nz) + i, j = @index(Global, NTuple) + @inbounds scaling[i, j, Nz+1] = (Lz + η[i, j, Nz+1]) / Lz +end + +@kernel function _update_z_star!(ΔzF, ΔzC, ΔzF₀, ΔzC₀, scaling) + i, j, k = @index(Global, NTuple) + @inbounds ΔzF[i, j, k] = scaling[i, j, Nz+1] * ΔzF₀[k] + @inbounds ΔzC[i, j, k] = scaling[i, j, Nz+1] * ΔzC₀[k] +end + +@kernel function _update_z_star!(ΔzF, ΔzC, ΔzF₀::Number, ΔzC₀::Number, scaling) + i, j, k = @index(Global, NTuple) + @inbounds ΔzF[i, j, k] = scaling[i, j, Nz+1] * ΔzF₀ + @inbounds ΔzC[i, j, k] = scaling[i, j, Nz+1] * ΔzC₀ +end + +import Oceananigans.Operators: Δzᶜᶜᶠ, Δzᶜᶜᶜ, Δzᶜᶠᶠ, Δzᶜᶠᶜ, Δzᶠᶜᶠ, Δzᶠᶜᶜ, Δzᶠᶠᶠ, Δzᶠᶠᶜ + +@inline Δzᶜᶜᶠ(i, j, k, grid::ZStarCoordinateGrid) = @inbounds grid.Δzᵃᵃᶠ.star_value[i, j, k] +@inline Δzᶜᶜᶜ(i, j, k, grid::ZStarCoordinateGrid) = @inbounds grid.Δzᵃᵃᶜ.star_value[i, j, k] + +@inline Δzᶜᶠᶠ(i, j, k, grid::ZStarCoordinateGrid) = ℑyᵃᶠᵃ(i, j, k, grid, grid.Δzᵃᵃᶠ.star_value) +@inline Δzᶜᶠᶜ(i, j, k, grid::ZStarCoordinateGrid) = ℑyᵃᶠᵃ(i, j, k, grid, grid.Δzᵃᵃᶜ.star_value) + +@inline Δzᶠᶜᶠ(i, j, k, grid::ZStarCoordinateGrid) = ℑxᶠᵃᵃ(i, j, k, grid, grid.Δzᵃᵃᶠ.star_value) +@inline Δzᶠᶜᶜ(i, j, k, grid::ZStarCoordinateGrid) = ℑxᶠᵃᵃ(i, j, k, grid, grid.Δzᵃᵃᶜ.star_value) + +@inline Δzᶠᶠᶠ(i, j, k, grid::ZStarCoordinateGrid) = ℑxyᶠᶠᵃ(i, j, k, grid, grid.Δzᵃᵃᶠ.star_value) +@inline Δzᶠᶠᶜ(i, j, k, grid::ZStarCoordinateGrid) = ℑxyᶠᶠᵃ(i, j, k, grid, grid.Δzᵃᵃᶜ.star_value) \ No newline at end of file diff --git a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl index f9888ae1b1..91514bff97 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl @@ -21,16 +21,16 @@ Update peripheral aspects of the model (auxiliary fields, halo regions, diffusiv hydrostatic pressure) to the current model state. If `callbacks` are provided (in an array), they are called in the end. """ -update_state!(model::HydrostaticFreeSurfaceModel, callbacks=[]) = update_state!(model, model.grid, callbacks) +update_state!(model::HydrostaticFreeSurfaceModel, Δt, callbacks=[]) = update_state!(model, model.grid, Δt, callbacks) -function update_state!(model::HydrostaticFreeSurfaceModel, grid, callbacks) +function update_state!(model::HydrostaticFreeSurfaceModel, grid, Δt, callbacks) @apply_regionally mask_immersed_model_fields!(model, grid) fill_halo_regions!(prognostic_fields(model), model.clock, fields(model)) fill_horizontal_velocity_halos!(model.velocities.u, model.velocities.v, model.architecture) - @apply_regionally compute_w_diffusivities_pressure!(model) + @apply_regionally compute_w_diffusivities_pressure!(model, Δt) fill_halo_regions!(model.velocities.w, model.clock, fields(model)) fill_halo_regions!(model.diffusivity_fields, model.clock, fields(model)) @@ -41,7 +41,8 @@ function update_state!(model::HydrostaticFreeSurfaceModel, grid, callbacks) end update_biogeochemical_state!(model.biogeochemistry, model) - + update_vertical_coordinate!(model, model.grid) + return nothing end @@ -60,8 +61,8 @@ function mask_immersed_model_fields!(model, grid) return nothing end -function compute_w_diffusivities_pressure!(model) - compute_w_from_continuity!(model) +function compute_w_diffusivities_pressure!(model, Δt) + compute_w_from_continuity!(model, Δt) calculate_diffusivities!(model.diffusivity_fields, model.closure, model) update_hydrostatic_pressure!(model.pressure.pHY′, model.architecture, model.grid, model.buoyancy, model.tracers) return nothing diff --git a/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl b/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl index 68f4d5e100..df2b12cb24 100644 --- a/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl +++ b/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl @@ -15,7 +15,7 @@ Update peripheral aspects of the model (halo regions, diffusivities, hydrostatic pressure) to the current model state. If `callbacks` are provided (in an array), they are called in the end. """ -function update_state!(model::NonhydrostaticModel, callbacks=[]) +function update_state!(model::NonhydrostaticModel, Δt, callbacks=[]) # Mask immersed tracers foreach(mask_immersed_field!, model.tracers) diff --git a/src/Models/ShallowWaterModels/update_shallow_water_state.jl b/src/Models/ShallowWaterModels/update_shallow_water_state.jl index baa5f184b8..b54d42edb0 100644 --- a/src/Models/ShallowWaterModels/update_shallow_water_state.jl +++ b/src/Models/ShallowWaterModels/update_shallow_water_state.jl @@ -9,7 +9,7 @@ import Oceananigans.TimeSteppers: update_state! Fill halo regions for `model.solution` and `model.tracers`. If `callbacks` are provided (in an array), they are called in the end. """ -function update_state!(model::ShallowWaterModel, callbacks=[]) +function update_state!(model::ShallowWaterModel, Δt, callbacks=[]) # Mask immersed fields foreach(mask_immersed_field!, model.solution) diff --git a/src/TimeSteppers/quasi_adams_bashforth_2.jl b/src/TimeSteppers/quasi_adams_bashforth_2.jl index b8a8dd4c5b..c62317c526 100644 --- a/src/TimeSteppers/quasi_adams_bashforth_2.jl +++ b/src/TimeSteppers/quasi_adams_bashforth_2.jl @@ -86,7 +86,7 @@ function time_step!(model::AbstractModel{<:QuasiAdamsBashforth2TimeStepper}, Δt model.timestepper.previous_Δt = Δt # Be paranoid and update state at iteration 0 - model.clock.iteration == 0 && update_state!(model, callbacks) + model.clock.iteration == 0 && update_state!(model, Δt, callbacks) @apply_regionally calculate_tendencies!(model, callbacks) @@ -96,7 +96,7 @@ function time_step!(model::AbstractModel{<:QuasiAdamsBashforth2TimeStepper}, Δt @apply_regionally correct_velocities_and_store_tendecies!(model, Δt) tick!(model.clock, Δt) - update_state!(model, callbacks) + update_state!(model, Δt, callbacks) step_lagrangian_particles!(model, Δt) return nothing diff --git a/src/TimeSteppers/runge_kutta_3.jl b/src/TimeSteppers/runge_kutta_3.jl index 731426e677..23c289382d 100644 --- a/src/TimeSteppers/runge_kutta_3.jl +++ b/src/TimeSteppers/runge_kutta_3.jl @@ -82,7 +82,7 @@ function time_step!(model::AbstractModel{<:RungeKutta3TimeStepper}, Δt; callbac Δt == 0 && @warn "Δt == 0 may cause model blowup!" # Be paranoid and update state at iteration 0, in case run! is not used: - model.clock.iteration == 0 && update_state!(model, callbacks) + model.clock.iteration == 0 && update_state!(model, Δt, callbacks) γ¹ = model.timestepper.γ¹ γ² = model.timestepper.γ² @@ -110,7 +110,7 @@ function time_step!(model::AbstractModel{<:RungeKutta3TimeStepper}, Δt; callbac tick!(model.clock, first_stage_Δt; stage=true) store_tendencies!(model) - update_state!(model, callbacks) + update_state!(model, Δt, callbacks) step_lagrangian_particles!(model, first_stage_Δt) # @@ -128,7 +128,7 @@ function time_step!(model::AbstractModel{<:RungeKutta3TimeStepper}, Δt; callbac tick!(model.clock, second_stage_Δt; stage=true) store_tendencies!(model) - update_state!(model, callbacks) + update_state!(model, Δt, callbacks) step_lagrangian_particles!(model, second_stage_Δt) # @@ -145,7 +145,7 @@ function time_step!(model::AbstractModel{<:RungeKutta3TimeStepper}, Δt; callbac pressure_correct_velocities!(model, third_stage_Δt) tick!(model.clock, third_stage_Δt) - update_state!(model, callbacks) + update_state!(model, Δt, callbacks) step_lagrangian_particles!(model, third_stage_Δt) return nothing From 2157efcb52c7610a3401321c505e8ec5027ece18 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 11 Dec 2023 10:14:46 -0500 Subject: [PATCH 054/567] small change --- src/Grids/latitude_longitude_grid.jl | 2 +- src/TimeSteppers/runge_kutta_3.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Grids/latitude_longitude_grid.jl b/src/Grids/latitude_longitude_grid.jl index 6119581a26..cef30deb5a 100644 --- a/src/Grids/latitude_longitude_grid.jl +++ b/src/Grids/latitude_longitude_grid.jl @@ -1,6 +1,6 @@ using KernelAbstractions: @kernel, @index -struct LatitudeLongitudeGrid{FT, TX, TY, TZ, M, MY, FX, FY, FZF, FZC, VX, VY, VZ, Arch} <: AbstractUnderlyingGrid{FT, TX, TY, TZ, Arch} +struct LatitudeLongitudeGrid{FT, TX, TY, TZ, M, MY, FX, FY, FZF, FZC, VX, VY, VZ, Arch} <: AbstractHorizontallyCurvilinearGrid{FT, TX, TY, TZ, Arch} architecture :: Arch Nx :: Int Ny :: Int diff --git a/src/TimeSteppers/runge_kutta_3.jl b/src/TimeSteppers/runge_kutta_3.jl index b575666551..ac28b59c21 100644 --- a/src/TimeSteppers/runge_kutta_3.jl +++ b/src/TimeSteppers/runge_kutta_3.jl @@ -82,7 +82,7 @@ function time_step!(model::AbstractModel{<:RungeKutta3TimeStepper}, Δt; callbac Δt == 0 && @warn "Δt == 0 may cause model blowup!" # Be paranoid and update state at iteration 0, in case run! is not used: - model.clock.iteration == 0 && update_state!(model, Δt, callbacks) + model.clock.iteration == 0 && update_state!(model, callbacks) γ¹ = model.timestepper.γ¹ γ² = model.timestepper.γ² From 731eada0b4dbfabf08ec0b0b205cd67da25c23b5 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 11 Dec 2023 11:13:03 -0500 Subject: [PATCH 055/567] test it out? --- src/Grids/rectilinear_grid.jl | 2 +- .../HydrostaticFreeSurfaceModels.jl | 1 + ...static_free_surface_boundary_tendencies.jl | 3 +- ...ute_hydrostatic_free_surface_tendencies.jl | 2 + .../hydrostatic_free_surface_model.jl | 6 ++ .../moving_vertical_coordinate.jl | 72 +++++++++++++------ .../split_explicit_free_surface_kernels.jl | 4 +- ...te_hydrostatic_free_surface_model_state.jl | 6 +- 8 files changed, 71 insertions(+), 25 deletions(-) diff --git a/src/Grids/rectilinear_grid.jl b/src/Grids/rectilinear_grid.jl index 639be605ff..c1eef184c6 100644 --- a/src/Grids/rectilinear_grid.jl +++ b/src/Grids/rectilinear_grid.jl @@ -36,7 +36,7 @@ struct RectilinearGrid{FT, TX, TY, TZ, FX, FY, FZF, FZC, VX, VY, VZ, Arch} <: Ab zᵃᵃᶠ :: VZ, zᵃᵃᶜ :: VZ) where {Arch, FT, TX, TY, TZ, FX, VX, FY, - VY, FZ, VZ} + VY, FZF, FZC, VZ} return new{FT, TX, TY, TZ, FX, FY, FZF, FZC, VX, VY, VZ, Arch}(arch, Nx, Ny, Nz, Hx, Hy, Hz, Lx, Ly, Lz, diff --git a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl index f5e1f793b6..f2e299ee8b 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl @@ -30,6 +30,7 @@ fill_horizontal_velocity_halos!(args...) = nothing FreeSurfaceDisplacementField(velocities, free_surface, grid) = ZFaceField(grid, indices = (:, :, size(grid, 3)+1)) FreeSurfaceDisplacementField(velocities, ::Nothing, grid) = nothing +include("moving_vertical_coordinate.jl") include("compute_w_from_continuity.jl") include("rigid_lid.jl") diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl index ab510f7c76..74c837a865 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl @@ -17,7 +17,8 @@ function compute_boundary_tendencies!(model::HydrostaticFreeSurfaceModel) p_parameters = boundary_p_kernel_parameters(grid, arch) κ_parameters = boundary_κ_kernel_parameters(grid, model.closure, arch) - # We need new values for `w`, `p` and `κ` + # We need new values for `w`, `p` and `κ` + update_vertical_coordinate!(model, model.grid; parameters = κ_parameters) compute_auxiliaries!(model; w_parameters, p_parameters, κ_parameters) # parameters for communicating North / South / East / West side diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl index 996eaf1ad4..6edc4320af 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl @@ -26,6 +26,8 @@ function compute_tendencies!(model::HydrostaticFreeSurfaceModel, callbacks) kernel_parameters = tuple(interior_tendency_kernel_parameters(model.grid)) + update_vertical_coordinate!(model, model.grid; parameters = kernel_parameters) + # Calculate contributions to momentum and tracer tendencies from fluxes and volume terms in the # interior of the domain compute_hydrostatic_free_surface_tendency_contributions!(model, kernel_parameters; diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl index 89c67d2f4d..f4f64c8ee4 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl @@ -65,6 +65,7 @@ end velocities = nothing, pressure = nothing, diffusivity_fields = nothing, + vertical_coordinate = ZCoordinate(), auxiliary_fields = NamedTuple(), ) @@ -91,6 +92,7 @@ Keyword arguments - `velocities`: The model velocities. Default: `nothing`. - `pressure`: Hydrostatic pressure field. Default: `nothing`. - `diffusivity_fields`: Diffusivity fields. Default: `nothing`. + - `vertical_coordinate`: choice between the default `ZCoordinate` and the free-surface following `ZStarCoordinate` - `auxiliary_fields`: `NamedTuple` of auxiliary fields. Default: `nothing`. """ function HydrostaticFreeSurfaceModel(; grid, @@ -109,12 +111,16 @@ function HydrostaticFreeSurfaceModel(; grid, velocities = nothing, pressure = nothing, diffusivity_fields = nothing, + vertical_coordinate = ZCoordinate(), auxiliary_fields = NamedTuple() ) # Check halos and throw an error if the grid's halo is too small @apply_regionally validate_model_halo(grid, momentum_advection, tracer_advection, closure) + # Introduce z-star coordinates if needed + grid = MovingCoordinateGrid(grid, vertical_coordinate) + arch = architecture(grid) @apply_regionally momentum_advection = validate_momentum_advection(momentum_advection, grid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl index 2e3ac8cb14..d5952fc237 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl @@ -1,6 +1,11 @@ -using Oceananigans.Grids: AbstractGrid +using Oceananigans +using Oceananigans.Grids +using Oceananigans.Grids: AbstractGrid, AbstractUnderlyingGrid, halo_size +using Oceananigans.ImmersedBoundaries +using Oceananigans.Utils: getnamewrapper +using Adapt -const RigidLidModel = HydrostaticFreeSurfaceModel{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Nothing} +struct ZCoordinate end struct ZStarCoordinate{R, S, Z} reference :: R @@ -8,6 +13,8 @@ struct ZStarCoordinate{R, S, Z} star_value :: Z end +ZStarCoordinate() = ZStarCoordinate(nothing, nothing, nothing) + Adapt.adapt_structure(to, coord::ZStarCoordinate) = ZStarCoordinate(Adapt.adapt(to, coord.reference), Adapt.adapt(to, coord.scaling), @@ -16,15 +23,28 @@ Adapt.adapt_structure(to, coord::ZStarCoordinate) = const ZStarCoordinateRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:ZStarCoordinate} const ZStarCoordinateLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:ZStarCoordinate} -const ZStarCoordinateGrid = Union{ZStarCoordinateRG, ZStarCoordinateLLG} +const ZStarCoordinateUnderlyingGrid = Union{ZStarCoordinateRG, ZStarCoordinateLLG} +const ZStarCoordinateImmersedGrid = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:ZStarCoordinateUnderlyingGrid} + +const ZStarCoordinateGrid = Union{ZStarCoordinateUnderlyingGrid, ZStarCoordinateImmersedGrid} + +MovingCoordinateGrid(grid, coord) = grid + +function MovingCoordinateGrid(grid::ImmersedBoundaryGrid, ::ZStarCoordinate) + underlying_grid = MovingCoordinateGrid(grid.underlying_grid, ZStarCoordinate()) + active_cells_map = !isnothing(grid.active_cells_map) + + return ImmersedBoundaryGrid(underlying_grid, grid.immersed_boundary; active_cells_map) +end -function MovingCoordinateGrid(grid::AbstractGrid{FT, TX, TY, TZ}, ::ZStarCoordinate) where {FT, TX, TY, TZ} +function MovingCoordinateGrid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, ::ZStarCoordinate) where {FT, TX, TY, TZ} + # Memory layout for Dz spacings is local in z ΔzF = ZFaceField(grid) ΔzC = CenterField(grid) scaling = ZFaceField(grid, indices = (:, :, grid.Nz + 1)) - Δzᵃᵃᶠ = ZStarCoordinate(grid.Δzᵃᵃᶠ, ΔzF, scaling) - Δzᵃᵃᶜ = ZStarCoordinate(grid.Δzᵃᵃᶜ, ΔzC, scaling) + Δzᵃᵃᶠ = ZStarCoordinate(grid.Δzᵃᵃᶠ, scaling, ΔzF) + Δzᵃᵃᶜ = ZStarCoordinate(grid.Δzᵃᵃᶜ, scaling, ΔzC) args = [] for prop in propertynames(grid) @@ -43,12 +63,9 @@ function MovingCoordinateGrid(grid::AbstractGrid{FT, TX, TY, TZ}, ::ZStarCoordin end # Fallback -update_vertical_coordinate!(model, grid) = nothing +update_vertical_coordinate!(model, grid; kwargs...) = nothing -# Do not update if rigid lid!! -update_vertical_coordinate!(::RigidLidModel, ::ZStarCoordinateGrid) = nothing - -function update_vertical_coordinate!(model, grid::ZStarCoordinateGrid) +function update_vertical_coordinate!(model, grid::ZStarCoordinateGrid; parameters = tuple(:xyz)) η = model.free_surface.η # Scaling @@ -62,28 +79,41 @@ function update_vertical_coordinate!(model, grid::ZStarCoordinateGrid) Δz₀ᵃᵃᶠ = grid.Δzᵃᵃᶠ.reference Δz₀ᵃᵃᶜ = grid.Δzᵃᵃᶜ.reference - launch!(architecture(grid), grid, _update_scaling!, :xy, - scaling, η, grid.Lz, grid.Nz) + # Update the scaling on the whole grid (from -H to N+H) + launch!(architecture(grid), grid, horizontal_parameters(grid), _update_scaling!, + scaling, η, grid) - # Update vertical coordinate - launch!(architecture(grid), grid, _update_z_star!, :xyz, - Δzᵃᵃᶠ, Δzᵃᵃᶜ, Δz₀ᵃᵃᶠ, Δz₀ᵃᵃᶜ, scaling) + # Update vertical coordinate with available parameters + for params in parameters + launch!(architecture(grid), grid, params, _update_z_star!, + Δzᵃᵃᶠ, Δzᵃᵃᶜ, Δz₀ᵃᵃᶠ, Δz₀ᵃᵃᶜ, scaling, grid.Nz) + end return nothing end -@kernel function update_scaling!(scaling, η, grid.Lz, grid.Nz) +function horizontal_parameters(grid) + halos = halo_size(grid)[1:2] + total_size = size(grid)[1:2] .+ 2 .* halos + return KernelParameters(total_size, .- halos) +end + +@kernel function _update_scaling!(scaling, η, grid) i, j = @index(Global, NTuple) - @inbounds scaling[i, j, Nz+1] = (Lz + η[i, j, Nz+1]) / Lz + bottom = bottom_height(grid, i, j) + @inbounds scaling[i, j, grid.Nz+1] = (bottom + η[i, j, grid.Nz+1]) / bottom end -@kernel function _update_z_star!(ΔzF, ΔzC, ΔzF₀, ΔzC₀, scaling) +bottom_height(grid, i, j) = grid.Lz +bottom_height(grid::ImmersedBoundaryGrid, i, j) = @inbounds - grid.immersed_boundary.bottom_height[i, j, 1] + +@kernel function _update_z_star!(ΔzF, ΔzC, ΔzF₀, ΔzC₀, scaling, Nz) i, j, k = @index(Global, NTuple) @inbounds ΔzF[i, j, k] = scaling[i, j, Nz+1] * ΔzF₀[k] @inbounds ΔzC[i, j, k] = scaling[i, j, Nz+1] * ΔzC₀[k] end -@kernel function _update_z_star!(ΔzF, ΔzC, ΔzF₀::Number, ΔzC₀::Number, scaling) +@kernel function _update_z_star!(ΔzF, ΔzC, ΔzF₀::Number, ΔzC₀::Number, scaling, Nz) i, j, k = @index(Global, NTuple) @inbounds ΔzF[i, j, k] = scaling[i, j, Nz+1] * ΔzF₀ @inbounds ΔzC[i, j, k] = scaling[i, j, Nz+1] * ΔzC₀ @@ -91,6 +121,8 @@ end import Oceananigans.Operators: Δzᶜᶜᶠ, Δzᶜᶜᶜ, Δzᶜᶠᶠ, Δzᶜᶠᶜ, Δzᶠᶜᶠ, Δzᶠᶜᶜ, Δzᶠᶠᶠ, Δzᶠᶠᶜ +# Very bad for GPU performance!!! (z-values are not coalesced in memory for z-derivatives anymore) +# TODO: make z-direction local in memory by not using Fields @inline Δzᶜᶜᶠ(i, j, k, grid::ZStarCoordinateGrid) = @inbounds grid.Δzᵃᵃᶠ.star_value[i, j, k] @inline Δzᶜᶜᶜ(i, j, k, grid::ZStarCoordinateGrid) = @inbounds grid.Δzᵃᵃᶜ.star_value[i, j, k] diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 1062cd43c9..6eee1f8a19 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -369,7 +369,7 @@ end # This function is called after `calculate_tendency` and before `ab2_step_velocities!` function setup_free_surface!(model, free_surface::SplitExplicitFreeSurface, χ) - free_surface_grid = free_surface.η.grid + grid = model.grid # we start the time integration of η from the average ηⁿ Gu⁻ = model.timestepper.G⁻.u @@ -379,7 +379,7 @@ function setup_free_surface!(model, free_surface::SplitExplicitFreeSurface, χ) auxiliary = free_surface.auxiliary - @apply_regionally setup_split_explicit_tendency!(auxiliary, free_surface_grid, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) + @apply_regionally setup_split_explicit_tendency!(auxiliary, grid, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) fields_to_fill = (auxiliary.Gᵁ, auxiliary.Gⱽ) fill_halo_regions!(fields_to_fill; async = true) diff --git a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl index b2f34da314..4843ecc571 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl @@ -36,7 +36,6 @@ function update_state!(model::HydrostaticFreeSurfaceModel, grid, callbacks; comp @apply_regionally update_model_field_time_series!(model, model.clock) fill_halo_regions!(prognostic_fields(model), model.clock, fields(model); async = true) - update_vertical_coordinate!(model, model.grid) @apply_regionally replace_horizontal_vector_halos!(model.velocities, model.grid) @apply_regionally compute_auxiliaries!(model) @@ -85,3 +84,8 @@ function compute_auxiliaries!(model::HydrostaticFreeSurfaceModel; w_parameters = end return nothing end + +# Do not update if rigid lid!! +const RigidLidModel = HydrostaticFreeSurfaceModel{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Nothing} + +update_vertical_coordinate!(::RigidLidModel, ::ZStarCoordinateGrid; kwargs...) = nothing From ef574a0d43ec699fa3e7741c4e496aa182a878a6 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 11 Dec 2023 11:18:00 -0500 Subject: [PATCH 056/567] fix boundaries --- .../HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl index d5952fc237..56c276a132 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl @@ -89,6 +89,9 @@ function update_vertical_coordinate!(model, grid::ZStarCoordinateGrid; parameter Δzᵃᵃᶠ, Δzᵃᵃᶜ, Δz₀ᵃᵃᶠ, Δz₀ᵃᵃᶜ, scaling, grid.Nz) end + fill_halo_regions!(Δzᵃᵃᶠ; only_local_halos = true) + fill_halo_regions!(Δzᵃᵃᶜ; only_local_halos = true) + return nothing end From 43bb6bffb9c43e5943662a31303b356b13570185 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 11 Dec 2023 11:19:09 -0500 Subject: [PATCH 057/567] test it like this --- .../moving_vertical_coordinate.jl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl index 56c276a132..00ee27b76e 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl @@ -89,9 +89,8 @@ function update_vertical_coordinate!(model, grid::ZStarCoordinateGrid; parameter Δzᵃᵃᶠ, Δzᵃᵃᶜ, Δz₀ᵃᵃᶠ, Δz₀ᵃᵃᶜ, scaling, grid.Nz) end - fill_halo_regions!(Δzᵃᵃᶠ; only_local_halos = true) - fill_halo_regions!(Δzᵃᵃᶜ; only_local_halos = true) - + fill_halo_regions!((Δzᵃᵃᶠ, Δzᵃᵃᶜ); only_local_halos = true) + return nothing end From 9c4fe387535b6a6fa958c462ec3e36be387b7992 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 11 Dec 2023 12:24:47 -0500 Subject: [PATCH 058/567] small change --- .../moving_vertical_coordinate.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl index 00ee27b76e..024cee9a29 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl @@ -135,4 +135,8 @@ import Oceananigans.Operators: Δzᶜᶜᶠ, Δzᶜᶜᶜ, Δzᶜᶠᶠ, Δzᶜ @inline Δzᶠᶜᶜ(i, j, k, grid::ZStarCoordinateGrid) = ℑxᶠᵃᵃ(i, j, k, grid, grid.Δzᵃᵃᶜ.star_value) @inline Δzᶠᶠᶠ(i, j, k, grid::ZStarCoordinateGrid) = ℑxyᶠᶠᵃ(i, j, k, grid, grid.Δzᵃᵃᶠ.star_value) -@inline Δzᶠᶠᶜ(i, j, k, grid::ZStarCoordinateGrid) = ℑxyᶠᶠᵃ(i, j, k, grid, grid.Δzᵃᵃᶜ.star_value) \ No newline at end of file +@inline Δzᶠᶠᶜ(i, j, k, grid::ZStarCoordinateGrid) = ℑxyᶠᶠᵃ(i, j, k, grid, grid.Δzᵃᵃᶜ.star_value) + +import Oceananigans.Architectures: arch_array + +arch_array(arch, coord::ZStarCoordinate) = ZStarCoordinate(arch_array(arch, coord.reference), coord.scaling, coord.star_value) \ No newline at end of file From 0a72cdde539383df5146f015fe8eab29e2ed3b47 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 11 Dec 2023 12:25:56 -0500 Subject: [PATCH 059/567] add stuff --- src/Fields/set!.jl | 2 +- .../HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Fields/set!.jl b/src/Fields/set!.jl index f6e8a8eb78..2b80b36121 100644 --- a/src/Fields/set!.jl +++ b/src/Fields/set!.jl @@ -40,7 +40,7 @@ function set!(u::Field, f::Function) # Form a FunctionField from `f` f_field = field(location(u), f, cpu_grid) - # Try to set the FuncitonField to cpu_u + # Try to set the FunctionField to cpu_u try set!(cpu_u, f_field) catch err diff --git a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl index 024cee9a29..824861b741 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl @@ -139,4 +139,5 @@ import Oceananigans.Operators: Δzᶜᶜᶠ, Δzᶜᶜᶜ, Δzᶜᶠᶠ, Δzᶜ import Oceananigans.Architectures: arch_array -arch_array(arch, coord::ZStarCoordinate) = ZStarCoordinate(arch_array(arch, coord.reference), coord.scaling, coord.star_value) \ No newline at end of file +arch_array(arch, coord::ZStarCoordinate) = + ZStarCoordinate(arch_array(arch, coord.reference), coord.scaling, coord.star_value) \ No newline at end of file From 37698738c27ba5c86f34d164fee377cbd0924a94 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 11 Dec 2023 12:56:21 -0500 Subject: [PATCH 060/567] small change --- .../split_explicit_free_surface_kernels.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index b3e2eb8eac..b1ae2f1476 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -272,7 +272,6 @@ function barotropic_split_explicit_corrector!(u, v, free_surface, grid) # !!!! reusing U and V for this storage since last timestep doesn't matter compute_barotropic_mode!(U, V, grid, u, v) # add in "good" barotropic mode - launch!(arch, grid, :xyz, _barotropic_split_explicit_corrector!, u, v, U̅, V̅, U, V, Hᶠᶜ, Hᶜᶠ, grid) From d60b64322192bf4a22b576d76cba353faaca2a61 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 11 Dec 2023 15:12:26 -0500 Subject: [PATCH 061/567] nvtx on fill halos --- src/DistributedComputations/halo_communication.jl | 5 ++++- .../split_explicit_free_surface_kernels.jl | 1 - .../update_hydrostatic_free_surface_model_state.jl | 7 +++++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/DistributedComputations/halo_communication.jl b/src/DistributedComputations/halo_communication.jl index 87865c75ab..0e27c0352f 100644 --- a/src/DistributedComputations/halo_communication.jl +++ b/src/DistributedComputations/halo_communication.jl @@ -189,7 +189,6 @@ function fill_halo_event!(c, fill_halos!, bcs, indices, loc, arch, grid::Distrib if !only_local_halos # Then we need to fill the `send` buffers fill_send_buffers!(c, buffers, grid, Val(buffer_side)) - sync_device!(arch) end # Calculate size and offset of the fill_halo kernel @@ -244,6 +243,8 @@ for (side, opposite_side) in zip([:west, :south], [:east, :north]) function $fill_both_halo!(c, bc_side::DCBCT, bc_opposite_side::DCBCT, size, offset, loc, arch::Distributed, grid::DistributedGrid, buffers, args...; only_local_halos = false, kwargs...) + sync_device!(arch) + only_local_halos && return nothing @assert bc_side.condition.from == bc_opposite_side.condition.from # Extra protection in case of bugs @@ -273,6 +274,8 @@ for side in [:west, :east, :south, :north] function $fill_side_halo!(c, bc_side::DCBCT, size, offset, loc, arch::Distributed, grid::DistributedGrid, buffers, args...; only_local_halos = false, kwargs...) + sync_device!(arch) + only_local_halos && return nothing child_arch = child_architecture(arch) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index b1ae2f1476..7b2e5507ee 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -161,7 +161,6 @@ end Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, timestepper) i, j = @index(Global, NTuple) - velocity_evolution!(i, j, grid, Δτ, η, U, V, η̅, U̅, V̅, averaging_weight, Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, diff --git a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl index 4477505945..7613b37745 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl @@ -12,6 +12,7 @@ import Oceananigans.TimeSteppers: update_state! import Oceananigans.Models.NonhydrostaticModels: compute_auxiliaries! using Oceananigans.Models: update_model_field_time_series! +using NVTX compute_auxiliary_fields!(auxiliary_fields) = Tuple(compute!(a) for a in auxiliary_fields) @@ -35,8 +36,10 @@ function update_state!(model::HydrostaticFreeSurfaceModel, grid, callbacks; comp # Update possible FieldTimeSeries used in the model @apply_regionally update_model_field_time_series!(model, model.clock) - fill_halo_regions!(prognostic_fields(model), model.clock, fields(model); async = true) - + NVTX.@range "fill_halo_regions!" begin + fill_halo_regions!(prognostic_fields(model), model.clock, fields(model); async = true) + end + @apply_regionally replace_horizontal_vector_halos!(model.velocities, model.grid) @apply_regionally compute_auxiliaries!(model) From 1b0a4404ac24eddee562c6511a64f31790ffcafa Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 11 Dec 2023 15:14:13 -0500 Subject: [PATCH 062/567] all NVTX --- .../halo_communication.jl | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/DistributedComputations/halo_communication.jl b/src/DistributedComputations/halo_communication.jl index 0e27c0352f..3a0f83d701 100644 --- a/src/DistributedComputations/halo_communication.jl +++ b/src/DistributedComputations/halo_communication.jl @@ -22,6 +22,8 @@ import Oceananigans.BoundaryConditions: fill_south_and_north_halo!, fill_bottom_and_top_halo! +using NVTX + ##### ##### MPI tags for halo communication BCs ##### @@ -104,7 +106,9 @@ function fill_halo_regions!(c::OffsetArray, bcs, indices, loc, grid::Distributed number_of_tasks = length(fill_halos!) for task = 1:number_of_tasks - fill_halo_event!(c, fill_halos![task], bcs[task], indices, loc, arch, grid, buffers, args...; kwargs...) + NVTX.@range "fill_halo_event" begin + fill_halo_event!(c, fill_halos![task], bcs[task], indices, loc, arch, grid, buffers, args...; kwargs...) + end end fill_corners!(c, arch.connectivity, indices, loc, arch, grid, buffers, args...; kwargs...) @@ -187,15 +191,21 @@ function fill_halo_event!(c, fill_halos!, bcs, indices, loc, arch, grid::Distrib buffer_side = communication_side(Val(fill_halos!)) - if !only_local_halos # Then we need to fill the `send` buffers - fill_send_buffers!(c, buffers, grid, Val(buffer_side)) + NVTX.@range "fill_send_halo" begin + if !only_local_halos # Then we need to fill the `send` buffers + fill_send_buffers!(c, buffers, grid, Val(buffer_side)) + end end # Calculate size and offset of the fill_halo kernel # We assume that the kernel size is the same for west and east boundaries, # south and north boundaries and bottom and top boundaries - size = fill_halo_size(c, fill_halos!, indices, bcs[1], loc, grid) - offset = fill_halo_offset(size, fill_halos!, indices) + NVTX.@range "fill_halo_size" begin + size = fill_halo_size(c, fill_halos!, indices, bcs[1], loc, grid) + end + NVTX.@range "fill_halo_offsets" begin + offset = fill_halo_offset(size, fill_halos!, indices) + end requests = fill_halos!(c, bcs..., size, offset, loc, arch, grid, buffers, args...; only_local_halos, kwargs...) From 6f9d400b57c0fc87f142fecbcd4e9ed1eddd23da Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 11 Dec 2023 15:15:10 -0500 Subject: [PATCH 063/567] fill it all --- src/DistributedComputations/halo_communication.jl | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/DistributedComputations/halo_communication.jl b/src/DistributedComputations/halo_communication.jl index 3a0f83d701..e80a448155 100644 --- a/src/DistributedComputations/halo_communication.jl +++ b/src/DistributedComputations/halo_communication.jl @@ -207,10 +207,14 @@ function fill_halo_event!(c, fill_halos!, bcs, indices, loc, arch, grid::Distrib offset = fill_halo_offset(size, fill_halos!, indices) end - requests = fill_halos!(c, bcs..., size, offset, loc, arch, grid, buffers, args...; only_local_halos, kwargs...) - - pool_requests_or_complete_comm!(c, arch, grid, buffers, requests, async, buffer_side) + NVTX.@range "actual fill_halos!" begin + requests = fill_halos!(c, bcs..., size, offset, loc, arch, grid, buffers, args...; only_local_halos, kwargs...) + end + NVTX.@range "pool_request" begin + pool_requests_or_complete_comm!(c, arch, grid, buffers, requests, async, buffer_side) + end + return nothing end From ea5e56bba0d89817b21ce52d846aa039f241924c Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 11 Dec 2023 15:26:55 -0500 Subject: [PATCH 064/567] check it out --- .../split_explicit_free_surface.jl | 30 ++++++++------ .../split_explicit_free_surface_kernels.jl | 39 +++++++++++-------- 2 files changed, 40 insertions(+), 29 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index ac64f01521..7ebccb14aa 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -79,7 +79,7 @@ SplitExplicitFreeSurface(FT::DataType = Float64; gravitational_acceleration = g_ function FreeSurface(free_surface::SplitExplicitFreeSurface, velocities, grid) η = FreeSurfaceDisplacementField(velocities, free_surface, grid) - return SplitExplicitFreeSurface(η, SplitExplicitState(grid), + return SplitExplicitFreeSurface(η, SplitExplicitState(grid, free_surface.settings.timestepper), SplitExplicitAuxiliaryFields(grid), free_surface.gravitational_acceleration, free_surface.settings) @@ -128,24 +128,23 @@ Note that `η̅` is solely used for setting the `η` at the next substep iterati acts as a filter for `η`. Values with superscripts `m-1` and `m-2` correspond to previous stored time steps to allow using a higher-order time stepping scheme, e.g., `AdamsBashforth3Scheme`. """ -function SplitExplicitState(grid::AbstractGrid) +function SplitExplicitState(grid::AbstractGrid, timestepper) η̅ = ZFaceField(grid, indices = (:, :, size(grid, 3)+1)) - ηᵐ = ZFaceField(grid, indices = (:, :, size(grid, 3)+1)) - ηᵐ⁻¹ = ZFaceField(grid, indices = (:, :, size(grid, 3)+1)) - ηᵐ⁻² = ZFaceField(grid, indices = (:, :, size(grid, 3)+1)) + ηᵐ = auxiliary_free_surface_field(grid, timestepper) + ηᵐ⁻¹ = auxiliary_free_surface_field(grid, timestepper) + ηᵐ⁻² = auxiliary_free_surface_field(grid, timestepper) U = ZFaceField(grid, indices = (:, :, size(grid, 3))) V = ZFaceField(grid, indices = (:, :, size(grid, 3))) - Uᵐ⁻¹ = ZFaceField(grid, indices = (:, :, size(grid, 3))) - Vᵐ⁻¹ = ZFaceField(grid, indices = (:, :, size(grid, 3))) + Uᵐ⁻¹ = auxiliary_barotropic_velocity_field(grid, timestepper) + Vᵐ⁻¹ = auxiliary_barotropic_velocity_field(grid, timestepper) + Uᵐ⁻² = auxiliary_barotropic_velocity_field(grid, timestepper) + Vᵐ⁻² = auxiliary_barotropic_velocity_field(grid, timestepper) - Uᵐ⁻² = ZFaceField(grid, indices = (:, :, size(grid, 3))) - Vᵐ⁻² = ZFaceField(grid, indices = (:, :, size(grid, 3))) - - U̅ = ZFaceField(grid, indices = (:, :, size(grid, 3))) - V̅ = ZFaceField(grid, indices = (:, :, size(grid, 3))) + U̅ = ZFaceField(grid, indices = (:, :, size(grid, 3))) + V̅ = ZFaceField(grid, indices = (:, :, size(grid, 3))) return SplitExplicitState(; ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅) end @@ -221,6 +220,13 @@ end struct AdamsBashforth3Scheme end struct ForwardBackwardScheme end + +auxiliary_free_surface_field(grid, ::AdamsBashforth3Scheme) = ZFaceField(grid, indices = (:, :, size(grid, 3)+1)) +auxiliary_free_surface_field(grid, ::ForwardBackwardScheme) = nothing + +auxiliary_barotropic_velocity_field(grid, ::AdamsBashforth3Scheme) = ZFaceField(grid, indices = (:, :, size(grid, 3))) +auxiliary_barotropic_velocity_field(grid, ::ForwardBackwardScheme) = nothing + # (p = 2, q = 4, r = 0.18927) minimize dispersion error from Shchepetkin and McWilliams (2005): https://doi.org/10.1016/j.ocemod.2004.08.002 @inline function averaging_shape_function(τ::FT; p = 2, q = 4, r = FT(0.18927)) where FT τ₀ = (p + 2) * (p + q + 2) / (p + 1) / (p + q + 1) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 7b2e5507ee..7bda6ca567 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -137,51 +137,54 @@ end using Oceananigans.DistributedComputations: Distributed using Printf -@kernel function _split_explicit_free_surface!(grid, Δτ, η, U, V, timestepper) +@kernel function _split_explicit_free_surface!(grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) i, j = @index(Global, NTuple) - free_surface_evolution!(i, j, grid, Δτ, η, U, V, timestepper) + free_surface_evolution!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) end -@inline function free_surface_evolution!(i, j, grid, Δτ, η, U, V, timestepper) +@inline function free_surface_evolution!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) k_top = grid.Nz+1 TX, TY, _ = topology(grid) @inbounds begin advance_previous_free_surface!(i, j, k_top, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) - η[i, j, k_top] -= Δτ * (div_xᶜᶜᶠ_U(i, j, k_top-1, grid, TX, U★, timestepper, U, 0, 0) + - div_yᶜᶜᶠ_V(i, j, k_top-1, grid, TY, U★, timestepper, V, 0, 0)) + η[i, j, k_top] -= Δτ * (div_xᶜᶜᶠ_U(i, j, k_top-1, grid, TX, U★, timestepper, U, Uᵐ⁻¹, Uᵐ⁻²) + + div_yᶜᶜᶠ_V(i, j, k_top-1, grid, TY, U★, timestepper, V, Vᵐ⁻¹, Vᵐ⁻²)) end return nothing end -@kernel function _split_explicit_barotropic_velocity!(grid, Δτ, η, U, V, +@kernel function _split_explicit_barotropic_velocity!(grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², + U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅, averaging_weight, Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, timestepper) i, j = @index(Global, NTuple) - velocity_evolution!(i, j, grid, Δτ, η, U, V, + velocity_evolution!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², + U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅, averaging_weight, Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, timestepper) end -@inline function velocity_evolution!(i, j, grid, Δτ, η, U, V, +@inline function velocity_evolution!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², + U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅, averaging_weight, Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, - timestepper ) + timestepper) k_top = grid.Nz+1 TX, TY, _ = topology(grid) @inbounds begin - # advance_previous_velocity!(i, j, k_top-1, timestepper, U, Uᵐ⁻¹, Uᵐ⁻²) - # advance_previous_velocity!(i, j, k_top-1, timestepper, V, Vᵐ⁻¹, Vᵐ⁻²) + advance_previous_velocity!(i, j, k_top-1, timestepper, U, Uᵐ⁻¹, Uᵐ⁻²) + advance_previous_velocity!(i, j, k_top-1, timestepper, V, Vᵐ⁻¹, Vᵐ⁻²) # ∂τ(U) = - ∇η + G - U[i, j, k_top-1] += Δτ * (- g * Hᶠᶜ[i, j, k_top-1] * ∂xᶠᶜᶠ_η(i, j, k_top, grid, TX, η★, timestepper, η, 0, 0, 0) + Gᵁ[i, j, k_top-1]) - V[i, j, k_top-1] += Δτ * (- g * Hᶜᶠ[i, j, k_top-1] * ∂yᶜᶠᶠ_η(i, j, k_top, grid, TY, η★, timestepper, η, 0, 0, 0) + Gⱽ[i, j, k_top-1]) + U[i, j, k_top-1] += Δτ * (- g * Hᶠᶜ[i, j, k_top-1] * ∂xᶠᶜᶠ_η(i, j, k_top, grid, TX, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gᵁ[i, j, k_top-1]) + V[i, j, k_top-1] += Δτ * (- g * Hᶜᶠ[i, j, k_top-1] * ∂yᶜᶠᶠ_η(i, j, k_top, grid, TY, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gⱽ[i, j, k_top-1]) # time-averaging η̅[i, j, k_top] += averaging_weight * η[i, j, k_top] @@ -370,9 +373,10 @@ function iterate_split_explicit!(free_surface::FixedSubstepsSplitExplicit{N}, gr averaging_weight = weights[substep] - free_surface_kernel!(grid, Δτᴮ, η, U, V, timestepper) + free_surface_kernel!(grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) - barotropic_velocity_kernel!(grid, Δτᴮ, η, U, V, + barotropic_velocity_kernel!(grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², + U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅, averaging_weight, Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, timestepper) @@ -417,9 +421,10 @@ function iterate_split_explicit!(free_surface, grid, Δt) averaging_weight = weights[substep] - free_surface_kernel!(grid, Δτᴮ, η, U, V, timestepper) + free_surface_kernel!(grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) - barotropic_velocity_kernel!(grid, Δτᴮ, η, U, V, + barotropic_velocity_kernel!(grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², + U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅, averaging_weight, Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, timestepper) From 47dd5698ed12f6735600bd497c2194472f42fc6a Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 11 Dec 2023 15:56:51 -0500 Subject: [PATCH 065/567] bugfixxed --- .../distributed_split_explicit_free_surface.jl | 2 +- .../HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl | 2 +- src/MultiRegion/multi_region_split_explicit_free_surface.jl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl index 679889a888..6f5067e555 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl @@ -75,7 +75,7 @@ function FreeSurface(free_surface::SplitExplicitFreeSurface, velocities, grid::D η = ZFaceField(new_grid, indices = (:, :, size(new_grid, 3)+1)) return SplitExplicitFreeSurface(η, - SplitExplicitState(new_grid), + SplitExplicitState(new_grid, settings.timestepper), SplitExplicitAuxiliaryFields(new_grid), free_surface.gravitational_acceleration, free_surface.settings) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index 7ebccb14aa..ca549d94b3 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -120,7 +120,7 @@ Base.@kwdef struct SplitExplicitState{𝒞𝒞, ℱ𝒞, 𝒞ℱ} end """ - SplitExplicitState(grid) + SplitExplicitState(grid, timestepper) Return the split-explicit state for `grid`. diff --git a/src/MultiRegion/multi_region_split_explicit_free_surface.jl b/src/MultiRegion/multi_region_split_explicit_free_surface.jl index 7349324fec..c645452038 100644 --- a/src/MultiRegion/multi_region_split_explicit_free_surface.jl +++ b/src/MultiRegion/multi_region_split_explicit_free_surface.jl @@ -55,7 +55,7 @@ function FreeSurface(free_surface::SplitExplicitFreeSurface, velocities, grid::M η = ZFaceField(new_grid, indices = (:, :, size(new_grid, 3)+1)) return SplitExplicitFreeSurface(η, - SplitExplicitState(new_grid), + SplitExplicitState(new_grid, free_surface.settings.timestepper), SplitExplicitAuxiliaryFields(new_grid), free_surface.gravitational_acceleration, free_surface.settings) From 5842685b3e40ded9fb88731129746b0d19d8e112 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 11 Dec 2023 17:01:41 -0500 Subject: [PATCH 066/567] bugfix --- .../moving_vertical_coordinate.jl | 2 +- .../immersed_boundaries/z_star_coordinate.jl | 52 +++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 validation/immersed_boundaries/z_star_coordinate.jl diff --git a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl index 824861b741..bb55aeeb45 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl @@ -32,7 +32,7 @@ MovingCoordinateGrid(grid, coord) = grid function MovingCoordinateGrid(grid::ImmersedBoundaryGrid, ::ZStarCoordinate) underlying_grid = MovingCoordinateGrid(grid.underlying_grid, ZStarCoordinate()) - active_cells_map = !isnothing(grid.active_cells_map) + active_cells_map = !isnothing(grid.interior_active_cells) return ImmersedBoundaryGrid(underlying_grid, grid.immersed_boundary; active_cells_map) end diff --git a/validation/immersed_boundaries/z_star_coordinate.jl b/validation/immersed_boundaries/z_star_coordinate.jl new file mode 100644 index 0000000000..ae83a9cd2c --- /dev/null +++ b/validation/immersed_boundaries/z_star_coordinate.jl @@ -0,0 +1,52 @@ +using Oceananigans +using Oceananigans.Units +using Printf +using Oceananigans.Models.HydrostaticFreeSurfaceModels: ZStarCoordinate + +grid = RectilinearGrid(size = (600, 20), x = (0, 18kilometers), z = (-200, 0), topology = (Periodic, Flat, Bounded), halo = (5, 5)) + +bottom(x) = - 200 + exp( - (x - 9kilometers)^2 / (1.2kilometers)^2) * 150 + +grid = ImmersedBoundaryGrid(grid, GridFittedBottom(bottom)) + +@inline u_bc(i, j, grid, clock, fields, p) = + @inbounds - 1 / p.λ * (0.1 - fields.u[i, j, grid.Nz]) + + +u_boundary = FluxBoundaryCondition(u_bc, discrete_form = true, parameters = (; λ = 10hour)) + +u_bcs = FieldBoundaryConditions(top = u_boundary) + +model = HydrostaticFreeSurfaceModel(; grid, + free_surface = SplitExplicitFreeSurface(; cfl = 0.7, grid), + momentum_advection = WENO(), + tracer_advection = WENO(; order = 7), + tracers = :b, + boundary_conditions = (; u = u_bcs), + closure = VerticalScalarDiffusivity(ν = 1e-3), + buoyancy = BuoyancyTracer(), + vertical_coordinate = ZStarCoordinate()) + +N² = 0.01 +bᵢ(x, z) = N² * z + +set!(model, b = bᵢ) + +simulation = Simulation(model, Δt = 1, stop_time = 2days, stop_iteration = 10000) + +function progress(sim) + u, v, w = sim.model.velocities + @info @sprintf("Time: %s, Δt: %.2f, iteration: %d, velocities: (%.2f, %.2f)", prettytime(sim.model.clock.time), + sim.Δt, sim.model.clock.iteration, maximum(abs, u), maximum(abs, w)) +end + +wizard = TimeStepWizard(; cfl = 0.2, max_change = 1.1) +simulation.callbacks[:wizard] = Callback(wizard, IterationInterval(2)) +simulation.callbacks[:progress] = Callback(progress, IterationInterval(1)) + +simulation.output_writers[:fields] = JLD2OutputWriter(model, merge(model.velocities, model.tracers), + filename = "z_star_coordinate", + overwrite_existing = true, + schedule = TimeInterval(0.5hour)) + +run!(simulation) \ No newline at end of file From 62dad92b1653ac250ab81cb3111d1dd543ed6e1d Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 11 Dec 2023 17:03:40 -0500 Subject: [PATCH 067/567] bugfixed --- .../split_explicit_free_surface.jl | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index ca549d94b3..d8173550da 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -92,31 +92,31 @@ A type containing the state fields for the split-explicit free surface. $(FIELDS) """ -Base.@kwdef struct SplitExplicitState{𝒞𝒞, ℱ𝒞, 𝒞ℱ} +Base.@kwdef struct SplitExplicitState{CC, ACC, FC, AFC, CF, ACF} "The free surface at time `m`. (`ReducedField` over ``z``)" - ηᵐ :: 𝒞𝒞 + ηᵐ :: ACC "The free surface at time `m-1`. (`ReducedField` over ``z``)" - ηᵐ⁻¹ :: 𝒞𝒞 + ηᵐ⁻¹ :: ACC "The free surface at time `m-2`. (`ReducedField` over ``z``)" - ηᵐ⁻² :: 𝒞𝒞 + ηᵐ⁻² :: ACC "The barotropic zonal velocity at time `m`. (`ReducedField` over ``z``)" - U :: ℱ𝒞 + U :: FC "The barotropic zonal velocity at time `m-1`. (`ReducedField` over ``z``)" - Uᵐ⁻¹ :: ℱ𝒞 + Uᵐ⁻¹ :: AFC "The barotropic zonal velocity at time `m-2`. (`ReducedField` over ``z``)" - Uᵐ⁻² :: ℱ𝒞 + Uᵐ⁻² :: AFC "The barotropic meridional velocity at time `m`. (`ReducedField` over ``z``)" - V :: 𝒞ℱ + V :: CF "The barotropic meridional velocity at time `m-1`. (`ReducedField` over ``z``)" - Vᵐ⁻¹ :: 𝒞ℱ + Vᵐ⁻¹ :: ACF "The barotropic meridional velocity at time `m-2`. (`ReducedField` over ``z``)" - Vᵐ⁻² :: 𝒞ℱ + Vᵐ⁻² :: ACF "The time-filtered free surface. (`ReducedField` over ``z``)" - η̅ :: 𝒞𝒞 + η̅ :: CC "The time-filtered barotropic zonal velocity. (`ReducedField` over ``z``)" - U̅ :: ℱ𝒞 + U̅ :: FC "The time-filtered barotropic meridional velocity. (`ReducedField` over ``z``)" - V̅ :: 𝒞ℱ + V̅ :: FC end """ From 9d5ada270477453ce4e71959b0cd8521156ed6c7 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 11 Dec 2023 20:42:34 -0500 Subject: [PATCH 068/567] bugfix --- .../split_explicit_free_surface_kernels.jl | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 7bda6ca567..154756ad5b 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -230,12 +230,23 @@ compute_barotropic_mode!(U, V, grid, u, v) = launch!(architecture(grid), grid, :xy, _barotropic_mode_kernel!, U, V, grid, u, v; only_active_cells = use_only_active_surface_cells(grid)) -function initialize_free_surface_state!(free_surface_state, η) - state = free_surface_state +function initialize_free_surface_state!(state, η, timestepper) parent(state.U) .= parent(state.U̅) parent(state.V) .= parent(state.V̅) + initialize_auxiliary_state!(state, η, timestepper) + + fill!(state.η̅, 0) + fill!(state.U̅, 0) + fill!(state.V̅, 0) + + return nothing +end + +initialize_auxiliary_state!(state, η, ::ForwardBackwardScheme) = nothing + +function initialize_auxiliary_state!(state, η, timestepper) parent(state.Uᵐ⁻¹) .= parent(state.U̅) parent(state.Vᵐ⁻¹) .= parent(state.V̅) @@ -246,10 +257,6 @@ function initialize_free_surface_state!(free_surface_state, η) parent(state.ηᵐ⁻¹) .= parent(η) parent(state.ηᵐ⁻²) .= parent(η) - fill!(state.η̅, 0) - fill!(state.U̅, 0) - fill!(state.V̅, 0) - return nothing end @@ -302,7 +309,7 @@ function split_explicit_free_surface_step!(free_surface::SplitExplicitFreeSurfac # reset free surface averages @apply_regionally begin - initialize_free_surface_state!(free_surface.state, free_surface.η) + initialize_free_surface_state!(free_surface.state, free_surface.η, free_surface.settings.timestepper) # Solve for the free surface at tⁿ⁺¹ iterate_split_explicit!(free_surface, free_surface_grid, Δt) # Reset eta for the next timestep @@ -373,9 +380,9 @@ function iterate_split_explicit!(free_surface::FixedSubstepsSplitExplicit{N}, gr averaging_weight = weights[substep] - free_surface_kernel!(grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) + free_surface_kernel!(grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) - barotropic_velocity_kernel!(grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², + barotropic_velocity_kernel!(grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅, averaging_weight, Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, @@ -421,9 +428,9 @@ function iterate_split_explicit!(free_surface, grid, Δt) averaging_weight = weights[substep] - free_surface_kernel!(grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) + free_surface_kernel!(grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) - barotropic_velocity_kernel!(grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², + barotropic_velocity_kernel!(grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅, averaging_weight, Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, From 76bfb5eb30eaf13dbb5c00e1fe4fc74ffe1d8178 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 12 Dec 2023 10:39:55 -0500 Subject: [PATCH 069/567] annotate the convert --- src/Architectures.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Architectures.jl b/src/Architectures.jl index 6d3da62098..95c9128b4b 100644 --- a/src/Architectures.jl +++ b/src/Architectures.jl @@ -8,6 +8,11 @@ using CUDA using KernelAbstractions using Adapt using OffsetArrays +using NVTX + +NVTX.@annotate "cudaconvert function" function KernelAbstractions.argconvert(k::KernelAbstractions.Kernel{CUDABackend}, arg) + CUDA.cudaconvert(arg) +end """ AbstractArchitecture From 3f645ce5e68f44f5fe911a3208b97891bf04cfb0 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 12 Dec 2023 10:40:26 -0500 Subject: [PATCH 070/567] bugfix --- src/Architectures.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Architectures.jl b/src/Architectures.jl index 95c9128b4b..49cb7fd56d 100644 --- a/src/Architectures.jl +++ b/src/Architectures.jl @@ -10,7 +10,7 @@ using Adapt using OffsetArrays using NVTX -NVTX.@annotate "cudaconvert function" function KernelAbstractions.argconvert(k::KernelAbstractions.Kernel{CUDABackend}, arg) +NVTX.@annotate "cudaconvert function" function KernelAbstractions.argconvert(k::KernelAbstractions.Kernel{CUDAl.CUDABackend}, arg) CUDA.cudaconvert(arg) end From 324aaefe11636d08e960c33c62c2bf4711e8e285 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 12 Dec 2023 10:41:40 -0500 Subject: [PATCH 071/567] bugfix --- src/Architectures.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Architectures.jl b/src/Architectures.jl index 49cb7fd56d..36e340d648 100644 --- a/src/Architectures.jl +++ b/src/Architectures.jl @@ -10,7 +10,7 @@ using Adapt using OffsetArrays using NVTX -NVTX.@annotate "cudaconvert function" function KernelAbstractions.argconvert(k::KernelAbstractions.Kernel{CUDAl.CUDABackend}, arg) +NVTX.@annotate "cudaconvert function" function KernelAbstractions.argconvert(k::KernelAbstractions.Kernel{CUDA.CUDABackend}, arg) CUDA.cudaconvert(arg) end From 67df158aff546983749dac1f781daf7c3bec1052 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 12 Dec 2023 11:25:49 -0500 Subject: [PATCH 072/567] add cudaconvert --- src/Architectures.jl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Architectures.jl b/src/Architectures.jl index 36e340d648..a5803cc163 100644 --- a/src/Architectures.jl +++ b/src/Architectures.jl @@ -14,6 +14,14 @@ NVTX.@annotate "cudaconvert function" function KernelAbstractions.argconvert(k:: CUDA.cudaconvert(arg) end +import CUDA: cudaconvert + +NVTX.@annotate "cudaconvert function" function CUDA.cudaconvert(arg) + NVTX.@range "inside convert function" begin + Adapt.adapt(CUDA.KernelAdaptor(), arg) + end +end + """ AbstractArchitecture From 74d3badaf1ade385da3fa267088784f5af625a48 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 12 Dec 2023 15:10:19 -0500 Subject: [PATCH 073/567] remove NVTX --- src/Architectures.jl | 12 --------- .../halo_communication.jl | 26 +++++-------------- ...te_hydrostatic_free_surface_model_state.jl | 4 +-- 3 files changed, 8 insertions(+), 34 deletions(-) diff --git a/src/Architectures.jl b/src/Architectures.jl index a5803cc163..54d7a95fc1 100644 --- a/src/Architectures.jl +++ b/src/Architectures.jl @@ -10,18 +10,6 @@ using Adapt using OffsetArrays using NVTX -NVTX.@annotate "cudaconvert function" function KernelAbstractions.argconvert(k::KernelAbstractions.Kernel{CUDA.CUDABackend}, arg) - CUDA.cudaconvert(arg) -end - -import CUDA: cudaconvert - -NVTX.@annotate "cudaconvert function" function CUDA.cudaconvert(arg) - NVTX.@range "inside convert function" begin - Adapt.adapt(CUDA.KernelAdaptor(), arg) - end -end - """ AbstractArchitecture diff --git a/src/DistributedComputations/halo_communication.jl b/src/DistributedComputations/halo_communication.jl index e80a448155..5d011acafe 100644 --- a/src/DistributedComputations/halo_communication.jl +++ b/src/DistributedComputations/halo_communication.jl @@ -106,9 +106,7 @@ function fill_halo_regions!(c::OffsetArray, bcs, indices, loc, grid::Distributed number_of_tasks = length(fill_halos!) for task = 1:number_of_tasks - NVTX.@range "fill_halo_event" begin - fill_halo_event!(c, fill_halos![task], bcs[task], indices, loc, arch, grid, buffers, args...; kwargs...) - end + fill_halo_event!(c, fill_halos![task], bcs[task], indices, loc, arch, grid, buffers, args...; kwargs...) end fill_corners!(c, arch.connectivity, indices, loc, arch, grid, buffers, args...; kwargs...) @@ -191,29 +189,19 @@ function fill_halo_event!(c, fill_halos!, bcs, indices, loc, arch, grid::Distrib buffer_side = communication_side(Val(fill_halos!)) - NVTX.@range "fill_send_halo" begin - if !only_local_halos # Then we need to fill the `send` buffers - fill_send_buffers!(c, buffers, grid, Val(buffer_side)) - end + if !only_local_halos # Then we need to fill the `send` buffers + fill_send_buffers!(c, buffers, grid, Val(buffer_side)) end # Calculate size and offset of the fill_halo kernel # We assume that the kernel size is the same for west and east boundaries, # south and north boundaries and bottom and top boundaries - NVTX.@range "fill_halo_size" begin - size = fill_halo_size(c, fill_halos!, indices, bcs[1], loc, grid) - end - NVTX.@range "fill_halo_offsets" begin - offset = fill_halo_offset(size, fill_halos!, indices) - end + size = fill_halo_size(c, fill_halos!, indices, bcs[1], loc, grid) + offset = fill_halo_offset(size, fill_halos!, indices) - NVTX.@range "actual fill_halos!" begin - requests = fill_halos!(c, bcs..., size, offset, loc, arch, grid, buffers, args...; only_local_halos, kwargs...) - end + requests = fill_halos!(c, bcs..., size, offset, loc, arch, grid, buffers, args...; only_local_halos, kwargs...) - NVTX.@range "pool_request" begin - pool_requests_or_complete_comm!(c, arch, grid, buffers, requests, async, buffer_side) - end + pool_requests_or_complete_comm!(c, arch, grid, buffers, requests, async, buffer_side) return nothing end diff --git a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl index 7613b37745..43f53a766f 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl @@ -36,9 +36,7 @@ function update_state!(model::HydrostaticFreeSurfaceModel, grid, callbacks; comp # Update possible FieldTimeSeries used in the model @apply_regionally update_model_field_time_series!(model, model.clock) - NVTX.@range "fill_halo_regions!" begin - fill_halo_regions!(prognostic_fields(model), model.clock, fields(model); async = true) - end + fill_halo_regions!(prognostic_fields(model), model.clock, fields(model); async = true) @apply_regionally replace_horizontal_vector_halos!(model.velocities, model.grid) @apply_regionally compute_auxiliaries!(model) From 955d2c16645c89a664f58c3884e52236d3291b0e Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 12 Dec 2023 15:49:51 -0500 Subject: [PATCH 074/567] model grid --- .../split_explicit_free_surface_kernels.jl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 154756ad5b..eb2b8bbbe0 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -477,8 +477,6 @@ end # Setting up the RHS for the barotropic step (tendencies of the barotopic velocity components) # This function is called after `calculate_tendency` and before `ab2_step_velocities!` function setup_free_surface!(model, free_surface::SplitExplicitFreeSurface, χ) - - free_surface_grid = free_surface.η.grid # we start the time integration of η from the average ηⁿ Gu⁻ = model.timestepper.G⁻.u @@ -488,7 +486,7 @@ function setup_free_surface!(model, free_surface::SplitExplicitFreeSurface, χ) auxiliary = free_surface.auxiliary - @apply_regionally setup_split_explicit_tendency!(auxiliary, free_surface_grid, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) + @apply_regionally setup_split_explicit_tendency!(auxiliary, model.grid, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) fields_to_fill = (auxiliary.Gᵁ, auxiliary.Gⱽ) fill_halo_regions!(fields_to_fill; async = true) From 15f60f73a2c79488faa3fea99607a3292c909f6a Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 12 Dec 2023 22:35:51 -0500 Subject: [PATCH 075/567] try like this? --- .../split_explicit_free_surface_kernels.jl | 96 +++++++------------ src/Utils/kernel_launching.jl | 34 +++++++ 2 files changed, 71 insertions(+), 59 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index eb2b8bbbe0..80a46304e0 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -193,6 +193,34 @@ end end end +@kernel function _iterate_split_explicit!(grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², + U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², + η̅, U̅, V̅, weights, + Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, + timestepper, + free_surface_kernel!, + barotropic_velocity_kernel!, + ::Val{N}) where N + + @unroll for substep in 1:N + Base.@_inline_meta + + averaging_weight = weights[substep] + + free_surface_kernel!(grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², + U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², + timestepper; + dynamic_launch=true) + + barotropic_velocity_kernel!(grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², + U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², + η̅, U̅, V̅, averaging_weight, + Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, + timestepper; + dynamic_launch=true) + end +end + # Barotropic Model Kernels # u_Δz = u * Δz @kernel function _barotropic_mode_kernel!(U, V, grid, u, v) @@ -346,52 +374,6 @@ const MINIMUM_SUBSTEPS = 5 const FixedSubstepsSetting{N} = SplitExplicitSettings{<:FixedSubstepNumber{<:Any, <:NTuple{N, <:Any}}} where N const FixedSubstepsSplitExplicit{F} = SplitExplicitFreeSurface{<:Any, <:Any, <:Any, <:Any, <:FixedSubstepsSetting{N}} where N -# For a fixed number of substeps it is possible to -function iterate_split_explicit!(free_surface::FixedSubstepsSplitExplicit{N}, grid, Δt) where N - arch = architecture(grid) - - η = free_surface.η - state = free_surface.state - auxiliary = free_surface.auxiliary - settings = free_surface.settings - g = free_surface.gravitational_acceleration - - weights = settings.substepping.averaging_weights - fractional_Δt = settings.substepping.fractional_step_size - - Δτᴮ = fractional_Δt * Δt # barotropic time step in seconds - - # unpack state quantities, parameters and forcing terms - U, V = state.U, state.V - Uᵐ⁻¹, Uᵐ⁻² = state.Uᵐ⁻¹, state.Uᵐ⁻² - Vᵐ⁻¹, Vᵐ⁻² = state.Vᵐ⁻¹, state.Vᵐ⁻² - ηᵐ, ηᵐ⁻¹, ηᵐ⁻² = state.ηᵐ, state.ηᵐ⁻¹, state.ηᵐ⁻² - η̅, U̅, V̅ = state.η̅, state.U̅, state.V̅ - Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ = auxiliary.Gᵁ, auxiliary.Gⱽ, auxiliary.Hᶠᶜ, auxiliary.Hᶜᶠ - - timestepper = settings.timestepper - parameters = auxiliary.kernel_parameters - - free_surface_kernel! = configured_kernel(arch, grid, parameters, _split_explicit_free_surface!) - barotropic_velocity_kernel! = configured_kernel(arch, grid, parameters, _split_explicit_barotropic_velocity!) - - @unroll for substep in 1:N - Base.@_inline_meta - - averaging_weight = weights[substep] - - free_surface_kernel!(grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) - - barotropic_velocity_kernel!(grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², - U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, averaging_weight, - Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, - timestepper) - end - - return nothing -end - function iterate_split_explicit!(free_surface, grid, Δt) arch = architecture(grid) @@ -423,19 +405,15 @@ function iterate_split_explicit!(free_surface, grid, Δt) free_surface_kernel! = configured_kernel(arch, grid, parameters, _split_explicit_free_surface!) barotropic_velocity_kernel! = configured_kernel(arch, grid, parameters, _split_explicit_barotropic_velocity!) - @unroll for substep in 1:N - Base.@_inline_meta - - averaging_weight = weights[substep] - - free_surface_kernel!(grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) - - barotropic_velocity_kernel!(grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², - U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, averaging_weight, - Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, - timestepper) - end + launch!(arch, grid, 1, _iterate_split_explicit!, + grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², + U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², + η̅, U̅, V̅, weights, + Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, + timestepper, + free_surface_kernel!, + barotropic_velocity_kernel!, + Val(N)) return nothing end diff --git a/src/Utils/kernel_launching.jl b/src/Utils/kernel_launching.jl index 543f977388..6d98f5d131 100644 --- a/src/Utils/kernel_launching.jl +++ b/src/Utils/kernel_launching.jl @@ -279,3 +279,37 @@ function partition(kernel::OffsetKernel, inrange, ingroupsize) return iterspace, dynamic end +##### +##### Add Dynamic kernels to KA +##### + +using KernelAbstractions +using CUDA: CUDABackend + +const KA = KernelAbstractions + +function (obj::KA.Kernel{CUDABackend})(args...; ndrange=nothing, workgroupsize=nothing, dynamic_launch=false) + backend = KA.backend(obj) + + ndrange, workgroupsize, iterspace, dynamic = KA.launch_config(obj, ndrange, workgroupsize) + # this might not be the final context, since we may tune the workgroupsize + ctx = KA.mkcontext(obj, ndrange, iterspace) + + maxthreads = prod(KA.get(KA.workgroupsize(obj))) + + kernel = @cuda launch=false always_inline=backend.always_inline maxthreads=maxthreads dynamic=dynamic_launch obj.f(ctx, args...) + + blocks = length(KA.blocks(iterspace)) + threads = length(KA.workitems(iterspace)) + + if blocks == 0 + return nothing + end + + # Launch kernel + kernel(ctx, args...; threads, blocks) + + return nothing +end + + From 3c8e34fe7c4f9fdd7cfff32b43a9c68c9b31afdc Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 12 Dec 2023 22:46:20 -0500 Subject: [PATCH 076/567] bugfix --- src/Utils/kernel_launching.jl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Utils/kernel_launching.jl b/src/Utils/kernel_launching.jl index 6d98f5d131..162e201e29 100644 --- a/src/Utils/kernel_launching.jl +++ b/src/Utils/kernel_launching.jl @@ -284,10 +284,13 @@ end ##### using KernelAbstractions -using CUDA: CUDABackend +using CUDA: CUDABackend, @cuda const KA = KernelAbstractions +(obj::KA.Kernel)(args...; ndrange=nothing, workgroupsize=nothing, dynamic_launch=false) = + obj(args...; ndrange, workgroupsize) + function (obj::KA.Kernel{CUDABackend})(args...; ndrange=nothing, workgroupsize=nothing, dynamic_launch=false) backend = KA.backend(obj) From 246c6d97d47eefa8cb69e466c238ed95127ee35e Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 12 Dec 2023 22:54:27 -0500 Subject: [PATCH 077/567] fix --- src/Utils/kernel_launching.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Utils/kernel_launching.jl b/src/Utils/kernel_launching.jl index 162e201e29..d925165509 100644 --- a/src/Utils/kernel_launching.jl +++ b/src/Utils/kernel_launching.jl @@ -300,7 +300,11 @@ function (obj::KA.Kernel{CUDABackend})(args...; ndrange=nothing, workgroupsize=n maxthreads = prod(KA.get(KA.workgroupsize(obj))) - kernel = @cuda launch=false always_inline=backend.always_inline maxthreads=maxthreads dynamic=dynamic_launch obj.f(ctx, args...) + kernel = if dynamic_launch + @cuda launch=false always_inline=backend.always_inline maxthreads=maxthreads dynamic=true obj.f(ctx, args...) + else + @cuda launch=false always_inline=backend.always_inline maxthreads=maxthreads obj.f(ctx, args...) + end blocks = length(KA.blocks(iterspace)) threads = length(KA.workitems(iterspace)) From 837a119c0ea6fa4ad1957fd7c2fe710e05577372 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 12 Dec 2023 22:57:16 -0500 Subject: [PATCH 078/567] should work? --- src/Utils/kernel_launching.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Utils/kernel_launching.jl b/src/Utils/kernel_launching.jl index d925165509..2102cca4b0 100644 --- a/src/Utils/kernel_launching.jl +++ b/src/Utils/kernel_launching.jl @@ -301,7 +301,7 @@ function (obj::KA.Kernel{CUDABackend})(args...; ndrange=nothing, workgroupsize=n maxthreads = prod(KA.get(KA.workgroupsize(obj))) kernel = if dynamic_launch - @cuda launch=false always_inline=backend.always_inline maxthreads=maxthreads dynamic=true obj.f(ctx, args...) + @cuda launch=false dynamic=true obj.f(ctx, args...) else @cuda launch=false always_inline=backend.always_inline maxthreads=maxthreads obj.f(ctx, args...) end From 148a2c87f16e35fff6c28b47accb07091be0dd27 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 12 Dec 2023 22:58:12 -0500 Subject: [PATCH 079/567] add here --- src/Utils/kernel_launching.jl | 42 ----------------------------------- 1 file changed, 42 deletions(-) diff --git a/src/Utils/kernel_launching.jl b/src/Utils/kernel_launching.jl index 2102cca4b0..f43be9a3d2 100644 --- a/src/Utils/kernel_launching.jl +++ b/src/Utils/kernel_launching.jl @@ -278,45 +278,3 @@ function partition(kernel::OffsetKernel, inrange, ingroupsize) return iterspace, dynamic end - -##### -##### Add Dynamic kernels to KA -##### - -using KernelAbstractions -using CUDA: CUDABackend, @cuda - -const KA = KernelAbstractions - -(obj::KA.Kernel)(args...; ndrange=nothing, workgroupsize=nothing, dynamic_launch=false) = - obj(args...; ndrange, workgroupsize) - -function (obj::KA.Kernel{CUDABackend})(args...; ndrange=nothing, workgroupsize=nothing, dynamic_launch=false) - backend = KA.backend(obj) - - ndrange, workgroupsize, iterspace, dynamic = KA.launch_config(obj, ndrange, workgroupsize) - # this might not be the final context, since we may tune the workgroupsize - ctx = KA.mkcontext(obj, ndrange, iterspace) - - maxthreads = prod(KA.get(KA.workgroupsize(obj))) - - kernel = if dynamic_launch - @cuda launch=false dynamic=true obj.f(ctx, args...) - else - @cuda launch=false always_inline=backend.always_inline maxthreads=maxthreads obj.f(ctx, args...) - end - - blocks = length(KA.blocks(iterspace)) - threads = length(KA.workitems(iterspace)) - - if blocks == 0 - return nothing - end - - # Launch kernel - kernel(ctx, args...; threads, blocks) - - return nothing -end - - From 0cf5c772b0303d4db694f2618bb83f6cc8f49193 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 12 Dec 2023 22:58:19 -0500 Subject: [PATCH 080/567] add here --- src/Utils/Utils.jl | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/Utils/Utils.jl b/src/Utils/Utils.jl index 761c8d23a6..1704375cd5 100644 --- a/src/Utils/Utils.jl +++ b/src/Utils/Utils.jl @@ -42,4 +42,46 @@ include("multi_region_transformation.jl") include("coordinate_transformations.jl") include("sum_of_arrays.jl") + +##### +##### Add Dynamic kernels to KA +##### + +using KernelAbstractions +using CUDA: CUDABackend, @cuda + +const KA = KernelAbstractions + +(obj::KA.Kernel)(args...; ndrange=nothing, workgroupsize=nothing, dynamic_launch=false) = + obj(args...; ndrange, workgroupsize) + +function (obj::KA.Kernel{CUDABackend})(args...; ndrange=nothing, workgroupsize=nothing, dynamic_launch=false) + backend = KA.backend(obj) + + ndrange, workgroupsize, iterspace, dynamic = KA.launch_config(obj, ndrange, workgroupsize) + # this might not be the final context, since we may tune the workgroupsize + ctx = KA.mkcontext(obj, ndrange, iterspace) + + maxthreads = prod(KA.get(KA.workgroupsize(obj))) + + kernel = if dynamic_launch + @cuda launch=false dynamic=true obj.f(ctx, args...) + else + @cuda launch=false always_inline=backend.always_inline maxthreads=maxthreads obj.f(ctx, args...) + end + + blocks = length(KA.blocks(iterspace)) + threads = length(KA.workitems(iterspace)) + + if blocks == 0 + return nothing + end + + # Launch kernel + kernel(ctx, args...; threads, blocks) + + return nothing +end + + end # module From d1f4f8380347ef8d3b1bd5a46850e12c35db4f85 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 12 Dec 2023 23:01:37 -0500 Subject: [PATCH 081/567] bugfix --- .../split_explicit_free_surface_kernels.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 80a46304e0..a7345120f5 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -413,7 +413,7 @@ function iterate_split_explicit!(free_surface, grid, Δt) timestepper, free_surface_kernel!, barotropic_velocity_kernel!, - Val(N)) + Val(Nsubsteps)) return nothing end From 41a085785f4684f9b11b105a08ca0115e50c8155 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Wed, 13 Dec 2023 11:24:41 -0500 Subject: [PATCH 082/567] back to how it was --- src/BoundaryConditions/fill_halo_regions.jl | 2 +- .../split_explicit_free_surface_kernels.jl | 52 ++++++------------- src/Utils/Utils.jl | 42 --------------- 3 files changed, 16 insertions(+), 80 deletions(-) diff --git a/src/BoundaryConditions/fill_halo_regions.jl b/src/BoundaryConditions/fill_halo_regions.jl index 9097cc0254..67c0c66117 100644 --- a/src/BoundaryConditions/fill_halo_regions.jl +++ b/src/BoundaryConditions/fill_halo_regions.jl @@ -50,7 +50,7 @@ function fill_halo_regions!(c::MaybeTupledData, boundary_conditions, indices, lo arch = architecture(grid) - fill_halos!, bcs = permute_boundary_conditions(boundary_conditions) + fill_halos!, bcs = permute_boundary_conditions(boundary_conditions) number_of_tasks = length(fill_halos!) # Fill halo in the three permuted directions (1, 2, and 3), making sure dependencies are fulfilled diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index a7345120f5..7237052967 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -193,34 +193,6 @@ end end end -@kernel function _iterate_split_explicit!(grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², - U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, weights, - Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, - timestepper, - free_surface_kernel!, - barotropic_velocity_kernel!, - ::Val{N}) where N - - @unroll for substep in 1:N - Base.@_inline_meta - - averaging_weight = weights[substep] - - free_surface_kernel!(grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², - U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², - timestepper; - dynamic_launch=true) - - barotropic_velocity_kernel!(grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², - U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, averaging_weight, - Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, - timestepper; - dynamic_launch=true) - end -end - # Barotropic Model Kernels # u_Δz = u * Δz @kernel function _barotropic_mode_kernel!(U, V, grid, u, v) @@ -405,15 +377,21 @@ function iterate_split_explicit!(free_surface, grid, Δt) free_surface_kernel! = configured_kernel(arch, grid, parameters, _split_explicit_free_surface!) barotropic_velocity_kernel! = configured_kernel(arch, grid, parameters, _split_explicit_barotropic_velocity!) - launch!(arch, grid, 1, _iterate_split_explicit!, - grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², - U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, weights, - Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, - timestepper, - free_surface_kernel!, - barotropic_velocity_kernel!, - Val(Nsubsteps)) + @unroll for substep in 1:N + Base.@_inline_meta + + averaging_weight = weights[substep] + + free_surface_kernel!(grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², + U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², + timestepper) + + barotropic_velocity_kernel!(grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², + U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², + η̅, U̅, V̅, averaging_weight, + Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, + timestepper) + end return nothing end diff --git a/src/Utils/Utils.jl b/src/Utils/Utils.jl index 1704375cd5..761c8d23a6 100644 --- a/src/Utils/Utils.jl +++ b/src/Utils/Utils.jl @@ -42,46 +42,4 @@ include("multi_region_transformation.jl") include("coordinate_transformations.jl") include("sum_of_arrays.jl") - -##### -##### Add Dynamic kernels to KA -##### - -using KernelAbstractions -using CUDA: CUDABackend, @cuda - -const KA = KernelAbstractions - -(obj::KA.Kernel)(args...; ndrange=nothing, workgroupsize=nothing, dynamic_launch=false) = - obj(args...; ndrange, workgroupsize) - -function (obj::KA.Kernel{CUDABackend})(args...; ndrange=nothing, workgroupsize=nothing, dynamic_launch=false) - backend = KA.backend(obj) - - ndrange, workgroupsize, iterspace, dynamic = KA.launch_config(obj, ndrange, workgroupsize) - # this might not be the final context, since we may tune the workgroupsize - ctx = KA.mkcontext(obj, ndrange, iterspace) - - maxthreads = prod(KA.get(KA.workgroupsize(obj))) - - kernel = if dynamic_launch - @cuda launch=false dynamic=true obj.f(ctx, args...) - else - @cuda launch=false always_inline=backend.always_inline maxthreads=maxthreads obj.f(ctx, args...) - end - - blocks = length(KA.blocks(iterspace)) - threads = length(KA.workitems(iterspace)) - - if blocks == 0 - return nothing - end - - # Launch kernel - kernel(ctx, args...; threads, blocks) - - return nothing -end - - end # module From ee97dde9603fc1e87696e6f0a418fe33efa3ecfa Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Wed, 13 Dec 2023 16:13:24 -0500 Subject: [PATCH 083/567] try it like this maybe? --- .../split_explicit_free_surface_kernels.jl | 37 +++++++++++-------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 7237052967..36a4320148 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -5,6 +5,7 @@ using Oceananigans.Utils using Oceananigans.AbstractOperations: Δz using Oceananigans.BoundaryConditions using Oceananigans.Operators +using CUDA: cudaconvert using Oceananigans.ImmersedBoundaries: peripheral_node, immersed_inactive_node using Oceananigans.ImmersedBoundaries: inactive_node, IBG, c, f, SurfaceMap using Oceananigans.ImmersedBoundaries: mask_immersed_field!, use_only_active_surface_cells, use_only_active_interior_cells @@ -156,10 +157,9 @@ end return nothing end -@kernel function _split_explicit_barotropic_velocity!(grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², +@kernel function _split_explicit_barotropic_velocity!(averaging_weight, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, averaging_weight, - Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, + η̅, U̅, V̅, Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, timestepper) i, j = @index(Global, NTuple) velocity_evolution!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², @@ -377,20 +377,27 @@ function iterate_split_explicit!(free_surface, grid, Δt) free_surface_kernel! = configured_kernel(arch, grid, parameters, _split_explicit_free_surface!) barotropic_velocity_kernel! = configured_kernel(arch, grid, parameters, _split_explicit_barotropic_velocity!) - @unroll for substep in 1:N - Base.@_inline_meta + η_args = (grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², + U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², + timestepper) - averaging_weight = weights[substep] + U_args = (grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², + U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², + η̅, U̅, V̅, Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, + timestepper) - free_surface_kernel!(grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², - U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², - timestepper) - - barotropic_velocity_kernel!(grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², - U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, averaging_weight, - Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, - timestepper) + GC.@preserve η_args U_args begin + converted_η_args = map(cudaconvert, η_args) + converted_U_args = map(cudaconvert, U_args) + + @unroll for substep in 1:N + Base.@_inline_meta + + averaging_weight = weights[substep] + + free_surface_kernel!(converted_η_args...) + barotropic_velocity_kernel!(averaging_weight, converted_U_args...) + end end return nothing From 6f5e6b769cadceb3dc29c294b84307187e45837e Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Wed, 13 Dec 2023 16:19:29 -0500 Subject: [PATCH 084/567] convert --- .../split_explicit_free_surface_kernels.jl | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 36a4320148..f370628931 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -387,8 +387,13 @@ function iterate_split_explicit!(free_surface, grid, Δt) timestepper) GC.@preserve η_args U_args begin - converted_η_args = map(cudaconvert, η_args) - converted_U_args = map(cudaconvert, U_args) + + # Since we need to perform ~50 time-steps which means + # launching ~100 very small kernels, we are limited by + # latency of argument conversion to GPU-compatible values + # To alleviate that penalty we convert first and then we substep! + converted_η_args = convert_args(arch, η_args) + converted_U_args = convert_args(arch, U_args) @unroll for substep in 1:N Base.@_inline_meta @@ -403,6 +408,10 @@ function iterate_split_explicit!(free_surface, grid, Δt) return nothing end +convert_args(::CPU, arg) = args +convert_args(::GPU, arg) = cudaconvert(arg) +convert_args(::GPU, arg::Tuple) = map(cudaconvert, args) + # Calculate RHS for the barotopic time step. @kernel function _compute_integrated_ab2_tendencies!(Gᵁ, Gⱽ, grid, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) i, j = @index(Global, NTuple) From acd1a541346ff8affd90d271fe25febb532efcfb Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Wed, 13 Dec 2023 16:25:04 -0500 Subject: [PATCH 085/567] fixxing --- .../split_explicit_free_surface_kernels.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index f370628931..93fe48fc8b 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -410,7 +410,7 @@ end convert_args(::CPU, arg) = args convert_args(::GPU, arg) = cudaconvert(arg) -convert_args(::GPU, arg::Tuple) = map(cudaconvert, args) +convert_args(::GPU, arg::Tuple) = map(cudaconvert, arg) # Calculate RHS for the barotopic time step. @kernel function _compute_integrated_ab2_tendencies!(Gᵁ, Gⱽ, grid, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) From ca7326891eb8e610311de870b948db981bbd6635 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Wed, 13 Dec 2023 16:28:06 -0500 Subject: [PATCH 086/567] try it now? --- .../split_explicit_free_surface_kernels.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 93fe48fc8b..a0ebe7e5d4 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -395,7 +395,7 @@ function iterate_split_explicit!(free_surface, grid, Δt) converted_η_args = convert_args(arch, η_args) converted_U_args = convert_args(arch, U_args) - @unroll for substep in 1:N + @unroll for substep in 1:Nsubsteps Base.@_inline_meta averaging_weight = weights[substep] From 2d8ae26a36befe74db0e5ea8e4675a97c908477a Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Wed, 13 Dec 2023 17:16:39 -0500 Subject: [PATCH 087/567] bugfix --- .../split_explicit_free_surface_kernels.jl | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index a0ebe7e5d4..24ff93f3d4 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -388,10 +388,10 @@ function iterate_split_explicit!(free_surface, grid, Δt) GC.@preserve η_args U_args begin - # Since we need to perform ~50 time-steps which means - # launching ~100 very small kernels, we are limited by - # latency of argument conversion to GPU-compatible values - # To alleviate that penalty we convert first and then we substep! + # We need to perform ~50 time-steps which means + # launching ~100 very small kernels: we are limited by + # latency of argument conversion to GPU-compatible values. + # To alleviate this penalty we convert first and then we substep! converted_η_args = convert_args(arch, η_args) converted_U_args = convert_args(arch, U_args) @@ -412,6 +412,8 @@ convert_args(::CPU, arg) = args convert_args(::GPU, arg) = cudaconvert(arg) convert_args(::GPU, arg::Tuple) = map(cudaconvert, arg) +convert_args(arch::Distributed, arg) = convert_args(child_architecture(arch), arg) + # Calculate RHS for the barotopic time step. @kernel function _compute_integrated_ab2_tendencies!(Gᵁ, Gⱽ, grid, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) i, j = @index(Global, NTuple) From 5341b713f82fc72992f0997fe88b31b9cdefe968 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Wed, 13 Dec 2023 17:49:53 -0500 Subject: [PATCH 088/567] add distributed --- .../split_explicit_free_surface_kernels.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 24ff93f3d4..1c2b85dbe4 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -10,6 +10,7 @@ using Oceananigans.ImmersedBoundaries: peripheral_node, immersed_inactive_node using Oceananigans.ImmersedBoundaries: inactive_node, IBG, c, f, SurfaceMap using Oceananigans.ImmersedBoundaries: mask_immersed_field!, use_only_active_surface_cells, use_only_active_interior_cells using Oceananigans.ImmersedBoundaries: active_linear_index_to_tuple, ActiveCellsIBG, ActiveSurfaceIBG +using Oceananigans.DistributedComputations: child_architecture # constants for AB3 time stepping scheme (from https://doi.org/10.1016/j.ocemod.2004.08.002) const β = 0.281105 From 44d96125154901da81ea56e52b683590e1b9db99 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 14 Dec 2023 05:10:36 -0500 Subject: [PATCH 089/567] bugfix --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 8bd1d7de48..e693e20e2d 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -102,6 +102,6 @@ function on_architecture(arch, ib::GridFittedBottom{<:Field}) return GridFittedBottom(new_bottom_height, ib.immersed_condition) end -Adapt.adapt_structure(to, ib::GridFittedBottom) = GridFittedBottom(adapt(to, ib.bottom_height.data), +Adapt.adapt_structure(to, ib::GridFittedBottom) = GridFittedBottom(adapt(to, ib.bottom_height), ib.immersed_condition) From 1ce6a5ae0b2deff8d5a250e9613d9a90f8ced517 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 14 Dec 2023 05:12:26 -0500 Subject: [PATCH 090/567] bugfix --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 8bd1d7de48..e693e20e2d 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -102,6 +102,6 @@ function on_architecture(arch, ib::GridFittedBottom{<:Field}) return GridFittedBottom(new_bottom_height, ib.immersed_condition) end -Adapt.adapt_structure(to, ib::GridFittedBottom) = GridFittedBottom(adapt(to, ib.bottom_height.data), +Adapt.adapt_structure(to, ib::GridFittedBottom) = GridFittedBottom(adapt(to, ib.bottom_height), ib.immersed_condition) From 53055d2e644ccb3d4516dc0515da64363b3aa93c Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 14 Dec 2023 08:35:07 -0500 Subject: [PATCH 091/567] allow unrolling --- .../split_explicit_free_surface_kernels.jl | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 1c2b85dbe4..b2ecf05d5f 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -310,9 +310,21 @@ function split_explicit_free_surface_step!(free_surface::SplitExplicitFreeSurfac # reset free surface averages @apply_regionally begin - initialize_free_surface_state!(free_surface.state, free_surface.η, free_surface.settings.timestepper) + settings = free_surface.settings + + initialize_free_surface_state!(free_surface.state, free_surface.η, settings.timestepper) + + Nsubsteps = calculate_substeps(settings.substepping, Δt) + + # barotropic time step as fraction of baroclinic step and averaging weights + fractional_Δt, weights = calculate_adaptive_settings(settings.substepping, Nsubsteps) + Nsubsteps = length(weights) + + # barotropic time step in seconds + Δτᴮ = fractional_Δt * Δt + # Solve for the free surface at tⁿ⁺¹ - iterate_split_explicit!(free_surface, free_surface_grid, Δt) + iterate_split_explicit!(free_surface, free_surface_grid, Δτᴮ, weights, Val(Nsubsteps)) # Reset eta for the next timestep set!(free_surface.η, free_surface.state.η̅) end @@ -347,7 +359,7 @@ const MINIMUM_SUBSTEPS = 5 const FixedSubstepsSetting{N} = SplitExplicitSettings{<:FixedSubstepNumber{<:Any, <:NTuple{N, <:Any}}} where N const FixedSubstepsSplitExplicit{F} = SplitExplicitFreeSurface{<:Any, <:Any, <:Any, <:Any, <:FixedSubstepsSetting{N}} where N -function iterate_split_explicit!(free_surface, grid, Δt) +function iterate_split_explicit!(free_surface, grid, Δτᴮ, weights, ::Val{Nsubsteps}) where Nsubsteps arch = architecture(grid) η = free_surface.η @@ -355,13 +367,6 @@ function iterate_split_explicit!(free_surface, grid, Δt) auxiliary = free_surface.auxiliary settings = free_surface.settings g = free_surface.gravitational_acceleration - - Nsubsteps = calculate_substeps(settings.substepping, Δt) - fractional_Δt, weights = calculate_adaptive_settings(settings.substepping, Nsubsteps) # barotropic time step in fraction of baroclinic step and averaging weights - - Nsubsteps = length(weights) - - Δτᴮ = fractional_Δt * Δt # barotropic time step in seconds # unpack state quantities, parameters and forcing terms U, V = state.U, state.V From 4185152bece1f847cb2d893d6ab1594d9d9f7e76 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 14 Dec 2023 09:26:50 -0500 Subject: [PATCH 092/567] convert in archs --- src/Architectures.jl | 9 +++++++++ .../split_explicit_free_surface_kernels.jl | 8 +------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/Architectures.jl b/src/Architectures.jl index 54d7a95fc1..ec9c6b319b 100644 --- a/src/Architectures.jl +++ b/src/Architectures.jl @@ -113,4 +113,13 @@ end @inline unsafe_free!(a::CuArray) = CUDA.unsafe_free!(a) @inline unsafe_free!(a) = nothing +# Convert arguments to GPU-compatible types + +@inline convert_args(::CPU, arg) = args +@inline convert_args(::GPU, arg) = CUDA.cudaconvert(arg) +@inline convert_args(::GPU, arg::Tuple) = map(CUDA.cudaconvert, arg) + +@inline convert_args(arch::Distributed, arg) = convert_args(child_architecture(arch), arg) + + end # module diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index b2ecf05d5f..3bbc5460c7 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -5,7 +5,7 @@ using Oceananigans.Utils using Oceananigans.AbstractOperations: Δz using Oceananigans.BoundaryConditions using Oceananigans.Operators -using CUDA: cudaconvert +using Oceananigans.Architectures: convert_args using Oceananigans.ImmersedBoundaries: peripheral_node, immersed_inactive_node using Oceananigans.ImmersedBoundaries: inactive_node, IBG, c, f, SurfaceMap using Oceananigans.ImmersedBoundaries: mask_immersed_field!, use_only_active_surface_cells, use_only_active_interior_cells @@ -414,12 +414,6 @@ function iterate_split_explicit!(free_surface, grid, Δτᴮ, weights, ::Val{Nsu return nothing end -convert_args(::CPU, arg) = args -convert_args(::GPU, arg) = cudaconvert(arg) -convert_args(::GPU, arg::Tuple) = map(cudaconvert, arg) - -convert_args(arch::Distributed, arg) = convert_args(child_architecture(arch), arg) - # Calculate RHS for the barotopic time step. @kernel function _compute_integrated_ab2_tendencies!(Gᵁ, Gⱽ, grid, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) i, j = @index(Global, NTuple) From 0aa5b10499d3a34393738ff3ca4b026a49e3fc76 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 14 Dec 2023 07:29:25 -0800 Subject: [PATCH 093/567] bugfix --- src/Architectures.jl | 2 -- src/DistributedComputations/distributed_architectures.jl | 3 ++- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Architectures.jl b/src/Architectures.jl index ec9c6b319b..0dc61f71ed 100644 --- a/src/Architectures.jl +++ b/src/Architectures.jl @@ -119,7 +119,5 @@ end @inline convert_args(::GPU, arg) = CUDA.cudaconvert(arg) @inline convert_args(::GPU, arg::Tuple) = map(CUDA.cudaconvert, arg) -@inline convert_args(arch::Distributed, arg) = convert_args(child_architecture(arch), arg) - end # module diff --git a/src/DistributedComputations/distributed_architectures.jl b/src/DistributedComputations/distributed_architectures.jl index 33fcfc3da8..c9046ca834 100644 --- a/src/DistributedComputations/distributed_architectures.jl +++ b/src/DistributedComputations/distributed_architectures.jl @@ -2,7 +2,7 @@ using Oceananigans.Architectures using Oceananigans.Grids: topology, validate_tupled_argument using CUDA: ndevices, device! -import Oceananigans.Architectures: device, cpu_architecture, arch_array, array_type, child_architecture +import Oceananigans.Architectures: device, cpu_architecture, arch_array, array_type, child_architecture, convert_args import Oceananigans.Grids: zeros import Oceananigans.Utils: sync_device!, tupleit @@ -264,6 +264,7 @@ arch_array(arch::Distributed, A) = arch_array(child_architecture(arch), A) zeros(FT, arch::Distributed, N...) = zeros(FT, child_architecture(arch), N...) array_type(arch::Distributed) = array_type(child_architecture(arch)) sync_device!(arch::Distributed) = sync_device!(arch.child_architecture) +convert_args(arch::Distributed, arg) = convert_args(child_architecture(arch), arg) cpu_architecture(arch::DistributedCPU) = arch cpu_architecture(arch::Distributed{A, S}) where {A, S} = From 71a1fc5393adb248d8ae1ed3894234c316b67224 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 14 Dec 2023 11:03:15 -0500 Subject: [PATCH 094/567] this might work --- .../compute_w_from_continuity.jl | 19 ++++++-- .../hydrostatic_free_surface_model.jl | 6 +-- ..._free_surface_tendency_kernel_functions.jl | 2 + .../moving_vertical_coordinate.jl | 44 ++++++++++++++----- .../prescribed_hydrostatic_velocity_fields.jl | 4 +- .../set_hydrostatic_free_surface_model.jl | 2 +- .../single_column_model_mode.jl | 2 +- ...te_hydrostatic_free_surface_model_state.jl | 18 ++++---- .../nonhydrostatic_model.jl | 2 +- .../set_nonhydrostatic_model.jl | 4 +- .../update_nonhydrostatic_model_state.jl | 4 +- .../set_shallow_water_model.jl | 2 +- .../ShallowWaterModels/shallow_water_model.jl | 2 +- .../update_shallow_water_state.jl | 2 +- src/Simulations/run.jl | 2 +- src/TimeSteppers/quasi_adams_bashforth_2.jl | 4 +- src/TimeSteppers/runge_kutta_3.jl | 8 ++-- .../geostrophic_adjustment_test.jl | 4 +- 18 files changed, 83 insertions(+), 48 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl index 8e8f2cc43a..b992a69436 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl @@ -12,12 +12,12 @@ Compute the vertical velocity ``w`` by integrating the continuity equation from w^{n+1} = -∫ [∂/∂x (u^{n+1}) + ∂/∂y (v^{n+1})] dz ``` """ -compute_w_from_continuity!(model; kwargs...) = compute_w_from_continuity!(model.velocities, model.architecture, model.grid; kwargs...) +compute_w_from_continuity!(model, Δt; kwargs...) = compute_w_from_continuity!(model.velocities, model.architecture, model.grid, Δt; kwargs...) -compute_w_from_continuity!(velocities, arch, grid; parameters = w_kernel_parameters(grid)) = - launch!(arch, grid, parameters, _compute_w_from_continuity!, velocities, grid) +compute_w_from_continuity!(velocities, arch, grid, Δt; parameters = w_kernel_parameters(grid)) = + launch!(arch, grid, parameters, _compute_w_from_continuity!, velocities, grid, Δt) -@kernel function _compute_w_from_continuity!(U, grid) +@kernel function _compute_w_from_continuity!(U, grid, Δt) i, j = @index(Global, NTuple) U.w[i, j, 1] = 0 @@ -26,6 +26,17 @@ compute_w_from_continuity!(velocities, arch, grid; parameters = w_kernel_paramet end end +@kernel function _compute_w_from_continuity!(U, grid::ZStarCoordinateGrid, Δt) + i, j = @index(Global, NTuple) + + U.w[i, j, 1] = 0 + @unroll for k in 2:grid.Nz+1 + @inbounds U.w[i, j, k] = U.w[i, j, k-1] + - Δzᶜᶜᶜ(i, j, k-1, grid) * div_xyᶜᶜᶜ(i, j, k-1, grid, U.u, U.v) + - (grid.Δzᵃᵃᶠ.previous_scaling[i, j, grid.Nz+1] - grid.Δzᵃᵃᶠ.previous_scaling[i, j, grid.Nz+1]) / Δt + end +end + ##### ##### Size and offsets for the w kernel ##### diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl index f4f64c8ee4..f17857698c 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl @@ -118,8 +118,8 @@ function HydrostaticFreeSurfaceModel(; grid, # Check halos and throw an error if the grid's halo is too small @apply_regionally validate_model_halo(grid, momentum_advection, tracer_advection, closure) - # Introduce z-star coordinates if needed - grid = MovingCoordinateGrid(grid, vertical_coordinate) + # Introduce z-star coordinates if needed (only is free_surface is not a nothing) + grid = !isnothing(free_surface) ? MovingCoordinateGrid(grid, vertical_coordinate) : grid arch = architecture(grid) @@ -198,7 +198,7 @@ function HydrostaticFreeSurfaceModel(; grid, free_surface, forcing, closure, particles, biogeochemistry, velocities, tracers, pressure, diffusivity_fields, timestepper, auxiliary_fields) - update_state!(model) + update_state!(model, 1.0) return model end diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_tendency_kernel_functions.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_tendency_kernel_functions.jl index 01ceaa21ec..967d6c5b35 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_tendency_kernel_functions.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_tendency_kernel_functions.jl @@ -48,6 +48,7 @@ implicitly during time-stepping. - explicit_barotropic_pressure_x_gradient(i, j, k, grid, free_surface) - x_f_cross_U(i, j, k, grid, coriolis, velocities) - ∂xᶠᶜᶜ(i, j, k, grid, hydrostatic_pressure_anomaly) + - free_surface_slope_x(i, j, k, grid, free_surface, buoyancy, model_fields) - ∂ⱼ_τ₁ⱼ(i, j, k, grid, closure, diffusivities, clock, model_fields, buoyancy) - immersed_∂ⱼ_τ₁ⱼ(i, j, k, grid, velocities, u_immersed_bc, closure, diffusivities, clock, model_fields) + forcings.u(i, j, k, grid, clock, hydrostatic_prognostic_fields(velocities, free_surface, tracers))) @@ -87,6 +88,7 @@ implicitly during time-stepping. - explicit_barotropic_pressure_y_gradient(i, j, k, grid, free_surface) - y_f_cross_U(i, j, k, grid, coriolis, velocities) - ∂yᶜᶠᶜ(i, j, k, grid, hydrostatic_pressure_anomaly) + - free_surface_slope_y(i, j, k, grid, free_surface, buoyancy, model_fields) - ∂ⱼ_τ₂ⱼ(i, j, k, grid, closure, diffusivities, clock, model_fields, buoyancy) - immersed_∂ⱼ_τ₂ⱼ(i, j, k, grid, velocities, v_immersed_bc, closure, diffusivities, clock, model_fields) + forcings.v(i, j, k, grid, clock, model_fields)) diff --git a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl index bb55aeeb45..24d815a633 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl @@ -1,5 +1,7 @@ using Oceananigans using Oceananigans.Grids +using Oceananigans.Operators +using Oceananigans.BuoyancyModels: buoyancy_perturbationᶜᶜᶜ using Oceananigans.Grids: AbstractGrid, AbstractUnderlyingGrid, halo_size using Oceananigans.ImmersedBoundaries using Oceananigans.Utils: getnamewrapper @@ -8,15 +10,17 @@ using Adapt struct ZCoordinate end struct ZStarCoordinate{R, S, Z} - reference :: R - scaling :: S - star_value :: Z + reference :: R +previous_scaling :: S + scaling :: S + star_value :: Z end -ZStarCoordinate() = ZStarCoordinate(nothing, nothing, nothing) +ZStarCoordinate() = ZStarCoordinate(nothing, nothing, nothing, nothing) Adapt.adapt_structure(to, coord::ZStarCoordinate) = ZStarCoordinate(Adapt.adapt(to, coord.reference), + Adapt.adapt(to, coord.scaling), Adapt.adapt(to, coord.scaling), Adapt.adapt(to, coord.star_value)) @@ -42,9 +46,10 @@ function MovingCoordinateGrid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, ::ZS ΔzF = ZFaceField(grid) ΔzC = CenterField(grid) scaling = ZFaceField(grid, indices = (:, :, grid.Nz + 1)) + previous_scaling = ZFaceField(grid, indices = (:, :, grid.Nz + 1)) - Δzᵃᵃᶠ = ZStarCoordinate(grid.Δzᵃᵃᶠ, scaling, ΔzF) - Δzᵃᵃᶜ = ZStarCoordinate(grid.Δzᵃᵃᶜ, scaling, ΔzC) + Δzᵃᵃᶠ = ZStarCoordinate(grid.Δzᵃᵃᶠ, previous_scaling, scaling, ΔzF) + Δzᵃᵃᶜ = ZStarCoordinate(grid.Δzᵃᵃᶜ, previous_scaling, scaling, ΔzC) args = [] for prop in propertynames(grid) @@ -70,6 +75,7 @@ function update_vertical_coordinate!(model, grid::ZStarCoordinateGrid; parameter # Scaling scaling = grid.Δzᵃᵃᶠ.scaling + previous_scaling = grid.Δzᵃᵃᶠ.previous_scaling # Moving coordinates Δzᵃᵃᶠ = grid.Δzᵃᵃᶠ.star_value @@ -81,7 +87,7 @@ function update_vertical_coordinate!(model, grid::ZStarCoordinateGrid; parameter # Update the scaling on the whole grid (from -H to N+H) launch!(architecture(grid), grid, horizontal_parameters(grid), _update_scaling!, - scaling, η, grid) + previous_scaling, scaling, η, grid) # Update vertical coordinate with available parameters for params in parameters @@ -102,12 +108,13 @@ end @kernel function _update_scaling!(scaling, η, grid) i, j = @index(Global, NTuple) - bottom = bottom_height(grid, i, j) + bottom = bottom_height(i, j, grid) + @inbounds previous_scaling[i, j, grid.Nz+1] = scaling[i, j, grid.Nz+1] @inbounds scaling[i, j, grid.Nz+1] = (bottom + η[i, j, grid.Nz+1]) / bottom end -bottom_height(grid, i, j) = grid.Lz -bottom_height(grid::ImmersedBoundaryGrid, i, j) = @inbounds - grid.immersed_boundary.bottom_height[i, j, 1] +bottom_height(i, j, grid) = grid.Lz +bottom_height(i, j, grid::ImmersedBoundaryGrid) = @inbounds - grid.immersed_boundary.bottom_height[i, j, 1] @kernel function _update_z_star!(ΔzF, ΔzC, ΔzF₀, ΔzC₀, scaling, Nz) i, j, k = @index(Global, NTuple) @@ -140,4 +147,19 @@ import Oceananigans.Operators: Δzᶜᶜᶠ, Δzᶜᶜᶜ, Δzᶜᶠᶠ, Δzᶜ import Oceananigans.Architectures: arch_array arch_array(arch, coord::ZStarCoordinate) = - ZStarCoordinate(arch_array(arch, coord.reference), coord.scaling, coord.star_value) \ No newline at end of file + ZStarCoordinate(arch_array(arch, coord.reference), coord.scaling, coord.star_value) + +# Adding the slope to the momentum-RHS +@inline free_surface_slope_x(i, j, k, grid, args...) = nothing +@inline free_surface_slope_y(i, j, k, grid, args...) = nothing + +@inline η_times_zᶜᶜᶜ(i, j, k, grid, η) = @inbounds η[i, j, grid.Nz+1] * (1 + grid.zᵃᵃᶜ[k] / bottom(i, j, grid)) + +@inline η_slope_xᶠᶜᶜ(i, j, k, grid, free_surface) = @inbounds ∂xᶠᶜᶜ(i, j, k, grid, η_times_zᶜᶜᶜ, free_surface.η) +@inline η_slope_yᶜᶠᶜ(i, j, k, grid, free_surface) = @inbounds ∂yᶜᶠᶜ(i, j, k, grid, η_times_zᶜᶜᶜ, free_surface.η) + +@inline free_surface_slope_x(i, j, k, grid::ZStarCoordinateGrid, free_surface, buoyancy, model_fields) = + ℑxᶠᵃᵃ(i, j, k, grid, buoyancy_perturbationᶜᶜᶜ, buoyancy, model_fields) * η_slope_xᶠᶜᶜ(i, j, k, grid, free_surface) + +@inline free_surface_slope_y(i, j, k, grid::ZStarCoordinateGrid, free_surface, buoyancy, model_fields) = + ℑyᵃᶠᵃ(i, j, k, grid, buoyancy_perturbationᶜᶜᶜ, buoyancy, model_fields) * η_slope_yᶜᶠᶜ(i, j, k, grid, free_surface) diff --git a/src/Models/HydrostaticFreeSurfaceModels/prescribed_hydrostatic_velocity_fields.jl b/src/Models/HydrostaticFreeSurfaceModels/prescribed_hydrostatic_velocity_fields.jl index 9905bfc586..68547fbcd5 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/prescribed_hydrostatic_velocity_fields.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/prescribed_hydrostatic_velocity_fields.jl @@ -121,8 +121,8 @@ function time_step!(model::OnlyParticleTrackingModel, Δt; callbacks = [], kwarg model.timestepper.previous_Δt = Δt tick!(model.clock, Δt) step_lagrangian_particles!(model, Δt) - update_state!(model, callbacks) + update_state!(model, Δt, callbacks) end -update_state!(model::OnlyParticleTrackingModel, callbacks) = +update_state!(model::OnlyParticleTrackingModel, Δt, callbacks) = [callback(model) for callback in callbacks if callback.callsite isa UpdateStateCallsite] diff --git a/src/Models/HydrostaticFreeSurfaceModels/set_hydrostatic_free_surface_model.jl b/src/Models/HydrostaticFreeSurfaceModels/set_hydrostatic_free_surface_model.jl index 80312aebe6..a7c0b378e9 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/set_hydrostatic_free_surface_model.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/set_hydrostatic_free_surface_model.jl @@ -60,7 +60,7 @@ function set!(model::HydrostaticFreeSurfaceModel; kwargs...) @apply_regionally set!(ϕ, value) end - update_state!(model) + update_state!(model, 1.0) return nothing end diff --git a/src/Models/HydrostaticFreeSurfaceModels/single_column_model_mode.jl b/src/Models/HydrostaticFreeSurfaceModels/single_column_model_mode.jl index b04d251af0..2ae9f8f1f0 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/single_column_model_mode.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/single_column_model_mode.jl @@ -59,7 +59,7 @@ compute_free_surface_tendency!(::SingleColumnGrid, ::SplitExplicitFreeSurfaceHFS # Fast state update and halo filling -function update_state!(model::HydrostaticFreeSurfaceModel, grid::SingleColumnGrid, callbacks) +function update_state!(model::HydrostaticFreeSurfaceModel, grid::SingleColumnGrid, Δt, callbacks) fill_halo_regions!(prognostic_fields(model), model.clock, fields(model)) diff --git a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl index 4843ecc571..cf80eba51a 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl @@ -19,16 +19,16 @@ compute_auxiliary_fields!(auxiliary_fields) = Tuple(compute!(a) for a in auxilia # single column models. """ - update_state!(model::HydrostaticFreeSurfaceModel, callbacks=[]) + update_state!(model::HydrostaticFreeSurfaceModel, Δt, callbacks=[]) Update peripheral aspects of the model (auxiliary fields, halo regions, diffusivities, hydrostatic pressure) to the current model state. If `callbacks` are provided (in an array), they are called in the end. """ -update_state!(model::HydrostaticFreeSurfaceModel, callbacks=[]; compute_tendencies = true) = - update_state!(model, model.grid, callbacks; compute_tendencies) +update_state!(model::HydrostaticFreeSurfaceModel, Δt, callbacks=[]; compute_tendencies = true) = + update_state!(model, model.grid, Δt, callbacks; compute_tendencies) -function update_state!(model::HydrostaticFreeSurfaceModel, grid, callbacks; compute_tendencies = true) +function update_state!(model::HydrostaticFreeSurfaceModel, grid, Δt, callbacks; compute_tendencies = true) @apply_regionally mask_immersed_model_fields!(model, grid) @@ -38,7 +38,7 @@ function update_state!(model::HydrostaticFreeSurfaceModel, grid, callbacks; comp fill_halo_regions!(prognostic_fields(model), model.clock, fields(model); async = true) @apply_regionally replace_horizontal_vector_halos!(model.velocities, model.grid) - @apply_regionally compute_auxiliaries!(model) + @apply_regionally compute_auxiliaries!(model, Δt) fill_halo_regions!(model.diffusivity_fields; only_local_halos = true) @@ -67,16 +67,16 @@ function mask_immersed_model_fields!(model, grid) return nothing end -function compute_auxiliaries!(model::HydrostaticFreeSurfaceModel; w_parameters = tuple(w_kernel_parameters(model.grid)), - p_parameters = tuple(p_kernel_parameters(model.grid)), - κ_parameters = tuple(:xyz)) +function compute_auxiliaries!(model::HydrostaticFreeSurfaceModel, Δt; w_parameters = tuple(w_kernel_parameters(model.grid)), + p_parameters = tuple(p_kernel_parameters(model.grid)), + κ_parameters = tuple(:xyz)) grid = model.grid closure = model.closure diffusivity = model.diffusivity_fields for (wpar, ppar, κpar) in zip(w_parameters, p_parameters, κ_parameters) - compute_w_from_continuity!(model; parameters = wpar) + compute_w_from_continuity!(model, Δt; parameters = wpar) compute_diffusivities!(diffusivity, closure, model; parameters = κpar) update_hydrostatic_pressure!(model.pressure.pHY′, architecture(grid), grid, model.buoyancy, model.tracers; diff --git a/src/Models/NonhydrostaticModels/nonhydrostatic_model.jl b/src/Models/NonhydrostaticModels/nonhydrostatic_model.jl index 2978403895..8e9a1102de 100644 --- a/src/Models/NonhydrostaticModels/nonhydrostatic_model.jl +++ b/src/Models/NonhydrostaticModels/nonhydrostatic_model.jl @@ -197,7 +197,7 @@ function NonhydrostaticModel(; grid, pressures, diffusivity_fields, timestepper, pressure_solver, immersed_boundary, auxiliary_fields) - update_state!(model) + update_state!(model, Δt) return model end diff --git a/src/Models/NonhydrostaticModels/set_nonhydrostatic_model.jl b/src/Models/NonhydrostaticModels/set_nonhydrostatic_model.jl index 19003a9438..58f0e4f8ec 100644 --- a/src/Models/NonhydrostaticModels/set_nonhydrostatic_model.jl +++ b/src/Models/NonhydrostaticModels/set_nonhydrostatic_model.jl @@ -44,13 +44,13 @@ function set!(model::NonhydrostaticModel; enforce_incompressibility=true, kwargs # Apply a mask foreach(mask_immersed_field!, model.tracers) foreach(mask_immersed_field!, model.velocities) - update_state!(model) + update_state!(model, 1.0) if enforce_incompressibility FT = eltype(model.grid) calculate_pressure_correction!(model, one(FT)) pressure_correct_velocities!(model, one(FT)) - update_state!(model) + update_state!(model, 1.0) end return nothing diff --git a/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl b/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl index b59a74eafe..ac655c480e 100644 --- a/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl +++ b/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl @@ -10,13 +10,13 @@ using Oceananigans.Models: update_model_field_time_series! import Oceananigans.TimeSteppers: update_state! """ - update_state!(model::NonhydrostaticModel, callbacks=[]) + update_state!(model::NonhydrostaticModel, Δt, callbacks=[]) Update peripheral aspects of the model (halo regions, diffusivities, hydrostatic pressure) to the current model state. If `callbacks` are provided (in an array), they are called in the end. """ -function update_state!(model::NonhydrostaticModel, callbacks=[]; compute_tendencies = true) +function update_state!(model::NonhydrostaticModel, Δt, callbacks=[]; compute_tendencies = true) # Mask immersed tracers foreach(model.tracers) do tracer diff --git a/src/Models/ShallowWaterModels/set_shallow_water_model.jl b/src/Models/ShallowWaterModels/set_shallow_water_model.jl index b68f279a04..ce2159bebf 100644 --- a/src/Models/ShallowWaterModels/set_shallow_water_model.jl +++ b/src/Models/ShallowWaterModels/set_shallow_water_model.jl @@ -14,7 +14,7 @@ function set!(model::ShallowWaterModel; kwargs...) set!(ϕ, value) end - update_state!(model) + update_state!(model, 1.0) return nothing end diff --git a/src/Models/ShallowWaterModels/shallow_water_model.jl b/src/Models/ShallowWaterModels/shallow_water_model.jl index d4c468e5c3..73c7f8f2e7 100644 --- a/src/Models/ShallowWaterModels/shallow_water_model.jl +++ b/src/Models/ShallowWaterModels/shallow_water_model.jl @@ -205,7 +205,7 @@ function ShallowWaterModel(; timestepper, formulation) - update_state!(model) + update_state!(model, 1.0) return model end diff --git a/src/Models/ShallowWaterModels/update_shallow_water_state.jl b/src/Models/ShallowWaterModels/update_shallow_water_state.jl index 1974dc5188..2a05f6527a 100644 --- a/src/Models/ShallowWaterModels/update_shallow_water_state.jl +++ b/src/Models/ShallowWaterModels/update_shallow_water_state.jl @@ -10,7 +10,7 @@ import Oceananigans.TimeSteppers: update_state! Fill halo regions for `model.solution` and `model.tracers`. If `callbacks` are provided (in an array), they are called in the end. """ -function update_state!(model::ShallowWaterModel, callbacks=[]; compute_tendencies = true) +function update_state!(model::ShallowWaterModel, Δt, callbacks=[]; compute_tendencies = true) # Mask immersed fields foreach(mask_immersed_field!, model.solution) diff --git a/src/Simulations/run.jl b/src/Simulations/run.jl index ab8f2ca156..9dbfd04dd9 100644 --- a/src/Simulations/run.jl +++ b/src/Simulations/run.jl @@ -188,7 +188,7 @@ function initialize!(sim::Simulation) model = sim.model clock = model.clock - update_state!(model) + update_state!(model, 1.0) # Output and diagnostics initialization [add_dependencies!(sim.diagnostics, writer) for writer in values(sim.output_writers)] diff --git a/src/TimeSteppers/quasi_adams_bashforth_2.jl b/src/TimeSteppers/quasi_adams_bashforth_2.jl index 6c3854e843..69aabaaa1b 100644 --- a/src/TimeSteppers/quasi_adams_bashforth_2.jl +++ b/src/TimeSteppers/quasi_adams_bashforth_2.jl @@ -88,7 +88,7 @@ function time_step!(model::AbstractModel{<:QuasiAdamsBashforth2TimeStepper}, Δt model.timestepper.previous_Δt = Δt # Be paranoid and update state at iteration 0 - model.clock.iteration == 0 && update_state!(model, callbacks) + model.clock.iteration == 0 && update_state!(model, Δt, callbacks) ab2_step!(model, Δt, χ) # full step for tracers, fractional step for velocities. calculate_pressure_correction!(model, Δt) @@ -96,7 +96,7 @@ function time_step!(model::AbstractModel{<:QuasiAdamsBashforth2TimeStepper}, Δt @apply_regionally correct_velocities_and_store_tendecies!(model, Δt) tick!(model.clock, Δt) - update_state!(model, callbacks; compute_tendencies) + update_state!(model, Δt, callbacks; compute_tendencies) step_lagrangian_particles!(model, Δt) return nothing diff --git a/src/TimeSteppers/runge_kutta_3.jl b/src/TimeSteppers/runge_kutta_3.jl index ac28b59c21..1ba1c5cac6 100644 --- a/src/TimeSteppers/runge_kutta_3.jl +++ b/src/TimeSteppers/runge_kutta_3.jl @@ -82,7 +82,7 @@ function time_step!(model::AbstractModel{<:RungeKutta3TimeStepper}, Δt; callbac Δt == 0 && @warn "Δt == 0 may cause model blowup!" # Be paranoid and update state at iteration 0, in case run! is not used: - model.clock.iteration == 0 && update_state!(model, callbacks) + model.clock.iteration == 0 && update_state!(model, Δt, callbacks) γ¹ = model.timestepper.γ¹ γ² = model.timestepper.γ² @@ -106,7 +106,7 @@ function time_step!(model::AbstractModel{<:RungeKutta3TimeStepper}, Δt; callbac tick!(model.clock, first_stage_Δt; stage=true) store_tendencies!(model) - update_state!(model, callbacks) + update_state!(model, Δt, callbacks) step_lagrangian_particles!(model, first_stage_Δt) # @@ -120,7 +120,7 @@ function time_step!(model::AbstractModel{<:RungeKutta3TimeStepper}, Δt; callbac tick!(model.clock, second_stage_Δt; stage=true) store_tendencies!(model) - update_state!(model, callbacks) + update_state!(model, Δt, callbacks) step_lagrangian_particles!(model, second_stage_Δt) # @@ -133,7 +133,7 @@ function time_step!(model::AbstractModel{<:RungeKutta3TimeStepper}, Δt; callbac pressure_correct_velocities!(model, third_stage_Δt) tick!(model.clock, third_stage_Δt) - update_state!(model, callbacks; compute_tendencies) + update_state!(model, Δt, callbacks; compute_tendencies) step_lagrangian_particles!(model, third_stage_Δt) return nothing diff --git a/validation/implicit_free_surface/geostrophic_adjustment_test.jl b/validation/implicit_free_surface/geostrophic_adjustment_test.jl index bb6f7a904c..7de77f6744 100644 --- a/validation/implicit_free_surface/geostrophic_adjustment_test.jl +++ b/validation/implicit_free_surface/geostrophic_adjustment_test.jl @@ -47,7 +47,7 @@ function geostrophic_adjustment_simulation(free_surface, topology, multi_region; L = grid.Lx / 40 # gaussian width x₀ = grid.Lx / 4 # gaussian center - vᵍ(x, y, z) = -U * (x - x₀) / L * gaussian(x - x₀, L) + vᵍ(x) = -U * (x - x₀) / L * gaussian(x - x₀, L) g = model.free_surface.gravitational_acceleration η = model.free_surface.η @@ -56,7 +56,7 @@ function geostrophic_adjustment_simulation(free_surface, topology, multi_region; ηᵍ(x) = η₀ * gaussian(x - x₀, L) - ηⁱ(x, y, z) = 2 * ηᵍ(x) + ηⁱ(x) = 2 * ηᵍ(x) set!(model, v = vᵍ) set!(model.free_surface.η, ηⁱ) From 6f47958df563f9e49d3b06539b976ef691df0cfd Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 14 Dec 2023 11:21:16 -0500 Subject: [PATCH 095/567] more stuff taken care of --- ...static_free_surface_boundary_tendencies.jl | 4 +-- ...ute_hydrostatic_free_surface_tendencies.jl | 8 ++--- .../compute_w_from_continuity.jl | 2 +- .../moving_vertical_coordinate.jl | 29 ++++++++++--------- ...te_hydrostatic_free_surface_model_state.jl | 10 +++---- ...pute_nonhydrostatic_boundary_tendencies.jl | 2 +- .../compute_nonhydrostatic_tendencies.jl | 4 +-- .../update_nonhydrostatic_model_state.jl | 2 +- ...nterleave_communication_and_computation.jl | 10 +++---- 9 files changed, 37 insertions(+), 34 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl index 74c837a865..d7e2640305 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl @@ -9,7 +9,7 @@ using Oceananigans.Models.NonhydrostaticModels: boundary_tendency_kernel_paramet import Oceananigans.Models: compute_boundary_tendencies! # We assume here that top/bottom BC are always synched (no partitioning in z) -function compute_boundary_tendencies!(model::HydrostaticFreeSurfaceModel) +function compute_boundary_tendencies!(model::HydrostaticFreeSurfaceModel, Δt) grid = model.grid arch = architecture(grid) @@ -18,7 +18,7 @@ function compute_boundary_tendencies!(model::HydrostaticFreeSurfaceModel) κ_parameters = boundary_κ_kernel_parameters(grid, model.closure, arch) # We need new values for `w`, `p` and `κ` - update_vertical_coordinate!(model, model.grid; parameters = κ_parameters) + update_vertical_coordinate!(model, model.grid, Δt; parameters = κ_parameters) compute_auxiliaries!(model; w_parameters, p_parameters, κ_parameters) # parameters for communicating North / South / East / West side diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl index 6edc4320af..53fe23f692 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl @@ -17,23 +17,23 @@ using Oceananigans.ImmersedBoundaries: use_only_active_interior_cells, ActiveCel InteriorMap, active_linear_index_to_interior_tuple """ - compute_tendencies!(model::HydrostaticFreeSurfaceModel, callbacks) + compute_tendencies!(model::HydrostaticFreeSurfaceModel, Δt, callbacks) Calculate the interior and boundary contributions to tendency terms without the contribution from non-hydrostatic pressure. """ -function compute_tendencies!(model::HydrostaticFreeSurfaceModel, callbacks) +function compute_tendencies!(model::HydrostaticFreeSurfaceModel, Δt, callbacks) kernel_parameters = tuple(interior_tendency_kernel_parameters(model.grid)) - update_vertical_coordinate!(model, model.grid; parameters = kernel_parameters) + update_vertical_coordinate!(model, model.grid, Δt; parameters = kernel_parameters) # Calculate contributions to momentum and tracer tendencies from fluxes and volume terms in the # interior of the domain compute_hydrostatic_free_surface_tendency_contributions!(model, kernel_parameters; only_active_cells = use_only_active_interior_cells(model.grid)) - complete_communication_and_compute_boundary!(model, model.grid, model.architecture) + complete_communication_and_compute_boundary!(model, model.grid, model.architecture, Δt) # Calculate contributions to momentum and tracer tendencies from user-prescribed fluxes across the # boundaries of the domain diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl index b992a69436..703d5233a8 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl @@ -33,7 +33,7 @@ end @unroll for k in 2:grid.Nz+1 @inbounds U.w[i, j, k] = U.w[i, j, k-1] - Δzᶜᶜᶜ(i, j, k-1, grid) * div_xyᶜᶜᶜ(i, j, k-1, grid, U.u, U.v) - - (grid.Δzᵃᵃᶠ.previous_scaling[i, j, grid.Nz+1] - grid.Δzᵃᵃᶠ.previous_scaling[i, j, grid.Nz+1]) / Δt + - grid.Δzᵃᵃᶠ.∂t_scaling[i, j, grid.Nz+1] end end diff --git a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl index 24d815a633..3fd496456d 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl @@ -10,10 +10,10 @@ using Adapt struct ZCoordinate end struct ZStarCoordinate{R, S, Z} - reference :: R -previous_scaling :: S - scaling :: S - star_value :: Z + reference :: R + star_value :: Z + scaling :: S + ∂t_scaling :: S end ZStarCoordinate() = ZStarCoordinate(nothing, nothing, nothing, nothing) @@ -46,10 +46,10 @@ function MovingCoordinateGrid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, ::ZS ΔzF = ZFaceField(grid) ΔzC = CenterField(grid) scaling = ZFaceField(grid, indices = (:, :, grid.Nz + 1)) - previous_scaling = ZFaceField(grid, indices = (:, :, grid.Nz + 1)) + ∂t_scaling = ZFaceField(grid, indices = (:, :, grid.Nz + 1)) - Δzᵃᵃᶠ = ZStarCoordinate(grid.Δzᵃᵃᶠ, previous_scaling, scaling, ΔzF) - Δzᵃᵃᶜ = ZStarCoordinate(grid.Δzᵃᵃᶜ, previous_scaling, scaling, ΔzC) + Δzᵃᵃᶠ = ZStarCoordinate(grid.Δzᵃᵃᶠ, ΔzF, scaling, ∂t_scaling) + Δzᵃᵃᶜ = ZStarCoordinate(grid.Δzᵃᵃᶜ, ΔzC, scaling, ∂t_scaling) args = [] for prop in propertynames(grid) @@ -70,12 +70,12 @@ end # Fallback update_vertical_coordinate!(model, grid; kwargs...) = nothing -function update_vertical_coordinate!(model, grid::ZStarCoordinateGrid; parameters = tuple(:xyz)) +function update_vertical_coordinate!(model, grid::ZStarCoordinateGrid, Δt; parameters = tuple(:xyz)) η = model.free_surface.η # Scaling scaling = grid.Δzᵃᵃᶠ.scaling - previous_scaling = grid.Δzᵃᵃᶠ.previous_scaling + ∂t_scaling = grid.Δzᵃᵃᶠ.previous_scaling # Moving coordinates Δzᵃᵃᶠ = grid.Δzᵃᵃᶠ.star_value @@ -87,7 +87,7 @@ function update_vertical_coordinate!(model, grid::ZStarCoordinateGrid; parameter # Update the scaling on the whole grid (from -H to N+H) launch!(architecture(grid), grid, horizontal_parameters(grid), _update_scaling!, - previous_scaling, scaling, η, grid) + scaling, ∂t_scaling, η, grid, Δt) # Update vertical coordinate with available parameters for params in parameters @@ -106,11 +106,14 @@ function horizontal_parameters(grid) return KernelParameters(total_size, .- halos) end -@kernel function _update_scaling!(scaling, η, grid) +@kernel function _update_scaling!(scaling, ∂t_scaling, η, grid, Δt) i, j = @index(Global, NTuple) bottom = bottom_height(i, j, grid) - @inbounds previous_scaling[i, j, grid.Nz+1] = scaling[i, j, grid.Nz+1] - @inbounds scaling[i, j, grid.Nz+1] = (bottom + η[i, j, grid.Nz+1]) / bottom + @inbounds begin + h = (bottom + η[i, j, grid.Nz+1]) / bottom + ∂t_scaling[i, j, grid.Nz+1] = (h - scaling[i, j, grid.Nz+1]) / Δt + scaling[i, j, grid.Nz+1] = h + end end bottom_height(i, j, grid) = grid.Lz diff --git a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl index cf80eba51a..e8277794e0 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl @@ -38,7 +38,7 @@ function update_state!(model::HydrostaticFreeSurfaceModel, grid, Δt, callbacks; fill_halo_regions!(prognostic_fields(model), model.clock, fields(model); async = true) @apply_regionally replace_horizontal_vector_halos!(model.velocities, model.grid) - @apply_regionally compute_auxiliaries!(model, Δt) + @apply_regionally compute_auxiliaries!(model) fill_halo_regions!(model.diffusivity_fields; only_local_halos = true) @@ -47,7 +47,7 @@ function update_state!(model::HydrostaticFreeSurfaceModel, grid, Δt, callbacks; update_biogeochemical_state!(model.biogeochemistry, model) compute_tendencies && - @apply_regionally compute_tendencies!(model, callbacks) + @apply_regionally compute_tendencies!(model, Δt, callbacks) return nothing end @@ -67,9 +67,9 @@ function mask_immersed_model_fields!(model, grid) return nothing end -function compute_auxiliaries!(model::HydrostaticFreeSurfaceModel, Δt; w_parameters = tuple(w_kernel_parameters(model.grid)), - p_parameters = tuple(p_kernel_parameters(model.grid)), - κ_parameters = tuple(:xyz)) +function compute_auxiliaries!(model::HydrostaticFreeSurfaceModel; w_parameters = tuple(w_kernel_parameters(model.grid)), + p_parameters = tuple(p_kernel_parameters(model.grid)), + κ_parameters = tuple(:xyz)) grid = model.grid closure = model.closure diff --git a/src/Models/NonhydrostaticModels/compute_nonhydrostatic_boundary_tendencies.jl b/src/Models/NonhydrostaticModels/compute_nonhydrostatic_boundary_tendencies.jl index ca8164bbe2..83e332c6ae 100644 --- a/src/Models/NonhydrostaticModels/compute_nonhydrostatic_boundary_tendencies.jl +++ b/src/Models/NonhydrostaticModels/compute_nonhydrostatic_boundary_tendencies.jl @@ -4,7 +4,7 @@ using Oceananigans.TurbulenceClosures: required_halo_size using Oceananigans.Grids: XFlatGrid, YFlatGrid # We assume here that top/bottom BC are always synched (no partitioning in z) -function compute_boundary_tendencies!(model::NonhydrostaticModel) +function compute_boundary_tendencies!(model::NonhydrostaticModel, Δt) grid = model.grid arch = architecture(grid) diff --git a/src/Models/NonhydrostaticModels/compute_nonhydrostatic_tendencies.jl b/src/Models/NonhydrostaticModels/compute_nonhydrostatic_tendencies.jl index 4e47a7b472..adbca4ab98 100644 --- a/src/Models/NonhydrostaticModels/compute_nonhydrostatic_tendencies.jl +++ b/src/Models/NonhydrostaticModels/compute_nonhydrostatic_tendencies.jl @@ -14,7 +14,7 @@ import Oceananigans.TimeSteppers: compute_tendencies! Calculate the interior and boundary contributions to tendency terms without the contribution from non-hydrostatic pressure. """ -function compute_tendencies!(model::NonhydrostaticModel, callbacks) +function compute_tendencies!(model::NonhydrostaticModel, Δt, callbacks) # Note: # @@ -29,7 +29,7 @@ function compute_tendencies!(model::NonhydrostaticModel, callbacks) kernel_parameters = tuple(interior_tendency_kernel_parameters(model.grid)) compute_interior_tendency_contributions!(model, kernel_parameters; only_active_cells = use_only_active_interior_cells(model.grid)) - complete_communication_and_compute_boundary!(model, model.grid, model.architecture) + complete_communication_and_compute_boundary!(model, model.grid, model.architecture, Δt) # Calculate contributions to momentum and tracer tendencies from user-prescribed fluxes across the # boundaries of the domain diff --git a/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl b/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl index ac655c480e..6e8988e262 100644 --- a/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl +++ b/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl @@ -45,7 +45,7 @@ function update_state!(model::NonhydrostaticModel, Δt, callbacks=[]; compute_te update_biogeochemical_state!(model.biogeochemistry, model) compute_tendencies && - @apply_regionally compute_tendencies!(model, callbacks) + @apply_regionally compute_tendencies!(model, Δt, callbacks) return nothing end diff --git a/src/Models/interleave_communication_and_computation.jl b/src/Models/interleave_communication_and_computation.jl index 2e36f54764..11a1e7cedf 100644 --- a/src/Models/interleave_communication_and_computation.jl +++ b/src/Models/interleave_communication_and_computation.jl @@ -6,7 +6,7 @@ using Oceananigans.DistributedComputations using Oceananigans.DistributedComputations: DistributedGrid using Oceananigans.DistributedComputations: synchronize_communication!, SynchronizedDistributed -function complete_communication_and_compute_boundary!(model, ::DistributedGrid, arch) +function complete_communication_and_compute_boundary!(model, ::DistributedGrid, arch, Δt) # Iterate over the fields to clear _ALL_ possible architectures for field in prognostic_fields(model) @@ -14,16 +14,16 @@ function complete_communication_and_compute_boundary!(model, ::DistributedGrid, end # Recompute tendencies near the boundary halos - compute_boundary_tendencies!(model) + compute_boundary_tendencies!(model, Δt) return nothing end # Fallback -complete_communication_and_compute_boundary!(model, ::DistributedGrid, ::SynchronizedDistributed) = nothing -complete_communication_and_compute_boundary!(model, grid, arch) = nothing +complete_communication_and_compute_boundary!(model, ::DistributedGrid, ::SynchronizedDistributed, Δt) = nothing +complete_communication_and_compute_boundary!(model, grid, arch, Δt) = nothing -compute_boundary_tendencies!(model) = nothing +compute_boundary_tendencies!(model, Δt) = nothing """ Kernel parameters for computing interior tendencies. """ interior_tendency_kernel_parameters(grid) = :xyz # fallback From d0caa7e56cde5cf27bf00cdc61c29ceeff07708b Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 14 Dec 2023 11:31:24 -0500 Subject: [PATCH 096/567] start from here --- .../compute_w_from_continuity.jl | 10 +++++----- .../moving_vertical_coordinate.jl | 7 +++++-- .../update_hydrostatic_free_surface_model_state.jl | 2 +- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl index 703d5233a8..e30ba78968 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl @@ -12,12 +12,12 @@ Compute the vertical velocity ``w`` by integrating the continuity equation from w^{n+1} = -∫ [∂/∂x (u^{n+1}) + ∂/∂y (v^{n+1})] dz ``` """ -compute_w_from_continuity!(model, Δt; kwargs...) = compute_w_from_continuity!(model.velocities, model.architecture, model.grid, Δt; kwargs...) +compute_w_from_continuity!(model; kwargs...) = compute_w_from_continuity!(model.velocities, model.architecture, model.grid; kwargs...) -compute_w_from_continuity!(velocities, arch, grid, Δt; parameters = w_kernel_parameters(grid)) = - launch!(arch, grid, parameters, _compute_w_from_continuity!, velocities, grid, Δt) +compute_w_from_continuity!(velocities, arch, grid; parameters = w_kernel_parameters(grid)) = + launch!(arch, grid, parameters, _compute_w_from_continuity!, velocities, grid) -@kernel function _compute_w_from_continuity!(U, grid, Δt) +@kernel function _compute_w_from_continuity!(U, grid) i, j = @index(Global, NTuple) U.w[i, j, 1] = 0 @@ -26,7 +26,7 @@ compute_w_from_continuity!(velocities, arch, grid, Δt; parameters = w_kernel_pa end end -@kernel function _compute_w_from_continuity!(U, grid::ZStarCoordinateGrid, Δt) +@kernel function _compute_w_from_continuity!(U, grid::ZStarCoordinateGrid) i, j = @index(Global, NTuple) U.w[i, j, 1] = 0 diff --git a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl index 3fd496456d..812efd7ddd 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl @@ -75,7 +75,7 @@ function update_vertical_coordinate!(model, grid::ZStarCoordinateGrid, Δt; para # Scaling scaling = grid.Δzᵃᵃᶠ.scaling - ∂t_scaling = grid.Δzᵃᵃᶠ.previous_scaling + ∂t_scaling = grid.Δzᵃᵃᶠ.∂t_scaling # Moving coordinates Δzᵃᵃᶠ = grid.Δzᵃᵃᶠ.star_value @@ -150,12 +150,15 @@ import Oceananigans.Operators: Δzᶜᶜᶠ, Δzᶜᶜᶜ, Δzᶜᶠᶠ, Δzᶜ import Oceananigans.Architectures: arch_array arch_array(arch, coord::ZStarCoordinate) = - ZStarCoordinate(arch_array(arch, coord.reference), coord.scaling, coord.star_value) + ZStarCoordinate(arch_array(arch, coord.reference), coord.star_value, coord.scaling, coord.∂t_scaling) # Adding the slope to the momentum-RHS @inline free_surface_slope_x(i, j, k, grid, args...) = nothing @inline free_surface_slope_y(i, j, k, grid, args...) = nothing +@inline free_surface_slope_x(i, j, k, grid::ZStarCoordinateGrid, free_surface, ::Nothing, model_fields) = zero(grid) +@inline free_surface_slope_y(i, j, k, grid::ZStarCoordinateGrid, free_surface, ::Nothing, model_fields) = zero(grid) + @inline η_times_zᶜᶜᶜ(i, j, k, grid, η) = @inbounds η[i, j, grid.Nz+1] * (1 + grid.zᵃᵃᶜ[k] / bottom(i, j, grid)) @inline η_slope_xᶠᶜᶜ(i, j, k, grid, free_surface) = @inbounds ∂xᶠᶜᶜ(i, j, k, grid, η_times_zᶜᶜᶜ, free_surface.η) diff --git a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl index e8277794e0..cc0589b6fe 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl @@ -76,7 +76,7 @@ function compute_auxiliaries!(model::HydrostaticFreeSurfaceModel; w_parameters = diffusivity = model.diffusivity_fields for (wpar, ppar, κpar) in zip(w_parameters, p_parameters, κ_parameters) - compute_w_from_continuity!(model, Δt; parameters = wpar) + compute_w_from_continuity!(model; parameters = wpar) compute_diffusivities!(diffusivity, closure, model; parameters = κpar) update_hydrostatic_pressure!(model.pressure.pHY′, architecture(grid), grid, model.buoyancy, model.tracers; From 742837468c4c0c6a164204b25b7d827fcde68fe7 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 18 Dec 2023 09:14:08 +0100 Subject: [PATCH 097/567] just for testing --- .../split_explicit_free_surface.jl | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index d8173550da..670c784bba 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -85,6 +85,20 @@ function FreeSurface(free_surface::SplitExplicitFreeSurface, velocities, grid) free_surface.settings) end +function SplitExplicitFreeSurface(grid; gravitational_acceleration = g_Earth, + settings = SplitExplicitSettings(eltype(grid); gravitational_acceleration, substeps = 200)) + + if eltype(settings) != eltype(grid) + @warn "Using $(eltype(settings)) settings for the SplitExplicitFreeSurface on a $(eltype(grid)) grid" + end + + η = ZFaceField(grid, indices = (:, :, size(grid, 3)+1)) + gravitational_acceleration = convert(eltype(grid), gravitational_acceleration) + + return SplitExplicitFreeSurface(η, SplitExplicitState(grid), SplitExplicitAuxiliaryFields(grid), + gravitational_acceleration, settings) +end + """ struct SplitExplicitState From b48d00c7d1a8bab7f7ac475c9dda08c11993c909 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 18 Dec 2023 09:16:55 +0100 Subject: [PATCH 098/567] removed useless particles --- .../inertial_particles.jl | 66 ------------------- 1 file changed, 66 deletions(-) delete mode 100644 validation/lagrangian_particles/inertial_particles.jl diff --git a/validation/lagrangian_particles/inertial_particles.jl b/validation/lagrangian_particles/inertial_particles.jl deleted file mode 100644 index e6bbfbe9d2..0000000000 --- a/validation/lagrangian_particles/inertial_particles.jl +++ /dev/null @@ -1,66 +0,0 @@ -using StructArrays -using Oceananigans -using Oceananigans: architecture -using Oceananigans.Models.LagrangianParticleTracking: AbstractParticle -using Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_w_from_continuity! -import Oceananigans.Models.LagrangianParticleTracking: particle_u_velocity, particle_v_velocity, particle_w_velocity - -struct InertialParticle{T} <: AbstractParticle - x :: T - y :: T - z :: T - u :: T - v :: T - w :: T - particle_respose_time :: T -end - -# 10 Particles with different inertia -x = ones(10) -y = ones(10) -z = ones(10) -u = zeros(10) -v = zeros(10) -w = zeros(10) - -particle_respose_time = range(0.1, 1.0, length = 10) - -properties = StructArray{InertialParticle}((x, y, z, u, v, w, particle_respose_time)) -particles = LagrangianParticles(properties) - -grid = RectilinearGrid(size = (50, 50, 50), x = (0, 2), y = (0, 2), z = (0, 2), topology = (Periodic, Periodic, Periodic)) - -u_fluid = XFaceField(grid) -v_fluid = YFaceField(grid) -w_fluid = ZFaceField(grid) - -@inline particles_u_velocity(u_fluid, particle, Δt) = particle.u + Δt / particles.particle_respose_time * (u_fluid - particle.u) -@inline particles_v_velocity(v_fluid, particle, Δt) = particle.v + Δt / particles.particle_respose_time * (v_fluid - particle.v) -@inline particles_w_velocity(w_fluid, particle, Δt) = particle.w + Δt / particles.particle_respose_time * (w_fluid - particle.w) - -set!(u_fluid, (x, y, z) -> rand()) -set!(v_fluid, (x, y, z) -> rand()) - -fill_halo_regions!((u_fluid, v_fluid)) - -compute_w_from_continuity!((; u = u_fluid, v = v_fluid, w = w_fluid), architecture(grid), grid) - -velocities = PrescribedVelocityFields(; u = u_fluid, v = v_fluid, w = w_fluid) - -model = HydrostaticFreeSurfaceModel(; grid, - tracers = (), - buoyancy = nothing, - particles, - velocities) - -simulation = Simulation(model, Δt = 1e-2, stop_time = 10) - -particles_save = [deepcopy(properties)] - -save_particles(sim) = - push!(particles_save, deepcopy(sim.model.particles.properties)) - -simulation.callbacks[:particles] = Callback(save_particles, IterationInterval(10)) - -run!(simulation) - From 4d36cc41bf5baf89fc4ee9cf33503de5dc8ea0c7 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 18 Dec 2023 09:17:37 +0100 Subject: [PATCH 099/567] removed bacthed stuff --- src/Solvers/batched_tridiagonal_solver.jl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Solvers/batched_tridiagonal_solver.jl b/src/Solvers/batched_tridiagonal_solver.jl index d69ad5ecfc..629638a98a 100644 --- a/src/Solvers/batched_tridiagonal_solver.jl +++ b/src/Solvers/batched_tridiagonal_solver.jl @@ -88,7 +88,7 @@ Reference implementation per Numerical Recipes, Press et al. 1992 (§ 2.4). Note a slightly different notation from Press et al. is used for indexing the off-diagonal elements; see [`BatchedTridiagonalSolver`](@ref). """ -function solve!(ϕ, solver::BatchedTridiagonalSolver, rhs, args...; only_active_cells = nothing) +function solve!(ϕ, solver::BatchedTridiagonalSolver, rhs, args...) launch_config = if solver.tridiagonal_direction isa XDirection :yz @@ -108,8 +108,7 @@ function solve!(ϕ, solver::BatchedTridiagonalSolver, rhs, args...; only_active_ solver.grid, solver.parameters, Tuple(args), - solver.tridiagonal_direction; - only_active_cells) + solver.tridiagonal_direction) return nothing end From 8842d055b2d263a7c72d75d9f0e6bdd7b7068bcc Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 18 Dec 2023 09:20:05 +0100 Subject: [PATCH 100/567] tracer advetion type --- src/Advection/tracer_advection_operators.jl | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/Advection/tracer_advection_operators.jl b/src/Advection/tracer_advection_operators.jl index f6bd834af5..82851cd598 100644 --- a/src/Advection/tracer_advection_operators.jl +++ b/src/Advection/tracer_advection_operators.jl @@ -1,22 +1,27 @@ using Oceananigans.Operators: Vᶜᶜᶜ using Oceananigans.Fields: ZeroField -struct ThreeDimensionalTracerAdvection{N, FT, A, B, C} <: AbstractAdvectionScheme{N, FT} +struct TracerAdvection{N, FT, A, B, C} <: AbstractAdvectionScheme{N, FT} x :: A y :: B z :: C - ThreeDimensionalTracerAdvection{N, FT}(x::A, y::B, z::C) where {N, FT, A, B, C} = new{N, FT, A, B, C}(x, y, z) + TracerAdvection{N, FT}(x::A, y::B, z::C) where {N, FT, A, B, C} = new{N, FT, A, B, C}(x, y, z) end -function ThreeDimensionalTracerAdvection(; x, y, z) +""" + function TracerAdvection(; x, y, z) + +builds a `TracerAdvection` type with different reconstructions in `x`, `y`, and `z` +""" +function TracerAdvection(; x, y, z) Nx = required_halo_size(x) Ny = required_halo_size(y) Nz = required_halo_size(z) FT = eltype(x) - return ThreeDimensionalTracerAdvection{max(Nx, Ny, Nz), FT}(x, y, z) + return TracerAdvection{max(Nx, Ny, Nz), FT}(x, y, z) end @inline _advective_tracer_flux_x(args...) = advective_tracer_flux_x(args...) @@ -51,7 +56,7 @@ which ends up at the location `ccc`. δzᵃᵃᶜ(i, j, k, grid, _advective_tracer_flux_z, advection, U.w, c)) end -@inline function div_Uc(i, j, k, grid, advection::ThreeDimensionalTracerAdvection, U, c) +@inline function div_Uc(i, j, k, grid, advection::TracerAdvection, U, c) return 1/Vᶜᶜᶜ(i, j, k, grid) * (δxᶜᵃᵃ(i, j, k, grid, _advective_tracer_flux_x, advection.x, U.u, c) + δyᵃᶜᵃ(i, j, k, grid, _advective_tracer_flux_y, advection.y, U.v, c) + δzᵃᵃᶜ(i, j, k, grid, _advective_tracer_flux_z, advection.z, U.w, c)) From c824b7d8b722e068d6bdbc3f155b5397039ab626 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 18 Dec 2023 11:39:56 +0100 Subject: [PATCH 101/567] well... this works! --- .../moving_vertical_coordinate.jl | 96 ++++++++++++++----- ...te_hydrostatic_free_surface_model_state.jl | 5 +- .../geostrophic_adjustment_test.jl | 2 +- .../moving_coordinates/z_star_coordinate.jl | 69 +++++++++++++ 4 files changed, 144 insertions(+), 28 deletions(-) create mode 100644 validation/moving_coordinates/z_star_coordinate.jl diff --git a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl index 812efd7ddd..740c302851 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl @@ -6,9 +6,20 @@ using Oceananigans.Grids: AbstractGrid, AbstractUnderlyingGrid, halo_size using Oceananigans.ImmersedBoundaries using Oceananigans.Utils: getnamewrapper using Adapt +using Printf +""" geopotential-following vertical coordinate """ struct ZCoordinate end +""" + ZStarCoordinate{R, S, Z} + +a _free surface following_ vertical coordinate system. +The fixed coordinate is stored in `reference`, while `star_value` +contains the actual free-surface-following z-coordinate +the `scaling` and `∂t_scaling` fields are the vertical derivative of +the vertical coordinate and it's time derivative +""" struct ZStarCoordinate{R, S, Z} reference :: R star_value :: Z @@ -18,11 +29,16 @@ end ZStarCoordinate() = ZStarCoordinate(nothing, nothing, nothing, nothing) +import Oceananigans.Grids: coordinate_summary + +coordinate_summary(Δ::ZStarCoordinate, name) = + @sprintf("Free-surface following with Δ%s=%s", name, prettysummary(Δ.reference)) + Adapt.adapt_structure(to, coord::ZStarCoordinate) = ZStarCoordinate(Adapt.adapt(to, coord.reference), + Adapt.adapt(to, coord.star_value), Adapt.adapt(to, coord.scaling), - Adapt.adapt(to, coord.scaling), - Adapt.adapt(to, coord.star_value)) + Adapt.adapt(to, coord.scaling)) const ZStarCoordinateRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:ZStarCoordinate} const ZStarCoordinateLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:ZStarCoordinate} @@ -41,12 +57,23 @@ function MovingCoordinateGrid(grid::ImmersedBoundaryGrid, ::ZStarCoordinate) return ImmersedBoundaryGrid(underlying_grid, grid.immersed_boundary; active_cells_map) end +# Replacing the z-coordinate with a moving vertical coordinate, defined by its reference spacing, +# the actual vertical spacing and a scaling function MovingCoordinateGrid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, ::ZStarCoordinate) where {FT, TX, TY, TZ} # Memory layout for Dz spacings is local in z ΔzF = ZFaceField(grid) ΔzC = CenterField(grid) - scaling = ZFaceField(grid, indices = (:, :, grid.Nz + 1)) - ∂t_scaling = ZFaceField(grid, indices = (:, :, grid.Nz + 1)) + scaling = ZFaceField(grid, indices = (:, :, grid.Nz+1)) + ∂t_scaling = ZFaceField(grid, indices = (:, :, grid.Nz+1)) + + # Initial "at-rest" conditions + launch!(architecture(grid), grid, :xy, _update_scaling!, + scaling, ∂t_scaling, ZeroField(grid), grid, 1) + + launch!(architecture(grid), grid, :xy,_update_z_star!, + ΔzF, ΔzC, grid.Δzᵃᵃᶠ, grid.Δzᵃᵃᶜ, scaling, Val(grid.Nz)) + + fill_halo_regions!((ΔzF, ΔzC, scaling, ∂t_scaling); only_local_halos = true) Δzᵃᵃᶠ = ZStarCoordinate(grid.Δzᵃᵃᶠ, ΔzF, scaling, ∂t_scaling) Δzᵃᵃᶜ = ZStarCoordinate(grid.Δzᵃᵃᶜ, ΔzC, scaling, ∂t_scaling) @@ -68,7 +95,7 @@ function MovingCoordinateGrid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, ::ZS end # Fallback -update_vertical_coordinate!(model, grid; kwargs...) = nothing +update_vertical_coordinate!(model, grid, Δt; kwargs...) = nothing function update_vertical_coordinate!(model, grid::ZStarCoordinateGrid, Δt; parameters = tuple(:xyz)) η = model.free_surface.η @@ -85,25 +112,38 @@ function update_vertical_coordinate!(model, grid::ZStarCoordinateGrid, Δt; para Δz₀ᵃᵃᶠ = grid.Δzᵃᵃᶠ.reference Δz₀ᵃᵃᶜ = grid.Δzᵃᵃᶜ.reference - # Update the scaling on the whole grid (from -H to N+H) - launch!(architecture(grid), grid, horizontal_parameters(grid), _update_scaling!, - scaling, ∂t_scaling, η, grid, Δt) - # Update vertical coordinate with available parameters for params in parameters - launch!(architecture(grid), grid, params, _update_z_star!, - Δzᵃᵃᶠ, Δzᵃᵃᶜ, Δz₀ᵃᵃᶠ, Δz₀ᵃᵃᶜ, scaling, grid.Nz) + # Update the scaling on the whole grid (from -H to N+H) + launch!(architecture(grid), grid, horizontal_parameters(params), _update_scaling!, + scaling, ∂t_scaling, η, grid, Δt) + + launch!(architecture(grid), grid, horizontal_parameters(params), _update_z_star!, + Δzᵃᵃᶠ, Δzᵃᵃᶜ, Δz₀ᵃᵃᶠ, Δz₀ᵃᵃᶜ, scaling, Val(grid.Nz)) end - fill_halo_regions!((Δzᵃᵃᶠ, Δzᵃᵃᶜ); only_local_halos = true) + fill_halo_regions!((Δzᵃᵃᶠ, Δzᵃᵃᶜ, scaling, ∂t_scaling); only_local_halos = true) return nothing end -function horizontal_parameters(grid) - halos = halo_size(grid)[1:2] - total_size = size(grid)[1:2] .+ 2 .* halos - return KernelParameters(total_size, .- halos) +horizontal_parameters(::Symbol) = :xy +horizontal_parameters(::KernelParameters{W, O}) where {W, O} = KernelParameters(W[1:2], O[1:2]) + +update_thickness_weighted_tracers!(tracers, grid) = nothing + +function update_thickness_weighted_tracers!(tracers, grid::ZStarCoordinateGrid) + arch = architecture(grid) + for tracer in tracers + launch!(arch, grid, :xyz, _update_tracers!, tracer, grid) + end + + return nothing +end + +@kernel function _update_tracers!(tracer, grid) + i, j, k = @index(Global, NTuple) + @inbounds tracer[i, j, k] /= grid.Δzᵃᵃᶠ.scaling[i, j, grid.Nz+1] end @kernel function _update_scaling!(scaling, ∂t_scaling, η, grid, Δt) @@ -119,16 +159,20 @@ end bottom_height(i, j, grid) = grid.Lz bottom_height(i, j, grid::ImmersedBoundaryGrid) = @inbounds - grid.immersed_boundary.bottom_height[i, j, 1] -@kernel function _update_z_star!(ΔzF, ΔzC, ΔzF₀, ΔzC₀, scaling, Nz) - i, j, k = @index(Global, NTuple) - @inbounds ΔzF[i, j, k] = scaling[i, j, Nz+1] * ΔzF₀[k] - @inbounds ΔzC[i, j, k] = scaling[i, j, Nz+1] * ΔzC₀[k] +@kernel function _update_z_star!(ΔzF, ΔzC, ΔzF₀, ΔzC₀, scaling, ::Val{Nz}) where Nz + i, j = @index(Global, NTuple) + @unroll for k in 1:Nz+1 + @inbounds ΔzF[i, j, k] = scaling[i, j, Nz+1] * ΔzF₀[k] + @inbounds ΔzC[i, j, k] = scaling[i, j, Nz+1] * ΔzC₀[k] + end end -@kernel function _update_z_star!(ΔzF, ΔzC, ΔzF₀::Number, ΔzC₀::Number, scaling, Nz) - i, j, k = @index(Global, NTuple) - @inbounds ΔzF[i, j, k] = scaling[i, j, Nz+1] * ΔzF₀ - @inbounds ΔzC[i, j, k] = scaling[i, j, Nz+1] * ΔzC₀ +@kernel function _update_z_star!(ΔzF, ΔzC, ΔzF₀::Number, ΔzC₀::Number, scaling, ::Val{Nz}) where Nz + i, j = @index(Global, NTuple) + @unroll for k in 1:Nz+1 + @inbounds ΔzF[i, j, k] = scaling[i, j, Nz+1] * ΔzF₀ + @inbounds ΔzC[i, j, k] = scaling[i, j, Nz+1] * ΔzC₀ + end end import Oceananigans.Operators: Δzᶜᶜᶠ, Δzᶜᶜᶜ, Δzᶜᶠᶠ, Δzᶜᶠᶜ, Δzᶠᶜᶠ, Δzᶠᶜᶜ, Δzᶠᶠᶠ, Δzᶠᶠᶜ @@ -153,8 +197,8 @@ arch_array(arch, coord::ZStarCoordinate) = ZStarCoordinate(arch_array(arch, coord.reference), coord.star_value, coord.scaling, coord.∂t_scaling) # Adding the slope to the momentum-RHS -@inline free_surface_slope_x(i, j, k, grid, args...) = nothing -@inline free_surface_slope_y(i, j, k, grid, args...) = nothing +@inline free_surface_slope_x(i, j, k, grid, args...) = zero(grid) +@inline free_surface_slope_y(i, j, k, grid, args...) = zero(grid) @inline free_surface_slope_x(i, j, k, grid::ZStarCoordinateGrid, free_surface, ::Nothing, model_fields) = zero(grid) @inline free_surface_slope_y(i, j, k, grid::ZStarCoordinateGrid, free_surface, ::Nothing, model_fields) = zero(grid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl index cc0589b6fe..f1623903bf 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl @@ -31,7 +31,10 @@ update_state!(model::HydrostaticFreeSurfaceModel, Δt, callbacks=[]; compute_ten function update_state!(model::HydrostaticFreeSurfaceModel, grid, Δt, callbacks; compute_tendencies = true) @apply_regionally mask_immersed_model_fields!(model, grid) - + + # Remove moving boundary from tracers + @apply_regionally update_thickness_weighted_tracers!(model.tracers, grid) + # Update possible FieldTimeSeries used in the model @apply_regionally update_model_field_time_series!(model, model.clock) diff --git a/validation/implicit_free_surface/geostrophic_adjustment_test.jl b/validation/implicit_free_surface/geostrophic_adjustment_test.jl index 7de77f6744..9383f35368 100644 --- a/validation/implicit_free_surface/geostrophic_adjustment_test.jl +++ b/validation/implicit_free_surface/geostrophic_adjustment_test.jl @@ -123,7 +123,7 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: averaging_shape_function splitexplicit_free_surface = SplitExplicitFreeSurface(substeps = 10, averaging_weighting_function = averaging_shape_function, timestepper = AdamsBashforth3Scheme()) -explicit_free_surface = ExplicitFreeSurface() +explicit_free_surface = ExplicitFreeSurface() topology_types = [(Bounded, Periodic, Bounded), (Periodic, Periodic, Bounded)] topology_types = [topology_types[2]] diff --git a/validation/moving_coordinates/z_star_coordinate.jl b/validation/moving_coordinates/z_star_coordinate.jl new file mode 100644 index 0000000000..5ea88e478a --- /dev/null +++ b/validation/moving_coordinates/z_star_coordinate.jl @@ -0,0 +1,69 @@ +using Oceananigans +using Oceananigans.Units +using Oceananigans.Utils: prettytime +using Oceananigans.Models.HydrostaticFreeSurfaceModels: ZStarCoordinate, ZCoordinate, ZStarCoordinateGrid +using Printf + +grid = RectilinearGrid(size = (300, 20), + x = (0, 100kilometers), + z = (-10, 0), + topology = (Periodic, Flat, Bounded)) + +model = HydrostaticFreeSurfaceModel(; grid, + vertical_coordinate = ZCoordinate(), + momentum_advection = WENO(), + tracer_advection = WENO(), + buoyancy = nothing, + tracers = (), + free_surface = SplitExplicitFreeSurface(; cfl = 0.5, grid)) + +ηᵢ(x, z) = exp(-(x - 50kilometers)^2 / (10kilometers)^2) + +set!(model, η = ηᵢ) + +gravity_wave_speed = sqrt(model.free_surface.gravitational_acceleration * grid.Lz) +barotropic_time_step = grid.Δxᶜᵃᵃ / gravity_wave_speed + +Δt = 0.5 * barotropic_time_step + +@info "the time step is $Δt" + +simulation = Simulation(model; Δt, stop_time = 10000Δt) + +field_outputs = if model.grid isa ZStarCoordinateGrid + merge(model.velocities, (; ΔzF = model.grid.Δzᵃᵃᶠ.star_value)) +else + model.velocities +end + +# simulation.output_writers[:free_surface] = JLD2OutputWriter(model, model.free_surface.η, +# overwrite_existing = true, +# schedule = IterationInterval(100), +# filename = "free_surface") + +simulation.output_writers[:other_variables] = JLD2OutputWriter(model, field_outputs, + overwrite_existing = true, + schedule = IterationInterval(100), + filename = "other_variables") + +function progress(sim) + w = interior(sim.model.velocities.w, :, :, sim.model.grid.Nz+1) + u = sim.model.velocities.u + + msg0 = @sprintf("Time: %s iteration %d ", prettytime(sim.model.clock.time), sim.model.clock.iteration) + msg1 = @sprintf("extrema w: %.2e %.2e ", maximum(w), minimum(w)) + msg2 = @sprintf("extrema u: %.2e %.2e ", maximum(u), minimum(u)) + if sim.model.grid isa ZStarCoordinateGrid + Δz = sim.model.grid.Δzᵃᵃᶠ.star_value + msg3 = @sprintf("extrema Δz: %.2e %.2e ", maximum(Δz), minimum(Δz)) + @info msg0 * msg1 * msg2 * msg3 + else + @info msg0 * msg1 * msg2 + end + + return nothing +end + +simulation.callbacks[:progress] = Callback(progress, IterationInterval(1)) + +run!(simulation) From 18da8d8a802aebd5f170e84646782a4355981a62 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 18 Dec 2023 11:56:23 +0100 Subject: [PATCH 102/567] It works! --- .../moving_vertical_coordinate.jl | 6 ++-- .../moving_coordinates/z_star_coordinate.jl | 28 +++++++++++-------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl index 740c302851..e24eeec1bf 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl @@ -203,13 +203,13 @@ arch_array(arch, coord::ZStarCoordinate) = @inline free_surface_slope_x(i, j, k, grid::ZStarCoordinateGrid, free_surface, ::Nothing, model_fields) = zero(grid) @inline free_surface_slope_y(i, j, k, grid::ZStarCoordinateGrid, free_surface, ::Nothing, model_fields) = zero(grid) -@inline η_times_zᶜᶜᶜ(i, j, k, grid, η) = @inbounds η[i, j, grid.Nz+1] * (1 + grid.zᵃᵃᶜ[k] / bottom(i, j, grid)) +@inline η_times_zᶜᶜᶜ(i, j, k, grid, η) = @inbounds η[i, j, grid.Nz+1] * (1 + grid.zᵃᵃᶜ[k] / bottom_height(i, j, grid)) @inline η_slope_xᶠᶜᶜ(i, j, k, grid, free_surface) = @inbounds ∂xᶠᶜᶜ(i, j, k, grid, η_times_zᶜᶜᶜ, free_surface.η) @inline η_slope_yᶜᶠᶜ(i, j, k, grid, free_surface) = @inbounds ∂yᶜᶠᶜ(i, j, k, grid, η_times_zᶜᶜᶜ, free_surface.η) @inline free_surface_slope_x(i, j, k, grid::ZStarCoordinateGrid, free_surface, buoyancy, model_fields) = - ℑxᶠᵃᵃ(i, j, k, grid, buoyancy_perturbationᶜᶜᶜ, buoyancy, model_fields) * η_slope_xᶠᶜᶜ(i, j, k, grid, free_surface) + ℑxᶠᵃᵃ(i, j, k, grid, buoyancy_perturbationᶜᶜᶜ, buoyancy.model, model_fields) * η_slope_xᶠᶜᶜ(i, j, k, grid, free_surface) @inline free_surface_slope_y(i, j, k, grid::ZStarCoordinateGrid, free_surface, buoyancy, model_fields) = - ℑyᵃᶠᵃ(i, j, k, grid, buoyancy_perturbationᶜᶜᶜ, buoyancy, model_fields) * η_slope_yᶜᶠᶜ(i, j, k, grid, free_surface) + ℑyᵃᶠᵃ(i, j, k, grid, buoyancy_perturbationᶜᶜᶜ, buoyancy.model, model_fields) * η_slope_yᶜᶠᶜ(i, j, k, grid, free_surface) diff --git a/validation/moving_coordinates/z_star_coordinate.jl b/validation/moving_coordinates/z_star_coordinate.jl index 5ea88e478a..76a5b04b53 100644 --- a/validation/moving_coordinates/z_star_coordinate.jl +++ b/validation/moving_coordinates/z_star_coordinate.jl @@ -10,16 +10,17 @@ grid = RectilinearGrid(size = (300, 20), topology = (Periodic, Flat, Bounded)) model = HydrostaticFreeSurfaceModel(; grid, - vertical_coordinate = ZCoordinate(), + vertical_coordinate = ZStarCoordinate(), momentum_advection = WENO(), tracer_advection = WENO(), - buoyancy = nothing, - tracers = (), + buoyancy = BuoyancyTracer(), + tracers = :b, free_surface = SplitExplicitFreeSurface(; cfl = 0.5, grid)) ηᵢ(x, z) = exp(-(x - 50kilometers)^2 / (10kilometers)^2) +bᵢ(x, z) = 1e-6 * z + ηᵢ(x, z) * 1e-8 -set!(model, η = ηᵢ) +set!(model, η = ηᵢ, b = bᵢ) gravity_wave_speed = sqrt(model.free_surface.gravitational_acceleration * grid.Lz) barotropic_time_step = grid.Δxᶜᵃᵃ / gravity_wave_speed @@ -31,20 +32,15 @@ barotropic_time_step = grid.Δxᶜᵃᵃ / gravity_wave_speed simulation = Simulation(model; Δt, stop_time = 10000Δt) field_outputs = if model.grid isa ZStarCoordinateGrid - merge(model.velocities, (; ΔzF = model.grid.Δzᵃᵃᶠ.star_value)) + merge(model.velocities, model.tracers, (; ΔzF = model.grid.Δzᵃᵃᶠ.star_value)) else - model.velocities + merge(model.velocities, model.tracers) end -# simulation.output_writers[:free_surface] = JLD2OutputWriter(model, model.free_surface.η, -# overwrite_existing = true, -# schedule = IterationInterval(100), -# filename = "free_surface") - simulation.output_writers[:other_variables] = JLD2OutputWriter(model, field_outputs, overwrite_existing = true, schedule = IterationInterval(100), - filename = "other_variables") + filename = "zstar_model") function progress(sim) w = interior(sim.model.velocities.w, :, :, sim.model.grid.Nz+1) @@ -67,3 +63,11 @@ end simulation.callbacks[:progress] = Callback(progress, IterationInterval(1)) run!(simulation) + +# Check conservation +b = FieldTimeSeries("zstar_model.jld2", "b") + +drift = [] +for t in 1:length(b.times) + push!(drift, sum(b[t] - b[1])) +end \ No newline at end of file From 6de2bc85c9bc275c57439d3d2d55ec736ade1418 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 18 Dec 2023 17:06:01 +0100 Subject: [PATCH 103/567] changes --- .../compute_w_from_continuity.jl | 6 +- .../hydrostatic_free_surface_ab2_step.jl | 7 +- ..._free_surface_tendency_kernel_functions.jl | 4 +- .../moving_vertical_coordinate.jl | 173 +++++++++++------- ...te_hydrostatic_free_surface_model_state.jl | 4 +- .../moving_coordinates/z_star_coordinate.jl | 44 +++-- 6 files changed, 148 insertions(+), 90 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl index e30ba78968..009d70f0c5 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl @@ -31,9 +31,9 @@ end U.w[i, j, 1] = 0 @unroll for k in 2:grid.Nz+1 - @inbounds U.w[i, j, k] = U.w[i, j, k-1] - - Δzᶜᶜᶜ(i, j, k-1, grid) * div_xyᶜᶜᶜ(i, j, k-1, grid, U.u, U.v) - - grid.Δzᵃᵃᶠ.∂t_scaling[i, j, grid.Nz+1] + @inbounds U.w[i, j, k] = U.w[i, j, k-1] - Δzᶜᶜᶜ(i, j, k-1, grid) * ( + div_xyᶜᶜᶜ(i, j, k-1, grid, U.u, U.v) + grid.Δzᵃᵃᶜ.∂t_s[i, j, grid.Nz+1] + ) end end diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl index 509b1fe0ea..336da2bb81 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl @@ -75,8 +75,7 @@ function ab2_step_tracers!(tracers, model, Δt, χ) tracer_field = tracers[tracer_name] closure = model.closure - launch!(model.architecture, model.grid, :xyz, - ab2_step_field!, tracer_field, Δt, χ, Gⁿ, G⁻) + ab2_step_tracer_field!(tracer_field, model.grid, Δt, χ, Gⁿ, G⁻) implicit_step!(tracer_field, model.timestepper.implicit_solver, @@ -90,3 +89,7 @@ function ab2_step_tracers!(tracers, model, Δt, χ) return nothing end +ab2_step_tracer_field!(tracer_field, grid, Δt, χ, Gⁿ, G⁻) = + launch!(architecture(grid), grid, :xyz, ab2_step_field!, tracer_field, Δt, χ, Gⁿ, G⁻) + + diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_tendency_kernel_functions.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_tendency_kernel_functions.jl index 967d6c5b35..10710c6318 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_tendency_kernel_functions.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_tendency_kernel_functions.jl @@ -44,7 +44,7 @@ implicitly during time-stepping. model_fields = merge(hydrostatic_fields(velocities, free_surface, tracers), auxiliary_fields) - return ( - U_dot_∇u(i, j, k, grid, advection, velocities) + return ( - U_dot_∇u(i, j, k, grid, advection, velocities) / scaling(i, j, k, grid) - explicit_barotropic_pressure_x_gradient(i, j, k, grid, free_surface) - x_f_cross_U(i, j, k, grid, coriolis, velocities) - ∂xᶠᶜᶜ(i, j, k, grid, hydrostatic_pressure_anomaly) @@ -84,7 +84,7 @@ implicitly during time-stepping. model_fields = merge(hydrostatic_fields(velocities, free_surface, tracers), auxiliary_fields) - return ( - U_dot_∇v(i, j, k, grid, advection, velocities) + return ( - U_dot_∇v(i, j, k, grid, advection, velocities) / scaling(i, j, k, grid) - explicit_barotropic_pressure_y_gradient(i, j, k, grid, free_surface) - y_f_cross_U(i, j, k, grid, coriolis, velocities) - ∂yᶜᶠᶜ(i, j, k, grid, hydrostatic_pressure_anomaly) diff --git a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl index e24eeec1bf..30b7c2e80e 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl @@ -21,24 +21,36 @@ the `scaling` and `∂t_scaling` fields are the vertical derivative of the vertical coordinate and it's time derivative """ struct ZStarCoordinate{R, S, Z} - reference :: R - star_value :: Z - scaling :: S - ∂t_scaling :: S + Δ★ :: R # Reference _non moving_ coordinate + Δ :: Z # moving vertical coordinate + s⁻ :: S # scaling term = ∂Δ★(Δ) at the start of the time step + sⁿ :: S # scaling term = ∂Δ★(Δ) at the end of the time step + ∂t_s :: S # Time derivative of the vertical coordinate scaling end -ZStarCoordinate() = ZStarCoordinate(nothing, nothing, nothing, nothing) +ZStarCoordinate() = ZStarCoordinate(nothing, nothing, nothing, nothing, nothing) import Oceananigans.Grids: coordinate_summary coordinate_summary(Δ::ZStarCoordinate, name) = - @sprintf("Free-surface following with Δ%s=%s", name, prettysummary(Δ.reference)) + @sprintf("Free-surface following with Δ%s=%s", name, prettysummary(Δ.Δz★)) Adapt.adapt_structure(to, coord::ZStarCoordinate) = - ZStarCoordinate(Adapt.adapt(to, coord.reference), - Adapt.adapt(to, coord.star_value), - Adapt.adapt(to, coord.scaling), - Adapt.adapt(to, coord.scaling)) + ZStarCoordinate(Adapt.adapt(to, coord.Δ★), + Adapt.adapt(to, coord.Δ), + Adapt.adapt(to, coord.s⁻), + Adapt.adapt(to, coord.sⁿ), + Adapt.adapt(to, coord.∂t_s)) + +import Oceananigans.Architectures: arch_array + +arch_array(arch, coord::ZStarCoordinate) = + ZStarCoordinate(arch_array(arch, coord.Δ★), + coord.Δ, + coord.s⁻, + coord.sⁿ, + coord.∂t_s) + const ZStarCoordinateRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:ZStarCoordinate} const ZStarCoordinateLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:ZStarCoordinate} @@ -60,23 +72,25 @@ end # Replacing the z-coordinate with a moving vertical coordinate, defined by its reference spacing, # the actual vertical spacing and a scaling function MovingCoordinateGrid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, ::ZStarCoordinate) where {FT, TX, TY, TZ} + # Memory layout for Dz spacings is local in z - ΔzF = ZFaceField(grid) - ΔzC = CenterField(grid) - scaling = ZFaceField(grid, indices = (:, :, grid.Nz+1)) - ∂t_scaling = ZFaceField(grid, indices = (:, :, grid.Nz+1)) + ΔzF = ZFaceField(grid) + ΔzC = CenterField(grid) + s⁻ = ZFaceField(grid, indices = (:, :, grid.Nz+1)) + sⁿ = ZFaceField(grid, indices = (:, :, grid.Nz+1)) + ∂t_s = ZFaceField(grid, indices = (:, :, grid.Nz+1)) # Initial "at-rest" conditions launch!(architecture(grid), grid, :xy, _update_scaling!, - scaling, ∂t_scaling, ZeroField(grid), grid, 1) + sⁿ, s⁻, ∂t_s, ZeroField(grid), grid, 1) launch!(architecture(grid), grid, :xy,_update_z_star!, - ΔzF, ΔzC, grid.Δzᵃᵃᶠ, grid.Δzᵃᵃᶜ, scaling, Val(grid.Nz)) + ΔzF, ΔzC, grid.Δzᵃᵃᶠ, grid.Δzᵃᵃᶜ, sⁿ, Val(grid.Nz)) - fill_halo_regions!((ΔzF, ΔzC, scaling, ∂t_scaling); only_local_halos = true) + fill_halo_regions!((ΔzF, ΔzC, s⁻, sⁿ, ∂t_s); only_local_halos = true) - Δzᵃᵃᶠ = ZStarCoordinate(grid.Δzᵃᵃᶠ, ΔzF, scaling, ∂t_scaling) - Δzᵃᵃᶜ = ZStarCoordinate(grid.Δzᵃᵃᶜ, ΔzC, scaling, ∂t_scaling) + Δzᵃᵃᶠ = ZStarCoordinate(grid.Δzᵃᵃᶠ, ΔzF, s⁻, sⁿ, ∂t_s) + Δzᵃᵃᶜ = ZStarCoordinate(grid.Δzᵃᵃᶜ, ΔzC, s⁻, sⁿ, ∂t_s) args = [] for prop in propertynames(grid) @@ -101,28 +115,28 @@ function update_vertical_coordinate!(model, grid::ZStarCoordinateGrid, Δt; para η = model.free_surface.η # Scaling - scaling = grid.Δzᵃᵃᶠ.scaling - ∂t_scaling = grid.Δzᵃᵃᶠ.∂t_scaling + s⁻ = grid.Δzᵃᵃᶠ.s⁻ + sⁿ = grid.Δzᵃᵃᶠ.sⁿ + ∂t_s = grid.Δzᵃᵃᶠ.∂t_s # Moving coordinates - Δzᵃᵃᶠ = grid.Δzᵃᵃᶠ.star_value - Δzᵃᵃᶜ = grid.Δzᵃᵃᶜ.star_value + Δzᵃᵃᶠ = grid.Δzᵃᵃᶠ.Δ + Δzᵃᵃᶜ = grid.Δzᵃᵃᶜ.Δ # Reference coordinates - Δz₀ᵃᵃᶠ = grid.Δzᵃᵃᶠ.reference - Δz₀ᵃᵃᶜ = grid.Δzᵃᵃᶜ.reference + Δz₀ᵃᵃᶠ = grid.Δzᵃᵃᶠ.Δ★ + Δz₀ᵃᵃᶜ = grid.Δzᵃᵃᶜ.Δ★ # Update vertical coordinate with available parameters for params in parameters - # Update the scaling on the whole grid (from -H to N+H) launch!(architecture(grid), grid, horizontal_parameters(params), _update_scaling!, - scaling, ∂t_scaling, η, grid, Δt) + sⁿ, s⁻, ∂t_s, η, grid, Δt) launch!(architecture(grid), grid, horizontal_parameters(params), _update_z_star!, - Δzᵃᵃᶠ, Δzᵃᵃᶜ, Δz₀ᵃᵃᶠ, Δz₀ᵃᵃᶜ, scaling, Val(grid.Nz)) + Δzᵃᵃᶠ, Δzᵃᵃᶜ, Δz₀ᵃᵃᶠ, Δz₀ᵃᵃᶜ, sⁿ, Val(grid.Nz)) end - fill_halo_regions!((Δzᵃᵃᶠ, Δzᵃᵃᶜ, scaling, ∂t_scaling); only_local_halos = true) + fill_halo_regions!((Δzᵃᵃᶠ, Δzᵃᵃᶜ, s⁻, sⁿ, ∂t_s); only_local_halos = true) return nothing end @@ -130,71 +144,74 @@ end horizontal_parameters(::Symbol) = :xy horizontal_parameters(::KernelParameters{W, O}) where {W, O} = KernelParameters(W[1:2], O[1:2]) -update_thickness_weighted_tracers!(tracers, grid) = nothing - -function update_thickness_weighted_tracers!(tracers, grid::ZStarCoordinateGrid) - arch = architecture(grid) - for tracer in tracers - launch!(arch, grid, :xyz, _update_tracers!, tracer, grid) - end - - return nothing -end - -@kernel function _update_tracers!(tracer, grid) - i, j, k = @index(Global, NTuple) - @inbounds tracer[i, j, k] /= grid.Δzᵃᵃᶠ.scaling[i, j, grid.Nz+1] -end - -@kernel function _update_scaling!(scaling, ∂t_scaling, η, grid, Δt) +@kernel function _update_scaling!(sⁿ, s⁻, ∂t_s, η, grid, Δt) i, j = @index(Global, NTuple) bottom = bottom_height(i, j, grid) @inbounds begin h = (bottom + η[i, j, grid.Nz+1]) / bottom - ∂t_scaling[i, j, grid.Nz+1] = (h - scaling[i, j, grid.Nz+1]) / Δt - scaling[i, j, grid.Nz+1] = h + + # update current and previous scaling + s⁻[i, j, grid.Nz+1] = sⁿ[i, j, grid.Nz+1] + sⁿ[i, j, grid.Nz+1] = h + + # Scaling derivative + ∂t_s[i, j, grid.Nz+1] = (h - s⁻[i, j, grid.Nz+1]) / Δt end end bottom_height(i, j, grid) = grid.Lz bottom_height(i, j, grid::ImmersedBoundaryGrid) = @inbounds - grid.immersed_boundary.bottom_height[i, j, 1] -@kernel function _update_z_star!(ΔzF, ΔzC, ΔzF₀, ΔzC₀, scaling, ::Val{Nz}) where Nz +@kernel function _update_z_star!(ΔzF, ΔzC, ΔzF₀, ΔzC₀, sⁿ, ::Val{Nz}) where Nz i, j = @index(Global, NTuple) @unroll for k in 1:Nz+1 - @inbounds ΔzF[i, j, k] = scaling[i, j, Nz+1] * ΔzF₀[k] - @inbounds ΔzC[i, j, k] = scaling[i, j, Nz+1] * ΔzC₀[k] + @inbounds ΔzF[i, j, k] = sⁿ[i, j, Nz+1] * ΔzF₀[k] + @inbounds ΔzC[i, j, k] = sⁿ[i, j, Nz+1] * ΔzC₀[k] end end -@kernel function _update_z_star!(ΔzF, ΔzC, ΔzF₀::Number, ΔzC₀::Number, scaling, ::Val{Nz}) where Nz +@kernel function _update_z_star!(ΔzF, ΔzC, ΔzF₀::Number, ΔzC₀::Number, sⁿ, ::Val{Nz}) where Nz i, j = @index(Global, NTuple) @unroll for k in 1:Nz+1 - @inbounds ΔzF[i, j, k] = scaling[i, j, Nz+1] * ΔzF₀ - @inbounds ΔzC[i, j, k] = scaling[i, j, Nz+1] * ΔzC₀ + @inbounds ΔzF[i, j, k] = sⁿ[i, j, Nz+1] * ΔzF₀ + @inbounds ΔzC[i, j, k] = sⁿ[i, j, Nz+1] * ΔzC₀ end end +@inline scaling(i, j, k, grid) = one(grid) +@inline scaling(i, j, k, grid::ZStarCoordinateGrid) = grid.Δzᵃᵃᶠ.sⁿ[i, j, grid.Nz+1] + + import Oceananigans.Operators: Δzᶜᶜᶠ, Δzᶜᶜᶜ, Δzᶜᶠᶠ, Δzᶜᶠᶜ, Δzᶠᶜᶠ, Δzᶠᶜᶜ, Δzᶠᶠᶠ, Δzᶠᶠᶜ +import Oceananigans.Operators: Vᶜᶜᶠ, Vᶜᶜᶜ, Vᶜᶠᶠ, Vᶜᶠᶜ, Vᶠᶜᶠ, Vᶠᶜᶜ, Vᶠᶠᶠ, Vᶠᶠᶜ # Very bad for GPU performance!!! (z-values are not coalesced in memory for z-derivatives anymore) # TODO: make z-direction local in memory by not using Fields -@inline Δzᶜᶜᶠ(i, j, k, grid::ZStarCoordinateGrid) = @inbounds grid.Δzᵃᵃᶠ.star_value[i, j, k] -@inline Δzᶜᶜᶜ(i, j, k, grid::ZStarCoordinateGrid) = @inbounds grid.Δzᵃᵃᶜ.star_value[i, j, k] +@inline Δzᶜᶜᶠ(i, j, k, grid::ZStarCoordinateGrid) = @inbounds grid.Δzᵃᵃᶠ.Δ[i, j, k] +@inline Δzᶜᶜᶜ(i, j, k, grid::ZStarCoordinateGrid) = @inbounds grid.Δzᵃᵃᶜ.Δ[i, j, k] -@inline Δzᶜᶠᶠ(i, j, k, grid::ZStarCoordinateGrid) = ℑyᵃᶠᵃ(i, j, k, grid, grid.Δzᵃᵃᶠ.star_value) -@inline Δzᶜᶠᶜ(i, j, k, grid::ZStarCoordinateGrid) = ℑyᵃᶠᵃ(i, j, k, grid, grid.Δzᵃᵃᶜ.star_value) +@inline Δzᶜᶠᶠ(i, j, k, grid::ZStarCoordinateGrid) = ℑyᵃᶠᵃ(i, j, k, grid, grid.Δzᵃᵃᶠ.Δ) +@inline Δzᶜᶠᶜ(i, j, k, grid::ZStarCoordinateGrid) = ℑyᵃᶠᵃ(i, j, k, grid, grid.Δzᵃᵃᶜ.Δ) -@inline Δzᶠᶜᶠ(i, j, k, grid::ZStarCoordinateGrid) = ℑxᶠᵃᵃ(i, j, k, grid, grid.Δzᵃᵃᶠ.star_value) -@inline Δzᶠᶜᶜ(i, j, k, grid::ZStarCoordinateGrid) = ℑxᶠᵃᵃ(i, j, k, grid, grid.Δzᵃᵃᶜ.star_value) +@inline Δzᶠᶜᶠ(i, j, k, grid::ZStarCoordinateGrid) = ℑxᶠᵃᵃ(i, j, k, grid, grid.Δzᵃᵃᶠ.Δ) +@inline Δzᶠᶜᶜ(i, j, k, grid::ZStarCoordinateGrid) = ℑxᶠᵃᵃ(i, j, k, grid, grid.Δzᵃᵃᶜ.Δ) -@inline Δzᶠᶠᶠ(i, j, k, grid::ZStarCoordinateGrid) = ℑxyᶠᶠᵃ(i, j, k, grid, grid.Δzᵃᵃᶠ.star_value) -@inline Δzᶠᶠᶜ(i, j, k, grid::ZStarCoordinateGrid) = ℑxyᶠᶠᵃ(i, j, k, grid, grid.Δzᵃᵃᶜ.star_value) +@inline Δzᶠᶠᶠ(i, j, k, grid::ZStarCoordinateGrid) = ℑxyᶠᶠᵃ(i, j, k, grid, grid.Δzᵃᵃᶠ.Δ) +@inline Δzᶠᶠᶜ(i, j, k, grid::ZStarCoordinateGrid) = ℑxyᶠᶠᵃ(i, j, k, grid, grid.Δzᵃᵃᶜ.Δ) -import Oceananigans.Architectures: arch_array +# Very bad for GPU performance!!! (z-values are not coalesced in memory for z-derivatives anymore) +# TODO: make z-direction local in memory by not using Fields +@inline Vᶜᶜᶠ(i, j, k, grid::ZStarCoordinateGrid) = Azᶜᶜᶠ(i, j, k, grid) * grid.Δzᵃᵃᶠ.Δ★ +@inline Vᶜᶜᶜ(i, j, k, grid::ZStarCoordinateGrid) = Azᶜᶜᶜ(i, j, k, grid) * grid.Δzᵃᵃᶜ.Δ★ -arch_array(arch, coord::ZStarCoordinate) = - ZStarCoordinate(arch_array(arch, coord.reference), coord.star_value, coord.scaling, coord.∂t_scaling) +@inline Vᶜᶠᶠ(i, j, k, grid::ZStarCoordinateGrid) = Azᶜᶠᶠ(i, j, k, grid) * grid.Δzᵃᵃᶠ.Δ★ +@inline Vᶜᶠᶜ(i, j, k, grid::ZStarCoordinateGrid) = Azᶜᶠᶜ(i, j, k, grid) * grid.Δzᵃᵃᶜ.Δ★ + +@inline Vᶠᶜᶠ(i, j, k, grid::ZStarCoordinateGrid) = Azᶠᶜᶠ(i, j, k, grid) * grid.Δzᵃᵃᶠ.Δ★ +@inline Vᶠᶜᶜ(i, j, k, grid::ZStarCoordinateGrid) = Azᶠᶜᶜ(i, j, k, grid) * grid.Δzᵃᵃᶜ.Δ★ + +@inline Vᶠᶠᶠ(i, j, k, grid::ZStarCoordinateGrid) = Azᶠᶠᶠ(i, j, k, grid) * grid.Δzᵃᵃᶠ.Δ★ +@inline Vᶠᶠᶜ(i, j, k, grid::ZStarCoordinateGrid) = Azᶠᶠᶜ(i, j, k, grid) * grid.Δzᵃᵃᶜ.Δ★ # Adding the slope to the momentum-RHS @inline free_surface_slope_x(i, j, k, grid, args...) = zero(grid) @@ -213,3 +230,29 @@ arch_array(arch, coord::ZStarCoordinate) = @inline free_surface_slope_y(i, j, k, grid::ZStarCoordinateGrid, free_surface, buoyancy, model_fields) = ℑyᵃᶠᵃ(i, j, k, grid, buoyancy_perturbationᶜᶜᶜ, buoyancy.model, model_fields) * η_slope_yᶜᶠᶜ(i, j, k, grid, free_surface) + +##### +##### In the Z-star coordinate framework the prognostic field is sθ! +##### + +@kernel function ab2_step_tracer_zstar!(θ, sⁿ, s⁻, Nz, Δt, χ, Gⁿ, G⁻) + i, j, k = @index(Global, NTuple) + + FT = eltype(χ) + one_point_five = convert(FT, 1.5) + oh_point_five = convert(FT, 0.5) + + @inbounds begin + ∂t_θ = (one_point_five + χ) * Gⁿ[i, j, k] - (oh_point_five + χ) * G⁻[i, j, k] + sθ = sⁿ[i, j, Nz+1] * θ[i, j, k] + convert(FT, Δt) * ∂t_θ + θ[i, j, k] = sθ / sⁿ[i, j, Nz+1] + end +end + +ab2_step_tracer_field!(tracer_field, grid::ZStarCoordinateGrid, Δt, χ, Gⁿ, G⁻) = + launch!(architecture(grid), grid, :xyz, ab2_step_tracer_zstar!, + tracer_field, + grid.Δzᵃᵃᶠ.sⁿ, + grid.Δzᵃᵃᶠ.s⁻, + grid.Nz, + Δt, χ, Gⁿ, G⁻) \ No newline at end of file diff --git a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl index f1623903bf..9559dacd72 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl @@ -32,9 +32,6 @@ function update_state!(model::HydrostaticFreeSurfaceModel, grid, Δt, callbacks; @apply_regionally mask_immersed_model_fields!(model, grid) - # Remove moving boundary from tracers - @apply_regionally update_thickness_weighted_tracers!(model.tracers, grid) - # Update possible FieldTimeSeries used in the model @apply_regionally update_model_field_time_series!(model, model.clock) @@ -92,3 +89,4 @@ end const RigidLidModel = HydrostaticFreeSurfaceModel{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Nothing} update_vertical_coordinate!(::RigidLidModel, ::ZStarCoordinateGrid; kwargs...) = nothing +update_tracer_thickness!(tracers, grid) = nothing diff --git a/validation/moving_coordinates/z_star_coordinate.jl b/validation/moving_coordinates/z_star_coordinate.jl index 76a5b04b53..2807ee2dd5 100644 --- a/validation/moving_coordinates/z_star_coordinate.jl +++ b/validation/moving_coordinates/z_star_coordinate.jl @@ -7,20 +7,20 @@ using Printf grid = RectilinearGrid(size = (300, 20), x = (0, 100kilometers), z = (-10, 0), - topology = (Periodic, Flat, Bounded)) + topology = (Bounded, Flat, Bounded)) model = HydrostaticFreeSurfaceModel(; grid, vertical_coordinate = ZStarCoordinate(), momentum_advection = WENO(), tracer_advection = WENO(), buoyancy = BuoyancyTracer(), - tracers = :b, - free_surface = SplitExplicitFreeSurface(; cfl = 0.5, grid)) + tracers = :b) + #free_surface = SplitExplicitFreeSurface(; cfl = 0.5, grid)) -ηᵢ(x, z) = exp(-(x - 50kilometers)^2 / (10kilometers)^2) -bᵢ(x, z) = 1e-6 * z + ηᵢ(x, z) * 1e-8 +# ηᵢ(x, z) = exp(-(x - 50kilometers)^2 / (10kilometers)^2) +bᵢ(x, z) = x > 50kilometers ? 0 : 1 -set!(model, η = ηᵢ, b = bᵢ) +set!(model, b = bᵢ) gravity_wave_speed = sqrt(model.free_surface.gravitational_acceleration * grid.Lz) barotropic_time_step = grid.Δxᶜᵃᵃ / gravity_wave_speed @@ -29,10 +29,10 @@ barotropic_time_step = grid.Δxᶜᵃᵃ / gravity_wave_speed @info "the time step is $Δt" -simulation = Simulation(model; Δt, stop_time = 10000Δt) +simulation = Simulation(model; Δt, stop_time = 10000Δt, stop_iteration = 10) field_outputs = if model.grid isa ZStarCoordinateGrid - merge(model.velocities, model.tracers, (; ΔzF = model.grid.Δzᵃᵃᶠ.star_value)) + merge(model.velocities, model.tracers, (; ΔzF = model.grid.Δzᵃᵃᶠ.Δ)) else merge(model.velocities, model.tracers) end @@ -50,7 +50,7 @@ function progress(sim) msg1 = @sprintf("extrema w: %.2e %.2e ", maximum(w), minimum(w)) msg2 = @sprintf("extrema u: %.2e %.2e ", maximum(u), minimum(u)) if sim.model.grid isa ZStarCoordinateGrid - Δz = sim.model.grid.Δzᵃᵃᶠ.star_value + Δz = sim.model.grid.Δzᵃᵃᶠ.Δ msg3 = @sprintf("extrema Δz: %.2e %.2e ", maximum(Δz), minimum(Δz)) @info msg0 * msg1 * msg2 * msg3 else @@ -60,14 +60,28 @@ function progress(sim) return nothing end -simulation.callbacks[:progress] = Callback(progress, IterationInterval(1)) - +simulation.callbacks[:progress] = Callback(progress, IterationInterval(100)) +simulation.callbacks[:wizard] = Callback(TimeStepWizard(; cfl = 0.2, max_change = 1.1), IterationInterval(10)) run!(simulation) # Check conservation -b = FieldTimeSeries("zstar_model.jld2", "b") +if model.grid isa ZStarCoordinateGrid + b = FieldTimeSeries("zstar_model.jld2", "b") + dz = FieldTimeSeries("zstar_model.jld2", "ΔzF") + + init = sum(b[1] * dz[1]) / sum(dz[1]) + drift = [] + for t in 1:length(b.times) + + push!(drift, sum(b[t] * dz[t]) / sum(dz[t]) - init) + end +else + b = FieldTimeSeries("zstar_model.jld2", "b") + + init = sum(b[1]) / prod(size(b[1])) + drift = [] + for t in 1:length(b.times) -drift = [] -for t in 1:length(b.times) - push!(drift, sum(b[t] - b[1])) + push!(drift, sum(b[t]) / prod(size(b[1])) - init) + end end \ No newline at end of file From fbbb0d5992e55bcf0bebddb7ea4c4fa1a32d4181 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 19 Dec 2023 10:30:27 +0100 Subject: [PATCH 104/567] at least it does not crash --- .../compute_w_from_continuity.jl | 3 +- .../moving_vertical_coordinate.jl | 29 ++++++------------- .../split_explicit_free_surface_kernels.jl | 19 ++++++++++++ .../moving_coordinates/z_star_coordinate.jl | 6 ++-- 4 files changed, 33 insertions(+), 24 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl index 009d70f0c5..4e8e5a183d 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl @@ -32,7 +32,8 @@ end U.w[i, j, 1] = 0 @unroll for k in 2:grid.Nz+1 @inbounds U.w[i, j, k] = U.w[i, j, k-1] - Δzᶜᶜᶜ(i, j, k-1, grid) * ( - div_xyᶜᶜᶜ(i, j, k-1, grid, U.u, U.v) + grid.Δzᵃᵃᶜ.∂t_s[i, j, grid.Nz+1] + div_xyᶜᶜᶜ(i, j, k-1, grid, U.u, U.v) + + grid.Δzᵃᵃᶜ.∂t_s[i, j, grid.Nz+1] / grid.Δzᵃᵃᶜ.sⁿ[i, j, grid.Nz+1] ) end end diff --git a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl index 30b7c2e80e..48097b801d 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl @@ -76,9 +76,9 @@ function MovingCoordinateGrid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, ::ZS # Memory layout for Dz spacings is local in z ΔzF = ZFaceField(grid) ΔzC = CenterField(grid) - s⁻ = ZFaceField(grid, indices = (:, :, grid.Nz+1)) - sⁿ = ZFaceField(grid, indices = (:, :, grid.Nz+1)) - ∂t_s = ZFaceField(grid, indices = (:, :, grid.Nz+1)) + s⁻ = ZFaceField(grid, indices = (:, :, grid.Nz+1)) + sⁿ = ZFaceField(grid, indices = (:, :, grid.Nz+1)) + ∂t_s = ZFaceField(grid, indices = (:, :, grid.Nz+1)) # Initial "at-rest" conditions launch!(architecture(grid), grid, :xy, _update_scaling!, @@ -129,9 +129,8 @@ function update_vertical_coordinate!(model, grid::ZStarCoordinateGrid, Δt; para # Update vertical coordinate with available parameters for params in parameters - launch!(architecture(grid), grid, horizontal_parameters(params), _update_scaling!, - sⁿ, s⁻, ∂t_s, η, grid, Δt) - + update_coordinate_scaling!(sⁿ, s⁻, ∂t_s, params, model.free_surface, grid, Δt) + launch!(architecture(grid), grid, horizontal_parameters(params), _update_z_star!, Δzᵃᵃᶠ, Δzᵃᵃᶜ, Δz₀ᵃᵃᶠ, Δz₀ᵃᵃᶜ, sⁿ, Val(grid.Nz)) end @@ -144,6 +143,10 @@ end horizontal_parameters(::Symbol) = :xy horizontal_parameters(::KernelParameters{W, O}) where {W, O} = KernelParameters(W[1:2], O[1:2]) +update_coordinate_scaling!(sⁿ, s⁻, ∂t_s, params, fs, grid, Δt) = + launch!(architecture(grid), grid, horizontal_parameters(params), _update_scaling!, + sⁿ, s⁻, ∂t_s, fs.η, grid, Δt) + @kernel function _update_scaling!(sⁿ, s⁻, ∂t_s, η, grid, Δt) i, j = @index(Global, NTuple) bottom = bottom_height(i, j, grid) @@ -199,20 +202,6 @@ import Oceananigans.Operators: Vᶜᶜᶠ, Vᶜᶜᶜ, Vᶜᶠᶠ, Vᶜᶠᶜ, V @inline Δzᶠᶠᶠ(i, j, k, grid::ZStarCoordinateGrid) = ℑxyᶠᶠᵃ(i, j, k, grid, grid.Δzᵃᵃᶠ.Δ) @inline Δzᶠᶠᶜ(i, j, k, grid::ZStarCoordinateGrid) = ℑxyᶠᶠᵃ(i, j, k, grid, grid.Δzᵃᵃᶜ.Δ) -# Very bad for GPU performance!!! (z-values are not coalesced in memory for z-derivatives anymore) -# TODO: make z-direction local in memory by not using Fields -@inline Vᶜᶜᶠ(i, j, k, grid::ZStarCoordinateGrid) = Azᶜᶜᶠ(i, j, k, grid) * grid.Δzᵃᵃᶠ.Δ★ -@inline Vᶜᶜᶜ(i, j, k, grid::ZStarCoordinateGrid) = Azᶜᶜᶜ(i, j, k, grid) * grid.Δzᵃᵃᶜ.Δ★ - -@inline Vᶜᶠᶠ(i, j, k, grid::ZStarCoordinateGrid) = Azᶜᶠᶠ(i, j, k, grid) * grid.Δzᵃᵃᶠ.Δ★ -@inline Vᶜᶠᶜ(i, j, k, grid::ZStarCoordinateGrid) = Azᶜᶠᶜ(i, j, k, grid) * grid.Δzᵃᵃᶜ.Δ★ - -@inline Vᶠᶜᶠ(i, j, k, grid::ZStarCoordinateGrid) = Azᶠᶜᶠ(i, j, k, grid) * grid.Δzᵃᵃᶠ.Δ★ -@inline Vᶠᶜᶜ(i, j, k, grid::ZStarCoordinateGrid) = Azᶠᶜᶜ(i, j, k, grid) * grid.Δzᵃᵃᶜ.Δ★ - -@inline Vᶠᶠᶠ(i, j, k, grid::ZStarCoordinateGrid) = Azᶠᶠᶠ(i, j, k, grid) * grid.Δzᵃᵃᶠ.Δ★ -@inline Vᶠᶠᶜ(i, j, k, grid::ZStarCoordinateGrid) = Azᶠᶠᶜ(i, j, k, grid) * grid.Δzᵃᵃᶜ.Δ★ - # Adding the slope to the momentum-RHS @inline free_surface_slope_x(i, j, k, grid, args...) = zero(grid) @inline free_surface_slope_y(i, j, k, grid, args...) = zero(grid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 6eee1f8a19..401d5ec645 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -391,3 +391,22 @@ setup_split_explicit_tendency!(auxiliary, grid, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) launch!(architecture(grid), grid, :xy, _compute_integrated_ab2_tendencies!, auxiliary.Gᵁ, auxiliary.Gⱽ, grid, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) wait_free_surface_communication!(free_surface, arch) = nothing + +update_coordinate_scaling!(sⁿ, s⁻, ∂t_s, params, fs::SplitExplicitFreeSurface, grid, Δt) = + launch!(architecture(grid), grid, horizontal_parameters(params), _update_split_explicit_scaling!, + sⁿ, s⁻, ∂t_s, fs.η, fs.state.U̅, fs.state.V̅, grid, Δt) + +@kernel function _update_split_explicit_scaling!(sⁿ, s⁻, ∂t_s, η, U̅, V̅, grid, Δt) + i, j = @index(Global, NTuple) + bottom = bottom_height(i, j, grid) + @inbounds begin + h = (bottom + η[i, j, grid.Nz+1]) / bottom + + # update current and previous scaling + s⁻[i, j, grid.Nz+1] = sⁿ[i, j, grid.Nz+1] + sⁿ[i, j, grid.Nz+1] = h + + # Scaling derivative + ∂t_s[i, j, grid.Nz+1] = - div_xyᶜᶜᶜ(i, j, 1, grid, U̅, V̅) / bottom + end +end diff --git a/validation/moving_coordinates/z_star_coordinate.jl b/validation/moving_coordinates/z_star_coordinate.jl index 2807ee2dd5..3f18cbf33e 100644 --- a/validation/moving_coordinates/z_star_coordinate.jl +++ b/validation/moving_coordinates/z_star_coordinate.jl @@ -14,8 +14,8 @@ model = HydrostaticFreeSurfaceModel(; grid, momentum_advection = WENO(), tracer_advection = WENO(), buoyancy = BuoyancyTracer(), - tracers = :b) - #free_surface = SplitExplicitFreeSurface(; cfl = 0.5, grid)) + tracers = :b, + free_surface = SplitExplicitFreeSurface(; substeps = 30)) # ηᵢ(x, z) = exp(-(x - 50kilometers)^2 / (10kilometers)^2) bᵢ(x, z) = x > 50kilometers ? 0 : 1 @@ -29,7 +29,7 @@ barotropic_time_step = grid.Δxᶜᵃᵃ / gravity_wave_speed @info "the time step is $Δt" -simulation = Simulation(model; Δt, stop_time = 10000Δt, stop_iteration = 10) +simulation = Simulation(model; Δt, stop_time = 10000Δt) #, stop_iteration = 10) field_outputs = if model.grid isa ZStarCoordinateGrid merge(model.velocities, model.tracers, (; ΔzF = model.grid.Δzᵃᵃᶠ.Δ)) From 6469529d61e5442a2edcd9dba7d674d3d632de61 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 21 Dec 2023 09:54:18 +0100 Subject: [PATCH 105/567] test it out --- .../moving_vertical_coordinate.jl | 70 ++++++++++--------- .../split_explicit_free_surface_kernels.jl | 4 +- .../moving_coordinates/z_star_coordinate.jl | 4 +- 3 files changed, 42 insertions(+), 36 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl index 48097b801d..097cab20dd 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl @@ -8,49 +8,55 @@ using Oceananigans.Utils: getnamewrapper using Adapt using Printf -""" geopotential-following vertical coordinate """ -struct ZCoordinate end """ - ZStarCoordinate{R, S, Z} + MovingVerticalCoordinate{R, S, Z} -a _free surface following_ vertical coordinate system. -The fixed coordinate is stored in `reference`, while `star_value` -contains the actual free-surface-following z-coordinate -the `scaling` and `∂t_scaling` fields are the vertical derivative of -the vertical coordinate and it's time derivative +spacings for a generalized vertical coordinate system. +The fixed coordinate is stored in `Δr`, while `Δ` contains the moving z-coordinate. +The `s⁻`, `sⁿ` and `∂t_s` fields are the vertical derivative of the vertical coordinate +at timestep `n-1` and `n` and it's time derivative. +`denomination` contains the """ -struct ZStarCoordinate{R, S, Z} - Δ★ :: R # Reference _non moving_ coordinate - Δ :: Z # moving vertical coordinate - s⁻ :: S # scaling term = ∂Δ★(Δ) at the start of the time step - sⁿ :: S # scaling term = ∂Δ★(Δ) at the end of the time step - ∂t_s :: S # Time derivative of the vertical coordinate scaling +struct MovingVerticalCoordinate{D, R, Z, S} + denomination :: D # The type of moving coordinate + Δr :: R # Reference _non moving_ vertical coordinate + Δ :: Z # moving vertical coordinate + s⁻ :: S # scaling term = ∂Δ/∂Δr at the start of the time step + sⁿ :: S # scaling term = ∂Δ/∂Δr at the end of the time step + ∂t_s :: S # Time derivative of the vertical coordinate scaling end -ZStarCoordinate() = ZStarCoordinate(nothing, nothing, nothing, nothing, nothing) +""" a _free-surface following_ vertical coordinate """ +struct ZStar end + +""" geopotential-following vertical coordinate """ +struct Z end import Oceananigans.Grids: coordinate_summary +const ZStarCoordinate = MovingVerticalCoordinate{<:ZStar} + coordinate_summary(Δ::ZStarCoordinate, name) = @sprintf("Free-surface following with Δ%s=%s", name, prettysummary(Δ.Δz★)) -Adapt.adapt_structure(to, coord::ZStarCoordinate) = - ZStarCoordinate(Adapt.adapt(to, coord.Δ★), - Adapt.adapt(to, coord.Δ), - Adapt.adapt(to, coord.s⁻), - Adapt.adapt(to, coord.sⁿ), - Adapt.adapt(to, coord.∂t_s)) +Adapt.adapt_structure(to, coord::MovingVerticalCoordinate) = + MovingVerticalCoordinate(nothing, + Adapt.adapt(to, coord.Δr), + Adapt.adapt(to, coord.Δ), + Adapt.adapt(to, coord.s⁻), + Adapt.adapt(to, coord.sⁿ), + Adapt.adapt(to, coord.∂t_s)) import Oceananigans.Architectures: arch_array -arch_array(arch, coord::ZStarCoordinate) = - ZStarCoordinate(arch_array(arch, coord.Δ★), - coord.Δ, - coord.s⁻, - coord.sⁿ, - coord.∂t_s) - +arch_array(arch, coord::MovingVerticalCoordinate) = + MovingVerticalCoordinate(coord.denomination, + arch_array(arch, coord.Δr), + coord.Δ, + coord.s⁻, + coord.sⁿ, + coord.∂t_s) const ZStarCoordinateRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:ZStarCoordinate} const ZStarCoordinateLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:ZStarCoordinate} @@ -62,7 +68,7 @@ const ZStarCoordinateGrid = Union{ZStarCoordinateUnderlyingGrid, ZStarCoordinate MovingCoordinateGrid(grid, coord) = grid -function MovingCoordinateGrid(grid::ImmersedBoundaryGrid, ::ZStarCoordinate) +function MovingCoordinateGrid(grid::ImmersedBoundaryGrid, ::ZStar) underlying_grid = MovingCoordinateGrid(grid.underlying_grid, ZStarCoordinate()) active_cells_map = !isnothing(grid.interior_active_cells) @@ -71,7 +77,7 @@ end # Replacing the z-coordinate with a moving vertical coordinate, defined by its reference spacing, # the actual vertical spacing and a scaling -function MovingCoordinateGrid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, ::ZStarCoordinate) where {FT, TX, TY, TZ} +function MovingCoordinateGrid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, ::ZStar) where {FT, TX, TY, TZ} # Memory layout for Dz spacings is local in z ΔzF = ZFaceField(grid) @@ -89,8 +95,8 @@ function MovingCoordinateGrid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, ::ZS fill_halo_regions!((ΔzF, ΔzC, s⁻, sⁿ, ∂t_s); only_local_halos = true) - Δzᵃᵃᶠ = ZStarCoordinate(grid.Δzᵃᵃᶠ, ΔzF, s⁻, sⁿ, ∂t_s) - Δzᵃᵃᶜ = ZStarCoordinate(grid.Δzᵃᵃᶜ, ΔzC, s⁻, sⁿ, ∂t_s) + Δzᵃᵃᶠ = MovingVerticalCoordinate(ZStar(), grid.Δzᵃᵃᶠ, ΔzF, s⁻, sⁿ, ∂t_s) + Δzᵃᵃᶜ = MovingVerticalCoordinate(ZStar(), grid.Δzᵃᵃᶜ, ΔzC, s⁻, sⁿ, ∂t_s) args = [] for prop in propertynames(grid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 401d5ec645..e690d6a6de 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -394,9 +394,9 @@ wait_free_surface_communication!(free_surface, arch) = nothing update_coordinate_scaling!(sⁿ, s⁻, ∂t_s, params, fs::SplitExplicitFreeSurface, grid, Δt) = launch!(architecture(grid), grid, horizontal_parameters(params), _update_split_explicit_scaling!, - sⁿ, s⁻, ∂t_s, fs.η, fs.state.U̅, fs.state.V̅, grid, Δt) + sⁿ, s⁻, ∂t_s, fs.η, fs.state.U̅, fs.state.V̅, grid) -@kernel function _update_split_explicit_scaling!(sⁿ, s⁻, ∂t_s, η, U̅, V̅, grid, Δt) +@kernel function _update_split_explicit_scaling!(sⁿ, s⁻, ∂t_s, η, U̅, V̅, grid) i, j = @index(Global, NTuple) bottom = bottom_height(i, j, grid) @inbounds begin diff --git a/validation/moving_coordinates/z_star_coordinate.jl b/validation/moving_coordinates/z_star_coordinate.jl index 3f18cbf33e..92d86e97ea 100644 --- a/validation/moving_coordinates/z_star_coordinate.jl +++ b/validation/moving_coordinates/z_star_coordinate.jl @@ -1,7 +1,7 @@ using Oceananigans using Oceananigans.Units using Oceananigans.Utils: prettytime -using Oceananigans.Models.HydrostaticFreeSurfaceModels: ZStarCoordinate, ZCoordinate, ZStarCoordinateGrid +using Oceananigans.Models.HydrostaticFreeSurfaceModels: ZStar, Z, ZStarCoordinateGrid using Printf grid = RectilinearGrid(size = (300, 20), @@ -10,7 +10,7 @@ grid = RectilinearGrid(size = (300, 20), topology = (Bounded, Flat, Bounded)) model = HydrostaticFreeSurfaceModel(; grid, - vertical_coordinate = ZStarCoordinate(), + vertical_coordinate = ZStar(), momentum_advection = WENO(), tracer_advection = WENO(), buoyancy = BuoyancyTracer(), From ef95f9c620c90e96c3cbf470c58f8a3f6a62a280 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 21 Dec 2023 10:20:11 +0100 Subject: [PATCH 106/567] changed to MovingVerticalCoordinate --- .../compute_w_from_continuity.jl | 17 ++---- ..._free_surface_tendency_kernel_functions.jl | 4 +- .../moving_vertical_coordinate.jl | 61 ++++++++++--------- 3 files changed, 38 insertions(+), 44 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl index 4e8e5a183d..3ebb200f02 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl @@ -22,21 +22,14 @@ compute_w_from_continuity!(velocities, arch, grid; parameters = w_kernel_paramet U.w[i, j, 1] = 0 @unroll for k in 2:grid.Nz+1 - @inbounds U.w[i, j, k] = U.w[i, j, k-1] - Δzᶜᶜᶜ(i, j, k-1, grid) * div_xyᶜᶜᶜ(i, j, k-1, grid, U.u, U.v) + @inbounds U.w[i, j, k] = U.w[i, j, k-1] - Δzᶜᶜᶜ(i, j, k-1, grid) * + ( div_xyᶜᶜᶜ(i, j, k-1, grid, U.u, U.v) + + metric_term(i, j, k-1, grid) ) end end -@kernel function _compute_w_from_continuity!(U, grid::ZStarCoordinateGrid) - i, j = @index(Global, NTuple) - - U.w[i, j, 1] = 0 - @unroll for k in 2:grid.Nz+1 - @inbounds U.w[i, j, k] = U.w[i, j, k-1] - Δzᶜᶜᶜ(i, j, k-1, grid) * ( - div_xyᶜᶜᶜ(i, j, k-1, grid, U.u, U.v) + - grid.Δzᵃᵃᶜ.∂t_s[i, j, grid.Nz+1] / grid.Δzᵃᵃᶜ.sⁿ[i, j, grid.Nz+1] - ) - end -end +metric_term(i, j, k, grid) = zero(grid) +metric_term(i, j, k, grid::MovingCoordinateGrid) = grid.Δzᵃᵃᶜ.∂t_s[i, j, grid.Nz+1] / grid.Δzᵃᵃᶜ.sⁿ[i, j, grid.Nz+1] ##### ##### Size and offsets for the w kernel diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_tendency_kernel_functions.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_tendency_kernel_functions.jl index 10710c6318..85b6ba7edb 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_tendency_kernel_functions.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_tendency_kernel_functions.jl @@ -48,7 +48,7 @@ implicitly during time-stepping. - explicit_barotropic_pressure_x_gradient(i, j, k, grid, free_surface) - x_f_cross_U(i, j, k, grid, coriolis, velocities) - ∂xᶠᶜᶜ(i, j, k, grid, hydrostatic_pressure_anomaly) - - free_surface_slope_x(i, j, k, grid, free_surface, buoyancy, model_fields) + - horizontal_surface_slope_x(i, j, k, grid, free_surface, buoyancy, model_fields) - ∂ⱼ_τ₁ⱼ(i, j, k, grid, closure, diffusivities, clock, model_fields, buoyancy) - immersed_∂ⱼ_τ₁ⱼ(i, j, k, grid, velocities, u_immersed_bc, closure, diffusivities, clock, model_fields) + forcings.u(i, j, k, grid, clock, hydrostatic_prognostic_fields(velocities, free_surface, tracers))) @@ -88,7 +88,7 @@ implicitly during time-stepping. - explicit_barotropic_pressure_y_gradient(i, j, k, grid, free_surface) - y_f_cross_U(i, j, k, grid, coriolis, velocities) - ∂yᶜᶠᶜ(i, j, k, grid, hydrostatic_pressure_anomaly) - - free_surface_slope_y(i, j, k, grid, free_surface, buoyancy, model_fields) + - horizontal_surface_slope_y(i, j, k, grid, free_surface, buoyancy, model_fields) - ∂ⱼ_τ₂ⱼ(i, j, k, grid, closure, diffusivities, clock, model_fields, buoyancy) - immersed_∂ⱼ_τ₂ⱼ(i, j, k, grid, velocities, v_immersed_bc, closure, diffusivities, clock, model_fields) + forcings.v(i, j, k, grid, clock, model_fields)) diff --git a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl index 097cab20dd..f5e5750b64 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl @@ -27,19 +27,8 @@ struct MovingVerticalCoordinate{D, R, Z, S} ∂t_s :: S # Time derivative of the vertical coordinate scaling end -""" a _free-surface following_ vertical coordinate """ -struct ZStar end - -""" geopotential-following vertical coordinate """ -struct Z end - import Oceananigans.Grids: coordinate_summary -const ZStarCoordinate = MovingVerticalCoordinate{<:ZStar} - -coordinate_summary(Δ::ZStarCoordinate, name) = - @sprintf("Free-surface following with Δ%s=%s", name, prettysummary(Δ.Δz★)) - Adapt.adapt_structure(to, coord::MovingVerticalCoordinate) = MovingVerticalCoordinate(nothing, Adapt.adapt(to, coord.Δr), @@ -58,13 +47,26 @@ arch_array(arch, coord::MovingVerticalCoordinate) = coord.sⁿ, coord.∂t_s) -const ZStarCoordinateRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:ZStarCoordinate} -const ZStarCoordinateLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:ZStarCoordinate} +const MovingCoordinateRG{D} = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:MovingVerticalCoordinate{D}} where D +const MovingCoordinateLLG{D} = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:MovingVerticalCoordinate{D}} where D + +const MovingCoordinateUnderlyingGrid{D} = Union{MovingCoordinateRG{D}, MovingCoordinateLLG{D}} where D +const MovingCoordinateImmersedGrid{D} = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:MovingCoordinateUnderlyingGrid{D}} where D -const ZStarCoordinateUnderlyingGrid = Union{ZStarCoordinateRG, ZStarCoordinateLLG} -const ZStarCoordinateImmersedGrid = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:ZStarCoordinateUnderlyingGrid} +const MovingCoordinateGrid{D} = Union{MovingCoordinateUnderlyingGrid{D}, MovingCoordinateImmersedGrid{D}} where D -const ZStarCoordinateGrid = Union{ZStarCoordinateUnderlyingGrid, ZStarCoordinateImmersedGrid} +""" a _free-surface following_ vertical coordinate """ +struct ZStar end + +""" geopotential-following vertical coordinate """ +struct Z end + +const ZStarCoordinate = MovingVerticalCoordinate{<:ZStar} + +coordinate_summary(Δ::ZStarCoordinate, name) = + @sprintf("Free-surface following with Δ%s=%s", name, prettysummary(Δ.Δz★)) + +const ZStarCoordinateGrid = MovingCoordinateGrid{<:ZStar} MovingCoordinateGrid(grid, coord) = grid @@ -126,22 +128,22 @@ function update_vertical_coordinate!(model, grid::ZStarCoordinateGrid, Δt; para ∂t_s = grid.Δzᵃᵃᶠ.∂t_s # Moving coordinates - Δzᵃᵃᶠ = grid.Δzᵃᵃᶠ.Δ - Δzᵃᵃᶜ = grid.Δzᵃᵃᶜ.Δ + ΔzF = grid.Δzᵃᵃᶠ.Δ + ΔzC = grid.Δzᵃᵃᶜ.Δ # Reference coordinates - Δz₀ᵃᵃᶠ = grid.Δzᵃᵃᶠ.Δ★ - Δz₀ᵃᵃᶜ = grid.Δzᵃᵃᶜ.Δ★ + Δz₀F = grid.Δzᵃᵃᶠ.Δr + Δz₀C = grid.Δzᵃᵃᶜ.Δr # Update vertical coordinate with available parameters for params in parameters update_coordinate_scaling!(sⁿ, s⁻, ∂t_s, params, model.free_surface, grid, Δt) launch!(architecture(grid), grid, horizontal_parameters(params), _update_z_star!, - Δzᵃᵃᶠ, Δzᵃᵃᶜ, Δz₀ᵃᵃᶠ, Δz₀ᵃᵃᶜ, sⁿ, Val(grid.Nz)) + ΔzF, ΔzC, Δz₀F, Δz₀C, sⁿ, Val(grid.Nz)) end - fill_halo_regions!((Δzᵃᵃᶠ, Δzᵃᵃᶜ, s⁻, sⁿ, ∂t_s); only_local_halos = true) + fill_halo_regions!((ΔzF, ΔzC, s⁻, sⁿ, ∂t_s); only_local_halos = true) return nothing end @@ -192,7 +194,6 @@ end import Oceananigans.Operators: Δzᶜᶜᶠ, Δzᶜᶜᶜ, Δzᶜᶠᶠ, Δzᶜᶠᶜ, Δzᶠᶜᶠ, Δzᶠᶜᶜ, Δzᶠᶠᶠ, Δzᶠᶠᶜ -import Oceananigans.Operators: Vᶜᶜᶠ, Vᶜᶜᶜ, Vᶜᶠᶠ, Vᶜᶠᶜ, Vᶠᶜᶠ, Vᶠᶜᶜ, Vᶠᶠᶠ, Vᶠᶠᶜ # Very bad for GPU performance!!! (z-values are not coalesced in memory for z-derivatives anymore) # TODO: make z-direction local in memory by not using Fields @@ -209,21 +210,21 @@ import Oceananigans.Operators: Vᶜᶜᶠ, Vᶜᶜᶜ, Vᶜᶠᶠ, Vᶜᶠᶜ, V @inline Δzᶠᶠᶜ(i, j, k, grid::ZStarCoordinateGrid) = ℑxyᶠᶠᵃ(i, j, k, grid, grid.Δzᵃᵃᶜ.Δ) # Adding the slope to the momentum-RHS -@inline free_surface_slope_x(i, j, k, grid, args...) = zero(grid) -@inline free_surface_slope_y(i, j, k, grid, args...) = zero(grid) +@inline horizontal_surface_slope_x(i, j, k, grid, args...) = zero(grid) +@inline horizontal_surface_slope_y(i, j, k, grid, args...) = zero(grid) -@inline free_surface_slope_x(i, j, k, grid::ZStarCoordinateGrid, free_surface, ::Nothing, model_fields) = zero(grid) -@inline free_surface_slope_y(i, j, k, grid::ZStarCoordinateGrid, free_surface, ::Nothing, model_fields) = zero(grid) +@inline horizontal_surface_slope_x(i, j, k, grid::ZStarCoordinateGrid, free_surface, ::Nothing, model_fields) = zero(grid) +@inline horizontal_surface_slope_y(i, j, k, grid::ZStarCoordinateGrid, free_surface, ::Nothing, model_fields) = zero(grid) @inline η_times_zᶜᶜᶜ(i, j, k, grid, η) = @inbounds η[i, j, grid.Nz+1] * (1 + grid.zᵃᵃᶜ[k] / bottom_height(i, j, grid)) @inline η_slope_xᶠᶜᶜ(i, j, k, grid, free_surface) = @inbounds ∂xᶠᶜᶜ(i, j, k, grid, η_times_zᶜᶜᶜ, free_surface.η) @inline η_slope_yᶜᶠᶜ(i, j, k, grid, free_surface) = @inbounds ∂yᶜᶠᶜ(i, j, k, grid, η_times_zᶜᶜᶜ, free_surface.η) -@inline free_surface_slope_x(i, j, k, grid::ZStarCoordinateGrid, free_surface, buoyancy, model_fields) = +@inline horizontal_surface_slope_x(i, j, k, grid::ZStarCoordinateGrid, free_surface, buoyancy, model_fields) = ℑxᶠᵃᵃ(i, j, k, grid, buoyancy_perturbationᶜᶜᶜ, buoyancy.model, model_fields) * η_slope_xᶠᶜᶜ(i, j, k, grid, free_surface) -@inline free_surface_slope_y(i, j, k, grid::ZStarCoordinateGrid, free_surface, buoyancy, model_fields) = +@inline horizontal_surface_slope_y(i, j, k, grid::ZStarCoordinateGrid, free_surface, buoyancy, model_fields) = ℑyᵃᶠᵃ(i, j, k, grid, buoyancy_perturbationᶜᶜᶜ, buoyancy.model, model_fields) * η_slope_yᶜᶠᶜ(i, j, k, grid, free_surface) ##### @@ -239,7 +240,7 @@ import Oceananigans.Operators: Vᶜᶜᶠ, Vᶜᶜᶜ, Vᶜᶠᶠ, Vᶜᶠᶜ, V @inbounds begin ∂t_θ = (one_point_five + χ) * Gⁿ[i, j, k] - (oh_point_five + χ) * G⁻[i, j, k] - sθ = sⁿ[i, j, Nz+1] * θ[i, j, k] + convert(FT, Δt) * ∂t_θ + sθ = s⁻[i, j, Nz+1] * θ[i, j, k] + convert(FT, Δt) * ∂t_θ θ[i, j, k] = sθ / sⁿ[i, j, Nz+1] end end From 9c5a907c48804f4bd592f5e8f62d044a20f71120 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 21 Dec 2023 10:47:16 +0100 Subject: [PATCH 107/567] should work --- src/Advection/vector_invariant_advection.jl | 5 +---- .../moving_vertical_coordinate.jl | 3 +-- .../split_explicit_free_surface_kernels.jl | 2 +- src/Models/NonhydrostaticModels/nonhydrostatic_model.jl | 2 +- src/Models/ShallowWaterModels/shallow_water_model.jl | 2 +- validation/moving_coordinates/z_star_coordinate.jl | 3 ++- 6 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/Advection/vector_invariant_advection.jl b/src/Advection/vector_invariant_advection.jl index a16af6f950..787303f5ce 100644 --- a/src/Advection/vector_invariant_advection.jl +++ b/src/Advection/vector_invariant_advection.jl @@ -167,10 +167,7 @@ Base.show(io::IO, a::VectorInvariant{N, FT}) where {N, FT} = ##### Convenience for WENO Vector Invariant ##### -# VectorInvariant{N, FT, M, Z (vorticity scheme), ZS, V (vertical scheme), K (kinetic energy gradient scheme) -const WENOVectorInvariant = VectorInvariant{<:Any, <:Any, <:Any, <:WENO, <:Any, <:WENO, <:WENO} - -nothing_to_default(user_value, default) = isnothing(user_value) ? default : user_value +nothing_to_default(user_value; default = nothing) = isnothing(user_value) ? default : user_value """ function WENOVectorInvariant(; upwinding = nothing, diff --git a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl index f5e5750b64..918244f1b2 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl @@ -240,8 +240,7 @@ import Oceananigans.Operators: Δzᶜᶜᶠ, Δzᶜᶜᶜ, Δzᶜᶠᶠ, Δzᶜ @inbounds begin ∂t_θ = (one_point_five + χ) * Gⁿ[i, j, k] - (oh_point_five + χ) * G⁻[i, j, k] - sθ = s⁻[i, j, Nz+1] * θ[i, j, k] + convert(FT, Δt) * ∂t_θ - θ[i, j, k] = sθ / sⁿ[i, j, Nz+1] + θ[i, j, k] = s⁻[i, j, Nz+1] * θ[i, j, k] / sⁿ[i, j, Nz+1] + convert(FT, Δt) * ∂t_θ end end diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index e690d6a6de..2621bdce0d 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -406,7 +406,7 @@ update_coordinate_scaling!(sⁿ, s⁻, ∂t_s, params, fs::SplitExplicitFreeSurf s⁻[i, j, grid.Nz+1] = sⁿ[i, j, grid.Nz+1] sⁿ[i, j, grid.Nz+1] = h - # Scaling derivative + # ∂(H + η / H)/∂t = - ∇ ⋅ ∫udz / H ∂t_s[i, j, grid.Nz+1] = - div_xyᶜᶜᶜ(i, j, 1, grid, U̅, V̅) / bottom end end diff --git a/src/Models/NonhydrostaticModels/nonhydrostatic_model.jl b/src/Models/NonhydrostaticModels/nonhydrostatic_model.jl index 8e9a1102de..ddf212c160 100644 --- a/src/Models/NonhydrostaticModels/nonhydrostatic_model.jl +++ b/src/Models/NonhydrostaticModels/nonhydrostatic_model.jl @@ -197,7 +197,7 @@ function NonhydrostaticModel(; grid, pressures, diffusivity_fields, timestepper, pressure_solver, immersed_boundary, auxiliary_fields) - update_state!(model, Δt) + update_state!(model, 1) return model end diff --git a/src/Models/ShallowWaterModels/shallow_water_model.jl b/src/Models/ShallowWaterModels/shallow_water_model.jl index 73c7f8f2e7..e905ecd929 100644 --- a/src/Models/ShallowWaterModels/shallow_water_model.jl +++ b/src/Models/ShallowWaterModels/shallow_water_model.jl @@ -205,7 +205,7 @@ function ShallowWaterModel(; timestepper, formulation) - update_state!(model, 1.0) + update_state!(model, 1) return model end diff --git a/validation/moving_coordinates/z_star_coordinate.jl b/validation/moving_coordinates/z_star_coordinate.jl index 92d86e97ea..4fb7cb7eb3 100644 --- a/validation/moving_coordinates/z_star_coordinate.jl +++ b/validation/moving_coordinates/z_star_coordinate.jl @@ -1,6 +1,7 @@ using Oceananigans using Oceananigans.Units using Oceananigans.Utils: prettytime +using Oceananigans.Advection: WENOVectorInvariant using Oceananigans.Models.HydrostaticFreeSurfaceModels: ZStar, Z, ZStarCoordinateGrid using Printf @@ -11,7 +12,7 @@ grid = RectilinearGrid(size = (300, 20), model = HydrostaticFreeSurfaceModel(; grid, vertical_coordinate = ZStar(), - momentum_advection = WENO(), + momentum_advection = WENOVectorInvariant(), tracer_advection = WENO(), buoyancy = BuoyancyTracer(), tracers = :b, From 1141671f4b6ec455dba9727b54f765e7ab81db4b Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 21 Dec 2023 11:03:11 +0100 Subject: [PATCH 108/567] change name --- .../moving_vertical_coordinate.jl | 6 ++---- .../{z_star_coordinate.jl => z_star_lock_release.jl} | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) rename validation/moving_coordinates/{z_star_coordinate.jl => z_star_lock_release.jl} (96%) diff --git a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl index 918244f1b2..9c1e9cc979 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl @@ -8,7 +8,6 @@ using Oceananigans.Utils: getnamewrapper using Adapt using Printf - """ MovingVerticalCoordinate{R, S, Z} @@ -27,7 +26,6 @@ struct MovingVerticalCoordinate{D, R, Z, S} ∂t_s :: S # Time derivative of the vertical coordinate scaling end -import Oceananigans.Grids: coordinate_summary Adapt.adapt_structure(to, coord::MovingVerticalCoordinate) = MovingVerticalCoordinate(nothing, @@ -63,8 +61,8 @@ struct Z end const ZStarCoordinate = MovingVerticalCoordinate{<:ZStar} -coordinate_summary(Δ::ZStarCoordinate, name) = - @sprintf("Free-surface following with Δ%s=%s", name, prettysummary(Δ.Δz★)) +Grids.coordinate_summary(Δ::ZStarCoordinate, name) = + @sprintf("Free-surface following with Δ%s=%s", name, prettysummary(Δ.Δr)) const ZStarCoordinateGrid = MovingCoordinateGrid{<:ZStar} diff --git a/validation/moving_coordinates/z_star_coordinate.jl b/validation/moving_coordinates/z_star_lock_release.jl similarity index 96% rename from validation/moving_coordinates/z_star_coordinate.jl rename to validation/moving_coordinates/z_star_lock_release.jl index 4fb7cb7eb3..00b0a07913 100644 --- a/validation/moving_coordinates/z_star_coordinate.jl +++ b/validation/moving_coordinates/z_star_lock_release.jl @@ -11,8 +11,8 @@ grid = RectilinearGrid(size = (300, 20), topology = (Bounded, Flat, Bounded)) model = HydrostaticFreeSurfaceModel(; grid, - vertical_coordinate = ZStar(), - momentum_advection = WENOVectorInvariant(), + vertical_coordinate = Z(), + momentum_advection = VectorInvariant(), tracer_advection = WENO(), buoyancy = BuoyancyTracer(), tracers = :b, From bda362859217bb10253d595a56a6f282a412e5aa Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 21 Dec 2023 11:04:22 +0100 Subject: [PATCH 109/567] bugfix --- .../hydrostatic_free_surface_model.jl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl index f17857698c..75a109b7cf 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl @@ -65,7 +65,7 @@ end velocities = nothing, pressure = nothing, diffusivity_fields = nothing, - vertical_coordinate = ZCoordinate(), + vertical_coordinate = Z(), auxiliary_fields = NamedTuple(), ) @@ -111,7 +111,7 @@ function HydrostaticFreeSurfaceModel(; grid, velocities = nothing, pressure = nothing, diffusivity_fields = nothing, - vertical_coordinate = ZCoordinate(), + vertical_coordinate = Z(), auxiliary_fields = NamedTuple() ) @@ -120,7 +120,6 @@ function HydrostaticFreeSurfaceModel(; grid, # Introduce z-star coordinates if needed (only is free_surface is not a nothing) grid = !isnothing(free_surface) ? MovingCoordinateGrid(grid, vertical_coordinate) : grid - arch = architecture(grid) @apply_regionally momentum_advection = validate_momentum_advection(momentum_advection, grid) From 5476d346c7c76fc188362fa39811385f487763a1 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 21 Dec 2023 11:42:37 +0100 Subject: [PATCH 110/567] chnage to advection --- .../vector_invariant_self_upwinding.jl | 7 +++++-- .../compute_w_from_continuity.jl | 2 +- .../moving_vertical_coordinate.jl | 17 ++++++++++++++--- .../split_explicit_free_surface_kernels.jl | 2 +- .../moving_coordinates/z_star_lock_release.jl | 7 ++++--- 5 files changed, 25 insertions(+), 10 deletions(-) diff --git a/src/Advection/vector_invariant_self_upwinding.jl b/src/Advection/vector_invariant_self_upwinding.jl index d8f798006e..a310ae02b7 100644 --- a/src/Advection/vector_invariant_self_upwinding.jl +++ b/src/Advection/vector_invariant_self_upwinding.jl @@ -5,6 +5,9 @@ @inline δx_U(i, j, k, grid, u, v) = δxᶜᵃᵃ(i, j, k, grid, Ax_qᶠᶜᶜ, u) @inline δy_V(i, j, k, grid, u, v) = δyᵃᶜᵃ(i, j, k, grid, Ay_qᶜᶠᶜ, v) +@inline δx_U_plus_metric(i, j, k, grid, u, v) = δxᶜᵃᵃ(i, j, k, grid, Ax_qᶠᶜᶜ, u) +@inline δy_V_plus_metric(i, j, k, grid, u, v) = δyᵃᶜᵃ(i, j, k, grid, Ay_qᶜᶠᶜ, v) + # Velocity smoothness for divergence upwinding @inline U_smoothness(i, j, k, grid, u, v) = ℑxᶜᵃᵃ(i, j, k, grid, Ax_qᶠᶜᶜ, u) @inline V_smoothness(i, j, k, grid, u, v) = ℑyᵃᶜᵃ(i, j, k, grid, Ay_qᶜᶠᶜ, v) @@ -18,7 +21,7 @@ cross_scheme = scheme.upwinding.cross_scheme @inbounds û = u[i, j, k] - δvˢ = _symmetric_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, cross_scheme, δy_V, u, v) + δvˢ = _symmetric_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, cross_scheme, δy_V_plus_metric, u, v) δuᴸ = _left_biased_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, δx_U, δU_stencil, u, v) δuᴿ = _right_biased_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, δx_U, δU_stencil, u, v) @@ -31,7 +34,7 @@ end cross_scheme = scheme.upwinding.cross_scheme @inbounds v̂ = v[i, j, k] - δuˢ = _symmetric_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, cross_scheme, δx_U, u, v) + δuˢ = _symmetric_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, cross_scheme, δy_V_plus_metric, u, v) δvᴸ = _left_biased_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, δy_V, δV_stencil, u, v) δvᴿ = _right_biased_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, δy_V, δV_stencil, u, v) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl index 3ebb200f02..60d9b40338 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl @@ -29,7 +29,7 @@ compute_w_from_continuity!(velocities, arch, grid; parameters = w_kernel_paramet end metric_term(i, j, k, grid) = zero(grid) -metric_term(i, j, k, grid::MovingCoordinateGrid) = grid.Δzᵃᵃᶜ.∂t_s[i, j, grid.Nz+1] / grid.Δzᵃᵃᶜ.sⁿ[i, j, grid.Nz+1] +metric_term(i, j, k, grid::MovingCoordinateGrid) = grid.Δzᵃᵃᶜ.∂t_s[i, j, grid.Nz+1] ##### ##### Size and offsets for the w kernel diff --git a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl index 9c1e9cc979..6f4af60e95 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl @@ -23,7 +23,7 @@ struct MovingVerticalCoordinate{D, R, Z, S} Δ :: Z # moving vertical coordinate s⁻ :: S # scaling term = ∂Δ/∂Δr at the start of the time step sⁿ :: S # scaling term = ∂Δ/∂Δr at the end of the time step - ∂t_s :: S # Time derivative of the vertical coordinate scaling + ∂t_s :: S # Time derivative of the vertical coordinate scaling divided by the scaling end @@ -164,7 +164,7 @@ update_coordinate_scaling!(sⁿ, s⁻, ∂t_s, params, fs, grid, Δt) = sⁿ[i, j, grid.Nz+1] = h # Scaling derivative - ∂t_s[i, j, grid.Nz+1] = (h - s⁻[i, j, grid.Nz+1]) / Δt + ∂t_s[i, j, grid.Nz+1] = (h - s⁻[i, j, grid.Nz+1]) / Δt / h end end @@ -248,4 +248,15 @@ ab2_step_tracer_field!(tracer_field, grid::ZStarCoordinateGrid, Δt, χ, Gⁿ, G grid.Δzᵃᵃᶠ.sⁿ, grid.Δzᵃᵃᶠ.s⁻, grid.Nz, - Δt, χ, Gⁿ, G⁻) \ No newline at end of file + Δt, χ, Gⁿ, G⁻) + +# When performing divergence upwinding we must include the +# metric term + +import Oceananigans.Advection: δx_U_plus_metric, δy_V_plus_metric + +@inline δx_U_plus_metric(i, j, k, grid::ZStarCoordinateGrid, u, v) = + @inbounds δxᶜᵃᵃ(i, j, k, grid, Ax_qᶠᶜᶜ, u) + grid.Δzᵃᵃᶜ.∂t_s[i, j, grid.Nz+1] * Vᶜᶜᶜ(i, j, k, grid) + +@inline δy_V_plus_metric(i, j, k, grid::ZStarCoordinateGrid, u, v) = + @inbounds δyᵃᶜᵃ(i, j, k, grid, Ay_qᶜᶠᶜ, v) + grid.Δzᵃᵃᶜ.∂t_s[i, j, grid.Nz+1] * Vᶜᶜᶜ(i, j, k, grid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 2621bdce0d..db2f692657 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -407,6 +407,6 @@ update_coordinate_scaling!(sⁿ, s⁻, ∂t_s, params, fs::SplitExplicitFreeSurf sⁿ[i, j, grid.Nz+1] = h # ∂(H + η / H)/∂t = - ∇ ⋅ ∫udz / H - ∂t_s[i, j, grid.Nz+1] = - div_xyᶜᶜᶜ(i, j, 1, grid, U̅, V̅) / bottom + ∂t_s[i, j, grid.Nz+1] = - div_xyᶜᶜᶜ(i, j, 1, grid, U̅, V̅) / bottom / h end end diff --git a/validation/moving_coordinates/z_star_lock_release.jl b/validation/moving_coordinates/z_star_lock_release.jl index 00b0a07913..6d007b49a6 100644 --- a/validation/moving_coordinates/z_star_lock_release.jl +++ b/validation/moving_coordinates/z_star_lock_release.jl @@ -8,12 +8,13 @@ using Printf grid = RectilinearGrid(size = (300, 20), x = (0, 100kilometers), z = (-10, 0), + halo = (6, 6), topology = (Bounded, Flat, Bounded)) model = HydrostaticFreeSurfaceModel(; grid, - vertical_coordinate = Z(), - momentum_advection = VectorInvariant(), - tracer_advection = WENO(), + vertical_coordinate = ZStar(), + momentum_advection = WENOVectorInvariant(), + tracer_advection = WENO(order = 9), buoyancy = BuoyancyTracer(), tracers = :b, free_surface = SplitExplicitFreeSurface(; substeps = 30)) From 12240fe1ec5aced8f0c5f5c871960ef34636e943 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 21 Dec 2023 11:54:08 +0100 Subject: [PATCH 111/567] fixing some numerics --- src/Advection/vector_invariant_cross_upwinding.jl | 4 ++-- src/Advection/vector_invariant_self_upwinding.jl | 11 +++++++---- .../compute_w_from_continuity.jl | 5 +---- .../moving_vertical_coordinate.jl | 15 +++++---------- 4 files changed, 15 insertions(+), 20 deletions(-) diff --git a/src/Advection/vector_invariant_cross_upwinding.jl b/src/Advection/vector_invariant_cross_upwinding.jl index 8ae27b8306..8241d77d07 100644 --- a/src/Advection/vector_invariant_cross_upwinding.jl +++ b/src/Advection/vector_invariant_cross_upwinding.jl @@ -24,7 +24,7 @@ δᴸ = _left_biased_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, flux_div_xyᶜᶜᶜ, δ_stencil, u, v) δᴿ = _right_biased_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, flux_div_xyᶜᶜᶜ, δ_stencil, u, v) - return upwind_biased_product(û, δᴸ, δᴿ) + return upwind_biased_product(û, δᴸ, δᴿ) + û * ℑxᶠᵃᵃ(i, j, k, grid, metric_term) end @inline function upwinded_divergence_flux_Vᶜᶠᶜ(i, j, k, grid, scheme::VectorInvariantCrossVerticalUpwinding, u, v) @@ -34,5 +34,5 @@ end δᴸ = _left_biased_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, flux_div_xyᶜᶜᶜ, δ_stencil, u, v) δᴿ = _right_biased_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, flux_div_xyᶜᶜᶜ, δ_stencil, u, v) - return upwind_biased_product(v̂, δᴸ, δᴿ) + return upwind_biased_product(v̂, δᴸ, δᴿ) + v̂ * ℑyᵃᶠᵃ(i, j, k, grid, metric_term) end diff --git a/src/Advection/vector_invariant_self_upwinding.jl b/src/Advection/vector_invariant_self_upwinding.jl index a310ae02b7..0599735b6f 100644 --- a/src/Advection/vector_invariant_self_upwinding.jl +++ b/src/Advection/vector_invariant_self_upwinding.jl @@ -15,17 +15,20 @@ # Divergence smoothness for divergence upwinding @inline divergence_smoothness(i, j, k, grid, u, v) = δx_U(i, j, k, grid, u, v) + δy_V(i, j, k, grid, u, v) +# Metric term for moving grids +metric_term(i, j, k, grid) = zero(grid) + @inline function upwinded_divergence_flux_Uᶠᶜᶜ(i, j, k, grid, scheme::VectorInvariantSelfVerticalUpwinding, u, v) δU_stencil = scheme.upwinding.δU_stencil cross_scheme = scheme.upwinding.cross_scheme @inbounds û = u[i, j, k] - δvˢ = _symmetric_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, cross_scheme, δy_V_plus_metric, u, v) + δvˢ = _symmetric_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, cross_scheme, δy_V, u, v) δuᴸ = _left_biased_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, δx_U, δU_stencil, u, v) δuᴿ = _right_biased_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, δx_U, δU_stencil, u, v) - return upwind_biased_product(û, δuᴸ, δuᴿ) + û * δvˢ + return upwind_biased_product(û, δuᴸ, δuᴿ) + û * (δvˢ + ℑxᶠᵃᵃ(i, j, k, grid, metric_term)) end @inline function upwinded_divergence_flux_Vᶜᶠᶜ(i, j, k, grid, scheme::VectorInvariantSelfVerticalUpwinding, u, v) @@ -34,11 +37,11 @@ end cross_scheme = scheme.upwinding.cross_scheme @inbounds v̂ = v[i, j, k] - δuˢ = _symmetric_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, cross_scheme, δy_V_plus_metric, u, v) + δuˢ = _symmetric_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, cross_scheme, δx_U, u, v) δvᴸ = _left_biased_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, δy_V, δV_stencil, u, v) δvᴿ = _right_biased_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, δy_V, δV_stencil, u, v) - return upwind_biased_product(v̂, δvᴸ, δvᴿ) + v̂ * δuˢ + return upwind_biased_product(v̂, δvᴸ, δvᴿ) + v̂ * (δuˢ + ℑyᵃᶠᵃ(i, j, k, grid, metric_term)) end ##### diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl index 60d9b40338..ff9f1d0939 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl @@ -24,13 +24,10 @@ compute_w_from_continuity!(velocities, arch, grid; parameters = w_kernel_paramet @unroll for k in 2:grid.Nz+1 @inbounds U.w[i, j, k] = U.w[i, j, k-1] - Δzᶜᶜᶜ(i, j, k-1, grid) * ( div_xyᶜᶜᶜ(i, j, k-1, grid, U.u, U.v) + - metric_term(i, j, k-1, grid) ) + metric_term(i, j, k-1, grid) / Vᶜᶜᶜ(i, j, k, grid) ) end end -metric_term(i, j, k, grid) = zero(grid) -metric_term(i, j, k, grid::MovingCoordinateGrid) = grid.Δzᵃᵃᶜ.∂t_s[i, j, grid.Nz+1] - ##### ##### Size and offsets for the w kernel ##### diff --git a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl index 6f4af60e95..9da8b91c2b 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl @@ -19,14 +19,13 @@ at timestep `n-1` and `n` and it's time derivative. """ struct MovingVerticalCoordinate{D, R, Z, S} denomination :: D # The type of moving coordinate - Δr :: R # Reference _non moving_ vertical coordinate - Δ :: Z # moving vertical coordinate + Δr :: R # Reference _non moving_ vertical coordinate (one-dimensional) + Δ :: Z # moving vertical coordinate (three-dimensional) s⁻ :: S # scaling term = ∂Δ/∂Δr at the start of the time step sⁿ :: S # scaling term = ∂Δ/∂Δr at the end of the time step - ∂t_s :: S # Time derivative of the vertical coordinate scaling divided by the scaling + ∂t_s :: S # Time derivative of the vertical coordinate scaling divided by the sⁿ end - Adapt.adapt_structure(to, coord::MovingVerticalCoordinate) = MovingVerticalCoordinate(nothing, Adapt.adapt(to, coord.Δr), @@ -253,10 +252,6 @@ ab2_step_tracer_field!(tracer_field, grid::ZStarCoordinateGrid, Δt, χ, Gⁿ, G # When performing divergence upwinding we must include the # metric term -import Oceananigans.Advection: δx_U_plus_metric, δy_V_plus_metric - -@inline δx_U_plus_metric(i, j, k, grid::ZStarCoordinateGrid, u, v) = - @inbounds δxᶜᵃᵃ(i, j, k, grid, Ax_qᶠᶜᶜ, u) + grid.Δzᵃᵃᶜ.∂t_s[i, j, grid.Nz+1] * Vᶜᶜᶜ(i, j, k, grid) +import Oceananigans.Advection: metric_term -@inline δy_V_plus_metric(i, j, k, grid::ZStarCoordinateGrid, u, v) = - @inbounds δyᵃᶜᵃ(i, j, k, grid, Ay_qᶜᶠᶜ, v) + grid.Δzᵃᵃᶜ.∂t_s[i, j, grid.Nz+1] * Vᶜᶜᶜ(i, j, k, grid) +metric_term(i, j, k, grid::MovingCoordinateGrid) = grid.Δzᵃᵃᶜ.∂t_s[i, j, grid.Nz+1] * Vᶜᶜᶜ(i, j, k, grid) From 4b98790355f6bdf5240987840283f1b94786d30c Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 21 Dec 2023 11:55:21 +0100 Subject: [PATCH 112/567] comments --- .../moving_vertical_coordinate.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl index 9da8b91c2b..5f49ea2fb2 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl @@ -15,7 +15,9 @@ spacings for a generalized vertical coordinate system. The fixed coordinate is stored in `Δr`, while `Δ` contains the moving z-coordinate. The `s⁻`, `sⁿ` and `∂t_s` fields are the vertical derivative of the vertical coordinate at timestep `n-1` and `n` and it's time derivative. -`denomination` contains the +`denomination` contains the "type" of generalized vertical coordinate, for example: +- Zstar: free-surface following +- sigma: terrain following """ struct MovingVerticalCoordinate{D, R, Z, S} denomination :: D # The type of moving coordinate From bb9c04833d0716160da14b47d512ea4694ff18e6 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 21 Dec 2023 11:57:33 +0100 Subject: [PATCH 113/567] changed to Generalized --- .../moving_vertical_coordinate.jl | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl index 5f49ea2fb2..a3db893e35 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl @@ -9,18 +9,18 @@ using Adapt using Printf """ - MovingVerticalCoordinate{R, S, Z} + GeneralizedVerticalCoordinate{R, S, Z} spacings for a generalized vertical coordinate system. -The fixed coordinate is stored in `Δr`, while `Δ` contains the moving z-coordinate. -The `s⁻`, `sⁿ` and `∂t_s` fields are the vertical derivative of the vertical coordinate +The reference coordinate is stored in `Δr`, while `Δ` contains the z-coordinate. +The `s⁻`, `sⁿ` and `∂t_s` fields are the vertical derivative of the vertical coordinate (∂Δ/∂Δr) at timestep `n-1` and `n` and it's time derivative. `denomination` contains the "type" of generalized vertical coordinate, for example: - Zstar: free-surface following - sigma: terrain following """ -struct MovingVerticalCoordinate{D, R, Z, S} - denomination :: D # The type of moving coordinate +struct GeneralizedVerticalCoordinate{D, R, Z, S} + denomination :: D # The type of generalized coordinate Δr :: R # Reference _non moving_ vertical coordinate (one-dimensional) Δ :: Z # moving vertical coordinate (three-dimensional) s⁻ :: S # scaling term = ∂Δ/∂Δr at the start of the time step @@ -28,8 +28,8 @@ struct MovingVerticalCoordinate{D, R, Z, S} ∂t_s :: S # Time derivative of the vertical coordinate scaling divided by the sⁿ end -Adapt.adapt_structure(to, coord::MovingVerticalCoordinate) = - MovingVerticalCoordinate(nothing, +Adapt.adapt_structure(to, coord::GeneralizedVerticalCoordinate) = + GeneralizedVerticalCoordinate(nothing, Adapt.adapt(to, coord.Δr), Adapt.adapt(to, coord.Δ), Adapt.adapt(to, coord.s⁻), @@ -38,29 +38,29 @@ Adapt.adapt_structure(to, coord::MovingVerticalCoordinate) = import Oceananigans.Architectures: arch_array -arch_array(arch, coord::MovingVerticalCoordinate) = - MovingVerticalCoordinate(coord.denomination, +arch_array(arch, coord::GeneralizedVerticalCoordinate) = + GeneralizedVerticalCoordinate(coord.denomination, arch_array(arch, coord.Δr), coord.Δ, coord.s⁻, coord.sⁿ, coord.∂t_s) -const MovingCoordinateRG{D} = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:MovingVerticalCoordinate{D}} where D -const MovingCoordinateLLG{D} = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:MovingVerticalCoordinate{D}} where D +const MovingCoordinateRG{D} = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:GeneralizedVerticalCoordinate{D}} where D +const MovingCoordinateLLG{D} = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:GeneralizedVerticalCoordinate{D}} where D const MovingCoordinateUnderlyingGrid{D} = Union{MovingCoordinateRG{D}, MovingCoordinateLLG{D}} where D const MovingCoordinateImmersedGrid{D} = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:MovingCoordinateUnderlyingGrid{D}} where D const MovingCoordinateGrid{D} = Union{MovingCoordinateUnderlyingGrid{D}, MovingCoordinateImmersedGrid{D}} where D -""" a _free-surface following_ vertical coordinate """ +""" free-surface following vertical coordinate """ struct ZStar end -""" geopotential-following vertical coordinate """ +""" geopotential following vertical coordinate """ struct Z end -const ZStarCoordinate = MovingVerticalCoordinate{<:ZStar} +const ZStarCoordinate = GeneralizedVerticalCoordinate{<:ZStar} Grids.coordinate_summary(Δ::ZStarCoordinate, name) = @sprintf("Free-surface following with Δ%s=%s", name, prettysummary(Δ.Δr)) @@ -96,8 +96,8 @@ function MovingCoordinateGrid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, ::ZS fill_halo_regions!((ΔzF, ΔzC, s⁻, sⁿ, ∂t_s); only_local_halos = true) - Δzᵃᵃᶠ = MovingVerticalCoordinate(ZStar(), grid.Δzᵃᵃᶠ, ΔzF, s⁻, sⁿ, ∂t_s) - Δzᵃᵃᶜ = MovingVerticalCoordinate(ZStar(), grid.Δzᵃᵃᶜ, ΔzC, s⁻, sⁿ, ∂t_s) + Δzᵃᵃᶠ = GeneralizedVerticalCoordinate(ZStar(), grid.Δzᵃᵃᶠ, ΔzF, s⁻, sⁿ, ∂t_s) + Δzᵃᵃᶜ = GeneralizedVerticalCoordinate(ZStar(), grid.Δzᵃᵃᶜ, ΔzC, s⁻, sⁿ, ∂t_s) args = [] for prop in propertynames(grid) From 9c86f6cc2bcfaee06fb5b6892d49114f4990e2ad Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 21 Dec 2023 11:59:40 +0100 Subject: [PATCH 114/567] removed scaling --- .../hydrostatic_free_surface_tendency_kernel_functions.jl | 4 ++-- .../moving_vertical_coordinate.jl | 4 ---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_tendency_kernel_functions.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_tendency_kernel_functions.jl index 85b6ba7edb..24492169f7 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_tendency_kernel_functions.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_tendency_kernel_functions.jl @@ -44,7 +44,7 @@ implicitly during time-stepping. model_fields = merge(hydrostatic_fields(velocities, free_surface, tracers), auxiliary_fields) - return ( - U_dot_∇u(i, j, k, grid, advection, velocities) / scaling(i, j, k, grid) + return ( - U_dot_∇u(i, j, k, grid, advection, velocities) - explicit_barotropic_pressure_x_gradient(i, j, k, grid, free_surface) - x_f_cross_U(i, j, k, grid, coriolis, velocities) - ∂xᶠᶜᶜ(i, j, k, grid, hydrostatic_pressure_anomaly) @@ -84,7 +84,7 @@ implicitly during time-stepping. model_fields = merge(hydrostatic_fields(velocities, free_surface, tracers), auxiliary_fields) - return ( - U_dot_∇v(i, j, k, grid, advection, velocities) / scaling(i, j, k, grid) + return ( - U_dot_∇v(i, j, k, grid, advection, velocities) - explicit_barotropic_pressure_y_gradient(i, j, k, grid, free_surface) - y_f_cross_U(i, j, k, grid, coriolis, velocities) - ∂yᶜᶠᶜ(i, j, k, grid, hydrostatic_pressure_anomaly) diff --git a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl index a3db893e35..caa850b226 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl @@ -188,10 +188,6 @@ end end end -@inline scaling(i, j, k, grid) = one(grid) -@inline scaling(i, j, k, grid::ZStarCoordinateGrid) = grid.Δzᵃᵃᶠ.sⁿ[i, j, grid.Nz+1] - - import Oceananigans.Operators: Δzᶜᶜᶠ, Δzᶜᶜᶜ, Δzᶜᶠᶠ, Δzᶜᶠᶜ, Δzᶠᶜᶠ, Δzᶠᶜᶜ, Δzᶠᶠᶠ, Δzᶠᶠᶜ # Very bad for GPU performance!!! (z-values are not coalesced in memory for z-derivatives anymore) From a161fa381dc316900182809380b67efaf24621d1 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 21 Dec 2023 12:01:47 +0100 Subject: [PATCH 115/567] generalized --- .../hydrostatic_free_surface_model.jl | 2 +- .../moving_vertical_coordinate.jl | 22 +++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl index 75a109b7cf..e0fefdcca8 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl @@ -119,7 +119,7 @@ function HydrostaticFreeSurfaceModel(; grid, @apply_regionally validate_model_halo(grid, momentum_advection, tracer_advection, closure) # Introduce z-star coordinates if needed (only is free_surface is not a nothing) - grid = !isnothing(free_surface) ? MovingCoordinateGrid(grid, vertical_coordinate) : grid + grid = !isnothing(free_surface) ? GeneralizedCoordinateGrid(grid, vertical_coordinate) : grid arch = architecture(grid) @apply_regionally momentum_advection = validate_momentum_advection(momentum_advection, grid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl index caa850b226..dc662400b9 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl @@ -46,13 +46,13 @@ arch_array(arch, coord::GeneralizedVerticalCoordinate) = coord.sⁿ, coord.∂t_s) -const MovingCoordinateRG{D} = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:GeneralizedVerticalCoordinate{D}} where D -const MovingCoordinateLLG{D} = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:GeneralizedVerticalCoordinate{D}} where D +const GeneralizedCoordinateRG{D} = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:GeneralizedVerticalCoordinate{D}} where D +const GeneralizedCoordinateLLG{D} = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:GeneralizedVerticalCoordinate{D}} where D -const MovingCoordinateUnderlyingGrid{D} = Union{MovingCoordinateRG{D}, MovingCoordinateLLG{D}} where D -const MovingCoordinateImmersedGrid{D} = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:MovingCoordinateUnderlyingGrid{D}} where D +const GeneralizedCoordinateUnderlyingGrid{D} = Union{GeneralizedCoordinateRG{D}, GeneralizedCoordinateLLG{D}} where D +const GeneralizedCoordinateImmersedGrid{D} = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:GeneralizedCoordinateUnderlyingGrid{D}} where D -const MovingCoordinateGrid{D} = Union{MovingCoordinateUnderlyingGrid{D}, MovingCoordinateImmersedGrid{D}} where D +const GeneralizedCoordinateGrid{D} = Union{GeneralizedCoordinateUnderlyingGrid{D}, GeneralizedCoordinateImmersedGrid{D}} where D """ free-surface following vertical coordinate """ struct ZStar end @@ -65,12 +65,12 @@ const ZStarCoordinate = GeneralizedVerticalCoordinate{<:ZStar} Grids.coordinate_summary(Δ::ZStarCoordinate, name) = @sprintf("Free-surface following with Δ%s=%s", name, prettysummary(Δ.Δr)) -const ZStarCoordinateGrid = MovingCoordinateGrid{<:ZStar} +const ZStarCoordinateGrid = GeneralizedCoordinateGrid{<:ZStar} -MovingCoordinateGrid(grid, coord) = grid +GeneralizedCoordinateGrid(grid, coord) = grid -function MovingCoordinateGrid(grid::ImmersedBoundaryGrid, ::ZStar) - underlying_grid = MovingCoordinateGrid(grid.underlying_grid, ZStarCoordinate()) +function GeneralizedCoordinateGrid(grid::ImmersedBoundaryGrid, ::ZStar) + underlying_grid = GeneralizedCoordinateGrid(grid.underlying_grid, ZStarCoordinate()) active_cells_map = !isnothing(grid.interior_active_cells) return ImmersedBoundaryGrid(underlying_grid, grid.immersed_boundary; active_cells_map) @@ -78,7 +78,7 @@ end # Replacing the z-coordinate with a moving vertical coordinate, defined by its reference spacing, # the actual vertical spacing and a scaling -function MovingCoordinateGrid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, ::ZStar) where {FT, TX, TY, TZ} +function GeneralizedCoordinateGrid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, ::ZStar) where {FT, TX, TY, TZ} # Memory layout for Dz spacings is local in z ΔzF = ZFaceField(grid) @@ -252,4 +252,4 @@ ab2_step_tracer_field!(tracer_field, grid::ZStarCoordinateGrid, Δt, χ, Gⁿ, G import Oceananigans.Advection: metric_term -metric_term(i, j, k, grid::MovingCoordinateGrid) = grid.Δzᵃᵃᶜ.∂t_s[i, j, grid.Nz+1] * Vᶜᶜᶜ(i, j, k, grid) +metric_term(i, j, k, grid::GeneralizedCoordinateGrid) = grid.Δzᵃᵃᶜ.∂t_s[i, j, grid.Nz+1] * Vᶜᶜᶜ(i, j, k, grid) From 1edea6344e0bf2fc0e000c40b597c3c1dbd8521e Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 21 Dec 2023 12:10:21 +0100 Subject: [PATCH 116/567] using reduced fields --- .../moving_vertical_coordinate.jl | 41 +++++++++---------- .../split_explicit_free_surface_kernels.jl | 12 +++--- 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl index dc662400b9..e20f5797e0 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl @@ -83,9 +83,9 @@ function GeneralizedCoordinateGrid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, # Memory layout for Dz spacings is local in z ΔzF = ZFaceField(grid) ΔzC = CenterField(grid) - s⁻ = ZFaceField(grid, indices = (:, :, grid.Nz+1)) - sⁿ = ZFaceField(grid, indices = (:, :, grid.Nz+1)) - ∂t_s = ZFaceField(grid, indices = (:, :, grid.Nz+1)) + s⁻ = Field{Center, Center, Nothing}(grid) + sⁿ = Field{Center, Center, Nothing}(grid) + ∂t_s = Field{Center, Center, Nothing}(grid) # Initial "at-rest" conditions launch!(architecture(grid), grid, :xy, _update_scaling!, @@ -131,15 +131,15 @@ function update_vertical_coordinate!(model, grid::ZStarCoordinateGrid, Δt; para ΔzC = grid.Δzᵃᵃᶜ.Δ # Reference coordinates - Δz₀F = grid.Δzᵃᵃᶠ.Δr - Δz₀C = grid.Δzᵃᵃᶜ.Δr + ΔzF₀ = grid.Δzᵃᵃᶠ.Δr + ΔzC₀ = grid.Δzᵃᵃᶜ.Δr # Update vertical coordinate with available parameters for params in parameters - update_coordinate_scaling!(sⁿ, s⁻, ∂t_s, params, model.free_surface, grid, Δt) + update_zstar_scaling!(sⁿ, s⁻, ∂t_s, params, model.free_surface, grid, Δt) launch!(architecture(grid), grid, horizontal_parameters(params), _update_z_star!, - ΔzF, ΔzC, Δz₀F, Δz₀C, sⁿ, Val(grid.Nz)) + ΔzF, ΔzC, ΔzF₀, ΔzC₀, sⁿ, Val(grid.Nz)) end fill_halo_regions!((ΔzF, ΔzC, s⁻, sⁿ, ∂t_s); only_local_halos = true) @@ -150,22 +150,22 @@ end horizontal_parameters(::Symbol) = :xy horizontal_parameters(::KernelParameters{W, O}) where {W, O} = KernelParameters(W[1:2], O[1:2]) -update_coordinate_scaling!(sⁿ, s⁻, ∂t_s, params, fs, grid, Δt) = - launch!(architecture(grid), grid, horizontal_parameters(params), _update_scaling!, +update_zstar_scaling!(sⁿ, s⁻, ∂t_s, params, fs, grid, Δt) = + launch!(architecture(grid), grid, horizontal_parameters(params), _update_zstar_scaling!, sⁿ, s⁻, ∂t_s, fs.η, grid, Δt) -@kernel function _update_scaling!(sⁿ, s⁻, ∂t_s, η, grid, Δt) +@kernel function _update_zstar_scaling!(sⁿ, s⁻, ∂t_s, η, grid, Δt) i, j = @index(Global, NTuple) bottom = bottom_height(i, j, grid) @inbounds begin h = (bottom + η[i, j, grid.Nz+1]) / bottom # update current and previous scaling - s⁻[i, j, grid.Nz+1] = sⁿ[i, j, grid.Nz+1] - sⁿ[i, j, grid.Nz+1] = h + s⁻[i, j, 1] = sⁿ[i, j, 1] + sⁿ[i, j, 1] = h # Scaling derivative - ∂t_s[i, j, grid.Nz+1] = (h - s⁻[i, j, grid.Nz+1]) / Δt / h + ∂t_s[i, j, 1] = (h - s⁻[i, j, 1]) / Δt / h end end @@ -175,16 +175,16 @@ bottom_height(i, j, grid::ImmersedBoundaryGrid) = @inbounds - grid.immersed_boun @kernel function _update_z_star!(ΔzF, ΔzC, ΔzF₀, ΔzC₀, sⁿ, ::Val{Nz}) where Nz i, j = @index(Global, NTuple) @unroll for k in 1:Nz+1 - @inbounds ΔzF[i, j, k] = sⁿ[i, j, Nz+1] * ΔzF₀[k] - @inbounds ΔzC[i, j, k] = sⁿ[i, j, Nz+1] * ΔzC₀[k] + @inbounds ΔzF[i, j, k] = sⁿ[i, j, 1] * ΔzF₀[k] + @inbounds ΔzC[i, j, k] = sⁿ[i, j, 1] * ΔzC₀[k] end end @kernel function _update_z_star!(ΔzF, ΔzC, ΔzF₀::Number, ΔzC₀::Number, sⁿ, ::Val{Nz}) where Nz i, j = @index(Global, NTuple) @unroll for k in 1:Nz+1 - @inbounds ΔzF[i, j, k] = sⁿ[i, j, Nz+1] * ΔzF₀ - @inbounds ΔzC[i, j, k] = sⁿ[i, j, Nz+1] * ΔzC₀ + @inbounds ΔzF[i, j, k] = sⁿ[i, j, 1] * ΔzF₀ + @inbounds ΔzC[i, j, k] = sⁿ[i, j, 1] * ΔzC₀ end end @@ -226,7 +226,7 @@ import Oceananigans.Operators: Δzᶜᶜᶠ, Δzᶜᶜᶜ, Δzᶜᶠᶠ, Δzᶜ ##### In the Z-star coordinate framework the prognostic field is sθ! ##### -@kernel function ab2_step_tracer_zstar!(θ, sⁿ, s⁻, Nz, Δt, χ, Gⁿ, G⁻) +@kernel function ab2_step_tracer_zstar!(θ, sⁿ, s⁻, Δt, χ, Gⁿ, G⁻) i, j, k = @index(Global, NTuple) FT = eltype(χ) @@ -235,7 +235,7 @@ import Oceananigans.Operators: Δzᶜᶜᶠ, Δzᶜᶜᶜ, Δzᶜᶠᶠ, Δzᶜ @inbounds begin ∂t_θ = (one_point_five + χ) * Gⁿ[i, j, k] - (oh_point_five + χ) * G⁻[i, j, k] - θ[i, j, k] = s⁻[i, j, Nz+1] * θ[i, j, k] / sⁿ[i, j, Nz+1] + convert(FT, Δt) * ∂t_θ + θ[i, j, k] = s⁻[i, j, 1] * θ[i, j, k] / sⁿ[i, j, 1] + convert(FT, Δt) * ∂t_θ end end @@ -244,7 +244,6 @@ ab2_step_tracer_field!(tracer_field, grid::ZStarCoordinateGrid, Δt, χ, Gⁿ, G tracer_field, grid.Δzᵃᵃᶠ.sⁿ, grid.Δzᵃᵃᶠ.s⁻, - grid.Nz, Δt, χ, Gⁿ, G⁻) # When performing divergence upwinding we must include the @@ -252,4 +251,4 @@ ab2_step_tracer_field!(tracer_field, grid::ZStarCoordinateGrid, Δt, χ, Gⁿ, G import Oceananigans.Advection: metric_term -metric_term(i, j, k, grid::GeneralizedCoordinateGrid) = grid.Δzᵃᵃᶜ.∂t_s[i, j, grid.Nz+1] * Vᶜᶜᶜ(i, j, k, grid) +metric_term(i, j, k, grid::GeneralizedCoordinateGrid) = grid.Δzᵃᵃᶜ.∂t_s[i, j, k] * Vᶜᶜᶜ(i, j, k, grid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index db2f692657..244f911d4f 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -392,21 +392,21 @@ setup_split_explicit_tendency!(auxiliary, grid, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) wait_free_surface_communication!(free_surface, arch) = nothing -update_coordinate_scaling!(sⁿ, s⁻, ∂t_s, params, fs::SplitExplicitFreeSurface, grid, Δt) = - launch!(architecture(grid), grid, horizontal_parameters(params), _update_split_explicit_scaling!, +update_zstar_scaling!(sⁿ, s⁻, ∂t_s, params, fs::SplitExplicitFreeSurface, grid, Δt) = + launch!(architecture(grid), grid, horizontal_parameters(params), _update_zstar_split_explicit_scaling!, sⁿ, s⁻, ∂t_s, fs.η, fs.state.U̅, fs.state.V̅, grid) -@kernel function _update_split_explicit_scaling!(sⁿ, s⁻, ∂t_s, η, U̅, V̅, grid) +@kernel function _update_zstar_split_explicit_scaling!(sⁿ, s⁻, ∂t_s, η, U̅, V̅, grid) i, j = @index(Global, NTuple) bottom = bottom_height(i, j, grid) @inbounds begin h = (bottom + η[i, j, grid.Nz+1]) / bottom # update current and previous scaling - s⁻[i, j, grid.Nz+1] = sⁿ[i, j, grid.Nz+1] - sⁿ[i, j, grid.Nz+1] = h + s⁻[i, j, 1] = sⁿ[i, j, 1] + sⁿ[i, j, 1] = h # ∂(H + η / H)/∂t = - ∇ ⋅ ∫udz / H - ∂t_s[i, j, grid.Nz+1] = - div_xyᶜᶜᶜ(i, j, 1, grid, U̅, V̅) / bottom / h + ∂t_s[i, j, 1] = - div_xyᶜᶜᶜ(i, j, 1, grid, U̅, V̅) / bottom / h end end From 69b76d4fea819bfffcd42156276b20e6d8274f1b Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 21 Dec 2023 17:47:02 +0100 Subject: [PATCH 117/567] only vector-invariant --- .../HydrostaticFreeSurfaceModels.jl | 2 +- .../compute_hydrostatic_free_surface_tendencies.jl | 2 +- .../compute_w_from_continuity.jl | 2 +- ...inate.jl => generalized_vertical_coordinate.jl} | 11 ++++++----- .../hydrostatic_free_surface_model.jl | 4 ++++ .../moving_coordinates/z_star_lock_release.jl | 14 +++++++++----- 6 files changed, 22 insertions(+), 13 deletions(-) rename src/Models/HydrostaticFreeSurfaceModels/{moving_vertical_coordinate.jl => generalized_vertical_coordinate.jl} (97%) diff --git a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl index f2e299ee8b..05110e5f16 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl @@ -30,7 +30,7 @@ fill_horizontal_velocity_halos!(args...) = nothing FreeSurfaceDisplacementField(velocities, free_surface, grid) = ZFaceField(grid, indices = (:, :, size(grid, 3)+1)) FreeSurfaceDisplacementField(velocities, ::Nothing, grid) = nothing -include("moving_vertical_coordinate.jl") +include("generalized_vertical_coordinate.jl") include("compute_w_from_continuity.jl") include("rigid_lid.jl") diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl index 53fe23f692..a552aea75c 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl @@ -27,7 +27,7 @@ function compute_tendencies!(model::HydrostaticFreeSurfaceModel, Δt, callbacks) kernel_parameters = tuple(interior_tendency_kernel_parameters(model.grid)) update_vertical_coordinate!(model, model.grid, Δt; parameters = kernel_parameters) - + # Calculate contributions to momentum and tracer tendencies from fluxes and volume terms in the # interior of the domain compute_hydrostatic_free_surface_tendency_contributions!(model, kernel_parameters; diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl index ff9f1d0939..cc8cd32aa6 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl @@ -24,7 +24,7 @@ compute_w_from_continuity!(velocities, arch, grid; parameters = w_kernel_paramet @unroll for k in 2:grid.Nz+1 @inbounds U.w[i, j, k] = U.w[i, j, k-1] - Δzᶜᶜᶜ(i, j, k-1, grid) * ( div_xyᶜᶜᶜ(i, j, k-1, grid, U.u, U.v) + - metric_term(i, j, k-1, grid) / Vᶜᶜᶜ(i, j, k, grid) ) + metric_term(i, j, k-1, grid) ) end end diff --git a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_coordinate.jl similarity index 97% rename from src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl rename to src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_coordinate.jl index e20f5797e0..817f6997cf 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/moving_vertical_coordinate.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_coordinate.jl @@ -88,13 +88,15 @@ function GeneralizedCoordinateGrid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, ∂t_s = Field{Center, Center, Nothing}(grid) # Initial "at-rest" conditions - launch!(architecture(grid), grid, :xy, _update_scaling!, - sⁿ, s⁻, ∂t_s, ZeroField(grid), grid, 1) + set!(sⁿ, 1) + set!(s⁻, 1) + + fill_halo_regions!((s⁻, sⁿ)) launch!(architecture(grid), grid, :xy,_update_z_star!, ΔzF, ΔzC, grid.Δzᵃᵃᶠ, grid.Δzᵃᵃᶜ, sⁿ, Val(grid.Nz)) - fill_halo_regions!((ΔzF, ΔzC, s⁻, sⁿ, ∂t_s); only_local_halos = true) + fill_halo_regions!((ΔzF, ΔzC)) Δzᵃᵃᶠ = GeneralizedVerticalCoordinate(ZStar(), grid.Δzᵃᵃᶠ, ΔzF, s⁻, sⁿ, ∂t_s) Δzᵃᵃᶜ = GeneralizedVerticalCoordinate(ZStar(), grid.Δzᵃᵃᶜ, ΔzC, s⁻, sⁿ, ∂t_s) @@ -119,7 +121,6 @@ end update_vertical_coordinate!(model, grid, Δt; kwargs...) = nothing function update_vertical_coordinate!(model, grid::ZStarCoordinateGrid, Δt; parameters = tuple(:xyz)) - η = model.free_surface.η # Scaling s⁻ = grid.Δzᵃᵃᶠ.s⁻ @@ -251,4 +252,4 @@ ab2_step_tracer_field!(tracer_field, grid::ZStarCoordinateGrid, Δt, χ, Gⁿ, G import Oceananigans.Advection: metric_term -metric_term(i, j, k, grid::GeneralizedCoordinateGrid) = grid.Δzᵃᵃᶜ.∂t_s[i, j, k] * Vᶜᶜᶜ(i, j, k, grid) +metric_term(i, j, k, grid::GeneralizedCoordinateGrid) = grid.Δzᵃᵃᶜ.∂t_s[i, j, k] / grid.Δzᵃᵃᶜ.sⁿ[i, j, k] diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl index e0fefdcca8..a2ac9f1db8 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl @@ -119,6 +119,10 @@ function HydrostaticFreeSurfaceModel(; grid, @apply_regionally validate_model_halo(grid, momentum_advection, tracer_advection, closure) # Introduce z-star coordinates if needed (only is free_surface is not a nothing) + if !(vertical_cordinate isa Z) && !(momentum_advection isa VectorInvariant) && isnothing(velocities) + throw(ArgumentError("Generalized vertical coordinates are supported only for the vector-invariant form of the momentum equations")) + end + grid = !isnothing(free_surface) ? GeneralizedCoordinateGrid(grid, vertical_coordinate) : grid arch = architecture(grid) diff --git a/validation/moving_coordinates/z_star_lock_release.jl b/validation/moving_coordinates/z_star_lock_release.jl index 6d007b49a6..98c81871ed 100644 --- a/validation/moving_coordinates/z_star_lock_release.jl +++ b/validation/moving_coordinates/z_star_lock_release.jl @@ -13,8 +13,8 @@ grid = RectilinearGrid(size = (300, 20), model = HydrostaticFreeSurfaceModel(; grid, vertical_coordinate = ZStar(), - momentum_advection = WENOVectorInvariant(), - tracer_advection = WENO(order = 9), + momentum_advection = VectorInvariant(), + tracer_advection = WENO(), buoyancy = BuoyancyTracer(), tracers = :b, free_surface = SplitExplicitFreeSurface(; substeps = 30)) @@ -47,16 +47,20 @@ simulation.output_writers[:other_variables] = JLD2OutputWriter(model, field_outp function progress(sim) w = interior(sim.model.velocities.w, :, :, sim.model.grid.Nz+1) u = sim.model.velocities.u + v = sim.model.velocities.v + η = sim.model.free_surface.η + b = sim.model.tracers.b msg0 = @sprintf("Time: %s iteration %d ", prettytime(sim.model.clock.time), sim.model.clock.iteration) msg1 = @sprintf("extrema w: %.2e %.2e ", maximum(w), minimum(w)) msg2 = @sprintf("extrema u: %.2e %.2e ", maximum(u), minimum(u)) + msg3 = @sprintf("extrema b: %.2e %.2e ", maximum(b), minimum(b)) if sim.model.grid isa ZStarCoordinateGrid Δz = sim.model.grid.Δzᵃᵃᶠ.Δ - msg3 = @sprintf("extrema Δz: %.2e %.2e ", maximum(Δz), minimum(Δz)) - @info msg0 * msg1 * msg2 * msg3 + msg4 = @sprintf("extrema Δz: %.2e %.2e ", maximum(Δz), minimum(Δz)) + @info msg0 * msg1 * msg2 * msg3 * msg4 else - @info msg0 * msg1 * msg2 + @info msg0 * msg1 * msg2 * msg3 end return nothing From f4681269d38d72f95518b926be7636f2f96cd9d5 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Fri, 22 Dec 2023 10:42:15 +0100 Subject: [PATCH 118/567] update after --- .../vector_invariant_cross_upwinding.jl | 4 +- .../vector_invariant_self_upwinding.jl | 8 +- ...static_free_surface_boundary_tendencies.jl | 3 +- ...ute_hydrostatic_free_surface_tendencies.jl | 2 - .../compute_w_from_continuity.jl | 2 +- .../generalized_vertical_coordinate.jl | 21 ++-- .../hydrostatic_free_surface_model.jl | 2 +- .../split_explicit_free_surface_kernels.jl | 2 +- ...te_hydrostatic_free_surface_model_state.jl | 9 +- .../immersed_boundaries/z_star_coordinate.jl | 52 --------- .../lock_release.jl} | 10 +- .../z_star_coordinate/tracer_conservation.jl | 103 ++++++++++++++++++ 12 files changed, 134 insertions(+), 84 deletions(-) delete mode 100644 validation/immersed_boundaries/z_star_coordinate.jl rename validation/{moving_coordinates/z_star_lock_release.jl => z_star_coordinate/lock_release.jl} (91%) create mode 100644 validation/z_star_coordinate/tracer_conservation.jl diff --git a/src/Advection/vector_invariant_cross_upwinding.jl b/src/Advection/vector_invariant_cross_upwinding.jl index 8241d77d07..6ed57a40e9 100644 --- a/src/Advection/vector_invariant_cross_upwinding.jl +++ b/src/Advection/vector_invariant_cross_upwinding.jl @@ -24,7 +24,7 @@ δᴸ = _left_biased_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, flux_div_xyᶜᶜᶜ, δ_stencil, u, v) δᴿ = _right_biased_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, flux_div_xyᶜᶜᶜ, δ_stencil, u, v) - return upwind_biased_product(û, δᴸ, δᴿ) + û * ℑxᶠᵃᵃ(i, j, k, grid, metric_term) + return upwind_biased_product(û, δᴸ, δᴿ) + û * ℑxᶠᵃᵃ(i, j, k, grid, V_times_∂t_∂s_grid) end @inline function upwinded_divergence_flux_Vᶜᶠᶜ(i, j, k, grid, scheme::VectorInvariantCrossVerticalUpwinding, u, v) @@ -34,5 +34,5 @@ end δᴸ = _left_biased_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, flux_div_xyᶜᶜᶜ, δ_stencil, u, v) δᴿ = _right_biased_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, flux_div_xyᶜᶜᶜ, δ_stencil, u, v) - return upwind_biased_product(v̂, δᴸ, δᴿ) + v̂ * ℑyᵃᶠᵃ(i, j, k, grid, metric_term) + return upwind_biased_product(v̂, δᴸ, δᴿ) + v̂ * ℑyᵃᶠᵃ(i, j, k, grid, V_times_∂t_∂s_grid) end diff --git a/src/Advection/vector_invariant_self_upwinding.jl b/src/Advection/vector_invariant_self_upwinding.jl index 0599735b6f..5704299f45 100644 --- a/src/Advection/vector_invariant_self_upwinding.jl +++ b/src/Advection/vector_invariant_self_upwinding.jl @@ -16,7 +16,9 @@ @inline divergence_smoothness(i, j, k, grid, u, v) = δx_U(i, j, k, grid, u, v) + δy_V(i, j, k, grid, u, v) # Metric term for moving grids -metric_term(i, j, k, grid) = zero(grid) +∂t_∂s_grid(i, j, k, grid) = zero(grid) + +@inline V_times_∂t_∂s_grid(i, j, k, grid) = ∂t_∂s_grid(i, j, k, grid) * Vᶜᶜᶜ(i, j, k, grid) @inline function upwinded_divergence_flux_Uᶠᶜᶜ(i, j, k, grid, scheme::VectorInvariantSelfVerticalUpwinding, u, v) @@ -28,7 +30,7 @@ metric_term(i, j, k, grid) = zero(grid) δuᴸ = _left_biased_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, δx_U, δU_stencil, u, v) δuᴿ = _right_biased_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, δx_U, δU_stencil, u, v) - return upwind_biased_product(û, δuᴸ, δuᴿ) + û * (δvˢ + ℑxᶠᵃᵃ(i, j, k, grid, metric_term)) + return upwind_biased_product(û, δuᴸ, δuᴿ) + û * (δvˢ + ℑxᶠᵃᵃ(i, j, k, grid, V_times_∂t_∂s_grid)) end @inline function upwinded_divergence_flux_Vᶜᶠᶜ(i, j, k, grid, scheme::VectorInvariantSelfVerticalUpwinding, u, v) @@ -41,7 +43,7 @@ end δvᴸ = _left_biased_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, δy_V, δV_stencil, u, v) δvᴿ = _right_biased_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, δy_V, δV_stencil, u, v) - return upwind_biased_product(v̂, δvᴸ, δvᴿ) + v̂ * (δuˢ + ℑyᵃᶠᵃ(i, j, k, grid, metric_term)) + return upwind_biased_product(v̂, δvᴸ, δvᴿ) + v̂ * (δuˢ + ℑyᵃᶠᵃ(i, j, k, grid, V_times_∂t_∂s_grid)) end ##### diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl index d7e2640305..33bea1a222 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl @@ -18,8 +18,7 @@ function compute_boundary_tendencies!(model::HydrostaticFreeSurfaceModel, Δt) κ_parameters = boundary_κ_kernel_parameters(grid, model.closure, arch) # We need new values for `w`, `p` and `κ` - update_vertical_coordinate!(model, model.grid, Δt; parameters = κ_parameters) - compute_auxiliaries!(model; w_parameters, p_parameters, κ_parameters) + compute_auxiliaries!(model, Δt; w_parameters, p_parameters, κ_parameters) # parameters for communicating North / South / East / West side kernel_parameters = boundary_tendency_kernel_parameters(grid, arch) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl index a552aea75c..1c7055a1f2 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl @@ -25,8 +25,6 @@ contribution from non-hydrostatic pressure. function compute_tendencies!(model::HydrostaticFreeSurfaceModel, Δt, callbacks) kernel_parameters = tuple(interior_tendency_kernel_parameters(model.grid)) - - update_vertical_coordinate!(model, model.grid, Δt; parameters = kernel_parameters) # Calculate contributions to momentum and tracer tendencies from fluxes and volume terms in the # interior of the domain diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl index cc8cd32aa6..85720ed6c0 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl @@ -24,7 +24,7 @@ compute_w_from_continuity!(velocities, arch, grid; parameters = w_kernel_paramet @unroll for k in 2:grid.Nz+1 @inbounds U.w[i, j, k] = U.w[i, j, k-1] - Δzᶜᶜᶜ(i, j, k-1, grid) * ( div_xyᶜᶜᶜ(i, j, k-1, grid, U.u, U.v) + - metric_term(i, j, k-1, grid) ) + ∂t_∂s_grid(i, j, k-1, grid) ) end end diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_coordinate.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_coordinate.jl index 817f6997cf..7e12b8479c 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_coordinate.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_coordinate.jl @@ -120,7 +120,7 @@ end # Fallback update_vertical_coordinate!(model, grid, Δt; kwargs...) = nothing -function update_vertical_coordinate!(model, grid::ZStarCoordinateGrid, Δt; parameters = tuple(:xyz)) +function update_vertical_coordinate!(model, grid::ZStarCoordinateGrid, Δt; parameters = tuple(:xy)) # Scaling s⁻ = grid.Δzᵃᵃᶠ.s⁻ @@ -136,23 +136,20 @@ function update_vertical_coordinate!(model, grid::ZStarCoordinateGrid, Δt; para ΔzC₀ = grid.Δzᵃᵃᶜ.Δr # Update vertical coordinate with available parameters + # No need to fill the halo as the scaling is updated _IN_ the halos + # as well for params in parameters update_zstar_scaling!(sⁿ, s⁻, ∂t_s, params, model.free_surface, grid, Δt) - launch!(architecture(grid), grid, horizontal_parameters(params), _update_z_star!, + launch!(architecture(grid), grid, params, _update_z_star!, ΔzF, ΔzC, ΔzF₀, ΔzC₀, sⁿ, Val(grid.Nz)) end - - fill_halo_regions!((ΔzF, ΔzC, s⁻, sⁿ, ∂t_s); only_local_halos = true) return nothing end -horizontal_parameters(::Symbol) = :xy -horizontal_parameters(::KernelParameters{W, O}) where {W, O} = KernelParameters(W[1:2], O[1:2]) - update_zstar_scaling!(sⁿ, s⁻, ∂t_s, params, fs, grid, Δt) = - launch!(architecture(grid), grid, horizontal_parameters(params), _update_zstar_scaling!, + launch!(architecture(grid), grid, params, _update_zstar_scaling!, sⁿ, s⁻, ∂t_s, fs.η, grid, Δt) @kernel function _update_zstar_scaling!(sⁿ, s⁻, ∂t_s, η, grid, Δt) @@ -235,8 +232,8 @@ import Oceananigans.Operators: Δzᶜᶜᶠ, Δzᶜᶜᶜ, Δzᶜᶠᶠ, Δzᶜ oh_point_five = convert(FT, 0.5) @inbounds begin - ∂t_θ = (one_point_five + χ) * Gⁿ[i, j, k] - (oh_point_five + χ) * G⁻[i, j, k] - θ[i, j, k] = s⁻[i, j, 1] * θ[i, j, k] / sⁿ[i, j, 1] + convert(FT, Δt) * ∂t_θ + ∂t_sθ = (one_point_five + χ) * Gⁿ[i, j, k] - (oh_point_five + χ) * G⁻[i, j, k] + θ[i, j, k] = s⁻[i, j, 1] * θ[i, j, k] / sⁿ[i, j, 1] + convert(FT, Δt) * ∂t_sθ end end @@ -250,6 +247,6 @@ ab2_step_tracer_field!(tracer_field, grid::ZStarCoordinateGrid, Δt, χ, Gⁿ, G # When performing divergence upwinding we must include the # metric term -import Oceananigans.Advection: metric_term +import Oceananigans.Advection: ∂t_∂s_grid -metric_term(i, j, k, grid::GeneralizedCoordinateGrid) = grid.Δzᵃᵃᶜ.∂t_s[i, j, k] / grid.Δzᵃᵃᶜ.sⁿ[i, j, k] +∂t_∂s_grid(i, j, k, grid::GeneralizedCoordinateGrid) = grid.Δzᵃᵃᶜ.∂t_s[i, j, k] diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl index a2ac9f1db8..a4ce0002b7 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl @@ -119,7 +119,7 @@ function HydrostaticFreeSurfaceModel(; grid, @apply_regionally validate_model_halo(grid, momentum_advection, tracer_advection, closure) # Introduce z-star coordinates if needed (only is free_surface is not a nothing) - if !(vertical_cordinate isa Z) && !(momentum_advection isa VectorInvariant) && isnothing(velocities) + if !(vertical_coordinate isa Z) && !(momentum_advection isa VectorInvariant) && isnothing(velocities) throw(ArgumentError("Generalized vertical coordinates are supported only for the vector-invariant form of the momentum equations")) end diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 244f911d4f..6608fd2b7e 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -393,7 +393,7 @@ setup_split_explicit_tendency!(auxiliary, grid, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) wait_free_surface_communication!(free_surface, arch) = nothing update_zstar_scaling!(sⁿ, s⁻, ∂t_s, params, fs::SplitExplicitFreeSurface, grid, Δt) = - launch!(architecture(grid), grid, horizontal_parameters(params), _update_zstar_split_explicit_scaling!, + launch!(architecture(grid), grid, params, _update_zstar_split_explicit_scaling!, sⁿ, s⁻, ∂t_s, fs.η, fs.state.U̅, fs.state.V̅, grid) @kernel function _update_zstar_split_explicit_scaling!(sⁿ, s⁻, ∂t_s, η, U̅, V̅, grid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl index 9559dacd72..8ccf3d5f81 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl @@ -38,7 +38,7 @@ function update_state!(model::HydrostaticFreeSurfaceModel, grid, Δt, callbacks; fill_halo_regions!(prognostic_fields(model), model.clock, fields(model); async = true) @apply_regionally replace_horizontal_vector_halos!(model.velocities, model.grid) - @apply_regionally compute_auxiliaries!(model) + @apply_regionally compute_auxiliaries!(model, Δt) fill_halo_regions!(model.diffusivity_fields; only_local_halos = true) @@ -67,15 +67,16 @@ function mask_immersed_model_fields!(model, grid) return nothing end -function compute_auxiliaries!(model::HydrostaticFreeSurfaceModel; w_parameters = tuple(w_kernel_parameters(model.grid)), - p_parameters = tuple(p_kernel_parameters(model.grid)), - κ_parameters = tuple(:xyz)) +function compute_auxiliaries!(model::HydrostaticFreeSurfaceModel, Δt; w_parameters = tuple(w_kernel_parameters(model.grid)), + p_parameters = tuple(p_kernel_parameters(model.grid)), + κ_parameters = tuple(:xyz)) grid = model.grid closure = model.closure diffusivity = model.diffusivity_fields for (wpar, ppar, κpar) in zip(w_parameters, p_parameters, κ_parameters) + update_vertical_coordinate!(model, grid, Δt; parameters = wpar) compute_w_from_continuity!(model; parameters = wpar) compute_diffusivities!(diffusivity, closure, model; parameters = κpar) update_hydrostatic_pressure!(model.pressure.pHY′, architecture(grid), diff --git a/validation/immersed_boundaries/z_star_coordinate.jl b/validation/immersed_boundaries/z_star_coordinate.jl deleted file mode 100644 index ae83a9cd2c..0000000000 --- a/validation/immersed_boundaries/z_star_coordinate.jl +++ /dev/null @@ -1,52 +0,0 @@ -using Oceananigans -using Oceananigans.Units -using Printf -using Oceananigans.Models.HydrostaticFreeSurfaceModels: ZStarCoordinate - -grid = RectilinearGrid(size = (600, 20), x = (0, 18kilometers), z = (-200, 0), topology = (Periodic, Flat, Bounded), halo = (5, 5)) - -bottom(x) = - 200 + exp( - (x - 9kilometers)^2 / (1.2kilometers)^2) * 150 - -grid = ImmersedBoundaryGrid(grid, GridFittedBottom(bottom)) - -@inline u_bc(i, j, grid, clock, fields, p) = - @inbounds - 1 / p.λ * (0.1 - fields.u[i, j, grid.Nz]) - - -u_boundary = FluxBoundaryCondition(u_bc, discrete_form = true, parameters = (; λ = 10hour)) - -u_bcs = FieldBoundaryConditions(top = u_boundary) - -model = HydrostaticFreeSurfaceModel(; grid, - free_surface = SplitExplicitFreeSurface(; cfl = 0.7, grid), - momentum_advection = WENO(), - tracer_advection = WENO(; order = 7), - tracers = :b, - boundary_conditions = (; u = u_bcs), - closure = VerticalScalarDiffusivity(ν = 1e-3), - buoyancy = BuoyancyTracer(), - vertical_coordinate = ZStarCoordinate()) - -N² = 0.01 -bᵢ(x, z) = N² * z - -set!(model, b = bᵢ) - -simulation = Simulation(model, Δt = 1, stop_time = 2days, stop_iteration = 10000) - -function progress(sim) - u, v, w = sim.model.velocities - @info @sprintf("Time: %s, Δt: %.2f, iteration: %d, velocities: (%.2f, %.2f)", prettytime(sim.model.clock.time), - sim.Δt, sim.model.clock.iteration, maximum(abs, u), maximum(abs, w)) -end - -wizard = TimeStepWizard(; cfl = 0.2, max_change = 1.1) -simulation.callbacks[:wizard] = Callback(wizard, IterationInterval(2)) -simulation.callbacks[:progress] = Callback(progress, IterationInterval(1)) - -simulation.output_writers[:fields] = JLD2OutputWriter(model, merge(model.velocities, model.tracers), - filename = "z_star_coordinate", - overwrite_existing = true, - schedule = TimeInterval(0.5hour)) - -run!(simulation) \ No newline at end of file diff --git a/validation/moving_coordinates/z_star_lock_release.jl b/validation/z_star_coordinate/lock_release.jl similarity index 91% rename from validation/moving_coordinates/z_star_lock_release.jl rename to validation/z_star_coordinate/lock_release.jl index 98c81871ed..f69b34e171 100644 --- a/validation/moving_coordinates/z_star_lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -15,23 +15,25 @@ model = HydrostaticFreeSurfaceModel(; grid, vertical_coordinate = ZStar(), momentum_advection = VectorInvariant(), tracer_advection = WENO(), + closure = HorizontalScalarDiffusivity(ν = 10), buoyancy = BuoyancyTracer(), tracers = :b, free_surface = SplitExplicitFreeSurface(; substeps = 30)) -# ηᵢ(x, z) = exp(-(x - 50kilometers)^2 / (10kilometers)^2) -bᵢ(x, z) = x > 50kilometers ? 0 : 1 +g = model.free_surface.gravitational_acceleration + +bᵢ(x, z) = x < 50kilometers ? 1 : 0 set!(model, b = bᵢ) -gravity_wave_speed = sqrt(model.free_surface.gravitational_acceleration * grid.Lz) +gravity_wave_speed = sqrt(g * grid.Lz) barotropic_time_step = grid.Δxᶜᵃᵃ / gravity_wave_speed Δt = 0.5 * barotropic_time_step @info "the time step is $Δt" -simulation = Simulation(model; Δt, stop_time = 10000Δt) #, stop_iteration = 10) +simulation = Simulation(model; Δt, stop_time = 1days) #, stop_iteration = 10) field_outputs = if model.grid isa ZStarCoordinateGrid merge(model.velocities, model.tracers, (; ΔzF = model.grid.Δzᵃᵃᶠ.Δ)) diff --git a/validation/z_star_coordinate/tracer_conservation.jl b/validation/z_star_coordinate/tracer_conservation.jl new file mode 100644 index 0000000000..756dd870fa --- /dev/null +++ b/validation/z_star_coordinate/tracer_conservation.jl @@ -0,0 +1,103 @@ +using Oceananigans +using Oceananigans.Units +using Oceananigans.Utils: prettytime +using Oceananigans.Advection: WENOVectorInvariant +using Oceananigans.Models.HydrostaticFreeSurfaceModels: ZStar, Z, ZStarCoordinateGrid +using Printf + +grid = RectilinearGrid(size = (300, 20), + x = (0, 100kilometers), + z = (-10, 0), + halo = (6, 6), + topology = (Bounded, Flat, Bounded)) + +model = HydrostaticFreeSurfaceModel(; grid, + vertical_coordinate = ZStar(), + momentum_advection = VectorInvariant(), + tracer_advection = WENO(), + closure = HorizontalScalarDiffusivity(ν = 10), + buoyancy = BuoyancyTracer(), + tracers = :b, + free_surface = SplitExplicitFreeSurface(; substeps = 30)) + +g = model.free_surface.gravitational_acceleration + +ηᵢ(x, z) = exp(-(x - 50kilometers)^2 / (10kilometers)^2) +bᵢ(x, z) = 1e-6 * z + +set!(model, b = bᵢ, η = ηᵢ) + +uᵢ = XFaceField(grid) +for i in 1:size(u, 1) + uᵢ[i, :, :] = - g * ∂xᶠᶜᶜ(i, 1, grid.Nz+1, grid, model.free_surface.η) +end + +set!(model, u = uᵢ) + +gravity_wave_speed = sqrt(g * grid.Lz) +barotropic_time_step = grid.Δxᶜᵃᵃ / gravity_wave_speed + +Δt = 0.5 * barotropic_time_step + +@info "the time step is $Δt" + +simulation = Simulation(model; Δt, stop_time = 1days) #, stop_iteration = 10) + +field_outputs = if model.grid isa ZStarCoordinateGrid + merge(model.velocities, model.tracers, (; ΔzF = model.grid.Δzᵃᵃᶠ.Δ)) +else + merge(model.velocities, model.tracers) +end + +simulation.output_writers[:other_variables] = JLD2OutputWriter(model, field_outputs, + overwrite_existing = true, + schedule = IterationInterval(100), + filename = "zstar_model") + +function progress(sim) + w = interior(sim.model.velocities.w, :, :, sim.model.grid.Nz+1) + u = sim.model.velocities.u + v = sim.model.velocities.v + η = sim.model.free_surface.η + b = sim.model.tracers.b + + msg0 = @sprintf("Time: %s iteration %d ", prettytime(sim.model.clock.time), sim.model.clock.iteration) + msg1 = @sprintf("extrema w: %.2e %.2e ", maximum(w), minimum(w)) + msg2 = @sprintf("extrema u: %.2e %.2e ", maximum(u), minimum(u)) + msg3 = @sprintf("extrema b: %.2e %.2e ", maximum(b), minimum(b)) + if sim.model.grid isa ZStarCoordinateGrid + Δz = sim.model.grid.Δzᵃᵃᶠ.Δ + msg4 = @sprintf("extrema Δz: %.2e %.2e ", maximum(Δz), minimum(Δz)) + @info msg0 * msg1 * msg2 * msg3 * msg4 + else + @info msg0 * msg1 * msg2 * msg3 + end + + return nothing +end + +simulation.callbacks[:progress] = Callback(progress, IterationInterval(100)) +simulation.callbacks[:wizard] = Callback(TimeStepWizard(; cfl = 0.2, max_change = 1.1), IterationInterval(10)) +run!(simulation) + +# Check conservation +if model.grid isa ZStarCoordinateGrid + b = FieldTimeSeries("zstar_model.jld2", "b") + dz = FieldTimeSeries("zstar_model.jld2", "ΔzF") + + init = sum(b[1] * dz[1]) / sum(dz[1]) + drift = [] + for t in 1:length(b.times) + + push!(drift, sum(b[t] * dz[t]) / sum(dz[t]) - init) + end +else + b = FieldTimeSeries("zstar_model.jld2", "b") + + init = sum(b[1]) / prod(size(b[1])) + drift = [] + for t in 1:length(b.times) + + push!(drift, sum(b[t]) / prod(size(b[1])) - init) + end +end \ No newline at end of file From 2f1ef4c64ce3f04f8bd3aa3946294bebd8204872 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Fri, 22 Dec 2023 11:16:07 +0100 Subject: [PATCH 119/567] cleaning up a bit --- .../HydrostaticFreeSurfaceModels.jl | 4 + .../generalized_vertical_coordinate.jl | 230 ++++-------------- .../hydrostatic_free_surface_model.jl | 4 +- ..._free_surface_tendency_kernel_functions.jl | 4 +- .../split_explicit_free_surface_kernels.jl | 8 +- ...te_hydrostatic_free_surface_model_state.jl | 4 +- .../z_star_vertical_coordinate.jl | 144 +++++++++++ 7 files changed, 210 insertions(+), 188 deletions(-) create mode 100644 src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_coordinate.jl diff --git a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl index 05110e5f16..9c7dd74058 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl @@ -30,8 +30,12 @@ fill_horizontal_velocity_halos!(args...) = nothing FreeSurfaceDisplacementField(velocities, free_surface, grid) = ZFaceField(grid, indices = (:, :, size(grid, 3)+1)) FreeSurfaceDisplacementField(velocities, ::Nothing, grid) = nothing +# Generalized vertical coordinate functionality include("generalized_vertical_coordinate.jl") +include("z_star_vertical_coordinate.jl") include("compute_w_from_continuity.jl") + +# Rigid-lid include("rigid_lid.jl") # Explicit free-surface solver functionality diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_coordinate.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_coordinate.jl index 7e12b8479c..0d235e06ca 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_coordinate.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_coordinate.jl @@ -8,223 +8,104 @@ using Oceananigans.Utils: getnamewrapper using Adapt using Printf +import Oceananigans.Operators: Δzᶜᶜᶠ, Δzᶜᶜᶜ, Δzᶜᶠᶠ, Δzᶜᶠᶜ, Δzᶠᶜᶠ, Δzᶠᶜᶜ, Δzᶠᶠᶠ, Δzᶠᶠᶜ +import Oceananigans.Advection: ∂t_∂s_grid + """ - GeneralizedVerticalCoordinate{R, S, Z} + GeneralizedVerticalSpacing{R, S, Z} spacings for a generalized vertical coordinate system. The reference coordinate is stored in `Δr`, while `Δ` contains the z-coordinate. -The `s⁻`, `sⁿ` and `∂t_s` fields are the vertical derivative of the vertical coordinate (∂Δ/∂Δr) +The `s⁻`, `sⁿ` and `∂t_∂s` fields are the vertical derivative of the vertical coordinate (∂Δ/∂Δr) at timestep `n-1` and `n` and it's time derivative. `denomination` contains the "type" of generalized vertical coordinate, for example: - Zstar: free-surface following - sigma: terrain following """ -struct GeneralizedVerticalCoordinate{D, R, Z, S} +struct GeneralizedVerticalSpacing{D, R, Z, S} denomination :: D # The type of generalized coordinate Δr :: R # Reference _non moving_ vertical coordinate (one-dimensional) Δ :: Z # moving vertical coordinate (three-dimensional) s⁻ :: S # scaling term = ∂Δ/∂Δr at the start of the time step sⁿ :: S # scaling term = ∂Δ/∂Δr at the end of the time step - ∂t_s :: S # Time derivative of the vertical coordinate scaling divided by the sⁿ + ∂t_∂s :: S # Time derivative of the vertical coordinate scaling divided by the sⁿ end -Adapt.adapt_structure(to, coord::GeneralizedVerticalCoordinate) = - GeneralizedVerticalCoordinate(nothing, +Adapt.adapt_structure(to, coord::GeneralizedVerticalSpacing) = + GeneralizedVerticalSpacing(nothing, Adapt.adapt(to, coord.Δr), Adapt.adapt(to, coord.Δ), Adapt.adapt(to, coord.s⁻), Adapt.adapt(to, coord.sⁿ), - Adapt.adapt(to, coord.∂t_s)) + Adapt.adapt(to, coord.∂t_∂s)) import Oceananigans.Architectures: arch_array -arch_array(arch, coord::GeneralizedVerticalCoordinate) = - GeneralizedVerticalCoordinate(coord.denomination, +arch_array(arch, coord::GeneralizedVerticalSpacing) = + GeneralizedVerticalSpacing(coord.denomination, arch_array(arch, coord.Δr), coord.Δ, coord.s⁻, coord.sⁿ, - coord.∂t_s) + coord.∂t_∂s) -const GeneralizedCoordinateRG{D} = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:GeneralizedVerticalCoordinate{D}} where D -const GeneralizedCoordinateLLG{D} = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:GeneralizedVerticalCoordinate{D}} where D +const GeneralizedSpacingRG{D} = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:GeneralizedVerticalSpacing{D}} where D +const GeneralizedSpacingLLG{D} = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:GeneralizedVerticalSpacing{D}} where D -const GeneralizedCoordinateUnderlyingGrid{D} = Union{GeneralizedCoordinateRG{D}, GeneralizedCoordinateLLG{D}} where D -const GeneralizedCoordinateImmersedGrid{D} = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:GeneralizedCoordinateUnderlyingGrid{D}} where D +const GeneralizedSpacingUnderlyingGrid{D} = Union{GeneralizedSpacingRG{D}, GeneralizedSpacingLLG{D}} where D +const GeneralizedSpacingImmersedGrid{D} = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:GeneralizedSpacingUnderlyingGrid{D}} where D -const GeneralizedCoordinateGrid{D} = Union{GeneralizedCoordinateUnderlyingGrid{D}, GeneralizedCoordinateImmersedGrid{D}} where D - -""" free-surface following vertical coordinate """ -struct ZStar end +const GeneralizedSpacingGrid{D} = Union{GeneralizedSpacingUnderlyingGrid{D}, GeneralizedSpacingImmersedGrid{D}} where D """ geopotential following vertical coordinate """ struct Z end -const ZStarCoordinate = GeneralizedVerticalCoordinate{<:ZStar} - -Grids.coordinate_summary(Δ::ZStarCoordinate, name) = - @sprintf("Free-surface following with Δ%s=%s", name, prettysummary(Δ.Δr)) - -const ZStarCoordinateGrid = GeneralizedCoordinateGrid{<:ZStar} - -GeneralizedCoordinateGrid(grid, coord) = grid - -function GeneralizedCoordinateGrid(grid::ImmersedBoundaryGrid, ::ZStar) - underlying_grid = GeneralizedCoordinateGrid(grid.underlying_grid, ZStarCoordinate()) - active_cells_map = !isnothing(grid.interior_active_cells) - - return ImmersedBoundaryGrid(underlying_grid, grid.immersed_boundary; active_cells_map) -end - -# Replacing the z-coordinate with a moving vertical coordinate, defined by its reference spacing, -# the actual vertical spacing and a scaling -function GeneralizedCoordinateGrid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, ::ZStar) where {FT, TX, TY, TZ} - - # Memory layout for Dz spacings is local in z - ΔzF = ZFaceField(grid) - ΔzC = CenterField(grid) - s⁻ = Field{Center, Center, Nothing}(grid) - sⁿ = Field{Center, Center, Nothing}(grid) - ∂t_s = Field{Center, Center, Nothing}(grid) - - # Initial "at-rest" conditions - set!(sⁿ, 1) - set!(s⁻, 1) - - fill_halo_regions!((s⁻, sⁿ)) - - launch!(architecture(grid), grid, :xy,_update_z_star!, - ΔzF, ΔzC, grid.Δzᵃᵃᶠ, grid.Δzᵃᵃᶜ, sⁿ, Val(grid.Nz)) - - fill_halo_regions!((ΔzF, ΔzC)) - - Δzᵃᵃᶠ = GeneralizedVerticalCoordinate(ZStar(), grid.Δzᵃᵃᶠ, ΔzF, s⁻, sⁿ, ∂t_s) - Δzᵃᵃᶜ = GeneralizedVerticalCoordinate(ZStar(), grid.Δzᵃᵃᶜ, ΔzC, s⁻, sⁿ, ∂t_s) - - args = [] - for prop in propertynames(grid) - if prop == :Δzᵃᵃᶠ - push!(args, Δzᵃᵃᶠ) - elseif prop == :Δzᵃᵃᶜ - push!(args, Δzᵃᵃᶜ) - else - push!(args, getproperty(grid, prop)) - end - end - - GridType = getnamewrapper(grid) - - return GridType{TX, TY, TZ}(args...) -end - -# Fallback -update_vertical_coordinate!(model, grid, Δt; kwargs...) = nothing - -function update_vertical_coordinate!(model, grid::ZStarCoordinateGrid, Δt; parameters = tuple(:xy)) - - # Scaling - s⁻ = grid.Δzᵃᵃᶠ.s⁻ - sⁿ = grid.Δzᵃᵃᶠ.sⁿ - ∂t_s = grid.Δzᵃᵃᶠ.∂t_s - - # Moving coordinates - ΔzF = grid.Δzᵃᵃᶠ.Δ - ΔzC = grid.Δzᵃᵃᶜ.Δ - - # Reference coordinates - ΔzF₀ = grid.Δzᵃᵃᶠ.Δr - ΔzC₀ = grid.Δzᵃᵃᶜ.Δr - - # Update vertical coordinate with available parameters - # No need to fill the halo as the scaling is updated _IN_ the halos - # as well - for params in parameters - update_zstar_scaling!(sⁿ, s⁻, ∂t_s, params, model.free_surface, grid, Δt) - - launch!(architecture(grid), grid, params, _update_z_star!, - ΔzF, ΔzC, ΔzF₀, ΔzC₀, sⁿ, Val(grid.Nz)) - end - - return nothing -end - -update_zstar_scaling!(sⁿ, s⁻, ∂t_s, params, fs, grid, Δt) = - launch!(architecture(grid), grid, params, _update_zstar_scaling!, - sⁿ, s⁻, ∂t_s, fs.η, grid, Δt) - -@kernel function _update_zstar_scaling!(sⁿ, s⁻, ∂t_s, η, grid, Δt) - i, j = @index(Global, NTuple) - bottom = bottom_height(i, j, grid) - @inbounds begin - h = (bottom + η[i, j, grid.Nz+1]) / bottom - - # update current and previous scaling - s⁻[i, j, 1] = sⁿ[i, j, 1] - sⁿ[i, j, 1] = h - - # Scaling derivative - ∂t_s[i, j, 1] = (h - s⁻[i, j, 1]) / Δt / h - end -end - -bottom_height(i, j, grid) = grid.Lz -bottom_height(i, j, grid::ImmersedBoundaryGrid) = @inbounds - grid.immersed_boundary.bottom_height[i, j, 1] - -@kernel function _update_z_star!(ΔzF, ΔzC, ΔzF₀, ΔzC₀, sⁿ, ::Val{Nz}) where Nz - i, j = @index(Global, NTuple) - @unroll for k in 1:Nz+1 - @inbounds ΔzF[i, j, k] = sⁿ[i, j, 1] * ΔzF₀[k] - @inbounds ΔzC[i, j, k] = sⁿ[i, j, 1] * ΔzC₀[k] - end -end +##### +##### General implementation +##### -@kernel function _update_z_star!(ΔzF, ΔzC, ΔzF₀::Number, ΔzC₀::Number, sⁿ, ::Val{Nz}) where Nz - i, j = @index(Global, NTuple) - @unroll for k in 1:Nz+1 - @inbounds ΔzF[i, j, k] = sⁿ[i, j, 1] * ΔzF₀ - @inbounds ΔzC[i, j, k] = sⁿ[i, j, 1] * ΔzC₀ - end -end +GeneralizedSpacingGrid(grid, coord) = grid +update_vertical_spacing!(model, grid, Δt; kwargs...) = nothing -import Oceananigans.Operators: Δzᶜᶜᶠ, Δzᶜᶜᶜ, Δzᶜᶠᶠ, Δzᶜᶠᶜ, Δzᶠᶜᶠ, Δzᶠᶜᶜ, Δzᶠᶠᶠ, Δzᶠᶠᶜ +##### +##### Vertical spacings for a generalized vertical coordinate system +##### # Very bad for GPU performance!!! (z-values are not coalesced in memory for z-derivatives anymore) # TODO: make z-direction local in memory by not using Fields -@inline Δzᶜᶜᶠ(i, j, k, grid::ZStarCoordinateGrid) = @inbounds grid.Δzᵃᵃᶠ.Δ[i, j, k] -@inline Δzᶜᶜᶜ(i, j, k, grid::ZStarCoordinateGrid) = @inbounds grid.Δzᵃᵃᶜ.Δ[i, j, k] - -@inline Δzᶜᶠᶠ(i, j, k, grid::ZStarCoordinateGrid) = ℑyᵃᶠᵃ(i, j, k, grid, grid.Δzᵃᵃᶠ.Δ) -@inline Δzᶜᶠᶜ(i, j, k, grid::ZStarCoordinateGrid) = ℑyᵃᶠᵃ(i, j, k, grid, grid.Δzᵃᵃᶜ.Δ) +@inline Δzᶜᶜᶠ(i, j, k, grid::GeneralizedSpacingGrid) = @inbounds grid.Δzᵃᵃᶠ.Δ[i, j, k] +@inline Δzᶜᶜᶜ(i, j, k, grid::GeneralizedSpacingGrid) = @inbounds grid.Δzᵃᵃᶜ.Δ[i, j, k] -@inline Δzᶠᶜᶠ(i, j, k, grid::ZStarCoordinateGrid) = ℑxᶠᵃᵃ(i, j, k, grid, grid.Δzᵃᵃᶠ.Δ) -@inline Δzᶠᶜᶜ(i, j, k, grid::ZStarCoordinateGrid) = ℑxᶠᵃᵃ(i, j, k, grid, grid.Δzᵃᵃᶜ.Δ) +@inline Δzᶜᶠᶠ(i, j, k, grid::GeneralizedSpacingGrid) = ℑyᵃᶠᵃ(i, j, k, grid, grid.Δzᵃᵃᶠ.Δ) +@inline Δzᶜᶠᶜ(i, j, k, grid::GeneralizedSpacingGrid) = ℑyᵃᶠᵃ(i, j, k, grid, grid.Δzᵃᵃᶜ.Δ) -@inline Δzᶠᶠᶠ(i, j, k, grid::ZStarCoordinateGrid) = ℑxyᶠᶠᵃ(i, j, k, grid, grid.Δzᵃᵃᶠ.Δ) -@inline Δzᶠᶠᶜ(i, j, k, grid::ZStarCoordinateGrid) = ℑxyᶠᶠᵃ(i, j, k, grid, grid.Δzᵃᵃᶜ.Δ) +@inline Δzᶠᶜᶠ(i, j, k, grid::GeneralizedSpacingGrid) = ℑxᶠᵃᵃ(i, j, k, grid, grid.Δzᵃᵃᶠ.Δ) +@inline Δzᶠᶜᶜ(i, j, k, grid::GeneralizedSpacingGrid) = ℑxᶠᵃᵃ(i, j, k, grid, grid.Δzᵃᵃᶜ.Δ) -# Adding the slope to the momentum-RHS -@inline horizontal_surface_slope_x(i, j, k, grid, args...) = zero(grid) -@inline horizontal_surface_slope_y(i, j, k, grid, args...) = zero(grid) +@inline Δzᶠᶠᶠ(i, j, k, grid::GeneralizedSpacingGrid) = ℑxyᶠᶠᵃ(i, j, k, grid, grid.Δzᵃᵃᶠ.Δ) +@inline Δzᶠᶠᶜ(i, j, k, grid::GeneralizedSpacingGrid) = ℑxyᶠᶠᵃ(i, j, k, grid, grid.Δzᵃᵃᶜ.Δ) -@inline horizontal_surface_slope_x(i, j, k, grid::ZStarCoordinateGrid, free_surface, ::Nothing, model_fields) = zero(grid) -@inline horizontal_surface_slope_y(i, j, k, grid::ZStarCoordinateGrid, free_surface, ::Nothing, model_fields) = zero(grid) +##### +##### Vertical velocity of the Δ-surfaces to be included in the continuity equation +##### -@inline η_times_zᶜᶜᶜ(i, j, k, grid, η) = @inbounds η[i, j, grid.Nz+1] * (1 + grid.zᵃᵃᶜ[k] / bottom_height(i, j, grid)) +∂t_∂s_grid(i, j, k, grid::GeneralizedSpacingGrid) = grid.Δzᵃᵃᶜ.∂t_∂s[i, j, k] -@inline η_slope_xᶠᶜᶜ(i, j, k, grid, free_surface) = @inbounds ∂xᶠᶜᶜ(i, j, k, grid, η_times_zᶜᶜᶜ, free_surface.η) -@inline η_slope_yᶜᶠᶜ(i, j, k, grid, free_surface) = @inbounds ∂yᶜᶠᶜ(i, j, k, grid, η_times_zᶜᶜᶜ, free_surface.η) +##### +##### Additional terms to be included in the momentum equations (fallbacks) +##### -@inline horizontal_surface_slope_x(i, j, k, grid::ZStarCoordinateGrid, free_surface, buoyancy, model_fields) = - ℑxᶠᵃᵃ(i, j, k, grid, buoyancy_perturbationᶜᶜᶜ, buoyancy.model, model_fields) * η_slope_xᶠᶜᶜ(i, j, k, grid, free_surface) +@inline grid_slope_contribution_x(i, j, k, grid, args...) = zero(grid) +@inline grid_slope_contribution_y(i, j, k, grid, args...) = zero(grid) -@inline horizontal_surface_slope_y(i, j, k, grid::ZStarCoordinateGrid, free_surface, buoyancy, model_fields) = - ℑyᵃᶠᵃ(i, j, k, grid, buoyancy_perturbationᶜᶜᶜ, buoyancy.model, model_fields) * η_slope_yᶜᶠᶜ(i, j, k, grid, free_surface) +@inline grid_slope_contribution_x(i, j, k, grid::GeneralizedSpacingGrid, free_surface, ::Nothing, model_fields) = zero(grid) +@inline grid_slope_contribution_y(i, j, k, grid::GeneralizedSpacingGrid, free_surface, ::Nothing, model_fields) = zero(grid) ##### -##### In the Z-star coordinate framework the prognostic field is sθ! +##### Handling tracer update in generalized vertical coordinates (we update sθ) ##### -@kernel function ab2_step_tracer_zstar!(θ, sⁿ, s⁻, Δt, χ, Gⁿ, G⁻) +@kernel function _ab2_step_tracer_generalized_spacing!(θ, sⁿ, s⁻, Δt, χ, Gⁿ, G⁻) i, j, k = @index(Global, NTuple) FT = eltype(χ) @@ -232,21 +113,14 @@ import Oceananigans.Operators: Δzᶜᶜᶠ, Δzᶜᶜᶜ, Δzᶜᶠᶠ, Δzᶜ oh_point_five = convert(FT, 0.5) @inbounds begin - ∂t_sθ = (one_point_five + χ) * Gⁿ[i, j, k] - (oh_point_five + χ) * G⁻[i, j, k] - θ[i, j, k] = s⁻[i, j, 1] * θ[i, j, k] / sⁿ[i, j, 1] + convert(FT, Δt) * ∂t_sθ + ∂t_∂sθ = (one_point_five + χ) * Gⁿ[i, j, k] - (oh_point_five + χ) * G⁻[i, j, k] + θ[i, j, k] = s⁻[i, j, k] * θ[i, j, k] / sⁿ[i, j, k] + convert(FT, Δt) * ∂t_∂sθ end end -ab2_step_tracer_field!(tracer_field, grid::ZStarCoordinateGrid, Δt, χ, Gⁿ, G⁻) = - launch!(architecture(grid), grid, :xyz, ab2_step_tracer_zstar!, +ab2_step_tracer_field!(tracer_field, grid::GeneralizedSpacingGrid, Δt, χ, Gⁿ, G⁻) = + launch!(architecture(grid), grid, :xyz, _ab2_step_tracer_generalized_spacing!, tracer_field, grid.Δzᵃᵃᶠ.sⁿ, grid.Δzᵃᵃᶠ.s⁻, - Δt, χ, Gⁿ, G⁻) - -# When performing divergence upwinding we must include the -# metric term - -import Oceananigans.Advection: ∂t_∂s_grid - -∂t_∂s_grid(i, j, k, grid::GeneralizedCoordinateGrid) = grid.Δzᵃᵃᶜ.∂t_s[i, j, k] + Δt, χ, Gⁿ, G⁻) \ No newline at end of file diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl index a4ce0002b7..cb4a547f57 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl @@ -92,7 +92,7 @@ Keyword arguments - `velocities`: The model velocities. Default: `nothing`. - `pressure`: Hydrostatic pressure field. Default: `nothing`. - `diffusivity_fields`: Diffusivity fields. Default: `nothing`. - - `vertical_coordinate`: choice between the default `ZCoordinate` and the free-surface following `ZStarCoordinate` + - `vertical_coordinate`: choice between the default `ZCoordinate` and the free-surface following `ZStarSpacing` - `auxiliary_fields`: `NamedTuple` of auxiliary fields. Default: `nothing`. """ function HydrostaticFreeSurfaceModel(; grid, @@ -123,7 +123,7 @@ function HydrostaticFreeSurfaceModel(; grid, throw(ArgumentError("Generalized vertical coordinates are supported only for the vector-invariant form of the momentum equations")) end - grid = !isnothing(free_surface) ? GeneralizedCoordinateGrid(grid, vertical_coordinate) : grid + grid = !isnothing(free_surface) ? GeneralizedSpacingGrid(grid, vertical_coordinate) : grid arch = architecture(grid) @apply_regionally momentum_advection = validate_momentum_advection(momentum_advection, grid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_tendency_kernel_functions.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_tendency_kernel_functions.jl index 24492169f7..13133103e6 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_tendency_kernel_functions.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_tendency_kernel_functions.jl @@ -48,7 +48,7 @@ implicitly during time-stepping. - explicit_barotropic_pressure_x_gradient(i, j, k, grid, free_surface) - x_f_cross_U(i, j, k, grid, coriolis, velocities) - ∂xᶠᶜᶜ(i, j, k, grid, hydrostatic_pressure_anomaly) - - horizontal_surface_slope_x(i, j, k, grid, free_surface, buoyancy, model_fields) + - grid_slope_contribution_x(i, j, k, grid, free_surface, buoyancy, model_fields) - ∂ⱼ_τ₁ⱼ(i, j, k, grid, closure, diffusivities, clock, model_fields, buoyancy) - immersed_∂ⱼ_τ₁ⱼ(i, j, k, grid, velocities, u_immersed_bc, closure, diffusivities, clock, model_fields) + forcings.u(i, j, k, grid, clock, hydrostatic_prognostic_fields(velocities, free_surface, tracers))) @@ -88,7 +88,7 @@ implicitly during time-stepping. - explicit_barotropic_pressure_y_gradient(i, j, k, grid, free_surface) - y_f_cross_U(i, j, k, grid, coriolis, velocities) - ∂yᶜᶠᶜ(i, j, k, grid, hydrostatic_pressure_anomaly) - - horizontal_surface_slope_y(i, j, k, grid, free_surface, buoyancy, model_fields) + - grid_slope_contribution_y(i, j, k, grid, free_surface, buoyancy, model_fields) - ∂ⱼ_τ₂ⱼ(i, j, k, grid, closure, diffusivities, clock, model_fields, buoyancy) - immersed_∂ⱼ_τ₂ⱼ(i, j, k, grid, velocities, v_immersed_bc, closure, diffusivities, clock, model_fields) + forcings.v(i, j, k, grid, clock, model_fields)) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 6608fd2b7e..29b7d88863 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -392,11 +392,11 @@ setup_split_explicit_tendency!(auxiliary, grid, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) wait_free_surface_communication!(free_surface, arch) = nothing -update_zstar_scaling!(sⁿ, s⁻, ∂t_s, params, fs::SplitExplicitFreeSurface, grid, Δt) = +update_zstar_scaling!(sⁿ, s⁻, ∂t_∂s, params, fs::SplitExplicitFreeSurface, grid, Δt) = launch!(architecture(grid), grid, params, _update_zstar_split_explicit_scaling!, - sⁿ, s⁻, ∂t_s, fs.η, fs.state.U̅, fs.state.V̅, grid) + sⁿ, s⁻, ∂t_∂s, fs.η, fs.state.U̅, fs.state.V̅, grid) -@kernel function _update_zstar_split_explicit_scaling!(sⁿ, s⁻, ∂t_s, η, U̅, V̅, grid) +@kernel function _update_zstar_split_explicit_scaling!(sⁿ, s⁻, ∂t_∂s, η, U̅, V̅, grid) i, j = @index(Global, NTuple) bottom = bottom_height(i, j, grid) @inbounds begin @@ -407,6 +407,6 @@ update_zstar_scaling!(sⁿ, s⁻, ∂t_s, params, fs::SplitExplicitFreeSurface, sⁿ[i, j, 1] = h # ∂(H + η / H)/∂t = - ∇ ⋅ ∫udz / H - ∂t_s[i, j, 1] = - div_xyᶜᶜᶜ(i, j, 1, grid, U̅, V̅) / bottom / h + ∂t_∂s[i, j, 1] = - div_xyᶜᶜᶜ(i, j, 1, grid, U̅, V̅) / bottom / h end end diff --git a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl index 8ccf3d5f81..b37590aa45 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl @@ -76,7 +76,7 @@ function compute_auxiliaries!(model::HydrostaticFreeSurfaceModel, Δt; w_paramet diffusivity = model.diffusivity_fields for (wpar, ppar, κpar) in zip(w_parameters, p_parameters, κ_parameters) - update_vertical_coordinate!(model, grid, Δt; parameters = wpar) + update_vertical_spacing!(model, grid, Δt; parameters = wpar) compute_w_from_continuity!(model; parameters = wpar) compute_diffusivities!(diffusivity, closure, model; parameters = κpar) update_hydrostatic_pressure!(model.pressure.pHY′, architecture(grid), @@ -89,5 +89,5 @@ end # Do not update if rigid lid!! const RigidLidModel = HydrostaticFreeSurfaceModel{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Nothing} -update_vertical_coordinate!(::RigidLidModel, ::ZStarCoordinateGrid; kwargs...) = nothing +update_vertical_spacing!(::RigidLidModel, ::ZStarSpacingGrid; kwargs...) = nothing update_tracer_thickness!(tracers, grid) = nothing diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_coordinate.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_coordinate.jl new file mode 100644 index 0000000000..b9f33d3040 --- /dev/null +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_coordinate.jl @@ -0,0 +1,144 @@ + +##### +##### ZStar coordinate and associated grid types +##### + +""" free-surface following vertical coordinate """ +struct ZStar end + +const ZStarSpacing = GeneralizedVerticalSpacing{<:ZStar} + +Grids.coordinate_summary(Δ::ZStarSpacing, name) = + @sprintf("Free-surface following with Δ%s=%s", name, prettysummary(Δ.Δr)) + +const ZStarSpacingGrid = GeneralizedSpacingGrid{<:ZStar} + +function GeneralizedSpacingGrid(grid::ImmersedBoundaryGrid, ::ZStar) + underlying_grid = GeneralizedSpacingGrid(grid.underlying_grid, ZStar()) + active_cells_map = !isnothing(grid.interior_active_cells) + + return ImmersedBoundaryGrid(underlying_grid, grid.immersed_boundary; active_cells_map) +end + +# Replacing the z-coordinate with a moving vertical coordinate, defined by its reference spacing, +# the actual vertical spacing and a scaling +function GeneralizedSpacingGrid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, ::ZStar) where {FT, TX, TY, TZ} + + # Memory layout for Dz spacings is local in z + ΔzF = ZFaceField(grid) + ΔzC = CenterField(grid) + s⁻ = Field{Center, Center, Nothing}(grid) + sⁿ = Field{Center, Center, Nothing}(grid) + ∂t_∂s = Field{Center, Center, Nothing}(grid) + + # Initial "at-rest" conditions + set!(sⁿ, 1) + set!(s⁻, 1) + + fill_halo_regions!((s⁻, sⁿ)) + + launch!(architecture(grid), grid, :xy,_update_z_star!, + ΔzF, ΔzC, grid.Δzᵃᵃᶠ, grid.Δzᵃᵃᶜ, sⁿ, Val(grid.Nz)) + + fill_halo_regions!((ΔzF, ΔzC)) + + Δzᵃᵃᶠ = GeneralizedVerticalSpacing(ZStar(), grid.Δzᵃᵃᶠ, ΔzF, s⁻, sⁿ, ∂t_∂s) + Δzᵃᵃᶜ = GeneralizedVerticalSpacing(ZStar(), grid.Δzᵃᵃᶜ, ΔzC, s⁻, sⁿ, ∂t_∂s) + + args = [] + for prop in propertynames(grid) + if prop == :Δzᵃᵃᶠ + push!(args, Δzᵃᵃᶠ) + elseif prop == :Δzᵃᵃᶜ + push!(args, Δzᵃᵃᶜ) + else + push!(args, getproperty(grid, prop)) + end + end + + GridType = getnamewrapper(grid) + + return GridType{TX, TY, TZ}(args...) +end + +##### +##### ZStar-specific vertical spacings update +##### + +function update_vertical_spacing!(model, grid::ZStarSpacingGrid, Δt; parameters = :xy) + + # Scaling + s⁻ = grid.Δzᵃᵃᶠ.s⁻ + sⁿ = grid.Δzᵃᵃᶠ.sⁿ + ∂t_∂s = grid.Δzᵃᵃᶠ.∂t_∂s + + # Generalized vertical spacing + ΔzF = grid.Δzᵃᵃᶠ.Δ + ΔzC = grid.Δzᵃᵃᶜ.Δ + + # Reference (non moving) spacing + ΔzF₀ = grid.Δzᵃᵃᶠ.Δr + ΔzC₀ = grid.Δzᵃᵃᶜ.Δr + + # Update vertical spacing with available parameters + # No need to fill the halo as the scaling is updated _IN_ the halos + update_zstar_scaling!(sⁿ, s⁻, ∂t_∂s, parameters, model.free_surface, grid, Δt) + + launch!(architecture(grid), grid, parameters, _update_z_star!, + ΔzF, ΔzC, ΔzF₀, ΔzC₀, sⁿ, Val(grid.Nz)) + + return nothing +end + +update_zstar_scaling!(sⁿ, s⁻, ∂t_∂s, params, fs, grid, Δt) = + launch!(architecture(grid), grid, params, _update_zstar_scaling!, + sⁿ, s⁻, ∂t_∂s, fs.η, grid, Δt) + +@kernel function _update_zstar_scaling!(sⁿ, s⁻, ∂t_∂s, η, grid, Δt) + i, j = @index(Global, NTuple) + bottom = bottom_height(i, j, grid) + @inbounds begin + h = (bottom + η[i, j, grid.Nz+1]) / bottom + + # update current and previous scaling + s⁻[i, j, 1] = sⁿ[i, j, 1] + sⁿ[i, j, 1] = h + + # Scaling derivative + ∂t_∂s[i, j, 1] = (h - s⁻[i, j, 1]) / Δt / h + end +end + +bottom_height(i, j, grid) = grid.Lz +bottom_height(i, j, grid::ImmersedBoundaryGrid) = @inbounds - grid.immersed_boundary.bottom_height[i, j, 1] + +@kernel function _update_z_star!(ΔzF, ΔzC, ΔzF₀, ΔzC₀, sⁿ, ::Val{Nz}) where Nz + i, j = @index(Global, NTuple) + @unroll for k in 1:Nz+1 + @inbounds ΔzF[i, j, k] = sⁿ[i, j, 1] * ΔzF₀[k] + @inbounds ΔzC[i, j, k] = sⁿ[i, j, 1] * ΔzC₀[k] + end +end + +@kernel function _update_z_star!(ΔzF, ΔzC, ΔzF₀::Number, ΔzC₀::Number, sⁿ, ::Val{Nz}) where Nz + i, j = @index(Global, NTuple) + @unroll for k in 1:Nz+1 + @inbounds ΔzF[i, j, k] = sⁿ[i, j, 1] * ΔzF₀ + @inbounds ΔzC[i, j, k] = sⁿ[i, j, 1] * ΔzC₀ + end +end + +##### +##### ZStar-specific implementation of the additional terms to be included in the momentum equations +##### + +@inline η_surfaceᶜᶜᶜ(i, j, k, grid, η) = @inbounds η[i, j, grid.Nz+1] * (1 + znode(i, j, k, grid, c, c, c) / bottom_height(i, j, grid)) + +@inline slope_xᶠᶜᶜ(i, j, k, grid, free_surface) = @inbounds ∂xᶠᶜᶜ(i, j, k, grid, η_surfaceᶜᶜᶜ, free_surface.η) +@inline slope_yᶜᶠᶜ(i, j, k, grid, free_surface) = @inbounds ∂yᶜᶠᶜ(i, j, k, grid, η_surfaceᶜᶜᶜ, free_surface.η) + +@inline grid_slope_contribution_x(i, j, k, grid::ZStarSpacingGrid, free_surface, buoyancy, model_fields) = + ℑxᶠᵃᵃ(i, j, k, grid, buoyancy_perturbationᶜᶜᶜ, buoyancy.model, model_fields) * slope_xᶠᶜᶜ(i, j, k, grid, free_surface) + +@inline grid_slope_contribution_y(i, j, k, grid::ZStarSpacingGrid, free_surface, buoyancy, model_fields) = + ℑyᵃᶠᵃ(i, j, k, grid, buoyancy_perturbationᶜᶜᶜ, buoyancy.model, model_fields) * slope_yᶜᶠᶜ(i, j, k, grid, free_surface) From 47ab44b6db63b598d51a7226a7f01d155e52c839 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 8 Jan 2024 12:53:42 -0500 Subject: [PATCH 120/567] bugfix --- .../HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index 670c784bba..a88bf409bf 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -95,7 +95,7 @@ function SplitExplicitFreeSurface(grid; gravitational_acceleration = g_Earth, η = ZFaceField(grid, indices = (:, :, size(grid, 3)+1)) gravitational_acceleration = convert(eltype(grid), gravitational_acceleration) - return SplitExplicitFreeSurface(η, SplitExplicitState(grid), SplitExplicitAuxiliaryFields(grid), + return SplitExplicitFreeSurface(η, SplitExplicitState(grid, settings.timestepper), SplitExplicitAuxiliaryFields(grid), gravitational_acceleration, settings) end From f6c2893ccc2a8f1b9ea4261f4f5f838032fffd28 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 8 Jan 2024 13:04:52 -0500 Subject: [PATCH 121/567] changes... --- .../vector_invariant_cross_upwinding.jl | 10 +- .../vector_invariant_self_upwinding.jl | 6 +- .../HydrostaticFreeSurfaceModels.jl | 4 +- ...distributed_split_explicit_free_surface.jl | 19 +--- .../explicit_free_surface.jl | 25 +++++ ...ate.jl => generalized_vertical_spacing.jl} | 75 +++++++------ .../hydrostatic_free_surface_model.jl | 42 +++---- ..._free_surface_tendency_kernel_functions.jl | 24 ---- .../split_explicit_free_surface.jl | 25 +---- .../split_explicit_free_surface_kernels.jl | 54 +++++---- ...ordinate.jl => z_star_vertical_spacing.jl} | 5 +- ...ulti_region_split_explicit_free_surface.jl | 19 +--- validation/z_star_coordinate/lock_release.jl | 17 ++- .../z_star_coordinate/tracer_conservation.jl | 103 ++++++++++-------- 14 files changed, 207 insertions(+), 221 deletions(-) rename src/Models/HydrostaticFreeSurfaceModels/{generalized_vertical_coordinate.jl => generalized_vertical_spacing.jl} (61%) rename src/Models/HydrostaticFreeSurfaceModels/{z_star_vertical_coordinate.jl => z_star_vertical_spacing.jl} (94%) diff --git a/src/Advection/vector_invariant_cross_upwinding.jl b/src/Advection/vector_invariant_cross_upwinding.jl index 6ed57a40e9..d88443b045 100644 --- a/src/Advection/vector_invariant_cross_upwinding.jl +++ b/src/Advection/vector_invariant_cross_upwinding.jl @@ -21,10 +21,11 @@ @inbounds û = u[i, j, k] δ_stencil = scheme.upwinding.divergence_stencil - δᴸ = _left_biased_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, flux_div_xyᶜᶜᶜ, δ_stencil, u, v) - δᴿ = _right_biased_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, flux_div_xyᶜᶜᶜ, δ_stencil, u, v) + δᴸ = _left_biased_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, flux_div_xyᶜᶜᶜ, δ_stencil, u, v) + δᴿ = _right_biased_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, flux_div_xyᶜᶜᶜ, δ_stencil, u, v) + ∂ts = _symmetric_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, cross_scheme, V_times_∂t_∂s_grid) - return upwind_biased_product(û, δᴸ, δᴿ) + û * ℑxᶠᵃᵃ(i, j, k, grid, V_times_∂t_∂s_grid) + return upwind_biased_product(û, δᴸ, δᴿ) + û * ∂ts end @inline function upwinded_divergence_flux_Vᶜᶠᶜ(i, j, k, grid, scheme::VectorInvariantCrossVerticalUpwinding, u, v) @@ -33,6 +34,7 @@ end δᴸ = _left_biased_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, flux_div_xyᶜᶜᶜ, δ_stencil, u, v) δᴿ = _right_biased_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, flux_div_xyᶜᶜᶜ, δ_stencil, u, v) + ∂ts = _symmetric_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, cross_scheme, V_times_∂t_∂s_grid) - return upwind_biased_product(v̂, δᴸ, δᴿ) + v̂ * ℑyᵃᶠᵃ(i, j, k, grid, V_times_∂t_∂s_grid) + return upwind_biased_product(v̂, δᴸ, δᴿ) + v̂ * ∂ts end diff --git a/src/Advection/vector_invariant_self_upwinding.jl b/src/Advection/vector_invariant_self_upwinding.jl index 5704299f45..f35bb775af 100644 --- a/src/Advection/vector_invariant_self_upwinding.jl +++ b/src/Advection/vector_invariant_self_upwinding.jl @@ -27,10 +27,11 @@ @inbounds û = u[i, j, k] δvˢ = _symmetric_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, cross_scheme, δy_V, u, v) + ∂ts = _symmetric_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, cross_scheme, V_times_∂t_∂s_grid) δuᴸ = _left_biased_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, δx_U, δU_stencil, u, v) δuᴿ = _right_biased_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, δx_U, δU_stencil, u, v) - return upwind_biased_product(û, δuᴸ, δuᴿ) + û * (δvˢ + ℑxᶠᵃᵃ(i, j, k, grid, V_times_∂t_∂s_grid)) + return upwind_biased_product(û, δuᴸ, δuᴿ) + û * (δvˢ + ∂ts) end @inline function upwinded_divergence_flux_Vᶜᶠᶜ(i, j, k, grid, scheme::VectorInvariantSelfVerticalUpwinding, u, v) @@ -40,10 +41,11 @@ end @inbounds v̂ = v[i, j, k] δuˢ = _symmetric_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, cross_scheme, δx_U, u, v) + ∂ts = _symmetric_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, cross_scheme, V_times_∂t_∂s_grid) δvᴸ = _left_biased_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, δy_V, δV_stencil, u, v) δvᴿ = _right_biased_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, δy_V, δV_stencil, u, v) - return upwind_biased_product(v̂, δvᴸ, δvᴿ) + v̂ * (δuˢ + ℑyᵃᶠᵃ(i, j, k, grid, V_times_∂t_∂s_grid)) + return upwind_biased_product(v̂, δvᴸ, δvᴿ) + v̂ * (δuˢ + ∂ts) end ##### diff --git a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl index 9c7dd74058..db475d0bc0 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl @@ -31,8 +31,8 @@ FreeSurfaceDisplacementField(velocities, free_surface, grid) = ZFaceField(grid, FreeSurfaceDisplacementField(velocities, ::Nothing, grid) = nothing # Generalized vertical coordinate functionality -include("generalized_vertical_coordinate.jl") -include("z_star_vertical_coordinate.jl") +include("generalized_vertical_spacing.jl") +include("z_star_vertical_spacing.jl") include("compute_w_from_continuity.jl") # Rigid-lid diff --git a/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl index 679889a888..7fa334dd24 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl @@ -9,17 +9,6 @@ function SplitExplicitAuxiliaryFields(grid::DistributedGrid) Gᵁ = Field((Face, Center, Nothing), grid) Gⱽ = Field((Center, Face, Nothing), grid) - - Hᶠᶜ = Field((Face, Center, Nothing), grid) - Hᶜᶠ = Field((Center, Face, Nothing), grid) - Hᶜᶜ = Field((Center, Center, Nothing), grid) - - calculate_column_height!(Hᶠᶜ, (Face, Center, Center)) - calculate_column_height!(Hᶜᶠ, (Center, Face, Center)) - - calculate_column_height!(Hᶜᶜ, (Center, Center, Center)) - - fill_halo_regions!((Hᶠᶜ, Hᶜᶠ, Hᶜᶜ)) # In a non-parallel grid we calculate only the interior kernel_size = augmented_kernel_size(grid) @@ -27,13 +16,7 @@ function SplitExplicitAuxiliaryFields(grid::DistributedGrid) kernel_parameters = KernelParameters(kernel_size, kernel_offsets) - return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, kernel_parameters) -end - -"""Integrate z at locations `location` and set! `height`` with the result""" -@inline function calculate_column_height!(height, location) - dz = GridMetricOperation(location, Δz, height.grid) - return sum!(height, dz) + return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, kernel_parameters) end @inline function augmented_kernel_size(grid::DistributedGrid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/explicit_free_surface.jl index 672322068f..0ebdffe562 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/explicit_free_surface.jl @@ -68,3 +68,28 @@ explicit_ab2_step_free_surface!(free_surface, model, Δt, χ) = η[i, j, Nz+1] += Δt * ((FT(1.5) + χ) * Gηⁿ[i, j, Nz+1] - (FT(0.5) + χ) * Gη⁻[i, j, Nz+1]) end end + + +""" +Return the tendency for an explicit free surface at horizontal grid point `i, j`. + +The tendency is called ``G_η`` and defined via + +``` +∂_t η = G_η +``` +""" +@inline function free_surface_tendency(i, j, grid, + velocities, + free_surface, + tracers, + auxiliary_fields, + forcings, + clock) + + k_top = grid.Nz + 1 + model_fields = merge(hydrostatic_fields(velocities, free_surface, tracers), auxiliary_fields) + + return @inbounds ( velocities.w[i, j, k_top] + + forcings.η(i, j, k_top, grid, clock, model_fields)) +end diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_coordinate.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl similarity index 61% rename from src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_coordinate.jl rename to src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index 0d235e06ca..af359f1c45 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_coordinate.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -10,44 +10,41 @@ using Printf import Oceananigans.Operators: Δzᶜᶜᶠ, Δzᶜᶜᶜ, Δzᶜᶠᶠ, Δzᶜᶠᶜ, Δzᶠᶜᶠ, Δzᶠᶜᶜ, Δzᶠᶠᶠ, Δzᶠᶠᶜ import Oceananigans.Advection: ∂t_∂s_grid +import Oceananigans.Architectures: arch_array """ - GeneralizedVerticalSpacing{R, S, Z} + GeneralizedVerticalSpacing{D, R, S, Z} -spacings for a generalized vertical coordinate system. -The reference coordinate is stored in `Δr`, while `Δ` contains the z-coordinate. -The `s⁻`, `sⁿ` and `∂t_∂s` fields are the vertical derivative of the vertical coordinate (∂Δ/∂Δr) +spacings for a generalized vertical coordinate system. The reference (non-moving) spacings are stored in `Δr`. +`Δ` contains the spacings associated with the moving coordinate system. +`s⁻`, `sⁿ` and `∂t_∂s` fields are the vertical derivative of the vertical coordinate (∂Δ/∂Δr) at timestep `n-1` and `n` and it's time derivative. -`denomination` contains the "type" of generalized vertical coordinate, for example: -- Zstar: free-surface following -- sigma: terrain following +`denomination` contains the "type" of generalized vertical coordinate (the only one implemented is `ZStar`) """ -struct GeneralizedVerticalSpacing{D, R, Z, S} - denomination :: D # The type of generalized coordinate - Δr :: R # Reference _non moving_ vertical coordinate (one-dimensional) - Δ :: Z # moving vertical coordinate (three-dimensional) - s⁻ :: S # scaling term = ∂Δ/∂Δr at the start of the time step - sⁿ :: S # scaling term = ∂Δ/∂Δr at the end of the time step - ∂t_∂s :: S # Time derivative of the vertical coordinate scaling divided by the sⁿ +struct GeneralizedVerticalSpacing{D, R, Z, S, SN, ST} + denomination :: D # The type of generalized coordinate + Δr :: R # Reference _non moving_ vertical coordinate (one-dimensional) + Δ :: Z # moving vertical coordinate (three-dimensional) + s⁻ :: S # scaling term = ∂Δ/∂Δr at the start of the time step + sⁿ :: SN # scaling term = ∂Δ/∂Δr at the end of the time step + ∂t_∂s :: ST # Time derivative of the vertical coordinate scaling end Adapt.adapt_structure(to, coord::GeneralizedVerticalSpacing) = - GeneralizedVerticalSpacing(nothing, - Adapt.adapt(to, coord.Δr), - Adapt.adapt(to, coord.Δ), - Adapt.adapt(to, coord.s⁻), - Adapt.adapt(to, coord.sⁿ), - Adapt.adapt(to, coord.∂t_∂s)) - -import Oceananigans.Architectures: arch_array + GeneralizedVerticalSpacing(coord.denomination, + Adapt.adapt(to, coord.Δr), + Adapt.adapt(to, coord.Δ), + Adapt.adapt(to, coord.s⁻), + Adapt.adapt(to, coord.sⁿ), + Adapt.adapt(to, coord.∂t_∂s)) arch_array(arch, coord::GeneralizedVerticalSpacing) = GeneralizedVerticalSpacing(coord.denomination, - arch_array(arch, coord.Δr), - coord.Δ, - coord.s⁻, - coord.sⁿ, - coord.∂t_∂s) + arch_array(arch, coord.Δr), + coord.Δ, + coord.s⁻, + coord.sⁿ, + coord.∂t_∂s) const GeneralizedSpacingRG{D} = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:GeneralizedVerticalSpacing{D}} where D const GeneralizedSpacingLLG{D} = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:GeneralizedVerticalSpacing{D}} where D @@ -57,9 +54,6 @@ const GeneralizedSpacingImmersedGrid{D} = ImmersedBoundaryGrid{<:Any, <:Any, <:A const GeneralizedSpacingGrid{D} = Union{GeneralizedSpacingUnderlyingGrid{D}, GeneralizedSpacingImmersedGrid{D}} where D -""" geopotential following vertical coordinate """ -struct Z end - ##### ##### General implementation ##### @@ -85,6 +79,24 @@ update_vertical_spacing!(model, grid, Δt; kwargs...) = nothing @inline Δzᶠᶠᶠ(i, j, k, grid::GeneralizedSpacingGrid) = ℑxyᶠᶠᵃ(i, j, k, grid, grid.Δzᵃᵃᶠ.Δ) @inline Δzᶠᶠᶜ(i, j, k, grid::GeneralizedSpacingGrid) = ℑxyᶠᶠᵃ(i, j, k, grid, grid.Δzᵃᵃᶜ.Δ) +@inline Δz_reference(i, j, k, Δz::Number) = Δz +@inline Δz_reference(i, j, k, Δz::AbstractVector) = Δz[k] + +@inline Δz_reference(i, j, k, Δz::GeneralizedVerticalSpacing{<:Any, <:Number}) = Δz.Δr +@inline Δz_reference(i, j, k, Δz::GeneralizedVerticalSpacing) = Δz.Δr[k] + +@inline Δzᶜᶜᶠ_reference(i, j, k, grid) = Δz_reference(i, j, k, grid.Δzᵃᵃᶠ) +@inline Δzᶜᶜᶜ_reference(i, j, k, grid) = Δz_reference(i, j, k, grid.Δzᵃᵃᶜ) + +@inline Δzᶜᶠᶠ_reference(i, j, k, grid) = ℑyᵃᶠᵃ(i, j, k, grid, Δzᶜᶜᶠ_reference) +@inline Δzᶜᶠᶜ_reference(i, j, k, grid) = ℑyᵃᶠᵃ(i, j, k, grid, Δzᶜᶜᶜ_reference) + +@inline Δzᶠᶜᶠ_reference(i, j, k, grid) = ℑxᶠᵃᵃ(i, j, k, grid, Δzᶜᶜᶠ_reference) +@inline Δzᶠᶜᶜ_reference(i, j, k, grid) = ℑxᶠᵃᵃ(i, j, k, grid, Δzᶜᶜᶜ_reference) + +@inline Δzᶠᶠᶠ_reference(i, j, k, grid) = ℑxyᶠᶠᵃ(i, j, k, grid, Δzᶜᶜᶠ_reference) +@inline Δzᶠᶠᶜ_reference(i, j, k, grid) = ℑxyᶠᶠᵃ(i, j, k, grid, Δzᶜᶜᶜ_reference) + ##### ##### Vertical velocity of the Δ-surfaces to be included in the continuity equation ##### @@ -98,9 +110,6 @@ update_vertical_spacing!(model, grid, Δt; kwargs...) = nothing @inline grid_slope_contribution_x(i, j, k, grid, args...) = zero(grid) @inline grid_slope_contribution_y(i, j, k, grid, args...) = zero(grid) -@inline grid_slope_contribution_x(i, j, k, grid::GeneralizedSpacingGrid, free_surface, ::Nothing, model_fields) = zero(grid) -@inline grid_slope_contribution_y(i, j, k, grid::GeneralizedSpacingGrid, free_surface, ::Nothing, model_fields) = zero(grid) - ##### ##### Handling tracer update in generalized vertical coordinates (we update sθ) ##### diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl index cb4a547f57..adaec837df 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl @@ -96,34 +96,34 @@ Keyword arguments - `auxiliary_fields`: `NamedTuple` of auxiliary fields. Default: `nothing`. """ function HydrostaticFreeSurfaceModel(; grid, - clock = Clock{eltype(grid)}(0, 0, 1), - momentum_advection = CenteredSecondOrder(), - tracer_advection = CenteredSecondOrder(), - buoyancy = SeawaterBuoyancy(eltype(grid)), - coriolis = nothing, - free_surface = ImplicitFreeSurface(gravitational_acceleration = g_Earth), - forcing::NamedTuple = NamedTuple(), - closure = nothing, - boundary_conditions::NamedTuple = NamedTuple(), - tracers = (:T, :S), - particles::ParticlesOrNothing = nothing, - biogeochemistry::AbstractBGCOrNothing = nothing, - velocities = nothing, - pressure = nothing, - diffusivity_fields = nothing, - vertical_coordinate = Z(), - auxiliary_fields = NamedTuple() + clock = Clock{eltype(grid)}(0, 0, 1), + momentum_advection = CenteredSecondOrder(), + tracer_advection = CenteredSecondOrder(), + buoyancy = SeawaterBuoyancy(eltype(grid)), + coriolis = nothing, + free_surface = ImplicitFreeSurface(gravitational_acceleration = g_Earth), + forcing::NamedTuple = NamedTuple(), + closure = nothing, + boundary_conditions::NamedTuple = NamedTuple(), + tracers = (:T, :S), + particles::ParticlesOrNothing = nothing, + biogeochemistry::AbstractBGCOrNothing = nothing, + velocities = nothing, + pressure = nothing, + diffusivity_fields = nothing, + generalized_vertical_coordinate = nothing, + auxiliary_fields = NamedTuple() ) # Check halos and throw an error if the grid's halo is too small @apply_regionally validate_model_halo(grid, momentum_advection, tracer_advection, closure) # Introduce z-star coordinates if needed (only is free_surface is not a nothing) - if !(vertical_coordinate isa Z) && !(momentum_advection isa VectorInvariant) && isnothing(velocities) - throw(ArgumentError("Generalized vertical coordinates are supported only for the vector-invariant form of the momentum equations")) - end + # if !isnothing(generalized_vertical_coordinate) && !(momentum_advection isa VectorInvariant) && isnothing(velocities) + # throw(ArgumentError("Generalized vertical coordinates are supported only for the vector-invariant form of the momentum equations")) + # end - grid = !isnothing(free_surface) ? GeneralizedSpacingGrid(grid, vertical_coordinate) : grid + grid = !isnothing(free_surface) ? GeneralizedSpacingGrid(grid, generalized_vertical_coordinate) : grid arch = architecture(grid) @apply_regionally momentum_advection = validate_momentum_advection(momentum_advection, grid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_tendency_kernel_functions.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_tendency_kernel_functions.jl index 13133103e6..557612b28a 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_tendency_kernel_functions.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_tendency_kernel_functions.jl @@ -141,30 +141,6 @@ where `c = C[tracer_index]`. + forcing(i, j, k, grid, clock, model_fields)) end -""" -Return the tendency for an explicit free surface at horizontal grid point `i, j`. - -The tendency is called ``G_η`` and defined via - -``` -∂_t η = G_η -``` -""" -@inline function free_surface_tendency(i, j, grid, - velocities, - free_surface, - tracers, - auxiliary_fields, - forcings, - clock) - - k_top = grid.Nz + 1 - model_fields = merge(hydrostatic_fields(velocities, free_surface, tracers), auxiliary_fields) - - return @inbounds ( velocities.w[i, j, k_top] - + forcings.η(i, j, k_top, grid, clock, model_fields)) -end - @inline function hydrostatic_turbulent_kinetic_energy_tendency(i, j, k, grid, val_tracer_index::Val{tracer_index}, val_tracer_name, diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index 8b78623673..84e44ab302 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -175,17 +175,11 @@ large (or `:xy` in case of a serial computation), and start computing from $(FIELDS) """ -Base.@kwdef struct SplitExplicitAuxiliaryFields{𝒞ℱ, ℱ𝒞, 𝒞𝒞, 𝒦} +Base.@kwdef struct SplitExplicitAuxiliaryFields{𝒞ℱ, ℱ𝒞, 𝒦} "Vertically-integrated slow barotropic forcing function for `U` (`ReducedField` over ``z``)" Gᵁ :: ℱ𝒞 "Vertically-integrated slow barotropic forcing function for `V` (`ReducedField` over ``z``)" Gⱽ :: 𝒞ℱ - "Depth at `(Face, Center)` (`ReducedField` over ``z``)" - Hᶠᶜ :: ℱ𝒞 - "Depth at `(Center, Face)` (`ReducedField` over ``z``)" - Hᶜᶠ :: 𝒞ℱ - "Depth at `(Center, Center)` (`ReducedField` over ``z``)" - Hᶜᶜ :: 𝒞𝒞 "kernel size for barotropic time stepping" kernel_parameters :: 𝒦 end @@ -200,24 +194,9 @@ function SplitExplicitAuxiliaryFields(grid::AbstractGrid) Gᵁ = Field((Face, Center, Nothing), grid) Gⱽ = Field((Center, Face, Nothing), grid) - Hᶠᶜ = Field((Face, Center, Nothing), grid) - Hᶜᶠ = Field((Center, Face, Nothing), grid) - Hᶜᶜ = Field((Center, Center, Nothing), grid) - - dz = GridMetricOperation((Face, Center, Center), Δz, grid) - sum!(Hᶠᶜ, dz) - - dz = GridMetricOperation((Center, Face, Center), Δz, grid) - sum!(Hᶜᶠ, dz) - - dz = GridMetricOperation((Center, Center, Center), Δz, grid) - sum!(Hᶜᶜ, dz) - - fill_halo_regions!((Hᶠᶜ, Hᶜᶠ, Hᶜᶜ)) - kernel_parameters = :xy - return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, kernel_parameters) + return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, kernel_parameters) end """ diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 29b7d88863..a669795b4e 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -138,7 +138,7 @@ using Printf @kernel function split_explicit_free_surface_evolution_kernel!(grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅, averaging_weight, - Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, + Gᵁ, Gⱽ, g, timestepper) i, j = @index(Global, NTuple) k_top = grid.Nz+1 @@ -153,9 +153,12 @@ using Printf end end +@inline dynamic_column_height(i, j, k, grid, η) = bottom_height(i, j, grid) +@inline dynamic_column_height(i, j, k, grid::ZStarSpacingGrid, H, η) = @inbounds bottom_height(i, j, grid) + η[i, j, k] + @kernel function split_explicit_barotropic_velocity_evolution_kernel!(grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅, averaging_weight, - Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, + Gᵁ, Gⱽ, g, timestepper) i, j = @index(Global, NTuple) k_top = grid.Nz+1 @@ -167,8 +170,8 @@ end advance_previous_velocity!(i, j, 1, timestepper, V, Vᵐ⁻¹, Vᵐ⁻²) # ∂τ(U) = - ∇η + G - U[i, j, 1] += Δτ * (- g * Hᶠᶜ[i, j] * ∂xᶠᶜᶠ_η(i, j, k_top, grid, TX, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gᵁ[i, j, 1]) - V[i, j, 1] += Δτ * (- g * Hᶜᶠ[i, j] * ∂yᶜᶠᶠ_η(i, j, k_top, grid, TY, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gⱽ[i, j, 1]) + U[i, j, 1] += Δτ * (- g * dynamic_column_height(i, j, k_top, grid, η) * ∂xᶠᶜᶠ_η(i, j, k_top, grid, TX, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gᵁ[i, j, 1]) + V[i, j, 1] += Δτ * (- g * dynamic_column_height(i, j, k_top, grid, η) * ∂yᶜᶠᶠ_η(i, j, k_top, grid, TY, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gⱽ[i, j, 1]) # time-averaging η̅[i, j, k_top] += averaging_weight * η[i, j, k_top] @@ -184,7 +187,7 @@ function split_explicit_free_surface_substep!(η, state, auxiliary, settings, we Vᵐ⁻¹, Vᵐ⁻² = state.Vᵐ⁻¹, state.Vᵐ⁻² ηᵐ, ηᵐ⁻¹, ηᵐ⁻² = state.ηᵐ, state.ηᵐ⁻¹, state.ηᵐ⁻² η̅, U̅, V̅ = state.η̅, state.U̅, state.V̅ - Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ = auxiliary.Gᵁ, auxiliary.Gⱽ, auxiliary.Hᶠᶜ, auxiliary.Hᶜᶠ + Gᵁ, Gⱽ = auxiliary.Gᵁ, auxiliary.Gⱽ timestepper = settings.timestepper averaging_weight = weights[substep_index] @@ -193,7 +196,7 @@ function split_explicit_free_surface_substep!(η, state, auxiliary, settings, we args = (grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅, averaging_weight, - Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, timestepper) + Gᵁ, Gⱽ, g, timestepper) launch!(arch, grid, parameters, split_explicit_free_surface_evolution_kernel!, args...) launch!(arch, grid, parameters, split_explicit_barotropic_velocity_evolution_kernel!, args...) @@ -203,22 +206,28 @@ end # Barotropic Model Kernels # u_Δz = u * Δz -@kernel function _barotropic_mode_kernel!(U, V, grid, u, v) +# For Zstar vertical spacing the vertical integral includes the dynamic height +# Remember, the vertical coordinate has not yet been updated! +# For this reason the integration has to be performed manually +@kernel function _barotropic_mode_kernel!(U, V, grid, u, v, η̅) i, j = @index(Global, NTuple) + k_top = grid.Nz + 1 + + scaling = dynamic_column_height(i, j, k_top, grid, η̅) / bottom_height(i, j, grid) # hand unroll first loop - @inbounds U[i, j, 1] = Δzᶠᶜᶜ(i, j, 1, grid) * u[i, j, 1] - @inbounds V[i, j, 1] = Δzᶜᶠᶜ(i, j, 1, grid) * v[i, j, 1] + @inbounds U[i, j, 1] = u[i, j, 1] * Δzᶠᶜᶜ_reference(i, j, 1, grid) * scaling + @inbounds V[i, j, 1] = v[i, j, 1] * Δzᶠᶜᶜ_reference(i, j, 1, grid) * scaling @unroll for k in 2:grid.Nz - @inbounds U[i, j, 1] += Δzᶠᶜᶜ(i, j, k, grid) * u[i, j, k] - @inbounds V[i, j, 1] += Δzᶜᶠᶜ(i, j, k, grid) * v[i, j, k] + @inbounds U[i, j, 1] += u[i, j, k] * Δzᶠᶜᶜ_reference(i, j, k, grid) * scaling + @inbounds V[i, j, 1] += v[i, j, k] * Δzᶜᶠᶜ_reference(i, j, k, grid) * scaling end end # may need to do Val(Nk) since it may not be known at compile -compute_barotropic_mode!(U, V, grid, u, v) = - launch!(architecture(grid), grid, :xy, _barotropic_mode_kernel!, U, V, grid, u, v) +compute_barotropic_mode!(U, V, grid, u, v, η) = + launch!(architecture(grid), grid, :xy, _barotropic_mode_kernel!, U, V, grid, u, v, η) function initialize_free_surface_state!(free_surface_state, η) state = free_surface_state @@ -243,28 +252,29 @@ function initialize_free_surface_state!(free_surface_state, η) return nothing end -@kernel function barotropic_split_explicit_corrector_kernel!(u, v, U̅, V̅, U, V, Hᶠᶜ, Hᶜᶠ) +@kernel function barotropic_split_explicit_corrector_kernel!(u, v, grid, U̅, V̅, U, V, η̅) i, j, k = @index(Global, NTuple) + k_top = grid.Nz + 1 + @inbounds begin - u[i, j, k] = u[i, j, k] + (U̅[i, j] - U[i, j]) / Hᶠᶜ[i, j] - v[i, j, k] = v[i, j, k] + (V̅[i, j] - V[i, j]) / Hᶜᶠ[i, j] + u[i, j, k] = u[i, j, k] + (U̅[i, j] - U[i, j]) / dynamic_column_height(i, j, k_top, grid, η̅) + v[i, j, k] = v[i, j, k] + (V̅[i, j] - V[i, j]) / dynamic_column_height(i, j, k_top, grid, η̅) end end # may need to do Val(Nk) since it may not be known at compile. Also figure out where to put H function barotropic_split_explicit_corrector!(u, v, free_surface, grid) sefs = free_surface.state - Hᶠᶜ, Hᶜᶠ = free_surface.auxiliary.Hᶠᶜ, free_surface.auxiliary.Hᶜᶠ U, V, U̅, V̅ = sefs.U, sefs.V, sefs.U̅, sefs.V̅ arch = architecture(grid) # take out "bad" barotropic mode, # !!!! reusing U and V for this storage since last timestep doesn't matter - compute_barotropic_mode!(U, V, grid, u, v) + compute_barotropic_mode!(U, V, grid, u, v, sefs.η̅) # add in "good" barotropic mode launch!(arch, grid, :xyz, barotropic_split_explicit_corrector_kernel!, - u, v, U̅, V̅, U, V, Hᶠᶜ, Hᶜᶠ) + u, v, grid, U̅, V̅, U, V, sefs.η̅) return nothing end @@ -276,7 +286,7 @@ ab2_step_free_surface!(free_surface::SplitExplicitFreeSurface, model, Δt, χ) = split_explicit_free_surface_step!(free_surface, model, Δt, χ) function initialize_free_surface!(sefs::SplitExplicitFreeSurface, grid, velocities) - @apply_regionally compute_barotropic_mode!(sefs.state.U̅, sefs.state.V̅, grid, velocities.u, velocities.v) + @apply_regionally compute_barotropic_mode!(sefs.state.U̅, sefs.state.V̅, grid, velocities.u, velocities.v, sefs.η) fill_halo_regions!((sefs.state.U̅, sefs.state.V̅, sefs.η)) end @@ -406,7 +416,7 @@ update_zstar_scaling!(sⁿ, s⁻, ∂t_∂s, params, fs::SplitExplicitFreeSurfac s⁻[i, j, 1] = sⁿ[i, j, 1] sⁿ[i, j, 1] = h - # ∂(H + η / H)/∂t = - ∇ ⋅ ∫udz / H - ∂t_∂s[i, j, 1] = - div_xyᶜᶜᶜ(i, j, 1, grid, U̅, V̅) / bottom / h + # ∂(η / H)/∂t = - ∇ ⋅ ∫udz / H + ∂t_∂s[i, j, 1] = - div_xyᶜᶜᶜ(i, j, 1, grid, U̅, V̅) / bottom end end diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_coordinate.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl similarity index 94% rename from src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_coordinate.jl rename to src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index b9f33d3040..403d66421c 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_coordinate.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -105,7 +105,7 @@ update_zstar_scaling!(sⁿ, s⁻, ∂t_∂s, params, fs, grid, Δt) = sⁿ[i, j, 1] = h # Scaling derivative - ∂t_∂s[i, j, 1] = (h - s⁻[i, j, 1]) / Δt / h + ∂t_∂s[i, j, 1] = (h - s⁻[i, j, 1]) / Δt end end @@ -137,6 +137,9 @@ end @inline slope_xᶠᶜᶜ(i, j, k, grid, free_surface) = @inbounds ∂xᶠᶜᶜ(i, j, k, grid, η_surfaceᶜᶜᶜ, free_surface.η) @inline slope_yᶜᶠᶜ(i, j, k, grid, free_surface) = @inbounds ∂yᶜᶠᶜ(i, j, k, grid, η_surfaceᶜᶜᶜ, free_surface.η) +@inline grid_slope_contribution_x(i, j, k, grid::ZStarSpacingGrid, free_surface, ::Nothing, model_fields) = zero(grid) +@inline grid_slope_contribution_y(i, j, k, grid::ZStarSpacingGrid, free_surface, ::Nothing, model_fields) = zero(grid) + @inline grid_slope_contribution_x(i, j, k, grid::ZStarSpacingGrid, free_surface, buoyancy, model_fields) = ℑxᶠᵃᵃ(i, j, k, grid, buoyancy_perturbationᶜᶜᶜ, buoyancy.model, model_fields) * slope_xᶠᶜᶜ(i, j, k, grid, free_surface) diff --git a/src/MultiRegion/multi_region_split_explicit_free_surface.jl b/src/MultiRegion/multi_region_split_explicit_free_surface.jl index 7349324fec..274bba5f80 100644 --- a/src/MultiRegion/multi_region_split_explicit_free_surface.jl +++ b/src/MultiRegion/multi_region_split_explicit_free_surface.jl @@ -9,30 +9,13 @@ function SplitExplicitAuxiliaryFields(grid::MultiRegionGrids) Gᵁ = Field((Face, Center, Nothing), grid) Gⱽ = Field((Center, Face, Nothing), grid) - Hᶠᶜ = Field((Face, Center, Nothing), grid) - Hᶜᶠ = Field((Center, Face, Nothing), grid) - Hᶜᶜ = Field((Center, Center, Nothing), grid) - - @apply_regionally calculate_column_height!(Hᶠᶜ, (Face, Center, Center)) - @apply_regionally calculate_column_height!(Hᶜᶠ, (Center, Face, Center)) - - @apply_regionally calculate_column_height!(Hᶜᶜ, (Center, Center, Center)) - - fill_halo_regions!((Hᶠᶜ, Hᶜᶠ, Hᶜᶜ)) - # In a non-parallel grid we calculate only the interior @apply_regionally kernel_size = augmented_kernel_size(grid, grid.partition) @apply_regionally kernel_offsets = augmented_kernel_offsets(grid, grid.partition) @apply_regionally kernel_parameters = KernelParameters(kernel_size, kernel_offsets) - return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, kernel_parameters) -end - -@inline function calculate_column_height!(height, location) - dz = GridMetricOperation(location, Δz, height.grid) - sum!(height, dz) - return nothing + return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, kernel_parameters) end @inline augmented_kernel_size(grid, ::XPartition) = (size(grid, 1) + 2halo_size(grid)[1]-2, size(grid, 2)) diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index f69b34e171..0a217a1e2e 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -2,7 +2,7 @@ using Oceananigans using Oceananigans.Units using Oceananigans.Utils: prettytime using Oceananigans.Advection: WENOVectorInvariant -using Oceananigans.Models.HydrostaticFreeSurfaceModels: ZStar, Z, ZStarCoordinateGrid +using Oceananigans.Models.HydrostaticFreeSurfaceModels: ZStar, ZStarSpacingGrid using Printf grid = RectilinearGrid(size = (300, 20), @@ -12,10 +12,10 @@ grid = RectilinearGrid(size = (300, 20), topology = (Bounded, Flat, Bounded)) model = HydrostaticFreeSurfaceModel(; grid, - vertical_coordinate = ZStar(), - momentum_advection = VectorInvariant(), + generalized_vertical_coordinate = ZStar(), + momentum_advection = WENOVectorInvariant(), tracer_advection = WENO(), - closure = HorizontalScalarDiffusivity(ν = 10), + # closure = HorizontalScalarDiffusivity(ν = 10), buoyancy = BuoyancyTracer(), tracers = :b, free_surface = SplitExplicitFreeSurface(; substeps = 30)) @@ -33,9 +33,9 @@ barotropic_time_step = grid.Δxᶜᵃᵃ / gravity_wave_speed @info "the time step is $Δt" -simulation = Simulation(model; Δt, stop_time = 1days) #, stop_iteration = 10) +simulation = Simulation(model; Δt, stop_time = 1days) -field_outputs = if model.grid isa ZStarCoordinateGrid +field_outputs = if model.grid isa ZStarSpacingGrid merge(model.velocities, model.tracers, (; ΔzF = model.grid.Δzᵃᵃᶠ.Δ)) else merge(model.velocities, model.tracers) @@ -57,7 +57,7 @@ function progress(sim) msg1 = @sprintf("extrema w: %.2e %.2e ", maximum(w), minimum(w)) msg2 = @sprintf("extrema u: %.2e %.2e ", maximum(u), minimum(u)) msg3 = @sprintf("extrema b: %.2e %.2e ", maximum(b), minimum(b)) - if sim.model.grid isa ZStarCoordinateGrid + if sim.model.grid isa ZStarSpacingGrid Δz = sim.model.grid.Δzᵃᵃᶠ.Δ msg4 = @sprintf("extrema Δz: %.2e %.2e ", maximum(Δz), minimum(Δz)) @info msg0 * msg1 * msg2 * msg3 * msg4 @@ -73,14 +73,13 @@ simulation.callbacks[:wizard] = Callback(TimeStepWizard(; cfl = 0.2, max_chang run!(simulation) # Check conservation -if model.grid isa ZStarCoordinateGrid +if model.grid isa ZStarSpacingGrid b = FieldTimeSeries("zstar_model.jld2", "b") dz = FieldTimeSeries("zstar_model.jld2", "ΔzF") init = sum(b[1] * dz[1]) / sum(dz[1]) drift = [] for t in 1:length(b.times) - push!(drift, sum(b[t] * dz[t]) / sum(dz[t]) - init) end else diff --git a/validation/z_star_coordinate/tracer_conservation.jl b/validation/z_star_coordinate/tracer_conservation.jl index 756dd870fa..32e73b3488 100644 --- a/validation/z_star_coordinate/tracer_conservation.jl +++ b/validation/z_star_coordinate/tracer_conservation.jl @@ -1,8 +1,11 @@ using Oceananigans using Oceananigans.Units using Oceananigans.Utils: prettytime +using Oceananigans.Operators using Oceananigans.Advection: WENOVectorInvariant -using Oceananigans.Models.HydrostaticFreeSurfaceModels: ZStar, Z, ZStarCoordinateGrid +using Oceananigans.Fields: ZeroField +using Oceananigans.Grids: architecture +using Oceananigans.Models.HydrostaticFreeSurfaceModels: ZStar, ZStarSpacingGrid, update_state! using Printf grid = RectilinearGrid(size = (300, 20), @@ -12,38 +15,74 @@ grid = RectilinearGrid(size = (300, 20), topology = (Bounded, Flat, Bounded)) model = HydrostaticFreeSurfaceModel(; grid, - vertical_coordinate = ZStar(), - momentum_advection = VectorInvariant(), - tracer_advection = WENO(), - closure = HorizontalScalarDiffusivity(ν = 10), - buoyancy = BuoyancyTracer(), - tracers = :b, - free_surface = SplitExplicitFreeSurface(; substeps = 30)) + generalized_vertical_coordinate = ZStar(), + momentum_advection = nothing, + tracer_advection = WENO(), + closure = nothing, + buoyancy = nothing, + tracers = (), + free_surface = SplitExplicitFreeSurface(; substeps = 30)) g = model.free_surface.gravitational_acceleration ηᵢ(x, z) = exp(-(x - 50kilometers)^2 / (10kilometers)^2) -bᵢ(x, z) = 1e-6 * z -set!(model, b = bᵢ, η = ηᵢ) +set!(model, η = ηᵢ) uᵢ = XFaceField(grid) -for i in 1:size(u, 1) - uᵢ[i, :, :] = - g * ∂xᶠᶜᶜ(i, 1, grid.Nz+1, grid, model.free_surface.η) +U̅ = Field((Face, Center, Nothing), grid) + +for i in 1:size(uᵢ, 1) + uᵢ[i, :, :] .= - g * ∂xᶠᶜᶜ(i, 1, grid.Nz+1, grid, model.free_surface.η) + U̅[i, 1, 1] = - g * ∂xᶠᶜᶜ(i, 1, grid.Nz+1, grid, model.free_surface.η) * grid.Lz end set!(model, u = uᵢ) +set!(model.free_surface.state.U̅, U̅) + +using Oceananigans.BoundaryConditions + +fill_halo_regions!(model.velocities) + +using Oceananigans.Utils +using Oceananigans.Models.HydrostaticFreeSurfaceModels: _update_zstar_split_explicit_scaling! +using Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_w_from_continuity!, _update_z_star! + +# # Scaling +# s⁻ = model.grid.Δzᵃᵃᶠ.s⁻ +# sⁿ = model.grid.Δzᵃᵃᶠ.sⁿ +# ∂t_∂s = model.grid.Δzᵃᵃᶠ.∂t_∂s + +# # Generalized vertical spacing +# ΔzF = model.grid.Δzᵃᵃᶠ.Δ +# ΔzC = model.grid.Δzᵃᵃᶜ.Δ + +# # Reference (non moving) spacing +# ΔzF₀ = model.grid.Δzᵃᵃᶠ.Δr +# ΔzC₀ = model.grid.Δzᵃᵃᶜ.Δr + +# launch!(architecture(model.grid), model.grid, :xy, _update_zstar_split_explicit_scaling!, +# sⁿ, s⁻, ∂t_∂s, model.free_surface.η, U̅, ZeroField(grid), model.grid) + +# fill_halo_regions!((sⁿ, s⁻, ∂t_∂s)) + +# launch!(architecture(grid), grid, :xy, _update_z_star!, +# ΔzF, ΔzC, ΔzF₀, ΔzC₀, sⁿ, Val(grid.Nz)) + +# fill_halo_regions!((ΔzF, ΔzC)) + +# compute_w_from_continuity!(model.velocities, CPU(), model.grid) gravity_wave_speed = sqrt(g * grid.Lz) barotropic_time_step = grid.Δxᶜᵃᵃ / gravity_wave_speed -Δt = 0.5 * barotropic_time_step +Δt = barotropic_time_step @info "the time step is $Δt" -simulation = Simulation(model; Δt, stop_time = 1days) #, stop_iteration = 10) +simulation = Simulation(model; Δt, stop_iteration = 1) -field_outputs = if model.grid isa ZStarCoordinateGrid +field_outputs = if model.grid isa ZStarSpacingGrid merge(model.velocities, model.tracers, (; ΔzF = model.grid.Δzᵃᵃᶠ.Δ)) else merge(model.velocities, model.tracers) @@ -51,7 +90,7 @@ end simulation.output_writers[:other_variables] = JLD2OutputWriter(model, field_outputs, overwrite_existing = true, - schedule = IterationInterval(100), + schedule = TimeInterval(30minutes), filename = "zstar_model") function progress(sim) @@ -59,45 +98,21 @@ function progress(sim) u = sim.model.velocities.u v = sim.model.velocities.v η = sim.model.free_surface.η - b = sim.model.tracers.b msg0 = @sprintf("Time: %s iteration %d ", prettytime(sim.model.clock.time), sim.model.clock.iteration) msg1 = @sprintf("extrema w: %.2e %.2e ", maximum(w), minimum(w)) msg2 = @sprintf("extrema u: %.2e %.2e ", maximum(u), minimum(u)) - msg3 = @sprintf("extrema b: %.2e %.2e ", maximum(b), minimum(b)) - if sim.model.grid isa ZStarCoordinateGrid + if sim.model.grid isa ZStarSpacingGrid Δz = sim.model.grid.Δzᵃᵃᶠ.Δ msg4 = @sprintf("extrema Δz: %.2e %.2e ", maximum(Δz), minimum(Δz)) - @info msg0 * msg1 * msg2 * msg3 * msg4 + @info msg0 * msg1 * msg2 * msg4 else - @info msg0 * msg1 * msg2 * msg3 + @info msg0 * msg1 * msg2 end return nothing end -simulation.callbacks[:progress] = Callback(progress, IterationInterval(100)) +simulation.callbacks[:progress] = Callback(progress, IterationInterval(1)) simulation.callbacks[:wizard] = Callback(TimeStepWizard(; cfl = 0.2, max_change = 1.1), IterationInterval(10)) run!(simulation) - -# Check conservation -if model.grid isa ZStarCoordinateGrid - b = FieldTimeSeries("zstar_model.jld2", "b") - dz = FieldTimeSeries("zstar_model.jld2", "ΔzF") - - init = sum(b[1] * dz[1]) / sum(dz[1]) - drift = [] - for t in 1:length(b.times) - - push!(drift, sum(b[t] * dz[t]) / sum(dz[t]) - init) - end -else - b = FieldTimeSeries("zstar_model.jld2", "b") - - init = sum(b[1]) / prod(size(b[1])) - drift = [] - for t in 1:length(b.times) - - push!(drift, sum(b[t]) / prod(size(b[1])) - init) - end -end \ No newline at end of file From 881bdb596aaaaf7e8bd32d73afcf0872ebfcee38 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 8 Jan 2024 13:47:05 -0500 Subject: [PATCH 122/567] bugfix --- .../split_explicit_free_surface_kernels.jl | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 3bbc5460c7..d0075e5aec 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -308,23 +308,24 @@ function split_explicit_free_surface_step!(free_surface::SplitExplicitFreeSurfac # Wait for previous set up wait_free_surface_communication!(free_surface, architecture(free_surface_grid)) + # Calculate the substepping parameterers + settings = free_surface.settings + Nsubsteps = calculate_substeps(settings.substepping, Δt) + + # barotropic time step as fraction of baroclinic step and averaging weights + fractional_Δt, weights = calculate_adaptive_settings(settings.substepping, Nsubsteps) + Nsubsteps = length(weights) + + # barotropic time step in seconds + Δτᴮ = fractional_Δt * Δt + # reset free surface averages @apply_regionally begin - settings = free_surface.settings - initialize_free_surface_state!(free_surface.state, free_surface.η, settings.timestepper) - Nsubsteps = calculate_substeps(settings.substepping, Δt) - - # barotropic time step as fraction of baroclinic step and averaging weights - fractional_Δt, weights = calculate_adaptive_settings(settings.substepping, Nsubsteps) - Nsubsteps = length(weights) - - # barotropic time step in seconds - Δτᴮ = fractional_Δt * Δt - # Solve for the free surface at tⁿ⁺¹ iterate_split_explicit!(free_surface, free_surface_grid, Δτᴮ, weights, Val(Nsubsteps)) + # Reset eta for the next timestep set!(free_surface.η, free_surface.state.η̅) end From f79a056ee9b18f2ca61075a30f5bb199686a1807 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 8 Jan 2024 14:19:01 -0500 Subject: [PATCH 123/567] other bugfix --- src/Architectures.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Architectures.jl b/src/Architectures.jl index 0dc61f71ed..be016b3f82 100644 --- a/src/Architectures.jl +++ b/src/Architectures.jl @@ -115,9 +115,9 @@ end # Convert arguments to GPU-compatible types -@inline convert_args(::CPU, arg) = args -@inline convert_args(::GPU, arg) = CUDA.cudaconvert(arg) -@inline convert_args(::GPU, arg::Tuple) = map(CUDA.cudaconvert, arg) +@inline convert_args(::CPU, args) = args +@inline convert_args(::GPU, args) = CUDA.cudaconvert(args) +@inline convert_args(::GPU, args::Tuple) = map(CUDA.cudaconvert, args) end # module From 446094af17789c691a79f30eb5d1962570852e45 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 8 Jan 2024 14:57:26 -0500 Subject: [PATCH 124/567] found the issue --- .../compute_w_from_continuity.jl | 2 +- .../generalized_vertical_spacing.jl | 7 +++ .../split_explicit_free_surface_kernels.jl | 14 ++--- .../z_star_vertical_spacing.jl | 58 ++++++------------- 4 files changed, 32 insertions(+), 49 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl index 85720ed6c0..6a92f1a4eb 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl @@ -23,7 +23,7 @@ compute_w_from_continuity!(velocities, arch, grid; parameters = w_kernel_paramet U.w[i, j, 1] = 0 @unroll for k in 2:grid.Nz+1 @inbounds U.w[i, j, k] = U.w[i, j, k-1] - Δzᶜᶜᶜ(i, j, k-1, grid) * - ( div_xyᶜᶜᶜ(i, j, k-1, grid, U.u, U.v) + + ( div_xyᶜᶜᶜ(i, j, k-1, grid, U.u, U.v) + ∂t_∂s_grid(i, j, k-1, grid) ) end end diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index af359f1c45..de4ea68419 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -97,6 +97,13 @@ update_vertical_spacing!(model, grid, Δt; kwargs...) = nothing @inline Δzᶠᶠᶠ_reference(i, j, k, grid) = ℑxyᶠᶠᵃ(i, j, k, grid, Δzᶜᶜᶠ_reference) @inline Δzᶠᶠᶜ_reference(i, j, k, grid) = ℑxyᶠᶠᵃ(i, j, k, grid, Δzᶜᶜᶜ_reference) +##### +##### Utility +##### + +bottom_height(i, j, grid) = grid.Lz +bottom_height(i, j, grid::ImmersedBoundaryGrid) = @inbounds - grid.immersed_boundary.bottom_height[i, j, 1] + ##### ##### Vertical velocity of the Δ-surfaces to be included in the continuity equation ##### diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index a669795b4e..8e1bdf312c 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -402,19 +402,15 @@ setup_split_explicit_tendency!(auxiliary, grid, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) wait_free_surface_communication!(free_surface, arch) = nothing -update_zstar_scaling!(sⁿ, s⁻, ∂t_∂s, params, fs::SplitExplicitFreeSurface, grid, Δt) = - launch!(architecture(grid), grid, params, _update_zstar_split_explicit_scaling!, - sⁿ, s⁻, ∂t_∂s, fs.η, fs.state.U̅, fs.state.V̅, grid) +# Special update ∂t_∂s for SplitExplicitFreeSurface where +# ∂(η / H)/∂t = - ∇ ⋅ U̅ / H +update_∂t_∂s!(∂t_∂s, parameters, grid, sⁿ, s⁻, Δt, fs::SplitExplicitFreeSurface) = + launch!(architecture(grid), grid, parameters, _update_∂t_∂s_split_explicit!, ∂t_∂s, fs.state.U̅, fs.state.V̅, grid) -@kernel function _update_zstar_split_explicit_scaling!(sⁿ, s⁻, ∂t_∂s, η, U̅, V̅, grid) +@kernel function _update_∂t_∂s_split_explicit!(∂t_∂s, U̅, V̅, grid) i, j = @index(Global, NTuple) bottom = bottom_height(i, j, grid) @inbounds begin - h = (bottom + η[i, j, grid.Nz+1]) / bottom - - # update current and previous scaling - s⁻[i, j, 1] = sⁿ[i, j, 1] - sⁿ[i, j, 1] = h # ∂(η / H)/∂t = - ∇ ⋅ ∫udz / H ∂t_∂s[i, j, 1] = - div_xyᶜᶜᶜ(i, j, 1, grid, U̅, V̅) / bottom diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 403d66421c..2a5107fec7 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -30,16 +30,12 @@ function GeneralizedSpacingGrid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, :: s⁻ = Field{Center, Center, Nothing}(grid) sⁿ = Field{Center, Center, Nothing}(grid) ∂t_∂s = Field{Center, Center, Nothing}(grid) - - # Initial "at-rest" conditions - set!(sⁿ, 1) - set!(s⁻, 1) - fill_halo_regions!((s⁻, sⁿ)) - - launch!(architecture(grid), grid, :xy,_update_z_star!, - ΔzF, ΔzC, grid.Δzᵃᵃᶠ, grid.Δzᵃᵃᶜ, sⁿ, Val(grid.Nz)) + # Initial "at-rest" conditions + launch!(architecture(grid), grid, :xy, _update_zstar!, + sⁿ, s⁻, ΔzF, ΔzC, ZeroField(grid), grid, Val(grid.Nz)) + fill_halo_regions!((s⁻, sⁿ)) fill_halo_regions!((ΔzF, ΔzC)) Δzᵃᵃᶠ = GeneralizedVerticalSpacing(ZStar(), grid.Δzᵃᵃᶠ, ΔzF, s⁻, sⁿ, ∂t_∂s) @@ -76,25 +72,18 @@ function update_vertical_spacing!(model, grid::ZStarSpacingGrid, Δt; parameters ΔzF = grid.Δzᵃᵃᶠ.Δ ΔzC = grid.Δzᵃᵃᶜ.Δ - # Reference (non moving) spacing - ΔzF₀ = grid.Δzᵃᵃᶠ.Δr - ΔzC₀ = grid.Δzᵃᵃᶜ.Δr - # Update vertical spacing with available parameters # No need to fill the halo as the scaling is updated _IN_ the halos - update_zstar_scaling!(sⁿ, s⁻, ∂t_∂s, parameters, model.free_surface, grid, Δt) - - launch!(architecture(grid), grid, parameters, _update_z_star!, - ΔzF, ΔzC, ΔzF₀, ΔzC₀, sⁿ, Val(grid.Nz)) + launch!(architecture(grid), grid, parameters, _update_zstar!, sⁿ, s⁻, ΔzF, ΔzC, + model.free_surface.η, grid, Val(grid.Nz)) + # Update scaling time derivative + update_∂t_∂s!(∂t_∂s, parameters, grid, sⁿ, s⁻, Δt, model.free_surface) + return nothing end -update_zstar_scaling!(sⁿ, s⁻, ∂t_∂s, params, fs, grid, Δt) = - launch!(architecture(grid), grid, params, _update_zstar_scaling!, - sⁿ, s⁻, ∂t_∂s, fs.η, grid, Δt) - -@kernel function _update_zstar_scaling!(sⁿ, s⁻, ∂t_∂s, η, grid, Δt) +@kernel function _update_zstar!(sⁿ, s⁻, ΔzF, ΔzC, η, grid, ::Val{Nz}) where Nz i, j = @index(Global, NTuple) bottom = bottom_height(i, j, grid) @inbounds begin @@ -103,29 +92,20 @@ update_zstar_scaling!(sⁿ, s⁻, ∂t_∂s, params, fs, grid, Δt) = # update current and previous scaling s⁻[i, j, 1] = sⁿ[i, j, 1] sⁿ[i, j, 1] = h - - # Scaling derivative - ∂t_∂s[i, j, 1] = (h - s⁻[i, j, 1]) / Δt + + @unroll for k in 1:Nz+1 + ΔzF[i, j, k] = h * Δzᶜᶜᶠ_reference(i, j, k, grid) + ΔzC[i, j, k] = h * Δzᶜᶜᶜ_reference(i, j, k, grid) + end end end -bottom_height(i, j, grid) = grid.Lz -bottom_height(i, j, grid::ImmersedBoundaryGrid) = @inbounds - grid.immersed_boundary.bottom_height[i, j, 1] - -@kernel function _update_z_star!(ΔzF, ΔzC, ΔzF₀, ΔzC₀, sⁿ, ::Val{Nz}) where Nz - i, j = @index(Global, NTuple) - @unroll for k in 1:Nz+1 - @inbounds ΔzF[i, j, k] = sⁿ[i, j, 1] * ΔzF₀[k] - @inbounds ΔzC[i, j, k] = sⁿ[i, j, 1] * ΔzC₀[k] - end -end +update_∂t_∂s!(∂t_∂s, parameters, grid, sⁿ, s⁻, Δt, fs) = + launch!(architecture(grid), grid, parameters, _update_∂t_∂s!, ∂t_∂s, sⁿ, s⁻, Δt) -@kernel function _update_z_star!(ΔzF, ΔzC, ΔzF₀::Number, ΔzC₀::Number, sⁿ, ::Val{Nz}) where Nz +@kernel function _update_∂t_∂s!(∂t_∂s, sⁿ, s⁻, Δt) i, j = @index(Global, NTuple) - @unroll for k in 1:Nz+1 - @inbounds ΔzF[i, j, k] = sⁿ[i, j, 1] * ΔzF₀ - @inbounds ΔzC[i, j, k] = sⁿ[i, j, 1] * ΔzC₀ - end + ∂t_∂s[i, j, 1] = (sⁿ[i, j, 1] - s⁻[i, j, 1]) / Δt end ##### From 044c2f83b3c6c93fee52ef3b31bad6594f0cc295 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 8 Jan 2024 15:02:48 -0500 Subject: [PATCH 125/567] remove useless code --- .../update_hydrostatic_free_surface_model_state.jl | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl index b37590aa45..e670e202a5 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl @@ -83,11 +83,6 @@ function compute_auxiliaries!(model::HydrostaticFreeSurfaceModel, Δt; w_paramet grid, model.buoyancy, model.tracers; parameters = ppar) end - return nothing -end -# Do not update if rigid lid!! -const RigidLidModel = HydrostaticFreeSurfaceModel{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Nothing} - -update_vertical_spacing!(::RigidLidModel, ::ZStarSpacingGrid; kwargs...) = nothing -update_tracer_thickness!(tracers, grid) = nothing + return nothing +end \ No newline at end of file From 31d26cab2bed9e5f3573314efceb1a39b2eb78d6 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 8 Jan 2024 16:55:55 -0500 Subject: [PATCH 126/567] perfect tracer conservation --- .../generalized_vertical_spacing.jl | 33 ++++++++++++++++--- ...te_hydrostatic_free_surface_model_state.jl | 10 ++++-- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index de4ea68419..f7886e87ea 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -118,7 +118,8 @@ bottom_height(i, j, grid::ImmersedBoundaryGrid) = @inbounds - grid.immersed_boun @inline grid_slope_contribution_y(i, j, k, grid, args...) = zero(grid) ##### -##### Handling tracer update in generalized vertical coordinates (we update sθ) +##### Tracer update in generalized vertical coordinates +##### We advance sθ but store θ once sⁿ⁺¹ is known ##### @kernel function _ab2_step_tracer_generalized_spacing!(θ, sⁿ, s⁻, Δt, χ, Gⁿ, G⁻) @@ -129,8 +130,10 @@ bottom_height(i, j, grid::ImmersedBoundaryGrid) = @inbounds - grid.immersed_boun oh_point_five = convert(FT, 0.5) @inbounds begin - ∂t_∂sθ = (one_point_five + χ) * Gⁿ[i, j, k] - (oh_point_five + χ) * G⁻[i, j, k] - θ[i, j, k] = s⁻[i, j, k] * θ[i, j, k] / sⁿ[i, j, k] + convert(FT, Δt) * ∂t_∂sθ + ∂t_∂sθ = (one_point_five + χ) * sⁿ[i, j, k] * Gⁿ[i, j, k] - (oh_point_five + χ) * s⁻[i, j, k] * G⁻[i, j, k] + + # We store temporarily sθ in θ. the unscaled θ will be retrived later on with `scale_tracers!` + θ[i, j, k] = sⁿ[i, j, k] * θ[i, j, k] + convert(FT, Δt) * ∂t_∂sθ end end @@ -139,4 +142,26 @@ ab2_step_tracer_field!(tracer_field, grid::GeneralizedSpacingGrid, Δt, χ, Gⁿ tracer_field, grid.Δzᵃᵃᶠ.sⁿ, grid.Δzᵃᵃᶠ.s⁻, - Δt, χ, Gⁿ, G⁻) \ No newline at end of file + Δt, χ, Gⁿ, G⁻) + +const EmptyTuples = Union{NamedTuple{(),Tuple{}}, Tuple{}} + +scale_tracers!(::EmptyTuples, ::GeneralizedSpacingGrid; kwargs...) = nothing + +tracer_scaling_parameters(param::Symbol, tracers, grid) = KernelParameters((size(grid, 1), size(grid, 2), length(tracers)), (0, 0, 0)) +tracer_scaling_parameters(param::KernelParameters{S, O}, tracers, grid) where {S, O} = KernelParameters((S..., length(tracers)), (O..., 0)) + +function scale_tracers!(tracers, grid::GeneralizedSpacingGrid; parameters = :xy) + parameters = tracer_scaling_parameters(parameters, tracers, grid) + launch!(architecture(grid), grid, parameters, _scale_tracers, tracers, grid.Δzᵃᵃᶠ.sⁿ, + Val(grid.Hz), Val(grid.Nz)) + return nothing +end + +@kernel function _scale_tracers(tracers, sⁿ, ::Val{Hz}, ::Val{Nz}) where {Hz, Nz} + i, j, n = @index(Global, NTuple) + + @unroll for k in -Hz+1:Nz+Hz + tracers[n][i, j, k] /= sⁿ[i, j, k] + end +end \ No newline at end of file diff --git a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl index e670e202a5..6c761e3d3a 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl @@ -71,12 +71,14 @@ function compute_auxiliaries!(model::HydrostaticFreeSurfaceModel, Δt; w_paramet p_parameters = tuple(p_kernel_parameters(model.grid)), κ_parameters = tuple(:xyz)) - grid = model.grid - closure = model.closure + grid = model.grid + closure = model.closure + tracers = model.tracers diffusivity = model.diffusivity_fields for (wpar, ppar, κpar) in zip(w_parameters, p_parameters, κ_parameters) update_vertical_spacing!(model, grid, Δt; parameters = wpar) + scale_tracers!(tracers, grid; parameters = wpar) compute_w_from_continuity!(model; parameters = wpar) compute_diffusivities!(diffusivity, closure, model; parameters = κpar) update_hydrostatic_pressure!(model.pressure.pHY′, architecture(grid), @@ -85,4 +87,6 @@ function compute_auxiliaries!(model::HydrostaticFreeSurfaceModel, Δt; w_paramet end return nothing -end \ No newline at end of file +end + +scale_tracers!(tracers, grid; kwargs...) = nothing From 870cc4621a37c1e462ce9094a208ed23136f52bf Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 8 Jan 2024 16:59:39 -0500 Subject: [PATCH 127/567] add lock release --- validation/z_star_coordinate/lock_release.jl | 1 - .../z_star_coordinate/tracer_conservation.jl | 118 ------------------ 2 files changed, 119 deletions(-) delete mode 100644 validation/z_star_coordinate/tracer_conservation.jl diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index 0a217a1e2e..b6b3efaea8 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -15,7 +15,6 @@ model = HydrostaticFreeSurfaceModel(; grid, generalized_vertical_coordinate = ZStar(), momentum_advection = WENOVectorInvariant(), tracer_advection = WENO(), - # closure = HorizontalScalarDiffusivity(ν = 10), buoyancy = BuoyancyTracer(), tracers = :b, free_surface = SplitExplicitFreeSurface(; substeps = 30)) diff --git a/validation/z_star_coordinate/tracer_conservation.jl b/validation/z_star_coordinate/tracer_conservation.jl deleted file mode 100644 index 32e73b3488..0000000000 --- a/validation/z_star_coordinate/tracer_conservation.jl +++ /dev/null @@ -1,118 +0,0 @@ -using Oceananigans -using Oceananigans.Units -using Oceananigans.Utils: prettytime -using Oceananigans.Operators -using Oceananigans.Advection: WENOVectorInvariant -using Oceananigans.Fields: ZeroField -using Oceananigans.Grids: architecture -using Oceananigans.Models.HydrostaticFreeSurfaceModels: ZStar, ZStarSpacingGrid, update_state! -using Printf - -grid = RectilinearGrid(size = (300, 20), - x = (0, 100kilometers), - z = (-10, 0), - halo = (6, 6), - topology = (Bounded, Flat, Bounded)) - -model = HydrostaticFreeSurfaceModel(; grid, - generalized_vertical_coordinate = ZStar(), - momentum_advection = nothing, - tracer_advection = WENO(), - closure = nothing, - buoyancy = nothing, - tracers = (), - free_surface = SplitExplicitFreeSurface(; substeps = 30)) - -g = model.free_surface.gravitational_acceleration - -ηᵢ(x, z) = exp(-(x - 50kilometers)^2 / (10kilometers)^2) - -set!(model, η = ηᵢ) - -uᵢ = XFaceField(grid) -U̅ = Field((Face, Center, Nothing), grid) - -for i in 1:size(uᵢ, 1) - uᵢ[i, :, :] .= - g * ∂xᶠᶜᶜ(i, 1, grid.Nz+1, grid, model.free_surface.η) - U̅[i, 1, 1] = - g * ∂xᶠᶜᶜ(i, 1, grid.Nz+1, grid, model.free_surface.η) * grid.Lz -end - -set!(model, u = uᵢ) -set!(model.free_surface.state.U̅, U̅) - -using Oceananigans.BoundaryConditions - -fill_halo_regions!(model.velocities) - -using Oceananigans.Utils -using Oceananigans.Models.HydrostaticFreeSurfaceModels: _update_zstar_split_explicit_scaling! -using Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_w_from_continuity!, _update_z_star! - -# # Scaling -# s⁻ = model.grid.Δzᵃᵃᶠ.s⁻ -# sⁿ = model.grid.Δzᵃᵃᶠ.sⁿ -# ∂t_∂s = model.grid.Δzᵃᵃᶠ.∂t_∂s - -# # Generalized vertical spacing -# ΔzF = model.grid.Δzᵃᵃᶠ.Δ -# ΔzC = model.grid.Δzᵃᵃᶜ.Δ - -# # Reference (non moving) spacing -# ΔzF₀ = model.grid.Δzᵃᵃᶠ.Δr -# ΔzC₀ = model.grid.Δzᵃᵃᶜ.Δr - -# launch!(architecture(model.grid), model.grid, :xy, _update_zstar_split_explicit_scaling!, -# sⁿ, s⁻, ∂t_∂s, model.free_surface.η, U̅, ZeroField(grid), model.grid) - -# fill_halo_regions!((sⁿ, s⁻, ∂t_∂s)) - -# launch!(architecture(grid), grid, :xy, _update_z_star!, -# ΔzF, ΔzC, ΔzF₀, ΔzC₀, sⁿ, Val(grid.Nz)) - -# fill_halo_regions!((ΔzF, ΔzC)) - -# compute_w_from_continuity!(model.velocities, CPU(), model.grid) - -gravity_wave_speed = sqrt(g * grid.Lz) -barotropic_time_step = grid.Δxᶜᵃᵃ / gravity_wave_speed - -Δt = barotropic_time_step - -@info "the time step is $Δt" - -simulation = Simulation(model; Δt, stop_iteration = 1) - -field_outputs = if model.grid isa ZStarSpacingGrid - merge(model.velocities, model.tracers, (; ΔzF = model.grid.Δzᵃᵃᶠ.Δ)) -else - merge(model.velocities, model.tracers) -end - -simulation.output_writers[:other_variables] = JLD2OutputWriter(model, field_outputs, - overwrite_existing = true, - schedule = TimeInterval(30minutes), - filename = "zstar_model") - -function progress(sim) - w = interior(sim.model.velocities.w, :, :, sim.model.grid.Nz+1) - u = sim.model.velocities.u - v = sim.model.velocities.v - η = sim.model.free_surface.η - - msg0 = @sprintf("Time: %s iteration %d ", prettytime(sim.model.clock.time), sim.model.clock.iteration) - msg1 = @sprintf("extrema w: %.2e %.2e ", maximum(w), minimum(w)) - msg2 = @sprintf("extrema u: %.2e %.2e ", maximum(u), minimum(u)) - if sim.model.grid isa ZStarSpacingGrid - Δz = sim.model.grid.Δzᵃᵃᶠ.Δ - msg4 = @sprintf("extrema Δz: %.2e %.2e ", maximum(Δz), minimum(Δz)) - @info msg0 * msg1 * msg2 * msg4 - else - @info msg0 * msg1 * msg2 - end - - return nothing -end - -simulation.callbacks[:progress] = Callback(progress, IterationInterval(1)) -simulation.callbacks[:wizard] = Callback(TimeStepWizard(; cfl = 0.2, max_change = 1.1), IterationInterval(10)) -run!(simulation) From be4f0c441111fa0a24fc74eefad3645b9408024e Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 8 Jan 2024 17:45:48 -0500 Subject: [PATCH 128/567] comments --- .../generalized_vertical_spacing.jl | 1 + .../HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl | 2 +- validation/z_star_coordinate/lock_release.jl | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index f7886e87ea..8ac2b2150c 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -67,6 +67,7 @@ update_vertical_spacing!(model, grid, Δt; kwargs...) = nothing # Very bad for GPU performance!!! (z-values are not coalesced in memory for z-derivatives anymore) # TODO: make z-direction local in memory by not using Fields + @inline Δzᶜᶜᶠ(i, j, k, grid::GeneralizedSpacingGrid) = @inbounds grid.Δzᵃᵃᶠ.Δ[i, j, k] @inline Δzᶜᶜᶜ(i, j, k, grid::GeneralizedSpacingGrid) = @inbounds grid.Δzᵃᵃᶜ.Δ[i, j, k] diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 2a5107fec7..d7170d2575 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -24,7 +24,7 @@ end # the actual vertical spacing and a scaling function GeneralizedSpacingGrid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, ::ZStar) where {FT, TX, TY, TZ} - # Memory layout for Dz spacings is local in z + # Memory layout for Δz spacings should be local in z instead of x ΔzF = ZFaceField(grid) ΔzC = CenterField(grid) s⁻ = Field{Center, Center, Nothing}(grid) diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index b6b3efaea8..dd0b197d33 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -71,7 +71,7 @@ simulation.callbacks[:progress] = Callback(progress, IterationInterval(100)) simulation.callbacks[:wizard] = Callback(TimeStepWizard(; cfl = 0.2, max_change = 1.1), IterationInterval(10)) run!(simulation) -# Check conservation +# Check tracer conservation if model.grid isa ZStarSpacingGrid b = FieldTimeSeries("zstar_model.jld2", "b") dz = FieldTimeSeries("zstar_model.jld2", "ΔzF") From 16290b485b65bb073a02006b97d7eb0fc9023abc Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 9 Jan 2024 11:03:52 -0500 Subject: [PATCH 129/567] add comments --- .../generalized_vertical_spacing.jl | 1 + validation/z_star_coordinate/lock_release.jl | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index 8ac2b2150c..bf6ea63c85 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -67,6 +67,7 @@ update_vertical_spacing!(model, grid, Δt; kwargs...) = nothing # Very bad for GPU performance!!! (z-values are not coalesced in memory for z-derivatives anymore) # TODO: make z-direction local in memory by not using Fields +# TODO: make it work with partial cells @inline Δzᶜᶜᶠ(i, j, k, grid::GeneralizedSpacingGrid) = @inbounds grid.Δzᵃᵃᶠ.Δ[i, j, k] @inline Δzᶜᶜᶜ(i, j, k, grid::GeneralizedSpacingGrid) = @inbounds grid.Δzᵃᵃᶜ.Δ[i, j, k] diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index dd0b197d33..40499467f0 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -17,7 +17,7 @@ model = HydrostaticFreeSurfaceModel(; grid, tracer_advection = WENO(), buoyancy = BuoyancyTracer(), tracers = :b, - free_surface = SplitExplicitFreeSurface(; substeps = 30)) + free_surface = SplitExplicitFreeSurface(; substeps = 10)) g = model.free_surface.gravitational_acceleration From c3a21a42d183cab09e6f19f15aaaf5867f7dfae6 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 9 Jan 2024 17:07:13 -0500 Subject: [PATCH 130/567] other small bugfix --- test/test_split_explicit_free_surface_solver.jl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/test_split_explicit_free_surface_solver.jl b/test/test_split_explicit_free_surface_solver.jl index ccb8aeb005..6ce237ccd7 100644 --- a/test/test_split_explicit_free_surface_solver.jl +++ b/test/test_split_explicit_free_surface_solver.jl @@ -42,9 +42,9 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, calc η₀(x, y, z) = sin(x) set!(η, η₀) - U₀(x, y) = 0 + U₀(x, y, z) = 0 set!(U, U₀) - V₀(x, y) = 0 + V₀(x, y, z) = 0 set!(V, V₀) η̅ .= 0 @@ -85,9 +85,9 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, calc # set!(η, f(x,y)) η₀(x, y, z) = sin(x) set!(η, η₀) - U₀(x, y) = 0 + U₀(x, y, z) = 0 set!(U, U₀) - V₀(x, y) = 0 + V₀(x, y, z) = 0 set!(V, V₀) η̅ .= 0 @@ -140,9 +140,9 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, calc V_avg = 3 η₀(x, y, z) = η_avg set!(η, η₀) - U₀(x, y) = U_avg + U₀(x, y, z) = U_avg set!(U, U₀) - V₀(x, y) = V_avg + V₀(x, y, z) = V_avg set!(V, V₀) η̅ .= 0 From 782f247fd8b3a0cef6d5e76e642403303aa9c090 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 9 Jan 2024 17:27:55 -0500 Subject: [PATCH 131/567] first bugfix --- test/test_split_explicit_free_surface_solver.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_split_explicit_free_surface_solver.jl b/test/test_split_explicit_free_surface_solver.jl index 6ce237ccd7..84f00003f9 100644 --- a/test/test_split_explicit_free_surface_solver.jl +++ b/test/test_split_explicit_free_surface_solver.jl @@ -246,11 +246,11 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, calc # ∂ₜₜ(η) = Δη η_exact = cos(ω * T) * (Array(η.data.parent)[2:Nx+1, 2:Ny+1] .- 1) .+ 1 - U₀(x, y) = kx * cos(kx * x) * sin(ky * y) # ∂ₜU = - ∂x(η), since we know η + U₀(x, y, z) = kx * cos(kx * x) * sin(ky * y) # ∂ₜU = - ∂x(η), since we know η set!(U, U₀) U_exact = -(sin(ω * T) * 1 / ω) .* Array(U.data.parent)[2:Nx+1, 2:Ny+1] .+ gu_c * T - V₀(x, y) = ky * sin(kx * x) * cos(ky * y) # ∂ₜV = - ∂y(η), since we know η + V₀(x, y, z) = ky * sin(kx * x) * cos(ky * y) # ∂ₜV = - ∂y(η), since we know η set!(V, V₀) V_exact = -(sin(ω * T) * 1 / ω) .* Array(V.data.parent)[2:Nx+1, 2:Ny+1] .+ gv_c * T From 7b92c64905288b973e7037fb7ffcb0ce56cd5a42 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 9 Jan 2024 17:47:38 -0500 Subject: [PATCH 132/567] correct error --- .../split_explicit_free_surface.jl | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index a88bf409bf..15f157e848 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -130,7 +130,7 @@ Base.@kwdef struct SplitExplicitState{CC, ACC, FC, AFC, CF, ACF} "The time-filtered barotropic zonal velocity. (`ReducedField` over ``z``)" U̅ :: FC "The time-filtered barotropic meridional velocity. (`ReducedField` over ``z``)" - V̅ :: FC + V̅ :: CF end """ @@ -149,16 +149,16 @@ function SplitExplicitState(grid::AbstractGrid, timestepper) ηᵐ⁻¹ = auxiliary_free_surface_field(grid, timestepper) ηᵐ⁻² = auxiliary_free_surface_field(grid, timestepper) - U = ZFaceField(grid, indices = (:, :, size(grid, 3))) - V = ZFaceField(grid, indices = (:, :, size(grid, 3))) + U = XFaceField(grid, indices = (:, :, size(grid, 3))) + V = YFaceField(grid, indices = (:, :, size(grid, 3))) - Uᵐ⁻¹ = auxiliary_barotropic_velocity_field(grid, timestepper) - Vᵐ⁻¹ = auxiliary_barotropic_velocity_field(grid, timestepper) - Uᵐ⁻² = auxiliary_barotropic_velocity_field(grid, timestepper) - Vᵐ⁻² = auxiliary_barotropic_velocity_field(grid, timestepper) + Uᵐ⁻¹ = auxiliary_barotropic_U_field(grid, timestepper) + Vᵐ⁻¹ = auxiliary_barotropic_V_field(grid, timestepper) + Uᵐ⁻² = auxiliary_barotropic_U_field(grid, timestepper) + Vᵐ⁻² = auxiliary_barotropic_V_field(grid, timestepper) - U̅ = ZFaceField(grid, indices = (:, :, size(grid, 3))) - V̅ = ZFaceField(grid, indices = (:, :, size(grid, 3))) + U̅ = XFaceField(grid, indices = (:, :, size(grid, 3))) + V̅ = YFaceField(grid, indices = (:, :, size(grid, 3))) return SplitExplicitState(; ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅) end @@ -196,12 +196,12 @@ Return the `SplitExplicitAuxiliaryFields` for `grid`. """ function SplitExplicitAuxiliaryFields(grid::AbstractGrid) - Gᵁ = ZFaceField(grid, indices = (:, :, size(grid, 3))) - Gⱽ = ZFaceField(grid, indices = (:, :, size(grid, 3))) + Gᵁ = XFaceField(grid, indices = (:, :, size(grid, 3))) + Gⱽ = YFaceField(grid, indices = (:, :, size(grid, 3))) - Hᶠᶜ = ZFaceField(grid, indices = (:, :, size(grid, 3))) - Hᶜᶠ = ZFaceField(grid, indices = (:, :, size(grid, 3))) - Hᶜᶜ = ZFaceField(grid, indices = (:, :, size(grid, 3))) + Hᶠᶜ = XFaceField(grid, indices = (:, :, size(grid, 3))) + Hᶜᶠ = YFaceField(grid, indices = (:, :, size(grid, 3))) + Hᶜᶜ = CenterField(grid, indices = (:, :, size(grid, 3))) dz = GridMetricOperation((Face, Center, Center), Δz, grid) Hᶠᶜ .= sum(dz; dims = 3) @@ -238,8 +238,10 @@ struct ForwardBackwardScheme end auxiliary_free_surface_field(grid, ::AdamsBashforth3Scheme) = ZFaceField(grid, indices = (:, :, size(grid, 3)+1)) auxiliary_free_surface_field(grid, ::ForwardBackwardScheme) = nothing -auxiliary_barotropic_velocity_field(grid, ::AdamsBashforth3Scheme) = ZFaceField(grid, indices = (:, :, size(grid, 3))) -auxiliary_barotropic_velocity_field(grid, ::ForwardBackwardScheme) = nothing +auxiliary_barotropic_U_field(grid, ::AdamsBashforth3Scheme) = XFaceField(grid, indices = (:, :, size(grid, 3))) +auxiliary_barotropic_U_field(grid, ::ForwardBackwardScheme) = nothing +auxiliary_barotropic_V_field(grid, ::AdamsBashforth3Scheme) = YFaceField(grid, indices = (:, :, size(grid, 3))) +auxiliary_barotropic_V_field(grid, ::ForwardBackwardScheme) = nothing # (p = 2, q = 4, r = 0.18927) minimize dispersion error from Shchepetkin and McWilliams (2005): https://doi.org/10.1016/j.ocemod.2004.08.002 @inline function averaging_shape_function(τ::FT; p = 2, q = 4, r = FT(0.18927)) where FT From 5e6dcb981a0d57f648513a032e6508bdfc0717da Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 9 Jan 2024 17:54:46 -0500 Subject: [PATCH 133/567] some bugfixes --- ...distributed_split_explicit_free_surface.jl | 19 +---------- .../split_explicit_free_surface.jl | 23 +------------ .../split_explicit_free_surface_kernels.jl | 34 +++++++++++-------- ...ulti_region_split_explicit_free_surface.jl | 13 +------ 4 files changed, 23 insertions(+), 66 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl index 6f5067e555..be65080829 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl @@ -10,30 +10,13 @@ function SplitExplicitAuxiliaryFields(grid::DistributedGrid) Gᵁ = Field((Face, Center, Nothing), grid) Gⱽ = Field((Center, Face, Nothing), grid) - Hᶠᶜ = Field((Face, Center, Nothing), grid) - Hᶜᶠ = Field((Center, Face, Nothing), grid) - Hᶜᶜ = Field((Center, Center, Nothing), grid) - - calculate_column_height!(Hᶠᶜ, (Face, Center, Center)) - calculate_column_height!(Hᶜᶠ, (Center, Face, Center)) - - calculate_column_height!(Hᶜᶜ, (Center, Center, Center)) - - fill_halo_regions!((Hᶠᶜ, Hᶜᶠ, Hᶜᶜ)) - # In a non-parallel grid we calculate only the interior kernel_size = augmented_kernel_size(grid) kernel_offsets = augmented_kernel_offsets(grid) kernel_parameters = KernelParameters(kernel_size, kernel_offsets) - return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, kernel_parameters) -end - -"""Integrate z at locations `location` and set! `height`` with the result""" -@inline function calculate_column_height!(height, location) - dz = GridMetricOperation(location, Δz, height.grid) - return sum!(height, dz) + return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, kernel_parameters) end @inline function augmented_kernel_size(grid::DistributedGrid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index 15f157e848..92edaff25d 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -179,12 +179,6 @@ Base.@kwdef struct SplitExplicitAuxiliaryFields{𝒞ℱ, ℱ𝒞, 𝒞𝒞, 𝒦 Gᵁ :: ℱ𝒞 "Vertically-integrated slow barotropic forcing function for `V` (`ReducedField` over ``z``)" Gⱽ :: 𝒞ℱ - "Depth at `(Face, Center)` (`ReducedField` over ``z``)" - Hᶠᶜ :: ℱ𝒞 - "Depth at `(Center, Face)` (`ReducedField` over ``z``)" - Hᶜᶠ :: 𝒞ℱ - "Depth at `(Center, Center)` (`ReducedField` over ``z``)" - Hᶜᶜ :: 𝒞𝒞 "kernel size for barotropic time stepping" kernel_parameters :: 𝒦 end @@ -199,24 +193,9 @@ function SplitExplicitAuxiliaryFields(grid::AbstractGrid) Gᵁ = XFaceField(grid, indices = (:, :, size(grid, 3))) Gⱽ = YFaceField(grid, indices = (:, :, size(grid, 3))) - Hᶠᶜ = XFaceField(grid, indices = (:, :, size(grid, 3))) - Hᶜᶠ = YFaceField(grid, indices = (:, :, size(grid, 3))) - Hᶜᶜ = CenterField(grid, indices = (:, :, size(grid, 3))) - - dz = GridMetricOperation((Face, Center, Center), Δz, grid) - Hᶠᶜ .= sum(dz; dims = 3) - - dz = GridMetricOperation((Center, Face, Center), Δz, grid) - Hᶜᶠ .= sum(dz; dims = 3) - - dz = GridMetricOperation((Center, Center, Center), Δz, grid) - Hᶜᶜ .= sum(dz; dims = 3) - - fill_halo_regions!((Hᶠᶜ, Hᶜᶠ, Hᶜᶜ)) - kernel_parameters = :xy - return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, kernel_parameters) + return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, kernel_parameters) end """ diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index d0075e5aec..ca98e44030 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -11,6 +11,8 @@ using Oceananigans.ImmersedBoundaries: inactive_node, IBG, c, f, SurfaceMap using Oceananigans.ImmersedBoundaries: mask_immersed_field!, use_only_active_surface_cells, use_only_active_interior_cells using Oceananigans.ImmersedBoundaries: active_linear_index_to_tuple, ActiveCellsIBG, ActiveSurfaceIBG using Oceananigans.DistributedComputations: child_architecture +using Oceananigans.DistributedComputations: Distributed +using Printf # constants for AB3 time stepping scheme (from https://doi.org/10.1016/j.ocemod.2004.08.002) const β = 0.281105 @@ -136,8 +138,13 @@ end return nothing end -using Oceananigans.DistributedComputations: Distributed -using Printf +# Column height for the split explicit solver +@inline column_heightᶜᶜ(i, j, k, grid) = grid.Lz +@inline column_heightᶜᶜ(i, j, k, grid::ImmersedBoundaryGrid) = min(grid.Lz, @inbounds grid.immersed_boundary.bottom_height[i, j, 1]) + +@inline column_heightᶠᶜ(i, j, grid) = ℑxᶠᵃᵃ(i, j, 1, grid, column_heightᶜᶜ) +@inline column_heightᶜᶠ(i, j, grid) = ℑyᵃᶠᵃ(i, j, 1, grid, column_heightᶜᶜ) + @kernel function _split_explicit_free_surface!(grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) i, j = @index(Global, NTuple) @@ -160,20 +167,20 @@ end @kernel function _split_explicit_barotropic_velocity!(averaging_weight, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, + η̅, U̅, V̅, Gᵁ, Gⱽ, g, timestepper) i, j = @index(Global, NTuple) velocity_evolution!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅, averaging_weight, - Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, + Gᵁ, Gⱽ, g, timestepper) end @inline function velocity_evolution!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅, averaging_weight, - Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, + Gᵁ, Gⱽ, g, timestepper) k_top = grid.Nz+1 @@ -184,8 +191,8 @@ end advance_previous_velocity!(i, j, k_top-1, timestepper, V, Vᵐ⁻¹, Vᵐ⁻²) # ∂τ(U) = - ∇η + G - U[i, j, k_top-1] += Δτ * (- g * Hᶠᶜ[i, j, k_top-1] * ∂xᶠᶜᶠ_η(i, j, k_top, grid, TX, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gᵁ[i, j, k_top-1]) - V[i, j, k_top-1] += Δτ * (- g * Hᶜᶠ[i, j, k_top-1] * ∂yᶜᶠᶠ_η(i, j, k_top, grid, TY, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gⱽ[i, j, k_top-1]) + U[i, j, k_top-1] += Δτ * (- g * column_heightᶠᶜ(i, j, grid) * ∂xᶠᶜᶠ_η(i, j, k_top, grid, TX, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gᵁ[i, j, k_top-1]) + V[i, j, k_top-1] += Δτ * (- g * column_heightᶜᶠ(i, j, grid) * ∂yᶜᶠᶠ_η(i, j, k_top, grid, TY, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gⱽ[i, j, k_top-1]) # time-averaging η̅[i, j, k_top] += averaging_weight * η[i, j, k_top] @@ -261,20 +268,19 @@ function initialize_auxiliary_state!(state, η, timestepper) return nothing end -@kernel function _barotropic_split_explicit_corrector!(u, v, U̅, V̅, U, V, Hᶠᶜ, Hᶜᶠ, grid) +@kernel function _barotropic_split_explicit_corrector!(u, v, U̅, V̅, U, V, grid) i, j, k = @index(Global, NTuple) k_top = grid.Nz+1 @inbounds begin - u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / Hᶠᶜ[i, j, k_top-1] - v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / Hᶜᶠ[i, j, k_top-1] + u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / column_heightᶠᶜ(i, j, grid) + v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / column_heightᶜᶠ(i, j, grid) end end # may need to do Val(Nk) since it may not be known at compile. Also figure out where to put H function barotropic_split_explicit_corrector!(u, v, free_surface, grid) sefs = free_surface.state - Hᶠᶜ, Hᶜᶠ = free_surface.auxiliary.Hᶠᶜ, free_surface.auxiliary.Hᶜᶠ U, V, U̅, V̅ = sefs.U, sefs.V, sefs.U̅, sefs.V̅ arch = architecture(grid) @@ -283,7 +289,7 @@ function barotropic_split_explicit_corrector!(u, v, free_surface, grid) compute_barotropic_mode!(U, V, grid, u, v) # add in "good" barotropic mode launch!(arch, grid, :xyz, _barotropic_split_explicit_corrector!, - u, v, U̅, V̅, U, V, Hᶠᶜ, Hᶜᶠ, grid) + u, v, U̅, V̅, U, V, grid) return nothing end @@ -375,7 +381,7 @@ function iterate_split_explicit!(free_surface, grid, Δτᴮ, weights, ::Val{Nsu Vᵐ⁻¹, Vᵐ⁻² = state.Vᵐ⁻¹, state.Vᵐ⁻² ηᵐ, ηᵐ⁻¹, ηᵐ⁻² = state.ηᵐ, state.ηᵐ⁻¹, state.ηᵐ⁻² η̅, U̅, V̅ = state.η̅, state.U̅, state.V̅ - Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ = auxiliary.Gᵁ, auxiliary.Gⱽ, auxiliary.Hᶠᶜ, auxiliary.Hᶜᶠ + Gᵁ, Gⱽ = auxiliary.Gᵁ, auxiliary.Gⱽ timestepper = settings.timestepper @@ -390,7 +396,7 @@ function iterate_split_explicit!(free_surface, grid, Δτᴮ, weights, ::Val{Nsu U_args = (grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, Gᵁ, Gⱽ, g, Hᶠᶜ, Hᶜᶠ, + η̅, U̅, V̅, Gᵁ, Gⱽ, g, timestepper) GC.@preserve η_args U_args begin diff --git a/src/MultiRegion/multi_region_split_explicit_free_surface.jl b/src/MultiRegion/multi_region_split_explicit_free_surface.jl index c645452038..59dd3770b1 100644 --- a/src/MultiRegion/multi_region_split_explicit_free_surface.jl +++ b/src/MultiRegion/multi_region_split_explicit_free_surface.jl @@ -9,24 +9,13 @@ function SplitExplicitAuxiliaryFields(grid::MultiRegionGrids) Gᵁ = Field((Face, Center, Nothing), grid) Gⱽ = Field((Center, Face, Nothing), grid) - Hᶠᶜ = Field((Face, Center, Nothing), grid) - Hᶜᶠ = Field((Center, Face, Nothing), grid) - Hᶜᶜ = Field((Center, Center, Nothing), grid) - - @apply_regionally calculate_column_height!(Hᶠᶜ, (Face, Center, Center)) - @apply_regionally calculate_column_height!(Hᶜᶠ, (Center, Face, Center)) - - @apply_regionally calculate_column_height!(Hᶜᶜ, (Center, Center, Center)) - - fill_halo_regions!((Hᶠᶜ, Hᶜᶠ, Hᶜᶜ)) - # In a non-parallel grid we calculate only the interior @apply_regionally kernel_size = augmented_kernel_size(grid, grid.partition) @apply_regionally kernel_offsets = augmented_kernel_offsets(grid, grid.partition) @apply_regionally kernel_parameters = KernelParameters(kernel_size, kernel_offsets) - return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, kernel_parameters) + return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, kernel_parameters) end @inline function calculate_column_height!(height, location) From 056def573322cac0782d055f56339d73ea98171b Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 9 Jan 2024 17:59:05 -0500 Subject: [PATCH 134/567] bugfix --- .../HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index 92edaff25d..e37e567fb4 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -174,7 +174,7 @@ large (or `:xy` in case of a serial computation), and start computing from $(FIELDS) """ -Base.@kwdef struct SplitExplicitAuxiliaryFields{𝒞ℱ, ℱ𝒞, 𝒞𝒞, 𝒦} +Base.@kwdef struct SplitExplicitAuxiliaryFields{𝒞ℱ, ℱ𝒞, 𝒦} "Vertically-integrated slow barotropic forcing function for `U` (`ReducedField` over ``z``)" Gᵁ :: ℱ𝒞 "Vertically-integrated slow barotropic forcing function for `V` (`ReducedField` over ``z``)" From 630f0fafab8ab334ed575629ba81db4a95c57412 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 9 Jan 2024 18:01:07 -0500 Subject: [PATCH 135/567] slightly more optim --- .../distributed_split_explicit_free_surface.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl index be65080829..f17331731e 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl @@ -7,8 +7,8 @@ import Oceananigans.Models.HydrostaticFreeSurfaceModels: FreeSurface, SplitExpli function SplitExplicitAuxiliaryFields(grid::DistributedGrid) - Gᵁ = Field((Face, Center, Nothing), grid) - Gⱽ = Field((Center, Face, Nothing), grid) + Gᵁ = XFaceField(grid, indices = (:, :, size(grid, 3))) + Gⱽ = YFaceField(grid, indices = (:, :, size(grid, 3))) # In a non-parallel grid we calculate only the interior kernel_size = augmented_kernel_size(grid) From e70a57d8a5ea5a2fb54473addef4fe9e2086e9ba Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 9 Jan 2024 18:21:35 -0500 Subject: [PATCH 136/567] simplifying more --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 10 ++++++++++ .../split_explicit_free_surface.jl | 2 +- .../split_explicit_free_surface_kernels.jl | 11 ++++++++--- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index e693e20e2d..bea53d46cd 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -72,12 +72,22 @@ Computes ib.bottom_height and wraps in an array. function ImmersedBoundaryGrid(grid, ib::GridFittedBottom) bottom_field = Field{Center, Center, Nothing}(grid) set!(bottom_field, ib.bottom_height) + launch!(architecture(grid), grid, :xy, _limit_bottom_heigth!, bottom_field, grid.Lz) + fill_halo_regions!(bottom_field) new_ib = GridFittedBottom(bottom_field, ib.immersed_condition) TX, TY, TZ = topology(grid) return ImmersedBoundaryGrid{TX, TY, TZ}(grid, new_ib) end +# Make sure that `abs(bottom_height) <= grid.Lz` to constrain the bottom +@kernel function _limit_bottom_heigth!(bottom_field, Lz) + i, j = @index(Global, NTuple) + if abs(bottom_field[i, j, 1]) > Lz + bottom_field[i, j, 1] = sign(bottom_field[i, j, 1]) * Lz + end +end + @inline function _immersed_cell(i, j, k, underlying_grid, ib::GridFittedBottom{<:Any, <:InterfaceImmersedCondition}) z = znode(i, j, k+1, underlying_grid, c, c, f) h = @inbounds ib.bottom_height[i, j, 1] diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index e37e567fb4..4132154897 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -68,7 +68,7 @@ Keyword Arguments is chosen. - `timestepper`: Time stepping scheme used for the barotropic advancement. Choose one of: - - `ForwardBackwardScheme()` (default): `η = f(U)` then `U = f(η)`, + - `ForwardBackwardScheme()` (default): `ηᵐ⁺¹ = f(U)` then `U = f(η)`, - `AdamsBashforth3Scheme()`: `η = f(U, Uᵐ⁻¹, Uᵐ⁻²)` then `U = f(η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²)`. """ SplitExplicitFreeSurface(FT::DataType = Float64; gravitational_acceleration = g_Earth, kwargs...) = diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index ca98e44030..c9910297b6 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -6,7 +6,7 @@ using Oceananigans.AbstractOperations: Δz using Oceananigans.BoundaryConditions using Oceananigans.Operators using Oceananigans.Architectures: convert_args -using Oceananigans.ImmersedBoundaries: peripheral_node, immersed_inactive_node +using Oceananigans.ImmersedBoundaries: peripheral_node, immersed_inactive_node, GFBIBG using Oceananigans.ImmersedBoundaries: inactive_node, IBG, c, f, SurfaceMap using Oceananigans.ImmersedBoundaries: mask_immersed_field!, use_only_active_surface_cells, use_only_active_interior_cells using Oceananigans.ImmersedBoundaries: active_linear_index_to_tuple, ActiveCellsIBG, ActiveSurfaceIBG @@ -139,8 +139,13 @@ end end # Column height for the split explicit solver -@inline column_heightᶜᶜ(i, j, k, grid) = grid.Lz -@inline column_heightᶜᶜ(i, j, k, grid::ImmersedBoundaryGrid) = min(grid.Lz, @inbounds grid.immersed_boundary.bottom_height[i, j, 1]) +@inline column_heightᶜᶜ(i, j, k, grid) = grid.Lz + +# Column height for an GridFitted bottom immersed boundary +@inline function column_heightᶜᶜ(i, j, k, grid::GFBIBG) + bottom = grid.immersed_boundary.bottom_height[i, j, 1] + return ifelse(bottom < 0, - bottom, grid.Lz - bottom) +end @inline column_heightᶠᶜ(i, j, grid) = ℑxᶠᵃᵃ(i, j, 1, grid, column_heightᶜᶜ) @inline column_heightᶜᶠ(i, j, grid) = ℑyᵃᶠᵃ(i, j, 1, grid, column_heightᶜᶜ) From c1c310157ee540f2b072a37f4e7b5d34bd5bd8bd Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 9 Jan 2024 19:46:17 -0500 Subject: [PATCH 137/567] all tests should be ok --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 8 +- ...distributed_split_explicit_free_surface.jl | 6 +- .../split_explicit_free_surface.jl | 22 +++-- ...ulti_region_split_explicit_free_surface.jl | 6 +- ...test_split_explicit_free_surface_solver.jl | 80 +++++-------------- .../test_split_explicit_vertical_integrals.jl | 27 +++---- 6 files changed, 59 insertions(+), 90 deletions(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index bea53d46cd..f1ca0f1e4b 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -72,8 +72,10 @@ Computes ib.bottom_height and wraps in an array. function ImmersedBoundaryGrid(grid, ib::GridFittedBottom) bottom_field = Field{Center, Center, Nothing}(grid) set!(bottom_field, ib.bottom_height) - launch!(architecture(grid), grid, :xy, _limit_bottom_heigth!, bottom_field, grid.Lz) - + + # Make sure that `abs(bottom_height) <= grid.Lz` to constrain the bottom + @apply_regionally launch!(architecture(grid), grid, :xy, _limit_bottom_height!, bottom_field, grid.Lz) + fill_halo_regions!(bottom_field) new_ib = GridFittedBottom(bottom_field, ib.immersed_condition) TX, TY, TZ = topology(grid) @@ -81,7 +83,7 @@ function ImmersedBoundaryGrid(grid, ib::GridFittedBottom) end # Make sure that `abs(bottom_height) <= grid.Lz` to constrain the bottom -@kernel function _limit_bottom_heigth!(bottom_field, Lz) +@kernel function _limit_bottom_height!(bottom_field, Lz) i, j = @index(Global, NTuple) if abs(bottom_field[i, j, 1]) > Lz bottom_field[i, j, 1] = sign(bottom_field[i, j, 1]) * Lz diff --git a/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl index f17331731e..d7f0fe42c5 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl @@ -7,8 +7,10 @@ import Oceananigans.Models.HydrostaticFreeSurfaceModels: FreeSurface, SplitExpli function SplitExplicitAuxiliaryFields(grid::DistributedGrid) - Gᵁ = XFaceField(grid, indices = (:, :, size(grid, 3))) - Gⱽ = YFaceField(grid, indices = (:, :, size(grid, 3))) + Nz = size(grid, 3) + + Gᵁ = XFaceField(grid, indices = (:, :, Nz)) + Gⱽ = YFaceField(grid, indices = (:, :, Nz)) # In a non-parallel grid we calculate only the interior kernel_size = augmented_kernel_size(grid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index 4132154897..3d88d92c18 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -92,7 +92,8 @@ function SplitExplicitFreeSurface(grid; gravitational_acceleration = g_Earth, @warn "Using $(eltype(settings)) settings for the SplitExplicitFreeSurface on a $(eltype(grid)) grid" end - η = ZFaceField(grid, indices = (:, :, size(grid, 3)+1)) + Nz = size(grid, 3) + η = ZFaceField(grid, indices = (:, :, Nz+1)) gravitational_acceleration = convert(eltype(grid), gravitational_acceleration) return SplitExplicitFreeSurface(η, SplitExplicitState(grid, settings.timestepper), SplitExplicitAuxiliaryFields(grid), @@ -143,22 +144,25 @@ acts as a filter for `η`. Values with superscripts `m-1` and `m-2` correspond t time steps to allow using a higher-order time stepping scheme, e.g., `AdamsBashforth3Scheme`. """ function SplitExplicitState(grid::AbstractGrid, timestepper) - η̅ = ZFaceField(grid, indices = (:, :, size(grid, 3)+1)) + + Nz = size(grid, 3) + + η̅ = ZFaceField(grid, indices = (:, :, Nz+1)) ηᵐ = auxiliary_free_surface_field(grid, timestepper) ηᵐ⁻¹ = auxiliary_free_surface_field(grid, timestepper) ηᵐ⁻² = auxiliary_free_surface_field(grid, timestepper) - U = XFaceField(grid, indices = (:, :, size(grid, 3))) - V = YFaceField(grid, indices = (:, :, size(grid, 3))) + U = XFaceField(grid, indices = (:, :, Nz)) + V = YFaceField(grid, indices = (:, :, Nz)) Uᵐ⁻¹ = auxiliary_barotropic_U_field(grid, timestepper) Vᵐ⁻¹ = auxiliary_barotropic_V_field(grid, timestepper) Uᵐ⁻² = auxiliary_barotropic_U_field(grid, timestepper) Vᵐ⁻² = auxiliary_barotropic_V_field(grid, timestepper) - U̅ = XFaceField(grid, indices = (:, :, size(grid, 3))) - V̅ = YFaceField(grid, indices = (:, :, size(grid, 3))) + U̅ = XFaceField(grid, indices = (:, :, Nz)) + V̅ = YFaceField(grid, indices = (:, :, Nz)) return SplitExplicitState(; ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅) end @@ -190,8 +194,10 @@ Return the `SplitExplicitAuxiliaryFields` for `grid`. """ function SplitExplicitAuxiliaryFields(grid::AbstractGrid) - Gᵁ = XFaceField(grid, indices = (:, :, size(grid, 3))) - Gⱽ = YFaceField(grid, indices = (:, :, size(grid, 3))) + Nz = size(grid, 3) + + Gᵁ = XFaceField(grid, indices = (:, :, Nz)) + Gⱽ = YFaceField(grid, indices = (:, :, Nz)) kernel_parameters = :xy diff --git a/src/MultiRegion/multi_region_split_explicit_free_surface.jl b/src/MultiRegion/multi_region_split_explicit_free_surface.jl index 59dd3770b1..3e3c4ec842 100644 --- a/src/MultiRegion/multi_region_split_explicit_free_surface.jl +++ b/src/MultiRegion/multi_region_split_explicit_free_surface.jl @@ -6,8 +6,10 @@ import Oceananigans.Models.HydrostaticFreeSurfaceModels: FreeSurface, SplitExpli function SplitExplicitAuxiliaryFields(grid::MultiRegionGrids) - Gᵁ = Field((Face, Center, Nothing), grid) - Gⱽ = Field((Center, Face, Nothing), grid) + Nz = size(grid, 3) + + Gᵁ = XFaceField(grid; indices = (:, :, Nz)) + Gⱽ = YFaceField(grid; indices = (:, :, Nz)) # In a non-parallel grid we calculate only the interior @apply_regionally kernel_size = augmented_kernel_size(grid, grid.partition) diff --git a/test/test_split_explicit_free_surface_solver.jl b/test/test_split_explicit_free_surface_solver.jl index 84f00003f9..4302a182c7 100644 --- a/test/test_split_explicit_free_surface_solver.jl +++ b/test/test_split_explicit_free_surface_solver.jl @@ -1,7 +1,7 @@ include("dependencies_for_runtests.jl") using Oceananigans.Models.HydrostaticFreeSurfaceModels import Oceananigans.Models.HydrostaticFreeSurfaceModels: SplitExplicitFreeSurface -import Oceananigans.Models.HydrostaticFreeSurfaceModels: SplitExplicitState, SplitExplicitAuxiliaryFields, SplitExplicitSettings, split_explicit_free_surface_substep! +import Oceananigans.Models.HydrostaticFreeSurfaceModels: SplitExplicitState, SplitExplicitAuxiliaryFields, SplitExplicitSettings, iterate_split_explicit! using Oceananigans.Models.HydrostaticFreeSurfaceModels: constant_averaging_kernel using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, calculate_adaptive_settings @@ -12,51 +12,35 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, calc for arch in archs topology = (Periodic, Periodic, Bounded) - Nx, Ny, Nz = 128, 64, 16 - Lx = Ly = Lz = 2π + Nx, Ny, Nz = 128, 1, 1 + Lx = Ly = 2π + Lz = 1 / Oceananigans.BuoyancyModels.g_Earth grid = RectilinearGrid(arch, FT; topology, size = (Nx, Ny, Nz), x = (0, Lx), y = (0, Ly), z = (-Lz, 0), halo=(1, 1, 1)) - settings = SplitExplicitSettings(; substeps = 200, averaging_kernel = constant_averaging_kernel) + settings = SplitExplicitSettings(eltype(grid); substeps = 200, averaging_kernel = constant_averaging_kernel) sefs = SplitExplicitFreeSurface(grid; settings) sefs.η .= 0 @testset " One timestep test " begin state = sefs.state - auxiliary = sefs.auxiliary U, V, η̅, U̅, V̅ = state.U, state.V, state.η̅, state.U̅, state.V̅ - Gᵁ, Gⱽ = auxiliary.Gᵁ, auxiliary.Gⱽ - Hᶠᶜ, Hᶜᶠ = sefs.auxiliary.Hᶠᶜ, sefs.auxiliary.Hᶜᶠ - g = sefs.gravitational_acceleration - Hᶠᶜ .= 1 / g - Hᶜᶠ .= 1 / g η = sefs.η - velocity_weight = 0.0 - free_surface_weight = 0.0 Δτ = 1.0 η₀(x, y, z) = sin(x) set!(η, η₀) - U₀(x, y, z) = 0 - set!(U, U₀) - V₀(x, y, z) = 0 - set!(V, V₀) - - η̅ .= 0 - U̅ .= 0 - V̅ .= 0 - Gᵁ .= 0 - Gⱽ .= 0 - - Nsubsteps = calculate_substeps(settings.substepping, 1) + + Nsubsteps = calculate_substeps(settings.substepping, 1) fractional_Δt, weights = calculate_adaptive_settings(settings.substepping, Nsubsteps) # barotropic time step in fraction of baroclinic step and averaging weights - split_explicit_free_surface_substep!(η, sefs.state, sefs.auxiliary, sefs.settings, weights, arch, grid, g, Δτ, 1) + iterate_split_explicit!(sefs, grid, Δτ, weights, Val(1)) + U_computed = Array(U.data.parent)[2:Nx+1, 2:Ny+1] U_exact = (reshape(-cos.(grid.xᶠᵃᵃ), (length(grid.xᶜᵃᵃ), 1)).+reshape(0 * grid.yᵃᶜᵃ, (1, length(grid.yᵃᶜᵃ))))[2:Nx+1, 2:Ny+1] @@ -68,19 +52,14 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, calc auxiliary = sefs.auxiliary U, V, η̅, U̅, V̅ = state.U, state.V, state.η̅, state.U̅, state.V̅ Gᵁ, Gⱽ = auxiliary.Gᵁ, auxiliary.Gⱽ - g = sefs.gravitational_acceleration - sefs.auxiliary.Hᶠᶜ .= 1 / g - sefs.auxiliary.Hᶜᶠ .= 1 / g η = sefs.η - velocity_weight = 0.0 - free_surface_weight = 0.0 - T = 2π - Δτ = 2π / maximum([Nx, Ny]) * 5e-2 # the last factor is essentially the order of accuracy + T = 2π + Δτ = 2π / maximum([Nx, Ny]) * 5e-1 # the last factor is essentially the order of accuracy Nt = floor(Int, T / Δτ) Δτ_end = T - Nt * Δτ - settings = SplitExplicitSettings(; substeps = Nt, averaging_kernel = constant_averaging_kernel) + settings = SplitExplicitSettings(eltype(grid); substeps = Nt, averaging_kernel = constant_averaging_kernel) # set!(η, f(x,y)) η₀(x, y, z) = sin(x) @@ -90,22 +69,17 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, calc V₀(x, y, z) = 0 set!(V, V₀) - η̅ .= 0 - U̅ .= 0 - V̅ .= 0 + η̅ .= 0 + U̅ .= 0 + V̅ .= 0 Gᵁ .= 0 Gⱽ .= 0 Nsubsteps = calculate_substeps(settings.substepping, 1) fractional_Δt, weights = calculate_adaptive_settings(settings.substepping, Nsubsteps) # barotropic time step in fraction of baroclinic step and averaging weights - - for i in 1:Nt - split_explicit_free_surface_substep!(η, sefs.state, sefs.auxiliary, settings, weights, arch, grid, g, Δτ, i) - end - - # + correction for exact time - split_explicit_free_surface_substep!(η, sefs.state, sefs.auxiliary, settings, weights, arch, grid, g, Δτ_end, 1) + iterate_split_explicit!(sefs, grid, Δτ, weights, Val(Nsubsteps)) + U_computed = Array(parent(U))[2:Nx+1, 2:Ny+1] η_computed = Array(parent(η))[2:Nx+1, 2:Ny+1] set!(η, η₀) @@ -126,13 +100,9 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, calc Gᵁ, Gⱽ = auxiliary.Gᵁ, auxiliary.Gⱽ g = sefs.gravitational_acceleration - sefs.auxiliary.Hᶠᶜ .= 1 / g - sefs.auxiliary.Hᶜᶠ .= 1 / g η = sefs.η - velocity_weight = 0.0 - free_surface_weight = 0.0 - Δτ = 2π / maximum([Nx, Ny]) * 5e-2 # the last factor is essentially the order of accuracy + Δτ = 2π / maximum([Nx, Ny]) * 5e-1 # the last factor is essentially the order of accuracy # set!(η, f(x, y)) η_avg = 1 @@ -155,9 +125,7 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, calc Nsubsteps = calculate_substeps(settings.substepping, 1) fractional_Δt, weights = calculate_adaptive_settings(settings.substepping, Nsubsteps) # barotropic time step in fraction of baroclinic step and averaging weights - for i in 1:Nsubsteps - split_explicit_free_surface_substep!(η, sefs.state, sefs.auxiliary, sefs.settings, weights, arch, grid, g, Δτ, i) - end + iterate_split_explicit!(sefs, grid, Δτ, weights, Val(Nsubsteps)) U_computed = Array(U.data.parent)[2:Nx+1, 2:Ny+1] V_computed = Array(V.data.parent)[2:Nx+1, 2:Ny+1] @@ -197,8 +165,6 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, calc Gᵁ, Gⱽ = auxiliary.Gᵁ, auxiliary.Gⱽ η = sefs.η g = sefs.gravitational_acceleration - sefs.auxiliary.Hᶠᶜ .= 1 / g # to make life easy - sefs.auxiliary.Hᶜᶠ .= 1 / g # to make life easy # set!(η, f(x,y)) k² = ω² gu_c = 1 @@ -221,12 +187,8 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, calc Nsubsteps = calculate_substeps(settings.substepping, 1) fractional_Δt, weights = calculate_adaptive_settings(settings.substepping, Nsubsteps) # barotropic time step in fraction of baroclinic step and averaging weights - - for i in 1:Nt - split_explicit_free_surface_substep!(η, sefs.state, sefs.auxiliary, sefs.settings, weights, arch, grid, g, Δτ, i) - end - # + correction for exact time - split_explicit_free_surface_substep!(η, sefs.state, sefs.auxiliary, sefs.settings, weights, arch, grid, g, Δτ_end, Nt + 1) + + iterate_split_explicit!(sefs, grid, Δτ, weights, Val(Nsubsteps)) η_mean_after = mean(Array(interior(η))) diff --git a/test/test_split_explicit_vertical_integrals.jl b/test/test_split_explicit_vertical_integrals.jl index aad8fbd8d6..6016053926 100644 --- a/test/test_split_explicit_vertical_integrals.jl +++ b/test/test_split_explicit_vertical_integrals.jl @@ -1,8 +1,8 @@ include("dependencies_for_runtests.jl") using Oceananigans.Models.HydrostaticFreeSurfaceModels -import Oceananigans.Models.HydrostaticFreeSurfaceModels: SplitExplicitFreeSurface -import Oceananigans.Models.HydrostaticFreeSurfaceModels: SplitExplicitState, SplitExplicitAuxiliaryFields, SplitExplicitSettings -import Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_barotropic_mode!, barotropic_split_explicit_corrector!, initialize_free_surface_state! +using Oceananigans.Models.HydrostaticFreeSurfaceModels: SplitExplicitFreeSurface +using Oceananigans.Models.HydrostaticFreeSurfaceModels: SplitExplicitState, SplitExplicitAuxiliaryFields, SplitExplicitSettings +using Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_barotropic_mode!, barotropic_split_explicit_corrector!, initialize_free_surface_state! @testset "Barotropic Kernels" begin @@ -17,7 +17,7 @@ import Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_barotropic_mode grid = RectilinearGrid(arch, topology = topology, size = (Nx, Ny, Nz), x = (0, Lx), y = (0, Ly), z = (-Lz, 0)) tmp = SplitExplicitFreeSurface(; substeps = 200) - sefs = SplitExplicitState(grid) + sefs = SplitExplicitState(grid, tmp.settings.timestepper) sefs = SplitExplicitAuxiliaryFields(grid) sefs = SplitExplicitFreeSurface(grid) @@ -25,7 +25,6 @@ import Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_barotropic_mode auxiliary = sefs.auxiliary U, V, η̅, U̅, V̅ = state.U, state.V, state.η̅, state.U̅, state.V̅ Gᵁ, Gⱽ = auxiliary.Gᵁ, auxiliary.Gⱽ - Hᶠᶜ, Hᶜᶠ = sefs.auxiliary.Hᶠᶜ, sefs.auxiliary.Hᶜᶠ u = Field{Face,Center,Center}(grid) v = Field{Center,Face,Center}(grid) @@ -34,7 +33,7 @@ import Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_barotropic_mode # set equal to something else η̅ .= U̅ .= V̅ .= 1.0 # now set equal to zero - initialize_free_surface_state!(sefs.state, sefs.η) + initialize_free_surface_state!(sefs.state, sefs.η, sefs.settings.timestepper) # don't forget the ghost points fill_halo_regions!(η̅) fill_halo_regions!(U̅) @@ -51,7 +50,7 @@ import Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_barotropic_mode Δz .= grid.Δzᵃᵃᶠ set_u_check(x, y, z) = cos((π / 2) * z / Lz) - set_U_check(x, y) = (sin(0) - (-2 * Lz / (π))) + set_U_check(x, y, z) = (sin(0) - (-2 * Lz / (π))) set!(u, set_u_check) exact_U = similar(U) set!(exact_U, set_U_check) @@ -60,7 +59,7 @@ import Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_barotropic_mode @test all((Array(interior(U) .- interior(exact_U))) .< tolerance) set_v_check(x, y, z) = sin(x * y) * cos((π / 2) * z / Lz) - set_V_check(x, y) = sin(x * y) * (sin(0) - (-2 * Lz / (π))) + set_V_check(x, y, z) = sin(x * y) * (sin(0) - (-2 * Lz / (π))) set!(v, set_v_check) exact_V = similar(V) set!(exact_V, set_V_check) @@ -83,7 +82,7 @@ import Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_barotropic_mode @test all(Array(interior(U)) .≈ Lz) set_u_check(x, y, z) = sin(x) - set_U_check(x, y) = sin(x) * Lz + set_U_check(x, y, z) = sin(x) * Lz set!(u, set_u_check) exact_U = similar(U) set!(exact_U, set_U_check) @@ -91,7 +90,7 @@ import Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_barotropic_mode @test all(Array(interior(U)) .≈ Array(interior(exact_U))) set_v_check(x, y, z) = sin(x) * z * cos(y) - set_V_check(x, y) = -sin(x) * Lz^2 / 2.0 * cos(y) + set_V_check(x, y, z) = -sin(x) * Lz^2 / 2.0 * cos(y) set!(v, set_v_check) exact_V = similar(V) set!(exact_V, set_V_check) @@ -116,7 +115,6 @@ import Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_barotropic_mode auxiliary = sefs.auxiliary U, V, η̅, U̅, V̅ = state.U, state.V, state.η̅, state.U̅, state.V̅ Gᵁ, Gⱽ = auxiliary.Gᵁ, auxiliary.Gⱽ - Hᶠᶜ, Hᶜᶠ = sefs.auxiliary.Hᶠᶜ, sefs.auxiliary.Hᶜᶠ u = Field{Face,Center,Center}(grid) v = Field{Center,Face,Center}(grid) @@ -124,22 +122,19 @@ import Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_barotropic_mode v_corrected = similar(v) set_u(x, y, z) = z + Lz / 2 + sin(x) - set_U̅(x, y) = cos(x) * Lz + set_U̅(x, y, z) = cos(x) * Lz set_u_corrected(x, y, z) = z + Lz / 2 + cos(x) set!(u, set_u) set!(U̅, set_U̅) set!(u_corrected, set_u_corrected) set_v(x, y, z) = (z + Lz / 2) * sin(y) + sin(x) - set_V̅(x, y) = (cos(x) + x) * Lz + set_V̅(x, y, z) = (cos(x) + x) * Lz set_v_corrected(x, y, z) = (z + Lz / 2) * sin(y) + cos(x) + x set!(v, set_v) set!(V̅, set_V̅) set!(v_corrected, set_v_corrected) - sefs.auxiliary.Hᶠᶜ .= Lz - sefs.auxiliary.Hᶜᶠ .= Lz - Δz = zeros(Nz) Δz .= grid.Δzᵃᵃᶜ From ff66175554ed89c560caeb0864ff50d3922463d3 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 9 Jan 2024 21:14:47 -0500 Subject: [PATCH 138/567] try it --- test/test_split_explicit_free_surface_solver.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_split_explicit_free_surface_solver.jl b/test/test_split_explicit_free_surface_solver.jl index 4302a182c7..756e03068c 100644 --- a/test/test_split_explicit_free_surface_solver.jl +++ b/test/test_split_explicit_free_surface_solver.jl @@ -182,7 +182,7 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, calc Gᵁ .= gu_c Gⱽ .= gv_c - settings = SplitExplicitSettings(substeps = Nt + 1, averaging_kernel = constant_averaging_kernel) + settings = SplitExplicitSettings(eltype(grid); substeps = Nt + 1, averaging_kernel = constant_averaging_kernel) sefs = sefs(settings) Nsubsteps = calculate_substeps(settings.substepping, 1) From 469224baa667d49731f02c006ea46af122e57914 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 9 Jan 2024 21:16:00 -0500 Subject: [PATCH 139/567] correct for last time --- test/test_split_explicit_free_surface_solver.jl | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/test_split_explicit_free_surface_solver.jl b/test/test_split_explicit_free_surface_solver.jl index 756e03068c..de7b80f251 100644 --- a/test/test_split_explicit_free_surface_solver.jl +++ b/test/test_split_explicit_free_surface_solver.jl @@ -185,10 +185,8 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, calc settings = SplitExplicitSettings(eltype(grid); substeps = Nt + 1, averaging_kernel = constant_averaging_kernel) sefs = sefs(settings) - Nsubsteps = calculate_substeps(settings.substepping, 1) - fractional_Δt, weights = calculate_adaptive_settings(settings.substepping, Nsubsteps) # barotropic time step in fraction of baroclinic step and averaging weights - - iterate_split_explicit!(sefs, grid, Δτ, weights, Val(Nsubsteps)) + iterate_split_explicit!(sefs, grid, Δτ, weights, Val(Nt)) + iterate_split_explicit!(sefs, grid, Δτ_end, weights, Val(1)) η_mean_after = mean(Array(interior(η))) From d09e5fe30b2aa6d30dd4400f184ed1a57f87e25a Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 9 Jan 2024 22:15:58 -0500 Subject: [PATCH 140/567] try again --- test/test_split_explicit_free_surface_solver.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/test_split_explicit_free_surface_solver.jl b/test/test_split_explicit_free_surface_solver.jl index de7b80f251..640e571624 100644 --- a/test/test_split_explicit_free_surface_solver.jl +++ b/test/test_split_explicit_free_surface_solver.jl @@ -184,6 +184,9 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, calc settings = SplitExplicitSettings(eltype(grid); substeps = Nt + 1, averaging_kernel = constant_averaging_kernel) sefs = sefs(settings) + + Nsubsteps = calculate_substeps(settings.substepping, 1) + fractional_Δt, weights = calculate_adaptive_settings(settings.substepping, Nsubsteps) # barotropic time step in fraction of baroclinic step and averaging weights iterate_split_explicit!(sefs, grid, Δτ, weights, Val(Nt)) iterate_split_explicit!(sefs, grid, Δτ_end, weights, Val(1)) From 73f8b09c746cae37c5dc034c11c85376af534365 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 9 Jan 2024 23:11:33 -0500 Subject: [PATCH 141/567] fixed --- .../split_explicit_free_surface.jl | 4 --- ...test_split_explicit_free_surface_solver.jl | 34 +++++++++---------- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index 3d88d92c18..90c7f1907f 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -88,10 +88,6 @@ end function SplitExplicitFreeSurface(grid; gravitational_acceleration = g_Earth, settings = SplitExplicitSettings(eltype(grid); gravitational_acceleration, substeps = 200)) - if eltype(settings) != eltype(grid) - @warn "Using $(eltype(settings)) settings for the SplitExplicitFreeSurface on a $(eltype(grid)) grid" - end - Nz = size(grid, 3) η = ZFaceField(grid, indices = (:, :, Nz+1)) gravitational_acceleration = convert(eltype(grid), gravitational_acceleration) diff --git a/test/test_split_explicit_free_surface_solver.jl b/test/test_split_explicit_free_surface_solver.jl index 640e571624..d2ca748c66 100644 --- a/test/test_split_explicit_free_surface_solver.jl +++ b/test/test_split_explicit_free_surface_solver.jl @@ -12,7 +12,7 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, calc for arch in archs topology = (Periodic, Periodic, Bounded) - Nx, Ny, Nz = 128, 1, 1 + Nx, Ny, Nz = 128, 64, 1 Lx = Ly = 2π Lz = 1 / Oceananigans.BuoyancyModels.g_Earth @@ -184,11 +184,11 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, calc settings = SplitExplicitSettings(eltype(grid); substeps = Nt + 1, averaging_kernel = constant_averaging_kernel) sefs = sefs(settings) - - Nsubsteps = calculate_substeps(settings.substepping, 1) - fractional_Δt, weights = calculate_adaptive_settings(settings.substepping, Nsubsteps) # barotropic time step in fraction of baroclinic step and averaging weights - iterate_split_explicit!(sefs, grid, Δτ, weights, Val(Nt)) + weights = settings.substepping.averaging_weights + for i in 1:Nt + iterate_split_explicit!(sefs, grid, Δτ, weights, Val(1)) + end iterate_split_explicit!(sefs, grid, Δτ_end, weights, Val(1)) η_mean_after = mean(Array(interior(η))) @@ -196,30 +196,30 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, calc tolerance = 10eps(FT) @test abs(η_mean_after - η_mean_before) < tolerance - η_computed = Array(η.data.parent)[2:Nx+1, 2:Ny+1] - U_computed = Array(U.data.parent)[2:Nx+1, 2:Ny+1] - V_computed = Array(V.data.parent)[2:Nx+1, 2:Ny+1] + η_computed = Array(deepcopy(interior(η, :, 1, 1))) + U_computed = Array(deepcopy(interior(U, :, 1, 1))) + V_computed = Array(deepcopy(interior(V, :, 1, 1))) - η̅_computed = Array(η̅.data.parent)[2:Nx+1, 2:Ny+1] - U̅_computed = Array(U̅.data.parent)[2:Nx+1, 2:Ny+1] - V̅_computed = Array(V̅.data.parent)[2:Nx+1, 2:Ny+1] + η̅_computed = Array(deepcopy(interior(η̅, :, 1, 1))) + U̅_computed = Array(deepcopy(interior(U̅, :, 1, 1))) + V̅_computed = Array(deepcopy(interior(V̅, :, 1, 1))) set!(η, η₀) # ∂ₜₜ(η) = Δη - η_exact = cos(ω * T) * (Array(η.data.parent)[2:Nx+1, 2:Ny+1] .- 1) .+ 1 + η_exact = cos(ω * T) * (Array(interior(η, :, 1, 1)) .- 1) .+ 1 U₀(x, y, z) = kx * cos(kx * x) * sin(ky * y) # ∂ₜU = - ∂x(η), since we know η set!(U, U₀) - U_exact = -(sin(ω * T) * 1 / ω) .* Array(U.data.parent)[2:Nx+1, 2:Ny+1] .+ gu_c * T + U_exact = -(sin(ω * T) * 1 / ω) .* Array(interior(U, :, 1, 1)) .+ gu_c * T V₀(x, y, z) = ky * sin(kx * x) * cos(ky * y) # ∂ₜV = - ∂y(η), since we know η set!(V, V₀) - V_exact = -(sin(ω * T) * 1 / ω) .* Array(V.data.parent)[2:Nx+1, 2:Ny+1] .+ gv_c * T + V_exact = -(sin(ω * T) * 1 / ω) .* Array(interior(V, :, 1, 1)) .+ gv_c * T - η̅_exact = (sin(ω * T) / ω - sin(ω * 0) / ω) / T * (Array(η.data.parent)[2:Nx+1, 2:Ny+1] .- 1) .+ 1 - U̅_exact = (cos(ω * T) * 1 / ω^2 - cos(ω * 0) * 1 / ω^2) / T * Array(U.data.parent)[2:Nx+1, 2:Ny+1] .+ gu_c * T / 2 - V̅_exact = (cos(ω * T) * 1 / ω^2 - cos(ω * 0) * 1 / ω^2) / T * Array(V.data.parent)[2:Nx+1, 2:Ny+1] .+ gv_c * T / 2 + η̅_exact = (sin(ω * T) / ω - sin(ω * 0) / ω) / T * (Array(interior(η, :, 1, 1)) .- 1) .+ 1 + U̅_exact = (cos(ω * T) * 1 / ω^2 - cos(ω * 0) * 1 / ω^2) / T * Array(interior(U, :, 1, 1)) .+ gu_c * T / 2 + V̅_exact = (cos(ω * T) * 1 / ω^2 - cos(ω * 0) * 1 / ω^2) / T * Array(interior(V, :, 1, 1)) .+ gv_c * T / 2 tolerance = 1e-2 From 69b9b98353dd3a34457bb311ae1fdeaeb8d7dc50 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Wed, 10 Jan 2024 00:11:59 -0500 Subject: [PATCH 142/567] tests fixxed --- test/test_split_explicit_free_surface_solver.jl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/test_split_explicit_free_surface_solver.jl b/test/test_split_explicit_free_surface_solver.jl index d2ca748c66..f30bd5769b 100644 --- a/test/test_split_explicit_free_surface_solver.jl +++ b/test/test_split_explicit_free_surface_solver.jl @@ -77,8 +77,11 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, calc Nsubsteps = calculate_substeps(settings.substepping, 1) fractional_Δt, weights = calculate_adaptive_settings(settings.substepping, Nsubsteps) # barotropic time step in fraction of baroclinic step and averaging weights - - iterate_split_explicit!(sefs, grid, Δτ, weights, Val(Nsubsteps)) + + for i in 1:Nt + iterate_split_explicit!(sefs, grid, Δτ, weights, Val(1)) + end + iterate_split_explicit!(sefs, grid, Δτ_end, weights, Val(1)) U_computed = Array(parent(U))[2:Nx+1, 2:Ny+1] η_computed = Array(parent(η))[2:Nx+1, 2:Ny+1] From b42b115f603df2a1faf6c718b63f9d72ca4a0e86 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Wed, 10 Jan 2024 10:22:26 -0500 Subject: [PATCH 143/567] finally tests fixed --- test/test_split_explicit_free_surface_solver.jl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/test_split_explicit_free_surface_solver.jl b/test/test_split_explicit_free_surface_solver.jl index f30bd5769b..a0d0aef3e6 100644 --- a/test/test_split_explicit_free_surface_solver.jl +++ b/test/test_split_explicit_free_surface_solver.jl @@ -55,11 +55,12 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, calc η = sefs.η T = 2π - Δτ = 2π / maximum([Nx, Ny]) * 5e-1 # the last factor is essentially the order of accuracy + Δτ = 2π / maximum([Nx, Ny]) * 5e-2 # the last factor is essentially the order of accuracy Nt = floor(Int, T / Δτ) Δτ_end = T - Nt * Δτ settings = SplitExplicitSettings(eltype(grid); substeps = Nt, averaging_kernel = constant_averaging_kernel) + sefs = sefs(settings) # set!(η, f(x,y)) η₀(x, y, z) = sin(x) @@ -75,20 +76,19 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, calc Gᵁ .= 0 Gⱽ .= 0 - Nsubsteps = calculate_substeps(settings.substepping, 1) - fractional_Δt, weights = calculate_adaptive_settings(settings.substepping, Nsubsteps) # barotropic time step in fraction of baroclinic step and averaging weights + weights = settings.substepping.averaging_weights for i in 1:Nt iterate_split_explicit!(sefs, grid, Δτ, weights, Val(1)) end iterate_split_explicit!(sefs, grid, Δτ_end, weights, Val(1)) - U_computed = Array(parent(U))[2:Nx+1, 2:Ny+1] - η_computed = Array(parent(η))[2:Nx+1, 2:Ny+1] + U_computed = Array(deepcopy(interior(U))) + η_computed = Array(deepcopy(interior(η))) set!(η, η₀) set!(U, U₀) - U_exact = Array(parent(U))[2:Nx+1, 2:Ny+1] - η_exact = Array(parent(η))[2:Nx+1, 2:Ny+1] + U_exact = Array(deepcopy(interior(U))) + η_exact = Array(deepcopy(interior(η))) @test maximum(abs.(U_computed - U_exact)) < 1e-3 @show maximum(abs.(η_computed)) From dcffb79f7859a7b087b6caa2e877aadf262a263a Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Wed, 10 Jan 2024 11:33:48 -0500 Subject: [PATCH 144/567] back to previous dt --- test/test_split_explicit_free_surface_solver.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_split_explicit_free_surface_solver.jl b/test/test_split_explicit_free_surface_solver.jl index a0d0aef3e6..ffb5b7c34b 100644 --- a/test/test_split_explicit_free_surface_solver.jl +++ b/test/test_split_explicit_free_surface_solver.jl @@ -105,7 +105,7 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, calc g = sefs.gravitational_acceleration η = sefs.η - Δτ = 2π / maximum([Nx, Ny]) * 5e-1 # the last factor is essentially the order of accuracy + Δτ = 2π / maximum([Nx, Ny]) * 5e-2 # the last factor is essentially the order of accuracy # set!(η, f(x, y)) η_avg = 1 @@ -127,7 +127,7 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, calc Nsubsteps = calculate_substeps(settings.substepping, 1) fractional_Δt, weights = calculate_adaptive_settings(settings.substepping, Nsubsteps) # barotropic time step in fraction of baroclinic step and averaging weights - + iterate_split_explicit!(sefs, grid, Δτ, weights, Val(Nsubsteps)) U_computed = Array(U.data.parent)[2:Nx+1, 2:Ny+1] From 9801ec04a6f0f41c0f91c1f059d923f03f6e9c70 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Fri, 12 Jan 2024 08:49:28 -0500 Subject: [PATCH 145/567] bugfix --- src/ImmersedBoundaries/active_cells_map.jl | 2 +- src/Utils/kernel_launching.jl | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/ImmersedBoundaries/active_cells_map.jl b/src/ImmersedBoundaries/active_cells_map.jl index 546902792d..5b255122a8 100644 --- a/src/ImmersedBoundaries/active_cells_map.jl +++ b/src/ImmersedBoundaries/active_cells_map.jl @@ -181,7 +181,7 @@ function active_cells_interior_map(ibg::ImmersedBoundaryGrid{<:Any, <:Any, <:Any ox = Rx == 1 || Tx == RightConnected ? 0 : Hx oy = Ry == 1 || Ty == RightConnected ? 0 : Hy - interior = active_interior_indices(ibg; parameters = KernelParameters((nx, ny, Nz), (ox, oy, 0))) + interior = active_interior_indices(ibg; parameters = KernelParameters((nx, ny, Nz), (ox, oy, 0))) return (; interior, west, east, south, north) end diff --git a/src/Utils/kernel_launching.jl b/src/Utils/kernel_launching.jl index f43be9a3d2..b0c6bf796c 100644 --- a/src/Utils/kernel_launching.jl +++ b/src/Utils/kernel_launching.jl @@ -128,7 +128,8 @@ function launch!(arch, grid, workspec, kernel!, kernel_args...; only_active_cells, kwargs...) - loop!(kernel_args...) + + !isnothing(loop!) && loop!(kernel_args...) return nothing end @@ -145,6 +146,7 @@ function configured_kernel(arch, grid, workspec, kernel!; reduced_dimensions, location) + offset = offsets(workspec) if !isnothing(only_active_cells) From c64f404210bf48f40dff19cb023d6a4af795475c Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 15 Jan 2024 10:28:04 -0500 Subject: [PATCH 146/567] tests fixed? --- test/test_split_explicit_free_surface_solver.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/test_split_explicit_free_surface_solver.jl b/test/test_split_explicit_free_surface_solver.jl index ffb5b7c34b..640d0596c6 100644 --- a/test/test_split_explicit_free_surface_solver.jl +++ b/test/test_split_explicit_free_surface_solver.jl @@ -128,7 +128,9 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, calc Nsubsteps = calculate_substeps(settings.substepping, 1) fractional_Δt, weights = calculate_adaptive_settings(settings.substepping, Nsubsteps) # barotropic time step in fraction of baroclinic step and averaging weights - iterate_split_explicit!(sefs, grid, Δτ, weights, Val(Nsubsteps)) + for step in 1:Nsubsteps + iterate_split_explicit!(sefs, grid, Δτ, weights, Val(1)) + end U_computed = Array(U.data.parent)[2:Nx+1, 2:Ny+1] V_computed = Array(V.data.parent)[2:Nx+1, 2:Ny+1] From 359a0832e8f59340b16289a73ed923b461bb15e5 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 15 Jan 2024 14:52:37 -0500 Subject: [PATCH 147/567] ale --- ...test_split_explicit_free_surface_solver.jl | 41 ++++++++++--------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/test/test_split_explicit_free_surface_solver.jl b/test/test_split_explicit_free_surface_solver.jl index 640d0596c6..b4e7a53b50 100644 --- a/test/test_split_explicit_free_surface_solver.jl +++ b/test/test_split_explicit_free_surface_solver.jl @@ -59,7 +59,7 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, calc Nt = floor(Int, T / Δτ) Δτ_end = T - Nt * Δτ - settings = SplitExplicitSettings(eltype(grid); substeps = Nt, averaging_kernel = constant_averaging_kernel) + settings = SplitExplicitSettings(; substeps = Nt, averaging_kernel = constant_averaging_kernel) sefs = sefs(settings) # set!(η, f(x,y)) @@ -96,6 +96,11 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, calc @test maximum(abs.(η_computed - η_exact)) < max(100eps(FT), 1e-6) end + settings = SplitExplicitSettings(eltype(grid); substeps = 200, averaging_kernel = constant_averaging_kernel) + sefs = SplitExplicitFreeSurface(grid; settings) + + sefs.η .= 0 + @testset "Averaging / Do Nothing test " begin state = sefs.state auxiliary = sefs.auxiliary @@ -105,24 +110,22 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, calc g = sefs.gravitational_acceleration η = sefs.η - Δτ = 2π / maximum([Nx, Ny]) * 5e-2 # the last factor is essentially the order of accuracy + Δτ = 2π / maximum([Nx, Ny]) * 1e-2 # the last factor is essentially the order of accuracy # set!(η, f(x, y)) η_avg = 1 U_avg = 2 V_avg = 3 - η₀(x, y, z) = η_avg - set!(η, η₀) - U₀(x, y, z) = U_avg - set!(U, U₀) - V₀(x, y, z) = V_avg - set!(V, V₀) + fill!(η, η_avg) + fill!(U, U_avg) + fill!(V, V_avg) + + fill!(η̅ , 0) + fill!(U̅ , 0) + fill!(V̅ , 0) + fill!(Gᵁ, 0) + fill!(Gⱽ, 0) - η̅ .= 0 - U̅ .= 0 - V̅ .= 0 - Gᵁ .= 0 - Gⱽ .= 0 settings = sefs.settings Nsubsteps = calculate_substeps(settings.substepping, 1) @@ -132,13 +135,13 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: calculate_substeps, calc iterate_split_explicit!(sefs, grid, Δτ, weights, Val(1)) end - U_computed = Array(U.data.parent)[2:Nx+1, 2:Ny+1] - V_computed = Array(V.data.parent)[2:Nx+1, 2:Ny+1] - η_computed = Array(η.data.parent)[2:Nx+1, 2:Ny+1] + U_computed = Array(deepcopy(interior(U))) + V_computed = Array(deepcopy(interior(V))) + η_computed = Array(deepcopy(interior(η))) - U̅_computed = Array(U̅.data.parent)[2:Nx+1, 2:Ny+1] - V̅_computed = Array(V̅.data.parent)[2:Nx+1, 2:Ny+1] - η̅_computed = Array(η̅.data.parent)[2:Nx+1, 2:Ny+1] + U̅_computed = Array(deepcopy(interior(U̅))) + V̅_computed = Array(deepcopy(interior(V̅))) + η̅_computed = Array(deepcopy(interior(η̅))) tolerance = 100eps(FT) From 72f286ef277b891436dfd230fc8b798d24a4816f Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 16 Jan 2024 09:21:12 -0500 Subject: [PATCH 148/567] Update src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl Co-authored-by: Navid C. Constantinou --- .../update_hydrostatic_free_surface_model_state.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl index 43f53a766f..4c082c31b5 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl @@ -37,7 +37,6 @@ function update_state!(model::HydrostaticFreeSurfaceModel, grid, callbacks; comp @apply_regionally update_model_field_time_series!(model, model.clock) fill_halo_regions!(prognostic_fields(model), model.clock, fields(model); async = true) - @apply_regionally replace_horizontal_vector_halos!(model.velocities, model.grid) @apply_regionally compute_auxiliaries!(model) From 46ef24cff94d244137c861c4414615e0ef6a9d80 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 16 Jan 2024 09:21:24 -0500 Subject: [PATCH 149/567] Update src/TimeSteppers/quasi_adams_bashforth_2.jl Co-authored-by: Navid C. Constantinou --- src/TimeSteppers/quasi_adams_bashforth_2.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TimeSteppers/quasi_adams_bashforth_2.jl b/src/TimeSteppers/quasi_adams_bashforth_2.jl index f63dcf817f..91a01ef7a3 100644 --- a/src/TimeSteppers/quasi_adams_bashforth_2.jl +++ b/src/TimeSteppers/quasi_adams_bashforth_2.jl @@ -159,4 +159,4 @@ Time step velocity fields via the 2nd-order quasi Adams-Bashforth method @inbounds u[i, j, k] += convert(FT, Δt) * ((one_point_five + χ) * Gⁿ[i, j, k] - (oh_point_five + χ) * G⁻[i, j, k]) end -@kernel ab2_step_field!(::FunctionField, Δt, χ, Gⁿ, G⁻, grid) = nothing +@kernel ab2_step_field!(::FunctionField, Δt, χ, Gⁿ, G⁻, grid) = nothing From 6b74e7ae13c2478bfdcb4743b08aeec22d684efb Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 16 Jan 2024 09:23:10 -0500 Subject: [PATCH 150/567] removed NVTX --- Project.toml | 1 - src/Utils/kernel_launching.jl | 1 - 2 files changed, 2 deletions(-) diff --git a/Project.toml b/Project.toml index 8f6c4a2351..2d0350b945 100644 --- a/Project.toml +++ b/Project.toml @@ -22,7 +22,6 @@ LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195" NCDatasets = "85f8d34a-cbdd-5861-8df4-14fed0d494ab" -NVTX = "5da4648a-3479-48b8-97b9-01cb529c0a1f" OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" PencilArrays = "0e08944d-e94e-41b1-9406-dcf66b6a9d2e" diff --git a/src/Utils/kernel_launching.jl b/src/Utils/kernel_launching.jl index b0c6bf796c..2fbf293971 100644 --- a/src/Utils/kernel_launching.jl +++ b/src/Utils/kernel_launching.jl @@ -7,7 +7,6 @@ using Oceananigans.Utils using Oceananigans.Grids using Oceananigans.Grids: AbstractGrid -using NVTX import Base struct KernelParameters{S, O} end From 2061300159815688e633ca87c1388501e90da431 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 16 Jan 2024 09:23:54 -0500 Subject: [PATCH 151/567] remove one line --- src/Utils/kernel_launching.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Utils/kernel_launching.jl b/src/Utils/kernel_launching.jl index 2fbf293971..d6da72b92f 100644 --- a/src/Utils/kernel_launching.jl +++ b/src/Utils/kernel_launching.jl @@ -145,7 +145,6 @@ function configured_kernel(arch, grid, workspec, kernel!; reduced_dimensions, location) - offset = offsets(workspec) if !isnothing(only_active_cells) From f46f7a9b85eb719ef384348a706eadfb85ba68e1 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 16 Jan 2024 09:24:48 -0500 Subject: [PATCH 152/567] if inside --- src/Utils/kernel_launching.jl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Utils/kernel_launching.jl b/src/Utils/kernel_launching.jl index d6da72b92f..caf1e2c251 100644 --- a/src/Utils/kernel_launching.jl +++ b/src/Utils/kernel_launching.jl @@ -150,12 +150,13 @@ function configured_kernel(arch, grid, workspec, kernel!; if !isnothing(only_active_cells) workgroup, worksize = active_cells_work_layout(workgroup, worksize, only_active_cells, grid) offset = nothing - end - if worksize == 0 - return nothing + # A fully immersed domain! + if worksize == 0 + return nothing + end end - + # We can only launch offset kernels with Static sizes!!!! loop! = isnothing(offset) ? kernel!(Architectures.device(arch), workgroup, worksize) : kernel!(Architectures.device(arch), StaticSize(workgroup), OffsetStaticSize(contiguousrange(worksize, offset))) From a276111dab5ce8d8807e07c5b52b341c6bcce8d5 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 16 Jan 2024 09:25:05 -0500 Subject: [PATCH 153/567] better comment --- src/Utils/kernel_launching.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Utils/kernel_launching.jl b/src/Utils/kernel_launching.jl index caf1e2c251..a6624e191a 100644 --- a/src/Utils/kernel_launching.jl +++ b/src/Utils/kernel_launching.jl @@ -151,7 +151,7 @@ function configured_kernel(arch, grid, workspec, kernel!; workgroup, worksize = active_cells_work_layout(workgroup, worksize, only_active_cells, grid) offset = nothing - # A fully immersed domain! + # A non active domain! if worksize == 0 return nothing end From 4fedc37d30b072d9b4c602027e0f4964816fb8a4 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 16 Jan 2024 10:12:53 -0500 Subject: [PATCH 154/567] some docstrings --- src/ImmersedBoundaries/active_cells_map.jl | 67 +++++++++++++++++----- 1 file changed, 52 insertions(+), 15 deletions(-) diff --git a/src/ImmersedBoundaries/active_cells_map.jl b/src/ImmersedBoundaries/active_cells_map.jl index 5b255122a8..48b157a436 100644 --- a/src/ImmersedBoundaries/active_cells_map.jl +++ b/src/ImmersedBoundaries/active_cells_map.jl @@ -37,23 +37,48 @@ active_map(::Val{:north}) = NorthMap() @inline use_only_active_interior_cells(::ActiveCellsIBG) = InteriorMap() @inline use_only_active_interior_cells(::DistributedActiveCellsIBG) = InteriorMap() -@inline active_cells_work_layout(group, size, ::InteriorMap, grid::ArrayActiveCellsIBG) = min(length(grid.interior_active_cells), 256), length(grid.interior_active_cells) +""" + active_cells_work_layout(group, size, map_type, grid) + +Compute the work layout for active cells based on the given map type and grid. + +# Arguments +- `group`: The previous workgroup. +- `size`: The previous worksize. +- `map_type`: The type of map (e.g., `InteriorMap`, `WestMap`, `EastMap`, `SouthMap`, `NorthMap`). +- `grid`: The grid containing the active cells. + +# Returns +- A tuple `(workgroup, worksize)` representing the work layout for active cells. +""" +@inline active_cells_work_layout(group, size, ::InteriorMap, grid::ArrayActiveCellsIBG) = min(length(grid.interior_active_cells), 256), length(grid.interior_active_cells) @inline active_cells_work_layout(group, size, ::InteriorMap, grid::NamedTupleActiveCellsIBG) = min(length(grid.interior_active_cells.interior), 256), length(grid.interior_active_cells.interior) @inline active_cells_work_layout(group, size, ::WestMap, grid::NamedTupleActiveCellsIBG) = min(length(grid.interior_active_cells.west), 256), length(grid.interior_active_cells.west) @inline active_cells_work_layout(group, size, ::EastMap, grid::NamedTupleActiveCellsIBG) = min(length(grid.interior_active_cells.east), 256), length(grid.interior_active_cells.east) @inline active_cells_work_layout(group, size, ::SouthMap, grid::NamedTupleActiveCellsIBG) = min(length(grid.interior_active_cells.south), 256), length(grid.interior_active_cells.south) @inline active_cells_work_layout(group, size, ::NorthMap, grid::NamedTupleActiveCellsIBG) = min(length(grid.interior_active_cells.north), 256), length(grid.interior_active_cells.north) +@inline active_cells_work_layout(group, size, ::SurfaceMap, grid::ActiveSurfaceIBG) = min(length(grid.surface_active_cells), 256), length(grid.surface_active_cells) + +""" + active_linear_index_to_tuple(idx, map, grid) +Converts a linear index to a tuple of indices based on the given map and grid. + +# Arguments +- `idx`: The linear index to convert. +- `map`: The map indicating the type of index conversion to perform. +- `grid`: The grid containing the active cells. + +# Returns +A tuple of indices corresponding to the linear index. +""" @inline active_linear_index_to_tuple(idx, ::InteriorMap, grid::ArrayActiveCellsIBG) = Base.map(Int, grid.interior_active_cells[idx]) @inline active_linear_index_to_tuple(idx, ::InteriorMap, grid::NamedTupleActiveCellsIBG) = Base.map(Int, grid.interior_active_cells.interior[idx]) @inline active_linear_index_to_tuple(idx, ::WestMap, grid::NamedTupleActiveCellsIBG) = Base.map(Int, grid.interior_active_cells.west[idx]) @inline active_linear_index_to_tuple(idx, ::EastMap, grid::NamedTupleActiveCellsIBG) = Base.map(Int, grid.interior_active_cells.east[idx]) @inline active_linear_index_to_tuple(idx, ::SouthMap, grid::NamedTupleActiveCellsIBG) = Base.map(Int, grid.interior_active_cells.south[idx]) @inline active_linear_index_to_tuple(idx, ::NorthMap, grid::NamedTupleActiveCellsIBG) = Base.map(Int, grid.interior_active_cells.north[idx]) - -@inline active_cells_work_layout(group, size, ::SurfaceMap, grid::ActiveSurfaceIBG) = min(length(grid.surface_active_cells), 256), length(grid.surface_active_cells) - -@inline active_linear_index_to_tuple(idx, ::SurfaceMap, grid::ActiveSurfaceIBG) = Base.map(Int, grid.surface_active_cells[idx]) +@inline active_linear_index_to_tuple(idx, ::SurfaceMap, grid::ActiveSurfaceIBG) = Base.map(Int, grid.surface_active_cells[idx]) function ImmersedBoundaryGrid(grid, ib; active_cells_map::Bool = true) @@ -62,9 +87,8 @@ function ImmersedBoundaryGrid(grid, ib; active_cells_map::Bool = true) # Create the cells map on the CPU, then switch it to the GPU if active_cells_map - interior_map = active_cells_interior_map(ibg) - surface_map = active_cells_surface_map(ibg) - surface_map = arch_array(architecture(ibg), surface_map) + interior_map = map_interior_active_cells(ibg) + surface_map = map_surface_active_cells(ibg) else interior_map = nothing surface_map = nothing @@ -107,6 +131,18 @@ const MAXUInt8 = 2^8 - 1 const MAXUInt16 = 2^16 - 1 const MAXUInt32 = 2^32 - 1 +""" + active_interior_indices(ibg; parameters = :xyz) + +Compute the indices of the active interior cells in the given immersed boundary grid. + +# Arguments +- `ibg`: The immersed boundary grid. +- `parameters`: (optional) The parameters to be used for computing the active cells. Default is `:xyz`. + +# Returns +An array of tuples representing the indices of the active interior cells. +""" function active_interior_indices(ibg; parameters = :xyz) active_cells_field = compute_interior_active_cells(ibg; parameters) @@ -146,11 +182,11 @@ end @inline add_3rd_index(t::Tuple, k) = (t[1], t[2], k) -active_cells_interior_map(ibg) = active_interior_indices(ibg; parameters = :xyz) +map_interior_active_cells(ibg) = active_interior_indices(ibg; parameters = :xyz) # In case of a `DistributedGrid` we want to have different maps depending on the # partitioning of the domain -function active_cells_interior_map(ibg::ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:DistributedGrid}) +function map_interior_active_cells(ibg::ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:DistributedGrid}) arch = architecture(ibg) Rx, Ry, _ = arch.ranks @@ -188,17 +224,18 @@ end # If we eventually want to perform also barotropic step, `w` computation and `p` # computation only on active `columns` -function active_cells_surface_map(ibg) +function map_surface_active_cells(ibg) active_cells_field = compute_surface_active_cells(ibg) interior_cells = arch_array(CPU(), interior(active_cells_field, :, :, 1)) full_indices = findall(interior_cells) - Nx, Ny, Nz = size(ibg) + Nx, Ny, _ = size(ibg) # Reduce the size of the active_cells_map (originally a tuple of Int64) N = max(Nx, Ny) IntType = N > MAXUInt8 ? (N > MAXUInt16 ? (N > MAXUInt32 ? UInt64 : UInt32) : UInt16) : UInt8 - smaller_indices = getproperty.(full_indices, Ref(:I)) .|> Tuple{IntType, IntType} - - return smaller_indices + surface_map = getproperty.(full_indices, Ref(:I)) .|> Tuple{IntType, IntType} + surface_map = arch_array(architecture(ibg), surface_map) + + return surface_map end From 8f342fa5a773aa1e64c79d5e5a3a0bc3af00c813 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 16 Jan 2024 11:35:31 -0500 Subject: [PATCH 155/567] remove NVTX --- src/Architectures.jl | 1 - src/DistributedComputations/halo_communication.jl | 2 -- .../update_hydrostatic_free_surface_model_state.jl | 1 - 3 files changed, 4 deletions(-) diff --git a/src/Architectures.jl b/src/Architectures.jl index be016b3f82..099769a04b 100644 --- a/src/Architectures.jl +++ b/src/Architectures.jl @@ -8,7 +8,6 @@ using CUDA using KernelAbstractions using Adapt using OffsetArrays -using NVTX """ AbstractArchitecture diff --git a/src/DistributedComputations/halo_communication.jl b/src/DistributedComputations/halo_communication.jl index 5d011acafe..45a520b5ee 100644 --- a/src/DistributedComputations/halo_communication.jl +++ b/src/DistributedComputations/halo_communication.jl @@ -22,8 +22,6 @@ import Oceananigans.BoundaryConditions: fill_south_and_north_halo!, fill_bottom_and_top_halo! -using NVTX - ##### ##### MPI tags for halo communication BCs ##### diff --git a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl index 4c082c31b5..4cf7c19521 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl @@ -12,7 +12,6 @@ import Oceananigans.TimeSteppers: update_state! import Oceananigans.Models.NonhydrostaticModels: compute_auxiliaries! using Oceananigans.Models: update_model_field_time_series! -using NVTX compute_auxiliary_fields!(auxiliary_fields) = Tuple(compute!(a) for a in auxiliary_fields) From b7c871a8d15435dcbf0b363cb3818992cf088b60 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 23 Jan 2024 18:32:16 -0500 Subject: [PATCH 156/567] test an hypothesis --- ...distributed_split_explicit_free_surface.jl | 16 +++++++- .../split_explicit_free_surface.jl | 17 +++++++- .../split_explicit_free_surface_kernels.jl | 39 +++++++------------ ...ulti_region_split_explicit_free_surface.jl | 10 ++++- 4 files changed, 54 insertions(+), 28 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl index d7f0fe42c5..66d30f9422 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl @@ -12,13 +12,27 @@ function SplitExplicitAuxiliaryFields(grid::DistributedGrid) Gᵁ = XFaceField(grid, indices = (:, :, Nz)) Gⱽ = YFaceField(grid, indices = (:, :, Nz)) + Hᶠᶜ = Field((Face, Center, Nothing), grid) + Hᶜᶠ = Field((Center, Face, Nothing), grid) + + calculate_column_height!(Hᶠᶜ, (Face, Center, Center)) + calculate_column_height!(Hᶜᶠ, (Center, Face, Center)) + + fill_halo_regions!((Hᶠᶜ, Hᶜᶠ)) + # In a non-parallel grid we calculate only the interior kernel_size = augmented_kernel_size(grid) kernel_offsets = augmented_kernel_offsets(grid) kernel_parameters = KernelParameters(kernel_size, kernel_offsets) - return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, kernel_parameters) + return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, kernel_parameters) +end + +"""Integrate z at locations `location` and set! `height`` with the result""" +@inline function calculate_column_height!(height, location) + dz = GridMetricOperation(location, Δz, height.grid) + return sum!(height, dz) end @inline function augmented_kernel_size(grid::DistributedGrid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index 90c7f1907f..252b335517 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -179,6 +179,10 @@ Base.@kwdef struct SplitExplicitAuxiliaryFields{𝒞ℱ, ℱ𝒞, 𝒦} Gᵁ :: ℱ𝒞 "Vertically-integrated slow barotropic forcing function for `V` (`ReducedField` over ``z``)" Gⱽ :: 𝒞ℱ + "Depth at `(Face, Center)` (`ReducedField` over ``z``)" + Hᶠᶜ :: ℱ𝒞 + "Depth at `(Center, Face)` (`ReducedField` over ``z``)" + Hᶜᶠ :: 𝒞ℱ "kernel size for barotropic time stepping" kernel_parameters :: 𝒦 end @@ -195,9 +199,20 @@ function SplitExplicitAuxiliaryFields(grid::AbstractGrid) Gᵁ = XFaceField(grid, indices = (:, :, Nz)) Gⱽ = YFaceField(grid, indices = (:, :, Nz)) + Hᶠᶜ = Field((Face, Center, Nothing), grid) + Hᶜᶠ = Field((Center, Face, Nothing), grid) + + dz = GridMetricOperation((Face, Center, Center), Δz, grid) + sum!(Hᶠᶜ, dz) + + dz = GridMetricOperation((Center, Face, Center), Δz, grid) + sum!(Hᶜᶠ, dz) + + fill_halo_regions!((Hᶠᶜ, Hᶜᶠ)) + kernel_parameters = :xy - return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, kernel_parameters) + return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, kernel_parameters) end """ diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 3bd50d9c34..64a0844ba7 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -138,19 +138,6 @@ end return nothing end -# Column height for the split explicit solver -@inline column_heightᶜᶜ(i, j, k, grid) = grid.Lz - -# Column height for an GridFitted bottom immersed boundary -@inline function column_heightᶜᶜ(i, j, k, grid::GFBIBG) - bottom = grid.immersed_boundary.bottom_height[i, j, 1] - return ifelse(bottom < 0, - bottom, grid.Lz - bottom) -end - -@inline column_heightᶠᶜ(i, j, grid) = ℑxᶠᵃᵃ(i, j, 1, grid, column_heightᶜᶜ) -@inline column_heightᶜᶠ(i, j, grid) = ℑyᵃᶠᵃ(i, j, 1, grid, column_heightᶜᶜ) - - @kernel function _split_explicit_free_surface!(grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) i, j = @index(Global, NTuple) free_surface_evolution!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) @@ -172,20 +159,20 @@ end @kernel function _split_explicit_barotropic_velocity!(averaging_weight, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, Gᵁ, Gⱽ, g, + η̅, U̅, V̅, Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, g, timestepper) i, j = @index(Global, NTuple) velocity_evolution!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅, averaging_weight, - Gᵁ, Gⱽ, g, + Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, g, timestepper) end @inline function velocity_evolution!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅, averaging_weight, - Gᵁ, Gⱽ, g, + Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, g, timestepper) k_top = grid.Nz+1 @@ -196,8 +183,8 @@ end advance_previous_velocity!(i, j, k_top-1, timestepper, V, Vᵐ⁻¹, Vᵐ⁻²) # ∂τ(U) = - ∇η + G - U[i, j, k_top-1] += Δτ * (- g * column_heightᶠᶜ(i, j, grid) * ∂xᶠᶜᶠ_η(i, j, k_top, grid, TX, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gᵁ[i, j, k_top-1]) - V[i, j, k_top-1] += Δτ * (- g * column_heightᶜᶠ(i, j, grid) * ∂yᶜᶠᶠ_η(i, j, k_top, grid, TY, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gⱽ[i, j, k_top-1]) + U[i, j, k_top-1] += Δτ * (- g * Hᶠᶜ[i, j, 1] * ∂xᶠᶜᶠ_η(i, j, k_top, grid, TX, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gᵁ[i, j, k_top-1]) + V[i, j, k_top-1] += Δτ * (- g * Hᶜᶠ[i, j, 1] * ∂yᶜᶠᶠ_η(i, j, k_top, grid, TY, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gⱽ[i, j, k_top-1]) # time-averaging η̅[i, j, k_top] += averaging_weight * η[i, j, k_top] @@ -273,13 +260,13 @@ function initialize_auxiliary_state!(state, η, timestepper) return nothing end -@kernel function _barotropic_split_explicit_corrector!(u, v, U̅, V̅, U, V, grid) +@kernel function _barotropic_split_explicit_corrector!(u, v, U̅, V̅, U, V, Hᶠᶜ, Hᶜᶠ, grid) i, j, k = @index(Global, NTuple) k_top = grid.Nz+1 @inbounds begin - u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / column_heightᶠᶜ(i, j, grid) - v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / column_heightᶜᶠ(i, j, grid) + u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / Hᶠᶜ[i, j, 1] + v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / Hᶜᶠ[i, j, 1] end end @@ -287,14 +274,16 @@ end function barotropic_split_explicit_corrector!(u, v, free_surface, grid) sefs = free_surface.state U, V, U̅, V̅ = sefs.U, sefs.V, sefs.U̅, sefs.V̅ + Hᶠᶜ, Hᶜᶠ = free_surface.auxiliary.Hᶠᶜ, free_surface.auxiliary.Hᶜᶠ arch = architecture(grid) + # take out "bad" barotropic mode, # !!!! reusing U and V for this storage since last timestep doesn't matter compute_barotropic_mode!(U, V, grid, u, v) # add in "good" barotropic mode launch!(arch, grid, :xyz, _barotropic_split_explicit_corrector!, - u, v, U̅, V̅, U, V, grid) + u, v, U̅, V̅, U, V, Hᶠᶜ, Hᶜᶠ, grid) return nothing end @@ -386,8 +375,8 @@ function iterate_split_explicit!(free_surface, grid, Δτᴮ, weights, ::Val{Nsu Vᵐ⁻¹, Vᵐ⁻² = state.Vᵐ⁻¹, state.Vᵐ⁻² ηᵐ, ηᵐ⁻¹, ηᵐ⁻² = state.ηᵐ, state.ηᵐ⁻¹, state.ηᵐ⁻² η̅, U̅, V̅ = state.η̅, state.U̅, state.V̅ - Gᵁ, Gⱽ = auxiliary.Gᵁ, auxiliary.Gⱽ - + Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ = auxiliary.Gᵁ, auxiliary.Gⱽ, auxiliary.Hᶠᶜ, auxiliary.Hᶜᶠ + timestepper = settings.timestepper parameters = auxiliary.kernel_parameters @@ -401,7 +390,7 @@ function iterate_split_explicit!(free_surface, grid, Δτᴮ, weights, ::Val{Nsu U_args = (grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, Gᵁ, Gⱽ, g, + η̅, U̅, V̅, Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, g, timestepper) GC.@preserve η_args U_args begin diff --git a/src/MultiRegion/multi_region_split_explicit_free_surface.jl b/src/MultiRegion/multi_region_split_explicit_free_surface.jl index 3e3c4ec842..493a2a2f9e 100644 --- a/src/MultiRegion/multi_region_split_explicit_free_surface.jl +++ b/src/MultiRegion/multi_region_split_explicit_free_surface.jl @@ -11,13 +11,21 @@ function SplitExplicitAuxiliaryFields(grid::MultiRegionGrids) Gᵁ = XFaceField(grid; indices = (:, :, Nz)) Gⱽ = YFaceField(grid; indices = (:, :, Nz)) + Hᶠᶜ = Field((Face, Center, Nothing), grid) + Hᶜᶠ = Field((Center, Face, Nothing), grid) + + @apply_regionally calculate_column_height!(Hᶠᶜ, (Face, Center, Center)) + @apply_regionally calculate_column_height!(Hᶜᶠ, (Center, Face, Center)) + + fill_halo_regions!((Hᶠᶜ, Hᶜᶠ)) + # In a non-parallel grid we calculate only the interior @apply_regionally kernel_size = augmented_kernel_size(grid, grid.partition) @apply_regionally kernel_offsets = augmented_kernel_offsets(grid, grid.partition) @apply_regionally kernel_parameters = KernelParameters(kernel_size, kernel_offsets) - return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, kernel_parameters) + return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, kernel_parameters) end @inline function calculate_column_height!(height, location) From 7ff259d652d8a385ca250e48e51562c856438de0 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 23 Jan 2024 18:37:11 -0500 Subject: [PATCH 157/567] test it now --- .../distributed_split_explicit_free_surface.jl | 6 ++---- .../split_explicit_free_surface.jl | 6 ++---- src/MultiRegion/multi_region_split_explicit_free_surface.jl | 6 ++---- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl index 66d30f9422..32b23f1256 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl @@ -7,10 +7,8 @@ import Oceananigans.Models.HydrostaticFreeSurfaceModels: FreeSurface, SplitExpli function SplitExplicitAuxiliaryFields(grid::DistributedGrid) - Nz = size(grid, 3) - - Gᵁ = XFaceField(grid, indices = (:, :, Nz)) - Gⱽ = YFaceField(grid, indices = (:, :, Nz)) + Gᵁ = Field((Face, Center, Nothing), grid) + Gⱽ = Field((Center, Face, Nothing), grid) Hᶠᶜ = Field((Face, Center, Nothing), grid) Hᶜᶠ = Field((Center, Face, Nothing), grid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index 252b335517..f98d116ba9 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -194,10 +194,8 @@ Return the `SplitExplicitAuxiliaryFields` for `grid`. """ function SplitExplicitAuxiliaryFields(grid::AbstractGrid) - Nz = size(grid, 3) - - Gᵁ = XFaceField(grid, indices = (:, :, Nz)) - Gⱽ = YFaceField(grid, indices = (:, :, Nz)) + Gᵁ = Field((Face, Center, Nothing), grid) + Gⱽ = Field((Center, Face, Nothing), grid) Hᶠᶜ = Field((Face, Center, Nothing), grid) Hᶜᶠ = Field((Center, Face, Nothing), grid) diff --git a/src/MultiRegion/multi_region_split_explicit_free_surface.jl b/src/MultiRegion/multi_region_split_explicit_free_surface.jl index 493a2a2f9e..d33f6bd1a6 100644 --- a/src/MultiRegion/multi_region_split_explicit_free_surface.jl +++ b/src/MultiRegion/multi_region_split_explicit_free_surface.jl @@ -6,10 +6,8 @@ import Oceananigans.Models.HydrostaticFreeSurfaceModels: FreeSurface, SplitExpli function SplitExplicitAuxiliaryFields(grid::MultiRegionGrids) - Nz = size(grid, 3) - - Gᵁ = XFaceField(grid; indices = (:, :, Nz)) - Gⱽ = YFaceField(grid; indices = (:, :, Nz)) + Gᵁ = Field((Face, Center, Nothing), grid) + Gⱽ = Field((Center, Face, Nothing), grid) Hᶠᶜ = Field((Face, Center, Nothing), grid) Hᶜᶠ = Field((Center, Face, Nothing), grid) From 06fa3464c948986ef263daf66975a3e8b5810a4e Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Wed, 24 Jan 2024 17:01:47 -0500 Subject: [PATCH 158/567] change name to ZColumnMap() --- src/ImmersedBoundaries/ImmersedBoundaries.jl | 6 ++-- src/ImmersedBoundaries/active_cells_map.jl | 28 ++++++++++++------- .../split_explicit_free_surface_kernels.jl | 12 ++++---- 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/src/ImmersedBoundaries/ImmersedBoundaries.jl b/src/ImmersedBoundaries/ImmersedBoundaries.jl index f7e8d3f429..f13bdabb4b 100644 --- a/src/ImmersedBoundaries/ImmersedBoundaries.jl +++ b/src/ImmersedBoundaries/ImmersedBoundaries.jl @@ -107,7 +107,7 @@ struct ImmersedBoundaryGrid{FT, TX, TY, TZ, G, I, M, S, Arch} <: AbstractGrid{FT underlying_grid :: G immersed_boundary :: I interior_active_cells :: M - surface_active_cells :: S + active_z_columns :: S # Internal interface function ImmersedBoundaryGrid{TX, TY, TZ}(grid::G, ib::I, mi::M, ms::S) where {TX, TY, TZ, G <: AbstractUnderlyingGrid, I, M, S} @@ -133,7 +133,7 @@ const IBG = ImmersedBoundaryGrid @inline get_ibg_property(ibg::IBG, ::Val{:immersed_boundary}) = getfield(ibg, :immersed_boundary) @inline get_ibg_property(ibg::IBG, ::Val{:underlying_grid}) = getfield(ibg, :underlying_grid) @inline get_ibg_property(ibg::IBG, ::Val{:interior_active_cells}) = getfield(ibg, :interior_active_cells) -@inline get_ibg_property(ibg::IBG, ::Val{:surface_active_cells}) = getfield(ibg, :surface_active_cells) +@inline get_ibg_property(ibg::IBG, ::Val{:active_z_columns}) = getfield(ibg, :active_z_columns) @inline architecture(ibg::IBG) = architecture(ibg.underlying_grid) @@ -145,7 +145,7 @@ Adapt.adapt_structure(to, ibg::IBG{FT, TX, TY, TZ}) where {FT, TX, TY, TZ} = ImmersedBoundaryGrid{TX, TY, TZ}(adapt(to, ibg.underlying_grid), adapt(to, ibg.immersed_boundary), adapt(to, ibg.interior_active_cells), - adapt(to, ibg.surface_active_cells)) + adapt(to, ibg.active_z_columns)) with_halo(halo, ibg::ImmersedBoundaryGrid) = ImmersedBoundaryGrid(with_halo(halo, ibg.underlying_grid), ibg.immersed_boundary) diff --git a/src/ImmersedBoundaries/active_cells_map.jl b/src/ImmersedBoundaries/active_cells_map.jl index 48b157a436..5555c14174 100644 --- a/src/ImmersedBoundaries/active_cells_map.jl +++ b/src/ImmersedBoundaries/active_cells_map.jl @@ -11,14 +11,22 @@ using Oceananigans.DistributedComputations: DistributedGrid import Oceananigans.Solvers: solve_batched_tridiagonal_system_kernel! -const ActiveSurfaceIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractArray} const DistributedActiveCellsIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:DistributedGrid, <:Any, <:NamedTuple} # Cannot be used to dispatch in kernels!!! const ArrayActiveCellsIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractArray} const NamedTupleActiveCellsIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:NamedTuple} -const ActiveCellsIBG = Union{DistributedActiveCellsIBG, ArrayActiveCellsIBG, NamedTupleActiveCellsIBG} + +""" +A constant representing an immersed boundary grid, where interior active cells are mapped to linear indices in grid.interior_active_cells +""" +const ActiveCellsIBG = Union{DistributedActiveCellsIBG, ArrayActiveCellsIBG, NamedTupleActiveCellsIBG} + +""" +A constant representing an immersed boundary grid, where active columns in the Z-direction are mapped to linear indices in grid.active_z_columns +""" +const ActiveZColumnsIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractArray} struct InteriorMap end -struct SurfaceMap end +struct ZColumnMap end struct WestMap end struct EastMap end @@ -31,7 +39,7 @@ active_map(::Val{:south}) = SouthMap() active_map(::Val{:north}) = NorthMap() @inline use_only_active_surface_cells(::AbstractGrid) = nothing -@inline use_only_active_surface_cells(::ActiveSurfaceIBG) = SurfaceMap() +@inline use_only_active_surface_cells(::ActiveZColumnsIBG) = ZColumnMap() @inline use_only_active_interior_cells(::AbstractGrid) = nothing @inline use_only_active_interior_cells(::ActiveCellsIBG) = InteriorMap() @@ -57,7 +65,7 @@ Compute the work layout for active cells based on the given map type and grid. @inline active_cells_work_layout(group, size, ::EastMap, grid::NamedTupleActiveCellsIBG) = min(length(grid.interior_active_cells.east), 256), length(grid.interior_active_cells.east) @inline active_cells_work_layout(group, size, ::SouthMap, grid::NamedTupleActiveCellsIBG) = min(length(grid.interior_active_cells.south), 256), length(grid.interior_active_cells.south) @inline active_cells_work_layout(group, size, ::NorthMap, grid::NamedTupleActiveCellsIBG) = min(length(grid.interior_active_cells.north), 256), length(grid.interior_active_cells.north) -@inline active_cells_work_layout(group, size, ::SurfaceMap, grid::ActiveSurfaceIBG) = min(length(grid.surface_active_cells), 256), length(grid.surface_active_cells) +@inline active_cells_work_layout(group, size, ::ZColumnMap, grid::ActiveZColumnsIBG) = min(length(grid.active_z_columns), 256), length(grid.active_z_columns) """ active_linear_index_to_tuple(idx, map, grid) @@ -78,7 +86,7 @@ A tuple of indices corresponding to the linear index. @inline active_linear_index_to_tuple(idx, ::EastMap, grid::NamedTupleActiveCellsIBG) = Base.map(Int, grid.interior_active_cells.east[idx]) @inline active_linear_index_to_tuple(idx, ::SouthMap, grid::NamedTupleActiveCellsIBG) = Base.map(Int, grid.interior_active_cells.south[idx]) @inline active_linear_index_to_tuple(idx, ::NorthMap, grid::NamedTupleActiveCellsIBG) = Base.map(Int, grid.interior_active_cells.north[idx]) -@inline active_linear_index_to_tuple(idx, ::SurfaceMap, grid::ActiveSurfaceIBG) = Base.map(Int, grid.surface_active_cells[idx]) +@inline active_linear_index_to_tuple(idx, ::ZColumnMap, grid::ActiveZColumnsIBG) = Base.map(Int, grid.active_z_columns[idx]) function ImmersedBoundaryGrid(grid, ib; active_cells_map::Bool = true) @@ -88,7 +96,7 @@ function ImmersedBoundaryGrid(grid, ib; active_cells_map::Bool = true) # Create the cells map on the CPU, then switch it to the GPU if active_cells_map interior_map = map_interior_active_cells(ibg) - surface_map = map_surface_active_cells(ibg) + surface_map = map_active_z_columns(ibg) else interior_map = nothing surface_map = nothing @@ -118,7 +126,7 @@ function compute_interior_active_cells(ibg; parameters = :xyz) return active_cells_field end -function compute_surface_active_cells(ibg) +function compute_active_z_columns(ibg) one_field = ConditionalOperation{Center, Center, Center}(OneField(Int), identity, ibg, NotImmersed(truefunc), 0) column = sum(one_field, dims = 3) is_immersed_column = KernelFunctionOperation{Center, Center, Nothing}(active_column, ibg, column) @@ -224,8 +232,8 @@ end # If we eventually want to perform also barotropic step, `w` computation and `p` # computation only on active `columns` -function map_surface_active_cells(ibg) - active_cells_field = compute_surface_active_cells(ibg) +function map_active_z_columns(ibg) + active_cells_field = compute_active_z_columns(ibg) interior_cells = arch_array(CPU(), interior(active_cells_field, :, :, 1)) full_indices = findall(interior_cells) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 64a0844ba7..63b75b8477 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -7,9 +7,9 @@ using Oceananigans.BoundaryConditions using Oceananigans.Operators using Oceananigans.Architectures: convert_args using Oceananigans.ImmersedBoundaries: peripheral_node, immersed_inactive_node, GFBIBG -using Oceananigans.ImmersedBoundaries: inactive_node, IBG, c, f, SurfaceMap +using Oceananigans.ImmersedBoundaries: inactive_node, IBG, c, f, ZColumnMap using Oceananigans.ImmersedBoundaries: mask_immersed_field!, use_only_active_surface_cells, use_only_active_interior_cells -using Oceananigans.ImmersedBoundaries: active_linear_index_to_tuple, ActiveCellsIBG, ActiveSurfaceIBG +using Oceananigans.ImmersedBoundaries: active_linear_index_to_tuple, ActiveCellsIBG, ActiveZColumnsIBGIBG using Oceananigans.DistributedComputations: child_architecture using Oceananigans.DistributedComputations: Distributed using Printf @@ -211,9 +211,9 @@ end # Barotropic Model Kernels # u_Δz = u * Δz -@kernel function _barotropic_mode_kernel!(U, V, grid::ActiveSurfaceIBG, u, v) +@kernel function _barotropic_mode_kernel!(U, V, grid::ActiveZColumnsIBGIBG, u, v) idx = @index(Global, Linear) - i, j = active_linear_index_to_tuple(idx, SurfaceMap(), grid) + i, j = active_linear_index_to_tuple(idx, ZColumnMap(), grid) k_top = grid.Nz+1 # hand unroll first loop @@ -431,9 +431,9 @@ end end # Calculate RHS for the barotopic time step. -@kernel function _compute_integrated_ab2_tendencies!(Gᵁ, Gⱽ, grid::ActiveSurfaceIBG, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) +@kernel function _compute_integrated_ab2_tendencies!(Gᵁ, Gⱽ, grid::ActiveZColumnsIBGIBG, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) idx = @index(Global, Linear) - i, j = active_linear_index_to_tuple(idx, SurfaceMap(), grid) + i, j = active_linear_index_to_tuple(idx, ZColumnMap(), grid) k_top = grid.Nz+1 # hand unroll first loop From 3e1d6f50c044141f54fd60ab3ecf38af37ea3922 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Wed, 24 Jan 2024 17:04:45 -0500 Subject: [PATCH 159/567] changed some function names --- src/ImmersedBoundaries/active_cells_map.jl | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/ImmersedBoundaries/active_cells_map.jl b/src/ImmersedBoundaries/active_cells_map.jl index 5555c14174..5ef49ee44c 100644 --- a/src/ImmersedBoundaries/active_cells_map.jl +++ b/src/ImmersedBoundaries/active_cells_map.jl @@ -96,16 +96,16 @@ function ImmersedBoundaryGrid(grid, ib; active_cells_map::Bool = true) # Create the cells map on the CPU, then switch it to the GPU if active_cells_map interior_map = map_interior_active_cells(ibg) - surface_map = map_active_z_columns(ibg) + column_map = map_active_z_columns(ibg) else interior_map = nothing - surface_map = nothing + column_map = nothing end return ImmersedBoundaryGrid{TX, TY, TZ}(ibg.underlying_grid, ibg.immersed_boundary, interior_map, - surface_map) + column_map) end with_halo(halo, ibg::ActiveCellsIBG) = @@ -140,7 +140,7 @@ const MAXUInt16 = 2^16 - 1 const MAXUInt32 = 2^32 - 1 """ - active_interior_indices(ibg; parameters = :xyz) + interior_active_indices(ibg; parameters = :xyz) Compute the indices of the active interior cells in the given immersed boundary grid. @@ -151,7 +151,7 @@ Compute the indices of the active interior cells in the given immersed boundary # Returns An array of tuples representing the indices of the active interior cells. """ -function active_interior_indices(ibg; parameters = :xyz) +function interior_active_indices(ibg; parameters = :xyz) active_cells_field = compute_interior_active_cells(ibg; parameters) N = maximum(size(ibg)) @@ -190,7 +190,7 @@ end @inline add_3rd_index(t::Tuple, k) = (t[1], t[2], k) -map_interior_active_cells(ibg) = active_interior_indices(ibg; parameters = :xyz) +map_interior_active_cells(ibg) = interior_active_indices(ibg; parameters = :xyz) # In case of a `DistributedGrid` we want to have different maps depending on the # partitioning of the domain @@ -214,10 +214,10 @@ function map_interior_active_cells(ibg::ImmersedBoundaryGrid{<:Any, <:Any, <:Any include_south = !isa(ibg, YFlatGrid) && (Ry != 1) && !(Ty == RightConnected) include_north = !isa(ibg, YFlatGrid) && (Ry != 1) && !(Ty == LeftConnected) - west = include_west ? active_interior_indices(ibg; parameters = KernelParameters(x_boundary, left_offsets)) : nothing - east = include_east ? active_interior_indices(ibg; parameters = KernelParameters(x_boundary, right_x_offsets)) : nothing - south = include_south ? active_interior_indices(ibg; parameters = KernelParameters(y_boundary, left_offsets)) : nothing - north = include_north ? active_interior_indices(ibg; parameters = KernelParameters(y_boundary, right_y_offsets)) : nothing + west = include_west ? interior_active_indices(ibg; parameters = KernelParameters(x_boundary, left_offsets)) : nothing + east = include_east ? interior_active_indices(ibg; parameters = KernelParameters(x_boundary, right_x_offsets)) : nothing + south = include_south ? interior_active_indices(ibg; parameters = KernelParameters(y_boundary, left_offsets)) : nothing + north = include_north ? interior_active_indices(ibg; parameters = KernelParameters(y_boundary, right_y_offsets)) : nothing nx = Rx == 1 ? Nx : (Tx == RightConnected || Tx == LeftConnected ? Nx - Hx : Nx - 2Hx) ny = Ry == 1 ? Ny : (Ty == RightConnected || Ty == LeftConnected ? Ny - Hy : Ny - 2Hy) @@ -225,7 +225,7 @@ function map_interior_active_cells(ibg::ImmersedBoundaryGrid{<:Any, <:Any, <:Any ox = Rx == 1 || Tx == RightConnected ? 0 : Hx oy = Ry == 1 || Ty == RightConnected ? 0 : Hy - interior = active_interior_indices(ibg; parameters = KernelParameters((nx, ny, Nz), (ox, oy, 0))) + interior = interior_active_indices(ibg; parameters = KernelParameters((nx, ny, Nz), (ox, oy, 0))) return (; interior, west, east, south, north) end From c74749fe9d5b3cab30cf401fb223831ddc73d394 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Wed, 24 Jan 2024 17:07:56 -0500 Subject: [PATCH 160/567] add comment --- src/DistributedComputations/distributed_grids.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/DistributedComputations/distributed_grids.jl b/src/DistributedComputations/distributed_grids.jl index b6fe50b5fb..933a91b198 100644 --- a/src/DistributedComputations/distributed_grids.jl +++ b/src/DistributedComputations/distributed_grids.jl @@ -41,8 +41,9 @@ end @inline local_sizes(N, R::Fractional) = Tuple(ceil(Int, N * r) for r in R.sizes) @inline function local_sizes(N, R::Sizes) if N != sum(R.sizes) - @warn "The domain size specified in the architecture $(sum(R.sizes)) is inconsistent - with the grid size $N: using the architecture-specified size" + @warn "The Sizes specified in the architecture $(R.sizes) is inconsistent \ + with the grid size: (N = $N != sum(Sizes) = $(sum(R.sizes))). \ + Using $(R.sizes)..." end return R.sizes end From 59dd773aa42f30a24d79f7f865b6bc746cd3c28e Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Wed, 24 Jan 2024 17:15:38 -0500 Subject: [PATCH 161/567] bugfix in warning --- src/DistributedComputations/distributed_grids.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/DistributedComputations/distributed_grids.jl b/src/DistributedComputations/distributed_grids.jl index 933a91b198..7a98830d7c 100644 --- a/src/DistributedComputations/distributed_grids.jl +++ b/src/DistributedComputations/distributed_grids.jl @@ -41,8 +41,8 @@ end @inline local_sizes(N, R::Fractional) = Tuple(ceil(Int, N * r) for r in R.sizes) @inline function local_sizes(N, R::Sizes) if N != sum(R.sizes) - @warn "The Sizes specified in the architecture $(R.sizes) is inconsistent \ - with the grid size: (N = $N != sum(Sizes) = $(sum(R.sizes))). \ + @warn "The Sizes specified in the architecture $(R.sizes) is inconsistent + with the grid size: (N = $N != sum(Sizes) = $(sum(R.sizes))). Using $(R.sizes)..." end return R.sizes From 3210d82951ac3d717cc5016bad2f3cda50e26e97 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Wed, 24 Jan 2024 17:26:13 -0500 Subject: [PATCH 162/567] bugfix --- .../split_explicit_free_surface_kernels.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 63b75b8477..9bf0ebac6a 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -9,7 +9,7 @@ using Oceananigans.Architectures: convert_args using Oceananigans.ImmersedBoundaries: peripheral_node, immersed_inactive_node, GFBIBG using Oceananigans.ImmersedBoundaries: inactive_node, IBG, c, f, ZColumnMap using Oceananigans.ImmersedBoundaries: mask_immersed_field!, use_only_active_surface_cells, use_only_active_interior_cells -using Oceananigans.ImmersedBoundaries: active_linear_index_to_tuple, ActiveCellsIBG, ActiveZColumnsIBGIBG +using Oceananigans.ImmersedBoundaries: active_linear_index_to_tuple, ActiveCellsIBG, ActiveZColumnsIBG using Oceananigans.DistributedComputations: child_architecture using Oceananigans.DistributedComputations: Distributed using Printf @@ -211,7 +211,7 @@ end # Barotropic Model Kernels # u_Δz = u * Δz -@kernel function _barotropic_mode_kernel!(U, V, grid::ActiveZColumnsIBGIBG, u, v) +@kernel function _barotropic_mode_kernel!(U, V, grid::ActiveZColumnsIBG, u, v) idx = @index(Global, Linear) i, j = active_linear_index_to_tuple(idx, ZColumnMap(), grid) k_top = grid.Nz+1 @@ -431,7 +431,7 @@ end end # Calculate RHS for the barotopic time step. -@kernel function _compute_integrated_ab2_tendencies!(Gᵁ, Gⱽ, grid::ActiveZColumnsIBGIBG, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) +@kernel function _compute_integrated_ab2_tendencies!(Gᵁ, Gⱽ, grid::ActiveZColumnsIBG, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) idx = @index(Global, Linear) i, j = active_linear_index_to_tuple(idx, ZColumnMap(), grid) k_top = grid.Nz+1 From 50f8b0608b2eed513ff8dcfaa21cf156e37f1865 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Wed, 24 Jan 2024 22:49:10 -0500 Subject: [PATCH 163/567] some problems with distributed --- src/DistributedComputations/distributed_grids.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DistributedComputations/distributed_grids.jl b/src/DistributedComputations/distributed_grids.jl index 7a98830d7c..ff61cadbfd 100644 --- a/src/DistributedComputations/distributed_grids.jl +++ b/src/DistributedComputations/distributed_grids.jl @@ -340,4 +340,4 @@ function reconstruct_global_topology(T, R, r, r1, r2, comm) else return Bounded end -end +end \ No newline at end of file From d55338c1249fed75b0565c20fb27418aa916be9c Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Fri, 2 Feb 2024 11:08:17 -0500 Subject: [PATCH 164/567] apparently no more splatting --- .../compute_hydrostatic_free_surface_tendencies.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl index a9997c6b38..fe40f1f2a0 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl @@ -255,12 +255,12 @@ end ##### """ Calculate the right-hand-side of the tracer advection-diffusion equation. """ -@kernel function compute_hydrostatic_free_surface_Gc!(Gc, grid, map, args) +@kernel function compute_hydrostatic_free_surface_Gc!(Gc, grid, map, args...) i, j, k = @index(Global, NTuple) @inbounds Gc[i, j, k] = hydrostatic_free_surface_tracer_tendency(i, j, k, grid, args...) end -@kernel function compute_hydrostatic_free_surface_Gc!(Gc, grid::ActiveCellsIBG, map, args) +@kernel function compute_hydrostatic_free_surface_Gc!(Gc, grid::ActiveCellsIBG, map, args...) idx = @index(Global, Linear) i, j, k = active_linear_index_to_tuple(idx, map, grid) @inbounds Gc[i, j, k] = hydrostatic_free_surface_tracer_tendency(i, j, k, grid, args...) @@ -272,7 +272,7 @@ end @inbounds Ge[i, j, k] = hydrostatic_turbulent_kinetic_energy_tendency(i, j, k, grid, args...) end -@kernel function compute_hydrostatic_free_surface_Ge!(Ge, grid::ActiveCellsIBG, map, args) +@kernel function compute_hydrostatic_free_surface_Ge!(Ge, grid::ActiveCellsIBG, map, args...) idx = @index(Global, Linear) i, j, k = active_linear_index_to_tuple(idx, map, grid) @inbounds Ge[i, j, k] = hydrostatic_turbulent_kinetic_energy_tendency(i, j, k, grid, args...) From dc8903bdd98abeebfb5040de182b76d2ce07b72e Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Fri, 2 Feb 2024 11:09:30 -0500 Subject: [PATCH 165/567] and also this splatting --- .../compute_hydrostatic_free_surface_tendencies.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl index fe40f1f2a0..9edc1d5963 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl @@ -267,7 +267,7 @@ end end """ Calculate the right-hand-side of the subgrid scale energy equation. """ -@kernel function compute_hydrostatic_free_surface_Ge!(Ge, grid, map, args) +@kernel function compute_hydrostatic_free_surface_Ge!(Ge, grid, map, args...) i, j, k = @index(Global, NTuple) @inbounds Ge[i, j, k] = hydrostatic_turbulent_kinetic_energy_tendency(i, j, k, grid, args...) end From 894d75346b37f92101833c25739dff0da987dd3d Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Fri, 2 Feb 2024 13:00:01 -0500 Subject: [PATCH 166/567] removes kernel splatting --- ...pute_hydrostatic_free_surface_tendencies.jl | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl index 9edc1d5963..e0c0fc49e6 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl @@ -122,7 +122,7 @@ function compute_hydrostatic_free_surface_tendency_contributions!(model, kernel_ c_tendency, grid, only_active_cells, - args...; + args; only_active_cells) end end @@ -155,7 +155,7 @@ function compute_free_surface_tendency!(grid, model, kernel_parameters) launch!(arch, grid, kernel_parameters, compute_hydrostatic_free_surface_Gη!, model.timestepper.Gⁿ.η, - grid, args...) + grid, args) return nothing end @@ -189,12 +189,12 @@ function compute_hydrostatic_momentum_tendencies!(model, velocities, kernel_para for parameters in kernel_parameters launch!(arch, grid, parameters, compute_hydrostatic_free_surface_Gu!, model.timestepper.Gⁿ.u, grid, - only_active_cells, u_kernel_args...; + only_active_cells, u_kernel_args; only_active_cells) launch!(arch, grid, parameters, compute_hydrostatic_free_surface_Gv!, model.timestepper.Gⁿ.v, grid, - only_active_cells, v_kernel_args...; + only_active_cells, v_kernel_args; only_active_cells) end @@ -255,24 +255,24 @@ end ##### """ Calculate the right-hand-side of the tracer advection-diffusion equation. """ -@kernel function compute_hydrostatic_free_surface_Gc!(Gc, grid, map, args...) +@kernel function compute_hydrostatic_free_surface_Gc!(Gc, grid, map, args) i, j, k = @index(Global, NTuple) @inbounds Gc[i, j, k] = hydrostatic_free_surface_tracer_tendency(i, j, k, grid, args...) end -@kernel function compute_hydrostatic_free_surface_Gc!(Gc, grid::ActiveCellsIBG, map, args...) +@kernel function compute_hydrostatic_free_surface_Gc!(Gc, grid::ActiveCellsIBG, map, args) idx = @index(Global, Linear) i, j, k = active_linear_index_to_tuple(idx, map, grid) @inbounds Gc[i, j, k] = hydrostatic_free_surface_tracer_tendency(i, j, k, grid, args...) end """ Calculate the right-hand-side of the subgrid scale energy equation. """ -@kernel function compute_hydrostatic_free_surface_Ge!(Ge, grid, map, args...) +@kernel function compute_hydrostatic_free_surface_Ge!(Ge, grid, map, args) i, j, k = @index(Global, NTuple) @inbounds Ge[i, j, k] = hydrostatic_turbulent_kinetic_energy_tendency(i, j, k, grid, args...) end -@kernel function compute_hydrostatic_free_surface_Ge!(Ge, grid::ActiveCellsIBG, map, args...) +@kernel function compute_hydrostatic_free_surface_Ge!(Ge, grid::ActiveCellsIBG, map, args) idx = @index(Global, Linear) i, j, k = active_linear_index_to_tuple(idx, map, grid) @inbounds Ge[i, j, k] = hydrostatic_turbulent_kinetic_energy_tendency(i, j, k, grid, args...) @@ -283,7 +283,7 @@ end ##### """ Calculate the right-hand-side of the free surface displacement (``η``) equation. """ -@kernel function compute_hydrostatic_free_surface_Gη!(Gη, grid, args...) +@kernel function compute_hydrostatic_free_surface_Gη!(Gη, grid, args) i, j = @index(Global, NTuple) @inbounds Gη[i, j, grid.Nz+1] = free_surface_tendency(i, j, grid, args...) end From 805af1eb33db85d1dd297b09ef2f772f17ae7345 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 12 Feb 2024 15:35:54 -0500 Subject: [PATCH 167/567] Update src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl Co-authored-by: Gregory L. Wagner --- .../HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index 1262eae75f..d32b5786d8 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -214,7 +214,7 @@ function SplitExplicitAuxiliaryFields(grid::AbstractGrid) kernel_parameters = :xy - return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, kernel_parameters) + return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, kernel_parameters) end """ From 16cbc27febc98f10a4d85f3726d51b738d331713 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 14 Feb 2024 07:38:21 -0500 Subject: [PATCH 168/567] Update tracer_advection_operators.jl --- src/Advection/tracer_advection_operators.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Advection/tracer_advection_operators.jl b/src/Advection/tracer_advection_operators.jl index 82851cd598..a528a2e0a8 100644 --- a/src/Advection/tracer_advection_operators.jl +++ b/src/Advection/tracer_advection_operators.jl @@ -12,7 +12,8 @@ end """ function TracerAdvection(; x, y, z) -builds a `TracerAdvection` type with different reconstructions in `x`, `y`, and `z` +Builds a `TracerAdvection` type with reconstructions schemes `x`, `y`, and `z` to be applied in +the x, y, and z direction, respectively. """ function TracerAdvection(; x, y, z) Nx = required_halo_size(x) From 6db2b20812f0916e41d4af9e9f0de384b853422b Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Fri, 16 Feb 2024 17:36:14 -0500 Subject: [PATCH 169/567] bugfix --- .../HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index d7170d2575..385e303b1f 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -33,7 +33,7 @@ function GeneralizedSpacingGrid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, :: # Initial "at-rest" conditions launch!(architecture(grid), grid, :xy, _update_zstar!, - sⁿ, s⁻, ΔzF, ΔzC, ZeroField(grid), grid, Val(grid.Nz)) + sⁿ, s⁻, ΔzF, ΔzC, ZeroField(), grid, Val(grid.Nz)) fill_halo_regions!((s⁻, sⁿ)) fill_halo_regions!((ΔzF, ΔzC)) From 943f9fe8c0ec5fec24526e4d3e9588f299a40601 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sun, 25 Feb 2024 11:00:05 -0500 Subject: [PATCH 170/567] small bugfix --- .../vector_invariant_self_upwinding.jl | 31 +++++++++---------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/src/Advection/vector_invariant_self_upwinding.jl b/src/Advection/vector_invariant_self_upwinding.jl index f35bb775af..55ed743ed0 100644 --- a/src/Advection/vector_invariant_self_upwinding.jl +++ b/src/Advection/vector_invariant_self_upwinding.jl @@ -2,11 +2,15 @@ ##### Self Upwinding of Divergence Flux, the best option! ##### -@inline δx_U(i, j, k, grid, u, v) = δxᶜᵃᵃ(i, j, k, grid, Ax_qᶠᶜᶜ, u) -@inline δy_V(i, j, k, grid, u, v) = δyᵃᶜᵃ(i, j, k, grid, Ay_qᶜᶠᶜ, v) +# Metric term for moving grids +∂t_∂s_grid(i, j, k, grid) = zero(grid) +@inline V_times_∂t_∂s_grid(i, j, k, grid) = ∂t_∂s_grid(i, j, k, grid) * Vᶜᶜᶜ(i, j, k, grid) -@inline δx_U_plus_metric(i, j, k, grid, u, v) = δxᶜᵃᵃ(i, j, k, grid, Ax_qᶠᶜᶜ, u) -@inline δy_V_plus_metric(i, j, k, grid, u, v) = δyᵃᶜᵃ(i, j, k, grid, Ay_qᶜᶠᶜ, v) +@inline δx_U(i, j, k, grid, u, v) = δxᶜᶜᶜ(i, j, k, grid, Ax_qᶠᶜᶜ, u) +@inline δy_V(i, j, k, grid, u, v) = δyᶜᶜᶜ(i, j, k, grid, Ay_qᶜᶠᶜ, v) + +@inline δx_U_plus_metric(i, j, k, grid, u, v) = δxᶜᶜᶜ(i, j, k, grid, Ax_qᶠᶜᶜ, u) + V_times_∂t_∂s_grid(i, j, k, grid) +@inline δy_V_plus_metric(i, j, k, grid, u, v) = δyᶜᶜᶜ(i, j, k, grid, Ay_qᶜᶠᶜ, v) + V_times_∂t_∂s_grid(i, j, k, grid) # Velocity smoothness for divergence upwinding @inline U_smoothness(i, j, k, grid, u, v) = ℑxᶜᵃᵃ(i, j, k, grid, Ax_qᶠᶜᶜ, u) @@ -15,11 +19,6 @@ # Divergence smoothness for divergence upwinding @inline divergence_smoothness(i, j, k, grid, u, v) = δx_U(i, j, k, grid, u, v) + δy_V(i, j, k, grid, u, v) -# Metric term for moving grids -∂t_∂s_grid(i, j, k, grid) = zero(grid) - -@inline V_times_∂t_∂s_grid(i, j, k, grid) = ∂t_∂s_grid(i, j, k, grid) * Vᶜᶜᶜ(i, j, k, grid) - @inline function upwinded_divergence_flux_Uᶠᶜᶜ(i, j, k, grid, scheme::VectorInvariantSelfVerticalUpwinding, u, v) δU_stencil = scheme.upwinding.δU_stencil @@ -27,11 +26,10 @@ @inbounds û = u[i, j, k] δvˢ = _symmetric_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, cross_scheme, δy_V, u, v) - ∂ts = _symmetric_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, cross_scheme, V_times_∂t_∂s_grid) - δuᴸ = _left_biased_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, δx_U, δU_stencil, u, v) - δuᴿ = _right_biased_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, δx_U, δU_stencil, u, v) + δuᴸ = _left_biased_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, δx_U_plus_metric, δU_stencil, u, v) + δuᴿ = _right_biased_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, δx_U_plus_metric, δU_stencil, u, v) - return upwind_biased_product(û, δuᴸ, δuᴿ) + û * (δvˢ + ∂ts) + return upwind_biased_product(û, δuᴸ, δuᴿ) + û * δvˢ end @inline function upwinded_divergence_flux_Vᶜᶠᶜ(i, j, k, grid, scheme::VectorInvariantSelfVerticalUpwinding, u, v) @@ -41,11 +39,10 @@ end @inbounds v̂ = v[i, j, k] δuˢ = _symmetric_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, cross_scheme, δx_U, u, v) - ∂ts = _symmetric_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, cross_scheme, V_times_∂t_∂s_grid) - δvᴸ = _left_biased_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, δy_V, δV_stencil, u, v) - δvᴿ = _right_biased_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, δy_V, δV_stencil, u, v) + δvᴸ = _left_biased_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, δy_V_plus_metric, δV_stencil, u, v) + δvᴿ = _right_biased_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, δy_V_plus_metric, δV_stencil, u, v) - return upwind_biased_product(v̂, δvᴸ, δvᴿ) + v̂ * (δuˢ + ∂ts) + return upwind_biased_product(v̂, δvᴸ, δvᴿ) + v̂ * δuˢ end ##### From fb606f88ac42d26774c8ea2f95891fb02c097cd2 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sun, 25 Feb 2024 15:46:51 -0500 Subject: [PATCH 171/567] small fix --- src/Advection/vector_invariant_self_upwinding.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Advection/vector_invariant_self_upwinding.jl b/src/Advection/vector_invariant_self_upwinding.jl index 55ed743ed0..41a861816e 100644 --- a/src/Advection/vector_invariant_self_upwinding.jl +++ b/src/Advection/vector_invariant_self_upwinding.jl @@ -17,7 +17,7 @@ @inline V_smoothness(i, j, k, grid, u, v) = ℑyᵃᶜᵃ(i, j, k, grid, Ay_qᶜᶠᶜ, v) # Divergence smoothness for divergence upwinding -@inline divergence_smoothness(i, j, k, grid, u, v) = δx_U(i, j, k, grid, u, v) + δy_V(i, j, k, grid, u, v) +@inline divergence_smoothness(i, j, k, grid, u, v) = δx_U(i, j, k, grid, u, v) + δy_V(i, j, k, grid, u, v) + V_times_∂t_∂s_grid(i, j, k, grid) @inline function upwinded_divergence_flux_Uᶠᶜᶜ(i, j, k, grid, scheme::VectorInvariantSelfVerticalUpwinding, u, v) From 375a51227181eea893e43b4aaa80ab7e4e5b32bf Mon Sep 17 00:00:00 2001 From: Gregory Wagner Date: Mon, 26 Feb 2024 12:01:54 -0700 Subject: [PATCH 172/567] Some minor clarifications --- src/Advection/tracer_advection_operators.jl | 13 +++++++------ src/Architectures.jl | 3 +-- .../distributed_architectures.jl | 2 +- .../halo_communication.jl | 16 +++++++--------- src/ImmersedBoundaries/grid_fitted_bottom.jl | 12 ------------ .../distributed_split_explicit_free_surface.jl | 11 ++++++----- .../split_explicit_free_surface_kernels.jl | 5 +---- 7 files changed, 23 insertions(+), 39 deletions(-) diff --git a/src/Advection/tracer_advection_operators.jl b/src/Advection/tracer_advection_operators.jl index a528a2e0a8..13cb8f2d05 100644 --- a/src/Advection/tracer_advection_operators.jl +++ b/src/Advection/tracer_advection_operators.jl @@ -10,19 +10,20 @@ struct TracerAdvection{N, FT, A, B, C} <: AbstractAdvectionScheme{N, FT} end """ - function TracerAdvection(; x, y, z) + function TracerAdvection(x, y, z) Builds a `TracerAdvection` type with reconstructions schemes `x`, `y`, and `z` to be applied in the x, y, and z direction, respectively. """ -function TracerAdvection(; x, y, z) - Nx = required_halo_size(x) - Ny = required_halo_size(y) - Nz = required_halo_size(z) +function TracerAdvection(x_advection, y_advection, z_advection) + Hx = required_halo_size(x_advection) + Hy = required_halo_size(y_advection) + Hz = required_halo_size(z_advection) FT = eltype(x) + H = max(Hx, Hy, Hz) - return TracerAdvection{max(Nx, Ny, Nz), FT}(x, y, z) + return TracerAdvection{H, FT}(x_advection, y_advection, z_advection) end @inline _advective_tracer_flux_x(args...) = advective_tracer_flux_x(args...) diff --git a/src/Architectures.jl b/src/Architectures.jl index 0f2883f2d3..78b55a067a 100644 --- a/src/Architectures.jl +++ b/src/Architectures.jl @@ -116,10 +116,9 @@ end @inline unsafe_free!(a) = nothing # Convert arguments to GPU-compatible types - @inline convert_args(::CPU, args) = args @inline convert_args(::GPU, args) = CUDA.cudaconvert(args) @inline convert_args(::GPU, args::Tuple) = map(CUDA.cudaconvert, args) - end # module + diff --git a/src/DistributedComputations/distributed_architectures.jl b/src/DistributedComputations/distributed_architectures.jl index b44b0e3371..2aa2a53e45 100644 --- a/src/DistributedComputations/distributed_architectures.jl +++ b/src/DistributedComputations/distributed_architectures.jl @@ -210,7 +210,7 @@ function Distributed(child_architecture = CPU(); partition = Partition(MPI.Comm_size(communicator))) if !(MPI.Initialized()) - @info "MPI has not been initialized, so we are calling MPI.Init()" + @info "MPI has not been initialized, so we are calling MPI.Init()." MPI.Init() end diff --git a/src/DistributedComputations/halo_communication.jl b/src/DistributedComputations/halo_communication.jl index 45a520b5ee..06264dd39c 100644 --- a/src/DistributedComputations/halo_communication.jl +++ b/src/DistributedComputations/halo_communication.jl @@ -142,11 +142,10 @@ end # corner passing routine function fill_corners!(c, connectivity, indices, loc, arch, grid, buffers, args...; async = false, only_local_halos = false, kwargs...) - if only_local_halos # No corner filling needed! - return nothing - end + # No corner filling needed! + only_local_halos && return nothing - # This has to be synchronized!! + # This has to be synchronized! fill_send_buffers!(c, buffers, grid, Val(:corners)) sync_device!(arch) @@ -243,9 +242,9 @@ for (side, opposite_side) in zip([:west, :south], [:east, :north]) function $fill_both_halo!(c, bc_side::DCBCT, bc_opposite_side::DCBCT, size, offset, loc, arch::Distributed, grid::DistributedGrid, buffers, args...; only_local_halos = false, kwargs...) - sync_device!(arch) - only_local_halos && return nothing + + sync_device!(arch) @assert bc_side.condition.from == bc_opposite_side.condition.from # Extra protection in case of bugs local_rank = bc_side.condition.from @@ -274,15 +273,14 @@ for side in [:west, :east, :south, :north] function $fill_side_halo!(c, bc_side::DCBCT, size, offset, loc, arch::Distributed, grid::DistributedGrid, buffers, args...; only_local_halos = false, kwargs...) - sync_device!(arch) - only_local_halos && return nothing + sync_device!(arch) + child_arch = child_architecture(arch) local_rank = bc_side.condition.from recv_req = $recv_and_fill_side_halo!(c, grid, arch, loc, local_rank, bc_side.condition.to, buffers) - send_req = $send_side_halo(c, grid, arch, loc, local_rank, bc_side.condition.to, buffers) return [send_req, recv_req] diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index f1ca0f1e4b..e693e20e2d 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -72,24 +72,12 @@ Computes ib.bottom_height and wraps in an array. function ImmersedBoundaryGrid(grid, ib::GridFittedBottom) bottom_field = Field{Center, Center, Nothing}(grid) set!(bottom_field, ib.bottom_height) - - # Make sure that `abs(bottom_height) <= grid.Lz` to constrain the bottom - @apply_regionally launch!(architecture(grid), grid, :xy, _limit_bottom_height!, bottom_field, grid.Lz) - fill_halo_regions!(bottom_field) new_ib = GridFittedBottom(bottom_field, ib.immersed_condition) TX, TY, TZ = topology(grid) return ImmersedBoundaryGrid{TX, TY, TZ}(grid, new_ib) end -# Make sure that `abs(bottom_height) <= grid.Lz` to constrain the bottom -@kernel function _limit_bottom_height!(bottom_field, Lz) - i, j = @index(Global, NTuple) - if abs(bottom_field[i, j, 1]) > Lz - bottom_field[i, j, 1] = sign(bottom_field[i, j, 1]) * Lz - end -end - @inline function _immersed_cell(i, j, k, underlying_grid, ib::GridFittedBottom{<:Any, <:InterfaceImmersedCondition}) z = znode(i, j, k+1, underlying_grid, c, c, f) h = @inbounds ib.bottom_height[i, j, 1] diff --git a/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl index 32b23f1256..4f14f69f2a 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl @@ -66,14 +66,15 @@ function FreeSurface(free_surface::SplitExplicitFreeSurface, velocities, grid::D old_halos = halo_size(grid) Nsubsteps = length(settings.substepping.averaging_weights) - new_halos = distributed_split_explicit_halos(old_halos, Nsubsteps+1, grid) - new_grid = with_halo(new_halos, grid) + extended_halos = distributed_split_explicit_halos(old_halos, Nsubsteps+1, grid) + extended_grid = with_halo(extended_halos, grid) - η = ZFaceField(new_grid, indices = (:, :, size(new_grid, 3)+1)) + Nze = size(extended_grid, 3) + η = ZFaceField(extended_grid, indices = (:, :, Nze+1)) return SplitExplicitFreeSurface(η, - SplitExplicitState(new_grid, settings.timestepper), - SplitExplicitAuxiliaryFields(new_grid), + SplitExplicitState(extended_grid, settings.timestepper), + SplitExplicitAuxiliaryFields(extended_grid), free_surface.gravitational_acceleration, free_surface.settings) end diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 9bf0ebac6a..370528c4e3 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -18,11 +18,10 @@ using Printf const β = 0.281105 const α = 1.5 + β const θ = - 0.5 - 2β - const γ = 0.088 const δ = 0.614 const ϵ = 0.013 -const μ = 1.0 - δ - γ - ϵ +const μ = 1 - δ - γ - ϵ # Evolution Kernels # @@ -404,9 +403,7 @@ function iterate_split_explicit!(free_surface, grid, Δτᴮ, weights, ::Val{Nsu @unroll for substep in 1:Nsubsteps Base.@_inline_meta - averaging_weight = weights[substep] - free_surface_kernel!(converted_η_args...) barotropic_velocity_kernel!(averaging_weight, converted_U_args...) end From f732dd171711885c580bd719d6fa64ac7c4b5089 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 26 Feb 2024 14:02:07 -0500 Subject: [PATCH 173/567] remove grid from ab2step --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 12 ------------ .../hydrostatic_free_surface_ab2_step.jl | 4 ++-- src/TimeSteppers/quasi_adams_bashforth_2.jl | 7 +++---- 3 files changed, 5 insertions(+), 18 deletions(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index f1ca0f1e4b..e693e20e2d 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -72,24 +72,12 @@ Computes ib.bottom_height and wraps in an array. function ImmersedBoundaryGrid(grid, ib::GridFittedBottom) bottom_field = Field{Center, Center, Nothing}(grid) set!(bottom_field, ib.bottom_height) - - # Make sure that `abs(bottom_height) <= grid.Lz` to constrain the bottom - @apply_regionally launch!(architecture(grid), grid, :xy, _limit_bottom_height!, bottom_field, grid.Lz) - fill_halo_regions!(bottom_field) new_ib = GridFittedBottom(bottom_field, ib.immersed_condition) TX, TY, TZ = topology(grid) return ImmersedBoundaryGrid{TX, TY, TZ}(grid, new_ib) end -# Make sure that `abs(bottom_height) <= grid.Lz` to constrain the bottom -@kernel function _limit_bottom_height!(bottom_field, Lz) - i, j = @index(Global, NTuple) - if abs(bottom_field[i, j, 1]) > Lz - bottom_field[i, j, 1] = sign(bottom_field[i, j, 1]) * Lz - end -end - @inline function _immersed_cell(i, j, k, underlying_grid, ib::GridFittedBottom{<:Any, <:InterfaceImmersedCondition}) z = znode(i, j, k+1, underlying_grid, c, c, f) h = @inbounds ib.bottom_height[i, j, 1] diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl index bc7dde1c31..14d8841f67 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl @@ -43,7 +43,7 @@ function ab2_step_velocities!(velocities, model, Δt, χ) velocity_field = model.velocities[name] launch!(model.architecture, model.grid, :xyz, - ab2_step_field!, velocity_field, Δt, χ, Gⁿ, G⁻, model.grid) + ab2_step_field!, velocity_field, Δt, χ, Gⁿ, G⁻) # TODO: let next implicit solve depend on previous solve + explicit velocity step # Need to distinguish between solver events and tendency calculation events. @@ -78,7 +78,7 @@ function ab2_step_tracers!(tracers, model, Δt, χ) closure = model.closure launch!(model.architecture, model.grid, :xyz, - ab2_step_field!, tracer_field, Δt, χ, Gⁿ, G⁻, model.grid) + ab2_step_field!, tracer_field, Δt, χ, Gⁿ, G⁻) implicit_step!(tracer_field, model.timestepper.implicit_solver, diff --git a/src/TimeSteppers/quasi_adams_bashforth_2.jl b/src/TimeSteppers/quasi_adams_bashforth_2.jl index 5e26927234..780df779b6 100644 --- a/src/TimeSteppers/quasi_adams_bashforth_2.jl +++ b/src/TimeSteppers/quasi_adams_bashforth_2.jl @@ -125,8 +125,7 @@ function ab2_step!(model, Δt, χ) step_field_kernel!(field, Δt, χ, model.timestepper.Gⁿ[i], - model.timestepper.G⁻[i], - model.grid) + model.timestepper.G⁻[i]) # TODO: function tracer_index(model, field_index) = field_index - 3, etc... tracer_index = Val(i - 3) # assumption @@ -149,7 +148,7 @@ Time step velocity fields via the 2nd-order quasi Adams-Bashforth method `U^{n+1} = U^n + Δt ((3/2 + χ) * G^{n} - (1/2 + χ) G^{n-1})` """ -@kernel function ab2_step_field!(u, Δt, χ, Gⁿ, G⁻, grid) +@kernel function ab2_step_field!(u, Δt, χ, Gⁿ, G⁻) i, j, k = @index(Global, NTuple) FT = eltype(χ) @@ -159,4 +158,4 @@ Time step velocity fields via the 2nd-order quasi Adams-Bashforth method @inbounds u[i, j, k] += convert(FT, Δt) * ((one_point_five + χ) * Gⁿ[i, j, k] - (oh_point_five + χ) * G⁻[i, j, k]) end -@kernel ab2_step_field!(::FunctionField, Δt, χ, Gⁿ, G⁻, grid) = nothing +@kernel ab2_step_field!(::FunctionField, Δt, χ, Gⁿ, G⁻) = nothing From eda141555d3bfcfbd12212959358d6b58cbc391f Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 26 Feb 2024 16:09:12 -0500 Subject: [PATCH 174/567] bugfix --- src/Advection/tracer_advection_operators.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Advection/tracer_advection_operators.jl b/src/Advection/tracer_advection_operators.jl index 13cb8f2d05..86cc43229b 100644 --- a/src/Advection/tracer_advection_operators.jl +++ b/src/Advection/tracer_advection_operators.jl @@ -20,7 +20,7 @@ function TracerAdvection(x_advection, y_advection, z_advection) Hy = required_halo_size(y_advection) Hz = required_halo_size(z_advection) - FT = eltype(x) + FT = eltype(x_advection) H = max(Hx, Hy, Hz) return TracerAdvection{H, FT}(x_advection, y_advection, z_advection) From b2497e71a26bbdd0b889bfd9d71e90a0ee02ee1d Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 26 Feb 2024 17:42:09 -0500 Subject: [PATCH 175/567] save data on CPU --- src/OutputReaders/on_disk_field_time_series.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OutputReaders/on_disk_field_time_series.jl b/src/OutputReaders/on_disk_field_time_series.jl index fed367e973..3bf3180db7 100644 --- a/src/OutputReaders/on_disk_field_time_series.jl +++ b/src/OutputReaders/on_disk_field_time_series.jl @@ -30,7 +30,7 @@ function set!(time_series::OnDiskFieldTimeSeries, f::Field, index::Int) jldopen(path, "a+") do file initialize_file!(file, name, time_series) maybe_write_property!(file, "timeseries/t/$index", time_series.times[index]) - maybe_write_property!(file, "timeseries/$(name)/$(index)", parent(f)) + maybe_write_property!(file, "timeseries/$(name)/$(index)", Array(parent(f))) end end From 1da125c92285f102c9f701fca82b930685dee8ee Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 27 Feb 2024 10:57:07 -0500 Subject: [PATCH 176/567] changing some names --- ext/OceananigansEnzymeExt.jl | 8 ++-- src/ImmersedBoundaries/active_cells_map.jl | 20 ++++----- ...static_free_surface_boundary_tendencies.jl | 11 +++-- ...ute_hydrostatic_free_surface_tendencies.jl | 22 ++++----- .../hydrostatic_free_surface_ab2_step.jl | 2 +- .../split_explicit_free_surface_kernels.jl | 6 +-- ...ore_hydrostatic_free_surface_tendencies.jl | 2 +- .../compute_nonhydrostatic_tendencies.jl | 22 ++++----- src/Utils/kernel_launching.jl | 45 ++++++++++++++----- .../distributed_hydrostatic_turbulence.jl | 2 +- 10 files changed, 85 insertions(+), 55 deletions(-) diff --git a/ext/OceananigansEnzymeExt.jl b/ext/OceananigansEnzymeExt.jl index ce1662d32f..c3b4954de8 100644 --- a/ext/OceananigansEnzymeExt.jl +++ b/ext/OceananigansEnzymeExt.jl @@ -228,7 +228,7 @@ function EnzymeCore.EnzymeRules.augmented_primal(config, include_right_boundaries = false, reduced_dimensions = (), location = nothing, - only_active_cells = nothing, + active_cells_map = nothing, kwargs...) where N @@ -239,8 +239,8 @@ function EnzymeCore.EnzymeRules.augmented_primal(config, offset = Oceananigans.Utils.offsets(workspec.val) - if !isnothing(only_active_cells) - workgroup, worksize = Oceananigans.Utils.active_cells_work_layout(workgroup, worksize, only_active_cells, grid.val) + if !isnothing(active_cells_map) + workgroup, worksize = Oceananigans.Utils.active_cells_work_layout(workgroup, worksize, active_cells_map, grid.val) offset = nothing end @@ -286,7 +286,7 @@ function EnzymeCore.EnzymeRules.reverse(config::EnzymeCore.EnzymeRules.ConfigWid include_right_boundaries = false, reduced_dimensions = (), location = nothing, - only_active_cells = nothing, + active_cells_map = nothing, kwargs...) where N subrets = if tape !== nothing diff --git a/src/ImmersedBoundaries/active_cells_map.jl b/src/ImmersedBoundaries/active_cells_map.jl index 5ef49ee44c..dbb4869368 100644 --- a/src/ImmersedBoundaries/active_cells_map.jl +++ b/src/ImmersedBoundaries/active_cells_map.jl @@ -33,17 +33,17 @@ struct EastMap end struct SouthMap end struct NorthMap end -active_map(::Val{:west}) = WestMap() -active_map(::Val{:east}) = EastMap() -active_map(::Val{:south}) = SouthMap() -active_map(::Val{:north}) = NorthMap() +@inline active_surface_map(::AbstractGrid) = nothing +@inline active_surface_map(::ActiveZColumnsIBG) = ZColumnMap() -@inline use_only_active_surface_cells(::AbstractGrid) = nothing -@inline use_only_active_surface_cells(::ActiveZColumnsIBG) = ZColumnMap() +@inline active_interior_map(::Val{:west}) = WestMap() +@inline active_interior_map(::Val{:east}) = EastMap() +@inline active_interior_map(::Val{:south}) = SouthMap() +@inline active_interior_map(::Val{:north}) = NorthMap() -@inline use_only_active_interior_cells(::AbstractGrid) = nothing -@inline use_only_active_interior_cells(::ActiveCellsIBG) = InteriorMap() -@inline use_only_active_interior_cells(::DistributedActiveCellsIBG) = InteriorMap() +@inline active_interior_map(::AbstractGrid) = nothing +@inline active_interior_map(::ActiveCellsIBG) = InteriorMap() +@inline active_interior_map(::DistributedActiveCellsIBG) = InteriorMap() """ active_cells_work_layout(group, size, map_type, grid) @@ -65,7 +65,7 @@ Compute the work layout for active cells based on the given map type and grid. @inline active_cells_work_layout(group, size, ::EastMap, grid::NamedTupleActiveCellsIBG) = min(length(grid.interior_active_cells.east), 256), length(grid.interior_active_cells.east) @inline active_cells_work_layout(group, size, ::SouthMap, grid::NamedTupleActiveCellsIBG) = min(length(grid.interior_active_cells.south), 256), length(grid.interior_active_cells.south) @inline active_cells_work_layout(group, size, ::NorthMap, grid::NamedTupleActiveCellsIBG) = min(length(grid.interior_active_cells.north), 256), length(grid.interior_active_cells.north) -@inline active_cells_work_layout(group, size, ::ZColumnMap, grid::ActiveZColumnsIBG) = min(length(grid.active_z_columns), 256), length(grid.active_z_columns) +@inline active_cells_work_layout(group, size, ::ZColumnMap, grid::ActiveZColumnsIBG) = min(length(grid.active_z_columns), 256), length(grid.active_z_columns) """ active_linear_index_to_tuple(idx, map, grid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl index 767454abc1..9e2715dcff 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl @@ -8,7 +8,7 @@ using Oceananigans.Models.NonhydrostaticModels: boundary_tendency_kernel_paramet import Oceananigans.Models: compute_boundary_tendencies! -using Oceananigans.ImmersedBoundaries: active_map, DistributedActiveCellsIBG +using Oceananigans.ImmersedBoundaries: active_interior_map, DistributedActiveCellsIBG # We assume here that top/bottom BC are always synched (no partitioning in z) function compute_boundary_tendencies!(model::HydrostaticFreeSurfaceModel) @@ -38,8 +38,13 @@ function compute_boundary_tendency_contributions!(grid::DistributedActiveCellsIB maps = grid.interior_active_cells for (name, map) in zip(keys(maps), maps) - if name != :interior && !isnothing(map) - compute_hydrostatic_free_surface_tendency_contributions!(model, tuple(:xyz); only_active_cells = active_map(Val(name))) + compute_boundary = (name != :interior) && !isnothing(map) + + # If there exists a boundary map, then we compute the boundary contributions + if compute_boundary + active_boundary_map = active_interior_map(Val(name)) + compute_hydrostatic_free_surface_tendency_contributions!(model, tuple(:xyz); + active_cells_map = active_boundary_map) end end diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl index 898fc8bcab..dc330fa1fd 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl @@ -13,7 +13,7 @@ import Oceananigans: tracer_tendency_kernel_function import Oceananigans.Models: complete_communication_and_compute_boundary! import Oceananigans.Models: interior_tendency_kernel_parameters -using Oceananigans.ImmersedBoundaries: use_only_active_interior_cells, ActiveCellsIBG, +using Oceananigans.ImmersedBoundaries: active_interior_map, ActiveCellsIBG, InteriorMap, active_linear_index_to_tuple """ @@ -29,7 +29,7 @@ function compute_tendencies!(model::HydrostaticFreeSurfaceModel, callbacks) # Calculate contributions to momentum and tracer tendencies from fluxes and volume terms in the # interior of the domain compute_hydrostatic_free_surface_tendency_contributions!(model, kernel_parameters; - only_active_cells = use_only_active_interior_cells(model.grid)) + active_cells_map = active_interior_map(model.grid)) complete_communication_and_compute_boundary!(model, model.grid, model.architecture) @@ -80,12 +80,12 @@ end end """ Store previous value of the source term and compute current source term. """ -function compute_hydrostatic_free_surface_tendency_contributions!(model, kernel_parameters; only_active_cells = nothing) +function compute_hydrostatic_free_surface_tendency_contributions!(model, kernel_parameters; active_cells_map = nothing) arch = model.architecture grid = model.grid - compute_hydrostatic_momentum_tendencies!(model, model.velocities, kernel_parameters; only_active_cells) + compute_hydrostatic_momentum_tendencies!(model, model.velocities, kernel_parameters; active_cells_map) top_tracer_bcs = top_tracer_boundary_conditions(grid, model.tracers) @@ -121,9 +121,9 @@ function compute_hydrostatic_free_surface_tendency_contributions!(model, kernel_ tendency_kernel!, c_tendency, grid, - only_active_cells, + active_cells_map, args; - only_active_cells) + active_cells_map) end end @@ -160,7 +160,7 @@ function compute_free_surface_tendency!(grid, model, kernel_parameters) end """ Calculate momentum tendencies if momentum is not prescribed.""" -function compute_hydrostatic_momentum_tendencies!(model, velocities, kernel_parameters; only_active_cells = nothing) +function compute_hydrostatic_momentum_tendencies!(model, velocities, kernel_parameters; active_cells_map = nothing) grid = model.grid arch = architecture(grid) @@ -188,13 +188,13 @@ function compute_hydrostatic_momentum_tendencies!(model, velocities, kernel_para for parameters in kernel_parameters launch!(arch, grid, parameters, compute_hydrostatic_free_surface_Gu!, model.timestepper.Gⁿ.u, grid, - only_active_cells, u_kernel_args; - only_active_cells) + active_cells_map, u_kernel_args; + active_cells_map) launch!(arch, grid, parameters, compute_hydrostatic_free_surface_Gv!, model.timestepper.Gⁿ.v, grid, - only_active_cells, v_kernel_args; - only_active_cells) + active_cells_map, v_kernel_args; + active_cells_map) end compute_free_surface_tendency!(grid, model, :xy) diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl index 14d8841f67..717bb1fd2d 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl @@ -2,7 +2,7 @@ using Oceananigans.Fields: location using Oceananigans.TimeSteppers: ab2_step_field! using Oceananigans.TurbulenceClosures: implicit_step! -using Oceananigans.ImmersedBoundaries: use_only_active_interior_cells, use_only_active_surface_cells +using Oceananigans.ImmersedBoundaries: active_interior_map, active_surface_map import Oceananigans.TimeSteppers: ab2_step! diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 370528c4e3..ff557339c4 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -8,7 +8,7 @@ using Oceananigans.Operators using Oceananigans.Architectures: convert_args using Oceananigans.ImmersedBoundaries: peripheral_node, immersed_inactive_node, GFBIBG using Oceananigans.ImmersedBoundaries: inactive_node, IBG, c, f, ZColumnMap -using Oceananigans.ImmersedBoundaries: mask_immersed_field!, use_only_active_surface_cells, use_only_active_interior_cells +using Oceananigans.ImmersedBoundaries: mask_immersed_field!, active_surface_map, active_interior_map using Oceananigans.ImmersedBoundaries: active_linear_index_to_tuple, ActiveCellsIBG, ActiveZColumnsIBG using Oceananigans.DistributedComputations: child_architecture using Oceananigans.DistributedComputations: Distributed @@ -227,7 +227,7 @@ end compute_barotropic_mode!(U, V, grid, u, v) = launch!(architecture(grid), grid, :xy, _barotropic_mode_kernel!, U, V, grid, u, v; - only_active_cells = use_only_active_surface_cells(grid)) + active_cells_map = active_surface_map(grid)) function initialize_free_surface_state!(state, η, timestepper) @@ -471,7 +471,7 @@ end setup_split_explicit_tendency!(auxiliary, grid, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) = launch!(architecture(grid), grid, :xy, _compute_integrated_ab2_tendencies!, auxiliary.Gᵁ, auxiliary.Gⱽ, grid, - Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ; only_active_cells = use_only_active_surface_cells(grid)) + Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ; active_cells_map = active_surface_map(grid)) wait_free_surface_communication!(free_surface, arch) = nothing diff --git a/src/Models/HydrostaticFreeSurfaceModels/store_hydrostatic_free_surface_tendencies.jl b/src/Models/HydrostaticFreeSurfaceModels/store_hydrostatic_free_surface_tendencies.jl index fb27432e7a..fa5e391a42 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/store_hydrostatic_free_surface_tendencies.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/store_hydrostatic_free_surface_tendencies.jl @@ -4,7 +4,7 @@ using Oceananigans.TimeSteppers: store_field_tendencies! using Oceananigans: prognostic_fields using Oceananigans.Grids: AbstractGrid -using Oceananigans.ImmersedBoundaries: use_only_active_interior_cells +using Oceananigans.ImmersedBoundaries: active_interior_map using Oceananigans.Utils: launch! diff --git a/src/Models/NonhydrostaticModels/compute_nonhydrostatic_tendencies.jl b/src/Models/NonhydrostaticModels/compute_nonhydrostatic_tendencies.jl index 7eebd2a4fc..70bc3b922a 100644 --- a/src/Models/NonhydrostaticModels/compute_nonhydrostatic_tendencies.jl +++ b/src/Models/NonhydrostaticModels/compute_nonhydrostatic_tendencies.jl @@ -3,7 +3,7 @@ using Oceananigans: fields, TendencyCallsite using Oceananigans.Utils: work_layout using Oceananigans.Models: complete_communication_and_compute_boundary!, interior_tendency_kernel_parameters -using Oceananigans.ImmersedBoundaries: use_only_active_interior_cells, ActiveCellsIBG, +using Oceananigans.ImmersedBoundaries: active_interior_map, ActiveCellsIBG, InteriorMap, active_linear_index_to_tuple import Oceananigans.TimeSteppers: compute_tendencies! @@ -28,7 +28,7 @@ function compute_tendencies!(model::NonhydrostaticModel, callbacks) # interior of the domain kernel_parameters = tuple(interior_tendency_kernel_parameters(model.grid)) - compute_interior_tendency_contributions!(model, kernel_parameters; only_active_cells = use_only_active_interior_cells(model.grid)) + compute_interior_tendency_contributions!(model, kernel_parameters; active_cells_map = active_interior_map(model.grid)) complete_communication_and_compute_boundary!(model, model.grid, model.architecture) # Calculate contributions to momentum and tracer tendencies from user-prescribed fluxes across the @@ -50,7 +50,7 @@ function compute_tendencies!(model::NonhydrostaticModel, callbacks) end """ Store previous value of the source term and compute current source term. """ -function compute_interior_tendency_contributions!(model, kernel_parameters; only_active_cells = nothing) +function compute_interior_tendency_contributions!(model, kernel_parameters; active_cells_map = nothing) tendencies = model.timestepper.Gⁿ arch = model.architecture @@ -91,16 +91,16 @@ function compute_interior_tendency_contributions!(model, kernel_parameters; only for parameters in kernel_parameters launch!(arch, grid, parameters, compute_Gu!, - tendencies.u, grid, only_active_cells, u_kernel_args; - only_active_cells) + tendencies.u, grid, active_cells_map, u_kernel_args; + active_cells_map) launch!(arch, grid, parameters, compute_Gv!, - tendencies.v, grid, only_active_cells, v_kernel_args; - only_active_cells) + tendencies.v, grid, active_cells_map, v_kernel_args; + active_cells_map) launch!(arch, grid, parameters, compute_Gw!, - tendencies.w, grid, only_active_cells, w_kernel_args; - only_active_cells) + tendencies.w, grid, active_cells_map, w_kernel_args; + active_cells_map) end start_tracer_kernel_args = (advection, closure) @@ -120,8 +120,8 @@ function compute_interior_tendency_contributions!(model, kernel_parameters; only for parameters in kernel_parameters launch!(arch, grid, parameters, compute_Gc!, - c_tendency, grid, only_active_cells, args; - only_active_cells) + c_tendency, grid, active_cells_map, args; + active_cells_map) end end diff --git a/src/Utils/kernel_launching.jl b/src/Utils/kernel_launching.jl index 0802acd53b..b0d3338669 100644 --- a/src/Utils/kernel_launching.jl +++ b/src/Utils/kernel_launching.jl @@ -105,7 +105,7 @@ function work_layout(grid, workdims::Symbol; include_right_boundaries=false, loc return workgroup, worksize end -@inline active_cells_work_layout(workgroup, worksize, only_active_cells, grid) = workgroup, worksize +@inline active_cells_work_layout(workgroup, worksize, active_cells_map, grid) = workgroup, worksize """ launch!(arch, grid, layout, kernel!, args...; kwargs...) @@ -117,15 +117,14 @@ function launch!(arch, grid, workspec, kernel!, kernel_args...; include_right_boundaries = false, reduced_dimensions = (), location = nothing, - only_active_cells = nothing, + active_cells_map = nothing, kwargs...) - loop! = configured_kernel(arch, grid, workspec, kernel!; include_right_boundaries, reduced_dimensions, location, - only_active_cells, + active_cells_map, kwargs...) @@ -134,22 +133,48 @@ function launch!(arch, grid, workspec, kernel!, kernel_args...; return nothing end + +""" + configured_kernel(arch, grid, workspec, kernel!; include_right_boundaries=false, reduced_dimensions=(), location=nothing, active_cells_map=nothing, kwargs...) + +Configures a kernel with the specified architecture, grid, workspec, and kernel function. +If `active_cells_map` is provided, the work distribution is adjusted to include only the active cells. +If the worksize is 0 after adjusting for active cells, the function returns `nothing`. + +# Arguments +============ + +- `arch`: The architecture on which the kernel will be launched. +- `grid`: The grid on which the kernel will be executed. +- `workspec`: The workspec that defines the work distribution. +- `kernel!`: The kernel function to be executed. + +# Keyword Arguments +==================== + +- `include_right_boundaries`: A boolean indicating whether to include right boundaries `(N + 1)`. Default is `false`. +- `reduced_dimensions`: A tuple specifying the dimensions to be reduced in the work distribution. Default is an empty tuple. +- `location`: The location of the kernel execution, needed for `include_right_boundaries`. Default is `nothing`. +- `active_cells_map`: A map indicating the active cells in the grid. If the map is not a nothing, the workspec will be disregarded and + the kernel is configured as a linear kernel with a worksize equal to the length of the active cell map. Default is `nothing`. +""" + function configured_kernel(arch, grid, workspec, kernel!; include_right_boundaries = false, reduced_dimensions = (), location = nothing, - only_active_cells = nothing, + active_cells_map = nothing, kwargs...) workgroup, worksize = work_layout(grid, workspec; - include_right_boundaries, - reduced_dimensions, - location) + include_right_boundaries, + reduced_dimensions, + location) offset = offsets(workspec) - if !isnothing(only_active_cells) - workgroup, worksize = active_cells_work_layout(workgroup, worksize, only_active_cells, grid) + if !isnothing(active_cells_map) + workgroup, worksize = active_cells_work_layout(workgroup, worksize, active_cells_map, grid) offset = nothing # A non active domain! diff --git a/validation/distributed_simulations/distributed_hydrostatic_turbulence.jl b/validation/distributed_simulations/distributed_hydrostatic_turbulence.jl index 01b899e87c..1358bb59cf 100644 --- a/validation/distributed_simulations/distributed_hydrostatic_turbulence.jl +++ b/validation/distributed_simulations/distributed_hydrostatic_turbulence.jl @@ -7,7 +7,7 @@ using Oceananigans.BoundaryConditions using Oceananigans.DistributedComputations using Random using JLD2 -using Oceananigans.ImmersedBoundaries: ActiveCellsIBG, use_only_active_interior_cells +using Oceananigans.ImmersedBoundaries: ActiveCellsIBG, active_interior_map # Run with # From def685ae1881fb6aa7b4e2a9d0794c620487b8a5 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 27 Feb 2024 16:30:36 -0500 Subject: [PATCH 177/567] starting --- src/Advection/shared_tracer_advection.jl | 52 ++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 src/Advection/shared_tracer_advection.jl diff --git a/src/Advection/shared_tracer_advection.jl b/src/Advection/shared_tracer_advection.jl new file mode 100644 index 0000000000..286e8f6266 --- /dev/null +++ b/src/Advection/shared_tracer_advection.jl @@ -0,0 +1,52 @@ + + +function tracer_advection(Gⁿ, tracers, grid, tracer_advection, velocities; parameters = :xyz) + + return nothing +end + +@kernel function tracer_advection_x(Gⁿ, tracers, grid, tracer_advection, U) + i, j, k = @index(Global, NTuple) + + # Allocate shared memory of the dimensions of the grid + + for (c, advection) in zip(tracers, tracer_advection) + # calculate fluxes into the shared memory + + __syncthreads() + # put div in memory + + # exit + end +end + +@kernel function tracer_advection_y(Gⁿ, tracers, grid, tracer_advection, V) + i, j, k = @index(Global, NTuple) + + # Allocate shared memory of the dimensions of the grid + + + for (c, advection) in zip(tracers, tracer_advection) + # calculate fluxes into the shared memory + + __syncthreads() + # put div in memory + + # exit + end +end + +@kernel function tracer_advection_z(Gⁿ, tracers, grid::ActiveCellsIBG, tracer_advection, W) + i, j, k = @index(Global, NTuple) + + # Allocate shared memory of the dimensions of the grid + + for (c, advection) in zip(tracers, tracer_advection) + # calculate fluxes into the shared memory + + __syncthreads() + # put div in memory + + # exit + end +end \ No newline at end of file From 218e78b7aa1043a1d5cc9148a2fa6021f6a0059a Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 27 Feb 2024 16:53:51 -0500 Subject: [PATCH 178/567] add shared --- src/Advection/shared_tracer_advection.jl | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/Advection/shared_tracer_advection.jl b/src/Advection/shared_tracer_advection.jl index 286e8f6266..a32ff58099 100644 --- a/src/Advection/shared_tracer_advection.jl +++ b/src/Advection/shared_tracer_advection.jl @@ -1,4 +1,4 @@ - +using KernelAbstractions: @index, @kernel function tracer_advection(Gⁿ, tracers, grid, tracer_advection, velocities; parameters = :xyz) @@ -8,14 +8,20 @@ end @kernel function tracer_advection_x(Gⁿ, tracers, grid, tracer_advection, U) i, j, k = @index(Global, NTuple) - # Allocate shared memory of the dimensions of the grid + glo_id = @index(Global) + loc_id = @index(Local) + + tracer_flux = @localmem FT (N, grp_size) + + @unroll for tracer_name in tracer_names + advection = retrieve_x_advection(tracer_advection, tracer_name) - for (c, advection) in zip(tracers, tracer_advection) # calculate fluxes into the shared memory + tracer_flux[] = _advective_tracer_flux_x(i, j, k, grid, advection, U, c) - __syncthreads() + @synchronize # put div in memory - + Gⁿ[i, j, k] = (tracer_flux[t] - tracer_flux[t-1]) / Vᶜᶜᶜ(i, j, k, grid) # exit end end From 0ab0a1d6787de79ed28591b14bfbf95d4f7f7058 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 27 Feb 2024 17:28:27 -0500 Subject: [PATCH 179/567] going ahead --- src/Advection/shared_tracer_advection.jl | 32 ++++++++++++++++-------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/src/Advection/shared_tracer_advection.jl b/src/Advection/shared_tracer_advection.jl index a32ff58099..5722e55a80 100644 --- a/src/Advection/shared_tracer_advection.jl +++ b/src/Advection/shared_tracer_advection.jl @@ -5,24 +5,36 @@ function tracer_advection(Gⁿ, tracers, grid, tracer_advection, velocities; par return nothing end -@kernel function tracer_advection_x(Gⁿ, tracers, grid, tracer_advection, U) - i, j, k = @index(Global, NTuple) - - glo_id = @index(Global) - loc_id = @index(Local) +@inline retrieve_x_advection(tracer_advection) = tracer_advection +@inline retrieve_y_advection(tracer_advection) = tracer_advection +@inline retrieve_z_advection(tracer_advection) = tracer_advection + +@inline retrieve_x_advection(tracer_advection::TracerAdvection) = tracer_advection.x +@inline retrieve_y_advection(tracer_advection::TracerAdvection) = tracer_advection.y +@inline retrieve_z_advection(tracer_advection::TracerAdvection) = tracer_advection.z - tracer_flux = @localmem FT (N, grp_size) +@kernel function tracer_advection_x(Gⁿ, tracers, grid, tracer_advection, U, ::Val{Ntracers}) where Ntracers + global_id = @index(Global, Linear) + block_id = + thread_id = @index(Local) - @unroll for tracer_name in tracer_names - advection = retrieve_x_advection(tracer_advection, tracer_name) + k_id = + + tracer_flux = @localmem FT (, ) + + @unroll for t in 1:Ntracers + tracer_name = @inbounds tracer_names[t] + tracer = @inbounds tracers[t] + advection = @inbounds tracer_advection[tracer_name] + tendency = @inbounds Gⁿ[tracer_name] + advection = retrieve_x_advection(advection) # calculate fluxes into the shared memory tracer_flux[] = _advective_tracer_flux_x(i, j, k, grid, advection, U, c) @synchronize # put div in memory - Gⁿ[i, j, k] = (tracer_flux[t] - tracer_flux[t-1]) / Vᶜᶜᶜ(i, j, k, grid) - # exit + tendency[i, j, k] = (tracer_flux[t] - tracer_flux[t-1]) / Vᶜᶜᶜ(i, j, k, grid) end end From 10632ee266846f45f93f90620b7eda733a37eaf8 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 29 Feb 2024 13:02:15 -0500 Subject: [PATCH 180/567] bugfix --- .../split_explicit_free_surface_kernels.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 0cd64a4d04..dd57cd0130 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -274,7 +274,7 @@ function initialize_auxiliary_state!(state, η, timestepper) return nothing end -@kernel function barotropic_split_explicit_corrector_kernel!(u, v, grid, U̅, V̅, U, V, Hᶠᶜ, Hᶜᶠ, η̅) +@kernel function _barotropic_split_explicit_corrector!(u, v, grid, U̅, V̅, U, V, Hᶠᶜ, Hᶜᶠ, η̅) i, j, k = @index(Global, NTuple) k_top = grid.Nz + 1 From 3ddb456cc3559bf4c42a3e90d3c622f02322bf00 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 29 Feb 2024 14:45:19 -0500 Subject: [PATCH 181/567] better --- .../split_explicit_free_surface_kernels.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index dd57cd0130..5d5f5d5d89 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -279,8 +279,8 @@ end k_top = grid.Nz + 1 @inbounds begin - u[i, j, k] = u[i, j, k] + (U̅[i, j] - U[i, j]) / dynamic_column_heightᶠᶜ(i, j, k_top, grid, Hᶠᶜ, η̅) - v[i, j, k] = v[i, j, k] + (V̅[i, j] - V[i, j]) / dynamic_column_heightᶜᶠ(i, j, k_top, grid, Hᶜᶠ, η̅) + u[i, j, k] = u[i, j, k] + (U̅[i, j, ktop-1] - U[i, j, ktop-1]) / dynamic_column_heightᶠᶜ(i, j, k_top, grid, Hᶠᶜ, η̅) + v[i, j, k] = v[i, j, k] + (V̅[i, j, ktop-1] - V[i, j, ktop-1]) / dynamic_column_heightᶜᶠ(i, j, k_top, grid, Hᶜᶠ, η̅) end end From 8ae009140bbd57800b4861f98df3ac9d377658c6 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 29 Feb 2024 15:44:19 -0500 Subject: [PATCH 182/567] bugfix --- .../split_explicit_free_surface_kernels.jl | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 5d5f5d5d89..b43112c185 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -213,12 +213,12 @@ end @inbounds scalingᶜᶠᶜ = dynamic_column_heightᶜᶠ(i, j, k_top, grid, Hᶜᶠ, η̅) / Hᶜᶠ[i, j, 1] # hand unroll first loop - @inbounds U[i, j, 1] = u[i, j, 1] * Δzᶠᶜᶜ_reference(i, j, 1, grid) * scalingᶠᶜᶜ - @inbounds V[i, j, 1] = v[i, j, 1] * Δzᶜᶠᶜ_reference(i, j, 1, grid) * scalingᶜᶠᶜ + @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δzᶠᶜᶜ_reference(i, j, 1, grid) * scalingᶠᶜᶜ + @inbounds V[i, j, k_top-1] = v[i, j, 1] * Δzᶜᶠᶜ_reference(i, j, 1, grid) * scalingᶜᶠᶜ @unroll for k in 2:grid.Nz - @inbounds U[i, j, 1] += u[i, j, k] * Δzᶠᶜᶜ_reference(i, j, k, grid) * scalingᶠᶜᶜ - @inbounds V[i, j, 1] += v[i, j, k] * Δzᶜᶠᶜ_reference(i, j, k, grid) * scalingᶜᶠᶜ + @inbounds U[i, j, k_top-1] += u[i, j, k] * Δzᶠᶜᶜ_reference(i, j, k, grid) * scalingᶠᶜᶜ + @inbounds V[i, j, k_top-1] += v[i, j, k] * Δzᶜᶠᶜ_reference(i, j, k, grid) * scalingᶜᶠᶜ end end @@ -231,12 +231,12 @@ end @inbounds scalingᶜᶠᶜ = dynamic_column_heightᶜᶠᶜ(i, j, k_top, grid, Hᶜᶠ, η̅) / Hᶜᶠ[i, j, 1] # hand unroll first loop - @inbounds U[i, j, 1] = u[i, j, 1] * Δzᶠᶜᶜ_reference(i, j, 1, grid) * scalingᶠᶜᶜ - @inbounds V[i, j, 1] = v[i, j, 1] * Δzᶜᶠᶜ_reference(i, j, 1, grid) * scalingᶜᶠᶜ + @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δzᶠᶜᶜ_reference(i, j, 1, grid) * scalingᶠᶜᶜ + @inbounds V[i, j, k_top-1] = v[i, j, 1] * Δzᶜᶠᶜ_reference(i, j, 1, grid) * scalingᶜᶠᶜ @unroll for k in 2:grid.Nz - @inbounds U[i, j, 1] += u[i, j, k] * Δzᶠᶜᶜ_reference(i, j, k, grid) * scalingᶠᶜᶜ - @inbounds V[i, j, 1] += v[i, j, k] * Δzᶜᶠᶜ_reference(i, j, k, grid) * scalingᶜᶠᶜ + @inbounds U[i, j, k_top-1] += u[i, j, k] * Δzᶠᶜᶜ_reference(i, j, k, grid) * scalingᶠᶜᶜ + @inbounds V[i, j, k_top-1] += v[i, j, k] * Δzᶜᶠᶜ_reference(i, j, k, grid) * scalingᶜᶠᶜ end end @@ -279,8 +279,8 @@ end k_top = grid.Nz + 1 @inbounds begin - u[i, j, k] = u[i, j, k] + (U̅[i, j, ktop-1] - U[i, j, ktop-1]) / dynamic_column_heightᶠᶜ(i, j, k_top, grid, Hᶠᶜ, η̅) - v[i, j, k] = v[i, j, k] + (V̅[i, j, ktop-1] - V[i, j, ktop-1]) / dynamic_column_heightᶜᶠ(i, j, k_top, grid, Hᶜᶠ, η̅) + u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / dynamic_column_heightᶠᶜ(i, j, k_top, grid, Hᶠᶜ, η̅) + v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / dynamic_column_heightᶜᶠ(i, j, k_top, grid, Hᶜᶠ, η̅) end end From e2213c9a7bd6dbb61453536353142de4bd37f26f Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 29 Feb 2024 18:48:50 -0500 Subject: [PATCH 183/567] now it doesn't work anymore --- src/ImmersedBoundaries/ImmersedBoundaries.jl | 6 ++- src/ImmersedBoundaries/grid_fitted_bottom.jl | 2 + src/ImmersedBoundaries/mask_immersed_field.jl | 1 - ...distributed_split_explicit_free_surface.jl | 10 ++-- .../generalized_vertical_spacing.jl | 50 ++++++++++++++++--- .../split_explicit_free_surface.jl | 12 +++-- .../split_explicit_free_surface_kernels.jl | 18 ++++--- .../z_star_vertical_spacing.jl | 27 ++++++---- ...ulti_region_split_explicit_free_surface.jl | 10 ++-- .../distributed_hydrostatic_turbulence.jl | 2 +- validation/z_star_coordinate/lock_release.jl | 4 +- 11 files changed, 101 insertions(+), 41 deletions(-) diff --git a/src/ImmersedBoundaries/ImmersedBoundaries.jl b/src/ImmersedBoundaries/ImmersedBoundaries.jl index f13bdabb4b..1d5a7c042d 100644 --- a/src/ImmersedBoundaries/ImmersedBoundaries.jl +++ b/src/ImmersedBoundaries/ImmersedBoundaries.jl @@ -147,8 +147,10 @@ Adapt.adapt_structure(to, ibg::IBG{FT, TX, TY, TZ}) where {FT, TX, TY, TZ} = adapt(to, ibg.interior_active_cells), adapt(to, ibg.active_z_columns)) -with_halo(halo, ibg::ImmersedBoundaryGrid) = - ImmersedBoundaryGrid(with_halo(halo, ibg.underlying_grid), ibg.immersed_boundary) +function with_halo(halo, ibg::ImmersedBoundaryGrid) + new_grid = with_halo(halo, ibg.underlying_grid) + ImmersedBoundaryGrid(new_grid, ibg.immersed_boundary) +end # ImmersedBoundaryGrids require an extra halo point to check the "inactivity" of a `Face` node at N + H # (which requires checking `Center` nodes at N + H and N + H + 1) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index e693e20e2d..05a9bc85a8 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -71,6 +71,8 @@ Computes ib.bottom_height and wraps in an array. """ function ImmersedBoundaryGrid(grid, ib::GridFittedBottom) bottom_field = Field{Center, Center, Nothing}(grid) + + @show ib.bottom_height, bottom_field, architecture(grid) set!(bottom_field, ib.bottom_height) fill_halo_regions!(bottom_field) new_ib = GridFittedBottom(bottom_field, ib.immersed_condition) diff --git a/src/ImmersedBoundaries/mask_immersed_field.jl b/src/ImmersedBoundaries/mask_immersed_field.jl index 4bd2878eec..9a4252a823 100644 --- a/src/ImmersedBoundaries/mask_immersed_field.jl +++ b/src/ImmersedBoundaries/mask_immersed_field.jl @@ -22,7 +22,6 @@ function mask_immersed_field!(field::Field, grid::ImmersedBoundaryGrid, loc, val return nothing end - @kernel function _mask_immersed_field!(field, loc, grid, value) i, j, k = @index(Global, NTuple) @inbounds field[i, j, k] = scalar_mask(i, j, k, grid, grid.immersed_boundary, loc..., value, field) diff --git a/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl index 4f14f69f2a..bcaeea7a32 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl @@ -12,11 +12,13 @@ function SplitExplicitAuxiliaryFields(grid::DistributedGrid) Hᶠᶜ = Field((Face, Center, Nothing), grid) Hᶜᶠ = Field((Center, Face, Nothing), grid) + Hᶜᶠ = Field((Center, Center, Nothing), grid) - calculate_column_height!(Hᶠᶜ, (Face, Center, Center)) - calculate_column_height!(Hᶜᶠ, (Center, Face, Center)) + calculate_column_height!(Hᶠᶜ, (Face, Center, Center)) + calculate_column_height!(Hᶜᶠ, (Center, Face, Center)) + calculate_column_height!(Hᶜᶜ, (Center, Center, Center)) - fill_halo_regions!((Hᶠᶜ, Hᶜᶠ)) + fill_halo_regions!((Hᶠᶜ, Hᶜᶠ, Hᶜᶜ)) # In a non-parallel grid we calculate only the interior kernel_size = augmented_kernel_size(grid) @@ -24,7 +26,7 @@ function SplitExplicitAuxiliaryFields(grid::DistributedGrid) kernel_parameters = KernelParameters(kernel_size, kernel_offsets) - return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, kernel_parameters) + return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, kernel_parameters) end """Integrate z at locations `location` and set! `height`` with the result""" diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index bf6ea63c85..f6c645cc0c 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -8,6 +8,7 @@ using Oceananigans.Utils: getnamewrapper using Adapt using Printf +import Oceananigans.Grids: with_halo import Oceananigans.Operators: Δzᶜᶜᶠ, Δzᶜᶜᶜ, Δzᶜᶠᶠ, Δzᶜᶠᶜ, Δzᶠᶜᶠ, Δzᶠᶜᶜ, Δzᶠᶠᶠ, Δzᶠᶠᶜ import Oceananigans.Advection: ∂t_∂s_grid import Oceananigans.Architectures: arch_array @@ -54,6 +55,48 @@ const GeneralizedSpacingImmersedGrid{D} = ImmersedBoundaryGrid{<:Any, <:Any, <:A const GeneralizedSpacingGrid{D} = Union{GeneralizedSpacingUnderlyingGrid{D}, GeneralizedSpacingImmersedGrid{D}} where D +denomination(grid::GeneralizedSpacingGrid) = grid.Δzᵃᵃᶠ.denomination + +##### +##### Original grid +##### + +retrieve_static_grid(grid) = grid + +function retrieve_static_grid(grid::GeneralizedSpacingGrid) + + Δzᵃᵃᶠ = grid.Δzᵃᵃᶠ.Δr + Δzᵃᵃᶜ = grid.Δzᵃᵃᶜ.Δr + + args = [] + for prop in propertynames(grid) + if prop == :Δzᵃᵃᶠ + push!(args, Δzᵃᵃᶠ) + elseif prop == :Δzᵃᵃᶜ + push!(args, Δzᵃᵃᶜ) + else + push!(args, getproperty(grid, prop)) + end + end + + GridType = getnamewrapper(grid) + + return GridType{TX, TY, TZ}(args...) +end + +##### +##### Some extensions +##### + +function with_halo(new_halo, grid::GeneralizedSpacingGrid) + old_static_grid = retrieve_static_grid(grid) + new_static_grid = with_halo(new_halo, old_static_grid) + vertical_coordinate = denomination(grid) + new_grid = GeneralizedSpacingGrid(new_static_grid, vertical_coordinate) + + return new_grid +end + ##### ##### General implementation ##### @@ -99,13 +142,6 @@ update_vertical_spacing!(model, grid, Δt; kwargs...) = nothing @inline Δzᶠᶠᶠ_reference(i, j, k, grid) = ℑxyᶠᶠᵃ(i, j, k, grid, Δzᶜᶜᶠ_reference) @inline Δzᶠᶠᶜ_reference(i, j, k, grid) = ℑxyᶠᶠᵃ(i, j, k, grid, Δzᶜᶜᶜ_reference) -##### -##### Utility -##### - -bottom_height(i, j, grid) = grid.Lz -bottom_height(i, j, grid::ImmersedBoundaryGrid) = @inbounds - grid.immersed_boundary.bottom_height[i, j, 1] - ##### ##### Vertical velocity of the Δ-surfaces to be included in the continuity equation ##### diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index d32b5786d8..a76a257089 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -178,7 +178,7 @@ large (or `:xy` in case of a serial computation), and start computing from $(FIELDS) """ -Base.@kwdef struct SplitExplicitAuxiliaryFields{𝒞ℱ, ℱ𝒞, 𝒦} +Base.@kwdef struct SplitExplicitAuxiliaryFields{𝒞ℱ, ℱ𝒞, 𝒞𝒞, 𝒦} "Vertically-integrated slow barotropic forcing function for `U` (`ReducedField` over ``z``)" Gᵁ :: ℱ𝒞 "Vertically-integrated slow barotropic forcing function for `V` (`ReducedField` over ``z``)" @@ -187,6 +187,8 @@ Base.@kwdef struct SplitExplicitAuxiliaryFields{𝒞ℱ, ℱ𝒞, 𝒦} Hᶠᶜ :: ℱ𝒞 "Depth at `(Center, Face)` (`ReducedField` over ``z``)" Hᶜᶠ :: 𝒞ℱ + "Depth at `(Center, Center)` (`ReducedField` over ``z``)" + Hᶜᶜ :: 𝒞𝒞 "kernel size for barotropic time stepping" kernel_parameters :: 𝒦 end @@ -203,6 +205,7 @@ function SplitExplicitAuxiliaryFields(grid::AbstractGrid) Hᶠᶜ = Field((Face, Center, Nothing), grid) Hᶜᶠ = Field((Center, Face, Nothing), grid) + Hᶜᶜ = Field((Center, Center, Nothing), grid) dz = GridMetricOperation((Face, Center, Center), Δz, grid) sum!(Hᶠᶜ, dz) @@ -210,11 +213,14 @@ function SplitExplicitAuxiliaryFields(grid::AbstractGrid) dz = GridMetricOperation((Center, Face, Center), Δz, grid) sum!(Hᶜᶠ, dz) - fill_halo_regions!((Hᶠᶜ, Hᶜᶠ)) + dz = GridMetricOperation((Center, Face, Center), Δz, grid) + sum!(Hᶜᶜ, dz) + + fill_halo_regions!((Hᶠᶜ, Hᶜᶠ, Hᶜᶜ)) kernel_parameters = :xy - return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, kernel_parameters) + return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, kernel_parameters) end """ diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index b43112c185..e2c844ef6e 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -502,15 +502,17 @@ wait_free_surface_communication!(free_surface, arch) = nothing # Special update ∂t_∂s for SplitExplicitFreeSurface where # ∂(η / H)/∂t = - ∇ ⋅ U̅ / H -update_∂t_∂s!(∂t_∂s, parameters, grid, sⁿ, s⁻, Δt, fs::SplitExplicitFreeSurface) = - launch!(architecture(grid), grid, parameters, _update_∂t_∂s_split_explicit!, ∂t_∂s, fs.state.U̅, fs.state.V̅, grid) +function update_∂t_∂s!(∂t_∂s, parameters, grid, sⁿ, s⁻, Δt, fs::SplitExplicitFreeSurface) + launch!(architecture(grid), grid, parameters, _update_∂t_∂s_split_explicit!, ∂t_∂s, fs.state.U̅, fs.state.V̅, fs.auxiliary.Hᶜᶜ, grid) + return nothing +end -@kernel function _update_∂t_∂s_split_explicit!(∂t_∂s, U̅, V̅, grid) - i, j = @index(Global, NTuple) - bottom = bottom_height(i, j, grid) - @inbounds begin +@kernel function _update_∂t_∂s_split_explicit!(∂t_∂s, U̅, V̅, Hᶜᶜ, grid) + i, j = @index(Global, NTuple) + k_top = grid.Nz + 1 + @inbounds begin # ∂(η / H)/∂t = - ∇ ⋅ ∫udz / H - ∂t_∂s[i, j, 1] = - div_xyᶜᶜᶜ(i, j, 1, grid, U̅, V̅) / bottom + ∂t_∂s[i, j, 1] = - div_xyᶜᶜᶜ(i, j, k_top - 1, grid, U̅, V̅) / Hᶜᶜ[i, j, 1] end -end +end \ No newline at end of file diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 385e303b1f..35e6119f5c 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -32,10 +32,11 @@ function GeneralizedSpacingGrid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, :: ∂t_∂s = Field{Center, Center, Nothing}(grid) # Initial "at-rest" conditions - launch!(architecture(grid), grid, :xy, _update_zstar!, - sⁿ, s⁻, ΔzF, ΔzC, ZeroField(), grid, Val(grid.Nz)) + fill!(s⁻, 1) + fill!(sⁿ, 1) + + launch!(architecture(grid), grid, :xyz, _initialize_zstar!, ΔzF, ΔzC, grid) - fill_halo_regions!((s⁻, sⁿ)) fill_halo_regions!((ΔzF, ΔzC)) Δzᵃᵃᶠ = GeneralizedVerticalSpacing(ZStar(), grid.Δzᵃᵃᶠ, ΔzF, s⁻, sⁿ, ∂t_∂s) @@ -57,6 +58,12 @@ function GeneralizedSpacingGrid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, :: return GridType{TX, TY, TZ}(args...) end +@kernel function _initialize_zstar!(ΔzF, ΔzC, grid) + i, j, k = @index(Global, NTuple) + @inbounds ΔzF[i, j, k] = grid.Δzᵃᵃᶠ[k] + @inbounds ΔzC[i, j, k] = grid.Δzᵃᵃᶜ[k] +end + ##### ##### ZStar-specific vertical spacings update ##### @@ -72,10 +79,12 @@ function update_vertical_spacing!(model, grid::ZStarSpacingGrid, Δt; parameters ΔzF = grid.Δzᵃᵃᶠ.Δ ΔzC = grid.Δzᵃᵃᶜ.Δ + Hᶜᶜ = model.free_surface.auxiliary.Hᶜᶜ + # Update vertical spacing with available parameters # No need to fill the halo as the scaling is updated _IN_ the halos launch!(architecture(grid), grid, parameters, _update_zstar!, sⁿ, s⁻, ΔzF, ΔzC, - model.free_surface.η, grid, Val(grid.Nz)) + model.free_surface.η, Hᶜᶜ, grid, Val(grid.Nz)) # Update scaling time derivative update_∂t_∂s!(∂t_∂s, parameters, grid, sⁿ, s⁻, Δt, model.free_surface) @@ -83,10 +92,10 @@ function update_vertical_spacing!(model, grid::ZStarSpacingGrid, Δt; parameters return nothing end -@kernel function _update_zstar!(sⁿ, s⁻, ΔzF, ΔzC, η, grid, ::Val{Nz}) where Nz +@kernel function _update_zstar!(sⁿ, s⁻, ΔzF, ΔzC, η, Hᶜᶜ, grid, ::Val{Nz}) where Nz i, j = @index(Global, NTuple) - bottom = bottom_height(i, j, grid) @inbounds begin + bottom = Hᶜᶜ[i, j, 1] h = (bottom + η[i, j, grid.Nz+1]) / bottom # update current and previous scaling @@ -112,10 +121,10 @@ end ##### ZStar-specific implementation of the additional terms to be included in the momentum equations ##### -@inline η_surfaceᶜᶜᶜ(i, j, k, grid, η) = @inbounds η[i, j, grid.Nz+1] * (1 + znode(i, j, k, grid, c, c, c) / bottom_height(i, j, grid)) +@inline η_surfaceᶜᶜᶜ(i, j, k, grid, η, Hᶜᶜ) = @inbounds η[i, j, grid.Nz+1] * (1 + znode(i, j, k, grid, c, c, c) / Hᶜᶜ[i, j, 1]) -@inline slope_xᶠᶜᶜ(i, j, k, grid, free_surface) = @inbounds ∂xᶠᶜᶜ(i, j, k, grid, η_surfaceᶜᶜᶜ, free_surface.η) -@inline slope_yᶜᶠᶜ(i, j, k, grid, free_surface) = @inbounds ∂yᶜᶠᶜ(i, j, k, grid, η_surfaceᶜᶜᶜ, free_surface.η) +@inline slope_xᶠᶜᶜ(i, j, k, grid, free_surface) = @inbounds ∂xᶠᶜᶜ(i, j, k, grid, η_surfaceᶜᶜᶜ, free_surface.η, free_surface.auxiliary.Hᶜᶜ) +@inline slope_yᶜᶠᶜ(i, j, k, grid, free_surface) = @inbounds ∂yᶜᶠᶜ(i, j, k, grid, η_surfaceᶜᶜᶜ, free_surface.η, free_surface.auxiliary.Hᶜᶜ) @inline grid_slope_contribution_x(i, j, k, grid::ZStarSpacingGrid, free_surface, ::Nothing, model_fields) = zero(grid) @inline grid_slope_contribution_y(i, j, k, grid::ZStarSpacingGrid, free_surface, ::Nothing, model_fields) = zero(grid) diff --git a/src/MultiRegion/multi_region_split_explicit_free_surface.jl b/src/MultiRegion/multi_region_split_explicit_free_surface.jl index d33f6bd1a6..4eed133830 100644 --- a/src/MultiRegion/multi_region_split_explicit_free_surface.jl +++ b/src/MultiRegion/multi_region_split_explicit_free_surface.jl @@ -11,11 +11,13 @@ function SplitExplicitAuxiliaryFields(grid::MultiRegionGrids) Hᶠᶜ = Field((Face, Center, Nothing), grid) Hᶜᶠ = Field((Center, Face, Nothing), grid) + Hᶜᶠ = Field((Center, Center, Nothing), grid) - @apply_regionally calculate_column_height!(Hᶠᶜ, (Face, Center, Center)) - @apply_regionally calculate_column_height!(Hᶜᶠ, (Center, Face, Center)) + @apply_regionally calculate_column_height!(Hᶠᶜ, (Face, Center, Center)) + @apply_regionally calculate_column_height!(Hᶜᶠ, (Center, Face, Center)) + @apply_regionally calculate_column_height!(Hᶜᶜ, (Center, Center, Center)) - fill_halo_regions!((Hᶠᶜ, Hᶜᶠ)) + fill_halo_regions!((Hᶠᶜ, Hᶜᶠ, Hᶜᶜ)) # In a non-parallel grid we calculate only the interior @apply_regionally kernel_size = augmented_kernel_size(grid, grid.partition) @@ -23,7 +25,7 @@ function SplitExplicitAuxiliaryFields(grid::MultiRegionGrids) @apply_regionally kernel_parameters = KernelParameters(kernel_size, kernel_offsets) - return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, kernel_parameters) + return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, kernel_parameters) end @inline function calculate_column_height!(height, location) diff --git a/validation/distributed_simulations/distributed_hydrostatic_turbulence.jl b/validation/distributed_simulations/distributed_hydrostatic_turbulence.jl index 1358bb59cf..ae0f7d4aa1 100644 --- a/validation/distributed_simulations/distributed_hydrostatic_turbulence.jl +++ b/validation/distributed_simulations/distributed_hydrostatic_turbulence.jl @@ -66,7 +66,7 @@ end Nx = 32 Ny = 32 -arch = Distributed(CPU(), partition = Partition(2, 2)) +arch = Distributed(CPU(), partition = Partition(2)) # Run the simulation run_simulation(Nx, Ny, arch) diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index 40499467f0..af79891a2d 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -13,7 +13,7 @@ grid = RectilinearGrid(size = (300, 20), model = HydrostaticFreeSurfaceModel(; grid, generalized_vertical_coordinate = ZStar(), - momentum_advection = WENOVectorInvariant(), + momentum_advection = WENO(), tracer_advection = WENO(), buoyancy = BuoyancyTracer(), tracers = :b, @@ -67,7 +67,7 @@ function progress(sim) return nothing end -simulation.callbacks[:progress] = Callback(progress, IterationInterval(100)) +simulation.callbacks[:progress] = Callback(progress, IterationInterval(1)) simulation.callbacks[:wizard] = Callback(TimeStepWizard(; cfl = 0.2, max_change = 1.1), IterationInterval(10)) run!(simulation) From a1e350851de4f2a31636214fb728461ddc78309a Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 29 Feb 2024 19:13:02 -0500 Subject: [PATCH 184/567] first commit --- test/test_distributed_hydrostatic_model.jl | 94 ++++++++++++---------- 1 file changed, 50 insertions(+), 44 deletions(-) diff --git a/test/test_distributed_hydrostatic_model.jl b/test/test_distributed_hydrostatic_model.jl index 8511a4c8ad..6dd53942e0 100644 --- a/test/test_distributed_hydrostatic_model.jl +++ b/test/test_distributed_hydrostatic_model.jl @@ -78,49 +78,55 @@ Ny = 32 for arch in archs @testset "Testing distributed solid body rotation" begin - grid = LatitudeLongitudeGrid(arch, size = (Nx, Ny, 1), - halo = (3, 3, 3), - latitude = (-80, 80), - longitude = (-160, 160), - z = (-1, 0), - radius = 1, - topology=(Bounded, Bounded, Bounded)) - - global_grid = reconstruct_global_grid(grid) - - # "s" for "serial" computation - us, vs, ws, cs, ηs = solid_body_rotation_test(global_grid) - - us = Array(interior(us)) - vs = Array(interior(vs)) - ws = Array(interior(ws)) - cs = Array(interior(cs)) - ηs = Array(interior(ηs)) - - @info " Testing distributed solid body rotation with architecture $arch" - u, v, w, c, η = solid_body_rotation_test(grid) - - u = Array(interior(u)) - v = Array(interior(v)) - w = Array(interior(w)) - c = Array(interior(c)) - η = Array(interior(η)) - - cpu_arch = cpu_architecture(arch) - - us = partition_global_array(cpu_arch, us, size(u)) - vs = partition_global_array(cpu_arch, vs, size(v)) - ws = partition_global_array(cpu_arch, ws, size(w)) - cs = partition_global_array(cpu_arch, cs, size(c)) - ηs = partition_global_array(cpu_arch, ηs, size(η)) - - atol = eps(eltype(grid)) - rtol = sqrt(eps(eltype(grid))) - - @test all(isapprox(u, us; atol, rtol)) - @test all(isapprox(v, vs; atol, rtol)) - @test all(isapprox(w, ws; atol, rtol)) - @test all(isapprox(c, cs; atol, rtol)) - @test all(isapprox(η, ηs; atol, rtol)) + underlying_grid = LatitudeLongitudeGrid(arch, size = (Nx, Ny, 1), + halo = (3, 3, 3), + latitude = (-80, 80), + longitude = (-160, 160), + z = (-1, 0), + radius = 1, + topology=(Bounded, Bounded, Bounded)) + + bottom(λ, φ) = -30 < λ < 30 && -40 < φ < 20 ? 0 : - 1 + + immersed_grid = ImmersedBoundaryGrid(underlying_grid, GridFittedBottom(bottom); active_cells_map = true) + + for grid in (underlying_grid, immersed_grid) + global_grid = reconstruct_global_grid(grid) + + # "s" for "serial" computation + us, vs, ws, cs, ηs = solid_body_rotation_test(global_grid) + + us = Array(interior(us)) + vs = Array(interior(vs)) + ws = Array(interior(ws)) + cs = Array(interior(cs)) + ηs = Array(interior(ηs)) + + @info " Testing distributed solid body rotation with architecture $arch" + u, v, w, c, η = solid_body_rotation_test(grid) + + u = Array(interior(u)) + v = Array(interior(v)) + w = Array(interior(w)) + c = Array(interior(c)) + η = Array(interior(η)) + + cpu_arch = cpu_architecture(arch) + + us = partition_global_array(cpu_arch, us, size(u)) + vs = partition_global_array(cpu_arch, vs, size(v)) + ws = partition_global_array(cpu_arch, ws, size(w)) + cs = partition_global_array(cpu_arch, cs, size(c)) + ηs = partition_global_array(cpu_arch, ηs, size(η)) + + atol = eps(eltype(grid)) + rtol = sqrt(eps(eltype(grid))) + + @test all(isapprox(u, us; atol, rtol)) + @test all(isapprox(v, vs; atol, rtol)) + @test all(isapprox(w, ws; atol, rtol)) + @test all(isapprox(c, cs; atol, rtol)) + @test all(isapprox(η, ηs; atol, rtol)) + end end end From 0602848d70d2c4fd65419b6a5633671032412ff9 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 29 Feb 2024 19:15:18 -0500 Subject: [PATCH 185/567] change the info message --- test/test_distributed_hydrostatic_model.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_distributed_hydrostatic_model.jl b/test/test_distributed_hydrostatic_model.jl index 6dd53942e0..cce7805c1d 100644 --- a/test/test_distributed_hydrostatic_model.jl +++ b/test/test_distributed_hydrostatic_model.jl @@ -102,7 +102,7 @@ for arch in archs cs = Array(interior(cs)) ηs = Array(interior(ηs)) - @info " Testing distributed solid body rotation with architecture $arch" + @info " Testing distributed solid body rotation with architecture $arch on $(typeof(grid).name.wrapper)" u, v, w, c, η = solid_body_rotation_test(grid) u = Array(interior(u)) From ce3820d74bbe229fed20a9fdc8acdf7d38c948d3 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 29 Feb 2024 19:36:00 -0500 Subject: [PATCH 186/567] first commit --- src/AbstractOperations/AbstractOperations.jl | 2 +- src/AbstractOperations/binary_operations.jl | 9 ++++ .../conditional_operations.jl | 11 ++++- src/AbstractOperations/derivatives.jl | 9 ++++ src/AbstractOperations/grid_metrics.jl | 5 +++ .../kernel_function_operation.jl | 5 +++ src/AbstractOperations/multiary_operations.jl | 7 ++++ src/AbstractOperations/unary_operations.jl | 6 +++ src/Advection/Advection.jl | 3 +- src/Advection/centered_reconstruction.jl | 6 +++ src/Advection/reconstruction_coefficients.jl | 8 ++-- src/Advection/stretched_weno_smoothness.jl | 4 +- src/Advection/upwind_biased_reconstruction.jl | 7 ++++ src/Advection/vector_invariant_advection.jl | 8 ++++ src/Advection/vector_invariant_upwinding.jl | 1 - src/Advection/weno_reconstruction.jl | 8 ++++ src/Architectures.jl | 42 +++++++++---------- src/BoundaryConditions/boundary_condition.jl | 6 +++ .../continuous_boundary_function.jl | 7 ++++ .../discrete_boundary_function.jl | 3 ++ .../distributed_architectures.jl | 4 +- .../distributed_fields.jl | 2 +- .../partition_assemble.jl | 10 ++--- src/Fields/Fields.jl | 2 +- src/Fields/field.jl | 14 +++++++ src/Fields/field_boundary_buffers.jl | 34 ++++++++++----- src/Fields/function_field.jl | 7 ++++ src/Fields/regridding_fields.jl | 2 +- src/Fields/set!.jl | 4 +- src/Forcings/advective_forcing.jl | 6 +++ src/Forcings/continuous_forcing.jl | 9 ++++ src/Forcings/discrete_forcing.jl | 4 ++ src/Forcings/multiple_forcings.jl | 1 + src/Grids/grid_generation.jl | 12 ++---- src/Grids/grid_utils.jl | 4 +- src/Grids/latitude_longitude_grid.jl | 8 ++-- src/Grids/orthogonal_spherical_shell_grid.jl | 12 +++--- src/Grids/rectilinear_grid.jl | 2 +- src/ImmersedBoundaries/active_cells_map.jl | 8 ++-- .../distributed_immersed_boundaries.jl | 2 +- src/ImmersedBoundaries/grid_fitted_bottom.jl | 2 +- src/ImmersedBoundaries/immersed_reductions.jl | 2 +- src/ImmersedBoundaries/partial_cell_bottom.jl | 2 +- .../matrix_implicit_free_surface_solver.jl | 12 +++--- .../multi_region_boundary_conditions.jl | 2 +- src/MultiRegion/multi_region_grid.jl | 4 +- src/MultiRegion/x_partitions.jl | 8 ++-- src/MultiRegion/y_partitions.jl | 8 ++-- src/OutputReaders/field_time_series.jl | 4 +- .../field_time_series_indexing.jl | 2 +- src/Solvers/Solvers.jl | 2 +- src/Solvers/batched_tridiagonal_solver.jl | 6 +-- src/Solvers/discrete_transforms.jl | 4 +- src/Solvers/fft_based_poisson_solver.jl | 8 ++-- .../fourier_tridiagonal_poisson_solver.jl | 12 +++--- src/Solvers/heptadiagonal_iterative_solver.jl | 18 ++++---- src/Solvers/sparse_preconditioners.jl | 6 +-- .../implicit_explicit_time_discretization.jl | 2 +- ...vective_adjustment_vertical_diffusivity.jl | 2 +- .../isopycnal_skew_symmetric_diffusivity.jl | 2 +- .../ri_based_vertical_diffusivity.jl | 2 +- test/test_abstract_operations.jl | 8 ++-- test/test_broadcasting.jl | 2 +- test/test_field.jl | 18 ++++---- test/test_field_reductions.jl | 2 +- ...static_free_surface_immersed_boundaries.jl | 2 +- ...face_immersed_boundaries_implicit_solve.jl | 4 +- test/test_lagrangian_particle_tracking.jl | 24 +++++------ test/test_matrix_poisson_solver.jl | 18 ++++---- test/test_multi_region_poisson_solver.jl | 4 +- test/test_sewater_density.jl | 2 +- test/test_shallow_water_models.jl | 2 +- validation/barotropic_gyre/barotropic_gyre.jl | 2 +- ...multi_region_near_global_quarter_degree.jl | 4 +- ...ear_global_shallow_water_quarter_degree.jl | 6 +-- 75 files changed, 327 insertions(+), 185 deletions(-) diff --git a/src/AbstractOperations/AbstractOperations.jl b/src/AbstractOperations/AbstractOperations.jl index f662d98899..9656716307 100644 --- a/src/AbstractOperations/AbstractOperations.jl +++ b/src/AbstractOperations/AbstractOperations.jl @@ -22,7 +22,7 @@ using Oceananigans.Operators: interpolation_operator using Oceananigans.Architectures: device using Oceananigans: AbstractModel -import Oceananigans.Architectures: architecture +import Oceananigans.Architectures: architecture, on_architecture import Oceananigans.BoundaryConditions: fill_halo_regions! import Oceananigans.Fields: compute_at!, indices diff --git a/src/AbstractOperations/binary_operations.jl b/src/AbstractOperations/binary_operations.jl index 2414a1c19c..31a0a80245 100644 --- a/src/AbstractOperations/binary_operations.jl +++ b/src/AbstractOperations/binary_operations.jl @@ -216,3 +216,12 @@ Adapt.adapt_structure(to, binary::BinaryOperation{LX, LY, LZ}) where {LX, LY, LZ Adapt.adapt(to, binary.▶a), Adapt.adapt(to, binary.▶b), Adapt.adapt(to, binary.grid)) + + +on_architecture(to, binary::BinaryOperation{LX, LY, LZ}) where {LX, LY, LZ} = + BinaryOperation{LX, LY, LZ}(on_architecture(to, binary.op), + on_architecture(to, binary.a), + on_architecture(to, binary.b), + on_architecture(to, binary.▶a), + on_architecture(to, binary.▶b), + on_architecture(to, binary.grid)) diff --git a/src/AbstractOperations/conditional_operations.jl b/src/AbstractOperations/conditional_operations.jl index cf183c7f29..e621bcd262 100644 --- a/src/AbstractOperations/conditional_operations.jl +++ b/src/AbstractOperations/conditional_operations.jl @@ -1,6 +1,6 @@ using Oceananigans.Fields: OneField using Oceananigans.Grids: architecture -using Oceananigans.Architectures: arch_array +using Oceananigans.Architectures: on_architecture import Oceananigans.Fields: condition_operand, conditional_length, set!, compute_at!, indices # For conditional reductions such as mean(u * v, condition = u .> 0)) @@ -106,7 +106,7 @@ end @inline condition_operand(func::Function, op::AbstractField, ::Nothing, mask) = ConditionalOperation(op; func, condition=TrueCondition(), mask) @inline function condition_operand(func::Function, operand::AbstractField, condition::AbstractArray, mask) - condition = arch_array(architecture(operand.grid), condition) + condition = on_architecture(architecture(operand.grid), condition) return ConditionalOperation(operand; func, condition, mask) end @@ -134,6 +134,13 @@ Adapt.adapt_structure(to, c::ConditionalOperation{LX, LY, LZ}) where {LX, LY, LZ adapt(to, c.condition), adapt(to, c.mask)) +on_architecture(to, c::ConditionalOperation{LX, LY, LZ}) where {LX, LY, LZ} = + ConditionalOperation{LX, LY, LZ}(on_architecture(to, c.operand), + on_architecture(to, c.func), + on_architecture(to, c.grid), + on_architecture(to, c.condition), + on_architecture(to, c.mask)) + Base.summary(c::ConditionalOperation) = string("ConditionalOperation of ", summary(c.operand), " with condition ", summary(c.condition)) compute_at!(c::ConditionalOperation, time) = compute_at!(c.operand, time) diff --git a/src/AbstractOperations/derivatives.jl b/src/AbstractOperations/derivatives.jl index 344ecd47de..751e794b94 100644 --- a/src/AbstractOperations/derivatives.jl +++ b/src/AbstractOperations/derivatives.jl @@ -125,3 +125,12 @@ Adapt.adapt_structure(to, deriv::Derivative{LX, LY, LZ}) where {LX, LY, LZ} = Adapt.adapt(to, deriv.▶), nothing, Adapt.adapt(to, deriv.grid)) + +"Adapt `Derivative` to work on the GPU via CUDAnative and CUDAdrv." +on_architecture(to, deriv::Derivative{LX, LY, LZ}) where {LX, LY, LZ} = + Derivative{LX, LY, LZ}(on_architecture(to, deriv.∂), + on_architecture(to, deriv.arg), + on_architecture(to, deriv.▶), + deriv.abstract_∂, + on_architecture(to, deriv.grid)) + \ No newline at end of file diff --git a/src/AbstractOperations/grid_metrics.jl b/src/AbstractOperations/grid_metrics.jl index a6eb5b8728..822c6df173 100644 --- a/src/AbstractOperations/grid_metrics.jl +++ b/src/AbstractOperations/grid_metrics.jl @@ -130,6 +130,11 @@ Adapt.adapt_structure(to, gm::GridMetricOperation{LX, LY, LZ}) where {LX, LY, LZ GridMetricOperation{LX, LY, LZ}(Adapt.adapt(to, gm.metric), Adapt.adapt(to, gm.grid)) +on_architecture(to, gm::GridMetricOperation{LX, LY, LZ}) where {LX, LY, LZ} = + GridMetricOperation{LX, LY, LZ}(on_architecture(to, gm.metric), + on_architecture(to, gm.grid)) + + @inline Base.getindex(gm::GridMetricOperation, i, j, k) = gm.metric(i, j, k, gm.grid) indices(::GridMetricOperation) = default_indices(3) diff --git a/src/AbstractOperations/kernel_function_operation.jl b/src/AbstractOperations/kernel_function_operation.jl index 821040a920..dbd89365ac 100644 --- a/src/AbstractOperations/kernel_function_operation.jl +++ b/src/AbstractOperations/kernel_function_operation.jl @@ -80,6 +80,11 @@ Adapt.adapt_structure(to, κ::KernelFunctionOperation{LX, LY, LZ}) where {LX, LY Adapt.adapt(to, κ.grid), Tuple(Adapt.adapt(to, a) for a in κ.arguments)...) +on_architecture(to, κ::KernelFunctionOperation{LX, LY, LZ}) where {LX, LY, LZ} = + KernelFunctionOperation{LX, LY, LZ}(on_architecture(to, κ.kernel_function), + on_architecture(to, κ.grid), + Tuple(on_architecture(to, a) for a in κ.arguments)...) + Base.show(io::IO, kfo::KernelFunctionOperation) = print(io, summary(kfo), '\n', diff --git a/src/AbstractOperations/multiary_operations.jl b/src/AbstractOperations/multiary_operations.jl index 3a05addc30..b09168c4d0 100644 --- a/src/AbstractOperations/multiary_operations.jl +++ b/src/AbstractOperations/multiary_operations.jl @@ -150,3 +150,10 @@ Adapt.adapt_structure(to, multiary::MultiaryOperation{LX, LY, LZ}) where {LX, LY Adapt.adapt(to, multiary.args), Adapt.adapt(to, multiary.▶), Adapt.adapt(to, multiary.grid)) + +on_architecture(to, multiary::MultiaryOperation{LX, LY, LZ}) where {LX, LY, LZ} = + MultiaryOperation{LX, LY, LZ}(on_architecture(to, multiary.op), + on_architecture(to, multiary.args), + on_architecture(to, multiary.▶), + on_architecture(to, multiary.grid)) + \ No newline at end of file diff --git a/src/AbstractOperations/unary_operations.jl b/src/AbstractOperations/unary_operations.jl index 9670339c25..7c917a2132 100644 --- a/src/AbstractOperations/unary_operations.jl +++ b/src/AbstractOperations/unary_operations.jl @@ -130,3 +130,9 @@ Adapt.adapt_structure(to, unary::UnaryOperation{LX, LY, LZ}) where {LX, LY, LZ} Adapt.adapt(to, unary.arg), Adapt.adapt(to, unary.▶), Adapt.adapt(to, unary.grid)) + +on_architecture(to, unary::UnaryOperation{LX, LY, LZ}) where {LX, LY, LZ} = + UnaryOperation{LX, LY, LZ}(on_architecture(to, unary.op), + on_architecture(to, unary.arg), + on_architecture(to, unary.▶), + on_architecture(to, unary.grid)) diff --git a/src/Advection/Advection.jl b/src/Advection/Advection.jl index c2e091c663..d949070438 100644 --- a/src/Advection/Advection.jl +++ b/src/Advection/Advection.jl @@ -32,12 +32,13 @@ using OffsetArrays using Oceananigans.Grids using Oceananigans.Grids: with_halo, coordinates -using Oceananigans.Architectures: arch_array, architecture, CPU +using Oceananigans.Architectures: on_architecture, architecture, CPU using Oceananigans.Operators import Base: show, summary import Oceananigans.Grids: required_halo_size +import Oceananigans.Architectures: on_architecture abstract type AbstractAdvectionScheme{B, FT} end abstract type AbstractCenteredAdvectionScheme{B, FT} <: AbstractAdvectionScheme{B, FT} end diff --git a/src/Advection/centered_reconstruction.jl b/src/Advection/centered_reconstruction.jl index 40cf60203b..b6f5010f80 100644 --- a/src/Advection/centered_reconstruction.jl +++ b/src/Advection/centered_reconstruction.jl @@ -76,6 +76,12 @@ Adapt.adapt_structure(to, scheme::Centered{N, FT}) where {N, FT} = Adapt.adapt(to, scheme.coeff_zᵃᵃᶠ), Adapt.adapt(to, scheme.coeff_zᵃᵃᶜ), Adapt.adapt(to, scheme.buffer_scheme)) +on_architecture(to, scheme::Centered{N, FT}) where {N, FT} = + Centered{N, FT}(on_architecture(to, scheme.coeff_xᶠᵃᵃ), on_architecture(to, scheme.coeff_xᶜᵃᵃ), + on_architecture(to, scheme.coeff_yᵃᶠᵃ), on_architecture(to, scheme.coeff_yᵃᶜᵃ), + on_architecture(to, scheme.coeff_zᵃᵃᶠ), on_architecture(to, scheme.coeff_zᵃᵃᶜ), + on_architecture(to, scheme.buffer_scheme)) + # Useful aliases Centered(grid, FT::DataType=Float64; kwargs...) = Centered(FT; grid, kwargs...) diff --git a/src/Advection/reconstruction_coefficients.jl b/src/Advection/reconstruction_coefficients.jl index e0c72a88f4..adcf18c925 100644 --- a/src/Advection/reconstruction_coefficients.jl +++ b/src/Advection/reconstruction_coefficients.jl @@ -243,7 +243,7 @@ end # Stretched reconstruction coefficients for `Centered` schemes @inline function calc_reconstruction_coefficients(FT, coord, arch, N, ::Val{1}; order) - cpu_coord = arch_array(CPU(), coord) + cpu_coord = on_architecture(CPU(), coord) r = ((order + 1) ÷ 2) - 1 s = create_reconstruction_coefficients(FT, r, cpu_coord, arch, N; order) return s @@ -251,7 +251,7 @@ end # Stretched reconstruction coefficients for `UpwindBiased` schemes @inline function calc_reconstruction_coefficients(FT, coord, arch, N, ::Val{2}; order) - cpu_coord = arch_array(CPU(), coord) + cpu_coord = on_architecture(CPU(), coord) rleft = ((order + 1) ÷ 2) - 2 rright = ((order + 1) ÷ 2) - 1 s = [] @@ -264,7 +264,7 @@ end # Stretched reconstruction coefficients for `WENO` schemes @inline function calc_reconstruction_coefficients(FT, coord, arch, N, ::Val{3}; order) - cpu_coord = arch_array(CPU(), coord) + cpu_coord = on_architecture(CPU(), coord) s = [] for r in -1:order-1 push!(s, create_reconstruction_coefficients(FT, r, cpu_coord, arch, N; order)) @@ -280,5 +280,5 @@ end push!(stencil, stencil_coefficients(i, r, cpu_coord, cpu_coord; order)) end end - return OffsetArray(arch_array(arch, stencil), -1) + return OffsetArray(on_architecture(arch, stencil), -1) end diff --git a/src/Advection/stretched_weno_smoothness.jl b/src/Advection/stretched_weno_smoothness.jl index 3b3c49bac8..ec37b87b6c 100644 --- a/src/Advection/stretched_weno_smoothness.jl +++ b/src/Advection/stretched_weno_smoothness.jl @@ -61,7 +61,7 @@ end function calc_smoothness_coefficients(FT, beta, coord, arch, N; order) - cpu_coord = arch_array(CPU(), coord) + cpu_coord = on_architecture(CPU(), coord) order == 3 || throw(ArgumentError("The stretched smoothness coefficients are only implemented for order == 3")) @@ -104,7 +104,7 @@ function create_smoothness_coefficients(FT, r, op, cpu_coord, arch, N; order) end end - return OffsetArray(arch_array(arch, stencil), -1) + return OffsetArray(on_architecture(arch, stencil), -1) end @inline dagger(ψ) = (ψ[2], ψ[3], ψ[1]) diff --git a/src/Advection/upwind_biased_reconstruction.jl b/src/Advection/upwind_biased_reconstruction.jl index 309709bcbc..e7b6955cc7 100644 --- a/src/Advection/upwind_biased_reconstruction.jl +++ b/src/Advection/upwind_biased_reconstruction.jl @@ -85,6 +85,13 @@ Adapt.adapt_structure(to, scheme::UpwindBiased{N, FT}) where {N, FT} = Adapt.adapt(to, scheme.buffer_scheme), Adapt.adapt(to, scheme.advecting_velocity_scheme)) +on_architecture(to, scheme::UpwindBiased{N, FT}) where {N, FT} = + UpwindBiased{N, FT}(on_architecture(to, scheme.coeff_xᶠᵃᵃ), on_architecture(to, scheme.coeff_xᶜᵃᵃ), + on_architecture(to, scheme.coeff_yᵃᶠᵃ), on_architecture(to, scheme.coeff_yᵃᶜᵃ), + on_architecture(to, scheme.coeff_zᵃᵃᶠ), on_architecture(to, scheme.coeff_zᵃᵃᶜ), + on_architecture(to, scheme.buffer_scheme), + on_architecture(to, scheme.advecting_velocity_scheme)) + # Useful aliases UpwindBiased(grid, FT::DataType=Float64; kwargs...) = UpwindBiased(FT; grid, kwargs...) diff --git a/src/Advection/vector_invariant_advection.jl b/src/Advection/vector_invariant_advection.jl index 7530e1788c..537dcf6eb3 100644 --- a/src/Advection/vector_invariant_advection.jl +++ b/src/Advection/vector_invariant_advection.jl @@ -229,6 +229,14 @@ Adapt.adapt_structure(to, scheme::VectorInvariant{N, FT, M}) where {N, FT, M} = Adapt.adapt(to, scheme.divergence_scheme), Adapt.adapt(to, scheme.upwinding)) +on_architecture(to, scheme::VectorInvariant{N, FT, M}) where {N, FT, M} = + VectorInvariant{N, FT, M}(on_architecture(to, scheme.vorticity_scheme), + on_architecture(to, scheme.vorticity_stencil), + on_architecture(to, scheme.vertical_scheme), + on_architecture(to, scheme.kinetic_energy_gradient_scheme), + on_architecture(to, scheme.divergence_scheme), + on_architecture(to, scheme.upwinding)) + @inline U_dot_∇u(i, j, k, grid, scheme::VectorInvariant, U) = horizontal_advection_U(i, j, k, grid, scheme, U.u, U.v) + vertical_advection_U(i, j, k, grid, scheme, U) + bernoulli_head_U(i, j, k, grid, scheme, U.u, U.v) diff --git a/src/Advection/vector_invariant_upwinding.jl b/src/Advection/vector_invariant_upwinding.jl index 02b1a4d4cb..b2b24470a0 100644 --- a/src/Advection/vector_invariant_upwinding.jl +++ b/src/Advection/vector_invariant_upwinding.jl @@ -140,7 +140,6 @@ Adapt.adapt_structure(to, scheme::CrossAndSelfUpwinding) = Adapt.adapt(to, scheme.δu²_stencil), Adapt.adapt(to, scheme.δv²_stencil)) - Base.show(io::IO, a::VelocityUpwinding) = print(io, summary(a), " \n", "KE gradient and Divergence flux cross terms reconstruction: ", "\n", diff --git a/src/Advection/weno_reconstruction.jl b/src/Advection/weno_reconstruction.jl index 2087794674..0ad255a68d 100644 --- a/src/Advection/weno_reconstruction.jl +++ b/src/Advection/weno_reconstruction.jl @@ -160,6 +160,14 @@ Adapt.adapt_structure(to, scheme::WENO{N, FT, XT, YT, ZT, WF, PP}) where {N, FT, Adapt.adapt(to, scheme.buffer_scheme), Adapt.adapt(to, scheme.advecting_velocity_scheme)) +on_architecture(to, scheme::WENO{N, FT, XT, YT, ZT, WF, PP}) where {N, FT, XT, YT, ZT, WF, PP} = + WENO{N, FT, WF}(on_architecture(to, scheme.coeff_xᶠᵃᵃ), on_architecture(to, scheme.coeff_xᶜᵃᵃ), + on_architecture(to, scheme.coeff_yᵃᶠᵃ), on_architecture(to, scheme.coeff_yᵃᶜᵃ), + on_architecture(to, scheme.coeff_zᵃᵃᶠ), on_architecture(to, scheme.coeff_zᵃᵃᶜ), + on_architecture(to, scheme.bounds), + on_architecture(to, scheme.buffer_scheme), + on_architecture(to, scheme.advecting_velocity_scheme)) + # Retrieve precomputed coefficients (+2 for julia's 1 based indices) @inline retrieve_coeff(scheme::WENO, r, ::Val{1}, i, ::Type{Face}) = @inbounds scheme.coeff_xᶠᵃᵃ[r+2][i] @inline retrieve_coeff(scheme::WENO, r, ::Val{1}, i, ::Type{Center}) = @inbounds scheme.coeff_xᶜᵃᵃ[r+2][i] diff --git a/src/Architectures.jl b/src/Architectures.jl index 78b55a067a..aebc19f4bb 100644 --- a/src/Architectures.jl +++ b/src/Architectures.jl @@ -2,7 +2,7 @@ module Architectures export AbstractArchitecture export CPU, GPU, MultiGPU -export device, architecture, array_type, arch_array, unified_array, device_copy_to! +export device, architecture, array_type, on_architecture, unified_array, device_copy_to! using CUDA using KernelAbstractions @@ -56,32 +56,32 @@ child_architecture(arch) = arch array_type(::CPU) = Array array_type(::GPU) = CuArray -arch_array(::CPU, a::Array) = a -arch_array(::CPU, a::CuArray) = Array(a) -arch_array(::GPU, a::Array) = CuArray(a) -arch_array(::GPU, a::CuArray) = a +on_architecture(::CPU, a::Array) = a +on_architecture(::CPU, a::CuArray) = Array(a) +on_architecture(::GPU, a::Array) = CuArray(a) +on_architecture(::GPU, a::CuArray) = a -arch_array(::CPU, a::BitArray) = a -arch_array(::GPU, a::BitArray) = CuArray(a) +on_architecture(::CPU, a::BitArray) = a +on_architecture(::GPU, a::BitArray) = CuArray(a) -arch_array(::GPU, a::SubArray{<:Any, <:Any, <:CuArray}) = a -arch_array(::CPU, a::SubArray{<:Any, <:Any, <:CuArray}) = Array(a) +on_architecture(::GPU, a::SubArray{<:Any, <:Any, <:CuArray}) = a +on_architecture(::CPU, a::SubArray{<:Any, <:Any, <:CuArray}) = Array(a) -arch_array(::GPU, a::SubArray{<:Any, <:Any, <:Array}) = CuArray(a) -arch_array(::CPU, a::SubArray{<:Any, <:Any, <:Array}) = a +on_architecture(::GPU, a::SubArray{<:Any, <:Any, <:Array}) = CuArray(a) +on_architecture(::CPU, a::SubArray{<:Any, <:Any, <:Array}) = a -arch_array(::CPU, a::AbstractRange) = a -arch_array(::CPU, ::Nothing) = nothing -arch_array(::CPU, a::Number) = a -arch_array(::CPU, a::Function) = a +on_architecture(::CPU, a::AbstractRange) = a +on_architecture(::CPU, ::Nothing) = nothing +on_architecture(::CPU, a::Number) = a +on_architecture(::CPU, a::Function) = a -arch_array(::GPU, a::AbstractRange) = a -arch_array(::GPU, ::Nothing) = nothing -arch_array(::GPU, a::Number) = a -arch_array(::GPU, a::Function) = a +on_architecture(::GPU, a::AbstractRange) = a +on_architecture(::GPU, ::Nothing) = nothing +on_architecture(::GPU, a::Number) = a +on_architecture(::GPU, a::Function) = a -arch_array(arch::CPU, a::OffsetArray) = OffsetArray(arch_array(arch, a.parent), a.offsets...) -arch_array(arch::GPU, a::OffsetArray) = OffsetArray(arch_array(arch, a.parent), a.offsets...) +on_architecture(arch::CPU, a::OffsetArray) = OffsetArray(on_architecture(arch, a.parent), a.offsets...) +on_architecture(arch::GPU, a::OffsetArray) = OffsetArray(on_architecture(arch, a.parent), a.offsets...) cpu_architecture(::CPU) = CPU() cpu_architecture(::GPU) = CPU() diff --git a/src/BoundaryConditions/boundary_condition.jl b/src/BoundaryConditions/boundary_condition.jl index 1e59c8f0be..86a81cbf25 100644 --- a/src/BoundaryConditions/boundary_condition.jl +++ b/src/BoundaryConditions/boundary_condition.jl @@ -1,4 +1,5 @@ import Adapt +import Oceananigans.Architectures: on_architecture """ struct BoundaryCondition{C<:AbstractBoundaryConditionClassification, T} @@ -69,6 +70,11 @@ end Adapt.adapt_structure(to, b::BoundaryCondition{Classification}) where Classification = BoundaryCondition(Classification(), Adapt.adapt(to, b.condition)) + +# Adapt boundary condition struct to be GPU friendly and passable to GPU kernels. +on_architecture(to, b::BoundaryCondition{Classification}) where Classification = + BoundaryCondition(Classification(), on_architecture(to, b.condition)) + ##### ##### Some abbreviations to make life easier. ##### diff --git a/src/BoundaryConditions/continuous_boundary_function.jl b/src/BoundaryConditions/continuous_boundary_function.jl index dcc4dbeea7..3023dbd2ac 100644 --- a/src/BoundaryConditions/continuous_boundary_function.jl +++ b/src/BoundaryConditions/continuous_boundary_function.jl @@ -217,3 +217,10 @@ Adapt.adapt_structure(to, bf::ContinuousBoundaryFunction{LX, LY, LZ, S}) where { nothing, Adapt.adapt(to, bf.field_dependencies_indices), Adapt.adapt(to, bf.field_dependencies_interp)) + +on_architecture(to, bf::ContinuousBoundaryFunction{LX, LY, LZ, S}) where {LX, LY, LZ, S} = + ContinuousBoundaryFunction{LX, LY, LZ, S}(on_architecture(to, bf.func), + on_architecture(to, bf.parameters), + on_architecture(to, bf.field_dependencies), + on_architecture(to, bf.field_dependencies_indices), + on_architecture(to, bf.field_dependencies_interp)) diff --git a/src/BoundaryConditions/discrete_boundary_function.jl b/src/BoundaryConditions/discrete_boundary_function.jl index 65ebf70323..39be642726 100644 --- a/src/BoundaryConditions/discrete_boundary_function.jl +++ b/src/BoundaryConditions/discrete_boundary_function.jl @@ -57,3 +57,6 @@ Base.summary(bf::DiscreteBoundaryFunction) = string("DiscreteBoundaryFunction ", Adapt.adapt_structure(to, bf::DiscreteBoundaryFunction) = DiscreteBoundaryFunction(Adapt.adapt(to, bf.func), Adapt.adapt(to, bf.parameters)) +on_architecture(to, bf::DiscreteBoundaryFunction) = DiscreteBoundaryFunction(on_architecture(to, bf.func), + on_architecture(to, bf.parameters)) + diff --git a/src/DistributedComputations/distributed_architectures.jl b/src/DistributedComputations/distributed_architectures.jl index 2aa2a53e45..7284ae9b66 100644 --- a/src/DistributedComputations/distributed_architectures.jl +++ b/src/DistributedComputations/distributed_architectures.jl @@ -2,7 +2,7 @@ using Oceananigans.Architectures using Oceananigans.Grids: topology, validate_tupled_argument using CUDA: ndevices, device! -import Oceananigans.Architectures: device, cpu_architecture, arch_array, array_type, child_architecture, convert_args +import Oceananigans.Architectures: device, cpu_architecture, on_architecture, array_type, child_architecture, convert_args import Oceananigans.Grids: zeros import Oceananigans.Utils: sync_device!, tupleit @@ -261,7 +261,7 @@ const SynchronizedDistributed = Distributed{<:Any, true} child_architecture(arch::Distributed) = arch.child_architecture device(arch::Distributed) = device(child_architecture(arch)) -arch_array(arch::Distributed, A) = arch_array(child_architecture(arch), A) +on_architecture(arch::Distributed, A) = on_architecture(child_architecture(arch), A) zeros(FT, arch::Distributed, N...) = zeros(FT, child_architecture(arch), N...) array_type(arch::Distributed) = array_type(child_architecture(arch)) sync_device!(arch::Distributed) = sync_device!(arch.child_architecture) diff --git a/src/DistributedComputations/distributed_fields.jl b/src/DistributedComputations/distributed_fields.jl index 878646662e..62b7fca628 100644 --- a/src/DistributedComputations/distributed_fields.jl +++ b/src/DistributedComputations/distributed_fields.jl @@ -45,7 +45,7 @@ function set!(u::DistributedField, v::Union{Array, CuArray}) return u else try - f = arch_array(architecture(u), v) + f = on_architecture(architecture(u), v) u .= f return u diff --git a/src/DistributedComputations/partition_assemble.jl b/src/DistributedComputations/partition_assemble.jl index c8b90e3cbb..d381743cb1 100644 --- a/src/DistributedComputations/partition_assemble.jl +++ b/src/DistributedComputations/partition_assemble.jl @@ -1,4 +1,4 @@ -using Oceananigans.Architectures: arch_array +using Oceananigans.Architectures: on_architecture all_reduce(op, val, arch::Distributed) = MPI.Allreduce(val, op, arch.communicator) @@ -116,7 +116,7 @@ partition_global_array(arch, c_global::Function, n) = c_global # Here we assume that we cannot partition in z (we should remove support for that) function partition_global_array(arch::Distributed, c_global::AbstractArray, n) - c_global = arch_array(CPU(), c_global) + c_global = on_architecture(CPU(), c_global) ri, rj, rk = arch.local_index @@ -137,7 +137,7 @@ function partition_global_array(arch::Distributed, c_global::AbstractArray, n) 1 + sum(ny[1:rj-1]) : sum(ny[1:rj]), 1:nz] end - return arch_array(child_architecture(arch), c_local) + return on_architecture(child_architecture(arch), c_local) end """ @@ -151,7 +151,7 @@ construct_global_array(arch, c_local::Function, N) = c_local # TODO: This does not work for 3D parallelizations!!! function construct_global_array(arch::Distributed, c_local::AbstractArray, n) - c_local = arch_array(CPU(), c_local) + c_local = on_architecture(CPU(), c_local) ri, rj, rk = arch.local_index @@ -180,5 +180,5 @@ function construct_global_array(arch::Distributed, c_local::AbstractArray, n) MPI.Allreduce!(c_global, +, arch.communicator) end - return arch_array(child_architecture(arch), c_global) + return on_architecture(child_architecture(arch), c_global) end diff --git a/src/Fields/Fields.jl b/src/Fields/Fields.jl index da749d6556..b974f00700 100644 --- a/src/Fields/Fields.jl +++ b/src/Fields/Fields.jl @@ -34,7 +34,7 @@ Build a field from `a` at `loc` and on `grid`. """ @inline function field(loc, a::AbstractArray, grid) f = Field(loc, grid) - a = arch_array(architecture(grid), a) + a = on_architecture(architecture(grid), a) try copyto!(parent(f), a) catch diff --git a/src/Fields/field.jl b/src/Fields/field.jl index 9337523cea..29f99af6dc 100644 --- a/src/Fields/field.jl +++ b/src/Fields/field.jl @@ -5,6 +5,7 @@ using Adapt using KernelAbstractions: @kernel, @index using Base: @propagate_inbounds +import Oceananigans.Architectures: on_architecture import Oceananigans.BoundaryConditions: fill_halo_regions!, getbc import Statistics: norm, mean, mean! import Base: == @@ -413,6 +414,19 @@ total_size(f::Field) = total_size(f.grid, location(f), f.indices) ==(a, f::Field) = a == interior(f) ==(a::Field, b::Field) = interior(a) == interior(b) +##### +##### Move Fields between architectures +##### + +on_architecture(arch, field::AbstractField) = + Field{LX, LY, LZ}(on_architecture(arch, field.grid), + on_architecture(arch, data), + on_architecture(arch, bcs), + on_architecture(arch, indices), + on_architecture(arch, op), + on_architecture(arch, status), + on_architecture(arch, buffers)) + ##### ##### Interface for field computations ##### diff --git a/src/Fields/field_boundary_buffers.jl b/src/Fields/field_boundary_buffers.jl index f11b2e9558..e514a3be3e 100644 --- a/src/Fields/field_boundary_buffers.jl +++ b/src/Fields/field_boundary_buffers.jl @@ -1,9 +1,11 @@ using Oceananigans.BoundaryConditions: MCBC, DCBC -using Oceananigans.Architectures: arch_array +using Oceananigans.Architectures: on_architecture using Oceananigans.Grids: halo_size, size using Oceananigans.Utils: launch! using KernelAbstractions: @kernel, @index +import Oceananigans.Architectures: on_architecture + struct FieldBoundaryBuffers{W, E, S, N, SW, SE, NW, NE} west :: W east :: E @@ -61,31 +63,31 @@ create_buffer_corner(arch, grid, data, Hx, Hy, ::Nothing, edge2) = nothing create_buffer_corner(arch, grid, data, Hx, Hy, ::Nothing, ::Nothing) = nothing function create_buffer_corner(arch, grid, data, Hx, Hy, edge1, edge2) - return (send = arch_array(arch, zeros(eltype(data), Hx, Hy, size(parent(data), 3))), - recv = arch_array(arch, zeros(eltype(data), Hx, Hy, size(parent(data), 3)))) + return (send = on_architecture(arch, zeros(eltype(data), Hx, Hy, size(parent(data), 3))), + recv = on_architecture(arch, zeros(eltype(data), Hx, Hy, size(parent(data), 3)))) end function create_buffer_x(arch, grid, data, H, ::DCBC) # Either we pass corners or it is a 1D parallelization in x size_y = arch.ranks[2] == 1 ? size(parent(data), 2) : size(grid, 2) - return (send = arch_array(arch, zeros(eltype(data), H, size_y, size(parent(data), 3))), - recv = arch_array(arch, zeros(eltype(data), H, size_y, size(parent(data), 3)))) + return (send = on_architecture(arch, zeros(eltype(data), H, size_y, size(parent(data), 3))), + recv = on_architecture(arch, zeros(eltype(data), H, size_y, size(parent(data), 3)))) end function create_buffer_y(arch, grid, data, H, ::DCBC) # Either we pass corners or it is a 1D parallelization in y size_x = arch.ranks[1] == 1 ? size(parent(data), 1) : size(grid, 1) - return (send = arch_array(arch, zeros(eltype(data), size_x, H, size(parent(data), 3))), - recv = arch_array(arch, zeros(eltype(data), size_x, H, size(parent(data), 3)))) + return (send = on_architecture(arch, zeros(eltype(data), size_x, H, size(parent(data), 3))), + recv = on_architecture(arch, zeros(eltype(data), size_x, H, size(parent(data), 3)))) end create_buffer_x(arch, grid, data, H, ::MCBC) = - (send = arch_array(arch, zeros(eltype(data), H, size(parent(data), 2), size(parent(data), 3))), - recv = arch_array(arch, zeros(eltype(data), H, size(parent(data), 2), size(parent(data), 3)))) + (send = on_architecture(arch, zeros(eltype(data), H, size(parent(data), 2), size(parent(data), 3))), + recv = on_architecture(arch, zeros(eltype(data), H, size(parent(data), 2), size(parent(data), 3)))) create_buffer_y(arch, grid, data, H, ::MCBC) = - (send = arch_array(arch, zeros(eltype(data), size(parent(data), 1), H, size(parent(data), 3))), - recv = arch_array(arch, zeros(eltype(data), size(parent(data), 1), H, size(parent(data), 3)))) + (send = on_architecture(arch, zeros(eltype(data), size(parent(data), 1), H, size(parent(data), 3))), + recv = on_architecture(arch, zeros(eltype(data), size(parent(data), 1), H, size(parent(data), 3)))) Adapt.adapt_structure(to, buff::FieldBoundaryBuffers) = FieldBoundaryBuffers(Adapt.adapt(to, buff.west), @@ -97,6 +99,16 @@ Adapt.adapt_structure(to, buff::FieldBoundaryBuffers) = Adapt.adapt(to, buff.northwest), Adapt.adapt(to, buff.northeast)) +on_architecture(arch, buff::FieldBoundaryBuffers) = + FieldBoundaryBuffers(on_architecture(arch, buff.west), + on_architecture(arch, buff.east), + on_architecture(arch, buff.north), + on_architecture(arch, buff.south), + on_architecture(arch, buff.southwest), + on_architecture(arch, buff.southeast), + on_architecture(arch, buff.northwest), + on_architecture(arch, buff.northeast)) + """ fill_send_buffers!(c::OffsetArray, buffers::FieldBoundaryBuffers, grid) diff --git a/src/Fields/function_field.jl b/src/Fields/function_field.jl index 39a3a78eab..0a35dfd0af 100644 --- a/src/Fields/function_field.jl +++ b/src/Fields/function_field.jl @@ -72,6 +72,13 @@ Adapt.adapt_structure(to, f::FunctionField{LX, LY, LZ}) where {LX, LY, LZ} = clock = Adapt.adapt(to, f.clock), parameters = Adapt.adapt(to, f.parameters)) + +on_architecture(to, f::FunctionField{LX, LY, LZ}) where {LX, LY, LZ} = + FunctionField{LX, LY, LZ}(on_architecture(to, f.func), + on_architecture(to, f.grid), + clock = on_architecture(to, f.clock), + parameters = on_architecture(to, f.parameters)) + Base.show(io::IO, field::FunctionField) = print(io, "FunctionField located at ", show_location(field), "\n", "├── func: $(prettysummary(field.func))", "\n", diff --git a/src/Fields/regridding_fields.jl b/src/Fields/regridding_fields.jl index a23c6e6a48..758257f369 100644 --- a/src/Fields/regridding_fields.jl +++ b/src/Fields/regridding_fields.jl @@ -1,6 +1,6 @@ using KernelAbstractions: @kernel, @index -using Oceananigans.Architectures: arch_array, architecture +using Oceananigans.Architectures: on_architecture, architecture using Oceananigans.Operators: Δzᶜᶜᶜ, Δyᶜᶜᶜ, Δxᶜᶜᶜ, Azᶜᶜᶜ using Oceananigans.Grids: hack_sind, ξnode, ηnode, rnode diff --git a/src/Fields/set!.jl b/src/Fields/set!.jl index f6e8a8eb78..f2e89a7817 100644 --- a/src/Fields/set!.jl +++ b/src/Fields/set!.jl @@ -70,7 +70,7 @@ function set!(u::Field, f::Function) end function set!(u::Field, f::Union{Array, CuArray, OffsetArray}) - f = arch_array(architecture(u), f) + f = on_architecture(architecture(u), f) u .= f return u end @@ -91,7 +91,7 @@ function set!(u::Field, v::Field) interior(u) .= interior(v) end else - v_data = arch_array(architecture(u), v.data) + v_data = on_architecture(architecture(u), v.data) # As above, we permit ourselves a little ambition and try to copy halo data: try diff --git a/src/Forcings/advective_forcing.jl b/src/Forcings/advective_forcing.jl index d1ba1a0394..ecf5191c47 100644 --- a/src/Forcings/advective_forcing.jl +++ b/src/Forcings/advective_forcing.jl @@ -3,6 +3,8 @@ using Oceananigans.Fields: ZeroField, ConstantField using Oceananigans.Utils: SumOfArrays using Adapt +import Oceananigans.Architectures: on_architecture + maybe_constant_field(u) = u maybe_constant_field(u::Number) = ConstantField(u) @@ -69,6 +71,10 @@ end Adapt.adapt_structure(to, af::AdvectiveForcing) = AdvectiveForcing(adapt(to, af.u), adapt(to, af.v), adapt(to, af.w)) +on_architecture(to, af::AdvectiveForcing) = + AdvectiveForcing(on_architecture(to, af.u), on_architecture(to, af.v), on_architecture(to, af.w)) + + # fallback @inline with_advective_forcing(forcing, total_velocities) = total_velocities diff --git a/src/Forcings/continuous_forcing.jl b/src/Forcings/continuous_forcing.jl index f9ee124754..6f3283358e 100644 --- a/src/Forcings/continuous_forcing.jl +++ b/src/Forcings/continuous_forcing.jl @@ -5,6 +5,8 @@ using Oceananigans.Operators: assumed_field_location, index_and_interp_dependenc using Oceananigans.Fields: show_location using Oceananigans.Utils: user_function_arguments, tupleit, prettysummary +import Oceananigans.Architectures: on_architecture + """ ContinuousForcing{LX, LY, LZ, P, F, D, I, ℑ} @@ -162,3 +164,10 @@ Adapt.adapt_structure(to, forcing::ContinuousForcing{LX, LY, LZ}) where {LX, LY, Adapt.adapt(to, forcing.field_dependencies_indices), Adapt.adapt(to, forcing.field_dependencies_interp)) +on_architecture(to, forcing::ContinuousForcing{LX, LY, LZ}) where {LX, LY, LZ} = + ContinuousForcing{LX, LY, LZ}(on_architecture(to, forcing.func), + on_architecture(to, forcing.parameters), + on_architecture(to, forcing.field_dependencies), + on_architecture(to, forcing.field_dependencies_indices), + on_architecture(to, forcing.field_dependencies_interp)) + diff --git a/src/Forcings/discrete_forcing.jl b/src/Forcings/discrete_forcing.jl index 5105c8f10b..435fc05cd1 100644 --- a/src/Forcings/discrete_forcing.jl +++ b/src/Forcings/discrete_forcing.jl @@ -60,3 +60,7 @@ Base.show(io::IO, forcing::DiscreteForcing{P}) where P = Adapt.adapt_structure(to, forcing::DiscreteForcing) = DiscreteForcing(Adapt.adapt(to, forcing.func), Adapt.adapt(to, forcing.parameters)) + +on_architecture(to, forcing::DiscreteForcing) = + DiscreteForcing(on_architecture(to, forcing.func), + on_architecture(to, forcing.parameters)) diff --git a/src/Forcings/multiple_forcings.jl b/src/Forcings/multiple_forcings.jl index 060e56a911..30123ec36d 100644 --- a/src/Forcings/multiple_forcings.jl +++ b/src/Forcings/multiple_forcings.jl @@ -5,6 +5,7 @@ struct MultipleForcings{N, F} end Adapt.adapt_structure(to, mf::MultipleForcings) = MultipleForcings(adapt(to, mf.forcings)) +on_architecture(to, mf::MultipleForcings) = MultipleForcings(on_architecture(to, mf.forcings)) Base.getindex(mf::MultipleForcings, i) = mf.forcings[i] diff --git a/src/Grids/grid_generation.jl b/src/Grids/grid_generation.jl index 98bd421d43..056c4fb7af 100644 --- a/src/Grids/grid_generation.jl +++ b/src/Grids/grid_generation.jl @@ -1,8 +1,4 @@ # Utilities to generate a grid with the following inputs - -@inline adapt_if_vector(to, var) = var -@inline adapt_if_vector(to, var::AbstractArray) = Adapt.adapt(to, var) - get_domain_extent(::Nothing, N) = (1, 1) get_domain_extent(coord, N) = (coord[1], coord[2]) get_domain_extent(coord::Function, N) = (coord(1), coord(N+1)) @@ -76,15 +72,15 @@ function generate_coordinate(FT, topo::AT, N, H, node_generator, coordinate_name Δᶠ[i] = Δᶠ[i-1] end - Δᶜ = OffsetArray(arch_array(arch, Δᶜ), -H) - Δᶠ = OffsetArray(arch_array(arch, Δᶠ), -H-1) + Δᶜ = OffsetArray(on_architecture(arch, Δᶜ), -H) + Δᶠ = OffsetArray(on_architecture(arch, Δᶠ), -H-1) F = OffsetArray(F, -H) C = OffsetArray(C, -H) # Convert to appropriate array type for arch - F = OffsetArray(arch_array(arch, F.parent), F.offsets...) - C = OffsetArray(arch_array(arch, C.parent), C.offsets...) + F = OffsetArray(on_architecture(arch, F.parent), F.offsets...) + C = OffsetArray(on_architecture(arch, C.parent), C.offsets...) return L, F, C, Δᶠ, Δᶜ end diff --git a/src/Grids/grid_utils.jl b/src/Grids/grid_utils.jl index fbcef761f7..90b34d603f 100644 --- a/src/Grids/grid_utils.jl +++ b/src/Grids/grid_utils.jl @@ -459,7 +459,7 @@ function add_halos(data, loc, topo, sz, halo_sz; warnings=true) arch = architecture(data) # bring to CPU - map(a -> arch_array(CPU(), a), data) + map(a -> on_architecture(CPU(), a), data) nx, ny, nz = total_length(loc[1](), topo[1](), sz[1], 0), total_length(loc[2](), topo[2](), sz[2], 0), @@ -484,7 +484,7 @@ function add_halos(data, loc, topo, sz, halo_sz; warnings=true) offset_array[1:nx, 1:ny, 1:nz] = data[1:nx, 1:ny, 1:nz] # return to data's original architecture - map(a -> arch_array(arch, a), offset_array) + map(a -> on_architecture(arch, a), offset_array) return offset_array end diff --git a/src/Grids/latitude_longitude_grid.jl b/src/Grids/latitude_longitude_grid.jl index f6e2d7a887..3e5fafd730 100644 --- a/src/Grids/latitude_longitude_grid.jl +++ b/src/Grids/latitude_longitude_grid.jl @@ -354,7 +354,7 @@ function on_architecture(new_arch::AbstractArchitecture, old_grid::LatitudeLongi old_grid.Δyᶠᶜᵃ, old_grid.Δyᶜᶠᵃ, old_grid.Azᶠᶜᵃ, old_grid.Azᶜᶠᵃ, old_grid.Azᶠᶠᵃ, old_grid.Azᶜᶜᵃ) - new_properties = Tuple(arch_array(new_arch, p) for p in old_properties) + new_properties = Tuple(on_architecture(new_arch, p) for p in old_properties) TX, TY, TZ = topology(old_grid) @@ -552,7 +552,7 @@ function allocate_metrics(grid::LatitudeLongitudeGrid) for metric in grid_metrics parentM = Symbol(metric, :_parent) @eval $parentM = zeros($FT, $metric_size...) - @eval $metric = OffsetArray(arch_array($arch, $parentM), $offsets...) + @eval $metric = OffsetArray(on_architecture($arch, $parentM), $offsets...) end if grid isa YRegularLLG @@ -561,8 +561,8 @@ function allocate_metrics(grid::LatitudeLongitudeGrid) else parentC = zeros(FT, length(grid.Δφᵃᶜᵃ)) parentF = zeros(FT, length(grid.Δφᵃᶜᵃ)) - Δyᶠᶜ = OffsetArray(arch_array(arch, parentC), grid.Δφᵃᶜᵃ.offsets[1]) - Δyᶜᶠ = OffsetArray(arch_array(arch, parentF), grid.Δφᵃᶜᵃ.offsets[1]) + Δyᶠᶜ = OffsetArray(on_architecture(arch, parentC), grid.Δφᵃᶜᵃ.offsets[1]) + Δyᶜᶠ = OffsetArray(on_architecture(arch, parentF), grid.Δφᵃᶜᵃ.offsets[1]) end return Δxᶠᶜ, Δxᶜᶠ, Δxᶠᶠ, Δxᶜᶜ, Δyᶠᶜ, Δyᶜᶠ, Azᶠᶜ, Azᶜᶠ, Azᶠᶠ, Azᶜᶜ diff --git a/src/Grids/orthogonal_spherical_shell_grid.jl b/src/Grids/orthogonal_spherical_shell_grid.jl index 5d9eb736ae..b8b3f0a723 100644 --- a/src/Grids/orthogonal_spherical_shell_grid.jl +++ b/src/Grids/orthogonal_spherical_shell_grid.jl @@ -589,13 +589,13 @@ function conformal_cubed_sphere_panel(architecture::AbstractArchitecture = CPU() # convert to coordinate_arrays = (λᶜᶜᵃ, λᶠᶜᵃ, λᶜᶠᵃ, λᶠᶠᵃ, φᶜᶜᵃ, φᶠᶜᵃ, φᶜᶠᵃ, φᶠᶠᵃ, zᵃᵃᶜ, zᵃᵃᶠ) - coordinate_arrays = map(a -> arch_array(architecture, a), coordinate_arrays) + coordinate_arrays = map(a -> on_architecture(architecture, a), coordinate_arrays) metric_arrays = (Δxᶜᶜᵃ, Δxᶠᶜᵃ, Δxᶜᶠᵃ, Δxᶠᶠᵃ, Δyᶜᶜᵃ, Δyᶜᶠᵃ, Δyᶠᶜᵃ, Δyᶠᶠᵃ, Δzᵃᵃᶜ, Δzᵃᵃᶠ, Azᶜᶜᵃ, Azᶠᶜᵃ, Azᶜᶠᵃ, Azᶠᶠᵃ) - metric_arrays = map(a -> arch_array(architecture, a), metric_arrays) + metric_arrays = map(a -> on_architecture(architecture, a), metric_arrays) conformal_mapping = (; ξ, η, rotation) @@ -777,7 +777,7 @@ function load_and_offset_cubed_sphere_data(file, FT, arch, field_name, loc, topo ii = interior_indices(loc[1](), topo[1](), N[1]) jj = interior_indices(loc[2](), topo[2](), N[2]) - interior_data = arch_array(arch, file[field_name][ii, jj]) + interior_data = on_architecture(arch, file[field_name][ii, jj]) underlying_data = zeros(FT, arch, total_length(loc[1](), topo[1](), N[1], H[1]), @@ -902,9 +902,9 @@ function on_architecture(arch::AbstractArchitecture, grid::OrthogonalSphericalSh :Azᶜᶠᵃ, :Azᶠᶠᵃ) - grid_spacing_data = Tuple(arch_array(arch, getproperty(grid, name)) for name in grid_spacings) - coordinate_data = Tuple(arch_array(arch, getproperty(grid, name)) for name in coordinates) - horizontal_area_data = Tuple(arch_array(arch, getproperty(grid, name)) for name in horizontal_areas) + grid_spacing_data = Tuple(on_architecture(arch, getproperty(grid, name)) for name in grid_spacings) + coordinate_data = Tuple(on_architecture(arch, getproperty(grid, name)) for name in coordinates) + horizontal_area_data = Tuple(on_architecture(arch, getproperty(grid, name)) for name in horizontal_areas) TX, TY, TZ = topology(grid) diff --git a/src/Grids/rectilinear_grid.jl b/src/Grids/rectilinear_grid.jl index 17eca771b9..99f35afe09 100644 --- a/src/Grids/rectilinear_grid.jl +++ b/src/Grids/rectilinear_grid.jl @@ -399,7 +399,7 @@ function on_architecture(new_arch::AbstractArchitecture, old_grid::RectilinearGr old_grid.Δyᵃᶠᵃ, old_grid.Δyᵃᶜᵃ, old_grid.yᵃᶠᵃ, old_grid.yᵃᶜᵃ, old_grid.Δzᵃᵃᶠ, old_grid.Δzᵃᵃᶜ, old_grid.zᵃᵃᶠ, old_grid.zᵃᵃᶜ) - new_properties = Tuple(arch_array(new_arch, p) for p in old_properties) + new_properties = Tuple(on_architecture(new_arch, p) for p in old_properties) TX, TY, TZ = topology(old_grid) diff --git a/src/ImmersedBoundaries/active_cells_map.jl b/src/ImmersedBoundaries/active_cells_map.jl index dbb4869368..5b4b516dde 100644 --- a/src/ImmersedBoundaries/active_cells_map.jl +++ b/src/ImmersedBoundaries/active_cells_map.jl @@ -162,7 +162,7 @@ function interior_active_indices(ibg; parameters = :xyz) # Cannot findall on the entire field because we incur on OOM errors active_indices = IndicesType[] active_indices = findall_active_indices!(active_indices, active_cells_field, ibg, IndicesType) - active_indices = arch_array(architecture(ibg), active_indices) + active_indices = on_architecture(architecture(ibg), active_indices) return active_indices end @@ -173,7 +173,7 @@ end function findall_active_indices!(active_indices, active_cells_field, ibg, IndicesType) for k in 1:size(ibg, 3) - interior_indices = findall(arch_array(CPU(), interior(active_cells_field, :, :, k:k))) + interior_indices = findall(on_architecture(CPU(), interior(active_cells_field, :, :, k:k))) interior_indices = convert_interior_indices(interior_indices, k, IndicesType) active_indices = vcat(active_indices, interior_indices) GC.gc() @@ -234,7 +234,7 @@ end # computation only on active `columns` function map_active_z_columns(ibg) active_cells_field = compute_active_z_columns(ibg) - interior_cells = arch_array(CPU(), interior(active_cells_field, :, :, 1)) + interior_cells = on_architecture(CPU(), interior(active_cells_field, :, :, 1)) full_indices = findall(interior_cells) @@ -243,7 +243,7 @@ function map_active_z_columns(ibg) N = max(Nx, Ny) IntType = N > MAXUInt8 ? (N > MAXUInt16 ? (N > MAXUInt32 ? UInt64 : UInt32) : UInt16) : UInt8 surface_map = getproperty.(full_indices, Ref(:I)) .|> Tuple{IntType, IntType} - surface_map = arch_array(architecture(ibg), surface_map) + surface_map = on_architecture(architecture(ibg), surface_map) return surface_map end diff --git a/src/ImmersedBoundaries/distributed_immersed_boundaries.jl b/src/ImmersedBoundaries/distributed_immersed_boundaries.jl index f524d8f7e4..726edda8be 100644 --- a/src/ImmersedBoundaries/distributed_immersed_boundaries.jl +++ b/src/ImmersedBoundaries/distributed_immersed_boundaries.jl @@ -75,7 +75,7 @@ function resize_immersed_boundary(ib::AbstractGridFittedBottom{<:OffsetArray}, g if any(size(ib.bottom_height) .!= bottom_heigth_size) @warn "Resizing the bottom field to match the grids' halos" bottom_field = Field((Center, Center, Nothing), grid) - cpu_bottom = arch_array(CPU(), ib.bottom_height)[1:Nx, 1:Ny] + cpu_bottom = on_architecture(CPU(), ib.bottom_height)[1:Nx, 1:Ny] set!(bottom_field, cpu_bottom) fill_halo_regions!(bottom_field) offset_bottom_array = dropdims(bottom_field.data, dims=3) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index e693e20e2d..a3461ac16d 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -4,7 +4,7 @@ using OffsetArrays: OffsetArray using Oceananigans.Utils: getnamewrapper using Oceananigans.Grids: total_size using Oceananigans.Fields: fill_halo_regions! -using Oceananigans.Architectures: arch_array +using Oceananigans.Architectures: on_architecture using Oceananigans.BoundaryConditions: FBC using Printf diff --git a/src/ImmersedBoundaries/immersed_reductions.jl b/src/ImmersedBoundaries/immersed_reductions.jl index e234f849ae..a9969363c6 100644 --- a/src/ImmersedBoundaries/immersed_reductions.jl +++ b/src/ImmersedBoundaries/immersed_reductions.jl @@ -23,7 +23,7 @@ const IF = AbstractField{<:Any, <:Any, <:Any, <:ImmersedBoundaryGrid} @inline function condition_operand(func::Function, op::IF, cond::AbstractArray, mask) arch = architecture(op.grid) - arch_condition = arch_array(arch, cond) + arch_condition = on_architecture(arch, cond) ni_condition = NotImmersed(arch_condition) return ConditionalOperation(op; func, condition=ni_condition, mask) end diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index 608c413679..94fff4cbca 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -1,6 +1,6 @@ using Oceananigans.Utils: prettysummary using Oceananigans.Fields: fill_halo_regions! -using Oceananigans.Architectures: arch_array +using Oceananigans.Architectures: on_architecture using Printf ##### diff --git a/src/Models/HydrostaticFreeSurfaceModels/matrix_implicit_free_surface_solver.jl b/src/Models/HydrostaticFreeSurfaceModels/matrix_implicit_free_surface_solver.jl index 41d0386160..65f4030132 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/matrix_implicit_free_surface_solver.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/matrix_implicit_free_surface_solver.jl @@ -48,7 +48,7 @@ function MatrixImplicitFreeSurfaceSolver(grid::AbstractGrid, settings, gravitati compute_vertically_integrated_lateral_areas!(vertically_integrated_lateral_areas) arch = architecture(grid) - right_hand_side = arch_array(arch, zeros(grid.Nx * grid.Ny)) # linearized RHS for matrix operations + right_hand_side = on_architecture(arch, zeros(grid.Nx * grid.Ny)) # linearized RHS for matrix operations storage = deepcopy(right_hand_side) @@ -112,11 +112,11 @@ function compute_matrix_coefficients(vertically_integrated_areas, grid, gravitat Nx, Ny = grid.Nx, grid.Ny - C = arch_array(arch, zeros(eltype(grid), Nx, Ny, 1)) - diag = arch_array(arch, zeros(eltype(grid), Nx, Ny, 1)) - Ax = arch_array(arch, zeros(eltype(grid), Nx, Ny, 1)) - Ay = arch_array(arch, zeros(eltype(grid), Nx, Ny, 1)) - Az = arch_array(arch, zeros(eltype(grid), Nx, Ny, 1)) + C = on_architecture(arch, zeros(eltype(grid), Nx, Ny, 1)) + diag = on_architecture(arch, zeros(eltype(grid), Nx, Ny, 1)) + Ax = on_architecture(arch, zeros(eltype(grid), Nx, Ny, 1)) + Ay = on_architecture(arch, zeros(eltype(grid), Nx, Ny, 1)) + Az = on_architecture(arch, zeros(eltype(grid), Nx, Ny, 1)) ∫Ax = vertically_integrated_areas.xᶠᶜᶜ ∫Ay = vertically_integrated_areas.yᶜᶠᶜ diff --git a/src/MultiRegion/multi_region_boundary_conditions.jl b/src/MultiRegion/multi_region_boundary_conditions.jl index 0c3e9456ba..1910c8847e 100644 --- a/src/MultiRegion/multi_region_boundary_conditions.jl +++ b/src/MultiRegion/multi_region_boundary_conditions.jl @@ -1,5 +1,5 @@ using Oceananigans: instantiated_location -using Oceananigans.Architectures: arch_array, device_copy_to! +using Oceananigans.Architectures: on_architecture, device_copy_to! using Oceananigans.Operators: assumed_field_location using Oceananigans.Fields: reduced_dimensions using Oceananigans.DistributedComputations: communication_side diff --git a/src/MultiRegion/multi_region_grid.jl b/src/MultiRegion/multi_region_grid.jl index d316f55729..707de5bd3c 100644 --- a/src/MultiRegion/multi_region_grid.jl +++ b/src/MultiRegion/multi_region_grid.jl @@ -197,13 +197,13 @@ Adapt an array `a` to be compatible with a `MultiRegionGrid`. function multi_region_object_from_array(a::AbstractArray, mrg::MultiRegionGrid) local_size = construct_regionally(size, mrg) arch = architecture(mrg) - a = arch_array(CPU(), a) + a = on_architecture(CPU(), a) ma = construct_regionally(partition_global_array, a, mrg.partition, local_size, Iterate(1:length(mrg)), arch) return ma end # Fallback! -multi_region_object_from_array(a::AbstractArray, grid) = arch_array(architecture(grid), a) +multi_region_object_from_array(a::AbstractArray, grid) = on_architecture(architecture(grid), a) #### #### Utilities for MultiRegionGrid diff --git a/src/MultiRegion/x_partitions.jl b/src/MultiRegion/x_partitions.jl index e5b056c539..92fb3cf930 100644 --- a/src/MultiRegion/x_partitions.jl +++ b/src/MultiRegion/x_partitions.jl @@ -59,13 +59,13 @@ partition_global_array(a::Field, p::EqualXPartition, args...) = partition_global function partition_global_array(a::AbstractArray, ::EqualXPartition, local_size, region, arch) idxs = default_indices(length(size(a))) - return arch_array(arch, a[local_size[1]*(region-1)+1:local_size[1]*region, idxs[2:end]...]) + return on_architecture(arch, a[local_size[1]*(region-1)+1:local_size[1]*region, idxs[2:end]...]) end function partition_global_array(a::OffsetArray, ::EqualXPartition, local_size, region, arch) idxs = default_indices(length(size(a))) offsets = (a.offsets[1], Tuple(0 for i in 1:length(idxs)-1)...) - return arch_array(arch, OffsetArray(a[local_size[1]*(region-1)+1+offsets[1]:local_size[1]*region-offsets[1], idxs[2:end]...], offsets...)) + return on_architecture(arch, OffsetArray(a[local_size[1]*(region-1)+1+offsets[1]:local_size[1]*region-offsets[1], idxs[2:end]...], offsets...)) end ##### @@ -114,10 +114,10 @@ function reconstruct_global_array(ma::ArrayMRO{T, N}, p::EqualXPartition, arch) for r = 1:length(p) init = Int(n * (r - 1) + 1) fin = Int(n * r) - arr_out[init:fin, idxs[2:end]...] .= arch_array(CPU(), ma[r])[1:fin-init+1, idxs[2:end]...] + arr_out[init:fin, idxs[2:end]...] .= on_architecture(CPU(), ma[r])[1:fin-init+1, idxs[2:end]...] end - return arch_array(arch, arr_out) + return on_architecture(arch, arr_out) end function compact_data!(global_field, global_grid, data::MultiRegionObject, p::EqualXPartition) diff --git a/src/MultiRegion/y_partitions.jl b/src/MultiRegion/y_partitions.jl index d854de3e73..06374a5a58 100644 --- a/src/MultiRegion/y_partitions.jl +++ b/src/MultiRegion/y_partitions.jl @@ -55,13 +55,13 @@ partition_global_array(a::Field, p::EqualYPartition, args...) = partition_global function partition_global_array(a::AbstractArray, ::EqualYPartition, local_size, region, arch) idxs = default_indices(length(size(a))) offsets = (a.offsets[1], Tuple(0 for i in 1:length(idxs)-1)...) - return arch_array(arch, OffsetArray(a[local_size[1]*(region-1)+1+offsets[1]:local_size[1]*region-offsets[1], idxs[2:end]...], offsets...)) + return on_architecture(arch, OffsetArray(a[local_size[1]*(region-1)+1+offsets[1]:local_size[1]*region-offsets[1], idxs[2:end]...], offsets...)) end function partition_global_array(a::OffsetArray, ::EqualYPartition, local_size, region, arch) idxs = default_indices(length(size(a))) offsets = (0, a.offsets[2], Tuple(0 for i in 1:length(idxs)-2)...) - return arch_array(arch, OffsetArray(a[idxs[1], local_size[2]*(region-1)+1+offsets[2]:local_size[2]*region-offsets[2], idxs[3:end]...], offsets...)) + return on_architecture(arch, OffsetArray(a[idxs[1], local_size[2]*(region-1)+1+offsets[2]:local_size[2]*region-offsets[2], idxs[3:end]...], offsets...)) end #### @@ -104,10 +104,10 @@ function reconstruct_global_array(ma::ArrayMRO{T, N}, p::EqualYPartition, arch) for r = 1:length(p) init = Int(n * (r - 1) + 1) fin = Int(n * r) - arr_out[idxs[1], init:fin, idxs[3:end]...] .= arch_array(CPU(), ma[r])[idxs[1], 1:fin-init+1, idxs[3:end]...] + arr_out[idxs[1], init:fin, idxs[3:end]...] .= on_architecture(CPU(), ma[r])[idxs[1], 1:fin-init+1, idxs[3:end]...] end - return arch_array(arch, arr_out) + return on_architecture(arch, arr_out) end function compact_data!(global_field, global_grid, data::MultiRegionObject, p::EqualYPartition) diff --git a/src/OutputReaders/field_time_series.jl b/src/OutputReaders/field_time_series.jl index 82aa537406..46dab7332d 100644 --- a/src/OutputReaders/field_time_series.jl +++ b/src/OutputReaders/field_time_series.jl @@ -247,7 +247,7 @@ mutable struct FieldTimeSeries{LX, LY, LZ, TI, K, I, D, G, ET, B, χ, P, N} <: A times = time_range end - times = arch_array(architecture(grid), times) + times = on_architecture(architecture(grid), times) end if time_indexing isa Cyclical{Nothing} # we have to infer the period @@ -551,7 +551,7 @@ function Field(location, path::String, name::String, iter; # Change grid to specified architecture? grid = on_architecture(architecture, grid) - raw_data = arch_array(architecture, raw_data) + raw_data = on_architecture(architecture, raw_data) data = offset_data(raw_data, grid, location, indices) return Field(location, grid; boundary_conditions, indices, data) diff --git a/src/OutputReaders/field_time_series_indexing.jl b/src/OutputReaders/field_time_series_indexing.jl index c50037f49b..3dcc20a29f 100644 --- a/src/OutputReaders/field_time_series_indexing.jl +++ b/src/OutputReaders/field_time_series_indexing.jl @@ -78,7 +78,7 @@ function getindex(fts::OnDiskFTS, n::Int) arch = architecture(fts) file = jldopen(fts.path) iter = keys(file["timeseries/t"])[n] - raw_data = arch_array(arch, file["timeseries/$(fts.name)/$iter"]) + raw_data = on_architecture(arch, file["timeseries/$(fts.name)/$iter"]) close(file) # Wrap Field diff --git a/src/Solvers/Solvers.jl b/src/Solvers/Solvers.jl index 9dac3b9e93..c5a5bf13ce 100644 --- a/src/Solvers/Solvers.jl +++ b/src/Solvers/Solvers.jl @@ -13,7 +13,7 @@ using CUDA using SparseArrays using KernelAbstractions -using Oceananigans.Architectures: device, CPU, GPU, array_type, arch_array +using Oceananigans.Architectures: device, CPU, GPU, array_type, on_architecture using Oceananigans.Utils using Oceananigans.Grids using Oceananigans.BoundaryConditions diff --git a/src/Solvers/batched_tridiagonal_solver.jl b/src/Solvers/batched_tridiagonal_solver.jl index ec5872a846..6316fc15d2 100644 --- a/src/Solvers/batched_tridiagonal_solver.jl +++ b/src/Solvers/batched_tridiagonal_solver.jl @@ -1,4 +1,4 @@ -using Oceananigans.Architectures: arch_array +using Oceananigans.Architectures: on_architecture using Oceananigans.Grids: XDirection, YDirection, ZDirection import Oceananigans.Architectures: architecture @@ -26,7 +26,7 @@ architecture(solver::BatchedTridiagonalSolver) = architecture(solver.grid) lower_diagonal, diagonal, upper_diagonal, - scratch = arch_array(architecture(grid), zeros(eltype(grid), size(grid)...)), + scratch = on_architecture(architecture(grid), zeros(eltype(grid), size(grid)...)), tridiagonal_direction = ZDirection() parameters = nothing) @@ -66,7 +66,7 @@ function BatchedTridiagonalSolver(grid; lower_diagonal, diagonal, upper_diagonal, - scratch = arch_array(architecture(grid), zeros(eltype(grid), grid.Nx, grid.Ny, grid.Nz)), + scratch = on_architecture(architecture(grid), zeros(eltype(grid), grid.Nx, grid.Ny, grid.Nz)), parameters = nothing, tridiagonal_direction = ZDirection()) diff --git a/src/Solvers/discrete_transforms.jl b/src/Solvers/discrete_transforms.jl index 744ea60d3d..9cf20a3c88 100644 --- a/src/Solvers/discrete_transforms.jl +++ b/src/Solvers/discrete_transforms.jl @@ -66,8 +66,8 @@ function twiddle_factors(arch::GPU, grid, dims) ω_4N⁻[1] *= 1/2 twiddle_factors = ( - forward = arch_array(arch, ω_4N⁺), - backward = arch_array(arch, ω_4N⁻) + forward = on_architecture(arch, ω_4N⁺), + backward = on_architecture(arch, ω_4N⁻) ) return twiddle_factors diff --git a/src/Solvers/fft_based_poisson_solver.jl b/src/Solvers/fft_based_poisson_solver.jl index 15f54704f5..22c1eddce0 100644 --- a/src/Solvers/fft_based_poisson_solver.jl +++ b/src/Solvers/fft_based_poisson_solver.jl @@ -56,11 +56,11 @@ function FFTBasedPoissonSolver(grid, planner_flag=FFTW.PATIENT) arch = architecture(grid) - eigenvalues = (λx = arch_array(arch, λx), - λy = arch_array(arch, λy), - λz = arch_array(arch, λz)) + eigenvalues = (λx = on_architecture(arch, λx), + λy = on_architecture(arch, λy), + λz = on_architecture(arch, λz)) - storage = arch_array(arch, zeros(complex(eltype(grid)), size(grid)...)) + storage = on_architecture(arch, zeros(complex(eltype(grid)), size(grid)...)) transforms = plan_transforms(grid, storage, planner_flag) diff --git a/src/Solvers/fourier_tridiagonal_poisson_solver.jl b/src/Solvers/fourier_tridiagonal_poisson_solver.jl index 2f93f2f51a..1971ab8d52 100644 --- a/src/Solvers/fourier_tridiagonal_poisson_solver.jl +++ b/src/Solvers/fourier_tridiagonal_poisson_solver.jl @@ -75,20 +75,20 @@ function FourierTridiagonalPoissonSolver(grid, planner_flag=FFTW.PATIENT) λ2 = poisson_eigenvalues(regular_siz2, regular_ext2, 2, regular_top2()) arch = architecture(grid) - λ1 = arch_array(arch, λ1) - λ2 = arch_array(arch, λ2) + λ1 = on_architecture(arch, λ1) + λ2 = on_architecture(arch, λ2) # Plan required transforms for x and y - sol_storage = arch_array(arch, zeros(complex(eltype(grid)), size(grid)...)) + sol_storage = on_architecture(arch, zeros(complex(eltype(grid)), size(grid)...)) transforms = plan_transforms(grid, sol_storage, planner_flag) # Lower and upper diagonals are the same lower_diagonal = CUDA.@allowscalar [ 1 / Δξᶠ(q, grid) for q in 2:size(grid, irreg_dim) ] - lower_diagonal = arch_array(arch, lower_diagonal) + lower_diagonal = on_architecture(arch, lower_diagonal) upper_diagonal = lower_diagonal # Compute diagonal coefficients for each grid point - diagonal = arch_array(arch, zeros(size(grid)...)) + diagonal = on_architecture(arch, zeros(size(grid)...)) launch_config = if grid isa YZRegularRG :yz elseif grid isa XZRegularRG @@ -108,7 +108,7 @@ function FourierTridiagonalPoissonSolver(grid, planner_flag=FFTW.PATIENT) buffer = buffer_needed ? similar(sol_storage) : nothing # Storage space for right hand side of Poisson equation - rhs = arch_array(arch, zeros(complex(eltype(grid)), size(grid)...)) + rhs = on_architecture(arch, zeros(complex(eltype(grid)), size(grid)...)) return FourierTridiagonalPoissonSolver(grid, btsolver, rhs, sol_storage, buffer, transforms) end diff --git a/src/Solvers/heptadiagonal_iterative_solver.jl b/src/Solvers/heptadiagonal_iterative_solver.jl index e192acba13..d3fb92f86a 100644 --- a/src/Solvers/heptadiagonal_iterative_solver.jl +++ b/src/Solvers/heptadiagonal_iterative_solver.jl @@ -1,5 +1,5 @@ using Oceananigans.Architectures -using Oceananigans.Architectures: architecture, arch_array, unsafe_free! +using Oceananigans.Architectures: architecture, on_architecture, unsafe_free! using Oceananigans.Grids: interior_parent_indices, topology using Oceananigans.Utils: heuristic_workgroup using KernelAbstractions: @kernel, @index @@ -36,7 +36,7 @@ end placeholder_timestep = -1.0, preconditioner_method = :Default, preconditioner_settings = nothing, - template = arch_array(architecture(grid), zeros(prod(size(grid)))), + template = on_architecture(architecture(grid), zeros(prod(size(grid)))), verbose = false) Return a `HeptadiagonalIterativeSolver` to solve the problem `A * x = b`, provided @@ -90,7 +90,7 @@ function HeptadiagonalIterativeSolver(coeffs; placeholder_timestep = -1.0, preconditioner_method = :Default, preconditioner_settings = nothing, - template = arch_array(architecture(grid), zeros(prod(size(grid)))), + template = on_architecture(architecture(grid), zeros(prod(size(grid)))), verbose = false) arch = architecture(grid) @@ -138,11 +138,11 @@ Return the sparse matrix constructors based on the pentadiagonal coeffients (`co function matrix_from_coefficients(arch, grid, coeffs, reduced_dim) Ax, Ay, Az, C, D = coeffs - Ax = arch_array(CPU(), Ax) - Ay = arch_array(CPU(), Ay) - Az = arch_array(CPU(), Az) - C = arch_array(CPU(), C) - D = arch_array(arch, D) + Ax = on_architecture(CPU(), Ax) + Ay = on_architecture(CPU(), Ay) + Az = on_architecture(CPU(), Az) + C = on_architecture(CPU(), C) + D = on_architecture(arch, D) N = size(grid) @@ -151,7 +151,7 @@ function matrix_from_coefficients(arch, grid, coeffs, reduced_dim) dims = validate_laplacian_direction.(N, topo, reduced_dim) Nx, Ny, Nz = N = validate_laplacian_size.(N, dims) M = prod(N) - diag = arch_array(arch, zeros(eltype(grid), M)) + diag = on_architecture(arch, zeros(eltype(grid), M)) # the following coefficients are the diagonals of the sparse matrix: # - coeff_d is the main diagonal (coefficents of ηᵢⱼₖ) diff --git a/src/Solvers/sparse_preconditioners.jl b/src/Solvers/sparse_preconditioners.jl index 8706469878..e1415dedae 100644 --- a/src/Solvers/sparse_preconditioners.jl +++ b/src/Solvers/sparse_preconditioners.jl @@ -131,13 +131,13 @@ function asymptotic_diagonal_inverse_preconditioner(A::AbstractMatrix; asymptoti M = size(A, 1) - invdiag = arch_array(arch, zeros(eltype(nzval), M)) + invdiag = on_architecture(arch, zeros(eltype(nzval), M)) loop! = _get_inv_diag!(dev, 256, M) loop!(invdiag, colptr, rowval, nzval) if asymptotic_order == 0 - Minv_cpu = spdiagm(0=>arch_array(CPU(), invdiag)) + Minv_cpu = spdiagm(0=>on_architecture(CPU(), invdiag)) Minv = arch_sparse_matrix(arch, Minv_cpu) elseif asymptotic_order == 1 loop! = _initialize_asymptotic_diagonal_inverse_preconditioner_first_order!(dev, 256, M) @@ -147,7 +147,7 @@ function asymptotic_diagonal_inverse_preconditioner(A::AbstractMatrix; asymptoti Minv = arch_sparse_matrix(arch, constructors(arch, M, M, constr_new)) else D = spdiagm(0=>diag(arch_sparse_matrix(CPU(), A))) - D⁻¹ = spdiagm(0=>arch_array(CPU(), invdiag)) + D⁻¹ = spdiagm(0=>on_architecture(CPU(), invdiag)) Minv_cpu = D⁻¹ * (I - (A - D) * D⁻¹ + (A - D) * D⁻¹ * (A - D) * D⁻¹) Minv = arch_sparse_matrix(arch, Minv_cpu) end diff --git a/src/TurbulenceClosures/implicit_explicit_time_discretization.jl b/src/TurbulenceClosures/implicit_explicit_time_discretization.jl index facfbfc562..33fe206185 100644 --- a/src/TurbulenceClosures/implicit_explicit_time_discretization.jl +++ b/src/TurbulenceClosures/implicit_explicit_time_discretization.jl @@ -1,4 +1,4 @@ -using Oceananigans.Utils: arch_array +using Oceananigans.Utils: on_architecture using Oceananigans.Grids: AbstractGrid abstract type AbstractTimeDiscretization end diff --git a/src/TurbulenceClosures/turbulence_closure_implementations/convective_adjustment_vertical_diffusivity.jl b/src/TurbulenceClosures/turbulence_closure_implementations/convective_adjustment_vertical_diffusivity.jl index f8da912926..a973edc0a0 100644 --- a/src/TurbulenceClosures/turbulence_closure_implementations/convective_adjustment_vertical_diffusivity.jl +++ b/src/TurbulenceClosures/turbulence_closure_implementations/convective_adjustment_vertical_diffusivity.jl @@ -1,4 +1,4 @@ -using Oceananigans.Architectures: architecture, arch_array +using Oceananigans.Architectures: architecture, on_architecture using Oceananigans.AbstractOperations: KernelFunctionOperation using Oceananigans.BuoyancyModels: ∂z_b using Oceananigans.Operators: ℑzᵃᵃᶜ diff --git a/src/TurbulenceClosures/turbulence_closure_implementations/isopycnal_skew_symmetric_diffusivity.jl b/src/TurbulenceClosures/turbulence_closure_implementations/isopycnal_skew_symmetric_diffusivity.jl index ed63c418c3..e79a1a76d1 100644 --- a/src/TurbulenceClosures/turbulence_closure_implementations/isopycnal_skew_symmetric_diffusivity.jl +++ b/src/TurbulenceClosures/turbulence_closure_implementations/isopycnal_skew_symmetric_diffusivity.jl @@ -68,7 +68,7 @@ function with_tracers(tracers, closure_vector::ISSDVector) Ex = length(closure_vector) closure_vector = [with_tracers(tracers, closure_vector[i]) for i=1:Ex] - return arch_array(arch, closure_vector) + return on_architecture(arch, closure_vector) end # Note: computing diffusivities at cell centers for now. diff --git a/src/TurbulenceClosures/turbulence_closure_implementations/ri_based_vertical_diffusivity.jl b/src/TurbulenceClosures/turbulence_closure_implementations/ri_based_vertical_diffusivity.jl index db17a26370..21f44f3000 100644 --- a/src/TurbulenceClosures/turbulence_closure_implementations/ri_based_vertical_diffusivity.jl +++ b/src/TurbulenceClosures/turbulence_closure_implementations/ri_based_vertical_diffusivity.jl @@ -1,4 +1,4 @@ -using Oceananigans.Architectures: architecture, arch_array +using Oceananigans.Architectures: architecture, on_architecture using Oceananigans.BuoyancyModels: ∂z_b using Oceananigans.Operators using Oceananigans.Grids: inactive_node diff --git a/test/test_abstract_operations.jl b/test/test_abstract_operations.jl index 36e90d8f07..3e9c7e6cab 100644 --- a/test/test_abstract_operations.jl +++ b/test/test_abstract_operations.jl @@ -23,7 +23,7 @@ function x_derivative(a) dx_a = ∂x(a) arch = architecture(a) - one_two_three = arch_array(arch, [1, 2, 3]) + one_two_three = on_architecture(arch, [1, 2, 3]) for k in 1:3 interior(a)[:, 1, k] .= one_two_three @@ -38,7 +38,7 @@ function y_derivative(a) dy_a = ∂y(a) arch = architecture(a) - one_three_five = arch_array(arch, [1, 3, 5]) + one_three_five = on_architecture(arch, [1, 3, 5]) for k in 1:3 interior(a)[1, :, k] .= one_three_five @@ -53,7 +53,7 @@ function z_derivative(a) dz_a = ∂z(a) arch = architecture(a) - one_four_seven = arch_array(arch, [1, 4, 7]) + one_four_seven = on_architecture(arch, [1, 4, 7]) for k in 1:3 interior(a)[1, k, :] .= one_four_seven @@ -69,7 +69,7 @@ function x_derivative_cell(arch) a = Field{Center, Center, Center}(grid) dx_a = ∂x(a) - one_four_four = arch_array(arch, [1, 4, 4]) + one_four_four = on_architecture(arch, [1, 4, 4]) for k in 1:3 interior(a)[:, 1, k] .= one_four_four diff --git a/test/test_broadcasting.jl b/test/test_broadcasting.jl index 5e9b5aa639..6f2e8a6788 100644 --- a/test/test_broadcasting.jl +++ b/test/test_broadcasting.jl @@ -108,7 +108,7 @@ include("dependencies_for_runtests.jl") two_two_two_grid = RectilinearGrid(arch, size=(2, 2, 2), extent=(1, 1, 1)) c = CenterField(two_two_two_grid) - random_column = arch_array(arch, reshape(rand(2), 1, 1, 2)) + random_column = on_architecture(arch, reshape(rand(2), 1, 1, 2)) c .= random_column # broadcast to every horizontal column in c diff --git a/test/test_field.jl b/test/test_field.jl index a470c6265b..2392ccc896 100644 --- a/test/test_field.jl +++ b/test/test_field.jl @@ -34,7 +34,7 @@ function correct_field_value_was_set(grid, FieldType, val::Number) arch = architecture(grid) f = FieldType(grid) set!(f, val) - return all(interior(f) .≈ val * arch_array(arch, ones(size(f)))) + return all(interior(f) .≈ val * on_architecture(arch, ones(size(f)))) end function run_field_reduction_tests(FT, arch) @@ -58,10 +58,10 @@ function run_field_reduction_tests(FT, arch) c_vals = f.(nodes(c, reshape=true)...) # Convert to CuArray if needed. - u_vals = arch_array(arch, u_vals) - v_vals = arch_array(arch, v_vals) - w_vals = arch_array(arch, w_vals) - c_vals = arch_array(arch, c_vals) + u_vals = on_architecture(arch, u_vals) + v_vals = on_architecture(arch, v_vals) + w_vals = on_architecture(arch, w_vals) + c_vals = on_architecture(arch, c_vals) ϕs_vals = (u_vals, v_vals, w_vals, c_vals) @@ -149,11 +149,11 @@ function run_field_interpolation_tests(grid) zs = Array(reshape([-1.3, 1.23, 2.1], (1, 1, 3))) X = [(xs[i], ys[j], zs[k]) for i=1:3, j=1:3, k=1:3] - X = arch_array(arch, X) + X = on_architecture(arch, X) - xs = arch_array(arch, xs) - ys = arch_array(arch, ys) - zs = arch_array(arch, zs) + xs = on_architecture(arch, xs) + ys = on_architecture(arch, ys) + zs = on_architecture(arch, zs) CUDA.@allowscalar begin for f in (u, v, w, c) diff --git a/test/test_field_reductions.jl b/test/test_field_reductions.jl index 3319cdb9d4..f44712ab68 100644 --- a/test/test_field_reductions.jl +++ b/test/test_field_reductions.jl @@ -1,7 +1,7 @@ include("dependencies_for_runtests.jl") using Statistics -using Oceananigans.Architectures: arch_array +using Oceananigans.Architectures: on_architecture using Oceananigans.AbstractOperations: BinaryOperation using Oceananigans.Fields: ReducedField, CenterField, ZFaceField, compute_at!, @compute using Oceananigans.BoundaryConditions: fill_halo_regions! diff --git a/test/test_hydrostatic_free_surface_immersed_boundaries.jl b/test/test_hydrostatic_free_surface_immersed_boundaries.jl index 76953dddf0..d262813b66 100644 --- a/test/test_hydrostatic_free_surface_immersed_boundaries.jl +++ b/test/test_hydrostatic_free_surface_immersed_boundaries.jl @@ -65,7 +65,7 @@ using Oceananigans.TurbulenceClosures bathymetry = zeros(Nx, Ny) .- 4000 view(bathymetry, 31:34, 43:47) .= 0 - bathymetry = arch_array(arch, bathymetry) + bathymetry = on_architecture(arch, bathymetry) grid = ImmersedBoundaryGrid(underlying_grid, GridFittedBottom(bathymetry)) diff --git a/test/test_hydrostatic_free_surface_immersed_boundaries_implicit_solve.jl b/test/test_hydrostatic_free_surface_immersed_boundaries_implicit_solve.jl index 06e0021986..9cf8116bbb 100644 --- a/test/test_hydrostatic_free_surface_immersed_boundaries_implicit_solve.jl +++ b/test/test_hydrostatic_free_surface_immersed_boundaries_implicit_solve.jl @@ -1,7 +1,7 @@ include("dependencies_for_runtests.jl") using Oceananigans.ImmersedBoundaries: ImmersedBoundaryGrid, GridFittedBottom -using Oceananigans.Architectures: arch_array +using Oceananigans.Architectures: on_architecture using Oceananigans.TurbulenceClosures using Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_vertically_integrated_volume_flux!, compute_implicit_free_surface_right_hand_side!, @@ -31,7 +31,7 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_vertically_integ bottom = [-1. for j=1:Ny, i=1:Nx] bottom[imm1-1:imp1+1, jmm1-1:jmp1+1] .= 0 - B = arch_array(arch, bottom) + B = on_architecture(arch, bottom) grid = ImmersedBoundaryGrid(underlying_grid, GridFittedBottom(B)) free_surfaces = [ImplicitFreeSurface(solver_method=:HeptadiagonalIterativeSolver, gravitational_acceleration=1.0), diff --git a/test/test_lagrangian_particle_tracking.jl b/test/test_lagrangian_particle_tracking.jl index 9af8ba9f31..1e6c9b6ac4 100644 --- a/test/test_lagrangian_particle_tracking.jl +++ b/test/test_lagrangian_particle_tracking.jl @@ -2,7 +2,7 @@ include("dependencies_for_runtests.jl") using NCDatasets using StructArrays -using Oceananigans.Architectures: arch_array +using Oceananigans.Architectures: on_architecture struct TestParticle{T} x :: T @@ -54,9 +54,9 @@ function run_simple_particle_tracking_tests(arch, timestepper; vertically_stretc ##### Test default particle ##### - xs = arch_array(arch, 0.6*ones(P)) - ys = arch_array(arch, 0.58*ones(P)) - zs = arch_array(arch, 0.8*ones(P)) + xs = on_architecture(arch, 0.6*ones(P)) + ys = on_architecture(arch, 0.58*ones(P)) + zs = on_architecture(arch, 0.8*ones(P)) particles = LagrangianParticles(x=xs, y=ys, z=zs) @test particles isa LagrangianParticles @@ -80,7 +80,7 @@ function run_simple_particle_tracking_tests(arch, timestepper; vertically_stretc initial_z = CUDA.@allowscalar grid.zᵃᵃᶜ[grid.Nz - 1] top_boundary = CUDA.@allowscalar grid.zᵃᵃᶠ[grid.Nz + 1] - x, y, z = arch_array.(Ref(arch), ([0.0], [0.0], [initial_z])) + x, y, z = on_architecture.(Ref(arch), ([0.0], [0.0], [initial_z])) particles = LagrangianParticles(; x, y, z) u, v, w = VelocityFields(grid) @@ -102,13 +102,13 @@ function run_simple_particle_tracking_tests(arch, timestepper; vertically_stretc ##### Test custom particle "SpeedTrackingParticle" ##### - xs = arch_array(arch, zeros(P)) - ys = arch_array(arch, zeros(P)) - zs = arch_array(arch, 0.5 * ones(P)) - us = arch_array(arch, zeros(P)) - vs = arch_array(arch, zeros(P)) - ws = arch_array(arch, zeros(P)) - ss = arch_array(arch, zeros(P)) + xs = on_architecture(arch, zeros(P)) + ys = on_architecture(arch, zeros(P)) + zs = on_architecture(arch, 0.5 * ones(P)) + us = on_architecture(arch, zeros(P)) + vs = on_architecture(arch, zeros(P)) + ws = on_architecture(arch, zeros(P)) + ss = on_architecture(arch, zeros(P)) # Test custom constructor particles = StructArray{TestParticle}((xs, ys, zs, us, vs, ws, ss)) diff --git a/test/test_matrix_poisson_solver.jl b/test/test_matrix_poisson_solver.jl index 79cbe01d7f..ad878cf24b 100644 --- a/test/test_matrix_poisson_solver.jl +++ b/test/test_matrix_poisson_solver.jl @@ -1,6 +1,6 @@ using Oceananigans.Solvers: solve!, HeptadiagonalIterativeSolver, sparse_approximate_inverse using Oceananigans.Operators: volume, Δyᶠᶜᵃ, Δyᶜᶠᵃ, Δyᶜᶜᵃ, Δxᶠᶜᵃ, Δxᶜᶠᵃ, Δxᶜᶜᵃ, Δyᵃᶜᵃ, Δxᶜᵃᵃ, Δzᵃᵃᶠ, Δzᵃᵃᶜ, ∇²ᶜᶜᶜ -using Oceananigans.Architectures: arch_array +using Oceananigans.Architectures: on_architecture using Oceananigans.Grids: architecture using KernelAbstractions: @kernel, @index using Statistics, LinearAlgebra, SparseArrays @@ -21,10 +21,10 @@ function run_identity_operator_test(grid) solver = HeptadiagonalIterativeSolver((A, A, A, C, D), grid = grid) - b = arch_array(architecture(grid), rand(M)) + b = on_architecture(architecture(grid), rand(M)) arch = architecture(grid) - storage = arch_array(arch, zeros(size(b))) + storage = on_architecture(arch, zeros(size(b))) solve!(storage, solver, b, 1.0) @test norm(Array(storage) .- Array(b)) .< solver.tolerance @@ -44,11 +44,11 @@ end function compute_poisson_weights(grid) N = size(grid) - Ax = arch_array(architecture(grid), zeros(N...)) - Ay = arch_array(architecture(grid), zeros(N...)) - Az = arch_array(architecture(grid), zeros(N...)) - C = arch_array(architecture(grid), zeros(grid, N...)) - D = arch_array(architecture(grid), zeros(grid, N...)) + Ax = on_architecture(architecture(grid), zeros(N...)) + Ay = on_architecture(architecture(grid), zeros(N...)) + Az = on_architecture(architecture(grid), zeros(N...)) + C = on_architecture(architecture(grid), zeros(grid, N...)) + D = on_architecture(architecture(grid), zeros(grid, N...)) launch!(architecture(grid), grid, :xyz, _compute_poisson_weights, Ax, Ay, Az, grid) @@ -85,7 +85,7 @@ function run_poisson_equation_test(grid) ϕ_solution = CenterField(grid) arch = architecture(grid) - storage = arch_array(arch, zeros(size(rhs))) + storage = on_architecture(arch, zeros(size(rhs))) solve!(storage, solver, rhs, 1.0) set!(ϕ_solution, reshape(storage, solver.problem_size...)) fill_halo_regions!(ϕ_solution) diff --git a/test/test_multi_region_poisson_solver.jl b/test/test_multi_region_poisson_solver.jl index 04d8bf065d..1d0716a11f 100644 --- a/test/test_multi_region_poisson_solver.jl +++ b/test/test_multi_region_poisson_solver.jl @@ -2,7 +2,7 @@ include("dependencies_for_runtests.jl") using Oceananigans.Solvers: solve!, HeptadiagonalIterativeSolver, sparse_approximate_inverse using Oceananigans.Operators: volume, Δyᶠᶜᵃ, Δyᶜᶠᵃ, Δyᶜᶜᵃ, Δxᶠᶜᵃ, Δxᶜᶠᵃ, Δxᶜᶜᵃ, Δyᵃᶜᵃ, Δxᶜᵃᵃ, Δzᵃᵃᶠ, Δzᵃᵃᶜ, ∇²ᶜᶜᶜ -using Oceananigans.Architectures: arch_array +using Oceananigans.Architectures: on_architecture using KernelAbstractions: @kernel, @index using Statistics, LinearAlgebra, SparseArrays @@ -55,7 +55,7 @@ function compute_poisson_weights(grid) Ay = zeros(N...) Az = zeros(N...) C = zeros(grid, N...) - D = arch_array(grid.architecture, ones(N...)) + D = on_architecture(grid.architecture, ones(N...)) for k = 1:grid.Nz, j = 1:grid.Ny, i = 1:grid.Nx Ax[i, j, k] = Δzᵃᵃᶜ(i, j, k, grid) * Δyᶠᶜᵃ(i, j, k, grid) / Δxᶠᶜᵃ(i, j, k, grid) diff --git a/test/test_sewater_density.jl b/test/test_sewater_density.jl index adf448f6a7..8a67ad05be 100644 --- a/test/test_sewater_density.jl +++ b/test/test_sewater_density.jl @@ -22,7 +22,7 @@ function grid_size_value(arch, grid, value) value_array = fill(value, size(grid)) - return arch_array(arch, value_array) + return on_architecture(arch, value_array) end diff --git a/test/test_shallow_water_models.jl b/test/test_shallow_water_models.jl index c23fd79af6..2e70999669 100644 --- a/test/test_shallow_water_models.jl +++ b/test/test_shallow_water_models.jl @@ -65,7 +65,7 @@ function test_shallow_water_diffusion_cosine(grid, formulation, fieldname, ξ) field = model.velocities[fieldname] - interior(field) .= arch_array(architecture(grid), cos.(m * ξ)) + interior(field) .= on_architecture(architecture(grid), cos.(m * ξ)) update_state!(model) # Step forward with small time-step relative to diff. time-scale diff --git a/validation/barotropic_gyre/barotropic_gyre.jl b/validation/barotropic_gyre/barotropic_gyre.jl index 2d74476713..db641b63c5 100644 --- a/validation/barotropic_gyre/barotropic_gyre.jl +++ b/validation/barotropic_gyre/barotropic_gyre.jl @@ -34,7 +34,7 @@ underlying_grid = LatitudeLongitudeGrid(size = (Nx, Ny, 1), ## bathymetry = zeros(Nx, Ny) .- 4000 ## view(bathymetry, 31:34, 43:47) .= 0 -## bathymetry = arch_array(arch, bathymetry) +## bathymetry = on_architecture(arch, bathymetry) ## grid = ImmersedBoundaryGrid(underlying_grid, GridFittedBottom(bathymetry) ) grid = underlying_grid diff --git a/validation/multi_region/multi_region_near_global_quarter_degree.jl b/validation/multi_region/multi_region_near_global_quarter_degree.jl index d5cb752b33..067b6f4c70 100644 --- a/validation/multi_region/multi_region_near_global_quarter_degree.jl +++ b/validation/multi_region/multi_region_near_global_quarter_degree.jl @@ -7,7 +7,7 @@ using Oceananigans.Units using Oceananigans.MultiRegion using Oceananigans.MultiRegion: multi_region_object_from_array using Oceananigans.Fields: interpolate, Field -using Oceananigans.Architectures: arch_array +using Oceananigans.Architectures: on_architecture using Oceananigans.Coriolis: HydrostaticSphericalCoriolis using Oceananigans.BoundaryConditions using Oceananigans.ImmersedBoundaries: ImmersedBoundaryGrid, GridFittedBottom, inactive_node, peripheral_node @@ -94,7 +94,7 @@ T★ = file_temp["field"] S★ = file_salt["field"] # Remember the convention!! On the surface a negative flux increases a positive decreases -bathymetry = arch_array(arch, bathymetry) +bathymetry = on_architecture(arch, bathymetry) # Stretched faces taken from ECCO Version 4 (50 levels in the vertical) z_faces = file_z_faces["z_faces"][3:end] diff --git a/validation/shallow_water_model/near_global_shallow_water_quarter_degree.jl b/validation/shallow_water_model/near_global_shallow_water_quarter_degree.jl index 0cd7ccd929..00ed065c13 100644 --- a/validation/shallow_water_model/near_global_shallow_water_quarter_degree.jl +++ b/validation/shallow_water_model/near_global_shallow_water_quarter_degree.jl @@ -7,7 +7,7 @@ using Oceananigans.Units using Oceananigans.MultiRegion using Oceananigans.MultiRegion: multi_region_object_from_array using Oceananigans.Fields: interpolate, Field -using Oceananigans.Architectures: arch_array +using Oceananigans.Architectures: on_architecture using Oceananigans.Coriolis: HydrostaticSphericalCoriolis using Oceananigans.BoundaryConditions using Oceananigans.Grids: boundary_node, inactive_node, peripheral_node @@ -84,8 +84,8 @@ end # Files contain 1 year (1992) of 12 monthly averages τˣ = file_tau_x["field"] ./ reference_density τʸ = file_tau_y["field"] ./ reference_density -τˣ = arch_array(arch, τˣ) -τʸ = arch_array(arch, τʸ) +τˣ = on_architecture(arch, τˣ) +τʸ = on_architecture(arch, τʸ) bat = file_bathymetry["bathymetry"] boundary = Int.(bat .> 0) From 8e5691b10622d4c60b27dbc5de0e37713056be91 Mon Sep 17 00:00:00 2001 From: "Navid C. Constantinou" Date: Fri, 1 Mar 2024 10:19:48 +0200 Subject: [PATCH 187/567] central -> new-central --- .buildkite/distributed/JuliaProject.toml | 15 --------------- .buildkite/distributed/pipeline.yml | 9 +++------ 2 files changed, 3 insertions(+), 21 deletions(-) delete mode 100644 .buildkite/distributed/JuliaProject.toml diff --git a/.buildkite/distributed/JuliaProject.toml b/.buildkite/distributed/JuliaProject.toml deleted file mode 100644 index fff554e83a..0000000000 --- a/.buildkite/distributed/JuliaProject.toml +++ /dev/null @@ -1,15 +0,0 @@ -[extras] -CUDA_Runtime_jll = "76a88914-d11a-5bdc-97e0-2f5a05c973a2" -CUDA_Driver_jll = "4ee394cb-3365-5eb0-8335-949819d2adfc" -MPIPreferences = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267" - -[preferences.CUDA_Driver_jll] -compat = false - -[preferences.CUDA_Runtime_jll] -version = "12.2" -local = "true" - -[preferences.MPIPreferences] -_format = "1.0" -binary = "MPItrampoline_jll" diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 27325c245a..bc6dd5c456 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -1,17 +1,14 @@ agents: - queue: central + queue: new-central slurm_mem: 8G - modules: julia/1.10.0 cuda/12.2 ucx/1.14.1_cuda-12.2 openmpi/4.1.5_cuda-12.2 nsight-systems/2023.2.1 + modules: climacommon/2024_02_27 + env: JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite/distributed" OPENBLAS_NUM_THREADS: 1 - JULIA_NVTX_CALLBACKS: gc OMPI_MCA_opal_warn_on_missing_libcuda: 0 JULIA_CPU_TARGET: 'broadwell;skylake' - JULIA_CUDA_MEMORY_POOL: none - MPITRAMPOLINE_LIB: /groups/esm/software/MPIwrapper/ompi4.1.5_cuda-12.2/lib64/libmpiwrapper.so - MPITRAMPOLINE_MPIEXEC: /groups/esm/software/MPIwrapper/ompi4.1.5_cuda-12.2/bin/mpiwrapperexec steps: - label: "initialize" From 5ae1a3fdbee68113e3ed23a8d5202528312b4919 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Fri, 1 Mar 2024 09:51:36 -0500 Subject: [PATCH 188/567] couple of more additions --- src/Architectures.jl | 9 +++++++++ src/ImmersedBoundaries/partial_cell_bottom.jl | 3 +++ .../explicit_free_surface.jl | 4 ++++ .../implicit_free_surface.jl | 8 ++++++++ .../prescribed_hydrostatic_velocity_fields.jl | 8 +++++++- .../split_explicit_free_surface.jl | 7 +++++++ .../shallow_water_diffusion_operators.jl | 4 ++++ src/OutputReaders/field_time_series.jl | 11 +++++++++++ src/TurbulenceClosures/discrete_diffusion_function.jl | 4 ++++ .../turbulent_kinetic_energy_equation.jl | 5 +++++ .../scalar_biharmonic_diffusivity.jl | 6 ++++++ .../scalar_diffusivity.jl | 6 ++++++ 12 files changed, 74 insertions(+), 1 deletion(-) diff --git a/src/Architectures.jl b/src/Architectures.jl index aebc19f4bb..600417edb4 100644 --- a/src/Architectures.jl +++ b/src/Architectures.jl @@ -56,6 +56,15 @@ child_architecture(arch) = arch array_type(::CPU) = Array array_type(::GPU) = CuArray +# Fallback +on_architecture(arch, a) = a + +# Tupled implementation +@inline on_architecture(arch, t::Tuple{}) = () +@inline on_architecture(arch, t::Tuple) = Tuple(on_architecture(arch, elem) for elem in t) +@inline on_architecture(arch, nt::NamedTuple) = NamedTuple{keys(nt)}(on_architecture(arch, Tuple(nt))) + +# On architecture for array types on_architecture(::CPU, a::Array) = a on_architecture(::CPU, a::CuArray) = Array(a) on_architecture(::GPU, a::Array) = CuArray(a) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index 94fff4cbca..f6ba0368e3 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -81,6 +81,9 @@ end Adapt.adapt_structure(to, ib::PartialCellBottom) = PartialCellBottom(adapt(to, ib.bottom_height.data), ib.minimum_fractional_cell_height) +on_architecture(to, ib::PartialCellBottom) = PartialCellBottom(on_architecture(to, ib.bottom_height.data), + on_architecture(to, ib.minimum_fractional_cell_height)) + """ --x-- diff --git a/src/Models/HydrostaticFreeSurfaceModels/explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/explicit_free_surface.jl index 672322068f..3d210e6778 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/explicit_free_surface.jl @@ -24,6 +24,10 @@ ExplicitFreeSurface(; gravitational_acceleration=g_Earth) = Adapt.adapt_structure(to, free_surface::ExplicitFreeSurface) = ExplicitFreeSurface(Adapt.adapt(to, free_surface.η), free_surface.gravitational_acceleration) +on_architecture(to, free_surface::ExplicitFreeSurface) = + ExplicitFreeSurface(on_architecture(to, free_surface.η), + on_architecture(to, free_surface.gravitational_acceleration)) + ##### ##### Interface to HydrostaticFreeSurfaceModel ##### diff --git a/src/Models/HydrostaticFreeSurfaceModels/implicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/implicit_free_surface.jl index d835e2a646..d745d4dc0a 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/implicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/implicit_free_surface.jl @@ -81,6 +81,14 @@ Adapt.adapt_structure(to, free_surface::ImplicitFreeSurface) = ImplicitFreeSurface(Adapt.adapt(to, free_surface.η), free_surface.gravitational_acceleration, nothing, nothing, nothing, nothing) +on_architecture(to, free_surface::ImplicitFreeSurface) = + ImplicitFreeSurface(on_architecture(to, free_surface.η), + on_architecture(to, free_surface.gravitational_acceleration), + on_architecture(to, free_surface.barotropic_volume_flux), + on_architecture(to, free_surface.implicit_step_solver), + on_architecture(to, free_surface.solver_methods), + on_architecture(to, free_surface.solver_settings)) + # Internal function for HydrostaticFreeSurfaceModel function FreeSurface(free_surface::ImplicitFreeSurface{Nothing}, velocities, grid) η = FreeSurfaceDisplacementField(velocities, free_surface, grid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/prescribed_hydrostatic_velocity_fields.jl b/src/Models/HydrostaticFreeSurfaceModels/prescribed_hydrostatic_velocity_fields.jl index 9905bfc586..6d1dc41de0 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/prescribed_hydrostatic_velocity_fields.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/prescribed_hydrostatic_velocity_fields.jl @@ -111,7 +111,13 @@ Adapt.adapt_structure(to, velocities::PrescribedVelocityFields) = PrescribedVelocityFields(Adapt.adapt(to, velocities.u), Adapt.adapt(to, velocities.v), Adapt.adapt(to, velocities.w), - nothing) + nothing) # Why are parameters not passed here? They probably should... + +on_architecture(to, velocities::PrescribedVelocityFields) = + PrescribedVelocityFields(on_architecture(to, velocities.u), + on_architecture(to, velocities.v), + on_architecture(to, velocities.w), + on_architecture(to, velocities.parameters)) # If the model only tracks particles... do nothing but that!!! const OnlyParticleTrackingModel = HydrostaticFreeSurfaceModel{TS, E, A, S, G, T, V, B, R, F, P, U, C} where diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index d32b5786d8..315238277a 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -364,3 +364,10 @@ end Adapt.adapt_structure(to, free_surface::SplitExplicitFreeSurface) = SplitExplicitFreeSurface(Adapt.adapt(to, free_surface.η), nothing, nothing, free_surface.gravitational_acceleration, nothing) + +on_architecture(to, free_surface::SplitExplicitFreeSurface) = + SplitExplicitFreeSurface(on_architecture(to, free_surface.η), + on_architecture(to, free_surface.state), + on_architecture(to, free_surface.auxiliary), + on_architecture(to, free_surface.gravitational_acceleration), + on_architecture(to, free_surface.settings)) diff --git a/src/Models/ShallowWaterModels/shallow_water_diffusion_operators.jl b/src/Models/ShallowWaterModels/shallow_water_diffusion_operators.jl index c3f2f82729..6e107b4e76 100644 --- a/src/Models/ShallowWaterModels/shallow_water_diffusion_operators.jl +++ b/src/Models/ShallowWaterModels/shallow_water_diffusion_operators.jl @@ -51,6 +51,10 @@ viscosity(closure::ShallowWaterScalarDiffusivity, K) = closure.ν Adapt.adapt_structure(to, closure::ShallowWaterScalarDiffusivity{B}) where B = ShallowWaterScalarDiffusivity{B}(Adapt.adapt(to, closure.ν), Adapt.adapt(to, closure.ξ)) +on_architecture(to, closure::ShallowWaterScalarDiffusivity{B}) where B = + ShallowWaterScalarDiffusivity{B}(on_architecture(to, closure.ν), on_architecture(to, closure.ξ)) + + # The diffusivity for the shallow water model is calculated as h*ν in order to have a viscous term in the form # h⁻¹ ∇ ⋅ (hν t) where t is the 2D stress tensor plus a trace => t = ∇u + (∇u)ᵀ - ξI⋅(∇⋅u) diff --git a/src/OutputReaders/field_time_series.jl b/src/OutputReaders/field_time_series.jl index 46dab7332d..9d015223e7 100644 --- a/src/OutputReaders/field_time_series.jl +++ b/src/OutputReaders/field_time_series.jl @@ -267,6 +267,17 @@ mutable struct FieldTimeSeries{LX, LY, LZ, TI, K, I, D, G, ET, B, χ, P, N} <: A end end +on_architecture(to, fts::FieldTimeSeries{LX, LY, LZ}) where {LX, LY, LZ} = + FieldTimeSeries{LX, LY, LZ}(on_architecture(to, data), + on_architecture(to, grid), + on_architecture(to, backend), + on_architecture(to, bcs), + on_architecture(to, indices), + on_architecture(to, times), + on_architecture(to, path), + on_architecture(to, name), + on_architecture(to, time_indexing)) + ##### ##### Minimal implementation of FieldTimeSeries for use in GPU kernels ##### diff --git a/src/TurbulenceClosures/discrete_diffusion_function.jl b/src/TurbulenceClosures/discrete_diffusion_function.jl index 9e7a87c108..19d8203783 100644 --- a/src/TurbulenceClosures/discrete_diffusion_function.jl +++ b/src/TurbulenceClosures/discrete_diffusion_function.jl @@ -101,3 +101,7 @@ end Adapt.adapt_structure(to, dd::DiscreteDiffusionFunction{LX, LY, LZ}) where {LX, LY, LZ} = DiscreteDiffusionFunction{LX, LY, LZ}(Adapt.adapt(to, dd.func), Adapt.adapt(to, dd.parameters)) + +on_architecture(to, dd::DiscreteDiffusionFunction{LX, LY, LZ}) where {LX, LY, LZ} = + DiscreteDiffusionFunction{LX, LY, LZ}(on_architecture(to, dd.func), + on_architecture(to, dd.parameters)) diff --git a/src/TurbulenceClosures/turbulence_closure_implementations/CATKEVerticalDiffusivities/turbulent_kinetic_energy_equation.jl b/src/TurbulenceClosures/turbulence_closure_implementations/CATKEVerticalDiffusivities/turbulent_kinetic_energy_equation.jl index 58d82af1af..3acb1763bf 100644 --- a/src/TurbulenceClosures/turbulence_closure_implementations/CATKEVerticalDiffusivities/turbulent_kinetic_energy_equation.jl +++ b/src/TurbulenceClosures/turbulence_closure_implementations/CATKEVerticalDiffusivities/turbulent_kinetic_energy_equation.jl @@ -248,6 +248,11 @@ end TKETopBoundaryConditionParameters(adapt(to, p.top_tracer_boundary_conditions), adapt(to, p.top_velocity_boundary_conditions)) +@inline on_architecture(to, p::TKETopBoundaryConditionParameters) = + TKETopBoundaryConditionParameters(on_architecture(to, p.top_tracer_boundary_conditions), + on_architecture(to, p.top_velocity_boundary_conditions)) + + using Oceananigans.BoundaryConditions: Flux const TKEBoundaryFunction = DiscreteBoundaryFunction{<:TKETopBoundaryConditionParameters} const TKEBoundaryCondition = BoundaryCondition{<:Flux, <:TKEBoundaryFunction} diff --git a/src/TurbulenceClosures/turbulence_closure_implementations/scalar_biharmonic_diffusivity.jl b/src/TurbulenceClosures/turbulence_closure_implementations/scalar_biharmonic_diffusivity.jl index 00c109db85..05d565022e 100644 --- a/src/TurbulenceClosures/turbulence_closure_implementations/scalar_biharmonic_diffusivity.jl +++ b/src/TurbulenceClosures/turbulence_closure_implementations/scalar_biharmonic_diffusivity.jl @@ -109,3 +109,9 @@ function Adapt.adapt_structure(to, closure::ScalarBiharmonicDiffusivity{F, <:Any return ScalarBiharmonicDiffusivity{F, N}(ν, κ) end +function on_architecture(to, closure::ScalarBiharmonicDiffusivity{F, <:Any, <:Any, N}) where {F, N} + ν = on_architecture(to, closure.ν) + κ = on_architecture(to, closure.κ) + return ScalarBiharmonicDiffusivity{F, N}(ν, κ) +end + diff --git a/src/TurbulenceClosures/turbulence_closure_implementations/scalar_diffusivity.jl b/src/TurbulenceClosures/turbulence_closure_implementations/scalar_diffusivity.jl index df66ead80b..aaeb2d50e0 100644 --- a/src/TurbulenceClosures/turbulence_closure_implementations/scalar_diffusivity.jl +++ b/src/TurbulenceClosures/turbulence_closure_implementations/scalar_diffusivity.jl @@ -208,3 +208,9 @@ function Adapt.adapt_structure(to, closure::ScalarDiffusivity{TD, F, <:Any, <:An κ = Adapt.adapt(to, closure.κ) return ScalarDiffusivity{TD, F, N}(ν, κ) end + +function on_architecture(to, closure::ScalarDiffusivity{TD, F, <:Any, <:Any, N}) where {TD, F, N} + ν = on_architecture(to, closure.ν) + κ = on_architecture(to, closure.κ) + return ScalarDiffusivity{TD, F, N}(ν, κ) +end From 920ef5ba9aab267fe70bc16a916457d5962b78f8 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Fri, 1 Mar 2024 09:55:10 -0500 Subject: [PATCH 189/567] importing on_architecture here and there --- src/Fields/Fields.jl | 2 ++ src/Forcings/Forcings.jl | 1 + src/Forcings/advective_forcing.jl | 2 -- src/Forcings/continuous_forcing.jl | 2 -- src/Grids/Grids.jl | 2 +- .../HydrostaticFreeSurfaceModels.jl | 1 + src/TurbulenceClosures/TurbulenceClosures.jl | 1 + 7 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Fields/Fields.jl b/src/Fields/Fields.jl index b974f00700..cae3a72ee9 100644 --- a/src/Fields/Fields.jl +++ b/src/Fields/Fields.jl @@ -14,6 +14,8 @@ using Oceananigans.Grids using Oceananigans.BoundaryConditions using Oceananigans.Utils +import Oceananigans.Architectures: on_architecture + include("abstract_field.jl") include("constant_field.jl") include("function_field.jl") diff --git a/src/Forcings/Forcings.jl b/src/Forcings/Forcings.jl index c0d99c2a8c..0df515ba82 100644 --- a/src/Forcings/Forcings.jl +++ b/src/Forcings/Forcings.jl @@ -3,6 +3,7 @@ module Forcings export Forcing, ContinuousForcing, DiscreteForcing, Relaxation, GaussianMask, LinearTarget, AdvectiveForcing using Oceananigans.Fields +import Oceananigans.Architectures: on_architecture include("multiple_forcings.jl") include("continuous_forcing.jl") diff --git a/src/Forcings/advective_forcing.jl b/src/Forcings/advective_forcing.jl index ecf5191c47..1bf2c2083d 100644 --- a/src/Forcings/advective_forcing.jl +++ b/src/Forcings/advective_forcing.jl @@ -3,8 +3,6 @@ using Oceananigans.Fields: ZeroField, ConstantField using Oceananigans.Utils: SumOfArrays using Adapt -import Oceananigans.Architectures: on_architecture - maybe_constant_field(u) = u maybe_constant_field(u::Number) = ConstantField(u) diff --git a/src/Forcings/continuous_forcing.jl b/src/Forcings/continuous_forcing.jl index 6f3283358e..388ed3958d 100644 --- a/src/Forcings/continuous_forcing.jl +++ b/src/Forcings/continuous_forcing.jl @@ -5,8 +5,6 @@ using Oceananigans.Operators: assumed_field_location, index_and_interp_dependenc using Oceananigans.Fields: show_location using Oceananigans.Utils: user_function_arguments, tupleit, prettysummary -import Oceananigans.Architectures: on_architecture - """ ContinuousForcing{LX, LY, LZ, P, F, D, I, ℑ} diff --git a/src/Grids/Grids.jl b/src/Grids/Grids.jl index ae89b82d6f..586ef84f86 100644 --- a/src/Grids/Grids.jl +++ b/src/Grids/Grids.jl @@ -30,7 +30,7 @@ using Oceananigans using Oceananigans.Architectures import Base: size, length, eltype, show, - -import Oceananigans.Architectures: architecture +import Oceananigans.Architectures: architecture, on_architecture # Physical constants for constructors. const R_Earth = 6371.0e3 # [m] Mean radius of the Earth https://en.wikipedia.org/wiki/Earth diff --git a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl index f5e1f793b6..5c1531d8a2 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl @@ -17,6 +17,7 @@ using DocStringExtensions import Oceananigans: fields, prognostic_fields, initialize! import Oceananigans.Advection: cell_advection_timescale import Oceananigans.TimeSteppers: step_lagrangian_particles! +import Oceananigans.Architectures: on_architecture abstract type AbstractFreeSurface{E, G} end diff --git a/src/TurbulenceClosures/TurbulenceClosures.jl b/src/TurbulenceClosures/TurbulenceClosures.jl index 60c27b9d3a..f401a8eab3 100644 --- a/src/TurbulenceClosures/TurbulenceClosures.jl +++ b/src/TurbulenceClosures/TurbulenceClosures.jl @@ -51,6 +51,7 @@ using Oceananigans.Utils using Oceananigans.Architectures: AbstractArchitecture, device using Oceananigans.Fields: FunctionField import Oceananigans.Advection: required_halo_size +import Oceananigans.Architectures: on_architecture const VerticallyBoundedGrid{FT} = AbstractGrid{FT, <:Any, <:Any, <:Bounded} From ce6099b3513336cc6a2515dbea934de2f38b776a Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Fri, 1 Mar 2024 10:00:04 -0500 Subject: [PATCH 190/567] just testing an hypothesis here --- test/test_distributed_hydrostatic_model.jl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/test_distributed_hydrostatic_model.jl b/test/test_distributed_hydrostatic_model.jl index cce7805c1d..a3a0671967 100644 --- a/test/test_distributed_hydrostatic_model.jl +++ b/test/test_distributed_hydrostatic_model.jl @@ -90,8 +90,10 @@ for arch in archs immersed_grid = ImmersedBoundaryGrid(underlying_grid, GridFittedBottom(bottom); active_cells_map = true) - for grid in (underlying_grid, immersed_grid) - global_grid = reconstruct_global_grid(grid) + global_underlying_grid = reconstruct_global_grid(underlying_grid) + global_immersed_grid = ImmersedBoundaryGrid(global_underlying_grid, GridFittedBottom(bottom); active_cells_map = true) + + for (grid, global_grid) in zip((underlying_grid, immersed_grid), (global_underlying_grid, global_immersed_grid)) # "s" for "serial" computation us, vs, ws, cs, ηs = solid_body_rotation_test(global_grid) From 9f20a5581c3e214c059c171026e211e3cec4f2e4 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Fri, 1 Mar 2024 15:22:47 -0500 Subject: [PATCH 191/567] on_architecture for split explicit --- .../split_explicit_free_surface.jl | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index 315238277a..0f36e03fa3 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -365,9 +365,19 @@ Adapt.adapt_structure(to, free_surface::SplitExplicitFreeSurface) = SplitExplicitFreeSurface(Adapt.adapt(to, free_surface.η), nothing, nothing, free_surface.gravitational_acceleration, nothing) -on_architecture(to, free_surface::SplitExplicitFreeSurface) = - SplitExplicitFreeSurface(on_architecture(to, free_surface.η), - on_architecture(to, free_surface.state), - on_architecture(to, free_surface.auxiliary), - on_architecture(to, free_surface.gravitational_acceleration), - on_architecture(to, free_surface.settings)) +for Type in (:SplitExplicitFreeSurface, + :SplitExplicitSettings, + :SplitExplicitState, + :SplitExplicitAuxiliaryFields, + :FixedTimeStepSize, + :FixedSubstepNumber) + + @eval begin + function on_architecture(to, settings::$Type) + args = Tuple(on_architecture(to, prop) for prop in propertynames(settings)) + return SplitExplicitState(args...) + end + end +end + + \ No newline at end of file From 6e384db94da25bb600c83a178ffac8b233157618 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Fri, 1 Mar 2024 15:25:58 -0500 Subject: [PATCH 192/567] keep deprecated arch_array --- src/Architectures.jl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Architectures.jl b/src/Architectures.jl index 600417edb4..d5de78e970 100644 --- a/src/Architectures.jl +++ b/src/Architectures.jl @@ -129,5 +129,12 @@ end @inline convert_args(::GPU, args) = CUDA.cudaconvert(args) @inline convert_args(::GPU, args::Tuple) = map(CUDA.cudaconvert, args) +# Deprecated functions + +function arch_array(arch, arr) + @warn "`arch_array` is deprecated. Use `on_architecture` instead" + return on_architecture(arch, arr) +end + end # module From a596b7d9954ce7bc28258636ff8b98141d1a6246 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Fri, 1 Mar 2024 15:26:23 -0500 Subject: [PATCH 193/567] typo --- src/Architectures.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Architectures.jl b/src/Architectures.jl index d5de78e970..b5fe68fde5 100644 --- a/src/Architectures.jl +++ b/src/Architectures.jl @@ -132,7 +132,7 @@ end # Deprecated functions function arch_array(arch, arr) - @warn "`arch_array` is deprecated. Use `on_architecture` instead" + @warn "`arch_array` is deprecated. Use `on_architecture` instead." return on_architecture(arch, arr) end From b31d84a5d9ef7e961ce9093818f67753f06599f9 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Fri, 1 Mar 2024 15:50:15 -0500 Subject: [PATCH 194/567] bugfix --- test/test_distributed_hydrostatic_model.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_distributed_hydrostatic_model.jl b/test/test_distributed_hydrostatic_model.jl index a3a0671967..2dcfb14779 100644 --- a/test/test_distributed_hydrostatic_model.jl +++ b/test/test_distributed_hydrostatic_model.jl @@ -79,7 +79,7 @@ Ny = 32 for arch in archs @testset "Testing distributed solid body rotation" begin underlying_grid = LatitudeLongitudeGrid(arch, size = (Nx, Ny, 1), - halo = (3, 3, 3), + halo = (4, 4, 4), latitude = (-80, 80), longitude = (-160, 160), z = (-1, 0), From 0689b9e1e93d0b741aeb66ff376b7df7562fbd7c Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sun, 3 Mar 2024 14:58:01 -0500 Subject: [PATCH 195/567] remove useless methods --- src/Architectures.jl | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/Architectures.jl b/src/Architectures.jl index b5fe68fde5..7c6842eab1 100644 --- a/src/Architectures.jl +++ b/src/Architectures.jl @@ -56,7 +56,7 @@ child_architecture(arch) = arch array_type(::CPU) = Array array_type(::GPU) = CuArray -# Fallback +# Fallback on_architecture(arch, a) = a # Tupled implementation @@ -79,16 +79,6 @@ on_architecture(::CPU, a::SubArray{<:Any, <:Any, <:CuArray}) = Array(a) on_architecture(::GPU, a::SubArray{<:Any, <:Any, <:Array}) = CuArray(a) on_architecture(::CPU, a::SubArray{<:Any, <:Any, <:Array}) = a -on_architecture(::CPU, a::AbstractRange) = a -on_architecture(::CPU, ::Nothing) = nothing -on_architecture(::CPU, a::Number) = a -on_architecture(::CPU, a::Function) = a - -on_architecture(::GPU, a::AbstractRange) = a -on_architecture(::GPU, ::Nothing) = nothing -on_architecture(::GPU, a::Number) = a -on_architecture(::GPU, a::Function) = a - on_architecture(arch::CPU, a::OffsetArray) = OffsetArray(on_architecture(arch, a.parent), a.offsets...) on_architecture(arch::GPU, a::OffsetArray) = OffsetArray(on_architecture(arch, a.parent), a.offsets...) From aa3bf068b49a26af21f00f006884a1556c8262e9 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sun, 3 Mar 2024 14:58:16 -0500 Subject: [PATCH 196/567] no inline for tuples --- src/Architectures.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Architectures.jl b/src/Architectures.jl index 7c6842eab1..2282fb136f 100644 --- a/src/Architectures.jl +++ b/src/Architectures.jl @@ -60,9 +60,9 @@ array_type(::GPU) = CuArray on_architecture(arch, a) = a # Tupled implementation -@inline on_architecture(arch, t::Tuple{}) = () -@inline on_architecture(arch, t::Tuple) = Tuple(on_architecture(arch, elem) for elem in t) -@inline on_architecture(arch, nt::NamedTuple) = NamedTuple{keys(nt)}(on_architecture(arch, Tuple(nt))) +on_architecture(arch, t::Tuple{}) = () +on_architecture(arch, t::Tuple) = Tuple(on_architecture(arch, elem) for elem in t) +on_architecture(arch, nt::NamedTuple) = NamedTuple{keys(nt)}(on_architecture(arch, Tuple(nt))) # On architecture for array types on_architecture(::CPU, a::Array) = a From 44c569184b77150cba23c86ee90a12c3ee21065a Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 4 Mar 2024 10:19:10 -0500 Subject: [PATCH 197/567] AbstractSerialArchitecture to disambiguate --- src/Architectures.jl | 11 +++++++++-- src/Grids/latitude_longitude_grid.jl | 2 +- src/Grids/orthogonal_spherical_shell_grid.jl | 2 +- src/Grids/rectilinear_grid.jl | 2 +- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/Architectures.jl b/src/Architectures.jl index 2282fb136f..52e47c2d4c 100644 --- a/src/Architectures.jl +++ b/src/Architectures.jl @@ -16,20 +16,27 @@ Abstract supertype for architectures supported by Oceananigans. """ abstract type AbstractArchitecture end +""" + AbstractSerialArchitecture + +Abstract supertype for serial architectures supported by Oceananigans. +""" +abstract type AbstractSerialArchitecture <: AbstractArchitecture end + """ CPU <: AbstractArchitecture Run Oceananigans on one CPU node. Uses multiple threads if the environment variable `JULIA_NUM_THREADS` is set. """ -struct CPU <: AbstractArchitecture end +struct CPU <: AbstractSerialArchitecture end """ GPU <: AbstractArchitecture Run Oceananigans on a single NVIDIA CUDA GPU. """ -struct GPU <: AbstractArchitecture end +struct GPU <: AbstractSerialArchitecture end ##### ##### These methods are extended in DistributedComputations.jl diff --git a/src/Grids/latitude_longitude_grid.jl b/src/Grids/latitude_longitude_grid.jl index 3e5fafd730..91e28b1318 100644 --- a/src/Grids/latitude_longitude_grid.jl +++ b/src/Grids/latitude_longitude_grid.jl @@ -346,7 +346,7 @@ function with_halo(new_halo, old_grid::LatitudeLongitudeGrid) return new_grid end -function on_architecture(new_arch::AbstractArchitecture, old_grid::LatitudeLongitudeGrid) +function on_architecture(new_arch::AbstractSerialArchitecture, old_grid::LatitudeLongitudeGrid) old_properties = (old_grid.Δλᶠᵃᵃ, old_grid.Δλᶜᵃᵃ, old_grid.λᶠᵃᵃ, old_grid.λᶜᵃᵃ, old_grid.Δφᵃᶠᵃ, old_grid.Δφᵃᶜᵃ, old_grid.φᵃᶠᵃ, old_grid.φᵃᶜᵃ, old_grid.Δzᵃᵃᶠ, old_grid.Δzᵃᵃᶜ, old_grid.zᵃᵃᶠ, old_grid.zᵃᵃᶜ, diff --git a/src/Grids/orthogonal_spherical_shell_grid.jl b/src/Grids/orthogonal_spherical_shell_grid.jl index b8b3f0a723..b0ec80a6ca 100644 --- a/src/Grids/orthogonal_spherical_shell_grid.jl +++ b/src/Grids/orthogonal_spherical_shell_grid.jl @@ -873,7 +873,7 @@ function conformal_cubed_sphere_panel(filepath::AbstractString, architecture = C conformal_mapping) end -function on_architecture(arch::AbstractArchitecture, grid::OrthogonalSphericalShellGrid) +function on_architecture(arch::AbstractSerialArchitecture, grid::OrthogonalSphericalShellGrid) coordinates = (:λᶜᶜᵃ, :λᶠᶜᵃ, diff --git a/src/Grids/rectilinear_grid.jl b/src/Grids/rectilinear_grid.jl index 99f35afe09..6d22ad81e7 100644 --- a/src/Grids/rectilinear_grid.jl +++ b/src/Grids/rectilinear_grid.jl @@ -394,7 +394,7 @@ function with_halo(new_halo, old_grid::RectilinearGrid) return new_grid end -function on_architecture(new_arch::AbstractArchitecture, old_grid::RectilinearGrid) +function on_architecture(new_arch::AbstractSerialArchitecture, old_grid::RectilinearGrid) old_properties = (old_grid.Δxᶠᵃᵃ, old_grid.Δxᶜᵃᵃ, old_grid.xᶠᵃᵃ, old_grid.xᶜᵃᵃ, old_grid.Δyᵃᶠᵃ, old_grid.Δyᵃᶜᵃ, old_grid.yᵃᶠᵃ, old_grid.yᵃᶜᵃ, old_grid.Δzᵃᵃᶠ, old_grid.Δzᵃᵃᶜ, old_grid.zᵃᵃᶠ, old_grid.zᵃᵃᶜ) From 8b2322af36a2396d24367c30bc464e038cec8887 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 4 Mar 2024 10:20:21 -0500 Subject: [PATCH 198/567] a comment --- src/DistributedComputations/distributed_architectures.jl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/DistributedComputations/distributed_architectures.jl b/src/DistributedComputations/distributed_architectures.jl index 7284ae9b66..2aa393c14b 100644 --- a/src/DistributedComputations/distributed_architectures.jl +++ b/src/DistributedComputations/distributed_architectures.jl @@ -261,7 +261,10 @@ const SynchronizedDistributed = Distributed{<:Any, true} child_architecture(arch::Distributed) = arch.child_architecture device(arch::Distributed) = device(child_architecture(arch)) -on_architecture(arch::Distributed, A) = on_architecture(child_architecture(arch), A) + +# We do not support switching from distributed and serial through `on_architecture`. +# We only support moving a type from CPU to GPU and the other way around +on_architecture(arch::Distributed, A) = on_architecture(child_architecture(arch), A) zeros(FT, arch::Distributed, N...) = zeros(FT, child_architecture(arch), N...) array_type(arch::Distributed) = array_type(child_architecture(arch)) sync_device!(arch::Distributed) = sync_device!(arch.child_architecture) From 6787bf8b31bd01c32426705138c48c7a0ca55f1b Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 4 Mar 2024 10:26:51 -0500 Subject: [PATCH 199/567] export AbstractSerialArchitecture --- src/Architectures.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Architectures.jl b/src/Architectures.jl index 52e47c2d4c..25e4334076 100644 --- a/src/Architectures.jl +++ b/src/Architectures.jl @@ -1,7 +1,7 @@ module Architectures -export AbstractArchitecture -export CPU, GPU, MultiGPU +export AbstractArchitecture, AbstractSerialArchitecture +export CPU, GPU export device, architecture, array_type, on_architecture, unified_array, device_copy_to! using CUDA From db94bdd0f70933b5bec3a1b92f1c0b3c9f25fa5e Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 4 Mar 2024 11:26:13 -0500 Subject: [PATCH 200/567] import on_architecture in OutputReaders --- src/OutputReaders/field_time_series.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OutputReaders/field_time_series.jl b/src/OutputReaders/field_time_series.jl index 9d015223e7..af6a34fb46 100644 --- a/src/OutputReaders/field_time_series.jl +++ b/src/OutputReaders/field_time_series.jl @@ -20,7 +20,7 @@ using Oceananigans.Fields: interior_view_indices, index_binary_search, using Oceananigans.Units: Time using Oceananigans.Utils: launch! -import Oceananigans.Architectures: architecture +import Oceananigans.Architectures: architecture, on_architecture import Oceananigans.BoundaryConditions: fill_halo_regions!, BoundaryCondition, getbc import Oceananigans.Fields: Field, set!, interior, indices, interpolate! From a90c600bc17324ac8ccfd0ff32b1791138af8cd3 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 4 Mar 2024 11:56:55 -0500 Subject: [PATCH 201/567] still export arch_array --- src/Architectures.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Architectures.jl b/src/Architectures.jl index 25e4334076..4a0964d6b5 100644 --- a/src/Architectures.jl +++ b/src/Architectures.jl @@ -2,7 +2,8 @@ module Architectures export AbstractArchitecture, AbstractSerialArchitecture export CPU, GPU -export device, architecture, array_type, on_architecture, unified_array, device_copy_to! +export device, architecture, unified_array, device_copy_to! +export array_type, on_architecture, arch_array using CUDA using KernelAbstractions @@ -127,7 +128,6 @@ end @inline convert_args(::GPU, args::Tuple) = map(CUDA.cudaconvert, args) # Deprecated functions - function arch_array(arch, arr) @warn "`arch_array` is deprecated. Use `on_architecture` instead." return on_architecture(arch, arr) From 62180dc178556a36cbcc0b37b0fb06211b6318be Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 5 Mar 2024 10:06:34 -0500 Subject: [PATCH 202/567] Update src/AbstractOperations/grid_metrics.jl Co-authored-by: Navid C. Constantinou --- src/AbstractOperations/grid_metrics.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/AbstractOperations/grid_metrics.jl b/src/AbstractOperations/grid_metrics.jl index 822c6df173..a04dcb50b1 100644 --- a/src/AbstractOperations/grid_metrics.jl +++ b/src/AbstractOperations/grid_metrics.jl @@ -131,8 +131,8 @@ Adapt.adapt_structure(to, gm::GridMetricOperation{LX, LY, LZ}) where {LX, LY, LZ Adapt.adapt(to, gm.grid)) on_architecture(to, gm::GridMetricOperation{LX, LY, LZ}) where {LX, LY, LZ} = - GridMetricOperation{LX, LY, LZ}(on_architecture(to, gm.metric), - on_architecture(to, gm.grid)) + GridMetricOperation{LX, LY, LZ}(on_architecture(to, gm.metric), + on_architecture(to, gm.grid)) @inline Base.getindex(gm::GridMetricOperation, i, j, k) = gm.metric(i, j, k, gm.grid) From c5e68a62e2428619126cb43ebac56d1069b0732b Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 5 Mar 2024 10:06:48 -0500 Subject: [PATCH 203/567] Update src/Architectures.jl Co-authored-by: Navid C. Constantinou --- src/Architectures.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Architectures.jl b/src/Architectures.jl index 4a0964d6b5..75454e88c5 100644 --- a/src/Architectures.jl +++ b/src/Architectures.jl @@ -74,8 +74,9 @@ on_architecture(arch, nt::NamedTuple) = NamedTuple{keys(nt)}(on_architecture(ar # On architecture for array types on_architecture(::CPU, a::Array) = a -on_architecture(::CPU, a::CuArray) = Array(a) on_architecture(::GPU, a::Array) = CuArray(a) + +on_architecture(::CPU, a::CuArray) = Array(a) on_architecture(::GPU, a::CuArray) = a on_architecture(::CPU, a::BitArray) = a From cbedd8882211993a89dee01af3b7efe65046a466 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 5 Mar 2024 10:07:01 -0500 Subject: [PATCH 204/567] Update src/Architectures.jl Co-authored-by: Navid C. Constantinou --- src/Architectures.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Architectures.jl b/src/Architectures.jl index 75454e88c5..2046ec8b4f 100644 --- a/src/Architectures.jl +++ b/src/Architectures.jl @@ -70,7 +70,7 @@ on_architecture(arch, a) = a # Tupled implementation on_architecture(arch, t::Tuple{}) = () on_architecture(arch, t::Tuple) = Tuple(on_architecture(arch, elem) for elem in t) -on_architecture(arch, nt::NamedTuple) = NamedTuple{keys(nt)}(on_architecture(arch, Tuple(nt))) +on_architecture(arch, nt::NamedTuple) = NamedTuple{keys(nt)}(on_architecture(arch, Tuple(nt))) # On architecture for array types on_architecture(::CPU, a::Array) = a From f2da3291b35f2660e73be2b33e141549f6bb8f77 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 5 Mar 2024 10:07:16 -0500 Subject: [PATCH 205/567] Update src/Advection/weno_reconstruction.jl Co-authored-by: Navid C. Constantinou --- src/Advection/weno_reconstruction.jl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Advection/weno_reconstruction.jl b/src/Advection/weno_reconstruction.jl index 0ad255a68d..49e00f9a0b 100644 --- a/src/Advection/weno_reconstruction.jl +++ b/src/Advection/weno_reconstruction.jl @@ -161,12 +161,12 @@ Adapt.adapt_structure(to, scheme::WENO{N, FT, XT, YT, ZT, WF, PP}) where {N, FT, Adapt.adapt(to, scheme.advecting_velocity_scheme)) on_architecture(to, scheme::WENO{N, FT, XT, YT, ZT, WF, PP}) where {N, FT, XT, YT, ZT, WF, PP} = - WENO{N, FT, WF}(on_architecture(to, scheme.coeff_xᶠᵃᵃ), on_architecture(to, scheme.coeff_xᶜᵃᵃ), - on_architecture(to, scheme.coeff_yᵃᶠᵃ), on_architecture(to, scheme.coeff_yᵃᶜᵃ), - on_architecture(to, scheme.coeff_zᵃᵃᶠ), on_architecture(to, scheme.coeff_zᵃᵃᶜ), - on_architecture(to, scheme.bounds), - on_architecture(to, scheme.buffer_scheme), - on_architecture(to, scheme.advecting_velocity_scheme)) + WENO{N, FT, WF}(on_architecture(to, scheme.coeff_xᶠᵃᵃ), on_architecture(to, scheme.coeff_xᶜᵃᵃ), + on_architecture(to, scheme.coeff_yᵃᶠᵃ), on_architecture(to, scheme.coeff_yᵃᶜᵃ), + on_architecture(to, scheme.coeff_zᵃᵃᶠ), on_architecture(to, scheme.coeff_zᵃᵃᶜ), + on_architecture(to, scheme.bounds), + on_architecture(to, scheme.buffer_scheme), + on_architecture(to, scheme.advecting_velocity_scheme)) # Retrieve precomputed coefficients (+2 for julia's 1 based indices) @inline retrieve_coeff(scheme::WENO, r, ::Val{1}, i, ::Type{Face}) = @inbounds scheme.coeff_xᶠᵃᵃ[r+2][i] From c1d0423b348000070843818b828ce1f05639667d Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 5 Mar 2024 10:08:03 -0500 Subject: [PATCH 206/567] Update src/Architectures.jl Co-authored-by: Navid C. Constantinou --- src/Architectures.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Architectures.jl b/src/Architectures.jl index 2046ec8b4f..1ede1f515c 100644 --- a/src/Architectures.jl +++ b/src/Architectures.jl @@ -68,7 +68,6 @@ array_type(::GPU) = CuArray on_architecture(arch, a) = a # Tupled implementation -on_architecture(arch, t::Tuple{}) = () on_architecture(arch, t::Tuple) = Tuple(on_architecture(arch, elem) for elem in t) on_architecture(arch, nt::NamedTuple) = NamedTuple{keys(nt)}(on_architecture(arch, Tuple(nt))) From 1a7debe4a917d15ef833ac7057559c6ade76d843 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 5 Mar 2024 10:35:24 -0500 Subject: [PATCH 207/567] some changes to distributed on_architecture --- src/Architectures.jl | 8 ++-- .../distributed_architectures.jl | 3 -- .../distributed_on_architecture.jl | 44 +++++++++++++++++++ 3 files changed, 47 insertions(+), 8 deletions(-) create mode 100644 src/DistributedComputations/distributed_on_architecture.jl diff --git a/src/Architectures.jl b/src/Architectures.jl index 4a0964d6b5..10e048f37a 100644 --- a/src/Architectures.jl +++ b/src/Architectures.jl @@ -68,9 +68,8 @@ array_type(::GPU) = CuArray on_architecture(arch, a) = a # Tupled implementation -on_architecture(arch, t::Tuple{}) = () -on_architecture(arch, t::Tuple) = Tuple(on_architecture(arch, elem) for elem in t) -on_architecture(arch, nt::NamedTuple) = NamedTuple{keys(nt)}(on_architecture(arch, Tuple(nt))) +on_architecture(arch::AbstractSerialArchitecture, t::Tuple) = Tuple(on_architecture(arch, elem) for elem in t) +on_architecture(arch::AbstractSerialArchitecture, nt::NamedTuple) = NamedTuple{keys(nt)}(on_architecture(arch, Tuple(nt))) # On architecture for array types on_architecture(::CPU, a::Array) = a @@ -87,8 +86,7 @@ on_architecture(::CPU, a::SubArray{<:Any, <:Any, <:CuArray}) = Array(a) on_architecture(::GPU, a::SubArray{<:Any, <:Any, <:Array}) = CuArray(a) on_architecture(::CPU, a::SubArray{<:Any, <:Any, <:Array}) = a -on_architecture(arch::CPU, a::OffsetArray) = OffsetArray(on_architecture(arch, a.parent), a.offsets...) -on_architecture(arch::GPU, a::OffsetArray) = OffsetArray(on_architecture(arch, a.parent), a.offsets...) +on_architecture(arch::AbstractSerialArchitecture, a::OffsetArray) = OffsetArray(on_architecture(arch, a.parent), a.offsets...) cpu_architecture(::CPU) = CPU() cpu_architecture(::GPU) = CPU() diff --git a/src/DistributedComputations/distributed_architectures.jl b/src/DistributedComputations/distributed_architectures.jl index 2aa393c14b..73ab013355 100644 --- a/src/DistributedComputations/distributed_architectures.jl +++ b/src/DistributedComputations/distributed_architectures.jl @@ -262,9 +262,6 @@ const SynchronizedDistributed = Distributed{<:Any, true} child_architecture(arch::Distributed) = arch.child_architecture device(arch::Distributed) = device(child_architecture(arch)) -# We do not support switching from distributed and serial through `on_architecture`. -# We only support moving a type from CPU to GPU and the other way around -on_architecture(arch::Distributed, A) = on_architecture(child_architecture(arch), A) zeros(FT, arch::Distributed, N...) = zeros(FT, child_architecture(arch), N...) array_type(arch::Distributed) = array_type(child_architecture(arch)) sync_device!(arch::Distributed) = sync_device!(arch.child_architecture) diff --git a/src/DistributedComputations/distributed_on_architecture.jl b/src/DistributedComputations/distributed_on_architecture.jl new file mode 100644 index 0000000000..15ceed1e62 --- /dev/null +++ b/src/DistributedComputations/distributed_on_architecture.jl @@ -0,0 +1,44 @@ +import Oceananigans.Architectures: on_architecture + +# We do not support switching from distributed and serial through `on_architecture`. +# We only support moving a type from CPU to GPU and the other way around +# TODO: support changing the number of ranks / the partitioning? +on_architecture(arch::Distributed, A) = on_architecture(child_architecture(arch), A) + +function on_architecture(new_arch::Distributed, old_grid::LatitudeLongitudeGrid) + child_arch = child_architecture(new_arch) + old_properties = (old_grid.Δλᶠᵃᵃ, old_grid.Δλᶜᵃᵃ, old_grid.λᶠᵃᵃ, old_grid.λᶜᵃᵃ, + old_grid.Δφᵃᶠᵃ, old_grid.Δφᵃᶜᵃ, old_grid.φᵃᶠᵃ, old_grid.φᵃᶜᵃ, + old_grid.Δzᵃᵃᶠ, old_grid.Δzᵃᵃᶜ, old_grid.zᵃᵃᶠ, old_grid.zᵃᵃᶜ, + old_grid.Δxᶠᶜᵃ, old_grid.Δxᶜᶠᵃ, old_grid.Δxᶠᶠᵃ, old_grid.Δxᶜᶜᵃ, + old_grid.Δyᶠᶜᵃ, old_grid.Δyᶜᶠᵃ, + old_grid.Azᶠᶜᵃ, old_grid.Azᶜᶠᵃ, old_grid.Azᶠᶠᵃ, old_grid.Azᶜᶜᵃ) + + new_properties = Tuple(on_architecture(child_arch, p) for p in old_properties) + + TX, TY, TZ = topology(old_grid) + + return LatitudeLongitudeGrid{TX, TY, TZ}(new_arch, + old_grid.Nx, old_grid.Ny, old_grid.Nz, + old_grid.Hx, old_grid.Hy, old_grid.Hz, + old_grid.Lx, old_grid.Ly, old_grid.Lz, + new_properties..., + old_grid.radius) +end + +function on_architecture(new_arch::Distributed, old_grid::RectilinearGrid) + child_arch = child_architecture(new_arch) + old_properties = (old_grid.Δxᶠᵃᵃ, old_grid.Δxᶜᵃᵃ, old_grid.xᶠᵃᵃ, old_grid.xᶜᵃᵃ, + old_grid.Δyᵃᶠᵃ, old_grid.Δyᵃᶜᵃ, old_grid.yᵃᶠᵃ, old_grid.yᵃᶜᵃ, + old_grid.Δzᵃᵃᶠ, old_grid.Δzᵃᵃᶜ, old_grid.zᵃᵃᶠ, old_grid.zᵃᵃᶜ) + + new_properties = Tuple(on_architecture(child_arch, p) for p in old_properties) + + TX, TY, TZ = topology(old_grid) + + return RectilinearGrid{TX, TY, TZ}(new_arch, + old_grid.Nx, old_grid.Ny, old_grid.Nz, + old_grid.Hx, old_grid.Hy, old_grid.Hz, + old_grid.Lx, old_grid.Ly, old_grid.Lz, + new_properties...) +end From d6fbc1f0fb3ec577c153dfd2b7b4408df71ff97a Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 5 Mar 2024 10:36:23 -0500 Subject: [PATCH 208/567] remove using on_architecture --- src/Advection/Advection.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Advection/Advection.jl b/src/Advection/Advection.jl index d949070438..b8d7d96469 100644 --- a/src/Advection/Advection.jl +++ b/src/Advection/Advection.jl @@ -32,7 +32,7 @@ using OffsetArrays using Oceananigans.Grids using Oceananigans.Grids: with_halo, coordinates -using Oceananigans.Architectures: on_architecture, architecture, CPU +using Oceananigans.Architectures: architecture, CPU using Oceananigans.Operators From 9672946506f7f5c21066b8f25e0a8e5553ecee5f Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 5 Mar 2024 10:37:41 -0500 Subject: [PATCH 209/567] remove wrong docstring --- src/AbstractOperations/derivatives.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/AbstractOperations/derivatives.jl b/src/AbstractOperations/derivatives.jl index 751e794b94..ee38f48fc8 100644 --- a/src/AbstractOperations/derivatives.jl +++ b/src/AbstractOperations/derivatives.jl @@ -118,7 +118,7 @@ compute_at!(∂::Derivative, time) = compute_at!(∂.arg, time) ##### GPU capabilities ##### -"Adapt `Derivative` to work on the GPU via CUDAnative and CUDAdrv." +"Adapt `Derivative` to work on the GPU." Adapt.adapt_structure(to, deriv::Derivative{LX, LY, LZ}) where {LX, LY, LZ} = Derivative{LX, LY, LZ}(Adapt.adapt(to, deriv.∂), Adapt.adapt(to, deriv.arg), @@ -126,7 +126,6 @@ Adapt.adapt_structure(to, deriv::Derivative{LX, LY, LZ}) where {LX, LY, LZ} = nothing, Adapt.adapt(to, deriv.grid)) -"Adapt `Derivative` to work on the GPU via CUDAnative and CUDAdrv." on_architecture(to, deriv::Derivative{LX, LY, LZ}) where {LX, LY, LZ} = Derivative{LX, LY, LZ}(on_architecture(to, deriv.∂), on_architecture(to, deriv.arg), From 41880bbc28b9509ba43a2d5acd5e469ea6746815 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 5 Mar 2024 10:39:55 -0500 Subject: [PATCH 210/567] some more cleaning --- src/Architectures.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Architectures.jl b/src/Architectures.jl index d3b5f838a1..9b03fbf7fa 100644 --- a/src/Architectures.jl +++ b/src/Architectures.jl @@ -72,8 +72,8 @@ on_architecture(arch::AbstractSerialArchitecture, t::Tuple) = Tuple(on_architect on_architecture(arch::AbstractSerialArchitecture, nt::NamedTuple) = NamedTuple{keys(nt)}(on_architecture(arch, Tuple(nt))) # On architecture for array types -on_architecture(::CPU, a::Array) = a -on_architecture(::GPU, a::Array) = CuArray(a) +on_architecture(::CPU, a::Array) = a +on_architecture(::GPU, a::Array) = CuArray(a) on_architecture(::CPU, a::CuArray) = Array(a) on_architecture(::GPU, a::CuArray) = a @@ -81,11 +81,11 @@ on_architecture(::GPU, a::CuArray) = a on_architecture(::CPU, a::BitArray) = a on_architecture(::GPU, a::BitArray) = CuArray(a) -on_architecture(::GPU, a::SubArray{<:Any, <:Any, <:CuArray}) = a on_architecture(::CPU, a::SubArray{<:Any, <:Any, <:CuArray}) = Array(a) +on_architecture(::GPU, a::SubArray{<:Any, <:Any, <:CuArray}) = a -on_architecture(::GPU, a::SubArray{<:Any, <:Any, <:Array}) = CuArray(a) on_architecture(::CPU, a::SubArray{<:Any, <:Any, <:Array}) = a +on_architecture(::GPU, a::SubArray{<:Any, <:Any, <:Array}) = CuArray(a) on_architecture(arch::AbstractSerialArchitecture, a::OffsetArray) = OffsetArray(on_architecture(arch, a.parent), a.offsets...) From 92f2644ca2b5be5572bc53219885daec58bd9027 Mon Sep 17 00:00:00 2001 From: "Navid C. Constantinou" Date: Tue, 5 Mar 2024 22:05:38 +0200 Subject: [PATCH 211/567] include("distributed_on_architecture.jl") --- src/DistributedComputations/DistributedComputations.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/DistributedComputations/DistributedComputations.jl b/src/DistributedComputations/DistributedComputations.jl index 70dbc90992..f80cd7973e 100644 --- a/src/DistributedComputations/DistributedComputations.jl +++ b/src/DistributedComputations/DistributedComputations.jl @@ -12,6 +12,7 @@ using Oceananigans.Utils using Oceananigans.Grids include("distributed_architectures.jl") +include("distributed_on_architecture.jl") include("partition_assemble.jl") include("distributed_grids.jl") include("distributed_kernel_launching.jl") From 3e4c6c2257824281fae96a3cd86be11695f45c40 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 5 Mar 2024 15:35:34 -0500 Subject: [PATCH 212/567] add distributed_on_architecture --- src/DistributedComputations/DistributedComputations.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/DistributedComputations/DistributedComputations.jl b/src/DistributedComputations/DistributedComputations.jl index 70dbc90992..b1e224c607 100644 --- a/src/DistributedComputations/DistributedComputations.jl +++ b/src/DistributedComputations/DistributedComputations.jl @@ -14,6 +14,7 @@ using Oceananigans.Grids include("distributed_architectures.jl") include("partition_assemble.jl") include("distributed_grids.jl") +include("distributed_on_architecture.jl") include("distributed_kernel_launching.jl") include("halo_communication_bcs.jl") include("distributed_fields.jl") From 362a26853ecce67fd665307166d72c8090a2caff Mon Sep 17 00:00:00 2001 From: "Navid C. Constantinou" Date: Wed, 6 Mar 2024 13:10:10 +0200 Subject: [PATCH 213/567] bump patch release --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 2004614377..2353752cc1 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "Oceananigans" uuid = "9e8cae18-63c1-5223-a75c-80ca9d6e9a09" authors = ["Climate Modeling Alliance and contributors"] -version = "0.90.9" +version = "0.90.10" [deps] Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" From 308f64fb491f8da50450ee1585583bb9249553df Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Wed, 6 Mar 2024 13:54:57 -0500 Subject: [PATCH 214/567] bugfix --- .../split_explicit_free_surface_kernels.jl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index e2c844ef6e..7bf8d493c2 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -227,8 +227,8 @@ end i, j = active_linear_index_to_tuple(idx, ZColumnMap(), grid) k_top = grid.Nz + 1 - @inbounds scalingᶠᶜᶜ = dynamic_column_heightᶠᶜᶜ(i, j, k_top, grid, Hᶠᶜ, η̅) / Hᶠᶜ[i, j, 1] - @inbounds scalingᶜᶠᶜ = dynamic_column_heightᶜᶠᶜ(i, j, k_top, grid, Hᶜᶠ, η̅) / Hᶜᶠ[i, j, 1] + @inbounds scalingᶠᶜᶜ = dynamic_column_heightᶠᶜ(i, j, k_top, grid, Hᶠᶜ, η̅) / Hᶠᶜ[i, j, 1] + @inbounds scalingᶜᶠᶜ = dynamic_column_heightᶜᶠ(i, j, k_top, grid, Hᶜᶠ, η̅) / Hᶜᶠ[i, j, 1] # hand unroll first loop @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δzᶠᶜᶜ_reference(i, j, 1, grid) * scalingᶠᶜᶜ @@ -507,7 +507,6 @@ function update_∂t_∂s!(∂t_∂s, parameters, grid, sⁿ, s⁻, Δt, fs::Spl return nothing end - @kernel function _update_∂t_∂s_split_explicit!(∂t_∂s, U̅, V̅, Hᶜᶜ, grid) i, j = @index(Global, NTuple) k_top = grid.Nz + 1 From 750caa19d50dc3d719cfe25d3b624d6224b63b17 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Wed, 6 Mar 2024 15:12:55 -0500 Subject: [PATCH 215/567] small bugfix --- .../HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 35e6119f5c..9ea2f9e994 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -60,8 +60,8 @@ end @kernel function _initialize_zstar!(ΔzF, ΔzC, grid) i, j, k = @index(Global, NTuple) - @inbounds ΔzF[i, j, k] = grid.Δzᵃᵃᶠ[k] - @inbounds ΔzC[i, j, k] = grid.Δzᵃᵃᶜ[k] + @inbounds ΔzF[i, j, k] = Δzᶜᶜᶠ(i, j, k, grid) + @inbounds ΔzC[i, j, k] = Δzᶜᶜᶜ(i, j, k, grid) end ##### From 13bc75db83dfe7e972cc50e80ada6ed02a51161b Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Wed, 6 Mar 2024 15:13:11 -0500 Subject: [PATCH 216/567] temporary addition of FFMPEG to remove --- Manifest.toml | 237 +++++++++++++++++++++++++++++++++++++++++++++++++- Project.toml | 1 + 2 files changed, 236 insertions(+), 2 deletions(-) diff --git a/Manifest.toml b/Manifest.toml index 6d88d843c4..c7390c435f 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -1,8 +1,8 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.10.1" +julia_version = "1.10.2" manifest_format = "2.0" -project_hash = "04d395caf937b0921325a77873167e8baa293a99" +project_hash = "f944c31d84f25261bfb827a874135fa96958b69e" [[deps.AbstractFFTs]] deps = ["LinearAlgebra"] @@ -127,6 +127,12 @@ git-tree-sha1 = "8e25c009d2bf16c2c31a70a6e9e8939f7325cc84" uuid = "76a88914-d11a-5bdc-97e0-2f5a05c973a2" version = "0.11.1+0" +[[deps.Cairo_jll]] +deps = ["Artifacts", "Bzip2_jll", "CompilerSupportLibraries_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "JLLWrappers", "LZO_jll", "Libdl", "Pixman_jll", "Pkg", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Zlib_jll", "libpng_jll"] +git-tree-sha1 = "4b859a208b2397a7a623a03449e4636bdb17bcf2" +uuid = "83423d85-b0ee-5818-9007-b63ccbeb887a" +version = "1.16.1+1" + [[deps.ColorTypes]] deps = ["FixedPointNumbers", "Random"] git-tree-sha1 = "eb7f0f8307f71fac7c606984ea5fb2817275d6e4" @@ -251,11 +257,29 @@ git-tree-sha1 = "71c79e77221ab3a29918aaf6db4f217b89138608" uuid = "b305315f-e792-5b7a-8f41-49f472929428" version = "1.0.1" +[[deps.Expat_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "4558ab818dcceaab612d1bb8c19cee87eda2b83c" +uuid = "2e619515-83b5-522b-bb60-26c02a35a201" +version = "2.5.0+0" + [[deps.ExprTools]] git-tree-sha1 = "27415f162e6028e81c72b82ef756bf321213b6ec" uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04" version = "0.1.10" +[[deps.FFMPEG]] +deps = ["FFMPEG_jll"] +git-tree-sha1 = "b57e3acbe22f8484b4b5ff66a7499717fe1a9cc8" +uuid = "c87230d0-a227-11e9-1b43-d7ebe4e7570a" +version = "0.4.1" + +[[deps.FFMPEG_jll]] +deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "JLLWrappers", "LAME_jll", "Libdl", "Ogg_jll", "OpenSSL_jll", "Opus_jll", "PCRE2_jll", "Zlib_jll", "libaom_jll", "libass_jll", "libfdk_aac_jll", "libvorbis_jll", "x264_jll", "x265_jll"] +git-tree-sha1 = "466d45dc38e15794ec7d5d63ec03d776a9aff36e" +uuid = "b22a6f82-2f65-5046-a5b2-351ab43fb4e5" +version = "4.4.4+1" + [[deps.FFTW]] deps = ["AbstractFFTs", "FFTW_jll", "LinearAlgebra", "MKL_jll", "Preferences", "Reexport"] git-tree-sha1 = "4820348781ae578893311153d69049a93d05f39d" @@ -283,6 +307,24 @@ git-tree-sha1 = "335bfdceacc84c5cdf16aadc768aa5ddfc5383cc" uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93" version = "0.8.4" +[[deps.Fontconfig_jll]] +deps = ["Artifacts", "Bzip2_jll", "Expat_jll", "FreeType2_jll", "JLLWrappers", "Libdl", "Libuuid_jll", "Pkg", "Zlib_jll"] +git-tree-sha1 = "21efd19106a55620a188615da6d3d06cd7f6ee03" +uuid = "a3f928ae-7b40-5064-980b-68af3947d34b" +version = "2.13.93+0" + +[[deps.FreeType2_jll]] +deps = ["Artifacts", "Bzip2_jll", "JLLWrappers", "Libdl", "Zlib_jll"] +git-tree-sha1 = "d8db6a5a2fe1381c1ea4ef2cab7c69c2de7f9ea0" +uuid = "d7e528f0-a631-5988-bf34-fe36492bcfd7" +version = "2.13.1+0" + +[[deps.FriBidi_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "aa31987c2ba8704e23c6c8ba8a4f769d5d7e4f91" +uuid = "559328eb-81f9-559d-9380-de523a88c83c" +version = "1.0.10+0" + [[deps.Future]] deps = ["Random"] uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" @@ -310,6 +352,18 @@ git-tree-sha1 = "a846f297ce9d09ccba02ead0cae70690e072a119" uuid = "61eb1bfa-7361-4325-ad38-22787b887f55" version = "0.25.0" +[[deps.Gettext_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Libiconv_jll", "Pkg", "XML2_jll"] +git-tree-sha1 = "9b02998aba7bf074d14de89f9d37ca24a1a0b046" +uuid = "78b55507-aeef-58d4-861c-77aaff3498b1" +version = "0.21.0+0" + +[[deps.Glib_jll]] +deps = ["Artifacts", "Gettext_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Libiconv_jll", "Libmount_jll", "PCRE2_jll", "Zlib_jll"] +git-tree-sha1 = "e94c92c7bf4819685eb80186d51c43e71d4afa17" +uuid = "7746bdde-850d-59dc-9ae8-88ece973131d" +version = "2.76.5+0" + [[deps.Glob]] git-tree-sha1 = "97285bbd5230dd766e9ef6749b80fc617126d496" uuid = "c27321d9-0574-5035-807b-f59d2c89b15c" @@ -321,12 +375,24 @@ git-tree-sha1 = "266fe9b2335527cbf569ba4fd0979e3d8c6fd491" uuid = "0951126a-58fd-58f1-b5b3-b08c7c4a876d" version = "3.7.8+1" +[[deps.Graphite2_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "344bf40dcab1073aca04aa0df4fb092f920e4011" +uuid = "3b182d85-2403-5c21-9c21-1e1f0cc25472" +version = "1.3.14+0" + [[deps.HDF5_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "LazyArtifacts", "LibCURL_jll", "Libdl", "MPICH_jll", "MPIPreferences", "MPItrampoline_jll", "MicrosoftMPI_jll", "OpenMPI_jll", "OpenSSL_jll", "TOML", "Zlib_jll", "libaec_jll"] git-tree-sha1 = "e4591176488495bf44d7456bd73179d87d5e6eab" uuid = "0234f1f7-429e-5d53-9886-15a909be8d59" version = "1.14.3+1" +[[deps.HarfBuzz_jll]] +deps = ["Artifacts", "Cairo_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "Graphite2_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Pkg"] +git-tree-sha1 = "129acf094d168394e80ee1dc4bc06ec835e510a3" +uuid = "2e76f6c2-a576-52d4-95c1-20adfe4de566" +version = "2.8.1+1" + [[deps.Hwloc_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] git-tree-sha1 = "ca0f6bf568b4bfc807e7537f081c81e35ceca114" @@ -423,6 +489,12 @@ version = "0.9.17" [deps.KernelAbstractions.weakdeps] EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869" +[[deps.LAME_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "f6250b16881adf048549549fba48b1161acdac8c" +uuid = "c1c5ebd0-6772-5130-a774-d5fcae4a789d" +version = "3.100.1+0" + [[deps.LLVM]] deps = ["CEnum", "LLVMExtra_jll", "Libdl", "Preferences", "Printf", "Requires", "Unicode"] git-tree-sha1 = "ddab4d40513bce53c8e3157825e245224f74fae7" @@ -444,6 +516,12 @@ git-tree-sha1 = "2e5c102cfc41f48ae4740c7eca7743cc7e7b75ea" uuid = "8b046642-f1f6-4319-8d3c-209ddc03c586" version = "1.0.0" +[[deps.LLVMOpenMP_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "d986ce2d884d49126836ea94ed5bfb0f12679713" +uuid = "1d63c593-3942-5779-bab2-d838dc0a180e" +version = "15.0.7+0" + [[deps.LRUCache]] git-tree-sha1 = "b3cc6698599b10e652832c2f23db3cab99d51b59" uuid = "8ac3fa9e-de4c-5943-b1dc-09c6b5f20637" @@ -453,6 +531,12 @@ weakdeps = ["Serialization"] [deps.LRUCache.extensions] SerializationExt = ["Serialization"] +[[deps.LZO_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "e5b909bcf985c5e2605737d2ce278ed791b89be6" +uuid = "dd4b983a-f0e5-5f8d-a1b7-129d4a5fb1ac" +version = "2.10.1+0" + [[deps.LaTeXStrings]] git-tree-sha1 = "50901ebc375ed41dbf8058da26f9de442febbbec" uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" @@ -489,12 +573,42 @@ version = "1.11.0+1" [[deps.Libdl]] uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +[[deps.Libffi_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "0b4a5d71f3e5200a7dff793393e09dfc2d874290" +uuid = "e9f186c6-92d2-5b65-8a66-fee21dc1b490" +version = "3.2.2+1" + +[[deps.Libgcrypt_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgpg_error_jll", "Pkg"] +git-tree-sha1 = "64613c82a59c120435c067c2b809fc61cf5166ae" +uuid = "d4300ac3-e22c-5743-9152-c294e39db1e4" +version = "1.8.7+0" + +[[deps.Libgpg_error_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "c333716e46366857753e273ce6a69ee0945a6db9" +uuid = "7add5ba3-2f88-524e-9cd5-f83b8a55f7b8" +version = "1.42.0+0" + [[deps.Libiconv_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] git-tree-sha1 = "f9557a255370125b405568f9767d6d195822a175" uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531" version = "1.17.0+0" +[[deps.Libmount_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "9c30530bf0effd46e15e0fdcf2b8636e78cbbd73" +uuid = "4b2f31a3-9ecc-558c-b454-b3730dcb73e9" +version = "2.35.0+0" + +[[deps.Libuuid_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "e5edc369a598dfde567269dc6add5812cfa10cd5" +uuid = "38a345b3-de98-5d2b-a5d3-14cd9215e700" +version = "2.39.3+0" + [[deps.LinearAlgebra]] deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" @@ -639,6 +753,12 @@ weakdeps = ["Adapt"] [deps.OffsetArrays.extensions] OffsetArraysAdaptExt = "Adapt" +[[deps.Ogg_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "887579a3eb005446d514ab7aeac5d1d027658b8f" +uuid = "e7412a2a-1a6e-54c0-be00-318e2571c051" +version = "1.3.5+1" + [[deps.OpenBLAS_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" @@ -667,6 +787,12 @@ git-tree-sha1 = "13652491f6856acfd2db29360e1bbcd4565d04f1" uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e" version = "0.5.5+0" +[[deps.Opus_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "51a08fb14ec28da2ec7a927c4337e4332c2a4720" +uuid = "91d4177d-7536-5919-b921-800302f37372" +version = "1.3.2+0" + [[deps.OrderedCollections]] git-tree-sha1 = "dfdf5519f235516220579f949664f1bf44e741c5" uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" @@ -678,6 +804,11 @@ git-tree-sha1 = "2cd396108e178f3ae8dedbd8e938a18726ab2fbf" uuid = "c2071276-7c44-58a7-b746-946036e04d0a" version = "0.24.1+0" +[[deps.PCRE2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "efcefdf7-47ab-520b-bdef-62a2eaa19f15" +version = "10.42.0+1" + [[deps.PackageExtensionCompat]] git-tree-sha1 = "fb28e33b8a95c4cee25ce296c817d89cc2e53518" uuid = "65ce6f38-6b18-4e1d-a461-8949797d7930" @@ -710,6 +841,12 @@ git-tree-sha1 = "bd69f3f0ee248cfb4241800aefb705b5ded592ff" uuid = "4a48f351-57a6-4416-9ec4-c37015456aae" version = "0.15.1" +[[deps.Pixman_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "LLVMOpenMP_jll", "Libdl"] +git-tree-sha1 = "64779bc4c9784fee475689a1752ef4d5747c5e87" +uuid = "30392449-352a-5448-841d-b1acce4e97dc" +version = "0.42.2+0" + [[deps.Pkg]] deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" @@ -1053,12 +1190,66 @@ git-tree-sha1 = "07e470dabc5a6a4254ffebc29a1b3fc01464e105" uuid = "02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a" version = "2.12.5+0" +[[deps.XSLT_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgcrypt_jll", "Libgpg_error_jll", "Libiconv_jll", "Pkg", "XML2_jll", "Zlib_jll"] +git-tree-sha1 = "91844873c4085240b95e795f692c4cec4d805f8a" +uuid = "aed1982a-8fda-507f-9586-7b0439959a61" +version = "1.1.34+0" + [[deps.XZ_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] git-tree-sha1 = "37195dcb94a5970397ad425b95a9a26d0befce3a" uuid = "ffd25f8a-64ca-5728-b0f7-c24cf3aae800" version = "5.6.0+0" +[[deps.Xorg_libX11_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libxcb_jll", "Xorg_xtrans_jll"] +git-tree-sha1 = "afead5aba5aa507ad5a3bf01f58f82c8d1403495" +uuid = "4f6342f7-b3d2-589e-9d20-edeb45f2b2bc" +version = "1.8.6+0" + +[[deps.Xorg_libXau_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "6035850dcc70518ca32f012e46015b9beeda49d8" +uuid = "0c0b7dd1-d40b-584c-a123-a41640f87eec" +version = "1.0.11+0" + +[[deps.Xorg_libXdmcp_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "34d526d318358a859d7de23da945578e8e8727b7" +uuid = "a3789734-cfe1-5b06-b2d0-1dd0d9d62d05" +version = "1.1.4+0" + +[[deps.Xorg_libXext_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"] +git-tree-sha1 = "b7c0aa8c376b31e4852b360222848637f481f8c3" +uuid = "1082639a-0dae-5f34-9b06-72781eeb8cb3" +version = "1.3.4+4" + +[[deps.Xorg_libXrender_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"] +git-tree-sha1 = "19560f30fd49f4d4efbe7002a1037f8c43d43b96" +uuid = "ea2f1a96-1ddc-540d-b46f-429655e07cfa" +version = "0.9.10+4" + +[[deps.Xorg_libpthread_stubs_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "8fdda4c692503d44d04a0603d9ac0982054635f9" +uuid = "14d82f49-176c-5ed1-bb49-ad3f5cbd8c74" +version = "0.1.1+0" + +[[deps.Xorg_libxcb_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "XSLT_jll", "Xorg_libXau_jll", "Xorg_libXdmcp_jll", "Xorg_libpthread_stubs_jll"] +git-tree-sha1 = "b4bfde5d5b652e22b9c790ad00af08b6d042b97d" +uuid = "c7cfdc94-dc32-55de-ac96-5a1b8d977c5b" +version = "1.15.0+0" + +[[deps.Xorg_xtrans_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "e92a1a012a10506618f10b7047e478403a046c77" +uuid = "c5fb5394-a638-5e4d-96e5-b29de1b5cf10" +version = "1.5.0+0" + [[deps.Zlib_jll]] deps = ["Libdl"] uuid = "83775a58-1f1d-513f-b197-d71354ab007a" @@ -1076,11 +1267,41 @@ git-tree-sha1 = "eddd19a8dea6b139ea97bdc8a0e2667d4b661720" uuid = "477f73a3-ac25-53e9-8cc3-50b2fa2566f0" version = "1.0.6+1" +[[deps.libaom_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "3a2ea60308f0996d26f1e5354e10c24e9ef905d4" +uuid = "a4ae2306-e953-59d6-aa16-d00cac43593b" +version = "3.4.0+0" + +[[deps.libass_jll]] +deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "HarfBuzz_jll", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"] +git-tree-sha1 = "5982a94fcba20f02f42ace44b9894ee2b140fe47" +uuid = "0ac62f75-1d6f-5e53-bd7c-93b484bb37c0" +version = "0.15.1+0" + [[deps.libblastrampoline_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" version = "5.8.0+1" +[[deps.libfdk_aac_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "daacc84a041563f965be61859a36e17c4e4fcd55" +uuid = "f638f0a6-7fb0-5443-88ba-1cc74229b280" +version = "2.0.2+0" + +[[deps.libpng_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Zlib_jll"] +git-tree-sha1 = "1ea2ebe8ffa31f9c324e8c1d6e86b4165b84a024" +uuid = "b53b4c65-9356-5827-b1ea-8c7a1a84506f" +version = "1.6.43+0" + +[[deps.libvorbis_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Ogg_jll", "Pkg"] +git-tree-sha1 = "b910cb81ef3fe6e78bf6acee440bda86fd6ae00c" +uuid = "f27f6e37-5d2b-51aa-960f-b287f2bc3b7a" +version = "1.3.7+1" + [[deps.libzip_jll]] deps = ["Artifacts", "Bzip2_jll", "GnuTLS_jll", "JLLWrappers", "Libdl", "XZ_jll", "Zlib_jll", "Zstd_jll"] git-tree-sha1 = "3282b7d16ae7ac3e57ec2f3fa8fafb564d8f9f7f" @@ -1096,3 +1317,15 @@ version = "1.52.0+1" deps = ["Artifacts", "Libdl"] uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" version = "17.4.0+2" + +[[deps.x264_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "4fea590b89e6ec504593146bf8b988b2c00922b2" +uuid = "1270edf5-f2f9-52d2-97e9-ab00b5d0237a" +version = "2021.5.5+0" + +[[deps.x265_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "ee567a171cce03570d77ad3a43e90218e38937a9" +uuid = "dfaa095f-4041-5dcd-9319-2fabd8486b76" +version = "3.5.0+0" diff --git a/Project.toml b/Project.toml index 2004614377..6ec2df9356 100644 --- a/Project.toml +++ b/Project.toml @@ -11,6 +11,7 @@ CubedSphere = "7445602f-e544-4518-8976-18f8e8ae6cdb" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Distances = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7" DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" +FFMPEG = "c87230d0-a227-11e9-1b43-d7ebe4e7570a" FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" Glob = "c27321d9-0574-5035-807b-f59d2c89b15c" IncompleteLU = "40713840-3770-5561-ab4c-a76e7d0d7895" From d6c1deddc24a28ff5138475b5831cb3a7dcc4238 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Wed, 6 Mar 2024 19:50:03 -0500 Subject: [PATCH 217/567] conserving tracers --- .../split_explicit_free_surface_kernels.jl | 29 ++++++++----------- ...te_hydrostatic_free_surface_model_state.jl | 3 ++ 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 7bf8d493c2..b10d77b4e4 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -158,6 +158,7 @@ end @inline dynamic_column_heightᶠᶜ(i, j, k, grid, H, η) = @inbounds H[i, j, 1] @inline dynamic_column_heightᶜᶠ(i, j, k, grid, H, η) = @inbounds H[i, j, 1] +# Non-linear free surface implementation @inline dynamic_column_heightᶠᶜ(i, j, k, grid::ZStarSpacingGrid, H, η) = @inbounds H[i, j, 1] + ℑxᶠᵃᵃ(i, j, k, grid, η) @inline dynamic_column_heightᶜᶠ(i, j, k, grid::ZStarSpacingGrid, H, η) = @inbounds H[i, j, 1] + ℑyᵃᶠᵃ(i, j, k, grid, η) @@ -189,7 +190,7 @@ end hᶠᶜ = dynamic_column_heightᶠᶜ(i, j, k_top, grid, Hᶠᶜ, η) hᶜᶠ = dynamic_column_heightᶠᶜ(i, j, k_top, grid, Hᶜᶠ, η) - # ∂τ(U) = - ∇η + G + # ∂τ(U) = - gH∇η + G U[i, j, k_top-1] += Δτ * (- g * hᶠᶜ * ∂xᶠᶜᶠ_η(i, j, k_top, grid, TX, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gᵁ[i, j, k_top-1]) V[i, j, k_top-1] += Δτ * (- g * hᶜᶠ * ∂yᶜᶠᶠ_η(i, j, k_top, grid, TY, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gⱽ[i, j, k_top-1]) @@ -209,16 +210,13 @@ end i, j = @index(Global, NTuple) k_top = grid.Nz + 1 - @inbounds scalingᶠᶜᶜ = dynamic_column_heightᶠᶜ(i, j, k_top, grid, Hᶠᶜ, η̅) / Hᶠᶜ[i, j, 1] - @inbounds scalingᶜᶠᶜ = dynamic_column_heightᶜᶠ(i, j, k_top, grid, Hᶜᶠ, η̅) / Hᶜᶠ[i, j, 1] - # hand unroll first loop - @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δzᶠᶜᶜ_reference(i, j, 1, grid) * scalingᶠᶜᶜ - @inbounds V[i, j, k_top-1] = v[i, j, 1] * Δzᶜᶠᶜ_reference(i, j, 1, grid) * scalingᶜᶠᶜ + @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δzᶠᶜᶜ_reference(i, j, 1, grid) + @inbounds V[i, j, k_top-1] = v[i, j, 1] * Δzᶜᶠᶜ_reference(i, j, 1, grid) @unroll for k in 2:grid.Nz - @inbounds U[i, j, k_top-1] += u[i, j, k] * Δzᶠᶜᶜ_reference(i, j, k, grid) * scalingᶠᶜᶜ - @inbounds V[i, j, k_top-1] += v[i, j, k] * Δzᶜᶠᶜ_reference(i, j, k, grid) * scalingᶜᶠᶜ + @inbounds U[i, j, k_top-1] += u[i, j, k] * Δzᶠᶜᶜ_reference(i, j, k, grid) + @inbounds V[i, j, k_top-1] += v[i, j, k] * Δzᶜᶠᶜ_reference(i, j, k, grid) end end @@ -227,16 +225,13 @@ end i, j = active_linear_index_to_tuple(idx, ZColumnMap(), grid) k_top = grid.Nz + 1 - @inbounds scalingᶠᶜᶜ = dynamic_column_heightᶠᶜ(i, j, k_top, grid, Hᶠᶜ, η̅) / Hᶠᶜ[i, j, 1] - @inbounds scalingᶜᶠᶜ = dynamic_column_heightᶜᶠ(i, j, k_top, grid, Hᶜᶠ, η̅) / Hᶜᶠ[i, j, 1] - # hand unroll first loop - @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δzᶠᶜᶜ_reference(i, j, 1, grid) * scalingᶠᶜᶜ - @inbounds V[i, j, k_top-1] = v[i, j, 1] * Δzᶜᶠᶜ_reference(i, j, 1, grid) * scalingᶜᶠᶜ + @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δzᶠᶜᶜ_reference(i, j, 1, grid) + @inbounds V[i, j, k_top-1] = v[i, j, 1] * Δzᶜᶠᶜ_reference(i, j, 1, grid) @unroll for k in 2:grid.Nz - @inbounds U[i, j, k_top-1] += u[i, j, k] * Δzᶠᶜᶜ_reference(i, j, k, grid) * scalingᶠᶜᶜ - @inbounds V[i, j, k_top-1] += v[i, j, k] * Δzᶜᶠᶜ_reference(i, j, k, grid) * scalingᶜᶠᶜ + @inbounds U[i, j, k_top-1] += u[i, j, k] * Δzᶠᶜᶜ_reference(i, j, k, grid) + @inbounds V[i, j, k_top-1] += v[i, j, k] * Δzᶜᶠᶜ_reference(i, j, k, grid) end end @@ -279,8 +274,8 @@ end k_top = grid.Nz + 1 @inbounds begin - u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / dynamic_column_heightᶠᶜ(i, j, k_top, grid, Hᶠᶜ, η̅) - v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / dynamic_column_heightᶜᶠ(i, j, k_top, grid, Hᶜᶠ, η̅) + u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / Hᶠᶜ[i, j, 1] + v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / Hᶜᶠ[i, j, 1] end end diff --git a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl index f1ced68f08..87ddb1366e 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl @@ -76,8 +76,11 @@ function compute_auxiliaries!(model::HydrostaticFreeSurfaceModel, Δt; w_paramet diffusivity = model.diffusivity_fields for (wpar, ppar, κpar) in zip(w_parameters, p_parameters, κ_parameters) + # Update the grid update_vertical_spacing!(model, grid, Δt; parameters = wpar) scale_tracers!(tracers, grid; parameters = wpar) + + # Update the other auxiliary terms compute_w_from_continuity!(model; parameters = wpar) compute_diffusivities!(diffusivity, closure, model; parameters = κpar) update_hydrostatic_pressure!(model.pressure.pHY′, architecture(grid), From 20e0b8a490696b8d0096e69c5f5ed6b55e4d3bc4 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Wed, 6 Mar 2024 20:22:21 -0500 Subject: [PATCH 218/567] height should be included everywhere --- ...distributed_split_explicit_free_surface.jl | 20 +++------ .../split_explicit_free_surface.jl | 43 ++++++++++++------- ...ulti_region_split_explicit_free_surface.jl | 14 +----- 3 files changed, 36 insertions(+), 41 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl index bcaeea7a32..61740ba210 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl @@ -7,18 +7,14 @@ import Oceananigans.Models.HydrostaticFreeSurfaceModels: FreeSurface, SplitExpli function SplitExplicitAuxiliaryFields(grid::DistributedGrid) - Gᵁ = Field((Face, Center, Nothing), grid) - Gⱽ = Field((Center, Face, Nothing), grid) + Gᵁ = Field{Face, Center, Nothing}(grid) + Gⱽ = Field{Center, Face, Nothing}(grid) - Hᶠᶜ = Field((Face, Center, Nothing), grid) - Hᶜᶠ = Field((Center, Face, Nothing), grid) - Hᶜᶠ = Field((Center, Center, Nothing), grid) + Hᶠᶜ = Field{Face, Center, Nothing}(grid) + Hᶜᶠ = Field{Center, Face, Nothing}(grid) + Hᶜᶠ = Field{Center, Center, Nothing}(grid) - calculate_column_height!(Hᶠᶜ, (Face, Center, Center)) - calculate_column_height!(Hᶜᶠ, (Center, Face, Center)) - calculate_column_height!(Hᶜᶜ, (Center, Center, Center)) - - fill_halo_regions!((Hᶠᶜ, Hᶜᶠ, Hᶜᶜ)) + calculate_column_height!(Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, grid) # In a non-parallel grid we calculate only the interior kernel_size = augmented_kernel_size(grid) @@ -30,10 +26,6 @@ function SplitExplicitAuxiliaryFields(grid::DistributedGrid) end """Integrate z at locations `location` and set! `height`` with the result""" -@inline function calculate_column_height!(height, location) - dz = GridMetricOperation(location, Δz, height.grid) - return sum!(height, dz) -end @inline function augmented_kernel_size(grid::DistributedGrid) Nx, Ny, _ = size(grid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index a76a257089..4eedf37b87 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -4,7 +4,7 @@ using Oceananigans.Fields using Oceananigans.Grids using Oceananigans.Grids: AbstractGrid using Oceananigans.AbstractOperations: Δz, GridMetricOperation - +using Oceananigans.ImmersedBoundaries: peripheral_node, c, f using Adapt using Base using KernelAbstractions: @index, @kernel @@ -200,27 +200,40 @@ Return the `SplitExplicitAuxiliaryFields` for `grid`. """ function SplitExplicitAuxiliaryFields(grid::AbstractGrid) - Gᵁ = Field((Face, Center, Nothing), grid) - Gⱽ = Field((Center, Face, Nothing), grid) + Gᵁ = Field{Face, Center, Nothing}(grid) + Gⱽ = Field{Center, Face, Nothing}(grid) - Hᶠᶜ = Field((Face, Center, Nothing), grid) - Hᶜᶠ = Field((Center, Face, Nothing), grid) - Hᶜᶜ = Field((Center, Center, Nothing), grid) + Hᶠᶜ = Field{Face, Center, Nothing}(grid) + Hᶜᶠ = Field{Center, Face, Nothing}(grid) + Hᶜᶜ = Field{Center, Center, Nothing}(grid) - dz = GridMetricOperation((Face, Center, Center), Δz, grid) - sum!(Hᶠᶜ, dz) + calculate_column_height!(Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, grid) + + kernel_parameters = :xy - dz = GridMetricOperation((Center, Face, Center), Δz, grid) - sum!(Hᶜᶠ, dz) + return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, kernel_parameters) +end - dz = GridMetricOperation((Center, Face, Center), Δz, grid) - sum!(Hᶜᶜ, dz) +function calculate_column_height!(Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, grid) + Nx, Ny, _ = size(grid) + Hx, Hy, _ = halo_size(grid) - fill_halo_regions!((Hᶠᶜ, Hᶜᶠ, Hᶜᶜ)) + arch = architecture(grid) + param = KernelParameters((Nx+2Hx, Ny+2Hy), (-Hx, -Hy)) - kernel_parameters = :xy + launch!(arch, grid, param, _compute_column_height!, Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, grid) - return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, kernel_parameters) + return nothing +end + +@kernel function _compute_column_height!(Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, grid) + i, j = @index(Global, NTuple) + Nz = size(grid, 3) + @inbounds for k in 1:Nz + Hᶠᶜ[i, j, 1] += ifelse(peripheral_node(i, j, k, grid, f, c, c), 0, Δzᶠᶜᶜ(i, j, k, grid)) + Hᶜᶠ[i, j, 1] += ifelse(peripheral_node(i, j, k, grid, c, f, c), 0, Δzᶜᶠᶜ(i, j, k, grid)) + Hᶜᶜ[i, j, 1] += ifelse(peripheral_node(i, j, k, grid, c, c, c), 0, Δzᶜᶜᶜ(i, j, k, grid)) + end end """ diff --git a/src/MultiRegion/multi_region_split_explicit_free_surface.jl b/src/MultiRegion/multi_region_split_explicit_free_surface.jl index 4eed133830..17c648a86d 100644 --- a/src/MultiRegion/multi_region_split_explicit_free_surface.jl +++ b/src/MultiRegion/multi_region_split_explicit_free_surface.jl @@ -1,6 +1,6 @@ using Oceananigans.Utils using Oceananigans.AbstractOperations: GridMetricOperation, Δz -using Oceananigans.Models.HydrostaticFreeSurfaceModels: SplitExplicitState, SplitExplicitFreeSurface +using Oceananigans.Models.HydrostaticFreeSurfaceModels: SplitExplicitState, SplitExplicitFreeSurface, calculate_column_height! import Oceananigans.Models.HydrostaticFreeSurfaceModels: FreeSurface, SplitExplicitAuxiliaryFields @@ -13,11 +13,7 @@ function SplitExplicitAuxiliaryFields(grid::MultiRegionGrids) Hᶜᶠ = Field((Center, Face, Nothing), grid) Hᶜᶠ = Field((Center, Center, Nothing), grid) - @apply_regionally calculate_column_height!(Hᶠᶜ, (Face, Center, Center)) - @apply_regionally calculate_column_height!(Hᶜᶠ, (Center, Face, Center)) - @apply_regionally calculate_column_height!(Hᶜᶜ, (Center, Center, Center)) - - fill_halo_regions!((Hᶠᶜ, Hᶜᶠ, Hᶜᶜ)) + @apply_regionally calculate_column_height!(Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, grid) # In a non-parallel grid we calculate only the interior @apply_regionally kernel_size = augmented_kernel_size(grid, grid.partition) @@ -28,12 +24,6 @@ function SplitExplicitAuxiliaryFields(grid::MultiRegionGrids) return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, kernel_parameters) end -@inline function calculate_column_height!(height, location) - dz = GridMetricOperation(location, Δz, height.grid) - sum!(height, dz) - return nothing -end - @inline augmented_kernel_size(grid, ::XPartition) = (size(grid, 1) + 2halo_size(grid)[1]-2, size(grid, 2)) @inline augmented_kernel_size(grid, ::YPartition) = (size(grid, 1), size(grid, 2) + 2halo_size(grid)[2]-2) @inline augmented_kernel_size(grid, ::CubedSpherePartition) = (size(grid, 1) + 2halo_size(grid)[1]-2, size(grid, 2) + 2halo_size(grid)[2]-2) From 0b88afd0e56d3095a56a8cacfebc5ed7a3f985db Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Wed, 6 Mar 2024 20:34:31 -0500 Subject: [PATCH 219/567] stable and correct, testing in neverworld --- src/ImmersedBoundaries/ImmersedBoundaries.jl | 5 ++++- .../split_explicit_free_surface.jl | 11 ++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/ImmersedBoundaries/ImmersedBoundaries.jl b/src/ImmersedBoundaries/ImmersedBoundaries.jl index 1d5a7c042d..c488d2de73 100644 --- a/src/ImmersedBoundaries/ImmersedBoundaries.jl +++ b/src/ImmersedBoundaries/ImmersedBoundaries.jl @@ -11,7 +11,7 @@ using Oceananigans.Utils using Oceananigans.Architectures using Oceananigans.TurbulenceClosures: AbstractTurbulenceClosure, time_discretization -using Oceananigans.Grids: size_summary, inactive_node, peripheral_node, AbstractGrid +using Oceananigans.Grids: size_summary, inactive_node, peripheral_node, AbstractGrid, AbstractUnderlyingGrid using Oceananigans.TurbulenceClosures: viscous_flux_ux, @@ -237,6 +237,9 @@ As well as @inline immersed_inactive_node(i, j, k, ibg::IBG, LX, LY, LZ) = inactive_node(i, j, k, ibg, LX, LY, LZ) & !inactive_node(i, j, k, ibg.underlying_grid, LX, LY, LZ) +# Underlying grids are never immersed! +@inline immersed_peripheral_node(i, j, k, grid::AbstractUnderlyingGrid, LX, LY, LZ) = false +@inline immersed_inactive_node(i, j, k, grid::AbstractUnderlyingGrid, LX, LY, LZ) = false ##### ##### Utilities diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index 4eedf37b87..4c29364069 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -4,7 +4,7 @@ using Oceananigans.Fields using Oceananigans.Grids using Oceananigans.Grids: AbstractGrid using Oceananigans.AbstractOperations: Δz, GridMetricOperation -using Oceananigans.ImmersedBoundaries: peripheral_node, c, f +using Oceananigans.ImmersedBoundaries: immersed_peripheral_node, c, f using Adapt using Base using KernelAbstractions: @index, @kernel @@ -208,7 +208,7 @@ function SplitExplicitAuxiliaryFields(grid::AbstractGrid) Hᶜᶜ = Field{Center, Center, Nothing}(grid) calculate_column_height!(Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, grid) - + kernel_parameters = :xy return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, kernel_parameters) @@ -221,6 +221,7 @@ function calculate_column_height!(Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, grid) arch = architecture(grid) param = KernelParameters((Nx+2Hx, Ny+2Hy), (-Hx, -Hy)) + @show param launch!(arch, grid, param, _compute_column_height!, Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, grid) return nothing @@ -230,9 +231,9 @@ end i, j = @index(Global, NTuple) Nz = size(grid, 3) @inbounds for k in 1:Nz - Hᶠᶜ[i, j, 1] += ifelse(peripheral_node(i, j, k, grid, f, c, c), 0, Δzᶠᶜᶜ(i, j, k, grid)) - Hᶜᶠ[i, j, 1] += ifelse(peripheral_node(i, j, k, grid, c, f, c), 0, Δzᶜᶠᶜ(i, j, k, grid)) - Hᶜᶜ[i, j, 1] += ifelse(peripheral_node(i, j, k, grid, c, c, c), 0, Δzᶜᶜᶜ(i, j, k, grid)) + Hᶠᶜ[i, j, 1] += ifelse(immersed_peripheral_node(i, j, k, grid, f, c, c), 0, Δzᶠᶜᶜ_reference(i, j, k, grid)) + Hᶜᶠ[i, j, 1] += ifelse(immersed_peripheral_node(i, j, k, grid, c, f, c), 0, Δzᶜᶠᶜ_reference(i, j, k, grid)) + Hᶜᶜ[i, j, 1] += ifelse(immersed_peripheral_node(i, j, k, grid, c, c, c), 0, Δzᶜᶜᶜ_reference(i, j, k, grid)) end end From 8e71f419ee29f548988d4c12b1ca690c48cb69b8 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Wed, 6 Mar 2024 20:39:55 -0500 Subject: [PATCH 220/567] test in lock-release --- validation/z_star_coordinate/lock_release.jl | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index af79891a2d..c50675fba5 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -5,7 +5,7 @@ using Oceananigans.Advection: WENOVectorInvariant using Oceananigans.Models.HydrostaticFreeSurfaceModels: ZStar, ZStarSpacingGrid using Printf -grid = RectilinearGrid(size = (300, 20), +grid = RectilinearGrid(size = (10, 10), x = (0, 100kilometers), z = (-10, 0), halo = (6, 6), @@ -13,8 +13,8 @@ grid = RectilinearGrid(size = (300, 20), model = HydrostaticFreeSurfaceModel(; grid, generalized_vertical_coordinate = ZStar(), - momentum_advection = WENO(), - tracer_advection = WENO(), + momentum_advection = Centered(), + tracer_advection = Centered(), buoyancy = BuoyancyTracer(), tracers = :b, free_surface = SplitExplicitFreeSurface(; substeps = 10)) @@ -32,7 +32,7 @@ barotropic_time_step = grid.Δxᶜᵃᵃ / gravity_wave_speed @info "the time step is $Δt" -simulation = Simulation(model; Δt, stop_time = 1days) +simulation = Simulation(model; Δt, stop_iteration = 100000, stop_time = 10days) field_outputs = if model.grid isa ZStarSpacingGrid merge(model.velocities, model.tracers, (; ΔzF = model.grid.Δzᵃᵃᶠ.Δ)) @@ -67,11 +67,12 @@ function progress(sim) return nothing end -simulation.callbacks[:progress] = Callback(progress, IterationInterval(1)) +simulation.callbacks[:progress] = Callback(progress, IterationInterval(100)) simulation.callbacks[:wizard] = Callback(TimeStepWizard(; cfl = 0.2, max_change = 1.1), IterationInterval(10)) + run!(simulation) -# Check tracer conservation +# # Check tracer conservation if model.grid isa ZStarSpacingGrid b = FieldTimeSeries("zstar_model.jld2", "b") dz = FieldTimeSeries("zstar_model.jld2", "ΔzF") From 971c49032944accddd54fe3b08392163de9dda72 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Wed, 6 Mar 2024 20:41:13 -0500 Subject: [PATCH 221/567] add a comment --- .../split_explicit_free_surface.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index 4c29364069..25a32e7815 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -218,10 +218,11 @@ function calculate_column_height!(Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, grid) Nx, Ny, _ = size(grid) Hx, Hy, _ = halo_size(grid) + # We compute the heights over all the + # domain including the halo points! arch = architecture(grid) param = KernelParameters((Nx+2Hx, Ny+2Hy), (-Hx, -Hy)) - @show param launch!(arch, grid, param, _compute_column_height!, Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, grid) return nothing From 3a78a2c0a8499d8add7c7e2f13575247a8a11c4c Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Wed, 6 Mar 2024 21:33:09 -0500 Subject: [PATCH 222/567] optimize zstar --- .../generalized_vertical_spacing.jl | 12 ++++---- .../z_star_vertical_spacing.jl | 30 +++++++------------ 2 files changed, 17 insertions(+), 25 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index f6c645cc0c..346b321a4d 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -115,14 +115,14 @@ update_vertical_spacing!(model, grid, Δt; kwargs...) = nothing @inline Δzᶜᶜᶠ(i, j, k, grid::GeneralizedSpacingGrid) = @inbounds grid.Δzᵃᵃᶠ.Δ[i, j, k] @inline Δzᶜᶜᶜ(i, j, k, grid::GeneralizedSpacingGrid) = @inbounds grid.Δzᵃᵃᶜ.Δ[i, j, k] -@inline Δzᶜᶠᶠ(i, j, k, grid::GeneralizedSpacingGrid) = ℑyᵃᶠᵃ(i, j, k, grid, grid.Δzᵃᵃᶠ.Δ) -@inline Δzᶜᶠᶜ(i, j, k, grid::GeneralizedSpacingGrid) = ℑyᵃᶠᵃ(i, j, k, grid, grid.Δzᵃᵃᶜ.Δ) +@inline Δzᶜᶠᶠ(i, j, k, grid::GeneralizedSpacingGrid) = ℑyᵃᶠᵃ(i, j, k, grid, Δzᶜᶜᶠ) +@inline Δzᶜᶠᶜ(i, j, k, grid::GeneralizedSpacingGrid) = ℑyᵃᶠᵃ(i, j, k, grid, Δzᶜᶜᶜ) -@inline Δzᶠᶜᶠ(i, j, k, grid::GeneralizedSpacingGrid) = ℑxᶠᵃᵃ(i, j, k, grid, grid.Δzᵃᵃᶠ.Δ) -@inline Δzᶠᶜᶜ(i, j, k, grid::GeneralizedSpacingGrid) = ℑxᶠᵃᵃ(i, j, k, grid, grid.Δzᵃᵃᶜ.Δ) +@inline Δzᶠᶜᶠ(i, j, k, grid::GeneralizedSpacingGrid) = ℑxᶠᵃᵃ(i, j, k, grid, Δzᶜᶜᶠ) +@inline Δzᶠᶜᶜ(i, j, k, grid::GeneralizedSpacingGrid) = ℑxᶠᵃᵃ(i, j, k, grid, Δzᶜᶜᶜ) -@inline Δzᶠᶠᶠ(i, j, k, grid::GeneralizedSpacingGrid) = ℑxyᶠᶠᵃ(i, j, k, grid, grid.Δzᵃᵃᶠ.Δ) -@inline Δzᶠᶠᶜ(i, j, k, grid::GeneralizedSpacingGrid) = ℑxyᶠᶠᵃ(i, j, k, grid, grid.Δzᵃᵃᶜ.Δ) +@inline Δzᶠᶠᶠ(i, j, k, grid::GeneralizedSpacingGrid) = ℑxyᶠᶠᵃ(i, j, k, grid, Δzᶜᶜᶠ) +@inline Δzᶠᶠᶜ(i, j, k, grid::GeneralizedSpacingGrid) = ℑxyᶠᶠᵃ(i, j, k, grid, Δzᶜᶜᶜ) @inline Δz_reference(i, j, k, Δz::Number) = Δz @inline Δz_reference(i, j, k, Δz::AbstractVector) = Δz[k] diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 9ea2f9e994..e46f6bcbcf 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -25,8 +25,6 @@ end function GeneralizedSpacingGrid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, ::ZStar) where {FT, TX, TY, TZ} # Memory layout for Δz spacings should be local in z instead of x - ΔzF = ZFaceField(grid) - ΔzC = CenterField(grid) s⁻ = Field{Center, Center, Nothing}(grid) sⁿ = Field{Center, Center, Nothing}(grid) ∂t_∂s = Field{Center, Center, Nothing}(grid) @@ -35,12 +33,8 @@ function GeneralizedSpacingGrid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, :: fill!(s⁻, 1) fill!(sⁿ, 1) - launch!(architecture(grid), grid, :xyz, _initialize_zstar!, ΔzF, ΔzC, grid) - - fill_halo_regions!((ΔzF, ΔzC)) - - Δzᵃᵃᶠ = GeneralizedVerticalSpacing(ZStar(), grid.Δzᵃᵃᶠ, ΔzF, s⁻, sⁿ, ∂t_∂s) - Δzᵃᵃᶜ = GeneralizedVerticalSpacing(ZStar(), grid.Δzᵃᵃᶜ, ΔzC, s⁻, sⁿ, ∂t_∂s) + Δzᵃᵃᶠ = GeneralizedVerticalSpacing(ZStar(), grid.Δzᵃᵃᶠ, nothing, s⁻, sⁿ, ∂t_∂s) + Δzᵃᵃᶜ = GeneralizedVerticalSpacing(ZStar(), grid.Δzᵃᵃᶜ, nothing, s⁻, sⁿ, ∂t_∂s) args = [] for prop in propertynames(grid) @@ -64,6 +58,13 @@ end @inbounds ΔzC[i, j, k] = Δzᶜᶜᶜ(i, j, k, grid) end +##### +##### ZStar-specific vertical spacing functions +##### + +@inline Δzᶜᶜᶠ(i, j, k, grid::ZStarSpacing) = @inbounds Δzᶜᶜᶠ_reference(i, j, k, grid) * grid.Δzᵃᵃᶠ.sⁿ[i, j, 1] +@inline Δzᶜᶜᶜ(i, j, k, grid::ZStarSpacing) = @inbounds Δzᶜᶜᶜ_reference(i, j, k, grid) * grid.Δzᵃᵃᶜ.sⁿ[i, j, 1] + ##### ##### ZStar-specific vertical spacings update ##### @@ -75,15 +76,11 @@ function update_vertical_spacing!(model, grid::ZStarSpacingGrid, Δt; parameters sⁿ = grid.Δzᵃᵃᶠ.sⁿ ∂t_∂s = grid.Δzᵃᵃᶠ.∂t_∂s - # Generalized vertical spacing - ΔzF = grid.Δzᵃᵃᶠ.Δ - ΔzC = grid.Δzᵃᵃᶜ.Δ - Hᶜᶜ = model.free_surface.auxiliary.Hᶜᶜ # Update vertical spacing with available parameters # No need to fill the halo as the scaling is updated _IN_ the halos - launch!(architecture(grid), grid, parameters, _update_zstar!, sⁿ, s⁻, ΔzF, ΔzC, + launch!(architecture(grid), grid, parameters, _update_zstar!, sⁿ, s⁻, model.free_surface.η, Hᶜᶜ, grid, Val(grid.Nz)) # Update scaling time derivative @@ -92,7 +89,7 @@ function update_vertical_spacing!(model, grid::ZStarSpacingGrid, Δt; parameters return nothing end -@kernel function _update_zstar!(sⁿ, s⁻, ΔzF, ΔzC, η, Hᶜᶜ, grid, ::Val{Nz}) where Nz +@kernel function _update_zstar!(sⁿ, s⁻, η, Hᶜᶜ, grid, ::Val{Nz}) where Nz i, j = @index(Global, NTuple) @inbounds begin bottom = Hᶜᶜ[i, j, 1] @@ -101,11 +98,6 @@ end # update current and previous scaling s⁻[i, j, 1] = sⁿ[i, j, 1] sⁿ[i, j, 1] = h - - @unroll for k in 1:Nz+1 - ΔzF[i, j, k] = h * Δzᶜᶜᶠ_reference(i, j, k, grid) - ΔzC[i, j, k] = h * Δzᶜᶜᶜ_reference(i, j, k, grid) - end end end From c637aea0ac4a5f4263f8ae051e17b9ab868aeb34 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Wed, 6 Mar 2024 21:38:29 -0500 Subject: [PATCH 223/567] this works better! --- .../HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index e46f6bcbcf..1afa4e3bae 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -62,8 +62,8 @@ end ##### ZStar-specific vertical spacing functions ##### -@inline Δzᶜᶜᶠ(i, j, k, grid::ZStarSpacing) = @inbounds Δzᶜᶜᶠ_reference(i, j, k, grid) * grid.Δzᵃᵃᶠ.sⁿ[i, j, 1] -@inline Δzᶜᶜᶜ(i, j, k, grid::ZStarSpacing) = @inbounds Δzᶜᶜᶜ_reference(i, j, k, grid) * grid.Δzᵃᵃᶜ.sⁿ[i, j, 1] +@inline Δzᶜᶜᶠ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δzᶜᶜᶠ_reference(i, j, k, grid) * grid.Δzᵃᵃᶠ.sⁿ[i, j, 1] +@inline Δzᶜᶜᶜ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δzᶜᶜᶜ_reference(i, j, k, grid) * grid.Δzᵃᵃᶜ.sⁿ[i, j, 1] ##### ##### ZStar-specific vertical spacings update From cf1423ab00e6e0f3116f6d2b16f51a3a01c189b0 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Wed, 6 Mar 2024 22:03:32 -0500 Subject: [PATCH 224/567] so we can save the down the grid --- .../generalized_vertical_spacing.jl | 14 +++++++------- validation/z_star_coordinate/lock_release.jl | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index 346b321a4d..ced13738f8 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -39,13 +39,13 @@ Adapt.adapt_structure(to, coord::GeneralizedVerticalSpacing) = Adapt.adapt(to, coord.sⁿ), Adapt.adapt(to, coord.∂t_∂s)) -arch_array(arch, coord::GeneralizedVerticalSpacing) = - GeneralizedVerticalSpacing(coord.denomination, - arch_array(arch, coord.Δr), - coord.Δ, - coord.s⁻, - coord.sⁿ, - coord.∂t_∂s) +on_architecture(arch, coord::GeneralizedVerticalSpacing) = + GeneralizedVerticalSpacing(on_architecture(arch, coord.denomination), + on_architecture(arch, coord.Δr), + on_architecture(arch, coord.Δ), + on_architecture(arch, coord.s⁻), + on_architecture(arch, coord.sⁿ), + on_architecture(arch, coord.∂t_∂s)) const GeneralizedSpacingRG{D} = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:GeneralizedVerticalSpacing{D}} where D const GeneralizedSpacingLLG{D} = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:GeneralizedVerticalSpacing{D}} where D diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index c50675fba5..492bb1f101 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -35,7 +35,7 @@ barotropic_time_step = grid.Δxᶜᵃᵃ / gravity_wave_speed simulation = Simulation(model; Δt, stop_iteration = 100000, stop_time = 10days) field_outputs = if model.grid isa ZStarSpacingGrid - merge(model.velocities, model.tracers, (; ΔzF = model.grid.Δzᵃᵃᶠ.Δ)) + merge(model.velocities, model.tracers, (; sⁿ = model.grid.Δzᵃᵃᶠ.sⁿ)) else merge(model.velocities, model.tracers) end @@ -57,7 +57,7 @@ function progress(sim) msg2 = @sprintf("extrema u: %.2e %.2e ", maximum(u), minimum(u)) msg3 = @sprintf("extrema b: %.2e %.2e ", maximum(b), minimum(b)) if sim.model.grid isa ZStarSpacingGrid - Δz = sim.model.grid.Δzᵃᵃᶠ.Δ + Δz = sim.model.grid.Δzᵃᵃᶠ.sⁿ msg4 = @sprintf("extrema Δz: %.2e %.2e ", maximum(Δz), minimum(Δz)) @info msg0 * msg1 * msg2 * msg3 * msg4 else @@ -75,7 +75,7 @@ run!(simulation) # # Check tracer conservation if model.grid isa ZStarSpacingGrid b = FieldTimeSeries("zstar_model.jld2", "b") - dz = FieldTimeSeries("zstar_model.jld2", "ΔzF") + dz = FieldTimeSeries("zstar_model.jld2", "sⁿ") init = sum(b[1] * dz[1]) / sum(dz[1]) drift = [] From 1e1f9ae065f3b61e5d06d8e1b6132bc64a3c934d Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Wed, 6 Mar 2024 22:07:50 -0500 Subject: [PATCH 225/567] changing integratoin limits --- .../HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index 9d7f1d7725..6a6f1fbcac 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -221,7 +221,7 @@ function calculate_column_height!(Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, grid) # We compute the heights over all the # domain including the halo points! arch = architecture(grid) - param = KernelParameters((Nx+2Hx, Ny+2Hy), (-Hx, -Hy)) + param = KernelParameters((Nx+2Hx-2, Ny+2Hy-2), (-Hx+1, -Hy+1)) launch!(arch, grid, param, _compute_column_height!, Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, grid) From de11f1014be0cfa8d6455666f538d90a74bda15e Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 7 Mar 2024 07:42:11 -0500 Subject: [PATCH 226/567] remove debugging show --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 2be0525c41..a3461ac16d 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -71,8 +71,6 @@ Computes ib.bottom_height and wraps in an array. """ function ImmersedBoundaryGrid(grid, ib::GridFittedBottom) bottom_field = Field{Center, Center, Nothing}(grid) - - @show ib.bottom_height, bottom_field, architecture(grid) set!(bottom_field, ib.bottom_height) fill_halo_regions!(bottom_field) new_ib = GridFittedBottom(bottom_field, ib.immersed_condition) From 4b3c61f95afac78a94cadd03e4622d650d531012 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 7 Mar 2024 09:45:07 -0500 Subject: [PATCH 227/567] bugfix --- src/Fields/field.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Fields/field.jl b/src/Fields/field.jl index 288a07f6ec..27bf1c2c3c 100644 --- a/src/Fields/field.jl +++ b/src/Fields/field.jl @@ -418,7 +418,7 @@ total_size(f::Field) = total_size(f.grid, location(f), f.indices) ##### Move Fields between architectures ##### -on_architecture(arch, field::AbstractField) = +on_architecture(arch, field::AbstractField{LX, LY, LZ}) where {LX, LY, LZ} = Field{LX, LY, LZ}(on_architecture(arch, field.grid), on_architecture(arch, data), on_architecture(arch, bcs), From 059a39d0f80b7c12b6a379369cf62ca580d79371 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 7 Mar 2024 09:46:27 -0500 Subject: [PATCH 228/567] bugfix --- src/Fields/field.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Fields/field.jl b/src/Fields/field.jl index 288a07f6ec..27bf1c2c3c 100644 --- a/src/Fields/field.jl +++ b/src/Fields/field.jl @@ -418,7 +418,7 @@ total_size(f::Field) = total_size(f.grid, location(f), f.indices) ##### Move Fields between architectures ##### -on_architecture(arch, field::AbstractField) = +on_architecture(arch, field::AbstractField{LX, LY, LZ}) where {LX, LY, LZ} = Field{LX, LY, LZ}(on_architecture(arch, field.grid), on_architecture(arch, data), on_architecture(arch, bcs), From 384128933022e326435854bdbc11603fd8295cc8 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 7 Mar 2024 12:04:55 -0500 Subject: [PATCH 229/567] on architecture --- test/dependencies_for_runtests.jl | 1 + test/test_distributed_hydrostatic_model.jl | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/test/dependencies_for_runtests.jl b/test/dependencies_for_runtests.jl index 1e7b206af9..6f5816e813 100644 --- a/test/dependencies_for_runtests.jl +++ b/test/dependencies_for_runtests.jl @@ -39,6 +39,7 @@ using Oceananigans.Units using Oceananigans.Utils using Oceananigans.MultiRegion using Oceananigans.Architectures: device, array_type # to resolve conflict with CUDA.device +using Oceananigans.Architectures: on_architecture using Oceananigans: Clock using Dates: DateTime, Nanosecond diff --git a/test/test_distributed_hydrostatic_model.jl b/test/test_distributed_hydrostatic_model.jl index 2dcfb14779..43d5d46e41 100644 --- a/test/test_distributed_hydrostatic_model.jl +++ b/test/test_distributed_hydrostatic_model.jl @@ -98,20 +98,20 @@ for arch in archs # "s" for "serial" computation us, vs, ws, cs, ηs = solid_body_rotation_test(global_grid) - us = Array(interior(us)) - vs = Array(interior(vs)) - ws = Array(interior(ws)) - cs = Array(interior(cs)) - ηs = Array(interior(ηs)) + us = interior(on_architecture(CPU(), us)) + vs = interior(on_architecture(CPU(), vs)) + ws = interior(on_architecture(CPU(), ws)) + cs = interior(on_architecture(CPU(), cs)) + ηs = interior(on_architecture(CPU(), ηs)) @info " Testing distributed solid body rotation with architecture $arch on $(typeof(grid).name.wrapper)" u, v, w, c, η = solid_body_rotation_test(grid) - u = Array(interior(u)) - v = Array(interior(v)) - w = Array(interior(w)) - c = Array(interior(c)) - η = Array(interior(η)) + u = interior(on_architecture(CPU(), u)) + v = interior(on_architecture(CPU(), v)) + w = interior(on_architecture(CPU(), w)) + c = interior(on_architecture(CPU(), c)) + η = interior(on_architecture(CPU(), η)) cpu_arch = cpu_architecture(arch) From f34327e31f2309273a054f04a8eb85b25f838109 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 7 Mar 2024 12:06:19 -0500 Subject: [PATCH 230/567] correct distributed on_architecture --- .../distributed_on_architecture.jl | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/DistributedComputations/distributed_on_architecture.jl b/src/DistributedComputations/distributed_on_architecture.jl index 15ceed1e62..ca780cf48a 100644 --- a/src/DistributedComputations/distributed_on_architecture.jl +++ b/src/DistributedComputations/distributed_on_architecture.jl @@ -1,9 +1,19 @@ -import Oceananigans.Architectures: on_architecture +using CUDA: CuArray +using OffsetArrays # We do not support switching from distributed and serial through `on_architecture`. # We only support moving a type from CPU to GPU and the other way around # TODO: support changing the number of ranks / the partitioning? -on_architecture(arch::Distributed, A) = on_architecture(child_architecture(arch), A) + +# Disambiguation for methods defined in `src/Architectures.jl` +DisambiguationTypes = Union{Array, + CuArray, + BitArray, + SubArray{<:Any, <:Any, <:CuArray}, + SubArray{<:Any, <:Any, <:Array}, + OffsetArray} + +on_architecture(::Distributed, a::DisambiguationTypes) = on_architecture(child_architecture(arch), a) function on_architecture(new_arch::Distributed, old_grid::LatitudeLongitudeGrid) child_arch = child_architecture(new_arch) From fcb6d9f447e89104596bdb075fac79a6380aedd1 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 7 Mar 2024 12:08:08 -0500 Subject: [PATCH 231/567] remove double include --- src/DistributedComputations/DistributedComputations.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/DistributedComputations/DistributedComputations.jl b/src/DistributedComputations/DistributedComputations.jl index 22d6cff7bb..b1e224c607 100644 --- a/src/DistributedComputations/DistributedComputations.jl +++ b/src/DistributedComputations/DistributedComputations.jl @@ -12,7 +12,6 @@ using Oceananigans.Utils using Oceananigans.Grids include("distributed_architectures.jl") -include("distributed_on_architecture.jl") include("partition_assemble.jl") include("distributed_grids.jl") include("distributed_on_architecture.jl") From bd3af47a139577631082786411be325fd4dfe46d Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 7 Mar 2024 12:22:47 -0500 Subject: [PATCH 232/567] hopefully last bugfix --- src/DistributedComputations/distributed_on_architecture.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DistributedComputations/distributed_on_architecture.jl b/src/DistributedComputations/distributed_on_architecture.jl index ca780cf48a..69c95ba455 100644 --- a/src/DistributedComputations/distributed_on_architecture.jl +++ b/src/DistributedComputations/distributed_on_architecture.jl @@ -13,7 +13,7 @@ DisambiguationTypes = Union{Array, SubArray{<:Any, <:Any, <:Array}, OffsetArray} -on_architecture(::Distributed, a::DisambiguationTypes) = on_architecture(child_architecture(arch), a) +on_architecture(arch::Distributed, a::DisambiguationTypes) = on_architecture(child_architecture(arch), a) function on_architecture(new_arch::Distributed, old_grid::LatitudeLongitudeGrid) child_arch = child_architecture(new_arch) From 3d469d23e8265f4be27f5b6a4a10ed7c7e852f74 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 7 Mar 2024 12:23:17 -0500 Subject: [PATCH 233/567] adding a couple of methods --- src/DistributedComputations/distributed_on_architecture.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/DistributedComputations/distributed_on_architecture.jl b/src/DistributedComputations/distributed_on_architecture.jl index 69c95ba455..1564547a59 100644 --- a/src/DistributedComputations/distributed_on_architecture.jl +++ b/src/DistributedComputations/distributed_on_architecture.jl @@ -11,7 +11,9 @@ DisambiguationTypes = Union{Array, BitArray, SubArray{<:Any, <:Any, <:CuArray}, SubArray{<:Any, <:Any, <:Array}, - OffsetArray} + OffsetArray, + Tuple, + NamedTuple} on_architecture(arch::Distributed, a::DisambiguationTypes) = on_architecture(child_architecture(arch), a) From 3fd3d73179633356edab5dae7ba74add0000d31f Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 7 Mar 2024 13:38:12 -0500 Subject: [PATCH 234/567] test this out on the neverworld --- .../split_explicit_free_surface.jl | 32 ++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index 6a6f1fbcac..071d0a38cd 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -217,24 +217,40 @@ end function calculate_column_height!(Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, grid) Nx, Ny, _ = size(grid) Hx, Hy, _ = halo_size(grid) + Tx, Ty, _ = topology(grid) + + Ax = Tx == Flat ? 0 : 1 + Ay = Ty == Flat ? 0 : 1 + + arch = architecture(grid) # We compute the heights over all the # domain including the halo points! - arch = architecture(grid) - param = KernelParameters((Nx+2Hx-2, Ny+2Hy-2), (-Hx+1, -Hy+1)) - - launch!(arch, grid, param, _compute_column_height!, Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, grid) + # Because of the different locations of the different heights, + # to encompass the whole grid, the kernel parameters must be different. + kernel_size = (Nx+2Hx-2Ax, Ny+2Hy) + kernel_offset = (-Hx+Ax, -Hy) + paramᶠᶜ = KernelParameters(kernel_size, kernel_offset) + launch!(arch, grid, paramᶠᶜ, _compute_column_height!, Hᶠᶜ, grid, f, c, Δzᶠᶜᶜ_reference) + + kernel_size = (Nx+2Hx, Ny+2Hy-2Ay) + kernel_offset = (-Hx, -Hy+Ay) + paramᶜᶠ = KernelParameters(kernel_size, kernel_offset) + launch!(arch, grid, paramᶜᶠ, _compute_column_height!, Hᶜᶠ, grid, c, f, Δzᶜᶠᶜ_reference) + + kernel_size = (Nx+2Hx, Ny+2Hy) + kernel_offset = (-Hx, -Hy) + paramᶜᶜ = KernelParameters(kernel_size, kernel_offset) + launch!(arch, grid, paramᶜᶜ, _compute_column_height!, Hᶜᶜ, grid, c, c, Δzᶜᶜᶜ_reference) return nothing end -@kernel function _compute_column_height!(Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, grid) +@kernel function _compute_column_height!(H, grid, LX, LY, Δz) i, j = @index(Global, NTuple) Nz = size(grid, 3) @inbounds for k in 1:Nz - Hᶠᶜ[i, j, 1] += ifelse(immersed_peripheral_node(i, j, k, grid, f, c, c), 0, Δzᶠᶜᶜ_reference(i, j, k, grid)) - Hᶜᶠ[i, j, 1] += ifelse(immersed_peripheral_node(i, j, k, grid, c, f, c), 0, Δzᶜᶠᶜ_reference(i, j, k, grid)) - Hᶜᶜ[i, j, 1] += ifelse(immersed_peripheral_node(i, j, k, grid, c, c, c), 0, Δzᶜᶜᶜ_reference(i, j, k, grid)) + H[i, j, 1] += ifelse(immersed_peripheral_node(i, j, k, grid, LX, LY, c), 0, Δz(i, j, k, grid)) end end From 7175789981356b5c9ab32a03326c794104ee50fb Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 7 Mar 2024 14:18:04 -0500 Subject: [PATCH 235/567] no zstar in the outputwriter --- .../generalized_vertical_spacing.jl | 4 +- src/OutputWriters/output_writer_utils.jl | 11 +++++ validation/z_star_coordinate/lock_release.jl | 42 ++++++++++--------- 3 files changed, 36 insertions(+), 21 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index ced13738f8..dac477f211 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -63,11 +63,13 @@ denomination(grid::GeneralizedSpacingGrid) = grid.Δzᵃᵃᶠ.denomination retrieve_static_grid(grid) = grid -function retrieve_static_grid(grid::GeneralizedSpacingGrid) +function retrieve_static_grid(grid::GeneralizedSpacingGrid) Δzᵃᵃᶠ = grid.Δzᵃᵃᶠ.Δr Δzᵃᵃᶜ = grid.Δzᵃᵃᶜ.Δr + TX, TY, TZ = topology(grid) + args = [] for prop in propertynames(grid) if prop == :Δzᵃᵃᶠ diff --git a/src/OutputWriters/output_writer_utils.jl b/src/OutputWriters/output_writer_utils.jl index 7225d69182..5c183ff5a5 100644 --- a/src/OutputWriters/output_writer_utils.jl +++ b/src/OutputWriters/output_writer_utils.jl @@ -6,6 +6,7 @@ using Oceananigans.Fields: AbstractField, indices, boundary_conditions, instanti using Oceananigans.BoundaryConditions: bc_str, FieldBoundaryConditions, ContinuousBoundaryFunction, DiscreteBoundaryFunction using Oceananigans.TimeSteppers: QuasiAdamsBashforth2TimeStepper, RungeKutta3TimeStepper using Oceananigans.Models.LagrangianParticleTracking: LagrangianParticles +using Oceananigans.Models.HydrostaticFreeSurfaceModels: GeneralizedSpacingGrid, retrieve_static_grid ##### ##### Output writer utilities @@ -48,6 +49,11 @@ function saveproperty!(file, address, grid::DistributedGrid) _saveproperty!(file, address, on_architecture(cpu_arch, grid)) end +function saveproperty!(file, address, grid::GeneralizedSpacingGrid) + static_grid = retrieve_static_grid(grid) + saveproperty!(file, address, static_grid) +end + # Special saveproperty! so boundary conditions are easily readable outside julia. function saveproperty!(file, address, bcs::FieldBoundaryConditions) for boundary in propertynames(bcs) @@ -89,6 +95,11 @@ function serializeproperty!(file, address, grid::DistributedGrid) file[address] = on_architecture(cpu_arch, grid) end +function serializeproperty!(file, address, grid::GeneralizedSpacingGrid) + static_grid = retrieve_static_grid(grid) + serializeproperty!(file, address, static_grid) +end + function serializeproperty!(file, address, p::FieldBoundaryConditions) # TODO: it'd be better to "filter" `FieldBoundaryCondition` and then serialize # rather than punting with `missing` instead. diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index 492bb1f101..10b3d3712e 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -2,7 +2,7 @@ using Oceananigans using Oceananigans.Units using Oceananigans.Utils: prettytime using Oceananigans.Advection: WENOVectorInvariant -using Oceananigans.Models.HydrostaticFreeSurfaceModels: ZStar, ZStarSpacingGrid +using Oceananigans.Models.HydrostaticFreeSurfaceModels: ZStar, ZStarSpacingGrid, Δzᶜᶜᶜ_reference using Printf grid = RectilinearGrid(size = (10, 10), @@ -12,7 +12,7 @@ grid = RectilinearGrid(size = (10, 10), topology = (Bounded, Flat, Bounded)) model = HydrostaticFreeSurfaceModel(; grid, - generalized_vertical_coordinate = ZStar(), + generalized_vertical_coordinate = nothing, momentum_advection = Centered(), tracer_advection = Centered(), buoyancy = BuoyancyTracer(), @@ -72,23 +72,25 @@ simulation.callbacks[:wizard] = Callback(TimeStepWizard(; cfl = 0.2, max_chang run!(simulation) -# # Check tracer conservation -if model.grid isa ZStarSpacingGrid - b = FieldTimeSeries("zstar_model.jld2", "b") - dz = FieldTimeSeries("zstar_model.jld2", "sⁿ") - - init = sum(b[1] * dz[1]) / sum(dz[1]) - drift = [] - for t in 1:length(b.times) - push!(drift, sum(b[t] * dz[t]) / sum(dz[t]) - init) - end -else - b = FieldTimeSeries("zstar_model.jld2", "b") +using Oceananigans.Utils + +@kernel function _compute_field!(tmp, s, b) + i, j, k = @index(Global, NTuple) + @inbounds tmp[i, j, k] = s[i, j, k] * b[i, j, k] +end - init = sum(b[1]) / prod(size(b[1])) - drift = [] - for t in 1:length(b.times) +using Oceananigans.Fields: OneField + +# # Check tracer conservation +b = FieldTimeSeries("zstar_model.jld2", "b") +# s = FieldTimeSeries("zstar_model.jld2", "sⁿ") +s = OneField() +tmpfield = CenterField(grid) +launch!(CPU(), grid, :xyz, _compute_field!, tmpfield, s, b[1]) +init = sum(tmpfield) / 10 # sum(s[1]) +drift = [] +for t in 1:length(b.times) + launch!(CPU(), grid, :xyz, _compute_field!, tmpfield, s, b[t]) + push!(drift, sum(tmpfield) / 10 - init) +end - push!(drift, sum(b[t]) / prod(size(b[1])) - init) - end -end \ No newline at end of file From c45617bf7cc711b715828e26ef946f209b1b19c5 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Fri, 8 Mar 2024 08:40:26 -0500 Subject: [PATCH 236/567] bugfix --- src/Fields/field.jl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Fields/field.jl b/src/Fields/field.jl index 27bf1c2c3c..0c19664430 100644 --- a/src/Fields/field.jl +++ b/src/Fields/field.jl @@ -420,12 +420,12 @@ total_size(f::Field) = total_size(f.grid, location(f), f.indices) on_architecture(arch, field::AbstractField{LX, LY, LZ}) where {LX, LY, LZ} = Field{LX, LY, LZ}(on_architecture(arch, field.grid), - on_architecture(arch, data), - on_architecture(arch, bcs), - on_architecture(arch, indices), - on_architecture(arch, op), - on_architecture(arch, status), - on_architecture(arch, buffers)) + on_architecture(arch, field.data), + on_architecture(arch, field.bcs), + on_architecture(arch, field.indices), + on_architecture(arch, field.op), + on_architecture(arch, field.status), + on_architecture(arch, field.buffers)) ##### ##### Interface for field computations From 824edd87e58c8e0fe37662c2cf76ee2ef6884460 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Fri, 8 Mar 2024 09:51:22 -0500 Subject: [PATCH 237/567] bugfix --- src/Fields/field.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Fields/field.jl b/src/Fields/field.jl index 0c19664430..f32f70fb8a 100644 --- a/src/Fields/field.jl +++ b/src/Fields/field.jl @@ -421,11 +421,11 @@ total_size(f::Field) = total_size(f.grid, location(f), f.indices) on_architecture(arch, field::AbstractField{LX, LY, LZ}) where {LX, LY, LZ} = Field{LX, LY, LZ}(on_architecture(arch, field.grid), on_architecture(arch, field.data), - on_architecture(arch, field.bcs), + on_architecture(arch, field.boundary_conditions), on_architecture(arch, field.indices), on_architecture(arch, field.op), on_architecture(arch, field.status), - on_architecture(arch, field.buffers)) + on_architecture(arch, field.boundary_buffers)) ##### ##### Interface for field computations From 0043d9cec106b047accb9309a1be3d8bf601b93b Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Fri, 8 Mar 2024 10:53:49 -0500 Subject: [PATCH 238/567] last bugfix --- src/Fields/field.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Fields/field.jl b/src/Fields/field.jl index f32f70fb8a..d0b82dcedd 100644 --- a/src/Fields/field.jl +++ b/src/Fields/field.jl @@ -423,7 +423,7 @@ on_architecture(arch, field::AbstractField{LX, LY, LZ}) where {LX, LY, LZ} = on_architecture(arch, field.data), on_architecture(arch, field.boundary_conditions), on_architecture(arch, field.indices), - on_architecture(arch, field.op), + on_architecture(arch, field.operand), on_architecture(arch, field.status), on_architecture(arch, field.boundary_buffers)) From 67ec92e4bb006e7f3001fd48c9c59ecd6b63ab76 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Fri, 8 Mar 2024 17:14:41 -0500 Subject: [PATCH 239/567] last bugfix --- test/test_distributed_hydrostatic_model.jl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/test_distributed_hydrostatic_model.jl b/test/test_distributed_hydrostatic_model.jl index 43d5d46e41..7c75436d3f 100644 --- a/test/test_distributed_hydrostatic_model.jl +++ b/test/test_distributed_hydrostatic_model.jl @@ -107,14 +107,14 @@ for arch in archs @info " Testing distributed solid body rotation with architecture $arch on $(typeof(grid).name.wrapper)" u, v, w, c, η = solid_body_rotation_test(grid) - u = interior(on_architecture(CPU(), u)) - v = interior(on_architecture(CPU(), v)) - w = interior(on_architecture(CPU(), w)) - c = interior(on_architecture(CPU(), c)) - η = interior(on_architecture(CPU(), η)) - cpu_arch = cpu_architecture(arch) + u = interior(on_architecture(cpu_arch, u)) + v = interior(on_architecture(cpu_arch, v)) + w = interior(on_architecture(cpu_arch, w)) + c = interior(on_architecture(cpu_arch, c)) + η = interior(on_architecture(cpu_arch, η)) + us = partition_global_array(cpu_arch, us, size(u)) vs = partition_global_array(cpu_arch, vs, size(v)) ws = partition_global_array(cpu_arch, ws, size(w)) From 9b3246b34b46e8ff7055decd6a434f4be5da483b Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Fri, 8 Mar 2024 18:56:44 -0500 Subject: [PATCH 240/567] small test --- src/Advection/weno_interpolants.jl | 47 ++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/src/Advection/weno_interpolants.jl b/src/Advection/weno_interpolants.jl index 7c96b00adb..d3ff94f740 100644 --- a/src/Advection/weno_interpolants.jl +++ b/src/Advection/weno_interpolants.jl @@ -307,7 +307,7 @@ julia> calc_weno_stencil(2, :right, :x) :(((ψ[i + 0, j, k], ψ[i + 1, j, k]), (ψ[i + -1, j, k], ψ[i + 0, j, k]))) """ -@inline function calc_weno_stencil(buffer, shift, dir, func::Bool = false) +@inline function calc_weno_stencil_function(buffer, shift, dir) N = buffer * 2 if shift != :none N -=1 @@ -322,25 +322,40 @@ julia> calc_weno_stencil(2, :right, :x) rngstencil = rng[stencil:stencil+buffer-1] for (idx, n) in enumerate(rngstencil) c = n - buffer - 1 - if func - stencil_point[idx] = dir == :x ? - :(@inbounds ψ(i + $c, j, k, args...)) : - dir == :y ? - :(@inbounds ψ(i, j + $c, k, args...)) : - :(@inbounds ψ(i, j, k + $c, args...)) - else - stencil_point[idx] = dir == :x ? - :(@inbounds ψ[i + $c, j, k]) : - dir == :y ? - :(@inbounds ψ[i, j + $c, k]) : - :(@inbounds ψ[i, j, k + $c]) - end + stencil_point[idx] = dir == :x ? + :(ψ(i + $c, j, k, args...)) : + dir == :y ? + :(ψ(i, j + $c, k, args...)) : + :(ψ(i, j, k + $c, args...)) end stencil_full[buffer - stencil + 1] = :($(stencil_point...), ) end return :($(stencil_full...),) end +@inline function calc_weno_stencil_variable(buffer, shift, dir) + N = buffer * 2 + if shift != :none + N -=1 + end + stencil_full = Vector(undef, buffer) + rng = 1:N + if shift == :right + rng = rng .+ 1 + end + for stencil in 1:buffer + rngstencil = rng[stencil:stencil+buffer-1] + c_min = rngstencil[1] - buffer - 1 + c_max = rngstencil[end] - buffer - 1 + stencil_full[buffer - stencil + 1] = dir == :x ? + :(view(ψ, i+$c_min:i+$c_max, j, k)) : + dir == :y ? + :(view(ψ, i, j+$c_min:j+$c_max, k)) : + :(view(ψ, i, j, k+$c_min:k+$c_max)) + end + return :($(stencil_full...),) +end + # Stencils for left and right biased reconstruction ((ψ̅ᵢ₋ᵣ₊ⱼ for j in 0:k) for r in 0:k) to calculate v̂ᵣ = ∑ⱼ(cᵣⱼψ̅ᵢ₋ᵣ₊ⱼ) # where `k = N - 1`. Coefficients (cᵣⱼ for j in 0:N) for stencil r are given by `coeff_side_p(scheme, Val(r), ...)` for side in (:left, :right), dir in (:x, :y, :z) @@ -348,8 +363,8 @@ for side in (:left, :right), dir in (:x, :y, :z) for buffer in [2, 3, 4, 5, 6] @eval begin - @inline $stencil(i, j, k, scheme::WENO{$buffer}, ψ, args...) = @inbounds $(calc_weno_stencil(buffer, side, dir, false)) - @inline $stencil(i, j, k, scheme::WENO{$buffer}, ψ::Function, args...) = @inbounds $(calc_weno_stencil(buffer, side, dir, true)) + @inline $stencil(i, j, k, scheme::WENO{$buffer}, ψ, args...) = @inbounds $(calc_weno_stencil_variable(buffer, side, dir)) + @inline $stencil(i, j, k, scheme::WENO{$buffer}, ψ::Function, args...) = @inbounds $(calc_weno_stencil_function(buffer, side, dir)) end end end From 8e1a99fb2274f00a7506bcde6060022fd20abcde Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Fri, 8 Mar 2024 19:08:48 -0500 Subject: [PATCH 241/567] new tests --- src/Advection/weno_interpolants.jl | 45 +++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/src/Advection/weno_interpolants.jl b/src/Advection/weno_interpolants.jl index d3ff94f740..dd19029673 100644 --- a/src/Advection/weno_interpolants.jl +++ b/src/Advection/weno_interpolants.jl @@ -114,8 +114,22 @@ for buffer in [2, 3, 4, 5, 6] # left biased and right biased reconstruction value for each stencil @eval begin - @inline left_biased_p(scheme::WENO{$buffer}, ::Val{$stencil}, ψ, T, dir, i, loc) = @inbounds sum(coeff_left_p(scheme, Val($stencil), T, dir, i, loc) .* ψ) - @inline right_biased_p(scheme::WENO{$buffer}, ::Val{$stencil}, ψ, T, dir, i, loc) = @inbounds sum(coeff_right_p(scheme, Val($stencil), T, dir, i, loc) .* ψ) + @inline function left_biased_p(scheme::WENO{$buffer}, ::Val{$stencil}, ψ, T, dir, i, loc) + r = 0 + C = coeff_left_p(scheme, Val($stencil), T, dir, i, loc) + ntuple(Val($buffer)) do n + @inbounds r += C[n] * ψ[n] + end + return r + end + @inline function right_biased_p(scheme::WENO{$buffer}, ::Val{$stencil}, ψ, T, dir, i, loc) + r = 0 + C = coeff_right_p(scheme, Val($stencil), T, dir, i, loc) + ntuple(Val($buffer)) do n + @inbounds r += C[n] * ψ[n] + end + return r + end end end end @@ -333,7 +347,7 @@ julia> calc_weno_stencil(2, :right, :x) return :($(stencil_full...),) end -@inline function calc_weno_stencil_variable(buffer, shift, dir) +@inline function calc_weno_stencil_variable(buffer, shift, dir, field::Bool = false) N = buffer * 2 if shift != :none N -=1 @@ -347,15 +361,25 @@ end rngstencil = rng[stencil:stencil+buffer-1] c_min = rngstencil[1] - buffer - 1 c_max = rngstencil[end] - buffer - 1 - stencil_full[buffer - stencil + 1] = dir == :x ? - :(view(ψ, i+$c_min:i+$c_max, j, k)) : - dir == :y ? - :(view(ψ, i, j+$c_min:j+$c_max, k)) : - :(view(ψ, i, j, k+$c_min:k+$c_max)) + if field + stencil_full[buffer - stencil + 1] = dir == :x ? + :(view(ψ.data, i+$c_min:i+$c_max, j, k)) : + dir == :y ? + :(view(ψ.data, i, j+$c_min:j+$c_max, k)) : + :(view(ψ.data, i, j, k+$c_min:k+$c_max)) + else + stencil_full[buffer - stencil + 1] = dir == :x ? + :(view(ψ, i+$c_min:i+$c_max, j, k)) : + dir == :y ? + :(view(ψ, i, j+$c_min:j+$c_max, k)) : + :(view(ψ, i, j, k+$c_min:k+$c_max)) + end end return :($(stencil_full...),) end +using Oceananigans.Fields: Field + # Stencils for left and right biased reconstruction ((ψ̅ᵢ₋ᵣ₊ⱼ for j in 0:k) for r in 0:k) to calculate v̂ᵣ = ∑ⱼ(cᵣⱼψ̅ᵢ₋ᵣ₊ⱼ) # where `k = N - 1`. Coefficients (cᵣⱼ for j in 0:N) for stencil r are given by `coeff_side_p(scheme, Val(r), ...)` for side in (:left, :right), dir in (:x, :y, :z) @@ -363,8 +387,9 @@ for side in (:left, :right), dir in (:x, :y, :z) for buffer in [2, 3, 4, 5, 6] @eval begin - @inline $stencil(i, j, k, scheme::WENO{$buffer}, ψ, args...) = @inbounds $(calc_weno_stencil_variable(buffer, side, dir)) - @inline $stencil(i, j, k, scheme::WENO{$buffer}, ψ::Function, args...) = @inbounds $(calc_weno_stencil_function(buffer, side, dir)) + @inline $stencil(i, j, k, scheme::WENO{$buffer}, ψ::Field, args...) = @inbounds $(calc_weno_stencil_variable(buffer, side, dir, false)) + @inline $stencil(i, j, k, scheme::WENO{$buffer}, ψ::OffsetArray, args...) = @inbounds $(calc_weno_stencil_variable(buffer, side, dir, true)) + @inline $stencil(i, j, k, scheme::WENO{$buffer}, ψ::Function, args...) = @inbounds $(calc_weno_stencil_function(buffer, side, dir)) end end end From c12ebae8474d4dce5ebaed985239f1a060ee0e1e Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sat, 9 Mar 2024 12:37:27 -0500 Subject: [PATCH 242/567] bugfix --- src/Advection/weno_interpolants.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Advection/weno_interpolants.jl b/src/Advection/weno_interpolants.jl index dd19029673..da87296723 100644 --- a/src/Advection/weno_interpolants.jl +++ b/src/Advection/weno_interpolants.jl @@ -387,8 +387,8 @@ for side in (:left, :right), dir in (:x, :y, :z) for buffer in [2, 3, 4, 5, 6] @eval begin - @inline $stencil(i, j, k, scheme::WENO{$buffer}, ψ::Field, args...) = @inbounds $(calc_weno_stencil_variable(buffer, side, dir, false)) - @inline $stencil(i, j, k, scheme::WENO{$buffer}, ψ::OffsetArray, args...) = @inbounds $(calc_weno_stencil_variable(buffer, side, dir, true)) + @inline $stencil(i, j, k, scheme::WENO{$buffer}, ψ::Field, args...) = @inbounds $(calc_weno_stencil_variable(buffer, side, dir, true)) + @inline $stencil(i, j, k, scheme::WENO{$buffer}, ψ::OffsetArray, args...) = @inbounds $(calc_weno_stencil_variable(buffer, side, dir, false)) @inline $stencil(i, j, k, scheme::WENO{$buffer}, ψ::Function, args...) = @inbounds $(calc_weno_stencil_function(buffer, side, dir)) end end From 9499655744ca380283704fb31e231888c5c6671c Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sat, 9 Mar 2024 14:28:24 -0500 Subject: [PATCH 243/567] try it like this --- src/Advection/weno_interpolants.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Advection/weno_interpolants.jl b/src/Advection/weno_interpolants.jl index da87296723..0e2f347f55 100644 --- a/src/Advection/weno_interpolants.jl +++ b/src/Advection/weno_interpolants.jl @@ -117,7 +117,7 @@ for buffer in [2, 3, 4, 5, 6] @inline function left_biased_p(scheme::WENO{$buffer}, ::Val{$stencil}, ψ, T, dir, i, loc) r = 0 C = coeff_left_p(scheme, Val($stencil), T, dir, i, loc) - ntuple(Val($buffer)) do n + @unroll for n in 1:$buffer @inbounds r += C[n] * ψ[n] end return r @@ -125,7 +125,7 @@ for buffer in [2, 3, 4, 5, 6] @inline function right_biased_p(scheme::WENO{$buffer}, ::Val{$stencil}, ψ, T, dir, i, loc) r = 0 C = coeff_right_p(scheme, Val($stencil), T, dir, i, loc) - ntuple(Val($buffer)) do n + @unroll for n in 1:$buffer @inbounds r += C[n] * ψ[n] end return r From cfede07e6129e1742d2d644c49c7c24cdd085a4c Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sat, 9 Mar 2024 14:30:44 -0500 Subject: [PATCH 244/567] bugfix --- src/Advection/weno_interpolants.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Advection/weno_interpolants.jl b/src/Advection/weno_interpolants.jl index 0e2f347f55..5b60073539 100644 --- a/src/Advection/weno_interpolants.jl +++ b/src/Advection/weno_interpolants.jl @@ -16,6 +16,8 @@ # where the weights wᵣ are calculated dynamically with `side_biased_weno_weights(ψ, scheme)`. # +using KernelAbstractions.Extras.LoopInfo: @unroll + """ `AbstractSmoothnessStencil`s specifies the polynomials used for diagnosing stencils' smoothness for weno weights calculation in the `VectorInvariant` advection formulation. From b50515e718b005a4011aa29d366298bc0f8ec911 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sat, 9 Mar 2024 14:47:28 -0500 Subject: [PATCH 245/567] bugfix? --- src/AbstractOperations/conditional_operations.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AbstractOperations/conditional_operations.jl b/src/AbstractOperations/conditional_operations.jl index e621bcd262..a6d40b7b53 100644 --- a/src/AbstractOperations/conditional_operations.jl +++ b/src/AbstractOperations/conditional_operations.jl @@ -1,7 +1,7 @@ using Oceananigans.Fields: OneField using Oceananigans.Grids: architecture -using Oceananigans.Architectures: on_architecture import Oceananigans.Fields: condition_operand, conditional_length, set!, compute_at!, indices +import Oceananigans.Architectures: architecture, on_architecture # For conditional reductions such as mean(u * v, condition = u .> 0)) struct ConditionalOperation{LX, LY, LZ, O, F, G, C, M, T} <: AbstractOperation{LX, LY, LZ, G, T} From 74b7b15dbf98f6ca1bcd00e1acbf155d0808bd13 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sat, 9 Mar 2024 14:51:44 -0500 Subject: [PATCH 246/567] another bugfix --- src/DistributedComputations/partition_assemble.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DistributedComputations/partition_assemble.jl b/src/DistributedComputations/partition_assemble.jl index d381743cb1..a6e730cf47 100644 --- a/src/DistributedComputations/partition_assemble.jl +++ b/src/DistributedComputations/partition_assemble.jl @@ -1,4 +1,4 @@ -using Oceananigans.Architectures: on_architecture +import Oceananigans.Architectures: on_architecture all_reduce(op, val, arch::Distributed) = MPI.Allreduce(val, op, arch.communicator) From 651191e383117190bc85edca8d08c39cbe29cf91 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sun, 10 Mar 2024 13:59:45 -0400 Subject: [PATCH 247/567] some fixes... --- src/Fields/field_boundary_buffers.jl | 1 - src/Fields/regridding_fields.jl | 2 +- src/Fields/set!.jl | 2 +- src/ImmersedBoundaries/grid_fitted_bottom.jl | 2 +- src/ImmersedBoundaries/partial_cell_bottom.jl | 3 ++- src/MultiRegion/multi_region_boundary_conditions.jl | 3 ++- src/MultiRegion/multi_region_grid.jl | 2 +- src/MultiRegion/unified_implicit_free_surface_solver.jl | 3 +-- src/TurbulenceClosures/discrete_diffusion_function.jl | 1 + .../implicit_explicit_time_discretization.jl | 3 ++- .../convective_adjustment_vertical_diffusivity.jl | 4 +++- .../ri_based_vertical_diffusivity.jl | 4 +++- 12 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/Fields/field_boundary_buffers.jl b/src/Fields/field_boundary_buffers.jl index e514a3be3e..f4bd99fcb8 100644 --- a/src/Fields/field_boundary_buffers.jl +++ b/src/Fields/field_boundary_buffers.jl @@ -1,5 +1,4 @@ using Oceananigans.BoundaryConditions: MCBC, DCBC -using Oceananigans.Architectures: on_architecture using Oceananigans.Grids: halo_size, size using Oceananigans.Utils: launch! using KernelAbstractions: @kernel, @index diff --git a/src/Fields/regridding_fields.jl b/src/Fields/regridding_fields.jl index 758257f369..4ab24aa591 100644 --- a/src/Fields/regridding_fields.jl +++ b/src/Fields/regridding_fields.jl @@ -1,6 +1,6 @@ using KernelAbstractions: @kernel, @index -using Oceananigans.Architectures: on_architecture, architecture +using Oceananigans.Architectures: architecture using Oceananigans.Operators: Δzᶜᶜᶜ, Δyᶜᶜᶜ, Δxᶜᶜᶜ, Azᶜᶜᶜ using Oceananigans.Grids: hack_sind, ξnode, ηnode, rnode diff --git a/src/Fields/set!.jl b/src/Fields/set!.jl index f2e89a7817..661e68f1c4 100644 --- a/src/Fields/set!.jl +++ b/src/Fields/set!.jl @@ -2,7 +2,7 @@ using CUDA using KernelAbstractions: @kernel, @index using Adapt: adapt_structure -using Oceananigans.Grids: on_architecture, node_names +using Oceananigans.Grids: node_names using Oceananigans.Architectures: device, GPU, CPU using Oceananigans.Utils: work_layout diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index a3461ac16d..c993a6fbb5 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -4,11 +4,11 @@ using OffsetArrays: OffsetArray using Oceananigans.Utils: getnamewrapper using Oceananigans.Grids: total_size using Oceananigans.Fields: fill_halo_regions! -using Oceananigans.Architectures: on_architecture using Oceananigans.BoundaryConditions: FBC using Printf import Oceananigans.TurbulenceClosures: z_bottom +import Oceananigans.Architectures: on_architecture ##### ##### GridFittedBottom (2.5D immersed boundary with modified bottom height) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index f6ba0368e3..cc30f3a10f 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -1,8 +1,9 @@ using Oceananigans.Utils: prettysummary using Oceananigans.Fields: fill_halo_regions! -using Oceananigans.Architectures: on_architecture using Printf +import Oceananigans.Architectures: on_architecture + ##### ##### PartialCellBottom ##### diff --git a/src/MultiRegion/multi_region_boundary_conditions.jl b/src/MultiRegion/multi_region_boundary_conditions.jl index 1910c8847e..58712a7c55 100644 --- a/src/MultiRegion/multi_region_boundary_conditions.jl +++ b/src/MultiRegion/multi_region_boundary_conditions.jl @@ -1,5 +1,5 @@ using Oceananigans: instantiated_location -using Oceananigans.Architectures: on_architecture, device_copy_to! +using Oceananigans.Architectures: device_copy_to! using Oceananigans.Operators: assumed_field_location using Oceananigans.Fields: reduced_dimensions using Oceananigans.DistributedComputations: communication_side @@ -15,6 +15,7 @@ using Oceananigans.BoundaryConditions: MCBC import Oceananigans.Fields: tupled_fill_halo_regions!, boundary_conditions, data, fill_send_buffers! +import Oceananigans.Architectures: on_architecture import Oceananigans.BoundaryConditions: fill_halo_regions!, diff --git a/src/MultiRegion/multi_region_grid.jl b/src/MultiRegion/multi_region_grid.jl index 707de5bd3c..7bc6fb3cea 100644 --- a/src/MultiRegion/multi_region_grid.jl +++ b/src/MultiRegion/multi_region_grid.jl @@ -1,4 +1,4 @@ -using Oceananigans.Grids: metrics_precomputed, on_architecture, pop_flat_elements, grid_name +using Oceananigans.Grids: metrics_precomputed, pop_flat_elements, grid_name using Oceananigans.ImmersedBoundaries: GridFittedBottom, PartialCellBottom, GridFittedBoundary import Oceananigans.Grids: architecture, size, new_data, halo_size diff --git a/src/MultiRegion/unified_implicit_free_surface_solver.jl b/src/MultiRegion/unified_implicit_free_surface_solver.jl index a6370ca97c..45080b3572 100644 --- a/src/MultiRegion/unified_implicit_free_surface_solver.jl +++ b/src/MultiRegion/unified_implicit_free_surface_solver.jl @@ -1,7 +1,6 @@ using Oceananigans.Solvers using Oceananigans.Operators using Oceananigans.Architectures -using Oceananigans.Grids: on_architecture using Oceananigans.Fields: Field using Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_vertically_integrated_lateral_areas!, @@ -12,7 +11,7 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_vertically_integ import Oceananigans.Models.HydrostaticFreeSurfaceModels: build_implicit_step_solver, compute_implicit_free_surface_right_hand_side! -import Oceananigans.Architectures: architecture +import Oceananigans.Architectures: architecture, on_architecture import Oceananigans.Solvers: solve! struct UnifiedImplicitFreeSurfaceSolver{S, R, T} diff --git a/src/TurbulenceClosures/discrete_diffusion_function.jl b/src/TurbulenceClosures/discrete_diffusion_function.jl index 19d8203783..8d5bc29d7b 100644 --- a/src/TurbulenceClosures/discrete_diffusion_function.jl +++ b/src/TurbulenceClosures/discrete_diffusion_function.jl @@ -1,5 +1,6 @@ using Oceananigans.Operators: ℑxyz using Oceananigans.Utils: instantiate +import Oceananigans.Architectures: on_architecture """ struct DiscreteDiffusionFunction{LX, LY, LZ, P, F} diff --git a/src/TurbulenceClosures/implicit_explicit_time_discretization.jl b/src/TurbulenceClosures/implicit_explicit_time_discretization.jl index 33fe206185..603ef75d55 100644 --- a/src/TurbulenceClosures/implicit_explicit_time_discretization.jl +++ b/src/TurbulenceClosures/implicit_explicit_time_discretization.jl @@ -1,6 +1,7 @@ -using Oceananigans.Utils: on_architecture using Oceananigans.Grids: AbstractGrid +import Oceananigans.Architectures: on_architecture + abstract type AbstractTimeDiscretization end """ diff --git a/src/TurbulenceClosures/turbulence_closure_implementations/convective_adjustment_vertical_diffusivity.jl b/src/TurbulenceClosures/turbulence_closure_implementations/convective_adjustment_vertical_diffusivity.jl index a973edc0a0..53df44fec9 100644 --- a/src/TurbulenceClosures/turbulence_closure_implementations/convective_adjustment_vertical_diffusivity.jl +++ b/src/TurbulenceClosures/turbulence_closure_implementations/convective_adjustment_vertical_diffusivity.jl @@ -1,8 +1,10 @@ -using Oceananigans.Architectures: architecture, on_architecture +using Oceananigans.Architectures: architecture using Oceananigans.AbstractOperations: KernelFunctionOperation using Oceananigans.BuoyancyModels: ∂z_b using Oceananigans.Operators: ℑzᵃᵃᶜ +import Oceananigans.Architectures: on_architecture + struct ConvectiveAdjustmentVerticalDiffusivity{TD, CK, CN, BK, BN} <: AbstractScalarDiffusivity{TD, VerticalFormulation, 1} convective_κz :: CK convective_νz :: CN diff --git a/src/TurbulenceClosures/turbulence_closure_implementations/ri_based_vertical_diffusivity.jl b/src/TurbulenceClosures/turbulence_closure_implementations/ri_based_vertical_diffusivity.jl index 21f44f3000..693a0ebf00 100644 --- a/src/TurbulenceClosures/turbulence_closure_implementations/ri_based_vertical_diffusivity.jl +++ b/src/TurbulenceClosures/turbulence_closure_implementations/ri_based_vertical_diffusivity.jl @@ -1,9 +1,11 @@ -using Oceananigans.Architectures: architecture, on_architecture +using Oceananigans.Architectures: architecture using Oceananigans.BuoyancyModels: ∂z_b using Oceananigans.Operators using Oceananigans.Grids: inactive_node using Oceananigans.Operators: ℑzᵃᵃᶜ +import Oceananigans.Architectures: on_architecture + struct RiBasedVerticalDiffusivity{TD, FT, R} <: AbstractScalarDiffusivity{TD, VerticalFormulation, 1} ν₀ :: FT κ₀ :: FT From 17fe7103edaa166968014734748729afdc97523e Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sun, 10 Mar 2024 15:34:34 -0400 Subject: [PATCH 248/567] try it out like this --- .../HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 1afa4e3bae..5ddfe42150 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -113,7 +113,7 @@ end ##### ZStar-specific implementation of the additional terms to be included in the momentum equations ##### -@inline η_surfaceᶜᶜᶜ(i, j, k, grid, η, Hᶜᶜ) = @inbounds η[i, j, grid.Nz+1] * (1 + znode(i, j, k, grid, c, c, c) / Hᶜᶜ[i, j, 1]) +@inline η_surfaceᶜᶜᶜ(i, j, k, grid, η, Hᶜᶜ) = @inbounds η[i, j, grid.Nz+1] * (1 + znode(i, j, k, grid, Center(), Center(), Center()) / Hᶜᶜ[i, j, 1]) @inline slope_xᶠᶜᶜ(i, j, k, grid, free_surface) = @inbounds ∂xᶠᶜᶜ(i, j, k, grid, η_surfaceᶜᶜᶜ, free_surface.η, free_surface.auxiliary.Hᶜᶜ) @inline slope_yᶜᶠᶜ(i, j, k, grid, free_surface) = @inbounds ∂yᶜᶠᶜ(i, j, k, grid, η_surfaceᶜᶜᶜ, free_surface.η, free_surface.auxiliary.Hᶜᶜ) From 2e3da975a23c7c6e80492518a16211d31f886add Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sun, 10 Mar 2024 15:56:40 -0400 Subject: [PATCH 249/567] adapt correctly --- .../split_explicit_free_surface.jl | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index 071d0a38cd..470ea3a5a8 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -399,9 +399,20 @@ end # Adapt Adapt.adapt_structure(to, free_surface::SplitExplicitFreeSurface) = - SplitExplicitFreeSurface(Adapt.adapt(to, free_surface.η), nothing, nothing, + SplitExplicitFreeSurface(Adapt.adapt(to, free_surface.η), + nothing, + Adapt.adapt(to, free_surface.auxiliary), free_surface.gravitational_acceleration, nothing) +# Adapt +Adapt.adapt_structure(to, auxiliary::SplitExplicitAuxiliaryFields) = + SplitExplicitAuxiliaryFields(nothing, + nothing, + Adapt.adapt(to, auxiliary.Hᶠᶜ), + Adapt.adapt(to, auxiliary.Hᶜᶠ), + Adapt.adapt(to, auxiliary.Hᶜᶜ), + nothing) + for Type in (:SplitExplicitFreeSurface, :SplitExplicitSettings, :SplitExplicitState, From eabeac735226ce7f7c68e44fd90fe21fd594745f Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sun, 10 Mar 2024 16:09:27 -0400 Subject: [PATCH 250/567] let's benchmark the performance --- src/Advection/shared_tracer_advection.jl | 70 --------------------- src/Advection/weno_interpolants.jl | 78 ++++++------------------ 2 files changed, 18 insertions(+), 130 deletions(-) delete mode 100644 src/Advection/shared_tracer_advection.jl diff --git a/src/Advection/shared_tracer_advection.jl b/src/Advection/shared_tracer_advection.jl deleted file mode 100644 index 5722e55a80..0000000000 --- a/src/Advection/shared_tracer_advection.jl +++ /dev/null @@ -1,70 +0,0 @@ -using KernelAbstractions: @index, @kernel - -function tracer_advection(Gⁿ, tracers, grid, tracer_advection, velocities; parameters = :xyz) - - return nothing -end - -@inline retrieve_x_advection(tracer_advection) = tracer_advection -@inline retrieve_y_advection(tracer_advection) = tracer_advection -@inline retrieve_z_advection(tracer_advection) = tracer_advection - -@inline retrieve_x_advection(tracer_advection::TracerAdvection) = tracer_advection.x -@inline retrieve_y_advection(tracer_advection::TracerAdvection) = tracer_advection.y -@inline retrieve_z_advection(tracer_advection::TracerAdvection) = tracer_advection.z - -@kernel function tracer_advection_x(Gⁿ, tracers, grid, tracer_advection, U, ::Val{Ntracers}) where Ntracers - global_id = @index(Global, Linear) - block_id = - thread_id = @index(Local) - - k_id = - - tracer_flux = @localmem FT (, ) - - @unroll for t in 1:Ntracers - tracer_name = @inbounds tracer_names[t] - tracer = @inbounds tracers[t] - advection = @inbounds tracer_advection[tracer_name] - tendency = @inbounds Gⁿ[tracer_name] - advection = retrieve_x_advection(advection) - - # calculate fluxes into the shared memory - tracer_flux[] = _advective_tracer_flux_x(i, j, k, grid, advection, U, c) - - @synchronize - # put div in memory - tendency[i, j, k] = (tracer_flux[t] - tracer_flux[t-1]) / Vᶜᶜᶜ(i, j, k, grid) - end -end - -@kernel function tracer_advection_y(Gⁿ, tracers, grid, tracer_advection, V) - i, j, k = @index(Global, NTuple) - - # Allocate shared memory of the dimensions of the grid - - - for (c, advection) in zip(tracers, tracer_advection) - # calculate fluxes into the shared memory - - __syncthreads() - # put div in memory - - # exit - end -end - -@kernel function tracer_advection_z(Gⁿ, tracers, grid::ActiveCellsIBG, tracer_advection, W) - i, j, k = @index(Global, NTuple) - - # Allocate shared memory of the dimensions of the grid - - for (c, advection) in zip(tracers, tracer_advection) - # calculate fluxes into the shared memory - - __syncthreads() - # put div in memory - - # exit - end -end \ No newline at end of file diff --git a/src/Advection/weno_interpolants.jl b/src/Advection/weno_interpolants.jl index 5b60073539..7c96b00adb 100644 --- a/src/Advection/weno_interpolants.jl +++ b/src/Advection/weno_interpolants.jl @@ -16,8 +16,6 @@ # where the weights wᵣ are calculated dynamically with `side_biased_weno_weights(ψ, scheme)`. # -using KernelAbstractions.Extras.LoopInfo: @unroll - """ `AbstractSmoothnessStencil`s specifies the polynomials used for diagnosing stencils' smoothness for weno weights calculation in the `VectorInvariant` advection formulation. @@ -116,22 +114,8 @@ for buffer in [2, 3, 4, 5, 6] # left biased and right biased reconstruction value for each stencil @eval begin - @inline function left_biased_p(scheme::WENO{$buffer}, ::Val{$stencil}, ψ, T, dir, i, loc) - r = 0 - C = coeff_left_p(scheme, Val($stencil), T, dir, i, loc) - @unroll for n in 1:$buffer - @inbounds r += C[n] * ψ[n] - end - return r - end - @inline function right_biased_p(scheme::WENO{$buffer}, ::Val{$stencil}, ψ, T, dir, i, loc) - r = 0 - C = coeff_right_p(scheme, Val($stencil), T, dir, i, loc) - @unroll for n in 1:$buffer - @inbounds r += C[n] * ψ[n] - end - return r - end + @inline left_biased_p(scheme::WENO{$buffer}, ::Val{$stencil}, ψ, T, dir, i, loc) = @inbounds sum(coeff_left_p(scheme, Val($stencil), T, dir, i, loc) .* ψ) + @inline right_biased_p(scheme::WENO{$buffer}, ::Val{$stencil}, ψ, T, dir, i, loc) = @inbounds sum(coeff_right_p(scheme, Val($stencil), T, dir, i, loc) .* ψ) end end end @@ -323,7 +307,7 @@ julia> calc_weno_stencil(2, :right, :x) :(((ψ[i + 0, j, k], ψ[i + 1, j, k]), (ψ[i + -1, j, k], ψ[i + 0, j, k]))) """ -@inline function calc_weno_stencil_function(buffer, shift, dir) +@inline function calc_weno_stencil(buffer, shift, dir, func::Bool = false) N = buffer * 2 if shift != :none N -=1 @@ -338,50 +322,25 @@ julia> calc_weno_stencil(2, :right, :x) rngstencil = rng[stencil:stencil+buffer-1] for (idx, n) in enumerate(rngstencil) c = n - buffer - 1 - stencil_point[idx] = dir == :x ? - :(ψ(i + $c, j, k, args...)) : - dir == :y ? - :(ψ(i, j + $c, k, args...)) : - :(ψ(i, j, k + $c, args...)) + if func + stencil_point[idx] = dir == :x ? + :(@inbounds ψ(i + $c, j, k, args...)) : + dir == :y ? + :(@inbounds ψ(i, j + $c, k, args...)) : + :(@inbounds ψ(i, j, k + $c, args...)) + else + stencil_point[idx] = dir == :x ? + :(@inbounds ψ[i + $c, j, k]) : + dir == :y ? + :(@inbounds ψ[i, j + $c, k]) : + :(@inbounds ψ[i, j, k + $c]) + end end stencil_full[buffer - stencil + 1] = :($(stencil_point...), ) end return :($(stencil_full...),) end -@inline function calc_weno_stencil_variable(buffer, shift, dir, field::Bool = false) - N = buffer * 2 - if shift != :none - N -=1 - end - stencil_full = Vector(undef, buffer) - rng = 1:N - if shift == :right - rng = rng .+ 1 - end - for stencil in 1:buffer - rngstencil = rng[stencil:stencil+buffer-1] - c_min = rngstencil[1] - buffer - 1 - c_max = rngstencil[end] - buffer - 1 - if field - stencil_full[buffer - stencil + 1] = dir == :x ? - :(view(ψ.data, i+$c_min:i+$c_max, j, k)) : - dir == :y ? - :(view(ψ.data, i, j+$c_min:j+$c_max, k)) : - :(view(ψ.data, i, j, k+$c_min:k+$c_max)) - else - stencil_full[buffer - stencil + 1] = dir == :x ? - :(view(ψ, i+$c_min:i+$c_max, j, k)) : - dir == :y ? - :(view(ψ, i, j+$c_min:j+$c_max, k)) : - :(view(ψ, i, j, k+$c_min:k+$c_max)) - end - end - return :($(stencil_full...),) -end - -using Oceananigans.Fields: Field - # Stencils for left and right biased reconstruction ((ψ̅ᵢ₋ᵣ₊ⱼ for j in 0:k) for r in 0:k) to calculate v̂ᵣ = ∑ⱼ(cᵣⱼψ̅ᵢ₋ᵣ₊ⱼ) # where `k = N - 1`. Coefficients (cᵣⱼ for j in 0:N) for stencil r are given by `coeff_side_p(scheme, Val(r), ...)` for side in (:left, :right), dir in (:x, :y, :z) @@ -389,9 +348,8 @@ for side in (:left, :right), dir in (:x, :y, :z) for buffer in [2, 3, 4, 5, 6] @eval begin - @inline $stencil(i, j, k, scheme::WENO{$buffer}, ψ::Field, args...) = @inbounds $(calc_weno_stencil_variable(buffer, side, dir, true)) - @inline $stencil(i, j, k, scheme::WENO{$buffer}, ψ::OffsetArray, args...) = @inbounds $(calc_weno_stencil_variable(buffer, side, dir, false)) - @inline $stencil(i, j, k, scheme::WENO{$buffer}, ψ::Function, args...) = @inbounds $(calc_weno_stencil_function(buffer, side, dir)) + @inline $stencil(i, j, k, scheme::WENO{$buffer}, ψ, args...) = @inbounds $(calc_weno_stencil(buffer, side, dir, false)) + @inline $stencil(i, j, k, scheme::WENO{$buffer}, ψ::Function, args...) = @inbounds $(calc_weno_stencil(buffer, side, dir, true)) end end end From 9eaa6abf0bbdab0a6c34e43414a644cce8e73b83 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sun, 10 Mar 2024 16:22:20 -0400 Subject: [PATCH 251/567] bugfix --- .../split_explicit_free_surface.jl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index 470ea3a5a8..b17cc04009 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -269,7 +269,6 @@ end struct AdamsBashforth3Scheme end struct ForwardBackwardScheme end - auxiliary_free_surface_field(grid, ::AdamsBashforth3Scheme) = ZFaceField(grid, indices = (:, :, size(grid, 3)+1)) auxiliary_free_surface_field(grid, ::ForwardBackwardScheme) = nothing @@ -406,8 +405,8 @@ Adapt.adapt_structure(to, free_surface::SplitExplicitFreeSurface) = # Adapt Adapt.adapt_structure(to, auxiliary::SplitExplicitAuxiliaryFields) = - SplitExplicitAuxiliaryFields(nothing, - nothing, + SplitExplicitAuxiliaryFields(Adapt.adapt(to, auxiliary.Gᵁ), + Adapt.adapt(to, auxiliary.Gⱽ), Adapt.adapt(to, auxiliary.Hᶠᶜ), Adapt.adapt(to, auxiliary.Hᶜᶠ), Adapt.adapt(to, auxiliary.Hᶜᶜ), From b27f1d772e2d0a01f02dff1f91da69007dd140bf Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sun, 10 Mar 2024 16:44:06 -0400 Subject: [PATCH 252/567] with halo for generalized immersed grid --- src/ImmersedBoundaries/ImmersedBoundaries.jl | 2 +- .../generalized_vertical_spacing.jl | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/ImmersedBoundaries/ImmersedBoundaries.jl b/src/ImmersedBoundaries/ImmersedBoundaries.jl index c488d2de73..74b6666201 100644 --- a/src/ImmersedBoundaries/ImmersedBoundaries.jl +++ b/src/ImmersedBoundaries/ImmersedBoundaries.jl @@ -149,7 +149,7 @@ Adapt.adapt_structure(to, ibg::IBG{FT, TX, TY, TZ}) where {FT, TX, TY, TZ} = function with_halo(halo, ibg::ImmersedBoundaryGrid) new_grid = with_halo(halo, ibg.underlying_grid) - ImmersedBoundaryGrid(new_grid, ibg.immersed_boundary) + return ImmersedBoundaryGrid(new_grid, ibg.immersed_boundary) end # ImmersedBoundaryGrids require an extra halo point to check the "inactivity" of a `Face` node at N + H diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index dac477f211..06b98b95f8 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -99,6 +99,18 @@ function with_halo(new_halo, grid::GeneralizedSpacingGrid) return new_grid end +function with_halo(halo, ibg::GeneralizedSpacingImmersedGrid) + underlying_grid = ibg.underlying_grid + immersed_boundary = ibg.immersed_boundary + old_static_grid = retrieve_static_grid(underlying_grid) + new_static_grid = with_halo(new_halo, underlying_grid) + vertical_coordinate = denomination(underlying_grid) + active_cells_map = isnothing(ibg.interior_active_cells) + new_grid = GeneralizedSpacingGrid(new_static_grid, vertical_coordinate) + new_ibg = ImmersedBoundaryGrid(new_grid, immersed_boundary; active_cells_map) + return new_ibg +end + ##### ##### General implementation ##### From cc8a25a54b0d94d1b517fe7cc0372b546eb3a966 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sun, 10 Mar 2024 16:53:21 -0400 Subject: [PATCH 253/567] small bugfix in with_halo --- .../generalized_vertical_spacing.jl | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index 06b98b95f8..4c0ad6e46d 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -90,7 +90,7 @@ end ##### Some extensions ##### -function with_halo(new_halo, grid::GeneralizedSpacingGrid) +function with_halo(new_halo, grid::GeneralizedSpacingUnderlyingGrid) old_static_grid = retrieve_static_grid(grid) new_static_grid = with_halo(new_halo, old_static_grid) vertical_coordinate = denomination(grid) @@ -99,15 +99,12 @@ function with_halo(new_halo, grid::GeneralizedSpacingGrid) return new_grid end -function with_halo(halo, ibg::GeneralizedSpacingImmersedGrid) +function with_halo(new_halo, ibg::GeneralizedSpacingImmersedGrid) underlying_grid = ibg.underlying_grid immersed_boundary = ibg.immersed_boundary - old_static_grid = retrieve_static_grid(underlying_grid) - new_static_grid = with_halo(new_halo, underlying_grid) - vertical_coordinate = denomination(underlying_grid) + new_underlying_grid = with_halo(new_halo, underlying_grid) active_cells_map = isnothing(ibg.interior_active_cells) - new_grid = GeneralizedSpacingGrid(new_static_grid, vertical_coordinate) - new_ibg = ImmersedBoundaryGrid(new_grid, immersed_boundary; active_cells_map) + new_ibg = ImmersedBoundaryGrid(new_underlying_grid, immersed_boundary; active_cells_map) return new_ibg end From bc7d1e7f2be739713f3db2705e34bdda493cb0f3 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sun, 10 Mar 2024 17:51:19 -0400 Subject: [PATCH 254/567] add scalings to auxiliary fields --- .../generalized_vertical_spacing.jl | 4 ++-- .../hydrostatic_free_surface_model.jl | 4 ++++ .../HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl | 6 ------ 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index 4c0ad6e46d..9ba69650d9 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -156,8 +156,8 @@ update_vertical_spacing!(model, grid, Δt; kwargs...) = nothing ##### ##### Vertical velocity of the Δ-surfaces to be included in the continuity equation ##### - -∂t_∂s_grid(i, j, k, grid::GeneralizedSpacingGrid) = grid.Δzᵃᵃᶜ.∂t_∂s[i, j, k] +@inline ∂t_∂s_grid(i, j, k, grid) = zero(grid) +@inline ∂t_∂s_grid(i, j, k, grid::GeneralizedSpacingGrid) = grid.Δzᵃᵃᶜ.∂t_∂s[i, j, k] ##### ##### Additional terms to be included in the momentum equations (fallbacks) diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl index e6f7639dbf..b49b70107e 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl @@ -124,6 +124,10 @@ function HydrostaticFreeSurfaceModel(; grid, # end grid = !isnothing(free_surface) ? GeneralizedSpacingGrid(grid, generalized_vertical_coordinate) : grid + + if grid isa GeneralizedSpacingGrid + auxiliary_fields = merge(auxiliary_fields, (; s⁻ = grid.Δzᵃᵃᶠ.s⁻, sⁿ = grid.Δzᵃᵃᶠ.sⁿ, ∂t_∂s = grid.Δzᵃᵃᶠ.∂t_∂s)) + end arch = architecture(grid) @apply_regionally momentum_advection = validate_momentum_advection(momentum_advection, grid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 5ddfe42150..a5b1550c27 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -52,12 +52,6 @@ function GeneralizedSpacingGrid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, :: return GridType{TX, TY, TZ}(args...) end -@kernel function _initialize_zstar!(ΔzF, ΔzC, grid) - i, j, k = @index(Global, NTuple) - @inbounds ΔzF[i, j, k] = Δzᶜᶜᶠ(i, j, k, grid) - @inbounds ΔzC[i, j, k] = Δzᶜᶜᶜ(i, j, k, grid) -end - ##### ##### ZStar-specific vertical spacing functions ##### From 1f33dac5d8d6e39f83eda21a0e90bf927d8af19e Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sun, 10 Mar 2024 17:53:55 -0400 Subject: [PATCH 255/567] another fix --- src/ImmersedBoundaries/ImmersedBoundaries.jl | 3 ++- src/ImmersedBoundaries/active_cells_map.jl | 3 --- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/ImmersedBoundaries/ImmersedBoundaries.jl b/src/ImmersedBoundaries/ImmersedBoundaries.jl index 74b6666201..1b44bd6143 100644 --- a/src/ImmersedBoundaries/ImmersedBoundaries.jl +++ b/src/ImmersedBoundaries/ImmersedBoundaries.jl @@ -149,7 +149,8 @@ Adapt.adapt_structure(to, ibg::IBG{FT, TX, TY, TZ}) where {FT, TX, TY, TZ} = function with_halo(halo, ibg::ImmersedBoundaryGrid) new_grid = with_halo(halo, ibg.underlying_grid) - return ImmersedBoundaryGrid(new_grid, ibg.immersed_boundary) + active_cells_map = !isnothing(ibg.interior_active_cells) + return ImmersedBoundaryGrid(new_grid, ibg.immersed_boundary; active_cells_map) end # ImmersedBoundaryGrids require an extra halo point to check the "inactivity" of a `Face` node at N + H diff --git a/src/ImmersedBoundaries/active_cells_map.jl b/src/ImmersedBoundaries/active_cells_map.jl index 5b4b516dde..4dee9b92bf 100644 --- a/src/ImmersedBoundaries/active_cells_map.jl +++ b/src/ImmersedBoundaries/active_cells_map.jl @@ -108,9 +108,6 @@ function ImmersedBoundaryGrid(grid, ib; active_cells_map::Bool = true) column_map) end -with_halo(halo, ibg::ActiveCellsIBG) = - ImmersedBoundaryGrid(with_halo(halo, ibg.underlying_grid), ibg.immersed_boundary; active_cells_map = true) - @inline active_cell(i, j, k, ibg) = !immersed_cell(i, j, k, ibg) @inline active_column(i, j, k, grid, column) = column[i, j, k] != 0 From 824b7db5ee0098164ee3aa9aec5c0b32c893677f Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Sun, 10 Mar 2024 18:11:11 -0400 Subject: [PATCH 256/567] other bugfix --- .../generalized_vertical_spacing.jl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index 9ba69650d9..d373cb0079 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -63,7 +63,13 @@ denomination(grid::GeneralizedSpacingGrid) = grid.Δzᵃᵃᶠ.denomination retrieve_static_grid(grid) = grid -function retrieve_static_grid(grid::GeneralizedSpacingGrid) +function retrieve_static_grid(grid::GeneralizedSpacingImmersedGrid) + underlying_grid = retrieve_static_grid(grid.underlying_grid) + active_cells_map = !isnothing(grid.interior_active_cells) + return ImmersedBoundaryGrid(underlying_grid, grid.immersed_boundary; active_cells_map) +end + +function retrieve_static_grid(grid::GeneralizedSpacingUnderlyingGrid) Δzᵃᵃᶠ = grid.Δzᵃᵃᶠ.Δr Δzᵃᵃᶜ = grid.Δzᵃᵃᶜ.Δr From d92c2ba9c660ec38d7e1ebecb566760af90eedd9 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 26 Mar 2024 13:35:07 -0400 Subject: [PATCH 257/567] will this allow enough parameter space on GPUs? --- .../generalized_vertical_spacing.jl | 55 +++++-------------- .../z_star_vertical_spacing.jl | 34 ++++++++++-- 2 files changed, 45 insertions(+), 44 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index d373cb0079..51f60d3f28 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -14,7 +14,7 @@ import Oceananigans.Advection: ∂t_∂s_grid import Oceananigans.Architectures: arch_array """ - GeneralizedVerticalSpacing{D, R, S, Z} + AbstractVerticalSpacing{R} spacings for a generalized vertical coordinate system. The reference (non-moving) spacings are stored in `Δr`. `Δ` contains the spacings associated with the moving coordinate system. @@ -22,40 +22,15 @@ spacings for a generalized vertical coordinate system. The reference (non-moving at timestep `n-1` and `n` and it's time derivative. `denomination` contains the "type" of generalized vertical coordinate (the only one implemented is `ZStar`) """ -struct GeneralizedVerticalSpacing{D, R, Z, S, SN, ST} - denomination :: D # The type of generalized coordinate - Δr :: R # Reference _non moving_ vertical coordinate (one-dimensional) - Δ :: Z # moving vertical coordinate (three-dimensional) - s⁻ :: S # scaling term = ∂Δ/∂Δr at the start of the time step - sⁿ :: SN # scaling term = ∂Δ/∂Δr at the end of the time step - ∂t_∂s :: ST # Time derivative of the vertical coordinate scaling -end - -Adapt.adapt_structure(to, coord::GeneralizedVerticalSpacing) = - GeneralizedVerticalSpacing(coord.denomination, - Adapt.adapt(to, coord.Δr), - Adapt.adapt(to, coord.Δ), - Adapt.adapt(to, coord.s⁻), - Adapt.adapt(to, coord.sⁿ), - Adapt.adapt(to, coord.∂t_∂s)) - -on_architecture(arch, coord::GeneralizedVerticalSpacing) = - GeneralizedVerticalSpacing(on_architecture(arch, coord.denomination), - on_architecture(arch, coord.Δr), - on_architecture(arch, coord.Δ), - on_architecture(arch, coord.s⁻), - on_architecture(arch, coord.sⁿ), - on_architecture(arch, coord.∂t_∂s)) - -const GeneralizedSpacingRG{D} = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:GeneralizedVerticalSpacing{D}} where D -const GeneralizedSpacingLLG{D} = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:GeneralizedVerticalSpacing{D}} where D +abstract type AbstractVerticalSpacing{R} end -const GeneralizedSpacingUnderlyingGrid{D} = Union{GeneralizedSpacingRG{D}, GeneralizedSpacingLLG{D}} where D -const GeneralizedSpacingImmersedGrid{D} = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:GeneralizedSpacingUnderlyingGrid{D}} where D +const GeneralizedSpacingRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractVerticalSpacing} +const GeneralizedSpacingLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractVerticalSpacing} -const GeneralizedSpacingGrid{D} = Union{GeneralizedSpacingUnderlyingGrid{D}, GeneralizedSpacingImmersedGrid{D}} where D +const GeneralizedSpacingUnderlyingGrid = Union{GeneralizedSpacingRG, GeneralizedSpacingLLG} +const GeneralizedSpacingImmersedGrid = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:GeneralizedSpacingUnderlyingGrid} -denomination(grid::GeneralizedSpacingGrid) = grid.Δzᵃᵃᶠ.denomination +const GeneralizedSpacingGrid = Union{GeneralizedSpacingUnderlyingGrid, GeneralizedSpacingImmersedGrid} ##### ##### Original grid @@ -69,10 +44,13 @@ function retrieve_static_grid(grid::GeneralizedSpacingImmersedGrid) return ImmersedBoundaryGrid(underlying_grid, grid.immersed_boundary; active_cells_map) end +reference_Δzᵃᵃᶠ(grid) = grid.Δzᵃᵃᶠ +reference_Δzᵃᵃᶜ(grid) = grid.Δzᵃᵃᶜ + function retrieve_static_grid(grid::GeneralizedSpacingUnderlyingGrid) - Δzᵃᵃᶠ = grid.Δzᵃᵃᶠ.Δr - Δzᵃᵃᶜ = grid.Δzᵃᵃᶜ.Δr + Δzᵃᵃᶠ = reference_Δzᵃᵃᶠ(grid) + Δzᵃᵃᶜ = reference_Δzᵃᵃᶜ(grid) TX, TY, TZ = topology(grid) @@ -129,9 +107,6 @@ update_vertical_spacing!(model, grid, Δt; kwargs...) = nothing # TODO: make z-direction local in memory by not using Fields # TODO: make it work with partial cells -@inline Δzᶜᶜᶠ(i, j, k, grid::GeneralizedSpacingGrid) = @inbounds grid.Δzᵃᵃᶠ.Δ[i, j, k] -@inline Δzᶜᶜᶜ(i, j, k, grid::GeneralizedSpacingGrid) = @inbounds grid.Δzᵃᵃᶜ.Δ[i, j, k] - @inline Δzᶜᶠᶠ(i, j, k, grid::GeneralizedSpacingGrid) = ℑyᵃᶠᵃ(i, j, k, grid, Δzᶜᶜᶠ) @inline Δzᶜᶠᶜ(i, j, k, grid::GeneralizedSpacingGrid) = ℑyᵃᶠᵃ(i, j, k, grid, Δzᶜᶜᶜ) @@ -144,8 +119,8 @@ update_vertical_spacing!(model, grid, Δt; kwargs...) = nothing @inline Δz_reference(i, j, k, Δz::Number) = Δz @inline Δz_reference(i, j, k, Δz::AbstractVector) = Δz[k] -@inline Δz_reference(i, j, k, Δz::GeneralizedVerticalSpacing{<:Any, <:Number}) = Δz.Δr -@inline Δz_reference(i, j, k, Δz::GeneralizedVerticalSpacing) = Δz.Δr[k] +@inline Δz_reference(i, j, k, Δz::AbstractVerticalSpacing{<:Number}) = Δz.Δr +@inline Δz_reference(i, j, k, Δz::AbstractVerticalSpacing) = Δz.Δr[k] @inline Δzᶜᶜᶠ_reference(i, j, k, grid) = Δz_reference(i, j, k, grid.Δzᵃᵃᶠ) @inline Δzᶜᶜᶜ_reference(i, j, k, grid) = Δz_reference(i, j, k, grid.Δzᵃᵃᶜ) @@ -162,8 +137,8 @@ update_vertical_spacing!(model, grid, Δt; kwargs...) = nothing ##### ##### Vertical velocity of the Δ-surfaces to be included in the continuity equation ##### + @inline ∂t_∂s_grid(i, j, k, grid) = zero(grid) -@inline ∂t_∂s_grid(i, j, k, grid::GeneralizedSpacingGrid) = grid.Δzᵃᵃᶜ.∂t_∂s[i, j, k] ##### ##### Additional terms to be included in the momentum equations (fallbacks) diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index a5b1550c27..0b7f7b6710 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -6,12 +6,36 @@ """ free-surface following vertical coordinate """ struct ZStar end -const ZStarSpacing = GeneralizedVerticalSpacing{<:ZStar} +struct ZStarSpacing{R, S} <: AbstractVerticalSpacing + Δr :: R + sⁿ :: S + s⁻ :: S + ∂t_∂s :: S +end + +Adapt.adapt_structure(to, coord::ZStarSpacing) = + ZStarSpacing(Adapt.adapt(to, coord.Δr), + Adapt.adapt(to, coord.s⁻), + Adapt.adapt(to, coord.sⁿ), + Adapt.adapt(to, coord.∂t_∂s)) + +on_architecture(arch, coord::ZStarSpacing) = + ZStarSpacing(on_architecture(arch, coord.Δr), + on_architecture(arch, coord.s⁻), + on_architecture(arch, coord.sⁿ), + on_architecture(arch, coord.∂t_∂s)) Grids.coordinate_summary(Δ::ZStarSpacing, name) = @sprintf("Free-surface following with Δ%s=%s", name, prettysummary(Δ.Δr)) -const ZStarSpacingGrid = GeneralizedSpacingGrid{<:ZStar} + +const ZStarSpacingRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:ZStarSpacing} +const ZStarSpacingLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:ZStarSpacing} + +const ZStarSpacingUnderlyingGrid = Union{GeneralizedSpacingRG, GeneralizedSpacingLLG} +const ZStarSpacingImmersedGrid = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:GeneralizedSpacingUnderlyingGrid} + +const ZStarSpacingGrid = Union{ZStarUnderlyingGrid, GeneralizedSpacingImmersedGrid} function GeneralizedSpacingGrid(grid::ImmersedBoundaryGrid, ::ZStar) underlying_grid = GeneralizedSpacingGrid(grid.underlying_grid, ZStar()) @@ -33,8 +57,8 @@ function GeneralizedSpacingGrid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, :: fill!(s⁻, 1) fill!(sⁿ, 1) - Δzᵃᵃᶠ = GeneralizedVerticalSpacing(ZStar(), grid.Δzᵃᵃᶠ, nothing, s⁻, sⁿ, ∂t_∂s) - Δzᵃᵃᶜ = GeneralizedVerticalSpacing(ZStar(), grid.Δzᵃᵃᶜ, nothing, s⁻, sⁿ, ∂t_∂s) + Δzᵃᵃᶠ = ZStarSpacing(grid.Δzᵃᵃᶠ, s⁻, sⁿ, ∂t_∂s) + Δzᵃᵃᶜ = ZStarSpacing(grid.Δzᵃᵃᶜ, s⁻, sⁿ, ∂t_∂s) args = [] for prop in propertynames(grid) @@ -59,6 +83,8 @@ end @inline Δzᶜᶜᶠ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δzᶜᶜᶠ_reference(i, j, k, grid) * grid.Δzᵃᵃᶠ.sⁿ[i, j, 1] @inline Δzᶜᶜᶜ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δzᶜᶜᶜ_reference(i, j, k, grid) * grid.Δzᵃᵃᶜ.sⁿ[i, j, 1] +@inline ∂t_∂s_grid(i, j, k, grid::ZStarSpacingGrid) = grid.Δzᵃᵃᶜ.∂t_∂s[i, j, k] + ##### ##### ZStar-specific vertical spacings update ##### From 2c511ab734e4c6790105a5c780089b24dad8820f Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 26 Mar 2024 13:38:16 -0400 Subject: [PATCH 258/567] specific z-star stuff --- .../generalized_vertical_spacing.jl | 36 +++++++++---------- .../z_star_vertical_spacing.jl | 3 ++ 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index 51f60d3f28..266bb9cf51 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -24,13 +24,13 @@ at timestep `n-1` and `n` and it's time derivative. """ abstract type AbstractVerticalSpacing{R} end -const GeneralizedSpacingRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractVerticalSpacing} -const GeneralizedSpacingLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractVerticalSpacing} +const AbstractVerticalSpacingRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractVerticalSpacing} +const AbstractVerticalSpacingLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractVerticalSpacing} -const GeneralizedSpacingUnderlyingGrid = Union{GeneralizedSpacingRG, GeneralizedSpacingLLG} -const GeneralizedSpacingImmersedGrid = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:GeneralizedSpacingUnderlyingGrid} +const AbstractVerticalSpacingUnderlyingGrid = Union{AbstractVerticalSpacingRG, AbstractVerticalSpacingLLG} +const AbstractVerticalSpacingImmersedGrid = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:AbstractVerticalSpacingUnderlyingGrid} -const GeneralizedSpacingGrid = Union{GeneralizedSpacingUnderlyingGrid, GeneralizedSpacingImmersedGrid} +const AbstractVerticalSpacingGrid = Union{AbstractVerticalSpacingUnderlyingGrid, AbstractVerticalSpacingImmersedGrid} ##### ##### Original grid @@ -38,7 +38,7 @@ const GeneralizedSpacingGrid = Union{GeneralizedSpacingUnderlyingGrid, Generaliz retrieve_static_grid(grid) = grid -function retrieve_static_grid(grid::GeneralizedSpacingImmersedGrid) +function retrieve_static_grid(grid::AbstractVerticalSpacingImmersedGrid) underlying_grid = retrieve_static_grid(grid.underlying_grid) active_cells_map = !isnothing(grid.interior_active_cells) return ImmersedBoundaryGrid(underlying_grid, grid.immersed_boundary; active_cells_map) @@ -47,7 +47,7 @@ end reference_Δzᵃᵃᶠ(grid) = grid.Δzᵃᵃᶠ reference_Δzᵃᵃᶜ(grid) = grid.Δzᵃᵃᶜ -function retrieve_static_grid(grid::GeneralizedSpacingUnderlyingGrid) +function retrieve_static_grid(grid::AbstractVerticalSpacingUnderlyingGrid) Δzᵃᵃᶠ = reference_Δzᵃᵃᶠ(grid) Δzᵃᵃᶜ = reference_Δzᵃᵃᶜ(grid) @@ -74,7 +74,7 @@ end ##### Some extensions ##### -function with_halo(new_halo, grid::GeneralizedSpacingUnderlyingGrid) +function with_halo(new_halo, grid::AbstractVerticalSpacingUnderlyingGrid) old_static_grid = retrieve_static_grid(grid) new_static_grid = with_halo(new_halo, old_static_grid) vertical_coordinate = denomination(grid) @@ -83,7 +83,7 @@ function with_halo(new_halo, grid::GeneralizedSpacingUnderlyingGrid) return new_grid end -function with_halo(new_halo, ibg::GeneralizedSpacingImmersedGrid) +function with_halo(new_halo, ibg::AbstractVerticalSpacingImmersedGrid) underlying_grid = ibg.underlying_grid immersed_boundary = ibg.immersed_boundary new_underlying_grid = with_halo(new_halo, underlying_grid) @@ -107,14 +107,14 @@ update_vertical_spacing!(model, grid, Δt; kwargs...) = nothing # TODO: make z-direction local in memory by not using Fields # TODO: make it work with partial cells -@inline Δzᶜᶠᶠ(i, j, k, grid::GeneralizedSpacingGrid) = ℑyᵃᶠᵃ(i, j, k, grid, Δzᶜᶜᶠ) -@inline Δzᶜᶠᶜ(i, j, k, grid::GeneralizedSpacingGrid) = ℑyᵃᶠᵃ(i, j, k, grid, Δzᶜᶜᶜ) +@inline Δzᶜᶠᶠ(i, j, k, grid::AbstractVerticalSpacingGrid) = ℑyᵃᶠᵃ(i, j, k, grid, Δzᶜᶜᶠ) +@inline Δzᶜᶠᶜ(i, j, k, grid::AbstractVerticalSpacingGrid) = ℑyᵃᶠᵃ(i, j, k, grid, Δzᶜᶜᶜ) -@inline Δzᶠᶜᶠ(i, j, k, grid::GeneralizedSpacingGrid) = ℑxᶠᵃᵃ(i, j, k, grid, Δzᶜᶜᶠ) -@inline Δzᶠᶜᶜ(i, j, k, grid::GeneralizedSpacingGrid) = ℑxᶠᵃᵃ(i, j, k, grid, Δzᶜᶜᶜ) +@inline Δzᶠᶜᶠ(i, j, k, grid::AbstractVerticalSpacingGrid) = ℑxᶠᵃᵃ(i, j, k, grid, Δzᶜᶜᶠ) +@inline Δzᶠᶜᶜ(i, j, k, grid::AbstractVerticalSpacingGrid) = ℑxᶠᵃᵃ(i, j, k, grid, Δzᶜᶜᶜ) -@inline Δzᶠᶠᶠ(i, j, k, grid::GeneralizedSpacingGrid) = ℑxyᶠᶠᵃ(i, j, k, grid, Δzᶜᶜᶠ) -@inline Δzᶠᶠᶜ(i, j, k, grid::GeneralizedSpacingGrid) = ℑxyᶠᶠᵃ(i, j, k, grid, Δzᶜᶜᶜ) +@inline Δzᶠᶠᶠ(i, j, k, grid::AbstractVerticalSpacingGrid) = ℑxyᶠᶠᵃ(i, j, k, grid, Δzᶜᶜᶠ) +@inline Δzᶠᶠᶜ(i, j, k, grid::AbstractVerticalSpacingGrid) = ℑxyᶠᶠᵃ(i, j, k, grid, Δzᶜᶜᶜ) @inline Δz_reference(i, j, k, Δz::Number) = Δz @inline Δz_reference(i, j, k, Δz::AbstractVector) = Δz[k] @@ -167,7 +167,7 @@ update_vertical_spacing!(model, grid, Δt; kwargs...) = nothing end end -ab2_step_tracer_field!(tracer_field, grid::GeneralizedSpacingGrid, Δt, χ, Gⁿ, G⁻) = +ab2_step_tracer_field!(tracer_field, grid::AbstractVerticalSpacingGrid, Δt, χ, Gⁿ, G⁻) = launch!(architecture(grid), grid, :xyz, _ab2_step_tracer_generalized_spacing!, tracer_field, grid.Δzᵃᵃᶠ.sⁿ, @@ -176,12 +176,12 @@ ab2_step_tracer_field!(tracer_field, grid::GeneralizedSpacingGrid, Δt, χ, Gⁿ const EmptyTuples = Union{NamedTuple{(),Tuple{}}, Tuple{}} -scale_tracers!(::EmptyTuples, ::GeneralizedSpacingGrid; kwargs...) = nothing +scale_tracers!(::EmptyTuples, ::AbstractVerticalSpacingGrid; kwargs...) = nothing tracer_scaling_parameters(param::Symbol, tracers, grid) = KernelParameters((size(grid, 1), size(grid, 2), length(tracers)), (0, 0, 0)) tracer_scaling_parameters(param::KernelParameters{S, O}, tracers, grid) where {S, O} = KernelParameters((S..., length(tracers)), (O..., 0)) -function scale_tracers!(tracers, grid::GeneralizedSpacingGrid; parameters = :xy) +function scale_tracers!(tracers, grid::AbstractVerticalSpacingGrid; parameters = :xy) parameters = tracer_scaling_parameters(parameters, tracers, grid) launch!(architecture(grid), grid, parameters, _scale_tracers, tracers, grid.Δzᵃᵃᶠ.sⁿ, Val(grid.Hz), Val(grid.Nz)) diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 0b7f7b6710..65a79b6521 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -80,6 +80,9 @@ end ##### ZStar-specific vertical spacing functions ##### +reference_Δzᵃᵃᶠ(grid::ZStarSpacingGrid) = grid.Δzᵃᵃᶠ.Δr +reference_Δzᵃᵃᶜ(grid::ZStarSpacingGrid) = grid.Δzᵃᵃᶜ.Δr + @inline Δzᶜᶜᶠ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δzᶜᶜᶠ_reference(i, j, k, grid) * grid.Δzᵃᵃᶠ.sⁿ[i, j, 1] @inline Δzᶜᶜᶜ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δzᶜᶜᶜ_reference(i, j, k, grid) * grid.Δzᵃᵃᶜ.sⁿ[i, j, 1] From 3bf4fe548a65664f2cc8cf57d90f872516b737c2 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 26 Mar 2024 13:46:00 -0400 Subject: [PATCH 259/567] bugfix --- .../HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 65a79b6521..93bfa8d0fb 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -6,7 +6,7 @@ """ free-surface following vertical coordinate """ struct ZStar end -struct ZStarSpacing{R, S} <: AbstractVerticalSpacing +struct ZStarSpacing{R, S} <: AbstractVerticalSpacing{R} Δr :: R sⁿ :: S s⁻ :: S From d5d08ed2d54338de33210057f7d6ddba001384bb Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 26 Mar 2024 13:48:02 -0400 Subject: [PATCH 260/567] another bugfix --- .../HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 93bfa8d0fb..05c94d3c8f 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -32,10 +32,10 @@ Grids.coordinate_summary(Δ::ZStarSpacing, name) = const ZStarSpacingRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:ZStarSpacing} const ZStarSpacingLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:ZStarSpacing} -const ZStarSpacingUnderlyingGrid = Union{GeneralizedSpacingRG, GeneralizedSpacingLLG} -const ZStarSpacingImmersedGrid = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:GeneralizedSpacingUnderlyingGrid} +const ZStarSpacingUnderlyingGrid = Union{ZStarSpacingRG, ZStarSpacingLLG} +const ZStarSpacingImmersedGrid = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:ZStarSpacingUnderlyingGrid} -const ZStarSpacingGrid = Union{ZStarUnderlyingGrid, GeneralizedSpacingImmersedGrid} +const ZStarSpacingGrid = Union{ZStarSpacingUnderlyingGrid, ZStarSpacingImmersedGrid} function GeneralizedSpacingGrid(grid::ImmersedBoundaryGrid, ::ZStar) underlying_grid = GeneralizedSpacingGrid(grid.underlying_grid, ZStar()) From 1def06d6dc4cfbf1e2ce0f58ccff0ce2aff20a46 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 26 Mar 2024 13:52:50 -0400 Subject: [PATCH 261/567] fixing output writers --- src/OutputWriters/output_writer_utils.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/OutputWriters/output_writer_utils.jl b/src/OutputWriters/output_writer_utils.jl index 5c183ff5a5..af5ebda5aa 100644 --- a/src/OutputWriters/output_writer_utils.jl +++ b/src/OutputWriters/output_writer_utils.jl @@ -6,7 +6,7 @@ using Oceananigans.Fields: AbstractField, indices, boundary_conditions, instanti using Oceananigans.BoundaryConditions: bc_str, FieldBoundaryConditions, ContinuousBoundaryFunction, DiscreteBoundaryFunction using Oceananigans.TimeSteppers: QuasiAdamsBashforth2TimeStepper, RungeKutta3TimeStepper using Oceananigans.Models.LagrangianParticleTracking: LagrangianParticles -using Oceananigans.Models.HydrostaticFreeSurfaceModels: GeneralizedSpacingGrid, retrieve_static_grid +using Oceananigans.Models.HydrostaticFreeSurfaceModels: AbstractVerticalSpacingGrid, retrieve_static_grid ##### ##### Output writer utilities @@ -49,7 +49,7 @@ function saveproperty!(file, address, grid::DistributedGrid) _saveproperty!(file, address, on_architecture(cpu_arch, grid)) end -function saveproperty!(file, address, grid::GeneralizedSpacingGrid) +function saveproperty!(file, address, grid::AbstractVerticalSpacingGrid) static_grid = retrieve_static_grid(grid) saveproperty!(file, address, static_grid) end @@ -95,7 +95,7 @@ function serializeproperty!(file, address, grid::DistributedGrid) file[address] = on_architecture(cpu_arch, grid) end -function serializeproperty!(file, address, grid::GeneralizedSpacingGrid) +function serializeproperty!(file, address, grid::AbstractVerticalSpacingGrid) static_grid = retrieve_static_grid(grid) serializeproperty!(file, address, static_grid) end From cfff03083ea43503abafc6692167c64fa6123ff7 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Tue, 26 Mar 2024 13:59:33 -0400 Subject: [PATCH 262/567] generalized_spacing_grid --- .../generalized_vertical_spacing.jl | 4 ++-- .../hydrostatic_free_surface_model.jl | 5 +---- .../HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl | 6 +++--- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index 266bb9cf51..de71db2fdf 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -78,7 +78,7 @@ function with_halo(new_halo, grid::AbstractVerticalSpacingUnderlyingGrid) old_static_grid = retrieve_static_grid(grid) new_static_grid = with_halo(new_halo, old_static_grid) vertical_coordinate = denomination(grid) - new_grid = GeneralizedSpacingGrid(new_static_grid, vertical_coordinate) + new_grid = generalized_spacing_grid(new_static_grid, vertical_coordinate) return new_grid end @@ -96,7 +96,7 @@ end ##### General implementation ##### -GeneralizedSpacingGrid(grid, coord) = grid +generalized_spacing_grid(grid, coord) = grid update_vertical_spacing!(model, grid, Δt; kwargs...) = nothing ##### diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl index b49b70107e..c60ac32e13 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl @@ -123,11 +123,8 @@ function HydrostaticFreeSurfaceModel(; grid, # throw(ArgumentError("Generalized vertical coordinates are supported only for the vector-invariant form of the momentum equations")) # end - grid = !isnothing(free_surface) ? GeneralizedSpacingGrid(grid, generalized_vertical_coordinate) : grid + grid = !isnothing(free_surface) ? generalized_spacing_grid(grid, generalized_vertical_coordinate) : grid - if grid isa GeneralizedSpacingGrid - auxiliary_fields = merge(auxiliary_fields, (; s⁻ = grid.Δzᵃᵃᶠ.s⁻, sⁿ = grid.Δzᵃᵃᶠ.sⁿ, ∂t_∂s = grid.Δzᵃᵃᶠ.∂t_∂s)) - end arch = architecture(grid) @apply_regionally momentum_advection = validate_momentum_advection(momentum_advection, grid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 05c94d3c8f..86cf05e486 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -37,8 +37,8 @@ const ZStarSpacingImmersedGrid = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:A const ZStarSpacingGrid = Union{ZStarSpacingUnderlyingGrid, ZStarSpacingImmersedGrid} -function GeneralizedSpacingGrid(grid::ImmersedBoundaryGrid, ::ZStar) - underlying_grid = GeneralizedSpacingGrid(grid.underlying_grid, ZStar()) +function generalized_spacing_grid(grid::ImmersedBoundaryGrid, ::ZStar) + underlying_grid = generalized_spacing_grid(grid.underlying_grid, ZStar()) active_cells_map = !isnothing(grid.interior_active_cells) return ImmersedBoundaryGrid(underlying_grid, grid.immersed_boundary; active_cells_map) @@ -46,7 +46,7 @@ end # Replacing the z-coordinate with a moving vertical coordinate, defined by its reference spacing, # the actual vertical spacing and a scaling -function GeneralizedSpacingGrid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, ::ZStar) where {FT, TX, TY, TZ} +function generalized_spacing_grid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, ::ZStar) where {FT, TX, TY, TZ} # Memory layout for Δz spacings should be local in z instead of x s⁻ = Field{Center, Center, Nothing}(grid) From a2102e10f3ca4d4ace8bc2679ff05922fcf5aaa2 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Wed, 27 Mar 2024 17:05:49 -0400 Subject: [PATCH 263/567] adding mass flux weights --- .../split_explicit_free_surface.jl | 20 +++++++ .../split_explicit_free_surface_kernels.jl | 55 +++++++++---------- 2 files changed, 47 insertions(+), 28 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index b17cc04009..16865a2849 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -367,6 +367,26 @@ function SplitExplicitSettings(FT::DataType=Float64; return SplitExplicitSettings(substepping, timestepper) end +#= +# mass_flux_weights = similar(averaging_weights) +# +# M = searchsortedfirst(τᶠ, 1) - 1 +# +# averaging_weights ./= sum(averaging_weights) +# +# for i in substeps:-1:1 +# mass_flux_weights[i] = 1 / M * sum(averaging_weights[i:substeps]) +# end +# +# mass_flux_weights ./= sum(mass_flux_weights) +# +# return SplitExplicitSettings(substeps, +# averaging_weights, +# mass_flux_weights, +# Δτ, +# timestepper) +=# + # Convenience Functions for grabbing free surface free_surface(free_surface::SplitExplicitFreeSurface) = free_surface.η diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 6e187d60b6..07d174f057 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -241,6 +241,33 @@ compute_barotropic_mode!(U, V, grid, u, v, Hᶠᶜ, Hᶜᶠ, η̅) = launch!(architecture(grid), grid, :xy, _barotropic_mode_kernel!, U, V, grid, u, v, Hᶠᶜ, Hᶜᶠ, η̅; active_cells_map = active_surface_map(grid)) +@kernel function _barotropic_split_explicit_corrector!(u, v, grid, U̅, V̅, U, V, Hᶠᶜ, Hᶜᶠ, η̅) + i, j, k = @index(Global, NTuple) + k_top = grid.Nz + 1 + + @inbounds begin + u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / Hᶠᶜ[i, j, 1] + v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / Hᶜᶠ[i, j, 1] + end +end + +function barotropic_split_explicit_corrector!(u, v, free_surface, grid) + sefs = free_surface.state + U, V, U̅, V̅ = sefs.U, sefs.V, sefs.U̅, sefs.V̅ + Hᶠᶜ, Hᶜᶠ = free_surface.auxiliary.Hᶠᶜ, free_surface.auxiliary.Hᶜᶠ + arch = architecture(grid) + + # take out "bad" barotropic mode, + # !!!! reusing U and V for this storage since last timestep doesn't matter + compute_barotropic_mode!(U, V, grid, u, v, Hᶠᶜ, Hᶜᶠ, sefs.η̅) + # add in "good" barotropic mode + + launch!(arch, grid, :xyz, _barotropic_split_explicit_corrector!, + u, v, grid, U̅, V̅, U, V, Hᶠᶜ, Hᶜᶠ, sefs.η̅) + + return nothing +end + function initialize_free_surface_state!(state, η, timestepper) parent(state.U) .= parent(state.U̅) @@ -271,34 +298,6 @@ function initialize_auxiliary_state!(state, η, timestepper) return nothing end -@kernel function _barotropic_split_explicit_corrector!(u, v, grid, U̅, V̅, U, V, Hᶠᶜ, Hᶜᶠ, η̅) - i, j, k = @index(Global, NTuple) - k_top = grid.Nz + 1 - - @inbounds begin - u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / Hᶠᶜ[i, j, 1] - v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / Hᶜᶠ[i, j, 1] - end -end - -function barotropic_split_explicit_corrector!(u, v, free_surface, grid) - sefs = free_surface.state - U, V, U̅, V̅ = sefs.U, sefs.V, sefs.U̅, sefs.V̅ - Hᶠᶜ, Hᶜᶠ = free_surface.auxiliary.Hᶠᶜ, free_surface.auxiliary.Hᶜᶠ - arch = architecture(grid) - - - # take out "bad" barotropic mode, - # !!!! reusing U and V for this storage since last timestep doesn't matter - compute_barotropic_mode!(U, V, grid, u, v, Hᶠᶜ, Hᶜᶠ, sefs.η̅) - # add in "good" barotropic mode - - launch!(arch, grid, :xyz, _barotropic_split_explicit_corrector!, - u, v, grid, U̅, V̅, U, V, Hᶠᶜ, Hᶜᶠ, sefs.η̅) - - return nothing -end - """ Explicitly step forward η in substeps. """ From 3efff4e783a0c6e8bc2f7b71f32c926a7ac3690e Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Fri, 12 Apr 2024 10:29:43 -0400 Subject: [PATCH 264/567] try a different upwinding --- src/Advection/vector_invariant_self_upwinding.jl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Advection/vector_invariant_self_upwinding.jl b/src/Advection/vector_invariant_self_upwinding.jl index 41a861816e..b9d33054e1 100644 --- a/src/Advection/vector_invariant_self_upwinding.jl +++ b/src/Advection/vector_invariant_self_upwinding.jl @@ -3,7 +3,7 @@ ##### # Metric term for moving grids -∂t_∂s_grid(i, j, k, grid) = zero(grid) +@inline ∂t_∂s_grid(i, j, k, grid) = zero(grid) @inline V_times_∂t_∂s_grid(i, j, k, grid) = ∂t_∂s_grid(i, j, k, grid) * Vᶜᶜᶜ(i, j, k, grid) @inline δx_U(i, j, k, grid, u, v) = δxᶜᶜᶜ(i, j, k, grid, Ax_qᶠᶜᶜ, u) @@ -25,9 +25,9 @@ cross_scheme = scheme.upwinding.cross_scheme @inbounds û = u[i, j, k] - δvˢ = _symmetric_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, cross_scheme, δy_V, u, v) - δuᴸ = _left_biased_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, δx_U_plus_metric, δU_stencil, u, v) - δuᴿ = _right_biased_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, δx_U_plus_metric, δU_stencil, u, v) + δvˢ = _symmetric_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, cross_scheme, δy_V_plus_metric, u, v) + δuᴸ = _left_biased_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, δx_U, δU_stencil, u, v) + δuᴿ = _right_biased_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, δx_U, δU_stencil, u, v) return upwind_biased_product(û, δuᴸ, δuᴿ) + û * δvˢ end @@ -38,9 +38,9 @@ end cross_scheme = scheme.upwinding.cross_scheme @inbounds v̂ = v[i, j, k] - δuˢ = _symmetric_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, cross_scheme, δx_U, u, v) - δvᴸ = _left_biased_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, δy_V_plus_metric, δV_stencil, u, v) - δvᴿ = _right_biased_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, δy_V_plus_metric, δV_stencil, u, v) + δuˢ = _symmetric_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, cross_scheme, δx_U_plus_metric, u, v) + δvᴸ = _left_biased_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, δy_V, δV_stencil, u, v) + δvᴿ = _right_biased_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, δy_V, δV_stencil, u, v) return upwind_biased_product(v̂, δvᴸ, δvᴿ) + v̂ * δuˢ end From a8eda445021d87c2a40457f08baf3ea5ed68a50e Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 6 Jun 2024 12:14:08 -0400 Subject: [PATCH 265/567] export ZStar --- .../HydrostaticFreeSurfaceModels.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl index 4dc8a5bf6f..c0fad04ff5 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl @@ -3,7 +3,8 @@ module HydrostaticFreeSurfaceModels export HydrostaticFreeSurfaceModel, ExplicitFreeSurface, ImplicitFreeSurface, SplitExplicitFreeSurface, - PrescribedVelocityFields + PrescribedVelocityFields, + ZStar using KernelAbstractions: @index, @kernel using KernelAbstractions.Extras.LoopInfo: @unroll From f8fdbc0ee22b5b9d5f07a94a3f364ba5307ae0cf Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 6 Jun 2024 12:16:03 -0400 Subject: [PATCH 266/567] some inbounds --- .../generalized_vertical_spacing.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index de71db2fdf..c6085e2c6a 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -117,10 +117,10 @@ update_vertical_spacing!(model, grid, Δt; kwargs...) = nothing @inline Δzᶠᶠᶜ(i, j, k, grid::AbstractVerticalSpacingGrid) = ℑxyᶠᶠᵃ(i, j, k, grid, Δzᶜᶜᶜ) @inline Δz_reference(i, j, k, Δz::Number) = Δz -@inline Δz_reference(i, j, k, Δz::AbstractVector) = Δz[k] +@inline Δz_reference(i, j, k, Δz::AbstractVector) = @inbounds Δz[k] @inline Δz_reference(i, j, k, Δz::AbstractVerticalSpacing{<:Number}) = Δz.Δr -@inline Δz_reference(i, j, k, Δz::AbstractVerticalSpacing) = Δz.Δr[k] +@inline Δz_reference(i, j, k, Δz::AbstractVerticalSpacing) = @inbounds Δz.Δr[k] @inline Δzᶜᶜᶠ_reference(i, j, k, grid) = Δz_reference(i, j, k, grid.Δzᵃᵃᶠ) @inline Δzᶜᶜᶜ_reference(i, j, k, grid) = Δz_reference(i, j, k, grid.Δzᵃᵃᶜ) From ab2062dca28d9795df6cb6a08322ae884d01ac8b Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 6 Jun 2024 12:17:30 -0400 Subject: [PATCH 267/567] small changes --- .../HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 86cf05e486..0f91223282 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -28,11 +28,11 @@ on_architecture(arch, coord::ZStarSpacing) = Grids.coordinate_summary(Δ::ZStarSpacing, name) = @sprintf("Free-surface following with Δ%s=%s", name, prettysummary(Δ.Δr)) - const ZStarSpacingRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:ZStarSpacing} const ZStarSpacingLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:ZStarSpacing} +const ZStarSpacingOSG = OrthogonalSphericalShellGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:ZStarSpacing} -const ZStarSpacingUnderlyingGrid = Union{ZStarSpacingRG, ZStarSpacingLLG} +const ZStarSpacingUnderlyingGrid = Union{ZStarSpacingRG, ZStarSpacingLLG, ZStarSpacingOSG} const ZStarSpacingImmersedGrid = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:ZStarSpacingUnderlyingGrid} const ZStarSpacingGrid = Union{ZStarSpacingUnderlyingGrid, ZStarSpacingImmersedGrid} From 733cf723eb01087e9c3d1322dc7ef65164f6d72b Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Thu, 6 Jun 2024 13:15:32 -0400 Subject: [PATCH 268/567] hmmm this was wrong? --- .../distributed_split_explicit_free_surface.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl index c78740d855..fcca388144 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl @@ -12,7 +12,7 @@ function SplitExplicitAuxiliaryFields(grid::DistributedGrid) Hᶠᶜ = Field{Face, Center, Nothing}(grid) Hᶜᶠ = Field{Center, Face, Nothing}(grid) - Hᶜᶠ = Field{Center, Center, Nothing}(grid) + Hᶜᶜ = Field{Center, Center, Nothing}(grid) calculate_column_height!(Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, grid) From 9e9db84e18e2c83f51d5814b5fabab959f1bd54f Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Fri, 14 Jun 2024 14:26:19 -0400 Subject: [PATCH 269/567] some comments --- .../z_star_vertical_spacing.jl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 0f91223282..2b7d6c19f1 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -44,11 +44,12 @@ function generalized_spacing_grid(grid::ImmersedBoundaryGrid, ::ZStar) return ImmersedBoundaryGrid(underlying_grid, grid.immersed_boundary; active_cells_map) end -# Replacing the z-coordinate with a moving vertical coordinate, defined by its reference spacing, -# the actual vertical spacing and a scaling +# Replacing the z-coordinate with a moving vertical coordinate, defined by +# - the reference spacing, +# - the scaling to apply to the reference +# - the derivative in time of the spacing function generalized_spacing_grid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, ::ZStar) where {FT, TX, TY, TZ} - # Memory layout for Δz spacings should be local in z instead of x s⁻ = Field{Center, Center, Nothing}(grid) sⁿ = Field{Center, Center, Nothing}(grid) ∂t_∂s = Field{Center, Center, Nothing}(grid) From 645c2b752ed44cd83cff205154e8b39bbe8ff136 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Fri, 14 Jun 2024 14:36:26 -0400 Subject: [PATCH 270/567] some docs --- .../generalized_vertical_spacing.jl | 2 +- .../z_star_vertical_spacing.jl | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index c6085e2c6a..f301fb1664 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -8,7 +8,7 @@ using Oceananigans.Utils: getnamewrapper using Adapt using Printf -import Oceananigans.Grids: with_halo +import Oceananigans.Grids: with_halo, znode import Oceananigans.Operators: Δzᶜᶜᶠ, Δzᶜᶜᶜ, Δzᶜᶠᶠ, Δzᶜᶠᶜ, Δzᶠᶜᶠ, Δzᶠᶜᶜ, Δzᶠᶠᶠ, Δzᶠᶠᶜ import Oceananigans.Advection: ∂t_∂s_grid import Oceananigans.Architectures: arch_array diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 2b7d6c19f1..4a7e658a26 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -6,6 +6,22 @@ """ free-surface following vertical coordinate """ struct ZStar end +""" + struct ZStarSpacing{R, S} <: AbstractVerticalSpacing{R} + +A vertical spacing for the hydrostatic free surface model that follows the free surface. +The vertical spacing is defined by a reference spacing `Δr` and a scaling `s` that obeys +```math +s = (η + H) / η +``` +where ``η`` is the free surface height and ``H`` the vertical depth of the water column + +# Fields +- `Δr`: reference vertical spacing with `η = 0` +- `sⁿ`: scaling of the vertical coordinate at time step `n` +- `s⁻`:scaling of the vertical coordinate at time step `n - 1` +- `∂t_∂s`: Time derivative of `s` +""" struct ZStarSpacing{R, S} <: AbstractVerticalSpacing{R} Δr :: R sⁿ :: S From c01ed53a9b5b6b5cc7b449204145bdbfbd6a7f65 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Fri, 26 Jul 2024 16:05:53 -0400 Subject: [PATCH 271/567] default for dt --- .../NonhydrostaticModels/update_nonhydrostatic_model_state.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl b/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl index 6e8988e262..961462a649 100644 --- a/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl +++ b/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl @@ -16,7 +16,7 @@ Update peripheral aspects of the model (halo regions, diffusivities, hydrostatic pressure) to the current model state. If `callbacks` are provided (in an array), they are called in the end. """ -function update_state!(model::NonhydrostaticModel, Δt, callbacks=[]; compute_tendencies = true) +function update_state!(model::NonhydrostaticModel, Δt=nothing, callbacks=[]; compute_tendencies = true) # Mask immersed tracers foreach(model.tracers) do tracer From 473b943f243c58d8d08c6a6d8ad54e6971387b4a Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 29 Jul 2024 13:48:47 -0400 Subject: [PATCH 272/567] uncorrectly scaled tracer equation --- .../generalized_vertical_spacing.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index f301fb1664..320ab870a1 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -160,9 +160,9 @@ update_vertical_spacing!(model, grid, Δt; kwargs...) = nothing oh_point_five = convert(FT, 0.5) @inbounds begin - ∂t_∂sθ = (one_point_five + χ) * sⁿ[i, j, k] * Gⁿ[i, j, k] - (oh_point_five + χ) * s⁻[i, j, k] * G⁻[i, j, k] + ∂t_∂sθ = (one_point_five + χ) * Gⁿ[i, j, k] - (oh_point_five + χ) * G⁻[i, j, k] - # We store temporarily sθ in θ. the unscaled θ will be retrived later on with `scale_tracers!` + # We store temporarily sθ in θ. the unscaled θ will be retrived later on with `unscale_tracers!` θ[i, j, k] = sⁿ[i, j, k] * θ[i, j, k] + convert(FT, Δt) * ∂t_∂sθ end end @@ -176,12 +176,12 @@ ab2_step_tracer_field!(tracer_field, grid::AbstractVerticalSpacingGrid, Δt, χ, const EmptyTuples = Union{NamedTuple{(),Tuple{}}, Tuple{}} -scale_tracers!(::EmptyTuples, ::AbstractVerticalSpacingGrid; kwargs...) = nothing +unscale_tracers!(::EmptyTuples, ::AbstractVerticalSpacingGrid; kwargs...) = nothing tracer_scaling_parameters(param::Symbol, tracers, grid) = KernelParameters((size(grid, 1), size(grid, 2), length(tracers)), (0, 0, 0)) tracer_scaling_parameters(param::KernelParameters{S, O}, tracers, grid) where {S, O} = KernelParameters((S..., length(tracers)), (O..., 0)) -function scale_tracers!(tracers, grid::AbstractVerticalSpacingGrid; parameters = :xy) +function unscale_tracers!(tracers, grid::AbstractVerticalSpacingGrid; parameters = :xy) parameters = tracer_scaling_parameters(parameters, tracers, grid) launch!(architecture(grid), grid, parameters, _scale_tracers, tracers, grid.Δzᵃᵃᶠ.sⁿ, Val(grid.Hz), Val(grid.Nz)) From e951c852b96ba37a88035b68fd7d18a06fc65324 Mon Sep 17 00:00:00 2001 From: Simone Silvestri <33547697+simone-silvestri@users.noreply.github.com> Date: Mon, 29 Jul 2024 15:10:52 -0400 Subject: [PATCH 273/567] why would this create problems? --- .../generalized_vertical_spacing.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index 320ab870a1..96cd3e4e07 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -156,11 +156,11 @@ update_vertical_spacing!(model, grid, Δt; kwargs...) = nothing i, j, k = @index(Global, NTuple) FT = eltype(χ) - one_point_five = convert(FT, 1.5) - oh_point_five = convert(FT, 0.5) + C₁ = convert(FT, 1.5) + χ + C₂ = convert(FT, 0.5) + χ @inbounds begin - ∂t_∂sθ = (one_point_five + χ) * Gⁿ[i, j, k] - (oh_point_five + χ) * G⁻[i, j, k] + ∂t_∂sθ = C₁ * sⁿ[i, j, k] * Gⁿ[i, j, k] - C₂ * s⁻[i, j, k] * G⁻[i, j, k] # We store temporarily sθ in θ. the unscaled θ will be retrived later on with `unscale_tracers!` θ[i, j, k] = sⁿ[i, j, k] * θ[i, j, k] + convert(FT, Δt) * ∂t_∂sθ From 047b423686a9c2e215c74437cf61f9f68250366a Mon Sep 17 00:00:00 2001 From: simone-silvestri Date: Tue, 30 Jul 2024 20:53:24 -0400 Subject: [PATCH 274/567] remove extra term --- .../generalized_vertical_spacing.jl | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index 96cd3e4e07..77d9427a1c 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -134,12 +134,6 @@ update_vertical_spacing!(model, grid, Δt; kwargs...) = nothing @inline Δzᶠᶠᶠ_reference(i, j, k, grid) = ℑxyᶠᶠᵃ(i, j, k, grid, Δzᶜᶜᶠ_reference) @inline Δzᶠᶠᶜ_reference(i, j, k, grid) = ℑxyᶠᶠᵃ(i, j, k, grid, Δzᶜᶜᶜ_reference) -##### -##### Vertical velocity of the Δ-surfaces to be included in the continuity equation -##### - -@inline ∂t_∂s_grid(i, j, k, grid) = zero(grid) - ##### ##### Additional terms to be included in the momentum equations (fallbacks) ##### From d977719d4a013ea0873fe7037ababb076607913b Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 8 Aug 2024 01:02:21 -0400 Subject: [PATCH 275/567] comvenience fix --- src/Advection/vector_invariant_self_upwinding.jl | 3 +-- .../generalized_vertical_spacing.jl | 4 ++-- .../HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl | 1 + 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Advection/vector_invariant_self_upwinding.jl b/src/Advection/vector_invariant_self_upwinding.jl index b3157baceb..9851eeed66 100644 --- a/src/Advection/vector_invariant_self_upwinding.jl +++ b/src/Advection/vector_invariant_self_upwinding.jl @@ -3,8 +3,7 @@ ##### # Metric term for moving grids -@inline ∂t_∂s_grid(i, j, k, grid) = zero(grid) -@inline V_times_∂t_∂s_grid(i, j, k, grid) = ∂t_∂s_grid(i, j, k, grid) * Vᶜᶜᶜ(i, j, k, grid) +@inline V_times_∂t_∂s_grid(i, j, k, grid) = zero(grid) @inline δx_U(i, j, k, grid, u, v) = δxᶜᶜᶜ(i, j, k, grid, Ax_qᶠᶜᶜ, u) @inline δy_V(i, j, k, grid, u, v) = δyᶜᶜᶜ(i, j, k, grid, Ay_qᶜᶠᶜ, v) diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index 77d9427a1c..594ba59c42 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -10,7 +10,7 @@ using Printf import Oceananigans.Grids: with_halo, znode import Oceananigans.Operators: Δzᶜᶜᶠ, Δzᶜᶜᶜ, Δzᶜᶠᶠ, Δzᶜᶠᶜ, Δzᶠᶜᶠ, Δzᶠᶜᶜ, Δzᶠᶠᶠ, Δzᶠᶠᶜ -import Oceananigans.Advection: ∂t_∂s_grid +import Oceananigans.Advection: V_times_∂t_∂s_grid import Oceananigans.Architectures: arch_array """ @@ -168,7 +168,7 @@ ab2_step_tracer_field!(tracer_field, grid::AbstractVerticalSpacingGrid, Δt, χ, grid.Δzᵃᵃᶠ.s⁻, Δt, χ, Gⁿ, G⁻) -const EmptyTuples = Union{NamedTuple{(),Tuple{}}, Tuple{}} +const EmptyTuples = Union{NamedTuple{(), Tuple{}}, Tuple{}} unscale_tracers!(::EmptyTuples, ::AbstractVerticalSpacingGrid; kwargs...) = nothing diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 4a7e658a26..6b186821f0 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -104,6 +104,7 @@ reference_Δzᵃᵃᶜ(grid::ZStarSpacingGrid) = grid.Δzᵃᵃᶜ.Δr @inline Δzᶜᶜᶜ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δzᶜᶜᶜ_reference(i, j, k, grid) * grid.Δzᵃᵃᶜ.sⁿ[i, j, 1] @inline ∂t_∂s_grid(i, j, k, grid::ZStarSpacingGrid) = grid.Δzᵃᵃᶜ.∂t_∂s[i, j, k] +@inline V_times_∂t_∂s_grid(i, j, k, grid::ZStarSpacingGrid) = ∂t_∂s_grid(i, j, k, grid) * Vᶜᶜᶜ(i, j, k, grid) ##### ##### ZStar-specific vertical spacings update From f6d10d126753a4ac578ffd40358f47d54bc29139 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 12 Aug 2024 10:48:33 -0400 Subject: [PATCH 276/567] ok this works --- ...ute_hydrostatic_free_surface_tendencies.jl | 23 +++++++++------- .../explicit_free_surface.jl | 2 +- .../generalized_vertical_spacing.jl | 9 +++++-- ..._free_surface_tendency_kernel_functions.jl | 27 +------------------ .../split_explicit_free_surface_kernels.jl | 6 ++--- ...te_hydrostatic_free_surface_model_state.jl | 4 +-- .../z_star_vertical_spacing.jl | 20 -------------- validation/z_star_coordinate/lock_release.jl | 23 +++++++++------- 8 files changed, 39 insertions(+), 75 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl index ef4a550a50..5b0af4027c 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl @@ -118,16 +118,19 @@ function compute_free_surface_tendency!(grid, model, kernel_parameters) arch = architecture(grid) - args = tuple(model.velocities, - model.free_surface, - model.tracers, - model.auxiliary_fields, - model.forcing, - model.clock) - - launch!(arch, grid, kernel_parameters, - compute_hydrostatic_free_surface_Gη!, model.timestepper.Gⁿ.η, - grid, args) + if model.free_surface isa ExplicitFreeSurface + + args = tuple(model.velocities, + model.free_surface, + model.tracers, + model.auxiliary_fields, + model.forcing, + model.clock) + + launch!(arch, grid, kernel_parameters, + compute_hydrostatic_free_surface_Gη!, model.timestepper.Gⁿ.η, + grid, args) + end return nothing end diff --git a/src/Models/HydrostaticFreeSurfaceModels/explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/explicit_free_surface.jl index 79ec3ee857..dbe0c3466d 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/explicit_free_surface.jl @@ -82,7 +82,7 @@ The tendency is called ``G_η`` and defined via """ @inline function free_surface_tendency(i, j, grid, velocities, - free_surface, + free_surface::ExplicitFreeSurface, tracers, auxiliary_fields, forcings, diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index 594ba59c42..31b909416b 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -141,6 +141,9 @@ update_vertical_spacing!(model, grid, Δt; kwargs...) = nothing @inline grid_slope_contribution_x(i, j, k, grid, args...) = zero(grid) @inline grid_slope_contribution_y(i, j, k, grid, args...) = zero(grid) +@inline ∂t_∂s_grid(i, j, k, grid) = zero(grid) +@inline V_times_∂t_∂s_grid(i, j, k, grid) = zero(grid) + ##### ##### Tracer update in generalized vertical coordinates ##### We advance sθ but store θ once sⁿ⁺¹ is known @@ -177,12 +180,14 @@ tracer_scaling_parameters(param::KernelParameters{S, O}, tracers, grid) where {S function unscale_tracers!(tracers, grid::AbstractVerticalSpacingGrid; parameters = :xy) parameters = tracer_scaling_parameters(parameters, tracers, grid) - launch!(architecture(grid), grid, parameters, _scale_tracers, tracers, grid.Δzᵃᵃᶠ.sⁿ, + + launch!(architecture(grid), grid, parameters, _unscale_tracers!, tracers, grid.Δzᵃᵃᶠ.sⁿ, Val(grid.Hz), Val(grid.Nz)) + return nothing end -@kernel function _scale_tracers(tracers, sⁿ, ::Val{Hz}, ::Val{Nz}) where {Hz, Nz} +@kernel function _unscale_tracers!(tracers, sⁿ, ::Val{Hz}, ::Val{Nz}) where {Hz, Nz} i, j, n = @index(Global, NTuple) @unroll for k in -Hz+1:Nz+Hz diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_tendency_kernel_functions.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_tendency_kernel_functions.jl index cec9e6abc8..954b06849e 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_tendency_kernel_functions.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_tendency_kernel_functions.jl @@ -137,29 +137,4 @@ where `c = C[tracer_index]`. - immersed_∇_dot_qᶜ(i, j, k, grid, c, c_immersed_bc, closure, diffusivities, val_tracer_index, clock, model_fields) + biogeochemical_transition(i, j, k, grid, biogeochemistry, val_tracer_name, clock, model_fields) + forcing(i, j, k, grid, clock, model_fields)) -end - -""" -Return the tendency for an explicit free surface at horizontal grid point `i, j`. - -The tendency is called ``G_η`` and defined via - -``` -∂_t η = G_η -``` -""" -@inline function free_surface_tendency(i, j, grid, - velocities, - free_surface, - tracers, - auxiliary_fields, - forcings, - clock) - - k_top = grid.Nz + 1 - model_fields = merge(hydrostatic_fields(velocities, free_surface, tracers), auxiliary_fields) - - return @inbounds ( velocities.w[i, j, k_top] - + forcings.η(i, j, k_top, grid, clock, model_fields)) -end - +end \ No newline at end of file diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 475824d81f..7b2f505ee2 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -494,11 +494,9 @@ wait_free_surface_communication!(free_surface, arch) = nothing # Special update ∂t_∂s for SplitExplicitFreeSurface where # ∂(η / H)/∂t = - ∇ ⋅ U̅ / H -function update_∂t_∂s!(∂t_∂s, parameters, grid, sⁿ, s⁻, Δt, fs::SplitExplicitFreeSurface) +update_∂t_∂s!(∂t_∂s, parameters, grid, sⁿ, s⁻, Δt, fs::SplitExplicitFreeSurface) = launch!(architecture(grid), grid, parameters, _update_∂t_∂s_split_explicit!, ∂t_∂s, fs.state.U̅, fs.state.V̅, fs.auxiliary.Hᶜᶜ, grid) - return nothing -end - + @kernel function _update_∂t_∂s_split_explicit!(∂t_∂s, U̅, V̅, Hᶜᶜ, grid) i, j = @index(Global, NTuple) k_top = grid.Nz + 1 diff --git a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl index 02f47ecbd4..91e7f3c18d 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl @@ -81,7 +81,7 @@ function compute_auxiliaries!(model::HydrostaticFreeSurfaceModel, Δt; w_paramet for (wpar, ppar, κpar) in zip(w_parameters, p_parameters, κ_parameters) # Update the grid update_vertical_spacing!(model, grid, Δt; parameters = wpar) - scale_tracers!(tracers, grid; parameters = wpar) + unscale_tracers!(tracers, grid; parameters = wpar) # Update the other auxiliary terms compute_w_from_continuity!(model; parameters = wpar) @@ -94,4 +94,4 @@ function compute_auxiliaries!(model::HydrostaticFreeSurfaceModel, Δt; w_paramet return nothing end -scale_tracers!(tracers, grid; kwargs...) = nothing +unscale_tracers!(tracers, grid; kwargs...) = nothing diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 6b186821f0..202c4fd5a7 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -130,26 +130,6 @@ function update_vertical_spacing!(model, grid::ZStarSpacingGrid, Δt; parameters return nothing end -@kernel function _update_zstar!(sⁿ, s⁻, η, Hᶜᶜ, grid, ::Val{Nz}) where Nz - i, j = @index(Global, NTuple) - @inbounds begin - bottom = Hᶜᶜ[i, j, 1] - h = (bottom + η[i, j, grid.Nz+1]) / bottom - - # update current and previous scaling - s⁻[i, j, 1] = sⁿ[i, j, 1] - sⁿ[i, j, 1] = h - end -end - -update_∂t_∂s!(∂t_∂s, parameters, grid, sⁿ, s⁻, Δt, fs) = - launch!(architecture(grid), grid, parameters, _update_∂t_∂s!, ∂t_∂s, sⁿ, s⁻, Δt) - -@kernel function _update_∂t_∂s!(∂t_∂s, sⁿ, s⁻, Δt) - i, j = @index(Global, NTuple) - ∂t_∂s[i, j, 1] = (sⁿ[i, j, 1] - s⁻[i, j, 1]) / Δt -end - ##### ##### ZStar-specific implementation of the additional terms to be included in the momentum equations ##### diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index 10b3d3712e..e059d64d45 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -12,9 +12,9 @@ grid = RectilinearGrid(size = (10, 10), topology = (Bounded, Flat, Bounded)) model = HydrostaticFreeSurfaceModel(; grid, - generalized_vertical_coordinate = nothing, - momentum_advection = Centered(), - tracer_advection = Centered(), + generalized_vertical_coordinate = ZStar(), + momentum_advection = WENO(), + tracer_advection = WENO(), buoyancy = BuoyancyTracer(), tracers = :b, free_surface = SplitExplicitFreeSurface(; substeps = 10)) @@ -67,12 +67,13 @@ function progress(sim) return nothing end -simulation.callbacks[:progress] = Callback(progress, IterationInterval(100)) +simulation.callbacks[:progress] = Callback(progress, IterationInterval(1)) simulation.callbacks[:wizard] = Callback(TimeStepWizard(; cfl = 0.2, max_change = 1.1), IterationInterval(10)) run!(simulation) using Oceananigans.Utils +using KernelAbstractions: @kernel, @index @kernel function _compute_field!(tmp, s, b) i, j, k = @index(Global, NTuple) @@ -83,14 +84,16 @@ using Oceananigans.Fields: OneField # # Check tracer conservation b = FieldTimeSeries("zstar_model.jld2", "b") -# s = FieldTimeSeries("zstar_model.jld2", "sⁿ") -s = OneField() +s = FieldTimeSeries("zstar_model.jld2", "sⁿ") + +# s = OneField() tmpfield = CenterField(grid) -launch!(CPU(), grid, :xyz, _compute_field!, tmpfield, s, b[1]) -init = sum(tmpfield) / 10 # sum(s[1]) +launch!(CPU(), grid, :xyz, _compute_field!, tmpfield, s[1], b[1]) +init = sum(tmpfield) / sum(s[1]) drift = [] + for t in 1:length(b.times) - launch!(CPU(), grid, :xyz, _compute_field!, tmpfield, s, b[t]) - push!(drift, sum(tmpfield) / 10 - init) + launch!(CPU(), grid, :xyz, _compute_field!, tmpfield, s[t], b[t]) + push!(drift, sum(tmpfield) / sum(s[t]) - init) end From 0c0f33fff2609e9a029fa8df7facf66270f24156 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 12 Aug 2024 11:17:56 -0400 Subject: [PATCH 277/567] do not overwrite --- .../HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index 31b909416b..dbbd19110f 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -142,7 +142,6 @@ update_vertical_spacing!(model, grid, Δt; kwargs...) = nothing @inline grid_slope_contribution_y(i, j, k, grid, args...) = zero(grid) @inline ∂t_∂s_grid(i, j, k, grid) = zero(grid) -@inline V_times_∂t_∂s_grid(i, j, k, grid) = zero(grid) ##### ##### Tracer update in generalized vertical coordinates From 68d2132faa98e77acfd027848d3cdcda67aaa93b Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 12 Aug 2024 14:30:08 -0400 Subject: [PATCH 278/567] bugfix --- .../z_star_vertical_spacing.jl | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 202c4fd5a7..321454642b 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -130,6 +130,18 @@ function update_vertical_spacing!(model, grid::ZStarSpacingGrid, Δt; parameters return nothing end +@kernel function _update_zstar!(sⁿ, s⁻, η, Hᶜᶜ, grid, ::Val{Nz}) where Nz + i, j = @index(Global, NTuple) + @inbounds begin + bottom = Hᶜᶜ[i, j, 1] + h = (bottom + η[i, j, grid.Nz+1]) / bottom + + # update current and previous scaling + s⁻[i, j, 1] = sⁿ[i, j, 1] + sⁿ[i, j, 1] = h + end +end + ##### ##### ZStar-specific implementation of the additional terms to be included in the momentum equations ##### From 742673b6fedffc4972796d2baeb062f76c315bc4 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 27 Aug 2024 10:36:27 -0400 Subject: [PATCH 279/567] update Manifest --- Manifest.toml | 209 ++++++++++++++++++++++++++------------------------ 1 file changed, 110 insertions(+), 99 deletions(-) diff --git a/Manifest.toml b/Manifest.toml index 077fbb4e4e..185a5f1334 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.10.4" manifest_format = "2.0" -project_hash = "fc80901e62b273c6815173b7a2bdcfd9bb441d24" +project_hash = "107cad82ede3e7be685ee86842b76275e6dc7373" [[deps.AbstractFFTs]] deps = ["LinearAlgebra"] @@ -90,10 +90,10 @@ version = "5.4.3" SpecialFunctions = "276daf66-3868-5448-9aa4-cd146d93841b" [[deps.CUDA_Driver_jll]] -deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl", "Pkg"] -git-tree-sha1 = "97df9d4d6be8ac6270cb8fd3b8fc413690820cbd" +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "325058b426c2b421e3d2df3d5fa646d72d2e3e7e" uuid = "4ee394cb-3365-5eb0-8335-949819d2adfc" -version = "0.9.1+1" +version = "0.9.2+0" [[deps.CUDA_Runtime_Discovery]] deps = ["Libdl"] @@ -108,10 +108,10 @@ uuid = "76a88914-d11a-5bdc-97e0-2f5a05c973a2" version = "0.14.1+0" [[deps.Cairo_jll]] -deps = ["Artifacts", "Bzip2_jll", "CompilerSupportLibraries_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "JLLWrappers", "LZO_jll", "Libdl", "Pixman_jll", "Pkg", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Zlib_jll", "libpng_jll"] -git-tree-sha1 = "4b859a208b2397a7a623a03449e4636bdb17bcf2" +deps = ["Artifacts", "Bzip2_jll", "CompilerSupportLibraries_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "JLLWrappers", "LZO_jll", "Libdl", "Pixman_jll", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Zlib_jll", "libpng_jll"] +git-tree-sha1 = "a2f1c8c668c8e3cb4cca4e57a8efdb09067bb3fd" uuid = "83423d85-b0ee-5818-9007-b63ccbeb887a" -version = "1.16.1+1" +version = "1.18.0+2" [[deps.ColorTypes]] deps = ["FixedPointNumbers", "Random"] @@ -133,9 +133,9 @@ version = "0.3.6" [[deps.Compat]] deps = ["TOML", "UUIDs"] -git-tree-sha1 = "b1c55339b7c6c350ee89f2c1604299660525b248" +git-tree-sha1 = "8ae8d32e09f0dcf42a36b90d4e17f5dd2e4c4215" uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" -version = "4.15.0" +version = "4.16.0" weakdeps = ["Dates", "LinearAlgebra"] [deps.Compat.extensions] @@ -148,9 +148,9 @@ version = "1.1.1+0" [[deps.ConstructionBase]] deps = ["LinearAlgebra"] -git-tree-sha1 = "d8a9c0b6ac2d9081bf76324b39c78ca3ce4f0c98" +git-tree-sha1 = "a33b7ced222c6165f624a3f2b55945fac5a598d9" uuid = "187b0558-2788-49d3-abe0-74a17ed4e7c9" -version = "1.5.6" +version = "1.5.7" [deps.ConstructionBase.extensions] ConstructionBaseIntervalSetsExt = "IntervalSets" @@ -239,9 +239,9 @@ version = "1.0.1" [[deps.Expat_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "4558ab818dcceaab612d1bb8c19cee87eda2b83c" +git-tree-sha1 = "1c6317308b9dc757616f0b5cb379db10494443a7" uuid = "2e619515-83b5-522b-bb60-26c02a35a201" -version = "2.5.0+0" +version = "2.6.2+0" [[deps.ExprTools]] git-tree-sha1 = "27415f162e6028e81c72b82ef756bf321213b6ec" @@ -288,22 +288,22 @@ uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93" version = "0.8.5" [[deps.Fontconfig_jll]] -deps = ["Artifacts", "Bzip2_jll", "Expat_jll", "FreeType2_jll", "JLLWrappers", "Libdl", "Libuuid_jll", "Pkg", "Zlib_jll"] -git-tree-sha1 = "21efd19106a55620a188615da6d3d06cd7f6ee03" +deps = ["Artifacts", "Bzip2_jll", "Expat_jll", "FreeType2_jll", "JLLWrappers", "Libdl", "Libuuid_jll", "Zlib_jll"] +git-tree-sha1 = "db16beca600632c95fc8aca29890d83788dd8b23" uuid = "a3f928ae-7b40-5064-980b-68af3947d34b" -version = "2.13.93+0" +version = "2.13.96+0" [[deps.FreeType2_jll]] deps = ["Artifacts", "Bzip2_jll", "JLLWrappers", "Libdl", "Zlib_jll"] -git-tree-sha1 = "d8db6a5a2fe1381c1ea4ef2cab7c69c2de7f9ea0" +git-tree-sha1 = "5c1d8ae0efc6c2e7b1fc502cbe25def8f661b7bc" uuid = "d7e528f0-a631-5988-bf34-fe36492bcfd7" -version = "2.13.1+0" +version = "2.13.2+0" [[deps.FriBidi_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "aa31987c2ba8704e23c6c8ba8a4f769d5d7e4f91" +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "1ed150b39aebcc805c26b93a8d0122c940f64ce2" uuid = "559328eb-81f9-559d-9380-de523a88c83c" -version = "1.0.10+0" +version = "1.0.14+0" [[deps.Future]] deps = ["Random"] @@ -316,9 +316,9 @@ version = "6.2.1+6" [[deps.GPUArrays]] deps = ["Adapt", "GPUArraysCore", "LLVM", "LinearAlgebra", "Printf", "Random", "Reexport", "Serialization", "Statistics"] -git-tree-sha1 = "a74c3f1cf56a3dfcdef0605f8cdb7015926aae30" +git-tree-sha1 = "62ee71528cca49be797076a76bdc654a170a523e" uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" -version = "10.3.0" +version = "10.3.1" [[deps.GPUArraysCore]] deps = ["Adapt"] @@ -340,9 +340,9 @@ version = "0.21.0+0" [[deps.Glib_jll]] deps = ["Artifacts", "Gettext_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Libiconv_jll", "Libmount_jll", "PCRE2_jll", "Zlib_jll"] -git-tree-sha1 = "e94c92c7bf4819685eb80186d51c43e71d4afa17" +git-tree-sha1 = "7c82e6a6cd34e9d935e9aa4051b66c6ff3af59ba" uuid = "7746bdde-850d-59dc-9ae8-88ece973131d" -version = "2.76.5+0" +version = "2.80.2+0" [[deps.Glob]] git-tree-sha1 = "97285bbd5230dd766e9ef6749b80fc617126d496" @@ -368,10 +368,10 @@ uuid = "0234f1f7-429e-5d53-9886-15a909be8d59" version = "1.14.3+3" [[deps.HarfBuzz_jll]] -deps = ["Artifacts", "Cairo_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "Graphite2_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Pkg"] -git-tree-sha1 = "129acf094d168394e80ee1dc4bc06ec835e510a3" +deps = ["Artifacts", "Cairo_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "Graphite2_jll", "JLLWrappers", "Libdl", "Libffi_jll"] +git-tree-sha1 = "401e4f3f30f43af2c8478fc008da50096ea5240f" uuid = "2e76f6c2-a576-52d4-95c1-20adfe4de566" -version = "2.8.1+1" +version = "8.3.1+0" [[deps.Hwloc_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] @@ -430,10 +430,10 @@ uuid = "82899510-4779-5014-852e-03e436cf321d" version = "1.0.0" [[deps.JLD2]] -deps = ["FileIO", "MacroTools", "Mmap", "OrderedCollections", "Pkg", "PrecompileTools", "Reexport", "Requires", "TranscodingStreams", "UUIDs", "Unicode"] -git-tree-sha1 = "5fe858cb863e211c6dedc8cce2dc0752d4ab6e2b" +deps = ["FileIO", "MacroTools", "Mmap", "OrderedCollections", "PrecompileTools", "Requires", "TranscodingStreams"] +git-tree-sha1 = "049950edff105ff73918d29dbf109220ff364157" uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819" -version = "0.4.50" +version = "0.4.52" [[deps.JLLWrappers]] deps = ["Artifacts", "Preferences"] @@ -448,28 +448,32 @@ uuid = "9c1d0b0a-7046-5b2e-a33f-ea22f176ac7e" version = "0.2.1+0" [[deps.KernelAbstractions]] -deps = ["Adapt", "Atomix", "InteractiveUtils", "LinearAlgebra", "MacroTools", "PrecompileTools", "Requires", "SparseArrays", "StaticArrays", "UUIDs", "UnsafeAtomics", "UnsafeAtomicsLLVM"] -git-tree-sha1 = "d0448cebd5919e06ca5edc7a264631790de810ec" +deps = ["Adapt", "Atomix", "InteractiveUtils", "MacroTools", "PrecompileTools", "Requires", "StaticArrays", "UUIDs", "UnsafeAtomics", "UnsafeAtomicsLLVM"] +git-tree-sha1 = "cb1cff88ef2f3a157cbad75bbe6b229e1975e498" uuid = "63c18a36-062a-441e-b654-da1e3ab1ce7c" -version = "0.9.22" +version = "0.9.25" [deps.KernelAbstractions.extensions] EnzymeExt = "EnzymeCore" + LinearAlgebraExt = "LinearAlgebra" + SparseArraysExt = "SparseArrays" [deps.KernelAbstractions.weakdeps] EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869" + LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" [[deps.LAME_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "f6250b16881adf048549549fba48b1161acdac8c" +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "170b660facf5df5de098d866564877e119141cbd" uuid = "c1c5ebd0-6772-5130-a774-d5fcae4a789d" -version = "3.100.1+0" +version = "3.100.2+0" [[deps.LLVM]] deps = ["CEnum", "LLVMExtra_jll", "Libdl", "Preferences", "Printf", "Requires", "Unicode"] -git-tree-sha1 = "020abd49586480c1be84f57da0017b5d3db73f7c" +git-tree-sha1 = "2470e69781ddd70b8878491233cd09bc1bd7fc96" uuid = "929cbde3-209d-540e-8aea-75f648917ca0" -version = "8.0.0" +version = "8.1.0" weakdeps = ["BFloat16s"] [deps.LLVM.extensions] @@ -477,9 +481,9 @@ weakdeps = ["BFloat16s"] [[deps.LLVMExtra_jll]] deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] -git-tree-sha1 = "c2636c264861edc6d305e6b4d528f09566d24c5e" +git-tree-sha1 = "597d1c758c9ae5d985ba4202386a607c675ee700" uuid = "dad2f222-ce93-54a1-a47d-0025e8a3acab" -version = "0.0.30+0" +version = "0.0.31+0" [[deps.LLVMLoopInfo]] git-tree-sha1 = "2e5c102cfc41f48ae4740c7eca7743cc7e7b75ea" @@ -488,9 +492,9 @@ version = "1.0.0" [[deps.LLVMOpenMP_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "d986ce2d884d49126836ea94ed5bfb0f12679713" +git-tree-sha1 = "e16271d212accd09d52ee0ae98956b8a05c4b626" uuid = "1d63c593-3942-5779-bab2-d838dc0a180e" -version = "15.0.7+0" +version = "17.0.6+0" [[deps.LRUCache]] git-tree-sha1 = "b3cc6698599b10e652832c2f23db3cab99d51b59" @@ -502,10 +506,10 @@ weakdeps = ["Serialization"] SerializationExt = ["Serialization"] [[deps.LZO_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "e5b909bcf985c5e2605737d2ce278ed791b89be6" +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "70c5da094887fd2cae843b8db33920bac4b6f07d" uuid = "dd4b983a-f0e5-5f8d-a1b7-129d4a5fb1ac" -version = "2.10.1+0" +version = "2.10.2+0" [[deps.LaTeXStrings]] git-tree-sha1 = "50901ebc375ed41dbf8058da26f9de442febbbec" @@ -550,16 +554,16 @@ uuid = "e9f186c6-92d2-5b65-8a66-fee21dc1b490" version = "3.2.2+1" [[deps.Libgcrypt_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgpg_error_jll", "Pkg"] -git-tree-sha1 = "64613c82a59c120435c067c2b809fc61cf5166ae" +deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgpg_error_jll"] +git-tree-sha1 = "9fd170c4bbfd8b935fdc5f8b7aa33532c991a673" uuid = "d4300ac3-e22c-5743-9152-c294e39db1e4" -version = "1.8.7+0" +version = "1.8.11+0" [[deps.Libgpg_error_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "c333716e46366857753e273ce6a69ee0945a6db9" +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "fbb1f2bef882392312feb1ede3615ddc1e9b99ed" uuid = "7add5ba3-2f88-524e-9cd5-f83b8a55f7b8" -version = "1.42.0+0" +version = "1.49.0+0" [[deps.Libiconv_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] @@ -568,16 +572,16 @@ uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531" version = "1.17.0+0" [[deps.Libmount_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "9c30530bf0effd46e15e0fdcf2b8636e78cbbd73" +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "0c4f9c4f1a50d8f35048fa0532dabbadf702f81e" uuid = "4b2f31a3-9ecc-558c-b454-b3730dcb73e9" -version = "2.35.0+0" +version = "2.40.1+0" [[deps.Libuuid_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "e5edc369a598dfde567269dc6add5812cfa10cd5" +git-tree-sha1 = "5ee6203157c120d79034c748a2acba45b82b8807" uuid = "38a345b3-de98-5d2b-a5d3-14cd9215e700" -version = "2.39.3+0" +version = "2.40.1+0" [[deps.LinearAlgebra]] deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] @@ -604,9 +608,9 @@ uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" [[deps.Lz4_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "6c26c5e8a4203d43b5497be3ec5d4e0c3cde240a" +git-tree-sha1 = "7f26c8fc5229e68484e0b3447312c98e16207d11" uuid = "5ced341a-0733-55b8-9ab6-a4889d929147" -version = "1.9.4+0" +version = "1.10.0+0" [[deps.MKL_jll]] deps = ["Artifacts", "IntelOpenMP_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "oneTBB_jll"] @@ -758,10 +762,10 @@ uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e" version = "0.5.5+0" [[deps.Opus_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "51a08fb14ec28da2ec7a927c4337e4332c2a4720" +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "6703a85cb3781bd5909d48730a67205f3f31a575" uuid = "91d4177d-7536-5919-b921-800302f37372" -version = "1.3.2+0" +version = "1.3.3+0" [[deps.OrderedCollections]] git-tree-sha1 = "dfdf5519f235516220579f949664f1bf44e741c5" @@ -774,6 +778,17 @@ git-tree-sha1 = "2cd396108e178f3ae8dedbd8e938a18726ab2fbf" uuid = "c2071276-7c44-58a7-b746-946036e04d0a" version = "0.24.1+0" +[[deps.PCRE2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "efcefdf7-47ab-520b-bdef-62a2eaa19f15" +version = "10.42.0+1" + +[[deps.Pixman_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "LLVMOpenMP_jll", "Libdl"] +git-tree-sha1 = "35621f10a7531bc8fa58f74610b1bfb70a3cfc6b" +uuid = "30392449-352a-5448-841d-b1acce4e97dc" +version = "0.43.4+0" + [[deps.Pkg]] deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" @@ -840,10 +855,10 @@ uuid = "74087812-796a-5b5d-8853-05524746bad3" version = "1.7.0" [[deps.RandomNumbers]] -deps = ["Random", "Requires"] -git-tree-sha1 = "043da614cc7e95c703498a491e2c21f58a2b8111" +deps = ["Random"] +git-tree-sha1 = "c6ec94d2aaba1ab2ff983052cf6a606ca5985902" uuid = "e6cf234a-135c-5ec9-84dd-332b85af5143" -version = "1.5.3" +version = "1.6.0" [[deps.RealDot]] deps = ["LinearAlgebra"] @@ -1027,13 +1042,9 @@ uuid = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" version = "0.5.24" [[deps.TranscodingStreams]] -git-tree-sha1 = "96612ac5365777520c3c5396314c8cf7408f436a" +git-tree-sha1 = "e84b3a11b9bece70d14cce63406bbc79ed3464d2" uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" -version = "0.11.1" -weakdeps = ["Random", "Test"] - - [deps.TranscodingStreams.extensions] - TestExt = ["Test", "Random"] +version = "0.11.2" [[deps.UUIDs]] deps = ["Random", "SHA"] @@ -1049,21 +1060,21 @@ version = "0.2.1" [[deps.UnsafeAtomicsLLVM]] deps = ["LLVM", "UnsafeAtomics"] -git-tree-sha1 = "bf2c553f25e954a9b38c9c0593a59bb13113f9e5" +git-tree-sha1 = "2d17fabcd17e67d7625ce9c531fb9f40b7c42ce4" uuid = "d80eeb9a-aca5-4d75-85e5-170c8b632249" -version = "0.1.5" +version = "0.2.1" [[deps.XML2_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Libiconv_jll", "Zlib_jll"] -git-tree-sha1 = "d9717ce3518dc68a99e6b96300813760d887a01d" +git-tree-sha1 = "1165b0443d0eca63ac1e32b8c0eb69ed2f4f8127" uuid = "02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a" -version = "2.13.1+0" +version = "2.13.3+0" [[deps.XSLT_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgcrypt_jll", "Libgpg_error_jll", "Libiconv_jll", "Pkg", "XML2_jll", "Zlib_jll"] -git-tree-sha1 = "91844873c4085240b95e795f692c4cec4d805f8a" +deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgcrypt_jll", "Libgpg_error_jll", "Libiconv_jll", "XML2_jll", "Zlib_jll"] +git-tree-sha1 = "a54ee957f4c86b526460a720dbc882fa5edcbefc" uuid = "aed1982a-8fda-507f-9586-7b0439959a61" -version = "1.1.34+0" +version = "1.1.41+0" [[deps.XZ_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] @@ -1090,16 +1101,16 @@ uuid = "a3789734-cfe1-5b06-b2d0-1dd0d9d62d05" version = "1.1.4+0" [[deps.Xorg_libXext_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"] -git-tree-sha1 = "b7c0aa8c376b31e4852b360222848637f481f8c3" +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libX11_jll"] +git-tree-sha1 = "d2d1a5c49fae4ba39983f63de6afcbea47194e85" uuid = "1082639a-0dae-5f34-9b06-72781eeb8cb3" -version = "1.3.4+4" +version = "1.3.6+0" [[deps.Xorg_libXrender_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"] -git-tree-sha1 = "19560f30fd49f4d4efbe7002a1037f8c43d43b96" +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libX11_jll"] +git-tree-sha1 = "47e45cd78224c53109495b3e324df0c37bb61fbe" uuid = "ea2f1a96-1ddc-540d-b46f-429655e07cfa" -version = "0.9.10+4" +version = "0.9.11+0" [[deps.Xorg_libpthread_stubs_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] @@ -1109,9 +1120,9 @@ version = "0.1.1+0" [[deps.Xorg_libxcb_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "XSLT_jll", "Xorg_libXau_jll", "Xorg_libXdmcp_jll", "Xorg_libpthread_stubs_jll"] -git-tree-sha1 = "b4bfde5d5b652e22b9c790ad00af08b6d042b97d" +git-tree-sha1 = "bcd466676fef0878338c61e655629fa7bbc69d8e" uuid = "c7cfdc94-dc32-55de-ac96-5a1b8d977c5b" -version = "1.15.0+0" +version = "1.17.0+0" [[deps.Xorg_xtrans_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] @@ -1137,16 +1148,16 @@ uuid = "477f73a3-ac25-53e9-8cc3-50b2fa2566f0" version = "1.1.2+0" [[deps.libaom_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "3a2ea60308f0996d26f1e5354e10c24e9ef905d4" +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "1827acba325fdcdf1d2647fc8d5301dd9ba43a9d" uuid = "a4ae2306-e953-59d6-aa16-d00cac43593b" -version = "3.4.0+0" +version = "3.9.0+0" [[deps.libass_jll]] -deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "HarfBuzz_jll", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"] -git-tree-sha1 = "5982a94fcba20f02f42ace44b9894ee2b140fe47" +deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "HarfBuzz_jll", "JLLWrappers", "Libdl", "Zlib_jll"] +git-tree-sha1 = "e17c115d55c5fbb7e52ebedb427a0dca79d4484e" uuid = "0ac62f75-1d6f-5e53-bd7c-93b484bb37c0" -version = "0.15.1+0" +version = "0.15.2+0" [[deps.libblastrampoline_jll]] deps = ["Artifacts", "Libdl"] @@ -1154,22 +1165,22 @@ uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" version = "5.8.0+1" [[deps.libfdk_aac_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "daacc84a041563f965be61859a36e17c4e4fcd55" +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "8a22cf860a7d27e4f3498a0fe0811a7957badb38" uuid = "f638f0a6-7fb0-5443-88ba-1cc74229b280" -version = "2.0.2+0" +version = "2.0.3+0" [[deps.libpng_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Zlib_jll"] -git-tree-sha1 = "1ea2ebe8ffa31f9c324e8c1d6e86b4165b84a024" +git-tree-sha1 = "d7015d2e18a5fd9a4f47de711837e980519781a4" uuid = "b53b4c65-9356-5827-b1ea-8c7a1a84506f" -version = "1.6.43+0" +version = "1.6.43+1" [[deps.libvorbis_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Ogg_jll", "Pkg"] -git-tree-sha1 = "b910cb81ef3fe6e78bf6acee440bda86fd6ae00c" +git-tree-sha1 = "490376214c4721cdaca654041f635213c6165cb3" uuid = "f27f6e37-5d2b-51aa-960f-b287f2bc3b7a" -version = "1.3.7+1" +version = "1.3.7+2" [[deps.libzip_jll]] deps = ["Artifacts", "Bzip2_jll", "GnuTLS_jll", "JLLWrappers", "Libdl", "XZ_jll", "Zlib_jll", "Zstd_jll"] From ca6ff376213f4360c70c8c5a47476c317d8c6190 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 27 Aug 2024 10:52:24 -0400 Subject: [PATCH 280/567] a small update --- ...static_free_surface_boundary_tendencies.jl | 4 ++-- ...ute_hydrostatic_free_surface_tendencies.jl | 4 ++-- .../generalized_vertical_spacing.jl | 2 +- .../hydrostatic_free_surface_model.jl | 8 +++++-- .../prescribed_hydrostatic_velocity_fields.jl | 4 ++-- .../set_hydrostatic_free_surface_model.jl | 2 +- .../single_column_model_mode.jl | 2 +- .../split_explicit_free_surface_kernels.jl | 16 +------------ ...te_hydrostatic_free_surface_model_state.jl | 20 ++++++++-------- .../z_star_vertical_spacing.jl | 23 +++++++++++++++---- ...pute_nonhydrostatic_boundary_tendencies.jl | 2 +- .../compute_nonhydrostatic_tendencies.jl | 4 ++-- .../nonhydrostatic_model.jl | 2 +- .../set_nonhydrostatic_model.jl | 4 ++-- .../update_nonhydrostatic_model_state.jl | 4 ++-- .../set_shallow_water_model.jl | 2 +- .../ShallowWaterModels/shallow_water_model.jl | 2 +- .../update_shallow_water_state.jl | 2 +- ...nterleave_communication_and_computation.jl | 10 ++++---- src/Simulations/run.jl | 2 +- src/TimeSteppers/runge_kutta_3.jl | 6 ++--- 21 files changed, 64 insertions(+), 61 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl index 3a335995fe..26bee78f21 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl @@ -13,7 +13,7 @@ using Oceananigans.TurbulenceClosures: required_halo_size using Oceananigans.ImmersedBoundaries: active_interior_map, DistributedActiveCellsIBG # We assume here that top/bottom BC are always synched (no partitioning in z) -function compute_boundary_tendencies!(model::HydrostaticFreeSurfaceModel, Δt) +function compute_boundary_tendencies!(model::HydrostaticFreeSurfaceModel) grid = model.grid arch = architecture(grid) @@ -22,7 +22,7 @@ function compute_boundary_tendencies!(model::HydrostaticFreeSurfaceModel, Δt) κ_parameters = boundary_κ_kernel_parameters(grid, model.closure, arch) # We need new values for `w`, `p` and `κ` - compute_auxiliaries!(model, Δt; w_parameters, p_parameters, κ_parameters) + compute_auxiliaries!(model; w_parameters, p_parameters, κ_parameters) # parameters for communicating North / South / East / West side compute_boundary_tendency_contributions!(grid, arch, model) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl index 5b0af4027c..da57aa2fcf 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl @@ -19,7 +19,7 @@ using Oceananigans.ImmersedBoundaries: active_interior_map, ActiveCellsIBG, Calculate the interior and boundary contributions to tendency terms without the contribution from non-hydrostatic pressure. """ -function compute_tendencies!(model::HydrostaticFreeSurfaceModel, Δt, callbacks) +function compute_tendencies!(model::HydrostaticFreeSurfaceModel, callbacks) kernel_parameters = tuple(interior_tendency_kernel_parameters(model.grid)) @@ -28,7 +28,7 @@ function compute_tendencies!(model::HydrostaticFreeSurfaceModel, Δt, callbacks) compute_hydrostatic_free_surface_tendency_contributions!(model, kernel_parameters; active_cells_map = active_interior_map(model.grid)) - complete_communication_and_compute_boundary!(model, model.grid, model.architecture, Δt) + complete_communication_and_compute_boundary!(model, model.grid, model.architecture) # Calculate contributions to momentum and tracer tendencies from user-prescribed fluxes across the # boundaries of the domain diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index dbbd19110f..c85d2fb1c0 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -97,7 +97,7 @@ end ##### generalized_spacing_grid(grid, coord) = grid -update_vertical_spacing!(model, grid, Δt; kwargs...) = nothing +update_vertical_spacing!(model, grid; kwargs...) = nothing ##### ##### Vertical spacings for a generalized vertical coordinate system diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl index f072a91c59..21e22226cc 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl @@ -132,7 +132,11 @@ function HydrostaticFreeSurfaceModel(; grid, # throw(ArgumentError("Generalized vertical coordinates are supported only for the vector-invariant form of the momentum equations")) # end - grid = !isnothing(free_surface) ? generalized_spacing_grid(grid, generalized_vertical_coordinate) : grid + if !(free_surface isa SplitExplicitFreeSurface) && !isnothing(generalized_vertical_coordinate) + @warn "Generalized vertical coordinates are supported only for the SplitExplicitFreeSurface at the moment. Ignoring the generalized_vertical_coordinate argument." + end + + grid = free_surface isa SplitExplicitFreeSurface ? generalized_spacing_grid(grid, generalized_vertical_coordinate) : grid arch = architecture(grid) @@ -211,7 +215,7 @@ function HydrostaticFreeSurfaceModel(; grid, free_surface, forcing, closure, particles, biogeochemistry, velocities, tracers, pressure, diffusivity_fields, timestepper, auxiliary_fields) - update_state!(model, 1.0) + update_state!(model) return model end diff --git a/src/Models/HydrostaticFreeSurfaceModels/prescribed_hydrostatic_velocity_fields.jl b/src/Models/HydrostaticFreeSurfaceModels/prescribed_hydrostatic_velocity_fields.jl index e67ec52f10..f5822af607 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/prescribed_hydrostatic_velocity_fields.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/prescribed_hydrostatic_velocity_fields.jl @@ -127,8 +127,8 @@ function time_step!(model::OnlyParticleTrackingModel, Δt; callbacks = [], kwarg tick!(model.clock, Δt) model.clock.last_Δt = Δt step_lagrangian_particles!(model, Δt) - update_state!(model, Δt, callbacks) + update_state!(model, callbacks) end -update_state!(model::OnlyParticleTrackingModel, Δt, callbacks) = +update_state!(model::OnlyParticleTrackingModel, callbacks) = [callback(model) for callback in callbacks if callback.callsite isa UpdateStateCallsite] diff --git a/src/Models/HydrostaticFreeSurfaceModels/set_hydrostatic_free_surface_model.jl b/src/Models/HydrostaticFreeSurfaceModels/set_hydrostatic_free_surface_model.jl index 744704a794..0238f876cb 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/set_hydrostatic_free_surface_model.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/set_hydrostatic_free_surface_model.jl @@ -61,7 +61,7 @@ model.velocities.u end initialize!(model) - update_state!(model, 1.0) + update_state!(model) return nothing end diff --git a/src/Models/HydrostaticFreeSurfaceModels/single_column_model_mode.jl b/src/Models/HydrostaticFreeSurfaceModels/single_column_model_mode.jl index 1fe5b9c0ea..8429887ea7 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/single_column_model_mode.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/single_column_model_mode.jl @@ -61,7 +61,7 @@ compute_free_surface_tendency!(::SingleColumnGrid, ::SplitExplicitFreeSurfaceHFS # Fast state update and halo filling -function update_state!(model::HydrostaticFreeSurfaceModel, grid::SingleColumnGrid, Δt, callbacks) +function update_state!(model::HydrostaticFreeSurfaceModel, grid::SingleColumnGrid, callbacks) fill_halo_regions!(prognostic_fields(model), model.clock, fields(model)) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 7b2f505ee2..5bcef4a993 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -490,18 +490,4 @@ setup_split_explicit_tendency!(auxiliary, grid, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) launch!(architecture(grid), grid, :xy, _compute_integrated_ab2_tendencies!, auxiliary.Gᵁ, auxiliary.Gⱽ, grid, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ; active_cells_map = active_surface_map(grid)) -wait_free_surface_communication!(free_surface, arch) = nothing - -# Special update ∂t_∂s for SplitExplicitFreeSurface where -# ∂(η / H)/∂t = - ∇ ⋅ U̅ / H -update_∂t_∂s!(∂t_∂s, parameters, grid, sⁿ, s⁻, Δt, fs::SplitExplicitFreeSurface) = - launch!(architecture(grid), grid, parameters, _update_∂t_∂s_split_explicit!, ∂t_∂s, fs.state.U̅, fs.state.V̅, fs.auxiliary.Hᶜᶜ, grid) - -@kernel function _update_∂t_∂s_split_explicit!(∂t_∂s, U̅, V̅, Hᶜᶜ, grid) - i, j = @index(Global, NTuple) - k_top = grid.Nz + 1 - @inbounds begin - # ∂(η / H)/∂t = - ∇ ⋅ ∫udz / H - ∂t_∂s[i, j, 1] = - div_xyᶜᶜᶜ(i, j, k_top - 1, grid, U̅, V̅) / Hᶜᶜ[i, j, 1] - end -end \ No newline at end of file +wait_free_surface_communication!(free_surface, arch) = nothing \ No newline at end of file diff --git a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl index 91e7f3c18d..4c37550d72 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl @@ -19,16 +19,16 @@ compute_auxiliary_fields!(auxiliary_fields) = Tuple(compute!(a) for a in auxilia # single column models. """ - update_state!(model::HydrostaticFreeSurfaceModel, Δt, callbacks=[]) + update_state!(model::HydrostaticFreeSurfaceModel, callbacks=[]) Update peripheral aspects of the model (auxiliary fields, halo regions, diffusivities, hydrostatic pressure) to the current model state. If `callbacks` are provided (in an array), they are called in the end. """ -update_state!(model::HydrostaticFreeSurfaceModel, Δt, callbacks=[]; compute_tendencies = true) = - update_state!(model, model.grid, Δt, callbacks; compute_tendencies) +update_state!(model::HydrostaticFreeSurfaceModel, callbacks=[]; compute_tendencies = true) = + update_state!(model, model.grid, callbacks; compute_tendencies) -function update_state!(model::HydrostaticFreeSurfaceModel, grid, Δt, callbacks; compute_tendencies = true) +function update_state!(model::HydrostaticFreeSurfaceModel, grid, callbacks; compute_tendencies = true) @apply_regionally mask_immersed_model_fields!(model, grid) @@ -40,7 +40,7 @@ function update_state!(model::HydrostaticFreeSurfaceModel, grid, Δt, callbacks; fill_halo_regions!(prognostic_fields(model), model.clock, fields(model); async = true) @apply_regionally replace_horizontal_vector_halos!(model.velocities, model.grid) - @apply_regionally compute_auxiliaries!(model, Δt) + @apply_regionally compute_auxiliaries!(model) fill_halo_regions!(model.diffusivity_fields; only_local_halos = true) @@ -49,7 +49,7 @@ function update_state!(model::HydrostaticFreeSurfaceModel, grid, Δt, callbacks; update_biogeochemical_state!(model.biogeochemistry, model) compute_tendencies && - @apply_regionally compute_tendencies!(model, Δt, callbacks) + @apply_regionally compute_tendencies!(model, callbacks) return nothing end @@ -69,9 +69,9 @@ function mask_immersed_model_fields!(model, grid) return nothing end -function compute_auxiliaries!(model::HydrostaticFreeSurfaceModel, Δt; w_parameters = tuple(w_kernel_parameters(model.grid)), - p_parameters = tuple(p_kernel_parameters(model.grid)), - κ_parameters = tuple(:xyz)) +function compute_auxiliaries!(model::HydrostaticFreeSurfaceModel; w_parameters = tuple(w_kernel_parameters(model.grid)), + p_parameters = tuple(p_kernel_parameters(model.grid)), + κ_parameters = tuple(:xyz)) grid = model.grid closure = model.closure @@ -80,7 +80,7 @@ function compute_auxiliaries!(model::HydrostaticFreeSurfaceModel, Δt; w_paramet for (wpar, ppar, κpar) in zip(w_parameters, p_parameters, κ_parameters) # Update the grid - update_vertical_spacing!(model, grid, Δt; parameters = wpar) + update_vertical_spacing!(model, grid; parameters = wpar) unscale_tracers!(tracers, grid; parameters = wpar) # Update the other auxiliary terms diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 321454642b..985ecec820 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -110,7 +110,7 @@ reference_Δzᵃᵃᶜ(grid::ZStarSpacingGrid) = grid.Δzᵃᵃᶜ.Δr ##### ZStar-specific vertical spacings update ##### -function update_vertical_spacing!(model, grid::ZStarSpacingGrid, Δt; parameters = :xy) +function update_vertical_spacing!(model, grid::ZStarSpacingGrid; parameters = :xy) # Scaling s⁻ = grid.Δzᵃᵃᶠ.s⁻ @@ -118,19 +118,32 @@ function update_vertical_spacing!(model, grid::ZStarSpacingGrid, Δt; parameters ∂t_∂s = grid.Δzᵃᵃᶠ.∂t_∂s Hᶜᶜ = model.free_surface.auxiliary.Hᶜᶜ + U̅ = model.free_surface.state.U̅, + V̅ = model.free_surface.state.V̅ # Update vertical spacing with available parameters # No need to fill the halo as the scaling is updated _IN_ the halos - launch!(architecture(grid), grid, parameters, _update_zstar!, sⁿ, s⁻, - model.free_surface.η, Hᶜᶜ, grid, Val(grid.Nz)) + launch!(architecture(grid), grid, parameters, _update_zstar!, + sⁿ, s⁻, model.free_surface.η, Hᶜᶜ, grid) # Update scaling time derivative - update_∂t_∂s!(∂t_∂s, parameters, grid, sⁿ, s⁻, Δt, model.free_surface) + launch!(architecture(grid), grid, parameters, _update_∂t_∂s!, + ∂t_∂s, U̅, V̅, Hᶜᶜ, grid) return nothing end -@kernel function _update_zstar!(sⁿ, s⁻, η, Hᶜᶜ, grid, ::Val{Nz}) where Nz +# NOTE: The ZStar vertical spacing works only for a SplitExplicitFreeSurface +@kernel function _update_∂t_∂s!(∂t_∂s, U̅, V̅, Hᶜᶜ, grid) + i, j = @index(Global, NTuple) + k_top = grid.Nz + 1 + @inbounds begin + # ∂(η / H)/∂t = - ∇ ⋅ ∫udz / H + ∂t_∂s[i, j, 1] = - div_xyᶜᶜᶜ(i, j, k_top - 1, grid, U̅, V̅) / Hᶜᶜ[i, j, 1] + end +end + +@kernel function _update_zstar!(sⁿ, s⁻, η, Hᶜᶜ, grid) i, j = @index(Global, NTuple) @inbounds begin bottom = Hᶜᶜ[i, j, 1] diff --git a/src/Models/NonhydrostaticModels/compute_nonhydrostatic_boundary_tendencies.jl b/src/Models/NonhydrostaticModels/compute_nonhydrostatic_boundary_tendencies.jl index f410ae2273..933f732d8d 100644 --- a/src/Models/NonhydrostaticModels/compute_nonhydrostatic_boundary_tendencies.jl +++ b/src/Models/NonhydrostaticModels/compute_nonhydrostatic_boundary_tendencies.jl @@ -7,7 +7,7 @@ using Oceananigans.Grids: XFlatGrid, YFlatGrid # Rewriting it may be helpful. # We assume here that top/bottom BC are always synched (no partitioning in z) -function compute_boundary_tendencies!(model::NonhydrostaticModel, Δt) +function compute_boundary_tendencies!(model::NonhydrostaticModel) grid = model.grid arch = architecture(grid) diff --git a/src/Models/NonhydrostaticModels/compute_nonhydrostatic_tendencies.jl b/src/Models/NonhydrostaticModels/compute_nonhydrostatic_tendencies.jl index 981b06fa7d..bf840e07b6 100644 --- a/src/Models/NonhydrostaticModels/compute_nonhydrostatic_tendencies.jl +++ b/src/Models/NonhydrostaticModels/compute_nonhydrostatic_tendencies.jl @@ -14,7 +14,7 @@ import Oceananigans.TimeSteppers: compute_tendencies! Calculate the interior and boundary contributions to tendency terms without the contribution from non-hydrostatic pressure. """ -function compute_tendencies!(model::NonhydrostaticModel, Δt, callbacks) +function compute_tendencies!(model::NonhydrostaticModel, callbacks) # Note: # @@ -29,7 +29,7 @@ function compute_tendencies!(model::NonhydrostaticModel, Δt, callbacks) kernel_parameters = tuple(interior_tendency_kernel_parameters(model.grid)) compute_interior_tendency_contributions!(model, kernel_parameters; active_cells_map = active_interior_map(model.grid)) - complete_communication_and_compute_boundary!(model, model.grid, model.architecture, Δt) + complete_communication_and_compute_boundary!(model, model.grid, model.architecture) # Calculate contributions to momentum and tracer tendencies from user-prescribed fluxes across the # boundaries of the domain diff --git a/src/Models/NonhydrostaticModels/nonhydrostatic_model.jl b/src/Models/NonhydrostaticModels/nonhydrostatic_model.jl index 6e407b9c7b..bfdadaf250 100644 --- a/src/Models/NonhydrostaticModels/nonhydrostatic_model.jl +++ b/src/Models/NonhydrostaticModels/nonhydrostatic_model.jl @@ -230,7 +230,7 @@ function NonhydrostaticModel(; grid, pressures, diffusivity_fields, timestepper, pressure_solver, immersed_boundary, auxiliary_fields) - update_state!(model, 0) + update_state!(model) return model end diff --git a/src/Models/NonhydrostaticModels/set_nonhydrostatic_model.jl b/src/Models/NonhydrostaticModels/set_nonhydrostatic_model.jl index 0a884503d8..e8011c9058 100644 --- a/src/Models/NonhydrostaticModels/set_nonhydrostatic_model.jl +++ b/src/Models/NonhydrostaticModels/set_nonhydrostatic_model.jl @@ -47,13 +47,13 @@ function set!(model::NonhydrostaticModel; enforce_incompressibility=true, kwargs # Apply a mask foreach(mask_immersed_field!, model.tracers) foreach(mask_immersed_field!, model.velocities) - update_state!(model, 1.0) + update_state!(model) if enforce_incompressibility FT = eltype(model.grid) calculate_pressure_correction!(model, one(FT)) pressure_correct_velocities!(model, one(FT)) - update_state!(model, 1.0) + update_state!(model) end return nothing diff --git a/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl b/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl index 89232be91e..2dacdecf29 100644 --- a/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl +++ b/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl @@ -17,7 +17,7 @@ Update peripheral aspects of the model (halo regions, diffusivities, hydrostatic pressure) to the current model state. If `callbacks` are provided (in an array), they are called in the end. """ -function update_state!(model::NonhydrostaticModel, Δt=nothing, callbacks=[]; compute_tendencies = true) +function update_state!(model::NonhydrostaticModel, callbacks=[]; compute_tendencies = true) # Mask immersed tracers foreach(model.tracers) do tracer @@ -50,7 +50,7 @@ function update_state!(model::NonhydrostaticModel, Δt=nothing, callbacks=[]; co update_biogeochemical_state!(model.biogeochemistry, model) compute_tendencies && - @apply_regionally compute_tendencies!(model, Δt, callbacks) + @apply_regionally compute_tendencies!(model, callbacks) return nothing end diff --git a/src/Models/ShallowWaterModels/set_shallow_water_model.jl b/src/Models/ShallowWaterModels/set_shallow_water_model.jl index ce2159bebf..b68f279a04 100644 --- a/src/Models/ShallowWaterModels/set_shallow_water_model.jl +++ b/src/Models/ShallowWaterModels/set_shallow_water_model.jl @@ -14,7 +14,7 @@ function set!(model::ShallowWaterModel; kwargs...) set!(ϕ, value) end - update_state!(model, 1.0) + update_state!(model) return nothing end diff --git a/src/Models/ShallowWaterModels/shallow_water_model.jl b/src/Models/ShallowWaterModels/shallow_water_model.jl index 6184218356..9c5778ac14 100644 --- a/src/Models/ShallowWaterModels/shallow_water_model.jl +++ b/src/Models/ShallowWaterModels/shallow_water_model.jl @@ -205,7 +205,7 @@ function ShallowWaterModel(; timestepper, formulation) - update_state!(model, 1) + update_state!(model) return model end diff --git a/src/Models/ShallowWaterModels/update_shallow_water_state.jl b/src/Models/ShallowWaterModels/update_shallow_water_state.jl index 8956a397b0..bb4fb8d06d 100644 --- a/src/Models/ShallowWaterModels/update_shallow_water_state.jl +++ b/src/Models/ShallowWaterModels/update_shallow_water_state.jl @@ -18,7 +18,7 @@ Next, `callbacks` are executed. Finally, tendencies are computed if `compute_tendencies=true`. """ -function update_state!(model::ShallowWaterModel, Δt, callbacks=[]; compute_tendencies = true) +function update_state!(model::ShallowWaterModel, callbacks=[]; compute_tendencies = true) # Mask immersed fields foreach(mask_immersed_field!, merge(model.solution, model.tracers)) diff --git a/src/Models/interleave_communication_and_computation.jl b/src/Models/interleave_communication_and_computation.jl index cdd6d760a0..286097e826 100644 --- a/src/Models/interleave_communication_and_computation.jl +++ b/src/Models/interleave_communication_and_computation.jl @@ -6,7 +6,7 @@ using Oceananigans.DistributedComputations using Oceananigans.DistributedComputations: DistributedGrid using Oceananigans.DistributedComputations: synchronize_communication!, SynchronizedDistributed -function complete_communication_and_compute_boundary!(model, ::DistributedGrid, arch, Δt) +function complete_communication_and_compute_boundary!(model, ::DistributedGrid, arch) # Iterate over the fields to clear _ALL_ possible architectures for field in prognostic_fields(model) @@ -14,16 +14,16 @@ function complete_communication_and_compute_boundary!(model, ::DistributedGrid, end # Recompute tendencies near the boundary halos - compute_boundary_tendencies!(model, Δt) + compute_boundary_tendencies!(model) return nothing end # Fallback -complete_communication_and_compute_boundary!(model, ::DistributedGrid, ::SynchronizedDistributed, Δt) = nothing -complete_communication_and_compute_boundary!(model, grid, arch, Δt) = nothing +complete_communication_and_compute_boundary!(model, ::DistributedGrid, ::SynchronizedDistributed) = nothing +complete_communication_and_compute_boundary!(model, grid, arch) = nothing -compute_boundary_tendencies!(model, Δt) = nothing +compute_boundary_tendencies!(model) = nothing """ Kernel parameters for computing interior tendencies. """ interior_tendency_kernel_parameters(grid) = :xyz # fallback diff --git a/src/Simulations/run.jl b/src/Simulations/run.jl index 8d5920729e..7d2d7a2aba 100644 --- a/src/Simulations/run.jl +++ b/src/Simulations/run.jl @@ -188,7 +188,7 @@ function initialize!(sim::Simulation) model = sim.model clock = model.clock - update_state!(model, 1.0) + update_state!(model) # Output and diagnostics initialization [add_dependencies!(sim.diagnostics, writer) for writer in values(sim.output_writers)] diff --git a/src/TimeSteppers/runge_kutta_3.jl b/src/TimeSteppers/runge_kutta_3.jl index 5cc1c5f95b..c54023e65a 100644 --- a/src/TimeSteppers/runge_kutta_3.jl +++ b/src/TimeSteppers/runge_kutta_3.jl @@ -82,7 +82,7 @@ function time_step!(model::AbstractModel{<:RungeKutta3TimeStepper}, Δt; callbac Δt == 0 && @warn "Δt == 0 may cause model blowup!" # Be paranoid and update state at iteration 0, in case run! is not used: - model.clock.iteration == 0 && update_state!(model, Δt, callbacks) + model.clock.iteration == 0 && update_state!(model, callbacks) γ¹ = model.timestepper.γ¹ γ² = model.timestepper.γ² @@ -111,7 +111,7 @@ function time_step!(model::AbstractModel{<:RungeKutta3TimeStepper}, Δt; callbac pressure_correct_velocities!(model, first_stage_Δt) store_tendencies!(model) - update_state!(model, Δt, callbacks) + update_state!(model, callbacks) step_lagrangian_particles!(model, first_stage_Δt) # @@ -127,7 +127,7 @@ function time_step!(model::AbstractModel{<:RungeKutta3TimeStepper}, Δt; callbac pressure_correct_velocities!(model, second_stage_Δt) store_tendencies!(model) - update_state!(model, Δt, callbacks) + update_state!(model, callbacks) step_lagrangian_particles!(model, second_stage_Δt) # From 4bd8c49cb8aa50c65e37001bdae5f3e349c7ad74 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 27 Aug 2024 10:54:40 -0400 Subject: [PATCH 281/567] removing ffmpeg --- Manifest.toml | 235 +------------------------------------------------- Project.toml | 5 +- 2 files changed, 3 insertions(+), 237 deletions(-) diff --git a/Manifest.toml b/Manifest.toml index 12c680a31b..e918cfa7c5 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.10.4" manifest_format = "2.0" -project_hash = "e1eb163664f6ca89a515c31f724661dff50fd8df" +project_hash = "9815a4091be3641fb9635ba2f3346c2241d20bcf" [[deps.AbstractFFTs]] deps = ["LinearAlgebra"] @@ -107,12 +107,6 @@ git-tree-sha1 = "afea94249b821dc754a8ca6695d3daed851e1f5a" uuid = "76a88914-d11a-5bdc-97e0-2f5a05c973a2" version = "0.14.1+0" -[[deps.Cairo_jll]] -deps = ["Artifacts", "Bzip2_jll", "CompilerSupportLibraries_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "JLLWrappers", "LZO_jll", "Libdl", "Pixman_jll", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Zlib_jll", "libpng_jll"] -git-tree-sha1 = "a2f1c8c668c8e3cb4cca4e57a8efdb09067bb3fd" -uuid = "83423d85-b0ee-5818-9007-b63ccbeb887a" -version = "1.18.0+2" - [[deps.ColorTypes]] deps = ["FixedPointNumbers", "Random"] git-tree-sha1 = "b10d0b65641d57b8b4d5e234446582de5047050d" @@ -237,29 +231,11 @@ git-tree-sha1 = "71c79e77221ab3a29918aaf6db4f217b89138608" uuid = "b305315f-e792-5b7a-8f41-49f472929428" version = "1.0.1" -[[deps.Expat_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "1c6317308b9dc757616f0b5cb379db10494443a7" -uuid = "2e619515-83b5-522b-bb60-26c02a35a201" -version = "2.6.2+0" - [[deps.ExprTools]] git-tree-sha1 = "27415f162e6028e81c72b82ef756bf321213b6ec" uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04" version = "0.1.10" -[[deps.FFMPEG]] -deps = ["FFMPEG_jll"] -git-tree-sha1 = "b57e3acbe22f8484b4b5ff66a7499717fe1a9cc8" -uuid = "c87230d0-a227-11e9-1b43-d7ebe4e7570a" -version = "0.4.1" - -[[deps.FFMPEG_jll]] -deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "JLLWrappers", "LAME_jll", "Libdl", "Ogg_jll", "OpenSSL_jll", "Opus_jll", "PCRE2_jll", "Zlib_jll", "libaom_jll", "libass_jll", "libfdk_aac_jll", "libvorbis_jll", "x264_jll", "x265_jll"] -git-tree-sha1 = "466d45dc38e15794ec7d5d63ec03d776a9aff36e" -uuid = "b22a6f82-2f65-5046-a5b2-351ab43fb4e5" -version = "4.4.4+1" - [[deps.FFTW]] deps = ["AbstractFFTs", "FFTW_jll", "LinearAlgebra", "MKL_jll", "Preferences", "Reexport"] git-tree-sha1 = "4820348781ae578893311153d69049a93d05f39d" @@ -287,24 +263,6 @@ git-tree-sha1 = "05882d6995ae5c12bb5f36dd2ed3f61c98cbb172" uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93" version = "0.8.5" -[[deps.Fontconfig_jll]] -deps = ["Artifacts", "Bzip2_jll", "Expat_jll", "FreeType2_jll", "JLLWrappers", "Libdl", "Libuuid_jll", "Zlib_jll"] -git-tree-sha1 = "db16beca600632c95fc8aca29890d83788dd8b23" -uuid = "a3f928ae-7b40-5064-980b-68af3947d34b" -version = "2.13.96+0" - -[[deps.FreeType2_jll]] -deps = ["Artifacts", "Bzip2_jll", "JLLWrappers", "Libdl", "Zlib_jll"] -git-tree-sha1 = "5c1d8ae0efc6c2e7b1fc502cbe25def8f661b7bc" -uuid = "d7e528f0-a631-5988-bf34-fe36492bcfd7" -version = "2.13.2+0" - -[[deps.FriBidi_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "1ed150b39aebcc805c26b93a8d0122c940f64ce2" -uuid = "559328eb-81f9-559d-9380-de523a88c83c" -version = "1.0.14+0" - [[deps.Future]] deps = ["Random"] uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" @@ -332,18 +290,6 @@ git-tree-sha1 = "ab29216184312f99ff957b32cd63c2fe9c928b91" uuid = "61eb1bfa-7361-4325-ad38-22787b887f55" version = "0.26.7" -[[deps.Gettext_jll]] -deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Libiconv_jll", "Pkg", "XML2_jll"] -git-tree-sha1 = "9b02998aba7bf074d14de89f9d37ca24a1a0b046" -uuid = "78b55507-aeef-58d4-861c-77aaff3498b1" -version = "0.21.0+0" - -[[deps.Glib_jll]] -deps = ["Artifacts", "Gettext_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Libiconv_jll", "Libmount_jll", "PCRE2_jll", "Zlib_jll"] -git-tree-sha1 = "7c82e6a6cd34e9d935e9aa4051b66c6ff3af59ba" -uuid = "7746bdde-850d-59dc-9ae8-88ece973131d" -version = "2.80.2+0" - [[deps.Glob]] git-tree-sha1 = "97285bbd5230dd766e9ef6749b80fc617126d496" uuid = "c27321d9-0574-5035-807b-f59d2c89b15c" @@ -355,24 +301,12 @@ git-tree-sha1 = "383db7d3f900f4c1f47a8a04115b053c095e48d3" uuid = "0951126a-58fd-58f1-b5b3-b08c7c4a876d" version = "3.8.4+0" -[[deps.Graphite2_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "344bf40dcab1073aca04aa0df4fb092f920e4011" -uuid = "3b182d85-2403-5c21-9c21-1e1f0cc25472" -version = "1.3.14+0" - [[deps.HDF5_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "LazyArtifacts", "LibCURL_jll", "Libdl", "MPICH_jll", "MPIPreferences", "MPItrampoline_jll", "MicrosoftMPI_jll", "OpenMPI_jll", "OpenSSL_jll", "TOML", "Zlib_jll", "libaec_jll"] git-tree-sha1 = "82a471768b513dc39e471540fdadc84ff80ff997" uuid = "0234f1f7-429e-5d53-9886-15a909be8d59" version = "1.14.3+3" -[[deps.HarfBuzz_jll]] -deps = ["Artifacts", "Cairo_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "Graphite2_jll", "JLLWrappers", "Libdl", "Libffi_jll"] -git-tree-sha1 = "401e4f3f30f43af2c8478fc008da50096ea5240f" -uuid = "2e76f6c2-a576-52d4-95c1-20adfe4de566" -version = "8.3.1+0" - [[deps.Hwloc_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] git-tree-sha1 = "5e19e1e4fa3e71b774ce746274364aef0234634e" @@ -463,12 +397,6 @@ version = "0.9.25" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" -[[deps.LAME_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "170b660facf5df5de098d866564877e119141cbd" -uuid = "c1c5ebd0-6772-5130-a774-d5fcae4a789d" -version = "3.100.2+0" - [[deps.LLVM]] deps = ["CEnum", "LLVMExtra_jll", "Libdl", "Preferences", "Printf", "Requires", "Unicode"] git-tree-sha1 = "2470e69781ddd70b8878491233cd09bc1bd7fc96" @@ -490,12 +418,6 @@ git-tree-sha1 = "2e5c102cfc41f48ae4740c7eca7743cc7e7b75ea" uuid = "8b046642-f1f6-4319-8d3c-209ddc03c586" version = "1.0.0" -[[deps.LLVMOpenMP_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "e16271d212accd09d52ee0ae98956b8a05c4b626" -uuid = "1d63c593-3942-5779-bab2-d838dc0a180e" -version = "17.0.6+0" - [[deps.LRUCache]] git-tree-sha1 = "b3cc6698599b10e652832c2f23db3cab99d51b59" uuid = "8ac3fa9e-de4c-5943-b1dc-09c6b5f20637" @@ -505,12 +427,6 @@ weakdeps = ["Serialization"] [deps.LRUCache.extensions] SerializationExt = ["Serialization"] -[[deps.LZO_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "70c5da094887fd2cae843b8db33920bac4b6f07d" -uuid = "dd4b983a-f0e5-5f8d-a1b7-129d4a5fb1ac" -version = "2.10.2+0" - [[deps.LaTeXStrings]] git-tree-sha1 = "50901ebc375ed41dbf8058da26f9de442febbbec" uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" @@ -547,42 +463,12 @@ version = "1.11.0+1" [[deps.Libdl]] uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" -[[deps.Libffi_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "0b4a5d71f3e5200a7dff793393e09dfc2d874290" -uuid = "e9f186c6-92d2-5b65-8a66-fee21dc1b490" -version = "3.2.2+1" - -[[deps.Libgcrypt_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgpg_error_jll"] -git-tree-sha1 = "9fd170c4bbfd8b935fdc5f8b7aa33532c991a673" -uuid = "d4300ac3-e22c-5743-9152-c294e39db1e4" -version = "1.8.11+0" - -[[deps.Libgpg_error_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "fbb1f2bef882392312feb1ede3615ddc1e9b99ed" -uuid = "7add5ba3-2f88-524e-9cd5-f83b8a55f7b8" -version = "1.49.0+0" - [[deps.Libiconv_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] git-tree-sha1 = "f9557a255370125b405568f9767d6d195822a175" uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531" version = "1.17.0+0" -[[deps.Libmount_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "0c4f9c4f1a50d8f35048fa0532dabbadf702f81e" -uuid = "4b2f31a3-9ecc-558c-b454-b3730dcb73e9" -version = "2.40.1+0" - -[[deps.Libuuid_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "5ee6203157c120d79034c748a2acba45b82b8807" -uuid = "38a345b3-de98-5d2b-a5d3-14cd9215e700" -version = "2.40.1+0" - [[deps.LinearAlgebra]] deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" @@ -727,12 +613,6 @@ weakdeps = ["Adapt"] [deps.OffsetArrays.extensions] OffsetArraysAdaptExt = "Adapt" -[[deps.Ogg_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "887579a3eb005446d514ab7aeac5d1d027658b8f" -uuid = "e7412a2a-1a6e-54c0-be00-318e2571c051" -version = "1.3.5+1" - [[deps.OpenBLAS_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" @@ -761,12 +641,6 @@ git-tree-sha1 = "13652491f6856acfd2db29360e1bbcd4565d04f1" uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e" version = "0.5.5+0" -[[deps.Opus_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "6703a85cb3781bd5909d48730a67205f3f31a575" -uuid = "91d4177d-7536-5919-b921-800302f37372" -version = "1.3.3+0" - [[deps.OrderedCollections]] git-tree-sha1 = "dfdf5519f235516220579f949664f1bf44e741c5" uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" @@ -778,17 +652,6 @@ git-tree-sha1 = "2cd396108e178f3ae8dedbd8e938a18726ab2fbf" uuid = "c2071276-7c44-58a7-b746-946036e04d0a" version = "0.24.1+0" -[[deps.PCRE2_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "efcefdf7-47ab-520b-bdef-62a2eaa19f15" -version = "10.42.0+1" - -[[deps.Pixman_jll]] -deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "LLVMOpenMP_jll", "Libdl"] -git-tree-sha1 = "35621f10a7531bc8fa58f74610b1bfb70a3cfc6b" -uuid = "30392449-352a-5448-841d-b1acce4e97dc" -version = "0.43.4+0" - [[deps.Pkg]] deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" @@ -1070,66 +933,12 @@ git-tree-sha1 = "1165b0443d0eca63ac1e32b8c0eb69ed2f4f8127" uuid = "02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a" version = "2.13.3+0" -[[deps.XSLT_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgcrypt_jll", "Libgpg_error_jll", "Libiconv_jll", "XML2_jll", "Zlib_jll"] -git-tree-sha1 = "a54ee957f4c86b526460a720dbc882fa5edcbefc" -uuid = "aed1982a-8fda-507f-9586-7b0439959a61" -version = "1.1.41+0" - [[deps.XZ_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] git-tree-sha1 = "ac88fb95ae6447c8dda6a5503f3bafd496ae8632" uuid = "ffd25f8a-64ca-5728-b0f7-c24cf3aae800" version = "5.4.6+0" -[[deps.Xorg_libX11_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libxcb_jll", "Xorg_xtrans_jll"] -git-tree-sha1 = "afead5aba5aa507ad5a3bf01f58f82c8d1403495" -uuid = "4f6342f7-b3d2-589e-9d20-edeb45f2b2bc" -version = "1.8.6+0" - -[[deps.Xorg_libXau_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "6035850dcc70518ca32f012e46015b9beeda49d8" -uuid = "0c0b7dd1-d40b-584c-a123-a41640f87eec" -version = "1.0.11+0" - -[[deps.Xorg_libXdmcp_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "34d526d318358a859d7de23da945578e8e8727b7" -uuid = "a3789734-cfe1-5b06-b2d0-1dd0d9d62d05" -version = "1.1.4+0" - -[[deps.Xorg_libXext_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libX11_jll"] -git-tree-sha1 = "d2d1a5c49fae4ba39983f63de6afcbea47194e85" -uuid = "1082639a-0dae-5f34-9b06-72781eeb8cb3" -version = "1.3.6+0" - -[[deps.Xorg_libXrender_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libX11_jll"] -git-tree-sha1 = "47e45cd78224c53109495b3e324df0c37bb61fbe" -uuid = "ea2f1a96-1ddc-540d-b46f-429655e07cfa" -version = "0.9.11+0" - -[[deps.Xorg_libpthread_stubs_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "8fdda4c692503d44d04a0603d9ac0982054635f9" -uuid = "14d82f49-176c-5ed1-bb49-ad3f5cbd8c74" -version = "0.1.1+0" - -[[deps.Xorg_libxcb_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "XSLT_jll", "Xorg_libXau_jll", "Xorg_libXdmcp_jll", "Xorg_libpthread_stubs_jll"] -git-tree-sha1 = "bcd466676fef0878338c61e655629fa7bbc69d8e" -uuid = "c7cfdc94-dc32-55de-ac96-5a1b8d977c5b" -version = "1.17.0+0" - -[[deps.Xorg_xtrans_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "e92a1a012a10506618f10b7047e478403a046c77" -uuid = "c5fb5394-a638-5e4d-96e5-b29de1b5cf10" -version = "1.5.0+0" - [[deps.Zlib_jll]] deps = ["Libdl"] uuid = "83775a58-1f1d-513f-b197-d71354ab007a" @@ -1147,41 +956,11 @@ git-tree-sha1 = "46bf7be2917b59b761247be3f317ddf75e50e997" uuid = "477f73a3-ac25-53e9-8cc3-50b2fa2566f0" version = "1.1.2+0" -[[deps.libaom_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "1827acba325fdcdf1d2647fc8d5301dd9ba43a9d" -uuid = "a4ae2306-e953-59d6-aa16-d00cac43593b" -version = "3.9.0+0" - -[[deps.libass_jll]] -deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "HarfBuzz_jll", "JLLWrappers", "Libdl", "Zlib_jll"] -git-tree-sha1 = "e17c115d55c5fbb7e52ebedb427a0dca79d4484e" -uuid = "0ac62f75-1d6f-5e53-bd7c-93b484bb37c0" -version = "0.15.2+0" - [[deps.libblastrampoline_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" version = "5.8.0+1" -[[deps.libfdk_aac_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "8a22cf860a7d27e4f3498a0fe0811a7957badb38" -uuid = "f638f0a6-7fb0-5443-88ba-1cc74229b280" -version = "2.0.3+0" - -[[deps.libpng_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Zlib_jll"] -git-tree-sha1 = "d7015d2e18a5fd9a4f47de711837e980519781a4" -uuid = "b53b4c65-9356-5827-b1ea-8c7a1a84506f" -version = "1.6.43+1" - -[[deps.libvorbis_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Ogg_jll", "Pkg"] -git-tree-sha1 = "490376214c4721cdaca654041f635213c6165cb3" -uuid = "f27f6e37-5d2b-51aa-960f-b287f2bc3b7a" -version = "1.3.7+2" - [[deps.libzip_jll]] deps = ["Artifacts", "Bzip2_jll", "GnuTLS_jll", "JLLWrappers", "Libdl", "XZ_jll", "Zlib_jll", "Zstd_jll"] git-tree-sha1 = "3282b7d16ae7ac3e57ec2f3fa8fafb564d8f9f7f" @@ -1203,15 +982,3 @@ version = "2021.12.0+0" deps = ["Artifacts", "Libdl"] uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" version = "17.4.0+2" - -[[deps.x264_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "4fea590b89e6ec504593146bf8b988b2c00922b2" -uuid = "1270edf5-f2f9-52d2-97e9-ab00b5d0237a" -version = "2021.5.5+0" - -[[deps.x265_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "ee567a171cce03570d77ad3a43e90218e38937a9" -uuid = "dfaa095f-4041-5dcd-9319-2fabd8486b76" -version = "3.5.0+0" diff --git a/Project.toml b/Project.toml index 26ac58cc51..612a9ff1f8 100644 --- a/Project.toml +++ b/Project.toml @@ -11,7 +11,6 @@ CubedSphere = "7445602f-e544-4518-8976-18f8e8ae6cdb" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Distances = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7" DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" -FFMPEG = "c87230d0-a227-11e9-1b43-d7ebe4e7570a" FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" Glob = "c27321d9-0574-5035-807b-f59d2c89b15c" IncompleteLU = "40713840-3770-5561-ab4c-a76e7d0d7895" @@ -36,8 +35,8 @@ StructArrays = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" [weakdeps] Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" -MakieCore = "20f20a25-4f0e-4fdf-b5d1-57303727442b" Makie = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a" +MakieCore = "20f20a25-4f0e-4fdf-b5d1-57303727442b" [extensions] OceananigansEnzymeExt = "Enzyme" @@ -62,8 +61,8 @@ KernelAbstractions = "0.9.21" LinearAlgebra = "1.9" Logging = "1.9" MPI = "0.16, 0.17, 0.18, 0.19, 0.20" -MakieCore = "0.7, 0.8" Makie = "0.21" +MakieCore = "0.7, 0.8" NCDatasets = "0.12.10, 0.13.1, 0.14" OffsetArrays = "1.4" OrderedCollections = "1.1" From 1bdd09eabe943cfa943c34e0644a2a3394d60a42 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 27 Aug 2024 10:56:37 -0400 Subject: [PATCH 282/567] typo in the description --- .../hydrostatic_free_surface_model.jl | 2 +- .../HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl index 21e22226cc..15a42769ae 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl @@ -137,7 +137,7 @@ function HydrostaticFreeSurfaceModel(; grid, end grid = free_surface isa SplitExplicitFreeSurface ? generalized_spacing_grid(grid, generalized_vertical_coordinate) : grid - + arch = architecture(grid) @apply_regionally momentum_advection = validate_momentum_advection(momentum_advection, grid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 985ecec820..6508db4e28 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -12,7 +12,7 @@ struct ZStar end A vertical spacing for the hydrostatic free surface model that follows the free surface. The vertical spacing is defined by a reference spacing `Δr` and a scaling `s` that obeys ```math -s = (η + H) / η +s = (η + H) / H ``` where ``η`` is the free surface height and ``H`` the vertical depth of the water column From 7c961eddf0ebfeeab8aead19dbac3570e4f80eb0 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 27 Aug 2024 10:59:29 -0400 Subject: [PATCH 283/567] bugfix --- .../HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 6508db4e28..b350a365b6 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -118,7 +118,7 @@ function update_vertical_spacing!(model, grid::ZStarSpacingGrid; parameters = :x ∂t_∂s = grid.Δzᵃᵃᶠ.∂t_∂s Hᶜᶜ = model.free_surface.auxiliary.Hᶜᶜ - U̅ = model.free_surface.state.U̅, + U̅ = model.free_surface.state.U̅ V̅ = model.free_surface.state.V̅ # Update vertical spacing with available parameters From 2e35052b198a73ef792afa2243b56635780512e1 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 27 Aug 2024 11:07:57 -0400 Subject: [PATCH 284/567] bugfix --- .../z_star_vertical_spacing.jl | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index b350a365b6..739c018052 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -113,20 +113,25 @@ reference_Δzᵃᵃᶜ(grid::ZStarSpacingGrid) = grid.Δzᵃᵃᶜ.Δr function update_vertical_spacing!(model, grid::ZStarSpacingGrid; parameters = :xy) # Scaling - s⁻ = grid.Δzᵃᵃᶠ.s⁻ - sⁿ = grid.Δzᵃᵃᶠ.sⁿ + s⁻ = grid.Δzᵃᵃᶠ.s⁻ + sⁿ = grid.Δzᵃᵃᶠ.sⁿ ∂t_∂s = grid.Δzᵃᵃᶠ.∂t_∂s - Hᶜᶜ = model.free_surface.auxiliary.Hᶜᶜ - U̅ = model.free_surface.state.U̅ - V̅ = model.free_surface.state.V̅ + # Free surface variables + Hᶜᶜ = model.free_surface.auxiliary.Hᶜᶜ + U̅ = model.free_surface.state.U̅ + V̅ = model.free_surface.state.V̅ + η = model.free_surface.η # Update vertical spacing with available parameters # No need to fill the halo as the scaling is updated _IN_ the halos launch!(architecture(grid), grid, parameters, _update_zstar!, - sⁿ, s⁻, model.free_surface.η, Hᶜᶜ, grid) + sⁿ, s⁻, η, Hᶜᶜ, grid) - # Update scaling time derivative + # Update the time derivative of the grid-scaling. Note that in this case we leverage the + # free surface evolution equation, where the time derivative of the free surface is equal + # to the divergence of the vertically integrated velocity field, such that + # ∂ₜ((H + η) / H) = H⁻¹ ∂ₜη = - H⁻¹ ∇ ⋅ ∫udz launch!(architecture(grid), grid, parameters, _update_∂t_∂s!, ∂t_∂s, U̅, V̅, Hᶜᶜ, grid) From f9d7aa870b26e3fff72e66c45e7b5d0ebd0a15dc Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 27 Aug 2024 12:41:17 -0400 Subject: [PATCH 285/567] fix non-distributed tests --- .../distributed_split_explicit_free_surface.jl | 2 +- .../split_explicit_free_surface.jl | 4 ++-- .../multi_region_split_explicit_free_surface.jl | 3 ++- test/test_split_explicit_vertical_integrals.jl | 14 ++++++++------ 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl index fcca388144..83d5f8a2d1 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl @@ -14,7 +14,7 @@ function SplitExplicitAuxiliaryFields(grid::DistributedGrid) Hᶜᶠ = Field{Center, Face, Nothing}(grid) Hᶜᶜ = Field{Center, Center, Nothing}(grid) - calculate_column_height!(Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, grid) + compute_column_height!(Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, grid) # In a non-parallel grid we calculate only the interior kernel_size = augmented_kernel_size(grid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index 573dcd60f0..a865b99c94 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -233,14 +233,14 @@ function SplitExplicitAuxiliaryFields(grid::AbstractGrid) Hᶜᶠ = Field{Center, Face, Nothing}(grid) Hᶜᶜ = Field{Center, Center, Nothing}(grid) - calculate_column_height!(Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, grid) + compute_column_height!(Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, grid) kernel_parameters = :xy return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, kernel_parameters) end -function calculate_column_height!(Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, grid) +function compute_column_height!(Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, grid) Nx, Ny, _ = size(grid) Hx, Hy, _ = halo_size(grid) Tx, Ty, _ = topology(grid) diff --git a/src/MultiRegion/multi_region_split_explicit_free_surface.jl b/src/MultiRegion/multi_region_split_explicit_free_surface.jl index d3d081c3ec..d2872d13e8 100644 --- a/src/MultiRegion/multi_region_split_explicit_free_surface.jl +++ b/src/MultiRegion/multi_region_split_explicit_free_surface.jl @@ -4,7 +4,8 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: SplitExplicitFreeSurface SplitExplicitSettings, SplitExplicitState, FixedSubstepNumber, FixedTimeStepSize, - calculate_substeps + calculate_substeps, + calculate_column_height! import Oceananigans.Models.HydrostaticFreeSurfaceModels: materialize_free_surface, SplitExplicitAuxiliaryFields diff --git a/test/test_split_explicit_vertical_integrals.jl b/test/test_split_explicit_vertical_integrals.jl index ca7713ad2b..c0a82b8a4d 100644 --- a/test/test_split_explicit_vertical_integrals.jl +++ b/test/test_split_explicit_vertical_integrals.jl @@ -62,7 +62,9 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_barotropic_mode! set!(u, set_u_check) exact_U = similar(U) set!(exact_U, set_U_check) - compute_barotropic_mode!(U, V, grid, u, v) + Hᶠᶜ = auxiliary.Hᶠᶜ + Hᶜᶠ = auxiliary.Hᶜᶠ + compute_barotropic_mode!(U, V, grid, u, v, Hᶠᶜ, Hᶜᶠ, η̅) tolerance = 1e-3 @test all((Array(interior(U) .- interior(exact_U))) .< tolerance) @@ -71,7 +73,7 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_barotropic_mode! set!(v, set_v_check) exact_V = similar(V) set!(exact_V, set_V_check) - compute_barotropic_mode!(U, V, grid, u, v) + compute_barotropic_mode!(U, V, grid, u, v, Hᶠᶜ, Hᶜᶠ, η̅) @test all((Array(interior(V) .- interior(exact_V))) .< tolerance) end @@ -81,12 +83,12 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_barotropic_mode! u .= 0.0 U .= 1.0 - compute_barotropic_mode!(U, V, grid, u, v) + compute_barotropic_mode!(U, V, grid, u, v, Hᶠᶜ, Hᶜᶠ, η̅) @test all(Array(U.data.parent) .== 0.0) u .= 1.0 U .= 1.0 - compute_barotropic_mode!(U, V, grid, u, v) + compute_barotropic_mode!(U, V, grid, u, v, Hᶠᶜ, Hᶜᶠ, η̅) @test all(Array(interior(U)) .≈ Lz) set_u_check(x, y, z) = sin(x) @@ -94,7 +96,7 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_barotropic_mode! set!(u, set_u_check) exact_U = similar(U) set!(exact_U, set_U_check) - compute_barotropic_mode!(U, V, grid, u, v) + compute_barotropic_mode!(U, V, grid, u, v, Hᶠᶜ, Hᶜᶠ, η̅) @test all(Array(interior(U)) .≈ Array(interior(exact_U))) set_v_check(x, y, z) = sin(x) * z * cos(y) @@ -102,7 +104,7 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_barotropic_mode! set!(v, set_v_check) exact_V = similar(V) set!(exact_V, set_V_check) - compute_barotropic_mode!(U, V, grid, u, v) + compute_barotropic_mode!(U, V, grid, u, v, Hᶠᶜ, Hᶜᶠ, η̅) @test all(Array(interior(V)) .≈ Array(interior(exact_V))) end From 18714840e82bba586ab25703902c4c2faf42c7d7 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 27 Aug 2024 13:08:58 -0400 Subject: [PATCH 286/567] bugfix --- src/MultiRegion/multi_region_split_explicit_free_surface.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/MultiRegion/multi_region_split_explicit_free_surface.jl b/src/MultiRegion/multi_region_split_explicit_free_surface.jl index d2872d13e8..c0eb342ac9 100644 --- a/src/MultiRegion/multi_region_split_explicit_free_surface.jl +++ b/src/MultiRegion/multi_region_split_explicit_free_surface.jl @@ -5,7 +5,7 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: SplitExplicitFreeSurface SplitExplicitState, FixedSubstepNumber, FixedTimeStepSize, calculate_substeps, - calculate_column_height! + compute_column_height! import Oceananigans.Models.HydrostaticFreeSurfaceModels: materialize_free_surface, SplitExplicitAuxiliaryFields @@ -18,7 +18,7 @@ function SplitExplicitAuxiliaryFields(grid::MultiRegionGrids) Hᶜᶠ = Field((Center, Face, Nothing), grid) Hᶜᶠ = Field((Center, Center, Nothing), grid) - @apply_regionally calculate_column_height!(Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, grid) + @apply_regionally compute_column_height!(Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, grid) # In a non-parallel grid we calculate only the interior @apply_regionally kernel_size = augmented_kernel_size(grid, grid.partition) From a450684d77eb9e66d8a9f6adb08dee5308a97a12 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Sun, 8 Sep 2024 12:31:26 -0400 Subject: [PATCH 287/567] new spacings --- .../z_star_vertical_spacing.jl | 57 ++++++++++++------- .../compute_nonhydrostatic_tendencies.jl | 2 +- 2 files changed, 39 insertions(+), 20 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 739c018052..286b4c731c 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -22,11 +22,13 @@ where ``η`` is the free surface height and ``H`` the vertical depth of the wate - `s⁻`:scaling of the vertical coordinate at time step `n - 1` - `∂t_∂s`: Time derivative of `s` """ -struct ZStarSpacing{R, S} <: AbstractVerticalSpacing{R} - Δr :: R - sⁿ :: S - s⁻ :: S - ∂t_∂s :: S +struct ZStarSpacing{R, SCC, SFC, SCF} <: AbstractVerticalSpacing{R} + Δr :: R + sᶜᶜⁿ :: SCC + sᶠᶜⁿ :: SFC + sᶜᶠⁿ :: SCF + sᶜᶜ⁻ :: SCC + ∂t_∂s :: SCC end Adapt.adapt_structure(to, coord::ZStarSpacing) = @@ -66,16 +68,18 @@ end # - the derivative in time of the spacing function generalized_spacing_grid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, ::ZStar) where {FT, TX, TY, TZ} - s⁻ = Field{Center, Center, Nothing}(grid) - sⁿ = Field{Center, Center, Nothing}(grid) + sᶜᶜ⁻ = Field{Center, Center, Nothing}(grid) + sᶜᶜⁿ = Field{Center, Center, Nothing}(grid) + sᶠᶜⁿ = Field{Face, Center, Nothing}(grid) + sᶜᶠⁿ = Field{Center, Face, Nothing}(grid) ∂t_∂s = Field{Center, Center, Nothing}(grid) # Initial "at-rest" conditions fill!(s⁻, 1) fill!(sⁿ, 1) - Δzᵃᵃᶠ = ZStarSpacing(grid.Δzᵃᵃᶠ, s⁻, sⁿ, ∂t_∂s) - Δzᵃᵃᶜ = ZStarSpacing(grid.Δzᵃᵃᶜ, s⁻, sⁿ, ∂t_∂s) + Δzᵃᵃᶠ = ZStarSpacing(grid.Δzᵃᵃᶠ, sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶜᶜ⁻, ∂t_∂s) + Δzᵃᵃᶜ = ZStarSpacing(grid.Δzᵃᵃᶜ, sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶜᶜ⁻, ∂t_∂s) args = [] for prop in propertynames(grid) @@ -100,8 +104,14 @@ end reference_Δzᵃᵃᶠ(grid::ZStarSpacingGrid) = grid.Δzᵃᵃᶠ.Δr reference_Δzᵃᵃᶜ(grid::ZStarSpacingGrid) = grid.Δzᵃᵃᶜ.Δr -@inline Δzᶜᶜᶠ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δzᶜᶜᶠ_reference(i, j, k, grid) * grid.Δzᵃᵃᶠ.sⁿ[i, j, 1] -@inline Δzᶜᶜᶜ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δzᶜᶜᶜ_reference(i, j, k, grid) * grid.Δzᵃᵃᶜ.sⁿ[i, j, 1] +@inline Δzᶜᶜᶠ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δzᶜᶜᶠ_reference(i, j, k, grid) * grid.Δzᵃᵃᶠ.sᶜᶜⁿ[i, j, 1] +@inline Δzᶜᶜᶜ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δzᶜᶜᶜ_reference(i, j, k, grid) * grid.Δzᵃᵃᶜ.sᶜᶜⁿ[i, j, 1] + +@inline Δzᶠᶜᶠ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δzᶜᶜᶠ_reference(i, j, k, grid) * grid.Δzᵃᵃᶠ.sᶠᶜⁿ[i, j, 1] +@inline Δzᶠᶜᶜ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δzᶜᶜᶜ_reference(i, j, k, grid) * grid.Δzᵃᵃᶜ.sᶠᶜⁿ[i, j, 1] + +@inline Δzᶜᶠᶠ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δzᶜᶜᶠ_reference(i, j, k, grid) * grid.Δzᵃᵃᶠ.sᶜᶠⁿ[i, j, 1] +@inline Δzᶜᶠᶜ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δzᶜᶜᶜ_reference(i, j, k, grid) * grid.Δzᵃᵃᶜ.sᶜᶠⁿ[i, j, 1] @inline ∂t_∂s_grid(i, j, k, grid::ZStarSpacingGrid) = grid.Δzᵃᵃᶜ.∂t_∂s[i, j, k] @inline V_times_∂t_∂s_grid(i, j, k, grid::ZStarSpacingGrid) = ∂t_∂s_grid(i, j, k, grid) * Vᶜᶜᶜ(i, j, k, grid) @@ -113,12 +123,16 @@ reference_Δzᵃᵃᶜ(grid::ZStarSpacingGrid) = grid.Δzᵃᵃᶜ.Δr function update_vertical_spacing!(model, grid::ZStarSpacingGrid; parameters = :xy) # Scaling - s⁻ = grid.Δzᵃᵃᶠ.s⁻ - sⁿ = grid.Δzᵃᵃᶠ.sⁿ + sᶜᶜ⁻ = grid.Δzᵃᵃᶠ.sᶜᶜ⁻ + sᶜᶜⁿ = grid.Δzᵃᵃᶠ.sᶜᶜⁿ + sᶠᶜⁿ = grid.Δzᵃᵃᶠ.sᶠᶜⁿ + sᶜᶠⁿ = grid.Δzᵃᵃᶠ.sᶜᶠⁿ ∂t_∂s = grid.Δzᵃᵃᶠ.∂t_∂s # Free surface variables Hᶜᶜ = model.free_surface.auxiliary.Hᶜᶜ + Hᶠᶜ = model.free_surface.auxiliary.Hᶠᶜ + Hᶜᶠ = model.free_surface.auxiliary.Hᶜᶠ U̅ = model.free_surface.state.U̅ V̅ = model.free_surface.state.V̅ η = model.free_surface.η @@ -126,7 +140,7 @@ function update_vertical_spacing!(model, grid::ZStarSpacingGrid; parameters = :x # Update vertical spacing with available parameters # No need to fill the halo as the scaling is updated _IN_ the halos launch!(architecture(grid), grid, parameters, _update_zstar!, - sⁿ, s⁻, η, Hᶜᶜ, grid) + sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶜᶜ⁻, η, Hᶜᶜ, Hᶠᶜ, Hᶜᶠ, grid) # Update the time derivative of the grid-scaling. Note that in this case we leverage the # free surface evolution equation, where the time derivative of the free surface is equal @@ -148,15 +162,20 @@ end end end -@kernel function _update_zstar!(sⁿ, s⁻, η, Hᶜᶜ, grid) +@kernel function _update_zstar!(sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶜᶜ⁻, η, Hᶜᶜ, Hᶠᶜ, Hᶜᶠ, grid) i, j = @index(Global, NTuple) + k_top = grid.Nz+1 @inbounds begin - bottom = Hᶜᶜ[i, j, 1] - h = (bottom + η[i, j, grid.Nz+1]) / bottom + hᶜᶜ = (Hᶜᶜ[i, j, 1] + η[i, j, k_top]) / Hᶜᶜ[i, j, 1] + hᶠᶜ = (Hᶠᶜ[i, j, 1] + ℑxᶠᵃᵃ(i, j, k_top, grid, η)) / Hᶠᶜ[i, j, 1] + hᶜᶠ = (Hᶜᶠ[i, j, 1] + ℑyᵃᶠᵃ(i, j, k_top, grid, η)) / Hᶜᶠ[i, j, 1] + sᶜᶜ⁻[i, j, 1] = sᶜᶜⁿ[i, j, 1] + # update current and previous scaling - s⁻[i, j, 1] = sⁿ[i, j, 1] - sⁿ[i, j, 1] = h + sᶜᶜⁿ[i, j, 1] = hᶜᶜ + sᶠᶜⁿ[i, j, 1] = hᶠᶜ + sᶜᶠⁿ[i, j, 1] = hᶜᶠ end end diff --git a/src/Models/NonhydrostaticModels/compute_nonhydrostatic_tendencies.jl b/src/Models/NonhydrostaticModels/compute_nonhydrostatic_tendencies.jl index bf840e07b6..2c19168270 100644 --- a/src/Models/NonhydrostaticModels/compute_nonhydrostatic_tendencies.jl +++ b/src/Models/NonhydrostaticModels/compute_nonhydrostatic_tendencies.jl @@ -115,7 +115,7 @@ function compute_interior_tendency_contributions!(model, kernel_parameters; acti end_tracer_kernel_args = (buoyancy, biogeochemistry, background_fields, velocities, tracers, auxiliary_fields, diffusivities) - for tracer_index in 1:length(tracers) + for tracer_index in eachindex(tracers) @inbounds c_tendency = tendencies[tracer_index + 3] @inbounds forcing = forcings[tracer_index + 3] @inbounds c_immersed_bc = tracers[tracer_index].boundary_conditions.immersed From a879db1414621e4ec4e22f0c18fa8113af7742a1 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Sun, 8 Sep 2024 12:31:49 -0400 Subject: [PATCH 288/567] better lock-release --- validation/z_star_coordinate/lock_release.jl | 24 ++++++++------------ 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index e059d64d45..1f29a4b9af 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -5,9 +5,9 @@ using Oceananigans.Advection: WENOVectorInvariant using Oceananigans.Models.HydrostaticFreeSurfaceModels: ZStar, ZStarSpacingGrid, Δzᶜᶜᶜ_reference using Printf -grid = RectilinearGrid(size = (10, 10), - x = (0, 100kilometers), - z = (-10, 0), +grid = RectilinearGrid(size = (128, 20), + x = (0, 64kilometers), + z = (-20, 0), halo = (6, 6), topology = (Bounded, Flat, Bounded)) @@ -17,22 +17,21 @@ model = HydrostaticFreeSurfaceModel(; grid, tracer_advection = WENO(), buoyancy = BuoyancyTracer(), tracers = :b, - free_surface = SplitExplicitFreeSurface(; substeps = 10)) + free_surface = SplitExplicitFreeSurface(; substeps = 120)) g = model.free_surface.gravitational_acceleration -bᵢ(x, z) = x < 50kilometers ? 1 : 0 +model.timestepper.χ = 0.0 -set!(model, b = bᵢ) +bᵢ(x, z) = x < 32kilometers ? 0.06 : 0.01 -gravity_wave_speed = sqrt(g * grid.Lz) -barotropic_time_step = grid.Δxᶜᵃᵃ / gravity_wave_speed +set!(model, b = bᵢ) -Δt = 0.5 * barotropic_time_step +Δt = 1 @info "the time step is $Δt" -simulation = Simulation(model; Δt, stop_iteration = 100000, stop_time = 10days) +simulation = Simulation(model; Δt, stop_iteration = 100000, stop_time = 17hours) field_outputs = if model.grid isa ZStarSpacingGrid merge(model.velocities, model.tracers, (; sⁿ = model.grid.Δzᵃᵃᶠ.sⁿ)) @@ -48,8 +47,6 @@ simulation.output_writers[:other_variables] = JLD2OutputWriter(model, field_outp function progress(sim) w = interior(sim.model.velocities.w, :, :, sim.model.grid.Nz+1) u = sim.model.velocities.u - v = sim.model.velocities.v - η = sim.model.free_surface.η b = sim.model.tracers.b msg0 = @sprintf("Time: %s iteration %d ", prettytime(sim.model.clock.time), sim.model.clock.iteration) @@ -67,8 +64,7 @@ function progress(sim) return nothing end -simulation.callbacks[:progress] = Callback(progress, IterationInterval(1)) -simulation.callbacks[:wizard] = Callback(TimeStepWizard(; cfl = 0.2, max_change = 1.1), IterationInterval(10)) +simulation.callbacks[:progress] = Callback(progress, IterationInterval(100)) run!(simulation) From 07cac8d112d2c734ca784db5df16441b1e113eb0 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Sun, 8 Sep 2024 12:45:11 -0400 Subject: [PATCH 289/567] bugfixes --- .../generalized_vertical_spacing.jl | 6 +++--- .../z_star_vertical_spacing.jl | 9 +++++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index c85d2fb1c0..5af0614264 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -166,8 +166,8 @@ end ab2_step_tracer_field!(tracer_field, grid::AbstractVerticalSpacingGrid, Δt, χ, Gⁿ, G⁻) = launch!(architecture(grid), grid, :xyz, _ab2_step_tracer_generalized_spacing!, tracer_field, - grid.Δzᵃᵃᶠ.sⁿ, - grid.Δzᵃᵃᶠ.s⁻, + vertical_scaling(grid), + previous_vertical_scaling(grid), Δt, χ, Gⁿ, G⁻) const EmptyTuples = Union{NamedTuple{(), Tuple{}}, Tuple{}} @@ -180,7 +180,7 @@ tracer_scaling_parameters(param::KernelParameters{S, O}, tracers, grid) where {S function unscale_tracers!(tracers, grid::AbstractVerticalSpacingGrid; parameters = :xy) parameters = tracer_scaling_parameters(parameters, tracers, grid) - launch!(architecture(grid), grid, parameters, _unscale_tracers!, tracers, grid.Δzᵃᵃᶠ.sⁿ, + launch!(architecture(grid), grid, parameters, _unscale_tracers!, tracers, vertical_scaling(grid), Val(grid.Hz), Val(grid.Nz)) return nothing diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 286b4c731c..1c547d3879 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -75,8 +75,10 @@ function generalized_spacing_grid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, ∂t_∂s = Field{Center, Center, Nothing}(grid) # Initial "at-rest" conditions - fill!(s⁻, 1) - fill!(sⁿ, 1) + fill!(sᶜᶜ⁻, 1) + fill!(sᶜᶜⁿ, 1) + fill!(sᶠᶜⁿ, 1) + fill!(sᶜᶠⁿ, 1) Δzᵃᵃᶠ = ZStarSpacing(grid.Δzᵃᵃᶠ, sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶜᶜ⁻, ∂t_∂s) Δzᵃᵃᶜ = ZStarSpacing(grid.Δzᵃᵃᶜ, sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶜᶜ⁻, ∂t_∂s) @@ -101,6 +103,9 @@ end ##### ZStar-specific vertical spacing functions ##### +@inline vertical_scaling(grid::ZStarSpacingGrid) = grid.Δzᵃᵃᶠ.sᶜᶜⁿ +@inline previous_vertical_scaling(grid::ZStarSpacingGrid) = grid.Δzᵃᵃᶠ.sᶜᶜ⁻ + reference_Δzᵃᵃᶠ(grid::ZStarSpacingGrid) = grid.Δzᵃᵃᶠ.Δr reference_Δzᵃᵃᶜ(grid::ZStarSpacingGrid) = grid.Δzᵃᵃᶜ.Δr From a402123848b6d2a31364ef4beaca499323b9030f Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Sun, 8 Sep 2024 12:58:38 -0400 Subject: [PATCH 290/567] add tests --- .../hydrostatic_free_surface_model.jl | 4 +- test/test_zstar_coordinate.jl | 56 +++++++++++++++++++ validation/z_star_coordinate/lock_release.jl | 8 +-- 3 files changed, 62 insertions(+), 6 deletions(-) create mode 100644 test/test_zstar_coordinate.jl diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl index 15a42769ae..7e2cd114c1 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl @@ -134,10 +134,10 @@ function HydrostaticFreeSurfaceModel(; grid, if !(free_surface isa SplitExplicitFreeSurface) && !isnothing(generalized_vertical_coordinate) @warn "Generalized vertical coordinates are supported only for the SplitExplicitFreeSurface at the moment. Ignoring the generalized_vertical_coordinate argument." + else + grid = generalized_spacing_grid(grid, generalized_vertical_coordinate) end - grid = free_surface isa SplitExplicitFreeSurface ? generalized_spacing_grid(grid, generalized_vertical_coordinate) : grid - arch = architecture(grid) @apply_regionally momentum_advection = validate_momentum_advection(momentum_advection, grid) diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl new file mode 100644 index 0000000000..1ec75e8107 --- /dev/null +++ b/test/test_zstar_coordinate.jl @@ -0,0 +1,56 @@ +include("dependencies_for_runtests.jl") + +using Oceananigans.Models.HydrostaticFreeSurfaceModels: ZStar + +function test_zstar_coordinate(model, Ni, Δt) + + ∫bᵢ = Field(Integral(model.tracers.b)) + w = model.velocities.w + Nz = model.grid.Nz + + for _ in 1:Ni + time_step!(model, Δt) + ∫b = Field(Integral(model.tracers.b)) + + @test ∫b[1, 1, 1] ≈ ∫bᵢ[1, 1, 1] + @test maximum(interior(w, :, :, Nz+1)) < eps(eltype(w)) + end + + return nothing +end + + +@testset "Testing z-star coordinates" begin + + for arch in archs + llg = LatitudeLongitudeGrid(arch; size = (10, 10, 10), latitude = (-10, 10), longitude = (-10, 10), z = (-10, 0)) + rtg = RectilinearGrid(arch; size = (10, 10, 10), x = (-10, 10), y = (-10, 10), z = (-10, 0)) + + llgv = LatitudeLongitudeGrid(arch; size = (10, 10, 10), latitude = (-10, 10), longitude = (-10, 10), z = collect(-10:0)) + rtgv = RectilinearGrid(arch; size = (10, 10, 10), x = (-10, 10), y = (-10, 10), z = collect(-10:0)) + + illg = ImmersedBoundaryGrid(llg, GridFittedBottom((x, y) -> - rand() - 5)) + irtg = ImmersedBoundaryGrid(rtg, GridFittedBottom((x, y) -> - rand() - 5)) + + illgv = ImmersedBoundaryGrid(llgv, GridFittedBottom((x, y) -> - rand() - 5)) + irtgv = ImmersedBoundaryGrid(rtgv, GridFittedBottom((x, y) -> - rand() - 5)) + + grids = [llg, rtg, llgv, rtgv, illg, irtg, illgv, irtgv] + + for grid in grids + + free_surface = SplitExplicitFreeSurface(grid; cfl = 0.75) + model = HydrostaticFreeSurfaceModel(; grid, + free_surface, + tracers = (:b, :c), + bouyancy = BuoyancTracer(), + generalized_vertical_coordinate = ZStar()) + + bᵢ(x, y, z) = x < grid.Lx / 2 ? 0.06 : 0.01 + + set!(model, c = (x, y, z) -> rand(), b = bᵢ) + + test_zstar_coordinate(model, 100, 10) + end + end +end \ No newline at end of file diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index 1f29a4b9af..1f5ef79455 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -13,11 +13,11 @@ grid = RectilinearGrid(size = (128, 20), model = HydrostaticFreeSurfaceModel(; grid, generalized_vertical_coordinate = ZStar(), - momentum_advection = WENO(), + momentum_advection = WENOVectorInvariant(), tracer_advection = WENO(), buoyancy = BuoyancyTracer(), tracers = :b, - free_surface = SplitExplicitFreeSurface(; substeps = 120)) + free_surface = SplitExplicitFreeSurface(; substeps = 100)) g = model.free_surface.gravitational_acceleration @@ -34,7 +34,7 @@ set!(model, b = bᵢ) simulation = Simulation(model; Δt, stop_iteration = 100000, stop_time = 17hours) field_outputs = if model.grid isa ZStarSpacingGrid - merge(model.velocities, model.tracers, (; sⁿ = model.grid.Δzᵃᵃᶠ.sⁿ)) + merge(model.velocities, model.tracers, (; sⁿ = model.grid.Δzᵃᵃᶠ.sᶜᶜⁿ)) else merge(model.velocities, model.tracers) end @@ -54,7 +54,7 @@ function progress(sim) msg2 = @sprintf("extrema u: %.2e %.2e ", maximum(u), minimum(u)) msg3 = @sprintf("extrema b: %.2e %.2e ", maximum(b), minimum(b)) if sim.model.grid isa ZStarSpacingGrid - Δz = sim.model.grid.Δzᵃᵃᶠ.sⁿ + Δz = sim.model.grid.Δzᵃᵃᶠ.sᶜᶜⁿ msg4 = @sprintf("extrema Δz: %.2e %.2e ", maximum(Δz), minimum(Δz)) @info msg0 * msg1 * msg2 * msg3 * msg4 else From 464feb1058480c9194d140e1ad5d5edc4ba22322 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 9 Sep 2024 09:54:45 -0600 Subject: [PATCH 291/567] adding some tests and numerics valid on an immersed boundary grid --- .../HydrostaticFreeSurfaceModels.jl | 4 ++++ ...distributed_split_explicit_free_surface.jl | 5 ++-- .../split_explicit_free_surface.jl | 15 ++++++++---- .../z_star_vertical_spacing.jl | 24 +++++++++++++------ ...ulti_region_split_explicit_free_surface.jl | 7 +++--- test/test_zstar_coordinate.jl | 9 +++++-- validation/z_star_coordinate/lock_release.jl | 9 ++++--- 7 files changed, 51 insertions(+), 22 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl index c0fad04ff5..f71808fd8b 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl @@ -86,6 +86,10 @@ Return a flattened `NamedTuple` of the prognostic fields associated with `Hydros @inline prognostic_fields(model::HydrostaticFreeSurfaceModel) = hydrostatic_prognostic_fields(model.velocities, model.free_surface, model.tracers) +@inline prognostic_fields(model::HydrostaticFreeSurfaceModel{<:Any, <:Any, <:Any, <:Any, ZStarSpacingGrid}) = + merge(hydrostatic_prognostic_fields(model.velocities, model.free_surface, model.tracers), + (; ∂t_∂s = grid.Δzᵃᵃᶠ.∂t_∂s)) + @inline hydrostatic_prognostic_fields(velocities, free_surface, tracers) = merge((u = velocities.u, v = velocities.v, η = free_surface.η), diff --git a/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl index 83d5f8a2d1..d1256d414a 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl @@ -13,8 +13,9 @@ function SplitExplicitAuxiliaryFields(grid::DistributedGrid) Hᶠᶜ = Field{Face, Center, Nothing}(grid) Hᶜᶠ = Field{Center, Face, Nothing}(grid) Hᶜᶜ = Field{Center, Center, Nothing}(grid) + Hᶠᶠ = Field{Face, Face, Nothing}(grid) - compute_column_height!(Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, grid) + compute_column_height!(Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, Hᶠᶠ, grid) # In a non-parallel grid we calculate only the interior kernel_size = augmented_kernel_size(grid) @@ -22,7 +23,7 @@ function SplitExplicitAuxiliaryFields(grid::DistributedGrid) kernel_parameters = KernelParameters(kernel_size, kernel_offsets) - return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, kernel_parameters) + return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, Hᶠᶠ, kernel_parameters) end """Integrate z at locations `location` and set! `height`` with the result""" diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index a865b99c94..3189dd12b6 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -204,7 +204,7 @@ large (or `:xy` in case of a serial computation), and start computing from $(FIELDS) """ -Base.@kwdef struct SplitExplicitAuxiliaryFields{𝒞ℱ, ℱ𝒞, 𝒞𝒞, 𝒦} +Base.@kwdef struct SplitExplicitAuxiliaryFields{𝒞ℱ, ℱ𝒞, 𝒞𝒞, ℱℱ, 𝒦} "Vertically-integrated slow barotropic forcing function for `U` (`ReducedField` over ``z``)" Gᵁ :: ℱ𝒞 "Vertically-integrated slow barotropic forcing function for `V` (`ReducedField` over ``z``)" @@ -216,6 +216,8 @@ Base.@kwdef struct SplitExplicitAuxiliaryFields{𝒞ℱ, ℱ𝒞, 𝒞𝒞, 𝒦 "Depth at `(Center, Center)` (`ReducedField` over ``z``)" Hᶜᶜ :: 𝒞𝒞 "kernel size for barotropic time stepping" + Hᶠᶠ :: ℱℱ + "kernel size for barotropic time stepping" kernel_parameters :: 𝒦 end @@ -232,15 +234,16 @@ function SplitExplicitAuxiliaryFields(grid::AbstractGrid) Hᶠᶜ = Field{Face, Center, Nothing}(grid) Hᶜᶠ = Field{Center, Face, Nothing}(grid) Hᶜᶜ = Field{Center, Center, Nothing}(grid) + Hᶠᶠ = Field{Face, Face, Nothing}(grid) - compute_column_height!(Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, grid) + compute_column_height!(Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, Hᶠᶠ, grid) kernel_parameters = :xy - return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, kernel_parameters) + return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, Hᶠᶠ, kernel_parameters) end -function compute_column_height!(Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, grid) +function compute_column_height!(Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, Hᶠᶠ, grid) Nx, Ny, _ = size(grid) Hx, Hy, _ = halo_size(grid) Tx, Ty, _ = topology(grid) @@ -268,7 +271,8 @@ function compute_column_height!(Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, grid) kernel_offset = (-Hx, -Hy) paramᶜᶜ = KernelParameters(kernel_size, kernel_offset) launch!(arch, grid, paramᶜᶜ, _compute_column_height!, Hᶜᶜ, grid, c, c, Δzᶜᶜᶜ_reference) - + launch!(arch, grid, paramᶜᶜ, _compute_column_height!, Hᶠᶠ, grid, c, c, Δzᶠᶠᶜ_reference) + return nothing end @@ -473,6 +477,7 @@ Adapt.adapt_structure(to, auxiliary::SplitExplicitAuxiliaryFields) = Adapt.adapt(to, auxiliary.Hᶠᶜ), Adapt.adapt(to, auxiliary.Hᶜᶠ), Adapt.adapt(to, auxiliary.Hᶜᶜ), + Adapt.adapt(to, auxiliary.Hᶠᶠ), nothing) for Type in (:SplitExplicitFreeSurface, diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 1c547d3879..9eb015fe5e 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -22,11 +22,12 @@ where ``η`` is the free surface height and ``H`` the vertical depth of the wate - `s⁻`:scaling of the vertical coordinate at time step `n - 1` - `∂t_∂s`: Time derivative of `s` """ -struct ZStarSpacing{R, SCC, SFC, SCF} <: AbstractVerticalSpacing{R} +struct ZStarSpacing{R, SCC, SFC, SCF, SFF} <: AbstractVerticalSpacing{R} Δr :: R sᶜᶜⁿ :: SCC sᶠᶜⁿ :: SFC sᶜᶠⁿ :: SCF + sᶠᶠⁿ :: SFF sᶜᶜ⁻ :: SCC ∂t_∂s :: SCC end @@ -72,6 +73,7 @@ function generalized_spacing_grid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, sᶜᶜⁿ = Field{Center, Center, Nothing}(grid) sᶠᶜⁿ = Field{Face, Center, Nothing}(grid) sᶜᶠⁿ = Field{Center, Face, Nothing}(grid) + sᶠᶠⁿ = Field{Face, Face, Nothing}(grid) ∂t_∂s = Field{Center, Center, Nothing}(grid) # Initial "at-rest" conditions @@ -79,9 +81,10 @@ function generalized_spacing_grid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, fill!(sᶜᶜⁿ, 1) fill!(sᶠᶜⁿ, 1) fill!(sᶜᶠⁿ, 1) + fill!(sᶠᶠⁿ, 1) - Δzᵃᵃᶠ = ZStarSpacing(grid.Δzᵃᵃᶠ, sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶜᶜ⁻, ∂t_∂s) - Δzᵃᵃᶜ = ZStarSpacing(grid.Δzᵃᵃᶜ, sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶜᶜ⁻, ∂t_∂s) + Δzᵃᵃᶠ = ZStarSpacing(grid.Δzᵃᵃᶠ, sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, ∂t_∂s) + Δzᵃᵃᶜ = ZStarSpacing(grid.Δzᵃᵃᶜ, sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, ∂t_∂s) args = [] for prop in propertynames(grid) @@ -118,6 +121,9 @@ reference_Δzᵃᵃᶜ(grid::ZStarSpacingGrid) = grid.Δzᵃᵃᶜ.Δr @inline Δzᶜᶠᶠ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δzᶜᶜᶠ_reference(i, j, k, grid) * grid.Δzᵃᵃᶠ.sᶜᶠⁿ[i, j, 1] @inline Δzᶜᶠᶜ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δzᶜᶜᶜ_reference(i, j, k, grid) * grid.Δzᵃᵃᶜ.sᶜᶠⁿ[i, j, 1] +@inline Δzᶠᶠᶠ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δzᶜᶜᶠ_reference(i, j, k, grid) * grid.Δzᵃᵃᶠ.sᶠᶠⁿ[i, j, 1] +@inline Δzᶠᶠᶜ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δzᶜᶜᶜ_reference(i, j, k, grid) * grid.Δzᵃᵃᶜ.sᶠᶠⁿ[i, j, 1] + @inline ∂t_∂s_grid(i, j, k, grid::ZStarSpacingGrid) = grid.Δzᵃᵃᶜ.∂t_∂s[i, j, k] @inline V_times_∂t_∂s_grid(i, j, k, grid::ZStarSpacingGrid) = ∂t_∂s_grid(i, j, k, grid) * Vᶜᶜᶜ(i, j, k, grid) @@ -132,12 +138,14 @@ function update_vertical_spacing!(model, grid::ZStarSpacingGrid; parameters = :x sᶜᶜⁿ = grid.Δzᵃᵃᶠ.sᶜᶜⁿ sᶠᶜⁿ = grid.Δzᵃᵃᶠ.sᶠᶜⁿ sᶜᶠⁿ = grid.Δzᵃᵃᶠ.sᶜᶠⁿ + sᶠᶠⁿ = grid.Δzᵃᵃᶠ.sᶠᶠⁿ ∂t_∂s = grid.Δzᵃᵃᶠ.∂t_∂s # Free surface variables Hᶜᶜ = model.free_surface.auxiliary.Hᶜᶜ Hᶠᶜ = model.free_surface.auxiliary.Hᶠᶜ Hᶜᶠ = model.free_surface.auxiliary.Hᶜᶠ + Hᶠᶠ = model.free_surface.auxiliary.Hᶠᶠ U̅ = model.free_surface.state.U̅ V̅ = model.free_surface.state.V̅ η = model.free_surface.η @@ -145,7 +153,7 @@ function update_vertical_spacing!(model, grid::ZStarSpacingGrid; parameters = :x # Update vertical spacing with available parameters # No need to fill the halo as the scaling is updated _IN_ the halos launch!(architecture(grid), grid, parameters, _update_zstar!, - sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶜᶜ⁻, η, Hᶜᶜ, Hᶠᶜ, Hᶜᶠ, grid) + sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, η, Hᶜᶜ, Hᶠᶜ, Hᶜᶠ, Hᶠᶠ, grid) # Update the time derivative of the grid-scaling. Note that in this case we leverage the # free surface evolution equation, where the time derivative of the free surface is equal @@ -167,13 +175,14 @@ end end end -@kernel function _update_zstar!(sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶜᶜ⁻, η, Hᶜᶜ, Hᶠᶜ, Hᶜᶠ, grid) +@kernel function _update_zstar!(sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, η, Hᶜᶜ, Hᶠᶜ, Hᶜᶠ, Hᶠᶠ, grid) i, j = @index(Global, NTuple) k_top = grid.Nz+1 @inbounds begin hᶜᶜ = (Hᶜᶜ[i, j, 1] + η[i, j, k_top]) / Hᶜᶜ[i, j, 1] - hᶠᶜ = (Hᶠᶜ[i, j, 1] + ℑxᶠᵃᵃ(i, j, k_top, grid, η)) / Hᶠᶜ[i, j, 1] - hᶜᶠ = (Hᶜᶠ[i, j, 1] + ℑyᵃᶠᵃ(i, j, k_top, grid, η)) / Hᶜᶠ[i, j, 1] + hᶠᶜ = (Hᶠᶜ[i, j, 1] + ℑxᶠᵃᵃ(i, j, k_top, grid, η)) / Hᶠᶜ[i, j, 1] + hᶜᶠ = (Hᶜᶠ[i, j, 1] + ℑyᵃᶠᵃ(i, j, k_top, grid, η)) / Hᶜᶠ[i, j, 1] + hᶠᶠ = (Hᶠᶠ[i, j, 1] + ℑxyᶠᶠᵃ(i, j, k_top, grid, η)) / Hᶠᶠ[i, j, 1] sᶜᶜ⁻[i, j, 1] = sᶜᶜⁿ[i, j, 1] @@ -181,6 +190,7 @@ end sᶜᶜⁿ[i, j, 1] = hᶜᶜ sᶠᶜⁿ[i, j, 1] = hᶠᶜ sᶜᶠⁿ[i, j, 1] = hᶜᶠ + sᶠᶠⁿ[i, j, 1] = hᶠᶠ end end diff --git a/src/MultiRegion/multi_region_split_explicit_free_surface.jl b/src/MultiRegion/multi_region_split_explicit_free_surface.jl index c0eb342ac9..f85b9b1241 100644 --- a/src/MultiRegion/multi_region_split_explicit_free_surface.jl +++ b/src/MultiRegion/multi_region_split_explicit_free_surface.jl @@ -16,9 +16,10 @@ function SplitExplicitAuxiliaryFields(grid::MultiRegionGrids) Hᶠᶜ = Field((Face, Center, Nothing), grid) Hᶜᶠ = Field((Center, Face, Nothing), grid) - Hᶜᶠ = Field((Center, Center, Nothing), grid) + Hᶜᶜ = Field((Center, Center, Nothing), grid) + Hᶠᶠ = Field((Center, Center, Nothing), grid) - @apply_regionally compute_column_height!(Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, grid) + @apply_regionally compute_column_height!(Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, Hᶠᶠ, grid) # In a non-parallel grid we calculate only the interior @apply_regionally kernel_size = augmented_kernel_size(grid, grid.partition) @@ -26,7 +27,7 @@ function SplitExplicitAuxiliaryFields(grid::MultiRegionGrids) @apply_regionally kernel_parameters = KernelParameters(kernel_size, kernel_offsets) - return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, kernel_parameters) + return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, Hᶠᶠ, kernel_parameters) end @inline augmented_kernel_size(grid, ::XPartition) = (size(grid, 1) + 2halo_size(grid)[1]-2, size(grid, 2)) diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index 1ec75e8107..2978573b08 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -5,14 +5,20 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: ZStar function test_zstar_coordinate(model, Ni, Δt) ∫bᵢ = Field(Integral(model.tracers.b)) + ∫cᵢ = Field(Integral(model.tracers.c)) w = model.velocities.w Nz = model.grid.Nz + # Testing that at each timestep + # (1) tracers are conserved down to machine precision + # (2) vertical velocities are zero at the top surface for _ in 1:Ni time_step!(model, Δt) ∫b = Field(Integral(model.tracers.b)) + ∫c = Field(Integral(model.tracers.c)) - @test ∫b[1, 1, 1] ≈ ∫bᵢ[1, 1, 1] + @test interior(∫b, 1, 1, 1) ≈ interior(∫bᵢ, 1, 1, 1) + @test interior(∫c, 1, 1, 1) ≈ interior(∫cᵢ, 1, 1, 1) @test maximum(interior(w, :, :, Nz+1)) < eps(eltype(w)) end @@ -38,7 +44,6 @@ end grids = [llg, rtg, llgv, rtgv, illg, irtg, illgv, irtgv] for grid in grids - free_surface = SplitExplicitFreeSurface(grid; cfl = 0.75) model = HydrostaticFreeSurfaceModel(; grid, free_surface, diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index 1f5ef79455..b0eed02717 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -11,13 +11,16 @@ grid = RectilinearGrid(size = (128, 20), halo = (6, 6), topology = (Bounded, Flat, Bounded)) +grid = ImmersedBoundaryGrid(grid, GridFittedBottom(x -> x < 32kilometers ? -10 : -20)) + model = HydrostaticFreeSurfaceModel(; grid, generalized_vertical_coordinate = ZStar(), - momentum_advection = WENOVectorInvariant(), - tracer_advection = WENO(), + momentum_advection = WENO(; order = 5), + tracer_advection = WENO(; order = 5), buoyancy = BuoyancyTracer(), + closure = nothing, tracers = :b, - free_surface = SplitExplicitFreeSurface(; substeps = 100)) + free_surface = SplitExplicitFreeSurface(; substeps = 120)) g = model.free_surface.gravitational_acceleration From 28658fb178bd116b88961c651a2a1ec1638d1edd Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 25 Sep 2024 12:14:10 -0400 Subject: [PATCH 292/567] change of notation --- Project.toml | 1 + ...static_free_surface_boundary_tendencies.jl | 2 +- .../generalized_vertical_spacing.jl | 36 ++++++++++--------- .../split_explicit_free_surface.jl | 8 ++--- .../split_explicit_free_surface_kernels.jl | 16 ++++----- .../z_star_vertical_spacing.jl | 34 ++++++++++-------- validation/z_star_coordinate/lock_release.jl | 4 +-- 7 files changed, 56 insertions(+), 45 deletions(-) diff --git a/Project.toml b/Project.toml index 51c93af24c..c2e5cf282b 100644 --- a/Project.toml +++ b/Project.toml @@ -11,6 +11,7 @@ CubedSphere = "7445602f-e544-4518-8976-18f8e8ae6cdb" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Distances = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7" DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" +FFMPEG = "c87230d0-a227-11e9-1b43-d7ebe4e7570a" FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" Glob = "c27321d9-0574-5035-807b-f59d2c89b15c" IncompleteLU = "40713840-3770-5561-ab4c-a76e7d0d7895" diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl index 4842ca95e6..7af5ca20ce 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_boundary_tendencies.jl @@ -10,7 +10,7 @@ using Oceananigans.Models.NonhydrostaticModels: boundary_tendency_kernel_paramet using Oceananigans.TurbulenceClosures: required_halo_size -using Oceananigans.ImmersedBoundaries: active_interior_map, DistributedActiveCellsIBG +using Oceananigans.ImmersedBoundaries: DistributedActiveCellsIBG # We assume here that top/bottom BC are always synched (no partitioning in z) function compute_boundary_tendencies!(model::HydrostaticFreeSurfaceModel) diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index 5af0614264..575e04756f 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -44,13 +44,13 @@ function retrieve_static_grid(grid::AbstractVerticalSpacingImmersedGrid) return ImmersedBoundaryGrid(underlying_grid, grid.immersed_boundary; active_cells_map) end -reference_Δzᵃᵃᶠ(grid) = grid.Δzᵃᵃᶠ -reference_Δzᵃᵃᶜ(grid) = grid.Δzᵃᵃᶜ +reference_zspacings(grid, ::Face) = grid.Δzᵃᵃᶠ +reference_zspacings(grid, ::Center) = grid.Δzᵃᵃᶜ function retrieve_static_grid(grid::AbstractVerticalSpacingUnderlyingGrid) - Δzᵃᵃᶠ = reference_Δzᵃᵃᶠ(grid) - Δzᵃᵃᶜ = reference_Δzᵃᵃᶜ(grid) + Δzᵃᵃᶠ = reference_zspacings(grid, Face()) + Δzᵃᵃᶜ = reference_zspacings(grid, Center()) TX, TY, TZ = topology(grid) @@ -116,23 +116,27 @@ update_vertical_spacing!(model, grid; kwargs...) = nothing @inline Δzᶠᶠᶠ(i, j, k, grid::AbstractVerticalSpacingGrid) = ℑxyᶠᶠᵃ(i, j, k, grid, Δzᶜᶜᶠ) @inline Δzᶠᶠᶜ(i, j, k, grid::AbstractVerticalSpacingGrid) = ℑxyᶠᶠᵃ(i, j, k, grid, Δzᶜᶜᶜ) -@inline Δz_reference(i, j, k, Δz::Number) = Δz -@inline Δz_reference(i, j, k, Δz::AbstractVector) = @inbounds Δz[k] +##### +##### Reference (local) vertical spacings in `r-coordinates` +##### + +@inline Δr(i, j, k, Δz::Number) = Δz +@inline Δr(i, j, k, Δz::AbstractVector) = @inbounds Δz[k] -@inline Δz_reference(i, j, k, Δz::AbstractVerticalSpacing{<:Number}) = Δz.Δr -@inline Δz_reference(i, j, k, Δz::AbstractVerticalSpacing) = @inbounds Δz.Δr[k] +@inline Δr(i, j, k, Δz::AbstractVerticalSpacing{<:Number}) = Δz.Δr +@inline Δr(i, j, k, Δz::AbstractVerticalSpacing) = @inbounds Δz.Δr[k] -@inline Δzᶜᶜᶠ_reference(i, j, k, grid) = Δz_reference(i, j, k, grid.Δzᵃᵃᶠ) -@inline Δzᶜᶜᶜ_reference(i, j, k, grid) = Δz_reference(i, j, k, grid.Δzᵃᵃᶜ) +@inline Δrᶜᶜᶠ(i, j, k, grid) = Δr(i, j, k, grid.Δzᵃᵃᶠ) +@inline Δrᶜᶜᶜ(i, j, k, grid) = Δr(i, j, k, grid.Δzᵃᵃᶜ) -@inline Δzᶜᶠᶠ_reference(i, j, k, grid) = ℑyᵃᶠᵃ(i, j, k, grid, Δzᶜᶜᶠ_reference) -@inline Δzᶜᶠᶜ_reference(i, j, k, grid) = ℑyᵃᶠᵃ(i, j, k, grid, Δzᶜᶜᶜ_reference) +@inline Δrᶜᶠᶠ(i, j, k, grid) = ℑyᵃᶠᵃ(i, j, k, grid, Δrᶜᶜᶠ) +@inline Δrᶜᶠᶜ(i, j, k, grid) = ℑyᵃᶠᵃ(i, j, k, grid, Δrᶜᶜᶜ) -@inline Δzᶠᶜᶠ_reference(i, j, k, grid) = ℑxᶠᵃᵃ(i, j, k, grid, Δzᶜᶜᶠ_reference) -@inline Δzᶠᶜᶜ_reference(i, j, k, grid) = ℑxᶠᵃᵃ(i, j, k, grid, Δzᶜᶜᶜ_reference) +@inline Δrᶠᶜᶠ(i, j, k, grid) = ℑxᶠᵃᵃ(i, j, k, grid, Δrᶜᶜᶠ) +@inline Δrᶠᶜᶜ(i, j, k, grid) = ℑxᶠᵃᵃ(i, j, k, grid, Δrᶜᶜᶜ) -@inline Δzᶠᶠᶠ_reference(i, j, k, grid) = ℑxyᶠᶠᵃ(i, j, k, grid, Δzᶜᶜᶠ_reference) -@inline Δzᶠᶠᶜ_reference(i, j, k, grid) = ℑxyᶠᶠᵃ(i, j, k, grid, Δzᶜᶜᶜ_reference) +@inline Δrᶠᶠᶠ(i, j, k, grid) = ℑxyᶠᶠᵃ(i, j, k, grid, Δrᶜᶜᶠ) +@inline Δrᶠᶠᶜ(i, j, k, grid) = ℑxyᶠᶠᵃ(i, j, k, grid, Δrᶜᶜᶜ) ##### ##### Additional terms to be included in the momentum equations (fallbacks) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index 3189dd12b6..ebc09c9ae9 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -260,18 +260,18 @@ function compute_column_height!(Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, Hᶠᶠ, grid) kernel_size = (Nx+2Hx-2Ax, Ny+2Hy) kernel_offset = (-Hx+Ax, -Hy) paramᶠᶜ = KernelParameters(kernel_size, kernel_offset) - launch!(arch, grid, paramᶠᶜ, _compute_column_height!, Hᶠᶜ, grid, f, c, Δzᶠᶜᶜ_reference) + launch!(arch, grid, paramᶠᶜ, _compute_column_height!, Hᶠᶜ, grid, f, c, Δrᶠᶜᶜ) kernel_size = (Nx+2Hx, Ny+2Hy-2Ay) kernel_offset = (-Hx, -Hy+Ay) paramᶜᶠ = KernelParameters(kernel_size, kernel_offset) - launch!(arch, grid, paramᶜᶠ, _compute_column_height!, Hᶜᶠ, grid, c, f, Δzᶜᶠᶜ_reference) + launch!(arch, grid, paramᶜᶠ, _compute_column_height!, Hᶜᶠ, grid, c, f, Δrᶜᶠᶜ) kernel_size = (Nx+2Hx, Ny+2Hy) kernel_offset = (-Hx, -Hy) paramᶜᶜ = KernelParameters(kernel_size, kernel_offset) - launch!(arch, grid, paramᶜᶜ, _compute_column_height!, Hᶜᶜ, grid, c, c, Δzᶜᶜᶜ_reference) - launch!(arch, grid, paramᶜᶜ, _compute_column_height!, Hᶠᶠ, grid, c, c, Δzᶠᶠᶜ_reference) + launch!(arch, grid, paramᶜᶜ, _compute_column_height!, Hᶜᶜ, grid, c, c, Δrᶜᶜᶜ) + launch!(arch, grid, paramᶜᶜ, _compute_column_height!, Hᶠᶠ, grid, c, c, Δrᶠᶠᶜ) return nothing end diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 43321f7cb7..1dec64c865 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -213,12 +213,12 @@ end k_top = grid.Nz + 1 # hand unroll first loop - @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δzᶠᶜᶜ_reference(i, j, 1, grid) - @inbounds V[i, j, k_top-1] = v[i, j, 1] * Δzᶜᶠᶜ_reference(i, j, 1, grid) + @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δrᶠᶜᶜ(i, j, 1, grid) + @inbounds V[i, j, k_top-1] = v[i, j, 1] * Δrᶜᶠᶜ(i, j, 1, grid) @unroll for k in 2:grid.Nz - @inbounds U[i, j, k_top-1] += u[i, j, k] * Δzᶠᶜᶜ_reference(i, j, k, grid) - @inbounds V[i, j, k_top-1] += v[i, j, k] * Δzᶜᶠᶜ_reference(i, j, k, grid) + @inbounds U[i, j, k_top-1] += u[i, j, k] * Δrᶠᶜᶜ(i, j, k, grid) + @inbounds V[i, j, k_top-1] += v[i, j, k] * Δrᶜᶠᶜ(i, j, k, grid) end end @@ -228,12 +228,12 @@ end k_top = grid.Nz+1 # hand unroll first loop - @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δzᶠᶜᶜ_reference(i, j, 1, grid) - @inbounds V[i, j, k_top-1] = v[i, j, 1] * Δzᶜᶠᶜ_reference(i, j, 1, grid) + @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δrᶠᶜᶜ(i, j, 1, grid) + @inbounds V[i, j, k_top-1] = v[i, j, 1] * Δrᶜᶠᶜ(i, j, 1, grid) @unroll for k in 2:grid.Nz - @inbounds U[i, j, k_top-1] += u[i, j, k] * Δzᶠᶜᶜ_reference(i, j, k, grid) - @inbounds V[i, j, k_top-1] += v[i, j, k] * Δzᶜᶠᶜ_reference(i, j, k, grid) + @inbounds U[i, j, k_top-1] += u[i, j, k] * Δrᶠᶜᶜ(i, j, k, grid) + @inbounds V[i, j, k_top-1] += v[i, j, k] * Δrᶜᶠᶜ(i, j, k, grid) end end diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 9eb015fe5e..7a5853c277 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -18,8 +18,11 @@ where ``η`` is the free surface height and ``H`` the vertical depth of the wate # Fields - `Δr`: reference vertical spacing with `η = 0` -- `sⁿ`: scaling of the vertical coordinate at time step `n` -- `s⁻`:scaling of the vertical coordinate at time step `n - 1` +- `sᶜᶜⁿ`: scaling of the vertical coordinate at time step `n` at `(Center, Center, Any)` location +- `sᶠᶜⁿ`: scaling of the vertical coordinate at time step `n` at `(Face, Center, Any)` location +- `sᶜᶠⁿ`: scaling of the vertical coordinate at time step `n` at `(Center, Face, Any)` location +- `sᶠᶠⁿ`: scaling of the vertical coordinate at time step `n` at `(Face, Face, Any)` location +- `s⁻`: scaling of the vertical coordinate at time step `n - 1` at `(Center, Center, Any)` location - `∂t_∂s`: Time derivative of `s` """ struct ZStarSpacing{R, SCC, SFC, SCF, SFF} <: AbstractVerticalSpacing{R} @@ -34,8 +37,11 @@ end Adapt.adapt_structure(to, coord::ZStarSpacing) = ZStarSpacing(Adapt.adapt(to, coord.Δr), - Adapt.adapt(to, coord.s⁻), - Adapt.adapt(to, coord.sⁿ), + Adapt.adapt(to, coord.sᶜᶜⁿ), + Adapt.adapt(to, coord.sᶠᶜⁿ), + Adapt.adapt(to, coord.sᶜᶠⁿ), + Adapt.adapt(to, coord.sᶠᶠⁿ), + Adapt.adapt(to, coord.sᶜᶜ⁻), Adapt.adapt(to, coord.∂t_∂s)) on_architecture(arch, coord::ZStarSpacing) = @@ -109,20 +115,20 @@ end @inline vertical_scaling(grid::ZStarSpacingGrid) = grid.Δzᵃᵃᶠ.sᶜᶜⁿ @inline previous_vertical_scaling(grid::ZStarSpacingGrid) = grid.Δzᵃᵃᶠ.sᶜᶜ⁻ -reference_Δzᵃᵃᶠ(grid::ZStarSpacingGrid) = grid.Δzᵃᵃᶠ.Δr -reference_Δzᵃᵃᶜ(grid::ZStarSpacingGrid) = grid.Δzᵃᵃᶜ.Δr +reference_zspacings(grid::ZStarSpacingGrid, ::Face) = grid.Δzᵃᵃᶠ.Δr +reference_zspacings(grid::ZStarSpacingGrid, ::Center) = grid.Δzᵃᵃᶜ.Δr -@inline Δzᶜᶜᶠ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δzᶜᶜᶠ_reference(i, j, k, grid) * grid.Δzᵃᵃᶠ.sᶜᶜⁿ[i, j, 1] -@inline Δzᶜᶜᶜ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δzᶜᶜᶜ_reference(i, j, k, grid) * grid.Δzᵃᵃᶜ.sᶜᶜⁿ[i, j, 1] +@inline Δzᶜᶜᶠ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δrᶜᶜᶠ(i, j, k, grid) * grid.Δzᵃᵃᶠ.sᶜᶜⁿ[i, j, 1] +@inline Δzᶜᶜᶜ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δrᶜᶜᶜ(i, j, k, grid) * grid.Δzᵃᵃᶜ.sᶜᶜⁿ[i, j, 1] -@inline Δzᶠᶜᶠ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δzᶜᶜᶠ_reference(i, j, k, grid) * grid.Δzᵃᵃᶠ.sᶠᶜⁿ[i, j, 1] -@inline Δzᶠᶜᶜ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δzᶜᶜᶜ_reference(i, j, k, grid) * grid.Δzᵃᵃᶜ.sᶠᶜⁿ[i, j, 1] +@inline Δzᶠᶜᶠ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δrᶜᶜᶠ(i, j, k, grid) * grid.Δzᵃᵃᶠ.sᶠᶜⁿ[i, j, 1] +@inline Δzᶠᶜᶜ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δrᶜᶜᶜ(i, j, k, grid) * grid.Δzᵃᵃᶜ.sᶠᶜⁿ[i, j, 1] -@inline Δzᶜᶠᶠ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δzᶜᶜᶠ_reference(i, j, k, grid) * grid.Δzᵃᵃᶠ.sᶜᶠⁿ[i, j, 1] -@inline Δzᶜᶠᶜ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δzᶜᶜᶜ_reference(i, j, k, grid) * grid.Δzᵃᵃᶜ.sᶜᶠⁿ[i, j, 1] +@inline Δzᶜᶠᶠ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δrᶜᶜᶠ(i, j, k, grid) * grid.Δzᵃᵃᶠ.sᶜᶠⁿ[i, j, 1] +@inline Δzᶜᶠᶜ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δrᶜᶜᶜ(i, j, k, grid) * grid.Δzᵃᵃᶜ.sᶜᶠⁿ[i, j, 1] -@inline Δzᶠᶠᶠ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δzᶜᶜᶠ_reference(i, j, k, grid) * grid.Δzᵃᵃᶠ.sᶠᶠⁿ[i, j, 1] -@inline Δzᶠᶠᶜ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δzᶜᶜᶜ_reference(i, j, k, grid) * grid.Δzᵃᵃᶜ.sᶠᶠⁿ[i, j, 1] +@inline Δzᶠᶠᶠ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δrᶜᶜᶠ(i, j, k, grid) * grid.Δzᵃᵃᶠ.sᶠᶠⁿ[i, j, 1] +@inline Δzᶠᶠᶜ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δrᶜᶜᶜ(i, j, k, grid) * grid.Δzᵃᵃᶜ.sᶠᶠⁿ[i, j, 1] @inline ∂t_∂s_grid(i, j, k, grid::ZStarSpacingGrid) = grid.Δzᵃᵃᶜ.∂t_∂s[i, j, k] @inline V_times_∂t_∂s_grid(i, j, k, grid::ZStarSpacingGrid) = ∂t_∂s_grid(i, j, k, grid) * Vᶜᶜᶜ(i, j, k, grid) diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index b0eed02717..154be46329 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -2,7 +2,7 @@ using Oceananigans using Oceananigans.Units using Oceananigans.Utils: prettytime using Oceananigans.Advection: WENOVectorInvariant -using Oceananigans.Models.HydrostaticFreeSurfaceModels: ZStar, ZStarSpacingGrid, Δzᶜᶜᶜ_reference +using Oceananigans.Models.HydrostaticFreeSurfaceModels: ZStar, ZStarSpacingGrid, Δrᶜᶜᶜ using Printf grid = RectilinearGrid(size = (128, 20), @@ -11,7 +11,7 @@ grid = RectilinearGrid(size = (128, 20), halo = (6, 6), topology = (Bounded, Flat, Bounded)) -grid = ImmersedBoundaryGrid(grid, GridFittedBottom(x -> x < 32kilometers ? -10 : -20)) +# grid = ImmersedBoundaryGrid(grid, GridFittedBottom(x -> x < 32kilometers ? -10 : -20)) model = HydrostaticFreeSurfaceModel(; grid, generalized_vertical_coordinate = ZStar(), From d63ee3167a893c4439573667e15dcd4c53e86382 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 25 Sep 2024 12:18:01 -0400 Subject: [PATCH 293/567] change name to vertical coordinate --- .../hydrostatic_free_surface_model.jl | 10 +++++----- test/test_zstar_coordinate.jl | 2 +- validation/z_star_coordinate/lock_release.jl | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl index 4717f231fb..768251cef5 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl @@ -120,7 +120,7 @@ function HydrostaticFreeSurfaceModel(; grid, velocities = nothing, pressure = nothing, diffusivity_fields = nothing, - generalized_vertical_coordinate = nothing, + vertical_coordinate = nothing, auxiliary_fields = NamedTuple() ) @@ -128,14 +128,14 @@ function HydrostaticFreeSurfaceModel(; grid, @apply_regionally validate_model_halo(grid, momentum_advection, tracer_advection, closure) # Introduce z-star coordinates if needed (only is free_surface is not a nothing) - # if !isnothing(generalized_vertical_coordinate) && !(momentum_advection isa VectorInvariant) && isnothing(velocities) + # if !isnothing(vertical_coordinate) && !(momentum_advection isa VectorInvariant) && isnothing(velocities) # throw(ArgumentError("Generalized vertical coordinates are supported only for the vector-invariant form of the momentum equations")) # end - if !(free_surface isa SplitExplicitFreeSurface) && !isnothing(generalized_vertical_coordinate) - @warn "Generalized vertical coordinates are supported only for the SplitExplicitFreeSurface at the moment. Ignoring the generalized_vertical_coordinate argument." + if !(free_surface isa SplitExplicitFreeSurface) && !isnothing(vertical_coordinate) + @warn "Generalized vertical coordinates are supported only for the SplitExplicitFreeSurface at the moment. Ignoring the vertical_coordinate argument." else - grid = generalized_spacing_grid(grid, generalized_vertical_coordinate) + grid = generalized_spacing_grid(grid, vertical_coordinate) end arch = architecture(grid) diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index 2978573b08..efdf35b6c6 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -49,7 +49,7 @@ end free_surface, tracers = (:b, :c), bouyancy = BuoyancTracer(), - generalized_vertical_coordinate = ZStar()) + vertical_coordinate = ZStar()) bᵢ(x, y, z) = x < grid.Lx / 2 ? 0.06 : 0.01 diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index 154be46329..300b213e68 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -14,7 +14,7 @@ grid = RectilinearGrid(size = (128, 20), # grid = ImmersedBoundaryGrid(grid, GridFittedBottom(x -> x < 32kilometers ? -10 : -20)) model = HydrostaticFreeSurfaceModel(; grid, - generalized_vertical_coordinate = ZStar(), + vertical_coordinate = ZStar(), momentum_advection = WENO(; order = 5), tracer_advection = WENO(; order = 5), buoyancy = BuoyancyTracer(), From 78adaf112d97d1055b017a7aa72d35577eaa464a Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 25 Sep 2024 12:26:07 -0400 Subject: [PATCH 294/567] add a dynamic column height --- .../split_explicit_free_surface_kernels.jl | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 1dec64c865..94ebdf3da1 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -157,13 +157,20 @@ end return nothing end +# Linear free surface implementation @inline dynamic_column_heightᶠᶜ(i, j, k, grid, H, η) = @inbounds H[i, j, 1] @inline dynamic_column_heightᶜᶠ(i, j, k, grid, H, η) = @inbounds H[i, j, 1] +@inline grid_scaling_factorᶠᶜ(i, j, k, grid, H, η) = one(grid) +@inline grid_scaling_factorᶜᶠ(i, j, k, grid, H, η) = one(grid) + # Non-linear free surface implementation @inline dynamic_column_heightᶠᶜ(i, j, k, grid::ZStarSpacingGrid, H, η) = @inbounds H[i, j, 1] + ℑxᶠᵃᵃ(i, j, k, grid, η) @inline dynamic_column_heightᶜᶠ(i, j, k, grid::ZStarSpacingGrid, H, η) = @inbounds H[i, j, 1] + ℑyᵃᶠᵃ(i, j, k, grid, η) +@inline grid_scaling_factorᶠᶜ(i, j, k, grid::ZStarSpacingGrid, H, η) = @inbounds dynamic_column_heightᶠᶜ(i, j, k, grid, H, η) / H[i, j, 1] +@inline grid_scaling_factorᶜᶠ(i, j, k, grid::ZStarSpacingGrid, H, η) = @inbounds dynamic_column_heightᶜᶠ(i, j, k, grid, H, η) / H[i, j, 1] + @kernel function _split_explicit_barotropic_velocity!(averaging_weight, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅, Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, g, @@ -212,13 +219,16 @@ end i, j = @index(Global, NTuple) k_top = grid.Nz + 1 + sᶠᶜ = grid_scaling_factorᶠᶜ(i, j, k_top, grid, Hᶠᶜ, η̅) + sᶜᶠ = grid_scaling_factorᶜᶠ(i, j, k_top, grid, Hᶜᶠ, η̅) + # hand unroll first loop - @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δrᶠᶜᶜ(i, j, 1, grid) - @inbounds V[i, j, k_top-1] = v[i, j, 1] * Δrᶜᶠᶜ(i, j, 1, grid) + @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δrᶠᶜᶜ(i, j, 1, grid) * sᶠᶜ + @inbounds V[i, j, k_top-1] = v[i, j, 1] * Δrᶜᶠᶜ(i, j, 1, grid) * sᶜᶠ @unroll for k in 2:grid.Nz - @inbounds U[i, j, k_top-1] += u[i, j, k] * Δrᶠᶜᶜ(i, j, k, grid) - @inbounds V[i, j, k_top-1] += v[i, j, k] * Δrᶜᶠᶜ(i, j, k, grid) + @inbounds U[i, j, k_top-1] += u[i, j, k] * Δrᶠᶜᶜ(i, j, k, grid) * sᶠᶜ + @inbounds V[i, j, k_top-1] += v[i, j, k] * Δrᶜᶠᶜ(i, j, k, grid) * sᶜᶠ end end @@ -227,13 +237,16 @@ end i, j = active_linear_index_to_tuple(idx, active_cells_map) k_top = grid.Nz+1 + sᶠᶜ = grid_scaling_factorᶠᶜ(i, j, k_top, grid, Hᶠᶜ, η̅) + sᶜᶠ = grid_scaling_factorᶜᶠ(i, j, k_top, grid, Hᶜᶠ, η̅) + # hand unroll first loop - @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δrᶠᶜᶜ(i, j, 1, grid) - @inbounds V[i, j, k_top-1] = v[i, j, 1] * Δrᶜᶠᶜ(i, j, 1, grid) + @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δrᶠᶜᶜ(i, j, 1, grid) * sᶠᶜ + @inbounds V[i, j, k_top-1] = v[i, j, 1] * Δrᶜᶠᶜ(i, j, 1, grid) * sᶜᶠ @unroll for k in 2:grid.Nz - @inbounds U[i, j, k_top-1] += u[i, j, k] * Δrᶠᶜᶜ(i, j, k, grid) - @inbounds V[i, j, k_top-1] += v[i, j, k] * Δrᶜᶠᶜ(i, j, k, grid) + @inbounds U[i, j, k_top-1] += u[i, j, k] * Δrᶠᶜᶜ(i, j, k, grid) * sᶠᶜ + @inbounds V[i, j, k_top-1] += v[i, j, k] * Δrᶜᶠᶜ(i, j, k, grid) * sᶜᶠ end end @@ -250,8 +263,8 @@ end k_top = grid.Nz + 1 @inbounds begin - u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / Hᶠᶜ[i, j, 1] - v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / Hᶜᶠ[i, j, 1] + u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / dynamic_column_heightᶠᶜ(i, j, k_top, grid, Hᶠᶜ, η̅) + v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / dynamic_column_heightᶜᶠ(i, j, k_top, grid, Hᶜᶠ, η̅) end end From 24739ee51b8f9a3de06cfd29e9c8fbed608000b6 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 25 Sep 2024 15:51:20 -0400 Subject: [PATCH 295/567] some chamges --- .../generalized_vertical_spacing.jl | 30 +++++-- .../split_explicit_free_surface.jl | 46 +++++----- .../split_explicit_free_surface_kernels.jl | 87 +++++++++++++------ .../z_star_vertical_spacing.jl | 60 ++++++++----- validation/z_star_coordinate/lock_release.jl | 2 +- 5 files changed, 143 insertions(+), 82 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index 575e04756f..ad3d065fff 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -138,6 +138,18 @@ update_vertical_spacing!(model, grid; kwargs...) = nothing @inline Δrᶠᶠᶠ(i, j, k, grid) = ℑxyᶠᶠᵃ(i, j, k, grid, Δrᶜᶜᶠ) @inline Δrᶠᶠᶜ(i, j, k, grid) = ℑxyᶠᶠᵃ(i, j, k, grid, Δrᶜᶜᶜ) +@inline rnode(i, j, k, grid, ℓx, ℓy, ℓz) = znode(i, j, k, grid, ℓx, ℓy, ℓz) + +@inline Δzᶜᶜᶠ(i, j, k, grid::AbstractVerticalSpacingGrid) = @inbounds Δrᶜᶜᶠ(i, j, k, grid) * vertical_scaling(i, j, k, grid, c, c, f) +@inline Δzᶜᶜᶜ(i, j, k, grid::AbstractVerticalSpacingGrid) = @inbounds Δrᶜᶜᶜ(i, j, k, grid) * vertical_scaling(i, j, k, grid, c, c, c) +@inline Δzᶠᶜᶠ(i, j, k, grid::AbstractVerticalSpacingGrid) = @inbounds Δrᶠᶜᶠ(i, j, k, grid) * vertical_scaling(i, j, k, grid, f, c, f) +@inline Δzᶠᶜᶜ(i, j, k, grid::AbstractVerticalSpacingGrid) = @inbounds Δrᶠᶜᶜ(i, j, k, grid) * vertical_scaling(i, j, k, grid, f, c, c) +@inline Δzᶜᶠᶠ(i, j, k, grid::AbstractVerticalSpacingGrid) = @inbounds Δrᶜᶠᶠ(i, j, k, grid) * vertical_scaling(i, j, k, grid, c, f, f) +@inline Δzᶜᶠᶜ(i, j, k, grid::AbstractVerticalSpacingGrid) = @inbounds Δrᶜᶠᶜ(i, j, k, grid) * vertical_scaling(i, j, k, grid, f, c, c) +@inline Δzᶠᶠᶠ(i, j, k, grid::AbstractVerticalSpacingGrid) = @inbounds Δrᶠᶠᶠ(i, j, k, grid) * vertical_scaling(i, j, k, grid, f, f, f) +@inline Δzᶠᶠᶜ(i, j, k, grid::AbstractVerticalSpacingGrid) = @inbounds Δrᶠᶠᶜ(i, j, k, grid) * vertical_scaling(i, j, k, grid, f, f, c) + + ##### ##### Additional terms to be included in the momentum equations (fallbacks) ##### @@ -152,26 +164,28 @@ update_vertical_spacing!(model, grid; kwargs...) = nothing ##### We advance sθ but store θ once sⁿ⁺¹ is known ##### -@kernel function _ab2_step_tracer_generalized_spacing!(θ, sⁿ, s⁻, Δt, χ, Gⁿ, G⁻) +@kernel function _ab2_step_tracer_generalized_spacing!(θ, grid, Δt, χ, Gⁿ, G⁻) i, j, k = @index(Global, NTuple) FT = eltype(χ) C₁ = convert(FT, 1.5) + χ C₂ = convert(FT, 0.5) + χ + sⁿ = vertical_scaling(i, j, k, grid, c, c, c) + s⁻ = previous_vertical_scaling(i, j, k, grid, f, f, f) + @inbounds begin - ∂t_∂sθ = C₁ * sⁿ[i, j, k] * Gⁿ[i, j, k] - C₂ * s⁻[i, j, k] * G⁻[i, j, k] + ∂t_∂sθ = C₁ * sⁿ * Gⁿ[i, j, k] - C₂ * s⁻ * G⁻[i, j, k] # We store temporarily sθ in θ. the unscaled θ will be retrived later on with `unscale_tracers!` - θ[i, j, k] = sⁿ[i, j, k] * θ[i, j, k] + convert(FT, Δt) * ∂t_∂sθ + θ[i, j, k] = sⁿ * θ[i, j, k] + convert(FT, Δt) * ∂t_∂sθ end end ab2_step_tracer_field!(tracer_field, grid::AbstractVerticalSpacingGrid, Δt, χ, Gⁿ, G⁻) = launch!(architecture(grid), grid, :xyz, _ab2_step_tracer_generalized_spacing!, tracer_field, - vertical_scaling(grid), - previous_vertical_scaling(grid), + grid, Δt, χ, Gⁿ, G⁻) const EmptyTuples = Union{NamedTuple{(), Tuple{}}, Tuple{}} @@ -184,16 +198,16 @@ tracer_scaling_parameters(param::KernelParameters{S, O}, tracers, grid) where {S function unscale_tracers!(tracers, grid::AbstractVerticalSpacingGrid; parameters = :xy) parameters = tracer_scaling_parameters(parameters, tracers, grid) - launch!(architecture(grid), grid, parameters, _unscale_tracers!, tracers, vertical_scaling(grid), + launch!(architecture(grid), grid, parameters, _unscale_tracers!, tracers, grid, Val(grid.Hz), Val(grid.Nz)) return nothing end -@kernel function _unscale_tracers!(tracers, sⁿ, ::Val{Hz}, ::Val{Nz}) where {Hz, Nz} +@kernel function _unscale_tracers!(tracers, grid, ::Val{Hz}, ::Val{Nz}) where {Hz, Nz} i, j, n = @index(Global, NTuple) @unroll for k in -Hz+1:Nz+Hz - tracers[n][i, j, k] /= sⁿ[i, j, k] + tracers[n][i, j, k] /= vertical_scaling(i, j, k, grid, c, c, c) end end \ No newline at end of file diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index ebc09c9ae9..a86c88d878 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -158,6 +158,10 @@ Base.@kwdef struct SplitExplicitState{CC, ACC, FC, AFC, CF, ACF} U̅ :: FC "The time-filtered barotropic meridional velocity. (`ReducedField` over ``z``)" V̅ :: CF + "The time-filtered barotropic zonal velocity. (`ReducedField` over ``z``)" + Ũ :: FC + "The time-filtered barotropic meridional velocity. (`ReducedField` over ``z``)" + Ṽ :: CF end """ @@ -190,7 +194,10 @@ function SplitExplicitState(grid::AbstractGrid, timestepper) U̅ = XFaceField(grid, indices = (:, :, Nz)) V̅ = YFaceField(grid, indices = (:, :, Nz)) - return SplitExplicitState(; ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅) + Ũ = XFaceField(grid, indices = (:, :, Nz)) + Ṽ = YFaceField(grid, indices = (:, :, Nz)) + + return SplitExplicitState(; ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅, Ũ, Ṽ) end """ @@ -330,6 +337,7 @@ a fixed number of substeps with time step size of `fractional_step_size * Δt_ba struct FixedSubstepNumber{B, F} fractional_step_size :: B averaging_weights :: F + mass_flux_weights :: F end function FixedTimeStepSize(grid; @@ -362,7 +370,17 @@ end averaging_weights = averaging_weights[1:idx] averaging_weights ./= sum(averaging_weights) - return Δτ, tuple(averaging_weights...) + mass_flux_weights = similar(averaging_weights) + + averaging_weights ./= sum(averaging_weights) + + for i in substeps:-1:1 + mass_flux_weights[i] = sum(averaging_weights[i:substeps]) + end + + mass_flux_weights ./= sum(mass_flux_weights) + + return Δτ, tuple(averaging_weights...), tuple(mass_flux_weights...) end function SplitExplicitSettings(grid = nothing; @@ -407,32 +425,12 @@ function SplitExplicitSettings(grid = nothing; end end - fractional_step_size, averaging_weights = weights_from_substeps(FT, substeps, averaging_kernel) - substepping = FixedSubstepNumber(fractional_step_size, averaging_weights) + fractional_step_size, averaging_weights, mass_flux_weights = weights_from_substeps(FT, substeps, averaging_kernel) + substepping = FixedSubstepNumber(fractional_step_size, averaging_weights, mass_flux_weights) return SplitExplicitSettings(substepping, timestepper, settings_kwargs) end -#= -# mass_flux_weights = similar(averaging_weights) -# -# M = searchsortedfirst(τᶠ, 1) - 1 -# -# averaging_weights ./= sum(averaging_weights) -# -# for i in substeps:-1:1 -# mass_flux_weights[i] = 1 / M * sum(averaging_weights[i:substeps]) -# end -# -# mass_flux_weights ./= sum(mass_flux_weights) -# -# return SplitExplicitSettings(substeps, -# averaging_weights, -# mass_flux_weights, -# Δτ, -# timestepper) -=# - # Convenience Functions for grabbing free surface free_surface(free_surface::SplitExplicitFreeSurface) = free_surface.η diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 94ebdf3da1..b473920e3e 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -164,28 +164,28 @@ end @inline grid_scaling_factorᶠᶜ(i, j, k, grid, H, η) = one(grid) @inline grid_scaling_factorᶜᶠ(i, j, k, grid, H, η) = one(grid) -# Non-linear free surface implementation -@inline dynamic_column_heightᶠᶜ(i, j, k, grid::ZStarSpacingGrid, H, η) = @inbounds H[i, j, 1] + ℑxᶠᵃᵃ(i, j, k, grid, η) -@inline dynamic_column_heightᶜᶠ(i, j, k, grid::ZStarSpacingGrid, H, η) = @inbounds H[i, j, 1] + ℑyᵃᶠᵃ(i, j, k, grid, η) +# # # Non-linear free surface implementation +# @inline dynamic_column_heightᶠᶜ(i, j, k, grid::ZStarSpacingGrid, H, η) = @inbounds H[i, j, 1] + ℑxᶠᵃᵃ(i, j, k, grid, η) +# @inline dynamic_column_heightᶜᶠ(i, j, k, grid::ZStarSpacingGrid, H, η) = @inbounds H[i, j, 1] + ℑyᵃᶠᵃ(i, j, k, grid, η) -@inline grid_scaling_factorᶠᶜ(i, j, k, grid::ZStarSpacingGrid, H, η) = @inbounds dynamic_column_heightᶠᶜ(i, j, k, grid, H, η) / H[i, j, 1] -@inline grid_scaling_factorᶜᶠ(i, j, k, grid::ZStarSpacingGrid, H, η) = @inbounds dynamic_column_heightᶜᶠ(i, j, k, grid, H, η) / H[i, j, 1] +# @inline grid_scaling_factorᶠᶜ(i, j, k, grid::ZStarSpacingGrid, H, η) = @inbounds (H[i, j, 1] + ℑxᶠᵃᵃ(i, j, k, grid, η)) / H[i, j, 1] +# @inline grid_scaling_factorᶜᶠ(i, j, k, grid::ZStarSpacingGrid, H, η) = @inbounds (H[i, j, 1] + ℑyᵃᶠᵃ(i, j, k, grid, η)) / H[i, j, 1] -@kernel function _split_explicit_barotropic_velocity!(averaging_weight, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², +@kernel function _split_explicit_barotropic_velocity!(averaging_weight, mass_flux_weight, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, g, + η̅, U̅, V̅, Ũ, Ṽ, Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, g, timestepper) i, j = @index(Global, NTuple) velocity_evolution!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, averaging_weight, + η̅, U̅, V̅, Ũ, Ṽ, averaging_weight, mass_flux_weight, Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, g, timestepper) end @inline function velocity_evolution!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, averaging_weight, + η̅, U̅, V̅, Ũ, Ṽ, averaging_weight, mass_flux_weight, Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, g, timestepper) k_top = grid.Nz+1 @@ -207,6 +207,8 @@ end η̅[i, j, k_top] += averaging_weight * η[i, j, k_top] U̅[i, j, k_top-1] += averaging_weight * U[i, j, k_top-1] V̅[i, j, k_top-1] += averaging_weight * V[i, j, k_top-1] + Ũ[i, j, k_top-1] += mass_flux_weight * U[i, j, k_top-1] + Ṽ[i, j, k_top-1] += mass_flux_weight * V[i, j, k_top-1] end end @@ -261,16 +263,19 @@ end @kernel function _barotropic_split_explicit_corrector!(u, v, grid, U̅, V̅, U, V, Hᶠᶜ, Hᶜᶠ, η̅) i, j, k = @index(Global, NTuple) k_top = grid.Nz + 1 + + hᶠᶜ = dynamic_column_heightᶠᶜ(i, j, k_top, grid, Hᶠᶜ, η̅) + hᶜᶠ = dynamic_column_heightᶜᶠ(i, j, k_top, grid, Hᶜᶠ, η̅) @inbounds begin - u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / dynamic_column_heightᶠᶜ(i, j, k_top, grid, Hᶠᶜ, η̅) - v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / dynamic_column_heightᶜᶠ(i, j, k_top, grid, Hᶜᶠ, η̅) + u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / hᶠᶜ + v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / hᶜᶠ end end function barotropic_split_explicit_corrector!(u, v, free_surface, grid) sefs = free_surface.state - U, V, U̅, V̅ = sefs.U, sefs.V, sefs.U̅, sefs.V̅ + U, V, U̅, V̅ = sefs.U, sefs.V, sefs.U̅, sefs.V̅ #, sefs.Ũ, sefs.Ṽ Hᶠᶜ, Hᶜᶠ = free_surface.auxiliary.Hᶠᶜ, free_surface.auxiliary.Hᶜᶠ arch = architecture(grid) @@ -295,6 +300,8 @@ function initialize_free_surface_state!(state, η, timestepper) fill!(state.η̅, 0) fill!(state.U̅, 0) fill!(state.V̅, 0) + fill!(state.Ũ, 0) + fill!(state.Ṽ, 0) return nothing end @@ -346,7 +353,7 @@ function split_explicit_free_surface_step!(free_surface::SplitExplicitFreeSurfac Nsubsteps = calculate_substeps(settings.substepping, Δt) # barotropic time step as fraction of baroclinic step and averaging weights - fractional_Δt, weights = calculate_adaptive_settings(settings.substepping, Nsubsteps) + fractional_Δt, weights, mass_flux_weights = calculate_adaptive_settings(settings.substepping, Nsubsteps) Nsubsteps = length(weights) # barotropic time step in seconds @@ -357,7 +364,7 @@ function split_explicit_free_surface_step!(free_surface::SplitExplicitFreeSurfac initialize_free_surface_state!(free_surface.state, free_surface.η, settings.timestepper) # Solve for the free surface at tⁿ⁺¹ - iterate_split_explicit!(free_surface, free_surface_grid, Δτᴮ, weights, Val(Nsubsteps)) + iterate_split_explicit!(free_surface, free_surface_grid, Δτᴮ, weights, mass_flux_weights, Val(Nsubsteps)) # Reset eta for the next timestep set!(free_surface.η, free_surface.state.η̅) @@ -386,14 +393,14 @@ const MINIMUM_SUBSTEPS = 5 @inline calculate_substeps(substepping::FNS, Δt=nothing) = length(substepping.averaging_weights) @inline calculate_substeps(substepping::FTS, Δt) = max(MINIMUM_SUBSTEPS, ceil(Int, 2 * Δt / substepping.Δt_barotropic)) -@inline calculate_adaptive_settings(substepping::FNS, substeps) = substepping.fractional_step_size, substepping.averaging_weights +@inline calculate_adaptive_settings(substepping::FNS, substeps) = substepping.fractional_step_size, substepping.averaging_weights, substepping.mass_flux_weights @inline calculate_adaptive_settings(substepping::FTS, substeps) = weights_from_substeps(eltype(substepping.Δt_barotropic), substeps, substepping.averaging_kernel) const FixedSubstepsSetting{N} = SplitExplicitSettings{<:FixedSubstepNumber{<:Any, <:NTuple{N, <:Any}}} where N const FixedSubstepsSplitExplicit{F} = SplitExplicitFreeSurface{<:Any, <:Any, <:Any, <:Any, <:FixedSubstepsSetting{N}} where N -function iterate_split_explicit!(free_surface, grid, Δτᴮ, weights, ::Val{Nsubsteps}) where Nsubsteps +function iterate_split_explicit!(free_surface, grid, Δτᴮ, weights, mass_flux_weights, ::Val{Nsubsteps}) where Nsubsteps arch = architecture(grid) η = free_surface.η @@ -407,7 +414,7 @@ function iterate_split_explicit!(free_surface, grid, Δτᴮ, weights, ::Val{Nsu Uᵐ⁻¹, Uᵐ⁻² = state.Uᵐ⁻¹, state.Uᵐ⁻² Vᵐ⁻¹, Vᵐ⁻² = state.Vᵐ⁻¹, state.Vᵐ⁻² ηᵐ, ηᵐ⁻¹, ηᵐ⁻² = state.ηᵐ, state.ηᵐ⁻¹, state.ηᵐ⁻² - η̅, U̅, V̅ = state.η̅, state.U̅, state.V̅ + η̅, U̅, V̅, Ũ, Ṽ = state.η̅, state.U̅, state.V̅, state.Ũ, state.Ṽ Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ = auxiliary.Gᵁ, auxiliary.Gⱽ, auxiliary.Hᶠᶜ, auxiliary.Hᶜᶠ timestepper = settings.timestepper @@ -423,7 +430,7 @@ function iterate_split_explicit!(free_surface, grid, Δτᴮ, weights, ::Val{Nsu U_args = (grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, g, + η̅, U̅, V̅, Ũ, Ṽ, Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, g, timestepper) GC.@preserve η_args U_args begin @@ -438,8 +445,9 @@ function iterate_split_explicit!(free_surface, grid, Δτᴮ, weights, ::Val{Nsu @unroll for substep in 1:Nsubsteps Base.@_inline_meta averaging_weight = weights[substep] + mass_flux_weight = mass_flux_weights[substep] free_surface_kernel!(converted_η_args...) - barotropic_velocity_kernel!(averaging_weight, converted_U_args...) + barotropic_velocity_kernel!(averaging_weight, mass_flux_weight, converted_U_args...) end end @@ -451,12 +459,12 @@ end i, j = @index(Global, NTuple) k_top = grid.Nz + 1 - @inbounds Gᵁ[i, j, k_top-1] = Δzᶠᶜᶜ(i, j, 1, grid) * ab2_step_Gu(i, j, 1, grid, Gu⁻, Guⁿ, χ) - @inbounds Gⱽ[i, j, k_top-1] = Δzᶜᶠᶜ(i, j, 1, grid) * ab2_step_Gv(i, j, 1, grid, Gv⁻, Gvⁿ, χ) + @inbounds Gᵁ[i, j, k_top-1] = Δrᶠᶜᶜ(i, j, 1, grid) * ab2_step_Gu(i, j, 1, grid, Gu⁻, Guⁿ, χ) + @inbounds Gⱽ[i, j, k_top-1] = Δrᶜᶠᶜ(i, j, 1, grid) * ab2_step_Gv(i, j, 1, grid, Gv⁻, Gvⁿ, χ) for k in 2:grid.Nz - @inbounds Gᵁ[i, j, k_top-1] += Δzᶠᶜᶜ(i, j, k, grid) * ab2_step_Gu(i, j, k, grid, Gu⁻, Guⁿ, χ) - @inbounds Gⱽ[i, j, k_top-1] += Δzᶜᶠᶜ(i, j, k, grid) * ab2_step_Gv(i, j, k, grid, Gv⁻, Gvⁿ, χ) + @inbounds Gᵁ[i, j, k_top-1] += Δrᶠᶜᶜ(i, j, k, grid) * ab2_step_Gu(i, j, k, grid, Gu⁻, Guⁿ, χ) + @inbounds Gⱽ[i, j, k_top-1] += Δrᶜᶠᶜ(i, j, k, grid) * ab2_step_Gv(i, j, k, grid, Gv⁻, Gvⁿ, χ) end end @@ -466,15 +474,40 @@ end i, j = active_linear_index_to_tuple(idx, active_cells_map) k_top = grid.Nz+1 - @inbounds Gᵁ[i, j, k_top-1] = Δzᶠᶜᶜ(i, j, 1, grid) * ab2_step_Gu(i, j, 1, grid, Gu⁻, Guⁿ, χ) - @inbounds Gⱽ[i, j, k_top-1] = Δzᶜᶠᶜ(i, j, 1, grid) * ab2_step_Gv(i, j, 1, grid, Gv⁻, Gvⁿ, χ) + @inbounds Gᵁ[i, j, k_top-1] = Δrᶠᶜᶜ(i, j, 1, grid) * ab2_step_Gu(i, j, 1, grid, Gu⁻, Guⁿ, χ) + @inbounds Gⱽ[i, j, k_top-1] = Δrᶜᶠᶜ(i, j, 1, grid) * ab2_step_Gv(i, j, 1, grid, Gv⁻, Gvⁿ, χ) for k in 2:grid.Nz - @inbounds Gᵁ[i, j, k_top-1] += Δzᶠᶜᶜ(i, j, k, grid) * ab2_step_Gu(i, j, k, grid, Gu⁻, Guⁿ, χ) - @inbounds Gⱽ[i, j, k_top-1] += Δzᶜᶠᶜ(i, j, k, grid) * ab2_step_Gv(i, j, k, grid, Gv⁻, Gvⁿ, χ) + @inbounds Gᵁ[i, j, k_top-1] += Δrᶠᶜᶜ(i, j, k, grid) * ab2_step_Gu(i, j, k, grid, Gu⁻, Guⁿ, χ) + @inbounds Gⱽ[i, j, k_top-1] += Δrᶜᶠᶜ(i, j, k, grid) * ab2_step_Gv(i, j, k, grid, Gv⁻, Gvⁿ, χ) end end +@inline function ab2_step_Gu(i, j, k, grid::ZStarSpacingGrid, G⁻, Gⁿ, χ::FT) where FT + C¹ = convert(FT, 3/2) + χ + C² = convert(FT, 1/2) + χ + + Fⁿ = @inbounds C¹ * Gⁿ[i, j, k] * grid.Δzᵃᵃᶠ.sᶠᶜⁿ[i, j, 1] + F⁻ = @inbounds C² * G⁻[i, j, k] * grid.Δzᵃᵃᶠ.sᶠᶜ⁻[i, j, 1] + + Gi = Fⁿ - F⁻ + + return ifelse(peripheral_node(i, j, k, grid, f, c, c), zero(grid), Gi) +end + +@inline function ab2_step_Gv(i, j, k, grid::ZStarSpacingGrid, G⁻, Gⁿ, χ::FT) where FT + C¹ = convert(FT, 3/2) + χ + C² = convert(FT, 1/2) + χ + + Fⁿ = @inbounds C¹ * Gⁿ[i, j, k] * grid.Δzᵃᵃᶠ.sᶜᶠⁿ[i, j, 1] + F⁻ = @inbounds C² * G⁻[i, j, k] * grid.Δzᵃᵃᶠ.sᶜᶠ⁻[i, j, 1] + + Gi = Fⁿ - F⁻ + + return ifelse(peripheral_node(i, j, k, grid, f, c, c), zero(grid), Gi) +end + +# Setting up the RHS for the barotropic step (tendencies of the barotropic velocity components) @inline ab2_step_Gu(i, j, k, grid, G⁻, Gⁿ, χ::FT) where FT = @inbounds ifelse(peripheral_node(i, j, k, grid, f, c, c), zero(grid), (convert(FT, 1.5) + χ) * Gⁿ[i, j, k] - G⁻[i, j, k] * (convert(FT, 0.5) + χ)) diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 7a5853c277..24305fad2e 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -26,13 +26,15 @@ where ``η`` is the free surface height and ``H`` the vertical depth of the wate - `∂t_∂s`: Time derivative of `s` """ struct ZStarSpacing{R, SCC, SFC, SCF, SFF} <: AbstractVerticalSpacing{R} - Δr :: R + Δr :: R sᶜᶜⁿ :: SCC sᶠᶜⁿ :: SFC sᶜᶠⁿ :: SCF sᶠᶠⁿ :: SFF sᶜᶜ⁻ :: SCC - ∂t_∂s :: SCC + sᶠᶜ⁻ :: SFC + sᶜᶠ⁻ :: SCF + ∂t_∂s :: SCC end Adapt.adapt_structure(to, coord::ZStarSpacing) = @@ -42,6 +44,8 @@ Adapt.adapt_structure(to, coord::ZStarSpacing) = Adapt.adapt(to, coord.sᶜᶠⁿ), Adapt.adapt(to, coord.sᶠᶠⁿ), Adapt.adapt(to, coord.sᶜᶜ⁻), + Adapt.adapt(to, coord.sᶠᶜ⁻), + Adapt.adapt(to, coord.sᶜᶠ⁻), Adapt.adapt(to, coord.∂t_∂s)) on_architecture(arch, coord::ZStarSpacing) = @@ -77,7 +81,9 @@ function generalized_spacing_grid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, sᶜᶜ⁻ = Field{Center, Center, Nothing}(grid) sᶜᶜⁿ = Field{Center, Center, Nothing}(grid) + sᶠᶜ⁻ = Field{Face, Center, Nothing}(grid) sᶠᶜⁿ = Field{Face, Center, Nothing}(grid) + sᶜᶠ⁻ = Field{Center, Face, Nothing}(grid) sᶜᶠⁿ = Field{Center, Face, Nothing}(grid) sᶠᶠⁿ = Field{Face, Face, Nothing}(grid) ∂t_∂s = Field{Center, Center, Nothing}(grid) @@ -87,10 +93,12 @@ function generalized_spacing_grid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, fill!(sᶜᶜⁿ, 1) fill!(sᶠᶜⁿ, 1) fill!(sᶜᶠⁿ, 1) + fill!(sᶠᶜ⁻, 1) + fill!(sᶜᶠ⁻, 1) fill!(sᶠᶠⁿ, 1) - Δzᵃᵃᶠ = ZStarSpacing(grid.Δzᵃᵃᶠ, sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, ∂t_∂s) - Δzᵃᵃᶜ = ZStarSpacing(grid.Δzᵃᵃᶜ, sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, ∂t_∂s) + Δzᵃᵃᶠ = ZStarSpacing(grid.Δzᵃᵃᶠ, sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, sᶠᶜ⁻, sᶜᶠ⁻, ∂t_∂s) + Δzᵃᵃᶜ = ZStarSpacing(grid.Δzᵃᵃᶜ, sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, sᶠᶜ⁻, sᶜᶠ⁻, ∂t_∂s) args = [] for prop in propertynames(grid) @@ -112,27 +120,31 @@ end ##### ZStar-specific vertical spacing functions ##### -@inline vertical_scaling(grid::ZStarSpacingGrid) = grid.Δzᵃᵃᶠ.sᶜᶜⁿ -@inline previous_vertical_scaling(grid::ZStarSpacingGrid) = grid.Δzᵃᵃᶠ.sᶜᶜ⁻ +@inline vertical_scaling(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) +@inline vertical_scaling(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) +@inline vertical_scaling(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) -reference_zspacings(grid::ZStarSpacingGrid, ::Face) = grid.Δzᵃᵃᶠ.Δr -reference_zspacings(grid::ZStarSpacingGrid, ::Center) = grid.Δzᵃᵃᶜ.Δr - -@inline Δzᶜᶜᶠ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δrᶜᶜᶠ(i, j, k, grid) * grid.Δzᵃᵃᶠ.sᶜᶜⁿ[i, j, 1] -@inline Δzᶜᶜᶜ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δrᶜᶜᶜ(i, j, k, grid) * grid.Δzᵃᵃᶜ.sᶜᶜⁿ[i, j, 1] +@inline vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Center, ::Center, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶜⁿ[i, j, 1] +@inline vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Face, ::Center, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶠᶜⁿ[i, j, 1] +@inline vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Center, ::Face, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶠⁿ[i, j, 1] -@inline Δzᶠᶜᶠ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δrᶜᶜᶠ(i, j, k, grid) * grid.Δzᵃᵃᶠ.sᶠᶜⁿ[i, j, 1] -@inline Δzᶠᶜᶜ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δrᶜᶜᶜ(i, j, k, grid) * grid.Δzᵃᵃᶜ.sᶠᶜⁿ[i, j, 1] +@inline previous_vertical_scaling(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) +@inline previous_vertical_scaling(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) +@inline previous_vertical_scaling(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) -@inline Δzᶜᶠᶠ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δrᶜᶜᶠ(i, j, k, grid) * grid.Δzᵃᵃᶠ.sᶜᶠⁿ[i, j, 1] -@inline Δzᶜᶠᶜ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δrᶜᶜᶜ(i, j, k, grid) * grid.Δzᵃᵃᶜ.sᶜᶠⁿ[i, j, 1] +@inline previous_vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Center, ::Center, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶜ⁻[i, j, 1] +@inline previous_vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Face, ::Center, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶠᶜ⁻[i, j, 1] +@inline previous_vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Center, ::Face, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶠ⁻[i, j, 1] -@inline Δzᶠᶠᶠ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δrᶜᶜᶠ(i, j, k, grid) * grid.Δzᵃᵃᶠ.sᶠᶠⁿ[i, j, 1] -@inline Δzᶠᶠᶜ(i, j, k, grid::ZStarSpacingGrid) = @inbounds Δrᶜᶜᶜ(i, j, k, grid) * grid.Δzᵃᵃᶜ.sᶠᶠⁿ[i, j, 1] +reference_zspacings(grid::ZStarSpacingGrid, ::Face) = grid.Δzᵃᵃᶠ.Δr +reference_zspacings(grid::ZStarSpacingGrid, ::Center) = grid.Δzᵃᵃᶜ.Δr -@inline ∂t_∂s_grid(i, j, k, grid::ZStarSpacingGrid) = grid.Δzᵃᵃᶜ.∂t_∂s[i, j, k] +@inline ∂t_∂s_grid(i, j, k, grid::ZStarSpacingGrid) = grid.Δzᵃᵃᶜ.∂t_∂s[i, j, 1] @inline V_times_∂t_∂s_grid(i, j, k, grid::ZStarSpacingGrid) = ∂t_∂s_grid(i, j, k, grid) * Vᶜᶜᶜ(i, j, k, grid) +@inline rnode(i, j, k, grid, ℓx, ℓy, ::Face) = getnode(grid.zᵃᵃᶠ, k) +@inline rnode(i, j, k, grid, ℓx, ℓy, ::Center) = getnode(grid.zᵃᵃᶜ, k) + ##### ##### ZStar-specific vertical spacings update ##### @@ -142,7 +154,9 @@ function update_vertical_spacing!(model, grid::ZStarSpacingGrid; parameters = :x # Scaling sᶜᶜ⁻ = grid.Δzᵃᵃᶠ.sᶜᶜ⁻ sᶜᶜⁿ = grid.Δzᵃᵃᶠ.sᶜᶜⁿ + sᶠᶜ⁻ = grid.Δzᵃᵃᶠ.sᶠᶜ⁻ sᶠᶜⁿ = grid.Δzᵃᵃᶠ.sᶠᶜⁿ + sᶜᶠ⁻ = grid.Δzᵃᵃᶠ.sᶜᶠ⁻ sᶜᶠⁿ = grid.Δzᵃᵃᶠ.sᶜᶠⁿ sᶠᶠⁿ = grid.Δzᵃᵃᶠ.sᶠᶠⁿ ∂t_∂s = grid.Δzᵃᵃᶠ.∂t_∂s @@ -152,14 +166,14 @@ function update_vertical_spacing!(model, grid::ZStarSpacingGrid; parameters = :x Hᶠᶜ = model.free_surface.auxiliary.Hᶠᶜ Hᶜᶠ = model.free_surface.auxiliary.Hᶜᶠ Hᶠᶠ = model.free_surface.auxiliary.Hᶠᶠ - U̅ = model.free_surface.state.U̅ - V̅ = model.free_surface.state.V̅ + U̅ = model.free_surface.state.U̅ # Ũ + V̅ = model.free_surface.state.V̅ # Ṽ η = model.free_surface.η # Update vertical spacing with available parameters # No need to fill the halo as the scaling is updated _IN_ the halos launch!(architecture(grid), grid, parameters, _update_zstar!, - sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, η, Hᶜᶜ, Hᶠᶜ, Hᶜᶠ, Hᶠᶠ, grid) + sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, sᶠᶜ⁻, sᶜᶠ⁻, η, Hᶜᶜ, Hᶠᶜ, Hᶜᶠ, Hᶠᶠ, grid) # Update the time derivative of the grid-scaling. Note that in this case we leverage the # free surface evolution equation, where the time derivative of the free surface is equal @@ -181,7 +195,7 @@ end end end -@kernel function _update_zstar!(sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, η, Hᶜᶜ, Hᶠᶜ, Hᶜᶠ, Hᶠᶠ, grid) +@kernel function _update_zstar!(sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, sᶠᶜ⁻, sᶜᶠ⁻, η, Hᶜᶜ, Hᶠᶜ, Hᶜᶠ, Hᶠᶠ, grid) i, j = @index(Global, NTuple) k_top = grid.Nz+1 @inbounds begin @@ -191,6 +205,8 @@ end hᶠᶠ = (Hᶠᶠ[i, j, 1] + ℑxyᶠᶠᵃ(i, j, k_top, grid, η)) / Hᶠᶠ[i, j, 1] sᶜᶜ⁻[i, j, 1] = sᶜᶜⁿ[i, j, 1] + sᶠᶜ⁻[i, j, 1] = sᶠᶜⁿ[i, j, 1] + sᶜᶠ⁻[i, j, 1] = sᶜᶠⁿ[i, j, 1] # update current and previous scaling sᶜᶜⁿ[i, j, 1] = hᶜᶜ diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index 300b213e68..334a973ea3 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -5,7 +5,7 @@ using Oceananigans.Advection: WENOVectorInvariant using Oceananigans.Models.HydrostaticFreeSurfaceModels: ZStar, ZStarSpacingGrid, Δrᶜᶜᶜ using Printf -grid = RectilinearGrid(size = (128, 20), +grid = RectilinearGrid(size = (10, 20), x = (0, 64kilometers), z = (-20, 0), halo = (6, 6), From a741108d689c51fbc06de7b2a901e486aee6b394 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 25 Sep 2024 17:03:41 -0400 Subject: [PATCH 296/567] getting there --- .../compute_w_from_continuity.jl | 18 +++-- .../generalized_vertical_spacing.jl | 26 +++---- .../split_explicit_free_surface_kernels.jl | 69 ++++++++----------- .../z_star_vertical_spacing.jl | 8 --- validation/z_star_coordinate/lock_release.jl | 2 +- 5 files changed, 50 insertions(+), 73 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl index f3049c48a8..c649288175 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl @@ -1,7 +1,7 @@ using Oceananigans.Architectures: device using Oceananigans.Grids: halo_size, topology using Oceananigans.Grids: XFlatGrid, YFlatGrid -using Oceananigans.Operators: div_xyᶜᶜᶜ, Δzᶜᶜᶜ +using Oceananigans.Operators: flux_div_xyᶜᶜᶜ, div_xyᶜᶜᶜ, Δzᶜᶜᶜ """ compute_w_from_continuity!(model) @@ -18,14 +18,24 @@ compute_w_from_continuity!(model; kwargs...) = compute_w_from_continuity!(velocities, arch, grid; parameters = w_kernel_parameters(grid)) = launch!(arch, grid, parameters, _compute_w_from_continuity!, velocities, grid) + +@inline Ax_uᶠᶜᶜ⁻(i, j, k, grid, u) = @inbounds vertical_scaling(i, j, k, grid, f, c, c) * Δrᶠᶜᶜ(i, j, k, grid) * Δyᶠᶜᶜ(i, j, k, grid) * u[i, j, k] +@inline Ay_vᶜᶠᶜ⁻(i, j, k, grid, v) = @inbounds vertical_scaling(i, j, k, grid, c, f, c) * Δrᶜᶠᶜ(i, j, k, grid) * Δxᶜᶠᶜ(i, j, k, grid) * v[i, j, k] + +@inline function scaled_velocity_divergenceᶜᶜᶜ(i, j, k, grid, u, v) + δU = δxᶜᶜᶜ(i, j, k, grid, Ax_uᶠᶜᶜ⁻, u) + δV = δyᶜᶜᶜ(i, j, k, grid, Ay_vᶜᶠᶜ⁻, v) + return (δU + δV) / Azᶜᶜᶜ(i, j, k, grid) +end + @kernel function _compute_w_from_continuity!(U, grid) i, j = @index(Global, NTuple) @inbounds U.w[i, j, 1] = 0 for k in 2:grid.Nz+1 - @inbounds U.w[i, j, k] = U.w[i, j, k-1] - Δzᶜᶜᶜ(i, j, k-1, grid) * - ( div_xyᶜᶜᶜ(i, j, k-1, grid, U.u, U.v) + - ∂t_∂s_grid(i, j, k-1, grid) ) + @inbounds U.w[i, j, k] = U.w[i, j, k-1] - + ( scaled_velocity_divergenceᶜᶜᶜ(i, j, k-1, grid, U.u, U.v) + + Δrᶜᶜᶜ(i, j, k-1, grid) * ∂t_∂s_grid(i, j, k-1, grid) ) end end diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index ad3d065fff..5f6a748cb8 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -99,23 +99,6 @@ end generalized_spacing_grid(grid, coord) = grid update_vertical_spacing!(model, grid; kwargs...) = nothing -##### -##### Vertical spacings for a generalized vertical coordinate system -##### - -# Very bad for GPU performance!!! (z-values are not coalesced in memory for z-derivatives anymore) -# TODO: make z-direction local in memory by not using Fields -# TODO: make it work with partial cells - -@inline Δzᶜᶠᶠ(i, j, k, grid::AbstractVerticalSpacingGrid) = ℑyᵃᶠᵃ(i, j, k, grid, Δzᶜᶜᶠ) -@inline Δzᶜᶠᶜ(i, j, k, grid::AbstractVerticalSpacingGrid) = ℑyᵃᶠᵃ(i, j, k, grid, Δzᶜᶜᶜ) - -@inline Δzᶠᶜᶠ(i, j, k, grid::AbstractVerticalSpacingGrid) = ℑxᶠᵃᵃ(i, j, k, grid, Δzᶜᶜᶠ) -@inline Δzᶠᶜᶜ(i, j, k, grid::AbstractVerticalSpacingGrid) = ℑxᶠᵃᵃ(i, j, k, grid, Δzᶜᶜᶜ) - -@inline Δzᶠᶠᶠ(i, j, k, grid::AbstractVerticalSpacingGrid) = ℑxyᶠᶠᵃ(i, j, k, grid, Δzᶜᶜᶠ) -@inline Δzᶠᶠᶜ(i, j, k, grid::AbstractVerticalSpacingGrid) = ℑxyᶠᶠᵃ(i, j, k, grid, Δzᶜᶜᶜ) - ##### ##### Reference (local) vertical spacings in `r-coordinates` ##### @@ -140,6 +123,14 @@ update_vertical_spacing!(model, grid; kwargs...) = nothing @inline rnode(i, j, k, grid, ℓx, ℓy, ℓz) = znode(i, j, k, grid, ℓx, ℓy, ℓz) +##### +##### Vertical spacings for a generalized vertical coordinate system +##### + +# Fallbacks +@inline previous_vertical_scaling(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) +@inline vertical_scaling(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) + @inline Δzᶜᶜᶠ(i, j, k, grid::AbstractVerticalSpacingGrid) = @inbounds Δrᶜᶜᶠ(i, j, k, grid) * vertical_scaling(i, j, k, grid, c, c, f) @inline Δzᶜᶜᶜ(i, j, k, grid::AbstractVerticalSpacingGrid) = @inbounds Δrᶜᶜᶜ(i, j, k, grid) * vertical_scaling(i, j, k, grid, c, c, c) @inline Δzᶠᶜᶠ(i, j, k, grid::AbstractVerticalSpacingGrid) = @inbounds Δrᶠᶜᶠ(i, j, k, grid) * vertical_scaling(i, j, k, grid, f, c, f) @@ -149,7 +140,6 @@ update_vertical_spacing!(model, grid; kwargs...) = nothing @inline Δzᶠᶠᶠ(i, j, k, grid::AbstractVerticalSpacingGrid) = @inbounds Δrᶠᶠᶠ(i, j, k, grid) * vertical_scaling(i, j, k, grid, f, f, f) @inline Δzᶠᶠᶜ(i, j, k, grid::AbstractVerticalSpacingGrid) = @inbounds Δrᶠᶠᶜ(i, j, k, grid) * vertical_scaling(i, j, k, grid, f, f, c) - ##### ##### Additional terms to be included in the momentum equations (fallbacks) ##### diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index b473920e3e..1293d44d0b 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -161,31 +161,27 @@ end @inline dynamic_column_heightᶠᶜ(i, j, k, grid, H, η) = @inbounds H[i, j, 1] @inline dynamic_column_heightᶜᶠ(i, j, k, grid, H, η) = @inbounds H[i, j, 1] -@inline grid_scaling_factorᶠᶜ(i, j, k, grid, H, η) = one(grid) -@inline grid_scaling_factorᶜᶠ(i, j, k, grid, H, η) = one(grid) +# Non-linear free surface implementation +# At the moment the dynamic height is frozen in time so it is in practice a linear equation in the substepping +# TODO: Implement the full non-linear free surface model by having the dynamic height evolve in time during substepping +@inline dynamic_column_heightᶠᶜ(i, j, k, grid::ZStarSpacingGrid, H, η) = @inbounds H[i, j, 1] * vertical_scaling(i, j, k, grid, f, c, c) +@inline dynamic_column_heightᶜᶠ(i, j, k, grid::ZStarSpacingGrid, H, η) = @inbounds H[i, j, 1] * vertical_scaling(i, j, k, grid, c, f, c) -# # # Non-linear free surface implementation -# @inline dynamic_column_heightᶠᶜ(i, j, k, grid::ZStarSpacingGrid, H, η) = @inbounds H[i, j, 1] + ℑxᶠᵃᵃ(i, j, k, grid, η) -# @inline dynamic_column_heightᶜᶠ(i, j, k, grid::ZStarSpacingGrid, H, η) = @inbounds H[i, j, 1] + ℑyᵃᶠᵃ(i, j, k, grid, η) - -# @inline grid_scaling_factorᶠᶜ(i, j, k, grid::ZStarSpacingGrid, H, η) = @inbounds (H[i, j, 1] + ℑxᶠᵃᵃ(i, j, k, grid, η)) / H[i, j, 1] -# @inline grid_scaling_factorᶜᶠ(i, j, k, grid::ZStarSpacingGrid, H, η) = @inbounds (H[i, j, 1] + ℑyᵃᶠᵃ(i, j, k, grid, η)) / H[i, j, 1] - -@kernel function _split_explicit_barotropic_velocity!(averaging_weight, mass_flux_weight, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², +@kernel function _split_explicit_barotropic_velocity!(averaging_weight, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, Ũ, Ṽ, Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, g, + η̅, U̅, V̅, Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, g, timestepper) i, j = @index(Global, NTuple) velocity_evolution!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, Ũ, Ṽ, averaging_weight, mass_flux_weight, + η̅, U̅, V̅, averaging_weight, Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, g, timestepper) end @inline function velocity_evolution!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, Ũ, Ṽ, averaging_weight, mass_flux_weight, + η̅, U̅, V̅, averaging_weight, Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, g, timestepper) k_top = grid.Nz+1 @@ -207,8 +203,6 @@ end η̅[i, j, k_top] += averaging_weight * η[i, j, k_top] U̅[i, j, k_top-1] += averaging_weight * U[i, j, k_top-1] V̅[i, j, k_top-1] += averaging_weight * V[i, j, k_top-1] - Ũ[i, j, k_top-1] += mass_flux_weight * U[i, j, k_top-1] - Ṽ[i, j, k_top-1] += mass_flux_weight * V[i, j, k_top-1] end end @@ -217,45 +211,39 @@ end # For Zstar vertical spacing the vertical integral includes the dynamic height # Remember, the vertical coordinate has not yet been updated! # For this reason the integration has to be performed manually -@kernel function _barotropic_mode_kernel!(U, V, grid, ::Nothing, u, v, Hᶠᶜ, Hᶜᶠ, η̅) +@kernel function _barotropic_mode_kernel!(U, V, grid, ::Nothing, u, v) i, j = @index(Global, NTuple) k_top = grid.Nz + 1 - sᶠᶜ = grid_scaling_factorᶠᶜ(i, j, k_top, grid, Hᶠᶜ, η̅) - sᶜᶠ = grid_scaling_factorᶜᶠ(i, j, k_top, grid, Hᶜᶠ, η̅) - # hand unroll first loop - @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δrᶠᶜᶜ(i, j, 1, grid) * sᶠᶜ - @inbounds V[i, j, k_top-1] = v[i, j, 1] * Δrᶜᶠᶜ(i, j, 1, grid) * sᶜᶠ + @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δzᶠᶜᶜ(i, j, 1, grid) + @inbounds V[i, j, k_top-1] = v[i, j, 1] * Δzᶜᶠᶜ(i, j, 1, grid) @unroll for k in 2:grid.Nz - @inbounds U[i, j, k_top-1] += u[i, j, k] * Δrᶠᶜᶜ(i, j, k, grid) * sᶠᶜ - @inbounds V[i, j, k_top-1] += v[i, j, k] * Δrᶜᶠᶜ(i, j, k, grid) * sᶜᶠ + @inbounds U[i, j, k_top-1] += u[i, j, k] * Δzᶠᶜᶜ(i, j, k, grid) + @inbounds V[i, j, k_top-1] += v[i, j, k] * Δzᶜᶠᶜ(i, j, k, grid) end end -@kernel function _barotropic_mode_kernel!(U, V, grid, active_cells_map, u, v, Hᶠᶜ, Hᶜᶠ, η̅) +@kernel function _barotropic_mode_kernel!(U, V, grid, active_cells_map, u, v) idx = @index(Global, Linear) i, j = active_linear_index_to_tuple(idx, active_cells_map) k_top = grid.Nz+1 - sᶠᶜ = grid_scaling_factorᶠᶜ(i, j, k_top, grid, Hᶠᶜ, η̅) - sᶜᶠ = grid_scaling_factorᶜᶠ(i, j, k_top, grid, Hᶜᶠ, η̅) - # hand unroll first loop - @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δrᶠᶜᶜ(i, j, 1, grid) * sᶠᶜ - @inbounds V[i, j, k_top-1] = v[i, j, 1] * Δrᶜᶠᶜ(i, j, 1, grid) * sᶜᶠ + @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δzᶠᶜᶜ(i, j, 1, grid) + @inbounds V[i, j, k_top-1] = v[i, j, 1] * Δzᶜᶠᶜ(i, j, 1, grid) @unroll for k in 2:grid.Nz - @inbounds U[i, j, k_top-1] += u[i, j, k] * Δrᶠᶜᶜ(i, j, k, grid) * sᶠᶜ - @inbounds V[i, j, k_top-1] += v[i, j, k] * Δrᶜᶠᶜ(i, j, k, grid) * sᶜᶠ + @inbounds U[i, j, k_top-1] += u[i, j, k] * Δzᶠᶜᶜ(i, j, k, grid) + @inbounds V[i, j, k_top-1] += v[i, j, k] * Δzᶜᶠᶜ(i, j, k, grid) end end @inline function compute_barotropic_mode!(U, V, grid, u, v, Hᶠᶜ, Hᶜᶠ, η̅) active_cells_map = retrieve_surface_active_cells_map(grid) - launch!(architecture(grid), grid, :xy, _barotropic_mode_kernel!, U, V, grid, active_cells_map, u, v, Hᶠᶜ, Hᶜᶠ, η̅; active_cells_map) + launch!(architecture(grid), grid, :xy, _barotropic_mode_kernel!, U, V, grid, active_cells_map, u, v; active_cells_map) return nothing end @@ -300,8 +288,6 @@ function initialize_free_surface_state!(state, η, timestepper) fill!(state.η̅, 0) fill!(state.U̅, 0) fill!(state.V̅, 0) - fill!(state.Ũ, 0) - fill!(state.Ṽ, 0) return nothing end @@ -353,7 +339,7 @@ function split_explicit_free_surface_step!(free_surface::SplitExplicitFreeSurfac Nsubsteps = calculate_substeps(settings.substepping, Δt) # barotropic time step as fraction of baroclinic step and averaging weights - fractional_Δt, weights, mass_flux_weights = calculate_adaptive_settings(settings.substepping, Nsubsteps) + fractional_Δt, weights = calculate_adaptive_settings(settings.substepping, Nsubsteps) Nsubsteps = length(weights) # barotropic time step in seconds @@ -364,7 +350,7 @@ function split_explicit_free_surface_step!(free_surface::SplitExplicitFreeSurfac initialize_free_surface_state!(free_surface.state, free_surface.η, settings.timestepper) # Solve for the free surface at tⁿ⁺¹ - iterate_split_explicit!(free_surface, free_surface_grid, Δτᴮ, weights, mass_flux_weights, Val(Nsubsteps)) + iterate_split_explicit!(free_surface, free_surface_grid, Δτᴮ, weights, Val(Nsubsteps)) # Reset eta for the next timestep set!(free_surface.η, free_surface.state.η̅) @@ -393,14 +379,14 @@ const MINIMUM_SUBSTEPS = 5 @inline calculate_substeps(substepping::FNS, Δt=nothing) = length(substepping.averaging_weights) @inline calculate_substeps(substepping::FTS, Δt) = max(MINIMUM_SUBSTEPS, ceil(Int, 2 * Δt / substepping.Δt_barotropic)) -@inline calculate_adaptive_settings(substepping::FNS, substeps) = substepping.fractional_step_size, substepping.averaging_weights, substepping.mass_flux_weights +@inline calculate_adaptive_settings(substepping::FNS, substeps) = substepping.fractional_step_size, substepping.averaging_weights @inline calculate_adaptive_settings(substepping::FTS, substeps) = weights_from_substeps(eltype(substepping.Δt_barotropic), substeps, substepping.averaging_kernel) const FixedSubstepsSetting{N} = SplitExplicitSettings{<:FixedSubstepNumber{<:Any, <:NTuple{N, <:Any}}} where N const FixedSubstepsSplitExplicit{F} = SplitExplicitFreeSurface{<:Any, <:Any, <:Any, <:Any, <:FixedSubstepsSetting{N}} where N -function iterate_split_explicit!(free_surface, grid, Δτᴮ, weights, mass_flux_weights, ::Val{Nsubsteps}) where Nsubsteps +function iterate_split_explicit!(free_surface, grid, Δτᴮ, weights, ::Val{Nsubsteps}) where Nsubsteps arch = architecture(grid) η = free_surface.η @@ -414,7 +400,7 @@ function iterate_split_explicit!(free_surface, grid, Δτᴮ, weights, mass_flux Uᵐ⁻¹, Uᵐ⁻² = state.Uᵐ⁻¹, state.Uᵐ⁻² Vᵐ⁻¹, Vᵐ⁻² = state.Vᵐ⁻¹, state.Vᵐ⁻² ηᵐ, ηᵐ⁻¹, ηᵐ⁻² = state.ηᵐ, state.ηᵐ⁻¹, state.ηᵐ⁻² - η̅, U̅, V̅, Ũ, Ṽ = state.η̅, state.U̅, state.V̅, state.Ũ, state.Ṽ + η̅, U̅, V̅ = state.η̅, state.U̅, state.V̅ Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ = auxiliary.Gᵁ, auxiliary.Gⱽ, auxiliary.Hᶠᶜ, auxiliary.Hᶜᶠ timestepper = settings.timestepper @@ -430,7 +416,7 @@ function iterate_split_explicit!(free_surface, grid, Δτᴮ, weights, mass_flux U_args = (grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, Ũ, Ṽ, Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, g, + η̅, U̅, V̅, Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, g, timestepper) GC.@preserve η_args U_args begin @@ -445,9 +431,8 @@ function iterate_split_explicit!(free_surface, grid, Δτᴮ, weights, mass_flux @unroll for substep in 1:Nsubsteps Base.@_inline_meta averaging_weight = weights[substep] - mass_flux_weight = mass_flux_weights[substep] free_surface_kernel!(converted_η_args...) - barotropic_velocity_kernel!(averaging_weight, mass_flux_weight, converted_U_args...) + barotropic_velocity_kernel!(averaging_weight, converted_U_args...) end end diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 24305fad2e..c42817b273 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -120,18 +120,10 @@ end ##### ZStar-specific vertical spacing functions ##### -@inline vertical_scaling(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) -@inline vertical_scaling(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) -@inline vertical_scaling(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) - @inline vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Center, ::Center, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶜⁿ[i, j, 1] @inline vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Face, ::Center, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶠᶜⁿ[i, j, 1] @inline vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Center, ::Face, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶠⁿ[i, j, 1] -@inline previous_vertical_scaling(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) -@inline previous_vertical_scaling(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) -@inline previous_vertical_scaling(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) - @inline previous_vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Center, ::Center, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶜ⁻[i, j, 1] @inline previous_vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Face, ::Center, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶠᶜ⁻[i, j, 1] @inline previous_vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Center, ::Face, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶠ⁻[i, j, 1] diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index 334a973ea3..300b213e68 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -5,7 +5,7 @@ using Oceananigans.Advection: WENOVectorInvariant using Oceananigans.Models.HydrostaticFreeSurfaceModels: ZStar, ZStarSpacingGrid, Δrᶜᶜᶜ using Printf -grid = RectilinearGrid(size = (10, 20), +grid = RectilinearGrid(size = (128, 20), x = (0, 64kilometers), z = (-20, 0), halo = (6, 6), From 644490f191088f698092373ede4437f9bee0d25a Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 25 Sep 2024 17:42:54 -0400 Subject: [PATCH 297/567] finally it works --- .../compute_w_from_continuity.jl | 4 ++-- .../z_star_vertical_spacing.jl | 19 +++++++++++-------- validation/z_star_coordinate/lock_release.jl | 2 +- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl index c649288175..cd984018de 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl @@ -19,8 +19,8 @@ compute_w_from_continuity!(velocities, arch, grid; parameters = w_kernel_paramet launch!(arch, grid, parameters, _compute_w_from_continuity!, velocities, grid) -@inline Ax_uᶠᶜᶜ⁻(i, j, k, grid, u) = @inbounds vertical_scaling(i, j, k, grid, f, c, c) * Δrᶠᶜᶜ(i, j, k, grid) * Δyᶠᶜᶜ(i, j, k, grid) * u[i, j, k] -@inline Ay_vᶜᶠᶜ⁻(i, j, k, grid, v) = @inbounds vertical_scaling(i, j, k, grid, c, f, c) * Δrᶜᶠᶜ(i, j, k, grid) * Δxᶜᶠᶜ(i, j, k, grid) * v[i, j, k] +@inline Ax_uᶠᶜᶜ⁻(i, j, k, grid, u) = @inbounds previous_vertical_scaling(i, j, k, grid, f, c, c) * Δrᶠᶜᶜ(i, j, k, grid) * Δyᶠᶜᶜ(i, j, k, grid) * u[i, j, k] +@inline Ay_vᶜᶠᶜ⁻(i, j, k, grid, v) = @inbounds previous_vertical_scaling(i, j, k, grid, c, f, c) * Δrᶜᶠᶜ(i, j, k, grid) * Δxᶜᶠᶜ(i, j, k, grid) * v[i, j, k] @inline function scaled_velocity_divergenceᶜᶜᶜ(i, j, k, grid, u, v) δU = δxᶜᶜᶜ(i, j, k, grid, Ax_uᶠᶜᶜ⁻, u) diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index c42817b273..5edbb15ffe 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -158,15 +158,10 @@ function update_vertical_spacing!(model, grid::ZStarSpacingGrid; parameters = :x Hᶠᶜ = model.free_surface.auxiliary.Hᶠᶜ Hᶜᶠ = model.free_surface.auxiliary.Hᶜᶠ Hᶠᶠ = model.free_surface.auxiliary.Hᶠᶠ - U̅ = model.free_surface.state.U̅ # Ũ - V̅ = model.free_surface.state.V̅ # Ṽ + U̅ = model.free_surface.state.U̅ + V̅ = model.free_surface.state.V̅ η = model.free_surface.η - # Update vertical spacing with available parameters - # No need to fill the halo as the scaling is updated _IN_ the halos - launch!(architecture(grid), grid, parameters, _update_zstar!, - sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, sᶠᶜ⁻, sᶜᶠ⁻, η, Hᶜᶜ, Hᶠᶜ, Hᶜᶠ, Hᶠᶠ, grid) - # Update the time derivative of the grid-scaling. Note that in this case we leverage the # free surface evolution equation, where the time derivative of the free surface is equal # to the divergence of the vertically integrated velocity field, such that @@ -174,16 +169,24 @@ function update_vertical_spacing!(model, grid::ZStarSpacingGrid; parameters = :x launch!(architecture(grid), grid, parameters, _update_∂t_∂s!, ∂t_∂s, U̅, V̅, Hᶜᶜ, grid) + # Update vertical spacing with available parameters + # No need to fill the halo as the scaling is updated _IN_ the halos + launch!(architecture(grid), grid, parameters, _update_zstar!, + sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, sᶠᶜ⁻, sᶜᶠ⁻, η, Hᶜᶜ, Hᶠᶜ, Hᶜᶠ, Hᶠᶠ, grid) + return nothing end +@inline ℑ(i, j, k, grid, U) = @inbounds U[i, j, k] + # NOTE: The ZStar vertical spacing works only for a SplitExplicitFreeSurface @kernel function _update_∂t_∂s!(∂t_∂s, U̅, V̅, Hᶜᶜ, grid) i, j = @index(Global, NTuple) k_top = grid.Nz + 1 @inbounds begin # ∂(η / H)/∂t = - ∇ ⋅ ∫udz / H - ∂t_∂s[i, j, 1] = - div_xyᶜᶜᶜ(i, j, k_top - 1, grid, U̅, V̅) / Hᶜᶜ[i, j, 1] + ∂t_∂s[i, j, 1] = - 1 / Azᶜᶜᶠ(i, j, k_top-1, grid) * (δxᶜᵃᵃ(i, j, k_top-1, grid, Δy_qᶠᶜᶠ, U̅) + + δyᵃᶜᵃ(i, j, k_top-1, grid, Δx_qᶜᶠᶠ, V̅)) / Hᶜᶜ[i, j, 1] end end diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index 300b213e68..ded3b47037 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -34,7 +34,7 @@ set!(model, b = bᵢ) @info "the time step is $Δt" -simulation = Simulation(model; Δt, stop_iteration = 100000, stop_time = 17hours) +simulation = Simulation(model; Δt, stop_iteration = 10000, stop_time = 17hours) field_outputs = if model.grid isa ZStarSpacingGrid merge(model.velocities, model.tracers, (; sⁿ = model.grid.Δzᵃᵃᶠ.sᶜᶜⁿ)) From 0df8e7216b72b0a2402048f5faa6e137069a2222 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 25 Sep 2024 17:46:04 -0400 Subject: [PATCH 298/567] add comments --- .../HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl index cd984018de..aaab57bf9e 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl @@ -18,7 +18,8 @@ compute_w_from_continuity!(model; kwargs...) = compute_w_from_continuity!(velocities, arch, grid; parameters = w_kernel_parameters(grid)) = launch!(arch, grid, parameters, _compute_w_from_continuity!, velocities, grid) - +# We use the previous vertical scaling in the continuity equation for +# consistency with how we have calculated the vertical integrals in the free surface step @inline Ax_uᶠᶜᶜ⁻(i, j, k, grid, u) = @inbounds previous_vertical_scaling(i, j, k, grid, f, c, c) * Δrᶠᶜᶜ(i, j, k, grid) * Δyᶠᶜᶜ(i, j, k, grid) * u[i, j, k] @inline Ay_vᶜᶠᶜ⁻(i, j, k, grid, v) = @inbounds previous_vertical_scaling(i, j, k, grid, c, f, c) * Δrᶜᶠᶜ(i, j, k, grid) * Δxᶜᶠᶜ(i, j, k, grid) * v[i, j, k] From 246a1e9d75080223fa0fbc2e6c77e012708aeeb6 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 25 Sep 2024 17:54:25 -0400 Subject: [PATCH 299/567] notation for partialt_s --- .../vector_invariant_cross_upwinding.jl | 4 +- .../vector_invariant_self_upwinding.jl | 8 ++-- .../HydrostaticFreeSurfaceModels.jl | 2 +- .../compute_w_from_continuity.jl | 2 +- .../generalized_vertical_spacing.jl | 10 ++--- .../z_star_vertical_spacing.jl | 43 ++++++++----------- validation/z_star_coordinate/lock_release.jl | 2 +- 7 files changed, 33 insertions(+), 38 deletions(-) diff --git a/src/Advection/vector_invariant_cross_upwinding.jl b/src/Advection/vector_invariant_cross_upwinding.jl index 7f40885fc1..6d0ef5b861 100644 --- a/src/Advection/vector_invariant_cross_upwinding.jl +++ b/src/Advection/vector_invariant_cross_upwinding.jl @@ -22,7 +22,7 @@ δ_stencil = scheme.upwinding.divergence_stencil δᴿ = _biased_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, bias(û), flux_div_xyᶜᶜᶜ, δ_stencil, u, v) - ∂ts = _symmetric_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, cross_scheme, V_times_∂t_∂s_grid) + ∂ts = _symmetric_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, cross_scheme, V_times_∂t_s_grid) return û * (δᴿ + ∂ts) end @@ -32,7 +32,7 @@ end δ_stencil = scheme.upwinding.divergence_stencil δᴿ = _biased_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, bias(v̂), flux_div_xyᶜᶜᶜ, δ_stencil, u, v) - ∂ts = _symmetric_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, cross_scheme, V_times_∂t_∂s_grid) + ∂ts = _symmetric_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, cross_scheme, V_times_∂t_s_grid) return v̂ * (δᴿ + ∂ts) diff --git a/src/Advection/vector_invariant_self_upwinding.jl b/src/Advection/vector_invariant_self_upwinding.jl index 9851eeed66..fde626c3ad 100644 --- a/src/Advection/vector_invariant_self_upwinding.jl +++ b/src/Advection/vector_invariant_self_upwinding.jl @@ -3,20 +3,20 @@ ##### # Metric term for moving grids -@inline V_times_∂t_∂s_grid(i, j, k, grid) = zero(grid) +@inline V_times_∂t_s_grid(i, j, k, grid) = zero(grid) @inline δx_U(i, j, k, grid, u, v) = δxᶜᶜᶜ(i, j, k, grid, Ax_qᶠᶜᶜ, u) @inline δy_V(i, j, k, grid, u, v) = δyᶜᶜᶜ(i, j, k, grid, Ay_qᶜᶠᶜ, v) -@inline δx_U_plus_metric(i, j, k, grid, u, v) = δxᶜᶜᶜ(i, j, k, grid, Ax_qᶠᶜᶜ, u) + V_times_∂t_∂s_grid(i, j, k, grid) -@inline δy_V_plus_metric(i, j, k, grid, u, v) = δyᶜᶜᶜ(i, j, k, grid, Ay_qᶜᶠᶜ, v) + V_times_∂t_∂s_grid(i, j, k, grid) +@inline δx_U_plus_metric(i, j, k, grid, u, v) = δxᶜᶜᶜ(i, j, k, grid, Ax_qᶠᶜᶜ, u) + V_times_∂t_s_grid(i, j, k, grid) +@inline δy_V_plus_metric(i, j, k, grid, u, v) = δyᶜᶜᶜ(i, j, k, grid, Ay_qᶜᶠᶜ, v) + V_times_∂t_s_grid(i, j, k, grid) # Velocity smoothness for divergence upwinding @inline U_smoothness(i, j, k, grid, u, v) = ℑxᶜᵃᵃ(i, j, k, grid, Ax_qᶠᶜᶜ, u) @inline V_smoothness(i, j, k, grid, u, v) = ℑyᵃᶜᵃ(i, j, k, grid, Ay_qᶜᶠᶜ, v) # Divergence smoothness for divergence upwinding -@inline divergence_smoothness(i, j, k, grid, u, v) = δx_U(i, j, k, grid, u, v) + δy_V(i, j, k, grid, u, v) + V_times_∂t_∂s_grid(i, j, k, grid) +@inline divergence_smoothness(i, j, k, grid, u, v) = δx_U(i, j, k, grid, u, v) + δy_V(i, j, k, grid, u, v) + V_times_∂t_s_grid(i, j, k, grid) @inline function upwinded_divergence_flux_Uᶠᶜᶜ(i, j, k, grid, scheme::VectorInvariantSelfVerticalUpwinding, u, v) diff --git a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl index f71808fd8b..7cfb4f60dc 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl @@ -88,7 +88,7 @@ Return a flattened `NamedTuple` of the prognostic fields associated with `Hydros @inline prognostic_fields(model::HydrostaticFreeSurfaceModel{<:Any, <:Any, <:Any, <:Any, ZStarSpacingGrid}) = merge(hydrostatic_prognostic_fields(model.velocities, model.free_surface, model.tracers), - (; ∂t_∂s = grid.Δzᵃᵃᶠ.∂t_∂s)) + (; ∂t_s = grid.Δzᵃᵃᶠ.∂t_s)) @inline hydrostatic_prognostic_fields(velocities, free_surface, tracers) = merge((u = velocities.u, v = velocities.v, diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl index aaab57bf9e..defb0d1d75 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl @@ -36,7 +36,7 @@ end for k in 2:grid.Nz+1 @inbounds U.w[i, j, k] = U.w[i, j, k-1] - ( scaled_velocity_divergenceᶜᶜᶜ(i, j, k-1, grid, U.u, U.v) + - Δrᶜᶜᶜ(i, j, k-1, grid) * ∂t_∂s_grid(i, j, k-1, grid) ) + Δrᶜᶜᶜ(i, j, k-1, grid) * ∂t_s_grid(i, j, k-1, grid) ) end end diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index 5f6a748cb8..4d8f230ecb 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -10,7 +10,7 @@ using Printf import Oceananigans.Grids: with_halo, znode import Oceananigans.Operators: Δzᶜᶜᶠ, Δzᶜᶜᶜ, Δzᶜᶠᶠ, Δzᶜᶠᶜ, Δzᶠᶜᶠ, Δzᶠᶜᶜ, Δzᶠᶠᶠ, Δzᶠᶠᶜ -import Oceananigans.Advection: V_times_∂t_∂s_grid +import Oceananigans.Advection: V_times_∂t_s_grid import Oceananigans.Architectures: arch_array """ @@ -18,7 +18,7 @@ import Oceananigans.Architectures: arch_array spacings for a generalized vertical coordinate system. The reference (non-moving) spacings are stored in `Δr`. `Δ` contains the spacings associated with the moving coordinate system. -`s⁻`, `sⁿ` and `∂t_∂s` fields are the vertical derivative of the vertical coordinate (∂Δ/∂Δr) +`s⁻`, `sⁿ` and `∂t_s` fields are the vertical derivative of the vertical coordinate (∂Δ/∂Δr) at timestep `n-1` and `n` and it's time derivative. `denomination` contains the "type" of generalized vertical coordinate (the only one implemented is `ZStar`) """ @@ -147,7 +147,7 @@ update_vertical_spacing!(model, grid; kwargs...) = nothing @inline grid_slope_contribution_x(i, j, k, grid, args...) = zero(grid) @inline grid_slope_contribution_y(i, j, k, grid, args...) = zero(grid) -@inline ∂t_∂s_grid(i, j, k, grid) = zero(grid) +@inline ∂t_s_grid(i, j, k, grid) = zero(grid) ##### ##### Tracer update in generalized vertical coordinates @@ -165,10 +165,10 @@ update_vertical_spacing!(model, grid; kwargs...) = nothing s⁻ = previous_vertical_scaling(i, j, k, grid, f, f, f) @inbounds begin - ∂t_∂sθ = C₁ * sⁿ * Gⁿ[i, j, k] - C₂ * s⁻ * G⁻[i, j, k] + ∂t_sθ = C₁ * sⁿ * Gⁿ[i, j, k] - C₂ * s⁻ * G⁻[i, j, k] # We store temporarily sθ in θ. the unscaled θ will be retrived later on with `unscale_tracers!` - θ[i, j, k] = sⁿ * θ[i, j, k] + convert(FT, Δt) * ∂t_∂sθ + θ[i, j, k] = sⁿ * θ[i, j, k] + convert(FT, Δt) * ∂t_sθ end end diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 5edbb15ffe..3b4314cdee 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -23,7 +23,7 @@ where ``η`` is the free surface height and ``H`` the vertical depth of the wate - `sᶜᶠⁿ`: scaling of the vertical coordinate at time step `n` at `(Center, Face, Any)` location - `sᶠᶠⁿ`: scaling of the vertical coordinate at time step `n` at `(Face, Face, Any)` location - `s⁻`: scaling of the vertical coordinate at time step `n - 1` at `(Center, Center, Any)` location -- `∂t_∂s`: Time derivative of `s` +- `∂t_s`: Time derivative of `s` """ struct ZStarSpacing{R, SCC, SFC, SCF, SFF} <: AbstractVerticalSpacing{R} Δr :: R @@ -34,7 +34,7 @@ struct ZStarSpacing{R, SCC, SFC, SCF, SFF} <: AbstractVerticalSpacing{R} sᶜᶜ⁻ :: SCC sᶠᶜ⁻ :: SFC sᶜᶠ⁻ :: SCF - ∂t_∂s :: SCC + ∂t_s :: SCC end Adapt.adapt_structure(to, coord::ZStarSpacing) = @@ -46,13 +46,13 @@ Adapt.adapt_structure(to, coord::ZStarSpacing) = Adapt.adapt(to, coord.sᶜᶜ⁻), Adapt.adapt(to, coord.sᶠᶜ⁻), Adapt.adapt(to, coord.sᶜᶠ⁻), - Adapt.adapt(to, coord.∂t_∂s)) + Adapt.adapt(to, coord.∂t_s)) on_architecture(arch, coord::ZStarSpacing) = ZStarSpacing(on_architecture(arch, coord.Δr), on_architecture(arch, coord.s⁻), on_architecture(arch, coord.sⁿ), - on_architecture(arch, coord.∂t_∂s)) + on_architecture(arch, coord.∂t_s)) Grids.coordinate_summary(Δ::ZStarSpacing, name) = @sprintf("Free-surface following with Δ%s=%s", name, prettysummary(Δ.Δr)) @@ -86,7 +86,7 @@ function generalized_spacing_grid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, sᶜᶠ⁻ = Field{Center, Face, Nothing}(grid) sᶜᶠⁿ = Field{Center, Face, Nothing}(grid) sᶠᶠⁿ = Field{Face, Face, Nothing}(grid) - ∂t_∂s = Field{Center, Center, Nothing}(grid) + ∂t_s = Field{Center, Center, Nothing}(grid) # Initial "at-rest" conditions fill!(sᶜᶜ⁻, 1) @@ -97,8 +97,8 @@ function generalized_spacing_grid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, fill!(sᶜᶠ⁻, 1) fill!(sᶠᶠⁿ, 1) - Δzᵃᵃᶠ = ZStarSpacing(grid.Δzᵃᵃᶠ, sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, sᶠᶜ⁻, sᶜᶠ⁻, ∂t_∂s) - Δzᵃᵃᶜ = ZStarSpacing(grid.Δzᵃᵃᶜ, sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, sᶠᶜ⁻, sᶜᶠ⁻, ∂t_∂s) + Δzᵃᵃᶠ = ZStarSpacing(grid.Δzᵃᵃᶠ, sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, sᶠᶜ⁻, sᶜᶠ⁻, ∂t_s) + Δzᵃᵃᶜ = ZStarSpacing(grid.Δzᵃᵃᶜ, sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, sᶠᶜ⁻, sᶜᶠ⁻, ∂t_s) args = [] for prop in propertynames(grid) @@ -131,11 +131,8 @@ end reference_zspacings(grid::ZStarSpacingGrid, ::Face) = grid.Δzᵃᵃᶠ.Δr reference_zspacings(grid::ZStarSpacingGrid, ::Center) = grid.Δzᵃᵃᶜ.Δr -@inline ∂t_∂s_grid(i, j, k, grid::ZStarSpacingGrid) = grid.Δzᵃᵃᶜ.∂t_∂s[i, j, 1] -@inline V_times_∂t_∂s_grid(i, j, k, grid::ZStarSpacingGrid) = ∂t_∂s_grid(i, j, k, grid) * Vᶜᶜᶜ(i, j, k, grid) - -@inline rnode(i, j, k, grid, ℓx, ℓy, ::Face) = getnode(grid.zᵃᵃᶠ, k) -@inline rnode(i, j, k, grid, ℓx, ℓy, ::Center) = getnode(grid.zᵃᵃᶜ, k) +@inline ∂t_s_grid(i, j, k, grid::ZStarSpacingGrid) = grid.Δzᵃᵃᶜ.∂t_s[i, j, 1] +@inline V_times_∂t_s_grid(i, j, k, grid::ZStarSpacingGrid) = ∂t_s_grid(i, j, k, grid) * Vᶜᶜᶜ(i, j, k, grid) ##### ##### ZStar-specific vertical spacings update @@ -151,7 +148,7 @@ function update_vertical_spacing!(model, grid::ZStarSpacingGrid; parameters = :x sᶜᶠ⁻ = grid.Δzᵃᵃᶠ.sᶜᶠ⁻ sᶜᶠⁿ = grid.Δzᵃᵃᶠ.sᶜᶠⁿ sᶠᶠⁿ = grid.Δzᵃᵃᶠ.sᶠᶠⁿ - ∂t_∂s = grid.Δzᵃᵃᶠ.∂t_∂s + ∂t_s = grid.Δzᵃᵃᶠ.∂t_s # Free surface variables Hᶜᶜ = model.free_surface.auxiliary.Hᶜᶜ @@ -162,30 +159,28 @@ function update_vertical_spacing!(model, grid::ZStarSpacingGrid; parameters = :x V̅ = model.free_surface.state.V̅ η = model.free_surface.η + # Update vertical spacing with available parameters + # No need to fill the halo as the scaling is updated _IN_ the halos + launch!(architecture(grid), grid, parameters, _update_zstar!, + sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, sᶠᶜ⁻, sᶜᶠ⁻, η, Hᶜᶜ, Hᶠᶜ, Hᶜᶠ, Hᶠᶠ, grid) + # Update the time derivative of the grid-scaling. Note that in this case we leverage the # free surface evolution equation, where the time derivative of the free surface is equal # to the divergence of the vertically integrated velocity field, such that # ∂ₜ((H + η) / H) = H⁻¹ ∂ₜη = - H⁻¹ ∇ ⋅ ∫udz - launch!(architecture(grid), grid, parameters, _update_∂t_∂s!, - ∂t_∂s, U̅, V̅, Hᶜᶜ, grid) + launch!(architecture(grid), grid, parameters, _update_∂t_s!, + ∂t_s, U̅, V̅, Hᶜᶜ, grid) - # Update vertical spacing with available parameters - # No need to fill the halo as the scaling is updated _IN_ the halos - launch!(architecture(grid), grid, parameters, _update_zstar!, - sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, sᶠᶜ⁻, sᶜᶠ⁻, η, Hᶜᶜ, Hᶠᶜ, Hᶜᶠ, Hᶠᶠ, grid) - return nothing end -@inline ℑ(i, j, k, grid, U) = @inbounds U[i, j, k] - # NOTE: The ZStar vertical spacing works only for a SplitExplicitFreeSurface -@kernel function _update_∂t_∂s!(∂t_∂s, U̅, V̅, Hᶜᶜ, grid) +@kernel function _update_∂t_s!(∂t_s, U̅, V̅, Hᶜᶜ, grid) i, j = @index(Global, NTuple) k_top = grid.Nz + 1 @inbounds begin # ∂(η / H)/∂t = - ∇ ⋅ ∫udz / H - ∂t_∂s[i, j, 1] = - 1 / Azᶜᶜᶠ(i, j, k_top-1, grid) * (δxᶜᵃᵃ(i, j, k_top-1, grid, Δy_qᶠᶜᶠ, U̅) + + ∂t_s[i, j, 1] = - 1 / Azᶜᶜᶠ(i, j, k_top-1, grid) * (δxᶜᵃᵃ(i, j, k_top-1, grid, Δy_qᶠᶜᶠ, U̅) + δyᵃᶜᵃ(i, j, k_top-1, grid, Δx_qᶜᶠᶠ, V̅)) / Hᶜᶜ[i, j, 1] end end diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index ded3b47037..6cec33c57b 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -34,7 +34,7 @@ set!(model, b = bᵢ) @info "the time step is $Δt" -simulation = Simulation(model; Δt, stop_iteration = 10000, stop_time = 17hours) +simulation = Simulation(model; Δt, stop_iteration = 1000000, stop_time = 17hours) field_outputs = if model.grid isa ZStarSpacingGrid merge(model.velocities, model.tracers, (; sⁿ = model.grid.Δzᵃᵃᶠ.sᶜᶜⁿ)) From 32e152f156460ef2b06773f654a456aa358051c3 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 25 Sep 2024 18:14:19 -0400 Subject: [PATCH 300/567] this should work now --- .../generalized_vertical_spacing.jl | 4 ++-- .../z_star_vertical_spacing.jl | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index 4d8f230ecb..e914ae9a2f 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -18,7 +18,7 @@ import Oceananigans.Architectures: arch_array spacings for a generalized vertical coordinate system. The reference (non-moving) spacings are stored in `Δr`. `Δ` contains the spacings associated with the moving coordinate system. -`s⁻`, `sⁿ` and `∂t_s` fields are the vertical derivative of the vertical coordinate (∂Δ/∂Δr) +`s⁻`, `sⁿ` and `∂t_s` fields are the vertical derivative of the vertical coordinate (∂Δr/∂Δz) at timestep `n-1` and `n` and it's time derivative. `denomination` contains the "type" of generalized vertical coordinate (the only one implemented is `ZStar`) """ @@ -162,7 +162,7 @@ update_vertical_spacing!(model, grid; kwargs...) = nothing C₂ = convert(FT, 0.5) + χ sⁿ = vertical_scaling(i, j, k, grid, c, c, c) - s⁻ = previous_vertical_scaling(i, j, k, grid, f, f, f) + s⁻ = previous_vertical_scaling(i, j, k, grid, c, c, c) @inbounds begin ∂t_sθ = C₁ * sⁿ * Gⁿ[i, j, k] - C₂ * s⁻ * G⁻[i, j, k] diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 3b4314cdee..0b0cb8b263 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -86,7 +86,7 @@ function generalized_spacing_grid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, sᶜᶠ⁻ = Field{Center, Face, Nothing}(grid) sᶜᶠⁿ = Field{Center, Face, Nothing}(grid) sᶠᶠⁿ = Field{Face, Face, Nothing}(grid) - ∂t_s = Field{Center, Center, Nothing}(grid) + ∂t_s = Field{Center, Center, Nothing}(grid) # Initial "at-rest" conditions fill!(sᶜᶜ⁻, 1) @@ -122,11 +122,11 @@ end @inline vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Center, ::Center, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶜⁿ[i, j, 1] @inline vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Face, ::Center, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶠᶜⁿ[i, j, 1] -@inline vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Center, ::Face, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶠⁿ[i, j, 1] +@inline vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Center, ::Face, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶠⁿ[i, j, 1] @inline previous_vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Center, ::Center, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶜ⁻[i, j, 1] @inline previous_vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Face, ::Center, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶠᶜ⁻[i, j, 1] -@inline previous_vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Center, ::Face, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶠ⁻[i, j, 1] +@inline previous_vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Center, ::Face, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶠ⁻[i, j, 1] reference_zspacings(grid::ZStarSpacingGrid, ::Face) = grid.Δzᵃᵃᶠ.Δr reference_zspacings(grid::ZStarSpacingGrid, ::Center) = grid.Δzᵃᵃᶜ.Δr @@ -148,7 +148,7 @@ function update_vertical_spacing!(model, grid::ZStarSpacingGrid; parameters = :x sᶜᶠ⁻ = grid.Δzᵃᵃᶠ.sᶜᶠ⁻ sᶜᶠⁿ = grid.Δzᵃᵃᶠ.sᶜᶠⁿ sᶠᶠⁿ = grid.Δzᵃᵃᶠ.sᶠᶠⁿ - ∂t_s = grid.Δzᵃᵃᶠ.∂t_s + ∂t_s = grid.Δzᵃᵃᶠ.∂t_s # Free surface variables Hᶜᶜ = model.free_surface.auxiliary.Hᶜᶜ @@ -181,7 +181,7 @@ end @inbounds begin # ∂(η / H)/∂t = - ∇ ⋅ ∫udz / H ∂t_s[i, j, 1] = - 1 / Azᶜᶜᶠ(i, j, k_top-1, grid) * (δxᶜᵃᵃ(i, j, k_top-1, grid, Δy_qᶠᶜᶠ, U̅) + - δyᵃᶜᵃ(i, j, k_top-1, grid, Δx_qᶜᶠᶠ, V̅)) / Hᶜᶜ[i, j, 1] + δyᵃᶜᵃ(i, j, k_top-1, grid, Δx_qᶜᶠᶠ, V̅)) / Hᶜᶜ[i, j, 1] end end From 3427460d552141431dbb1066627900830f962513 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 25 Sep 2024 18:25:41 -0400 Subject: [PATCH 301/567] lock release --- validation/z_star_coordinate/lock_release.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index 6cec33c57b..f02ea43796 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -11,7 +11,7 @@ grid = RectilinearGrid(size = (128, 20), halo = (6, 6), topology = (Bounded, Flat, Bounded)) -# grid = ImmersedBoundaryGrid(grid, GridFittedBottom(x -> x < 32kilometers ? -10 : -20)) +grid = ImmersedBoundaryGrid(grid, GridFittedBottom(x -> x < 32kilometers ? -10 : -20)) model = HydrostaticFreeSurfaceModel(; grid, vertical_coordinate = ZStar(), @@ -20,7 +20,7 @@ model = HydrostaticFreeSurfaceModel(; grid, buoyancy = BuoyancyTracer(), closure = nothing, tracers = :b, - free_surface = SplitExplicitFreeSurface(; substeps = 120)) + free_surface = SplitExplicitFreeSurface(; substeps = 10)) g = model.free_surface.gravitational_acceleration From 088bb7d33d052dad6d3c3859dfed8ce48363986f Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 25 Sep 2024 18:26:57 -0400 Subject: [PATCH 302/567] lock-release with less substeps --- validation/z_star_coordinate/lock_release.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index f02ea43796..889bbcf48e 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -11,7 +11,7 @@ grid = RectilinearGrid(size = (128, 20), halo = (6, 6), topology = (Bounded, Flat, Bounded)) -grid = ImmersedBoundaryGrid(grid, GridFittedBottom(x -> x < 32kilometers ? -10 : -20)) +# grid = ImmersedBoundaryGrid(grid, GridFittedBottom(x -> x < 32kilometers ? -10 : -20)) model = HydrostaticFreeSurfaceModel(; grid, vertical_coordinate = ZStar(), From 8259edce27ac5cef719cec7bab8bd0bd620a81b2 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 7 Oct 2024 12:43:32 +0200 Subject: [PATCH 303/567] do not change grids --- src/Grids/latitude_longitude_grid.jl | 30 ++++++------- src/Grids/rectilinear_grid.jl | 44 +++++++++---------- .../z_star_vertical_spacing.jl | 13 ++++-- 3 files changed, 45 insertions(+), 42 deletions(-) diff --git a/src/Grids/latitude_longitude_grid.jl b/src/Grids/latitude_longitude_grid.jl index 58a3063762..ebca324492 100644 --- a/src/Grids/latitude_longitude_grid.jl +++ b/src/Grids/latitude_longitude_grid.jl @@ -1,6 +1,6 @@ using KernelAbstractions: @kernel, @index -struct LatitudeLongitudeGrid{FT, TX, TY, TZ, M, MY, FX, FY, FZF, FZC, VX, VY, VZ, Arch} <: AbstractHorizontallyCurvilinearGrid{FT, TX, TY, TZ, Arch} +struct LatitudeLongitudeGrid{FT, TX, TY, TZ, M, MY, FX, FY, FZ, VX, VY, VZ, Arch} <: AbstractHorizontallyCurvilinearGrid{FT, TX, TY, TZ, Arch} architecture :: Arch Nx :: Int Ny :: Int @@ -21,8 +21,8 @@ struct LatitudeLongitudeGrid{FT, TX, TY, TZ, M, MY, FX, FY, FZF, FZC, VX, VY, VZ Δφᵃᶜᵃ :: FY φᵃᶠᵃ :: VY φᵃᶜᵃ :: VY - Δzᵃᵃᶠ :: FZF - Δzᵃᵃᶜ :: FZC + Δzᵃᵃᶠ :: FZ + Δzᵃᵃᶜ :: FZ zᵃᵃᶠ :: VZ zᵃᵃᶜ :: VZ # Precomputed metrics M <: Nothing means metrics will be computed on the fly @@ -47,7 +47,7 @@ struct LatitudeLongitudeGrid{FT, TX, TY, TZ, M, MY, FX, FY, FZF, FZC, VX, VY, VZ λᶠᵃᵃ :: VX, λᶜᵃᵃ :: VX, Δφᵃᶠᵃ :: FY, Δφᵃᶜᵃ :: FY, φᵃᶠᵃ :: VY, φᵃᶜᵃ :: VY, - Δzᵃᵃᶠ :: FZF, Δzᵃᵃᶜ :: FZC, + Δzᵃᵃᶠ :: FZ, Δzᵃᵃᶜ :: FZ, zᵃᵃᶠ :: VZ, zᵃᵃᶜ :: VZ, Δxᶠᶜᵃ :: M, Δxᶜᶠᵃ :: M, Δxᶠᶠᵃ :: M, Δxᶜᶜᵃ :: M, @@ -55,18 +55,18 @@ struct LatitudeLongitudeGrid{FT, TX, TY, TZ, M, MY, FX, FY, FZF, FZC, VX, VY, VZ Azᶠᶜᵃ :: M, Azᶜᶠᵃ :: M, Azᶠᶠᵃ :: M, Azᶜᶜᵃ :: M, radius :: FT) where {Arch, FT, TX, TY, TZ, - FX, FY, FZF, FZC, VX, VY, VZ, + FX, FY, FZ, VX, VY, VZ, M, MY} = - new{FT, TX, TY, TZ, M, MY, FX, FY, FZF, FZC, VX, VY, VZ, Arch}(architecture, - Nλ, Nφ, Nz, - Hλ, Hφ, Hz, - Lλ, Lφ, Lz, - Δλᶠᵃᵃ, Δλᶜᵃᵃ, λᶠᵃᵃ, λᶜᵃᵃ, - Δφᵃᶠᵃ, Δφᵃᶜᵃ, φᵃᶠᵃ, φᵃᶜᵃ, - Δzᵃᵃᶠ, Δzᵃᵃᶜ, zᵃᵃᶠ, zᵃᵃᶜ, - Δxᶠᶜᵃ, Δxᶜᶠᵃ, Δxᶠᶠᵃ, Δxᶜᶜᵃ, - Δyᶠᶜᵃ, Δyᶜᶠᵃ, - Azᶠᶜᵃ, Azᶜᶠᵃ, Azᶠᶠᵃ, Azᶜᶜᵃ, radius) + new{FT, TX, TY, TZ, M, MY, FX, FY, FZ, VX, VY, VZ, Arch}(architecture, + Nλ, Nφ, Nz, + Hλ, Hφ, Hz, + Lλ, Lφ, Lz, + Δλᶠᵃᵃ, Δλᶜᵃᵃ, λᶠᵃᵃ, λᶜᵃᵃ, + Δφᵃᶠᵃ, Δφᵃᶜᵃ, φᵃᶠᵃ, φᵃᶜᵃ, + Δzᵃᵃᶠ, Δzᵃᵃᶜ, zᵃᵃᶠ, zᵃᵃᶜ, + Δxᶠᶜᵃ, Δxᶜᶠᵃ, Δxᶠᶠᵃ, Δxᶜᶜᵃ, + Δyᶠᶜᵃ, Δyᶜᶠᵃ, + Azᶠᶜᵃ, Azᶜᶠᵃ, Azᶠᶠᵃ, Azᶜᶜᵃ, radius) end const LLG = LatitudeLongitudeGrid diff --git a/src/Grids/rectilinear_grid.jl b/src/Grids/rectilinear_grid.jl index a1dcb44acd..57313d87a4 100644 --- a/src/Grids/rectilinear_grid.jl +++ b/src/Grids/rectilinear_grid.jl @@ -1,4 +1,4 @@ -struct RectilinearGrid{FT, TX, TY, TZ, FX, FY, FZF, FZC, VX, VY, VZ, Arch} <: AbstractUnderlyingGrid{FT, TX, TY, TZ, Arch} +struct RectilinearGrid{FT, TX, TY, TZ, FX, FY, FZ, VX, VY, VZ, Arch} <: AbstractUnderlyingGrid{FT, TX, TY, TZ, Arch} architecture :: Arch Nx :: Int Ny :: Int @@ -19,31 +19,29 @@ struct RectilinearGrid{FT, TX, TY, TZ, FX, FY, FZF, FZC, VX, VY, VZ, Arch} <: Ab Δyᵃᶜᵃ :: FY yᵃᶠᵃ :: VY yᵃᶜᵃ :: VY - Δzᵃᵃᶠ :: FZF - Δzᵃᵃᶜ :: FZC + Δzᵃᵃᶠ :: FZ + Δzᵃᵃᶜ :: FZ zᵃᵃᶠ :: VZ zᵃᵃᶜ :: VZ - function RectilinearGrid{TX, TY, TZ}(arch::Arch, - Nx, Ny, Nz, - Hx, Hy, Hz, - Lx::FT, Ly::FT, Lz::FT, - Δxᶠᵃᵃ :: FX, Δxᶜᵃᵃ :: FX, - xᶠᵃᵃ :: VX, xᶜᵃᵃ :: VX, - Δyᵃᶠᵃ :: FY, Δyᵃᶜᵃ :: FY, - yᵃᶠᵃ :: VY, yᵃᶜᵃ :: VY, - Δzᵃᵃᶠ :: FZF, Δzᵃᵃᶜ :: FZC, - zᵃᵃᶠ :: VZ, zᵃᵃᶜ :: VZ) where {Arch, FT, - TX, TY, TZ, - FX, VX, FY, - VY, FZF, FZC, VZ} - - return new{FT, TX, TY, TZ, FX, FY, FZF, FZC, VX, VY, VZ, Arch}(arch, Nx, Ny, Nz, - Hx, Hy, Hz, Lx, Ly, Lz, - Δxᶠᵃᵃ, Δxᶜᵃᵃ, xᶠᵃᵃ, xᶜᵃᵃ, - Δyᵃᶠᵃ, Δyᵃᶜᵃ, yᵃᶠᵃ, yᵃᶜᵃ, - Δzᵃᵃᶠ, Δzᵃᵃᶜ, zᵃᵃᶠ, zᵃᵃᶜ) - end + RectilinearGrid{TX, TY, TZ}(arch::Arch, + Nx, Ny, Nz, + Hx, Hy, Hz, + Lx :: FT, Ly :: FT, Lz :: FT, + Δxᶠᵃᵃ :: FX, Δxᶜᵃᵃ :: FX, + xᶠᵃᵃ :: VX, xᶜᵃᵃ :: VX, + Δyᵃᶠᵃ :: FY, Δyᵃᶜᵃ :: FY, + yᵃᶠᵃ :: VY, yᵃᶜᵃ :: VY, + Δzᵃᵃᶠ :: FZ, Δzᵃᵃᶜ :: FZ, + zᵃᵃᶠ :: VZ, zᵃᵃᶜ :: VZ) where {Arch, FT, + TX, TY, TZ, + FX, VX, FY, + VY, FZ, VZ} = + new{FT, TX, TY, TZ, FX, FY, FZ, VX, VY, VZ, Arch}(arch, Nx, Ny, Nz, + Hx, Hy, Hz, Lx, Ly, Lz, + Δxᶠᵃᵃ, Δxᶜᵃᵃ, xᶠᵃᵃ, xᶜᵃᵃ, + Δyᵃᶠᵃ, Δyᵃᶜᵃ, yᵃᶠᵃ, yᵃᶜᵃ, + Δzᵃᵃᶠ, Δzᵃᵃᶜ, zᵃᵃᶠ, zᵃᵃᶜ) end const RG = RectilinearGrid diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 0b0cb8b263..29676d04d2 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -49,9 +49,14 @@ Adapt.adapt_structure(to, coord::ZStarSpacing) = Adapt.adapt(to, coord.∂t_s)) on_architecture(arch, coord::ZStarSpacing) = - ZStarSpacing(on_architecture(arch, coord.Δr), - on_architecture(arch, coord.s⁻), - on_architecture(arch, coord.sⁿ), + ZStarSpacing(on_architecture(arch, coord.Δr), + on_architecture(arch, coord.sᶜᶜⁿ), + on_architecture(arch, coord.sᶠᶜⁿ), + on_architecture(arch, coord.sᶜᶠⁿ), + on_architecture(arch, coord.sᶠᶠⁿ), + on_architecture(arch, coord.sᶜᶜ⁻), + on_architecture(arch, coord.sᶠᶜ⁻), + on_architecture(arch, coord.sᶜᶠ⁻), on_architecture(arch, coord.∂t_s)) Grids.coordinate_summary(Δ::ZStarSpacing, name) = @@ -131,7 +136,7 @@ end reference_zspacings(grid::ZStarSpacingGrid, ::Face) = grid.Δzᵃᵃᶠ.Δr reference_zspacings(grid::ZStarSpacingGrid, ::Center) = grid.Δzᵃᵃᶜ.Δr -@inline ∂t_s_grid(i, j, k, grid::ZStarSpacingGrid) = grid.Δzᵃᵃᶜ.∂t_s[i, j, 1] +@inline ∂t_s_grid(i, j, k, grid::ZStarSpacingGrid) = @inbounds grid.Δzᵃᵃᶜ.∂t_s[i, j, 1] @inline V_times_∂t_s_grid(i, j, k, grid::ZStarSpacingGrid) = ∂t_s_grid(i, j, k, grid) * Vᶜᶜᶜ(i, j, k, grid) ##### From c9ffe305f8330e99b608d0993b882fd5d43038b8 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 7 Oct 2024 12:44:49 +0200 Subject: [PATCH 304/567] bugfix --- .../HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 29676d04d2..fa0100e85c 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -128,6 +128,7 @@ end @inline vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Center, ::Center, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶜⁿ[i, j, 1] @inline vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Face, ::Center, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶠᶜⁿ[i, j, 1] @inline vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Center, ::Face, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶠⁿ[i, j, 1] +@inline vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Face, ::Face, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶠᶠⁿ[i, j, 1] @inline previous_vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Center, ::Center, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶜ⁻[i, j, 1] @inline previous_vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Face, ::Center, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶠᶜ⁻[i, j, 1] From ef99d1b77feab21a0b9cb5b8d04a9f9b093d3a74 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 7 Oct 2024 12:45:12 +0200 Subject: [PATCH 305/567] remove rnode for now --- .../generalized_vertical_spacing.jl | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index e914ae9a2f..c83d8ab64b 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -121,8 +121,6 @@ update_vertical_spacing!(model, grid; kwargs...) = nothing @inline Δrᶠᶠᶠ(i, j, k, grid) = ℑxyᶠᶠᵃ(i, j, k, grid, Δrᶜᶜᶠ) @inline Δrᶠᶠᶜ(i, j, k, grid) = ℑxyᶠᶠᵃ(i, j, k, grid, Δrᶜᶜᶜ) -@inline rnode(i, j, k, grid, ℓx, ℓy, ℓz) = znode(i, j, k, grid, ℓx, ℓy, ℓz) - ##### ##### Vertical spacings for a generalized vertical coordinate system ##### From 2009fd492740d1dce7ae9074c832e42b700c18fd Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 7 Oct 2024 13:01:09 +0200 Subject: [PATCH 306/567] full non-linear free surface --- .../compute_w_from_continuity.jl | 4 +-- .../split_explicit_free_surface_kernels.jl | 34 +++++++++++-------- .../z_star_vertical_spacing.jl | 28 ++++++++------- validation/z_star_coordinate/lock_release.jl | 2 +- 4 files changed, 38 insertions(+), 30 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl index defb0d1d75..2f29124221 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl @@ -20,8 +20,8 @@ compute_w_from_continuity!(velocities, arch, grid; parameters = w_kernel_paramet # We use the previous vertical scaling in the continuity equation for # consistency with how we have calculated the vertical integrals in the free surface step -@inline Ax_uᶠᶜᶜ⁻(i, j, k, grid, u) = @inbounds previous_vertical_scaling(i, j, k, grid, f, c, c) * Δrᶠᶜᶜ(i, j, k, grid) * Δyᶠᶜᶜ(i, j, k, grid) * u[i, j, k] -@inline Ay_vᶜᶠᶜ⁻(i, j, k, grid, v) = @inbounds previous_vertical_scaling(i, j, k, grid, c, f, c) * Δrᶜᶠᶜ(i, j, k, grid) * Δxᶜᶠᶜ(i, j, k, grid) * v[i, j, k] +@inline Ax_uᶠᶜᶜ⁻(i, j, k, grid, u) = @inbounds vertical_scaling(i, j, k, grid, f, c, c) * Δrᶠᶜᶜ(i, j, k, grid) * Δyᶠᶜᶜ(i, j, k, grid) * u[i, j, k] +@inline Ay_vᶜᶠᶜ⁻(i, j, k, grid, v) = @inbounds vertical_scaling(i, j, k, grid, c, f, c) * Δrᶜᶠᶜ(i, j, k, grid) * Δxᶜᶠᶜ(i, j, k, grid) * v[i, j, k] @inline function scaled_velocity_divergenceᶜᶜᶜ(i, j, k, grid, u, v) δU = δxᶜᶜᶜ(i, j, k, grid, Ax_uᶠᶜᶜ⁻, u) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 1293d44d0b..5659b84cc0 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -162,10 +162,8 @@ end @inline dynamic_column_heightᶜᶠ(i, j, k, grid, H, η) = @inbounds H[i, j, 1] # Non-linear free surface implementation -# At the moment the dynamic height is frozen in time so it is in practice a linear equation in the substepping -# TODO: Implement the full non-linear free surface model by having the dynamic height evolve in time during substepping -@inline dynamic_column_heightᶠᶜ(i, j, k, grid::ZStarSpacingGrid, H, η) = @inbounds H[i, j, 1] * vertical_scaling(i, j, k, grid, f, c, c) -@inline dynamic_column_heightᶜᶠ(i, j, k, grid::ZStarSpacingGrid, H, η) = @inbounds H[i, j, 1] * vertical_scaling(i, j, k, grid, c, f, c) +@inline dynamic_column_heightᶠᶜ(i, j, k, grid::ZStarSpacingGrid, H, η) = @inbounds H[i, j, 1] + ℑxᶠᵃᵃ(i, j, k, grid, η) +@inline dynamic_column_heightᶜᶠ(i, j, k, grid::ZStarSpacingGrid, H, η) = @inbounds H[i, j, 1] + ℑyᵃᶠᵃ(i, j, k, grid, η) @kernel function _split_explicit_barotropic_velocity!(averaging_weight, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², @@ -211,39 +209,45 @@ end # For Zstar vertical spacing the vertical integral includes the dynamic height # Remember, the vertical coordinate has not yet been updated! # For this reason the integration has to be performed manually -@kernel function _barotropic_mode_kernel!(U, V, grid, ::Nothing, u, v) +@kernel function _barotropic_mode_kernel!(U, V, grid, ::Nothing, u, v, Hᶠᶜ, Hᶜᶠ, η̅) i, j = @index(Global, NTuple) k_top = grid.Nz + 1 + sᶠᶜ = @inbounds dynamic_column_heightᶠᶜ(i, j, k_top, grid, Hᶠᶜ, η̅) / Hᶠᶜ[i, j, 1] + sᶜᶠ = @inbounds dynamic_column_heightᶜᶠ(i, j, k_top, grid, Hᶜᶠ, η̅) / Hᶜᶠ[i, j, 1] + # hand unroll first loop - @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δzᶠᶜᶜ(i, j, 1, grid) - @inbounds V[i, j, k_top-1] = v[i, j, 1] * Δzᶜᶠᶜ(i, j, 1, grid) + @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δrᶠᶜᶜ(i, j, 1, grid) * sᶠᶜ + @inbounds V[i, j, k_top-1] = v[i, j, 1] * Δrᶜᶠᶜ(i, j, 1, grid) * sᶜᶠ @unroll for k in 2:grid.Nz - @inbounds U[i, j, k_top-1] += u[i, j, k] * Δzᶠᶜᶜ(i, j, k, grid) - @inbounds V[i, j, k_top-1] += v[i, j, k] * Δzᶜᶠᶜ(i, j, k, grid) + @inbounds U[i, j, k_top-1] += u[i, j, k] * Δrᶠᶜᶜ(i, j, k, grid) * sᶠᶜ + @inbounds V[i, j, k_top-1] += v[i, j, k] * Δrᶜᶠᶜ(i, j, k, grid) * sᶜᶠ end end -@kernel function _barotropic_mode_kernel!(U, V, grid, active_cells_map, u, v) +@kernel function _barotropic_mode_kernel!(U, V, grid, active_cells_map, u, v, Hᶠᶜ, Hᶜᶠ, η̅) idx = @index(Global, Linear) i, j = active_linear_index_to_tuple(idx, active_cells_map) k_top = grid.Nz+1 + sᶠᶜ = @inbounds dynamic_column_heightᶠᶜ(i, j, k_top, grid, Hᶠᶜ, η̅) / Hᶠᶜ[i, j, 1] + sᶜᶠ = @inbounds dynamic_column_heightᶜᶠ(i, j, k_top, grid, Hᶜᶠ, η̅) / Hᶜᶠ[i, j, 1] + # hand unroll first loop - @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δzᶠᶜᶜ(i, j, 1, grid) - @inbounds V[i, j, k_top-1] = v[i, j, 1] * Δzᶜᶠᶜ(i, j, 1, grid) + @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δrᶠᶜᶜ(i, j, 1, grid) * sᶠᶜ + @inbounds V[i, j, k_top-1] = v[i, j, 1] * Δrᶜᶠᶜ(i, j, 1, grid) * sᶜᶠ @unroll for k in 2:grid.Nz - @inbounds U[i, j, k_top-1] += u[i, j, k] * Δzᶠᶜᶜ(i, j, k, grid) - @inbounds V[i, j, k_top-1] += v[i, j, k] * Δzᶜᶠᶜ(i, j, k, grid) + @inbounds U[i, j, k_top-1] += u[i, j, k] * Δrᶠᶜᶜ(i, j, k, grid) * sᶠᶜ + @inbounds V[i, j, k_top-1] += v[i, j, k] * Δrᶜᶠᶜ(i, j, k, grid) * sᶜᶠ end end @inline function compute_barotropic_mode!(U, V, grid, u, v, Hᶠᶜ, Hᶜᶠ, η̅) active_cells_map = retrieve_surface_active_cells_map(grid) - launch!(architecture(grid), grid, :xy, _barotropic_mode_kernel!, U, V, grid, active_cells_map, u, v; active_cells_map) + launch!(architecture(grid), grid, :xy, _barotropic_mode_kernel!, U, V, grid, active_cells_map, u, v, Hᶠᶜ, Hᶜᶠ, η̅; active_cells_map) return nothing end diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index fa0100e85c..8b259cadab 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -125,6 +125,9 @@ end ##### ZStar-specific vertical spacing functions ##### +reference_zspacings(grid::ZStarSpacingGrid, ::Face) = grid.Δzᵃᵃᶠ.Δr +reference_zspacings(grid::ZStarSpacingGrid, ::Center) = grid.Δzᵃᵃᶜ.Δr + @inline vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Center, ::Center, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶜⁿ[i, j, 1] @inline vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Face, ::Center, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶠᶜⁿ[i, j, 1] @inline vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Center, ::Face, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶠⁿ[i, j, 1] @@ -134,8 +137,9 @@ end @inline previous_vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Face, ::Center, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶠᶜ⁻[i, j, 1] @inline previous_vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Center, ::Face, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶠ⁻[i, j, 1] -reference_zspacings(grid::ZStarSpacingGrid, ::Face) = grid.Δzᵃᵃᶠ.Δr -reference_zspacings(grid::ZStarSpacingGrid, ::Center) = grid.Δzᵃᵃᶜ.Δr +@inline previous_vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Center, ::Center, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶜ⁻[i, j, 1] +@inline previous_vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Face, ::Center, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶠᶜ⁻[i, j, 1] +@inline previous_vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Center, ::Face, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶠ⁻[i, j, 1] @inline ∂t_s_grid(i, j, k, grid::ZStarSpacingGrid) = @inbounds grid.Δzᵃᵃᶜ.∂t_s[i, j, 1] @inline V_times_∂t_s_grid(i, j, k, grid::ZStarSpacingGrid) = ∂t_s_grid(i, j, k, grid) * Vᶜᶜᶜ(i, j, k, grid) @@ -147,14 +151,14 @@ reference_zspacings(grid::ZStarSpacingGrid, ::Center) = grid.Δzᵃᵃᶜ.Δr function update_vertical_spacing!(model, grid::ZStarSpacingGrid; parameters = :xy) # Scaling - sᶜᶜ⁻ = grid.Δzᵃᵃᶠ.sᶜᶜ⁻ - sᶜᶜⁿ = grid.Δzᵃᵃᶠ.sᶜᶜⁿ - sᶠᶜ⁻ = grid.Δzᵃᵃᶠ.sᶠᶜ⁻ - sᶠᶜⁿ = grid.Δzᵃᵃᶠ.sᶠᶜⁿ - sᶜᶠ⁻ = grid.Δzᵃᵃᶠ.sᶜᶠ⁻ - sᶜᶠⁿ = grid.Δzᵃᵃᶠ.sᶜᶠⁿ - sᶠᶠⁿ = grid.Δzᵃᵃᶠ.sᶠᶠⁿ - ∂t_s = grid.Δzᵃᵃᶠ.∂t_s + sᶜᶜ⁻ = grid.Δzᵃᵃᶠ.sᶜᶜ⁻ + sᶜᶜⁿ = grid.Δzᵃᵃᶠ.sᶜᶜⁿ + sᶠᶜ⁻ = grid.Δzᵃᵃᶠ.sᶠᶜ⁻ + sᶠᶜⁿ = grid.Δzᵃᵃᶠ.sᶠᶜⁿ + sᶜᶠ⁻ = grid.Δzᵃᵃᶠ.sᶜᶠ⁻ + sᶜᶠⁿ = grid.Δzᵃᵃᶠ.sᶜᶠⁿ + sᶠᶠⁿ = grid.Δzᵃᵃᶠ.sᶠᶠⁿ + ∂t_s = grid.Δzᵃᵃᶠ.∂t_s # Free surface variables Hᶜᶜ = model.free_surface.auxiliary.Hᶜᶜ @@ -186,8 +190,8 @@ end k_top = grid.Nz + 1 @inbounds begin # ∂(η / H)/∂t = - ∇ ⋅ ∫udz / H - ∂t_s[i, j, 1] = - 1 / Azᶜᶜᶠ(i, j, k_top-1, grid) * (δxᶜᵃᵃ(i, j, k_top-1, grid, Δy_qᶠᶜᶠ, U̅) + - δyᵃᶜᵃ(i, j, k_top-1, grid, Δx_qᶜᶠᶠ, V̅)) / Hᶜᶜ[i, j, 1] + ∂t_s[i, j, 1] = - 1 / Azᶜᶜᶠ(i, j, k_top-1, grid) * (δxᶜᵃᵃ(i, j, k_top-1, grid, Δy_qᶠᶜᶠ, U̅) + + δyᵃᶜᵃ(i, j, k_top-1, grid, Δx_qᶜᶠᶠ, V̅)) / Hᶜᶜ[i, j, 1] end end diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index 889bbcf48e..2a221bec3b 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -34,7 +34,7 @@ set!(model, b = bᵢ) @info "the time step is $Δt" -simulation = Simulation(model; Δt, stop_iteration = 1000000, stop_time = 17hours) +simulation = Simulation(model; Δt, stop_iteration = 10000, stop_time = 17hours) field_outputs = if model.grid isa ZStarSpacingGrid merge(model.velocities, model.tracers, (; sⁿ = model.grid.Δzᵃᵃᶠ.sᶜᶜⁿ)) From 3b016356be73fe04f54cace68866f3b48587dbe8 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 7 Oct 2024 13:03:14 +0200 Subject: [PATCH 307/567] simplify a bit --- .../compute_w_from_continuity.jl | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl index 2f29124221..b40984b466 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl @@ -18,16 +18,7 @@ compute_w_from_continuity!(model; kwargs...) = compute_w_from_continuity!(velocities, arch, grid; parameters = w_kernel_parameters(grid)) = launch!(arch, grid, parameters, _compute_w_from_continuity!, velocities, grid) -# We use the previous vertical scaling in the continuity equation for -# consistency with how we have calculated the vertical integrals in the free surface step -@inline Ax_uᶠᶜᶜ⁻(i, j, k, grid, u) = @inbounds vertical_scaling(i, j, k, grid, f, c, c) * Δrᶠᶜᶜ(i, j, k, grid) * Δyᶠᶜᶜ(i, j, k, grid) * u[i, j, k] -@inline Ay_vᶜᶠᶜ⁻(i, j, k, grid, v) = @inbounds vertical_scaling(i, j, k, grid, c, f, c) * Δrᶜᶠᶜ(i, j, k, grid) * Δxᶜᶠᶜ(i, j, k, grid) * v[i, j, k] - -@inline function scaled_velocity_divergenceᶜᶜᶜ(i, j, k, grid, u, v) - δU = δxᶜᶜᶜ(i, j, k, grid, Ax_uᶠᶜᶜ⁻, u) - δV = δyᶜᶜᶜ(i, j, k, grid, Ay_vᶜᶠᶜ⁻, v) - return (δU + δV) / Azᶜᶜᶜ(i, j, k, grid) -end +@inline velocity_divergenceᶜᶜᶜ(i, j, k, grid, u, v) = flux_div_xyᶜᶜᶜ(i, j, k, grid, u, v) / Azᶜᶜᶜ(i, j, k, grid) @kernel function _compute_w_from_continuity!(U, grid) i, j = @index(Global, NTuple) @@ -35,7 +26,7 @@ end @inbounds U.w[i, j, 1] = 0 for k in 2:grid.Nz+1 @inbounds U.w[i, j, k] = U.w[i, j, k-1] - - ( scaled_velocity_divergenceᶜᶜᶜ(i, j, k-1, grid, U.u, U.v) + + ( velocity_divergenceᶜᶜᶜ(i, j, k-1, grid, U.u, U.v) + Δrᶜᶜᶜ(i, j, k-1, grid) * ∂t_s_grid(i, j, k-1, grid) ) end end From 229976a29218414be75f20fe0aae3b0c4e89028d Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 7 Oct 2024 13:03:40 +0200 Subject: [PATCH 308/567] still scaled --- .../HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl index b40984b466..4627f68df4 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl @@ -18,7 +18,7 @@ compute_w_from_continuity!(model; kwargs...) = compute_w_from_continuity!(velocities, arch, grid; parameters = w_kernel_parameters(grid)) = launch!(arch, grid, parameters, _compute_w_from_continuity!, velocities, grid) -@inline velocity_divergenceᶜᶜᶜ(i, j, k, grid, u, v) = flux_div_xyᶜᶜᶜ(i, j, k, grid, u, v) / Azᶜᶜᶜ(i, j, k, grid) +@inline scaled_velocity_divergenceᶜᶜᶜ(i, j, k, grid, u, v) = flux_div_xyᶜᶜᶜ(i, j, k, grid, u, v) / Azᶜᶜᶜ(i, j, k, grid) @kernel function _compute_w_from_continuity!(U, grid) i, j = @index(Global, NTuple) @@ -26,7 +26,7 @@ compute_w_from_continuity!(velocities, arch, grid; parameters = w_kernel_paramet @inbounds U.w[i, j, 1] = 0 for k in 2:grid.Nz+1 @inbounds U.w[i, j, k] = U.w[i, j, k-1] - - ( velocity_divergenceᶜᶜᶜ(i, j, k-1, grid, U.u, U.v) + + ( scaled_velocity_divergenceᶜᶜᶜ(i, j, k-1, grid, U.u, U.v) + Δrᶜᶜᶜ(i, j, k-1, grid) * ∂t_s_grid(i, j, k-1, grid) ) end end From 7c6a57e0697f4427bba2b32e6a3bcd794a0065ef Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 7 Oct 2024 13:04:19 +0200 Subject: [PATCH 309/567] simplify even more --- .../compute_w_from_continuity.jl | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl index 4627f68df4..eca29f4933 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl @@ -18,16 +18,14 @@ compute_w_from_continuity!(model; kwargs...) = compute_w_from_continuity!(velocities, arch, grid; parameters = w_kernel_parameters(grid)) = launch!(arch, grid, parameters, _compute_w_from_continuity!, velocities, grid) -@inline scaled_velocity_divergenceᶜᶜᶜ(i, j, k, grid, u, v) = flux_div_xyᶜᶜᶜ(i, j, k, grid, u, v) / Azᶜᶜᶜ(i, j, k, grid) - @kernel function _compute_w_from_continuity!(U, grid) i, j = @index(Global, NTuple) @inbounds U.w[i, j, 1] = 0 for k in 2:grid.Nz+1 @inbounds U.w[i, j, k] = U.w[i, j, k-1] - - ( scaled_velocity_divergenceᶜᶜᶜ(i, j, k-1, grid, U.u, U.v) + - Δrᶜᶜᶜ(i, j, k-1, grid) * ∂t_s_grid(i, j, k-1, grid) ) + (flux_div_xyᶜᶜᶜ(i, j, k, grid, u, v) / Azᶜᶜᶜ(i, j, k, grid) + + Δrᶜᶜᶜ(i, j, k-1, grid) * ∂t_s_grid(i, j, k-1, grid) ) end end From 3ba2bd63bab0a28198460557794a59f022a6dc55 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 7 Oct 2024 14:14:25 +0200 Subject: [PATCH 310/567] some bugfixes --- .../compute_w_from_continuity.jl | 7 +-- .../generalized_vertical_spacing.jl | 2 +- .../split_explicit_free_surface.jl | 33 +++----------- .../z_star_vertical_spacing.jl | 19 ++++---- src/Utils/kernel_launching.jl | 2 +- validation/z_star_coordinate/lock_release.jl | 43 ++++++++----------- 6 files changed, 39 insertions(+), 67 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl index eca29f4933..f85f57ba7d 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl @@ -23,9 +23,10 @@ compute_w_from_continuity!(velocities, arch, grid; parameters = w_kernel_paramet @inbounds U.w[i, j, 1] = 0 for k in 2:grid.Nz+1 - @inbounds U.w[i, j, k] = U.w[i, j, k-1] - - (flux_div_xyᶜᶜᶜ(i, j, k, grid, u, v) / Azᶜᶜᶜ(i, j, k, grid) + - Δrᶜᶜᶜ(i, j, k-1, grid) * ∂t_s_grid(i, j, k-1, grid) ) + δ_Uh = flux_div_xyᶜᶜᶜ(i, j, k-1, grid, U.u, U.v) / Azᶜᶜᶜ(i, j, k-1, grid) + ∂t_s = Δrᶜᶜᶜ(i, j, k-1, grid) * ∂t_s_grid(i, j, k-1, grid) + + @inbounds U.w[i, j, k] = U.w[i, j, k-1] - ∂t_s - δ_Uh end end diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index c83d8ab64b..f7d9f74991 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -134,7 +134,7 @@ update_vertical_spacing!(model, grid; kwargs...) = nothing @inline Δzᶠᶜᶠ(i, j, k, grid::AbstractVerticalSpacingGrid) = @inbounds Δrᶠᶜᶠ(i, j, k, grid) * vertical_scaling(i, j, k, grid, f, c, f) @inline Δzᶠᶜᶜ(i, j, k, grid::AbstractVerticalSpacingGrid) = @inbounds Δrᶠᶜᶜ(i, j, k, grid) * vertical_scaling(i, j, k, grid, f, c, c) @inline Δzᶜᶠᶠ(i, j, k, grid::AbstractVerticalSpacingGrid) = @inbounds Δrᶜᶠᶠ(i, j, k, grid) * vertical_scaling(i, j, k, grid, c, f, f) -@inline Δzᶜᶠᶜ(i, j, k, grid::AbstractVerticalSpacingGrid) = @inbounds Δrᶜᶠᶜ(i, j, k, grid) * vertical_scaling(i, j, k, grid, f, c, c) +@inline Δzᶜᶠᶜ(i, j, k, grid::AbstractVerticalSpacingGrid) = @inbounds Δrᶜᶠᶜ(i, j, k, grid) * vertical_scaling(i, j, k, grid, c, f, c) @inline Δzᶠᶠᶠ(i, j, k, grid::AbstractVerticalSpacingGrid) = @inbounds Δrᶠᶠᶠ(i, j, k, grid) * vertical_scaling(i, j, k, grid, f, f, f) @inline Δzᶠᶠᶜ(i, j, k, grid::AbstractVerticalSpacingGrid) = @inbounds Δrᶠᶠᶜ(i, j, k, grid) * vertical_scaling(i, j, k, grid, f, f, c) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index a86c88d878..8b44cd1764 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -3,6 +3,7 @@ using Oceananigans.Architectures using Oceananigans.Fields using Oceananigans.Grids using Oceananigans.Grids: AbstractGrid +using Oceananigans.BoundaryConditions: default_prognostic_bc using Oceananigans.AbstractOperations: Δz, GridMetricOperation using Oceananigans.ImmersedBoundaries: immersed_peripheral_node, c, f using Adapt @@ -251,35 +252,15 @@ function SplitExplicitAuxiliaryFields(grid::AbstractGrid) end function compute_column_height!(Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, Hᶠᶠ, grid) - Nx, Ny, _ = size(grid) - Hx, Hy, _ = halo_size(grid) - Tx, Ty, _ = topology(grid) - - Ax = Tx == Flat ? 0 : 1 - Ay = Ty == Flat ? 0 : 1 arch = architecture(grid) - - # We compute the heights over all the - # domain including the halo points! - # Because of the different locations of the different heights, - # to encompass the whole grid, the kernel parameters must be different. - kernel_size = (Nx+2Hx-2Ax, Ny+2Hy) - kernel_offset = (-Hx+Ax, -Hy) - paramᶠᶜ = KernelParameters(kernel_size, kernel_offset) - launch!(arch, grid, paramᶠᶜ, _compute_column_height!, Hᶠᶜ, grid, f, c, Δrᶠᶜᶜ) - - kernel_size = (Nx+2Hx, Ny+2Hy-2Ay) - kernel_offset = (-Hx, -Hy+Ay) - paramᶜᶠ = KernelParameters(kernel_size, kernel_offset) - launch!(arch, grid, paramᶜᶠ, _compute_column_height!, Hᶜᶠ, grid, c, f, Δrᶜᶠᶜ) - - kernel_size = (Nx+2Hx, Ny+2Hy) - kernel_offset = (-Hx, -Hy) - paramᶜᶜ = KernelParameters(kernel_size, kernel_offset) - launch!(arch, grid, paramᶜᶜ, _compute_column_height!, Hᶜᶜ, grid, c, c, Δrᶜᶜᶜ) - launch!(arch, grid, paramᶜᶜ, _compute_column_height!, Hᶠᶠ, grid, c, c, Δrᶠᶠᶜ) + launch!(arch, grid, :xy, _compute_column_height!, Hᶠᶜ, grid, f, c, Δrᶠᶜᶜ; include_right_boundaries = true, location = (Face, Center, Nothing)) + launch!(arch, grid, :xy, _compute_column_height!, Hᶜᶠ, grid, c, f, Δrᶜᶠᶜ; include_right_boundaries = true, location = (Center, Face, Nothing)) + launch!(arch, grid, :xy, _compute_column_height!, Hᶜᶜ, grid, c, c, Δrᶜᶜᶜ; include_right_boundaries = true, location = (Center, Center, Nothing)) + launch!(arch, grid, :xy, _compute_column_height!, Hᶠᶠ, grid, f, f, Δrᶠᶠᶜ; include_right_boundaries = true, location = (Face, Face, Nothing)) + fill_halo_regions!((Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, Hᶠᶠ)) + return nothing end diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 8b259cadab..42af9a08ba 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -59,7 +59,7 @@ on_architecture(arch, coord::ZStarSpacing) = on_architecture(arch, coord.sᶜᶠ⁻), on_architecture(arch, coord.∂t_s)) -Grids.coordinate_summary(Δ::ZStarSpacing, name) = +Grids.coordinate_summary(::Bounded, Δ::ZStarSpacing, name) = @sprintf("Free-surface following with Δ%s=%s", name, prettysummary(Δ.Δr)) const ZStarSpacingRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:ZStarSpacing} @@ -137,10 +137,6 @@ reference_zspacings(grid::ZStarSpacingGrid, ::Center) = grid.Δzᵃᵃᶜ.Δr @inline previous_vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Face, ::Center, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶠᶜ⁻[i, j, 1] @inline previous_vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Center, ::Face, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶠ⁻[i, j, 1] -@inline previous_vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Center, ::Center, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶜ⁻[i, j, 1] -@inline previous_vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Face, ::Center, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶠᶜ⁻[i, j, 1] -@inline previous_vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Center, ::Face, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶠ⁻[i, j, 1] - @inline ∂t_s_grid(i, j, k, grid::ZStarSpacingGrid) = @inbounds grid.Δzᵃᵃᶜ.∂t_s[i, j, 1] @inline V_times_∂t_s_grid(i, j, k, grid::ZStarSpacingGrid) = ∂t_s_grid(i, j, k, grid) * Vᶜᶜᶜ(i, j, k, grid) @@ -184,7 +180,8 @@ function update_vertical_spacing!(model, grid::ZStarSpacingGrid; parameters = :x return nothing end -# NOTE: The ZStar vertical spacing works only for a SplitExplicitFreeSurface +# NOTE: The ZStar vertical spacing only supports a SplitExplicitFreeSurface +# TODO: extend to support other free surface solvers @kernel function _update_∂t_s!(∂t_s, U̅, V̅, Hᶜᶜ, grid) i, j = @index(Global, NTuple) k_top = grid.Nz + 1 @@ -220,16 +217,16 @@ end ##### ZStar-specific implementation of the additional terms to be included in the momentum equations ##### -@inline η_surfaceᶜᶜᶜ(i, j, k, grid, η, Hᶜᶜ) = @inbounds η[i, j, grid.Nz+1] * (1 + znode(i, j, k, grid, Center(), Center(), Center()) / Hᶜᶜ[i, j, 1]) +@inline z_minus_rᶜᶜᶜ(i, j, k, grid, η, Hᶜᶜ) = @inbounds η[i, j, grid.Nz+1] * (1 + znode(i, j, k, grid, Center(), Center(), Center()) / Hᶜᶜ[i, j, 1]) -@inline slope_xᶠᶜᶜ(i, j, k, grid, free_surface) = @inbounds ∂xᶠᶜᶜ(i, j, k, grid, η_surfaceᶜᶜᶜ, free_surface.η, free_surface.auxiliary.Hᶜᶜ) -@inline slope_yᶜᶠᶜ(i, j, k, grid, free_surface) = @inbounds ∂yᶜᶠᶜ(i, j, k, grid, η_surfaceᶜᶜᶜ, free_surface.η, free_surface.auxiliary.Hᶜᶜ) +@inline ∂x_z(i, j, k, grid, free_surface) = @inbounds ∂xᶠᶜᶜ(i, j, k, grid, z_minus_rᶜᶜᶜ, free_surface.η, free_surface.auxiliary.Hᶜᶜ) +@inline ∂y_z(i, j, k, grid, free_surface) = @inbounds ∂yᶜᶠᶜ(i, j, k, grid, z_minus_rᶜᶜᶜ, free_surface.η, free_surface.auxiliary.Hᶜᶜ) @inline grid_slope_contribution_x(i, j, k, grid::ZStarSpacingGrid, free_surface, ::Nothing, model_fields) = zero(grid) @inline grid_slope_contribution_y(i, j, k, grid::ZStarSpacingGrid, free_surface, ::Nothing, model_fields) = zero(grid) @inline grid_slope_contribution_x(i, j, k, grid::ZStarSpacingGrid, free_surface, buoyancy, model_fields) = - ℑxᶠᵃᵃ(i, j, k, grid, buoyancy_perturbationᶜᶜᶜ, buoyancy.model, model_fields) * slope_xᶠᶜᶜ(i, j, k, grid, free_surface) + ℑxᶠᵃᵃ(i, j, k, grid, buoyancy_perturbationᶜᶜᶜ, buoyancy.model, model_fields) * ∂x_z(i, j, k, grid, free_surface) @inline grid_slope_contribution_y(i, j, k, grid::ZStarSpacingGrid, free_surface, buoyancy, model_fields) = - ℑyᵃᶠᵃ(i, j, k, grid, buoyancy_perturbationᶜᶜᶜ, buoyancy.model, model_fields) * slope_yᶜᶠᶜ(i, j, k, grid, free_surface) + ℑyᵃᶠᵃ(i, j, k, grid, buoyancy_perturbationᶜᶜᶜ, buoyancy.model, model_fields) * ∂y_z(i, j, k, grid, free_surface) diff --git a/src/Utils/kernel_launching.jl b/src/Utils/kernel_launching.jl index 48108ec236..e769061114 100644 --- a/src/Utils/kernel_launching.jl +++ b/src/Utils/kernel_launching.jl @@ -129,7 +129,7 @@ For more information, see: https://github.com/CliMA/Oceananigans.jl/pull/308 """ function work_layout(grid, workdims::Symbol; include_right_boundaries=false, location=nothing, reduced_dimensions=()) - Nx′, Ny′, Nz′ = include_right_boundaries ? size(location, grid) : size(grid) + Nx′, Ny′, Nz′ = include_right_boundaries ? size(grid, location) : size(grid) Nx′, Ny′, Nz′ = flatten_reduced_dimensions((Nx′, Ny′, Nz′), reduced_dimensions) workgroup = heuristic_workgroup(Nx′, Ny′, Nz′) diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index 2a221bec3b..2053c955b6 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -2,16 +2,17 @@ using Oceananigans using Oceananigans.Units using Oceananigans.Utils: prettytime using Oceananigans.Advection: WENOVectorInvariant +using Oceananigans.AbstractOperations: GridMetricOperation using Oceananigans.Models.HydrostaticFreeSurfaceModels: ZStar, ZStarSpacingGrid, Δrᶜᶜᶜ using Printf -grid = RectilinearGrid(size = (128, 20), +grid = RectilinearGrid(size = (10, 20), x = (0, 64kilometers), z = (-20, 0), halo = (6, 6), topology = (Bounded, Flat, Bounded)) -# grid = ImmersedBoundaryGrid(grid, GridFittedBottom(x -> x < 32kilometers ? -10 : -20)) +grid = ImmersedBoundaryGrid(grid, GridFittedBottom(x -> x < 32kilometers ? -10 : -20)) model = HydrostaticFreeSurfaceModel(; grid, vertical_coordinate = ZStar(), @@ -34,13 +35,11 @@ set!(model, b = bᵢ) @info "the time step is $Δt" -simulation = Simulation(model; Δt, stop_iteration = 10000, stop_time = 17hours) +simulation = Simulation(model; Δt, stop_iteration = 100, stop_time = 17hours) -field_outputs = if model.grid isa ZStarSpacingGrid - merge(model.velocities, model.tracers, (; sⁿ = model.grid.Δzᵃᵃᶠ.sᶜᶜⁿ)) -else - merge(model.velocities, model.tracers) -end +Δz = GridMetricOperation((Center, Center, Center), Oceananigans.AbstractOperations.Δz, model.grid) + +field_outputs = merge(model.velocities, model.tracers, (; Δz)) simulation.output_writers[:other_variables] = JLD2OutputWriter(model, field_outputs, overwrite_existing = true, @@ -53,16 +52,11 @@ function progress(sim) b = sim.model.tracers.b msg0 = @sprintf("Time: %s iteration %d ", prettytime(sim.model.clock.time), sim.model.clock.iteration) - msg1 = @sprintf("extrema w: %.2e %.2e ", maximum(w), minimum(w)) - msg2 = @sprintf("extrema u: %.2e %.2e ", maximum(u), minimum(u)) - msg3 = @sprintf("extrema b: %.2e %.2e ", maximum(b), minimum(b)) - if sim.model.grid isa ZStarSpacingGrid - Δz = sim.model.grid.Δzᵃᵃᶠ.sᶜᶜⁿ - msg4 = @sprintf("extrema Δz: %.2e %.2e ", maximum(Δz), minimum(Δz)) - @info msg0 * msg1 * msg2 * msg3 * msg4 - else - @info msg0 * msg1 * msg2 * msg3 - end + msg1 = @sprintf("extrema w: %.2e %.2e ", maximum(w), minimum(w)) + msg2 = @sprintf("extrema u: %.2e %.2e ", maximum(u), minimum(u)) + msg3 = @sprintf("extrema b: %.2e %.2e ", maximum(b), minimum(b)) + msg4 = @sprintf("extrema Δz: %.2e %.2e ", maximum(Δz), minimum(Δz)) + @info msg0 * msg1 * msg2 * msg3 * msg4 return nothing end @@ -82,17 +76,16 @@ end using Oceananigans.Fields: OneField # # Check tracer conservation -b = FieldTimeSeries("zstar_model.jld2", "b") -s = FieldTimeSeries("zstar_model.jld2", "sⁿ") +b = FieldTimeSeries("zstar_model.jld2", "b") +Δz = FieldTimeSeries("zstar_model.jld2", "Δz") -# s = OneField() tmpfield = CenterField(grid) -launch!(CPU(), grid, :xyz, _compute_field!, tmpfield, s[1], b[1]) -init = sum(tmpfield) / sum(s[1]) +launch!(CPU(), grid, :xyz, _compute_field!, tmpfield, Δz[1], b[1]) +init = sum(tmpfield) / sum(Δz[1]) drift = [] for t in 1:length(b.times) - launch!(CPU(), grid, :xyz, _compute_field!, tmpfield, s[t], b[t]) - push!(drift, sum(tmpfield) / sum(s[t]) - init) + launch!(CPU(), grid, :xyz, _compute_field!, tmpfield, Δz[t], b[t]) + push!(drift, sum(tmpfield) / sum(Δz[t]) - init) end From 6776ded98fa9ba0a4dc45751368b567322b10636 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 7 Oct 2024 14:26:59 +0200 Subject: [PATCH 311/567] works for immersed boundaries --- .../compute_w_from_continuity.jl | 4 +++- .../split_explicit_free_surface.jl | 16 +++++++++------- validation/z_star_coordinate/lock_release.jl | 2 +- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl index f85f57ba7d..eafd7a2784 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl @@ -2,6 +2,7 @@ using Oceananigans.Architectures: device using Oceananigans.Grids: halo_size, topology using Oceananigans.Grids: XFlatGrid, YFlatGrid using Oceananigans.Operators: flux_div_xyᶜᶜᶜ, div_xyᶜᶜᶜ, Δzᶜᶜᶜ +using Oceananigans.ImmersedBoundaries: immersed_cell """ compute_w_from_continuity!(model) @@ -25,8 +26,9 @@ compute_w_from_continuity!(velocities, arch, grid; parameters = w_kernel_paramet for k in 2:grid.Nz+1 δ_Uh = flux_div_xyᶜᶜᶜ(i, j, k-1, grid, U.u, U.v) / Azᶜᶜᶜ(i, j, k-1, grid) ∂t_s = Δrᶜᶜᶜ(i, j, k-1, grid) * ∂t_s_grid(i, j, k-1, grid) + immersed = immersed_cell(i, j, k-1, grid) - @inbounds U.w[i, j, k] = U.w[i, j, k-1] - ∂t_s - δ_Uh + @inbounds U.w[i, j, k] = U.w[i, j, k-1] - ifelse(immersed, 0, ∂t_s + δ_Uh) end end diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index 8b44cd1764..c65e66e13c 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -239,25 +239,27 @@ function SplitExplicitAuxiliaryFields(grid::AbstractGrid) Gᵁ = Field{Face, Center, Nothing}(grid) Gⱽ = Field{Center, Face, Nothing}(grid) + Hᶜᶜ = Field{Center, Center, Nothing}(grid) Hᶠᶜ = Field{Face, Center, Nothing}(grid) Hᶜᶠ = Field{Center, Face, Nothing}(grid) - Hᶜᶜ = Field{Center, Center, Nothing}(grid) Hᶠᶠ = Field{Face, Face, Nothing}(grid) - compute_column_height!(Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, Hᶠᶠ, grid) + compute_column_height!(Hᶜᶜ, Hᶠᶜ, Hᶜᶠ, Hᶠᶠ, grid) kernel_parameters = :xy return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, Hᶠᶠ, kernel_parameters) end -function compute_column_height!(Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, Hᶠᶠ, grid) +function compute_column_height!(Hᶜᶜ, Hᶠᶜ, Hᶜᶠ, Hᶠᶠ, grid) arch = architecture(grid) - launch!(arch, grid, :xy, _compute_column_height!, Hᶠᶜ, grid, f, c, Δrᶠᶜᶜ; include_right_boundaries = true, location = (Face, Center, Nothing)) - launch!(arch, grid, :xy, _compute_column_height!, Hᶜᶠ, grid, c, f, Δrᶜᶠᶜ; include_right_boundaries = true, location = (Center, Face, Nothing)) - launch!(arch, grid, :xy, _compute_column_height!, Hᶜᶜ, grid, c, c, Δrᶜᶜᶜ; include_right_boundaries = true, location = (Center, Center, Nothing)) - launch!(arch, grid, :xy, _compute_column_height!, Hᶠᶠ, grid, f, f, Δrᶠᶠᶜ; include_right_boundaries = true, location = (Face, Face, Nothing)) + include_right_boundaries = true + + launch!(arch, grid, :xy, _compute_column_height!, Hᶜᶜ, grid, c, c, Δrᶜᶜᶜ; include_right_boundaries, location = (Center, Center, Nothing)) + launch!(arch, grid, :xy, _compute_column_height!, Hᶠᶜ, grid, f, c, Δrᶠᶜᶜ; include_right_boundaries, location = (Face, Center, Nothing)) + launch!(arch, grid, :xy, _compute_column_height!, Hᶜᶠ, grid, c, f, Δrᶜᶠᶜ; include_right_boundaries, location = (Center, Face, Nothing)) + launch!(arch, grid, :xy, _compute_column_height!, Hᶠᶠ, grid, f, f, Δrᶠᶠᶜ; include_right_boundaries, location = (Face, Face, Nothing)) fill_halo_regions!((Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, Hᶠᶠ)) diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index 2053c955b6..0e04249009 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -35,7 +35,7 @@ set!(model, b = bᵢ) @info "the time step is $Δt" -simulation = Simulation(model; Δt, stop_iteration = 100, stop_time = 17hours) +simulation = Simulation(model; Δt, stop_iteration = 10000, stop_time = 17hours) Δz = GridMetricOperation((Center, Center, Center), Oceananigans.AbstractOperations.Δz, model.grid) From 7c68ad1c0efccc75d55527daf8f59397bd5bbe80 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 7 Oct 2024 14:29:22 +0200 Subject: [PATCH 312/567] make sure continuity is respected --- .../HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl index eafd7a2784..3c1228648c 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl @@ -26,9 +26,11 @@ compute_w_from_continuity!(velocities, arch, grid; parameters = w_kernel_paramet for k in 2:grid.Nz+1 δ_Uh = flux_div_xyᶜᶜᶜ(i, j, k-1, grid, U.u, U.v) / Azᶜᶜᶜ(i, j, k-1, grid) ∂t_s = Δrᶜᶜᶜ(i, j, k-1, grid) * ∂t_s_grid(i, j, k-1, grid) + immersed = immersed_cell(i, j, k-1, grid) + Δw = ifelse(immersed, 0, δ_Uh + ∂t_s) - @inbounds U.w[i, j, k] = U.w[i, j, k-1] - ifelse(immersed, 0, ∂t_s + δ_Uh) + @inbounds U.w[i, j, k] = U.w[i, j, k-1] - Δw end end From 0ed0516e9379ae46318029cca2d5b1ec3a7996bf Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 7 Oct 2024 15:55:09 +0200 Subject: [PATCH 313/567] works also for periodic --- .../compute_w_from_continuity.jl | 6 ++-- .../split_explicit_free_surface_kernels.jl | 36 +++++++------------ .../z_star_vertical_spacing.jl | 6 ++-- validation/z_star_coordinate/lock_release.jl | 6 ++-- 4 files changed, 23 insertions(+), 31 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl index 3c1228648c..3f1d6b2321 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl @@ -28,7 +28,7 @@ compute_w_from_continuity!(velocities, arch, grid; parameters = w_kernel_paramet ∂t_s = Δrᶜᶜᶜ(i, j, k-1, grid) * ∂t_s_grid(i, j, k-1, grid) immersed = immersed_cell(i, j, k-1, grid) - Δw = ifelse(immersed, 0, δ_Uh + ∂t_s) + Δw = δ_Uh + ifelse(immersed, 0, ∂t_s) # We do not account for grid changes in immersed cells @inbounds U.w[i, j, k] = U.w[i, j, k-1] - Δw end @@ -45,8 +45,8 @@ end Hx, Hy, _ = halo_size(grid) Tx, Ty, _ = topology(grid) - ii = ifelse(Tx == Flat, 1:Nx, -Hx+2:Nx+Hx-1) - jj = ifelse(Ty == Flat, 1:Ny, -Hy+2:Ny+Hy-1) + ii = ifelse(Tx == Flat, 1:Nx, -Hx+1:Nx+Hx) + jj = ifelse(Ty == Flat, 1:Ny, -Hy+1:Ny+Hy) return KernelParameters(ii, jj) end diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 5659b84cc0..780bdbf0b9 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -252,12 +252,12 @@ end return nothing end -@kernel function _barotropic_split_explicit_corrector!(u, v, grid, U̅, V̅, U, V, Hᶠᶜ, Hᶜᶠ, η̅) +@kernel function _barotropic_split_explicit_corrector!(u, v, grid, U̅, V̅, U, V, Hᶠᶜ, Hᶜᶠ, η) i, j, k = @index(Global, NTuple) k_top = grid.Nz + 1 - hᶠᶜ = dynamic_column_heightᶠᶜ(i, j, k_top, grid, Hᶠᶜ, η̅) - hᶜᶠ = dynamic_column_heightᶜᶠ(i, j, k_top, grid, Hᶜᶠ, η̅) + hᶠᶜ = dynamic_column_heightᶠᶜ(i, j, k_top, grid, Hᶠᶜ, η) + hᶜᶠ = dynamic_column_heightᶜᶠ(i, j, k_top, grid, Hᶜᶠ, η) @inbounds begin u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / hᶠᶜ @@ -267,17 +267,17 @@ end function barotropic_split_explicit_corrector!(u, v, free_surface, grid) sefs = free_surface.state - U, V, U̅, V̅ = sefs.U, sefs.V, sefs.U̅, sefs.V̅ #, sefs.Ũ, sefs.Ṽ + U, V, U̅, V̅ = sefs.U, sefs.V, sefs.U̅, sefs.V̅ Hᶠᶜ, Hᶜᶠ = free_surface.auxiliary.Hᶠᶜ, free_surface.auxiliary.Hᶜᶠ arch = architecture(grid) # take out "bad" barotropic mode, # !!!! reusing U and V for this storage since last timestep doesn't matter - compute_barotropic_mode!(U, V, grid, u, v, Hᶠᶜ, Hᶜᶠ, sefs.η̅) + compute_barotropic_mode!(U, V, grid, u, v, Hᶠᶜ, Hᶜᶠ, free_surface.η) # add in "good" barotropic mode launch!(arch, grid, :xyz, _barotropic_split_explicit_corrector!, - u, v, grid, U̅, V̅, U, V, Hᶠᶜ, Hᶜᶠ, sefs.η̅) + u, v, grid, U̅, V̅, U, V, Hᶠᶜ, Hᶜᶠ, free_surface.η) return nothing end @@ -360,7 +360,7 @@ function split_explicit_free_surface_step!(free_surface::SplitExplicitFreeSurfac set!(free_surface.η, free_surface.state.η̅) end - fields_to_fill = (free_surface.state.U̅, free_surface.state.V̅) + fields_to_fill = (free_surface.state.U̅, free_surface.state.V̅, free_surface.η) fill_halo_regions!(fields_to_fill; async = true) # Preparing velocities for the barotropic correction @@ -387,9 +387,6 @@ const MINIMUM_SUBSTEPS = 5 @inline calculate_adaptive_settings(substepping::FTS, substeps) = weights_from_substeps(eltype(substepping.Δt_barotropic), substeps, substepping.averaging_kernel) -const FixedSubstepsSetting{N} = SplitExplicitSettings{<:FixedSubstepNumber{<:Any, <:NTuple{N, <:Any}}} where N -const FixedSubstepsSplitExplicit{F} = SplitExplicitFreeSurface{<:Any, <:Any, <:Any, <:Any, <:FixedSubstepsSetting{N}} where N - function iterate_split_explicit!(free_surface, grid, Δτᴮ, weights, ::Val{Nsubsteps}) where Nsubsteps arch = architecture(grid) @@ -472,37 +469,30 @@ end end end -@inline function ab2_step_Gu(i, j, k, grid::ZStarSpacingGrid, G⁻, Gⁿ, χ::FT) where FT +@inline function ab2_step_Gu(i, j, k, grid, G⁻, Gⁿ, χ::FT) where FT C¹ = convert(FT, 3/2) + χ C² = convert(FT, 1/2) + χ - Fⁿ = @inbounds C¹ * Gⁿ[i, j, k] * grid.Δzᵃᵃᶠ.sᶠᶜⁿ[i, j, 1] - F⁻ = @inbounds C² * G⁻[i, j, k] * grid.Δzᵃᵃᶠ.sᶠᶜ⁻[i, j, 1] + Fⁿ = @inbounds C¹ * Gⁿ[i, j, k] * vertical_scaling(i, j, k, grid, Face(), Center(), Center()) + F⁻ = @inbounds C² * G⁻[i, j, k] * previous_vertical_scaling(i, j, k, grid, Face(), Center(), Center()) Gi = Fⁿ - F⁻ return ifelse(peripheral_node(i, j, k, grid, f, c, c), zero(grid), Gi) end -@inline function ab2_step_Gv(i, j, k, grid::ZStarSpacingGrid, G⁻, Gⁿ, χ::FT) where FT +@inline function ab2_step_Gv(i, j, k, grid, G⁻, Gⁿ, χ::FT) where FT C¹ = convert(FT, 3/2) + χ C² = convert(FT, 1/2) + χ - Fⁿ = @inbounds C¹ * Gⁿ[i, j, k] * grid.Δzᵃᵃᶠ.sᶜᶠⁿ[i, j, 1] - F⁻ = @inbounds C² * G⁻[i, j, k] * grid.Δzᵃᵃᶠ.sᶜᶠ⁻[i, j, 1] + Fⁿ = @inbounds C¹ * Gⁿ[i, j, k] * vertical_scaling(i, j, k, grid, Center(), Face(), Center()) + F⁻ = @inbounds C² * G⁻[i, j, k] * previous_vertical_scaling(i, j, k, grid, Center(), Face(), Center()) Gi = Fⁿ - F⁻ return ifelse(peripheral_node(i, j, k, grid, f, c, c), zero(grid), Gi) end -# Setting up the RHS for the barotropic step (tendencies of the barotropic velocity components) -@inline ab2_step_Gu(i, j, k, grid, G⁻, Gⁿ, χ::FT) where FT = - @inbounds ifelse(peripheral_node(i, j, k, grid, f, c, c), zero(grid), (convert(FT, 1.5) + χ) * Gⁿ[i, j, k] - G⁻[i, j, k] * (convert(FT, 0.5) + χ)) - -@inline ab2_step_Gv(i, j, k, grid, G⁻, Gⁿ, χ::FT) where FT = - @inbounds ifelse(peripheral_node(i, j, k, grid, c, f, c), zero(grid), (convert(FT, 1.5) + χ) * Gⁿ[i, j, k] - G⁻[i, j, k] * (convert(FT, 0.5) + χ)) - # Setting up the RHS for the barotropic step (tendencies of the barotropic velocity components) # This function is called after `calculate_tendency` and before `ab2_step_velocities!` function setup_free_surface!(model, free_surface::SplitExplicitFreeSurface, χ) diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 42af9a08ba..95d9b42857 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -185,10 +185,12 @@ end @kernel function _update_∂t_s!(∂t_s, U̅, V̅, Hᶜᶜ, grid) i, j = @index(Global, NTuple) k_top = grid.Nz + 1 + TX, TY, _ = topology(grid) + @inbounds begin # ∂(η / H)/∂t = - ∇ ⋅ ∫udz / H - ∂t_s[i, j, 1] = - 1 / Azᶜᶜᶠ(i, j, k_top-1, grid) * (δxᶜᵃᵃ(i, j, k_top-1, grid, Δy_qᶠᶜᶠ, U̅) + - δyᵃᶜᵃ(i, j, k_top-1, grid, Δx_qᶜᶠᶠ, V̅)) / Hᶜᶜ[i, j, 1] + ∂t_s[i, j, 1] = - 1 / Azᶜᶜᶠ(i, j, k_top-1, grid) * (δxᶜᶜᶠ(i, j, k_top-1, grid, Δy_qᶠᶜᶠ, U̅) + + δyᶜᶜᶠ(i, j, k_top-1, grid, Δx_qᶜᶠᶠ, V̅)) / Hᶜᶜ[i, j, 1] end end diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index 0e04249009..b4e55a97f0 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -6,11 +6,11 @@ using Oceananigans.AbstractOperations: GridMetricOperation using Oceananigans.Models.HydrostaticFreeSurfaceModels: ZStar, ZStarSpacingGrid, Δrᶜᶜᶜ using Printf -grid = RectilinearGrid(size = (10, 20), +grid = RectilinearGrid(size = (20, 20), x = (0, 64kilometers), z = (-20, 0), halo = (6, 6), - topology = (Bounded, Flat, Bounded)) + topology = (Periodic, Flat, Bounded)) grid = ImmersedBoundaryGrid(grid, GridFittedBottom(x -> x < 32kilometers ? -10 : -20)) @@ -31,7 +31,7 @@ bᵢ(x, z) = x < 32kilometers ? 0.06 : 0.01 set!(model, b = bᵢ) -Δt = 1 +Δt = 10 @info "the time step is $Δt" From 21e7f0373251d068a0cc35cccf9f2f4eaa6fc986 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 7 Oct 2024 15:57:41 +0200 Subject: [PATCH 314/567] some comment --- .../split_explicit_free_surface_kernels.jl | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 780bdbf0b9..2e3dd3699c 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -209,12 +209,12 @@ end # For Zstar vertical spacing the vertical integral includes the dynamic height # Remember, the vertical coordinate has not yet been updated! # For this reason the integration has to be performed manually -@kernel function _barotropic_mode_kernel!(U, V, grid, ::Nothing, u, v, Hᶠᶜ, Hᶜᶠ, η̅) +@kernel function _barotropic_mode_kernel!(U, V, grid, ::Nothing, u, v, Hᶠᶜ, Hᶜᶠ, η) i, j = @index(Global, NTuple) k_top = grid.Nz + 1 - sᶠᶜ = @inbounds dynamic_column_heightᶠᶜ(i, j, k_top, grid, Hᶠᶜ, η̅) / Hᶠᶜ[i, j, 1] - sᶜᶠ = @inbounds dynamic_column_heightᶜᶠ(i, j, k_top, grid, Hᶜᶠ, η̅) / Hᶜᶠ[i, j, 1] + sᶠᶜ = @inbounds dynamic_column_heightᶠᶜ(i, j, k_top, grid, Hᶠᶜ, η) / Hᶠᶜ[i, j, 1] + sᶜᶠ = @inbounds dynamic_column_heightᶜᶠ(i, j, k_top, grid, Hᶜᶠ, η) / Hᶜᶠ[i, j, 1] # hand unroll first loop @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δrᶠᶜᶜ(i, j, 1, grid) * sᶠᶜ @@ -226,13 +226,13 @@ end end end -@kernel function _barotropic_mode_kernel!(U, V, grid, active_cells_map, u, v, Hᶠᶜ, Hᶜᶠ, η̅) +@kernel function _barotropic_mode_kernel!(U, V, grid, active_cells_map, u, v, Hᶠᶜ, Hᶜᶠ, η) idx = @index(Global, Linear) i, j = active_linear_index_to_tuple(idx, active_cells_map) k_top = grid.Nz+1 - sᶠᶜ = @inbounds dynamic_column_heightᶠᶜ(i, j, k_top, grid, Hᶠᶜ, η̅) / Hᶠᶜ[i, j, 1] - sᶜᶠ = @inbounds dynamic_column_heightᶜᶠ(i, j, k_top, grid, Hᶜᶠ, η̅) / Hᶜᶠ[i, j, 1] + sᶠᶜ = @inbounds dynamic_column_heightᶠᶜ(i, j, k_top, grid, Hᶠᶜ, η) / Hᶠᶜ[i, j, 1] + sᶜᶠ = @inbounds dynamic_column_heightᶜᶠ(i, j, k_top, grid, Hᶜᶠ, η) / Hᶜᶠ[i, j, 1] # hand unroll first loop @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δrᶠᶜᶜ(i, j, 1, grid) * sᶠᶜ @@ -244,10 +244,10 @@ end end end -@inline function compute_barotropic_mode!(U, V, grid, u, v, Hᶠᶜ, Hᶜᶠ, η̅) +@inline function compute_barotropic_mode!(U, V, grid, u, v, Hᶠᶜ, Hᶜᶠ, η) active_cells_map = retrieve_surface_active_cells_map(grid) - launch!(architecture(grid), grid, :xy, _barotropic_mode_kernel!, U, V, grid, active_cells_map, u, v, Hᶠᶜ, Hᶜᶠ, η̅; active_cells_map) + launch!(architecture(grid), grid, :xy, _barotropic_mode_kernel!, U, V, grid, active_cells_map, u, v, Hᶠᶜ, Hᶜᶠ, η; active_cells_map) return nothing end @@ -360,7 +360,11 @@ function split_explicit_free_surface_step!(free_surface::SplitExplicitFreeSurfac set!(free_surface.η, free_surface.state.η̅) end - fields_to_fill = (free_surface.state.U̅, free_surface.state.V̅, free_surface.η) + # This is needed for the barotropic mode calculations, so it cannot be done asynchronously + fill_halo_region!(free_surface.η) + + # Velocities can be passed asynchronously + fields_to_fill = (free_surface.state.U̅, free_surface.state.V̅) fill_halo_regions!(fields_to_fill; async = true) # Preparing velocities for the barotropic correction From 6df83c84003821f13e3b5957ea30daf9f1805bbc Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 7 Oct 2024 17:09:13 +0200 Subject: [PATCH 315/567] bugfix --- .../split_explicit_free_surface_kernels.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 2e3dd3699c..4daf98d46d 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -361,7 +361,7 @@ function split_explicit_free_surface_step!(free_surface::SplitExplicitFreeSurfac end # This is needed for the barotropic mode calculations, so it cannot be done asynchronously - fill_halo_region!(free_surface.η) + fill_halo_regions!(free_surface.η) # Velocities can be passed asynchronously fields_to_fill = (free_surface.state.U̅, free_surface.state.V̅) From cce4ccc09154c2fb1e7712f720cf1cd9b06f9fee Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 8 Oct 2024 09:43:20 +0200 Subject: [PATCH 316/567] include in different PR but usefull --- .../split_explicit_free_surface.jl | 13 +++++-------- src/Utils/kernel_launching.jl | 11 +++++++++++ validation/z_star_coordinate/lock_release.jl | 4 ++-- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index c65e66e13c..4700123c30 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -254,14 +254,11 @@ end function compute_column_height!(Hᶜᶜ, Hᶠᶜ, Hᶜᶠ, Hᶠᶠ, grid) arch = architecture(grid) - include_right_boundaries = true - - launch!(arch, grid, :xy, _compute_column_height!, Hᶜᶜ, grid, c, c, Δrᶜᶜᶜ; include_right_boundaries, location = (Center, Center, Nothing)) - launch!(arch, grid, :xy, _compute_column_height!, Hᶠᶜ, grid, f, c, Δrᶠᶜᶜ; include_right_boundaries, location = (Face, Center, Nothing)) - launch!(arch, grid, :xy, _compute_column_height!, Hᶜᶠ, grid, c, f, Δrᶜᶠᶜ; include_right_boundaries, location = (Center, Face, Nothing)) - launch!(arch, grid, :xy, _compute_column_height!, Hᶠᶠ, grid, f, f, Δrᶠᶠᶜ; include_right_boundaries, location = (Face, Face, Nothing)) - - fill_halo_regions!((Hᶠᶜ, Hᶜᶠ, Hᶜᶜ, Hᶠᶠ)) + + launch!(arch, grid, KernelParameters(Hᶜᶜ), _compute_column_height!, Hᶜᶜ, grid, c, c, Δrᶜᶜᶜ) + launch!(arch, grid, KernelParameters(Hᶠᶜ), _compute_column_height!, Hᶠᶜ, grid, f, c, Δrᶠᶜᶜ) + launch!(arch, grid, KernelParameters(Hᶜᶠ), _compute_column_height!, Hᶜᶠ, grid, c, f, Δrᶜᶠᶜ) + launch!(arch, grid, KernelParameters(Hᶠᶠ), _compute_column_height!, Hᶠᶠ, grid, f, f, Δrᶠᶠᶜ) return nothing end diff --git a/src/Utils/kernel_launching.jl b/src/Utils/kernel_launching.jl index e769061114..f054aba0c0 100644 --- a/src/Utils/kernel_launching.jl +++ b/src/Utils/kernel_launching.jl @@ -2,6 +2,7 @@ ##### Utilities for launching kernels ##### +using Oceananigans using Oceananigans.Architectures using Oceananigans.Utils using Oceananigans.Grids @@ -75,6 +76,16 @@ function KernelParameters(r1::UnitRange, r2::UnitRange, r3::UnitRange) return KernelParameters(size, offsets) end +function KernelParameters(f::Oceananigans.Fields.Field) + size = Base.size(f.data) + offsets = f.data.offsets + + rdims = Oceananigans.Fields.reduced_dimensions(f) + dims = filter(n -> !(n ∈ rdims), [1, 2, 3]) + + return KernelParameters(size[dims], offsets[dims]) +end + offsets(::KernelParameters{S, O}) where {S, O} = O offsets(workspec) = nothing diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index b4e55a97f0..902ed9b5b1 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -7,10 +7,10 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: ZStar, ZStarSpacingGrid, using Printf grid = RectilinearGrid(size = (20, 20), - x = (0, 64kilometers), + y = (0, 64kilometers), z = (-20, 0), halo = (6, 6), - topology = (Periodic, Flat, Bounded)) + topology = (Flat, Periodic, Bounded)) grid = ImmersedBoundaryGrid(grid, GridFittedBottom(x -> x < 32kilometers ? -10 : -20)) From fc8468029fb9c014d7edc6c185a27c392ea2ba34 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 8 Oct 2024 14:55:29 +0200 Subject: [PATCH 317/567] revert to previous syntax --- .../split_explicit_free_surface.jl | 12 ++++++++---- src/Utils/kernel_launching.jl | 10 ---------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index 4700123c30..adbec374e1 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -254,11 +254,15 @@ end function compute_column_height!(Hᶜᶜ, Hᶠᶜ, Hᶜᶠ, Hᶠᶠ, grid) arch = architecture(grid) + Hx, Hy, _ = halo_size(grid) + Nx, Ny, _ = size(grid) - launch!(arch, grid, KernelParameters(Hᶜᶜ), _compute_column_height!, Hᶜᶜ, grid, c, c, Δrᶜᶜᶜ) - launch!(arch, grid, KernelParameters(Hᶠᶜ), _compute_column_height!, Hᶠᶜ, grid, f, c, Δrᶠᶜᶜ) - launch!(arch, grid, KernelParameters(Hᶜᶠ), _compute_column_height!, Hᶜᶠ, grid, c, f, Δrᶜᶠᶜ) - launch!(arch, grid, KernelParameters(Hᶠᶠ), _compute_column_height!, Hᶠᶠ, grid, f, f, Δrᶠᶠᶜ) + params = KernelParameters(-Hx+1:Nx+Hx, -Hy+1:Ny+Hy) + + launch!(arch, grid, params, _compute_column_height!, Hᶜᶜ, grid, c, c, Δrᶜᶜᶜ) + launch!(arch, grid, params, _compute_column_height!, Hᶠᶜ, grid, f, c, Δrᶠᶜᶜ) + launch!(arch, grid, params, _compute_column_height!, Hᶜᶠ, grid, c, f, Δrᶜᶠᶜ) + launch!(arch, grid, params, _compute_column_height!, Hᶠᶠ, grid, f, f, Δrᶠᶠᶜ) return nothing end diff --git a/src/Utils/kernel_launching.jl b/src/Utils/kernel_launching.jl index f054aba0c0..366a5f1de7 100644 --- a/src/Utils/kernel_launching.jl +++ b/src/Utils/kernel_launching.jl @@ -76,16 +76,6 @@ function KernelParameters(r1::UnitRange, r2::UnitRange, r3::UnitRange) return KernelParameters(size, offsets) end -function KernelParameters(f::Oceananigans.Fields.Field) - size = Base.size(f.data) - offsets = f.data.offsets - - rdims = Oceananigans.Fields.reduced_dimensions(f) - dims = filter(n -> !(n ∈ rdims), [1, 2, 3]) - - return KernelParameters(size[dims], offsets[dims]) -end - offsets(::KernelParameters{S, O}) where {S, O} = O offsets(workspec) = nothing From 5741b73f53aedec35856b84ed3c9fb3e3141f6a5 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 8 Oct 2024 16:15:35 +0200 Subject: [PATCH 318/567] fix the offsets --- src/BoundaryConditions/fill_halo_regions.jl | 6 ++-- .../fill_halo_regions_open.jl | 9 ++++-- .../split_explicit_free_surface.jl | 32 +++++++++---------- .../split_explicit_free_surface_kernels.jl | 24 ++++++++++++-- 4 files changed, 47 insertions(+), 24 deletions(-) diff --git a/src/BoundaryConditions/fill_halo_regions.jl b/src/BoundaryConditions/fill_halo_regions.jl index 44d72c2bd3..a96873a249 100644 --- a/src/BoundaryConditions/fill_halo_regions.jl +++ b/src/BoundaryConditions/fill_halo_regions.jl @@ -341,9 +341,9 @@ end ##### Calculate kernel size and offset for Windowed and Sliced Fields ##### -const WEB = Union{typeof(fill_west_and_east_halo!), typeof(fill_west_halo!), typeof(fill_east_halo!)} -const SNB = Union{typeof(fill_south_and_north_halo!), typeof(fill_south_halo!), typeof(fill_north_halo!)} -const TBB = Union{typeof(fill_bottom_and_top_halo!), typeof(fill_bottom_halo!), typeof(fill_top_halo!)} +const WEB = Union{typeof(fill_west_and_east_halo!), typeof(fill_west_halo!), typeof(fill_east_halo!)} +const SNB = Union{typeof(fill_south_and_north_halo!), typeof(fill_south_halo!), typeof(fill_north_halo!)} +const TBB = Union{typeof(fill_bottom_and_top_halo!), typeof(fill_bottom_halo!), typeof(fill_top_halo!)} # Tupled halo filling _only_ deals with full fields! @inline fill_halo_size(::Tuple, ::WEB, args...) = :yz diff --git a/src/BoundaryConditions/fill_halo_regions_open.jl b/src/BoundaryConditions/fill_halo_regions_open.jl index 152ebd29b7..76c1f5a9b9 100644 --- a/src/BoundaryConditions/fill_halo_regions_open.jl +++ b/src/BoundaryConditions/fill_halo_regions_open.jl @@ -15,9 +15,11 @@ function fill_open_boundary_regions!(field, boundary_conditions, indices, loc, g # gets `open_fill`, the function which fills open boundaries at `loc`, as well as `regular_fill` # which is the function which fills non-open boundaries at `loc` which informs `fill_halo_size` open_fill, regular_fill = get_open_halo_filling_functions(loc) - fill_size = fill_halo_size(field, regular_fill, indices, boundary_conditions, loc, grid) + size = fill_halo_size(field, regular_fill, indices, boundary_conditions, loc, grid) + offset = fill_halo_offset(size, regular_fill, indices) + params = KernelParameters(size, offset) - launch!(arch, grid, fill_size, open_fill, field, left_bc, right_bc, loc, grid, args) + launch!(arch, grid, params, open_fill, field, left_bc, right_bc, loc, grid, args) return nothing end @@ -52,7 +54,8 @@ fill_open_boundary_regions!(fields::NTuple, boundary_conditions, indices, loc, g @kernel _no_fill!(args...) = nothing -@inline fill_halo_size(field, ::typeof(_no_fill!), args...) = (0, 0) +@inline fill_halo_size(field, ::typeof(_no_fill!), args...) = (0, 0) +@inline fill_halo_offset(size, ::typeof(_no_fill!), args...) = (0, 0) @kernel function _fill_west_and_east_open_halo!(c, west_bc, east_bc, loc, grid, args) j, k = @index(Global, NTuple) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index adbec374e1..c78be2c5b5 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -184,19 +184,21 @@ function SplitExplicitState(grid::AbstractGrid, timestepper) ηᵐ⁻¹ = auxiliary_free_surface_field(grid, timestepper) ηᵐ⁻² = auxiliary_free_surface_field(grid, timestepper) - U = XFaceField(grid, indices = (:, :, Nz)) - V = YFaceField(grid, indices = (:, :, Nz)) - - Uᵐ⁻¹ = auxiliary_barotropic_U_field(grid, timestepper) - Vᵐ⁻¹ = auxiliary_barotropic_V_field(grid, timestepper) - Uᵐ⁻² = auxiliary_barotropic_U_field(grid, timestepper) - Vᵐ⁻² = auxiliary_barotropic_V_field(grid, timestepper) - - U̅ = XFaceField(grid, indices = (:, :, Nz)) - V̅ = YFaceField(grid, indices = (:, :, Nz)) + 𝒰 = VelocityFields(grid) + u_bcs = 𝒰.u.boundary_conditions + + U = Field(𝒰.u, indices = (:, :, Nz)) + V = Field(𝒰.v, indices = (:, :, Nz)) - Ũ = XFaceField(grid, indices = (:, :, Nz)) - Ṽ = YFaceField(grid, indices = (:, :, Nz)) + Uᵐ⁻¹ = auxiliary_barotropic_velocity_field(𝒰.u, timestepper) + Vᵐ⁻¹ = auxiliary_barotropic_velocity_field(𝒰.v, timestepper) + Uᵐ⁻² = auxiliary_barotropic_velocity_field(𝒰.u, timestepper) + Vᵐ⁻² = auxiliary_barotropic_velocity_field(𝒰.v, timestepper) + + U̅ = Field(𝒰.u, indices = (:, :, Nz)) + V̅ = Field(𝒰.v, indices = (:, :, Nz)) + Ũ = Field(𝒰.u, indices = (:, :, Nz)) + Ṽ = Field(𝒰.v, indices = (:, :, Nz)) return SplitExplicitState(; ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅, Ũ, Ṽ) end @@ -294,10 +296,8 @@ struct ForwardBackwardScheme end auxiliary_free_surface_field(grid, ::AdamsBashforth3Scheme) = ZFaceField(grid, indices = (:, :, size(grid, 3)+1)) auxiliary_free_surface_field(grid, ::ForwardBackwardScheme) = nothing -auxiliary_barotropic_U_field(grid, ::AdamsBashforth3Scheme) = XFaceField(grid, indices = (:, :, size(grid, 3))) -auxiliary_barotropic_U_field(grid, ::ForwardBackwardScheme) = nothing -auxiliary_barotropic_V_field(grid, ::AdamsBashforth3Scheme) = YFaceField(grid, indices = (:, :, size(grid, 3))) -auxiliary_barotropic_V_field(grid, ::ForwardBackwardScheme) = nothing +auxiliary_barotropic_velocity_field(u, ::AdamsBashforth3Scheme) = Field(u) +auxiliary_barotropic_velocity_field(u, ::ForwardBackwardScheme) = nothing # (p = 2, q = 4, r = 0.18927) minimize dispersion error from Shchepetkin and McWilliams (2005): https://doi.org/10.1016/j.ocemod.2004.08.002 @inline function averaging_shape_function(τ::FT; p = 2, q = 4, r = FT(0.18927)) where FT diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 4daf98d46d..a07a8a2806 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -42,6 +42,7 @@ const μ = 1 - δ - γ - ϵ # `δyᵃᶜᵃ_V` : Hardcodes NoPenetration or Periodic boundary conditions for the meridional barotropic velocity V in y direction # # The functions `η★` `U★` and `V★` represent the value of free surface, barotropic zonal and meridional velocity at time step m+1/2 + @inline δxᶠᵃᵃ_η(i, j, k, grid, T, η★::Function, args...) = δxᶠᵃᵃ(i, j, k, grid, η★, args...) @inline δyᵃᶠᵃ_η(i, j, k, grid, T, η★::Function, args...) = δyᵃᶠᵃ(i, j, k, grid, η★, args...) @inline δxᶜᵃᵃ_U(i, j, k, grid, T, U★::Function, args...) = δxᶜᵃᵃ(i, j, k, grid, U★, args...) @@ -102,6 +103,25 @@ for Topo in [:Periodic, :Bounded, :RightConnected, :LeftConnected] end end +# Interpolation + +const PGX = AbstractGrid{<:Any, <:Periodic} +const BGX = AbstractGrid{<:Any, <:Bounded} +const RGX = AbstractGrid{<:Any, <:RightConnected} + +const PGY = AbstractGrid{<:Any, <:Periodic} +const BGY = AbstractGrid{<:Any, <:Bounded} +const RGY = AbstractGrid{<:Any, <:RightConnected} + +@inline ℑxᶠᵃᵃ_η(i, j, k, grid, η) = ℑxᶠᵃᵃ(i, j, k, grid, η) +@inline ℑyᵃᶠᵃ_η(i, j, k, grid, η) = ℑyᵃᶠᵃ(i, j, k, grid, η) +@inline ℑxᶠᵃᵃ_η(i, j, k, grid::PGX, η) = ifelse(i == 1, (η[1, j, k] + η[grid.Nx, j, k]) / 2, ℑxᶠᵃᵃ(i, j, k, grid, η)) +@inline ℑyᵃᶠᵃ_η(i, j, k, grid::PGY, η) = ifelse(j == 1, (η[i, 1, k] + η[i, grid.Ny, k]) / 2, ℑyᵃᶠᵃ(i, j, k, grid, η)) +@inline ℑxᶠᵃᵃ_η(i, j, k, grid::BGX, η) = ifelse(i == 1, η[1, j, k], ℑxᶠᵃᵃ(i, j, k, grid, η)) +@inline ℑyᵃᶠᵃ_η(i, j, k, grid::BGY, η) = ifelse(j == 1, η[i, 1, k], ℑyᵃᶠᵃ(i, j, k, grid, η)) +@inline ℑxᶠᵃᵃ_η(i, j, k, grid::RGX, η) = ifelse(i == 1, η[1, j, k], ℑxᶠᵃᵃ(i, j, k, grid, η)) +@inline ℑyᵃᶠᵃ_η(i, j, k, grid::RGY, η) = ifelse(j == 1, η[i, 1, k], ℑyᵃᶠᵃ(i, j, k, grid, η)) + # Time stepping extrapolation U★, and η★ # AB3 step @@ -162,8 +182,8 @@ end @inline dynamic_column_heightᶜᶠ(i, j, k, grid, H, η) = @inbounds H[i, j, 1] # Non-linear free surface implementation -@inline dynamic_column_heightᶠᶜ(i, j, k, grid::ZStarSpacingGrid, H, η) = @inbounds H[i, j, 1] + ℑxᶠᵃᵃ(i, j, k, grid, η) -@inline dynamic_column_heightᶜᶠ(i, j, k, grid::ZStarSpacingGrid, H, η) = @inbounds H[i, j, 1] + ℑyᵃᶠᵃ(i, j, k, grid, η) +@inline dynamic_column_heightᶠᶜ(i, j, k, grid::ZStarSpacingGrid, H, η) = @inbounds H[i, j, 1] + ℑxᶠᵃᵃ_η(i, j, k, grid, η) +@inline dynamic_column_heightᶜᶠ(i, j, k, grid::ZStarSpacingGrid, H, η) = @inbounds H[i, j, 1] + ℑyᵃᶠᵃ_η(i, j, k, grid, η) @kernel function _split_explicit_barotropic_velocity!(averaging_weight, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², From 5ee08be712578ef850b058a29e19e471029d2e4e Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 8 Oct 2024 16:32:37 +0200 Subject: [PATCH 319/567] finally it works --- .../split_explicit_free_surface.jl | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index c78be2c5b5..5ae4a1df08 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -190,15 +190,15 @@ function SplitExplicitState(grid::AbstractGrid, timestepper) U = Field(𝒰.u, indices = (:, :, Nz)) V = Field(𝒰.v, indices = (:, :, Nz)) - Uᵐ⁻¹ = auxiliary_barotropic_velocity_field(𝒰.u, timestepper) - Vᵐ⁻¹ = auxiliary_barotropic_velocity_field(𝒰.v, timestepper) - Uᵐ⁻² = auxiliary_barotropic_velocity_field(𝒰.u, timestepper) - Vᵐ⁻² = auxiliary_barotropic_velocity_field(𝒰.v, timestepper) + Uᵐ⁻¹ = auxiliary_barotropic_velocity_field(U, timestepper) + Vᵐ⁻¹ = auxiliary_barotropic_velocity_field(V, timestepper) + Uᵐ⁻² = auxiliary_barotropic_velocity_field(U, timestepper) + Vᵐ⁻² = auxiliary_barotropic_velocity_field(V, timestepper) - U̅ = Field(𝒰.u, indices = (:, :, Nz)) - V̅ = Field(𝒰.v, indices = (:, :, Nz)) - Ũ = Field(𝒰.u, indices = (:, :, Nz)) - Ṽ = Field(𝒰.v, indices = (:, :, Nz)) + U̅ = deepcopy(U) + V̅ = deepcopy(V) + Ũ = deepcopy(U) + Ṽ = deepcopy(V) return SplitExplicitState(; ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅, Ũ, Ṽ) end @@ -296,8 +296,8 @@ struct ForwardBackwardScheme end auxiliary_free_surface_field(grid, ::AdamsBashforth3Scheme) = ZFaceField(grid, indices = (:, :, size(grid, 3)+1)) auxiliary_free_surface_field(grid, ::ForwardBackwardScheme) = nothing -auxiliary_barotropic_velocity_field(u, ::AdamsBashforth3Scheme) = Field(u) -auxiliary_barotropic_velocity_field(u, ::ForwardBackwardScheme) = nothing +auxiliary_barotropic_velocity_field(U, ::AdamsBashforth3Scheme) = deecopy(u) +auxiliary_barotropic_velocity_field(U, ::ForwardBackwardScheme) = nothing # (p = 2, q = 4, r = 0.18927) minimize dispersion error from Shchepetkin and McWilliams (2005): https://doi.org/10.1016/j.ocemod.2004.08.002 @inline function averaging_shape_function(τ::FT; p = 2, q = 4, r = FT(0.18927)) where FT From e73cadf82ffad063eb0cd770248e5819463b2c42 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 8 Oct 2024 16:33:32 +0200 Subject: [PATCH 320/567] add a validation --- .../baroclinic_double_gyre.jl | 161 ++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 validation/z_star_coordinate/baroclinic_double_gyre.jl diff --git a/validation/z_star_coordinate/baroclinic_double_gyre.jl b/validation/z_star_coordinate/baroclinic_double_gyre.jl new file mode 100644 index 0000000000..543068fd9f --- /dev/null +++ b/validation/z_star_coordinate/baroclinic_double_gyre.jl @@ -0,0 +1,161 @@ +using Oceananigans +using Oceananigans.Units +using Oceananigans.Operators +using Oceananigans.Grids: φnode +using Oceananigans.Models.HydrostaticFreeSurfaceModels: ZStar +using Oceananigans.AbstractOperations: GridMetricOperation +using Printf + +arch = CPU() +grid = LatitudeLongitudeGrid(arch; size = (60, 60, 18), + latitude = (15, 75), + longitude = (0, 60), + halo = (5, 5, 5), + z = (-1800, 0)) + +##### +##### Parameters +##### + +θ⁺ = 30 # ᵒC maximum temperature +θ⁻ = 0 # ᵒC maximum temperature +α = 2e-4 # ᵒC⁻¹ thermal expansion coefficient +ρ₀ = 1000 # kg m⁻³ reference density +g = 9.80665 # m s⁻² gravitational acceleration +λ = 30days # time scale for restoring + +##### +##### Numerics +##### + +Δt = 20minutes + +Δx = minimum_xspacing(grid) +Δy = minimum_yspacing(grid) + +Δs = sqrt(1 / (1 / Δx^2 + 1 / Δy^2)) +sp = sqrt(g * grid.Lz) +CFL = 0.75 +Δτ = Δs / sp * CFL + +substeps = ceil(Int, 3 * Δt / Δτ) + +coriolis = HydrostaticSphericalCoriolis() +momentum_advection = WENOVectorInvariant(vorticity_order = 5) +tracer_advection = WENO(order = 5) +vertical_coordinate = ZStar() +free_surface = SplitExplicitFreeSurface(grid; substeps) + +numerics = (; coriolis, free_surface, momentum_advection, tracer_advection, vertical_coordinate) + +##### +##### Closure +##### + +closure = ConvectiveAdjustmentVerticalDiffusivity(convective_κz = 1.0, + background_κz = 1e-5, + convective_νz = 1e-2, + background_νz = 1e-2) + +##### +##### Boundary Conditions +##### + +@inline function wind_stress(i, j, grid, clock, fields, p) + τ₀ = p.τ₀ + y = (φnode(j, grid, Center()) - p.φ₀) / grid.Ly + + return τ₀ * cos(2π * y) +end + +@inline function buoyancy_restoring(i, j, grid, clock, fields, p) + b = @inbounds fields.b[i, j, 1] + y = (φnode(j, grid, Center()) - p.φ₀) / grid.Ly + b★ = p.Δb * y + + return p.𝓋 * (b - b★) +end + +Δz₀ = Δzᶜᶜᶜ(1, 1, grid.Nz, grid) # Surface layer thickness + +Δb = α * g * (θ⁺ - θ⁻) # Buoyancy difference + +parameters = (; τ₀ = 0.1 / ρ₀, # Wind stress + φ₀ = 15, # Latitude of southern edge + Δb, # Buoyancy difference + 𝓋 = Δz₀ / λ) # Pumping velocity for restoring + +u_boundary = FluxBoundaryCondition(wind_stress; discrete_form = true, parameters) +b_boundary = FluxBoundaryCondition(buoyancy_restoring; discrete_form = true, parameters) + +no_slip = ValueBoundaryCondition(0.0) + +u_bcs = FieldBoundaryConditions(north = no_slip, south = no_slip, top = u_boundary) +v_bcs = FieldBoundaryConditions(west = no_slip, east = no_slip) +b_bcs = FieldBoundaryConditions(top = b_boundary) + +##### +##### Model +##### + +model = HydrostaticFreeSurfaceModel(; grid, + boundary_conditions = (u = u_bcs, v = v_bcs, b = b_bcs), + buoyancy = BuoyancyTracer(), + tracers = :b, + numerics..., + closure) + +N² = Δb / grid.Lz +bᵢ(x, y, z) = N² * (grid.Lz + z) + +set!(model, b = bᵢ) + +##### +##### Simulation +##### + +simulation = Simulation(model; Δt, stop_time = 2000days) + +##### +##### Output +##### + + +Δzᶜᶜ = GridMetricOperation((Center, Center, Center), Oceananigans.AbstractOperations.Δz, model.grid) +Δzᶠᶜ = GridMetricOperation((Face, Center, Center), Oceananigans.AbstractOperations.Δz, model.grid) +Δzᶜᶠ = GridMetricOperation((Center, Face, Center), Oceananigans.AbstractOperations.Δz, model.grid) + +field_outputs = merge(model.velocities, + model.tracers, + model.pressure, + (; Δzᶜᶜ, Δzᶠᶜ, Δzᶜᶠ)) + +function progress(sim) + w = interior(sim.model.velocities.w, :, :, sim.model.grid.Nz+1) + u = sim.model.velocities.u + b = sim.model.tracers.b + + msg0 = @sprintf("Time: %s iteration %d ", prettytime(sim.model.clock.time), sim.model.clock.iteration) + msg1 = @sprintf("extrema w: %.2e %.2e ", maximum(w), minimum(w)) + msg2 = @sprintf("extrema u: %.2e %.2e ", maximum(u), minimum(u)) + msg3 = @sprintf("extrema b: %.2e %.2e ", maximum(b), minimum(b)) + msg4 = @sprintf("extrema Δz: %.2e %.2e ", maximum(Δzᶜᶜ), minimum(Δzᶜᶜ)) + @info msg0 * msg1 * msg2 * msg3 * msg4 + + return nothing +end + +simulation.callbacks[:progress] = Callback(progress, IterationInterval(1)) + +simulation.output_writers[:snapshots] = JLD2OutputWriter(model, field_outputs, + overwrite_existing = true, + schedule = TimeInterval(60days), + filename = "baroclinic_double_gyre") + +simulation.output_writers[:free_surface] = JLD2OutputWriter(model, (; η = model.free_surface.η), + overwrite_existing = true, + indices = (:, :, grid.Nz+1), + schedule = TimeInterval(60days), + filename = "baroclinic_double_gyre_free_surface") + +run!(simulation) \ No newline at end of file From 65ee3421305997cd165ca14c518fb4622542ded4 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 8 Oct 2024 16:34:07 +0200 Subject: [PATCH 321/567] not so much progress --- validation/z_star_coordinate/baroclinic_double_gyre.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/validation/z_star_coordinate/baroclinic_double_gyre.jl b/validation/z_star_coordinate/baroclinic_double_gyre.jl index 543068fd9f..0a16de900e 100644 --- a/validation/z_star_coordinate/baroclinic_double_gyre.jl +++ b/validation/z_star_coordinate/baroclinic_double_gyre.jl @@ -120,7 +120,6 @@ simulation = Simulation(model; Δt, stop_time = 2000days) ##### Output ##### - Δzᶜᶜ = GridMetricOperation((Center, Center, Center), Oceananigans.AbstractOperations.Δz, model.grid) Δzᶠᶜ = GridMetricOperation((Face, Center, Center), Oceananigans.AbstractOperations.Δz, model.grid) Δzᶜᶠ = GridMetricOperation((Center, Face, Center), Oceananigans.AbstractOperations.Δz, model.grid) @@ -145,7 +144,7 @@ function progress(sim) return nothing end -simulation.callbacks[:progress] = Callback(progress, IterationInterval(1)) +simulation.callbacks[:progress] = Callback(progress, IterationInterval(100)) simulation.output_writers[:snapshots] = JLD2OutputWriter(model, field_outputs, overwrite_existing = true, From 4044e1648a71ab86f19d6114aaff0ef802676f6c Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 10:09:36 +0200 Subject: [PATCH 322/567] new bottom height --- docs/src/grids.md | 4 +- examples/internal_tide.jl | 2 +- src/Grids/grid_utils.jl | 9 ++ .../abstract_grid_fitted_boundary.jl | 18 --- .../distributed_immersed_boundaries.jl | 10 +- src/ImmersedBoundaries/grid_fitted_bottom.jl | 105 +++++++++--------- src/ImmersedBoundaries/partial_cell_bottom.jl | 56 ++++++---- src/MultiRegion/multi_region_grid.jl | 4 +- .../immersed_couette_flow.jl | 4 +- 9 files changed, 105 insertions(+), 107 deletions(-) diff --git a/docs/src/grids.md b/docs/src/grids.md index 71ec02adc0..5e25849b05 100644 --- a/docs/src/grids.md +++ b/docs/src/grids.md @@ -207,11 +207,11 @@ mountain_grid = ImmersedBoundaryGrid(grid, GridFittedBottom(mountain)) ```@example grids using CairoMakie -h = mountain_grid.immersed_boundary.bottom_height +h = mountain_grid.immersed_boundary.z_bottom fig = Figure(size=(600, 600)) ax = Axis(fig[2, 1], xlabel="x (m)", ylabel="y (m)", aspect=1) -hm = heatmap!(ax, h) +hm = heatmap!(ax, - h) Colorbar(fig[1, 1], hm, vertical=false, label="Bottom height (m)") current_figure() diff --git a/examples/internal_tide.jl b/examples/internal_tide.jl index 5b31f18418..a08dedd65a 100644 --- a/examples/internal_tide.jl +++ b/examples/internal_tide.jl @@ -52,7 +52,7 @@ grid = ImmersedBoundaryGrid(underlying_grid, PartialCellBottom(bottom)) # Let's see how the domain with the bathymetry is. x = xnodes(grid, Center()) -bottom_boundary = interior(grid.immersed_boundary.bottom_height, :, 1, 1) +bottom_boundary = interior(grid.immersed_boundary.z_bottom, :, 1, 1) top_boundary = 0 * x using CairoMakie diff --git a/src/Grids/grid_utils.jl b/src/Grids/grid_utils.jl index c881ebeb86..afd199abe0 100644 --- a/src/Grids/grid_utils.jl +++ b/src/Grids/grid_utils.jl @@ -314,6 +314,15 @@ coordinate_summary(topo, Δ::Union{AbstractVector, AbstractMatrix}, name) = name, prettysummary(minimum(parent(Δ))), name, prettysummary(maximum(parent(Δ)))) +##### +##### Bottom height +##### + +@inline bottom_heightᶜᶜᵃ(i, j, k, grid) = grid.Lz +@inline bottom_heightᶜᶠᵃ(i, j, k, grid) = grid.Lz +@inline bottom_heightᶠᶜᵃ(i, j, k, grid) = grid.Lz +@inline bottom_heightᶠᶠᵃ(i, j, k, grid) = grid.Lz + ##### ##### Spherical geometry ##### diff --git a/src/ImmersedBoundaries/abstract_grid_fitted_boundary.jl b/src/ImmersedBoundaries/abstract_grid_fitted_boundary.jl index dcc442b6ac..67d9de83d3 100644 --- a/src/ImmersedBoundaries/abstract_grid_fitted_boundary.jl +++ b/src/ImmersedBoundaries/abstract_grid_fitted_boundary.jl @@ -52,21 +52,3 @@ const AGFB = AbstractGridFittedBoundary @inline immersed_cell(i, j, k, grid::AbstractGrid{<:Any, <:Any, Flat, Flat}, ib::AGFB) = _immersed_cell(i, 1, 1, grid, ib) @inline immersed_cell(i, j, k, grid::AbstractGrid{<:Any, Flat, Flat, Flat}, ib::AGFB) = _immersed_cell(1, 1, 1, grid, ib) end - -function clamp_bottom_height!(bottom_field, grid) - launch!(architecture(grid), grid, :xy, _clamp_bottom_height!, bottom_field, grid) - return nothing -end - -const c = Center() -const f = Face() - -@kernel function _clamp_bottom_height!(z, grid) - i, j = @index(Global, NTuple) - Nz = size(grid, 3) - zmin = znode(i, j, 1, grid, c, c, f) - zmax = znode(i, j, Nz+1, grid, c, c, f) - @inbounds z[i, j, 1] = clamp(z[i, j, 1], zmin, zmax) -end - - diff --git a/src/ImmersedBoundaries/distributed_immersed_boundaries.jl b/src/ImmersedBoundaries/distributed_immersed_boundaries.jl index 4b19202b92..10473ba015 100644 --- a/src/ImmersedBoundaries/distributed_immersed_boundaries.jl +++ b/src/ImmersedBoundaries/distributed_immersed_boundaries.jl @@ -14,7 +14,7 @@ function reconstruct_global_grid(grid::ImmersedBoundaryGrid) arch = grid.architecture local_ib = grid.immersed_boundary global_ug = reconstruct_global_grid(grid.underlying_grid) - global_ib = getnamewrapper(local_ib)(construct_global_array(arch, local_ib.bottom_height, size(grid))) + global_ib = getnamewrapper(local_ib)(construct_global_array(arch, local_ib.z_bottom, size(grid))) return ImmersedBoundaryGrid(global_ug, global_ib) end @@ -33,9 +33,9 @@ function scatter_local_grids(global_grid::ImmersedBoundaryGrid, arch::Distribute local_ug = scatter_local_grids(ug, arch, local_size) # Kinda hacky - local_bottom_height = partition(ib.bottom_height, arch, local_size) + local_z_bottom = partition(ib.z_bottom, arch, local_size) ImmersedBoundaryConstructor = getnamewrapper(ib) - local_ib = ImmersedBoundaryConstructor(local_bottom_height) + local_ib = ImmersedBoundaryConstructor(local_z_bottom) return ImmersedBoundaryGrid(local_ug, local_ib) end @@ -76,10 +76,10 @@ function resize_immersed_boundary(ib::AbstractGridFittedBottom{<:OffsetArray}, g # Check that the size of a bottom field are # consistent with the size of the grid - if any(size(ib.bottom_height) .!= bottom_heigth_size) + if any(size(ib.z_bottom) .!= bottom_heigth_size) @warn "Resizing the bottom field to match the grids' halos" bottom_field = Field((Center, Center, Nothing), grid) - cpu_bottom = on_architecture(CPU(), ib.bottom_height)[1:Nx, 1:Ny] + cpu_bottom = on_architecture(CPU(), ib.z_bottom)[1:Nx, 1:Ny] set!(bottom_field, cpu_bottom) fill_halo_regions!(bottom_field) offset_bottom_array = dropdims(bottom_field.data, dims=3) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index e007f461a1..b21609dd2b 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -15,48 +15,29 @@ import Oceananigans.TurbulenceClosures: z_bottom abstract type AbstractGridFittedBottom{H} <: AbstractGridFittedBoundary end -# To enable comparison with PartialCellBottom in the limiting case that -# fractional cell height is 1.0. -struct CenterImmersedCondition end -struct InterfaceImmersedCondition end - -Base.summary(::CenterImmersedCondition) = "CenterImmersedCondition" -Base.summary(::InterfaceImmersedCondition) = "InterfaceImmersedCondition" - -struct GridFittedBottom{H, I} <: AbstractGridFittedBottom{H} - bottom_height :: H - immersed_condition :: I +struct GridFittedBottom{H} <: AbstractGridFittedBottom{H} + z_bottom :: H end const GFBIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:GridFittedBottom} """ - GridFittedBottom(bottom_height, [immersed_condition=CenterImmersedCondition()]) + GridFittedBottom(z_bottom) Return a bottom immersed boundary. Keyword Arguments ================= -* `bottom_height`: an array or function that gives the height of the - bottom in absolute ``z`` coordinates. - -* `immersed_condition`: Determine whether the part of the domain that is - immersed are all the cell centers that lie below - `bottom_height` (`CenterImmersedCondition()`; default) - or all the cell faces that lie below `bottom_height` - (`InterfaceImmersedCondition()`). The only purpose of - `immersed_condition` to allow `GridFittedBottom` and - `PartialCellBottom` to have the same behavior when the - minimum fractional cell height for partial cells is set - to 0. +* `z_bottom`: an array or function that gives the height of the + bottom in absolute ``z`` coordinates. This is the height of + the bottom interface of the last ``fluid`` cell. """ -GridFittedBottom(bottom_height) = GridFittedBottom(bottom_height, CenterImmersedCondition()) function Base.summary(ib::GridFittedBottom) - zmax = maximum(ib.bottom_height) - zmin = minimum(ib.bottom_height) - zmean = mean(ib.bottom_height) + zmax = maximum(ib.z_bottom) + zmin = minimum(ib.z_bottom) + zmean = mean(ib.z_bottom) summary1 = "GridFittedBottom(" @@ -69,55 +50,69 @@ function Base.summary(ib::GridFittedBottom) return summary1 * summary2 * summary3 end -Base.summary(ib::GridFittedBottom{<:Function}) = @sprintf("GridFittedBottom(%s)", ib.bottom_height) +Base.summary(ib::GridFittedBottom{<:Function}) = @sprintf("GridFittedBottom(%s)", ib.z_bottom) function Base.show(io::IO, ib::GridFittedBottom) print(io, summary(ib), '\n') - print(io, "├── bottom_height: ", prettysummary(ib.bottom_height), '\n') - print(io, "└── immersed_condition: ", summary(ib.immersed_condition)) + print(io, "├── z_bottom: ", prettysummary(ib.z_bottom), '\n') +end + +on_architecture(arch, ib::GridFittedBottom) = GridFittedBottom(on_architecture(ib.bottom_height)) + +function on_architecture(arch, ib::GridFittedBottom{<:Field}) + architecture(ib.bottom_height) == arch && return ib + arch_grid = on_architecture(arch, ib.bottom_height.grid) + new_bottom_height = Field{Center, Center, Nothing}(arch_grid) + set!(new_bottom_height, ib.bottom_height) + fill_halo_regions!(new_bottom_height) + return GridFittedBottom(new_bottom_height, ib.immersed_condition) end +Adapt.adapt_structure(to, ib::GridFittedBottom) = GridFittedBottom(adapt(to, ib.z_bottom)) + """ ImmersedBoundaryGrid(grid, ib::GridFittedBottom) Return a grid with `GridFittedBottom` immersed boundary (`ib`). -Computes `ib.bottom_height` and wraps it in a Field. +Computes `ib.z_bottom` and wraps it in a Field. """ function ImmersedBoundaryGrid(grid, ib::GridFittedBottom) bottom_field = Field{Center, Center, Nothing}(grid) - set!(bottom_field, ib.bottom_height) - @apply_regionally clamp_bottom_height!(bottom_field, grid) + set!(bottom_field, ib.z_bottom) + @apply_regionally correct_z_bottom!(bottom_field, grid) fill_halo_regions!(bottom_field) - new_ib = GridFittedBottom(bottom_field, ib.immersed_condition) + new_ib = GridFittedBottom(bottom_field) TX, TY, TZ = topology(grid) return ImmersedBoundaryGrid{TX, TY, TZ}(grid, new_ib) end -@inline function _immersed_cell(i, j, k, underlying_grid, ib::GridFittedBottom{<:Any, <:InterfaceImmersedCondition}) - z = znode(i, j, k+1, underlying_grid, c, c, f) - h = @inbounds ib.bottom_height[i, j, 1] - return z ≤ h +correct_z_bottom!(bottom_field, grid, ib) = + launch!(architecture(grid), grid, :xy, _correct_z_bottom!, bottom_field, grid, ib) + +@kernel function _correct_z_bottom!(bottom_field, grid, ::GridFittedBottom) + i, j = @index(grid) + zb = @inbounds bottom_field[i, j, 1] + for k in 1:grid.Nz + z⁺ = znode(i, j, k+1, grid, c, c, f) + bottom_cell = zb < z⁺ + @inbounds bottom_field[i, j, 1] = ifelse(bottom_cell, z⁺, zb) + end end -@inline function _immersed_cell(i, j, k, underlying_grid, ib::GridFittedBottom{<:Any, <:CenterImmersedCondition}) - z = znode(i, j, k, underlying_grid, c, c, c) - h = @inbounds ib.bottom_height[i, j, 1] +@inline function _immersed_cell(i, j, k, underlying_grid, ib::GridFittedBottom) + z = znode(i, j, k+1, underlying_grid, c, c, f) + h = @inbounds ib.z_bottom[i, j, 1] return z ≤ h end -@inline z_bottom(i, j, ibg::GFBIBG) = @inbounds ibg.immersed_boundary.bottom_height[i, j, 1] +@inline z_bottom(i, j, ibg::GFBIBG) = @inbounds ibg.immersed_boundary.z_bottom[i, j, 1] -on_architecture(arch, ib::GridFittedBottom) = GridFittedBottom(ib.bottom_height, ib.immersed_condition) - -function on_architecture(arch, ib::GridFittedBottom{<:Field}) - architecture(ib.bottom_height) == arch && return ib - arch_grid = on_architecture(arch, ib.bottom_height.grid) - new_bottom_height = Field{Center, Center, Nothing}(arch_grid) - set!(new_bottom_height, ib.bottom_height) - fill_halo_regions!(new_bottom_height) - return GridFittedBottom(new_bottom_height, ib.immersed_condition) -end +##### +##### Bottom height +##### -Adapt.adapt_structure(to, ib::GridFittedBottom) = GridFittedBottom(adapt(to, ib.bottom_height), - ib.immersed_condition) +@inline bottom_heightᶜᶜᵃ(i, j, k, ibg::GFBIBG) = ibg.immersed_boundary.bottom_height[i, j, 1] +@inline bottom_heightᶜᶠᵃ(i, j, k, ibg::GFBIBG) = min(bottom_heightᶜᶜᵃ(i, j-1, k, ibg), bottom_heightᶜᶜᵃ(i, j, k, ibg)) +@inline bottom_heightᶠᶜᵃ(i, j, k, ibg::GFBIBG) = min(bottom_heightᶜᶜᵃ(i-1, j, k, ibg), bottom_heightᶜᶜᵃ(i, j, k, ibg)) +@inline bottom_heightᶠᶠᵃ(i, j, k, ibg::GFBIBG) = min(bottom_heightᶠᶜᵃ(i, j-1, k, ibg), bottom_heightᶠᶜᵃ(i, j, k, ibg)) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index d08f6db815..d3581b1780 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -7,16 +7,16 @@ using Printf ##### struct PartialCellBottom{H, E} <: AbstractGridFittedBottom{H} - bottom_height :: H + z_bottom :: H minimum_fractional_cell_height :: E end const PCBIBG{FT, TX, TY, TZ} = ImmersedBoundaryGrid{FT, TX, TY, TZ, <:Any, <:PartialCellBottom} where {FT, TX, TY, TZ} function Base.summary(ib::PartialCellBottom) - zmax = maximum(parent(ib.bottom_height)) - zmin = minimum(parent(ib.bottom_height)) - zmean = mean(parent(ib.bottom_height)) + zmax = maximum(parent(ib.z_bottom)) + zmin = minimum(parent(ib.z_bottom)) + zmean = mean(parent(ib.z_bottom)) summary1 = "PartialCellBottom(" @@ -31,21 +31,21 @@ function Base.summary(ib::PartialCellBottom) end Base.summary(ib::PartialCellBottom{<:Function}) = @sprintf("PartialCellBottom(%s, ϵ=%.1f)", - prettysummary(ib.bottom_height, false), + prettysummary(ib.z_bottom, false), prettysummary(ib.minimum_fractional_cell_height)) function Base.show(io::IO, ib::PartialCellBottom) print(io, summary(ib), '\n') - print(io, "├── bottom_height: ", prettysummary(ib.bottom_height), '\n') + print(io, "├── z_bottom: ", prettysummary(ib.z_bottom), '\n') print(io, "└── minimum_fractional_cell_height: ", prettysummary(ib.minimum_fractional_cell_height)) end """ - PartialCellBottom(bottom_height; minimum_fractional_cell_height=0.2) + PartialCellBottom(z_bottom; minimum_fractional_cell_height=0.2) Return `PartialCellBottom` representing an immersed boundary with "partial" bottom cells. That is, the height of the bottommost cell in each column is reduced -to fit the provided `bottom_height`, which may be a `Field`, `Array`, or function +to fit the provided `z_bottom`, which may be a `Field`, `Array`, or function of `(x, y)`. The height of partial bottom cells is greater than @@ -56,32 +56,44 @@ minimum_fractional_cell_height * Δz, where `Δz` is the original height of the bottom cell underlying grid. """ -function PartialCellBottom(bottom_height; minimum_fractional_cell_height=0.2) - return PartialCellBottom(bottom_height, minimum_fractional_cell_height) +function PartialCellBottom(z_bottom; minimum_fractional_cell_height=0.2) + return PartialCellBottom(z_bottom, minimum_fractional_cell_height) end function ImmersedBoundaryGrid(grid, ib::PartialCellBottom) bottom_field = Field{Center, Center, Nothing}(grid) - set!(bottom_field, ib.bottom_height) - @apply_regionally clamp_bottom_height!(bottom_field, grid) + set!(bottom_field, ib.z_bottom) + @apply_regionally correct_z_bottom!(bottom_field, grid, ib.minimum_fractional_cell_height) fill_halo_regions!(bottom_field) new_ib = PartialCellBottom(bottom_field, ib.minimum_fractional_cell_height) TX, TY, TZ = topology(grid) return ImmersedBoundaryGrid{TX, TY, TZ}(grid, new_ib) end +@kernel function _correct_z_bottom!(bottom_field, grid, ib::PartialCellBottom) + i, j = @index(grid) + zb = @inbounds bottom_field[i, j, 1] + ϵ = ib.minimum_fractional_cell_height + for k in 1:grid.Nz + z⁻ = znode(i, j, k, grid, c, c, f) + Δz = Δzᶜᶜᶜ(i, j, k, underlying_grid) + bottom_cell = zb < z⁻ + Δz * (1 - ϵ) + @inbounds bottom_field[i, j, 1] = ifelse(bottom_cell, z⁻ + Δz * (1 - ϵ), zb) + end +end + function on_architecture(arch, ib::PartialCellBottom{<:Field}) - architecture(ib.bottom_height) == arch && return ib - arch_grid = on_architecture(arch, ib.bottom_height.grid) - new_bottom_height = Field{Center, Center, Nothing}(arch_grid) - copyto!(parent(new_bottom_height), parent(ib.bottom_height)) - return PartialCellBottom(new_bottom_height, ib.minimum_fractional_cell_height) + architecture(ib.z_bottom) == arch && return ib + arch_grid = on_architecture(arch, ib.z_bottom.grid) + new_z_bottom = Field{Center, Center, Nothing}(arch_grid) + copyto!(parent(new_z_bottom), parent(ib.z_bottom)) + return PartialCellBottom(new_z_bottom, ib.minimum_fractional_cell_height) end -Adapt.adapt_structure(to, ib::PartialCellBottom) = PartialCellBottom(adapt(to, ib.bottom_height), +Adapt.adapt_structure(to, ib::PartialCellBottom) = PartialCellBottom(adapt(to, ib.z_bottom), ib.minimum_fractional_cell_height) -on_architecture(to, ib::PartialCellBottom) = PartialCellBottom(on_architecture(to, ib.bottom_height), +on_architecture(to, ib::PartialCellBottom) = PartialCellBottom(on_architecture(to, ib.z_bottom), on_architecture(to, ib.minimum_fractional_cell_height)) """ @@ -108,7 +120,7 @@ Criterion is zb ≥ z - ϵ Δz @inline function _immersed_cell(i, j, k, underlying_grid, ib::PartialCellBottom) # Face node below current cell z = znode(i, j, k, underlying_grid, c, c, f) - zb = @inbounds ib.bottom_height[i, j, 1] + zb = @inbounds ib.z_bottom[i, j, 1] ϵ = ib.minimum_fractional_cell_height # z + Δz is equal to the face above the current cell Δz = Δzᶜᶜᶜ(i, j, k, underlying_grid) @@ -130,7 +142,7 @@ end z = znode(i, j, k+1, underlying_grid, c, c, f) # Get bottom height and fractional Δz parameter - h = @inbounds ib.bottom_height[i, j, 1] + h = @inbounds ib.z_bottom[i, j, 1] ϵ = ibg.immersed_boundary.minimum_fractional_cell_height # Are we in a bottom cell? @@ -176,5 +188,5 @@ YFlatPCBIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Flat, <:Any, <:Any, <:Partial @inline Δzᶠᶠᶜ(i, j, k, ibg::XFlatPCBIBG) = Δzᶜᶠᶜ(i, j, k, ibg) @inline Δzᶠᶠᶜ(i, j, k, ibg::YFlatPCBIBG) = Δzᶠᶜᶜ(i, j, k, ibg) -@inline z_bottom(i, j, ibg::PCBIBG) = @inbounds ibg.immersed_boundary.bottom_height[i, j, 1] +@inline z_bottom(i, j, ibg::PCBIBG) = @inbounds ibg.immersed_boundary.z_bottom[i, j, 1] diff --git a/src/MultiRegion/multi_region_grid.jl b/src/MultiRegion/multi_region_grid.jl index 0c67d75a49..a65a8b5dd8 100644 --- a/src/MultiRegion/multi_region_grid.jl +++ b/src/MultiRegion/multi_region_grid.jl @@ -188,8 +188,8 @@ function reconstruct_global_grid(mrg::ImmersedMultiRegionGrid) return ImmersedBoundaryGrid(global_grid, global_boundary) end -reconstruct_global_boundary(g::GridFittedBottom{<:Field}) = GridFittedBottom(reconstruct_global_field(g.bottom_height), g.immersed_condition) -reconstruct_global_boundary(g::PartialCellBottom{<:Field}) = PartialCellBottom(reconstruct_global_field(g.bottom_height), g.minimum_fractional_cell_height) +reconstruct_global_boundary(g::GridFittedBottom{<:Field}) = GridFittedBottom(reconstruct_global_field(g.z_bottom), g.immersed_condition) +reconstruct_global_boundary(g::PartialCellBottom{<:Field}) = PartialCellBottom(reconstruct_global_field(g.z_bottom), g.minimum_fractional_cell_height) reconstruct_global_boundary(g::GridFittedBoundary{<:Field}) = GridFittedBoundary(reconstruct_global_field(g.mask)) @inline getregion(mrg::ImmersedMultiRegionGrid{FT, TX, TY, TZ}, r) where {FT, TX, TY, TZ} = ImmersedBoundaryGrid{TX, TY, TZ}(_getregion(mrg.underlying_grid, r), _getregion(mrg.immersed_boundary, r)) diff --git a/validation/immersed_boundaries/immersed_couette_flow.jl b/validation/immersed_boundaries/immersed_couette_flow.jl index 90098d6ee4..cd1671f6e7 100644 --- a/validation/immersed_boundaries/immersed_couette_flow.jl +++ b/validation/immersed_boundaries/immersed_couette_flow.jl @@ -10,8 +10,8 @@ Lz = 1 underlying_grid = RectilinearGrid(size=Nz, z=(0, Lz), topology = (Flat, Flat, Bounded)) -@inline bottom_height(x, y) = 0.1 -grid = ImmersedBoundaryGrid(underlying_grid, GridFittedBottom(bottom_height)) +@inline z_bottom(x, y) = 0.1 +grid = ImmersedBoundaryGrid(underlying_grid, GridFittedBottom(z_bottom)) c_immersed_bc = ValueBoundaryCondition(1) c_top_bc = ValueBoundaryCondition(-1) From b83a6ae5098520d28f19268082d9801c6705ce87 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 10:18:09 +0200 Subject: [PATCH 323/567] now it should work --- src/Grids/Grids.jl | 1 + src/ImmersedBoundaries/ImmersedBoundaries.jl | 5 +++- src/ImmersedBoundaries/grid_fitted_bottom.jl | 10 ++++---- src/ImmersedBoundaries/partial_cell_bottom.jl | 2 +- ...distributed_split_explicit_free_surface.jl | 16 +----------- .../split_explicit_free_surface.jl | 17 +------------ .../split_explicit_free_surface_kernels.jl | 25 +++++++++++-------- ...ulti_region_split_explicit_free_surface.jl | 16 +----------- 8 files changed, 29 insertions(+), 63 deletions(-) diff --git a/src/Grids/Grids.jl b/src/Grids/Grids.jl index fd8aebbf21..bbc40581ce 100644 --- a/src/Grids/Grids.jl +++ b/src/Grids/Grids.jl @@ -18,6 +18,7 @@ export xnodes, ynodes, znodes, λnodes, φnodes export spacings export xspacings, yspacings, zspacings, xspacing, yspacing, zspacing export minimum_xspacing, minimum_yspacing, minimum_zspacing +export bottom_heightᶜᶜᵃ, bottom_heightᶠᶜᵃ, bottom_heightᶜᶠᵃ, bottom_heightᶠᶠᵃ export offset_data, new_data export on_architecture diff --git a/src/ImmersedBoundaries/ImmersedBoundaries.jl b/src/ImmersedBoundaries/ImmersedBoundaries.jl index e9281eac32..8cf5e90b08 100644 --- a/src/ImmersedBoundaries/ImmersedBoundaries.jl +++ b/src/ImmersedBoundaries/ImmersedBoundaries.jl @@ -47,16 +47,19 @@ import Oceananigans.Advection: cell_advection_timescale import Oceananigans.Grids: cpu_face_constructor_x, cpu_face_constructor_y, cpu_face_constructor_z, x_domain, y_domain, z_domain -import Oceananigans.Grids: architecture, on_architecture, with_halo, inflate_halo_size_one_dimension, +import Oceananigans.Grids: architecture, with_halo, inflate_halo_size_one_dimension, xnode, ynode, znode, λnode, φnode, node, ξnode, ηnode, rnode, ξname, ηname, rname, node_names, xnodes, ynodes, znodes, λnodes, φnodes, nodes, ξnodes, ηnodes, rnodes, + bottom_heightᶜᶜᵃ, bottom_heightᶠᶜᵃ, bottom_heightᶜᶠᵃ, bottom_heightᶠᶠᵃ, inactive_cell import Oceananigans.Coriolis: φᶠᶠᵃ +import Oceananigans.Architectures: on_architecture + import Oceananigans.Advection: _advective_momentum_flux_Uu, _advective_momentum_flux_Uv, diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index b21609dd2b..1851e39d95 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -91,7 +91,7 @@ correct_z_bottom!(bottom_field, grid, ib) = launch!(architecture(grid), grid, :xy, _correct_z_bottom!, bottom_field, grid, ib) @kernel function _correct_z_bottom!(bottom_field, grid, ::GridFittedBottom) - i, j = @index(grid) + i, j = @index(Global, NTuple) zb = @inbounds bottom_field[i, j, 1] for k in 1:grid.Nz z⁺ = znode(i, j, k+1, grid, c, c, f) @@ -112,7 +112,7 @@ end ##### Bottom height ##### -@inline bottom_heightᶜᶜᵃ(i, j, k, ibg::GFBIBG) = ibg.immersed_boundary.bottom_height[i, j, 1] -@inline bottom_heightᶜᶠᵃ(i, j, k, ibg::GFBIBG) = min(bottom_heightᶜᶜᵃ(i, j-1, k, ibg), bottom_heightᶜᶜᵃ(i, j, k, ibg)) -@inline bottom_heightᶠᶜᵃ(i, j, k, ibg::GFBIBG) = min(bottom_heightᶜᶜᵃ(i-1, j, k, ibg), bottom_heightᶜᶜᵃ(i, j, k, ibg)) -@inline bottom_heightᶠᶠᵃ(i, j, k, ibg::GFBIBG) = min(bottom_heightᶠᶜᵃ(i, j-1, k, ibg), bottom_heightᶠᶜᵃ(i, j, k, ibg)) +@inline bottom_heightᶜᶜᵃ(i, j, k, ibg::AbstractGridFittedBottom) = ibg.immersed_boundary.bottom_height[i, j, 1] +@inline bottom_heightᶜᶠᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(bottom_heightᶜᶜᵃ(i, j-1, k, ibg), bottom_heightᶜᶜᵃ(i, j, k, ibg)) +@inline bottom_heightᶠᶜᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(bottom_heightᶜᶜᵃ(i-1, j, k, ibg), bottom_heightᶜᶜᵃ(i, j, k, ibg)) +@inline bottom_heightᶠᶠᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(bottom_heightᶠᶜᵃ(i, j-1, k, ibg), bottom_heightᶠᶜᵃ(i, j, k, ibg)) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index d3581b1780..00088a1c74 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -71,7 +71,7 @@ function ImmersedBoundaryGrid(grid, ib::PartialCellBottom) end @kernel function _correct_z_bottom!(bottom_field, grid, ib::PartialCellBottom) - i, j = @index(grid) + i, j = @index(Global, NTuple) zb = @inbounds bottom_field[i, j, 1] ϵ = ib.minimum_fractional_cell_height for k in 1:grid.Nz diff --git a/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl index 6f0afa5a26..d8b472fec8 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/distributed_split_explicit_free_surface.jl @@ -10,27 +10,13 @@ function SplitExplicitAuxiliaryFields(grid::DistributedGrid) Gᵁ = Field((Face, Center, Nothing), grid) Gⱽ = Field((Center, Face, Nothing), grid) - Hᶠᶜ = Field((Face, Center, Nothing), grid) - Hᶜᶠ = Field((Center, Face, Nothing), grid) - - calculate_column_height!(Hᶠᶜ, (Face, Center, Center)) - calculate_column_height!(Hᶜᶠ, (Center, Face, Center)) - - fill_halo_regions!((Hᶠᶜ, Hᶜᶠ)) - # In a non-parallel grid we calculate only the interior kernel_size = augmented_kernel_size(grid) kernel_offsets = augmented_kernel_offsets(grid) kernel_parameters = KernelParameters(kernel_size, kernel_offsets) - return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, kernel_parameters) -end - -"""Integrate z at locations `location` and set! `height`` with the result""" -@inline function calculate_column_height!(height, location) - dz = GridMetricOperation(location, Δz, height.grid) - return sum!(height, dz) + return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, kernel_parameters) end @inline function augmented_kernel_size(grid::DistributedGrid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index 5d9a00f2ed..073fbfc4ea 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -209,10 +209,6 @@ Base.@kwdef struct SplitExplicitAuxiliaryFields{𝒞ℱ, ℱ𝒞, 𝒦} Gᵁ :: ℱ𝒞 "Vertically-integrated slow barotropic forcing function for `V` (`ReducedField` over ``z``)" Gⱽ :: 𝒞ℱ - "Depth at `(Face, Center)` (`ReducedField` over ``z``)" - Hᶠᶜ :: ℱ𝒞 - "Depth at `(Center, Face)` (`ReducedField` over ``z``)" - Hᶜᶠ :: 𝒞ℱ "kernel size for barotropic time stepping" kernel_parameters :: 𝒦 end @@ -227,20 +223,9 @@ function SplitExplicitAuxiliaryFields(grid::AbstractGrid) Gᵁ = Field((Face, Center, Nothing), grid) Gⱽ = Field((Center, Face, Nothing), grid) - Hᶠᶜ = Field((Face, Center, Nothing), grid) - Hᶜᶠ = Field((Center, Face, Nothing), grid) - - dz = GridMetricOperation((Face, Center, Center), Δz, grid) - sum!(Hᶠᶜ, dz) - - dz = GridMetricOperation((Center, Face, Center), Δz, grid) - sum!(Hᶜᶠ, dz) - - fill_halo_regions!((Hᶠᶜ, Hᶜᶠ)) - kernel_parameters = :xy - return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, kernel_parameters) + return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, kernel_parameters) end """ diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index baca09bbfc..9f793d9168 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -1,3 +1,4 @@ +using Oceananigans.Grids using Oceananigans.Grids: topology using Oceananigans.Utils using Oceananigans.AbstractOperations: Δz @@ -159,20 +160,20 @@ end @kernel function _split_explicit_barotropic_velocity!(averaging_weight, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, g, + η̅, U̅, V̅, Gᵁ, Gⱽ, g, timestepper) i, j = @index(Global, NTuple) velocity_evolution!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅, averaging_weight, - Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, g, + Gᵁ, Gⱽ, g, timestepper) end @inline function velocity_evolution!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅, averaging_weight, - Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, g, + Gᵁ, Gⱽ, g, timestepper) k_top = grid.Nz+1 @@ -182,9 +183,12 @@ end advance_previous_velocity!(i, j, k_top-1, timestepper, U, Uᵐ⁻¹, Uᵐ⁻²) advance_previous_velocity!(i, j, k_top-1, timestepper, V, Vᵐ⁻¹, Vᵐ⁻²) + Hᶠᶜ = bottom_heightᶠᶜᵃ(i, j, k_top-1, grid) + Hᶜᶠ = bottom_heightᶜᶠᵃ(i, j, k_top-1, grid) + # ∂τ(U) = - ∇η + G - U[i, j, k_top-1] += Δτ * (- g * Hᶠᶜ[i, j, 1] * ∂xᶠᶜᶠ_η(i, j, k_top, grid, TX, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gᵁ[i, j, k_top-1]) - V[i, j, k_top-1] += Δτ * (- g * Hᶜᶠ[i, j, 1] * ∂yᶜᶠᶠ_η(i, j, k_top, grid, TY, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gⱽ[i, j, k_top-1]) + U[i, j, k_top-1] += Δτ * (- g * Hᶠᶜ * ∂xᶠᶜᶠ_η(i, j, k_top, grid, TX, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gᵁ[i, j, k_top-1]) + V[i, j, k_top-1] += Δτ * (- g * Hᶜᶠ * ∂yᶜᶠᶠ_η(i, j, k_top, grid, TY, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gⱽ[i, j, k_top-1]) # time-averaging η̅[i, j, k_top] += averaging_weight * η[i, j, k_top] @@ -267,24 +271,25 @@ end k_top = grid.Nz+1 @inbounds begin - u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / Hᶠᶜ[i, j, 1] - v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / Hᶜᶠ[i, j, 1] + Hᶠᶜ = bottom_heightᶠᶜᵃ(i, j, k_top-1, grid) + Hᶜᶠ = bottom_heightᶜᶠᵃ(i, j, k_top-1, grid) + + u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / Hᶠᶜ + v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / Hᶜᶠ end end function barotropic_split_explicit_corrector!(u, v, free_surface, grid) sefs = free_surface.state U, V, U̅, V̅ = sefs.U, sefs.V, sefs.U̅, sefs.V̅ - Hᶠᶜ, Hᶜᶠ = free_surface.auxiliary.Hᶠᶜ, free_surface.auxiliary.Hᶜᶠ arch = architecture(grid) - # take out "bad" barotropic mode, # !!!! reusing U and V for this storage since last timestep doesn't matter compute_barotropic_mode!(U, V, grid, u, v) # add in "good" barotropic mode launch!(arch, grid, :xyz, _barotropic_split_explicit_corrector!, - u, v, U̅, V̅, U, V, Hᶠᶜ, Hᶜᶠ, grid) + u, v, U̅, V̅, U, V, grid) return nothing end diff --git a/src/MultiRegion/multi_region_split_explicit_free_surface.jl b/src/MultiRegion/multi_region_split_explicit_free_surface.jl index fb36463b39..10ba130b33 100644 --- a/src/MultiRegion/multi_region_split_explicit_free_surface.jl +++ b/src/MultiRegion/multi_region_split_explicit_free_surface.jl @@ -13,27 +13,13 @@ function SplitExplicitAuxiliaryFields(grid::MultiRegionGrids) Gᵁ = Field((Face, Center, Nothing), grid) Gⱽ = Field((Center, Face, Nothing), grid) - Hᶠᶜ = Field((Face, Center, Nothing), grid) - Hᶜᶠ = Field((Center, Face, Nothing), grid) - - @apply_regionally calculate_column_height!(Hᶠᶜ, (Face, Center, Center)) - @apply_regionally calculate_column_height!(Hᶜᶠ, (Center, Face, Center)) - - fill_halo_regions!((Hᶠᶜ, Hᶜᶠ)) - # In a non-parallel grid we calculate only the interior @apply_regionally kernel_size = augmented_kernel_size(grid, grid.partition) @apply_regionally kernel_offsets = augmented_kernel_offsets(grid, grid.partition) @apply_regionally kernel_parameters = KernelParameters(kernel_size, kernel_offsets) - return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, kernel_parameters) -end - -@inline function calculate_column_height!(height, location) - dz = GridMetricOperation(location, Δz, height.grid) - sum!(height, dz) - return nothing + return SplitExplicitAuxiliaryFields(Gᵁ, Gⱽ, kernel_parameters) end @inline augmented_kernel_size(grid, ::XPartition) = (size(grid, 1) + 2halo_size(grid)[1]-2, size(grid, 2)) From 5ebaa84481e6dd8e144b651a3dfa42dd08b34890 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 10:19:54 +0200 Subject: [PATCH 324/567] comment --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 1851e39d95..1824d3b56f 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -30,8 +30,7 @@ Keyword Arguments ================= * `z_bottom`: an array or function that gives the height of the - bottom in absolute ``z`` coordinates. This is the height of - the bottom interface of the last ``fluid`` cell. + bottom in absolute ``z`` coordinates. """ function Base.summary(ib::GridFittedBottom) From f9ad2279e5f720c3a27e6a051321d4128bf87445 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 10:24:55 +0200 Subject: [PATCH 325/567] comment --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 1824d3b56f..8dc208e790 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -74,7 +74,8 @@ Adapt.adapt_structure(to, ib::GridFittedBottom) = GridFittedBottom(adapt(to, ib. Return a grid with `GridFittedBottom` immersed boundary (`ib`). -Computes `ib.z_bottom` and wraps it in a Field. +Computes `ib.z_bottom` and wraps it in a Field. `ib.z_bottom` is the z-coordinate of top-most interface +of the last ``immersed`` cell in the column. """ function ImmersedBoundaryGrid(grid, ib::GridFittedBottom) bottom_field = Field{Center, Center, Nothing}(grid) From cbdb713a649b59313b903f643c78dc19e76a483e Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 10:54:06 +0200 Subject: [PATCH 326/567] remove circular dependency for now --- Project.toml | 3 +-- test/test_cubed_spheres.jl | 6 +----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/Project.toml b/Project.toml index 13f1fb6317..06a8553f22 100644 --- a/Project.toml +++ b/Project.toml @@ -79,10 +79,9 @@ julia = "1.9" [extras] DataDeps = "124859b0-ceae-595e-8997-d05f6a7a8dfe" Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" -OrthogonalSphericalShellGrids = "c2be9673-fb75-4747-82dc-aa2bb9f4aed0" SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" TimesDates = "bdfc003b-8df8-5c39-adcd-3a9087f5df4a" [targets] -test = ["DataDeps", "Enzyme", "OrthogonalSphericalShellGrids", "SafeTestsets", "Test", "TimesDates"] +test = ["DataDeps", "Enzyme", "SafeTestsets", "Test", "TimesDates"] diff --git a/test/test_cubed_spheres.jl b/test/test_cubed_spheres.jl index d8984cb0a1..e907dedf68 100644 --- a/test/test_cubed_spheres.jl +++ b/test/test_cubed_spheres.jl @@ -7,8 +7,6 @@ using Oceananigans.CubedSpheres using Oceananigans.Models.HydrostaticFreeSurfaceModels using Oceananigans.Models.HydrostaticFreeSurfaceModels: VerticalVorticityField -using OrthogonalSphericalShellGrids - # To be used in the test below as `KernelFunctionOperation`s @inline intrinsic_vector_x_component(i, j, k, grid, uₑ, vₑ) = @inbounds intrinsic_vector(i, j, k, grid, uₑ, vₑ)[1] @@ -172,12 +170,10 @@ end @testset "Conversion from Intrinsic to Extrinsic reference frame [$(typeof(arch))]" begin @info " Testing the conversion of a vector between the Intrinsic and Extrinsic reference frame" - trg_grid = TripolarGrid(arch, size = (20, 20, 1), z = (0, 1)) - test_vector_rotation(grid) - test_vector_rotation(trg_grid) end + @testset "CubedSphereData and CubedSphereFields [$(typeof(arch))]" begin @info " Testing CubedSphereData and CubedSphereFields [$(typeof(arch))]..." c = model.tracers.c From 7f087680d561eeccb5c959820bb55e951490d820 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 11:20:56 +0200 Subject: [PATCH 327/567] some bugfixes --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 2 +- src/ImmersedBoundaries/partial_cell_bottom.jl | 2 +- .../split_explicit_free_surface_kernels.jl | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 8dc208e790..fdae0c6a02 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -80,7 +80,7 @@ of the last ``immersed`` cell in the column. function ImmersedBoundaryGrid(grid, ib::GridFittedBottom) bottom_field = Field{Center, Center, Nothing}(grid) set!(bottom_field, ib.z_bottom) - @apply_regionally correct_z_bottom!(bottom_field, grid) + @apply_regionally correct_z_bottom!(bottom_field, grid, ib) fill_halo_regions!(bottom_field) new_ib = GridFittedBottom(bottom_field) TX, TY, TZ = topology(grid) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index 00088a1c74..5117adddf7 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -63,7 +63,7 @@ end function ImmersedBoundaryGrid(grid, ib::PartialCellBottom) bottom_field = Field{Center, Center, Nothing}(grid) set!(bottom_field, ib.z_bottom) - @apply_regionally correct_z_bottom!(bottom_field, grid, ib.minimum_fractional_cell_height) + @apply_regionally correct_z_bottom!(bottom_field, grid, ib) fill_halo_regions!(bottom_field) new_ib = PartialCellBottom(bottom_field, ib.minimum_fractional_cell_height) TX, TY, TZ = topology(grid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 9f793d9168..6d31c95973 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -381,7 +381,7 @@ function iterate_split_explicit!(free_surface, grid, Δτᴮ, weights, ::Val{Nsu Vᵐ⁻¹, Vᵐ⁻² = state.Vᵐ⁻¹, state.Vᵐ⁻² ηᵐ, ηᵐ⁻¹, ηᵐ⁻² = state.ηᵐ, state.ηᵐ⁻¹, state.ηᵐ⁻² η̅, U̅, V̅ = state.η̅, state.U̅, state.V̅ - Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ = auxiliary.Gᵁ, auxiliary.Gⱽ, auxiliary.Hᶠᶜ, auxiliary.Hᶜᶠ + Gᵁ, Gⱽ = auxiliary.Gᵁ, auxiliary.Gⱽ timestepper = settings.timestepper @@ -396,7 +396,7 @@ function iterate_split_explicit!(free_surface, grid, Δτᴮ, weights, ::Val{Nsu U_args = (grid, Δτᴮ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, Gᵁ, Gⱽ, Hᶠᶜ, Hᶜᶠ, g, + η̅, U̅, V̅, Gᵁ, Gⱽ, g, timestepper) GC.@preserve η_args U_args begin From 4b7926fd74822f87630ed80eda921b59c102a94f Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 11:26:30 +0200 Subject: [PATCH 328/567] change name to column_height --- src/Grids/Grids.jl | 2 +- src/Grids/grid_utils.jl | 8 +++---- src/ImmersedBoundaries/ImmersedBoundaries.jl | 2 +- src/ImmersedBoundaries/grid_fitted_bottom.jl | 22 +++++++++---------- .../split_explicit_free_surface_kernels.jl | 8 +++---- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/Grids/Grids.jl b/src/Grids/Grids.jl index bbc40581ce..fcc6270001 100644 --- a/src/Grids/Grids.jl +++ b/src/Grids/Grids.jl @@ -18,7 +18,7 @@ export xnodes, ynodes, znodes, λnodes, φnodes export spacings export xspacings, yspacings, zspacings, xspacing, yspacing, zspacing export minimum_xspacing, minimum_yspacing, minimum_zspacing -export bottom_heightᶜᶜᵃ, bottom_heightᶠᶜᵃ, bottom_heightᶜᶠᵃ, bottom_heightᶠᶠᵃ +export column_heightᶜᶜᵃ, column_heightᶠᶜᵃ, column_heightᶜᶠᵃ, column_heightᶠᶠᵃ export offset_data, new_data export on_architecture diff --git a/src/Grids/grid_utils.jl b/src/Grids/grid_utils.jl index afd199abe0..33d60205d1 100644 --- a/src/Grids/grid_utils.jl +++ b/src/Grids/grid_utils.jl @@ -318,10 +318,10 @@ coordinate_summary(topo, Δ::Union{AbstractVector, AbstractMatrix}, name) = ##### Bottom height ##### -@inline bottom_heightᶜᶜᵃ(i, j, k, grid) = grid.Lz -@inline bottom_heightᶜᶠᵃ(i, j, k, grid) = grid.Lz -@inline bottom_heightᶠᶜᵃ(i, j, k, grid) = grid.Lz -@inline bottom_heightᶠᶠᵃ(i, j, k, grid) = grid.Lz +@inline column_heightᶜᶜᵃ(i, j, k, grid) = grid.Lz +@inline column_heightᶜᶠᵃ(i, j, k, grid) = grid.Lz +@inline column_heightᶠᶜᵃ(i, j, k, grid) = grid.Lz +@inline column_heightᶠᶠᵃ(i, j, k, grid) = grid.Lz ##### ##### Spherical geometry diff --git a/src/ImmersedBoundaries/ImmersedBoundaries.jl b/src/ImmersedBoundaries/ImmersedBoundaries.jl index 8cf5e90b08..4a011d1beb 100644 --- a/src/ImmersedBoundaries/ImmersedBoundaries.jl +++ b/src/ImmersedBoundaries/ImmersedBoundaries.jl @@ -53,7 +53,7 @@ import Oceananigans.Grids: architecture, with_halo, inflate_halo_size_one_dimens ξname, ηname, rname, node_names, xnodes, ynodes, znodes, λnodes, φnodes, nodes, ξnodes, ηnodes, rnodes, - bottom_heightᶜᶜᵃ, bottom_heightᶠᶜᵃ, bottom_heightᶜᶠᵃ, bottom_heightᶠᶠᵃ, + column_heightᶜᶜᵃ, column_heightᶠᶜᵃ, column_heightᶜᶠᵃ, column_heightᶠᶠᵃ, inactive_cell import Oceananigans.Coriolis: φᶠᶠᵃ diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index fdae0c6a02..ca9722a073 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -56,15 +56,15 @@ function Base.show(io::IO, ib::GridFittedBottom) print(io, "├── z_bottom: ", prettysummary(ib.z_bottom), '\n') end -on_architecture(arch, ib::GridFittedBottom) = GridFittedBottom(on_architecture(ib.bottom_height)) +on_architecture(arch, ib::GridFittedBottom) = GridFittedBottom(on_architecture(ib.z_bottom)) function on_architecture(arch, ib::GridFittedBottom{<:Field}) - architecture(ib.bottom_height) == arch && return ib - arch_grid = on_architecture(arch, ib.bottom_height.grid) - new_bottom_height = Field{Center, Center, Nothing}(arch_grid) - set!(new_bottom_height, ib.bottom_height) - fill_halo_regions!(new_bottom_height) - return GridFittedBottom(new_bottom_height, ib.immersed_condition) + architecture(ib.z_bottom) == arch && return ib + arch_grid = on_architecture(arch, ib.z_bottom.grid) + new_z_bottom = Field{Center, Center, Nothing}(arch_grid) + set!(new_z_bottom, ib.z_bottom) + fill_halo_regions!(new_z_bottom) + return GridFittedBottom(new_z_bottom) end Adapt.adapt_structure(to, ib::GridFittedBottom) = GridFittedBottom(adapt(to, ib.z_bottom)) @@ -112,7 +112,7 @@ end ##### Bottom height ##### -@inline bottom_heightᶜᶜᵃ(i, j, k, ibg::AbstractGridFittedBottom) = ibg.immersed_boundary.bottom_height[i, j, 1] -@inline bottom_heightᶜᶠᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(bottom_heightᶜᶜᵃ(i, j-1, k, ibg), bottom_heightᶜᶜᵃ(i, j, k, ibg)) -@inline bottom_heightᶠᶜᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(bottom_heightᶜᶜᵃ(i-1, j, k, ibg), bottom_heightᶜᶜᵃ(i, j, k, ibg)) -@inline bottom_heightᶠᶠᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(bottom_heightᶠᶜᵃ(i, j-1, k, ibg), bottom_heightᶠᶜᵃ(i, j, k, ibg)) +@inline column_heightᶜᶜᵃ(i, j, k, ibg::AbstractGridFittedBottom) = ibg.immersed_boundary.column_height[i, j, 1] +@inline column_heightᶜᶠᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(column_heightᶜᶜᵃ(i, j-1, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) +@inline column_heightᶠᶜᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(column_heightᶜᶜᵃ(i-1, j, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) +@inline column_heightᶠᶠᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(column_heightᶠᶜᵃ(i, j-1, k, ibg), column_heightᶠᶜᵃ(i, j, k, ibg)) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 6d31c95973..58c0231f00 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -183,8 +183,8 @@ end advance_previous_velocity!(i, j, k_top-1, timestepper, U, Uᵐ⁻¹, Uᵐ⁻²) advance_previous_velocity!(i, j, k_top-1, timestepper, V, Vᵐ⁻¹, Vᵐ⁻²) - Hᶠᶜ = bottom_heightᶠᶜᵃ(i, j, k_top-1, grid) - Hᶜᶠ = bottom_heightᶜᶠᵃ(i, j, k_top-1, grid) + Hᶠᶜ = column_heightᶠᶜᵃ(i, j, k_top-1, grid) + Hᶜᶠ = column_heightᶜᶠᵃ(i, j, k_top-1, grid) # ∂τ(U) = - ∇η + G U[i, j, k_top-1] += Δτ * (- g * Hᶠᶜ * ∂xᶠᶜᶠ_η(i, j, k_top, grid, TX, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gᵁ[i, j, k_top-1]) @@ -271,8 +271,8 @@ end k_top = grid.Nz+1 @inbounds begin - Hᶠᶜ = bottom_heightᶠᶜᵃ(i, j, k_top-1, grid) - Hᶜᶠ = bottom_heightᶜᶠᵃ(i, j, k_top-1, grid) + Hᶠᶜ = column_heightᶠᶜᵃ(i, j, k_top-1, grid) + Hᶜᶠ = column_heightᶜᶠᵃ(i, j, k_top-1, grid) u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / Hᶠᶜ v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / Hᶜᶠ From 6b7b27ca46ff6c0222ec0a105db401bdafb9a5d6 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 11:30:22 +0200 Subject: [PATCH 329/567] correct column height --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index ca9722a073..8cbe5e998d 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -112,7 +112,8 @@ end ##### Bottom height ##### -@inline column_heightᶜᶜᵃ(i, j, k, ibg::AbstractGridFittedBottom) = ibg.immersed_boundary.column_height[i, j, 1] +@inline column_heightᶜᶜᵃ(i, j, k, ibg::AbstractGridFittedBottom) = znode(i, j, grid.Nz, c, c, f) - ibg.immersed_boundary.column_height[i, j, 1] @inline column_heightᶜᶠᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(column_heightᶜᶜᵃ(i, j-1, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) @inline column_heightᶠᶜᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(column_heightᶜᶜᵃ(i-1, j, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) @inline column_heightᶠᶠᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(column_heightᶠᶜᵃ(i, j-1, k, ibg), column_heightᶠᶜᵃ(i, j, k, ibg)) + From fdee36641a1b786a974627babe06fa3503f54432 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 11:31:11 +0200 Subject: [PATCH 330/567] whoops --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 8cbe5e998d..9f743195e9 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -112,7 +112,7 @@ end ##### Bottom height ##### -@inline column_heightᶜᶜᵃ(i, j, k, ibg::AbstractGridFittedBottom) = znode(i, j, grid.Nz, c, c, f) - ibg.immersed_boundary.column_height[i, j, 1] +@inline column_heightᶜᶜᵃ(i, j, k, ibg::AbstractGridFittedBottom) = znode(i, j, grid.Nz, c, c, f) - ibg.immersed_boundary.z_bottom[i, j, 1] @inline column_heightᶜᶠᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(column_heightᶜᶜᵃ(i, j-1, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) @inline column_heightᶠᶜᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(column_heightᶜᶜᵃ(i-1, j, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) @inline column_heightᶠᶠᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(column_heightᶠᶜᵃ(i, j-1, k, ibg), column_heightᶠᶜᵃ(i, j, k, ibg)) From d1c145c60af4a6c754a822c632e417e636de2c2d Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 11:31:32 +0200 Subject: [PATCH 331/567] another correction --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 9f743195e9..e46cb0fe43 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -112,7 +112,7 @@ end ##### Bottom height ##### -@inline column_heightᶜᶜᵃ(i, j, k, ibg::AbstractGridFittedBottom) = znode(i, j, grid.Nz, c, c, f) - ibg.immersed_boundary.z_bottom[i, j, 1] +@inline column_heightᶜᶜᵃ(i, j, k, ibg::AbstractGridFittedBottom) = znode(i, j, grid.Nz, ibg, c, c, f) - ibg.immersed_boundary.z_bottom[i, j, 1] @inline column_heightᶜᶠᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(column_heightᶜᶜᵃ(i, j-1, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) @inline column_heightᶠᶜᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(column_heightᶜᶜᵃ(i-1, j, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) @inline column_heightᶠᶠᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(column_heightᶠᶜᵃ(i, j-1, k, ibg), column_heightᶠᶜᵃ(i, j, k, ibg)) From dbb411ad78a19c4e08bea88f162c8213262a4bf8 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 11:34:25 +0200 Subject: [PATCH 332/567] some more changes --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 12 +++++++----- src/ImmersedBoundaries/partial_cell_bottom.jl | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index e46cb0fe43..e518714eaa 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -95,7 +95,7 @@ correct_z_bottom!(bottom_field, grid, ib) = zb = @inbounds bottom_field[i, j, 1] for k in 1:grid.Nz z⁺ = znode(i, j, k+1, grid, c, c, f) - bottom_cell = zb < z⁺ + bottom_cell = zb ≤ z⁺ @inbounds bottom_field[i, j, 1] = ifelse(bottom_cell, z⁺, zb) end end @@ -112,8 +112,10 @@ end ##### Bottom height ##### -@inline column_heightᶜᶜᵃ(i, j, k, ibg::AbstractGridFittedBottom) = znode(i, j, grid.Nz, ibg, c, c, f) - ibg.immersed_boundary.z_bottom[i, j, 1] -@inline column_heightᶜᶠᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(column_heightᶜᶜᵃ(i, j-1, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) -@inline column_heightᶠᶜᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(column_heightᶜᶜᵃ(i-1, j, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) -@inline column_heightᶠᶠᵃ(i, j, k, ibg::AbstractGridFittedBottom) = min(column_heightᶠᶜᵃ(i, j-1, k, ibg), column_heightᶠᶜᵃ(i, j, k, ibg)) +const AGFBIB = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} + +@inline column_heightᶜᶜᵃ(i, j, k, ibg::AGFBIB) = @inbounds znode(i, j, grid.Nz, ibg, c, c, f) - ibg.immersed_boundary.z_bottom[i, j, 1] +@inline column_heightᶜᶠᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶜᶜᵃ(i, j-1, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) +@inline column_heightᶠᶜᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶜᶜᵃ(i-1, j, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) +@inline column_heightᶠᶠᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶠᶜᵃ(i, j-1, k, ibg), column_heightᶠᶜᵃ(i, j, k, ibg)) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index 5117adddf7..d23fe199a7 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -77,7 +77,7 @@ end for k in 1:grid.Nz z⁻ = znode(i, j, k, grid, c, c, f) Δz = Δzᶜᶜᶜ(i, j, k, underlying_grid) - bottom_cell = zb < z⁻ + Δz * (1 - ϵ) + bottom_cell = zb ≤ z⁻ + Δz * (1 - ϵ) @inbounds bottom_field[i, j, 1] = ifelse(bottom_cell, z⁻ + Δz * (1 - ϵ), zb) end end From 3ec05dff7ac595441aa7055d24684764db5eb46e Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 11:44:44 +0200 Subject: [PATCH 333/567] another correction --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index e518714eaa..83c23b2df4 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -114,7 +114,7 @@ end const AGFBIB = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} -@inline column_heightᶜᶜᵃ(i, j, k, ibg::AGFBIB) = @inbounds znode(i, j, grid.Nz, ibg, c, c, f) - ibg.immersed_boundary.z_bottom[i, j, 1] +@inline column_heightᶜᶜᵃ(i, j, k, ibg::AGFBIB) = @inbounds znode(i, j, grid.Nz+1, ibg, c, c, f) - ibg.immersed_boundary.z_bottom[i, j, 1] @inline column_heightᶜᶠᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶜᶜᵃ(i, j-1, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) @inline column_heightᶠᶜᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶜᶜᵃ(i-1, j, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) @inline column_heightᶠᶠᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶠᶜᵃ(i, j-1, k, ibg), column_heightᶠᶜᵃ(i, j, k, ibg)) From cfd877c9e40550c061eacec38edc722e6e464f6e Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 12:09:00 +0200 Subject: [PATCH 334/567] start with some changes --- src/Grids/z_star_vertical_coordinate.jl | 106 ++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 src/Grids/z_star_vertical_coordinate.jl diff --git a/src/Grids/z_star_vertical_coordinate.jl b/src/Grids/z_star_vertical_coordinate.jl new file mode 100644 index 0000000000..fed39ecafd --- /dev/null +++ b/src/Grids/z_star_vertical_coordinate.jl @@ -0,0 +1,106 @@ + +##### +##### ZStar coordinate and associated grid types +##### + +""" + struct ZStarSpacing{R, S} <: AbstractVerticalSpacing{R} + +A vertical spacing for the hydrostatic free surface model that follows the free surface. +The vertical spacing is defined by a reference spacing `Δr` and a scaling `s` that obeys +```math +s = (η + H) / H +``` +where ``η`` is the free surface height and ``H`` the vertical depth of the water column + +# Fields +- `Δr`: reference vertical spacing with `η = 0` +- `sᶜᶜⁿ`: scaling of the vertical coordinate at time step `n` at `(Center, Center, Any)` location +- `sᶠᶜⁿ`: scaling of the vertical coordinate at time step `n` at `(Face, Center, Any)` location +- `sᶜᶠⁿ`: scaling of the vertical coordinate at time step `n` at `(Center, Face, Any)` location +- `sᶠᶠⁿ`: scaling of the vertical coordinate at time step `n` at `(Face, Face, Any)` location +- `s⁻`: scaling of the vertical coordinate at time step `n - 1` at `(Center, Center, Any)` location +- `∂t_s`: Time derivative of `s` +""" +struct ZStarVerticalCoordinate{R, SCC, SFC, SCF, SFF} <: AbstractVerticalCoordinate{R} + reference :: R + sᶜᶜⁿ :: SCC + sᶠᶜⁿ :: SFC + sᶜᶠⁿ :: SCF + sᶠᶠⁿ :: SFF + sᶜᶜ⁻ :: SCC + sᶠᶜ⁻ :: SFC + sᶜᶠ⁻ :: SCF + ∂t_s :: SCC +end + +ZStarVerticalCoordinate(r_faces) = ZStarVerticalCoordinate(r_faces, nothing, nothing, nothing, nothing, nothing, nothing, nothing, nothing) + +Adapt.adapt_structure(to, coord::ZStarVerticalCoordinate) = + ZStarSpacing(Adapt.adapt(to, coord.reference), + Adapt.adapt(to, coord.sᶜᶜⁿ), + Adapt.adapt(to, coord.sᶠᶜⁿ), + Adapt.adapt(to, coord.sᶜᶠⁿ), + Adapt.adapt(to, coord.sᶠᶠⁿ), + Adapt.adapt(to, coord.sᶜᶜ⁻), + Adapt.adapt(to, coord.sᶠᶜ⁻), + Adapt.adapt(to, coord.sᶜᶠ⁻), + Adapt.adapt(to, coord.∂t_s)) + +on_architecture(arch, coord::ZStarVerticalCoordinate) = + ZStarSpacing(on_architecture(arch, coord.reference), + on_architecture(arch, coord.sᶜᶜⁿ), + on_architecture(arch, coord.sᶠᶜⁿ), + on_architecture(arch, coord.sᶜᶠⁿ), + on_architecture(arch, coord.sᶠᶠⁿ), + on_architecture(arch, coord.sᶜᶜ⁻), + on_architecture(arch, coord.sᶠᶜ⁻), + on_architecture(arch, coord.sᶜᶠ⁻), + on_architecture(arch, coord.∂t_s)) + +Grids.coordinate_summary(::Bounded, Δ::ZStarVerticalCoordinate, name) = + @sprintf("Free-surface following with Δ%s=%s", name, prettysummary(Δ.reference)) + + +generate_coordinate(FT, topo::Periodic, N, H, coordinate::ZStarVerticalCoordinate, coordinate_name, arch, args...) = + throw(ArgumentError("Periodic domains are not supported for ZStarVerticalCoordinate")) + +# Generate a regularly-spaced coordinate passing the domain extent (2-tuple) and number of points +function generate_coordinate(FT, topo::Bounded, Nz, Hz, coordinate::ZStarVerticalCoordinate, coordinate_name, arch, Nx, Ny, Hx, Hy) + + if coordinate_name != :z + msg = "Only z-coordinate is supported for ZStarVerticalCoordinate" + throw(ArgumentError(msg)) + end + + r_faces = coordinate.reference + + Lr, rᵃᵃᶠ, rᵃᵃᶜ, Δrᵃᵃᶠ, Δrᵃᵃᶜ = generate_coordinate(FT, topo[3](), Nz, Hz, r_faces, :z, arch) + + args = (topo, (Nx, Ny, Nz), (Hx, Hy, Hz)) + + szᶜᶜᵃ = new_data(FT, arch, (Center, Center, Nothing), args...) + szᶜᶠᵃ = new_data(FT, arch, (Center, Face, Nothing), args...) + szᶠᶜᵃ = new_data(FT, arch, (Face, Center, Nothing), args...) + szᶠᶠᵃ = new_data(FT, arch, (Face, Face, Nothing), args...) + + sΔzᶜᶜᵃ = new_data(FT, arch, (Center, Center, Nothing), args...) + sΔzᶜᶠᵃ = new_data(FT, arch, (Center, Face, Nothing), args...) + sΔzᶠᶜᵃ = new_data(FT, arch, (Face, Center, Nothing), args...) + sΔzᶠᶠᵃ = new_data(FT, arch, (Face, Face, Nothing), args...) + + sΔzᶜᶜᵃ₋ = new_data(FT, arch, (Center, Center, Nothing), args...) + sΔzᶜᶠᵃ₋ = new_data(FT, arch, (Center, Face, Nothing), args...) + sΔzᶠᶜᵃ₋ = new_data(FT, arch, (Face, Center, Nothing), args...) + + ∂t_s = new_data(FT, arch, (Center, Center, Nothing), args...) + + zᵃᵃᶠ = ZStarVerticalCoordinate(rᵃᵃᶠ, szᶜᶜᵃ, szᶠᶜᵃ, szᶜᶠᵃ, szᶠᶠᵃ, nothing, nothing, nothing, nothing) + zᵃᵃᶜ = ZStarVerticalCoordinate(rᵃᵃᶜ, szᶜᶜᵃ, szᶠᶜᵃ, szᶜᶠᵃ, szᶠᶠᵃ, nothing, nothing, nothing, nothing) + + Δzᵃᵃᶠ = ZStarVerticalCoordinate(Δrᵃᵃᶠ, sΔzᶜᶜᵃ, sΔzᶠᶜᵃ, sΔzᶜᶠᵃ, sΔzᶠᶠᵃ, sΔzᶜᶜᵃ₋, sΔzᶠᶜᵃ₋, sΔzᶜᶠᵃ₋, ∂t_s) + Δzᵃᵃᶜ = ZStarVerticalCoordinate(Δrᵃᵃᶜ, sΔzᶜᶜᵃ, sΔzᶠᶜᵃ, sΔzᶜᶠᵃ, sΔzᶠᶠᵃ, sΔzᶜᶜᵃ₋, sΔzᶠᶜᵃ₋, sΔzᶜᶠᵃ₋, ∂t_s) + + return Lr, zᵃᵃᶠ, zᵃᵃᶜ, Δzᵃᵃᶠ, Δzᵃᵃᶜ +end + From 4e4d40b91052f6cb3352990b906210700a607f49 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 12:28:15 +0200 Subject: [PATCH 335/567] couple of more bugfixes --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 2 +- src/ImmersedBoundaries/partial_cell_bottom.jl | 2 +- .../split_explicit_free_surface_kernels.jl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 83c23b2df4..e75df50573 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -95,7 +95,7 @@ correct_z_bottom!(bottom_field, grid, ib) = zb = @inbounds bottom_field[i, j, 1] for k in 1:grid.Nz z⁺ = znode(i, j, k+1, grid, c, c, f) - bottom_cell = zb ≤ z⁺ + bottom_cell = z⁺ ≤ zb @inbounds bottom_field[i, j, 1] = ifelse(bottom_cell, z⁺, zb) end end diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index d23fe199a7..926b026972 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -77,7 +77,7 @@ end for k in 1:grid.Nz z⁻ = znode(i, j, k, grid, c, c, f) Δz = Δzᶜᶜᶜ(i, j, k, underlying_grid) - bottom_cell = zb ≤ z⁻ + Δz * (1 - ϵ) + bottom_cell = z⁻ + Δz * (1 - ϵ) ≤ zb @inbounds bottom_field[i, j, 1] = ifelse(bottom_cell, z⁻ + Δz * (1 - ϵ), zb) end end diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 58c0231f00..a0226762f8 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -266,7 +266,7 @@ function initialize_auxiliary_state!(state, η, timestepper) return nothing end -@kernel function _barotropic_split_explicit_corrector!(u, v, U̅, V̅, U, V, Hᶠᶜ, Hᶜᶠ, grid) +@kernel function _barotropic_split_explicit_corrector!(u, v, U̅, V̅, U, V, grid) i, j, k = @index(Global, NTuple) k_top = grid.Nz+1 From a6362eeae2e1b32b6567a91eddc1f5f43bc21d81 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 13:24:03 +0200 Subject: [PATCH 336/567] more bugfixes --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 2 +- src/MultiRegion/multi_region_grid.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index e75df50573..806a4fb22f 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -114,7 +114,7 @@ end const AGFBIB = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} -@inline column_heightᶜᶜᵃ(i, j, k, ibg::AGFBIB) = @inbounds znode(i, j, grid.Nz+1, ibg, c, c, f) - ibg.immersed_boundary.z_bottom[i, j, 1] +@inline column_heightᶜᶜᵃ(i, j, k, ibg::AGFBIB) = @inbounds znode(i, j, ibg.Nz+1, ibg, c, c, f) - ibg.immersed_boundary.z_bottom[i, j, 1] @inline column_heightᶜᶠᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶜᶜᵃ(i, j-1, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) @inline column_heightᶠᶜᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶜᶜᵃ(i-1, j, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) @inline column_heightᶠᶠᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶠᶜᵃ(i, j-1, k, ibg), column_heightᶠᶜᵃ(i, j, k, ibg)) diff --git a/src/MultiRegion/multi_region_grid.jl b/src/MultiRegion/multi_region_grid.jl index a65a8b5dd8..7d28ee6100 100644 --- a/src/MultiRegion/multi_region_grid.jl +++ b/src/MultiRegion/multi_region_grid.jl @@ -188,7 +188,7 @@ function reconstruct_global_grid(mrg::ImmersedMultiRegionGrid) return ImmersedBoundaryGrid(global_grid, global_boundary) end -reconstruct_global_boundary(g::GridFittedBottom{<:Field}) = GridFittedBottom(reconstruct_global_field(g.z_bottom), g.immersed_condition) +reconstruct_global_boundary(g::GridFittedBottom{<:Field}) = GridFittedBottom(reconstruct_global_field(g.z_bottom)) reconstruct_global_boundary(g::PartialCellBottom{<:Field}) = PartialCellBottom(reconstruct_global_field(g.z_bottom), g.minimum_fractional_cell_height) reconstruct_global_boundary(g::GridFittedBoundary{<:Field}) = GridFittedBoundary(reconstruct_global_field(g.mask)) From 3bce6fc823ee375ba42ac05835bb8f557c522569 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 14:14:40 +0200 Subject: [PATCH 337/567] this should make it work --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 806a4fb22f..a67cecf4b0 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -101,7 +101,7 @@ correct_z_bottom!(bottom_field, grid, ib) = end @inline function _immersed_cell(i, j, k, underlying_grid, ib::GridFittedBottom) - z = znode(i, j, k+1, underlying_grid, c, c, f) + z = znode(i, j, k, underlying_grid, c, c, c) h = @inbounds ib.z_bottom[i, j, 1] return z ≤ h end From 0fa1a67e29d3333ad7086c1b5a2510e5be37e7c8 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 14:16:24 +0200 Subject: [PATCH 338/567] unify the formulations --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 13 +++++++------ src/ImmersedBoundaries/partial_cell_bottom.jl | 9 --------- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index a67cecf4b0..ae9d0b525a 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -100,20 +100,21 @@ correct_z_bottom!(bottom_field, grid, ib) = end end -@inline function _immersed_cell(i, j, k, underlying_grid, ib::GridFittedBottom) - z = znode(i, j, k, underlying_grid, c, c, c) - h = @inbounds ib.z_bottom[i, j, 1] - return z ≤ h -end @inline z_bottom(i, j, ibg::GFBIBG) = @inbounds ibg.immersed_boundary.z_bottom[i, j, 1] ##### -##### Bottom height +##### _immersed_cell and column height ##### const AGFBIB = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} +@inline function _immersed_cell(i, j, k, underlying_grid, ib::AGFBIB) + z = znode(i, j, k, underlying_grid, c, c, c) + h = @inbounds ib.z_bottom[i, j, 1] + return z ≤ h +end + @inline column_heightᶜᶜᵃ(i, j, k, ibg::AGFBIB) = @inbounds znode(i, j, ibg.Nz+1, ibg, c, c, f) - ibg.immersed_boundary.z_bottom[i, j, 1] @inline column_heightᶜᶠᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶜᶜᵃ(i, j-1, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) @inline column_heightᶠᶜᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶜᶜᵃ(i-1, j, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index 926b026972..5d1ddf1210 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -117,15 +117,6 @@ on_architecture(to, ib::PartialCellBottom) = PartialCellBottom(on_architecture(t Criterion is zb ≥ z - ϵ Δz """ -@inline function _immersed_cell(i, j, k, underlying_grid, ib::PartialCellBottom) - # Face node below current cell - z = znode(i, j, k, underlying_grid, c, c, f) - zb = @inbounds ib.z_bottom[i, j, 1] - ϵ = ib.minimum_fractional_cell_height - # z + Δz is equal to the face above the current cell - Δz = Δzᶜᶜᶜ(i, j, k, underlying_grid) - return (z + Δz * (1 - ϵ)) ≤ zb -end @inline function bottom_cell(i, j, k, ibg::PCBIBG) grid = ibg.underlying_grid From 135cac345b32d6e9931c318564e1cabe1ea91030 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 14:18:08 +0200 Subject: [PATCH 339/567] correct implementation --- src/ImmersedBoundaries/partial_cell_bottom.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index 5d1ddf1210..966ae98ac4 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -117,6 +117,11 @@ on_architecture(to, ib::PartialCellBottom) = PartialCellBottom(on_architecture(t Criterion is zb ≥ z - ϵ Δz """ +@inline function _immersed_cell(i, j, k, underlying_grid, ib::PartialCellBottom) + z = znode(i, j, k+1, underlying_grid, c, c, f) + zb = @inbounds ib.z_bottom[i, j, 1] + return z ≤ zb +end @inline function bottom_cell(i, j, k, ibg::PCBIBG) grid = ibg.underlying_grid From 156dada1aa5da8f193c171d43dca7b24e3d0aff6 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 14:18:16 +0200 Subject: [PATCH 340/567] correct implementation --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index ae9d0b525a..a67cecf4b0 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -100,21 +100,20 @@ correct_z_bottom!(bottom_field, grid, ib) = end end +@inline function _immersed_cell(i, j, k, underlying_grid, ib::GridFittedBottom) + z = znode(i, j, k, underlying_grid, c, c, c) + h = @inbounds ib.z_bottom[i, j, 1] + return z ≤ h +end @inline z_bottom(i, j, ibg::GFBIBG) = @inbounds ibg.immersed_boundary.z_bottom[i, j, 1] ##### -##### _immersed_cell and column height +##### Bottom height ##### const AGFBIB = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} -@inline function _immersed_cell(i, j, k, underlying_grid, ib::AGFBIB) - z = znode(i, j, k, underlying_grid, c, c, c) - h = @inbounds ib.z_bottom[i, j, 1] - return z ≤ h -end - @inline column_heightᶜᶜᵃ(i, j, k, ibg::AGFBIB) = @inbounds znode(i, j, ibg.Nz+1, ibg, c, c, f) - ibg.immersed_boundary.z_bottom[i, j, 1] @inline column_heightᶜᶠᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶜᶜᵃ(i, j-1, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) @inline column_heightᶠᶜᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶜᶜᵃ(i-1, j, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) From 04e717070cec3c218ca7eec2a590ed118c8fb0b8 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 14:32:21 +0200 Subject: [PATCH 341/567] correct partial cell bottom --- src/ImmersedBoundaries/partial_cell_bottom.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index 966ae98ac4..3ece188427 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -76,7 +76,7 @@ end ϵ = ib.minimum_fractional_cell_height for k in 1:grid.Nz z⁻ = znode(i, j, k, grid, c, c, f) - Δz = Δzᶜᶜᶜ(i, j, k, underlying_grid) + Δz = Δzᶜᶜᶜ(i, j, k, grid) bottom_cell = z⁻ + Δz * (1 - ϵ) ≤ zb @inbounds bottom_field[i, j, 1] = ifelse(bottom_cell, z⁻ + Δz * (1 - ϵ), zb) end From 9258572332bf6acfc51469e601bbd707ec4f1aef Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 14:33:49 +0200 Subject: [PATCH 342/567] use center immersed condition for grid fitted boundary --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index a67cecf4b0..d76e5b6444 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -95,7 +95,8 @@ correct_z_bottom!(bottom_field, grid, ib) = zb = @inbounds bottom_field[i, j, 1] for k in 1:grid.Nz z⁺ = znode(i, j, k+1, grid, c, c, f) - bottom_cell = z⁺ ≤ zb + z = znode(i, j, k+1, grid, c, c, c) + bottom_cell = z ≤ zb @inbounds bottom_field[i, j, 1] = ifelse(bottom_cell, z⁺, zb) end end From 9df333a62383c607efbdf01500901f792d7ae13f Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 14:34:24 +0200 Subject: [PATCH 343/567] use the *correct* center node --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index d76e5b6444..95a1d98efb 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -95,7 +95,7 @@ correct_z_bottom!(bottom_field, grid, ib) = zb = @inbounds bottom_field[i, j, 1] for k in 1:grid.Nz z⁺ = znode(i, j, k+1, grid, c, c, f) - z = znode(i, j, k+1, grid, c, c, c) + z = znode(i, j, k, grid, c, c, c) bottom_cell = z ≤ zb @inbounds bottom_field[i, j, 1] = ifelse(bottom_cell, z⁺, zb) end From 053b26faf7b4c74f912b33d36c15f94dabdfdaa4 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 14:35:33 +0200 Subject: [PATCH 344/567] no h for z-values! --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 6 +++--- src/ImmersedBoundaries/partial_cell_bottom.jl | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 95a1d98efb..5ddfe41ddd 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -102,9 +102,9 @@ correct_z_bottom!(bottom_field, grid, ib) = end @inline function _immersed_cell(i, j, k, underlying_grid, ib::GridFittedBottom) - z = znode(i, j, k, underlying_grid, c, c, c) - h = @inbounds ib.z_bottom[i, j, 1] - return z ≤ h + z = znode(i, j, k, underlying_grid, c, c, c) + zb = @inbounds ib.z_bottom[i, j, 1] + return z ≤ zb end @inline z_bottom(i, j, ibg::GFBIBG) = @inbounds ibg.immersed_boundary.z_bottom[i, j, 1] diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index 3ece188427..51f01ffb08 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -137,9 +137,9 @@ end # Get node at face above and defining nodes on c,c,f z = znode(i, j, k+1, underlying_grid, c, c, f) - # Get bottom height and fractional Δz parameter - h = @inbounds ib.z_bottom[i, j, 1] - ϵ = ibg.immersed_boundary.minimum_fractional_cell_height + # Get bottom z-coordinate and fractional Δz parameter + zb = @inbounds ib.z_bottom[i, j, 1] + ϵ = ibg.immersed_boundary.minimum_fractional_cell_height # Are we in a bottom cell? at_the_bottom = bottom_cell(i, j, k, ibg) From 61b7e7f0a7729badd5519d4e71e3fa69cb7b722a Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 14:37:10 +0200 Subject: [PATCH 345/567] simplify partial cells --- src/ImmersedBoundaries/partial_cell_bottom.jl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index 51f01ffb08..2e6e9ba50b 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -139,13 +139,12 @@ end # Get bottom z-coordinate and fractional Δz parameter zb = @inbounds ib.z_bottom[i, j, 1] - ϵ = ibg.immersed_boundary.minimum_fractional_cell_height # Are we in a bottom cell? at_the_bottom = bottom_cell(i, j, k, ibg) - full_Δz = Δzᶜᶜᶜ(i, j, k, ibg.underlying_grid) - partial_Δz = max(ϵ * full_Δz, z - h) + full_Δz = Δzᶜᶜᶜ(i, j, k, ibg.underlying_grid) + partial_Δz = z - zb return ifelse(at_the_bottom, partial_Δz, full_Δz) end From 1098c58cc850dbe8c07e636565053aa76eca2d36 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 14:55:41 +0200 Subject: [PATCH 346/567] make sure we don't go out of bounds --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 5ddfe41ddd..4fd1efd05a 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -120,3 +120,12 @@ const AGFBIB = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Abstrac @inline column_heightᶠᶜᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶜᶜᵃ(i-1, j, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) @inline column_heightᶠᶠᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶠᶜᵃ(i, j-1, k, ibg), column_heightᶠᶜᵃ(i, j, k, ibg)) +# Make sure Δz works for horizontally-Flat topologies. +# (There's no point in using z-Flat with PartialCellBottom). +XFlatAGFIBG = ImmersedBoundaryGrid{<:Any, <:Flat, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} +YFlatAGFIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Flat, <:Any, <:Any, <:AbstractGridFittedBottom} + +@inline column_heightᶠᶜᵃ(i, j, k, ibg::XFlatAGFIBG) = column_heightᶜᶜᵃ(i, j, k, ibg) +@inline column_heightᶜᶠᵃ(i, j, k, ibg::YFlatAGFIBG) = column_heightᶜᶜᵃ(i, j, k, ibg) +@inline column_heightᶠᶠᵃ(i, j, k, ibg::XFlatAGFIBG) = column_heightᶜᶠᵃ(i, j, k, ibg) +@inline column_heightᶠᶠᵃ(i, j, k, ibg::YFlatAGFIBG) = column_heightᶠᶜᵃ(i, j, k, ibg) From 9e35af6a62c66462de51fcce3b47aa6b4b3a693f Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 15:11:19 +0200 Subject: [PATCH 347/567] back to immersed condition --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 33 +++++++++++++++----- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 4fd1efd05a..1a2cc89e57 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -15,10 +15,22 @@ import Oceananigans.TurbulenceClosures: z_bottom abstract type AbstractGridFittedBottom{H} <: AbstractGridFittedBoundary end -struct GridFittedBottom{H} <: AbstractGridFittedBottom{H} +# To enable comparison with PartialCellBottom in the limiting case that +# fractional cell height is 1.0. +struct InterfaceImmersedCondition end +struct CenterImmersedCondition end + +struct GridFittedBottom{H, I} <: AbstractGridFittedBottom{H} z_bottom :: H + immersed_condition :: I end + +GridFittedBottom(z_bottom) = GridFittedBottom(z_bottom, CenterImmersedCondition()) + +Base.summary(::CenterImmersedCondition) = "CenterImmersedCondition" +Base.summary(::InterfaceImmersedCondition) = "InterfaceImmersedCondition" + const GFBIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:GridFittedBottom} """ @@ -31,6 +43,11 @@ Keyword Arguments * `z_bottom`: an array or function that gives the height of the bottom in absolute ``z`` coordinates. + +* `immersed_condition`: Determine whether the part of the domain that is immersed are all the cell centers that lie below + `z_bottom` (`CenterImmersedCondition()`; default) or all the cell faces that lie below `bottom_height` (`InterfaceImmersedCondition()`). + The only purpose of `immersed_condition` to allow `GridFittedBottom` and `PartialCellBottom` to have the same behavior when the + minimum fractional cell height for partial cells is set to 0. """ function Base.summary(ib::GridFittedBottom) @@ -56,7 +73,7 @@ function Base.show(io::IO, ib::GridFittedBottom) print(io, "├── z_bottom: ", prettysummary(ib.z_bottom), '\n') end -on_architecture(arch, ib::GridFittedBottom) = GridFittedBottom(on_architecture(ib.z_bottom)) +on_architecture(arch, ib::GridFittedBottom) = GridFittedBottom(on_architecture(ib.z_bottom), ib.immersed_condition) function on_architecture(arch, ib::GridFittedBottom{<:Field}) architecture(ib.z_bottom) == arch && return ib @@ -64,10 +81,10 @@ function on_architecture(arch, ib::GridFittedBottom{<:Field}) new_z_bottom = Field{Center, Center, Nothing}(arch_grid) set!(new_z_bottom, ib.z_bottom) fill_halo_regions!(new_z_bottom) - return GridFittedBottom(new_z_bottom) + return GridFittedBottom(new_z_bottom, ib.immersed_condition) end -Adapt.adapt_structure(to, ib::GridFittedBottom) = GridFittedBottom(adapt(to, ib.z_bottom)) +Adapt.adapt_structure(to, ib::GridFittedBottom) = GridFittedBottom(adapt(to, ib.z_bottom), ib.immersed_condition) """ ImmersedBoundaryGrid(grid, ib::GridFittedBottom) @@ -90,13 +107,14 @@ end correct_z_bottom!(bottom_field, grid, ib) = launch!(architecture(grid), grid, :xy, _correct_z_bottom!, bottom_field, grid, ib) -@kernel function _correct_z_bottom!(bottom_field, grid, ::GridFittedBottom) +@kernel function _correct_z_bottom!(bottom_field, grid, ib::GridFittedBottom) i, j = @index(Global, NTuple) zb = @inbounds bottom_field[i, j, 1] + condition = ib.immersed_condition for k in 1:grid.Nz z⁺ = znode(i, j, k+1, grid, c, c, f) z = znode(i, j, k, grid, c, c, c) - bottom_cell = z ≤ zb + bottom_cell = ifelse(condition isa CenterImmersedCondition, z ≤ zb, z⁺ ≤ zb) @inbounds bottom_field[i, j, 1] = ifelse(bottom_cell, z⁺, zb) end end @@ -120,8 +138,7 @@ const AGFBIB = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Abstrac @inline column_heightᶠᶜᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶜᶜᵃ(i-1, j, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) @inline column_heightᶠᶠᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶠᶜᵃ(i, j-1, k, ibg), column_heightᶠᶜᵃ(i, j, k, ibg)) -# Make sure Δz works for horizontally-Flat topologies. -# (There's no point in using z-Flat with PartialCellBottom). +# Make sure column_height works for horizontally-Flat topologies. XFlatAGFIBG = ImmersedBoundaryGrid{<:Any, <:Flat, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} YFlatAGFIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Flat, <:Any, <:Any, <:AbstractGridFittedBottom} From b9abd22979dc82c6a47a1337512625ff976d51d3 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 16:40:27 +0200 Subject: [PATCH 348/567] change some stuff --- .../vector_invariant_cross_upwinding.jl | 4 +- .../vector_invariant_self_upwinding.jl | 11 +- src/Grids/Grids.jl | 2 + src/Grids/z_star_vertical_coordinate.jl | 125 +++++++++-- .../compute_w_from_continuity.jl | 2 +- .../generalized_vertical_spacing.jl | 127 +----------- .../z_star_vertical_spacing.jl | 194 +++--------------- src/Operators/Operators.jl | 1 + .../spacings_and_areas_and_volumes.jl | 36 +++- 9 files changed, 186 insertions(+), 316 deletions(-) diff --git a/src/Advection/vector_invariant_cross_upwinding.jl b/src/Advection/vector_invariant_cross_upwinding.jl index 6d0ef5b861..fdd4e6a975 100644 --- a/src/Advection/vector_invariant_cross_upwinding.jl +++ b/src/Advection/vector_invariant_cross_upwinding.jl @@ -22,7 +22,7 @@ δ_stencil = scheme.upwinding.divergence_stencil δᴿ = _biased_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, bias(û), flux_div_xyᶜᶜᶜ, δ_stencil, u, v) - ∂ts = _symmetric_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, cross_scheme, V_times_∂t_s_grid) + ∂ts = _symmetric_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, cross_scheme, V_times_∂t_grid) return û * (δᴿ + ∂ts) end @@ -32,7 +32,7 @@ end δ_stencil = scheme.upwinding.divergence_stencil δᴿ = _biased_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, bias(v̂), flux_div_xyᶜᶜᶜ, δ_stencil, u, v) - ∂ts = _symmetric_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, cross_scheme, V_times_∂t_s_grid) + ∂ts = _symmetric_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, cross_scheme, V_times_∂t_grid) return v̂ * (δᴿ + ∂ts) diff --git a/src/Advection/vector_invariant_self_upwinding.jl b/src/Advection/vector_invariant_self_upwinding.jl index fde626c3ad..5fd44967bb 100644 --- a/src/Advection/vector_invariant_self_upwinding.jl +++ b/src/Advection/vector_invariant_self_upwinding.jl @@ -1,22 +1,21 @@ +using Oceananigans.Grids: V_times_∂t_grid + ##### ##### Self Upwinding of Divergence Flux, the best option! ##### -# Metric term for moving grids -@inline V_times_∂t_s_grid(i, j, k, grid) = zero(grid) - @inline δx_U(i, j, k, grid, u, v) = δxᶜᶜᶜ(i, j, k, grid, Ax_qᶠᶜᶜ, u) @inline δy_V(i, j, k, grid, u, v) = δyᶜᶜᶜ(i, j, k, grid, Ay_qᶜᶠᶜ, v) -@inline δx_U_plus_metric(i, j, k, grid, u, v) = δxᶜᶜᶜ(i, j, k, grid, Ax_qᶠᶜᶜ, u) + V_times_∂t_s_grid(i, j, k, grid) -@inline δy_V_plus_metric(i, j, k, grid, u, v) = δyᶜᶜᶜ(i, j, k, grid, Ay_qᶜᶠᶜ, v) + V_times_∂t_s_grid(i, j, k, grid) +@inline δx_U_plus_metric(i, j, k, grid, u, v) = δxᶜᶜᶜ(i, j, k, grid, Ax_qᶠᶜᶜ, u) + V_times_∂t_grid(i, j, k, grid) +@inline δy_V_plus_metric(i, j, k, grid, u, v) = δyᶜᶜᶜ(i, j, k, grid, Ay_qᶜᶠᶜ, v) + V_times_∂t_grid(i, j, k, grid) # Velocity smoothness for divergence upwinding @inline U_smoothness(i, j, k, grid, u, v) = ℑxᶜᵃᵃ(i, j, k, grid, Ax_qᶠᶜᶜ, u) @inline V_smoothness(i, j, k, grid, u, v) = ℑyᵃᶜᵃ(i, j, k, grid, Ay_qᶜᶠᶜ, v) # Divergence smoothness for divergence upwinding -@inline divergence_smoothness(i, j, k, grid, u, v) = δx_U(i, j, k, grid, u, v) + δy_V(i, j, k, grid, u, v) + V_times_∂t_s_grid(i, j, k, grid) +@inline divergence_smoothness(i, j, k, grid, u, v) = δx_U(i, j, k, grid, u, v) + δy_V(i, j, k, grid, u, v) + V_times_∂t_grid(i, j, k, grid) @inline function upwinded_divergence_flux_Uᶠᶜᶜ(i, j, k, grid, scheme::VectorInvariantSelfVerticalUpwinding, u, v) diff --git a/src/Grids/Grids.jl b/src/Grids/Grids.jl index fd8aebbf21..9c3830f6ab 100644 --- a/src/Grids/Grids.jl +++ b/src/Grids/Grids.jl @@ -18,6 +18,7 @@ export xnodes, ynodes, znodes, λnodes, φnodes export spacings export xspacings, yspacings, zspacings, xspacing, yspacing, zspacing export minimum_xspacing, minimum_yspacing, minimum_zspacing +export ZStarVerticalCoordinate, vertical_scaling, previous_vertical_scaling, reference_zspacings export offset_data, new_data export on_architecture @@ -127,5 +128,6 @@ include("grid_generation.jl") include("rectilinear_grid.jl") include("orthogonal_spherical_shell_grid.jl") include("latitude_longitude_grid.jl") +include("z_star_vertical_coordinate.jl") end # module diff --git a/src/Grids/z_star_vertical_coordinate.jl b/src/Grids/z_star_vertical_coordinate.jl index fed39ecafd..a2104de563 100644 --- a/src/Grids/z_star_vertical_coordinate.jl +++ b/src/Grids/z_star_vertical_coordinate.jl @@ -1,12 +1,27 @@ +##### +##### ZStar coordinate and associated types +##### + +abstract type AbstractVerticalCoordinate end ##### -##### ZStar coordinate and associated grid types +##### AbstractVerticalCoordinate grid definitions ##### +const AVLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractVerticalCoordinate} +const AVOSSG = OrthogonalSphericalShellGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractVerticalCoordinate} +const AVRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractVerticalCoordinate} + +const AbstractVerticalCoordinateGrid = Union{AVLLG, AVOSSG, AVRG} + +# rnode for an AbstractVerticalCoordinate grid is the reference node +@inline rnode(i, j, k, grid::AbstractVerticalCoordinateGrid, ℓx, ℓy, ::Center) = @inbounds grid.zᵃᵃᶜ[k] +@inline rnode(i, j, k, grid::AbstractVerticalCoordinateGrid, ℓx, ℓy, ::Face) = @inbounds grid.zᵃᵃᶠ[k] + """ - struct ZStarSpacing{R, S} <: AbstractVerticalSpacing{R} + struct ZStarVerticalCoordinate{R, S} <: AbstractVerticalSpacing{R} -A vertical spacing for the hydrostatic free surface model that follows the free surface. +A vertical coordinate for the hydrostatic free surface model that follows the free surface. The vertical spacing is defined by a reference spacing `Δr` and a scaling `s` that obeys ```math s = (η + H) / H @@ -22,7 +37,7 @@ where ``η`` is the free surface height and ``H`` the vertical depth of the wate - `s⁻`: scaling of the vertical coordinate at time step `n - 1` at `(Center, Center, Any)` location - `∂t_s`: Time derivative of `s` """ -struct ZStarVerticalCoordinate{R, SCC, SFC, SCF, SFF} <: AbstractVerticalCoordinate{R} +struct ZStarVerticalCoordinate{R, SCC, SFC, SCF, SFF} <: AbstractVerticalCoordinate reference :: R sᶜᶜⁿ :: SCC sᶠᶜⁿ :: SFC @@ -61,8 +76,7 @@ on_architecture(arch, coord::ZStarVerticalCoordinate) = Grids.coordinate_summary(::Bounded, Δ::ZStarVerticalCoordinate, name) = @sprintf("Free-surface following with Δ%s=%s", name, prettysummary(Δ.reference)) - -generate_coordinate(FT, topo::Periodic, N, H, coordinate::ZStarVerticalCoordinate, coordinate_name, arch, args...) = +generate_coordinate(FT, ::Periodic, N, H, ::ZStarVerticalCoordinate, coordinate_name, arch, args...) = throw(ArgumentError("Periodic domains are not supported for ZStarVerticalCoordinate")) # Generate a regularly-spaced coordinate passing the domain extent (2-tuple) and number of points @@ -79,11 +93,6 @@ function generate_coordinate(FT, topo::Bounded, Nz, Hz, coordinate::ZStarVertica args = (topo, (Nx, Ny, Nz), (Hx, Hy, Hz)) - szᶜᶜᵃ = new_data(FT, arch, (Center, Center, Nothing), args...) - szᶜᶠᵃ = new_data(FT, arch, (Center, Face, Nothing), args...) - szᶠᶜᵃ = new_data(FT, arch, (Face, Center, Nothing), args...) - szᶠᶠᵃ = new_data(FT, arch, (Face, Face, Nothing), args...) - sΔzᶜᶜᵃ = new_data(FT, arch, (Center, Center, Nothing), args...) sΔzᶜᶠᵃ = new_data(FT, arch, (Center, Face, Nothing), args...) sΔzᶠᶜᵃ = new_data(FT, arch, (Face, Center, Nothing), args...) @@ -94,9 +103,11 @@ function generate_coordinate(FT, topo::Bounded, Nz, Hz, coordinate::ZStarVertica sΔzᶠᶜᵃ₋ = new_data(FT, arch, (Face, Center, Nothing), args...) ∂t_s = new_data(FT, arch, (Center, Center, Nothing), args...) + η = new_data(FT, arch, (Center, Center, Nothing), args...) # Storage place for the free surface height - zᵃᵃᶠ = ZStarVerticalCoordinate(rᵃᵃᶠ, szᶜᶜᵃ, szᶠᶜᵃ, szᶜᶠᵃ, szᶠᶠᵃ, nothing, nothing, nothing, nothing) - zᵃᵃᶜ = ZStarVerticalCoordinate(rᵃᵃᶜ, szᶜᶜᵃ, szᶠᶜᵃ, szᶜᶠᵃ, szᶠᶠᵃ, nothing, nothing, nothing, nothing) + # The scaling is the same for everyone (H + \eta) / H + zᵃᵃᶠ = ZStarVerticalCoordinate(rᵃᵃᶠ, sΔzᶜᶜᵃ, sΔzᶠᶜᵃ, sΔzᶜᶠᵃ, sΔzᶠᶠᵃ, sΔzᶜᶜᵃ₋, sΔzᶠᶜᵃ₋, sΔzᶜᶠᵃ₋, η) + zᵃᵃᶜ = ZStarVerticalCoordinate(rᵃᵃᶜ, sΔzᶜᶜᵃ, sΔzᶠᶜᵃ, sΔzᶜᶠᵃ, sΔzᶠᶠᵃ, sΔzᶜᶜᵃ₋, sΔzᶠᶜᵃ₋, sΔzᶜᶠᵃ₋, η) Δzᵃᵃᶠ = ZStarVerticalCoordinate(Δrᵃᵃᶠ, sΔzᶜᶜᵃ, sΔzᶠᶜᵃ, sΔzᶜᶠᵃ, sΔzᶠᶠᵃ, sΔzᶜᶜᵃ₋, sΔzᶠᶜᵃ₋, sΔzᶜᶠᵃ₋, ∂t_s) Δzᵃᵃᶜ = ZStarVerticalCoordinate(Δrᵃᵃᶜ, sΔzᶜᶜᵃ, sΔzᶠᶜᵃ, sΔzᶜᶠᵃ, sΔzᶠᶠᵃ, sΔzᶜᶜᵃ₋, sΔzᶠᶜᵃ₋, sΔzᶜᶠᵃ₋, ∂t_s) @@ -104,3 +115,91 @@ function generate_coordinate(FT, topo::Bounded, Nz, Hz, coordinate::ZStarVertica return Lr, zᵃᵃᶠ, zᵃᵃᶜ, Δzᵃᵃᶠ, Δzᵃᵃᶜ end +const ZStarLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:ZStarVerticalCoordinate} +const ZStarOSSG = OrthogonalSphericalShellGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:ZStarVerticalCoordinate} +const ZStarRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:ZStarVerticalCoordinate} + +const ZStarUnderlyingGrid = Union{ZStarLLG, ZStarOSSG, ZStarRG} + +##### +##### ZStar-specific vertical spacing functions +##### + +const C = Center +const F = Face + +const ZSG = ZStarUnderlyingGrid + +# Fallbacks +reference_zspacings(grid, ::C) = grid.Δzᵃᵃᶜ +reference_zspacings(grid, ::F) = grid.Δzᵃᵃᶠ + +@inline vertical_scaling(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) +@inline previous_vertical_scaling(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) + +@inline ∂t_grid(i, j, k, grid) = zero(grid) +@inline V_times_∂t_grid(i, j, k, grid) = zero(grid) + +reference_zspacings(grid::ZSG, ::C) = grid.Δzᵃᵃᶜ.Δr +reference_zspacings(grid::ZSG, ::F) = grid.Δzᵃᵃᶠ.Δr + +@inline vertical_scaling(i, j, k, grid::ZSG, ::C, ::C, ::C) = @inbounds grid.Δzᵃᵃᶜ.sᶜᶜⁿ[i, j] +@inline vertical_scaling(i, j, k, grid::ZSG, ::F, ::C, ::C) = @inbounds grid.Δzᵃᵃᶜ.sᶠᶜⁿ[i, j] +@inline vertical_scaling(i, j, k, grid::ZSG, ::C, ::F, ::C) = @inbounds grid.Δzᵃᵃᶜ.sᶜᶠⁿ[i, j] +@inline vertical_scaling(i, j, k, grid::ZSG, ::F, ::F, ::C) = @inbounds grid.Δzᵃᵃᶜ.sᶠᶠⁿ[i, j] + +@inline vertical_scaling(i, j, k, grid::ZSG, ::C, ::C, ::F) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶜⁿ[i, j] +@inline vertical_scaling(i, j, k, grid::ZSG, ::F, ::C, ::F) = @inbounds grid.Δzᵃᵃᶠ.sᶠᶜⁿ[i, j] +@inline vertical_scaling(i, j, k, grid::ZSG, ::C, ::F, ::F) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶠⁿ[i, j] +@inline vertical_scaling(i, j, k, grid::ZSG, ::F, ::F, ::F) = @inbounds grid.Δzᵃᵃᶠ.sᶠᶠⁿ[i, j] + +@inline previous_vertical_scaling(i, j, k, grid::ZSG, ::C, ::C, ::C) = @inbounds grid.Δzᵃᵃᶜ.sᶜᶜ⁻[i, j] +@inline previous_vertical_scaling(i, j, k, grid::ZSG, ::F, ::C, ::C) = @inbounds grid.Δzᵃᵃᶜ.sᶠᶜ⁻[i, j] +@inline previous_vertical_scaling(i, j, k, grid::ZSG, ::C, ::F, ::C) = @inbounds grid.Δzᵃᵃᶜ.sᶜᶠ⁻[i, j] + +@inline previous_vertical_scaling(i, j, k, grid::ZSG, ::C, ::C, ::F) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶜ⁻[i, j] +@inline previous_vertical_scaling(i, j, k, grid::ZSG, ::F, ::C, ::F) = @inbounds grid.Δzᵃᵃᶠ.sᶠᶜ⁻[i, j] +@inline previous_vertical_scaling(i, j, k, grid::ZSG, ::C, ::F, ::F) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶠ⁻[i, j] + +@inline ∂t_grid(i, j, k, grid::ZSG) = @inbounds grid.Δzᵃᵃᶜ.∂t_s[i, j] +@inline V_times_∂t_grid(i, j, k, grid::ZSG) = ∂t_grid(i, j, k, grid) * Vᶜᶜᶜ(i, j, k, grid) + +##### +##### znode +##### + +const c = Center() +const f = Face() + +# rnode for an AbstractVerticalCoordinate grid is the reference node +@inline znode(i, j, k, grid::ZSG, ::C, ::C, ::C) = @inbounds grid.zᵃᵃᶜ.reference[k] * vertical_scaling(i, j, k, grid, c, c, c) + grid.zᵃᵃᶜ.∂t_s[i, j] +@inline znode(i, j, k, grid::ZSG, ::C, ::F, ::C) = @inbounds grid.zᵃᵃᶜ.reference[k] * vertical_scaling(i, j, k, grid, c, f, c) + grid.zᵃᵃᶜ.∂t_s[i, j] +@inline znode(i, j, k, grid::ZSG, ::F, ::C, ::C) = @inbounds grid.zᵃᵃᶜ.reference[k] * vertical_scaling(i, j, k, grid, f, c, c) + grid.zᵃᵃᶜ.∂t_s[i, j] +@inline znode(i, j, k, grid::ZSG, ::F, ::F, ::C) = @inbounds grid.zᵃᵃᶜ.reference[k] * vertical_scaling(i, j, k, grid, f, f, c) + grid.zᵃᵃᶜ.∂t_s[i, j] +@inline znode(i, j, k, grid::ZSG, ::C, ::C, ::F) = @inbounds grid.zᵃᵃᶠ.reference[k] * vertical_scaling(i, j, k, grid, c, c, c) + grid.zᵃᵃᶠ.∂t_s[i, j] +@inline znode(i, j, k, grid::ZSG, ::C, ::F, ::F) = @inbounds grid.zᵃᵃᶠ.reference[k] * vertical_scaling(i, j, k, grid, c, f, c) + grid.zᵃᵃᶠ.∂t_s[i, j] +@inline znode(i, j, k, grid::ZSG, ::F, ::C, ::F) = @inbounds grid.zᵃᵃᶠ.reference[k] * vertical_scaling(i, j, k, grid, f, c, c) + grid.zᵃᵃᶠ.∂t_s[i, j] +@inline znode(i, j, k, grid::ZSG, ::F, ::F, ::F) = @inbounds grid.zᵃᵃᶠ.reference[k] * vertical_scaling(i, j, k, grid, f, f, c) + grid.zᵃᵃᶠ.∂t_s[i, j] + +function retrieve_static_grid(grid::ZStarUnderlyingGrid) + + Δzᵃᵃᶠ = reference_zspacings(grid, Face()) + Δzᵃᵃᶜ = reference_zspacings(grid, Center()) + + TX, TY, TZ = topology(grid) + + args = [] + for prop in propertynames(grid) + if prop == :Δzᵃᵃᶠ + push!(args, Δzᵃᵃᶠ) + elseif prop == :Δzᵃᵃᶜ + push!(args, Δzᵃᵃᶜ) + else + push!(args, getproperty(grid, prop)) + end + end + + GridType = getnamewrapper(grid) + + return GridType{TX, TY, TZ}(args...) +end \ No newline at end of file diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl index 3f1d6b2321..29b2291429 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl @@ -25,7 +25,7 @@ compute_w_from_continuity!(velocities, arch, grid; parameters = w_kernel_paramet @inbounds U.w[i, j, 1] = 0 for k in 2:grid.Nz+1 δ_Uh = flux_div_xyᶜᶜᶜ(i, j, k-1, grid, U.u, U.v) / Azᶜᶜᶜ(i, j, k-1, grid) - ∂t_s = Δrᶜᶜᶜ(i, j, k-1, grid) * ∂t_s_grid(i, j, k-1, grid) + ∂t_s = Δrᶜᶜᶜ(i, j, k-1, grid) * ∂t_grid(i, j, k-1, grid) immersed = immersed_cell(i, j, k-1, grid) Δw = δ_Uh + ifelse(immersed, 0, ∂t_s) # We do not account for grid changes in immersed cells diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index f7d9f74991..0a02b5b5a5 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -8,135 +8,14 @@ using Oceananigans.Utils: getnamewrapper using Adapt using Printf -import Oceananigans.Grids: with_halo, znode -import Oceananigans.Operators: Δzᶜᶜᶠ, Δzᶜᶜᶜ, Δzᶜᶠᶠ, Δzᶜᶠᶜ, Δzᶠᶜᶠ, Δzᶠᶜᶜ, Δzᶠᶠᶠ, Δzᶠᶠᶜ -import Oceananigans.Advection: V_times_∂t_s_grid +import Oceananigans.Grids: with_halo, znode, ∂t_grid, vertical_scaling, previous_vertical_scaling import Oceananigans.Architectures: arch_array -""" - AbstractVerticalSpacing{R} - -spacings for a generalized vertical coordinate system. The reference (non-moving) spacings are stored in `Δr`. -`Δ` contains the spacings associated with the moving coordinate system. -`s⁻`, `sⁿ` and `∂t_s` fields are the vertical derivative of the vertical coordinate (∂Δr/∂Δz) -at timestep `n-1` and `n` and it's time derivative. -`denomination` contains the "type" of generalized vertical coordinate (the only one implemented is `ZStar`) -""" -abstract type AbstractVerticalSpacing{R} end - -const AbstractVerticalSpacingRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractVerticalSpacing} -const AbstractVerticalSpacingLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractVerticalSpacing} - -const AbstractVerticalSpacingUnderlyingGrid = Union{AbstractVerticalSpacingRG, AbstractVerticalSpacingLLG} -const AbstractVerticalSpacingImmersedGrid = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:AbstractVerticalSpacingUnderlyingGrid} - -const AbstractVerticalSpacingGrid = Union{AbstractVerticalSpacingUnderlyingGrid, AbstractVerticalSpacingImmersedGrid} - -##### -##### Original grid -##### - -retrieve_static_grid(grid) = grid - -function retrieve_static_grid(grid::AbstractVerticalSpacingImmersedGrid) - underlying_grid = retrieve_static_grid(grid.underlying_grid) - active_cells_map = !isnothing(grid.interior_active_cells) - return ImmersedBoundaryGrid(underlying_grid, grid.immersed_boundary; active_cells_map) -end - -reference_zspacings(grid, ::Face) = grid.Δzᵃᵃᶠ -reference_zspacings(grid, ::Center) = grid.Δzᵃᵃᶜ - -function retrieve_static_grid(grid::AbstractVerticalSpacingUnderlyingGrid) - - Δzᵃᵃᶠ = reference_zspacings(grid, Face()) - Δzᵃᵃᶜ = reference_zspacings(grid, Center()) - - TX, TY, TZ = topology(grid) - - args = [] - for prop in propertynames(grid) - if prop == :Δzᵃᵃᶠ - push!(args, Δzᵃᵃᶠ) - elseif prop == :Δzᵃᵃᶜ - push!(args, Δzᵃᵃᶜ) - else - push!(args, getproperty(grid, prop)) - end - end - - GridType = getnamewrapper(grid) - - return GridType{TX, TY, TZ}(args...) -end - -##### -##### Some extensions -##### - -function with_halo(new_halo, grid::AbstractVerticalSpacingUnderlyingGrid) - old_static_grid = retrieve_static_grid(grid) - new_static_grid = with_halo(new_halo, old_static_grid) - vertical_coordinate = denomination(grid) - new_grid = generalized_spacing_grid(new_static_grid, vertical_coordinate) - - return new_grid -end - -function with_halo(new_halo, ibg::AbstractVerticalSpacingImmersedGrid) - underlying_grid = ibg.underlying_grid - immersed_boundary = ibg.immersed_boundary - new_underlying_grid = with_halo(new_halo, underlying_grid) - active_cells_map = isnothing(ibg.interior_active_cells) - new_ibg = ImmersedBoundaryGrid(new_underlying_grid, immersed_boundary; active_cells_map) - return new_ibg -end - ##### ##### General implementation ##### -generalized_spacing_grid(grid, coord) = grid -update_vertical_spacing!(model, grid; kwargs...) = nothing - -##### -##### Reference (local) vertical spacings in `r-coordinates` -##### - -@inline Δr(i, j, k, Δz::Number) = Δz -@inline Δr(i, j, k, Δz::AbstractVector) = @inbounds Δz[k] - -@inline Δr(i, j, k, Δz::AbstractVerticalSpacing{<:Number}) = Δz.Δr -@inline Δr(i, j, k, Δz::AbstractVerticalSpacing) = @inbounds Δz.Δr[k] - -@inline Δrᶜᶜᶠ(i, j, k, grid) = Δr(i, j, k, grid.Δzᵃᵃᶠ) -@inline Δrᶜᶜᶜ(i, j, k, grid) = Δr(i, j, k, grid.Δzᵃᵃᶜ) - -@inline Δrᶜᶠᶠ(i, j, k, grid) = ℑyᵃᶠᵃ(i, j, k, grid, Δrᶜᶜᶠ) -@inline Δrᶜᶠᶜ(i, j, k, grid) = ℑyᵃᶠᵃ(i, j, k, grid, Δrᶜᶜᶜ) - -@inline Δrᶠᶜᶠ(i, j, k, grid) = ℑxᶠᵃᵃ(i, j, k, grid, Δrᶜᶜᶠ) -@inline Δrᶠᶜᶜ(i, j, k, grid) = ℑxᶠᵃᵃ(i, j, k, grid, Δrᶜᶜᶜ) - -@inline Δrᶠᶠᶠ(i, j, k, grid) = ℑxyᶠᶠᵃ(i, j, k, grid, Δrᶜᶜᶠ) -@inline Δrᶠᶠᶜ(i, j, k, grid) = ℑxyᶠᶠᵃ(i, j, k, grid, Δrᶜᶜᶜ) - -##### -##### Vertical spacings for a generalized vertical coordinate system -##### - -# Fallbacks -@inline previous_vertical_scaling(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) -@inline vertical_scaling(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) - -@inline Δzᶜᶜᶠ(i, j, k, grid::AbstractVerticalSpacingGrid) = @inbounds Δrᶜᶜᶠ(i, j, k, grid) * vertical_scaling(i, j, k, grid, c, c, f) -@inline Δzᶜᶜᶜ(i, j, k, grid::AbstractVerticalSpacingGrid) = @inbounds Δrᶜᶜᶜ(i, j, k, grid) * vertical_scaling(i, j, k, grid, c, c, c) -@inline Δzᶠᶜᶠ(i, j, k, grid::AbstractVerticalSpacingGrid) = @inbounds Δrᶠᶜᶠ(i, j, k, grid) * vertical_scaling(i, j, k, grid, f, c, f) -@inline Δzᶠᶜᶜ(i, j, k, grid::AbstractVerticalSpacingGrid) = @inbounds Δrᶠᶜᶜ(i, j, k, grid) * vertical_scaling(i, j, k, grid, f, c, c) -@inline Δzᶜᶠᶠ(i, j, k, grid::AbstractVerticalSpacingGrid) = @inbounds Δrᶜᶠᶠ(i, j, k, grid) * vertical_scaling(i, j, k, grid, c, f, f) -@inline Δzᶜᶠᶜ(i, j, k, grid::AbstractVerticalSpacingGrid) = @inbounds Δrᶜᶠᶜ(i, j, k, grid) * vertical_scaling(i, j, k, grid, c, f, c) -@inline Δzᶠᶠᶠ(i, j, k, grid::AbstractVerticalSpacingGrid) = @inbounds Δrᶠᶠᶠ(i, j, k, grid) * vertical_scaling(i, j, k, grid, f, f, f) -@inline Δzᶠᶠᶜ(i, j, k, grid::AbstractVerticalSpacingGrid) = @inbounds Δrᶠᶠᶜ(i, j, k, grid) * vertical_scaling(i, j, k, grid, f, f, c) +update_grid!(model, grid; kwargs...) = nothing ##### ##### Additional terms to be included in the momentum equations (fallbacks) @@ -145,8 +24,6 @@ update_vertical_spacing!(model, grid; kwargs...) = nothing @inline grid_slope_contribution_x(i, j, k, grid, args...) = zero(grid) @inline grid_slope_contribution_y(i, j, k, grid, args...) = zero(grid) -@inline ∂t_s_grid(i, j, k, grid) = zero(grid) - ##### ##### Tracer update in generalized vertical coordinates ##### We advance sθ but store θ once sⁿ⁺¹ is known diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 95d9b42857..660975e179 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -1,160 +1,23 @@ +using Oceananigans.Grids: ZStarUnderlyingGrid, rnode -##### -##### ZStar coordinate and associated grid types -##### - -""" free-surface following vertical coordinate """ -struct ZStar end - -""" - struct ZStarSpacing{R, S} <: AbstractVerticalSpacing{R} - -A vertical spacing for the hydrostatic free surface model that follows the free surface. -The vertical spacing is defined by a reference spacing `Δr` and a scaling `s` that obeys -```math -s = (η + H) / H -``` -where ``η`` is the free surface height and ``H`` the vertical depth of the water column - -# Fields -- `Δr`: reference vertical spacing with `η = 0` -- `sᶜᶜⁿ`: scaling of the vertical coordinate at time step `n` at `(Center, Center, Any)` location -- `sᶠᶜⁿ`: scaling of the vertical coordinate at time step `n` at `(Face, Center, Any)` location -- `sᶜᶠⁿ`: scaling of the vertical coordinate at time step `n` at `(Center, Face, Any)` location -- `sᶠᶠⁿ`: scaling of the vertical coordinate at time step `n` at `(Face, Face, Any)` location -- `s⁻`: scaling of the vertical coordinate at time step `n - 1` at `(Center, Center, Any)` location -- `∂t_s`: Time derivative of `s` -""" -struct ZStarSpacing{R, SCC, SFC, SCF, SFF} <: AbstractVerticalSpacing{R} - Δr :: R - sᶜᶜⁿ :: SCC - sᶠᶜⁿ :: SFC - sᶜᶠⁿ :: SCF - sᶠᶠⁿ :: SFF - sᶜᶜ⁻ :: SCC - sᶠᶜ⁻ :: SFC - sᶜᶠ⁻ :: SCF - ∂t_s :: SCC -end - -Adapt.adapt_structure(to, coord::ZStarSpacing) = - ZStarSpacing(Adapt.adapt(to, coord.Δr), - Adapt.adapt(to, coord.sᶜᶜⁿ), - Adapt.adapt(to, coord.sᶠᶜⁿ), - Adapt.adapt(to, coord.sᶜᶠⁿ), - Adapt.adapt(to, coord.sᶠᶠⁿ), - Adapt.adapt(to, coord.sᶜᶜ⁻), - Adapt.adapt(to, coord.sᶠᶜ⁻), - Adapt.adapt(to, coord.sᶜᶠ⁻), - Adapt.adapt(to, coord.∂t_s)) - -on_architecture(arch, coord::ZStarSpacing) = - ZStarSpacing(on_architecture(arch, coord.Δr), - on_architecture(arch, coord.sᶜᶜⁿ), - on_architecture(arch, coord.sᶠᶜⁿ), - on_architecture(arch, coord.sᶜᶠⁿ), - on_architecture(arch, coord.sᶠᶠⁿ), - on_architecture(arch, coord.sᶜᶜ⁻), - on_architecture(arch, coord.sᶠᶜ⁻), - on_architecture(arch, coord.sᶜᶠ⁻), - on_architecture(arch, coord.∂t_s)) - -Grids.coordinate_summary(::Bounded, Δ::ZStarSpacing, name) = - @sprintf("Free-surface following with Δ%s=%s", name, prettysummary(Δ.Δr)) - -const ZStarSpacingRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:ZStarSpacing} -const ZStarSpacingLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:ZStarSpacing} -const ZStarSpacingOSG = OrthogonalSphericalShellGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:ZStarSpacing} - -const ZStarSpacingUnderlyingGrid = Union{ZStarSpacingRG, ZStarSpacingLLG, ZStarSpacingOSG} -const ZStarSpacingImmersedGrid = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:ZStarSpacingUnderlyingGrid} - -const ZStarSpacingGrid = Union{ZStarSpacingUnderlyingGrid, ZStarSpacingImmersedGrid} - -function generalized_spacing_grid(grid::ImmersedBoundaryGrid, ::ZStar) - underlying_grid = generalized_spacing_grid(grid.underlying_grid, ZStar()) - active_cells_map = !isnothing(grid.interior_active_cells) - - return ImmersedBoundaryGrid(underlying_grid, grid.immersed_boundary; active_cells_map) -end - -# Replacing the z-coordinate with a moving vertical coordinate, defined by -# - the reference spacing, -# - the scaling to apply to the reference -# - the derivative in time of the spacing -function generalized_spacing_grid(grid::AbstractUnderlyingGrid{FT, TX, TY, TZ}, ::ZStar) where {FT, TX, TY, TZ} - - sᶜᶜ⁻ = Field{Center, Center, Nothing}(grid) - sᶜᶜⁿ = Field{Center, Center, Nothing}(grid) - sᶠᶜ⁻ = Field{Face, Center, Nothing}(grid) - sᶠᶜⁿ = Field{Face, Center, Nothing}(grid) - sᶜᶠ⁻ = Field{Center, Face, Nothing}(grid) - sᶜᶠⁿ = Field{Center, Face, Nothing}(grid) - sᶠᶠⁿ = Field{Face, Face, Nothing}(grid) - ∂t_s = Field{Center, Center, Nothing}(grid) - - # Initial "at-rest" conditions - fill!(sᶜᶜ⁻, 1) - fill!(sᶜᶜⁿ, 1) - fill!(sᶠᶜⁿ, 1) - fill!(sᶜᶠⁿ, 1) - fill!(sᶠᶜ⁻, 1) - fill!(sᶜᶠ⁻, 1) - fill!(sᶠᶠⁿ, 1) - - Δzᵃᵃᶠ = ZStarSpacing(grid.Δzᵃᵃᶠ, sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, sᶠᶜ⁻, sᶜᶠ⁻, ∂t_s) - Δzᵃᵃᶜ = ZStarSpacing(grid.Δzᵃᵃᶜ, sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, sᶠᶜ⁻, sᶜᶠ⁻, ∂t_s) - - args = [] - for prop in propertynames(grid) - if prop == :Δzᵃᵃᶠ - push!(args, Δzᵃᵃᶠ) - elseif prop == :Δzᵃᵃᶜ - push!(args, Δzᵃᵃᶜ) - else - push!(args, getproperty(grid, prop)) - end - end - - GridType = getnamewrapper(grid) - - return GridType{TX, TY, TZ}(args...) -end - -##### -##### ZStar-specific vertical spacing functions -##### - -reference_zspacings(grid::ZStarSpacingGrid, ::Face) = grid.Δzᵃᵃᶠ.Δr -reference_zspacings(grid::ZStarSpacingGrid, ::Center) = grid.Δzᵃᵃᶜ.Δr - -@inline vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Center, ::Center, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶜⁿ[i, j, 1] -@inline vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Face, ::Center, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶠᶜⁿ[i, j, 1] -@inline vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Center, ::Face, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶠⁿ[i, j, 1] -@inline vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Face, ::Face, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶠᶠⁿ[i, j, 1] - -@inline previous_vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Center, ::Center, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶜ⁻[i, j, 1] -@inline previous_vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Face, ::Center, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶠᶜ⁻[i, j, 1] -@inline previous_vertical_scaling(i, j, k, grid::ZStarSpacingGrid, ::Center, ::Face, ℓz) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶠ⁻[i, j, 1] - -@inline ∂t_s_grid(i, j, k, grid::ZStarSpacingGrid) = @inbounds grid.Δzᵃᵃᶜ.∂t_s[i, j, 1] -@inline V_times_∂t_s_grid(i, j, k, grid::ZStarSpacingGrid) = ∂t_s_grid(i, j, k, grid) * Vᶜᶜᶜ(i, j, k, grid) +ZStarSpacingGrid = ZStarUnderlyingGrid ##### ##### ZStar-specific vertical spacings update ##### -function update_vertical_spacing!(model, grid::ZStarSpacingGrid; parameters = :xy) +function update_grid!(model, grid::ZStarUnderlyingGrid; parameters = :xy) - # Scaling - sᶜᶜ⁻ = grid.Δzᵃᵃᶠ.sᶜᶜ⁻ - sᶜᶜⁿ = grid.Δzᵃᵃᶠ.sᶜᶜⁿ - sᶠᶜ⁻ = grid.Δzᵃᵃᶠ.sᶠᶜ⁻ - sᶠᶜⁿ = grid.Δzᵃᵃᶠ.sᶠᶜⁿ - sᶜᶠ⁻ = grid.Δzᵃᵃᶠ.sᶜᶠ⁻ - sᶜᶠⁿ = grid.Δzᵃᵃᶠ.sᶜᶠⁿ - sᶠᶠⁿ = grid.Δzᵃᵃᶠ.sᶠᶠⁿ - ∂t_s = grid.Δzᵃᵃᶠ.∂t_s + # Scaling (just update once, they are the same for all the metrics) + sᶜᶜ⁻ = grid.Δzᵃᵃᶠ.sᶜᶜ⁻ + sᶜᶜⁿ = grid.Δzᵃᵃᶠ.sᶜᶜⁿ + sᶠᶜ⁻ = grid.Δzᵃᵃᶠ.sᶠᶜ⁻ + sᶠᶜⁿ = grid.Δzᵃᵃᶠ.sᶠᶜⁿ + sᶜᶠ⁻ = grid.Δzᵃᵃᶠ.sᶜᶠ⁻ + sᶜᶠⁿ = grid.Δzᵃᵃᶠ.sᶜᶠⁿ + sᶠᶠⁿ = grid.Δzᵃᵃᶠ.sᶠᶠⁿ + ∂t_s = grid.Δzᵃᵃᶠ.∂t_s + η_grid = grid.zᵃᵃᶠ.∂t_s # Free surface variables Hᶜᶜ = model.free_surface.auxiliary.Hᶜᶜ @@ -168,7 +31,7 @@ function update_vertical_spacing!(model, grid::ZStarSpacingGrid; parameters = :x # Update vertical spacing with available parameters # No need to fill the halo as the scaling is updated _IN_ the halos launch!(architecture(grid), grid, parameters, _update_zstar!, - sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, sᶠᶜ⁻, sᶜᶠ⁻, η, Hᶜᶜ, Hᶠᶜ, Hᶜᶠ, Hᶠᶠ, grid) + sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, sᶠᶜ⁻, sᶜᶠ⁻, η_grid, η, Hᶜᶜ, Hᶠᶜ, Hᶜᶠ, Hᶠᶠ, grid) # Update the time derivative of the grid-scaling. Note that in this case we leverage the # free surface evolution equation, where the time derivative of the free surface is equal @@ -194,7 +57,7 @@ end end end -@kernel function _update_zstar!(sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, sᶠᶜ⁻, sᶜᶠ⁻, η, Hᶜᶜ, Hᶠᶜ, Hᶜᶠ, Hᶠᶠ, grid) +@kernel function _update_zstar!(sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, sᶠᶜ⁻, sᶜᶠ⁻, η_grid, η, Hᶜᶜ, Hᶠᶜ, Hᶜᶠ, Hᶠᶠ, grid) i, j = @index(Global, NTuple) k_top = grid.Nz+1 @inbounds begin @@ -203,15 +66,18 @@ end hᶜᶠ = (Hᶜᶠ[i, j, 1] + ℑyᵃᶠᵃ(i, j, k_top, grid, η)) / Hᶜᶠ[i, j, 1] hᶠᶠ = (Hᶠᶠ[i, j, 1] + ℑxyᶠᶠᵃ(i, j, k_top, grid, η)) / Hᶠᶠ[i, j, 1] - sᶜᶜ⁻[i, j, 1] = sᶜᶜⁿ[i, j, 1] - sᶠᶜ⁻[i, j, 1] = sᶠᶜⁿ[i, j, 1] - sᶜᶠ⁻[i, j, 1] = sᶜᶠⁿ[i, j, 1] + sᶜᶜ⁻[i, j] = sᶜᶜⁿ[i, j] + sᶠᶜ⁻[i, j] = sᶠᶜⁿ[i, j] + sᶜᶠ⁻[i, j] = sᶜᶠⁿ[i, j] # update current and previous scaling - sᶜᶜⁿ[i, j, 1] = hᶜᶜ - sᶠᶜⁿ[i, j, 1] = hᶠᶜ - sᶜᶠⁿ[i, j, 1] = hᶜᶠ - sᶠᶠⁿ[i, j, 1] = hᶠᶠ + sᶜᶜⁿ[i, j] = hᶜᶜ + sᶠᶜⁿ[i, j] = hᶠᶜ + sᶜᶠⁿ[i, j] = hᶜᶠ + sᶠᶠⁿ[i, j] = hᶠᶠ + + # Update η in the grid + η_grid[i, j] = η[i, j, k_top] end end @@ -219,16 +85,16 @@ end ##### ZStar-specific implementation of the additional terms to be included in the momentum equations ##### -@inline z_minus_rᶜᶜᶜ(i, j, k, grid, η, Hᶜᶜ) = @inbounds η[i, j, grid.Nz+1] * (1 + znode(i, j, k, grid, Center(), Center(), Center()) / Hᶜᶜ[i, j, 1]) +@inline z_minus_rᶜᶜᶜ(i, j, k, grid, η, Hᶜᶜ) = @inbounds η[i, j, grid.Nz+1] * (1 + rnode(i, j, k, grid, Center(), Center(), Center()) / Hᶜᶜ[i, j, 1]) @inline ∂x_z(i, j, k, grid, free_surface) = @inbounds ∂xᶠᶜᶜ(i, j, k, grid, z_minus_rᶜᶜᶜ, free_surface.η, free_surface.auxiliary.Hᶜᶜ) @inline ∂y_z(i, j, k, grid, free_surface) = @inbounds ∂yᶜᶠᶜ(i, j, k, grid, z_minus_rᶜᶜᶜ, free_surface.η, free_surface.auxiliary.Hᶜᶜ) -@inline grid_slope_contribution_x(i, j, k, grid::ZStarSpacingGrid, free_surface, ::Nothing, model_fields) = zero(grid) -@inline grid_slope_contribution_y(i, j, k, grid::ZStarSpacingGrid, free_surface, ::Nothing, model_fields) = zero(grid) +@inline grid_slope_contribution_x(i, j, k, grid::ZStarUnderlyingGrid, free_surface, ::Nothing, model_fields) = zero(grid) +@inline grid_slope_contribution_y(i, j, k, grid::ZStarUnderlyingGrid, free_surface, ::Nothing, model_fields) = zero(grid) -@inline grid_slope_contribution_x(i, j, k, grid::ZStarSpacingGrid, free_surface, buoyancy, model_fields) = +@inline grid_slope_contribution_x(i, j, k, grid::ZStarUnderlyingGrid, free_surface, buoyancy, model_fields) = ℑxᶠᵃᵃ(i, j, k, grid, buoyancy_perturbationᶜᶜᶜ, buoyancy.model, model_fields) * ∂x_z(i, j, k, grid, free_surface) -@inline grid_slope_contribution_y(i, j, k, grid::ZStarSpacingGrid, free_surface, buoyancy, model_fields) = +@inline grid_slope_contribution_y(i, j, k, grid::ZStarUnderlyingGrid, free_surface, buoyancy, model_fields) = ℑyᵃᶠᵃ(i, j, k, grid, buoyancy_perturbationᶜᶜᶜ, buoyancy.model, model_fields) * ∂y_z(i, j, k, grid, free_surface) diff --git a/src/Operators/Operators.jl b/src/Operators/Operators.jl index 570d43dd6b..2e10c6126f 100644 --- a/src/Operators/Operators.jl +++ b/src/Operators/Operators.jl @@ -4,6 +4,7 @@ module Operators export Δxᶠᶠᶠ, Δxᶠᶠᶜ, Δxᶠᶜᶠ, Δxᶠᶜᶜ, Δxᶜᶠᶠ, Δxᶜᶠᶜ, Δxᶜᶜᶠ, Δxᶜᶜᶜ export Δyᶠᶠᶠ, Δyᶠᶠᶜ, Δyᶠᶜᶠ, Δyᶠᶜᶜ, Δyᶜᶠᶠ, Δyᶜᶠᶜ, Δyᶜᶜᶠ, Δyᶜᶜᶜ export Δzᶠᶠᶠ, Δzᶠᶠᶜ, Δzᶠᶜᶠ, Δzᶠᶜᶜ, Δzᶜᶠᶠ, Δzᶜᶠᶜ, Δzᶜᶜᶠ, Δzᶜᶜᶜ +export Δrᶠᶠᶠ, Δrᶠᶠᶜ, Δrᶠᶜᶠ, Δrᶠᶜᶜ, Δrᶜᶠᶠ, Δrᶜᶠᶜ, Δrᶜᶜᶠ, Δrᶜᶜᶜ # Areas export Axᶠᶠᶠ, Axᶠᶠᶜ, Axᶠᶜᶠ, Axᶠᶜᶜ, Axᶜᶠᶠ, Axᶜᶠᶜ, Axᶜᶜᶠ, Axᶜᶜᶜ diff --git a/src/Operators/spacings_and_areas_and_volumes.jl b/src/Operators/spacings_and_areas_and_volumes.jl index 5735444c63..5cb6a9fc70 100644 --- a/src/Operators/spacings_and_areas_and_volumes.jl +++ b/src/Operators/spacings_and_areas_and_volumes.jl @@ -1,4 +1,5 @@ using Oceananigans.Grids: Center, Face +using Oceananigans.Grids: AbstractVerticalCoordinateGrid const RG = RectilinearGrid const RGX = XRegularRG @@ -58,7 +59,7 @@ The operators in this file fall into three categories: @inline Δyᵃᶠᵃ(i, j, k, grid) = nothing @inline Δyᵃᶜᵃ(i, j, k, grid) = nothing -const ZRG = Union{LLGZ, RGZ} +const ZRG = Union{LLGZ, RGZ, OSSGZ} @inline Δzᵃᵃᶠ(i, j, k, grid) = @inbounds grid.Δzᵃᵃᶠ[k] @inline Δzᵃᵃᶜ(i, j, k, grid) = @inbounds grid.Δzᵃᵃᶜ[k] @@ -66,11 +67,15 @@ const ZRG = Union{LLGZ, RGZ} @inline Δzᵃᵃᶠ(i, j, k, grid::ZRG) = grid.Δzᵃᵃᶠ @inline Δzᵃᵃᶜ(i, j, k, grid::ZRG) = grid.Δzᵃᵃᶜ -@inline Δzᵃᵃᶜ(i, j, k, grid::OSSG) = @inbounds grid.Δzᵃᵃᶜ[k] -@inline Δzᵃᵃᶠ(i, j, k, grid::OSSG) = @inbounds grid.Δzᵃᵃᶠ[k] +# Reference spacing, they differ only for a ZStar vertical coordinate grid +@inline Δrᵃᵃᶜ(i, j, k, grid) = Δzᵃᵃᶜ(i, j, k, grid) +@inline Δrᵃᵃᶠ(i, j, k, grid) = Δzᵃᵃᶠ(i, j, k, grid) -@inline Δzᵃᵃᶜ(i, j, k, grid::OSSGZ) = grid.Δzᵃᵃᶜ -@inline Δzᵃᵃᶠ(i, j, k, grid::OSSGZ) = grid.Δzᵃᵃᶠ +@inline getspacing(k, Δz::AbstractVector) = @inbounds Δz[k] +@inline getspacing(k, Δz::Number) = @inbounds Δz + +@inline Δrᵃᵃᶜ(i, j, k, grid::ZRG) = getspacing(k, grid.Δzᵃᵃᶜ.reference) +@inline Δrᵃᵃᶠ(i, j, k, grid::ZRG) = getspacing(k, grid.Δzᵃᵃᶠ.reference) # Convenience Functions for all grids for LX in (:ᶜ, :ᶠ), LY in (:ᶜ, :ᶠ) @@ -93,14 +98,35 @@ for LX in (:ᶜ, :ᶠ), LY in (:ᶜ, :ᶠ) z_spacing_1D = Symbol(:Δz, :ᵃ, :ᵃ, LZ) z_spacing_3D = Symbol(:Δz, LX, LY, LZ) + r_spacing_1D = Symbol(:Δr, :ᵃ, :ᵃ, LZ) + r_spacing_3D = Symbol(:Δr, LX, LY, LZ) + @eval begin @inline $x_spacing_3D(i, j, k, grid) = $x_spacing_2D(i, j, k, grid) @inline $y_spacing_3D(i, j, k, grid) = $y_spacing_2D(i, j, k, grid) @inline $z_spacing_3D(i, j, k, grid) = $z_spacing_1D(i, j, k, grid) + @inline $r_spacing_3D(i, j, k, grid) = $r_spacing_1D(i, j, k, grid) end end end +##### +##### 3D spacings for AbstractVerticalCoordinate grids +##### + +const AVCG = AbstractVerticalCoordinateGrid +const c = Center() +const f = Face() + +@inline Δzᶜᶜᶠ(i, j, k, grid::AVCG) = @inbounds Δrᶜᶜᶠ(i, j, k, grid) * vertical_scaling(i, j, k, grid, c, c, f) +@inline Δzᶜᶜᶜ(i, j, k, grid::AVCG) = @inbounds Δrᶜᶜᶜ(i, j, k, grid) * vertical_scaling(i, j, k, grid, c, c, c) +@inline Δzᶠᶜᶠ(i, j, k, grid::AVCG) = @inbounds Δrᶠᶜᶠ(i, j, k, grid) * vertical_scaling(i, j, k, grid, f, c, f) +@inline Δzᶠᶜᶜ(i, j, k, grid::AVCG) = @inbounds Δrᶠᶜᶜ(i, j, k, grid) * vertical_scaling(i, j, k, grid, f, c, c) +@inline Δzᶜᶠᶠ(i, j, k, grid::AVCG) = @inbounds Δrᶜᶠᶠ(i, j, k, grid) * vertical_scaling(i, j, k, grid, c, f, f) +@inline Δzᶜᶠᶜ(i, j, k, grid::AVCG) = @inbounds Δrᶜᶠᶜ(i, j, k, grid) * vertical_scaling(i, j, k, grid, c, f, c) +@inline Δzᶠᶠᶠ(i, j, k, grid::AVCG) = @inbounds Δrᶠᶠᶠ(i, j, k, grid) * vertical_scaling(i, j, k, grid, f, f, f) +@inline Δzᶠᶠᶜ(i, j, k, grid::AVCG) = @inbounds Δrᶠᶠᶜ(i, j, k, grid) * vertical_scaling(i, j, k, grid, f, f, c) + ##### ##### Rectilinear Grids (Flat grids already have Δ = 1) ##### From ac7c96d02bf8e159d4080153d33ced04866db2d3 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 10 Oct 2024 09:49:43 +0200 Subject: [PATCH 349/567] name changes --- docs/src/grids.md | 2 +- examples/internal_tide.jl | 2 +- src/Advection/adapt_advection_order.jl | 98 +++++++++++++++++++ src/BoundaryConditions/apply_flux_bcs.jl | 6 +- src/ImmersedBoundaries/ImmersedBoundaries.jl | 4 +- .../distributed_immersed_boundaries.jl | 10 +- src/ImmersedBoundaries/grid_fitted_bottom.jl | 56 +++++------ src/ImmersedBoundaries/partial_cell_bottom.jl | 46 ++++----- src/MultiRegion/multi_region_grid.jl | 4 +- src/TurbulenceClosures/TurbulenceClosures.jl | 8 +- .../immersed_couette_flow.jl | 4 +- ...multi_region_near_global_quarter_degree.jl | 2 +- .../heterogeneous_windy_convection.jl | 4 +- 13 files changed, 172 insertions(+), 74 deletions(-) create mode 100644 src/Advection/adapt_advection_order.jl diff --git a/docs/src/grids.md b/docs/src/grids.md index 5e25849b05..a6686e5767 100644 --- a/docs/src/grids.md +++ b/docs/src/grids.md @@ -207,7 +207,7 @@ mountain_grid = ImmersedBoundaryGrid(grid, GridFittedBottom(mountain)) ```@example grids using CairoMakie -h = mountain_grid.immersed_boundary.z_bottom +h = mountain_grid.immersed_boundary.bottom_height fig = Figure(size=(600, 600)) ax = Axis(fig[2, 1], xlabel="x (m)", ylabel="y (m)", aspect=1) diff --git a/examples/internal_tide.jl b/examples/internal_tide.jl index a08dedd65a..5b31f18418 100644 --- a/examples/internal_tide.jl +++ b/examples/internal_tide.jl @@ -52,7 +52,7 @@ grid = ImmersedBoundaryGrid(underlying_grid, PartialCellBottom(bottom)) # Let's see how the domain with the bathymetry is. x = xnodes(grid, Center()) -bottom_boundary = interior(grid.immersed_boundary.z_bottom, :, 1, 1) +bottom_boundary = interior(grid.immersed_boundary.bottom_height, :, 1, 1) top_boundary = 0 * x using CairoMakie diff --git a/src/Advection/adapt_advection_order.jl b/src/Advection/adapt_advection_order.jl new file mode 100644 index 0000000000..41f3f60c9a --- /dev/null +++ b/src/Advection/adapt_advection_order.jl @@ -0,0 +1,98 @@ +using Oceananigans.Grids: topology + +""" + adapt_advection_order(advection, grid::AbstractGrid) + +Adapts the advection operator `advection` based on the grid `grid` by adjusting the order of advection in each direction. +For example, if the grid has only one point in the x-direction, the advection operator in the x-direction is set to first order +upwind or 2nd order centered scheme, depending on the original user-specified advection scheme. A high order advection sheme +is reduced to a lower order advection scheme if the grid has fewer points in that direction. + +# Arguments +- `advection`: The original advection scheme. +- `grid::AbstractGrid`: The grid on which the advection scheme is applied. + +If the order of advection is changed in at least one direction, the adapted advection scheme with adjusted advection order returned +by this function is a `FluxFormAdvection`. +""" +function adapt_advection_order(advection, grid::AbstractGrid) + advection_x = x_advection(advection) + advection_y = y_advection(advection) + advection_z = z_advection(advection) + + new_advection_x = adapt_advection_order(advection_x, size(grid, 1), grid) + new_advection_y = adapt_advection_order(advection_y, size(grid, 2), grid) + new_advection_z = adapt_advection_order(advection_z, size(grid, 3), grid) + + # Check that we indeed changed the advection operator + changed_x = new_advection_x != advection_x + changed_y = new_advection_y != advection_y + changed_z = new_advection_z != advection_z + + new_advection = FluxFormAdvection(new_advection_x, new_advection_y, new_advection_z) + changed_advection = any((changed_x, changed_y, changed_z)) + + if changed_x + @info "Using the advection scheme $(summary(new_advection.x)) in the x-direction because size(grid, 1) = $(size(grid, 1))" + end + if changed_y + @info "Using the advection scheme $(summary(new_advection.y)) in the y-direction because size(grid, 2) = $(size(grid, 2))" + end + if changed_z + @info "Using the advection scheme $(summary(new_advection.z)) in the x-direction because size(grid, 3) = $(size(grid, 3))" + end + + return ifelse(changed_advection, new_advection, advection) +end + + +x_advection(flux_form::FluxFormAdvection) = flux_form.x +y_advection(flux_form::FluxFormAdvection) = flux_form.y +z_advection(flux_form::FluxFormAdvection) = flux_form.z + +x_advection(advection) = advection +y_advection(advection) = advection +z_advection(advection) = advection + +# For the moment, we do not adapt the advection order for the VectorInvariant advection scheme +adapt_advection_order(advection::VectorInvariant, grid::AbstractGrid) = advection +adapt_advection_order(advection::Nothing, grid::AbstractGrid) = nothing +adapt_advection_order(advection::Nothing, N::Int, grid::AbstractGrid) = nothing + +##### +##### Directional adapt advection order +##### + +function adapt_advection_order(advection::Centered{B}, N::Int, grid::AbstractGrid) where B + if N >= B + return advection + else + return Centered(; order = 2N) + end +end + +function adapt_advection_order(advection::UpwindBiased{B}, N::Int, grid::AbstractGrid) where B + if N >= B + return advection + else + return UpwindBiased(; order = 2N - 1) + end +end + +""" + new_weno_scheme(grid, order, bounds, XT, YT, ZT) + +Constructs a new WENO scheme based on the given parameters. `XT`, `YT`, and `ZT` is the type of the precomputed weno coefficients in the +x-direction, y-direction and z-direction. A _non-stretched_ WENO scheme has `T` equal to `Nothing` everywhere. In case of a non-stretched WENO scheme, +we rebuild the advection without passing the grid information, otherwise we use the grid to account for stretched directions. +""" +new_weno_scheme(::WENO, grid, order, bounds, ::Type{Nothing}, ::Type{Nothing}, ::Type{Nothing},) = WENO(; order, bounds) +new_weno_scheme(::WENO, grid, order, bounds, XT, YT, ZT) = WENO(grid; order, bounds) + +function adapt_advection_order(advection::WENO{B, FT, XT, YT, ZT}, N::Int, grid::AbstractGrid) where {B, FT, XT, YT, ZT} + if N >= B + return advection + else + return new_weno_scheme(advection, grid, 2N - 1, advection.bounds, XT, YT, ZT) + end +end \ No newline at end of file diff --git a/src/BoundaryConditions/apply_flux_bcs.jl b/src/BoundaryConditions/apply_flux_bcs.jl index c1866fd84e..1ed09e8b43 100644 --- a/src/BoundaryConditions/apply_flux_bcs.jl +++ b/src/BoundaryConditions/apply_flux_bcs.jl @@ -78,7 +78,7 @@ Apply a top and/or bottom boundary condition to variable `c`. """ @kernel function _apply_z_bcs!(Gc, loc, grid, bottom_bc, top_bc, args) i, j = @index(Global, NTuple) - apply_z_bottom_bc!(Gc, loc, bottom_bc, i, j, grid, args...) + apply_bottom_height_bc!(Gc, loc, bottom_bc, i, j, grid, args...) apply_z_top_bc!(Gc, loc, top_bc, i, j, grid, args...) end @@ -88,7 +88,7 @@ end @inline apply_y_north_bc!(Gc, loc, ::NotFluxBC, args...) = nothing @inline apply_y_south_bc!(Gc, loc, ::NotFluxBC, args...) = nothing @inline apply_z_top_bc!(Gc, loc, ::NotFluxBC, args...) = nothing -@inline apply_z_bottom_bc!(Gc, loc, ::NotFluxBC, args...) = nothing +@inline apply_bottom_height_bc!(Gc, loc, ::NotFluxBC, args...) = nothing @inline flip(::Center) = Face() @inline flip(::Face) = Center() @@ -120,7 +120,7 @@ end return nothing end -@inline function apply_z_bottom_bc!(Gc, loc, bottom_flux::BC{<:Flux}, i, j, grid, args...) +@inline function apply_bottom_height_bc!(Gc, loc, bottom_flux::BC{<:Flux}, i, j, grid, args...) LX, LY, LZ = loc @inbounds Gc[i, j, 1] += getbc(bottom_flux, i, j, grid, args...) * Az(i, j, 1, grid, LX, LY, flip(LZ)) / volume(i, j, 1, grid, LX, LY, LZ) return nothing diff --git a/src/ImmersedBoundaries/ImmersedBoundaries.jl b/src/ImmersedBoundaries/ImmersedBoundaries.jl index 4a011d1beb..869b44dd8b 100644 --- a/src/ImmersedBoundaries/ImmersedBoundaries.jl +++ b/src/ImmersedBoundaries/ImmersedBoundaries.jl @@ -94,7 +94,7 @@ import Oceananigans.TurbulenceClosures: νᶠᶠᶜ, νᶜᶠᶠ, νᶠᶜᶠ, - z_bottom + bottom_height import Oceananigans.Fields: fractional_x_index, fractional_y_index, fractional_z_index @@ -162,7 +162,7 @@ with_halo(halo, ibg::ImmersedBoundaryGrid) = inflate_halo_size_one_dimension(req_H, old_H, _, ::IBG) = max(req_H + 1, old_H) inflate_halo_size_one_dimension(req_H, old_H, ::Type{Flat}, ::IBG) = 0 -@inline z_bottom(i, j, ibg::IBG) = error("The function `bottom` has not been defined for $(summary(ibg))!") +@inline bottom_height(i, j, ibg::IBG) = error("The function `bottom` has not been defined for $(summary(ibg))!") function Base.summary(grid::ImmersedBoundaryGrid) FT = eltype(grid) diff --git a/src/ImmersedBoundaries/distributed_immersed_boundaries.jl b/src/ImmersedBoundaries/distributed_immersed_boundaries.jl index 10473ba015..4b19202b92 100644 --- a/src/ImmersedBoundaries/distributed_immersed_boundaries.jl +++ b/src/ImmersedBoundaries/distributed_immersed_boundaries.jl @@ -14,7 +14,7 @@ function reconstruct_global_grid(grid::ImmersedBoundaryGrid) arch = grid.architecture local_ib = grid.immersed_boundary global_ug = reconstruct_global_grid(grid.underlying_grid) - global_ib = getnamewrapper(local_ib)(construct_global_array(arch, local_ib.z_bottom, size(grid))) + global_ib = getnamewrapper(local_ib)(construct_global_array(arch, local_ib.bottom_height, size(grid))) return ImmersedBoundaryGrid(global_ug, global_ib) end @@ -33,9 +33,9 @@ function scatter_local_grids(global_grid::ImmersedBoundaryGrid, arch::Distribute local_ug = scatter_local_grids(ug, arch, local_size) # Kinda hacky - local_z_bottom = partition(ib.z_bottom, arch, local_size) + local_bottom_height = partition(ib.bottom_height, arch, local_size) ImmersedBoundaryConstructor = getnamewrapper(ib) - local_ib = ImmersedBoundaryConstructor(local_z_bottom) + local_ib = ImmersedBoundaryConstructor(local_bottom_height) return ImmersedBoundaryGrid(local_ug, local_ib) end @@ -76,10 +76,10 @@ function resize_immersed_boundary(ib::AbstractGridFittedBottom{<:OffsetArray}, g # Check that the size of a bottom field are # consistent with the size of the grid - if any(size(ib.z_bottom) .!= bottom_heigth_size) + if any(size(ib.bottom_height) .!= bottom_heigth_size) @warn "Resizing the bottom field to match the grids' halos" bottom_field = Field((Center, Center, Nothing), grid) - cpu_bottom = on_architecture(CPU(), ib.z_bottom)[1:Nx, 1:Ny] + cpu_bottom = on_architecture(CPU(), ib.bottom_height)[1:Nx, 1:Ny] set!(bottom_field, cpu_bottom) fill_halo_regions!(bottom_field) offset_bottom_array = dropdims(bottom_field.data, dims=3) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 1a2cc89e57..b3ac5f35fc 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -7,7 +7,7 @@ using Oceananigans.Fields: fill_halo_regions! using Oceananigans.BoundaryConditions: FBC using Printf -import Oceananigans.TurbulenceClosures: z_bottom +import Oceananigans.TurbulenceClosures: bottom_height ##### ##### GridFittedBottom (2.5D immersed boundary with modified bottom height) @@ -21,12 +21,12 @@ struct InterfaceImmersedCondition end struct CenterImmersedCondition end struct GridFittedBottom{H, I} <: AbstractGridFittedBottom{H} - z_bottom :: H + bottom_height :: H immersed_condition :: I end -GridFittedBottom(z_bottom) = GridFittedBottom(z_bottom, CenterImmersedCondition()) +GridFittedBottom(bottom_height) = GridFittedBottom(bottom_height, CenterImmersedCondition()) Base.summary(::CenterImmersedCondition) = "CenterImmersedCondition" Base.summary(::InterfaceImmersedCondition) = "InterfaceImmersedCondition" @@ -34,26 +34,26 @@ Base.summary(::InterfaceImmersedCondition) = "InterfaceImmersedCondition" const GFBIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:GridFittedBottom} """ - GridFittedBottom(z_bottom) + GridFittedBottom(bottom_height) Return a bottom immersed boundary. Keyword Arguments ================= -* `z_bottom`: an array or function that gives the height of the +* `bottom_height`: an array or function that gives the height of the bottom in absolute ``z`` coordinates. * `immersed_condition`: Determine whether the part of the domain that is immersed are all the cell centers that lie below - `z_bottom` (`CenterImmersedCondition()`; default) or all the cell faces that lie below `bottom_height` (`InterfaceImmersedCondition()`). + `bottom_height` (`CenterImmersedCondition()`; default) or all the cell faces that lie below `bottom_height` (`InterfaceImmersedCondition()`). The only purpose of `immersed_condition` to allow `GridFittedBottom` and `PartialCellBottom` to have the same behavior when the minimum fractional cell height for partial cells is set to 0. """ function Base.summary(ib::GridFittedBottom) - zmax = maximum(ib.z_bottom) - zmin = minimum(ib.z_bottom) - zmean = mean(ib.z_bottom) + zmax = maximum(ib.bottom_height) + zmin = minimum(ib.bottom_height) + zmean = mean(ib.bottom_height) summary1 = "GridFittedBottom(" @@ -66,48 +66,48 @@ function Base.summary(ib::GridFittedBottom) return summary1 * summary2 * summary3 end -Base.summary(ib::GridFittedBottom{<:Function}) = @sprintf("GridFittedBottom(%s)", ib.z_bottom) +Base.summary(ib::GridFittedBottom{<:Function}) = @sprintf("GridFittedBottom(%s)", ib.bottom_height) function Base.show(io::IO, ib::GridFittedBottom) print(io, summary(ib), '\n') - print(io, "├── z_bottom: ", prettysummary(ib.z_bottom), '\n') + print(io, "├── bottom_height: ", prettysummary(ib.bottom_height), '\n') end -on_architecture(arch, ib::GridFittedBottom) = GridFittedBottom(on_architecture(ib.z_bottom), ib.immersed_condition) +on_architecture(arch, ib::GridFittedBottom) = GridFittedBottom(on_architecture(ib.bottom_height), ib.immersed_condition) function on_architecture(arch, ib::GridFittedBottom{<:Field}) - architecture(ib.z_bottom) == arch && return ib - arch_grid = on_architecture(arch, ib.z_bottom.grid) - new_z_bottom = Field{Center, Center, Nothing}(arch_grid) - set!(new_z_bottom, ib.z_bottom) - fill_halo_regions!(new_z_bottom) - return GridFittedBottom(new_z_bottom, ib.immersed_condition) + architecture(ib.bottom_height) == arch && return ib + arch_grid = on_architecture(arch, ib.bottom_height.grid) + new_bottom_height = Field{Center, Center, Nothing}(arch_grid) + set!(new_bottom_height, ib.bottom_height) + fill_halo_regions!(new_bottom_height) + return GridFittedBottom(new_bottom_height, ib.immersed_condition) end -Adapt.adapt_structure(to, ib::GridFittedBottom) = GridFittedBottom(adapt(to, ib.z_bottom), ib.immersed_condition) +Adapt.adapt_structure(to, ib::GridFittedBottom) = GridFittedBottom(adapt(to, ib.bottom_height), ib.immersed_condition) """ ImmersedBoundaryGrid(grid, ib::GridFittedBottom) Return a grid with `GridFittedBottom` immersed boundary (`ib`). -Computes `ib.z_bottom` and wraps it in a Field. `ib.z_bottom` is the z-coordinate of top-most interface +Computes `ib.bottom_height` and wraps it in a Field. `ib.bottom_height` is the z-coordinate of top-most interface of the last ``immersed`` cell in the column. """ function ImmersedBoundaryGrid(grid, ib::GridFittedBottom) bottom_field = Field{Center, Center, Nothing}(grid) - set!(bottom_field, ib.z_bottom) - @apply_regionally correct_z_bottom!(bottom_field, grid, ib) + set!(bottom_field, ib.bottom_height) + @apply_regionally correct_bottom_height!(bottom_field, grid, ib) fill_halo_regions!(bottom_field) new_ib = GridFittedBottom(bottom_field) TX, TY, TZ = topology(grid) return ImmersedBoundaryGrid{TX, TY, TZ}(grid, new_ib) end -correct_z_bottom!(bottom_field, grid, ib) = - launch!(architecture(grid), grid, :xy, _correct_z_bottom!, bottom_field, grid, ib) +correct_bottom_height!(bottom_field, grid, ib) = + launch!(architecture(grid), grid, :xy, _correct_bottom_height!, bottom_field, grid, ib) -@kernel function _correct_z_bottom!(bottom_field, grid, ib::GridFittedBottom) +@kernel function _correct_bottom_height!(bottom_field, grid, ib::GridFittedBottom) i, j = @index(Global, NTuple) zb = @inbounds bottom_field[i, j, 1] condition = ib.immersed_condition @@ -121,11 +121,11 @@ end @inline function _immersed_cell(i, j, k, underlying_grid, ib::GridFittedBottom) z = znode(i, j, k, underlying_grid, c, c, c) - zb = @inbounds ib.z_bottom[i, j, 1] + zb = @inbounds ib.bottom_height[i, j, 1] return z ≤ zb end -@inline z_bottom(i, j, ibg::GFBIBG) = @inbounds ibg.immersed_boundary.z_bottom[i, j, 1] +@inline bottom_height(i, j, ibg::GFBIBG) = @inbounds ibg.immersed_boundary.bottom_height[i, j, 1] ##### ##### Bottom height @@ -133,7 +133,7 @@ end const AGFBIB = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} -@inline column_heightᶜᶜᵃ(i, j, k, ibg::AGFBIB) = @inbounds znode(i, j, ibg.Nz+1, ibg, c, c, f) - ibg.immersed_boundary.z_bottom[i, j, 1] +@inline column_heightᶜᶜᵃ(i, j, k, ibg::AGFBIB) = @inbounds znode(i, j, ibg.Nz+1, ibg, c, c, f) - ibg.immersed_boundary.bottom_height[i, j, 1] @inline column_heightᶜᶠᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶜᶜᵃ(i, j-1, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) @inline column_heightᶠᶜᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶜᶜᵃ(i-1, j, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) @inline column_heightᶠᶠᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶠᶜᵃ(i, j-1, k, ibg), column_heightᶠᶜᵃ(i, j, k, ibg)) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index 2e6e9ba50b..7988fa9ec9 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -7,16 +7,16 @@ using Printf ##### struct PartialCellBottom{H, E} <: AbstractGridFittedBottom{H} - z_bottom :: H + bottom_height :: H minimum_fractional_cell_height :: E end const PCBIBG{FT, TX, TY, TZ} = ImmersedBoundaryGrid{FT, TX, TY, TZ, <:Any, <:PartialCellBottom} where {FT, TX, TY, TZ} function Base.summary(ib::PartialCellBottom) - zmax = maximum(parent(ib.z_bottom)) - zmin = minimum(parent(ib.z_bottom)) - zmean = mean(parent(ib.z_bottom)) + zmax = maximum(parent(ib.bottom_height)) + zmin = minimum(parent(ib.bottom_height)) + zmean = mean(parent(ib.bottom_height)) summary1 = "PartialCellBottom(" @@ -31,21 +31,21 @@ function Base.summary(ib::PartialCellBottom) end Base.summary(ib::PartialCellBottom{<:Function}) = @sprintf("PartialCellBottom(%s, ϵ=%.1f)", - prettysummary(ib.z_bottom, false), + prettysummary(ib.bottom_height, false), prettysummary(ib.minimum_fractional_cell_height)) function Base.show(io::IO, ib::PartialCellBottom) print(io, summary(ib), '\n') - print(io, "├── z_bottom: ", prettysummary(ib.z_bottom), '\n') + print(io, "├── bottom_height: ", prettysummary(ib.bottom_height), '\n') print(io, "└── minimum_fractional_cell_height: ", prettysummary(ib.minimum_fractional_cell_height)) end """ - PartialCellBottom(z_bottom; minimum_fractional_cell_height=0.2) + PartialCellBottom(bottom_height; minimum_fractional_cell_height=0.2) Return `PartialCellBottom` representing an immersed boundary with "partial" bottom cells. That is, the height of the bottommost cell in each column is reduced -to fit the provided `z_bottom`, which may be a `Field`, `Array`, or function +to fit the provided `bottom_height`, which may be a `Field`, `Array`, or function of `(x, y)`. The height of partial bottom cells is greater than @@ -56,21 +56,21 @@ minimum_fractional_cell_height * Δz, where `Δz` is the original height of the bottom cell underlying grid. """ -function PartialCellBottom(z_bottom; minimum_fractional_cell_height=0.2) - return PartialCellBottom(z_bottom, minimum_fractional_cell_height) +function PartialCellBottom(bottom_height; minimum_fractional_cell_height=0.2) + return PartialCellBottom(bottom_height, minimum_fractional_cell_height) end function ImmersedBoundaryGrid(grid, ib::PartialCellBottom) bottom_field = Field{Center, Center, Nothing}(grid) - set!(bottom_field, ib.z_bottom) - @apply_regionally correct_z_bottom!(bottom_field, grid, ib) + set!(bottom_field, ib.bottom_height) + @apply_regionally correct_bottom_height!(bottom_field, grid, ib) fill_halo_regions!(bottom_field) new_ib = PartialCellBottom(bottom_field, ib.minimum_fractional_cell_height) TX, TY, TZ = topology(grid) return ImmersedBoundaryGrid{TX, TY, TZ}(grid, new_ib) end -@kernel function _correct_z_bottom!(bottom_field, grid, ib::PartialCellBottom) +@kernel function _correct_bottom_height!(bottom_field, grid, ib::PartialCellBottom) i, j = @index(Global, NTuple) zb = @inbounds bottom_field[i, j, 1] ϵ = ib.minimum_fractional_cell_height @@ -83,17 +83,17 @@ end end function on_architecture(arch, ib::PartialCellBottom{<:Field}) - architecture(ib.z_bottom) == arch && return ib - arch_grid = on_architecture(arch, ib.z_bottom.grid) - new_z_bottom = Field{Center, Center, Nothing}(arch_grid) - copyto!(parent(new_z_bottom), parent(ib.z_bottom)) - return PartialCellBottom(new_z_bottom, ib.minimum_fractional_cell_height) + architecture(ib.bottom_height) == arch && return ib + arch_grid = on_architecture(arch, ib.bottom_height.grid) + new_bottom_height = Field{Center, Center, Nothing}(arch_grid) + copyto!(parent(new_bottom_height), parent(ib.bottom_height)) + return PartialCellBottom(new_bottom_height, ib.minimum_fractional_cell_height) end -Adapt.adapt_structure(to, ib::PartialCellBottom) = PartialCellBottom(adapt(to, ib.z_bottom), +Adapt.adapt_structure(to, ib::PartialCellBottom) = PartialCellBottom(adapt(to, ib.bottom_height), ib.minimum_fractional_cell_height) -on_architecture(to, ib::PartialCellBottom) = PartialCellBottom(on_architecture(to, ib.z_bottom), +on_architecture(to, ib::PartialCellBottom) = PartialCellBottom(on_architecture(to, ib.bottom_height), on_architecture(to, ib.minimum_fractional_cell_height)) """ @@ -119,7 +119,7 @@ Criterion is zb ≥ z - ϵ Δz """ @inline function _immersed_cell(i, j, k, underlying_grid, ib::PartialCellBottom) z = znode(i, j, k+1, underlying_grid, c, c, f) - zb = @inbounds ib.z_bottom[i, j, 1] + zb = @inbounds ib.bottom_height[i, j, 1] return z ≤ zb end @@ -138,7 +138,7 @@ end z = znode(i, j, k+1, underlying_grid, c, c, f) # Get bottom z-coordinate and fractional Δz parameter - zb = @inbounds ib.z_bottom[i, j, 1] + zb = @inbounds ib.bottom_height[i, j, 1] # Are we in a bottom cell? at_the_bottom = bottom_cell(i, j, k, ibg) @@ -183,5 +183,5 @@ YFlatPCBIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Flat, <:Any, <:Any, <:Partial @inline Δzᶠᶠᶜ(i, j, k, ibg::XFlatPCBIBG) = Δzᶜᶠᶜ(i, j, k, ibg) @inline Δzᶠᶠᶜ(i, j, k, ibg::YFlatPCBIBG) = Δzᶠᶜᶜ(i, j, k, ibg) -@inline z_bottom(i, j, ibg::PCBIBG) = @inbounds ibg.immersed_boundary.z_bottom[i, j, 1] +@inline bottom_height(i, j, ibg::PCBIBG) = @inbounds ibg.immersed_boundary.bottom_height[i, j, 1] diff --git a/src/MultiRegion/multi_region_grid.jl b/src/MultiRegion/multi_region_grid.jl index 7d28ee6100..f338750b24 100644 --- a/src/MultiRegion/multi_region_grid.jl +++ b/src/MultiRegion/multi_region_grid.jl @@ -188,8 +188,8 @@ function reconstruct_global_grid(mrg::ImmersedMultiRegionGrid) return ImmersedBoundaryGrid(global_grid, global_boundary) end -reconstruct_global_boundary(g::GridFittedBottom{<:Field}) = GridFittedBottom(reconstruct_global_field(g.z_bottom)) -reconstruct_global_boundary(g::PartialCellBottom{<:Field}) = PartialCellBottom(reconstruct_global_field(g.z_bottom), g.minimum_fractional_cell_height) +reconstruct_global_boundary(g::GridFittedBottom{<:Field}) = GridFittedBottom(reconstruct_global_field(g.bottom_height)) +reconstruct_global_boundary(g::PartialCellBottom{<:Field}) = PartialCellBottom(reconstruct_global_field(g.bottom_height), g.minimum_fractional_cell_height) reconstruct_global_boundary(g::GridFittedBoundary{<:Field}) = GridFittedBoundary(reconstruct_global_field(g.mask)) @inline getregion(mrg::ImmersedMultiRegionGrid{FT, TX, TY, TZ}, r) where {FT, TX, TY, TZ} = ImmersedBoundaryGrid{TX, TY, TZ}(_getregion(mrg.underlying_grid, r), _getregion(mrg.immersed_boundary, r)) diff --git a/src/TurbulenceClosures/TurbulenceClosures.jl b/src/TurbulenceClosures/TurbulenceClosures.jl index f810ce5e2f..7046a2f1cc 100644 --- a/src/TurbulenceClosures/TurbulenceClosures.jl +++ b/src/TurbulenceClosures/TurbulenceClosures.jl @@ -120,14 +120,14 @@ const c = Center() const f = Face() @inline z_top(i, j, grid) = znode(i, j, grid.Nz+1, grid, c, c, f) -@inline z_bottom(i, j, grid) = znode(i, j, 1, grid, c, c, f) +@inline bottom_height(i, j, grid) = znode(i, j, 1, grid, c, c, f) @inline depthᶜᶜᶠ(i, j, k, grid) = clip(z_top(i, j, grid) - znode(i, j, k, grid, c, c, f)) @inline depthᶜᶜᶜ(i, j, k, grid) = clip(z_top(i, j, grid) - znode(i, j, k, grid, c, c, c)) -@inline total_depthᶜᶜᵃ(i, j, grid) = clip(z_top(i, j, grid) - z_bottom(i, j, grid)) +@inline total_depthᶜᶜᵃ(i, j, grid) = clip(z_top(i, j, grid) - bottom_height(i, j, grid)) @inline function height_above_bottomᶜᶜᶠ(i, j, k, grid) - h = znode(i, j, k, grid, c, c, f) - z_bottom(i, j, grid) + h = znode(i, j, k, grid, c, c, f) - bottom_height(i, j, grid) # Limit by thickness of cell below Δz = Δzᶜᶜᶜ(i, j, k-1, grid) @@ -136,7 +136,7 @@ end @inline function height_above_bottomᶜᶜᶜ(i, j, k, grid) Δz = Δzᶜᶜᶜ(i, j, k, grid) - h = znode(i, j, k, grid, c, c, c) - z_bottom(i, j, grid) + h = znode(i, j, k, grid, c, c, c) - bottom_height(i, j, grid) return max(Δz/2, h) end diff --git a/validation/immersed_boundaries/immersed_couette_flow.jl b/validation/immersed_boundaries/immersed_couette_flow.jl index cd1671f6e7..90098d6ee4 100644 --- a/validation/immersed_boundaries/immersed_couette_flow.jl +++ b/validation/immersed_boundaries/immersed_couette_flow.jl @@ -10,8 +10,8 @@ Lz = 1 underlying_grid = RectilinearGrid(size=Nz, z=(0, Lz), topology = (Flat, Flat, Bounded)) -@inline z_bottom(x, y) = 0.1 -grid = ImmersedBoundaryGrid(underlying_grid, GridFittedBottom(z_bottom)) +@inline bottom_height(x, y) = 0.1 +grid = ImmersedBoundaryGrid(underlying_grid, GridFittedBottom(bottom_height)) c_immersed_bc = ValueBoundaryCondition(1) c_top_bc = ValueBoundaryCondition(-1) diff --git a/validation/multi_region/multi_region_near_global_quarter_degree.jl b/validation/multi_region/multi_region_near_global_quarter_degree.jl index 6bb671cb8c..ce62c65f82 100644 --- a/validation/multi_region/multi_region_near_global_quarter_degree.jl +++ b/validation/multi_region/multi_region_near_global_quarter_degree.jl @@ -147,7 +147,7 @@ biharmonic_viscosity = HorizontalScalarBiharmonicDiffusivity(ν=νhb, discrete @inline cyclic_interpolate(u₁::Number, u₂, time) = u₁ + mod(time / thirty_days, 1) * (u₂ - u₁) Δz_top = @allowscalar Δzᵃᵃᶜ(1, 1, grid.Nz, grid.underlying_grid) -Δz_bottom = @allowscalar Δzᵃᵃᶜ(1, 1, 1, grid.underlying_grid) +Δbottom_height = @allowscalar Δzᵃᵃᶜ(1, 1, 1, grid.underlying_grid) @inline function surface_wind_stress(i, j, grid, clock, fields, τ) time = clock.time diff --git a/validation/vertical_mixing_closures/heterogeneous_windy_convection.jl b/validation/vertical_mixing_closures/heterogeneous_windy_convection.jl index 6faf2950d0..ce55c42dee 100644 --- a/validation/vertical_mixing_closures/heterogeneous_windy_convection.jl +++ b/validation/vertical_mixing_closures/heterogeneous_windy_convection.jl @@ -34,8 +34,8 @@ grid = RectilinearGrid(size = (Nx, Ny, Nz), z = z, topology=(Periodic, Bounded, Bounded)) -z_bottom(x, y) = - Lz * (1 - (2y / Ly)^2) -grid = ImmersedBoundaryGrid(grid, PartialCellBottom(z_bottom, minimum_fractional_cell_height=0.1)) +bottom_height(x, y) = - Lz * (1 - (2y / Ly)^2) +grid = ImmersedBoundaryGrid(grid, PartialCellBottom(bottom_height, minimum_fractional_cell_height=0.1)) @show grid @inline Qᵇ(x, y, t) = 1e-7 From 605880601b1bf6a90c1a7fcfa4d37cb54cb4ab82 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 10 Oct 2024 12:46:23 +0200 Subject: [PATCH 350/567] massive change of interface --- src/Grids/grid_generation.jl | 4 + src/Grids/grid_utils.jl | 3 + src/Grids/latitude_longitude_grid.jl | 6 +- src/Grids/rectilinear_grid.jl | 6 +- src/Grids/z_star_vertical_coordinate.jl | 108 +++++++++++------- src/ImmersedBoundaries/ImmersedBoundaries.jl | 3 + .../abstract_zstar_immersed_grid.jl | 21 ++++ .../generalized_vertical_spacing.jl | 15 ++- .../hydrostatic_free_surface_model.jl | 7 -- ...te_hydrostatic_free_surface_model_state.jl | 4 +- .../z_star_vertical_spacing.jl | 13 ++- src/Oceananigans.jl | 1 + .../spacings_and_areas_and_volumes.jl | 10 +- src/OutputWriters/output_writer_utils.jl | 9 +- validation/z_star_coordinate/lock_release.jl | 31 ++--- 15 files changed, 143 insertions(+), 98 deletions(-) create mode 100644 src/ImmersedBoundaries/abstract_zstar_immersed_grid.jl diff --git a/src/Grids/grid_generation.jl b/src/Grids/grid_generation.jl index b0ba1ad7dc..ccc5d68498 100644 --- a/src/Grids/grid_generation.jl +++ b/src/Grids/grid_generation.jl @@ -26,6 +26,10 @@ total_interior_length(::BoundedTopology, N) = N + 1 bad_coordinate_message(ξ::Function, name) = "The values of $name(index) must increase as the index increases!" bad_coordinate_message(ξ::AbstractArray, name) = "The elements of $name must be increasing!" +# General generate_coordinate +generate_coordinate(FT, topology, size, halo, nodes, coordinate_name, dim::Int, arch) = + generate_coordinate(FT, topology[dim](), size[dim], halo[dim], nodes, coordinate_name, arch) + # generate a variably-spaced coordinate passing the explicit coord faces as vector or function function generate_coordinate(FT, topo::AT, N, H, node_generator, coordinate_name, arch) diff --git a/src/Grids/grid_utils.jl b/src/Grids/grid_utils.jl index c881ebeb86..5bd859384c 100644 --- a/src/Grids/grid_utils.jl +++ b/src/Grids/grid_utils.jl @@ -124,6 +124,9 @@ constant grid spacing `Δ`, and interior extent `L`. @inline domain(::Flat, N, ξ::Number) = ξ @inline domain(::Flat, N, ::Nothing) = nothing +# TODO: is this the correct definition? +@inline domain(topo, N, ξ::ZStarVerticalCoordinate) = domain(topo, N, ξ.reference) + @inline x_domain(grid) = domain(topology(grid, 1)(), grid.Nx, grid.xᶠᵃᵃ) @inline y_domain(grid) = domain(topology(grid, 2)(), grid.Ny, grid.yᵃᶠᵃ) @inline z_domain(grid) = domain(topology(grid, 3)(), grid.Nz, grid.zᵃᵃᶠ) diff --git a/src/Grids/latitude_longitude_grid.jl b/src/Grids/latitude_longitude_grid.jl index ebca324492..48d0da6ce0 100644 --- a/src/Grids/latitude_longitude_grid.jl +++ b/src/Grids/latitude_longitude_grid.jl @@ -197,9 +197,9 @@ function LatitudeLongitudeGrid(architecture::AbstractArchitecture = CPU(), # it is stretched if being passed is a function or vector (as for the VerticallyStretchedRectilinearGrid) TX, TY, TZ = topology - Lλ, λᶠᵃᵃ, λᶜᵃᵃ, Δλᶠᵃᵃ, Δλᶜᵃᵃ = generate_coordinate(FT, TX(), Nλ, Hλ, longitude, :longitude, architecture) - Lφ, φᵃᶠᵃ, φᵃᶜᵃ, Δφᵃᶠᵃ, Δφᵃᶜᵃ = generate_coordinate(FT, TY(), Nφ, Hφ, latitude, :latitude, architecture) - Lz, zᵃᵃᶠ, zᵃᵃᶜ, Δzᵃᵃᶠ, Δzᵃᵃᶜ = generate_coordinate(FT, TZ(), Nz, Hz, z, :z, architecture) + Lλ, λᶠᵃᵃ, λᶜᵃᵃ, Δλᶠᵃᵃ, Δλᶜᵃᵃ = generate_coordinate(FT, TX(), Nλ, Hλ, longitude, :longitude, 1, architecture) + Lφ, φᵃᶠᵃ, φᵃᶜᵃ, Δφᵃᶠᵃ, Δφᵃᶜᵃ = generate_coordinate(FT, TY(), Nφ, Hφ, latitude, :latitude, 2, architecture) + Lz, zᵃᵃᶠ, zᵃᵃᶜ, Δzᵃᵃᶠ, Δzᵃᵃᶜ = generate_coordinate(FT, TZ(), Nz, Hz, z, :z, 3, architecture) preliminary_grid = LatitudeLongitudeGrid{TX, TY, TZ}(architecture, Nλ, Nφ, Nz, diff --git a/src/Grids/rectilinear_grid.jl b/src/Grids/rectilinear_grid.jl index 57313d87a4..70f2407960 100644 --- a/src/Grids/rectilinear_grid.jl +++ b/src/Grids/rectilinear_grid.jl @@ -271,9 +271,9 @@ function RectilinearGrid(architecture::AbstractArchitecture = CPU(), Nx, Ny, Nz = size Hx, Hy, Hz = halo - Lx, xᶠᵃᵃ, xᶜᵃᵃ, Δxᶠᵃᵃ, Δxᶜᵃᵃ = generate_coordinate(FT, TX(), Nx, Hx, x, :x, architecture) - Ly, yᵃᶠᵃ, yᵃᶜᵃ, Δyᵃᶠᵃ, Δyᵃᶜᵃ = generate_coordinate(FT, TY(), Ny, Hy, y, :y, architecture) - Lz, zᵃᵃᶠ, zᵃᵃᶜ, Δzᵃᵃᶠ, Δzᵃᵃᶜ = generate_coordinate(FT, TZ(), Nz, Hz, z, :z, architecture) + Lx, xᶠᵃᵃ, xᶜᵃᵃ, Δxᶠᵃᵃ, Δxᶜᵃᵃ = generate_coordinate(FT, topology, size, halo, x, :x, 1, architecture) + Ly, yᵃᶠᵃ, yᵃᶜᵃ, Δyᵃᶠᵃ, Δyᵃᶜᵃ = generate_coordinate(FT, topology, size, halo, y, :y, 2, architecture) + Lz, zᵃᵃᶠ, zᵃᵃᶜ, Δzᵃᵃᶠ, Δzᵃᵃᶜ = generate_coordinate(FT, topology, size, halo, z, :z, 3, architecture) return RectilinearGrid{TX, TY, TZ}(architecture, Nx, Ny, Nz, diff --git a/src/Grids/z_star_vertical_coordinate.jl b/src/Grids/z_star_vertical_coordinate.jl index a2104de563..19a368d50e 100644 --- a/src/Grids/z_star_vertical_coordinate.jl +++ b/src/Grids/z_star_vertical_coordinate.jl @@ -12,11 +12,34 @@ const AVLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, < const AVOSSG = OrthogonalSphericalShellGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractVerticalCoordinate} const AVRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractVerticalCoordinate} -const AbstractVerticalCoordinateGrid = Union{AVLLG, AVOSSG, AVRG} +const AbstractVerticalCoordinateUnderlyingGrid = Union{AVLLG, AVOSSG, AVRG} # rnode for an AbstractVerticalCoordinate grid is the reference node -@inline rnode(i, j, k, grid::AbstractVerticalCoordinateGrid, ℓx, ℓy, ::Center) = @inbounds grid.zᵃᵃᶜ[k] -@inline rnode(i, j, k, grid::AbstractVerticalCoordinateGrid, ℓx, ℓy, ::Face) = @inbounds grid.zᵃᵃᶠ[k] +@inline rnode(i, j, k, grid::AbstractVerticalCoordinateUnderlyingGrid, ℓx, ℓy, ::Center) = @inbounds grid.zᵃᵃᶜ.reference[k] +@inline rnode(i, j, k, grid::AbstractVerticalCoordinateUnderlyingGrid, ℓx, ℓy, ::Face) = @inbounds grid.zᵃᵃᶠ.reference[k] + +function retrieve_static_grid(grid::AbstractVerticalCoordinateUnderlyingGrid) + + Δzᵃᵃᶠ = reference_zspacings(grid, Face()) + Δzᵃᵃᶜ = reference_zspacings(grid, Center()) + + TX, TY, TZ = topology(grid) + + args = [] + for prop in propertynames(grid) + if prop == :Δzᵃᵃᶠ + push!(args, Δzᵃᵃᶠ) + elseif prop == :Δzᵃᵃᶜ + push!(args, Δzᵃᵃᶜ) + else + push!(args, getproperty(grid, prop)) + end + end + + GridType = getnamewrapper(grid) + + return GridType{TX, TY, TZ}(args...) +end """ struct ZStarVerticalCoordinate{R, S} <: AbstractVerticalSpacing{R} @@ -80,7 +103,15 @@ generate_coordinate(FT, ::Periodic, N, H, ::ZStarVerticalCoordinate, coordinate_ throw(ArgumentError("Periodic domains are not supported for ZStarVerticalCoordinate")) # Generate a regularly-spaced coordinate passing the domain extent (2-tuple) and number of points -function generate_coordinate(FT, topo::Bounded, Nz, Hz, coordinate::ZStarVerticalCoordinate, coordinate_name, arch, Nx, Ny, Hx, Hy) +function generate_coordinate(FT, topo, size, halo, coordinate::ZStarVerticalCoordinate, coordinate_name, dim::Int, arch) + + Nx, Ny, Nz = size + Hx, Hy, Hz = halo + + if dim != 3 + msg = "ZStarVerticalCoordinate is supported only in the third dimension (z)" + throw(ArgumentError(msg)) + end if coordinate_name != :z msg = "Only z-coordinate is supported for ZStarVerticalCoordinate" @@ -93,28 +124,42 @@ function generate_coordinate(FT, topo::Bounded, Nz, Hz, coordinate::ZStarVertica args = (topo, (Nx, Ny, Nz), (Hx, Hy, Hz)) - sΔzᶜᶜᵃ = new_data(FT, arch, (Center, Center, Nothing), args...) - sΔzᶜᶠᵃ = new_data(FT, arch, (Center, Face, Nothing), args...) - sΔzᶠᶜᵃ = new_data(FT, arch, (Face, Center, Nothing), args...) - sΔzᶠᶠᵃ = new_data(FT, arch, (Face, Face, Nothing), args...) + sᶜᶜᵃ = new_data(FT, arch, (Center, Center, Nothing), args...) + sᶜᶠᵃ = new_data(FT, arch, (Center, Face, Nothing), args...) + sᶠᶜᵃ = new_data(FT, arch, (Face, Center, Nothing), args...) + sᶠᶠᵃ = new_data(FT, arch, (Face, Face, Nothing), args...) - sΔzᶜᶜᵃ₋ = new_data(FT, arch, (Center, Center, Nothing), args...) - sΔzᶜᶠᵃ₋ = new_data(FT, arch, (Center, Face, Nothing), args...) - sΔzᶠᶜᵃ₋ = new_data(FT, arch, (Face, Center, Nothing), args...) + sᶜᶜᵃ₋ = new_data(FT, arch, (Center, Center, Nothing), args...) + sᶜᶠᵃ₋ = new_data(FT, arch, (Center, Face, Nothing), args...) + sᶠᶜᵃ₋ = new_data(FT, arch, (Face, Center, Nothing), args...) ∂t_s = new_data(FT, arch, (Center, Center, Nothing), args...) - η = new_data(FT, arch, (Center, Center, Nothing), args...) # Storage place for the free surface height + # Storage place for the free surface height? Probably find a better way to call this + η = new_data(FT, arch, (Center, Center, Nothing), args...) - # The scaling is the same for everyone (H + \eta) / H - zᵃᵃᶠ = ZStarVerticalCoordinate(rᵃᵃᶠ, sΔzᶜᶜᵃ, sΔzᶠᶜᵃ, sΔzᶜᶠᵃ, sΔzᶠᶠᵃ, sΔzᶜᶜᵃ₋, sΔzᶠᶜᵃ₋, sΔzᶜᶠᵃ₋, η) - zᵃᵃᶜ = ZStarVerticalCoordinate(rᵃᵃᶜ, sΔzᶜᶜᵃ, sΔzᶠᶜᵃ, sΔzᶜᶠᵃ, sΔzᶠᶠᵃ, sΔzᶜᶜᵃ₋, sΔzᶠᶜᵃ₋, sΔzᶜᶠᵃ₋, η) + # fill all the scalings with 1 + for s in (sᶜᶜᵃ, sᶜᶠᵃ, sᶠᶜᵃ, sᶠᶠᵃ, sᶜᶜᵃ₋, sᶜᶠᵃ₋, sᶠᶜᵃ₋) + fill!(s, 1) + end + + # The scaling is the same for everyone (H + \eta) / H, the vertical coordinate requires + # to add the free surface to retrieve the znode. + zᵃᵃᶠ = ZStarVerticalCoordinate(rᵃᵃᶠ, sᶜᶜᵃ, sᶠᶜᵃ, sᶜᶠᵃ, sᶠᶠᵃ, sᶜᶜᵃ₋, sᶠᶜᵃ₋, sᶜᶠᵃ₋, η) + zᵃᵃᶜ = ZStarVerticalCoordinate(rᵃᵃᶜ, sᶜᶜᵃ, sᶠᶜᵃ, sᶜᶠᵃ, sᶠᶠᵃ, sᶜᶜᵃ₋, sᶠᶜᵃ₋, sᶜᶠᵃ₋, η) - Δzᵃᵃᶠ = ZStarVerticalCoordinate(Δrᵃᵃᶠ, sΔzᶜᶜᵃ, sΔzᶠᶜᵃ, sΔzᶜᶠᵃ, sΔzᶠᶠᵃ, sΔzᶜᶜᵃ₋, sΔzᶠᶜᵃ₋, sΔzᶜᶠᵃ₋, ∂t_s) - Δzᵃᵃᶜ = ZStarVerticalCoordinate(Δrᵃᵃᶜ, sΔzᶜᶜᵃ, sΔzᶠᶜᵃ, sΔzᶜᶠᵃ, sΔzᶠᶠᵃ, sΔzᶜᶜᵃ₋, sΔzᶠᶜᵃ₋, sΔzᶜᶠᵃ₋, ∂t_s) + Δzᵃᵃᶠ = ZStarVerticalCoordinate(Δrᵃᵃᶠ, sᶜᶜᵃ, sᶠᶜᵃ, sᶜᶠᵃ, sᶠᶠᵃ, sᶜᶜᵃ₋, sᶠᶜᵃ₋, sᶜᶠᵃ₋, ∂t_s) + Δzᵃᵃᶜ = ZStarVerticalCoordinate(Δrᵃᵃᶜ, sᶜᶜᵃ, sᶠᶜᵃ, sᶜᶠᵃ, sᶠᶠᵃ, sᶜᶜᵃ₋, sᶠᶜᵃ₋, sᶜᶠᵃ₋, ∂t_s) return Lr, zᵃᵃᶠ, zᵃᵃᶜ, Δzᵃᵃᶠ, Δzᵃᵃᶜ end +function validate_dimension_specification(T, ξ::ZStarVerticalCoordinate, dir, N, FT) + reference = validate_dimension_specification(T, ξ.reference, dir, N, FT) + args = Tuple(getproperty(ξ, prop) for prop in propertynames(ξ)) + + return ZStarVerticalCoordinate(reference, args[2:end]...) +end + const ZStarLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:ZStarVerticalCoordinate} const ZStarOSSG = OrthogonalSphericalShellGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:ZStarVerticalCoordinate} const ZStarRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:ZStarVerticalCoordinate} @@ -140,8 +185,8 @@ reference_zspacings(grid, ::F) = grid.Δzᵃᵃᶠ @inline ∂t_grid(i, j, k, grid) = zero(grid) @inline V_times_∂t_grid(i, j, k, grid) = zero(grid) -reference_zspacings(grid::ZSG, ::C) = grid.Δzᵃᵃᶜ.Δr -reference_zspacings(grid::ZSG, ::F) = grid.Δzᵃᵃᶠ.Δr +reference_zspacings(grid::ZSG, ::C) = grid.Δzᵃᵃᶜ.reference +reference_zspacings(grid::ZSG, ::F) = grid.Δzᵃᵃᶠ.reference @inline vertical_scaling(i, j, k, grid::ZSG, ::C, ::C, ::C) = @inbounds grid.Δzᵃᵃᶜ.sᶜᶜⁿ[i, j] @inline vertical_scaling(i, j, k, grid::ZSG, ::F, ::C, ::C) = @inbounds grid.Δzᵃᵃᶜ.sᶠᶜⁿ[i, j] @@ -179,27 +224,4 @@ const f = Face() @inline znode(i, j, k, grid::ZSG, ::C, ::C, ::F) = @inbounds grid.zᵃᵃᶠ.reference[k] * vertical_scaling(i, j, k, grid, c, c, c) + grid.zᵃᵃᶠ.∂t_s[i, j] @inline znode(i, j, k, grid::ZSG, ::C, ::F, ::F) = @inbounds grid.zᵃᵃᶠ.reference[k] * vertical_scaling(i, j, k, grid, c, f, c) + grid.zᵃᵃᶠ.∂t_s[i, j] @inline znode(i, j, k, grid::ZSG, ::F, ::C, ::F) = @inbounds grid.zᵃᵃᶠ.reference[k] * vertical_scaling(i, j, k, grid, f, c, c) + grid.zᵃᵃᶠ.∂t_s[i, j] -@inline znode(i, j, k, grid::ZSG, ::F, ::F, ::F) = @inbounds grid.zᵃᵃᶠ.reference[k] * vertical_scaling(i, j, k, grid, f, f, c) + grid.zᵃᵃᶠ.∂t_s[i, j] - -function retrieve_static_grid(grid::ZStarUnderlyingGrid) - - Δzᵃᵃᶠ = reference_zspacings(grid, Face()) - Δzᵃᵃᶜ = reference_zspacings(grid, Center()) - - TX, TY, TZ = topology(grid) - - args = [] - for prop in propertynames(grid) - if prop == :Δzᵃᵃᶠ - push!(args, Δzᵃᵃᶠ) - elseif prop == :Δzᵃᵃᶜ - push!(args, Δzᵃᵃᶜ) - else - push!(args, getproperty(grid, prop)) - end - end - - GridType = getnamewrapper(grid) - - return GridType{TX, TY, TZ}(args...) -end \ No newline at end of file +@inline znode(i, j, k, grid::ZSG, ::F, ::F, ::F) = @inbounds grid.zᵃᵃᶠ.reference[k] * vertical_scaling(i, j, k, grid, f, f, c) + grid.zᵃᵃᶠ.∂t_s[i, j] \ No newline at end of file diff --git a/src/ImmersedBoundaries/ImmersedBoundaries.jl b/src/ImmersedBoundaries/ImmersedBoundaries.jl index 7d21f6c911..9093508fe5 100644 --- a/src/ImmersedBoundaries/ImmersedBoundaries.jl +++ b/src/ImmersedBoundaries/ImmersedBoundaries.jl @@ -337,5 +337,8 @@ include("immersed_reductions.jl") # Extension to distributed Immersed boundaries include("distributed_immersed_boundaries.jl") +# Extension to Immersed boundaries with turbulence closures +include("abstract_zstar_immersed_grid.jl") + end # module diff --git a/src/ImmersedBoundaries/abstract_zstar_immersed_grid.jl b/src/ImmersedBoundaries/abstract_zstar_immersed_grid.jl new file mode 100644 index 0000000000..f820d25189 --- /dev/null +++ b/src/ImmersedBoundaries/abstract_zstar_immersed_grid.jl @@ -0,0 +1,21 @@ +using Oceananigans.Grids: AbstractVerticalCoordinateUnderlyingGrid, ZStarUnderlyingGrid + +import Oceananigans.Grids: retrieve_static_grid +import Oceananigans.Grids: reference_zspacings, vertical_scaling, previous_vertical_scaling + +const ImmersedAbstractVerticalCoordinateGrid = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:AbstractVerticalCoordinateUnderlyingGrid} +const ImmersedZStarGrid = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:ZStarUnderlyingGrid} + + +reference_zspacings(grid::ImmersedBoundaryGrid, ℓz) = reference_zspacings(grid.underlying_grid) + +@inline vertical_scaling(i, j, k, grid::ImmersedBoundaryGrid, ℓx, ℓy, ℓz) = vertical_scaling(i, j, k, grid.underlying_grid, ℓx, ℓy, ℓz) +@inline previous_vertical_scaling(i, j, k, grid::ImmersedBoundaryGrid, ℓx, ℓy, ℓz) = previous_vertical_scaling(i, j, k, grid.underlying_grid, ℓx, ℓy, ℓz) + +function retrieve_static_grid(ib::ImmersedAbstractVerticalCoordinateGrid) + immersed_boundary = ib.immersed_boundary + active_cells_map = !isnothing(ib.active_cells_map) + underlying_grid = retrieve_static_grid(ib.underlying_grid) + + return ImmersedBoundaryGrid(underlying_grid, immersed_boundary; active_cells_map) +end \ No newline at end of file diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index 0a02b5b5a5..0f4be4e521 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -2,15 +2,18 @@ using Oceananigans using Oceananigans.Grids using Oceananigans.Operators using Oceananigans.BuoyancyModels: buoyancy_perturbationᶜᶜᶜ -using Oceananigans.Grids: AbstractGrid, AbstractUnderlyingGrid, halo_size +using Oceananigans.Grids: AbstractGrid, AbstractUnderlyingGrid, halo_size, AbstractVerticalCoordinateUnderlyingGrid using Oceananigans.ImmersedBoundaries +using Oceananigans.ImmersedBoundaries: ImmersedAbstractVerticalCoordinateGrid using Oceananigans.Utils: getnamewrapper +using Oceananigans.Grids: with_halo, znode, ∂t_grid, vertical_scaling, previous_vertical_scaling using Adapt using Printf -import Oceananigans.Grids: with_halo, znode, ∂t_grid, vertical_scaling, previous_vertical_scaling import Oceananigans.Architectures: arch_array +const AbstractVerticalCoordinateGrid = Union{AbstractVerticalCoordinateUnderlyingGrid, ImmersedAbstractVerticalCoordinateGrid} + ##### ##### General implementation ##### @@ -47,7 +50,7 @@ update_grid!(model, grid; kwargs...) = nothing end end -ab2_step_tracer_field!(tracer_field, grid::AbstractVerticalSpacingGrid, Δt, χ, Gⁿ, G⁻) = +ab2_step_tracer_field!(tracer_field, grid::AbstractVerticalCoordinateGrid, Δt, χ, Gⁿ, G⁻) = launch!(architecture(grid), grid, :xyz, _ab2_step_tracer_generalized_spacing!, tracer_field, grid, @@ -55,12 +58,14 @@ ab2_step_tracer_field!(tracer_field, grid::AbstractVerticalSpacingGrid, Δt, χ, const EmptyTuples = Union{NamedTuple{(), Tuple{}}, Tuple{}} -unscale_tracers!(::EmptyTuples, ::AbstractVerticalSpacingGrid; kwargs...) = nothing +# Fallbacks +unscale_tracers!(tracers, grid; kwargs...) = nothing +unscale_tracers!(::EmptyTuples, ::AbstractVerticalCoordinateGrid; kwargs...) = nothing tracer_scaling_parameters(param::Symbol, tracers, grid) = KernelParameters((size(grid, 1), size(grid, 2), length(tracers)), (0, 0, 0)) tracer_scaling_parameters(param::KernelParameters{S, O}, tracers, grid) where {S, O} = KernelParameters((S..., length(tracers)), (O..., 0)) -function unscale_tracers!(tracers, grid::AbstractVerticalSpacingGrid; parameters = :xy) +function unscale_tracers!(tracers, grid::AbstractVerticalCoordinateGrid; parameters = :xy) parameters = tracer_scaling_parameters(parameters, tracers, grid) launch!(architecture(grid), grid, parameters, _unscale_tracers!, tracers, grid, diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl index 768251cef5..5aa16cfb92 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl @@ -120,7 +120,6 @@ function HydrostaticFreeSurfaceModel(; grid, velocities = nothing, pressure = nothing, diffusivity_fields = nothing, - vertical_coordinate = nothing, auxiliary_fields = NamedTuple() ) @@ -132,12 +131,6 @@ function HydrostaticFreeSurfaceModel(; grid, # throw(ArgumentError("Generalized vertical coordinates are supported only for the vector-invariant form of the momentum equations")) # end - if !(free_surface isa SplitExplicitFreeSurface) && !isnothing(vertical_coordinate) - @warn "Generalized vertical coordinates are supported only for the SplitExplicitFreeSurface at the moment. Ignoring the vertical_coordinate argument." - else - grid = generalized_spacing_grid(grid, vertical_coordinate) - end - arch = architecture(grid) @apply_regionally momentum_advection = validate_momentum_advection(momentum_advection, grid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl index 06e482396d..eeffb2a256 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl @@ -81,7 +81,7 @@ function compute_auxiliaries!(model::HydrostaticFreeSurfaceModel; w_parameters = for (wpar, ppar, κpar) in zip(w_parameters, p_parameters, κ_parameters) # Update the grid - update_vertical_spacing!(model, grid; parameters = wpar) + update_grid!(model, grid; parameters = wpar) unscale_tracers!(tracers, grid; parameters = wpar) # Update the other auxiliary terms @@ -94,5 +94,3 @@ function compute_auxiliaries!(model::HydrostaticFreeSurfaceModel; w_parameters = return nothing end - -unscale_tracers!(tracers, grid; kwargs...) = nothing diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 660975e179..3a0f04c899 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -1,12 +1,13 @@ using Oceananigans.Grids: ZStarUnderlyingGrid, rnode +using Oceananigans.ImmersedBoundaries: ImmersedZStarGrid -ZStarSpacingGrid = ZStarUnderlyingGrid +const ZStarSpacingGrid = Union{ZStarUnderlyingGrid, ImmersedZStarGrid} ##### ##### ZStar-specific vertical spacings update ##### -function update_grid!(model, grid::ZStarUnderlyingGrid; parameters = :xy) +function update_grid!(model, grid::ZStarSpacingGrid; parameters = :xy) # Scaling (just update once, they are the same for all the metrics) sᶜᶜ⁻ = grid.Δzᵃᵃᶠ.sᶜᶜ⁻ @@ -90,11 +91,11 @@ end @inline ∂x_z(i, j, k, grid, free_surface) = @inbounds ∂xᶠᶜᶜ(i, j, k, grid, z_minus_rᶜᶜᶜ, free_surface.η, free_surface.auxiliary.Hᶜᶜ) @inline ∂y_z(i, j, k, grid, free_surface) = @inbounds ∂yᶜᶠᶜ(i, j, k, grid, z_minus_rᶜᶜᶜ, free_surface.η, free_surface.auxiliary.Hᶜᶜ) -@inline grid_slope_contribution_x(i, j, k, grid::ZStarUnderlyingGrid, free_surface, ::Nothing, model_fields) = zero(grid) -@inline grid_slope_contribution_y(i, j, k, grid::ZStarUnderlyingGrid, free_surface, ::Nothing, model_fields) = zero(grid) +@inline grid_slope_contribution_x(i, j, k, grid::ZStarSpacingGrid, free_surface, ::Nothing, model_fields) = zero(grid) +@inline grid_slope_contribution_y(i, j, k, grid::ZStarSpacingGrid, free_surface, ::Nothing, model_fields) = zero(grid) -@inline grid_slope_contribution_x(i, j, k, grid::ZStarUnderlyingGrid, free_surface, buoyancy, model_fields) = +@inline grid_slope_contribution_x(i, j, k, grid::ZStarSpacingGrid, free_surface, buoyancy, model_fields) = ℑxᶠᵃᵃ(i, j, k, grid, buoyancy_perturbationᶜᶜᶜ, buoyancy.model, model_fields) * ∂x_z(i, j, k, grid, free_surface) -@inline grid_slope_contribution_y(i, j, k, grid::ZStarUnderlyingGrid, free_surface, buoyancy, model_fields) = +@inline grid_slope_contribution_y(i, j, k, grid::ZStarSpacingGrid, free_surface, buoyancy, model_fields) = ℑyᵃᶠᵃ(i, j, k, grid, buoyancy_perturbationᶜᶜᶜ, buoyancy.model, model_fields) * ∂y_z(i, j, k, grid, free_surface) diff --git a/src/Oceananigans.jl b/src/Oceananigans.jl index 2abd97e7a0..095dc7c360 100644 --- a/src/Oceananigans.jl +++ b/src/Oceananigans.jl @@ -17,6 +17,7 @@ export FullyConnected, LeftConnected, RightConnected, RectilinearGrid, LatitudeLongitudeGrid, + ZStarVerticalCoordinate, OrthogonalSphericalShellGrid, xnodes, ynodes, znodes, nodes, λnodes, φnodes, diff --git a/src/Operators/spacings_and_areas_and_volumes.jl b/src/Operators/spacings_and_areas_and_volumes.jl index 5cb6a9fc70..3b793f48f1 100644 --- a/src/Operators/spacings_and_areas_and_volumes.jl +++ b/src/Operators/spacings_and_areas_and_volumes.jl @@ -1,5 +1,5 @@ using Oceananigans.Grids: Center, Face -using Oceananigans.Grids: AbstractVerticalCoordinateGrid +using Oceananigans.Grids: AbstractVerticalCoordinateUnderlyingGrid, ZStarUnderlyingGrid const RG = RectilinearGrid const RGX = XRegularRG @@ -14,6 +14,8 @@ const LLGX = XRegularLLG const LLGY = YRegularLLG const LLGZ = ZRegularLLG +const ZSG = ZStarUnderlyingGrid + # On the fly calculations of metrics const LLGF = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Nothing} const LLGFX = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Nothing, <:Any, <:Number} @@ -74,8 +76,8 @@ const ZRG = Union{LLGZ, RGZ, OSSGZ} @inline getspacing(k, Δz::AbstractVector) = @inbounds Δz[k] @inline getspacing(k, Δz::Number) = @inbounds Δz -@inline Δrᵃᵃᶜ(i, j, k, grid::ZRG) = getspacing(k, grid.Δzᵃᵃᶜ.reference) -@inline Δrᵃᵃᶠ(i, j, k, grid::ZRG) = getspacing(k, grid.Δzᵃᵃᶠ.reference) +@inline Δrᵃᵃᶜ(i, j, k, grid::ZSG) = getspacing(k, grid.Δzᵃᵃᶜ.reference) +@inline Δrᵃᵃᶠ(i, j, k, grid::ZSG) = getspacing(k, grid.Δzᵃᵃᶠ.reference) # Convenience Functions for all grids for LX in (:ᶜ, :ᶠ), LY in (:ᶜ, :ᶠ) @@ -114,7 +116,7 @@ end ##### 3D spacings for AbstractVerticalCoordinate grids ##### -const AVCG = AbstractVerticalCoordinateGrid +const AVCG = AbstractVerticalCoordinateUnderlyingGrid const c = Center() const f = Face() diff --git a/src/OutputWriters/output_writer_utils.jl b/src/OutputWriters/output_writer_utils.jl index a2de7b33b3..a9c5b51342 100644 --- a/src/OutputWriters/output_writer_utils.jl +++ b/src/OutputWriters/output_writer_utils.jl @@ -1,12 +1,13 @@ +using Oceananigans.DistributedComputations using StructArrays: StructArray, replace_storage using Oceananigans.Grids: on_architecture, architecture -using Oceananigans.DistributedComputations +using Oceananigans.Grids: retrieve_static_grid using Oceananigans.DistributedComputations: DistributedGrid, Partition using Oceananigans.Fields: AbstractField, indices, boundary_conditions, instantiated_location using Oceananigans.BoundaryConditions: bc_str, FieldBoundaryConditions, ContinuousBoundaryFunction, DiscreteBoundaryFunction using Oceananigans.TimeSteppers: QuasiAdamsBashforth2TimeStepper, RungeKutta3TimeStepper +using Oceananigans.Models.HydrostaticFreeSurfaceModels: AbstractVerticalCoordinateGrid using Oceananigans.Models.LagrangianParticleTracking: LagrangianParticles -using Oceananigans.Models.HydrostaticFreeSurfaceModels: AbstractVerticalSpacingGrid, retrieve_static_grid using Oceananigans.Utils: AbstractSchedule ##### @@ -90,7 +91,7 @@ function saveproperty!(file, address, grid::DistributedGrid) _saveproperty!(file, address, on_architecture(cpu_arch, grid)) end -function saveproperty!(file, address, grid::AbstractVerticalSpacingGrid) +function saveproperty!(file, address, grid::AbstractVerticalCoordinateGrid) static_grid = retrieve_static_grid(grid) saveproperty!(file, address, static_grid) end @@ -136,7 +137,7 @@ function serializeproperty!(file, address, grid::DistributedGrid) file[address] = on_architecture(cpu_arch, grid) end -function serializeproperty!(file, address, grid::AbstractVerticalSpacingGrid) +function serializeproperty!(file, address, grid::AbstractVerticalCoordinateGrid) static_grid = retrieve_static_grid(grid) serializeproperty!(file, address, static_grid) end diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index 902ed9b5b1..088149a0e4 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -3,19 +3,21 @@ using Oceananigans.Units using Oceananigans.Utils: prettytime using Oceananigans.Advection: WENOVectorInvariant using Oceananigans.AbstractOperations: GridMetricOperation -using Oceananigans.Models.HydrostaticFreeSurfaceModels: ZStar, ZStarSpacingGrid, Δrᶜᶜᶜ +using Oceananigans.Models.HydrostaticFreeSurfaceModels: ZStarSpacingGrid using Printf +r_faces = (-20, 0) +z_faces = ZStarVerticalCoordinate(r_faces) + grid = RectilinearGrid(size = (20, 20), - y = (0, 64kilometers), - z = (-20, 0), + x = (0, 64kilometers), + z = z_faces, halo = (6, 6), - topology = (Flat, Periodic, Bounded)) + topology = (Bounded, Flat, Bounded)) -grid = ImmersedBoundaryGrid(grid, GridFittedBottom(x -> x < 32kilometers ? -10 : -20)) +# grid = ImmersedBoundaryGrid(grid, GridFittedBottom(x -> x < 32kilometers ? -10 : -20)) model = HydrostaticFreeSurfaceModel(; grid, - vertical_coordinate = ZStar(), momentum_advection = WENO(; order = 5), tracer_advection = WENO(; order = 5), buoyancy = BuoyancyTracer(), @@ -61,31 +63,20 @@ function progress(sim) return nothing end -simulation.callbacks[:progress] = Callback(progress, IterationInterval(100)) +simulation.callbacks[:progress] = Callback(progress, IterationInterval(1)) run!(simulation) -using Oceananigans.Utils -using KernelAbstractions: @kernel, @index - -@kernel function _compute_field!(tmp, s, b) - i, j, k = @index(Global, NTuple) - @inbounds tmp[i, j, k] = s[i, j, k] * b[i, j, k] -end - using Oceananigans.Fields: OneField # # Check tracer conservation b = FieldTimeSeries("zstar_model.jld2", "b") Δz = FieldTimeSeries("zstar_model.jld2", "Δz") -tmpfield = CenterField(grid) -launch!(CPU(), grid, :xyz, _compute_field!, tmpfield, Δz[1], b[1]) -init = sum(tmpfield) / sum(Δz[1]) +init = sum(Δz[1] * b[1]) / sum(Δz[1]) drift = [] for t in 1:length(b.times) - launch!(CPU(), grid, :xyz, _compute_field!, tmpfield, Δz[t], b[t]) - push!(drift, sum(tmpfield) / sum(Δz[t]) - init) + push!(drift, sum(Δz[t] * b[t]) / sum(Δz[t]) - init) end From 508306d6757684fef656124df1d7c052a8e75f45 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 10 Oct 2024 12:51:13 +0200 Subject: [PATCH 351/567] just need to fix binary operatinos now --- src/Grids/z_star_vertical_coordinate.jl | 6 ++++-- src/OutputWriters/jld2_output_writer.jl | 8 ++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/Grids/z_star_vertical_coordinate.jl b/src/Grids/z_star_vertical_coordinate.jl index 19a368d50e..db6c9ad89f 100644 --- a/src/Grids/z_star_vertical_coordinate.jl +++ b/src/Grids/z_star_vertical_coordinate.jl @@ -1,3 +1,5 @@ +using Oceananigans.Utils: getnamewrapper + ##### ##### ZStar coordinate and associated types ##### @@ -102,7 +104,7 @@ Grids.coordinate_summary(::Bounded, Δ::ZStarVerticalCoordinate, name) = generate_coordinate(FT, ::Periodic, N, H, ::ZStarVerticalCoordinate, coordinate_name, arch, args...) = throw(ArgumentError("Periodic domains are not supported for ZStarVerticalCoordinate")) -# Generate a regularly-spaced coordinate passing the domain extent (2-tuple) and number of points +# Generate a moving coordinate with evolving scaling (`s`) for spacings and znodes function generate_coordinate(FT, topo, size, halo, coordinate::ZStarVerticalCoordinate, coordinate_name, dim::Int, arch) Nx, Ny, Nz = size @@ -142,7 +144,7 @@ function generate_coordinate(FT, topo, size, halo, coordinate::ZStarVerticalCoor fill!(s, 1) end - # The scaling is the same for everyone (H + \eta) / H, the vertical coordinate requires + # The scaling is the same for everyone, the vertical coordinate requires # to add the free surface to retrieve the znode. zᵃᵃᶠ = ZStarVerticalCoordinate(rᵃᵃᶠ, sᶜᶜᵃ, sᶠᶜᵃ, sᶜᶠᵃ, sᶠᶠᵃ, sᶜᶜᵃ₋, sᶠᶜᵃ₋, sᶜᶠᵃ₋, η) zᵃᵃᶜ = ZStarVerticalCoordinate(rᵃᵃᶜ, sᶜᶜᵃ, sᶠᶜᵃ, sᶜᶠᵃ, sᶠᶠᵃ, sᶜᶜᵃ₋, sᶠᶜᵃ₋, sᶜᶠᵃ₋, η) diff --git a/src/OutputWriters/jld2_output_writer.jl b/src/OutputWriters/jld2_output_writer.jl index 50a0a31937..a2559832f5 100644 --- a/src/OutputWriters/jld2_output_writer.jl +++ b/src/OutputWriters/jld2_output_writer.jl @@ -203,7 +203,7 @@ function initialize_jld2_file!(filepath, init, jld2_kw, including, outputs, mode @warn """Failed to execute user `init` for $filepath because $(typeof(err)): $(sprint(showerror, err))""" end - try + # try jldopen(filepath, "a+"; jld2_kw...) do file saveproperties!(file, model, including) @@ -212,9 +212,9 @@ function initialize_jld2_file!(filepath, init, jld2_kw, including, outputs, mode serializeproperty!(file, "serialized/$property", getproperty(model, property)) end end - catch err - @warn """Failed to save and serialize $including in $filepath because $(typeof(err)): $(sprint(showerror, err))""" - end + # catch err + # @warn """Failed to save and serialize $including in $filepath because $(typeof(err)): $(sprint(showerror, err))""" + # end # Serialize the location and boundary conditions of each output. for (name, field) in pairs(outputs) From a715038bdff922472e565baee523216fed8868bd Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 10 Oct 2024 13:37:04 +0200 Subject: [PATCH 352/567] now it should work --- src/Grids/grid_utils.jl | 3 -- src/Grids/z_star_vertical_coordinate.jl | 31 ++++++++++++++----- .../abstract_zstar_immersed_grid.jl | 4 +-- .../immersed_grid_metrics.jl | 9 ++++++ validation/z_star_coordinate/lock_release.jl | 4 +-- 5 files changed, 37 insertions(+), 14 deletions(-) diff --git a/src/Grids/grid_utils.jl b/src/Grids/grid_utils.jl index 5bd859384c..c881ebeb86 100644 --- a/src/Grids/grid_utils.jl +++ b/src/Grids/grid_utils.jl @@ -124,9 +124,6 @@ constant grid spacing `Δ`, and interior extent `L`. @inline domain(::Flat, N, ξ::Number) = ξ @inline domain(::Flat, N, ::Nothing) = nothing -# TODO: is this the correct definition? -@inline domain(topo, N, ξ::ZStarVerticalCoordinate) = domain(topo, N, ξ.reference) - @inline x_domain(grid) = domain(topology(grid, 1)(), grid.Nx, grid.xᶠᵃᵃ) @inline y_domain(grid) = domain(topology(grid, 2)(), grid.Ny, grid.yᵃᶠᵃ) @inline z_domain(grid) = domain(topology(grid, 3)(), grid.Nz, grid.zᵃᵃᶠ) diff --git a/src/Grids/z_star_vertical_coordinate.jl b/src/Grids/z_star_vertical_coordinate.jl index db6c9ad89f..2239440a86 100644 --- a/src/Grids/z_star_vertical_coordinate.jl +++ b/src/Grids/z_star_vertical_coordinate.jl @@ -1,5 +1,3 @@ -using Oceananigans.Utils: getnamewrapper - ##### ##### ZStar coordinate and associated types ##### @@ -22,6 +20,9 @@ const AbstractVerticalCoordinateUnderlyingGrid = Union{AVLLG, AVOSSG, AVRG} function retrieve_static_grid(grid::AbstractVerticalCoordinateUnderlyingGrid) + zᵃᵃᶠ = reference_znodes(grid, Face()) + zᵃᵃᶜ = reference_znodes(grid, Center()) + Δzᵃᵃᶠ = reference_zspacings(grid, Face()) Δzᵃᵃᶜ = reference_zspacings(grid, Center()) @@ -29,7 +30,11 @@ function retrieve_static_grid(grid::AbstractVerticalCoordinateUnderlyingGrid) args = [] for prop in propertynames(grid) - if prop == :Δzᵃᵃᶠ + if prop == :zᵃᵃᶠ + push!(args, zᵃᵃᶠ) + elseif prop == :zᵃᵃᶜ + push!(args, zᵃᵃᶜ) + elseif prop == :Δzᵃᵃᶠ push!(args, Δzᵃᵃᶠ) elseif prop == :Δzᵃᵃᶜ push!(args, Δzᵃᵃᶜ) @@ -38,11 +43,13 @@ function retrieve_static_grid(grid::AbstractVerticalCoordinateUnderlyingGrid) end end - GridType = getnamewrapper(grid) - - return GridType{TX, TY, TZ}(args...) + return construct_grid(grid, TX, TY, TZ, args...) end +construct_grid(::RectilinearGrid, TX, TY, TZ, args...) = RectilinearGrid{TX, TY, TZ}(args...) +construct_grid(::LatitudeLongitudeGrid, TX, TY, TZ, args...) = LatitudeLongitudeGrid{TX, TY, TZ}(args...) +construct_grid(::OrthogonalSphericalShellGrid, TX, TY, TZ, args...) = OrthogonalSphericalShellGrid{TX, TY, TZ}(args...) + """ struct ZStarVerticalCoordinate{R, S} <: AbstractVerticalSpacing{R} @@ -136,7 +143,8 @@ function generate_coordinate(FT, topo, size, halo, coordinate::ZStarVerticalCoor sᶠᶜᵃ₋ = new_data(FT, arch, (Face, Center, Nothing), args...) ∂t_s = new_data(FT, arch, (Center, Center, Nothing), args...) - # Storage place for the free surface height? Probably find a better way to call this + # Storage place for the free surface height? + # TODO: Probably find a better way to call this or to store this η = new_data(FT, arch, (Center, Center, Nothing), args...) # fill all the scalings with 1 @@ -162,6 +170,9 @@ function validate_dimension_specification(T, ξ::ZStarVerticalCoordinate, dir, N return ZStarVerticalCoordinate(reference, args[2:end]...) end +# TODO: is this the correct definition? +@inline domain(topo, N, ξ::ZStarVerticalCoordinate) = domain(topo, N, ξ.reference) + const ZStarLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:ZStarVerticalCoordinate} const ZStarOSSG = OrthogonalSphericalShellGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:ZStarVerticalCoordinate} const ZStarRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:ZStarVerticalCoordinate} @@ -178,6 +189,9 @@ const F = Face const ZSG = ZStarUnderlyingGrid # Fallbacks +reference_znodes(grid, ::C) = grid.zᵃᵃᶜ +reference_znodes(grid, ::F) = grid.zᵃᵃᶠ + reference_zspacings(grid, ::C) = grid.Δzᵃᵃᶜ reference_zspacings(grid, ::F) = grid.Δzᵃᵃᶠ @@ -190,6 +204,9 @@ reference_zspacings(grid, ::F) = grid.Δzᵃᵃᶠ reference_zspacings(grid::ZSG, ::C) = grid.Δzᵃᵃᶜ.reference reference_zspacings(grid::ZSG, ::F) = grid.Δzᵃᵃᶠ.reference +reference_znodes(grid::ZSG, ::C) = grid.zᵃᵃᶜ.reference +reference_znodes(grid::ZSG, ::F) = grid.zᵃᵃᶠ.reference + @inline vertical_scaling(i, j, k, grid::ZSG, ::C, ::C, ::C) = @inbounds grid.Δzᵃᵃᶜ.sᶜᶜⁿ[i, j] @inline vertical_scaling(i, j, k, grid::ZSG, ::F, ::C, ::C) = @inbounds grid.Δzᵃᵃᶜ.sᶠᶜⁿ[i, j] @inline vertical_scaling(i, j, k, grid::ZSG, ::C, ::F, ::C) = @inbounds grid.Δzᵃᵃᶜ.sᶜᶠⁿ[i, j] diff --git a/src/ImmersedBoundaries/abstract_zstar_immersed_grid.jl b/src/ImmersedBoundaries/abstract_zstar_immersed_grid.jl index f820d25189..65fd99b90a 100644 --- a/src/ImmersedBoundaries/abstract_zstar_immersed_grid.jl +++ b/src/ImmersedBoundaries/abstract_zstar_immersed_grid.jl @@ -1,12 +1,12 @@ using Oceananigans.Grids: AbstractVerticalCoordinateUnderlyingGrid, ZStarUnderlyingGrid import Oceananigans.Grids: retrieve_static_grid -import Oceananigans.Grids: reference_zspacings, vertical_scaling, previous_vertical_scaling +import Oceananigans.Grids: reference_zspacings, reference_znodes, vertical_scaling, previous_vertical_scaling const ImmersedAbstractVerticalCoordinateGrid = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:AbstractVerticalCoordinateUnderlyingGrid} const ImmersedZStarGrid = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:ZStarUnderlyingGrid} - +reference_znodes(grid::ImmersedBoundaryGrid, ℓz) = reference_znodes(grid.underlying_grid) reference_zspacings(grid::ImmersedBoundaryGrid, ℓz) = reference_zspacings(grid.underlying_grid) @inline vertical_scaling(i, j, k, grid::ImmersedBoundaryGrid, ℓx, ℓy, ℓz) = vertical_scaling(i, j, k, grid.underlying_grid, ℓx, ℓy, ℓz) diff --git a/src/ImmersedBoundaries/immersed_grid_metrics.jl b/src/ImmersedBoundaries/immersed_grid_metrics.jl index a7dfbcbe10..14f496d0be 100644 --- a/src/ImmersedBoundaries/immersed_grid_metrics.jl +++ b/src/ImmersedBoundaries/immersed_grid_metrics.jl @@ -1,6 +1,7 @@ using Oceananigans.AbstractOperations: GridMetricOperation import Oceananigans.Grids: coordinates +import Oceananigans.Operators: Δrᵃᵃᶜ, Δrᵃᵃᶠ, Δzᵃᵃᶜ, Δzᵃᵃᶠ const c = Center() const f = Face() @@ -23,6 +24,12 @@ for LX in (:ᶜ, :ᶠ), LY in (:ᶜ, :ᶠ), LZ in (:ᶜ, :ᶠ) end end + metric = Symbol(:Δr, LX, LY, LZ) + @eval begin + import Oceananigans.Operators: $metric + @inline $metric(i, j, k, ibg::IBG) = $metric(i, j, k, ibg.underlying_grid) + end + volume = Symbol(:V, LX, LY, LZ) @eval begin import Oceananigans.Operators: $volume @@ -30,6 +37,8 @@ for LX in (:ᶜ, :ᶠ), LY in (:ᶜ, :ᶠ), LZ in (:ᶜ, :ᶠ) end end +@inline Δrᵃᵃᶜ(i, j, k, ibg::IBG) = Δrᵃᵃᶜ(i, j, k, ibg.underlying_grid) +@inline Δrᵃᵃᶠ(i, j, k, ibg::IBG) = Δrᵃᵃᶠ(i, j, k, ibg.underlying_grid) @inline Δzᵃᵃᶜ(i, j, k, ibg::IBG) = Δzᵃᵃᶜ(i, j, k, ibg.underlying_grid) @inline Δzᵃᵃᶠ(i, j, k, ibg::IBG) = Δzᵃᵃᶠ(i, j, k, ibg.underlying_grid) diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index 088149a0e4..85ccd084b5 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -13,9 +13,9 @@ grid = RectilinearGrid(size = (20, 20), x = (0, 64kilometers), z = z_faces, halo = (6, 6), - topology = (Bounded, Flat, Bounded)) + topology = (Periodic, Flat, Bounded)) -# grid = ImmersedBoundaryGrid(grid, GridFittedBottom(x -> x < 32kilometers ? -10 : -20)) +grid = ImmersedBoundaryGrid(grid, GridFittedBottom(x -> x < 32kilometers ? -10 : -20)) model = HydrostaticFreeSurfaceModel(; grid, momentum_advection = WENO(; order = 5), From 402e04d3eab16f354d241714d0d31f40642f824c Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 10 Oct 2024 13:58:28 +0200 Subject: [PATCH 353/567] just have to fix the immersed boundary --- .../abstract_zstar_immersed_grid.jl | 6 +++--- test/test_zstar_coordinate.jl | 14 ++++++++------ .../z_star_coordinate/baroclinic_double_gyre.jl | 15 +++++++-------- validation/z_star_coordinate/lock_release.jl | 2 +- 4 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/ImmersedBoundaries/abstract_zstar_immersed_grid.jl b/src/ImmersedBoundaries/abstract_zstar_immersed_grid.jl index 65fd99b90a..876a009e63 100644 --- a/src/ImmersedBoundaries/abstract_zstar_immersed_grid.jl +++ b/src/ImmersedBoundaries/abstract_zstar_immersed_grid.jl @@ -6,15 +6,15 @@ import Oceananigans.Grids: reference_zspacings, reference_znodes, vertical_scali const ImmersedAbstractVerticalCoordinateGrid = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:AbstractVerticalCoordinateUnderlyingGrid} const ImmersedZStarGrid = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:ZStarUnderlyingGrid} -reference_znodes(grid::ImmersedBoundaryGrid, ℓz) = reference_znodes(grid.underlying_grid) -reference_zspacings(grid::ImmersedBoundaryGrid, ℓz) = reference_zspacings(grid.underlying_grid) +reference_znodes(grid::ImmersedBoundaryGrid, ℓz) = reference_znodes(grid.underlying_grid, ℓz) +reference_zspacings(grid::ImmersedBoundaryGrid, ℓz) = reference_zspacings(grid.underlying_grid, ℓz) @inline vertical_scaling(i, j, k, grid::ImmersedBoundaryGrid, ℓx, ℓy, ℓz) = vertical_scaling(i, j, k, grid.underlying_grid, ℓx, ℓy, ℓz) @inline previous_vertical_scaling(i, j, k, grid::ImmersedBoundaryGrid, ℓx, ℓy, ℓz) = previous_vertical_scaling(i, j, k, grid.underlying_grid, ℓx, ℓy, ℓz) function retrieve_static_grid(ib::ImmersedAbstractVerticalCoordinateGrid) immersed_boundary = ib.immersed_boundary - active_cells_map = !isnothing(ib.active_cells_map) + active_cells_map = !isnothing(ib.interior_active_cells) underlying_grid = retrieve_static_grid(ib.underlying_grid) return ImmersedBoundaryGrid(underlying_grid, immersed_boundary; active_cells_map) diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index efdf35b6c6..fb0552f4fd 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -28,12 +28,15 @@ end @testset "Testing z-star coordinates" begin + z_uniform = ZStarVerticalCoordinate((-10, 0)) + z_stretched = ZStarVerticalCoordinate(collect(-10:0)) + for arch in archs - llg = LatitudeLongitudeGrid(arch; size = (10, 10, 10), latitude = (-10, 10), longitude = (-10, 10), z = (-10, 0)) - rtg = RectilinearGrid(arch; size = (10, 10, 10), x = (-10, 10), y = (-10, 10), z = (-10, 0)) + llg = LatitudeLongitudeGrid(arch; size = (10, 10, 10), latitude = (-10, 10), longitude = (-10, 10), z = z_uniform) + rtg = RectilinearGrid(arch; size = (10, 10, 10), x = (-10, 10), y = (-10, 10), z = z_uniform) - llgv = LatitudeLongitudeGrid(arch; size = (10, 10, 10), latitude = (-10, 10), longitude = (-10, 10), z = collect(-10:0)) - rtgv = RectilinearGrid(arch; size = (10, 10, 10), x = (-10, 10), y = (-10, 10), z = collect(-10:0)) + llgv = LatitudeLongitudeGrid(arch; size = (10, 10, 10), latitude = (-10, 10), longitude = (-10, 10), z = z_stretched) + rtgv = RectilinearGrid(arch; size = (10, 10, 10), x = (-10, 10), y = (-10, 10), z = z_stretched) illg = ImmersedBoundaryGrid(llg, GridFittedBottom((x, y) -> - rand() - 5)) irtg = ImmersedBoundaryGrid(rtg, GridFittedBottom((x, y) -> - rand() - 5)) @@ -48,8 +51,7 @@ end model = HydrostaticFreeSurfaceModel(; grid, free_surface, tracers = (:b, :c), - bouyancy = BuoyancTracer(), - vertical_coordinate = ZStar()) + bouyancy = BuoyancTracer()) bᵢ(x, y, z) = x < grid.Lx / 2 ? 0.06 : 0.01 diff --git a/validation/z_star_coordinate/baroclinic_double_gyre.jl b/validation/z_star_coordinate/baroclinic_double_gyre.jl index 0a16de900e..1071f938b3 100644 --- a/validation/z_star_coordinate/baroclinic_double_gyre.jl +++ b/validation/z_star_coordinate/baroclinic_double_gyre.jl @@ -2,16 +2,16 @@ using Oceananigans using Oceananigans.Units using Oceananigans.Operators using Oceananigans.Grids: φnode -using Oceananigans.Models.HydrostaticFreeSurfaceModels: ZStar using Oceananigans.AbstractOperations: GridMetricOperation using Printf -arch = CPU() -grid = LatitudeLongitudeGrid(arch; size = (60, 60, 18), - latitude = (15, 75), - longitude = (0, 60), - halo = (5, 5, 5), - z = (-1800, 0)) +arch = CPU() +z_faces = ZStarVerticalCoordinate((-1800, 0)) +grid = LatitudeLongitudeGrid(arch; size = (60, 60, 18), + latitude = (15, 75), + longitude = (0, 60), + halo = (5, 5, 5), + z = z_faces) ##### ##### Parameters @@ -43,7 +43,6 @@ substeps = ceil(Int, 3 * Δt / Δτ) coriolis = HydrostaticSphericalCoriolis() momentum_advection = WENOVectorInvariant(vorticity_order = 5) tracer_advection = WENO(order = 5) -vertical_coordinate = ZStar() free_surface = SplitExplicitFreeSurface(grid; substeps) numerics = (; coriolis, free_surface, momentum_advection, tracer_advection, vertical_coordinate) diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index 85ccd084b5..e25f2fec00 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -13,7 +13,7 @@ grid = RectilinearGrid(size = (20, 20), x = (0, 64kilometers), z = z_faces, halo = (6, 6), - topology = (Periodic, Flat, Bounded)) + topology = (Bounded, Flat, Bounded)) grid = ImmersedBoundaryGrid(grid, GridFittedBottom(x -> x < 32kilometers ? -10 : -20)) From 1c477bfab306606a430dcde75539fc3ec733ea76 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 10 Oct 2024 14:11:08 +0200 Subject: [PATCH 354/567] now it should work with everything --- src/Advection/vector_invariant_self_upwinding.jl | 8 ++++---- src/Grids/z_star_vertical_coordinate.jl | 2 -- src/ImmersedBoundaries/abstract_zstar_immersed_grid.jl | 3 ++- validation/z_star_coordinate/lock_release.jl | 4 ++-- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/Advection/vector_invariant_self_upwinding.jl b/src/Advection/vector_invariant_self_upwinding.jl index 5fd44967bb..91f30d7395 100644 --- a/src/Advection/vector_invariant_self_upwinding.jl +++ b/src/Advection/vector_invariant_self_upwinding.jl @@ -1,4 +1,4 @@ -using Oceananigans.Grids: V_times_∂t_grid +using Oceananigans.Grids: ∂t_grid ##### ##### Self Upwinding of Divergence Flux, the best option! @@ -7,15 +7,15 @@ using Oceananigans.Grids: V_times_∂t_grid @inline δx_U(i, j, k, grid, u, v) = δxᶜᶜᶜ(i, j, k, grid, Ax_qᶠᶜᶜ, u) @inline δy_V(i, j, k, grid, u, v) = δyᶜᶜᶜ(i, j, k, grid, Ay_qᶜᶠᶜ, v) -@inline δx_U_plus_metric(i, j, k, grid, u, v) = δxᶜᶜᶜ(i, j, k, grid, Ax_qᶠᶜᶜ, u) + V_times_∂t_grid(i, j, k, grid) -@inline δy_V_plus_metric(i, j, k, grid, u, v) = δyᶜᶜᶜ(i, j, k, grid, Ay_qᶜᶠᶜ, v) + V_times_∂t_grid(i, j, k, grid) +@inline δx_U_plus_metric(i, j, k, grid, u, v) = δxᶜᶜᶜ(i, j, k, grid, Ax_qᶠᶜᶜ, u) + Vᶜᶜᶜ(i, j, k, grid) * ∂t_grid(i, j, k, grid) +@inline δy_V_plus_metric(i, j, k, grid, u, v) = δyᶜᶜᶜ(i, j, k, grid, Ay_qᶜᶠᶜ, v) + Vᶜᶜᶜ(i, j, k, grid) * ∂t_grid(i, j, k, grid) # Velocity smoothness for divergence upwinding @inline U_smoothness(i, j, k, grid, u, v) = ℑxᶜᵃᵃ(i, j, k, grid, Ax_qᶠᶜᶜ, u) @inline V_smoothness(i, j, k, grid, u, v) = ℑyᵃᶜᵃ(i, j, k, grid, Ay_qᶜᶠᶜ, v) # Divergence smoothness for divergence upwinding -@inline divergence_smoothness(i, j, k, grid, u, v) = δx_U(i, j, k, grid, u, v) + δy_V(i, j, k, grid, u, v) + V_times_∂t_grid(i, j, k, grid) +@inline divergence_smoothness(i, j, k, grid, u, v) = δx_U(i, j, k, grid, u, v) + δy_V(i, j, k, grid, u, v) + Vᶜᶜᶜ(i, j, k, grid) * ∂t_grid(i, j, k, grid) @inline function upwinded_divergence_flux_Uᶠᶜᶜ(i, j, k, grid, scheme::VectorInvariantSelfVerticalUpwinding, u, v) diff --git a/src/Grids/z_star_vertical_coordinate.jl b/src/Grids/z_star_vertical_coordinate.jl index 2239440a86..be7d401232 100644 --- a/src/Grids/z_star_vertical_coordinate.jl +++ b/src/Grids/z_star_vertical_coordinate.jl @@ -199,7 +199,6 @@ reference_zspacings(grid, ::F) = grid.Δzᵃᵃᶠ @inline previous_vertical_scaling(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) @inline ∂t_grid(i, j, k, grid) = zero(grid) -@inline V_times_∂t_grid(i, j, k, grid) = zero(grid) reference_zspacings(grid::ZSG, ::C) = grid.Δzᵃᵃᶜ.reference reference_zspacings(grid::ZSG, ::F) = grid.Δzᵃᵃᶠ.reference @@ -226,7 +225,6 @@ reference_znodes(grid::ZSG, ::F) = grid.zᵃᵃᶠ.reference @inline previous_vertical_scaling(i, j, k, grid::ZSG, ::C, ::F, ::F) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶠ⁻[i, j] @inline ∂t_grid(i, j, k, grid::ZSG) = @inbounds grid.Δzᵃᵃᶜ.∂t_s[i, j] -@inline V_times_∂t_grid(i, j, k, grid::ZSG) = ∂t_grid(i, j, k, grid) * Vᶜᶜᶜ(i, j, k, grid) ##### ##### znode diff --git a/src/ImmersedBoundaries/abstract_zstar_immersed_grid.jl b/src/ImmersedBoundaries/abstract_zstar_immersed_grid.jl index 876a009e63..d3b765520d 100644 --- a/src/ImmersedBoundaries/abstract_zstar_immersed_grid.jl +++ b/src/ImmersedBoundaries/abstract_zstar_immersed_grid.jl @@ -1,7 +1,7 @@ using Oceananigans.Grids: AbstractVerticalCoordinateUnderlyingGrid, ZStarUnderlyingGrid import Oceananigans.Grids: retrieve_static_grid -import Oceananigans.Grids: reference_zspacings, reference_znodes, vertical_scaling, previous_vertical_scaling +import Oceananigans.Grids: reference_zspacings, reference_znodes, vertical_scaling, previous_vertical_scaling, ∂t_grid const ImmersedAbstractVerticalCoordinateGrid = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:AbstractVerticalCoordinateUnderlyingGrid} const ImmersedZStarGrid = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:ZStarUnderlyingGrid} @@ -11,6 +11,7 @@ reference_zspacings(grid::ImmersedBoundaryGrid, ℓz) = reference_zspacings(grid @inline vertical_scaling(i, j, k, grid::ImmersedBoundaryGrid, ℓx, ℓy, ℓz) = vertical_scaling(i, j, k, grid.underlying_grid, ℓx, ℓy, ℓz) @inline previous_vertical_scaling(i, j, k, grid::ImmersedBoundaryGrid, ℓx, ℓy, ℓz) = previous_vertical_scaling(i, j, k, grid.underlying_grid, ℓx, ℓy, ℓz) +@inline ∂t_grid(i, j, k, grid::ImmersedBoundaryGrid) = ∂t_grid(i, j, k, grid.underlying_grid) function retrieve_static_grid(ib::ImmersedAbstractVerticalCoordinateGrid) immersed_boundary = ib.immersed_boundary diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index e25f2fec00..017d04fc77 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -37,7 +37,7 @@ set!(model, b = bᵢ) @info "the time step is $Δt" -simulation = Simulation(model; Δt, stop_iteration = 10000, stop_time = 17hours) +simulation = Simulation(model; Δt, stop_iteration = 10000000, stop_time = 17hours) Δz = GridMetricOperation((Center, Center, Center), Oceananigans.AbstractOperations.Δz, model.grid) @@ -63,7 +63,7 @@ function progress(sim) return nothing end -simulation.callbacks[:progress] = Callback(progress, IterationInterval(1)) +simulation.callbacks[:progress] = Callback(progress, IterationInterval(100)) run!(simulation) From f8d5c1857f054f687b3cb91f3bea21bf37525aee Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 10 Oct 2024 14:22:26 +0200 Subject: [PATCH 355/567] bugfix --- ext/OceananigansMakieExt.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/OceananigansMakieExt.jl b/ext/OceananigansMakieExt.jl index 9c8fe12bad..398f847fec 100644 --- a/ext/OceananigansMakieExt.jl +++ b/ext/OceananigansMakieExt.jl @@ -37,7 +37,7 @@ end convert_arguments(pl::Type{<:AbstractPlot}, f::Field) = convert_arguments(pl, convert_field_argument(f)...) -function convert_arguments(pl::Type{<:AbstractPlot}, fop::AbstractOperation) +function convert_arguments(pl::Type{<:AbstractPlot}, op::AbstractOperation) f = Field(op) compute!(f) return convert_arguments(pl, f) From fd8a0287b34df94397b3e9eb5f6ef336c381198a Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 10 Oct 2024 14:26:02 +0200 Subject: [PATCH 356/567] another bugfix --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index b3ac5f35fc..455d35666c 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -25,7 +25,6 @@ struct GridFittedBottom{H, I} <: AbstractGridFittedBottom{H} immersed_condition :: I end - GridFittedBottom(bottom_height) = GridFittedBottom(bottom_height, CenterImmersedCondition()) Base.summary(::CenterImmersedCondition) = "CenterImmersedCondition" @@ -73,7 +72,7 @@ function Base.show(io::IO, ib::GridFittedBottom) print(io, "├── bottom_height: ", prettysummary(ib.bottom_height), '\n') end -on_architecture(arch, ib::GridFittedBottom) = GridFittedBottom(on_architecture(ib.bottom_height), ib.immersed_condition) +on_architecture(arch, ib::GridFittedBottom) = GridFittedBottom(on_architecture(arch, ib.bottom_height), ib.immersed_condition) function on_architecture(arch, ib::GridFittedBottom{<:Field}) architecture(ib.bottom_height) == arch && return ib @@ -84,7 +83,7 @@ function on_architecture(arch, ib::GridFittedBottom{<:Field}) return GridFittedBottom(new_bottom_height, ib.immersed_condition) end -Adapt.adapt_structure(to, ib::GridFittedBottom) = GridFittedBottom(adapt(to, ib.bottom_height), ib.immersed_condition) +Adapt.adapt_structure(to, ib::GridFittedBottom) = GridFittedBottom(adapt(to, ib.bottom_height), adapt(to, ib.immersed_condition)) """ ImmersedBoundaryGrid(grid, ib::GridFittedBottom) From 5b3d3bfd8b4ac3e1e694a1e84f6478e16d7503fa Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 10 Oct 2024 14:41:44 +0200 Subject: [PATCH 357/567] remove tripolar grid just to check tests --- Project.toml | 3 +-- src/Grids/latitude_longitude_grid.jl | 6 +++--- test/test_cubed_spheres.jl | 6 +++--- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/Project.toml b/Project.toml index 8692d7f1f9..91f916318e 100644 --- a/Project.toml +++ b/Project.toml @@ -80,10 +80,9 @@ julia = "1.9" [extras] DataDeps = "124859b0-ceae-595e-8997-d05f6a7a8dfe" Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" -OrthogonalSphericalShellGrids = "c2be9673-fb75-4747-82dc-aa2bb9f4aed0" SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" TimesDates = "bdfc003b-8df8-5c39-adcd-3a9087f5df4a" [targets] -test = ["DataDeps", "Enzyme", "OrthogonalSphericalShellGrids", "SafeTestsets", "Test", "TimesDates"] +test = ["DataDeps", "Enzyme", "SafeTestsets", "Test", "TimesDates"] diff --git a/src/Grids/latitude_longitude_grid.jl b/src/Grids/latitude_longitude_grid.jl index 48d0da6ce0..1dcef52943 100644 --- a/src/Grids/latitude_longitude_grid.jl +++ b/src/Grids/latitude_longitude_grid.jl @@ -197,9 +197,9 @@ function LatitudeLongitudeGrid(architecture::AbstractArchitecture = CPU(), # it is stretched if being passed is a function or vector (as for the VerticallyStretchedRectilinearGrid) TX, TY, TZ = topology - Lλ, λᶠᵃᵃ, λᶜᵃᵃ, Δλᶠᵃᵃ, Δλᶜᵃᵃ = generate_coordinate(FT, TX(), Nλ, Hλ, longitude, :longitude, 1, architecture) - Lφ, φᵃᶠᵃ, φᵃᶜᵃ, Δφᵃᶠᵃ, Δφᵃᶜᵃ = generate_coordinate(FT, TY(), Nφ, Hφ, latitude, :latitude, 2, architecture) - Lz, zᵃᵃᶠ, zᵃᵃᶜ, Δzᵃᵃᶠ, Δzᵃᵃᶜ = generate_coordinate(FT, TZ(), Nz, Hz, z, :z, 3, architecture) + Lλ, λᶠᵃᵃ, λᶜᵃᵃ, Δλᶠᵃᵃ, Δλᶜᵃᵃ = generate_coordinate(FT, topology, size, halo, x, :longitude, 1, architecture) + Lφ, φᵃᶠᵃ, φᵃᶜᵃ, Δφᵃᶠᵃ, Δφᵃᶜᵃ = generate_coordinate(FT, topology, size, halo, y, :latitude, 2, architecture) + Lz, zᵃᵃᶠ, zᵃᵃᶜ, Δzᵃᵃᶠ, Δzᵃᵃᶜ = generate_coordinate(FT, topology, size, halo, z, :z, 3, architecture) preliminary_grid = LatitudeLongitudeGrid{TX, TY, TZ}(architecture, Nλ, Nφ, Nz, diff --git a/test/test_cubed_spheres.jl b/test/test_cubed_spheres.jl index d8984cb0a1..09319d665c 100644 --- a/test/test_cubed_spheres.jl +++ b/test/test_cubed_spheres.jl @@ -7,7 +7,7 @@ using Oceananigans.CubedSpheres using Oceananigans.Models.HydrostaticFreeSurfaceModels using Oceananigans.Models.HydrostaticFreeSurfaceModels: VerticalVorticityField -using OrthogonalSphericalShellGrids +# using OrthogonalSphericalShellGrids # To be used in the test below as `KernelFunctionOperation`s @inline intrinsic_vector_x_component(i, j, k, grid, uₑ, vₑ) = @@ -172,10 +172,10 @@ end @testset "Conversion from Intrinsic to Extrinsic reference frame [$(typeof(arch))]" begin @info " Testing the conversion of a vector between the Intrinsic and Extrinsic reference frame" - trg_grid = TripolarGrid(arch, size = (20, 20, 1), z = (0, 1)) + # trg_grid = TripolarGrid(arch, size = (20, 20, 1), z = (0, 1)) test_vector_rotation(grid) - test_vector_rotation(trg_grid) + # test_vector_rotation(trg_grid) end @testset "CubedSphereData and CubedSphereFields [$(typeof(arch))]" begin From a9535b6f9a0601679cea7c43f3f9547a990a94b0 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 10 Oct 2024 14:42:51 +0200 Subject: [PATCH 358/567] bugfix in latlon grid --- src/Grids/latitude_longitude_grid.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Grids/latitude_longitude_grid.jl b/src/Grids/latitude_longitude_grid.jl index 1dcef52943..8469c63de6 100644 --- a/src/Grids/latitude_longitude_grid.jl +++ b/src/Grids/latitude_longitude_grid.jl @@ -197,9 +197,9 @@ function LatitudeLongitudeGrid(architecture::AbstractArchitecture = CPU(), # it is stretched if being passed is a function or vector (as for the VerticallyStretchedRectilinearGrid) TX, TY, TZ = topology - Lλ, λᶠᵃᵃ, λᶜᵃᵃ, Δλᶠᵃᵃ, Δλᶜᵃᵃ = generate_coordinate(FT, topology, size, halo, x, :longitude, 1, architecture) - Lφ, φᵃᶠᵃ, φᵃᶜᵃ, Δφᵃᶠᵃ, Δφᵃᶜᵃ = generate_coordinate(FT, topology, size, halo, y, :latitude, 2, architecture) - Lz, zᵃᵃᶠ, zᵃᵃᶜ, Δzᵃᵃᶠ, Δzᵃᵃᶜ = generate_coordinate(FT, topology, size, halo, z, :z, 3, architecture) + Lλ, λᶠᵃᵃ, λᶜᵃᵃ, Δλᶠᵃᵃ, Δλᶜᵃᵃ = generate_coordinate(FT, topology, size, halo, longitude, :longitude, 1, architecture) + Lφ, φᵃᶠᵃ, φᵃᶜᵃ, Δφᵃᶠᵃ, Δφᵃᶜᵃ = generate_coordinate(FT, topology, size, halo, latitude, :latitude, 2, architecture) + Lz, zᵃᵃᶠ, zᵃᵃᶜ, Δzᵃᵃᶠ, Δzᵃᵃᶜ = generate_coordinate(FT, topology, size, halo, z, :z, 3, architecture) preliminary_grid = LatitudeLongitudeGrid{TX, TY, TZ}(architecture, Nλ, Nφ, Nz, From 09d1a34bcea276ade694ae9ddc075ad283b826c8 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 10 Oct 2024 16:09:43 +0200 Subject: [PATCH 359/567] more fixes --- src/Grids/z_star_vertical_coordinate.jl | 11 ++++++----- .../compute_w_from_continuity.jl | 4 ++-- .../z_star_coordinate/baroclinic_double_gyre.jl | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/Grids/z_star_vertical_coordinate.jl b/src/Grids/z_star_vertical_coordinate.jl index be7d401232..911a281807 100644 --- a/src/Grids/z_star_vertical_coordinate.jl +++ b/src/Grids/z_star_vertical_coordinate.jl @@ -14,10 +14,6 @@ const AVRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Abstr const AbstractVerticalCoordinateUnderlyingGrid = Union{AVLLG, AVOSSG, AVRG} -# rnode for an AbstractVerticalCoordinate grid is the reference node -@inline rnode(i, j, k, grid::AbstractVerticalCoordinateUnderlyingGrid, ℓx, ℓy, ::Center) = @inbounds grid.zᵃᵃᶜ.reference[k] -@inline rnode(i, j, k, grid::AbstractVerticalCoordinateUnderlyingGrid, ℓx, ℓy, ::Face) = @inbounds grid.zᵃᵃᶠ.reference[k] - function retrieve_static_grid(grid::AbstractVerticalCoordinateUnderlyingGrid) zᵃᵃᶠ = reference_znodes(grid, Face()) @@ -233,7 +229,12 @@ reference_znodes(grid::ZSG, ::F) = grid.zᵃᵃᶠ.reference const c = Center() const f = Face() -# rnode for an AbstractVerticalCoordinate grid is the reference node +# rnode for an ZStarUnderlyingGrid is the reference node +@inline rnode(i, j, k, grid::ZSG, ℓx, ℓy, ::Center) = @inbounds grid.zᵃᵃᶜ.reference[k] +@inline rnode(i, j, k, grid::ZSG, ℓx, ℓy, ::Face) = @inbounds grid.zᵃᵃᶠ.reference[k] + +# rnode for an ZStarUnderlyingGrid grid is scaled +# TODO: fix this when bottom height is implemented @inline znode(i, j, k, grid::ZSG, ::C, ::C, ::C) = @inbounds grid.zᵃᵃᶜ.reference[k] * vertical_scaling(i, j, k, grid, c, c, c) + grid.zᵃᵃᶜ.∂t_s[i, j] @inline znode(i, j, k, grid::ZSG, ::C, ::F, ::C) = @inbounds grid.zᵃᵃᶜ.reference[k] * vertical_scaling(i, j, k, grid, c, f, c) + grid.zᵃᵃᶜ.∂t_s[i, j] @inline znode(i, j, k, grid::ZSG, ::F, ::C, ::C) = @inbounds grid.zᵃᵃᶜ.reference[k] * vertical_scaling(i, j, k, grid, f, c, c) + grid.zᵃᵃᶜ.∂t_s[i, j] diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl index 29b2291429..8b7e0cf134 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl @@ -45,8 +45,8 @@ end Hx, Hy, _ = halo_size(grid) Tx, Ty, _ = topology(grid) - ii = ifelse(Tx == Flat, 1:Nx, -Hx+1:Nx+Hx) - jj = ifelse(Ty == Flat, 1:Ny, -Hy+1:Ny+Hy) + ii = ifelse(Tx == Flat, 1:Nx, -Hx+2:Nx+Hx-1) + jj = ifelse(Ty == Flat, 1:Ny, -Hy+2:Ny+Hy-1) return KernelParameters(ii, jj) end diff --git a/validation/z_star_coordinate/baroclinic_double_gyre.jl b/validation/z_star_coordinate/baroclinic_double_gyre.jl index 1071f938b3..022a084ad6 100644 --- a/validation/z_star_coordinate/baroclinic_double_gyre.jl +++ b/validation/z_star_coordinate/baroclinic_double_gyre.jl @@ -45,7 +45,7 @@ momentum_advection = WENOVectorInvariant(vorticity_order = 5) tracer_advection = WENO(order = 5) free_surface = SplitExplicitFreeSurface(grid; substeps) -numerics = (; coriolis, free_surface, momentum_advection, tracer_advection, vertical_coordinate) +numerics = (; coriolis, free_surface, momentum_advection, tracer_advection) ##### ##### Closure From 6de18a671f8e32cef1fb6b808bdbdf502c909ba4 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 10 Oct 2024 17:21:44 +0200 Subject: [PATCH 360/567] remove vestigial code --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 1 - .../HydrostaticFreeSurfaceModels.jl | 4 ---- .../compute_hydrostatic_free_surface_tendencies.jl | 2 +- .../hydrostatic_free_surface_model.jl | 13 ++++--------- .../split_explicit_free_surface.jl | 2 +- 5 files changed, 6 insertions(+), 16 deletions(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index ef4f36c170..e007f461a1 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -8,7 +8,6 @@ using Oceananigans.BoundaryConditions: FBC using Printf import Oceananigans.TurbulenceClosures: z_bottom -import Oceananigans.Architectures: on_architecture ##### ##### GridFittedBottom (2.5D immersed boundary with modified bottom height) diff --git a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl index 7cfb4f60dc..c0fad04ff5 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl @@ -86,10 +86,6 @@ Return a flattened `NamedTuple` of the prognostic fields associated with `Hydros @inline prognostic_fields(model::HydrostaticFreeSurfaceModel) = hydrostatic_prognostic_fields(model.velocities, model.free_surface, model.tracers) -@inline prognostic_fields(model::HydrostaticFreeSurfaceModel{<:Any, <:Any, <:Any, <:Any, ZStarSpacingGrid}) = - merge(hydrostatic_prognostic_fields(model.velocities, model.free_surface, model.tracers), - (; ∂t_s = grid.Δzᵃᵃᶠ.∂t_s)) - @inline hydrostatic_prognostic_fields(velocities, free_surface, tracers) = merge((u = velocities.u, v = velocities.v, η = free_surface.η), diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl index fbd2b9eeef..1069201345 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_tendencies.jl @@ -14,7 +14,7 @@ using Oceananigans.ImmersedBoundaries: retrieve_interior_active_cells_map, Activ active_linear_index_to_tuple """ - compute_tendencies!(model::HydrostaticFreeSurfaceModel, Δt, callbacks) + compute_tendencies!(model::HydrostaticFreeSurfaceModel, callbacks) Calculate the interior and boundary contributions to tendency terms without the contribution from non-hydrostatic pressure. diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl index 5aa16cfb92..ffb2cfb3ba 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl @@ -58,13 +58,13 @@ default_free_surface(grid; gravitational_acceleration=g_Earth) = clock = Clock{eltype(grid)}(time = 0), momentum_advection = CenteredSecondOrder(), tracer_advection = CenteredSecondOrder(), - buoyancy = SeawaterBuoyancy(eltype(grid)), + buoyancy = nothing, coriolis = nothing, free_surface = default_free_surface(grid, gravitational_acceleration=g_Earth), forcing::NamedTuple = NamedTuple(), closure = nothing, boundary_conditions::NamedTuple = NamedTuple(), - tracers = (:T, :S), + tracers = nothing, particles::ParticlesOrNothing = nothing, biogeochemistry::AbstractBGCOrNothing = nothing, velocities = nothing, @@ -108,10 +108,10 @@ function HydrostaticFreeSurfaceModel(; grid, clock = Clock{eltype(grid)}(time = 0), momentum_advection = CenteredSecondOrder(), tracer_advection = CenteredSecondOrder(), - buoyancy = SeawaterBuoyancy(eltype(grid)), + buoyancy = nothing, coriolis = nothing, free_surface = default_free_surface(grid, gravitational_acceleration=g_Earth), - tracers = (:T, :S), + tracers = nothing, forcing::NamedTuple = NamedTuple(), closure = nothing, boundary_conditions::NamedTuple = NamedTuple(), @@ -126,11 +126,6 @@ function HydrostaticFreeSurfaceModel(; grid, # Check halos and throw an error if the grid's halo is too small @apply_regionally validate_model_halo(grid, momentum_advection, tracer_advection, closure) - # Introduce z-star coordinates if needed (only is free_surface is not a nothing) - # if !isnothing(vertical_coordinate) && !(momentum_advection isa VectorInvariant) && isnothing(velocities) - # throw(ArgumentError("Generalized vertical coordinates are supported only for the vector-invariant form of the momentum equations")) - # end - arch = architecture(grid) @apply_regionally momentum_advection = validate_momentum_advection(momentum_advection, grid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index 5ae4a1df08..fcfaca5ac4 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -259,7 +259,7 @@ function compute_column_height!(Hᶜᶜ, Hᶠᶜ, Hᶜᶠ, Hᶠᶠ, grid) Hx, Hy, _ = halo_size(grid) Nx, Ny, _ = size(grid) - params = KernelParameters(-Hx+1:Nx+Hx, -Hy+1:Ny+Hy) + params = KernelParameters(-Hx+2:Nx+Hx-1, -Hy+2:Ny+Hy-1) launch!(arch, grid, params, _compute_column_height!, Hᶜᶜ, grid, c, c, Δrᶜᶜᶜ) launch!(arch, grid, params, _compute_column_height!, Hᶠᶜ, grid, f, c, Δrᶠᶜᶜ) From f518d414b1d843ff7f58a72a797770fffeeeb8e9 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 11 Oct 2024 09:29:58 +0200 Subject: [PATCH 361/567] fix bugs --- src/MultiRegion/multi_region_field.jl | 2 ++ src/MultiRegion/multi_region_grid.jl | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/MultiRegion/multi_region_field.jl b/src/MultiRegion/multi_region_field.jl index cb6d06cda8..6abbee7ce2 100644 --- a/src/MultiRegion/multi_region_field.jl +++ b/src/MultiRegion/multi_region_field.jl @@ -70,6 +70,8 @@ Base.size(f::GriddedMultiRegionField) = size(getregion(f.grid, 1)) Reconstruct a global field from `mrf::MultiRegionField` on the `CPU`. """ function reconstruct_global_field(mrf::MultiRegionField) + + # TODO: Is this correct? Shall we reconstruct a global field on the architecture of the grid? global_grid = on_architecture(CPU(), reconstruct_global_grid(mrf.grid)) indices = reconstruct_global_indices(mrf.indices, mrf.grid.partition, size(global_grid)) global_field = Field(location(mrf), global_grid; indices) diff --git a/src/MultiRegion/multi_region_grid.jl b/src/MultiRegion/multi_region_grid.jl index f338750b24..61e419c16a 100644 --- a/src/MultiRegion/multi_region_grid.jl +++ b/src/MultiRegion/multi_region_grid.jl @@ -184,11 +184,12 @@ end function reconstruct_global_grid(mrg::ImmersedMultiRegionGrid) global_grid = reconstruct_global_grid(mrg.underlying_grid) global_boundary = reconstruct_global_boundary(mrg.immersed_boundary) + global_boundary = on_architecture(architecture(mrg), global_boundary) return ImmersedBoundaryGrid(global_grid, global_boundary) end -reconstruct_global_boundary(g::GridFittedBottom{<:Field}) = GridFittedBottom(reconstruct_global_field(g.bottom_height)) +reconstruct_global_boundary(g::GridFittedBottom{<:Field}) = GridFittedBottom(reconstruct_global_field(g.bottom_height), g.immersed_condition) reconstruct_global_boundary(g::PartialCellBottom{<:Field}) = PartialCellBottom(reconstruct_global_field(g.bottom_height), g.minimum_fractional_cell_height) reconstruct_global_boundary(g::GridFittedBoundary{<:Field}) = GridFittedBoundary(reconstruct_global_field(g.mask)) From 983b9560c0334c7f12de8797482f24b011902aab Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 11 Oct 2024 09:33:35 +0200 Subject: [PATCH 362/567] some corrections --- docs/src/grids.md | 2 +- src/Advection/adapt_advection_order.jl | 98 ------------------------ src/BoundaryConditions/apply_flux_bcs.jl | 4 +- 3 files changed, 3 insertions(+), 101 deletions(-) delete mode 100644 src/Advection/adapt_advection_order.jl diff --git a/docs/src/grids.md b/docs/src/grids.md index a6686e5767..71ec02adc0 100644 --- a/docs/src/grids.md +++ b/docs/src/grids.md @@ -211,7 +211,7 @@ h = mountain_grid.immersed_boundary.bottom_height fig = Figure(size=(600, 600)) ax = Axis(fig[2, 1], xlabel="x (m)", ylabel="y (m)", aspect=1) -hm = heatmap!(ax, - h) +hm = heatmap!(ax, h) Colorbar(fig[1, 1], hm, vertical=false, label="Bottom height (m)") current_figure() diff --git a/src/Advection/adapt_advection_order.jl b/src/Advection/adapt_advection_order.jl deleted file mode 100644 index 41f3f60c9a..0000000000 --- a/src/Advection/adapt_advection_order.jl +++ /dev/null @@ -1,98 +0,0 @@ -using Oceananigans.Grids: topology - -""" - adapt_advection_order(advection, grid::AbstractGrid) - -Adapts the advection operator `advection` based on the grid `grid` by adjusting the order of advection in each direction. -For example, if the grid has only one point in the x-direction, the advection operator in the x-direction is set to first order -upwind or 2nd order centered scheme, depending on the original user-specified advection scheme. A high order advection sheme -is reduced to a lower order advection scheme if the grid has fewer points in that direction. - -# Arguments -- `advection`: The original advection scheme. -- `grid::AbstractGrid`: The grid on which the advection scheme is applied. - -If the order of advection is changed in at least one direction, the adapted advection scheme with adjusted advection order returned -by this function is a `FluxFormAdvection`. -""" -function adapt_advection_order(advection, grid::AbstractGrid) - advection_x = x_advection(advection) - advection_y = y_advection(advection) - advection_z = z_advection(advection) - - new_advection_x = adapt_advection_order(advection_x, size(grid, 1), grid) - new_advection_y = adapt_advection_order(advection_y, size(grid, 2), grid) - new_advection_z = adapt_advection_order(advection_z, size(grid, 3), grid) - - # Check that we indeed changed the advection operator - changed_x = new_advection_x != advection_x - changed_y = new_advection_y != advection_y - changed_z = new_advection_z != advection_z - - new_advection = FluxFormAdvection(new_advection_x, new_advection_y, new_advection_z) - changed_advection = any((changed_x, changed_y, changed_z)) - - if changed_x - @info "Using the advection scheme $(summary(new_advection.x)) in the x-direction because size(grid, 1) = $(size(grid, 1))" - end - if changed_y - @info "Using the advection scheme $(summary(new_advection.y)) in the y-direction because size(grid, 2) = $(size(grid, 2))" - end - if changed_z - @info "Using the advection scheme $(summary(new_advection.z)) in the x-direction because size(grid, 3) = $(size(grid, 3))" - end - - return ifelse(changed_advection, new_advection, advection) -end - - -x_advection(flux_form::FluxFormAdvection) = flux_form.x -y_advection(flux_form::FluxFormAdvection) = flux_form.y -z_advection(flux_form::FluxFormAdvection) = flux_form.z - -x_advection(advection) = advection -y_advection(advection) = advection -z_advection(advection) = advection - -# For the moment, we do not adapt the advection order for the VectorInvariant advection scheme -adapt_advection_order(advection::VectorInvariant, grid::AbstractGrid) = advection -adapt_advection_order(advection::Nothing, grid::AbstractGrid) = nothing -adapt_advection_order(advection::Nothing, N::Int, grid::AbstractGrid) = nothing - -##### -##### Directional adapt advection order -##### - -function adapt_advection_order(advection::Centered{B}, N::Int, grid::AbstractGrid) where B - if N >= B - return advection - else - return Centered(; order = 2N) - end -end - -function adapt_advection_order(advection::UpwindBiased{B}, N::Int, grid::AbstractGrid) where B - if N >= B - return advection - else - return UpwindBiased(; order = 2N - 1) - end -end - -""" - new_weno_scheme(grid, order, bounds, XT, YT, ZT) - -Constructs a new WENO scheme based on the given parameters. `XT`, `YT`, and `ZT` is the type of the precomputed weno coefficients in the -x-direction, y-direction and z-direction. A _non-stretched_ WENO scheme has `T` equal to `Nothing` everywhere. In case of a non-stretched WENO scheme, -we rebuild the advection without passing the grid information, otherwise we use the grid to account for stretched directions. -""" -new_weno_scheme(::WENO, grid, order, bounds, ::Type{Nothing}, ::Type{Nothing}, ::Type{Nothing},) = WENO(; order, bounds) -new_weno_scheme(::WENO, grid, order, bounds, XT, YT, ZT) = WENO(grid; order, bounds) - -function adapt_advection_order(advection::WENO{B, FT, XT, YT, ZT}, N::Int, grid::AbstractGrid) where {B, FT, XT, YT, ZT} - if N >= B - return advection - else - return new_weno_scheme(advection, grid, 2N - 1, advection.bounds, XT, YT, ZT) - end -end \ No newline at end of file diff --git a/src/BoundaryConditions/apply_flux_bcs.jl b/src/BoundaryConditions/apply_flux_bcs.jl index 1ed09e8b43..2a69fad07c 100644 --- a/src/BoundaryConditions/apply_flux_bcs.jl +++ b/src/BoundaryConditions/apply_flux_bcs.jl @@ -78,7 +78,7 @@ Apply a top and/or bottom boundary condition to variable `c`. """ @kernel function _apply_z_bcs!(Gc, loc, grid, bottom_bc, top_bc, args) i, j = @index(Global, NTuple) - apply_bottom_height_bc!(Gc, loc, bottom_bc, i, j, grid, args...) + apply_z_bottom_bc!(Gc, loc, bottom_bc, i, j, grid, args...) apply_z_top_bc!(Gc, loc, top_bc, i, j, grid, args...) end @@ -88,7 +88,7 @@ end @inline apply_y_north_bc!(Gc, loc, ::NotFluxBC, args...) = nothing @inline apply_y_south_bc!(Gc, loc, ::NotFluxBC, args...) = nothing @inline apply_z_top_bc!(Gc, loc, ::NotFluxBC, args...) = nothing -@inline apply_bottom_height_bc!(Gc, loc, ::NotFluxBC, args...) = nothing +@inline apply_z_bottom_bc!(Gc, loc, ::NotFluxBC, args...) = nothing @inline flip(::Center) = Face() @inline flip(::Face) = Center() From dfada799cb91a1945a95b99100288e04bcc2d4cb Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 11 Oct 2024 12:19:34 +0200 Subject: [PATCH 363/567] another bugfix --- src/BoundaryConditions/apply_flux_bcs.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BoundaryConditions/apply_flux_bcs.jl b/src/BoundaryConditions/apply_flux_bcs.jl index 2a69fad07c..c1866fd84e 100644 --- a/src/BoundaryConditions/apply_flux_bcs.jl +++ b/src/BoundaryConditions/apply_flux_bcs.jl @@ -120,7 +120,7 @@ end return nothing end -@inline function apply_bottom_height_bc!(Gc, loc, bottom_flux::BC{<:Flux}, i, j, grid, args...) +@inline function apply_z_bottom_bc!(Gc, loc, bottom_flux::BC{<:Flux}, i, j, grid, args...) LX, LY, LZ = loc @inbounds Gc[i, j, 1] += getbc(bottom_flux, i, j, grid, args...) * Az(i, j, 1, grid, LX, LY, flip(LZ)) / volume(i, j, 1, grid, LX, LY, LZ) return nothing From 2705b03818c459ab23d7cc61e3e3f6c5f241109b Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 11 Oct 2024 16:46:53 +0200 Subject: [PATCH 364/567] domain_depth --- src/Grids/Grids.jl | 2 +- src/Grids/grid_utils.jl | 8 ++++---- src/ImmersedBoundaries/ImmersedBoundaries.jl | 2 +- src/ImmersedBoundaries/grid_fitted_bottom.jl | 18 +++++++++--------- .../split_explicit_free_surface_kernels.jl | 8 ++++---- src/TurbulenceClosures/TurbulenceClosures.jl | 13 ++++++------- .../catke_equation.jl | 2 +- .../catke_mixing_length.jl | 8 ++++---- 8 files changed, 30 insertions(+), 31 deletions(-) diff --git a/src/Grids/Grids.jl b/src/Grids/Grids.jl index fcc6270001..e88ca07980 100644 --- a/src/Grids/Grids.jl +++ b/src/Grids/Grids.jl @@ -18,7 +18,7 @@ export xnodes, ynodes, znodes, λnodes, φnodes export spacings export xspacings, yspacings, zspacings, xspacing, yspacing, zspacing export minimum_xspacing, minimum_yspacing, minimum_zspacing -export column_heightᶜᶜᵃ, column_heightᶠᶜᵃ, column_heightᶜᶠᵃ, column_heightᶠᶠᵃ +export domain_depthᶜᶜᵃ, domain_depthᶠᶜᵃ, domain_depthᶜᶠᵃ, domain_depthᶠᶠᵃ export offset_data, new_data export on_architecture diff --git a/src/Grids/grid_utils.jl b/src/Grids/grid_utils.jl index 33d60205d1..efcefd2297 100644 --- a/src/Grids/grid_utils.jl +++ b/src/Grids/grid_utils.jl @@ -318,10 +318,10 @@ coordinate_summary(topo, Δ::Union{AbstractVector, AbstractMatrix}, name) = ##### Bottom height ##### -@inline column_heightᶜᶜᵃ(i, j, k, grid) = grid.Lz -@inline column_heightᶜᶠᵃ(i, j, k, grid) = grid.Lz -@inline column_heightᶠᶜᵃ(i, j, k, grid) = grid.Lz -@inline column_heightᶠᶠᵃ(i, j, k, grid) = grid.Lz +@inline domain_depthᶜᶜᵃ(i, j, grid) = grid.Lz +@inline domain_depthᶜᶠᵃ(i, j, grid) = grid.Lz +@inline domain_depthᶠᶜᵃ(i, j, grid) = grid.Lz +@inline domain_depthᶠᶠᵃ(i, j, grid) = grid.Lz ##### ##### Spherical geometry diff --git a/src/ImmersedBoundaries/ImmersedBoundaries.jl b/src/ImmersedBoundaries/ImmersedBoundaries.jl index 869b44dd8b..5805e04e05 100644 --- a/src/ImmersedBoundaries/ImmersedBoundaries.jl +++ b/src/ImmersedBoundaries/ImmersedBoundaries.jl @@ -53,7 +53,7 @@ import Oceananigans.Grids: architecture, with_halo, inflate_halo_size_one_dimens ξname, ηname, rname, node_names, xnodes, ynodes, znodes, λnodes, φnodes, nodes, ξnodes, ηnodes, rnodes, - column_heightᶜᶜᵃ, column_heightᶠᶜᵃ, column_heightᶜᶠᵃ, column_heightᶠᶠᵃ, + domain_depthᶜᶜᵃ, domain_depthᶠᶜᵃ, domain_depthᶜᶠᵃ, domain_depthᶠᶠᵃ, inactive_cell import Oceananigans.Coriolis: φᶠᶠᵃ diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 455d35666c..d70ac77cff 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -124,7 +124,7 @@ end return z ≤ zb end -@inline bottom_height(i, j, ibg::GFBIBG) = @inbounds ibg.immersed_boundary.bottom_height[i, j, 1] +@inline z_bottom(i, j, ibg::GFBIBG) = @inbounds ibg.immersed_boundary.bottom_height[i, j, 1] ##### ##### Bottom height @@ -132,16 +132,16 @@ end const AGFBIB = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} -@inline column_heightᶜᶜᵃ(i, j, k, ibg::AGFBIB) = @inbounds znode(i, j, ibg.Nz+1, ibg, c, c, f) - ibg.immersed_boundary.bottom_height[i, j, 1] -@inline column_heightᶜᶠᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶜᶜᵃ(i, j-1, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) -@inline column_heightᶠᶜᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶜᶜᵃ(i-1, j, k, ibg), column_heightᶜᶜᵃ(i, j, k, ibg)) -@inline column_heightᶠᶠᵃ(i, j, k, ibg::AGFBIB) = min(column_heightᶠᶜᵃ(i, j-1, k, ibg), column_heightᶠᶜᵃ(i, j, k, ibg)) +@inline domain_depthᶜᶜᵃ(i, j, ibg::AGFBIB) = @inbounds znode(i, j, ibg.Nz+1, ibg, c, c, f) - ibg.immersed_boundary.bottom_height[i, j, 1] +@inline domain_depthᶜᶠᵃ(i, j, ibg::AGFBIB) = min(domain_depthᶜᶜᵃ(i, j-1, ibg), domain_depthᶜᶜᵃ(i, j, ibg)) +@inline domain_depthᶠᶜᵃ(i, j, ibg::AGFBIB) = min(domain_depthᶜᶜᵃ(i-1, j, ibg), domain_depthᶜᶜᵃ(i, j, ibg)) +@inline domain_depthᶠᶠᵃ(i, j, ibg::AGFBIB) = min(domain_depthᶠᶜᵃ(i, j-1, ibg), domain_depthᶠᶜᵃ(i, j, ibg)) # Make sure column_height works for horizontally-Flat topologies. XFlatAGFIBG = ImmersedBoundaryGrid{<:Any, <:Flat, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} YFlatAGFIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Flat, <:Any, <:Any, <:AbstractGridFittedBottom} -@inline column_heightᶠᶜᵃ(i, j, k, ibg::XFlatAGFIBG) = column_heightᶜᶜᵃ(i, j, k, ibg) -@inline column_heightᶜᶠᵃ(i, j, k, ibg::YFlatAGFIBG) = column_heightᶜᶜᵃ(i, j, k, ibg) -@inline column_heightᶠᶠᵃ(i, j, k, ibg::XFlatAGFIBG) = column_heightᶜᶠᵃ(i, j, k, ibg) -@inline column_heightᶠᶠᵃ(i, j, k, ibg::YFlatAGFIBG) = column_heightᶠᶜᵃ(i, j, k, ibg) +@inline domain_depthᶠᶜᵃ(i, j, k, ibg::XFlatAGFIBG) = domain_depthᶜᶜᵃ(i, j, ibg) +@inline domain_depthᶜᶠᵃ(i, j, k, ibg::YFlatAGFIBG) = domain_depthᶜᶜᵃ(i, j, ibg) +@inline domain_depthᶠᶠᵃ(i, j, k, ibg::XFlatAGFIBG) = domain_depthᶜᶠᵃ(i, j, ibg) +@inline domain_depthᶠᶠᵃ(i, j, k, ibg::YFlatAGFIBG) = domain_depthᶠᶜᵃ(i, j, ibg) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index a0226762f8..6b118d0868 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -183,8 +183,8 @@ end advance_previous_velocity!(i, j, k_top-1, timestepper, U, Uᵐ⁻¹, Uᵐ⁻²) advance_previous_velocity!(i, j, k_top-1, timestepper, V, Vᵐ⁻¹, Vᵐ⁻²) - Hᶠᶜ = column_heightᶠᶜᵃ(i, j, k_top-1, grid) - Hᶜᶠ = column_heightᶜᶠᵃ(i, j, k_top-1, grid) + Hᶠᶜ = domain_depthᶠᶜᵃ(i, j, grid) + Hᶜᶠ = domain_depthᶜᶠᵃ(i, j, grid) # ∂τ(U) = - ∇η + G U[i, j, k_top-1] += Δτ * (- g * Hᶠᶜ * ∂xᶠᶜᶠ_η(i, j, k_top, grid, TX, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gᵁ[i, j, k_top-1]) @@ -271,8 +271,8 @@ end k_top = grid.Nz+1 @inbounds begin - Hᶠᶜ = column_heightᶠᶜᵃ(i, j, k_top-1, grid) - Hᶜᶠ = column_heightᶜᶠᵃ(i, j, k_top-1, grid) + Hᶠᶜ = domain_depthᶠᶜᵃ(i, j, grid) + Hᶜᶠ = domain_depthᶜᶠᵃ(i, j, grid) u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / Hᶠᶜ v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / Hᶜᶠ diff --git a/src/TurbulenceClosures/TurbulenceClosures.jl b/src/TurbulenceClosures/TurbulenceClosures.jl index 7046a2f1cc..c32faf9bae 100644 --- a/src/TurbulenceClosures/TurbulenceClosures.jl +++ b/src/TurbulenceClosures/TurbulenceClosures.jl @@ -119,15 +119,14 @@ end const c = Center() const f = Face() -@inline z_top(i, j, grid) = znode(i, j, grid.Nz+1, grid, c, c, f) -@inline bottom_height(i, j, grid) = znode(i, j, 1, grid, c, c, f) +@inline z_top(i, j, grid) = znode(i, j, grid.Nz+1, grid, c, c, f) +@inline z_bottom(i, j, grid) = znode(i, j, 1, grid, c, c, f) -@inline depthᶜᶜᶠ(i, j, k, grid) = clip(z_top(i, j, grid) - znode(i, j, k, grid, c, c, f)) -@inline depthᶜᶜᶜ(i, j, k, grid) = clip(z_top(i, j, grid) - znode(i, j, k, grid, c, c, c)) -@inline total_depthᶜᶜᵃ(i, j, grid) = clip(z_top(i, j, grid) - bottom_height(i, j, grid)) +@inline depthᶜᶜᶠ(i, j, k, grid) = clip(z_top(i, j, grid) - znode(i, j, k, grid, c, c, f)) +@inline depthᶜᶜᶜ(i, j, k, grid) = clip(z_top(i, j, grid) - znode(i, j, k, grid, c, c, c)) @inline function height_above_bottomᶜᶜᶠ(i, j, k, grid) - h = znode(i, j, k, grid, c, c, f) - bottom_height(i, j, grid) + h = znode(i, j, k, grid, c, c, f) - z_bottom(i, j, grid) # Limit by thickness of cell below Δz = Δzᶜᶜᶜ(i, j, k-1, grid) @@ -136,7 +135,7 @@ end @inline function height_above_bottomᶜᶜᶜ(i, j, k, grid) Δz = Δzᶜᶜᶜ(i, j, k, grid) - h = znode(i, j, k, grid, c, c, c) - bottom_height(i, j, grid) + h = znode(i, j, k, grid, c, c, c) - z_bottom(i, j, grid) return max(Δz/2, h) end diff --git a/src/TurbulenceClosures/turbulence_closure_implementations/TKEBasedVerticalDiffusivities/catke_equation.jl b/src/TurbulenceClosures/turbulence_closure_implementations/TKEBasedVerticalDiffusivities/catke_equation.jl index a08fb4b969..73ddb5310c 100644 --- a/src/TurbulenceClosures/turbulence_closure_implementations/TKEBasedVerticalDiffusivities/catke_equation.jl +++ b/src/TurbulenceClosures/turbulence_closure_implementations/TKEBasedVerticalDiffusivities/catke_equation.jl @@ -58,7 +58,7 @@ end ℓ★ = ifelse(isnan(ℓ★), zero(grid), ℓ★) ℓᴰ = max(ℓ★, ℓʰ) - H = total_depthᶜᶜᵃ(i, j, grid) + H = domain_depthᶜᶜᵃ(i, j, grid) return min(H, ℓᴰ) end diff --git a/src/TurbulenceClosures/turbulence_closure_implementations/TKEBasedVerticalDiffusivities/catke_mixing_length.jl b/src/TurbulenceClosures/turbulence_closure_implementations/TKEBasedVerticalDiffusivities/catke_mixing_length.jl index 8b2620f6b0..19e81270fb 100644 --- a/src/TurbulenceClosures/turbulence_closure_implementations/TKEBasedVerticalDiffusivities/catke_mixing_length.jl +++ b/src/TurbulenceClosures/turbulence_closure_implementations/TKEBasedVerticalDiffusivities/catke_mixing_length.jl @@ -5,7 +5,7 @@ using ..TurbulenceClosures: height_above_bottomᶜᶜᶠ, depthᶜᶜᶜ, height_above_bottomᶜᶜᶜ, - total_depthᶜᶜᵃ + domain_depthᶜᶜᵃ """ struct CATKEMixingLength{FT} @@ -232,7 +232,7 @@ end ℓ★ = ifelse(isnan(ℓ★), zero(grid), ℓ★) ℓu = max(ℓ★, ℓʰ) - H = total_depthᶜᶜᵃ(i, j, grid) + H = domain_depthᶜᶜᵃ(i, j, grid) return min(H, ℓu) end @@ -252,7 +252,7 @@ end ℓ★ = ifelse(isnan(ℓ★), zero(grid), ℓ★) ℓc = max(ℓ★, ℓʰ) - H = total_depthᶜᶜᵃ(i, j, grid) + H = domain_depthᶜᶜᵃ(i, j, grid) return min(H, ℓc) end @@ -272,7 +272,7 @@ end ℓ★ = ifelse(isnan(ℓ★), zero(grid), ℓ★) ℓe = max(ℓ★, ℓʰ) - H = total_depthᶜᶜᵃ(i, j, grid) + H = domain_depthᶜᶜᵃ(i, j, grid) return min(H, ℓe) end From cc9cd4ff0697019dcfee2be7c17cfcc02f970479 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 11 Oct 2024 17:13:23 +0200 Subject: [PATCH 365/567] some remaining `z_bottom`s --- src/ImmersedBoundaries/ImmersedBoundaries.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ImmersedBoundaries/ImmersedBoundaries.jl b/src/ImmersedBoundaries/ImmersedBoundaries.jl index 5805e04e05..f823f54dfd 100644 --- a/src/ImmersedBoundaries/ImmersedBoundaries.jl +++ b/src/ImmersedBoundaries/ImmersedBoundaries.jl @@ -94,7 +94,7 @@ import Oceananigans.TurbulenceClosures: νᶠᶠᶜ, νᶜᶠᶠ, νᶠᶜᶠ, - bottom_height + z_bottom import Oceananigans.Fields: fractional_x_index, fractional_y_index, fractional_z_index @@ -162,7 +162,7 @@ with_halo(halo, ibg::ImmersedBoundaryGrid) = inflate_halo_size_one_dimension(req_H, old_H, _, ::IBG) = max(req_H + 1, old_H) inflate_halo_size_one_dimension(req_H, old_H, ::Type{Flat}, ::IBG) = 0 -@inline bottom_height(i, j, ibg::IBG) = error("The function `bottom` has not been defined for $(summary(ibg))!") +@inline z_bottom(i, j, ibg::IBG) = error("The function `bottom` has not been defined for $(summary(ibg))!") function Base.summary(grid::ImmersedBoundaryGrid) FT = eltype(grid) From 45b5cd4296e9c646df15e2bffaa4573682632ad1 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 11 Oct 2024 17:14:23 +0200 Subject: [PATCH 366/567] back as it was --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index d70ac77cff..b54e75921e 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -33,7 +33,7 @@ Base.summary(::InterfaceImmersedCondition) = "InterfaceImmersedCondition" const GFBIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:GridFittedBottom} """ - GridFittedBottom(bottom_height) + GridFittedBottom(bottom_height, [immersed_condition=CenterImmersedCondition()]) Return a bottom immersed boundary. From 3151408d0b0d0dd635f0c3b81db6f9b3e6aeb115 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 16 Oct 2024 12:04:04 +0200 Subject: [PATCH 367/567] use new bottom --- .../z_star_vertical_spacing.jl | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 3a0f04c899..4bc2c4b7f2 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -21,10 +21,6 @@ function update_grid!(model, grid::ZStarSpacingGrid; parameters = :xy) η_grid = grid.zᵃᵃᶠ.∂t_s # Free surface variables - Hᶜᶜ = model.free_surface.auxiliary.Hᶜᶜ - Hᶠᶜ = model.free_surface.auxiliary.Hᶠᶜ - Hᶜᶠ = model.free_surface.auxiliary.Hᶜᶠ - Hᶠᶠ = model.free_surface.auxiliary.Hᶠᶠ U̅ = model.free_surface.state.U̅ V̅ = model.free_surface.state.V̅ η = model.free_surface.η @@ -32,14 +28,14 @@ function update_grid!(model, grid::ZStarSpacingGrid; parameters = :xy) # Update vertical spacing with available parameters # No need to fill the halo as the scaling is updated _IN_ the halos launch!(architecture(grid), grid, parameters, _update_zstar!, - sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, sᶠᶜ⁻, sᶜᶠ⁻, η_grid, η, Hᶜᶜ, Hᶠᶜ, Hᶜᶠ, Hᶠᶠ, grid) + sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, sᶠᶜ⁻, sᶜᶠ⁻, η_grid, η, grid) # Update the time derivative of the grid-scaling. Note that in this case we leverage the # free surface evolution equation, where the time derivative of the free surface is equal # to the divergence of the vertically integrated velocity field, such that # ∂ₜ((H + η) / H) = H⁻¹ ∂ₜη = - H⁻¹ ∇ ⋅ ∫udz launch!(architecture(grid), grid, parameters, _update_∂t_s!, - ∂t_s, U̅, V̅, Hᶜᶜ, grid) + ∂t_s, U̅, V̅, grid) return nothing end @@ -49,23 +45,28 @@ end @kernel function _update_∂t_s!(∂t_s, U̅, V̅, Hᶜᶜ, grid) i, j = @index(Global, NTuple) k_top = grid.Nz + 1 - TX, TY, _ = topology(grid) + Hᶜᶜ = domain_depthᶜᶜᵃ(i, j, grid) @inbounds begin # ∂(η / H)/∂t = - ∇ ⋅ ∫udz / H ∂t_s[i, j, 1] = - 1 / Azᶜᶜᶠ(i, j, k_top-1, grid) * (δxᶜᶜᶠ(i, j, k_top-1, grid, Δy_qᶠᶜᶠ, U̅) + - δyᶜᶜᶠ(i, j, k_top-1, grid, Δx_qᶜᶠᶠ, V̅)) / Hᶜᶜ[i, j, 1] - end + δyᶜᶜᶠ(i, j, k_top-1, grid, Δx_qᶜᶠᶠ, V̅)) / Hᶜᶜ end -@kernel function _update_zstar!(sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, sᶠᶜ⁻, sᶜᶠ⁻, η_grid, η, Hᶜᶜ, Hᶠᶜ, Hᶜᶠ, Hᶠᶠ, grid) +@kernel function _update_zstar!(sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, sᶠᶜ⁻, sᶜᶠ⁻, η_grid, η, grid) i, j = @index(Global, NTuple) k_top = grid.Nz+1 + + Hᶜᶜ = domain_depthᶜᶜᵃ(i, j, grid) + Hᶠᶜ = domain_depthᶠᶜᵃ(i, j, grid) + Hᶜᶠ = domain_depthᶜᶠᵃ(i, j, grid) + Hᶠᶠ = domain_depthᶠᶠᵃ(i, j, grid) + @inbounds begin - hᶜᶜ = (Hᶜᶜ[i, j, 1] + η[i, j, k_top]) / Hᶜᶜ[i, j, 1] - hᶠᶜ = (Hᶠᶜ[i, j, 1] + ℑxᶠᵃᵃ(i, j, k_top, grid, η)) / Hᶠᶜ[i, j, 1] - hᶜᶠ = (Hᶜᶠ[i, j, 1] + ℑyᵃᶠᵃ(i, j, k_top, grid, η)) / Hᶜᶠ[i, j, 1] - hᶠᶠ = (Hᶠᶠ[i, j, 1] + ℑxyᶠᶠᵃ(i, j, k_top, grid, η)) / Hᶠᶠ[i, j, 1] + hᶜᶜ = (Hᶜᶜ + η[i, j, k_top]) / Hᶜᶜ + hᶠᶜ = (Hᶠᶜ + ℑxᶠᵃᵃ(i, j, k_top, grid, η)) / Hᶠᶜ + hᶜᶠ = (Hᶜᶠ + ℑyᵃᶠᵃ(i, j, k_top, grid, η)) / Hᶜᶠ + hᶠᶠ = (Hᶠᶠ + ℑxyᶠᶠᵃ(i, j, k_top, grid, η)) / Hᶠᶠ sᶜᶜ⁻[i, j] = sᶜᶜⁿ[i, j] sᶠᶜ⁻[i, j] = sᶠᶜⁿ[i, j] @@ -86,10 +87,10 @@ end ##### ZStar-specific implementation of the additional terms to be included in the momentum equations ##### -@inline z_minus_rᶜᶜᶜ(i, j, k, grid, η, Hᶜᶜ) = @inbounds η[i, j, grid.Nz+1] * (1 + rnode(i, j, k, grid, Center(), Center(), Center()) / Hᶜᶜ[i, j, 1]) +@inline z_minus_rᶜᶜᶜ(i, j, k, grid, η) = @inbounds η[i, j, grid.Nz+1] * (1 + rnode(i, j, k, grid, Center(), Center(), Center()) / domain_depthᶜᶜᵃ(i, j, grid)) -@inline ∂x_z(i, j, k, grid, free_surface) = @inbounds ∂xᶠᶜᶜ(i, j, k, grid, z_minus_rᶜᶜᶜ, free_surface.η, free_surface.auxiliary.Hᶜᶜ) -@inline ∂y_z(i, j, k, grid, free_surface) = @inbounds ∂yᶜᶠᶜ(i, j, k, grid, z_minus_rᶜᶜᶜ, free_surface.η, free_surface.auxiliary.Hᶜᶜ) +@inline ∂x_z(i, j, k, grid, free_surface) = @inbounds ∂xᶠᶜᶜ(i, j, k, grid, z_minus_rᶜᶜᶜ, free_surface.η) +@inline ∂y_z(i, j, k, grid, free_surface) = @inbounds ∂yᶜᶠᶜ(i, j, k, grid, z_minus_rᶜᶜᶜ, free_surface.η) @inline grid_slope_contribution_x(i, j, k, grid::ZStarSpacingGrid, free_surface, ::Nothing, model_fields) = zero(grid) @inline grid_slope_contribution_y(i, j, k, grid::ZStarSpacingGrid, free_surface, ::Nothing, model_fields) = zero(grid) From 52870c9aba9764fafed95b41632727a1a173d57f Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 16 Oct 2024 12:25:28 +0200 Subject: [PATCH 368/567] some correction --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 2 +- .../split_explicit_free_surface.jl | 2 +- .../split_explicit_free_surface_kernels.jl | 17 +++++----------- .../z_star_vertical_spacing.jl | 20 ++++++++++--------- ...ulti_region_split_explicit_free_surface.jl | 3 +-- 5 files changed, 19 insertions(+), 25 deletions(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index b54e75921e..1865672dce 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -7,7 +7,7 @@ using Oceananigans.Fields: fill_halo_regions! using Oceananigans.BoundaryConditions: FBC using Printf -import Oceananigans.TurbulenceClosures: bottom_height +import Oceananigans.TurbulenceClosures: z_bottom ##### ##### GridFittedBottom (2.5D immersed boundary with modified bottom height) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index 9f7046b32d..3a5661410b 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -214,7 +214,7 @@ large (or `:xy` in case of a serial computation), and start computing from $(FIELDS) """ -Base.@kwdef struct SplitExplicitAuxiliaryFields{𝒞ℱ, ℱ𝒞, 𝒞𝒞, ℱℱ, 𝒦} +Base.@kwdef struct SplitExplicitAuxiliaryFields{ℱ𝒞, 𝒞ℱ, 𝒦} "Vertically-integrated slow barotropic forcing function for `U` (`ReducedField` over ``z``)" Gᵁ :: ℱ𝒞 "Vertically-integrated slow barotropic forcing function for `V` (`ReducedField` over ``z``)" diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 55fb3b500a..0e80dab13e 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -105,7 +105,6 @@ for Topo in [:Periodic, :Bounded, :RightConnected, :LeftConnected] end # Interpolation - const PGX = AbstractGrid{<:Any, <:Periodic} const BGX = AbstractGrid{<:Any, <:Bounded} const RGX = AbstractGrid{<:Any, <:RightConnected} @@ -234,11 +233,8 @@ end i, j = @index(Global, NTuple) k_top = grid.Nz + 1 - Hᶠᶜ = domain_depthᶠᶜᵃ(i, j, grid) - Hᶜᶠ = domain_depthᶜᶠᵃ(i, j, grid) - - sᶠᶜ = @inbounds (η[i, j, k_top] + Hᶠᶜ) / Hᶠᶜ - sᶜᶠ = @inbounds (η[i, j, k_top] + Hᶜᶠ) / Hᶜᶠ + sᶠᶜ = dynamic_domain_depthᶜᶠᵃ(i, j, k_top, grid, η) / domain_depthᶠᶜᵃ(i, j, grid) + sᶜᶠ = dynamic_domain_depthᶜᶠᵃ(i, j, k_top, grid, η) / domain_depthᶜᶠᵃ(i, j, grid) # hand unroll first loop @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δrᶠᶜᶜ(i, j, 1, grid) * sᶠᶜ @@ -255,11 +251,8 @@ end i, j = active_linear_index_to_tuple(idx, active_cells_map) k_top = grid.Nz+1 - Hᶠᶜ = domain_depthᶠᶜᵃ(i, j, grid) - Hᶜᶠ = domain_depthᶜᶠᵃ(i, j, grid) - - sᶠᶜ = @inbounds (η[i, j, k_top] + Hᶠᶜ) / Hᶠᶜ - sᶜᶠ = @inbounds (η[i, j, k_top] + Hᶜᶠ) / Hᶜᶠ + sᶠᶜ = dynamic_domain_depthᶜᶠᵃ(i, j, k_top, grid, η) / domain_depthᶠᶜᵃ(i, j, grid) + sᶜᶠ = dynamic_domain_depthᶜᶠᵃ(i, j, k_top, grid, η) / domain_depthᶜᶠᵃ(i, j, grid) # hand unroll first loop @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δrᶠᶜᶜ(i, j, 1, grid) * sᶠᶜ @@ -289,7 +282,7 @@ end @inbounds begin u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / Hᶠᶜ v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / Hᶜᶠ - enH + end end function barotropic_split_explicit_corrector!(u, v, free_surface, grid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 4bc2c4b7f2..fb7060a4fa 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -1,3 +1,4 @@ +using Oceananigans.Grids using Oceananigans.Grids: ZStarUnderlyingGrid, rnode using Oceananigans.ImmersedBoundaries: ImmersedZStarGrid @@ -42,7 +43,7 @@ end # NOTE: The ZStar vertical spacing only supports a SplitExplicitFreeSurface # TODO: extend to support other free surface solvers -@kernel function _update_∂t_s!(∂t_s, U̅, V̅, Hᶜᶜ, grid) +@kernel function _update_∂t_s!(∂t_s, U̅, V̅, grid) i, j = @index(Global, NTuple) k_top = grid.Nz + 1 Hᶜᶜ = domain_depthᶜᶜᵃ(i, j, grid) @@ -51,22 +52,23 @@ end # ∂(η / H)/∂t = - ∇ ⋅ ∫udz / H ∂t_s[i, j, 1] = - 1 / Azᶜᶜᶠ(i, j, k_top-1, grid) * (δxᶜᶜᶠ(i, j, k_top-1, grid, Δy_qᶠᶜᶠ, U̅) + δyᶜᶜᶠ(i, j, k_top-1, grid, Δx_qᶜᶠᶠ, V̅)) / Hᶜᶜ + end end @kernel function _update_zstar!(sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, sᶠᶜ⁻, sᶜᶠ⁻, η_grid, η, grid) i, j = @index(Global, NTuple) k_top = grid.Nz+1 - Hᶜᶜ = domain_depthᶜᶜᵃ(i, j, grid) - Hᶠᶜ = domain_depthᶠᶜᵃ(i, j, grid) - Hᶜᶠ = domain_depthᶜᶠᵃ(i, j, grid) - Hᶠᶠ = domain_depthᶠᶠᵃ(i, j, grid) + hᶜᶜ = domain_depthᶜᶜᵃ(i, j, grid) + hᶠᶜ = domain_depthᶠᶜᵃ(i, j, grid) + hᶜᶠ = domain_depthᶜᶠᵃ(i, j, grid) + hᶠᶠ = domain_depthᶠᶠᵃ(i, j, grid) @inbounds begin - hᶜᶜ = (Hᶜᶜ + η[i, j, k_top]) / Hᶜᶜ - hᶠᶜ = (Hᶠᶜ + ℑxᶠᵃᵃ(i, j, k_top, grid, η)) / Hᶠᶜ - hᶜᶠ = (Hᶜᶠ + ℑyᵃᶠᵃ(i, j, k_top, grid, η)) / Hᶜᶠ - hᶠᶠ = (Hᶠᶠ + ℑxyᶠᶠᵃ(i, j, k_top, grid, η)) / Hᶠᶠ + Hᶜᶜ = (hᶜᶜ + η[i, j, k_top]) / hᶜᶜ + Hᶠᶜ = (hᶠᶜ + ℑxᶠᵃᵃ(i, j, k_top, grid, η)) / hᶠᶜ + Hᶜᶠ = (hᶜᶠ + ℑyᵃᶠᵃ(i, j, k_top, grid, η)) / hᶜᶠ + Hᶠᶠ = (hᶠᶠ + ℑxyᶠᶠᵃ(i, j, k_top, grid, η)) / hᶠᶠ sᶜᶜ⁻[i, j] = sᶜᶜⁿ[i, j] sᶠᶜ⁻[i, j] = sᶠᶜⁿ[i, j] diff --git a/src/MultiRegion/multi_region_split_explicit_free_surface.jl b/src/MultiRegion/multi_region_split_explicit_free_surface.jl index 45d8f42a34..10ba130b33 100644 --- a/src/MultiRegion/multi_region_split_explicit_free_surface.jl +++ b/src/MultiRegion/multi_region_split_explicit_free_surface.jl @@ -4,8 +4,7 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: SplitExplicitFreeSurface SplitExplicitSettings, SplitExplicitState, FixedSubstepNumber, FixedTimeStepSize, - calculate_substeps, - compute_column_height! + calculate_substeps import Oceananigans.Models.HydrostaticFreeSurfaceModels: materialize_free_surface, SplitExplicitAuxiliaryFields From 9f97e901ff29c918282b2372b3c766526437b650 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 16 Oct 2024 12:33:31 +0200 Subject: [PATCH 369/567] constructor had an error --- src/Advection/vector_invariant_advection.jl | 8 +++++++- .../z_star_vertical_spacing.jl | 15 ++++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/Advection/vector_invariant_advection.jl b/src/Advection/vector_invariant_advection.jl index a69251fd8a..09883e07be 100644 --- a/src/Advection/vector_invariant_advection.jl +++ b/src/Advection/vector_invariant_advection.jl @@ -212,7 +212,13 @@ function WENOVectorInvariant(FT::DataType = Float64; upwinding = nothing_to_default(upwinding; default = default_upwinding) schemes = (vorticity_scheme, vertical_scheme, kinetic_energy_gradient_scheme, divergence_scheme) - N = maximum(required_halo_size(s) for s in schemes) + + NX = maximum(required_halo_size_x(s) for s in schemes) + NY = maximum(required_halo_size_y(s) for s in schemes) + NZ = maximum(required_halo_size_z(s) for s in schemes) + + N = max(NX, NY, NZ) + FT = eltype(vorticity_scheme) # assumption return VectorInvariant{N, FT, multi_dimensional_stencil}(vorticity_scheme, diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index fb7060a4fa..3cb9890dce 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -46,12 +46,13 @@ end @kernel function _update_∂t_s!(∂t_s, U̅, V̅, grid) i, j = @index(Global, NTuple) k_top = grid.Nz + 1 - Hᶜᶜ = domain_depthᶜᶜᵃ(i, j, grid) @inbounds begin # ∂(η / H)/∂t = - ∇ ⋅ ∫udz / H - ∂t_s[i, j, 1] = - 1 / Azᶜᶜᶠ(i, j, k_top-1, grid) * (δxᶜᶜᶠ(i, j, k_top-1, grid, Δy_qᶠᶜᶠ, U̅) + - δyᶜᶜᶠ(i, j, k_top-1, grid, Δx_qᶜᶠᶠ, V̅)) / Hᶜᶜ + δ_U̅h = (δxᶜᶜᶠ(i, j, k_top-1, grid, Δy_qᶠᶜᶠ, U̅) + + δyᶜᶜᶠ(i, j, k_top-1, grid, Δx_qᶜᶠᶠ, V̅)) / Azᶜᶜᶠ(i, j, k_top-1, grid) + + ∂t_s[i, j, 1] = - δ_U̅h / domain_depthᶜᶜᵃ(i, j, grid) end end @@ -75,10 +76,10 @@ end sᶜᶠ⁻[i, j] = sᶜᶠⁿ[i, j] # update current and previous scaling - sᶜᶜⁿ[i, j] = hᶜᶜ - sᶠᶜⁿ[i, j] = hᶠᶜ - sᶜᶠⁿ[i, j] = hᶜᶠ - sᶠᶠⁿ[i, j] = hᶠᶠ + sᶜᶜⁿ[i, j] = Hᶜᶜ + sᶠᶜⁿ[i, j] = Hᶠᶜ + sᶜᶠⁿ[i, j] = Hᶜᶠ + sᶠᶠⁿ[i, j] = Hᶠᶠ # Update η in the grid η_grid[i, j] = η[i, j, k_top] From 53c87d7458d08c1ca8bbccf33ac37cdda78be5da Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 16 Oct 2024 13:17:38 +0200 Subject: [PATCH 370/567] getting there --- .../vector_invariant_cross_upwinding.jl | 3 +- .../fill_halo_regions_flux.jl | 38 +++++++++++++++++++ src/ImmersedBoundaries/grid_fitted_bottom.jl | 4 +- .../generalized_vertical_spacing.jl | 2 +- .../split_explicit_free_surface.jl | 31 +++++---------- .../split_explicit_free_surface_kernels.jl | 29 ++------------ .../z_star_vertical_spacing.jl | 19 ++++++---- .../spacings_and_areas_and_volumes.jl | 5 +++ validation/z_star_coordinate/lock_release.jl | 10 ++--- 9 files changed, 76 insertions(+), 65 deletions(-) diff --git a/src/Advection/vector_invariant_cross_upwinding.jl b/src/Advection/vector_invariant_cross_upwinding.jl index fdd4e6a975..052ae63be4 100644 --- a/src/Advection/vector_invariant_cross_upwinding.jl +++ b/src/Advection/vector_invariant_cross_upwinding.jl @@ -17,6 +17,8 @@ ##### Cross and Self Upwinding of the Divergence flux ##### +@inline V_times_∂t_grid(i, j, k, grid) = Vᶜᶜᶜ(i, j, k, grid) * ∂t_grid(i, j, k, grid) + @inline function upwinded_divergence_flux_Uᶠᶜᶜ(i, j, k, grid, scheme::VectorInvariantCrossVerticalUpwinding, u, v) @inbounds û = u[i, j, k] δ_stencil = scheme.upwinding.divergence_stencil @@ -34,6 +36,5 @@ end δᴿ = _biased_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, bias(v̂), flux_div_xyᶜᶜᶜ, δ_stencil, u, v) ∂ts = _symmetric_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, cross_scheme, V_times_∂t_grid) - return v̂ * (δᴿ + ∂ts) end diff --git a/src/BoundaryConditions/fill_halo_regions_flux.jl b/src/BoundaryConditions/fill_halo_regions_flux.jl index 64cbaf9fb2..13ec79d9e6 100644 --- a/src/BoundaryConditions/fill_halo_regions_flux.jl +++ b/src/BoundaryConditions/fill_halo_regions_flux.jl @@ -31,3 +31,41 @@ @inline _fill_north_halo!(i, k, grid, c, ::FBC, args...) = _fill_flux_north_halo!(i, 1, k, grid, c) @inline _fill_bottom_halo!(i, j, grid, c, ::FBC, args...) = _fill_flux_bottom_halo!(i, j, 1, grid, c) @inline _fill_top_halo!(i, j, grid, c, ::FBC, args...) = _fill_flux_top_halo!(i, j, 1, grid, c) + + +# @inline function _fill_west_halo!(j, k, grid, c, ::FBC, args...) +# for i in 1:grid.Hx +# _fill_flux_west_halo!(i, j, k, grid, c) +# end +# end + +# @inline function _fill_east_halo!(j, k, grid, c, ::FBC, args...) +# for i in 1:grid.Hx +# _fill_flux_east_halo!(i, j, k, grid, c) +# end +# end + +# @inline function _fill_south_halo!(i, k, grid, c, ::FBC, args...) +# for j in 1:grid.Hz +# _fill_flux_south_halo!(i, j, k, grid, c) +# end +# end + +# @inline function _fill_north_halo!(i, k, grid, c, ::FBC, args...) +# for j in 1:grid.Hz +# _fill_flux_north_halo!(i, j, k, grid, c) +# end +# end + +# @inline function _fill_bottom_halo!(i, j, grid, c, ::FBC, args...) +# for k in 1:grid.Hz +# _fill_flux_bottom_halo!(i, j, k, grid, c) +# end +# end + +# @inline function _fill_top_halo!(i, j, grid, c, ::FBC, args...) +# for k in 1:grid.Hz +# _fill_flux_top_halo!(i, j, k, grid, c) +# end +# end + diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 1865672dce..51ca7cf47b 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -2,7 +2,7 @@ using Adapt using CUDA: CuArray using OffsetArrays: OffsetArray using Oceananigans.Utils: getnamewrapper -using Oceananigans.Grids: total_size +using Oceananigans.Grids: total_size, rnode using Oceananigans.Fields: fill_halo_regions! using Oceananigans.BoundaryConditions: FBC using Printf @@ -132,7 +132,7 @@ end const AGFBIB = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} -@inline domain_depthᶜᶜᵃ(i, j, ibg::AGFBIB) = @inbounds znode(i, j, ibg.Nz+1, ibg, c, c, f) - ibg.immersed_boundary.bottom_height[i, j, 1] +@inline domain_depthᶜᶜᵃ(i, j, ibg::AGFBIB) = @inbounds rnode(i, j, ibg.Nz+1, ibg, c, c, f) - ibg.immersed_boundary.bottom_height[i, j, 1] @inline domain_depthᶜᶠᵃ(i, j, ibg::AGFBIB) = min(domain_depthᶜᶜᵃ(i, j-1, ibg), domain_depthᶜᶜᵃ(i, j, ibg)) @inline domain_depthᶠᶜᵃ(i, j, ibg::AGFBIB) = min(domain_depthᶜᶜᵃ(i-1, j, ibg), domain_depthᶜᶜᵃ(i, j, ibg)) @inline domain_depthᶠᶠᵃ(i, j, ibg::AGFBIB) = min(domain_depthᶠᶜᵃ(i, j-1, ibg), domain_depthᶠᶜᵃ(i, j, ibg)) diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index 0f4be4e521..045e0b7b63 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -6,7 +6,7 @@ using Oceananigans.Grids: AbstractGrid, AbstractUnderlyingGrid, halo_size, Abstr using Oceananigans.ImmersedBoundaries using Oceananigans.ImmersedBoundaries: ImmersedAbstractVerticalCoordinateGrid using Oceananigans.Utils: getnamewrapper -using Oceananigans.Grids: with_halo, znode, ∂t_grid, vertical_scaling, previous_vertical_scaling +using Oceananigans.Grids: with_halo, ∂t_grid, vertical_scaling, previous_vertical_scaling using Adapt using Printf diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index 3a5661410b..a69a505821 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -159,10 +159,10 @@ Base.@kwdef struct SplitExplicitState{CC, ACC, FC, AFC, CF, ACF} U̅ :: FC "The time-filtered barotropic meridional velocity. (`ReducedField` over ``z``)" V̅ :: CF - "The time-filtered barotropic zonal velocity. (`ReducedField` over ``z``)" - Ũ :: FC - "The time-filtered barotropic meridional velocity. (`ReducedField` over ``z``)" - Ṽ :: CF + # "The time-filtered barotropic zonal velocity. (`ReducedField` over ``z``)" + # Ũ :: FC + # "The time-filtered barotropic meridional velocity. (`ReducedField` over ``z``)" + # Ṽ :: CF end """ @@ -197,10 +197,8 @@ function SplitExplicitState(grid::AbstractGrid, timestepper) U̅ = deepcopy(U) V̅ = deepcopy(V) - Ũ = deepcopy(U) - Ṽ = deepcopy(V) - return SplitExplicitState(; ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅, Ũ, Ṽ) + return SplitExplicitState(; ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅) end """ @@ -282,7 +280,6 @@ a fixed number of substeps with time step size of `fractional_step_size * Δt_ba struct FixedSubstepNumber{B, F} fractional_step_size :: B averaging_weights :: F - mass_flux_weights :: F end function FixedTimeStepSize(grid; @@ -314,18 +311,8 @@ end averaging_weights = averaging_weights[1:idx] averaging_weights ./= sum(averaging_weights) - - mass_flux_weights = similar(averaging_weights) - - averaging_weights ./= sum(averaging_weights) - - for i in substeps:-1:1 - mass_flux_weights[i] = sum(averaging_weights[i:substeps]) - end - - mass_flux_weights ./= sum(mass_flux_weights) - - return Δτ, tuple(averaging_weights...), tuple(mass_flux_weights...) + + return Δτ, tuple(averaging_weights...) end function SplitExplicitSettings(grid = nothing; @@ -370,8 +357,8 @@ function SplitExplicitSettings(grid = nothing; end end - fractional_step_size, averaging_weights, mass_flux_weights = weights_from_substeps(FT, substeps, averaging_kernel) - substepping = FixedSubstepNumber(fractional_step_size, averaging_weights, mass_flux_weights) + fractional_step_size, averaging_weights = weights_from_substeps(FT, substeps, averaging_kernel) + substepping = FixedSubstepNumber(fractional_step_size, averaging_weights) return SplitExplicitSettings(substepping, timestepper, settings_kwargs) end diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 0e80dab13e..92e2d83875 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -229,7 +229,7 @@ end # For Zstar vertical spacing the vertical integral includes the dynamic height # Remember, the vertical coordinate has not yet been updated! # For this reason the integration has to be performed manually -@kernel function _barotropic_mode_kernel!(U, V, grid, ::Nothing, u, v, η) +@kernel function _barotropic_mode_kernel!(U, V, grid, u, v, η) i, j = @index(Global, NTuple) k_top = grid.Nz + 1 @@ -246,31 +246,8 @@ end end end -@kernel function _barotropic_mode_kernel!(U, V, grid, active_cells_map, u, v, η) - idx = @index(Global, Linear) - i, j = active_linear_index_to_tuple(idx, active_cells_map) - k_top = grid.Nz+1 - - sᶠᶜ = dynamic_domain_depthᶜᶠᵃ(i, j, k_top, grid, η) / domain_depthᶠᶜᵃ(i, j, grid) - sᶜᶠ = dynamic_domain_depthᶜᶠᵃ(i, j, k_top, grid, η) / domain_depthᶜᶠᵃ(i, j, grid) - - # hand unroll first loop - @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δrᶠᶜᶜ(i, j, 1, grid) * sᶠᶜ - @inbounds V[i, j, k_top-1] = v[i, j, 1] * Δrᶜᶠᶜ(i, j, 1, grid) * sᶜᶠ - - @unroll for k in 2:grid.Nz - @inbounds U[i, j, k_top-1] += u[i, j, k] * Δrᶠᶜᶜ(i, j, k, grid) * sᶠᶜ - @inbounds V[i, j, k_top-1] += v[i, j, k] * Δrᶜᶠᶜ(i, j, k, grid) * sᶜᶠ - end -end - -@inline function compute_barotropic_mode!(U, V, grid, u, v, η) - active_cells_map = retrieve_surface_active_cells_map(grid) - - launch!(architecture(grid), grid, :xy, _barotropic_mode_kernel!, U, V, grid, active_cells_map, u, v, η; active_cells_map) - - return nothing -end +@inline compute_barotropic_mode!(U, V, grid, u, v, η) = + launch!(architecture(grid), grid, :xy, _barotropic_mode_kernel!, U, V, grid, u, v, η) @kernel function _barotropic_split_explicit_corrector!(u, v, grid, U̅, V̅, U, V, η) i, j, k = @index(Global, NTuple) diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 3cb9890dce..d3511238de 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -21,7 +21,10 @@ function update_grid!(model, grid::ZStarSpacingGrid; parameters = :xy) ∂t_s = grid.Δzᵃᵃᶠ.∂t_s η_grid = grid.zᵃᵃᶠ.∂t_s - # Free surface variables + # Free surface variables + # At the moment only SplitExplicitFreeSurface is supported, + # but this can be extended to other free surface solvers by calculating + # the barotropic velocity in this step U̅ = model.free_surface.state.U̅ V̅ = model.free_surface.state.V̅ η = model.free_surface.η @@ -46,14 +49,14 @@ end @kernel function _update_∂t_s!(∂t_s, U̅, V̅, grid) i, j = @index(Global, NTuple) k_top = grid.Nz + 1 + # ∂(η / H)/∂t = - ∇ ⋅ ∫udz / H + δx_U = ∂xᶠᶜᶜ(i, j, k_top-1, grid, U̅) + δy_V = ∂yᶜᶠᶜ(i, j, k_top-1, grid, V̅) - @inbounds begin - # ∂(η / H)/∂t = - ∇ ⋅ ∫udz / H - δ_U̅h = (δxᶜᶜᶠ(i, j, k_top-1, grid, Δy_qᶠᶜᶠ, U̅) + - δyᶜᶜᶠ(i, j, k_top-1, grid, Δx_qᶜᶠᶠ, V̅)) / Azᶜᶜᶠ(i, j, k_top-1, grid) + δ_U̅h = (δx_U + δy_V) / Azᶜᶜᶠ(i, j, k_top-1, grid) + H = domain_depthᶜᶜᵃ(i, j, grid) - ∂t_s[i, j, 1] = - δ_U̅h / domain_depthᶜᶜᵃ(i, j, grid) - end + @inbounds ∂t_s[i, j] = - δ_U̅h / H end @kernel function _update_zstar!(sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, sᶠᶜ⁻, sᶜᶠ⁻, η_grid, η, grid) @@ -66,7 +69,7 @@ end hᶠᶠ = domain_depthᶠᶠᵃ(i, j, grid) @inbounds begin - Hᶜᶜ = (hᶜᶜ + η[i, j, k_top]) / hᶜᶜ + Hᶜᶜ = (hᶜᶜ + η[i, j, k_top]) / hᶜᶜ Hᶠᶜ = (hᶠᶜ + ℑxᶠᵃᵃ(i, j, k_top, grid, η)) / hᶠᶜ Hᶜᶠ = (hᶜᶠ + ℑyᵃᶠᵃ(i, j, k_top, grid, η)) / hᶜᶠ Hᶠᶠ = (hᶠᶠ + ℑxyᶠᶠᵃ(i, j, k_top, grid, η)) / hᶠᶠ diff --git a/src/Operators/spacings_and_areas_and_volumes.jl b/src/Operators/spacings_and_areas_and_volumes.jl index 3b793f48f1..d85f427f01 100644 --- a/src/Operators/spacings_and_areas_and_volumes.jl +++ b/src/Operators/spacings_and_areas_and_volumes.jl @@ -296,6 +296,11 @@ for LX in (:Center, :Face) @inline $func(i, j, k, grid, ::$LX, ::$LY, ::$LZ) = $metric(i, j, k, grid) end end + + metric_function = Symbol(:Δr, location_code(LXe, LYe, LZe)) + @eval begin + @inline Δr(i, j, k, grid, ::$LX, ::$LY, ::$LZ) = $metric_function(i, j, k, grid) + end end end end diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index 017d04fc77..75d9f04291 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -15,11 +15,11 @@ grid = RectilinearGrid(size = (20, 20), halo = (6, 6), topology = (Bounded, Flat, Bounded)) -grid = ImmersedBoundaryGrid(grid, GridFittedBottom(x -> x < 32kilometers ? -10 : -20)) +# grid = ImmersedBoundaryGrid(grid, GridFittedBottom(x -> x < 32kilometers ? -10 : -20)) model = HydrostaticFreeSurfaceModel(; grid, - momentum_advection = WENO(; order = 5), - tracer_advection = WENO(; order = 5), + momentum_advection = FluxFormAdvection(WENO(; order = 5), nothing, WENO(; order = 5)), + tracer_advection = FluxFormAdvection(WENO(; order = 5), nothing, WENO(; order = 5)), buoyancy = BuoyancyTracer(), closure = nothing, tracers = :b, @@ -37,7 +37,7 @@ set!(model, b = bᵢ) @info "the time step is $Δt" -simulation = Simulation(model; Δt, stop_iteration = 10000000, stop_time = 17hours) +simulation = Simulation(model; Δt, stop_iteration = 1, stop_time = 17hours) Δz = GridMetricOperation((Center, Center, Center), Oceananigans.AbstractOperations.Δz, model.grid) @@ -63,7 +63,7 @@ function progress(sim) return nothing end -simulation.callbacks[:progress] = Callback(progress, IterationInterval(100)) +simulation.callbacks[:progress] = Callback(progress, IterationInterval(1)) run!(simulation) From f668c5a75f7096bb66b7452e2f65c9bb60b0b114 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 16 Oct 2024 13:29:02 +0200 Subject: [PATCH 371/567] another test --- src/Grids/z_star_vertical_coordinate.jl | 1 + .../split_explicit_free_surface.jl | 1 - .../z_star_vertical_spacing.jl | 9 ++++++--- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/Grids/z_star_vertical_coordinate.jl b/src/Grids/z_star_vertical_coordinate.jl index 911a281807..d8f1b9da39 100644 --- a/src/Grids/z_star_vertical_coordinate.jl +++ b/src/Grids/z_star_vertical_coordinate.jl @@ -239,6 +239,7 @@ const f = Face() @inline znode(i, j, k, grid::ZSG, ::C, ::F, ::C) = @inbounds grid.zᵃᵃᶜ.reference[k] * vertical_scaling(i, j, k, grid, c, f, c) + grid.zᵃᵃᶜ.∂t_s[i, j] @inline znode(i, j, k, grid::ZSG, ::F, ::C, ::C) = @inbounds grid.zᵃᵃᶜ.reference[k] * vertical_scaling(i, j, k, grid, f, c, c) + grid.zᵃᵃᶜ.∂t_s[i, j] @inline znode(i, j, k, grid::ZSG, ::F, ::F, ::C) = @inbounds grid.zᵃᵃᶜ.reference[k] * vertical_scaling(i, j, k, grid, f, f, c) + grid.zᵃᵃᶜ.∂t_s[i, j] + @inline znode(i, j, k, grid::ZSG, ::C, ::C, ::F) = @inbounds grid.zᵃᵃᶠ.reference[k] * vertical_scaling(i, j, k, grid, c, c, c) + grid.zᵃᵃᶠ.∂t_s[i, j] @inline znode(i, j, k, grid::ZSG, ::C, ::F, ::F) = @inbounds grid.zᵃᵃᶠ.reference[k] * vertical_scaling(i, j, k, grid, c, f, c) + grid.zᵃᵃᶠ.∂t_s[i, j] @inline znode(i, j, k, grid::ZSG, ::F, ::C, ::F) = @inbounds grid.zᵃᵃᶠ.reference[k] * vertical_scaling(i, j, k, grid, f, c, c) + grid.zᵃᵃᶠ.∂t_s[i, j] diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index a69a505821..feb0d4a499 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -185,7 +185,6 @@ function SplitExplicitState(grid::AbstractGrid, timestepper) ηᵐ⁻² = auxiliary_free_surface_field(grid, timestepper) 𝒰 = VelocityFields(grid) - u_bcs = 𝒰.u.boundary_conditions U = Field(𝒰.u, indices = (:, :, Nz)) V = Field(𝒰.v, indices = (:, :, Nz)) diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index d3511238de..9778c96da5 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -21,9 +21,9 @@ function update_grid!(model, grid::ZStarSpacingGrid; parameters = :xy) ∂t_s = grid.Δzᵃᵃᶠ.∂t_s η_grid = grid.zᵃᵃᶠ.∂t_s - # Free surface variables - # At the moment only SplitExplicitFreeSurface is supported, - # but this can be extended to other free surface solvers by calculating + # Free surface variables: + # TODO: At the moment only SplitExplicitFreeSurface is supported, + # but zstar can be extended to other free surface solvers by calculating # the barotropic velocity in this step U̅ = model.free_surface.state.U̅ V̅ = model.free_surface.state.V̅ @@ -49,6 +49,7 @@ end @kernel function _update_∂t_s!(∂t_s, U̅, V̅, grid) i, j = @index(Global, NTuple) k_top = grid.Nz + 1 + # ∂(η / H)/∂t = - ∇ ⋅ ∫udz / H δx_U = ∂xᶠᶜᶜ(i, j, k_top-1, grid, U̅) δy_V = ∂yᶜᶠᶜ(i, j, k_top-1, grid, V̅) @@ -56,6 +57,8 @@ end δ_U̅h = (δx_U + δy_V) / Azᶜᶜᶠ(i, j, k_top-1, grid) H = domain_depthᶜᶜᵃ(i, j, grid) + @show δ_U̅h, H + @inbounds ∂t_s[i, j] = - δ_U̅h / H end From 1a6babd27585b5bd551e2f34cdab4f024506374c Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 16 Oct 2024 13:30:19 +0200 Subject: [PATCH 372/567] bugfix --- .../HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 9778c96da5..44d76283eb 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -51,14 +51,12 @@ end k_top = grid.Nz + 1 # ∂(η / H)/∂t = - ∇ ⋅ ∫udz / H - δx_U = ∂xᶠᶜᶜ(i, j, k_top-1, grid, U̅) - δy_V = ∂yᶜᶠᶜ(i, j, k_top-1, grid, V̅) + δx_U = ∂xᶠᶜᶜ(i, j, k_top-1, grid, Δy_qᶠᶜᶠ, U̅) + δy_V = ∂yᶜᶠᶜ(i, j, k_top-1, grid, Δx_qᶜᶠᶠ, V̅) δ_U̅h = (δx_U + δy_V) / Azᶜᶜᶠ(i, j, k_top-1, grid) H = domain_depthᶜᶜᵃ(i, j, grid) - @show δ_U̅h, H - @inbounds ∂t_s[i, j] = - δ_U̅h / H end From d89aac91816dbe573ad4c2fde11ffef799a132b4 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 16 Oct 2024 13:31:44 +0200 Subject: [PATCH 373/567] better, still not ok --- .../HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 44d76283eb..c4e61ff5fe 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -51,8 +51,8 @@ end k_top = grid.Nz + 1 # ∂(η / H)/∂t = - ∇ ⋅ ∫udz / H - δx_U = ∂xᶠᶜᶜ(i, j, k_top-1, grid, Δy_qᶠᶜᶠ, U̅) - δy_V = ∂yᶜᶠᶜ(i, j, k_top-1, grid, Δx_qᶜᶠᶠ, V̅) + δx_U = δxᶜᶜᶠ(i, j, k_top-1, grid, Δy_qᶠᶜᶠ, U̅) + δy_V = δyᶜᶜᶠ(i, j, k_top-1, grid, Δx_qᶜᶠᶠ, V̅) δ_U̅h = (δx_U + δy_V) / Azᶜᶜᶠ(i, j, k_top-1, grid) H = domain_depthᶜᶜᵃ(i, j, grid) From 1d09bed641e2651b8215da673528edee70c1951d Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 16 Oct 2024 14:19:53 +0200 Subject: [PATCH 374/567] some name changes --- .../generalized_vertical_spacing.jl | 6 +++--- .../z_star_vertical_spacing.jl | 19 ++++++++++--------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index 045e0b7b63..02a9923415 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -39,8 +39,8 @@ update_grid!(model, grid; kwargs...) = nothing C₁ = convert(FT, 1.5) + χ C₂ = convert(FT, 0.5) + χ - sⁿ = vertical_scaling(i, j, k, grid, c, c, c) - s⁻ = previous_vertical_scaling(i, j, k, grid, c, c, c) + sⁿ = vertical_scaling(i, j, k, grid, Center(), Center(), Center()) + s⁻ = previous_vertical_scaling(i, j, k, grid, Center(), Center(), Center()) @inbounds begin ∂t_sθ = C₁ * sⁿ * Gⁿ[i, j, k] - C₂ * s⁻ * G⁻[i, j, k] @@ -78,6 +78,6 @@ end i, j, n = @index(Global, NTuple) @unroll for k in -Hz+1:Nz+Hz - tracers[n][i, j, k] /= vertical_scaling(i, j, k, grid, c, c, c) + tracers[n][i, j, k] /= vertical_scaling(i, j, k, grid, Center(), Center(), Center()) end end \ No newline at end of file diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index c4e61ff5fe..6da82ddc50 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -70,20 +70,21 @@ end hᶠᶠ = domain_depthᶠᶠᵃ(i, j, grid) @inbounds begin - Hᶜᶜ = (hᶜᶜ + η[i, j, k_top]) / hᶜᶜ - Hᶠᶜ = (hᶠᶜ + ℑxᶠᵃᵃ(i, j, k_top, grid, η)) / hᶠᶜ - Hᶜᶠ = (hᶜᶠ + ℑyᵃᶠᵃ(i, j, k_top, grid, η)) / hᶜᶠ - Hᶠᶠ = (hᶠᶠ + ℑxyᶠᶠᵃ(i, j, k_top, grid, η)) / hᶠᶠ + sᶜᶜ = (hᶜᶜ + η[i, j, k_top]) / hᶜᶜ + sᶠᶜ = (hᶠᶜ + ℑxᶠᵃᵃ(i, j, k_top, grid, η)) / hᶠᶜ + sᶜᶠ = (hᶜᶠ + ℑyᵃᶠᵃ(i, j, k_top, grid, η)) / hᶜᶠ + sᶠᶠ = (hᶠᶠ + ℑxyᶠᶠᵃ(i, j, k_top, grid, η)) / hᶠᶠ + # Update previous scaling sᶜᶜ⁻[i, j] = sᶜᶜⁿ[i, j] sᶠᶜ⁻[i, j] = sᶠᶜⁿ[i, j] sᶜᶠ⁻[i, j] = sᶜᶠⁿ[i, j] - # update current and previous scaling - sᶜᶜⁿ[i, j] = Hᶜᶜ - sᶠᶜⁿ[i, j] = Hᶠᶜ - sᶜᶠⁿ[i, j] = Hᶜᶠ - sᶠᶠⁿ[i, j] = Hᶠᶠ + # update current scaling + sᶜᶜⁿ[i, j] = sᶜᶜ + sᶠᶜⁿ[i, j] = sᶠᶜ + sᶜᶠⁿ[i, j] = sᶜᶠ + sᶠᶠⁿ[i, j] = sᶠᶠ # Update η in the grid η_grid[i, j] = η[i, j, k_top] From cd16d6c701419be2e33668b8891e3cf6f02d2336 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 16 Oct 2024 17:04:18 +0200 Subject: [PATCH 375/567] corrected everything --- .../compute_w_from_continuity.jl | 4 ++-- .../split_explicit_free_surface_kernels.jl | 6 +++--- .../HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl index 8b7e0cf134..335d6ccc52 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl @@ -24,11 +24,11 @@ compute_w_from_continuity!(velocities, arch, grid; parameters = w_kernel_paramet @inbounds U.w[i, j, 1] = 0 for k in 2:grid.Nz+1 - δ_Uh = flux_div_xyᶜᶜᶜ(i, j, k-1, grid, U.u, U.v) / Azᶜᶜᶜ(i, j, k-1, grid) + δh_u = flux_div_xyᶜᶜᶜ(i, j, k-1, grid, U.u, U.v) / Azᶜᶜᶜ(i, j, k-1, grid) ∂t_s = Δrᶜᶜᶜ(i, j, k-1, grid) * ∂t_grid(i, j, k-1, grid) immersed = immersed_cell(i, j, k-1, grid) - Δw = δ_Uh + ifelse(immersed, 0, ∂t_s) # We do not account for grid changes in immersed cells + Δw = δh_u + ifelse(immersed, 0, ∂t_s) # We do not account for grid changes in immersed cells @inbounds U.w[i, j, k] = U.w[i, j, k-1] - Δw end diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 92e2d83875..7d56c77a53 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -233,9 +233,9 @@ end i, j = @index(Global, NTuple) k_top = grid.Nz + 1 - sᶠᶜ = dynamic_domain_depthᶜᶠᵃ(i, j, k_top, grid, η) / domain_depthᶠᶜᵃ(i, j, grid) + sᶠᶜ = dynamic_domain_depthᶠᶜᵃ(i, j, k_top, grid, η) / domain_depthᶠᶜᵃ(i, j, grid) sᶜᶠ = dynamic_domain_depthᶜᶠᵃ(i, j, k_top, grid, η) / domain_depthᶜᶠᵃ(i, j, grid) - + # hand unroll first loop @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δrᶠᶜᶜ(i, j, 1, grid) * sᶠᶜ @inbounds V[i, j, k_top-1] = v[i, j, 1] * Δrᶜᶠᶜ(i, j, 1, grid) * sᶜᶠ @@ -255,7 +255,7 @@ end Hᶠᶜ = dynamic_domain_depthᶠᶜᵃ(i, j, k_top, grid, η) Hᶜᶠ = dynamic_domain_depthᶜᶠᵃ(i, j, k_top, grid, η) - + @inbounds begin u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / Hᶠᶜ v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / Hᶜᶠ diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 6da82ddc50..bcf4656873 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -54,10 +54,10 @@ end δx_U = δxᶜᶜᶠ(i, j, k_top-1, grid, Δy_qᶠᶜᶠ, U̅) δy_V = δyᶜᶜᶠ(i, j, k_top-1, grid, Δx_qᶜᶠᶠ, V̅) - δ_U̅h = (δx_U + δy_V) / Azᶜᶜᶠ(i, j, k_top-1, grid) + δh_U = (δx_U + δy_V) / Azᶜᶜᶠ(i, j, k_top-1, grid) H = domain_depthᶜᶜᵃ(i, j, grid) - @inbounds ∂t_s[i, j] = - δ_U̅h / H + @inbounds ∂t_s[i, j] = - δh_U / H end @kernel function _update_zstar!(sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, sᶠᶜ⁻, sᶜᶠ⁻, η_grid, η, grid) From bf95000e5d69cb5280561a8a6b427331c0fbc124 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 16 Oct 2024 17:17:21 +0200 Subject: [PATCH 376/567] some bugfix now works also for immersed boundaries --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 8 ++++---- validation/z_star_coordinate/lock_release.jl | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 51ca7cf47b..5417b9bbfd 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -141,7 +141,7 @@ const AGFBIB = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Abstrac XFlatAGFIBG = ImmersedBoundaryGrid{<:Any, <:Flat, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} YFlatAGFIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Flat, <:Any, <:Any, <:AbstractGridFittedBottom} -@inline domain_depthᶠᶜᵃ(i, j, k, ibg::XFlatAGFIBG) = domain_depthᶜᶜᵃ(i, j, ibg) -@inline domain_depthᶜᶠᵃ(i, j, k, ibg::YFlatAGFIBG) = domain_depthᶜᶜᵃ(i, j, ibg) -@inline domain_depthᶠᶠᵃ(i, j, k, ibg::XFlatAGFIBG) = domain_depthᶜᶠᵃ(i, j, ibg) -@inline domain_depthᶠᶠᵃ(i, j, k, ibg::YFlatAGFIBG) = domain_depthᶠᶜᵃ(i, j, ibg) +@inline domain_depthᶠᶜᵃ(i, j, ibg::XFlatAGFIBG) = domain_depthᶜᶜᵃ(i, j, ibg) +@inline domain_depthᶜᶠᵃ(i, j, ibg::YFlatAGFIBG) = domain_depthᶜᶜᵃ(i, j, ibg) +@inline domain_depthᶠᶠᵃ(i, j, ibg::XFlatAGFIBG) = domain_depthᶜᶠᵃ(i, j, ibg) +@inline domain_depthᶠᶠᵃ(i, j, ibg::YFlatAGFIBG) = domain_depthᶠᶜᵃ(i, j, ibg) diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index 75d9f04291..41cea8ad4e 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -15,7 +15,7 @@ grid = RectilinearGrid(size = (20, 20), halo = (6, 6), topology = (Bounded, Flat, Bounded)) -# grid = ImmersedBoundaryGrid(grid, GridFittedBottom(x -> x < 32kilometers ? -10 : -20)) +grid = ImmersedBoundaryGrid(grid, GridFittedBottom(x -> x < 32kilometers ? -10 : -20)) model = HydrostaticFreeSurfaceModel(; grid, momentum_advection = FluxFormAdvection(WENO(; order = 5), nothing, WENO(; order = 5)), @@ -37,7 +37,7 @@ set!(model, b = bᵢ) @info "the time step is $Δt" -simulation = Simulation(model; Δt, stop_iteration = 1, stop_time = 17hours) +simulation = Simulation(model; Δt, stop_iteration = 10000, stop_time = 17hours) Δz = GridMetricOperation((Center, Center, Center), Oceananigans.AbstractOperations.Δz, model.grid) @@ -63,7 +63,7 @@ function progress(sim) return nothing end -simulation.callbacks[:progress] = Callback(progress, IterationInterval(1)) +simulation.callbacks[:progress] = Callback(progress, IterationInterval(100)) run!(simulation) From d381037662c46649caead41b9e3ff8a0d2294a4b Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 16 Oct 2024 17:18:36 +0200 Subject: [PATCH 377/567] bugfix --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index b54e75921e..e6d75bc7fa 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -141,7 +141,7 @@ const AGFBIB = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Abstrac XFlatAGFIBG = ImmersedBoundaryGrid{<:Any, <:Flat, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} YFlatAGFIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Flat, <:Any, <:Any, <:AbstractGridFittedBottom} -@inline domain_depthᶠᶜᵃ(i, j, k, ibg::XFlatAGFIBG) = domain_depthᶜᶜᵃ(i, j, ibg) -@inline domain_depthᶜᶠᵃ(i, j, k, ibg::YFlatAGFIBG) = domain_depthᶜᶜᵃ(i, j, ibg) -@inline domain_depthᶠᶠᵃ(i, j, k, ibg::XFlatAGFIBG) = domain_depthᶜᶠᵃ(i, j, ibg) -@inline domain_depthᶠᶠᵃ(i, j, k, ibg::YFlatAGFIBG) = domain_depthᶠᶜᵃ(i, j, ibg) +@inline domain_depthᶠᶜᵃ(i, j, ibg::XFlatAGFIBG) = domain_depthᶜᶜᵃ(i, j, ibg) +@inline domain_depthᶜᶠᵃ(i, j, ibg::YFlatAGFIBG) = domain_depthᶜᶜᵃ(i, j, ibg) +@inline domain_depthᶠᶠᵃ(i, j, ibg::XFlatAGFIBG) = domain_depthᶜᶠᵃ(i, j, ibg) +@inline domain_depthᶠᶠᵃ(i, j, ibg::YFlatAGFIBG) = domain_depthᶠᶜᵃ(i, j, ibg) From 14d0a22c9d2b4dd5a130791d028e1d3cf5413f0f Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 16 Oct 2024 17:39:57 +0200 Subject: [PATCH 378/567] make sure the cells do not wet or dry --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 4 +++- src/ImmersedBoundaries/partial_cell_bottom.jl | 7 +++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 5417b9bbfd..4860b65d9c 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -119,7 +119,9 @@ correct_bottom_height!(bottom_field, grid, ib) = end @inline function _immersed_cell(i, j, k, underlying_grid, ib::GridFittedBottom) - z = znode(i, j, k, underlying_grid, c, c, c) + # We use `rnode` for the `immersed_cell` because we do not want to have + # wetting or drying that could happen for a moving grid if we use znode + z = rnode(i, j, k, underlying_grid, c, c, c) zb = @inbounds ib.bottom_height[i, j, 1] return z ≤ zb end diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index 51620ed2c9..712ddf8534 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -77,8 +77,11 @@ end zb = @inbounds bottom_field[i, j, 1] ϵ = ib.minimum_fractional_cell_height for k in 1:grid.Nz - z⁻ = znode(i, j, k, grid, c, c, f) - Δz = Δzᶜᶜᶜ(i, j, k, grid) + # We use `rnode` for the `immersed_cell` because we do not want to have + # wetting or drying that could happen for a moving grid if we use znode + z⁻ = rnode(i, j, k, underlying_grid, c, c, c) + # For the same reason, here we use `Δrᶜᶜᶜ` instead of `Δzᶜᶜᶜ` + Δz = Δrᶜᶜᶜ(i, j, k, grid) bottom_cell = z⁻ + Δz * (1 - ϵ) ≤ zb @inbounds bottom_field[i, j, 1] = ifelse(bottom_cell, z⁻ + Δz * (1 - ϵ), zb) end From 58ed7f408597d164b9bdd092b035ef698ad47361 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 16 Oct 2024 18:06:38 +0200 Subject: [PATCH 379/567] rnodes and rspacings --- src/Grids/Grids.jl | 4 +- src/Grids/latitude_longitude_grid.jl | 12 +- src/Grids/nodes_and_spacings.jl | 11 + src/Grids/orthogonal_spherical_shell_grid.jl | 15 +- src/Grids/rectilinear_grid.jl | 12 +- src/Grids/z_star_vertical_coordinate.jl | 17 +- .../abstract_zstar_immersed_grid.jl | 3 - src/ImmersedBoundaries/partial_cell_bottom.jl | 4 +- validation/z_star_coordinate/internal_tide.jl | 207 ++++++++++++++++++ validation/z_star_coordinate/lock_release.jl | 3 +- 10 files changed, 263 insertions(+), 25 deletions(-) create mode 100644 validation/z_star_coordinate/internal_tide.jl diff --git a/src/Grids/Grids.jl b/src/Grids/Grids.jl index 215adf9671..d3b51ac7ab 100644 --- a/src/Grids/Grids.jl +++ b/src/Grids/Grids.jl @@ -14,9 +14,9 @@ export conformal_cubed_sphere_panel export node, nodes export ξnode, ηnode, rnode export xnode, ynode, znode, λnode, φnode -export xnodes, ynodes, znodes, λnodes, φnodes +export xnodes, ynodes, znodes, λnodes, φnodes, rnodes export spacings -export xspacings, yspacings, zspacings, xspacing, yspacing, zspacing +export xspacings, yspacings, zspacings, rspacings, xspacing, yspacing, zspacing, rspacing export minimum_xspacing, minimum_yspacing, minimum_zspacing export ZStarVerticalCoordinate, vertical_scaling, previous_vertical_scaling, reference_zspacings export domain_depthᶜᶜᵃ, domain_depthᶠᶜᵃ, domain_depthᶜᶠᵃ, domain_depthᶠᶠᵃ diff --git a/src/Grids/latitude_longitude_grid.jl b/src/Grids/latitude_longitude_grid.jl index c875906030..cb5fb65777 100644 --- a/src/Grids/latitude_longitude_grid.jl +++ b/src/Grids/latitude_longitude_grid.jl @@ -604,7 +604,10 @@ rname(::LLG) = :z function nodes(grid::LLG, ℓx, ℓy, ℓz; reshape=false, with_halos=false) λ = λnodes(grid, ℓx, ℓy, ℓz; with_halos) φ = φnodes(grid, ℓx, ℓy, ℓz; with_halos) - z = znodes(grid, ℓx, ℓy, ℓz; with_halos) + # We use rnodes here. This is intentional. znodes return the actual zcoordinate + # which might be moving in case of a ZStarVerticalCoordinate. rnodes return the + # reference zcoordinate which is constant. + z = rnodes(grid, ℓx, ℓy, ℓz; with_halos) if reshape # Here we have to deal with the fact that Flat directions may have @@ -647,6 +650,8 @@ end @inline znodes(grid::LLG, ℓz::F; with_halos=false) = _property(grid.zᵃᵃᶠ, ℓz, topology(grid, 3), size(grid, 3), with_halos) @inline znodes(grid::LLG, ℓz::C; with_halos=false) = _property(grid.zᵃᵃᶜ, ℓz, topology(grid, 3), size(grid, 3), with_halos) +@inline rnodes(grid::LLG, ℓz::F; with_halos=false) = _property(grid.zᵃᵃᶠ, ℓz, topology(grid, 3), size(grid, 3), with_halos) +@inline rnodes(grid::LLG, ℓz::C; with_halos=false) = _property(grid.zᵃᵃᶜ, ℓz, topology(grid, 3), size(grid, 3), with_halos) # Convenience @inline λnodes(grid::LLG, ℓx, ℓy, ℓz; with_halos=false) = λnodes(grid, ℓx; with_halos) @@ -662,7 +667,7 @@ end @inline ξnodes(grid::LLG, ℓx, ℓy, ℓz; kwargs...) = λnodes(grid, ℓx; kwargs...) @inline ηnodes(grid::LLG, ℓx, ℓy, ℓz; kwargs...) = φnodes(grid, ℓy; kwargs...) -@inline rnodes(grid::LLG, ℓx, ℓy, ℓz; kwargs...) = znodes(grid, ℓz; kwargs...) +@inline rnodes(grid::LLG, ℓx, ℓy, ℓz; kwargs...) = rnodes(grid, ℓz; kwargs...) ##### ##### Grid spacings in x, y, z (in meters) @@ -710,6 +715,7 @@ end @inline xspacings(grid::LLG, ℓx, ℓy, ℓz; kwargs...) = xspacings(grid, ℓx, ℓy; kwargs...) @inline yspacings(grid::LLG, ℓx, ℓy, ℓz; kwargs...) = yspacings(grid, ℓx, ℓy; kwargs...) @inline zspacings(grid::LLG, ℓx, ℓy, ℓz; kwargs...) = zspacings(grid, ℓz; kwargs...) +@inline rspacings(grid::LLG, ℓx, ℓy, ℓz; kwargs...) = rspacings(grid, ℓz; kwargs...) ##### ##### Grid spacings in λ, φ (in degrees) @@ -730,6 +736,8 @@ end @inline φspacings(grid::LLG, ℓy::F; with_halos=false) = _property(grid.Δφᵃᶠᵃ, ℓy, topology(grid, 2), size(grid, 2), with_halos) @inline zspacings(grid::LLG, ℓz::C; with_halos=false) = _property(grid.Δzᵃᵃᶜ, ℓz, topology(grid, 3), size(grid, 3), with_halos) @inline zspacings(grid::LLG, ℓz::F; with_halos=false) = _property(grid.Δzᵃᵃᶠ, ℓz, topology(grid, 3), size(grid, 3), with_halos) +@inline rspacings(grid::LLG, ℓz::C; with_halos=false) = _property(grid.Δzᵃᵃᶜ, ℓz, topology(grid, 3), size(grid, 3), with_halos) +@inline rspacings(grid::LLG, ℓz::F; with_halos=false) = _property(grid.Δzᵃᵃᶠ, ℓz, topology(grid, 3), size(grid, 3), with_halos) @inline λspacings(grid::LLG, ℓx, ℓy, ℓz; kwargs...) = λspacings(grid, ℓx; kwargs...) @inline φspacings(grid::LLG, ℓx, ℓy, ℓz; kwargs...) = φspacings(grid, ℓy; kwargs...) diff --git a/src/Grids/nodes_and_spacings.jl b/src/Grids/nodes_and_spacings.jl index e348c0cb47..d54b2f1b93 100644 --- a/src/Grids/nodes_and_spacings.jl +++ b/src/Grids/nodes_and_spacings.jl @@ -233,6 +233,17 @@ julia> zspacings(grid, Center(), Center(), Center()) """ @inline zspacings(grid, ℓx, ℓy, ℓz; with_halos=true) = zspacings(grid, ℓz; with_halos) + +""" + rspacings(grid, ℓx, ℓy, ℓz; with_halos=true) + +Return the "reference" spacings over the interior nodes on `grid` in the ``z``-direction for the location `ℓx`, +`ℓy`, `ℓz`. For `Bounded` directions, `Face` nodes include the boundary points. These are equal to the `zspacings` +for a _static_ grid. +""" +@inline rspacings(grid, ℓx, ℓy, ℓz; with_halos=true) = rspacings(grid, ℓz; with_halos) + + destantiate(::Face) = Face destantiate(::Center) = Center diff --git a/src/Grids/orthogonal_spherical_shell_grid.jl b/src/Grids/orthogonal_spherical_shell_grid.jl index 1567b2a5fb..c54cb11abf 100644 --- a/src/Grids/orthogonal_spherical_shell_grid.jl +++ b/src/Grids/orthogonal_spherical_shell_grid.jl @@ -1119,7 +1119,10 @@ end function nodes(grid::OSSG, ℓx, ℓy, ℓz; reshape=false, with_halos=false) λ = λnodes(grid, ℓx, ℓy, ℓz; with_halos) φ = φnodes(grid, ℓx, ℓy, ℓz; with_halos) - z = znodes(grid, ℓx, ℓy, ℓz; with_halos) + # We use rnodes here. This is intentional. znodes return the actual zcoordinate + # which might be moving in case of a ZStarVerticalCoordinate. rnodes return the + # reference zcoordinate which is constant. + z = rnodes(grid, ℓx, ℓy, ℓz; with_halos) if reshape # λ and φ are 2D arrays @@ -1158,10 +1161,16 @@ end @inline znodes(grid::OSSG, ℓz::Center; with_halos=false) = with_halos ? grid.zᵃᵃᶜ : view(grid.zᵃᵃᶜ, interior_indices(ℓz, topology(grid, 3)(), grid.Nz)) +@inline rnodes(grid::OSSG, ℓz::Face ; with_halos=false) = with_halos ? grid.zᵃᵃᶠ : + view(grid.zᵃᵃᶠ, interior_indices(ℓz, topology(grid, 3)(), grid.Nz)) +@inline rnodes(grid::OSSG, ℓz::Center; with_halos=false) = with_halos ? grid.zᵃᵃᶜ : + view(grid.zᵃᵃᶜ, interior_indices(ℓz, topology(grid, 3)(), grid.Nz)) + # convenience @inline λnodes(grid::OSSG, ℓx, ℓy, ℓz; with_halos=false) = λnodes(grid, ℓx, ℓy; with_halos) @inline φnodes(grid::OSSG, ℓx, ℓy, ℓz; with_halos=false) = φnodes(grid, ℓx, ℓy; with_halos) @inline znodes(grid::OSSG, ℓx, ℓy, ℓz; with_halos=false) = znodes(grid, ℓz ; with_halos) +@inline rnodes(grid::OSSG, ℓx, ℓy, ℓz; with_halos=false) = rnodes(grid, ℓz ; with_halos) @inline xnodes(grid::OSSG, ℓx, ℓy, ℓz; with_halos=false) = xnodes(grid, ℓx, ℓy; with_halos) @inline ynodes(grid::OSSG, ℓx, ℓy, ℓz; with_halos=false) = ynodes(grid, ℓx, ℓy; with_halos) @@ -1225,7 +1234,11 @@ rname(::OSSG) = :z @inline zspacings(grid::OSSG, ℓz::Face; with_halos=false) = with_halos ? grid.Δzᵃᵃᶠ : view(grid.Δzᵃᵃᶠ, interior_indices(ℓz, topology(grid, 3)(), grid.Nz)) @inline zspacings(grid::ZRegOSSG, ℓz::Face; with_halos=false) = grid.Δzᵃᵃᶠ +@inline rspacings(grid::OSSG, ℓz::Face; with_halos=false) = with_halos ? grid.Δzᵃᵃᶠ : + view(grid.Δzᵃᵃᶠ, interior_indices(ℓz, topology(grid, 3)(), grid.Nz)) +@inline rspacings(grid::ZRegOSSG, ℓz::Face; with_halos=false) = grid.Δzᵃᵃᶠ @inline xspacings(grid::OSSG, ℓx, ℓy, ℓz; with_halos=false) = xspacings(grid, ℓx, ℓy; with_halos) @inline yspacings(grid::OSSG, ℓx, ℓy, ℓz; with_halos=false) = yspacings(grid, ℓx, ℓy; with_halos) @inline zspacings(grid::OSSG, ℓx, ℓy, ℓz; with_halos=false) = zspacings(grid, ℓz; with_halos) +@inline rspacings(grid::OSSG, ℓx, ℓy, ℓz; with_halos=false) = rspacings(grid, ℓz; with_halos) diff --git a/src/Grids/rectilinear_grid.jl b/src/Grids/rectilinear_grid.jl index 5dbee4385a..313fcec1ca 100644 --- a/src/Grids/rectilinear_grid.jl +++ b/src/Grids/rectilinear_grid.jl @@ -469,7 +469,10 @@ rname(::RG) = :z function nodes(grid::RectilinearGrid, ℓx, ℓy, ℓz; reshape=false, with_halos=false) x = xnodes(grid, ℓx, ℓy, ℓz; with_halos) y = ynodes(grid, ℓx, ℓy, ℓz; with_halos) - z = znodes(grid, ℓx, ℓy, ℓz; with_halos) + # We use rnodes here. This is intentional. znodes return the actual zcoordinate + # which might be moving in case of a ZStarVerticalCoordinate. rnodes return the + # reference zcoordinate which is constant. + z = rnodes(grid, ℓx, ℓy, ℓz; with_halos) if reshape # Here we have to deal with the fact that Flat directions may have @@ -503,6 +506,8 @@ const C = Center @inline ynodes(grid::RG, ℓy::C; with_halos=false) = _property(grid.yᵃᶜᵃ, ℓy, topology(grid, 2), size(grid, 2), with_halos) @inline znodes(grid::RG, ℓz::F; with_halos=false) = _property(grid.zᵃᵃᶠ, ℓz, topology(grid, 3), size(grid, 3), with_halos) @inline znodes(grid::RG, ℓz::C; with_halos=false) = _property(grid.zᵃᵃᶜ, ℓz, topology(grid, 3), size(grid, 3), with_halos) +@inline rnodes(grid::RG, ℓz::F; with_halos=false) = _property(grid.zᵃᵃᶠ, ℓz, topology(grid, 3), size(grid, 3), with_halos) +@inline rnodes(grid::RG, ℓz::C; with_halos=false) = _property(grid.zᵃᵃᶜ, ℓz, topology(grid, 3), size(grid, 3), with_halos) # convenience @inline xnodes(grid::RG, ℓx, ℓy, ℓz; with_halos=false) = xnodes(grid, ℓx; with_halos) @@ -516,7 +521,7 @@ const C = Center @inline ξnodes(grid::RG, ℓx, ℓy, ℓz; kwargs...) = xnodes(grid, ℓx; kwargs...) @inline ηnodes(grid::RG, ℓx, ℓy, ℓz; kwargs...) = ynodes(grid, ℓy; kwargs...) -@inline rnodes(grid::RG, ℓx, ℓy, ℓz; kwargs...) = znodes(grid, ℓz; kwargs...) +@inline rnodes(grid::RG, ℓx, ℓy, ℓz; kwargs...) = rnodes(grid, ℓz; kwargs...) ##### ##### Grid spacings @@ -528,9 +533,12 @@ const C = Center @inline yspacings(grid::RG, ℓy::F; with_halos=false) = _property(grid.Δyᵃᶠᵃ, ℓy, topology(grid, 2), size(grid, 2), with_halos) @inline zspacings(grid::RG, ℓz::C; with_halos=false) = _property(grid.Δzᵃᵃᶜ, ℓz, topology(grid, 3), size(grid, 3), with_halos) @inline zspacings(grid::RG, ℓz::F; with_halos=false) = _property(grid.Δzᵃᵃᶠ, ℓz, topology(grid, 3), size(grid, 3), with_halos) +@inline rspacings(grid::RG, ℓz::C; with_halos=false) = _property(grid.Δzᵃᵃᶜ, ℓz, topology(grid, 3), size(grid, 3), with_halos) +@inline rspacings(grid::RG, ℓz::F; with_halos=false) = _property(grid.Δzᵃᵃᶠ, ℓz, topology(grid, 3), size(grid, 3), with_halos) @inline xspacings(grid::RG, ℓx, ℓy, ℓz; kwargs...) = xspacings(grid, ℓx; kwargs...) @inline yspacings(grid::RG, ℓx, ℓy, ℓz; kwargs...) = yspacings(grid, ℓy; kwargs...) @inline zspacings(grid::RG, ℓx, ℓy, ℓz; kwargs...) = zspacings(grid, ℓz; kwargs...) +@inline rspacings(grid::RG, ℓx, ℓy, ℓz; kwargs...) = zspacings(grid, ℓz; kwargs...) @inline isrectilinear(::RG) = true diff --git a/src/Grids/z_star_vertical_coordinate.jl b/src/Grids/z_star_vertical_coordinate.jl index d8f1b9da39..cf6a406101 100644 --- a/src/Grids/z_star_vertical_coordinate.jl +++ b/src/Grids/z_star_vertical_coordinate.jl @@ -77,7 +77,8 @@ struct ZStarVerticalCoordinate{R, SCC, SFC, SCF, SFF} <: AbstractVerticalCoordin ∂t_s :: SCC end -ZStarVerticalCoordinate(r_faces) = ZStarVerticalCoordinate(r_faces, nothing, nothing, nothing, nothing, nothing, nothing, nothing, nothing) +ZStarVerticalCoordinate(r_faces::Union{Tuple, AbstractVector}) = ZStarVerticalCoordinate(r_faces, nothing, nothing, nothing, nothing, nothing, nothing, nothing, nothing) +ZStarVerticalCoordinate(r⁻::Number, r⁺::Number) = ZStarVerticalCoordinate((r⁻, r⁺), nothing, nothing, nothing, nothing, nothing, nothing, nothing, nothing) Adapt.adapt_structure(to, coord::ZStarVerticalCoordinate) = ZStarSpacing(Adapt.adapt(to, coord.reference), @@ -185,22 +186,16 @@ const F = Face const ZSG = ZStarUnderlyingGrid # Fallbacks -reference_znodes(grid, ::C) = grid.zᵃᵃᶜ -reference_znodes(grid, ::F) = grid.zᵃᵃᶠ - -reference_zspacings(grid, ::C) = grid.Δzᵃᵃᶜ -reference_zspacings(grid, ::F) = grid.Δzᵃᵃᶠ - @inline vertical_scaling(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) @inline previous_vertical_scaling(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) @inline ∂t_grid(i, j, k, grid) = zero(grid) -reference_zspacings(grid::ZSG, ::C) = grid.Δzᵃᵃᶜ.reference -reference_zspacings(grid::ZSG, ::F) = grid.Δzᵃᵃᶠ.reference +@inline rnodes(grid::ZSG, ℓz::C; with_halos=false) = _property(grid.zᵃᵃᶜ.reference, ℓz, topology(grid, 3), size(grid, 3), with_halos) +@inline rnodes(grid::ZSG, ℓz::F; with_halos=false) = _property(grid.zᵃᵃᶠ.reference, ℓz, topology(grid, 3), size(grid, 3), with_halos) -reference_znodes(grid::ZSG, ::C) = grid.zᵃᵃᶜ.reference -reference_znodes(grid::ZSG, ::F) = grid.zᵃᵃᶠ.reference +@inline rspacings(grid::ZSG, ℓz::C; with_halos=false) = _property(grid.Δzᵃᵃᶜ.reference, ℓz, topology(grid, 3), size(grid, 3), with_halos) +@inline rspacings(grid::ZSG, ℓz::F; with_halos=false) = _property(grid.Δzᵃᵃᶠ.reference, ℓz, topology(grid, 3), size(grid, 3), with_halos) @inline vertical_scaling(i, j, k, grid::ZSG, ::C, ::C, ::C) = @inbounds grid.Δzᵃᵃᶜ.sᶜᶜⁿ[i, j] @inline vertical_scaling(i, j, k, grid::ZSG, ::F, ::C, ::C) = @inbounds grid.Δzᵃᵃᶜ.sᶠᶜⁿ[i, j] diff --git a/src/ImmersedBoundaries/abstract_zstar_immersed_grid.jl b/src/ImmersedBoundaries/abstract_zstar_immersed_grid.jl index d3b765520d..df8c24766f 100644 --- a/src/ImmersedBoundaries/abstract_zstar_immersed_grid.jl +++ b/src/ImmersedBoundaries/abstract_zstar_immersed_grid.jl @@ -6,9 +6,6 @@ import Oceananigans.Grids: reference_zspacings, reference_znodes, vertical_scali const ImmersedAbstractVerticalCoordinateGrid = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:AbstractVerticalCoordinateUnderlyingGrid} const ImmersedZStarGrid = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:ZStarUnderlyingGrid} -reference_znodes(grid::ImmersedBoundaryGrid, ℓz) = reference_znodes(grid.underlying_grid, ℓz) -reference_zspacings(grid::ImmersedBoundaryGrid, ℓz) = reference_zspacings(grid.underlying_grid, ℓz) - @inline vertical_scaling(i, j, k, grid::ImmersedBoundaryGrid, ℓx, ℓy, ℓz) = vertical_scaling(i, j, k, grid.underlying_grid, ℓx, ℓy, ℓz) @inline previous_vertical_scaling(i, j, k, grid::ImmersedBoundaryGrid, ℓx, ℓy, ℓz) = previous_vertical_scaling(i, j, k, grid.underlying_grid, ℓx, ℓy, ℓz) @inline ∂t_grid(i, j, k, grid::ImmersedBoundaryGrid) = ∂t_grid(i, j, k, grid.underlying_grid) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index 712ddf8534..b81d5098f9 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -79,7 +79,7 @@ end for k in 1:grid.Nz # We use `rnode` for the `immersed_cell` because we do not want to have # wetting or drying that could happen for a moving grid if we use znode - z⁻ = rnode(i, j, k, underlying_grid, c, c, c) + z⁻ = rnode(i, j, k, grid, c, c, c) # For the same reason, here we use `Δrᶜᶜᶜ` instead of `Δzᶜᶜᶜ` Δz = Δrᶜᶜᶜ(i, j, k, grid) bottom_cell = z⁻ + Δz * (1 - ϵ) ≤ zb @@ -123,7 +123,7 @@ Criterion is zb ≥ z - ϵ Δz """ @inline function _immersed_cell(i, j, k, underlying_grid, ib::PartialCellBottom) - z = znode(i, j, k+1, underlying_grid, c, c, f) + z = rnode(i, j, k+1, underlying_grid, c, c, f) zb = @inbounds ib.bottom_height[i, j, 1] return z ≤ zb end diff --git a/validation/z_star_coordinate/internal_tide.jl b/validation/z_star_coordinate/internal_tide.jl new file mode 100644 index 0000000000..ccb9e826b6 --- /dev/null +++ b/validation/z_star_coordinate/internal_tide.jl @@ -0,0 +1,207 @@ +using CairoMakie +using Oceananigans +using Oceananigans.Units +using Oceananigans.ImmersedBoundaries: PartialCellBottom +using Oceananigans.AbstractOperations: GridMetricOperation + +Nx, Nz = 250, 125 + +H = 2kilometers +z_faces = ZStarVerticalCoordinate(-H, 0) + +underlying_grid = RectilinearGrid(size = (Nx, Nz), + x = (-1000kilometers, 1000kilometers), + z = z_faces, + halo = (4, 4), + topology = (Periodic, Flat, Bounded)) + +h₀ = 250meters +width = 20kilometers +hill(x) = h₀ * exp(-x^2 / 2width^2) +bottom(x) = - H + hill(x) + +grid = ImmersedBoundaryGrid(underlying_grid, PartialCellBottom(bottom)) + +# Let's see how the domain with the bathymetry is. + +x = xnodes(grid, Center()) +bottom_boundary = interior(grid.immersed_boundary.bottom_height, :, 1, 1) +top_boundary = 0 * x + + +coriolis = FPlane(latitude = -45) + +# Now we have everything we require to construct the tidal forcing given a value of the +# excursion parameter. + +T₂ = 12.421hours +ω₂ = 2π / T₂ # radians/sec + +ϵ = 0.1 # excursion parameter + +U_tidal = ϵ * ω₂ * width + +tidal_forcing_amplitude = U_tidal * (ω₂^2 - coriolis.f^2) / ω₂ + +@inline tidal_forcing(x, z, t, p) = p.tidal_forcing_amplitude * sin(p.ω₂ * t) + +u_forcing = Forcing(tidal_forcing, parameters=(; tidal_forcing_amplitude, ω₂)) + +# ## Model + +# We built a `HydrostaticFreeSurfaceModel`: + +model = HydrostaticFreeSurfaceModel(; grid, coriolis, + buoyancy = BuoyancyTracer(), + tracers = :b, + momentum_advection = WENO(), + tracer_advection = WENO(), + forcing = (; u = u_forcing)) + +# We initialize the model with the tidal flow and a linear stratification. + +uᵢ(x, z) = U_tidal + +Nᵢ² = 1e-4 # [s⁻²] initial buoyancy frequency / stratification +bᵢ(x, z) = Nᵢ² * z + +set!(model, u=uᵢ, b=bᵢ) + +# Now let's build a `Simulation`. + +Δt = 5minutes +stop_time = 4days + +simulation = Simulation(model; Δt, stop_time) + +# We add a callback to print a message about how the simulation is going, + +using Printf + +wall_clock = Ref(time_ns()) + +dz = GridMetricOperation((Center, Center, Center), Oceananigans.AbstractOperations.Δz, model.grid) +∫b_init = sum(model.tracers.b * dz) / sum(dz) + +function progress(sim) + elapsed = 1e-9 * (time_ns() - wall_clock[]) + + ∫b = sum(model.tracers.b * dz) / sum(dz) + + msg = @sprintf("iteration: %d, time: %s, wall time: %s, max|w|: %6.3e, m s⁻¹, drift: %6.3\n", + iteration(sim), prettytime(sim), prettytime(elapsed), + maximum(abs, sim.model.velocities.w), + ∫b - ∫b_init) + + wall_clock[] = time_ns() + + @info msg + + return nothing +end + +b = model.tracers.b +u, v, w = model.velocities + +U = Field(Average(u)) + +u′ = u - U + +N² = ∂z(b) + +filename = "internal_tide" +save_fields_interval = 30minutes + +simulation.output_writers[:fields] = JLD2OutputWriter(model, (; u, u′, w, b, N²); + filename, + schedule = TimeInterval(save_fields_interval), + overwrite_existing = true) + +# We are ready -- let's run! +run!(simulation) + +# ## Load output + +# First, we load the saved velocities and stratification output as `FieldTimeSeries`es. + +saved_output_filename = filename * ".jld2" + +u′_t = FieldTimeSeries(saved_output_filename, "u′") + w_t = FieldTimeSeries(saved_output_filename, "w") +N²_t = FieldTimeSeries(saved_output_filename, "N²") + +umax = maximum(abs, u′_t[end]) +wmax = maximum(abs, w_t[end]) + +times = u′_t.times +nothing #hide + +# We retrieve each field's coordinates and convert from meters to kilometers. + +xu, _, zu = nodes(u′_t[1]) +xw, _, zw = nodes(w_t[1]) +xN², _, zN² = nodes(N²_t[1]) + +xu = xu ./ 1e3 +xw = xw ./ 1e3 +xN² = xN² ./ 1e3 +zu = zu ./ 1e3 +zw = zw ./ 1e3 +zN² = zN² ./ 1e3 +nothing #hide + +# ## Visualize + +# Now we can visualize our resutls! We use `CairoMakie` here. On a system with OpenGL +# `using GLMakie` is more convenient as figures will be displayed on the screen. +# +# We use Makie's `Observable` to animate the data. To dive into how `Observable`s work we +# refer to [Makie.jl's Documentation](https://makie.juliaplots.org/stable/documentation/nodes/index.html). + +using CairoMakie + +n = Observable(1) + +title = @lift @sprintf("t = %1.2f days = %1.2f T₂", + round(times[$n] / day, digits=2) , round(times[$n] / T₂, digits=2)) + +u′n = @lift u′_t[$n] + wn = @lift w_t[$n] +N²n = @lift N²_t[$n] + +axis_kwargs = (xlabel = "x [km]", + ylabel = "z [km]", + limits = ((-grid.Lx/2e3, grid.Lx/2e3), (-grid.Lz/1e3, 0)), # note conversion to kilometers + titlesize = 20) + +fig = Figure(size = (700, 900)) + +fig[1, :] = Label(fig, title, fontsize=24, tellwidth=false) + +ax_u = Axis(fig[2, 1]; title = "u'-velocity", axis_kwargs...) +hm_u = heatmap!(ax_u, xu, zu, u′n; nan_color=:gray, colorrange=(-umax, umax), colormap=:balance) +Colorbar(fig[2, 2], hm_u, label = "m s⁻¹") + +ax_w = Axis(fig[3, 1]; title = "w-velocity", axis_kwargs...) +hm_w = heatmap!(ax_w, xw, zw, wn; nan_color=:gray, colorrange=(-wmax, wmax), colormap=:balance) +Colorbar(fig[3, 2], hm_w, label = "m s⁻¹") + +ax_N² = Axis(fig[4, 1]; title = "stratification N²", axis_kwargs...) +hm_N² = heatmap!(ax_N², xN², zN², N²n; nan_color=:gray, colorrange=(0.9Nᵢ², 1.1Nᵢ²), colormap=:magma) +Colorbar(fig[4, 2], hm_N², label = "s⁻²") + +fig + +# Finally, we can record a movie. + +@info "Making an animation from saved data..." + +frames = 1:length(times) + +record(fig, filename * ".mp4", frames, framerate=16) do i + @info string("Plotting frame ", i, " of ", frames[end]) + n[] = i +end +nothing #hide + +# ![](internal_tide.mp4) diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index 41cea8ad4e..f4b06aac8a 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -6,8 +6,7 @@ using Oceananigans.AbstractOperations: GridMetricOperation using Oceananigans.Models.HydrostaticFreeSurfaceModels: ZStarSpacingGrid using Printf -r_faces = (-20, 0) -z_faces = ZStarVerticalCoordinate(r_faces) +z_faces = ZStarVerticalCoordinate(-20, 0) grid = RectilinearGrid(size = (20, 20), x = (0, 64kilometers), From d0de5a9ffe46f0d084ee1887f62eca7a52267ede Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 16 Oct 2024 18:41:08 +0200 Subject: [PATCH 380/567] removing some NaNs --- src/Grids/z_star_vertical_coordinate.jl | 8 +- .../hydrostatic_free_surface_model.jl | 20 --- .../split_explicit_free_surface_kernels.jl | 7 +- .../z_star_vertical_spacing.jl | 10 +- validation/z_star_coordinate/internal_tide.jl | 138 +++++++++--------- validation/z_star_coordinate/lock_release.jl | 6 +- 6 files changed, 84 insertions(+), 105 deletions(-) diff --git a/src/Grids/z_star_vertical_coordinate.jl b/src/Grids/z_star_vertical_coordinate.jl index cf6a406101..a304f9eb21 100644 --- a/src/Grids/z_star_vertical_coordinate.jl +++ b/src/Grids/z_star_vertical_coordinate.jl @@ -16,11 +16,11 @@ const AbstractVerticalCoordinateUnderlyingGrid = Union{AVLLG, AVOSSG, AVRG} function retrieve_static_grid(grid::AbstractVerticalCoordinateUnderlyingGrid) - zᵃᵃᶠ = reference_znodes(grid, Face()) - zᵃᵃᶜ = reference_znodes(grid, Center()) + zᵃᵃᶠ = rnodes(grid, Face(); with_halos = true) + zᵃᵃᶜ = rnodes(grid, Center(); with_halos = true) - Δzᵃᵃᶠ = reference_zspacings(grid, Face()) - Δzᵃᵃᶜ = reference_zspacings(grid, Center()) + Δzᵃᵃᶠ = rspacings(grid, Face(); with_halos = true) + Δzᵃᵃᶜ = rspacings(grid, Center(); with_halos = true) TX, TY, TZ = topology(grid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl index 7785b9447b..764674f219 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl @@ -55,25 +55,6 @@ default_free_surface(grid; gravitational_acceleration=g_Earth) = """ HydrostaticFreeSurfaceModel(; grid, -<<<<<<< HEAD - clock = Clock{eltype(grid)}(time = 0), - momentum_advection = CenteredSecondOrder(), - tracer_advection = CenteredSecondOrder(), - buoyancy = nothing, - coriolis = nothing, - free_surface = default_free_surface(grid, gravitational_acceleration=g_Earth), - forcing::NamedTuple = NamedTuple(), - closure = nothing, - boundary_conditions::NamedTuple = NamedTuple(), - tracers = nothing, - particles::ParticlesOrNothing = nothing, - biogeochemistry::AbstractBGCOrNothing = nothing, - velocities = nothing, - pressure = nothing, - diffusivity_fields = nothing, - vertical_coordinate = Z(), - auxiliary_fields = NamedTuple(), -======= clock = Clock{eltype(grid)}(time = 0), momentum_advection = VectorInvariant(), tracer_advection = CenteredSecondOrder(), @@ -90,7 +71,6 @@ biogeochemistry::AbstractBGCOrNothing = nothing, pressure = nothing, diffusivity_fields = nothing, auxiliary_fields = NamedTuple(), ->>>>>>> origin/main ) Construct a hydrostatic model with a free surface on `grid`. diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 7d56c77a53..9b1ea2dc33 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -233,8 +233,11 @@ end i, j = @index(Global, NTuple) k_top = grid.Nz + 1 - sᶠᶜ = dynamic_domain_depthᶠᶜᵃ(i, j, k_top, grid, η) / domain_depthᶠᶜᵃ(i, j, grid) - sᶜᶠ = dynamic_domain_depthᶜᶠᵃ(i, j, k_top, grid, η) / domain_depthᶜᶠᵃ(i, j, grid) + hᶠᶜ = domain_depthᶠᶜᵃ(i, j, grid) + hᶜᶠ = domain_depthᶜᶠᵃ(i, j, grid) + + sᶠᶜ = ifelse(hᶠᶜ == 0, one(grid), dynamic_domain_depthᶠᶜᵃ(i, j, k_top, grid, η) / hᶠᶜ) + sᶜᶠ = ifelse(hᶜᶠ == 0, one(grid), dynamic_domain_depthᶜᶠᵃ(i, j, k_top, grid, η) / hᶜᶠ) # hand unroll first loop @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δrᶠᶜᶜ(i, j, 1, grid) * sᶠᶜ diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index bcf4656873..87373c3914 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -57,7 +57,7 @@ end δh_U = (δx_U + δy_V) / Azᶜᶜᶠ(i, j, k_top-1, grid) H = domain_depthᶜᶜᵃ(i, j, grid) - @inbounds ∂t_s[i, j] = - δh_U / H + @inbounds ∂t_s[i, j] = ifelse(H == 0, zero(grid), - δh_U / H) end @kernel function _update_zstar!(sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, sᶠᶜ⁻, sᶜᶠ⁻, η_grid, η, grid) @@ -70,10 +70,10 @@ end hᶠᶠ = domain_depthᶠᶠᵃ(i, j, grid) @inbounds begin - sᶜᶜ = (hᶜᶜ + η[i, j, k_top]) / hᶜᶜ - sᶠᶜ = (hᶠᶜ + ℑxᶠᵃᵃ(i, j, k_top, grid, η)) / hᶠᶜ - sᶜᶠ = (hᶜᶠ + ℑyᵃᶠᵃ(i, j, k_top, grid, η)) / hᶜᶠ - sᶠᶠ = (hᶠᶠ + ℑxyᶠᶠᵃ(i, j, k_top, grid, η)) / hᶠᶠ + sᶜᶜ = ifelse(hᶜᶜ == 0, one(grid), (hᶜᶜ + η[i, j, k_top]) / hᶜᶜ) + sᶠᶜ = ifelse(hᶠᶜ == 0, one(grid), (hᶠᶜ + ℑxᶠᵃᵃ(i, j, k_top, grid, η)) / hᶠᶜ) + sᶜᶠ = ifelse(hᶜᶠ == 0, one(grid), (hᶜᶠ + ℑyᵃᶠᵃ(i, j, k_top, grid, η)) / hᶜᶠ) + sᶠᶠ = ifelse(hᶠᶠ == 0, one(grid), (hᶠᶠ + ℑxyᶠᶠᵃ(i, j, k_top, grid, η)) / hᶠᶠ) # Update previous scaling sᶜᶜ⁻[i, j] = sᶜᶜⁿ[i, j] diff --git a/validation/z_star_coordinate/internal_tide.jl b/validation/z_star_coordinate/internal_tide.jl index ccb9e826b6..95db1f92d4 100644 --- a/validation/z_star_coordinate/internal_tide.jl +++ b/validation/z_star_coordinate/internal_tide.jl @@ -20,14 +20,7 @@ width = 20kilometers hill(x) = h₀ * exp(-x^2 / 2width^2) bottom(x) = - H + hill(x) -grid = ImmersedBoundaryGrid(underlying_grid, PartialCellBottom(bottom)) - -# Let's see how the domain with the bathymetry is. - -x = xnodes(grid, Center()) -bottom_boundary = interior(grid.immersed_boundary.bottom_height, :, 1, 1) -top_boundary = 0 * x - +grid = ImmersedBoundaryGrid(underlying_grid, GridFittedBottom(bottom)) coriolis = FPlane(latitude = -45) @@ -54,13 +47,14 @@ u_forcing = Forcing(tidal_forcing, parameters=(; tidal_forcing_amplitude, ω₂) model = HydrostaticFreeSurfaceModel(; grid, coriolis, buoyancy = BuoyancyTracer(), tracers = :b, + free_surface = SplitExplicitFreeSurface(grid; substeps = 20), momentum_advection = WENO(), tracer_advection = WENO(), forcing = (; u = u_forcing)) # We initialize the model with the tidal flow and a linear stratification. -uᵢ(x, z) = U_tidal +uᵢ(x, z) = 0 Nᵢ² = 1e-4 # [s⁻²] initial buoyancy frequency / stratification bᵢ(x, z) = Nᵢ² * z @@ -72,7 +66,7 @@ set!(model, u=uᵢ, b=bᵢ) Δt = 5minutes stop_time = 4days -simulation = Simulation(model; Δt, stop_time) +simulation = Simulation(model; Δt, stop_time, stop_iteration = 2) # We add a callback to print a message about how the simulation is going, @@ -88,9 +82,9 @@ function progress(sim) ∫b = sum(model.tracers.b * dz) / sum(dz) - msg = @sprintf("iteration: %d, time: %s, wall time: %s, max|w|: %6.3e, m s⁻¹, drift: %6.3\n", + msg = @sprintf("iteration: %d, time: %s, wall time: %s, max|w|: %6.3e, max|u|: %6.3e, drift: %6.3e\n", iteration(sim), prettytime(sim), prettytime(elapsed), - maximum(abs, sim.model.velocities.w), + maximum(abs, interior(w, :, :, grid.Nz+1)), maximum(abs, simulation.model.velocities.u), ∫b - ∫b_init) wall_clock[] = time_ns() @@ -100,6 +94,8 @@ function progress(sim) return nothing end +simulation.callbacks[:progress] = Callback(progress, IterationInterval(1)) + b = model.tracers.b u, v, w = model.velocities @@ -120,88 +116,88 @@ simulation.output_writers[:fields] = JLD2OutputWriter(model, (; u, u′, w, b, N # We are ready -- let's run! run!(simulation) -# ## Load output +# # ## Load output -# First, we load the saved velocities and stratification output as `FieldTimeSeries`es. +# # First, we load the saved velocities and stratification output as `FieldTimeSeries`es. -saved_output_filename = filename * ".jld2" +# saved_output_filename = filename * ".jld2" -u′_t = FieldTimeSeries(saved_output_filename, "u′") - w_t = FieldTimeSeries(saved_output_filename, "w") -N²_t = FieldTimeSeries(saved_output_filename, "N²") +# u′_t = FieldTimeSeries(saved_output_filename, "u′") +# w_t = FieldTimeSeries(saved_output_filename, "w") +# N²_t = FieldTimeSeries(saved_output_filename, "N²") -umax = maximum(abs, u′_t[end]) -wmax = maximum(abs, w_t[end]) +# umax = maximum(abs, u′_t[end]) +# wmax = maximum(abs, w_t[end]) -times = u′_t.times -nothing #hide +# times = u′_t.times +# nothing #hide -# We retrieve each field's coordinates and convert from meters to kilometers. +# # We retrieve each field's coordinates and convert from meters to kilometers. -xu, _, zu = nodes(u′_t[1]) -xw, _, zw = nodes(w_t[1]) -xN², _, zN² = nodes(N²_t[1]) +# xu, _, zu = nodes(u′_t[1]) +# xw, _, zw = nodes(w_t[1]) +# xN², _, zN² = nodes(N²_t[1]) -xu = xu ./ 1e3 -xw = xw ./ 1e3 -xN² = xN² ./ 1e3 -zu = zu ./ 1e3 -zw = zw ./ 1e3 -zN² = zN² ./ 1e3 -nothing #hide +# xu = xu ./ 1e3 +# xw = xw ./ 1e3 +# xN² = xN² ./ 1e3 +# zu = zu ./ 1e3 +# zw = zw ./ 1e3 +# zN² = zN² ./ 1e3 +# nothing #hide -# ## Visualize +# # ## Visualize -# Now we can visualize our resutls! We use `CairoMakie` here. On a system with OpenGL -# `using GLMakie` is more convenient as figures will be displayed on the screen. -# -# We use Makie's `Observable` to animate the data. To dive into how `Observable`s work we -# refer to [Makie.jl's Documentation](https://makie.juliaplots.org/stable/documentation/nodes/index.html). +# # Now we can visualize our resutls! We use `CairoMakie` here. On a system with OpenGL +# # `using GLMakie` is more convenient as figures will be displayed on the screen. +# # +# # We use Makie's `Observable` to animate the data. To dive into how `Observable`s work we +# # refer to [Makie.jl's Documentation](https://makie.juliaplots.org/stable/documentation/nodes/index.html). -using CairoMakie +# using CairoMakie -n = Observable(1) +# n = Observable(1) -title = @lift @sprintf("t = %1.2f days = %1.2f T₂", - round(times[$n] / day, digits=2) , round(times[$n] / T₂, digits=2)) +# title = @lift @sprintf("t = %1.2f days = %1.2f T₂", +# round(times[$n] / day, digits=2) , round(times[$n] / T₂, digits=2)) -u′n = @lift u′_t[$n] - wn = @lift w_t[$n] -N²n = @lift N²_t[$n] +# u′n = @lift u′_t[$n] +# wn = @lift w_t[$n] +# N²n = @lift N²_t[$n] -axis_kwargs = (xlabel = "x [km]", - ylabel = "z [km]", - limits = ((-grid.Lx/2e3, grid.Lx/2e3), (-grid.Lz/1e3, 0)), # note conversion to kilometers - titlesize = 20) +# axis_kwargs = (xlabel = "x [km]", +# ylabel = "z [km]", +# limits = ((-grid.Lx/2e3, grid.Lx/2e3), (-grid.Lz/1e3, 0)), # note conversion to kilometers +# titlesize = 20) -fig = Figure(size = (700, 900)) +# fig = Figure(size = (700, 900)) -fig[1, :] = Label(fig, title, fontsize=24, tellwidth=false) +# fig[1, :] = Label(fig, title, fontsize=24, tellwidth=false) -ax_u = Axis(fig[2, 1]; title = "u'-velocity", axis_kwargs...) -hm_u = heatmap!(ax_u, xu, zu, u′n; nan_color=:gray, colorrange=(-umax, umax), colormap=:balance) -Colorbar(fig[2, 2], hm_u, label = "m s⁻¹") +# ax_u = Axis(fig[2, 1]; title = "u'-velocity", axis_kwargs...) +# hm_u = heatmap!(ax_u, xu, zu, u′n; nan_color=:gray, colorrange=(-umax, umax), colormap=:balance) +# Colorbar(fig[2, 2], hm_u, label = "m s⁻¹") -ax_w = Axis(fig[3, 1]; title = "w-velocity", axis_kwargs...) -hm_w = heatmap!(ax_w, xw, zw, wn; nan_color=:gray, colorrange=(-wmax, wmax), colormap=:balance) -Colorbar(fig[3, 2], hm_w, label = "m s⁻¹") +# ax_w = Axis(fig[3, 1]; title = "w-velocity", axis_kwargs...) +# hm_w = heatmap!(ax_w, xw, zw, wn; nan_color=:gray, colorrange=(-wmax, wmax), colormap=:balance) +# Colorbar(fig[3, 2], hm_w, label = "m s⁻¹") -ax_N² = Axis(fig[4, 1]; title = "stratification N²", axis_kwargs...) -hm_N² = heatmap!(ax_N², xN², zN², N²n; nan_color=:gray, colorrange=(0.9Nᵢ², 1.1Nᵢ²), colormap=:magma) -Colorbar(fig[4, 2], hm_N², label = "s⁻²") +# ax_N² = Axis(fig[4, 1]; title = "stratification N²", axis_kwargs...) +# hm_N² = heatmap!(ax_N², xN², zN², N²n; nan_color=:gray, colorrange=(0.9Nᵢ², 1.1Nᵢ²), colormap=:magma) +# Colorbar(fig[4, 2], hm_N², label = "s⁻²") -fig +# fig -# Finally, we can record a movie. +# # Finally, we can record a movie. -@info "Making an animation from saved data..." +# @info "Making an animation from saved data..." -frames = 1:length(times) +# frames = 1:length(times) -record(fig, filename * ".mp4", frames, framerate=16) do i - @info string("Plotting frame ", i, " of ", frames[end]) - n[] = i -end -nothing #hide +# record(fig, filename * ".mp4", frames, framerate=16) do i +# @info string("Plotting frame ", i, " of ", frames[end]) +# n[] = i +# end +# nothing #hide -# ![](internal_tide.mp4) +# # ![](internal_tide.mp4) diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index f4b06aac8a..34356ad4ce 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -14,7 +14,7 @@ grid = RectilinearGrid(size = (20, 20), halo = (6, 6), topology = (Bounded, Flat, Bounded)) -grid = ImmersedBoundaryGrid(grid, GridFittedBottom(x -> x < 32kilometers ? -10 : -20)) +grid = ImmersedBoundaryGrid(grid, GridFittedBottom(x -> - (64kilometers - x) / 64kilometers * 20)) model = HydrostaticFreeSurfaceModel(; grid, momentum_advection = FluxFormAdvection(WENO(; order = 5), nothing, WENO(; order = 5)), @@ -36,7 +36,7 @@ set!(model, b = bᵢ) @info "the time step is $Δt" -simulation = Simulation(model; Δt, stop_iteration = 10000, stop_time = 17hours) +simulation = Simulation(model; Δt, stop_iteration = 1, stop_time = 17hours) Δz = GridMetricOperation((Center, Center, Center), Oceananigans.AbstractOperations.Δz, model.grid) @@ -62,7 +62,7 @@ function progress(sim) return nothing end -simulation.callbacks[:progress] = Callback(progress, IterationInterval(100)) +simulation.callbacks[:progress] = Callback(progress, IterationInterval(1)) run!(simulation) From 9b5811edf9e9dcba87de8f1ab8145d3fdac095f4 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 16 Oct 2024 18:53:17 +0200 Subject: [PATCH 381/567] =?UTF-8?q?it's=20`=CE=94r`=20for=20partial=20cell?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../abstract_zstar_immersed_grid.jl | 2 +- src/ImmersedBoundaries/grid_fitted_bottom.jl | 4 +-- src/ImmersedBoundaries/partial_cell_bottom.jl | 34 +++++++++---------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/ImmersedBoundaries/abstract_zstar_immersed_grid.jl b/src/ImmersedBoundaries/abstract_zstar_immersed_grid.jl index df8c24766f..51396d11ee 100644 --- a/src/ImmersedBoundaries/abstract_zstar_immersed_grid.jl +++ b/src/ImmersedBoundaries/abstract_zstar_immersed_grid.jl @@ -16,4 +16,4 @@ function retrieve_static_grid(ib::ImmersedAbstractVerticalCoordinateGrid) underlying_grid = retrieve_static_grid(ib.underlying_grid) return ImmersedBoundaryGrid(underlying_grid, immersed_boundary; active_cells_map) -end \ No newline at end of file +end diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 4860b65d9c..52e5ac817c 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -111,8 +111,8 @@ correct_bottom_height!(bottom_field, grid, ib) = zb = @inbounds bottom_field[i, j, 1] condition = ib.immersed_condition for k in 1:grid.Nz - z⁺ = znode(i, j, k+1, grid, c, c, f) - z = znode(i, j, k, grid, c, c, c) + z⁺ = rnode(i, j, k+1, grid, c, c, f) + z = rnode(i, j, k, grid, c, c, c) bottom_cell = ifelse(condition isa CenterImmersedCondition, z ≤ zb, z⁺ ≤ zb) @inbounds bottom_field[i, j, 1] = ifelse(bottom_cell, z⁺, zb) end diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index b81d5098f9..0c9cdda11b 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -135,7 +135,7 @@ end return !immersed_cell(i, j, k, grid, ib) & immersed_cell(i, j, k-1, grid, ib) end -@inline function Δzᶜᶜᶜ(i, j, k, ibg::PCBIBG) +@inline function Δrᶜᶜᶜ(i, j, k, ibg::PCBIBG) underlying_grid = ibg.underlying_grid ib = ibg.immersed_boundary @@ -148,45 +148,45 @@ end # Are we in a bottom cell? at_the_bottom = bottom_cell(i, j, k, ibg) - full_Δz = Δzᶜᶜᶜ(i, j, k, ibg.underlying_grid) + full_Δz = Δrᶜᶜᶜ(i, j, k, ibg.underlying_grid) partial_Δz = z - zb return ifelse(at_the_bottom, partial_Δz, full_Δz) end -@inline function Δzᶜᶜᶠ(i, j, k, ibg::PCBIBG) +@inline function Δrᶜᶜᶠ(i, j, k, ibg::PCBIBG) just_above_bottom = bottom_cell(i, j, k-1, ibg) zc = znode(i, j, k, ibg.underlying_grid, c, c, c) zf = znode(i, j, k, ibg.underlying_grid, c, c, f) - full_Δz = Δzᶜᶜᶠ(i, j, k, ibg.underlying_grid) - partial_Δz = zc - zf + Δzᶜᶜᶜ(i, j, k-1, ibg) / 2 + full_Δz = Δrᶜᶜᶠ(i, j, k, ibg.underlying_grid) + partial_Δz = zc - zf + Δrᶜᶜᶜ(i, j, k-1, ibg) / 2 Δz = ifelse(just_above_bottom, partial_Δz, full_Δz) return Δz end -@inline Δzᶠᶜᶜ(i, j, k, ibg::PCBIBG) = min(Δzᶜᶜᶜ(i-1, j, k, ibg), Δzᶜᶜᶜ(i, j, k, ibg)) -@inline Δzᶜᶠᶜ(i, j, k, ibg::PCBIBG) = min(Δzᶜᶜᶜ(i, j-1, k, ibg), Δzᶜᶜᶜ(i, j, k, ibg)) -@inline Δzᶠᶠᶜ(i, j, k, ibg::PCBIBG) = min(Δzᶠᶜᶜ(i, j-1, k, ibg), Δzᶠᶜᶜ(i, j, k, ibg)) +@inline Δrᶠᶜᶜ(i, j, k, ibg::PCBIBG) = min(Δrᶜᶜᶜ(i-1, j, k, ibg), Δrᶜᶜᶜ(i, j, k, ibg)) +@inline Δrᶜᶠᶜ(i, j, k, ibg::PCBIBG) = min(Δrᶜᶜᶜ(i, j-1, k, ibg), Δrᶜᶜᶜ(i, j, k, ibg)) +@inline Δrᶠᶠᶜ(i, j, k, ibg::PCBIBG) = min(Δrᶠᶜᶜ(i, j-1, k, ibg), Δrᶠᶜᶜ(i, j, k, ibg)) -@inline Δzᶠᶜᶠ(i, j, k, ibg::PCBIBG) = min(Δzᶜᶜᶠ(i-1, j, k, ibg), Δzᶜᶜᶠ(i, j, k, ibg)) -@inline Δzᶜᶠᶠ(i, j, k, ibg::PCBIBG) = min(Δzᶜᶜᶠ(i, j-1, k, ibg), Δzᶜᶜᶠ(i, j, k, ibg)) -@inline Δzᶠᶠᶠ(i, j, k, ibg::PCBIBG) = min(Δzᶠᶜᶠ(i, j-1, k, ibg), Δzᶠᶜᶠ(i, j, k, ibg)) +@inline Δrᶠᶜᶠ(i, j, k, ibg::PCBIBG) = min(Δrᶜᶜᶠ(i-1, j, k, ibg), Δrᶜᶜᶠ(i, j, k, ibg)) +@inline Δrᶜᶠᶠ(i, j, k, ibg::PCBIBG) = min(Δrᶜᶜᶠ(i, j-1, k, ibg), Δrᶜᶜᶠ(i, j, k, ibg)) +@inline Δrᶠᶠᶠ(i, j, k, ibg::PCBIBG) = min(Δrᶠᶜᶠ(i, j-1, k, ibg), Δrᶠᶜᶠ(i, j, k, ibg)) # Make sure Δz works for horizontally-Flat topologies. # (There's no point in using z-Flat with PartialCellBottom). XFlatPCBIBG = ImmersedBoundaryGrid{<:Any, <:Flat, <:Any, <:Any, <:Any, <:PartialCellBottom} YFlatPCBIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Flat, <:Any, <:Any, <:PartialCellBottom} -@inline Δzᶠᶜᶜ(i, j, k, ibg::XFlatPCBIBG) = Δzᶜᶜᶜ(i, j, k, ibg) -@inline Δzᶠᶜᶠ(i, j, k, ibg::XFlatPCBIBG) = Δzᶜᶜᶠ(i, j, k, ibg) -@inline Δzᶜᶠᶜ(i, j, k, ibg::YFlatPCBIBG) = Δzᶜᶜᶜ(i, j, k, ibg) +@inline Δrᶠᶜᶜ(i, j, k, ibg::XFlatPCBIBG) = Δrᶜᶜᶜ(i, j, k, ibg) +@inline Δrᶠᶜᶠ(i, j, k, ibg::XFlatPCBIBG) = Δrᶜᶜᶠ(i, j, k, ibg) +@inline Δrᶜᶠᶜ(i, j, k, ibg::YFlatPCBIBG) = Δrᶜᶜᶜ(i, j, k, ibg) -@inline Δzᶜᶠᶠ(i, j, k, ibg::YFlatPCBIBG) = Δzᶜᶜᶠ(i, j, k, ibg) -@inline Δzᶠᶠᶜ(i, j, k, ibg::XFlatPCBIBG) = Δzᶜᶠᶜ(i, j, k, ibg) -@inline Δzᶠᶠᶜ(i, j, k, ibg::YFlatPCBIBG) = Δzᶠᶜᶜ(i, j, k, ibg) +@inline Δrᶜᶠᶠ(i, j, k, ibg::YFlatPCBIBG) = Δrᶜᶜᶠ(i, j, k, ibg) +@inline Δrᶠᶠᶜ(i, j, k, ibg::XFlatPCBIBG) = Δrᶜᶠᶜ(i, j, k, ibg) +@inline Δrᶠᶠᶜ(i, j, k, ibg::YFlatPCBIBG) = Δrᶠᶜᶜ(i, j, k, ibg) @inline bottom_height(i, j, ibg::PCBIBG) = @inbounds ibg.immersed_boundary.bottom_height[i, j, 1] From 764119ff60dd7c060a9f8d3fa393b447586af071 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 16 Oct 2024 19:13:49 +0200 Subject: [PATCH 382/567] some changes --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 2 +- src/ImmersedBoundaries/partial_cell_bottom.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 52e5ac817c..2488fa033c 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -114,7 +114,7 @@ correct_bottom_height!(bottom_field, grid, ib) = z⁺ = rnode(i, j, k+1, grid, c, c, f) z = rnode(i, j, k, grid, c, c, c) bottom_cell = ifelse(condition isa CenterImmersedCondition, z ≤ zb, z⁺ ≤ zb) - @inbounds bottom_field[i, j, 1] = ifelse(bottom_cell, z⁺, zb) + @inbounds bottom_field[i, j, 1] = ifelse(bottom_cell, z⁺, bottom_field[i, j, 1]) end end diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index 0c9cdda11b..5ff4fbd391 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -83,7 +83,7 @@ end # For the same reason, here we use `Δrᶜᶜᶜ` instead of `Δzᶜᶜᶜ` Δz = Δrᶜᶜᶜ(i, j, k, grid) bottom_cell = z⁻ + Δz * (1 - ϵ) ≤ zb - @inbounds bottom_field[i, j, 1] = ifelse(bottom_cell, z⁻ + Δz * (1 - ϵ), zb) + @inbounds bottom_field[i, j, 1] = ifelse(bottom_cell, z⁻ + Δz * (1 - ϵ), bottom_field[i, j, 1]) end end From c63a7bc50b14b6c6a6d14dbb98164155d7c578c4 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 16 Oct 2024 19:53:50 +0200 Subject: [PATCH 383/567] another small bugfix --- .../abstract_zstar_immersed_grid.jl | 2 +- src/ImmersedBoundaries/grid_fitted_bottom.jl | 1 + src/ImmersedBoundaries/partial_cell_bottom.jl | 7 +++--- .../split_explicit_free_surface_kernels.jl | 22 ++----------------- validation/z_star_coordinate/internal_tide.jl | 2 +- validation/z_star_coordinate/lock_release.jl | 15 +++++++------ 6 files changed, 17 insertions(+), 32 deletions(-) diff --git a/src/ImmersedBoundaries/abstract_zstar_immersed_grid.jl b/src/ImmersedBoundaries/abstract_zstar_immersed_grid.jl index 51396d11ee..f6d2f58c97 100644 --- a/src/ImmersedBoundaries/abstract_zstar_immersed_grid.jl +++ b/src/ImmersedBoundaries/abstract_zstar_immersed_grid.jl @@ -1,7 +1,7 @@ using Oceananigans.Grids: AbstractVerticalCoordinateUnderlyingGrid, ZStarUnderlyingGrid import Oceananigans.Grids: retrieve_static_grid -import Oceananigans.Grids: reference_zspacings, reference_znodes, vertical_scaling, previous_vertical_scaling, ∂t_grid +import Oceananigans.Grids: vertical_scaling, previous_vertical_scaling, ∂t_grid const ImmersedAbstractVerticalCoordinateGrid = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:AbstractVerticalCoordinateUnderlyingGrid} const ImmersedZStarGrid = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:ZStarUnderlyingGrid} diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 2488fa033c..45c409e018 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -109,6 +109,7 @@ correct_bottom_height!(bottom_field, grid, ib) = @kernel function _correct_bottom_height!(bottom_field, grid, ib::GridFittedBottom) i, j = @index(Global, NTuple) zb = @inbounds bottom_field[i, j, 1] + @inbounds bottom_field[i, j, 1] = rnode(i, j, 1, grid, c, c, f) condition = ib.immersed_condition for k in 1:grid.Nz z⁺ = rnode(i, j, k+1, grid, c, c, f) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index 5ff4fbd391..f2e8aa6067 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -76,6 +76,7 @@ end i, j = @index(Global, NTuple) zb = @inbounds bottom_field[i, j, 1] ϵ = ib.minimum_fractional_cell_height + @inbounds bottom_field[i, j, 1] = rnode(i, j, 1, grid, c, c, f) for k in 1:grid.Nz # We use `rnode` for the `immersed_cell` because we do not want to have # wetting or drying that could happen for a moving grid if we use znode @@ -140,7 +141,7 @@ end ib = ibg.immersed_boundary # Get node at face above and defining nodes on c,c,f - z = znode(i, j, k+1, underlying_grid, c, c, f) + z = rnode(i, j, k+1, underlying_grid, c, c, f) # Get bottom z-coordinate and fractional Δz parameter zb = @inbounds ib.bottom_height[i, j, 1] @@ -156,8 +157,8 @@ end @inline function Δrᶜᶜᶠ(i, j, k, ibg::PCBIBG) just_above_bottom = bottom_cell(i, j, k-1, ibg) - zc = znode(i, j, k, ibg.underlying_grid, c, c, c) - zf = znode(i, j, k, ibg.underlying_grid, c, c, f) + zc = rnode(i, j, k, ibg.underlying_grid, c, c, c) + zf = rnode(i, j, k, ibg.underlying_grid, c, c, f) full_Δz = Δrᶜᶜᶠ(i, j, k, ibg.underlying_grid) partial_Δz = zc - zf + Δrᶜᶜᶜ(i, j, k-1, ibg) / 2 diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 9b1ea2dc33..daeaa5a0a9 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -104,24 +104,6 @@ for Topo in [:Periodic, :Bounded, :RightConnected, :LeftConnected] end end -# Interpolation -const PGX = AbstractGrid{<:Any, <:Periodic} -const BGX = AbstractGrid{<:Any, <:Bounded} -const RGX = AbstractGrid{<:Any, <:RightConnected} - -const PGY = AbstractGrid{<:Any, <:Periodic} -const BGY = AbstractGrid{<:Any, <:Bounded} -const RGY = AbstractGrid{<:Any, <:RightConnected} - -@inline ℑxᶠᵃᵃ_η(i, j, k, grid, η) = ℑxᶠᵃᵃ(i, j, k, grid, η) -@inline ℑyᵃᶠᵃ_η(i, j, k, grid, η) = ℑyᵃᶠᵃ(i, j, k, grid, η) -@inline ℑxᶠᵃᵃ_η(i, j, k, grid::PGX, η) = ifelse(i == 1, (η[1, j, k] + η[grid.Nx, j, k]) / 2, ℑxᶠᵃᵃ(i, j, k, grid, η)) -@inline ℑyᵃᶠᵃ_η(i, j, k, grid::PGY, η) = ifelse(j == 1, (η[i, 1, k] + η[i, grid.Ny, k]) / 2, ℑyᵃᶠᵃ(i, j, k, grid, η)) -@inline ℑxᶠᵃᵃ_η(i, j, k, grid::BGX, η) = ifelse(i == 1, η[1, j, k], ℑxᶠᵃᵃ(i, j, k, grid, η)) -@inline ℑyᵃᶠᵃ_η(i, j, k, grid::BGY, η) = ifelse(j == 1, η[i, 1, k], ℑyᵃᶠᵃ(i, j, k, grid, η)) -@inline ℑxᶠᵃᵃ_η(i, j, k, grid::RGX, η) = ifelse(i == 1, η[1, j, k], ℑxᶠᵃᵃ(i, j, k, grid, η)) -@inline ℑyᵃᶠᵃ_η(i, j, k, grid::RGY, η) = ifelse(j == 1, η[i, 1, k], ℑyᵃᶠᵃ(i, j, k, grid, η)) - # Time stepping extrapolation U★, and η★ # AB3 step @@ -182,8 +164,8 @@ end @inline dynamic_domain_depthᶜᶠᵃ(i, j, k, grid, η) = domain_depthᶜᶠᵃ(i, j, grid) # Non-linear free surface implementation -@inline dynamic_domain_depthᶠᶜᵃ(i, j, k, grid::ZStarSpacingGrid, η) = domain_depthᶠᶜᵃ(i, j, grid) + ℑxᶠᵃᵃ_η(i, j, k, grid, η) -@inline dynamic_domain_depthᶜᶠᵃ(i, j, k, grid::ZStarSpacingGrid, η) = domain_depthᶜᶠᵃ(i, j, grid) + ℑyᵃᶠᵃ_η(i, j, k, grid, η) +@inline dynamic_domain_depthᶠᶜᵃ(i, j, k, grid::ZStarSpacingGrid, η) = domain_depthᶠᶜᵃ(i, j, grid) + ℑxᶠᵃᵃ(i, j, k, grid, η) +@inline dynamic_domain_depthᶜᶠᵃ(i, j, k, grid::ZStarSpacingGrid, η) = domain_depthᶜᶠᵃ(i, j, grid) + ℑyᵃᶠᵃ(i, j, k, grid, η) @kernel function _split_explicit_barotropic_velocity!(averaging_weight, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², diff --git a/validation/z_star_coordinate/internal_tide.jl b/validation/z_star_coordinate/internal_tide.jl index 95db1f92d4..0cbb12d177 100644 --- a/validation/z_star_coordinate/internal_tide.jl +++ b/validation/z_star_coordinate/internal_tide.jl @@ -66,7 +66,7 @@ set!(model, u=uᵢ, b=bᵢ) Δt = 5minutes stop_time = 4days -simulation = Simulation(model; Δt, stop_time, stop_iteration = 2) +simulation = Simulation(model; Δt, stop_time) # We add a callback to print a message about how the simulation is going, diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index 34356ad4ce..71eb49ea67 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -8,11 +8,11 @@ using Printf z_faces = ZStarVerticalCoordinate(-20, 0) -grid = RectilinearGrid(size = (20, 20), +grid = RectilinearGrid(size = (128, 20), x = (0, 64kilometers), z = z_faces, halo = (6, 6), - topology = (Bounded, Flat, Bounded)) + topology = (Periodic, Flat, Bounded)) grid = ImmersedBoundaryGrid(grid, GridFittedBottom(x -> - (64kilometers - x) / 64kilometers * 20)) @@ -36,9 +36,10 @@ set!(model, b = bᵢ) @info "the time step is $Δt" -simulation = Simulation(model; Δt, stop_iteration = 1, stop_time = 17hours) +simulation = Simulation(model; Δt, stop_iteration = 10000, stop_time = 17hours) Δz = GridMetricOperation((Center, Center, Center), Oceananigans.AbstractOperations.Δz, model.grid) +∫b_init = sum(model.tracers.b * Δz) / sum(Δz) field_outputs = merge(model.velocities, model.tracers, (; Δz)) @@ -50,19 +51,19 @@ simulation.output_writers[:other_variables] = JLD2OutputWriter(model, field_outp function progress(sim) w = interior(sim.model.velocities.w, :, :, sim.model.grid.Nz+1) u = sim.model.velocities.u - b = sim.model.tracers.b - + ∫b = sum(model.tracers.b * Δz) / sum(Δz) + msg0 = @sprintf("Time: %s iteration %d ", prettytime(sim.model.clock.time), sim.model.clock.iteration) msg1 = @sprintf("extrema w: %.2e %.2e ", maximum(w), minimum(w)) msg2 = @sprintf("extrema u: %.2e %.2e ", maximum(u), minimum(u)) - msg3 = @sprintf("extrema b: %.2e %.2e ", maximum(b), minimum(b)) + msg3 = @sprintf("drift b: %6.3e ", ∫b - ∫b_init) msg4 = @sprintf("extrema Δz: %.2e %.2e ", maximum(Δz), minimum(Δz)) @info msg0 * msg1 * msg2 * msg3 * msg4 return nothing end -simulation.callbacks[:progress] = Callback(progress, IterationInterval(1)) +simulation.callbacks[:progress] = Callback(progress, IterationInterval(100)) run!(simulation) From a5442278bb6ae4f7ca31fad594ce6607a200f1c9 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 16 Oct 2024 19:55:29 +0200 Subject: [PATCH 384/567] figure this out later --- .../split_explicit_free_surface_kernels.jl | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index daeaa5a0a9..821b8a58db 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -104,6 +104,25 @@ for Topo in [:Periodic, :Bounded, :RightConnected, :LeftConnected] end end +# Interpolation +# TODO: Figure out a nice way to do this. +const PGX = AbstractGrid{<:Any, <:Periodic} +const BGX = AbstractGrid{<:Any, <:Bounded} +const RGX = AbstractGrid{<:Any, <:RightConnected} + +const PGY = AbstractGrid{<:Any, <:Periodic} +const BGY = AbstractGrid{<:Any, <:Bounded} +const RGY = AbstractGrid{<:Any, <:RightConnected} + +@inline ℑxᶠᵃᵃ_η(i, j, k, grid, η) = ℑxᶠᵃᵃ(i, j, k, grid, η) +@inline ℑyᵃᶠᵃ_η(i, j, k, grid, η) = ℑyᵃᶠᵃ(i, j, k, grid, η) +@inline ℑxᶠᵃᵃ_η(i, j, k, grid::PGX, η) = ifelse(i == 1, (η[1, j, k] + η[grid.Nx, j, k]) / 2, ℑxᶠᵃᵃ(i, j, k, grid, η)) +@inline ℑyᵃᶠᵃ_η(i, j, k, grid::PGY, η) = ifelse(j == 1, (η[i, 1, k] + η[i, grid.Ny, k]) / 2, ℑyᵃᶠᵃ(i, j, k, grid, η)) +@inline ℑxᶠᵃᵃ_η(i, j, k, grid::BGX, η) = ifelse(i == 1, η[1, j, k], ℑxᶠᵃᵃ(i, j, k, grid, η)) +@inline ℑyᵃᶠᵃ_η(i, j, k, grid::BGY, η) = ifelse(j == 1, η[i, 1, k], ℑyᵃᶠᵃ(i, j, k, grid, η)) +@inline ℑxᶠᵃᵃ_η(i, j, k, grid::RGX, η) = ifelse(i == 1, η[1, j, k], ℑxᶠᵃᵃ(i, j, k, grid, η)) +@inline ℑyᵃᶠᵃ_η(i, j, k, grid::RGY, η) = ifelse(j == 1, η[i, 1, k], ℑyᵃᶠᵃ(i, j, k, grid, η)) + # Time stepping extrapolation U★, and η★ # AB3 step @@ -164,8 +183,8 @@ end @inline dynamic_domain_depthᶜᶠᵃ(i, j, k, grid, η) = domain_depthᶜᶠᵃ(i, j, grid) # Non-linear free surface implementation -@inline dynamic_domain_depthᶠᶜᵃ(i, j, k, grid::ZStarSpacingGrid, η) = domain_depthᶠᶜᵃ(i, j, grid) + ℑxᶠᵃᵃ(i, j, k, grid, η) -@inline dynamic_domain_depthᶜᶠᵃ(i, j, k, grid::ZStarSpacingGrid, η) = domain_depthᶜᶠᵃ(i, j, grid) + ℑyᵃᶠᵃ(i, j, k, grid, η) +@inline dynamic_domain_depthᶠᶜᵃ(i, j, k, grid::ZStarSpacingGrid, η) = domain_depthᶠᶜᵃ(i, j, grid) + ℑxᶠᵃᵃ_η(i, j, k, grid, η) +@inline dynamic_domain_depthᶜᶠᵃ(i, j, k, grid::ZStarSpacingGrid, η) = domain_depthᶜᶠᵃ(i, j, grid) + ℑyᵃᶠᵃ_η(i, j, k, grid, η) @kernel function _split_explicit_barotropic_velocity!(averaging_weight, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², From e8f285517e7c02c14657d36d0858d4355e1578eb Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 16 Oct 2024 20:13:13 +0200 Subject: [PATCH 385/567] internal tide works --- validation/z_star_coordinate/internal_tide.jl | 116 +++++++++--------- 1 file changed, 58 insertions(+), 58 deletions(-) diff --git a/validation/z_star_coordinate/internal_tide.jl b/validation/z_star_coordinate/internal_tide.jl index 0cbb12d177..d244ae0296 100644 --- a/validation/z_star_coordinate/internal_tide.jl +++ b/validation/z_star_coordinate/internal_tide.jl @@ -94,7 +94,7 @@ function progress(sim) return nothing end -simulation.callbacks[:progress] = Callback(progress, IterationInterval(1)) +simulation.callbacks[:progress] = Callback(progress, IterationInterval(100)) b = model.tracers.b u, v, w = model.velocities @@ -120,84 +120,84 @@ run!(simulation) # # First, we load the saved velocities and stratification output as `FieldTimeSeries`es. -# saved_output_filename = filename * ".jld2" +saved_output_filename = filename * ".jld2" -# u′_t = FieldTimeSeries(saved_output_filename, "u′") -# w_t = FieldTimeSeries(saved_output_filename, "w") -# N²_t = FieldTimeSeries(saved_output_filename, "N²") +u′_t = FieldTimeSeries(saved_output_filename, "u′") + w_t = FieldTimeSeries(saved_output_filename, "w") +N²_t = FieldTimeSeries(saved_output_filename, "N²") -# umax = maximum(abs, u′_t[end]) -# wmax = maximum(abs, w_t[end]) +umax = maximum(abs, u′_t[end]) +wmax = maximum(abs, w_t[end]) -# times = u′_t.times -# nothing #hide +times = u′_t.times +nothing #hide -# # We retrieve each field's coordinates and convert from meters to kilometers. +# We retrieve each field's coordinates and convert from meters to kilometers. -# xu, _, zu = nodes(u′_t[1]) -# xw, _, zw = nodes(w_t[1]) -# xN², _, zN² = nodes(N²_t[1]) +xu, _, zu = nodes(u′_t[1]) +xw, _, zw = nodes(w_t[1]) +xN², _, zN² = nodes(N²_t[1]) -# xu = xu ./ 1e3 -# xw = xw ./ 1e3 -# xN² = xN² ./ 1e3 -# zu = zu ./ 1e3 -# zw = zw ./ 1e3 -# zN² = zN² ./ 1e3 -# nothing #hide +xu = xu ./ 1e3 +xw = xw ./ 1e3 +xN² = xN² ./ 1e3 +zu = zu ./ 1e3 +zw = zw ./ 1e3 +zN² = zN² ./ 1e3 +nothing #hide -# # ## Visualize +# ## Visualize -# # Now we can visualize our resutls! We use `CairoMakie` here. On a system with OpenGL -# # `using GLMakie` is more convenient as figures will be displayed on the screen. -# # -# # We use Makie's `Observable` to animate the data. To dive into how `Observable`s work we -# # refer to [Makie.jl's Documentation](https://makie.juliaplots.org/stable/documentation/nodes/index.html). +# Now we can visualize our resutls! We use `CairoMakie` here. On a system with OpenGL +# `using GLMakie` is more convenient as figures will be displayed on the screen. +# +# We use Makie's `Observable` to animate the data. To dive into how `Observable`s work we +# refer to [Makie.jl's Documentation](https://makie.juliaplots.org/stable/documentation/nodes/index.html). -# using CairoMakie +using CairoMakie -# n = Observable(1) +n = Observable(1) -# title = @lift @sprintf("t = %1.2f days = %1.2f T₂", -# round(times[$n] / day, digits=2) , round(times[$n] / T₂, digits=2)) +title = @lift @sprintf("t = %1.2f days = %1.2f T₂", + round(times[$n] / day, digits=2) , round(times[$n] / T₂, digits=2)) -# u′n = @lift u′_t[$n] -# wn = @lift w_t[$n] -# N²n = @lift N²_t[$n] +u′n = @lift u′_t[$n] + wn = @lift w_t[$n] +N²n = @lift N²_t[$n] -# axis_kwargs = (xlabel = "x [km]", -# ylabel = "z [km]", -# limits = ((-grid.Lx/2e3, grid.Lx/2e3), (-grid.Lz/1e3, 0)), # note conversion to kilometers -# titlesize = 20) +axis_kwargs = (xlabel = "x [km]", + ylabel = "z [km]", + limits = ((-grid.Lx/2e3, grid.Lx/2e3), (-grid.Lz/1e3, 0)), # note conversion to kilometers + titlesize = 20) -# fig = Figure(size = (700, 900)) +fig = Figure(size = (700, 900)) -# fig[1, :] = Label(fig, title, fontsize=24, tellwidth=false) +fig[1, :] = Label(fig, title, fontsize=24, tellwidth=false) -# ax_u = Axis(fig[2, 1]; title = "u'-velocity", axis_kwargs...) -# hm_u = heatmap!(ax_u, xu, zu, u′n; nan_color=:gray, colorrange=(-umax, umax), colormap=:balance) -# Colorbar(fig[2, 2], hm_u, label = "m s⁻¹") +ax_u = Axis(fig[2, 1]; title = "u'-velocity", axis_kwargs...) +hm_u = heatmap!(ax_u, xu, zu, u′n; nan_color=:gray, colorrange=(-umax, umax), colormap=:balance) +Colorbar(fig[2, 2], hm_u, label = "m s⁻¹") -# ax_w = Axis(fig[3, 1]; title = "w-velocity", axis_kwargs...) -# hm_w = heatmap!(ax_w, xw, zw, wn; nan_color=:gray, colorrange=(-wmax, wmax), colormap=:balance) -# Colorbar(fig[3, 2], hm_w, label = "m s⁻¹") +ax_w = Axis(fig[3, 1]; title = "w-velocity", axis_kwargs...) +hm_w = heatmap!(ax_w, xw, zw, wn; nan_color=:gray, colorrange=(-wmax, wmax), colormap=:balance) +Colorbar(fig[3, 2], hm_w, label = "m s⁻¹") -# ax_N² = Axis(fig[4, 1]; title = "stratification N²", axis_kwargs...) -# hm_N² = heatmap!(ax_N², xN², zN², N²n; nan_color=:gray, colorrange=(0.9Nᵢ², 1.1Nᵢ²), colormap=:magma) -# Colorbar(fig[4, 2], hm_N², label = "s⁻²") +ax_N² = Axis(fig[4, 1]; title = "stratification N²", axis_kwargs...) +hm_N² = heatmap!(ax_N², xN², zN², N²n; nan_color=:gray, colorrange=(0.9Nᵢ², 1.1Nᵢ²), colormap=:magma) +Colorbar(fig[4, 2], hm_N², label = "s⁻²") -# fig +fig -# # Finally, we can record a movie. +# Finally, we can record a movie. -# @info "Making an animation from saved data..." +@info "Making an animation from saved data..." -# frames = 1:length(times) +frames = 1:length(times) -# record(fig, filename * ".mp4", frames, framerate=16) do i -# @info string("Plotting frame ", i, " of ", frames[end]) -# n[] = i -# end -# nothing #hide +record(fig, filename * ".mp4", frames, framerate=16) do i + @info string("Plotting frame ", i, " of ", frames[end]) + n[] = i +end +nothing #hide -# # ![](internal_tide.mp4) +# ![](internal_tide.mp4) From b8401cc0cc87d31241fe9c67948ecf35576ff387 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 16 Oct 2024 20:14:03 +0200 Subject: [PATCH 386/567] correct correction --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 1 + src/ImmersedBoundaries/partial_cell_bottom.jl | 1 + 2 files changed, 2 insertions(+) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index e6d75bc7fa..29452164c0 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -109,6 +109,7 @@ correct_bottom_height!(bottom_field, grid, ib) = @kernel function _correct_bottom_height!(bottom_field, grid, ib::GridFittedBottom) i, j = @index(Global, NTuple) zb = @inbounds bottom_field[i, j, 1] + @inbounds bottom_field[i, j, 1] = znode(i, j, 1, grid, c, c, f) condition = ib.immersed_condition for k in 1:grid.Nz z⁺ = znode(i, j, k+1, grid, c, c, f) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index 7988fa9ec9..7034c01dc6 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -73,6 +73,7 @@ end @kernel function _correct_bottom_height!(bottom_field, grid, ib::PartialCellBottom) i, j = @index(Global, NTuple) zb = @inbounds bottom_field[i, j, 1] + @inbounds bottom_field[i, j, 1] = znode(i, j, 1, grid, c, c, f) ϵ = ib.minimum_fractional_cell_height for k in 1:grid.Nz z⁻ = znode(i, j, k, grid, c, c, f) From eb3dd44065db6336640063f3f84a65445b70852f Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 17 Oct 2024 09:46:18 +0200 Subject: [PATCH 387/567] on_architecture --- src/Grids/z_star_vertical_coordinate.jl | 36 ++++++++++++------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/Grids/z_star_vertical_coordinate.jl b/src/Grids/z_star_vertical_coordinate.jl index a304f9eb21..0402c7c7e1 100644 --- a/src/Grids/z_star_vertical_coordinate.jl +++ b/src/Grids/z_star_vertical_coordinate.jl @@ -81,26 +81,26 @@ ZStarVerticalCoordinate(r_faces::Union{Tuple, AbstractVector}) = ZStarVerticalCo ZStarVerticalCoordinate(r⁻::Number, r⁺::Number) = ZStarVerticalCoordinate((r⁻, r⁺), nothing, nothing, nothing, nothing, nothing, nothing, nothing, nothing) Adapt.adapt_structure(to, coord::ZStarVerticalCoordinate) = - ZStarSpacing(Adapt.adapt(to, coord.reference), - Adapt.adapt(to, coord.sᶜᶜⁿ), - Adapt.adapt(to, coord.sᶠᶜⁿ), - Adapt.adapt(to, coord.sᶜᶠⁿ), - Adapt.adapt(to, coord.sᶠᶠⁿ), - Adapt.adapt(to, coord.sᶜᶜ⁻), - Adapt.adapt(to, coord.sᶠᶜ⁻), - Adapt.adapt(to, coord.sᶜᶠ⁻), - Adapt.adapt(to, coord.∂t_s)) + ZStarVerticalCoordinate(Adapt.adapt(to, coord.reference), + Adapt.adapt(to, coord.sᶜᶜⁿ), + Adapt.adapt(to, coord.sᶠᶜⁿ), + Adapt.adapt(to, coord.sᶜᶠⁿ), + Adapt.adapt(to, coord.sᶠᶠⁿ), + Adapt.adapt(to, coord.sᶜᶜ⁻), + Adapt.adapt(to, coord.sᶠᶜ⁻), + Adapt.adapt(to, coord.sᶜᶠ⁻), + Adapt.adapt(to, coord.∂t_s)) on_architecture(arch, coord::ZStarVerticalCoordinate) = - ZStarSpacing(on_architecture(arch, coord.reference), - on_architecture(arch, coord.sᶜᶜⁿ), - on_architecture(arch, coord.sᶠᶜⁿ), - on_architecture(arch, coord.sᶜᶠⁿ), - on_architecture(arch, coord.sᶠᶠⁿ), - on_architecture(arch, coord.sᶜᶜ⁻), - on_architecture(arch, coord.sᶠᶜ⁻), - on_architecture(arch, coord.sᶜᶠ⁻), - on_architecture(arch, coord.∂t_s)) + ZStarVerticalCoordinate(on_architecture(arch, coord.reference), + on_architecture(arch, coord.sᶜᶜⁿ), + on_architecture(arch, coord.sᶠᶜⁿ), + on_architecture(arch, coord.sᶜᶠⁿ), + on_architecture(arch, coord.sᶠᶠⁿ), + on_architecture(arch, coord.sᶜᶜ⁻), + on_architecture(arch, coord.sᶠᶜ⁻), + on_architecture(arch, coord.sᶜᶠ⁻), + on_architecture(arch, coord.∂t_s)) Grids.coordinate_summary(::Bounded, Δ::ZStarVerticalCoordinate, name) = @sprintf("Free-surface following with Δ%s=%s", name, prettysummary(Δ.reference)) From 2d36ed7ab098c32af11f954a5d07dfd3868d0334 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 17 Oct 2024 10:09:41 +0200 Subject: [PATCH 388/567] correct nonhydrostatic tendency --- src/Grids/z_star_vertical_coordinate.jl | 1 + .../hydrostatic_free_surface_model.jl | 1 - .../NonhydrostaticModels/compute_nonhydrostatic_tendencies.jl | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Grids/z_star_vertical_coordinate.jl b/src/Grids/z_star_vertical_coordinate.jl index 0402c7c7e1..f36ebb7192 100644 --- a/src/Grids/z_star_vertical_coordinate.jl +++ b/src/Grids/z_star_vertical_coordinate.jl @@ -77,6 +77,7 @@ struct ZStarVerticalCoordinate{R, SCC, SFC, SCF, SFF} <: AbstractVerticalCoordin ∂t_s :: SCC end +# Convenience constructors ZStarVerticalCoordinate(r_faces::Union{Tuple, AbstractVector}) = ZStarVerticalCoordinate(r_faces, nothing, nothing, nothing, nothing, nothing, nothing, nothing, nothing) ZStarVerticalCoordinate(r⁻::Number, r⁺::Number) = ZStarVerticalCoordinate((r⁻, r⁺), nothing, nothing, nothing, nothing, nothing, nothing, nothing, nothing) diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl index 764674f219..6b92127487 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_model.jl @@ -100,7 +100,6 @@ Keyword arguments - `velocities`: The model velocities. Default: `nothing`. - `pressure`: Hydrostatic pressure field. Default: `nothing`. - `diffusivity_fields`: Diffusivity fields. Default: `nothing`. - - `vertical_coordinate`: choice between the default `ZCoordinate` and the free-surface following `ZStarSpacing` - `auxiliary_fields`: `NamedTuple` of auxiliary fields. Default: `nothing`. """ function HydrostaticFreeSurfaceModel(; grid, diff --git a/src/Models/NonhydrostaticModels/compute_nonhydrostatic_tendencies.jl b/src/Models/NonhydrostaticModels/compute_nonhydrostatic_tendencies.jl index cc71605cb0..fb7e89ee64 100644 --- a/src/Models/NonhydrostaticModels/compute_nonhydrostatic_tendencies.jl +++ b/src/Models/NonhydrostaticModels/compute_nonhydrostatic_tendencies.jl @@ -116,7 +116,7 @@ function compute_interior_tendency_contributions!(model, kernel_parameters; acti end_tracer_kernel_args = (buoyancy, biogeochemistry, background_fields, velocities, tracers, auxiliary_fields, diffusivities) - for tracer_index in eachindex(tracers) + for tracer_index in 1:length(tracers) @inbounds c_tendency = tendencies[tracer_index + 3] @inbounds forcing = forcings[tracer_index + 3] @inbounds c_immersed_bc = tracers[tracer_index].boundary_conditions.immersed From 275a642bf07acffe0525d6f5ff3ebda79356fdca Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 17 Oct 2024 11:14:50 +0200 Subject: [PATCH 389/567] some more fixes plus a testset --- .buildkite/pipeline.yml | 35 +++++++++ .../split_explicit_free_surface.jl | 4 - test/dependencies_for_runtests.jl | 2 +- test/runtests.jl | 6 ++ .../test_split_explicit_vertical_integrals.jl | 21 ++--- test/test_zstar_coordinate.jl | 78 ++++++++++++------- 6 files changed, 96 insertions(+), 50 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 32ae563fae..2b82ba4541 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -661,6 +661,41 @@ steps: limit: 1 depends_on: "init_cpu" +##### +##### Vertical Coordinates tests +##### + + - label: "🦧 gpu vertical coordinate" + env: + JULIA_DEPOT_PATH: "$SVERDRUP_HOME/.julia-$BUILDKITE_BUILD_NUMBER" + TEST_GROUP: "vertical_coordinate" + commands: + - "$SVERDRUP_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" + agents: + queue: Oceananigans + architecture: GPU + retry: + automatic: + - exit_status: 1 + limit: 1 + depends_on: "init_gpu" + + - label: "🦍 cpu vertical coordinate" + env: + JULIA_DEPOT_PATH: "$TARTARUS_HOME/.julia-$BUILDKITE_BUILD_NUMBER" + TEST_GROUP: "vertical_coordinate" + CUDA_VISIBLE_DEVICES: "-1" + commands: + - "$TARTARUS_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" + agents: + queue: Oceananigans + architecture: CPU + retry: + automatic: + - exit_status: 1 + limit: 1 + depends_on: "init_cpu" + ##### ##### Enzyme extension tests ##### diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index feb0d4a499..6992a410de 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -403,10 +403,6 @@ Adapt.adapt_structure(to, free_surface::SplitExplicitFreeSurface) = Adapt.adapt_structure(to, auxiliary::SplitExplicitAuxiliaryFields) = SplitExplicitAuxiliaryFields(Adapt.adapt(to, auxiliary.Gᵁ), Adapt.adapt(to, auxiliary.Gⱽ), - Adapt.adapt(to, auxiliary.Hᶠᶜ), - Adapt.adapt(to, auxiliary.Hᶜᶠ), - Adapt.adapt(to, auxiliary.Hᶜᶜ), - Adapt.adapt(to, auxiliary.Hᶠᶠ), nothing) for Type in (:SplitExplicitFreeSurface, diff --git a/test/dependencies_for_runtests.jl b/test/dependencies_for_runtests.jl index 3e15e0a3d8..54da7bbc52 100644 --- a/test/dependencies_for_runtests.jl +++ b/test/dependencies_for_runtests.jl @@ -5,7 +5,7 @@ using Random using Statistics using LinearAlgebra using Logging -using Enzyme +# using Enzyme using SparseArrays using JLD2 using FFTW diff --git a/test/runtests.jl b/test/runtests.jl index c78eeefbb6..ec98953fd5 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -208,6 +208,12 @@ CUDA.allowscalar() do end end + if group == :vertical_coordinate || group == :all + @testset "Vertical coordinate tests" begin + include("test_zstar_coordinate.jl") + end + end + # Tests for Enzyme extension if group == :enzyme || group == :all @testset "Enzyme extension tests" begin diff --git a/test/test_split_explicit_vertical_integrals.jl b/test/test_split_explicit_vertical_integrals.jl index c0a82b8a4d..40f8920a4b 100644 --- a/test/test_split_explicit_vertical_integrals.jl +++ b/test/test_split_explicit_vertical_integrals.jl @@ -54,17 +54,13 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_barotropic_mode! @testset "Inexact integration" begin # Test 2: Check that vertical integrals work on the CPU(). The following should be "inexact" - Δz = zeros(Nz) - Δz .= grid.Δzᵃᵃᶠ set_u_check(x, y, z) = cos((π / 2) * z / Lz) set_U_check(x, y, z) = (sin(0) - (-2 * Lz / (π))) set!(u, set_u_check) exact_U = similar(U) set!(exact_U, set_U_check) - Hᶠᶜ = auxiliary.Hᶠᶜ - Hᶜᶠ = auxiliary.Hᶜᶠ - compute_barotropic_mode!(U, V, grid, u, v, Hᶠᶜ, Hᶜᶠ, η̅) + compute_barotropic_mode!(U, V, grid, u, v, η̅) tolerance = 1e-3 @test all((Array(interior(U) .- interior(exact_U))) .< tolerance) @@ -73,22 +69,20 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_barotropic_mode! set!(v, set_v_check) exact_V = similar(V) set!(exact_V, set_V_check) - compute_barotropic_mode!(U, V, grid, u, v, Hᶠᶜ, Hᶜᶠ, η̅) + compute_barotropic_mode!(U, V, grid, u, v, η̅) @test all((Array(interior(V) .- interior(exact_V))) .< tolerance) end @testset "Vertical Integral " begin - Δz = zeros(Nz) - Δz .= grid.Δzᵃᵃᶜ u .= 0.0 U .= 1.0 - compute_barotropic_mode!(U, V, grid, u, v, Hᶠᶜ, Hᶜᶠ, η̅) + compute_barotropic_mode!(U, V, grid, u, v, η̅) @test all(Array(U.data.parent) .== 0.0) u .= 1.0 U .= 1.0 - compute_barotropic_mode!(U, V, grid, u, v, Hᶠᶜ, Hᶜᶠ, η̅) + compute_barotropic_mode!(U, V, grid, u, v, η̅) @test all(Array(interior(U)) .≈ Lz) set_u_check(x, y, z) = sin(x) @@ -96,7 +90,7 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_barotropic_mode! set!(u, set_u_check) exact_U = similar(U) set!(exact_U, set_U_check) - compute_barotropic_mode!(U, V, grid, u, v, Hᶠᶜ, Hᶜᶠ, η̅) + compute_barotropic_mode!(U, V, grid, u, v, η̅) @test all(Array(interior(U)) .≈ Array(interior(exact_U))) set_v_check(x, y, z) = sin(x) * z * cos(y) @@ -104,7 +98,7 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_barotropic_mode! set!(v, set_v_check) exact_V = similar(V) set!(exact_V, set_V_check) - compute_barotropic_mode!(U, V, grid, u, v, Hᶠᶜ, Hᶜᶠ, η̅) + compute_barotropic_mode!(U, V, grid, u, v, η̅) @test all(Array(interior(V)) .≈ Array(interior(exact_V))) end @@ -144,9 +138,6 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_barotropic_mode! set!(V̅, set_V̅) set!(v_corrected, set_v_corrected) - Δz = zeros(Nz) - Δz .= grid.Δzᵃᵃᶜ - barotropic_split_explicit_corrector!(u, v, sefs, grid) @test all(Array((interior(u) .- interior(u_corrected))) .< 1e-14) @test all(Array((interior(v) .- interior(v_corrected))) .< 1e-14) diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index fb0552f4fd..47e80839e5 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -1,6 +1,7 @@ include("dependencies_for_runtests.jl") -using Oceananigans.Models.HydrostaticFreeSurfaceModels: ZStar +using Random +using Oceananigans.ImmersedBoundaries: PartialCellBottom function test_zstar_coordinate(model, Ni, Δt) @@ -25,39 +26,56 @@ function test_zstar_coordinate(model, Ni, Δt) return nothing end +@testset "ZStar coordinate testset" begin -@testset "Testing z-star coordinates" begin - - z_uniform = ZStarVerticalCoordinate((-10, 0)) + z_uniform = ZStarVerticalCoordinate(-10, 0) z_stretched = ZStarVerticalCoordinate(collect(-10:0)) + topologies = ((Periodic, Periodic, Bounded), + (Periodic, Bounded, Bounded), + (Bounded, Periodic, Bounded), + (Bounded, Bounded, Bounded)) for arch in archs - llg = LatitudeLongitudeGrid(arch; size = (10, 10, 10), latitude = (-10, 10), longitude = (-10, 10), z = z_uniform) - rtg = RectilinearGrid(arch; size = (10, 10, 10), x = (-10, 10), y = (-10, 10), z = z_uniform) - - llgv = LatitudeLongitudeGrid(arch; size = (10, 10, 10), latitude = (-10, 10), longitude = (-10, 10), z = z_stretched) - rtgv = RectilinearGrid(arch; size = (10, 10, 10), x = (-10, 10), y = (-10, 10), z = z_stretched) - - illg = ImmersedBoundaryGrid(llg, GridFittedBottom((x, y) -> - rand() - 5)) - irtg = ImmersedBoundaryGrid(rtg, GridFittedBottom((x, y) -> - rand() - 5)) - - illgv = ImmersedBoundaryGrid(llgv, GridFittedBottom((x, y) -> - rand() - 5)) - irtgv = ImmersedBoundaryGrid(rtgv, GridFittedBottom((x, y) -> - rand() - 5)) - - grids = [llg, rtg, llgv, rtgv, illg, irtg, illgv, irtgv] - - for grid in grids - free_surface = SplitExplicitFreeSurface(grid; cfl = 0.75) - model = HydrostaticFreeSurfaceModel(; grid, - free_surface, - tracers = (:b, :c), - bouyancy = BuoyancTracer()) - - bᵢ(x, y, z) = x < grid.Lx / 2 ? 0.06 : 0.01 - - set!(model, c = (x, y, z) -> rand(), b = bᵢ) - - test_zstar_coordinate(model, 100, 10) + for topology in topologies + Random.seed!(1234) + + rtg = RectilinearGrid(arch; size = (10, 10, 10), x = (-10, 10), y = (-10, 10), topology, z = z_uniform) + rtgv = RectilinearGrid(arch; size = (10, 10, 10), x = (-10, 10), y = (-10, 10), topology, z = z_stretched) + + irtg = ImmersedBoundaryGrid(rtg, GridFittedBottom((x, y) -> - rand() - 5)) + irtgv = ImmersedBoundaryGrid(rtgv, GridFittedBottom((x, y) -> - rand() - 5)) + prtg = ImmersedBoundaryGrid(rtg, PartialCellBottom((x, y) -> - rand() - 5)) + prtgv = ImmersedBoundaryGrid(rtgv, PartialCellBottom((x, y) -> - rand() - 5)) + + if topology[2] == Bounded + llg = LatitudeLongitudeGrid(arch; size = (10, 10, 10), latitude = (-10, 10), longitude = (-180, 180), topology, z = z_uniform) + llgv = LatitudeLongitudeGrid(arch; size = (10, 10, 10), latitude = (-10, 10), longitude = (-180, 180), topology, z = z_stretched) + + illg = ImmersedBoundaryGrid(llg, GridFittedBottom((x, y) -> - rand() - 5)) + illgv = ImmersedBoundaryGrid(llgv, GridFittedBottom((x, y) -> - rand() - 5)) + pllg = ImmersedBoundaryGrid(llg, PartialCellBottom((x, y) -> - rand() - 5)) + pllgv = ImmersedBoundaryGrid(llgv, PartialCellBottom((x, y) -> - rand() - 5)) + + grids = [llg, rtg, llgv, rtgv, illg, irtg, illgv, irtgv, pllg, prtg, pllgv, prtgv] + else + grids = [rtg, rtgv, irtg, irtgv, prtg, prtgv] + end + + for grid in grids + @info " Testing z-star coordinates on $(summary(grid))..." + + free_surface = SplitExplicitFreeSurface(grid; cfl = 0.75) + model = HydrostaticFreeSurfaceModel(; grid, + free_surface, + tracers = (:b, :c), + buoyancy = BuoyancyTracer()) + + bᵢ(x, y, z) = x < grid.Lx / 2 ? 0.06 : 0.01 + + set!(model, c = (x, y, z) -> rand(), b = bᵢ) + + test_zstar_coordinate(model, 100, 10) + end end end end \ No newline at end of file From 3e99c79e8484c794963f3d1a0c3d553637006a8e Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 17 Oct 2024 11:15:15 +0200 Subject: [PATCH 390/567] whoops --- test/dependencies_for_runtests.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/dependencies_for_runtests.jl b/test/dependencies_for_runtests.jl index 54da7bbc52..3e15e0a3d8 100644 --- a/test/dependencies_for_runtests.jl +++ b/test/dependencies_for_runtests.jl @@ -5,7 +5,7 @@ using Random using Statistics using LinearAlgebra using Logging -# using Enzyme +using Enzyme using SparseArrays using JLD2 using FFTW From 01ddfdfdd9b7f563c0c81a77c779fa4b31fee214 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 17 Oct 2024 11:26:05 +0200 Subject: [PATCH 391/567] revert the outputwriter --- src/OutputWriters/jld2_output_writer.jl | 8 ++++---- test/test_zstar_coordinate.jl | 24 +++++++++++++++++------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/src/OutputWriters/jld2_output_writer.jl b/src/OutputWriters/jld2_output_writer.jl index a2559832f5..50a0a31937 100644 --- a/src/OutputWriters/jld2_output_writer.jl +++ b/src/OutputWriters/jld2_output_writer.jl @@ -203,7 +203,7 @@ function initialize_jld2_file!(filepath, init, jld2_kw, including, outputs, mode @warn """Failed to execute user `init` for $filepath because $(typeof(err)): $(sprint(showerror, err))""" end - # try + try jldopen(filepath, "a+"; jld2_kw...) do file saveproperties!(file, model, including) @@ -212,9 +212,9 @@ function initialize_jld2_file!(filepath, init, jld2_kw, including, outputs, mode serializeproperty!(file, "serialized/$property", getproperty(model, property)) end end - # catch err - # @warn """Failed to save and serialize $including in $filepath because $(typeof(err)): $(sprint(showerror, err))""" - # end + catch err + @warn """Failed to save and serialize $including in $filepath because $(typeof(err)): $(sprint(showerror, err))""" + end # Serialize the location and boundary conditions of each output. for (name, field) in pairs(outputs) diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index 47e80839e5..7a1b140a91 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -15,17 +15,27 @@ function test_zstar_coordinate(model, Ni, Δt) # (2) vertical velocities are zero at the top surface for _ in 1:Ni time_step!(model, Δt) - ∫b = Field(Integral(model.tracers.b)) - ∫c = Field(Integral(model.tracers.c)) - - @test interior(∫b, 1, 1, 1) ≈ interior(∫bᵢ, 1, 1, 1) - @test interior(∫c, 1, 1, 1) ≈ interior(∫cᵢ, 1, 1, 1) - @test maximum(interior(w, :, :, Nz+1)) < eps(eltype(w)) end + ∫b = Field(Integral(model.tracers.b)) + ∫c = Field(Integral(model.tracers.c)) + + @test interior(∫b, 1, 1, 1) ≈ interior(∫bᵢ, 1, 1, 1) + @test interior(∫c, 1, 1, 1) ≈ interior(∫cᵢ, 1, 1, 1) + @test maximum(interior(w, :, :, Nz+1)) < eps(eltype(w)) + return nothing end +function info_message(grid) + msg1 = "Testing z-star coordinates on $(architecture(grid)) on a " + msg2 = string(getnamewrapper(grid)) + msg3 = grid.Δzᵃᵃᶠ.reference isa Number ? " with uniform spacing" : " with stretched spacing" + msg4 = grid isa ImmersedBoundaryGrid ? " and $(string(getnamewrapper(grid.immersed_boundary))) immersed boundary" : "" + + return msg1 * msg2 * msg3 * msg4 +end + @testset "ZStar coordinate testset" begin z_uniform = ZStarVerticalCoordinate(-10, 0) @@ -62,7 +72,7 @@ end end for grid in grids - @info " Testing z-star coordinates on $(summary(grid))..." + @info info_message(grid) free_surface = SplitExplicitFreeSurface(grid; cfl = 0.75) model = HydrostaticFreeSurfaceModel(; grid, From 0f4dfc52dceb6cd4049461cc32f2123dcd7d42ad Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 17 Oct 2024 13:52:41 +0200 Subject: [PATCH 392/567] back to the previous split_explicit barotropic mode --- .../split_explicit_free_surface_kernels.jl | 36 ++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 821b8a58db..31180f44e6 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -230,9 +230,9 @@ end # For Zstar vertical spacing the vertical integral includes the dynamic height # Remember, the vertical coordinate has not yet been updated! # For this reason the integration has to be performed manually -@kernel function _barotropic_mode_kernel!(U, V, grid, u, v, η) +@kernel function _barotropic_mode_kernel!(U, V, grid, ::Nothing, u, v, η) i, j = @index(Global, NTuple) - k_top = grid.Nz + 1 + k_top = grid.Nz+1 hᶠᶜ = domain_depthᶠᶜᵃ(i, j, grid) hᶜᶠ = domain_depthᶜᶠᵃ(i, j, grid) @@ -250,8 +250,36 @@ end end end -@inline compute_barotropic_mode!(U, V, grid, u, v, η) = - launch!(architecture(grid), grid, :xy, _barotropic_mode_kernel!, U, V, grid, u, v, η) +# Barotropic Model Kernels +# u_Δz = u * Δz +@kernel function _barotropic_mode_kernel!(U, V, grid, active_cells_map, u, v, η) + idx = @index(Global, Linear) + i, j = active_linear_index_to_tuple(idx, active_cells_map) + k_top = grid.Nz+1 + + hᶠᶜ = domain_depthᶠᶜᵃ(i, j, grid) + hᶜᶠ = domain_depthᶜᶠᵃ(i, j, grid) + + sᶠᶜ = ifelse(hᶠᶜ == 0, one(grid), dynamic_domain_depthᶠᶜᵃ(i, j, k_top, grid, η) / hᶠᶜ) + sᶜᶠ = ifelse(hᶜᶠ == 0, one(grid), dynamic_domain_depthᶜᶠᵃ(i, j, k_top, grid, η) / hᶜᶠ) + + # hand unroll first loop + @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δrᶠᶜᶜ(i, j, 1, grid) * sᶠᶜ + @inbounds V[i, j, k_top-1] = v[i, j, 1] * Δrᶜᶠᶜ(i, j, 1, grid) * sᶜᶠ + + @unroll for k in 2:grid.Nz + @inbounds U[i, j, k_top-1] += u[i, j, k] * Δrᶠᶜᶜ(i, j, k, grid) * sᶠᶜ + @inbounds V[i, j, k_top-1] += v[i, j, k] * Δrᶜᶠᶜ(i, j, k, grid) * sᶜᶠ + end +end + +@inline function compute_barotropic_mode!(U, V, grid, u, v, η) + active_cells_map = retrieve_surface_active_cells_map(grid) + + launch!(architecture(grid), grid, :xy, _barotropic_mode_kernel!, U, V, grid, active_cells_map, u, v, η; active_cells_map) + + return nothing +end @kernel function _barotropic_split_explicit_corrector!(u, v, grid, U̅, V̅, U, V, η) i, j, k = @index(Global, NTuple) From 9fea102822f3a9cae111377d7b6771f540c20f17 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 17 Oct 2024 16:45:06 +0200 Subject: [PATCH 393/567] simpler baroclinic gyre --- validation/z_star_coordinate/baroclinic_double_gyre.jl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/validation/z_star_coordinate/baroclinic_double_gyre.jl b/validation/z_star_coordinate/baroclinic_double_gyre.jl index 022a084ad6..bf85e64ed6 100644 --- a/validation/z_star_coordinate/baroclinic_double_gyre.jl +++ b/validation/z_star_coordinate/baroclinic_double_gyre.jl @@ -7,7 +7,7 @@ using Printf arch = CPU() z_faces = ZStarVerticalCoordinate((-1800, 0)) -grid = LatitudeLongitudeGrid(arch; size = (60, 60, 18), +grid = LatitudeLongitudeGrid(arch; size = (60, 60, 2), latitude = (15, 75), longitude = (0, 60), halo = (5, 5, 5), @@ -75,7 +75,7 @@ end return p.𝓋 * (b - b★) end -Δz₀ = Δzᶜᶜᶜ(1, 1, grid.Nz, grid) # Surface layer thickness +Δz₀ = 10 # Surface layer thickness [m] Δb = α * g * (θ⁺ - θ⁻) # Buoyancy difference @@ -104,6 +104,7 @@ model = HydrostaticFreeSurfaceModel(; grid, numerics..., closure) + N² = Δb / grid.Lz bᵢ(x, y, z) = N² * (grid.Lz + z) @@ -148,12 +149,12 @@ simulation.callbacks[:progress] = Callback(progress, IterationInterval(100)) simulation.output_writers[:snapshots] = JLD2OutputWriter(model, field_outputs, overwrite_existing = true, schedule = TimeInterval(60days), - filename = "baroclinic_double_gyre") + filename = "baroclinic_double_gyre_new") simulation.output_writers[:free_surface] = JLD2OutputWriter(model, (; η = model.free_surface.η), overwrite_existing = true, indices = (:, :, grid.Nz+1), schedule = TimeInterval(60days), - filename = "baroclinic_double_gyre_free_surface") + filename = "baroclinic_double_gyre_free_surface_new") run!(simulation) \ No newline at end of file From b967754bd00596f44aeadc480509e55414508953 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 21 Oct 2024 09:28:19 +0200 Subject: [PATCH 394/567] static_column_depth --- src/Grids/Grids.jl | 2 +- src/Grids/grid_utils.jl | 8 +- src/ImmersedBoundaries/ImmersedBoundaries.jl | 2 +- src/ImmersedBoundaries/grid_fitted_bottom.jl | 16 ++-- ...ompute_hydrostatic_free_surface_buffers.jl | 73 +++++++++++++++++++ .../split_explicit_free_surface_kernels.jl | 8 +- .../catke_equation.jl | 2 +- .../catke_mixing_length.jl | 8 +- 8 files changed, 96 insertions(+), 23 deletions(-) create mode 100644 src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_buffers.jl diff --git a/src/Grids/Grids.jl b/src/Grids/Grids.jl index e88ca07980..27bd8f3a4a 100644 --- a/src/Grids/Grids.jl +++ b/src/Grids/Grids.jl @@ -18,7 +18,7 @@ export xnodes, ynodes, znodes, λnodes, φnodes export spacings export xspacings, yspacings, zspacings, xspacing, yspacing, zspacing export minimum_xspacing, minimum_yspacing, minimum_zspacing -export domain_depthᶜᶜᵃ, domain_depthᶠᶜᵃ, domain_depthᶜᶠᵃ, domain_depthᶠᶠᵃ +export static_column_depthᶜᶜᵃ, static_column_depthᶠᶜᵃ, static_column_depthᶜᶠᵃ, static_column_depthᶠᶠᵃ export offset_data, new_data export on_architecture diff --git a/src/Grids/grid_utils.jl b/src/Grids/grid_utils.jl index efcefd2297..4bee71f311 100644 --- a/src/Grids/grid_utils.jl +++ b/src/Grids/grid_utils.jl @@ -318,10 +318,10 @@ coordinate_summary(topo, Δ::Union{AbstractVector, AbstractMatrix}, name) = ##### Bottom height ##### -@inline domain_depthᶜᶜᵃ(i, j, grid) = grid.Lz -@inline domain_depthᶜᶠᵃ(i, j, grid) = grid.Lz -@inline domain_depthᶠᶜᵃ(i, j, grid) = grid.Lz -@inline domain_depthᶠᶠᵃ(i, j, grid) = grid.Lz +@inline static_column_depthᶜᶜᵃ(i, j, grid) = grid.Lz +@inline static_column_depthᶜᶠᵃ(i, j, grid) = grid.Lz +@inline static_column_depthᶠᶜᵃ(i, j, grid) = grid.Lz +@inline static_column_depthᶠᶠᵃ(i, j, grid) = grid.Lz ##### ##### Spherical geometry diff --git a/src/ImmersedBoundaries/ImmersedBoundaries.jl b/src/ImmersedBoundaries/ImmersedBoundaries.jl index f823f54dfd..afad4b152a 100644 --- a/src/ImmersedBoundaries/ImmersedBoundaries.jl +++ b/src/ImmersedBoundaries/ImmersedBoundaries.jl @@ -53,7 +53,7 @@ import Oceananigans.Grids: architecture, with_halo, inflate_halo_size_one_dimens ξname, ηname, rname, node_names, xnodes, ynodes, znodes, λnodes, φnodes, nodes, ξnodes, ηnodes, rnodes, - domain_depthᶜᶜᵃ, domain_depthᶠᶜᵃ, domain_depthᶜᶠᵃ, domain_depthᶠᶠᵃ, + static_column_depthᶜᶜᵃ, static_column_depthᶠᶜᵃ, static_column_depthᶜᶠᵃ, static_column_depthᶠᶠᵃ, inactive_cell import Oceananigans.Coriolis: φᶠᶠᵃ diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 29452164c0..b44d253bb6 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -133,16 +133,16 @@ end const AGFBIB = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} -@inline domain_depthᶜᶜᵃ(i, j, ibg::AGFBIB) = @inbounds znode(i, j, ibg.Nz+1, ibg, c, c, f) - ibg.immersed_boundary.bottom_height[i, j, 1] -@inline domain_depthᶜᶠᵃ(i, j, ibg::AGFBIB) = min(domain_depthᶜᶜᵃ(i, j-1, ibg), domain_depthᶜᶜᵃ(i, j, ibg)) -@inline domain_depthᶠᶜᵃ(i, j, ibg::AGFBIB) = min(domain_depthᶜᶜᵃ(i-1, j, ibg), domain_depthᶜᶜᵃ(i, j, ibg)) -@inline domain_depthᶠᶠᵃ(i, j, ibg::AGFBIB) = min(domain_depthᶠᶜᵃ(i, j-1, ibg), domain_depthᶠᶜᵃ(i, j, ibg)) +@inline static_column_depthᶜᶜᵃ(i, j, ibg::AGFBIB) = @inbounds znode(i, j, ibg.Nz+1, ibg, c, c, f) - ibg.immersed_boundary.bottom_height[i, j, 1] +@inline static_column_depthᶜᶠᵃ(i, j, ibg::AGFBIB) = min(static_column_depthᶜᶜᵃ(i, j-1, ibg), static_column_depthᶜᶜᵃ(i, j, ibg)) +@inline static_column_depthᶠᶜᵃ(i, j, ibg::AGFBIB) = min(static_column_depthᶜᶜᵃ(i-1, j, ibg), static_column_depthᶜᶜᵃ(i, j, ibg)) +@inline static_column_depthᶠᶠᵃ(i, j, ibg::AGFBIB) = min(static_column_depthᶠᶜᵃ(i, j-1, ibg), static_column_depthᶠᶜᵃ(i, j, ibg)) # Make sure column_height works for horizontally-Flat topologies. XFlatAGFIBG = ImmersedBoundaryGrid{<:Any, <:Flat, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} YFlatAGFIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Flat, <:Any, <:Any, <:AbstractGridFittedBottom} -@inline domain_depthᶠᶜᵃ(i, j, ibg::XFlatAGFIBG) = domain_depthᶜᶜᵃ(i, j, ibg) -@inline domain_depthᶜᶠᵃ(i, j, ibg::YFlatAGFIBG) = domain_depthᶜᶜᵃ(i, j, ibg) -@inline domain_depthᶠᶠᵃ(i, j, ibg::XFlatAGFIBG) = domain_depthᶜᶠᵃ(i, j, ibg) -@inline domain_depthᶠᶠᵃ(i, j, ibg::YFlatAGFIBG) = domain_depthᶠᶜᵃ(i, j, ibg) +@inline static_column_depthᶠᶜᵃ(i, j, ibg::XFlatAGFIBG) = static_column_depthᶜᶜᵃ(i, j, ibg) +@inline static_column_depthᶜᶠᵃ(i, j, ibg::YFlatAGFIBG) = static_column_depthᶜᶜᵃ(i, j, ibg) +@inline static_column_depthᶠᶠᵃ(i, j, ibg::XFlatAGFIBG) = static_column_depthᶜᶠᵃ(i, j, ibg) +@inline static_column_depthᶠᶠᵃ(i, j, ibg::YFlatAGFIBG) = static_column_depthᶠᶜᵃ(i, j, ibg) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_buffers.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_buffers.jl new file mode 100644 index 0000000000..7dc110d662 --- /dev/null +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_hydrostatic_free_surface_buffers.jl @@ -0,0 +1,73 @@ +import Oceananigans.Models: compute_buffer_tendencies! + +using Oceananigans.Grids: halo_size +using Oceananigans.DistributedComputations: DistributedActiveCellsIBG +using Oceananigans.ImmersedBoundaries: retrieve_interior_active_cells_map +using Oceananigans.Models.NonhydrostaticModels: buffer_tendency_kernel_parameters, + buffer_p_kernel_parameters, + buffer_κ_kernel_parameters, + buffer_parameters + +# We assume here that top/bottom BC are always synchronized (no partitioning in z) +function compute_buffer_tendencies!(model::HydrostaticFreeSurfaceModel) + grid = model.grid + arch = architecture(grid) + + w_parameters = buffer_w_kernel_parameters(grid, arch) + p_parameters = buffer_p_kernel_parameters(grid, arch) + κ_parameters = buffer_κ_kernel_parameters(grid, model.closure, arch) + + # We need new values for `w`, `p` and `κ` + compute_auxiliaries!(model; w_parameters, p_parameters, κ_parameters) + + # parameters for communicating North / South / East / West side + compute_buffer_tendency_contributions!(grid, arch, model) + + return nothing +end + +function compute_buffer_tendency_contributions!(grid, arch, model) + kernel_parameters = buffer_tendency_kernel_parameters(grid, arch) + compute_hydrostatic_free_surface_tendency_contributions!(model, kernel_parameters) + return nothing +end + +function compute_buffer_tendency_contributions!(grid::DistributedActiveCellsIBG, arch, model) + maps = grid.interior_active_cells + + for (name, map) in zip(keys(maps), maps) + + # If there exists a buffer map, then we compute the buffer contributions. If not, the + # buffer contributions have already been calculated. We exclude the interior because it has + # already been calculated + compute_buffer = (name != :interior) && !isnothing(map) + + if compute_buffer + active_cells_map = retrieve_interior_active_cells_map(grid, Val(name)) + compute_hydrostatic_free_surface_tendency_contributions!(model, :xyz; active_cells_map) + end + end + + return nothing +end + +# w needs computing in the range - H + 1 : 0 and N - 1 : N + H - 1 +function buffer_w_kernel_parameters(grid, arch) + Nx, Ny, _ = size(grid) + Hx, Hy, _ = halo_size(grid) + + Sx = (Hx, Ny+2) + Sy = (Nx+2, Hy) + + # Offsets in tangential direction are == -1 to + # cover the required corners + param_west = (-Hx+2:1, 0:Ny+1) + param_east = (Nx:Nx+Hx-1, 0:Ny+1) + param_south = (0:Nx+1, -Hy+2:1) + param_north = (0:Nx+1, Ny:Ny+Hy-1) + + params = (param_west, param_east, param_south, param_north) + + return buffer_parameters(params, grid, arch) +end + diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 6b118d0868..10a41cc4c8 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -183,8 +183,8 @@ end advance_previous_velocity!(i, j, k_top-1, timestepper, U, Uᵐ⁻¹, Uᵐ⁻²) advance_previous_velocity!(i, j, k_top-1, timestepper, V, Vᵐ⁻¹, Vᵐ⁻²) - Hᶠᶜ = domain_depthᶠᶜᵃ(i, j, grid) - Hᶜᶠ = domain_depthᶜᶠᵃ(i, j, grid) + Hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) + Hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) # ∂τ(U) = - ∇η + G U[i, j, k_top-1] += Δτ * (- g * Hᶠᶜ * ∂xᶠᶜᶠ_η(i, j, k_top, grid, TX, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gᵁ[i, j, k_top-1]) @@ -271,8 +271,8 @@ end k_top = grid.Nz+1 @inbounds begin - Hᶠᶜ = domain_depthᶠᶜᵃ(i, j, grid) - Hᶜᶠ = domain_depthᶜᶠᵃ(i, j, grid) + Hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) + Hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / Hᶠᶜ v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / Hᶜᶠ diff --git a/src/TurbulenceClosures/turbulence_closure_implementations/TKEBasedVerticalDiffusivities/catke_equation.jl b/src/TurbulenceClosures/turbulence_closure_implementations/TKEBasedVerticalDiffusivities/catke_equation.jl index 73ddb5310c..bf2aaa7bfd 100644 --- a/src/TurbulenceClosures/turbulence_closure_implementations/TKEBasedVerticalDiffusivities/catke_equation.jl +++ b/src/TurbulenceClosures/turbulence_closure_implementations/TKEBasedVerticalDiffusivities/catke_equation.jl @@ -58,7 +58,7 @@ end ℓ★ = ifelse(isnan(ℓ★), zero(grid), ℓ★) ℓᴰ = max(ℓ★, ℓʰ) - H = domain_depthᶜᶜᵃ(i, j, grid) + H = static_column_depthᶜᶜᵃ(i, j, grid) return min(H, ℓᴰ) end diff --git a/src/TurbulenceClosures/turbulence_closure_implementations/TKEBasedVerticalDiffusivities/catke_mixing_length.jl b/src/TurbulenceClosures/turbulence_closure_implementations/TKEBasedVerticalDiffusivities/catke_mixing_length.jl index 19e81270fb..16d0366d37 100644 --- a/src/TurbulenceClosures/turbulence_closure_implementations/TKEBasedVerticalDiffusivities/catke_mixing_length.jl +++ b/src/TurbulenceClosures/turbulence_closure_implementations/TKEBasedVerticalDiffusivities/catke_mixing_length.jl @@ -5,7 +5,7 @@ using ..TurbulenceClosures: height_above_bottomᶜᶜᶠ, depthᶜᶜᶜ, height_above_bottomᶜᶜᶜ, - domain_depthᶜᶜᵃ + static_column_depthᶜᶜᵃ """ struct CATKEMixingLength{FT} @@ -232,7 +232,7 @@ end ℓ★ = ifelse(isnan(ℓ★), zero(grid), ℓ★) ℓu = max(ℓ★, ℓʰ) - H = domain_depthᶜᶜᵃ(i, j, grid) + H = static_column_depthᶜᶜᵃ(i, j, grid) return min(H, ℓu) end @@ -252,7 +252,7 @@ end ℓ★ = ifelse(isnan(ℓ★), zero(grid), ℓ★) ℓc = max(ℓ★, ℓʰ) - H = domain_depthᶜᶜᵃ(i, j, grid) + H = static_column_depthᶜᶜᵃ(i, j, grid) return min(H, ℓc) end @@ -272,7 +272,7 @@ end ℓ★ = ifelse(isnan(ℓ★), zero(grid), ℓ★) ℓe = max(ℓ★, ℓʰ) - H = domain_depthᶜᶜᵃ(i, j, grid) + H = static_column_depthᶜᶜᵃ(i, j, grid) return min(H, ℓe) end From 6046d2d084c0fe2d053d0a7f2dead31aa9f234b5 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 21 Oct 2024 16:22:51 +0200 Subject: [PATCH 395/567] Update src/Grids/grid_utils.jl Co-authored-by: Gregory L. Wagner --- src/Grids/grid_utils.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Grids/grid_utils.jl b/src/Grids/grid_utils.jl index 4bee71f311..178d7ab87c 100644 --- a/src/Grids/grid_utils.jl +++ b/src/Grids/grid_utils.jl @@ -315,7 +315,7 @@ coordinate_summary(topo, Δ::Union{AbstractVector, AbstractMatrix}, name) = name, prettysummary(maximum(parent(Δ)))) ##### -##### Bottom height +##### Static column depth ##### @inline static_column_depthᶜᶜᵃ(i, j, grid) = grid.Lz From a10df1613cc5a888ff4b75f67725ad3215cb426e Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Oct 2024 10:49:54 +0200 Subject: [PATCH 396/567] address comments --- Project.toml | 3 ++- src/ImmersedBoundaries/grid_fitted_bottom.jl | 25 +++++++++++-------- src/ImmersedBoundaries/partial_cell_bottom.jl | 7 ++---- src/MultiRegion/multi_region_grid.jl | 12 ++++----- ...multi_region_near_global_quarter_degree.jl | 2 +- 5 files changed, 26 insertions(+), 23 deletions(-) diff --git a/Project.toml b/Project.toml index 06a8553f22..13f1fb6317 100644 --- a/Project.toml +++ b/Project.toml @@ -79,9 +79,10 @@ julia = "1.9" [extras] DataDeps = "124859b0-ceae-595e-8997-d05f6a7a8dfe" Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" +OrthogonalSphericalShellGrids = "c2be9673-fb75-4747-82dc-aa2bb9f4aed0" SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" TimesDates = "bdfc003b-8df8-5c39-adcd-3a9087f5df4a" [targets] -test = ["DataDeps", "Enzyme", "SafeTestsets", "Test", "TimesDates"] +test = ["DataDeps", "Enzyme", "OrthogonalSphericalShellGrids", "SafeTestsets", "Test", "TimesDates"] diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 9679d1bc5a..c21106c175 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -23,8 +23,6 @@ struct GridFittedBottom{H, I} <: AbstractGridFittedBottom{H} immersed_condition :: I end -GridFittedBottom(bottom_height) = GridFittedBottom(bottom_height, CenterImmersedCondition()) - Base.summary(::CenterImmersedCondition) = "CenterImmersedCondition" Base.summary(::InterfaceImmersedCondition) = "InterfaceImmersedCondition" @@ -38,14 +36,21 @@ Return a bottom immersed boundary. Keyword Arguments ================= -* `bottom_height`: an array or function that gives the height of the - bottom in absolute ``z`` coordinates. -* `immersed_condition`: Determine whether the part of the domain that is immersed are all the cell centers that lie below - `bottom_height` (`CenterImmersedCondition()`; default) or all the cell faces that lie below `bottom_height` (`InterfaceImmersedCondition()`). - The only purpose of `immersed_condition` to allow `GridFittedBottom` and `PartialCellBottom` to have the same behavior when the - minimum fractional cell height for partial cells is set to 0. +* `bottom_height`: an array or function that gives the height of the + bottom in absolute ``z`` coordinates. + +* `immersed_condition`: Determine whether the part of the domain that is + immersed are all the cell centers that lie below + `bottom_height` (`CenterImmersedCondition()`; default) + or all the cell faces that lie below `bottom_height` + (`InterfaceImmersedCondition()`). The only purpose of + `immersed_condition` to allow `GridFittedBottom` and + `PartialCellBottom` to have the same behavior when the + minimum fractional cell height for partial cells is set + to 0. """ +GridFittedBottom(bottom_height) = GridFittedBottom(bottom_height, CenterImmersedCondition()) function Base.summary(ib::GridFittedBottom) zmax = maximum(ib.bottom_height) @@ -123,14 +128,14 @@ end return z ≤ zb end -@inline z_bottom(i, j, ibg::GFBIBG) = @inbounds ibg.immersed_boundary.bottom_height[i, j, 1] - ##### ##### Bottom height ##### const AGFBIB = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} +@inline z_bottom(i, j, ibg::AGFBIBG) = @inbounds ibg.immersed_boundary.bottom_height[i, j, 1] + @inline static_column_depthᶜᶜᵃ(i, j, ibg::AGFBIB) = @inbounds znode(i, j, ibg.Nz+1, ibg, c, c, f) - ibg.immersed_boundary.bottom_height[i, j, 1] @inline static_column_depthᶜᶠᵃ(i, j, ibg::AGFBIB) = min(static_column_depthᶜᶜᵃ(i, j-1, ibg), static_column_depthᶜᶜᵃ(i, j, ibg)) @inline static_column_depthᶠᶜᵃ(i, j, ibg::AGFBIB) = min(static_column_depthᶜᶜᵃ(i-1, j, ibg), static_column_depthᶜᶜᵃ(i, j, ibg)) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index 7034c01dc6..cd8a261ec3 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -63,14 +63,14 @@ end function ImmersedBoundaryGrid(grid, ib::PartialCellBottom) bottom_field = Field{Center, Center, Nothing}(grid) set!(bottom_field, ib.bottom_height) - @apply_regionally correct_bottom_height!(bottom_field, grid, ib) + @apply_regionally compute_numerical_bottom_height!(bottom_field, grid, ib) fill_halo_regions!(bottom_field) new_ib = PartialCellBottom(bottom_field, ib.minimum_fractional_cell_height) TX, TY, TZ = topology(grid) return ImmersedBoundaryGrid{TX, TY, TZ}(grid, new_ib) end -@kernel function _correct_bottom_height!(bottom_field, grid, ib::PartialCellBottom) +@kernel function _compute_numerical_bottom_height!(bottom_field, grid, ib::PartialCellBottom) i, j = @index(Global, NTuple) zb = @inbounds bottom_field[i, j, 1] @inbounds bottom_field[i, j, 1] = znode(i, j, 1, grid, c, c, f) @@ -183,6 +183,3 @@ YFlatPCBIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Flat, <:Any, <:Any, <:Partial @inline Δzᶜᶠᶠ(i, j, k, ibg::YFlatPCBIBG) = Δzᶜᶜᶠ(i, j, k, ibg) @inline Δzᶠᶠᶜ(i, j, k, ibg::XFlatPCBIBG) = Δzᶜᶠᶜ(i, j, k, ibg) @inline Δzᶠᶠᶜ(i, j, k, ibg::YFlatPCBIBG) = Δzᶠᶜᶜ(i, j, k, ibg) - -@inline bottom_height(i, j, ibg::PCBIBG) = @inbounds ibg.immersed_boundary.bottom_height[i, j, 1] - diff --git a/src/MultiRegion/multi_region_grid.jl b/src/MultiRegion/multi_region_grid.jl index 61e419c16a..0702b20d43 100644 --- a/src/MultiRegion/multi_region_grid.jl +++ b/src/MultiRegion/multi_region_grid.jl @@ -183,15 +183,15 @@ end function reconstruct_global_grid(mrg::ImmersedMultiRegionGrid) global_grid = reconstruct_global_grid(mrg.underlying_grid) - global_boundary = reconstruct_global_boundary(mrg.immersed_boundary) - global_boundary = on_architecture(architecture(mrg), global_boundary) + global_immersed_boundary = reconstruct_global_immersed_boundary(mrg.immersed_boundary) + global_immersed_boundary = on_architecture(architecture(mrg), global_immersed_boundary) - return ImmersedBoundaryGrid(global_grid, global_boundary) + return ImmersedBoundaryGrid(global_grid, global_immersed_boundary) end -reconstruct_global_boundary(g::GridFittedBottom{<:Field}) = GridFittedBottom(reconstruct_global_field(g.bottom_height), g.immersed_condition) -reconstruct_global_boundary(g::PartialCellBottom{<:Field}) = PartialCellBottom(reconstruct_global_field(g.bottom_height), g.minimum_fractional_cell_height) -reconstruct_global_boundary(g::GridFittedBoundary{<:Field}) = GridFittedBoundary(reconstruct_global_field(g.mask)) +reconstruct_global_immersed_boundary(g::GridFittedBottom{<:Field}) = GridFittedBottom(reconstruct_global_field(g.bottom_height), g.immersed_condition) +reconstruct_global_immersed_boundary(g::PartialCellBottom{<:Field}) = PartialCellBottom(reconstruct_global_field(g.bottom_height), g.minimum_fractional_cell_height) +reconstruct_global_immersed_boundary(g::GridFittedBoundary{<:Field}) = GridFittedBoundary(reconstruct_global_field(g.mask)) @inline getregion(mrg::ImmersedMultiRegionGrid{FT, TX, TY, TZ}, r) where {FT, TX, TY, TZ} = ImmersedBoundaryGrid{TX, TY, TZ}(_getregion(mrg.underlying_grid, r), _getregion(mrg.immersed_boundary, r)) @inline _getregion(mrg::ImmersedMultiRegionGrid{FT, TX, TY, TZ}, r) where {FT, TX, TY, TZ} = ImmersedBoundaryGrid{TX, TY, TZ}( getregion(mrg.underlying_grid, r), getregion(mrg.immersed_boundary, r)) diff --git a/validation/multi_region/multi_region_near_global_quarter_degree.jl b/validation/multi_region/multi_region_near_global_quarter_degree.jl index ce62c65f82..6bb671cb8c 100644 --- a/validation/multi_region/multi_region_near_global_quarter_degree.jl +++ b/validation/multi_region/multi_region_near_global_quarter_degree.jl @@ -147,7 +147,7 @@ biharmonic_viscosity = HorizontalScalarBiharmonicDiffusivity(ν=νhb, discrete @inline cyclic_interpolate(u₁::Number, u₂, time) = u₁ + mod(time / thirty_days, 1) * (u₂ - u₁) Δz_top = @allowscalar Δzᵃᵃᶜ(1, 1, grid.Nz, grid.underlying_grid) -Δbottom_height = @allowscalar Δzᵃᵃᶜ(1, 1, 1, grid.underlying_grid) +Δz_bottom = @allowscalar Δzᵃᵃᶜ(1, 1, 1, grid.underlying_grid) @inline function surface_wind_stress(i, j, grid, clock, fields, τ) time = clock.time From 5ee7623516c00d0132c1d87fcbd2e400deef7684 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Oct 2024 10:55:40 +0200 Subject: [PATCH 397/567] new comment --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index c21106c175..8ef8ab25df 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -128,14 +128,18 @@ end return z ≤ zb end -##### -##### Bottom height +##### +##### Utilities for `AbstractGridFittedBottom` ImmersedBoundaryGrids ##### const AGFBIB = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} @inline z_bottom(i, j, ibg::AGFBIBG) = @inbounds ibg.immersed_boundary.bottom_height[i, j, 1] +##### +##### Static column depth +##### + @inline static_column_depthᶜᶜᵃ(i, j, ibg::AGFBIB) = @inbounds znode(i, j, ibg.Nz+1, ibg, c, c, f) - ibg.immersed_boundary.bottom_height[i, j, 1] @inline static_column_depthᶜᶠᵃ(i, j, ibg::AGFBIB) = min(static_column_depthᶜᶜᵃ(i, j-1, ibg), static_column_depthᶜᶜᵃ(i, j, ibg)) @inline static_column_depthᶠᶜᵃ(i, j, ibg::AGFBIB) = min(static_column_depthᶜᶜᵃ(i-1, j, ibg), static_column_depthᶜᶜᵃ(i, j, ibg)) From 9777ee6460dc15f944e2cafd4dc8aba6db0f9cc1 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Oct 2024 10:58:20 +0200 Subject: [PATCH 398/567] another name change --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 8ef8ab25df..1644e91fab 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -99,17 +99,17 @@ of the last ``immersed`` cell in the column. function ImmersedBoundaryGrid(grid, ib::GridFittedBottom) bottom_field = Field{Center, Center, Nothing}(grid) set!(bottom_field, ib.bottom_height) - @apply_regionally correct_bottom_height!(bottom_field, grid, ib) + @apply_regionally compute_numerical_bottom_height!(bottom_field, grid, ib) fill_halo_regions!(bottom_field) new_ib = GridFittedBottom(bottom_field) TX, TY, TZ = topology(grid) return ImmersedBoundaryGrid{TX, TY, TZ}(grid, new_ib) end -correct_bottom_height!(bottom_field, grid, ib) = - launch!(architecture(grid), grid, :xy, _correct_bottom_height!, bottom_field, grid, ib) +compute_numerical_bottom_height!(bottom_field, grid, ib) = + launch!(architecture(grid), grid, :xy, _compute_numerical_bottom_height!, bottom_field, grid, ib) -@kernel function _correct_bottom_height!(bottom_field, grid, ib::GridFittedBottom) +@kernel function _compute_numerical_bottom_height!(bottom_field, grid, ib::GridFittedBottom) i, j = @index(Global, NTuple) zb = @inbounds bottom_field[i, j, 1] @inbounds bottom_field[i, j, 1] = znode(i, j, 1, grid, c, c, f) From 1d219af3159ebc31802bc102bce4d2f876b29da6 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Oct 2024 11:07:42 +0200 Subject: [PATCH 399/567] AGFBIBG istead of AGFBIB and z_bottom only in TurbulenceClosures --- src/ImmersedBoundaries/ImmersedBoundaries.jl | 4 ---- src/ImmersedBoundaries/grid_fitted_bottom.jl | 18 ++++++------------ src/TurbulenceClosures/TurbulenceClosures.jl | 5 +++-- 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/src/ImmersedBoundaries/ImmersedBoundaries.jl b/src/ImmersedBoundaries/ImmersedBoundaries.jl index 3976b79453..7047948e31 100644 --- a/src/ImmersedBoundaries/ImmersedBoundaries.jl +++ b/src/ImmersedBoundaries/ImmersedBoundaries.jl @@ -95,10 +95,6 @@ with_halo(halo, ibg::ImmersedBoundaryGrid) = inflate_halo_size_one_dimension(req_H, old_H, _, ::IBG) = max(req_H + 1, old_H) inflate_halo_size_one_dimension(req_H, old_H, ::Type{Flat}, ::IBG) = 0 -# Defining the bottom -@inline z_bottom(i, j, grid) = znode(i, j, 1, grid, c, c, f) -@inline z_bottom(i, j, ibg::IBG) = error("The function `bottom` has not been defined for $(summary(ibg))!") - function Base.summary(grid::ImmersedBoundaryGrid) FT = eltype(grid) TX, TY, TZ = topology(grid) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index 1644e91fab..af62726ab3 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -128,22 +128,16 @@ end return z ≤ zb end -##### -##### Utilities for `AbstractGridFittedBottom` ImmersedBoundaryGrids -##### - -const AGFBIB = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} - -@inline z_bottom(i, j, ibg::AGFBIBG) = @inbounds ibg.immersed_boundary.bottom_height[i, j, 1] - ##### ##### Static column depth ##### -@inline static_column_depthᶜᶜᵃ(i, j, ibg::AGFBIB) = @inbounds znode(i, j, ibg.Nz+1, ibg, c, c, f) - ibg.immersed_boundary.bottom_height[i, j, 1] -@inline static_column_depthᶜᶠᵃ(i, j, ibg::AGFBIB) = min(static_column_depthᶜᶜᵃ(i, j-1, ibg), static_column_depthᶜᶜᵃ(i, j, ibg)) -@inline static_column_depthᶠᶜᵃ(i, j, ibg::AGFBIB) = min(static_column_depthᶜᶜᵃ(i-1, j, ibg), static_column_depthᶜᶜᵃ(i, j, ibg)) -@inline static_column_depthᶠᶠᵃ(i, j, ibg::AGFBIB) = min(static_column_depthᶠᶜᵃ(i, j-1, ibg), static_column_depthᶠᶜᵃ(i, j, ibg)) +const AGFBIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} + +@inline static_column_depthᶜᶜᵃ(i, j, ibg::AGFBIBG) = @inbounds znode(i, j, ibg.Nz+1, ibg, c, c, f) - ibg.immersed_boundary.bottom_height[i, j, 1] +@inline static_column_depthᶜᶠᵃ(i, j, ibg::AGFBIBG) = min(static_column_depthᶜᶜᵃ(i, j-1, ibg), static_column_depthᶜᶜᵃ(i, j, ibg)) +@inline static_column_depthᶠᶜᵃ(i, j, ibg::AGFBIBG) = min(static_column_depthᶜᶜᵃ(i-1, j, ibg), static_column_depthᶜᶜᵃ(i, j, ibg)) +@inline static_column_depthᶠᶠᵃ(i, j, ibg::AGFBIBG) = min(static_column_depthᶠᶜᵃ(i, j-1, ibg), static_column_depthᶠᶜᵃ(i, j, ibg)) # Make sure column_height works for horizontally-Flat topologies. XFlatAGFIBG = ImmersedBoundaryGrid{<:Any, <:Flat, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} diff --git a/src/TurbulenceClosures/TurbulenceClosures.jl b/src/TurbulenceClosures/TurbulenceClosures.jl index eb683c11d7..e451b7d455 100644 --- a/src/TurbulenceClosures/TurbulenceClosures.jl +++ b/src/TurbulenceClosures/TurbulenceClosures.jl @@ -50,12 +50,10 @@ using Oceananigans.Utils using Oceananigans.Architectures: AbstractArchitecture, device using Oceananigans.Fields: FunctionField -using Oceananigans.ImmersedBoundaries: z_bottom import Oceananigans.Grids: required_halo_size_x, required_halo_size_y, required_halo_size_z import Oceananigans.Architectures: on_architecture -import Oceananigans.ImmersedBoundaries: z_bottom const VerticallyBoundedGrid{FT} = AbstractGrid{FT, <:Any, <:Any, <:Bounded} ##### @@ -124,8 +122,11 @@ end const c = Center() const f = Face() +const AGFBIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractGridFittedBottom} + @inline z_top(i, j, grid) = znode(i, j, grid.Nz+1, grid, c, c, f) @inline z_bottom(i, j, grid) = znode(i, j, 1, grid, c, c, f) +@inline z_bottom(i, j, ibg::ACFBIGB) = @inbounds ibg.immersed_boundary.bottom_height[i, j, 1] @inline depthᶜᶜᶠ(i, j, k, grid) = clip(z_top(i, j, grid) - znode(i, j, k, grid, c, c, f)) @inline depthᶜᶜᶜ(i, j, k, grid) = clip(z_top(i, j, grid) - znode(i, j, k, grid, c, c, c)) From 238fa741339582200c1a31613a2af452ff257399 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Oct 2024 11:13:19 +0200 Subject: [PATCH 400/567] some bugfixes --- src/TurbulenceClosures/TurbulenceClosures.jl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/TurbulenceClosures/TurbulenceClosures.jl b/src/TurbulenceClosures/TurbulenceClosures.jl index e451b7d455..6effcffc72 100644 --- a/src/TurbulenceClosures/TurbulenceClosures.jl +++ b/src/TurbulenceClosures/TurbulenceClosures.jl @@ -50,6 +50,8 @@ using Oceananigans.Utils using Oceananigans.Architectures: AbstractArchitecture, device using Oceananigans.Fields: FunctionField +using Oceananigans.ImmersedBoundaries +using Oceananigans.ImmersedBoundaries: AbstractGridFittedBottom import Oceananigans.Grids: required_halo_size_x, required_halo_size_y, required_halo_size_z import Oceananigans.Architectures: on_architecture @@ -119,6 +121,10 @@ end @inline clip(x) = max(zero(x), x) +##### +##### Height, Depth and Bottom interfaces +##### + const c = Center() const f = Face() @@ -126,7 +132,7 @@ const AGFBIBG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Abstra @inline z_top(i, j, grid) = znode(i, j, grid.Nz+1, grid, c, c, f) @inline z_bottom(i, j, grid) = znode(i, j, 1, grid, c, c, f) -@inline z_bottom(i, j, ibg::ACFBIGB) = @inbounds ibg.immersed_boundary.bottom_height[i, j, 1] +@inline z_bottom(i, j, ibg::AGFBIBG) = @inbounds ibg.immersed_boundary.bottom_height[i, j, 1] @inline depthᶜᶜᶠ(i, j, k, grid) = clip(z_top(i, j, grid) - znode(i, j, k, grid, c, c, f)) @inline depthᶜᶜᶜ(i, j, k, grid) = clip(z_top(i, j, grid) - znode(i, j, k, grid, c, c, c)) From d12fea0f6fac0e1575ac2d9deda9a5a7f40b1629 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Oct 2024 11:16:03 +0200 Subject: [PATCH 401/567] adapting to new interface --- .../split_explicit_free_surface_kernels.jl | 32 +++++++++---------- .../z_star_vertical_spacing.jl | 12 +++---- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index c26814f6de..8ea38b9ba5 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -179,12 +179,12 @@ end end # Linear free surface implementation -@inline dynamic_domain_depthᶠᶜᵃ(i, j, k, grid, η) = domain_depthᶠᶜᵃ(i, j, grid) -@inline dynamic_domain_depthᶜᶠᵃ(i, j, k, grid, η) = domain_depthᶜᶠᵃ(i, j, grid) +@inline dynamic_static_column_depthᶠᶜᵃ(i, j, k, grid, η) = static_column_depthᶠᶜᵃ(i, j, grid) +@inline dynamic_static_column_depthᶜᶠᵃ(i, j, k, grid, η) = static_column_depthᶜᶠᵃ(i, j, grid) # Non-linear free surface implementation -@inline dynamic_domain_depthᶠᶜᵃ(i, j, k, grid::ZStarSpacingGrid, η) = domain_depthᶠᶜᵃ(i, j, grid) + ℑxᶠᵃᵃ_η(i, j, k, grid, η) -@inline dynamic_domain_depthᶜᶠᵃ(i, j, k, grid::ZStarSpacingGrid, η) = domain_depthᶜᶠᵃ(i, j, grid) + ℑyᵃᶠᵃ_η(i, j, k, grid, η) +@inline dynamic_static_column_depthᶠᶜᵃ(i, j, k, grid::ZStarSpacingGrid, η) = static_column_depthᶠᶜᵃ(i, j, grid) + ℑxᶠᵃᵃ_η(i, j, k, grid, η) +@inline dynamic_static_column_depthᶜᶠᵃ(i, j, k, grid::ZStarSpacingGrid, η) = static_column_depthᶜᶠᵃ(i, j, grid) + ℑyᵃᶠᵃ_η(i, j, k, grid, η) @kernel function _split_explicit_barotropic_velocity!(averaging_weight, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², @@ -211,8 +211,8 @@ end advance_previous_velocity!(i, j, k_top-1, timestepper, U, Uᵐ⁻¹, Uᵐ⁻²) advance_previous_velocity!(i, j, k_top-1, timestepper, V, Vᵐ⁻¹, Vᵐ⁻²) - Hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) - Hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) + Hᶠᶜ = dynamic_column_depthᶠᶜᵃ(i, j, grid) + Hᶜᶠ = dynamic_column_depthᶜᶠᵃ(i, j, grid) # ∂τ(U) = - ∇η + G U[i, j, k_top-1] += Δτ * (- g * Hᶠᶜ * ∂xᶠᶜᶠ_η(i, j, k_top, grid, TX, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gᵁ[i, j, k_top-1]) @@ -234,11 +234,11 @@ end i, j = @index(Global, NTuple) k_top = grid.Nz+1 - hᶠᶜ = domain_depthᶠᶜᵃ(i, j, grid) - hᶜᶠ = domain_depthᶜᶠᵃ(i, j, grid) + hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) + hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) - sᶠᶜ = ifelse(hᶠᶜ == 0, one(grid), dynamic_domain_depthᶠᶜᵃ(i, j, k_top, grid, η) / hᶠᶜ) - sᶜᶠ = ifelse(hᶜᶠ == 0, one(grid), dynamic_domain_depthᶜᶠᵃ(i, j, k_top, grid, η) / hᶜᶠ) + sᶠᶜ = ifelse(hᶠᶜ == 0, one(grid), dynamic_static_column_depthᶠᶜᵃ(i, j, k_top, grid, η) / hᶠᶜ) + sᶜᶠ = ifelse(hᶜᶠ == 0, one(grid), dynamic_static_column_depthᶜᶠᵃ(i, j, k_top, grid, η) / hᶜᶠ) # hand unroll first loop @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δrᶠᶜᶜ(i, j, 1, grid) * sᶠᶜ @@ -257,11 +257,11 @@ end i, j = active_linear_index_to_tuple(idx, active_cells_map) k_top = grid.Nz+1 - hᶠᶜ = domain_depthᶠᶜᵃ(i, j, grid) - hᶜᶠ = domain_depthᶜᶠᵃ(i, j, grid) + hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) + hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) - sᶠᶜ = ifelse(hᶠᶜ == 0, one(grid), dynamic_domain_depthᶠᶜᵃ(i, j, k_top, grid, η) / hᶠᶜ) - sᶜᶠ = ifelse(hᶜᶠ == 0, one(grid), dynamic_domain_depthᶜᶠᵃ(i, j, k_top, grid, η) / hᶜᶠ) + sᶠᶜ = ifelse(hᶠᶜ == 0, one(grid), dynamic_static_column_depthᶠᶜᵃ(i, j, k_top, grid, η) / hᶠᶜ) + sᶜᶠ = ifelse(hᶜᶠ == 0, one(grid), dynamic_static_column_depthᶜᶠᵃ(i, j, k_top, grid, η) / hᶜᶠ) # hand unroll first loop @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δrᶠᶜᶜ(i, j, 1, grid) * sᶠᶜ @@ -285,8 +285,8 @@ end i, j, k = @index(Global, NTuple) k_top = grid.Nz + 1 - Hᶠᶜ = dynamic_domain_depthᶠᶜᵃ(i, j, k_top, grid, η) - Hᶜᶠ = dynamic_domain_depthᶜᶠᵃ(i, j, k_top, grid, η) + Hᶠᶜ = dynamic_static_column_depthᶠᶜᵃ(i, j, k_top, grid, η) + Hᶜᶠ = dynamic_static_column_depthᶜᶠᵃ(i, j, k_top, grid, η) @inbounds begin u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / Hᶠᶜ diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 87373c3914..6e43918816 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -55,7 +55,7 @@ end δy_V = δyᶜᶜᶠ(i, j, k_top-1, grid, Δx_qᶜᶠᶠ, V̅) δh_U = (δx_U + δy_V) / Azᶜᶜᶠ(i, j, k_top-1, grid) - H = domain_depthᶜᶜᵃ(i, j, grid) + H = static_column_depthᶜᶜᵃ(i, j, grid) @inbounds ∂t_s[i, j] = ifelse(H == 0, zero(grid), - δh_U / H) end @@ -64,10 +64,10 @@ end i, j = @index(Global, NTuple) k_top = grid.Nz+1 - hᶜᶜ = domain_depthᶜᶜᵃ(i, j, grid) - hᶠᶜ = domain_depthᶠᶜᵃ(i, j, grid) - hᶜᶠ = domain_depthᶜᶠᵃ(i, j, grid) - hᶠᶠ = domain_depthᶠᶠᵃ(i, j, grid) + hᶜᶜ = static_column_depthᶜᶜᵃ(i, j, grid) + hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) + hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) + hᶠᶠ = static_column_depthᶠᶠᵃ(i, j, grid) @inbounds begin sᶜᶜ = ifelse(hᶜᶜ == 0, one(grid), (hᶜᶜ + η[i, j, k_top]) / hᶜᶜ) @@ -95,7 +95,7 @@ end ##### ZStar-specific implementation of the additional terms to be included in the momentum equations ##### -@inline z_minus_rᶜᶜᶜ(i, j, k, grid, η) = @inbounds η[i, j, grid.Nz+1] * (1 + rnode(i, j, k, grid, Center(), Center(), Center()) / domain_depthᶜᶜᵃ(i, j, grid)) +@inline z_minus_rᶜᶜᶜ(i, j, k, grid, η) = @inbounds η[i, j, grid.Nz+1] * (1 + rnode(i, j, k, grid, Center(), Center(), Center()) / static_column_depthᶜᶜᵃ(i, j, grid)) @inline ∂x_z(i, j, k, grid, free_surface) = @inbounds ∂xᶠᶜᶜ(i, j, k, grid, z_minus_rᶜᶜᶜ, free_surface.η) @inline ∂y_z(i, j, k, grid, free_surface) = @inbounds ∂yᶜᶠᶜ(i, j, k, grid, z_minus_rᶜᶜᶜ, free_surface.η) From 52864406a3c4c0cebae3b40a68f48de55241d2d9 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Oct 2024 11:43:40 +0200 Subject: [PATCH 402/567] should change these operators --- src/ImmersedBoundaries/partial_cell_bottom.jl | 1 - .../split_explicit_free_surface_kernels.jl | 33 ++++++++++++------- validation/z_star_coordinate/lock_release.jl | 2 +- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index 27d97acb79..64caf522d5 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -75,7 +75,6 @@ end @kernel function _compute_numerical_bottom_height!(bottom_field, grid, ib::PartialCellBottom) i, j = @index(Global, NTuple) zb = @inbounds bottom_field[i, j, 1] - @inbounds bottom_field[i, j, 1] = znode(i, j, 1, grid, c, c, f) ϵ = ib.minimum_fractional_cell_height @inbounds bottom_field[i, j, 1] = rnode(i, j, 1, grid, c, c, f) for k in 1:grid.Nz diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 8ea38b9ba5..7c2f26ae18 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -123,6 +123,15 @@ const RGY = AbstractGrid{<:Any, <:RightConnected} @inline ℑxᶠᵃᵃ_η(i, j, k, grid::RGX, η) = ifelse(i == 1, η[1, j, k], ℑxᶠᵃᵃ(i, j, k, grid, η)) @inline ℑyᵃᶠᵃ_η(i, j, k, grid::RGY, η) = ifelse(j == 1, η[i, 1, k], ℑyᵃᶠᵃ(i, j, k, grid, η)) +@inline ℑxᶠᵃᵃ_η(i, j, k, grid, η::Function, args...) = ℑxᶠᵃᵃ(i, j, k, grid, η, args...) +@inline ℑyᵃᶠᵃ_η(i, j, k, grid, η::Function, args...) = ℑyᵃᶠᵃ(i, j, k, grid, η, args...) +@inline ℑxᶠᵃᵃ_η(i, j, k, grid::PGX, η::Function, args...) = ifelse(i == 1, (η(1, j, k, grid, args...) + η(grid.Nx, j, k, grid, args...)) / 2, ℑxᶠᵃᵃ(i, j, k, grid, η, args...)) +@inline ℑyᵃᶠᵃ_η(i, j, k, grid::PGY, η::Function, args...) = ifelse(j == 1, (η(i, 1, k, grid, args...) + η(i, grid.Ny, k, grid, args...)) / 2, ℑyᵃᶠᵃ(i, j, k, grid, η, args...)) +@inline ℑxᶠᵃᵃ_η(i, j, k, grid::BGX, η::Function, args...) = ifelse(i == 1, η(1, j, k, grid, args...), ℑxᶠᵃᵃ(i, j, k, grid, η, args...)) +@inline ℑyᵃᶠᵃ_η(i, j, k, grid::BGY, η::Function, args...) = ifelse(j == 1, η(i, 1, k, grid, args...), ℑyᵃᶠᵃ(i, j, k, grid, η, args...)) +@inline ℑxᶠᵃᵃ_η(i, j, k, grid::RGX, η::Function, args...) = ifelse(i == 1, η(1, j, k, grid, args...), ℑxᶠᵃᵃ(i, j, k, grid, η, args...)) +@inline ℑyᵃᶠᵃ_η(i, j, k, grid::RGY, η::Function, args...) = ifelse(j == 1, η(i, 1, k, grid, args...), ℑyᵃᶠᵃ(i, j, k, grid, η, args...)) + # Time stepping extrapolation U★, and η★ # AB3 step @@ -179,12 +188,12 @@ end end # Linear free surface implementation -@inline dynamic_static_column_depthᶠᶜᵃ(i, j, k, grid, η) = static_column_depthᶠᶜᵃ(i, j, grid) -@inline dynamic_static_column_depthᶜᶠᵃ(i, j, k, grid, η) = static_column_depthᶜᶠᵃ(i, j, grid) +@inline dynamic_column_depthᶠᶜᵃ(i, j, k, grid, η, args...) = static_column_depthᶠᶜᵃ(i, j, grid) +@inline dynamic_column_depthᶜᶠᵃ(i, j, k, grid, η, args...) = static_column_depthᶜᶠᵃ(i, j, grid) # Non-linear free surface implementation -@inline dynamic_static_column_depthᶠᶜᵃ(i, j, k, grid::ZStarSpacingGrid, η) = static_column_depthᶠᶜᵃ(i, j, grid) + ℑxᶠᵃᵃ_η(i, j, k, grid, η) -@inline dynamic_static_column_depthᶜᶠᵃ(i, j, k, grid::ZStarSpacingGrid, η) = static_column_depthᶜᶠᵃ(i, j, grid) + ℑyᵃᶠᵃ_η(i, j, k, grid, η) +@inline dynamic_column_depthᶠᶜᵃ(i, j, k, grid::ZStarSpacingGrid, η, args...) = static_column_depthᶠᶜᵃ(i, j, grid) + ℑxᶠᵃᵃ_η(i, j, k, grid, η, args...) +@inline dynamic_column_depthᶜᶠᵃ(i, j, k, grid::ZStarSpacingGrid, η, args...) = static_column_depthᶜᶠᵃ(i, j, grid) + ℑyᵃᶠᵃ_η(i, j, k, grid, η, args...) @kernel function _split_explicit_barotropic_velocity!(averaging_weight, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², @@ -211,8 +220,8 @@ end advance_previous_velocity!(i, j, k_top-1, timestepper, U, Uᵐ⁻¹, Uᵐ⁻²) advance_previous_velocity!(i, j, k_top-1, timestepper, V, Vᵐ⁻¹, Vᵐ⁻²) - Hᶠᶜ = dynamic_column_depthᶠᶜᵃ(i, j, grid) - Hᶜᶠ = dynamic_column_depthᶜᶠᵃ(i, j, grid) + Hᶠᶜ = dynamic_column_depthᶠᶜᵃ(i, j, k_top, grid, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Hᶜᶠ = dynamic_column_depthᶜᶠᵃ(i, j, k_top, grid, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) # ∂τ(U) = - ∇η + G U[i, j, k_top-1] += Δτ * (- g * Hᶠᶜ * ∂xᶠᶜᶠ_η(i, j, k_top, grid, TX, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gᵁ[i, j, k_top-1]) @@ -237,8 +246,8 @@ end hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) - sᶠᶜ = ifelse(hᶠᶜ == 0, one(grid), dynamic_static_column_depthᶠᶜᵃ(i, j, k_top, grid, η) / hᶠᶜ) - sᶜᶠ = ifelse(hᶜᶠ == 0, one(grid), dynamic_static_column_depthᶜᶠᵃ(i, j, k_top, grid, η) / hᶜᶠ) + sᶠᶜ = ifelse(hᶠᶜ == 0, one(grid), dynamic_column_depthᶠᶜᵃ(i, j, k_top, grid, η) / hᶠᶜ) + sᶜᶠ = ifelse(hᶜᶠ == 0, one(grid), dynamic_column_depthᶜᶠᵃ(i, j, k_top, grid, η) / hᶜᶠ) # hand unroll first loop @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δrᶠᶜᶜ(i, j, 1, grid) * sᶠᶜ @@ -260,8 +269,8 @@ end hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) - sᶠᶜ = ifelse(hᶠᶜ == 0, one(grid), dynamic_static_column_depthᶠᶜᵃ(i, j, k_top, grid, η) / hᶠᶜ) - sᶜᶠ = ifelse(hᶜᶠ == 0, one(grid), dynamic_static_column_depthᶜᶠᵃ(i, j, k_top, grid, η) / hᶜᶠ) + sᶠᶜ = ifelse(hᶠᶜ == 0, one(grid), dynamic_column_depthᶠᶜᵃ(i, j, k_top, grid, η) / hᶠᶜ) + sᶜᶠ = ifelse(hᶜᶠ == 0, one(grid), dynamic_column_depthᶜᶠᵃ(i, j, k_top, grid, η) / hᶜᶠ) # hand unroll first loop @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δrᶠᶜᶜ(i, j, 1, grid) * sᶠᶜ @@ -285,8 +294,8 @@ end i, j, k = @index(Global, NTuple) k_top = grid.Nz + 1 - Hᶠᶜ = dynamic_static_column_depthᶠᶜᵃ(i, j, k_top, grid, η) - Hᶜᶠ = dynamic_static_column_depthᶜᶠᵃ(i, j, k_top, grid, η) + Hᶠᶜ = dynamic_column_depthᶠᶜᵃ(i, j, k_top, grid, η) + Hᶜᶠ = dynamic_column_depthᶜᶠᵃ(i, j, k_top, grid, η) @inbounds begin u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / Hᶠᶜ diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index 71eb49ea67..a3d14696cb 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -12,7 +12,7 @@ grid = RectilinearGrid(size = (128, 20), x = (0, 64kilometers), z = z_faces, halo = (6, 6), - topology = (Periodic, Flat, Bounded)) + topology = (Bounded, Flat, Bounded)) grid = ImmersedBoundaryGrid(grid, GridFittedBottom(x -> - (64kilometers - x) / 64kilometers * 20)) From f1aab48f8080ac3d819f61882734d0166f73f5d7 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Oct 2024 11:48:23 +0200 Subject: [PATCH 403/567] remove orthogonalsphericalshellgrids for the moment --- Project.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index 8692d7f1f9..91f916318e 100644 --- a/Project.toml +++ b/Project.toml @@ -80,10 +80,9 @@ julia = "1.9" [extras] DataDeps = "124859b0-ceae-595e-8997-d05f6a7a8dfe" Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" -OrthogonalSphericalShellGrids = "c2be9673-fb75-4747-82dc-aa2bb9f4aed0" SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" TimesDates = "bdfc003b-8df8-5c39-adcd-3a9087f5df4a" [targets] -test = ["DataDeps", "Enzyme", "OrthogonalSphericalShellGrids", "SafeTestsets", "Test", "TimesDates"] +test = ["DataDeps", "Enzyme", "SafeTestsets", "Test", "TimesDates"] From ace678b1492bbfac2667ac316fd30be6f2de707e Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Oct 2024 13:48:20 +0200 Subject: [PATCH 404/567] better bottom height --- src/ImmersedBoundaries/partial_cell_bottom.jl | 21 ++- validation/z_star_coordinate/internal_tide.jl | 120 +++++++++--------- 2 files changed, 75 insertions(+), 66 deletions(-) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index 64caf522d5..66e89293a7 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -74,17 +74,26 @@ end @kernel function _compute_numerical_bottom_height!(bottom_field, grid, ib::PartialCellBottom) i, j = @index(Global, NTuple) + + # Save analytical bottom height zb = @inbounds bottom_field[i, j, 1] + + # Cap bottom height at Lz and at rnode(i, j, grid.Nz+1, grid, c, c, f) + domain_bottom = rnode(i, j, 1, grid, c, c, f) + domain_top = rnode(i, j, grid.Nz+1, grid, c, c, f) + @inbounds bottom_field[i, j, 1] = clamp(zb, domain_bottom, domain_top) + ϵ = ib.minimum_fractional_cell_height - @inbounds bottom_field[i, j, 1] = rnode(i, j, 1, grid, c, c, f) for k in 1:grid.Nz - # We use `rnode` for the `immersed_cell` because we do not want to have - # wetting or drying that could happen for a moving grid if we use znode - z⁻ = rnode(i, j, k, grid, c, c, c) + # We use `rnode` (reference vertical coordinate) instead of `znode` + # because we do not want to possibly using a moving grid to compute the bottom height + z⁻ = rnode(i, j, k, grid, c, c, f) + z⁺ = rnode(i, j, k+1, grid, c, c, f) # For the same reason, here we use `Δrᶜᶜᶜ` instead of `Δzᶜᶜᶜ` Δz = Δrᶜᶜᶜ(i, j, k, grid) - bottom_cell = z⁻ + Δz * (1 - ϵ) ≤ zb - @inbounds bottom_field[i, j, 1] = ifelse(bottom_cell, z⁻ + Δz * (1 - ϵ), bottom_field[i, j, 1]) + bottom_cell = (z⁻ ≤ zb) & (z⁺ ≥ zb) + capped_zb = ifelse(z⁻ + Δz * ϵ > zb, z⁻, zb) + @inbounds bottom_field[i, j, 1] = ifelse(bottom_cell, capped_zb, bottom_field[i, j, 1]) end end diff --git a/validation/z_star_coordinate/internal_tide.jl b/validation/z_star_coordinate/internal_tide.jl index d244ae0296..7f4ec1e734 100644 --- a/validation/z_star_coordinate/internal_tide.jl +++ b/validation/z_star_coordinate/internal_tide.jl @@ -20,7 +20,7 @@ width = 20kilometers hill(x) = h₀ * exp(-x^2 / 2width^2) bottom(x) = - H + hill(x) -grid = ImmersedBoundaryGrid(underlying_grid, GridFittedBottom(bottom)) +grid = ImmersedBoundaryGrid(underlying_grid, PartialCellBottom(bottom)) coriolis = FPlane(latitude = -45) @@ -66,7 +66,7 @@ set!(model, u=uᵢ, b=bᵢ) Δt = 5minutes stop_time = 4days -simulation = Simulation(model; Δt, stop_time) +simulation = Simulation(model; Δt, stop_time, stop_iteration = 2) # We add a callback to print a message about how the simulation is going, @@ -118,86 +118,86 @@ run!(simulation) # # ## Load output -# # First, we load the saved velocities and stratification output as `FieldTimeSeries`es. +# # # First, we load the saved velocities and stratification output as `FieldTimeSeries`es. -saved_output_filename = filename * ".jld2" +# saved_output_filename = filename * ".jld2" -u′_t = FieldTimeSeries(saved_output_filename, "u′") - w_t = FieldTimeSeries(saved_output_filename, "w") -N²_t = FieldTimeSeries(saved_output_filename, "N²") +# u′_t = FieldTimeSeries(saved_output_filename, "u′") +# w_t = FieldTimeSeries(saved_output_filename, "w") +# N²_t = FieldTimeSeries(saved_output_filename, "N²") -umax = maximum(abs, u′_t[end]) -wmax = maximum(abs, w_t[end]) +# umax = maximum(abs, u′_t[end]) +# wmax = maximum(abs, w_t[end]) -times = u′_t.times -nothing #hide +# times = u′_t.times +# nothing #hide -# We retrieve each field's coordinates and convert from meters to kilometers. +# # We retrieve each field's coordinates and convert from meters to kilometers. -xu, _, zu = nodes(u′_t[1]) -xw, _, zw = nodes(w_t[1]) -xN², _, zN² = nodes(N²_t[1]) +# xu, _, zu = nodes(u′_t[1]) +# xw, _, zw = nodes(w_t[1]) +# xN², _, zN² = nodes(N²_t[1]) -xu = xu ./ 1e3 -xw = xw ./ 1e3 -xN² = xN² ./ 1e3 -zu = zu ./ 1e3 -zw = zw ./ 1e3 -zN² = zN² ./ 1e3 -nothing #hide +# xu = xu ./ 1e3 +# xw = xw ./ 1e3 +# xN² = xN² ./ 1e3 +# zu = zu ./ 1e3 +# zw = zw ./ 1e3 +# zN² = zN² ./ 1e3 +# nothing #hide -# ## Visualize +# # ## Visualize -# Now we can visualize our resutls! We use `CairoMakie` here. On a system with OpenGL -# `using GLMakie` is more convenient as figures will be displayed on the screen. -# -# We use Makie's `Observable` to animate the data. To dive into how `Observable`s work we -# refer to [Makie.jl's Documentation](https://makie.juliaplots.org/stable/documentation/nodes/index.html). +# # Now we can visualize our resutls! We use `CairoMakie` here. On a system with OpenGL +# # `using GLMakie` is more convenient as figures will be displayed on the screen. +# # +# # We use Makie's `Observable` to animate the data. To dive into how `Observable`s work we +# # refer to [Makie.jl's Documentation](https://makie.juliaplots.org/stable/documentation/nodes/index.html). -using CairoMakie +# using CairoMakie -n = Observable(1) +# n = Observable(1) -title = @lift @sprintf("t = %1.2f days = %1.2f T₂", - round(times[$n] / day, digits=2) , round(times[$n] / T₂, digits=2)) +# title = @lift @sprintf("t = %1.2f days = %1.2f T₂", +# round(times[$n] / day, digits=2) , round(times[$n] / T₂, digits=2)) -u′n = @lift u′_t[$n] - wn = @lift w_t[$n] -N²n = @lift N²_t[$n] +# u′n = @lift u′_t[$n] +# wn = @lift w_t[$n] +# N²n = @lift N²_t[$n] -axis_kwargs = (xlabel = "x [km]", - ylabel = "z [km]", - limits = ((-grid.Lx/2e3, grid.Lx/2e3), (-grid.Lz/1e3, 0)), # note conversion to kilometers - titlesize = 20) +# axis_kwargs = (xlabel = "x [km]", +# ylabel = "z [km]", +# limits = ((-grid.Lx/2e3, grid.Lx/2e3), (-grid.Lz/1e3, 0)), # note conversion to kilometers +# titlesize = 20) -fig = Figure(size = (700, 900)) +# fig = Figure(size = (700, 900)) -fig[1, :] = Label(fig, title, fontsize=24, tellwidth=false) +# fig[1, :] = Label(fig, title, fontsize=24, tellwidth=false) -ax_u = Axis(fig[2, 1]; title = "u'-velocity", axis_kwargs...) -hm_u = heatmap!(ax_u, xu, zu, u′n; nan_color=:gray, colorrange=(-umax, umax), colormap=:balance) -Colorbar(fig[2, 2], hm_u, label = "m s⁻¹") +# ax_u = Axis(fig[2, 1]; title = "u'-velocity", axis_kwargs...) +# hm_u = heatmap!(ax_u, xu, zu, u′n; nan_color=:gray, colorrange=(-umax, umax), colormap=:balance) +# Colorbar(fig[2, 2], hm_u, label = "m s⁻¹") -ax_w = Axis(fig[3, 1]; title = "w-velocity", axis_kwargs...) -hm_w = heatmap!(ax_w, xw, zw, wn; nan_color=:gray, colorrange=(-wmax, wmax), colormap=:balance) -Colorbar(fig[3, 2], hm_w, label = "m s⁻¹") +# ax_w = Axis(fig[3, 1]; title = "w-velocity", axis_kwargs...) +# hm_w = heatmap!(ax_w, xw, zw, wn; nan_color=:gray, colorrange=(-wmax, wmax), colormap=:balance) +# Colorbar(fig[3, 2], hm_w, label = "m s⁻¹") -ax_N² = Axis(fig[4, 1]; title = "stratification N²", axis_kwargs...) -hm_N² = heatmap!(ax_N², xN², zN², N²n; nan_color=:gray, colorrange=(0.9Nᵢ², 1.1Nᵢ²), colormap=:magma) -Colorbar(fig[4, 2], hm_N², label = "s⁻²") +# ax_N² = Axis(fig[4, 1]; title = "stratification N²", axis_kwargs...) +# hm_N² = heatmap!(ax_N², xN², zN², N²n; nan_color=:gray, colorrange=(0.9Nᵢ², 1.1Nᵢ²), colormap=:magma) +# Colorbar(fig[4, 2], hm_N², label = "s⁻²") -fig +# fig -# Finally, we can record a movie. +# # Finally, we can record a movie. -@info "Making an animation from saved data..." +# @info "Making an animation from saved data..." -frames = 1:length(times) +# frames = 1:length(times) -record(fig, filename * ".mp4", frames, framerate=16) do i - @info string("Plotting frame ", i, " of ", frames[end]) - n[] = i -end -nothing #hide +# record(fig, filename * ".mp4", frames, framerate=16) do i +# @info string("Plotting frame ", i, " of ", frames[end]) +# n[] = i +# end +# nothing #hide -# ![](internal_tide.mp4) +# # ![](internal_tide.mp4) From 210ead14a202005482bb68c9a3d9bfd610a304ae Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Oct 2024 13:56:58 +0200 Subject: [PATCH 405/567] bugfix in bottom height --- src/ImmersedBoundaries/partial_cell_bottom.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index 66e89293a7..0423ef4606 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -92,7 +92,8 @@ end # For the same reason, here we use `Δrᶜᶜᶜ` instead of `Δzᶜᶜᶜ` Δz = Δrᶜᶜᶜ(i, j, k, grid) bottom_cell = (z⁻ ≤ zb) & (z⁺ ≥ zb) - capped_zb = ifelse(z⁻ + Δz * ϵ > zb, z⁻, zb) + capped_zb = ifelse(zb < z⁻ + Δz * (1 - ϵ), zb, z⁺) + @inbounds bottom_field[i, j, 1] = ifelse(bottom_cell, capped_zb, bottom_field[i, j, 1]) end end From 90dfa7af1d1cf0c4f2de33e2db969ab913905ad5 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Oct 2024 13:58:26 +0200 Subject: [PATCH 406/567] better definition of bottom height --- src/ImmersedBoundaries/partial_cell_bottom.jl | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index cd8a261ec3..92fd2ac305 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -72,14 +72,28 @@ end @kernel function _compute_numerical_bottom_height!(bottom_field, grid, ib::PartialCellBottom) i, j = @index(Global, NTuple) + + # Save analytical bottom height zb = @inbounds bottom_field[i, j, 1] - @inbounds bottom_field[i, j, 1] = znode(i, j, 1, grid, c, c, f) + + # Cap bottom height at Lz and at rnode(i, j, grid.Nz+1, grid, c, c, f) + domain_bottom = znode(i, j, 1, grid, c, c, f) + domain_top = znode(i, j, grid.Nz+1, grid, c, c, f) + @inbounds bottom_field[i, j, 1] = clamp(zb, domain_bottom, domain_top) + ϵ = ib.minimum_fractional_cell_height for k in 1:grid.Nz - z⁻ = znode(i, j, k, grid, c, c, f) + z⁻ = znode(i, j, k, grid, c, c, f) + z⁺ = znode(i, j, k+1, grid, c, c, f) Δz = Δzᶜᶜᶜ(i, j, k, grid) - bottom_cell = z⁻ + Δz * (1 - ϵ) ≤ zb - @inbounds bottom_field[i, j, 1] = ifelse(bottom_cell, z⁻ + Δz * (1 - ϵ), zb) + bottom_cell = (z⁻ ≤ zb) & (z⁺ ≥ zb) + + # If the size of the bottom cell is less than ϵ Δz, use + # a simple `GridFittedBottom` approach where the bottom + # height is the top interface of cell k. + capped_zb = ifelse(zb < z⁻ + Δz * (1 - ϵ), zb, z⁺) + + @inbounds bottom_field[i, j, 1] = ifelse(bottom_cell, capped_zb, bottom_field[i, j, 1]) end end From 99a152afb004237bef0f6ffe70f2a8e6b62f70f4 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Oct 2024 14:49:01 +0200 Subject: [PATCH 407/567] fixed partial cells? --- src/Fields/abstract_field.jl | 4 +++- src/ImmersedBoundaries/partial_cell_bottom.jl | 9 ++++++--- validation/z_star_coordinate/internal_tide.jl | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/Fields/abstract_field.jl b/src/Fields/abstract_field.jl index c8c9b7b5f1..4ecf6a3d58 100644 --- a/src/Fields/abstract_field.jl +++ b/src/Fields/abstract_field.jl @@ -12,7 +12,7 @@ import Base: minimum, maximum, extrema import Oceananigans: location, instantiated_location import Oceananigans.Architectures: architecture, child_architecture import Oceananigans.Grids: interior_x_indices, interior_y_indices, interior_z_indices -import Oceananigans.Grids: total_size, topology, nodes, xnodes, ynodes, znodes, node, xnode, ynode, znode +import Oceananigans.Grids: total_size, topology, nodes, xnodes, ynodes, znodes, rnodes, node, xnode, ynode, znode, rnode import Oceananigans.Utils: datatuple const ArchOrNothing = Union{AbstractArchitecture, Nothing} @@ -110,10 +110,12 @@ interior(f::AbstractField) = f @propagate_inbounds xnode(i, j, k, ψ::AbstractField) = xnode(i, j, k, ψ.grid, instantiated_location(ψ)...) @propagate_inbounds ynode(i, j, k, ψ::AbstractField) = ynode(i, j, k, ψ.grid, instantiated_location(ψ)...) @propagate_inbounds znode(i, j, k, ψ::AbstractField) = znode(i, j, k, ψ.grid, instantiated_location(ψ)...) +@propagate_inbounds rnode(i, j, k, ψ::AbstractField) = rnode(i, j, k, ψ.grid, instantiated_location(ψ)...) xnodes(ψ::AbstractField; kwargs...) = xnodes(ψ.grid, instantiated_location(ψ)...; kwargs...) ynodes(ψ::AbstractField; kwargs...) = ynodes(ψ.grid, instantiated_location(ψ)...; kwargs...) znodes(ψ::AbstractField; kwargs...) = znodes(ψ.grid, instantiated_location(ψ)...; kwargs...) +rnodes(ψ::AbstractField; kwargs...) = rnodes(ψ.grid, instantiated_location(ψ)...; kwargs...) nodes(ψ::AbstractField; kwargs...) = nodes(ψ.grid, instantiated_location(ψ); kwargs...) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index 0423ef4606..461bfb72ad 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -91,7 +91,7 @@ end z⁺ = rnode(i, j, k+1, grid, c, c, f) # For the same reason, here we use `Δrᶜᶜᶜ` instead of `Δzᶜᶜᶜ` Δz = Δrᶜᶜᶜ(i, j, k, grid) - bottom_cell = (z⁻ ≤ zb) & (z⁺ ≥ zb) + bottom_cell = (z⁻ < zb) & (z⁺ ≥ zb) capped_zb = ifelse(zb < z⁻ + Δz * (1 - ϵ), zb, z⁺) @inbounds bottom_field[i, j, 1] = ifelse(bottom_cell, capped_zb, bottom_field[i, j, 1]) @@ -134,9 +134,12 @@ Criterion is zb ≥ z - ϵ Δz """ @inline function _immersed_cell(i, j, k, underlying_grid, ib::PartialCellBottom) - z = rnode(i, j, k+1, underlying_grid, c, c, f) + z⁻ = rnode(i, j, k, underlying_grid, c, c, f) + ϵ = ib.minimum_fractional_cell_height + Δr = Δrᶜᶜᶜ(i, j, k, underlying_grid) + z★ = z⁻ + Δr * (1 - ϵ) zb = @inbounds ib.bottom_height[i, j, 1] - return z ≤ zb + return z★ < zb end @inline function bottom_cell(i, j, k, ibg::PCBIBG) diff --git a/validation/z_star_coordinate/internal_tide.jl b/validation/z_star_coordinate/internal_tide.jl index 7f4ec1e734..dac756bdae 100644 --- a/validation/z_star_coordinate/internal_tide.jl +++ b/validation/z_star_coordinate/internal_tide.jl @@ -66,7 +66,7 @@ set!(model, u=uᵢ, b=bᵢ) Δt = 5minutes stop_time = 4days -simulation = Simulation(model; Δt, stop_time, stop_iteration = 2) +simulation = Simulation(model; Δt, stop_time, stop_iteration = 10) # We add a callback to print a message about how the simulation is going, From f2402f0b4505e097101c1396e90ae91fbfd3896f Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Oct 2024 14:49:41 +0200 Subject: [PATCH 408/567] fixed partial cell --- src/ImmersedBoundaries/partial_cell_bottom.jl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index 92fd2ac305..40c6bed202 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -133,9 +133,12 @@ Criterion is zb ≥ z - ϵ Δz """ @inline function _immersed_cell(i, j, k, underlying_grid, ib::PartialCellBottom) - z = znode(i, j, k+1, underlying_grid, c, c, f) + z⁻ = znode(i, j, k, underlying_grid, c, c, f) + ϵ = ib.minimum_fractional_cell_height + Δrz= Δzᶜᶜᶜ(i, j, k, underlying_grid) + z★ = z⁻ + Δz * (1 - ϵ) zb = @inbounds ib.bottom_height[i, j, 1] - return z ≤ zb + return z★ < zb end @inline function bottom_cell(i, j, k, ibg::PCBIBG) From 55d7432ecb42bd4db63350845a01aaf12b5d596f Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Oct 2024 14:50:13 +0200 Subject: [PATCH 409/567] fixed partial cells --- src/ImmersedBoundaries/partial_cell_bottom.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ImmersedBoundaries/partial_cell_bottom.jl b/src/ImmersedBoundaries/partial_cell_bottom.jl index 40c6bed202..decaa17763 100644 --- a/src/ImmersedBoundaries/partial_cell_bottom.jl +++ b/src/ImmersedBoundaries/partial_cell_bottom.jl @@ -135,7 +135,7 @@ Criterion is zb ≥ z - ϵ Δz @inline function _immersed_cell(i, j, k, underlying_grid, ib::PartialCellBottom) z⁻ = znode(i, j, k, underlying_grid, c, c, f) ϵ = ib.minimum_fractional_cell_height - Δrz= Δzᶜᶜᶜ(i, j, k, underlying_grid) + Δz = Δzᶜᶜᶜ(i, j, k, underlying_grid) z★ = z⁻ + Δz * (1 - ϵ) zb = @inbounds ib.bottom_height[i, j, 1] return z★ < zb From 525f4be6b3e4532b44447d25f78fd6c64d5ebc0c Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Oct 2024 15:04:21 +0200 Subject: [PATCH 410/567] fix split explicit tests --- .../test_split_explicit_vertical_integrals.jl | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/test/test_split_explicit_vertical_integrals.jl b/test/test_split_explicit_vertical_integrals.jl index 40f8920a4b..76f573255e 100644 --- a/test/test_split_explicit_vertical_integrals.jl +++ b/test/test_split_explicit_vertical_integrals.jl @@ -7,7 +7,8 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: SplitExplicitState, SplitExplicitSettings using Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_barotropic_mode!, barotropic_split_explicit_corrector!, - initialize_free_surface_state! + initialize_free_surface_state!, + materialize_free_surface @testset "Barotropic Kernels" begin @@ -36,7 +37,9 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_barotropic_mode! @testset "Average to zero" begin # set equal to something else - η̅ .= U̅ .= V̅ .= 1.0 + η̅ .= 1 + U̅ .= 1 + V̅ .= 1 # now set equal to zero initialize_free_surface_state!(sefs.state, sefs.η, sefs.settings.timestepper) @@ -48,13 +51,12 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_barotropic_mode! # check @test all(Array(η̅.data.parent) .== 0.0) - @test all(Array(U̅.data.parent .== 0.0)) - @test all(Array(V̅.data.parent .== 0.0)) + @test all(Array(U̅.data.parent) .== 0.0) + @test all(Array(V̅.data.parent) .== 0.0) end @testset "Inexact integration" begin # Test 2: Check that vertical integrals work on the CPU(). The following should be "inexact" - set_u_check(x, y, z) = cos((π / 2) * z / Lz) set_U_check(x, y, z) = (sin(0) - (-2 * Lz / (π))) set!(u, set_u_check) @@ -75,13 +77,13 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: compute_barotropic_mode! @testset "Vertical Integral " begin - u .= 0.0 - U .= 1.0 + set!(u, 0.0) + set!(U, 1.0) compute_barotropic_mode!(U, V, grid, u, v, η̅) - @test all(Array(U.data.parent) .== 0.0) + @test all(Array(interior(U)) .== 0.0) - u .= 1.0 - U .= 1.0 + set!(u, 1.0) + set!(U, 1.0) compute_barotropic_mode!(U, V, grid, u, v, η̅) @test all(Array(interior(U)) .≈ Lz) From c25d67d9a6e6c26e4d58f435cd4fbb1ee2450c67 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Oct 2024 15:06:58 +0200 Subject: [PATCH 411/567] check if these pass on a GPU for the moment --- test/test_zstar_coordinate.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index 7a1b140a91..885e267ae4 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -74,7 +74,9 @@ end for grid in grids @info info_message(grid) - free_surface = SplitExplicitFreeSurface(grid; cfl = 0.75) + # TODO: minimum_xspacing(grid) on a Immersed GPU grid with ZStarVerticalCoordinate + # fails because it uses too much parameter space. Figure out a way to reduce it + free_surface = SplitExplicitFreeSurface(grid; substeps = 10) model = HydrostaticFreeSurfaceModel(; grid, free_surface, tracers = (:b, :c), From 849c253d1e2e2638140c857b578fb2bccb987d18 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Oct 2024 15:16:52 +0200 Subject: [PATCH 412/567] remove OrthogonalSphericalShellGrids while we decide what to do --- Project.toml | 3 +- difference | 302 +++++++++++++++++++++++++++++++++++++ difference_se | 126 ++++++++++++++++ mynew_model.jl | 139 +++++++++++++++++ test/test_cubed_spheres.jl | 1 - 5 files changed, 568 insertions(+), 3 deletions(-) create mode 100644 difference create mode 100644 difference_se create mode 100644 mynew_model.jl diff --git a/Project.toml b/Project.toml index 13f1fb6317..06a8553f22 100644 --- a/Project.toml +++ b/Project.toml @@ -79,10 +79,9 @@ julia = "1.9" [extras] DataDeps = "124859b0-ceae-595e-8997-d05f6a7a8dfe" Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" -OrthogonalSphericalShellGrids = "c2be9673-fb75-4747-82dc-aa2bb9f4aed0" SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" TimesDates = "bdfc003b-8df8-5c39-adcd-3a9087f5df4a" [targets] -test = ["DataDeps", "Enzyme", "OrthogonalSphericalShellGrids", "SafeTestsets", "Test", "TimesDates"] +test = ["DataDeps", "Enzyme", "SafeTestsets", "Test", "TimesDates"] diff --git a/difference b/difference new file mode 100644 index 0000000000..5ef0c89b63 --- /dev/null +++ b/difference @@ -0,0 +1,302 @@ + ++const PGX = AbstractGrid{<:Any, <:Periodic} ++const BGX = AbstractGrid{<:Any, <:Bounded} ++const RGX = AbstractGrid{<:Any, <:RightConnected} ++ ++const PGY = AbstractGrid{<:Any, <:Periodic} ++const BGY = AbstractGrid{<:Any, <:Bounded} ++const RGY = AbstractGrid{<:Any, <:RightConnected} ++ ++@inline ℑxᶠᵃᵃ_η(i, j, k, grid, η) = ℑxᶠᵃᵃ(i, j, k, grid, η) ++@inline ℑyᵃᶠᵃ_η(i, j, k, grid, η) = ℑyᵃᶠᵃ(i, j, k, grid, η) ++@inline ℑxᶠᵃᵃ_η(i, j, k, grid::PGX, η) = ifelse(i == 1, (η[1, j, k] + η[grid.Nx, j, k]) / 2, ℑxᶠᵃᵃ(i, j, k, grid, η)) ++@inline ℑyᵃᶠᵃ_η(i, j, k, grid::PGY, η) = ifelse(j == 1, (η[i, 1, k] + η[i, grid.Ny, k]) / 2, ℑyᵃᶠᵃ(i, j, k, grid, η)) ++@inline ℑxᶠᵃᵃ_η(i, j, k, grid::BGX, η) = ifelse(i == 1, η[1, j, k], ℑxᶠᵃᵃ(i, j, k, grid, η)) ++@inline ℑyᵃᶠᵃ_η(i, j, k, grid::BGY, η) = ifelse(j == 1, η[i, 1, k], ℑyᵃᶠᵃ(i, j, k, grid, η)) ++@inline ℑxᶠᵃᵃ_η(i, j, k, grid::RGX, η) = ifelse(i == 1, η[1, j, k], ℑxᶠᵃᵃ(i, j, k, grid, η)) ++@inline ℑyᵃᶠᵃ_η(i, j, k, grid::RGY, η) = ifelse(j == 1, η[i, 1, k], ℑyᵃᶠᵃ(i, j, k, grid, η)) ++ + # Time stepping extrapolation U★, and η★ + + # AB3 step +@@ -158,6 +178,14 @@ end + return nothing + end + ++# Linear free surface implementation ++@inline dynamic_column_depthᶠᶜᵃ(i, j, k, grid, η) = static_column_depthᶠᶜᵃ(i, j, grid) ++@inline dynamic_column_depthᶜᶠᵃ(i, j, k, grid, η) = static_column_depthᶜᶠᵃ(i, j, grid) ++ ++# Non-linear free surface implementation ++@inline dynamic_column_depthᶠᶜᵃ(i, j, k, grid::ZStarSpacingGrid, η) = static_column_depthᶠᶜᵃ(i, j, grid) + ℑxᶠᵃᵃ_η(i, j, k, grid, η) ++@inline dynamic_column_depthᶜᶠᵃ(i, j, k, grid::ZStarSpacingGrid, η) = static_column_depthᶜᶠᵃ(i, j, grid) + ℑyᵃᶠᵃ_η(i, j, k, grid, η) ++ + @kernel function _split_explicit_barotropic_velocity!(averaging_weight, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², + U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², + η̅, U̅, V̅, Gᵁ, Gⱽ, g, +@@ -183,9 +211,9 @@ end + advance_previous_velocity!(i, j, k_top-1, timestepper, U, Uᵐ⁻¹, Uᵐ⁻²) + advance_previous_velocity!(i, j, k_top-1, timestepper, V, Vᵐ⁻¹, Vᵐ⁻²) + +- Hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) +- Hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) +- ++ Hᶠᶜ = dynamic_column_depthᶠᶜᵃ(i, j, k_top, grid, η) ++ Hᶜᶠ = dynamic_column_depthᶠᶜᵃ(i, j, k_top, grid, η) ++ + # ∂τ(U) = - ∇η + G + U[i, j, k_top-1] += Δτ * (- g * Hᶠᶜ * ∂xᶠᶜᶠ_η(i, j, k_top, grid, TX, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gᵁ[i, j, k_top-1]) + V[i, j, k_top-1] += Δτ * (- g * Hᶜᶠ * ∂yᶜᶠᶠ_η(i, j, k_top, grid, TY, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gⱽ[i, j, k_top-1]) +@@ -199,16 +227,26 @@ end + + # Barotropic Model Kernels + # u_Δz = u * Δz ++# For Zstar vertical spacing the vertical integral includes the dynamic height ++# Remember, the vertical coordinate has not yet been updated! ++# For this reason the integration has to be performed manually + @kernel function _barotropic_mode_kernel!(U, V, grid, ::Nothing, u, v) + i, j = @index(Global, NTuple) + k_top = grid.Nz+1 + +- @inbounds U[i, j, k_top-1] = Δzᶠᶜᶜ(i, j, 1, grid) * u[i, j, 1] +- @inbounds V[i, j, k_top-1] = Δzᶜᶠᶜ(i, j, 1, grid) * v[i, j, 1] ++ hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) ++ hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) ++ ++ sᶠᶜ = ifelse(hᶠᶜ == 0, one(grid), dynamic_column_depthᶠᶜᵃ(i, j, k_top, grid, η) / hᶠᶜ) ++ sᶜᶠ = ifelse(hᶜᶠ == 0, one(grid), dynamic_column_depthᶜᶠᵃ(i, j, k_top, grid, η) / hᶜᶠ) + +- for k in 2:grid.Nz +- @inbounds U[i, j, k_top-1] += Δzᶠᶜᶜ(i, j, k, grid) * u[i, j, k] +- @inbounds V[i, j, k_top-1] += Δzᶜᶠᶜ(i, j, k, grid) * v[i, j, k] ++ # hand unroll first loop ++ @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δrᶠᶜᶜ(i, j, 1, grid) * sᶠᶜ ++ @inbounds V[i, j, k_top-1] = v[i, j, 1] * Δrᶜᶠᶜ(i, j, 1, grid) * sᶜᶠ ++ ++ @unroll for k in 2:grid.Nz ++ @inbounds U[i, j, k_top-1] += u[i, j, k] * Δrᶠᶜᶜ(i, j, k, grid) * sᶠᶜ ++ @inbounds V[i, j, k_top-1] += v[i, j, k] * Δrᶜᶠᶜ(i, j, k, grid) * sᶜᶠ + end + end + +@@ -219,19 +257,55 @@ end + i, j = active_linear_index_to_tuple(idx, active_cells_map) + k_top = grid.Nz+1 + +- @inbounds U[i, j, k_top-1] = Δzᶠᶜᶜ(i, j, 1, grid) * u[i, j, 1] +- @inbounds V[i, j, k_top-1] = Δzᶜᶠᶜ(i, j, 1, grid) * v[i, j, 1] ++ hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) ++ hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) ++ ++ sᶠᶜ = ifelse(hᶠᶜ == 0, one(grid), dynamic_column_depthᶠᶜᵃ(i, j, k_top, grid, η) / hᶠᶜ) ++ sᶜᶠ = ifelse(hᶜᶠ == 0, one(grid), dynamic_column_depthᶜᶠᵃ(i, j, k_top, grid, η) / hᶜᶠ) + +- for k in 2:grid.Nz +- @inbounds U[i, j, k_top-1] += Δzᶠᶜᶜ(i, j, k, grid) * u[i, j, k] +- @inbounds V[i, j, k_top-1] += Δzᶜᶠᶜ(i, j, k, grid) * v[i, j, k] ++ # hand unroll first loop ++ @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δrᶠᶜᶜ(i, j, 1, grid) * sᶠᶜ ++ @inbounds V[i, j, k_top-1] = v[i, j, 1] * Δrᶜᶠᶜ(i, j, 1, grid) * sᶜᶠ ++ ++ @unroll for k in 2:grid.Nz ++ @inbounds U[i, j, k_top-1] += u[i, j, k] * Δrᶠᶜᶜ(i, j, k, grid) * sᶠᶜ ++ @inbounds V[i, j, k_top-1] += v[i, j, k] * Δrᶜᶠᶜ(i, j, k, grid) * sᶜᶠ + end + end + +-@inline function compute_barotropic_mode!(U, V, grid, u, v) ++@inline function compute_barotropic_mode!(U, V, grid, u, v, η) + active_cells_map = retrieve_surface_active_cells_map(grid) + +- launch!(architecture(grid), grid, :xy, _barotropic_mode_kernel!, U, V, grid, active_cells_map, u, v; active_cells_map) ++ launch!(architecture(grid), grid, :xy, _barotropic_mode_kernel!, U, V, grid, active_cells_map, u, v, η; active_cells_map) ++ ++ return nothing ++end ++ ++@kernel function _barotropic_split_explicit_corrector!(u, v, grid, U̅, V̅, U, V, η) ++ i, j, k = @index(Global, NTuple) ++ k_top = grid.Nz + 1 ++ ++ Hᶠᶜ = dynamic_column_depthᶠᶜᵃ(i, j, k_top, grid, η) ++ Hᶜᶠ = dynamic_column_depthᶜᶠᵃ(i, j, k_top, grid, η) ++ ++ @inbounds begin ++ u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / Hᶠᶜ ++ v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / Hᶜᶠ ++ end ++end ++ ++function barotropic_split_explicit_corrector!(u, v, free_surface, grid) ++ sefs = free_surface.state ++ U, V, U̅, V̅ = sefs.U, sefs.V, sefs.U̅, sefs.V̅ ++ arch = architecture(grid) ++ ++ # take out "bad" barotropic mode, ++ # !!!! reusing U and V for this storage since last timestep doesn't matter ++ compute_barotropic_mode!(U, V, grid, u, v, free_surface.η) ++ # add in "good" barotropic mode ++ ++ launch!(arch, grid, :xyz, _barotropic_split_explicit_corrector!, ++ u, v, grid, U̅, V̅, U, V, free_surface.η) + + return nothing + end +@@ -266,34 +340,6 @@ function initialize_auxiliary_state!(state, η, timestepper) + return nothing + end + +-@kernel function _barotropic_split_explicit_corrector!(u, v, U̅, V̅, U, V, grid) +- i, j, k = @index(Global, NTuple) +- k_top = grid.Nz+1 +- +- @inbounds begin +- Hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) +- Hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) +- +- u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / Hᶠᶜ +- v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / Hᶜᶠ +- end +-end +- +-function barotropic_split_explicit_corrector!(u, v, free_surface, grid) +- sefs = free_surface.state +- U, V, U̅, V̅ = sefs.U, sefs.V, sefs.U̅, sefs.V̅ +- arch = architecture(grid) +- +- # take out "bad" barotropic mode, +- # !!!! reusing U and V for this storage since last timestep doesn't matter +- compute_barotropic_mode!(U, V, grid, u, v) +- # add in "good" barotropic mode +- launch!(arch, grid, :xyz, _barotropic_split_explicit_corrector!, +- u, v, U̅, V̅, U, V, grid) +- +- return nothing +-end +- + """ + Explicitly step forward η in substeps. + """ +@@ -301,8 +347,13 @@ ab2_step_free_surface!(free_surface::SplitExplicitFreeSurface, model, Δt, χ) = + split_explicit_free_surface_step!(free_surface, model, Δt, χ) + + function initialize_free_surface!(sefs::SplitExplicitFreeSurface, grid, velocities) +- @apply_regionally compute_barotropic_mode!(sefs.state.U̅, sefs.state.V̅, grid, velocities.u, velocities.v) ++ U̅, V̅ = sefs.state.U̅, sefs.state.V̅ ++ u, v, _ = velocities ++ ++ @apply_regionally compute_barotropic_mode!(U̅, V̅, grid, u, v, sefs.η) + fill_halo_regions!((sefs.state.U̅, sefs.state.V̅, sefs.η)) ++ ++ return nothing + end + + function split_explicit_free_surface_step!(free_surface::SplitExplicitFreeSurface, model, Δt, χ) +@@ -336,6 +387,10 @@ function split_explicit_free_surface_step!(free_surface::SplitExplicitFreeSurfac + set!(free_surface.η, free_surface.state.η̅) + end + ++ # This is needed for the barotropic mode calculations, so it cannot be done asynchronously ++ fill_halo_regions!(free_surface.η) ++ ++ # Velocities can be passed asynchronously + fields_to_fill = (free_surface.state.U̅, free_surface.state.V̅) + fill_halo_regions!(fields_to_fill; async = true) + +@@ -363,9 +418,6 @@ const MINIMUM_SUBSTEPS = 5 + @inline calculate_adaptive_settings(substepping::FTS, substeps) = weights_from_substeps(eltype(substepping.Δt_barotropic), + substeps, substepping.averaging_kernel) + +-const FixedSubstepsSetting{N} = SplitExplicitSettings{<:FixedSubstepNumber{<:Any, <:NTuple{N, <:Any}}} where N +-const FixedSubstepsSplitExplicit{F} = SplitExplicitFreeSurface{<:Any, <:Any, <:Any, <:Any, <:FixedSubstepsSetting{N}} where N +- + function iterate_split_explicit!(free_surface, grid, Δτᴮ, weights, ::Val{Nsubsteps}) where Nsubsteps + arch = architecture(grid) + +@@ -424,12 +476,12 @@ end + i, j = @index(Global, NTuple) + k_top = grid.Nz + 1 + +- @inbounds Gᵁ[i, j, k_top-1] = Δzᶠᶜᶜ(i, j, 1, grid) * ab2_step_Gu(i, j, 1, grid, Gu⁻, Guⁿ, χ) +- @inbounds Gⱽ[i, j, k_top-1] = Δzᶜᶠᶜ(i, j, 1, grid) * ab2_step_Gv(i, j, 1, grid, Gv⁻, Gvⁿ, χ) ++ @inbounds Gᵁ[i, j, k_top-1] = Δrᶠᶜᶜ(i, j, 1, grid) * ab2_step_Gu(i, j, 1, grid, Gu⁻, Guⁿ, χ) ++ @inbounds Gⱽ[i, j, k_top-1] = Δrᶜᶠᶜ(i, j, 1, grid) * ab2_step_Gv(i, j, 1, grid, Gv⁻, Gvⁿ, χ) + + for k in 2:grid.Nz +- @inbounds Gᵁ[i, j, k_top-1] += Δzᶠᶜᶜ(i, j, k, grid) * ab2_step_Gu(i, j, k, grid, Gu⁻, Guⁿ, χ) +- @inbounds Gⱽ[i, j, k_top-1] += Δzᶜᶠᶜ(i, j, k, grid) * ab2_step_Gv(i, j, k, grid, Gv⁻, Gvⁿ, χ) ++ @inbounds Gᵁ[i, j, k_top-1] += Δrᶠᶜᶜ(i, j, k, grid) * ab2_step_Gu(i, j, k, grid, Gu⁻, Guⁿ, χ) ++ @inbounds Gⱽ[i, j, k_top-1] += Δrᶜᶠᶜ(i, j, k, grid) * ab2_step_Gv(i, j, k, grid, Gv⁻, Gvⁿ, χ) + end + end + +@@ -439,24 +491,44 @@ end + i, j = active_linear_index_to_tuple(idx, active_cells_map) + k_top = grid.Nz+1 + +- @inbounds Gᵁ[i, j, k_top-1] = Δzᶠᶜᶜ(i, j, 1, grid) * ab2_step_Gu(i, j, 1, grid, Gu⁻, Guⁿ, χ) +- @inbounds Gⱽ[i, j, k_top-1] = Δzᶜᶠᶜ(i, j, 1, grid) * ab2_step_Gv(i, j, 1, grid, Gv⁻, Gvⁿ, χ) ++ @inbounds Gᵁ[i, j, k_top-1] = Δrᶠᶜᶜ(i, j, 1, grid) * ab2_step_Gu(i, j, 1, grid, Gu⁻, Guⁿ, χ) ++ @inbounds Gⱽ[i, j, k_top-1] = Δrᶜᶠᶜ(i, j, 1, grid) * ab2_step_Gv(i, j, 1, grid, Gv⁻, Gvⁿ, χ) + + for k in 2:grid.Nz +- @inbounds Gᵁ[i, j, k_top-1] += Δzᶠᶜᶜ(i, j, k, grid) * ab2_step_Gu(i, j, k, grid, Gu⁻, Guⁿ, χ) +- @inbounds Gⱽ[i, j, k_top-1] += Δzᶜᶠᶜ(i, j, k, grid) * ab2_step_Gv(i, j, k, grid, Gv⁻, Gvⁿ, χ) ++ @inbounds Gᵁ[i, j, k_top-1] += Δrᶠᶜᶜ(i, j, k, grid) * ab2_step_Gu(i, j, k, grid, Gu⁻, Guⁿ, χ) ++ @inbounds Gⱽ[i, j, k_top-1] += Δrᶜᶠᶜ(i, j, k, grid) * ab2_step_Gv(i, j, k, grid, Gv⁻, Gvⁿ, χ) + end + end + +-@inline ab2_step_Gu(i, j, k, grid, G⁻, Gⁿ, χ::FT) where FT = +- @inbounds ifelse(peripheral_node(i, j, k, grid, f, c, c), zero(grid), (convert(FT, 1.5) + χ) * Gⁿ[i, j, k] - G⁻[i, j, k] * (convert(FT, 0.5) + χ)) ++@inline function ab2_step_Gu(i, j, k, grid, G⁻, Gⁿ, χ::FT) where FT ++ C¹ = convert(FT, 3/2) + χ ++ C² = convert(FT, 1/2) + χ ++ ++ Fⁿ = @inbounds C¹ * Gⁿ[i, j, k] * vertical_scaling(i, j, k, grid, Face(), Center(), Center()) ++ F⁻ = @inbounds C² * G⁻[i, j, k] * previous_vertical_scaling(i, j, k, grid, Face(), Center(), Center()) ++ ++ Gi = Fⁿ - F⁻ ++ ++ return ifelse(peripheral_node(i, j, k, grid, f, c, c), zero(grid), Gi) ++end ++ ++@inline function ab2_step_Gv(i, j, k, grid, G⁻, Gⁿ, χ::FT) where FT ++ C¹ = convert(FT, 3/2) + χ ++ C² = convert(FT, 1/2) + χ ++ ++ Fⁿ = @inbounds C¹ * Gⁿ[i, j, k] * vertical_scaling(i, j, k, grid, Center(), Face(), Center()) ++ F⁻ = @inbounds C² * G⁻[i, j, k] * previous_vertical_scaling(i, j, k, grid, Center(), Face(), Center()) + +-@inline ab2_step_Gv(i, j, k, grid, G⁻, Gⁿ, χ::FT) where FT = +- @inbounds ifelse(peripheral_node(i, j, k, grid, c, f, c), zero(grid), (convert(FT, 1.5) + χ) * Gⁿ[i, j, k] - G⁻[i, j, k] * (convert(FT, 0.5) + χ)) ++ Gi = Fⁿ - F⁻ ++ ++ return ifelse(peripheral_node(i, j, k, grid, f, c, c), zero(grid), Gi) ++end + + # Setting up the RHS for the barotropic step (tendencies of the barotropic velocity components) + # This function is called after `calculate_tendency` and before `ab2_step_velocities!` + function setup_free_surface!(model, free_surface::SplitExplicitFreeSurface, χ) ++ ++ grid = model.grid + + # we start the time integration of η from the average ηⁿ + Gu⁻ = model.timestepper.G⁻.u +@@ -466,7 +538,7 @@ function setup_free_surface!(model, free_surface::SplitExplicitFreeSurface, χ) + + auxiliary = free_surface.auxiliary + +- @apply_regionally setup_split_explicit_tendency!(auxiliary, model.grid, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) ++ @apply_regionally setup_split_explicit_tendency!(auxiliary, grid, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) + + fields_to_fill = (auxiliary.Gᵁ, auxiliary.Gⱽ) + fill_halo_regions!(fields_to_fill; async = true) +@@ -483,5 +555,4 @@ end + return nothing + end + +-wait_free_surface_communication!(free_surface, arch) = nothing +- ++wait_free_surface_communication!(free_surface, arch) = nothing +\ No newline at end of file diff --git a/difference_se b/difference_se new file mode 100644 index 0000000000..75b7e8da6d --- /dev/null +++ b/difference_se @@ -0,0 +1,126 @@ +diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +index 073fbfc4e..6992a410d 100644 +--- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl ++++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +@@ -3,8 +3,9 @@ using Oceananigans.Architectures + using Oceananigans.Fields + using Oceananigans.Grids + using Oceananigans.Grids: AbstractGrid ++using Oceananigans.BoundaryConditions: default_prognostic_bc + using Oceananigans.AbstractOperations: Δz, GridMetricOperation +- ++using Oceananigans.ImmersedBoundaries: immersed_peripheral_node, c, f + using Adapt + using Base + using KernelAbstractions: @index, @kernel +@@ -158,6 +159,10 @@ Base.@kwdef struct SplitExplicitState{CC, ACC, FC, AFC, CF, ACF} + U̅ :: FC + "The time-filtered barotropic meridional velocity. (`ReducedField` over ``z``)" + V̅ :: CF ++ # "The time-filtered barotropic zonal velocity. (`ReducedField` over ``z``)" ++ # Ũ :: FC ++ # "The time-filtered barotropic meridional velocity. (`ReducedField` over ``z``)" ++ # Ṽ :: CF + end + + """ +@@ -179,18 +184,20 @@ function SplitExplicitState(grid::AbstractGrid, timestepper) + ηᵐ⁻¹ = auxiliary_free_surface_field(grid, timestepper) + ηᵐ⁻² = auxiliary_free_surface_field(grid, timestepper) + +- U = XFaceField(grid, indices = (:, :, Nz)) +- V = YFaceField(grid, indices = (:, :, Nz)) +- +- Uᵐ⁻¹ = auxiliary_barotropic_U_field(grid, timestepper) +- Vᵐ⁻¹ = auxiliary_barotropic_V_field(grid, timestepper) +- Uᵐ⁻² = auxiliary_barotropic_U_field(grid, timestepper) +- Vᵐ⁻² = auxiliary_barotropic_V_field(grid, timestepper) ++ 𝒰 = VelocityFields(grid) ++ ++ U = Field(𝒰.u, indices = (:, :, Nz)) ++ V = Field(𝒰.v, indices = (:, :, Nz)) + +- U̅ = XFaceField(grid, indices = (:, :, Nz)) +- V̅ = YFaceField(grid, indices = (:, :, Nz)) ++ Uᵐ⁻¹ = auxiliary_barotropic_velocity_field(U, timestepper) ++ Vᵐ⁻¹ = auxiliary_barotropic_velocity_field(V, timestepper) ++ Uᵐ⁻² = auxiliary_barotropic_velocity_field(U, timestepper) ++ Vᵐ⁻² = auxiliary_barotropic_velocity_field(V, timestepper) ++ ++ U̅ = deepcopy(U) ++ V̅ = deepcopy(V) + +- return SplitExplicitState(; ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅) ++ return SplitExplicitState(; ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅) + end + + """ +@@ -204,7 +211,7 @@ large (or `:xy` in case of a serial computation), and start computing from + + $(FIELDS) + """ +-Base.@kwdef struct SplitExplicitAuxiliaryFields{𝒞ℱ, ℱ𝒞, 𝒦} ++Base.@kwdef struct SplitExplicitAuxiliaryFields{ℱ𝒞, 𝒞ℱ, 𝒦} + "Vertically-integrated slow barotropic forcing function for `U` (`ReducedField` over ``z``)" + Gᵁ :: ℱ𝒞 + "Vertically-integrated slow barotropic forcing function for `V` (`ReducedField` over ``z``)" +@@ -220,8 +227,8 @@ Return the `SplitExplicitAuxiliaryFields` for `grid`. + """ + function SplitExplicitAuxiliaryFields(grid::AbstractGrid) + +- Gᵁ = Field((Face, Center, Nothing), grid) +- Gⱽ = Field((Center, Face, Nothing), grid) ++ Gᵁ = Field{Face, Center, Nothing}(grid) ++ Gⱽ = Field{Center, Face, Nothing}(grid) + + kernel_parameters = :xy + +@@ -244,14 +251,11 @@ end + struct AdamsBashforth3Scheme end + struct ForwardBackwardScheme end + +- + auxiliary_free_surface_field(grid, ::AdamsBashforth3Scheme) = ZFaceField(grid, indices = (:, :, size(grid, 3)+1)) + auxiliary_free_surface_field(grid, ::ForwardBackwardScheme) = nothing + +-auxiliary_barotropic_U_field(grid, ::AdamsBashforth3Scheme) = XFaceField(grid, indices = (:, :, size(grid, 3))) +-auxiliary_barotropic_U_field(grid, ::ForwardBackwardScheme) = nothing +-auxiliary_barotropic_V_field(grid, ::AdamsBashforth3Scheme) = YFaceField(grid, indices = (:, :, size(grid, 3))) +-auxiliary_barotropic_V_field(grid, ::ForwardBackwardScheme) = nothing ++auxiliary_barotropic_velocity_field(U, ::AdamsBashforth3Scheme) = deecopy(u) ++auxiliary_barotropic_velocity_field(U, ::ForwardBackwardScheme) = nothing + + # (p = 2, q = 4, r = 0.18927) minimize dispersion error from Shchepetkin and McWilliams (2005): https://doi.org/10.1016/j.ocemod.2004.08.002 + @inline function averaging_shape_function(τ::FT; p = 2, q = 4, r = FT(0.18927)) where FT +@@ -306,7 +310,7 @@ end + + averaging_weights = averaging_weights[1:idx] + averaging_weights ./= sum(averaging_weights) +- ++ + return Δτ, tuple(averaging_weights...) + end + +@@ -390,11 +394,19 @@ end + + # Adapt + Adapt.adapt_structure(to, free_surface::SplitExplicitFreeSurface) = +- SplitExplicitFreeSurface(Adapt.adapt(to, free_surface.η), nothing, nothing, ++ SplitExplicitFreeSurface(Adapt.adapt(to, free_surface.η), ++ nothing, ++ Adapt.adapt(to, free_surface.auxiliary), + free_surface.gravitational_acceleration, nothing) + +-for Type in (:SplitExplicitFreeSurface, +- :SplitExplicitSettings, ++# Adapt ++Adapt.adapt_structure(to, auxiliary::SplitExplicitAuxiliaryFields) = ++ SplitExplicitAuxiliaryFields(Adapt.adapt(to, auxiliary.Gᵁ), ++ Adapt.adapt(to, auxiliary.Gⱽ), ++ nothing) ++ ++for Type in (:SplitExplicitFreeSurface, ++ :SplitExplicitSettings, + :SplitExplicitState, + :SplitExplicitAuxiliaryFields, + :FixedTimeStepSize, diff --git a/mynew_model.jl b/mynew_model.jl new file mode 100644 index 0000000000..a9467b4cb0 --- /dev/null +++ b/mynew_model.jl @@ -0,0 +1,139 @@ +using Oceananigans +using Oceananigans: tupleit +using Oceananigans.Grids +using Oceananigans.Architectures: architecture +using Oceananigans.Advection: div_Uc +using Oceananigans.BoundaryConditions +using Oceananigans.Utils +using Oceananigans.Fields: TracerFields +using Oceananigans.Models.HydrostaticFreeSurfaceModels: PrescribedVelocityFields, tracernames, validate_tracer_advection, with_tracers, tracernames +using KernelAbstractions: @index, @kernel + +import Oceananigans.Models: prognostic_fields + +using Oceananigans.TimeSteppers: RungeKutta3TimeStepper, store_tendencies!, rk3_substep_field! +import Oceananigans.TimeSteppers: update_state!, time_step!, compute_tendencies!, rk3_substep! + +mutable struct AdvectiveModel{G, A, V, T, B} + grid :: G + advection :: A + velocities :: V + tracers :: T + timestepper :: B +end + +struct DirectSpaceTimeAder end + +function AdvectiveModel(; grid, + velocities = PrescribedVelocityFields(; u = 1), + tracers = :b, + timestepper = nothing, + advection = WENO(; order = 5)) + + # Next, we form a list of default boundary conditions: + tracers = tupleit(tracers) # supports tracers=:c keyword argument (for example) + + tracers = TracerFields(tracers, grid, NamedTuple()) + Gⁿ = deepcopy(tracers) + G⁻ = deepcopy(tracers) + + if timestepper isa Nothing + timestepper = DirectSpaceTime() + else + timestepper = RungeKutta3TimeStepper(grid, tracernames(tracers); Gⁿ = Gⁿ, G⁻ = G⁻) + end + + model = AdvectiveModel(grid, advection, velocities, tracers, timestepper) + update_state!(model) + + return model +end + +prognostic_fields(model::AdvectiveModel) = model.tracers + +update_state!(model::AdvectiveModel) = + fill_halo_regions!(model.tracers) + +function time_step!(model::AdvectiveModel, Δt) + Δt == 0 && @warn "Δt == 0 may cause model blowup!" + + # Be paranoid and update state at iteration 0, in case run! is not used: + update_state!(model) + + γ¹ = model.timestepper.γ¹ + γ² = model.timestepper.γ² + γ³ = model.timestepper.γ³ + + ζ² = model.timestepper.ζ² + ζ³ = model.timestepper.ζ³ + + compute_tendencies!(model) + rk3_substep!(model, Δt, γ¹, nothing) + store_tendencies!(model) + update_state!(model) + + # + # Second stage + # + + compute_tendencies!(model) + rk3_substep!(model, Δt, γ², ζ²) + store_tendencies!(model) + update_state!(model) + + # + # Third stage + # + + compute_tendencies!(model) + rk3_substep!(model, Δt, γ³, ζ³) + update_state!(model) + + return nothing +end + +function rk3_substep!(model::AdvectiveModel, Δt, γ, ζ) + grid = model.grid + arch = architecture(grid) + tracers = model.tracers + velocities = model.velocities + + for (tracer, Gⁿ, G⁻) in zip(tracers, model.timestepper.Gⁿ, model.timestepper.G⁻) + launch!(arch, grid, :xyz, rk3_substep_field!, tracer, Δt, γ, ζ, Gⁿ, G⁻) + end + + return nothing +end + +function compute_tendencies!(model::AdvectiveModel) + grid = model.grid + arch = architecture(grid) + tendencies = model.timestepper.Gⁿ + tracers = model.tracers + velocities = model.velocities + + for (tracer, G) in zip(tracers, tendencies) + launch!(arch, grid, :xyz, _compute_tendency!, G, grid, model.advection, velocities, tracer) + end + + return nothing +end + +@kernel function _compute_tendency!(G, grid, advection, U, c) + i, j, k = @index(Global, NTuple) + @inbounds G[i, j, k] = - div_Uc(i, j, k, grid, advection, U, c) +end + +function time_step!(model::AdvectiveModel{G, A, V, <:DirectSpaceTimeAder, B}, Δt) where {G, A, V, B} + Δt == 0 && @warn "Δt == 0 may cause model blowup!" + + # Be paranoid and update state at iteration 0, in case run! is not used: + update_state!(model) + + compute_tendencies!(model) + rk3_substep!(model, Δt, γ¹, nothing) + store_tendencies!(model) + update_state!(model) + + return nothing +end diff --git a/test/test_cubed_spheres.jl b/test/test_cubed_spheres.jl index e907dedf68..a950dc5cbf 100644 --- a/test/test_cubed_spheres.jl +++ b/test/test_cubed_spheres.jl @@ -173,7 +173,6 @@ end test_vector_rotation(grid) end - @testset "CubedSphereData and CubedSphereFields [$(typeof(arch))]" begin @info " Testing CubedSphereData and CubedSphereFields [$(typeof(arch))]..." c = model.tracers.c From 510a858cc8ff520894ef1a8e0e551cf188e67077 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Oct 2024 15:18:30 +0200 Subject: [PATCH 413/567] these files shouldn't go here --- difference | 302 ------------------------------------------------- difference_se | 126 --------------------- mynew_model.jl | 139 ----------------------- 3 files changed, 567 deletions(-) delete mode 100644 difference delete mode 100644 difference_se delete mode 100644 mynew_model.jl diff --git a/difference b/difference deleted file mode 100644 index 5ef0c89b63..0000000000 --- a/difference +++ /dev/null @@ -1,302 +0,0 @@ - -+const PGX = AbstractGrid{<:Any, <:Periodic} -+const BGX = AbstractGrid{<:Any, <:Bounded} -+const RGX = AbstractGrid{<:Any, <:RightConnected} -+ -+const PGY = AbstractGrid{<:Any, <:Periodic} -+const BGY = AbstractGrid{<:Any, <:Bounded} -+const RGY = AbstractGrid{<:Any, <:RightConnected} -+ -+@inline ℑxᶠᵃᵃ_η(i, j, k, grid, η) = ℑxᶠᵃᵃ(i, j, k, grid, η) -+@inline ℑyᵃᶠᵃ_η(i, j, k, grid, η) = ℑyᵃᶠᵃ(i, j, k, grid, η) -+@inline ℑxᶠᵃᵃ_η(i, j, k, grid::PGX, η) = ifelse(i == 1, (η[1, j, k] + η[grid.Nx, j, k]) / 2, ℑxᶠᵃᵃ(i, j, k, grid, η)) -+@inline ℑyᵃᶠᵃ_η(i, j, k, grid::PGY, η) = ifelse(j == 1, (η[i, 1, k] + η[i, grid.Ny, k]) / 2, ℑyᵃᶠᵃ(i, j, k, grid, η)) -+@inline ℑxᶠᵃᵃ_η(i, j, k, grid::BGX, η) = ifelse(i == 1, η[1, j, k], ℑxᶠᵃᵃ(i, j, k, grid, η)) -+@inline ℑyᵃᶠᵃ_η(i, j, k, grid::BGY, η) = ifelse(j == 1, η[i, 1, k], ℑyᵃᶠᵃ(i, j, k, grid, η)) -+@inline ℑxᶠᵃᵃ_η(i, j, k, grid::RGX, η) = ifelse(i == 1, η[1, j, k], ℑxᶠᵃᵃ(i, j, k, grid, η)) -+@inline ℑyᵃᶠᵃ_η(i, j, k, grid::RGY, η) = ifelse(j == 1, η[i, 1, k], ℑyᵃᶠᵃ(i, j, k, grid, η)) -+ - # Time stepping extrapolation U★, and η★ - - # AB3 step -@@ -158,6 +178,14 @@ end - return nothing - end - -+# Linear free surface implementation -+@inline dynamic_column_depthᶠᶜᵃ(i, j, k, grid, η) = static_column_depthᶠᶜᵃ(i, j, grid) -+@inline dynamic_column_depthᶜᶠᵃ(i, j, k, grid, η) = static_column_depthᶜᶠᵃ(i, j, grid) -+ -+# Non-linear free surface implementation -+@inline dynamic_column_depthᶠᶜᵃ(i, j, k, grid::ZStarSpacingGrid, η) = static_column_depthᶠᶜᵃ(i, j, grid) + ℑxᶠᵃᵃ_η(i, j, k, grid, η) -+@inline dynamic_column_depthᶜᶠᵃ(i, j, k, grid::ZStarSpacingGrid, η) = static_column_depthᶜᶠᵃ(i, j, grid) + ℑyᵃᶠᵃ_η(i, j, k, grid, η) -+ - @kernel function _split_explicit_barotropic_velocity!(averaging_weight, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², - U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², - η̅, U̅, V̅, Gᵁ, Gⱽ, g, -@@ -183,9 +211,9 @@ end - advance_previous_velocity!(i, j, k_top-1, timestepper, U, Uᵐ⁻¹, Uᵐ⁻²) - advance_previous_velocity!(i, j, k_top-1, timestepper, V, Vᵐ⁻¹, Vᵐ⁻²) - -- Hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) -- Hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) -- -+ Hᶠᶜ = dynamic_column_depthᶠᶜᵃ(i, j, k_top, grid, η) -+ Hᶜᶠ = dynamic_column_depthᶠᶜᵃ(i, j, k_top, grid, η) -+ - # ∂τ(U) = - ∇η + G - U[i, j, k_top-1] += Δτ * (- g * Hᶠᶜ * ∂xᶠᶜᶠ_η(i, j, k_top, grid, TX, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gᵁ[i, j, k_top-1]) - V[i, j, k_top-1] += Δτ * (- g * Hᶜᶠ * ∂yᶜᶠᶠ_η(i, j, k_top, grid, TY, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gⱽ[i, j, k_top-1]) -@@ -199,16 +227,26 @@ end - - # Barotropic Model Kernels - # u_Δz = u * Δz -+# For Zstar vertical spacing the vertical integral includes the dynamic height -+# Remember, the vertical coordinate has not yet been updated! -+# For this reason the integration has to be performed manually - @kernel function _barotropic_mode_kernel!(U, V, grid, ::Nothing, u, v) - i, j = @index(Global, NTuple) - k_top = grid.Nz+1 - -- @inbounds U[i, j, k_top-1] = Δzᶠᶜᶜ(i, j, 1, grid) * u[i, j, 1] -- @inbounds V[i, j, k_top-1] = Δzᶜᶠᶜ(i, j, 1, grid) * v[i, j, 1] -+ hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) -+ hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) -+ -+ sᶠᶜ = ifelse(hᶠᶜ == 0, one(grid), dynamic_column_depthᶠᶜᵃ(i, j, k_top, grid, η) / hᶠᶜ) -+ sᶜᶠ = ifelse(hᶜᶠ == 0, one(grid), dynamic_column_depthᶜᶠᵃ(i, j, k_top, grid, η) / hᶜᶠ) - -- for k in 2:grid.Nz -- @inbounds U[i, j, k_top-1] += Δzᶠᶜᶜ(i, j, k, grid) * u[i, j, k] -- @inbounds V[i, j, k_top-1] += Δzᶜᶠᶜ(i, j, k, grid) * v[i, j, k] -+ # hand unroll first loop -+ @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δrᶠᶜᶜ(i, j, 1, grid) * sᶠᶜ -+ @inbounds V[i, j, k_top-1] = v[i, j, 1] * Δrᶜᶠᶜ(i, j, 1, grid) * sᶜᶠ -+ -+ @unroll for k in 2:grid.Nz -+ @inbounds U[i, j, k_top-1] += u[i, j, k] * Δrᶠᶜᶜ(i, j, k, grid) * sᶠᶜ -+ @inbounds V[i, j, k_top-1] += v[i, j, k] * Δrᶜᶠᶜ(i, j, k, grid) * sᶜᶠ - end - end - -@@ -219,19 +257,55 @@ end - i, j = active_linear_index_to_tuple(idx, active_cells_map) - k_top = grid.Nz+1 - -- @inbounds U[i, j, k_top-1] = Δzᶠᶜᶜ(i, j, 1, grid) * u[i, j, 1] -- @inbounds V[i, j, k_top-1] = Δzᶜᶠᶜ(i, j, 1, grid) * v[i, j, 1] -+ hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) -+ hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) -+ -+ sᶠᶜ = ifelse(hᶠᶜ == 0, one(grid), dynamic_column_depthᶠᶜᵃ(i, j, k_top, grid, η) / hᶠᶜ) -+ sᶜᶠ = ifelse(hᶜᶠ == 0, one(grid), dynamic_column_depthᶜᶠᵃ(i, j, k_top, grid, η) / hᶜᶠ) - -- for k in 2:grid.Nz -- @inbounds U[i, j, k_top-1] += Δzᶠᶜᶜ(i, j, k, grid) * u[i, j, k] -- @inbounds V[i, j, k_top-1] += Δzᶜᶠᶜ(i, j, k, grid) * v[i, j, k] -+ # hand unroll first loop -+ @inbounds U[i, j, k_top-1] = u[i, j, 1] * Δrᶠᶜᶜ(i, j, 1, grid) * sᶠᶜ -+ @inbounds V[i, j, k_top-1] = v[i, j, 1] * Δrᶜᶠᶜ(i, j, 1, grid) * sᶜᶠ -+ -+ @unroll for k in 2:grid.Nz -+ @inbounds U[i, j, k_top-1] += u[i, j, k] * Δrᶠᶜᶜ(i, j, k, grid) * sᶠᶜ -+ @inbounds V[i, j, k_top-1] += v[i, j, k] * Δrᶜᶠᶜ(i, j, k, grid) * sᶜᶠ - end - end - --@inline function compute_barotropic_mode!(U, V, grid, u, v) -+@inline function compute_barotropic_mode!(U, V, grid, u, v, η) - active_cells_map = retrieve_surface_active_cells_map(grid) - -- launch!(architecture(grid), grid, :xy, _barotropic_mode_kernel!, U, V, grid, active_cells_map, u, v; active_cells_map) -+ launch!(architecture(grid), grid, :xy, _barotropic_mode_kernel!, U, V, grid, active_cells_map, u, v, η; active_cells_map) -+ -+ return nothing -+end -+ -+@kernel function _barotropic_split_explicit_corrector!(u, v, grid, U̅, V̅, U, V, η) -+ i, j, k = @index(Global, NTuple) -+ k_top = grid.Nz + 1 -+ -+ Hᶠᶜ = dynamic_column_depthᶠᶜᵃ(i, j, k_top, grid, η) -+ Hᶜᶠ = dynamic_column_depthᶜᶠᵃ(i, j, k_top, grid, η) -+ -+ @inbounds begin -+ u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / Hᶠᶜ -+ v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / Hᶜᶠ -+ end -+end -+ -+function barotropic_split_explicit_corrector!(u, v, free_surface, grid) -+ sefs = free_surface.state -+ U, V, U̅, V̅ = sefs.U, sefs.V, sefs.U̅, sefs.V̅ -+ arch = architecture(grid) -+ -+ # take out "bad" barotropic mode, -+ # !!!! reusing U and V for this storage since last timestep doesn't matter -+ compute_barotropic_mode!(U, V, grid, u, v, free_surface.η) -+ # add in "good" barotropic mode -+ -+ launch!(arch, grid, :xyz, _barotropic_split_explicit_corrector!, -+ u, v, grid, U̅, V̅, U, V, free_surface.η) - - return nothing - end -@@ -266,34 +340,6 @@ function initialize_auxiliary_state!(state, η, timestepper) - return nothing - end - --@kernel function _barotropic_split_explicit_corrector!(u, v, U̅, V̅, U, V, grid) -- i, j, k = @index(Global, NTuple) -- k_top = grid.Nz+1 -- -- @inbounds begin -- Hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) -- Hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) -- -- u[i, j, k] = u[i, j, k] + (U̅[i, j, k_top-1] - U[i, j, k_top-1]) / Hᶠᶜ -- v[i, j, k] = v[i, j, k] + (V̅[i, j, k_top-1] - V[i, j, k_top-1]) / Hᶜᶠ -- end --end -- --function barotropic_split_explicit_corrector!(u, v, free_surface, grid) -- sefs = free_surface.state -- U, V, U̅, V̅ = sefs.U, sefs.V, sefs.U̅, sefs.V̅ -- arch = architecture(grid) -- -- # take out "bad" barotropic mode, -- # !!!! reusing U and V for this storage since last timestep doesn't matter -- compute_barotropic_mode!(U, V, grid, u, v) -- # add in "good" barotropic mode -- launch!(arch, grid, :xyz, _barotropic_split_explicit_corrector!, -- u, v, U̅, V̅, U, V, grid) -- -- return nothing --end -- - """ - Explicitly step forward η in substeps. - """ -@@ -301,8 +347,13 @@ ab2_step_free_surface!(free_surface::SplitExplicitFreeSurface, model, Δt, χ) = - split_explicit_free_surface_step!(free_surface, model, Δt, χ) - - function initialize_free_surface!(sefs::SplitExplicitFreeSurface, grid, velocities) -- @apply_regionally compute_barotropic_mode!(sefs.state.U̅, sefs.state.V̅, grid, velocities.u, velocities.v) -+ U̅, V̅ = sefs.state.U̅, sefs.state.V̅ -+ u, v, _ = velocities -+ -+ @apply_regionally compute_barotropic_mode!(U̅, V̅, grid, u, v, sefs.η) - fill_halo_regions!((sefs.state.U̅, sefs.state.V̅, sefs.η)) -+ -+ return nothing - end - - function split_explicit_free_surface_step!(free_surface::SplitExplicitFreeSurface, model, Δt, χ) -@@ -336,6 +387,10 @@ function split_explicit_free_surface_step!(free_surface::SplitExplicitFreeSurfac - set!(free_surface.η, free_surface.state.η̅) - end - -+ # This is needed for the barotropic mode calculations, so it cannot be done asynchronously -+ fill_halo_regions!(free_surface.η) -+ -+ # Velocities can be passed asynchronously - fields_to_fill = (free_surface.state.U̅, free_surface.state.V̅) - fill_halo_regions!(fields_to_fill; async = true) - -@@ -363,9 +418,6 @@ const MINIMUM_SUBSTEPS = 5 - @inline calculate_adaptive_settings(substepping::FTS, substeps) = weights_from_substeps(eltype(substepping.Δt_barotropic), - substeps, substepping.averaging_kernel) - --const FixedSubstepsSetting{N} = SplitExplicitSettings{<:FixedSubstepNumber{<:Any, <:NTuple{N, <:Any}}} where N --const FixedSubstepsSplitExplicit{F} = SplitExplicitFreeSurface{<:Any, <:Any, <:Any, <:Any, <:FixedSubstepsSetting{N}} where N -- - function iterate_split_explicit!(free_surface, grid, Δτᴮ, weights, ::Val{Nsubsteps}) where Nsubsteps - arch = architecture(grid) - -@@ -424,12 +476,12 @@ end - i, j = @index(Global, NTuple) - k_top = grid.Nz + 1 - -- @inbounds Gᵁ[i, j, k_top-1] = Δzᶠᶜᶜ(i, j, 1, grid) * ab2_step_Gu(i, j, 1, grid, Gu⁻, Guⁿ, χ) -- @inbounds Gⱽ[i, j, k_top-1] = Δzᶜᶠᶜ(i, j, 1, grid) * ab2_step_Gv(i, j, 1, grid, Gv⁻, Gvⁿ, χ) -+ @inbounds Gᵁ[i, j, k_top-1] = Δrᶠᶜᶜ(i, j, 1, grid) * ab2_step_Gu(i, j, 1, grid, Gu⁻, Guⁿ, χ) -+ @inbounds Gⱽ[i, j, k_top-1] = Δrᶜᶠᶜ(i, j, 1, grid) * ab2_step_Gv(i, j, 1, grid, Gv⁻, Gvⁿ, χ) - - for k in 2:grid.Nz -- @inbounds Gᵁ[i, j, k_top-1] += Δzᶠᶜᶜ(i, j, k, grid) * ab2_step_Gu(i, j, k, grid, Gu⁻, Guⁿ, χ) -- @inbounds Gⱽ[i, j, k_top-1] += Δzᶜᶠᶜ(i, j, k, grid) * ab2_step_Gv(i, j, k, grid, Gv⁻, Gvⁿ, χ) -+ @inbounds Gᵁ[i, j, k_top-1] += Δrᶠᶜᶜ(i, j, k, grid) * ab2_step_Gu(i, j, k, grid, Gu⁻, Guⁿ, χ) -+ @inbounds Gⱽ[i, j, k_top-1] += Δrᶜᶠᶜ(i, j, k, grid) * ab2_step_Gv(i, j, k, grid, Gv⁻, Gvⁿ, χ) - end - end - -@@ -439,24 +491,44 @@ end - i, j = active_linear_index_to_tuple(idx, active_cells_map) - k_top = grid.Nz+1 - -- @inbounds Gᵁ[i, j, k_top-1] = Δzᶠᶜᶜ(i, j, 1, grid) * ab2_step_Gu(i, j, 1, grid, Gu⁻, Guⁿ, χ) -- @inbounds Gⱽ[i, j, k_top-1] = Δzᶜᶠᶜ(i, j, 1, grid) * ab2_step_Gv(i, j, 1, grid, Gv⁻, Gvⁿ, χ) -+ @inbounds Gᵁ[i, j, k_top-1] = Δrᶠᶜᶜ(i, j, 1, grid) * ab2_step_Gu(i, j, 1, grid, Gu⁻, Guⁿ, χ) -+ @inbounds Gⱽ[i, j, k_top-1] = Δrᶜᶠᶜ(i, j, 1, grid) * ab2_step_Gv(i, j, 1, grid, Gv⁻, Gvⁿ, χ) - - for k in 2:grid.Nz -- @inbounds Gᵁ[i, j, k_top-1] += Δzᶠᶜᶜ(i, j, k, grid) * ab2_step_Gu(i, j, k, grid, Gu⁻, Guⁿ, χ) -- @inbounds Gⱽ[i, j, k_top-1] += Δzᶜᶠᶜ(i, j, k, grid) * ab2_step_Gv(i, j, k, grid, Gv⁻, Gvⁿ, χ) -+ @inbounds Gᵁ[i, j, k_top-1] += Δrᶠᶜᶜ(i, j, k, grid) * ab2_step_Gu(i, j, k, grid, Gu⁻, Guⁿ, χ) -+ @inbounds Gⱽ[i, j, k_top-1] += Δrᶜᶠᶜ(i, j, k, grid) * ab2_step_Gv(i, j, k, grid, Gv⁻, Gvⁿ, χ) - end - end - --@inline ab2_step_Gu(i, j, k, grid, G⁻, Gⁿ, χ::FT) where FT = -- @inbounds ifelse(peripheral_node(i, j, k, grid, f, c, c), zero(grid), (convert(FT, 1.5) + χ) * Gⁿ[i, j, k] - G⁻[i, j, k] * (convert(FT, 0.5) + χ)) -+@inline function ab2_step_Gu(i, j, k, grid, G⁻, Gⁿ, χ::FT) where FT -+ C¹ = convert(FT, 3/2) + χ -+ C² = convert(FT, 1/2) + χ -+ -+ Fⁿ = @inbounds C¹ * Gⁿ[i, j, k] * vertical_scaling(i, j, k, grid, Face(), Center(), Center()) -+ F⁻ = @inbounds C² * G⁻[i, j, k] * previous_vertical_scaling(i, j, k, grid, Face(), Center(), Center()) -+ -+ Gi = Fⁿ - F⁻ -+ -+ return ifelse(peripheral_node(i, j, k, grid, f, c, c), zero(grid), Gi) -+end -+ -+@inline function ab2_step_Gv(i, j, k, grid, G⁻, Gⁿ, χ::FT) where FT -+ C¹ = convert(FT, 3/2) + χ -+ C² = convert(FT, 1/2) + χ -+ -+ Fⁿ = @inbounds C¹ * Gⁿ[i, j, k] * vertical_scaling(i, j, k, grid, Center(), Face(), Center()) -+ F⁻ = @inbounds C² * G⁻[i, j, k] * previous_vertical_scaling(i, j, k, grid, Center(), Face(), Center()) - --@inline ab2_step_Gv(i, j, k, grid, G⁻, Gⁿ, χ::FT) where FT = -- @inbounds ifelse(peripheral_node(i, j, k, grid, c, f, c), zero(grid), (convert(FT, 1.5) + χ) * Gⁿ[i, j, k] - G⁻[i, j, k] * (convert(FT, 0.5) + χ)) -+ Gi = Fⁿ - F⁻ -+ -+ return ifelse(peripheral_node(i, j, k, grid, f, c, c), zero(grid), Gi) -+end - - # Setting up the RHS for the barotropic step (tendencies of the barotropic velocity components) - # This function is called after `calculate_tendency` and before `ab2_step_velocities!` - function setup_free_surface!(model, free_surface::SplitExplicitFreeSurface, χ) -+ -+ grid = model.grid - - # we start the time integration of η from the average ηⁿ - Gu⁻ = model.timestepper.G⁻.u -@@ -466,7 +538,7 @@ function setup_free_surface!(model, free_surface::SplitExplicitFreeSurface, χ) - - auxiliary = free_surface.auxiliary - -- @apply_regionally setup_split_explicit_tendency!(auxiliary, model.grid, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) -+ @apply_regionally setup_split_explicit_tendency!(auxiliary, grid, Gu⁻, Gv⁻, Guⁿ, Gvⁿ, χ) - - fields_to_fill = (auxiliary.Gᵁ, auxiliary.Gⱽ) - fill_halo_regions!(fields_to_fill; async = true) -@@ -483,5 +555,4 @@ end - return nothing - end - --wait_free_surface_communication!(free_surface, arch) = nothing -- -+wait_free_surface_communication!(free_surface, arch) = nothing -\ No newline at end of file diff --git a/difference_se b/difference_se deleted file mode 100644 index 75b7e8da6d..0000000000 --- a/difference_se +++ /dev/null @@ -1,126 +0,0 @@ -diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl -index 073fbfc4e..6992a410d 100644 ---- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl -+++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl -@@ -3,8 +3,9 @@ using Oceananigans.Architectures - using Oceananigans.Fields - using Oceananigans.Grids - using Oceananigans.Grids: AbstractGrid -+using Oceananigans.BoundaryConditions: default_prognostic_bc - using Oceananigans.AbstractOperations: Δz, GridMetricOperation -- -+using Oceananigans.ImmersedBoundaries: immersed_peripheral_node, c, f - using Adapt - using Base - using KernelAbstractions: @index, @kernel -@@ -158,6 +159,10 @@ Base.@kwdef struct SplitExplicitState{CC, ACC, FC, AFC, CF, ACF} - U̅ :: FC - "The time-filtered barotropic meridional velocity. (`ReducedField` over ``z``)" - V̅ :: CF -+ # "The time-filtered barotropic zonal velocity. (`ReducedField` over ``z``)" -+ # Ũ :: FC -+ # "The time-filtered barotropic meridional velocity. (`ReducedField` over ``z``)" -+ # Ṽ :: CF - end - - """ -@@ -179,18 +184,20 @@ function SplitExplicitState(grid::AbstractGrid, timestepper) - ηᵐ⁻¹ = auxiliary_free_surface_field(grid, timestepper) - ηᵐ⁻² = auxiliary_free_surface_field(grid, timestepper) - -- U = XFaceField(grid, indices = (:, :, Nz)) -- V = YFaceField(grid, indices = (:, :, Nz)) -- -- Uᵐ⁻¹ = auxiliary_barotropic_U_field(grid, timestepper) -- Vᵐ⁻¹ = auxiliary_barotropic_V_field(grid, timestepper) -- Uᵐ⁻² = auxiliary_barotropic_U_field(grid, timestepper) -- Vᵐ⁻² = auxiliary_barotropic_V_field(grid, timestepper) -+ 𝒰 = VelocityFields(grid) -+ -+ U = Field(𝒰.u, indices = (:, :, Nz)) -+ V = Field(𝒰.v, indices = (:, :, Nz)) - -- U̅ = XFaceField(grid, indices = (:, :, Nz)) -- V̅ = YFaceField(grid, indices = (:, :, Nz)) -+ Uᵐ⁻¹ = auxiliary_barotropic_velocity_field(U, timestepper) -+ Vᵐ⁻¹ = auxiliary_barotropic_velocity_field(V, timestepper) -+ Uᵐ⁻² = auxiliary_barotropic_velocity_field(U, timestepper) -+ Vᵐ⁻² = auxiliary_barotropic_velocity_field(V, timestepper) -+ -+ U̅ = deepcopy(U) -+ V̅ = deepcopy(V) - -- return SplitExplicitState(; ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅) -+ return SplitExplicitState(; ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅) - end - - """ -@@ -204,7 +211,7 @@ large (or `:xy` in case of a serial computation), and start computing from - - $(FIELDS) - """ --Base.@kwdef struct SplitExplicitAuxiliaryFields{𝒞ℱ, ℱ𝒞, 𝒦} -+Base.@kwdef struct SplitExplicitAuxiliaryFields{ℱ𝒞, 𝒞ℱ, 𝒦} - "Vertically-integrated slow barotropic forcing function for `U` (`ReducedField` over ``z``)" - Gᵁ :: ℱ𝒞 - "Vertically-integrated slow barotropic forcing function for `V` (`ReducedField` over ``z``)" -@@ -220,8 +227,8 @@ Return the `SplitExplicitAuxiliaryFields` for `grid`. - """ - function SplitExplicitAuxiliaryFields(grid::AbstractGrid) - -- Gᵁ = Field((Face, Center, Nothing), grid) -- Gⱽ = Field((Center, Face, Nothing), grid) -+ Gᵁ = Field{Face, Center, Nothing}(grid) -+ Gⱽ = Field{Center, Face, Nothing}(grid) - - kernel_parameters = :xy - -@@ -244,14 +251,11 @@ end - struct AdamsBashforth3Scheme end - struct ForwardBackwardScheme end - -- - auxiliary_free_surface_field(grid, ::AdamsBashforth3Scheme) = ZFaceField(grid, indices = (:, :, size(grid, 3)+1)) - auxiliary_free_surface_field(grid, ::ForwardBackwardScheme) = nothing - --auxiliary_barotropic_U_field(grid, ::AdamsBashforth3Scheme) = XFaceField(grid, indices = (:, :, size(grid, 3))) --auxiliary_barotropic_U_field(grid, ::ForwardBackwardScheme) = nothing --auxiliary_barotropic_V_field(grid, ::AdamsBashforth3Scheme) = YFaceField(grid, indices = (:, :, size(grid, 3))) --auxiliary_barotropic_V_field(grid, ::ForwardBackwardScheme) = nothing -+auxiliary_barotropic_velocity_field(U, ::AdamsBashforth3Scheme) = deecopy(u) -+auxiliary_barotropic_velocity_field(U, ::ForwardBackwardScheme) = nothing - - # (p = 2, q = 4, r = 0.18927) minimize dispersion error from Shchepetkin and McWilliams (2005): https://doi.org/10.1016/j.ocemod.2004.08.002 - @inline function averaging_shape_function(τ::FT; p = 2, q = 4, r = FT(0.18927)) where FT -@@ -306,7 +310,7 @@ end - - averaging_weights = averaging_weights[1:idx] - averaging_weights ./= sum(averaging_weights) -- -+ - return Δτ, tuple(averaging_weights...) - end - -@@ -390,11 +394,19 @@ end - - # Adapt - Adapt.adapt_structure(to, free_surface::SplitExplicitFreeSurface) = -- SplitExplicitFreeSurface(Adapt.adapt(to, free_surface.η), nothing, nothing, -+ SplitExplicitFreeSurface(Adapt.adapt(to, free_surface.η), -+ nothing, -+ Adapt.adapt(to, free_surface.auxiliary), - free_surface.gravitational_acceleration, nothing) - --for Type in (:SplitExplicitFreeSurface, -- :SplitExplicitSettings, -+# Adapt -+Adapt.adapt_structure(to, auxiliary::SplitExplicitAuxiliaryFields) = -+ SplitExplicitAuxiliaryFields(Adapt.adapt(to, auxiliary.Gᵁ), -+ Adapt.adapt(to, auxiliary.Gⱽ), -+ nothing) -+ -+for Type in (:SplitExplicitFreeSurface, -+ :SplitExplicitSettings, - :SplitExplicitState, - :SplitExplicitAuxiliaryFields, - :FixedTimeStepSize, diff --git a/mynew_model.jl b/mynew_model.jl deleted file mode 100644 index a9467b4cb0..0000000000 --- a/mynew_model.jl +++ /dev/null @@ -1,139 +0,0 @@ -using Oceananigans -using Oceananigans: tupleit -using Oceananigans.Grids -using Oceananigans.Architectures: architecture -using Oceananigans.Advection: div_Uc -using Oceananigans.BoundaryConditions -using Oceananigans.Utils -using Oceananigans.Fields: TracerFields -using Oceananigans.Models.HydrostaticFreeSurfaceModels: PrescribedVelocityFields, tracernames, validate_tracer_advection, with_tracers, tracernames -using KernelAbstractions: @index, @kernel - -import Oceananigans.Models: prognostic_fields - -using Oceananigans.TimeSteppers: RungeKutta3TimeStepper, store_tendencies!, rk3_substep_field! -import Oceananigans.TimeSteppers: update_state!, time_step!, compute_tendencies!, rk3_substep! - -mutable struct AdvectiveModel{G, A, V, T, B} - grid :: G - advection :: A - velocities :: V - tracers :: T - timestepper :: B -end - -struct DirectSpaceTimeAder end - -function AdvectiveModel(; grid, - velocities = PrescribedVelocityFields(; u = 1), - tracers = :b, - timestepper = nothing, - advection = WENO(; order = 5)) - - # Next, we form a list of default boundary conditions: - tracers = tupleit(tracers) # supports tracers=:c keyword argument (for example) - - tracers = TracerFields(tracers, grid, NamedTuple()) - Gⁿ = deepcopy(tracers) - G⁻ = deepcopy(tracers) - - if timestepper isa Nothing - timestepper = DirectSpaceTime() - else - timestepper = RungeKutta3TimeStepper(grid, tracernames(tracers); Gⁿ = Gⁿ, G⁻ = G⁻) - end - - model = AdvectiveModel(grid, advection, velocities, tracers, timestepper) - update_state!(model) - - return model -end - -prognostic_fields(model::AdvectiveModel) = model.tracers - -update_state!(model::AdvectiveModel) = - fill_halo_regions!(model.tracers) - -function time_step!(model::AdvectiveModel, Δt) - Δt == 0 && @warn "Δt == 0 may cause model blowup!" - - # Be paranoid and update state at iteration 0, in case run! is not used: - update_state!(model) - - γ¹ = model.timestepper.γ¹ - γ² = model.timestepper.γ² - γ³ = model.timestepper.γ³ - - ζ² = model.timestepper.ζ² - ζ³ = model.timestepper.ζ³ - - compute_tendencies!(model) - rk3_substep!(model, Δt, γ¹, nothing) - store_tendencies!(model) - update_state!(model) - - # - # Second stage - # - - compute_tendencies!(model) - rk3_substep!(model, Δt, γ², ζ²) - store_tendencies!(model) - update_state!(model) - - # - # Third stage - # - - compute_tendencies!(model) - rk3_substep!(model, Δt, γ³, ζ³) - update_state!(model) - - return nothing -end - -function rk3_substep!(model::AdvectiveModel, Δt, γ, ζ) - grid = model.grid - arch = architecture(grid) - tracers = model.tracers - velocities = model.velocities - - for (tracer, Gⁿ, G⁻) in zip(tracers, model.timestepper.Gⁿ, model.timestepper.G⁻) - launch!(arch, grid, :xyz, rk3_substep_field!, tracer, Δt, γ, ζ, Gⁿ, G⁻) - end - - return nothing -end - -function compute_tendencies!(model::AdvectiveModel) - grid = model.grid - arch = architecture(grid) - tendencies = model.timestepper.Gⁿ - tracers = model.tracers - velocities = model.velocities - - for (tracer, G) in zip(tracers, tendencies) - launch!(arch, grid, :xyz, _compute_tendency!, G, grid, model.advection, velocities, tracer) - end - - return nothing -end - -@kernel function _compute_tendency!(G, grid, advection, U, c) - i, j, k = @index(Global, NTuple) - @inbounds G[i, j, k] = - div_Uc(i, j, k, grid, advection, U, c) -end - -function time_step!(model::AdvectiveModel{G, A, V, <:DirectSpaceTimeAder, B}, Δt) where {G, A, V, B} - Δt == 0 && @warn "Δt == 0 may cause model blowup!" - - # Be paranoid and update state at iteration 0, in case run! is not used: - update_state!(model) - - compute_tendencies!(model) - rk3_substep!(model, Δt, γ¹, nothing) - store_tendencies!(model) - update_state!(model) - - return nothing -end From c2102dc54d2314b7f6a72de897ee70107832f05d Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 23 Oct 2024 17:12:18 +0200 Subject: [PATCH 414/567] nothing was happening --- test/test_zstar_coordinate.jl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index 885e267ae4..faa802f898 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -15,6 +15,7 @@ function test_zstar_coordinate(model, Ni, Δt) # (2) vertical velocities are zero at the top surface for _ in 1:Ni time_step!(model, Δt) + @show extrema(interior(model.velocities.w, :, :, model.grid.Nz+1)) end ∫b = Field(Integral(model.tracers.b)) @@ -49,8 +50,8 @@ end for topology in topologies Random.seed!(1234) - rtg = RectilinearGrid(arch; size = (10, 10, 10), x = (-10, 10), y = (-10, 10), topology, z = z_uniform) - rtgv = RectilinearGrid(arch; size = (10, 10, 10), x = (-10, 10), y = (-10, 10), topology, z = z_stretched) + rtg = RectilinearGrid(arch; size = (10, 10, 10), x = (0, 20), y = (-10, 10), topology, z = z_uniform) + rtgv = RectilinearGrid(arch; size = (10, 10, 10), x = (0, 20), y = (-10, 10), topology, z = z_stretched) irtg = ImmersedBoundaryGrid(rtg, GridFittedBottom((x, y) -> - rand() - 5)) irtgv = ImmersedBoundaryGrid(rtgv, GridFittedBottom((x, y) -> - rand() - 5)) @@ -58,8 +59,8 @@ end prtgv = ImmersedBoundaryGrid(rtgv, PartialCellBottom((x, y) -> - rand() - 5)) if topology[2] == Bounded - llg = LatitudeLongitudeGrid(arch; size = (10, 10, 10), latitude = (-10, 10), longitude = (-180, 180), topology, z = z_uniform) - llgv = LatitudeLongitudeGrid(arch; size = (10, 10, 10), latitude = (-10, 10), longitude = (-180, 180), topology, z = z_stretched) + llg = LatitudeLongitudeGrid(arch; size = (10, 10, 10), latitude = (0, 20), longitude = (-180, 180), topology, z = z_uniform) + llgv = LatitudeLongitudeGrid(arch; size = (10, 10, 10), latitude = (0, 20), longitude = (-180, 180), topology, z = z_stretched) illg = ImmersedBoundaryGrid(llg, GridFittedBottom((x, y) -> - rand() - 5)) illgv = ImmersedBoundaryGrid(llgv, GridFittedBottom((x, y) -> - rand() - 5)) From 6e560a41456fba64018d2caaec32642ad4ea02ab Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 24 Oct 2024 08:56:16 +0200 Subject: [PATCH 415/567] correct coordinate test --- test/test_zstar_coordinate.jl | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index faa802f898..8b7dfcdfb3 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -30,17 +30,18 @@ end function info_message(grid) msg1 = "Testing z-star coordinates on $(architecture(grid)) on a " - msg2 = string(getnamewrapper(grid)) - msg3 = grid.Δzᵃᵃᶠ.reference isa Number ? " with uniform spacing" : " with stretched spacing" - msg4 = grid isa ImmersedBoundaryGrid ? " and $(string(getnamewrapper(grid.immersed_boundary))) immersed boundary" : "" + msg2 = string(getnamewrapper(grid)) + msg3 = grid isa ImmersedBoundaryGrid ? " on a " * string(getnamewrapper(grid.underlying_grid)) : "" + msg4 = grid.Δzᵃᵃᶠ.reference isa Number ? " with uniform spacing" : " with stretched spacing" + msg5 = grid isa ImmersedBoundaryGrid ? " and $(string(getnamewrapper(grid.immersed_boundary))) immersed boundary" : "" - return msg1 * msg2 * msg3 * msg4 + return msg1 * msg2 * msg3 * msg4 * msg5 end @testset "ZStar coordinate testset" begin - z_uniform = ZStarVerticalCoordinate(-10, 0) - z_stretched = ZStarVerticalCoordinate(collect(-10:0)) + z_uniform = ZStarVerticalCoordinate(-20, 0) + z_stretched = ZStarVerticalCoordinate(collect(-20:0)) topologies = ((Periodic, Periodic, Bounded), (Periodic, Bounded, Bounded), (Bounded, Periodic, Bounded), @@ -50,17 +51,17 @@ end for topology in topologies Random.seed!(1234) - rtg = RectilinearGrid(arch; size = (10, 10, 10), x = (0, 20), y = (-10, 10), topology, z = z_uniform) - rtgv = RectilinearGrid(arch; size = (10, 10, 10), x = (0, 20), y = (-10, 10), topology, z = z_stretched) + rtg = RectilinearGrid(arch; size = (10, 10, 10), x = (0, 100kilometers), y = (-10kilometers, 10kilometers), topology, z = z_uniform) + rtgv = RectilinearGrid(arch; size = (10, 10, 10), x = (0, 100kilometers), y = (-10kilometers, 10kilometers), topology, z = z_stretched) - irtg = ImmersedBoundaryGrid(rtg, GridFittedBottom((x, y) -> - rand() - 5)) - irtgv = ImmersedBoundaryGrid(rtgv, GridFittedBottom((x, y) -> - rand() - 5)) - prtg = ImmersedBoundaryGrid(rtg, PartialCellBottom((x, y) -> - rand() - 5)) - prtgv = ImmersedBoundaryGrid(rtgv, PartialCellBottom((x, y) -> - rand() - 5)) + irtg = ImmersedBoundaryGrid(rtg, GridFittedBottom((x, y) -> - rand() - 10)) + irtgv = ImmersedBoundaryGrid(rtgv, GridFittedBottom((x, y) -> - rand() - 10)) + prtg = ImmersedBoundaryGrid(rtg, PartialCellBottom((x, y) -> - rand() - 10)) + prtgv = ImmersedBoundaryGrid(rtgv, PartialCellBottom((x, y) -> - rand() - 10)) if topology[2] == Bounded - llg = LatitudeLongitudeGrid(arch; size = (10, 10, 10), latitude = (0, 20), longitude = (-180, 180), topology, z = z_uniform) - llgv = LatitudeLongitudeGrid(arch; size = (10, 10, 10), latitude = (0, 20), longitude = (-180, 180), topology, z = z_stretched) + llg = LatitudeLongitudeGrid(arch; size = (10, 10, 10), latitude = (0, 1), longitude = (0, 1), topology, z = z_uniform) + llgv = LatitudeLongitudeGrid(arch; size = (10, 10, 10), latitude = (0, 1), longitude = (0, 1), topology, z = z_stretched) illg = ImmersedBoundaryGrid(llg, GridFittedBottom((x, y) -> - rand() - 5)) illgv = ImmersedBoundaryGrid(llgv, GridFittedBottom((x, y) -> - rand() - 5)) @@ -77,7 +78,7 @@ end # TODO: minimum_xspacing(grid) on a Immersed GPU grid with ZStarVerticalCoordinate # fails because it uses too much parameter space. Figure out a way to reduce it - free_surface = SplitExplicitFreeSurface(grid; substeps = 10) + free_surface = SplitExplicitFreeSurface(grid; substeps = 20) model = HydrostaticFreeSurfaceModel(; grid, free_surface, tracers = (:b, :c), From fe2d552fc5216f0565c9e0c4b7eeadc25c6be49c Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 28 Oct 2024 18:45:54 +0100 Subject: [PATCH 416/567] new operators --- .../split_explicit_free_surface_kernels.jl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 4bf9428d72..0042755123 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -83,7 +83,6 @@ end @inline function free_surface_evolution!(i, j, grid, Δτ, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, V, Uᵐ⁻¹, Uᵐ⁻², Vᵐ⁻¹, Vᵐ⁻², timestepper) k_top = grid.Nz+1 - TX, TY, _ = topology(grid) @inbounds begin advance_previous_free_surface!(i, j, k_top, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) @@ -122,8 +121,8 @@ end Hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) # ∂τ(U) = - ∇η + G - U[i, j, k_top-1] += Δτ * (- g * Hᶠᶜ * ∂xᶠᶜᶠ_η(i, j, k_top, grid, TX, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gᵁ[i, j, k_top-1]) - V[i, j, k_top-1] += Δτ * (- g * Hᶜᶠ * ∂yᶜᶠᶠ_η(i, j, k_top, grid, TY, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gⱽ[i, j, k_top-1]) + U[i, j, k_top-1] += Δτ * (- g * Hᶠᶜ * ∂xTᶠᶜᶠ(i, j, k_top, grid, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gᵁ[i, j, k_top-1]) + V[i, j, k_top-1] += Δτ * (- g * Hᶜᶠ * ∂yTᶜᶠᶠ(i, j, k_top, grid, η★, timestepper, η, ηᵐ, ηᵐ⁻¹, ηᵐ⁻²) + Gⱽ[i, j, k_top-1]) # time-averaging η̅[i, j, k_top] += averaging_weight * η[i, j, k_top] From 5227a1369321791687e35ce9925f33d0a9643075 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 29 Oct 2024 08:46:15 +0100 Subject: [PATCH 417/567] fix pipeline --- .buildkite/distributed/pipeline.yml | 18 +++++++++++++++--- test/utils_for_runtests.jl | 2 +- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index fe38f7bea1..2a0f00841a 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -17,10 +17,18 @@ steps: env: TEST_GROUP: "init" command: - - echo "--- Instantiate project" - - "julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" + - "echo '--- Instantiate project'" + - "julia --project -e 'using Pkg; Pkg.instantiate(verbose=true)'" + + - "echo '--- Precompile project'" + - "julia --project -e 'using Pkg; Pkg.precompile(strict=true)'" + - "julia --project -e 'using Pkg; Pkg.status()'" + + # Force the initialization of the CUDA runtime as it is lazily loaded by default: + - "echo '--- Initialize the CUDA runtime'" + - "julia --project -e 'using CUDA; CUDA.precompile_runtime()'" + - "julia --project -e 'using Pkg; Pkg.test()'" agents: - slurm_mem: 120G slurm_gpus: 1 slurm_cpus_per_task: 8 @@ -40,6 +48,7 @@ steps: key: "distributed_gpu" env: TEST_GROUP: "distributed" + GPU_TEST: "true" commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -66,6 +75,7 @@ steps: key: "distributed_solvers_gpu" env: TEST_GROUP: "distributed_solvers" + GPU_TEST: "true" commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -91,6 +101,7 @@ steps: key: "distributed_hydrostatic_model_gpu" env: TEST_GROUP: "distributed_hydrostatic_model" + GPU_TEST: "true" commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -116,6 +127,7 @@ steps: key: "distributed_nonhydrostatic_regression_gpu" env: TEST_GROUP: "distributed_nonhydrostatic_regression" + GPU_TEST: "true" commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: diff --git a/test/utils_for_runtests.jl b/test/utils_for_runtests.jl index 7bc06e528e..298f68db16 100644 --- a/test/utils_for_runtests.jl +++ b/test/utils_for_runtests.jl @@ -3,7 +3,7 @@ using Oceananigans.DistributedComputations: Distributed, Partition, child_archit import Oceananigans.Fields: interior -test_child_arch() = CUDA.has_cuda() ? GPU() : CPU() +test_child_arch() = parse(Bool, get(ENV, "GPU_TEST", "false")) ? GPU() : CPU() function test_architectures() child_arch = test_child_arch() From 1225061aa8fa9d20fe851e4de804ea30b4f07567 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 29 Oct 2024 08:53:10 +0100 Subject: [PATCH 418/567] mpi test and gpu test --- .buildkite/distributed/pipeline.yml | 8 ++++++++ test/utils_for_runtests.jl | 31 +++++++++++++++++++---------- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 2a0f00841a..3ebd4de52c 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -38,6 +38,7 @@ steps: key: "distributed_cpu" env: TEST_GROUP: "distributed" + MPI_TEST: "true" commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -49,6 +50,7 @@ steps: env: TEST_GROUP: "distributed" GPU_TEST: "true" + MPI_TEST: "true" commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -65,6 +67,7 @@ steps: key: "distributed_solvers_cpu" env: TEST_GROUP: "distributed_solvers" + MPI_TEST: "true" commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -76,6 +79,7 @@ steps: env: TEST_GROUP: "distributed_solvers" GPU_TEST: "true" + MPI_TEST: "true" commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -91,6 +95,7 @@ steps: key: "distributed_hydrostatic_model_cpu" env: TEST_GROUP: "distributed_hydrostatic_model" + MPI_TEST: "true" commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -102,6 +107,7 @@ steps: env: TEST_GROUP: "distributed_hydrostatic_model" GPU_TEST: "true" + MPI_TEST: "true" commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -117,6 +123,7 @@ steps: key: "distributed_nonhydrostatic_regression_cpu" env: TEST_GROUP: "distributed_nonhydrostatic_regression" + MPI_TEST: "true" commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: @@ -128,6 +135,7 @@ steps: env: TEST_GROUP: "distributed_nonhydrostatic_regression" GPU_TEST: "true" + MPI_TEST: "true" commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: diff --git a/test/utils_for_runtests.jl b/test/utils_for_runtests.jl index 298f68db16..e8741e4839 100644 --- a/test/utils_for_runtests.jl +++ b/test/utils_for_runtests.jl @@ -4,6 +4,7 @@ using Oceananigans.DistributedComputations: Distributed, Partition, child_archit import Oceananigans.Fields: interior test_child_arch() = parse(Bool, get(ENV, "GPU_TEST", "false")) ? GPU() : CPU() +mpi_test() = parse(Bool, get(ENV, "MPI_TEST", "false")) function test_architectures() child_arch = test_child_arch() @@ -11,13 +12,17 @@ function test_architectures() # If MPI is initialized with MPI.Comm_size > 0, we are running in parallel. # We test several different configurations: `Partition(x = 4)`, `Partition(y = 4)`, # `Partition(x = 2, y = 2)`, and different fractional subdivisions in x, y and xy - if MPI.Initialized() && MPI.Comm_size(MPI.COMM_WORLD) == 4 - return (Distributed(child_arch; partition = Partition(4)), - Distributed(child_arch; partition = Partition(1, 4)), - Distributed(child_arch; partition = Partition(2, 2)), - Distributed(child_arch; partition = Partition(x = Fractional(1, 2, 3, 4))), - Distributed(child_arch; partition = Partition(y = Fractional(1, 2, 3, 4))), - Distributed(child_arch; partition = Partition(x = Fractional(1, 2), y = Equal()))) + if mpi_test() + if MPI.Initialized() && MPI.Comm_size(MPI.COMM_WORLD) == 4 + return (Distributed(child_arch; partition = Partition(4)), + Distributed(child_arch; partition = Partition(1, 4)), + Distributed(child_arch; partition = Partition(2, 2)), + Distributed(child_arch; partition = Partition(x = Fractional(1, 2, 3, 4))), + Distributed(child_arch; partition = Partition(y = Fractional(1, 2, 3, 4))), + Distributed(child_arch; partition = Partition(x = Fractional(1, 2), y = Equal()))) + else + return throw("The MPI partitioning is not correctly configured.") + end else return tuple(child_arch) end @@ -31,10 +36,14 @@ function nonhydrostatic_regression_test_architectures() # If MPI is initialized with MPI.Comm_size > 0, we are running in parallel. # We test 3 different configurations: `Partition(x = 4)`, `Partition(y = 4)` # and `Partition(x = 2, y = 2)` - if MPI.Initialized() && MPI.Comm_size(MPI.COMM_WORLD) == 4 - return (Distributed(child_arch; partition = Partition(4)), - Distributed(child_arch; partition = Partition(1, 4)), - Distributed(child_arch; partition = Partition(2, 2))) + if mpi_test() + if MPI.Initialized() && MPI.Comm_size(MPI.COMM_WORLD) == 4 + return (Distributed(child_arch; partition = Partition(4)), + Distributed(child_arch; partition = Partition(1, 4)), + Distributed(child_arch; partition = Partition(2, 2))) + else + return throw("The MPI partitioning is not correctly configured.") + end else return tuple(child_arch) end From 1652c6bd8948c7d53e7ea580068395769716f6ff Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 29 Oct 2024 11:22:17 +0100 Subject: [PATCH 419/567] do we need to precompile it inside? --- .buildkite/distributed/pipeline.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 3ebd4de52c..eb9b58bd5a 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -52,6 +52,7 @@ steps: GPU_TEST: "true" MPI_TEST: "true" commands: + - "julia -O0 --color=yes --project -e 'using CUDA; CUDA.precompile_runtime()'" - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: slurm_mem: 120G From 93232037fc96a05f92b7294c8c0d843fe6442b49 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 29 Oct 2024 11:49:55 +0100 Subject: [PATCH 420/567] precompile inside the node --- .buildkite/distributed/pipeline.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index eb9b58bd5a..77731654a9 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -18,18 +18,20 @@ steps: TEST_GROUP: "init" command: - "echo '--- Instantiate project'" - - "julia --project -e 'using Pkg; Pkg.instantiate(verbose=true)'" + - "srun julia --project -e 'using Pkg; Pkg.instantiate(verbose=true)'" - "echo '--- Precompile project'" - - "julia --project -e 'using Pkg; Pkg.precompile(strict=true)'" - - "julia --project -e 'using Pkg; Pkg.status()'" + - "srun julia --project -e 'using Pkg; Pkg.precompile(strict=true)'" + - "srun julia --project -e 'using Pkg; Pkg.status()'" # Force the initialization of the CUDA runtime as it is lazily loaded by default: - "echo '--- Initialize the CUDA runtime'" - - "julia --project -e 'using CUDA; CUDA.precompile_runtime()'" - - "julia --project -e 'using Pkg; Pkg.test()'" + - "srun julia --project -e 'using CUDA; CUDA.precompile_runtime()'" + - "srun julia --project -e 'using Pkg; Pkg.test()'" agents: + slurm_ntasks: 1 slurm_gpus: 1 + slurm_gpus_per_task: 1 slurm_cpus_per_task: 8 - wait @@ -52,7 +54,6 @@ steps: GPU_TEST: "true" MPI_TEST: "true" commands: - - "julia -O0 --color=yes --project -e 'using CUDA; CUDA.precompile_runtime()'" - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: slurm_mem: 120G From 37b17ff714adab1d76b434e8619a2e035d4da42c Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 29 Oct 2024 11:51:39 +0100 Subject: [PATCH 421/567] try previous climacommon version --- .buildkite/distributed/pipeline.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 77731654a9..5187915441 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -1,7 +1,7 @@ agents: queue: new-central slurm_mem: 8G - modules: climacommon/2024_10_09 + modules: climacommon/2024_05_27 env: JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite/distributed" @@ -18,20 +18,18 @@ steps: TEST_GROUP: "init" command: - "echo '--- Instantiate project'" - - "srun julia --project -e 'using Pkg; Pkg.instantiate(verbose=true)'" + - "julia --project -e 'using Pkg; Pkg.instantiate(verbose=true)'" - "echo '--- Precompile project'" - - "srun julia --project -e 'using Pkg; Pkg.precompile(strict=true)'" - - "srun julia --project -e 'using Pkg; Pkg.status()'" + - "julia --project -e 'using Pkg; Pkg.precompile(strict=true)'" + - "julia --project -e 'using Pkg; Pkg.status()'" # Force the initialization of the CUDA runtime as it is lazily loaded by default: - "echo '--- Initialize the CUDA runtime'" - - "srun julia --project -e 'using CUDA; CUDA.precompile_runtime()'" - - "srun julia --project -e 'using Pkg; Pkg.test()'" + - "julia --project -e 'using CUDA; CUDA.precompile_runtime()'" + - "julia --project -e 'using Pkg; Pkg.test()'" agents: - slurm_ntasks: 1 slurm_gpus: 1 - slurm_gpus_per_task: 1 slurm_cpus_per_task: 8 - wait From 2ac8cdedc51183633fd174f3c2cdb0c77c428ce9 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 29 Oct 2024 12:33:30 +0100 Subject: [PATCH 422/567] go even more back --- .buildkite/distributed/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 5187915441..c01f6560e5 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -1,7 +1,7 @@ agents: queue: new-central slurm_mem: 8G - modules: climacommon/2024_05_27 + modules: climacommon/2024_02_27 env: JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite/distributed" From 0eb2720e2822234f6da1b75c89b593a5fb5fa59b Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 29 Oct 2024 13:19:05 +0100 Subject: [PATCH 423/567] use the ClimaOcean implementation --- .buildkite/distributed/pipeline.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index c01f6560e5..7787aa351e 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -1,19 +1,18 @@ agents: queue: new-central slurm_mem: 8G - modules: climacommon/2024_02_27 + modules: climacommon/2024_05_27 + +timeout_in_minutes: 1440 env: - JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite/distributed" OPENBLAS_NUM_THREADS: 1 - JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager - JULIA_NUM_PRECOMPILE_TASKS: 8 - JULIA_NUM_THREADS: 8 OMPI_MCA_opal_warn_on_missing_libcuda: 0 + JULIA_NUM_PRECOMPILE_TASKS: 8 steps: - label: "initialize" - key: "init_central" + key: "init" env: TEST_GROUP: "init" command: From 50d0ec3c08c6a8e2c2ea2d7d0728773bbe0dd34f Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 29 Oct 2024 13:19:20 +0100 Subject: [PATCH 424/567] using the ClimaOcean implementation --- .buildkite/distributed/pipeline.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 7787aa351e..20753bc6f1 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -3,8 +3,6 @@ agents: slurm_mem: 8G modules: climacommon/2024_05_27 -timeout_in_minutes: 1440 - env: OPENBLAS_NUM_THREADS: 1 OMPI_MCA_opal_warn_on_missing_libcuda: 0 From 070c1791735d36ba6879d395bb4b286eef343805 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 29 Oct 2024 17:16:38 +0100 Subject: [PATCH 425/567] try removing boundary conditoin on barotropic velocity --- .../split_explicit_free_surface.jl | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index 6992a410de..8e7af56f3f 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -159,10 +159,6 @@ Base.@kwdef struct SplitExplicitState{CC, ACC, FC, AFC, CF, ACF} U̅ :: FC "The time-filtered barotropic meridional velocity. (`ReducedField` over ``z``)" V̅ :: CF - # "The time-filtered barotropic zonal velocity. (`ReducedField` over ``z``)" - # Ũ :: FC - # "The time-filtered barotropic meridional velocity. (`ReducedField` over ``z``)" - # Ṽ :: CF end """ @@ -186,8 +182,8 @@ function SplitExplicitState(grid::AbstractGrid, timestepper) 𝒰 = VelocityFields(grid) - U = Field(𝒰.u, indices = (:, :, Nz)) - V = Field(𝒰.v, indices = (:, :, Nz)) + U = XFaceField(grid, indices = (:, :, Nz)) + V = YFaceField(grid, indices = (:, :, Nz)) Uᵐ⁻¹ = auxiliary_barotropic_velocity_field(U, timestepper) Vᵐ⁻¹ = auxiliary_barotropic_velocity_field(V, timestepper) From 8feff8bc0b4d7cb6c76e83aa0b708bcf0bfcc608 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 29 Oct 2024 18:16:30 +0100 Subject: [PATCH 426/567] found the bug --- .../split_explicit_free_surface.jl | 28 +++++++------------ .../split_explicit_free_surface_kernels.jl | 2 +- 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl index 8e7af56f3f..901b28b4e9 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface.jl @@ -3,9 +3,8 @@ using Oceananigans.Architectures using Oceananigans.Fields using Oceananigans.Grids using Oceananigans.Grids: AbstractGrid -using Oceananigans.BoundaryConditions: default_prognostic_bc using Oceananigans.AbstractOperations: Δz, GridMetricOperation -using Oceananigans.ImmersedBoundaries: immersed_peripheral_node, c, f + using Adapt using Base using KernelAbstractions: @index, @kernel @@ -182,8 +181,8 @@ function SplitExplicitState(grid::AbstractGrid, timestepper) 𝒰 = VelocityFields(grid) - U = XFaceField(grid, indices = (:, :, Nz)) - V = YFaceField(grid, indices = (:, :, Nz)) + U = Field(𝒰.u, indices = (:, :, Nz)) + V = Field(𝒰.v, indices = (:, :, Nz)) Uᵐ⁻¹ = auxiliary_barotropic_velocity_field(U, timestepper) Vᵐ⁻¹ = auxiliary_barotropic_velocity_field(V, timestepper) @@ -193,7 +192,7 @@ function SplitExplicitState(grid::AbstractGrid, timestepper) U̅ = deepcopy(U) V̅ = deepcopy(V) - return SplitExplicitState(; ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅) + return SplitExplicitState(; ηᵐ, ηᵐ⁻¹, ηᵐ⁻², U, Uᵐ⁻¹, Uᵐ⁻², V, Vᵐ⁻¹, Vᵐ⁻², η̅, U̅, V̅) end """ @@ -207,7 +206,7 @@ large (or `:xy` in case of a serial computation), and start computing from $(FIELDS) """ -Base.@kwdef struct SplitExplicitAuxiliaryFields{ℱ𝒞, 𝒞ℱ, 𝒦} +Base.@kwdef struct SplitExplicitAuxiliaryFields{𝒞ℱ, ℱ𝒞, 𝒦} "Vertically-integrated slow barotropic forcing function for `U` (`ReducedField` over ``z``)" Gᵁ :: ℱ𝒞 "Vertically-integrated slow barotropic forcing function for `V` (`ReducedField` over ``z``)" @@ -247,6 +246,7 @@ end struct AdamsBashforth3Scheme end struct ForwardBackwardScheme end + auxiliary_free_surface_field(grid, ::AdamsBashforth3Scheme) = ZFaceField(grid, indices = (:, :, size(grid, 3)+1)) auxiliary_free_surface_field(grid, ::ForwardBackwardScheme) = nothing @@ -306,7 +306,7 @@ end averaging_weights = averaging_weights[1:idx] averaging_weights ./= sum(averaging_weights) - + return Δτ, tuple(averaging_weights...) end @@ -390,19 +390,11 @@ end # Adapt Adapt.adapt_structure(to, free_surface::SplitExplicitFreeSurface) = - SplitExplicitFreeSurface(Adapt.adapt(to, free_surface.η), - nothing, - Adapt.adapt(to, free_surface.auxiliary), + SplitExplicitFreeSurface(Adapt.adapt(to, free_surface.η), nothing, nothing, free_surface.gravitational_acceleration, nothing) -# Adapt -Adapt.adapt_structure(to, auxiliary::SplitExplicitAuxiliaryFields) = - SplitExplicitAuxiliaryFields(Adapt.adapt(to, auxiliary.Gᵁ), - Adapt.adapt(to, auxiliary.Gⱽ), - nothing) - -for Type in (:SplitExplicitFreeSurface, - :SplitExplicitSettings, +for Type in (:SplitExplicitFreeSurface, + :SplitExplicitSettings, :SplitExplicitState, :SplitExplicitAuxiliaryFields, :FixedTimeStepSize, diff --git a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl index 942e1b40f1..6245170f93 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/split_explicit_free_surface_kernels.jl @@ -463,7 +463,7 @@ end Gi = Fⁿ - F⁻ - return ifelse(peripheral_node(i, j, k, grid, f, c, c), zero(grid), Gi) + return ifelse(peripheral_node(i, j, k, grid, c, f, c), zero(grid), Gi) end # Setting up the RHS for the barotropic step (tendencies of the barotropic velocity components) From dea8f3049fd3c56d1863824a0bf5ab0136a297fd Mon Sep 17 00:00:00 2001 From: simone-silvestri Date: Tue, 29 Oct 2024 13:46:22 -0400 Subject: [PATCH 427/567] remove the show --- test/test_zstar_coordinate.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index 8b7dfcdfb3..1399ea683c 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -15,7 +15,6 @@ function test_zstar_coordinate(model, Ni, Δt) # (2) vertical velocities are zero at the top surface for _ in 1:Ni time_step!(model, Δt) - @show extrema(interior(model.velocities.w, :, :, model.grid.Nz+1)) end ∫b = Field(Integral(model.tracers.b)) From 97b41e4fc5399faf971d7b8c5948022fee3a1698 Mon Sep 17 00:00:00 2001 From: simone-silvestri Date: Tue, 29 Oct 2024 13:47:27 -0400 Subject: [PATCH 428/567] check where it is failing --- test/test_zstar_coordinate.jl | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index 1399ea683c..a24d434e3a 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -73,21 +73,22 @@ end end for grid in grids - @info info_message(grid) + @testset info_message(grid) begin - # TODO: minimum_xspacing(grid) on a Immersed GPU grid with ZStarVerticalCoordinate - # fails because it uses too much parameter space. Figure out a way to reduce it - free_surface = SplitExplicitFreeSurface(grid; substeps = 20) - model = HydrostaticFreeSurfaceModel(; grid, - free_surface, - tracers = (:b, :c), - buoyancy = BuoyancyTracer()) + # TODO: minimum_xspacing(grid) on a Immersed GPU grid with ZStarVerticalCoordinate + # fails because it uses too much parameter space. Figure out a way to reduce it + free_surface = SplitExplicitFreeSurface(grid; substeps = 20) + model = HydrostaticFreeSurfaceModel(; grid, + free_surface, + tracers = (:b, :c), + buoyancy = BuoyancyTracer()) - bᵢ(x, y, z) = x < grid.Lx / 2 ? 0.06 : 0.01 + bᵢ(x, y, z) = x < grid.Lx / 2 ? 0.06 : 0.01 - set!(model, c = (x, y, z) -> rand(), b = bᵢ) + set!(model, c = (x, y, z) -> rand(), b = bᵢ) - test_zstar_coordinate(model, 100, 10) + test_zstar_coordinate(model, 100, 10) + end end end end From 040bcad314cdb69de5d994d588aa11c45de7dfb3 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 30 Oct 2024 09:59:34 +0100 Subject: [PATCH 429/567] fairer comparison --- test/test_zstar_coordinate.jl | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index a24d434e3a..5c0205a1de 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -22,7 +22,7 @@ function test_zstar_coordinate(model, Ni, Δt) @test interior(∫b, 1, 1, 1) ≈ interior(∫bᵢ, 1, 1, 1) @test interior(∫c, 1, 1, 1) ≈ interior(∫cᵢ, 1, 1, 1) - @test maximum(interior(w, :, :, Nz+1)) < eps(eltype(w)) + @test maximum(interior(w, :, :, Nz+1)) < model.grid.Nz * eps(eltype(w)) return nothing end @@ -50,22 +50,22 @@ end for topology in topologies Random.seed!(1234) - rtg = RectilinearGrid(arch; size = (10, 10, 10), x = (0, 100kilometers), y = (-10kilometers, 10kilometers), topology, z = z_uniform) - rtgv = RectilinearGrid(arch; size = (10, 10, 10), x = (0, 100kilometers), y = (-10kilometers, 10kilometers), topology, z = z_stretched) + rtg = RectilinearGrid(arch; size = (10, 10, 20), x = (0, 100kilometers), y = (-10kilometers, 10kilometers), topology, z = z_uniform) + rtgv = RectilinearGrid(arch; size = (10, 10, 20), x = (0, 100kilometers), y = (-10kilometers, 10kilometers), topology, z = z_stretched) - irtg = ImmersedBoundaryGrid(rtg, GridFittedBottom((x, y) -> - rand() - 10)) - irtgv = ImmersedBoundaryGrid(rtgv, GridFittedBottom((x, y) -> - rand() - 10)) - prtg = ImmersedBoundaryGrid(rtg, PartialCellBottom((x, y) -> - rand() - 10)) - prtgv = ImmersedBoundaryGrid(rtgv, PartialCellBottom((x, y) -> - rand() - 10)) + irtg = ImmersedBoundaryGrid(rtg, GridFittedBottom((x, y) -> rand() - 10)) + irtgv = ImmersedBoundaryGrid(rtgv, GridFittedBottom((x, y) -> rand() - 10)) + prtg = ImmersedBoundaryGrid(rtg, PartialCellBottom((x, y) -> rand() - 10)) + prtgv = ImmersedBoundaryGrid(rtgv, PartialCellBottom((x, y) -> rand() - 10)) if topology[2] == Bounded - llg = LatitudeLongitudeGrid(arch; size = (10, 10, 10), latitude = (0, 1), longitude = (0, 1), topology, z = z_uniform) - llgv = LatitudeLongitudeGrid(arch; size = (10, 10, 10), latitude = (0, 1), longitude = (0, 1), topology, z = z_stretched) + llg = LatitudeLongitudeGrid(arch; size = (10, 10, 20), latitude = (0, 1), longitude = (0, 1), topology, z = z_uniform) + llgv = LatitudeLongitudeGrid(arch; size = (10, 10, 20), latitude = (0, 1), longitude = (0, 1), topology, z = z_stretched) - illg = ImmersedBoundaryGrid(llg, GridFittedBottom((x, y) -> - rand() - 5)) - illgv = ImmersedBoundaryGrid(llgv, GridFittedBottom((x, y) -> - rand() - 5)) - pllg = ImmersedBoundaryGrid(llg, PartialCellBottom((x, y) -> - rand() - 5)) - pllgv = ImmersedBoundaryGrid(llgv, PartialCellBottom((x, y) -> - rand() - 5)) + illg = ImmersedBoundaryGrid(llg, GridFittedBottom((x, y) -> rand() - 10)) + illgv = ImmersedBoundaryGrid(llgv, GridFittedBottom((x, y) -> rand() - 10)) + pllg = ImmersedBoundaryGrid(llg, PartialCellBottom((x, y) -> rand() - 10)) + pllgv = ImmersedBoundaryGrid(llgv, PartialCellBottom((x, y) -> rand() - 10)) grids = [llg, rtg, llgv, rtgv, illg, irtg, illgv, irtgv, pllg, prtg, pllgv, prtgv] else @@ -73,7 +73,9 @@ end end for grid in grids - @testset info_message(grid) begin + info_msg = info_message(grid) + @testset "$info_msg" begin + @info " $info_msg" # TODO: minimum_xspacing(grid) on a Immersed GPU grid with ZStarVerticalCoordinate # fails because it uses too much parameter space. Figure out a way to reduce it From 6e183bd3161a42b3cfcd25a34b772e3c13ddd145 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 30 Oct 2024 13:39:41 +0100 Subject: [PATCH 430/567] see if this test passes --- .buildkite/distributed/pipeline.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 20753bc6f1..57859f199d 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -49,16 +49,21 @@ steps: GPU_TEST: "true" MPI_TEST: "true" commands: + - "echo '--- Instantiate project'" + - "julia --project -e 'using Pkg; Pkg.instantiate(verbose=true)'" + + - "echo '--- Precompile project'" + - "julia --project -e 'using Pkg; Pkg.precompile(strict=true)'" + - "julia --project -e 'using Pkg; Pkg.status()'" + + # Force the initialization of the CUDA runtime as it is lazily loaded by default: + - "echo '--- Initialize the CUDA runtime'" + - "julia --project -e 'using CUDA; CUDA.precompile_runtime()'" - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: slurm_mem: 120G slurm_ntasks: 4 slurm_gpus_per_task: 1 - retry: - automatic: - - exit_status: 1 - limit: 1 - - label: "🦾 cpu distributed solvers tests" key: "distributed_solvers_cpu" From c57a321c9deed6a187bab5a7cf869d5f116002ea Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 30 Oct 2024 14:29:26 +0100 Subject: [PATCH 431/567] this was removed --- test/test_cubed_spheres.jl | 227 ------------------------------------- 1 file changed, 227 deletions(-) delete mode 100644 test/test_cubed_spheres.jl diff --git a/test/test_cubed_spheres.jl b/test/test_cubed_spheres.jl deleted file mode 100644 index a950dc5cbf..0000000000 --- a/test/test_cubed_spheres.jl +++ /dev/null @@ -1,227 +0,0 @@ -include("dependencies_for_runtests.jl") -include("data_dependencies.jl") - -using Statistics: mean -using Oceananigans.Operators -using Oceananigans.CubedSpheres -using Oceananigans.Models.HydrostaticFreeSurfaceModels -using Oceananigans.Models.HydrostaticFreeSurfaceModels: VerticalVorticityField - -# To be used in the test below as `KernelFunctionOperation`s -@inline intrinsic_vector_x_component(i, j, k, grid, uₑ, vₑ) = - @inbounds intrinsic_vector(i, j, k, grid, uₑ, vₑ)[1] - -@inline intrinsic_vector_y_component(i, j, k, grid, uₑ, vₑ) = - @inbounds intrinsic_vector(i, j, k, grid, uₑ, vₑ)[2] - -@inline extrinsic_vector_x_component(i, j, k, grid, uₑ, vₑ) = - @inbounds intrinsic_vector(i, j, k, grid, uₑ, vₑ)[1] - -@inline extrinsic_vector_y_component(i, j, k, grid, uₑ, vₑ) = - @inbounds intrinsic_vector(i, j, k, grid, uₑ, vₑ)[2] - -function kinetic_energy(u, v) - ke = Field(0.5 * (u * u + v * v)) - return compute!(ke) -end - -function test_vector_rotation(grid) - u = XFaceField(grid) - v = YFaceField(grid) - - # Purely longitudinal flow in the extrinsic coordinate system - set!(u, 1) - set!(v, 0) - - # Convert it to an "Instrinsic" reference frame - uᵢ = KernelFunctionOperation{Face, Center, Center}(intrinsic_vector_x_component, grid, u, v) - vᵢ = KernelFunctionOperation{Center, Face, Center}(intrinsic_vector_y_component, grid, u, v) - - uᵢ = compute!(Field(uᵢ)) - vᵢ = compute!(Field(vᵢ)) - - # The extrema of u and v, as well as their mean value should - # be equivalent on an "intrinsic" frame - @test maximum(uᵢ) ≈ maximum(vᵢ) - @test minimum(uᵢ) ≈ minimum(vᵢ) - @test mean(uᵢ) ≈ mean(vᵢ) - @test mean(uᵢ) > 0 # The mean value should be positive - - # Kinetic energy should remain the same - KE = kinetic_energy(uᵢ, vᵢ) - @test all(on_architecture(CPU(), interior(KE)) .≈ 0.5) - - # Convert it back to a purely zonal velocity (vₑ == 0) - uₑ = KernelFunctionOperation{Face, Center, Center}(extrinsic_vector_x_component, grid, uᵢ, vᵢ) - vₑ = KernelFunctionOperation{Center, Face, Center}(extrinsic_vector_y_component, grid, uᵢ, vᵢ) - - uₑ = compute!(Field(uₑ)) - vₑ = compute!(Field(vₑ)) - - # Make sure that the flow was converted back to a - # purely zonal flow in the extrensic frame (v ≈ 0) - @test all(on_architecture(CPU(), interior(vₑ)) .≈ 0) - @test all(on_architecture(CPU(), interior(uₑ)) .≈ 1) - - # Purely meridional flow in the extrinsic coordinate system - set!(u, 0) - set!(v, 1) - - # Convert it to an "Instrinsic" reference frame - uᵢ = KernelFunctionOperation{Face, Center, Center}(intrinsic_vector_x_component, grid, u, v) - vᵢ = KernelFunctionOperation{Center, Face, Center}(intrinsic_vector_y_component, grid, u, v) - - uᵢ = compute!(Field(uᵢ)) - vᵢ = compute!(Field(vᵢ)) - - # The extrema of u and v, as well as their mean value should - # be equivalent on an "intrinsic" frame - @test maximum(uᵢ) ≈ maximum(vᵢ) - @test minimum(uᵢ) ≈ minimum(vᵢ) - @test mean(uᵢ) ≈ mean(vᵢ) - @test mean(vᵢ) > 0 # The mean value should be positive - - # Kinetic energy should remain the same - KE = kinetic_energy(uᵢ, vᵢ) - @test all(on_architecture(CPU(), interior(KE)) .≈ 0.5) - - # Convert it back to a purely zonal velocity (vₑ == 0) - uₑ = KernelFunctionOperation{Face, Center, Center}(extrinsic_vector_x_component, grid, uᵢ, vᵢ) - vₑ = KernelFunctionOperation{Center, Face, Center}(extrinsic_vector_y_component, grid, uᵢ, vᵢ) - - uₑ = compute!(Field(uₑ)) - vₑ = compute!(Field(vₑ)) - - # Make sure that the flow was converted back to a - # purely zonal flow in the extrensic frame (v ≈ 0) - @test all(on_architecture(CPU(), interior(vₑ)) .≈ 1) - @test all(on_architecture(CPU(), interior(uₑ)) .≈ 0) - - # Mixed zonal and meridional flow. - set!(u, 0.5) - set!(v, 0.5) - - # Convert it to an "Instrinsic" reference frame - uᵢ = KernelFunctionOperation{Face, Center, Center}(intrinsic_vector_x_component, grid, u, v) - vᵢ = KernelFunctionOperation{Center, Face, Center}(intrinsic_vector_y_component, grid, u, v) - - uᵢ = compute!(Field(uᵢ)) - vᵢ = compute!(Field(vᵢ)) - - # The extrema of u and v, as well as their mean value should - # be equivalent on an "intrinsic" frame - @test maximum(uᵢ) ≈ maximum(vᵢ) - @test minimum(uᵢ) ≈ minimum(vᵢ) - @test mean(uᵢ) ≈ mean(vᵢ) - @test mean(vᵢ) > 0 # The mean value should be positive - - # Kinetic energy should remain the same - KE = kinetic_energy(uᵢ, vᵢ) - @test all(on_architecture(CPU(), interior(KE)) .≈ 0.25) - - # Convert it back to a purely zonal velocity (vₑ == 0) - uₑ = KernelFunctionOperation{Face, Center, Center}(extrinsic_vector_x_component, grid, uᵢ, vᵢ) - vₑ = KernelFunctionOperation{Center, Face, Center}(extrinsic_vector_y_component, grid, uᵢ, vᵢ) - - uₑ = compute!(Field(uₑ)) - vₑ = compute!(Field(vₑ)) - - # Make sure that the flow was converted back to a - # purely zonal flow in the extrensic frame (v ≈ 0) - @test all(on_architecture(CPU(), interior(vₑ)) .≈ 0.5) - @test all(on_architecture(CPU(), interior(uₑ)) .≈ 0.5) -end - - -@testset "Cubed spheres" begin - - @testset "Conformal cubed sphere grid" begin - @info " Testing conformal cubed sphere grid..." - - grid = ConformalCubedSphereGrid(panel_size=(10, 10, 1), z=(-1, 0)) - @test try show(grid); println(); true; catch; false; end - end - - for arch in archs - - @info " Constructing a ConformalCubedSphereGrid from file [$(typeof(arch))]..." - - # These tests cause an undefined `Bound Access Error` on GPU's CI with the new CUDA version. - # The error is not reproducible neither on Tartarus nor on Sverdrup. - # These are excised for the moment (PR #2253) as Cubed sphere will be reworked - if !(arch isa GPU) - # Prototype grid and model for subsequent tests - cs32_filepath = datadep"cubed_sphere_32_grid/cubed_sphere_32_grid.jld2" - grid = OldConformalCubedSphereGrid(cs32_filepath, arch, Nz=1, z=(-1, 0)) - - @info " Constructing a HydrostaticFreeSurfaceModel on a ConformalCubedSphereGrid [$(typeof(arch))]..." - - free_surface = ExplicitFreeSurface(gravitational_acceleration=0.1) - model = HydrostaticFreeSurfaceModel(; grid, free_surface, - momentum_advection = VectorInvariant(), - coriolis = nothing, - closure = nothing, - tracers = :c, - buoyancy = nothing) - - @testset "Constructing a grid from file [$(typeof(arch))]" begin - @test grid isa ConformalCubedSphereGrid - end - - @testset "Conversion from Intrinsic to Extrinsic reference frame [$(typeof(arch))]" begin - @info " Testing the conversion of a vector between the Intrinsic and Extrinsic reference frame" - test_vector_rotation(grid) - end - - @testset "CubedSphereData and CubedSphereFields [$(typeof(arch))]" begin - @info " Testing CubedSphereData and CubedSphereFields [$(typeof(arch))]..." - c = model.tracers.c - η = model.free_surface.η - - set!(c, 0) - set!(η, 0) - - CUDA.allowscalar(true) - @test all(all(face_c .== 0) for face_c in faces(c)) - @test all(all(face_η .== 0) for face_η in faces(η)) - CUDA.allowscalar(false) - - @test maximum(abs, c) == 0 - @test minimum(abs, c) == 0 - @test mean(c) == 0 - - @test maximum(abs, η) == 0 - @test minimum(abs, η) == 0 - @test mean(η) == 0 - end - - @testset "Constructing a HydrostaticFreeSurfaceModel [$(typeof(arch))]" begin - @test model isa HydrostaticFreeSurfaceModel - end - - @testset "Time stepping a HydrostaticFreeSurfaceModel [$(typeof(arch))]" begin - @info " Time-stepping HydrostaticFreeSurfaceModel on a ConformalCubedSphereGrid [$(typeof(arch))]..." - time_step!(model, 1) - @test try time_step!(model, 1); true; catch; false; end - end - - @testset "VerticalVorticityField on ConformalCubedSphereGrid [$(typeof(arch))]" begin - @info " Testing VerticalVorticityField on a ConformalCubedSphereGrid [$(typeof(arch))]..." - ζ = VerticalVorticityField(model) - - @test ζ isa Field - - set!(model, u = (x, y, z) -> rand()) - - @test try - compute!(ζ) - true - catch err - println(sprint(showerror, err)) - false - end - @test maximum(abs, ζ) > 0 # fingers crossed - end - end - end -end From c56b15b16c1561a1bfa6ea43ca250cb3e2ad7f39 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 31 Oct 2024 15:54:23 +0100 Subject: [PATCH 432/567] maybe precompiling before... --- .buildkite/distributed/pipeline.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 57859f199d..4d5a88ebc0 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -49,16 +49,7 @@ steps: GPU_TEST: "true" MPI_TEST: "true" commands: - - "echo '--- Instantiate project'" - - "julia --project -e 'using Pkg; Pkg.instantiate(verbose=true)'" - - - "echo '--- Precompile project'" - - "julia --project -e 'using Pkg; Pkg.precompile(strict=true)'" - - "julia --project -e 'using Pkg; Pkg.status()'" - - # Force the initialization of the CUDA runtime as it is lazily loaded by default: - - "echo '--- Initialize the CUDA runtime'" - - "julia --project -e 'using CUDA; CUDA.precompile_runtime()'" + - "julia -O0 -O0 --color=yes --project -e 'pkg = Base.PkgId(Base.UUID("76a88914-d11a-5bdc-97e0-2f5a05c973a2"), "CUDA_Runtime_jll"); Base.compilecache(pkg)'" - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: slurm_mem: 120G From e30973feb600dda1564db9a093bdbba8da6c8dc8 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 31 Oct 2024 16:48:52 +0100 Subject: [PATCH 433/567] double O0 --- .buildkite/distributed/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 4d5a88ebc0..ec12aad398 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -49,7 +49,7 @@ steps: GPU_TEST: "true" MPI_TEST: "true" commands: - - "julia -O0 -O0 --color=yes --project -e 'pkg = Base.PkgId(Base.UUID("76a88914-d11a-5bdc-97e0-2f5a05c973a2"), "CUDA_Runtime_jll"); Base.compilecache(pkg)'" + - "julia -O0 --color=yes --project -e 'pkg = Base.PkgId(Base.UUID("76a88914-d11a-5bdc-97e0-2f5a05c973a2"), "CUDA_Runtime_jll"); Base.compilecache(pkg)'" - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: slurm_mem: 120G From e4cb16ecb53dbdf52f7721baec7f10b3f1f0902b Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 31 Oct 2024 17:46:25 +0100 Subject: [PATCH 434/567] back to previous clima_common --- .buildkite/distributed/pipeline.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index ec12aad398..0c0fb40d8d 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -1,7 +1,7 @@ agents: queue: new-central slurm_mem: 8G - modules: climacommon/2024_05_27 + modules: climacommon/2024_10_09 env: OPENBLAS_NUM_THREADS: 1 @@ -49,7 +49,6 @@ steps: GPU_TEST: "true" MPI_TEST: "true" commands: - - "julia -O0 --color=yes --project -e 'pkg = Base.PkgId(Base.UUID("76a88914-d11a-5bdc-97e0-2f5a05c973a2"), "CUDA_Runtime_jll"); Base.compilecache(pkg)'" - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: slurm_mem: 120G From 0c1f01c79a704fced0e52bad34ce027f8fb7d94d Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 1 Nov 2024 09:33:23 +0100 Subject: [PATCH 435/567] another quick test --- test/runtests.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/runtests.jl b/test/runtests.jl index 816d8d578a..11a4e553e8 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -173,11 +173,13 @@ CUDA.allowscalar() do if group == :distributed || group == :all MPI.Initialized() || MPI.Init() archs = test_architectures() + CUDA.set_runtime_version!(v"12.6") include("test_distributed_models.jl") end if group == :distributed_solvers || group == :all MPI.Initialized() || MPI.Init() + CUDA.set_runtime_version!(v"12.6") include("test_distributed_transpose.jl") include("test_distributed_poisson_solvers.jl") end @@ -185,12 +187,14 @@ CUDA.allowscalar() do if group == :distributed_hydrostatic_model || group == :all MPI.Initialized() || MPI.Init() archs = test_architectures() + CUDA.set_runtime_version!(v"12.6") include("test_hydrostatic_regression.jl") include("test_distributed_hydrostatic_model.jl") end if group == :distributed_nonhydrostatic_regression || group == :all MPI.Initialized() || MPI.Init() + CUDA.set_runtime_version!(v"12.6") archs = nonhydrostatic_regression_test_architectures() include("test_nonhydrostatic_regression.jl") end From bec1cd1e0728ed6da36b9b3e80d4b408ec078eaf Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 1 Nov 2024 14:40:18 +0100 Subject: [PATCH 436/567] change environment --- .buildkite/distributed/pipeline.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 0c0fb40d8d..c59078ca24 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -4,15 +4,18 @@ agents: modules: climacommon/2024_10_09 env: + JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite/distributed" OPENBLAS_NUM_THREADS: 1 - OMPI_MCA_opal_warn_on_missing_libcuda: 0 + JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager JULIA_NUM_PRECOMPILE_TASKS: 8 + JULIA_NUM_THREADS: 8 + OMPI_MCA_opal_warn_on_missing_libcuda: 0 steps: - label: "initialize" key: "init" env: - TEST_GROUP: "init" + TEST_GROUP: "init_central" command: - "echo '--- Instantiate project'" - "julia --project -e 'using Pkg; Pkg.instantiate(verbose=true)'" @@ -26,6 +29,7 @@ steps: - "julia --project -e 'using CUDA; CUDA.precompile_runtime()'" - "julia --project -e 'using Pkg; Pkg.test()'" agents: + slurm_mem: 120G slurm_gpus: 1 slurm_cpus_per_task: 8 From 75546af3166e5e9969c2d8fe3282d83395aa30b8 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 1 Nov 2024 14:42:40 +0100 Subject: [PATCH 437/567] correct the utils --- test/utils_for_runtests.jl | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/test/utils_for_runtests.jl b/test/utils_for_runtests.jl index e8741e4839..a64d44263b 100644 --- a/test/utils_for_runtests.jl +++ b/test/utils_for_runtests.jl @@ -3,16 +3,16 @@ using Oceananigans.DistributedComputations: Distributed, Partition, child_archit import Oceananigans.Fields: interior -test_child_arch() = parse(Bool, get(ENV, "GPU_TEST", "false")) ? GPU() : CPU() -mpi_test() = parse(Bool, get(ENV, "MPI_TEST", "false")) +# Are the test running on the GPUs? +# Are the test running in parallel? +child_arch = parse(Bool, get(ENV, "GPU_TEST", "false")) ? GPU() : CPU() +mpi_test = parse(Bool, get(ENV, "MPI_TEST", "false")) function test_architectures() - child_arch = test_child_arch() - # If MPI is initialized with MPI.Comm_size > 0, we are running in parallel. # We test several different configurations: `Partition(x = 4)`, `Partition(y = 4)`, # `Partition(x = 2, y = 2)`, and different fractional subdivisions in x, y and xy - if mpi_test() + if mpi_test if MPI.Initialized() && MPI.Comm_size(MPI.COMM_WORLD) == 4 return (Distributed(child_arch; partition = Partition(4)), Distributed(child_arch; partition = Partition(1, 4)), @@ -31,12 +31,10 @@ end # For nonhydrostatic simulations we cannot use `Fractional` at the moment (requirements # for the tranpose are more stringent than for hydrostatic simulations). function nonhydrostatic_regression_test_architectures() - child_arch = test_child_arch() - # If MPI is initialized with MPI.Comm_size > 0, we are running in parallel. # We test 3 different configurations: `Partition(x = 4)`, `Partition(y = 4)` # and `Partition(x = 2, y = 2)` - if mpi_test() + if mpi_test if MPI.Initialized() && MPI.Comm_size(MPI.COMM_WORLD) == 4 return (Distributed(child_arch; partition = Partition(4)), Distributed(child_arch; partition = Partition(1, 4)), From 9b334af5cf796ab4eb08d708374152c7dcb90bb8 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 4 Nov 2024 15:25:24 +0100 Subject: [PATCH 438/567] this should load mpitrampoline --- .buildkite/distributed/pipeline.yml | 32 ++++++++---------------- test/test_distributed_poisson_solvers.jl | 4 +-- test/test_distributed_transpose.jl | 2 -- 3 files changed, 12 insertions(+), 26 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index c59078ca24..0fcd204243 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -1,38 +1,28 @@ agents: queue: new-central slurm_mem: 8G - modules: climacommon/2024_10_09 + modules: climacommon/2024_05_27 env: JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite/distributed" OPENBLAS_NUM_THREADS: 1 - JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager - JULIA_NUM_PRECOMPILE_TASKS: 8 - JULIA_NUM_THREADS: 8 OMPI_MCA_opal_warn_on_missing_libcuda: 0 steps: - label: "initialize" - key: "init" - env: - TEST_GROUP: "init_central" + key: "init_central" command: - - "echo '--- Instantiate project'" - - "julia --project -e 'using Pkg; Pkg.instantiate(verbose=true)'" - - - "echo '--- Precompile project'" - - "julia --project -e 'using Pkg; Pkg.precompile(strict=true)'" + - echo "--- Instantiate project" + - "julia --project -e 'using Pkg; Pkg.instantiate(;verbose=true); Pkg.precompile(;strict=true)'" + # force the initialization of the CUDA runtime as it is lazily loaded by default + - "julia --project -e 'using CUDA; CUDA.precompile_runtime(); CUDA.versioninfo()'" + - "julia --project -e 'using MPI; MPI.versioninfo()'" + # Download artifacts by running an empty testgroup and thereby executing /test/runtests.jl + - "julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" + + - echo "--- Instantiate status" - "julia --project -e 'using Pkg; Pkg.status()'" - # Force the initialization of the CUDA runtime as it is lazily loaded by default: - - "echo '--- Initialize the CUDA runtime'" - - "julia --project -e 'using CUDA; CUDA.precompile_runtime()'" - - "julia --project -e 'using Pkg; Pkg.test()'" - agents: - slurm_mem: 120G - slurm_gpus: 1 - slurm_cpus_per_task: 8 - - wait - label: "🐉 cpu distributed unit tests" diff --git a/test/test_distributed_poisson_solvers.jl b/test/test_distributed_poisson_solvers.jl index dca406d8c3..5ab7c90a7e 100644 --- a/test/test_distributed_poisson_solvers.jl +++ b/test/test_distributed_poisson_solvers.jl @@ -121,9 +121,7 @@ function divergence_free_poisson_tridiagonal_solution(grid_points, ranks, stretc return Array(interior(∇²ϕ)) ≈ Array(R) end -@testset "Distributed FFT-based Poisson solver" begin - child_arch = test_child_arch() - +@testset "Distributed FFT-based Poisson solver" begin for topology in ((Periodic, Periodic, Periodic), (Periodic, Periodic, Bounded), (Periodic, Bounded, Bounded), diff --git a/test/test_distributed_transpose.jl b/test/test_distributed_transpose.jl index 5bc0fe6292..2eff450b27 100644 --- a/test/test_distributed_transpose.jl +++ b/test/test_distributed_transpose.jl @@ -38,8 +38,6 @@ function test_transpose(grid_points, ranks, topo, child_arch) end @testset "Distributed Transpose" begin - child_arch = test_child_arch() - for topology in ((Periodic, Periodic, Periodic), (Periodic, Periodic, Bounded), (Periodic, Bounded, Bounded), From f8c64016b94e0758f05d0cef9d7e641a3d6bae14 Mon Sep 17 00:00:00 2001 From: Gregory Wagner Date: Mon, 4 Nov 2024 15:03:56 -0700 Subject: [PATCH 439/567] Fix formatting --- .buildkite/distributed/pipeline.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 0fcd204243..02223bf2dc 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -13,10 +13,12 @@ steps: key: "init_central" command: - echo "--- Instantiate project" - - "julia --project -e 'using Pkg; Pkg.instantiate(;verbose=true); Pkg.precompile(;strict=true)'" - # force the initialization of the CUDA runtime as it is lazily loaded by default + - "julia --project -e 'using Pkg; Pkg.instantiate(; verbose=true); Pkg.precompile(; strict=true)'" + + # Force the initialization of the CUDA runtime as it is lazily loaded by default - "julia --project -e 'using CUDA; CUDA.precompile_runtime(); CUDA.versioninfo()'" - "julia --project -e 'using MPI; MPI.versioninfo()'" + # Download artifacts by running an empty testgroup and thereby executing /test/runtests.jl - "julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" From 1dc42bba5b2e15ec24eb5f8916bb122c0d0b99eb Mon Sep 17 00:00:00 2001 From: Gregory Wagner Date: Mon, 4 Nov 2024 15:24:56 -0700 Subject: [PATCH 440/567] Go back to latest climacommon --- .buildkite/distributed/pipeline.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 02223bf2dc..e0ea62eef4 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -1,10 +1,11 @@ agents: queue: new-central slurm_mem: 8G - modules: climacommon/2024_05_27 + modules: climacommon/2024_10_08 env: JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite/distributed" + JULIA_MPI_HAS_CUDA: "true" OPENBLAS_NUM_THREADS: 1 OMPI_MCA_opal_warn_on_missing_libcuda: 0 From 5a870e787b76e40e2ba8bb4d1ec2d81ee1b71bb2 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 5 Nov 2024 09:54:12 +0100 Subject: [PATCH 441/567] try adding Manifest --- Manifest.toml | 972 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 972 insertions(+) create mode 100644 Manifest.toml diff --git a/Manifest.toml b/Manifest.toml new file mode 100644 index 0000000000..35f1827b15 --- /dev/null +++ b/Manifest.toml @@ -0,0 +1,972 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.11.1" +manifest_format = "2.0" +project_hash = "f3d268da10603d34af6d3fea2014bfc2339ad299" + +[[deps.AbstractFFTs]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "d92ad398961a3ed262d8bf04a1a2b8340f915fef" +uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c" +version = "1.5.0" + + [deps.AbstractFFTs.extensions] + AbstractFFTsChainRulesCoreExt = "ChainRulesCore" + AbstractFFTsTestExt = "Test" + + [deps.AbstractFFTs.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[deps.Adapt]] +deps = ["LinearAlgebra", "Requires"] +git-tree-sha1 = "50c3c56a52972d78e8be9fd135bfb91c9574c140" +uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" +version = "4.1.1" +weakdeps = ["StaticArrays"] + + [deps.Adapt.extensions] + AdaptStaticArraysExt = "StaticArrays" + +[[deps.ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.2" + +[[deps.Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +version = "1.11.0" + +[[deps.Atomix]] +deps = ["UnsafeAtomics"] +git-tree-sha1 = "c06a868224ecba914baa6942988e2f2aade419be" +uuid = "a9b6321e-bd34-4604-b9c9-b65b8de01458" +version = "0.1.0" + +[[deps.BFloat16s]] +deps = ["LinearAlgebra", "Printf", "Random", "Test"] +git-tree-sha1 = "2c7cc21e8678eff479978a0a2ef5ce2f51b63dff" +uuid = "ab4f0b2a-ad5b-11e8-123f-65d77653426b" +version = "0.5.0" + +[[deps.Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +version = "1.11.0" + +[[deps.Blosc_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Lz4_jll", "Zlib_jll", "Zstd_jll"] +git-tree-sha1 = "ef12cdd1c7fb7e1dfd6fa8fd60d4db6bc61d2f23" +uuid = "0b7ba130-8d10-5ba8-a3d6-c5182647fed9" +version = "1.21.6+0" + +[[deps.Bzip2_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "8873e196c2eb87962a2048b3b8e08946535864a1" +uuid = "6e34b625-4abd-537c-b88f-471c36dfa7a0" +version = "1.0.8+2" + +[[deps.CEnum]] +git-tree-sha1 = "389ad5c84de1ae7cf0e28e381131c98ea87d54fc" +uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82" +version = "0.5.0" + +[[deps.CFTime]] +deps = ["Dates", "Printf"] +git-tree-sha1 = "5afb5c5ba2688ca43a9ad2e5a91cbb93921ccfa1" +uuid = "179af706-886a-5703-950a-314cd64e0468" +version = "0.1.3" + +[[deps.CUDA]] +deps = ["AbstractFFTs", "Adapt", "BFloat16s", "CEnum", "CUDA_Driver_jll", "CUDA_Runtime_Discovery", "CUDA_Runtime_jll", "Crayons", "DataFrames", "ExprTools", "GPUArrays", "GPUCompiler", "KernelAbstractions", "LLVM", "LLVMLoopInfo", "LazyArtifacts", "Libdl", "LinearAlgebra", "Logging", "NVTX", "Preferences", "PrettyTables", "Printf", "Random", "Random123", "RandomNumbers", "Reexport", "Requires", "SparseArrays", "StaticArrays", "Statistics", "demumble_jll"] +git-tree-sha1 = "e0725a467822697171af4dae15cec10b4fc19053" +uuid = "052768ef-5323-5732-b1bb-66c8b64840ba" +version = "5.5.2" + + [deps.CUDA.extensions] + ChainRulesCoreExt = "ChainRulesCore" + EnzymeCoreExt = "EnzymeCore" + SpecialFunctionsExt = "SpecialFunctions" + + [deps.CUDA.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869" + SpecialFunctions = "276daf66-3868-5448-9aa4-cd146d93841b" + +[[deps.CUDA_Driver_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "ccd1e54610c222fadfd4737dac66bff786f63656" +uuid = "4ee394cb-3365-5eb0-8335-949819d2adfc" +version = "0.10.3+0" + +[[deps.CUDA_Runtime_Discovery]] +deps = ["Libdl"] +git-tree-sha1 = "33576c7c1b2500f8e7e6baa082e04563203b3a45" +uuid = "1af6417a-86b4-443c-805f-a4643ffb695f" +version = "0.3.5" + +[[deps.CUDA_Runtime_jll]] +deps = ["Artifacts", "CUDA_Driver_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] +git-tree-sha1 = "e43727b237b2879a34391eeb81887699a26f8f2f" +uuid = "76a88914-d11a-5bdc-97e0-2f5a05c973a2" +version = "0.15.3+0" + +[[deps.ColorTypes]] +deps = ["FixedPointNumbers", "Random"] +git-tree-sha1 = "b10d0b65641d57b8b4d5e234446582de5047050d" +uuid = "3da002f7-5984-5a60-b8a6-cbb66c0b333f" +version = "0.11.5" + +[[deps.Colors]] +deps = ["ColorTypes", "FixedPointNumbers", "Reexport"] +git-tree-sha1 = "362a287c3aa50601b0bc359053d5c2468f0e7ce0" +uuid = "5ae59095-9a9b-59fe-a467-6f913c188581" +version = "0.12.11" + +[[deps.CommonDataModel]] +deps = ["CFTime", "DataStructures", "Dates", "Preferences", "Printf", "Statistics"] +git-tree-sha1 = "d6fb5bf939a2753c74984b11434ea25d6c397a58" +uuid = "1fbeeb36-5f17-413c-809b-666fb144f157" +version = "0.3.6" + +[[deps.Compat]] +deps = ["TOML", "UUIDs"] +git-tree-sha1 = "8ae8d32e09f0dcf42a36b90d4e17f5dd2e4c4215" +uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" +version = "4.16.0" +weakdeps = ["Dates", "LinearAlgebra"] + + [deps.Compat.extensions] + CompatLinearAlgebraExt = "LinearAlgebra" + +[[deps.CompilerSupportLibraries_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" +version = "1.1.1+0" + +[[deps.ConstructionBase]] +git-tree-sha1 = "76219f1ed5771adbb096743bff43fb5fdd4c1157" +uuid = "187b0558-2788-49d3-abe0-74a17ed4e7c9" +version = "1.5.8" + + [deps.ConstructionBase.extensions] + ConstructionBaseIntervalSetsExt = "IntervalSets" + ConstructionBaseLinearAlgebraExt = "LinearAlgebra" + ConstructionBaseStaticArraysExt = "StaticArrays" + + [deps.ConstructionBase.weakdeps] + IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953" + LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" + +[[deps.Crayons]] +git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" +uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" +version = "4.1.1" + +[[deps.CubedSphere]] +deps = ["TaylorSeries"] +git-tree-sha1 = "51bb25de518b4c62b7cdf26e5fbb84601bb27a60" +uuid = "7445602f-e544-4518-8976-18f8e8ae6cdb" +version = "0.3.0" + +[[deps.DataAPI]] +git-tree-sha1 = "abe83f3a2f1b857aac70ef8b269080af17764bbe" +uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" +version = "1.16.0" + +[[deps.DataFrames]] +deps = ["Compat", "DataAPI", "DataStructures", "Future", "InlineStrings", "InvertedIndices", "IteratorInterfaceExtensions", "LinearAlgebra", "Markdown", "Missings", "PooledArrays", "PrecompileTools", "PrettyTables", "Printf", "Random", "Reexport", "SentinelArrays", "SortingAlgorithms", "Statistics", "TableTraits", "Tables", "Unicode"] +git-tree-sha1 = "fb61b4812c49343d7ef0b533ba982c46021938a6" +uuid = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +version = "1.7.0" + +[[deps.DataStructures]] +deps = ["Compat", "InteractiveUtils", "OrderedCollections"] +git-tree-sha1 = "1d0a14036acb104d9e89698bd408f63ab58cdc82" +uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +version = "0.18.20" + +[[deps.DataValueInterfaces]] +git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" +uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464" +version = "1.0.0" + +[[deps.Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" +version = "1.11.0" + +[[deps.DiskArrays]] +deps = ["LRUCache", "OffsetArrays"] +git-tree-sha1 = "e0e89a60637a62d13aa2107f0acd169b9b9b77e7" +uuid = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3" +version = "0.4.6" + +[[deps.Distances]] +deps = ["LinearAlgebra", "Statistics", "StatsAPI"] +git-tree-sha1 = "c7e3a542b999843086e2f29dac96a618c105be1d" +uuid = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7" +version = "0.10.12" + + [deps.Distances.extensions] + DistancesChainRulesCoreExt = "ChainRulesCore" + DistancesSparseArraysExt = "SparseArrays" + + [deps.Distances.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + +[[deps.Distributed]] +deps = ["Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" +version = "1.11.0" + +[[deps.DocStringExtensions]] +deps = ["LibGit2"] +git-tree-sha1 = "2fb1e02f2b635d0845df5d7c167fec4dd739b00d" +uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" +version = "0.9.3" + +[[deps.Downloads]] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.6.0" + +[[deps.ExprTools]] +git-tree-sha1 = "27415f162e6028e81c72b82ef756bf321213b6ec" +uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04" +version = "0.1.10" + +[[deps.FFTW]] +deps = ["AbstractFFTs", "FFTW_jll", "LinearAlgebra", "MKL_jll", "Preferences", "Reexport"] +git-tree-sha1 = "4820348781ae578893311153d69049a93d05f39d" +uuid = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" +version = "1.8.0" + +[[deps.FFTW_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "4d81ed14783ec49ce9f2e168208a12ce1815aa25" +uuid = "f5851436-0d7a-5f13-b9de-f02708fd171a" +version = "3.3.10+1" + +[[deps.FileIO]] +deps = ["Pkg", "Requires", "UUIDs"] +git-tree-sha1 = "62ca0547a14c57e98154423419d8a342dca75ca9" +uuid = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" +version = "1.16.4" + +[[deps.FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" +version = "1.11.0" + +[[deps.FixedPointNumbers]] +deps = ["Statistics"] +git-tree-sha1 = "05882d6995ae5c12bb5f36dd2ed3f61c98cbb172" +uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93" +version = "0.8.5" + +[[deps.Future]] +deps = ["Random"] +uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" +version = "1.11.0" + +[[deps.GMP_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "781609d7-10c4-51f6-84f2-b8444358ff6d" +version = "6.3.0+0" + +[[deps.GPUArrays]] +deps = ["Adapt", "GPUArraysCore", "LLVM", "LinearAlgebra", "Printf", "Random", "Reexport", "Serialization", "Statistics"] +git-tree-sha1 = "62ee71528cca49be797076a76bdc654a170a523e" +uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" +version = "10.3.1" + +[[deps.GPUArraysCore]] +deps = ["Adapt"] +git-tree-sha1 = "ec632f177c0d990e64d955ccc1b8c04c485a0950" +uuid = "46192b85-c4d5-4398-a991-12ede77f4527" +version = "0.1.6" + +[[deps.GPUCompiler]] +deps = ["ExprTools", "InteractiveUtils", "LLVM", "Libdl", "Logging", "PrecompileTools", "Preferences", "Scratch", "Serialization", "TOML", "TimerOutputs", "UUIDs"] +git-tree-sha1 = "1d6f290a5eb1201cd63574fbc4440c788d5cb38f" +uuid = "61eb1bfa-7361-4325-ad38-22787b887f55" +version = "0.27.8" + +[[deps.Glob]] +git-tree-sha1 = "97285bbd5230dd766e9ef6749b80fc617126d496" +uuid = "c27321d9-0574-5035-807b-f59d2c89b15c" +version = "1.3.1" + +[[deps.GnuTLS_jll]] +deps = ["Artifacts", "GMP_jll", "JLLWrappers", "Libdl", "Nettle_jll", "P11Kit_jll", "Zlib_jll"] +git-tree-sha1 = "383db7d3f900f4c1f47a8a04115b053c095e48d3" +uuid = "0951126a-58fd-58f1-b5b3-b08c7c4a876d" +version = "3.8.4+0" + +[[deps.HDF5_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "LLVMOpenMP_jll", "LazyArtifacts", "LibCURL_jll", "Libdl", "MPICH_jll", "MPIPreferences", "MPItrampoline_jll", "MicrosoftMPI_jll", "OpenMPI_jll", "OpenSSL_jll", "TOML", "Zlib_jll", "libaec_jll"] +git-tree-sha1 = "38c8874692d48d5440d5752d6c74b0c6b0b60739" +uuid = "0234f1f7-429e-5d53-9886-15a909be8d59" +version = "1.14.2+1" + +[[deps.Hwloc_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "dd3b49277ec2bb2c6b94eb1604d4d0616016f7a6" +uuid = "e33a78d0-f292-5ffc-b300-72abe9b543c8" +version = "2.11.2+0" + +[[deps.IncompleteLU]] +deps = ["LinearAlgebra", "SparseArrays"] +git-tree-sha1 = "6c676e79f98abb6d33fa28122cad099f1e464afe" +uuid = "40713840-3770-5561-ab4c-a76e7d0d7895" +version = "0.2.1" + +[[deps.InlineStrings]] +git-tree-sha1 = "45521d31238e87ee9f9732561bfee12d4eebd52d" +uuid = "842dd82b-1e85-43dc-bf29-5d0ee9dffc48" +version = "1.4.2" + + [deps.InlineStrings.extensions] + ArrowTypesExt = "ArrowTypes" + ParsersExt = "Parsers" + + [deps.InlineStrings.weakdeps] + ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd" + Parsers = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" + +[[deps.IntelOpenMP_jll]] +deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl"] +git-tree-sha1 = "10bd689145d2c3b2a9844005d01087cc1194e79e" +uuid = "1d5cc7b8-4909-519e-a0f8-d0f5ad9712d0" +version = "2024.2.1+0" + +[[deps.InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +version = "1.11.0" + +[[deps.InvertedIndices]] +git-tree-sha1 = "0dc7b50b8d436461be01300fd8cd45aa0274b038" +uuid = "41ab1584-1d38-5bbf-9106-f11c6c58b48f" +version = "1.3.0" + +[[deps.IterativeSolvers]] +deps = ["LinearAlgebra", "Printf", "Random", "RecipesBase", "SparseArrays"] +git-tree-sha1 = "59545b0a2b27208b0650df0a46b8e3019f85055b" +uuid = "42fd0dbc-a981-5370-80f2-aaf504508153" +version = "0.9.4" + +[[deps.IteratorInterfaceExtensions]] +git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" +uuid = "82899510-4779-5014-852e-03e436cf321d" +version = "1.0.0" + +[[deps.JLD2]] +deps = ["FileIO", "MacroTools", "Mmap", "OrderedCollections", "PrecompileTools", "Requires", "TranscodingStreams"] +git-tree-sha1 = "783c1be5213a09609b23237a0c9e5dfd258ae6f2" +uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819" +version = "0.5.7" + +[[deps.JLLWrappers]] +deps = ["Artifacts", "Preferences"] +git-tree-sha1 = "be3dc50a92e5a386872a493a10050136d4703f9b" +uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +version = "1.6.1" + +[[deps.JuliaNVTXCallbacks_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "af433a10f3942e882d3c671aacb203e006a5808f" +uuid = "9c1d0b0a-7046-5b2e-a33f-ea22f176ac7e" +version = "0.2.1+0" + +[[deps.KernelAbstractions]] +deps = ["Adapt", "Atomix", "InteractiveUtils", "MacroTools", "PrecompileTools", "Requires", "StaticArrays", "UUIDs", "UnsafeAtomics", "UnsafeAtomicsLLVM"] +git-tree-sha1 = "e73a077abc7fe798fe940deabe30ef6c66bdde52" +uuid = "63c18a36-062a-441e-b654-da1e3ab1ce7c" +version = "0.9.29" + + [deps.KernelAbstractions.extensions] + EnzymeExt = "EnzymeCore" + LinearAlgebraExt = "LinearAlgebra" + SparseArraysExt = "SparseArrays" + + [deps.KernelAbstractions.weakdeps] + EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869" + LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + +[[deps.LLVM]] +deps = ["CEnum", "LLVMExtra_jll", "Libdl", "Preferences", "Printf", "Unicode"] +git-tree-sha1 = "d422dfd9707bec6617335dc2ea3c5172a87d5908" +uuid = "929cbde3-209d-540e-8aea-75f648917ca0" +version = "9.1.3" +weakdeps = ["BFloat16s"] + + [deps.LLVM.extensions] + BFloat16sExt = "BFloat16s" + +[[deps.LLVMExtra_jll]] +deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] +git-tree-sha1 = "05a8bd5a42309a9ec82f700876903abce1017dd3" +uuid = "dad2f222-ce93-54a1-a47d-0025e8a3acab" +version = "0.0.34+0" + +[[deps.LLVMLoopInfo]] +git-tree-sha1 = "2e5c102cfc41f48ae4740c7eca7743cc7e7b75ea" +uuid = "8b046642-f1f6-4319-8d3c-209ddc03c586" +version = "1.0.0" + +[[deps.LLVMOpenMP_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "78211fb6cbc872f77cad3fc0b6cf647d923f4929" +uuid = "1d63c593-3942-5779-bab2-d838dc0a180e" +version = "18.1.7+0" + +[[deps.LRUCache]] +git-tree-sha1 = "b3cc6698599b10e652832c2f23db3cab99d51b59" +uuid = "8ac3fa9e-de4c-5943-b1dc-09c6b5f20637" +version = "1.6.1" +weakdeps = ["Serialization"] + + [deps.LRUCache.extensions] + SerializationExt = ["Serialization"] + +[[deps.LaTeXStrings]] +git-tree-sha1 = "dda21b8cbd6a6c40d9d02a73230f9d70fed6918c" +uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" +version = "1.4.0" + +[[deps.LazyArtifacts]] +deps = ["Artifacts", "Pkg"] +uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" +version = "1.11.0" + +[[deps.LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.4" + +[[deps.LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "8.6.0+0" + +[[deps.LibGit2]] +deps = ["Base64", "LibGit2_jll", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" +version = "1.11.0" + +[[deps.LibGit2_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"] +uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" +version = "1.7.2+0" + +[[deps.LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "MbedTLS_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.11.0+1" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +version = "1.11.0" + +[[deps.Libiconv_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "61dfdba58e585066d8bce214c5a51eaa0539f269" +uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531" +version = "1.17.0+1" + +[[deps.LinearAlgebra]] +deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +version = "1.11.0" + +[[deps.Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" +version = "1.11.0" + +[[deps.Lz4_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "abf88ff67f4fd89839efcae2f4c39cbc4ecd0846" +uuid = "5ced341a-0733-55b8-9ab6-a4889d929147" +version = "1.10.0+1" + +[[deps.MKL_jll]] +deps = ["Artifacts", "IntelOpenMP_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "oneTBB_jll"] +git-tree-sha1 = "f046ccd0c6db2832a9f639e2c669c6fe867e5f4f" +uuid = "856f044c-d86e-5d09-b602-aeab76dc8ba7" +version = "2024.2.0+0" + +[[deps.MPI]] +deps = ["Distributed", "DocStringExtensions", "Libdl", "MPICH_jll", "MPIPreferences", "MPItrampoline_jll", "MicrosoftMPI_jll", "OpenMPI_jll", "PkgVersion", "PrecompileTools", "Requires", "Serialization", "Sockets"] +git-tree-sha1 = "892676019c58f34e38743bc989b0eca5bce5edc5" +uuid = "da04e1cc-30fd-572f-bb4f-1f8673147195" +version = "0.20.22" + + [deps.MPI.extensions] + AMDGPUExt = "AMDGPU" + CUDAExt = "CUDA" + + [deps.MPI.weakdeps] + AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e" + CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" + +[[deps.MPICH_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Hwloc_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "MPIPreferences", "TOML"] +git-tree-sha1 = "7715e65c47ba3941c502bffb7f266a41a7f54423" +uuid = "7cb0a576-ebde-5e09-9194-50597f1243b4" +version = "4.2.3+0" + +[[deps.MPIPreferences]] +deps = ["Libdl", "Preferences"] +git-tree-sha1 = "c105fe467859e7f6e9a852cb15cb4301126fac07" +uuid = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267" +version = "0.1.11" + +[[deps.MPItrampoline_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "MPIPreferences", "TOML"] +git-tree-sha1 = "70e830dab5d0775183c99fc75e4c24c614ed7142" +uuid = "f1f71cc9-e9ae-5b93-9b94-4fe0e1ad3748" +version = "5.5.1+0" + +[[deps.MacroTools]] +deps = ["Markdown", "Random"] +git-tree-sha1 = "2fa9ee3e63fd3a4f7a9a4f4744a52f4856de82df" +uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" +version = "0.5.13" + +[[deps.Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" +version = "1.11.0" + +[[deps.MbedTLS_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +version = "2.28.6+0" + +[[deps.MicrosoftMPI_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "f12a29c4400ba812841c6ace3f4efbb6dbb3ba01" +uuid = "9237b28f-5490-5468-be7b-bb81f5f5e6cf" +version = "10.1.4+2" + +[[deps.Missings]] +deps = ["DataAPI"] +git-tree-sha1 = "ec4f7fbeab05d7747bdf98eb74d130a2a2ed298d" +uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" +version = "1.2.0" + +[[deps.Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" +version = "1.11.0" + +[[deps.MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2023.12.12" + +[[deps.NCDatasets]] +deps = ["CFTime", "CommonDataModel", "DataStructures", "Dates", "DiskArrays", "NetCDF_jll", "NetworkOptions", "Printf"] +git-tree-sha1 = "2c9dc92001ac06d432f363f37ff5552954d9947c" +uuid = "85f8d34a-cbdd-5861-8df4-14fed0d494ab" +version = "0.14.6" + +[[deps.NVTX]] +deps = ["Colors", "JuliaNVTXCallbacks_jll", "Libdl", "NVTX_jll"] +git-tree-sha1 = "53046f0483375e3ed78e49190f1154fa0a4083a1" +uuid = "5da4648a-3479-48b8-97b9-01cb529c0a1f" +version = "0.3.4" + +[[deps.NVTX_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "ce3269ed42816bf18d500c9f63418d4b0d9f5a3b" +uuid = "e98f9f5b-d649-5603-91fd-7774390e6439" +version = "3.1.0+2" + +[[deps.NetCDF_jll]] +deps = ["Artifacts", "Blosc_jll", "Bzip2_jll", "HDF5_jll", "JLLWrappers", "LibCURL_jll", "Libdl", "OpenMPI_jll", "XML2_jll", "Zlib_jll", "Zstd_jll", "libzip_jll"] +git-tree-sha1 = "a8af1798e4eb9ff768ce7fdefc0e957097793f15" +uuid = "7243133f-43d8-5620-bbf4-c2c921802cf3" +version = "400.902.209+0" + +[[deps.Nettle_jll]] +deps = ["Artifacts", "GMP_jll", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "eca63e3847dad608cfa6a3329b95ef674c7160b4" +uuid = "4c82536e-c426-54e4-b420-14f461c4ed8b" +version = "3.7.2+0" + +[[deps.NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.2.0" + +[[deps.OffsetArrays]] +git-tree-sha1 = "1a27764e945a152f7ca7efa04de513d473e9542e" +uuid = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" +version = "1.14.1" +weakdeps = ["Adapt"] + + [deps.OffsetArrays.extensions] + OffsetArraysAdaptExt = "Adapt" + +[[deps.OpenBLAS_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" +version = "0.3.27+1" + +[[deps.OpenMPI_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Hwloc_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "MPIPreferences", "TOML", "Zlib_jll"] +git-tree-sha1 = "bfce6d523861a6c562721b262c0d1aaeead2647f" +uuid = "fe0851c0-eecd-5654-98d4-656369965a5c" +version = "5.0.5+0" + +[[deps.OpenSSL_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "7493f61f55a6cce7325f197443aa80d32554ba10" +uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" +version = "3.0.15+1" + +[[deps.OrderedCollections]] +git-tree-sha1 = "dfdf5519f235516220579f949664f1bf44e741c5" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.6.3" + +[[deps.P11Kit_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "2cd396108e178f3ae8dedbd8e938a18726ab2fbf" +uuid = "c2071276-7c44-58a7-b746-946036e04d0a" +version = "0.24.1+0" + +[[deps.Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.11.0" + + [deps.Pkg.extensions] + REPLExt = "REPL" + + [deps.Pkg.weakdeps] + REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[[deps.PkgVersion]] +deps = ["Pkg"] +git-tree-sha1 = "f9501cc0430a26bc3d156ae1b5b0c1b47af4d6da" +uuid = "eebad327-c553-4316-9ea0-9fa01ccd7688" +version = "0.3.3" + +[[deps.PooledArrays]] +deps = ["DataAPI", "Future"] +git-tree-sha1 = "36d8b4b899628fb92c2749eb488d884a926614d3" +uuid = "2dfb63ee-cc39-5dd5-95bd-886bf059d720" +version = "1.4.3" + +[[deps.PrecompileTools]] +deps = ["Preferences"] +git-tree-sha1 = "5aa36f7049a63a1528fe8f7c3f2113413ffd4e1f" +uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" +version = "1.2.1" + +[[deps.Preferences]] +deps = ["TOML"] +git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.4.3" + +[[deps.PrettyTables]] +deps = ["Crayons", "LaTeXStrings", "Markdown", "PrecompileTools", "Printf", "Reexport", "StringManipulation", "Tables"] +git-tree-sha1 = "1101cd475833706e4d0e7b122218257178f48f34" +uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" +version = "2.4.0" + +[[deps.Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" +version = "1.11.0" + +[[deps.Quaternions]] +deps = ["LinearAlgebra", "Random", "RealDot"] +git-tree-sha1 = "994cc27cdacca10e68feb291673ec3a76aa2fae9" +uuid = "94ee1d12-ae83-5a48-8b1c-48b8ff168ae0" +version = "0.7.6" + +[[deps.Random]] +deps = ["SHA"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +version = "1.11.0" + +[[deps.Random123]] +deps = ["Random", "RandomNumbers"] +git-tree-sha1 = "4743b43e5a9c4a2ede372de7061eed81795b12e7" +uuid = "74087812-796a-5b5d-8853-05524746bad3" +version = "1.7.0" + +[[deps.RandomNumbers]] +deps = ["Random"] +git-tree-sha1 = "c6ec94d2aaba1ab2ff983052cf6a606ca5985902" +uuid = "e6cf234a-135c-5ec9-84dd-332b85af5143" +version = "1.6.0" + +[[deps.RealDot]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "9f0a1b71baaf7650f4fa8a1d168c7fb6ee41f0c9" +uuid = "c1ae055f-0cd5-4b69-90a6-9a35b1a98df9" +version = "0.1.0" + +[[deps.RecipesBase]] +deps = ["PrecompileTools"] +git-tree-sha1 = "5c3d09cc4f31f5fc6af001c250bf1278733100ff" +uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" +version = "1.3.4" + +[[deps.Reexport]] +git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" +uuid = "189a3867-3050-52da-a836-e630ba90ab69" +version = "1.2.2" + +[[deps.Requires]] +deps = ["UUIDs"] +git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7" +uuid = "ae029012-a4dd-5104-9daa-d747884805df" +version = "1.3.0" + +[[deps.Rotations]] +deps = ["LinearAlgebra", "Quaternions", "Random", "StaticArrays"] +git-tree-sha1 = "5680a9276685d392c87407df00d57c9924d9f11e" +uuid = "6038ab10-8711-5258-84ad-4b1120ba62dc" +version = "1.7.1" +weakdeps = ["RecipesBase"] + + [deps.Rotations.extensions] + RotationsRecipesBaseExt = "RecipesBase" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.Scratch]] +deps = ["Dates"] +git-tree-sha1 = "3bac05bc7e74a75fd9cba4295cde4045d9fe2386" +uuid = "6c6a2e73-6563-6170-7368-637461726353" +version = "1.2.1" + +[[deps.SeawaterPolynomials]] +git-tree-sha1 = "78f965a2f0cd5250a20c9aba9979346dd2b35734" +uuid = "d496a93d-167e-4197-9f49-d3af4ff8fe40" +version = "0.3.5" + +[[deps.SentinelArrays]] +deps = ["Dates", "Random"] +git-tree-sha1 = "d0553ce4031a081cc42387a9b9c8441b7d99f32d" +uuid = "91c51154-3ec4-41a3-a24f-3f23e20d615c" +version = "1.4.7" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +version = "1.11.0" + +[[deps.Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" +version = "1.11.0" + +[[deps.SortingAlgorithms]] +deps = ["DataStructures"] +git-tree-sha1 = "66e0a8e672a0bdfca2c3f5937efb8538b9ddc085" +uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" +version = "1.2.1" + +[[deps.SparseArrays]] +deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] +uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +version = "1.11.0" + +[[deps.StaticArrays]] +deps = ["LinearAlgebra", "PrecompileTools", "Random", "StaticArraysCore"] +git-tree-sha1 = "777657803913ffc7e8cc20f0fd04b634f871af8f" +uuid = "90137ffa-7385-5640-81b9-e52037218182" +version = "1.9.8" + + [deps.StaticArrays.extensions] + StaticArraysChainRulesCoreExt = "ChainRulesCore" + StaticArraysStatisticsExt = "Statistics" + + [deps.StaticArrays.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" + +[[deps.StaticArraysCore]] +git-tree-sha1 = "192954ef1208c7019899fbf8049e717f92959682" +uuid = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" +version = "1.4.3" + +[[deps.Statistics]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "ae3bb1eb3bba077cd276bc5cfc337cc65c3075c0" +uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +version = "1.11.1" +weakdeps = ["SparseArrays"] + + [deps.Statistics.extensions] + SparseArraysExt = ["SparseArrays"] + +[[deps.StatsAPI]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "1ff449ad350c9c4cbc756624d6f8a8c3ef56d3ed" +uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0" +version = "1.7.0" + +[[deps.StringManipulation]] +deps = ["PrecompileTools"] +git-tree-sha1 = "a6b1675a536c5ad1a60e5a5153e1fee12eb146e3" +uuid = "892a3eda-7b42-436c-8928-eab12a02cf0e" +version = "0.4.0" + +[[deps.StructArrays]] +deps = ["ConstructionBase", "DataAPI", "Tables"] +git-tree-sha1 = "f4dc295e983502292c4c3f951dbb4e985e35b3be" +uuid = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" +version = "0.6.18" +weakdeps = ["Adapt", "GPUArraysCore", "SparseArrays", "StaticArrays"] + + [deps.StructArrays.extensions] + StructArraysAdaptExt = "Adapt" + StructArraysGPUArraysCoreExt = "GPUArraysCore" + StructArraysSparseArraysExt = "SparseArrays" + StructArraysStaticArraysExt = "StaticArrays" + +[[deps.SuiteSparse_jll]] +deps = ["Artifacts", "Libdl", "libblastrampoline_jll"] +uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" +version = "7.7.0+0" + +[[deps.TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" + +[[deps.TableTraits]] +deps = ["IteratorInterfaceExtensions"] +git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39" +uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c" +version = "1.0.1" + +[[deps.Tables]] +deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "OrderedCollections", "TableTraits"] +git-tree-sha1 = "598cd7c1f68d1e205689b1c2fe65a9f85846f297" +uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" +version = "1.12.0" + +[[deps.Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.0" + +[[deps.TaylorSeries]] +deps = ["LinearAlgebra", "Markdown", "Requires", "SparseArrays"] +git-tree-sha1 = "abc13c4d3cccd1703335ba03abdaf0dc8ecc97e2" +uuid = "6aa5eb33-94cf-58f4-a9d0-e4b2c4fc25ea" +version = "0.17.3" + + [deps.TaylorSeries.extensions] + TaylorSeriesIAExt = "IntervalArithmetic" + TaylorSeriesSAExt = "StaticArrays" + + [deps.TaylorSeries.weakdeps] + IntervalArithmetic = "d1acc4aa-44c8-5952-acd4-ba5d80a2a253" + StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" + +[[deps.Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +version = "1.11.0" + +[[deps.TimerOutputs]] +deps = ["ExprTools", "Printf"] +git-tree-sha1 = "3a6f063d690135f5c1ba351412c82bae4d1402bf" +uuid = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" +version = "0.5.25" + +[[deps.TranscodingStreams]] +git-tree-sha1 = "0c45878dcfdcfa8480052b6ab162cdd138781742" +uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" +version = "0.11.3" + +[[deps.UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" +version = "1.11.0" + +[[deps.Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" +version = "1.11.0" + +[[deps.UnsafeAtomics]] +git-tree-sha1 = "6331ac3440856ea1988316b46045303bef658278" +uuid = "013be700-e6cd-48c3-b4a1-df204f14c38f" +version = "0.2.1" + +[[deps.UnsafeAtomicsLLVM]] +deps = ["LLVM", "UnsafeAtomics"] +git-tree-sha1 = "2d17fabcd17e67d7625ce9c531fb9f40b7c42ce4" +uuid = "d80eeb9a-aca5-4d75-85e5-170c8b632249" +version = "0.2.1" + +[[deps.XML2_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Libiconv_jll", "Zlib_jll"] +git-tree-sha1 = "6a451c6f33a176150f315726eba8b92fbfdb9ae7" +uuid = "02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a" +version = "2.13.4+0" + +[[deps.XZ_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "15e637a697345f6743674f1322beefbc5dcd5cfc" +uuid = "ffd25f8a-64ca-5728-b0f7-c24cf3aae800" +version = "5.6.3+0" + +[[deps.Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.2.13+1" + +[[deps.Zstd_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "555d1076590a6cc2fdee2ef1469451f872d8b41b" +uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" +version = "1.5.6+1" + +[[deps.demumble_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "6498e3581023f8e530f34760d18f75a69e3a4ea8" +uuid = "1e29f10c-031c-5a83-9565-69cddfc27673" +version = "1.3.0+0" + +[[deps.libaec_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "46bf7be2917b59b761247be3f317ddf75e50e997" +uuid = "477f73a3-ac25-53e9-8cc3-50b2fa2566f0" +version = "1.1.2+0" + +[[deps.libblastrampoline_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" +version = "5.11.0+0" + +[[deps.libzip_jll]] +deps = ["Artifacts", "Bzip2_jll", "GnuTLS_jll", "JLLWrappers", "Libdl", "XZ_jll", "Zlib_jll", "Zstd_jll"] +git-tree-sha1 = "668ac0297e6bd8f4d53dfdcd3ace71f2e00f4a35" +uuid = "337d8026-41b4-5cde-a456-74a10e5b31d1" +version = "1.11.1+0" + +[[deps.nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.59.0+0" + +[[deps.oneTBB_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "7d0ea0f4895ef2f5cb83645fa689e52cb55cf493" +uuid = "1317d2d5-d96f-522e-a858-c73665f53c3e" +version = "2021.12.0+0" + +[[deps.p7zip_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.4.0+2" From 9e63f564d4b4c2564d22ae979622dc0a927f1cf6 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 5 Nov 2024 10:03:33 +0100 Subject: [PATCH 442/567] Manifest from julia 1.10 --- Manifest.toml | 68 +++++++++++++++------------------------------------ 1 file changed, 20 insertions(+), 48 deletions(-) diff --git a/Manifest.toml b/Manifest.toml index 35f1827b15..45d969c32d 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -1,8 +1,8 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.11.1" +julia_version = "1.10.6" manifest_format = "2.0" -project_hash = "f3d268da10603d34af6d3fea2014bfc2339ad299" +project_hash = "475d8d28ec1bf53471638e448deed724aee15ccd" [[deps.AbstractFFTs]] deps = ["LinearAlgebra"] @@ -30,11 +30,10 @@ weakdeps = ["StaticArrays"] [[deps.ArgTools]] uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" -version = "1.1.2" +version = "1.1.1" [[deps.Artifacts]] uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" -version = "1.11.0" [[deps.Atomix]] deps = ["UnsafeAtomics"] @@ -50,7 +49,6 @@ version = "0.5.0" [[deps.Base64]] uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" -version = "1.11.0" [[deps.Blosc_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Lz4_jll", "Zlib_jll", "Zstd_jll"] @@ -193,7 +191,6 @@ version = "1.0.0" [[deps.Dates]] deps = ["Printf"] uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" -version = "1.11.0" [[deps.DiskArrays]] deps = ["LRUCache", "OffsetArrays"] @@ -218,7 +215,6 @@ version = "0.10.12" [[deps.Distributed]] deps = ["Random", "Serialization", "Sockets"] uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" -version = "1.11.0" [[deps.DocStringExtensions]] deps = ["LibGit2"] @@ -256,7 +252,6 @@ version = "1.16.4" [[deps.FileWatching]] uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" -version = "1.11.0" [[deps.FixedPointNumbers]] deps = ["Statistics"] @@ -267,12 +262,11 @@ version = "0.8.5" [[deps.Future]] deps = ["Random"] uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" -version = "1.11.0" [[deps.GMP_jll]] deps = ["Artifacts", "Libdl"] uuid = "781609d7-10c4-51f6-84f2-b8444358ff6d" -version = "6.3.0+0" +version = "6.2.1+6" [[deps.GPUArrays]] deps = ["Adapt", "GPUArraysCore", "LLVM", "LinearAlgebra", "Printf", "Random", "Reexport", "Serialization", "Statistics"] @@ -343,7 +337,6 @@ version = "2024.2.1+0" [[deps.InteractiveUtils]] deps = ["Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" -version = "1.11.0" [[deps.InvertedIndices]] git-tree-sha1 = "0dc7b50b8d436461be01300fd8cd45aa0274b038" @@ -439,7 +432,6 @@ version = "1.4.0" [[deps.LazyArtifacts]] deps = ["Artifacts", "Pkg"] uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" -version = "1.11.0" [[deps.LibCURL]] deps = ["LibCURL_jll", "MozillaCACerts_jll"] @@ -449,17 +441,16 @@ version = "0.6.4" [[deps.LibCURL_jll]] deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" -version = "8.6.0+0" +version = "8.4.0+0" [[deps.LibGit2]] deps = ["Base64", "LibGit2_jll", "NetworkOptions", "Printf", "SHA"] uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" -version = "1.11.0" [[deps.LibGit2_jll]] deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"] uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" -version = "1.7.2+0" +version = "1.6.4+0" [[deps.LibSSH2_jll]] deps = ["Artifacts", "Libdl", "MbedTLS_jll"] @@ -468,7 +459,6 @@ version = "1.11.0+1" [[deps.Libdl]] uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" -version = "1.11.0" [[deps.Libiconv_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] @@ -479,11 +469,9 @@ version = "1.17.0+1" [[deps.LinearAlgebra]] deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" -version = "1.11.0" [[deps.Logging]] uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" -version = "1.11.0" [[deps.Lz4_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] @@ -538,12 +526,11 @@ version = "0.5.13" [[deps.Markdown]] deps = ["Base64"] uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" -version = "1.11.0" [[deps.MbedTLS_jll]] deps = ["Artifacts", "Libdl"] uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" -version = "2.28.6+0" +version = "2.28.2+1" [[deps.MicrosoftMPI_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] @@ -559,11 +546,10 @@ version = "1.2.0" [[deps.Mmap]] uuid = "a63ad114-7e13-5084-954f-fe012c677804" -version = "1.11.0" [[deps.MozillaCACerts_jll]] uuid = "14a3606d-f60d-562e-9121-12d972cd8159" -version = "2023.12.12" +version = "2023.1.10" [[deps.NCDatasets]] deps = ["CFTime", "CommonDataModel", "DataStructures", "Dates", "DiskArrays", "NetCDF_jll", "NetworkOptions", "Printf"] @@ -611,7 +597,7 @@ weakdeps = ["Adapt"] [[deps.OpenBLAS_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" -version = "0.3.27+1" +version = "0.3.23+4" [[deps.OpenMPI_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "Hwloc_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "MPIPreferences", "TOML", "Zlib_jll"] @@ -637,15 +623,9 @@ uuid = "c2071276-7c44-58a7-b746-946036e04d0a" version = "0.24.1+0" [[deps.Pkg]] -deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "TOML", "Tar", "UUIDs", "p7zip_jll"] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -version = "1.11.0" - - [deps.Pkg.extensions] - REPLExt = "REPL" - - [deps.Pkg.weakdeps] - REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" +version = "1.10.0" [[deps.PkgVersion]] deps = ["Pkg"] @@ -680,7 +660,6 @@ version = "2.4.0" [[deps.Printf]] deps = ["Unicode"] uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" -version = "1.11.0" [[deps.Quaternions]] deps = ["LinearAlgebra", "Random", "RealDot"] @@ -688,10 +667,13 @@ git-tree-sha1 = "994cc27cdacca10e68feb291673ec3a76aa2fae9" uuid = "94ee1d12-ae83-5a48-8b1c-48b8ff168ae0" version = "0.7.6" +[[deps.REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + [[deps.Random]] deps = ["SHA"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" -version = "1.11.0" [[deps.Random123]] deps = ["Random", "RandomNumbers"] @@ -761,11 +743,9 @@ version = "1.4.7" [[deps.Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" -version = "1.11.0" [[deps.Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" -version = "1.11.0" [[deps.SortingAlgorithms]] deps = ["DataStructures"] @@ -776,7 +756,7 @@ version = "1.2.1" [[deps.SparseArrays]] deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" -version = "1.11.0" +version = "1.10.0" [[deps.StaticArrays]] deps = ["LinearAlgebra", "PrecompileTools", "Random", "StaticArraysCore"] @@ -798,14 +778,9 @@ uuid = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" version = "1.4.3" [[deps.Statistics]] -deps = ["LinearAlgebra"] -git-tree-sha1 = "ae3bb1eb3bba077cd276bc5cfc337cc65c3075c0" +deps = ["LinearAlgebra", "SparseArrays"] uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" -version = "1.11.1" -weakdeps = ["SparseArrays"] - - [deps.Statistics.extensions] - SparseArraysExt = ["SparseArrays"] +version = "1.10.0" [[deps.StatsAPI]] deps = ["LinearAlgebra"] @@ -835,7 +810,7 @@ weakdeps = ["Adapt", "GPUArraysCore", "SparseArrays", "StaticArrays"] [[deps.SuiteSparse_jll]] deps = ["Artifacts", "Libdl", "libblastrampoline_jll"] uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" -version = "7.7.0+0" +version = "7.2.1+1" [[deps.TOML]] deps = ["Dates"] @@ -876,7 +851,6 @@ version = "0.17.3" [[deps.Test]] deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" -version = "1.11.0" [[deps.TimerOutputs]] deps = ["ExprTools", "Printf"] @@ -892,11 +866,9 @@ version = "0.11.3" [[deps.UUIDs]] deps = ["Random", "SHA"] uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" -version = "1.11.0" [[deps.Unicode]] uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" -version = "1.11.0" [[deps.UnsafeAtomics]] git-tree-sha1 = "6331ac3440856ea1988316b46045303bef658278" @@ -958,7 +930,7 @@ version = "1.11.1+0" [[deps.nghttp2_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" -version = "1.59.0+0" +version = "1.52.0+1" [[deps.oneTBB_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] From 59548f880a81a59c172f20dbcc61a737d3cd24e0 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 5 Nov 2024 10:06:01 +0100 Subject: [PATCH 443/567] we probably need to initialize on a GPU --- .buildkite/distributed/pipeline.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index e0ea62eef4..6c82558a3e 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -5,13 +5,15 @@ agents: env: JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite/distributed" - JULIA_MPI_HAS_CUDA: "true" OPENBLAS_NUM_THREADS: 1 OMPI_MCA_opal_warn_on_missing_libcuda: 0 steps: - label: "initialize" key: "init_central" + env: + TEST_GROUP: "init" + GPU_TEST: "true" command: - echo "--- Instantiate project" - "julia --project -e 'using Pkg; Pkg.instantiate(; verbose=true); Pkg.precompile(; strict=true)'" @@ -25,6 +27,10 @@ steps: - echo "--- Instantiate status" - "julia --project -e 'using Pkg; Pkg.status()'" + agents: + slurm_mem: 120G + slurm_ntasks: 1 + slurm_gpus_per_task: 1 - wait From 642cfd9d594dc7503e4e5ad187cda800c87a5b26 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 5 Nov 2024 10:06:43 +0100 Subject: [PATCH 444/567] these options should not create problems --- .buildkite/distributed/pipeline.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 6c82558a3e..6b993bbec2 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -6,6 +6,9 @@ agents: env: JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite/distributed" OPENBLAS_NUM_THREADS: 1 + JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager + JULIA_NUM_PRECOMPILE_TASKS: 8 + JULIA_NUM_THREADS: 8 OMPI_MCA_opal_warn_on_missing_libcuda: 0 steps: From db58da1730a71310646671a145e24b1a2d5b6048 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 5 Nov 2024 11:41:04 +0100 Subject: [PATCH 445/567] restart the tests --- src/ImmersedBoundaries/grid_fitted_bottom.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ImmersedBoundaries/grid_fitted_bottom.jl b/src/ImmersedBoundaries/grid_fitted_bottom.jl index af62726ab3..b2b1b2d2fc 100644 --- a/src/ImmersedBoundaries/grid_fitted_bottom.jl +++ b/src/ImmersedBoundaries/grid_fitted_bottom.jl @@ -15,8 +15,8 @@ abstract type AbstractGridFittedBottom{H} <: AbstractGridFittedBoundary end # To enable comparison with PartialCellBottom in the limiting case that # fractional cell height is 1.0. -struct InterfaceImmersedCondition end struct CenterImmersedCondition end +struct InterfaceImmersedCondition end struct GridFittedBottom{H, I} <: AbstractGridFittedBottom{H} bottom_height :: H From 4cee49ab8be4408fbba12cd23f0c826c48b927d1 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 5 Nov 2024 17:18:36 +0100 Subject: [PATCH 446/567] let's see if this differs --- test/runtests.jl | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index 11a4e553e8..f4f0d532ca 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -179,7 +179,18 @@ CUDA.allowscalar() do if group == :distributed_solvers || group == :all MPI.Initialized() || MPI.Init() - CUDA.set_runtime_version!(v"12.6") + Pkg.instantiate(; verbose=true) + Pkg.precompile(; strict=true) + Pkg.status() + + try + MPI.versioninfo() + catch; end + + try + CUDA.precompile_runtime() + CUDA.versioninfo() + catch; end include("test_distributed_transpose.jl") include("test_distributed_poisson_solvers.jl") end From a46b25dee593075c6a60e0f58728cd6d7866accc Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 5 Nov 2024 18:57:36 +0100 Subject: [PATCH 447/567] just version infos --- test/runtests.jl | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index f4f0d532ca..69de014ec9 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -28,8 +28,6 @@ CUDA.allowscalar() do # Initialization steps if group == :init || group == :all - Pkg.instantiate(; verbose=true) - Pkg.precompile(; strict=true) Pkg.status() try @@ -37,7 +35,6 @@ CUDA.allowscalar() do catch; end try - CUDA.precompile_runtime() CUDA.versioninfo() catch; end end From 4dffbe5d40cf36bb54c974ef3cf51da9013315c0 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 6 Nov 2024 09:28:16 +0100 Subject: [PATCH 448/567] fiddling with O0 --- .buildkite/distributed/pipeline.yml | 6 +- Manifest.toml | 944 ---------------------------- 2 files changed, 3 insertions(+), 947 deletions(-) delete mode 100644 Manifest.toml diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 6b993bbec2..7c892803a1 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -19,11 +19,11 @@ steps: GPU_TEST: "true" command: - echo "--- Instantiate project" - - "julia --project -e 'using Pkg; Pkg.instantiate(; verbose=true); Pkg.precompile(; strict=true)'" + - "julia -O0 --project -e 'using Pkg; Pkg.instantiate(; verbose=true); Pkg.precompile(; strict=true)'" # Force the initialization of the CUDA runtime as it is lazily loaded by default - - "julia --project -e 'using CUDA; CUDA.precompile_runtime(); CUDA.versioninfo()'" - - "julia --project -e 'using MPI; MPI.versioninfo()'" + - "julia -O0 --project -e 'using CUDA; CUDA.precompile_runtime(); CUDA.versioninfo()'" + - "julia -O0 --project -e 'using MPI; MPI.versioninfo()'" # Download artifacts by running an empty testgroup and thereby executing /test/runtests.jl - "julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" diff --git a/Manifest.toml b/Manifest.toml deleted file mode 100644 index 45d969c32d..0000000000 --- a/Manifest.toml +++ /dev/null @@ -1,944 +0,0 @@ -# This file is machine-generated - editing it directly is not advised - -julia_version = "1.10.6" -manifest_format = "2.0" -project_hash = "475d8d28ec1bf53471638e448deed724aee15ccd" - -[[deps.AbstractFFTs]] -deps = ["LinearAlgebra"] -git-tree-sha1 = "d92ad398961a3ed262d8bf04a1a2b8340f915fef" -uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c" -version = "1.5.0" - - [deps.AbstractFFTs.extensions] - AbstractFFTsChainRulesCoreExt = "ChainRulesCore" - AbstractFFTsTestExt = "Test" - - [deps.AbstractFFTs.weakdeps] - ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" - Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" - -[[deps.Adapt]] -deps = ["LinearAlgebra", "Requires"] -git-tree-sha1 = "50c3c56a52972d78e8be9fd135bfb91c9574c140" -uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" -version = "4.1.1" -weakdeps = ["StaticArrays"] - - [deps.Adapt.extensions] - AdaptStaticArraysExt = "StaticArrays" - -[[deps.ArgTools]] -uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" -version = "1.1.1" - -[[deps.Artifacts]] -uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" - -[[deps.Atomix]] -deps = ["UnsafeAtomics"] -git-tree-sha1 = "c06a868224ecba914baa6942988e2f2aade419be" -uuid = "a9b6321e-bd34-4604-b9c9-b65b8de01458" -version = "0.1.0" - -[[deps.BFloat16s]] -deps = ["LinearAlgebra", "Printf", "Random", "Test"] -git-tree-sha1 = "2c7cc21e8678eff479978a0a2ef5ce2f51b63dff" -uuid = "ab4f0b2a-ad5b-11e8-123f-65d77653426b" -version = "0.5.0" - -[[deps.Base64]] -uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" - -[[deps.Blosc_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Lz4_jll", "Zlib_jll", "Zstd_jll"] -git-tree-sha1 = "ef12cdd1c7fb7e1dfd6fa8fd60d4db6bc61d2f23" -uuid = "0b7ba130-8d10-5ba8-a3d6-c5182647fed9" -version = "1.21.6+0" - -[[deps.Bzip2_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "8873e196c2eb87962a2048b3b8e08946535864a1" -uuid = "6e34b625-4abd-537c-b88f-471c36dfa7a0" -version = "1.0.8+2" - -[[deps.CEnum]] -git-tree-sha1 = "389ad5c84de1ae7cf0e28e381131c98ea87d54fc" -uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82" -version = "0.5.0" - -[[deps.CFTime]] -deps = ["Dates", "Printf"] -git-tree-sha1 = "5afb5c5ba2688ca43a9ad2e5a91cbb93921ccfa1" -uuid = "179af706-886a-5703-950a-314cd64e0468" -version = "0.1.3" - -[[deps.CUDA]] -deps = ["AbstractFFTs", "Adapt", "BFloat16s", "CEnum", "CUDA_Driver_jll", "CUDA_Runtime_Discovery", "CUDA_Runtime_jll", "Crayons", "DataFrames", "ExprTools", "GPUArrays", "GPUCompiler", "KernelAbstractions", "LLVM", "LLVMLoopInfo", "LazyArtifacts", "Libdl", "LinearAlgebra", "Logging", "NVTX", "Preferences", "PrettyTables", "Printf", "Random", "Random123", "RandomNumbers", "Reexport", "Requires", "SparseArrays", "StaticArrays", "Statistics", "demumble_jll"] -git-tree-sha1 = "e0725a467822697171af4dae15cec10b4fc19053" -uuid = "052768ef-5323-5732-b1bb-66c8b64840ba" -version = "5.5.2" - - [deps.CUDA.extensions] - ChainRulesCoreExt = "ChainRulesCore" - EnzymeCoreExt = "EnzymeCore" - SpecialFunctionsExt = "SpecialFunctions" - - [deps.CUDA.weakdeps] - ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" - EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869" - SpecialFunctions = "276daf66-3868-5448-9aa4-cd146d93841b" - -[[deps.CUDA_Driver_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "ccd1e54610c222fadfd4737dac66bff786f63656" -uuid = "4ee394cb-3365-5eb0-8335-949819d2adfc" -version = "0.10.3+0" - -[[deps.CUDA_Runtime_Discovery]] -deps = ["Libdl"] -git-tree-sha1 = "33576c7c1b2500f8e7e6baa082e04563203b3a45" -uuid = "1af6417a-86b4-443c-805f-a4643ffb695f" -version = "0.3.5" - -[[deps.CUDA_Runtime_jll]] -deps = ["Artifacts", "CUDA_Driver_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] -git-tree-sha1 = "e43727b237b2879a34391eeb81887699a26f8f2f" -uuid = "76a88914-d11a-5bdc-97e0-2f5a05c973a2" -version = "0.15.3+0" - -[[deps.ColorTypes]] -deps = ["FixedPointNumbers", "Random"] -git-tree-sha1 = "b10d0b65641d57b8b4d5e234446582de5047050d" -uuid = "3da002f7-5984-5a60-b8a6-cbb66c0b333f" -version = "0.11.5" - -[[deps.Colors]] -deps = ["ColorTypes", "FixedPointNumbers", "Reexport"] -git-tree-sha1 = "362a287c3aa50601b0bc359053d5c2468f0e7ce0" -uuid = "5ae59095-9a9b-59fe-a467-6f913c188581" -version = "0.12.11" - -[[deps.CommonDataModel]] -deps = ["CFTime", "DataStructures", "Dates", "Preferences", "Printf", "Statistics"] -git-tree-sha1 = "d6fb5bf939a2753c74984b11434ea25d6c397a58" -uuid = "1fbeeb36-5f17-413c-809b-666fb144f157" -version = "0.3.6" - -[[deps.Compat]] -deps = ["TOML", "UUIDs"] -git-tree-sha1 = "8ae8d32e09f0dcf42a36b90d4e17f5dd2e4c4215" -uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" -version = "4.16.0" -weakdeps = ["Dates", "LinearAlgebra"] - - [deps.Compat.extensions] - CompatLinearAlgebraExt = "LinearAlgebra" - -[[deps.CompilerSupportLibraries_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" -version = "1.1.1+0" - -[[deps.ConstructionBase]] -git-tree-sha1 = "76219f1ed5771adbb096743bff43fb5fdd4c1157" -uuid = "187b0558-2788-49d3-abe0-74a17ed4e7c9" -version = "1.5.8" - - [deps.ConstructionBase.extensions] - ConstructionBaseIntervalSetsExt = "IntervalSets" - ConstructionBaseLinearAlgebraExt = "LinearAlgebra" - ConstructionBaseStaticArraysExt = "StaticArrays" - - [deps.ConstructionBase.weakdeps] - IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953" - LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" - StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" - -[[deps.Crayons]] -git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" -uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" -version = "4.1.1" - -[[deps.CubedSphere]] -deps = ["TaylorSeries"] -git-tree-sha1 = "51bb25de518b4c62b7cdf26e5fbb84601bb27a60" -uuid = "7445602f-e544-4518-8976-18f8e8ae6cdb" -version = "0.3.0" - -[[deps.DataAPI]] -git-tree-sha1 = "abe83f3a2f1b857aac70ef8b269080af17764bbe" -uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" -version = "1.16.0" - -[[deps.DataFrames]] -deps = ["Compat", "DataAPI", "DataStructures", "Future", "InlineStrings", "InvertedIndices", "IteratorInterfaceExtensions", "LinearAlgebra", "Markdown", "Missings", "PooledArrays", "PrecompileTools", "PrettyTables", "Printf", "Random", "Reexport", "SentinelArrays", "SortingAlgorithms", "Statistics", "TableTraits", "Tables", "Unicode"] -git-tree-sha1 = "fb61b4812c49343d7ef0b533ba982c46021938a6" -uuid = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" -version = "1.7.0" - -[[deps.DataStructures]] -deps = ["Compat", "InteractiveUtils", "OrderedCollections"] -git-tree-sha1 = "1d0a14036acb104d9e89698bd408f63ab58cdc82" -uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" -version = "0.18.20" - -[[deps.DataValueInterfaces]] -git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" -uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464" -version = "1.0.0" - -[[deps.Dates]] -deps = ["Printf"] -uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" - -[[deps.DiskArrays]] -deps = ["LRUCache", "OffsetArrays"] -git-tree-sha1 = "e0e89a60637a62d13aa2107f0acd169b9b9b77e7" -uuid = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3" -version = "0.4.6" - -[[deps.Distances]] -deps = ["LinearAlgebra", "Statistics", "StatsAPI"] -git-tree-sha1 = "c7e3a542b999843086e2f29dac96a618c105be1d" -uuid = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7" -version = "0.10.12" - - [deps.Distances.extensions] - DistancesChainRulesCoreExt = "ChainRulesCore" - DistancesSparseArraysExt = "SparseArrays" - - [deps.Distances.weakdeps] - ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" - SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" - -[[deps.Distributed]] -deps = ["Random", "Serialization", "Sockets"] -uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" - -[[deps.DocStringExtensions]] -deps = ["LibGit2"] -git-tree-sha1 = "2fb1e02f2b635d0845df5d7c167fec4dd739b00d" -uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" -version = "0.9.3" - -[[deps.Downloads]] -deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] -uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" -version = "1.6.0" - -[[deps.ExprTools]] -git-tree-sha1 = "27415f162e6028e81c72b82ef756bf321213b6ec" -uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04" -version = "0.1.10" - -[[deps.FFTW]] -deps = ["AbstractFFTs", "FFTW_jll", "LinearAlgebra", "MKL_jll", "Preferences", "Reexport"] -git-tree-sha1 = "4820348781ae578893311153d69049a93d05f39d" -uuid = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" -version = "1.8.0" - -[[deps.FFTW_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "4d81ed14783ec49ce9f2e168208a12ce1815aa25" -uuid = "f5851436-0d7a-5f13-b9de-f02708fd171a" -version = "3.3.10+1" - -[[deps.FileIO]] -deps = ["Pkg", "Requires", "UUIDs"] -git-tree-sha1 = "62ca0547a14c57e98154423419d8a342dca75ca9" -uuid = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" -version = "1.16.4" - -[[deps.FileWatching]] -uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" - -[[deps.FixedPointNumbers]] -deps = ["Statistics"] -git-tree-sha1 = "05882d6995ae5c12bb5f36dd2ed3f61c98cbb172" -uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93" -version = "0.8.5" - -[[deps.Future]] -deps = ["Random"] -uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" - -[[deps.GMP_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "781609d7-10c4-51f6-84f2-b8444358ff6d" -version = "6.2.1+6" - -[[deps.GPUArrays]] -deps = ["Adapt", "GPUArraysCore", "LLVM", "LinearAlgebra", "Printf", "Random", "Reexport", "Serialization", "Statistics"] -git-tree-sha1 = "62ee71528cca49be797076a76bdc654a170a523e" -uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" -version = "10.3.1" - -[[deps.GPUArraysCore]] -deps = ["Adapt"] -git-tree-sha1 = "ec632f177c0d990e64d955ccc1b8c04c485a0950" -uuid = "46192b85-c4d5-4398-a991-12ede77f4527" -version = "0.1.6" - -[[deps.GPUCompiler]] -deps = ["ExprTools", "InteractiveUtils", "LLVM", "Libdl", "Logging", "PrecompileTools", "Preferences", "Scratch", "Serialization", "TOML", "TimerOutputs", "UUIDs"] -git-tree-sha1 = "1d6f290a5eb1201cd63574fbc4440c788d5cb38f" -uuid = "61eb1bfa-7361-4325-ad38-22787b887f55" -version = "0.27.8" - -[[deps.Glob]] -git-tree-sha1 = "97285bbd5230dd766e9ef6749b80fc617126d496" -uuid = "c27321d9-0574-5035-807b-f59d2c89b15c" -version = "1.3.1" - -[[deps.GnuTLS_jll]] -deps = ["Artifacts", "GMP_jll", "JLLWrappers", "Libdl", "Nettle_jll", "P11Kit_jll", "Zlib_jll"] -git-tree-sha1 = "383db7d3f900f4c1f47a8a04115b053c095e48d3" -uuid = "0951126a-58fd-58f1-b5b3-b08c7c4a876d" -version = "3.8.4+0" - -[[deps.HDF5_jll]] -deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "LLVMOpenMP_jll", "LazyArtifacts", "LibCURL_jll", "Libdl", "MPICH_jll", "MPIPreferences", "MPItrampoline_jll", "MicrosoftMPI_jll", "OpenMPI_jll", "OpenSSL_jll", "TOML", "Zlib_jll", "libaec_jll"] -git-tree-sha1 = "38c8874692d48d5440d5752d6c74b0c6b0b60739" -uuid = "0234f1f7-429e-5d53-9886-15a909be8d59" -version = "1.14.2+1" - -[[deps.Hwloc_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "dd3b49277ec2bb2c6b94eb1604d4d0616016f7a6" -uuid = "e33a78d0-f292-5ffc-b300-72abe9b543c8" -version = "2.11.2+0" - -[[deps.IncompleteLU]] -deps = ["LinearAlgebra", "SparseArrays"] -git-tree-sha1 = "6c676e79f98abb6d33fa28122cad099f1e464afe" -uuid = "40713840-3770-5561-ab4c-a76e7d0d7895" -version = "0.2.1" - -[[deps.InlineStrings]] -git-tree-sha1 = "45521d31238e87ee9f9732561bfee12d4eebd52d" -uuid = "842dd82b-1e85-43dc-bf29-5d0ee9dffc48" -version = "1.4.2" - - [deps.InlineStrings.extensions] - ArrowTypesExt = "ArrowTypes" - ParsersExt = "Parsers" - - [deps.InlineStrings.weakdeps] - ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd" - Parsers = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" - -[[deps.IntelOpenMP_jll]] -deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl"] -git-tree-sha1 = "10bd689145d2c3b2a9844005d01087cc1194e79e" -uuid = "1d5cc7b8-4909-519e-a0f8-d0f5ad9712d0" -version = "2024.2.1+0" - -[[deps.InteractiveUtils]] -deps = ["Markdown"] -uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" - -[[deps.InvertedIndices]] -git-tree-sha1 = "0dc7b50b8d436461be01300fd8cd45aa0274b038" -uuid = "41ab1584-1d38-5bbf-9106-f11c6c58b48f" -version = "1.3.0" - -[[deps.IterativeSolvers]] -deps = ["LinearAlgebra", "Printf", "Random", "RecipesBase", "SparseArrays"] -git-tree-sha1 = "59545b0a2b27208b0650df0a46b8e3019f85055b" -uuid = "42fd0dbc-a981-5370-80f2-aaf504508153" -version = "0.9.4" - -[[deps.IteratorInterfaceExtensions]] -git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" -uuid = "82899510-4779-5014-852e-03e436cf321d" -version = "1.0.0" - -[[deps.JLD2]] -deps = ["FileIO", "MacroTools", "Mmap", "OrderedCollections", "PrecompileTools", "Requires", "TranscodingStreams"] -git-tree-sha1 = "783c1be5213a09609b23237a0c9e5dfd258ae6f2" -uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819" -version = "0.5.7" - -[[deps.JLLWrappers]] -deps = ["Artifacts", "Preferences"] -git-tree-sha1 = "be3dc50a92e5a386872a493a10050136d4703f9b" -uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.6.1" - -[[deps.JuliaNVTXCallbacks_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "af433a10f3942e882d3c671aacb203e006a5808f" -uuid = "9c1d0b0a-7046-5b2e-a33f-ea22f176ac7e" -version = "0.2.1+0" - -[[deps.KernelAbstractions]] -deps = ["Adapt", "Atomix", "InteractiveUtils", "MacroTools", "PrecompileTools", "Requires", "StaticArrays", "UUIDs", "UnsafeAtomics", "UnsafeAtomicsLLVM"] -git-tree-sha1 = "e73a077abc7fe798fe940deabe30ef6c66bdde52" -uuid = "63c18a36-062a-441e-b654-da1e3ab1ce7c" -version = "0.9.29" - - [deps.KernelAbstractions.extensions] - EnzymeExt = "EnzymeCore" - LinearAlgebraExt = "LinearAlgebra" - SparseArraysExt = "SparseArrays" - - [deps.KernelAbstractions.weakdeps] - EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869" - LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" - SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" - -[[deps.LLVM]] -deps = ["CEnum", "LLVMExtra_jll", "Libdl", "Preferences", "Printf", "Unicode"] -git-tree-sha1 = "d422dfd9707bec6617335dc2ea3c5172a87d5908" -uuid = "929cbde3-209d-540e-8aea-75f648917ca0" -version = "9.1.3" -weakdeps = ["BFloat16s"] - - [deps.LLVM.extensions] - BFloat16sExt = "BFloat16s" - -[[deps.LLVMExtra_jll]] -deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] -git-tree-sha1 = "05a8bd5a42309a9ec82f700876903abce1017dd3" -uuid = "dad2f222-ce93-54a1-a47d-0025e8a3acab" -version = "0.0.34+0" - -[[deps.LLVMLoopInfo]] -git-tree-sha1 = "2e5c102cfc41f48ae4740c7eca7743cc7e7b75ea" -uuid = "8b046642-f1f6-4319-8d3c-209ddc03c586" -version = "1.0.0" - -[[deps.LLVMOpenMP_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "78211fb6cbc872f77cad3fc0b6cf647d923f4929" -uuid = "1d63c593-3942-5779-bab2-d838dc0a180e" -version = "18.1.7+0" - -[[deps.LRUCache]] -git-tree-sha1 = "b3cc6698599b10e652832c2f23db3cab99d51b59" -uuid = "8ac3fa9e-de4c-5943-b1dc-09c6b5f20637" -version = "1.6.1" -weakdeps = ["Serialization"] - - [deps.LRUCache.extensions] - SerializationExt = ["Serialization"] - -[[deps.LaTeXStrings]] -git-tree-sha1 = "dda21b8cbd6a6c40d9d02a73230f9d70fed6918c" -uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" -version = "1.4.0" - -[[deps.LazyArtifacts]] -deps = ["Artifacts", "Pkg"] -uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" - -[[deps.LibCURL]] -deps = ["LibCURL_jll", "MozillaCACerts_jll"] -uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" -version = "0.6.4" - -[[deps.LibCURL_jll]] -deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] -uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" -version = "8.4.0+0" - -[[deps.LibGit2]] -deps = ["Base64", "LibGit2_jll", "NetworkOptions", "Printf", "SHA"] -uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" - -[[deps.LibGit2_jll]] -deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"] -uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" -version = "1.6.4+0" - -[[deps.LibSSH2_jll]] -deps = ["Artifacts", "Libdl", "MbedTLS_jll"] -uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" -version = "1.11.0+1" - -[[deps.Libdl]] -uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" - -[[deps.Libiconv_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "61dfdba58e585066d8bce214c5a51eaa0539f269" -uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531" -version = "1.17.0+1" - -[[deps.LinearAlgebra]] -deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] -uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" - -[[deps.Logging]] -uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" - -[[deps.Lz4_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "abf88ff67f4fd89839efcae2f4c39cbc4ecd0846" -uuid = "5ced341a-0733-55b8-9ab6-a4889d929147" -version = "1.10.0+1" - -[[deps.MKL_jll]] -deps = ["Artifacts", "IntelOpenMP_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "oneTBB_jll"] -git-tree-sha1 = "f046ccd0c6db2832a9f639e2c669c6fe867e5f4f" -uuid = "856f044c-d86e-5d09-b602-aeab76dc8ba7" -version = "2024.2.0+0" - -[[deps.MPI]] -deps = ["Distributed", "DocStringExtensions", "Libdl", "MPICH_jll", "MPIPreferences", "MPItrampoline_jll", "MicrosoftMPI_jll", "OpenMPI_jll", "PkgVersion", "PrecompileTools", "Requires", "Serialization", "Sockets"] -git-tree-sha1 = "892676019c58f34e38743bc989b0eca5bce5edc5" -uuid = "da04e1cc-30fd-572f-bb4f-1f8673147195" -version = "0.20.22" - - [deps.MPI.extensions] - AMDGPUExt = "AMDGPU" - CUDAExt = "CUDA" - - [deps.MPI.weakdeps] - AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e" - CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" - -[[deps.MPICH_jll]] -deps = ["Artifacts", "CompilerSupportLibraries_jll", "Hwloc_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "MPIPreferences", "TOML"] -git-tree-sha1 = "7715e65c47ba3941c502bffb7f266a41a7f54423" -uuid = "7cb0a576-ebde-5e09-9194-50597f1243b4" -version = "4.2.3+0" - -[[deps.MPIPreferences]] -deps = ["Libdl", "Preferences"] -git-tree-sha1 = "c105fe467859e7f6e9a852cb15cb4301126fac07" -uuid = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267" -version = "0.1.11" - -[[deps.MPItrampoline_jll]] -deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "MPIPreferences", "TOML"] -git-tree-sha1 = "70e830dab5d0775183c99fc75e4c24c614ed7142" -uuid = "f1f71cc9-e9ae-5b93-9b94-4fe0e1ad3748" -version = "5.5.1+0" - -[[deps.MacroTools]] -deps = ["Markdown", "Random"] -git-tree-sha1 = "2fa9ee3e63fd3a4f7a9a4f4744a52f4856de82df" -uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" -version = "0.5.13" - -[[deps.Markdown]] -deps = ["Base64"] -uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" - -[[deps.MbedTLS_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" -version = "2.28.2+1" - -[[deps.MicrosoftMPI_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "f12a29c4400ba812841c6ace3f4efbb6dbb3ba01" -uuid = "9237b28f-5490-5468-be7b-bb81f5f5e6cf" -version = "10.1.4+2" - -[[deps.Missings]] -deps = ["DataAPI"] -git-tree-sha1 = "ec4f7fbeab05d7747bdf98eb74d130a2a2ed298d" -uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" -version = "1.2.0" - -[[deps.Mmap]] -uuid = "a63ad114-7e13-5084-954f-fe012c677804" - -[[deps.MozillaCACerts_jll]] -uuid = "14a3606d-f60d-562e-9121-12d972cd8159" -version = "2023.1.10" - -[[deps.NCDatasets]] -deps = ["CFTime", "CommonDataModel", "DataStructures", "Dates", "DiskArrays", "NetCDF_jll", "NetworkOptions", "Printf"] -git-tree-sha1 = "2c9dc92001ac06d432f363f37ff5552954d9947c" -uuid = "85f8d34a-cbdd-5861-8df4-14fed0d494ab" -version = "0.14.6" - -[[deps.NVTX]] -deps = ["Colors", "JuliaNVTXCallbacks_jll", "Libdl", "NVTX_jll"] -git-tree-sha1 = "53046f0483375e3ed78e49190f1154fa0a4083a1" -uuid = "5da4648a-3479-48b8-97b9-01cb529c0a1f" -version = "0.3.4" - -[[deps.NVTX_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "ce3269ed42816bf18d500c9f63418d4b0d9f5a3b" -uuid = "e98f9f5b-d649-5603-91fd-7774390e6439" -version = "3.1.0+2" - -[[deps.NetCDF_jll]] -deps = ["Artifacts", "Blosc_jll", "Bzip2_jll", "HDF5_jll", "JLLWrappers", "LibCURL_jll", "Libdl", "OpenMPI_jll", "XML2_jll", "Zlib_jll", "Zstd_jll", "libzip_jll"] -git-tree-sha1 = "a8af1798e4eb9ff768ce7fdefc0e957097793f15" -uuid = "7243133f-43d8-5620-bbf4-c2c921802cf3" -version = "400.902.209+0" - -[[deps.Nettle_jll]] -deps = ["Artifacts", "GMP_jll", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "eca63e3847dad608cfa6a3329b95ef674c7160b4" -uuid = "4c82536e-c426-54e4-b420-14f461c4ed8b" -version = "3.7.2+0" - -[[deps.NetworkOptions]] -uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" -version = "1.2.0" - -[[deps.OffsetArrays]] -git-tree-sha1 = "1a27764e945a152f7ca7efa04de513d473e9542e" -uuid = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" -version = "1.14.1" -weakdeps = ["Adapt"] - - [deps.OffsetArrays.extensions] - OffsetArraysAdaptExt = "Adapt" - -[[deps.OpenBLAS_jll]] -deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] -uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" -version = "0.3.23+4" - -[[deps.OpenMPI_jll]] -deps = ["Artifacts", "CompilerSupportLibraries_jll", "Hwloc_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "MPIPreferences", "TOML", "Zlib_jll"] -git-tree-sha1 = "bfce6d523861a6c562721b262c0d1aaeead2647f" -uuid = "fe0851c0-eecd-5654-98d4-656369965a5c" -version = "5.0.5+0" - -[[deps.OpenSSL_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "7493f61f55a6cce7325f197443aa80d32554ba10" -uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "3.0.15+1" - -[[deps.OrderedCollections]] -git-tree-sha1 = "dfdf5519f235516220579f949664f1bf44e741c5" -uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" -version = "1.6.3" - -[[deps.P11Kit_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "2cd396108e178f3ae8dedbd8e938a18726ab2fbf" -uuid = "c2071276-7c44-58a7-b746-946036e04d0a" -version = "0.24.1+0" - -[[deps.Pkg]] -deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] -uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -version = "1.10.0" - -[[deps.PkgVersion]] -deps = ["Pkg"] -git-tree-sha1 = "f9501cc0430a26bc3d156ae1b5b0c1b47af4d6da" -uuid = "eebad327-c553-4316-9ea0-9fa01ccd7688" -version = "0.3.3" - -[[deps.PooledArrays]] -deps = ["DataAPI", "Future"] -git-tree-sha1 = "36d8b4b899628fb92c2749eb488d884a926614d3" -uuid = "2dfb63ee-cc39-5dd5-95bd-886bf059d720" -version = "1.4.3" - -[[deps.PrecompileTools]] -deps = ["Preferences"] -git-tree-sha1 = "5aa36f7049a63a1528fe8f7c3f2113413ffd4e1f" -uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" -version = "1.2.1" - -[[deps.Preferences]] -deps = ["TOML"] -git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6" -uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.4.3" - -[[deps.PrettyTables]] -deps = ["Crayons", "LaTeXStrings", "Markdown", "PrecompileTools", "Printf", "Reexport", "StringManipulation", "Tables"] -git-tree-sha1 = "1101cd475833706e4d0e7b122218257178f48f34" -uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" -version = "2.4.0" - -[[deps.Printf]] -deps = ["Unicode"] -uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" - -[[deps.Quaternions]] -deps = ["LinearAlgebra", "Random", "RealDot"] -git-tree-sha1 = "994cc27cdacca10e68feb291673ec3a76aa2fae9" -uuid = "94ee1d12-ae83-5a48-8b1c-48b8ff168ae0" -version = "0.7.6" - -[[deps.REPL]] -deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] -uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" - -[[deps.Random]] -deps = ["SHA"] -uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" - -[[deps.Random123]] -deps = ["Random", "RandomNumbers"] -git-tree-sha1 = "4743b43e5a9c4a2ede372de7061eed81795b12e7" -uuid = "74087812-796a-5b5d-8853-05524746bad3" -version = "1.7.0" - -[[deps.RandomNumbers]] -deps = ["Random"] -git-tree-sha1 = "c6ec94d2aaba1ab2ff983052cf6a606ca5985902" -uuid = "e6cf234a-135c-5ec9-84dd-332b85af5143" -version = "1.6.0" - -[[deps.RealDot]] -deps = ["LinearAlgebra"] -git-tree-sha1 = "9f0a1b71baaf7650f4fa8a1d168c7fb6ee41f0c9" -uuid = "c1ae055f-0cd5-4b69-90a6-9a35b1a98df9" -version = "0.1.0" - -[[deps.RecipesBase]] -deps = ["PrecompileTools"] -git-tree-sha1 = "5c3d09cc4f31f5fc6af001c250bf1278733100ff" -uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" -version = "1.3.4" - -[[deps.Reexport]] -git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" -uuid = "189a3867-3050-52da-a836-e630ba90ab69" -version = "1.2.2" - -[[deps.Requires]] -deps = ["UUIDs"] -git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7" -uuid = "ae029012-a4dd-5104-9daa-d747884805df" -version = "1.3.0" - -[[deps.Rotations]] -deps = ["LinearAlgebra", "Quaternions", "Random", "StaticArrays"] -git-tree-sha1 = "5680a9276685d392c87407df00d57c9924d9f11e" -uuid = "6038ab10-8711-5258-84ad-4b1120ba62dc" -version = "1.7.1" -weakdeps = ["RecipesBase"] - - [deps.Rotations.extensions] - RotationsRecipesBaseExt = "RecipesBase" - -[[deps.SHA]] -uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" -version = "0.7.0" - -[[deps.Scratch]] -deps = ["Dates"] -git-tree-sha1 = "3bac05bc7e74a75fd9cba4295cde4045d9fe2386" -uuid = "6c6a2e73-6563-6170-7368-637461726353" -version = "1.2.1" - -[[deps.SeawaterPolynomials]] -git-tree-sha1 = "78f965a2f0cd5250a20c9aba9979346dd2b35734" -uuid = "d496a93d-167e-4197-9f49-d3af4ff8fe40" -version = "0.3.5" - -[[deps.SentinelArrays]] -deps = ["Dates", "Random"] -git-tree-sha1 = "d0553ce4031a081cc42387a9b9c8441b7d99f32d" -uuid = "91c51154-3ec4-41a3-a24f-3f23e20d615c" -version = "1.4.7" - -[[deps.Serialization]] -uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" - -[[deps.Sockets]] -uuid = "6462fe0b-24de-5631-8697-dd941f90decc" - -[[deps.SortingAlgorithms]] -deps = ["DataStructures"] -git-tree-sha1 = "66e0a8e672a0bdfca2c3f5937efb8538b9ddc085" -uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" -version = "1.2.1" - -[[deps.SparseArrays]] -deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] -uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" -version = "1.10.0" - -[[deps.StaticArrays]] -deps = ["LinearAlgebra", "PrecompileTools", "Random", "StaticArraysCore"] -git-tree-sha1 = "777657803913ffc7e8cc20f0fd04b634f871af8f" -uuid = "90137ffa-7385-5640-81b9-e52037218182" -version = "1.9.8" - - [deps.StaticArrays.extensions] - StaticArraysChainRulesCoreExt = "ChainRulesCore" - StaticArraysStatisticsExt = "Statistics" - - [deps.StaticArrays.weakdeps] - ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" - Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" - -[[deps.StaticArraysCore]] -git-tree-sha1 = "192954ef1208c7019899fbf8049e717f92959682" -uuid = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" -version = "1.4.3" - -[[deps.Statistics]] -deps = ["LinearAlgebra", "SparseArrays"] -uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" -version = "1.10.0" - -[[deps.StatsAPI]] -deps = ["LinearAlgebra"] -git-tree-sha1 = "1ff449ad350c9c4cbc756624d6f8a8c3ef56d3ed" -uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0" -version = "1.7.0" - -[[deps.StringManipulation]] -deps = ["PrecompileTools"] -git-tree-sha1 = "a6b1675a536c5ad1a60e5a5153e1fee12eb146e3" -uuid = "892a3eda-7b42-436c-8928-eab12a02cf0e" -version = "0.4.0" - -[[deps.StructArrays]] -deps = ["ConstructionBase", "DataAPI", "Tables"] -git-tree-sha1 = "f4dc295e983502292c4c3f951dbb4e985e35b3be" -uuid = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" -version = "0.6.18" -weakdeps = ["Adapt", "GPUArraysCore", "SparseArrays", "StaticArrays"] - - [deps.StructArrays.extensions] - StructArraysAdaptExt = "Adapt" - StructArraysGPUArraysCoreExt = "GPUArraysCore" - StructArraysSparseArraysExt = "SparseArrays" - StructArraysStaticArraysExt = "StaticArrays" - -[[deps.SuiteSparse_jll]] -deps = ["Artifacts", "Libdl", "libblastrampoline_jll"] -uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" -version = "7.2.1+1" - -[[deps.TOML]] -deps = ["Dates"] -uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" -version = "1.0.3" - -[[deps.TableTraits]] -deps = ["IteratorInterfaceExtensions"] -git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39" -uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c" -version = "1.0.1" - -[[deps.Tables]] -deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "OrderedCollections", "TableTraits"] -git-tree-sha1 = "598cd7c1f68d1e205689b1c2fe65a9f85846f297" -uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" -version = "1.12.0" - -[[deps.Tar]] -deps = ["ArgTools", "SHA"] -uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" -version = "1.10.0" - -[[deps.TaylorSeries]] -deps = ["LinearAlgebra", "Markdown", "Requires", "SparseArrays"] -git-tree-sha1 = "abc13c4d3cccd1703335ba03abdaf0dc8ecc97e2" -uuid = "6aa5eb33-94cf-58f4-a9d0-e4b2c4fc25ea" -version = "0.17.3" - - [deps.TaylorSeries.extensions] - TaylorSeriesIAExt = "IntervalArithmetic" - TaylorSeriesSAExt = "StaticArrays" - - [deps.TaylorSeries.weakdeps] - IntervalArithmetic = "d1acc4aa-44c8-5952-acd4-ba5d80a2a253" - StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" - -[[deps.Test]] -deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] -uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" - -[[deps.TimerOutputs]] -deps = ["ExprTools", "Printf"] -git-tree-sha1 = "3a6f063d690135f5c1ba351412c82bae4d1402bf" -uuid = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" -version = "0.5.25" - -[[deps.TranscodingStreams]] -git-tree-sha1 = "0c45878dcfdcfa8480052b6ab162cdd138781742" -uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" -version = "0.11.3" - -[[deps.UUIDs]] -deps = ["Random", "SHA"] -uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" - -[[deps.Unicode]] -uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" - -[[deps.UnsafeAtomics]] -git-tree-sha1 = "6331ac3440856ea1988316b46045303bef658278" -uuid = "013be700-e6cd-48c3-b4a1-df204f14c38f" -version = "0.2.1" - -[[deps.UnsafeAtomicsLLVM]] -deps = ["LLVM", "UnsafeAtomics"] -git-tree-sha1 = "2d17fabcd17e67d7625ce9c531fb9f40b7c42ce4" -uuid = "d80eeb9a-aca5-4d75-85e5-170c8b632249" -version = "0.2.1" - -[[deps.XML2_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Libiconv_jll", "Zlib_jll"] -git-tree-sha1 = "6a451c6f33a176150f315726eba8b92fbfdb9ae7" -uuid = "02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a" -version = "2.13.4+0" - -[[deps.XZ_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "15e637a697345f6743674f1322beefbc5dcd5cfc" -uuid = "ffd25f8a-64ca-5728-b0f7-c24cf3aae800" -version = "5.6.3+0" - -[[deps.Zlib_jll]] -deps = ["Libdl"] -uuid = "83775a58-1f1d-513f-b197-d71354ab007a" -version = "1.2.13+1" - -[[deps.Zstd_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "555d1076590a6cc2fdee2ef1469451f872d8b41b" -uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" -version = "1.5.6+1" - -[[deps.demumble_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "6498e3581023f8e530f34760d18f75a69e3a4ea8" -uuid = "1e29f10c-031c-5a83-9565-69cddfc27673" -version = "1.3.0+0" - -[[deps.libaec_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "46bf7be2917b59b761247be3f317ddf75e50e997" -uuid = "477f73a3-ac25-53e9-8cc3-50b2fa2566f0" -version = "1.1.2+0" - -[[deps.libblastrampoline_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" -version = "5.11.0+0" - -[[deps.libzip_jll]] -deps = ["Artifacts", "Bzip2_jll", "GnuTLS_jll", "JLLWrappers", "Libdl", "XZ_jll", "Zlib_jll", "Zstd_jll"] -git-tree-sha1 = "668ac0297e6bd8f4d53dfdcd3ace71f2e00f4a35" -uuid = "337d8026-41b4-5cde-a456-74a10e5b31d1" -version = "1.11.1+0" - -[[deps.nghttp2_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" -version = "1.52.0+1" - -[[deps.oneTBB_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "7d0ea0f4895ef2f5cb83645fa689e52cb55cf493" -uuid = "1317d2d5-d96f-522e-a858-c73665f53c3e" -version = "2021.12.0+0" - -[[deps.p7zip_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" -version = "17.4.0+2" From 9c3c6cd75c4f5892cbf8d5be61a5b78ad0754c9b Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 6 Nov 2024 10:56:50 +0100 Subject: [PATCH 449/567] why are we using 8 threads? --- .buildkite/distributed/pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 7c892803a1..874f939bb1 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -7,8 +7,8 @@ env: JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite/distributed" OPENBLAS_NUM_THREADS: 1 JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager - JULIA_NUM_PRECOMPILE_TASKS: 8 - JULIA_NUM_THREADS: 8 + JULIA_NUM_PRECOMPILE_TASKS: 1 + JULIA_NUM_THREADS: 1 OMPI_MCA_opal_warn_on_missing_libcuda: 0 steps: From 3b28ecb2c4d7cdf9be60def791e3f2d03f79e444 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 6 Nov 2024 11:03:56 +0100 Subject: [PATCH 450/567] memory requirements are not this huge --- .buildkite/distributed/pipeline.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 874f939bb1..20d2eed28a 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -31,7 +31,7 @@ steps: - echo "--- Instantiate status" - "julia --project -e 'using Pkg; Pkg.status()'" agents: - slurm_mem: 120G + slurm_mem: 8G slurm_ntasks: 1 slurm_gpus_per_task: 1 @@ -45,7 +45,7 @@ steps: commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: - slurm_mem: 120G + slurm_mem: 8G slurm_ntasks: 4 - label: "🐲 gpu distributed unit tests" @@ -57,7 +57,7 @@ steps: commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: - slurm_mem: 120G + slurm_mem: 8G slurm_ntasks: 4 slurm_gpus_per_task: 1 @@ -69,7 +69,7 @@ steps: commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: - slurm_mem: 120G + slurm_mem: 8G slurm_ntasks: 4 - label: "🛸 gpu distributed solvers tests" @@ -81,7 +81,7 @@ steps: commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: - slurm_mem: 120G + slurm_mem: 8G slurm_ntasks: 4 slurm_gpus_per_task: 1 retry: @@ -97,7 +97,7 @@ steps: commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: - slurm_mem: 120G + slurm_mem: 8G slurm_ntasks: 4 - label: "🦏 gpu distributed hydrostatic model tests" @@ -109,7 +109,7 @@ steps: commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: - slurm_mem: 120G + slurm_mem: 8G slurm_ntasks: 4 slurm_gpus_per_task: 1 retry: @@ -125,7 +125,7 @@ steps: commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: - slurm_mem: 120G + slurm_mem: 8G slurm_ntasks: 4 - label: "🕺 gpu distributed nonhydrostatic regression" @@ -137,7 +137,7 @@ steps: commands: - "srun julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: - slurm_mem: 120G + slurm_mem: 8G slurm_ntasks: 4 slurm_gpus_per_task: 1 retry: From 00f87a4e91466ed3bea3a9aa73c000be2584bcc8 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 6 Nov 2024 11:06:12 +0100 Subject: [PATCH 451/567] update enzyme --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 4d80cee5b7..45b005cea6 100644 --- a/Project.toml +++ b/Project.toml @@ -50,7 +50,7 @@ CubedSphere = "0.2, 0.3" Dates = "1.9" Distances = "0.10" DocStringExtensions = "0.8, 0.9" -Enzyme = "0.13.3" +Enzyme = "0.13.14" FFTW = "1" Glob = "1.3" IncompleteLU = "0.2" From 7126c7cb5c99c464625b6c970733bc0dc4478254 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 6 Nov 2024 11:49:08 +0100 Subject: [PATCH 452/567] speed up the precompilation a bit, to revert later --- .buildkite/distributed/pipeline.yml | 6 ------ Project.toml | 3 +-- test/dependencies_for_runtests.jl | 2 +- test/runtests.jl | 12 ++++++------ 4 files changed, 8 insertions(+), 15 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 20d2eed28a..ce9f53d027 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -24,12 +24,6 @@ steps: # Force the initialization of the CUDA runtime as it is lazily loaded by default - "julia -O0 --project -e 'using CUDA; CUDA.precompile_runtime(); CUDA.versioninfo()'" - "julia -O0 --project -e 'using MPI; MPI.versioninfo()'" - - # Download artifacts by running an empty testgroup and thereby executing /test/runtests.jl - - "julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" - - - echo "--- Instantiate status" - - "julia --project -e 'using Pkg; Pkg.status()'" agents: slurm_mem: 8G slurm_ntasks: 1 diff --git a/Project.toml b/Project.toml index 4d80cee5b7..33fa9f8b01 100644 --- a/Project.toml +++ b/Project.toml @@ -77,10 +77,9 @@ julia = "1.9" [extras] DataDeps = "124859b0-ceae-595e-8997-d05f6a7a8dfe" -Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" TimesDates = "bdfc003b-8df8-5c39-adcd-3a9087f5df4a" [targets] -test = ["DataDeps", "Enzyme", "SafeTestsets", "Test", "TimesDates"] +test = ["DataDeps", "SafeTestsets", "Test", "TimesDates"] diff --git a/test/dependencies_for_runtests.jl b/test/dependencies_for_runtests.jl index 3e15e0a3d8..54da7bbc52 100644 --- a/test/dependencies_for_runtests.jl +++ b/test/dependencies_for_runtests.jl @@ -5,7 +5,7 @@ using Random using Statistics using LinearAlgebra using Logging -using Enzyme +# using Enzyme using SparseArrays using JLD2 using FFTW diff --git a/test/runtests.jl b/test/runtests.jl index 69de014ec9..28bf877c2a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -221,12 +221,12 @@ CUDA.allowscalar() do end end - # Tests for Enzyme extension - if group == :enzyme || group == :all - @testset "Enzyme extension tests" begin - include("test_enzyme.jl") - end - end + # # Tests for Enzyme extension + # if group == :enzyme || group == :all + # @testset "Enzyme extension tests" begin + # include("test_enzyme.jl") + # end + # end if group == :convergence include("test_convergence.jl") From 733ab2bc614cd338a96bfc37f620f5e118d4d17e Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 6 Nov 2024 12:02:32 +0100 Subject: [PATCH 453/567] might this be the culprit? --- .buildkite/distributed/pipeline.yml | 3 +++ Project.toml | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index ce9f53d027..f13d8ae8cb 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -24,6 +24,9 @@ steps: # Force the initialization of the CUDA runtime as it is lazily loaded by default - "julia -O0 --project -e 'using CUDA; CUDA.precompile_runtime(); CUDA.versioninfo()'" - "julia -O0 --project -e 'using MPI; MPI.versioninfo()'" + + - echo "--- Initialize tests" + - "julia -O0 --project -e 'using Pkg; Pkg.test()'" agents: slurm_mem: 8G slurm_ntasks: 1 diff --git a/Project.toml b/Project.toml index 33fa9f8b01..a0b9ffd19f 100644 --- a/Project.toml +++ b/Project.toml @@ -80,6 +80,7 @@ DataDeps = "124859b0-ceae-595e-8997-d05f6a7a8dfe" SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" TimesDates = "bdfc003b-8df8-5c39-adcd-3a9087f5df4a" +MPIPreferences = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267" [targets] -test = ["DataDeps", "SafeTestsets", "Test", "TimesDates"] +test = ["DataDeps", "SafeTestsets", "Test", "MPIPreferences", "TimesDates"] From 2dbf1a0c5da4338257f7b90c715b4743ba6fcfb6 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 6 Nov 2024 12:03:25 +0100 Subject: [PATCH 454/567] revert to 8 tasks to precompile --- .buildkite/distributed/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index f13d8ae8cb..1eb73a04ec 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -7,7 +7,7 @@ env: JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite/distributed" OPENBLAS_NUM_THREADS: 1 JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager - JULIA_NUM_PRECOMPILE_TASKS: 1 + JULIA_NUM_PRECOMPILE_TASKS: 8 JULIA_NUM_THREADS: 1 OMPI_MCA_opal_warn_on_missing_libcuda: 0 From a4b129ade57a5cc4d1efbdc8460d6b5cbd0e4655 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 6 Nov 2024 12:04:19 +0100 Subject: [PATCH 455/567] final version? --- .buildkite/distributed/pipeline.yml | 20 ++++++++++++++++++++ test/runtests.jl | 3 --- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 1eb73a04ec..fa620cd04d 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -44,6 +44,10 @@ steps: agents: slurm_mem: 8G slurm_ntasks: 4 + retry: + automatic: + - exit_status: 1 + limit: 1 - label: "🐲 gpu distributed unit tests" key: "distributed_gpu" @@ -57,6 +61,10 @@ steps: slurm_mem: 8G slurm_ntasks: 4 slurm_gpus_per_task: 1 + retry: + automatic: + - exit_status: 1 + limit: 1 - label: "🦾 cpu distributed solvers tests" key: "distributed_solvers_cpu" @@ -68,6 +76,10 @@ steps: agents: slurm_mem: 8G slurm_ntasks: 4 + retry: + automatic: + - exit_status: 1 + limit: 1 - label: "🛸 gpu distributed solvers tests" key: "distributed_solvers_gpu" @@ -96,6 +108,10 @@ steps: agents: slurm_mem: 8G slurm_ntasks: 4 + retry: + automatic: + - exit_status: 1 + limit: 1 - label: "🦏 gpu distributed hydrostatic model tests" key: "distributed_hydrostatic_model_gpu" @@ -124,6 +140,10 @@ steps: agents: slurm_mem: 8G slurm_ntasks: 4 + retry: + automatic: + - exit_status: 1 + limit: 1 - label: "🕺 gpu distributed nonhydrostatic regression" key: "distributed_nonhydrostatic_regression_gpu" diff --git a/test/runtests.jl b/test/runtests.jl index 28bf877c2a..9661096837 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -170,7 +170,6 @@ CUDA.allowscalar() do if group == :distributed || group == :all MPI.Initialized() || MPI.Init() archs = test_architectures() - CUDA.set_runtime_version!(v"12.6") include("test_distributed_models.jl") end @@ -195,14 +194,12 @@ CUDA.allowscalar() do if group == :distributed_hydrostatic_model || group == :all MPI.Initialized() || MPI.Init() archs = test_architectures() - CUDA.set_runtime_version!(v"12.6") include("test_hydrostatic_regression.jl") include("test_distributed_hydrostatic_model.jl") end if group == :distributed_nonhydrostatic_regression || group == :all MPI.Initialized() || MPI.Init() - CUDA.set_runtime_version!(v"12.6") archs = nonhydrostatic_regression_test_architectures() include("test_nonhydrostatic_regression.jl") end From 29f7d69a7d2814aec9f7acabefc3191001aac4f3 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 6 Nov 2024 12:28:55 +0100 Subject: [PATCH 456/567] return to previous state of affairs --- Project.toml | 5 +++-- test/runtests.jl | 24 ++++++------------------ test/utils_for_runtests.jl | 4 ++-- 3 files changed, 11 insertions(+), 22 deletions(-) diff --git a/Project.toml b/Project.toml index a0b9ffd19f..c12724ce34 100644 --- a/Project.toml +++ b/Project.toml @@ -50,7 +50,7 @@ CubedSphere = "0.2, 0.3" Dates = "1.9" Distances = "0.10" DocStringExtensions = "0.8, 0.9" -Enzyme = "0.13.3" +Enzyme = "0.13.14" FFTW = "1" Glob = "1.3" IncompleteLU = "0.2" @@ -80,7 +80,8 @@ DataDeps = "124859b0-ceae-595e-8997-d05f6a7a8dfe" SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" TimesDates = "bdfc003b-8df8-5c39-adcd-3a9087f5df4a" +Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" MPIPreferences = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267" [targets] -test = ["DataDeps", "SafeTestsets", "Test", "MPIPreferences", "TimesDates"] +test = ["DataDeps", "SafeTestsets", "Test", "Enzyme", "MPIPreferences", "TimesDates"] diff --git a/test/runtests.jl b/test/runtests.jl index 9661096837..bbf6415976 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -175,18 +175,6 @@ CUDA.allowscalar() do if group == :distributed_solvers || group == :all MPI.Initialized() || MPI.Init() - Pkg.instantiate(; verbose=true) - Pkg.precompile(; strict=true) - Pkg.status() - - try - MPI.versioninfo() - catch; end - - try - CUDA.precompile_runtime() - CUDA.versioninfo() - catch; end include("test_distributed_transpose.jl") include("test_distributed_poisson_solvers.jl") end @@ -218,12 +206,12 @@ CUDA.allowscalar() do end end - # # Tests for Enzyme extension - # if group == :enzyme || group == :all - # @testset "Enzyme extension tests" begin - # include("test_enzyme.jl") - # end - # end + # Tests for Enzyme extension + if group == :enzyme || group == :all + @testset "Enzyme extension tests" begin + include("test_enzyme.jl") + end + end if group == :convergence include("test_convergence.jl") diff --git a/test/utils_for_runtests.jl b/test/utils_for_runtests.jl index a64d44263b..f27ba77054 100644 --- a/test/utils_for_runtests.jl +++ b/test/utils_for_runtests.jl @@ -5,8 +5,8 @@ import Oceananigans.Fields: interior # Are the test running on the GPUs? # Are the test running in parallel? -child_arch = parse(Bool, get(ENV, "GPU_TEST", "false")) ? GPU() : CPU() -mpi_test = parse(Bool, get(ENV, "MPI_TEST", "false")) +child_arch = get(ENV, "GPU_TEST", nothing) == "true" ? GPU() : CPU() +mpi_test = get(ENV, "MPI_TEST", nothing) == "true" function test_architectures() # If MPI is initialized with MPI.Comm_size > 0, we are running in parallel. From 3154a347a6c19b533ec8f6c8c25ea74bd66db6b0 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 6 Nov 2024 12:32:27 +0100 Subject: [PATCH 457/567] bugfix --- test/dependencies_for_runtests.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/dependencies_for_runtests.jl b/test/dependencies_for_runtests.jl index 54da7bbc52..3e15e0a3d8 100644 --- a/test/dependencies_for_runtests.jl +++ b/test/dependencies_for_runtests.jl @@ -5,7 +5,7 @@ using Random using Statistics using LinearAlgebra using Logging -# using Enzyme +using Enzyme using SparseArrays using JLD2 using FFTW From 3803ff34df5452b87438ef49f4d5d5586ba9d870 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 6 Nov 2024 14:49:47 +0100 Subject: [PATCH 458/567] cuda runtime version --- test/runtests.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/runtests.jl b/test/runtests.jl index bbf6415976..8a1cff05f3 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -169,18 +169,21 @@ CUDA.allowscalar() do if group == :distributed || group == :all MPI.Initialized() || MPI.Init() + CUDA.set_runtime_version!(v"12.2"; local_toolkit = true) archs = test_architectures() include("test_distributed_models.jl") end if group == :distributed_solvers || group == :all MPI.Initialized() || MPI.Init() + CUDA.set_runtime_version!(v"12.2"; local_toolkit = true) include("test_distributed_transpose.jl") include("test_distributed_poisson_solvers.jl") end if group == :distributed_hydrostatic_model || group == :all MPI.Initialized() || MPI.Init() + CUDA.set_runtime_version!(v"12.2"; local_toolkit = true) archs = test_architectures() include("test_hydrostatic_regression.jl") include("test_distributed_hydrostatic_model.jl") @@ -188,6 +191,7 @@ CUDA.allowscalar() do if group == :distributed_nonhydrostatic_regression || group == :all MPI.Initialized() || MPI.Init() + CUDA.set_runtime_version!(v"12.2"; local_toolkit = true) archs = nonhydrostatic_regression_test_architectures() include("test_nonhydrostatic_regression.jl") end From 349f71dfb30313a317b9401b15ff23cc98d095c8 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 15 Nov 2024 17:44:46 +0100 Subject: [PATCH 459/567] bugfix --- src/ImmersedBoundaries/ImmersedBoundaries.jl | 110 ------------------- 1 file changed, 110 deletions(-) diff --git a/src/ImmersedBoundaries/ImmersedBoundaries.jl b/src/ImmersedBoundaries/ImmersedBoundaries.jl index fbc4ab6846..50d8eb7d58 100644 --- a/src/ImmersedBoundaries/ImmersedBoundaries.jl +++ b/src/ImmersedBoundaries/ImmersedBoundaries.jl @@ -31,116 +31,6 @@ import Oceananigans.Architectures: on_architecture import Oceananigans.Fields: fractional_x_index, fractional_y_index, fractional_z_index - -Consider the configuration - -``` - Immersed Fluid - =========== ⋅⋅⋅⋅⋅⋅⋅⋅⋅⋅⋅⋅ - - c c - i-1 i - - | ========= | | - × === ∘ === × ∘ × - | ========= | | - -i-1 i - f f f -``` - -We then have - -* `inactive_node(i, 1, 1, grid, f, c, c) = false` - -As well as - -* `inactive_node(i, 1, 1, grid, c, c, c) = false` -* `inactive_node(i-1, 1, 1, grid, c, c, c) = true` -* `inactive_node(i-1, 1, 1, grid, f, c, c) = true` -""" -@inline inactive_cell(i, j, k, ibg::IBG) = immersed_cell(i, j, k, ibg) | inactive_cell(i, j, k, ibg.underlying_grid) - -# Isolate periphery of the immersed boundary -@inline immersed_peripheral_node(i, j, k, ibg::IBG, LX, LY, LZ) = peripheral_node(i, j, k, ibg, LX, LY, LZ) & - !peripheral_node(i, j, k, ibg.underlying_grid, LX, LY, LZ) - -@inline immersed_inactive_node(i, j, k, ibg::IBG, LX, LY, LZ) = inactive_node(i, j, k, ibg, LX, LY, LZ) & - !inactive_node(i, j, k, ibg.underlying_grid, LX, LY, LZ) - -# Underlying grids are never immersed! -@inline immersed_peripheral_node(i, j, k, grid::AbstractUnderlyingGrid, LX, LY, LZ) = false -@inline immersed_inactive_node(i, j, k, grid::AbstractUnderlyingGrid, LX, LY, LZ) = false - -##### -##### Utilities -##### - -const c = Center() -const f = Face() - -@inline Base.zero(ibg::IBG) = zero(ibg.underlying_grid) - -@inline xnode(i, j, k, ibg::IBG, ℓx, ℓy, ℓz) = xnode(i, j, k, ibg.underlying_grid, ℓx, ℓy, ℓz) -@inline ynode(i, j, k, ibg::IBG, ℓx, ℓy, ℓz) = ynode(i, j, k, ibg.underlying_grid, ℓx, ℓy, ℓz) -@inline znode(i, j, k, ibg::IBG, ℓx, ℓy, ℓz) = znode(i, j, k, ibg.underlying_grid, ℓx, ℓy, ℓz) - -@inline λnode(i, j, k, ibg::IBG, ℓx, ℓy, ℓz) = λnode(i, j, k, ibg.underlying_grid, ℓx, ℓy, ℓz) -@inline φnode(i, j, k, ibg::IBG, ℓx, ℓy, ℓz) = φnode(i, j, k, ibg.underlying_grid, ℓx, ℓy, ℓz) - -@inline ξnode(i, j, k, ibg::IBG, ℓx, ℓy, ℓz) = ξnode(i, j, k, ibg.underlying_grid, ℓx, ℓy, ℓz) -@inline ηnode(i, j, k, ibg::IBG, ℓx, ℓy, ℓz) = ηnode(i, j, k, ibg.underlying_grid, ℓx, ℓy, ℓz) -@inline rnode(i, j, k, ibg::IBG, ℓx, ℓy, ℓz) = rnode(i, j, k, ibg.underlying_grid, ℓx, ℓy, ℓz) - -@inline node(i, j, k, ibg::IBG, ℓx, ℓy, ℓz) = node(i, j, k, ibg.underlying_grid, ℓx, ℓy, ℓz) - -nodes(ibg::IBG, ℓx, ℓy, ℓz; kwargs...) = nodes(ibg.underlying_grid, ℓx, ℓy, ℓz; kwargs...) -nodes(ibg::IBG, (ℓx, ℓy, ℓz); kwargs...) = nodes(ibg, ℓx, ℓy, ℓz; kwargs...) - -xnodes(ibg::IBG, loc; kwargs...) = xnodes(ibg.underlying_grid, loc; kwargs...) -ynodes(ibg::IBG, loc; kwargs...) = ynodes(ibg.underlying_grid, loc; kwargs...) -znodes(ibg::IBG, loc; kwargs...) = znodes(ibg.underlying_grid, loc; kwargs...) - -λnodes(ibg::IBG, loc; kwargs...) = λnodes(ibg.underlying_grid, loc; kwargs...) -φnodes(ibg::IBG, loc; kwargs...) = φnodes(ibg.underlying_grid, loc; kwargs...) - -ξnodes(ibg::IBG, loc; kwargs...) = ξnodes(ibg.underlying_grid, loc; kwargs...) -ηnodes(ibg::IBG, loc; kwargs...) = ηnodes(ibg.underlying_grid, loc; kwargs...) -rnodes(ibg::IBG, loc; kwargs...) = rnodes(ibg.underlying_grid, loc; kwargs...) - -xnodes(ibg::IBG, ℓx, ℓy, ℓz; kwargs...) = xnodes(ibg.underlying_grid, ℓx, ℓy, ℓz; kwargs...) -ynodes(ibg::IBG, ℓx, ℓy, ℓz; kwargs...) = ynodes(ibg.underlying_grid, ℓx, ℓy, ℓz; kwargs...) -znodes(ibg::IBG, ℓx, ℓy, ℓz; kwargs...) = znodes(ibg.underlying_grid, ℓx, ℓy, ℓz; kwargs...) - -λnodes(ibg::IBG, ℓx, ℓy, ℓz; kwargs...) = λnodes(ibg.underlying_grid, ℓx, ℓy, ℓz; kwargs...) -φnodes(ibg::IBG, ℓx, ℓy, ℓz; kwargs...) = φnodes(ibg.underlying_grid, ℓx, ℓy, ℓz; kwargs...) - -ξnodes(ibg::IBG, ℓx, ℓy, ℓz; kwargs...) = ξnodes(ibg.underlying_grid, ℓx, ℓy, ℓz; kwargs...) -ηnodes(ibg::IBG, ℓx, ℓy, ℓz; kwargs...) = ηnodes(ibg.underlying_grid, ℓx, ℓy, ℓz; kwargs...) -rnodes(ibg::IBG, ℓx, ℓy, ℓz; kwargs...) = rnodes(ibg.underlying_grid, ℓx, ℓy, ℓz; kwargs...) - -@inline cpu_face_constructor_x(ibg::IBG) = cpu_face_constructor_x(ibg.underlying_grid) -@inline cpu_face_constructor_y(ibg::IBG) = cpu_face_constructor_y(ibg.underlying_grid) -@inline cpu_face_constructor_z(ibg::IBG) = cpu_face_constructor_z(ibg.underlying_grid) - -node_names(ibg::IBG, ℓx, ℓy, ℓz) = node_names(ibg.underlying_grid, ℓx, ℓy, ℓz) -ξname(ibg::IBG) = ξname(ibg.underlying_grid) -ηname(ibg::IBG) = ηname(ibg.underlying_grid) -rname(ibg::IBG) = rname(ibg.underlying_grid) - -function on_architecture(arch, ibg::IBG) - underlying_grid = on_architecture(arch, ibg.underlying_grid) - immersed_boundary = on_architecture(arch, ibg.immersed_boundary) - return ImmersedBoundaryGrid(underlying_grid, immersed_boundary) -end - -isrectilinear(ibg::IBG) = isrectilinear(ibg.underlying_grid) - -@inline fractional_x_index(x, locs, grid::ImmersedBoundaryGrid) = fractional_x_index(x, locs, grid.underlying_grid) -@inline fractional_y_index(x, locs, grid::ImmersedBoundaryGrid) = fractional_y_index(x, locs, grid.underlying_grid) -@inline fractional_z_index(x, locs, grid::ImmersedBoundaryGrid) = fractional_z_index(x, locs, grid.underlying_grid) - -======= include("immersed_boundary_grid.jl") include("immersed_boundary_interface.jl") include("immersed_boundary_nodes.jl") From cf7b6390d913ee47c463c76248a4bedd81b53429 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 15 Nov 2024 17:52:00 +0100 Subject: [PATCH 460/567] some bugfixes --- Project.toml | 3 +-- src/Operators/spacings_and_areas_and_volumes.jl | 4 ++-- src/OutputWriters/output_writer_utils.jl | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Project.toml b/Project.toml index 8bcaae0ff2..ad01c82946 100644 --- a/Project.toml +++ b/Project.toml @@ -11,7 +11,6 @@ CubedSphere = "7445602f-e544-4518-8976-18f8e8ae6cdb" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Distances = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7" DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" -FFMPEG = "c87230d0-a227-11e9-1b43-d7ebe4e7570a" FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" Glob = "c27321d9-0574-5035-807b-f59d2c89b15c" IncompleteLU = "40713840-3770-5561-ab4c-a76e7d0d7895" @@ -77,9 +76,9 @@ StructArrays = "0.4, 0.5, 0.6" julia = "1.9" [extras] +CUDA_Runtime_jll = "76a88914-d11a-5bdc-97e0-2f5a05c973a2" DataDeps = "124859b0-ceae-595e-8997-d05f6a7a8dfe" Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" -CUDA_Runtime_jll = "76a88914-d11a-5bdc-97e0-2f5a05c973a2" MPIPreferences = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267" SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/src/Operators/spacings_and_areas_and_volumes.jl b/src/Operators/spacings_and_areas_and_volumes.jl index a939c7fcce..130a0e9a35 100644 --- a/src/Operators/spacings_and_areas_and_volumes.jl +++ b/src/Operators/spacings_and_areas_and_volumes.jl @@ -56,8 +56,8 @@ const ZRG = Union{LLGZ, RGZ, OSSGZ} @inline getspacing(k, Δz::AbstractVector) = @inbounds Δz[k] @inline getspacing(k, Δz::Number) = @inbounds Δz -@inline Δrᵃᵃᶜ(i, j, k, grid::ZSG) = getspacing(k, grid.Δzᵃᵃᶜ.reference) -@inline Δrᵃᵃᶠ(i, j, k, grid::ZSG) = getspacing(k, grid.Δzᵃᵃᶠ.reference) +@inline Δrᵃᵃᶜ(i, j, k, grid::ZStarUnderlyingGrid) = getspacing(k, grid.Δzᵃᵃᶜ.reference) +@inline Δrᵃᵃᶠ(i, j, k, grid::ZStarUnderlyingGrid) = getspacing(k, grid.Δzᵃᵃᶠ.reference) # Convenience Functions for all grids for LX in (:ᶜ, :ᶠ), LY in (:ᶜ, :ᶠ) diff --git a/src/OutputWriters/output_writer_utils.jl b/src/OutputWriters/output_writer_utils.jl index 0ca4102d20..6a237375f1 100644 --- a/src/OutputWriters/output_writer_utils.jl +++ b/src/OutputWriters/output_writer_utils.jl @@ -1,7 +1,7 @@ using Oceananigans.DistributedComputations using StructArrays: StructArray, replace_storage using Oceananigans.Grids: on_architecture, architecture -using Oceananigans.Grids: retrieve_static_grid +using Oceananigans.Grids: retrieve_static_grid, AbstractVerticalCoordinateGrid using Oceananigans.DistributedComputations: DistributedGrid, Partition using Oceananigans.Fields: AbstractField, indices, boundary_conditions, instantiated_location using Oceananigans.BoundaryConditions: bc_str, FieldBoundaryConditions, ContinuousBoundaryFunction, DiscreteBoundaryFunction From 0edb48ffcb74b579aab13f3534f6e9df9d6b93d8 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Sat, 16 Nov 2024 18:25:32 +0100 Subject: [PATCH 461/567] more fixes --- .../immersed_grid_metrics.jl | 5 +--- .../SplitExplicitFreeSurfaces.jl | 15 ++++++++++- .../barotropic_split_explicit_corrector.jl | 19 ++++++++------ .../step_split_explicit_free_surface.jl | 4 +-- .../z_star_vertical_spacing.jl | 26 +++++++++++++------ src/OutputWriters/output_writer_utils.jl | 2 +- validation/z_star_coordinate/lock_release.jl | 2 +- 7 files changed, 48 insertions(+), 25 deletions(-) diff --git a/src/ImmersedBoundaries/immersed_grid_metrics.jl b/src/ImmersedBoundaries/immersed_grid_metrics.jl index 37fd1e5b20..76346f26fb 100644 --- a/src/ImmersedBoundaries/immersed_grid_metrics.jl +++ b/src/ImmersedBoundaries/immersed_grid_metrics.jl @@ -38,7 +38,4 @@ end @inline Δzᵃᵃᶜ(i, j, k, ibg::IBG) = Δzᵃᵃᶜ(i, j, k, ibg.underlying_grid) @inline Δzᵃᵃᶠ(i, j, k, ibg::IBG) = Δzᵃᵃᶠ(i, j, k, ibg.underlying_grid) -coordinates(grid::IBG) = coordinates(grid.underlying_grid) - -@inline Δzᵃᵃᶠ(i, j, k, ibg::IBG) = Δzᵃᵃᶠ(i, j, k, ibg.underlying_grid) -@inline Δzᵃᵃᶜ(i, j, k, ibg::IBG) = Δzᵃᵃᶜ(i, j, k, ibg.underlying_grid) +coordinates(grid::IBG) = coordinates(grid.underlying_grid) \ No newline at end of file diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl index e622246489..0513578641 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl @@ -11,10 +11,12 @@ using Oceananigans.Utils using Oceananigans.Grids using Oceananigans.Operators using Oceananigans.BoundaryConditions +using Oceananigans.ImmersedBoundaries using Oceananigans.Grids: AbstractGrid, topology using Oceananigans.ImmersedBoundaries: active_linear_index_to_tuple, mask_immersed_field! using Oceananigans.Models.HydrostaticFreeSurfaceModels: AbstractFreeSurface, - free_surface_displacement_field + free_surface_displacement_field, + ZStarSpacingGrid using Adapt using Base @@ -29,6 +31,17 @@ import Oceananigans.Models.HydrostaticFreeSurfaceModels: initialize_free_surface explicit_barotropic_pressure_x_gradient, explicit_barotropic_pressure_y_gradient + +@inline dynamic_column_depthᶜᶜᵃ(i, j, grid, η) = static_column_depthᶜᶜᵃ(i, j, grid) +@inline dynamic_column_depthᶜᶠᵃ(i, j, grid, η) = static_column_depthᶜᶠᵃ(i, j, grid) +@inline dynamic_column_depthᶠᶜᵃ(i, j, grid, η) = static_column_depthᶠᶜᵃ(i, j, grid) +@inline dynamic_column_depthᶠᶠᵃ(i, j, grid, η) = static_column_depthᶠᶠᵃ(i, j, grid) + +@inline dynamic_column_depthᶜᶜᵃ(i, j, grid::ZStarSpacingGrid, η) = static_column_depthᶜᶜᵃ(i, j, grid) + @inbounds η[i, j, grid.Nz+1] +@inline dynamic_column_depthᶜᶠᵃ(i, j, grid::ZStarSpacingGrid, η) = static_column_depthᶜᶠᵃ(i, j, grid) + ℑxᶠᵃᵃ(i, j, grid.Nz+1, grid, η) +@inline dynamic_column_depthᶠᶜᵃ(i, j, grid::ZStarSpacingGrid, η) = static_column_depthᶠᶜᵃ(i, j, grid) + ℑyᵃᶠᵃ(i, j, grid.Nz+1, grid, η) +@inline dynamic_column_depthᶠᶠᵃ(i, j, grid::ZStarSpacingGrid, η) = static_column_depthᶠᶠᵃ(i, j, grid) + ℑxyᶠᶠᵃ(i, j, grid.Nz+1, grid, η) + include("split_explicit_timesteppers.jl") include("split_explicit_free_surface.jl") include("distributed_split_explicit_free_surface.jl") diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_split_explicit_corrector.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_split_explicit_corrector.jl index 2b7f7845f6..d6c6a0cc7c 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_split_explicit_corrector.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_split_explicit_corrector.jl @@ -11,12 +11,14 @@ end end @inline function barotropic_mode_kernel!(U, V, i, j, grid, u, v) - @inbounds U[i, j, 1] = Δzᶠᶜᶜ(i, j, 1, grid) * u[i, j, 1] - @inbounds V[i, j, 1] = Δzᶜᶠᶜ(i, j, 1, grid) * v[i, j, 1] + # TODO: + + @inbounds U[i, j, 1] = Δrᶠᶜᶜ(i, j, 1, grid) * u[i, j, 1] + @inbounds V[i, j, 1] = Δrᶜᶠᶜ(i, j, 1, grid) * v[i, j, 1] for k in 2:grid.Nz - @inbounds U[i, j, 1] += Δzᶠᶜᶜ(i, j, k, grid) * u[i, j, k] - @inbounds V[i, j, 1] += Δzᶜᶠᶜ(i, j, k, grid) * v[i, j, k] + @inbounds U[i, j, 1] += Δrᶠᶜᶜ(i, j, k, grid) * u[i, j, k] + @inbounds V[i, j, 1] += Δrᶜᶠᶜ(i, j, k, grid) * v[i, j, k] end return nothing @@ -28,12 +30,12 @@ end return nothing end -@kernel function _barotropic_split_explicit_corrector!(u, v, U, V, U̅, V̅, grid) +@kernel function _barotropic_split_explicit_corrector!(u, v, U, V, U̅, V̅, η, grid) i, j, k = @index(Global, NTuple) @inbounds begin - Hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) - Hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) + Hᶠᶜ = dynamic_column_depthᶠᶜᵃ(i, j, grid, η) + Hᶜᶠ = dynamic_column_depthᶜᶠᵃ(i, j, grid, η) u[i, j, k] = u[i, j, k] + (U[i, j, 1] - U̅[i, j, 1]) / Hᶠᶜ v[i, j, k] = v[i, j, k] + (V[i, j, 1] - V̅[i, j, 1]) / Hᶜᶠ @@ -43,6 +45,7 @@ end # Correcting `u` and `v` with the barotropic mode computed in `free_surface` function barotropic_split_explicit_corrector!(u, v, free_surface, grid) state = free_surface.filtered_state + η = free_surface.η U, V = free_surface.barotropic_velocities U̅, V̅ = state.U, state.V arch = architecture(grid) @@ -54,7 +57,7 @@ function barotropic_split_explicit_corrector!(u, v, free_surface, grid) # add in "good" barotropic mode launch!(arch, grid, :xyz, _barotropic_split_explicit_corrector!, - u, v, U, V, U̅, V̅, grid) + u, v, U, V, U̅, V̅, η, grid) return nothing end diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/step_split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/step_split_explicit_free_surface.jl index 07896866e1..8c6a33d5f3 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/step_split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/step_split_explicit_free_surface.jl @@ -26,8 +26,8 @@ end store_previous_velocities!(timestepper, i, j, 1, U) store_previous_velocities!(timestepper, i, j, 1, V) - Hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) - Hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) + Hᶠᶜ = dynamic_column_depthᶠᶜᵃ(i, j, grid, η) + Hᶜᶠ = dynamic_column_depthᶜᶠᵃ(i, j, grid, η) @inbounds begin # ∂τ(U) = - ∇η + G diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 6e43918816..998b0edcb3 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -2,6 +2,11 @@ using Oceananigans.Grids using Oceananigans.Grids: ZStarUnderlyingGrid, rnode using Oceananigans.ImmersedBoundaries: ImmersedZStarGrid +using Oceananigans.Models.SplitExplicitFreeSurfaces: dynamic_column_depthᶜᶜᵃ, + dynamic_column_depthᶜᶠᵃ, + dynamic_column_depthᶠᶜᵃ, + dynamic_column_depthᶠᶠᵃ + const ZStarSpacingGrid = Union{ZStarUnderlyingGrid, ImmersedZStarGrid} ##### @@ -25,8 +30,8 @@ function update_grid!(model, grid::ZStarSpacingGrid; parameters = :xy) # TODO: At the moment only SplitExplicitFreeSurface is supported, # but zstar can be extended to other free surface solvers by calculating # the barotropic velocity in this step - U̅ = model.free_surface.state.U̅ - V̅ = model.free_surface.state.V̅ + U̅ = model.free_surface.barotropic_velocities.U + V̅ = model.free_surface.barotropic_velocities.V η = model.free_surface.η # Update vertical spacing with available parameters @@ -51,8 +56,8 @@ end k_top = grid.Nz + 1 # ∂(η / H)/∂t = - ∇ ⋅ ∫udz / H - δx_U = δxᶜᶜᶠ(i, j, k_top-1, grid, Δy_qᶠᶜᶠ, U̅) - δy_V = δyᶜᶜᶠ(i, j, k_top-1, grid, Δx_qᶜᶠᶠ, V̅) + δx_U = δxᶜᶜᶠ(i, j, k_top-1, grid, Δy_qᶠᶜᶠ, U) + δy_V = δyᶜᶜᶠ(i, j, k_top-1, grid, Δx_qᶜᶠᶠ, V) δh_U = (δx_U + δy_V) / Azᶜᶜᶠ(i, j, k_top-1, grid) H = static_column_depthᶜᶜᵃ(i, j, grid) @@ -69,11 +74,16 @@ end hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) hᶠᶠ = static_column_depthᶠᶠᵃ(i, j, grid) + Hᶜᶜ = dynamic_column_depthᶜᶜᵃ(i, j, k_top, grid, η) + Hᶠᶜ = dynamic_column_depthᶠᶜᵃ(i, j, k_top, grid, η) + Hᶜᶠ = dynamic_column_depthᶜᶠᵃ(i, j, k_top, grid, η) + Hᶠᶠ = dynamic_column_depthᶠᶠᵃ(i, j, k_top, grid, η) + @inbounds begin - sᶜᶜ = ifelse(hᶜᶜ == 0, one(grid), (hᶜᶜ + η[i, j, k_top]) / hᶜᶜ) - sᶠᶜ = ifelse(hᶠᶜ == 0, one(grid), (hᶠᶜ + ℑxᶠᵃᵃ(i, j, k_top, grid, η)) / hᶠᶜ) - sᶜᶠ = ifelse(hᶜᶠ == 0, one(grid), (hᶜᶠ + ℑyᵃᶠᵃ(i, j, k_top, grid, η)) / hᶜᶠ) - sᶠᶠ = ifelse(hᶠᶠ == 0, one(grid), (hᶠᶠ + ℑxyᶠᶠᵃ(i, j, k_top, grid, η)) / hᶠᶠ) + sᶜᶜ = ifelse(hᶜᶜ == 0, one(grid), Hᶜᶜ / hᶜᶜ) + sᶠᶜ = ifelse(hᶠᶜ == 0, one(grid), Hᶠᶜ / hᶠᶜ) + sᶜᶠ = ifelse(hᶜᶠ == 0, one(grid), Hᶜᶠ / hᶜᶠ) + sᶠᶠ = ifelse(hᶠᶠ == 0, one(grid), Hᶠᶠ / hᶠᶠ) # Update previous scaling sᶜᶜ⁻[i, j] = sᶜᶜⁿ[i, j] diff --git a/src/OutputWriters/output_writer_utils.jl b/src/OutputWriters/output_writer_utils.jl index 6a237375f1..0ca4102d20 100644 --- a/src/OutputWriters/output_writer_utils.jl +++ b/src/OutputWriters/output_writer_utils.jl @@ -1,7 +1,7 @@ using Oceananigans.DistributedComputations using StructArrays: StructArray, replace_storage using Oceananigans.Grids: on_architecture, architecture -using Oceananigans.Grids: retrieve_static_grid, AbstractVerticalCoordinateGrid +using Oceananigans.Grids: retrieve_static_grid using Oceananigans.DistributedComputations: DistributedGrid, Partition using Oceananigans.Fields: AbstractField, indices, boundary_conditions, instantiated_location using Oceananigans.BoundaryConditions: bc_str, FieldBoundaryConditions, ContinuousBoundaryFunction, DiscreteBoundaryFunction diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index a3d14696cb..50d1c13380 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -1,9 +1,9 @@ using Oceananigans +using Oceananigans.Grids using Oceananigans.Units using Oceananigans.Utils: prettytime using Oceananigans.Advection: WENOVectorInvariant using Oceananigans.AbstractOperations: GridMetricOperation -using Oceananigans.Models.HydrostaticFreeSurfaceModels: ZStarSpacingGrid using Printf z_faces = ZStarVerticalCoordinate(-20, 0) From 4b62393f1164c137ef7c08c121ce6d60a3c231d5 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Sun, 17 Nov 2024 15:43:45 +0000 Subject: [PATCH 462/567] just need to fix immersed boundaries --- .../HydrostaticFreeSurfaceModels.jl | 10 ++++- .../SplitExplicitFreeSurfaces.jl | 2 - .../barotropic_split_explicit_corrector.jl | 28 +++++++------ .../initialize_split_explicit_substepping.jl | 2 +- .../step_split_explicit_free_surface.jl | 3 ++ .../generalized_vertical_spacing.jl | 8 ++-- .../z_star_vertical_spacing.jl | 40 +++++++++---------- src/Models/Models.jl | 6 +++ src/OutputWriters/OutputWriters.jl | 1 + src/OutputWriters/output_writer_utils.jl | 1 - validation/z_star_coordinate/lock_release.jl | 2 +- 11 files changed, 57 insertions(+), 46 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl index d74235e3d0..2864437827 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl @@ -11,7 +11,8 @@ using KernelAbstractions.Extras.LoopInfo: @unroll using Oceananigans.Utils using Oceananigans.Utils: launch!, SumOfArrays -using Oceananigans.Grids: AbstractGrid +using Oceananigans.Grids: AbstractGrid, ZStarUnderlyingGrid, rnode +using Oceananigans.ImmersedBoundaries: ImmersedZStarGrid using DocStringExtensions @@ -20,6 +21,8 @@ import Oceananigans.Advection: cell_advection_timescale import Oceananigans.TimeSteppers: step_lagrangian_particles! import Oceananigans.Architectures: on_architecture +const ZStarSpacingGrid = Union{ZStarUnderlyingGrid, ImmersedZStarGrid} + abstract type AbstractFreeSurface{E, G} end # This is only used by the cubed sphere for now. @@ -35,6 +38,9 @@ free_surface_displacement_field(velocities, ::Nothing, grid) = nothing # free surface initialization functions initialize_free_surface!(free_surface, grid, velocities) = nothing +# ZStar implementation +include("generalized_vertical_spacing.jl") + include("compute_w_from_continuity.jl") # No free surface @@ -61,6 +67,8 @@ include("hydrostatic_free_surface_model.jl") include("show_hydrostatic_free_surface_model.jl") include("set_hydrostatic_free_surface_model.jl") +include("z_star_vertical_spacing.jl") + ##### ##### AbstractModel interface ##### diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl index 0513578641..b25bc44800 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl @@ -24,14 +24,12 @@ using KernelAbstractions: @index, @kernel using KernelAbstractions.Extras.LoopInfo: @unroll import Oceananigans.Models.HydrostaticFreeSurfaceModels: initialize_free_surface!, - setup_free_surface!, materialize_free_surface, ab2_step_free_surface!, compute_free_surface_tendency!, explicit_barotropic_pressure_x_gradient, explicit_barotropic_pressure_y_gradient - @inline dynamic_column_depthᶜᶜᵃ(i, j, grid, η) = static_column_depthᶜᶜᵃ(i, j, grid) @inline dynamic_column_depthᶜᶠᵃ(i, j, grid, η) = static_column_depthᶜᶠᵃ(i, j, grid) @inline dynamic_column_depthᶠᶜᵃ(i, j, grid, η) = static_column_depthᶠᶜᵃ(i, j, grid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_split_explicit_corrector.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_split_explicit_corrector.jl index d6c6a0cc7c..40b60793f3 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_split_explicit_corrector.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_split_explicit_corrector.jl @@ -1,32 +1,34 @@ # Kernels to compute the vertical integral of the velocities -@kernel function _barotropic_mode_kernel!(U, V, grid, ::Nothing, u, v) +@kernel function _barotropic_mode_kernel!(u, v, grid, ::Nothing, U, V, η) i, j = @index(Global, NTuple) - barotropic_mode_kernel!(U, V, i, j, grid, u, v) + barotropic_mode_kernel!(u, v, i, j, grid, U, V, η) end -@kernel function _barotropic_mode_kernel!(U, V, grid, active_cells_map, u, v) +@kernel function _barotropic_mode_kernel!(u, v, grid, active_cells_map, U, V, η) idx = @index(Global, Linear) i, j = active_linear_index_to_tuple(idx, active_cells_map) - barotropic_mode_kernel!(U, V, i, j, grid, u, v) + barotropic_mode_kernel!(U, V, i, j, grid, u, v, η) end -@inline function barotropic_mode_kernel!(U, V, i, j, grid, u, v) - # TODO: +@inline function barotropic_mode_kernel!(U, V, i, j, grid, u, v, η) + + sᶠᶜ = dynamic_column_depthᶠᶜᵃ(i, j, grid, η) / static_column_depthᶠᶜᵃ(i, j, grid) + sᶜᶠ = dynamic_column_depthᶜᶠᵃ(i, j, grid, η) / static_column_depthᶜᶠᵃ(i, j, grid) - @inbounds U[i, j, 1] = Δrᶠᶜᶜ(i, j, 1, grid) * u[i, j, 1] - @inbounds V[i, j, 1] = Δrᶜᶠᶜ(i, j, 1, grid) * v[i, j, 1] + @inbounds U[i, j, 1] = Δrᶠᶜᶜ(i, j, 1, grid) * u[i, j, 1] * sᶠᶜ + @inbounds V[i, j, 1] = Δrᶜᶠᶜ(i, j, 1, grid) * v[i, j, 1] * sᶜᶠ for k in 2:grid.Nz - @inbounds U[i, j, 1] += Δrᶠᶜᶜ(i, j, k, grid) * u[i, j, k] - @inbounds V[i, j, 1] += Δrᶜᶠᶜ(i, j, k, grid) * v[i, j, k] + @inbounds U[i, j, 1] += Δrᶠᶜᶜ(i, j, k, grid) * u[i, j, k] * sᶠᶜ + @inbounds V[i, j, 1] += Δrᶜᶠᶜ(i, j, k, grid) * v[i, j, k] * sᶜᶠ end return nothing end -@inline function compute_barotropic_mode!(U, V, grid, u, v) +@inline function compute_barotropic_mode!(U, V, grid, u, v, η) active_cells_map = retrieve_surface_active_cells_map(grid) - launch!(architecture(grid), grid, :xy, _barotropic_mode_kernel!, U, V, grid, active_cells_map, u, v; active_cells_map) + launch!(architecture(grid), grid, :xy, _barotropic_mode_kernel!, U, V, grid, active_cells_map, u, v, η; active_cells_map) return nothing end @@ -53,7 +55,7 @@ function barotropic_split_explicit_corrector!(u, v, free_surface, grid) # NOTE: the filtered `U̅` and `V̅` have been copied in the instantaneous `U` and `V`, # so we use the filtered velocities as "work arrays" to store the vertical integrals # of the instantaneous velocities `u` and `v`. - compute_barotropic_mode!(U̅, V̅, grid, u, v) + compute_barotropic_mode!(U̅, V̅, grid, u, v, η) # add in "good" barotropic mode launch!(arch, grid, :xyz, _barotropic_split_explicit_corrector!, diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/initialize_split_explicit_substepping.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/initialize_split_explicit_substepping.jl index 521fd02737..56f1b6b959 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/initialize_split_explicit_substepping.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/initialize_split_explicit_substepping.jl @@ -12,7 +12,7 @@ using Oceananigans.ImmersedBoundaries: retrieve_surface_active_cells_map, periph # from the initial velocity conditions. function initialize_free_surface!(sefs::SplitExplicitFreeSurface, grid, velocities) barotropic_velocities = sefs.barotropic_velocities - @apply_regionally compute_barotropic_mode!(barotropic_velocities.U, barotropic_velocities.V, grid, velocities.u, velocities.v) + @apply_regionally compute_barotropic_mode!(barotropic_velocities.U, barotropic_velocities.V, grid, velocities.u, velocities.v, sefs.η) fill_halo_regions!((barotropic_velocities.U, barotropic_velocities.V)) return nothing end diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/step_split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/step_split_explicit_free_surface.jl index 8c6a33d5f3..143668ae09 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/step_split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/step_split_explicit_free_surface.jl @@ -170,5 +170,8 @@ function split_explicit_free_surface_step!(free_surface::SplitExplicitFreeSurfac mask_immersed_field!(model.velocities.v) end + # Needed for ZStar + fill_halo_regions!(η) + return nothing end diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index 02a9923415..e45947be89 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -2,9 +2,9 @@ using Oceananigans using Oceananigans.Grids using Oceananigans.Operators using Oceananigans.BuoyancyModels: buoyancy_perturbationᶜᶜᶜ -using Oceananigans.Grids: AbstractGrid, AbstractUnderlyingGrid, halo_size, AbstractVerticalCoordinateUnderlyingGrid +using Oceananigans.Grids: AbstractGrid, AbstractUnderlyingGrid, halo_size using Oceananigans.ImmersedBoundaries -using Oceananigans.ImmersedBoundaries: ImmersedAbstractVerticalCoordinateGrid +using Oceananigans.Models: AbstractVerticalCoordinateGrid using Oceananigans.Utils: getnamewrapper using Oceananigans.Grids: with_halo, ∂t_grid, vertical_scaling, previous_vertical_scaling using Adapt @@ -12,13 +12,11 @@ using Printf import Oceananigans.Architectures: arch_array -const AbstractVerticalCoordinateGrid = Union{AbstractVerticalCoordinateUnderlyingGrid, ImmersedAbstractVerticalCoordinateGrid} - ##### ##### General implementation ##### -update_grid!(model, grid; kwargs...) = nothing +update_grid!(model, grid; parameters = :xy) = nothing ##### ##### Additional terms to be included in the momentum equations (fallbacks) diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 998b0edcb3..8d14dada59 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -1,20 +1,16 @@ using Oceananigans.Grids -using Oceananigans.Grids: ZStarUnderlyingGrid, rnode -using Oceananigans.ImmersedBoundaries: ImmersedZStarGrid -using Oceananigans.Models.SplitExplicitFreeSurfaces: dynamic_column_depthᶜᶜᵃ, - dynamic_column_depthᶜᶠᵃ, - dynamic_column_depthᶠᶜᵃ, - dynamic_column_depthᶠᶠᵃ - -const ZStarSpacingGrid = Union{ZStarUnderlyingGrid, ImmersedZStarGrid} +using Oceananigans.Models.HydrostaticFreeSurfaceModels.SplitExplicitFreeSurfaces: dynamic_column_depthᶜᶜᵃ, + dynamic_column_depthᶜᶠᵃ, + dynamic_column_depthᶠᶜᵃ, + dynamic_column_depthᶠᶠᵃ ##### ##### ZStar-specific vertical spacings update ##### function update_grid!(model, grid::ZStarSpacingGrid; parameters = :xy) - + # Scaling (just update once, they are the same for all the metrics) sᶜᶜ⁻ = grid.Δzᵃᵃᶠ.sᶜᶜ⁻ sᶜᶜⁿ = grid.Δzᵃᵃᶠ.sᶜᶜⁿ @@ -30,9 +26,9 @@ function update_grid!(model, grid::ZStarSpacingGrid; parameters = :xy) # TODO: At the moment only SplitExplicitFreeSurface is supported, # but zstar can be extended to other free surface solvers by calculating # the barotropic velocity in this step - U̅ = model.free_surface.barotropic_velocities.U - V̅ = model.free_surface.barotropic_velocities.V - η = model.free_surface.η + U = model.free_surface.barotropic_velocities.U + V = model.free_surface.barotropic_velocities.V + η = model.free_surface.η # Update vertical spacing with available parameters # No need to fill the halo as the scaling is updated _IN_ the halos @@ -44,22 +40,22 @@ function update_grid!(model, grid::ZStarSpacingGrid; parameters = :xy) # to the divergence of the vertically integrated velocity field, such that # ∂ₜ((H + η) / H) = H⁻¹ ∂ₜη = - H⁻¹ ∇ ⋅ ∫udz launch!(architecture(grid), grid, parameters, _update_∂t_s!, - ∂t_s, U̅, V̅, grid) + ∂t_s, U, V, grid) return nothing end # NOTE: The ZStar vertical spacing only supports a SplitExplicitFreeSurface # TODO: extend to support other free surface solvers -@kernel function _update_∂t_s!(∂t_s, U̅, V̅, grid) +@kernel function _update_∂t_s!(∂t_s, U, V, grid) i, j = @index(Global, NTuple) - k_top = grid.Nz + 1 + kᴺ = size(grid, 3) # ∂(η / H)/∂t = - ∇ ⋅ ∫udz / H - δx_U = δxᶜᶜᶠ(i, j, k_top-1, grid, Δy_qᶠᶜᶠ, U) - δy_V = δyᶜᶜᶠ(i, j, k_top-1, grid, Δx_qᶜᶠᶠ, V) + δx_U = δxᶜᶜᶜ(i, j, kᴺ, grid, Δy_qᶠᶜᶜ, U) + δy_V = δyᶜᶜᶜ(i, j, kᴺ, grid, Δx_qᶜᶠᶜ, V) - δh_U = (δx_U + δy_V) / Azᶜᶜᶠ(i, j, k_top-1, grid) + δh_U = (δx_U + δy_V) / Azᶜᶜᶜ(i, j, kᴺ, grid) H = static_column_depthᶜᶜᵃ(i, j, grid) @inbounds ∂t_s[i, j] = ifelse(H == 0, zero(grid), - δh_U / H) @@ -74,10 +70,10 @@ end hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) hᶠᶠ = static_column_depthᶠᶠᵃ(i, j, grid) - Hᶜᶜ = dynamic_column_depthᶜᶜᵃ(i, j, k_top, grid, η) - Hᶠᶜ = dynamic_column_depthᶠᶜᵃ(i, j, k_top, grid, η) - Hᶜᶠ = dynamic_column_depthᶜᶠᵃ(i, j, k_top, grid, η) - Hᶠᶠ = dynamic_column_depthᶠᶠᵃ(i, j, k_top, grid, η) + Hᶜᶜ = dynamic_column_depthᶜᶜᵃ(i, j, grid, η) + Hᶠᶜ = dynamic_column_depthᶠᶜᵃ(i, j, grid, η) + Hᶜᶠ = dynamic_column_depthᶜᶠᵃ(i, j, grid, η) + Hᶠᶠ = dynamic_column_depthᶠᶠᵃ(i, j, grid, η) @inbounds begin sᶜᶜ = ifelse(hᶜᶜ == 0, one(grid), Hᶜᶜ / hᶜᶜ) diff --git a/src/Models/Models.jl b/src/Models/Models.jl index 4981986dd5..d742538b1e 100644 --- a/src/Models/Models.jl +++ b/src/Models/Models.jl @@ -23,6 +23,12 @@ import Oceananigans.Architectures: architecture import Oceananigans.TimeSteppers: reset! import Oceananigans.Solvers: iteration +using Oceananigans.Grids: AbstractVerticalCoordinateUnderlyingGrid +using Oceananigans.ImmersedBoundaries: ImmersedAbstractVerticalCoordinateGrid + +const AbstractVerticalCoordinateGrid = Union{AbstractVerticalCoordinateUnderlyingGrid, ImmersedAbstractVerticalCoordinateGrid} + + # A prototype interface for AbstractModel. # # TODO: decide if we like this. diff --git a/src/OutputWriters/OutputWriters.jl b/src/OutputWriters/OutputWriters.jl index d1d12cb23f..a6b346d016 100644 --- a/src/OutputWriters/OutputWriters.jl +++ b/src/OutputWriters/OutputWriters.jl @@ -16,6 +16,7 @@ using Oceananigans: AbstractOutputWriter using Oceananigans.Grids: interior_indices using Oceananigans.Utils: TimeInterval, IterationInterval, WallTimeInterval, instantiate using Oceananigans.Utils: pretty_filesize +using Oceananigans.Models: AbstractVerticalCoordinateGrid using OffsetArrays diff --git a/src/OutputWriters/output_writer_utils.jl b/src/OutputWriters/output_writer_utils.jl index 0ca4102d20..8ededd10a2 100644 --- a/src/OutputWriters/output_writer_utils.jl +++ b/src/OutputWriters/output_writer_utils.jl @@ -6,7 +6,6 @@ using Oceananigans.DistributedComputations: DistributedGrid, Partition using Oceananigans.Fields: AbstractField, indices, boundary_conditions, instantiated_location using Oceananigans.BoundaryConditions: bc_str, FieldBoundaryConditions, ContinuousBoundaryFunction, DiscreteBoundaryFunction using Oceananigans.TimeSteppers: QuasiAdamsBashforth2TimeStepper, RungeKutta3TimeStepper -using Oceananigans.Models.HydrostaticFreeSurfaceModels: AbstractVerticalCoordinateGrid using Oceananigans.Models.LagrangianParticleTracking: LagrangianParticles using Oceananigans.Utils: AbstractSchedule diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index 50d1c13380..b7fe8f79e2 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -14,7 +14,7 @@ grid = RectilinearGrid(size = (128, 20), halo = (6, 6), topology = (Bounded, Flat, Bounded)) -grid = ImmersedBoundaryGrid(grid, GridFittedBottom(x -> - (64kilometers - x) / 64kilometers * 20)) +# grid = ImmersedBoundaryGrid(grid, GridFittedBottom(x -> - (64kilometers - x) / 64kilometers * 20)) model = HydrostaticFreeSurfaceModel(; grid, momentum_advection = FluxFormAdvection(WENO(; order = 5), nothing, WENO(; order = 5)), From 0f69fcb934ba7f5c53fed518c8a253ced66b4b35 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Sun, 17 Nov 2024 17:27:11 +0000 Subject: [PATCH 463/567] new zstar --- src/Grids/abstract_grid.jl | 8 +-- src/Grids/grid_generation.jl | 65 +++++++++++++++++-- src/Grids/grid_utils.jl | 6 ++ src/Grids/latitude_longitude_grid.jl | 15 ++--- src/Grids/orthogonal_spherical_shell_grid.jl | 5 +- src/Grids/rectilinear_grid.jl | 6 +- src/Grids/vertical_coordinates.jl | 53 +++++++++++++++ src/Grids/z_star_vertical_coordinate.jl | 41 +----------- .../SplitExplicitFreeSurfaces.jl | 10 +-- src/Operators/variable_grid_operators.jl | 51 +++++++++++++++ validation/z_star_coordinate/lock_release.jl | 6 +- 11 files changed, 191 insertions(+), 75 deletions(-) create mode 100644 src/Grids/vertical_coordinates.jl create mode 100644 src/Operators/variable_grid_operators.jl diff --git a/src/Grids/abstract_grid.jl b/src/Grids/abstract_grid.jl index 92b0c3f21b..9285b3c8c2 100644 --- a/src/Grids/abstract_grid.jl +++ b/src/Grids/abstract_grid.jl @@ -3,7 +3,7 @@ Abstract supertype for grids with elements of type `FT` and topology `{TX, TY, TZ}`. """ -abstract type AbstractGrid{FT, TX, TY, TZ, Arch} end +abstract type AbstractGrid{FT, TX, TY, TZ, CZ, Arch} end """ AbstractUnderlyingGrid{FT, TX, TY, TZ} @@ -11,21 +11,21 @@ abstract type AbstractGrid{FT, TX, TY, TZ, Arch} end Abstract supertype for "primary" grids (as opposed to grids with immersed boundaries) with elements of type `FT` and topology `{TX, TY, TZ}`. """ -abstract type AbstractUnderlyingGrid{FT, TX, TY, TZ, Arch} <: AbstractGrid{FT, TX, TY, TZ, Arch} end +abstract type AbstractUnderlyingGrid{FT, TX, TY, TZ, CZ, Arch} <: AbstractGrid{FT, TX, TY, TZ, CZ, Arch} end """ AbstractCurvilinearGrid{FT, TX, TY, TZ} Abstract supertype for curvilinear grids with elements of type `FT` and topology `{TX, TY, TZ}`. """ -abstract type AbstractCurvilinearGrid{FT, TX, TY, TZ, Arch} <: AbstractUnderlyingGrid{FT, TX, TY, TZ, Arch} end +abstract type AbstractCurvilinearGrid{FT, TX, TY, TZ, CZ, Arch} <: AbstractUnderlyingGrid{FT, TX, TY, TZ, CZ, Arch} end """ AbstractHorizontallyCurvilinearGrid{FT, TX, TY, TZ} Abstract supertype for horizontally-curvilinear grids with elements of type `FT` and topology `{TX, TY, TZ}`. """ -abstract type AbstractHorizontallyCurvilinearGrid{FT, TX, TY, TZ, Arch} <: AbstractCurvilinearGrid{FT, TX, TY, TZ, Arch} end +abstract type AbstractHorizontallyCurvilinearGrid{FT, TX, TY, TZ, CZ, Arch} <: AbstractCurvilinearGrid{FT, TX, TY, TZ, CZ, Arch} end const XFlatGrid = AbstractGrid{<:Any, Flat} const YFlatGrid = AbstractGrid{<:Any, <:Any, Flat} diff --git a/src/Grids/grid_generation.jl b/src/Grids/grid_generation.jl index 6e468f2738..8e3eda6a0d 100644 --- a/src/Grids/grid_generation.jl +++ b/src/Grids/grid_generation.jl @@ -87,7 +87,11 @@ function generate_coordinate(FT, topo::AT, N, H, node_generator, coordinate_name F = OffsetArray(on_architecture(arch, F.parent), F.offsets...) C = OffsetArray(on_architecture(arch, C.parent), C.offsets...) - return L, F, C, Δᶠ, Δᶜ + if coordinate_name == :z + return L, StaticVerticalCoordinate(F, C, Δᶠ, Δᶜ) + else + return L, F, C, Δᶠ, Δᶜ + end end # Generate a regularly-spaced coordinate passing the domain extent (2-tuple) and number of points @@ -120,16 +124,65 @@ function generate_coordinate(FT, topo::AT, N, H, node_interval::Tuple{<:Number, F = OffsetArray(F, -H) C = OffsetArray(C, -H) - return FT(L), F, C, FT(Δᶠ), FT(Δᶜ) + if coordinate_name == :z + return FT(L), StaticVerticalCoordinate(F, C, FT(Δᶠ), FT(Δᶜ)) + else + return FT(L), F, C, FT(Δᶠ), FT(Δᶜ) + end end # Flat domains -generate_coordinate(FT, ::Flat, N, H, c::Number, coordinate_name, arch) = - FT(1), range(FT(c), FT(c), length=N), range(FT(c), FT(c), length=N), FT(1), FT(1) +function generate_coordinate(FT, ::Flat, N, H, c::Number, coordinate_name, arch) + if coordinate_name == :z + return FT(1), StaticVerticalCoordinate(range(FT(c), FT(c), length=N), range(FT(c), FT(c), length=N), FT(1), FT(1)) + else + return FT(1), range(FT(c), FT(c), length=N), range(FT(c), FT(c), length=N), FT(1), FT(1) + end +end # What's the use case for this? # generate_coordinate(FT, ::Flat, N, H, c::Tuple{Number, Number}, coordinate_name, arch) = # FT(1), c, c, FT(1), FT(1) +function generate_coordinate(FT, ::Flat, N, H, ::Nothing, coordinate_name, arch) + if coordinate_name == :z + return FT(1), StaticVerticalCoordinate(nothing, nothing, FT(1), FT(1)) + else + return FT(1), nothing, nothing, FT(1), FT(1) + end +end + +#### +#### ZStarVerticalCoordinate +#### + +generate_coordinate(FT, ::Periodic, N, H, ::ZStarVerticalCoordinate, coordinate_name, arch, args...) = + throw(ArgumentError("Periodic domains are not supported for ZStarVerticalCoordinate")) + +# Generate a moving coordinate with evolving scaling (`s`) for spacings and znodes +function generate_coordinate(FT, topo, size, halo, coordinate::ZStarVerticalCoordinate, coordinate_name, dim::Int, arch) + + Nx, Ny, Nz = size + Hx, Hy, Hz = halo + + if dim != 3 + msg = "ZStarVerticalCoordinate is supported only in the third dimension (z)" + throw(ArgumentError(msg)) + end + + if coordinate_name != :z + msg = "Only z-coordinate is supported for ZStarVerticalCoordinate" + throw(ArgumentError(msg)) + end -generate_coordinate(FT, ::Flat, N, H, ::Nothing, coordinate_name, arch) = - FT(1), nothing, nothing, FT(1), FT(1) + r_faces = coordinate.reference + + Lr, rᵃᵃᶠ, rᵃᵃᶜ, Δrᵃᵃᶠ, Δrᵃᵃᶜ = generate_coordinate(FT, topo[3](), Nz, Hz, r_faces, :z, arch) + + args = (topo, (Nx, Ny, Nz), (Hx, Hy, Hz)) + + sᶜᶜᵃ = new_data(FT, arch, (Center, Center, Nothing), args...) + sᶜᶜᵃ₋ = new_data(FT, arch, (Center, Center, Nothing), args...) + ∂t_s = new_data(FT, arch, (Center, Center, Nothing), args...) + + return Lr, ZStarVerticalCoordinate(rᵃᵃᶠ, rᵃᵃᶜ, Δrᵃᵃᶠ, Δrᵃᵃᶜ, sᶜᶜᵃ, sᶜᶜᵃ₋, ∂t_s) +end diff --git a/src/Grids/grid_utils.jl b/src/Grids/grid_utils.jl index 178d7ab87c..4b52d73a71 100644 --- a/src/Grids/grid_utils.jl +++ b/src/Grids/grid_utils.jl @@ -300,6 +300,12 @@ function domain_summary(topo, name, (left, right)) prettysummary(right), interval) end +function dimension_summary(topo, name, dom, z::AbstractVerticalCoordinate, pad_domain=0) + prefix = domain_summary(topo, name, dom) + padding = " "^(pad_domain+1) + return string(prefix, padding, coordinate_summary(topo, z.Δᶜ, name)) +end + function dimension_summary(topo, name, dom, spacing, pad_domain=0) prefix = domain_summary(topo, name, dom) padding = " "^(pad_domain+1) diff --git a/src/Grids/latitude_longitude_grid.jl b/src/Grids/latitude_longitude_grid.jl index 93bab354e0..70879acf17 100644 --- a/src/Grids/latitude_longitude_grid.jl +++ b/src/Grids/latitude_longitude_grid.jl @@ -321,7 +321,7 @@ function Base.show(io::IO, grid::LatitudeLongitudeGrid, withsummary=true) x_summary = "longitude: " * dimension_summary(TX(), "λ", Ωλ, grid.Δλᶜᵃᵃ, longest - length(x_summary)) y_summary = "latitude: " * dimension_summary(TY(), "φ", Ωφ, grid.Δφᵃᶜᵃ, longest - length(y_summary)) - z_summary = "z: " * dimension_summary(TZ(), "z", Ωz, grid.Δzᵃᵃᶜ, longest - length(z_summary)) + z_summary = "z: " * dimension_summary(TZ(), "z", Ωz, grid.z, longest - length(z_summary)) if withsummary print(io, summary(grid), "\n") @@ -583,14 +583,10 @@ rname(::LLG) = :z @inline λnode(i, grid::LLG, ::Face) = getnode(grid.λᶠᵃᵃ, i) @inline φnode(j, grid::LLG, ::Center) = getnode(grid.φᵃᶜᵃ, j) @inline φnode(j, grid::LLG, ::Face) = getnode(grid.φᵃᶠᵃ, j) -@inline znode(k, grid::LLG, ::Center) = getnode(grid.zᵃᵃᶜ, k) -@inline znode(k, grid::LLG, ::Face) = getnode(grid.zᵃᵃᶠ, k) # Definitions for node @inline ξnode(i, j, k, grid::LLG, ℓx, ℓy, ℓz) = λnode(i, grid, ℓx) @inline ηnode(i, j, k, grid::LLG, ℓx, ℓy, ℓz) = φnode(j, grid, ℓy) -@inline rnode(i, j, k, grid::LLG, ℓx, ℓy, ℓz) = znode(k, grid, ℓz) - @inline xnode(i, j, grid::LLG, ℓx, ℓy) = grid.radius * deg2rad(λnode(i, grid, ℓx)) * hack_cosd((φnode(j, grid, ℓy))) @inline ynode(j, grid::LLG, ℓy) = grid.radius * deg2rad(φnode(j, grid, ℓy)) @@ -599,7 +595,6 @@ rname(::LLG) = :z @inline φnode(i, j, k, grid::LLG, ℓx, ℓy, ℓz) = φnode(j, grid, ℓy) @inline xnode(i, j, k, grid::LLG, ℓx, ℓy, ℓz) = xnode(i, j, grid, ℓx, ℓy) @inline ynode(i, j, k, grid::LLG, ℓx, ℓy, ℓz) = ynode(j, grid, ℓy) -@inline znode(i, j, k, grid::LLG, ℓx, ℓy, ℓz) = znode(k, grid, ℓz) function nodes(grid::LLG, ℓx, ℓy, ℓz; reshape=false, with_halos=false) λ = λnodes(grid, ℓx, ℓy, ℓz; with_halos) @@ -648,10 +643,10 @@ end return @. R * deg2rad(φ) end -@inline znodes(grid::LLG, ℓz::F; with_halos=false) = _property(grid.zᵃᵃᶠ, ℓz, topology(grid, 3), size(grid, 3), with_halos) -@inline znodes(grid::LLG, ℓz::C; with_halos=false) = _property(grid.zᵃᵃᶜ, ℓz, topology(grid, 3), size(grid, 3), with_halos) -@inline rnodes(grid::LLG, ℓz::F; with_halos=false) = _property(grid.zᵃᵃᶠ, ℓz, topology(grid, 3), size(grid, 3), with_halos) -@inline rnodes(grid::LLG, ℓz::C; with_halos=false) = _property(grid.zᵃᵃᶜ, ℓz, topology(grid, 3), size(grid, 3), with_halos) +@inline znodes(grid::LLG, ℓz::F; with_halos=false) = _property(grid.z.cᶠ, ℓz, topology(grid, 3), size(grid, 3), with_halos) +@inline znodes(grid::LLG, ℓz::C; with_halos=false) = _property(grid.z.cᶜ, ℓz, topology(grid, 3), size(grid, 3), with_halos) +@inline rnodes(grid::LLG, ℓz::F; with_halos=false) = _property(grid.z.cᶠ, ℓz, topology(grid, 3), size(grid, 3), with_halos) +@inline rnodes(grid::LLG, ℓz::C; with_halos=false) = _property(grid.z.cᶜ, ℓz, topology(grid, 3), size(grid, 3), with_halos) # Convenience @inline λnodes(grid::LLG, ℓx, ℓy, ℓz; with_halos=false) = λnodes(grid, ℓx; with_halos) diff --git a/src/Grids/orthogonal_spherical_shell_grid.jl b/src/Grids/orthogonal_spherical_shell_grid.jl index fff0aa04d4..d6aecbb2e7 100644 --- a/src/Grids/orthogonal_spherical_shell_grid.jl +++ b/src/Grids/orthogonal_spherical_shell_grid.jl @@ -1081,7 +1081,7 @@ function Base.show(io::IO, grid::OrthogonalSphericalShellGrid, withsummary=true) φ_summary = "latitude: $(TY) extent $(prettysummary(extent_φ)) degrees" * padding_φ * " " * coordinate_summary(TY, rad2deg.(grid.Δyᶠᶠᵃ[1:Nx_face, 1:Ny_face] ./ grid.radius), "φ") - z_summary = "z: " * dimension_summary(TZ(), "z", Ωz, grid.Δzᵃᵃᶜ, longest - length(z_summary)) + z_summary = "z: " * dimension_summary(TZ(), "z", Ωz, grid.z, longest - length(z_summary)) if withsummary print(io, summary(grid), "\n") @@ -1187,9 +1187,6 @@ end @inline xnode(i, j, grid::OSSG, ℓx, ℓy) = grid.radius * deg2rad(λnode(i, j, grid, ℓx, ℓy)) * hack_cosd((φnode(i, j, grid, ℓx, ℓy))) @inline ynode(i, j, grid::OSSG, ℓx, ℓy) = grid.radius * deg2rad(φnode(i, j, grid, ℓx, ℓy)) -@inline znode(k, grid::OSSG, ::Center) = @inbounds grid.zᵃᵃᶜ[k] -@inline znode(k, grid::OSSG, ::Face ) = @inbounds grid.zᵃᵃᶠ[k] - # convenience @inline λnode(i, j, k, grid::OSSG, ℓx, ℓy, ℓz) = λnode(i, j, grid, ℓx, ℓy) @inline φnode(i, j, k, grid::OSSG, ℓx, ℓy, ℓz) = φnode(i, j, grid, ℓx, ℓy) diff --git a/src/Grids/rectilinear_grid.jl b/src/Grids/rectilinear_grid.jl index 1934972ba3..d6a17c473f 100644 --- a/src/Grids/rectilinear_grid.jl +++ b/src/Grids/rectilinear_grid.jl @@ -332,7 +332,7 @@ function Base.show(io::IO, grid::RectilinearGrid, withsummary=true) x_summary = dimension_summary(TX(), "x", Ωx, grid.Δxᶜᵃᵃ, longest - length(x_summary)) y_summary = dimension_summary(TY(), "y", Ωy, grid.Δyᵃᶜᵃ, longest - length(y_summary)) - z_summary = dimension_summary(TZ(), "z", Ωz, grid.Δzᵃᵃᶜ, longest - length(z_summary)) + z_summary = dimension_summary(TZ(), "z", Ωz, grid.z, longest - length(z_summary)) if withsummary print(io, summary(grid), "\n") @@ -454,17 +454,13 @@ rname(::RG) = :z @inline xnode(i, grid::RG, ::Face) = getnode(grid.xᶠᵃᵃ, i) @inline ynode(j, grid::RG, ::Center) = getnode(grid.yᵃᶜᵃ, j) @inline ynode(j, grid::RG, ::Face) = getnode(grid.yᵃᶠᵃ, j) -@inline znode(k, grid::RG, ::Center) = getnode(grid.zᵃᵃᶜ, k) -@inline znode(k, grid::RG, ::Face) = getnode(grid.zᵃᵃᶠ, k) @inline ξnode(i, j, k, grid::RG, ℓx, ℓy, ℓz) = xnode(i, grid, ℓx) @inline ηnode(i, j, k, grid::RG, ℓx, ℓy, ℓz) = ynode(j, grid, ℓy) -@inline rnode(i, j, k, grid::RG, ℓx, ℓy, ℓz) = znode(k, grid, ℓz) # Convenience definitions for x, y, znode @inline xnode(i, j, k, grid::RG, ℓx, ℓy, ℓz) = xnode(i, grid, ℓx) @inline ynode(i, j, k, grid::RG, ℓx, ℓy, ℓz) = ynode(j, grid, ℓy) -@inline znode(i, j, k, grid::RG, ℓx, ℓy, ℓz) = znode(k, grid, ℓz) function nodes(grid::RectilinearGrid, ℓx, ℓy, ℓz; reshape=false, with_halos=false) x = xnodes(grid, ℓx, ℓy, ℓz; with_halos) diff --git a/src/Grids/vertical_coordinates.jl b/src/Grids/vertical_coordinates.jl new file mode 100644 index 0000000000..50ef7821d5 --- /dev/null +++ b/src/Grids/vertical_coordinates.jl @@ -0,0 +1,53 @@ +abstract type AbstractVerticalCoordinate end + +struct StaticVerticalCoordinate{C, D} <: AbstractVerticalCoordinate + cᶠ :: C + cᶜ :: C + Δᶠ :: D + Δᶠ :: D +end + +struct ZStarVerticalCoordinate{C, D, E, S} <: AbstractVerticalCoordinate + cᶠ :: C + cᶜ :: C + Δᶠ :: D + Δᶠ :: D + ηⁿ :: E + η⁻ :: E + ∂t_s :: S +end + +const AbstractZStarGrid = AbstractGrid{<:Any, <:Any, <:Any, <:Bounded, <:ZStarVerticalCoordinate} + +##### +##### vertical nodes... +##### + +@inline rnode(i, j, k, grid, ℓx, ℓy, ℓz) = rnode(k, grid, ℓz) +@inline rnode(k, grid, ::Center) = getnode(grid.z.cᶜ, k) +@inline rnode(k, grid, ::Face) = getnode(grid.z.cᶠ, k) + +@inline znode(k, grid, ℓz) = rnode(k, grid, ℓz) +@inline znode(i, j, k, grid, ℓx, ℓy, ℓz) = rnode(k, grid, ℓz) + +# TO extend in operators +@inline znode(i, j, k, grid::AbstractZStarGrid, ℓx, ℓy, ℓz) = znode(k, grid, ℓz) + +@inline znode(k, grid, ℓx) = getnode(grid.z, k) + +# Convenience constructors +ZStarVerticalCoordinate(r_faces::Union{Tuple, AbstractVector}) = ZStarVerticalCoordinate(r_faces, r_faces, nothing, nothing, nothing, nothing, nothing) +ZStarVerticalCoordinate(r⁻::Number, r⁺::Number) = ZStarVerticalCoordinate((r⁻, r⁺), (r⁻, r⁺), nothing, nothing, nothing, nothing, nothing) + +Grids.coordinate_summary(::Bounded, Δ::ZStarVerticalCoordinate, name) = + @sprintf("Free-surface following with Δ%s=%s", name, prettysummary(Δ.reference)) + +Grids.coordinate_summary(::Bounded, Δ::ZStarVerticalCoordinate, name) = + @sprintf("Free-surface following with Δ%s=%s", name, prettysummary(Δ.reference)) + +function validate_dimension_specification(T, ξ::ZStarVerticalCoordinate, dir, N, FT) + reference = validate_dimension_specification(T, ξ.reference, dir, N, FT) + args = Tuple(getproperty(ξ, prop) for prop in propertynames(ξ)) + + return ZStarVerticalCoordinate(reference, args[2:end]...) +end diff --git a/src/Grids/z_star_vertical_coordinate.jl b/src/Grids/z_star_vertical_coordinate.jl index f36ebb7192..748400f349 100644 --- a/src/Grids/z_star_vertical_coordinate.jl +++ b/src/Grids/z_star_vertical_coordinate.jl @@ -152,8 +152,8 @@ function generate_coordinate(FT, topo, size, halo, coordinate::ZStarVerticalCoor # The scaling is the same for everyone, the vertical coordinate requires # to add the free surface to retrieve the znode. - zᵃᵃᶠ = ZStarVerticalCoordinate(rᵃᵃᶠ, sᶜᶜᵃ, sᶠᶜᵃ, sᶜᶠᵃ, sᶠᶠᵃ, sᶜᶜᵃ₋, sᶠᶜᵃ₋, sᶜᶠᵃ₋, η) - zᵃᵃᶜ = ZStarVerticalCoordinate(rᵃᵃᶜ, sᶜᶜᵃ, sᶠᶜᵃ, sᶜᶠᵃ, sᶠᶠᵃ, sᶜᶜᵃ₋, sᶠᶜᵃ₋, sᶜᶠᵃ₋, η) + zᵃᵃᶠ = ZStarVerticalCoordinate(rᵃᵃᶠ, ηᶜᶜᵃ, sᶠᶜᵃ, sᶜᶠᵃ, sᶠᶠᵃ, sᶜᶜᵃ₋, sᶠᶜᵃ₋, sᶜᶠᵃ₋, η) + zᵃᵃᶜ = ZStarVerticalCoordinate(rᵃᵃᶜ, ηᶜᶜᵃ, sᶠᶜᵃ, sᶜᶠᵃ, sᶠᶠᵃ, sᶜᶜᵃ₋, sᶠᶜᵃ₋, sᶜᶠᵃ₋, η) Δzᵃᵃᶠ = ZStarVerticalCoordinate(Δrᵃᵃᶠ, sᶜᶜᵃ, sᶠᶜᵃ, sᶜᶠᵃ, sᶠᶠᵃ, sᶜᶜᵃ₋, sᶠᶜᵃ₋, sᶜᶠᵃ₋, ∂t_s) Δzᵃᵃᶜ = ZStarVerticalCoordinate(Δrᵃᵃᶜ, sᶜᶜᵃ, sᶠᶜᵃ, sᶜᶠᵃ, sᶠᶠᵃ, sᶜᶜᵃ₋, sᶠᶜᵃ₋, sᶜᶠᵃ₋, ∂t_s) @@ -186,36 +186,13 @@ const F = Face const ZSG = ZStarUnderlyingGrid -# Fallbacks -@inline vertical_scaling(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) -@inline previous_vertical_scaling(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) - -@inline ∂t_grid(i, j, k, grid) = zero(grid) - @inline rnodes(grid::ZSG, ℓz::C; with_halos=false) = _property(grid.zᵃᵃᶜ.reference, ℓz, topology(grid, 3), size(grid, 3), with_halos) @inline rnodes(grid::ZSG, ℓz::F; with_halos=false) = _property(grid.zᵃᵃᶠ.reference, ℓz, topology(grid, 3), size(grid, 3), with_halos) @inline rspacings(grid::ZSG, ℓz::C; with_halos=false) = _property(grid.Δzᵃᵃᶜ.reference, ℓz, topology(grid, 3), size(grid, 3), with_halos) @inline rspacings(grid::ZSG, ℓz::F; with_halos=false) = _property(grid.Δzᵃᵃᶠ.reference, ℓz, topology(grid, 3), size(grid, 3), with_halos) -@inline vertical_scaling(i, j, k, grid::ZSG, ::C, ::C, ::C) = @inbounds grid.Δzᵃᵃᶜ.sᶜᶜⁿ[i, j] -@inline vertical_scaling(i, j, k, grid::ZSG, ::F, ::C, ::C) = @inbounds grid.Δzᵃᵃᶜ.sᶠᶜⁿ[i, j] -@inline vertical_scaling(i, j, k, grid::ZSG, ::C, ::F, ::C) = @inbounds grid.Δzᵃᵃᶜ.sᶜᶠⁿ[i, j] -@inline vertical_scaling(i, j, k, grid::ZSG, ::F, ::F, ::C) = @inbounds grid.Δzᵃᵃᶜ.sᶠᶠⁿ[i, j] - -@inline vertical_scaling(i, j, k, grid::ZSG, ::C, ::C, ::F) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶜⁿ[i, j] -@inline vertical_scaling(i, j, k, grid::ZSG, ::F, ::C, ::F) = @inbounds grid.Δzᵃᵃᶠ.sᶠᶜⁿ[i, j] -@inline vertical_scaling(i, j, k, grid::ZSG, ::C, ::F, ::F) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶠⁿ[i, j] -@inline vertical_scaling(i, j, k, grid::ZSG, ::F, ::F, ::F) = @inbounds grid.Δzᵃᵃᶠ.sᶠᶠⁿ[i, j] - -@inline previous_vertical_scaling(i, j, k, grid::ZSG, ::C, ::C, ::C) = @inbounds grid.Δzᵃᵃᶜ.sᶜᶜ⁻[i, j] -@inline previous_vertical_scaling(i, j, k, grid::ZSG, ::F, ::C, ::C) = @inbounds grid.Δzᵃᵃᶜ.sᶠᶜ⁻[i, j] -@inline previous_vertical_scaling(i, j, k, grid::ZSG, ::C, ::F, ::C) = @inbounds grid.Δzᵃᵃᶜ.sᶜᶠ⁻[i, j] - -@inline previous_vertical_scaling(i, j, k, grid::ZSG, ::C, ::C, ::F) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶜ⁻[i, j] -@inline previous_vertical_scaling(i, j, k, grid::ZSG, ::F, ::C, ::F) = @inbounds grid.Δzᵃᵃᶠ.sᶠᶜ⁻[i, j] -@inline previous_vertical_scaling(i, j, k, grid::ZSG, ::C, ::F, ::F) = @inbounds grid.Δzᵃᵃᶠ.sᶜᶠ⁻[i, j] - +@inline ∂t_grid(i, j, k, grid) = zero(grid) @inline ∂t_grid(i, j, k, grid::ZSG) = @inbounds grid.Δzᵃᵃᶜ.∂t_s[i, j] ##### @@ -228,15 +205,3 @@ const f = Face() # rnode for an ZStarUnderlyingGrid is the reference node @inline rnode(i, j, k, grid::ZSG, ℓx, ℓy, ::Center) = @inbounds grid.zᵃᵃᶜ.reference[k] @inline rnode(i, j, k, grid::ZSG, ℓx, ℓy, ::Face) = @inbounds grid.zᵃᵃᶠ.reference[k] - -# rnode for an ZStarUnderlyingGrid grid is scaled -# TODO: fix this when bottom height is implemented -@inline znode(i, j, k, grid::ZSG, ::C, ::C, ::C) = @inbounds grid.zᵃᵃᶜ.reference[k] * vertical_scaling(i, j, k, grid, c, c, c) + grid.zᵃᵃᶜ.∂t_s[i, j] -@inline znode(i, j, k, grid::ZSG, ::C, ::F, ::C) = @inbounds grid.zᵃᵃᶜ.reference[k] * vertical_scaling(i, j, k, grid, c, f, c) + grid.zᵃᵃᶜ.∂t_s[i, j] -@inline znode(i, j, k, grid::ZSG, ::F, ::C, ::C) = @inbounds grid.zᵃᵃᶜ.reference[k] * vertical_scaling(i, j, k, grid, f, c, c) + grid.zᵃᵃᶜ.∂t_s[i, j] -@inline znode(i, j, k, grid::ZSG, ::F, ::F, ::C) = @inbounds grid.zᵃᵃᶜ.reference[k] * vertical_scaling(i, j, k, grid, f, f, c) + grid.zᵃᵃᶜ.∂t_s[i, j] - -@inline znode(i, j, k, grid::ZSG, ::C, ::C, ::F) = @inbounds grid.zᵃᵃᶠ.reference[k] * vertical_scaling(i, j, k, grid, c, c, c) + grid.zᵃᵃᶠ.∂t_s[i, j] -@inline znode(i, j, k, grid::ZSG, ::C, ::F, ::F) = @inbounds grid.zᵃᵃᶠ.reference[k] * vertical_scaling(i, j, k, grid, c, f, c) + grid.zᵃᵃᶠ.∂t_s[i, j] -@inline znode(i, j, k, grid::ZSG, ::F, ::C, ::F) = @inbounds grid.zᵃᵃᶠ.reference[k] * vertical_scaling(i, j, k, grid, f, c, c) + grid.zᵃᵃᶠ.∂t_s[i, j] -@inline znode(i, j, k, grid::ZSG, ::F, ::F, ::F) = @inbounds grid.zᵃᵃᶠ.reference[k] * vertical_scaling(i, j, k, grid, f, f, c) + grid.zᵃᵃᶠ.∂t_s[i, j] \ No newline at end of file diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl index b25bc44800..82622b438c 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl @@ -23,6 +23,11 @@ using Base using KernelAbstractions: @index, @kernel using KernelAbstractions.Extras.LoopInfo: @unroll +import Oceananigans.Grids: dynamic_column_depthᶜᶜᵃ, + dynamic_column_depthᶜᶠᵃ, + dynamic_column_depthᶠᶜᵃ, + dynamic_column_depthᶠᶠᵃ + import Oceananigans.Models.HydrostaticFreeSurfaceModels: initialize_free_surface!, materialize_free_surface, ab2_step_free_surface!, @@ -30,11 +35,6 @@ import Oceananigans.Models.HydrostaticFreeSurfaceModels: initialize_free_surface explicit_barotropic_pressure_x_gradient, explicit_barotropic_pressure_y_gradient -@inline dynamic_column_depthᶜᶜᵃ(i, j, grid, η) = static_column_depthᶜᶜᵃ(i, j, grid) -@inline dynamic_column_depthᶜᶠᵃ(i, j, grid, η) = static_column_depthᶜᶠᵃ(i, j, grid) -@inline dynamic_column_depthᶠᶜᵃ(i, j, grid, η) = static_column_depthᶠᶜᵃ(i, j, grid) -@inline dynamic_column_depthᶠᶠᵃ(i, j, grid, η) = static_column_depthᶠᶠᵃ(i, j, grid) - @inline dynamic_column_depthᶜᶜᵃ(i, j, grid::ZStarSpacingGrid, η) = static_column_depthᶜᶜᵃ(i, j, grid) + @inbounds η[i, j, grid.Nz+1] @inline dynamic_column_depthᶜᶠᵃ(i, j, grid::ZStarSpacingGrid, η) = static_column_depthᶜᶠᵃ(i, j, grid) + ℑxᶠᵃᵃ(i, j, grid.Nz+1, grid, η) @inline dynamic_column_depthᶠᶜᵃ(i, j, grid::ZStarSpacingGrid, η) = static_column_depthᶠᶜᵃ(i, j, grid) + ℑyᵃᶠᵃ(i, j, grid.Nz+1, grid, η) diff --git a/src/Operators/variable_grid_operators.jl b/src/Operators/variable_grid_operators.jl new file mode 100644 index 0000000000..53d289c418 --- /dev/null +++ b/src/Operators/variable_grid_operators.jl @@ -0,0 +1,51 @@ +using Oceananigans.Grids: ZStarUnderlyingGrid +import Oceananigans.Grids: znode + +##### +##### ZStar-specific vertical spacing functions +##### + +const C = Center +const F = Face + +const ZSG = ZStarUnderlyingGrid + +@inline dynamic_column_depthᶜᶜᵃ(i, j, grid, η) = static_column_depthᶜᶜᵃ(i, j, grid) +@inline dynamic_column_depthᶜᶠᵃ(i, j, grid, η) = static_column_depthᶜᶠᵃ(i, j, grid) +@inline dynamic_column_depthᶠᶜᵃ(i, j, grid, η) = static_column_depthᶠᶜᵃ(i, j, grid) +@inline dynamic_column_depthᶠᶠᵃ(i, j, grid, η) = static_column_depthᶠᶠᵃ(i, j, grid) + +# Fallbacks +@inline e₃ⁿ(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) +@inline e₃⁻(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) + +@inline e₃ⁿ(i, j, k, grid::ZSG, ::C, ::C, ℓz) = dynamic_bottom_heightᶜᶜᵃ(i, j, 1, grid, grid.zᵃᵃᶠ.ηⁿ) / static_bottom_heightᶜᶜᵃ(i, j, grid) +@inline e₃ⁿ(i, j, k, grid::ZSG, ::F, ::C, ℓz) = dynamic_bottom_heightᶠᶜᵃ(i, j, 1, grid, grid.zᵃᵃᶠ.ηⁿ) / static_bottom_heightᶠᶜᵃ(i, j, grid) +@inline e₃ⁿ(i, j, k, grid::ZSG, ::C, ::F, ℓz) = dynamic_bottom_heightᶜᶠᵃ(i, j, 1, grid, grid.zᵃᵃᶠ.ηⁿ) / static_bottom_heightᶜᶠᵃ(i, j, grid) +@inline e₃ⁿ(i, j, k, grid::ZSG, ::F, ::F, ℓz) = dynamic_bottom_heightᶠᶠᵃ(i, j, 1, grid, grid.zᵃᵃᶠ.ηⁿ) / static_bottom_heightᶠᶠᵃ(i, j, grid) + +@inline e₃⁻(i, j, k, grid::ZSG, ::C, ::C, ℓz) = dynamic_bottom_heightᶜᶜᵃ(i, j, 1, grid, grid.zᵃᵃᶠ.η⁻) / static_bottom_heightᶜᶜᵃ(i, j, grid) +@inline e₃⁻(i, j, k, grid::ZSG, ::F, ::C, ℓz) = dynamic_bottom_heightᶠᶜᵃ(i, j, 1, grid, grid.zᵃᵃᶠ.η⁻) / static_bottom_heightᶠᶜᵃ(i, j, grid) +@inline e₃⁻(i, j, k, grid::ZSG, ::C, ::F, ℓz) = dynamic_bottom_heightᶜᶠᵃ(i, j, 1, grid, grid.zᵃᵃᶠ.η⁻) / static_bottom_heightᶜᶠᵃ(i, j, grid) +@inline e₃⁻(i, j, k, grid::ZSG, ::F, ::F, ℓz) = dynamic_bottom_heightᶠᶠᵃ(i, j, 1, grid, grid.zᵃᵃᶠ.η⁻) / static_bottom_heightᶠᶠᵃ(i, j, grid) + +@inline ∂t_grid(i, j, k, grid) = zero(grid) +@inline ∂t_grid(i, j, k, grid::ZSG) = @inbounds grid.Δzᵃᵃᶜ.∂t_s[i, j] + +@inline ηⁿ(i, j, grid, ℓx, ℓy) = one(grid) +@inline η⁻(i, j, grid, ℓx, ℓy) = one(grid) + +@inline ηⁿ(i, j, grid::ZSG, ::C, ::C) = @inbounds grid.zᵃᵃᶠ.ηⁿ[i, j, 1] +@inline ηⁿ(i, j, grid::ZSG, ::F, ::C) = ℑxᶠᵃᵃ(i, j, 1, grid.zᵃᵃᶠ.η) +@inline ηⁿ(i, j, grid::ZSG, ::C, ::F) = ℑyᵃᶠᵃ(i, j, 1, grid.zᵃᵃᶠ.η) +@inline ηⁿ(i, j, grid::ZSG, ::F, ::F) = ℑxyᶠᶠᵃ(i, j, 1, grid.zᵃᵃᶠ.η) + +@inline η⁻(i, j, grid::ZSG, ::C, ::C) = @inbounds grid.zᵃᵃᶠ.η⁻[i, j, 1] +@inline η⁻(i, j, grid::ZSG, ::F, ::C) = ℑxᶠᵃᵃ(i, j, 1, grid.zᵃᵃᶠ.η) +@inline η⁻(i, j, grid::ZSG, ::C, ::F) = ℑyᵃᶠᵃ(i, j, 1, grid.zᵃᵃᶠ.η) +@inline η⁻(i, j, grid::ZSG, ::F, ::F) = ℑxyᶠᶠᵃ(i, j, 1, grid.zᵃᵃᶠ.η) + + +# rnode for an ZStarUnderlyingGrid grid is scaled +# TODO: fix this when bottom height is implemented +@inline znode(i, j, k, grid::ZSG, ℓx, ℓx, ℓx) = rnode(i, j, k, grid, ℓx, ℓy, ℓz) * e₃ⁿ(i, j, k, grid, c, c, c) + ηⁿ(i, j, grid, ℓx, ℓy) \ No newline at end of file diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index b7fe8f79e2..3a5ef8551f 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -17,7 +17,7 @@ grid = RectilinearGrid(size = (128, 20), # grid = ImmersedBoundaryGrid(grid, GridFittedBottom(x -> - (64kilometers - x) / 64kilometers * 20)) model = HydrostaticFreeSurfaceModel(; grid, - momentum_advection = FluxFormAdvection(WENO(; order = 5), nothing, WENO(; order = 5)), + momentum_advection = WENO(order = 5), tracer_advection = FluxFormAdvection(WENO(; order = 5), nothing, WENO(; order = 5)), buoyancy = BuoyancyTracer(), closure = nothing, @@ -26,7 +26,7 @@ model = HydrostaticFreeSurfaceModel(; grid, g = model.free_surface.gravitational_acceleration -model.timestepper.χ = 0.0 +model.timestepper.χ = 0.1 bᵢ(x, z) = x < 32kilometers ? 0.06 : 0.01 @@ -63,7 +63,7 @@ function progress(sim) return nothing end -simulation.callbacks[:progress] = Callback(progress, IterationInterval(100)) +simulation.callbacks[:progress] = Callback(progress, IterationInterval(1)) run!(simulation) From 6321fbaa10cb052f4e4db3c32220aaef97259e81 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Sat, 23 Nov 2024 07:22:13 -0500 Subject: [PATCH 464/567] much better! --- src/Grids/latitude_longitude_grid.jl | 36 +++---- src/Grids/rectilinear_grid.jl | 40 ++++--- src/Grids/vertical_coordinates.jl | 102 +++++++++++++++--- src/Grids/z_star_vertical_coordinate.jl | 21 ---- .../spacings_and_areas_and_volumes.jl | 4 +- src/Operators/variable_grid_operators.jl | 58 +++++----- 6 files changed, 154 insertions(+), 107 deletions(-) diff --git a/src/Grids/latitude_longitude_grid.jl b/src/Grids/latitude_longitude_grid.jl index 70879acf17..44b9ed3e8f 100644 --- a/src/Grids/latitude_longitude_grid.jl +++ b/src/Grids/latitude_longitude_grid.jl @@ -1,6 +1,6 @@ using KernelAbstractions: @kernel, @index -struct LatitudeLongitudeGrid{FT, TX, TY, TZ, M, MY, FX, FY, FZ, VX, VY, VZ, Arch} <: AbstractHorizontallyCurvilinearGrid{FT, TX, TY, TZ, Arch} +struct LatitudeLongitudeGrid{FT, TX, TY, TZ, FZ, M, MY, FX, FY, VX, VY, Arch} <: AbstractHorizontallyCurvilinearGrid{FT, TX, TY, TZ, Arch} architecture :: Arch Nx :: Int Ny :: Int @@ -20,11 +20,7 @@ struct LatitudeLongitudeGrid{FT, TX, TY, TZ, M, MY, FX, FY, FZ, VX, VY, VZ, Arch Δφᵃᶠᵃ :: FY Δφᵃᶜᵃ :: FY φᵃᶠᵃ :: VY - φᵃᶜᵃ :: VY - Δzᵃᵃᶠ :: FZ - Δzᵃᵃᶜ :: FZ - zᵃᵃᶠ :: VZ - zᵃᵃᶜ :: VZ + z :: FZ # Precomputed metrics M <: Nothing means metrics will be computed on the fly Δxᶠᶜᵃ :: M Δxᶜᶠᵃ :: M @@ -48,25 +44,23 @@ struct LatitudeLongitudeGrid{FT, TX, TY, TZ, M, MY, FX, FY, FZ, VX, VY, VZ, Arch Δφᵃᶠᵃ :: FY, Δφᵃᶜᵃ :: FY, φᵃᶠᵃ :: VY, φᵃᶜᵃ :: VY, Δzᵃᵃᶠ :: FZ, Δzᵃᵃᶜ :: FZ, - zᵃᵃᶠ :: VZ, zᵃᵃᶜ :: VZ, Δxᶠᶜᵃ :: M, Δxᶜᶠᵃ :: M, Δxᶠᶠᵃ :: M, Δxᶜᶜᵃ :: M, Δyᶠᶜᵃ :: MY, Δyᶜᶠᵃ :: MY, - Azᶠᶜᵃ :: M, Azᶜᶠᵃ :: M, - Azᶠᶠᵃ :: M, Azᶜᶜᵃ :: M, - radius :: FT) where {Arch, FT, TX, TY, TZ, - FX, FY, FZ, VX, VY, VZ, + z :: FZ, + radius :: FT) where {Arch, FT, TX, TY, TZ, + FX, FY, FZ, VX, VY, M, MY} = - new{FT, TX, TY, TZ, M, MY, FX, FY, FZ, VX, VY, VZ, Arch}(architecture, - Nλ, Nφ, Nz, - Hλ, Hφ, Hz, - Lλ, Lφ, Lz, - Δλᶠᵃᵃ, Δλᶜᵃᵃ, λᶠᵃᵃ, λᶜᵃᵃ, - Δφᵃᶠᵃ, Δφᵃᶜᵃ, φᵃᶠᵃ, φᵃᶜᵃ, - Δzᵃᵃᶠ, Δzᵃᵃᶜ, zᵃᵃᶠ, zᵃᵃᶜ, - Δxᶠᶜᵃ, Δxᶜᶠᵃ, Δxᶠᶠᵃ, Δxᶜᶜᵃ, - Δyᶠᶜᵃ, Δyᶜᶠᵃ, - Azᶠᶜᵃ, Azᶜᶠᵃ, Azᶠᶠᵃ, Azᶜᶜᵃ, radius) + new{FT, TX, TY, TZ, FZ, M, MY, FX, FY, VX, VY, Arch}(architecture, + Nλ, Nφ, Nz, + Hλ, Hφ, Hz, + Lλ, Lφ, Lz, + Δλᶠᵃᵃ, Δλᶜᵃᵃ, λᶠᵃᵃ, λᶜᵃᵃ, + Δφᵃᶠᵃ, Δφᵃᶜᵃ, φᵃᶠᵃ, φᵃᶜᵃ, + Δzᵃᵃᶠ, Δzᵃᵃᶜ, zᵃᵃᶠ, zᵃᵃᶜ, + Δxᶠᶜᵃ, Δxᶜᶠᵃ, Δxᶠᶠᵃ, Δxᶜᶜᵃ, + Δyᶠᶜᵃ, Δyᶜᶠᵃ, + Azᶠᶜᵃ, Azᶜᶠᵃ, Azᶠᶠᵃ, Azᶜᶜᵃ, radius) end const LLG = LatitudeLongitudeGrid diff --git a/src/Grids/rectilinear_grid.jl b/src/Grids/rectilinear_grid.jl index d6a17c473f..f7e3a30667 100644 --- a/src/Grids/rectilinear_grid.jl +++ b/src/Grids/rectilinear_grid.jl @@ -1,4 +1,4 @@ -struct RectilinearGrid{FT, TX, TY, TZ, FX, FY, FZ, VX, VY, VZ, Arch} <: AbstractUnderlyingGrid{FT, TX, TY, TZ, Arch} +struct RectilinearGrid{FT, TX, TY, TZ, FZ, FX, FY, VX, VY, Arch} <: AbstractUnderlyingGrid{FT, TX, TY, TZ, Arch} architecture :: Arch Nx :: Int Ny :: Int @@ -19,10 +19,7 @@ struct RectilinearGrid{FT, TX, TY, TZ, FX, FY, FZ, VX, VY, VZ, Arch} <: Abstract Δyᵃᶜᵃ :: FY yᵃᶠᵃ :: VY yᵃᶜᵃ :: VY - Δzᵃᵃᶠ :: FZ - Δzᵃᵃᶜ :: FZ - zᵃᵃᶠ :: VZ - zᵃᵃᶜ :: VZ + z :: FZ RectilinearGrid{TX, TY, TZ}(arch::Arch, Nx, Ny, Nz, @@ -32,27 +29,26 @@ struct RectilinearGrid{FT, TX, TY, TZ, FX, FY, FZ, VX, VY, VZ, Arch} <: Abstract xᶠᵃᵃ :: VX, xᶜᵃᵃ :: VX, Δyᵃᶠᵃ :: FY, Δyᵃᶜᵃ :: FY, yᵃᶠᵃ :: VY, yᵃᶜᵃ :: VY, - Δzᵃᵃᶠ :: FZ, Δzᵃᵃᶜ :: FZ, - zᵃᵃᶠ :: VZ, zᵃᵃᶜ :: VZ) where {Arch, FT, - TX, TY, TZ, - FX, VX, FY, - VY, FZ, VZ} = - new{FT, TX, TY, TZ, FX, FY, FZ, VX, VY, VZ, Arch}(arch, Nx, Ny, Nz, - Hx, Hy, Hz, Lx, Ly, Lz, - Δxᶠᵃᵃ, Δxᶜᵃᵃ, xᶠᵃᵃ, xᶜᵃᵃ, - Δyᵃᶠᵃ, Δyᵃᶜᵃ, yᵃᶠᵃ, yᵃᶜᵃ, - Δzᵃᵃᶠ, Δzᵃᵃᶜ, zᵃᵃᶠ, zᵃᵃᶜ) + z :: FZ) where {Arch, FT, + TX, TY, TZ, + FX, VX, FY, + VY, FZ} = + new{FT, TX, TY, TZ, FZ, FX, FY, VX, VY, Arch}(arch, Nx, Ny, Nz, + Hx, Hy, Hz, Lx, Ly, Lz, + Δxᶠᵃᵃ, Δxᶜᵃᵃ, xᶠᵃᵃ, xᶜᵃᵃ, + Δyᵃᶠᵃ, Δyᵃᶜᵃ, yᵃᶠᵃ, yᵃᶜᵃ, + z) end const RG = RectilinearGrid -const XRegularRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Number} -const YRegularRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Number} -const ZRegularRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Number} -const XYRegularRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Number, <:Number} -const XZRegularRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Number, <:Any, <:Number} -const YZRegularRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Number, <:Number} -const XYZRegularRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Number, <:Number, <:Number} +const XRegularRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Number} +const YRegularRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Number} +const ZRegularRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:RegularVerticalCoordinate} +const XYRegularRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Number, <:Number} +const XZRegularRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:RegularVerticalCoordinate, <:Number} +const YZRegularRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:RegularVerticalCoordinate, <:Any, <:Number} +const XYZRegularRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:RegularVerticalCoordinate, <:Number, <:Number} regular_dimensions(::XRegularRG) = tuple(1) regular_dimensions(::YRegularRG) = tuple(2) diff --git a/src/Grids/vertical_coordinates.jl b/src/Grids/vertical_coordinates.jl index 50ef7821d5..0af34180de 100644 --- a/src/Grids/vertical_coordinates.jl +++ b/src/Grids/vertical_coordinates.jl @@ -1,23 +1,90 @@ abstract type AbstractVerticalCoordinate end +# Represents a static vertical coordinate system. +# +# # Fields +# - `cᶠ::C`: Face-centered coordinates. +# - `cᶜ::C`: Cell-centered coordinates. +# - `Δᶠ::D`: Face-centered grid spacing. +# - `Δᶠ::D`: Face-centered grid spacing (duplicate field, consider renaming or removing). +# +# # Type Parameters +# - `C`: Type of the face-centered and cell-centered coordinates. +# - `D`: Type of the face-centered grid spacing. struct StaticVerticalCoordinate{C, D} <: AbstractVerticalCoordinate cᶠ :: C cᶜ :: C - Δᶠ :: D + Δᶜ :: D Δᶠ :: D end -struct ZStarVerticalCoordinate{C, D, E, S} <: AbstractVerticalCoordinate - cᶠ :: C - cᶜ :: C - Δᶠ :: D - Δᶠ :: D - ηⁿ :: E - η⁻ :: E - ∂t_s :: S +# Represents a z-star vertical coordinate system. +# +# # Fields +# - `cᶠ::C`: Face-centered coordinates. +# - `cᶜ::C`: Cell-centered coordinates. +# - `Δᶠ::D`: Face-centered grid spacing. +# - `Δᶠ::D`: Face-centered grid spacing (duplicate field, consider renaming or removing). +# - `ηⁿ::E`: Surface elevation at the current time step. +# - `η⁻::E`: Surface elevation at the previous time step. +# - `e₃ᶜᶜⁿ::CC`: Vertical grid scaling at cell centers at the current time step. +# - `e₃ᶠᶜⁿ::FC`: Vertical grid scaling at face centers at the current time step. +# - `e₃ᶜᶠⁿ::CF`: Vertical grid scaling at cell-face interfaces at the current time step. +# - `e₃ᶠᶠⁿ::FF`: Vertical grid scaling at face-face interfaces at the current time step. +# - `e₃ᶜᶜ⁻::CC`: Vertical grid scaling at cell centers at the previous time step. +# - `∂t_e₃::CC`: Time derivative of the vertical grid scaling at cell centers. +struct ZStarVerticalCoordinate{C, D, E, CC, FC, CF, FF} <: AbstractVerticalCoordinate + cᶠ :: C + cᶜ :: C + Δᶜ :: D + Δᶠ :: D + ηⁿ :: E + η⁻ :: E + e₃ᶜᶜⁿ :: CC + e₃ᶠᶜⁿ :: FC + e₃ᶜᶠⁿ :: CF + e₃ᶠᶠⁿ :: FF + e₃ᶜᶜ⁻ :: CC + ∂t_e₃ :: CC end -const AbstractZStarGrid = AbstractGrid{<:Any, <:Any, <:Any, <:Bounded, <:ZStarVerticalCoordinate} +#### +#### Some usefull aliases +#### + +const RegularStaticVerticalCoordinate = StaticVerticalCoordinate{<:Any, <:Number} +const RegularZstarVerticalCoordinate = ZStarVerticalCoordinate{<:Any, <:Number} + +const RegularVerticalCoordinate = Union{RegularStaticVerticalCoordinate, RegularZstarVerticalCoordinate} + +const AbstractZStarGrid = AbstractGrid{<:Any, <:Any, <:Any, <:Bounded, <:ZStarVerticalCoordinate} +const AbstractStaticGrid = AbstractGrid{<:Any, <:Any, <:Any, <:Bounded, <:StaticVerticalCoordinate} +const RegularVerticalGrid = AbstractGrid{<:Any, <:Any, <:Any, <:Bounded, <:RegularVerticalCoordinate} + +#### +#### Adapting +#### + + +Adapt.adapt_structure(to, coord::StaticVerticalCoordinate) = + StaticVerticalCoordinate(Adapt.adapt(to, coord.cᶠ), + Adapt.adapt(to, coord.cᶜ), + Adapt.adapt(to, coord.Δᶠ), + Adapt.adapt(to, coord.Δᶠ)) + +Adapt.adapt_structure(to, coord::ZStarVerticalCoordinate) = + ZStarVerticalCoordinate(Adapt.adapt(to, coord.cᶠ), + Adapt.adapt(to, coord.cᶜ), + Adapt.adapt(to, coord.Δᶠ), + Adapt.adapt(to, coord.Δᶠ), + Adapt.adapt(to, coord.ηⁿ), + Adapt.adapt(to, coord.η⁻), + Adapt.adapt(to, coord.e₃ᶜᶜⁿ), + Adapt.adapt(to, coord.e₃ᶠᶜⁿ), + Adapt.adapt(to, coord.e₃ᶜᶠⁿ), + Adapt.adapt(to, coord.e₃ᶠᶠⁿ), + Adapt.adapt(to, coord.e₃ᶜᶜ⁻), + Adapt.adapt(to, coord.∂t_e₃)) ##### ##### vertical nodes... @@ -25,20 +92,21 @@ const AbstractZStarGrid = AbstractGrid{<:Any, <:Any, <:Any, <:Bounded, <:ZStarVe @inline rnode(i, j, k, grid, ℓx, ℓy, ℓz) = rnode(k, grid, ℓz) @inline rnode(k, grid, ::Center) = getnode(grid.z.cᶜ, k) -@inline rnode(k, grid, ::Face) = getnode(grid.z.cᶠ, k) +@inline rnode(k, grid, ::Face) = getnode(grid.z.cᶠ, k) -@inline znode(k, grid, ℓz) = rnode(k, grid, ℓz) +@inline znode(k, grid, ℓz) = rnode(k, grid, ℓz) @inline znode(i, j, k, grid, ℓx, ℓy, ℓz) = rnode(k, grid, ℓz) -# TO extend in operators -@inline znode(i, j, k, grid::AbstractZStarGrid, ℓx, ℓy, ℓz) = znode(k, grid, ℓz) - -@inline znode(k, grid, ℓx) = getnode(grid.z, k) +# Extended for ZStargrids in operators +@inline znode(i, j, k, grid, ℓx, ℓy, ℓz) = znode(k, grid, ℓz) +@inline znode(k, grid, ℓx) = getnode(grid.z, k) -# Convenience constructors +# Convenience constructors for Zstar vertical coordinate ZStarVerticalCoordinate(r_faces::Union{Tuple, AbstractVector}) = ZStarVerticalCoordinate(r_faces, r_faces, nothing, nothing, nothing, nothing, nothing) ZStarVerticalCoordinate(r⁻::Number, r⁺::Number) = ZStarVerticalCoordinate((r⁻, r⁺), (r⁻, r⁺), nothing, nothing, nothing, nothing, nothing) +# Summaries + Grids.coordinate_summary(::Bounded, Δ::ZStarVerticalCoordinate, name) = @sprintf("Free-surface following with Δ%s=%s", name, prettysummary(Δ.reference)) diff --git a/src/Grids/z_star_vertical_coordinate.jl b/src/Grids/z_star_vertical_coordinate.jl index 748400f349..31291efb0d 100644 --- a/src/Grids/z_star_vertical_coordinate.jl +++ b/src/Grids/z_star_vertical_coordinate.jl @@ -81,27 +81,6 @@ end ZStarVerticalCoordinate(r_faces::Union{Tuple, AbstractVector}) = ZStarVerticalCoordinate(r_faces, nothing, nothing, nothing, nothing, nothing, nothing, nothing, nothing) ZStarVerticalCoordinate(r⁻::Number, r⁺::Number) = ZStarVerticalCoordinate((r⁻, r⁺), nothing, nothing, nothing, nothing, nothing, nothing, nothing, nothing) -Adapt.adapt_structure(to, coord::ZStarVerticalCoordinate) = - ZStarVerticalCoordinate(Adapt.adapt(to, coord.reference), - Adapt.adapt(to, coord.sᶜᶜⁿ), - Adapt.adapt(to, coord.sᶠᶜⁿ), - Adapt.adapt(to, coord.sᶜᶠⁿ), - Adapt.adapt(to, coord.sᶠᶠⁿ), - Adapt.adapt(to, coord.sᶜᶜ⁻), - Adapt.adapt(to, coord.sᶠᶜ⁻), - Adapt.adapt(to, coord.sᶜᶠ⁻), - Adapt.adapt(to, coord.∂t_s)) - -on_architecture(arch, coord::ZStarVerticalCoordinate) = - ZStarVerticalCoordinate(on_architecture(arch, coord.reference), - on_architecture(arch, coord.sᶜᶜⁿ), - on_architecture(arch, coord.sᶠᶜⁿ), - on_architecture(arch, coord.sᶜᶠⁿ), - on_architecture(arch, coord.sᶠᶠⁿ), - on_architecture(arch, coord.sᶜᶜ⁻), - on_architecture(arch, coord.sᶠᶜ⁻), - on_architecture(arch, coord.sᶜᶠ⁻), - on_architecture(arch, coord.∂t_s)) Grids.coordinate_summary(::Bounded, Δ::ZStarVerticalCoordinate, name) = @sprintf("Free-surface following with Δ%s=%s", name, prettysummary(Δ.reference)) diff --git a/src/Operators/spacings_and_areas_and_volumes.jl b/src/Operators/spacings_and_areas_and_volumes.jl index 948c7710f9..bfd551a1ec 100644 --- a/src/Operators/spacings_and_areas_and_volumes.jl +++ b/src/Operators/spacings_and_areas_and_volumes.jl @@ -56,8 +56,8 @@ const ZRG = Union{LLGZ, RGZ, OSSGZ} @inline getspacing(k, Δz::AbstractVector) = @inbounds Δz[k] @inline getspacing(k, Δz::Number) = @inbounds Δz -@inline Δrᵃᵃᶜ(i, j, k, grid::ZStarUnderlyingGrid) = getspacing(k, grid.Δzᵃᵃᶜ.reference) -@inline Δrᵃᵃᶠ(i, j, k, grid::ZStarUnderlyingGrid) = getspacing(k, grid.Δzᵃᵃᶠ.reference) +@inline Δrᵃᵃᶜ(i, j, k, grid::ZStarUnderlyingGrid) = getspacing(k, grid.z.Δᶜ) +@inline Δrᵃᵃᶠ(i, j, k, grid::ZStarUnderlyingGrid) = getspacing(k, grid.z.Δᶠ) # Convenience Functions for all grids for LX in (:ᶜ, :ᶠ, :ᵃ), LY in (:ᶜ, :ᶠ, :ᵃ) diff --git a/src/Operators/variable_grid_operators.jl b/src/Operators/variable_grid_operators.jl index 53d289c418..932aa76bd1 100644 --- a/src/Operators/variable_grid_operators.jl +++ b/src/Operators/variable_grid_operators.jl @@ -8,7 +8,7 @@ import Oceananigans.Grids: znode const C = Center const F = Face -const ZSG = ZStarUnderlyingGrid +const ZSG = AbstractZStarGrid @inline dynamic_column_depthᶜᶜᵃ(i, j, grid, η) = static_column_depthᶜᶜᵃ(i, j, grid) @inline dynamic_column_depthᶜᶠᵃ(i, j, grid, η) = static_column_depthᶜᶠᵃ(i, j, grid) @@ -19,33 +19,43 @@ const ZSG = ZStarUnderlyingGrid @inline e₃ⁿ(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) @inline e₃⁻(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) -@inline e₃ⁿ(i, j, k, grid::ZSG, ::C, ::C, ℓz) = dynamic_bottom_heightᶜᶜᵃ(i, j, 1, grid, grid.zᵃᵃᶠ.ηⁿ) / static_bottom_heightᶜᶜᵃ(i, j, grid) -@inline e₃ⁿ(i, j, k, grid::ZSG, ::F, ::C, ℓz) = dynamic_bottom_heightᶠᶜᵃ(i, j, 1, grid, grid.zᵃᵃᶠ.ηⁿ) / static_bottom_heightᶠᶜᵃ(i, j, grid) -@inline e₃ⁿ(i, j, k, grid::ZSG, ::C, ::F, ℓz) = dynamic_bottom_heightᶜᶠᵃ(i, j, 1, grid, grid.zᵃᵃᶠ.ηⁿ) / static_bottom_heightᶜᶠᵃ(i, j, grid) -@inline e₃ⁿ(i, j, k, grid::ZSG, ::F, ::F, ℓz) = dynamic_bottom_heightᶠᶠᵃ(i, j, 1, grid, grid.zᵃᵃᶠ.ηⁿ) / static_bottom_heightᶠᶠᵃ(i, j, grid) +@inline e₃ⁿ(i, j, k, grid::ZSG, ::C, ::C, ℓz) = @inbounds grid.z.eᶜᶜⁿ[i, j, 1] +@inline e₃ⁿ(i, j, k, grid::ZSG, ::F, ::C, ℓz) = @inbounds grid.z.eᶠᶜⁿ[i, j, 1] +@inline e₃ⁿ(i, j, k, grid::ZSG, ::C, ::F, ℓz) = @inbounds grid.z.eᶜᶠⁿ[i, j, 1] +@inline e₃ⁿ(i, j, k, grid::ZSG, ::F, ::F, ℓz) = @inbounds grid.z.eᶠᶠⁿ[i, j, 1] -@inline e₃⁻(i, j, k, grid::ZSG, ::C, ::C, ℓz) = dynamic_bottom_heightᶜᶜᵃ(i, j, 1, grid, grid.zᵃᵃᶠ.η⁻) / static_bottom_heightᶜᶜᵃ(i, j, grid) -@inline e₃⁻(i, j, k, grid::ZSG, ::F, ::C, ℓz) = dynamic_bottom_heightᶠᶜᵃ(i, j, 1, grid, grid.zᵃᵃᶠ.η⁻) / static_bottom_heightᶠᶜᵃ(i, j, grid) -@inline e₃⁻(i, j, k, grid::ZSG, ::C, ::F, ℓz) = dynamic_bottom_heightᶜᶠᵃ(i, j, 1, grid, grid.zᵃᵃᶠ.η⁻) / static_bottom_heightᶜᶠᵃ(i, j, grid) -@inline e₃⁻(i, j, k, grid::ZSG, ::F, ::F, ℓz) = dynamic_bottom_heightᶠᶠᵃ(i, j, 1, grid, grid.zᵃᵃᶠ.η⁻) / static_bottom_heightᶠᶠᵃ(i, j, grid) +# e₃⁻ is needed only at centers +@inline e₃⁻(i, j, k, grid::ZSG, ::C, ::C, ℓz) = @inbounds grid.z.eᶜᶜ⁻[i, j, 1] -@inline ∂t_grid(i, j, k, grid) = zero(grid) -@inline ∂t_grid(i, j, k, grid::ZSG) = @inbounds grid.Δzᵃᵃᶜ.∂t_s[i, j] +@inline ∂t_e₃(i, j, k, grid) = zero(grid) +@inline ∂t_e₃(i, j, k, grid::ZSG) = @inbounds grid.z.∂t_e₃[i, j, 1] -@inline ηⁿ(i, j, grid, ℓx, ℓy) = one(grid) -@inline η⁻(i, j, grid, ℓx, ℓy) = one(grid) +# rnode for an ZStarUnderlyingGrid grid is scaled +# TODO: fix this when bottom height is implemented +@inline znode(i, j, k, grid::ZSG, ℓx, ℓx, ℓx) = @inbounds rnode(i, j, k, grid, ℓx, ℓy, ℓz) * e₃ⁿ(i, j, k, grid, ℓx, ℓy, ℓz) + grid.z.ηⁿ[i, j, 1] -@inline ηⁿ(i, j, grid::ZSG, ::C, ::C) = @inbounds grid.zᵃᵃᶠ.ηⁿ[i, j, 1] -@inline ηⁿ(i, j, grid::ZSG, ::F, ::C) = ℑxᶠᵃᵃ(i, j, 1, grid.zᵃᵃᶠ.η) -@inline ηⁿ(i, j, grid::ZSG, ::C, ::F) = ℑyᵃᶠᵃ(i, j, 1, grid.zᵃᵃᶠ.η) -@inline ηⁿ(i, j, grid::ZSG, ::F, ::F) = ℑxyᶠᶠᵃ(i, j, 1, grid.zᵃᵃᶠ.η) +#### +#### Vertical spacing functions +#### -@inline η⁻(i, j, grid::ZSG, ::C, ::C) = @inbounds grid.zᵃᵃᶠ.η⁻[i, j, 1] -@inline η⁻(i, j, grid::ZSG, ::F, ::C) = ℑxᶠᵃᵃ(i, j, 1, grid.zᵃᵃᶠ.η) -@inline η⁻(i, j, grid::ZSG, ::C, ::F) = ℑyᵃᶠᵃ(i, j, 1, grid.zᵃᵃᶠ.η) -@inline η⁻(i, j, grid::ZSG, ::F, ::F) = ℑxyᶠᶠᵃ(i, j, 1, grid.zᵃᵃᶠ.η) +const ZSRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:ZStarVerticalCoordinate} +const ZSLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:ZStarVerticalCoordinate} +const ZSOSG = OrthogonalSphericalShellGrid{<:Any, <:Any, <:Any, <:Any, <:ZStarVerticalCoordinate} +location(s::Symbol) = s == :ᶜ ? C() : F() + +for Lx in (:ᶠ, :ᶜ), Lx in (:ᶠ, :ᶜ), Lx in (:ᶠ, :ᶜ) + zspacing = Symbol(:Δz, Lx, Ly, Lz) + rspacing = Symbol(:Δr, Lx, Ly, Lz) + + ℓx = location(Lx) + ℓy = location(Ly) + ℓz = location(Lz) + + @eval begin + @inline $zspacing(i, j, k, grid::ZSRG) = $rspacing(i, j, k, grid) * e₃ⁿ(i, j, k, grid, ℓx, ℓy, ℓz) + @inline $zspacing(i, j, k, grid::ZSLLG) = $rspacing(i, j, k, grid) * e₃ⁿ(i, j, k, grid, ℓx, ℓy, ℓz) + @inline $zspacing(i, j, k, grid::ZSOSG) = $rspacing(i, j, k, grid) * e₃ⁿ(i, j, k, grid, ℓx, ℓy, ℓz) + end +end -# rnode for an ZStarUnderlyingGrid grid is scaled -# TODO: fix this when bottom height is implemented -@inline znode(i, j, k, grid::ZSG, ℓx, ℓx, ℓx) = rnode(i, j, k, grid, ℓx, ℓy, ℓz) * e₃ⁿ(i, j, k, grid, c, c, c) + ηⁿ(i, j, grid, ℓx, ℓy) \ No newline at end of file From 9cbf5d4586ec38b61d7ec6cc17b1bdc1ff1e951d Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Sat, 23 Nov 2024 07:35:43 -0500 Subject: [PATCH 465/567] this should work nicely --- src/Grids/Grids.jl | 1 + src/Grids/grid_utils.jl | 7 +- src/Grids/vertical_coordinates.jl | 3 - .../generalized_vertical_spacing.jl | 10 +-- .../z_star_vertical_spacing.jl | 73 +++++++------------ src/Operators/Operators.jl | 3 + src/Operators/variable_grid_operators.jl | 25 +++++-- 7 files changed, 60 insertions(+), 62 deletions(-) diff --git a/src/Grids/Grids.jl b/src/Grids/Grids.jl index 04bea6727c..af23b4d8c7 100644 --- a/src/Grids/Grids.jl +++ b/src/Grids/Grids.jl @@ -21,6 +21,7 @@ export xspacings, yspacings, zspacings, λspacings, φspacings, rspacings export minimum_xspacing, minimum_yspacing, minimum_zspacing export ZStarVerticalCoordinate, vertical_scaling, previous_vertical_scaling, reference_zspacings export static_column_depthᶜᶜᵃ, static_column_depthᶠᶜᵃ, static_column_depthᶜᶠᵃ, static_column_depthᶠᶠᵃ +export dynamic_column_depthᶜᶜᵃ, dynamic_column_depthᶠᶜᵃ, dynamic_column_depthᶜᶠᵃ, dynamic_column_depthᶠᶠᵃ export offset_data, new_data export on_architecture diff --git a/src/Grids/grid_utils.jl b/src/Grids/grid_utils.jl index 4b52d73a71..ebe0b4f050 100644 --- a/src/Grids/grid_utils.jl +++ b/src/Grids/grid_utils.jl @@ -321,7 +321,7 @@ coordinate_summary(topo, Δ::Union{AbstractVector, AbstractMatrix}, name) = name, prettysummary(maximum(parent(Δ)))) ##### -##### Static column depth +##### Static and Dynamic column depths ##### @inline static_column_depthᶜᶜᵃ(i, j, grid) = grid.Lz @@ -329,6 +329,11 @@ coordinate_summary(topo, Δ::Union{AbstractVector, AbstractMatrix}, name) = @inline static_column_depthᶠᶜᵃ(i, j, grid) = grid.Lz @inline static_column_depthᶠᶠᵃ(i, j, grid) = grid.Lz +@inline dynamic_column_depthᶜᶜᵃ(i, j, grid, η) = static_column_depthᶜᶜᵃ(i, j, grid) +@inline dynamic_column_depthᶜᶠᵃ(i, j, grid, η) = static_column_depthᶜᶠᵃ(i, j, grid) +@inline dynamic_column_depthᶠᶜᵃ(i, j, grid, η) = static_column_depthᶠᶜᵃ(i, j, grid) +@inline dynamic_column_depthᶠᶠᵃ(i, j, grid, η) = static_column_depthᶠᶠᵃ(i, j, grid) + ##### ##### Spherical geometry ##### diff --git a/src/Grids/vertical_coordinates.jl b/src/Grids/vertical_coordinates.jl index 0af34180de..63bbf1fd25 100644 --- a/src/Grids/vertical_coordinates.jl +++ b/src/Grids/vertical_coordinates.jl @@ -39,7 +39,6 @@ struct ZStarVerticalCoordinate{C, D, E, CC, FC, CF, FF} <: AbstractVerticalCoord Δᶜ :: D Δᶠ :: D ηⁿ :: E - η⁻ :: E e₃ᶜᶜⁿ :: CC e₃ᶠᶜⁿ :: FC e₃ᶜᶠⁿ :: CF @@ -65,7 +64,6 @@ const RegularVerticalGrid = AbstractGrid{<:Any, <:Any, <:Any, <:Bounded, <:Regul #### Adapting #### - Adapt.adapt_structure(to, coord::StaticVerticalCoordinate) = StaticVerticalCoordinate(Adapt.adapt(to, coord.cᶠ), Adapt.adapt(to, coord.cᶜ), @@ -78,7 +76,6 @@ Adapt.adapt_structure(to, coord::ZStarVerticalCoordinate) = Adapt.adapt(to, coord.Δᶠ), Adapt.adapt(to, coord.Δᶠ), Adapt.adapt(to, coord.ηⁿ), - Adapt.adapt(to, coord.η⁻), Adapt.adapt(to, coord.e₃ᶜᶜⁿ), Adapt.adapt(to, coord.e₃ᶠᶜⁿ), Adapt.adapt(to, coord.e₃ᶜᶠⁿ), diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index e45947be89..093214c27f 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -37,14 +37,14 @@ update_grid!(model, grid; parameters = :xy) = nothing C₁ = convert(FT, 1.5) + χ C₂ = convert(FT, 0.5) + χ - sⁿ = vertical_scaling(i, j, k, grid, Center(), Center(), Center()) - s⁻ = previous_vertical_scaling(i, j, k, grid, Center(), Center(), Center()) + eⁿ = e₃ⁿ(i, j, k, grid, Center(), Center(), Center()) + e⁻ = e₃⁻(i, j, k, grid, Center(), Center(), Center()) @inbounds begin - ∂t_sθ = C₁ * sⁿ * Gⁿ[i, j, k] - C₂ * s⁻ * G⁻[i, j, k] + ∂t_sθ = C₁ * eⁿ * Gⁿ[i, j, k] - C₂ * e⁻ * G⁻[i, j, k] # We store temporarily sθ in θ. the unscaled θ will be retrived later on with `unscale_tracers!` - θ[i, j, k] = sⁿ * θ[i, j, k] + convert(FT, Δt) * ∂t_sθ + θ[i, j, k] = eⁿ * θ[i, j, k] + convert(FT, Δt) * ∂t_sθ end end @@ -76,6 +76,6 @@ end i, j, n = @index(Global, NTuple) @unroll for k in -Hz+1:Nz+Hz - tracers[n][i, j, k] /= vertical_scaling(i, j, k, grid, Center(), Center(), Center()) + tracers[n][i, j, k] /= e₃ⁿ(i, j, k, grid, Center(), Center(), Center()) end end \ No newline at end of file diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 8d14dada59..2c6bbdd301 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -12,15 +12,13 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels.SplitExplicitFreeSurfaces function update_grid!(model, grid::ZStarSpacingGrid; parameters = :xy) # Scaling (just update once, they are the same for all the metrics) - sᶜᶜ⁻ = grid.Δzᵃᵃᶠ.sᶜᶜ⁻ - sᶜᶜⁿ = grid.Δzᵃᵃᶠ.sᶜᶜⁿ - sᶠᶜ⁻ = grid.Δzᵃᵃᶠ.sᶠᶜ⁻ - sᶠᶜⁿ = grid.Δzᵃᵃᶠ.sᶠᶜⁿ - sᶜᶠ⁻ = grid.Δzᵃᵃᶠ.sᶜᶠ⁻ - sᶜᶠⁿ = grid.Δzᵃᵃᶠ.sᶜᶠⁿ - sᶠᶠⁿ = grid.Δzᵃᵃᶠ.sᶠᶠⁿ - ∂t_s = grid.Δzᵃᵃᶠ.∂t_s - η_grid = grid.zᵃᵃᶠ.∂t_s + e₃ᶜᶜ⁻ = grid.z.e₃ᶜᶜ⁻ + e₃ᶜᶜⁿ = grid.z.e₃ᶜᶜⁿ + e₃ᶠᶜⁿ = grid.z.e₃ᶠᶜⁿ + e₃ᶜᶠⁿ = grid.z.e₃ᶜᶠⁿ + e₃ᶠᶠⁿ = grid.z.e₃ᶠᶠⁿ + ∂t_e₃ = grid.z.∂t_e₃ + ηⁿ = grid.z.ηⁿ # Free surface variables: # TODO: At the moment only SplitExplicitFreeSurface is supported, @@ -33,37 +31,16 @@ function update_grid!(model, grid::ZStarSpacingGrid; parameters = :xy) # Update vertical spacing with available parameters # No need to fill the halo as the scaling is updated _IN_ the halos launch!(architecture(grid), grid, parameters, _update_zstar!, - sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, sᶠᶜ⁻, sᶜᶠ⁻, η_grid, η, grid) - - # Update the time derivative of the grid-scaling. Note that in this case we leverage the - # free surface evolution equation, where the time derivative of the free surface is equal - # to the divergence of the vertically integrated velocity field, such that - # ∂ₜ((H + η) / H) = H⁻¹ ∂ₜη = - H⁻¹ ∇ ⋅ ∫udz - launch!(architecture(grid), grid, parameters, _update_∂t_s!, - ∂t_s, U, V, grid) + e₃ᶜᶜⁿ, e₃ᶠᶜⁿ, e₃ᶜᶠⁿ, e₃ᶠᶠⁿ, e₃ᶜᶜ⁻, ηⁿ, ∂t_e₃, grid, η, U, V) return nothing end # NOTE: The ZStar vertical spacing only supports a SplitExplicitFreeSurface # TODO: extend to support other free surface solvers -@kernel function _update_∂t_s!(∂t_s, U, V, grid) - i, j = @index(Global, NTuple) - kᴺ = size(grid, 3) - - # ∂(η / H)/∂t = - ∇ ⋅ ∫udz / H - δx_U = δxᶜᶜᶜ(i, j, kᴺ, grid, Δy_qᶠᶜᶜ, U) - δy_V = δyᶜᶜᶜ(i, j, kᴺ, grid, Δx_qᶜᶠᶜ, V) - - δh_U = (δx_U + δy_V) / Azᶜᶜᶜ(i, j, kᴺ, grid) - H = static_column_depthᶜᶜᵃ(i, j, grid) - - @inbounds ∂t_s[i, j] = ifelse(H == 0, zero(grid), - δh_U / H) -end - -@kernel function _update_zstar!(sᶜᶜⁿ, sᶠᶜⁿ, sᶜᶠⁿ, sᶠᶠⁿ, sᶜᶜ⁻, sᶠᶜ⁻, sᶜᶠ⁻, η_grid, η, grid) +@kernel function _update_zstar!(e₃ᶜᶜⁿ, e₃ᶠᶜⁿ, e₃ᶜᶠⁿ, e₃ᶠᶠⁿ, e₃ᶜᶜ⁻, ηⁿ, ∂t_e₃, grid, η, U, V) i, j = @index(Global, NTuple) - k_top = grid.Nz+1 + kᴺ = size(grid, 3) hᶜᶜ = static_column_depthᶜᶜᵃ(i, j, grid) hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) @@ -76,24 +53,30 @@ end Hᶠᶠ = dynamic_column_depthᶠᶠᵃ(i, j, grid, η) @inbounds begin - sᶜᶜ = ifelse(hᶜᶜ == 0, one(grid), Hᶜᶜ / hᶜᶜ) - sᶠᶜ = ifelse(hᶠᶜ == 0, one(grid), Hᶠᶜ / hᶠᶜ) - sᶜᶠ = ifelse(hᶜᶠ == 0, one(grid), Hᶜᶠ / hᶜᶠ) - sᶠᶠ = ifelse(hᶠᶠ == 0, one(grid), Hᶠᶠ / hᶠᶠ) + e₃ᶜᶜ = ifelse(hᶜᶜ == 0, one(grid), Hᶜᶜ / hᶜᶜ) + e₃ᶠᶜ = ifelse(hᶠᶜ == 0, one(grid), Hᶠᶜ / hᶠᶜ) + e₃ᶜᶠ = ifelse(hᶜᶠ == 0, one(grid), Hᶜᶠ / hᶜᶠ) + e₃ᶠᶠ = ifelse(hᶠᶠ == 0, one(grid), Hᶠᶠ / hᶠᶠ) # Update previous scaling - sᶜᶜ⁻[i, j] = sᶜᶜⁿ[i, j] - sᶠᶜ⁻[i, j] = sᶠᶜⁿ[i, j] - sᶜᶠ⁻[i, j] = sᶜᶠⁿ[i, j] + e₃ᶜᶜ⁻[i, j, 1] = e₃ᶜᶜⁿ[i, j, 1] # update current scaling - sᶜᶜⁿ[i, j] = sᶜᶜ - sᶠᶜⁿ[i, j] = sᶠᶜ - sᶜᶠⁿ[i, j] = sᶜᶠ - sᶠᶠⁿ[i, j] = sᶠᶠ + e₃ᶜᶜⁿ[i, j, 1] = e₃ᶜᶜ + e₃ᶠᶜⁿ[i, j, 1] = e₃ᶠᶜ + e₃ᶜᶠⁿ[i, j, 1] = e₃ᶜᶠ + e₃ᶠᶠⁿ[i, j, 1] = e₃ᶠᶠ # Update η in the grid - η_grid[i, j] = η[i, j, k_top] + ηⁿ[i, j, 1] = η[i, j, kᴺ+1] + + # ∂(η / H)/∂t = - ∇ ⋅ ∫udz / H + δx_U = δxᶜᶜᶜ(i, j, kᴺ, grid, Δy_qᶠᶜᶜ, U) + δy_V = δyᶜᶜᶜ(i, j, kᴺ, grid, Δx_qᶜᶠᶜ, V) + + δh_U = (δx_U + δy_V) / Azᶜᶜᶜ(i, j, kᴺ, grid) + + ∂t_e₃[i, j, 1] = ifelse(hᶜᶜ == 0, zero(grid), - δh_U / hᶜᶜ) end end diff --git a/src/Operators/Operators.jl b/src/Operators/Operators.jl index a57a1d1a4b..7b982317d3 100644 --- a/src/Operators/Operators.jl +++ b/src/Operators/Operators.jl @@ -74,6 +74,9 @@ export ∂xTᶠᶜᶠ, ∂yTᶜᶠᶠ # Reference frame conversion export intrinsic_vector, extrinsic_vector +# Variable grid operators +export e₃ⁿ, e₃⁻, ∂t_e₃ + using Oceananigans.Grids import Oceananigans.Grids: xspacing, yspacing, zspacing diff --git a/src/Operators/variable_grid_operators.jl b/src/Operators/variable_grid_operators.jl index 932aa76bd1..cef6080ebb 100644 --- a/src/Operators/variable_grid_operators.jl +++ b/src/Operators/variable_grid_operators.jl @@ -1,6 +1,11 @@ using Oceananigans.Grids: ZStarUnderlyingGrid import Oceananigans.Grids: znode +import Oceananigans.Grids: dynamic_column_depthᶜᶜᵃ, + dynamic_column_depthᶜᶠᵃ, + dynamic_column_depthᶠᶜᵃ, + dynamic_column_depthᶠᶠᵃ + ##### ##### ZStar-specific vertical spacing functions ##### @@ -10,10 +15,16 @@ const F = Face const ZSG = AbstractZStarGrid -@inline dynamic_column_depthᶜᶜᵃ(i, j, grid, η) = static_column_depthᶜᶜᵃ(i, j, grid) -@inline dynamic_column_depthᶜᶠᵃ(i, j, grid, η) = static_column_depthᶜᶠᵃ(i, j, grid) -@inline dynamic_column_depthᶠᶜᵃ(i, j, grid, η) = static_column_depthᶠᶜᵃ(i, j, grid) -@inline dynamic_column_depthᶠᶠᵃ(i, j, grid, η) = static_column_depthᶠᶠᵃ(i, j, grid) +@inline dynamic_column_depthᶜᶜᵃ(i, j, grid::ZSG, η) = @inbounds static_column_depthᶜᶜᵃ(i, j, grid) + η[i, j, grid.Nz+1] +@inline dynamic_column_depthᶜᶠᵃ(i, j, grid::ZSG, η) = @inbounds static_column_depthᶜᶠᵃ(i, j, grid) + ℑxᶠᵃᵃ(i, j, grid.Nz+1, η) +@inline dynamic_column_depthᶠᶜᵃ(i, j, grid::ZSG, η) = @inbounds static_column_depthᶠᶜᵃ(i, j, grid) + ℑyᵃᶠᵃ(i, j, grid.Nz+1, η) +@inline dynamic_column_depthᶠᶠᵃ(i, j, grid::ZSG, η) = @inbounds static_column_depthᶠᶠᵃ(i, j, grid) + ℑxyᶠᶠᵃ(i, j, grid.Nz+1, η) + +# Convenience +@inline dynamic_column_depthᶜᶜᵃ(i, j, grid::ZSG) = dynamic_column_depthᶜᶜᵃ(i, j, grid, grid.z.ηⁿ) +@inline dynamic_column_depthᶜᶠᵃ(i, j, grid::ZSG) = dynamic_column_depthᶜᶠᵃ(i, j, grid, grid.z.ηⁿ) +@inline dynamic_column_depthᶠᶜᵃ(i, j, grid::ZSG) = dynamic_column_depthᶠᶜᵃ(i, j, grid, grid.z.ηⁿ) +@inline dynamic_column_depthᶠᶠᵃ(i, j, grid::ZSG) = dynamic_column_depthᶠᶠᵃ(i, j, grid, grid.z.ηⁿ) # Fallbacks @inline e₃ⁿ(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) @@ -30,10 +41,6 @@ const ZSG = AbstractZStarGrid @inline ∂t_e₃(i, j, k, grid) = zero(grid) @inline ∂t_e₃(i, j, k, grid::ZSG) = @inbounds grid.z.∂t_e₃[i, j, 1] -# rnode for an ZStarUnderlyingGrid grid is scaled -# TODO: fix this when bottom height is implemented -@inline znode(i, j, k, grid::ZSG, ℓx, ℓx, ℓx) = @inbounds rnode(i, j, k, grid, ℓx, ℓy, ℓz) * e₃ⁿ(i, j, k, grid, ℓx, ℓy, ℓz) + grid.z.ηⁿ[i, j, 1] - #### #### Vertical spacing functions #### @@ -59,3 +66,5 @@ for Lx in (:ᶠ, :ᶜ), Lx in (:ᶠ, :ᶜ), Lx in (:ᶠ, :ᶜ) end end +# rnode for an ZStarUnderlyingGrid grid is scaled +@inline znode(i, j, k, grid::ZSG, ℓx, ℓy, ℓz) = @inbounds rnode(i, j, k, grid, ℓx, ℓy, ℓz) * e₃ⁿ(i, j, k, grid, ℓx, ℓy, ℓz) + grid.z.ηⁿ[i, j, 1] From 96f8d663d03f8c50e6aed98919600ad263c5009c Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Sat, 23 Nov 2024 07:38:59 -0500 Subject: [PATCH 466/567] more working --- src/Grids/abstract_grid.jl | 11 +++++++---- src/Grids/orthogonal_spherical_shell_grid.jl | 15 ++++++--------- src/Grids/rectilinear_grid.jl | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/Grids/abstract_grid.jl b/src/Grids/abstract_grid.jl index 9285b3c8c2..0c9eaa9f66 100644 --- a/src/Grids/abstract_grid.jl +++ b/src/Grids/abstract_grid.jl @@ -1,7 +1,8 @@ """ AbstractGrid{FT, TX, TY, TZ} -Abstract supertype for grids with elements of type `FT` and topology `{TX, TY, TZ}`. +Abstract supertype for grids with elements of type `FT`, +topology `{TX, TY, TZ}` and vertical coordinate `CZ`. """ abstract type AbstractGrid{FT, TX, TY, TZ, CZ, Arch} end @@ -9,21 +10,23 @@ abstract type AbstractGrid{FT, TX, TY, TZ, CZ, Arch} end AbstractUnderlyingGrid{FT, TX, TY, TZ} Abstract supertype for "primary" grids (as opposed to grids with immersed boundaries) -with elements of type `FT` and topology `{TX, TY, TZ}`. +with elements of type `FT`, topology `{TX, TY, TZ}` and vertical coordinate `CZ`. """ abstract type AbstractUnderlyingGrid{FT, TX, TY, TZ, CZ, Arch} <: AbstractGrid{FT, TX, TY, TZ, CZ, Arch} end """ AbstractCurvilinearGrid{FT, TX, TY, TZ} -Abstract supertype for curvilinear grids with elements of type `FT` and topology `{TX, TY, TZ}`. +Abstract supertype for curvilinear grids with elements of type `FT`, +topology `{TX, TY, TZ}`, and vertical coordinate `CZ`. """ abstract type AbstractCurvilinearGrid{FT, TX, TY, TZ, CZ, Arch} <: AbstractUnderlyingGrid{FT, TX, TY, TZ, CZ, Arch} end """ AbstractHorizontallyCurvilinearGrid{FT, TX, TY, TZ} -Abstract supertype for horizontally-curvilinear grids with elements of type `FT` and topology `{TX, TY, TZ}`. +Abstract supertype for horizontally-curvilinear grids with elements of type `FT`, +topology `{TX, TY, TZ}` and vertical coordinate `CZ`. """ abstract type AbstractHorizontallyCurvilinearGrid{FT, TX, TY, TZ, CZ, Arch} <: AbstractCurvilinearGrid{FT, TX, TY, TZ, CZ, Arch} end diff --git a/src/Grids/orthogonal_spherical_shell_grid.jl b/src/Grids/orthogonal_spherical_shell_grid.jl index d6aecbb2e7..fcfa93ac67 100644 --- a/src/Grids/orthogonal_spherical_shell_grid.jl +++ b/src/Grids/orthogonal_spherical_shell_grid.jl @@ -15,7 +15,7 @@ struct CubedSphereConformalMapping{FT, Rotation} rotation :: Rotation end -struct OrthogonalSphericalShellGrid{FT, TX, TY, TZ, A, R, FR, C, Arch} <: AbstractHorizontallyCurvilinearGrid{FT, TX, TY, TZ, Arch} +struct OrthogonalSphericalShellGrid{FT, TX, TY, TZ, CZ, A, C, Arch} <: AbstractHorizontallyCurvilinearGrid{FT, TX, TY, TZ, CZ, Arch} architecture :: Arch Nx :: Int Ny :: Int @@ -32,8 +32,7 @@ struct OrthogonalSphericalShellGrid{FT, TX, TY, TZ, A, R, FR, C, Arch} <: Abstra φᶠᶜᵃ :: A φᶜᶠᵃ :: A φᶠᶠᵃ :: A - zᵃᵃᶜ :: R - zᵃᵃᶠ :: R + z :: CZ Δxᶜᶜᵃ :: A Δxᶠᶜᵃ :: A Δxᶜᶠᵃ :: A @@ -42,8 +41,6 @@ struct OrthogonalSphericalShellGrid{FT, TX, TY, TZ, A, R, FR, C, Arch} <: Abstra Δyᶜᶠᵃ :: A Δyᶠᶜᵃ :: A Δyᶠᶠᵃ :: A - Δzᵃᵃᶜ :: FR - Δzᵃᵃᶠ :: FR Azᶜᶜᵃ :: A Azᶠᶜᵃ :: A Azᶜᶠᵃ :: A @@ -56,13 +53,13 @@ struct OrthogonalSphericalShellGrid{FT, TX, TY, TZ, A, R, FR, C, Arch} <: Abstra Hx, Hy, Hz, Lz :: FT, λᶜᶜᵃ :: A, λᶠᶜᵃ :: A, λᶜᶠᵃ :: A, λᶠᶠᵃ :: A, - φᶜᶜᵃ :: A, φᶠᶜᵃ :: A, φᶜᶠᵃ :: A, φᶠᶠᵃ :: A, zᵃᵃᶜ :: R, zᵃᵃᶠ :: R, + φᶜᶜᵃ :: A, φᶠᶜᵃ :: A, φᶜᶠᵃ :: A, φᶠᶠᵃ :: A, z :: CZ, Δxᶜᶜᵃ :: A, Δxᶠᶜᵃ :: A, Δxᶜᶠᵃ :: A, Δxᶠᶠᵃ :: A, - Δyᶜᶜᵃ :: A, Δyᶜᶠᵃ :: A, Δyᶠᶜᵃ :: A, Δyᶠᶠᵃ :: A, Δzᵃᵃᶜ :: FR, Δzᵃᵃᶠ :: FR, + Δyᶜᶜᵃ :: A, Δyᶜᶠᵃ :: A, Δyᶠᶜᵃ :: A, Δyᶠᶠᵃ :: A, Azᶜᶜᵃ :: A, Azᶠᶜᵃ :: A, Azᶜᶠᵃ :: A, Azᶠᶠᵃ :: A, radius :: FT, - conformal_mapping :: C) where {TX, TY, TZ, FT, A, R, FR, C, Arch} = - new{FT, TX, TY, TZ, A, R, FR, C, Arch}(architecture, + conformal_mapping :: C) where {TX, TY, TZ, FT, CZ, A, C, Arch} = + new{FT, TX, TY, TZ, CZ, A, C, Arch}(architecture, Nx, Ny, Nz, Hx, Hy, Hz, Lz, diff --git a/src/Grids/rectilinear_grid.jl b/src/Grids/rectilinear_grid.jl index f7e3a30667..d416ed76a3 100644 --- a/src/Grids/rectilinear_grid.jl +++ b/src/Grids/rectilinear_grid.jl @@ -1,4 +1,4 @@ -struct RectilinearGrid{FT, TX, TY, TZ, FZ, FX, FY, VX, VY, Arch} <: AbstractUnderlyingGrid{FT, TX, TY, TZ, Arch} +struct RectilinearGrid{FT, TX, TY, TZ, CZ, FX, FY, VX, VY, Arch} <: AbstractUnderlyingGrid{FT, TX, TY, TZ, CZ, Arch} architecture :: Arch Nx :: Int Ny :: Int From 1bfe7d736a098409be4219b4babf4725c96f9240 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Sat, 23 Nov 2024 07:48:20 -0500 Subject: [PATCH 467/567] fx grid generation --- src/Grids/grid_generation.jl | 14 +++++++++----- src/Grids/vertical_coordinates.jl | 18 +++++++----------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/Grids/grid_generation.jl b/src/Grids/grid_generation.jl index 8e3eda6a0d..178ad44403 100644 --- a/src/Grids/grid_generation.jl +++ b/src/Grids/grid_generation.jl @@ -180,9 +180,13 @@ function generate_coordinate(FT, topo, size, halo, coordinate::ZStarVerticalCoor args = (topo, (Nx, Ny, Nz), (Hx, Hy, Hz)) - sᶜᶜᵃ = new_data(FT, arch, (Center, Center, Nothing), args...) - sᶜᶜᵃ₋ = new_data(FT, arch, (Center, Center, Nothing), args...) - ∂t_s = new_data(FT, arch, (Center, Center, Nothing), args...) - - return Lr, ZStarVerticalCoordinate(rᵃᵃᶠ, rᵃᵃᶜ, Δrᵃᵃᶠ, Δrᵃᵃᶜ, sᶜᶜᵃ, sᶜᶜᵃ₋, ∂t_s) + e₃ᶜᶜ⁻ = new_data(FT, arch, (Center, Center, Nothing), args...) + e₃ᶜᶜⁿ = new_data(FT, arch, (Center, Center, Nothing), args...) + e₃ᶠᶜⁿ = new_data(FT, arch, (Face, Center, Nothing), args...) + e₃ᶜᶠⁿ = new_data(FT, arch, (Center, Face, Nothing), args...) + e₃ᶠᶠⁿ = new_data(FT, arch, (Face, Face, Nothing), args...) + ηⁿ = new_data(FT, arch, (Center, Center, Nothing), args...) + ∂t_e₃ = new_data(FT, arch, (Center, Center, Nothing), args...) + + return Lr, ZStarVerticalCoordinate(rᵃᵃᶠ, rᵃᵃᶜ, Δrᵃᵃᶠ, Δrᵃᵃᶜ, ηⁿ, e₃ᶜᶜⁿ, e₃ᶠᶜⁿ, e₃ᶜᶠⁿ, e₃ᶠᶠⁿ, e₃ᶜᶜ⁻, ∂t_e₃) end diff --git a/src/Grids/vertical_coordinates.jl b/src/Grids/vertical_coordinates.jl index 63bbf1fd25..16de56e5b6 100644 --- a/src/Grids/vertical_coordinates.jl +++ b/src/Grids/vertical_coordinates.jl @@ -47,6 +47,10 @@ struct ZStarVerticalCoordinate{C, D, E, CC, FC, CF, FF} <: AbstractVerticalCoord ∂t_e₃ :: CC end +# Convenience constructors for Zstar vertical coordinate +ZStarVerticalCoordinate(r_faces::Union{Tuple, AbstractVector}) = ZStarVerticalCoordinate(r_faces, r_faces, nothing, nothing, nothing, nothing, nothing) +ZStarVerticalCoordinate(r⁻::Number, r⁺::Number) = ZStarVerticalCoordinate((r⁻, r⁺), (r⁻, r⁺), nothing, nothing, nothing, nothing, nothing) + #### #### Some usefull aliases #### @@ -94,21 +98,13 @@ Adapt.adapt_structure(to, coord::ZStarVerticalCoordinate) = @inline znode(k, grid, ℓz) = rnode(k, grid, ℓz) @inline znode(i, j, k, grid, ℓx, ℓy, ℓz) = rnode(k, grid, ℓz) -# Extended for ZStargrids in operators +# Extended for ZStarGrids in the Operators module @inline znode(i, j, k, grid, ℓx, ℓy, ℓz) = znode(k, grid, ℓz) @inline znode(k, grid, ℓx) = getnode(grid.z, k) -# Convenience constructors for Zstar vertical coordinate -ZStarVerticalCoordinate(r_faces::Union{Tuple, AbstractVector}) = ZStarVerticalCoordinate(r_faces, r_faces, nothing, nothing, nothing, nothing, nothing) -ZStarVerticalCoordinate(r⁻::Number, r⁺::Number) = ZStarVerticalCoordinate((r⁻, r⁺), (r⁻, r⁺), nothing, nothing, nothing, nothing, nothing) - # Summaries - -Grids.coordinate_summary(::Bounded, Δ::ZStarVerticalCoordinate, name) = - @sprintf("Free-surface following with Δ%s=%s", name, prettysummary(Δ.reference)) - -Grids.coordinate_summary(::Bounded, Δ::ZStarVerticalCoordinate, name) = - @sprintf("Free-surface following with Δ%s=%s", name, prettysummary(Δ.reference)) +Grids.coordinate_summary(::Bounded, z::ZStarVerticalCoordinate, name) = + @sprintf("Free-surface following with Δ%s=%s", name, prettysummary(z.Δᶜ)) function validate_dimension_specification(T, ξ::ZStarVerticalCoordinate, dir, N, FT) reference = validate_dimension_specification(T, ξ.reference, dir, N, FT) From a8b122895befcfc76f752a7625ac6b49389b2c8c Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Sat, 23 Nov 2024 07:48:33 -0500 Subject: [PATCH 468/567] delete zstar --- src/Grids/z_star_vertical_coordinate.jl | 186 ------------------------ 1 file changed, 186 deletions(-) delete mode 100644 src/Grids/z_star_vertical_coordinate.jl diff --git a/src/Grids/z_star_vertical_coordinate.jl b/src/Grids/z_star_vertical_coordinate.jl deleted file mode 100644 index 31291efb0d..0000000000 --- a/src/Grids/z_star_vertical_coordinate.jl +++ /dev/null @@ -1,186 +0,0 @@ -##### -##### ZStar coordinate and associated types -##### - -abstract type AbstractVerticalCoordinate end - -##### -##### AbstractVerticalCoordinate grid definitions -##### - -const AVLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractVerticalCoordinate} -const AVOSSG = OrthogonalSphericalShellGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractVerticalCoordinate} -const AVRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractVerticalCoordinate} - -const AbstractVerticalCoordinateUnderlyingGrid = Union{AVLLG, AVOSSG, AVRG} - -function retrieve_static_grid(grid::AbstractVerticalCoordinateUnderlyingGrid) - - zᵃᵃᶠ = rnodes(grid, Face(); with_halos = true) - zᵃᵃᶜ = rnodes(grid, Center(); with_halos = true) - - Δzᵃᵃᶠ = rspacings(grid, Face(); with_halos = true) - Δzᵃᵃᶜ = rspacings(grid, Center(); with_halos = true) - - TX, TY, TZ = topology(grid) - - args = [] - for prop in propertynames(grid) - if prop == :zᵃᵃᶠ - push!(args, zᵃᵃᶠ) - elseif prop == :zᵃᵃᶜ - push!(args, zᵃᵃᶜ) - elseif prop == :Δzᵃᵃᶠ - push!(args, Δzᵃᵃᶠ) - elseif prop == :Δzᵃᵃᶜ - push!(args, Δzᵃᵃᶜ) - else - push!(args, getproperty(grid, prop)) - end - end - - return construct_grid(grid, TX, TY, TZ, args...) -end - -construct_grid(::RectilinearGrid, TX, TY, TZ, args...) = RectilinearGrid{TX, TY, TZ}(args...) -construct_grid(::LatitudeLongitudeGrid, TX, TY, TZ, args...) = LatitudeLongitudeGrid{TX, TY, TZ}(args...) -construct_grid(::OrthogonalSphericalShellGrid, TX, TY, TZ, args...) = OrthogonalSphericalShellGrid{TX, TY, TZ}(args...) - -""" - struct ZStarVerticalCoordinate{R, S} <: AbstractVerticalSpacing{R} - -A vertical coordinate for the hydrostatic free surface model that follows the free surface. -The vertical spacing is defined by a reference spacing `Δr` and a scaling `s` that obeys -```math -s = (η + H) / H -``` -where ``η`` is the free surface height and ``H`` the vertical depth of the water column - -# Fields -- `Δr`: reference vertical spacing with `η = 0` -- `sᶜᶜⁿ`: scaling of the vertical coordinate at time step `n` at `(Center, Center, Any)` location -- `sᶠᶜⁿ`: scaling of the vertical coordinate at time step `n` at `(Face, Center, Any)` location -- `sᶜᶠⁿ`: scaling of the vertical coordinate at time step `n` at `(Center, Face, Any)` location -- `sᶠᶠⁿ`: scaling of the vertical coordinate at time step `n` at `(Face, Face, Any)` location -- `s⁻`: scaling of the vertical coordinate at time step `n - 1` at `(Center, Center, Any)` location -- `∂t_s`: Time derivative of `s` -""" -struct ZStarVerticalCoordinate{R, SCC, SFC, SCF, SFF} <: AbstractVerticalCoordinate - reference :: R - sᶜᶜⁿ :: SCC - sᶠᶜⁿ :: SFC - sᶜᶠⁿ :: SCF - sᶠᶠⁿ :: SFF - sᶜᶜ⁻ :: SCC - sᶠᶜ⁻ :: SFC - sᶜᶠ⁻ :: SCF - ∂t_s :: SCC -end - -# Convenience constructors -ZStarVerticalCoordinate(r_faces::Union{Tuple, AbstractVector}) = ZStarVerticalCoordinate(r_faces, nothing, nothing, nothing, nothing, nothing, nothing, nothing, nothing) -ZStarVerticalCoordinate(r⁻::Number, r⁺::Number) = ZStarVerticalCoordinate((r⁻, r⁺), nothing, nothing, nothing, nothing, nothing, nothing, nothing, nothing) - - -Grids.coordinate_summary(::Bounded, Δ::ZStarVerticalCoordinate, name) = - @sprintf("Free-surface following with Δ%s=%s", name, prettysummary(Δ.reference)) - -generate_coordinate(FT, ::Periodic, N, H, ::ZStarVerticalCoordinate, coordinate_name, arch, args...) = - throw(ArgumentError("Periodic domains are not supported for ZStarVerticalCoordinate")) - -# Generate a moving coordinate with evolving scaling (`s`) for spacings and znodes -function generate_coordinate(FT, topo, size, halo, coordinate::ZStarVerticalCoordinate, coordinate_name, dim::Int, arch) - - Nx, Ny, Nz = size - Hx, Hy, Hz = halo - - if dim != 3 - msg = "ZStarVerticalCoordinate is supported only in the third dimension (z)" - throw(ArgumentError(msg)) - end - - if coordinate_name != :z - msg = "Only z-coordinate is supported for ZStarVerticalCoordinate" - throw(ArgumentError(msg)) - end - - r_faces = coordinate.reference - - Lr, rᵃᵃᶠ, rᵃᵃᶜ, Δrᵃᵃᶠ, Δrᵃᵃᶜ = generate_coordinate(FT, topo[3](), Nz, Hz, r_faces, :z, arch) - - args = (topo, (Nx, Ny, Nz), (Hx, Hy, Hz)) - - sᶜᶜᵃ = new_data(FT, arch, (Center, Center, Nothing), args...) - sᶜᶠᵃ = new_data(FT, arch, (Center, Face, Nothing), args...) - sᶠᶜᵃ = new_data(FT, arch, (Face, Center, Nothing), args...) - sᶠᶠᵃ = new_data(FT, arch, (Face, Face, Nothing), args...) - - sᶜᶜᵃ₋ = new_data(FT, arch, (Center, Center, Nothing), args...) - sᶜᶠᵃ₋ = new_data(FT, arch, (Center, Face, Nothing), args...) - sᶠᶜᵃ₋ = new_data(FT, arch, (Face, Center, Nothing), args...) - - ∂t_s = new_data(FT, arch, (Center, Center, Nothing), args...) - # Storage place for the free surface height? - # TODO: Probably find a better way to call this or to store this - η = new_data(FT, arch, (Center, Center, Nothing), args...) - - # fill all the scalings with 1 - for s in (sᶜᶜᵃ, sᶜᶠᵃ, sᶠᶜᵃ, sᶠᶠᵃ, sᶜᶜᵃ₋, sᶜᶠᵃ₋, sᶠᶜᵃ₋) - fill!(s, 1) - end - - # The scaling is the same for everyone, the vertical coordinate requires - # to add the free surface to retrieve the znode. - zᵃᵃᶠ = ZStarVerticalCoordinate(rᵃᵃᶠ, ηᶜᶜᵃ, sᶠᶜᵃ, sᶜᶠᵃ, sᶠᶠᵃ, sᶜᶜᵃ₋, sᶠᶜᵃ₋, sᶜᶠᵃ₋, η) - zᵃᵃᶜ = ZStarVerticalCoordinate(rᵃᵃᶜ, ηᶜᶜᵃ, sᶠᶜᵃ, sᶜᶠᵃ, sᶠᶠᵃ, sᶜᶜᵃ₋, sᶠᶜᵃ₋, sᶜᶠᵃ₋, η) - - Δzᵃᵃᶠ = ZStarVerticalCoordinate(Δrᵃᵃᶠ, sᶜᶜᵃ, sᶠᶜᵃ, sᶜᶠᵃ, sᶠᶠᵃ, sᶜᶜᵃ₋, sᶠᶜᵃ₋, sᶜᶠᵃ₋, ∂t_s) - Δzᵃᵃᶜ = ZStarVerticalCoordinate(Δrᵃᵃᶜ, sᶜᶜᵃ, sᶠᶜᵃ, sᶜᶠᵃ, sᶠᶠᵃ, sᶜᶜᵃ₋, sᶠᶜᵃ₋, sᶜᶠᵃ₋, ∂t_s) - - return Lr, zᵃᵃᶠ, zᵃᵃᶜ, Δzᵃᵃᶠ, Δzᵃᵃᶜ -end - -function validate_dimension_specification(T, ξ::ZStarVerticalCoordinate, dir, N, FT) - reference = validate_dimension_specification(T, ξ.reference, dir, N, FT) - args = Tuple(getproperty(ξ, prop) for prop in propertynames(ξ)) - - return ZStarVerticalCoordinate(reference, args[2:end]...) -end - -# TODO: is this the correct definition? -@inline domain(topo, N, ξ::ZStarVerticalCoordinate) = domain(topo, N, ξ.reference) - -const ZStarLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:ZStarVerticalCoordinate} -const ZStarOSSG = OrthogonalSphericalShellGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:ZStarVerticalCoordinate} -const ZStarRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:ZStarVerticalCoordinate} - -const ZStarUnderlyingGrid = Union{ZStarLLG, ZStarOSSG, ZStarRG} - -##### -##### ZStar-specific vertical spacing functions -##### - -const C = Center -const F = Face - -const ZSG = ZStarUnderlyingGrid - -@inline rnodes(grid::ZSG, ℓz::C; with_halos=false) = _property(grid.zᵃᵃᶜ.reference, ℓz, topology(grid, 3), size(grid, 3), with_halos) -@inline rnodes(grid::ZSG, ℓz::F; with_halos=false) = _property(grid.zᵃᵃᶠ.reference, ℓz, topology(grid, 3), size(grid, 3), with_halos) - -@inline rspacings(grid::ZSG, ℓz::C; with_halos=false) = _property(grid.Δzᵃᵃᶜ.reference, ℓz, topology(grid, 3), size(grid, 3), with_halos) -@inline rspacings(grid::ZSG, ℓz::F; with_halos=false) = _property(grid.Δzᵃᵃᶠ.reference, ℓz, topology(grid, 3), size(grid, 3), with_halos) - -@inline ∂t_grid(i, j, k, grid) = zero(grid) -@inline ∂t_grid(i, j, k, grid::ZSG) = @inbounds grid.Δzᵃᵃᶜ.∂t_s[i, j] - -##### -##### znode -##### - -const c = Center() -const f = Face() - -# rnode for an ZStarUnderlyingGrid is the reference node -@inline rnode(i, j, k, grid::ZSG, ℓx, ℓy, ::Center) = @inbounds grid.zᵃᵃᶜ.reference[k] -@inline rnode(i, j, k, grid::ZSG, ℓx, ℓy, ::Face) = @inbounds grid.zᵃᵃᶠ.reference[k] From d42e313484b1c25bf3365b49dac7537141e27d07 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Sat, 23 Nov 2024 10:17:03 -0500 Subject: [PATCH 469/567] should work like this --- src/Grids/abstract_grid.jl | 2 +- src/Grids/vertical_coordinates.jl | 6 ++--- src/ImmersedBoundaries/ImmersedBoundaries.jl | 2 +- .../abstract_zstar_immersed_grid.jl | 19 ------------- .../immersed_grid_metrics.jl | 1 - src/ImmersedBoundaries/zstar_immersed_grid.jl | 27 +++++++++++++++++++ src/Operators/variable_grid_operators.jl | 11 -------- 7 files changed, 32 insertions(+), 36 deletions(-) delete mode 100644 src/ImmersedBoundaries/abstract_zstar_immersed_grid.jl create mode 100644 src/ImmersedBoundaries/zstar_immersed_grid.jl diff --git a/src/Grids/abstract_grid.jl b/src/Grids/abstract_grid.jl index 0c9eaa9f66..38ce6ac9db 100644 --- a/src/Grids/abstract_grid.jl +++ b/src/Grids/abstract_grid.jl @@ -12,7 +12,7 @@ abstract type AbstractGrid{FT, TX, TY, TZ, CZ, Arch} end Abstract supertype for "primary" grids (as opposed to grids with immersed boundaries) with elements of type `FT`, topology `{TX, TY, TZ}` and vertical coordinate `CZ`. """ -abstract type AbstractUnderlyingGrid{FT, TX, TY, TZ, CZ, Arch} <: AbstractGrid{FT, TX, TY, TZ, CZ, Arch} end +abstract type AbstractUnderlyingGrid{FT, TX, TY, TZ, CZ, Arch} <: AbstractGrid{FT, TX, TY, TZ, Arch} end """ AbstractCurvilinearGrid{FT, TX, TY, TZ} diff --git a/src/Grids/vertical_coordinates.jl b/src/Grids/vertical_coordinates.jl index 16de56e5b6..0dc16daffe 100644 --- a/src/Grids/vertical_coordinates.jl +++ b/src/Grids/vertical_coordinates.jl @@ -60,9 +60,9 @@ const RegularZstarVerticalCoordinate = ZStarVerticalCoordinate{<:Any, <:Number const RegularVerticalCoordinate = Union{RegularStaticVerticalCoordinate, RegularZstarVerticalCoordinate} -const AbstractZStarGrid = AbstractGrid{<:Any, <:Any, <:Any, <:Bounded, <:ZStarVerticalCoordinate} -const AbstractStaticGrid = AbstractGrid{<:Any, <:Any, <:Any, <:Bounded, <:StaticVerticalCoordinate} -const RegularVerticalGrid = AbstractGrid{<:Any, <:Any, <:Any, <:Bounded, <:RegularVerticalCoordinate} +const AbstractZStarGrid = AbstractUnderlyingGrid{<:Any, <:Any, <:Any, <:Bounded, <:ZStarVerticalCoordinate} +const AbstractStaticGrid = AbstractUnderlyingGrid{<:Any, <:Any, <:Any, <:Any, <:StaticVerticalCoordinate} +const RegularVerticalGrid = AbstractUnderlyingGrid{<:Any, <:Any, <:Any, <:Any, <:RegularVerticalCoordinate} #### #### Adapting diff --git a/src/ImmersedBoundaries/ImmersedBoundaries.jl b/src/ImmersedBoundaries/ImmersedBoundaries.jl index 50d8eb7d58..f0908bfacb 100644 --- a/src/ImmersedBoundaries/ImmersedBoundaries.jl +++ b/src/ImmersedBoundaries/ImmersedBoundaries.jl @@ -46,6 +46,6 @@ include("mask_immersed_field.jl") include("immersed_reductions.jl") # Extension to Immersed boundaries with zstar -include("abstract_zstar_immersed_grid.jl") +include("zstar_immersed_grid.jl") end # module diff --git a/src/ImmersedBoundaries/abstract_zstar_immersed_grid.jl b/src/ImmersedBoundaries/abstract_zstar_immersed_grid.jl deleted file mode 100644 index f6d2f58c97..0000000000 --- a/src/ImmersedBoundaries/abstract_zstar_immersed_grid.jl +++ /dev/null @@ -1,19 +0,0 @@ -using Oceananigans.Grids: AbstractVerticalCoordinateUnderlyingGrid, ZStarUnderlyingGrid - -import Oceananigans.Grids: retrieve_static_grid -import Oceananigans.Grids: vertical_scaling, previous_vertical_scaling, ∂t_grid - -const ImmersedAbstractVerticalCoordinateGrid = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:AbstractVerticalCoordinateUnderlyingGrid} -const ImmersedZStarGrid = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:ZStarUnderlyingGrid} - -@inline vertical_scaling(i, j, k, grid::ImmersedBoundaryGrid, ℓx, ℓy, ℓz) = vertical_scaling(i, j, k, grid.underlying_grid, ℓx, ℓy, ℓz) -@inline previous_vertical_scaling(i, j, k, grid::ImmersedBoundaryGrid, ℓx, ℓy, ℓz) = previous_vertical_scaling(i, j, k, grid.underlying_grid, ℓx, ℓy, ℓz) -@inline ∂t_grid(i, j, k, grid::ImmersedBoundaryGrid) = ∂t_grid(i, j, k, grid.underlying_grid) - -function retrieve_static_grid(ib::ImmersedAbstractVerticalCoordinateGrid) - immersed_boundary = ib.immersed_boundary - active_cells_map = !isnothing(ib.interior_active_cells) - underlying_grid = retrieve_static_grid(ib.underlying_grid) - - return ImmersedBoundaryGrid(underlying_grid, immersed_boundary; active_cells_map) -end diff --git a/src/ImmersedBoundaries/immersed_grid_metrics.jl b/src/ImmersedBoundaries/immersed_grid_metrics.jl index 93a0089e80..a5d8248115 100644 --- a/src/ImmersedBoundaries/immersed_grid_metrics.jl +++ b/src/ImmersedBoundaries/immersed_grid_metrics.jl @@ -13,7 +13,6 @@ import Oceananigans.Operators: intrinsic_vector, extrinsic_vector for LX in (:ᶜ, :ᶠ), LY in (:ᶜ, :ᶠ), LZ in (:ᶜ, :ᶠ) for dir in (:x, :y, :z), operator in (:Δ, :A) - metric = Symbol(operator, dir, LX, LY, LZ) @eval begin import Oceananigans.Operators: $metric diff --git a/src/ImmersedBoundaries/zstar_immersed_grid.jl b/src/ImmersedBoundaries/zstar_immersed_grid.jl new file mode 100644 index 0000000000..9b00c217da --- /dev/null +++ b/src/ImmersedBoundaries/zstar_immersed_grid.jl @@ -0,0 +1,27 @@ +using Oceananigans.Grids: ZStarGrid +using Oceananigans.Operators + +import Oceananigans.Grids: dynamic_column_depthᶜᶜᵃ, + dynamic_column_depthᶜᶠᵃ, + dynamic_column_depthᶠᶜᵃ, + dynamic_column_depthᶠᶠᵃ + +const ZStarImmersedGrid = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:ZStarGrid} +const ZStarGridOfSomeKind = Union{ZStarImmersedGrid, ZStarGrid} + +@inline dynamic_column_depthᶜᶜᵃ(i, j, grid::ZStarGridOfSomeKind, η) = @inbounds static_column_depthᶜᶜᵃ(i, j, grid) + η[i, j, grid.Nz+1] +@inline dynamic_column_depthᶜᶠᵃ(i, j, grid::ZStarGridOfSomeKind, η) = @inbounds static_column_depthᶜᶠᵃ(i, j, grid) + ℑxᶠᵃᵃ(i, j, grid.Nz+1, grid, η) +@inline dynamic_column_depthᶠᶜᵃ(i, j, grid::ZStarGridOfSomeKind, η) = @inbounds static_column_depthᶠᶜᵃ(i, j, grid) + ℑyᵃᶠᵃ(i, j, grid.Nz+1, grid, η) +@inline dynamic_column_depthᶠᶠᵃ(i, j, grid::ZStarGridOfSomeKind, η) = @inbounds static_column_depthᶠᶠᵃ(i, j, grid) + ℑxyᶠᶠᵃ(i, j, grid.Nz+1, grid, η) + +# Convenience +@inline dynamic_column_depthᶜᶜᵃ(i, j, grid::ZStarGridOfSomeKind) = dynamic_column_depthᶜᶜᵃ(i, j, grid, grid.z.ηⁿ) +@inline dynamic_column_depthᶜᶠᵃ(i, j, grid::ZStarGridOfSomeKind) = dynamic_column_depthᶜᶠᵃ(i, j, grid, grid.z.ηⁿ) +@inline dynamic_column_depthᶠᶜᵃ(i, j, grid::ZStarGridOfSomeKind) = dynamic_column_depthᶠᶜᵃ(i, j, grid, grid.z.ηⁿ) +@inline dynamic_column_depthᶠᶠᵃ(i, j, grid::ZStarGridOfSomeKind) = dynamic_column_depthᶠᶠᵃ(i, j, grid, grid.z.ηⁿ) + +# Fallbacks +@inline e₃ⁿ(i, j, k, grid::IBG, ℓx, ℓy, ℓz) = e₃ⁿ(i, j, k, ibg.underlying_grid, ℓx, ℓy, ℓz) +@inline e₃⁻(i, j, k, grid::IBG, ℓx, ℓy, ℓz) = e₃⁻(i, j, k, ibg.underlying_grid, ℓx, ℓy, ℓz) + +@inline ∂t_e₃(i, j, k, grid::IBG) = ∂t_e₃(i, j, k, grid.underlying_grid) diff --git a/src/Operators/variable_grid_operators.jl b/src/Operators/variable_grid_operators.jl index cef6080ebb..763ac8cd0e 100644 --- a/src/Operators/variable_grid_operators.jl +++ b/src/Operators/variable_grid_operators.jl @@ -15,17 +15,6 @@ const F = Face const ZSG = AbstractZStarGrid -@inline dynamic_column_depthᶜᶜᵃ(i, j, grid::ZSG, η) = @inbounds static_column_depthᶜᶜᵃ(i, j, grid) + η[i, j, grid.Nz+1] -@inline dynamic_column_depthᶜᶠᵃ(i, j, grid::ZSG, η) = @inbounds static_column_depthᶜᶠᵃ(i, j, grid) + ℑxᶠᵃᵃ(i, j, grid.Nz+1, η) -@inline dynamic_column_depthᶠᶜᵃ(i, j, grid::ZSG, η) = @inbounds static_column_depthᶠᶜᵃ(i, j, grid) + ℑyᵃᶠᵃ(i, j, grid.Nz+1, η) -@inline dynamic_column_depthᶠᶠᵃ(i, j, grid::ZSG, η) = @inbounds static_column_depthᶠᶠᵃ(i, j, grid) + ℑxyᶠᶠᵃ(i, j, grid.Nz+1, η) - -# Convenience -@inline dynamic_column_depthᶜᶜᵃ(i, j, grid::ZSG) = dynamic_column_depthᶜᶜᵃ(i, j, grid, grid.z.ηⁿ) -@inline dynamic_column_depthᶜᶠᵃ(i, j, grid::ZSG) = dynamic_column_depthᶜᶠᵃ(i, j, grid, grid.z.ηⁿ) -@inline dynamic_column_depthᶠᶜᵃ(i, j, grid::ZSG) = dynamic_column_depthᶠᶜᵃ(i, j, grid, grid.z.ηⁿ) -@inline dynamic_column_depthᶠᶠᵃ(i, j, grid::ZSG) = dynamic_column_depthᶠᶠᵃ(i, j, grid, grid.z.ηⁿ) - # Fallbacks @inline e₃ⁿ(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) @inline e₃⁻(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) From 50cd210a22aaab38708f482fea22cca306bd51a0 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Sat, 23 Nov 2024 10:20:46 -0500 Subject: [PATCH 470/567] going on --- Project.toml | 2 +- src/Grids/Grids.jl | 2 +- src/Grids/abstract_grid.jl | 2 +- .../{vertical_coordinates.jl => vertical_coordinate.jl} | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) rename src/Grids/{vertical_coordinates.jl => vertical_coordinate.jl} (97%) diff --git a/Project.toml b/Project.toml index cec2f85661..093b3e7a42 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "Oceananigans" uuid = "9e8cae18-63c1-5223-a75c-80ca9d6e9a09" authors = ["Climate Modeling Alliance and contributors"] -version = "0.94.3" +version = "0.95.0" [deps] Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" diff --git a/src/Grids/Grids.jl b/src/Grids/Grids.jl index af23b4d8c7..6acc611481 100644 --- a/src/Grids/Grids.jl +++ b/src/Grids/Grids.jl @@ -120,6 +120,7 @@ struct ZDirection <: AbstractDirection end struct NegativeZDirection <: AbstractDirection end include("abstract_grid.jl") +include("vertical_coordinate.jl") include("grid_utils.jl") include("nodes_and_spacings.jl") include("zeros_and_ones.jl") @@ -131,6 +132,5 @@ include("grid_generation.jl") include("rectilinear_grid.jl") include("orthogonal_spherical_shell_grid.jl") include("latitude_longitude_grid.jl") -include("z_star_vertical_coordinate.jl") end # module diff --git a/src/Grids/abstract_grid.jl b/src/Grids/abstract_grid.jl index 38ce6ac9db..aa67be1621 100644 --- a/src/Grids/abstract_grid.jl +++ b/src/Grids/abstract_grid.jl @@ -4,7 +4,7 @@ Abstract supertype for grids with elements of type `FT`, topology `{TX, TY, TZ}` and vertical coordinate `CZ`. """ -abstract type AbstractGrid{FT, TX, TY, TZ, CZ, Arch} end +abstract type AbstractGrid{FT, TX, TY, TZ, Arch} end """ AbstractUnderlyingGrid{FT, TX, TY, TZ} diff --git a/src/Grids/vertical_coordinates.jl b/src/Grids/vertical_coordinate.jl similarity index 97% rename from src/Grids/vertical_coordinates.jl rename to src/Grids/vertical_coordinate.jl index 0dc16daffe..db0e75d3cb 100644 --- a/src/Grids/vertical_coordinates.jl +++ b/src/Grids/vertical_coordinate.jl @@ -3,8 +3,8 @@ abstract type AbstractVerticalCoordinate end # Represents a static vertical coordinate system. # # # Fields -# - `cᶠ::C`: Face-centered coordinates. -# - `cᶜ::C`: Cell-centered coordinates. +# - `cᶠ::C`: Face-centered coordinate. +# - `cᶜ::C`: Cell-centered coordinate. # - `Δᶠ::D`: Face-centered grid spacing. # - `Δᶠ::D`: Face-centered grid spacing (duplicate field, consider renaming or removing). # @@ -88,7 +88,7 @@ Adapt.adapt_structure(to, coord::ZStarVerticalCoordinate) = Adapt.adapt(to, coord.∂t_e₃)) ##### -##### vertical nodes... +##### Vertical nodes... ##### @inline rnode(i, j, k, grid, ℓx, ℓy, ℓz) = rnode(k, grid, ℓz) From 1fd22c86fa598f7eedba789278c0c2a382648eb2 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Sat, 23 Nov 2024 12:39:36 -0500 Subject: [PATCH 471/567] some simplification --- src/Grids/Grids.jl | 1 + src/Grids/grid_utils.jl | 2 - src/Grids/latitude_longitude_grid.jl | 13 ++-- src/Grids/rectilinear_grid.jl | 8 +-- src/Grids/vertical_coordinate.jl | 21 +++--- .../HydrostaticFreeSurfaceModels.jl | 4 +- src/Models/Models.jl | 6 -- .../spacings_and_areas_and_volumes.jl | 65 ++----------------- src/Operators/variable_grid_operators.jl | 10 +-- 9 files changed, 33 insertions(+), 97 deletions(-) diff --git a/src/Grids/Grids.jl b/src/Grids/Grids.jl index 6acc611481..decf316586 100644 --- a/src/Grids/Grids.jl +++ b/src/Grids/Grids.jl @@ -29,6 +29,7 @@ using CUDA using CUDA: has_cuda using Adapt using OffsetArrays +using Printf using Oceananigans using Oceananigans.Architectures diff --git a/src/Grids/grid_utils.jl b/src/Grids/grid_utils.jl index ebe0b4f050..d629d5f873 100644 --- a/src/Grids/grid_utils.jl +++ b/src/Grids/grid_utils.jl @@ -1,5 +1,3 @@ -using CUDA -using Printf using Base.Ryu: writeshortest using LinearAlgebra: dot, cross using OffsetArrays: IdOffsetRange diff --git a/src/Grids/latitude_longitude_grid.jl b/src/Grids/latitude_longitude_grid.jl index 44b9ed3e8f..7b36989ec1 100644 --- a/src/Grids/latitude_longitude_grid.jl +++ b/src/Grids/latitude_longitude_grid.jl @@ -1,6 +1,6 @@ using KernelAbstractions: @kernel, @index -struct LatitudeLongitudeGrid{FT, TX, TY, TZ, FZ, M, MY, FX, FY, VX, VY, Arch} <: AbstractHorizontallyCurvilinearGrid{FT, TX, TY, TZ, Arch} +struct LatitudeLongitudeGrid{FT, TX, TY, TZ, CZ, M, MY, FX, FY, VX, VY, Arch} <: AbstractHorizontallyCurvilinearGrid{FT, TX, TY, TZ, CZ, Arch} architecture :: Arch Nx :: Int Ny :: Int @@ -20,7 +20,7 @@ struct LatitudeLongitudeGrid{FT, TX, TY, TZ, FZ, M, MY, FX, FY, VX, VY, Arch} <: Δφᵃᶠᵃ :: FY Δφᵃᶜᵃ :: FY φᵃᶠᵃ :: VY - z :: FZ + z :: CZ # Precomputed metrics M <: Nothing means metrics will be computed on the fly Δxᶠᶜᵃ :: M Δxᶜᶠᵃ :: M @@ -43,21 +43,20 @@ struct LatitudeLongitudeGrid{FT, TX, TY, TZ, FZ, M, MY, FX, FY, VX, VY, Arch} <: λᶠᵃᵃ :: VX, λᶜᵃᵃ :: VX, Δφᵃᶠᵃ :: FY, Δφᵃᶜᵃ :: FY, φᵃᶠᵃ :: VY, φᵃᶜᵃ :: VY, - Δzᵃᵃᶠ :: FZ, Δzᵃᵃᶜ :: FZ, + z :: CZ, Δxᶠᶜᵃ :: M, Δxᶜᶠᵃ :: M, Δxᶠᶠᵃ :: M, Δxᶜᶜᵃ :: M, Δyᶠᶜᵃ :: MY, Δyᶜᶠᵃ :: MY, - z :: FZ, radius :: FT) where {Arch, FT, TX, TY, TZ, - FX, FY, FZ, VX, VY, + FX, FY, CZ, VX, VY, M, MY} = - new{FT, TX, TY, TZ, FZ, M, MY, FX, FY, VX, VY, Arch}(architecture, + new{FT, TX, TY, TZ, CZ, M, MY, FX, FY, VX, VY, Arch}(architecture, Nλ, Nφ, Nz, Hλ, Hφ, Hz, Lλ, Lφ, Lz, Δλᶠᵃᵃ, Δλᶜᵃᵃ, λᶠᵃᵃ, λᶜᵃᵃ, Δφᵃᶠᵃ, Δφᵃᶜᵃ, φᵃᶠᵃ, φᵃᶜᵃ, - Δzᵃᵃᶠ, Δzᵃᵃᶜ, zᵃᵃᶠ, zᵃᵃᶜ, + z, Δxᶠᶜᵃ, Δxᶜᶠᵃ, Δxᶠᶠᵃ, Δxᶜᶜᵃ, Δyᶠᶜᵃ, Δyᶜᶠᵃ, Azᶠᶜᵃ, Azᶜᶠᵃ, Azᶠᶠᵃ, Azᶜᶜᵃ, radius) diff --git a/src/Grids/rectilinear_grid.jl b/src/Grids/rectilinear_grid.jl index d416ed76a3..35a24f37e8 100644 --- a/src/Grids/rectilinear_grid.jl +++ b/src/Grids/rectilinear_grid.jl @@ -19,7 +19,7 @@ struct RectilinearGrid{FT, TX, TY, TZ, CZ, FX, FY, VX, VY, Arch} <: AbstractUnde Δyᵃᶜᵃ :: FY yᵃᶠᵃ :: VY yᵃᶜᵃ :: VY - z :: FZ + z :: CZ RectilinearGrid{TX, TY, TZ}(arch::Arch, Nx, Ny, Nz, @@ -29,11 +29,11 @@ struct RectilinearGrid{FT, TX, TY, TZ, CZ, FX, FY, VX, VY, Arch} <: AbstractUnde xᶠᵃᵃ :: VX, xᶜᵃᵃ :: VX, Δyᵃᶠᵃ :: FY, Δyᵃᶜᵃ :: FY, yᵃᶠᵃ :: VY, yᵃᶜᵃ :: VY, - z :: FZ) where {Arch, FT, + z :: CZ) where {Arch, FT, TX, TY, TZ, FX, VX, FY, - VY, FZ} = - new{FT, TX, TY, TZ, FZ, FX, FY, VX, VY, Arch}(arch, Nx, Ny, Nz, + VY, CZ} = + new{FT, TX, TY, TZ, CZ, FX, FY, VX, VY, Arch}(arch, Nx, Ny, Nz, Hx, Hy, Hz, Lx, Ly, Lz, Δxᶠᵃᵃ, Δxᶜᵃᵃ, xᶠᵃᵃ, xᶜᵃᵃ, Δyᵃᶠᵃ, Δyᵃᶜᵃ, yᵃᶠᵃ, yᵃᶜᵃ, diff --git a/src/Grids/vertical_coordinate.jl b/src/Grids/vertical_coordinate.jl index db0e75d3cb..320f3b485f 100644 --- a/src/Grids/vertical_coordinate.jl +++ b/src/Grids/vertical_coordinate.jl @@ -1,6 +1,6 @@ abstract type AbstractVerticalCoordinate end -# Represents a static vertical coordinate system. +# Represents a static one-dimensional vertical coordinate. # # # Fields # - `cᶠ::C`: Face-centered coordinate. @@ -18,7 +18,7 @@ struct StaticVerticalCoordinate{C, D} <: AbstractVerticalCoordinate Δᶠ :: D end -# Represents a z-star vertical coordinate system. +# Represents a z-star three-dimensional vertical coordinate. # # # Fields # - `cᶠ::C`: Face-centered coordinates. @@ -48,8 +48,8 @@ struct ZStarVerticalCoordinate{C, D, E, CC, FC, CF, FF} <: AbstractVerticalCoord end # Convenience constructors for Zstar vertical coordinate -ZStarVerticalCoordinate(r_faces::Union{Tuple, AbstractVector}) = ZStarVerticalCoordinate(r_faces, r_faces, nothing, nothing, nothing, nothing, nothing) -ZStarVerticalCoordinate(r⁻::Number, r⁺::Number) = ZStarVerticalCoordinate((r⁻, r⁺), (r⁻, r⁺), nothing, nothing, nothing, nothing, nothing) +ZStarVerticalCoordinate(r_faces::Union{Tuple, AbstractVector}) = ZStarVerticalCoordinate(r_faces, r_faces, [nothing for i in 1:9]...) +ZStarVerticalCoordinate(r⁻::Number, r⁺::Number) = ZStarVerticalCoordinate((r⁻, r⁺), (r⁻, r⁺), [nothing for i in 1:9]...) #### #### Some usefull aliases @@ -95,20 +95,17 @@ Adapt.adapt_structure(to, coord::ZStarVerticalCoordinate) = @inline rnode(k, grid, ::Center) = getnode(grid.z.cᶜ, k) @inline rnode(k, grid, ::Face) = getnode(grid.z.cᶠ, k) +# These will be extended in the Operators module @inline znode(k, grid, ℓz) = rnode(k, grid, ℓz) @inline znode(i, j, k, grid, ℓx, ℓy, ℓz) = rnode(k, grid, ℓz) -# Extended for ZStarGrids in the Operators module -@inline znode(i, j, k, grid, ℓx, ℓy, ℓz) = znode(k, grid, ℓz) -@inline znode(k, grid, ℓx) = getnode(grid.z, k) - -# Summaries -Grids.coordinate_summary(::Bounded, z::ZStarVerticalCoordinate, name) = - @sprintf("Free-surface following with Δ%s=%s", name, prettysummary(z.Δᶜ)) - function validate_dimension_specification(T, ξ::ZStarVerticalCoordinate, dir, N, FT) reference = validate_dimension_specification(T, ξ.reference, dir, N, FT) args = Tuple(getproperty(ξ, prop) for prop in propertynames(ξ)) return ZStarVerticalCoordinate(reference, args[2:end]...) end + +# Summaries +coordinate_summary(::Bounded, z::AbstractVerticalCoordinate, name) = + @sprintf("Free-surface following with Δ%s=%s", name, prettysummary(z.Δᶜ)) diff --git a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl index f38f375bbb..9acb6d1e8f 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl @@ -11,7 +11,7 @@ using KernelAbstractions.Extras.LoopInfo: @unroll using Oceananigans.Utils using Oceananigans.Utils: launch!, SumOfArrays -using Oceananigans.Grids: AbstractGrid, ZStarUnderlyingGrid, rnode +using Oceananigans.Grids: AbstractGrid, rnode using Oceananigans.ImmersedBoundaries: ImmersedZStarGrid using Oceananigans.TimeSteppers: SplitRungeKutta3TimeStepper, QuasiAdamsBashforth2TimeStepper @@ -22,8 +22,6 @@ import Oceananigans.Advection: cell_advection_timescale import Oceananigans.TimeSteppers: step_lagrangian_particles! import Oceananigans.Architectures: on_architecture -const ZStarSpacingGrid = Union{ZStarUnderlyingGrid, ImmersedZStarGrid} - abstract type AbstractFreeSurface{E, G} end # This is only used by the cubed sphere for now. diff --git a/src/Models/Models.jl b/src/Models/Models.jl index d742538b1e..4981986dd5 100644 --- a/src/Models/Models.jl +++ b/src/Models/Models.jl @@ -23,12 +23,6 @@ import Oceananigans.Architectures: architecture import Oceananigans.TimeSteppers: reset! import Oceananigans.Solvers: iteration -using Oceananigans.Grids: AbstractVerticalCoordinateUnderlyingGrid -using Oceananigans.ImmersedBoundaries: ImmersedAbstractVerticalCoordinateGrid - -const AbstractVerticalCoordinateGrid = Union{AbstractVerticalCoordinateUnderlyingGrid, ImmersedAbstractVerticalCoordinateGrid} - - # A prototype interface for AbstractModel. # # TODO: decide if we like this. diff --git a/src/Operators/spacings_and_areas_and_volumes.jl b/src/Operators/spacings_and_areas_and_volumes.jl index bfd551a1ec..c7b1cfa6a0 100644 --- a/src/Operators/spacings_and_areas_and_volumes.jl +++ b/src/Operators/spacings_and_areas_and_volumes.jl @@ -1,5 +1,5 @@ using Oceananigans.Grids: Center, Face -using Oceananigans.Grids: AbstractVerticalCoordinateUnderlyingGrid, ZStarUnderlyingGrid +using Oceananigans.Grids: AbstractZStarGrid @inline hack_cosd(φ) = cos(π * φ / 180) @inline hack_sind(φ) = sin(π * φ / 180) @@ -43,21 +43,14 @@ The operators in this file fall into three categories: const ZRG = Union{LLGZ, RGZ, OSSGZ} -@inline Δzᵃᵃᶠ(i, j, k, grid) = @inbounds grid.Δzᵃᵃᶠ[k] -@inline Δzᵃᵃᶜ(i, j, k, grid) = @inbounds grid.Δzᵃᵃᶜ[k] - -@inline Δzᵃᵃᶠ(i, j, k, grid::ZRG) = grid.Δzᵃᵃᶠ -@inline Δzᵃᵃᶜ(i, j, k, grid::ZRG) = grid.Δzᵃᵃᶜ - -# Reference spacing, they differ only for a ZStar vertical coordinate grid -@inline Δrᵃᵃᶜ(i, j, k, grid) = Δzᵃᵃᶜ(i, j, k, grid) -@inline Δrᵃᵃᶠ(i, j, k, grid) = Δzᵃᵃᶠ(i, j, k, grid) - @inline getspacing(k, Δz::AbstractVector) = @inbounds Δz[k] @inline getspacing(k, Δz::Number) = @inbounds Δz -@inline Δrᵃᵃᶜ(i, j, k, grid::ZStarUnderlyingGrid) = getspacing(k, grid.z.Δᶜ) -@inline Δrᵃᵃᶠ(i, j, k, grid::ZStarUnderlyingGrid) = getspacing(k, grid.z.Δᶠ) +@inline Δrᵃᵃᶜ(i, j, k, grid) = getspacing(k, grid.z.Δᶜ) +@inline Δrᵃᵃᶠ(i, j, k, grid) = getspacing(k, grid.z.Δᶠ) + +@inline Δzᵃᵃᶠ(i, j, k, grid) = getspacing(k, grid.z.Δᶜ) +@inline Δzᵃᵃᶜ(i, j, k, grid) = getspacing(k, grid.z.Δᶠ) # Convenience Functions for all grids for LX in (:ᶜ, :ᶠ, :ᵃ), LY in (:ᶜ, :ᶠ, :ᵃ) @@ -92,23 +85,6 @@ for LX in (:ᶜ, :ᶠ, :ᵃ), LY in (:ᶜ, :ᶠ, :ᵃ) end end -##### -##### 3D spacings for AbstractVerticalCoordinate grids -##### - -const AVCG = AbstractVerticalCoordinateUnderlyingGrid -const c = Center() -const f = Face() - -@inline Δzᶜᶜᶠ(i, j, k, grid::AVCG) = @inbounds Δrᶜᶜᶠ(i, j, k, grid) * vertical_scaling(i, j, k, grid, c, c, f) -@inline Δzᶜᶜᶜ(i, j, k, grid::AVCG) = @inbounds Δrᶜᶜᶜ(i, j, k, grid) * vertical_scaling(i, j, k, grid, c, c, c) -@inline Δzᶠᶜᶠ(i, j, k, grid::AVCG) = @inbounds Δrᶠᶜᶠ(i, j, k, grid) * vertical_scaling(i, j, k, grid, f, c, f) -@inline Δzᶠᶜᶜ(i, j, k, grid::AVCG) = @inbounds Δrᶠᶜᶜ(i, j, k, grid) * vertical_scaling(i, j, k, grid, f, c, c) -@inline Δzᶜᶠᶠ(i, j, k, grid::AVCG) = @inbounds Δrᶜᶠᶠ(i, j, k, grid) * vertical_scaling(i, j, k, grid, c, f, f) -@inline Δzᶜᶠᶜ(i, j, k, grid::AVCG) = @inbounds Δrᶜᶠᶜ(i, j, k, grid) * vertical_scaling(i, j, k, grid, c, f, c) -@inline Δzᶠᶠᶠ(i, j, k, grid::AVCG) = @inbounds Δrᶠᶠᶠ(i, j, k, grid) * vertical_scaling(i, j, k, grid, f, f, f) -@inline Δzᶠᶠᶜ(i, j, k, grid::AVCG) = @inbounds Δrᶠᶠᶜ(i, j, k, grid) * vertical_scaling(i, j, k, grid, f, f, c) - ##### ##### Rectilinear Grids (Flat grids already have Δ = 1) ##### @@ -125,12 +101,6 @@ const f = Face() @inline Δyᶠᵃᶜ(i, j, k, grid::RG) = @inbounds grid.Δyᵃᶜᵃ[j] @inline Δyᶜᵃᶠ(i, j, k, grid::RG) = @inbounds grid.Δyᵃᶜᵃ[j] -@inline Δzᵃᵃᶠ(i, j, k, grid::RG) = @inbounds grid.Δzᵃᵃᶠ[k] -@inline Δzᵃᵃᶜ(i, j, k, grid::RG) = @inbounds grid.Δzᵃᵃᶜ[k] -@inline Δzᶜᵃᶜ(i, j, k, grid::RG) = @inbounds grid.Δzᵃᵃᶜ[k] -@inline Δzᶠᵃᶜ(i, j, k, grid::RG) = @inbounds grid.Δzᵃᵃᶜ[k] -@inline Δzᶜᵃᶠ(i, j, k, grid::RG) = @inbounds grid.Δzᵃᵃᶠ[k] - ## XRegularRG @inline Δxᶠᵃᵃ(i, j, k, grid::RGX) = grid.Δxᶠᵃᵃ @@ -149,15 +119,6 @@ const f = Face() @inline Δyᶠᵃᶜ(i, j, k, grid::RGY) = grid.Δyᵃᶜᵃ @inline Δyᶜᵃᶠ(i, j, k, grid::RGY) = grid.Δyᵃᶜᵃ -## ZRegularRG - -@inline Δzᵃᵃᶠ(i, j, k, grid::RGZ) = grid.Δzᵃᵃᶠ -@inline Δzᵃᵃᶜ(i, j, k, grid::RGZ) = grid.Δzᵃᵃᶜ - -@inline Δzᶜᵃᶜ(i, j, k, grid::RGZ) = grid.Δzᵃᵃᶜ -@inline Δzᶠᵃᶜ(i, j, k, grid::RGZ) = grid.Δzᵃᵃᶜ -@inline Δzᶜᵃᶠ(i, j, k, grid::RGZ) = grid.Δzᵃᵃᶠ - ##### ##### LatitudeLongitudeGrid ##### @@ -174,9 +135,6 @@ const f = Face() @inline Δyᶜᶜᵃ(i, j, k, grid::LLG) = Δyᶠᶜᵃ(i, j, k, grid) @inline Δyᶠᶠᵃ(i, j, k, grid::LLG) = Δyᶜᶠᵃ(i, j, k, grid) -@inline Δzᵃᵃᶠ(i, j, k, grid::LLG) = @inbounds grid.Δzᵃᵃᶠ[k] -@inline Δzᵃᵃᶜ(i, j, k, grid::LLG) = @inbounds grid.Δzᵃᵃᶜ[k] - ### XRegularLLG with pre-computed metrics @inline Δxᶠᶜᵃ(i, j, k, grid::LLGX) = @inbounds grid.Δxᶠᶜᵃ[j] @@ -189,11 +147,6 @@ const f = Face() @inline Δyᶜᶠᵃ(i, j, k, grid::LLGY) = grid.Δyᶜᶠᵃ @inline Δyᶠᶜᵃ(i, j, k, grid::LLGY) = grid.Δyᶠᶜᵃ -### ZRegularLLG with pre-computed metrics - -@inline Δzᵃᵃᶠ(i, j, k, grid::LLGZ) = grid.Δzᵃᵃᶠ -@inline Δzᵃᵃᶜ(i, j, k, grid::LLGZ) = grid.Δzᵃᵃᶜ - ## On the fly metrics @inline Δxᶠᶜᵃ(i, j, k, grid::LLGF) = @inbounds grid.radius * deg2rad(grid.Δλᶠᵃᵃ[i]) * hack_cosd(grid.φᵃᶜᵃ[j]) @@ -230,12 +183,6 @@ const f = Face() @inline Δyᶜᶠᵃ(i, j, k, grid::OSSG) = @inbounds grid.Δyᶜᶠᵃ[i, j] @inline Δyᶠᶠᵃ(i, j, k, grid::OSSG) = @inbounds grid.Δyᶠᶠᵃ[i, j] -@inline Δzᵃᵃᶜ(i, j, k, grid::OSSG) = @inbounds grid.Δzᵃᵃᶜ[k] -@inline Δzᵃᵃᶠ(i, j, k, grid::OSSG) = @inbounds grid.Δzᵃᵃᶠ[k] - -@inline Δzᵃᵃᶜ(i, j, k, grid::OSSGZ) = grid.Δzᵃᵃᶜ -@inline Δzᵃᵃᶠ(i, j, k, grid::OSSGZ) = grid.Δzᵃᵃᶠ - ##### ##### ##### Areas!! diff --git a/src/Operators/variable_grid_operators.jl b/src/Operators/variable_grid_operators.jl index 763ac8cd0e..cc80db03f6 100644 --- a/src/Operators/variable_grid_operators.jl +++ b/src/Operators/variable_grid_operators.jl @@ -1,5 +1,4 @@ -using Oceananigans.Grids: ZStarUnderlyingGrid -import Oceananigans.Grids: znode +import Oceananigans.Grids: znode, AbstractZStarGrid import Oceananigans.Grids: dynamic_column_depthᶜᶜᵃ, dynamic_column_depthᶜᶠᵃ, @@ -55,5 +54,8 @@ for Lx in (:ᶠ, :ᶜ), Lx in (:ᶠ, :ᶜ), Lx in (:ᶠ, :ᶜ) end end -# rnode for an ZStarUnderlyingGrid grid is scaled -@inline znode(i, j, k, grid::ZSG, ℓx, ℓy, ℓz) = @inbounds rnode(i, j, k, grid, ℓx, ℓy, ℓz) * e₃ⁿ(i, j, k, grid, ℓx, ℓy, ℓz) + grid.z.ηⁿ[i, j, 1] +# rnode for an AbstractZStarGrid grid is scaled +@inline znode(i, j, k, grid::ZSG, ::C, ::C, ℓz) = @inbounds rnode(i, j, k, grid, ℓx, ℓy, ℓz) * e₃ⁿ(i, j, k, grid, ℓx, ℓy, ℓz) + grid.z.ηⁿ[i, j, 1] +@inline znode(i, j, k, grid::ZSG, ::F, ::C, ℓz) = @inbounds rnode(i, j, k, grid, ℓx, ℓy, ℓz) * e₃ⁿ(i, j, k, grid, ℓx, ℓy, ℓz) + ℑxᶠᵃᵃ(i, j, 1, grid, grid.z.ηⁿ) +@inline znode(i, j, k, grid::ZSG, ::C, ::F, ℓz) = @inbounds rnode(i, j, k, grid, ℓx, ℓy, ℓz) * e₃ⁿ(i, j, k, grid, ℓx, ℓy, ℓz) + ℑyᵃᶠᵃ(i, j, 1, grid, grid.z.ηⁿ) +@inline znode(i, j, k, grid::ZSG, ::F, ::F, ℓz) = @inbounds rnode(i, j, k, grid, ℓx, ℓy, ℓz) * e₃ⁿ(i, j, k, grid, ℓx, ℓy, ℓz) + ℑxyᶠᶠᵃ(i, j, 1, grid, grid.z.ηⁿ) From a339f6ac41ece1e66315edbbf3f281c75a5589c4 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Sat, 23 Nov 2024 12:42:06 -0500 Subject: [PATCH 472/567] this to finish --- src/Grids/latitude_longitude_grid.jl | 10 ---------- src/Grids/orthogonal_spherical_shell_grid.jl | 13 ------------- src/Grids/rectilinear_grid.jl | 8 -------- src/Grids/vertical_coordinate.jl | 9 +++++++++ 4 files changed, 9 insertions(+), 31 deletions(-) diff --git a/src/Grids/latitude_longitude_grid.jl b/src/Grids/latitude_longitude_grid.jl index 7b36989ec1..ac1c61d5ab 100644 --- a/src/Grids/latitude_longitude_grid.jl +++ b/src/Grids/latitude_longitude_grid.jl @@ -636,15 +636,9 @@ end return @. R * deg2rad(φ) end -@inline znodes(grid::LLG, ℓz::F; with_halos=false) = _property(grid.z.cᶠ, ℓz, topology(grid, 3), size(grid, 3), with_halos) -@inline znodes(grid::LLG, ℓz::C; with_halos=false) = _property(grid.z.cᶜ, ℓz, topology(grid, 3), size(grid, 3), with_halos) -@inline rnodes(grid::LLG, ℓz::F; with_halos=false) = _property(grid.z.cᶠ, ℓz, topology(grid, 3), size(grid, 3), with_halos) -@inline rnodes(grid::LLG, ℓz::C; with_halos=false) = _property(grid.z.cᶜ, ℓz, topology(grid, 3), size(grid, 3), with_halos) - # Convenience @inline λnodes(grid::LLG, ℓx, ℓy, ℓz; with_halos=false) = λnodes(grid, ℓx; with_halos) @inline φnodes(grid::LLG, ℓx, ℓy, ℓz; with_halos=false) = φnodes(grid, ℓy; with_halos) -@inline znodes(grid::LLG, ℓx, ℓy, ℓz; with_halos=false) = znodes(grid, ℓz; with_halos) @inline xnodes(grid::LLG, ℓx, ℓy, ℓz; with_halos=false) = xnodes(grid, ℓx, ℓy; with_halos) @inline ynodes(grid::LLG, ℓx, ℓy, ℓz; with_halos=false) = ynodes(grid, ℓy; with_halos) @@ -656,11 +650,9 @@ end # Generalized coordinates @inline ξnodes(grid::LLG, ℓx; kwargs...) = λnodes(grid, ℓx; kwargs...) @inline ηnodes(grid::LLG, ℓy; kwargs...) = φnodes(grid, ℓy; kwargs...) -@inline rnodes(grid::LLG, ℓz; kwargs...) = znodes(grid, ℓz; kwargs...) @inline ξnodes(grid::LLG, ℓx, ℓy, ℓz; kwargs...) = λnodes(grid, ℓx; kwargs...) @inline ηnodes(grid::LLG, ℓx, ℓy, ℓz; kwargs...) = φnodes(grid, ℓy; kwargs...) -@inline rnodes(grid::LLG, ℓx, ℓy, ℓz; kwargs...) = rnodes(grid, ℓz; kwargs...) ##### ##### Grid spacings @@ -681,8 +673,6 @@ end @inline xspacings(grid::LLG, ℓx, ℓy) = xspacings(grid, ℓx, ℓy, nothing) @inline yspacings(grid::LLG, ℓx, ℓy) = yspacings(grid, ℓx, ℓy, nothing) -@inline zspacings(grid::LLG, ℓz) = zspacings(grid, nothing, nothing, ℓz) -@inline rspacings(grid::LLG, ℓz) = rspacings(grid, ℓz; kwargs...) @inline λspacings(grid::LLG, ℓx) = λspacings(grid, ℓx, nothing, nothing) @inline φspacings(grid::LLG, ℓy) = φspacings(grid, nothing, ℓy, nothing) diff --git a/src/Grids/orthogonal_spherical_shell_grid.jl b/src/Grids/orthogonal_spherical_shell_grid.jl index fcfa93ac67..694d694333 100644 --- a/src/Grids/orthogonal_spherical_shell_grid.jl +++ b/src/Grids/orthogonal_spherical_shell_grid.jl @@ -1153,20 +1153,9 @@ end @inline xnodes(grid::OSSG, ℓx, ℓy; with_halos=false) = grid.radius * deg2rad.(λnodes(grid, ℓx, ℓy; with_halos=with_halos)) .* hack_cosd.(φnodes(grid, ℓx, ℓy; with_halos=with_halos)) @inline ynodes(grid::OSSG, ℓx, ℓy; with_halos=false) = grid.radius * deg2rad.(φnodes(grid, ℓx, ℓy; with_halos=with_halos)) -@inline znodes(grid::OSSG, ℓz::Face ; with_halos=false) = with_halos ? grid.zᵃᵃᶠ : - view(grid.zᵃᵃᶠ, interior_indices(ℓz, topology(grid, 3)(), grid.Nz)) -@inline znodes(grid::OSSG, ℓz::Center; with_halos=false) = with_halos ? grid.zᵃᵃᶜ : - view(grid.zᵃᵃᶜ, interior_indices(ℓz, topology(grid, 3)(), grid.Nz)) - -@inline rnodes(grid::OSSG, ℓz::Face ; with_halos=false) = with_halos ? grid.zᵃᵃᶠ : - view(grid.zᵃᵃᶠ, interior_indices(ℓz, topology(grid, 3)(), grid.Nz)) -@inline rnodes(grid::OSSG, ℓz::Center; with_halos=false) = with_halos ? grid.zᵃᵃᶜ : - view(grid.zᵃᵃᶜ, interior_indices(ℓz, topology(grid, 3)(), grid.Nz)) - # convenience @inline λnodes(grid::OSSG, ℓx, ℓy, ℓz; with_halos=false) = λnodes(grid, ℓx, ℓy; with_halos) @inline φnodes(grid::OSSG, ℓx, ℓy, ℓz; with_halos=false) = φnodes(grid, ℓx, ℓy; with_halos) -@inline znodes(grid::OSSG, ℓx, ℓy, ℓz; with_halos=false) = znodes(grid, ℓz ; with_halos) @inline rnodes(grid::OSSG, ℓx, ℓy, ℓz; with_halos=false) = rnodes(grid, ℓz ; with_halos) @inline xnodes(grid::OSSG, ℓx, ℓy, ℓz; with_halos=false) = xnodes(grid, ℓx, ℓy; with_halos) @inline ynodes(grid::OSSG, ℓx, ℓy, ℓz; with_halos=false) = ynodes(grid, ℓx, ℓy; with_halos) @@ -1187,14 +1176,12 @@ end # convenience @inline λnode(i, j, k, grid::OSSG, ℓx, ℓy, ℓz) = λnode(i, j, grid, ℓx, ℓy) @inline φnode(i, j, k, grid::OSSG, ℓx, ℓy, ℓz) = φnode(i, j, grid, ℓx, ℓy) -@inline znode(i, j, k, grid::OSSG, ℓx, ℓy, ℓz) = znode(k, grid, ℓz) @inline xnode(i, j, k, grid::OSSG, ℓx, ℓy, ℓz) = xnode(i, j, grid, ℓx, ℓy) @inline ynode(i, j, k, grid::OSSG, ℓx, ℓy, ℓz) = ynode(i, j, grid, ℓx, ℓy) # Definitions for node @inline ξnode(i, j, k, grid::OSSG, ℓx, ℓy, ℓz) = λnode(i, j, grid, ℓx, ℓy) @inline ηnode(i, j, k, grid::OSSG, ℓx, ℓy, ℓz) = φnode(i, j, grid, ℓx, ℓy) -@inline rnode(i, j, k, grid::OSSG, ℓx, ℓy, ℓz) = znode(k, grid, ℓz) ξname(::OSSG) = :λ ηname(::OSSG) = :φ diff --git a/src/Grids/rectilinear_grid.jl b/src/Grids/rectilinear_grid.jl index 35a24f37e8..e143ea7383 100644 --- a/src/Grids/rectilinear_grid.jl +++ b/src/Grids/rectilinear_grid.jl @@ -496,24 +496,17 @@ const C = Center @inline xnodes(grid::RG, ℓx::C; with_halos=false) = _property(grid.xᶜᵃᵃ, ℓx, topology(grid, 1), size(grid, 1), with_halos) @inline ynodes(grid::RG, ℓy::F; with_halos=false) = _property(grid.yᵃᶠᵃ, ℓy, topology(grid, 2), size(grid, 2), with_halos) @inline ynodes(grid::RG, ℓy::C; with_halos=false) = _property(grid.yᵃᶜᵃ, ℓy, topology(grid, 2), size(grid, 2), with_halos) -@inline znodes(grid::RG, ℓz::F; with_halos=false) = _property(grid.zᵃᵃᶠ, ℓz, topology(grid, 3), size(grid, 3), with_halos) -@inline znodes(grid::RG, ℓz::C; with_halos=false) = _property(grid.zᵃᵃᶜ, ℓz, topology(grid, 3), size(grid, 3), with_halos) -@inline rnodes(grid::RG, ℓz::F; with_halos=false) = _property(grid.zᵃᵃᶠ, ℓz, topology(grid, 3), size(grid, 3), with_halos) -@inline rnodes(grid::RG, ℓz::C; with_halos=false) = _property(grid.zᵃᵃᶜ, ℓz, topology(grid, 3), size(grid, 3), with_halos) # convenience @inline xnodes(grid::RG, ℓx, ℓy, ℓz; with_halos=false) = xnodes(grid, ℓx; with_halos) @inline ynodes(grid::RG, ℓx, ℓy, ℓz; with_halos=false) = ynodes(grid, ℓy; with_halos) -@inline znodes(grid::RG, ℓx, ℓy, ℓz; with_halos=false) = znodes(grid, ℓz; with_halos) # Generalized coordinates @inline ξnodes(grid::RG, ℓx; kwargs...) = xnodes(grid, ℓx; kwargs...) @inline ηnodes(grid::RG, ℓy; kwargs...) = ynodes(grid, ℓy; kwargs...) -@inline rnodes(grid::RG, ℓz; kwargs...) = znodes(grid, ℓz; kwargs...) @inline ξnodes(grid::RG, ℓx, ℓy, ℓz; kwargs...) = xnodes(grid, ℓx; kwargs...) @inline ηnodes(grid::RG, ℓx, ℓy, ℓz; kwargs...) = ynodes(grid, ℓy; kwargs...) -@inline rnodes(grid::RG, ℓx, ℓy, ℓz; kwargs...) = rnodes(grid, ℓz; kwargs...) @inline isrectilinear(::RG) = true @@ -523,4 +516,3 @@ const C = Center @inline xspacings(grid::RG, ℓx) = xspacings(grid, ℓx, nothing, nothing) @inline yspacings(grid::RG, ℓy) = yspacings(grid, nothing, ℓy, nothing) -@inline zspacings(grid::RG, ℓz) = zspacings(grid, nothing, nothing, ℓz) diff --git a/src/Grids/vertical_coordinate.jl b/src/Grids/vertical_coordinate.jl index 320f3b485f..e3f3669f9c 100644 --- a/src/Grids/vertical_coordinate.jl +++ b/src/Grids/vertical_coordinate.jl @@ -109,3 +109,12 @@ end # Summaries coordinate_summary(::Bounded, z::AbstractVerticalCoordinate, name) = @sprintf("Free-surface following with Δ%s=%s", name, prettysummary(z.Δᶜ)) + +#### +#### Nodes and spacings... +#### + +# znodes(....) +# rnodes(....) +# zspacings(....) +# rspacings(....) \ No newline at end of file From 03708642cc7f9ac185159c4f91258a3e01f2686e Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 25 Nov 2024 11:16:19 +0100 Subject: [PATCH 473/567] should compile --- src/Advection/Advection.jl | 1 + src/Advection/vector_invariant_advection.jl | 3 -- .../vector_invariant_cross_upwinding.jl | 6 ++-- .../vector_invariant_self_upwinding.jl | 8 ++--- .../distributed_grids.jl | 32 +++++++++++-------- src/ImmersedBoundaries/zstar_immersed_grid.jl | 8 +++-- .../compute_w_from_continuity.jl | 2 +- .../generalized_vertical_spacing.jl | 4 +-- src/Operators/Operators.jl | 1 + .../spacings_and_areas_and_volumes.jl | 7 +--- src/Operators/variable_grid_operators.jl | 14 ++++---- 11 files changed, 43 insertions(+), 43 deletions(-) diff --git a/src/Advection/Advection.jl b/src/Advection/Advection.jl index 6473915aed..d813b6d7b9 100644 --- a/src/Advection/Advection.jl +++ b/src/Advection/Advection.jl @@ -34,6 +34,7 @@ using Oceananigans.Grids: with_halo, coordinates using Oceananigans.Architectures: architecture, CPU using Oceananigans.Operators +using Oceananigans.Operators: flux_div_xyᶜᶜᶜ, Γᶠᶠᶜ, ∂t_e₃ import Base: show, summary import Oceananigans.Grids: required_halo_size_x, required_halo_size_y, required_halo_size_z diff --git a/src/Advection/vector_invariant_advection.jl b/src/Advection/vector_invariant_advection.jl index c756e14843..7f02532200 100644 --- a/src/Advection/vector_invariant_advection.jl +++ b/src/Advection/vector_invariant_advection.jl @@ -1,6 +1,3 @@ -using Oceananigans.Operators -using Oceananigans.Operators: flux_div_xyᶜᶜᶜ, Γᶠᶠᶜ - # These are also used in Coriolis/hydrostatic_spherical_coriolis.jl struct EnergyConserving{FT} <: AbstractAdvectionScheme{1, FT} end struct EnstrophyConserving{FT} <: AbstractAdvectionScheme{1, FT} end diff --git a/src/Advection/vector_invariant_cross_upwinding.jl b/src/Advection/vector_invariant_cross_upwinding.jl index 052ae63be4..5c0c459a25 100644 --- a/src/Advection/vector_invariant_cross_upwinding.jl +++ b/src/Advection/vector_invariant_cross_upwinding.jl @@ -17,14 +17,14 @@ ##### Cross and Self Upwinding of the Divergence flux ##### -@inline V_times_∂t_grid(i, j, k, grid) = Vᶜᶜᶜ(i, j, k, grid) * ∂t_grid(i, j, k, grid) +@inline V_times_∂t_e₃(i, j, k, grid) = Vᶜᶜᶜ(i, j, k, grid) * ∂t_e₃(i, j, k, grid) @inline function upwinded_divergence_flux_Uᶠᶜᶜ(i, j, k, grid, scheme::VectorInvariantCrossVerticalUpwinding, u, v) @inbounds û = u[i, j, k] δ_stencil = scheme.upwinding.divergence_stencil δᴿ = _biased_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, bias(û), flux_div_xyᶜᶜᶜ, δ_stencil, u, v) - ∂ts = _symmetric_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, cross_scheme, V_times_∂t_grid) + ∂ts = _symmetric_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, cross_scheme, V_times_∂t_e₃) return û * (δᴿ + ∂ts) end @@ -34,7 +34,7 @@ end δ_stencil = scheme.upwinding.divergence_stencil δᴿ = _biased_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, bias(v̂), flux_div_xyᶜᶜᶜ, δ_stencil, u, v) - ∂ts = _symmetric_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, cross_scheme, V_times_∂t_grid) + ∂ts = _symmetric_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, cross_scheme, V_times_∂t_e₃) return v̂ * (δᴿ + ∂ts) end diff --git a/src/Advection/vector_invariant_self_upwinding.jl b/src/Advection/vector_invariant_self_upwinding.jl index 91f30d7395..fa60ee949a 100644 --- a/src/Advection/vector_invariant_self_upwinding.jl +++ b/src/Advection/vector_invariant_self_upwinding.jl @@ -1,5 +1,3 @@ -using Oceananigans.Grids: ∂t_grid - ##### ##### Self Upwinding of Divergence Flux, the best option! ##### @@ -7,15 +5,15 @@ using Oceananigans.Grids: ∂t_grid @inline δx_U(i, j, k, grid, u, v) = δxᶜᶜᶜ(i, j, k, grid, Ax_qᶠᶜᶜ, u) @inline δy_V(i, j, k, grid, u, v) = δyᶜᶜᶜ(i, j, k, grid, Ay_qᶜᶠᶜ, v) -@inline δx_U_plus_metric(i, j, k, grid, u, v) = δxᶜᶜᶜ(i, j, k, grid, Ax_qᶠᶜᶜ, u) + Vᶜᶜᶜ(i, j, k, grid) * ∂t_grid(i, j, k, grid) -@inline δy_V_plus_metric(i, j, k, grid, u, v) = δyᶜᶜᶜ(i, j, k, grid, Ay_qᶜᶠᶜ, v) + Vᶜᶜᶜ(i, j, k, grid) * ∂t_grid(i, j, k, grid) +@inline δx_U_plus_metric(i, j, k, grid, u, v) = δxᶜᶜᶜ(i, j, k, grid, Ax_qᶠᶜᶜ, u) + Vᶜᶜᶜ(i, j, k, grid) * ∂t_e₃(i, j, k, grid) +@inline δy_V_plus_metric(i, j, k, grid, u, v) = δyᶜᶜᶜ(i, j, k, grid, Ay_qᶜᶠᶜ, v) + Vᶜᶜᶜ(i, j, k, grid) * ∂t_e₃(i, j, k, grid) # Velocity smoothness for divergence upwinding @inline U_smoothness(i, j, k, grid, u, v) = ℑxᶜᵃᵃ(i, j, k, grid, Ax_qᶠᶜᶜ, u) @inline V_smoothness(i, j, k, grid, u, v) = ℑyᵃᶜᵃ(i, j, k, grid, Ay_qᶜᶠᶜ, v) # Divergence smoothness for divergence upwinding -@inline divergence_smoothness(i, j, k, grid, u, v) = δx_U(i, j, k, grid, u, v) + δy_V(i, j, k, grid, u, v) + Vᶜᶜᶜ(i, j, k, grid) * ∂t_grid(i, j, k, grid) +@inline divergence_smoothness(i, j, k, grid, u, v) = δx_U(i, j, k, grid, u, v) + δy_V(i, j, k, grid, u, v) + Vᶜᶜᶜ(i, j, k, grid) * ∂t_e₃(i, j, k, grid) @inline function upwinded_divergence_flux_Uᶠᶜᶜ(i, j, k, grid, scheme::VectorInvariantSelfVerticalUpwinding, u, v) diff --git a/src/DistributedComputations/distributed_grids.jl b/src/DistributedComputations/distributed_grids.jl index 1b52477149..7658a2485f 100644 --- a/src/DistributedComputations/distributed_grids.jl +++ b/src/DistributedComputations/distributed_grids.jl @@ -81,12 +81,15 @@ function RectilinearGrid(arch::Distributed, TY = insert_connected_topology(topology[2], Ry, rj) TZ = insert_connected_topology(topology[3], Rz, rk) + local_topo = (TX, TY, TZ) + xl = Rx == 1 ? x : partition_coordinate(x, nx, arch, 1) yl = Ry == 1 ? y : partition_coordinate(y, ny, arch, 2) zl = Rz == 1 ? z : partition_coordinate(z, nz, arch, 3) - Lx, xᶠᵃᵃ, xᶜᵃᵃ, Δxᶠᵃᵃ, Δxᶜᵃᵃ = generate_coordinate(FT, topology[1](), nx, Hx, xl, :x, child_architecture(arch)) - Ly, yᵃᶠᵃ, yᵃᶜᵃ, Δyᵃᶠᵃ, Δyᵃᶜᵃ = generate_coordinate(FT, topology[2](), ny, Hy, yl, :y, child_architecture(arch)) - Lz, zᵃᵃᶠ, zᵃᵃᶜ, Δzᵃᵃᶠ, Δzᵃᵃᶜ = generate_coordinate(FT, topology[3](), nz, Hz, zl, :z, child_architecture(arch)) + + Lx, xᶠᵃᵃ, xᶜᵃᵃ, Δxᶠᵃᵃ, Δxᶜᵃᵃ = generate_coordinate(FT, local_topo, local_sz, halo, xl, :x, 1, child_architecture(arch)) + Ly, yᵃᶠᵃ, yᵃᶜᵃ, Δyᵃᶠᵃ, Δyᵃᶜᵃ = generate_coordinate(FT, local_topo, local_sz, halo, yl, :y, 2, child_architecture(arch)) + Lz, z = generate_coordinate(FT, local_topo, local_sz, halo, zl, :z, 3, child_architecture(arch)) return RectilinearGrid{TX, TY, TZ}(arch, nx, ny, nz, @@ -94,7 +97,7 @@ function RectilinearGrid(arch::Distributed, Lx, Ly, Lz, Δxᶠᵃᵃ, Δxᶜᵃᵃ, xᶠᵃᵃ, xᶜᵃᵃ, Δyᵃᶜᵃ, Δyᵃᶠᵃ, yᵃᶠᵃ, yᵃᶜᵃ, - Δzᵃᵃᶠ, Δzᵃᵃᶜ, zᵃᵃᶠ, zᵃᵃᶜ) + z) end """ @@ -127,6 +130,8 @@ function LatitudeLongitudeGrid(arch::Distributed, TY = insert_connected_topology(topology[2], Ry, rj) TZ = insert_connected_topology(topology[3], Rz, rk) + local_topo = (TX, TY, TZ) + λl = Rx == 1 ? longitude : partition_coordinate(longitude, nλ, arch, 1) φl = Ry == 1 ? latitude : partition_coordinate(latitude, nφ, arch, 2) zl = Rz == 1 ? z : partition_coordinate(z, nz, arch, 3) @@ -134,8 +139,8 @@ function LatitudeLongitudeGrid(arch::Distributed, # Calculate all direction (which might be stretched) # A direction is regular if the domain passed is a Tuple{<:Real, <:Real}, # it is stretched if being passed is a function or vector (as for the VerticallyStretchedRectilinearGrid) - Lλ, λᶠᵃᵃ, λᶜᵃᵃ, Δλᶠᵃᵃ, Δλᶜᵃᵃ = generate_coordinate(FT, TX(), nλ, Hλ, λl, :longitude, arch.child_architecture) - Lz, zᵃᵃᶠ, zᵃᵃᶜ, Δzᵃᵃᶠ, Δzᵃᵃᶜ = generate_coordinate(FT, TZ(), nz, Hz, zl, :z, arch.child_architecture) + Lλ, λᶠᵃᵃ, λᶜᵃᵃ, Δλᶠᵃᵃ, Δλᶜᵃᵃ = generate_coordinate(FT, local_topo, local_sz, halo, λl, :longitude, 1, child_architecture(arch)) + Lz, z = generate_coordinate(FT, local_topo, local_sz, halo, zl, :z, 3, child_architecture(arch)) # The Latitudinal direction is _special_: # precompute_metrics assumes that `length(φᵃᶠᵃ) = length(φᵃᶜᵃ) + 1`, which is always the case in a @@ -145,7 +150,8 @@ function LatitudeLongitudeGrid(arch::Distributed, # we disregard the topology when constructing the metrics and add a halo point! # Furthermore, the `LatitudeLongitudeGrid` requires an extra halo on it's latitudinal coordinate to allow calculating # the z-area on halo cells. (see: Az = R^2 * Δλ * (sin(φ[j]) - sin(φ[j-1])) - Lφ, φᵃᶠᵃ, φᵃᶜᵃ, Δφᵃᶠᵃ, Δφᵃᶜᵃ = generate_coordinate(FT, Bounded(), nφ, Hφ + 1, φl, :latitude, arch.child_architecture) + Lφ, φᵃᶠᵃ, φᵃᶜᵃ, Δφᵃᶠᵃ, Δφᵃᶜᵃ = generate_coordinate(FT, Bounded(), nφ, Hφ + 1, φl, :latitude, child_architecture(arch)) + preliminary_grid = LatitudeLongitudeGrid{TX, TY, TZ}(arch, nλ, nφ, nz, @@ -153,7 +159,7 @@ function LatitudeLongitudeGrid(arch::Distributed, Lλ, Lφ, Lz, Δλᶠᵃᵃ, Δλᶜᵃᵃ, λᶠᵃᵃ, λᶜᵃᵃ, Δφᵃᶠᵃ, Δφᵃᶜᵃ, φᵃᶠᵃ, φᵃᶜᵃ, - Δzᵃᵃᶠ, Δzᵃᵃᶜ, zᵃᵃᶠ, zᵃᵃᶜ, + z, (nothing for i=1:10)..., convert(FT, radius)) return !precompute_metrics ? preliminary_grid : with_precomputed_metrics(preliminary_grid) @@ -199,7 +205,7 @@ function reconstruct_global_grid(grid::DistributedRectilinearGrid) Lx, xᶠᵃᵃ, xᶜᵃᵃ, Δxᶠᵃᵃ, Δxᶜᵃᵃ = generate_coordinate(FT, TX(), Nx, Hx, xG, :x, child_arch) Ly, yᵃᶠᵃ, yᵃᶜᵃ, Δyᵃᶠᵃ, Δyᵃᶜᵃ = generate_coordinate(FT, TY(), Ny, Hy, yG, :y, child_arch) - Lz, zᵃᵃᶠ, zᵃᵃᶜ, Δzᵃᵃᶠ, Δzᵃᵃᶜ = generate_coordinate(FT, TZ(), Nz, Hz, zG, :z, child_arch) + Lz, z = generate_coordinate(FT, (TX, TY, TZ), (Nx, Ny, Nz), H, zG, :z, 3, child_arch) return RectilinearGrid{TX, TY, TZ}(child_arch, Nx, Ny, Nz, @@ -207,7 +213,7 @@ function reconstruct_global_grid(grid::DistributedRectilinearGrid) Lx, Ly, Lz, Δxᶠᵃᵃ, Δxᶜᵃᵃ, xᶠᵃᵃ, xᶜᵃᵃ, Δyᵃᶠᵃ, Δyᵃᶜᵃ, yᵃᶠᵃ, yᵃᶜᵃ, - Δzᵃᵃᶠ, Δzᵃᵃᶜ, zᵃᵃᶠ, zᵃᵃᶜ) + z) end function reconstruct_global_grid(grid::DistributedLatitudeLongitudeGrid) @@ -245,8 +251,8 @@ function reconstruct_global_grid(grid::DistributedLatitudeLongitudeGrid) # it is stretched if being passed is a function or vector Lλ, λᶠᵃᵃ, λᶜᵃᵃ, Δλᶠᵃᵃ, Δλᶜᵃᵃ = generate_coordinate(FT, TX(), Nλ, Hλ, λG, :longitude, child_arch) Lφ, φᵃᶠᵃ, φᵃᶜᵃ, Δφᵃᶠᵃ, Δφᵃᶜᵃ = generate_coordinate(FT, TY(), Nφ, Hφ, φG, :latitude, child_arch) - Lz, zᵃᵃᶠ, zᵃᵃᶜ, Δzᵃᵃᶠ, Δzᵃᵃᶜ = generate_coordinate(FT, TZ(), Nz, Hz, zG, :z, child_arch) - + Lz, z = generate_coordinate(FT, (TX, TY, TZ), (Nx, Ny, Nz), H, zG, :z, 3, child_arch) + precompute_metrics = metrics_precomputed(grid) preliminary_grid = LatitudeLongitudeGrid{TX, TY, TZ}(child_arch, @@ -255,7 +261,7 @@ function reconstruct_global_grid(grid::DistributedLatitudeLongitudeGrid) Lλ, Lφ, Lz, Δλᶠᵃᵃ, Δλᶜᵃᵃ, λᶠᵃᵃ, λᶜᵃᵃ, Δφᵃᶠᵃ, Δφᵃᶜᵃ, φᵃᶠᵃ, φᵃᶜᵃ, - Δzᵃᵃᶠ, Δzᵃᵃᶜ, zᵃᵃᶠ, zᵃᵃᶜ, + z, (nothing for i=1:10)..., grid.radius) return !precompute_metrics ? preliminary_grid : with_precomputed_metrics(preliminary_grid) diff --git a/src/ImmersedBoundaries/zstar_immersed_grid.jl b/src/ImmersedBoundaries/zstar_immersed_grid.jl index 9b00c217da..7aaed57ffe 100644 --- a/src/ImmersedBoundaries/zstar_immersed_grid.jl +++ b/src/ImmersedBoundaries/zstar_immersed_grid.jl @@ -1,4 +1,4 @@ -using Oceananigans.Grids: ZStarGrid +using Oceananigans.Grids: AbstractZStarGrid using Oceananigans.Operators import Oceananigans.Grids: dynamic_column_depthᶜᶜᵃ, @@ -6,8 +6,10 @@ import Oceananigans.Grids: dynamic_column_depthᶜᶜᵃ, dynamic_column_depthᶠᶜᵃ, dynamic_column_depthᶠᶠᵃ -const ZStarImmersedGrid = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:ZStarGrid} -const ZStarGridOfSomeKind = Union{ZStarImmersedGrid, ZStarGrid} +import Oceananigans.Operators: e₃ⁿ, e₃⁻, ∂t_e₃ + +const ZStarImmersedGrid = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:AbstractZStarGrid} +const ZStarGridOfSomeKind = Union{ZStarImmersedGrid, AbstractZStarGrid} @inline dynamic_column_depthᶜᶜᵃ(i, j, grid::ZStarGridOfSomeKind, η) = @inbounds static_column_depthᶜᶜᵃ(i, j, grid) + η[i, j, grid.Nz+1] @inline dynamic_column_depthᶜᶠᵃ(i, j, grid::ZStarGridOfSomeKind, η) = @inbounds static_column_depthᶜᶠᵃ(i, j, grid) + ℑxᶠᵃᵃ(i, j, grid.Nz+1, grid, η) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl index 335d6ccc52..fa013b9425 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl @@ -25,7 +25,7 @@ compute_w_from_continuity!(velocities, arch, grid; parameters = w_kernel_paramet @inbounds U.w[i, j, 1] = 0 for k in 2:grid.Nz+1 δh_u = flux_div_xyᶜᶜᶜ(i, j, k-1, grid, U.u, U.v) / Azᶜᶜᶜ(i, j, k-1, grid) - ∂t_s = Δrᶜᶜᶜ(i, j, k-1, grid) * ∂t_grid(i, j, k-1, grid) + ∂t_s = Δrᶜᶜᶜ(i, j, k-1, grid) * ∂t_e₃(i, j, k-1, grid) immersed = immersed_cell(i, j, k-1, grid) Δw = δh_u + ifelse(immersed, 0, ∂t_s) # We do not account for grid changes in immersed cells diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index 093214c27f..1da202e069 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -2,11 +2,11 @@ using Oceananigans using Oceananigans.Grids using Oceananigans.Operators using Oceananigans.BuoyancyModels: buoyancy_perturbationᶜᶜᶜ -using Oceananigans.Grids: AbstractGrid, AbstractUnderlyingGrid, halo_size +using Oceananigans.Grids: AbstractGrid, AbstractUnderlyingGrid, halo_size, with_halo using Oceananigans.ImmersedBoundaries using Oceananigans.Models: AbstractVerticalCoordinateGrid using Oceananigans.Utils: getnamewrapper -using Oceananigans.Grids: with_halo, ∂t_grid, vertical_scaling, previous_vertical_scaling +using Oceananigans.Operators: ∂t_e₃ using Adapt using Printf diff --git a/src/Operators/Operators.jl b/src/Operators/Operators.jl index 7b982317d3..d53539c408 100644 --- a/src/Operators/Operators.jl +++ b/src/Operators/Operators.jl @@ -122,6 +122,7 @@ include("topology_aware_operators.jl") include("vorticity_operators.jl") include("laplacian_operators.jl") +include("variable_grid_operators.jl") include("vector_rotation_operators.jl") end # module diff --git a/src/Operators/spacings_and_areas_and_volumes.jl b/src/Operators/spacings_and_areas_and_volumes.jl index c7b1cfa6a0..9464ab6033 100644 --- a/src/Operators/spacings_and_areas_and_volumes.jl +++ b/src/Operators/spacings_and_areas_and_volumes.jl @@ -36,11 +36,6 @@ The operators in this file fall into three categories: ##### ##### -@inline Δxᶠᵃᵃ(i, j, k, grid) = nothing -@inline Δxᶜᵃᵃ(i, j, k, grid) = nothing -@inline Δyᵃᶠᵃ(i, j, k, grid) = nothing -@inline Δyᵃᶜᵃ(i, j, k, grid) = nothing - const ZRG = Union{LLGZ, RGZ, OSSGZ} @inline getspacing(k, Δz::AbstractVector) = @inbounds Δz[k] @@ -53,7 +48,7 @@ const ZRG = Union{LLGZ, RGZ, OSSGZ} @inline Δzᵃᵃᶜ(i, j, k, grid) = getspacing(k, grid.z.Δᶠ) # Convenience Functions for all grids -for LX in (:ᶜ, :ᶠ, :ᵃ), LY in (:ᶜ, :ᶠ, :ᵃ) +for LX in (:ᶜ, :ᶠ), LY in (:ᶜ, :ᶠ) x_spacing_1D = Symbol(:Δx, LX, :ᵃ, :ᵃ) x_spacing_2D = Symbol(:Δx, LX, LY, :ᵃ) diff --git a/src/Operators/variable_grid_operators.jl b/src/Operators/variable_grid_operators.jl index cc80db03f6..f08c73dc5e 100644 --- a/src/Operators/variable_grid_operators.jl +++ b/src/Operators/variable_grid_operators.jl @@ -37,15 +37,15 @@ const ZSRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:ZStarVerticalCoordina const ZSLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:ZStarVerticalCoordinate} const ZSOSG = OrthogonalSphericalShellGrid{<:Any, <:Any, <:Any, <:Any, <:ZStarVerticalCoordinate} -location(s::Symbol) = s == :ᶜ ? C() : F() +superscript_location(s::Symbol) = s == :ᶜ ? C() : F() -for Lx in (:ᶠ, :ᶜ), Lx in (:ᶠ, :ᶜ), Lx in (:ᶠ, :ᶜ) - zspacing = Symbol(:Δz, Lx, Ly, Lz) - rspacing = Symbol(:Δr, Lx, Ly, Lz) +for LX in (:ᶠ, :ᶜ), LY in (:ᶠ, :ᶜ), LZ in (:ᶠ, :ᶜ) + zspacing = Symbol(:Δz, LX, LY, LZ) + rspacing = Symbol(:Δr, LX, LY, LZ) - ℓx = location(Lx) - ℓy = location(Ly) - ℓz = location(Lz) + ℓx = superscript_location(LX) + ℓy = superscript_location(LY) + ℓz = superscript_location(LZ) @eval begin @inline $zspacing(i, j, k, grid::ZSRG) = $rspacing(i, j, k, grid) * e₃ⁿ(i, j, k, grid, ℓx, ℓy, ℓz) From d46cb797a1a5562b48a197ebb151c85bd9ac6604 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 25 Nov 2024 11:25:45 +0100 Subject: [PATCH 474/567] better --- .../distributed_grids.jl | 10 ++-- .../HydrostaticFreeSurfaceModels.jl | 1 - .../generalized_vertical_spacing.jl | 58 +------------------ .../hydrostatic_free_surface_ab2_step.jl | 29 +++++++++- ..._free_surface_tendency_kernel_functions.jl | 4 +- .../z_star_vertical_spacing.jl | 48 +++++++++++---- 6 files changed, 73 insertions(+), 77 deletions(-) diff --git a/src/DistributedComputations/distributed_grids.jl b/src/DistributedComputations/distributed_grids.jl index 7658a2485f..19446d4c99 100644 --- a/src/DistributedComputations/distributed_grids.jl +++ b/src/DistributedComputations/distributed_grids.jl @@ -13,11 +13,11 @@ import Oceananigans.Grids: RectilinearGrid, LatitudeLongitudeGrid, with_halo const DistributedGrid{FT, TX, TY, TZ} = AbstractGrid{FT, TX, TY, TZ, <:Distributed} -const DistributedRectilinearGrid{FT, TX, TY, TZ, FX, FY, FZ, VX, VY, VZ} = - RectilinearGrid{FT, TX, TY, TZ, FX, FY, FZ, VX, VY, VZ, <:Distributed} where {FT, TX, TY, TZ, FX, FY, FZ, VX, VY, VZ} +const DistributedRectilinearGrid{FT, TX, TY, TZ, CZ, FX, FY, VX, VY} = + RectilinearGrid{FT, TX, TY, TZ, CZ, FX, FY, VX, VY, <:Distributed} where {FT, TX, TY, TZ, CZ, FX, FY, VX, VY} -const DistributedLatitudeLongitudeGrid{FT, TX, TY, TZ, M, MY, FX, FY, FZ, VX, VY, VZ} = - LatitudeLongitudeGrid{FT, TX, TY, TZ, M, MY, FX, FY, FZ, VX, VY, VZ, <:Distributed} where {FT, TX, TY, TZ, M, MY, FX, FY, FZ, VX, VY, VZ} +const DistributedLatitudeLongitudeGrid{FT, TX, TY, TZ, CZ, M, MY, FX, FY, VX, VY} = + LatitudeLongitudeGrid{FT, TX, TY, TZ, CZ, M, MY, FX, FY, VX, VY, <:Distributed} where {FT, TX, TY, TZ, CZ, M, MY, FX, FY, VX, VY} # Local size from global size and architecture local_size(arch::Distributed, global_sz) = (local_size(global_sz[1], arch.partition.x, arch.local_index[1]), @@ -252,7 +252,7 @@ function reconstruct_global_grid(grid::DistributedLatitudeLongitudeGrid) Lλ, λᶠᵃᵃ, λᶜᵃᵃ, Δλᶠᵃᵃ, Δλᶜᵃᵃ = generate_coordinate(FT, TX(), Nλ, Hλ, λG, :longitude, child_arch) Lφ, φᵃᶠᵃ, φᵃᶜᵃ, Δφᵃᶠᵃ, Δφᵃᶜᵃ = generate_coordinate(FT, TY(), Nφ, Hφ, φG, :latitude, child_arch) Lz, z = generate_coordinate(FT, (TX, TY, TZ), (Nx, Ny, Nz), H, zG, :z, 3, child_arch) - + precompute_metrics = metrics_precomputed(grid) preliminary_grid = LatitudeLongitudeGrid{TX, TY, TZ}(child_arch, diff --git a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl index 9acb6d1e8f..72210df379 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl @@ -12,7 +12,6 @@ using KernelAbstractions.Extras.LoopInfo: @unroll using Oceananigans.Utils using Oceananigans.Utils: launch!, SumOfArrays using Oceananigans.Grids: AbstractGrid, rnode -using Oceananigans.ImmersedBoundaries: ImmersedZStarGrid using Oceananigans.TimeSteppers: SplitRungeKutta3TimeStepper, QuasiAdamsBashforth2TimeStepper using DocStringExtensions diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl index 1da202e069..8745e175b0 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl @@ -4,7 +4,6 @@ using Oceananigans.Operators using Oceananigans.BuoyancyModels: buoyancy_perturbationᶜᶜᶜ using Oceananigans.Grids: AbstractGrid, AbstractUnderlyingGrid, halo_size, with_halo using Oceananigans.ImmersedBoundaries -using Oceananigans.Models: AbstractVerticalCoordinateGrid using Oceananigans.Utils: getnamewrapper using Oceananigans.Operators: ∂t_e₃ using Adapt @@ -23,59 +22,4 @@ update_grid!(model, grid; parameters = :xy) = nothing ##### @inline grid_slope_contribution_x(i, j, k, grid, args...) = zero(grid) -@inline grid_slope_contribution_y(i, j, k, grid, args...) = zero(grid) - -##### -##### Tracer update in generalized vertical coordinates -##### We advance sθ but store θ once sⁿ⁺¹ is known -##### - -@kernel function _ab2_step_tracer_generalized_spacing!(θ, grid, Δt, χ, Gⁿ, G⁻) - i, j, k = @index(Global, NTuple) - - FT = eltype(χ) - C₁ = convert(FT, 1.5) + χ - C₂ = convert(FT, 0.5) + χ - - eⁿ = e₃ⁿ(i, j, k, grid, Center(), Center(), Center()) - e⁻ = e₃⁻(i, j, k, grid, Center(), Center(), Center()) - - @inbounds begin - ∂t_sθ = C₁ * eⁿ * Gⁿ[i, j, k] - C₂ * e⁻ * G⁻[i, j, k] - - # We store temporarily sθ in θ. the unscaled θ will be retrived later on with `unscale_tracers!` - θ[i, j, k] = eⁿ * θ[i, j, k] + convert(FT, Δt) * ∂t_sθ - end -end - -ab2_step_tracer_field!(tracer_field, grid::AbstractVerticalCoordinateGrid, Δt, χ, Gⁿ, G⁻) = - launch!(architecture(grid), grid, :xyz, _ab2_step_tracer_generalized_spacing!, - tracer_field, - grid, - Δt, χ, Gⁿ, G⁻) - -const EmptyTuples = Union{NamedTuple{(), Tuple{}}, Tuple{}} - -# Fallbacks -unscale_tracers!(tracers, grid; kwargs...) = nothing -unscale_tracers!(::EmptyTuples, ::AbstractVerticalCoordinateGrid; kwargs...) = nothing - -tracer_scaling_parameters(param::Symbol, tracers, grid) = KernelParameters((size(grid, 1), size(grid, 2), length(tracers)), (0, 0, 0)) -tracer_scaling_parameters(param::KernelParameters{S, O}, tracers, grid) where {S, O} = KernelParameters((S..., length(tracers)), (O..., 0)) - -function unscale_tracers!(tracers, grid::AbstractVerticalCoordinateGrid; parameters = :xy) - parameters = tracer_scaling_parameters(parameters, tracers, grid) - - launch!(architecture(grid), grid, parameters, _unscale_tracers!, tracers, grid, - Val(grid.Hz), Val(grid.Nz)) - - return nothing -end - -@kernel function _unscale_tracers!(tracers, grid, ::Val{Hz}, ::Val{Nz}) where {Hz, Nz} - i, j, n = @index(Global, NTuple) - - @unroll for k in -Hz+1:Nz+Hz - tracers[n][i, j, k] /= e₃ⁿ(i, j, k, grid, Center(), Center(), Center()) - end -end \ No newline at end of file +@inline grid_slope_contribution_y(i, j, k, grid, args...) = zero(grid) \ No newline at end of file diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl index 20a1f6ef0a..cb4cd8aaff 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl @@ -85,7 +85,7 @@ function ab2_step_tracers!(tracers, model, Δt, χ) tracer_field = tracers[tracer_name] closure = model.closure - ab2_step_tracer_field!(tracer_field, model.grid, Δt, χ, Gⁿ, G⁻) + ab2_step_tracer_field!(tracer_field, model.grid, Δt, χ, Gⁿ, G⁻) implicit_step!(tracer_field, model.timestepper.implicit_solver, @@ -101,6 +101,31 @@ function ab2_step_tracers!(tracers, model, Δt, χ) end ab2_step_tracer_field!(tracer_field, grid, Δt, χ, Gⁿ, G⁻) = - launch!(architecture(grid), grid, :xyz, ab2_step_field!, tracer_field, Δt, χ, Gⁿ, G⁻) + launch!(architecture(grid), grid, :xyz, _ab2_step_tracer_field!, tracer_field, Δt, χ, Gⁿ, G⁻) +##### +##### Tracer update in generalized vertical coordinates +##### We advance e₃θ but store θ once e₃ⁿ⁺¹ is known +##### + +@kernel function _ab2_step_tracer_field!(θ, grid, Δt, χ, Gⁿ, G⁻) + i, j, k = @index(Global, NTuple) + + FT = eltype(χ) + C₁ = convert(FT, 1.5) + χ + C₂ = convert(FT, 0.5) + χ + + eⁿ = e₃ⁿ(i, j, k, grid, Center(), Center(), Center()) + e⁻ = e₃⁻(i, j, k, grid, Center(), Center(), Center()) + + @inbounds begin + ∂t_sθ = C₁ * eⁿ * Gⁿ[i, j, k] - C₂ * e⁻ * G⁻[i, j, k] + + # We store temporarily sθ in θ. the unscaled θ will be retrived later on with `unscale_tracers!` + θ[i, j, k] = eⁿ * θ[i, j, k] + convert(FT, Δt) * ∂t_sθ + end +end +# Fallback! We need to unscale the tracers only in case of +# a grid with a moving vertical cocrdinate, i.e. where e₃ is not constant +unscale_tracers!(tracers, grid; kwargs...) = nothing diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_tendency_kernel_functions.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_tendency_kernel_functions.jl index 2eea388d1e..81fac3bf19 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_tendency_kernel_functions.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_tendency_kernel_functions.jl @@ -47,7 +47,7 @@ implicitly during time-stepping. - explicit_barotropic_pressure_x_gradient(i, j, k, grid, free_surface) - x_f_cross_U(i, j, k, grid, coriolis, velocities) - ∂xᶠᶜᶜ(i, j, k, grid, hydrostatic_pressure_anomaly) - - grid_slope_contribution_x(i, j, k, grid, free_surface, buoyancy, model_fields) + - grid_slope_contribution_x(i, j, k, grid, buoyancy, model_fields) - ∂ⱼ_τ₁ⱼ(i, j, k, grid, closure, diffusivities, clock, model_fields, buoyancy) - immersed_∂ⱼ_τ₁ⱼ(i, j, k, grid, velocities, u_immersed_bc, closure, diffusivities, clock, model_fields) + forcing(i, j, k, grid, clock, hydrostatic_prognostic_fields(velocities, free_surface, tracers))) @@ -87,7 +87,7 @@ implicitly during time-stepping. - explicit_barotropic_pressure_y_gradient(i, j, k, grid, free_surface) - y_f_cross_U(i, j, k, grid, coriolis, velocities) - ∂yᶜᶠᶜ(i, j, k, grid, hydrostatic_pressure_anomaly) - - grid_slope_contribution_y(i, j, k, grid, free_surface, buoyancy, model_fields) + - grid_slope_contribution_y(i, j, k, grid, buoyancy, model_fields) - ∂ⱼ_τ₂ⱼ(i, j, k, grid, closure, diffusivities, clock, model_fields, buoyancy) - immersed_∂ⱼ_τ₂ⱼ(i, j, k, grid, velocities, v_immersed_bc, closure, diffusivities, clock, model_fields) + forcing(i, j, k, grid, clock, model_fields)) diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 2c6bbdd301..69151b6e4e 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -1,5 +1,7 @@ using Oceananigans.Grids +using Oceananigans.ImmersedBoundaries: ZStarGridOfSomeKind + using Oceananigans.Models.HydrostaticFreeSurfaceModels.SplitExplicitFreeSurfaces: dynamic_column_depthᶜᶜᵃ, dynamic_column_depthᶜᶠᵃ, dynamic_column_depthᶠᶜᵃ, @@ -9,7 +11,7 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels.SplitExplicitFreeSurfaces ##### ZStar-specific vertical spacings update ##### -function update_grid!(model, grid::ZStarSpacingGrid; parameters = :xy) +function update_grid!(model, grid::ZStarGridOfSomeKind; parameters = :xy) # Scaling (just update once, they are the same for all the metrics) e₃ᶜᶜ⁻ = grid.z.e₃ᶜᶜ⁻ @@ -84,16 +86,42 @@ end ##### ZStar-specific implementation of the additional terms to be included in the momentum equations ##### -@inline z_minus_rᶜᶜᶜ(i, j, k, grid, η) = @inbounds η[i, j, grid.Nz+1] * (1 + rnode(i, j, k, grid, Center(), Center(), Center()) / static_column_depthᶜᶜᵃ(i, j, grid)) +@inline ∂x_z(i, j, k, grid) = @inbounds ∂xᶠᶜᶜ(i, j, k, grid, znode) +@inline ∂y_z(i, j, k, grid) = @inbounds ∂yᶜᶠᶜ(i, j, k, grid, znode) + +@inline grid_slope_contribution_x(i, j, k, grid::ZStarGridOfSomeKind, ::Nothing, model_fields) = zero(grid) +@inline grid_slope_contribution_y(i, j, k, grid::ZStarGridOfSomeKind, ::Nothing, model_fields) = zero(grid) + +@inline grid_slope_contribution_x(i, j, k, grid::ZStarGridOfSomeKind, buoyancy, model_fields) = + ℑxᶠᵃᵃ(i, j, k, grid, buoyancy_perturbationᶜᶜᶜ, buoyancy.model, model_fields) * ∂x_z(i, j, k, grid) + +@inline grid_slope_contribution_y(i, j, k, grid::ZStarGridOfSomeKind, buoyancy, model_fields) = + ℑyᵃᶠᵃ(i, j, k, grid, buoyancy_perturbationᶜᶜᶜ, buoyancy.model, model_fields) * ∂y_z(i, j, k, grid) + +#### +#### Removing the scaling of the vertical coordinate from the tracer fields +#### -@inline ∂x_z(i, j, k, grid, free_surface) = @inbounds ∂xᶠᶜᶜ(i, j, k, grid, z_minus_rᶜᶜᶜ, free_surface.η) -@inline ∂y_z(i, j, k, grid, free_surface) = @inbounds ∂yᶜᶠᶜ(i, j, k, grid, z_minus_rᶜᶜᶜ, free_surface.η) +const EmptyTuples = Union{NamedTuple{(), Tuple{}}, Tuple{}} -@inline grid_slope_contribution_x(i, j, k, grid::ZStarSpacingGrid, free_surface, ::Nothing, model_fields) = zero(grid) -@inline grid_slope_contribution_y(i, j, k, grid::ZStarSpacingGrid, free_surface, ::Nothing, model_fields) = zero(grid) +unscale_tracers!(::EmptyTuples, ::ZStarGridOfSomeKind; kwargs...) = nothing -@inline grid_slope_contribution_x(i, j, k, grid::ZStarSpacingGrid, free_surface, buoyancy, model_fields) = - ℑxᶠᵃᵃ(i, j, k, grid, buoyancy_perturbationᶜᶜᶜ, buoyancy.model, model_fields) * ∂x_z(i, j, k, grid, free_surface) +tracer_scaling_parameters(param::Symbol, tracers, grid) = KernelParameters((size(grid, 1), size(grid, 2), length(tracers)), (0, 0, 0)) +tracer_scaling_parameters(param::KernelParameters{S, O}, tracers, grid) where {S, O} = KernelParameters((S..., length(tracers)), (O..., 0)) -@inline grid_slope_contribution_y(i, j, k, grid::ZStarSpacingGrid, free_surface, buoyancy, model_fields) = - ℑyᵃᶠᵃ(i, j, k, grid, buoyancy_perturbationᶜᶜᶜ, buoyancy.model, model_fields) * ∂y_z(i, j, k, grid, free_surface) +function unscale_tracers!(tracers, grid::AbstractVerticalCoordinateGrid; parameters = :xy) + parameters = tracer_scaling_parameters(parameters, tracers, grid) + + launch!(architecture(grid), grid, parameters, _unscale_tracers!, tracers, grid, + Val(grid.Hz), Val(grid.Nz)) + + return nothing +end + +@kernel function _unscale_tracers!(tracers, grid, ::Val{Hz}, ::Val{Nz}) where {Hz, Nz} + i, j, n = @index(Global, NTuple) + + @unroll for k in -Hz+1:Nz+Hz + tracers[n][i, j, k] /= e₃ⁿ(i, j, k, grid, Center(), Center(), Center()) + end +end \ No newline at end of file From 3520d00831c698ac674cdcc1dc6290c9ef1019bd Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 25 Nov 2024 11:29:53 +0100 Subject: [PATCH 475/567] add new definitions --- .../SplitExplicitFreeSurfaces.jl | 16 +++++----------- .../z_star_vertical_spacing.jl | 2 +- src/Operators/variable_grid_operators.jl | 5 ----- src/OutputWriters/OutputWriters.jl | 1 - src/OutputWriters/output_writer_utils.jl | 10 ---------- 5 files changed, 6 insertions(+), 28 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl index 9466f25d2a..01807d4f14 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/SplitExplicitFreeSurfaces.jl @@ -15,18 +15,17 @@ using Oceananigans.ImmersedBoundaries using Oceananigans.Grids: AbstractGrid, topology using Oceananigans.ImmersedBoundaries: active_linear_index_to_tuple, mask_immersed_field! using Oceananigans.Models.HydrostaticFreeSurfaceModels: AbstractFreeSurface, - free_surface_displacement_field, - ZStarSpacingGrid + free_surface_displacement_field using Adapt using Base using KernelAbstractions: @index, @kernel using KernelAbstractions.Extras.LoopInfo: @unroll -import Oceananigans.Grids: dynamic_column_depthᶜᶜᵃ, - dynamic_column_depthᶜᶠᵃ, - dynamic_column_depthᶠᶜᵃ, - dynamic_column_depthᶠᶠᵃ +using Oceananigans.Grids: dynamic_column_depthᶜᶜᵃ, + dynamic_column_depthᶜᶠᵃ, + dynamic_column_depthᶠᶜᵃ, + dynamic_column_depthᶠᶠᵃ import Oceananigans.Models.HydrostaticFreeSurfaceModels: initialize_free_surface!, materialize_free_surface, @@ -35,11 +34,6 @@ import Oceananigans.Models.HydrostaticFreeSurfaceModels: initialize_free_surface explicit_barotropic_pressure_x_gradient, explicit_barotropic_pressure_y_gradient -@inline dynamic_column_depthᶜᶜᵃ(i, j, grid::ZStarSpacingGrid, η) = static_column_depthᶜᶜᵃ(i, j, grid) + @inbounds η[i, j, grid.Nz+1] -@inline dynamic_column_depthᶜᶠᵃ(i, j, grid::ZStarSpacingGrid, η) = static_column_depthᶜᶠᵃ(i, j, grid) + ℑxᶠᵃᵃ(i, j, grid.Nz+1, grid, η) -@inline dynamic_column_depthᶠᶜᵃ(i, j, grid::ZStarSpacingGrid, η) = static_column_depthᶠᶜᵃ(i, j, grid) + ℑyᵃᶠᵃ(i, j, grid.Nz+1, grid, η) -@inline dynamic_column_depthᶠᶠᵃ(i, j, grid::ZStarSpacingGrid, η) = static_column_depthᶠᶠᵃ(i, j, grid) + ℑxyᶠᶠᵃ(i, j, grid.Nz+1, grid, η) - include("split_explicit_timesteppers.jl") include("split_explicit_free_surface.jl") include("distributed_split_explicit_free_surface.jl") diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 69151b6e4e..fe70f05f59 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -109,7 +109,7 @@ unscale_tracers!(::EmptyTuples, ::ZStarGridOfSomeKind; kwargs...) = nothing tracer_scaling_parameters(param::Symbol, tracers, grid) = KernelParameters((size(grid, 1), size(grid, 2), length(tracers)), (0, 0, 0)) tracer_scaling_parameters(param::KernelParameters{S, O}, tracers, grid) where {S, O} = KernelParameters((S..., length(tracers)), (O..., 0)) -function unscale_tracers!(tracers, grid::AbstractVerticalCoordinateGrid; parameters = :xy) +function unscale_tracers!(tracers, grid::ZStarGridOfSomeKind; parameters = :xy) parameters = tracer_scaling_parameters(parameters, tracers, grid) launch!(architecture(grid), grid, parameters, _unscale_tracers!, tracers, grid, diff --git a/src/Operators/variable_grid_operators.jl b/src/Operators/variable_grid_operators.jl index f08c73dc5e..9663065ff9 100644 --- a/src/Operators/variable_grid_operators.jl +++ b/src/Operators/variable_grid_operators.jl @@ -1,10 +1,5 @@ import Oceananigans.Grids: znode, AbstractZStarGrid -import Oceananigans.Grids: dynamic_column_depthᶜᶜᵃ, - dynamic_column_depthᶜᶠᵃ, - dynamic_column_depthᶠᶜᵃ, - dynamic_column_depthᶠᶠᵃ - ##### ##### ZStar-specific vertical spacing functions ##### diff --git a/src/OutputWriters/OutputWriters.jl b/src/OutputWriters/OutputWriters.jl index a6b346d016..d1d12cb23f 100644 --- a/src/OutputWriters/OutputWriters.jl +++ b/src/OutputWriters/OutputWriters.jl @@ -16,7 +16,6 @@ using Oceananigans: AbstractOutputWriter using Oceananigans.Grids: interior_indices using Oceananigans.Utils: TimeInterval, IterationInterval, WallTimeInterval, instantiate using Oceananigans.Utils: pretty_filesize -using Oceananigans.Models: AbstractVerticalCoordinateGrid using OffsetArrays diff --git a/src/OutputWriters/output_writer_utils.jl b/src/OutputWriters/output_writer_utils.jl index 8ededd10a2..29c6526980 100644 --- a/src/OutputWriters/output_writer_utils.jl +++ b/src/OutputWriters/output_writer_utils.jl @@ -90,11 +90,6 @@ function saveproperty!(file, address, grid::DistributedGrid) _saveproperty!(file, address, on_architecture(cpu_arch, grid)) end -function saveproperty!(file, address, grid::AbstractVerticalCoordinateGrid) - static_grid = retrieve_static_grid(grid) - saveproperty!(file, address, static_grid) -end - # Special saveproperty! so boundary conditions are easily readable outside julia. function saveproperty!(file, address, bcs::FieldBoundaryConditions) for boundary in propertynames(bcs) @@ -136,11 +131,6 @@ function serializeproperty!(file, address, grid::DistributedGrid) file[address] = on_architecture(cpu_arch, grid) end -function serializeproperty!(file, address, grid::AbstractVerticalCoordinateGrid) - static_grid = retrieve_static_grid(grid) - serializeproperty!(file, address, static_grid) -end - function serializeproperty!(file, address, fbcs::FieldBoundaryConditions) # TODO: it'd be better to "filter" `FieldBoundaryCondition` and then serialize # rather than punting with `missing` instead. From 1863ef73a9895d8550ba6ccde34ad4ae5b5b4440 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 25 Nov 2024 11:33:09 +0100 Subject: [PATCH 476/567] should work? --- src/Grids/vertical_coordinate.jl | 2 +- src/MultiRegion/MultiRegion.jl | 2 +- src/OutputWriters/output_writer_utils.jl | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Grids/vertical_coordinate.jl b/src/Grids/vertical_coordinate.jl index e3f3669f9c..bdb8aa1bd0 100644 --- a/src/Grids/vertical_coordinate.jl +++ b/src/Grids/vertical_coordinate.jl @@ -100,7 +100,7 @@ Adapt.adapt_structure(to, coord::ZStarVerticalCoordinate) = @inline znode(i, j, k, grid, ℓx, ℓy, ℓz) = rnode(k, grid, ℓz) function validate_dimension_specification(T, ξ::ZStarVerticalCoordinate, dir, N, FT) - reference = validate_dimension_specification(T, ξ.reference, dir, N, FT) + reference = validate_dimension_specification(T, ξ.cᶠ, dir, N, FT) args = Tuple(getproperty(ξ, prop) for prop in propertynames(ξ)) return ZStarVerticalCoordinate(reference, args[2:end]...) diff --git a/src/MultiRegion/MultiRegion.jl b/src/MultiRegion/MultiRegion.jl index 4bb0d2e7eb..671eacc12e 100644 --- a/src/MultiRegion/MultiRegion.jl +++ b/src/MultiRegion/MultiRegion.jl @@ -35,7 +35,7 @@ import Oceananigans.Utils: _getregion, sync_all_devices! -abstract type AbstractMultiRegionGrid{FT, TX, TY, TZ, Arch} <: AbstractUnderlyingGrid{FT, TX, TY, TZ, Arch} end +abstract type AbstractMultiRegionGrid{FT, TX, TY, TZ, Arch} <: AbstractGrid{FT, TX, TY, TZ, Arch} end abstract type AbstractPartition end diff --git a/src/OutputWriters/output_writer_utils.jl b/src/OutputWriters/output_writer_utils.jl index 29c6526980..8c5dc6b596 100644 --- a/src/OutputWriters/output_writer_utils.jl +++ b/src/OutputWriters/output_writer_utils.jl @@ -1,7 +1,6 @@ using Oceananigans.DistributedComputations using StructArrays: StructArray, replace_storage using Oceananigans.Grids: on_architecture, architecture -using Oceananigans.Grids: retrieve_static_grid using Oceananigans.DistributedComputations: DistributedGrid, Partition using Oceananigans.Fields: AbstractField, indices, boundary_conditions, instantiated_location using Oceananigans.BoundaryConditions: bc_str, FieldBoundaryConditions, ContinuousBoundaryFunction, DiscreteBoundaryFunction From ca65b6bb30f5947096b90fe96d797982b804cda0 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 25 Nov 2024 11:44:27 +0100 Subject: [PATCH 477/567] This works --- src/Grids/grid_generation.jl | 4 ++-- src/Grids/latitude_longitude_grid.jl | 4 ++-- src/Grids/orthogonal_spherical_shell_grid.jl | 15 +++++++-------- src/Grids/rectilinear_grid.jl | 6 +++--- src/Grids/vertical_coordinate.jl | 10 +++++++++- src/Operators/variable_grid_operators.jl | 18 +++++++++--------- 6 files changed, 32 insertions(+), 25 deletions(-) diff --git a/src/Grids/grid_generation.jl b/src/Grids/grid_generation.jl index 178ad44403..bb8a188aed 100644 --- a/src/Grids/grid_generation.jl +++ b/src/Grids/grid_generation.jl @@ -174,9 +174,9 @@ function generate_coordinate(FT, topo, size, halo, coordinate::ZStarVerticalCoor throw(ArgumentError(msg)) end - r_faces = coordinate.reference + r_faces = coordinate.cᶠ - Lr, rᵃᵃᶠ, rᵃᵃᶜ, Δrᵃᵃᶠ, Δrᵃᵃᶜ = generate_coordinate(FT, topo[3](), Nz, Hz, r_faces, :z, arch) + Lr, rᵃᵃᶠ, rᵃᵃᶜ, Δrᵃᵃᶠ, Δrᵃᵃᶜ = generate_coordinate(FT, topo[3](), Nz, Hz, r_faces, :r, arch) args = (topo, (Nx, Ny, Nz), (Hx, Hy, Hz)) diff --git a/src/Grids/latitude_longitude_grid.jl b/src/Grids/latitude_longitude_grid.jl index ac1c61d5ab..642a2c4a3b 100644 --- a/src/Grids/latitude_longitude_grid.jl +++ b/src/Grids/latitude_longitude_grid.jl @@ -192,7 +192,7 @@ function LatitudeLongitudeGrid(architecture::AbstractArchitecture = CPU(), Lλ, λᶠᵃᵃ, λᶜᵃᵃ, Δλᶠᵃᵃ, Δλᶜᵃᵃ = generate_coordinate(FT, topology, size, halo, longitude, :longitude, 1, architecture) Lφ, φᵃᶠᵃ, φᵃᶜᵃ, Δφᵃᶠᵃ, Δφᵃᶜᵃ = generate_coordinate(FT, topology, size, halo, latitude, :latitude, 2, architecture) - Lz, zᵃᵃᶠ, zᵃᵃᶜ, Δzᵃᵃᶠ, Δzᵃᵃᶜ = generate_coordinate(FT, topology, size, halo, z, :z, 3, architecture) + Lz, z = generate_coordinate(FT, topology, size, halo, z, :z, 3, architecture) preliminary_grid = LatitudeLongitudeGrid{TX, TY, TZ}(architecture, Nλ, Nφ, Nz, @@ -200,7 +200,7 @@ function LatitudeLongitudeGrid(architecture::AbstractArchitecture = CPU(), Lλ, Lφ, Lz, Δλᶠᵃᵃ, Δλᶜᵃᵃ, λᶠᵃᵃ, λᶜᵃᵃ, Δφᵃᶠᵃ, Δφᵃᶜᵃ, φᵃᶠᵃ, φᵃᶜᵃ, - Δzᵃᵃᶠ, Δzᵃᵃᶜ, zᵃᵃᶠ, zᵃᵃᶜ, + z, (nothing for i=1:10)..., FT(radius)) if !precompute_metrics diff --git a/src/Grids/orthogonal_spherical_shell_grid.jl b/src/Grids/orthogonal_spherical_shell_grid.jl index 694d694333..c696f66fe7 100644 --- a/src/Grids/orthogonal_spherical_shell_grid.jl +++ b/src/Grids/orthogonal_spherical_shell_grid.jl @@ -64,25 +64,25 @@ struct OrthogonalSphericalShellGrid{FT, TX, TY, TZ, CZ, A, C, Arch} <: AbstractH Hx, Hy, Hz, Lz, λᶜᶜᵃ, λᶠᶜᵃ, λᶜᶠᵃ, λᶠᶠᵃ, - φᶜᶜᵃ, φᶠᶜᵃ, φᶜᶠᵃ, φᶠᶠᵃ, zᵃᵃᶜ, zᵃᵃᶠ, + φᶜᶜᵃ, φᶠᶜᵃ, φᶜᶠᵃ, φᶠᶠᵃ, z, Δxᶜᶜᵃ, Δxᶠᶜᵃ, Δxᶜᶠᵃ, Δxᶠᶠᵃ, - Δyᶜᶜᵃ, Δyᶜᶠᵃ, Δyᶠᶜᵃ, Δyᶠᶠᵃ, Δzᵃᵃᶜ, Δzᵃᵃᶠ, + Δyᶜᶜᵃ, Δyᶜᶠᵃ, Δyᶠᶜᵃ, Δyᶠᶠᵃ, Azᶜᶜᵃ, Azᶠᶜᵃ, Azᶜᶠᵃ, Azᶠᶠᵃ, radius, conformal_mapping) end const OSSG = OrthogonalSphericalShellGrid -const ZRegOSSG = OrthogonalSphericalShellGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Number} +const ZRegOSSG = OrthogonalSphericalShellGrid{<:Any, <:Any, <:Any, <:Any, <:RegularVerticalCoordinate} const ZRegOrthogonalSphericalShellGrid = ZRegOSSG const ConformalCubedSpherePanel = OrthogonalSphericalShellGrid{<:Any, FullyConnected, FullyConnected, <:Any, <:Any, <:Any, <:Any, <:CubedSphereConformalMapping} # convenience constructor for OSSG without any conformal_mapping properties OrthogonalSphericalShellGrid(architecture, Nx, Ny, Nz, Hx, Hy, Hz, Lz, - λᶜᶜᵃ, λᶠᶜᵃ, λᶜᶠᵃ, λᶠᶠᵃ, φᶜᶜᵃ, φᶠᶜᵃ, φᶜᶠᵃ, φᶠᶠᵃ, zᵃᵃᶜ, zᵃᵃᶠ, - Δxᶜᶜᵃ, Δxᶠᶜᵃ, Δxᶜᶠᵃ, Δxᶠᶠᵃ, Δyᶜᶜᵃ, Δyᶜᶠᵃ, Δyᶠᶜᵃ, Δyᶠᶠᵃ, Δzᵃᵃᶜ, Δzᵃᵃᶠ, + λᶜᶜᵃ, λᶠᶜᵃ, λᶜᶠᵃ, λᶠᶠᵃ, φᶜᶜᵃ, φᶠᶜᵃ, φᶜᶠᵃ, φᶠᶠᵃ, z, + Δxᶜᶜᵃ, Δxᶠᶜᵃ, Δxᶜᶠᵃ, Δxᶠᶠᵃ, Δyᶜᶜᵃ, Δyᶜᶠᵃ, Δyᶠᶜᵃ, Δyᶠᶠᵃ, Azᶜᶜᵃ, Azᶠᶜᵃ, Azᶜᶠᵃ, Azᶠᶠᵃ, radius) = OrthogonalSphericalShellGrid(architecture, Nx, Ny, Nz, Hx, Hy, Hz, Lz, - λᶜᶜᵃ, λᶠᶜᵃ, λᶜᶠᵃ, λᶠᶠᵃ, φᶜᶜᵃ, φᶠᶜᵃ, φᶜᶠᵃ, φᶠᶠᵃ, zᵃᵃᶜ, zᵃᵃᶠ, - Δxᶜᶜᵃ, Δxᶠᶜᵃ, Δxᶜᶠᵃ, Δxᶠᶠᵃ, Δyᶜᶜᵃ, Δyᶜᶠᵃ, Δyᶠᶜᵃ, Δyᶠᶠᵃ, Δzᵃᵃᶜ, Δzᵃᵃᶠ, + λᶜᶜᵃ, λᶠᶜᵃ, λᶜᶠᵃ, λᶠᶠᵃ, φᶜᶜᵃ, φᶠᶜᵃ, φᶜᶠᵃ, φᶠᶠᵃ, z, + Δxᶜᶜᵃ, Δxᶠᶜᵃ, Δxᶜᶠᵃ, Δxᶠᶠᵃ, Δyᶜᶜᵃ, Δyᶜᶠᵃ, Δyᶠᶜᵃ, Δyᶠᶠᵃ, Azᶜᶜᵃ, Azᶠᶜᵃ, Azᶜᶠᵃ, Azᶠᶠᵃ, radius, nothing) """ @@ -1156,7 +1156,6 @@ end # convenience @inline λnodes(grid::OSSG, ℓx, ℓy, ℓz; with_halos=false) = λnodes(grid, ℓx, ℓy; with_halos) @inline φnodes(grid::OSSG, ℓx, ℓy, ℓz; with_halos=false) = φnodes(grid, ℓx, ℓy; with_halos) -@inline rnodes(grid::OSSG, ℓx, ℓy, ℓz; with_halos=false) = rnodes(grid, ℓz ; with_halos) @inline xnodes(grid::OSSG, ℓx, ℓy, ℓz; with_halos=false) = xnodes(grid, ℓx, ℓy; with_halos) @inline ynodes(grid::OSSG, ℓx, ℓy, ℓz; with_halos=false) = ynodes(grid, ℓx, ℓy; with_halos) diff --git a/src/Grids/rectilinear_grid.jl b/src/Grids/rectilinear_grid.jl index e143ea7383..ec3dd0c684 100644 --- a/src/Grids/rectilinear_grid.jl +++ b/src/Grids/rectilinear_grid.jl @@ -269,7 +269,7 @@ function RectilinearGrid(architecture::AbstractArchitecture = CPU(), Lx, xᶠᵃᵃ, xᶜᵃᵃ, Δxᶠᵃᵃ, Δxᶜᵃᵃ = generate_coordinate(FT, topology, size, halo, x, :x, 1, architecture) Ly, yᵃᶠᵃ, yᵃᶜᵃ, Δyᵃᶠᵃ, Δyᵃᶜᵃ = generate_coordinate(FT, topology, size, halo, y, :y, 2, architecture) - Lz, zᵃᵃᶠ, zᵃᵃᶜ, Δzᵃᵃᶠ, Δzᵃᵃᶜ = generate_coordinate(FT, topology, size, halo, z, :z, 3, architecture) + Lz, z = generate_coordinate(FT, topology, size, halo, z, :z, 3, architecture) return RectilinearGrid{TX, TY, TZ}(architecture, Nx, Ny, Nz, @@ -277,7 +277,7 @@ function RectilinearGrid(architecture::AbstractArchitecture = CPU(), Lx, Ly, Lz, Δxᶠᵃᵃ, Δxᶜᵃᵃ, xᶠᵃᵃ, xᶜᵃᵃ, Δyᵃᶠᵃ, Δyᵃᶜᵃ, yᵃᶠᵃ, yᵃᶜᵃ, - Δzᵃᵃᶠ, Δzᵃᵃᶜ, zᵃᵃᶠ, zᵃᵃᶜ) + z) end """ Validate user input arguments to the `RectilinearGrid` constructor. """ @@ -298,7 +298,7 @@ end x_domain(grid::RectilinearGrid) = domain(topology(grid, 1)(), grid.Nx, grid.xᶠᵃᵃ) y_domain(grid::RectilinearGrid) = domain(topology(grid, 2)(), grid.Ny, grid.yᵃᶠᵃ) -z_domain(grid::RectilinearGrid) = domain(topology(grid, 3)(), grid.Nz, grid.zᵃᵃᶠ) +z_domain(grid::RectilinearGrid) = domain(topology(grid, 3)(), grid.Nz, grid.z.cᶠ) # architecture = CPU() default, assuming that a DataType positional arg # is specifying the floating point type. diff --git a/src/Grids/vertical_coordinate.jl b/src/Grids/vertical_coordinate.jl index bdb8aa1bd0..6d4cd73b7e 100644 --- a/src/Grids/vertical_coordinate.jl +++ b/src/Grids/vertical_coordinate.jl @@ -103,7 +103,7 @@ function validate_dimension_specification(T, ξ::ZStarVerticalCoordinate, dir, N reference = validate_dimension_specification(T, ξ.cᶠ, dir, N, FT) args = Tuple(getproperty(ξ, prop) for prop in propertynames(ξ)) - return ZStarVerticalCoordinate(reference, args[2:end]...) + return ZStarVerticalCoordinate(reference, reference, args[3:end]...) end # Summaries @@ -114,6 +114,14 @@ coordinate_summary(::Bounded, z::AbstractVerticalCoordinate, name) = #### Nodes and spacings... #### +@inline rnodes(grid, ℓz::F; with_halos=false) = _property(grid.z.cᶠ, ℓz, topology(grid, 3), size(grid, 3), with_halos) +@inline rnodes(grid, ℓz::C; with_halos=false) = _property(grid.z.cᶜ, ℓz, topology(grid, 3), size(grid, 3), with_halos) + +@inline rnodes(grid, ℓx, ℓy, ℓz; with_halos=false) = rnodes(grid, ℓz; with_halos) + +# Extended in the Operators module +@inline znodes(grid, ℓx, ℓy, ℓz; with_halos=false) = rnodes(grid, ℓz; with_halos) + # znodes(....) # rnodes(....) # zspacings(....) diff --git a/src/Operators/variable_grid_operators.jl b/src/Operators/variable_grid_operators.jl index 9663065ff9..713e6d26f3 100644 --- a/src/Operators/variable_grid_operators.jl +++ b/src/Operators/variable_grid_operators.jl @@ -13,13 +13,13 @@ const ZSG = AbstractZStarGrid @inline e₃ⁿ(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) @inline e₃⁻(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) -@inline e₃ⁿ(i, j, k, grid::ZSG, ::C, ::C, ℓz) = @inbounds grid.z.eᶜᶜⁿ[i, j, 1] -@inline e₃ⁿ(i, j, k, grid::ZSG, ::F, ::C, ℓz) = @inbounds grid.z.eᶠᶜⁿ[i, j, 1] -@inline e₃ⁿ(i, j, k, grid::ZSG, ::C, ::F, ℓz) = @inbounds grid.z.eᶜᶠⁿ[i, j, 1] -@inline e₃ⁿ(i, j, k, grid::ZSG, ::F, ::F, ℓz) = @inbounds grid.z.eᶠᶠⁿ[i, j, 1] +@inline e₃ⁿ(i, j, k, grid::ZSG, ::C, ::C, ℓz) = @inbounds grid.z.e₃ᶜᶜⁿ[i, j, 1] +@inline e₃ⁿ(i, j, k, grid::ZSG, ::F, ::C, ℓz) = @inbounds grid.z.e₃ᶠᶜⁿ[i, j, 1] +@inline e₃ⁿ(i, j, k, grid::ZSG, ::C, ::F, ℓz) = @inbounds grid.z.e₃ᶜᶠⁿ[i, j, 1] +@inline e₃ⁿ(i, j, k, grid::ZSG, ::F, ::F, ℓz) = @inbounds grid.z.e₃ᶠᶠⁿ[i, j, 1] # e₃⁻ is needed only at centers -@inline e₃⁻(i, j, k, grid::ZSG, ::C, ::C, ℓz) = @inbounds grid.z.eᶜᶜ⁻[i, j, 1] +@inline e₃⁻(i, j, k, grid::ZSG, ::C, ::C, ℓz) = @inbounds grid.z.e₃ᶜᶜ⁻[i, j, 1] @inline ∂t_e₃(i, j, k, grid) = zero(grid) @inline ∂t_e₃(i, j, k, grid::ZSG) = @inbounds grid.z.∂t_e₃[i, j, 1] @@ -32,7 +32,7 @@ const ZSRG = RectilinearGrid{<:Any, <:Any, <:Any, <:Any, <:ZStarVerticalCoordina const ZSLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:ZStarVerticalCoordinate} const ZSOSG = OrthogonalSphericalShellGrid{<:Any, <:Any, <:Any, <:Any, <:ZStarVerticalCoordinate} -superscript_location(s::Symbol) = s == :ᶜ ? C() : F() +superscript_location(s::Symbol) = s == :ᶜ ? :Center : :Face for LX in (:ᶠ, :ᶜ), LY in (:ᶠ, :ᶜ), LZ in (:ᶠ, :ᶜ) zspacing = Symbol(:Δz, LX, LY, LZ) @@ -43,9 +43,9 @@ for LX in (:ᶠ, :ᶜ), LY in (:ᶠ, :ᶜ), LZ in (:ᶠ, :ᶜ) ℓz = superscript_location(LZ) @eval begin - @inline $zspacing(i, j, k, grid::ZSRG) = $rspacing(i, j, k, grid) * e₃ⁿ(i, j, k, grid, ℓx, ℓy, ℓz) - @inline $zspacing(i, j, k, grid::ZSLLG) = $rspacing(i, j, k, grid) * e₃ⁿ(i, j, k, grid, ℓx, ℓy, ℓz) - @inline $zspacing(i, j, k, grid::ZSOSG) = $rspacing(i, j, k, grid) * e₃ⁿ(i, j, k, grid, ℓx, ℓy, ℓz) + @inline $zspacing(i, j, k, grid::ZSRG) = $rspacing(i, j, k, grid) * e₃ⁿ(i, j, k, grid, $ℓx(), $ℓy(), $ℓz()) + @inline $zspacing(i, j, k, grid::ZSLLG) = $rspacing(i, j, k, grid) * e₃ⁿ(i, j, k, grid, $ℓx(), $ℓy(), $ℓz()) + @inline $zspacing(i, j, k, grid::ZSOSG) = $rspacing(i, j, k, grid) * e₃ⁿ(i, j, k, grid, $ℓx(), $ℓy(), $ℓz()) end end From c15ebfd2f231132540fa57d840b855f85d54cb82 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 25 Nov 2024 11:57:15 +0100 Subject: [PATCH 478/567] This seems to work --- src/Grids/nodes_and_spacings.jl | 44 ------------------- src/Grids/vertical_coordinate.jl | 28 ++++++++---- .../hydrostatic_free_surface_ab2_step.jl | 2 +- .../z_star_vertical_spacing.jl | 4 +- src/Operators/variable_grid_operators.jl | 8 ++-- 5 files changed, 26 insertions(+), 60 deletions(-) diff --git a/src/Grids/nodes_and_spacings.jl b/src/Grids/nodes_and_spacings.jl index 29d6b3a16a..003385f18a 100644 --- a/src/Grids/nodes_and_spacings.jl +++ b/src/Grids/nodes_and_spacings.jl @@ -66,7 +66,6 @@ _node_names(grid, ::Nothing, ::Nothing, ::Nothing) = tuple() xnodes(grid, ::Nothing; kwargs...) = 1:1 ynodes(grid, ::Nothing; kwargs...) = 1:1 -znodes(grid, ::Nothing; kwargs...) = 1:1 """ xnodes(grid, ℓx, ℓy, ℓz, with_halos=false) @@ -88,36 +87,6 @@ See [`znodes`](@ref) for examples. """ @inline ynodes(grid, ℓx, ℓy, ℓz; kwargs...) = ynodes(grid, ℓy; kwargs...) -""" - znodes(grid, ℓx, ℓy, ℓz; with_halos=false) - -Return the positions over the interior nodes on `grid` in the ``z``-direction for the location `ℓx`, -`ℓy`, `ℓz`. For `Bounded` directions, `Face` nodes include the boundary points. - -```jldoctest znodes -julia> using Oceananigans - -julia> horz_periodic_grid = RectilinearGrid(size=(3, 3, 3), extent=(2π, 2π, 1), halo=(1, 1, 1), - topology=(Periodic, Periodic, Bounded)); - -julia> zC = znodes(horz_periodic_grid, Center()) -3-element view(OffsetArray(::StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}, 0:4), 1:3) with eltype Float64: - -0.8333333333333334 - -0.5 - -0.16666666666666666 - -julia> zC = znodes(horz_periodic_grid, Center(), Center(), Center()) -3-element view(OffsetArray(::StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}, 0:4), 1:3) with eltype Float64: - -0.8333333333333334 - -0.5 - -0.16666666666666666 - -julia> zC = znodes(horz_periodic_grid, Center(), Center(), Center(), with_halos=true) --1.1666666666666667:0.3333333333333333:0.16666666666666666 with indices 0:4 -``` -""" -@inline znodes(grid, ℓx, ℓy, ℓz; kwargs...) = znodes(grid, ℓz; kwargs...) - """ λnodes(grid::AbstractCurvilinearGrid, ℓx, ℓy, ℓz, with_halos=false) @@ -165,27 +134,14 @@ nodes(grid::AbstractGrid, (ℓx, ℓy, ℓz); reshape=false, with_halos=false) = # see Oceananigans/AbstractOperations/grid_metrics.jl for definitions function xspacing end function yspacing end -function zspacing end function λspacing end function φspacing end function xspacings end function yspacings end -function zspacings end function λspacings end function φspacings end - -""" - rspacings(grid, ℓx, ℓy, ℓz; with_halos=true) - -Return the "reference" spacings over the interior nodes on `grid` in the ``z``-direction for the location `ℓx`, -`ℓy`, `ℓz`. For `Bounded` directions, `Face` nodes include the boundary points. These are equal to the `zspacings` -for a _static_ grid. -""" -@inline rspacings(grid, ℓx, ℓy, ℓz; with_halos=true) = rspacings(grid, ℓz; with_halos) - - destantiate(::Face) = Face destantiate(::Center) = Center diff --git a/src/Grids/vertical_coordinate.jl b/src/Grids/vertical_coordinate.jl index 6d4cd73b7e..8e1ce1eba0 100644 --- a/src/Grids/vertical_coordinate.jl +++ b/src/Grids/vertical_coordinate.jl @@ -114,15 +114,25 @@ coordinate_summary(::Bounded, z::AbstractVerticalCoordinate, name) = #### Nodes and spacings... #### -@inline rnodes(grid, ℓz::F; with_halos=false) = _property(grid.z.cᶠ, ℓz, topology(grid, 3), size(grid, 3), with_halos) -@inline rnodes(grid, ℓz::C; with_halos=false) = _property(grid.z.cᶜ, ℓz, topology(grid, 3), size(grid, 3), with_halos) - +@inline rnodes(grid, ℓz::Face; with_halos=false) = _property(grid.z.cᶠ, ℓz, topology(grid, 3), size(grid, 3), with_halos) +@inline rnodes(grid, ℓz::Center; with_halos=false) = _property(grid.z.cᶜ, ℓz, topology(grid, 3), size(grid, 3), with_halos) @inline rnodes(grid, ℓx, ℓy, ℓz; with_halos=false) = rnodes(grid, ℓz; with_halos) -# Extended in the Operators module -@inline znodes(grid, ℓx, ℓy, ℓz; with_halos=false) = rnodes(grid, ℓz; with_halos) +rnodes(grid, ::Nothing; kwargs...) = 1:1 +znodes(grid, ::Nothing; kwargs...) = 1:1 + +# TODO: extend in the Operators module +@inline znodes(grid, ℓz; kwargs...) = rnodes(grid, ℓz; kwargs...) +@inline znodes(grid, ℓx, ℓy, ℓz; kwargs...) = rnodes(grid, ℓx, ℓy, ℓz; kwargs...) + +function zspacing end +function zspacings end + +""" + rspacings(grid, ℓx, ℓy, ℓz; with_halos=true) -# znodes(....) -# rnodes(....) -# zspacings(....) -# rspacings(....) \ No newline at end of file +Return the "reference" spacings over the interior nodes on `grid` in the ``z``-direction for the location `ℓx`, +`ℓy`, `ℓz`. For `Bounded` directions, `Face` nodes include the boundary points. These are equal to the `zspacings` +for a _static_ grid. +""" +@inline rspacings(grid, ℓx, ℓy, ℓz; with_halos=true) = rspacings(grid, ℓz; with_halos) diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl index cb4cd8aaff..7cc936db07 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl @@ -101,7 +101,7 @@ function ab2_step_tracers!(tracers, model, Δt, χ) end ab2_step_tracer_field!(tracer_field, grid, Δt, χ, Gⁿ, G⁻) = - launch!(architecture(grid), grid, :xyz, _ab2_step_tracer_field!, tracer_field, Δt, χ, Gⁿ, G⁻) + launch!(architecture(grid), grid, :xyz, _ab2_step_tracer_field!, tracer_field, grid, Δt, χ, Gⁿ, G⁻) ##### ##### Tracer update in generalized vertical coordinates diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index fe70f05f59..a7f6a65e94 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -86,8 +86,8 @@ end ##### ZStar-specific implementation of the additional terms to be included in the momentum equations ##### -@inline ∂x_z(i, j, k, grid) = @inbounds ∂xᶠᶜᶜ(i, j, k, grid, znode) -@inline ∂y_z(i, j, k, grid) = @inbounds ∂yᶜᶠᶜ(i, j, k, grid, znode) +@inline ∂x_z(i, j, k, grid) = @inbounds ∂xᶠᶜᶜ(i, j, k, grid, znode, Center(), Center(), Center()) +@inline ∂y_z(i, j, k, grid) = @inbounds ∂yᶜᶠᶜ(i, j, k, grid, znode, Center(), Center(), Center()) @inline grid_slope_contribution_x(i, j, k, grid::ZStarGridOfSomeKind, ::Nothing, model_fields) = zero(grid) @inline grid_slope_contribution_y(i, j, k, grid::ZStarGridOfSomeKind, ::Nothing, model_fields) = zero(grid) diff --git a/src/Operators/variable_grid_operators.jl b/src/Operators/variable_grid_operators.jl index 713e6d26f3..3785aa2d79 100644 --- a/src/Operators/variable_grid_operators.jl +++ b/src/Operators/variable_grid_operators.jl @@ -50,7 +50,7 @@ for LX in (:ᶠ, :ᶜ), LY in (:ᶠ, :ᶜ), LZ in (:ᶠ, :ᶜ) end # rnode for an AbstractZStarGrid grid is scaled -@inline znode(i, j, k, grid::ZSG, ::C, ::C, ℓz) = @inbounds rnode(i, j, k, grid, ℓx, ℓy, ℓz) * e₃ⁿ(i, j, k, grid, ℓx, ℓy, ℓz) + grid.z.ηⁿ[i, j, 1] -@inline znode(i, j, k, grid::ZSG, ::F, ::C, ℓz) = @inbounds rnode(i, j, k, grid, ℓx, ℓy, ℓz) * e₃ⁿ(i, j, k, grid, ℓx, ℓy, ℓz) + ℑxᶠᵃᵃ(i, j, 1, grid, grid.z.ηⁿ) -@inline znode(i, j, k, grid::ZSG, ::C, ::F, ℓz) = @inbounds rnode(i, j, k, grid, ℓx, ℓy, ℓz) * e₃ⁿ(i, j, k, grid, ℓx, ℓy, ℓz) + ℑyᵃᶠᵃ(i, j, 1, grid, grid.z.ηⁿ) -@inline znode(i, j, k, grid::ZSG, ::F, ::F, ℓz) = @inbounds rnode(i, j, k, grid, ℓx, ℓy, ℓz) * e₃ⁿ(i, j, k, grid, ℓx, ℓy, ℓz) + ℑxyᶠᶠᵃ(i, j, 1, grid, grid.z.ηⁿ) +@inline znode(i, j, k, grid::ZSG, ::C, ::C, ℓz) = @inbounds rnode(i, j, k, grid, C(), C(), ℓz) * e₃ⁿ(i, j, k, grid, C(), C(), ℓz) + grid.z.ηⁿ[i, j, 1] +@inline znode(i, j, k, grid::ZSG, ::F, ::C, ℓz) = @inbounds rnode(i, j, k, grid, F(), C(), ℓz) * e₃ⁿ(i, j, k, grid, F(), C(), ℓz) + ℑxᶠᵃᵃ(i, j, 1, grid, grid.z.ηⁿ) +@inline znode(i, j, k, grid::ZSG, ::C, ::F, ℓz) = @inbounds rnode(i, j, k, grid, C(), F(), ℓz) * e₃ⁿ(i, j, k, grid, C(), F(), ℓz) + ℑyᵃᶠᵃ(i, j, 1, grid, grid.z.ηⁿ) +@inline znode(i, j, k, grid::ZSG, ::F, ::F, ℓz) = @inbounds rnode(i, j, k, grid, F(), F(), ℓz) * e₃ⁿ(i, j, k, grid, F(), F(), ℓz) + ℑxyᶠᶠᵃ(i, j, 1, grid, grid.z.ηⁿ) From 9e05e21921affdf3c8bdeb5a1f58da926550e907 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 25 Nov 2024 12:08:03 +0100 Subject: [PATCH 479/567] make sure implicit operator works --- .../hydrostatic_free_surface_rk3_step.jl | 2 +- .../abstract_scalar_diffusivity_closure.jl | 1 - .../vertically_implicit_diffusion_solver.jl | 30 +++++++++++-------- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_rk3_step.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_rk3_step.jl index 0a02235f89..fecf8fa785 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_rk3_step.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_rk3_step.jl @@ -107,7 +107,7 @@ function rk3_substep_tracers!(tracers, model, Δt, γⁿ, ζⁿ) closure = model.closure launch!(architecture(grid), grid, :xyz, - _split_rk3_substep_field!, tracer_field, Δt, γⁿ, ζⁿ, Gⁿ, Ψ⁻) + _split_rk3_substep_tracer_field!, tracer_field, Δt, γⁿ, ζⁿ, Gⁿ, Ψ⁻) implicit_step!(tracer_field, model.timestepper.implicit_solver, diff --git a/src/TurbulenceClosures/abstract_scalar_diffusivity_closure.jl b/src/TurbulenceClosures/abstract_scalar_diffusivity_closure.jl index 01e322645d..b3d6f95222 100644 --- a/src/TurbulenceClosures/abstract_scalar_diffusivity_closure.jl +++ b/src/TurbulenceClosures/abstract_scalar_diffusivity_closure.jl @@ -259,7 +259,6 @@ end ##### Products of viscosity and stress, divergence, vorticity ##### - @inline κ_σᶠᶜᶜ(i, j, k, grid, closure, K, id, clock, fields, σᶠᶜᶜ, args...) = κᶠᶜᶜ(i, j, k, grid, closure, K, id, clock, fields) * σᶠᶜᶜ(i, j, k, grid, args...) @inline κ_σᶜᶠᶜ(i, j, k, grid, closure, K, id, clock, fields, σᶜᶠᶜ, args...) = κᶜᶠᶜ(i, j, k, grid, closure, K, id, clock, fields) * σᶜᶠᶜ(i, j, k, grid, args...) @inline κ_σᶜᶜᶠ(i, j, k, grid, closure, K, id, clock, fields, σᶜᶜᶠ, args...) = κᶜᶜᶠ(i, j, k, grid, closure, K, id, clock, fields) * σᶜᶜᶠ(i, j, k, grid, args...) diff --git a/src/TurbulenceClosures/vertically_implicit_diffusion_solver.jl b/src/TurbulenceClosures/vertically_implicit_diffusion_solver.jl index 04582652ee..1e8637a56f 100644 --- a/src/TurbulenceClosures/vertically_implicit_diffusion_solver.jl +++ b/src/TurbulenceClosures/vertically_implicit_diffusion_solver.jl @@ -45,13 +45,19 @@ implicit_diffusion_solver(::ExplicitTimeDiscretization, args...; kwargs...) = no const c = Center() const f = Face() +# The vertical spacing used here is Δz for velocities and Δr for tracers, since the +# implicit solver operator is applied to the scaled tracer e₃θ instead of just θ + +@inline vertical_spacing(i, j, k, grid, ℓx, ℓy, ℓz) = Δz(i, j, k, grid, ℓx, ℓy, ℓz) +@inline vertical_spacing(i, j, k, grid, ::Center, ::Center, ℓz) = Δr(i, j, k, grid, c, c, ℓz) + # Tracers and horizontal velocities at cell centers in z @inline function ivd_upper_diagonal(i, j, k, grid, closure, K, id, ℓx, ℓy, ::Center, clock, Δt, κz) closure_ij = getclosure(i, j, closure) κᵏ⁺¹ = κz(i, j, k+1, grid, closure_ij, K, id, clock) - Δzᶜₖ = Δz(i, j, k, grid, ℓx, ℓy, c) - Δzᶠₖ₊₁ = Δz(i, j, k+1, grid, ℓx, ℓy, f) - du = - Δt * κᵏ⁺¹ / (Δzᶜₖ * Δzᶠₖ₊₁) + Δzᶜₖ = vertical_spacing(i, j, k, grid, ℓx, ℓy, c) + Δzᶠₖ₊₁ = vertical_spacing(i, j, k+1, grid, ℓx, ℓy, f) + du = - Δt * κᵏ⁺¹ / (Δrᶜₖ * Δrᶠₖ₊₁) # This conditional ensures the diagonal is correct return ifelse(k > grid.Nz-1, zero(grid), du) @@ -61,9 +67,9 @@ end k = k′ + 1 # Shift index to match LinearAlgebra.Tridiagonal indexing convenction closure_ij = getclosure(i, j, closure) κᵏ = κz(i, j, k, grid, closure_ij, K, id, clock) - Δzᶜₖ = Δz(i, j, k, grid, ℓx, ℓy, c) - Δzᶠₖ = Δz(i, j, k, grid, ℓx, ℓy, f) - dl = - Δt * κᵏ / (Δzᶜₖ * Δzᶠₖ) + Δrᶜₖ = vertical_spacing(i, j, k, grid, ℓx, ℓy, c) + Δrᶠₖ = vertical_spacing(i, j, k, grid, ℓx, ℓy, f) + dl = - Δt * κᵏ / (Δrᶜₖ * Δrᶠₖ) # This conditional ensures the diagonal is correct: the lower diagonal does not # exist for k′ = 0. (Note we use LinearAlgebra.Tridiagonal indexing convention, @@ -80,9 +86,9 @@ end @inline function ivd_upper_diagonal(i, j, k, grid, closure, K, id, ℓx, ℓy, ::Face, clock, Δt, νzᶜᶜᶜ) closure_ij = getclosure(i, j, closure) νᵏ = νzᶜᶜᶜ(i, j, k, grid, closure_ij, K, clock) - Δzᶜₖ = Δz(i, j, k, grid, ℓx, ℓy, c) - Δzᶠₖ = Δz(i, j, k, grid, ℓx, ℓy, f) - du = - Δt * νᵏ / (Δzᶜₖ * Δzᶠₖ) + Δrᶜₖ = vertical_spacing(i, j, k, grid, ℓx, ℓy, c) + Δrᶠₖ = vertical_spacing(i, j, k, grid, ℓx, ℓy, f) + du = - Δt * νᵏ / (Δrᶜₖ * Δrᶠₖ) return ifelse(k < 1, zero(grid), du) end @@ -90,9 +96,9 @@ end k′ = k + 2 # Shift to adjust for Tridiagonal indexing convention closure_ij = getclosure(i, j, closure) νᵏ⁻¹ = νzᶜᶜᶜ(i, j, k′-1, grid, closure_ij, K, clock) - Δzᶜₖ = Δz(i, j, k′, grid, ℓx, ℓy, c) - Δzᶠₖ₋₁ = Δz(i, j, k′-1, grid, ℓx, ℓy, f) - dl = - Δt * νᵏ⁻¹ / (Δzᶜₖ * Δzᶠₖ₋₁) + Δrᶜₖ = vertical_spacing(i, j, k′, grid, ℓx, ℓy, c) + Δrᶠₖ₋₁ = vertical_spacing(i, j, k′-1, grid, ℓx, ℓy, f) + dl = - Δt * νᵏ⁻¹ / (Δrᶜₖ * Δrᶠₖ₋₁) return ifelse(k < 1, zero(grid), dl) end From e32956a1299de8024d51fafbb0e6449ae6d99a23 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 25 Nov 2024 12:39:26 +0100 Subject: [PATCH 480/567] do not change this --- .buildkite/distributed/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/distributed/pipeline.yml b/.buildkite/distributed/pipeline.yml index 7bfc3b4aac..9f7869de63 100644 --- a/.buildkite/distributed/pipeline.yml +++ b/.buildkite/distributed/pipeline.yml @@ -8,7 +8,7 @@ env: OPENBLAS_NUM_THREADS: 1 JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager JULIA_NUM_PRECOMPILE_TASKS: 8 - JULIA_NUM_THREADS: 1 + JULIA_NUM_THREADS: 8 OMPI_MCA_opal_warn_on_missing_libcuda: 0 steps: From 8262e16e8a2976aad1ce7e3def3de4f9f023fab8 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 25 Nov 2024 12:40:02 +0100 Subject: [PATCH 481/567] changes --- src/BoundaryConditions/fill_halo_regions.jl | 6 +-- .../fill_halo_regions_flux.jl | 37 ------------------- .../compute_w_from_continuity.jl | 4 +- 3 files changed, 5 insertions(+), 42 deletions(-) diff --git a/src/BoundaryConditions/fill_halo_regions.jl b/src/BoundaryConditions/fill_halo_regions.jl index 3467ac4124..7ac3e79ea5 100644 --- a/src/BoundaryConditions/fill_halo_regions.jl +++ b/src/BoundaryConditions/fill_halo_regions.jl @@ -344,9 +344,9 @@ end ##### Calculate kernel size and offset for Windowed and Sliced Fields ##### -const WEB = Union{typeof(fill_west_and_east_halo!), typeof(fill_west_halo!), typeof(fill_east_halo!)} -const SNB = Union{typeof(fill_south_and_north_halo!), typeof(fill_south_halo!), typeof(fill_north_halo!)} -const TBB = Union{typeof(fill_bottom_and_top_halo!), typeof(fill_bottom_halo!), typeof(fill_top_halo!)} +const WEB = Union{typeof(fill_west_and_east_halo!), typeof(fill_west_halo!), typeof(fill_east_halo!)} +const SNB = Union{typeof(fill_south_and_north_halo!), typeof(fill_south_halo!), typeof(fill_north_halo!)} +const TBB = Union{typeof(fill_bottom_and_top_halo!), typeof(fill_bottom_halo!), typeof(fill_top_halo!)} # Tupled halo filling _only_ deals with full fields! @inline fill_halo_size(::Tuple, ::WEB, args...) = :yz diff --git a/src/BoundaryConditions/fill_halo_regions_flux.jl b/src/BoundaryConditions/fill_halo_regions_flux.jl index 13ec79d9e6..08fe4f2ce4 100644 --- a/src/BoundaryConditions/fill_halo_regions_flux.jl +++ b/src/BoundaryConditions/fill_halo_regions_flux.jl @@ -32,40 +32,3 @@ @inline _fill_bottom_halo!(i, j, grid, c, ::FBC, args...) = _fill_flux_bottom_halo!(i, j, 1, grid, c) @inline _fill_top_halo!(i, j, grid, c, ::FBC, args...) = _fill_flux_top_halo!(i, j, 1, grid, c) - -# @inline function _fill_west_halo!(j, k, grid, c, ::FBC, args...) -# for i in 1:grid.Hx -# _fill_flux_west_halo!(i, j, k, grid, c) -# end -# end - -# @inline function _fill_east_halo!(j, k, grid, c, ::FBC, args...) -# for i in 1:grid.Hx -# _fill_flux_east_halo!(i, j, k, grid, c) -# end -# end - -# @inline function _fill_south_halo!(i, k, grid, c, ::FBC, args...) -# for j in 1:grid.Hz -# _fill_flux_south_halo!(i, j, k, grid, c) -# end -# end - -# @inline function _fill_north_halo!(i, k, grid, c, ::FBC, args...) -# for j in 1:grid.Hz -# _fill_flux_north_halo!(i, j, k, grid, c) -# end -# end - -# @inline function _fill_bottom_halo!(i, j, grid, c, ::FBC, args...) -# for k in 1:grid.Hz -# _fill_flux_bottom_halo!(i, j, k, grid, c) -# end -# end - -# @inline function _fill_top_halo!(i, j, grid, c, ::FBC, args...) -# for k in 1:grid.Hz -# _fill_flux_top_halo!(i, j, k, grid, c) -# end -# end - diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl index fa013b9425..073855b6aa 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl @@ -25,10 +25,10 @@ compute_w_from_continuity!(velocities, arch, grid; parameters = w_kernel_paramet @inbounds U.w[i, j, 1] = 0 for k in 2:grid.Nz+1 δh_u = flux_div_xyᶜᶜᶜ(i, j, k-1, grid, U.u, U.v) / Azᶜᶜᶜ(i, j, k-1, grid) - ∂t_s = Δrᶜᶜᶜ(i, j, k-1, grid) * ∂t_e₃(i, j, k-1, grid) + ∂te₃ = Δrᶜᶜᶜ(i, j, k-1, grid) * ∂t_e₃(i, j, k-1, grid) immersed = immersed_cell(i, j, k-1, grid) - Δw = δh_u + ifelse(immersed, 0, ∂t_s) # We do not account for grid changes in immersed cells + Δw = δh_u + ifelse(immersed, 0, ∂te₃) # We do not account for grid changes in immersed cells @inbounds U.w[i, j, k] = U.w[i, j, k-1] - Δw end From eeff71e2dc7bd631d50f13ea08bc224b06484bbc Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 25 Nov 2024 13:12:29 +0100 Subject: [PATCH 482/567] some changes --- src/AbstractOperations/conditional_operations.jl | 3 ++- src/Fields/field_boundary_buffers.jl | 1 + src/Fields/regridding_fields.jl | 2 +- src/ImmersedBoundaries/ImmersedBoundaries.jl | 3 --- .../HydrostaticFreeSurfaceModels.jl | 12 ++---------- .../hydrostatic_free_surface_rk3_step.jl | 2 +- .../prescribed_hydrostatic_velocity_fields.jl | 7 +++---- 7 files changed, 10 insertions(+), 20 deletions(-) diff --git a/src/AbstractOperations/conditional_operations.jl b/src/AbstractOperations/conditional_operations.jl index 890592993f..7eba3699f7 100644 --- a/src/AbstractOperations/conditional_operations.jl +++ b/src/AbstractOperations/conditional_operations.jl @@ -1,7 +1,8 @@ using Oceananigans.Fields: OneField using Oceananigans.Grids: architecture + +import Oceananigans.Architectures: on_architecture import Oceananigans.Fields: condition_operand, conditional_length, set!, compute_at!, indices -import Oceananigans.Architectures: architecture, on_architecture # For conditional reductions such as mean(u * v, condition = u .> 0)) struct ConditionalOperation{LX, LY, LZ, O, F, G, C, M, T} <: AbstractOperation{LX, LY, LZ, G, T} diff --git a/src/Fields/field_boundary_buffers.jl b/src/Fields/field_boundary_buffers.jl index f4bd99fcb8..e514a3be3e 100644 --- a/src/Fields/field_boundary_buffers.jl +++ b/src/Fields/field_boundary_buffers.jl @@ -1,4 +1,5 @@ using Oceananigans.BoundaryConditions: MCBC, DCBC +using Oceananigans.Architectures: on_architecture using Oceananigans.Grids: halo_size, size using Oceananigans.Utils: launch! using KernelAbstractions: @kernel, @index diff --git a/src/Fields/regridding_fields.jl b/src/Fields/regridding_fields.jl index 4ab24aa591..758257f369 100644 --- a/src/Fields/regridding_fields.jl +++ b/src/Fields/regridding_fields.jl @@ -1,6 +1,6 @@ using KernelAbstractions: @kernel, @index -using Oceananigans.Architectures: architecture +using Oceananigans.Architectures: on_architecture, architecture using Oceananigans.Operators: Δzᶜᶜᶜ, Δyᶜᶜᶜ, Δxᶜᶜᶜ, Azᶜᶜᶜ using Oceananigans.Grids: hack_sind, ξnode, ηnode, rnode diff --git a/src/ImmersedBoundaries/ImmersedBoundaries.jl b/src/ImmersedBoundaries/ImmersedBoundaries.jl index f0908bfacb..2b45f4af9a 100644 --- a/src/ImmersedBoundaries/ImmersedBoundaries.jl +++ b/src/ImmersedBoundaries/ImmersedBoundaries.jl @@ -45,7 +45,4 @@ include("conditional_differences.jl") include("mask_immersed_field.jl") include("immersed_reductions.jl") -# Extension to Immersed boundaries with zstar -include("zstar_immersed_grid.jl") - end # module diff --git a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl index 72210df379..e3e63fa736 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl @@ -3,16 +3,14 @@ module HydrostaticFreeSurfaceModels export HydrostaticFreeSurfaceModel, ExplicitFreeSurface, ImplicitFreeSurface, SplitExplicitFreeSurface, - PrescribedVelocityFields, - ZStar + PrescribedVelocityFields using KernelAbstractions: @index, @kernel using KernelAbstractions.Extras.LoopInfo: @unroll using Oceananigans.Utils using Oceananigans.Utils: launch!, SumOfArrays -using Oceananigans.Grids: AbstractGrid, rnode -using Oceananigans.TimeSteppers: SplitRungeKutta3TimeStepper, QuasiAdamsBashforth2TimeStepper +using Oceananigans.Grids: AbstractGrid using DocStringExtensions @@ -36,9 +34,6 @@ free_surface_displacement_field(velocities, ::Nothing, grid) = nothing # free surface initialization functions initialize_free_surface!(free_surface, grid, velocities) = nothing -# ZStar implementation -include("generalized_vertical_spacing.jl") - include("compute_w_from_continuity.jl") # No free surface @@ -65,8 +60,6 @@ include("hydrostatic_free_surface_model.jl") include("show_hydrostatic_free_surface_model.jl") include("set_hydrostatic_free_surface_model.jl") -include("z_star_vertical_spacing.jl") - ##### ##### AbstractModel interface ##### @@ -139,7 +132,6 @@ include("compute_hydrostatic_free_surface_tendencies.jl") include("compute_hydrostatic_free_surface_buffers.jl") include("update_hydrostatic_free_surface_model_state.jl") include("hydrostatic_free_surface_ab2_step.jl") -include("hydrostatic_free_surface_rk3_step.jl") include("store_hydrostatic_free_surface_tendencies.jl") include("prescribed_hydrostatic_velocity_fields.jl") include("single_column_model_mode.jl") diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_rk3_step.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_rk3_step.jl index fecf8fa785..0a02235f89 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_rk3_step.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_rk3_step.jl @@ -107,7 +107,7 @@ function rk3_substep_tracers!(tracers, model, Δt, γⁿ, ζⁿ) closure = model.closure launch!(architecture(grid), grid, :xyz, - _split_rk3_substep_tracer_field!, tracer_field, Δt, γⁿ, ζⁿ, Gⁿ, Ψ⁻) + _split_rk3_substep_field!, tracer_field, Δt, γⁿ, ζⁿ, Gⁿ, Ψ⁻) implicit_step!(tracer_field, model.timestepper.implicit_solver, diff --git a/src/Models/HydrostaticFreeSurfaceModels/prescribed_hydrostatic_velocity_fields.jl b/src/Models/HydrostaticFreeSurfaceModels/prescribed_hydrostatic_velocity_fields.jl index 2f145b73b5..94403dc62c 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/prescribed_hydrostatic_velocity_fields.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/prescribed_hydrostatic_velocity_fields.jl @@ -74,7 +74,7 @@ end function HydrostaticFreeSurfaceTendencyFields(::PrescribedVelocityFields, free_surface, grid, tracer_names) tracer_tendencies = TracerFields(tracer_names, grid) - momentum_tendencies = (u = nothing, v = nothing) + momentum_tendencies = (u = nothing, v = nothing, η = nothing) return merge(momentum_tendencies, tracer_tendencies) end @@ -89,8 +89,7 @@ end @inline datatuple(obj::PrescribedVelocityFields) = (; u = datatuple(obj.u), v = datatuple(obj.v), w = datatuple(obj.w)) ab2_step_velocities!(::PrescribedVelocityFields, args...) = nothing -rk3_substep_velocities!(::PrescribedVelocityFields, args...) = nothing -step_free_surface!(::Nothing, model, timestepper, Δt) = nothing +ab2_step_free_surface!(::Nothing, model, Δt, χ) = nothing compute_w_from_continuity!(::PrescribedVelocityFields, args...; kwargs...) = nothing validate_velocity_boundary_conditions(grid, ::PrescribedVelocityFields) = nothing @@ -131,5 +130,5 @@ function time_step!(model::OnlyParticleTrackingModel, Δt; callbacks = [], kwarg update_state!(model, callbacks) end -update_state!(model::OnlyParticleTrackingModel, callbacks) = +update_state!(model::OnlyParticleTrackingModel, callbacks) = [callback(model) for callback in callbacks if callback.callsite isa UpdateStateCallsite] From df33f47d505d857858630a82275f20d216acd4b8 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 25 Nov 2024 13:12:35 +0100 Subject: [PATCH 483/567] more changes --- src/Grids/abstract_grid.jl | 3 +- src/Grids/grid_utils.jl | 1 + .../HydrostaticFreeSurfaceModels.jl | 3 + .../step_split_explicit_free_surface.jl | 3 +- .../generalized_vertical_spacing.jl | 25 -------- .../z_star_vertical_spacing.jl | 62 +++++++++++-------- 6 files changed, 44 insertions(+), 53 deletions(-) delete mode 100644 src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl diff --git a/src/Grids/abstract_grid.jl b/src/Grids/abstract_grid.jl index aa67be1621..4a2398cb4f 100644 --- a/src/Grids/abstract_grid.jl +++ b/src/Grids/abstract_grid.jl @@ -1,8 +1,7 @@ """ AbstractGrid{FT, TX, TY, TZ} -Abstract supertype for grids with elements of type `FT`, -topology `{TX, TY, TZ}` and vertical coordinate `CZ`. +Abstract supertype for grids with elements of type `FT` and topology `{TX, TY, TZ}`. """ abstract type AbstractGrid{FT, TX, TY, TZ, Arch} end diff --git a/src/Grids/grid_utils.jl b/src/Grids/grid_utils.jl index d629d5f873..5fa238683a 100644 --- a/src/Grids/grid_utils.jl +++ b/src/Grids/grid_utils.jl @@ -327,6 +327,7 @@ coordinate_summary(topo, Δ::Union{AbstractVector, AbstractMatrix}, name) = @inline static_column_depthᶠᶜᵃ(i, j, grid) = grid.Lz @inline static_column_depthᶠᶠᵃ(i, j, grid) = grid.Lz +# Will be extended in the `ImmersedBoundaries` module for any ZStar grid type @inline dynamic_column_depthᶜᶜᵃ(i, j, grid, η) = static_column_depthᶜᶜᵃ(i, j, grid) @inline dynamic_column_depthᶜᶠᵃ(i, j, grid, η) = static_column_depthᶜᶠᵃ(i, j, grid) @inline dynamic_column_depthᶠᶜᵃ(i, j, grid, η) = static_column_depthᶠᶜᵃ(i, j, grid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl index e3e63fa736..9e4b398cbb 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl @@ -60,6 +60,9 @@ include("hydrostatic_free_surface_model.jl") include("show_hydrostatic_free_surface_model.jl") include("set_hydrostatic_free_surface_model.jl") +# ZStar implementation +include("z_star_vertical_spacing.jl") + ##### ##### AbstractModel interface ##### diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/step_split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/step_split_explicit_free_surface.jl index 9eaa5231bb..aefcd89f8f 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/step_split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/step_split_explicit_free_surface.jl @@ -164,7 +164,8 @@ function step_free_surface!(free_surface::SplitExplicitFreeSurface, model, baroc mask_immersed_field!(model.velocities.v) end - # Needed for ZStar + # Needed for ZStar to compute the barotropic correction? + # Can we remove it? fill_halo_regions!(η) return nothing diff --git a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl deleted file mode 100644 index 8745e175b0..0000000000 --- a/src/Models/HydrostaticFreeSurfaceModels/generalized_vertical_spacing.jl +++ /dev/null @@ -1,25 +0,0 @@ -using Oceananigans -using Oceananigans.Grids -using Oceananigans.Operators -using Oceananigans.BuoyancyModels: buoyancy_perturbationᶜᶜᶜ -using Oceananigans.Grids: AbstractGrid, AbstractUnderlyingGrid, halo_size, with_halo -using Oceananigans.ImmersedBoundaries -using Oceananigans.Utils: getnamewrapper -using Oceananigans.Operators: ∂t_e₃ -using Adapt -using Printf - -import Oceananigans.Architectures: arch_array - -##### -##### General implementation -##### - -update_grid!(model, grid; parameters = :xy) = nothing - -##### -##### Additional terms to be included in the momentum equations (fallbacks) -##### - -@inline grid_slope_contribution_x(i, j, k, grid, args...) = zero(grid) -@inline grid_slope_contribution_y(i, j, k, grid, args...) = zero(grid) \ No newline at end of file diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index a7f6a65e94..4ef23351a0 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -1,16 +1,18 @@ using Oceananigans.Grids - using Oceananigans.ImmersedBoundaries: ZStarGridOfSomeKind -using Oceananigans.Models.HydrostaticFreeSurfaceModels.SplitExplicitFreeSurfaces: dynamic_column_depthᶜᶜᵃ, - dynamic_column_depthᶜᶠᵃ, - dynamic_column_depthᶠᶜᵃ, - dynamic_column_depthᶠᶠᵃ +using Oceananigans.Models.HydrostaticFreeSurfaceModels.SplitExplicitFreeSurfaces: compute_barotropic_mode! ##### ##### ZStar-specific vertical spacings update ##### +# The easy case +retrieve_barotropic_velocity(model, free_surface::SplitExplicitFreeSurface) = free_surface.barotropic_velocities + +# Fallback +update_grid!(model, grid::ZStarGridOfSomeKind) = nothing + function update_grid!(model, grid::ZStarGridOfSomeKind; parameters = :xy) # Scaling (just update once, they are the same for all the metrics) @@ -21,26 +23,25 @@ function update_grid!(model, grid::ZStarGridOfSomeKind; parameters = :xy) e₃ᶠᶠⁿ = grid.z.e₃ᶠᶠⁿ ∂t_e₃ = grid.z.∂t_e₃ ηⁿ = grid.z.ηⁿ + η = model.free_surface.η - # Free surface variables: - # TODO: At the moment only SplitExplicitFreeSurface is supported, - # but zstar can be extended to other free surface solvers by calculating - # the barotropic velocity in this step - U = model.free_surface.barotropic_velocities.U - V = model.free_surface.barotropic_velocities.V - η = model.free_surface.η + launch!(architecture(grid), grid, parameters, _update_grid!, + e₃ᶜᶜ⁻, e₃ᶜᶜⁿ, e₃ᶠᶜⁿ, e₃ᶜᶠⁿ, e₃ᶠᶠⁿ, e₃ᶜᶜ⁻, ηⁿ, grid, η) + + # the barotropic velocity are retrieved from the free surface model for a + # SplitExplicitFreeSurface and are calculated for other free surface models + # For the moment only the `SplitExplicitFreeSurface` is supported + # TODO: find a way to support other free surface models by storing the barotropic velocities shomewhere + U, V = retrieve_barotropic_velocity(model, free_surface) # Update vertical spacing with available parameters # No need to fill the halo as the scaling is updated _IN_ the halos - launch!(architecture(grid), grid, parameters, _update_zstar!, - e₃ᶜᶜⁿ, e₃ᶠᶜⁿ, e₃ᶜᶠⁿ, e₃ᶠᶠⁿ, e₃ᶜᶜ⁻, ηⁿ, ∂t_e₃, grid, η, U, V) + launch!(architecture(grid), grid, parameters, _update_grid_vertical_velocity!, ∂t_e₃, grid, U, V) return nothing end -# NOTE: The ZStar vertical spacing only supports a SplitExplicitFreeSurface -# TODO: extend to support other free surface solvers -@kernel function _update_zstar!(e₃ᶜᶜⁿ, e₃ᶠᶜⁿ, e₃ᶜᶠⁿ, e₃ᶠᶠⁿ, e₃ᶜᶜ⁻, ηⁿ, ∂t_e₃, grid, η, U, V) +@kernel function _update_grid!(e₃ᶜᶜⁿ, e₃ᶠᶜⁿ, e₃ᶜᶠⁿ, e₃ᶠᶠⁿ, e₃ᶜᶜ⁻, ηⁿ, grid, η) i, j = @index(Global, NTuple) kᴺ = size(grid, 3) @@ -71,27 +72,38 @@ end # Update η in the grid ηⁿ[i, j, 1] = η[i, j, kᴺ+1] + end +end - # ∂(η / H)/∂t = - ∇ ⋅ ∫udz / H - δx_U = δxᶜᶜᶜ(i, j, kᴺ, grid, Δy_qᶠᶜᶜ, U) - δy_V = δyᶜᶜᶜ(i, j, kᴺ, grid, Δx_qᶜᶠᶜ, V) +@kernel function _update_grid_vertical_velocity!(∂t_e₃, grid, U, V) + i, j = @index(Global, NTuple) + kᴺ = size(grid, 3) - δh_U = (δx_U + δy_V) / Azᶜᶜᶜ(i, j, kᴺ, grid) + hᶜᶜ = static_column_depthᶜᶜᵃ(i, j, grid) - ∂t_e₃[i, j, 1] = ifelse(hᶜᶜ == 0, zero(grid), - δh_U / hᶜᶜ) - end + # ∂(η / H)/∂t = - ∇ ⋅ ∫udz / H + δx_U = δxᶜᶜᶜ(i, j, kᴺ, grid, Δy_qᶠᶜᶜ, U) + δy_V = δyᶜᶜᶜ(i, j, kᴺ, grid, Δx_qᶜᶠᶜ, V) + + δh_U = (δx_U + δy_V) / Azᶜᶜᶜ(i, j, kᴺ, grid) + + @inbounds ∂t_e₃[i, j, 1] = ifelse(hᶜᶜ == 0, zero(grid), - δh_U / hᶜᶜ) end ##### ##### ZStar-specific implementation of the additional terms to be included in the momentum equations ##### -@inline ∂x_z(i, j, k, grid) = @inbounds ∂xᶠᶜᶜ(i, j, k, grid, znode, Center(), Center(), Center()) -@inline ∂y_z(i, j, k, grid) = @inbounds ∂yᶜᶠᶜ(i, j, k, grid, znode, Center(), Center(), Center()) +# Fallbacks +@inline grid_slope_contribution_x(i, j, k, grid, buoyancy, model_fields) = zero(grid) +@inline grid_slope_contribution_y(i, j, k, grid, buoyancy, model_fields) = zero(grid) @inline grid_slope_contribution_x(i, j, k, grid::ZStarGridOfSomeKind, ::Nothing, model_fields) = zero(grid) @inline grid_slope_contribution_y(i, j, k, grid::ZStarGridOfSomeKind, ::Nothing, model_fields) = zero(grid) +@inline ∂x_z(i, j, k, grid) = @inbounds ∂xᶠᶜᶜ(i, j, k, grid, znode, Center(), Center(), Center()) +@inline ∂y_z(i, j, k, grid) = @inbounds ∂yᶜᶠᶜ(i, j, k, grid, znode, Center(), Center(), Center()) + @inline grid_slope_contribution_x(i, j, k, grid::ZStarGridOfSomeKind, buoyancy, model_fields) = ℑxᶠᵃᵃ(i, j, k, grid, buoyancy_perturbationᶜᶜᶜ, buoyancy.model, model_fields) * ∂x_z(i, j, k, grid) From 0555e28c3690bc0825ef9fe9b6e8b5f81cdbaf45 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 25 Nov 2024 13:19:40 +0100 Subject: [PATCH 484/567] more changes --- .../update_nonhydrostatic_model_state.jl | 2 +- .../update_shallow_water_state.jl | 2 +- .../multi_region_boundary_conditions.jl | 3 +-- src/MultiRegion/multi_region_grid.jl | 2 +- .../spacings_and_areas_and_volumes.jl | 4 ++-- src/Operators/variable_grid_operators.jl | 2 +- .../vertically_implicit_diffusion_solver.jl | 20 +++++++++---------- src/Utils/kernel_launching.jl | 1 - 8 files changed, 17 insertions(+), 19 deletions(-) diff --git a/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl b/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl index 37eb1fa4e3..705ca507ac 100644 --- a/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl +++ b/src/Models/NonhydrostaticModels/update_nonhydrostatic_model_state.jl @@ -11,7 +11,7 @@ using Oceananigans.Models: update_model_field_time_series! import Oceananigans.TimeSteppers: update_state! """ - update_state!(model::NonhydrostaticModel, Δt, callbacks=[]) + update_state!(model::NonhydrostaticModel, callbacks=[]) Update peripheral aspects of the model (halo regions, diffusivities, hydrostatic pressure) to the current model state. If `callbacks` are provided (in an array), diff --git a/src/Models/ShallowWaterModels/update_shallow_water_state.jl b/src/Models/ShallowWaterModels/update_shallow_water_state.jl index bb4fb8d06d..51d9267264 100644 --- a/src/Models/ShallowWaterModels/update_shallow_water_state.jl +++ b/src/Models/ShallowWaterModels/update_shallow_water_state.jl @@ -18,7 +18,7 @@ Next, `callbacks` are executed. Finally, tendencies are computed if `compute_tendencies=true`. """ -function update_state!(model::ShallowWaterModel, callbacks=[]; compute_tendencies = true) +function update_state!(model::ShallowWaterModel, callbacks=[]; compute_tendencies=true) # Mask immersed fields foreach(mask_immersed_field!, merge(model.solution, model.tracers)) diff --git a/src/MultiRegion/multi_region_boundary_conditions.jl b/src/MultiRegion/multi_region_boundary_conditions.jl index 8ea7b5639c..92342a1106 100644 --- a/src/MultiRegion/multi_region_boundary_conditions.jl +++ b/src/MultiRegion/multi_region_boundary_conditions.jl @@ -1,5 +1,5 @@ using Oceananigans: instantiated_location -using Oceananigans.Architectures: device_copy_to! +using Oceananigans.Architectures: on_architecture, device_copy_to! using Oceananigans.Operators: assumed_field_location using Oceananigans.Fields: reduced_dimensions using Oceananigans.DistributedComputations: communication_side @@ -16,7 +16,6 @@ using Oceananigans.BoundaryConditions: fill_open_boundary_regions! import Oceananigans.Fields: tupled_fill_halo_regions!, boundary_conditions, data, fill_send_buffers! -import Oceananigans.Architectures: on_architecture import Oceananigans.BoundaryConditions: fill_halo_regions!, diff --git a/src/MultiRegion/multi_region_grid.jl b/src/MultiRegion/multi_region_grid.jl index 0b73fb84a6..264b6a29be 100644 --- a/src/MultiRegion/multi_region_grid.jl +++ b/src/MultiRegion/multi_region_grid.jl @@ -1,4 +1,4 @@ -using Oceananigans.Grids: metrics_precomputed, pop_flat_elements, grid_name +using Oceananigans.Grids: metrics_precomputed, on_architecture, pop_flat_elements, grid_name using Oceananigans.ImmersedBoundaries: GridFittedBottom, PartialCellBottom, GridFittedBoundary import Oceananigans.Grids: architecture, size, new_data, halo_size diff --git a/src/Operators/spacings_and_areas_and_volumes.jl b/src/Operators/spacings_and_areas_and_volumes.jl index 9464ab6033..f8af81628b 100644 --- a/src/Operators/spacings_and_areas_and_volumes.jl +++ b/src/Operators/spacings_and_areas_and_volumes.jl @@ -44,8 +44,8 @@ const ZRG = Union{LLGZ, RGZ, OSSGZ} @inline Δrᵃᵃᶜ(i, j, k, grid) = getspacing(k, grid.z.Δᶜ) @inline Δrᵃᵃᶠ(i, j, k, grid) = getspacing(k, grid.z.Δᶠ) -@inline Δzᵃᵃᶠ(i, j, k, grid) = getspacing(k, grid.z.Δᶜ) -@inline Δzᵃᵃᶜ(i, j, k, grid) = getspacing(k, grid.z.Δᶠ) +@inline Δzᵃᵃᶜ(i, j, k, grid) = getspacing(k, grid.z.Δᶜ) +@inline Δzᵃᵃᶠ(i, j, k, grid) = getspacing(k, grid.z.Δᶠ) # Convenience Functions for all grids for LX in (:ᶜ, :ᶠ), LY in (:ᶜ, :ᶠ) diff --git a/src/Operators/variable_grid_operators.jl b/src/Operators/variable_grid_operators.jl index 3785aa2d79..24c3eda599 100644 --- a/src/Operators/variable_grid_operators.jl +++ b/src/Operators/variable_grid_operators.jl @@ -49,7 +49,7 @@ for LX in (:ᶠ, :ᶜ), LY in (:ᶠ, :ᶜ), LZ in (:ᶠ, :ᶜ) end end -# rnode for an AbstractZStarGrid grid is scaled +# znode for an AbstractZStarGrid grid is scaled by the free surface @inline znode(i, j, k, grid::ZSG, ::C, ::C, ℓz) = @inbounds rnode(i, j, k, grid, C(), C(), ℓz) * e₃ⁿ(i, j, k, grid, C(), C(), ℓz) + grid.z.ηⁿ[i, j, 1] @inline znode(i, j, k, grid::ZSG, ::F, ::C, ℓz) = @inbounds rnode(i, j, k, grid, F(), C(), ℓz) * e₃ⁿ(i, j, k, grid, F(), C(), ℓz) + ℑxᶠᵃᵃ(i, j, 1, grid, grid.z.ηⁿ) @inline znode(i, j, k, grid::ZSG, ::C, ::F, ℓz) = @inbounds rnode(i, j, k, grid, C(), F(), ℓz) * e₃ⁿ(i, j, k, grid, C(), F(), ℓz) + ℑyᵃᶠᵃ(i, j, 1, grid, grid.z.ηⁿ) diff --git a/src/TurbulenceClosures/vertically_implicit_diffusion_solver.jl b/src/TurbulenceClosures/vertically_implicit_diffusion_solver.jl index 1e8637a56f..6272398057 100644 --- a/src/TurbulenceClosures/vertically_implicit_diffusion_solver.jl +++ b/src/TurbulenceClosures/vertically_implicit_diffusion_solver.jl @@ -57,7 +57,7 @@ const f = Face() κᵏ⁺¹ = κz(i, j, k+1, grid, closure_ij, K, id, clock) Δzᶜₖ = vertical_spacing(i, j, k, grid, ℓx, ℓy, c) Δzᶠₖ₊₁ = vertical_spacing(i, j, k+1, grid, ℓx, ℓy, f) - du = - Δt * κᵏ⁺¹ / (Δrᶜₖ * Δrᶠₖ₊₁) + du = - Δt * κᵏ⁺¹ / (Δzᶜₖ * Δzᶠₖ₊₁) # This conditional ensures the diagonal is correct return ifelse(k > grid.Nz-1, zero(grid), du) @@ -67,9 +67,9 @@ end k = k′ + 1 # Shift index to match LinearAlgebra.Tridiagonal indexing convenction closure_ij = getclosure(i, j, closure) κᵏ = κz(i, j, k, grid, closure_ij, K, id, clock) - Δrᶜₖ = vertical_spacing(i, j, k, grid, ℓx, ℓy, c) - Δrᶠₖ = vertical_spacing(i, j, k, grid, ℓx, ℓy, f) - dl = - Δt * κᵏ / (Δrᶜₖ * Δrᶠₖ) + Δzᶜₖ = vertical_spacing(i, j, k, grid, ℓx, ℓy, c) + Δzᶠₖ = vertical_spacing(i, j, k, grid, ℓx, ℓy, f) + dl = - Δt * κᵏ / (Δzᶜₖ * Δzᶠₖ) # This conditional ensures the diagonal is correct: the lower diagonal does not # exist for k′ = 0. (Note we use LinearAlgebra.Tridiagonal indexing convention, @@ -86,9 +86,9 @@ end @inline function ivd_upper_diagonal(i, j, k, grid, closure, K, id, ℓx, ℓy, ::Face, clock, Δt, νzᶜᶜᶜ) closure_ij = getclosure(i, j, closure) νᵏ = νzᶜᶜᶜ(i, j, k, grid, closure_ij, K, clock) - Δrᶜₖ = vertical_spacing(i, j, k, grid, ℓx, ℓy, c) - Δrᶠₖ = vertical_spacing(i, j, k, grid, ℓx, ℓy, f) - du = - Δt * νᵏ / (Δrᶜₖ * Δrᶠₖ) + Δzᶜₖ = vertical_spacing(i, j, k, grid, ℓx, ℓy, c) + Δzᶠₖ = vertical_spacing(i, j, k, grid, ℓx, ℓy, f) + du = - Δt * νᵏ / (Δzᶜₖ * Δzᶠₖ) return ifelse(k < 1, zero(grid), du) end @@ -96,9 +96,9 @@ end k′ = k + 2 # Shift to adjust for Tridiagonal indexing convention closure_ij = getclosure(i, j, closure) νᵏ⁻¹ = νzᶜᶜᶜ(i, j, k′-1, grid, closure_ij, K, clock) - Δrᶜₖ = vertical_spacing(i, j, k′, grid, ℓx, ℓy, c) - Δrᶠₖ₋₁ = vertical_spacing(i, j, k′-1, grid, ℓx, ℓy, f) - dl = - Δt * νᵏ⁻¹ / (Δrᶜₖ * Δrᶠₖ₋₁) + Δzᶜₖ = vertical_spacing(i, j, k′, grid, ℓx, ℓy, c) + Δzᶠₖ₋₁ = vertical_spacing(i, j, k′-1, grid, ℓx, ℓy, f) + dl = - Δt * νᵏ⁻¹ / (Δzᶜₖ * Δzᶠₖ₋₁) return ifelse(k < 1, zero(grid), dl) end diff --git a/src/Utils/kernel_launching.jl b/src/Utils/kernel_launching.jl index cc7f1378f1..097ce8fe00 100644 --- a/src/Utils/kernel_launching.jl +++ b/src/Utils/kernel_launching.jl @@ -2,7 +2,6 @@ ##### Utilities for launching kernels ##### -using Oceananigans using Oceananigans: location using Oceananigans.Architectures using Oceananigans.Grids From 662c78abe759e0e8d87397bc849a81f65c6217d8 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 25 Nov 2024 13:28:59 +0100 Subject: [PATCH 485/567] add some more tests --- test/test_zstar_coordinate.jl | 45 ++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index 5c0205a1de..df6ada0eee 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -37,8 +37,51 @@ function info_message(grid) return msg1 * msg2 * msg3 * msg4 * msg5 end -@testset "ZStar coordinate testset" begin +const C = Center +@testset "ZStar coordinate scaling tests" begin + @info "testing the ZStar coordinate scalings" + + z = ZStarVerticalCoordinate(-20, 0) + + grid = RectilinearGrid(size = (2, 1, 20), + x = (0, 2), + y = (0, 1), + z = z, + topology = (Periodic, Periodic, Bounded)) + + grid = ImmersedBoundaryGrid(grid, GridFittedBottom((x, y) -> -10)) + + model = HydrostaticFreeSurfaceModel(grid, + free_surface = SplitExplicitFreeSurface(grid; substeps = 20)) + + @test znode(1, 1, 21, grid, C(), C(), F()) == 0 + @test dynamic_column_depthᶜᶜᵃ(1, 1, grid) == 10 + @test static_column_depthᶜᶜᵃ(1, 1, grid) == 10 + + set!(model, η = [1, 2]) + set!(model, u = (x, y, z) -> x) + + initialize!(model) + update_state!(model) + + @test e₃ⁿ(1, 1, 1, grid, C(), C(), C()) == 11 / 10 + @test e₃ⁿ(2, 1, 1, grid, C(), C(), C()) == 12 / 10 + @test e₃⁻(1, 1, 1, grid, C(), C(), C()) == 1 + @test e₃⁻(2, 1, 1, grid, C(), C(), C()) == 1 + + @test znode(1, 1, 21, grid, C(), C(), C()) == 1 + @test znode(2, 1, 21, grid, C(), C(), C()) == 2 + @test rnode(1, 1, 21, grid, C(), C(), F()) == 0 + @test dynamic_column_depthᶜᶜᵃ(1, 1, grid) == 11 + @test dynamic_column_depthᶜᶜᵃ(2, 1, grid) == 12 + @test static_column_depthᶜᶜᵃ(1, 1, grid) == 10 + @test static_column_depthᶜᶜᵃ(2, 1, grid) == 10 + + @test ∂t_e₃(1, 1, 1, grid) == 1 / 10 +end + +@testset "ZStar coordinate simulation testset" begin z_uniform = ZStarVerticalCoordinate(-20, 0) z_stretched = ZStarVerticalCoordinate(collect(-20:0)) topologies = ((Periodic, Periodic, Bounded), From 6b836517b3519c9ceed261d40b94147cf30b39a8 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 25 Nov 2024 13:29:45 +0100 Subject: [PATCH 486/567] better --- test/test_zstar_coordinate.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index df6ada0eee..0a4af3f517 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -48,7 +48,7 @@ const C = Center x = (0, 2), y = (0, 1), z = z, - topology = (Periodic, Periodic, Bounded)) + topology = (Bounded, Periodic, Bounded)) grid = ImmersedBoundaryGrid(grid, GridFittedBottom((x, y) -> -10)) From 1ef6eefd78be95ba0edf785dc2ea079f00f17293 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 25 Nov 2024 13:42:50 +0100 Subject: [PATCH 487/567] make it compile --- .../HydrostaticFreeSurfaceModels.jl | 2 ++ test/runtests.jl | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl index 9e4b398cbb..262af463cd 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl @@ -19,6 +19,8 @@ import Oceananigans.Advection: cell_advection_timescale import Oceananigans.TimeSteppers: step_lagrangian_particles! import Oceananigans.Architectures: on_architecture +using Oceananigans.TimeSteppers: SplitRungeKutta3TimeStepper, QuasiAdamsBashforth2TimeStepper + abstract type AbstractFreeSurface{E, G} end # This is only used by the cubed sphere for now. diff --git a/test/runtests.jl b/test/runtests.jl index bb88600416..0146132dbe 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -28,6 +28,8 @@ CUDA.allowscalar() do # Initialization steps if group == :init || group == :all + Pkg.instantiate(; verbose=true) + Pkg.precompile(; strict=true) Pkg.status() try @@ -35,10 +37,11 @@ CUDA.allowscalar() do catch; end try + CUDA.precompile_runtime() CUDA.versioninfo() catch; end end - + # Core Oceananigans if group == :unit || group == :all @testset "Unit tests" begin From 1cf932378b660356628fd345e200c8c8bf6ca917 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 25 Nov 2024 13:53:05 +0100 Subject: [PATCH 488/567] make sure adapts work --- src/Grids/latitude_longitude_grid.jl | 5 +---- src/Grids/orthogonal_spherical_shell_grid.jl | 5 +---- src/Grids/rectilinear_grid.jl | 5 +---- src/ImmersedBoundaries/ImmersedBoundaries.jl | 1 + .../HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl | 4 ++-- 5 files changed, 6 insertions(+), 14 deletions(-) diff --git a/src/Grids/latitude_longitude_grid.jl b/src/Grids/latitude_longitude_grid.jl index 642a2c4a3b..a05f215a2f 100644 --- a/src/Grids/latitude_longitude_grid.jl +++ b/src/Grids/latitude_longitude_grid.jl @@ -403,10 +403,7 @@ function Adapt.adapt_structure(to, grid::LatitudeLongitudeGrid) Adapt.adapt(to, grid.Δφᵃᶜᵃ), Adapt.adapt(to, grid.φᵃᶠᵃ), Adapt.adapt(to, grid.φᵃᶜᵃ), - Adapt.adapt(to, grid.Δzᵃᵃᶠ), - Adapt.adapt(to, grid.Δzᵃᵃᶜ), - Adapt.adapt(to, grid.zᵃᵃᶠ), - Adapt.adapt(to, grid.zᵃᵃᶜ), + Adapt.adapt(to, grid.z), Adapt.adapt(to, grid.Δxᶠᶜᵃ), Adapt.adapt(to, grid.Δxᶜᶠᵃ), Adapt.adapt(to, grid.Δxᶠᶠᵃ), diff --git a/src/Grids/orthogonal_spherical_shell_grid.jl b/src/Grids/orthogonal_spherical_shell_grid.jl index c696f66fe7..a748379ed6 100644 --- a/src/Grids/orthogonal_spherical_shell_grid.jl +++ b/src/Grids/orthogonal_spherical_shell_grid.jl @@ -958,8 +958,7 @@ function Adapt.adapt_structure(to, grid::OrthogonalSphericalShellGrid) adapt(to, grid.φᶠᶜᵃ), adapt(to, grid.φᶜᶠᵃ), adapt(to, grid.φᶠᶠᵃ), - adapt(to, grid.zᵃᵃᶜ), - adapt(to, grid.zᵃᵃᶠ), + adapt(to, grid.z), adapt(to, grid.Δxᶜᶜᵃ), adapt(to, grid.Δxᶠᶜᵃ), adapt(to, grid.Δxᶜᶠᵃ), @@ -968,8 +967,6 @@ function Adapt.adapt_structure(to, grid::OrthogonalSphericalShellGrid) adapt(to, grid.Δyᶜᶠᵃ), adapt(to, grid.Δyᶠᶜᵃ), adapt(to, grid.Δyᶠᶠᵃ), - adapt(to, grid.Δzᵃᵃᶜ), - adapt(to, grid.Δzᵃᵃᶠ), adapt(to, grid.Azᶜᶜᵃ), adapt(to, grid.Azᶠᶜᵃ), adapt(to, grid.Azᶜᶠᵃ), diff --git a/src/Grids/rectilinear_grid.jl b/src/Grids/rectilinear_grid.jl index ec3dd0c684..ac4dea3df1 100644 --- a/src/Grids/rectilinear_grid.jl +++ b/src/Grids/rectilinear_grid.jl @@ -357,10 +357,7 @@ function Adapt.adapt_structure(to, grid::RectilinearGrid) Adapt.adapt(to, grid.Δyᵃᶜᵃ), Adapt.adapt(to, grid.yᵃᶠᵃ), Adapt.adapt(to, grid.yᵃᶜᵃ), - Adapt.adapt(to, grid.Δzᵃᵃᶠ), - Adapt.adapt(to, grid.Δzᵃᵃᶜ), - Adapt.adapt(to, grid.zᵃᵃᶠ), - Adapt.adapt(to, grid.zᵃᵃᶜ)) + Adapt.adapt(to, grid.z)) end cpu_face_constructor_x(grid::XRegularRG) = x_domain(grid) diff --git a/src/ImmersedBoundaries/ImmersedBoundaries.jl b/src/ImmersedBoundaries/ImmersedBoundaries.jl index 2b45f4af9a..de714053e2 100644 --- a/src/ImmersedBoundaries/ImmersedBoundaries.jl +++ b/src/ImmersedBoundaries/ImmersedBoundaries.jl @@ -44,5 +44,6 @@ include("immersed_boundary_condition.jl") include("conditional_differences.jl") include("mask_immersed_field.jl") include("immersed_reductions.jl") +include("zstar_immersed_grid.jl") end # module diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 4ef23351a0..a81a5abff9 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -11,11 +11,11 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels.SplitExplicitFreeSurfaces retrieve_barotropic_velocity(model, free_surface::SplitExplicitFreeSurface) = free_surface.barotropic_velocities # Fallback -update_grid!(model, grid::ZStarGridOfSomeKind) = nothing +update_grid!(model, grid; parameters) = nothing function update_grid!(model, grid::ZStarGridOfSomeKind; parameters = :xy) - # Scaling (just update once, they are the same for all the metrics) + # Scalings and free surface e₃ᶜᶜ⁻ = grid.z.e₃ᶜᶜ⁻ e₃ᶜᶜⁿ = grid.z.e₃ᶜᶜⁿ e₃ᶠᶜⁿ = grid.z.e₃ᶠᶜⁿ From 91c811cc8bd243e995919f27863b8c1d792be77b Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 25 Nov 2024 13:59:40 +0100 Subject: [PATCH 489/567] some more corrections --- src/Grids/grid_utils.jl | 1 - src/Grids/latitude_longitude_grid.jl | 2 -- src/Grids/orthogonal_spherical_shell_grid.jl | 30 ++++++-------------- src/Grids/rectilinear_grid.jl | 2 -- src/Grids/vertical_coordinate.jl | 7 +++++ 5 files changed, 15 insertions(+), 27 deletions(-) diff --git a/src/Grids/grid_utils.jl b/src/Grids/grid_utils.jl index 5fa238683a..a501738935 100644 --- a/src/Grids/grid_utils.jl +++ b/src/Grids/grid_utils.jl @@ -124,7 +124,6 @@ constant grid spacing `Δ`, and interior extent `L`. @inline x_domain(grid) = domain(topology(grid, 1)(), grid.Nx, grid.xᶠᵃᵃ) @inline y_domain(grid) = domain(topology(grid, 2)(), grid.Ny, grid.yᵃᶠᵃ) -@inline z_domain(grid) = domain(topology(grid, 3)(), grid.Nz, grid.zᵃᵃᶠ) regular_dimensions(grid) = () diff --git a/src/Grids/latitude_longitude_grid.jl b/src/Grids/latitude_longitude_grid.jl index a05f215a2f..49ca15cd65 100644 --- a/src/Grids/latitude_longitude_grid.jl +++ b/src/Grids/latitude_longitude_grid.jl @@ -327,11 +327,9 @@ end @inline x_domain(grid::LLG) = domain(topology(grid, 1)(), grid.Nx, grid.λᶠᵃᵃ) @inline y_domain(grid::LLG) = domain(topology(grid, 2)(), grid.Ny, grid.φᵃᶠᵃ) -@inline z_domain(grid::LLG) = domain(topology(grid, 3)(), grid.Nz, grid.zᵃᵃᶠ) @inline cpu_face_constructor_x(grid::XRegularLLG) = x_domain(grid) @inline cpu_face_constructor_y(grid::YRegularLLG) = y_domain(grid) -@inline cpu_face_constructor_z(grid::ZRegularLLG) = z_domain(grid) function constructor_arguments(grid::LatitudeLongitudeGrid) arch = architecture(grid) diff --git a/src/Grids/orthogonal_spherical_shell_grid.jl b/src/Grids/orthogonal_spherical_shell_grid.jl index a748379ed6..3f54d2b1c7 100644 --- a/src/Grids/orthogonal_spherical_shell_grid.jl +++ b/src/Grids/orthogonal_spherical_shell_grid.jl @@ -821,17 +821,7 @@ function conformal_cubed_sphere_panel(filepath::AbstractString, architecture = C TX, TY, TZ = topology Hx, Hy, Hz = halo - ## The vertical coordinates can come out of the regular rectilinear grid! - - z_grid = RectilinearGrid(architecture, FT; size = Nz, z, topology=(Flat, Flat, topology[3]), halo=halo[3]) - - zᵃᵃᶠ = z_grid.zᵃᵃᶠ - zᵃᵃᶜ = z_grid.zᵃᵃᶜ - Δzᵃᵃᶜ = z_grid.Δzᵃᵃᶜ - Δzᵃᵃᶠ = z_grid.Δzᵃᵃᶠ - Lz = z_grid.Lz - - ## Read everything else from the file + ## Read everything from the file except the z-coordinates file = jldopen(filepath, "r")["panel$panel"] @@ -880,16 +870,18 @@ function conformal_cubed_sphere_panel(filepath::AbstractString, architecture = C φᶠᶜᵃ = offset_data(zeros(FT, architecture, Txᶠᶜ, Tyᶠᶜ), loc_fc, topology[1:2], N[1:2], H[1:2]) φᶜᶠᵃ = offset_data(zeros(FT, architecture, Txᶜᶠ, Tyᶜᶠ), loc_cf, topology[1:2], N[1:2], H[1:2]) + ## The vertical coordinates can come out of the regular rectilinear grid! + Lz, z = generate_coordinate(FT, topology, (Nξ, Nη, Nz), halo, z, :z, 3, architecture) + ξ, η = (-1, 1), (-1, 1) conformal_mapping = CubedSphereConformalMapping(ξ, η, rotation) return OrthogonalSphericalShellGrid{TX, TY, TZ}(architecture, Nξ, Nη, Nz, Hx, Hy, Hz, Lz, λᶜᶜᵃ, λᶠᶜᵃ, λᶜᶠᵃ, λᶠᶠᵃ, φᶜᶜᵃ, φᶠᶜᵃ, φᶜᶠᵃ, φᶠᶠᵃ, - zᵃᵃᶜ, zᵃᵃᶠ, + z, Δxᶜᶜᵃ, Δxᶠᶜᵃ, Δxᶜᶠᵃ, Δxᶠᶠᵃ, Δyᶜᶜᵃ, Δyᶜᶠᵃ, Δyᶠᶜᵃ, Δyᶠᶠᵃ, - Δzᵃᵃᶜ, Δzᵃᵃᶠ, Azᶜᶜᵃ, Azᶠᶜᵃ, Azᶜᶠᵃ, Azᶠᶠᵃ, radius, conformal_mapping) @@ -904,9 +896,8 @@ function on_architecture(arch::AbstractSerialArchitecture, grid::OrthogonalSpher :φᶜᶜᵃ, :φᶠᶜᵃ, :φᶜᶠᵃ, - :φᶠᶠᵃ, - :zᵃᵃᶜ, - :zᵃᵃᶠ) + :φᶠᶠᵃ + :z) grid_spacings = (:Δxᶜᶜᵃ, :Δxᶠᶜᵃ, @@ -915,9 +906,7 @@ function on_architecture(arch::AbstractSerialArchitecture, grid::OrthogonalSpher :Δyᶜᶜᵃ, :Δyᶜᶠᵃ, :Δyᶠᶜᵃ, - :Δyᶠᶠᵃ, - :Δzᵃᵃᶜ, - :Δzᵃᵃᶜ) + :Δyᶠᶠᵃ) horizontal_areas = (:Azᶜᶜᵃ, :Azᶠᶜᵃ, @@ -1087,9 +1076,6 @@ function Base.show(io::IO, grid::OrthogonalSphericalShellGrid, withsummary=true) "└── ", z_summary) end -@inline z_domain(grid::OrthogonalSphericalShellGrid{FT, TX, TY, TZ}) where {FT, TX, TY, TZ} = domain(TZ, grid.Nz, grid.zᵃᵃᶠ) -@inline cpu_face_constructor_z(grid::ZRegOrthogonalSphericalShellGrid) = z_domain(grid) - function with_halo(new_halo, old_grid::OrthogonalSphericalShellGrid; rotation=nothing) size = (old_grid.Nx, old_grid.Ny, old_grid.Nz) diff --git a/src/Grids/rectilinear_grid.jl b/src/Grids/rectilinear_grid.jl index ac4dea3df1..aaae4c2a85 100644 --- a/src/Grids/rectilinear_grid.jl +++ b/src/Grids/rectilinear_grid.jl @@ -298,7 +298,6 @@ end x_domain(grid::RectilinearGrid) = domain(topology(grid, 1)(), grid.Nx, grid.xᶠᵃᵃ) y_domain(grid::RectilinearGrid) = domain(topology(grid, 2)(), grid.Ny, grid.yᵃᶠᵃ) -z_domain(grid::RectilinearGrid) = domain(topology(grid, 3)(), grid.Nz, grid.z.cᶠ) # architecture = CPU() default, assuming that a DataType positional arg # is specifying the floating point type. @@ -362,7 +361,6 @@ end cpu_face_constructor_x(grid::XRegularRG) = x_domain(grid) cpu_face_constructor_y(grid::YRegularRG) = y_domain(grid) -cpu_face_constructor_z(grid::ZRegularRG) = z_domain(grid) function constructor_arguments(grid::RectilinearGrid) arch = architecture(grid) diff --git a/src/Grids/vertical_coordinate.jl b/src/Grids/vertical_coordinate.jl index 8e1ce1eba0..e1f533b4b1 100644 --- a/src/Grids/vertical_coordinate.jl +++ b/src/Grids/vertical_coordinate.jl @@ -110,6 +110,13 @@ end coordinate_summary(::Bounded, z::AbstractVerticalCoordinate, name) = @sprintf("Free-surface following with Δ%s=%s", name, prettysummary(z.Δᶜ)) +#### +#### z_domain (independent of ZStar or not) +#### + +z_domain(grid) = domain(topology(grid, 3)(), grid.Nz, grid.z.cᶠ) +cpu_face_constructor_z(grid) = z_domain(grid) + #### #### Nodes and spacings... #### From d4effbc2b6c76673a1c9df6e92e35626b65c7a53 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 25 Nov 2024 14:01:39 +0100 Subject: [PATCH 490/567] a comment --- src/Grids/vertical_coordinate.jl | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/Grids/vertical_coordinate.jl b/src/Grids/vertical_coordinate.jl index e1f533b4b1..9cf9f98d6a 100644 --- a/src/Grids/vertical_coordinate.jl +++ b/src/Grids/vertical_coordinate.jl @@ -1,3 +1,12 @@ +#### +#### Vertical coordinates +#### + +# This file implements everything related to vertical coordinates in Oceananigans. +# Vertical coordinates are independent of the underlying grid type as we support grids that are +# "unstructured" or "curviliear" only in the horizontal direction. +# For this reason the vertical coodinate is _special_, and it can be implemented once for all grid types. + abstract type AbstractVerticalCoordinate end # Represents a static one-dimensional vertical coordinate. From a2fba4145474dd56cff31315469a901f07d6e679 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 25 Nov 2024 14:04:25 +0100 Subject: [PATCH 491/567] remove duplicate field --- src/Grids/vertical_coordinate.jl | 39 ++++++++++++++------------------ 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/src/Grids/vertical_coordinate.jl b/src/Grids/vertical_coordinate.jl index 9cf9f98d6a..7a899ec429 100644 --- a/src/Grids/vertical_coordinate.jl +++ b/src/Grids/vertical_coordinate.jl @@ -4,7 +4,7 @@ # This file implements everything related to vertical coordinates in Oceananigans. # Vertical coordinates are independent of the underlying grid type as we support grids that are -# "unstructured" or "curviliear" only in the horizontal direction. +# "unstructured" or "curvilinear" only in the horizontal direction. # For this reason the vertical coodinate is _special_, and it can be implemented once for all grid types. abstract type AbstractVerticalCoordinate end @@ -12,17 +12,13 @@ abstract type AbstractVerticalCoordinate end # Represents a static one-dimensional vertical coordinate. # # # Fields -# - `cᶠ::C`: Face-centered coordinate. # - `cᶜ::C`: Cell-centered coordinate. +# - `cᶠ::C`: Face-centered coordinate. +# - `Δᶜ::D`: Cell-centered grid spacing. # - `Δᶠ::D`: Face-centered grid spacing. -# - `Δᶠ::D`: Face-centered grid spacing (duplicate field, consider renaming or removing). -# -# # Type Parameters -# - `C`: Type of the face-centered and cell-centered coordinates. -# - `D`: Type of the face-centered grid spacing. struct StaticVerticalCoordinate{C, D} <: AbstractVerticalCoordinate - cᶠ :: C cᶜ :: C + cᶠ :: C Δᶜ :: D Δᶠ :: D end @@ -30,21 +26,20 @@ end # Represents a z-star three-dimensional vertical coordinate. # # # Fields -# - `cᶠ::C`: Face-centered coordinates. -# - `cᶜ::C`: Cell-centered coordinates. +# - `cᶜ::C`: Cell-centered coordinate. +# - `cᶠ::C`: Face-centered coordinate. +# - `Δᶜ::D`: Cell-centered grid spacing. # - `Δᶠ::D`: Face-centered grid spacing. -# - `Δᶠ::D`: Face-centered grid spacing (duplicate field, consider renaming or removing). # - `ηⁿ::E`: Surface elevation at the current time step. -# - `η⁻::E`: Surface elevation at the previous time step. -# - `e₃ᶜᶜⁿ::CC`: Vertical grid scaling at cell centers at the current time step. -# - `e₃ᶠᶜⁿ::FC`: Vertical grid scaling at face centers at the current time step. -# - `e₃ᶜᶠⁿ::CF`: Vertical grid scaling at cell-face interfaces at the current time step. -# - `e₃ᶠᶠⁿ::FF`: Vertical grid scaling at face-face interfaces at the current time step. -# - `e₃ᶜᶜ⁻::CC`: Vertical grid scaling at cell centers at the previous time step. +# - `e₃ᶜᶜⁿ::CC`: Vertical grid scaling at center-center at the current time step. +# - `e₃ᶠᶜⁿ::FC`: Vertical grid scaling at face-center at the current time step. +# - `e₃ᶜᶠⁿ::CF`: Vertical grid scaling at center-face at the current time step. +# - `e₃ᶠᶠⁿ::FF`: Vertical grid scaling at face-face at the current time step. +# - `e₃ᶜᶜ⁻::CC`: Vertical grid scaling at center-center at the previous time step. # - `∂t_e₃::CC`: Time derivative of the vertical grid scaling at cell centers. struct ZStarVerticalCoordinate{C, D, E, CC, FC, CF, FF} <: AbstractVerticalCoordinate - cᶠ :: C cᶜ :: C + cᶠ :: C Δᶜ :: D Δᶠ :: D ηⁿ :: E @@ -78,16 +73,16 @@ const RegularVerticalGrid = AbstractUnderlyingGrid{<:Any, <:Any, <:Any, <:Any, #### Adapt.adapt_structure(to, coord::StaticVerticalCoordinate) = - StaticVerticalCoordinate(Adapt.adapt(to, coord.cᶠ), - Adapt.adapt(to, coord.cᶜ), - Adapt.adapt(to, coord.Δᶠ), + StaticVerticalCoordinate(Adapt.adapt(to, coord.cᶜ), + Adapt.adapt(to, coord.cᶠ), + Adapt.adapt(to, coord.Δᶜ), Adapt.adapt(to, coord.Δᶠ)) Adapt.adapt_structure(to, coord::ZStarVerticalCoordinate) = ZStarVerticalCoordinate(Adapt.adapt(to, coord.cᶠ), Adapt.adapt(to, coord.cᶜ), Adapt.adapt(to, coord.Δᶠ), - Adapt.adapt(to, coord.Δᶠ), + Adapt.adapt(to, coord.Δᶜ), Adapt.adapt(to, coord.ηⁿ), Adapt.adapt(to, coord.e₃ᶜᶜⁿ), Adapt.adapt(to, coord.e₃ᶠᶜⁿ), From eee652f795341406ddba997f12328ec3857c4c66 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 25 Nov 2024 14:11:38 +0100 Subject: [PATCH 492/567] where is zspacing??? --- src/AbstractOperations/grid_metrics.jl | 2 ++ src/Grids/vertical_coordinate.jl | 5 +++-- src/Oceananigans.jl | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/AbstractOperations/grid_metrics.jl b/src/AbstractOperations/grid_metrics.jl index 4205135ef0..9fbfb79570 100644 --- a/src/AbstractOperations/grid_metrics.jl +++ b/src/AbstractOperations/grid_metrics.jl @@ -290,6 +290,7 @@ end @inline zspacings(field::AbstractField) = zspacings(field.grid, location(field)...) @inline λspacings(field::AbstractField) = λspacings(field.grid, location(field)...) @inline φspacings(field::AbstractField) = φspacings(field.grid, location(field)...) +@inline rspacings(field::AbstractField) = rspacings(field.grid, location(field)...) # Some defaults for e.g. easy CFL computations. @inline xspacings(grid::AbstractGrid) = xspacings(grid, Center(), Center(), Center()) @@ -297,3 +298,4 @@ end @inline zspacings(grid::AbstractGrid) = zspacings(grid, Center(), Center(), Center()) @inline λspacings(grid::AbstractGrid) = λspacings(grid, Center(), Center(), Center()) @inline φspacings(grid::AbstractGrid) = φspacings(grid, Center(), Center(), Center()) +@inline rspacings(grid::AbstractGrid) = rspacings(grid, Center(), Center(), Center()) diff --git a/src/Grids/vertical_coordinate.jl b/src/Grids/vertical_coordinate.jl index 7a899ec429..be50329a59 100644 --- a/src/Grids/vertical_coordinate.jl +++ b/src/Grids/vertical_coordinate.jl @@ -97,7 +97,7 @@ Adapt.adapt_structure(to, coord::ZStarVerticalCoordinate) = @inline rnode(i, j, k, grid, ℓx, ℓy, ℓz) = rnode(k, grid, ℓz) @inline rnode(k, grid, ::Center) = getnode(grid.z.cᶜ, k) -@inline rnode(k, grid, ::Face) = getnode(grid.z.cᶠ, k) +@inline rnode(k, grid, ::Face) = getnode(grid.z.cᶠ, k) # These will be extended in the Operators module @inline znode(k, grid, ℓz) = rnode(k, grid, ℓz) @@ -136,7 +136,7 @@ znodes(grid, ::Nothing; kwargs...) = 1:1 @inline znodes(grid, ℓz; kwargs...) = rnodes(grid, ℓz; kwargs...) @inline znodes(grid, ℓx, ℓy, ℓz; kwargs...) = rnodes(grid, ℓx, ℓy, ℓz; kwargs...) -function zspacing end +function rspacings end function zspacings end """ @@ -147,3 +147,4 @@ Return the "reference" spacings over the interior nodes on `grid` in the ``z``-d for a _static_ grid. """ @inline rspacings(grid, ℓx, ℓy, ℓz; with_halos=true) = rspacings(grid, ℓz; with_halos) +@inline zspacings(grid, ℓx, ℓy, ℓz; with_halos=true) = rspacings(grid, ℓz; with_halos) diff --git a/src/Oceananigans.jl b/src/Oceananigans.jl index b6c48a0554..990fa3e3ec 100644 --- a/src/Oceananigans.jl +++ b/src/Oceananigans.jl @@ -13,8 +13,8 @@ export Periodic, Bounded, Flat, ZStarVerticalCoordinate, RectilinearGrid, LatitudeLongitudeGrid, OrthogonalSphericalShellGrid, - nodes, xnodes, ynodes, znodes, λnodes, φnodes, - xspacings, yspacings, zspacings, λspacings, φspacings, + nodes, xnodes, ynodes, rnodes, znodes, λnodes, φnodes, + xspacings, yspacings, rspacings, zspacings, λspacings, φspacings, minimum_xspacing, minimum_yspacing, minimum_zspacing, # Immersed boundaries From e6a35c4cc46ff46a65da779ff1ca54ca69353cd0 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 25 Nov 2024 23:38:21 +0100 Subject: [PATCH 493/567] works --- src/Grids/nodes_and_spacings.jl | 1 + src/Grids/vertical_coordinate.jl | 1 - src/ImmersedBoundaries/zstar_immersed_grid.jl | 6 +++--- .../z_star_vertical_spacing.jl | 8 ++++---- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Grids/nodes_and_spacings.jl b/src/Grids/nodes_and_spacings.jl index 003385f18a..0d0056c09b 100644 --- a/src/Grids/nodes_and_spacings.jl +++ b/src/Grids/nodes_and_spacings.jl @@ -134,6 +134,7 @@ nodes(grid::AbstractGrid, (ℓx, ℓy, ℓz); reshape=false, with_halos=false) = # see Oceananigans/AbstractOperations/grid_metrics.jl for definitions function xspacing end function yspacing end +function zspacing end function λspacing end function φspacing end diff --git a/src/Grids/vertical_coordinate.jl b/src/Grids/vertical_coordinate.jl index be50329a59..5d00a2ffcd 100644 --- a/src/Grids/vertical_coordinate.jl +++ b/src/Grids/vertical_coordinate.jl @@ -119,7 +119,6 @@ coordinate_summary(::Bounded, z::AbstractVerticalCoordinate, name) = #### z_domain(grid) = domain(topology(grid, 3)(), grid.Nz, grid.z.cᶠ) -cpu_face_constructor_z(grid) = z_domain(grid) #### #### Nodes and spacings... diff --git a/src/ImmersedBoundaries/zstar_immersed_grid.jl b/src/ImmersedBoundaries/zstar_immersed_grid.jl index 7aaed57ffe..841cae216d 100644 --- a/src/ImmersedBoundaries/zstar_immersed_grid.jl +++ b/src/ImmersedBoundaries/zstar_immersed_grid.jl @@ -23,7 +23,7 @@ const ZStarGridOfSomeKind = Union{ZStarImmersedGrid, AbstractZStarGrid} @inline dynamic_column_depthᶠᶠᵃ(i, j, grid::ZStarGridOfSomeKind) = dynamic_column_depthᶠᶠᵃ(i, j, grid, grid.z.ηⁿ) # Fallbacks -@inline e₃ⁿ(i, j, k, grid::IBG, ℓx, ℓy, ℓz) = e₃ⁿ(i, j, k, ibg.underlying_grid, ℓx, ℓy, ℓz) -@inline e₃⁻(i, j, k, grid::IBG, ℓx, ℓy, ℓz) = e₃⁻(i, j, k, ibg.underlying_grid, ℓx, ℓy, ℓz) +@inline e₃ⁿ(i, j, k, ibg::IBG, ℓx, ℓy, ℓz) = e₃ⁿ(i, j, k, ibg.underlying_grid, ℓx, ℓy, ℓz) +@inline e₃⁻(i, j, k, ibg::IBG, ℓx, ℓy, ℓz) = e₃⁻(i, j, k, ibg.underlying_grid, ℓx, ℓy, ℓz) -@inline ∂t_e₃(i, j, k, grid::IBG) = ∂t_e₃(i, j, k, grid.underlying_grid) +@inline ∂t_e₃(i, j, k, ibg::IBG) = ∂t_e₃(i, j, k, ibg.underlying_grid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index a81a5abff9..5071f80562 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -25,14 +25,14 @@ function update_grid!(model, grid::ZStarGridOfSomeKind; parameters = :xy) ηⁿ = grid.z.ηⁿ η = model.free_surface.η - launch!(architecture(grid), grid, parameters, _update_grid!, - e₃ᶜᶜ⁻, e₃ᶜᶜⁿ, e₃ᶠᶜⁿ, e₃ᶜᶠⁿ, e₃ᶠᶠⁿ, e₃ᶜᶜ⁻, ηⁿ, grid, η) + launch!(architecture(grid), grid, parameters, _update_grid_scaling!, + e₃ᶜᶜⁿ, e₃ᶠᶜⁿ, e₃ᶜᶠⁿ, e₃ᶠᶠⁿ, e₃ᶜᶜ⁻, ηⁿ, grid, η) # the barotropic velocity are retrieved from the free surface model for a # SplitExplicitFreeSurface and are calculated for other free surface models # For the moment only the `SplitExplicitFreeSurface` is supported # TODO: find a way to support other free surface models by storing the barotropic velocities shomewhere - U, V = retrieve_barotropic_velocity(model, free_surface) + U, V = retrieve_barotropic_velocity(model, model.free_surface) # Update vertical spacing with available parameters # No need to fill the halo as the scaling is updated _IN_ the halos @@ -41,7 +41,7 @@ function update_grid!(model, grid::ZStarGridOfSomeKind; parameters = :xy) return nothing end -@kernel function _update_grid!(e₃ᶜᶜⁿ, e₃ᶠᶜⁿ, e₃ᶜᶠⁿ, e₃ᶠᶠⁿ, e₃ᶜᶜ⁻, ηⁿ, grid, η) +@kernel function _update_grid_scaling!(e₃ᶜᶜⁿ, e₃ᶠᶜⁿ, e₃ᶜᶠⁿ, e₃ᶠᶠⁿ, e₃ᶜᶜ⁻, ηⁿ, grid, η) i, j = @index(Global, NTuple) kᴺ = size(grid, 3) From 0585e3b16ef9195a73ac58654b1cef98149b871d Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 26 Nov 2024 09:31:57 +0100 Subject: [PATCH 494/567] add cpu face constructor --- src/ImmersedBoundaries/immersed_boundary_nodes.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ImmersedBoundaries/immersed_boundary_nodes.jl b/src/ImmersedBoundaries/immersed_boundary_nodes.jl index 62f39894f8..9149dd4d55 100644 --- a/src/ImmersedBoundaries/immersed_boundary_nodes.jl +++ b/src/ImmersedBoundaries/immersed_boundary_nodes.jl @@ -1,3 +1,4 @@ +import Oceananigans.Grids: cpu_face_constructor_x, cpu_face_constructor_y, cpu_face_constructor_z import Oceananigans.Grids: xspacings, yspacings, zspacings const c = Center() From b1276fbf061cd87510347c4ac09d74481cd22980 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 28 Nov 2024 11:01:14 +0100 Subject: [PATCH 495/567] correct aliases --- src/Grids/latitude_longitude_grid.jl | 17 ++++++++++------- src/Grids/orthogonal_spherical_shell_grid.jl | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/Grids/latitude_longitude_grid.jl b/src/Grids/latitude_longitude_grid.jl index 49ca15cd65..f00dae9913 100644 --- a/src/Grids/latitude_longitude_grid.jl +++ b/src/Grids/latitude_longitude_grid.jl @@ -20,6 +20,7 @@ struct LatitudeLongitudeGrid{FT, TX, TY, TZ, CZ, M, MY, FX, FY, VX, VY, Arch} <: Δφᵃᶠᵃ :: FY Δφᵃᶜᵃ :: FY φᵃᶠᵃ :: VY + φᵃᶜᵃ :: VY z :: CZ # Precomputed metrics M <: Nothing means metrics will be computed on the fly Δxᶠᶜᵃ :: M @@ -47,6 +48,8 @@ struct LatitudeLongitudeGrid{FT, TX, TY, TZ, CZ, M, MY, FX, FY, VX, VY, Arch} <: Δxᶠᶜᵃ :: M, Δxᶜᶠᵃ :: M, Δxᶠᶠᵃ :: M, Δxᶜᶜᵃ :: M, Δyᶠᶜᵃ :: MY, Δyᶜᶠᵃ :: MY, + Azᶠᶜᵃ :: M, Azᶜᶠᵃ :: M, + Azᶠᶠᵃ :: M, Azᶜᶜᵃ :: M, radius :: FT) where {Arch, FT, TX, TY, TZ, FX, FY, CZ, VX, VY, M, MY} = @@ -63,12 +66,12 @@ struct LatitudeLongitudeGrid{FT, TX, TY, TZ, CZ, M, MY, FX, FY, VX, VY, Arch} <: end const LLG = LatitudeLongitudeGrid -const XRegularLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Number} -const YRegularLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Number} -const ZRegularLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Number} -const HRegularLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Number, <:Number} -const HNonRegularLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractArray, <:AbstractArray} -const YNonRegularLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Number, <:AbstractArray} +const XRegularLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Number} +const YRegularLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Number} +const ZRegularLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:RegularVerticalCoordinate} +const HRegularLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Number, <:Number} +const HNonRegularLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:AbstractArray, <:AbstractArray} +const YNonRegularLLG = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Any, <:Number, <:AbstractArray} regular_dimensions(::ZRegularLLG) = tuple(3) @@ -241,7 +244,7 @@ function with_precomputed_metrics(grid) grid.Lx, grid.Ly, grid.Lz, grid.Δλᶠᵃᵃ, grid.Δλᶜᵃᵃ, grid.λᶠᵃᵃ, grid.λᶜᵃᵃ, grid.Δφᵃᶠᵃ, grid.Δφᵃᶜᵃ, grid.φᵃᶠᵃ, grid.φᵃᶜᵃ, - grid.Δzᵃᵃᶠ, grid.Δzᵃᵃᶜ, grid.zᵃᵃᶠ, grid.zᵃᵃᶜ, + grid.z, Δxᶠᶜᵃ, Δxᶜᶠᵃ, Δxᶠᶠᵃ, Δxᶜᶜᵃ, Δyᶠᶜᵃ, Δyᶜᶠᵃ, Azᶠᶜᵃ, Azᶜᶠᵃ, Azᶠᶠᵃ, Azᶜᶜᵃ, grid.radius) end diff --git a/src/Grids/orthogonal_spherical_shell_grid.jl b/src/Grids/orthogonal_spherical_shell_grid.jl index 3f54d2b1c7..c19fc226ca 100644 --- a/src/Grids/orthogonal_spherical_shell_grid.jl +++ b/src/Grids/orthogonal_spherical_shell_grid.jl @@ -73,7 +73,7 @@ end const OSSG = OrthogonalSphericalShellGrid const ZRegOSSG = OrthogonalSphericalShellGrid{<:Any, <:Any, <:Any, <:Any, <:RegularVerticalCoordinate} const ZRegOrthogonalSphericalShellGrid = ZRegOSSG -const ConformalCubedSpherePanel = OrthogonalSphericalShellGrid{<:Any, FullyConnected, FullyConnected, <:Any, <:Any, <:Any, <:Any, <:CubedSphereConformalMapping} +const ConformalCubedSpherePanel = OrthogonalSphericalShellGrid{<:Any, FullyConnected, FullyConnected, <:Any, <:Any, <:Any, <:CubedSphereConformalMapping} # convenience constructor for OSSG without any conformal_mapping properties OrthogonalSphericalShellGrid(architecture, Nx, Ny, Nz, Hx, Hy, Hz, Lz, From f0511df81e495726afc5902c124d4605cc0f3b89 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 28 Nov 2024 13:47:34 +0100 Subject: [PATCH 496/567] at least some tests should pass --- src/Grids/vertical_coordinate.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Grids/vertical_coordinate.jl b/src/Grids/vertical_coordinate.jl index 5d00a2ffcd..4ab3edec60 100644 --- a/src/Grids/vertical_coordinate.jl +++ b/src/Grids/vertical_coordinate.jl @@ -17,10 +17,10 @@ abstract type AbstractVerticalCoordinate end # - `Δᶜ::D`: Cell-centered grid spacing. # - `Δᶠ::D`: Face-centered grid spacing. struct StaticVerticalCoordinate{C, D} <: AbstractVerticalCoordinate - cᶜ :: C cᶠ :: C - Δᶜ :: D + cᶜ :: C Δᶠ :: D + Δᶜ :: D end # Represents a z-star three-dimensional vertical coordinate. @@ -38,10 +38,10 @@ end # - `e₃ᶜᶜ⁻::CC`: Vertical grid scaling at center-center at the previous time step. # - `∂t_e₃::CC`: Time derivative of the vertical grid scaling at cell centers. struct ZStarVerticalCoordinate{C, D, E, CC, FC, CF, FF} <: AbstractVerticalCoordinate - cᶜ :: C cᶠ :: C - Δᶜ :: D + cᶜ :: C Δᶠ :: D + Δᶜ :: D ηⁿ :: E e₃ᶜᶜⁿ :: CC e₃ᶠᶜⁿ :: FC From a1a7a4e65f0c130dd7941ae9d3d0c8268b1f03d0 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 28 Nov 2024 15:37:28 +0100 Subject: [PATCH 497/567] sloggin along --- examples/langmuir_turbulence.jl | 2 +- .../distributed_grids.jl | 2 +- .../distributed_on_architecture.jl | 4 +- src/Forcings/forcing.jl | 2 +- src/Grids/grid_utils.jl | 1 - src/Grids/latitude_longitude_grid.jl | 2 +- src/Grids/orthogonal_spherical_shell_grid.jl | 15 +-- src/Grids/rectilinear_grid.jl | 2 +- src/Grids/vertical_coordinate.jl | 32 ++++++- src/Operators/Operators.jl | 6 +- test/test_diagnostics.jl | 4 +- test/test_dynamics.jl | 8 +- ...nsemble_hydrostatic_free_surface_models.jl | 2 +- test/test_grids.jl | 96 ++++++++++--------- test/test_hydrostatic_regression.jl | 2 +- test/test_lagrangian_particle_tracking.jl | 4 +- test/test_netcdf_output_writer.jl | 48 +++++----- test/test_zstar_coordinate.jl | 2 +- 18 files changed, 126 insertions(+), 108 deletions(-) diff --git a/examples/langmuir_turbulence.jl b/examples/langmuir_turbulence.jl index 9886c2fca9..6824ddf484 100644 --- a/examples/langmuir_turbulence.jl +++ b/examples/langmuir_turbulence.jl @@ -313,7 +313,7 @@ Vₙ = @lift view(time_series.V[$n], 1, 1, :) wuₙ = @lift view(time_series.wu[$n], 1, 1, :) wvₙ = @lift view(time_series.wv[$n], 1, 1, :) -k = searchsortedfirst(grid.zᵃᵃᶠ[:], -8) +k = searchsortedfirst(znodes(grid, Center(); with_halos=true), -8) wxyₙ = @lift view(time_series.w[$n], :, :, k) wxzₙ = @lift view(time_series.w[$n], :, 1, :) uxzₙ = @lift view(time_series.u[$n], :, 1, :) diff --git a/src/DistributedComputations/distributed_grids.jl b/src/DistributedComputations/distributed_grids.jl index 19446d4c99..4fae2d30f4 100644 --- a/src/DistributedComputations/distributed_grids.jl +++ b/src/DistributedComputations/distributed_grids.jl @@ -251,7 +251,7 @@ function reconstruct_global_grid(grid::DistributedLatitudeLongitudeGrid) # it is stretched if being passed is a function or vector Lλ, λᶠᵃᵃ, λᶜᵃᵃ, Δλᶠᵃᵃ, Δλᶜᵃᵃ = generate_coordinate(FT, TX(), Nλ, Hλ, λG, :longitude, child_arch) Lφ, φᵃᶠᵃ, φᵃᶜᵃ, Δφᵃᶠᵃ, Δφᵃᶜᵃ = generate_coordinate(FT, TY(), Nφ, Hφ, φG, :latitude, child_arch) - Lz, z = generate_coordinate(FT, (TX, TY, TZ), (Nx, Ny, Nz), H, zG, :z, 3, child_arch) + Lz, z = generate_coordinate(FT, (TX, TY, TZ), (Nλ, Nφ, Nz), H, zG, :z, 3, child_arch) precompute_metrics = metrics_precomputed(grid) diff --git a/src/DistributedComputations/distributed_on_architecture.jl b/src/DistributedComputations/distributed_on_architecture.jl index f4229b59f2..75b936fb9f 100644 --- a/src/DistributedComputations/distributed_on_architecture.jl +++ b/src/DistributedComputations/distributed_on_architecture.jl @@ -23,7 +23,7 @@ function on_architecture(new_arch::Distributed, old_grid::LatitudeLongitudeGrid) child_arch = child_architecture(new_arch) old_properties = (old_grid.Δλᶠᵃᵃ, old_grid.Δλᶜᵃᵃ, old_grid.λᶠᵃᵃ, old_grid.λᶜᵃᵃ, old_grid.Δφᵃᶠᵃ, old_grid.Δφᵃᶜᵃ, old_grid.φᵃᶠᵃ, old_grid.φᵃᶜᵃ, - old_grid.Δzᵃᵃᶠ, old_grid.Δzᵃᵃᶜ, old_grid.zᵃᵃᶠ, old_grid.zᵃᵃᶜ, + old_grid.z, old_grid.Δxᶠᶜᵃ, old_grid.Δxᶜᶠᵃ, old_grid.Δxᶠᶠᵃ, old_grid.Δxᶜᶜᵃ, old_grid.Δyᶠᶜᵃ, old_grid.Δyᶜᶠᵃ, old_grid.Azᶠᶜᵃ, old_grid.Azᶜᶠᵃ, old_grid.Azᶠᶠᵃ, old_grid.Azᶜᶜᵃ) @@ -44,7 +44,7 @@ function on_architecture(new_arch::Distributed, old_grid::RectilinearGrid) child_arch = child_architecture(new_arch) old_properties = (old_grid.Δxᶠᵃᵃ, old_grid.Δxᶜᵃᵃ, old_grid.xᶠᵃᵃ, old_grid.xᶜᵃᵃ, old_grid.Δyᵃᶠᵃ, old_grid.Δyᵃᶜᵃ, old_grid.yᵃᶠᵃ, old_grid.yᵃᶜᵃ, - old_grid.Δzᵃᵃᶠ, old_grid.Δzᵃᵃᶜ, old_grid.zᵃᵃᶠ, old_grid.zᵃᵃᶜ) + old_grid.z) new_properties = Tuple(on_architecture(child_arch, p) for p in old_properties) diff --git a/src/Forcings/forcing.jl b/src/Forcings/forcing.jl index c3f405be62..24c2eeb838 100644 --- a/src/Forcings/forcing.jl +++ b/src/Forcings/forcing.jl @@ -143,7 +143,7 @@ DiscreteForcing{Nothing} ```jldoctest forcing # Discrete-form forcing function with parameters masked_damping(i, j, k, grid, clock, model_fields, parameters) = - @inbounds - parameters.μ * exp(grid.zᵃᵃᶜ[k] / parameters.λ) * model_fields.u[i, j, k] + @inbounds - parameters.μ * exp(grid.z.cᶜ[k] / parameters.λ) * model_fields.u[i, j, k] masked_damping_forcing = Forcing(masked_damping, parameters=(μ=42, λ=π), discrete_form=true) diff --git a/src/Grids/grid_utils.jl b/src/Grids/grid_utils.jl index a501738935..379fafb8ac 100644 --- a/src/Grids/grid_utils.jl +++ b/src/Grids/grid_utils.jl @@ -231,7 +231,6 @@ const f = Face() # What's going on here? @inline cpu_face_constructor_x(grid) = Array(getindex(nodes(grid, f, c, c; with_halos=true), 1)[1:size(grid, 1)+1]) @inline cpu_face_constructor_y(grid) = Array(getindex(nodes(grid, c, f, c; with_halos=true), 2)[1:size(grid, 2)+1]) -@inline cpu_face_constructor_z(grid) = Array(getindex(nodes(grid, c, c, f; with_halos=true), 3)[1:size(grid, 3)+1]) ##### ##### Convenience functions diff --git a/src/Grids/latitude_longitude_grid.jl b/src/Grids/latitude_longitude_grid.jl index f00dae9913..9fa86f6fdb 100644 --- a/src/Grids/latitude_longitude_grid.jl +++ b/src/Grids/latitude_longitude_grid.jl @@ -307,7 +307,7 @@ function Base.show(io::IO, grid::LatitudeLongitudeGrid, withsummary=true) Ωλ = domain(TX(), size(grid, 1), grid.λᶠᵃᵃ) Ωφ = domain(TY(), size(grid, 2), grid.φᵃᶠᵃ) - Ωz = domain(TZ(), size(grid, 3), grid.zᵃᵃᶠ) + Ωz = domain(TZ(), size(grid, 3), grid.z.cᶠ) x_summary = domain_summary(TX(), "λ", Ωλ) y_summary = domain_summary(TY(), "φ", Ωφ) diff --git a/src/Grids/orthogonal_spherical_shell_grid.jl b/src/Grids/orthogonal_spherical_shell_grid.jl index c19fc226ca..acf798d793 100644 --- a/src/Grids/orthogonal_spherical_shell_grid.jl +++ b/src/Grids/orthogonal_spherical_shell_grid.jl @@ -199,11 +199,8 @@ function conformal_cubed_sphere_panel(architecture::AbstractArchitecture = CPU() ηᵃᶜᵃ = ynodes(ξη_grid, Center()) ## The vertical coordinates and metrics can come out of the regular rectilinear grid! - zᵃᵃᶠ = ξη_grid.zᵃᵃᶠ - zᵃᵃᶜ = ξη_grid.zᵃᵃᶜ - Δzᵃᵃᶜ = ξη_grid.Δzᵃᵃᶜ - Δzᵃᵃᶠ = ξη_grid.Δzᵃᵃᶠ - Lz = ξη_grid.Lz + zc = ξη_grid.z + Lz = ξη_grid.Lz ## Compute staggered grid latitude-longitude (φ, λ) coordinates. @@ -600,11 +597,10 @@ function conformal_cubed_sphere_panel(architecture::AbstractArchitecture = CPU() coordinate_arrays = (λᶜᶜᵃ, λᶠᶜᵃ, λᶜᶠᵃ, λᶠᶠᵃ, φᶜᶜᵃ, φᶠᶜᵃ, φᶜᶠᵃ, φᶠᶠᵃ, - zᵃᵃᶜ, zᵃᵃᶠ) + zc) metric_arrays = (Δxᶜᶜᵃ, Δxᶠᶜᵃ, Δxᶜᶠᵃ, Δxᶠᶠᵃ, Δyᶜᶜᵃ, Δyᶜᶠᵃ, Δyᶠᶜᵃ, Δyᶠᶠᵃ, - Δzᵃᵃᶜ, Δzᵃᵃᶠ, Azᶜᶜᵃ, Azᶠᶜᵃ, Azᶜᶠᵃ, Azᶠᶠᵃ) conformal_mapping = CubedSphereConformalMapping(ξ, η, rotation) @@ -621,11 +617,10 @@ function conformal_cubed_sphere_panel(architecture::AbstractArchitecture = CPU() coordinate_arrays = (grid.λᶜᶜᵃ, grid.λᶠᶜᵃ, grid.λᶜᶠᵃ, grid.λᶠᶠᵃ, grid.φᶜᶜᵃ, grid.φᶠᶜᵃ, grid.φᶜᶠᵃ, grid.φᶠᶠᵃ, - grid.zᵃᵃᶜ, grid.zᵃᵃᶠ) + grid.zᶠ) metric_arrays = (grid.Δxᶜᶜᵃ, grid.Δxᶠᶜᵃ, grid.Δxᶜᶠᵃ, grid.Δxᶠᶠᵃ, grid.Δyᶜᶜᵃ, grid.Δyᶜᶠᵃ, grid.Δyᶠᶜᵃ, grid.Δyᶠᶠᵃ, - grid.Δzᵃᵃᶜ, grid.Δzᵃᵃᶠ, grid.Azᶜᶜᵃ, grid.Azᶠᶜᵃ, grid.Azᶜᶠᵃ, grid.Azᶠᶠᵃ) coordinate_arrays = map(a -> on_architecture(architecture, a), coordinate_arrays) @@ -1029,7 +1024,7 @@ function Base.show(io::IO, grid::OrthogonalSphericalShellGrid, withsummary=true) λ₁, λ₂ = minimum(grid.λᶠᶠᵃ[1:Nx_face, 1:Ny_face]), maximum(grid.λᶠᶠᵃ[1:Nx_face, 1:Ny_face]) φ₁, φ₂ = minimum(grid.φᶠᶠᵃ[1:Nx_face, 1:Ny_face]), maximum(grid.φᶠᶠᵃ[1:Nx_face, 1:Ny_face]) - Ωz = domain(topology(grid, 3)(), Nz, grid.zᵃᵃᶠ) + Ωz = domain(topology(grid, 3)(), Nz, grid.z.cᶠ) (λ_center, φ_center), (extent_λ, extent_φ) = get_center_and_extents_of_shell(grid) diff --git a/src/Grids/rectilinear_grid.jl b/src/Grids/rectilinear_grid.jl index aaae4c2a85..f9af2b13a2 100644 --- a/src/Grids/rectilinear_grid.jl +++ b/src/Grids/rectilinear_grid.jl @@ -317,7 +317,7 @@ function Base.show(io::IO, grid::RectilinearGrid, withsummary=true) Ωx = domain(TX(), grid.Nx, grid.xᶠᵃᵃ) Ωy = domain(TY(), grid.Ny, grid.yᵃᶠᵃ) - Ωz = domain(TZ(), grid.Nz, grid.zᵃᵃᶠ) + Ωz = domain(TZ(), grid.Nz, grid.z.cᶠ) x_summary = domain_summary(TX(), "x", Ωx) y_summary = domain_summary(TY(), "y", Ωy) diff --git a/src/Grids/vertical_coordinate.jl b/src/Grids/vertical_coordinate.jl index 4ab3edec60..e1cec16f96 100644 --- a/src/Grids/vertical_coordinate.jl +++ b/src/Grids/vertical_coordinate.jl @@ -69,14 +69,14 @@ const AbstractStaticGrid = AbstractUnderlyingGrid{<:Any, <:Any, <:Any, <:Any, const RegularVerticalGrid = AbstractUnderlyingGrid{<:Any, <:Any, <:Any, <:Any, <:RegularVerticalCoordinate} #### -#### Adapting +#### Adapt and on_architecture #### Adapt.adapt_structure(to, coord::StaticVerticalCoordinate) = - StaticVerticalCoordinate(Adapt.adapt(to, coord.cᶜ), - Adapt.adapt(to, coord.cᶠ), - Adapt.adapt(to, coord.Δᶜ), - Adapt.adapt(to, coord.Δᶠ)) + StaticVerticalCoordinate(Adapt.adapt(to, coord.cᶠ), + Adapt.adapt(to, coord.cᶜ), + Adapt.adapt(to, coord.Δᶠ), + Adapt.adapt(to, coord.Δᶜ)) Adapt.adapt_structure(to, coord::ZStarVerticalCoordinate) = ZStarVerticalCoordinate(Adapt.adapt(to, coord.cᶠ), @@ -91,6 +91,25 @@ Adapt.adapt_structure(to, coord::ZStarVerticalCoordinate) = Adapt.adapt(to, coord.e₃ᶜᶜ⁻), Adapt.adapt(to, coord.∂t_e₃)) +on_architecture(arch, coord::StaticVerticalCoordinate) = + StaticVerticalCoordinate(on_architecture(arch, coord.cᶠ), + on_architecture(arch, coord.cᶜ), + on_architecture(arch, coord.Δᶠ), + on_architecture(arch, coord.Δᶜ)) + +on_architecture(arch, coord::ZStarVerticalCoordinate) = + ZStarVerticalCoordinate(on_architecture(arch, coord.cᶠ), + on_architecture(arch, coord.cᶜ), + on_architecture(arch, coord.Δᶠ), + on_architecture(arch, coord.Δᶜ), + on_architecture(arch, coord.ηⁿ), + on_architecture(arch, coord.e₃ᶜᶜⁿ), + on_architecture(arch, coord.e₃ᶠᶜⁿ), + on_architecture(arch, coord.e₃ᶜᶠⁿ), + on_architecture(arch, coord.e₃ᶠᶠⁿ), + on_architecture(arch, coord.e₃ᶜᶜ⁻), + on_architecture(arch, coord.∂t_e₃)) + ##### ##### Vertical nodes... ##### @@ -120,6 +139,9 @@ coordinate_summary(::Bounded, z::AbstractVerticalCoordinate, name) = z_domain(grid) = domain(topology(grid, 3)(), grid.Nz, grid.z.cᶠ) +# We construct on r not on z +cpu_face_constructor_z(grid) = Array(rnodes(grid, Face())) + #### #### Nodes and spacings... #### diff --git a/src/Operators/Operators.jl b/src/Operators/Operators.jl index d53539c408..6c5ff40c26 100644 --- a/src/Operators/Operators.jl +++ b/src/Operators/Operators.jl @@ -105,9 +105,9 @@ const LLGY = YRegularLLG const LLGZ = ZRegularLLG # On the fly calculations of metrics -const LLGF = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Nothing} -const LLGFX = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Nothing, <:Any, <:Number} -const LLGFY = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Nothing, <:Any, <:Any, <:Number} +const LLGF = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Nothing} +const LLGFX = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Nothing, <:Any, <:Number} +const LLGFY = LatitudeLongitudeGrid{<:Any, <:Any, <:Any, <:Any, <:Any, <:Nothing, <:Any, <:Any, <:Number} include("difference_operators.jl") include("interpolation_operators.jl") diff --git a/test/test_diagnostics.jl b/test/test_diagnostics.jl index 91a37c532e..d402e29c48 100644 --- a/test/test_diagnostics.jl +++ b/test/test_diagnostics.jl @@ -67,7 +67,7 @@ function advective_timescale_cfl_on_regular_grid(arch, FT) Δx = model.grid.Δxᶜᵃᵃ Δy = model.grid.Δyᵃᶜᵃ - Δz = model.grid.Δzᵃᵃᶜ + Δz = model.grid.z.Δᶜ u₀ = FT(1.2) v₀ = FT(-2.5) @@ -121,7 +121,7 @@ function advective_timescale_cfl_on_lat_lon_grid(arch, FT) # Will be the same at every grid point. Δy_min = CUDA.@allowscalar Oceananigans.Operators.Δyᶜᶠᵃ(1, 1, 1, grid) - Δz = model.grid.Δzᵃᵃᶠ + Δz = model.grid.z.Δᶠ u₀ = FT(1.2) v₀ = FT(-2.5) diff --git a/test/test_dynamics.jl b/test/test_dynamics.jl index 9097150946..974c0d6453 100644 --- a/test/test_dynamics.jl +++ b/test/test_dynamics.jl @@ -52,14 +52,14 @@ function test_ScalarDiffusivity_budget(fieldname, model) set!(model; Dict(fieldname => (x, y, z) -> rand())...) field = fields(model)[fieldname] ν = viscosity(model.closure, nothing) - return test_diffusion_budget(fieldname, field, model, ν, model.grid.Δzᵃᵃᶜ) + return test_diffusion_budget(fieldname, field, model, ν, model.grid.z.Δᶜ) end function test_ScalarBiharmonicDiffusivity_budget(fieldname, model) set!(model; u=0, v=0, w=0, c=0) set!(model; Dict(fieldname => (x, y, z) -> rand())...) field = fields(model)[fieldname] - return test_diffusion_budget(fieldname, field, model, model.closure.ν, model.grid.Δzᵃᵃᶜ, 4) + return test_diffusion_budget(fieldname, field, model, model.closure.ν, model.grid.z.Δᶜ, 4) end function test_diffusion_cosine(fieldname, grid, closure, ξ, tracers=:c) @@ -90,7 +90,7 @@ function test_immersed_diffusion(Nz, z, time_discretization) underlying_grid = RectilinearGrid(size=Nz, z=z, topology=(Flat, Flat, Bounded)) grid = ImmersedBoundaryGrid(underlying_grid, GridFittedBottom(() -> 0); active_cells_map = true) - Δz_min = minimum(underlying_grid.Δzᵃᵃᶜ) + Δz_min = minimum(underlying_grid.z.Δᶜ) model_kwargs = (tracers=:c, buoyancy=nothing, velocities=PrescribedVelocityFields()) full_model = HydrostaticFreeSurfaceModel(; grid=underlying_grid, closure, model_kwargs...) @@ -133,7 +133,7 @@ function test_3D_immersed_diffusion(Nz, z, time_discretization) underlying_grid = RectilinearGrid(size=(9, 9, Nz), x=(0, 1), y=(0, 1), z=z, topology=(Periodic, Periodic, Bounded)) grid = ImmersedBoundaryGrid(underlying_grid, GridFittedBottom(bathymetry); active_cells_map = true) - Δz_min = minimum(grid.underlying_grid.Δzᵃᵃᶜ) + Δz_min = minimum(grid.underlying_grid.z.Δᶜ) model_kwargs = (tracers=:c, buoyancy=nothing, velocities=PrescribedVelocityFields()) full_model = HydrostaticFreeSurfaceModel(; grid=underlying_grid, closure, model_kwargs...) diff --git a/test/test_ensemble_hydrostatic_free_surface_models.jl b/test/test_ensemble_hydrostatic_free_surface_models.jl index 95ab1b701c..d0ea1561e6 100644 --- a/test/test_ensemble_hydrostatic_free_surface_models.jl +++ b/test/test_ensemble_hydrostatic_free_surface_models.jl @@ -55,7 +55,7 @@ end @test size(closures) == (3, 2) @test closures[2, 1].background_κz == 1.2 - Δt = 0.01 * grid.Δzᵃᵃᶜ^2 + Δt = 0.01 * grid.z.Δᶜ^2 model_kwargs = (; tracers=:c, buoyancy=nothing, coriolis=nothing) simulation_kwargs = (; Δt, stop_iteration=100) diff --git a/test/test_grids.jl b/test/test_grids.jl index b30b48c1bb..d850b776cb 100644 --- a/test/test_grids.jl +++ b/test/test_grids.jl @@ -46,10 +46,11 @@ function test_regular_rectilinear_correct_coordinate_lengths(FT) @test length(grid.xᶜᵃᵃ) == Nx + 2Hx @test length(grid.yᵃᶜᵃ) == Ny + 2Hy - @test length(grid.zᵃᵃᶜ) == Nz + 2Hz @test length(grid.xᶠᵃᵃ) == Nx + 2Hx @test length(grid.yᵃᶠᵃ) == Ny + 2Hy + 1 - @test length(grid.zᵃᵃᶠ) == Nz + 2Hz + 1 + + @test length(grid.z.cᶜ) == Nz + 2Hz + @test length(grid.z.cᶠ) == Nz + 2Hz + 1 return nothing end @@ -94,7 +95,7 @@ function test_regular_rectilinear_correct_first_cells(FT) @test grid.xᶜᵃᵃ[1] == Δ/2 @test grid.yᵃᶜᵃ[1] == Δ/2 - @test grid.zᵃᵃᶜ[1] == Δ/2 + @test grid.z.cᶜ[1] == Δ/2 return nothing end @@ -123,10 +124,11 @@ function test_regular_rectilinear_ranges_have_correct_length(FT) @test length(grid.xᶜᵃᵃ) == Nx + 2Hx @test length(grid.yᵃᶜᵃ) == Ny + 2Hy - @test length(grid.zᵃᵃᶜ) == Nz + 2Hz @test length(grid.xᶠᵃᵃ) == Nx + 1 + 2Hx @test length(grid.yᵃᶠᵃ) == Ny + 1 + 2Hy - @test length(grid.zᵃᵃᶠ) == Nz + 1 + 2Hz + + @test length(grid.z.cᶜ) == Nz + 2Hz + @test length(grid.z.cᶠ) == Nz + 1 + 2Hz return nothing end @@ -139,8 +141,8 @@ function test_regular_rectilinear_no_roundoff_error_in_ranges(FT) grid = RectilinearGrid(CPU(), FT, size=(Nx, Ny, Nz), extent=(1, 1, π/2), halo=(1, 1, Hz)) - @test length(grid.zᵃᵃᶜ) == Nz + 2Hz - @test length(grid.zᵃᵃᶠ) == Nz + 2Hz + 1 + @test length(grid.z.cᶜ) == Nz + 2Hz + @test length(grid.z.cᶠ) == Nz + 2Hz + 1 return nothing end @@ -153,14 +155,14 @@ function test_regular_rectilinear_grid_properties_are_same_type(FT) @test grid.Lz isa FT @test grid.Δxᶠᵃᵃ isa FT @test grid.Δyᵃᶠᵃ isa FT - @test grid.Δzᵃᵃᶠ isa FT + @test grid.z.Δᶠ isa FT @test eltype(grid.xᶠᵃᵃ) == FT @test eltype(grid.yᵃᶠᵃ) == FT @test eltype(grid.zᵃᵃᶠ) == FT @test eltype(grid.xᶜᵃᵃ) == FT @test eltype(grid.yᵃᶜᵃ) == FT - @test eltype(grid.zᵃᵃᶜ) == FT + @test eltype(grid.z.cᶜ) == FT return nothing end @@ -359,11 +361,11 @@ function test_vertically_stretched_grid_properties_are_same_type(FT, arch) @test eltype(grid.xᶜᵃᵃ) == FT @test eltype(grid.yᵃᶠᵃ) == FT @test eltype(grid.yᵃᶜᵃ) == FT - @test eltype(grid.zᵃᵃᶠ) == FT - @test eltype(grid.zᵃᵃᶜ) == FT + @test eltype(grid.z.cᶠ) == FT + @test eltype(grid.z.cᶜ) == FT - @test eltype(grid.Δzᵃᵃᶜ) == FT - @test eltype(grid.Δzᵃᵃᶠ) == FT + @test eltype(grid.z.Δᶜ) == FT + @test eltype(grid.z.Δᶠ) == FT return nothing end @@ -372,10 +374,10 @@ function test_architecturally_correct_stretched_grid(FT, arch, zᵃᵃᶠ) grid = RectilinearGrid(arch, FT, size=(1, 1, length(zᵃᵃᶠ)-1), x=(0, 1), y=(0, 1), z=zᵃᵃᶠ) ArrayType = array_type(arch) - @test grid.zᵃᵃᶠ isa OffsetArray{FT, 1, <:ArrayType} - @test grid.zᵃᵃᶜ isa OffsetArray{FT, 1, <:ArrayType} - @test grid.Δzᵃᵃᶠ isa OffsetArray{FT, 1, <:ArrayType} - @test grid.Δzᵃᵃᶜ isa OffsetArray{FT, 1, <:ArrayType} + @test grid.z.cᶠ isa OffsetArray{FT, 1, <:ArrayType} + @test grid.z.cᶜ isa OffsetArray{FT, 1, <:ArrayType} + @test grid.z.Δᶠ isa OffsetArray{FT, 1, <:ArrayType} + @test grid.z.Δᶜ isa OffsetArray{FT, 1, <:ArrayType} return nothing end @@ -407,20 +409,20 @@ function test_rectilinear_grid_correct_spacings(FT, N) Δzᵃᵃᶜ(k) = zᵃᵃᶠ(k+1) - zᵃᵃᶠ(k) Δzᵃᵃᶠ(k) = zᵃᵃᶜ(k) - zᵃᵃᶜ(k-1) - @test all(isapprox.( grid.zᵃᵃᶠ[1:N+1], zᵃᵃᶠ.(1:N+1) )) - @test all(isapprox.( grid.zᵃᵃᶜ[1:N], zᵃᵃᶜ.(1:N) )) - @test all(isapprox.( grid.Δzᵃᵃᶜ[1:N], Δzᵃᵃᶜ.(1:N) )) + @test all(isapprox.( grid.z.cᶠ[1:N+1], zᵃᵃᶠ.(1:N+1) )) + @test all(isapprox.( grid.z.cᶜ[1:N], zᵃᵃᶜ.(1:N) )) + @test all(isapprox.( grid.z.Δᶜ[1:N], Δzᵃᵃᶜ.(1:N) )) - @test all(isapprox.(zspacings(grid, Face()), reshape(grid.Δzᵃᵃᶠ[1:N+1], 1, 1, N+1))) - @test all(isapprox.(zspacings(grid, Center()), reshape(grid.Δzᵃᵃᶜ[1:N], 1, 1, N))) + @test all(isapprox.(zspacings(grid, Face()), reshape(grid.z.cᶠ[1:N+1], 1, 1, N+1))) + @test all(isapprox.(zspacings(grid, Center()), reshape(grid.z.cᶜ[1:N], 1, 1, N))) - @test zspacing(1, 1, 2, grid, Center(), Center(), Face()) == grid.Δzᵃᵃᶠ[2] + @test zspacing(1, 1, 2, grid, Center(), Center(), Face()) == grid.z.Δᶠ[2] - @test minimum_zspacing(grid, Center(), Center(), Center()) ≈ minimum(grid.Δzᵃᵃᶜ[1:grid.Nz]) + @test minimum_zspacing(grid, Center(), Center(), Center()) ≈ minimum(grid.z.Δᶜ[1:grid.Nz]) # Note that Δzᵃᵃᶠ[1] involves a halo point, which is not directly determined by # the user-supplied zᵃᵃᶠ - @test all(isapprox.( grid.Δzᵃᵃᶠ[2:N], Δzᵃᵃᶠ.(2:N) )) + @test all(isapprox.( grid.z.Δᶠ[2:N], Δzᵃᵃᶠ.(2:N) )) return nothing end @@ -447,8 +449,8 @@ function test_basic_lat_lon_bounded_domain(FT) @test grid.Δλᶠᵃᵃ == 10 @test grid.Δφᵃᶠᵃ == 5 - @test grid.Δzᵃᵃᶜ == 1 - @test grid.Δzᵃᵃᶠ == 1 + @test grid.z.Δᶜ == 1 + @test grid.z.Δᶠ == 1 @test length(grid.λᶠᵃᵃ) == Nλ + 2Hλ + 1 @test length(grid.λᶜᵃᵃ) == Nλ + 2Hλ @@ -496,8 +498,8 @@ function test_basic_lat_lon_periodic_domain(FT) @test grid.Δλᶠᵃᵃ == 10 @test grid.Δφᵃᶠᵃ == 5 - @test grid.Δzᵃᵃᶜ == 1 - @test grid.Δzᵃᵃᶠ == 1 + @test grid.z.Δᶜ == 1 + @test grid.z.Δᶠ == 1 @test length(grid.λᶠᵃᵃ) == Nλ + 2Hλ @test length(grid.λᶜᵃᵃ) == Nλ + 2Hλ @@ -542,7 +544,7 @@ function test_basic_lat_lon_general_grid(FT) grid_reg = LatitudeLongitudeGrid(CPU(), FT, size=grid_size, halo=halo, latitude=lat, longitude=lon, z=zᵣ) - @test typeof(grid_reg.Δzᵃᵃᶜ) == typeof(grid_reg.Δzᵃᵃᶠ) == FT + @test typeof(grid_reg.z.Δᶜ) == typeof(grid_reg.z.Δᶠ) == FT @test all(xspacings(grid_reg, Center(), Center()) .== reshape(grid_reg.Δxᶜᶜᵃ[1:Nφ], 1, Nφ, 1)) @test all(xspacings(grid_reg, Center(), Face() ) .== reshape(grid_reg.Δxᶜᶠᵃ[1:Nφ+1], 1, Nφ+1, 1)) @@ -550,8 +552,8 @@ function test_basic_lat_lon_general_grid(FT) @test all(xspacings(grid_reg, Face(), Face()) .== reshape(grid_reg.Δxᶠᶠᵃ[1:Nφ+1], 1, Nφ+1, 1)) @test all(yspacings(grid_reg, Center(), Face()) .== grid_reg.Δyᶜᶠᵃ) @test all(yspacings(grid_reg, Face(), Center()) .== grid_reg.Δyᶠᶜᵃ) - @test all(zspacings(grid_reg, Center()) .== grid_reg.Δzᵃᵃᶜ) - @test all(zspacings(grid_reg, Face()) .== grid_reg.Δzᵃᵃᶠ) + @test all(zspacings(grid_reg, Center()) .== grid_reg.z.Δᶜ) + @test all(zspacings(grid_reg, Face()) .== grid_reg.z.Δᶠ) @test all(xspacings(grid_reg, Center(), Center(), Center()) .== xspacings(grid_reg, Center(), Center())) @test all(xspacings(grid_reg, Face(), Face(), Center()) .== xspacings(grid_reg, Face(), Face())) @@ -564,8 +566,8 @@ function test_basic_lat_lon_general_grid(FT) @test xspacing(1, 2, 3, grid_reg, Center(), Face(), Center()) == grid_reg.Δxᶜᶠᵃ[2] @test yspacing(1, 2, 3, grid_reg, Center(), Face(), Center()) == grid_reg.Δyᶜᶠᵃ @test yspacing(1, 2, 3, grid_reg, Face(), Center(), Center()) == grid_reg.Δyᶠᶜᵃ - @test zspacing(1, 2, 3, grid_reg, Center(), Center(), Face() ) == grid_reg.Δzᵃᵃᶠ - @test zspacing(1, 2, 3, grid_reg, Center(), Center(), Center()) == grid_reg.Δzᵃᵃᶜ + @test zspacing(1, 2, 3, grid_reg, Center(), Center(), Face() ) == grid_reg.z.Δᶠ + @test zspacing(1, 2, 3, grid_reg, Center(), Center(), Center()) == grid_reg.z.Δᶜ @test all(λspacings(grid_reg, Center()) .== grid_reg.Δλᶜᵃᵃ) @test all(λspacings(grid_reg, Face()) .== grid_reg.Δλᶠᵃᵃ) @@ -578,7 +580,7 @@ function test_basic_lat_lon_general_grid(FT) Δλ = grid_reg.Δλᶠᵃᵃ λₛ = (-grid_reg.Lx/2):Δλ:(grid_reg.Lx/2) - Δz = grid_reg.Δzᵃᵃᶜ + Δz = grid_reg.z.Δᶜ zₛ = -Lz:Δz:0 grid_str = LatitudeLongitudeGrid(CPU(), FT, size=grid_size, halo=halo, latitude=lat, longitude=λₛ, z=zₛ) @@ -589,21 +591,21 @@ function test_basic_lat_lon_general_grid(FT) @test length(grid_str.φᵃᶠᵃ) == length(grid_reg.φᵃᶠᵃ) == Nφ + 2Hφ + 1 @test length(grid_str.φᵃᶜᵃ) == length(grid_reg.φᵃᶜᵃ) == Nφ + 2Hφ - @test length(grid_str.zᵃᵃᶠ) == length(grid_reg.zᵃᵃᶠ) == Nz + 2Hz + 1 - @test length(grid_str.zᵃᵃᶜ) == length(grid_reg.zᵃᵃᶜ) == Nz + 2Hz + @test length(grid_str.z.cᶠ) == length(grid_reg.z.cᶠ) == Nz + 2Hz + 1 + @test length(grid_str.z.cᶜ) == length(grid_reg.z.cᶜ) == Nz + 2Hz - @test length(grid_str.Δzᵃᵃᶠ) == Nz + 2Hz + 1 - @test length(grid_str.Δzᵃᵃᶜ) == Nz + 2Hz + @test length(grid_str.z.Δᶠ) == Nz + 2Hz + 1 + @test length(grid_str.z.Δᶜ) == Nz + 2Hz @test all(grid_str.λᶜᵃᵃ == grid_reg.λᶜᵃᵃ) @test all(grid_str.λᶠᵃᵃ == grid_reg.λᶠᵃᵃ) @test all(grid_str.φᵃᶜᵃ == grid_reg.φᵃᶜᵃ) @test all(grid_str.φᵃᶠᵃ == grid_reg.φᵃᶠᵃ) - @test all(grid_str.zᵃᵃᶜ == grid_reg.zᵃᵃᶜ) - @test all(grid_str.zᵃᵃᶠ == grid_reg.zᵃᵃᶠ) + @test all(grid_str.z.cᶜ == grid_reg.z.cᶜ) + @test all(grid_str.z.cᶠ == grid_reg.z.cᶠ) - @test sum(grid_str.Δzᵃᵃᶜ) == grid_reg.Δzᵃᵃᶜ * length(grid_str.Δzᵃᵃᶜ) - @test sum(grid_str.Δzᵃᵃᶠ) == grid_reg.Δzᵃᵃᶠ * length(grid_str.Δzᵃᵃᶠ) + @test sum(grid_str.z.Δᶜ) == grid_reg.z.Δᶜ * length(grid_str.z.Δᶜ) + @test sum(grid_str.z.Δᶠ) == grid_reg.z.Δᶠ * length(grid_str.z.Δᶠ) @test all(xspacings(grid_str, Center(), Center()) .== reshape(grid_str.Δxᶜᶜᵃ[1:Nλ, 1:Nφ], Nλ, Nφ, 1)) @test all(xspacings(grid_str, Center(), Face()) .== reshape(grid_str.Δxᶜᶠᵃ[1:Nλ, 1:Nφ+1], Nλ, Nφ+1, 1)) @@ -613,8 +615,8 @@ function test_basic_lat_lon_general_grid(FT) @test all(yspacings(grid_str, Center(), Face()) .== grid_str.Δyᶜᶠᵃ) @test all(yspacings(grid_str, Face(), Center()) .== grid_str.Δyᶠᶜᵃ) - @test all(zspacings(grid_str, Center()) .== reshape(grid_str.Δzᵃᵃᶜ[1:Nz], 1, 1, Nz)) - @test all(zspacings(grid_str, Face()) .== reshape(grid_str.Δzᵃᵃᶠ[1:Nz+1], 1, 1, Nz+1)) + @test all(zspacings(grid_str, Center()) .== reshape(grid_str.z.Δᶜ[1:Nz], 1, 1, Nz)) + @test all(zspacings(grid_str, Face()) .== reshape(grid_str.z.Δᶠ[1:Nz+1], 1, 1, Nz+1)) @test all(zspacings(grid_str, Center()) .== zspacings(grid_str, Center(), Center(), Center())) @test all(zspacings(grid_str, Face()) .== zspacings(grid_str, Face(), Center(), Face())) @@ -749,8 +751,8 @@ function test_orthogonal_shell_grid_array_sizes_and_spacings(FT) @test all(yspacings(grid, Face(), Center(), Face()) .== yspacings(grid, Face(), Center()) .== grid.Δyᶠᶜᵃ[1:Nx+1, 1:Ny]) @test all(yspacings(grid, Face(), Face(), Face()) .== yspacings(grid, Face(), Face() ) .== grid.Δyᶠᶠᵃ[1:Nx+1, 1:Ny+1]) - @test all(zspacings(grid, Center(), Center(), Face() ) .== zspacings(grid, Face() ) .== grid.Δzᵃᵃᶠ) - @test all(zspacings(grid, Center(), Center(), Center()) .== zspacings(grid, Center()) .== grid.Δzᵃᵃᶜ) + @test all(zspacings(grid, Center(), Center(), Face() ) .== zspacings(grid, Face() ) .== grid.z.Δᶠ) + @test all(zspacings(grid, Center(), Center(), Center()) .== zspacings(grid, Center()) .== grid.z.Δᶜ) return nothing end diff --git a/test/test_hydrostatic_regression.jl b/test/test_hydrostatic_regression.jl index d989ba6c27..7e3b7e7d61 100644 --- a/test/test_hydrostatic_regression.jl +++ b/test/test_hydrostatic_regression.jl @@ -1,4 +1,4 @@ -# include("dependencies_for_runtests.jl") +include("dependencies_for_runtests.jl") include("data_dependencies.jl") using Oceananigans.Grids: topology, XRegularLLG, YRegularLLG, ZRegularLLG diff --git a/test/test_lagrangian_particle_tracking.jl b/test/test_lagrangian_particle_tracking.jl index 6ee6492a3f..e813c9d79f 100644 --- a/test/test_lagrangian_particle_tracking.jl +++ b/test/test_lagrangian_particle_tracking.jl @@ -75,8 +75,8 @@ function run_simple_particle_tracking_tests(grid, timestepper=:QuasiAdamsBashfor ##### Test Boundary restitution ##### - initial_z = CUDA.@allowscalar grid.zᵃᵃᶜ[grid.Nz-1] - top_boundary = CUDA.@allowscalar grid.zᵃᵃᶠ[grid.Nz+1] + initial_z = CUDA.@allowscalar grid.z.cᶜ[grid.Nz-1] + top_boundary = CUDA.@allowscalar grid.z.cᶠ[grid.Nz+1] x, y, z = on_architecture.(Ref(arch), ([0.0], [0.0], [initial_z])) diff --git a/test/test_netcdf_output_writer.jl b/test/test_netcdf_output_writer.jl index a354766176..7d95282f2b 100644 --- a/test/test_netcdf_output_writer.jl +++ b/test/test_netcdf_output_writer.jl @@ -241,15 +241,15 @@ function test_thermal_bubble_netcdf_output(arch) @test ds3["xF"][1] == grid.xᶠᵃᵃ[1] @test ds3["yC"][1] == grid.yᵃᶜᵃ[1] @test ds3["yF"][1] == grid.yᵃᶠᵃ[1] - @test ds3["zC"][1] == grid.zᵃᵃᶜ[1] - @test ds3["zF"][1] == grid.zᵃᵃᶠ[1] + @test ds3["zC"][1] == grid.z.cᶜ[1] + @test ds3["zF"][1] == grid.z.cᶠ[1] @test ds3["xC"][end] == grid.xᶜᵃᵃ[Nx] @test ds3["xF"][end] == grid.xᶠᵃᵃ[Nx] @test ds3["yC"][end] == grid.yᵃᶜᵃ[Ny] @test ds3["yF"][end] == grid.yᵃᶠᵃ[Ny] - @test ds3["zC"][end] == grid.zᵃᵃᶜ[Nz] - @test ds3["zF"][end] == grid.zᵃᵃᶠ[Nz+1] # z is Bounded + @test ds3["zC"][end] == grid.z.cᶜ[Nz] + @test ds3["zF"][end] == grid.z.cᶠ[Nz+1] # z is Bounded @test eltype(ds3["u"]) == Float64 @test eltype(ds3["v"]) == Float64 @@ -300,15 +300,15 @@ function test_thermal_bubble_netcdf_output(arch) @test ds2["xF"][1] == grid.xᶠᵃᵃ[i_slice[1]] @test ds2["yC"][1] == grid.yᵃᶜᵃ[j_slice[1]] @test ds2["yF"][1] == grid.yᵃᶠᵃ[j_slice[1]] - @test ds2["zC"][1] == grid.zᵃᵃᶜ[k_slice[1]] - @test ds2["zF"][1] == grid.zᵃᵃᶠ[k_slice[1]] + @test ds2["zC"][1] == grid.z.cᶜ[k_slice[1]] + @test ds2["zF"][1] == grid.z.cᶠ[k_slice[1]] @test ds2["xC"][end] == grid.xᶜᵃᵃ[i_slice[end]] @test ds2["xF"][end] == grid.xᶠᵃᵃ[i_slice[end]] @test ds2["yC"][end] == grid.yᵃᶜᵃ[j_slice[end]] @test ds2["yF"][end] == grid.yᵃᶠᵃ[j_slice[end]] - @test ds2["zC"][end] == grid.zᵃᵃᶜ[k_slice[end]] - @test ds2["zF"][end] == grid.zᵃᵃᶠ[k_slice[end]] + @test ds2["zC"][end] == grid.z.cᶜ[k_slice[end]] + @test ds2["zF"][end] == grid.z.cᶠ[k_slice[end]] @test eltype(ds2["u"]) == Float32 @test eltype(ds2["v"]) == Float32 @@ -395,15 +395,15 @@ function test_thermal_bubble_netcdf_output_with_halos(arch) @test ds["xF"][1] == grid.xᶠᵃᵃ[1-Hx] @test ds["yC"][1] == grid.yᵃᶜᵃ[1-Hy] @test ds["yF"][1] == grid.yᵃᶠᵃ[1-Hy] - @test ds["zC"][1] == grid.zᵃᵃᶜ[1-Hz] - @test ds["zF"][1] == grid.zᵃᵃᶠ[1-Hz] + @test ds["zC"][1] == grid.z.cᶜ[1-Hz] + @test ds["zF"][1] == grid.z.cᶠ[1-Hz] @test ds["xC"][end] == grid.xᶜᵃᵃ[Nx+Hx] @test ds["xF"][end] == grid.xᶠᵃᵃ[Nx+Hx] @test ds["yC"][end] == grid.yᵃᶜᵃ[Ny+Hy] @test ds["yF"][end] == grid.yᵃᶠᵃ[Ny+Hy] - @test ds["zC"][end] == grid.zᵃᵃᶜ[Nz+Hz] - @test ds["zF"][end] == grid.zᵃᵃᶠ[Nz+Hz+1] # z is Bounded + @test ds["zC"][end] == grid.z.cᶜ[Nz+Hz] + @test ds["zF"][end] == grid.z.cᶠ[Nz+Hz+1] # z is Bounded @test eltype(ds["u"]) == Float64 @test eltype(ds["v"]) == Float64 @@ -507,15 +507,15 @@ function test_netcdf_function_output(arch) @test ds["xF"][1] == grid.xᶠᵃᵃ[1] @test ds["yC"][1] == grid.yᵃᶜᵃ[1] @test ds["yF"][1] == grid.yᵃᶠᵃ[1] - @test ds["zC"][1] == grid.zᵃᵃᶜ[1] - @test ds["zF"][1] == grid.zᵃᵃᶠ[1] + @test ds["zC"][1] == grid.z.cᶜ[1] + @test ds["zF"][1] == grid.z.cᶠ[1] @test ds["xC"][end] == grid.xᶜᵃᵃ[N] @test ds["yC"][end] == grid.yᵃᶜᵃ[N] - @test ds["zC"][end] == grid.zᵃᵃᶜ[N] @test ds["xF"][end] == grid.xᶠᵃᵃ[N] @test ds["yF"][end] == grid.yᵃᶠᵃ[N] - @test ds["zF"][end] == grid.zᵃᵃᶠ[N+1] # z is Bounded + @test ds["zC"][end] == grid.z.cᶜ[N] + @test ds["zF"][end] == grid.z.cᶠ[N+1] # z is Bounded @test ds.attrib["location"] == "Bay of Fundy" @test ds.attrib["onions"] == 7 @@ -849,16 +849,16 @@ function test_netcdf_vertically_stretched_grid_output(arch) @test ds["yC"][1] == grid.yᵃᶜᵃ[1] @test ds["yF"][1] == grid.yᵃᶠᵃ[1] - @test CUDA.@allowscalar ds["zC"][1] == grid.zᵃᵃᶜ[1] - @test CUDA.@allowscalar ds["zF"][1] == grid.zᵃᵃᶠ[1] + @test CUDA.@allowscalar ds["zC"][1] == grid.z.cᶜ[1] + @test CUDA.@allowscalar ds["zF"][1] == grid.z.cᶠ[1] @test ds["xC"][end] == grid.xᶜᵃᵃ[Nx] @test ds["xF"][end] == grid.xᶠᵃᵃ[Nx] @test ds["yC"][end] == grid.yᵃᶜᵃ[Ny] @test ds["yF"][end] == grid.yᵃᶠᵃ[Ny] - @test CUDA.@allowscalar ds["zC"][end] == grid.zᵃᵃᶜ[Nz] - @test CUDA.@allowscalar ds["zF"][end] == grid.zᵃᵃᶠ[Nz+1] # z is Bounded + @test CUDA.@allowscalar ds["zC"][end] == grid.z.cᶜ[Nz] + @test CUDA.@allowscalar ds["zF"][end] == grid.z.cᶠ[Nz+1] # z is Bounded close(ds) rm(nc_filepath) @@ -907,15 +907,15 @@ function test_netcdf_regular_lat_lon_grid_output(arch; immersed = false) @test ds["xF"][1] == grid.λᶠᵃᵃ[1] @test ds["yC"][1] == grid.φᵃᶜᵃ[1] @test ds["yF"][1] == grid.φᵃᶠᵃ[1] - @test ds["zC"][1] == grid.zᵃᵃᶜ[1] - @test ds["zF"][1] == grid.zᵃᵃᶠ[1] + @test ds["zC"][1] == grid.z.cᶜ[1] + @test ds["zF"][1] == grid.z.cᶠ[1] @test ds["xC"][end] == grid.λᶜᵃᵃ[Nx] @test ds["xF"][end] == grid.λᶠᵃᵃ[Nx] @test ds["yC"][end] == grid.φᵃᶜᵃ[Ny] @test ds["yF"][end] == grid.φᵃᶠᵃ[Ny+1] # y is Bounded - @test ds["zC"][end] == grid.zᵃᵃᶜ[Nz] - @test ds["zF"][end] == grid.zᵃᵃᶠ[Nz+1] # z is Bounded + @test ds["zC"][end] == grid.z.cᶜ[Nz] + @test ds["zF"][end] == grid.z.cᶠ[Nz+1] # z is Bounded close(ds) rm(nc_filepath) diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index 0a4af3f517..44fe7ecb56 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -31,7 +31,7 @@ function info_message(grid) msg1 = "Testing z-star coordinates on $(architecture(grid)) on a " msg2 = string(getnamewrapper(grid)) msg3 = grid isa ImmersedBoundaryGrid ? " on a " * string(getnamewrapper(grid.underlying_grid)) : "" - msg4 = grid.Δzᵃᵃᶠ.reference isa Number ? " with uniform spacing" : " with stretched spacing" + msg4 = grid.z.Δᶠ isa Number ? " with uniform spacing" : " with stretched spacing" msg5 = grid isa ImmersedBoundaryGrid ? " and $(string(getnamewrapper(grid.immersed_boundary))) immersed boundary" : "" return msg1 * msg2 * msg3 * msg4 * msg5 From 8ab0d1dd3405b02d4242927e6c8421b3ace30ddd Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 28 Nov 2024 16:04:07 +0100 Subject: [PATCH 498/567] another bugfix --- src/Grids/orthogonal_spherical_shell_grid.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Grids/orthogonal_spherical_shell_grid.jl b/src/Grids/orthogonal_spherical_shell_grid.jl index acf798d793..f381fc22bd 100644 --- a/src/Grids/orthogonal_spherical_shell_grid.jl +++ b/src/Grids/orthogonal_spherical_shell_grid.jl @@ -617,7 +617,7 @@ function conformal_cubed_sphere_panel(architecture::AbstractArchitecture = CPU() coordinate_arrays = (grid.λᶜᶜᵃ, grid.λᶠᶜᵃ, grid.λᶜᶠᵃ, grid.λᶠᶠᵃ, grid.φᶜᶜᵃ, grid.φᶠᶜᵃ, grid.φᶜᶠᵃ, grid.φᶠᶠᵃ, - grid.zᶠ) + grid.z) metric_arrays = (grid.Δxᶜᶜᵃ, grid.Δxᶠᶜᵃ, grid.Δxᶜᶠᵃ, grid.Δxᶠᶠᵃ, grid.Δyᶜᶜᵃ, grid.Δyᶜᶠᵃ, grid.Δyᶠᶜᵃ, grid.Δyᶠᶠᵃ, From abf72ce81214b0b9cf7d782f7143567bfc56e2ed Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 28 Nov 2024 16:49:29 +0100 Subject: [PATCH 499/567] most of the tests should pass --- src/Grids/orthogonal_spherical_shell_grid.jl | 1 - src/Grids/vertical_coordinate.jl | 13 +++---------- test/test_distributed_models.jl | 12 ++++++------ test/test_grids.jl | 12 ++++++------ 4 files changed, 15 insertions(+), 23 deletions(-) diff --git a/src/Grids/orthogonal_spherical_shell_grid.jl b/src/Grids/orthogonal_spherical_shell_grid.jl index f381fc22bd..83afbb81da 100644 --- a/src/Grids/orthogonal_spherical_shell_grid.jl +++ b/src/Grids/orthogonal_spherical_shell_grid.jl @@ -1170,7 +1170,6 @@ rname(::OSSG) = :z @inline xspacings(grid::OSSG, ℓx, ℓy) = xspacings(grid, ℓx, ℓy, nothing) @inline yspacings(grid::OSSG, ℓx, ℓy) = yspacings(grid, ℓx, ℓy, nothing) -@inline zspacings(grid::OSSG, ℓz) = zspacings(grid, nothing, nothing, ℓz) @inline λspacings(grid::OSSG, ℓx, ℓy) = λspacings(grid, ℓx, ℓy, nothing) @inline φspacings(grid::OSSG, ℓx, ℓy) = φspacings(grid, ℓx, ℓy, nothing) diff --git a/src/Grids/vertical_coordinate.jl b/src/Grids/vertical_coordinate.jl index e1cec16f96..b6690f54ae 100644 --- a/src/Grids/vertical_coordinate.jl +++ b/src/Grids/vertical_coordinate.jl @@ -143,7 +143,7 @@ z_domain(grid) = domain(topology(grid, 3)(), grid.Nz, grid.z.cᶠ) cpu_face_constructor_z(grid) = Array(rnodes(grid, Face())) #### -#### Nodes and spacings... +#### Nodes and spacings (common to every grid)... #### @inline rnodes(grid, ℓz::Face; with_halos=false) = _property(grid.z.cᶠ, ℓz, topology(grid, 3), size(grid, 3), with_halos) @@ -160,12 +160,5 @@ znodes(grid, ::Nothing; kwargs...) = 1:1 function rspacings end function zspacings end -""" - rspacings(grid, ℓx, ℓy, ℓz; with_halos=true) - -Return the "reference" spacings over the interior nodes on `grid` in the ``z``-direction for the location `ℓx`, -`ℓy`, `ℓz`. For `Bounded` directions, `Face` nodes include the boundary points. These are equal to the `zspacings` -for a _static_ grid. -""" -@inline rspacings(grid, ℓx, ℓy, ℓz; with_halos=true) = rspacings(grid, ℓz; with_halos) -@inline zspacings(grid, ℓx, ℓy, ℓz; with_halos=true) = rspacings(grid, ℓz; with_halos) +@inline rspacings(grid, ℓz) = rspacings(grid, nothing, nothing, ℓz) +@inline zspacings(grid, ℓz) = zspacings(grid, nothing, nothing, ℓz) \ No newline at end of file diff --git a/test/test_distributed_models.jl b/test/test_distributed_models.jl index 12d45191c9..7d291b2eba 100644 --- a/test/test_distributed_models.jl +++ b/test/test_distributed_models.jl @@ -237,8 +237,8 @@ function test_triply_periodic_local_grid_with_411_ranks() @test local_grid.xᶠᵃᵃ[nx+1] == 0.25*(local_rank+1) @test local_grid.yᵃᶠᵃ[1] == 0 @test local_grid.yᵃᶠᵃ[ny+1] == 2 - @test local_grid.zᵃᵃᶠ[1] == -3 - @test local_grid.zᵃᵃᶠ[nz+1] == 0 + @test local_grid.z.cᶠ[1] == -3 + @test local_grid.z.cᶠ[nz+1] == 0 return nothing end @@ -254,8 +254,8 @@ function test_triply_periodic_local_grid_with_141_ranks() @test local_grid.xᶠᵃᵃ[nx+1] == 1 @test local_grid.yᵃᶠᵃ[1] == 0.5*local_rank @test local_grid.yᵃᶠᵃ[ny+1] == 0.5*(local_rank+1) - @test local_grid.zᵃᵃᶠ[1] == -3 - @test local_grid.zᵃᵃᶠ[nz+1] == 0 + @test local_grid.z.cᶠ[1] == -3 + @test local_grid.z.cᶠ[nz+1] == 0 return nothing end @@ -271,8 +271,8 @@ function test_triply_periodic_local_grid_with_221_ranks() @test local_grid.xᶠᵃᵃ[nx+1] == 0.5*i @test local_grid.yᵃᶠᵃ[1] == j-1 @test local_grid.yᵃᶠᵃ[ny+1] == j - @test local_grid.zᵃᵃᶠ[1] == -3 - @test local_grid.zᵃᵃᶠ[nz+1] == 0 + @test local_grid.z.cᶠ[1] == -3 + @test local_grid.z.cᶠ[nz+1] == 0 return nothing end diff --git a/test/test_grids.jl b/test/test_grids.jl index d850b776cb..ad14d96192 100644 --- a/test/test_grids.jl +++ b/test/test_grids.jl @@ -76,11 +76,11 @@ function test_regular_rectilinear_correct_halo_faces(FT) @test grid.xᶠᵃᵃ[0] == - H * Δ @test grid.yᵃᶠᵃ[0] == - H * Δ - @test grid.zᵃᵃᶠ[0] == - H * Δ + @test grid.z.cᶠ[0] == - H * Δ @test grid.xᶠᵃᵃ[N+1] == L # Periodic @test grid.yᵃᶠᵃ[N+2] == L + H * Δ - @test grid.zᵃᵃᶠ[N+2] == L + H * Δ + @test grid.z.cᶠ[N+2] == L + H * Δ return nothing end @@ -110,7 +110,7 @@ function test_regular_rectilinear_correct_end_faces(FT) @test grid.xᶠᵃᵃ[N+1] == L @test grid.yᵃᶠᵃ[N+2] == L + Δ - @test grid.zᵃᵃᶠ[N+2] == L + Δ + @test grid.z.cᶠ[N+2] == L + Δ return nothing end @@ -159,7 +159,7 @@ function test_regular_rectilinear_grid_properties_are_same_type(FT) @test eltype(grid.xᶠᵃᵃ) == FT @test eltype(grid.yᵃᶠᵃ) == FT - @test eltype(grid.zᵃᵃᶠ) == FT + @test eltype(grid.z.cᶠ) == FT @test eltype(grid.xᶜᵃᵃ) == FT @test eltype(grid.yᵃᶜᵃ) == FT @test eltype(grid.z.cᶜ) == FT @@ -413,8 +413,8 @@ function test_rectilinear_grid_correct_spacings(FT, N) @test all(isapprox.( grid.z.cᶜ[1:N], zᵃᵃᶜ.(1:N) )) @test all(isapprox.( grid.z.Δᶜ[1:N], Δzᵃᵃᶜ.(1:N) )) - @test all(isapprox.(zspacings(grid, Face()), reshape(grid.z.cᶠ[1:N+1], 1, 1, N+1))) - @test all(isapprox.(zspacings(grid, Center()), reshape(grid.z.cᶜ[1:N], 1, 1, N))) + @test all(isapprox.(zspacings(grid, Face()), reshape(grid.z.Δᶠ[1:N+1], 1, 1, N+1))) + @test all(isapprox.(zspacings(grid, Center()), reshape(grid.z.Δᶜ[1:N], 1, 1, N))) @test zspacing(1, 1, 2, grid, Center(), Center(), Face()) == grid.z.Δᶠ[2] From 8080929135d9318575b10917a84374cd7cd93f5a Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 28 Nov 2024 16:57:05 +0100 Subject: [PATCH 500/567] remove partial cells from this business --- test/test_zstar_coordinate.jl | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index 44fe7ecb56..34aab4be4c 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -38,6 +38,7 @@ function info_message(grid) end const C = Center +const F = Face @testset "ZStar coordinate scaling tests" begin @info "testing the ZStar coordinate scalings" @@ -110,16 +111,18 @@ end pllg = ImmersedBoundaryGrid(llg, PartialCellBottom((x, y) -> rand() - 10)) pllgv = ImmersedBoundaryGrid(llgv, PartialCellBottom((x, y) -> rand() - 10)) - grids = [llg, rtg, llgv, rtgv, illg, irtg, illgv, irtgv, pllg, prtg, pllgv, prtgv] + # Partial cell bottom are broken at the moment and do not account for the Δz in the volumes + # and vertical areas (see https://github.com/CliMA/Oceananigans.jl/issues/3958) + # When this is issue is fixed we can add the partial cells to the testing. + grids = [llg, rtg, llgv, rtgv, illg, irtg, illgv, irtgv] # , pllg, prtg, pllgv, prtgv] else - grids = [rtg, rtgv, irtg, irtgv, prtg, prtgv] + grids = [rtg, rtgv, irtg, irtgv] #, prtg, prtgv] end for grid in grids info_msg = info_message(grid) @testset "$info_msg" begin @info " $info_msg" - # TODO: minimum_xspacing(grid) on a Immersed GPU grid with ZStarVerticalCoordinate # fails because it uses too much parameter space. Figure out a way to reduce it free_surface = SplitExplicitFreeSurface(grid; substeps = 20) From 9b6eed226fa37955e0a403f8fa9b601be59c79bc Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 28 Nov 2024 17:06:52 +0100 Subject: [PATCH 501/567] transport the correct term --- src/Advection/vector_invariant_cross_upwinding.jl | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/Advection/vector_invariant_cross_upwinding.jl b/src/Advection/vector_invariant_cross_upwinding.jl index 5c0c459a25..7f82a509a7 100644 --- a/src/Advection/vector_invariant_cross_upwinding.jl +++ b/src/Advection/vector_invariant_cross_upwinding.jl @@ -17,7 +17,20 @@ ##### Cross and Self Upwinding of the Divergence flux ##### -@inline V_times_∂t_e₃(i, j, k, grid) = Vᶜᶜᶜ(i, j, k, grid) * ∂t_e₃(i, j, k, grid) +# This is the additional term appearing in the continuity equation due to a moving grid. +# The discrete divergence is calculated as: +# +# wᵏ⁺¹ - wᵏ δx(Ax u) + δy(Ay v) Δrᶜᶜᶜ ∂t_e₃ +# ---------- = - --------------------- - ------------- +# Δzᶜᶜᶜ Vᶜᶜᶜ Δzᶜᶜᶜ +# +# We upwind with the discrete divergence `δx(Ax u) + δy(Ay v)` and then divide by the volume, +# therefore, the correct term to be added to the divergence transport due to the moving grid is: +# +# Azᶜᶜᶜ Δrᶜᶜᶜ ∂t_e₃ +# +# which represents the static volume times the time derivative of the vertical grid scaling. +@inline V_times_∂t_e₃(i, j, k, grid) = Azᶜᶜᶜ(i, j, k, grid) * Δrᶜᶜᶜ(i, j, k, grid) * ∂t_e₃(i, j, k, grid) @inline function upwinded_divergence_flux_Uᶠᶜᶜ(i, j, k, grid, scheme::VectorInvariantCrossVerticalUpwinding, u, v) @inbounds û = u[i, j, k] From b62e4a20bc030b5b984a905a8fdd7e0bb9ba88d0 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 28 Nov 2024 17:32:50 +0100 Subject: [PATCH 502/567] some more comment --- .../compute_w_from_continuity.jl | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl index 073855b6aa..d8629050f3 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl @@ -19,6 +19,25 @@ compute_w_from_continuity!(model; kwargs...) = compute_w_from_continuity!(velocities, arch, grid; parameters = w_kernel_parameters(grid)) = launch!(arch, grid, parameters, _compute_w_from_continuity!, velocities, grid) + +# Since the derivative of the moving grid is: +# +# δx(Ax U) + δy(Ay V) ∇ ⋅ U +# ∂t_e₃ = - --------------------- = - -------- +# Azᶜᶜᶜ ⋅ Hᶜᶜ Hᶜᶜ +# +# The discrete divergence is calculated as: +# +# wᵏ⁺¹ - wᵏ δx(Ax u) + δy(Ay v) Δrᶜᶜᶜ ∂t_e₃ +# ---------- = - --------------------- - ------------- +# Δzᶜᶜᶜ Vᶜᶜᶜ Δzᶜᶜᶜ +# +# This makes sure that if we sum up till the top of the domain, we get +# +# ∇ ⋅ U +# wᴺᶻ⁺¹ = w⁰ + ∂t_e₃ + ------- = 0 (if w⁰ == 0) +# Hᶜᶜ +# @kernel function _compute_w_from_continuity!(U, grid) i, j = @index(Global, NTuple) From f9d71ae14d7f4626914668d2b19af15cee3e23f9 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 28 Nov 2024 17:36:36 +0100 Subject: [PATCH 503/567] better comment --- .../compute_w_from_continuity.jl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl index d8629050f3..4fcd297b7c 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl @@ -22,21 +22,21 @@ compute_w_from_continuity!(velocities, arch, grid; parameters = w_kernel_paramet # Since the derivative of the moving grid is: # -# δx(Ax U) + δy(Ay V) ∇ ⋅ U +# δx(Δy U) + δy(Δx V) ∇ ⋅ U # ∂t_e₃ = - --------------------- = - -------- -# Azᶜᶜᶜ ⋅ Hᶜᶜ Hᶜᶜ +# Az ⋅ H H # # The discrete divergence is calculated as: # -# wᵏ⁺¹ - wᵏ δx(Ax u) + δy(Ay v) Δrᶜᶜᶜ ∂t_e₃ -# ---------- = - --------------------- - ------------- -# Δzᶜᶜᶜ Vᶜᶜᶜ Δzᶜᶜᶜ +# wᵏ⁺¹ - wᵏ δx(Ax u) + δy(Ay v) Δr ∂t_e₃ +# ---------- = - --------------------- - ---------- +# Δz V Δz # # This makes sure that if we sum up till the top of the domain, we get # -# ∇ ⋅ U -# wᴺᶻ⁺¹ = w⁰ + ∂t_e₃ + ------- = 0 (if w⁰ == 0) -# Hᶜᶜ +# ∇ ⋅ U +# wᴺᶻ⁺¹ = w⁰ - ------- - ∂t_e₃ ≈ 0 (if w⁰ == 0) +# H # @kernel function _compute_w_from_continuity!(U, grid) i, j = @index(Global, NTuple) From 5d300649727875933db5799ea577ecc0e7f9dc1c Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 28 Nov 2024 23:59:51 +0100 Subject: [PATCH 504/567] some more bugfixes --- src/Grids/orthogonal_spherical_shell_grid.jl | 2 +- src/Solvers/batched_tridiagonal_solver.jl | 1 - src/TurbulenceClosures/vertically_implicit_diffusion_solver.jl | 2 +- test/test_zstar_coordinate.jl | 3 +-- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/Grids/orthogonal_spherical_shell_grid.jl b/src/Grids/orthogonal_spherical_shell_grid.jl index 83afbb81da..8d6dc67eff 100644 --- a/src/Grids/orthogonal_spherical_shell_grid.jl +++ b/src/Grids/orthogonal_spherical_shell_grid.jl @@ -891,7 +891,7 @@ function on_architecture(arch::AbstractSerialArchitecture, grid::OrthogonalSpher :φᶜᶜᵃ, :φᶠᶜᵃ, :φᶜᶠᵃ, - :φᶠᶠᵃ + :φᶠᶠᵃ, :z) grid_spacings = (:Δxᶜᶜᵃ, diff --git a/src/Solvers/batched_tridiagonal_solver.jl b/src/Solvers/batched_tridiagonal_solver.jl index 2886847938..bebc7c74ce 100644 --- a/src/Solvers/batched_tridiagonal_solver.jl +++ b/src/Solvers/batched_tridiagonal_solver.jl @@ -1,6 +1,5 @@ using Oceananigans.Architectures: on_architecture using Oceananigans.Grids: XDirection, YDirection, ZDirection - import Oceananigans.Architectures: architecture """ diff --git a/src/TurbulenceClosures/vertically_implicit_diffusion_solver.jl b/src/TurbulenceClosures/vertically_implicit_diffusion_solver.jl index 6272398057..86795b9e24 100644 --- a/src/TurbulenceClosures/vertically_implicit_diffusion_solver.jl +++ b/src/TurbulenceClosures/vertically_implicit_diffusion_solver.jl @@ -1,4 +1,4 @@ -using Oceananigans.Operators: Δz +using Oceananigans.Operators: Δz, Δr using Oceananigans.Solvers: BatchedTridiagonalSolver, solve! using Oceananigans.ImmersedBoundaries: immersed_peripheral_node, ImmersedBoundaryGrid using Oceananigans.Grids: ZDirection diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index 34aab4be4c..9ea6b3a5fd 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -53,8 +53,7 @@ const F = Face grid = ImmersedBoundaryGrid(grid, GridFittedBottom((x, y) -> -10)) - model = HydrostaticFreeSurfaceModel(grid, - free_surface = SplitExplicitFreeSurface(grid; substeps = 20)) + model = HydrostaticFreeSurfaceModel(; grid, free_surface = SplitExplicitFreeSurface(grid; substeps = 20)) @test znode(1, 1, 21, grid, C(), C(), F()) == 0 @test dynamic_column_depthᶜᶜᵃ(1, 1, grid) == 10 From ad0f4531f247916c969d9e6497082413a2af6280 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 29 Nov 2024 09:26:09 +0100 Subject: [PATCH 505/567] add z face constructor for nothing --- src/Grids/vertical_coordinate.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Grids/vertical_coordinate.jl b/src/Grids/vertical_coordinate.jl index b6690f54ae..a6cdf2069a 100644 --- a/src/Grids/vertical_coordinate.jl +++ b/src/Grids/vertical_coordinate.jl @@ -141,6 +141,7 @@ z_domain(grid) = domain(topology(grid, 3)(), grid.Nz, grid.z.cᶠ) # We construct on r not on z cpu_face_constructor_z(grid) = Array(rnodes(grid, Face())) +cpu_face_constructor_z(::ZFlatGrid) = nothing #### #### Nodes and spacings (common to every grid)... From fb54ac72e955ba975478e18d5f9817354ec798aa Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 29 Nov 2024 09:39:30 +0100 Subject: [PATCH 506/567] a couple of bugfixes --- src/Grids/vertical_coordinate.jl | 52 ++++++++++--------- .../prescribed_hydrostatic_velocity_fields.jl | 2 +- 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/src/Grids/vertical_coordinate.jl b/src/Grids/vertical_coordinate.jl index a6cdf2069a..13fbd17b5d 100644 --- a/src/Grids/vertical_coordinate.jl +++ b/src/Grids/vertical_coordinate.jl @@ -111,27 +111,35 @@ on_architecture(arch, coord::ZStarVerticalCoordinate) = on_architecture(arch, coord.∂t_e₃)) ##### -##### Vertical nodes... +##### Nodes and spacings (common to every grid)... ##### +AUG = AbstractUnderlyingGrid + @inline rnode(i, j, k, grid, ℓx, ℓy, ℓz) = rnode(k, grid, ℓz) @inline rnode(k, grid, ::Center) = getnode(grid.z.cᶜ, k) @inline rnode(k, grid, ::Face) = getnode(grid.z.cᶠ, k) # These will be extended in the Operators module @inline znode(k, grid, ℓz) = rnode(k, grid, ℓz) -@inline znode(i, j, k, grid, ℓx, ℓy, ℓz) = rnode(k, grid, ℓz) +@inline znode(i, j, k, grid, ℓx, ℓy, ℓz) = rnode(i, j, k, grid, ℓx, ℓy, ℓz) -function validate_dimension_specification(T, ξ::ZStarVerticalCoordinate, dir, N, FT) - reference = validate_dimension_specification(T, ξ.cᶠ, dir, N, FT) - args = Tuple(getproperty(ξ, prop) for prop in propertynames(ξ)) +@inline rnodes(grid::AUG, ℓz::Face; with_halos=false) = _property(grid.z.cᶠ, ℓz, topology(grid, 3), size(grid, 3), with_halos) +@inline rnodes(grid::AUG, ℓz::Center; with_halos=false) = _property(grid.z.cᶜ, ℓz, topology(grid, 3), size(grid, 3), with_halos) +@inline rnodes(grid::AUG, ℓx, ℓy, ℓz; with_halos=false) = rnodes(grid, ℓz; with_halos) - return ZStarVerticalCoordinate(reference, reference, args[3:end]...) -end +rnodes(grid::AUG, ::Nothing; kwargs...) = 1:1 +znodes(grid::AUG, ::Nothing; kwargs...) = 1:1 -# Summaries -coordinate_summary(::Bounded, z::AbstractVerticalCoordinate, name) = - @sprintf("Free-surface following with Δ%s=%s", name, prettysummary(z.Δᶜ)) +# TODO: extend in the Operators module +@inline znodes(grid::AUG, ℓz; kwargs...) = rnodes(grid, ℓz; kwargs...) +@inline znodes(grid::AUG, ℓx, ℓy, ℓz; kwargs...) = rnodes(grid, ℓx, ℓy, ℓz; kwargs...) + +function rspacings end +function zspacings end + +@inline rspacings(grid, ℓz) = rspacings(grid, nothing, nothing, ℓz) +@inline zspacings(grid, ℓz) = zspacings(grid, nothing, nothing, ℓz) #### #### z_domain (independent of ZStar or not) @@ -144,22 +152,16 @@ cpu_face_constructor_z(grid) = Array(rnodes(grid, Face())) cpu_face_constructor_z(::ZFlatGrid) = nothing #### -#### Nodes and spacings (common to every grid)... +#### Utilities #### -@inline rnodes(grid, ℓz::Face; with_halos=false) = _property(grid.z.cᶠ, ℓz, topology(grid, 3), size(grid, 3), with_halos) -@inline rnodes(grid, ℓz::Center; with_halos=false) = _property(grid.z.cᶜ, ℓz, topology(grid, 3), size(grid, 3), with_halos) -@inline rnodes(grid, ℓx, ℓy, ℓz; with_halos=false) = rnodes(grid, ℓz; with_halos) - -rnodes(grid, ::Nothing; kwargs...) = 1:1 -znodes(grid, ::Nothing; kwargs...) = 1:1 - -# TODO: extend in the Operators module -@inline znodes(grid, ℓz; kwargs...) = rnodes(grid, ℓz; kwargs...) -@inline znodes(grid, ℓx, ℓy, ℓz; kwargs...) = rnodes(grid, ℓx, ℓy, ℓz; kwargs...) +function validate_dimension_specification(T, ξ::ZStarVerticalCoordinate, dir, N, FT) + reference = validate_dimension_specification(T, ξ.cᶠ, dir, N, FT) + args = Tuple(getproperty(ξ, prop) for prop in propertynames(ξ)) -function rspacings end -function zspacings end + return ZStarVerticalCoordinate(reference, reference, args[3:end]...) +end -@inline rspacings(grid, ℓz) = rspacings(grid, nothing, nothing, ℓz) -@inline zspacings(grid, ℓz) = zspacings(grid, nothing, nothing, ℓz) \ No newline at end of file +# Summaries +coordinate_summary(::Bounded, z::AbstractVerticalCoordinate, name) = + @sprintf("Free-surface following with Δ%s=%s", name, prettysummary(z.Δᶜ)) diff --git a/src/Models/HydrostaticFreeSurfaceModels/prescribed_hydrostatic_velocity_fields.jl b/src/Models/HydrostaticFreeSurfaceModels/prescribed_hydrostatic_velocity_fields.jl index 94403dc62c..38b5ca925c 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/prescribed_hydrostatic_velocity_fields.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/prescribed_hydrostatic_velocity_fields.jl @@ -89,7 +89,7 @@ end @inline datatuple(obj::PrescribedVelocityFields) = (; u = datatuple(obj.u), v = datatuple(obj.v), w = datatuple(obj.w)) ab2_step_velocities!(::PrescribedVelocityFields, args...) = nothing -ab2_step_free_surface!(::Nothing, model, Δt, χ) = nothing +step_free_surface!(::Nothing, model, timestepper, Δt) = nothing compute_w_from_continuity!(::PrescribedVelocityFields, args...; kwargs...) = nothing validate_velocity_boundary_conditions(grid, ::PrescribedVelocityFields) = nothing From cb54a6f467737fa531bb18387b15be4475b613a4 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 29 Nov 2024 09:40:03 +0100 Subject: [PATCH 507/567] another bugfix --- .../prescribed_hydrostatic_velocity_fields.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Models/HydrostaticFreeSurfaceModels/prescribed_hydrostatic_velocity_fields.jl b/src/Models/HydrostaticFreeSurfaceModels/prescribed_hydrostatic_velocity_fields.jl index 38b5ca925c..85e919b464 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/prescribed_hydrostatic_velocity_fields.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/prescribed_hydrostatic_velocity_fields.jl @@ -98,6 +98,7 @@ extract_boundary_conditions(::PrescribedVelocityFields) = NamedTuple() free_surface_displacement_field(::PrescribedVelocityFields, ::Nothing, grid) = nothing HorizontalVelocityFields(::PrescribedVelocityFields, grid) = nothing, nothing +materialize_free_surface(::Nothing, velocities, grid) = nothing materialize_free_surface(::ExplicitFreeSurface{Nothing}, ::PrescribedVelocityFields, grid) = nothing materialize_free_surface(::ImplicitFreeSurface{Nothing}, ::PrescribedVelocityFields, grid) = nothing materialize_free_surface(::SplitExplicitFreeSurface, ::PrescribedVelocityFields, grid) = nothing From 0e0c5e7d8bf8a4a4d652f5204fda1e0efdc0d4dc Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 29 Nov 2024 10:01:04 +0100 Subject: [PATCH 508/567] it works for all free surfaces!! --- test/test_zstar_coordinate.jl | 36 ++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index 9ea6b3a5fd..a289aaa1ed 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -120,21 +120,27 @@ end for grid in grids info_msg = info_message(grid) - @testset "$info_msg" begin - @info " $info_msg" - # TODO: minimum_xspacing(grid) on a Immersed GPU grid with ZStarVerticalCoordinate - # fails because it uses too much parameter space. Figure out a way to reduce it - free_surface = SplitExplicitFreeSurface(grid; substeps = 20) - model = HydrostaticFreeSurfaceModel(; grid, - free_surface, - tracers = (:b, :c), - buoyancy = BuoyancyTracer()) - - bᵢ(x, y, z) = x < grid.Lx / 2 ? 0.06 : 0.01 - - set!(model, c = (x, y, z) -> rand(), b = bᵢ) - - test_zstar_coordinate(model, 100, 10) + + split_free_surface = SplitExplicitFreeSurface(grid; substeps = 20) + implicit_free_surface = ImplicitFreeSurface() + explicit_free_surface = ExplicitFreeSurface() + + for free_surface in [split_free_surface, implicit_free_surface, explicit_free_surface] + @testset "$info_msg" begin + @info " $info_msg of $(free_surface)" + # TODO: minimum_xspacing(grid) on a Immersed GPU grid with ZStarVerticalCoordinate + # fails because it uses too much parameter space. Figure out a way to reduce it + model = HydrostaticFreeSurfaceModel(; grid, + free_surface, + tracers = (:b, :c), + buoyancy = BuoyancyTracer()) + + bᵢ(x, y, z) = x < grid.Lx / 2 ? 0.06 : 0.01 + + set!(model, c = (x, y, z) -> rand(), b = bᵢ) + + test_zstar_coordinate(model, 100, 10) + end end end end From b85287f98bbb1d7b54abc7440dd2edc6a777fada Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 29 Nov 2024 10:01:15 +0100 Subject: [PATCH 509/567] it works for all free surfaces --- src/Grids/input_validation.jl | 5 +++ src/Grids/vertical_coordinate.jl | 7 ++-- .../z_star_vertical_spacing.jl | 39 ++++++++++++++----- validation/z_star_coordinate/lock_release.jl | 4 +- 4 files changed, 41 insertions(+), 14 deletions(-) diff --git a/src/Grids/input_validation.jl b/src/Grids/input_validation.jl index 9624086bba..1a81f7e5d6 100644 --- a/src/Grids/input_validation.jl +++ b/src/Grids/input_validation.jl @@ -145,6 +145,11 @@ function validate_dimension_specification(T, ξ::AbstractVector, dir, N, FT) return ξ end +function validate_dimension_specification(T, ξ::Tuple, dir, N, FT) + ξ[2] ≥ ξ[1] || throw(ArgumentError("$dir should have increasing values.")) + return ξ +end + function validate_dimension_specification(T, ξ::Function, dir, N, FT) ξ(N) ≥ ξ(1) || throw(ArgumentError("$dir should have increasing values.")) return ξ diff --git a/src/Grids/vertical_coordinate.jl b/src/Grids/vertical_coordinate.jl index 13fbd17b5d..bf72a19fb0 100644 --- a/src/Grids/vertical_coordinate.jl +++ b/src/Grids/vertical_coordinate.jl @@ -156,10 +156,11 @@ cpu_face_constructor_z(::ZFlatGrid) = nothing #### function validate_dimension_specification(T, ξ::ZStarVerticalCoordinate, dir, N, FT) - reference = validate_dimension_specification(T, ξ.cᶠ, dir, N, FT) - args = Tuple(getproperty(ξ, prop) for prop in propertynames(ξ)) + cᶠ = validate_dimension_specification(T, ξ.cᶠ, dir, N, FT) + cᶜ = validate_dimension_specification(T, ξ.cᶜ, dir, N, FT) + args = Tuple(getproperty(ξ, prop) for prop in propertynames(ξ)) - return ZStarVerticalCoordinate(reference, reference, args[3:end]...) + return ZStarVerticalCoordinate(cᶠ, cᶜ, args[3:end]...) end # Summaries diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 5071f80562..4e3dfb9e50 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -8,7 +8,8 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels.SplitExplicitFreeSurfaces ##### # The easy case -retrieve_barotropic_velocity(model, free_surface::SplitExplicitFreeSurface) = free_surface.barotropic_velocities +barotropic_velocities(free_surface::SplitExplicitFreeSurface) = free_surface.barotropic_velocities +barotropic_velocities(free_surface) = nothing, nothing # Fallback update_grid!(model, grid; parameters) = nothing @@ -28,15 +29,14 @@ function update_grid!(model, grid::ZStarGridOfSomeKind; parameters = :xy) launch!(architecture(grid), grid, parameters, _update_grid_scaling!, e₃ᶜᶜⁿ, e₃ᶠᶜⁿ, e₃ᶜᶠⁿ, e₃ᶠᶠⁿ, e₃ᶜᶜ⁻, ηⁿ, grid, η) - # the barotropic velocity are retrieved from the free surface model for a + # the barotropic velocities are retrieved from the free surface model for a # SplitExplicitFreeSurface and are calculated for other free surface models - # For the moment only the `SplitExplicitFreeSurface` is supported - # TODO: find a way to support other free surface models by storing the barotropic velocities shomewhere - U, V = retrieve_barotropic_velocity(model, model.free_surface) + U, V = barotropic_velocities(model.free_surface) + u, v, _ = model.velocities # Update vertical spacing with available parameters # No need to fill the halo as the scaling is updated _IN_ the halos - launch!(architecture(grid), grid, parameters, _update_grid_vertical_velocity!, ∂t_e₃, grid, U, V) + launch!(architecture(grid), grid, parameters, _update_grid_vertical_velocity!, ∂t_e₃, grid, U, V, u, v) return nothing end @@ -75,21 +75,42 @@ end end end -@kernel function _update_grid_vertical_velocity!(∂t_e₃, grid, U, V) +@kernel function _update_grid_vertical_velocity!(∂t_e₃, grid, U, V, u, v) i, j = @index(Global, NTuple) kᴺ = size(grid, 3) hᶜᶜ = static_column_depthᶜᶜᵃ(i, j, grid) # ∂(η / H)/∂t = - ∇ ⋅ ∫udz / H - δx_U = δxᶜᶜᶜ(i, j, kᴺ, grid, Δy_qᶠᶜᶜ, U) - δy_V = δyᶜᶜᶜ(i, j, kᴺ, grid, Δx_qᶜᶠᶜ, V) + δx_U = δxᶜᶜᶜ(i, j, kᴺ, grid, Δy_qᶠᶜᶜ, barotropic_U, U, u) + δy_V = δyᶜᶜᶜ(i, j, kᴺ, grid, Δx_qᶜᶠᶜ, barotropic_V, V, v) δh_U = (δx_U + δy_V) / Azᶜᶜᶜ(i, j, kᴺ, grid) @inbounds ∂t_e₃[i, j, 1] = ifelse(hᶜᶜ == 0, zero(grid), - δh_U / hᶜᶜ) end +# If U and V exist, we just take them +@inline barotropic_U(i, j, k, grid, U, u) = @inbounds U[i, j, k] +@inline barotropic_V(i, j, k, grid, V, v) = @inbounds V[i, j, k] + +# If U and V are not available, we compute them +@inline function barotropic_U(i, j, k, grid, ::Nothing, u) + U = 0 + for k in 1:size(grid, 3) + U += u[i, j, k] * Δzᶠᶜᶜ(i, j, k, grid) + end + return U +end + +@inline function barotropic_V(i, j, k, grid, ::Nothing, v) + V = 0 + for k in 1:size(grid, 3) + V += v[i, j, k] * Δzᶠᶜᶜ(i, j, k, grid) + end + return V +end + ##### ##### ZStar-specific implementation of the additional terms to be included in the momentum equations ##### diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index 3a5ef8551f..99f804e1c5 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -14,7 +14,7 @@ grid = RectilinearGrid(size = (128, 20), halo = (6, 6), topology = (Bounded, Flat, Bounded)) -# grid = ImmersedBoundaryGrid(grid, GridFittedBottom(x -> - (64kilometers - x) / 64kilometers * 20)) +grid = ImmersedBoundaryGrid(grid, GridFittedBottom(x -> - (64kilometers - x) / 64kilometers * 20)) model = HydrostaticFreeSurfaceModel(; grid, momentum_advection = WENO(order = 5), @@ -22,7 +22,7 @@ model = HydrostaticFreeSurfaceModel(; grid, buoyancy = BuoyancyTracer(), closure = nothing, tracers = :b, - free_surface = SplitExplicitFreeSurface(; substeps = 10)) + free_surface = ImplicitFreeSurface()) #SplitExplicitFreeSurface(; substeps = 10)) g = model.free_surface.gravitational_acceleration From f4681090ccc532759c355ab184fd40fbb411c4fd Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 29 Nov 2024 10:43:40 +0100 Subject: [PATCH 510/567] back to previous cpu face constructor --- src/Grids/grid_utils.jl | 1 + src/Grids/vertical_coordinate.jl | 4 ---- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Grids/grid_utils.jl b/src/Grids/grid_utils.jl index 379fafb8ac..923f465916 100644 --- a/src/Grids/grid_utils.jl +++ b/src/Grids/grid_utils.jl @@ -231,6 +231,7 @@ const f = Face() # What's going on here? @inline cpu_face_constructor_x(grid) = Array(getindex(nodes(grid, f, c, c; with_halos=true), 1)[1:size(grid, 1)+1]) @inline cpu_face_constructor_y(grid) = Array(getindex(nodes(grid, c, f, c; with_halos=true), 2)[1:size(grid, 2)+1]) +@inline cpu_face_constructor_z(grid) = Array(getindex(nodes(grid, c, c, f; with_halos=true), 2)[1:size(grid, 2)+1]) ##### ##### Convenience functions diff --git a/src/Grids/vertical_coordinate.jl b/src/Grids/vertical_coordinate.jl index bf72a19fb0..0faad3401e 100644 --- a/src/Grids/vertical_coordinate.jl +++ b/src/Grids/vertical_coordinate.jl @@ -147,10 +147,6 @@ function zspacings end z_domain(grid) = domain(topology(grid, 3)(), grid.Nz, grid.z.cᶠ) -# We construct on r not on z -cpu_face_constructor_z(grid) = Array(rnodes(grid, Face())) -cpu_face_constructor_z(::ZFlatGrid) = nothing - #### #### Utilities #### From 0c06253b0c71a50474fe397836d184fb7233b3b0 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 29 Nov 2024 10:51:21 +0100 Subject: [PATCH 511/567] change vector invariant upwinding --- src/Advection/vector_invariant_self_upwinding.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Advection/vector_invariant_self_upwinding.jl b/src/Advection/vector_invariant_self_upwinding.jl index fa60ee949a..c530fbc4b1 100644 --- a/src/Advection/vector_invariant_self_upwinding.jl +++ b/src/Advection/vector_invariant_self_upwinding.jl @@ -5,15 +5,15 @@ @inline δx_U(i, j, k, grid, u, v) = δxᶜᶜᶜ(i, j, k, grid, Ax_qᶠᶜᶜ, u) @inline δy_V(i, j, k, grid, u, v) = δyᶜᶜᶜ(i, j, k, grid, Ay_qᶜᶠᶜ, v) -@inline δx_U_plus_metric(i, j, k, grid, u, v) = δxᶜᶜᶜ(i, j, k, grid, Ax_qᶠᶜᶜ, u) + Vᶜᶜᶜ(i, j, k, grid) * ∂t_e₃(i, j, k, grid) -@inline δy_V_plus_metric(i, j, k, grid, u, v) = δyᶜᶜᶜ(i, j, k, grid, Ay_qᶜᶠᶜ, v) + Vᶜᶜᶜ(i, j, k, grid) * ∂t_e₃(i, j, k, grid) +@inline δx_U_plus_metric(i, j, k, grid, u, v) = δxᶜᶜᶜ(i, j, k, grid, Ax_qᶠᶜᶜ, u) + V_times_∂t_e₃(i, j, k, grid) +@inline δy_V_plus_metric(i, j, k, grid, u, v) = δyᶜᶜᶜ(i, j, k, grid, Ay_qᶜᶠᶜ, v) + V_times_∂t_e₃(i, j, k, grid) # Velocity smoothness for divergence upwinding @inline U_smoothness(i, j, k, grid, u, v) = ℑxᶜᵃᵃ(i, j, k, grid, Ax_qᶠᶜᶜ, u) @inline V_smoothness(i, j, k, grid, u, v) = ℑyᵃᶜᵃ(i, j, k, grid, Ay_qᶜᶠᶜ, v) # Divergence smoothness for divergence upwinding -@inline divergence_smoothness(i, j, k, grid, u, v) = δx_U(i, j, k, grid, u, v) + δy_V(i, j, k, grid, u, v) + Vᶜᶜᶜ(i, j, k, grid) * ∂t_e₃(i, j, k, grid) +@inline divergence_smoothness(i, j, k, grid, u, v) = δx_U(i, j, k, grid, u, v) + δy_V(i, j, k, grid, u, v) + V_times_∂t_e₃(i, j, k, grid) @inline function upwinded_divergence_flux_Uᶠᶜᶜ(i, j, k, grid, scheme::VectorInvariantSelfVerticalUpwinding, u, v) From 5a2f91b69da12097757444dc79b23f9fc617802f Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 29 Nov 2024 10:58:20 +0100 Subject: [PATCH 512/567] no need for a loop --- ...te_hydrostatic_free_surface_model_state.jl | 26 +++++++++---------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl index 3357c36834..e5a1933dd6 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl @@ -71,27 +71,25 @@ function mask_immersed_model_fields!(model, grid) return nothing end -function compute_auxiliaries!(model::HydrostaticFreeSurfaceModel; w_parameters = tuple(w_kernel_parameters(model.grid)), - p_parameters = tuple(p_kernel_parameters(model.grid)), - κ_parameters = tuple(:xyz)) +function compute_auxiliaries!(model::HydrostaticFreeSurfaceModel; w_params = w_kernel_parameters(model.grid), + p_params = p_kernel_parameters(model.grid), + κ_params = :xyz) grid = model.grid closure = model.closure tracers = model.tracers diffusivity = model.diffusivity_fields - for (wpar, ppar, κpar) in zip(w_parameters, p_parameters, κ_parameters) - # Update the grid - update_grid!(model, grid; parameters = wpar) - unscale_tracers!(tracers, grid; parameters = wpar) + # Update the grid and unscale the tracers + update_grid!(model, grid; parameters = w_params) + unscale_tracers!(tracers, grid; parameters = w_params) - # Update the other auxiliary terms - compute_w_from_continuity!(model; parameters = wpar) - compute_diffusivities!(diffusivity, closure, model; parameters = κpar) - update_hydrostatic_pressure!(model.pressure.pHY′, architecture(grid), - grid, model.buoyancy, model.tracers; - parameters = ppar) - end + # Update the other auxiliary terms + compute_w_from_continuity!(model; parameters = w_params) + compute_diffusivities!(diffusivity, closure, model; parameters = κ_params) + update_hydrostatic_pressure!(model.pressure.pHY′, architecture(grid), + grid, model.buoyancy, model.tracers; + parameters = p_params) return nothing end From b5a8ed80a1ecaf43b6fd02e8e315114d44177b8d Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 29 Nov 2024 11:04:57 +0100 Subject: [PATCH 513/567] simplify --- .../update_hydrostatic_free_surface_model_state.jl | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl index e5a1933dd6..034cd0b6d8 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl @@ -84,12 +84,10 @@ function compute_auxiliaries!(model::HydrostaticFreeSurfaceModel; w_params = w_k update_grid!(model, grid; parameters = w_params) unscale_tracers!(tracers, grid; parameters = w_params) - # Update the other auxiliary terms + # Update the other auxiliary terms (w, κ, p) compute_w_from_continuity!(model; parameters = w_params) compute_diffusivities!(diffusivity, closure, model; parameters = κ_params) - update_hydrostatic_pressure!(model.pressure.pHY′, architecture(grid), - grid, model.buoyancy, model.tracers; - parameters = p_params) + update_hydrostatic_pressure!(grid, model; parameters = p_params) return nothing end From e3ac54c27502917bf6d837619c5595d1a6d40025 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 29 Nov 2024 11:28:57 +0100 Subject: [PATCH 514/567] couple of fixes --- .../update_hydrostatic_free_surface_model_state.jl | 14 ++++++++++---- .../z_star_vertical_spacing.jl | 4 ++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl index 034cd0b6d8..43bc612e85 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl @@ -79,15 +79,21 @@ function compute_auxiliaries!(model::HydrostaticFreeSurfaceModel; w_params = w_k closure = model.closure tracers = model.tracers diffusivity = model.diffusivity_fields + buoyancy = model.buoyancy + + P = model.pressure.pHY′ + arch = architecture(grid) # Update the grid and unscale the tracers update_grid!(model, grid; parameters = w_params) unscale_tracers!(tracers, grid; parameters = w_params) - - # Update the other auxiliary terms (w, κ, p) + + # Advance diagnostic quantities compute_w_from_continuity!(model; parameters = w_params) - compute_diffusivities!(diffusivity, closure, model; parameters = κ_params) - update_hydrostatic_pressure!(grid, model; parameters = p_params) + update_hydrostatic_pressure!(P, arch, grid, buoyancy, tracers; parameters = p_params) + # Update closure diffusivities + compute_diffusivities!(diffusivity, closure, model; parameters = κ_params) + return nothing end diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 4e3dfb9e50..69115d0410 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -1,14 +1,14 @@ using Oceananigans.Grids using Oceananigans.ImmersedBoundaries: ZStarGridOfSomeKind -using Oceananigans.Models.HydrostaticFreeSurfaceModels.SplitExplicitFreeSurfaces: compute_barotropic_mode! - ##### ##### ZStar-specific vertical spacings update ##### # The easy case barotropic_velocities(free_surface::SplitExplicitFreeSurface) = free_surface.barotropic_velocities + +# The "harder" case, barotropic velocities are computed on the fly barotropic_velocities(free_surface) = nothing, nothing # Fallback From 5e3a411f62a66ddad884637b151e0c26bc77824d Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 2 Dec 2024 11:05:40 +0100 Subject: [PATCH 515/567] some improvements --- src/Grids/grid_generation.jl | 5 ++++ src/Grids/grid_utils.jl | 1 - src/Grids/input_validation.jl | 25 ++++++++----------- src/Grids/rectilinear_grid.jl | 2 +- src/Grids/vertical_coordinate.jl | 8 ++++++ src/ImmersedBoundaries/zstar_immersed_grid.jl | 18 ++++++------- .../barotropic_split_explicit_corrector.jl | 12 +++++---- .../step_split_explicit_free_surface.jl | 4 +-- .../set_hydrostatic_free_surface_model.jl | 2 +- .../z_star_vertical_spacing.jl | 10 ++++---- test/test_zstar_coordinate.jl | 8 +++--- 11 files changed, 51 insertions(+), 44 deletions(-) diff --git a/src/Grids/grid_generation.jl b/src/Grids/grid_generation.jl index bb8a188aed..51831b4313 100644 --- a/src/Grids/grid_generation.jl +++ b/src/Grids/grid_generation.jl @@ -188,5 +188,10 @@ function generate_coordinate(FT, topo, size, halo, coordinate::ZStarVerticalCoor ηⁿ = new_data(FT, arch, (Center, Center, Nothing), args...) ∂t_e₃ = new_data(FT, arch, (Center, Center, Nothing), args...) + # Fill all the scalings with one (at rest coordinate) + for e₃ in (e₃ᶜᶜ⁻, e₃ᶜᶜⁿ, e₃ᶠᶜⁿ, e₃ᶜᶠⁿ, e₃ᶠᶠⁿ) + fill!(e₃, 1) + end + return Lr, ZStarVerticalCoordinate(rᵃᵃᶠ, rᵃᵃᶜ, Δrᵃᵃᶠ, Δrᵃᵃᶜ, ηⁿ, e₃ᶜᶜⁿ, e₃ᶠᶜⁿ, e₃ᶜᶠⁿ, e₃ᶠᶠⁿ, e₃ᶜᶜ⁻, ∂t_e₃) end diff --git a/src/Grids/grid_utils.jl b/src/Grids/grid_utils.jl index 923f465916..379fafb8ac 100644 --- a/src/Grids/grid_utils.jl +++ b/src/Grids/grid_utils.jl @@ -231,7 +231,6 @@ const f = Face() # What's going on here? @inline cpu_face_constructor_x(grid) = Array(getindex(nodes(grid, f, c, c; with_halos=true), 1)[1:size(grid, 1)+1]) @inline cpu_face_constructor_y(grid) = Array(getindex(nodes(grid, c, f, c; with_halos=true), 2)[1:size(grid, 2)+1]) -@inline cpu_face_constructor_z(grid) = Array(getindex(nodes(grid, c, c, f; with_halos=true), 2)[1:size(grid, 2)+1]) ##### ##### Convenience functions diff --git a/src/Grids/input_validation.jl b/src/Grids/input_validation.jl index 1a81f7e5d6..0ef5a9e018 100644 --- a/src/Grids/input_validation.jl +++ b/src/Grids/input_validation.jl @@ -90,16 +90,6 @@ function validate_halo(TX, TY, TZ, size, halo) return halo end -function validate_dimension_specification(T, ξ, dir, N, FT) - - isnothing(ξ) && throw(ArgumentError("Must supply extent or $dir keyword when $dir-direction is $T")) - length(ξ) == 2 || throw(ArgumentError("$dir length($ξ) must be 2.")) - all(isa.(ξ, Number)) || throw(ArgumentError("$dir=$ξ should contain numbers.")) - ξ[2] ≥ ξ[1] || throw(ArgumentError("$dir=$ξ should be an increasing interval.")) - - return FT.(ξ) -end - function validate_rectilinear_domain(TX, TY, TZ, FT, size, extent, x, y, z) # Find domain endpoints or domain extent, depending on user input: @@ -126,6 +116,16 @@ function validate_rectilinear_domain(TX, TY, TZ, FT, size, extent, x, y, z) return x, y, z end +function validate_dimension_specification(T, ξ, dir, N, FT) + + isnothing(ξ) && throw(ArgumentError("Must supply extent or $dir keyword when $dir-direction is $T")) + length(ξ) == 2 || throw(ArgumentError("$dir length($ξ) must be 2.")) + all(isa.(ξ, Number)) || throw(ArgumentError("$dir=$ξ should contain numbers.")) + ξ[2] ≥ ξ[1] || throw(ArgumentError("$dir=$ξ should be an increasing interval.")) + + return FT.(ξ) +end + function validate_dimension_specification(T, ξ::AbstractVector, dir, N, FT) ξ = FT.(ξ) @@ -145,11 +145,6 @@ function validate_dimension_specification(T, ξ::AbstractVector, dir, N, FT) return ξ end -function validate_dimension_specification(T, ξ::Tuple, dir, N, FT) - ξ[2] ≥ ξ[1] || throw(ArgumentError("$dir should have increasing values.")) - return ξ -end - function validate_dimension_specification(T, ξ::Function, dir, N, FT) ξ(N) ≥ ξ(1) || throw(ArgumentError("$dir should have increasing values.")) return ξ diff --git a/src/Grids/rectilinear_grid.jl b/src/Grids/rectilinear_grid.jl index f9af2b13a2..fde840ebd3 100644 --- a/src/Grids/rectilinear_grid.jl +++ b/src/Grids/rectilinear_grid.jl @@ -412,7 +412,7 @@ function with_halo(halo, grid::RectilinearGrid) halo = pop_flat_elements(halo, topology(grid)) kwargs[:halo] = halo arch = args[:architecture] - FT = args[:number_type] + FT = args[:number_type] return RectilinearGrid(arch, FT; kwargs...) end diff --git a/src/Grids/vertical_coordinate.jl b/src/Grids/vertical_coordinate.jl index 0faad3401e..e490c85875 100644 --- a/src/Grids/vertical_coordinate.jl +++ b/src/Grids/vertical_coordinate.jl @@ -147,6 +147,14 @@ function zspacings end z_domain(grid) = domain(topology(grid, 3)(), grid.Nz, grid.z.cᶠ) +@inline cpu_face_constructor_z(grid) = Array(getindex(nodes(grid, c, c, f; with_halos=true), 3)[1:size(grid, 3)+1]) + +# In case of an AbstractZStarGrid return a ZStarVerticalCoordinate +@inline function cpu_face_constructor_z(grid::AbstractZStarGrid) + r_faces = Array(getindex(nodes(grid, c, c, f; with_halos=true), 3)[1:size(grid, 3)+1]) + return ZStarVerticalCoordinate(r_faces) +end + #### #### Utilities #### diff --git a/src/ImmersedBoundaries/zstar_immersed_grid.jl b/src/ImmersedBoundaries/zstar_immersed_grid.jl index 841cae216d..4596c2d8ad 100644 --- a/src/ImmersedBoundaries/zstar_immersed_grid.jl +++ b/src/ImmersedBoundaries/zstar_immersed_grid.jl @@ -11,16 +11,16 @@ import Oceananigans.Operators: e₃ⁿ, e₃⁻, ∂t_e₃ const ZStarImmersedGrid = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:AbstractZStarGrid} const ZStarGridOfSomeKind = Union{ZStarImmersedGrid, AbstractZStarGrid} -@inline dynamic_column_depthᶜᶜᵃ(i, j, grid::ZStarGridOfSomeKind, η) = @inbounds static_column_depthᶜᶜᵃ(i, j, grid) + η[i, j, grid.Nz+1] -@inline dynamic_column_depthᶜᶠᵃ(i, j, grid::ZStarGridOfSomeKind, η) = @inbounds static_column_depthᶜᶠᵃ(i, j, grid) + ℑxᶠᵃᵃ(i, j, grid.Nz+1, grid, η) -@inline dynamic_column_depthᶠᶜᵃ(i, j, grid::ZStarGridOfSomeKind, η) = @inbounds static_column_depthᶠᶜᵃ(i, j, grid) + ℑyᵃᶠᵃ(i, j, grid.Nz+1, grid, η) -@inline dynamic_column_depthᶠᶠᵃ(i, j, grid::ZStarGridOfSomeKind, η) = @inbounds static_column_depthᶠᶠᵃ(i, j, grid) + ℑxyᶠᶠᵃ(i, j, grid.Nz+1, grid, η) +@inline dynamic_column_depthᶜᶜᵃ(i, j, k, grid::ZStarGridOfSomeKind, η) = @inbounds static_column_depthᶜᶜᵃ(i, j, grid) + η[i, j, k] +@inline dynamic_column_depthᶜᶠᵃ(i, j, k, grid::ZStarGridOfSomeKind, η) = @inbounds static_column_depthᶠᶜᵃ(i, j, grid) + ℑyᵃᶠᵃ(i, j, k, grid, η) +@inline dynamic_column_depthᶠᶜᵃ(i, j, k, grid::ZStarGridOfSomeKind, η) = @inbounds static_column_depthᶜᶠᵃ(i, j, grid) + ℑxᶠᵃᵃ(i, j, k, grid, η) +@inline dynamic_column_depthᶠᶠᵃ(i, j, k, grid::ZStarGridOfSomeKind, η) = @inbounds static_column_depthᶠᶠᵃ(i, j, grid) + ℑxyᶠᶠᵃ(i, j, k, grid, η) -# Convenience -@inline dynamic_column_depthᶜᶜᵃ(i, j, grid::ZStarGridOfSomeKind) = dynamic_column_depthᶜᶜᵃ(i, j, grid, grid.z.ηⁿ) -@inline dynamic_column_depthᶜᶠᵃ(i, j, grid::ZStarGridOfSomeKind) = dynamic_column_depthᶜᶠᵃ(i, j, grid, grid.z.ηⁿ) -@inline dynamic_column_depthᶠᶜᵃ(i, j, grid::ZStarGridOfSomeKind) = dynamic_column_depthᶠᶜᵃ(i, j, grid, grid.z.ηⁿ) -@inline dynamic_column_depthᶠᶠᵃ(i, j, grid::ZStarGridOfSomeKind) = dynamic_column_depthᶠᶠᵃ(i, j, grid, grid.z.ηⁿ) +# Convenience functions +@inline dynamic_column_depthᶜᶜᵃ(i, j, grid::ZStarGridOfSomeKind) = dynamic_column_depthᶜᶜᵃ(i, j, 1, grid, grid.z.ηⁿ) +@inline dynamic_column_depthᶜᶠᵃ(i, j, grid::ZStarGridOfSomeKind) = dynamic_column_depthᶜᶠᵃ(i, j, 1, grid, grid.z.ηⁿ) +@inline dynamic_column_depthᶠᶜᵃ(i, j, grid::ZStarGridOfSomeKind) = dynamic_column_depthᶠᶜᵃ(i, j, 1, grid, grid.z.ηⁿ) +@inline dynamic_column_depthᶠᶠᵃ(i, j, grid::ZStarGridOfSomeKind) = dynamic_column_depthᶠᶠᵃ(i, j, 1, grid, grid.z.ηⁿ) # Fallbacks @inline e₃ⁿ(i, j, k, ibg::IBG, ℓx, ℓy, ℓz) = e₃ⁿ(i, j, k, ibg.underlying_grid, ℓx, ℓy, ℓz) diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_split_explicit_corrector.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_split_explicit_corrector.jl index 40b60793f3..20a6dca582 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_split_explicit_corrector.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_split_explicit_corrector.jl @@ -11,9 +11,10 @@ end end @inline function barotropic_mode_kernel!(U, V, i, j, grid, u, v, η) - - sᶠᶜ = dynamic_column_depthᶠᶜᵃ(i, j, grid, η) / static_column_depthᶠᶜᵃ(i, j, grid) - sᶜᶠ = dynamic_column_depthᶜᶠᵃ(i, j, grid, η) / static_column_depthᶜᶠᵃ(i, j, grid) + k_top = size(grid, 3) + 1 + + sᶠᶜ = dynamic_column_depthᶠᶜᵃ(i, j, k_top, grid, η) / static_column_depthᶠᶜᵃ(i, j, grid) + sᶜᶠ = dynamic_column_depthᶜᶠᵃ(i, j, k_top, grid, η) / static_column_depthᶜᶠᵃ(i, j, grid) @inbounds U[i, j, 1] = Δrᶠᶜᶜ(i, j, 1, grid) * u[i, j, 1] * sᶠᶜ @inbounds V[i, j, 1] = Δrᶜᶠᶜ(i, j, 1, grid) * v[i, j, 1] * sᶜᶠ @@ -34,10 +35,11 @@ end @kernel function _barotropic_split_explicit_corrector!(u, v, U, V, U̅, V̅, η, grid) i, j, k = @index(Global, NTuple) + k_top = size(grid, 3) + 1 @inbounds begin - Hᶠᶜ = dynamic_column_depthᶠᶜᵃ(i, j, grid, η) - Hᶜᶠ = dynamic_column_depthᶜᶠᵃ(i, j, grid, η) + Hᶠᶜ = dynamic_column_depthᶠᶜᵃ(i, j, k_top, grid, η) + Hᶜᶠ = dynamic_column_depthᶜᶠᵃ(i, j, k_top, grid, η) u[i, j, k] = u[i, j, k] + (U[i, j, 1] - U̅[i, j, 1]) / Hᶠᶜ v[i, j, k] = v[i, j, k] + (V[i, j, 1] - V̅[i, j, 1]) / Hᶜᶠ diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/step_split_explicit_free_surface.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/step_split_explicit_free_surface.jl index aefcd89f8f..65ecd3fff6 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/step_split_explicit_free_surface.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/step_split_explicit_free_surface.jl @@ -26,8 +26,8 @@ end store_previous_velocities!(timestepper, i, j, 1, U) store_previous_velocities!(timestepper, i, j, 1, V) - Hᶠᶜ = dynamic_column_depthᶠᶜᵃ(i, j, grid, η) - Hᶜᶠ = dynamic_column_depthᶜᶠᵃ(i, j, grid, η) + Hᶠᶜ = dynamic_column_depthᶠᶜᵃ(i, j, k_top, grid, η) + Hᶜᶠ = dynamic_column_depthᶜᶠᵃ(i, j, k_top, grid, η) @inbounds begin # ∂τ(U) = - ∇η + G diff --git a/src/Models/HydrostaticFreeSurfaceModels/set_hydrostatic_free_surface_model.jl b/src/Models/HydrostaticFreeSurfaceModels/set_hydrostatic_free_surface_model.jl index f216d312c7..d54c807e41 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/set_hydrostatic_free_surface_model.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/set_hydrostatic_free_surface_model.jl @@ -60,8 +60,8 @@ model.velocities.u @apply_regionally set!(ϕ, value) end - initialize!(model) update_state!(model; compute_tendencies = false) + initialize!(model) return nothing end diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 69115d0410..2e5828461f 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -43,17 +43,17 @@ end @kernel function _update_grid_scaling!(e₃ᶜᶜⁿ, e₃ᶠᶜⁿ, e₃ᶜᶠⁿ, e₃ᶠᶠⁿ, e₃ᶜᶜ⁻, ηⁿ, grid, η) i, j = @index(Global, NTuple) - kᴺ = size(grid, 3) + k_top = size(grid, 3) + 1 hᶜᶜ = static_column_depthᶜᶜᵃ(i, j, grid) hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) hᶠᶠ = static_column_depthᶠᶠᵃ(i, j, grid) - Hᶜᶜ = dynamic_column_depthᶜᶜᵃ(i, j, grid, η) - Hᶠᶜ = dynamic_column_depthᶠᶜᵃ(i, j, grid, η) - Hᶜᶠ = dynamic_column_depthᶜᶠᵃ(i, j, grid, η) - Hᶠᶠ = dynamic_column_depthᶠᶠᵃ(i, j, grid, η) + Hᶜᶜ = dynamic_column_depthᶜᶜᵃ(i, j, k_top, grid, η) + Hᶠᶜ = dynamic_column_depthᶠᶜᵃ(i, j, k_top, grid, η) + Hᶜᶠ = dynamic_column_depthᶜᶠᵃ(i, j, k_top, grid, η) + Hᶠᶠ = dynamic_column_depthᶠᶠᵃ(i, j, k_top, grid, η) @inbounds begin e₃ᶜᶜ = ifelse(hᶜᶜ == 0, one(grid), Hᶜᶜ / hᶜᶜ) diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index a289aaa1ed..89af20e963 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -1,6 +1,7 @@ include("dependencies_for_runtests.jl") using Random +using Oceananigans: initialize! using Oceananigans.ImmersedBoundaries: PartialCellBottom function test_zstar_coordinate(model, Ni, Δt) @@ -45,7 +46,7 @@ const F = Face z = ZStarVerticalCoordinate(-20, 0) - grid = RectilinearGrid(size = (2, 1, 20), + grid = RectilinearGrid(size = (2, 2, 20), x = (0, 2), y = (0, 1), z = z, @@ -59,12 +60,9 @@ const F = Face @test dynamic_column_depthᶜᶜᵃ(1, 1, grid) == 10 @test static_column_depthᶜᶜᵃ(1, 1, grid) == 10 - set!(model, η = [1, 2]) + set!(model, η = [1 1; 2 2]) set!(model, u = (x, y, z) -> x) - initialize!(model) - update_state!(model) - @test e₃ⁿ(1, 1, 1, grid, C(), C(), C()) == 11 / 10 @test e₃ⁿ(2, 1, 1, grid, C(), C(), C()) == 12 / 10 @test e₃⁻(1, 1, 1, grid, C(), C(), C()) == 1 From c309254a925a717bedbe6330f387ffe0889bb4cd Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 2 Dec 2024 13:33:27 +0100 Subject: [PATCH 516/567] regression tests should pass --- src/Grids/grid_utils.jl | 8 ++++---- src/ImmersedBoundaries/zstar_immersed_grid.jl | 7 ++++++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/Grids/grid_utils.jl b/src/Grids/grid_utils.jl index 379fafb8ac..d6cc7c87d1 100644 --- a/src/Grids/grid_utils.jl +++ b/src/Grids/grid_utils.jl @@ -326,10 +326,10 @@ coordinate_summary(topo, Δ::Union{AbstractVector, AbstractMatrix}, name) = @inline static_column_depthᶠᶠᵃ(i, j, grid) = grid.Lz # Will be extended in the `ImmersedBoundaries` module for any ZStar grid type -@inline dynamic_column_depthᶜᶜᵃ(i, j, grid, η) = static_column_depthᶜᶜᵃ(i, j, grid) -@inline dynamic_column_depthᶜᶠᵃ(i, j, grid, η) = static_column_depthᶜᶠᵃ(i, j, grid) -@inline dynamic_column_depthᶠᶜᵃ(i, j, grid, η) = static_column_depthᶠᶜᵃ(i, j, grid) -@inline dynamic_column_depthᶠᶠᵃ(i, j, grid, η) = static_column_depthᶠᶠᵃ(i, j, grid) +@inline dynamic_column_depthᶜᶜᵃ(i, j, k, grid, η) = static_column_depthᶜᶜᵃ(i, j, grid) +@inline dynamic_column_depthᶜᶠᵃ(i, j, k, grid, η) = static_column_depthᶜᶠᵃ(i, j, grid) +@inline dynamic_column_depthᶠᶜᵃ(i, j, k, grid, η) = static_column_depthᶠᶜᵃ(i, j, grid) +@inline dynamic_column_depthᶠᶠᵃ(i, j, k, grid, η) = static_column_depthᶠᶠᵃ(i, j, grid) ##### ##### Spherical geometry diff --git a/src/ImmersedBoundaries/zstar_immersed_grid.jl b/src/ImmersedBoundaries/zstar_immersed_grid.jl index 4596c2d8ad..385db35497 100644 --- a/src/ImmersedBoundaries/zstar_immersed_grid.jl +++ b/src/ImmersedBoundaries/zstar_immersed_grid.jl @@ -16,7 +16,12 @@ const ZStarGridOfSomeKind = Union{ZStarImmersedGrid, AbstractZStarGrid} @inline dynamic_column_depthᶠᶜᵃ(i, j, k, grid::ZStarGridOfSomeKind, η) = @inbounds static_column_depthᶜᶠᵃ(i, j, grid) + ℑxᶠᵃᵃ(i, j, k, grid, η) @inline dynamic_column_depthᶠᶠᵃ(i, j, k, grid::ZStarGridOfSomeKind, η) = @inbounds static_column_depthᶠᶠᵃ(i, j, grid) + ℑxyᶠᶠᵃ(i, j, k, grid, η) -# Convenience functions +# Convenience methods +@inline dynamic_column_depthᶜᶜᵃ(i, j, grid) = static_column_depthᶜᶜᵃ(i, j, grid) +@inline dynamic_column_depthᶜᶠᵃ(i, j, grid) = static_column_depthᶜᶠᵃ(i, j, grid) +@inline dynamic_column_depthᶠᶜᵃ(i, j, grid) = static_column_depthᶠᶜᵃ(i, j, grid) +@inline dynamic_column_depthᶠᶠᵃ(i, j, grid) = static_column_depthᶠᶠᵃ(i, j, grid) + @inline dynamic_column_depthᶜᶜᵃ(i, j, grid::ZStarGridOfSomeKind) = dynamic_column_depthᶜᶜᵃ(i, j, 1, grid, grid.z.ηⁿ) @inline dynamic_column_depthᶜᶠᵃ(i, j, grid::ZStarGridOfSomeKind) = dynamic_column_depthᶜᶠᵃ(i, j, 1, grid, grid.z.ηⁿ) @inline dynamic_column_depthᶠᶜᵃ(i, j, grid::ZStarGridOfSomeKind) = dynamic_column_depthᶠᶜᵃ(i, j, 1, grid, grid.z.ηⁿ) From e55cac56e38a8ef8f7f007c090716e2221380c11 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 2 Dec 2024 14:12:29 +0100 Subject: [PATCH 517/567] rework the constructor --- src/Grids/latitude_longitude_grid.jl | 5 +---- src/Grids/orthogonal_spherical_shell_grid.jl | 5 +---- src/Grids/rectilinear_grid.jl | 5 +---- src/Grids/vertical_coordinate.jl | 15 +++++++++++---- 4 files changed, 14 insertions(+), 16 deletions(-) diff --git a/src/Grids/latitude_longitude_grid.jl b/src/Grids/latitude_longitude_grid.jl index 9fa86f6fdb..6c12486e6e 100644 --- a/src/Grids/latitude_longitude_grid.jl +++ b/src/Grids/latitude_longitude_grid.jl @@ -590,10 +590,7 @@ rname(::LLG) = :z function nodes(grid::LLG, ℓx, ℓy, ℓz; reshape=false, with_halos=false) λ = λnodes(grid, ℓx, ℓy, ℓz; with_halos) φ = φnodes(grid, ℓx, ℓy, ℓz; with_halos) - # We use rnodes here. This is intentional. znodes return the actual zcoordinate - # which might be moving in case of a ZStarVerticalCoordinate. rnodes return the - # reference zcoordinate which is constant. - z = rnodes(grid, ℓx, ℓy, ℓz; with_halos) + z = znodes(grid, ℓx, ℓy, ℓz; with_halos) if reshape # Here we have to deal with the fact that Flat directions may have diff --git a/src/Grids/orthogonal_spherical_shell_grid.jl b/src/Grids/orthogonal_spherical_shell_grid.jl index 8d6dc67eff..93db54f105 100644 --- a/src/Grids/orthogonal_spherical_shell_grid.jl +++ b/src/Grids/orthogonal_spherical_shell_grid.jl @@ -1094,10 +1094,7 @@ end function nodes(grid::OSSG, ℓx, ℓy, ℓz; reshape=false, with_halos=false) λ = λnodes(grid, ℓx, ℓy, ℓz; with_halos) φ = φnodes(grid, ℓx, ℓy, ℓz; with_halos) - # We use rnodes here. This is intentional. znodes return the actual zcoordinate - # which might be moving in case of a ZStarVerticalCoordinate. rnodes return the - # reference zcoordinate which is constant. - z = rnodes(grid, ℓx, ℓy, ℓz; with_halos) + z = znodes(grid, ℓx, ℓy, ℓz; with_halos) if reshape # λ and φ are 2D arrays diff --git a/src/Grids/rectilinear_grid.jl b/src/Grids/rectilinear_grid.jl index fde840ebd3..a6bb01cc4b 100644 --- a/src/Grids/rectilinear_grid.jl +++ b/src/Grids/rectilinear_grid.jl @@ -456,10 +456,7 @@ rname(::RG) = :z function nodes(grid::RectilinearGrid, ℓx, ℓy, ℓz; reshape=false, with_halos=false) x = xnodes(grid, ℓx, ℓy, ℓz; with_halos) y = ynodes(grid, ℓx, ℓy, ℓz; with_halos) - # We use rnodes here. This is intentional. znodes return the actual zcoordinate - # which might be moving in case of a ZStarVerticalCoordinate. rnodes return the - # reference zcoordinate which is constant. - z = rnodes(grid, ℓx, ℓy, ℓz; with_halos) + z = znodes(grid, ℓx, ℓy, ℓz; with_halos) if reshape # Here we have to deal with the fact that Flat directions may have diff --git a/src/Grids/vertical_coordinate.jl b/src/Grids/vertical_coordinate.jl index e490c85875..d70f117650 100644 --- a/src/Grids/vertical_coordinate.jl +++ b/src/Grids/vertical_coordinate.jl @@ -60,14 +60,17 @@ ZStarVerticalCoordinate(r⁻::Number, r⁺::Number) = ZStarVerticalCoordinate((r #### const RegularStaticVerticalCoordinate = StaticVerticalCoordinate{<:Any, <:Number} -const RegularZstarVerticalCoordinate = ZStarVerticalCoordinate{<:Any, <:Number} +const RegularZStarVerticalCoordinate = ZStarVerticalCoordinate{<:Any, <:Number} -const RegularVerticalCoordinate = Union{RegularStaticVerticalCoordinate, RegularZstarVerticalCoordinate} +const RegularVerticalCoordinate = Union{RegularStaticVerticalCoordinate, RegularZStarVerticalCoordinate} const AbstractZStarGrid = AbstractUnderlyingGrid{<:Any, <:Any, <:Any, <:Bounded, <:ZStarVerticalCoordinate} const AbstractStaticGrid = AbstractUnderlyingGrid{<:Any, <:Any, <:Any, <:Any, <:StaticVerticalCoordinate} const RegularVerticalGrid = AbstractUnderlyingGrid{<:Any, <:Any, <:Any, <:Any, <:RegularVerticalCoordinate} +const RegularStaticVerticalGrid = AbstractUnderlyingGrid{<:Any, <:Any, <:Any, <:Any, <:RegularStaticVerticalCoordinate} +const RegularZStarVerticalGrid = AbstractUnderlyingGrid{<:Any, <:Any, <:Any, <:Any, <:RegularZStarVerticalCoordinate} + #### #### Adapt and on_architecture #### @@ -147,14 +150,18 @@ function zspacings end z_domain(grid) = domain(topology(grid, 3)(), grid.Nz, grid.z.cᶠ) -@inline cpu_face_constructor_z(grid) = Array(getindex(nodes(grid, c, c, f; with_halos=true), 3)[1:size(grid, 3)+1]) +@inline cpu_face_constructor_z(grid) = on_architecture(CPU(), rnodes(grid, Face())) # In case of an AbstractZStarGrid return a ZStarVerticalCoordinate @inline function cpu_face_constructor_z(grid::AbstractZStarGrid) - r_faces = Array(getindex(nodes(grid, c, c, f; with_halos=true), 3)[1:size(grid, 3)+1]) + r_faces = on_architecture(CPU(), rnodes(grid, Face())) return ZStarVerticalCoordinate(r_faces) end +# Easier for regular grids +@inline cpu_face_constructor_z(grid::RegularStaticVerticalGrid) = z_domain(grid) +@inline cpu_face_constructor_z(grid::RegularZStarVerticalGrid) = ZStarVerticalCoordinate(z_domain(grid)) + #### #### Utilities #### From ff2e3c00a254bb113600ab4f7da787b3cdc7f87a Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 2 Dec 2024 14:14:22 +0100 Subject: [PATCH 518/567] simplify a bit --- src/Grids/vertical_coordinate.jl | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/src/Grids/vertical_coordinate.jl b/src/Grids/vertical_coordinate.jl index d70f117650..279c70fa27 100644 --- a/src/Grids/vertical_coordinate.jl +++ b/src/Grids/vertical_coordinate.jl @@ -68,9 +68,6 @@ const AbstractZStarGrid = AbstractUnderlyingGrid{<:Any, <:Any, <:Any, <:Bounde const AbstractStaticGrid = AbstractUnderlyingGrid{<:Any, <:Any, <:Any, <:Any, <:StaticVerticalCoordinate} const RegularVerticalGrid = AbstractUnderlyingGrid{<:Any, <:Any, <:Any, <:Any, <:RegularVerticalCoordinate} -const RegularStaticVerticalGrid = AbstractUnderlyingGrid{<:Any, <:Any, <:Any, <:Any, <:RegularStaticVerticalCoordinate} -const RegularZStarVerticalGrid = AbstractUnderlyingGrid{<:Any, <:Any, <:Any, <:Any, <:RegularZStarVerticalCoordinate} - #### #### Adapt and on_architecture #### @@ -150,17 +147,11 @@ function zspacings end z_domain(grid) = domain(topology(grid, 3)(), grid.Nz, grid.z.cᶠ) -@inline cpu_face_constructor_z(grid) = on_architecture(CPU(), rnodes(grid, Face())) - -# In case of an AbstractZStarGrid return a ZStarVerticalCoordinate -@inline function cpu_face_constructor_z(grid::AbstractZStarGrid) - r_faces = on_architecture(CPU(), rnodes(grid, Face())) - return ZStarVerticalCoordinate(r_faces) -end +@inline rfaces(grid::RegularGrid) = z_domain(grid) +@inline rfaces(grid) = on_architecture(CPU(), rnodes(grid, Face())) -# Easier for regular grids -@inline cpu_face_constructor_z(grid::RegularStaticVerticalGrid) = z_domain(grid) -@inline cpu_face_constructor_z(grid::RegularZStarVerticalGrid) = ZStarVerticalCoordinate(z_domain(grid)) +@inline cpu_face_constructor_z(grid) = rfaces(grid) +@inline cpu_face_constructor_z(grid::AbstractZStarGrid) = ZStarVerticalCoordinate(rfaces(grid)) #### #### Utilities From 322373ed73f885112d8a850718b34b4fee3a86c8 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 2 Dec 2024 14:47:39 +0100 Subject: [PATCH 519/567] change face constructor --- src/Grids/vertical_coordinate.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Grids/vertical_coordinate.jl b/src/Grids/vertical_coordinate.jl index 279c70fa27..62330dceb5 100644 --- a/src/Grids/vertical_coordinate.jl +++ b/src/Grids/vertical_coordinate.jl @@ -147,11 +147,11 @@ function zspacings end z_domain(grid) = domain(topology(grid, 3)(), grid.Nz, grid.z.cᶠ) -@inline rfaces(grid::RegularGrid) = z_domain(grid) -@inline rfaces(grid) = on_architecture(CPU(), rnodes(grid, Face())) +@inline cpu_face_constructor_r(grid::RegularVerticalGrid) = z_domain(grid) +@inline cpu_face_constructor_r(grid) = on_architecture(CPU(), rnodes(grid, Face())) -@inline cpu_face_constructor_z(grid) = rfaces(grid) -@inline cpu_face_constructor_z(grid::AbstractZStarGrid) = ZStarVerticalCoordinate(rfaces(grid)) +@inline cpu_face_constructor_z(grid) = cpu_face_constructor_r(grid) +@inline cpu_face_constructor_z(grid::AbstractZStarGrid) = ZStarVerticalCoordinate(cpu_face_constructor_r(grid)) #### #### Utilities From 6d71d86123141c3eb58f29d9d17a05c9d7d4fd37 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 2 Dec 2024 16:06:23 +0100 Subject: [PATCH 520/567] fix distributed tests --- ...pdate_hydrostatic_free_surface_model_state.jl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl index 43bc612e85..5ecaead713 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/update_hydrostatic_free_surface_model_state.jl @@ -71,9 +71,9 @@ function mask_immersed_model_fields!(model, grid) return nothing end -function compute_auxiliaries!(model::HydrostaticFreeSurfaceModel; w_params = w_kernel_parameters(model.grid), - p_params = p_kernel_parameters(model.grid), - κ_params = :xyz) +function compute_auxiliaries!(model::HydrostaticFreeSurfaceModel; w_parameters = w_kernel_parameters(model.grid), + p_parameters = p_kernel_parameters(model.grid), + κ_parameters = :xyz) grid = model.grid closure = model.closure @@ -85,15 +85,15 @@ function compute_auxiliaries!(model::HydrostaticFreeSurfaceModel; w_params = w_k arch = architecture(grid) # Update the grid and unscale the tracers - update_grid!(model, grid; parameters = w_params) - unscale_tracers!(tracers, grid; parameters = w_params) + update_grid!(model, grid; parameters = w_parameters) + unscale_tracers!(tracers, grid; parameters = w_parameters) # Advance diagnostic quantities - compute_w_from_continuity!(model; parameters = w_params) - update_hydrostatic_pressure!(P, arch, grid, buoyancy, tracers; parameters = p_params) + compute_w_from_continuity!(model; parameters = w_parameters) + update_hydrostatic_pressure!(P, arch, grid, buoyancy, tracers; parameters = p_parameters) # Update closure diffusivities - compute_diffusivities!(diffusivity, closure, model; parameters = κ_params) + compute_diffusivities!(diffusivity, closure, model; parameters = κ_parameters) return nothing end From 8ca1f7f0dcbad33ec56a537a318e27919943541d Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Mon, 2 Dec 2024 17:43:46 +0100 Subject: [PATCH 521/567] still debugging --- .../set_hydrostatic_free_surface_model.jl | 2 +- .../z_star_vertical_spacing.jl | 4 ++-- test/test_zstar_coordinate.jl | 11 ++++------- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/set_hydrostatic_free_surface_model.jl b/src/Models/HydrostaticFreeSurfaceModels/set_hydrostatic_free_surface_model.jl index d54c807e41..f216d312c7 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/set_hydrostatic_free_surface_model.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/set_hydrostatic_free_surface_model.jl @@ -60,8 +60,8 @@ model.velocities.u @apply_regionally set!(ϕ, value) end - update_state!(model; compute_tendencies = false) initialize!(model) + update_state!(model; compute_tendencies = false) return nothing end diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 2e5828461f..87166d6cf6 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -71,7 +71,7 @@ end e₃ᶠᶠⁿ[i, j, 1] = e₃ᶠᶠ # Update η in the grid - ηⁿ[i, j, 1] = η[i, j, kᴺ+1] + ηⁿ[i, j, 1] = η[i, j, k_top] end end @@ -86,7 +86,7 @@ end δy_V = δyᶜᶜᶜ(i, j, kᴺ, grid, Δx_qᶜᶠᶜ, barotropic_V, V, v) δh_U = (δx_U + δy_V) / Azᶜᶜᶜ(i, j, kᴺ, grid) - + @inbounds ∂t_e₃[i, j, 1] = ifelse(hᶜᶜ == 0, zero(grid), - δh_U / hᶜᶜ) end diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index 89af20e963..e73b034457 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -61,22 +61,19 @@ const F = Face @test static_column_depthᶜᶜᵃ(1, 1, grid) == 10 set!(model, η = [1 1; 2 2]) - set!(model, u = (x, y, z) -> x) + set!(model, u = (x, y, z) -> x, v = (x, y, z) -> y) + update_state!(model) @test e₃ⁿ(1, 1, 1, grid, C(), C(), C()) == 11 / 10 @test e₃ⁿ(2, 1, 1, grid, C(), C(), C()) == 12 / 10 - @test e₃⁻(1, 1, 1, grid, C(), C(), C()) == 1 - @test e₃⁻(2, 1, 1, grid, C(), C(), C()) == 1 - @test znode(1, 1, 21, grid, C(), C(), C()) == 1 - @test znode(2, 1, 21, grid, C(), C(), C()) == 2 + @test znode(1, 1, 21, grid, C(), C(), F()) == 1 + @test znode(2, 1, 21, grid, C(), C(), F()) == 2 @test rnode(1, 1, 21, grid, C(), C(), F()) == 0 @test dynamic_column_depthᶜᶜᵃ(1, 1, grid) == 11 @test dynamic_column_depthᶜᶜᵃ(2, 1, grid) == 12 @test static_column_depthᶜᶜᵃ(1, 1, grid) == 10 @test static_column_depthᶜᶜᵃ(2, 1, grid) == 10 - - @test ∂t_e₃(1, 1, 1, grid) == 1 / 10 end @testset "ZStar coordinate simulation testset" begin From f1e56c7d3e8e16b149db2fabc33637c9bd258a96 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 3 Dec 2024 08:34:38 +0100 Subject: [PATCH 522/567] fix gpu scripts --- .../stratified_couette_flow/stratified_couette_flow.jl | 6 +++--- validation/thermal_bubble/thermal_bubble.jl | 2 +- validation/z_star_coordinate/lock_release.jl | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/validation/stratified_couette_flow/stratified_couette_flow.jl b/validation/stratified_couette_flow/stratified_couette_flow.jl index ca8a7ca600..81e025294b 100644 --- a/validation/stratified_couette_flow/stratified_couette_flow.jl +++ b/validation/stratified_couette_flow/stratified_couette_flow.jl @@ -11,7 +11,7 @@ using Oceananigans.Advection: cell_advection_timescale """ Friction velocity. See equation (16) of Vreugdenhil & Taylor (2018). """ function uτ(model, Uavg, U_wall, n) - Nz, Hz, Δz = model.grid.Nz, model.grid.Hz, model.grid.Δzᵃᵃᶜ + Nz, Hz, Δz = model.grid.Nz, model.grid.Hz, model.grid.z.Δzᶜ ν = model.closure[n].ν compute!(Uavg) @@ -30,7 +30,7 @@ end """ Heat flux at the wall. See equation (16) of Vreugdenhil & Taylor (2018). """ function q_wall(model, Tavg, Θ_wall, n) - Nz, Hz, Δz = model.grid.Nz, model.grid.Hz, model.grid.Δzᵃᵃᶜ + Nz, Hz, Δz = model.grid.Nz, model.grid.Hz, model.grid.z.Δᶜ # TODO: interface function for extracting diffusivity? κ = model.closure[n].κ.T @@ -261,7 +261,7 @@ function simulate_stratified_couette_flow(; Nxy, Nz, arch=GPU(), h=1, U_wall=1, wmax = maximum(abs, model.velocities.w.data.parent) CFL = simulation.Δt / cell_advection_timescale(model) - Δ = min(model.grid.Δxᶜᵃᵃ, model.grid.Δyᵃᶜᵃ, model.grid.Δzᵃᵃᶜ) + Δ = min(model.grid.Δxᶜᵃᵃ, model.grid.Δyᵃᶜᵃ, model.grid.z.Δᶜ) νmax = maximum(model.diffusivity_fields[n_amd].νₑ.data.parent) κmax = maximum(model.diffusivity_fields[n_amd].κₑ.T.data.parent) νCFL = simulation.Δt / (Δ^2 / νmax) diff --git a/validation/thermal_bubble/thermal_bubble.jl b/validation/thermal_bubble/thermal_bubble.jl index 5358786ae0..eb747510dc 100644 --- a/validation/thermal_bubble/thermal_bubble.jl +++ b/validation/thermal_bubble/thermal_bubble.jl @@ -62,7 +62,7 @@ function print_progress(simulation) u_max = maximum(abs, interior(model.velocities.u)) w_max = maximum(abs, interior(model.velocities.w)) T_min, T_max = extrema(interior(model.tracers.T)) - CFL = max(u_max, w_max) * simulation.Δt / min(model.grid.Δxᶜᵃᵃ, model.grid.Δzᵃᵃᶜ) + CFL = max(u_max, w_max) * simulation.Δt / min(model.grid.Δxᶜᵃᵃ, model.grid.z.Δᶜ) i, t = model.clock.iteration, model.clock.time @info @sprintf("[%06.2f%%] i: %d, t: %.4f, U_max: (%.2e, %.2e), T: (min=%.5f, max=%.5f), CFL: %.4f", diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index 99f804e1c5..ec4da9a51b 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -22,7 +22,7 @@ model = HydrostaticFreeSurfaceModel(; grid, buoyancy = BuoyancyTracer(), closure = nothing, tracers = :b, - free_surface = ImplicitFreeSurface()) #SplitExplicitFreeSurface(; substeps = 10)) + free_surface = SplitExplicitFreeSurface(; substeps = 10)) g = model.free_surface.gravitational_acceleration From 8a6937fef95a4e3551b5534ac46a6b5a74159802 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 3 Dec 2024 09:44:37 +0100 Subject: [PATCH 523/567] correct the face constructor --- src/Grids/vertical_coordinate.jl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Grids/vertical_coordinate.jl b/src/Grids/vertical_coordinate.jl index 62330dceb5..36803b4a3b 100644 --- a/src/Grids/vertical_coordinate.jl +++ b/src/Grids/vertical_coordinate.jl @@ -148,7 +148,13 @@ function zspacings end z_domain(grid) = domain(topology(grid, 3)(), grid.Nz, grid.z.cᶠ) @inline cpu_face_constructor_r(grid::RegularVerticalGrid) = z_domain(grid) -@inline cpu_face_constructor_r(grid) = on_architecture(CPU(), rnodes(grid, Face())) + +@inline function cpu_face_constructor_r(grid) + Nz = size(grid, 3) + nodes = rnodes(grid, Face(); with_halos=true) + cpu_nodes = on_architecture(CPU(), nodes) + return cpu_nodes[1:Nz+1] +end @inline cpu_face_constructor_z(grid) = cpu_face_constructor_r(grid) @inline cpu_face_constructor_z(grid::AbstractZStarGrid) = ZStarVerticalCoordinate(cpu_face_constructor_r(grid)) From 4da8c7512ea41ba7c721450c1f28323b34fb6b73 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 3 Dec 2024 09:46:05 +0100 Subject: [PATCH 524/567] comment --- src/Grids/vertical_coordinate.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Grids/vertical_coordinate.jl b/src/Grids/vertical_coordinate.jl index 36803b4a3b..0167f0d376 100644 --- a/src/Grids/vertical_coordinate.jl +++ b/src/Grids/vertical_coordinate.jl @@ -142,7 +142,7 @@ function zspacings end @inline zspacings(grid, ℓz) = zspacings(grid, nothing, nothing, ℓz) #### -#### z_domain (independent of ZStar or not) +#### `z_domain` (independent of ZStar or not) and `cpu_face_constructor_z` #### z_domain(grid) = domain(topology(grid, 3)(), grid.Nz, grid.z.cᶠ) From dd988753421e68ddb517b444da0c375e3c07865b Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 3 Dec 2024 15:59:32 +0100 Subject: [PATCH 525/567] all tests should be fixed now except for the distributed --- .buildkite/pipeline.yml | 1 + .../z_star_vertical_spacing.jl | 6 ++++++ src/MultiRegion/cubed_sphere_grid.jl | 19 ++++++++++++------- src/MultiRegion/multi_region_grid.jl | 11 +++++++---- .../spacings_and_areas_and_volumes.jl | 2 +- test/test_multi_region_unit.jl | 2 +- test/test_zstar_coordinate.jl | 16 ++++++++++++---- .../stratified_couette_flow.jl | 2 +- 8 files changed, 41 insertions(+), 18 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index aa8fee0a3b..00167e1a10 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -669,6 +669,7 @@ steps: env: JULIA_DEPOT_PATH: "$SVERDRUP_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "vertical_coordinate" + GPU_TEST: "true" commands: - "$SVERDRUP_HOME/julia-$JULIA_VERSION/bin/julia -O0 --color=yes --project -e 'using Pkg; Pkg.test()'" agents: diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 87166d6cf6..e7917b1f16 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -1,6 +1,12 @@ using Oceananigans.Grids using Oceananigans.ImmersedBoundaries: ZStarGridOfSomeKind +# TODO: ZStar is not currently working for an ImplicitFreeSurface on +# - `LatitudeLongitudeGrid`s +# - `ImmersedBoundaryGrid`s +# and for an ExplicitFreeSurface on +# - `ImmersedBoundaryGrid`s + ##### ##### ZStar-specific vertical spacings update ##### diff --git a/src/MultiRegion/cubed_sphere_grid.jl b/src/MultiRegion/cubed_sphere_grid.jl index 0cfa3db30e..e68a2621f3 100644 --- a/src/MultiRegion/cubed_sphere_grid.jl +++ b/src/MultiRegion/cubed_sphere_grid.jl @@ -11,7 +11,7 @@ using Distances import Oceananigans.Grids: grid_name -const ConformalCubedSphereGrid{FT, TX, TY, TZ} = MultiRegionGrid{FT, TX, TY, TZ, <:CubedSpherePartition} +const ConformalCubedSphereGrid{FT, TX, TY, TZ, CZ} = MultiRegionGrid{FT, TX, TY, TZ, CZ, <:CubedSpherePartition} """ ConformalCubedSphereGrid(arch=CPU(), FT=Float64; @@ -343,11 +343,14 @@ function ConformalCubedSphereGrid(arch::AbstractArchitecture=CPU(), FT=Float64; new_region_grids = MultiRegionObject(new_region_grids.regional_objects, new_devices) - new_grid = MultiRegionGrid{FT, region_topology...}(arch, - partition, - connectivity, - new_region_grids, - new_devices) + # Propagate the vertical coordinate type in the `MultiRegionGrid` + CZ = typeof(getregion(region_grids, 1).z) + + new_grid = MultiRegionGrid{FT, region_topology..., CZ}(arch, + partition, + connectivity, + new_region_grids, + new_devices) return new_grid end @@ -390,7 +393,9 @@ function ConformalCubedSphereGrid(filepath::AbstractString, arch::AbstractArchit connectivity = CubedSphereConnectivity(devices, partition) - return MultiRegionGrid{FT, panel_topology...}(arch, partition, connectivity, region_grids, devices) + CZ = typeof(getregion(region_grids, 1).z) + + return MultiRegionGrid{FT, panel_topology..., CZ}(arch, partition, connectivity, region_grids, devices) end function with_halo(new_halo, csg::ConformalCubedSphereGrid) diff --git a/src/MultiRegion/multi_region_grid.jl b/src/MultiRegion/multi_region_grid.jl index 264b6a29be..7f21e6c02a 100644 --- a/src/MultiRegion/multi_region_grid.jl +++ b/src/MultiRegion/multi_region_grid.jl @@ -8,15 +8,15 @@ import Oceananigans.Grids: minimum_xspacing, minimum_yspacing, minimum_zspacing import Oceananigans.Models.HydrostaticFreeSurfaceModels: default_free_surface import Oceananigans.DistributedComputations: reconstruct_global_grid -struct MultiRegionGrid{FT, TX, TY, TZ, P, C, G, D, Arch} <: AbstractMultiRegionGrid{FT, TX, TY, TZ, Arch} +struct MultiRegionGrid{FT, TX, TY, TZ, CZ, P, C, G, D, Arch} <: AbstractUnderlyingGrid{FT, TX, TY, TZ, CZ, Arch} architecture :: Arch partition :: P connectivity :: C region_grids :: G devices :: D - MultiRegionGrid{FT, TX, TY, TZ}(arch::A, partition::P, connectivity::C, - region_grids::G, devices::D) where {FT, TX, TY, TZ, P, C, G, D, A} = + MultiRegionGrid{FT, TX, TY, TZ, CZ}(arch::A, partition::P, connectivity::C, + region_grids::G, devices::D) where {FT, TX, TY, TZ, P, C, G, D, A} = new{FT, TX, TY, TZ, P, C, G, D, A}(arch, partition, connectivity, region_grids, devices) end @@ -140,10 +140,13 @@ function MultiRegionGrid(global_grid; partition = XPartition(2), region_grids = construct_regionally(construct_grid, args...) + # Propagate the vertical coordinate type in the `MultiRegionGrid` + CZ = typeof(global_grid.z) + ## If we are on GPUs we want to enable peer access, which we do by just copying fake arrays between all devices maybe_enable_peer_access!(devices) - return MultiRegionGrid{FT, global_topo[1], global_topo[2], global_topo[3]}(arch, partition, connectivity, region_grids, devices) + return MultiRegionGrid{FT, global_topo[1], global_topo[2], global_topo[3], CZ}(arch, partition, connectivity, region_grids, devices) end function construct_grid(grid::RectilinearGrid, child_arch, topo, size, extent, args...) diff --git a/src/Operators/spacings_and_areas_and_volumes.jl b/src/Operators/spacings_and_areas_and_volumes.jl index f8af81628b..ecea5deb10 100644 --- a/src/Operators/spacings_and_areas_and_volumes.jl +++ b/src/Operators/spacings_and_areas_and_volumes.jl @@ -48,7 +48,7 @@ const ZRG = Union{LLGZ, RGZ, OSSGZ} @inline Δzᵃᵃᶠ(i, j, k, grid) = getspacing(k, grid.z.Δᶠ) # Convenience Functions for all grids -for LX in (:ᶜ, :ᶠ), LY in (:ᶜ, :ᶠ) +for LX in (:ᶜ, :ᶠ, :ᵃ), LY in (:ᶜ, :ᶠ, :ᵃ) x_spacing_1D = Symbol(:Δx, LX, :ᵃ, :ᵃ) x_spacing_2D = Symbol(:Δx, LX, LY, :ᵃ) diff --git a/test/test_multi_region_unit.jl b/test/test_multi_region_unit.jl index f5cc3496a3..c0624d3282 100644 --- a/test/test_multi_region_unit.jl +++ b/test/test_multi_region_unit.jl @@ -53,7 +53,7 @@ devices(::GPU, num) = Tuple(0 for i in 1:num) for immersed_boundary in immersed_boundaries @info "Testing multi region immersed boundaries on $(getnamewrapper(grid)) on $regions $(Partition)s" ibg = ImmersedBoundaryGrid(grid, immersed_boundary) - mrg = MultiRegionGrid(grid, partition = Partition(region), devices = devices(arch, region)) + mrg = MultiRegionGrid(grid, partition = XPartition(region), devices = devices(arch, region)) mribg = ImmersedBoundaryGrid(mrg, immersed_boundary) @test reconstruct_global_grid(mribg) == ibg diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index e73b034457..ed01c803d7 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -23,7 +23,14 @@ function test_zstar_coordinate(model, Ni, Δt) @test interior(∫b, 1, 1, 1) ≈ interior(∫bᵢ, 1, 1, 1) @test interior(∫c, 1, 1, 1) ≈ interior(∫cᵢ, 1, 1, 1) - @test maximum(interior(w, :, :, Nz+1)) < model.grid.Nz * eps(eltype(w)) + + # TODO: This test errors for all grids except RectilinearGrid with the Implicit and Explicit + # free surfaces (the velocity at the top is not approximately zero) + if model.free_surface isa SplitExplicitFreeSurface + @test maximum(interior(w, :, :, Nz+1)) < model.grid.Nz * eps(eltype(w)) + else + @test_broken maximum(interior(w, :, :, Nz+1)) < model.grid.Nz * eps(eltype(w)) + end return nothing end @@ -105,7 +112,7 @@ end pllg = ImmersedBoundaryGrid(llg, PartialCellBottom((x, y) -> rand() - 10)) pllgv = ImmersedBoundaryGrid(llgv, PartialCellBottom((x, y) -> rand() - 10)) - # Partial cell bottom are broken at the moment and do not account for the Δz in the volumes + # TODO: Partial cell bottom are broken at the moment and do not account for the Δz in the volumes # and vertical areas (see https://github.com/CliMA/Oceananigans.jl/issues/3958) # When this is issue is fixed we can add the partial cells to the testing. grids = [llg, rtg, llgv, rtgv, illg, irtg, illgv, irtgv] # , pllg, prtg, pllgv, prtgv] @@ -116,12 +123,13 @@ end for grid in grids info_msg = info_message(grid) - split_free_surface = SplitExplicitFreeSurface(grid; substeps = 20) + split_free_surface = SplitExplicitFreeSurface(grid; substeps = 20) + implicit_free_surface = ImplicitFreeSurface() explicit_free_surface = ExplicitFreeSurface() for free_surface in [split_free_surface, implicit_free_surface, explicit_free_surface] - @testset "$info_msg" begin + @testset "$info_msg on $(free_surface)" begin @info " $info_msg of $(free_surface)" # TODO: minimum_xspacing(grid) on a Immersed GPU grid with ZStarVerticalCoordinate # fails because it uses too much parameter space. Figure out a way to reduce it diff --git a/validation/stratified_couette_flow/stratified_couette_flow.jl b/validation/stratified_couette_flow/stratified_couette_flow.jl index 81e025294b..82558572b1 100644 --- a/validation/stratified_couette_flow/stratified_couette_flow.jl +++ b/validation/stratified_couette_flow/stratified_couette_flow.jl @@ -11,7 +11,7 @@ using Oceananigans.Advection: cell_advection_timescale """ Friction velocity. See equation (16) of Vreugdenhil & Taylor (2018). """ function uτ(model, Uavg, U_wall, n) - Nz, Hz, Δz = model.grid.Nz, model.grid.Hz, model.grid.z.Δzᶜ + Nz, Hz, Δz = model.grid.Nz, model.grid.Hz, model.grid.z.Δᶜ ν = model.closure[n].ν compute!(Uavg) From bda37bfdb32df92efd9cf481f628ef14e244e8f3 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 3 Dec 2024 16:01:49 +0100 Subject: [PATCH 526/567] last bugfix --- src/MultiRegion/multi_region_grid.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/MultiRegion/multi_region_grid.jl b/src/MultiRegion/multi_region_grid.jl index 7f21e6c02a..bfd6cb061b 100644 --- a/src/MultiRegion/multi_region_grid.jl +++ b/src/MultiRegion/multi_region_grid.jl @@ -17,7 +17,7 @@ struct MultiRegionGrid{FT, TX, TY, TZ, CZ, P, C, G, D, Arch} <: AbstractUnderlyi MultiRegionGrid{FT, TX, TY, TZ, CZ}(arch::A, partition::P, connectivity::C, region_grids::G, devices::D) where {FT, TX, TY, TZ, P, C, G, D, A} = - new{FT, TX, TY, TZ, P, C, G, D, A}(arch, partition, connectivity, region_grids, devices) + new{FT, TX, TY, TZ, CZ, P, C, G, D, A}(arch, partition, connectivity, region_grids, devices) end const ImmersedMultiRegionGrid = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:MultiRegionGrid} From ae012deee5364793a590ead6e25fae4b4d2b56b3 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 3 Dec 2024 16:07:09 +0100 Subject: [PATCH 527/567] now it should work --- src/MultiRegion/multi_region_grid.jl | 2 +- test/test_multi_region_unit.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/MultiRegion/multi_region_grid.jl b/src/MultiRegion/multi_region_grid.jl index bfd6cb061b..9974e03a2e 100644 --- a/src/MultiRegion/multi_region_grid.jl +++ b/src/MultiRegion/multi_region_grid.jl @@ -16,7 +16,7 @@ struct MultiRegionGrid{FT, TX, TY, TZ, CZ, P, C, G, D, Arch} <: AbstractUnderlyi devices :: D MultiRegionGrid{FT, TX, TY, TZ, CZ}(arch::A, partition::P, connectivity::C, - region_grids::G, devices::D) where {FT, TX, TY, TZ, P, C, G, D, A} = + region_grids::G, devices::D) where {FT, TX, TY, TZ, CZ, P, C, G, D, A} = new{FT, TX, TY, TZ, CZ, P, C, G, D, A}(arch, partition, connectivity, region_grids, devices) end diff --git a/test/test_multi_region_unit.jl b/test/test_multi_region_unit.jl index c0624d3282..f5cc3496a3 100644 --- a/test/test_multi_region_unit.jl +++ b/test/test_multi_region_unit.jl @@ -53,7 +53,7 @@ devices(::GPU, num) = Tuple(0 for i in 1:num) for immersed_boundary in immersed_boundaries @info "Testing multi region immersed boundaries on $(getnamewrapper(grid)) on $regions $(Partition)s" ibg = ImmersedBoundaryGrid(grid, immersed_boundary) - mrg = MultiRegionGrid(grid, partition = XPartition(region), devices = devices(arch, region)) + mrg = MultiRegionGrid(grid, partition = Partition(region), devices = devices(arch, region)) mribg = ImmersedBoundaryGrid(mrg, immersed_boundary) @test reconstruct_global_grid(mribg) == ibg From ee8866ddb36d5f08e84bee63a6815ad51440037d Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 3 Dec 2024 16:53:41 +0100 Subject: [PATCH 528/567] check which is the test that errors --- test/test_distributed_hydrostatic_model.jl | 74 ++++++++++++---------- 1 file changed, 39 insertions(+), 35 deletions(-) diff --git a/test/test_distributed_hydrostatic_model.jl b/test/test_distributed_hydrostatic_model.jl index c11d3c5248..7e93a1643a 100644 --- a/test/test_distributed_hydrostatic_model.jl +++ b/test/test_distributed_hydrostatic_model.jl @@ -76,6 +76,9 @@ end Nx = 32 Ny = 32 +active_cells_map(grid) = false +active_cells_map(grid::ImmersedBoundaryGrid) = !isnothing(grid.interior_active_cells) + for arch in archs @testset "Testing distributed solid body rotation" begin underlying_grid = LatitudeLongitudeGrid(arch, size = (Nx, Ny, 1), @@ -95,41 +98,42 @@ for arch in archs global_immersed_grid = ImmersedBoundaryGrid(global_underlying_grid, GridFittedBottom(bottom)) for (grid, global_grid) in zip((underlying_grid, immersed_grid, immersed_active_grid), (global_underlying_grid, global_immersed_grid, global_immersed_grid)) - - # "s" for "serial" computation - us, vs, ws, cs, ηs = solid_body_rotation_test(global_grid) - - us = interior(on_architecture(CPU(), us)) - vs = interior(on_architecture(CPU(), vs)) - ws = interior(on_architecture(CPU(), ws)) - cs = interior(on_architecture(CPU(), cs)) - ηs = interior(on_architecture(CPU(), ηs)) - - @info " Testing distributed solid body rotation with architecture $arch on $(typeof(grid).name.wrapper)" - u, v, w, c, η = solid_body_rotation_test(grid) - - cpu_arch = cpu_architecture(arch) - - u = interior(on_architecture(cpu_arch, u)) - v = interior(on_architecture(cpu_arch, v)) - w = interior(on_architecture(cpu_arch, w)) - c = interior(on_architecture(cpu_arch, c)) - η = interior(on_architecture(cpu_arch, η)) - - us = partition(us, cpu_arch, size(u)) - vs = partition(vs, cpu_arch, size(v)) - ws = partition(ws, cpu_arch, size(w)) - cs = partition(cs, cpu_arch, size(c)) - ηs = partition(ηs, cpu_arch, size(η)) - - atol = eps(eltype(grid)) - rtol = sqrt(eps(eltype(grid))) - - @test all(isapprox(u, us; atol, rtol)) - @test all(isapprox(v, vs; atol, rtol)) - @test all(isapprox(w, ws; atol, rtol)) - @test all(isapprox(c, cs; atol, rtol)) - @test all(isapprox(η, ηs; atol, rtol)) + @testset "Test solid rotation on $(summary(grid)) with $(active_cells_map(grid)))" begin + # "s" for "serial" computation + us, vs, ws, cs, ηs = solid_body_rotation_test(global_grid) + + us = interior(on_architecture(CPU(), us)) + vs = interior(on_architecture(CPU(), vs)) + ws = interior(on_architecture(CPU(), ws)) + cs = interior(on_architecture(CPU(), cs)) + ηs = interior(on_architecture(CPU(), ηs)) + + @info " Testing distributed solid body rotation with architecture $arch on $(typeof(grid).name.wrapper)" + u, v, w, c, η = solid_body_rotation_test(grid) + + cpu_arch = cpu_architecture(arch) + + u = interior(on_architecture(cpu_arch, u)) + v = interior(on_architecture(cpu_arch, v)) + w = interior(on_architecture(cpu_arch, w)) + c = interior(on_architecture(cpu_arch, c)) + η = interior(on_architecture(cpu_arch, η)) + + us = partition(us, cpu_arch, size(u)) + vs = partition(vs, cpu_arch, size(v)) + ws = partition(ws, cpu_arch, size(w)) + cs = partition(cs, cpu_arch, size(c)) + ηs = partition(ηs, cpu_arch, size(η)) + + atol = eps(eltype(grid)) + rtol = sqrt(eps(eltype(grid))) + + @test all(isapprox(u, us; atol, rtol)) + @test all(isapprox(v, vs; atol, rtol)) + @test all(isapprox(w, ws; atol, rtol)) + @test all(isapprox(c, cs; atol, rtol)) + @test all(isapprox(η, ηs; atol, rtol)) + end end end end From 846b27b255a17e9f090e02a42fb8b57931bb5f94 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 3 Dec 2024 17:01:11 +0100 Subject: [PATCH 529/567] remove stackoverflow --- src/Operators/spacings_and_areas_and_volumes.jl | 10 +++++----- test/test_distributed_hydrostatic_model.jl | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Operators/spacings_and_areas_and_volumes.jl b/src/Operators/spacings_and_areas_and_volumes.jl index ecea5deb10..9d8b7026bd 100644 --- a/src/Operators/spacings_and_areas_and_volumes.jl +++ b/src/Operators/spacings_and_areas_and_volumes.jl @@ -1,4 +1,4 @@ -using Oceananigans.Grids: Center, Face +using Oceananigans.Grids: Center, Face, AbstractGrid using Oceananigans.Grids: AbstractZStarGrid @inline hack_cosd(φ) = cos(π * φ / 180) @@ -41,11 +41,11 @@ const ZRG = Union{LLGZ, RGZ, OSSGZ} @inline getspacing(k, Δz::AbstractVector) = @inbounds Δz[k] @inline getspacing(k, Δz::Number) = @inbounds Δz -@inline Δrᵃᵃᶜ(i, j, k, grid) = getspacing(k, grid.z.Δᶜ) -@inline Δrᵃᵃᶠ(i, j, k, grid) = getspacing(k, grid.z.Δᶠ) +@inline Δrᵃᵃᶜ(i, j, k, grid::AbstractGrid) = getspacing(k, grid.z.Δᶜ) +@inline Δrᵃᵃᶠ(i, j, k, grid::AbstractGrid) = getspacing(k, grid.z.Δᶠ) -@inline Δzᵃᵃᶜ(i, j, k, grid) = getspacing(k, grid.z.Δᶜ) -@inline Δzᵃᵃᶠ(i, j, k, grid) = getspacing(k, grid.z.Δᶠ) +@inline Δzᵃᵃᶜ(i, j, k, grid::AbstractGrid) = getspacing(k, grid.z.Δᶜ) +@inline Δzᵃᵃᶠ(i, j, k, grid::AbstractGrid) = getspacing(k, grid.z.Δᶠ) # Convenience Functions for all grids for LX in (:ᶜ, :ᶠ, :ᵃ), LY in (:ᶜ, :ᶠ, :ᵃ) diff --git a/test/test_distributed_hydrostatic_model.jl b/test/test_distributed_hydrostatic_model.jl index 7e93a1643a..9f530d0702 100644 --- a/test/test_distributed_hydrostatic_model.jl +++ b/test/test_distributed_hydrostatic_model.jl @@ -98,7 +98,7 @@ for arch in archs global_immersed_grid = ImmersedBoundaryGrid(global_underlying_grid, GridFittedBottom(bottom)) for (grid, global_grid) in zip((underlying_grid, immersed_grid, immersed_active_grid), (global_underlying_grid, global_immersed_grid, global_immersed_grid)) - @testset "Test solid rotation on $(summary(grid)) with $(active_cells_map(grid)))" begin + @testset "Test solid rotation on $(summary(grid)) with $(active_cells_map(grid)) on $arch)" begin # "s" for "serial" computation us, vs, ws, cs, ηs = solid_body_rotation_test(global_grid) From 3f2de733676f3b642ebd563b69e757747fd93437 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 4 Dec 2024 08:25:06 +0100 Subject: [PATCH 530/567] fix tests --- src/MultiRegion/multi_region_grid.jl | 4 ++-- test/test_zstar_coordinate.jl | 14 ++++---------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/src/MultiRegion/multi_region_grid.jl b/src/MultiRegion/multi_region_grid.jl index 9974e03a2e..ce7c54d5e2 100644 --- a/src/MultiRegion/multi_region_grid.jl +++ b/src/MultiRegion/multi_region_grid.jl @@ -235,10 +235,10 @@ function with_halo(new_halo, mrg::MultiRegionGrid) return MultiRegionGrid(new_global; partition, devices, validate = false) end -function on_architecture(::CPU, mrg::MultiRegionGrid{FT, TX, TY, TZ}) where {FT, TX, TY, TZ} +function on_architecture(::CPU, mrg::MultiRegionGrid{FT, TX, TY, TZ, CZ}) where {FT, TX, TY, TZ, CZ} new_grids = construct_regionally(on_architecture, CPU(), mrg) devices = Tuple(CPU() for i in 1:length(mrg)) - return MultiRegionGrid{FT, TX, TY, TZ}(CPU(), mrg.partition, mrg.connectivity, new_grids, devices) + return MultiRegionGrid{FT, TX, TY, TZ, CZ}(CPU(), mrg.partition, mrg.connectivity, new_grids, devices) end Base.summary(mrg::MultiRegionGrid{FT, TX, TY, TZ}) where {FT, TX, TY, TZ} = diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index ed01c803d7..e47ef65830 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -22,16 +22,9 @@ function test_zstar_coordinate(model, Ni, Δt) ∫c = Field(Integral(model.tracers.c)) @test interior(∫b, 1, 1, 1) ≈ interior(∫bᵢ, 1, 1, 1) - @test interior(∫c, 1, 1, 1) ≈ interior(∫cᵢ, 1, 1, 1) + @test interior(∫c, 1, 1, 1) ≈ interior(∫cᵢ, 1, 1, 1) + @test maximum(interior(w, :, :, Nz+1)) < model.grid.Nz * eps(eltype(w)) - # TODO: This test errors for all grids except RectilinearGrid with the Implicit and Explicit - # free surfaces (the velocity at the top is not approximately zero) - if model.free_surface isa SplitExplicitFreeSurface - @test maximum(interior(w, :, :, Nz+1)) < model.grid.Nz * eps(eltype(w)) - else - @test_broken maximum(interior(w, :, :, Nz+1)) < model.grid.Nz * eps(eltype(w)) - end - return nothing end @@ -125,10 +118,11 @@ end split_free_surface = SplitExplicitFreeSurface(grid; substeps = 20) + # TODO: Implicit and Explicit free surfaces are not fully supported yet implicit_free_surface = ImplicitFreeSurface() explicit_free_surface = ExplicitFreeSurface() - for free_surface in [split_free_surface, implicit_free_surface, explicit_free_surface] + for free_surface in [split_free_surface] #, implicit_free_surface, explicit_free_surface] @testset "$info_msg on $(free_surface)" begin @info " $info_msg of $(free_surface)" # TODO: minimum_xspacing(grid) on a Immersed GPU grid with ZStarVerticalCoordinate From 2d61d0d42ac09d15fe142e66110d2c7719c174e2 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 11 Dec 2024 09:03:15 +0100 Subject: [PATCH 531/567] revert file --- src/Grids/input_validation.jl | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Grids/input_validation.jl b/src/Grids/input_validation.jl index 0ef5a9e018..9624086bba 100644 --- a/src/Grids/input_validation.jl +++ b/src/Grids/input_validation.jl @@ -90,6 +90,16 @@ function validate_halo(TX, TY, TZ, size, halo) return halo end +function validate_dimension_specification(T, ξ, dir, N, FT) + + isnothing(ξ) && throw(ArgumentError("Must supply extent or $dir keyword when $dir-direction is $T")) + length(ξ) == 2 || throw(ArgumentError("$dir length($ξ) must be 2.")) + all(isa.(ξ, Number)) || throw(ArgumentError("$dir=$ξ should contain numbers.")) + ξ[2] ≥ ξ[1] || throw(ArgumentError("$dir=$ξ should be an increasing interval.")) + + return FT.(ξ) +end + function validate_rectilinear_domain(TX, TY, TZ, FT, size, extent, x, y, z) # Find domain endpoints or domain extent, depending on user input: @@ -116,16 +126,6 @@ function validate_rectilinear_domain(TX, TY, TZ, FT, size, extent, x, y, z) return x, y, z end -function validate_dimension_specification(T, ξ, dir, N, FT) - - isnothing(ξ) && throw(ArgumentError("Must supply extent or $dir keyword when $dir-direction is $T")) - length(ξ) == 2 || throw(ArgumentError("$dir length($ξ) must be 2.")) - all(isa.(ξ, Number)) || throw(ArgumentError("$dir=$ξ should contain numbers.")) - ξ[2] ≥ ξ[1] || throw(ArgumentError("$dir=$ξ should be an increasing interval.")) - - return FT.(ξ) -end - function validate_dimension_specification(T, ξ::AbstractVector, dir, N, FT) ξ = FT.(ξ) From 500339a5854c11a614db5ae53398ed4d94649324 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 11 Dec 2024 09:04:12 +0100 Subject: [PATCH 532/567] revert file --- src/ImmersedBoundaries/active_cells_map.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ImmersedBoundaries/active_cells_map.jl b/src/ImmersedBoundaries/active_cells_map.jl index 2225937528..ce1d186eb8 100644 --- a/src/ImmersedBoundaries/active_cells_map.jl +++ b/src/ImmersedBoundaries/active_cells_map.jl @@ -72,6 +72,9 @@ function ImmersedBoundaryGrid(grid, ib; active_cells_map::Bool = true) column_map) end +with_halo(halo, ibg::ActiveCellsIBG) = + ImmersedBoundaryGrid(with_halo(halo, ibg.underlying_grid), ibg.immersed_boundary; active_cells_map = true) + @inline active_cell(i, j, k, ibg) = !immersed_cell(i, j, k, ibg) @inline active_column(i, j, k, grid, column) = column[i, j, k] != 0 From 9c5564731a110cd1924205a6a0527081328606d4 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 11 Dec 2024 09:05:21 +0100 Subject: [PATCH 533/567] removed rk3 by mistake --- .../HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl index 2b340a0e96..58ef588bc9 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/HydrostaticFreeSurfaceModels.jl @@ -137,6 +137,7 @@ include("compute_hydrostatic_free_surface_tendencies.jl") include("compute_hydrostatic_free_surface_buffers.jl") include("update_hydrostatic_free_surface_model_state.jl") include("hydrostatic_free_surface_ab2_step.jl") +include("hydrostatic_free_surface_rk3_step.jl") include("store_hydrostatic_free_surface_tendencies.jl") include("prescribed_hydrostatic_velocity_fields.jl") include("single_column_model_mode.jl") From 7fed979ea1816fd4feec8c555358ba4f4571c4fc Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 11 Dec 2024 09:15:15 +0100 Subject: [PATCH 534/567] revert files --- src/OutputWriters/output_writer_utils.jl | 2 +- src/Solvers/batched_tridiagonal_solver.jl | 1 + src/TimeSteppers/quasi_adams_bashforth_2.jl | 1 - .../abstract_scalar_diffusivity_closure.jl | 1 + .../discrete_diffusion_function.jl | 1 - .../implicit_explicit_time_discretization.jl | 2 - ...vective_adjustment_vertical_diffusivity.jl | 2 - .../ri_based_vertical_diffusivity.jl | 2 - test/test_distributed_hydrostatic_model.jl | 74 +++++++++---------- .../distributed_hydrostatic_turbulence.jl | 2 +- .../geostrophic_adjustment_test.jl | 11 +-- validation/thermal_bubble/thermal_bubble.jl | 2 +- 12 files changed, 43 insertions(+), 58 deletions(-) diff --git a/src/OutputWriters/output_writer_utils.jl b/src/OutputWriters/output_writer_utils.jl index 8c5dc6b596..7b79755b3b 100644 --- a/src/OutputWriters/output_writer_utils.jl +++ b/src/OutputWriters/output_writer_utils.jl @@ -1,6 +1,6 @@ -using Oceananigans.DistributedComputations using StructArrays: StructArray, replace_storage using Oceananigans.Grids: on_architecture, architecture +using Oceananigans.DistributedComputations using Oceananigans.DistributedComputations: DistributedGrid, Partition using Oceananigans.Fields: AbstractField, indices, boundary_conditions, instantiated_location using Oceananigans.BoundaryConditions: bc_str, FieldBoundaryConditions, ContinuousBoundaryFunction, DiscreteBoundaryFunction diff --git a/src/Solvers/batched_tridiagonal_solver.jl b/src/Solvers/batched_tridiagonal_solver.jl index bebc7c74ce..2886847938 100644 --- a/src/Solvers/batched_tridiagonal_solver.jl +++ b/src/Solvers/batched_tridiagonal_solver.jl @@ -1,5 +1,6 @@ using Oceananigans.Architectures: on_architecture using Oceananigans.Grids: XDirection, YDirection, ZDirection + import Oceananigans.Architectures: architecture """ diff --git a/src/TimeSteppers/quasi_adams_bashforth_2.jl b/src/TimeSteppers/quasi_adams_bashforth_2.jl index 181c3c3048..d099ba7a8d 100644 --- a/src/TimeSteppers/quasi_adams_bashforth_2.jl +++ b/src/TimeSteppers/quasi_adams_bashforth_2.jl @@ -99,7 +99,6 @@ function time_step!(model::AbstractModel{<:QuasiAdamsBashforth2TimeStepper}, Δt ab2_step!(model, Δt) tick!(model.clock, Δt) - model.clock.last_Δt = Δt model.clock.last_stage_Δt = Δt # just one stage diff --git a/src/TurbulenceClosures/abstract_scalar_diffusivity_closure.jl b/src/TurbulenceClosures/abstract_scalar_diffusivity_closure.jl index 6a28d542cc..5b1b781d8e 100644 --- a/src/TurbulenceClosures/abstract_scalar_diffusivity_closure.jl +++ b/src/TurbulenceClosures/abstract_scalar_diffusivity_closure.jl @@ -259,6 +259,7 @@ end ##### Products of viscosity and stress, divergence, vorticity ##### + @inline κ_σᶠᶜᶜ(i, j, k, grid, closure, K, id, clock, fields, σᶠᶜᶜ, args...) = κᶠᶜᶜ(i, j, k, grid, closure, K, id, clock, fields) * σᶠᶜᶜ(i, j, k, grid, args...) @inline κ_σᶜᶠᶜ(i, j, k, grid, closure, K, id, clock, fields, σᶜᶠᶜ, args...) = κᶜᶠᶜ(i, j, k, grid, closure, K, id, clock, fields) * σᶜᶠᶜ(i, j, k, grid, args...) @inline κ_σᶜᶜᶠ(i, j, k, grid, closure, K, id, clock, fields, σᶜᶜᶠ, args...) = κᶜᶜᶠ(i, j, k, grid, closure, K, id, clock, fields) * σᶜᶜᶠ(i, j, k, grid, args...) diff --git a/src/TurbulenceClosures/discrete_diffusion_function.jl b/src/TurbulenceClosures/discrete_diffusion_function.jl index 8d5bc29d7b..19d8203783 100644 --- a/src/TurbulenceClosures/discrete_diffusion_function.jl +++ b/src/TurbulenceClosures/discrete_diffusion_function.jl @@ -1,6 +1,5 @@ using Oceananigans.Operators: ℑxyz using Oceananigans.Utils: instantiate -import Oceananigans.Architectures: on_architecture """ struct DiscreteDiffusionFunction{LX, LY, LZ, P, F} diff --git a/src/TurbulenceClosures/implicit_explicit_time_discretization.jl b/src/TurbulenceClosures/implicit_explicit_time_discretization.jl index 603ef75d55..3fa9d7a54e 100644 --- a/src/TurbulenceClosures/implicit_explicit_time_discretization.jl +++ b/src/TurbulenceClosures/implicit_explicit_time_discretization.jl @@ -1,7 +1,5 @@ using Oceananigans.Grids: AbstractGrid -import Oceananigans.Architectures: on_architecture - abstract type AbstractTimeDiscretization end """ diff --git a/src/TurbulenceClosures/turbulence_closure_implementations/convective_adjustment_vertical_diffusivity.jl b/src/TurbulenceClosures/turbulence_closure_implementations/convective_adjustment_vertical_diffusivity.jl index 53df44fec9..cfdd407523 100644 --- a/src/TurbulenceClosures/turbulence_closure_implementations/convective_adjustment_vertical_diffusivity.jl +++ b/src/TurbulenceClosures/turbulence_closure_implementations/convective_adjustment_vertical_diffusivity.jl @@ -3,8 +3,6 @@ using Oceananigans.AbstractOperations: KernelFunctionOperation using Oceananigans.BuoyancyModels: ∂z_b using Oceananigans.Operators: ℑzᵃᵃᶜ -import Oceananigans.Architectures: on_architecture - struct ConvectiveAdjustmentVerticalDiffusivity{TD, CK, CN, BK, BN} <: AbstractScalarDiffusivity{TD, VerticalFormulation, 1} convective_κz :: CK convective_νz :: CN diff --git a/src/TurbulenceClosures/turbulence_closure_implementations/ri_based_vertical_diffusivity.jl b/src/TurbulenceClosures/turbulence_closure_implementations/ri_based_vertical_diffusivity.jl index 9706c49843..2d4b7bb2eb 100644 --- a/src/TurbulenceClosures/turbulence_closure_implementations/ri_based_vertical_diffusivity.jl +++ b/src/TurbulenceClosures/turbulence_closure_implementations/ri_based_vertical_diffusivity.jl @@ -4,8 +4,6 @@ using Oceananigans.Operators using Oceananigans.Grids: inactive_node using Oceananigans.Operators: ℑzᵃᵃᶜ -import Oceananigans.Architectures: on_architecture - struct RiBasedVerticalDiffusivity{TD, FT, R, HR} <: AbstractScalarDiffusivity{TD, VerticalFormulation, 1} ν₀ :: FT κ₀ :: FT diff --git a/test/test_distributed_hydrostatic_model.jl b/test/test_distributed_hydrostatic_model.jl index 9f530d0702..c11d3c5248 100644 --- a/test/test_distributed_hydrostatic_model.jl +++ b/test/test_distributed_hydrostatic_model.jl @@ -76,9 +76,6 @@ end Nx = 32 Ny = 32 -active_cells_map(grid) = false -active_cells_map(grid::ImmersedBoundaryGrid) = !isnothing(grid.interior_active_cells) - for arch in archs @testset "Testing distributed solid body rotation" begin underlying_grid = LatitudeLongitudeGrid(arch, size = (Nx, Ny, 1), @@ -98,42 +95,41 @@ for arch in archs global_immersed_grid = ImmersedBoundaryGrid(global_underlying_grid, GridFittedBottom(bottom)) for (grid, global_grid) in zip((underlying_grid, immersed_grid, immersed_active_grid), (global_underlying_grid, global_immersed_grid, global_immersed_grid)) - @testset "Test solid rotation on $(summary(grid)) with $(active_cells_map(grid)) on $arch)" begin - # "s" for "serial" computation - us, vs, ws, cs, ηs = solid_body_rotation_test(global_grid) - - us = interior(on_architecture(CPU(), us)) - vs = interior(on_architecture(CPU(), vs)) - ws = interior(on_architecture(CPU(), ws)) - cs = interior(on_architecture(CPU(), cs)) - ηs = interior(on_architecture(CPU(), ηs)) - - @info " Testing distributed solid body rotation with architecture $arch on $(typeof(grid).name.wrapper)" - u, v, w, c, η = solid_body_rotation_test(grid) - - cpu_arch = cpu_architecture(arch) - - u = interior(on_architecture(cpu_arch, u)) - v = interior(on_architecture(cpu_arch, v)) - w = interior(on_architecture(cpu_arch, w)) - c = interior(on_architecture(cpu_arch, c)) - η = interior(on_architecture(cpu_arch, η)) - - us = partition(us, cpu_arch, size(u)) - vs = partition(vs, cpu_arch, size(v)) - ws = partition(ws, cpu_arch, size(w)) - cs = partition(cs, cpu_arch, size(c)) - ηs = partition(ηs, cpu_arch, size(η)) - - atol = eps(eltype(grid)) - rtol = sqrt(eps(eltype(grid))) - - @test all(isapprox(u, us; atol, rtol)) - @test all(isapprox(v, vs; atol, rtol)) - @test all(isapprox(w, ws; atol, rtol)) - @test all(isapprox(c, cs; atol, rtol)) - @test all(isapprox(η, ηs; atol, rtol)) - end + + # "s" for "serial" computation + us, vs, ws, cs, ηs = solid_body_rotation_test(global_grid) + + us = interior(on_architecture(CPU(), us)) + vs = interior(on_architecture(CPU(), vs)) + ws = interior(on_architecture(CPU(), ws)) + cs = interior(on_architecture(CPU(), cs)) + ηs = interior(on_architecture(CPU(), ηs)) + + @info " Testing distributed solid body rotation with architecture $arch on $(typeof(grid).name.wrapper)" + u, v, w, c, η = solid_body_rotation_test(grid) + + cpu_arch = cpu_architecture(arch) + + u = interior(on_architecture(cpu_arch, u)) + v = interior(on_architecture(cpu_arch, v)) + w = interior(on_architecture(cpu_arch, w)) + c = interior(on_architecture(cpu_arch, c)) + η = interior(on_architecture(cpu_arch, η)) + + us = partition(us, cpu_arch, size(u)) + vs = partition(vs, cpu_arch, size(v)) + ws = partition(ws, cpu_arch, size(w)) + cs = partition(cs, cpu_arch, size(c)) + ηs = partition(ηs, cpu_arch, size(η)) + + atol = eps(eltype(grid)) + rtol = sqrt(eps(eltype(grid))) + + @test all(isapprox(u, us; atol, rtol)) + @test all(isapprox(v, vs; atol, rtol)) + @test all(isapprox(w, ws; atol, rtol)) + @test all(isapprox(c, cs; atol, rtol)) + @test all(isapprox(η, ηs; atol, rtol)) end end end diff --git a/validation/distributed_simulations/distributed_hydrostatic_turbulence.jl b/validation/distributed_simulations/distributed_hydrostatic_turbulence.jl index 0156951505..6e88bdd049 100644 --- a/validation/distributed_simulations/distributed_hydrostatic_turbulence.jl +++ b/validation/distributed_simulations/distributed_hydrostatic_turbulence.jl @@ -65,7 +65,7 @@ end Nx = 32 Ny = 32 -arch = Distributed(CPU(), partition = Partition(2)) +arch = Distributed(CPU(), partition = Partition(2, 2)) # Run the simulation run_simulation(Nx, Ny, arch) diff --git a/validation/implicit_free_surface/geostrophic_adjustment_test.jl b/validation/implicit_free_surface/geostrophic_adjustment_test.jl index a3290a8f83..45d38d8987 100644 --- a/validation/implicit_free_surface/geostrophic_adjustment_test.jl +++ b/validation/implicit_free_surface/geostrophic_adjustment_test.jl @@ -47,7 +47,7 @@ function geostrophic_adjustment_simulation(free_surface, topology, multi_region; L = grid.Lx / 40 # gaussian width x₀ = grid.Lx / 4 # gaussian center - vᵍ(x) = -U * (x - x₀) / L * gaussian(x - x₀, L) + vᵍ(x, y, z) = -U * (x - x₀) / L * gaussian(x - x₀, L) g = model.free_surface.gravitational_acceleration η = model.free_surface.η @@ -56,7 +56,7 @@ function geostrophic_adjustment_simulation(free_surface, topology, multi_region; ηᵍ(x) = η₀ * gaussian(x - x₀, L) - ηⁱ(x) = 2 * ηᵍ(x) + ηⁱ(x, y, z) = 2 * ηᵍ(x) set!(model, v = vᵍ) set!(model.free_surface.η, ηⁱ) @@ -112,12 +112,7 @@ using Oceananigans.Models.HydrostaticFreeSurfaceModels: averaging_shape_function AdamsBashforth3Scheme # @inline new_function(t) = averaging_shape_function(t; r = 0.15766, p = 2, q = 2) -@inline new_function(t) = t >= 1/2 && t <= 3/2 ? 1.0 : 0.0 - -splitexplicit_free_surface = SplitExplicitFreeSurface(substeps = 10, - averaging_weighting_function = averaging_shape_function, - timestepper = AdamsBashforth3Scheme()) -explicit_free_surface = ExplicitFreeSurface() +@inline new_function(t) = t ≥ 1/2 && t ≤ 3/2 ? 1.0 : 0.0 topology_types = [(Bounded, Periodic, Bounded), (Periodic, Periodic, Bounded)] topology_types = [topology_types[2]] diff --git a/validation/thermal_bubble/thermal_bubble.jl b/validation/thermal_bubble/thermal_bubble.jl index eb747510dc..5358786ae0 100644 --- a/validation/thermal_bubble/thermal_bubble.jl +++ b/validation/thermal_bubble/thermal_bubble.jl @@ -62,7 +62,7 @@ function print_progress(simulation) u_max = maximum(abs, interior(model.velocities.u)) w_max = maximum(abs, interior(model.velocities.w)) T_min, T_max = extrema(interior(model.tracers.T)) - CFL = max(u_max, w_max) * simulation.Δt / min(model.grid.Δxᶜᵃᵃ, model.grid.z.Δᶜ) + CFL = max(u_max, w_max) * simulation.Δt / min(model.grid.Δxᶜᵃᵃ, model.grid.Δzᵃᵃᶜ) i, t = model.clock.iteration, model.clock.time @info @sprintf("[%06.2f%%] i: %d, t: %.4f, U_max: (%.2e, %.2e), T: (min=%.5f, max=%.5f), CFL: %.4f", From bb7d8ab81a753c2a1e6398ac46fa929ee9f7eeb6 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 11 Dec 2024 09:15:25 +0100 Subject: [PATCH 535/567] revert to rk3 --- .../prescribed_hydrostatic_velocity_fields.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Models/HydrostaticFreeSurfaceModels/prescribed_hydrostatic_velocity_fields.jl b/src/Models/HydrostaticFreeSurfaceModels/prescribed_hydrostatic_velocity_fields.jl index 7ccd753800..b63b797321 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/prescribed_hydrostatic_velocity_fields.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/prescribed_hydrostatic_velocity_fields.jl @@ -90,6 +90,7 @@ hydrostatic_tendency_fields(::PrescribedVelocityFields, free_surface, grid, trac @inline sum_of_velocities(U1, U2, U3::PrescribedVelocityFields) = sum_of_velocities(U1, U2, velocities(U3)) ab2_step_velocities!(::PrescribedVelocityFields, args...) = nothing +rk3_substep_velocities!(::PrescribedVelocityFields, args...) = nothing step_free_surface!(::Nothing, model, timestepper, Δt) = nothing compute_w_from_continuity!(::PrescribedVelocityFields, args...; kwargs...) = nothing From eebc1f5c75c20a6ce13175932e85e4049c72a54f Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 11 Dec 2024 09:37:12 +0100 Subject: [PATCH 536/567] make sure we do not need to use regular grids --- src/Grids/grid_generation.jl | 2 +- .../spacings_and_areas_and_volumes.jl | 18 ++++++------------ 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/src/Grids/grid_generation.jl b/src/Grids/grid_generation.jl index 51831b4313..e23b11dffe 100644 --- a/src/Grids/grid_generation.jl +++ b/src/Grids/grid_generation.jl @@ -174,7 +174,7 @@ function generate_coordinate(FT, topo, size, halo, coordinate::ZStarVerticalCoor throw(ArgumentError(msg)) end - r_faces = coordinate.cᶠ + r_faces = coordinate.cᵃᵃᶠ Lr, rᵃᵃᶠ, rᵃᵃᶜ, Δrᵃᵃᶠ, Δrᵃᵃᶜ = generate_coordinate(FT, topo[3](), Nz, Hz, r_faces, :r, arch) diff --git a/src/Operators/spacings_and_areas_and_volumes.jl b/src/Operators/spacings_and_areas_and_volumes.jl index b759807408..9f8f866a3e 100644 --- a/src/Operators/spacings_and_areas_and_volumes.jl +++ b/src/Operators/spacings_and_areas_and_volumes.jl @@ -103,20 +103,14 @@ end ##### One - dimensional Vertical spacing (same for all grids) ##### +@inline getspacing(k, Δz::Number) = Δz +@inline getspacing(k, Δz::AbstractVector) = @inbounds Δz[k] -const ZRG = Union{LLGZ, RGZ, OSSGZ} +@inline Δrᵃᵃᶜ(i, j, k, grid) = getspacing(k, grid.z.Δᵃᵃᶜ) +@inline Δrᵃᵃᶠ(i, j, k, grid) = getspacing(k, grid.z.Δᵃᵃᶠ) -@inline Δrᵃᵃᶜ(i, j, k, grid) = @inbounds grid.z.Δᵃᵃᶜ[k] -@inline Δrᵃᵃᶠ(i, j, k, grid) = @inbounds grid.z.Δᵃᵃᶠ[k] - -@inline Δrᵃᵃᶜ(i, j, k, grid::ZRG) = grid.z.Δᵃᵃᶜ -@inline Δrᵃᵃᶠ(i, j, k, grid::ZRG) = grid.z.Δᵃᵃᶠ - -@inline Δzᵃᵃᶜ(i, j, k, grid) = @inbounds grid.z.Δᵃᵃᶜ[k] -@inline Δzᵃᵃᶠ(i, j, k, grid) = @inbounds grid.z.Δᵃᵃᶠ[k] - -@inline Δzᵃᵃᶜ(i, j, k, grid::ZRG) = grid.z.Δᵃᵃᶜ -@inline Δzᵃᵃᶠ(i, j, k, grid::ZRG) = grid.z.Δᵃᵃᶠ +@inline Δzᵃᵃᶜ(i, j, k, grid) = getspacing(k, grid.z.Δᵃᵃᶜ) +@inline Δzᵃᵃᶠ(i, j, k, grid) = getspacing(k, grid.z.Δᵃᵃᶠ) ##### ##### From 569b9b356bbcba7da918675ce97da2bf864bda3f Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 11 Dec 2024 09:38:15 +0100 Subject: [PATCH 537/567] =?UTF-8?q?change=20e=E2=82=83=20to=20=CF=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/Advection/Advection.jl | 2 +- .../vector_invariant_cross_upwinding.jl | 10 ++-- .../vector_invariant_self_upwinding.jl | 6 +-- src/Grids/grid_generation.jl | 18 +++---- src/Grids/vertical_coordinate.jl | 48 +++++++++---------- src/ImmersedBoundaries/zstar_immersed_grid.jl | 8 ++-- .../compute_w_from_continuity.jl | 10 ++-- .../hydrostatic_free_surface_ab2_step.jl | 8 ++-- .../z_star_vertical_spacing.jl | 42 ++++++++-------- src/Operators/Operators.jl | 2 +- src/Operators/variable_grid_operators.jl | 34 ++++++------- .../vertically_implicit_diffusion_solver.jl | 2 +- test/test_zstar_coordinate.jl | 4 +- 13 files changed, 97 insertions(+), 97 deletions(-) diff --git a/src/Advection/Advection.jl b/src/Advection/Advection.jl index c056710ccd..d528aa8610 100644 --- a/src/Advection/Advection.jl +++ b/src/Advection/Advection.jl @@ -34,7 +34,7 @@ using Oceananigans.Grids: with_halo using Oceananigans.Architectures: architecture, CPU using Oceananigans.Operators -using Oceananigans.Operators: flux_div_xyᶜᶜᶜ, Γᶠᶠᶜ, ∂t_e₃ +using Oceananigans.Operators: flux_div_xyᶜᶜᶜ, Γᶠᶠᶜ, ∂t_σ import Base: show, summary import Oceananigans.Grids: required_halo_size_x, required_halo_size_y, required_halo_size_z diff --git a/src/Advection/vector_invariant_cross_upwinding.jl b/src/Advection/vector_invariant_cross_upwinding.jl index 7f82a509a7..b5aaa079f7 100644 --- a/src/Advection/vector_invariant_cross_upwinding.jl +++ b/src/Advection/vector_invariant_cross_upwinding.jl @@ -20,24 +20,24 @@ # This is the additional term appearing in the continuity equation due to a moving grid. # The discrete divergence is calculated as: # -# wᵏ⁺¹ - wᵏ δx(Ax u) + δy(Ay v) Δrᶜᶜᶜ ∂t_e₃ +# wᵏ⁺¹ - wᵏ δx(Ax u) + δy(Ay v) Δrᶜᶜᶜ ∂t_σ # ---------- = - --------------------- - ------------- # Δzᶜᶜᶜ Vᶜᶜᶜ Δzᶜᶜᶜ # # We upwind with the discrete divergence `δx(Ax u) + δy(Ay v)` and then divide by the volume, # therefore, the correct term to be added to the divergence transport due to the moving grid is: # -# Azᶜᶜᶜ Δrᶜᶜᶜ ∂t_e₃ +# Azᶜᶜᶜ Δrᶜᶜᶜ ∂t_σ # # which represents the static volume times the time derivative of the vertical grid scaling. -@inline V_times_∂t_e₃(i, j, k, grid) = Azᶜᶜᶜ(i, j, k, grid) * Δrᶜᶜᶜ(i, j, k, grid) * ∂t_e₃(i, j, k, grid) +@inline V_times_∂t_σ(i, j, k, grid) = Azᶜᶜᶜ(i, j, k, grid) * Δrᶜᶜᶜ(i, j, k, grid) * ∂t_σ(i, j, k, grid) @inline function upwinded_divergence_flux_Uᶠᶜᶜ(i, j, k, grid, scheme::VectorInvariantCrossVerticalUpwinding, u, v) @inbounds û = u[i, j, k] δ_stencil = scheme.upwinding.divergence_stencil δᴿ = _biased_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, bias(û), flux_div_xyᶜᶜᶜ, δ_stencil, u, v) - ∂ts = _symmetric_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, cross_scheme, V_times_∂t_e₃) + ∂ts = _symmetric_interpolate_xᶠᵃᵃ(i, j, k, grid, scheme, cross_scheme, V_times_∂t_σ) return û * (δᴿ + ∂ts) end @@ -47,7 +47,7 @@ end δ_stencil = scheme.upwinding.divergence_stencil δᴿ = _biased_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, scheme.divergence_scheme, bias(v̂), flux_div_xyᶜᶜᶜ, δ_stencil, u, v) - ∂ts = _symmetric_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, cross_scheme, V_times_∂t_e₃) + ∂ts = _symmetric_interpolate_yᵃᶠᵃ(i, j, k, grid, scheme, cross_scheme, V_times_∂t_σ) return v̂ * (δᴿ + ∂ts) end diff --git a/src/Advection/vector_invariant_self_upwinding.jl b/src/Advection/vector_invariant_self_upwinding.jl index c530fbc4b1..aed2d6ad91 100644 --- a/src/Advection/vector_invariant_self_upwinding.jl +++ b/src/Advection/vector_invariant_self_upwinding.jl @@ -5,15 +5,15 @@ @inline δx_U(i, j, k, grid, u, v) = δxᶜᶜᶜ(i, j, k, grid, Ax_qᶠᶜᶜ, u) @inline δy_V(i, j, k, grid, u, v) = δyᶜᶜᶜ(i, j, k, grid, Ay_qᶜᶠᶜ, v) -@inline δx_U_plus_metric(i, j, k, grid, u, v) = δxᶜᶜᶜ(i, j, k, grid, Ax_qᶠᶜᶜ, u) + V_times_∂t_e₃(i, j, k, grid) -@inline δy_V_plus_metric(i, j, k, grid, u, v) = δyᶜᶜᶜ(i, j, k, grid, Ay_qᶜᶠᶜ, v) + V_times_∂t_e₃(i, j, k, grid) +@inline δx_U_plus_metric(i, j, k, grid, u, v) = δxᶜᶜᶜ(i, j, k, grid, Ax_qᶠᶜᶜ, u) + V_times_∂t_σ(i, j, k, grid) +@inline δy_V_plus_metric(i, j, k, grid, u, v) = δyᶜᶜᶜ(i, j, k, grid, Ay_qᶜᶠᶜ, v) + V_times_∂t_σ(i, j, k, grid) # Velocity smoothness for divergence upwinding @inline U_smoothness(i, j, k, grid, u, v) = ℑxᶜᵃᵃ(i, j, k, grid, Ax_qᶠᶜᶜ, u) @inline V_smoothness(i, j, k, grid, u, v) = ℑyᵃᶜᵃ(i, j, k, grid, Ay_qᶜᶠᶜ, v) # Divergence smoothness for divergence upwinding -@inline divergence_smoothness(i, j, k, grid, u, v) = δx_U(i, j, k, grid, u, v) + δy_V(i, j, k, grid, u, v) + V_times_∂t_e₃(i, j, k, grid) +@inline divergence_smoothness(i, j, k, grid, u, v) = δx_U(i, j, k, grid, u, v) + δy_V(i, j, k, grid, u, v) + V_times_∂t_σ(i, j, k, grid) @inline function upwinded_divergence_flux_Uᶠᶜᶜ(i, j, k, grid, scheme::VectorInvariantSelfVerticalUpwinding, u, v) diff --git a/src/Grids/grid_generation.jl b/src/Grids/grid_generation.jl index e23b11dffe..46176517e7 100644 --- a/src/Grids/grid_generation.jl +++ b/src/Grids/grid_generation.jl @@ -180,18 +180,18 @@ function generate_coordinate(FT, topo, size, halo, coordinate::ZStarVerticalCoor args = (topo, (Nx, Ny, Nz), (Hx, Hy, Hz)) - e₃ᶜᶜ⁻ = new_data(FT, arch, (Center, Center, Nothing), args...) - e₃ᶜᶜⁿ = new_data(FT, arch, (Center, Center, Nothing), args...) - e₃ᶠᶜⁿ = new_data(FT, arch, (Face, Center, Nothing), args...) - e₃ᶜᶠⁿ = new_data(FT, arch, (Center, Face, Nothing), args...) - e₃ᶠᶠⁿ = new_data(FT, arch, (Face, Face, Nothing), args...) + σᶜᶜ⁻ = new_data(FT, arch, (Center, Center, Nothing), args...) + σᶜᶜⁿ = new_data(FT, arch, (Center, Center, Nothing), args...) + σᶠᶜⁿ = new_data(FT, arch, (Face, Center, Nothing), args...) + σᶜᶠⁿ = new_data(FT, arch, (Center, Face, Nothing), args...) + σᶠᶠⁿ = new_data(FT, arch, (Face, Face, Nothing), args...) ηⁿ = new_data(FT, arch, (Center, Center, Nothing), args...) - ∂t_e₃ = new_data(FT, arch, (Center, Center, Nothing), args...) + ∂t_σ = new_data(FT, arch, (Center, Center, Nothing), args...) # Fill all the scalings with one (at rest coordinate) - for e₃ in (e₃ᶜᶜ⁻, e₃ᶜᶜⁿ, e₃ᶠᶜⁿ, e₃ᶜᶠⁿ, e₃ᶠᶠⁿ) - fill!(e₃, 1) + for σ in (σᶜᶜ⁻, σᶜᶜⁿ, σᶠᶜⁿ, σᶜᶠⁿ, σᶠᶠⁿ) + fill!(σ, 1) end - return Lr, ZStarVerticalCoordinate(rᵃᵃᶠ, rᵃᵃᶜ, Δrᵃᵃᶠ, Δrᵃᵃᶜ, ηⁿ, e₃ᶜᶜⁿ, e₃ᶠᶜⁿ, e₃ᶜᶠⁿ, e₃ᶠᶠⁿ, e₃ᶜᶜ⁻, ∂t_e₃) + return Lr, ZStarVerticalCoordinate(rᵃᵃᶠ, rᵃᵃᶜ, Δrᵃᵃᶠ, Δrᵃᵃᶜ, ηⁿ, σᶜᶜⁿ, σᶠᶜⁿ, σᶜᶠⁿ, σᶠᶠⁿ, σᶜᶜ⁻, ∂t_σ) end diff --git a/src/Grids/vertical_coordinate.jl b/src/Grids/vertical_coordinate.jl index b4b9b86999..cd8af3171b 100644 --- a/src/Grids/vertical_coordinate.jl +++ b/src/Grids/vertical_coordinate.jl @@ -31,24 +31,24 @@ end # - `Δᶜ::D`: Cell-centered grid spacing. # - `Δᶠ::D`: Face-centered grid spacing. # - `ηⁿ::E`: Surface elevation at the current time step. -# - `e₃ᶜᶜⁿ::CC`: Vertical grid scaling at center-center at the current time step. -# - `e₃ᶠᶜⁿ::FC`: Vertical grid scaling at face-center at the current time step. -# - `e₃ᶜᶠⁿ::CF`: Vertical grid scaling at center-face at the current time step. -# - `e₃ᶠᶠⁿ::FF`: Vertical grid scaling at face-face at the current time step. -# - `e₃ᶜᶜ⁻::CC`: Vertical grid scaling at center-center at the previous time step. -# - `∂t_e₃::CC`: Time derivative of the vertical grid scaling at cell centers. +# - `σᶜᶜⁿ::CC`: Vertical grid scaling at center-center at the current time step. +# - `σᶠᶜⁿ::FC`: Vertical grid scaling at face-center at the current time step. +# - `σᶜᶠⁿ::CF`: Vertical grid scaling at center-face at the current time step. +# - `σᶠᶠⁿ::FF`: Vertical grid scaling at face-face at the current time step. +# - `σᶜᶜ⁻::CC`: Vertical grid scaling at center-center at the previous time step. +# - `∂t_σ::CC`: Time derivative of the vertical grid scaling at cell centers. struct ZStarVerticalCoordinate{C, D, E, CC, FC, CF, FF} <: AbstractVerticalCoordinate cᵃᵃᶠ :: C cᵃᵃᶜ :: C Δᵃᵃᶠ :: D Δᵃᵃᶜ :: D ηⁿ :: E - e₃ᶜᶜⁿ :: CC - e₃ᶠᶜⁿ :: FC - e₃ᶜᶠⁿ :: CF - e₃ᶠᶠⁿ :: FF - e₃ᶜᶜ⁻ :: CC - ∂t_e₃ :: CC + σᶜᶜⁿ :: CC + σᶠᶜⁿ :: FC + σᶜᶠⁿ :: CF + σᶠᶠⁿ :: FF + σᶜᶜ⁻ :: CC + ∂t_σ :: CC end # Convenience constructors for Zstar vertical coordinate @@ -90,12 +90,12 @@ Adapt.adapt_structure(to, coord::ZStarVerticalCoordinate) = Adapt.adapt(to, coord.Δᵃᵃᶠ), Adapt.adapt(to, coord.Δᵃᵃᶜ), Adapt.adapt(to, coord.ηⁿ), - Adapt.adapt(to, coord.e₃ᶜᶜⁿ), - Adapt.adapt(to, coord.e₃ᶠᶜⁿ), - Adapt.adapt(to, coord.e₃ᶜᶠⁿ), - Adapt.adapt(to, coord.e₃ᶠᶠⁿ), - Adapt.adapt(to, coord.e₃ᶜᶜ⁻), - Adapt.adapt(to, coord.∂t_e₃)) + Adapt.adapt(to, coord.σᶜᶜⁿ), + Adapt.adapt(to, coord.σᶠᶜⁿ), + Adapt.adapt(to, coord.σᶜᶠⁿ), + Adapt.adapt(to, coord.σᶠᶠⁿ), + Adapt.adapt(to, coord.σᶜᶜ⁻), + Adapt.adapt(to, coord.∂t_σ)) on_architecture(arch, coord::ZStarVerticalCoordinate) = ZStarVerticalCoordinate(on_architecture(arch, coord.cᵃᵃᶠ), @@ -103,12 +103,12 @@ on_architecture(arch, coord::ZStarVerticalCoordinate) = on_architecture(arch, coord.Δᵃᵃᶠ), on_architecture(arch, coord.Δᵃᵃᶜ), on_architecture(arch, coord.ηⁿ), - on_architecture(arch, coord.e₃ᶜᶜⁿ), - on_architecture(arch, coord.e₃ᶠᶜⁿ), - on_architecture(arch, coord.e₃ᶜᶠⁿ), - on_architecture(arch, coord.e₃ᶠᶠⁿ), - on_architecture(arch, coord.e₃ᶜᶜ⁻), - on_architecture(arch, coord.∂t_e₃)) + on_architecture(arch, coord.σᶜᶜⁿ), + on_architecture(arch, coord.σᶠᶜⁿ), + on_architecture(arch, coord.σᶜᶠⁿ), + on_architecture(arch, coord.σᶠᶠⁿ), + on_architecture(arch, coord.σᶜᶜ⁻), + on_architecture(arch, coord.∂t_σ)) ##### ##### Nodes and spacings (common to every grid)... diff --git a/src/ImmersedBoundaries/zstar_immersed_grid.jl b/src/ImmersedBoundaries/zstar_immersed_grid.jl index 385db35497..4eb1fc6959 100644 --- a/src/ImmersedBoundaries/zstar_immersed_grid.jl +++ b/src/ImmersedBoundaries/zstar_immersed_grid.jl @@ -6,7 +6,7 @@ import Oceananigans.Grids: dynamic_column_depthᶜᶜᵃ, dynamic_column_depthᶠᶜᵃ, dynamic_column_depthᶠᶠᵃ -import Oceananigans.Operators: e₃ⁿ, e₃⁻, ∂t_e₃ +import Oceananigans.Operators: σⁿ, σ⁻, ∂t_σ const ZStarImmersedGrid = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:AbstractZStarGrid} const ZStarGridOfSomeKind = Union{ZStarImmersedGrid, AbstractZStarGrid} @@ -28,7 +28,7 @@ const ZStarGridOfSomeKind = Union{ZStarImmersedGrid, AbstractZStarGrid} @inline dynamic_column_depthᶠᶠᵃ(i, j, grid::ZStarGridOfSomeKind) = dynamic_column_depthᶠᶠᵃ(i, j, 1, grid, grid.z.ηⁿ) # Fallbacks -@inline e₃ⁿ(i, j, k, ibg::IBG, ℓx, ℓy, ℓz) = e₃ⁿ(i, j, k, ibg.underlying_grid, ℓx, ℓy, ℓz) -@inline e₃⁻(i, j, k, ibg::IBG, ℓx, ℓy, ℓz) = e₃⁻(i, j, k, ibg.underlying_grid, ℓx, ℓy, ℓz) +@inline σⁿ(i, j, k, ibg::IBG, ℓx, ℓy, ℓz) = σⁿ(i, j, k, ibg.underlying_grid, ℓx, ℓy, ℓz) +@inline σ⁻(i, j, k, ibg::IBG, ℓx, ℓy, ℓz) = σ⁻(i, j, k, ibg.underlying_grid, ℓx, ℓy, ℓz) -@inline ∂t_e₃(i, j, k, ibg::IBG) = ∂t_e₃(i, j, k, ibg.underlying_grid) +@inline ∂t_σ(i, j, k, ibg::IBG) = ∂t_σ(i, j, k, ibg.underlying_grid) diff --git a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl index 4fcd297b7c..85919ea441 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/compute_w_from_continuity.jl @@ -23,19 +23,19 @@ compute_w_from_continuity!(velocities, arch, grid; parameters = w_kernel_paramet # Since the derivative of the moving grid is: # # δx(Δy U) + δy(Δx V) ∇ ⋅ U -# ∂t_e₃ = - --------------------- = - -------- +# ∂t_σ = - --------------------- = - -------- # Az ⋅ H H # # The discrete divergence is calculated as: # -# wᵏ⁺¹ - wᵏ δx(Ax u) + δy(Ay v) Δr ∂t_e₃ +# wᵏ⁺¹ - wᵏ δx(Ax u) + δy(Ay v) Δr ∂t_σ # ---------- = - --------------------- - ---------- # Δz V Δz # # This makes sure that if we sum up till the top of the domain, we get # # ∇ ⋅ U -# wᴺᶻ⁺¹ = w⁰ - ------- - ∂t_e₃ ≈ 0 (if w⁰ == 0) +# wᴺᶻ⁺¹ = w⁰ - ------- - ∂t_σ ≈ 0 (if w⁰ == 0) # H # @kernel function _compute_w_from_continuity!(U, grid) @@ -44,10 +44,10 @@ compute_w_from_continuity!(velocities, arch, grid; parameters = w_kernel_paramet @inbounds U.w[i, j, 1] = 0 for k in 2:grid.Nz+1 δh_u = flux_div_xyᶜᶜᶜ(i, j, k-1, grid, U.u, U.v) / Azᶜᶜᶜ(i, j, k-1, grid) - ∂te₃ = Δrᶜᶜᶜ(i, j, k-1, grid) * ∂t_e₃(i, j, k-1, grid) + ∂tσ = Δrᶜᶜᶜ(i, j, k-1, grid) * ∂t_σ(i, j, k-1, grid) immersed = immersed_cell(i, j, k-1, grid) - Δw = δh_u + ifelse(immersed, 0, ∂te₃) # We do not account for grid changes in immersed cells + Δw = δh_u + ifelse(immersed, 0, ∂tσ) # We do not account for grid changes in immersed cells @inbounds U.w[i, j, k] = U.w[i, j, k-1] - Δw end diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl index af20e011c3..4faab520f0 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl @@ -110,7 +110,7 @@ ab2_step_tracer_field!(tracer_field, grid, Δt, χ, Gⁿ, G⁻) = ##### ##### Tracer update in generalized vertical coordinates -##### We advance e₃θ but store θ once e₃ⁿ⁺¹ is known +##### We advance σθ but store θ once σⁿ⁺¹ is known ##### @kernel function _ab2_step_tracer_field!(θ, grid, Δt, χ, Gⁿ, G⁻) @@ -120,8 +120,8 @@ ab2_step_tracer_field!(tracer_field, grid, Δt, χ, Gⁿ, G⁻) = C₁ = convert(FT, 1.5) + χ C₂ = convert(FT, 0.5) + χ - eⁿ = e₃ⁿ(i, j, k, grid, Center(), Center(), Center()) - e⁻ = e₃⁻(i, j, k, grid, Center(), Center(), Center()) + eⁿ = σⁿ(i, j, k, grid, Center(), Center(), Center()) + e⁻ = σ⁻(i, j, k, grid, Center(), Center(), Center()) @inbounds begin ∂t_sθ = C₁ * eⁿ * Gⁿ[i, j, k] - C₂ * e⁻ * G⁻[i, j, k] @@ -132,5 +132,5 @@ ab2_step_tracer_field!(tracer_field, grid, Δt, χ, Gⁿ, G⁻) = end # Fallback! We need to unscale the tracers only in case of -# a grid with a moving vertical cocrdinate, i.e. where e₃ is not constant +# a grid with a moving vertical cocrdinate, i.e. where σ is not constant unscale_tracers!(tracers, grid; kwargs...) = nothing diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index e7917b1f16..67dba6e2cb 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -23,17 +23,17 @@ update_grid!(model, grid; parameters) = nothing function update_grid!(model, grid::ZStarGridOfSomeKind; parameters = :xy) # Scalings and free surface - e₃ᶜᶜ⁻ = grid.z.e₃ᶜᶜ⁻ - e₃ᶜᶜⁿ = grid.z.e₃ᶜᶜⁿ - e₃ᶠᶜⁿ = grid.z.e₃ᶠᶜⁿ - e₃ᶜᶠⁿ = grid.z.e₃ᶜᶠⁿ - e₃ᶠᶠⁿ = grid.z.e₃ᶠᶠⁿ - ∂t_e₃ = grid.z.∂t_e₃ + σᶜᶜ⁻ = grid.z.σᶜᶜ⁻ + σᶜᶜⁿ = grid.z.σᶜᶜⁿ + σᶠᶜⁿ = grid.z.σᶠᶜⁿ + σᶜᶠⁿ = grid.z.σᶜᶠⁿ + σᶠᶠⁿ = grid.z.σᶠᶠⁿ + ∂t_σ = grid.z.∂t_σ ηⁿ = grid.z.ηⁿ η = model.free_surface.η launch!(architecture(grid), grid, parameters, _update_grid_scaling!, - e₃ᶜᶜⁿ, e₃ᶠᶜⁿ, e₃ᶜᶠⁿ, e₃ᶠᶠⁿ, e₃ᶜᶜ⁻, ηⁿ, grid, η) + σᶜᶜⁿ, σᶠᶜⁿ, σᶜᶠⁿ, σᶠᶠⁿ, σᶜᶜ⁻, ηⁿ, grid, η) # the barotropic velocities are retrieved from the free surface model for a # SplitExplicitFreeSurface and are calculated for other free surface models @@ -42,12 +42,12 @@ function update_grid!(model, grid::ZStarGridOfSomeKind; parameters = :xy) # Update vertical spacing with available parameters # No need to fill the halo as the scaling is updated _IN_ the halos - launch!(architecture(grid), grid, parameters, _update_grid_vertical_velocity!, ∂t_e₃, grid, U, V, u, v) + launch!(architecture(grid), grid, parameters, _update_grid_vertical_velocity!, ∂t_σ, grid, U, V, u, v) return nothing end -@kernel function _update_grid_scaling!(e₃ᶜᶜⁿ, e₃ᶠᶜⁿ, e₃ᶜᶠⁿ, e₃ᶠᶠⁿ, e₃ᶜᶜ⁻, ηⁿ, grid, η) +@kernel function _update_grid_scaling!(σᶜᶜⁿ, σᶠᶜⁿ, σᶜᶠⁿ, σᶠᶠⁿ, σᶜᶜ⁻, ηⁿ, grid, η) i, j = @index(Global, NTuple) k_top = size(grid, 3) + 1 @@ -62,26 +62,26 @@ end Hᶠᶠ = dynamic_column_depthᶠᶠᵃ(i, j, k_top, grid, η) @inbounds begin - e₃ᶜᶜ = ifelse(hᶜᶜ == 0, one(grid), Hᶜᶜ / hᶜᶜ) - e₃ᶠᶜ = ifelse(hᶠᶜ == 0, one(grid), Hᶠᶜ / hᶠᶜ) - e₃ᶜᶠ = ifelse(hᶜᶠ == 0, one(grid), Hᶜᶠ / hᶜᶠ) - e₃ᶠᶠ = ifelse(hᶠᶠ == 0, one(grid), Hᶠᶠ / hᶠᶠ) + σᶜᶜ = ifelse(hᶜᶜ == 0, one(grid), Hᶜᶜ / hᶜᶜ) + σᶠᶜ = ifelse(hᶠᶜ == 0, one(grid), Hᶠᶜ / hᶠᶜ) + σᶜᶠ = ifelse(hᶜᶠ == 0, one(grid), Hᶜᶠ / hᶜᶠ) + σᶠᶠ = ifelse(hᶠᶠ == 0, one(grid), Hᶠᶠ / hᶠᶠ) # Update previous scaling - e₃ᶜᶜ⁻[i, j, 1] = e₃ᶜᶜⁿ[i, j, 1] + σᶜᶜ⁻[i, j, 1] = σᶜᶜⁿ[i, j, 1] # update current scaling - e₃ᶜᶜⁿ[i, j, 1] = e₃ᶜᶜ - e₃ᶠᶜⁿ[i, j, 1] = e₃ᶠᶜ - e₃ᶜᶠⁿ[i, j, 1] = e₃ᶜᶠ - e₃ᶠᶠⁿ[i, j, 1] = e₃ᶠᶠ + σᶜᶜⁿ[i, j, 1] = σᶜᶜ + σᶠᶜⁿ[i, j, 1] = σᶠᶜ + σᶜᶠⁿ[i, j, 1] = σᶜᶠ + σᶠᶠⁿ[i, j, 1] = σᶠᶠ # Update η in the grid ηⁿ[i, j, 1] = η[i, j, k_top] end end -@kernel function _update_grid_vertical_velocity!(∂t_e₃, grid, U, V, u, v) +@kernel function _update_grid_vertical_velocity!(∂t_σ, grid, U, V, u, v) i, j = @index(Global, NTuple) kᴺ = size(grid, 3) @@ -93,7 +93,7 @@ end δh_U = (δx_U + δy_V) / Azᶜᶜᶜ(i, j, kᴺ, grid) - @inbounds ∂t_e₃[i, j, 1] = ifelse(hᶜᶜ == 0, zero(grid), - δh_U / hᶜᶜ) + @inbounds ∂t_σ[i, j, 1] = ifelse(hᶜᶜ == 0, zero(grid), - δh_U / hᶜᶜ) end # If U and V exist, we just take them @@ -161,6 +161,6 @@ end i, j, n = @index(Global, NTuple) @unroll for k in -Hz+1:Nz+Hz - tracers[n][i, j, k] /= e₃ⁿ(i, j, k, grid, Center(), Center(), Center()) + tracers[n][i, j, k] /= σⁿ(i, j, k, grid, Center(), Center(), Center()) end end \ No newline at end of file diff --git a/src/Operators/Operators.jl b/src/Operators/Operators.jl index 6df0efe866..6069cdef99 100644 --- a/src/Operators/Operators.jl +++ b/src/Operators/Operators.jl @@ -78,7 +78,7 @@ export ∂xTᶠᶜᶠ, ∂yTᶜᶠᶠ export intrinsic_vector, extrinsic_vector # Variable grid operators -export e₃ⁿ, e₃⁻, ∂t_e₃ +export σⁿ, σ⁻, ∂t_σ using Oceananigans.Grids diff --git a/src/Operators/variable_grid_operators.jl b/src/Operators/variable_grid_operators.jl index 24c3eda599..b2034bae0e 100644 --- a/src/Operators/variable_grid_operators.jl +++ b/src/Operators/variable_grid_operators.jl @@ -10,19 +10,19 @@ const F = Face const ZSG = AbstractZStarGrid # Fallbacks -@inline e₃ⁿ(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) -@inline e₃⁻(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) +@inline σⁿ(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) +@inline σ⁻(i, j, k, grid, ℓx, ℓy, ℓz) = one(grid) -@inline e₃ⁿ(i, j, k, grid::ZSG, ::C, ::C, ℓz) = @inbounds grid.z.e₃ᶜᶜⁿ[i, j, 1] -@inline e₃ⁿ(i, j, k, grid::ZSG, ::F, ::C, ℓz) = @inbounds grid.z.e₃ᶠᶜⁿ[i, j, 1] -@inline e₃ⁿ(i, j, k, grid::ZSG, ::C, ::F, ℓz) = @inbounds grid.z.e₃ᶜᶠⁿ[i, j, 1] -@inline e₃ⁿ(i, j, k, grid::ZSG, ::F, ::F, ℓz) = @inbounds grid.z.e₃ᶠᶠⁿ[i, j, 1] +@inline σⁿ(i, j, k, grid::ZSG, ::C, ::C, ℓz) = @inbounds grid.z.σᶜᶜⁿ[i, j, 1] +@inline σⁿ(i, j, k, grid::ZSG, ::F, ::C, ℓz) = @inbounds grid.z.σᶠᶜⁿ[i, j, 1] +@inline σⁿ(i, j, k, grid::ZSG, ::C, ::F, ℓz) = @inbounds grid.z.σᶜᶠⁿ[i, j, 1] +@inline σⁿ(i, j, k, grid::ZSG, ::F, ::F, ℓz) = @inbounds grid.z.σᶠᶠⁿ[i, j, 1] -# e₃⁻ is needed only at centers -@inline e₃⁻(i, j, k, grid::ZSG, ::C, ::C, ℓz) = @inbounds grid.z.e₃ᶜᶜ⁻[i, j, 1] +# σ⁻ is needed only at centers +@inline σ⁻(i, j, k, grid::ZSG, ::C, ::C, ℓz) = @inbounds grid.z.σᶜᶜ⁻[i, j, 1] -@inline ∂t_e₃(i, j, k, grid) = zero(grid) -@inline ∂t_e₃(i, j, k, grid::ZSG) = @inbounds grid.z.∂t_e₃[i, j, 1] +@inline ∂t_σ(i, j, k, grid) = zero(grid) +@inline ∂t_σ(i, j, k, grid::ZSG) = @inbounds grid.z.∂t_σ[i, j, 1] #### #### Vertical spacing functions @@ -43,14 +43,14 @@ for LX in (:ᶠ, :ᶜ), LY in (:ᶠ, :ᶜ), LZ in (:ᶠ, :ᶜ) ℓz = superscript_location(LZ) @eval begin - @inline $zspacing(i, j, k, grid::ZSRG) = $rspacing(i, j, k, grid) * e₃ⁿ(i, j, k, grid, $ℓx(), $ℓy(), $ℓz()) - @inline $zspacing(i, j, k, grid::ZSLLG) = $rspacing(i, j, k, grid) * e₃ⁿ(i, j, k, grid, $ℓx(), $ℓy(), $ℓz()) - @inline $zspacing(i, j, k, grid::ZSOSG) = $rspacing(i, j, k, grid) * e₃ⁿ(i, j, k, grid, $ℓx(), $ℓy(), $ℓz()) + @inline $zspacing(i, j, k, grid::ZSRG) = $rspacing(i, j, k, grid) * σⁿ(i, j, k, grid, $ℓx(), $ℓy(), $ℓz()) + @inline $zspacing(i, j, k, grid::ZSLLG) = $rspacing(i, j, k, grid) * σⁿ(i, j, k, grid, $ℓx(), $ℓy(), $ℓz()) + @inline $zspacing(i, j, k, grid::ZSOSG) = $rspacing(i, j, k, grid) * σⁿ(i, j, k, grid, $ℓx(), $ℓy(), $ℓz()) end end # znode for an AbstractZStarGrid grid is scaled by the free surface -@inline znode(i, j, k, grid::ZSG, ::C, ::C, ℓz) = @inbounds rnode(i, j, k, grid, C(), C(), ℓz) * e₃ⁿ(i, j, k, grid, C(), C(), ℓz) + grid.z.ηⁿ[i, j, 1] -@inline znode(i, j, k, grid::ZSG, ::F, ::C, ℓz) = @inbounds rnode(i, j, k, grid, F(), C(), ℓz) * e₃ⁿ(i, j, k, grid, F(), C(), ℓz) + ℑxᶠᵃᵃ(i, j, 1, grid, grid.z.ηⁿ) -@inline znode(i, j, k, grid::ZSG, ::C, ::F, ℓz) = @inbounds rnode(i, j, k, grid, C(), F(), ℓz) * e₃ⁿ(i, j, k, grid, C(), F(), ℓz) + ℑyᵃᶠᵃ(i, j, 1, grid, grid.z.ηⁿ) -@inline znode(i, j, k, grid::ZSG, ::F, ::F, ℓz) = @inbounds rnode(i, j, k, grid, F(), F(), ℓz) * e₃ⁿ(i, j, k, grid, F(), F(), ℓz) + ℑxyᶠᶠᵃ(i, j, 1, grid, grid.z.ηⁿ) +@inline znode(i, j, k, grid::ZSG, ::C, ::C, ℓz) = @inbounds rnode(i, j, k, grid, C(), C(), ℓz) * σⁿ(i, j, k, grid, C(), C(), ℓz) + grid.z.ηⁿ[i, j, 1] +@inline znode(i, j, k, grid::ZSG, ::F, ::C, ℓz) = @inbounds rnode(i, j, k, grid, F(), C(), ℓz) * σⁿ(i, j, k, grid, F(), C(), ℓz) + ℑxᶠᵃᵃ(i, j, 1, grid, grid.z.ηⁿ) +@inline znode(i, j, k, grid::ZSG, ::C, ::F, ℓz) = @inbounds rnode(i, j, k, grid, C(), F(), ℓz) * σⁿ(i, j, k, grid, C(), F(), ℓz) + ℑyᵃᶠᵃ(i, j, 1, grid, grid.z.ηⁿ) +@inline znode(i, j, k, grid::ZSG, ::F, ::F, ℓz) = @inbounds rnode(i, j, k, grid, F(), F(), ℓz) * σⁿ(i, j, k, grid, F(), F(), ℓz) + ℑxyᶠᶠᵃ(i, j, 1, grid, grid.z.ηⁿ) diff --git a/src/TurbulenceClosures/vertically_implicit_diffusion_solver.jl b/src/TurbulenceClosures/vertically_implicit_diffusion_solver.jl index 86795b9e24..3aaa551622 100644 --- a/src/TurbulenceClosures/vertically_implicit_diffusion_solver.jl +++ b/src/TurbulenceClosures/vertically_implicit_diffusion_solver.jl @@ -46,7 +46,7 @@ const c = Center() const f = Face() # The vertical spacing used here is Δz for velocities and Δr for tracers, since the -# implicit solver operator is applied to the scaled tracer e₃θ instead of just θ +# implicit solver operator is applied to the scaled tracer σθ instead of just θ @inline vertical_spacing(i, j, k, grid, ℓx, ℓy, ℓz) = Δz(i, j, k, grid, ℓx, ℓy, ℓz) @inline vertical_spacing(i, j, k, grid, ::Center, ::Center, ℓz) = Δr(i, j, k, grid, c, c, ℓz) diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index e47ef65830..8789af01af 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -64,8 +64,8 @@ const F = Face set!(model, u = (x, y, z) -> x, v = (x, y, z) -> y) update_state!(model) - @test e₃ⁿ(1, 1, 1, grid, C(), C(), C()) == 11 / 10 - @test e₃ⁿ(2, 1, 1, grid, C(), C(), C()) == 12 / 10 + @test σⁿ(1, 1, 1, grid, C(), C(), C()) == 11 / 10 + @test σⁿ(2, 1, 1, grid, C(), C(), C()) == 12 / 10 @test znode(1, 1, 21, grid, C(), C(), F()) == 1 @test znode(2, 1, 21, grid, C(), C(), F()) == 2 From 1425c5be4dc929ac23ecd30780bd48c98ddb46df Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 11 Dec 2024 09:39:34 +0100 Subject: [PATCH 538/567] chenga name --- .../hydrostatic_free_surface_ab2_step.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl index 4faab520f0..57b344948d 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_ab2_step.jl @@ -124,10 +124,10 @@ ab2_step_tracer_field!(tracer_field, grid, Δt, χ, Gⁿ, G⁻) = e⁻ = σ⁻(i, j, k, grid, Center(), Center(), Center()) @inbounds begin - ∂t_sθ = C₁ * eⁿ * Gⁿ[i, j, k] - C₂ * e⁻ * G⁻[i, j, k] + ∂t_σθ = C₁ * eⁿ * Gⁿ[i, j, k] - C₂ * e⁻ * G⁻[i, j, k] # We store temporarily sθ in θ. the unscaled θ will be retrived later on with `unscale_tracers!` - θ[i, j, k] = eⁿ * θ[i, j, k] + convert(FT, Δt) * ∂t_sθ + θ[i, j, k] = eⁿ * θ[i, j, k] + convert(FT, Δt) * ∂t_σθ end end From 35780bff7025eb4d36e1fdf3d605437893c937f9 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 11 Dec 2024 09:54:35 +0100 Subject: [PATCH 539/567] more corrections --- src/ImmersedBoundaries/immersed_grid_metrics.jl | 6 +----- test/test_zstar_coordinate.jl | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/ImmersedBoundaries/immersed_grid_metrics.jl b/src/ImmersedBoundaries/immersed_grid_metrics.jl index f69cfc28ae..7857d20016 100644 --- a/src/ImmersedBoundaries/immersed_grid_metrics.jl +++ b/src/ImmersedBoundaries/immersed_grid_metrics.jl @@ -1,7 +1,6 @@ using Oceananigans.AbstractOperations: GridMetricOperation -import Oceananigans.Grids: coordinates -import Oceananigans.Operators: Δrᵃᵃᶠ, Δrᵃᵃᶜ +import Oceananigans.Operators: Δrᵃᵃᶠ, Δrᵃᵃᶜ, Δzᵃᵃᶠ, Δzᵃᵃᶜ import Oceananigans.Operators: Δxᶠᵃᵃ, Δxᶜᵃᵃ, Δxᶠᶜᵃ, Δxᶜᶠᵃ, Δxᶠᶠᵃ, Δxᶜᶜᵃ import Oceananigans.Operators: Δyᵃᶠᵃ, Δyᵃᶜᵃ, Δyᶠᶜᵃ, Δyᶜᶠᵃ, Δyᶠᶠᵃ, Δyᶜᶜᵃ import Oceananigans.Operators: Azᶠᶜᵃ, Azᶜᶠᵃ, Azᶠᶠᵃ, Azᶜᶜᵃ @@ -23,12 +22,9 @@ import Oceananigans.Operators: intrinsic_vector, extrinsic_vector @inline Δrᵃᵃᶠ(i, j, k, ibg::IBG) = Δrᵃᵃᶠ(i, j, k, ibg.underlying_grid) @inline Δrᵃᵃᶜ(i, j, k, ibg::IBG) = Δrᵃᵃᶜ(i, j, k, ibg.underlying_grid) -@inline Δzᵃᵃᶠ(i, j, k, ibg::IBG) = Δzᵃᵃᶠ(i, j, k, ibg.underlying_grid) @inline Δzᵃᵃᶜ(i, j, k, ibg::IBG) = Δzᵃᵃᶜ(i, j, k, ibg.underlying_grid) @inline Δzᵃᵃᶠ(i, j, k, ibg::IBG) = Δzᵃᵃᶠ(i, j, k, ibg.underlying_grid) -coordinates(grid::IBG) = coordinates(grid.underlying_grid) - # 1D Horizontal spacings @inline Δxᶠᵃᵃ(i, j, k, ibg::RGIBG) = Δxᶠᵃᵃ(i, j, k, ibg.underlying_grid) diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index 8789af01af..d399d2d122 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -32,7 +32,7 @@ function info_message(grid) msg1 = "Testing z-star coordinates on $(architecture(grid)) on a " msg2 = string(getnamewrapper(grid)) msg3 = grid isa ImmersedBoundaryGrid ? " on a " * string(getnamewrapper(grid.underlying_grid)) : "" - msg4 = grid.z.Δᶠ isa Number ? " with uniform spacing" : " with stretched spacing" + msg4 = grid.z.Δᵃᵃᶠ isa Number ? " with uniform spacing" : " with stretched spacing" msg5 = grid isa ImmersedBoundaryGrid ? " and $(string(getnamewrapper(grid.immersed_boundary))) immersed boundary" : "" return msg1 * msg2 * msg3 * msg4 * msg5 From bfd24cce7390e229f070ffe153b1e6bd1c5f9732 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 11 Dec 2024 11:41:13 +0100 Subject: [PATCH 540/567] add a test for active_cells_map --- test/runtests.jl | 1 + test/test_active_cells_map.jl | 95 +++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 test/test_active_cells_map.jl diff --git a/test/runtests.jl b/test/runtests.jl index 0146132dbe..fbfbe6dac7 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -120,6 +120,7 @@ CUDA.allowscalar() do @testset "Model and time stepping tests (part 1)" begin include("test_nonhydrostatic_models.jl") include("test_time_stepping.jl") + include("test_active_cells_map.jl") end end diff --git a/test/test_active_cells_map.jl b/test/test_active_cells_map.jl new file mode 100644 index 0000000000..da9fccdfd6 --- /dev/null +++ b/test/test_active_cells_map.jl @@ -0,0 +1,95 @@ +include("dependencies_for_runtests.jl") + +using Oceananigans.Operators: hack_cosd + +function Δ_min(grid) + Δx_min = minimum_xspacing(grid, Center(), Center(), Center()) + Δy_min = minimum_yspacing(grid, Center(), Center(), Center()) + return min(Δx_min, Δy_min) +end + +@inline Gaussian(x, y, L) = exp(-(x^2 + y^2) / L^2) + +function solid_body_rotation_test(grid) + + free_surface = SplitExplicitFreeSurface(grid; substeps = 5, gravitational_acceleration = 1) + coriolis = HydrostaticSphericalCoriolis(rotation_rate = 1) + + model = HydrostaticFreeSurfaceModel(; grid, + momentum_advection = VectorInvariant(), + free_surface = free_surface, + coriolis = coriolis, + tracers = :c, + tracer_advection = WENO(), + buoyancy = nothing, + closure = nothing) + + g = model.free_surface.gravitational_acceleration + R = grid.radius + Ω = model.coriolis.rotation_rate + + uᵢ(λ, φ, z) = 0.1 * cosd(φ) * sind(λ) + ηᵢ(λ, φ, z) = (R * Ω * 0.1 + 0.1^2 / 2) * sind(φ)^2 / g * sind(λ) + + # Gaussian leads to values with O(1e-60), + # too small for repetible testing. We cap it at 0.1 + cᵢ(λ, φ, z) = max(Gaussian(λ, φ - 5, 10), 0.1) + vᵢ(λ, φ, z) = 0.1 + + set!(model, u=uᵢ, η=ηᵢ, c=cᵢ) + + Δt = 0.1 * Δ_min(grid) / sqrt(g * grid.Lz) + + simulation = Simulation(model; Δt, stop_iteration = 10) + run!(simulation) + + return merge(model.velocities, model.tracers, (; η = model.free_surface.η)) +end + +Nx = 32 +Ny = 32 + +for arch in archs + @testset "Active cells map solid body rotation" begin + underlying_grid = LatitudeLongitudeGrid(arch, size = (Nx, Ny, 10), + halo = (4, 4, 4), + latitude = (-80, 80), + longitude = (-160, 160), + z = (-10, 0), + radius = 1, + topology=(Bounded, Bounded, Bounded)) + + # Make sure the bottom is the same + bottom_height = zeros(Nx, Ny) + for i in 1:Nx, j in 1:Ny + bottom_height[i, j] = - rand() * 5 - 5 + end + + immersed_grid = ImmersedBoundaryGrid(underlying_grid, GridFittedBottom(bottom_height)) + immersed_active_grid = ImmersedBoundaryGrid(underlying_grid, GridFittedBottom(bottom_height); active_cells_map = true) + + ua, va, wa, ca, ηa = solid_body_rotation_test(immersed_active_grid) + u, v, w, c, η = solid_body_rotation_test(immersed_grid) + + ua = interior(on_architecture(CPU(), ua)) + va = interior(on_architecture(CPU(), va)) + wa = interior(on_architecture(CPU(), wa)) + ca = interior(on_architecture(CPU(), ca)) + ηa = interior(on_architecture(CPU(), ηa)) + + u = interior(on_architecture(CPU(), u)) + v = interior(on_architecture(CPU(), v)) + w = interior(on_architecture(CPU(), w)) + c = interior(on_architecture(CPU(), c)) + η = interior(on_architecture(CPU(), η)) + + atol = eps(eltype(immersed_grid)) + rtol = sqrt(eps(eltype(immersed_grid))) + + @test all(isapprox(u, ua; atol, rtol)) + @test all(isapprox(v, va; atol, rtol)) + @test all(isapprox(w, wa; atol, rtol)) + @test all(isapprox(c, ca; atol, rtol)) + @test all(isapprox(η, ηa; atol, rtol)) + end +end From e576565d6880dc67d5b7102201ed8f9c84ea6789 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 11 Dec 2024 11:59:08 +0100 Subject: [PATCH 541/567] make the test a bit smaller --- test/test_active_cells_map.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_active_cells_map.jl b/test/test_active_cells_map.jl index da9fccdfd6..bc44d3debf 100644 --- a/test/test_active_cells_map.jl +++ b/test/test_active_cells_map.jl @@ -46,8 +46,8 @@ function solid_body_rotation_test(grid) return merge(model.velocities, model.tracers, (; η = model.free_surface.η)) end -Nx = 32 -Ny = 32 +Nx = 16 +Ny = 16 for arch in archs @testset "Active cells map solid body rotation" begin From 1e5573c06b0b0017dbece8120a5f61820de142bd Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 11 Dec 2024 12:35:34 +0100 Subject: [PATCH 542/567] improve test --- test/test_active_cells_map.jl | 84 +++++++++++++++++++++++------------ 1 file changed, 56 insertions(+), 28 deletions(-) diff --git a/test/test_active_cells_map.jl b/test/test_active_cells_map.jl index bc44d3debf..1fca7f8fad 100644 --- a/test/test_active_cells_map.jl +++ b/test/test_active_cells_map.jl @@ -1,6 +1,9 @@ include("dependencies_for_runtests.jl") using Oceananigans.Operators: hack_cosd +using Oceananigans.ImmersedBoundaries: retrieve_surface_active_cells_map, + retrieve_interior_active_cells_map, + immersed_cell function Δ_min(grid) Δx_min = minimum_xspacing(grid, Center(), Center(), Center()) @@ -12,7 +15,7 @@ end function solid_body_rotation_test(grid) - free_surface = SplitExplicitFreeSurface(grid; substeps = 5, gravitational_acceleration = 1) + free_surface = SplitExplicitFreeSurface(grid; substeps = 10, gravitational_acceleration = 1) coriolis = HydrostaticSphericalCoriolis(rotation_rate = 1) model = HydrostaticFreeSurfaceModel(; grid, @@ -48,10 +51,11 @@ end Nx = 16 Ny = 16 +Nz = 10 -for arch in archs - @testset "Active cells map solid body rotation" begin - underlying_grid = LatitudeLongitudeGrid(arch, size = (Nx, Ny, 10), +@testset "Active cells map" begin + for arch in archs + underlying_grid = LatitudeLongitudeGrid(arch, size = (Nx, Ny, Nz), halo = (4, 4, 4), latitude = (-80, 80), longitude = (-160, 160), @@ -68,28 +72,52 @@ for arch in archs immersed_grid = ImmersedBoundaryGrid(underlying_grid, GridFittedBottom(bottom_height)) immersed_active_grid = ImmersedBoundaryGrid(underlying_grid, GridFittedBottom(bottom_height); active_cells_map = true) - ua, va, wa, ca, ηa = solid_body_rotation_test(immersed_active_grid) - u, v, w, c, η = solid_body_rotation_test(immersed_grid) - - ua = interior(on_architecture(CPU(), ua)) - va = interior(on_architecture(CPU(), va)) - wa = interior(on_architecture(CPU(), wa)) - ca = interior(on_architecture(CPU(), ca)) - ηa = interior(on_architecture(CPU(), ηa)) - - u = interior(on_architecture(CPU(), u)) - v = interior(on_architecture(CPU(), v)) - w = interior(on_architecture(CPU(), w)) - c = interior(on_architecture(CPU(), c)) - η = interior(on_architecture(CPU(), η)) - - atol = eps(eltype(immersed_grid)) - rtol = sqrt(eps(eltype(immersed_grid))) - - @test all(isapprox(u, ua; atol, rtol)) - @test all(isapprox(v, va; atol, rtol)) - @test all(isapprox(w, wa; atol, rtol)) - @test all(isapprox(c, ca; atol, rtol)) - @test all(isapprox(η, ηa; atol, rtol)) + @testset "Active cells map construction" begin + surface_active_cells_map = retrieve_surface_active_cells_map(immersed_active_grid) + interior_active_cells_map = retrieve_interior_active_cells_map(immersed_active_grid, Val(:interior)) + + surface_active_cells_map = on_architecture(CPU(), surface_active_cells_map) + interior_active_cells_map = on_architecture(CPU(), interior_active_cells_map) + grid = on_architecture(CPU(), immersed_grid) + + for i in 1:Nx, j in 1:Ny, k in 1:Nz + immersed = immersed_cell(i, j, k, grid) + active = (i, j, k) ∈ interior_active_cells_map + @test immersed | active + end + + for i in 1:Nx, j in 1:Ny + immersed = all(immersed_cell(i, j, k, grid) for k in 1:Nz) + active = (i, j) ∈ surface_active_cells_map + @test immersed | active + end + end + + @testset "Active cells map solid body rotation" begin + + ua, va, wa, ca, ηa = solid_body_rotation_test(immersed_active_grid) + u, v, w, c, η = solid_body_rotation_test(immersed_grid) + + ua = interior(on_architecture(CPU(), ua)) + va = interior(on_architecture(CPU(), va)) + wa = interior(on_architecture(CPU(), wa)) + ca = interior(on_architecture(CPU(), ca)) + ηa = interior(on_architecture(CPU(), ηa)) + + u = interior(on_architecture(CPU(), u)) + v = interior(on_architecture(CPU(), v)) + w = interior(on_architecture(CPU(), w)) + c = interior(on_architecture(CPU(), c)) + η = interior(on_architecture(CPU(), η)) + + atol = eps(eltype(immersed_grid)) + rtol = sqrt(eps(eltype(immersed_grid))) + + @test all(isapprox(u, ua; atol, rtol)) + @test all(isapprox(v, va; atol, rtol)) + @test all(isapprox(w, wa; atol, rtol)) + @test all(isapprox(c, ca; atol, rtol)) + @test all(isapprox(η, ηa; atol, rtol)) + end end -end +end \ No newline at end of file From 5552bcff6f765114dfbc46e7c188d204fe4d1fc4 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 11 Dec 2024 12:37:25 +0100 Subject: [PATCH 543/567] xor instead of or --- test/test_active_cells_map.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_active_cells_map.jl b/test/test_active_cells_map.jl index 1fca7f8fad..2084755093 100644 --- a/test/test_active_cells_map.jl +++ b/test/test_active_cells_map.jl @@ -83,13 +83,13 @@ Nz = 10 for i in 1:Nx, j in 1:Ny, k in 1:Nz immersed = immersed_cell(i, j, k, grid) active = (i, j, k) ∈ interior_active_cells_map - @test immersed | active + @test immersed ⊻ active end for i in 1:Nx, j in 1:Ny immersed = all(immersed_cell(i, j, k, grid) for k in 1:Nz) active = (i, j) ∈ surface_active_cells_map - @test immersed | active + @test immersed ⊻ active end end From 70ef066b2a2e819329bc763e4c2571e9f603ef20 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 11 Dec 2024 12:52:09 +0100 Subject: [PATCH 544/567] correct all tests --- .../barotropic_split_explicit_corrector.jl | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_split_explicit_corrector.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_split_explicit_corrector.jl index 20a6dca582..23f25d6379 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_split_explicit_corrector.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_split_explicit_corrector.jl @@ -1,35 +1,35 @@ # Kernels to compute the vertical integral of the velocities -@kernel function _barotropic_mode_kernel!(u, v, grid, ::Nothing, U, V, η) +@kernel function _barotropic_mode_kernel!(U̅, V̅, grid, ::Nothing, u, v, η) i, j = @index(Global, NTuple) - barotropic_mode_kernel!(u, v, i, j, grid, U, V, η) + barotropic_mode_kernel!(U̅, V̅, i, j, grid, u, v, η) end -@kernel function _barotropic_mode_kernel!(u, v, grid, active_cells_map, U, V, η) +@kernel function _barotropic_mode_kernel!(U̅, V̅, grid, active_cells_map, u, v, η) idx = @index(Global, Linear) i, j = active_linear_index_to_tuple(idx, active_cells_map) - barotropic_mode_kernel!(U, V, i, j, grid, u, v, η) + barotropic_mode_kernel!(U̅, V̅, i, j, grid, u, v, η) end -@inline function barotropic_mode_kernel!(U, V, i, j, grid, u, v, η) +@inline function barotropic_mode_kernel!(U̅, V̅, i, j, grid, u, v, η) k_top = size(grid, 3) + 1 sᶠᶜ = dynamic_column_depthᶠᶜᵃ(i, j, k_top, grid, η) / static_column_depthᶠᶜᵃ(i, j, grid) sᶜᶠ = dynamic_column_depthᶜᶠᵃ(i, j, k_top, grid, η) / static_column_depthᶜᶠᵃ(i, j, grid) - @inbounds U[i, j, 1] = Δrᶠᶜᶜ(i, j, 1, grid) * u[i, j, 1] * sᶠᶜ - @inbounds V[i, j, 1] = Δrᶜᶠᶜ(i, j, 1, grid) * v[i, j, 1] * sᶜᶠ + @inbounds U̅[i, j, 1] = Δrᶠᶜᶜ(i, j, 1, grid) * u[i, j, 1] * sᶠᶜ + @inbounds V̅[i, j, 1] = Δrᶜᶠᶜ(i, j, 1, grid) * v[i, j, 1] * sᶜᶠ for k in 2:grid.Nz - @inbounds U[i, j, 1] += Δrᶠᶜᶜ(i, j, k, grid) * u[i, j, k] * sᶠᶜ - @inbounds V[i, j, 1] += Δrᶜᶠᶜ(i, j, k, grid) * v[i, j, k] * sᶜᶠ + @inbounds U̅[i, j, 1] += Δrᶠᶜᶜ(i, j, k, grid) * u[i, j, k] * sᶠᶜ + @inbounds V̅[i, j, 1] += Δrᶜᶠᶜ(i, j, k, grid) * v[i, j, k] * sᶜᶠ end return nothing end -@inline function compute_barotropic_mode!(U, V, grid, u, v, η) +@inline function compute_barotropic_mode!(U̅, V̅, grid, u, v, η) active_cells_map = retrieve_surface_active_cells_map(grid) - launch!(architecture(grid), grid, :xy, _barotropic_mode_kernel!, U, V, grid, active_cells_map, u, v, η; active_cells_map) + launch!(architecture(grid), grid, :xy, _barotropic_mode_kernel!, U̅, V̅, grid, active_cells_map, u, v, η; active_cells_map) return nothing end From 071e14a256f72bdc212706592d0ef03b66fc671f Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 11 Dec 2024 13:56:41 +0100 Subject: [PATCH 545/567] on architecture --- test/test_active_cells_map.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_active_cells_map.jl b/test/test_active_cells_map.jl index 2084755093..581a50ff74 100644 --- a/test/test_active_cells_map.jl +++ b/test/test_active_cells_map.jl @@ -69,6 +69,7 @@ Nz = 10 bottom_height[i, j] = - rand() * 5 - 5 end + bottom_height = on_architecture(arch, bottom_height) immersed_grid = ImmersedBoundaryGrid(underlying_grid, GridFittedBottom(bottom_height)) immersed_active_grid = ImmersedBoundaryGrid(underlying_grid, GridFittedBottom(bottom_height); active_cells_map = true) From 998ce3bff9da6d6fd89787d0d208034196799785 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 11 Dec 2024 14:23:22 +0100 Subject: [PATCH 546/567] removed the drift in immersed boundary grids --- src/ImmersedBoundaries/zstar_immersed_grid.jl | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/ImmersedBoundaries/zstar_immersed_grid.jl b/src/ImmersedBoundaries/zstar_immersed_grid.jl index 4eb1fc6959..39fc271318 100644 --- a/src/ImmersedBoundaries/zstar_immersed_grid.jl +++ b/src/ImmersedBoundaries/zstar_immersed_grid.jl @@ -1,5 +1,6 @@ using Oceananigans.Grids: AbstractZStarGrid using Oceananigans.Operators +using Oceananigans.Operators: ZSRG, ZSLLG, ZSOSG, superscript_location import Oceananigans.Grids: dynamic_column_depthᶜᶜᵃ, dynamic_column_depthᶜᶠᵃ, @@ -32,3 +33,26 @@ const ZStarGridOfSomeKind = Union{ZStarImmersedGrid, AbstractZStarGrid} @inline σ⁻(i, j, k, ibg::IBG, ℓx, ℓy, ℓz) = σ⁻(i, j, k, ibg.underlying_grid, ℓx, ℓy, ℓz) @inline ∂t_σ(i, j, k, ibg::IBG) = ∂t_σ(i, j, k, ibg.underlying_grid) + +# Extend the 3D vertical spacing operators on an Immersed ZStar grid +const IZSRG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:ZSRG} +const IZSLLG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:ZSLLG} +const IZSOSG = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:ZSOSG} + +for LX in (:ᶠ, :ᶜ), LY in (:ᶠ, :ᶜ), LZ in (:ᶠ, :ᶜ) + zspacing = Symbol(:Δz, LX, LY, LZ) + rspacing = Symbol(:Δr, LX, LY, LZ) + + ℓx = superscript_location(LX) + ℓy = superscript_location(LY) + ℓz = superscript_location(LZ) + + @eval begin + using Oceananigans.Operators: $rspacing + import Oceananigans.Operators: $zspacing + + @inline $zspacing(i, j, k, grid::IZSRG) = $rspacing(i, j, k, grid) * σⁿ(i, j, k, grid, $ℓx(), $ℓy(), $ℓz()) + @inline $zspacing(i, j, k, grid::IZSLLG) = $rspacing(i, j, k, grid) * σⁿ(i, j, k, grid, $ℓx(), $ℓy(), $ℓz()) + @inline $zspacing(i, j, k, grid::IZSOSG) = $rspacing(i, j, k, grid) * σⁿ(i, j, k, grid, $ℓx(), $ℓy(), $ℓz()) + end +end \ No newline at end of file From dc11dd44461c1eda55cec678f3ea3b1e7ce21a41 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 11 Dec 2024 14:24:07 +0100 Subject: [PATCH 547/567] better lock_release validation --- validation/z_star_coordinate/lock_release.jl | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/validation/z_star_coordinate/lock_release.jl b/validation/z_star_coordinate/lock_release.jl index ec4da9a51b..d7867746f3 100644 --- a/validation/z_star_coordinate/lock_release.jl +++ b/validation/z_star_coordinate/lock_release.jl @@ -18,16 +18,14 @@ grid = ImmersedBoundaryGrid(grid, GridFittedBottom(x -> - (64kilometers - x) / 6 model = HydrostaticFreeSurfaceModel(; grid, momentum_advection = WENO(order = 5), - tracer_advection = FluxFormAdvection(WENO(; order = 5), nothing, WENO(; order = 5)), + tracer_advection = WENO(order = 5), buoyancy = BuoyancyTracer(), closure = nothing, tracers = :b, - free_surface = SplitExplicitFreeSurface(; substeps = 10)) + free_surface = SplitExplicitFreeSurface(grid; substeps = 10)) g = model.free_surface.gravitational_acceleration -model.timestepper.χ = 0.1 - bᵢ(x, z) = x < 32kilometers ? 0.06 : 0.01 set!(model, b = bᵢ) @@ -38,7 +36,7 @@ set!(model, b = bᵢ) simulation = Simulation(model; Δt, stop_iteration = 10000, stop_time = 17hours) -Δz = GridMetricOperation((Center, Center, Center), Oceananigans.AbstractOperations.Δz, model.grid) +Δz = zspacings(grid, Center(), Center(), Center()) ∫b_init = sum(model.tracers.b * Δz) / sum(Δz) field_outputs = merge(model.velocities, model.tracers, (; Δz)) @@ -63,7 +61,7 @@ function progress(sim) return nothing end -simulation.callbacks[:progress] = Callback(progress, IterationInterval(1)) +simulation.callbacks[:progress] = Callback(progress, IterationInterval(100)) run!(simulation) @@ -71,12 +69,12 @@ using Oceananigans.Fields: OneField # # Check tracer conservation b = FieldTimeSeries("zstar_model.jld2", "b") -Δz = FieldTimeSeries("zstar_model.jld2", "Δz") +dz = FieldTimeSeries("zstar_model.jld2", "Δz") -init = sum(Δz[1] * b[1]) / sum(Δz[1]) +init = sum(dz[1] * b[1]) / sum(dz[1]) drift = [] for t in 1:length(b.times) - push!(drift, sum(Δz[t] * b[t]) / sum(Δz[t]) - init) + push!(drift, sum(dz[t] * b[t]) / sum(dz[t]) - init) end From e8c6a34fa4933b613a8bbd0e1e94b805b870fe57 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 11 Dec 2024 14:34:05 +0100 Subject: [PATCH 548/567] remove extra validation examples --- .../baroclinic_double_gyre.jl | 160 -------------- validation/z_star_coordinate/internal_tide.jl | 203 ------------------ 2 files changed, 363 deletions(-) delete mode 100644 validation/z_star_coordinate/baroclinic_double_gyre.jl delete mode 100644 validation/z_star_coordinate/internal_tide.jl diff --git a/validation/z_star_coordinate/baroclinic_double_gyre.jl b/validation/z_star_coordinate/baroclinic_double_gyre.jl deleted file mode 100644 index bf85e64ed6..0000000000 --- a/validation/z_star_coordinate/baroclinic_double_gyre.jl +++ /dev/null @@ -1,160 +0,0 @@ -using Oceananigans -using Oceananigans.Units -using Oceananigans.Operators -using Oceananigans.Grids: φnode -using Oceananigans.AbstractOperations: GridMetricOperation -using Printf - -arch = CPU() -z_faces = ZStarVerticalCoordinate((-1800, 0)) -grid = LatitudeLongitudeGrid(arch; size = (60, 60, 2), - latitude = (15, 75), - longitude = (0, 60), - halo = (5, 5, 5), - z = z_faces) - -##### -##### Parameters -##### - -θ⁺ = 30 # ᵒC maximum temperature -θ⁻ = 0 # ᵒC maximum temperature -α = 2e-4 # ᵒC⁻¹ thermal expansion coefficient -ρ₀ = 1000 # kg m⁻³ reference density -g = 9.80665 # m s⁻² gravitational acceleration -λ = 30days # time scale for restoring - -##### -##### Numerics -##### - -Δt = 20minutes - -Δx = minimum_xspacing(grid) -Δy = minimum_yspacing(grid) - -Δs = sqrt(1 / (1 / Δx^2 + 1 / Δy^2)) -sp = sqrt(g * grid.Lz) -CFL = 0.75 -Δτ = Δs / sp * CFL - -substeps = ceil(Int, 3 * Δt / Δτ) - -coriolis = HydrostaticSphericalCoriolis() -momentum_advection = WENOVectorInvariant(vorticity_order = 5) -tracer_advection = WENO(order = 5) -free_surface = SplitExplicitFreeSurface(grid; substeps) - -numerics = (; coriolis, free_surface, momentum_advection, tracer_advection) - -##### -##### Closure -##### - -closure = ConvectiveAdjustmentVerticalDiffusivity(convective_κz = 1.0, - background_κz = 1e-5, - convective_νz = 1e-2, - background_νz = 1e-2) - -##### -##### Boundary Conditions -##### - -@inline function wind_stress(i, j, grid, clock, fields, p) - τ₀ = p.τ₀ - y = (φnode(j, grid, Center()) - p.φ₀) / grid.Ly - - return τ₀ * cos(2π * y) -end - -@inline function buoyancy_restoring(i, j, grid, clock, fields, p) - b = @inbounds fields.b[i, j, 1] - y = (φnode(j, grid, Center()) - p.φ₀) / grid.Ly - b★ = p.Δb * y - - return p.𝓋 * (b - b★) -end - -Δz₀ = 10 # Surface layer thickness [m] - -Δb = α * g * (θ⁺ - θ⁻) # Buoyancy difference - -parameters = (; τ₀ = 0.1 / ρ₀, # Wind stress - φ₀ = 15, # Latitude of southern edge - Δb, # Buoyancy difference - 𝓋 = Δz₀ / λ) # Pumping velocity for restoring - -u_boundary = FluxBoundaryCondition(wind_stress; discrete_form = true, parameters) -b_boundary = FluxBoundaryCondition(buoyancy_restoring; discrete_form = true, parameters) - -no_slip = ValueBoundaryCondition(0.0) - -u_bcs = FieldBoundaryConditions(north = no_slip, south = no_slip, top = u_boundary) -v_bcs = FieldBoundaryConditions(west = no_slip, east = no_slip) -b_bcs = FieldBoundaryConditions(top = b_boundary) - -##### -##### Model -##### - -model = HydrostaticFreeSurfaceModel(; grid, - boundary_conditions = (u = u_bcs, v = v_bcs, b = b_bcs), - buoyancy = BuoyancyTracer(), - tracers = :b, - numerics..., - closure) - - -N² = Δb / grid.Lz -bᵢ(x, y, z) = N² * (grid.Lz + z) - -set!(model, b = bᵢ) - -##### -##### Simulation -##### - -simulation = Simulation(model; Δt, stop_time = 2000days) - -##### -##### Output -##### - -Δzᶜᶜ = GridMetricOperation((Center, Center, Center), Oceananigans.AbstractOperations.Δz, model.grid) -Δzᶠᶜ = GridMetricOperation((Face, Center, Center), Oceananigans.AbstractOperations.Δz, model.grid) -Δzᶜᶠ = GridMetricOperation((Center, Face, Center), Oceananigans.AbstractOperations.Δz, model.grid) - -field_outputs = merge(model.velocities, - model.tracers, - model.pressure, - (; Δzᶜᶜ, Δzᶠᶜ, Δzᶜᶠ)) - -function progress(sim) - w = interior(sim.model.velocities.w, :, :, sim.model.grid.Nz+1) - u = sim.model.velocities.u - b = sim.model.tracers.b - - msg0 = @sprintf("Time: %s iteration %d ", prettytime(sim.model.clock.time), sim.model.clock.iteration) - msg1 = @sprintf("extrema w: %.2e %.2e ", maximum(w), minimum(w)) - msg2 = @sprintf("extrema u: %.2e %.2e ", maximum(u), minimum(u)) - msg3 = @sprintf("extrema b: %.2e %.2e ", maximum(b), minimum(b)) - msg4 = @sprintf("extrema Δz: %.2e %.2e ", maximum(Δzᶜᶜ), minimum(Δzᶜᶜ)) - @info msg0 * msg1 * msg2 * msg3 * msg4 - - return nothing -end - -simulation.callbacks[:progress] = Callback(progress, IterationInterval(100)) - -simulation.output_writers[:snapshots] = JLD2OutputWriter(model, field_outputs, - overwrite_existing = true, - schedule = TimeInterval(60days), - filename = "baroclinic_double_gyre_new") - -simulation.output_writers[:free_surface] = JLD2OutputWriter(model, (; η = model.free_surface.η), - overwrite_existing = true, - indices = (:, :, grid.Nz+1), - schedule = TimeInterval(60days), - filename = "baroclinic_double_gyre_free_surface_new") - -run!(simulation) \ No newline at end of file diff --git a/validation/z_star_coordinate/internal_tide.jl b/validation/z_star_coordinate/internal_tide.jl deleted file mode 100644 index dac756bdae..0000000000 --- a/validation/z_star_coordinate/internal_tide.jl +++ /dev/null @@ -1,203 +0,0 @@ -using CairoMakie -using Oceananigans -using Oceananigans.Units -using Oceananigans.ImmersedBoundaries: PartialCellBottom -using Oceananigans.AbstractOperations: GridMetricOperation - -Nx, Nz = 250, 125 - -H = 2kilometers -z_faces = ZStarVerticalCoordinate(-H, 0) - -underlying_grid = RectilinearGrid(size = (Nx, Nz), - x = (-1000kilometers, 1000kilometers), - z = z_faces, - halo = (4, 4), - topology = (Periodic, Flat, Bounded)) - -h₀ = 250meters -width = 20kilometers -hill(x) = h₀ * exp(-x^2 / 2width^2) -bottom(x) = - H + hill(x) - -grid = ImmersedBoundaryGrid(underlying_grid, PartialCellBottom(bottom)) - -coriolis = FPlane(latitude = -45) - -# Now we have everything we require to construct the tidal forcing given a value of the -# excursion parameter. - -T₂ = 12.421hours -ω₂ = 2π / T₂ # radians/sec - -ϵ = 0.1 # excursion parameter - -U_tidal = ϵ * ω₂ * width - -tidal_forcing_amplitude = U_tidal * (ω₂^2 - coriolis.f^2) / ω₂ - -@inline tidal_forcing(x, z, t, p) = p.tidal_forcing_amplitude * sin(p.ω₂ * t) - -u_forcing = Forcing(tidal_forcing, parameters=(; tidal_forcing_amplitude, ω₂)) - -# ## Model - -# We built a `HydrostaticFreeSurfaceModel`: - -model = HydrostaticFreeSurfaceModel(; grid, coriolis, - buoyancy = BuoyancyTracer(), - tracers = :b, - free_surface = SplitExplicitFreeSurface(grid; substeps = 20), - momentum_advection = WENO(), - tracer_advection = WENO(), - forcing = (; u = u_forcing)) - -# We initialize the model with the tidal flow and a linear stratification. - -uᵢ(x, z) = 0 - -Nᵢ² = 1e-4 # [s⁻²] initial buoyancy frequency / stratification -bᵢ(x, z) = Nᵢ² * z - -set!(model, u=uᵢ, b=bᵢ) - -# Now let's build a `Simulation`. - -Δt = 5minutes -stop_time = 4days - -simulation = Simulation(model; Δt, stop_time, stop_iteration = 10) - -# We add a callback to print a message about how the simulation is going, - -using Printf - -wall_clock = Ref(time_ns()) - -dz = GridMetricOperation((Center, Center, Center), Oceananigans.AbstractOperations.Δz, model.grid) -∫b_init = sum(model.tracers.b * dz) / sum(dz) - -function progress(sim) - elapsed = 1e-9 * (time_ns() - wall_clock[]) - - ∫b = sum(model.tracers.b * dz) / sum(dz) - - msg = @sprintf("iteration: %d, time: %s, wall time: %s, max|w|: %6.3e, max|u|: %6.3e, drift: %6.3e\n", - iteration(sim), prettytime(sim), prettytime(elapsed), - maximum(abs, interior(w, :, :, grid.Nz+1)), maximum(abs, simulation.model.velocities.u), - ∫b - ∫b_init) - - wall_clock[] = time_ns() - - @info msg - - return nothing -end - -simulation.callbacks[:progress] = Callback(progress, IterationInterval(100)) - -b = model.tracers.b -u, v, w = model.velocities - -U = Field(Average(u)) - -u′ = u - U - -N² = ∂z(b) - -filename = "internal_tide" -save_fields_interval = 30minutes - -simulation.output_writers[:fields] = JLD2OutputWriter(model, (; u, u′, w, b, N²); - filename, - schedule = TimeInterval(save_fields_interval), - overwrite_existing = true) - -# We are ready -- let's run! -run!(simulation) - -# # ## Load output - -# # # First, we load the saved velocities and stratification output as `FieldTimeSeries`es. - -# saved_output_filename = filename * ".jld2" - -# u′_t = FieldTimeSeries(saved_output_filename, "u′") -# w_t = FieldTimeSeries(saved_output_filename, "w") -# N²_t = FieldTimeSeries(saved_output_filename, "N²") - -# umax = maximum(abs, u′_t[end]) -# wmax = maximum(abs, w_t[end]) - -# times = u′_t.times -# nothing #hide - -# # We retrieve each field's coordinates and convert from meters to kilometers. - -# xu, _, zu = nodes(u′_t[1]) -# xw, _, zw = nodes(w_t[1]) -# xN², _, zN² = nodes(N²_t[1]) - -# xu = xu ./ 1e3 -# xw = xw ./ 1e3 -# xN² = xN² ./ 1e3 -# zu = zu ./ 1e3 -# zw = zw ./ 1e3 -# zN² = zN² ./ 1e3 -# nothing #hide - -# # ## Visualize - -# # Now we can visualize our resutls! We use `CairoMakie` here. On a system with OpenGL -# # `using GLMakie` is more convenient as figures will be displayed on the screen. -# # -# # We use Makie's `Observable` to animate the data. To dive into how `Observable`s work we -# # refer to [Makie.jl's Documentation](https://makie.juliaplots.org/stable/documentation/nodes/index.html). - -# using CairoMakie - -# n = Observable(1) - -# title = @lift @sprintf("t = %1.2f days = %1.2f T₂", -# round(times[$n] / day, digits=2) , round(times[$n] / T₂, digits=2)) - -# u′n = @lift u′_t[$n] -# wn = @lift w_t[$n] -# N²n = @lift N²_t[$n] - -# axis_kwargs = (xlabel = "x [km]", -# ylabel = "z [km]", -# limits = ((-grid.Lx/2e3, grid.Lx/2e3), (-grid.Lz/1e3, 0)), # note conversion to kilometers -# titlesize = 20) - -# fig = Figure(size = (700, 900)) - -# fig[1, :] = Label(fig, title, fontsize=24, tellwidth=false) - -# ax_u = Axis(fig[2, 1]; title = "u'-velocity", axis_kwargs...) -# hm_u = heatmap!(ax_u, xu, zu, u′n; nan_color=:gray, colorrange=(-umax, umax), colormap=:balance) -# Colorbar(fig[2, 2], hm_u, label = "m s⁻¹") - -# ax_w = Axis(fig[3, 1]; title = "w-velocity", axis_kwargs...) -# hm_w = heatmap!(ax_w, xw, zw, wn; nan_color=:gray, colorrange=(-wmax, wmax), colormap=:balance) -# Colorbar(fig[3, 2], hm_w, label = "m s⁻¹") - -# ax_N² = Axis(fig[4, 1]; title = "stratification N²", axis_kwargs...) -# hm_N² = heatmap!(ax_N², xN², zN², N²n; nan_color=:gray, colorrange=(0.9Nᵢ², 1.1Nᵢ²), colormap=:magma) -# Colorbar(fig[4, 2], hm_N², label = "s⁻²") - -# fig - -# # Finally, we can record a movie. - -# @info "Making an animation from saved data..." - -# frames = 1:length(times) - -# record(fig, filename * ".mp4", frames, framerate=16) do i -# @info string("Plotting frame ", i, " of ", frames[end]) -# n[] = i -# end -# nothing #hide - -# # ![](internal_tide.mp4) From 0e6f26e5b82bc68ee42d3cc7430fa8ccfc7a1793 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 11 Dec 2024 15:10:51 +0100 Subject: [PATCH 549/567] update tests --- test/test_zstar_coordinate.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index d399d2d122..d523626a58 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -23,7 +23,7 @@ function test_zstar_coordinate(model, Ni, Δt) @test interior(∫b, 1, 1, 1) ≈ interior(∫bᵢ, 1, 1, 1) @test interior(∫c, 1, 1, 1) ≈ interior(∫cᵢ, 1, 1, 1) - @test maximum(interior(w, :, :, Nz+1)) < model.grid.Nz * eps(eltype(w)) + @test maximum(abs, interior(w, :, :, Nz+1)) < eps(eltype(w)) return nothing end From 01856eaf7802f34b2c94e15534cb14111fa9281b Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 11 Dec 2024 15:11:49 +0100 Subject: [PATCH 550/567] increase the time step --- test/test_zstar_coordinate.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index d523626a58..11c27b8f20 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -116,7 +116,7 @@ end for grid in grids info_msg = info_message(grid) - split_free_surface = SplitExplicitFreeSurface(grid; substeps = 20) + split_free_surface = SplitExplicitFreeSurface(grid; cfl = 0.75) # TODO: Implicit and Explicit free surfaces are not fully supported yet implicit_free_surface = ImplicitFreeSurface() @@ -136,7 +136,7 @@ end set!(model, c = (x, y, z) -> rand(), b = bᵢ) - test_zstar_coordinate(model, 100, 10) + test_zstar_coordinate(model, 100, 5minutes) end end end From 7eef5cd14c6f70088c48748591a30e0a5f1071e0 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 12 Dec 2024 09:24:50 +0100 Subject: [PATCH 551/567] works for all free surfaces --- .../z_star_vertical_spacing.jl | 8 +------- test/test_zstar_coordinate.jl | 9 ++++----- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index 67dba6e2cb..a0b2575cf0 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -1,12 +1,6 @@ using Oceananigans.Grids using Oceananigans.ImmersedBoundaries: ZStarGridOfSomeKind -# TODO: ZStar is not currently working for an ImplicitFreeSurface on -# - `LatitudeLongitudeGrid`s -# - `ImmersedBoundaryGrid`s -# and for an ExplicitFreeSurface on -# - `ImmersedBoundaryGrid`s - ##### ##### ZStar-specific vertical spacings update ##### @@ -112,7 +106,7 @@ end @inline function barotropic_V(i, j, k, grid, ::Nothing, v) V = 0 for k in 1:size(grid, 3) - V += v[i, j, k] * Δzᶠᶜᶜ(i, j, k, grid) + V += v[i, j, k] * Δzᶜᶠᶜ(i, j, k, grid) end return V end diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index 11c27b8f20..d2855479ac 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -79,9 +79,9 @@ end @testset "ZStar coordinate simulation testset" begin z_uniform = ZStarVerticalCoordinate(-20, 0) z_stretched = ZStarVerticalCoordinate(collect(-20:0)) - topologies = ((Periodic, Periodic, Bounded), + topologies = (#(Periodic, Periodic, Bounded), (Periodic, Bounded, Bounded), - (Bounded, Periodic, Bounded), + #(Bounded, Periodic, Bounded), (Bounded, Bounded, Bounded)) for arch in archs @@ -109,6 +109,7 @@ end # and vertical areas (see https://github.com/CliMA/Oceananigans.jl/issues/3958) # When this is issue is fixed we can add the partial cells to the testing. grids = [llg, rtg, llgv, rtgv, illg, irtg, illgv, irtgv] # , pllg, prtg, pllgv, prtgv] + grids = [llg, llgv, illg, illgv] # , pllg, prtg, pllgv, prtgv] else grids = [rtg, rtgv, irtg, irtgv] #, prtg, prtgv] end @@ -117,12 +118,10 @@ end info_msg = info_message(grid) split_free_surface = SplitExplicitFreeSurface(grid; cfl = 0.75) - - # TODO: Implicit and Explicit free surfaces are not fully supported yet implicit_free_surface = ImplicitFreeSurface() explicit_free_surface = ExplicitFreeSurface() - for free_surface in [split_free_surface] #, implicit_free_surface, explicit_free_surface] + for free_surface in [explicit_free_surface, implicit_free_surface, explicit_free_surface] @testset "$info_msg on $(free_surface)" begin @info " $info_msg of $(free_surface)" # TODO: minimum_xspacing(grid) on a Immersed GPU grid with ZStarVerticalCoordinate From f66a66a04934aff5012de605de25769dfa5eed21 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 12 Dec 2024 09:46:08 +0100 Subject: [PATCH 552/567] test also rectilinear grids --- test/test_zstar_coordinate.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index d2855479ac..4813822f69 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -109,7 +109,6 @@ end # and vertical areas (see https://github.com/CliMA/Oceananigans.jl/issues/3958) # When this is issue is fixed we can add the partial cells to the testing. grids = [llg, rtg, llgv, rtgv, illg, irtg, illgv, irtgv] # , pllg, prtg, pllgv, prtgv] - grids = [llg, llgv, illg, illgv] # , pllg, prtg, pllgv, prtgv] else grids = [rtg, rtgv, irtg, irtgv] #, prtg, prtgv] end From e3d7f936e46036af138c88cd0d9cb90a780dda5b Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 12 Dec 2024 09:48:11 +0100 Subject: [PATCH 553/567] better info msg --- test/test_zstar_coordinate.jl | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index 4813822f69..c6c577aa0a 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -28,14 +28,14 @@ function test_zstar_coordinate(model, Ni, Δt) return nothing end -function info_message(grid) - msg1 = "Testing z-star coordinates on $(architecture(grid)) on a " +function info_message(grid, free_surface) + msg1 = "$(architecture(grid)) " msg2 = string(getnamewrapper(grid)) msg3 = grid isa ImmersedBoundaryGrid ? " on a " * string(getnamewrapper(grid.underlying_grid)) : "" msg4 = grid.z.Δᵃᵃᶠ isa Number ? " with uniform spacing" : " with stretched spacing" msg5 = grid isa ImmersedBoundaryGrid ? " and $(string(getnamewrapper(grid.immersed_boundary))) immersed boundary" : "" - - return msg1 * msg2 * msg3 * msg4 * msg5 + msg6 = " using a " * string(getnamewrapper(free_surface)) + return msg1 * msg2 * msg3 * msg4 * msg5 * msg6 end const C = Center @@ -114,15 +114,14 @@ end end for grid in grids - info_msg = info_message(grid) - split_free_surface = SplitExplicitFreeSurface(grid; cfl = 0.75) implicit_free_surface = ImplicitFreeSurface() explicit_free_surface = ExplicitFreeSurface() for free_surface in [explicit_free_surface, implicit_free_surface, explicit_free_surface] - @testset "$info_msg on $(free_surface)" begin - @info " $info_msg of $(free_surface)" + info_msg = info_message(grid, free_surface) + @testset "$info_msg" begin + @info " Testing a $info_msg" # TODO: minimum_xspacing(grid) on a Immersed GPU grid with ZStarVerticalCoordinate # fails because it uses too much parameter space. Figure out a way to reduce it model = HydrostaticFreeSurfaceModel(; grid, From 5c137ed828a37e57a05695144560112eb5dc4668 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 12 Dec 2024 09:48:31 +0100 Subject: [PATCH 554/567] back to all the grids --- test/test_zstar_coordinate.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index c6c577aa0a..cbd6247879 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -79,9 +79,9 @@ end @testset "ZStar coordinate simulation testset" begin z_uniform = ZStarVerticalCoordinate(-20, 0) z_stretched = ZStarVerticalCoordinate(collect(-20:0)) - topologies = (#(Periodic, Periodic, Bounded), + topologies = ((Periodic, Periodic, Bounded), (Periodic, Bounded, Bounded), - #(Bounded, Periodic, Bounded), + (Bounded, Periodic, Bounded), (Bounded, Bounded, Bounded)) for arch in archs From 2835aea589bc1ab07f391a09a6bf4e591259969f Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 12 Dec 2024 09:51:10 +0100 Subject: [PATCH 555/567] deepcopy before integrating --- test/test_zstar_coordinate.jl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index cbd6247879..ef56bdbd2c 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -6,8 +6,11 @@ using Oceananigans.ImmersedBoundaries: PartialCellBottom function test_zstar_coordinate(model, Ni, Δt) - ∫bᵢ = Field(Integral(model.tracers.b)) - ∫cᵢ = Field(Integral(model.tracers.c)) + bᵢ = deepcopy(model.tracers.b) + cᵢ = deepcopy(model.tracers.c) + + ∫bᵢ = Field(Integral(bᵢ)) + ∫cᵢ = Field(Integral(cᵢ)) w = model.velocities.w Nz = model.grid.Nz From 6224b7d55bcfc6014dbb4553e6794d3c306213ba Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 12 Dec 2024 10:12:25 +0100 Subject: [PATCH 556/567] make sure tests do not crash --- test/test_zstar_coordinate.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index ef56bdbd2c..1915538a5c 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -118,6 +118,7 @@ end for grid in grids split_free_surface = SplitExplicitFreeSurface(grid; cfl = 0.75) + implicit_free_surface = ImplicitFreeSurface() explicit_free_surface = ExplicitFreeSurface() @@ -136,6 +137,7 @@ end set!(model, c = (x, y, z) -> rand(), b = bᵢ) + Δt = free_surface isa ExplicitFreeSurface ? 10 : 5minutes test_zstar_coordinate(model, 100, 5minutes) end end From 9066f1f7d355090bd96c8ee278e1e909b2c47548 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 12 Dec 2024 10:12:43 +0100 Subject: [PATCH 557/567] remove space --- test/test_zstar_coordinate.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index 1915538a5c..3dfca839bd 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -118,7 +118,6 @@ end for grid in grids split_free_surface = SplitExplicitFreeSurface(grid; cfl = 0.75) - implicit_free_surface = ImplicitFreeSurface() explicit_free_surface = ExplicitFreeSurface() From cdfa58f0258c120f82c211924888e3c7bbc1aa99 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 12 Dec 2024 10:38:10 +0100 Subject: [PATCH 558/567] use correct timesteps --- test/test_zstar_coordinate.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index 3dfca839bd..1acf9e2978 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -121,7 +121,7 @@ end implicit_free_surface = ImplicitFreeSurface() explicit_free_surface = ExplicitFreeSurface() - for free_surface in [explicit_free_surface, implicit_free_surface, explicit_free_surface] + for free_surface in [split_free_surface, implicit_free_surface, explicit_free_surface] info_msg = info_message(grid, free_surface) @testset "$info_msg" begin @info " Testing a $info_msg" @@ -136,8 +136,8 @@ end set!(model, c = (x, y, z) -> rand(), b = bᵢ) - Δt = free_surface isa ExplicitFreeSurface ? 10 : 5minutes - test_zstar_coordinate(model, 100, 5minutes) + Δt = free_surface isa ExplicitFreeSurface ? 10 : 2minutes + test_zstar_coordinate(model, 100, Δt) end end end From c2d572a9d23bad1b5a4c8b752caf4c67eb04c0a1 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 12 Dec 2024 16:49:51 +0100 Subject: [PATCH 559/567] zstar following coordinate --- src/Grids/vertical_coordinate.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Grids/vertical_coordinate.jl b/src/Grids/vertical_coordinate.jl index cd8af3171b..d5ce3e6f94 100644 --- a/src/Grids/vertical_coordinate.jl +++ b/src/Grids/vertical_coordinate.jl @@ -172,5 +172,5 @@ function validate_dimension_specification(T, ξ::ZStarVerticalCoordinate, dir, N end # Summaries -coordinate_summary(::Bounded, z::AbstractVerticalCoordinate, name) = +coordinate_summary(::Bounded, z::ZStarVerticalCoordinate, name) = @sprintf("Free-surface following with Δ%s=%s", name, prettysummary(z.Δᵃᵃᶜ)) From 3e5ac724b9a1ad8dd6db16ab7c1eb786113a4251 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 13 Dec 2024 09:11:32 +0100 Subject: [PATCH 560/567] skip the test --- test/test_zstar_coordinate.jl | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index 1acf9e2978..b1563442f7 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -122,6 +122,19 @@ end explicit_free_surface = ExplicitFreeSurface() for free_surface in [split_free_surface, implicit_free_surface, explicit_free_surface] + + # TODO: There are parameter space issues with ImmersedBoundaryGrid and a immersed LatitudeLongitudeGrid + # with a stretched vertical coordinate. For the moment we are skipping these tests. + if (arch == GPU) && + (free_surface isa ImplicitFreeSurface) && + (grid isa ImmersedBoundaryGrid) && + (grid.underlying_grid isa LatitudeLongitudeGrid) && + (grid.z.Δᵃᵃᶠ isa AbstractArray) + + @info " Skipping $(info_message(grid, free_surface)) because of parameter space issues" + continue + end + info_msg = info_message(grid, free_surface) @testset "$info_msg" begin @info " Testing a $info_msg" From 63d1be394a1d25ed761175469d13277fc9d91edc Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 13 Dec 2024 09:35:47 +0100 Subject: [PATCH 561/567] chnage emojii --- .buildkite/pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 00167e1a10..985e003ae4 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -665,7 +665,7 @@ steps: ##### Vertical Coordinates tests ##### - - label: "🦧 gpu vertical coordinate" + - label: "🥑 gpu vertical coordinate" env: JULIA_DEPOT_PATH: "$SVERDRUP_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "vertical_coordinate" @@ -681,7 +681,7 @@ steps: limit: 1 depends_on: "init_gpu" - - label: "🦍 cpu vertical coordinate" + - label: "🥒 cpu vertical coordinate" env: JULIA_DEPOT_PATH: "$TARTARUS_HOME/.julia-$BUILDKITE_BUILD_NUMBER" TEST_GROUP: "vertical_coordinate" From 6377362ccccf7e61c1935c9261b312bf8e467c76 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 13 Dec 2024 10:51:21 +0100 Subject: [PATCH 562/567] update to new syntax --- .../HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index a0b2575cf0..cacd3bfdda 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -126,10 +126,10 @@ end @inline ∂y_z(i, j, k, grid) = @inbounds ∂yᶜᶠᶜ(i, j, k, grid, znode, Center(), Center(), Center()) @inline grid_slope_contribution_x(i, j, k, grid::ZStarGridOfSomeKind, buoyancy, model_fields) = - ℑxᶠᵃᵃ(i, j, k, grid, buoyancy_perturbationᶜᶜᶜ, buoyancy.model, model_fields) * ∂x_z(i, j, k, grid) + ℑxᶠᵃᵃ(i, j, k, grid, buoyancy_perturbationᶜᶜᶜ, buoyancy.formulation, model_fields) * ∂x_z(i, j, k, grid) @inline grid_slope_contribution_y(i, j, k, grid::ZStarGridOfSomeKind, buoyancy, model_fields) = - ℑyᵃᶠᵃ(i, j, k, grid, buoyancy_perturbationᶜᶜᶜ, buoyancy.model, model_fields) * ∂y_z(i, j, k, grid) + ℑyᵃᶠᵃ(i, j, k, grid, buoyancy_perturbationᶜᶜᶜ, buoyancy.formulation, model_fields) * ∂y_z(i, j, k, grid) #### #### Removing the scaling of the vertical coordinate from the tracer fields From 2953b066c5fdaaa3514a2dd2e588adf411590488 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 13 Dec 2024 10:52:08 +0100 Subject: [PATCH 563/567] align --- .../HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl index cacd3bfdda..e4b8402cc2 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/z_star_vertical_spacing.jl @@ -23,8 +23,8 @@ function update_grid!(model, grid::ZStarGridOfSomeKind; parameters = :xy) σᶜᶠⁿ = grid.z.σᶜᶠⁿ σᶠᶠⁿ = grid.z.σᶠᶠⁿ ∂t_σ = grid.z.∂t_σ - ηⁿ = grid.z.ηⁿ - η = model.free_surface.η + ηⁿ = grid.z.ηⁿ + η = model.free_surface.η launch!(architecture(grid), grid, parameters, _update_grid_scaling!, σᶜᶜⁿ, σᶠᶜⁿ, σᶜᶠⁿ, σᶠᶠⁿ, σᶜᶜ⁻, ηⁿ, grid, η) From c94606ae4f716150d0d72071010da52c0ec32a2e Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 13 Dec 2024 12:32:43 +0100 Subject: [PATCH 564/567] skip the correct tests --- test/test_zstar_coordinate.jl | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index b1563442f7..a6c8a91c1c 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -123,13 +123,12 @@ end for free_surface in [split_free_surface, implicit_free_surface, explicit_free_surface] - # TODO: There are parameter space issues with ImmersedBoundaryGrid and a immersed LatitudeLongitudeGrid - # with a stretched vertical coordinate. For the moment we are skipping these tests. + # TODO: There are parameter space issues with ImplicitFreeSurface and a immersed LatitudeLongitudeGrid + # For the moment we are skipping these tests. if (arch == GPU) && (free_surface isa ImplicitFreeSurface) && (grid isa ImmersedBoundaryGrid) && - (grid.underlying_grid isa LatitudeLongitudeGrid) && - (grid.z.Δᵃᵃᶠ isa AbstractArray) + (grid.underlying_grid isa LatitudeLongitudeGrid) @info " Skipping $(info_message(grid, free_surface)) because of parameter space issues" continue From 08ac22ebc2895dd7c91cebb8e65ff656738149b7 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 13 Dec 2024 16:50:56 +0100 Subject: [PATCH 565/567] remove correct tests --- test/test_zstar_coordinate.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_zstar_coordinate.jl b/test/test_zstar_coordinate.jl index a6c8a91c1c..427d5bcff5 100644 --- a/test/test_zstar_coordinate.jl +++ b/test/test_zstar_coordinate.jl @@ -125,7 +125,7 @@ end # TODO: There are parameter space issues with ImplicitFreeSurface and a immersed LatitudeLongitudeGrid # For the moment we are skipping these tests. - if (arch == GPU) && + if (arch isa GPU) && (free_surface isa ImplicitFreeSurface) && (grid isa ImmersedBoundaryGrid) && (grid.underlying_grid isa LatitudeLongitudeGrid) From 08cbad8f6a8dfd72dd8e7fe3f5ce2109255b4027 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Sun, 15 Dec 2024 13:42:43 +0100 Subject: [PATCH 566/567] make sure we don't hit nans --- .../barotropic_split_explicit_corrector.jl | 10 ++++++++-- .../split_explicit_timesteppers.jl | 4 ++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_split_explicit_corrector.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_split_explicit_corrector.jl index 23f25d6379..e6dc7fe483 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_split_explicit_corrector.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/barotropic_split_explicit_corrector.jl @@ -13,8 +13,14 @@ end @inline function barotropic_mode_kernel!(U̅, V̅, i, j, grid, u, v, η) k_top = size(grid, 3) + 1 - sᶠᶜ = dynamic_column_depthᶠᶜᵃ(i, j, k_top, grid, η) / static_column_depthᶠᶜᵃ(i, j, grid) - sᶜᶠ = dynamic_column_depthᶜᶠᵃ(i, j, k_top, grid, η) / static_column_depthᶜᶠᵃ(i, j, grid) + hᶠᶜ = static_column_depthᶠᶜᵃ(i, j, grid) + hᶜᶠ = static_column_depthᶜᶠᵃ(i, j, grid) + + Hᶠᶜ = dynamic_column_depthᶠᶜᵃ(i, j, k_top, grid, η) + Hᶜᶠ = dynamic_column_depthᶜᶠᵃ(i, j, k_top, grid, η) + + sᶠᶜ = ifelse(hᶠᶜ == 0, zero(grid), Hᶠᶜ / hᶠᶜ) + sᶜᶠ = ifelse(hᶜᶠ == 0, zero(grid), Hᶜᶠ / hᶜᶠ) @inbounds U̅[i, j, 1] = Δrᶠᶜᶜ(i, j, 1, grid) * u[i, j, 1] * sᶠᶜ @inbounds V̅[i, j, 1] = Δrᶜᶠᶜ(i, j, 1, grid) * v[i, j, 1] * sᶜᶠ diff --git a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_timesteppers.jl b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_timesteppers.jl index e624054d5e..a450e99de6 100644 --- a/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_timesteppers.jl +++ b/src/Models/HydrostaticFreeSurfaceModels/SplitExplicitFreeSurfaces/split_explicit_timesteppers.jl @@ -135,10 +135,10 @@ function initialize_free_surface_timestepper!(timestepper::AdamsBashforth3Scheme end # The functions `η★` `U★` and `V★` represent the value of free surface, barotropic zonal and meridional velocity at time step m+1/2 -@inline U★(i, j, k, grid, t::ForwardBackwardScheme, Uᵐ) = @inbounds Uᵐ[i, j, k] +@inline U★(i, j, k, grid, ::ForwardBackwardScheme, Uᵐ) = @inbounds Uᵐ[i, j, k] @inline U★(i, j, k, grid, t::AdamsBashforth3Scheme, Uᵐ) = @inbounds t.α * Uᵐ[i, j, k] + t.θ * t.Uᵐ⁻¹[i, j, k] + t.β * t.Uᵐ⁻²[i, j, k] -@inline η★(i, j, k, grid, t::ForwardBackwardScheme, ηᵐ⁺¹) = @inbounds ηᵐ⁺¹[i, j, k] +@inline η★(i, j, k, grid, ::ForwardBackwardScheme, ηᵐ⁺¹) = @inbounds ηᵐ⁺¹[i, j, k] @inline η★(i, j, k, grid, t::AdamsBashforth3Scheme, ηᵐ⁺¹) = @inbounds t.δ * ηᵐ⁺¹[i, j, k] + t.μ * t.ηᵐ[i, j, k] + t.γ * t.ηᵐ⁻¹[i, j, k] + t.ϵ * t.ηᵐ⁻²[i, j, k] @inline store_previous_velocities!(::ForwardBackwardScheme, i, j, k, U) = nothing From c0f5bab170a2f963cb780aa154c8bc46a059e1fc Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Sun, 15 Dec 2024 14:16:57 +0100 Subject: [PATCH 567/567] bugfix --- src/Grids/grid_utils.jl | 2 +- src/ImmersedBoundaries/zstar_immersed_grid.jl | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Grids/grid_utils.jl b/src/Grids/grid_utils.jl index b7b09d6212..4c3bbe9df6 100644 --- a/src/Grids/grid_utils.jl +++ b/src/Grids/grid_utils.jl @@ -327,8 +327,8 @@ coordinate_summary(topo, Δ::Union{AbstractVector, AbstractMatrix}, name) = # Will be extended in the `ImmersedBoundaries` module for any ZStar grid type @inline dynamic_column_depthᶜᶜᵃ(i, j, k, grid, η) = static_column_depthᶜᶜᵃ(i, j, grid) -@inline dynamic_column_depthᶜᶠᵃ(i, j, k, grid, η) = static_column_depthᶜᶠᵃ(i, j, grid) @inline dynamic_column_depthᶠᶜᵃ(i, j, k, grid, η) = static_column_depthᶠᶜᵃ(i, j, grid) +@inline dynamic_column_depthᶜᶠᵃ(i, j, k, grid, η) = static_column_depthᶜᶠᵃ(i, j, grid) @inline dynamic_column_depthᶠᶠᵃ(i, j, k, grid, η) = static_column_depthᶠᶠᵃ(i, j, grid) ##### diff --git a/src/ImmersedBoundaries/zstar_immersed_grid.jl b/src/ImmersedBoundaries/zstar_immersed_grid.jl index 39fc271318..2296cd3eae 100644 --- a/src/ImmersedBoundaries/zstar_immersed_grid.jl +++ b/src/ImmersedBoundaries/zstar_immersed_grid.jl @@ -13,8 +13,8 @@ const ZStarImmersedGrid = ImmersedBoundaryGrid{<:Any, <:Any, <:Any, <:Any, <:A const ZStarGridOfSomeKind = Union{ZStarImmersedGrid, AbstractZStarGrid} @inline dynamic_column_depthᶜᶜᵃ(i, j, k, grid::ZStarGridOfSomeKind, η) = @inbounds static_column_depthᶜᶜᵃ(i, j, grid) + η[i, j, k] -@inline dynamic_column_depthᶜᶠᵃ(i, j, k, grid::ZStarGridOfSomeKind, η) = @inbounds static_column_depthᶠᶜᵃ(i, j, grid) + ℑyᵃᶠᵃ(i, j, k, grid, η) -@inline dynamic_column_depthᶠᶜᵃ(i, j, k, grid::ZStarGridOfSomeKind, η) = @inbounds static_column_depthᶜᶠᵃ(i, j, grid) + ℑxᶠᵃᵃ(i, j, k, grid, η) +@inline dynamic_column_depthᶠᶜᵃ(i, j, k, grid::ZStarGridOfSomeKind, η) = @inbounds static_column_depthᶠᶜᵃ(i, j, grid) + ℑxᶠᵃᵃ(i, j, k, grid, η) +@inline dynamic_column_depthᶜᶠᵃ(i, j, k, grid::ZStarGridOfSomeKind, η) = @inbounds static_column_depthᶜᶠᵃ(i, j, grid) + ℑyᵃᶠᵃ(i, j, k, grid, η) @inline dynamic_column_depthᶠᶠᵃ(i, j, k, grid::ZStarGridOfSomeKind, η) = @inbounds static_column_depthᶠᶠᵃ(i, j, grid) + ℑxyᶠᶠᵃ(i, j, k, grid, η) # Convenience methods