From 10c8d948a9fce3991bf6f21ac761f513de978a56 Mon Sep 17 00:00:00 2001 From: Daniel Doehring Date: Tue, 10 Oct 2023 11:59:50 +0200 Subject: [PATCH 1/4] Print addition to DOF count (#1669) --- src/callbacks_step/analysis.jl | 2 +- src/semidiscretization/semidiscretization_coupled.jl | 2 +- src/semidiscretization/semidiscretization_hyperbolic.jl | 2 +- .../semidiscretization_hyperbolic_parabolic.jl | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/callbacks_step/analysis.jl b/src/callbacks_step/analysis.jl index fad42b11098..a3e49007fb6 100644 --- a/src/callbacks_step/analysis.jl +++ b/src/callbacks_step/analysis.jl @@ -297,7 +297,7 @@ function (analysis_callback::AnalysisCallback)(u_ode, du_ode, integrator, semi) mpi_println(" " * " " * " " * " PID: " * @sprintf("%10.8e s", performance_index)) - mpi_println(" #DOF: " * @sprintf("% 14d", ndofs(semi)) * + mpi_println(" #DOFs per field:" * @sprintf("% 14d", ndofs(semi)) * " " * " alloc'd memory: " * @sprintf("%14.3f MiB", memory_use)) mpi_println(" #elements: " * diff --git a/src/semidiscretization/semidiscretization_coupled.jl b/src/semidiscretization/semidiscretization_coupled.jl index b7adff78425..49763b12e5d 100644 --- a/src/semidiscretization/semidiscretization_coupled.jl +++ b/src/semidiscretization/semidiscretization_coupled.jl @@ -71,7 +71,7 @@ function Base.show(io::IO, ::MIME"text/plain", semi::SemidiscretizationCoupled) summary_line(increment_indent(io), "source terms", semi.semis[i].source_terms) summary_line(increment_indent(io), "solver", solver |> typeof |> nameof) end - summary_line(io, "total #DOFs", ndofs(semi)) + summary_line(io, "total #DOFs per field", ndofs(semi)) summary_footer(io) end end diff --git a/src/semidiscretization/semidiscretization_hyperbolic.jl b/src/semidiscretization/semidiscretization_hyperbolic.jl index 50b2c21c14e..9d465bfcc5f 100644 --- a/src/semidiscretization/semidiscretization_hyperbolic.jl +++ b/src/semidiscretization/semidiscretization_hyperbolic.jl @@ -243,7 +243,7 @@ function Base.show(io::IO, ::MIME"text/plain", semi::SemidiscretizationHyperboli summary_line(io, "source terms", semi.source_terms) summary_line(io, "solver", semi.solver |> typeof |> nameof) - summary_line(io, "total #DOFs", ndofs(semi)) + summary_line(io, "total #DOFs per field", ndofs(semi)) summary_footer(io) end end diff --git a/src/semidiscretization/semidiscretization_hyperbolic_parabolic.jl b/src/semidiscretization/semidiscretization_hyperbolic_parabolic.jl index b12ecadb58b..35340d65b1e 100644 --- a/src/semidiscretization/semidiscretization_hyperbolic_parabolic.jl +++ b/src/semidiscretization/semidiscretization_hyperbolic_parabolic.jl @@ -228,7 +228,7 @@ function Base.show(io::IO, ::MIME"text/plain", summary_line(io, "source terms", semi.source_terms) summary_line(io, "solver", semi.solver |> typeof |> nameof) summary_line(io, "parabolic solver", semi.solver_parabolic |> typeof |> nameof) - summary_line(io, "total #DOFs", ndofs(semi)) + summary_line(io, "total #DOFs per field", ndofs(semi)) summary_footer(io) end end From 8bc5469b3d883be0ce088386e1ec794e448003ef Mon Sep 17 00:00:00 2001 From: Jesse Chan <1156048+jlchan@users.noreply.github.com> Date: Wed, 11 Oct 2023 06:22:49 -0500 Subject: [PATCH 2/4] fix bug in `apply_jacobian_parabolic!` for `P4estMesh` (#1668) * fix bug in parabolic jacobian computations * formatting * updating test results and removing duplicate test --------- Co-authored-by: Hendrik Ranocha --- src/solvers/dgsem_tree/dg_2d_parabolic.jl | 20 +++++++++++++++++++- src/solvers/dgsem_tree/dg_3d_parabolic.jl | 23 +++++++++++++++++++++-- test/test_parabolic_2d.jl | 16 ++++------------ 3 files changed, 44 insertions(+), 15 deletions(-) diff --git a/src/solvers/dgsem_tree/dg_2d_parabolic.jl b/src/solvers/dgsem_tree/dg_2d_parabolic.jl index 3dbc55412ad..1c32703c7c3 100644 --- a/src/solvers/dgsem_tree/dg_2d_parabolic.jl +++ b/src/solvers/dgsem_tree/dg_2d_parabolic.jl @@ -935,7 +935,7 @@ end # This is because the parabolic fluxes are assumed to be of the form # `du/dt + df/dx = dg/dx + source(x,t)`, # where f(u) is the inviscid flux and g(u) is the viscous flux. -function apply_jacobian_parabolic!(du, mesh::Union{TreeMesh{2}, P4estMesh{2}}, +function apply_jacobian_parabolic!(du, mesh::TreeMesh{2}, equations::AbstractEquationsParabolic, dg::DG, cache) @unpack inverse_jacobian = cache.elements @@ -951,4 +951,22 @@ function apply_jacobian_parabolic!(du, mesh::Union{TreeMesh{2}, P4estMesh{2}}, return nothing end + +function apply_jacobian_parabolic!(du, mesh::P4estMesh{2}, + equations::AbstractEquationsParabolic, + dg::DG, cache) + @unpack inverse_jacobian = cache.elements + + @threaded for element in eachelement(dg, cache) + for j in eachnode(dg), i in eachnode(dg) + factor = inverse_jacobian[i, j, element] + + for v in eachvariable(equations) + du[v, i, j, element] *= factor + end + end + end + + return nothing +end end # @muladd diff --git a/src/solvers/dgsem_tree/dg_3d_parabolic.jl b/src/solvers/dgsem_tree/dg_3d_parabolic.jl index 9817e0e5f0e..37492dbcb91 100644 --- a/src/solvers/dgsem_tree/dg_3d_parabolic.jl +++ b/src/solvers/dgsem_tree/dg_3d_parabolic.jl @@ -1125,8 +1125,9 @@ end # This is because the parabolic fluxes are assumed to be of the form # `du/dt + df/dx = dg/dx + source(x,t)`, # where f(u) is the inviscid flux and g(u) is the viscous flux. -function apply_jacobian_parabolic!(du, mesh::Union{TreeMesh{3}, P4estMesh{3}}, - equations::AbstractEquationsParabolic, dg::DG, cache) +function apply_jacobian_parabolic!(du, mesh::TreeMesh{3}, + equations::AbstractEquationsParabolic, + dg::DG, cache) @unpack inverse_jacobian = cache.elements @threaded for element in eachelement(dg, cache) @@ -1141,4 +1142,22 @@ function apply_jacobian_parabolic!(du, mesh::Union{TreeMesh{3}, P4estMesh{3}}, return nothing end + +function apply_jacobian_parabolic!(du, mesh::P4estMesh{3}, + equations::AbstractEquationsParabolic, + dg::DG, cache) + @unpack inverse_jacobian = cache.elements + + @threaded for element in eachelement(dg, cache) + for k in eachnode(dg), j in eachnode(dg), i in eachnode(dg) + factor = inverse_jacobian[i, j, k, element] + + for v in eachvariable(equations) + du[v, i, j, k, element] *= factor + end + end + end + + return nothing +end end # @muladd diff --git a/test/test_parabolic_2d.jl b/test/test_parabolic_2d.jl index 967aa1069a0..98aa337afb4 100644 --- a/test/test_parabolic_2d.jl +++ b/test/test_parabolic_2d.jl @@ -260,24 +260,16 @@ isdir(outdir) && rm(outdir, recursive=true) @trixi_testset "P4estMesh2D: elixir_advection_diffusion_periodic_curved.jl" begin @test_trixi_include(joinpath(examples_dir(), "p4est_2d_dgsem", "elixir_advection_diffusion_periodic_curved.jl"), trees_per_dimension = (1, 1), initial_refinement_level = 2, tspan=(0.0, 0.5), - l2 = [0.012380458938507371], - linf = [0.10860506906472567] - ) - end - - @trixi_testset "P4estMesh2D: elixir_advection_diffusion_periodic_curved.jl" begin - @test_trixi_include(joinpath(examples_dir(), "p4est_2d_dgsem", "elixir_advection_diffusion_periodic_curved.jl"), - trees_per_dimension = (1, 1), initial_refinement_level = 2, tspan=(0.0, 0.5), - l2 = [0.012380458938507371], - linf = [0.10860506906472567] + l2 = [0.006708147442490916], + linf = [0.04807038397976693] ) end @trixi_testset "P4estMesh2D: elixir_advection_diffusion_nonperiodic_curved.jl" begin @test_trixi_include(joinpath(examples_dir(), "p4est_2d_dgsem", "elixir_advection_diffusion_nonperiodic_curved.jl"), trees_per_dimension = (1, 1), initial_refinement_level = 2, tspan=(0.0, 0.5), - l2 = [0.04933902988507035], - linf = [0.2550261714590271] + l2 = [0.00919917034843865], + linf = [0.14186297438393505] ) end From 1727b072fa9f269bcde73b516bc7b00f13fc9c99 Mon Sep 17 00:00:00 2001 From: Joshua Lampert <51029046+JoshuaLampert@users.noreply.github.com> Date: Wed, 11 Oct 2023 16:23:12 +0200 Subject: [PATCH 3/4] Show percentage of simulation progress (#1659) * show percentage of simulation progress * add notes about AnalysisCallback in docs * generalize for general initial time --------- Co-authored-by: Hendrik Ranocha --- docs/src/callbacks.md | 5 +++++ src/callbacks_step/alive.jl | 10 ++++++++-- src/callbacks_step/analysis.jl | 10 ++++++++-- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/docs/src/callbacks.md b/docs/src/callbacks.md index 7f44dfd5925..f018bcf7c39 100644 --- a/docs/src/callbacks.md +++ b/docs/src/callbacks.md @@ -30,6 +30,11 @@ An example elixir using AMR can be found at [`examples/tree_2d_dgsem/elixir_adve The [`AnalysisCallback`](@ref) can be used to analyze the numerical solution, e.g. calculate errors or user-specified integrals, and print the results to the screen. The results can also be saved in a file. An example can be found at [`examples/tree_2d_dgsem/elixir_euler_vortex.jl`](https://github.com/trixi-framework/Trixi.jl/blob/main/examples/tree_2d_dgsem/elixir_euler_vortex.jl). +Note that the errors (e.g. `L2 error` or `Linf error`) are computed with respect to the initial condition. +The percentage of the simulation time refers to the ratio of the current time and the final time, i.e. it does +not consider the maximal number of iterations. So the simulation could finish before 100% are reached. +Note that, e.g., due to AMR or smaller time step sizes, the simulation can actually take longer than +the percentage indicates. In [Performance metrics of the `AnalysisCallback`](@ref performance-metrics) you can find a detailed description of the different performance metrics the `AnalysisCallback` computes. diff --git a/src/callbacks_step/alive.jl b/src/callbacks_step/alive.jl index eeacd9681d8..9df7181521e 100644 --- a/src/callbacks_step/alive.jl +++ b/src/callbacks_step/alive.jl @@ -86,9 +86,15 @@ function (alive_callback::AliveCallback)(integrator) println("─"^100) println() elseif mpi_isroot() + t = integrator.t + t_initial = first(integrator.sol.prob.tspan) + t_final = last(integrator.sol.prob.tspan) + sim_time_percentage = (t - t_initial) / (t_final - t_initial) * 100 runtime_absolute = 1.0e-9 * (time_ns() - alive_callback.start_time) - @printf("#timesteps: %6d │ Δt: %.4e │ sim. time: %.4e │ run time: %.4e s\n", - integrator.stats.naccept, integrator.dt, integrator.t, runtime_absolute) + println(rpad(@sprintf("#timesteps: %6d │ Δt: %.4e │ sim. time: %.4e (%5.3f%%)", + integrator.stats.naccept, integrator.dt, t, + sim_time_percentage), 71) * + @sprintf("│ run time: %.4e s", runtime_absolute)) end # avoid re-evaluating possible FSAL stages diff --git a/src/callbacks_step/analysis.jl b/src/callbacks_step/analysis.jl index a3e49007fb6..e5b4a01a885 100644 --- a/src/callbacks_step/analysis.jl +++ b/src/callbacks_step/analysis.jl @@ -232,6 +232,12 @@ function (analysis_callback::AnalysisCallback)(u_ode, du_ode, integrator, semi) @unpack dt, t = integrator iter = integrator.stats.naccept + # Compute the percentage of the simulation that is done + t = integrator.t + t_initial = first(integrator.sol.prob.tspan) + t_final = last(integrator.sol.prob.tspan) + sim_time_percentage = (t - t_initial) / (t_final - t_initial) * 100 + # Record performance measurements and compute performance index (PID) runtime_since_last_analysis = 1.0e-9 * (time_ns() - analysis_callback.start_time_last_analysis) @@ -291,8 +297,8 @@ function (analysis_callback::AnalysisCallback)(u_ode, du_ode, integrator, semi) " " * " └── GC time: " * @sprintf("%10.8e s (%5.3f%%)", gc_time_absolute, gc_time_percentage)) - mpi_println(" sim. time: " * @sprintf("%10.8e", t) * - " " * + mpi_println(rpad(" sim. time: " * + @sprintf("%10.8e (%5.3f%%)", t, sim_time_percentage), 46) * " time/DOF/rhs!: " * @sprintf("%10.8e s", runtime_relative)) mpi_println(" " * " " * " " * From 7fd4503af9dfdf7022ae75f7398cac8fda811c85 Mon Sep 17 00:00:00 2001 From: Benjamin Bolm <74359358+bennibolm@users.noreply.github.com> Date: Thu, 12 Oct 2023 15:31:43 +0200 Subject: [PATCH 4/4] Subcell limiting: Revise order of bounds using a `Dict` (#1649) * Implement order of bounds as Dict * Implement `variable_bounds` directly as Dictionary * Remove unnecessary line of code * Use Symbols instead of Strings --- src/solvers/dgsem_tree/containers_2d.jl | 31 ++++++++++--------- src/solvers/dgsem_tree/subcell_limiters.jl | 5 +-- src/solvers/dgsem_tree/subcell_limiters_2d.jl | 31 ++++++++----------- 3 files changed, 32 insertions(+), 35 deletions(-) diff --git a/src/solvers/dgsem_tree/containers_2d.jl b/src/solvers/dgsem_tree/containers_2d.jl index 9148b936312..fc916b41dd2 100644 --- a/src/solvers/dgsem_tree/containers_2d.jl +++ b/src/solvers/dgsem_tree/containers_2d.jl @@ -1325,16 +1325,16 @@ mutable struct ContainerSubcellLimiterIDP2D{uEltype <: Real} alpha::Array{uEltype, 3} # [i, j, element] alpha1::Array{uEltype, 3} alpha2::Array{uEltype, 3} - variable_bounds::Vector{Array{uEltype, 3}} + variable_bounds::Dict{Symbol, Array{uEltype, 3}} # internal `resize!`able storage _alpha::Vector{uEltype} _alpha1::Vector{uEltype} _alpha2::Vector{uEltype} - _variable_bounds::Vector{Vector{uEltype}} + _variable_bounds::Dict{Symbol, Vector{uEltype}} end function ContainerSubcellLimiterIDP2D{uEltype}(capacity::Integer, n_nodes, - length) where {uEltype <: Real} + bound_keys) where {uEltype <: Real} nan_uEltype = convert(uEltype, NaN) # Initialize fields with defaults @@ -1345,12 +1345,12 @@ function ContainerSubcellLimiterIDP2D{uEltype}(capacity::Integer, n_nodes, _alpha2 = fill(nan_uEltype, n_nodes * (n_nodes + 1) * capacity) alpha2 = unsafe_wrap(Array, pointer(_alpha2), (n_nodes, n_nodes + 1, capacity)) - _variable_bounds = Vector{Vector{uEltype}}(undef, length) - variable_bounds = Vector{Array{uEltype, 3}}(undef, length) - for i in 1:length - _variable_bounds[i] = fill(nan_uEltype, n_nodes * n_nodes * capacity) - variable_bounds[i] = unsafe_wrap(Array, pointer(_variable_bounds[i]), - (n_nodes, n_nodes, capacity)) + _variable_bounds = Dict{Symbol, Vector{uEltype}}() + variable_bounds = Dict{Symbol, Array{uEltype, 3}}() + for key in bound_keys + _variable_bounds[key] = fill(nan_uEltype, n_nodes * n_nodes * capacity) + variable_bounds[key] = unsafe_wrap(Array, pointer(_variable_bounds[key]), + (n_nodes, n_nodes, capacity)) end return ContainerSubcellLimiterIDP2D{uEltype}(alpha, alpha1, alpha2, @@ -1369,7 +1369,7 @@ nnodes(container::ContainerSubcellLimiterIDP2D) = size(container.alpha, 1) function Base.resize!(container::ContainerSubcellLimiterIDP2D, capacity) n_nodes = nnodes(container) - @unpack _alpha, _alpha1, _alpha2 = container + (; _alpha, _alpha1, _alpha2) = container resize!(_alpha, n_nodes * n_nodes * capacity) container.alpha = unsafe_wrap(Array, pointer(_alpha), (n_nodes, n_nodes, capacity)) resize!(_alpha1, (n_nodes + 1) * n_nodes * capacity) @@ -1379,11 +1379,12 @@ function Base.resize!(container::ContainerSubcellLimiterIDP2D, capacity) container.alpha2 = unsafe_wrap(Array, pointer(_alpha2), (n_nodes, n_nodes + 1, capacity)) - @unpack _variable_bounds = container - for i in 1:length(_variable_bounds) - resize!(_variable_bounds[i], n_nodes * n_nodes * capacity) - container.variable_bounds[i] = unsafe_wrap(Array, pointer(_variable_bounds[i]), - (n_nodes, n_nodes, capacity)) + (; _variable_bounds) = container + for (key, _) in _variable_bounds + resize!(_variable_bounds[key], n_nodes * n_nodes * capacity) + container.variable_bounds[key] = unsafe_wrap(Array, + pointer(_variable_bounds[key]), + (n_nodes, n_nodes, capacity)) end return nothing diff --git a/src/solvers/dgsem_tree/subcell_limiters.jl b/src/solvers/dgsem_tree/subcell_limiters.jl index 3a707de3bc7..4d9eec25a89 100644 --- a/src/solvers/dgsem_tree/subcell_limiters.jl +++ b/src/solvers/dgsem_tree/subcell_limiters.jl @@ -52,9 +52,10 @@ function SubcellLimiterIDP(equations::AbstractEquations, basis; positivity_variables_cons = [], positivity_correction_factor = 0.1) positivity = (length(positivity_variables_cons) > 0) - number_bounds = length(positivity_variables_cons) - cache = create_cache(SubcellLimiterIDP, equations, basis, number_bounds) + bound_keys = Tuple(Symbol("$(i)_min") for i in positivity_variables_cons) + + cache = create_cache(SubcellLimiterIDP, equations, basis, bound_keys) SubcellLimiterIDP{typeof(positivity_correction_factor), typeof(cache)}(positivity, positivity_variables_cons, diff --git a/src/solvers/dgsem_tree/subcell_limiters_2d.jl b/src/solvers/dgsem_tree/subcell_limiters_2d.jl index 09ab84ed11a..4497217fb56 100644 --- a/src/solvers/dgsem_tree/subcell_limiters_2d.jl +++ b/src/solvers/dgsem_tree/subcell_limiters_2d.jl @@ -6,17 +6,14 @@ #! format: noindent # this method is used when the limiter is constructed as for shock-capturing volume integrals -function create_cache(indicator::Type{SubcellLimiterIDP}, - equations::AbstractEquations{2}, - basis::LobattoLegendreBasis, number_bounds) +function create_cache(limiter::Type{SubcellLimiterIDP}, equations::AbstractEquations{2}, + basis::LobattoLegendreBasis, bound_keys) subcell_limiter_coefficients = Trixi.ContainerSubcellLimiterIDP2D{real(basis) }(0, nnodes(basis), - number_bounds) + bound_keys) - cache = (; subcell_limiter_coefficients) - - return cache + return (; subcell_limiter_coefficients) end function (limiter::SubcellLimiterIDP)(u::AbstractArray{<:Any, 4}, semi, dg::DGSEM, t, @@ -26,8 +23,7 @@ function (limiter::SubcellLimiterIDP)(u::AbstractArray{<:Any, 4}, semi, dg::DGSE alpha .= zero(eltype(alpha)) if limiter.positivity - @trixi_timeit timer() "positivity" idp_positivity!(alpha, limiter, u, dt, - semi) + @trixi_timeit timer() "positivity" idp_positivity!(alpha, limiter, u, dt, semi) end # Calculate alpha1 and alpha2 @@ -50,22 +46,21 @@ end @inline function idp_positivity!(alpha, limiter, u, dt, semi) # Conservative variables - for (index, variable) in enumerate(limiter.positivity_variables_cons) - idp_positivity!(alpha, limiter, u, dt, semi, variable, index) + for variable in limiter.positivity_variables_cons + idp_positivity!(alpha, limiter, u, dt, semi, variable) end return nothing end -@inline function idp_positivity!(alpha, limiter, u, dt, semi, variable, index) +@inline function idp_positivity!(alpha, limiter, u, dt, semi, variable) mesh, equations, dg, cache = mesh_equations_solver_cache(semi) - @unpack antidiffusive_flux1, antidiffusive_flux2 = cache.antidiffusive_fluxes - @unpack inverse_weights = dg.basis - @unpack positivity_correction_factor = limiter - - @unpack variable_bounds = limiter.cache.subcell_limiter_coefficients + (; antidiffusive_flux1, antidiffusive_flux2) = cache.antidiffusive_fluxes + (; inverse_weights) = dg.basis + (; positivity_correction_factor) = limiter - var_min = variable_bounds[index] + (; variable_bounds) = limiter.cache.subcell_limiter_coefficients + var_min = variable_bounds[Symbol("$(variable)_min")] @threaded for element in eachelement(dg, semi.cache) inverse_jacobian = cache.elements.inverse_jacobian[element]