From c5a37c0c760a5571cefa3b8bbe70fa3ae2145e02 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 28 Feb 2022 16:56:50 +0100 Subject: [PATCH 001/129] sgs_fra solver declaration --- .../solvers/mpdata_rhs_vip_prs_sgs_fra.hpp | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp diff --git a/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp b/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp new file mode 100644 index 00000000..00e9da0c --- /dev/null +++ b/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp @@ -0,0 +1,45 @@ +/** + * @file + * @copyright University of Warsaw + * @section LICENSE + * GPLv3+ (see the COPYING file or http://www.gnu.org/licenses/) + * + */ + +#pragma once + +#include + +namespace libmpdataxx +{ + namespace solvers + { + struct mpdata_rhs_vip_prs_sgs_fra_family_tag {}; + + template + class mpdata_rhs_vip_prs_sgs_fra; + + template + class mpdata_rhs_vip_prs_sgs_fra< + ct_params_t, minhalo, + typename std::enable_if<(int)ct_params_t::fra_rec == 0>::type // fra_rec is the number of fractal reconstructions + > : public mpdata_rhs_vip_prs_sgs + { + using parent_t = mpdata_rhs_vip_prs_sgs; + using parent_t::parent_t; // inheriting constructors + }; + + template + class mpdata_rhs_vip_prs_sgs_fra< + ct_params_t, minhalo, + typename std::enable_if<(int)ct_params_t::fra_rec > 0>::type + > : public detail::mpdata_rhs_vip_prs_sgs_fra + { + using parent_t = detail::mpdata_rhs_vip_prs_sgs_fra; + using parent_t::parent_t; // inheriting constructors + + protected: + using solver_family = mpdata_rhs_vip_prs_sgs_fra_family_tag; + }; + } // namespace solvers +} // namespace libmpdataxx From f9158753d0707b3f44b7fd652e2a009208370b5d Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 1 Mar 2022 16:51:58 +0100 Subject: [PATCH 002/129] wip on fractal refinement --- libmpdata++/concurr/detail/concurr_common.hpp | 1 + libmpdata++/concurr/detail/sharedmem.hpp | 1 + .../concurr/detail/sharedmem_refined.hpp | 483 ++++++++++++++++++ libmpdata++/concurr/openmp.hpp | 3 +- .../detail/mpdata_rhs_vip_prs_sgs_fra.hpp | 68 +++ libmpdata++/solvers/detail/solver_3d.hpp | 25 +- .../solvers/mpdata_rhs_vip_prs_sgs_fra.hpp | 4 +- 7 files changed, 571 insertions(+), 14 deletions(-) create mode 100644 libmpdata++/concurr/detail/sharedmem_refined.hpp create mode 100644 libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra.hpp diff --git a/libmpdata++/concurr/detail/concurr_common.hpp b/libmpdata++/concurr/detail/concurr_common.hpp index 49e53eeb..4e224785 100644 --- a/libmpdata++/concurr/detail/concurr_common.hpp +++ b/libmpdata++/concurr/detail/concurr_common.hpp @@ -154,6 +154,7 @@ namespace libmpdataxx typename solver_t::real_t, solver_t::n_dims, solver_t::n_tlev + if solver family is _fra use sharedmem_rec, else use sharedmem > mem_t; // member fields diff --git a/libmpdata++/concurr/detail/sharedmem.hpp b/libmpdata++/concurr/detail/sharedmem.hpp index a934f790..653302d1 100644 --- a/libmpdata++/concurr/detail/sharedmem.hpp +++ b/libmpdata++/concurr/detail/sharedmem.hpp @@ -309,6 +309,7 @@ namespace libmpdataxx } virtual arr_t advectee(int e = 0) = 0; + virtual const arr_t advectee_global(int e = 0) = 0; void advectee_global_set(const arr_t arr, int e = 0) { diff --git a/libmpdata++/concurr/detail/sharedmem_refined.hpp b/libmpdata++/concurr/detail/sharedmem_refined.hpp new file mode 100644 index 00000000..f75a150f --- /dev/null +++ b/libmpdata++/concurr/detail/sharedmem_refined.hpp @@ -0,0 +1,483 @@ +/** @file + * @copyright University of Warsaw + * @section LICENSE + * GPLv3+ (see the COPYING file or http://www.gnu.org/licenses/) + */ + +// memory management with (fractal) grid refinement + +#pragma once + +#include + +namespace libmpdataxx +{ + namespace concurr + { + namespace detail + { + template < + typename real_t, + int n_dims, + int n_tlev + > + class sharedmem_refined_common :: sharedmem + { + using parent_t = sharedmem; + using arr_t = typename parent_t::parent_t::arr_t; + + protected: + + const int n_ref; // number of equal divisions of the large cell (in each direction) + blitz::TinyVector origin_ref; + + public: + + std::array grid_size_ref; + // TODO: these are public because used from outside in alloc - could friendship help? + arrvec_t GC_ref, psi_ref; + + // ctors + sharedmem_refined_common(const std::array &grid_size, const int &size, const int &n_ref) + : parent_t(grid_size, size), n_ref(n_ref) + { + for (int d = 0; d < n_dims; ++d) + { + grid_size_ref[d] = refine_grid_size(this->grid_size, n_ref) + origin_ref[d] = grid_size_ref[d].first(); + } + } + + virtual arr_t advectee_ref(int e = 0) = 0; + virtual const arr_t advectee_global_ref(int e = 0) = 0; + + public: + static rng_t refine_grid_size( + const rng_t &grid_size, + const int &n_ref + ) { + return rng_t( + grid_size.first() * n_ref, + grid_size.first() + (grid_size.last() - grid_size.first()) * n_ref + ); + } + }; + + + + + + + + +/* + + + + + template + class sharedmem + {}; + + template + class sharedmem : public sharedmem_common + { + using parent_t = sharedmem_common; + using parent_t::parent_t; // inheriting ctors + + public: + + // accessor methods + blitz::Array advectee(int e = 0) + { + assert(this->n < n_tlev); + + // returning just the domain interior, i.e. without halos + // reindexing so that element 0 is at 0 + return this->psi[e][ this->n ]( + this->grid_size[0] + ).reindex(this->origin); + } + + const blitz::Array advectee_global(int e = 0) + { +#if defined(USE_MPI) + if(this->distmem.size() > 1) + { +// TODO: move some of that to distmem... +// TODO: a lot of common code betwee 1,2 and 3 dimensions... + + // a vector of number of elements to be sent by each non-root process + std::vector sizes(this->distmem.size()); + std::iota(sizes.begin(), sizes.end(), 0); // fill with 1,2,3,... + for(auto &size : sizes) { size = this->slab(rng_t(0, this->distmem.grid_size[0]-1), size, this->distmem.size()).length();} + // calc displacement + std::vector displ(sizes.size()); + std::partial_sum(sizes.begin(), sizes.end(), displ.begin()); + std::transform(displ.begin(), displ.end(), sizes.begin(), displ.begin(), std::minus()); // exclusive_scan is c++17 + // a vector that will store the received data, relevant only on process rank=0 + std::vector out_values(this->distmem.grid_size[0]); + // create an array that will store advectee to be sent in a contiguous memory block + std::vector in_values_vec(advectee(e).size()); + std::copy(advectee(e).begin(), advectee(e).end(), in_values_vec.begin()); + + // gather the data from all processes on rank=0 + boost::mpi::gatherv(this->distmem.mpicom, in_values_vec, out_values.data(), sizes, displ, 0); + // send the result to other processes + boost::mpi::broadcast(this->distmem.mpicom, out_values, 0); + + blitz::Array res(out_values.data(), blitz::shape(this->distmem.grid_size[0]), blitz::duplicateData); + return res; + } + else +#endif + return advectee(e); + } + + blitz::Array advector(int d = 0) + { + using namespace arakawa_c; + assert(d == 0); + // returning just the domain interior, i.e. without halos + // reindexed to make it more intuitive when working with index placeholders + // (i.e. border between cell 0 and cell 1 is indexed with 0) + auto orgn = decltype(this->origin)({ + this->origin[0] - 1 + }); + + return this->GC[d]( + this->distmem_ext(this->grid_size[0]^(-1)^h) + ).reindex( + this->distmem.rank() > 0 + ? decltype(this->origin)({this->origin[0] - 1}) + : orgn + ); + } + + blitz::Array g_factor() + { + // a sanity check + if (this->G.get() == nullptr) + throw std::runtime_error("g_factor() called with nug option unset?"); + + // the same logic as in advectee() - see above + return (*this->G)( + this->grid_size[0] + ).reindex(this->origin); + } + + blitz::Array vab_coefficient() + { + throw std::logic_error("absorber not yet implemented in 1d"); + } + + blitz::Array vab_relaxed_state(int d = 0) + { + throw std::logic_error("absorber not yet implemented in 1d"); + } + + blitz::Array sclr_array(const std::string& name, int n = 0) + { + return this->tmp.at(this->avail_tmp[name].first)[this->avail_tmp[name].second][n]( + this->grid_size[0] + ).reindex(this->origin); + } + }; + + template + class sharedmem : public sharedmem_common + { + using parent_t = sharedmem_common; + using parent_t::parent_t; // inheriting ctors + + public: + + blitz::Array advectee(int e = 0) + { + assert(this->n < n_tlev); + + return this->psi[e][ this->n ]( + this->grid_size[0], + this->grid_size[1] + ).reindex(this->origin); + } + + const blitz::Array advectee_global(int e = 0) + { +#if defined(USE_MPI) + if(this->distmem.size() > 1) + { +// TODO: move some of that to distmem... +// TODO: a lot of common code betwee 1,2 and 3 dimensions... + + // a vector of number of elements to be sent by each non-root process + std::vector sizes(this->distmem.size()); + std::iota(sizes.begin(), sizes.end(), 0); // fill with 1,2,3,... + for(auto &size : sizes) + { + size = this->slab(rng_t(0, this->distmem.grid_size[0]-1), size, this->distmem.size()).length() + * this->grid_size[1].length(); + } + // calc displacement + std::vector displ(sizes.size()); + std::partial_sum(sizes.begin(), sizes.end(), displ.begin()); + std::transform(displ.begin(), displ.end(), sizes.begin(), displ.begin(), std::minus()); // exclusive_scan is c++17 + // a vector that will store the received data, relevant only on process rank=0 + std::vector out_values(this->distmem.grid_size[0] * this->grid_size[1].length()); + // create an array that will store advectee to be sent in a contiguous memory block + std::vector in_values_vec(advectee(e).size()); + std::copy(advectee(e).begin(), advectee(e).end(), in_values_vec.begin()); + + // gather the data from all processes on rank=0 + boost::mpi::gatherv(this->distmem.mpicom, in_values_vec, out_values.data(), sizes, displ, 0); + // send the result to other processes + boost::mpi::broadcast(this->distmem.mpicom, out_values, 0); + + blitz::Array res(out_values.data(), blitz::shape( + this->distmem.grid_size[0], this->grid_size[1].length()), + blitz::duplicateData); + return res; + } + else +#endif + return advectee(e); + } + + blitz::Array advector(int d = 0) + { + using namespace arakawa_c; + assert(d == 0 || d== 1); + // returning just the domain interior, i.e. without halos + // reindexed to make it more intuitive when working with index placeholders + auto orgn = decltype(this->origin)({ + this->origin[0] - 1, + this->origin[1] + }); + + switch (d) + { + case 0: + return this->GC[d]( + this->distmem_ext(this->grid_size[0]^(-1)^h), + this->grid_size[1] + ).reindex(orgn); + case 1: + return this->GC[d]( + this->distmem_ext(this->grid_size[0]), + this->grid_size[1]^(-1)^h + ).reindex(orgn); + default: assert(false); throw; + } + } + + blitz::Array g_factor() + { + // a sanity check + if (this->G.get() == nullptr) + throw std::runtime_error("g_factor() called with nug option unset?"); + + // the same logic as in advectee() - see above + return (*this->G)( + this->grid_size[0], + this->grid_size[1] + ).reindex(this->origin); + } + + blitz::Array vab_coefficient() + { + // a sanity check + if (this->vab_coeff.get() == nullptr) + throw std::runtime_error("vab_coeff() called with option vip_vab unset?"); + + // the same logic as in advectee() - see above + return (*this->vab_coeff)( + this->grid_size[0], + this->grid_size[1] + ).reindex(this->origin); + } + + blitz::Array vab_relaxed_state(int d = 0) + { + assert(d == 0 || d== 1); + // a sanity check + if (this->vab_coeff.get() == nullptr) + throw std::runtime_error("vab_relaxed_state() called with option vip_vab unset?"); + // the same logic as in advectee() - see above + return this->vab_relax[d]( + this->grid_size[0], + this->grid_size[1] + ).reindex(this->origin); + } + + blitz::Array sclr_array(const std::string& name, int n = 0) + { + return this->tmp.at(this->avail_tmp[name].first)[this->avail_tmp[name].second][n]( + this->grid_size[0], + this->grid_size[1] + ).reindex(this->origin); + } + }; + + template + class sharedmem : public sharedmem_common + { + using parent_t = sharedmem_common; + using arr_t = typename parent_t::arr_t; + using parent_t::parent_t; // inheriting ctors + + public: + + virtual arr_t *never_delete(arr_t *arg) override + { + arr_t *ret = new arr_t(arg->dataFirst(), arg->shape(), blitz::neverDeleteData, blitz::GeneralArrayStorage<3>(arg->ordering(), {true, true, true})); + ret->reindexSelf(arg->base()); + return ret; + } + + blitz::Array advectee(int e = 0) + { + assert(this->n < n_tlev); + + return this->psi[e][ this->n ]( + this->grid_size[0], + this->grid_size[1], + this->grid_size[2] + ).reindex(this->origin); + } + + const blitz::Array advectee_global(int e = 0) + { +#if defined(USE_MPI) + if(this->distmem.size() > 1) + { +// TODO: move some of that to distmem... +// TODO: a lot of common code betwee 1,2 and 3 dimensions... + + // a vector of number of elements to be sent by each non-root process + std::vector sizes(this->distmem.size()); + std::iota(sizes.begin(), sizes.end(), 0); // fill with 1,2,3,... + for(auto &size : sizes) + { + size = this->slab(rng_t(0, this->distmem.grid_size[0]-1), size, this->distmem.size()).length() + * this->grid_size[1].length() * this->grid_size[2].length(); + } + // calc displacement + std::vector displ(sizes.size()); + std::partial_sum(sizes.begin(), sizes.end(), displ.begin()); + std::transform(displ.begin(), displ.end(), sizes.begin(), displ.begin(), std::minus()); // exclusive_scan is c++17 + // a vector that will store the received data, relevant only on process rank=0 + std::vector out_values(this->distmem.grid_size[0] * this->grid_size[1].length() * this->grid_size[2].length()); + // create an array that will store advectee to be sent in a contiguous memory block using the (default) kji storage order + // NOTE: libmpdata++ 3d blitz arrays, like advectee, are in the kij order + blitz::Array in_values_arr(advectee(e).shape()); + in_values_arr = advectee(e); + // wrap in_values_arr in a std::vector + std::vector in_values_vec(in_values_arr.begin(), in_values_arr.end()); + + // gather the data from all processes on rank=0 + boost::mpi::gatherv(this->distmem.mpicom, in_values_vec, out_values.data(), sizes, displ, 0); + // send the result to other processes + boost::mpi::broadcast(this->distmem.mpicom, out_values, 0); + + blitz::Array res(out_values.data(), blitz::shape( + this->distmem.grid_size[0], this->grid_size[1].length(), this->grid_size[2].length()), + blitz::duplicateData + ); + return res; + } + else +#endif + return advectee(e); + } + + blitz::Array advector(int d = 0) + { + using namespace arakawa_c; + assert(d == 0 || d == 1 || d == 2); + // returning just the domain interior, i.e. without halos + // reindexed to make it more intuitive when working with index placeholders + auto orgn = decltype(this->origin)({ + this->origin[0] - 1, + this->origin[1], + this->origin[2] + }); + + switch (d) + { + case 0: + return this->GC[d]( + this->distmem_ext(this->grid_size[0]^(-1)^h), + this->grid_size[1], + this->grid_size[2] + ).reindex(orgn); + case 1: + return this->GC[d]( + this->distmem_ext(this->grid_size[0]), + this->grid_size[1]^(-1)^h, + this->grid_size[2] + ).reindex(orgn); + case 2: + return this->GC[d]( + this->distmem_ext(this->grid_size[0]), + this->grid_size[1], + this->grid_size[2]^(-1)^h + ).reindex(orgn); + default: assert(false); throw; + } + } + + blitz::Array g_factor() + { + // a sanity check + if (this->G.get() == nullptr) + throw std::runtime_error("g_factor() called with nug option unset?"); + + // the same logic as in advectee() - see above + return (*this->G)( + this->grid_size[0], + this->grid_size[1], + this->grid_size[2] + ).reindex(this->origin); + } + + blitz::Array vab_coefficient() + { + // a sanity check + if (this->vab_coeff.get() == nullptr) + throw std::runtime_error("vab_coeff() called with option vip_vab unset?"); + + // the same logic as in advectee() - see above + return (*this->vab_coeff)( + this->grid_size[0], + this->grid_size[1], + this->grid_size[2] + ).reindex(this->origin); + } + + blitz::Array vab_relaxed_state(int d = 0) + { + assert(d == 0 || d == 1 || d == 2); + // a sanity check + if (this->vab_coeff.get() == nullptr) + throw std::runtime_error("vab_relaxed_state() called with option vip_vab unset?"); + // the same logic as in advectee() - see above + return this->vab_relax[d]( + this->grid_size[0], + this->grid_size[1], + this->grid_size[2] + ).reindex(this->origin); + } + + blitz::Array sclr_array(const std::string& name, int n = 0) + { + return this->tmp.at(this->avail_tmp[name].first)[this->avail_tmp[name].second][n]( + this->grid_size[0], + this->grid_size[1], + this->grid_size[2] + ).reindex(this->origin); + } + }; + */ + } // namespace detail + } // namespace concurr +} // namespace libmpdataxx diff --git a/libmpdata++/concurr/openmp.hpp b/libmpdata++/concurr/openmp.hpp index d33cda98..81b17bca 100644 --- a/libmpdata++/concurr/openmp.hpp +++ b/libmpdata++/concurr/openmp.hpp @@ -58,7 +58,8 @@ namespace libmpdataxx } // ctors - mem_t(const std::array &grid_size) : parent_t::mem_t(grid_size, size(grid_size[0])) {}; + // TODO: n_ref only for a run with grid refinement? + mem_t(const std::array &grid_size, const int n_ref=1) : parent_t::mem_t(grid_size, size(grid_size[0]), n_ref) {}; }; void solve(typename parent_t::advance_arg_t nt) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra.hpp new file mode 100644 index 00000000..5d0b89d1 --- /dev/null +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra.hpp @@ -0,0 +1,68 @@ +/** + * @file + * @copyright University of Warsaw + * @section LICENSE + * GPLv3+ (see the COPYING file or http://www.gnu.org/licenses/) + * + */ + +#pragma once + +#include +#include + +namespace libmpdataxx +{ + namespace solvers + { + namespace detail + { + template + class mpdata_rhs_vip_prs_sgs_fra : public mpdata_rhs_vip_prs_sgs + { + using parent_t = mpdata_rhs_vip_prs_sgs; + + public: + + using real_t = typename ct_params_t::real_t; + + protected: + +// const int n_fra; // number of fields with fractal reconstruction +// constexpr int n_rec_cell = pow(2, ct_params::n_fra_rec) ; // number of reconstructed cells in each direction + + // idx_t ijkm; + + public: + + struct rt_params_t : parent_t::rt_params_t + { + int n_fra_iter = 0; // number of iterations of fractal reconstruction + }; + + // ctor + mpdata_rhs_vip_prs_sgs_fra( + typename parent_t::ctor_args_t args, + const rt_params_t &p + ) : + parent_t(args, p), + mem->psi_ref(args.mem->tmp[__FILE__][0]), + mem->GC_ref(args.mem->tmp[__FILE__][1]) + { + std::cerr << "psi_ref: " << mem->psi_ref << std::endl; + std::cerr << "GC_ref: " << mem->psi_ref << std::endl; + // for (int d = 0; d < ct_params_t::n_dims; ++d) + } + + static void alloc( + typename parent_t::mem_t *mem, + const int &n_iters + ) { + parent_t::alloc(mem, n_iters); + parent_t::alloc_tmp_sclr(mem, __FILE__, ct_params_t::n_eqns, "", false, mem->grid_size_ref); // rec_psi + parent_t::alloc_tmp_vctr(mem, __FILE__, false, mem->grid_size_ref); // rec_GC + } + }; + } // namespace detail + } // namespace solvers +} // namespace libmpdataxx diff --git a/libmpdata++/solvers/detail/solver_3d.hpp b/libmpdata++/solvers/detail/solver_3d.hpp index ea62aac9..1fae88d5 100644 --- a/libmpdata++/solvers/detail/solver_3d.hpp +++ b/libmpdata++/solvers/detail/solver_3d.hpp @@ -488,18 +488,19 @@ namespace libmpdataxx const char * __file__, const int n_arr, const std::vector> &stgr, - bool srfc = false // allocate only surface data + bool srfc = false, // allocate only surface data + const std::array grid_size = mem->grid_size // custom grid size ) { mem->tmp[__file__].push_back(new arrvec_t()); for (int n = 0; n < n_arr; ++n) { mem->tmp[__file__].back().push_back(mem->old(new typename parent_t::arr_t( - stgr[n][0] ? parent_t::rng_vctr(mem->grid_size[0]) : parent_t::rng_sclr(mem->grid_size[0]), - stgr[n][1] ? parent_t::rng_vctr(mem->grid_size[1]) : parent_t::rng_sclr(mem->grid_size[1]), + stgr[n][0] ? parent_t::rng_vctr(grid_size[0]) : parent_t::rng_sclr(grid_size[0]), + stgr[n][1] ? parent_t::rng_vctr(grid_size[1]) : parent_t::rng_sclr(grid_size[1]), srfc ? rng_t(0, 0) : - stgr[n][2] ? parent_t::rng_vctr(mem->grid_size[2]) : - parent_t::rng_sclr(mem->grid_size[2]), + stgr[n][2] ? parent_t::rng_vctr(grid_size[2]) : + parent_t::rng_sclr(grid_size[2]), arr3D_storage ))); } @@ -508,10 +509,11 @@ namespace libmpdataxx // helper method to allocate a temporary space composed of vector-component arrays static void alloc_tmp_vctr( typename parent_t::mem_t *mem, - const char * __file__ + const char * __file__, + const std::array grid_size = mem->grid_size // custom grid size ) { - alloc_tmp_stgr(mem, __file__, 3, {{true, false, false}, {false, true, false}, {false, false, true}}); + alloc_tmp_stgr(mem, __file__, 3, {{true, false, false}, {false, true, false}, {false, false, true}, false, grid_size}); } // helper method to allocate n_arr scalar temporary arrays @@ -519,7 +521,8 @@ namespace libmpdataxx typename parent_t::mem_t *mem, const char * __file__, const int n_arr, std::string name = "", - bool srfc = false // allocate only surface data + bool srfc = false, // allocate only surface data + const std::array grid_size = mem->grid_size // custom grid size ) { mem->tmp[__file__].push_back(new arrvec_t()); @@ -528,9 +531,9 @@ namespace libmpdataxx for (int n = 0; n < n_arr; ++n) mem->tmp[__file__].back().push_back(mem->old(new typename parent_t::arr_t( - parent_t::rng_sclr(mem->grid_size[0]), - parent_t::rng_sclr(mem->grid_size[1]), - srfc ? rng_t(0, 0) : parent_t::rng_sclr(mem->grid_size[2]), + parent_t::rng_sclr(grid_size[0]), + parent_t::rng_sclr(grid_size[1]), + srfc ? rng_t(0, 0) : parent_t::rng_sclr(grid_size[2]), arr3D_storage ))); } diff --git a/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp b/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp index 00e9da0c..24057b9e 100644 --- a/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp +++ b/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp @@ -22,7 +22,7 @@ namespace libmpdataxx template class mpdata_rhs_vip_prs_sgs_fra< ct_params_t, minhalo, - typename std::enable_if<(int)ct_params_t::fra_rec == 0>::type // fra_rec is the number of fractal reconstructions + typename std::enable_if<(int)ct_params_t::n_fra_rec == 0>::type > : public mpdata_rhs_vip_prs_sgs { using parent_t = mpdata_rhs_vip_prs_sgs; @@ -32,7 +32,7 @@ namespace libmpdataxx template class mpdata_rhs_vip_prs_sgs_fra< ct_params_t, minhalo, - typename std::enable_if<(int)ct_params_t::fra_rec > 0>::type + typename std::enable_if<(int)ct_params_t::n_fra_rec > 0>::type > : public detail::mpdata_rhs_vip_prs_sgs_fra { using parent_t = detail::mpdata_rhs_vip_prs_sgs_fra; From c1b497d6fcaf40bf0edba862b17178b692c91738 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 2 Mar 2022 15:30:22 +0100 Subject: [PATCH 003/129] working allocation of refined arrays --- libmpdata++/concurr/detail/concurr_common.hpp | 6 +- libmpdata++/concurr/detail/sharedmem.hpp | 4 +- .../concurr/detail/sharedmem_refined.hpp | 430 +----------------- libmpdata++/concurr/openmp.hpp | 2 +- .../solvers/detail/boussinesq_common.hpp | 6 +- .../detail/mpdata_rhs_vip_prs_sgs_fra.hpp | 21 +- libmpdata++/solvers/detail/solver_3d.hpp | 43 +- libmpdata++/solvers/detail/solver_common.hpp | 4 +- .../solvers/mpdata_rhs_vip_prs_sgs_fra.hpp | 46 +- 9 files changed, 98 insertions(+), 464 deletions(-) diff --git a/libmpdata++/concurr/detail/concurr_common.hpp b/libmpdata++/concurr/detail/concurr_common.hpp index 4e224785..b722f3b5 100644 --- a/libmpdata++/concurr/detail/concurr_common.hpp +++ b/libmpdata++/concurr/detail/concurr_common.hpp @@ -15,6 +15,7 @@ #include #include +#include #include #include @@ -150,11 +151,12 @@ namespace libmpdataxx protected: // (cannot be nested due to templates) - typedef sharedmem< + typedef sharedmem_refined_common< + //typedef sharedmem< typename solver_t::real_t, solver_t::n_dims, solver_t::n_tlev - if solver family is _fra use sharedmem_rec, else use sharedmem +// if solver family is _fra use sharedmem_rec, else use sharedmem > mem_t; // member fields diff --git a/libmpdata++/concurr/detail/sharedmem.hpp b/libmpdata++/concurr/detail/sharedmem.hpp index 653302d1..30cddbe8 100644 --- a/libmpdata++/concurr/detail/sharedmem.hpp +++ b/libmpdata++/concurr/detail/sharedmem.hpp @@ -579,9 +579,11 @@ namespace libmpdataxx class sharedmem : public sharedmem_common { using parent_t = sharedmem_common; - using arr_t = typename parent_t::arr_t; using parent_t::parent_t; // inheriting ctors + protected: + using arr_t = typename parent_t::arr_t; + public: virtual arr_t *never_delete(arr_t *arg) override diff --git a/libmpdata++/concurr/detail/sharedmem_refined.hpp b/libmpdata++/concurr/detail/sharedmem_refined.hpp index f75a150f..5af20ce6 100644 --- a/libmpdata++/concurr/detail/sharedmem_refined.hpp +++ b/libmpdata++/concurr/detail/sharedmem_refined.hpp @@ -8,7 +8,7 @@ #pragma once -#include +#include "sharedmem.hpp" namespace libmpdataxx { @@ -21,10 +21,10 @@ namespace libmpdataxx int n_dims, int n_tlev > - class sharedmem_refined_common :: sharedmem + class sharedmem_refined_common : public sharedmem { using parent_t = sharedmem; - using arr_t = typename parent_t::parent_t::arr_t; + using arr_t = typename parent_t::arr_t; protected: @@ -43,13 +43,15 @@ namespace libmpdataxx { for (int d = 0; d < n_dims; ++d) { - grid_size_ref[d] = refine_grid_size(this->grid_size, n_ref) + grid_size_ref[d] = refine_grid_size(this->grid_size[d], n_ref); origin_ref[d] = grid_size_ref[d].first(); } } - virtual arr_t advectee_ref(int e = 0) = 0; - virtual const arr_t advectee_global_ref(int e = 0) = 0; + // virtual arr_t advectee_ref(int e = 0) = 0; + // virtual const arr_t advectee_global_ref(int e = 0) = 0; + // using parent_t::advectee; +// using parent_t::sclr_array; public: static rng_t refine_grid_size( @@ -62,422 +64,6 @@ namespace libmpdataxx ); } }; - - - - - - - - -/* - - - - - template - class sharedmem - {}; - - template - class sharedmem : public sharedmem_common - { - using parent_t = sharedmem_common; - using parent_t::parent_t; // inheriting ctors - - public: - - // accessor methods - blitz::Array advectee(int e = 0) - { - assert(this->n < n_tlev); - - // returning just the domain interior, i.e. without halos - // reindexing so that element 0 is at 0 - return this->psi[e][ this->n ]( - this->grid_size[0] - ).reindex(this->origin); - } - - const blitz::Array advectee_global(int e = 0) - { -#if defined(USE_MPI) - if(this->distmem.size() > 1) - { -// TODO: move some of that to distmem... -// TODO: a lot of common code betwee 1,2 and 3 dimensions... - - // a vector of number of elements to be sent by each non-root process - std::vector sizes(this->distmem.size()); - std::iota(sizes.begin(), sizes.end(), 0); // fill with 1,2,3,... - for(auto &size : sizes) { size = this->slab(rng_t(0, this->distmem.grid_size[0]-1), size, this->distmem.size()).length();} - // calc displacement - std::vector displ(sizes.size()); - std::partial_sum(sizes.begin(), sizes.end(), displ.begin()); - std::transform(displ.begin(), displ.end(), sizes.begin(), displ.begin(), std::minus()); // exclusive_scan is c++17 - // a vector that will store the received data, relevant only on process rank=0 - std::vector out_values(this->distmem.grid_size[0]); - // create an array that will store advectee to be sent in a contiguous memory block - std::vector in_values_vec(advectee(e).size()); - std::copy(advectee(e).begin(), advectee(e).end(), in_values_vec.begin()); - - // gather the data from all processes on rank=0 - boost::mpi::gatherv(this->distmem.mpicom, in_values_vec, out_values.data(), sizes, displ, 0); - // send the result to other processes - boost::mpi::broadcast(this->distmem.mpicom, out_values, 0); - - blitz::Array res(out_values.data(), blitz::shape(this->distmem.grid_size[0]), blitz::duplicateData); - return res; - } - else -#endif - return advectee(e); - } - - blitz::Array advector(int d = 0) - { - using namespace arakawa_c; - assert(d == 0); - // returning just the domain interior, i.e. without halos - // reindexed to make it more intuitive when working with index placeholders - // (i.e. border between cell 0 and cell 1 is indexed with 0) - auto orgn = decltype(this->origin)({ - this->origin[0] - 1 - }); - - return this->GC[d]( - this->distmem_ext(this->grid_size[0]^(-1)^h) - ).reindex( - this->distmem.rank() > 0 - ? decltype(this->origin)({this->origin[0] - 1}) - : orgn - ); - } - - blitz::Array g_factor() - { - // a sanity check - if (this->G.get() == nullptr) - throw std::runtime_error("g_factor() called with nug option unset?"); - - // the same logic as in advectee() - see above - return (*this->G)( - this->grid_size[0] - ).reindex(this->origin); - } - - blitz::Array vab_coefficient() - { - throw std::logic_error("absorber not yet implemented in 1d"); - } - - blitz::Array vab_relaxed_state(int d = 0) - { - throw std::logic_error("absorber not yet implemented in 1d"); - } - - blitz::Array sclr_array(const std::string& name, int n = 0) - { - return this->tmp.at(this->avail_tmp[name].first)[this->avail_tmp[name].second][n]( - this->grid_size[0] - ).reindex(this->origin); - } - }; - - template - class sharedmem : public sharedmem_common - { - using parent_t = sharedmem_common; - using parent_t::parent_t; // inheriting ctors - - public: - - blitz::Array advectee(int e = 0) - { - assert(this->n < n_tlev); - - return this->psi[e][ this->n ]( - this->grid_size[0], - this->grid_size[1] - ).reindex(this->origin); - } - - const blitz::Array advectee_global(int e = 0) - { -#if defined(USE_MPI) - if(this->distmem.size() > 1) - { -// TODO: move some of that to distmem... -// TODO: a lot of common code betwee 1,2 and 3 dimensions... - - // a vector of number of elements to be sent by each non-root process - std::vector sizes(this->distmem.size()); - std::iota(sizes.begin(), sizes.end(), 0); // fill with 1,2,3,... - for(auto &size : sizes) - { - size = this->slab(rng_t(0, this->distmem.grid_size[0]-1), size, this->distmem.size()).length() - * this->grid_size[1].length(); - } - // calc displacement - std::vector displ(sizes.size()); - std::partial_sum(sizes.begin(), sizes.end(), displ.begin()); - std::transform(displ.begin(), displ.end(), sizes.begin(), displ.begin(), std::minus()); // exclusive_scan is c++17 - // a vector that will store the received data, relevant only on process rank=0 - std::vector out_values(this->distmem.grid_size[0] * this->grid_size[1].length()); - // create an array that will store advectee to be sent in a contiguous memory block - std::vector in_values_vec(advectee(e).size()); - std::copy(advectee(e).begin(), advectee(e).end(), in_values_vec.begin()); - - // gather the data from all processes on rank=0 - boost::mpi::gatherv(this->distmem.mpicom, in_values_vec, out_values.data(), sizes, displ, 0); - // send the result to other processes - boost::mpi::broadcast(this->distmem.mpicom, out_values, 0); - - blitz::Array res(out_values.data(), blitz::shape( - this->distmem.grid_size[0], this->grid_size[1].length()), - blitz::duplicateData); - return res; - } - else -#endif - return advectee(e); - } - - blitz::Array advector(int d = 0) - { - using namespace arakawa_c; - assert(d == 0 || d== 1); - // returning just the domain interior, i.e. without halos - // reindexed to make it more intuitive when working with index placeholders - auto orgn = decltype(this->origin)({ - this->origin[0] - 1, - this->origin[1] - }); - - switch (d) - { - case 0: - return this->GC[d]( - this->distmem_ext(this->grid_size[0]^(-1)^h), - this->grid_size[1] - ).reindex(orgn); - case 1: - return this->GC[d]( - this->distmem_ext(this->grid_size[0]), - this->grid_size[1]^(-1)^h - ).reindex(orgn); - default: assert(false); throw; - } - } - - blitz::Array g_factor() - { - // a sanity check - if (this->G.get() == nullptr) - throw std::runtime_error("g_factor() called with nug option unset?"); - - // the same logic as in advectee() - see above - return (*this->G)( - this->grid_size[0], - this->grid_size[1] - ).reindex(this->origin); - } - - blitz::Array vab_coefficient() - { - // a sanity check - if (this->vab_coeff.get() == nullptr) - throw std::runtime_error("vab_coeff() called with option vip_vab unset?"); - - // the same logic as in advectee() - see above - return (*this->vab_coeff)( - this->grid_size[0], - this->grid_size[1] - ).reindex(this->origin); - } - - blitz::Array vab_relaxed_state(int d = 0) - { - assert(d == 0 || d== 1); - // a sanity check - if (this->vab_coeff.get() == nullptr) - throw std::runtime_error("vab_relaxed_state() called with option vip_vab unset?"); - // the same logic as in advectee() - see above - return this->vab_relax[d]( - this->grid_size[0], - this->grid_size[1] - ).reindex(this->origin); - } - - blitz::Array sclr_array(const std::string& name, int n = 0) - { - return this->tmp.at(this->avail_tmp[name].first)[this->avail_tmp[name].second][n]( - this->grid_size[0], - this->grid_size[1] - ).reindex(this->origin); - } - }; - - template - class sharedmem : public sharedmem_common - { - using parent_t = sharedmem_common; - using arr_t = typename parent_t::arr_t; - using parent_t::parent_t; // inheriting ctors - - public: - - virtual arr_t *never_delete(arr_t *arg) override - { - arr_t *ret = new arr_t(arg->dataFirst(), arg->shape(), blitz::neverDeleteData, blitz::GeneralArrayStorage<3>(arg->ordering(), {true, true, true})); - ret->reindexSelf(arg->base()); - return ret; - } - - blitz::Array advectee(int e = 0) - { - assert(this->n < n_tlev); - - return this->psi[e][ this->n ]( - this->grid_size[0], - this->grid_size[1], - this->grid_size[2] - ).reindex(this->origin); - } - - const blitz::Array advectee_global(int e = 0) - { -#if defined(USE_MPI) - if(this->distmem.size() > 1) - { -// TODO: move some of that to distmem... -// TODO: a lot of common code betwee 1,2 and 3 dimensions... - - // a vector of number of elements to be sent by each non-root process - std::vector sizes(this->distmem.size()); - std::iota(sizes.begin(), sizes.end(), 0); // fill with 1,2,3,... - for(auto &size : sizes) - { - size = this->slab(rng_t(0, this->distmem.grid_size[0]-1), size, this->distmem.size()).length() - * this->grid_size[1].length() * this->grid_size[2].length(); - } - // calc displacement - std::vector displ(sizes.size()); - std::partial_sum(sizes.begin(), sizes.end(), displ.begin()); - std::transform(displ.begin(), displ.end(), sizes.begin(), displ.begin(), std::minus()); // exclusive_scan is c++17 - // a vector that will store the received data, relevant only on process rank=0 - std::vector out_values(this->distmem.grid_size[0] * this->grid_size[1].length() * this->grid_size[2].length()); - // create an array that will store advectee to be sent in a contiguous memory block using the (default) kji storage order - // NOTE: libmpdata++ 3d blitz arrays, like advectee, are in the kij order - blitz::Array in_values_arr(advectee(e).shape()); - in_values_arr = advectee(e); - // wrap in_values_arr in a std::vector - std::vector in_values_vec(in_values_arr.begin(), in_values_arr.end()); - - // gather the data from all processes on rank=0 - boost::mpi::gatherv(this->distmem.mpicom, in_values_vec, out_values.data(), sizes, displ, 0); - // send the result to other processes - boost::mpi::broadcast(this->distmem.mpicom, out_values, 0); - - blitz::Array res(out_values.data(), blitz::shape( - this->distmem.grid_size[0], this->grid_size[1].length(), this->grid_size[2].length()), - blitz::duplicateData - ); - return res; - } - else -#endif - return advectee(e); - } - - blitz::Array advector(int d = 0) - { - using namespace arakawa_c; - assert(d == 0 || d == 1 || d == 2); - // returning just the domain interior, i.e. without halos - // reindexed to make it more intuitive when working with index placeholders - auto orgn = decltype(this->origin)({ - this->origin[0] - 1, - this->origin[1], - this->origin[2] - }); - - switch (d) - { - case 0: - return this->GC[d]( - this->distmem_ext(this->grid_size[0]^(-1)^h), - this->grid_size[1], - this->grid_size[2] - ).reindex(orgn); - case 1: - return this->GC[d]( - this->distmem_ext(this->grid_size[0]), - this->grid_size[1]^(-1)^h, - this->grid_size[2] - ).reindex(orgn); - case 2: - return this->GC[d]( - this->distmem_ext(this->grid_size[0]), - this->grid_size[1], - this->grid_size[2]^(-1)^h - ).reindex(orgn); - default: assert(false); throw; - } - } - - blitz::Array g_factor() - { - // a sanity check - if (this->G.get() == nullptr) - throw std::runtime_error("g_factor() called with nug option unset?"); - - // the same logic as in advectee() - see above - return (*this->G)( - this->grid_size[0], - this->grid_size[1], - this->grid_size[2] - ).reindex(this->origin); - } - - blitz::Array vab_coefficient() - { - // a sanity check - if (this->vab_coeff.get() == nullptr) - throw std::runtime_error("vab_coeff() called with option vip_vab unset?"); - - // the same logic as in advectee() - see above - return (*this->vab_coeff)( - this->grid_size[0], - this->grid_size[1], - this->grid_size[2] - ).reindex(this->origin); - } - - blitz::Array vab_relaxed_state(int d = 0) - { - assert(d == 0 || d == 1 || d == 2); - // a sanity check - if (this->vab_coeff.get() == nullptr) - throw std::runtime_error("vab_relaxed_state() called with option vip_vab unset?"); - // the same logic as in advectee() - see above - return this->vab_relax[d]( - this->grid_size[0], - this->grid_size[1], - this->grid_size[2] - ).reindex(this->origin); - } - - blitz::Array sclr_array(const std::string& name, int n = 0) - { - return this->tmp.at(this->avail_tmp[name].first)[this->avail_tmp[name].second][n]( - this->grid_size[0], - this->grid_size[1], - this->grid_size[2] - ).reindex(this->origin); - } - }; - */ } // namespace detail } // namespace concurr } // namespace libmpdataxx diff --git a/libmpdata++/concurr/openmp.hpp b/libmpdata++/concurr/openmp.hpp index 81b17bca..a27aad4d 100644 --- a/libmpdata++/concurr/openmp.hpp +++ b/libmpdata++/concurr/openmp.hpp @@ -78,7 +78,7 @@ namespace libmpdataxx // ctor openmp(const typename solver_t::rt_params_t &p) : - parent_t(p, new mem_t(p.grid_size), mem_t::size(p.grid_size[solver_t::n_dims < 3 ? 0 : 1])) // note 3D domain decomposition in y direction + parent_t(p, new mem_t(p.grid_size, 2), mem_t::size(p.grid_size[solver_t::n_dims < 3 ? 0 : 1])) // note 3D domain decomposition in y direction {} }; diff --git a/libmpdata++/solvers/detail/boussinesq_common.hpp b/libmpdata++/solvers/detail/boussinesq_common.hpp index c7c1a6c0..e88203fa 100644 --- a/libmpdata++/solvers/detail/boussinesq_common.hpp +++ b/libmpdata++/solvers/detail/boussinesq_common.hpp @@ -5,7 +5,7 @@ * GPLv3+ (see the COPYING file or http://www.gnu.org/licenses/) */ #pragma once -#include +#include namespace libmpdataxx { @@ -14,9 +14,9 @@ namespace libmpdataxx namespace detail { template - class boussinesq_common : public libmpdataxx::solvers::mpdata_rhs_vip_prs_sgs + class boussinesq_common : public libmpdataxx::solvers::mpdata_rhs_vip_prs_sgs_fra { - using parent_t = libmpdataxx::solvers::mpdata_rhs_vip_prs_sgs; + using parent_t = libmpdataxx::solvers::mpdata_rhs_vip_prs_sgs_fra; public: using real_t = typename ct_params_t::real_t; diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra.hpp index 5d0b89d1..70012e4a 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra.hpp @@ -8,8 +8,9 @@ #pragma once -#include -#include +#include +//#include +//#include namespace libmpdataxx { @@ -45,12 +46,8 @@ namespace libmpdataxx typename parent_t::ctor_args_t args, const rt_params_t &p ) : - parent_t(args, p), - mem->psi_ref(args.mem->tmp[__FILE__][0]), - mem->GC_ref(args.mem->tmp[__FILE__][1]) + parent_t(args, p) { - std::cerr << "psi_ref: " << mem->psi_ref << std::endl; - std::cerr << "GC_ref: " << mem->psi_ref << std::endl; // for (int d = 0; d < ct_params_t::n_dims; ++d) } @@ -59,8 +56,14 @@ namespace libmpdataxx const int &n_iters ) { parent_t::alloc(mem, n_iters); - parent_t::alloc_tmp_sclr(mem, __FILE__, ct_params_t::n_eqns, "", false, mem->grid_size_ref); // rec_psi - parent_t::alloc_tmp_vctr(mem, __FILE__, false, mem->grid_size_ref); // rec_GC + parent_t::alloc_tmp_sclr(mem, __FILE__, ct_params_t::n_eqns, mem->grid_size_ref, "", false); // rec_psi + parent_t::alloc_tmp_vctr(mem, __FILE__, mem->grid_size_ref); // rec_GC + + mem->psi_ref = mem->tmp[__FILE__][0]; + mem->GC_ref = mem->tmp[__FILE__][1]; + + std::cerr << "psi_ref: " << mem->psi_ref[0] << std::endl; + std::cerr << "GC_ref: " << mem->GC_ref[0] << std::endl; } }; } // namespace detail diff --git a/libmpdata++/solvers/detail/solver_3d.hpp b/libmpdata++/solvers/detail/solver_3d.hpp index 1fae88d5..6d13a34c 100644 --- a/libmpdata++/solvers/detail/solver_3d.hpp +++ b/libmpdata++/solvers/detail/solver_3d.hpp @@ -488,8 +488,8 @@ namespace libmpdataxx const char * __file__, const int n_arr, const std::vector> &stgr, - bool srfc = false, // allocate only surface data - const std::array grid_size = mem->grid_size // custom grid size + const std::array grid_size, + bool srfc = false // allocate only surface data ) { mem->tmp[__file__].push_back(new arrvec_t()); @@ -506,23 +506,43 @@ namespace libmpdataxx } } + // version with default grid size + static void alloc_tmp_stgr( + typename parent_t::mem_t *mem, + const char * __file__, + const int n_arr, + const std::vector> &stgr, + bool srfc = false // allocate only surface data + ) + { + alloc_tmp_stgr(mem, __file__, n_arr, stgr, mem->grid_size, srfc); + } + // helper method to allocate a temporary space composed of vector-component arrays static void alloc_tmp_vctr( typename parent_t::mem_t *mem, const char * __file__, - const std::array grid_size = mem->grid_size // custom grid size + const std::array grid_size ) { - alloc_tmp_stgr(mem, __file__, 3, {{true, false, false}, {false, true, false}, {false, false, true}, false, grid_size}); + alloc_tmp_stgr(mem, __file__, 3, {{true, false, false}, {false, true, false}, {false, false, true}}, grid_size, false); + } + + static void alloc_tmp_vctr( + typename parent_t::mem_t *mem, + const char * __file__ + ) + { + alloc_tmp_stgr(mem, __file__, 3, {{true, false, false}, {false, true, false}, {false, false, true}}, false); } // helper method to allocate n_arr scalar temporary arrays static void alloc_tmp_sclr( typename parent_t::mem_t *mem, const char * __file__, const int n_arr, + const std::array grid_size, std::string name = "", - bool srfc = false, // allocate only surface data - const std::array grid_size = mem->grid_size // custom grid size + bool srfc = false // allocate only surface data ) { mem->tmp[__file__].push_back(new arrvec_t()); @@ -537,6 +557,17 @@ namespace libmpdataxx arr3D_storage ))); } + + // with default grid_size + static void alloc_tmp_sclr( + typename parent_t::mem_t *mem, + const char * __file__, const int n_arr, + std::string name = "", + bool srfc = false + ) + { + alloc_tmp_sclr(mem, __file__, n_arr, mem->grid_size, name, srfc); + } }; } // namespace detail } // namespace solvers diff --git a/libmpdata++/solvers/detail/solver_common.hpp b/libmpdata++/solvers/detail/solver_common.hpp index 5aab5f07..0e072960 100644 --- a/libmpdata++/solvers/detail/solver_common.hpp +++ b/libmpdata++/solvers/detail/solver_common.hpp @@ -8,7 +8,7 @@ #include #include -#include +#include #include @@ -72,7 +72,7 @@ namespace libmpdataxx real_t time = 0; std::vector n; - typedef concurr::detail::sharedmem mem_t; + typedef concurr::detail::sharedmem_refined_common mem_t; mem_t *mem; // helper methods invoked by solve() diff --git a/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp b/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp index 24057b9e..961d5c0e 100644 --- a/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp +++ b/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp @@ -8,7 +8,7 @@ #pragma once -#include +#include namespace libmpdataxx { @@ -16,24 +16,34 @@ namespace libmpdataxx { struct mpdata_rhs_vip_prs_sgs_fra_family_tag {}; - template - class mpdata_rhs_vip_prs_sgs_fra; +// template +// class mpdata_rhs_vip_prs_sgs_fra; +// +// template +// class mpdata_rhs_vip_prs_sgs_fra< +// ct_params_t, minhalo, +// typename std::enable_if<(int)ct_params_t::n_fra_rec == 0>::type +// > : public mpdata_rhs_vip_prs_sgs +// { +// using parent_t = mpdata_rhs_vip_prs_sgs; +// using parent_t::parent_t; // inheriting constructors +// }; +// +// template +// class mpdata_rhs_vip_prs_sgs_fra< +// ct_params_t, minhalo, +// typename std::enable_if<(int)ct_params_t::n_fra_rec > 0>::type +// > : public detail::mpdata_rhs_vip_prs_sgs_fra +// { +// using parent_t = detail::mpdata_rhs_vip_prs_sgs_fra; +// using parent_t::parent_t; // inheriting constructors +// +// protected: +// using solver_family = mpdata_rhs_vip_prs_sgs_fra_family_tag; +// }; - template - class mpdata_rhs_vip_prs_sgs_fra< - ct_params_t, minhalo, - typename std::enable_if<(int)ct_params_t::n_fra_rec == 0>::type - > : public mpdata_rhs_vip_prs_sgs - { - using parent_t = mpdata_rhs_vip_prs_sgs; - using parent_t::parent_t; // inheriting constructors - }; - - template - class mpdata_rhs_vip_prs_sgs_fra< - ct_params_t, minhalo, - typename std::enable_if<(int)ct_params_t::n_fra_rec > 0>::type - > : public detail::mpdata_rhs_vip_prs_sgs_fra + template + class mpdata_rhs_vip_prs_sgs_fra : public detail::mpdata_rhs_vip_prs_sgs_fra { using parent_t = detail::mpdata_rhs_vip_prs_sgs_fra; using parent_t::parent_t; // inheriting constructors From 487b356a034052af21eb6a885d804fc0f1c774e7 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 2 Mar 2022 15:40:12 +0100 Subject: [PATCH 004/129] refined grid size dependant on the number of fractal reconstructions --- libmpdata++/concurr/openmp.hpp | 2 +- libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libmpdata++/concurr/openmp.hpp b/libmpdata++/concurr/openmp.hpp index a27aad4d..7c915a5b 100644 --- a/libmpdata++/concurr/openmp.hpp +++ b/libmpdata++/concurr/openmp.hpp @@ -78,7 +78,7 @@ namespace libmpdataxx // ctor openmp(const typename solver_t::rt_params_t &p) : - parent_t(p, new mem_t(p.grid_size, 2), mem_t::size(p.grid_size[solver_t::n_dims < 3 ? 0 : 1])) // note 3D domain decomposition in y direction + parent_t(p, new mem_t(p.grid_size, pow(2, p.n_fra_iter)), mem_t::size(p.grid_size[solver_t::n_dims < 3 ? 0 : 1])) // note 3D domain decomposition in y direction {} }; diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra.hpp index 70012e4a..ca0da71d 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra.hpp @@ -38,7 +38,7 @@ namespace libmpdataxx struct rt_params_t : parent_t::rt_params_t { - int n_fra_iter = 0; // number of iterations of fractal reconstruction + unsigned int n_fra_iter = 1; // number of iterations of fractal reconstruction }; // ctor From 1ad25f3fdde7b968a5a46c9a103db820b725f6b3 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 2 Mar 2022 15:51:48 +0100 Subject: [PATCH 005/129] fix refine_grid_size() --- libmpdata++/concurr/detail/sharedmem_refined.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmpdata++/concurr/detail/sharedmem_refined.hpp b/libmpdata++/concurr/detail/sharedmem_refined.hpp index 5af20ce6..6a22c86b 100644 --- a/libmpdata++/concurr/detail/sharedmem_refined.hpp +++ b/libmpdata++/concurr/detail/sharedmem_refined.hpp @@ -60,7 +60,7 @@ namespace libmpdataxx ) { return rng_t( grid_size.first() * n_ref, - grid_size.first() + (grid_size.last() - grid_size.first()) * n_ref + (grid_size.last() + 1) * n_ref - 1 ); } }; From f3885aa919c90b35f195512baadd8be6c13b3b9b Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 2 Mar 2022 17:53:38 +0100 Subject: [PATCH 006/129] mem_factory function --- libmpdata++-config.cmake | 6 ++--- .../concurr/detail/sharedmem_refined.hpp | 6 ++--- libmpdata++/concurr/openmp.hpp | 23 +++++++++++++++++-- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/libmpdata++-config.cmake b/libmpdata++-config.cmake index 32c4e83a..bca0af5e 100644 --- a/libmpdata++-config.cmake +++ b/libmpdata++-config.cmake @@ -32,7 +32,7 @@ set(libmpdataxx_INCLUDE_DIRS "${CMAKE_CURRENT_LIST_DIR}/../../include/") ############################################################################################ # debug mode compiler flags -set(libmpdataxx_CXX_FLAGS_DEBUG "${libmpdataxx_CXX_FLAGS_DEBUG} -std=c++14 -DBZ_DEBUG -g -Wno-enum-compare") #TODO: -Og if compiler supports it? +set(libmpdataxx_CXX_FLAGS_DEBUG "${libmpdataxx_CXX_FLAGS_DEBUG} -std=c++17 -DBZ_DEBUG -g -Wno-enum-compare") #TODO: -Og if compiler supports it? ############################################################################################ @@ -42,7 +42,7 @@ if( CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" ) - set(libmpdataxx_CXX_FLAGS_RELEASE "${libmpdataxx_CXX_FLAGS_RELEASE} -std=c++14 -DNDEBUG -Ofast -march=native -Wno-enum-compare") + set(libmpdataxx_CXX_FLAGS_RELEASE "${libmpdataxx_CXX_FLAGS_RELEASE} -std=c++17 -DNDEBUG -Ofast -march=native -Wno-enum-compare") # preventing Kahan summation from being optimised out if ( @@ -58,7 +58,7 @@ if( CMAKE_CXX_COMPILER_ID STREQUAL "Intel" ) # flags taken from -fast but without -static - set(libmpdataxx_CXX_FLAGS_RELEASE "${libmpdataxx_CXX_FLAGS_RELEASE} -std=gnu++14 -DNDEBUG -xHOST -O3 -ipo -no-prec-div -fp-model fast=2") + set(libmpdataxx_CXX_FLAGS_RELEASE "${libmpdataxx_CXX_FLAGS_RELEASE} -std=gnu++17 -DNDEBUG -xHOST -O3 -ipo -no-prec-div -fp-model fast=2") endif() diff --git a/libmpdata++/concurr/detail/sharedmem_refined.hpp b/libmpdata++/concurr/detail/sharedmem_refined.hpp index 6a22c86b..d8cd12b6 100644 --- a/libmpdata++/concurr/detail/sharedmem_refined.hpp +++ b/libmpdata++/concurr/detail/sharedmem_refined.hpp @@ -48,10 +48,8 @@ namespace libmpdataxx } } - // virtual arr_t advectee_ref(int e = 0) = 0; - // virtual const arr_t advectee_global_ref(int e = 0) = 0; - // using parent_t::advectee; -// using parent_t::sclr_array; + // virtual arr_t refinee(int e = 0) = 0; + // virtual const arr_t refinee_global_ref(int e = 0) = 0; public: static rng_t refine_grid_size( diff --git a/libmpdata++/concurr/openmp.hpp b/libmpdata++/concurr/openmp.hpp index 7c915a5b..82098d0a 100644 --- a/libmpdata++/concurr/openmp.hpp +++ b/libmpdata++/concurr/openmp.hpp @@ -9,6 +9,7 @@ #include #include +#include #ifdef _OPENMP # include @@ -18,6 +19,25 @@ namespace libmpdataxx { namespace concurr { + namespace detail + { + // helper function that detect if the solver uses fractal reconstruction + template< class, class = void > + struct slvr_with_frac_recn : std::false_type { }; + + template< class solver_t > + struct slvr_with_frac_recn> : std::true_type { }; + + template< class mem_t, class solver_t> + mem_t* mem_factory(const typename solver_t::rt_params_t &p) + { + if constexpr (slvr_with_frac_recn()) + return new mem_t(p.grid_size, pow(2, p.n_fra_iter)); + else + return new mem_t(p.grid_size); + } + }; + template < class solver_t, bcond::bcond_e bcxl, @@ -76,9 +96,8 @@ namespace libmpdataxx public: - // ctor openmp(const typename solver_t::rt_params_t &p) : - parent_t(p, new mem_t(p.grid_size, pow(2, p.n_fra_iter)), mem_t::size(p.grid_size[solver_t::n_dims < 3 ? 0 : 1])) // note 3D domain decomposition in y direction + parent_t(p, detail::mem_factory(p), mem_t::size(p.grid_size[solver_t::n_dims < 3 ? 0 : 1])) // note 3D domain decomposition in y direction {} }; From 47348d7ad3e1d05cafecb1ab93336133056acd0d Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 2 Mar 2022 18:19:32 +0100 Subject: [PATCH 007/129] consitional choice of shmem / shmem_refined --- libmpdata++/concurr/detail/concurr_common.hpp | 26 ++++++++++++++----- libmpdata++/concurr/openmp.hpp | 10 +------ .../solvers/detail/solver_type_traits.hpp | 17 ++++++++++++ 3 files changed, 37 insertions(+), 16 deletions(-) create mode 100644 libmpdata++/solvers/detail/solver_type_traits.hpp diff --git a/libmpdata++/concurr/detail/concurr_common.hpp b/libmpdata++/concurr/detail/concurr_common.hpp index b722f3b5..850a8bfc 100644 --- a/libmpdata++/concurr/detail/concurr_common.hpp +++ b/libmpdata++/concurr/detail/concurr_common.hpp @@ -35,6 +35,8 @@ #include #include +#include + namespace libmpdataxx { namespace concurr @@ -150,14 +152,24 @@ namespace libmpdataxx protected: + using shmem_ref_t = + sharedmem_refined_common< + typename solver_t::real_t, + solver_t::n_dims, + solver_t::n_tlev + >; + + using shmem_t = + sharedmem< + typename solver_t::real_t, + solver_t::n_dims, + solver_t::n_tlev + >; + // (cannot be nested due to templates) - typedef sharedmem_refined_common< - //typedef sharedmem< - typename solver_t::real_t, - solver_t::n_dims, - solver_t::n_tlev -// if solver family is _fra use sharedmem_rec, else use sharedmem - > mem_t; + using mem_t = typename std::conditional::value, shmem_ref_t, shmem_t>::type; + //using mem_t = typename std::conditional::type; + //using mem_t = shmem_ref_t; // member fields boost::ptr_vector algos; diff --git a/libmpdata++/concurr/openmp.hpp b/libmpdata++/concurr/openmp.hpp index 82098d0a..d9da77e1 100644 --- a/libmpdata++/concurr/openmp.hpp +++ b/libmpdata++/concurr/openmp.hpp @@ -21,17 +21,10 @@ namespace libmpdataxx { namespace detail { - // helper function that detect if the solver uses fractal reconstruction - template< class, class = void > - struct slvr_with_frac_recn : std::false_type { }; - - template< class solver_t > - struct slvr_with_frac_recn> : std::true_type { }; - template< class mem_t, class solver_t> mem_t* mem_factory(const typename solver_t::rt_params_t &p) { - if constexpr (slvr_with_frac_recn()) + if constexpr (libmpdataxx::solvers::detail::slvr_with_frac_recn()) return new mem_t(p.grid_size, pow(2, p.n_fra_iter)); else return new mem_t(p.grid_size); @@ -51,7 +44,6 @@ namespace libmpdataxx { using parent_t = detail::concurr_common; - struct mem_t : parent_t::mem_t { static int size(const unsigned max_threads = std::numeric_limits::max()) diff --git a/libmpdata++/solvers/detail/solver_type_traits.hpp b/libmpdata++/solvers/detail/solver_type_traits.hpp new file mode 100644 index 00000000..bd027c3b --- /dev/null +++ b/libmpdata++/solvers/detail/solver_type_traits.hpp @@ -0,0 +1,17 @@ +#pragma once + +namespace libmpdataxx +{ + namespace solvers + { + namespace detail + { + // helper function that detect if the solver uses fractal reconstruction + template< class, class = void > + struct slvr_with_frac_recn : std::false_type { }; + + template< class solver_t > + struct slvr_with_frac_recn> : std::true_type { }; + } + } +} From a6e8b435192e73495b1275e1ffa8c2c1a367a335 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 3 Mar 2022 14:45:40 +0100 Subject: [PATCH 008/129] consitional choice of shmem / shmem_refined - upgraded --- libmpdata++/concurr/detail/concurr_common.hpp | 39 +++++++----- libmpdata++/concurr/openmp.hpp | 5 +- libmpdata++/opts.hpp | 1 + libmpdata++/solvers/detail/solver_common.hpp | 15 ++++- .../solvers/detail/solver_type_traits.hpp | 12 +++- libmpdata++/solvers/mpdata.hpp | 4 +- libmpdata++/solvers/mpdata_rhs.hpp | 5 +- libmpdata++/solvers/mpdata_rhs_vip.hpp | 12 +++- libmpdata++/solvers/mpdata_rhs_vip_prs.hpp | 8 +-- .../solvers/mpdata_rhs_vip_prs_sgs.hpp | 4 +- .../solvers/mpdata_rhs_vip_prs_sgs_fra.hpp | 61 ++++++++++--------- tests/sandbox/pbl/pbl_test_def.hpp | 1 + 12 files changed, 106 insertions(+), 61 deletions(-) diff --git a/libmpdata++/concurr/detail/concurr_common.hpp b/libmpdata++/concurr/detail/concurr_common.hpp index 850a8bfc..befb956f 100644 --- a/libmpdata++/concurr/detail/concurr_common.hpp +++ b/libmpdata++/concurr/detail/concurr_common.hpp @@ -152,22 +152,29 @@ namespace libmpdataxx protected: - using shmem_ref_t = - sharedmem_refined_common< - typename solver_t::real_t, - solver_t::n_dims, - solver_t::n_tlev - >; - - using shmem_t = - sharedmem< - typename solver_t::real_t, - solver_t::n_dims, - solver_t::n_tlev - >; - - // (cannot be nested due to templates) - using mem_t = typename std::conditional::value, shmem_ref_t, shmem_t>::type; +// using shmem_ref_t = +// sharedmem_refined_common< +// typename solver_t::real_t, +// solver_t::n_dims, +// solver_t::n_tlev +// >; +// +// using shmem_t = +// sharedmem< +// typename solver_t::real_t, +// solver_t::n_dims, +// solver_t::n_tlev +// >; +// +// // if solver is derived from mpdata_rhs_vip_prs_sgs_fra, use sharedmem version with grid refinement +// using mem_t = typename std::conditional_t< +// libmpdataxx::solvers::detail::slvr_with_frac_recn_v, +// shmem_ref_t, shmem_t>; + + using mem_t = typename solver_t::mem_t; + +// libmpdataxx::solvers::detail::slvr_with_frac_recn::value, shmem_ref_t, shmem_t>::type; +// using mem_t = typename std::conditional::value, shmem_ref_t, shmem_t>::type; //using mem_t = typename std::conditional::type; //using mem_t = shmem_ref_t; diff --git a/libmpdata++/concurr/openmp.hpp b/libmpdata++/concurr/openmp.hpp index d9da77e1..d56bf726 100644 --- a/libmpdata++/concurr/openmp.hpp +++ b/libmpdata++/concurr/openmp.hpp @@ -71,7 +71,10 @@ namespace libmpdataxx // ctors // TODO: n_ref only for a run with grid refinement? - mem_t(const std::array &grid_size, const int n_ref=1) : parent_t::mem_t(grid_size, size(grid_size[0]), n_ref) {}; + mem_t(const std::array &grid_size, const int n_ref) : parent_t::mem_t(grid_size, size(grid_size[0]), n_ref) {}; + mem_t(const std::array &grid_size) : parent_t::mem_t(grid_size, size(grid_size[0])) {}; + +// using parent_t::mem_t::mem_t; }; void solve(typename parent_t::advance_arg_t nt) diff --git a/libmpdata++/opts.hpp b/libmpdata++/opts.hpp index 35a89ea2..16e526da 100644 --- a/libmpdata++/opts.hpp +++ b/libmpdata++/opts.hpp @@ -83,6 +83,7 @@ namespace libmpdataxx enum { opts = opts::iga | opts::fct }; enum { hint_norhs = 0 }; enum { delayed_step = 0 }; + enum { fractal_recon = 0 }; struct ix {}; static constexpr int hint_scale(const int &e) { return 0; } // base-2 logarithm enum { var_dt = false}; diff --git a/libmpdata++/solvers/detail/solver_common.hpp b/libmpdata++/solvers/detail/solver_common.hpp index 0e072960..fb203714 100644 --- a/libmpdata++/solvers/detail/solver_common.hpp +++ b/libmpdata++/solvers/detail/solver_common.hpp @@ -11,6 +11,7 @@ #include #include +#include #include @@ -72,7 +73,19 @@ namespace libmpdataxx real_t time = 0; std::vector n; - typedef concurr::detail::sharedmem_refined_common mem_t; + using shmem_ref_t = concurr::detail::sharedmem_refined_common; + using shmem_t = concurr::detail::sharedmem; + + public: + + // if fractal reconstruction of some variables is required in ct_params, use special type of sharedmem + using mem_t = typename std::conditional_t< + detail::slvr_with_frac_recn_v, + shmem_ref_t, shmem_t>; + + protected: + + //typedef concurr::detail::sharedmem mem_t; mem_t *mem; // helper methods invoked by solve() diff --git a/libmpdata++/solvers/detail/solver_type_traits.hpp b/libmpdata++/solvers/detail/solver_type_traits.hpp index bd027c3b..bbf60361 100644 --- a/libmpdata++/solvers/detail/solver_type_traits.hpp +++ b/libmpdata++/solvers/detail/solver_type_traits.hpp @@ -6,12 +6,18 @@ namespace libmpdataxx { namespace detail { - // helper function that detect if the solver uses fractal reconstruction + // helpers that detect if the solver uses fractal reconstruction template< class, class = void > struct slvr_with_frac_recn : std::false_type { }; - template< class solver_t > - struct slvr_with_frac_recn> : std::true_type { }; + template< class ct_params_t > + struct slvr_with_frac_recn< + ct_params_t, + typename std::enable_if_t<(int)ct_params_t::fractal_recon != (int)0 > > : std::true_type { }; + //std::enable_if_t< std::is_base_of_v< libmpdataxx::solvers::mpdata_rhs_vip_prs_sgs_fra_family_tag, solver_t > > > : std::true_type { }; + + template + inline constexpr bool slvr_with_frac_recn_v = slvr_with_frac_recn::value; } } } diff --git a/libmpdata++/solvers/mpdata.hpp b/libmpdata++/solvers/mpdata.hpp index 1e19bde3..538c93ad 100644 --- a/libmpdata++/solvers/mpdata.hpp +++ b/libmpdata++/solvers/mpdata.hpp @@ -35,7 +35,7 @@ namespace libmpdataxx using parent_t = detail::mpdata_osc; using parent_t::parent_t; // inheriting constructors - protected: + public: using solver_family = mpdata_family_tag; }; @@ -49,7 +49,7 @@ namespace libmpdataxx using parent_t = detail::mpdata_fct; using parent_t::parent_t; // inheriting constructors - protected: + public: using solver_family = mpdata_family_tag; }; } // namespace solvers diff --git a/libmpdata++/solvers/mpdata_rhs.hpp b/libmpdata++/solvers/mpdata_rhs.hpp index 351d7c43..0f4341e5 100644 --- a/libmpdata++/solvers/mpdata_rhs.hpp +++ b/libmpdata++/solvers/mpdata_rhs.hpp @@ -44,9 +44,12 @@ namespace libmpdataxx bool update_rhs_called = true; // so that it nt=0 there's no complain #endif - protected: + public: + using solver_family = mpdata_rhs_family_tag; + protected: + // member fields arrvec_t &rhs; diff --git a/libmpdata++/solvers/mpdata_rhs_vip.hpp b/libmpdata++/solvers/mpdata_rhs_vip.hpp index a037049e..c4c20474 100644 --- a/libmpdata++/solvers/mpdata_rhs_vip.hpp +++ b/libmpdata++/solvers/mpdata_rhs_vip.hpp @@ -28,9 +28,11 @@ namespace libmpdataxx { using ix = typename ct_params_t::ix; + public: + using solver_family = mpdata_rhs_vip_family_tag; + protected: - using solver_family = mpdata_rhs_vip_family_tag; using parent_t = detail::mpdata_rhs_vip_common; // member fields @@ -87,9 +89,11 @@ namespace libmpdataxx { using ix = typename ct_params_t::ix; + public: + using solver_family = mpdata_rhs_vip_family_tag; + protected: - using solver_family = mpdata_rhs_vip_family_tag; using parent_t = detail::mpdata_rhs_vip_common; // member fields @@ -197,9 +201,11 @@ namespace libmpdataxx { using ix = typename ct_params_t::ix; + public: + using solver_family = mpdata_rhs_vip_family_tag; + protected: - using solver_family = mpdata_rhs_vip_family_tag; using parent_t = detail::mpdata_rhs_vip_common; // member fields diff --git a/libmpdata++/solvers/mpdata_rhs_vip_prs.hpp b/libmpdata++/solvers/mpdata_rhs_vip_prs.hpp index 52e1f9d1..af17bec4 100644 --- a/libmpdata++/solvers/mpdata_rhs_vip_prs.hpp +++ b/libmpdata++/solvers/mpdata_rhs_vip_prs.hpp @@ -48,7 +48,7 @@ namespace libmpdataxx using parent_t = detail::mpdata_rhs_vip_prs_mr; using parent_t::parent_t; // inheriting constructors - protected: + public: using solver_family = mpdata_rhs_vip_prs_family_tag; }; @@ -62,7 +62,7 @@ namespace libmpdataxx using parent_t = detail::mpdata_rhs_vip_prs_gcrk; using parent_t::parent_t; // inheriting constructors - protected: + public: using solver_family = mpdata_rhs_vip_prs_family_tag; }; @@ -76,7 +76,7 @@ namespace libmpdataxx using parent_t = detail::mpdata_rhs_vip_prs_gcrk; using parent_t::parent_t; // inheriting constructors - protected: + public: using solver_family = mpdata_rhs_vip_prs_family_tag; }; @@ -90,7 +90,7 @@ namespace libmpdataxx using parent_t = detail::mpdata_rhs_vip_prs_pc; using parent_t::parent_t; // inheriting constructors - protected: + public: using solver_family = mpdata_rhs_vip_prs_family_tag; }; } // namespace solvers diff --git a/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs.hpp b/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs.hpp index fba000d7..334a9cca 100644 --- a/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs.hpp +++ b/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs.hpp @@ -55,7 +55,7 @@ namespace libmpdataxx using parent_t = detail::mpdata_rhs_vip_prs_sgs_dns; using parent_t::parent_t; // inheriting constructors - protected: + public: using solver_family = mpdata_rhs_vip_prs_sgs_dns_family_tag; }; @@ -68,7 +68,7 @@ namespace libmpdataxx using parent_t = detail::mpdata_rhs_vip_prs_sgs_smg; using parent_t::parent_t; // inheriting constructors - protected: + public: using solver_family = mpdata_rhs_vip_prs_sgs_smg_family_tag; }; } // namespace solvers diff --git a/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp b/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp index 961d5c0e..6af47cbd 100644 --- a/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp +++ b/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp @@ -16,40 +16,45 @@ namespace libmpdataxx { struct mpdata_rhs_vip_prs_sgs_fra_family_tag {}; -// template -// class mpdata_rhs_vip_prs_sgs_fra; -// -// template -// class mpdata_rhs_vip_prs_sgs_fra< -// ct_params_t, minhalo, -// typename std::enable_if<(int)ct_params_t::n_fra_rec == 0>::type -// > : public mpdata_rhs_vip_prs_sgs -// { -// using parent_t = mpdata_rhs_vip_prs_sgs; -// using parent_t::parent_t; // inheriting constructors -// }; -// -// template -// class mpdata_rhs_vip_prs_sgs_fra< -// ct_params_t, minhalo, -// typename std::enable_if<(int)ct_params_t::n_fra_rec > 0>::type -// > : public detail::mpdata_rhs_vip_prs_sgs_fra -// { -// using parent_t = detail::mpdata_rhs_vip_prs_sgs_fra; -// using parent_t::parent_t; // inheriting constructors -// -// protected: -// using solver_family = mpdata_rhs_vip_prs_sgs_fra_family_tag; -// }; + template + class mpdata_rhs_vip_prs_sgs_fra; - template - class mpdata_rhs_vip_prs_sgs_fra : public detail::mpdata_rhs_vip_prs_sgs_fra + template + class mpdata_rhs_vip_prs_sgs_fra< + ct_params_t, minhalo, + typename std::enable_if_t<(int)ct_params_t::fractal_recon == (int)0> // no fields with fractal reconstruction +// typename std::enable_if_t<(int)ct_params_t::fractal_recon == (int) 0> + // std::enable_if_t<(int)ct_params_t::fractal_recon == 0, bool> = true + > : public mpdata_rhs_vip_prs_sgs + { + using parent_t = mpdata_rhs_vip_prs_sgs; + using parent_t::parent_t; // inheriting constructors + }; + + template + class mpdata_rhs_vip_prs_sgs_fra< + ct_params_t, minhalo, + typename std::enable_if_t<(int)ct_params_t::fractal_recon != (int)0> + //typename std::enable_if_t<(int)ct_params_t::fractal_recon != (int) 0> + // std::enable_if_t<(int)ct_params_t::fractal_recon > 0, bool> = true + > : public detail::mpdata_rhs_vip_prs_sgs_fra, + public mpdata_rhs_vip_prs_sgs_fra_family_tag // allows checking if derived solvers use fractal reconstruction { using parent_t = detail::mpdata_rhs_vip_prs_sgs_fra; using parent_t::parent_t; // inheriting constructors - protected: + private: using solver_family = mpdata_rhs_vip_prs_sgs_fra_family_tag; }; + +// template +// class mpdata_rhs_vip_prs_sgs_fra : public detail::mpdata_rhs_vip_prs_sgs_fra +// { +// using parent_t = detail::mpdata_rhs_vip_prs_sgs_fra; +// using parent_t::parent_t; // inheriting constructors +// +// protected: +// using solver_family = mpdata_rhs_vip_prs_sgs_fra_family_tag; +// }; } // namespace solvers } // namespace libmpdataxx diff --git a/tests/sandbox/pbl/pbl_test_def.hpp b/tests/sandbox/pbl/pbl_test_def.hpp index ef63d592..bb0270e1 100644 --- a/tests/sandbox/pbl/pbl_test_def.hpp +++ b/tests/sandbox/pbl/pbl_test_def.hpp @@ -54,6 +54,7 @@ void test(const std::string &dirname, const int np, const int nt) u, v, w, tht, vip_i=u, vip_j=v, vip_k=w, vip_den=-1 }; }; + enum { fractal_recon = libmpdataxx::opts::bit(ix::tht) }; }; using ix = typename ct_params_t::ix; From be93f457ff42578119e20d6ad2a037ecdde205c8 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 3 Mar 2022 15:50:36 +0100 Subject: [PATCH 009/129] working shmem / shmem_recon stuff? (openmp and 3d only for now) --- libmpdata++/concurr/openmp.hpp | 2 +- tests/sandbox/pbl/pbl_test_def.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libmpdata++/concurr/openmp.hpp b/libmpdata++/concurr/openmp.hpp index d56bf726..1ad4c959 100644 --- a/libmpdata++/concurr/openmp.hpp +++ b/libmpdata++/concurr/openmp.hpp @@ -24,7 +24,7 @@ namespace libmpdataxx template< class mem_t, class solver_t> mem_t* mem_factory(const typename solver_t::rt_params_t &p) { - if constexpr (libmpdataxx::solvers::detail::slvr_with_frac_recn()) + if constexpr (solvers::detail::slvr_with_frac_recn()) return new mem_t(p.grid_size, pow(2, p.n_fra_iter)); else return new mem_t(p.grid_size); diff --git a/tests/sandbox/pbl/pbl_test_def.hpp b/tests/sandbox/pbl/pbl_test_def.hpp index bb0270e1..c34d5d05 100644 --- a/tests/sandbox/pbl/pbl_test_def.hpp +++ b/tests/sandbox/pbl/pbl_test_def.hpp @@ -54,7 +54,7 @@ void test(const std::string &dirname, const int np, const int nt) u, v, w, tht, vip_i=u, vip_j=v, vip_k=w, vip_den=-1 }; }; - enum { fractal_recon = libmpdataxx::opts::bit(ix::tht) }; + // enum { fractal_recon = libmpdataxx::opts::bit(ix::tht) }; }; using ix = typename ct_params_t::ix; From 963be45d9a32082a2a2167d74fe4402e0982746c Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 3 Mar 2022 16:08:37 +0100 Subject: [PATCH 010/129] solver_family protected again + solver doesnt inherit from family tag --- libmpdata++/concurr/detail/concurr_common.hpp | 25 ------------------- libmpdata++/concurr/openmp.hpp | 1 - libmpdata++/solvers/detail/solver_common.hpp | 1 - libmpdata++/solvers/mpdata.hpp | 4 +-- libmpdata++/solvers/mpdata_rhs.hpp | 4 +-- libmpdata++/solvers/mpdata_rhs_vip.hpp | 12 +++------ libmpdata++/solvers/mpdata_rhs_vip_prs.hpp | 8 +++--- .../solvers/mpdata_rhs_vip_prs_sgs.hpp | 4 +-- .../solvers/mpdata_rhs_vip_prs_sgs_fra.hpp | 5 ++-- tests/sandbox/pbl/pbl_test_def.hpp | 2 +- 10 files changed, 15 insertions(+), 51 deletions(-) diff --git a/libmpdata++/concurr/detail/concurr_common.hpp b/libmpdata++/concurr/detail/concurr_common.hpp index befb956f..76cdb509 100644 --- a/libmpdata++/concurr/detail/concurr_common.hpp +++ b/libmpdata++/concurr/detail/concurr_common.hpp @@ -14,7 +14,6 @@ #include #include -#include #include #include #include @@ -152,31 +151,7 @@ namespace libmpdataxx protected: -// using shmem_ref_t = -// sharedmem_refined_common< -// typename solver_t::real_t, -// solver_t::n_dims, -// solver_t::n_tlev -// >; -// -// using shmem_t = -// sharedmem< -// typename solver_t::real_t, -// solver_t::n_dims, -// solver_t::n_tlev -// >; -// -// // if solver is derived from mpdata_rhs_vip_prs_sgs_fra, use sharedmem version with grid refinement -// using mem_t = typename std::conditional_t< -// libmpdataxx::solvers::detail::slvr_with_frac_recn_v, -// shmem_ref_t, shmem_t>; - using mem_t = typename solver_t::mem_t; - -// libmpdataxx::solvers::detail::slvr_with_frac_recn::value, shmem_ref_t, shmem_t>::type; -// using mem_t = typename std::conditional::value, shmem_ref_t, shmem_t>::type; - //using mem_t = typename std::conditional::type; - //using mem_t = shmem_ref_t; // member fields boost::ptr_vector algos; diff --git a/libmpdata++/concurr/openmp.hpp b/libmpdata++/concurr/openmp.hpp index 1ad4c959..08972e13 100644 --- a/libmpdata++/concurr/openmp.hpp +++ b/libmpdata++/concurr/openmp.hpp @@ -9,7 +9,6 @@ #include #include -#include #ifdef _OPENMP # include diff --git a/libmpdata++/solvers/detail/solver_common.hpp b/libmpdata++/solvers/detail/solver_common.hpp index fb203714..82407a43 100644 --- a/libmpdata++/solvers/detail/solver_common.hpp +++ b/libmpdata++/solvers/detail/solver_common.hpp @@ -85,7 +85,6 @@ namespace libmpdataxx protected: - //typedef concurr::detail::sharedmem mem_t; mem_t *mem; // helper methods invoked by solve() diff --git a/libmpdata++/solvers/mpdata.hpp b/libmpdata++/solvers/mpdata.hpp index 538c93ad..1e19bde3 100644 --- a/libmpdata++/solvers/mpdata.hpp +++ b/libmpdata++/solvers/mpdata.hpp @@ -35,7 +35,7 @@ namespace libmpdataxx using parent_t = detail::mpdata_osc; using parent_t::parent_t; // inheriting constructors - public: + protected: using solver_family = mpdata_family_tag; }; @@ -49,7 +49,7 @@ namespace libmpdataxx using parent_t = detail::mpdata_fct; using parent_t::parent_t; // inheriting constructors - public: + protected: using solver_family = mpdata_family_tag; }; } // namespace solvers diff --git a/libmpdata++/solvers/mpdata_rhs.hpp b/libmpdata++/solvers/mpdata_rhs.hpp index 0f4341e5..d247ae1a 100644 --- a/libmpdata++/solvers/mpdata_rhs.hpp +++ b/libmpdata++/solvers/mpdata_rhs.hpp @@ -44,12 +44,10 @@ namespace libmpdataxx bool update_rhs_called = true; // so that it nt=0 there's no complain #endif - public: + protected: using solver_family = mpdata_rhs_family_tag; - protected: - // member fields arrvec_t &rhs; diff --git a/libmpdata++/solvers/mpdata_rhs_vip.hpp b/libmpdata++/solvers/mpdata_rhs_vip.hpp index c4c20474..a037049e 100644 --- a/libmpdata++/solvers/mpdata_rhs_vip.hpp +++ b/libmpdata++/solvers/mpdata_rhs_vip.hpp @@ -28,11 +28,9 @@ namespace libmpdataxx { using ix = typename ct_params_t::ix; - public: - using solver_family = mpdata_rhs_vip_family_tag; - protected: + using solver_family = mpdata_rhs_vip_family_tag; using parent_t = detail::mpdata_rhs_vip_common; // member fields @@ -89,11 +87,9 @@ namespace libmpdataxx { using ix = typename ct_params_t::ix; - public: - using solver_family = mpdata_rhs_vip_family_tag; - protected: + using solver_family = mpdata_rhs_vip_family_tag; using parent_t = detail::mpdata_rhs_vip_common; // member fields @@ -201,11 +197,9 @@ namespace libmpdataxx { using ix = typename ct_params_t::ix; - public: - using solver_family = mpdata_rhs_vip_family_tag; - protected: + using solver_family = mpdata_rhs_vip_family_tag; using parent_t = detail::mpdata_rhs_vip_common; // member fields diff --git a/libmpdata++/solvers/mpdata_rhs_vip_prs.hpp b/libmpdata++/solvers/mpdata_rhs_vip_prs.hpp index af17bec4..52e1f9d1 100644 --- a/libmpdata++/solvers/mpdata_rhs_vip_prs.hpp +++ b/libmpdata++/solvers/mpdata_rhs_vip_prs.hpp @@ -48,7 +48,7 @@ namespace libmpdataxx using parent_t = detail::mpdata_rhs_vip_prs_mr; using parent_t::parent_t; // inheriting constructors - public: + protected: using solver_family = mpdata_rhs_vip_prs_family_tag; }; @@ -62,7 +62,7 @@ namespace libmpdataxx using parent_t = detail::mpdata_rhs_vip_prs_gcrk; using parent_t::parent_t; // inheriting constructors - public: + protected: using solver_family = mpdata_rhs_vip_prs_family_tag; }; @@ -76,7 +76,7 @@ namespace libmpdataxx using parent_t = detail::mpdata_rhs_vip_prs_gcrk; using parent_t::parent_t; // inheriting constructors - public: + protected: using solver_family = mpdata_rhs_vip_prs_family_tag; }; @@ -90,7 +90,7 @@ namespace libmpdataxx using parent_t = detail::mpdata_rhs_vip_prs_pc; using parent_t::parent_t; // inheriting constructors - public: + protected: using solver_family = mpdata_rhs_vip_prs_family_tag; }; } // namespace solvers diff --git a/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs.hpp b/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs.hpp index 334a9cca..fba000d7 100644 --- a/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs.hpp +++ b/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs.hpp @@ -55,7 +55,7 @@ namespace libmpdataxx using parent_t = detail::mpdata_rhs_vip_prs_sgs_dns; using parent_t::parent_t; // inheriting constructors - public: + protected: using solver_family = mpdata_rhs_vip_prs_sgs_dns_family_tag; }; @@ -68,7 +68,7 @@ namespace libmpdataxx using parent_t = detail::mpdata_rhs_vip_prs_sgs_smg; using parent_t::parent_t; // inheriting constructors - public: + protected: using solver_family = mpdata_rhs_vip_prs_sgs_smg_family_tag; }; } // namespace solvers diff --git a/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp b/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp index 6af47cbd..4fa8a1b7 100644 --- a/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp +++ b/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp @@ -37,13 +37,12 @@ namespace libmpdataxx typename std::enable_if_t<(int)ct_params_t::fractal_recon != (int)0> //typename std::enable_if_t<(int)ct_params_t::fractal_recon != (int) 0> // std::enable_if_t<(int)ct_params_t::fractal_recon > 0, bool> = true - > : public detail::mpdata_rhs_vip_prs_sgs_fra, - public mpdata_rhs_vip_prs_sgs_fra_family_tag // allows checking if derived solvers use fractal reconstruction + > : public detail::mpdata_rhs_vip_prs_sgs_fra { using parent_t = detail::mpdata_rhs_vip_prs_sgs_fra; using parent_t::parent_t; // inheriting constructors - private: + protected: using solver_family = mpdata_rhs_vip_prs_sgs_fra_family_tag; }; diff --git a/tests/sandbox/pbl/pbl_test_def.hpp b/tests/sandbox/pbl/pbl_test_def.hpp index c34d5d05..bb0270e1 100644 --- a/tests/sandbox/pbl/pbl_test_def.hpp +++ b/tests/sandbox/pbl/pbl_test_def.hpp @@ -54,7 +54,7 @@ void test(const std::string &dirname, const int np, const int nt) u, v, w, tht, vip_i=u, vip_j=v, vip_k=w, vip_den=-1 }; }; - // enum { fractal_recon = libmpdataxx::opts::bit(ix::tht) }; + enum { fractal_recon = libmpdataxx::opts::bit(ix::tht) }; }; using ix = typename ct_params_t::ix; From d36191951e4bbb7b84bff021d02352e6db1c0aab Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 3 Mar 2022 16:19:52 +0100 Subject: [PATCH 011/129] use mem_factory in all concurr types --- libmpdata++/concurr/boost_thread.hpp | 2 +- libmpdata++/concurr/cxx11_thread.hpp | 2 +- libmpdata++/concurr/detail/concurr_common.hpp | 9 +++++++++ libmpdata++/concurr/openmp.hpp | 15 --------------- libmpdata++/concurr/serial.hpp | 2 +- 5 files changed, 12 insertions(+), 18 deletions(-) diff --git a/libmpdata++/concurr/boost_thread.hpp b/libmpdata++/concurr/boost_thread.hpp index f1d7c7a6..dc53f012 100644 --- a/libmpdata++/concurr/boost_thread.hpp +++ b/libmpdata++/concurr/boost_thread.hpp @@ -81,7 +81,7 @@ namespace libmpdataxx // ctor boost_thread(const typename solver_t::rt_params_t &p) : - parent_t(p, new mem_t(p.grid_size), mem_t::size(p.grid_size[solver_t::n_dims < 3 ? 0 : 1])) // note 3D domain decomposition in y direction + parent_t(p, detail::mem_factory(p), mem_t::size(p.grid_size[solver_t::n_dims < 3 ? 0 : 1])) // note 3D domain decomposition in y direction {} }; diff --git a/libmpdata++/concurr/cxx11_thread.hpp b/libmpdata++/concurr/cxx11_thread.hpp index 77dc47d9..66e177f0 100644 --- a/libmpdata++/concurr/cxx11_thread.hpp +++ b/libmpdata++/concurr/cxx11_thread.hpp @@ -119,7 +119,7 @@ namespace libmpdataxx // ctor cxx11_thread(const typename solver_t::rt_params_t &p) : - parent_t(p, new mem_t(p.grid_size), mem_t::size(p.grid_size[solver_t::n_dims < 3 ? 0 : 1])) // note 3D domain decomposition in y direction + parent_t(p, detail::mem_factory(p), mem_t::size(p.grid_size[solver_t::n_dims < 3 ? 0 : 1])) // note 3D domain decomposition in y direction {} }; diff --git a/libmpdata++/concurr/detail/concurr_common.hpp b/libmpdata++/concurr/detail/concurr_common.hpp index 76cdb509..5981803a 100644 --- a/libmpdata++/concurr/detail/concurr_common.hpp +++ b/libmpdata++/concurr/detail/concurr_common.hpp @@ -449,6 +449,15 @@ namespace libmpdataxx return mem->max(mem->advectee(e)); } }; + + template< class mem_t, class solver_t> + mem_t* mem_factory(const typename solver_t::rt_params_t &p) + { + if constexpr (solvers::detail::slvr_with_frac_recn()) + return new mem_t(p.grid_size, pow(2, p.n_fra_iter)); + else + return new mem_t(p.grid_size); + } } // namespace detail } // namespace concurr } // namespace libmpdataxx diff --git a/libmpdata++/concurr/openmp.hpp b/libmpdata++/concurr/openmp.hpp index 08972e13..ce65d8aa 100644 --- a/libmpdata++/concurr/openmp.hpp +++ b/libmpdata++/concurr/openmp.hpp @@ -18,18 +18,6 @@ namespace libmpdataxx { namespace concurr { - namespace detail - { - template< class mem_t, class solver_t> - mem_t* mem_factory(const typename solver_t::rt_params_t &p) - { - if constexpr (solvers::detail::slvr_with_frac_recn()) - return new mem_t(p.grid_size, pow(2, p.n_fra_iter)); - else - return new mem_t(p.grid_size); - } - }; - template < class solver_t, bcond::bcond_e bcxl, @@ -69,11 +57,8 @@ namespace libmpdataxx } // ctors - // TODO: n_ref only for a run with grid refinement? mem_t(const std::array &grid_size, const int n_ref) : parent_t::mem_t(grid_size, size(grid_size[0]), n_ref) {}; mem_t(const std::array &grid_size) : parent_t::mem_t(grid_size, size(grid_size[0])) {}; - -// using parent_t::mem_t::mem_t; }; void solve(typename parent_t::advance_arg_t nt) diff --git a/libmpdata++/concurr/serial.hpp b/libmpdata++/concurr/serial.hpp index f6abc1ea..135fc3ae 100644 --- a/libmpdata++/concurr/serial.hpp +++ b/libmpdata++/concurr/serial.hpp @@ -47,7 +47,7 @@ namespace libmpdataxx // ctor serial(const typename solver_t::rt_params_t &p) : - parent_t(p, new mem_t(p.grid_size), mem_t::size()) + parent_t(p, detail::mem_factory(p), mem_t::size()) {} }; From 2d956d1fabb75058bf603182d9a2c6ecd5f57fcb Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 3 Mar 2022 16:23:59 +0100 Subject: [PATCH 012/129] second ctor for shmem_refined in mem_t in each concurr type --- libmpdata++/concurr/boost_thread.hpp | 6 ++---- libmpdata++/concurr/cxx11_thread.hpp | 6 ++---- libmpdata++/concurr/openmp.hpp | 2 +- libmpdata++/concurr/serial.hpp | 5 ++--- 4 files changed, 7 insertions(+), 12 deletions(-) diff --git a/libmpdata++/concurr/boost_thread.hpp b/libmpdata++/concurr/boost_thread.hpp index dc53f012..c804e9bd 100644 --- a/libmpdata++/concurr/boost_thread.hpp +++ b/libmpdata++/concurr/boost_thread.hpp @@ -51,10 +51,8 @@ namespace libmpdataxx // ctor - mem_t(const std::array &grid_size) : - b(size(grid_size[0])), - parent_t::mem_t(grid_size, size(grid_size[0])) - {}; + mem_t(const std::array &grid_size, const int n_ref) : b(size(grid_size[0])), parent_t::mem_t(grid_size, size(grid_size[0]), n_ref) {}; + mem_t(const std::array &grid_size) : b(size(grid_size[0])), parent_t::mem_t(grid_size, size(grid_size[0])) {}; void barrier() { diff --git a/libmpdata++/concurr/cxx11_thread.hpp b/libmpdata++/concurr/cxx11_thread.hpp index 66e177f0..408b7f3b 100644 --- a/libmpdata++/concurr/cxx11_thread.hpp +++ b/libmpdata++/concurr/cxx11_thread.hpp @@ -92,10 +92,8 @@ namespace libmpdataxx } // ctor - mem_t(const std::array &grid_size) : - b(size(grid_size[0])), - parent_t::mem_t(grid_size, size(grid_size[0])) - {}; + mem_t(const std::array &grid_size, const int n_ref) : b(size(grid_size[0])), parent_t::mem_t(grid_size, size(grid_size[0]), n_ref) {}; + mem_t(const std::array &grid_size) : b(size(grid_size[0])), parent_t::mem_t(grid_size, size(grid_size[0])) {}; void barrier() { diff --git a/libmpdata++/concurr/openmp.hpp b/libmpdata++/concurr/openmp.hpp index ce65d8aa..96362754 100644 --- a/libmpdata++/concurr/openmp.hpp +++ b/libmpdata++/concurr/openmp.hpp @@ -58,7 +58,7 @@ namespace libmpdataxx // ctors mem_t(const std::array &grid_size, const int n_ref) : parent_t::mem_t(grid_size, size(grid_size[0]), n_ref) {}; - mem_t(const std::array &grid_size) : parent_t::mem_t(grid_size, size(grid_size[0])) {}; + mem_t(const std::array &grid_size) : parent_t::mem_t(grid_size, size(grid_size[0])) {}; }; void solve(typename parent_t::advance_arg_t nt) diff --git a/libmpdata++/concurr/serial.hpp b/libmpdata++/concurr/serial.hpp index 135fc3ae..1a5fe18b 100644 --- a/libmpdata++/concurr/serial.hpp +++ b/libmpdata++/concurr/serial.hpp @@ -33,9 +33,8 @@ namespace libmpdataxx void barrier() { } // ctors - mem_t(const std::array &grid_size) - : parent_t::mem_t(grid_size, size()) - {}; + mem_t(const std::array &grid_size, const int n_ref) : parent_t::mem_t(grid_size, size(grid_size[0]), n_ref) {}; + mem_t(const std::array &grid_size) : parent_t::mem_t(grid_size, size(grid_size[0])) {}; }; void solve(typename parent_t::advance_arg_t nt) From 7d54514871a6d1228a80cfb227d83fad4b38ccf8 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 3 Mar 2022 16:35:46 +0100 Subject: [PATCH 013/129] alloc tmp with custom grid size in 1d and 2d --- libmpdata++/solvers/detail/solver_1d.hpp | 18 ++++++++++ libmpdata++/solvers/detail/solver_2d.hpp | 42 +++++++++++++++++++++--- 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/libmpdata++/solvers/detail/solver_1d.hpp b/libmpdata++/solvers/detail/solver_1d.hpp index 1b007aa4..4adfe74c 100644 --- a/libmpdata++/solvers/detail/solver_1d.hpp +++ b/libmpdata++/solvers/detail/solver_1d.hpp @@ -199,6 +199,15 @@ namespace libmpdataxx protected: // helper method to allocate a vector-component temporary array + static void alloc_tmp_vctr( + typename parent_t::mem_t *mem, + const char * __file__, + const std::array grid_size + ) + { + alloc_tmp(mem, __file__, 1, parent_t::rng_vctr(grid_size[0])); // always one-component vectors + } + static void alloc_tmp_vctr( typename parent_t::mem_t *mem, const char * __file__ @@ -208,6 +217,15 @@ namespace libmpdataxx } // helper method to allocate n_arr scalar temporary arrays + static void alloc_tmp_sclr( + typename parent_t::mem_t *mem, + const char * __file__, const int n_arr, + const std::array grid_size + ) + { + alloc_tmp(mem, __file__, n_arr, parent_t::rng_sclr(grid_size[0])); + } + static void alloc_tmp_sclr( typename parent_t::mem_t *mem, const char * __file__, const int n_arr diff --git a/libmpdata++/solvers/detail/solver_2d.hpp b/libmpdata++/solvers/detail/solver_2d.hpp index fa8876f4..f76aea4d 100644 --- a/libmpdata++/solvers/detail/solver_2d.hpp +++ b/libmpdata++/solvers/detail/solver_2d.hpp @@ -364,6 +364,7 @@ namespace libmpdataxx const char * __file__, const int n_arr, const std::vector> &stgr, + const std::array grid_size, bool srfc = false ) { @@ -371,27 +372,49 @@ namespace libmpdataxx for (int n = 0; n < n_arr; ++n) { mem->tmp[__file__].back().push_back(mem->old(new typename parent_t::arr_t( - stgr[n][0] ? parent_t::rng_vctr(mem->grid_size[0]) : parent_t::rng_sclr(mem->grid_size[0]), + stgr[n][0] ? parent_t::rng_vctr(grid_size[0]) : parent_t::rng_sclr(grid_size[0]), srfc ? rng_t(0, 0) : - stgr[n][1] ? parent_t::rng_vctr(mem->grid_size[1]) : - parent_t::rng_sclr(mem->grid_size[1]) + stgr[n][1] ? parent_t::rng_vctr(grid_size[1]) : + parent_t::rng_sclr(grid_size[1]) ))); } } + // version with default grid size + static void alloc_tmp_stgr( + typename parent_t::mem_t *mem, + const char * __file__, + const int n_arr, + const std::vector> &stgr, + bool srfc = false // allocate only surface data + ) + { + alloc_tmp_stgr(mem, __file__, n_arr, stgr, mem->grid_size, srfc); + } + // helper method to allocate a temporary space composed of vector-component arrays + static void alloc_tmp_vctr( + typename parent_t::mem_t *mem, + const char * __file__, + const std::array grid_size + ) + { + alloc_tmp_stgr(mem, __file__, 2, {{true, false}, {false, true}}, grid_size, false); + } + static void alloc_tmp_vctr( typename parent_t::mem_t *mem, const char * __file__ ) { - alloc_tmp_stgr(mem, __file__, 2, {{true, false}, {false, true}}); + alloc_tmp_stgr(mem, __file__, 2, {{true, false}, {false, true}}, false); } // helper method to allocate n_arr scalar temporary arrays static void alloc_tmp_sclr( typename parent_t::mem_t *mem, const char * __file__, const int n_arr, + const std::array grid_size, std::string name = "", bool srfc = false ) @@ -406,6 +429,17 @@ namespace libmpdataxx srfc ? rng_t(0, 0) : parent_t::rng_sclr(mem->grid_size[1]) ))); } + + // with default grid_size + static void alloc_tmp_sclr( + typename parent_t::mem_t *mem, + const char * __file__, const int n_arr, + std::string name = "", + bool srfc = false + ) + { + alloc_tmp_sclr(mem, __file__, n_arr, mem->grid_size, name, srfc); + } }; } // namespace detail } // namespace solvers From 3f81179662f472998b11dd405da3edefd537d387 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 7 Mar 2022 13:20:46 +0100 Subject: [PATCH 014/129] hdf5: store refined data --- libmpdata++/concurr/detail/distmem.hpp | 1 + .../concurr/detail/sharedmem_refined.hpp | 39 ++++++++++- libmpdata++/output/hdf5.hpp | 66 +++++++++++++------ .../detail/mpdata_rhs_vip_prs_sgs_fra.hpp | 3 - libmpdata++/solvers/detail/solver_common.hpp | 2 +- tests/sandbox/pbl/pbl.hpp | 1 + 6 files changed, 85 insertions(+), 27 deletions(-) diff --git a/libmpdata++/concurr/detail/distmem.hpp b/libmpdata++/concurr/detail/distmem.hpp index c8d6e0b2..878cbd63 100644 --- a/libmpdata++/concurr/detail/distmem.hpp +++ b/libmpdata++/concurr/detail/distmem.hpp @@ -48,6 +48,7 @@ namespace libmpdataxx public: std::array grid_size; + std::array grid_size_ref; int rank() { diff --git a/libmpdata++/concurr/detail/sharedmem_refined.hpp b/libmpdata++/concurr/detail/sharedmem_refined.hpp index d8cd12b6..77eb17e5 100644 --- a/libmpdata++/concurr/detail/sharedmem_refined.hpp +++ b/libmpdata++/concurr/detail/sharedmem_refined.hpp @@ -24,10 +24,11 @@ namespace libmpdataxx class sharedmem_refined_common : public sharedmem { using parent_t = sharedmem; - using arr_t = typename parent_t::arr_t; protected: + using arr_t = typename parent_t::arr_t; + const int n_ref; // number of equal divisions of the large cell (in each direction) blitz::TinyVector origin_ref; @@ -45,10 +46,11 @@ namespace libmpdataxx { grid_size_ref[d] = refine_grid_size(this->grid_size[d], n_ref); origin_ref[d] = grid_size_ref[d].first(); + this->distmem.grid_size_ref[d] = grid_size_ref[d].last() - grid_size_ref[d].first() + 1; } } - // virtual arr_t refinee(int e = 0) = 0; + virtual arr_t refinee(int e = 0) = 0; // virtual const arr_t refinee_global_ref(int e = 0) = 0; public: @@ -62,6 +64,39 @@ namespace libmpdataxx ); } }; + + + + template + class sharedmem_refined + {}; + + template + class sharedmem_refined : public sharedmem_refined_common + { + using parent_t = sharedmem_refined_common; + using parent_t::parent_t; // inheriting ctors + + protected: + using arr_t = typename parent_t::arr_t; + + public: + arr_t refinee(int e = 0) override + { + std::cerr << "refinee: " << this->psi_ref[e]( + this->grid_size_ref[0], + this->grid_size_ref[1], + this->grid_size_ref[2] + ).reindex(this->origin_ref); + + return this->psi_ref[e]( + this->grid_size_ref[0], + this->grid_size_ref[1], + this->grid_size_ref[2] + ).reindex(this->origin_ref); + } + }; + } // namespace detail } // namespace concurr } // namespace libmpdataxx diff --git a/libmpdata++/output/hdf5.hpp b/libmpdata++/output/hdf5.hpp index 6d8b906b..ae734156 100644 --- a/libmpdata++/output/hdf5.hpp +++ b/libmpdata++/output/hdf5.hpp @@ -55,10 +55,10 @@ namespace libmpdataxx H5::PredType::NATIVE_FLOAT, flttype_output = H5::PredType::NATIVE_FLOAT; // using floats not to waste disk space - blitz::TinyVector cshape, shape, chunk, srfcshape, srfcchunk, offst; + blitz::TinyVector cshape, shape, chunk, srfcshape, srfcchunk, offst, shape_ref, cshape_ref, chunk_ref, offst_ref; H5::DSetCreatPropList params; - H5::DataSpace sspace, cspace, srfcspace; + H5::DataSpace sspace, cspace, srfcspace, sspace_ref; #if defined(USE_MPI) hid_t fapl_id; #endif @@ -96,35 +96,49 @@ namespace libmpdataxx // creating the dimensions // x,y,z offst = 0; + offst_ref = 0; for (int d = 0; d < parent_t::n_dims; ++d) + { shape[d] = this->mem->distmem.grid_size[d]; + shape_ref[d] = this->mem->distmem.grid_size_ref[d]; + } chunk = shape; + chunk_ref = shape_ref; // there is one more coordinate than cell index in each dimension cshape = shape + 1; + cshape_ref = shape_ref + 1; srfcshape = shape; *(srfcshape.end()-1) = 1; sspace = H5::DataSpace(parent_t::n_dims, shape.data()); - srfcspace = H5::DataSpace(parent_t::n_dims, srfcshape.data()); cspace = H5::DataSpace(parent_t::n_dims, cshape.data()); + srfcspace = H5::DataSpace(parent_t::n_dims, srfcshape.data()); + sspace_ref = H5::DataSpace(parent_t::n_dims, shape_ref.data()); #if defined(USE_MPI) if (this->mem->distmem.size() > 1) { shape[0] = this->mem->grid_size[0].length(); cshape[0] = this->mem->grid_size[0].length(); + shape_ref[0] = this->mem->grid_size_ref[0].length(); + cshape_ref[0] = this->mem->grid_size_ref[0].length(); if (this->mem->distmem.rank() == this->mem->distmem.size() - 1) + { cshape[0] += 1; + cshape_ref[0] += 1; + } offst[0] = this->mem->grid_size[0].first(); + offst_ref[0] = this->mem->grid_size_ref[0].first(); // chunk size has to be common to all processes ! // TODO: something better ? chunk[0] = ( (typename solver_t::real_t) (this->mem->distmem.grid_size[0])) / this->mem->distmem.size() + 0.5 ; + chunk_ref[0] = ( (typename solver_t::real_t) (this->mem->distmem.grid_size_ref[0])) / this->mem->distmem.size() + 0.5 ; } #endif @@ -287,33 +301,35 @@ namespace libmpdataxx }; } - void record_dsc_helper(const H5::DataSet &dset, const typename solver_t::arr_t &arr) + void record_dsc_helper(const H5::DataSet &dset, const typename solver_t::arr_t &arr, const bool refined = false) { H5::DataSpace space = dset.getSpace(); - space.selectHyperslab(H5S_SELECT_SET, shape.data(), offst.data()); - // TODO: some permutation of grid_size instead of the switch + const auto _grid_size(refined ? this->mem->grid_size_ref : this->mem->grid_size); + const auto _shape(refined ? shape_ref : shape); + space.selectHyperslab(H5S_SELECT_SET, _shape.data(), refined ? offst_ref.data() : offst.data()); + + // TODO: some permutation of grid_size instead of the switch switch (int(solver_t::n_dims)) { case 1: { - typename solver_t::arr_t contiguous_arr = arr(this->mem->grid_size[0]).copy(); // create a copy that is contiguous - dset.write(contiguous_arr.data(), flttype_solver, H5::DataSpace(parent_t::n_dims, shape.data()), space, dxpl_id); + typename solver_t::arr_t contiguous_arr = arr(_grid_size[0]).copy(); // create a copy that is contiguous + dset.write(contiguous_arr.data(), flttype_solver, H5::DataSpace(parent_t::n_dims, _shape.data()), space, dxpl_id); break; } case 2: { - typename solver_t::arr_t contiguous_arr = arr(this->mem->grid_size[0], this->mem->grid_size[1]).copy(); // create a copy that is contiguous - dset.write(contiguous_arr.data(), flttype_solver, H5::DataSpace(parent_t::n_dims, shape.data()), space, dxpl_id); + typename solver_t::arr_t contiguous_arr = arr(_grid_size[0], _grid_size[1]).copy(); // create a copy that is contiguous + dset.write(contiguous_arr.data(), flttype_solver, H5::DataSpace(parent_t::n_dims, _shape.data()), space, dxpl_id); break; } case 3: { // create a copy that is contiguous and has the C-style (kji) storage order as required by HDF5 - typename solver_t::arr_t contiguous_arr(this->mem->grid_size[0], this->mem->grid_size[1], this->mem->grid_size[2]); - contiguous_arr = arr(this->mem->grid_size[0], this->mem->grid_size[1], this->mem->grid_size[2]); - - dset.write(contiguous_arr.data(), flttype_solver, H5::DataSpace(parent_t::n_dims, shape.data()), space, dxpl_id); + typename solver_t::arr_t contiguous_arr(_grid_size[0], _grid_size[1], _grid_size[2]); + contiguous_arr = arr(_grid_size[0], _grid_size[1], _grid_size[2]); + dset.write(contiguous_arr.data(), flttype_solver, H5::DataSpace(parent_t::n_dims, _shape.data()), space, dxpl_id); break; } default: assert(false); @@ -338,28 +354,31 @@ namespace libmpdataxx } // for discontiguous array with halos - void record_aux_dsc_hlpr(const std::string &name, const typename solver_t::arr_t &arr, H5::H5File hdf, bool srfc = false) + void record_aux_dsc_hlpr(const std::string &name, const typename solver_t::arr_t &arr, H5::H5File hdf, bool srfc = false, bool refined = false) { assert(this->rank == 0); + assert(!(refined && srfc)); if(srfc) params.setChunk(parent_t::n_dims, srfcchunk.data()); + else if (refined) + params.setChunk(parent_t::n_dims, chunk_ref.data()); auto aux = hdf.createDataSet( name, flttype_output, - srfc ? srfcspace : sspace, + srfc ? srfcspace : refined ? sspace_ref : sspace, params ); if(srfc) - { - // revert to default chunk - params.setChunk(parent_t::n_dims, chunk.data()); record_dsc_srfc_helper(aux, arr); - } else - record_dsc_helper(aux, arr); + record_dsc_helper(aux, arr, refined); + + // revert to default chunk + if(srfc || refined) + params.setChunk(parent_t::n_dims, chunk.data()); } void record_scalar_hlpr(const std::string &name, const std::string &group_name, typename solver_t::real_t data, H5::H5File hdf) @@ -413,6 +432,11 @@ namespace libmpdataxx record_aux_dsc_hlpr(name, arr, *hdfp, srfc); } + void record_aux_dsc_refined(const std::string &name, const typename solver_t::arr_t &arr) + { + record_aux_dsc_hlpr(name, arr, *hdfp, false, true); + } + void record_aux_scalar(const std::string &name, const std::string &group_name, typename solver_t::real_t data) { record_scalar_hlpr(name, group_name, data, *hdfp); diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra.hpp index ca0da71d..b05ec22b 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra.hpp @@ -61,9 +61,6 @@ namespace libmpdataxx mem->psi_ref = mem->tmp[__FILE__][0]; mem->GC_ref = mem->tmp[__FILE__][1]; - - std::cerr << "psi_ref: " << mem->psi_ref[0] << std::endl; - std::cerr << "GC_ref: " << mem->GC_ref[0] << std::endl; } }; } // namespace detail diff --git a/libmpdata++/solvers/detail/solver_common.hpp b/libmpdata++/solvers/detail/solver_common.hpp index 82407a43..90126ab2 100644 --- a/libmpdata++/solvers/detail/solver_common.hpp +++ b/libmpdata++/solvers/detail/solver_common.hpp @@ -73,7 +73,7 @@ namespace libmpdataxx real_t time = 0; std::vector n; - using shmem_ref_t = concurr::detail::sharedmem_refined_common; + using shmem_ref_t = concurr::detail::sharedmem_refined; using shmem_t = concurr::detail::sharedmem; public: diff --git a/tests/sandbox/pbl/pbl.hpp b/tests/sandbox/pbl/pbl.hpp index 0b2f7079..481f1f6f 100644 --- a/tests/sandbox/pbl/pbl.hpp +++ b/tests/sandbox/pbl/pbl.hpp @@ -68,6 +68,7 @@ class pbl : public libmpdataxx::output::hdf5_xdmfrecord_aux_dsc("tke", this->tke); } this->record_aux_dsc("p", this->Phi); + this->record_aux_dsc_refined("tht refined", this->mem->refinee(ix::tht)); } this->mem->barrier(); } From 71f4d70c7b4ecb583f7cd47bf8d03e32f51800cd Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 7 Mar 2022 14:24:10 +0100 Subject: [PATCH 015/129] xmf: record refined aux --- libmpdata++/output/hdf5_xdmf.hpp | 10 ++++++++++ tests/sandbox/pbl/pbl.hpp | 1 + 2 files changed, 11 insertions(+) diff --git a/libmpdata++/output/hdf5_xdmf.hpp b/libmpdata++/output/hdf5_xdmf.hpp index 91425fce..5d94cd3e 100644 --- a/libmpdata++/output/hdf5_xdmf.hpp +++ b/libmpdata++/output/hdf5_xdmf.hpp @@ -103,6 +103,16 @@ namespace libmpdataxx parent_t::record_aux_dsc(name, arr, srfc); } + void record_aux_dsc_refined(const std::string &name, const typename solver_t::arr_t &arr) + { + auto shape = this->mem->distmem.grid_size_ref; +#if defined(USE_MPI) + if (this->mem->distmem.rank() == 0) +#endif + xdmfw.add_attribute(name, this->hdf_name(), shape.data()); + parent_t::record_aux_dsc_refined(name, arr); + } + public: // ctor diff --git a/tests/sandbox/pbl/pbl.hpp b/tests/sandbox/pbl/pbl.hpp index 481f1f6f..5251ada0 100644 --- a/tests/sandbox/pbl/pbl.hpp +++ b/tests/sandbox/pbl/pbl.hpp @@ -68,6 +68,7 @@ class pbl : public libmpdataxx::output::hdf5_xdmfrecord_aux_dsc("tke", this->tke); } this->record_aux_dsc("p", this->Phi); + this->mem->refinee(ix::tht) = blitz::tensor::k; this->record_aux_dsc_refined("tht refined", this->mem->refinee(ix::tht)); } this->mem->barrier(); From 0fe0730ebdf458c56835b38ac158ca8c27cdc7b6 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 7 Mar 2022 14:30:31 +0100 Subject: [PATCH 016/129] hdf5: store refined X,Y,Z TODO: make it conditional --- .../concurr/detail/sharedmem_refined.hpp | 3 +- libmpdata++/output/hdf5.hpp | 38 ++++++++++++++++++- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/libmpdata++/concurr/detail/sharedmem_refined.hpp b/libmpdata++/concurr/detail/sharedmem_refined.hpp index 77eb17e5..7e100bcc 100644 --- a/libmpdata++/concurr/detail/sharedmem_refined.hpp +++ b/libmpdata++/concurr/detail/sharedmem_refined.hpp @@ -29,11 +29,12 @@ namespace libmpdataxx using arr_t = typename parent_t::arr_t; - const int n_ref; // number of equal divisions of the large cell (in each direction) blitz::TinyVector origin_ref; public: + const int n_ref; // number of equal divisions of the large cell (in each direction) + std::array grid_size_ref; // TODO: these are public because used from outside in alloc - could friendship help? arrvec_t GC_ref, psi_ref; diff --git a/libmpdata++/output/hdf5.hpp b/libmpdata++/output/hdf5.hpp index ae734156..1cafa591 100644 --- a/libmpdata++/output/hdf5.hpp +++ b/libmpdata++/output/hdf5.hpp @@ -55,10 +55,10 @@ namespace libmpdataxx H5::PredType::NATIVE_FLOAT, flttype_output = H5::PredType::NATIVE_FLOAT; // using floats not to waste disk space - blitz::TinyVector cshape, shape, chunk, srfcshape, srfcchunk, offst, shape_ref, cshape_ref, chunk_ref, offst_ref; + blitz::TinyVector cshape, shape, chunk, srfcshape, srfcchunk, offst, shape_ref, cshape_ref, chunk_ref, offst_ref; // what if grid refinement is not done??? H5::DSetCreatPropList params; - H5::DataSpace sspace, cspace, srfcspace, sspace_ref; + H5::DataSpace sspace, cspace, srfcspace, sspace_ref, cspace_ref; #if defined(USE_MPI) hid_t fapl_id; #endif @@ -117,6 +117,7 @@ namespace libmpdataxx cspace = H5::DataSpace(parent_t::n_dims, cshape.data()); srfcspace = H5::DataSpace(parent_t::n_dims, srfcshape.data()); sspace_ref = H5::DataSpace(parent_t::n_dims, shape_ref.data()); + cspace_ref = H5::DataSpace(parent_t::n_dims, cshape_ref.data()); #if defined(USE_MPI) if (this->mem->distmem.size() > 1) @@ -188,6 +189,39 @@ namespace libmpdataxx curr_dim.write(coord.data(), flttype_solver, H5::DataSpace(parent_t::n_dims, cshape.data()), dim_space, dxpl_id); } + // refined X, Y, Z, TODO: very similar to X, Y, Z + for (int i = 0; i < parent_t::n_dims; ++i) + { + + blitz::Array coord(cshape_ref); +#if defined(USE_MPI) + coord.reindexSelf(offst_ref); +#endif + std::string name; + switch (i) + { + case 0 : coord = this->di / this->mem->n_ref * blitz::firstIndex(); + name = "X refined"; + dim_names[i] = name; + break; + case 1 : coord = this->dj / this->mem->n_ref * blitz::secondIndex(); + name = "Y refined"; + dim_names[i] = name; + break; + case 2 : coord = this->dk / this->mem->n_ref * blitz::thirdIndex(); + name = "Z refined"; + dim_names[i] = name; + break; + default : break; + } + + auto curr_dim = (*hdfp).createDataSet(name, flttype_output, cspace_ref); + + H5::DataSpace dim_space = curr_dim.getSpace(); + dim_space.selectHyperslab(H5S_SELECT_SET, cshape_ref.data(), offst_ref.data()); + curr_dim.write(coord.data(), flttype_solver, H5::DataSpace(parent_t::n_dims, cshape_ref.data()), dim_space, dxpl_id); + } + // T { const hsize_t From caaffb4a1fed301fcc1a485b6b6d335d91503a8b Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 7 Mar 2022 16:37:03 +0100 Subject: [PATCH 017/129] xmf writer: add reference writing --- libmpdata++/output/detail/xdmf_writer.hpp | 187 +++++++++++++--------- libmpdata++/output/hdf5_xdmf.hpp | 10 +- 2 files changed, 116 insertions(+), 81 deletions(-) diff --git a/libmpdata++/output/detail/xdmf_writer.hpp b/libmpdata++/output/detail/xdmf_writer.hpp index 173ebd4a..84c38ded 100644 --- a/libmpdata++/output/detail/xdmf_writer.hpp +++ b/libmpdata++/output/detail/xdmf_writer.hpp @@ -18,88 +18,101 @@ namespace libmpdataxx namespace detail { template - class xdmf_writer + struct data_item { using ptree = boost::property_tree::ptree; -#if (BOOST_VERSION >= 105600) - using xml_writer_settings = boost::property_tree::xml_writer_settings; -#else - using xml_writer_settings = boost::property_tree::xml_writer_settings; -#endif - struct data_item + blitz::TinyVector dimensions; + const std::string number_type = "Float"; + const std::string format = "HDF"; + std::string data; + void add(ptree& node) { - blitz::TinyVector dimensions; - const std::string number_type = "Float"; - const std::string format = "HDF"; - std::string data; - void add(ptree& node) - { - std::stringstream ss; - for (auto d : dimensions) - ss << d << ' '; - ptree& dat_node = node.add("DataItem", data); - dat_node.put(".Dimensions", ss.str()); - dat_node.put(".NumberType", number_type); - dat_node.put(".Format", format); - } - }; + std::stringstream ss; + for (auto d : dimensions) + ss << d << ' '; + ptree& dat_node = node.add("DataItem", data); + dat_node.put(".Dimensions", ss.str()); + dat_node.put(".NumberType", number_type); + dat_node.put(".Format", format); + } + }; - struct topology + template + struct topology + { + using ptree = boost::property_tree::ptree; + + const std::string topology_type = dim == 2 ? "2DSMesh" : "3DSMesh"; + blitz::TinyVector dimensions; + void add(ptree& node) { - const std::string topology_type = dim == 2 ? "2DSMesh" : "3DSMesh"; - blitz::TinyVector dimensions; - void add(ptree& node) - { - node.put("Topology..TopologyType", topology_type); - std::stringstream ss; - for (auto d : dimensions) - ss << d << ' '; - node.put("Topology..Dimensions", ss.str()); - } - }; + node.put("Topology..TopologyType", topology_type); + std::stringstream ss; + for (auto d : dimensions) + ss << d << ' '; + node.put("Topology..Dimensions", ss.str()); + } + }; + + template + struct geometry + { + using ptree = boost::property_tree::ptree; - struct geometry + const std::string geometry_type = dim == 2 ? "X_Y" : "X_Y_Z"; + std::array, dim> coords; + void add(ptree& node) { - const std::string geometry_type = dim == 2 ? "X_Y" : "X_Y_Z"; - std::array coords; - void add(ptree& node) - { - ptree& geo_node = node.add("Geometry", ""); - geo_node.put(".GeometryType", geometry_type); - for (auto &c : coords) - c.add(geo_node); - } - }; + ptree& geo_node = node.add("Geometry", ""); + geo_node.put(".GeometryType", geometry_type); + for (auto &c : coords) + c.add(geo_node); + } + }; + + template + struct attribute_t + { + using ptree = boost::property_tree::ptree; - struct attribute + std::string name; + const std::string attribute_type = "Scalar"; + const std::string center = "Cell"; + mutable data_item item; + void add(ptree& node) { - std::string name; - const std::string attribute_type = "Scalar"; - const std::string center = "Cell"; - mutable data_item item; - void add(ptree& node) - { - ptree& attr_node = node.add("Attribute", ""); - attr_node.put(".Name", name); - attr_node.put(".AttributeType", attribute_type); - attr_node.put(".Center", center); - item.add(attr_node); - } + ptree& attr_node = node.add("Attribute", ""); + attr_node.put(".Name", name); + attr_node.put(".AttributeType", attribute_type); + attr_node.put(".Center", center); + item.add(attr_node); + } - // to allow storing attributes in std::set - friend bool operator<(const attribute &lhs, const attribute &rhs) - { - return lhs.name < rhs.name; - } - }; + // to allow storing attributes in std::set + friend bool operator<(const attribute_t &lhs, const attribute_t &rhs) + { + return lhs.name < rhs.name; + } + }; + + template + class xdmf_writer + { + using ptree = boost::property_tree::ptree; +#if (BOOST_VERSION >= 105600) + using xml_writer_settings = boost::property_tree::xml_writer_settings; +#else + using xml_writer_settings = boost::property_tree::xml_writer_settings; +#endif + using attribute = attribute_t; const std::string name = "Grid"; const std::string grid_type = "Uniform"; - topology top; - geometry geo; - std::set attrs; - std::set c_attrs; + topology top, top_ref; + geometry geo, geo_ref; + std::set attrs, attrs_ref; + std::set c_attrs, c_attrs_ref; attribute make_attribute(const std::string& name, const blitz::TinyVector& dimensions) @@ -134,11 +147,13 @@ namespace libmpdataxx void add_attribute(const std::string& name, const std::string& hdf_name, - const blitz::TinyVector& dimensions) + const blitz::TinyVector& dimensions, + const bool refined = false) { + auto &_attrs(refined ? attrs_ref : attrs); attribute a = make_attribute(name, dimensions); a.item.data = hdf_name + ":/" + a.name; - attrs.insert(a); + _attrs.insert(a); } void add_const_attribute(const std::string& name, @@ -151,9 +166,15 @@ namespace libmpdataxx c_attrs.insert(a); } - void write(const std::string& xmf_name, const std::string& hdf_name, const double time) + void write_helper(const std::string& xmf_name, const std::string& hdf_name, const double time, const bool refined) { - for (auto& a : attrs) + const auto &_attrs(refined ? attrs_ref : attrs); + const auto &_c_attrs(refined ? c_attrs_ref : c_attrs); + auto &_top(refined ? top_ref : top); + auto &_geo(refined ? geo_ref : geo); + + + for (auto& a : _attrs) { a.item.data = hdf_name + ":/" + a.name; } @@ -165,21 +186,31 @@ namespace libmpdataxx grid_node.put(".xml:id", "gid"); grid_node.put("Time..Value", std::to_string(time)); - top.add(grid_node); + _top.add(grid_node); - geo.add(grid_node); + _geo.add(grid_node); - for (auto a : attrs) + for (auto a : _attrs) a.add(grid_node); - for (auto ca : c_attrs) + for (auto ca : _c_attrs) ca.add(grid_node); xml_writer_settings settings('\t', 1); write_xml(xmf_name, pt, std::locale(), settings); } - void write_temporal(const std::string& xmf_name, const std::vector& timesteps) + void write(const std::string& xmf_name, const std::string& hdf_name, const double time) + { + write_helper(xmf_name, hdf_name, time, false); + } + + void write_refined(const std::string& xmf_name, const std::string& hdf_name, const double time) + { + write_helper(xmf_name, hdf_name, time, true); + } + + void write_temporal(const std::string& xmf_name, const std::vector& timesteps, const std::string ×teps_suffix = "") { ptree pt; @@ -193,7 +224,7 @@ namespace libmpdataxx for (auto ts : timesteps) { ptree& ts_node = grid_node.add("xi:include", ""); - ts_node.put(".href", ts); + ts_node.put(".href", ts+timesteps_suffix); ts_node.put(".xpointer", "gid"); } diff --git a/libmpdata++/output/hdf5_xdmf.hpp b/libmpdata++/output/hdf5_xdmf.hpp index 5d94cd3e..e56f0e3e 100644 --- a/libmpdata++/output/hdf5_xdmf.hpp +++ b/libmpdata++/output/hdf5_xdmf.hpp @@ -70,10 +70,14 @@ namespace libmpdataxx std::string xmf_name = this->base_name() + ".xmf"; xdmfw.write(this->outdir + "/" + xmf_name, this->hdf_name(), this->record_time); + std::string xmf_ref_name = this->base_name() + "_ref.xmf"; + xdmfw.write_refined(this->outdir + "/" + xmf_ref_name, this->hdf_name(), this->record_time); + // save the xmf filename for temporal write - timesteps.push_back(xmf_name); + timesteps.push_back(this->base_name()); // write temporal xmf - xdmfw.write_temporal(this->outdir + "/temp.xmf", timesteps); + xdmfw.write_temporal(this->outdir + "/temp.xmf", timesteps, ".xmf"); + xdmfw.write_temporal(this->outdir + "/temp_ref.xmf", timesteps, "_ref.xmf"); // separate temporal with refined data } } @@ -109,7 +113,7 @@ namespace libmpdataxx #if defined(USE_MPI) if (this->mem->distmem.rank() == 0) #endif - xdmfw.add_attribute(name, this->hdf_name(), shape.data()); + xdmfw.add_attribute(name, this->hdf_name(), shape.data(), true); parent_t::record_aux_dsc_refined(name, arr); } From 1beeef17368a8219d72e8d31cf4ff029cd272f84 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 7 Mar 2022 17:01:37 +0100 Subject: [PATCH 018/129] xmf: write refined data using separate xmf_writer --- libmpdata++/output/detail/xdmf_writer.hpp | 45 +++++++---------------- libmpdata++/output/hdf5.hpp | 8 ++-- libmpdata++/output/hdf5_xdmf.hpp | 13 ++++--- 3 files changed, 24 insertions(+), 42 deletions(-) diff --git a/libmpdata++/output/detail/xdmf_writer.hpp b/libmpdata++/output/detail/xdmf_writer.hpp index 84c38ded..cd7a1c7d 100644 --- a/libmpdata++/output/detail/xdmf_writer.hpp +++ b/libmpdata++/output/detail/xdmf_writer.hpp @@ -109,10 +109,10 @@ namespace libmpdataxx const std::string name = "Grid"; const std::string grid_type = "Uniform"; - topology top, top_ref; - geometry geo, geo_ref; - std::set attrs, attrs_ref; - std::set c_attrs, c_attrs_ref; + topology top; + geometry geo; + std::set attrs; + std::set c_attrs; attribute make_attribute(const std::string& name, const blitz::TinyVector& dimensions) @@ -147,13 +147,11 @@ namespace libmpdataxx void add_attribute(const std::string& name, const std::string& hdf_name, - const blitz::TinyVector& dimensions, - const bool refined = false) + const blitz::TinyVector& dimensions) { - auto &_attrs(refined ? attrs_ref : attrs); attribute a = make_attribute(name, dimensions); a.item.data = hdf_name + ":/" + a.name; - _attrs.insert(a); + attrs.insert(a); } void add_const_attribute(const std::string& name, @@ -166,15 +164,9 @@ namespace libmpdataxx c_attrs.insert(a); } - void write_helper(const std::string& xmf_name, const std::string& hdf_name, const double time, const bool refined) + void write(const std::string& xmf_name, const std::string& hdf_name, const double time) { - const auto &_attrs(refined ? attrs_ref : attrs); - const auto &_c_attrs(refined ? c_attrs_ref : c_attrs); - auto &_top(refined ? top_ref : top); - auto &_geo(refined ? geo_ref : geo); - - - for (auto& a : _attrs) + for (auto& a : attrs) { a.item.data = hdf_name + ":/" + a.name; } @@ -186,30 +178,20 @@ namespace libmpdataxx grid_node.put(".xml:id", "gid"); grid_node.put("Time..Value", std::to_string(time)); - _top.add(grid_node); + top.add(grid_node); - _geo.add(grid_node); + geo.add(grid_node); - for (auto a : _attrs) + for (auto a : attrs) a.add(grid_node); - for (auto ca : _c_attrs) + for (auto ca : c_attrs) ca.add(grid_node); xml_writer_settings settings('\t', 1); write_xml(xmf_name, pt, std::locale(), settings); } - void write(const std::string& xmf_name, const std::string& hdf_name, const double time) - { - write_helper(xmf_name, hdf_name, time, false); - } - - void write_refined(const std::string& xmf_name, const std::string& hdf_name, const double time) - { - write_helper(xmf_name, hdf_name, time, true); - } - void write_temporal(const std::string& xmf_name, const std::vector& timesteps, const std::string ×teps_suffix = "") { @@ -224,14 +206,13 @@ namespace libmpdataxx for (auto ts : timesteps) { ptree& ts_node = grid_node.add("xi:include", ""); - ts_node.put(".href", ts+timesteps_suffix); + ts_node.put(".href", ts + timesteps_suffix); ts_node.put(".xpointer", "gid"); } xml_writer_settings settings('\t', 1); write_xml(xmf_name, pt, std::locale(), settings); } - }; } diff --git a/libmpdata++/output/hdf5.hpp b/libmpdata++/output/hdf5.hpp index 1cafa591..b066fcee 100644 --- a/libmpdata++/output/hdf5.hpp +++ b/libmpdata++/output/hdf5.hpp @@ -40,7 +40,7 @@ namespace libmpdataxx using output_t = hdf5; std::unique_ptr hdfp; - std::map dim_names; + std::map dim_names, dim_names_ref; const std::string const_name = "const.h5"; std::string const_file; const hsize_t zero = 0, one = 1; @@ -202,15 +202,15 @@ namespace libmpdataxx { case 0 : coord = this->di / this->mem->n_ref * blitz::firstIndex(); name = "X refined"; - dim_names[i] = name; + dim_names_ref[i] = name; break; case 1 : coord = this->dj / this->mem->n_ref * blitz::secondIndex(); name = "Y refined"; - dim_names[i] = name; + dim_names_ref[i] = name; break; case 2 : coord = this->dk / this->mem->n_ref * blitz::thirdIndex(); name = "Z refined"; - dim_names[i] = name; + dim_names_ref[i] = name; break; default : break; } diff --git a/libmpdata++/output/hdf5_xdmf.hpp b/libmpdata++/output/hdf5_xdmf.hpp index e56f0e3e..2eb3452a 100644 --- a/libmpdata++/output/hdf5_xdmf.hpp +++ b/libmpdata++/output/hdf5_xdmf.hpp @@ -36,8 +36,8 @@ namespace libmpdataxx static_assert(parent_t::n_dims > 1, "only 2D and 3D output supported"); std::vector timesteps; - //xdmf writer - detail::xdmf_writer xdmfw; + //xdmf writer, additional one for refined data (separate .xmf files describing the refined grid, TODO: use two grids in one file? Paraview AMR dataset could help?) + detail::xdmf_writer xdmfw, xdmfw_ref; void start(const typename parent_t::advance_arg_t nt) { @@ -56,7 +56,8 @@ namespace libmpdataxx if (this->mem->G.get() != nullptr) xdmfw.add_const_attribute("G", this->const_name, this->mem->distmem.grid_size.data()); - xdmfw.setup(this->const_name, this->dim_names, attr_names, this->mem->distmem.grid_size.data()); + xdmfw.setup( this->const_name, this->dim_names, attr_names, this->mem->distmem.grid_size.data() ); + xdmfw_ref.setup(this->const_name, this->dim_names_ref, {}, this->mem->distmem.grid_size_ref.data()); } } @@ -71,13 +72,13 @@ namespace libmpdataxx xdmfw.write(this->outdir + "/" + xmf_name, this->hdf_name(), this->record_time); std::string xmf_ref_name = this->base_name() + "_ref.xmf"; - xdmfw.write_refined(this->outdir + "/" + xmf_ref_name, this->hdf_name(), this->record_time); + xdmfw_ref.write(this->outdir + "/" + xmf_ref_name, this->hdf_name(), this->record_time); // save the xmf filename for temporal write timesteps.push_back(this->base_name()); // write temporal xmf xdmfw.write_temporal(this->outdir + "/temp.xmf", timesteps, ".xmf"); - xdmfw.write_temporal(this->outdir + "/temp_ref.xmf", timesteps, "_ref.xmf"); // separate temporal with refined data + xdmfw_ref.write_temporal(this->outdir + "/temp_ref.xmf", timesteps, "_ref.xmf"); } } @@ -113,7 +114,7 @@ namespace libmpdataxx #if defined(USE_MPI) if (this->mem->distmem.rank() == 0) #endif - xdmfw.add_attribute(name, this->hdf_name(), shape.data(), true); + xdmfw_ref.add_attribute(name, this->hdf_name(), shape.data()); parent_t::record_aux_dsc_refined(name, arr); } From 54e392b71702077e6e3d3cc499a18fe609a60faa Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 16 Mar 2022 12:22:04 +0100 Subject: [PATCH 019/129] fix grid size refinement strategy --- libmpdata++/concurr/detail/sharedmem.hpp | 2 -- .../concurr/detail/sharedmem_refined.hpp | 33 ++++++++++++------- .../detail/mpdata_rhs_vip_prs_sgs_fra.hpp | 4 +-- 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/libmpdata++/concurr/detail/sharedmem.hpp b/libmpdata++/concurr/detail/sharedmem.hpp index 32249738..9e888025 100644 --- a/libmpdata++/concurr/detail/sharedmem.hpp +++ b/libmpdata++/concurr/detail/sharedmem.hpp @@ -97,8 +97,6 @@ namespace libmpdataxx rng_t(0, grid_size[d]-1), d == 0 ? distmem.rank() : 0, // decomposition along x, because that's MPI decomposition d == 0 ? distmem.size() : 1 - // d == shmem_decomp_dim ? distmem.rank() : 0, - // d == shmem_decomp_dim ? distmem.size() : 1 ); origin[d] = this->grid_size[d].first(); } diff --git a/libmpdata++/concurr/detail/sharedmem_refined.hpp b/libmpdata++/concurr/detail/sharedmem_refined.hpp index 7e100bcc..2266ab7e 100644 --- a/libmpdata++/concurr/detail/sharedmem_refined.hpp +++ b/libmpdata++/concurr/detail/sharedmem_refined.hpp @@ -33,7 +33,12 @@ namespace libmpdataxx public: - const int n_ref; // number of equal divisions of the large cell (in each direction) + const int n_ref; // number of equal divisions of the large cell (in each direction), refined resolution is dx / n_ref; + // every n_ref scalar of the refined grid is at the same position as a scalar of the normal grid + // no refinement done in the halo, because there are no SD in the halo (it's not real space) + // what about MPI boundaries? there are refined points exactly at the boundary (since n_ref has to be even) + // note: if there is a refined cell that is divided by the MPI boundary, o we need to add contributions from microphysics to both processes on both sides? + // maybe not, because microphysics contrbutions will affect the large cells, which are not divided by the MPI boundary... std::array grid_size_ref; // TODO: these are public because used from outside in alloc - could friendship help? @@ -43,11 +48,18 @@ namespace libmpdataxx sharedmem_refined_common(const std::array &grid_size, const int &size, const int &n_ref) : parent_t(grid_size, size), n_ref(n_ref) { + assert(n_ref % 2 == 0); // only division into even number of cells, because we assume that one of the refined scalar points is at the MPI boundary, which is in the middle between normal grid scalars for (int d = 0; d < n_dims; ++d) { - grid_size_ref[d] = refine_grid_size(this->grid_size[d], n_ref); + grid_size_ref[d] = refine_grid_size( + this->grid_size[d], + n_ref, + d == 0 ? this->distmem.rank() : 0, + d == 0 ? this->distmem.size() : 1 + ); origin_ref[d] = grid_size_ref[d].first(); - this->distmem.grid_size_ref[d] = grid_size_ref[d].last() - grid_size_ref[d].first() + 1; + + this->distmem.grid_size_ref[d] = refine_grid_size(rng_t(0,grid_size[d]-1), n_ref, 0, 1).length(); } } @@ -57,11 +69,14 @@ namespace libmpdataxx public: static rng_t refine_grid_size( const rng_t &grid_size, - const int &n_ref + const int &n_ref, + const int &mpi_rank, + const int &mpi_size ) { + assert(n_ref % 2 == 0); return rng_t( - grid_size.first() * n_ref, - (grid_size.last() + 1) * n_ref - 1 + mpi_rank == 0 ? grid_size.first() * n_ref : grid_size.first() * n_ref - (n_ref / 2), + mpi_rank == mpi_size-1 ? grid_size.last() * n_ref : grid_size.last() * n_ref + (n_ref / 2) ); } }; @@ -84,12 +99,6 @@ namespace libmpdataxx public: arr_t refinee(int e = 0) override { - std::cerr << "refinee: " << this->psi_ref[e]( - this->grid_size_ref[0], - this->grid_size_ref[1], - this->grid_size_ref[2] - ).reindex(this->origin_ref); - return this->psi_ref[e]( this->grid_size_ref[0], this->grid_size_ref[1], diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra.hpp index b05ec22b..9b4c24d0 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra.hpp @@ -56,8 +56,8 @@ namespace libmpdataxx const int &n_iters ) { parent_t::alloc(mem, n_iters); - parent_t::alloc_tmp_sclr(mem, __FILE__, ct_params_t::n_eqns, mem->grid_size_ref, "", false); // rec_psi - parent_t::alloc_tmp_vctr(mem, __FILE__, mem->grid_size_ref); // rec_GC + parent_t::alloc_tmp_sclr(mem, __FILE__, ct_params_t::n_eqns, mem->grid_size_ref, "", false); // psi_ref + parent_t::alloc_tmp_vctr(mem, __FILE__, mem->grid_size_ref); // GC_ref mem->psi_ref = mem->tmp[__FILE__][0]; mem->GC_ref = mem->tmp[__FILE__][1]; From fb08f524ee90ae0798747163b08ede4d5488acdb Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 16 Mar 2022 17:31:23 +0100 Subject: [PATCH 020/129] wip on refinement: fixed positions in output, added refined ijk --- libmpdata++/blitz.hpp | 1 + libmpdata++/formulae/domain_decomposition.hpp | 6 +-- libmpdata++/output/hdf5.hpp | 6 +-- .../detail/mpdata_rhs_vip_prs_sgs_common.hpp | 4 +- .../detail/mpdata_rhs_vip_prs_sgs_fra.hpp | 45 +++++++++++++++++-- tests/sandbox/pbl/pbl.hpp | 11 ++++- tests/sandbox/pbl/pbl_test_def.hpp | 1 + 7 files changed, 62 insertions(+), 12 deletions(-) diff --git a/libmpdata++/blitz.hpp b/libmpdata++/blitz.hpp index 53273bde..8a54410a 100644 --- a/libmpdata++/blitz.hpp +++ b/libmpdata++/blitz.hpp @@ -51,6 +51,7 @@ namespace libmpdataxx { template using idx_t = blitz::RectDomain; + template using idxs_t = blitz::StridedDomain; using rng_t = blitz::Range; // non-int ix_t means either rng_t or idx_t diff --git a/libmpdata++/formulae/domain_decomposition.hpp b/libmpdata++/formulae/domain_decomposition.hpp index 21594610..ccf47b3e 100644 --- a/libmpdata++/formulae/domain_decomposition.hpp +++ b/libmpdata++/formulae/domain_decomposition.hpp @@ -15,19 +15,19 @@ namespace libmpdataxx namespace detail { // helper methods to define subdomain ranges - static int min(const int &span, const int &rank, const int &size) + int min(const int &span, const int &rank, const int &size) { return rank * span / size; } - static int max(const int &span, const int &rank, const int &size) + int max(const int &span, const int &rank, const int &size) { return min(span, rank + 1, size) - 1; } }; // get part of 'span' assigned to 'rank' (out of 'size' ranks) - static rng_t slab( + rng_t slab( const rng_t &span, const int &rank = 0, const int &size = 1 diff --git a/libmpdata++/output/hdf5.hpp b/libmpdata++/output/hdf5.hpp index b066fcee..87f07ac7 100644 --- a/libmpdata++/output/hdf5.hpp +++ b/libmpdata++/output/hdf5.hpp @@ -200,15 +200,15 @@ namespace libmpdataxx std::string name; switch (i) { - case 0 : coord = this->di / this->mem->n_ref * blitz::firstIndex(); + case 0 : coord = this->di / 4. + this->di / this->mem->n_ref * blitz::firstIndex(); name = "X refined"; dim_names_ref[i] = name; break; - case 1 : coord = this->dj / this->mem->n_ref * blitz::secondIndex(); + case 1 : coord = this->dj / 4. + this->dj / this->mem->n_ref * blitz::secondIndex(); name = "Y refined"; dim_names_ref[i] = name; break; - case 2 : coord = this->dk / this->mem->n_ref * blitz::thirdIndex(); + case 2 : coord = this->dk / 4. + this->dk / this->mem->n_ref * blitz::thirdIndex(); name = "Z refined"; dim_names_ref[i] = name; break; diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_common.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_common.hpp index 07d15f50..421dbeff 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_common.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_common.hpp @@ -50,6 +50,8 @@ namespace libmpdataxx arrvec_t &drv; arrvec_t &wrk; + // TODO: make all ijk... objects const + // like ijk, but containing vectors at the lower/left/fre edge // CAUTION: on sharedmem, ijkm contains overlapping ranges idx_t ijkm; @@ -60,7 +62,7 @@ namespace libmpdataxx // like ijk, but with range in x direction extended by 1 to the left for MPI compliance. // MPI requires that vector between two process domains is calculated by the process to the right of it (c.f. remote_2d.hpp fill_halos_vctr_alng) // TODO: change MPI logic to assume that it is calculated by the process to the left? then, ijk_vec would not be needed(?) - std::array ijk_vec; + std::array ijk_vec; // TODO: replace std::array with idx_t real_t cdrag; diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra.hpp index 9b4c24d0..26bb8963 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra.hpp @@ -6,6 +6,8 @@ * */ +// solver with fractal reconstruction of SGS fields + #pragma once #include @@ -30,9 +32,11 @@ namespace libmpdataxx protected: // const int n_fra; // number of fields with fractal reconstruction -// constexpr int n_rec_cell = pow(2, ct_params::n_fra_rec) ; // number of reconstructed cells in each direction + const int n_ref; // number of refinements; refined resolution is dx / n_ref - // idx_t ijkm; + // TODO: make these const! + idx_t ijk_ref; // range of refinee handled by given solver, excluding halos + const idxs_t ijk_r2r; // resolved to refined; refined scalars at the same position as resolved scalars public: @@ -46,9 +50,42 @@ namespace libmpdataxx typename parent_t::ctor_args_t args, const rt_params_t &p ) : - parent_t(args, p) + parent_t(args, p), + n_ref(this->mem->n_ref), + ijk_r2r{ + {this->ijk[0].first() * n_ref, this->ijk[1].first() * n_ref, this->ijk[2].first() * n_ref}, // lbound + {this->ijk[0].last() * n_ref, this->ijk[1].last() * n_ref, this->ijk[2].last() * n_ref}, // ubound + {n_ref, n_ref, n_ref}, // stride + } { - // for (int d = 0; d < ct_params_t::n_dims; ++d) + for (int d = 0; d < ct_params_t::n_dims; ++d) + { + // ijk_ref init below assumes 3D (shmem decomp dim along y); + // TODO: move this to concurr_common::init()? add something to ctor_args_t? + + if(d==1) + { + ijk_ref.lbound(d) = this->mem->slab(this->mem->grid_size_ref[d], this->rank, this->mem->size).first();// this->ijk.lbound(d) * n_ref; + ijk_ref.ubound(d) = this->mem->slab(this->mem->grid_size_ref[d], this->rank, this->mem->size).last(); + } + else + { + ijk_ref.lbound(d) = this->mem->grid_size_ref[d].first(); + ijk_ref.ubound(d) = this->mem->grid_size_ref[d].last(); + } + + std::cerr << "ijk_ref: " << ijk_ref[d] << std::endl; + std::cerr << "grid_size_ref: " << this->mem->grid_size_ref[d] << std::endl; + +// StidedDomain can only be constructed, cant be modified +// ijk_r2r.lbound(d) = this->ijk[d].first() * n_ref; +// ijk_r2r.ubound(d) = this->ijk[d].last() * n_ref; +// ijk_r2r.stride(d) = n_ref; + + std::cerr << "ijk_r2r: " << ijk_r2r[d] << std::endl; + + std::cerr << this->ijk[d].first() * n_ref << " " << this->ijk[d].last() * n_ref << " " << n_ref << std::endl; + } } static void alloc( diff --git a/tests/sandbox/pbl/pbl.hpp b/tests/sandbox/pbl/pbl.hpp index 5251ada0..25f6fe58 100644 --- a/tests/sandbox/pbl/pbl.hpp +++ b/tests/sandbox/pbl/pbl.hpp @@ -60,6 +60,16 @@ class pbl : public libmpdataxx::output::hdf5_xdmftimestep % static_cast(this->outfreq) == 0) { if (this->rank == 0) std::cout << this->timestep << std::endl; + + // test filling of refinee + this->mem->refinee(ix::tht)(this->ijk_ref) = blitz::tensor::k; // anything + //this->mem->refinee(ix::tht) = blitz::tensor::k; // anything + // copy tht to refined tht at position where they overlap + std::cerr << "refinee: " << this->mem->refinee(ix::tht); + std::cerr << "refinee(ijk_ref): " << this->mem->refinee(ix::tht)(this->ijk_ref); + std::cerr << "refinee(ijk_r2r): " << this->mem->refinee(ix::tht)(this->ijk_r2r); + this->mem->refinee(ix::tht)(this->ijk_r2r) = this->mem->advectee(ix::tht)(this->ijk); + this->mem->barrier(); if (this->rank == 0) { @@ -68,7 +78,6 @@ class pbl : public libmpdataxx::output::hdf5_xdmfrecord_aux_dsc("tke", this->tke); } this->record_aux_dsc("p", this->Phi); - this->mem->refinee(ix::tht) = blitz::tensor::k; this->record_aux_dsc_refined("tht refined", this->mem->refinee(ix::tht)); } this->mem->barrier(); diff --git a/tests/sandbox/pbl/pbl_test_def.hpp b/tests/sandbox/pbl/pbl_test_def.hpp index bb0270e1..c8601420 100644 --- a/tests/sandbox/pbl/pbl_test_def.hpp +++ b/tests/sandbox/pbl/pbl_test_def.hpp @@ -22,6 +22,7 @@ template void set_sgs_specific(params_t &p, iles_tag) { p.iles_cdrag = 0.1; + p.n_fra_iter = 1; } // smagorinsky From e9984b57a74838443a05234cdac9bb625f278914 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 17 Mar 2022 15:13:43 +0100 Subject: [PATCH 021/129] refined fields without halos --- .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 95 +++++++++++++++++++ ... => mpdata_rhs_vip_prs_sgs_fra_common.hpp} | 44 +++------ libmpdata++/solvers/detail/solver_1d.hpp | 9 -- libmpdata++/solvers/detail/solver_2d.hpp | 14 +-- libmpdata++/solvers/detail/solver_3d.hpp | 20 +--- .../solvers/mpdata_rhs_vip_prs_sgs_fra.hpp | 35 ++++--- tests/sandbox/pbl/pbl_test_def.hpp | 2 +- 7 files changed, 132 insertions(+), 87 deletions(-) create mode 100644 libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp rename libmpdata++/solvers/detail/{mpdata_rhs_vip_prs_sgs_fra.hpp => mpdata_rhs_vip_prs_sgs_fra_common.hpp} (62%) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp new file mode 100644 index 00000000..86459af5 --- /dev/null +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -0,0 +1,95 @@ +/** @file + * @copyright University of Warsaw + * @section LICENSE + * GPLv3+ (see the COPYING file or http://www.gnu.org/licenses/) + */ + +#pragma once + +#include + +namespace libmpdataxx +{ + namespace solvers + { + namespace detail + { + template + class mpdata_rhs_vip_prs_sgs_fra_dim + { +// static_assert(false); + }; + + template + class mpdata_rhs_vip_prs_sgs_fra_dim< + ct_params_t, + minhalo, + typename std::enable_if_t + > : public detail::mpdata_rhs_vip_prs_sgs_fra_common + { + using parent_t = detail::mpdata_rhs_vip_prs_sgs_fra_common; + using parent_t::parent_t; // inheriting constructors + + protected: + + // interpolation similar to mpdata_rhs_vip... + template + void intrp( + arr_t &dst, + const arr_t &src, + const rng_t &i, + const rng_t &j, + const rng_t &k, + const int &dist + ) + { + using idxperm::pi; + using namespace arakawa_c; + using real_t = typename ct_params_t::real_t; + + dst(pi(i, j, k)) = real_t(.5) * ( + src(pi(i - dist, j, k)) + + src(pi(i + dist, j, k)) + ); + } + + void interpolate_refinee(arrvec_t &dst, + const arrvec_t &src, + const int stride) + { + using namespace arakawa_c; // for rng_t operator^ + assert(dist % 2 == 0); + const auto hstride = stride / 2; + +// auto ex = this->halo - 1; + intrp<0>(dst[0], src[0], this->ijk_r2r[0]^hstride, this->ijk_r2r[1], this->ijk_r2r[2], hstride); + intrp<1>(dst[1], src[1], this->ijk_r2r[1]^hstride, this->ijk_r2r[2], this->ijk_r2r[0], hstride); + intrp<2>(dst[2], src[2], this->ijk_r2r[2]^hstride, this->ijk_r2r[0], this->ijk_r2r[1], hstride); + } + + public: + + // helper method to allocate n_arr refined scalar temporary arrays + static void alloc_tmp_sclr_ref( + typename parent_t::mem_t *mem, + const char * __file__, const int n_arr, + std::string name = "", + bool srfc = false + ) + { + mem->tmp[__file__].push_back(new arrvec_t()); + + if (!name.empty()) mem->avail_tmp[name] = std::make_pair(__file__, mem->tmp[__file__].size() - 1); + + for (int n = 0; n < n_arr; ++n) + mem->tmp[__file__].back().push_back(mem->old(new typename parent_t::arr_t( + mem->grid_size_ref[0], // NOTE: no halo + mem->grid_size_ref[1], + srfc ? rng_t(0, 0) : mem->grid_size_ref[2], + arr3D_storage + ))); + } + }; + } // namespace detail + } // namespace solvers +} // namespace libmpdataxx diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp similarity index 62% rename from libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra.hpp rename to libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp index 26bb8963..6f26eed6 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp @@ -21,7 +21,7 @@ namespace libmpdataxx namespace detail { template - class mpdata_rhs_vip_prs_sgs_fra : public mpdata_rhs_vip_prs_sgs + class mpdata_rhs_vip_prs_sgs_fra_common : public mpdata_rhs_vip_prs_sgs { using parent_t = mpdata_rhs_vip_prs_sgs; @@ -33,11 +33,16 @@ namespace libmpdataxx // const int n_fra; // number of fields with fractal reconstruction const int n_ref; // number of refinements; refined resolution is dx / n_ref + + // refined arrays have no halo, because halo size needs to be known at compile time (why is it so? probably not really necessary?) +// const int halo_ref; // size of halos of refined scalar (and vector?) arrays // TODO: make these const! - idx_t ijk_ref; // range of refinee handled by given solver, excluding halos + idx_t ijk_ref; // range of refinee handled by given solver const idxs_t ijk_r2r; // resolved to refined; refined scalars at the same position as resolved scalars +// static rng_t rng_sclr_ref(const rng_t &rng) { return rng^halo_ref; } + public: struct rt_params_t : parent_t::rt_params_t @@ -46,23 +51,26 @@ namespace libmpdataxx }; // ctor - mpdata_rhs_vip_prs_sgs_fra( + mpdata_rhs_vip_prs_sgs_fra_common( typename parent_t::ctor_args_t args, const rt_params_t &p ) : parent_t(args, p), n_ref(this->mem->n_ref), +// halo_ref(n_ref / 2), + // ijk_ref init below assumes 3D (shmem decomp dim along y); + // TODO: move this to concurr_common::init()? add something to ctor_args_t? ijk_r2r{ {this->ijk[0].first() * n_ref, this->ijk[1].first() * n_ref, this->ijk[2].first() * n_ref}, // lbound {this->ijk[0].last() * n_ref, this->ijk[1].last() * n_ref, this->ijk[2].last() * n_ref}, // ubound {n_ref, n_ref, n_ref}, // stride } { + assert(p.n_fra_iter > 0); + assert(n_ref & 2 == 0); + for (int d = 0; d < ct_params_t::n_dims; ++d) { - // ijk_ref init below assumes 3D (shmem decomp dim along y); - // TODO: move this to concurr_common::init()? add something to ctor_args_t? - if(d==1) { ijk_ref.lbound(d) = this->mem->slab(this->mem->grid_size_ref[d], this->rank, this->mem->size).first();// this->ijk.lbound(d) * n_ref; @@ -73,32 +81,8 @@ namespace libmpdataxx ijk_ref.lbound(d) = this->mem->grid_size_ref[d].first(); ijk_ref.ubound(d) = this->mem->grid_size_ref[d].last(); } - - std::cerr << "ijk_ref: " << ijk_ref[d] << std::endl; - std::cerr << "grid_size_ref: " << this->mem->grid_size_ref[d] << std::endl; - -// StidedDomain can only be constructed, cant be modified -// ijk_r2r.lbound(d) = this->ijk[d].first() * n_ref; -// ijk_r2r.ubound(d) = this->ijk[d].last() * n_ref; -// ijk_r2r.stride(d) = n_ref; - - std::cerr << "ijk_r2r: " << ijk_r2r[d] << std::endl; - - std::cerr << this->ijk[d].first() * n_ref << " " << this->ijk[d].last() * n_ref << " " << n_ref << std::endl; } } - - static void alloc( - typename parent_t::mem_t *mem, - const int &n_iters - ) { - parent_t::alloc(mem, n_iters); - parent_t::alloc_tmp_sclr(mem, __FILE__, ct_params_t::n_eqns, mem->grid_size_ref, "", false); // psi_ref - parent_t::alloc_tmp_vctr(mem, __FILE__, mem->grid_size_ref); // GC_ref - - mem->psi_ref = mem->tmp[__FILE__][0]; - mem->GC_ref = mem->tmp[__FILE__][1]; - } }; } // namespace detail } // namespace solvers diff --git a/libmpdata++/solvers/detail/solver_1d.hpp b/libmpdata++/solvers/detail/solver_1d.hpp index 4adfe74c..336284fd 100644 --- a/libmpdata++/solvers/detail/solver_1d.hpp +++ b/libmpdata++/solvers/detail/solver_1d.hpp @@ -217,15 +217,6 @@ namespace libmpdataxx } // helper method to allocate n_arr scalar temporary arrays - static void alloc_tmp_sclr( - typename parent_t::mem_t *mem, - const char * __file__, const int n_arr, - const std::array grid_size - ) - { - alloc_tmp(mem, __file__, n_arr, parent_t::rng_sclr(grid_size[0])); - } - static void alloc_tmp_sclr( typename parent_t::mem_t *mem, const char * __file__, const int n_arr diff --git a/libmpdata++/solvers/detail/solver_2d.hpp b/libmpdata++/solvers/detail/solver_2d.hpp index f76aea4d..e9e515e6 100644 --- a/libmpdata++/solvers/detail/solver_2d.hpp +++ b/libmpdata++/solvers/detail/solver_2d.hpp @@ -414,9 +414,8 @@ namespace libmpdataxx static void alloc_tmp_sclr( typename parent_t::mem_t *mem, const char * __file__, const int n_arr, - const std::array grid_size, std::string name = "", - bool srfc = false + bool srfc = false ) { mem->tmp[__file__].push_back(new arrvec_t()); @@ -429,17 +428,6 @@ namespace libmpdataxx srfc ? rng_t(0, 0) : parent_t::rng_sclr(mem->grid_size[1]) ))); } - - // with default grid_size - static void alloc_tmp_sclr( - typename parent_t::mem_t *mem, - const char * __file__, const int n_arr, - std::string name = "", - bool srfc = false - ) - { - alloc_tmp_sclr(mem, __file__, n_arr, mem->grid_size, name, srfc); - } }; } // namespace detail } // namespace solvers diff --git a/libmpdata++/solvers/detail/solver_3d.hpp b/libmpdata++/solvers/detail/solver_3d.hpp index 6d13a34c..3391dd2b 100644 --- a/libmpdata++/solvers/detail/solver_3d.hpp +++ b/libmpdata++/solvers/detail/solver_3d.hpp @@ -540,9 +540,8 @@ namespace libmpdataxx static void alloc_tmp_sclr( typename parent_t::mem_t *mem, const char * __file__, const int n_arr, - const std::array grid_size, std::string name = "", - bool srfc = false // allocate only surface data + bool srfc = false ) { mem->tmp[__file__].push_back(new arrvec_t()); @@ -551,23 +550,12 @@ namespace libmpdataxx for (int n = 0; n < n_arr; ++n) mem->tmp[__file__].back().push_back(mem->old(new typename parent_t::arr_t( - parent_t::rng_sclr(grid_size[0]), - parent_t::rng_sclr(grid_size[1]), - srfc ? rng_t(0, 0) : parent_t::rng_sclr(grid_size[2]), + parent_t::rng_sclr(mem->grid_size[0]), + parent_t::rng_sclr(mem->grid_size[1]), + srfc ? rng_t(0, 0) : parent_t::rng_sclr(mem->grid_size[2]), arr3D_storage ))); } - - // with default grid_size - static void alloc_tmp_sclr( - typename parent_t::mem_t *mem, - const char * __file__, const int n_arr, - std::string name = "", - bool srfc = false - ) - { - alloc_tmp_sclr(mem, __file__, n_arr, mem->grid_size, name, srfc); - } }; } // namespace detail } // namespace solvers diff --git a/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp b/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp index 4fa8a1b7..1fa3534d 100644 --- a/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp +++ b/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp @@ -8,7 +8,7 @@ #pragma once -#include +#include namespace libmpdataxx { @@ -19,12 +19,11 @@ namespace libmpdataxx template class mpdata_rhs_vip_prs_sgs_fra; + // no fields with fractal reconstruction, inherit from mpdata_rhs_vip_prs_sgs template class mpdata_rhs_vip_prs_sgs_fra< ct_params_t, minhalo, - typename std::enable_if_t<(int)ct_params_t::fractal_recon == (int)0> // no fields with fractal reconstruction -// typename std::enable_if_t<(int)ct_params_t::fractal_recon == (int) 0> - // std::enable_if_t<(int)ct_params_t::fractal_recon == 0, bool> = true + typename std::enable_if_t<(int)ct_params_t::fractal_recon == (int)0> > : public mpdata_rhs_vip_prs_sgs { using parent_t = mpdata_rhs_vip_prs_sgs; @@ -35,25 +34,25 @@ namespace libmpdataxx class mpdata_rhs_vip_prs_sgs_fra< ct_params_t, minhalo, typename std::enable_if_t<(int)ct_params_t::fractal_recon != (int)0> - //typename std::enable_if_t<(int)ct_params_t::fractal_recon != (int) 0> - // std::enable_if_t<(int)ct_params_t::fractal_recon > 0, bool> = true - > : public detail::mpdata_rhs_vip_prs_sgs_fra + > : public detail::mpdata_rhs_vip_prs_sgs_fra_dim { - using parent_t = detail::mpdata_rhs_vip_prs_sgs_fra; + using parent_t = detail::mpdata_rhs_vip_prs_sgs_fra_dim; using parent_t::parent_t; // inheriting constructors protected: using solver_family = mpdata_rhs_vip_prs_sgs_fra_family_tag; - }; -// template -// class mpdata_rhs_vip_prs_sgs_fra : public detail::mpdata_rhs_vip_prs_sgs_fra -// { -// using parent_t = detail::mpdata_rhs_vip_prs_sgs_fra; -// using parent_t::parent_t; // inheriting constructors -// -// protected: -// using solver_family = mpdata_rhs_vip_prs_sgs_fra_family_tag; -// }; + static void alloc( + typename parent_t::mem_t *mem, + const int &n_iters + ) { + parent_t::alloc(mem, n_iters); + parent_t::alloc_tmp_sclr_ref(mem, __FILE__, ct_params_t::n_eqns); // psi_ref +// parent_t::alloc_tmp_vctr(mem, __FILE__, mem->grid_size_ref); // GC_ref + + mem->psi_ref = mem->tmp[__FILE__][0]; +// mem->GC_ref = mem->tmp[__FILE__][1]; + } + }; } // namespace solvers } // namespace libmpdataxx diff --git a/tests/sandbox/pbl/pbl_test_def.hpp b/tests/sandbox/pbl/pbl_test_def.hpp index c8601420..c853f78b 100644 --- a/tests/sandbox/pbl/pbl_test_def.hpp +++ b/tests/sandbox/pbl/pbl_test_def.hpp @@ -22,7 +22,7 @@ template void set_sgs_specific(params_t &p, iles_tag) { p.iles_cdrag = 0.1; - p.n_fra_iter = 1; + p.n_fra_iter = 2; } // smagorinsky From 3f540a959e39e235c2c515dcf2baf45aefd0abdf Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 17 Mar 2022 16:06:09 +0100 Subject: [PATCH 022/129] refined grid: halo of n_ref/2 --- .../solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 6 +++--- .../solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 86459af5..de4d74d1 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -83,9 +83,9 @@ namespace libmpdataxx for (int n = 0; n < n_arr; ++n) mem->tmp[__file__].back().push_back(mem->old(new typename parent_t::arr_t( - mem->grid_size_ref[0], // NOTE: no halo - mem->grid_size_ref[1], - srfc ? rng_t(0, 0) : mem->grid_size_ref[2], + mem->grid_size_ref[0]^(mem->n_ref/2), // NOTE: halo of size n_ref / 2 + mem->grid_size_ref[1]^(mem->n_ref/2), + srfc ? rng_t(0, 0) : mem->grid_size_ref[2]^(mem->n_ref/2), arr3D_storage ))); } diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp index 6f26eed6..726a87d4 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp @@ -34,8 +34,8 @@ namespace libmpdataxx // const int n_fra; // number of fields with fractal reconstruction const int n_ref; // number of refinements; refined resolution is dx / n_ref - // refined arrays have no halo, because halo size needs to be known at compile time (why is it so? probably not really necessary?) -// const int halo_ref; // size of halos of refined scalar (and vector?) arrays + // TODO: do refined arrays really need to have halos? + const int halo_ref; // size of halos of refined scalar (and vector?) arrays // TODO: make these const! idx_t ijk_ref; // range of refinee handled by given solver @@ -57,7 +57,7 @@ namespace libmpdataxx ) : parent_t(args, p), n_ref(this->mem->n_ref), -// halo_ref(n_ref / 2), + halo_ref(n_ref / 2), // ijk_ref init below assumes 3D (shmem decomp dim along y); // TODO: move this to concurr_common::init()? add something to ctor_args_t? ijk_r2r{ From 87df456e7e225c79d4241b18d622c31fb1ca4bef Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 17 Mar 2022 16:23:00 +0100 Subject: [PATCH 023/129] Revert "refined grid: halo of n_ref/2" This reverts commit 3f540a959e39e235c2c515dcf2baf45aefd0abdf. --- .../solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 6 +++--- .../solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index de4d74d1..86459af5 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -83,9 +83,9 @@ namespace libmpdataxx for (int n = 0; n < n_arr; ++n) mem->tmp[__file__].back().push_back(mem->old(new typename parent_t::arr_t( - mem->grid_size_ref[0]^(mem->n_ref/2), // NOTE: halo of size n_ref / 2 - mem->grid_size_ref[1]^(mem->n_ref/2), - srfc ? rng_t(0, 0) : mem->grid_size_ref[2]^(mem->n_ref/2), + mem->grid_size_ref[0], // NOTE: no halo + mem->grid_size_ref[1], + srfc ? rng_t(0, 0) : mem->grid_size_ref[2], arr3D_storage ))); } diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp index 726a87d4..6f26eed6 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp @@ -34,8 +34,8 @@ namespace libmpdataxx // const int n_fra; // number of fields with fractal reconstruction const int n_ref; // number of refinements; refined resolution is dx / n_ref - // TODO: do refined arrays really need to have halos? - const int halo_ref; // size of halos of refined scalar (and vector?) arrays + // refined arrays have no halo, because halo size needs to be known at compile time (why is it so? probably not really necessary?) +// const int halo_ref; // size of halos of refined scalar (and vector?) arrays // TODO: make these const! idx_t ijk_ref; // range of refinee handled by given solver @@ -57,7 +57,7 @@ namespace libmpdataxx ) : parent_t(args, p), n_ref(this->mem->n_ref), - halo_ref(n_ref / 2), +// halo_ref(n_ref / 2), // ijk_ref init below assumes 3D (shmem decomp dim along y); // TODO: move this to concurr_common::init()? add something to ctor_args_t? ijk_r2r{ From aba74b3b564eec52e9ade816b93b3828f4ba4ecd Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 17 Mar 2022 17:39:40 +0100 Subject: [PATCH 024/129] wip on refined interpolation --- .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 41 +++++++++++++------ .../mpdata_rhs_vip_prs_sgs_fra_common.hpp | 9 +++- tests/sandbox/pbl/pbl.hpp | 4 +- 3 files changed, 39 insertions(+), 15 deletions(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 86459af5..390fc21d 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -35,8 +35,7 @@ namespace libmpdataxx // interpolation similar to mpdata_rhs_vip... template void intrp( - arr_t &dst, - const arr_t &src, + arr_t arr, const rng_t &i, const rng_t &j, const rng_t &k, @@ -47,24 +46,40 @@ namespace libmpdataxx using namespace arakawa_c; using real_t = typename ct_params_t::real_t; - dst(pi(i, j, k)) = real_t(.5) * ( - src(pi(i - dist, j, k)) + - src(pi(i + dist, j, k)) + arr(pi(i, j, k)) = real_t(.5) * ( + arr(pi(i - dist, j, k)) + + arr(pi(i + dist, j, k)) ); } - void interpolate_refinee(arrvec_t &dst, - const arrvec_t &src, - const int stride) + void interpolate_refinee(const int e = 0) +// arrvec_t &dst, +// const arrvec_t &src, +// const int stride) { using namespace arakawa_c; // for rng_t operator^ - assert(dist % 2 == 0); + const auto stride = this->ijk_r2r[0].stride(); + assert(stride % 2 == 0); + assert(this->ijk_r2r[0].stride() == this->ijk_r2r[1].stride() == this->ijk_r2r[2].stride()); const auto hstride = stride / 2; -// auto ex = this->halo - 1; - intrp<0>(dst[0], src[0], this->ijk_r2r[0]^hstride, this->ijk_r2r[1], this->ijk_r2r[2], hstride); - intrp<1>(dst[1], src[1], this->ijk_r2r[1]^hstride, this->ijk_r2r[2], this->ijk_r2r[0], hstride); - intrp<2>(dst[2], src[2], this->ijk_r2r[2]^hstride, this->ijk_r2r[0], this->ijk_r2r[1], hstride); + // first round of interpolation +// interpolate_refinee_on_edges(); // with MPI, some refined points are at the edges of the domain; calculate them using halos of non-refined arrays + + // interpolate between points of the large grid + intrp<0>(this->mem->refinee(e), this->rng_midpoints(this->ijk_r2r[0]), this->ijk_r2r[1], this->ijk_r2r[2], hstride); +// intrp<1>(this->mem->refinee(e), this->rng_midpoints(this->ijk_r2r[1]), this->ijk_r2r[2], this->ijk_r2r[0], hstride); +// intrp<2>(this->mem->refinee(e), this->rng_midpoints(this->ijk_r2r[2]), this->ijk_r2r[0], this->ijk_r2r[1], hstride); + +// std::cerr << "midpoints 0: " << this->rng_midpoints(this->ijk_r2r[0]) << std::endl; +// std::cerr << "midpoints 1: " << this->rng_midpoints(this->ijk_r2r[1]) << std::endl; +// std::cerr << "midpoints 2: " << this->rng_midpoints(this->ijk_r2r[2]) << std::endl; + + // interpolate between points calculated in the previous step + + // interpolate between points calculated in the previous step + + // subsequent rounds of interpolation ... } public: diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp index 6f26eed6..bd4c4e16 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp @@ -34,7 +34,7 @@ namespace libmpdataxx // const int n_fra; // number of fields with fractal reconstruction const int n_ref; // number of refinements; refined resolution is dx / n_ref - // refined arrays have no halo, because halo size needs to be known at compile time (why is it so? probably not really necessary?) + // refined arrays have no halos (do they need them? halos can be added by extending grid_size in alloc() in mpdata_rhs_vip_prs_sgs_fra.hpp by e.g. mem->n_ref/2) // const int halo_ref; // size of halos of refined scalar (and vector?) arrays // TODO: make these const! @@ -43,6 +43,13 @@ namespace libmpdataxx // static rng_t rng_sclr_ref(const rng_t &rng) { return rng^halo_ref; } + // what if + static rng_t rng_midpoints(const rng_t &rng) + { + if(rng.last() - rng.first() < rng.stride()) return rng; + else return rng_t(rng.first() + rng.stride() / 2, rng.last() - rng.stride() / 2, rng.stride()); + } + public: struct rt_params_t : parent_t::rt_params_t diff --git a/tests/sandbox/pbl/pbl.hpp b/tests/sandbox/pbl/pbl.hpp index 25f6fe58..21a122a1 100644 --- a/tests/sandbox/pbl/pbl.hpp +++ b/tests/sandbox/pbl/pbl.hpp @@ -64,12 +64,14 @@ class pbl : public libmpdataxx::output::hdf5_xdmfmem->refinee(ix::tht)(this->ijk_ref) = blitz::tensor::k; // anything //this->mem->refinee(ix::tht) = blitz::tensor::k; // anything - // copy tht to refined tht at position where they overlap std::cerr << "refinee: " << this->mem->refinee(ix::tht); std::cerr << "refinee(ijk_ref): " << this->mem->refinee(ix::tht)(this->ijk_ref); std::cerr << "refinee(ijk_r2r): " << this->mem->refinee(ix::tht)(this->ijk_r2r); + // copy tht to refined tht at position where they overlap this->mem->refinee(ix::tht)(this->ijk_r2r) = this->mem->advectee(ix::tht)(this->ijk); + this->interpolate_refinee(ix::tht); + this->mem->barrier(); if (this->rank == 0) { From 6aab97c36a38937a987c67de8de197abba67e8e8 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 18 Mar 2022 12:59:46 +0100 Subject: [PATCH 025/129] wip on interpolation --- libmpdata++/formulae/idxperm.hpp | 13 +++++++++ libmpdata++/output/hdf5.hpp | 6 ++-- .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 28 ++++++++++++++----- tests/sandbox/pbl/pbl.hpp | 11 +++++--- 4 files changed, 44 insertions(+), 14 deletions(-) diff --git a/libmpdata++/formulae/idxperm.hpp b/libmpdata++/formulae/idxperm.hpp index 5c5c1c74..1d58574d 100644 --- a/libmpdata++/formulae/idxperm.hpp +++ b/libmpdata++/formulae/idxperm.hpp @@ -69,5 +69,18 @@ namespace libmpdataxx template<> inline int_idx_t<3> pi<2>(const int k, const int i, const int j) { return int_idx_t<3>({i,j,k}); } + + // 3D - strided ranges + template + inline idxs_t<3> pis(const rng_t &i, const rng_t &j, const rng_t &k); + + template<> + inline idxs_t<3> pis<0>(const rng_t &i, const rng_t &j, const rng_t &k) { return idxs_t<3>{{i.first(), j.first(), k.first()}, {i.last(), j.last(), k.last()}, {i.stride(), j.stride(), k.stride()}}; } + + template<> + inline idxs_t<3> pis<1>(const rng_t &j, const rng_t &k, const rng_t &i) { return idxs_t<3>{{i.first(), j.first(), k.first()}, {i.last(), j.last(), k.last()}, {i.stride(), j.stride(), k.stride()}}; } + + template<> + inline idxs_t<3> pis<2>(const rng_t &k, const rng_t &i, const rng_t &j) { return idxs_t<3>{{i.first(), j.first(), k.first()}, {i.last(), j.last(), k.last()}, {i.stride(), j.stride(), k.stride()}}; } } // namespace idxperm } // namespace libmpdataxx diff --git a/libmpdata++/output/hdf5.hpp b/libmpdata++/output/hdf5.hpp index 87f07ac7..7ba87f90 100644 --- a/libmpdata++/output/hdf5.hpp +++ b/libmpdata++/output/hdf5.hpp @@ -200,15 +200,15 @@ namespace libmpdataxx std::string name; switch (i) { - case 0 : coord = this->di / 4. + this->di / this->mem->n_ref * blitz::firstIndex(); + case 0 : coord = this->di / 2. + this->di / this->mem->n_ref * (blitz::firstIndex() - .5); name = "X refined"; dim_names_ref[i] = name; break; - case 1 : coord = this->dj / 4. + this->dj / this->mem->n_ref * blitz::secondIndex(); + case 1 : coord = this->dj / 2. + this->dj / this->mem->n_ref * (blitz::secondIndex() - .5); name = "Y refined"; dim_names_ref[i] = name; break; - case 2 : coord = this->dk / 4. + this->dk / this->mem->n_ref * blitz::thirdIndex(); + case 2 : coord = this->dk / 2. + this->dk / this->mem->n_ref * (blitz::thirdIndex() - .5); name = "Z refined"; dim_names_ref[i] = name; break; diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 390fc21d..38527839 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -42,13 +42,25 @@ namespace libmpdataxx const int &dist ) { - using idxperm::pi; - using namespace arakawa_c; + using idxperm::pis; +// using namespace arakawa_c; using real_t = typename ct_params_t::real_t; - arr(pi(i, j, k)) = real_t(.5) * ( - arr(pi(i - dist, j, k)) + - arr(pi(i + dist, j, k)) +// std::cerr << "range: " << i << " " << j << " " << k << std::endl; +// std::cerr << "range - dist: " << i - dist << " " << j << " " << k << std::endl; +// std::cerr << "range + dist: " << i + dist << " " << j << " " << k << std::endl; +// +// std::cerr << "arr range: " << arr(i, j, k) << std::endl; +// std::cerr << "arr range - dist: " << arr(i - dist, j, k) << std::endl; +// std::cerr << "arr range + dist: " << arr(i + dist, j, k) << std::endl; +// +// std::cerr << "arr range pi: " << arr(pis(i, j, k)) << std::endl; +// std::cerr << "arr range - dist pi: " << arr(pis(i - dist, j, k)) << std::endl; +// std::cerr << "arr range + dist pi: " << arr(pis(i + dist, j, k)) << std::endl; + + arr(pis(i, j, k)) = real_t(.5) * ( + arr(pis(i - dist, j, k)) + + arr(pis(i + dist, j, k)) ); } @@ -62,16 +74,18 @@ namespace libmpdataxx assert(stride % 2 == 0); assert(this->ijk_r2r[0].stride() == this->ijk_r2r[1].stride() == this->ijk_r2r[2].stride()); const auto hstride = stride / 2; +// std::cerr << "hstride: " << hstride << std::endl; // first round of interpolation // interpolate_refinee_on_edges(); // with MPI, some refined points are at the edges of the domain; calculate them using halos of non-refined arrays // interpolate between points of the large grid intrp<0>(this->mem->refinee(e), this->rng_midpoints(this->ijk_r2r[0]), this->ijk_r2r[1], this->ijk_r2r[2], hstride); -// intrp<1>(this->mem->refinee(e), this->rng_midpoints(this->ijk_r2r[1]), this->ijk_r2r[2], this->ijk_r2r[0], hstride); -// intrp<2>(this->mem->refinee(e), this->rng_midpoints(this->ijk_r2r[2]), this->ijk_r2r[0], this->ijk_r2r[1], hstride); + intrp<1>(this->mem->refinee(e), this->rng_midpoints(this->ijk_r2r[1]), this->ijk_r2r[2], this->ijk_r2r[0], hstride); + intrp<2>(this->mem->refinee(e), this->rng_midpoints(this->ijk_r2r[2]), this->ijk_r2r[0], this->ijk_r2r[1], hstride); // std::cerr << "midpoints 0: " << this->rng_midpoints(this->ijk_r2r[0]) << std::endl; +// std::cerr << "refinee at midpoints 0: " << this->mem->refinee(e)(this->rng_midpoints(this->ijk_r2r[0]), this->ijk_r2r[1], this->ijk_r2r[2]) << std::endl; // std::cerr << "midpoints 1: " << this->rng_midpoints(this->ijk_r2r[1]) << std::endl; // std::cerr << "midpoints 2: " << this->rng_midpoints(this->ijk_r2r[2]) << std::endl; diff --git a/tests/sandbox/pbl/pbl.hpp b/tests/sandbox/pbl/pbl.hpp index 21a122a1..505b3dbc 100644 --- a/tests/sandbox/pbl/pbl.hpp +++ b/tests/sandbox/pbl/pbl.hpp @@ -62,14 +62,17 @@ class pbl : public libmpdataxx::output::hdf5_xdmfrank == 0) std::cout << this->timestep << std::endl; // test filling of refinee - this->mem->refinee(ix::tht)(this->ijk_ref) = blitz::tensor::k; // anything + //this->mem->refinee(ix::tht)(this->ijk_ref) = blitz::tensor::k * 1e-3; // anything + this->mem->refinee(ix::tht)(this->ijk_ref) = -1; //this->mem->refinee(ix::tht) = blitz::tensor::k; // anything - std::cerr << "refinee: " << this->mem->refinee(ix::tht); - std::cerr << "refinee(ijk_ref): " << this->mem->refinee(ix::tht)(this->ijk_ref); - std::cerr << "refinee(ijk_r2r): " << this->mem->refinee(ix::tht)(this->ijk_r2r); + // copy tht to refined tht at position where they overlap this->mem->refinee(ix::tht)(this->ijk_r2r) = this->mem->advectee(ix::tht)(this->ijk); +// std::cerr << "refinee: " << this->mem->refinee(ix::tht); +// std::cerr << "refinee(ijk_ref): " << this->mem->refinee(ix::tht)(this->ijk_ref); +// std::cerr << "refinee(ijk_r2r): " << this->mem->refinee(ix::tht)(this->ijk_r2r); + this->interpolate_refinee(ix::tht); this->mem->barrier(); From e622df1b184e9d3c703db2e846d8f5bc6ff29194 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 18 Mar 2022 15:02:40 +0100 Subject: [PATCH 026/129] interpolation: hardocode two iterations, still no interpolation at sharedmem and MPI boundaries --- .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 65 +++++++++++++++---- .../mpdata_rhs_vip_prs_sgs_fra_common.hpp | 15 ++++- 2 files changed, 65 insertions(+), 15 deletions(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 38527839..6183d120 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -70,30 +70,67 @@ namespace libmpdataxx // const int stride) { using namespace arakawa_c; // for rng_t operator^ - const auto stride = this->ijk_r2r[0].stride(); + +// interpolate_refinee_on_edges(); // with MPI, some refined points are at the edges of the domain; calculate them using halos of non-refined arrays + + // first round of interpolation + auto stride = this->ijk_r2r[0].stride(); assert(stride % 2 == 0); assert(this->ijk_r2r[0].stride() == this->ijk_r2r[1].stride() == this->ijk_r2r[2].stride()); - const auto hstride = stride / 2; -// std::cerr << "hstride: " << hstride << std::endl; + auto hstride = stride / 2; - // first round of interpolation -// interpolate_refinee_on_edges(); // with MPI, some refined points are at the edges of the domain; calculate them using halos of non-refined arrays + const auto mid_ijk_r2r_0 = this->rng_midpoints(this->ijk_r2r[0]); + const auto mid_ijk_r2r_1 = this->rng_midpoints(this->ijk_r2r[1]); + const auto mid_ijk_r2r_2 = this->rng_midpoints(this->ijk_r2r[2]); // interpolate between points of the large grid - intrp<0>(this->mem->refinee(e), this->rng_midpoints(this->ijk_r2r[0]), this->ijk_r2r[1], this->ijk_r2r[2], hstride); - intrp<1>(this->mem->refinee(e), this->rng_midpoints(this->ijk_r2r[1]), this->ijk_r2r[2], this->ijk_r2r[0], hstride); - intrp<2>(this->mem->refinee(e), this->rng_midpoints(this->ijk_r2r[2]), this->ijk_r2r[0], this->ijk_r2r[1], hstride); + intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, this->ijk_r2r[1], this->ijk_r2r[2], hstride); + intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, this->ijk_r2r[2], this->ijk_r2r[0], hstride); + intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, this->ijk_r2r[0], this->ijk_r2r[1], hstride); -// std::cerr << "midpoints 0: " << this->rng_midpoints(this->ijk_r2r[0]) << std::endl; -// std::cerr << "refinee at midpoints 0: " << this->mem->refinee(e)(this->rng_midpoints(this->ijk_r2r[0]), this->ijk_r2r[1], this->ijk_r2r[2]) << std::endl; -// std::cerr << "midpoints 1: " << this->rng_midpoints(this->ijk_r2r[1]) << std::endl; -// std::cerr << "midpoints 2: " << this->rng_midpoints(this->ijk_r2r[2]) << std::endl; + // interpolate between just interpolated points + intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, mid_ijk_r2r_1, this->ijk_r2r[2], hstride); + intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, mid_ijk_r2r_2, this->ijk_r2r[0], hstride); + intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, mid_ijk_r2r_0, this->ijk_r2r[1], hstride); - // interpolate between points calculated in the previous step + intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, this->ijk_r2r[1], mid_ijk_r2r_2, hstride); + intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, this->ijk_r2r[2], mid_ijk_r2r_0, hstride); + intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, this->ijk_r2r[0], mid_ijk_r2r_1, hstride); - // interpolate between points calculated in the previous step + intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, mid_ijk_r2r_1, mid_ijk_r2r_2, hstride); + intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, mid_ijk_r2r_2, mid_ijk_r2r_0, hstride); + intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, mid_ijk_r2r_0, mid_ijk_r2r_1, hstride); // subsequent rounds of interpolation ... + + stride = mid_ijk_r2r_0_2.stride(); + assert(stride % 2 == 0); + assert(mid_ijk_r2r_0_2.stride() == mid_ijk_r2r_1_2.stride() == mid_ijk_r2r_2_2.stride()); + hstride = stride / 2; + + const auto mid_ijk_r2r_0_2 = this->rng_midpoints_out(mid_ijk_r2r_0); + const auto mid_ijk_r2r_1_2 = this->rng_midpoints_out(mid_ijk_r2r_1); + const auto mid_ijk_r2r_2_2 = this->rng_midpoints_out(mid_ijk_r2r_2); + + const auto ijk_r2r_0_h = this->rng_half_stride(this->ijk_r2r[0]); + const auto ijk_r2r_1_h = this->rng_half_stride(this->ijk_r2r[1]); + const auto ijk_r2r_2_h = this->rng_half_stride(this->ijk_r2r[2]); + + intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0_2, ijk_r2r_1_h, ijk_r2r_2_h, hstride); + intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1_2, ijk_r2r_2_h, ijk_r2r_0_h, hstride); + intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2_2, ijk_r2r_0_h, ijk_r2r_1_h, hstride); + + intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0_2, mid_ijk_r2r_1_2, ijk_r2r_2_h, hstride); + intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1_2, mid_ijk_r2r_2_2, ijk_r2r_0_h, hstride); + intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2_2, mid_ijk_r2r_0_2, ijk_r2r_1_h, hstride); + + intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0_2, ijk_r2r_1_h, mid_ijk_r2r_2_2, hstride); + intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1_2, ijk_r2r_2_h, mid_ijk_r2r_0_2, hstride); + intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2_2, ijk_r2r_0_h, mid_ijk_r2r_1_2, hstride); + + intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0_2, mid_ijk_r2r_1_2, mid_ijk_r2r_2_2, hstride); + intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1_2, mid_ijk_r2r_2_2, mid_ijk_r2r_0_2, hstride); + intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2_2, mid_ijk_r2r_0_2, mid_ijk_r2r_1_2, hstride); } public: diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp index bd4c4e16..e3b6502c 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp @@ -43,13 +43,26 @@ namespace libmpdataxx // static rng_t rng_sclr_ref(const rng_t &rng) { return rng^halo_ref; } - // what if static rng_t rng_midpoints(const rng_t &rng) { + assert(rng.stride() % 2 == 0); if(rng.last() - rng.first() < rng.stride()) return rng; else return rng_t(rng.first() + rng.stride() / 2, rng.last() - rng.stride() / 2, rng.stride()); } + static rng_t rng_midpoints_out(const rng_t &rng) + { + assert(rng.stride() % 4 == 0); + if(rng.last() - rng.first() < rng.stride()) return rng; + else return rng_t(rng.first() - rng.stride() / 4, rng.last() + rng.stride() / 4, rng.stride() / 2); + } + + static rng_t rng_half_stride(const rng_t &rng) + { + assert(rng.stride() % 2 == 0); + return rng_t(rng.first(), rng.last(), rng.stride() / 2); + } + public: struct rt_params_t : parent_t::rt_params_t From 669194ad619103ac1ec0087fb1c949fa994e2885 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 21 Mar 2022 16:12:02 +0100 Subject: [PATCH 027/129] wip on interpolation at edges --- .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 113 ++++++++---------- .../mpdata_rhs_vip_prs_sgs_fra_common.hpp | 11 +- tests/sandbox/pbl/pbl_test_def.hpp | 2 +- 3 files changed, 61 insertions(+), 65 deletions(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 6183d120..fcf39cc5 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -65,72 +65,63 @@ namespace libmpdataxx } void interpolate_refinee(const int e = 0) -// arrvec_t &dst, -// const arrvec_t &src, -// const int stride) { using namespace arakawa_c; // for rng_t operator^ -// interpolate_refinee_on_edges(); // with MPI, some refined points are at the edges of the domain; calculate them using halos of non-refined arrays - - // first round of interpolation - auto stride = this->ijk_r2r[0].stride(); - assert(stride % 2 == 0); - assert(this->ijk_r2r[0].stride() == this->ijk_r2r[1].stride() == this->ijk_r2r[2].stride()); - auto hstride = stride / 2; - - const auto mid_ijk_r2r_0 = this->rng_midpoints(this->ijk_r2r[0]); - const auto mid_ijk_r2r_1 = this->rng_midpoints(this->ijk_r2r[1]); - const auto mid_ijk_r2r_2 = this->rng_midpoints(this->ijk_r2r[2]); - - // interpolate between points of the large grid - intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, this->ijk_r2r[1], this->ijk_r2r[2], hstride); - intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, this->ijk_r2r[2], this->ijk_r2r[0], hstride); - intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, this->ijk_r2r[0], this->ijk_r2r[1], hstride); - - // interpolate between just interpolated points - intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, mid_ijk_r2r_1, this->ijk_r2r[2], hstride); - intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, mid_ijk_r2r_2, this->ijk_r2r[0], hstride); - intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, mid_ijk_r2r_0, this->ijk_r2r[1], hstride); - - intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, this->ijk_r2r[1], mid_ijk_r2r_2, hstride); - intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, this->ijk_r2r[2], mid_ijk_r2r_0, hstride); - intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, this->ijk_r2r[0], mid_ijk_r2r_1, hstride); - - intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, mid_ijk_r2r_1, mid_ijk_r2r_2, hstride); - intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, mid_ijk_r2r_2, mid_ijk_r2r_0, hstride); - intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, mid_ijk_r2r_0, mid_ijk_r2r_1, hstride); + rng_t mid_ijk_r2r_0, mid_ijk_r2r_1, mid_ijk_r2r_2; // positions between already known values (to be filled during given iteration) + rng_t ijk_r2r_0_h, ijk_r2r_1_h, ijk_r2r_2_h; // all positions at resolution of given iteration + int stride, hstride; - // subsequent rounds of interpolation ... + // fill refined array at position where it overlaps with the resolved array + this->mem->refinee(e)(this->ijk_r2r) = this->mem->advectee(e)(this->ijk); - stride = mid_ijk_r2r_0_2.stride(); - assert(stride % 2 == 0); - assert(mid_ijk_r2r_0_2.stride() == mid_ijk_r2r_1_2.stride() == mid_ijk_r2r_2_2.stride()); - hstride = stride / 2; - - const auto mid_ijk_r2r_0_2 = this->rng_midpoints_out(mid_ijk_r2r_0); - const auto mid_ijk_r2r_1_2 = this->rng_midpoints_out(mid_ijk_r2r_1); - const auto mid_ijk_r2r_2_2 = this->rng_midpoints_out(mid_ijk_r2r_2); - - const auto ijk_r2r_0_h = this->rng_half_stride(this->ijk_r2r[0]); - const auto ijk_r2r_1_h = this->rng_half_stride(this->ijk_r2r[1]); - const auto ijk_r2r_2_h = this->rng_half_stride(this->ijk_r2r[2]); - - intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0_2, ijk_r2r_1_h, ijk_r2r_2_h, hstride); - intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1_2, ijk_r2r_2_h, ijk_r2r_0_h, hstride); - intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2_2, ijk_r2r_0_h, ijk_r2r_1_h, hstride); - - intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0_2, mid_ijk_r2r_1_2, ijk_r2r_2_h, hstride); - intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1_2, mid_ijk_r2r_2_2, ijk_r2r_0_h, hstride); - intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2_2, mid_ijk_r2r_0_2, ijk_r2r_1_h, hstride); - - intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0_2, ijk_r2r_1_h, mid_ijk_r2r_2_2, hstride); - intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1_2, ijk_r2r_2_h, mid_ijk_r2r_0_2, hstride); - intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2_2, ijk_r2r_0_h, mid_ijk_r2r_1_2, hstride); - - intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0_2, mid_ijk_r2r_1_2, mid_ijk_r2r_2_2, hstride); - intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1_2, mid_ijk_r2r_2_2, mid_ijk_r2r_0_2, hstride); - intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2_2, mid_ijk_r2r_0_2, mid_ijk_r2r_1_2, hstride); +// interpolate_refinee_on_edges(); // with MPI, some refined points are at the edges of the domain; calculate them using halos of non-refined arrays +// TODO: fill halo sclr ? + + for(int i=0; in_fra_iter; ++i) + { + if(i==0) + { + mid_ijk_r2r_0 = this->rng_midpoints(this->ijk_r2r[0]);//, this->mem->distmem.rank(), this->mem->distmem.size()); + mid_ijk_r2r_1 = this->rng_midpoints(this->ijk_r2r[1]);//, this->rank, this->mem->size); + mid_ijk_r2r_2 = this->rng_midpoints(this->ijk_r2r[2]); + + ijk_r2r_0_h = this->ijk_r2r[0]; + ijk_r2r_1_h = this->ijk_r2r[1]; + ijk_r2r_2_h = this->ijk_r2r[2]; + } + else + { + mid_ijk_r2r_0 = this->rng_midpoints_out(mid_ijk_r2r_0); + mid_ijk_r2r_1 = this->rng_midpoints_out(mid_ijk_r2r_1); + mid_ijk_r2r_2 = this->rng_midpoints_out(mid_ijk_r2r_2); + + ijk_r2r_0_h = this->rng_half_stride(ijk_r2r_0_h); + ijk_r2r_1_h = this->rng_half_stride(ijk_r2r_1_h); + ijk_r2r_2_h = this->rng_half_stride(ijk_r2r_2_h); + } + + stride = ijk_r2r_0_h.stride(); + assert(ijk_r2r_0_h.stride() == ijk_r2r_1_h.stride() == ijk_r2r_2_h.stride()); + assert(stride % 2 == 0); + hstride = stride / 2; + + intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, ijk_r2r_1_h, ijk_r2r_2_h, hstride); + intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, ijk_r2r_2_h, ijk_r2r_0_h, hstride); + intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, ijk_r2r_0_h, ijk_r2r_1_h, hstride); + + intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, mid_ijk_r2r_1, ijk_r2r_2_h, hstride); + intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, mid_ijk_r2r_2, ijk_r2r_0_h, hstride); + intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, mid_ijk_r2r_0, ijk_r2r_1_h, hstride); + + intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, ijk_r2r_1_h, mid_ijk_r2r_2, hstride); + intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, ijk_r2r_2_h, mid_ijk_r2r_0, hstride); + intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, ijk_r2r_0_h, mid_ijk_r2r_1, hstride); + + intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, mid_ijk_r2r_1, mid_ijk_r2r_2, hstride); + intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, mid_ijk_r2r_2, mid_ijk_r2r_0, hstride); + intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, mid_ijk_r2r_0, mid_ijk_r2r_1, hstride); + } } public: diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp index e3b6502c..f2d823ba 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp @@ -32,7 +32,8 @@ namespace libmpdataxx protected: // const int n_fra; // number of fields with fractal reconstruction - const int n_ref; // number of refinements; refined resolution is dx / n_ref + const int n_ref, // number of refinements; refined resolution is dx / n_ref + n_fra_iter; // refined arrays have no halos (do they need them? halos can be added by extending grid_size in alloc() in mpdata_rhs_vip_prs_sgs_fra.hpp by e.g. mem->n_ref/2) // const int halo_ref; // size of halos of refined scalar (and vector?) arrays @@ -43,11 +44,14 @@ namespace libmpdataxx // static rng_t rng_sclr_ref(const rng_t &rng) { return rng^halo_ref; } - static rng_t rng_midpoints(const rng_t &rng) + static rng_t rng_midpoints(const rng_t &rng, const int rank = 0, const int size = 1) { assert(rng.stride() % 2 == 0); if(rng.last() - rng.first() < rng.stride()) return rng; - else return rng_t(rng.first() + rng.stride() / 2, rng.last() - rng.stride() / 2, rng.stride()); + else return rng_t( + rank == 0 ? rng.first() + rng.stride() / 2 : rng.first() - rng.stride() / 2, + rank == size - 1 ? rng.last() - rng.stride() / 2 : rng.last() + rng.stride() / 2, + rng.stride()); } static rng_t rng_midpoints_out(const rng_t &rng) @@ -76,6 +80,7 @@ namespace libmpdataxx const rt_params_t &p ) : parent_t(args, p), + n_fra_iter(p.n_fra_iter), n_ref(this->mem->n_ref), // halo_ref(n_ref / 2), // ijk_ref init below assumes 3D (shmem decomp dim along y); diff --git a/tests/sandbox/pbl/pbl_test_def.hpp b/tests/sandbox/pbl/pbl_test_def.hpp index c853f78b..c8601420 100644 --- a/tests/sandbox/pbl/pbl_test_def.hpp +++ b/tests/sandbox/pbl/pbl_test_def.hpp @@ -22,7 +22,7 @@ template void set_sgs_specific(params_t &p, iles_tag) { p.iles_cdrag = 0.1; - p.n_fra_iter = 2; + p.n_fra_iter = 1; } // smagorinsky From fe9d8f81b46c1ba5fca6496a153538c489e0f5c4 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 31 Mar 2022 13:18:33 +0200 Subject: [PATCH 028/129] shmem_ref: require grid size that makes fractal reconstruction easier --- libmpdata++/concurr/detail/sharedmem_refined.hpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/libmpdata++/concurr/detail/sharedmem_refined.hpp b/libmpdata++/concurr/detail/sharedmem_refined.hpp index 2266ab7e..f3e9f1a5 100644 --- a/libmpdata++/concurr/detail/sharedmem_refined.hpp +++ b/libmpdata++/concurr/detail/sharedmem_refined.hpp @@ -49,6 +49,13 @@ namespace libmpdataxx : parent_t(grid_size, size), n_ref(n_ref) { assert(n_ref % 2 == 0); // only division into even number of cells, because we assume that one of the refined scalar points is at the MPI boundary, which is in the middle between normal grid scalars + + // for now, require a grid_size that is convenient for fractal reconstruction (which calculates 2 points based on 3 points) + // NOTE: fix this with proper halos (cyclic is easy, but what about rigid?) + // NOTE2: this is actually a requirement for fractal reconstruction, not for any grid refinement, so move this somewhere else + for (int d = 0; d < n_dims; ++d) + if((grid_size[d] - 3) % 2 != 0) throw std::runtime_error("Fractal grid refinement requires nx/ny/nz = 3 + 2 * i, where i = 0,1,2,3,..."); + for (int d = 0; d < n_dims; ++d) { grid_size_ref[d] = refine_grid_size( From 275789c39dd28918a0d6fc551692a0c0e43fb067 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 31 Mar 2022 15:33:05 +0200 Subject: [PATCH 029/129] refinement: wip --- .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 36 +++++++++++++------ .../mpdata_rhs_vip_prs_sgs_fra_common.hpp | 7 ++-- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index fcf39cc5..eb90b277 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -46,7 +46,13 @@ namespace libmpdataxx // using namespace arakawa_c; using real_t = typename ct_params_t::real_t; -// std::cerr << "range: " << i << " " << j << " " << k << std::endl; +// if(d==0) +// std::cerr << "range<" << d << ">: " << i << " " << j << " " << k << std::endl; +// if(d==1) +// std::cerr << "range<" << d << ">: " << k << " " << i << " " << j << std::endl; +// if(d==2) +// std::cerr << "range<" << d << ">: " << j << " " << k << " " << i << std::endl; + // std::cerr << "range - dist: " << i - dist << " " << j << " " << k << std::endl; // std::cerr << "range + dist: " << i + dist << " " << j << " " << k << std::endl; // @@ -80,10 +86,11 @@ namespace libmpdataxx for(int i=0; in_fra_iter; ++i) { + // messy, because in domain decomposition (sharedmem and distmem) some refined scalars are on the edge of the subdomain... if(i==0) { - mid_ijk_r2r_0 = this->rng_midpoints(this->ijk_r2r[0]);//, this->mem->distmem.rank(), this->mem->distmem.size()); - mid_ijk_r2r_1 = this->rng_midpoints(this->ijk_r2r[1]);//, this->rank, this->mem->size); + mid_ijk_r2r_0 = this->rng_midpoints(this->ijk_r2r[0], this->mem->distmem.rank(), this->mem->distmem.size()); + mid_ijk_r2r_1 = this->rng_midpoints(this->ijk_r2r[1], this->rank, this->mem->size); mid_ijk_r2r_2 = this->rng_midpoints(this->ijk_r2r[2]); ijk_r2r_0_h = this->ijk_r2r[0]; @@ -92,9 +99,18 @@ namespace libmpdataxx } else { - mid_ijk_r2r_0 = this->rng_midpoints_out(mid_ijk_r2r_0); - mid_ijk_r2r_1 = this->rng_midpoints_out(mid_ijk_r2r_1); - mid_ijk_r2r_2 = this->rng_midpoints_out(mid_ijk_r2r_2); + if(i==1) + { + mid_ijk_r2r_0 = this->rng_midpoints_out(mid_ijk_r2r_0, this->mem->distmem.rank(), this->mem->distmem.size()); + mid_ijk_r2r_1 = this->rng_midpoints_out(mid_ijk_r2r_1, this->rank, this->mem->size); + mid_ijk_r2r_2 = this->rng_midpoints_out(mid_ijk_r2r_2); + } + else + { + mid_ijk_r2r_0 = this->rng_midpoints_out(mid_ijk_r2r_0); + mid_ijk_r2r_1 = this->rng_midpoints_out(mid_ijk_r2r_1); + mid_ijk_r2r_2 = this->rng_midpoints_out(mid_ijk_r2r_2); + } ijk_r2r_0_h = this->rng_half_stride(ijk_r2r_0_h); ijk_r2r_1_h = this->rng_half_stride(ijk_r2r_1_h); @@ -115,12 +131,12 @@ namespace libmpdataxx intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, mid_ijk_r2r_0, ijk_r2r_1_h, hstride); intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, ijk_r2r_1_h, mid_ijk_r2r_2, hstride); - intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, ijk_r2r_2_h, mid_ijk_r2r_0, hstride); - intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, ijk_r2r_0_h, mid_ijk_r2r_1, hstride); +// intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, ijk_r2r_2_h, mid_ijk_r2r_0, hstride); +// intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, ijk_r2r_0_h, mid_ijk_r2r_1, hstride); intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, mid_ijk_r2r_1, mid_ijk_r2r_2, hstride); - intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, mid_ijk_r2r_2, mid_ijk_r2r_0, hstride); - intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, mid_ijk_r2r_0, mid_ijk_r2r_1, hstride); +// intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, mid_ijk_r2r_2, mid_ijk_r2r_0, hstride); +// intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, mid_ijk_r2r_0, mid_ijk_r2r_1, hstride); } } diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp index f2d823ba..a9f84007 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp @@ -54,11 +54,14 @@ namespace libmpdataxx rng.stride()); } - static rng_t rng_midpoints_out(const rng_t &rng) + static rng_t rng_midpoints_out(const rng_t &rng, const int rank = 0, const int size = 1) { assert(rng.stride() % 4 == 0); if(rng.last() - rng.first() < rng.stride()) return rng; - else return rng_t(rng.first() - rng.stride() / 4, rng.last() + rng.stride() / 4, rng.stride() / 2); + else return rng_t( + rank == 0 ? rng.first() - rng.stride() / 4 : rng.first() + rng.stride() / 4, + rank == size - 1 ? rng.last() + rng.stride() / 4 : rng.last() - rng.stride() / 4, + rng.stride() / 2); } static rng_t rng_half_stride(const rng_t &rng) From e5213f1443a924d581ef0bad40a8f179f70e8ed5 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 1 Apr 2022 21:37:34 +0200 Subject: [PATCH 030/129] refinement: fix interpolation for distmem! --- .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 56 ++++++++++++++++--- tests/sandbox/pbl/pbl.hpp | 2 +- 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index eb90b277..5d250443 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -78,6 +78,47 @@ namespace libmpdataxx rng_t ijk_r2r_0_h, ijk_r2r_1_h, ijk_r2r_2_h; // all positions at resolution of given iteration int stride, hstride; + // TODO: fill halos of advectee perpendicular to axis x? (distmem) + + this->mem->psi_ref[e] = -1000; + + // fill distmem halos of refinee + // TODO: move to bcond or sth? would be filled only by remote bcond + this->mem->psi_ref[e]( + this->mem->grid_size_ref[0].first()-1, + this->ijk_r2r[1], + this->ijk_r2r[2] + ) = + this->mem->psi[e][0]( + this->ijk[0].first()-1, + this->ijk[1], + this->ijk[2] + ); + this->mem->psi_ref[e]( + this->mem->grid_size_ref[0].last()+1, + this->ijk_r2r[1], + this->ijk_r2r[2] + ) = + this->mem->psi[e][0]( + this->ijk[0].last()+1, + this->ijk[1], + this->ijk[2] + ); + +// std::cerr << "post left halo fill psi_ref: " << this->mem->psi_ref[e]; +// std::cerr << "post left halo fill psi: " << this->mem->psi[e][0]; +// std::cerr << this->mem->psi_ref[e]( +// this->ijk_r2r[0].first()-1, +// this->ijk_r2r[1], +// this->ijk_r2r[2] +// ); +// std::cerr << this->mem->psi[e][0]( +// this->ijk[0].first()-1, +// this->ijk[1], +// this->ijk[2] +// ); + + // fill refined array at position where it overlaps with the resolved array this->mem->refinee(e)(this->ijk_r2r) = this->mem->advectee(e)(this->ijk); @@ -126,16 +167,17 @@ namespace libmpdataxx intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, ijk_r2r_2_h, ijk_r2r_0_h, hstride); intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, ijk_r2r_0_h, ijk_r2r_1_h, hstride); - intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, mid_ijk_r2r_1, ijk_r2r_2_h, hstride); - intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, mid_ijk_r2r_2, ijk_r2r_0_h, hstride); intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, mid_ijk_r2r_0, ijk_r2r_1_h, hstride); + intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, mid_ijk_r2r_2, ijk_r2r_0_h, hstride); +// intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, mid_ijk_r2r_1, ijk_r2r_2_h, hstride); - intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, ijk_r2r_1_h, mid_ijk_r2r_2, hstride); -// intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, ijk_r2r_2_h, mid_ijk_r2r_0, hstride); +// intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, ijk_r2r_1_h, mid_ijk_r2r_2, hstride); + intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, ijk_r2r_2_h, mid_ijk_r2r_0, hstride); // intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, ijk_r2r_0_h, mid_ijk_r2r_1, hstride); - intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, mid_ijk_r2r_1, mid_ijk_r2r_2, hstride); -// intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, mid_ijk_r2r_2, mid_ijk_r2r_0, hstride); +// necessary? + intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, mid_ijk_r2r_2, mid_ijk_r2r_0, hstride); +// intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, mid_ijk_r2r_1, mid_ijk_r2r_2, hstride); // intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, mid_ijk_r2r_0, mid_ijk_r2r_1, hstride); } } @@ -156,7 +198,7 @@ namespace libmpdataxx for (int n = 0; n < n_arr; ++n) mem->tmp[__file__].back().push_back(mem->old(new typename parent_t::arr_t( - mem->grid_size_ref[0], // NOTE: no halo + mem->grid_size_ref[0]^1, // NOTE: halo size 1 along x, because in distmem reconstructed point is at the edge of a domain mem->grid_size_ref[1], srfc ? rng_t(0, 0) : mem->grid_size_ref[2], arr3D_storage diff --git a/tests/sandbox/pbl/pbl.hpp b/tests/sandbox/pbl/pbl.hpp index 505b3dbc..edebd0ff 100644 --- a/tests/sandbox/pbl/pbl.hpp +++ b/tests/sandbox/pbl/pbl.hpp @@ -67,7 +67,7 @@ class pbl : public libmpdataxx::output::hdf5_xdmfmem->refinee(ix::tht) = blitz::tensor::k; // anything // copy tht to refined tht at position where they overlap - this->mem->refinee(ix::tht)(this->ijk_r2r) = this->mem->advectee(ix::tht)(this->ijk); +// this->mem->refinee(ix::tht)(this->ijk_r2r) = this->mem->advectee(ix::tht)(this->ijk); // std::cerr << "refinee: " << this->mem->refinee(ix::tht); // std::cerr << "refinee(ijk_ref): " << this->mem->refinee(ix::tht)(this->ijk_ref); From 5225da2864f007221d73b0a5c004eb7522de1776 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 19 Apr 2022 13:46:49 +0200 Subject: [PATCH 031/129] working interpolation of scalars --- .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 75 ++++++++++++++----- .../mpdata_rhs_vip_prs_sgs_fra_common.hpp | 29 +++++-- tests/sandbox/pbl/pbl_test_def.hpp | 2 +- 3 files changed, 79 insertions(+), 27 deletions(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 5d250443..cb154e3e 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -46,12 +46,12 @@ namespace libmpdataxx // using namespace arakawa_c; using real_t = typename ct_params_t::real_t; -// if(d==0) -// std::cerr << "range<" << d << ">: " << i << " " << j << " " << k << std::endl; -// if(d==1) -// std::cerr << "range<" << d << ">: " << k << " " << i << " " << j << std::endl; -// if(d==2) -// std::cerr << "range<" << d << ">: " << j << " " << k << " " << i << std::endl; + if(d==0) + std::cerr << "range<" << d << ">: " << i << " " << j << " " << k << std::endl; + if(d==1) + std::cerr << "range<" << d << ">: " << k << " " << i << " " << j << std::endl; + if(d==2) + std::cerr << "range<" << d << ">: " << j << " " << k << " " << i << std::endl; // std::cerr << "range - dist: " << i - dist << " " << j << " " << k << std::endl; // std::cerr << "range + dist: " << i + dist << " " << j << " " << k << std::endl; @@ -81,11 +81,12 @@ namespace libmpdataxx // TODO: fill halos of advectee perpendicular to axis x? (distmem) this->mem->psi_ref[e] = -1000; + this->mem->barrier(); // fill distmem halos of refinee // TODO: move to bcond or sth? would be filled only by remote bcond this->mem->psi_ref[e]( - this->mem->grid_size_ref[0].first()-1, + this->mem->grid_size_ref[0].first() - this->mem->n_ref/2, this->ijk_r2r[1], this->ijk_r2r[2] ) = @@ -95,7 +96,7 @@ namespace libmpdataxx this->ijk[2] ); this->mem->psi_ref[e]( - this->mem->grid_size_ref[0].last()+1, + this->mem->grid_size_ref[0].last() + this->mem->n_ref/2, this->ijk_r2r[1], this->ijk_r2r[2] ) = @@ -131,7 +132,7 @@ namespace libmpdataxx if(i==0) { mid_ijk_r2r_0 = this->rng_midpoints(this->ijk_r2r[0], this->mem->distmem.rank(), this->mem->distmem.size()); - mid_ijk_r2r_1 = this->rng_midpoints(this->ijk_r2r[1], this->rank, this->mem->size); + mid_ijk_r2r_1 = this->rng_midpoints(this->ijk_r2r[1], this->rank, this->mem->size, false); mid_ijk_r2r_2 = this->rng_midpoints(this->ijk_r2r[2]); ijk_r2r_0_h = this->ijk_r2r[0]; @@ -143,6 +144,8 @@ namespace libmpdataxx if(i==1) { mid_ijk_r2r_0 = this->rng_midpoints_out(mid_ijk_r2r_0, this->mem->distmem.rank(), this->mem->distmem.size()); + if(this->rank > 0) + mid_ijk_r2r_1 = rng_t(mid_ijk_r2r_1.first() - mid_ijk_r2r_1.stride(), mid_ijk_r2r_1.last(), mid_ijk_r2r_1.stride()); // shift back to an overlapping range along y mid_ijk_r2r_1 = this->rng_midpoints_out(mid_ijk_r2r_1, this->rank, this->mem->size); mid_ijk_r2r_2 = this->rng_midpoints_out(mid_ijk_r2r_2); } @@ -153,8 +156,8 @@ namespace libmpdataxx mid_ijk_r2r_2 = this->rng_midpoints_out(mid_ijk_r2r_2); } - ijk_r2r_0_h = this->rng_half_stride(ijk_r2r_0_h); - ijk_r2r_1_h = this->rng_half_stride(ijk_r2r_1_h); + ijk_r2r_0_h = this->rng_half_stride(ijk_r2r_0_h, this->mem->distmem.rank(), this->mem->distmem.size()); + ijk_r2r_1_h = this->rng_half_stride(ijk_r2r_1_h, this->rank, this->mem->size, false); ijk_r2r_2_h = this->rng_half_stride(ijk_r2r_2_h); } @@ -168,17 +171,51 @@ namespace libmpdataxx intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, ijk_r2r_0_h, ijk_r2r_1_h, hstride); intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, mid_ijk_r2r_0, ijk_r2r_1_h, hstride); - intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, mid_ijk_r2r_2, ijk_r2r_0_h, hstride); -// intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, mid_ijk_r2r_1, ijk_r2r_2_h, hstride); + intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, ijk_r2r_0_h, mid_ijk_r2r_1, hstride); + this->mem->barrier(); // necessary before interpolation along sharedmem y direction + intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, ijk_r2r_2_h, mid_ijk_r2r_0, hstride); + + intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, mid_ijk_r2r_0, mid_ijk_r2r_1, hstride); + +/* + if(this->rank == 0) + { + + intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, ijk_r2r_1_h, ijk_r2r_2_h, hstride); + intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, ijk_r2r_2_h, ijk_r2r_0_h, hstride); + intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, ijk_r2r_0_h, ijk_r2r_1_h, hstride); -// intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, ijk_r2r_1_h, mid_ijk_r2r_2, hstride); + intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, mid_ijk_r2r_0, ijk_r2r_1_h, hstride); + //intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, mid_ijk_r2r_2, ijk_r2r_0_h, hstride); + intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, ijk_r2r_0_h, mid_ijk_r2r_1, hstride); intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, ijk_r2r_2_h, mid_ijk_r2r_0, hstride); -// intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, ijk_r2r_0_h, mid_ijk_r2r_1, hstride); -// necessary? - intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, mid_ijk_r2r_2, mid_ijk_r2r_0, hstride); -// intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, mid_ijk_r2r_1, mid_ijk_r2r_2, hstride); -// intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, mid_ijk_r2r_0, mid_ijk_r2r_1, hstride); + //intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, mid_ijk_r2r_2, mid_ijk_r2r_0, hstride); + intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, mid_ijk_r2r_0, mid_ijk_r2r_1, hstride); + + std::cerr << std::endl; + } + this->mem->barrier(); + if(this->rank == 1) + { + + intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, ijk_r2r_1_h, ijk_r2r_2_h, hstride); + intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, ijk_r2r_2_h, ijk_r2r_0_h, hstride); + intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, ijk_r2r_0_h, ijk_r2r_1_h, hstride); + + intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, mid_ijk_r2r_0, ijk_r2r_1_h, hstride); + //intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, mid_ijk_r2r_2, ijk_r2r_0_h, hstride); + intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, ijk_r2r_0_h, mid_ijk_r2r_1, hstride); + intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, ijk_r2r_2_h, mid_ijk_r2r_0, hstride); + + //intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, mid_ijk_r2r_2, mid_ijk_r2r_0, hstride); + intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, mid_ijk_r2r_0, mid_ijk_r2r_1, hstride); + + std::cerr << std::endl; + } + */ + + this->mem->barrier(); } } diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp index a9f84007..6c42e4fe 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp @@ -44,14 +44,20 @@ namespace libmpdataxx // static rng_t rng_sclr_ref(const rng_t &rng) { return rng^halo_ref; } - static rng_t rng_midpoints(const rng_t &rng, const int rank = 0, const int size = 1) + static rng_t rng_midpoints(const rng_t &rng, const int rank = 0, const int size = 1, const bool overlap = true) // input range, rank and size of thread / MPI process, should created ranges have overlaps between different ranks { assert(rng.stride() % 2 == 0); if(rng.last() - rng.first() < rng.stride()) return rng; - else return rng_t( - rank == 0 ? rng.first() + rng.stride() / 2 : rng.first() - rng.stride() / 2, - rank == size - 1 ? rng.last() - rng.stride() / 2 : rng.last() + rng.stride() / 2, - rng.stride()); + else if (overlap) + return rng_t( + rank == 0 ? rng.first() + rng.stride() / 2 : rng.first() - rng.stride() / 2, + rank == size - 1 ? rng.last() - rng.stride() / 2 : rng.last() + rng.stride() / 2, + rng.stride()); + else + return rng_t( + rng.first() + rng.stride() / 2, + rank == size - 1 ? rng.last() - rng.stride() / 2 : rng.last() + rng.stride() / 2, + rng.stride()); } static rng_t rng_midpoints_out(const rng_t &rng, const int rank = 0, const int size = 1) @@ -64,10 +70,19 @@ namespace libmpdataxx rng.stride() / 2); } - static rng_t rng_half_stride(const rng_t &rng) + static rng_t rng_half_stride(const rng_t &rng, const int rank = 0, const int size = 1, const bool overlap = true) { assert(rng.stride() % 2 == 0); - return rng_t(rng.first(), rng.last(), rng.stride() / 2); + if(overlap) + return rng_t( + rank > 0 ? rng.first() - rng.stride() / 2 : rng.first(), + rank < size - 1 ? rng.last() + rng.stride() / 2 : rng.last(), + rng.stride() / 2); + else + return rng_t( + rng.first(), + rank < size - 1 ? rng.last() + rng.stride() / 2 : rng.last(), + rng.stride() / 2); } public: diff --git a/tests/sandbox/pbl/pbl_test_def.hpp b/tests/sandbox/pbl/pbl_test_def.hpp index c8601420..c853f78b 100644 --- a/tests/sandbox/pbl/pbl_test_def.hpp +++ b/tests/sandbox/pbl/pbl_test_def.hpp @@ -22,7 +22,7 @@ template void set_sgs_specific(params_t &p, iles_tag) { p.iles_cdrag = 0.1; - p.n_fra_iter = 1; + p.n_fra_iter = 2; } // smagorinsky From c7eb23a0decebc3010b4617c5ebd09e89fafeff6 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 19 Apr 2022 14:10:20 +0200 Subject: [PATCH 032/129] pbl: refine u too --- .../solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 12 ++++++------ tests/sandbox/pbl/pbl.hpp | 4 +++- tests/sandbox/pbl/pbl_test_def.hpp | 2 +- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index cb154e3e..0b665e1d 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -46,12 +46,12 @@ namespace libmpdataxx // using namespace arakawa_c; using real_t = typename ct_params_t::real_t; - if(d==0) - std::cerr << "range<" << d << ">: " << i << " " << j << " " << k << std::endl; - if(d==1) - std::cerr << "range<" << d << ">: " << k << " " << i << " " << j << std::endl; - if(d==2) - std::cerr << "range<" << d << ">: " << j << " " << k << " " << i << std::endl; +// if(d==0) +// std::cerr << "range<" << d << ">: " << i << " " << j << " " << k << std::endl; +// if(d==1) +// std::cerr << "range<" << d << ">: " << k << " " << i << " " << j << std::endl; +// if(d==2) +// std::cerr << "range<" << d << ">: " << j << " " << k << " " << i << std::endl; // std::cerr << "range - dist: " << i - dist << " " << j << " " << k << std::endl; // std::cerr << "range + dist: " << i + dist << " " << j << " " << k << std::endl; diff --git a/tests/sandbox/pbl/pbl.hpp b/tests/sandbox/pbl/pbl.hpp index edebd0ff..32cc3687 100644 --- a/tests/sandbox/pbl/pbl.hpp +++ b/tests/sandbox/pbl/pbl.hpp @@ -63,7 +63,7 @@ class pbl : public libmpdataxx::output::hdf5_xdmfmem->refinee(ix::tht)(this->ijk_ref) = blitz::tensor::k * 1e-3; // anything - this->mem->refinee(ix::tht)(this->ijk_ref) = -1; +// this->mem->refinee(ix::tht)(this->ijk_ref) = -1; //this->mem->refinee(ix::tht) = blitz::tensor::k; // anything // copy tht to refined tht at position where they overlap @@ -74,6 +74,7 @@ class pbl : public libmpdataxx::output::hdf5_xdmfmem->refinee(ix::tht)(this->ijk_r2r); this->interpolate_refinee(ix::tht); + this->interpolate_refinee(ix::u); this->mem->barrier(); if (this->rank == 0) @@ -84,6 +85,7 @@ class pbl : public libmpdataxx::output::hdf5_xdmfrecord_aux_dsc("p", this->Phi); this->record_aux_dsc_refined("tht refined", this->mem->refinee(ix::tht)); + this->record_aux_dsc_refined("u refined", this->mem->refinee(ix::u)); } this->mem->barrier(); } diff --git a/tests/sandbox/pbl/pbl_test_def.hpp b/tests/sandbox/pbl/pbl_test_def.hpp index c853f78b..6fce5efb 100644 --- a/tests/sandbox/pbl/pbl_test_def.hpp +++ b/tests/sandbox/pbl/pbl_test_def.hpp @@ -55,7 +55,7 @@ void test(const std::string &dirname, const int np, const int nt) u, v, w, tht, vip_i=u, vip_j=v, vip_k=w, vip_den=-1 }; }; - enum { fractal_recon = libmpdataxx::opts::bit(ix::tht) }; + enum { fractal_recon = libmpdataxx::opts::bit(ix::tht) | libmpdataxx::opts::bit(ix::u) }; }; using ix = typename ct_params_t::ix; From b0fa3f3747845892129f374724c89d9cb295b6c0 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 20 Apr 2022 10:36:44 +0200 Subject: [PATCH 033/129] fra rec wip --- .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 181 +++++++++++++++++- .../mpdata_rhs_vip_prs_sgs_fra_common.hpp | 5 + tests/sandbox/pbl/pbl.hpp | 8 + 3 files changed, 189 insertions(+), 5 deletions(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 0b665e1d..1d5285b7 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -70,6 +70,36 @@ namespace libmpdataxx ); } + // fractral reconstruction (calculate 2 points based on 3) + template + void rcnstrct( + arr_t arr, + const rng_t &i, + const rng_t &j, + const rng_t &k, + const int &dist + ) + { + + // TODO: move some to formulas:: + + using idxperm::pis; +// using namespace arakawa_c; + using real_t = typename ct_params_t::real_t; + + const rng_t i_next = i + 2*dist; + + arr(pis(i, j, k)) = real_t(.5) * ( + arr(pis(i - dist, j, k)) + + arr(pis(i + dist, j, k)) + ); + + arr(pis(i_next, j, k)) = real_t(.5) * ( + arr(pis(i_next - dist, j, k)) + + arr(pis(i_next + dist, j, k)) + ); + } + void interpolate_refinee(const int e = 0) { using namespace arakawa_c; // for rng_t operator^ @@ -78,8 +108,7 @@ namespace libmpdataxx rng_t ijk_r2r_0_h, ijk_r2r_1_h, ijk_r2r_2_h; // all positions at resolution of given iteration int stride, hstride; - // TODO: fill halos of advectee perpendicular to axis x? (distmem) - + // TEMPORARY this->mem->psi_ref[e] = -1000; this->mem->barrier(); @@ -123,9 +152,6 @@ namespace libmpdataxx // fill refined array at position where it overlaps with the resolved array this->mem->refinee(e)(this->ijk_r2r) = this->mem->advectee(e)(this->ijk); -// interpolate_refinee_on_edges(); // with MPI, some refined points are at the edges of the domain; calculate them using halos of non-refined arrays -// TODO: fill halo sclr ? - for(int i=0; in_fra_iter; ++i) { // messy, because in domain decomposition (sharedmem and distmem) some refined scalars are on the edge of the subdomain... @@ -177,6 +203,151 @@ namespace libmpdataxx intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, mid_ijk_r2r_0, mid_ijk_r2r_1, hstride); +/* + if(this->rank == 0) + { + + intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, ijk_r2r_1_h, ijk_r2r_2_h, hstride); + intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, ijk_r2r_2_h, ijk_r2r_0_h, hstride); + intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, ijk_r2r_0_h, ijk_r2r_1_h, hstride); + + intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, mid_ijk_r2r_0, ijk_r2r_1_h, hstride); + //intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, mid_ijk_r2r_2, ijk_r2r_0_h, hstride); + intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, ijk_r2r_0_h, mid_ijk_r2r_1, hstride); + intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, ijk_r2r_2_h, mid_ijk_r2r_0, hstride); + + //intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, mid_ijk_r2r_2, mid_ijk_r2r_0, hstride); + intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, mid_ijk_r2r_0, mid_ijk_r2r_1, hstride); + + std::cerr << std::endl; + } + this->mem->barrier(); + if(this->rank == 1) + { + + intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, ijk_r2r_1_h, ijk_r2r_2_h, hstride); + intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, ijk_r2r_2_h, ijk_r2r_0_h, hstride); + intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, ijk_r2r_0_h, ijk_r2r_1_h, hstride); + + intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, mid_ijk_r2r_0, ijk_r2r_1_h, hstride); + //intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, mid_ijk_r2r_2, ijk_r2r_0_h, hstride); + intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, ijk_r2r_0_h, mid_ijk_r2r_1, hstride); + intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, ijk_r2r_2_h, mid_ijk_r2r_0, hstride); + + //intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, mid_ijk_r2r_2, mid_ijk_r2r_0, hstride); + intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, mid_ijk_r2r_0, mid_ijk_r2r_1, hstride); + + std::cerr << std::endl; + } + */ + + this->mem->barrier(); + } + } + + void reconstruct_refinee(const int e = 0) + { + using namespace arakawa_c; // for rng_t operator^ + + rng_t mid_ijk_r2r_0, mid_ijk_r2r_1, mid_ijk_r2r_2; // positions between already known values (to be filled during given iteration) + rng_t ijk_r2r_0_h, ijk_r2r_1_h, ijk_r2r_2_h; // all positions at resolution of given iteration + int stride, hstride; + + // TEMPORARY + this->mem->psi_ref[e] = -1000; + this->mem->barrier(); + + // fill distmem halos of refinee + // TODO: move to bcond or sth? would be filled only by remote bcond + this->mem->psi_ref[e]( + this->mem->grid_size_ref[0].first() - this->mem->n_ref/2, + this->ijk_r2r[1], + this->ijk_r2r[2] + ) = + this->mem->psi[e][0]( + this->ijk[0].first()-1, + this->ijk[1], + this->ijk[2] + ); + this->mem->psi_ref[e]( + this->mem->grid_size_ref[0].last() + this->mem->n_ref/2, + this->ijk_r2r[1], + this->ijk_r2r[2] + ) = + this->mem->psi[e][0]( + this->ijk[0].last()+1, + this->ijk[1], + this->ijk[2] + ); + +// std::cerr << "post left halo fill psi_ref: " << this->mem->psi_ref[e]; +// std::cerr << "post left halo fill psi: " << this->mem->psi[e][0]; +// std::cerr << this->mem->psi_ref[e]( +// this->ijk_r2r[0].first()-1, +// this->ijk_r2r[1], +// this->ijk_r2r[2] +// ); +// std::cerr << this->mem->psi[e][0]( +// this->ijk[0].first()-1, +// this->ijk[1], +// this->ijk[2] +// ); + + + // fill refined array at position where it overlaps with the resolved array + this->mem->refinee(e)(this->ijk_r2r) = this->mem->advectee(e)(this->ijk); + + for(int i=0; in_fra_iter; ++i) + { + // messy, because in domain decomposition (sharedmem and distmem) some refined scalars are on the edge of the subdomain... + if(i==0) + { + mid_ijk_r2r_0 = this->rng_midpoints(this->ijk_r2r[0], this->mem->distmem.rank(), this->mem->distmem.size()); + mid_ijk_r2r_1 = this->rng_midpoints(this->ijk_r2r[1], this->rank, this->mem->size, false); + mid_ijk_r2r_2 = this->rng_midpoints(this->ijk_r2r[2]); + + ijk_r2r_0_h = this->ijk_r2r[0]; + ijk_r2r_1_h = this->ijk_r2r[1]; + ijk_r2r_2_h = this->ijk_r2r[2]; + } + else + { + if(i==1) + { + mid_ijk_r2r_0 = this->rng_midpoints_out(mid_ijk_r2r_0, this->mem->distmem.rank(), this->mem->distmem.size()); + if(this->rank > 0) + mid_ijk_r2r_1 = rng_t(mid_ijk_r2r_1.first() - mid_ijk_r2r_1.stride(), mid_ijk_r2r_1.last(), mid_ijk_r2r_1.stride()); // shift back to an overlapping range along y + mid_ijk_r2r_1 = this->rng_midpoints_out(mid_ijk_r2r_1, this->rank, this->mem->size); + mid_ijk_r2r_2 = this->rng_midpoints_out(mid_ijk_r2r_2); + } + else + { + mid_ijk_r2r_0 = this->rng_midpoints_out(mid_ijk_r2r_0); + mid_ijk_r2r_1 = this->rng_midpoints_out(mid_ijk_r2r_1); + mid_ijk_r2r_2 = this->rng_midpoints_out(mid_ijk_r2r_2); + } + + ijk_r2r_0_h = this->rng_half_stride(ijk_r2r_0_h, this->mem->distmem.rank(), this->mem->distmem.size()); + ijk_r2r_1_h = this->rng_half_stride(ijk_r2r_1_h, this->rank, this->mem->size, false); + ijk_r2r_2_h = this->rng_half_stride(ijk_r2r_2_h); + } + + stride = ijk_r2r_0_h.stride(); + assert(ijk_r2r_0_h.stride() == ijk_r2r_1_h.stride() == ijk_r2r_2_h.stride()); + assert(stride % 2 == 0); + hstride = stride / 2; + + rcnstrct<0>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_0), ijk_r2r_1_h, ijk_r2r_2_h, hstride); + rcnstrct<1>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_1), ijk_r2r_2_h, ijk_r2r_0_h, hstride); + rcnstrct<2>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_2), ijk_r2r_0_h, ijk_r2r_1_h, hstride); + + rcnstrct<2>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_2), mid_ijk_r2r_0, ijk_r2r_1_h, hstride); + rcnstrct<2>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_2), ijk_r2r_0_h, mid_ijk_r2r_1, hstride); + this->mem->barrier(); // necessary before interpolation along sharedmem y direction + rcnstrct<1>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_1), ijk_r2r_2_h, mid_ijk_r2r_0, hstride); + + rcnstrct<2>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_2), mid_ijk_r2r_0, mid_ijk_r2r_1, hstride); + /* if(this->rank == 0) { diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp index 6c42e4fe..e8e9453b 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp @@ -85,6 +85,11 @@ namespace libmpdataxx rng.stride() / 2); } + static rng_t rng_dbl_stride(const rng_t &rng) + { + return rng_t(rng.first(), rng.last(), 2*rng.stride()); + } + public: struct rt_params_t : parent_t::rt_params_t diff --git a/tests/sandbox/pbl/pbl.hpp b/tests/sandbox/pbl/pbl.hpp index 32cc3687..1ff156e4 100644 --- a/tests/sandbox/pbl/pbl.hpp +++ b/tests/sandbox/pbl/pbl.hpp @@ -88,6 +88,14 @@ class pbl : public libmpdataxx::output::hdf5_xdmfrecord_aux_dsc_refined("u refined", this->mem->refinee(ix::u)); } this->mem->barrier(); + + + this->reconstruct_refinee(ix::tht); + this->mem->barrier(); + if (this->rank == 0) + { + this->record_aux_dsc_refined("tht reconstructed", this->mem->refinee(ix::tht)); + } } } From 94af7cbb1ee66af22210c9cabb023d18039939b9 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 20 Apr 2022 11:01:57 +0200 Subject: [PATCH 034/129] fra rec in 3 steps: --- .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 33 ++++++++++++++----- .../mpdata_rhs_vip_prs_sgs_fra_common.hpp | 9 +++++ 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 1d5285b7..b17eea9f 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -192,6 +192,15 @@ namespace libmpdataxx assert(stride % 2 == 0); hstride = stride / 2; + + intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, ijk_r2r_1_h, ijk_r2r_2_h, hstride); + this->mem->barrier(); + intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, ijk_r2r_2_h, this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), hstride); + intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); + + +/* + intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, ijk_r2r_1_h, ijk_r2r_2_h, hstride); intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, ijk_r2r_2_h, ijk_r2r_0_h, hstride); intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, ijk_r2r_0_h, ijk_r2r_1_h, hstride); @@ -202,6 +211,7 @@ namespace libmpdataxx intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, ijk_r2r_2_h, mid_ijk_r2r_0, hstride); intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, mid_ijk_r2r_0, mid_ijk_r2r_1, hstride); + */ /* if(this->rank == 0) @@ -337,16 +347,23 @@ namespace libmpdataxx assert(stride % 2 == 0); hstride = stride / 2; - rcnstrct<0>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_0), ijk_r2r_1_h, ijk_r2r_2_h, hstride); - rcnstrct<1>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_1), ijk_r2r_2_h, ijk_r2r_0_h, hstride); - rcnstrct<2>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_2), ijk_r2r_0_h, ijk_r2r_1_h, hstride); - rcnstrct<2>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_2), mid_ijk_r2r_0, ijk_r2r_1_h, hstride); - rcnstrct<2>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_2), ijk_r2r_0_h, mid_ijk_r2r_1, hstride); - this->mem->barrier(); // necessary before interpolation along sharedmem y direction - rcnstrct<1>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_1), ijk_r2r_2_h, mid_ijk_r2r_0, hstride); + intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, ijk_r2r_1_h, ijk_r2r_2_h, hstride); + this->mem->barrier(); + intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, ijk_r2r_2_h, this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), hstride); + intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); - rcnstrct<2>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_2), mid_ijk_r2r_0, mid_ijk_r2r_1, hstride); + +// rcnstrct<0>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_0), ijk_r2r_1_h, ijk_r2r_2_h, hstride); +// rcnstrct<1>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_1), ijk_r2r_2_h, ijk_r2r_0_h, hstride); +// rcnstrct<2>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_2), ijk_r2r_0_h, ijk_r2r_1_h, hstride); +// +// rcnstrct<2>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_2), mid_ijk_r2r_0, ijk_r2r_1_h, hstride); +// rcnstrct<2>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_2), ijk_r2r_0_h, mid_ijk_r2r_1, hstride); +// this->mem->barrier(); // necessary before interpolation along sharedmem y direction +// rcnstrct<1>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_1), ijk_r2r_2_h, mid_ijk_r2r_0, hstride); +// +// rcnstrct<2>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_2), mid_ijk_r2r_0, mid_ijk_r2r_1, hstride); /* if(this->rank == 0) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp index e8e9453b..e2c7c771 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp @@ -90,6 +90,15 @@ namespace libmpdataxx return rng_t(rng.first(), rng.last(), 2*rng.stride()); } + static rng_t rng_merge(const rng_t &rng1, const rng_t &rng2) + { + assert(rng1.stride() == rng2.stride()); + return rng_t( + std::min(rng1.first(), rng2.first()), + std::max(rng1.last(), rng2.last()), + rng1.stride() / 2); + } + public: struct rt_params_t : parent_t::rt_params_t From 27017225ab6c00285cd1d8b207e582efb5cd8c81 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 20 Apr 2022 16:06:30 +0200 Subject: [PATCH 035/129] fra rec wip --- .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 60 ++++++++++++++++--- .../mpdata_rhs_vip_prs_sgs_fra_common.hpp | 26 ++++++-- .../solvers/mpdata_rhs_vip_prs_sgs_fra.hpp | 16 ++++- 3 files changed, 87 insertions(+), 15 deletions(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index b17eea9f..30eccd0a 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -70,25 +70,65 @@ namespace libmpdataxx ); } - // fractral reconstruction (calculate 2 points based on 3) + // fractral reconstruction (calculates 2 mid-points based on 3) + // notation following Akinlabi et al. "Fractal reconstruciton..." 2019 template void rcnstrct( arr_t arr, const rng_t &i, const rng_t &j, const rng_t &k, - const int &dist + const int &dist // number of indices between a reconstructed point and a known point ) { - // TODO: move some to formulas:: using idxperm::pis; // using namespace arakawa_c; using real_t = typename ct_params_t::real_t; + // stretching parameter, TODO: draw it from the distribution + const real_t d_1 = -pow(2, -1./3.), + d_2 = pow(2, -1./3.); + + const int x[3] = {0, 1, 2}; + + // second interpolated position (j=2, between i=1 and i=2) const rng_t i_next = i + 2*dist; + // helper references, follows Akinlabi et al. + const arr_t u_0 = arr(pis(i - dist, j, k)), + u_1 = arr(pis(i + dist, j, k)), + u_2 = arr(pis(i_next + dist, j, k)); + + arr_t c_1 = this->c_j(pis(i, j, k)), + c_2 = this->c_j(pis(i_next, j, k)), + f_1 = this->f_j(pis(i, j, k)), + f_2 = this->f_j(pis(i_next, j, k)); + + // Eqs. (5) and (6) Akinlabi et al. + c_1 = (u_1 - u_0) / (4 * dist) - d_1 * (u_2 - u_0) / (4 * dist); + c_2 = (u_2 - u_1) / (4 * dist) - d_2 * (u_2 - u_0) / (4 * dist); + f_1 = (x[2] * u_0 - x[0] * u_1) / (4 * dist) - d_1 * (x[2] * u_0 - x[0] * u_2) / (4 * dist); + f_2 = (x[2] * u_1 - x[0] * u_2) / (4 * dist) - d_2 * (x[2] * u_0 - x[0] * u_2) / (4 * dist); + + // new u, at j=1, between i=0 and i=1, result of w_j=1(u_i=1), Eq. (1) in Akinlabi et al. + arr(pis(i , j, k)) = c_1 * 2*dist + d_1 * u_1 + f_1; + // new u, at j=2, between i=1 and i=2, result of w_j=2(u_i=1), Eq. (1) in Akinlabi et al. + arr(pis(i_next, j, k)) = c_2 * 2*dist + d_2 * u_1 + f_2; + // not sure about * 2*dist here (is x_1 == 2*dist?) + +/* + c_j(pis(i, j, k)) * x_j + + d_j[0] * arr(pis(i + dist, j, k)) + // coarse-grained u at i=1 + f_j(pis(i, j, k)); + */ + + + // new u, at j=2, between i=1 and i=2, result of w2(u1) + // TODO + +/* arr(pis(i, j, k)) = real_t(.5) * ( arr(pis(i - dist, j, k)) + arr(pis(i + dist, j, k)) @@ -98,6 +138,7 @@ namespace libmpdataxx arr(pis(i_next - dist, j, k)) + arr(pis(i_next + dist, j, k)) ); + */ } void interpolate_refinee(const int e = 0) @@ -188,7 +229,7 @@ namespace libmpdataxx } stride = ijk_r2r_0_h.stride(); - assert(ijk_r2r_0_h.stride() == ijk_r2r_1_h.stride() == ijk_r2r_2_h.stride()); + assert(ijk_r2r_0_h.stride() == ijk_r2r_1_h.stride() && ijk_r2r_1_h.stride() == ijk_r2r_2_h.stride()); assert(stride % 2 == 0); hstride = stride / 2; @@ -255,6 +296,7 @@ namespace libmpdataxx } } + // TODO: very similar to interpolate refinee void reconstruct_refinee(const int e = 0) { using namespace arakawa_c; // for rng_t operator^ @@ -343,15 +385,15 @@ namespace libmpdataxx } stride = ijk_r2r_0_h.stride(); - assert(ijk_r2r_0_h.stride() == ijk_r2r_1_h.stride() == ijk_r2r_2_h.stride()); + assert(ijk_r2r_0_h.stride() == ijk_r2r_1_h.stride() && ijk_r2r_1_h.stride() == ijk_r2r_2_h.stride()); assert(stride % 2 == 0); hstride = stride / 2; - intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, ijk_r2r_1_h, ijk_r2r_2_h, hstride); + rcnstrct<0>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_0), ijk_r2r_1_h, ijk_r2r_2_h, hstride); this->mem->barrier(); - intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, ijk_r2r_2_h, this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), hstride); - intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); + rcnstrct<1>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_1), ijk_r2r_2_h, this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), hstride); + rcnstrct<2>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_2), this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); // rcnstrct<0>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_0), ijk_r2r_1_h, ijk_r2r_2_h, hstride); @@ -423,7 +465,7 @@ namespace libmpdataxx for (int n = 0; n < n_arr; ++n) mem->tmp[__file__].back().push_back(mem->old(new typename parent_t::arr_t( - mem->grid_size_ref[0]^1, // NOTE: halo size 1 along x, because in distmem reconstructed point is at the edge of a domain + mem->grid_size_ref[0]^(mem->n_ref/2), // NOTE: halo size 1 along x, because in distmem reconstructed point is at the edge of a domain mem->grid_size_ref[1], srfc ? rng_t(0, 0) : mem->grid_size_ref[2], arr3D_storage diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp index e2c7c771..11f9acce 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp @@ -31,19 +31,26 @@ namespace libmpdataxx protected: + typename parent_t::arr_t c_j, f_j; // parameters used in fractal reconstruction, Akinlabi et al. 2019 + + // const int n_fra; // number of fields with fractal reconstruction - const int n_ref, // number of refinements; refined resolution is dx / n_ref - n_fra_iter; + const int n_ref, // number of refinements; refined resolution is dx / n_ref + n_fra_iter; // number of iterations of grid refinement // refined arrays have no halos (do they need them? halos can be added by extending grid_size in alloc() in mpdata_rhs_vip_prs_sgs_fra.hpp by e.g. mem->n_ref/2) // const int halo_ref; // size of halos of refined scalar (and vector?) arrays + // herlper ranges // TODO: make these const! idx_t ijk_ref; // range of refinee handled by given solver const idxs_t ijk_r2r; // resolved to refined; refined scalars at the same position as resolved scalars // static rng_t rng_sclr_ref(const rng_t &rng) { return rng^halo_ref; } + + // range modifying methods used in grid refinement + static rng_t rng_midpoints(const rng_t &rng, const int rank = 0, const int size = 1, const bool overlap = true) // input range, rank and size of thread / MPI process, should created ranges have overlaps between different ranks { assert(rng.stride() % 2 == 0); @@ -85,9 +92,18 @@ namespace libmpdataxx rng.stride() / 2); } + // reconstruction based on 3 points + // input is an array pointing to midpoints (to be reconstructed), but with a stride jumping on each one + // returned range points to the first from the pair of reconstructed points + // to avoid boundary conditions, in y and z directions it is assumed that the number of points is 3+i*2, i=0,1,2,... (this check is somewhere else) + // however in the x direction there can be any number of points, because the domain is divided between mpi processes... static rng_t rng_dbl_stride(const rng_t &rng) { - return rng_t(rng.first(), rng.last(), 2*rng.stride()); + assert(rng.last() != rng.first()); // we need at least 2 midpoints + if( ((rng.last() - rng.first()) / rng.stride() + 1) % 2 == 0) // even number of midpoints; y and z directions (and sometimes x) + return rng_t(rng.first(), rng.last() - rng.stride(), 2*rng.stride()); + else // uneven number of midpoints + return rng_t(rng.first(), rng.last(), 2*rng.stride()); // rely on the halo along x direction } static rng_t rng_merge(const rng_t &rng1, const rng_t &rng2) @@ -113,7 +129,7 @@ namespace libmpdataxx ) : parent_t(args, p), n_fra_iter(p.n_fra_iter), - n_ref(this->mem->n_ref), + n_ref(args.mem->n_ref), // halo_ref(n_ref / 2), // ijk_ref init below assumes 3D (shmem decomp dim along y); // TODO: move this to concurr_common::init()? add something to ctor_args_t? @@ -124,7 +140,7 @@ namespace libmpdataxx } { assert(p.n_fra_iter > 0); - assert(n_ref & 2 == 0); + assert(n_ref % 2 == 0); for (int d = 0; d < ct_params_t::n_dims; ++d) { diff --git a/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp b/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp index 1fa3534d..5b503e96 100644 --- a/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp +++ b/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp @@ -37,17 +37,31 @@ namespace libmpdataxx > : public detail::mpdata_rhs_vip_prs_sgs_fra_dim { using parent_t = detail::mpdata_rhs_vip_prs_sgs_fra_dim; - using parent_t::parent_t; // inheriting constructors +// using parent_t::parent_t; // inheriting constructors protected: using solver_family = mpdata_rhs_vip_prs_sgs_fra_family_tag; + //ctor + mpdata_rhs_vip_prs_sgs_fra( + typename parent_t::ctor_args_t args, + const typename parent_t::rt_params_t &p + ) : + parent_t(args, p) + { + this->c_j.reference(args.mem->tmp[__FILE__][1][0]); + this->f_j.reference(args.mem->tmp[__FILE__][1][1]); + } + + // why alloc here? static void alloc( typename parent_t::mem_t *mem, const int &n_iters ) { parent_t::alloc(mem, n_iters); + // TODO: allocate only for fields that we want to reconstruct, defined in ct_params_t::fractal_recon, will this mess with field numbering? e.g. ix::tht_ref.... parent_t::alloc_tmp_sclr_ref(mem, __FILE__, ct_params_t::n_eqns); // psi_ref + parent_t::alloc_tmp_sclr_ref(mem, __FILE__, 2); // c_j, f_j // parent_t::alloc_tmp_vctr(mem, __FILE__, mem->grid_size_ref); // GC_ref mem->psi_ref = mem->tmp[__FILE__][0]; From 93294f27ce1c9e3af9df3f1cec47fa0f1154ab3a Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 21 Apr 2022 14:48:44 +0200 Subject: [PATCH 036/129] fra rec wip --- .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 102 ++++++++++++++++-- .../mpdata_rhs_vip_prs_sgs_fra_common.hpp | 2 +- tests/sandbox/pbl/pbl_test_def.hpp | 2 +- 3 files changed, 94 insertions(+), 12 deletions(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 30eccd0a..0805f76a 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -46,12 +46,14 @@ namespace libmpdataxx // using namespace arakawa_c; using real_t = typename ct_params_t::real_t; -// if(d==0) -// std::cerr << "range<" << d << ">: " << i << " " << j << " " << k << std::endl; -// if(d==1) -// std::cerr << "range<" << d << ">: " << k << " " << i << " " << j << std::endl; -// if(d==2) -// std::cerr << "range<" << d << ">: " << j << " " << k << " " << i << std::endl; + // debug output + std::cerr << "ranges in interpolation" << std::endl; + if(d==0) + std::cerr << "range<" << d << ">: " << i << " " << j << " " << k << std::endl; + if(d==1) + std::cerr << "range<" << d << ">: " << k << " " << i << " " << j << std::endl; + if(d==2) + std::cerr << "range<" << d << ">: " << j << " " << k << " " << i << std::endl; // std::cerr << "range - dist: " << i - dist << " " << j << " " << k << std::endl; // std::cerr << "range + dist: " << i + dist << " " << j << " " << k << std::endl; @@ -87,16 +89,95 @@ namespace libmpdataxx // using namespace arakawa_c; using real_t = typename ct_params_t::real_t; + // second reconstructed position (j=2, between i=1 and i=2) + const rng_t i_next = i + 2*dist; + + // debug output + std::cerr << "ranges in rcnstrct" << std::endl; + if(d==0) + std::cerr << "range<" << d << ">: " << i << " " << j << " " << k << std::endl; + if(d==1) + std::cerr << "range<" << d << ">: " << k << " " << i << " " << j << std::endl; + if(d==2) + std::cerr << "range<" << d << ">: " << j << " " << k << " " << i << std::endl; + // do interpolation, useful for testing if ranges are correct + /* + arr(pis(i, j, k)) = real_t(.5) * ( + arr(pis(i - dist, j, k)) + + arr(pis(i + dist, j, k)) + ); + arr(pis(i_next, j, k)) = real_t(.5) * ( + arr(pis(i_next - dist, j, k)) + + arr(pis(i_next + dist, j, k)) + ); + return; + */ + // end of the interpolation test + + // stretching parameter, TODO: draw it from the distribution const real_t d_1 = -pow(2, -1./3.), d_2 = pow(2, -1./3.); - const int x[3] = {0, 1, 2}; - // second interpolated position (j=2, between i=1 and i=2) - const rng_t i_next = i + 2*dist; +// solution following Scotti and Meneveau Physica D 1999 + + // helper references + // NOTE: these are actually the resolved values only in the first iteration, in the subsequent iterations some of these are reconstructed values + // should we point to the resolved values in all iterations?? + const arr_t u_im1 = arr(pis(i - dist, j, k)), // resolved u_(i-1) + u_i = arr(pis(i + dist, j, k)), // resolved u_(i) + u_ip1 = arr(pis(i_next + dist, j, k)); // resolved u_(i+1) + + // c_j and f_j naming comes from the Akinlabi paper, TODO: rename to a_i b_i + // NOTE: in multiple iterations of reconstruction, they only change due to random stretching parameters? + // or the stretching parameter should be drawn from the distro once? then no need for c_j and f_j to have the same size as reconstructed fields... + arr_t a_1 = this->c_j(pis(i, j, k)), + a_2 = this->c_j(pis(i_next, j, k)), + b_1 = this->f_j(pis(i, j, k)), + b_2 = this->f_j(pis(i_next, j, k)); + + // Eqs. (13) and (14) from Scotti Meneveau 1999 + a_1 = u_i - u_im1 - d_1*(u_ip1 - u_im1); + a_2 = u_ip1 - u_i - d_2*(u_ip1 - u_im1); + b_1 = u_im1*(real_t(1) - d_1); + b_2 = u_i - d_2*u_im1; + + // W(u_0(xi)) at xi=1/4, between u_i-1 and u_i + // u_0(2*xi) = u_i-1 + 1/2 * (u_i+1 - u_i-1) + // Eq. (6) and caption of fig. 1 Scotti 1999 + // NOTE: in multiple iterations, should xi be between the 3 resolved points? or should we consider the reconstructed points as new resolved points? + arr(pis(i, j, k)) = d_1 * +// (u_im1 + 0.5 * (u_ip1 - u_im1)) + // u_0(2*xi) + u_i + + 0.5 * a_1 + b_1; // q_1(2*xi) + // W(u_0(xi)) at xi=3/4, between u_i and u_i+1 + // u_0(2*xi-1) = u_i-1 + 1/2 * (u_i+1 - u_i-1) + // NOTE: u_0(2*3/4-1) == u_0(2*1/4), so calculate it once in the previous step and store it... + arr(pis(i_next, j, k)) = d_2 * +// (u_im1 + 0.5 * (u_ip1 - u_im1)) + // u_0(2*xi-1) + u_i + + 0.5 * a_2 + b_2; // q_2(2*xi-1) + + // Eq. (5) Scotti and Meneveau PRL 1997, a_1 and a_2 differ from Scotti Meneveau 1999 ! + // NOTE: in the 1999 paper, a_1 and a_2 are multipled by 2*xi and 2*xi-1, respectively + // in the 1997 paper, the are both multipled by xi + /* + a_1 = 2 * (u_im1 - u_i - d_1*(u_ip1 - u_im1)); + a_2 = 2 * (u_ip1 - u_i - d_2*(u_ip1 - u_im1)); + b_1 = u_im1*(real_t(1) - d_1); + b_2 = u_i - d_2*u_im1; + */ + + //const real_t xi[3] = {0, 0.5, 1}; + - // helper references, follows Akinlabi et al. +// solution following Akinlabi et al. +/* + + const int x[3] = {0, 1, 2}; + + // helper references const arr_t u_0 = arr(pis(i - dist, j, k)), u_1 = arr(pis(i + dist, j, k)), u_2 = arr(pis(i_next + dist, j, k)); @@ -117,6 +198,7 @@ namespace libmpdataxx // new u, at j=2, between i=1 and i=2, result of w_j=2(u_i=1), Eq. (1) in Akinlabi et al. arr(pis(i_next, j, k)) = c_2 * 2*dist + d_2 * u_1 + f_2; // not sure about * 2*dist here (is x_1 == 2*dist?) + */ /* c_j(pis(i, j, k)) * x_j + diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp index 11f9acce..24d2baa9 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp @@ -102,7 +102,7 @@ namespace libmpdataxx assert(rng.last() != rng.first()); // we need at least 2 midpoints if( ((rng.last() - rng.first()) / rng.stride() + 1) % 2 == 0) // even number of midpoints; y and z directions (and sometimes x) return rng_t(rng.first(), rng.last() - rng.stride(), 2*rng.stride()); - else // uneven number of midpoints + else // odd number of midpoints return rng_t(rng.first(), rng.last(), 2*rng.stride()); // rely on the halo along x direction } diff --git a/tests/sandbox/pbl/pbl_test_def.hpp b/tests/sandbox/pbl/pbl_test_def.hpp index 6fce5efb..32ebf6f5 100644 --- a/tests/sandbox/pbl/pbl_test_def.hpp +++ b/tests/sandbox/pbl/pbl_test_def.hpp @@ -22,7 +22,7 @@ template void set_sgs_specific(params_t &p, iles_tag) { p.iles_cdrag = 0.1; - p.n_fra_iter = 2; + p.n_fra_iter = 1; } // smagorinsky From 9866e7c9d9167b9065aa7d28af396e97745d821b Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 25 Apr 2022 15:49:46 +0200 Subject: [PATCH 037/129] fra rec wip --- .../concurr/detail/sharedmem_refined.hpp | 5 +- .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 86 +++++++++++++++---- .../mpdata_rhs_vip_prs_sgs_fra_common.hpp | 2 +- tests/sandbox/pbl/pbl_iles_short.cpp | 2 +- tests/sandbox/pbl/pbl_test_def.hpp | 2 +- 5 files changed, 76 insertions(+), 21 deletions(-) diff --git a/libmpdata++/concurr/detail/sharedmem_refined.hpp b/libmpdata++/concurr/detail/sharedmem_refined.hpp index f3e9f1a5..b63ae176 100644 --- a/libmpdata++/concurr/detail/sharedmem_refined.hpp +++ b/libmpdata++/concurr/detail/sharedmem_refined.hpp @@ -81,9 +81,10 @@ namespace libmpdataxx const int &mpi_size ) { assert(n_ref % 2 == 0); + // NOTE: overlapping points inbetween MPI domains return rng_t( - mpi_rank == 0 ? grid_size.first() * n_ref : grid_size.first() * n_ref - (n_ref / 2), - mpi_rank == mpi_size-1 ? grid_size.last() * n_ref : grid_size.last() * n_ref + (n_ref / 2) + mpi_rank == 0 ? grid_size.first() * n_ref : grid_size.first() * n_ref, + mpi_rank == mpi_size-1 ? grid_size.last() * n_ref : grid_size.last() * n_ref + n_ref - 1 // refined points between MPI domains belong to the MPI process on the left ); } }; diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 0805f76a..0db27ab7 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -237,26 +237,55 @@ namespace libmpdataxx // fill distmem halos of refinee // TODO: move to bcond or sth? would be filled only by remote bcond +//if(this->mem->distmem.rank < size-1) +//{ this->mem->psi_ref[e]( - this->mem->grid_size_ref[0].first() - this->mem->n_ref/2, + this->mem->grid_size_ref[0].last() + 1, this->ijk_r2r[1], this->ijk_r2r[2] ) = this->mem->psi[e][0]( - this->ijk[0].first()-1, + this->ijk[0].last()+1, this->ijk[1], this->ijk[2] ); this->mem->psi_ref[e]( - this->mem->grid_size_ref[0].last() + this->mem->n_ref/2, + this->mem->grid_size_ref[0].last() + 1 + this->mem->n_ref , this->ijk_r2r[1], this->ijk_r2r[2] ) = this->mem->psi[e][0]( - this->ijk[0].last()+1, + this->ijk[0].last()+2, // MPI halo size 2 needed! this->ijk[1], this->ijk[2] ); +//} + +//if(this->mem->distmem.rank > 0) +//{ + this->mem->psi_ref[e]( + this->mem->grid_size_ref[0].first() - this->mem->n_ref, + this->ijk_r2r[1], + this->ijk_r2r[2] + ) = + this->mem->psi[e][0]( + this->ijk[0].first()-1, // MPI halo size 2 needed! + this->ijk[1], + this->ijk[2] + ); + /* + this->mem->psi_ref[e]( + this->mem->grid_size_ref[0].first() - this->mem->n_ref/2, + this->ijk_r2r[1], + this->ijk_r2r[2] + ) = + this->mem->psi[e][0]( + this->ijk[0].first()-1, + this->ijk[1], + this->ijk[2] + ); + */ +//} // std::cerr << "post left halo fill psi_ref: " << this->mem->psi_ref[e]; // std::cerr << "post left halo fill psi: " << this->mem->psi[e][0]; @@ -316,10 +345,18 @@ namespace libmpdataxx hstride = stride / 2; - intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, ijk_r2r_1_h, ijk_r2r_2_h, hstride); +// intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, ijk_r2r_1_h, ijk_r2r_2_h, hstride); +// this->mem->barrier(); +// intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, ijk_r2r_2_h, this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), hstride); +// intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); +std::cerr << "this->mem->grid_size[0]: " << this->mem->grid_size[0] << std::endl; +std::cerr << "this->mem->grid_size_ref[0]: " << this->mem->grid_size_ref[0] << std::endl; +std::cerr << "this->ijk_r2r[0]: " << this->ijk_r2r[0] << std::endl; +std::cerr << "mid_ijk_r2r_0: " << mid_ijk_r2r_0 << std::endl; + intrp<0>(this->mem->psi_ref[e], mid_ijk_r2r_0, ijk_r2r_1_h, ijk_r2r_2_h, hstride); this->mem->barrier(); - intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, ijk_r2r_2_h, this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), hstride); - intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); + intrp<1>(this->mem->psi_ref[e], mid_ijk_r2r_1, ijk_r2r_2_h, this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), hstride); + intrp<2>(this->mem->psi_ref[e], mid_ijk_r2r_2, this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); /* @@ -393,26 +430,43 @@ namespace libmpdataxx // fill distmem halos of refinee // TODO: move to bcond or sth? would be filled only by remote bcond + this->mem->psi_ref[e]( - this->mem->grid_size_ref[0].first() - this->mem->n_ref/2, + this->mem->grid_size_ref[0].last() + 1, this->ijk_r2r[1], this->ijk_r2r[2] ) = this->mem->psi[e][0]( - this->ijk[0].first()-1, + this->ijk[0].last()+1, this->ijk[1], this->ijk[2] ); this->mem->psi_ref[e]( - this->mem->grid_size_ref[0].last() + this->mem->n_ref/2, + this->mem->grid_size_ref[0].last() + 1 + this->mem->n_ref , this->ijk_r2r[1], this->ijk_r2r[2] ) = this->mem->psi[e][0]( - this->ijk[0].last()+1, + this->ijk[0].last()+2, // MPI halo size 2 needed! this->ijk[1], this->ijk[2] ); +//} + +//if(this->mem->distmem.rank > 0) +//{ + this->mem->psi_ref[e]( + this->mem->grid_size_ref[0].first() - this->mem->n_ref, + this->ijk_r2r[1], + this->ijk_r2r[2] + ) = + this->mem->psi[e][0]( + this->ijk[0].first()-1, // MPI halo size 2 needed! + this->ijk[1], + this->ijk[2] + ); + + // std::cerr << "post left halo fill psi_ref: " << this->mem->psi_ref[e]; // std::cerr << "post left halo fill psi: " << this->mem->psi[e][0]; @@ -429,7 +483,7 @@ namespace libmpdataxx // fill refined array at position where it overlaps with the resolved array - this->mem->refinee(e)(this->ijk_r2r) = this->mem->advectee(e)(this->ijk); +// this->mem->refinee(e)(this->ijk_r2r) = this->mem->advectee(e)(this->ijk); for(int i=0; in_fra_iter; ++i) { @@ -472,10 +526,10 @@ namespace libmpdataxx hstride = stride / 2; - rcnstrct<0>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_0), ijk_r2r_1_h, ijk_r2r_2_h, hstride); + rcnstrct<0>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_0), ijk_r2r_1_h, ijk_r2r_2_h, hstride); this->mem->barrier(); - rcnstrct<1>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_1), ijk_r2r_2_h, this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), hstride); - rcnstrct<2>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_2), this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); +// rcnstrct<1>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_1), ijk_r2r_2_h, this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), hstride); +// rcnstrct<2>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_2), this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); // rcnstrct<0>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_0), ijk_r2r_1_h, ijk_r2r_2_h, hstride); @@ -547,7 +601,7 @@ namespace libmpdataxx for (int n = 0; n < n_arr; ++n) mem->tmp[__file__].back().push_back(mem->old(new typename parent_t::arr_t( - mem->grid_size_ref[0]^(mem->n_ref/2), // NOTE: halo size 1 along x, because in distmem reconstructed point is at the edge of a domain + mem->grid_size_ref[0]^(mem->n_ref+1), // reconstruction based on 3 points, we need up to 2 resolved points outside of the domain mem->grid_size_ref[1], srfc ? rng_t(0, 0) : mem->grid_size_ref[2], arr3D_storage diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp index 24d2baa9..f2e75173 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp @@ -55,7 +55,7 @@ namespace libmpdataxx { assert(rng.stride() % 2 == 0); if(rng.last() - rng.first() < rng.stride()) return rng; - else if (overlap) + if (overlap) return rng_t( rank == 0 ? rng.first() + rng.stride() / 2 : rng.first() - rng.stride() / 2, rank == size - 1 ? rng.last() - rng.stride() / 2 : rng.last() + rng.stride() / 2, diff --git a/tests/sandbox/pbl/pbl_iles_short.cpp b/tests/sandbox/pbl/pbl_iles_short.cpp index 96bd847c..82cce4e6 100644 --- a/tests/sandbox/pbl/pbl_iles_short.cpp +++ b/tests/sandbox/pbl/pbl_iles_short.cpp @@ -9,5 +9,5 @@ int main() { - test("out_pbl_iles_short", 33, 601); + test("out_pbl_iles_short", 33, 5); } diff --git a/tests/sandbox/pbl/pbl_test_def.hpp b/tests/sandbox/pbl/pbl_test_def.hpp index 32ebf6f5..580f2b97 100644 --- a/tests/sandbox/pbl/pbl_test_def.hpp +++ b/tests/sandbox/pbl/pbl_test_def.hpp @@ -82,7 +82,7 @@ void test(const std::string &dirname, const int np, const int nt) double mixed_length = 500; double st = 1e-4 / p.g; - p.outfreq = 150; + p.outfreq = 1; p.outwindow = 1; p.outvars = { {ix::u, { "u", "m/s"}}, From ab01ef02544da464b02f08e0009323a370d5a987 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 25 Apr 2022 18:18:24 +0200 Subject: [PATCH 038/129] new ref ranges working interpolation --- .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 61 ++++++++++++------- .../mpdata_rhs_vip_prs_sgs_fra_common.hpp | 10 ++- tests/sandbox/pbl/pbl.hpp | 12 ++-- tests/sandbox/pbl/pbl_test_def.hpp | 2 +- 4 files changed, 54 insertions(+), 31 deletions(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 0db27ab7..b1cb9062 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -239,6 +239,10 @@ namespace libmpdataxx // TODO: move to bcond or sth? would be filled only by remote bcond //if(this->mem->distmem.rank < size-1) //{ + + // TODO: we only need to xchng along distmem direction (x) + this->xchng(e); + this->mem->psi_ref[e]( this->mem->grid_size_ref[0].last() + 1, this->ijk_r2r[1], @@ -249,16 +253,16 @@ namespace libmpdataxx this->ijk[1], this->ijk[2] ); - this->mem->psi_ref[e]( - this->mem->grid_size_ref[0].last() + 1 + this->mem->n_ref , - this->ijk_r2r[1], - this->ijk_r2r[2] - ) = - this->mem->psi[e][0]( - this->ijk[0].last()+2, // MPI halo size 2 needed! - this->ijk[1], - this->ijk[2] - ); +// this->mem->psi_ref[e]( +// this->mem->grid_size_ref[0].last() + 1 + this->mem->n_ref , +// this->ijk_r2r[1], +// this->ijk_r2r[2] +// ) = +// this->mem->psi[e][0]( +// this->ijk[0].last()+2, // MPI halo size 2 needed! +// this->ijk[1], +// this->ijk[2] +// ); //} //if(this->mem->distmem.rank > 0) @@ -309,11 +313,11 @@ namespace libmpdataxx // messy, because in domain decomposition (sharedmem and distmem) some refined scalars are on the edge of the subdomain... if(i==0) { - mid_ijk_r2r_0 = this->rng_midpoints(this->ijk_r2r[0], this->mem->distmem.rank(), this->mem->distmem.size()); + mid_ijk_r2r_0 = this->rng_midpoints(this->ijk_r2r[0], this->mem->distmem.rank(), this->mem->distmem.size(), false); mid_ijk_r2r_1 = this->rng_midpoints(this->ijk_r2r[1], this->rank, this->mem->size, false); mid_ijk_r2r_2 = this->rng_midpoints(this->ijk_r2r[2]); - ijk_r2r_0_h = this->ijk_r2r[0]; + // ijk_r2r_0_h = this->ijk_r2r[0]; ijk_r2r_1_h = this->ijk_r2r[1]; ijk_r2r_2_h = this->ijk_r2r[2]; } @@ -322,8 +326,8 @@ namespace libmpdataxx if(i==1) { mid_ijk_r2r_0 = this->rng_midpoints_out(mid_ijk_r2r_0, this->mem->distmem.rank(), this->mem->distmem.size()); - if(this->rank > 0) - mid_ijk_r2r_1 = rng_t(mid_ijk_r2r_1.first() - mid_ijk_r2r_1.stride(), mid_ijk_r2r_1.last(), mid_ijk_r2r_1.stride()); // shift back to an overlapping range along y +// if(this->rank > 0) + // mid_ijk_r2r_1 = rng_t(mid_ijk_r2r_1.first() - mid_ijk_r2r_1.stride(), mid_ijk_r2r_1.last(), mid_ijk_r2r_1.stride()); // shift back to an overlapping range along y mid_ijk_r2r_1 = this->rng_midpoints_out(mid_ijk_r2r_1, this->rank, this->mem->size); mid_ijk_r2r_2 = this->rng_midpoints_out(mid_ijk_r2r_2); } @@ -334,13 +338,15 @@ namespace libmpdataxx mid_ijk_r2r_2 = this->rng_midpoints_out(mid_ijk_r2r_2); } - ijk_r2r_0_h = this->rng_half_stride(ijk_r2r_0_h, this->mem->distmem.rank(), this->mem->distmem.size()); +// ijk_r2r_0_h = this->rng_half_stride(ijk_r2r_0_h, this->mem->distmem.rank(), this->mem->distmem.size()); ijk_r2r_1_h = this->rng_half_stride(ijk_r2r_1_h, this->rank, this->mem->size, false); ijk_r2r_2_h = this->rng_half_stride(ijk_r2r_2_h); } - stride = ijk_r2r_0_h.stride(); - assert(ijk_r2r_0_h.stride() == ijk_r2r_1_h.stride() && ijk_r2r_1_h.stride() == ijk_r2r_2_h.stride()); + //stride = ijk_r2r_0_h.stride(); + stride = ijk_r2r_1_h.stride(); + //assert(ijk_r2r_0_h.stride() == ijk_r2r_1_h.stride() && ijk_r2r_1_h.stride() == ijk_r2r_2_h.stride()); + assert(ijk_r2r_1_h.stride() == ijk_r2r_2_h.stride()); assert(stride % 2 == 0); hstride = stride / 2; @@ -353,10 +359,21 @@ std::cerr << "this->mem->grid_size[0]: " << this->mem->grid_size[0] << std::endl std::cerr << "this->mem->grid_size_ref[0]: " << this->mem->grid_size_ref[0] << std::endl; std::cerr << "this->ijk_r2r[0]: " << this->ijk_r2r[0] << std::endl; std::cerr << "mid_ijk_r2r_0: " << mid_ijk_r2r_0 << std::endl; + +if(this->rank==0) + std::cerr << "(63,2,0) before intrp iter " << i << " : " << this->mem->psi_ref[e](63,2,0) << std::endl; intrp<0>(this->mem->psi_ref[e], mid_ijk_r2r_0, ijk_r2r_1_h, ijk_r2r_2_h, hstride); + std::cerr << "(63,2,0) after intrp<0> iter " << i << " : " << this->mem->psi_ref[e](63,2,0) << std::endl; this->mem->barrier(); - intrp<1>(this->mem->psi_ref[e], mid_ijk_r2r_1, ijk_r2r_2_h, this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), hstride); - intrp<2>(this->mem->psi_ref[e], mid_ijk_r2r_2, this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); + +// intrp<1>(this->mem->psi_ref[e], mid_ijk_r2r_1, ijk_r2r_2_h, this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), hstride); + const int halo_size = 1; // 2 for reconstruction + const rng_t ijk_0_with_halo_h(this->ijk_r2r[0].first(), this->ijk_r2r[0].last() + halo_size * this->mem->n_ref, hstride); + intrp<1>(this->mem->psi_ref[e], mid_ijk_r2r_1, ijk_r2r_2_h, ijk_0_with_halo_h, hstride); + + std::cerr << "(63,2,0) after intrp<1> iter " << i << " : " << this->mem->psi_ref[e](63,2,0) << std::endl; + //intrp<2>(this->mem->psi_ref[e], mid_ijk_r2r_2, this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); + intrp<2>(this->mem->psi_ref[e], mid_ijk_r2r_2, ijk_0_with_halo_h, this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); /* @@ -431,6 +448,8 @@ std::cerr << "mid_ijk_r2r_0: " << mid_ijk_r2r_0 << std::endl; // fill distmem halos of refinee // TODO: move to bcond or sth? would be filled only by remote bcond + // TODO: we only need to xchng along distmem direction (x) + this->xchng(e); this->mem->psi_ref[e]( this->mem->grid_size_ref[0].last() + 1, this->ijk_r2r[1], @@ -528,8 +547,8 @@ std::cerr << "mid_ijk_r2r_0: " << mid_ijk_r2r_0 << std::endl; rcnstrct<0>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_0), ijk_r2r_1_h, ijk_r2r_2_h, hstride); this->mem->barrier(); -// rcnstrct<1>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_1), ijk_r2r_2_h, this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), hstride); -// rcnstrct<2>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_2), this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); + rcnstrct<1>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_1), ijk_r2r_2_h, this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), hstride); + rcnstrct<2>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_2), this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); // rcnstrct<0>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_0), ijk_r2r_1_h, ijk_r2r_2_h, hstride); diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp index f2e75173..143fde1d 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp @@ -20,6 +20,7 @@ namespace libmpdataxx { namespace detail { + // TODO: minimum halo size is 2 template class mpdata_rhs_vip_prs_sgs_fra_common : public mpdata_rhs_vip_prs_sgs { @@ -72,8 +73,10 @@ namespace libmpdataxx assert(rng.stride() % 4 == 0); if(rng.last() - rng.first() < rng.stride()) return rng; else return rng_t( - rank == 0 ? rng.first() - rng.stride() / 4 : rng.first() + rng.stride() / 4, - rank == size - 1 ? rng.last() + rng.stride() / 4 : rng.last() - rng.stride() / 4, + //rank == 0 ? rng.first() - rng.stride() / 4 : rng.first() + rng.stride() / 4, + rng.first() - rng.stride() / 4, + //rank == size - 1 ? rng.last() + rng.stride() / 4 : rng.last() - rng.stride() / 4, + rng.last() + rng.stride() / 4, rng.stride() / 2); } @@ -82,7 +85,8 @@ namespace libmpdataxx assert(rng.stride() % 2 == 0); if(overlap) return rng_t( - rank > 0 ? rng.first() - rng.stride() / 2 : rng.first(), +// rank > 0 ? rng.first() - rng.stride() / 2 : rng.first(), + rng.first(), rank < size - 1 ? rng.last() + rng.stride() / 2 : rng.last(), rng.stride() / 2); else diff --git a/tests/sandbox/pbl/pbl.hpp b/tests/sandbox/pbl/pbl.hpp index 1ff156e4..34236fe9 100644 --- a/tests/sandbox/pbl/pbl.hpp +++ b/tests/sandbox/pbl/pbl.hpp @@ -90,12 +90,12 @@ class pbl : public libmpdataxx::output::hdf5_xdmfmem->barrier(); - this->reconstruct_refinee(ix::tht); - this->mem->barrier(); - if (this->rank == 0) - { - this->record_aux_dsc_refined("tht reconstructed", this->mem->refinee(ix::tht)); - } +// this->reconstruct_refinee(ix::tht); +// this->mem->barrier(); +// if (this->rank == 0) +// { +// this->record_aux_dsc_refined("tht reconstructed", this->mem->refinee(ix::tht)); +// } } } diff --git a/tests/sandbox/pbl/pbl_test_def.hpp b/tests/sandbox/pbl/pbl_test_def.hpp index 580f2b97..1f89b43d 100644 --- a/tests/sandbox/pbl/pbl_test_def.hpp +++ b/tests/sandbox/pbl/pbl_test_def.hpp @@ -22,7 +22,7 @@ template void set_sgs_specific(params_t &p, iles_tag) { p.iles_cdrag = 0.1; - p.n_fra_iter = 1; + p.n_fra_iter = 2; } // smagorinsky From c620de47bffa410070bfbab701ca9f1876703322 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 25 Apr 2022 20:56:49 +0200 Subject: [PATCH 039/129] cleanup --- .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 33 +++++++---------- .../mpdata_rhs_vip_prs_sgs_fra_common.hpp | 37 ++++++------------- 2 files changed, 25 insertions(+), 45 deletions(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index b1cb9062..4cc739d0 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -313,8 +313,8 @@ namespace libmpdataxx // messy, because in domain decomposition (sharedmem and distmem) some refined scalars are on the edge of the subdomain... if(i==0) { - mid_ijk_r2r_0 = this->rng_midpoints(this->ijk_r2r[0], this->mem->distmem.rank(), this->mem->distmem.size(), false); - mid_ijk_r2r_1 = this->rng_midpoints(this->ijk_r2r[1], this->rank, this->mem->size, false); + mid_ijk_r2r_0 = this->rng_midpoints(this->ijk_r2r[0], this->mem->distmem.rank(), this->mem->distmem.size()); + mid_ijk_r2r_1 = this->rng_midpoints(this->ijk_r2r[1], this->rank, this->mem->size); mid_ijk_r2r_2 = this->rng_midpoints(this->ijk_r2r[2]); // ijk_r2r_0_h = this->ijk_r2r[0]; @@ -325,10 +325,8 @@ namespace libmpdataxx { if(i==1) { - mid_ijk_r2r_0 = this->rng_midpoints_out(mid_ijk_r2r_0, this->mem->distmem.rank(), this->mem->distmem.size()); -// if(this->rank > 0) - // mid_ijk_r2r_1 = rng_t(mid_ijk_r2r_1.first() - mid_ijk_r2r_1.stride(), mid_ijk_r2r_1.last(), mid_ijk_r2r_1.stride()); // shift back to an overlapping range along y - mid_ijk_r2r_1 = this->rng_midpoints_out(mid_ijk_r2r_1, this->rank, this->mem->size); + mid_ijk_r2r_0 = this->rng_midpoints_out(mid_ijk_r2r_0); + mid_ijk_r2r_1 = this->rng_midpoints_out(mid_ijk_r2r_1); mid_ijk_r2r_2 = this->rng_midpoints_out(mid_ijk_r2r_2); } else @@ -339,7 +337,7 @@ namespace libmpdataxx } // ijk_r2r_0_h = this->rng_half_stride(ijk_r2r_0_h, this->mem->distmem.rank(), this->mem->distmem.size()); - ijk_r2r_1_h = this->rng_half_stride(ijk_r2r_1_h, this->rank, this->mem->size, false); + ijk_r2r_1_h = this->rng_half_stride(ijk_r2r_1_h, this->rank, this->mem->size); ijk_r2r_2_h = this->rng_half_stride(ijk_r2r_2_h); } @@ -350,30 +348,27 @@ namespace libmpdataxx assert(stride % 2 == 0); hstride = stride / 2; + const int halo_size = 1; // 2 for reconstruction + const rng_t ijk_r2r_0_h_with_halo(this->ijk_r2r[0].first(), this->ijk_r2r[0].last() + halo_size * this->mem->n_ref, hstride); + // intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, ijk_r2r_1_h, ijk_r2r_2_h, hstride); // this->mem->barrier(); // intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, ijk_r2r_2_h, this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), hstride); // intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); -std::cerr << "this->mem->grid_size[0]: " << this->mem->grid_size[0] << std::endl; -std::cerr << "this->mem->grid_size_ref[0]: " << this->mem->grid_size_ref[0] << std::endl; -std::cerr << "this->ijk_r2r[0]: " << this->ijk_r2r[0] << std::endl; -std::cerr << "mid_ijk_r2r_0: " << mid_ijk_r2r_0 << std::endl; +//std::cerr << "this->mem->grid_size[0]: " << this->mem->grid_size[0] << std::endl; +//std::cerr << "this->mem->grid_size_ref[0]: " << this->mem->grid_size_ref[0] << std::endl; +//std::cerr << "this->ijk_r2r[0]: " << this->ijk_r2r[0] << std::endl; +//std::cerr << "mid_ijk_r2r_0: " << mid_ijk_r2r_0 << std::endl; -if(this->rank==0) - std::cerr << "(63,2,0) before intrp iter " << i << " : " << this->mem->psi_ref[e](63,2,0) << std::endl; intrp<0>(this->mem->psi_ref[e], mid_ijk_r2r_0, ijk_r2r_1_h, ijk_r2r_2_h, hstride); - std::cerr << "(63,2,0) after intrp<0> iter " << i << " : " << this->mem->psi_ref[e](63,2,0) << std::endl; this->mem->barrier(); // intrp<1>(this->mem->psi_ref[e], mid_ijk_r2r_1, ijk_r2r_2_h, this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), hstride); - const int halo_size = 1; // 2 for reconstruction - const rng_t ijk_0_with_halo_h(this->ijk_r2r[0].first(), this->ijk_r2r[0].last() + halo_size * this->mem->n_ref, hstride); - intrp<1>(this->mem->psi_ref[e], mid_ijk_r2r_1, ijk_r2r_2_h, ijk_0_with_halo_h, hstride); + intrp<1>(this->mem->psi_ref[e], mid_ijk_r2r_1, ijk_r2r_2_h, ijk_r2r_0_h_with_halo, hstride); - std::cerr << "(63,2,0) after intrp<1> iter " << i << " : " << this->mem->psi_ref[e](63,2,0) << std::endl; //intrp<2>(this->mem->psi_ref[e], mid_ijk_r2r_2, this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); - intrp<2>(this->mem->psi_ref[e], mid_ijk_r2r_2, ijk_0_with_halo_h, this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); + intrp<2>(this->mem->psi_ref[e], mid_ijk_r2r_2, ijk_r2r_0_h_with_halo, this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); /* diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp index 143fde1d..5b4927e3 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp @@ -52,48 +52,33 @@ namespace libmpdataxx // range modifying methods used in grid refinement - static rng_t rng_midpoints(const rng_t &rng, const int rank = 0, const int size = 1, const bool overlap = true) // input range, rank and size of thread / MPI process, should created ranges have overlaps between different ranks + static rng_t rng_midpoints(const rng_t &rng, const int rank = 0, const int size = 1) { assert(rng.stride() % 2 == 0); if(rng.last() - rng.first() < rng.stride()) return rng; - if (overlap) - return rng_t( - rank == 0 ? rng.first() + rng.stride() / 2 : rng.first() - rng.stride() / 2, - rank == size - 1 ? rng.last() - rng.stride() / 2 : rng.last() + rng.stride() / 2, - rng.stride()); - else - return rng_t( - rng.first() + rng.stride() / 2, - rank == size - 1 ? rng.last() - rng.stride() / 2 : rng.last() + rng.stride() / 2, - rng.stride()); + return rng_t( + rng.first() + rng.stride() / 2, + rank == size - 1 ? rng.last() - rng.stride() / 2 : rng.last() + rng.stride() / 2, + rng.stride()); } - static rng_t rng_midpoints_out(const rng_t &rng, const int rank = 0, const int size = 1) + static rng_t rng_midpoints_out(const rng_t &rng) { assert(rng.stride() % 4 == 0); if(rng.last() - rng.first() < rng.stride()) return rng; else return rng_t( - //rank == 0 ? rng.first() - rng.stride() / 4 : rng.first() + rng.stride() / 4, rng.first() - rng.stride() / 4, - //rank == size - 1 ? rng.last() + rng.stride() / 4 : rng.last() - rng.stride() / 4, rng.last() + rng.stride() / 4, rng.stride() / 2); } - static rng_t rng_half_stride(const rng_t &rng, const int rank = 0, const int size = 1, const bool overlap = true) + static rng_t rng_half_stride(const rng_t &rng, const int rank = 0, const int size = 1) { assert(rng.stride() % 2 == 0); - if(overlap) - return rng_t( -// rank > 0 ? rng.first() - rng.stride() / 2 : rng.first(), - rng.first(), - rank < size - 1 ? rng.last() + rng.stride() / 2 : rng.last(), - rng.stride() / 2); - else - return rng_t( - rng.first(), - rank < size - 1 ? rng.last() + rng.stride() / 2 : rng.last(), - rng.stride() / 2); + return rng_t( + rng.first(), + rank < size - 1 ? rng.last() + rng.stride() / 2 : rng.last(), + rng.stride() / 2); } // reconstruction based on 3 points From eeaba3f4278fe7e2093bc1358a01a7e786944dae Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 25 Apr 2022 21:09:12 +0200 Subject: [PATCH 040/129] cleanup + wip on reconstruction --- .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 219 ++---------------- .../mpdata_rhs_vip_prs_sgs_fra_common.hpp | 3 - tests/sandbox/pbl/pbl.hpp | 12 +- tests/sandbox/pbl/pbl_test_def.hpp | 2 +- 4 files changed, 30 insertions(+), 206 deletions(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 4cc739d0..11f35c92 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -55,17 +55,6 @@ namespace libmpdataxx if(d==2) std::cerr << "range<" << d << ">: " << j << " " << k << " " << i << std::endl; -// std::cerr << "range - dist: " << i - dist << " " << j << " " << k << std::endl; -// std::cerr << "range + dist: " << i + dist << " " << j << " " << k << std::endl; -// -// std::cerr << "arr range: " << arr(i, j, k) << std::endl; -// std::cerr << "arr range - dist: " << arr(i - dist, j, k) << std::endl; -// std::cerr << "arr range + dist: " << arr(i + dist, j, k) << std::endl; -// -// std::cerr << "arr range pi: " << arr(pis(i, j, k)) << std::endl; -// std::cerr << "arr range - dist pi: " << arr(pis(i - dist, j, k)) << std::endl; -// std::cerr << "arr range + dist pi: " << arr(pis(i + dist, j, k)) << std::endl; - arr(pis(i, j, k)) = real_t(.5) * ( arr(pis(i - dist, j, k)) + arr(pis(i + dist, j, k)) @@ -86,7 +75,6 @@ namespace libmpdataxx // TODO: move some to formulas:: using idxperm::pis; -// using namespace arakawa_c; using real_t = typename ct_params_t::real_t; // second reconstructed position (j=2, between i=1 and i=2) @@ -228,7 +216,7 @@ namespace libmpdataxx using namespace arakawa_c; // for rng_t operator^ rng_t mid_ijk_r2r_0, mid_ijk_r2r_1, mid_ijk_r2r_2; // positions between already known values (to be filled during given iteration) - rng_t ijk_r2r_0_h, ijk_r2r_1_h, ijk_r2r_2_h; // all positions at resolution of given iteration + rng_t ijk_r2r_1_h, ijk_r2r_2_h; // all positions at resolution of given iteration int stride, hstride; // TEMPORARY @@ -237,9 +225,6 @@ namespace libmpdataxx // fill distmem halos of refinee // TODO: move to bcond or sth? would be filled only by remote bcond -//if(this->mem->distmem.rank < size-1) -//{ - // TODO: we only need to xchng along distmem direction (x) this->xchng(e); @@ -248,62 +233,22 @@ namespace libmpdataxx this->ijk_r2r[1], this->ijk_r2r[2] ) = - this->mem->psi[e][0]( - this->ijk[0].last()+1, - this->ijk[1], - this->ijk[2] - ); -// this->mem->psi_ref[e]( -// this->mem->grid_size_ref[0].last() + 1 + this->mem->n_ref , -// this->ijk_r2r[1], -// this->ijk_r2r[2] -// ) = -// this->mem->psi[e][0]( -// this->ijk[0].last()+2, // MPI halo size 2 needed! -// this->ijk[1], -// this->ijk[2] -// ); -//} - -//if(this->mem->distmem.rank > 0) -//{ + this->mem->psi[e][0]( + this->ijk[0].last()+1, + this->ijk[1], + this->ijk[2] + ); + this->mem->psi_ref[e]( this->mem->grid_size_ref[0].first() - this->mem->n_ref, this->ijk_r2r[1], this->ijk_r2r[2] ) = - this->mem->psi[e][0]( - this->ijk[0].first()-1, // MPI halo size 2 needed! - this->ijk[1], - this->ijk[2] - ); - /* - this->mem->psi_ref[e]( - this->mem->grid_size_ref[0].first() - this->mem->n_ref/2, - this->ijk_r2r[1], - this->ijk_r2r[2] - ) = - this->mem->psi[e][0]( - this->ijk[0].first()-1, - this->ijk[1], - this->ijk[2] - ); - */ -//} - -// std::cerr << "post left halo fill psi_ref: " << this->mem->psi_ref[e]; -// std::cerr << "post left halo fill psi: " << this->mem->psi[e][0]; -// std::cerr << this->mem->psi_ref[e]( -// this->ijk_r2r[0].first()-1, -// this->ijk_r2r[1], -// this->ijk_r2r[2] -// ); -// std::cerr << this->mem->psi[e][0]( -// this->ijk[0].first()-1, -// this->ijk[1], -// this->ijk[2] -// ); - + this->mem->psi[e][0]( + this->ijk[0].first()-1, + this->ijk[1], + this->ijk[2] + ); // fill refined array at position where it overlaps with the resolved array this->mem->refinee(e)(this->ijk_r2r) = this->mem->advectee(e)(this->ijk); @@ -317,7 +262,6 @@ namespace libmpdataxx mid_ijk_r2r_1 = this->rng_midpoints(this->ijk_r2r[1], this->rank, this->mem->size); mid_ijk_r2r_2 = this->rng_midpoints(this->ijk_r2r[2]); - // ijk_r2r_0_h = this->ijk_r2r[0]; ijk_r2r_1_h = this->ijk_r2r[1]; ijk_r2r_2_h = this->ijk_r2r[2]; } @@ -336,14 +280,11 @@ namespace libmpdataxx mid_ijk_r2r_2 = this->rng_midpoints_out(mid_ijk_r2r_2); } -// ijk_r2r_0_h = this->rng_half_stride(ijk_r2r_0_h, this->mem->distmem.rank(), this->mem->distmem.size()); ijk_r2r_1_h = this->rng_half_stride(ijk_r2r_1_h, this->rank, this->mem->size); ijk_r2r_2_h = this->rng_half_stride(ijk_r2r_2_h); } - //stride = ijk_r2r_0_h.stride(); stride = ijk_r2r_1_h.stride(); - //assert(ijk_r2r_0_h.stride() == ijk_r2r_1_h.stride() && ijk_r2r_1_h.stride() == ijk_r2r_2_h.stride()); assert(ijk_r2r_1_h.stride() == ijk_r2r_2_h.stride()); assert(stride % 2 == 0); hstride = stride / 2; @@ -351,26 +292,11 @@ namespace libmpdataxx const int halo_size = 1; // 2 for reconstruction const rng_t ijk_r2r_0_h_with_halo(this->ijk_r2r[0].first(), this->ijk_r2r[0].last() + halo_size * this->mem->n_ref, hstride); - -// intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, ijk_r2r_1_h, ijk_r2r_2_h, hstride); -// this->mem->barrier(); -// intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, ijk_r2r_2_h, this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), hstride); -// intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); -//std::cerr << "this->mem->grid_size[0]: " << this->mem->grid_size[0] << std::endl; -//std::cerr << "this->mem->grid_size_ref[0]: " << this->mem->grid_size_ref[0] << std::endl; -//std::cerr << "this->ijk_r2r[0]: " << this->ijk_r2r[0] << std::endl; -//std::cerr << "mid_ijk_r2r_0: " << mid_ijk_r2r_0 << std::endl; - intrp<0>(this->mem->psi_ref[e], mid_ijk_r2r_0, ijk_r2r_1_h, ijk_r2r_2_h, hstride); this->mem->barrier(); - -// intrp<1>(this->mem->psi_ref[e], mid_ijk_r2r_1, ijk_r2r_2_h, this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), hstride); intrp<1>(this->mem->psi_ref[e], mid_ijk_r2r_1, ijk_r2r_2_h, ijk_r2r_0_h_with_halo, hstride); - - //intrp<2>(this->mem->psi_ref[e], mid_ijk_r2r_2, this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); intrp<2>(this->mem->psi_ref[e], mid_ijk_r2r_2, ijk_r2r_0_h_with_halo, this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); - /* intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, ijk_r2r_1_h, ijk_r2r_2_h, hstride); @@ -384,45 +310,6 @@ namespace libmpdataxx intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, mid_ijk_r2r_0, mid_ijk_r2r_1, hstride); */ - -/* - if(this->rank == 0) - { - - intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, ijk_r2r_1_h, ijk_r2r_2_h, hstride); - intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, ijk_r2r_2_h, ijk_r2r_0_h, hstride); - intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, ijk_r2r_0_h, ijk_r2r_1_h, hstride); - - intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, mid_ijk_r2r_0, ijk_r2r_1_h, hstride); - //intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, mid_ijk_r2r_2, ijk_r2r_0_h, hstride); - intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, ijk_r2r_0_h, mid_ijk_r2r_1, hstride); - intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, ijk_r2r_2_h, mid_ijk_r2r_0, hstride); - - //intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, mid_ijk_r2r_2, mid_ijk_r2r_0, hstride); - intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, mid_ijk_r2r_0, mid_ijk_r2r_1, hstride); - - std::cerr << std::endl; - } - this->mem->barrier(); - if(this->rank == 1) - { - - intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, ijk_r2r_1_h, ijk_r2r_2_h, hstride); - intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, ijk_r2r_2_h, ijk_r2r_0_h, hstride); - intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, ijk_r2r_0_h, ijk_r2r_1_h, hstride); - - intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, mid_ijk_r2r_0, ijk_r2r_1_h, hstride); - //intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, mid_ijk_r2r_2, ijk_r2r_0_h, hstride); - intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, ijk_r2r_0_h, mid_ijk_r2r_1, hstride); - intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, ijk_r2r_2_h, mid_ijk_r2r_0, hstride); - - //intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, mid_ijk_r2r_2, mid_ijk_r2r_0, hstride); - intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, mid_ijk_r2r_0, mid_ijk_r2r_1, hstride); - - std::cerr << std::endl; - } - */ - this->mem->barrier(); } } @@ -442,7 +329,6 @@ namespace libmpdataxx // fill distmem halos of refinee // TODO: move to bcond or sth? would be filled only by remote bcond - // TODO: we only need to xchng along distmem direction (x) this->xchng(e); this->mem->psi_ref[e]( @@ -465,50 +351,30 @@ namespace libmpdataxx this->ijk[1], this->ijk[2] ); -//} - -//if(this->mem->distmem.rank > 0) -//{ this->mem->psi_ref[e]( this->mem->grid_size_ref[0].first() - this->mem->n_ref, this->ijk_r2r[1], this->ijk_r2r[2] ) = this->mem->psi[e][0]( - this->ijk[0].first()-1, // MPI halo size 2 needed! + this->ijk[0].first()-1, this->ijk[1], this->ijk[2] ); - - -// std::cerr << "post left halo fill psi_ref: " << this->mem->psi_ref[e]; -// std::cerr << "post left halo fill psi: " << this->mem->psi[e][0]; -// std::cerr << this->mem->psi_ref[e]( -// this->ijk_r2r[0].first()-1, -// this->ijk_r2r[1], -// this->ijk_r2r[2] -// ); -// std::cerr << this->mem->psi[e][0]( -// this->ijk[0].first()-1, -// this->ijk[1], -// this->ijk[2] -// ); - - // fill refined array at position where it overlaps with the resolved array -// this->mem->refinee(e)(this->ijk_r2r) = this->mem->advectee(e)(this->ijk); + this->mem->refinee(e)(this->ijk_r2r) = this->mem->advectee(e)(this->ijk); + // TODO: starting point of ranges of each MPI process should be at an odd number! for(int i=0; in_fra_iter; ++i) { // messy, because in domain decomposition (sharedmem and distmem) some refined scalars are on the edge of the subdomain... if(i==0) { mid_ijk_r2r_0 = this->rng_midpoints(this->ijk_r2r[0], this->mem->distmem.rank(), this->mem->distmem.size()); - mid_ijk_r2r_1 = this->rng_midpoints(this->ijk_r2r[1], this->rank, this->mem->size, false); + mid_ijk_r2r_1 = this->rng_midpoints(this->ijk_r2r[1], this->rank, this->mem->size); mid_ijk_r2r_2 = this->rng_midpoints(this->ijk_r2r[2]); - ijk_r2r_0_h = this->ijk_r2r[0]; ijk_r2r_1_h = this->ijk_r2r[1]; ijk_r2r_2_h = this->ijk_r2r[2]; } @@ -516,10 +382,8 @@ namespace libmpdataxx { if(i==1) { - mid_ijk_r2r_0 = this->rng_midpoints_out(mid_ijk_r2r_0, this->mem->distmem.rank(), this->mem->distmem.size()); - if(this->rank > 0) - mid_ijk_r2r_1 = rng_t(mid_ijk_r2r_1.first() - mid_ijk_r2r_1.stride(), mid_ijk_r2r_1.last(), mid_ijk_r2r_1.stride()); // shift back to an overlapping range along y - mid_ijk_r2r_1 = this->rng_midpoints_out(mid_ijk_r2r_1, this->rank, this->mem->size); + mid_ijk_r2r_0 = this->rng_midpoints_out(mid_ijk_r2r_0); + mid_ijk_r2r_1 = this->rng_midpoints_out(mid_ijk_r2r_1); mid_ijk_r2r_2 = this->rng_midpoints_out(mid_ijk_r2r_2); } else @@ -529,16 +393,17 @@ namespace libmpdataxx mid_ijk_r2r_2 = this->rng_midpoints_out(mid_ijk_r2r_2); } - ijk_r2r_0_h = this->rng_half_stride(ijk_r2r_0_h, this->mem->distmem.rank(), this->mem->distmem.size()); - ijk_r2r_1_h = this->rng_half_stride(ijk_r2r_1_h, this->rank, this->mem->size, false); + ijk_r2r_1_h = this->rng_half_stride(ijk_r2r_1_h, this->rank, this->mem->size); ijk_r2r_2_h = this->rng_half_stride(ijk_r2r_2_h); } - stride = ijk_r2r_0_h.stride(); - assert(ijk_r2r_0_h.stride() == ijk_r2r_1_h.stride() && ijk_r2r_1_h.stride() == ijk_r2r_2_h.stride()); + stride = ijk_r2r_1_h.stride(); + assert(ijk_r2r_1_h.stride() == ijk_r2r_2_h.stride()); assert(stride % 2 == 0); hstride = stride / 2; + const int halo_size = 2; + const rng_t ijk_r2r_0_h_with_halo(this->ijk_r2r[0].first(), this->ijk_r2r[0].last() + halo_size * this->mem->n_ref, hstride); rcnstrct<0>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_0), ijk_r2r_1_h, ijk_r2r_2_h, hstride); this->mem->barrier(); @@ -557,44 +422,6 @@ namespace libmpdataxx // // rcnstrct<2>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_2), mid_ijk_r2r_0, mid_ijk_r2r_1, hstride); -/* - if(this->rank == 0) - { - - intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, ijk_r2r_1_h, ijk_r2r_2_h, hstride); - intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, ijk_r2r_2_h, ijk_r2r_0_h, hstride); - intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, ijk_r2r_0_h, ijk_r2r_1_h, hstride); - - intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, mid_ijk_r2r_0, ijk_r2r_1_h, hstride); - //intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, mid_ijk_r2r_2, ijk_r2r_0_h, hstride); - intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, ijk_r2r_0_h, mid_ijk_r2r_1, hstride); - intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, ijk_r2r_2_h, mid_ijk_r2r_0, hstride); - - //intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, mid_ijk_r2r_2, mid_ijk_r2r_0, hstride); - intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, mid_ijk_r2r_0, mid_ijk_r2r_1, hstride); - - std::cerr << std::endl; - } - this->mem->barrier(); - if(this->rank == 1) - { - - intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, ijk_r2r_1_h, ijk_r2r_2_h, hstride); - intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, ijk_r2r_2_h, ijk_r2r_0_h, hstride); - intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, ijk_r2r_0_h, ijk_r2r_1_h, hstride); - - intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, mid_ijk_r2r_0, ijk_r2r_1_h, hstride); - //intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, mid_ijk_r2r_2, ijk_r2r_0_h, hstride); - intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, ijk_r2r_0_h, mid_ijk_r2r_1, hstride); - intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, ijk_r2r_2_h, mid_ijk_r2r_0, hstride); - - //intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, mid_ijk_r2r_2, mid_ijk_r2r_0, hstride); - intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, mid_ijk_r2r_0, mid_ijk_r2r_1, hstride); - - std::cerr << std::endl; - } - */ - this->mem->barrier(); } } diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp index 5b4927e3..98a48252 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp @@ -47,9 +47,6 @@ namespace libmpdataxx idx_t ijk_ref; // range of refinee handled by given solver const idxs_t ijk_r2r; // resolved to refined; refined scalars at the same position as resolved scalars -// static rng_t rng_sclr_ref(const rng_t &rng) { return rng^halo_ref; } - - // range modifying methods used in grid refinement static rng_t rng_midpoints(const rng_t &rng, const int rank = 0, const int size = 1) diff --git a/tests/sandbox/pbl/pbl.hpp b/tests/sandbox/pbl/pbl.hpp index 34236fe9..1ff156e4 100644 --- a/tests/sandbox/pbl/pbl.hpp +++ b/tests/sandbox/pbl/pbl.hpp @@ -90,12 +90,12 @@ class pbl : public libmpdataxx::output::hdf5_xdmfmem->barrier(); -// this->reconstruct_refinee(ix::tht); -// this->mem->barrier(); -// if (this->rank == 0) -// { -// this->record_aux_dsc_refined("tht reconstructed", this->mem->refinee(ix::tht)); -// } + this->reconstruct_refinee(ix::tht); + this->mem->barrier(); + if (this->rank == 0) + { + this->record_aux_dsc_refined("tht reconstructed", this->mem->refinee(ix::tht)); + } } } diff --git a/tests/sandbox/pbl/pbl_test_def.hpp b/tests/sandbox/pbl/pbl_test_def.hpp index 1f89b43d..580f2b97 100644 --- a/tests/sandbox/pbl/pbl_test_def.hpp +++ b/tests/sandbox/pbl/pbl_test_def.hpp @@ -22,7 +22,7 @@ template void set_sgs_specific(params_t &p, iles_tag) { p.iles_cdrag = 0.1; - p.n_fra_iter = 2; + p.n_fra_iter = 1; } // smagorinsky From 34337eb4db6670b715f9b4b2f4fef294e1fccca9 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 26 Apr 2022 13:22:37 +0200 Subject: [PATCH 041/129] working reconstruction? at least for 1 iteration --- .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 53 ++++++++----------- .../mpdata_rhs_vip_prs_sgs_fra_common.hpp | 10 ++-- 2 files changed, 29 insertions(+), 34 deletions(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 11f35c92..283f5066 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -267,18 +267,9 @@ namespace libmpdataxx } else { - if(i==1) - { - mid_ijk_r2r_0 = this->rng_midpoints_out(mid_ijk_r2r_0); - mid_ijk_r2r_1 = this->rng_midpoints_out(mid_ijk_r2r_1); - mid_ijk_r2r_2 = this->rng_midpoints_out(mid_ijk_r2r_2); - } - else - { - mid_ijk_r2r_0 = this->rng_midpoints_out(mid_ijk_r2r_0); - mid_ijk_r2r_1 = this->rng_midpoints_out(mid_ijk_r2r_1); - mid_ijk_r2r_2 = this->rng_midpoints_out(mid_ijk_r2r_2); - } + mid_ijk_r2r_0 = this->rng_midpoints_out(mid_ijk_r2r_0); + mid_ijk_r2r_1 = this->rng_midpoints_out(mid_ijk_r2r_1); + mid_ijk_r2r_2 = this->rng_midpoints_out(mid_ijk_r2r_2); ijk_r2r_1_h = this->rng_half_stride(ijk_r2r_1_h, this->rank, this->mem->size); ijk_r2r_2_h = this->rng_half_stride(ijk_r2r_2_h); @@ -289,8 +280,10 @@ namespace libmpdataxx assert(stride % 2 == 0); hstride = stride / 2; - const int halo_size = 1; // 2 for reconstruction - const rng_t ijk_r2r_0_h_with_halo(this->ijk_r2r[0].first(), this->ijk_r2r[0].last() + halo_size * this->mem->n_ref, hstride); + const int halo_size = 1; + const rng_t ijk_r2r_0_h_with_halo = this->mem->distmem.rank() < this->mem->distmem.size() - 1 ? + rng_t(this->ijk_r2r[0].first(), this->ijk_r2r[0].last() + halo_size * this->mem->n_ref, hstride) : + rng_t(this->ijk_r2r[0].first(), this->ijk_r2r[0].last(), hstride); intrp<0>(this->mem->psi_ref[e], mid_ijk_r2r_0, ijk_r2r_1_h, ijk_r2r_2_h, hstride); this->mem->barrier(); @@ -365,6 +358,8 @@ namespace libmpdataxx // fill refined array at position where it overlaps with the resolved array this->mem->refinee(e)(this->ijk_r2r) = this->mem->advectee(e)(this->ijk); + int offset; + // TODO: starting point of ranges of each MPI process should be at an odd number! for(int i=0; in_fra_iter; ++i) { @@ -380,18 +375,9 @@ namespace libmpdataxx } else { - if(i==1) - { - mid_ijk_r2r_0 = this->rng_midpoints_out(mid_ijk_r2r_0); - mid_ijk_r2r_1 = this->rng_midpoints_out(mid_ijk_r2r_1); - mid_ijk_r2r_2 = this->rng_midpoints_out(mid_ijk_r2r_2); - } - else - { - mid_ijk_r2r_0 = this->rng_midpoints_out(mid_ijk_r2r_0); - mid_ijk_r2r_1 = this->rng_midpoints_out(mid_ijk_r2r_1); - mid_ijk_r2r_2 = this->rng_midpoints_out(mid_ijk_r2r_2); - } + mid_ijk_r2r_0 = this->rng_midpoints_out(mid_ijk_r2r_0); + mid_ijk_r2r_1 = this->rng_midpoints_out(mid_ijk_r2r_1); + mid_ijk_r2r_2 = this->rng_midpoints_out(mid_ijk_r2r_2); ijk_r2r_1_h = this->rng_half_stride(ijk_r2r_1_h, this->rank, this->mem->size); ijk_r2r_2_h = this->rng_half_stride(ijk_r2r_2_h); @@ -402,13 +388,20 @@ namespace libmpdataxx assert(stride % 2 == 0); hstride = stride / 2; + if(i==0 && this->mem->grid_size_ref[0].first() > 0 && (this->mem->grid_size_ref[0].first() / stride) % 2 != 0) + offset = stride; // MPI domain starts with a point in the middle of a triple - we need to start calculating from a point to the left (in halo) + else + offset = 0; + const int halo_size = 2; - const rng_t ijk_r2r_0_h_with_halo(this->ijk_r2r[0].first(), this->ijk_r2r[0].last() + halo_size * this->mem->n_ref, hstride); + const rng_t ijk_r2r_0_h_with_halo = this->mem->distmem.rank() < this->mem->distmem.size() - 1 ? + rng_t(this->ijk_r2r[0].first(), this->ijk_r2r[0].last() + halo_size * this->mem->n_ref, hstride) : + rng_t(this->ijk_r2r[0].first(), this->ijk_r2r[0].last(), hstride); - rcnstrct<0>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_0), ijk_r2r_1_h, ijk_r2r_2_h, hstride); + rcnstrct<0>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_0, offset), ijk_r2r_1_h, ijk_r2r_2_h, hstride); this->mem->barrier(); - rcnstrct<1>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_1), ijk_r2r_2_h, this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), hstride); - rcnstrct<2>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_2), this->rng_merge(ijk_r2r_0_h, mid_ijk_r2r_0), this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); + rcnstrct<1>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_1) , ijk_r2r_2_h, ijk_r2r_0_h_with_halo, hstride); + rcnstrct<2>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_2) , ijk_r2r_0_h_with_halo, this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); // rcnstrct<0>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_0), ijk_r2r_1_h, ijk_r2r_2_h, hstride); diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp index 98a48252..d1ac967d 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp @@ -83,13 +83,15 @@ namespace libmpdataxx // returned range points to the first from the pair of reconstructed points // to avoid boundary conditions, in y and z directions it is assumed that the number of points is 3+i*2, i=0,1,2,... (this check is somewhere else) // however in the x direction there can be any number of points, because the domain is divided between mpi processes... - static rng_t rng_dbl_stride(const rng_t &rng) + static rng_t rng_dbl_stride(const rng_t &rng, const int offset = 0) { assert(rng.last() != rng.first()); // we need at least 2 midpoints - if( ((rng.last() - rng.first()) / rng.stride() + 1) % 2 == 0) // even number of midpoints; y and z directions (and sometimes x) - return rng_t(rng.first(), rng.last() - rng.stride(), 2*rng.stride()); + +// return rng_t(rng.first() - offset, rng.last(), 2*rng.stride()); + if( ((rng.last() - rng.first() - offset) / rng.stride() + 1) % 2 == 0) // even number of midpoints; y and z directions (and sometimes x) + return rng_t(rng.first() - offset, rng.last() - rng.stride(), 2*rng.stride()); else // odd number of midpoints - return rng_t(rng.first(), rng.last(), 2*rng.stride()); // rely on the halo along x direction + return rng_t(rng.first() - offset, rng.last(), 2*rng.stride()); // rely on the halo along x direction } static rng_t rng_merge(const rng_t &rng1, const rng_t &rng2) From 6c06aed10f4118323867b528a729f879a325ab80 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Tue, 26 Apr 2022 16:10:44 +0200 Subject: [PATCH 042/129] working recon, at least 1 iter --- .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 283f5066..2226c8e9 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -109,6 +109,7 @@ namespace libmpdataxx // solution following Scotti and Meneveau Physica D 1999 +/* // helper references // NOTE: these are actually the resolved values only in the first iteration, in the subsequent iterations some of these are reconstructed values @@ -146,7 +147,7 @@ namespace libmpdataxx // (u_im1 + 0.5 * (u_ip1 - u_im1)) + // u_0(2*xi-1) u_i + 0.5 * a_2 + b_2; // q_2(2*xi-1) - +*/ // Eq. (5) Scotti and Meneveau PRL 1997, a_1 and a_2 differ from Scotti Meneveau 1999 ! // NOTE: in the 1999 paper, a_1 and a_2 are multipled by 2*xi and 2*xi-1, respectively // in the 1997 paper, the are both multipled by xi @@ -161,7 +162,7 @@ namespace libmpdataxx // solution following Akinlabi et al. -/* + const int x[3] = {0, 1, 2}; @@ -176,17 +177,21 @@ namespace libmpdataxx f_2 = this->f_j(pis(i_next, j, k)); // Eqs. (5) and (6) Akinlabi et al. - c_1 = (u_1 - u_0) / (4 * dist) - d_1 * (u_2 - u_0) / (4 * dist); - c_2 = (u_2 - u_1) / (4 * dist) - d_2 * (u_2 - u_0) / (4 * dist); - f_1 = (x[2] * u_0 - x[0] * u_1) / (4 * dist) - d_1 * (x[2] * u_0 - x[0] * u_2) / (4 * dist); - f_2 = (x[2] * u_1 - x[0] * u_2) / (4 * dist) - d_2 * (x[2] * u_0 - x[0] * u_2) / (4 * dist); + c_1 = (u_1 - u_0) / (2.) - d_1 * (u_2 - u_0) / (2.); + c_2 = (u_2 - u_1) / (2.) - d_2 * (u_2 - u_0) / (2.); + f_1 = (x[2] * u_0 - x[0] * u_1) / (2.) - d_1 * (x[2] * u_0 - x[0] * u_2) / (2.); + f_2 = (x[2] * u_1 - x[0] * u_2) / (2.) - d_2 * (x[2] * u_0 - x[0] * u_2) / (2.); +// c_1 = (u_1 - u_0) / (4 * dist) - d_1 * (u_2 - u_0) / (4 * dist); +// c_2 = (u_2 - u_1) / (4 * dist) - d_2 * (u_2 - u_0) / (4 * dist); +// f_1 = (x[2] * u_0 - x[0] * u_1) / (4 * dist) - d_1 * (x[2] * u_0 - x[0] * u_2) / (4 * dist); +// f_2 = (x[2] * u_1 - x[0] * u_2) / (4 * dist) - d_2 * (x[2] * u_0 - x[0] * u_2) / (4 * dist); // new u, at j=1, between i=0 and i=1, result of w_j=1(u_i=1), Eq. (1) in Akinlabi et al. - arr(pis(i , j, k)) = c_1 * 2*dist + d_1 * u_1 + f_1; + arr(pis(i , j, k)) = c_1 * 1 + d_1 * u_1 + f_1; // new u, at j=2, between i=1 and i=2, result of w_j=2(u_i=1), Eq. (1) in Akinlabi et al. - arr(pis(i_next, j, k)) = c_2 * 2*dist + d_2 * u_1 + f_2; + arr(pis(i_next, j, k)) = c_2 * 1 + d_2 * u_1 + f_2; // not sure about * 2*dist here (is x_1 == 2*dist?) - */ + /* c_j(pis(i, j, k)) * x_j + From 42c26ac6f05d135c54265ed41c990a779c56eb92 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 27 Apr 2022 16:28:23 +0200 Subject: [PATCH 043/129] working recon --- tests/sandbox/pbl/pbl.hpp | 10 ++++++++++ tests/sandbox/pbl/pbl_iles_short.cpp | 2 +- tests/sandbox/pbl/pbl_test_def.hpp | 2 +- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/sandbox/pbl/pbl.hpp b/tests/sandbox/pbl/pbl.hpp index 1ff156e4..a856a14c 100644 --- a/tests/sandbox/pbl/pbl.hpp +++ b/tests/sandbox/pbl/pbl.hpp @@ -75,6 +75,8 @@ class pbl : public libmpdataxx::output::hdf5_xdmfinterpolate_refinee(ix::tht); this->interpolate_refinee(ix::u); + this->interpolate_refinee(ix::v); + this->interpolate_refinee(ix::w); this->mem->barrier(); if (this->rank == 0) @@ -86,15 +88,23 @@ class pbl : public libmpdataxx::output::hdf5_xdmfrecord_aux_dsc("p", this->Phi); this->record_aux_dsc_refined("tht refined", this->mem->refinee(ix::tht)); this->record_aux_dsc_refined("u refined", this->mem->refinee(ix::u)); + this->record_aux_dsc_refined("v refined", this->mem->refinee(ix::v)); + this->record_aux_dsc_refined("w refined", this->mem->refinee(ix::w)); } this->mem->barrier(); this->reconstruct_refinee(ix::tht); + this->reconstruct_refinee(ix::u); + this->reconstruct_refinee(ix::v); + this->reconstruct_refinee(ix::w); this->mem->barrier(); if (this->rank == 0) { this->record_aux_dsc_refined("tht reconstructed", this->mem->refinee(ix::tht)); + this->record_aux_dsc_refined("u reconstructed", this->mem->refinee(ix::u)); + this->record_aux_dsc_refined("v reconstructed", this->mem->refinee(ix::v)); + this->record_aux_dsc_refined("w reconstructed", this->mem->refinee(ix::w)); } } } diff --git a/tests/sandbox/pbl/pbl_iles_short.cpp b/tests/sandbox/pbl/pbl_iles_short.cpp index 82cce4e6..96bd847c 100644 --- a/tests/sandbox/pbl/pbl_iles_short.cpp +++ b/tests/sandbox/pbl/pbl_iles_short.cpp @@ -9,5 +9,5 @@ int main() { - test("out_pbl_iles_short", 33, 5); + test("out_pbl_iles_short", 33, 601); } diff --git a/tests/sandbox/pbl/pbl_test_def.hpp b/tests/sandbox/pbl/pbl_test_def.hpp index 580f2b97..32ebf6f5 100644 --- a/tests/sandbox/pbl/pbl_test_def.hpp +++ b/tests/sandbox/pbl/pbl_test_def.hpp @@ -82,7 +82,7 @@ void test(const std::string &dirname, const int np, const int nt) double mixed_length = 500; double st = 1e-4 / p.g; - p.outfreq = 1; + p.outfreq = 150; p.outwindow = 1; p.outvars = { {ix::u, { "u", "m/s"}}, From 72ead36618435ba07d1dbee2f1aa9bafdd0c5f9f Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 27 Apr 2022 17:53:30 +0200 Subject: [PATCH 044/129] reconstruction and interpolation: move similar code to functions --- .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 270 +++++++----------- 1 file changed, 102 insertions(+), 168 deletions(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 2226c8e9..acf15628 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -31,6 +31,11 @@ namespace libmpdataxx using parent_t::parent_t; // inheriting constructors protected: + + // helper variables for grid refinement + rng_t mid_ijk_r2r_0, mid_ijk_r2r_1, mid_ijk_r2r_2; // positions between already known values (to be filled during given iteration) + rng_t ijk_r2r_0_h_with_halo, ijk_r2r_1_h, ijk_r2r_2_h; // all positions at resolution of given iteration + int stride, hstride; // stride and half stride // interpolation similar to mpdata_rhs_vip... template @@ -216,212 +221,141 @@ namespace libmpdataxx */ } - void interpolate_refinee(const int e = 0) + // fill distmem halos of refinee + // TODO: move to bcond or sth? would be filled only by remote bcond + void fill_refinee_distmem_halos(const int e, const int halo_size) { - using namespace arakawa_c; // for rng_t operator^ + // TODO: we only need to xchng along distmem direction (x) + this->xchng(e); - rng_t mid_ijk_r2r_0, mid_ijk_r2r_1, mid_ijk_r2r_2; // positions between already known values (to be filled during given iteration) - rng_t ijk_r2r_1_h, ijk_r2r_2_h; // all positions at resolution of given iteration - int stride, hstride; + // TODO: assert halo_size <= solver halo size - // TEMPORARY - this->mem->psi_ref[e] = -1000; - this->mem->barrier(); + switch(halo_size) + { + case 2: + this->mem->psi_ref[e]( + this->mem->grid_size_ref[0].last() + 1 + this->mem->n_ref , + this->ijk_r2r[1], + this->ijk_r2r[2] + ) = + this->mem->psi[e][0]( + this->ijk[0].last()+2, + this->ijk[1], + this->ijk[2] + ); + // no break intentionally + case 1: + this->mem->psi_ref[e]( + this->mem->grid_size_ref[0].first() - this->mem->n_ref, + this->ijk_r2r[1], + this->ijk_r2r[2] + ) = + this->mem->psi[e][0]( + this->ijk[0].first()-1, + this->ijk[1], + this->ijk[2] + ); + + this->mem->psi_ref[e]( + this->mem->grid_size_ref[0].last() + 1, + this->ijk_r2r[1], + this->ijk_r2r[2] + ) = + this->mem->psi[e][0]( + this->ijk[0].last()+1, + this->ijk[1], + this->ijk[2] + ); + break; + default: + assert(false); + break; + } + } - // fill distmem halos of refinee - // TODO: move to bcond or sth? would be filled only by remote bcond - // TODO: we only need to xchng along distmem direction (x) - this->xchng(e); + void refinement_ranges(const int iter, const int halo_size) + { + // messy, because in domain decomposition (sharedmem and distmem) some refined scalars are on the edge of the subdomain... + if(iter==0) + { + mid_ijk_r2r_0 = this->rng_midpoints(this->ijk_r2r[0], this->mem->distmem.rank(), this->mem->distmem.size()); + mid_ijk_r2r_1 = this->rng_midpoints(this->ijk_r2r[1], this->rank, this->mem->size); + mid_ijk_r2r_2 = this->rng_midpoints(this->ijk_r2r[2]); - this->mem->psi_ref[e]( - this->mem->grid_size_ref[0].last() + 1, - this->ijk_r2r[1], - this->ijk_r2r[2] - ) = - this->mem->psi[e][0]( - this->ijk[0].last()+1, - this->ijk[1], - this->ijk[2] - ); + ijk_r2r_1_h = this->ijk_r2r[1]; + ijk_r2r_2_h = this->ijk_r2r[2]; + } + else + { + mid_ijk_r2r_0 = this->rng_midpoints_out(mid_ijk_r2r_0); + mid_ijk_r2r_1 = this->rng_midpoints_out(mid_ijk_r2r_1); + mid_ijk_r2r_2 = this->rng_midpoints_out(mid_ijk_r2r_2); - this->mem->psi_ref[e]( - this->mem->grid_size_ref[0].first() - this->mem->n_ref, - this->ijk_r2r[1], - this->ijk_r2r[2] - ) = - this->mem->psi[e][0]( - this->ijk[0].first()-1, - this->ijk[1], - this->ijk[2] - ); + ijk_r2r_1_h = this->rng_half_stride(ijk_r2r_1_h, this->rank, this->mem->size); + ijk_r2r_2_h = this->rng_half_stride(ijk_r2r_2_h); + } + + stride = ijk_r2r_1_h.stride(); + assert(ijk_r2r_1_h.stride() == ijk_r2r_2_h.stride()); + assert(stride % 2 == 0); + hstride = stride / 2; + + ijk_r2r_0_h_with_halo = this->mem->distmem.rank() < this->mem->distmem.size() - 1 ? + rng_t(this->ijk_r2r[0].first(), this->ijk_r2r[0].last() + halo_size * this->mem->n_ref, hstride) : + rng_t(this->ijk_r2r[0].first(), this->ijk_r2r[0].last(), hstride); + } + + void interpolate_refinee(const int e = 0) + { + const int halo_size = 1; + + // TEMPORARY + this->mem->psi_ref[e] = -1000; + this->mem->barrier(); + + fill_refinee_distmem_halos(e, halo_size); // fill refined array at position where it overlaps with the resolved array this->mem->refinee(e)(this->ijk_r2r) = this->mem->advectee(e)(this->ijk); for(int i=0; in_fra_iter; ++i) { - // messy, because in domain decomposition (sharedmem and distmem) some refined scalars are on the edge of the subdomain... - if(i==0) - { - mid_ijk_r2r_0 = this->rng_midpoints(this->ijk_r2r[0], this->mem->distmem.rank(), this->mem->distmem.size()); - mid_ijk_r2r_1 = this->rng_midpoints(this->ijk_r2r[1], this->rank, this->mem->size); - mid_ijk_r2r_2 = this->rng_midpoints(this->ijk_r2r[2]); - - ijk_r2r_1_h = this->ijk_r2r[1]; - ijk_r2r_2_h = this->ijk_r2r[2]; - } - else - { - mid_ijk_r2r_0 = this->rng_midpoints_out(mid_ijk_r2r_0); - mid_ijk_r2r_1 = this->rng_midpoints_out(mid_ijk_r2r_1); - mid_ijk_r2r_2 = this->rng_midpoints_out(mid_ijk_r2r_2); - - ijk_r2r_1_h = this->rng_half_stride(ijk_r2r_1_h, this->rank, this->mem->size); - ijk_r2r_2_h = this->rng_half_stride(ijk_r2r_2_h); - } - - stride = ijk_r2r_1_h.stride(); - assert(ijk_r2r_1_h.stride() == ijk_r2r_2_h.stride()); - assert(stride % 2 == 0); - hstride = stride / 2; - - const int halo_size = 1; - const rng_t ijk_r2r_0_h_with_halo = this->mem->distmem.rank() < this->mem->distmem.size() - 1 ? - rng_t(this->ijk_r2r[0].first(), this->ijk_r2r[0].last() + halo_size * this->mem->n_ref, hstride) : - rng_t(this->ijk_r2r[0].first(), this->ijk_r2r[0].last(), hstride); + refinement_ranges(i, halo_size); intrp<0>(this->mem->psi_ref[e], mid_ijk_r2r_0, ijk_r2r_1_h, ijk_r2r_2_h, hstride); this->mem->barrier(); intrp<1>(this->mem->psi_ref[e], mid_ijk_r2r_1, ijk_r2r_2_h, ijk_r2r_0_h_with_halo, hstride); intrp<2>(this->mem->psi_ref[e], mid_ijk_r2r_2, ijk_r2r_0_h_with_halo, this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); - -/* - - intrp<0>(this->mem->refinee(e), mid_ijk_r2r_0, ijk_r2r_1_h, ijk_r2r_2_h, hstride); - intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, ijk_r2r_2_h, ijk_r2r_0_h, hstride); - intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, ijk_r2r_0_h, ijk_r2r_1_h, hstride); - - intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, mid_ijk_r2r_0, ijk_r2r_1_h, hstride); - intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, ijk_r2r_0_h, mid_ijk_r2r_1, hstride); - this->mem->barrier(); // necessary before interpolation along sharedmem y direction - intrp<1>(this->mem->refinee(e), mid_ijk_r2r_1, ijk_r2r_2_h, mid_ijk_r2r_0, hstride); - - intrp<2>(this->mem->refinee(e), mid_ijk_r2r_2, mid_ijk_r2r_0, mid_ijk_r2r_1, hstride); - */ - this->mem->barrier(); } + this->mem->barrier(); } - // TODO: very similar to interpolate refinee void reconstruct_refinee(const int e = 0) { - using namespace arakawa_c; // for rng_t operator^ - - rng_t mid_ijk_r2r_0, mid_ijk_r2r_1, mid_ijk_r2r_2; // positions between already known values (to be filled during given iteration) - rng_t ijk_r2r_0_h, ijk_r2r_1_h, ijk_r2r_2_h; // all positions at resolution of given iteration - int stride, hstride; + const int halo_size = 2; // TEMPORARY this->mem->psi_ref[e] = -1000; - this->mem->barrier(); + this->mem->barrier(); - // fill distmem halos of refinee - // TODO: move to bcond or sth? would be filled only by remote bcond - // TODO: we only need to xchng along distmem direction (x) - this->xchng(e); - this->mem->psi_ref[e]( - this->mem->grid_size_ref[0].last() + 1, - this->ijk_r2r[1], - this->ijk_r2r[2] - ) = - this->mem->psi[e][0]( - this->ijk[0].last()+1, - this->ijk[1], - this->ijk[2] - ); - this->mem->psi_ref[e]( - this->mem->grid_size_ref[0].last() + 1 + this->mem->n_ref , - this->ijk_r2r[1], - this->ijk_r2r[2] - ) = - this->mem->psi[e][0]( - this->ijk[0].last()+2, // MPI halo size 2 needed! - this->ijk[1], - this->ijk[2] - ); - this->mem->psi_ref[e]( - this->mem->grid_size_ref[0].first() - this->mem->n_ref, - this->ijk_r2r[1], - this->ijk_r2r[2] - ) = - this->mem->psi[e][0]( - this->ijk[0].first()-1, - this->ijk[1], - this->ijk[2] - ); + fill_refinee_distmem_halos(e, halo_size); // fill refined array at position where it overlaps with the resolved array this->mem->refinee(e)(this->ijk_r2r) = this->mem->advectee(e)(this->ijk); - int offset; - - // TODO: starting point of ranges of each MPI process should be at an odd number! for(int i=0; in_fra_iter; ++i) { - // messy, because in domain decomposition (sharedmem and distmem) some refined scalars are on the edge of the subdomain... - if(i==0) - { - mid_ijk_r2r_0 = this->rng_midpoints(this->ijk_r2r[0], this->mem->distmem.rank(), this->mem->distmem.size()); - mid_ijk_r2r_1 = this->rng_midpoints(this->ijk_r2r[1], this->rank, this->mem->size); - mid_ijk_r2r_2 = this->rng_midpoints(this->ijk_r2r[2]); - - ijk_r2r_1_h = this->ijk_r2r[1]; - ijk_r2r_2_h = this->ijk_r2r[2]; - } - else - { - mid_ijk_r2r_0 = this->rng_midpoints_out(mid_ijk_r2r_0); - mid_ijk_r2r_1 = this->rng_midpoints_out(mid_ijk_r2r_1); - mid_ijk_r2r_2 = this->rng_midpoints_out(mid_ijk_r2r_2); - - ijk_r2r_1_h = this->rng_half_stride(ijk_r2r_1_h, this->rank, this->mem->size); - ijk_r2r_2_h = this->rng_half_stride(ijk_r2r_2_h); - } - - stride = ijk_r2r_1_h.stride(); - assert(ijk_r2r_1_h.stride() == ijk_r2r_2_h.stride()); - assert(stride % 2 == 0); - hstride = stride / 2; - - if(i==0 && this->mem->grid_size_ref[0].first() > 0 && (this->mem->grid_size_ref[0].first() / stride) % 2 != 0) - offset = stride; // MPI domain starts with a point in the middle of a triple - we need to start calculating from a point to the left (in halo) - else - offset = 0; - - const int halo_size = 2; - const rng_t ijk_r2r_0_h_with_halo = this->mem->distmem.rank() < this->mem->distmem.size() - 1 ? - rng_t(this->ijk_r2r[0].first(), this->ijk_r2r[0].last() + halo_size * this->mem->n_ref, hstride) : - rng_t(this->ijk_r2r[0].first(), this->ijk_r2r[0].last(), hstride); + refinement_ranges(i, halo_size); + + // if MPI domain starts with a point in the middle of a triple - we need to start calculating from a point to the left (in halo) + const int offset = (i==0 && this->mem->grid_size_ref[0].first() > 0 && (this->mem->grid_size_ref[0].first() / stride) % 2 != 0) ? stride : 0; rcnstrct<0>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_0, offset), ijk_r2r_1_h, ijk_r2r_2_h, hstride); this->mem->barrier(); rcnstrct<1>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_1) , ijk_r2r_2_h, ijk_r2r_0_h_with_halo, hstride); rcnstrct<2>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_2) , ijk_r2r_0_h_with_halo, this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); - - -// rcnstrct<0>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_0), ijk_r2r_1_h, ijk_r2r_2_h, hstride); -// rcnstrct<1>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_1), ijk_r2r_2_h, ijk_r2r_0_h, hstride); -// rcnstrct<2>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_2), ijk_r2r_0_h, ijk_r2r_1_h, hstride); -// -// rcnstrct<2>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_2), mid_ijk_r2r_0, ijk_r2r_1_h, hstride); -// rcnstrct<2>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_2), ijk_r2r_0_h, mid_ijk_r2r_1, hstride); -// this->mem->barrier(); // necessary before interpolation along sharedmem y direction -// rcnstrct<1>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_1), ijk_r2r_2_h, mid_ijk_r2r_0, hstride); -// -// rcnstrct<2>(this->mem->refinee(e), this->rng_dbl_stride(mid_ijk_r2r_2), mid_ijk_r2r_0, mid_ijk_r2r_1, hstride); - - this->mem->barrier(); } + this->mem->barrier(); } public: From 12766ed15106c97e1b890f63bcba161e7ce83d43 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Thu, 28 Apr 2022 18:04:01 +0200 Subject: [PATCH 045/129] random stretching parameter --- .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 53 ++++++++++++++++--- .../mpdata_rhs_vip_prs_sgs_fra_common.hpp | 2 +- .../solvers/mpdata_rhs_vip_prs_sgs_fra.hpp | 5 +- tests/sandbox/pbl/pbl_test_def.hpp | 2 +- 4 files changed, 52 insertions(+), 10 deletions(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index acf15628..035f77f0 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -29,6 +29,7 @@ namespace libmpdataxx { using parent_t = detail::mpdata_rhs_vip_prs_sgs_fra_common; using parent_t::parent_t; // inheriting constructors + using real_t = typename ct_params_t::real_t; protected: @@ -37,6 +38,29 @@ namespace libmpdataxx rng_t ijk_r2r_0_h_with_halo, ijk_r2r_1_h, ijk_r2r_2_h; // all positions at resolution of given iteration int stride, hstride; // stride and half stride + // CDF of stretching parameter d; assuming CDF = A d^B + C under condition that CDF(0.5)=0 and CDF(1)=1 + static real_t CDF_of_d(const real_t &d) + { + const real_t B = -0.39091161; // comes from a fit to the E. Akinlabi data + return (pow(d,B) - pow(0.5,B)) / (1. - pow(0.5,B)); + } + // inverse function: d of a CDF + static real_t d_of_CDF(const real_t &CDF) + { + const real_t B = -0.39091161; // comes from a fit to the E. Akinlabi data + return pow((1.-pow(0.5,B))*CDF + pow(0.5,B), 1./B); + } + + struct d_of_CDF_fctr + { + real_t operator()(const real_t &CDF) const + { + return CDF < 0 ? -d_of_CDF(-CDF) : d_of_CDF(CDF); + } + BZ_DECLARE_FUNCTOR(d_of_CDF_fctr); + }; + + // interpolation similar to mpdata_rhs_vip... template void intrp( @@ -48,8 +72,6 @@ namespace libmpdataxx ) { using idxperm::pis; -// using namespace arakawa_c; - using real_t = typename ct_params_t::real_t; // debug output std::cerr << "ranges in interpolation" << std::endl; @@ -78,9 +100,7 @@ namespace libmpdataxx ) { // TODO: move some to formulas:: - using idxperm::pis; - using real_t = typename ct_params_t::real_t; // second reconstructed position (j=2, between i=1 and i=2) const rng_t i_next = i + 2*dist; @@ -109,8 +129,8 @@ namespace libmpdataxx // stretching parameter, TODO: draw it from the distribution - const real_t d_1 = -pow(2, -1./3.), - d_2 = pow(2, -1./3.); +// const real_t d_1 = -pow(2, -1./3.), +// d_2 = pow(2, -1./3.); // solution following Scotti and Meneveau Physica D 1999 @@ -178,9 +198,14 @@ namespace libmpdataxx arr_t c_1 = this->c_j(pis(i, j, k)), c_2 = this->c_j(pis(i_next, j, k)), + d_1 = this->d_j(pis(i, j, k)), + d_2 = this->d_j(pis(i_next, j, k)), f_1 = this->f_j(pis(i, j, k)), f_2 = this->f_j(pis(i_next, j, k)); + + + // Eqs. (5) and (6) Akinlabi et al. c_1 = (u_1 - u_0) / (2.) - d_1 * (u_2 - u_0) / (2.); c_2 = (u_2 - u_1) / (2.) - d_2 * (u_2 - u_0) / (2.); @@ -305,6 +330,18 @@ namespace libmpdataxx rng_t(this->ijk_r2r[0].first(), this->ijk_r2r[0].last(), hstride); } + // TODO: move some formulas + // TODO: move to common? + void generate_stretching_parameters(const int rng_seed = 44) + { + std::mt19937 gen(rng_seed); + std::uniform_real_distribution<> dis(-1, 1); // [-1,1), but whatever + auto rand = std::bind(dis, gen); + std::generate(this->d_j(this->ijk_ref).begin(), this->d_j(this->ijk_ref).end(), rand); + this->d_j(this->ijk_ref) = d_of_CDF_fctr{}(this->d_j(this->ijk_ref)); + // TODO: randomly convert to negative + } + void interpolate_refinee(const int e = 0) { const int halo_size = 1; @@ -318,6 +355,8 @@ namespace libmpdataxx // fill refined array at position where it overlaps with the resolved array this->mem->refinee(e)(this->ijk_r2r) = this->mem->advectee(e)(this->ijk); + generate_stretching_parameters(); + for(int i=0; in_fra_iter; ++i) { refinement_ranges(i, halo_size); @@ -343,6 +382,8 @@ namespace libmpdataxx // fill refined array at position where it overlaps with the resolved array this->mem->refinee(e)(this->ijk_r2r) = this->mem->advectee(e)(this->ijk); + generate_stretching_parameters(); + for(int i=0; in_fra_iter; ++i) { refinement_ranges(i, halo_size); diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp index d1ac967d..47c6b4f0 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp @@ -32,7 +32,7 @@ namespace libmpdataxx protected: - typename parent_t::arr_t c_j, f_j; // parameters used in fractal reconstruction, Akinlabi et al. 2019 + typename parent_t::arr_t c_j, d_j, f_j; // parameters used in fractal reconstruction, Akinlabi et al. 2019 // const int n_fra; // number of fields with fractal reconstruction diff --git a/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp b/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp index 5b503e96..f70842d5 100644 --- a/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp +++ b/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp @@ -50,7 +50,8 @@ namespace libmpdataxx parent_t(args, p) { this->c_j.reference(args.mem->tmp[__FILE__][1][0]); - this->f_j.reference(args.mem->tmp[__FILE__][1][1]); + this->d_j.reference(args.mem->tmp[__FILE__][1][1]); + this->f_j.reference(args.mem->tmp[__FILE__][1][2]); } // why alloc here? @@ -61,7 +62,7 @@ namespace libmpdataxx parent_t::alloc(mem, n_iters); // TODO: allocate only for fields that we want to reconstruct, defined in ct_params_t::fractal_recon, will this mess with field numbering? e.g. ix::tht_ref.... parent_t::alloc_tmp_sclr_ref(mem, __FILE__, ct_params_t::n_eqns); // psi_ref - parent_t::alloc_tmp_sclr_ref(mem, __FILE__, 2); // c_j, f_j + parent_t::alloc_tmp_sclr_ref(mem, __FILE__, 3); // c_j, d_j, f_j // parent_t::alloc_tmp_vctr(mem, __FILE__, mem->grid_size_ref); // GC_ref mem->psi_ref = mem->tmp[__FILE__][0]; diff --git a/tests/sandbox/pbl/pbl_test_def.hpp b/tests/sandbox/pbl/pbl_test_def.hpp index 32ebf6f5..6fce5efb 100644 --- a/tests/sandbox/pbl/pbl_test_def.hpp +++ b/tests/sandbox/pbl/pbl_test_def.hpp @@ -22,7 +22,7 @@ template void set_sgs_specific(params_t &p, iles_tag) { p.iles_cdrag = 0.1; - p.n_fra_iter = 1; + p.n_fra_iter = 2; } // smagorinsky From 549a59265b49d161840a7c1320ff90bfe056785d Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 29 Apr 2022 17:05:50 +0200 Subject: [PATCH 046/129] move fractal reconstruction formulas to formulae/ --- .../formulae/fractal_reconstruction.hpp | 191 +++++++++++++++ .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 226 +----------------- 2 files changed, 199 insertions(+), 218 deletions(-) create mode 100644 libmpdata++/formulae/fractal_reconstruction.hpp diff --git a/libmpdata++/formulae/fractal_reconstruction.hpp b/libmpdata++/formulae/fractal_reconstruction.hpp new file mode 100644 index 00000000..b722b520 --- /dev/null +++ b/libmpdata++/formulae/fractal_reconstruction.hpp @@ -0,0 +1,191 @@ +/** @file +* @copyright University of Warsaw +* @section LICENSE +* GPLv3+ (see the COPYING file or http://www.gnu.org/licenses/) +*/ + +#pragma once + +#include + +namespace libmpdataxx +{ + namespace formulae + { + namespace fractal + { + // CDF of stretching parameter d; assuming CDF = A d^B + C under condition that CDF(0.5)=0 and CDF(1)=1 + template + static real_t CDF_of_d(const real_t &d) + { + const real_t B = -0.39091161; // comes from a fit to the E. Akinlabi data + return (pow(d,B) - pow(0.5,B)) / (1. - pow(0.5,B)); + } + // inverse function: d of a CDF + template + static real_t d_of_CDF(const real_t &CDF) + { + const real_t B = -0.39091161; // comes from a fit to the E. Akinlabi data + return pow((1.-pow(0.5,B))*CDF + pow(0.5,B), 1./B); + } + + template + struct d_of_CDF_fctr + { + real_t operator()(const real_t &CDF) const + { + return CDF < 0 ? -d_of_CDF(-CDF) : d_of_CDF(CDF); + } + BZ_DECLARE_FUNCTOR(d_of_CDF_fctr); + }; + + // (tri-)linear interpolation + // 3d version + template + void intrp( + arr_t arr, // array to be interpolated + const rng_t &i, // index of the point to be interpolated to + const rng_t &j, // ditto + const rng_t &k, // ditto + const int &dist // number of indices between an interpolated point and a known point + ) + { + using idxperm::pis; + + // debug output +// std::cerr << "ranges in interpolation" << std::endl; +// if(d==0) +// std::cerr << "range<" << d << ">: " << i << " " << j << " " << k << std::endl; +// if(d==1) +// std::cerr << "range<" << d << ">: " << k << " " << i << " " << j << std::endl; +// if(d==2) +// std::cerr << "range<" << d << ">: " << j << " " << k << " " << i << std::endl; + + arr(pis(i, j, k)) = real_t(.5) * ( + arr(pis(i - dist, j, k)) + + arr(pis(i + dist, j, k)) + ); + } + + // fractral reconstruction (calculates 2 mid-points based on 3) + // following Akinlabi et al. "Fractal reconstruciton..." 2019 + // 3d version + template + void rcnstrct( + arr_t arr, // array to be reconstructed + const rng_t &i, // index of the first point from the pair to be reconstructed + const rng_t &j, // ditto + const rng_t &k, // ditto + const int &dist, // number of indices between a reconstructed point and a known point + arr_t c_j, // parameter c + const arr_t d_j, // parameter d (stretching) - needs to be calculated before + arr_t f_j // parameter f + ) + { + using idxperm::pis; + + // second reconstructed position (j=2, between i=1 and i=2) + const rng_t i_next = i + 2*dist; + + const int x[3] = {0, 1, 2}; + + // helper references + const arr_t u_0 = arr(pis(i - dist, j, k)), + u_1 = arr(pis(i + dist, j, k)), + u_2 = arr(pis(i_next + dist, j, k)); + + arr_t c_1 = c_j(pis(i, j, k)), + c_2 = c_j(pis(i_next, j, k)), + d_1 = d_j(pis(i, j, k)), + d_2 = d_j(pis(i_next, j, k)), + f_1 = f_j(pis(i, j, k)), + f_2 = f_j(pis(i_next, j, k)); + + // Eqs. (5) and (6) Akinlabi et al. + c_1 = (u_1 - u_0) / (2.) - d_1 * (u_2 - u_0) / (2.); + c_2 = (u_2 - u_1) / (2.) - d_2 * (u_2 - u_0) / (2.); + f_1 = (x[2] * u_0 - x[0] * u_1) / (2.) - d_1 * (x[2] * u_0 - x[0] * u_2) / (2.); + f_2 = (x[2] * u_1 - x[0] * u_2) / (2.) - d_2 * (x[2] * u_0 - x[0] * u_2) / (2.); + + // new u, at j=1, between i=0 and i=1, result of w_j=1(u_i=1), Eq. (1) in Akinlabi et al. + arr(pis(i , j, k)) = c_1 * 1 + d_1 * u_1 + f_1; + // new u, at j=2, between i=1 and i=2, result of w_j=2(u_i=1), Eq. (1) in Akinlabi et al. + arr(pis(i_next, j, k)) = c_2 * 1 + d_2 * u_1 + f_2; + + // debug output +// std::cerr << "ranges in rcnstrct" << std::endl; +// if(d==0) +// std::cerr << "range<" << d << ">: " << i << " " << j << " " << k << std::endl; +// if(d==1) +// std::cerr << "range<" << d << ">: " << k << " " << i << " " << j << std::endl; +// if(d==2) +// std::cerr << "range<" << d << ">: " << j << " " << k << " " << i << std::endl; + + // do interpolation, useful for testing if ranges are correct + /* + arr(pis(i, j, k)) = real_t(.5) * ( + arr(pis(i - dist, j, k)) + + arr(pis(i + dist, j, k)) + ); + arr(pis(i_next, j, k)) = real_t(.5) * ( + arr(pis(i_next - dist, j, k)) + + arr(pis(i_next + dist, j, k)) + ); + return; + */ + // end of the interpolation test + + +// alternative solution following Scotti and Meneveau Physica D 1999 +/* + + // helper references + // NOTE: these are actually the resolved values only in the first iteration, in the subsequent iterations some of these are reconstructed values + // should we point to the resolved values in all iterations?? + const arr_t u_im1 = arr(pis(i - dist, j, k)), // resolved u_(i-1) + u_i = arr(pis(i + dist, j, k)), // resolved u_(i) + u_ip1 = arr(pis(i_next + dist, j, k)); // resolved u_(i+1) + + // c_j and f_j naming comes from the Akinlabi paper, TODO: rename to a_i b_i + // NOTE: in multiple iterations of reconstruction, they only change due to random stretching parameters? + // or the stretching parameter should be drawn from the distro once? then no need for c_j and f_j to have the same size as reconstructed fields... + arr_t a_1 = this->c_j(pis(i, j, k)), + a_2 = this->c_j(pis(i_next, j, k)), + b_1 = this->f_j(pis(i, j, k)), + b_2 = this->f_j(pis(i_next, j, k)); + + // Eqs. (13) and (14) from Scotti Meneveau 1999 + a_1 = u_i - u_im1 - d_1*(u_ip1 - u_im1); + a_2 = u_ip1 - u_i - d_2*(u_ip1 - u_im1); + b_1 = u_im1*(real_t(1) - d_1); + b_2 = u_i - d_2*u_im1; + + // W(u_0(xi)) at xi=1/4, between u_i-1 and u_i + // u_0(2*xi) = u_i-1 + 1/2 * (u_i+1 - u_i-1) + // Eq. (6) and caption of fig. 1 Scotti 1999 + // NOTE: in multiple iterations, should xi be between the 3 resolved points? or should we consider the reconstructed points as new resolved points? + arr(pis(i, j, k)) = d_1 * +// (u_im1 + 0.5 * (u_ip1 - u_im1)) + // u_0(2*xi) + u_i + + 0.5 * a_1 + b_1; // q_1(2*xi) + // W(u_0(xi)) at xi=3/4, between u_i and u_i+1 + // u_0(2*xi-1) = u_i-1 + 1/2 * (u_i+1 - u_i-1) + // NOTE: u_0(2*3/4-1) == u_0(2*1/4), so calculate it once in the previous step and store it... + arr(pis(i_next, j, k)) = d_2 * +// (u_im1 + 0.5 * (u_ip1 - u_im1)) + // u_0(2*xi-1) + u_i + + 0.5 * a_2 + b_2; // q_2(2*xi-1) +*/ + // Eq. (5) Scotti and Meneveau PRL 1997, a_1 and a_2 differ from Scotti Meneveau 1999 ! + // NOTE: in the 1999 paper, a_1 and a_2 are multipled by 2*xi and 2*xi-1, respectively + // in the 1997 paper, the are both multipled by xi + /* + a_1 = 2 * (u_im1 - u_i - d_1*(u_ip1 - u_im1)); + a_2 = 2 * (u_ip1 - u_i - d_2*(u_ip1 - u_im1)); + b_1 = u_im1*(real_t(1) - d_1); + b_2 = u_i - d_2*u_im1; + */ + } + }; + }; +}; diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 035f77f0..ba612424 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -7,6 +7,7 @@ #pragma once #include +#include namespace libmpdataxx { @@ -38,214 +39,6 @@ namespace libmpdataxx rng_t ijk_r2r_0_h_with_halo, ijk_r2r_1_h, ijk_r2r_2_h; // all positions at resolution of given iteration int stride, hstride; // stride and half stride - // CDF of stretching parameter d; assuming CDF = A d^B + C under condition that CDF(0.5)=0 and CDF(1)=1 - static real_t CDF_of_d(const real_t &d) - { - const real_t B = -0.39091161; // comes from a fit to the E. Akinlabi data - return (pow(d,B) - pow(0.5,B)) / (1. - pow(0.5,B)); - } - // inverse function: d of a CDF - static real_t d_of_CDF(const real_t &CDF) - { - const real_t B = -0.39091161; // comes from a fit to the E. Akinlabi data - return pow((1.-pow(0.5,B))*CDF + pow(0.5,B), 1./B); - } - - struct d_of_CDF_fctr - { - real_t operator()(const real_t &CDF) const - { - return CDF < 0 ? -d_of_CDF(-CDF) : d_of_CDF(CDF); - } - BZ_DECLARE_FUNCTOR(d_of_CDF_fctr); - }; - - - // interpolation similar to mpdata_rhs_vip... - template - void intrp( - arr_t arr, - const rng_t &i, - const rng_t &j, - const rng_t &k, - const int &dist - ) - { - using idxperm::pis; - - // debug output - std::cerr << "ranges in interpolation" << std::endl; - if(d==0) - std::cerr << "range<" << d << ">: " << i << " " << j << " " << k << std::endl; - if(d==1) - std::cerr << "range<" << d << ">: " << k << " " << i << " " << j << std::endl; - if(d==2) - std::cerr << "range<" << d << ">: " << j << " " << k << " " << i << std::endl; - - arr(pis(i, j, k)) = real_t(.5) * ( - arr(pis(i - dist, j, k)) + - arr(pis(i + dist, j, k)) - ); - } - - // fractral reconstruction (calculates 2 mid-points based on 3) - // notation following Akinlabi et al. "Fractal reconstruciton..." 2019 - template - void rcnstrct( - arr_t arr, - const rng_t &i, - const rng_t &j, - const rng_t &k, - const int &dist // number of indices between a reconstructed point and a known point - ) - { - // TODO: move some to formulas:: - using idxperm::pis; - - // second reconstructed position (j=2, between i=1 and i=2) - const rng_t i_next = i + 2*dist; - - // debug output - std::cerr << "ranges in rcnstrct" << std::endl; - if(d==0) - std::cerr << "range<" << d << ">: " << i << " " << j << " " << k << std::endl; - if(d==1) - std::cerr << "range<" << d << ">: " << k << " " << i << " " << j << std::endl; - if(d==2) - std::cerr << "range<" << d << ">: " << j << " " << k << " " << i << std::endl; - // do interpolation, useful for testing if ranges are correct - /* - arr(pis(i, j, k)) = real_t(.5) * ( - arr(pis(i - dist, j, k)) + - arr(pis(i + dist, j, k)) - ); - arr(pis(i_next, j, k)) = real_t(.5) * ( - arr(pis(i_next - dist, j, k)) + - arr(pis(i_next + dist, j, k)) - ); - return; - */ - // end of the interpolation test - - - // stretching parameter, TODO: draw it from the distribution -// const real_t d_1 = -pow(2, -1./3.), -// d_2 = pow(2, -1./3.); - - -// solution following Scotti and Meneveau Physica D 1999 -/* - - // helper references - // NOTE: these are actually the resolved values only in the first iteration, in the subsequent iterations some of these are reconstructed values - // should we point to the resolved values in all iterations?? - const arr_t u_im1 = arr(pis(i - dist, j, k)), // resolved u_(i-1) - u_i = arr(pis(i + dist, j, k)), // resolved u_(i) - u_ip1 = arr(pis(i_next + dist, j, k)); // resolved u_(i+1) - - // c_j and f_j naming comes from the Akinlabi paper, TODO: rename to a_i b_i - // NOTE: in multiple iterations of reconstruction, they only change due to random stretching parameters? - // or the stretching parameter should be drawn from the distro once? then no need for c_j and f_j to have the same size as reconstructed fields... - arr_t a_1 = this->c_j(pis(i, j, k)), - a_2 = this->c_j(pis(i_next, j, k)), - b_1 = this->f_j(pis(i, j, k)), - b_2 = this->f_j(pis(i_next, j, k)); - - // Eqs. (13) and (14) from Scotti Meneveau 1999 - a_1 = u_i - u_im1 - d_1*(u_ip1 - u_im1); - a_2 = u_ip1 - u_i - d_2*(u_ip1 - u_im1); - b_1 = u_im1*(real_t(1) - d_1); - b_2 = u_i - d_2*u_im1; - - // W(u_0(xi)) at xi=1/4, between u_i-1 and u_i - // u_0(2*xi) = u_i-1 + 1/2 * (u_i+1 - u_i-1) - // Eq. (6) and caption of fig. 1 Scotti 1999 - // NOTE: in multiple iterations, should xi be between the 3 resolved points? or should we consider the reconstructed points as new resolved points? - arr(pis(i, j, k)) = d_1 * -// (u_im1 + 0.5 * (u_ip1 - u_im1)) + // u_0(2*xi) - u_i + - 0.5 * a_1 + b_1; // q_1(2*xi) - // W(u_0(xi)) at xi=3/4, between u_i and u_i+1 - // u_0(2*xi-1) = u_i-1 + 1/2 * (u_i+1 - u_i-1) - // NOTE: u_0(2*3/4-1) == u_0(2*1/4), so calculate it once in the previous step and store it... - arr(pis(i_next, j, k)) = d_2 * -// (u_im1 + 0.5 * (u_ip1 - u_im1)) + // u_0(2*xi-1) - u_i + - 0.5 * a_2 + b_2; // q_2(2*xi-1) -*/ - // Eq. (5) Scotti and Meneveau PRL 1997, a_1 and a_2 differ from Scotti Meneveau 1999 ! - // NOTE: in the 1999 paper, a_1 and a_2 are multipled by 2*xi and 2*xi-1, respectively - // in the 1997 paper, the are both multipled by xi - /* - a_1 = 2 * (u_im1 - u_i - d_1*(u_ip1 - u_im1)); - a_2 = 2 * (u_ip1 - u_i - d_2*(u_ip1 - u_im1)); - b_1 = u_im1*(real_t(1) - d_1); - b_2 = u_i - d_2*u_im1; - */ - - //const real_t xi[3] = {0, 0.5, 1}; - - -// solution following Akinlabi et al. - - - const int x[3] = {0, 1, 2}; - - // helper references - const arr_t u_0 = arr(pis(i - dist, j, k)), - u_1 = arr(pis(i + dist, j, k)), - u_2 = arr(pis(i_next + dist, j, k)); - - arr_t c_1 = this->c_j(pis(i, j, k)), - c_2 = this->c_j(pis(i_next, j, k)), - d_1 = this->d_j(pis(i, j, k)), - d_2 = this->d_j(pis(i_next, j, k)), - f_1 = this->f_j(pis(i, j, k)), - f_2 = this->f_j(pis(i_next, j, k)); - - - - - // Eqs. (5) and (6) Akinlabi et al. - c_1 = (u_1 - u_0) / (2.) - d_1 * (u_2 - u_0) / (2.); - c_2 = (u_2 - u_1) / (2.) - d_2 * (u_2 - u_0) / (2.); - f_1 = (x[2] * u_0 - x[0] * u_1) / (2.) - d_1 * (x[2] * u_0 - x[0] * u_2) / (2.); - f_2 = (x[2] * u_1 - x[0] * u_2) / (2.) - d_2 * (x[2] * u_0 - x[0] * u_2) / (2.); -// c_1 = (u_1 - u_0) / (4 * dist) - d_1 * (u_2 - u_0) / (4 * dist); -// c_2 = (u_2 - u_1) / (4 * dist) - d_2 * (u_2 - u_0) / (4 * dist); -// f_1 = (x[2] * u_0 - x[0] * u_1) / (4 * dist) - d_1 * (x[2] * u_0 - x[0] * u_2) / (4 * dist); -// f_2 = (x[2] * u_1 - x[0] * u_2) / (4 * dist) - d_2 * (x[2] * u_0 - x[0] * u_2) / (4 * dist); - - // new u, at j=1, between i=0 and i=1, result of w_j=1(u_i=1), Eq. (1) in Akinlabi et al. - arr(pis(i , j, k)) = c_1 * 1 + d_1 * u_1 + f_1; - // new u, at j=2, between i=1 and i=2, result of w_j=2(u_i=1), Eq. (1) in Akinlabi et al. - arr(pis(i_next, j, k)) = c_2 * 1 + d_2 * u_1 + f_2; - // not sure about * 2*dist here (is x_1 == 2*dist?) - - -/* - c_j(pis(i, j, k)) * x_j + - d_j[0] * arr(pis(i + dist, j, k)) + // coarse-grained u at i=1 - f_j(pis(i, j, k)); - */ - - - // new u, at j=2, between i=1 and i=2, result of w2(u1) - // TODO - -/* - arr(pis(i, j, k)) = real_t(.5) * ( - arr(pis(i - dist, j, k)) + - arr(pis(i + dist, j, k)) - ); - - arr(pis(i_next, j, k)) = real_t(.5) * ( - arr(pis(i_next - dist, j, k)) + - arr(pis(i_next + dist, j, k)) - ); - */ - } - // fill distmem halos of refinee // TODO: move to bcond or sth? would be filled only by remote bcond void fill_refinee_distmem_halos(const int e, const int halo_size) @@ -330,16 +123,13 @@ namespace libmpdataxx rng_t(this->ijk_r2r[0].first(), this->ijk_r2r[0].last(), hstride); } - // TODO: move some formulas - // TODO: move to common? void generate_stretching_parameters(const int rng_seed = 44) { std::mt19937 gen(rng_seed); std::uniform_real_distribution<> dis(-1, 1); // [-1,1), but whatever auto rand = std::bind(dis, gen); std::generate(this->d_j(this->ijk_ref).begin(), this->d_j(this->ijk_ref).end(), rand); - this->d_j(this->ijk_ref) = d_of_CDF_fctr{}(this->d_j(this->ijk_ref)); - // TODO: randomly convert to negative + this->d_j(this->ijk_ref) = formulae::fractal::d_of_CDF_fctr{}(this->d_j(this->ijk_ref)); } void interpolate_refinee(const int e = 0) @@ -361,10 +151,10 @@ namespace libmpdataxx { refinement_ranges(i, halo_size); - intrp<0>(this->mem->psi_ref[e], mid_ijk_r2r_0, ijk_r2r_1_h, ijk_r2r_2_h, hstride); + formulae::fractal::intrp<0, real_t>(this->mem->psi_ref[e], mid_ijk_r2r_0, ijk_r2r_1_h, ijk_r2r_2_h, hstride); this->mem->barrier(); - intrp<1>(this->mem->psi_ref[e], mid_ijk_r2r_1, ijk_r2r_2_h, ijk_r2r_0_h_with_halo, hstride); - intrp<2>(this->mem->psi_ref[e], mid_ijk_r2r_2, ijk_r2r_0_h_with_halo, this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); + formulae::fractal::intrp<1, real_t>(this->mem->psi_ref[e], mid_ijk_r2r_1, ijk_r2r_2_h, ijk_r2r_0_h_with_halo, hstride); + formulae::fractal::intrp<2, real_t>(this->mem->psi_ref[e], mid_ijk_r2r_2, ijk_r2r_0_h_with_halo, this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); } this->mem->barrier(); } @@ -391,10 +181,10 @@ namespace libmpdataxx // if MPI domain starts with a point in the middle of a triple - we need to start calculating from a point to the left (in halo) const int offset = (i==0 && this->mem->grid_size_ref[0].first() > 0 && (this->mem->grid_size_ref[0].first() / stride) % 2 != 0) ? stride : 0; - rcnstrct<0>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_0, offset), ijk_r2r_1_h, ijk_r2r_2_h, hstride); + formulae::fractal::rcnstrct<0, real_t>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_0, offset), ijk_r2r_1_h, ijk_r2r_2_h, hstride, this->c_j, this->d_j, this->f_j); this->mem->barrier(); - rcnstrct<1>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_1) , ijk_r2r_2_h, ijk_r2r_0_h_with_halo, hstride); - rcnstrct<2>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_2) , ijk_r2r_0_h_with_halo, this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); + formulae::fractal::rcnstrct<1, real_t>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_1) , ijk_r2r_2_h, ijk_r2r_0_h_with_halo, hstride, this->c_j, this->d_j, this->f_j); + formulae::fractal::rcnstrct<2, real_t>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_2) , ijk_r2r_0_h_with_halo, this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride, this->c_j, this->d_j, this->f_j); } this->mem->barrier(); } From 0fd42ec4805fb49ec7f6d1ae16f3be721a592762 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 29 Apr 2022 17:05:55 +0200 Subject: [PATCH 047/129] Revert "move fractal reconstruction formulas to formulae/" This reverts commit 549a59265b49d161840a7c1320ff90bfe056785d. --- .../formulae/fractal_reconstruction.hpp | 191 --------------- .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 226 +++++++++++++++++- 2 files changed, 218 insertions(+), 199 deletions(-) delete mode 100644 libmpdata++/formulae/fractal_reconstruction.hpp diff --git a/libmpdata++/formulae/fractal_reconstruction.hpp b/libmpdata++/formulae/fractal_reconstruction.hpp deleted file mode 100644 index b722b520..00000000 --- a/libmpdata++/formulae/fractal_reconstruction.hpp +++ /dev/null @@ -1,191 +0,0 @@ -/** @file -* @copyright University of Warsaw -* @section LICENSE -* GPLv3+ (see the COPYING file or http://www.gnu.org/licenses/) -*/ - -#pragma once - -#include - -namespace libmpdataxx -{ - namespace formulae - { - namespace fractal - { - // CDF of stretching parameter d; assuming CDF = A d^B + C under condition that CDF(0.5)=0 and CDF(1)=1 - template - static real_t CDF_of_d(const real_t &d) - { - const real_t B = -0.39091161; // comes from a fit to the E. Akinlabi data - return (pow(d,B) - pow(0.5,B)) / (1. - pow(0.5,B)); - } - // inverse function: d of a CDF - template - static real_t d_of_CDF(const real_t &CDF) - { - const real_t B = -0.39091161; // comes from a fit to the E. Akinlabi data - return pow((1.-pow(0.5,B))*CDF + pow(0.5,B), 1./B); - } - - template - struct d_of_CDF_fctr - { - real_t operator()(const real_t &CDF) const - { - return CDF < 0 ? -d_of_CDF(-CDF) : d_of_CDF(CDF); - } - BZ_DECLARE_FUNCTOR(d_of_CDF_fctr); - }; - - // (tri-)linear interpolation - // 3d version - template - void intrp( - arr_t arr, // array to be interpolated - const rng_t &i, // index of the point to be interpolated to - const rng_t &j, // ditto - const rng_t &k, // ditto - const int &dist // number of indices between an interpolated point and a known point - ) - { - using idxperm::pis; - - // debug output -// std::cerr << "ranges in interpolation" << std::endl; -// if(d==0) -// std::cerr << "range<" << d << ">: " << i << " " << j << " " << k << std::endl; -// if(d==1) -// std::cerr << "range<" << d << ">: " << k << " " << i << " " << j << std::endl; -// if(d==2) -// std::cerr << "range<" << d << ">: " << j << " " << k << " " << i << std::endl; - - arr(pis(i, j, k)) = real_t(.5) * ( - arr(pis(i - dist, j, k)) + - arr(pis(i + dist, j, k)) - ); - } - - // fractral reconstruction (calculates 2 mid-points based on 3) - // following Akinlabi et al. "Fractal reconstruciton..." 2019 - // 3d version - template - void rcnstrct( - arr_t arr, // array to be reconstructed - const rng_t &i, // index of the first point from the pair to be reconstructed - const rng_t &j, // ditto - const rng_t &k, // ditto - const int &dist, // number of indices between a reconstructed point and a known point - arr_t c_j, // parameter c - const arr_t d_j, // parameter d (stretching) - needs to be calculated before - arr_t f_j // parameter f - ) - { - using idxperm::pis; - - // second reconstructed position (j=2, between i=1 and i=2) - const rng_t i_next = i + 2*dist; - - const int x[3] = {0, 1, 2}; - - // helper references - const arr_t u_0 = arr(pis(i - dist, j, k)), - u_1 = arr(pis(i + dist, j, k)), - u_2 = arr(pis(i_next + dist, j, k)); - - arr_t c_1 = c_j(pis(i, j, k)), - c_2 = c_j(pis(i_next, j, k)), - d_1 = d_j(pis(i, j, k)), - d_2 = d_j(pis(i_next, j, k)), - f_1 = f_j(pis(i, j, k)), - f_2 = f_j(pis(i_next, j, k)); - - // Eqs. (5) and (6) Akinlabi et al. - c_1 = (u_1 - u_0) / (2.) - d_1 * (u_2 - u_0) / (2.); - c_2 = (u_2 - u_1) / (2.) - d_2 * (u_2 - u_0) / (2.); - f_1 = (x[2] * u_0 - x[0] * u_1) / (2.) - d_1 * (x[2] * u_0 - x[0] * u_2) / (2.); - f_2 = (x[2] * u_1 - x[0] * u_2) / (2.) - d_2 * (x[2] * u_0 - x[0] * u_2) / (2.); - - // new u, at j=1, between i=0 and i=1, result of w_j=1(u_i=1), Eq. (1) in Akinlabi et al. - arr(pis(i , j, k)) = c_1 * 1 + d_1 * u_1 + f_1; - // new u, at j=2, between i=1 and i=2, result of w_j=2(u_i=1), Eq. (1) in Akinlabi et al. - arr(pis(i_next, j, k)) = c_2 * 1 + d_2 * u_1 + f_2; - - // debug output -// std::cerr << "ranges in rcnstrct" << std::endl; -// if(d==0) -// std::cerr << "range<" << d << ">: " << i << " " << j << " " << k << std::endl; -// if(d==1) -// std::cerr << "range<" << d << ">: " << k << " " << i << " " << j << std::endl; -// if(d==2) -// std::cerr << "range<" << d << ">: " << j << " " << k << " " << i << std::endl; - - // do interpolation, useful for testing if ranges are correct - /* - arr(pis(i, j, k)) = real_t(.5) * ( - arr(pis(i - dist, j, k)) + - arr(pis(i + dist, j, k)) - ); - arr(pis(i_next, j, k)) = real_t(.5) * ( - arr(pis(i_next - dist, j, k)) + - arr(pis(i_next + dist, j, k)) - ); - return; - */ - // end of the interpolation test - - -// alternative solution following Scotti and Meneveau Physica D 1999 -/* - - // helper references - // NOTE: these are actually the resolved values only in the first iteration, in the subsequent iterations some of these are reconstructed values - // should we point to the resolved values in all iterations?? - const arr_t u_im1 = arr(pis(i - dist, j, k)), // resolved u_(i-1) - u_i = arr(pis(i + dist, j, k)), // resolved u_(i) - u_ip1 = arr(pis(i_next + dist, j, k)); // resolved u_(i+1) - - // c_j and f_j naming comes from the Akinlabi paper, TODO: rename to a_i b_i - // NOTE: in multiple iterations of reconstruction, they only change due to random stretching parameters? - // or the stretching parameter should be drawn from the distro once? then no need for c_j and f_j to have the same size as reconstructed fields... - arr_t a_1 = this->c_j(pis(i, j, k)), - a_2 = this->c_j(pis(i_next, j, k)), - b_1 = this->f_j(pis(i, j, k)), - b_2 = this->f_j(pis(i_next, j, k)); - - // Eqs. (13) and (14) from Scotti Meneveau 1999 - a_1 = u_i - u_im1 - d_1*(u_ip1 - u_im1); - a_2 = u_ip1 - u_i - d_2*(u_ip1 - u_im1); - b_1 = u_im1*(real_t(1) - d_1); - b_2 = u_i - d_2*u_im1; - - // W(u_0(xi)) at xi=1/4, between u_i-1 and u_i - // u_0(2*xi) = u_i-1 + 1/2 * (u_i+1 - u_i-1) - // Eq. (6) and caption of fig. 1 Scotti 1999 - // NOTE: in multiple iterations, should xi be between the 3 resolved points? or should we consider the reconstructed points as new resolved points? - arr(pis(i, j, k)) = d_1 * -// (u_im1 + 0.5 * (u_ip1 - u_im1)) + // u_0(2*xi) - u_i + - 0.5 * a_1 + b_1; // q_1(2*xi) - // W(u_0(xi)) at xi=3/4, between u_i and u_i+1 - // u_0(2*xi-1) = u_i-1 + 1/2 * (u_i+1 - u_i-1) - // NOTE: u_0(2*3/4-1) == u_0(2*1/4), so calculate it once in the previous step and store it... - arr(pis(i_next, j, k)) = d_2 * -// (u_im1 + 0.5 * (u_ip1 - u_im1)) + // u_0(2*xi-1) - u_i + - 0.5 * a_2 + b_2; // q_2(2*xi-1) -*/ - // Eq. (5) Scotti and Meneveau PRL 1997, a_1 and a_2 differ from Scotti Meneveau 1999 ! - // NOTE: in the 1999 paper, a_1 and a_2 are multipled by 2*xi and 2*xi-1, respectively - // in the 1997 paper, the are both multipled by xi - /* - a_1 = 2 * (u_im1 - u_i - d_1*(u_ip1 - u_im1)); - a_2 = 2 * (u_ip1 - u_i - d_2*(u_ip1 - u_im1)); - b_1 = u_im1*(real_t(1) - d_1); - b_2 = u_i - d_2*u_im1; - */ - } - }; - }; -}; diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index ba612424..035f77f0 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -7,7 +7,6 @@ #pragma once #include -#include namespace libmpdataxx { @@ -39,6 +38,214 @@ namespace libmpdataxx rng_t ijk_r2r_0_h_with_halo, ijk_r2r_1_h, ijk_r2r_2_h; // all positions at resolution of given iteration int stride, hstride; // stride and half stride + // CDF of stretching parameter d; assuming CDF = A d^B + C under condition that CDF(0.5)=0 and CDF(1)=1 + static real_t CDF_of_d(const real_t &d) + { + const real_t B = -0.39091161; // comes from a fit to the E. Akinlabi data + return (pow(d,B) - pow(0.5,B)) / (1. - pow(0.5,B)); + } + // inverse function: d of a CDF + static real_t d_of_CDF(const real_t &CDF) + { + const real_t B = -0.39091161; // comes from a fit to the E. Akinlabi data + return pow((1.-pow(0.5,B))*CDF + pow(0.5,B), 1./B); + } + + struct d_of_CDF_fctr + { + real_t operator()(const real_t &CDF) const + { + return CDF < 0 ? -d_of_CDF(-CDF) : d_of_CDF(CDF); + } + BZ_DECLARE_FUNCTOR(d_of_CDF_fctr); + }; + + + // interpolation similar to mpdata_rhs_vip... + template + void intrp( + arr_t arr, + const rng_t &i, + const rng_t &j, + const rng_t &k, + const int &dist + ) + { + using idxperm::pis; + + // debug output + std::cerr << "ranges in interpolation" << std::endl; + if(d==0) + std::cerr << "range<" << d << ">: " << i << " " << j << " " << k << std::endl; + if(d==1) + std::cerr << "range<" << d << ">: " << k << " " << i << " " << j << std::endl; + if(d==2) + std::cerr << "range<" << d << ">: " << j << " " << k << " " << i << std::endl; + + arr(pis(i, j, k)) = real_t(.5) * ( + arr(pis(i - dist, j, k)) + + arr(pis(i + dist, j, k)) + ); + } + + // fractral reconstruction (calculates 2 mid-points based on 3) + // notation following Akinlabi et al. "Fractal reconstruciton..." 2019 + template + void rcnstrct( + arr_t arr, + const rng_t &i, + const rng_t &j, + const rng_t &k, + const int &dist // number of indices between a reconstructed point and a known point + ) + { + // TODO: move some to formulas:: + using idxperm::pis; + + // second reconstructed position (j=2, between i=1 and i=2) + const rng_t i_next = i + 2*dist; + + // debug output + std::cerr << "ranges in rcnstrct" << std::endl; + if(d==0) + std::cerr << "range<" << d << ">: " << i << " " << j << " " << k << std::endl; + if(d==1) + std::cerr << "range<" << d << ">: " << k << " " << i << " " << j << std::endl; + if(d==2) + std::cerr << "range<" << d << ">: " << j << " " << k << " " << i << std::endl; + // do interpolation, useful for testing if ranges are correct + /* + arr(pis(i, j, k)) = real_t(.5) * ( + arr(pis(i - dist, j, k)) + + arr(pis(i + dist, j, k)) + ); + arr(pis(i_next, j, k)) = real_t(.5) * ( + arr(pis(i_next - dist, j, k)) + + arr(pis(i_next + dist, j, k)) + ); + return; + */ + // end of the interpolation test + + + // stretching parameter, TODO: draw it from the distribution +// const real_t d_1 = -pow(2, -1./3.), +// d_2 = pow(2, -1./3.); + + +// solution following Scotti and Meneveau Physica D 1999 +/* + + // helper references + // NOTE: these are actually the resolved values only in the first iteration, in the subsequent iterations some of these are reconstructed values + // should we point to the resolved values in all iterations?? + const arr_t u_im1 = arr(pis(i - dist, j, k)), // resolved u_(i-1) + u_i = arr(pis(i + dist, j, k)), // resolved u_(i) + u_ip1 = arr(pis(i_next + dist, j, k)); // resolved u_(i+1) + + // c_j and f_j naming comes from the Akinlabi paper, TODO: rename to a_i b_i + // NOTE: in multiple iterations of reconstruction, they only change due to random stretching parameters? + // or the stretching parameter should be drawn from the distro once? then no need for c_j and f_j to have the same size as reconstructed fields... + arr_t a_1 = this->c_j(pis(i, j, k)), + a_2 = this->c_j(pis(i_next, j, k)), + b_1 = this->f_j(pis(i, j, k)), + b_2 = this->f_j(pis(i_next, j, k)); + + // Eqs. (13) and (14) from Scotti Meneveau 1999 + a_1 = u_i - u_im1 - d_1*(u_ip1 - u_im1); + a_2 = u_ip1 - u_i - d_2*(u_ip1 - u_im1); + b_1 = u_im1*(real_t(1) - d_1); + b_2 = u_i - d_2*u_im1; + + // W(u_0(xi)) at xi=1/4, between u_i-1 and u_i + // u_0(2*xi) = u_i-1 + 1/2 * (u_i+1 - u_i-1) + // Eq. (6) and caption of fig. 1 Scotti 1999 + // NOTE: in multiple iterations, should xi be between the 3 resolved points? or should we consider the reconstructed points as new resolved points? + arr(pis(i, j, k)) = d_1 * +// (u_im1 + 0.5 * (u_ip1 - u_im1)) + // u_0(2*xi) + u_i + + 0.5 * a_1 + b_1; // q_1(2*xi) + // W(u_0(xi)) at xi=3/4, between u_i and u_i+1 + // u_0(2*xi-1) = u_i-1 + 1/2 * (u_i+1 - u_i-1) + // NOTE: u_0(2*3/4-1) == u_0(2*1/4), so calculate it once in the previous step and store it... + arr(pis(i_next, j, k)) = d_2 * +// (u_im1 + 0.5 * (u_ip1 - u_im1)) + // u_0(2*xi-1) + u_i + + 0.5 * a_2 + b_2; // q_2(2*xi-1) +*/ + // Eq. (5) Scotti and Meneveau PRL 1997, a_1 and a_2 differ from Scotti Meneveau 1999 ! + // NOTE: in the 1999 paper, a_1 and a_2 are multipled by 2*xi and 2*xi-1, respectively + // in the 1997 paper, the are both multipled by xi + /* + a_1 = 2 * (u_im1 - u_i - d_1*(u_ip1 - u_im1)); + a_2 = 2 * (u_ip1 - u_i - d_2*(u_ip1 - u_im1)); + b_1 = u_im1*(real_t(1) - d_1); + b_2 = u_i - d_2*u_im1; + */ + + //const real_t xi[3] = {0, 0.5, 1}; + + +// solution following Akinlabi et al. + + + const int x[3] = {0, 1, 2}; + + // helper references + const arr_t u_0 = arr(pis(i - dist, j, k)), + u_1 = arr(pis(i + dist, j, k)), + u_2 = arr(pis(i_next + dist, j, k)); + + arr_t c_1 = this->c_j(pis(i, j, k)), + c_2 = this->c_j(pis(i_next, j, k)), + d_1 = this->d_j(pis(i, j, k)), + d_2 = this->d_j(pis(i_next, j, k)), + f_1 = this->f_j(pis(i, j, k)), + f_2 = this->f_j(pis(i_next, j, k)); + + + + + // Eqs. (5) and (6) Akinlabi et al. + c_1 = (u_1 - u_0) / (2.) - d_1 * (u_2 - u_0) / (2.); + c_2 = (u_2 - u_1) / (2.) - d_2 * (u_2 - u_0) / (2.); + f_1 = (x[2] * u_0 - x[0] * u_1) / (2.) - d_1 * (x[2] * u_0 - x[0] * u_2) / (2.); + f_2 = (x[2] * u_1 - x[0] * u_2) / (2.) - d_2 * (x[2] * u_0 - x[0] * u_2) / (2.); +// c_1 = (u_1 - u_0) / (4 * dist) - d_1 * (u_2 - u_0) / (4 * dist); +// c_2 = (u_2 - u_1) / (4 * dist) - d_2 * (u_2 - u_0) / (4 * dist); +// f_1 = (x[2] * u_0 - x[0] * u_1) / (4 * dist) - d_1 * (x[2] * u_0 - x[0] * u_2) / (4 * dist); +// f_2 = (x[2] * u_1 - x[0] * u_2) / (4 * dist) - d_2 * (x[2] * u_0 - x[0] * u_2) / (4 * dist); + + // new u, at j=1, between i=0 and i=1, result of w_j=1(u_i=1), Eq. (1) in Akinlabi et al. + arr(pis(i , j, k)) = c_1 * 1 + d_1 * u_1 + f_1; + // new u, at j=2, between i=1 and i=2, result of w_j=2(u_i=1), Eq. (1) in Akinlabi et al. + arr(pis(i_next, j, k)) = c_2 * 1 + d_2 * u_1 + f_2; + // not sure about * 2*dist here (is x_1 == 2*dist?) + + +/* + c_j(pis(i, j, k)) * x_j + + d_j[0] * arr(pis(i + dist, j, k)) + // coarse-grained u at i=1 + f_j(pis(i, j, k)); + */ + + + // new u, at j=2, between i=1 and i=2, result of w2(u1) + // TODO + +/* + arr(pis(i, j, k)) = real_t(.5) * ( + arr(pis(i - dist, j, k)) + + arr(pis(i + dist, j, k)) + ); + + arr(pis(i_next, j, k)) = real_t(.5) * ( + arr(pis(i_next - dist, j, k)) + + arr(pis(i_next + dist, j, k)) + ); + */ + } + // fill distmem halos of refinee // TODO: move to bcond or sth? would be filled only by remote bcond void fill_refinee_distmem_halos(const int e, const int halo_size) @@ -123,13 +330,16 @@ namespace libmpdataxx rng_t(this->ijk_r2r[0].first(), this->ijk_r2r[0].last(), hstride); } + // TODO: move some formulas + // TODO: move to common? void generate_stretching_parameters(const int rng_seed = 44) { std::mt19937 gen(rng_seed); std::uniform_real_distribution<> dis(-1, 1); // [-1,1), but whatever auto rand = std::bind(dis, gen); std::generate(this->d_j(this->ijk_ref).begin(), this->d_j(this->ijk_ref).end(), rand); - this->d_j(this->ijk_ref) = formulae::fractal::d_of_CDF_fctr{}(this->d_j(this->ijk_ref)); + this->d_j(this->ijk_ref) = d_of_CDF_fctr{}(this->d_j(this->ijk_ref)); + // TODO: randomly convert to negative } void interpolate_refinee(const int e = 0) @@ -151,10 +361,10 @@ namespace libmpdataxx { refinement_ranges(i, halo_size); - formulae::fractal::intrp<0, real_t>(this->mem->psi_ref[e], mid_ijk_r2r_0, ijk_r2r_1_h, ijk_r2r_2_h, hstride); + intrp<0>(this->mem->psi_ref[e], mid_ijk_r2r_0, ijk_r2r_1_h, ijk_r2r_2_h, hstride); this->mem->barrier(); - formulae::fractal::intrp<1, real_t>(this->mem->psi_ref[e], mid_ijk_r2r_1, ijk_r2r_2_h, ijk_r2r_0_h_with_halo, hstride); - formulae::fractal::intrp<2, real_t>(this->mem->psi_ref[e], mid_ijk_r2r_2, ijk_r2r_0_h_with_halo, this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); + intrp<1>(this->mem->psi_ref[e], mid_ijk_r2r_1, ijk_r2r_2_h, ijk_r2r_0_h_with_halo, hstride); + intrp<2>(this->mem->psi_ref[e], mid_ijk_r2r_2, ijk_r2r_0_h_with_halo, this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); } this->mem->barrier(); } @@ -181,10 +391,10 @@ namespace libmpdataxx // if MPI domain starts with a point in the middle of a triple - we need to start calculating from a point to the left (in halo) const int offset = (i==0 && this->mem->grid_size_ref[0].first() > 0 && (this->mem->grid_size_ref[0].first() / stride) % 2 != 0) ? stride : 0; - formulae::fractal::rcnstrct<0, real_t>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_0, offset), ijk_r2r_1_h, ijk_r2r_2_h, hstride, this->c_j, this->d_j, this->f_j); + rcnstrct<0>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_0, offset), ijk_r2r_1_h, ijk_r2r_2_h, hstride); this->mem->barrier(); - formulae::fractal::rcnstrct<1, real_t>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_1) , ijk_r2r_2_h, ijk_r2r_0_h_with_halo, hstride, this->c_j, this->d_j, this->f_j); - formulae::fractal::rcnstrct<2, real_t>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_2) , ijk_r2r_0_h_with_halo, this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride, this->c_j, this->d_j, this->f_j); + rcnstrct<1>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_1) , ijk_r2r_2_h, ijk_r2r_0_h_with_halo, hstride); + rcnstrct<2>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_2) , ijk_r2r_0_h_with_halo, this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); } this->mem->barrier(); } From 59b3babe83b319f45b66e56fde30c4166cb751b9 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 29 Apr 2022 17:08:09 +0200 Subject: [PATCH 048/129] Revert "Revert "move fractal reconstruction formulas to formulae/"" This reverts commit 0fd42ec4805fb49ec7f6d1ae16f3be721a592762. --- .../formulae/fractal_reconstruction.hpp | 191 +++++++++++++++ .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 226 +----------------- 2 files changed, 199 insertions(+), 218 deletions(-) create mode 100644 libmpdata++/formulae/fractal_reconstruction.hpp diff --git a/libmpdata++/formulae/fractal_reconstruction.hpp b/libmpdata++/formulae/fractal_reconstruction.hpp new file mode 100644 index 00000000..b722b520 --- /dev/null +++ b/libmpdata++/formulae/fractal_reconstruction.hpp @@ -0,0 +1,191 @@ +/** @file +* @copyright University of Warsaw +* @section LICENSE +* GPLv3+ (see the COPYING file or http://www.gnu.org/licenses/) +*/ + +#pragma once + +#include + +namespace libmpdataxx +{ + namespace formulae + { + namespace fractal + { + // CDF of stretching parameter d; assuming CDF = A d^B + C under condition that CDF(0.5)=0 and CDF(1)=1 + template + static real_t CDF_of_d(const real_t &d) + { + const real_t B = -0.39091161; // comes from a fit to the E. Akinlabi data + return (pow(d,B) - pow(0.5,B)) / (1. - pow(0.5,B)); + } + // inverse function: d of a CDF + template + static real_t d_of_CDF(const real_t &CDF) + { + const real_t B = -0.39091161; // comes from a fit to the E. Akinlabi data + return pow((1.-pow(0.5,B))*CDF + pow(0.5,B), 1./B); + } + + template + struct d_of_CDF_fctr + { + real_t operator()(const real_t &CDF) const + { + return CDF < 0 ? -d_of_CDF(-CDF) : d_of_CDF(CDF); + } + BZ_DECLARE_FUNCTOR(d_of_CDF_fctr); + }; + + // (tri-)linear interpolation + // 3d version + template + void intrp( + arr_t arr, // array to be interpolated + const rng_t &i, // index of the point to be interpolated to + const rng_t &j, // ditto + const rng_t &k, // ditto + const int &dist // number of indices between an interpolated point and a known point + ) + { + using idxperm::pis; + + // debug output +// std::cerr << "ranges in interpolation" << std::endl; +// if(d==0) +// std::cerr << "range<" << d << ">: " << i << " " << j << " " << k << std::endl; +// if(d==1) +// std::cerr << "range<" << d << ">: " << k << " " << i << " " << j << std::endl; +// if(d==2) +// std::cerr << "range<" << d << ">: " << j << " " << k << " " << i << std::endl; + + arr(pis(i, j, k)) = real_t(.5) * ( + arr(pis(i - dist, j, k)) + + arr(pis(i + dist, j, k)) + ); + } + + // fractral reconstruction (calculates 2 mid-points based on 3) + // following Akinlabi et al. "Fractal reconstruciton..." 2019 + // 3d version + template + void rcnstrct( + arr_t arr, // array to be reconstructed + const rng_t &i, // index of the first point from the pair to be reconstructed + const rng_t &j, // ditto + const rng_t &k, // ditto + const int &dist, // number of indices between a reconstructed point and a known point + arr_t c_j, // parameter c + const arr_t d_j, // parameter d (stretching) - needs to be calculated before + arr_t f_j // parameter f + ) + { + using idxperm::pis; + + // second reconstructed position (j=2, between i=1 and i=2) + const rng_t i_next = i + 2*dist; + + const int x[3] = {0, 1, 2}; + + // helper references + const arr_t u_0 = arr(pis(i - dist, j, k)), + u_1 = arr(pis(i + dist, j, k)), + u_2 = arr(pis(i_next + dist, j, k)); + + arr_t c_1 = c_j(pis(i, j, k)), + c_2 = c_j(pis(i_next, j, k)), + d_1 = d_j(pis(i, j, k)), + d_2 = d_j(pis(i_next, j, k)), + f_1 = f_j(pis(i, j, k)), + f_2 = f_j(pis(i_next, j, k)); + + // Eqs. (5) and (6) Akinlabi et al. + c_1 = (u_1 - u_0) / (2.) - d_1 * (u_2 - u_0) / (2.); + c_2 = (u_2 - u_1) / (2.) - d_2 * (u_2 - u_0) / (2.); + f_1 = (x[2] * u_0 - x[0] * u_1) / (2.) - d_1 * (x[2] * u_0 - x[0] * u_2) / (2.); + f_2 = (x[2] * u_1 - x[0] * u_2) / (2.) - d_2 * (x[2] * u_0 - x[0] * u_2) / (2.); + + // new u, at j=1, between i=0 and i=1, result of w_j=1(u_i=1), Eq. (1) in Akinlabi et al. + arr(pis(i , j, k)) = c_1 * 1 + d_1 * u_1 + f_1; + // new u, at j=2, between i=1 and i=2, result of w_j=2(u_i=1), Eq. (1) in Akinlabi et al. + arr(pis(i_next, j, k)) = c_2 * 1 + d_2 * u_1 + f_2; + + // debug output +// std::cerr << "ranges in rcnstrct" << std::endl; +// if(d==0) +// std::cerr << "range<" << d << ">: " << i << " " << j << " " << k << std::endl; +// if(d==1) +// std::cerr << "range<" << d << ">: " << k << " " << i << " " << j << std::endl; +// if(d==2) +// std::cerr << "range<" << d << ">: " << j << " " << k << " " << i << std::endl; + + // do interpolation, useful for testing if ranges are correct + /* + arr(pis(i, j, k)) = real_t(.5) * ( + arr(pis(i - dist, j, k)) + + arr(pis(i + dist, j, k)) + ); + arr(pis(i_next, j, k)) = real_t(.5) * ( + arr(pis(i_next - dist, j, k)) + + arr(pis(i_next + dist, j, k)) + ); + return; + */ + // end of the interpolation test + + +// alternative solution following Scotti and Meneveau Physica D 1999 +/* + + // helper references + // NOTE: these are actually the resolved values only in the first iteration, in the subsequent iterations some of these are reconstructed values + // should we point to the resolved values in all iterations?? + const arr_t u_im1 = arr(pis(i - dist, j, k)), // resolved u_(i-1) + u_i = arr(pis(i + dist, j, k)), // resolved u_(i) + u_ip1 = arr(pis(i_next + dist, j, k)); // resolved u_(i+1) + + // c_j and f_j naming comes from the Akinlabi paper, TODO: rename to a_i b_i + // NOTE: in multiple iterations of reconstruction, they only change due to random stretching parameters? + // or the stretching parameter should be drawn from the distro once? then no need for c_j and f_j to have the same size as reconstructed fields... + arr_t a_1 = this->c_j(pis(i, j, k)), + a_2 = this->c_j(pis(i_next, j, k)), + b_1 = this->f_j(pis(i, j, k)), + b_2 = this->f_j(pis(i_next, j, k)); + + // Eqs. (13) and (14) from Scotti Meneveau 1999 + a_1 = u_i - u_im1 - d_1*(u_ip1 - u_im1); + a_2 = u_ip1 - u_i - d_2*(u_ip1 - u_im1); + b_1 = u_im1*(real_t(1) - d_1); + b_2 = u_i - d_2*u_im1; + + // W(u_0(xi)) at xi=1/4, between u_i-1 and u_i + // u_0(2*xi) = u_i-1 + 1/2 * (u_i+1 - u_i-1) + // Eq. (6) and caption of fig. 1 Scotti 1999 + // NOTE: in multiple iterations, should xi be between the 3 resolved points? or should we consider the reconstructed points as new resolved points? + arr(pis(i, j, k)) = d_1 * +// (u_im1 + 0.5 * (u_ip1 - u_im1)) + // u_0(2*xi) + u_i + + 0.5 * a_1 + b_1; // q_1(2*xi) + // W(u_0(xi)) at xi=3/4, between u_i and u_i+1 + // u_0(2*xi-1) = u_i-1 + 1/2 * (u_i+1 - u_i-1) + // NOTE: u_0(2*3/4-1) == u_0(2*1/4), so calculate it once in the previous step and store it... + arr(pis(i_next, j, k)) = d_2 * +// (u_im1 + 0.5 * (u_ip1 - u_im1)) + // u_0(2*xi-1) + u_i + + 0.5 * a_2 + b_2; // q_2(2*xi-1) +*/ + // Eq. (5) Scotti and Meneveau PRL 1997, a_1 and a_2 differ from Scotti Meneveau 1999 ! + // NOTE: in the 1999 paper, a_1 and a_2 are multipled by 2*xi and 2*xi-1, respectively + // in the 1997 paper, the are both multipled by xi + /* + a_1 = 2 * (u_im1 - u_i - d_1*(u_ip1 - u_im1)); + a_2 = 2 * (u_ip1 - u_i - d_2*(u_ip1 - u_im1)); + b_1 = u_im1*(real_t(1) - d_1); + b_2 = u_i - d_2*u_im1; + */ + } + }; + }; +}; diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 035f77f0..ba612424 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -7,6 +7,7 @@ #pragma once #include +#include namespace libmpdataxx { @@ -38,214 +39,6 @@ namespace libmpdataxx rng_t ijk_r2r_0_h_with_halo, ijk_r2r_1_h, ijk_r2r_2_h; // all positions at resolution of given iteration int stride, hstride; // stride and half stride - // CDF of stretching parameter d; assuming CDF = A d^B + C under condition that CDF(0.5)=0 and CDF(1)=1 - static real_t CDF_of_d(const real_t &d) - { - const real_t B = -0.39091161; // comes from a fit to the E. Akinlabi data - return (pow(d,B) - pow(0.5,B)) / (1. - pow(0.5,B)); - } - // inverse function: d of a CDF - static real_t d_of_CDF(const real_t &CDF) - { - const real_t B = -0.39091161; // comes from a fit to the E. Akinlabi data - return pow((1.-pow(0.5,B))*CDF + pow(0.5,B), 1./B); - } - - struct d_of_CDF_fctr - { - real_t operator()(const real_t &CDF) const - { - return CDF < 0 ? -d_of_CDF(-CDF) : d_of_CDF(CDF); - } - BZ_DECLARE_FUNCTOR(d_of_CDF_fctr); - }; - - - // interpolation similar to mpdata_rhs_vip... - template - void intrp( - arr_t arr, - const rng_t &i, - const rng_t &j, - const rng_t &k, - const int &dist - ) - { - using idxperm::pis; - - // debug output - std::cerr << "ranges in interpolation" << std::endl; - if(d==0) - std::cerr << "range<" << d << ">: " << i << " " << j << " " << k << std::endl; - if(d==1) - std::cerr << "range<" << d << ">: " << k << " " << i << " " << j << std::endl; - if(d==2) - std::cerr << "range<" << d << ">: " << j << " " << k << " " << i << std::endl; - - arr(pis(i, j, k)) = real_t(.5) * ( - arr(pis(i - dist, j, k)) + - arr(pis(i + dist, j, k)) - ); - } - - // fractral reconstruction (calculates 2 mid-points based on 3) - // notation following Akinlabi et al. "Fractal reconstruciton..." 2019 - template - void rcnstrct( - arr_t arr, - const rng_t &i, - const rng_t &j, - const rng_t &k, - const int &dist // number of indices between a reconstructed point and a known point - ) - { - // TODO: move some to formulas:: - using idxperm::pis; - - // second reconstructed position (j=2, between i=1 and i=2) - const rng_t i_next = i + 2*dist; - - // debug output - std::cerr << "ranges in rcnstrct" << std::endl; - if(d==0) - std::cerr << "range<" << d << ">: " << i << " " << j << " " << k << std::endl; - if(d==1) - std::cerr << "range<" << d << ">: " << k << " " << i << " " << j << std::endl; - if(d==2) - std::cerr << "range<" << d << ">: " << j << " " << k << " " << i << std::endl; - // do interpolation, useful for testing if ranges are correct - /* - arr(pis(i, j, k)) = real_t(.5) * ( - arr(pis(i - dist, j, k)) + - arr(pis(i + dist, j, k)) - ); - arr(pis(i_next, j, k)) = real_t(.5) * ( - arr(pis(i_next - dist, j, k)) + - arr(pis(i_next + dist, j, k)) - ); - return; - */ - // end of the interpolation test - - - // stretching parameter, TODO: draw it from the distribution -// const real_t d_1 = -pow(2, -1./3.), -// d_2 = pow(2, -1./3.); - - -// solution following Scotti and Meneveau Physica D 1999 -/* - - // helper references - // NOTE: these are actually the resolved values only in the first iteration, in the subsequent iterations some of these are reconstructed values - // should we point to the resolved values in all iterations?? - const arr_t u_im1 = arr(pis(i - dist, j, k)), // resolved u_(i-1) - u_i = arr(pis(i + dist, j, k)), // resolved u_(i) - u_ip1 = arr(pis(i_next + dist, j, k)); // resolved u_(i+1) - - // c_j and f_j naming comes from the Akinlabi paper, TODO: rename to a_i b_i - // NOTE: in multiple iterations of reconstruction, they only change due to random stretching parameters? - // or the stretching parameter should be drawn from the distro once? then no need for c_j and f_j to have the same size as reconstructed fields... - arr_t a_1 = this->c_j(pis(i, j, k)), - a_2 = this->c_j(pis(i_next, j, k)), - b_1 = this->f_j(pis(i, j, k)), - b_2 = this->f_j(pis(i_next, j, k)); - - // Eqs. (13) and (14) from Scotti Meneveau 1999 - a_1 = u_i - u_im1 - d_1*(u_ip1 - u_im1); - a_2 = u_ip1 - u_i - d_2*(u_ip1 - u_im1); - b_1 = u_im1*(real_t(1) - d_1); - b_2 = u_i - d_2*u_im1; - - // W(u_0(xi)) at xi=1/4, between u_i-1 and u_i - // u_0(2*xi) = u_i-1 + 1/2 * (u_i+1 - u_i-1) - // Eq. (6) and caption of fig. 1 Scotti 1999 - // NOTE: in multiple iterations, should xi be between the 3 resolved points? or should we consider the reconstructed points as new resolved points? - arr(pis(i, j, k)) = d_1 * -// (u_im1 + 0.5 * (u_ip1 - u_im1)) + // u_0(2*xi) - u_i + - 0.5 * a_1 + b_1; // q_1(2*xi) - // W(u_0(xi)) at xi=3/4, between u_i and u_i+1 - // u_0(2*xi-1) = u_i-1 + 1/2 * (u_i+1 - u_i-1) - // NOTE: u_0(2*3/4-1) == u_0(2*1/4), so calculate it once in the previous step and store it... - arr(pis(i_next, j, k)) = d_2 * -// (u_im1 + 0.5 * (u_ip1 - u_im1)) + // u_0(2*xi-1) - u_i + - 0.5 * a_2 + b_2; // q_2(2*xi-1) -*/ - // Eq. (5) Scotti and Meneveau PRL 1997, a_1 and a_2 differ from Scotti Meneveau 1999 ! - // NOTE: in the 1999 paper, a_1 and a_2 are multipled by 2*xi and 2*xi-1, respectively - // in the 1997 paper, the are both multipled by xi - /* - a_1 = 2 * (u_im1 - u_i - d_1*(u_ip1 - u_im1)); - a_2 = 2 * (u_ip1 - u_i - d_2*(u_ip1 - u_im1)); - b_1 = u_im1*(real_t(1) - d_1); - b_2 = u_i - d_2*u_im1; - */ - - //const real_t xi[3] = {0, 0.5, 1}; - - -// solution following Akinlabi et al. - - - const int x[3] = {0, 1, 2}; - - // helper references - const arr_t u_0 = arr(pis(i - dist, j, k)), - u_1 = arr(pis(i + dist, j, k)), - u_2 = arr(pis(i_next + dist, j, k)); - - arr_t c_1 = this->c_j(pis(i, j, k)), - c_2 = this->c_j(pis(i_next, j, k)), - d_1 = this->d_j(pis(i, j, k)), - d_2 = this->d_j(pis(i_next, j, k)), - f_1 = this->f_j(pis(i, j, k)), - f_2 = this->f_j(pis(i_next, j, k)); - - - - - // Eqs. (5) and (6) Akinlabi et al. - c_1 = (u_1 - u_0) / (2.) - d_1 * (u_2 - u_0) / (2.); - c_2 = (u_2 - u_1) / (2.) - d_2 * (u_2 - u_0) / (2.); - f_1 = (x[2] * u_0 - x[0] * u_1) / (2.) - d_1 * (x[2] * u_0 - x[0] * u_2) / (2.); - f_2 = (x[2] * u_1 - x[0] * u_2) / (2.) - d_2 * (x[2] * u_0 - x[0] * u_2) / (2.); -// c_1 = (u_1 - u_0) / (4 * dist) - d_1 * (u_2 - u_0) / (4 * dist); -// c_2 = (u_2 - u_1) / (4 * dist) - d_2 * (u_2 - u_0) / (4 * dist); -// f_1 = (x[2] * u_0 - x[0] * u_1) / (4 * dist) - d_1 * (x[2] * u_0 - x[0] * u_2) / (4 * dist); -// f_2 = (x[2] * u_1 - x[0] * u_2) / (4 * dist) - d_2 * (x[2] * u_0 - x[0] * u_2) / (4 * dist); - - // new u, at j=1, between i=0 and i=1, result of w_j=1(u_i=1), Eq. (1) in Akinlabi et al. - arr(pis(i , j, k)) = c_1 * 1 + d_1 * u_1 + f_1; - // new u, at j=2, between i=1 and i=2, result of w_j=2(u_i=1), Eq. (1) in Akinlabi et al. - arr(pis(i_next, j, k)) = c_2 * 1 + d_2 * u_1 + f_2; - // not sure about * 2*dist here (is x_1 == 2*dist?) - - -/* - c_j(pis(i, j, k)) * x_j + - d_j[0] * arr(pis(i + dist, j, k)) + // coarse-grained u at i=1 - f_j(pis(i, j, k)); - */ - - - // new u, at j=2, between i=1 and i=2, result of w2(u1) - // TODO - -/* - arr(pis(i, j, k)) = real_t(.5) * ( - arr(pis(i - dist, j, k)) + - arr(pis(i + dist, j, k)) - ); - - arr(pis(i_next, j, k)) = real_t(.5) * ( - arr(pis(i_next - dist, j, k)) + - arr(pis(i_next + dist, j, k)) - ); - */ - } - // fill distmem halos of refinee // TODO: move to bcond or sth? would be filled only by remote bcond void fill_refinee_distmem_halos(const int e, const int halo_size) @@ -330,16 +123,13 @@ namespace libmpdataxx rng_t(this->ijk_r2r[0].first(), this->ijk_r2r[0].last(), hstride); } - // TODO: move some formulas - // TODO: move to common? void generate_stretching_parameters(const int rng_seed = 44) { std::mt19937 gen(rng_seed); std::uniform_real_distribution<> dis(-1, 1); // [-1,1), but whatever auto rand = std::bind(dis, gen); std::generate(this->d_j(this->ijk_ref).begin(), this->d_j(this->ijk_ref).end(), rand); - this->d_j(this->ijk_ref) = d_of_CDF_fctr{}(this->d_j(this->ijk_ref)); - // TODO: randomly convert to negative + this->d_j(this->ijk_ref) = formulae::fractal::d_of_CDF_fctr{}(this->d_j(this->ijk_ref)); } void interpolate_refinee(const int e = 0) @@ -361,10 +151,10 @@ namespace libmpdataxx { refinement_ranges(i, halo_size); - intrp<0>(this->mem->psi_ref[e], mid_ijk_r2r_0, ijk_r2r_1_h, ijk_r2r_2_h, hstride); + formulae::fractal::intrp<0, real_t>(this->mem->psi_ref[e], mid_ijk_r2r_0, ijk_r2r_1_h, ijk_r2r_2_h, hstride); this->mem->barrier(); - intrp<1>(this->mem->psi_ref[e], mid_ijk_r2r_1, ijk_r2r_2_h, ijk_r2r_0_h_with_halo, hstride); - intrp<2>(this->mem->psi_ref[e], mid_ijk_r2r_2, ijk_r2r_0_h_with_halo, this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); + formulae::fractal::intrp<1, real_t>(this->mem->psi_ref[e], mid_ijk_r2r_1, ijk_r2r_2_h, ijk_r2r_0_h_with_halo, hstride); + formulae::fractal::intrp<2, real_t>(this->mem->psi_ref[e], mid_ijk_r2r_2, ijk_r2r_0_h_with_halo, this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); } this->mem->barrier(); } @@ -391,10 +181,10 @@ namespace libmpdataxx // if MPI domain starts with a point in the middle of a triple - we need to start calculating from a point to the left (in halo) const int offset = (i==0 && this->mem->grid_size_ref[0].first() > 0 && (this->mem->grid_size_ref[0].first() / stride) % 2 != 0) ? stride : 0; - rcnstrct<0>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_0, offset), ijk_r2r_1_h, ijk_r2r_2_h, hstride); + formulae::fractal::rcnstrct<0, real_t>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_0, offset), ijk_r2r_1_h, ijk_r2r_2_h, hstride, this->c_j, this->d_j, this->f_j); this->mem->barrier(); - rcnstrct<1>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_1) , ijk_r2r_2_h, ijk_r2r_0_h_with_halo, hstride); - rcnstrct<2>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_2) , ijk_r2r_0_h_with_halo, this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); + formulae::fractal::rcnstrct<1, real_t>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_1) , ijk_r2r_2_h, ijk_r2r_0_h_with_halo, hstride, this->c_j, this->d_j, this->f_j); + formulae::fractal::rcnstrct<2, real_t>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_2) , ijk_r2r_0_h_with_halo, this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride, this->c_j, this->d_j, this->f_j); } this->mem->barrier(); } From a788a4d9dac024e4688a5e1e2c2394f70f5511d4 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 29 Apr 2022 17:32:11 +0200 Subject: [PATCH 049/129] pbl smg: set fra iter --- tests/sandbox/pbl/pbl_test_def.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/sandbox/pbl/pbl_test_def.hpp b/tests/sandbox/pbl/pbl_test_def.hpp index 6fce5efb..73e9300a 100644 --- a/tests/sandbox/pbl/pbl_test_def.hpp +++ b/tests/sandbox/pbl/pbl_test_def.hpp @@ -33,6 +33,7 @@ void set_sgs_specific(params_t &p, smg_tag) p.smg_c = 0.165; p.prandtl_num = 0.42; p.cdrag = 0.1; + p.n_fra_iter = 10; } template From 34ad9a5f554381557e1388dbe63493e7de013043 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Fri, 29 Apr 2022 17:43:01 +0200 Subject: [PATCH 050/129] pbl smg: fra iter 4 --- tests/sandbox/pbl/pbl_test_def.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/sandbox/pbl/pbl_test_def.hpp b/tests/sandbox/pbl/pbl_test_def.hpp index 73e9300a..981fd640 100644 --- a/tests/sandbox/pbl/pbl_test_def.hpp +++ b/tests/sandbox/pbl/pbl_test_def.hpp @@ -33,7 +33,7 @@ void set_sgs_specific(params_t &p, smg_tag) p.smg_c = 0.165; p.prandtl_num = 0.42; p.cdrag = 0.1; - p.n_fra_iter = 10; + p.n_fra_iter = 4; } template From be4a6a30b250c5f1f818cfd4cb4cf11e230ae71f Mon Sep 17 00:00:00 2001 From: pdziekan Date: Wed, 4 May 2022 16:47:38 +0200 Subject: [PATCH 051/129] alloc refined arrays only for required advectees --- .../concurr/detail/sharedmem_refined.hpp | 4 +- libmpdata++/opts.hpp | 11 ++++++ .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 20 +++++----- .../mpdata_rhs_vip_prs_sgs_fra_common.hpp | 13 +++++++ .../solvers/mpdata_rhs_vip_prs_sgs_fra.hpp | 4 +- tests/sandbox/pbl/pbl.hpp | 38 ++----------------- tests/sandbox/pbl/pbl_test_def.hpp | 2 +- 7 files changed, 41 insertions(+), 51 deletions(-) diff --git a/libmpdata++/concurr/detail/sharedmem_refined.hpp b/libmpdata++/concurr/detail/sharedmem_refined.hpp index b63ae176..d9e60484 100644 --- a/libmpdata++/concurr/detail/sharedmem_refined.hpp +++ b/libmpdata++/concurr/detail/sharedmem_refined.hpp @@ -42,7 +42,8 @@ namespace libmpdataxx std::array grid_size_ref; // TODO: these are public because used from outside in alloc - could friendship help? - arrvec_t GC_ref, psi_ref; + //arrvec_t GC_ref, psi_ref; + arrvec_t psi_ref; // ctors sharedmem_refined_common(const std::array &grid_size, const int &size, const int &n_ref) @@ -70,6 +71,7 @@ namespace libmpdataxx } } + // NOTE: not all advectees are refined, so e (numbering) in refinee is different than in advectee virtual arr_t refinee(int e = 0) = 0; // virtual const arr_t refinee_global_ref(int e = 0) = 0; diff --git a/libmpdata++/opts.hpp b/libmpdata++/opts.hpp index 16e526da..33f24dc5 100644 --- a/libmpdata++/opts.hpp +++ b/libmpdata++/opts.hpp @@ -31,6 +31,17 @@ namespace libmpdataxx return x == 0 ? 0. : int(log(x)/log(2)) + 1; } + // https://www.geeksforgeeks.org/count-set-bits-in-an-integer/ + constexpr unsigned int nset(opts_t x) + { + unsigned int count = 0; + while (x) { + count += x & 1; + x >>= 1; + } + return count; + } + enum { fct = opts::bit(0), // flux-corrected transport diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index ba612424..dae480aa 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -32,7 +32,7 @@ namespace libmpdataxx using parent_t::parent_t; // inheriting constructors using real_t = typename ct_params_t::real_t; - protected: + private: // helper variables for grid refinement rng_t mid_ijk_r2r_0, mid_ijk_r2r_1, mid_ijk_r2r_2; // positions between already known values (to be filled during given iteration) @@ -132,13 +132,13 @@ namespace libmpdataxx this->d_j(this->ijk_ref) = formulae::fractal::d_of_CDF_fctr{}(this->d_j(this->ijk_ref)); } - void interpolate_refinee(const int e = 0) + protected: + + void interpolate_refinee(const int _e = 0) { + assert(opts::isset(ct_params_t::fractal_recon, opts::bit(_e))); const int halo_size = 1; - - // TEMPORARY - this->mem->psi_ref[e] = -1000; - this->mem->barrier(); + const int e = this->ix_r2r.at(_e); fill_refinee_distmem_halos(e, halo_size); @@ -159,13 +159,11 @@ namespace libmpdataxx this->mem->barrier(); } - void reconstruct_refinee(const int e = 0) + void reconstruct_refinee(const int _e = 0) { + assert(opts::isset(ct_params_t::fractal_recon, opts::bit(_e))); const int halo_size = 2; - - // TEMPORARY - this->mem->psi_ref[e] = -1000; - this->mem->barrier(); + const int e = this->ix_r2r.at(_e); fill_refinee_distmem_halos(e, halo_size); diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp index 47c6b4f0..f8fd0fd3 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp @@ -42,6 +42,18 @@ namespace libmpdataxx // refined arrays have no halos (do they need them? halos can be added by extending grid_size in alloc() in mpdata_rhs_vip_prs_sgs_fra.hpp by e.g. mem->n_ref/2) // const int halo_ref; // size of halos of refined scalar (and vector?) arrays + const std::array ix_r2r; + constexpr std::array get_ix_r2r() + { + std::array ix_r2r{}; + int j = 0; + for(opts::opts_t e=0; e ijk_ref; // range of refinee handled by given solver @@ -119,6 +131,7 @@ namespace libmpdataxx n_fra_iter(p.n_fra_iter), n_ref(args.mem->n_ref), // halo_ref(n_ref / 2), + ix_r2r(get_ix_r2r()), // ijk_ref init below assumes 3D (shmem decomp dim along y); // TODO: move this to concurr_common::init()? add something to ctor_args_t? ijk_r2r{ diff --git a/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp b/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp index f70842d5..4397571b 100644 --- a/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp +++ b/libmpdata++/solvers/mpdata_rhs_vip_prs_sgs_fra.hpp @@ -37,7 +37,6 @@ namespace libmpdataxx > : public detail::mpdata_rhs_vip_prs_sgs_fra_dim { using parent_t = detail::mpdata_rhs_vip_prs_sgs_fra_dim; -// using parent_t::parent_t; // inheriting constructors protected: using solver_family = mpdata_rhs_vip_prs_sgs_fra_family_tag; @@ -60,8 +59,7 @@ namespace libmpdataxx const int &n_iters ) { parent_t::alloc(mem, n_iters); - // TODO: allocate only for fields that we want to reconstruct, defined in ct_params_t::fractal_recon, will this mess with field numbering? e.g. ix::tht_ref.... - parent_t::alloc_tmp_sclr_ref(mem, __FILE__, ct_params_t::n_eqns); // psi_ref + parent_t::alloc_tmp_sclr_ref(mem, __FILE__, opts::nset(ct_params_t::fractal_recon)); parent_t::alloc_tmp_sclr_ref(mem, __FILE__, 3); // c_j, d_j, f_j // parent_t::alloc_tmp_vctr(mem, __FILE__, mem->grid_size_ref); // GC_ref diff --git a/tests/sandbox/pbl/pbl.hpp b/tests/sandbox/pbl/pbl.hpp index a856a14c..f12a5d90 100644 --- a/tests/sandbox/pbl/pbl.hpp +++ b/tests/sandbox/pbl/pbl.hpp @@ -59,24 +59,9 @@ class pbl : public libmpdataxx::output::hdf5_xdmftimestep % static_cast(this->outfreq) == 0) { - if (this->rank == 0) std::cout << this->timestep << std::endl; - - // test filling of refinee - //this->mem->refinee(ix::tht)(this->ijk_ref) = blitz::tensor::k * 1e-3; // anything -// this->mem->refinee(ix::tht)(this->ijk_ref) = -1; - //this->mem->refinee(ix::tht) = blitz::tensor::k; // anything - - // copy tht to refined tht at position where they overlap -// this->mem->refinee(ix::tht)(this->ijk_r2r) = this->mem->advectee(ix::tht)(this->ijk); - -// std::cerr << "refinee: " << this->mem->refinee(ix::tht); -// std::cerr << "refinee(ijk_ref): " << this->mem->refinee(ix::tht)(this->ijk_ref); -// std::cerr << "refinee(ijk_r2r): " << this->mem->refinee(ix::tht)(this->ijk_r2r); + this->reconstruct_refinee(ix::w); - this->interpolate_refinee(ix::tht); - this->interpolate_refinee(ix::u); - this->interpolate_refinee(ix::v); - this->interpolate_refinee(ix::w); + if (this->rank == 0) std::cout << this->timestep << std::endl; this->mem->barrier(); if (this->rank == 0) @@ -86,26 +71,9 @@ class pbl : public libmpdataxx::output::hdf5_xdmfrecord_aux_dsc("tke", this->tke); } this->record_aux_dsc("p", this->Phi); - this->record_aux_dsc_refined("tht refined", this->mem->refinee(ix::tht)); - this->record_aux_dsc_refined("u refined", this->mem->refinee(ix::u)); - this->record_aux_dsc_refined("v refined", this->mem->refinee(ix::v)); - this->record_aux_dsc_refined("w refined", this->mem->refinee(ix::w)); + this->record_aux_dsc_refined("w reconstructed", this->mem->refinee(this->ix_r2r.at(ix::w))); } this->mem->barrier(); - - - this->reconstruct_refinee(ix::tht); - this->reconstruct_refinee(ix::u); - this->reconstruct_refinee(ix::v); - this->reconstruct_refinee(ix::w); - this->mem->barrier(); - if (this->rank == 0) - { - this->record_aux_dsc_refined("tht reconstructed", this->mem->refinee(ix::tht)); - this->record_aux_dsc_refined("u reconstructed", this->mem->refinee(ix::u)); - this->record_aux_dsc_refined("v reconstructed", this->mem->refinee(ix::v)); - this->record_aux_dsc_refined("w reconstructed", this->mem->refinee(ix::w)); - } } } diff --git a/tests/sandbox/pbl/pbl_test_def.hpp b/tests/sandbox/pbl/pbl_test_def.hpp index 981fd640..5cc2834b 100644 --- a/tests/sandbox/pbl/pbl_test_def.hpp +++ b/tests/sandbox/pbl/pbl_test_def.hpp @@ -56,7 +56,7 @@ void test(const std::string &dirname, const int np, const int nt) u, v, w, tht, vip_i=u, vip_j=v, vip_k=w, vip_den=-1 }; }; - enum { fractal_recon = libmpdataxx::opts::bit(ix::tht) | libmpdataxx::opts::bit(ix::u) }; + enum { fractal_recon = libmpdataxx::opts::bit(ix::w) }; }; using ix = typename ct_params_t::ix; From 6e5ba14190f07963edc2042234acf8277b8260e9 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 9 May 2022 14:25:18 +0200 Subject: [PATCH 052/129] grid refinement: refined boundary in the middle between mpi domains --- .../concurr/detail/sharedmem_refined.hpp | 4 +- .../formulae/fractal_reconstruction.hpp | 55 ++-- libmpdata++/output/hdf5.hpp | 6 +- .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 243 ++++++++++++++---- .../mpdata_rhs_vip_prs_sgs_fra_common.hpp | 53 +++- tests/sandbox/pbl/pbl.hpp | 15 +- tests/sandbox/pbl/pbl_iles_short.cpp | 2 +- tests/sandbox/pbl/pbl_test_def.hpp | 5 +- 8 files changed, 282 insertions(+), 101 deletions(-) diff --git a/libmpdata++/concurr/detail/sharedmem_refined.hpp b/libmpdata++/concurr/detail/sharedmem_refined.hpp index d9e60484..6c393802 100644 --- a/libmpdata++/concurr/detail/sharedmem_refined.hpp +++ b/libmpdata++/concurr/detail/sharedmem_refined.hpp @@ -85,8 +85,8 @@ namespace libmpdataxx assert(n_ref % 2 == 0); // NOTE: overlapping points inbetween MPI domains return rng_t( - mpi_rank == 0 ? grid_size.first() * n_ref : grid_size.first() * n_ref, - mpi_rank == mpi_size-1 ? grid_size.last() * n_ref : grid_size.last() * n_ref + n_ref - 1 // refined points between MPI domains belong to the MPI process on the left + mpi_rank == 0 ? grid_size.first() * n_ref : grid_size.first() * n_ref - n_ref / 2, + mpi_rank == mpi_size-1 ? grid_size.last() * n_ref : grid_size.last() * n_ref + n_ref / 2 // refined points between MPI domains are evenly divided between MPI processes ); } }; diff --git a/libmpdata++/formulae/fractal_reconstruction.hpp b/libmpdata++/formulae/fractal_reconstruction.hpp index b722b520..fdae5c69 100644 --- a/libmpdata++/formulae/fractal_reconstruction.hpp +++ b/libmpdata++/formulae/fractal_reconstruction.hpp @@ -53,13 +53,13 @@ namespace libmpdataxx using idxperm::pis; // debug output -// std::cerr << "ranges in interpolation" << std::endl; -// if(d==0) -// std::cerr << "range<" << d << ">: " << i << " " << j << " " << k << std::endl; -// if(d==1) -// std::cerr << "range<" << d << ">: " << k << " " << i << " " << j << std::endl; -// if(d==2) -// std::cerr << "range<" << d << ">: " << j << " " << k << " " << i << std::endl; + std::cerr << "ranges in interpolation" << std::endl; + if(d==0) + std::cerr << "range<" << d << ">: " << i << " " << j << " " << k << std::endl; + if(d==1) + std::cerr << "range<" << d << ">: " << k << " " << i << " " << j << std::endl; + if(d==2) + std::cerr << "range<" << d << ">: " << j << " " << k << " " << i << std::endl; arr(pis(i, j, k)) = real_t(.5) * ( arr(pis(i - dist, j, k)) + @@ -82,7 +82,16 @@ namespace libmpdataxx arr_t f_j // parameter f ) { - using idxperm::pis; + // debug output + std::cerr << "ranges in rcnstrct" << std::endl; + if(d==0) + std::cerr << "range<" << d << ">: " << i << " " << j << " " << k << std::endl; + if(d==1) + std::cerr << "range<" << d << ">: " << k << " " << i << " " << j << std::endl; + if(d==2) + std::cerr << "range<" << d << ">: " << j << " " << k << " " << i << std::endl; + + using idxperm::pis; // second reconstructed position (j=2, between i=1 and i=2) const rng_t i_next = i + 2*dist; @@ -111,28 +120,14 @@ namespace libmpdataxx arr(pis(i , j, k)) = c_1 * 1 + d_1 * u_1 + f_1; // new u, at j=2, between i=1 and i=2, result of w_j=2(u_i=1), Eq. (1) in Akinlabi et al. arr(pis(i_next, j, k)) = c_2 * 1 + d_2 * u_1 + f_2; - - // debug output -// std::cerr << "ranges in rcnstrct" << std::endl; -// if(d==0) -// std::cerr << "range<" << d << ">: " << i << " " << j << " " << k << std::endl; -// if(d==1) -// std::cerr << "range<" << d << ">: " << k << " " << i << " " << j << std::endl; -// if(d==2) -// std::cerr << "range<" << d << ">: " << j << " " << k << " " << i << std::endl; - - // do interpolation, useful for testing if ranges are correct - /* - arr(pis(i, j, k)) = real_t(.5) * ( - arr(pis(i - dist, j, k)) + - arr(pis(i + dist, j, k)) - ); - arr(pis(i_next, j, k)) = real_t(.5) * ( - arr(pis(i_next - dist, j, k)) + - arr(pis(i_next + dist, j, k)) - ); - return; - */ + + // DEBUG: do interpolation, useful for testing if ranges are correct +// arr(pis(i, j, k)) = real_t(.5) * (u_0 + u_1); +// arr(pis(i_next, j, k)) = real_t(.5) * (u_1 + u_2); +// +// std::cerr << "new u_j=1:" << arr(pis(i, j, k)); +// std::cerr << "new u_j=2:" << arr(pis(i_next, j, k)); +// return; // end of the interpolation test diff --git a/libmpdata++/output/hdf5.hpp b/libmpdata++/output/hdf5.hpp index 7ba87f90..c3656241 100644 --- a/libmpdata++/output/hdf5.hpp +++ b/libmpdata++/output/hdf5.hpp @@ -124,8 +124,10 @@ namespace libmpdataxx { shape[0] = this->mem->grid_size[0].length(); cshape[0] = this->mem->grid_size[0].length(); - shape_ref[0] = this->mem->grid_size_ref[0].length(); - cshape_ref[0] = this->mem->grid_size_ref[0].length(); + shape_ref[0] = this->mem->distmem.rank() < this->mem->distmem.size() - 1 ? + this->mem->grid_size_ref[0].length()-1 : // -1 to make ranges nonoverlapping + this->mem->grid_size_ref[0].length(); + cshape_ref[0] = shape_ref[0]; if (this->mem->distmem.rank() == this->mem->distmem.size() - 1) { diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index dae480aa..4857e484 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -43,52 +43,89 @@ namespace libmpdataxx // TODO: move to bcond or sth? would be filled only by remote bcond void fill_refinee_distmem_halos(const int e, const int halo_size) { + const int e_ref = this->ix_r2r.at(e); // TODO: we only need to xchng along distmem direction (x) this->xchng(e); // TODO: assert halo_size <= solver halo size + // TODO: DRY! switch(halo_size) { case 2: - this->mem->psi_ref[e]( - this->mem->grid_size_ref[0].last() + 1 + this->mem->n_ref , - this->ijk_r2r[1], - this->ijk_r2r[2] - ) = - this->mem->psi[e][0]( - this->ijk[0].last()+2, - this->ijk[1], - this->ijk[2] - ); + if(this->mem->distmem.rank() < this->mem->distmem.size() - 1) + this->mem->psi_ref[e_ref]( + this->mem->grid_size_ref[0].last() + this->mem->n_ref / 2 + this->mem->n_ref , + this->ijk_r2r[1], + this->ijk_r2r[2] + ) = + this->mem->psi[e][0]( + this->ijk[0].last()+2, + this->ijk[1], + this->ijk[2] + ); + if(this->mem->distmem.rank() > 0) + this->mem->psi_ref[e_ref]( + this->mem->grid_size_ref[0].first() - this->mem->n_ref / 2 - this->mem->n_ref , + this->ijk_r2r[1], + this->ijk_r2r[2] + ) = + this->mem->psi[e][0]( + this->ijk[0].first()-2, + this->ijk[1], + this->ijk[2] + ); // no break intentionally case 1: - this->mem->psi_ref[e]( - this->mem->grid_size_ref[0].first() - this->mem->n_ref, - this->ijk_r2r[1], - this->ijk_r2r[2] - ) = - this->mem->psi[e][0]( - this->ijk[0].first()-1, - this->ijk[1], - this->ijk[2] - ); - - this->mem->psi_ref[e]( - this->mem->grid_size_ref[0].last() + 1, - this->ijk_r2r[1], - this->ijk_r2r[2] - ) = - this->mem->psi[e][0]( - this->ijk[0].last()+1, - this->ijk[1], - this->ijk[2] - ); + if(this->mem->distmem.rank() < this->mem->distmem.size() - 1) + this->mem->psi_ref[e_ref]( + this->mem->grid_size_ref[0].last() + this->mem->n_ref / 2, + this->ijk_r2r[1], + this->ijk_r2r[2] + ) = + this->mem->psi[e][0]( + this->ijk[0].last()+1, + this->ijk[1], + this->ijk[2] + ); + if(this->mem->distmem.rank() > 0) + this->mem->psi_ref[e_ref]( + this->mem->grid_size_ref[0].first() - this->mem->n_ref / 2, + this->ijk_r2r[1], + this->ijk_r2r[2] + ) = + this->mem->psi[e][0]( + this->ijk[0].first()-1, + this->ijk[1], + this->ijk[2] + ); break; default: assert(false); break; } +// std::cerr << "this->mem->psi_ref[e_ref](this->mem->grid_size_ref[0].first() - this->mem->n_ref / 2 - this->mem->n_ref , this->ijk_r2r[1], this->ijk_r2r[2])" << this->mem->psi_ref[e_ref]( +// this->mem->grid_size_ref[0].first() - this->mem->n_ref / 2 - this->mem->n_ref , +// this->ijk_r2r[1], +// this->ijk_r2r[2] +// ) << std::endl; +// +// std::cerr << "this->mem->psi_ref[e_ref](this->mem->grid_size_ref[0].first() - this->mem->n_ref / 2 , this->ijk_r2r[1], this->ijk_r2r[2])" << this->mem->psi_ref[e_ref]( +// this->mem->grid_size_ref[0].first() - this->mem->n_ref / 2 , +// this->ijk_r2r[1], +// this->ijk_r2r[2] +// ) << std::endl; +// +// std::cerr << "this->mem->psi_ref[e_ref](this->mem->grid_size_ref[0].last() + this->mem->n_ref / 2 , this->ijk_r2r[1], this->ijk_r2r[2])" << this->mem->psi_ref[e_ref]( +// this->mem->grid_size_ref[0].last() + this->mem->n_ref / 2 , +// this->ijk_r2r[1], +// this->ijk_r2r[2] +// ) << std::endl; +// std::cerr << "this->mem->psi_ref[e_ref](this->mem->grid_size_ref[0].last() + this->mem->n_ref / 2 + this->mem->n_ref , this->ijk_r2r[1], this->ijk_r2r[2])" << this->mem->psi_ref[e_ref]( +// this->mem->grid_size_ref[0].last() + this->mem->n_ref / 2 + this->mem->n_ref , +// this->ijk_r2r[1], +// this->ijk_r2r[2] +// ) << std::endl; } void refinement_ranges(const int iter, const int halo_size) @@ -97,7 +134,7 @@ namespace libmpdataxx if(iter==0) { mid_ijk_r2r_0 = this->rng_midpoints(this->ijk_r2r[0], this->mem->distmem.rank(), this->mem->distmem.size()); - mid_ijk_r2r_1 = this->rng_midpoints(this->ijk_r2r[1], this->rank, this->mem->size); + mid_ijk_r2r_1 = this->rng_midpoints_in_out(this->ijk_r2r[1], this->rank, this->mem->size); // different because we dont want overlapping ranges in y mid_ijk_r2r_2 = this->rng_midpoints(this->ijk_r2r[2]); ijk_r2r_1_h = this->ijk_r2r[1]; @@ -105,7 +142,11 @@ namespace libmpdataxx } else { - mid_ijk_r2r_0 = this->rng_midpoints_out(mid_ijk_r2r_0); + if(iter==1) + mid_ijk_r2r_0 = this->rng_midpoints_in(mid_ijk_r2r_0, this->mem->distmem.rank(), this->mem->distmem.size()); + else + mid_ijk_r2r_0 = this->rng_midpoints_out(mid_ijk_r2r_0); + mid_ijk_r2r_1 = this->rng_midpoints_out(mid_ijk_r2r_1); mid_ijk_r2r_2 = this->rng_midpoints_out(mid_ijk_r2r_2); @@ -118,32 +159,42 @@ namespace libmpdataxx assert(stride % 2 == 0); hstride = stride / 2; - ijk_r2r_0_h_with_halo = this->mem->distmem.rank() < this->mem->distmem.size() - 1 ? - rng_t(this->ijk_r2r[0].first(), this->ijk_r2r[0].last() + halo_size * this->mem->n_ref, hstride) : - rng_t(this->ijk_r2r[0].first(), this->ijk_r2r[0].last(), hstride); + ijk_r2r_0_h_with_halo = rng_t( + this->mem->distmem.rank() == 0 ? this->ijk_r2r[0].first() : this->ijk_r2r[0].first() - halo_size * this->mem->n_ref, + this->mem->distmem.rank() == this->mem->distmem.size() - 1 ? this->ijk_r2r[0].last() : this->ijk_r2r[0].last() + halo_size * this->mem->n_ref, + hstride + ); } + // TODO: stretching parameters at the overlaps of the reconstructed and resolved arrays (ijk_r2r) are not used, do not generate them void generate_stretching_parameters(const int rng_seed = 44) { std::mt19937 gen(rng_seed); std::uniform_real_distribution<> dis(-1, 1); // [-1,1), but whatever auto rand = std::bind(dis, gen); - std::generate(this->d_j(this->ijk_ref).begin(), this->d_j(this->ijk_ref).end(), rand); - this->d_j(this->ijk_ref) = formulae::fractal::d_of_CDF_fctr{}(this->d_j(this->ijk_ref)); + std::generate(this->d_j(this->ijk_ref_with_halo).begin(), this->d_j(this->ijk_ref_with_halo).end(), rand); + this->d_j(this->ijk_ref_with_halo) = formulae::fractal::d_of_CDF_fctr{}(this->d_j(this->ijk_ref_with_halo)); } protected: - void interpolate_refinee(const int _e = 0) + void interpolate_refinee(const int e = 0) { - assert(opts::isset(ct_params_t::fractal_recon, opts::bit(_e))); +return; + + assert(opts::isset(ct_params_t::fractal_recon, opts::bit(e))); const int halo_size = 1; - const int e = this->ix_r2r.at(_e); + const int e_ref = this->ix_r2r.at(e); + + // TEMPORARY + this->mem->psi_ref[e_ref] = -1000; + this->mem->barrier(); + fill_refinee_distmem_halos(e, halo_size); // fill refined array at position where it overlaps with the resolved array - this->mem->refinee(e)(this->ijk_r2r) = this->mem->advectee(e)(this->ijk); + this->mem->refinee(e_ref)(this->ijk_r2r) = this->mem->advectee(e)(this->ijk); generate_stretching_parameters(); @@ -151,38 +202,120 @@ namespace libmpdataxx { refinement_ranges(i, halo_size); - formulae::fractal::intrp<0, real_t>(this->mem->psi_ref[e], mid_ijk_r2r_0, ijk_r2r_1_h, ijk_r2r_2_h, hstride); + formulae::fractal::intrp<0, real_t>(this->mem->psi_ref[e_ref], mid_ijk_r2r_0, ijk_r2r_1_h, ijk_r2r_2_h, hstride); this->mem->barrier(); - formulae::fractal::intrp<1, real_t>(this->mem->psi_ref[e], mid_ijk_r2r_1, ijk_r2r_2_h, ijk_r2r_0_h_with_halo, hstride); - formulae::fractal::intrp<2, real_t>(this->mem->psi_ref[e], mid_ijk_r2r_2, ijk_r2r_0_h_with_halo, this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); + formulae::fractal::intrp<1, real_t>(this->mem->psi_ref[e_ref], mid_ijk_r2r_1, ijk_r2r_2_h, ijk_r2r_0_h_with_halo, hstride); + formulae::fractal::intrp<2, real_t>(this->mem->psi_ref[e_ref], mid_ijk_r2r_2, ijk_r2r_0_h_with_halo, this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); } this->mem->barrier(); } - void reconstruct_refinee(const int _e = 0) + void reconstruct_refinee(const int e = 0) { - assert(opts::isset(ct_params_t::fractal_recon, opts::bit(_e))); + assert(opts::isset(ct_params_t::fractal_recon, opts::bit(e))); const int halo_size = 2; - const int e = this->ix_r2r.at(_e); + const int e_ref = this->ix_r2r.at(e); + + // TEMPORARY + this->mem->psi_ref[e_ref] = -1000; + this->mem->barrier(); + + //std::cerr << "mem->grid_size_ref[0]: " << this->mem->grid_size_ref[0] << std::endl; fill_refinee_distmem_halos(e, halo_size); // fill refined array at position where it overlaps with the resolved array - this->mem->refinee(e)(this->ijk_r2r) = this->mem->advectee(e)(this->ijk); + this->mem->refinee(e_ref)(this->ijk_r2r) = this->mem->advectee(e)(this->ijk); generate_stretching_parameters(); for(int i=0; in_fra_iter; ++i) { +// if(i>0) continue; refinement_ranges(i, halo_size); - // if MPI domain starts with a point in the middle of a triple - we need to start calculating from a point to the left (in halo) - const int offset = (i==0 && this->mem->grid_size_ref[0].first() > 0 && (this->mem->grid_size_ref[0].first() / stride) % 2 != 0) ? stride : 0; + formulae::fractal::rcnstrct<0, real_t>(this->mem->psi_ref[e_ref], this->rng_dbl_stride(mid_ijk_r2r_0), ijk_r2r_1_h, ijk_r2r_2_h, hstride, this->c_j, this->d_j, this->f_j); +// +// std::cerr << "this->mem->psi_ref[e_ref](this->mem->grid_size_ref[0].first() - this->mem->n_ref / 2 - this->mem->n_ref , this->ijk_ref[1], this->ijk_ref[2])" << this->mem->psi_ref[e_ref]( +// this->mem->grid_size_ref[0].first() - this->mem->n_ref / 2 - this->mem->n_ref , +// this->ijk_ref[1], +// this->ijk_ref[2] +// ) << std::endl; +// +// std::cerr << "this->mem->psi_ref[e_ref](this->mem->grid_size_ref[0].first() - this->mem->n_ref / 2 , this->ijk_ref[1], this->ijk_ref[2])" << this->mem->psi_ref[e_ref]( +// this->mem->grid_size_ref[0].first() - this->mem->n_ref / 2 , +// this->ijk_ref[1], +// this->ijk_ref[2] +// ) << std::endl; +// +// std::cerr << "this->mem->psi_ref[e_ref](this->mem->grid_size_ref[0].last() + this->mem->n_ref / 2 , this->ijk_ref[1], this->ijk_ref[2])" << this->mem->psi_ref[e_ref]( +// this->mem->grid_size_ref[0].last() + this->mem->n_ref / 2 , +// this->ijk_ref[1], +// this->ijk_ref[2] +// ) << std::endl; +// std::cerr << "this->mem->psi_ref[e_ref](this->mem->grid_size_ref[0].last() + this->mem->n_ref / 2 + this->mem->n_ref , this->ijk_ref[1], this->ijk_ref[2])" << this->mem->psi_ref[e_ref]( +// this->mem->grid_size_ref[0].last() + this->mem->n_ref / 2 + this->mem->n_ref , +// this->ijk_ref[1], +// this->ijk_ref[2] +// ) << std::endl; - formulae::fractal::rcnstrct<0, real_t>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_0, offset), ijk_r2r_1_h, ijk_r2r_2_h, hstride, this->c_j, this->d_j, this->f_j); this->mem->barrier(); - formulae::fractal::rcnstrct<1, real_t>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_1) , ijk_r2r_2_h, ijk_r2r_0_h_with_halo, hstride, this->c_j, this->d_j, this->f_j); - formulae::fractal::rcnstrct<2, real_t>(this->mem->psi_ref[e], this->rng_dbl_stride(mid_ijk_r2r_2) , ijk_r2r_0_h_with_halo, this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride, this->c_j, this->d_j, this->f_j); + formulae::fractal::rcnstrct<1, real_t>(this->mem->psi_ref[e_ref], this->rng_dbl_stride(mid_ijk_r2r_1) , ijk_r2r_2_h, ijk_r2r_0_h_with_halo, hstride, this->c_j, this->d_j, this->f_j); + + //std::cerr << "this->c_j: " << this->c_j << std::endl; + // formulae::fractal::rcnstrct<1, real_t>(this->mem->psi_ref[e_ref], + // rng_t(this->rng_dbl_stride(mid_ijk_r2r_1).first(), this->rng_dbl_stride(mid_ijk_r2r_1).first()), + // rng_t(ijk_r2r_2_h.first(), ijk_r2r_2_h.first()), + // rng_t(ijk_r2r_0_h_with_halo.first(), ijk_r2r_0_h_with_halo.first()) , hstride, this->c_j, this->d_j, this->f_j); +// +// std::cerr << "this->mem->psi_ref[e_ref](this->mem->grid_size_ref[0].first() - this->mem->n_ref / 2 - this->mem->n_ref , this->ijk_ref[1], this->ijk_ref[2])" << this->mem->psi_ref[e_ref]( +// this->mem->grid_size_ref[0].first() - this->mem->n_ref / 2 - this->mem->n_ref , +// this->ijk_ref[1], +// this->ijk_ref[2] +// ) << std::endl; +// +// std::cerr << "this->mem->psi_ref[e_ref](this->mem->grid_size_ref[0].first() - this->mem->n_ref / 2 , this->ijk_ref[1], this->ijk_ref[2])" << this->mem->psi_ref[e_ref]( +// this->mem->grid_size_ref[0].first() - this->mem->n_ref / 2 , +// this->ijk_ref[1], +// this->ijk_ref[2] +// ) << std::endl; +// +// std::cerr << "this->mem->psi_ref[e_ref](this->mem->grid_size_ref[0].last() + this->mem->n_ref / 2 , this->ijk_ref[1], this->ijk_ref[2])" << this->mem->psi_ref[e_ref]( +// this->mem->grid_size_ref[0].last() + this->mem->n_ref / 2 , +// this->ijk_ref[1], +// this->ijk_ref[2] +// ) << std::endl; +// std::cerr << "this->mem->psi_ref[e_ref](this->mem->grid_size_ref[0].last() + this->mem->n_ref / 2 + this->mem->n_ref , this->ijk_ref[1], this->ijk_ref[2])" << this->mem->psi_ref[e_ref]( +// this->mem->grid_size_ref[0].last() + this->mem->n_ref / 2 + this->mem->n_ref , +// this->ijk_ref[1], +// this->ijk_ref[2] +// ) << std::endl; + + formulae::fractal::rcnstrct<2, real_t>(this->mem->psi_ref[e_ref], this->rng_dbl_stride(mid_ijk_r2r_2) , ijk_r2r_0_h_with_halo, this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride, this->c_j, this->d_j, this->f_j); + +// std::cerr << "this->mem->psi_ref[e_ref](this->mem->grid_size_ref[0].first() - this->mem->n_ref / 2 - this->mem->n_ref , this->ijk_ref[1], this->ijk_ref[2])" << this->mem->psi_ref[e_ref]( +// this->mem->grid_size_ref[0].first() - this->mem->n_ref / 2 - this->mem->n_ref , +// this->ijk_ref[1], +// this->ijk_ref[2] +// ) << std::endl; +// +// std::cerr << "this->mem->psi_ref[e_ref](this->mem->grid_size_ref[0].first() - this->mem->n_ref / 2 , this->ijk_ref[1], this->ijk_ref[2])" << this->mem->psi_ref[e_ref]( +// this->mem->grid_size_ref[0].first() - this->mem->n_ref / 2 , +// this->ijk_ref[1], +// this->ijk_ref[2] +// ) << std::endl; +// +// std::cerr << "this->mem->psi_ref[e_ref](this->mem->grid_size_ref[0].last() + this->mem->n_ref / 2 , this->ijk_ref[1], this->ijk_ref[2])" << this->mem->psi_ref[e_ref]( +// this->mem->grid_size_ref[0].last() + this->mem->n_ref / 2 , +// this->ijk_ref[1], +// this->ijk_ref[2] +// ) << std::endl; +// std::cerr << "this->mem->psi_ref[e_ref](this->mem->grid_size_ref[0].last() + this->mem->n_ref / 2 + this->mem->n_ref , this->ijk_ref[1], this->ijk_ref[2])" << this->mem->psi_ref[e_ref]( +// this->mem->grid_size_ref[0].last() + this->mem->n_ref / 2 + this->mem->n_ref , +// this->ijk_ref[1], +// this->ijk_ref[2] +// ) << std::endl; + } this->mem->barrier(); } @@ -203,7 +336,7 @@ namespace libmpdataxx for (int n = 0; n < n_arr; ++n) mem->tmp[__file__].back().push_back(mem->old(new typename parent_t::arr_t( - mem->grid_size_ref[0]^(mem->n_ref+1), // reconstruction based on 3 points, we need up to 2 resolved points outside of the domain + parent_t::rng_ref_distmem_halo(mem->grid_size_ref[0], mem->n_ref, mem->distmem.rank(), mem->distmem.size()), mem->grid_size_ref[1], srfc ? rng_t(0, 0) : mem->grid_size_ref[2], arr3D_storage diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp index f8fd0fd3..248e3a97 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp @@ -56,21 +56,43 @@ namespace libmpdataxx // herlper ranges // TODO: make these const! - idx_t ijk_ref; // range of refinee handled by given solver + idx_t ijk_ref, // range of refinee handled by given solver + ijk_ref_with_halo; // same but with a halo in x direction between MPI processes const idxs_t ijk_r2r; // resolved to refined; refined scalars at the same position as resolved scalars // range modifying methods used in grid refinement + // TODO: unify static rng_t rng_midpoints(const rng_t &rng, const int rank = 0, const int size = 1) + { + assert(rng.stride() % 2 == 0); + if(rng.last() - rng.first() < rng.stride()) return rng; + return rng_t( + rank == 0 ? rng.first() + rng.stride() / 2 : rng.first() - rng.stride() / 2, + rank == size - 1 ? rng.last() - rng.stride() / 2 : rng.last() + rng.stride() / 2, + rng.stride()); + } + + static rng_t rng_midpoints_in_out(const rng_t &rng, const int rank = 0, const int size = 1) { assert(rng.stride() % 2 == 0); if(rng.last() - rng.first() < rng.stride()) return rng; return rng_t( rng.first() + rng.stride() / 2, - rank == size - 1 ? rng.last() - rng.stride() / 2 : rng.last() + rng.stride() / 2, + rank == size - 1 ? rng.last() - rng.stride() / 2 : rng.last() + rng.stride() / 2, rng.stride()); } + static rng_t rng_midpoints_in(const rng_t &rng, const int rank = 0, const int size = 1) + { + assert(rng.stride() % 4 == 0); + if(rng.last() - rng.first() < rng.stride()) return rng; + else return rng_t( + rank == 0 ? rng.first() - rng.stride() / 4 : rng.first() + rng.stride() / 4, + rank == size - 1 ? rng.last() + rng.stride() / 4 : rng.last() - rng.stride() / 4, + rng.stride() / 2); + } + static rng_t rng_midpoints_out(const rng_t &rng) { assert(rng.stride() % 4 == 0); @@ -95,15 +117,19 @@ namespace libmpdataxx // returned range points to the first from the pair of reconstructed points // to avoid boundary conditions, in y and z directions it is assumed that the number of points is 3+i*2, i=0,1,2,... (this check is somewhere else) // however in the x direction there can be any number of points, because the domain is divided between mpi processes... - static rng_t rng_dbl_stride(const rng_t &rng, const int offset = 0) + static rng_t rng_dbl_stride(const rng_t &rng) { assert(rng.last() != rng.first()); // we need at least 2 midpoints + assert(rng.stride() % 2 == 0); -// return rng_t(rng.first() - offset, rng.last(), 2*rng.stride()); - if( ((rng.last() - rng.first() - offset) / rng.stride() + 1) % 2 == 0) // even number of midpoints; y and z directions (and sometimes x) - return rng_t(rng.first() - offset, rng.last() - rng.stride(), 2*rng.stride()); + // if domain starts with a point in the middle of a triple - we need to start calculating from a point to the left + const int offset = ( (rng.first() - rng.stride() / 2) / rng.stride() ) % 2 == 0 ? 0 : - rng.stride(); + const auto first = rng.first() + offset; + + if( ((rng.last() - first) / rng.stride() + 1) % 2 == 0) // even number of midpoints; y and z directions (and sometimes x) + return rng_t(first, rng.last() - rng.stride(), 2*rng.stride()); else // odd number of midpoints - return rng_t(rng.first() - offset, rng.last(), 2*rng.stride()); // rely on the halo along x direction + return rng_t(first, rng.last(), 2*rng.stride()); // rely on the halo along x direction } static rng_t rng_merge(const rng_t &rng1, const rng_t &rng2) @@ -115,6 +141,15 @@ namespace libmpdataxx rng1.stride() / 2); } + // reconstruction based on 3 points, we need up to 2 resolved points between MPI domains + static rng_t rng_ref_distmem_halo(const rng_t &rng, const int &n_ref, const int rank = 0, const int size = 1) + { + return rng_t( + rank == 0 ? rng.first() : rng.first() - (n_ref + n_ref/2), + rank < size - 1 ? rng.last() + (n_ref + n_ref/2) : rng.last(), + rng.stride()); + } + public: struct rt_params_t : parent_t::rt_params_t @@ -156,6 +191,10 @@ namespace libmpdataxx ijk_ref.ubound(d) = this->mem->grid_size_ref[d].last(); } } + ijk_ref_with_halo = ijk_ref; + const auto ijk_ref_with_halo_rng_0 = rng_ref_distmem_halo(ijk_ref[0], this->mem->n_ref, this->mem->distmem.rank(), this->mem->distmem.size()); + ijk_ref_with_halo.lbound(0) = ijk_ref_with_halo_rng_0.first(); + ijk_ref_with_halo.ubound(0) = ijk_ref_with_halo_rng_0.last(); } }; } // namespace detail diff --git a/tests/sandbox/pbl/pbl.hpp b/tests/sandbox/pbl/pbl.hpp index f12a5d90..7ff0e255 100644 --- a/tests/sandbox/pbl/pbl.hpp +++ b/tests/sandbox/pbl/pbl.hpp @@ -59,7 +59,8 @@ class pbl : public libmpdataxx::output::hdf5_xdmftimestep % static_cast(this->outfreq) == 0) { - this->reconstruct_refinee(ix::w); + //this->reconstruct_refinee(ix::w); + this->reconstruct_refinee(ix::tht); if (this->rank == 0) std::cout << this->timestep << std::endl; @@ -71,7 +72,17 @@ class pbl : public libmpdataxx::output::hdf5_xdmfrecord_aux_dsc("tke", this->tke); } this->record_aux_dsc("p", this->Phi); - this->record_aux_dsc_refined("w reconstructed", this->mem->refinee(this->ix_r2r.at(ix::w))); + //this->record_aux_dsc_refined("w reconstructed", this->mem->refinee(this->ix_r2r.at(ix::w))); + this->record_aux_dsc_refined("tht reconstructed", this->mem->refinee(this->ix_r2r.at(ix::tht))); + } + this->mem->barrier(); + + this->interpolate_refinee(ix::tht); + + this->mem->barrier(); + if (this->rank == 0) + { + this->record_aux_dsc_refined("tht interpolated", this->mem->refinee(this->ix_r2r.at(ix::tht))); } this->mem->barrier(); } diff --git a/tests/sandbox/pbl/pbl_iles_short.cpp b/tests/sandbox/pbl/pbl_iles_short.cpp index 96bd847c..65e5fbaa 100644 --- a/tests/sandbox/pbl/pbl_iles_short.cpp +++ b/tests/sandbox/pbl/pbl_iles_short.cpp @@ -9,5 +9,5 @@ int main() { - test("out_pbl_iles_short", 33, 601); + test("out_pbl_iles_short", 33, 2); } diff --git a/tests/sandbox/pbl/pbl_test_def.hpp b/tests/sandbox/pbl/pbl_test_def.hpp index 5cc2834b..1cca03b1 100644 --- a/tests/sandbox/pbl/pbl_test_def.hpp +++ b/tests/sandbox/pbl/pbl_test_def.hpp @@ -56,7 +56,8 @@ void test(const std::string &dirname, const int np, const int nt) u, v, w, tht, vip_i=u, vip_j=v, vip_k=w, vip_den=-1 }; }; - enum { fractal_recon = libmpdataxx::opts::bit(ix::w) }; + //enum { fractal_recon = libmpdataxx::opts::bit(ix::w) }; + enum { fractal_recon = libmpdataxx::opts::bit(ix::tht) }; }; using ix = typename ct_params_t::ix; @@ -83,7 +84,7 @@ void test(const std::string &dirname, const int np, const int nt) double mixed_length = 500; double st = 1e-4 / p.g; - p.outfreq = 150; + p.outfreq = 1; p.outwindow = 1; p.outvars = { {ix::u, { "u", "m/s"}}, From 4275cea21543217d09605b1545766dcf2d91055a Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 9 May 2022 14:30:33 +0200 Subject: [PATCH 053/129] cleanup --- .../formulae/fractal_reconstruction.hpp | 33 +++-- .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 115 +----------------- .../mpdata_rhs_vip_prs_sgs_fra_common.hpp | 2 - tests/sandbox/pbl/pbl_iles_short.cpp | 2 +- tests/sandbox/pbl/pbl_test_def.hpp | 2 +- 5 files changed, 22 insertions(+), 132 deletions(-) diff --git a/libmpdata++/formulae/fractal_reconstruction.hpp b/libmpdata++/formulae/fractal_reconstruction.hpp index fdae5c69..15fbacc6 100644 --- a/libmpdata++/formulae/fractal_reconstruction.hpp +++ b/libmpdata++/formulae/fractal_reconstruction.hpp @@ -53,13 +53,13 @@ namespace libmpdataxx using idxperm::pis; // debug output - std::cerr << "ranges in interpolation" << std::endl; - if(d==0) - std::cerr << "range<" << d << ">: " << i << " " << j << " " << k << std::endl; - if(d==1) - std::cerr << "range<" << d << ">: " << k << " " << i << " " << j << std::endl; - if(d==2) - std::cerr << "range<" << d << ">: " << j << " " << k << " " << i << std::endl; + // std::cerr << "ranges in interpolation" << std::endl; + // if(d==0) + // std::cerr << "range<" << d << ">: " << i << " " << j << " " << k << std::endl; + // if(d==1) + // std::cerr << "range<" << d << ">: " << k << " " << i << " " << j << std::endl; + // if(d==2) + // std::cerr << "range<" << d << ">: " << j << " " << k << " " << i << std::endl; arr(pis(i, j, k)) = real_t(.5) * ( arr(pis(i - dist, j, k)) + @@ -83,13 +83,13 @@ namespace libmpdataxx ) { // debug output - std::cerr << "ranges in rcnstrct" << std::endl; - if(d==0) - std::cerr << "range<" << d << ">: " << i << " " << j << " " << k << std::endl; - if(d==1) - std::cerr << "range<" << d << ">: " << k << " " << i << " " << j << std::endl; - if(d==2) - std::cerr << "range<" << d << ">: " << j << " " << k << " " << i << std::endl; +// std::cerr << "ranges in rcnstrct" << std::endl; +// if(d==0) +// std::cerr << "range<" << d << ">: " << i << " " << j << " " << k << std::endl; +// if(d==1) +// std::cerr << "range<" << d << ">: " << k << " " << i << " " << j << std::endl; +// if(d==2) +// std::cerr << "range<" << d << ">: " << j << " " << k << " " << i << std::endl; using idxperm::pis; @@ -124,11 +124,6 @@ namespace libmpdataxx // DEBUG: do interpolation, useful for testing if ranges are correct // arr(pis(i, j, k)) = real_t(.5) * (u_0 + u_1); // arr(pis(i_next, j, k)) = real_t(.5) * (u_1 + u_2); -// -// std::cerr << "new u_j=1:" << arr(pis(i, j, k)); -// std::cerr << "new u_j=2:" << arr(pis(i_next, j, k)); -// return; - // end of the interpolation test // alternative solution following Scotti and Meneveau Physica D 1999 diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 4857e484..479ff977 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -104,28 +104,6 @@ namespace libmpdataxx assert(false); break; } -// std::cerr << "this->mem->psi_ref[e_ref](this->mem->grid_size_ref[0].first() - this->mem->n_ref / 2 - this->mem->n_ref , this->ijk_r2r[1], this->ijk_r2r[2])" << this->mem->psi_ref[e_ref]( -// this->mem->grid_size_ref[0].first() - this->mem->n_ref / 2 - this->mem->n_ref , -// this->ijk_r2r[1], -// this->ijk_r2r[2] -// ) << std::endl; -// -// std::cerr << "this->mem->psi_ref[e_ref](this->mem->grid_size_ref[0].first() - this->mem->n_ref / 2 , this->ijk_r2r[1], this->ijk_r2r[2])" << this->mem->psi_ref[e_ref]( -// this->mem->grid_size_ref[0].first() - this->mem->n_ref / 2 , -// this->ijk_r2r[1], -// this->ijk_r2r[2] -// ) << std::endl; -// -// std::cerr << "this->mem->psi_ref[e_ref](this->mem->grid_size_ref[0].last() + this->mem->n_ref / 2 , this->ijk_r2r[1], this->ijk_r2r[2])" << this->mem->psi_ref[e_ref]( -// this->mem->grid_size_ref[0].last() + this->mem->n_ref / 2 , -// this->ijk_r2r[1], -// this->ijk_r2r[2] -// ) << std::endl; -// std::cerr << "this->mem->psi_ref[e_ref](this->mem->grid_size_ref[0].last() + this->mem->n_ref / 2 + this->mem->n_ref , this->ijk_r2r[1], this->ijk_r2r[2])" << this->mem->psi_ref[e_ref]( -// this->mem->grid_size_ref[0].last() + this->mem->n_ref / 2 + this->mem->n_ref , -// this->ijk_r2r[1], -// this->ijk_r2r[2] -// ) << std::endl; } void refinement_ranges(const int iter, const int halo_size) @@ -180,15 +158,13 @@ namespace libmpdataxx void interpolate_refinee(const int e = 0) { -return; - assert(opts::isset(ct_params_t::fractal_recon, opts::bit(e))); const int halo_size = 1; const int e_ref = this->ix_r2r.at(e); - // TEMPORARY - this->mem->psi_ref[e_ref] = -1000; - this->mem->barrier(); + //// TEMPORARY + //this->mem->psi_ref[e_ref] = -1000; + //this->mem->barrier(); fill_refinee_distmem_halos(e, halo_size); @@ -216,9 +192,9 @@ return; const int halo_size = 2; const int e_ref = this->ix_r2r.at(e); - // TEMPORARY - this->mem->psi_ref[e_ref] = -1000; - this->mem->barrier(); + //// TEMPORARY + //this->mem->psi_ref[e_ref] = -1000; + //this->mem->barrier(); //std::cerr << "mem->grid_size_ref[0]: " << this->mem->grid_size_ref[0] << std::endl; @@ -231,91 +207,12 @@ return; for(int i=0; in_fra_iter; ++i) { -// if(i>0) continue; refinement_ranges(i, halo_size); formulae::fractal::rcnstrct<0, real_t>(this->mem->psi_ref[e_ref], this->rng_dbl_stride(mid_ijk_r2r_0), ijk_r2r_1_h, ijk_r2r_2_h, hstride, this->c_j, this->d_j, this->f_j); -// -// std::cerr << "this->mem->psi_ref[e_ref](this->mem->grid_size_ref[0].first() - this->mem->n_ref / 2 - this->mem->n_ref , this->ijk_ref[1], this->ijk_ref[2])" << this->mem->psi_ref[e_ref]( -// this->mem->grid_size_ref[0].first() - this->mem->n_ref / 2 - this->mem->n_ref , -// this->ijk_ref[1], -// this->ijk_ref[2] -// ) << std::endl; -// -// std::cerr << "this->mem->psi_ref[e_ref](this->mem->grid_size_ref[0].first() - this->mem->n_ref / 2 , this->ijk_ref[1], this->ijk_ref[2])" << this->mem->psi_ref[e_ref]( -// this->mem->grid_size_ref[0].first() - this->mem->n_ref / 2 , -// this->ijk_ref[1], -// this->ijk_ref[2] -// ) << std::endl; -// -// std::cerr << "this->mem->psi_ref[e_ref](this->mem->grid_size_ref[0].last() + this->mem->n_ref / 2 , this->ijk_ref[1], this->ijk_ref[2])" << this->mem->psi_ref[e_ref]( -// this->mem->grid_size_ref[0].last() + this->mem->n_ref / 2 , -// this->ijk_ref[1], -// this->ijk_ref[2] -// ) << std::endl; -// std::cerr << "this->mem->psi_ref[e_ref](this->mem->grid_size_ref[0].last() + this->mem->n_ref / 2 + this->mem->n_ref , this->ijk_ref[1], this->ijk_ref[2])" << this->mem->psi_ref[e_ref]( -// this->mem->grid_size_ref[0].last() + this->mem->n_ref / 2 + this->mem->n_ref , -// this->ijk_ref[1], -// this->ijk_ref[2] -// ) << std::endl; - this->mem->barrier(); formulae::fractal::rcnstrct<1, real_t>(this->mem->psi_ref[e_ref], this->rng_dbl_stride(mid_ijk_r2r_1) , ijk_r2r_2_h, ijk_r2r_0_h_with_halo, hstride, this->c_j, this->d_j, this->f_j); - - //std::cerr << "this->c_j: " << this->c_j << std::endl; - // formulae::fractal::rcnstrct<1, real_t>(this->mem->psi_ref[e_ref], - // rng_t(this->rng_dbl_stride(mid_ijk_r2r_1).first(), this->rng_dbl_stride(mid_ijk_r2r_1).first()), - // rng_t(ijk_r2r_2_h.first(), ijk_r2r_2_h.first()), - // rng_t(ijk_r2r_0_h_with_halo.first(), ijk_r2r_0_h_with_halo.first()) , hstride, this->c_j, this->d_j, this->f_j); -// -// std::cerr << "this->mem->psi_ref[e_ref](this->mem->grid_size_ref[0].first() - this->mem->n_ref / 2 - this->mem->n_ref , this->ijk_ref[1], this->ijk_ref[2])" << this->mem->psi_ref[e_ref]( -// this->mem->grid_size_ref[0].first() - this->mem->n_ref / 2 - this->mem->n_ref , -// this->ijk_ref[1], -// this->ijk_ref[2] -// ) << std::endl; -// -// std::cerr << "this->mem->psi_ref[e_ref](this->mem->grid_size_ref[0].first() - this->mem->n_ref / 2 , this->ijk_ref[1], this->ijk_ref[2])" << this->mem->psi_ref[e_ref]( -// this->mem->grid_size_ref[0].first() - this->mem->n_ref / 2 , -// this->ijk_ref[1], -// this->ijk_ref[2] -// ) << std::endl; -// -// std::cerr << "this->mem->psi_ref[e_ref](this->mem->grid_size_ref[0].last() + this->mem->n_ref / 2 , this->ijk_ref[1], this->ijk_ref[2])" << this->mem->psi_ref[e_ref]( -// this->mem->grid_size_ref[0].last() + this->mem->n_ref / 2 , -// this->ijk_ref[1], -// this->ijk_ref[2] -// ) << std::endl; -// std::cerr << "this->mem->psi_ref[e_ref](this->mem->grid_size_ref[0].last() + this->mem->n_ref / 2 + this->mem->n_ref , this->ijk_ref[1], this->ijk_ref[2])" << this->mem->psi_ref[e_ref]( -// this->mem->grid_size_ref[0].last() + this->mem->n_ref / 2 + this->mem->n_ref , -// this->ijk_ref[1], -// this->ijk_ref[2] -// ) << std::endl; - formulae::fractal::rcnstrct<2, real_t>(this->mem->psi_ref[e_ref], this->rng_dbl_stride(mid_ijk_r2r_2) , ijk_r2r_0_h_with_halo, this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride, this->c_j, this->d_j, this->f_j); - -// std::cerr << "this->mem->psi_ref[e_ref](this->mem->grid_size_ref[0].first() - this->mem->n_ref / 2 - this->mem->n_ref , this->ijk_ref[1], this->ijk_ref[2])" << this->mem->psi_ref[e_ref]( -// this->mem->grid_size_ref[0].first() - this->mem->n_ref / 2 - this->mem->n_ref , -// this->ijk_ref[1], -// this->ijk_ref[2] -// ) << std::endl; -// -// std::cerr << "this->mem->psi_ref[e_ref](this->mem->grid_size_ref[0].first() - this->mem->n_ref / 2 , this->ijk_ref[1], this->ijk_ref[2])" << this->mem->psi_ref[e_ref]( -// this->mem->grid_size_ref[0].first() - this->mem->n_ref / 2 , -// this->ijk_ref[1], -// this->ijk_ref[2] -// ) << std::endl; -// -// std::cerr << "this->mem->psi_ref[e_ref](this->mem->grid_size_ref[0].last() + this->mem->n_ref / 2 , this->ijk_ref[1], this->ijk_ref[2])" << this->mem->psi_ref[e_ref]( -// this->mem->grid_size_ref[0].last() + this->mem->n_ref / 2 , -// this->ijk_ref[1], -// this->ijk_ref[2] -// ) << std::endl; -// std::cerr << "this->mem->psi_ref[e_ref](this->mem->grid_size_ref[0].last() + this->mem->n_ref / 2 + this->mem->n_ref , this->ijk_ref[1], this->ijk_ref[2])" << this->mem->psi_ref[e_ref]( -// this->mem->grid_size_ref[0].last() + this->mem->n_ref / 2 + this->mem->n_ref , -// this->ijk_ref[1], -// this->ijk_ref[2] -// ) << std::endl; - } this->mem->barrier(); } diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp index 248e3a97..0ab70d55 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp @@ -35,11 +35,9 @@ namespace libmpdataxx typename parent_t::arr_t c_j, d_j, f_j; // parameters used in fractal reconstruction, Akinlabi et al. 2019 -// const int n_fra; // number of fields with fractal reconstruction const int n_ref, // number of refinements; refined resolution is dx / n_ref n_fra_iter; // number of iterations of grid refinement - // refined arrays have no halos (do they need them? halos can be added by extending grid_size in alloc() in mpdata_rhs_vip_prs_sgs_fra.hpp by e.g. mem->n_ref/2) // const int halo_ref; // size of halos of refined scalar (and vector?) arrays const std::array ix_r2r; diff --git a/tests/sandbox/pbl/pbl_iles_short.cpp b/tests/sandbox/pbl/pbl_iles_short.cpp index 65e5fbaa..96bd847c 100644 --- a/tests/sandbox/pbl/pbl_iles_short.cpp +++ b/tests/sandbox/pbl/pbl_iles_short.cpp @@ -9,5 +9,5 @@ int main() { - test("out_pbl_iles_short", 33, 2); + test("out_pbl_iles_short", 33, 601); } diff --git a/tests/sandbox/pbl/pbl_test_def.hpp b/tests/sandbox/pbl/pbl_test_def.hpp index 1cca03b1..9409c4bb 100644 --- a/tests/sandbox/pbl/pbl_test_def.hpp +++ b/tests/sandbox/pbl/pbl_test_def.hpp @@ -84,7 +84,7 @@ void test(const std::string &dirname, const int np, const int nt) double mixed_length = 500; double st = 1e-4 / p.g; - p.outfreq = 1; + p.outfreq = 150; p.outwindow = 1; p.outvars = { {ix::u, { "u", "m/s"}}, From 079a75ef8c299ca67450a6ef59797113827504e2 Mon Sep 17 00:00:00 2001 From: pdziekan Date: Mon, 6 Jun 2022 14:45:53 +0200 Subject: [PATCH 054/129] comments --- libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 479ff977..2c3763ea 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -145,6 +145,7 @@ namespace libmpdataxx } // TODO: stretching parameters at the overlaps of the reconstructed and resolved arrays (ijk_r2r) are not used, do not generate them + // also not all parameters in the halo are needed (but some are!) void generate_stretching_parameters(const int rng_seed = 44) { std::mt19937 gen(rng_seed); From 36452f969db90d77664a09cae0c0024aa85eb65f Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Tue, 27 Sep 2022 15:17:38 +0200 Subject: [PATCH 055/129] include in fractal reconstruction --- libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp index 0ab70d55..efbb63b1 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp @@ -13,6 +13,7 @@ #include //#include //#include +#include namespace libmpdataxx { From 10a335db8b5a9056d1544e751c874337807c0b24 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Tue, 27 Sep 2022 15:38:07 +0200 Subject: [PATCH 056/129] stop on first compilation error in Debug mode --- libmpdata++-config.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmpdata++-config.cmake b/libmpdata++-config.cmake index bca0af5e..0415f1ec 100644 --- a/libmpdata++-config.cmake +++ b/libmpdata++-config.cmake @@ -32,7 +32,7 @@ set(libmpdataxx_INCLUDE_DIRS "${CMAKE_CURRENT_LIST_DIR}/../../include/") ############################################################################################ # debug mode compiler flags -set(libmpdataxx_CXX_FLAGS_DEBUG "${libmpdataxx_CXX_FLAGS_DEBUG} -std=c++17 -DBZ_DEBUG -g -Wno-enum-compare") #TODO: -Og if compiler supports it? +set(libmpdataxx_CXX_FLAGS_DEBUG "${libmpdataxx_CXX_FLAGS_DEBUG} -std=c++17 -DBZ_DEBUG -g -Wno-enum-compare -Wfatal-errors") #TODO: -Og if compiler supports it? ############################################################################################ From 60cd21f103dda11591f810872379fb82732e5dff Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Tue, 27 Sep 2022 17:14:59 +0200 Subject: [PATCH 057/129] sgs fra make typdefs public --- libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 2c3763ea..eb257f9f 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -28,6 +28,8 @@ namespace libmpdataxx typename std::enable_if_t > : public detail::mpdata_rhs_vip_prs_sgs_fra_common { + public: + using parent_t = detail::mpdata_rhs_vip_prs_sgs_fra_common; using parent_t::parent_t; // inheriting constructors using real_t = typename ct_params_t::real_t; From e5e4879852fb2e9e75c8678fa8aba20a97e49636 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Wed, 28 Sep 2022 11:17:20 +0200 Subject: [PATCH 058/129] hdf5 output: handle sgs_fra family tag --- libmpdata++/output/hdf5.hpp | 27 ++++++++++++++++--- .../mpdata_rhs_vip_prs_sgs_fra_common.hpp | 2 +- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/libmpdata++/output/hdf5.hpp b/libmpdata++/output/hdf5.hpp index c3656241..0f2cd2c7 100644 --- a/libmpdata++/output/hdf5.hpp +++ b/libmpdata++/output/hdf5.hpp @@ -687,13 +687,32 @@ namespace libmpdataxx } } + // as above but for solvers with fractal grid refinement + void record_params(const H5::H5File &hdfcp, typename solvers::mpdata_rhs_vip_prs_sgs_fra_family_tag) + { + record_params(hdfcp, typename std::conditional + (parent_t::ct_params_t_::sgs_scheme) == solvers::iles, + typename solvers::mpdata_rhs_vip_prs_family_tag, // iles + typename std::conditional + (parent_t::ct_params_t_::sgs_scheme) == solvers::smg, + typename solvers::mpdata_rhs_vip_prs_sgs_smg_family_tag, // smg + typename solvers::mpdata_rhs_vip_prs_sgs_dns_family_tag // dns + >::type + >::type{} + ); + hdfcp.createGroup("fractal"); + const auto &group = hdfcp.openGroup("fractal"); + { + const auto type = H5::PredType::NATIVE_INT; + group.createAttribute("n_fra_iter", type, H5::DataSpace(1, &one)).write(type, &this->n_fra_iter); + } + } + // as above but for the boussinesq solver void record_params(const H5::H5File &hdfcp, typename solvers::mpdata_boussinesq_family_tag) { - record_params(hdfcp, typename std::conditional - (parent_t::ct_params_t_::sgs_scheme) == solvers::iles, - typename solvers::mpdata_rhs_vip_prs_family_tag, - typename solvers::mpdata_rhs_vip_prs_sgs_smg_family_tag>::type{}); + record_params(hdfcp, typename solvers::mpdata_rhs_vip_prs_sgs_fra_family_tag{}); + hdfcp.createGroup("boussinesq"); const auto &group = hdfcp.openGroup("boussinesq"); { diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp index efbb63b1..a01780cf 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp @@ -153,7 +153,7 @@ namespace libmpdataxx struct rt_params_t : parent_t::rt_params_t { - unsigned int n_fra_iter = 1; // number of iterations of fractal reconstruction + int n_fra_iter = 1; // number of iterations of fractal reconstruction }; // ctor From 2c532ec18d9299dcbb27e0404373dece46bc1a21 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Wed, 28 Sep 2022 11:22:20 +0200 Subject: [PATCH 059/129] domain decmposition functions static --- libmpdata++/formulae/domain_decomposition.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libmpdata++/formulae/domain_decomposition.hpp b/libmpdata++/formulae/domain_decomposition.hpp index ccf47b3e..21594610 100644 --- a/libmpdata++/formulae/domain_decomposition.hpp +++ b/libmpdata++/formulae/domain_decomposition.hpp @@ -15,19 +15,19 @@ namespace libmpdataxx namespace detail { // helper methods to define subdomain ranges - int min(const int &span, const int &rank, const int &size) + static int min(const int &span, const int &rank, const int &size) { return rank * span / size; } - int max(const int &span, const int &rank, const int &size) + static int max(const int &span, const int &rank, const int &size) { return min(span, rank + 1, size) - 1; } }; // get part of 'span' assigned to 'rank' (out of 'size' ranks) - rng_t slab( + static rng_t slab( const rng_t &span, const int &rank = 0, const int &size = 1 From eee81352646cd71a708d54d37a523d1fda8efca7 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Wed, 28 Sep 2022 13:19:22 +0200 Subject: [PATCH 060/129] add hook_ante_record_all --- libmpdata++/output/detail/output_common.hpp | 24 ++++++++++++--------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/libmpdata++/output/detail/output_common.hpp b/libmpdata++/output/detail/output_common.hpp index 47920259..2ac3b6a5 100644 --- a/libmpdata++/output/detail/output_common.hpp +++ b/libmpdata++/output/detail/output_common.hpp @@ -38,6 +38,7 @@ namespace libmpdataxx virtual void record(const int var) {} virtual void start(const typename parent_t::advance_arg_t nt) {} + virtual void hook_pre_record_all() {} typename parent_t::arr_t out_data(const int var) { @@ -55,6 +56,7 @@ namespace libmpdataxx this->mem->barrier(); } + hook_pre_record_all(); if (this->rank == 0) { record_time = this->time; @@ -145,20 +147,22 @@ namespace libmpdataxx this->mem->barrier(); } - if (this->rank == 0) + if (this->var_dt && do_record_cnt == 1) { - //TODO: output of solver statistics every timesteps could probably go here - - if (this->var_dt && do_record_cnt == 1) - { + hook_pre_record_all(); + if (this->rank == 0) record_all(); - } - else if (!this->var_dt) + } + else if (!this->var_dt) + { + record_time = this->time; + for (int t = 0; t < outwindow; ++t) { - record_time = this->time; - for (int t = 0; t < outwindow; ++t) + if ((this->timestep - t) % static_cast(outfreq) == 0) { - if ((this->timestep - t) % static_cast(outfreq) == 0) record_all(); + hook_pre_record_all(); + if (this->rank == 0) + record_all(); } } } From 5015952c189a17ac41bef7da0c811664b2cd8400 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Wed, 28 Sep 2022 13:54:15 +0200 Subject: [PATCH 061/129] hook_pre_record_all -> hook_ante_record_all --- libmpdata++/output/detail/output_common.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libmpdata++/output/detail/output_common.hpp b/libmpdata++/output/detail/output_common.hpp index 2ac3b6a5..4ebcee19 100644 --- a/libmpdata++/output/detail/output_common.hpp +++ b/libmpdata++/output/detail/output_common.hpp @@ -38,7 +38,7 @@ namespace libmpdataxx virtual void record(const int var) {} virtual void start(const typename parent_t::advance_arg_t nt) {} - virtual void hook_pre_record_all() {} + virtual void hook_ante_record_all() {} typename parent_t::arr_t out_data(const int var) { @@ -56,7 +56,7 @@ namespace libmpdataxx this->mem->barrier(); } - hook_pre_record_all(); + hook_ante_record_all(); if (this->rank == 0) { record_time = this->time; @@ -149,7 +149,7 @@ namespace libmpdataxx if (this->var_dt && do_record_cnt == 1) { - hook_pre_record_all(); + hook_ante_record_all(); if (this->rank == 0) record_all(); } @@ -160,7 +160,7 @@ namespace libmpdataxx { if ((this->timestep - t) % static_cast(outfreq) == 0) { - hook_pre_record_all(); + hook_ante_record_all(); if (this->rank == 0) record_all(); } From 01942999be607b6f76987496a13e1859e07996a9 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Wed, 28 Sep 2022 15:27:15 +0200 Subject: [PATCH 062/129] barrier in output common --- libmpdata++/output/detail/output_common.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libmpdata++/output/detail/output_common.hpp b/libmpdata++/output/detail/output_common.hpp index 4ebcee19..430306d5 100644 --- a/libmpdata++/output/detail/output_common.hpp +++ b/libmpdata++/output/detail/output_common.hpp @@ -160,6 +160,7 @@ namespace libmpdataxx { if ((this->timestep - t) % static_cast(outfreq) == 0) { + this->mem->barrier(); hook_ante_record_all(); if (this->rank == 0) record_all(); From bbe52f9737082ff4eee019d96a9fdf46d8f703dc Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Fri, 30 Sep 2022 12:09:43 +0200 Subject: [PATCH 063/129] interpolate ref: dont calc stretchin params --- libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index eb257f9f..508381ee 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -175,7 +175,7 @@ namespace libmpdataxx // fill refined array at position where it overlaps with the resolved array this->mem->refinee(e_ref)(this->ijk_r2r) = this->mem->advectee(e)(this->ijk); - generate_stretching_parameters(); +// generate_stretching_parameters(); for(int i=0; in_fra_iter; ++i) { From c9b3b62ef685981f1154b3f88854e53db27994d6 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Mon, 3 Oct 2022 15:03:33 +0200 Subject: [PATCH 064/129] record prof refined --- libmpdata++/output/hdf5.hpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/libmpdata++/output/hdf5.hpp b/libmpdata++/output/hdf5.hpp index 0f2cd2c7..d9d918bc 100644 --- a/libmpdata++/output/hdf5.hpp +++ b/libmpdata++/output/hdf5.hpp @@ -531,9 +531,11 @@ namespace libmpdataxx } // see above, also assumes that z is the last dimension - void record_prof_const(const std::string &name, typename solver_t::real_t *data) + void record_prof_const(const std::string &name, typename solver_t::real_t *data, const bool refined = false) { assert(this->rank == 0); + const auto _shape(refined ? shape_ref : shape); + const auto _offst(refined ? offst_ref : offst); H5::H5File hdfcp(const_file, H5F_ACC_RDWR #if defined(USE_MPI) @@ -544,7 +546,7 @@ namespace libmpdataxx auto aux = hdfcp.createDataSet( name, flttype_output, - H5::DataSpace(1, &shape[parent_t::n_dims - 1]) + H5::DataSpace(1, &_shape[parent_t::n_dims - 1]) ); #if defined(USE_MPI) @@ -552,8 +554,8 @@ namespace libmpdataxx #endif { auto space = aux.getSpace(); - space.selectHyperslab(H5S_SELECT_SET, &shape[parent_t::n_dims - 1], &offst[parent_t::n_dims - 1]); - aux.write(data, flttype_solver, H5::DataSpace(1, &shape[parent_t::n_dims - 1]), space); + space.selectHyperslab(H5S_SELECT_SET, &_shape[parent_t::n_dims - 1], &_offst[parent_t::n_dims - 1]); + aux.write(data, flttype_solver, H5::DataSpace(1, &_shape[parent_t::n_dims - 1]), space); } } From be08975ce8b42352b95eb7f3d36e7d96f5a39c27 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Wed, 5 Oct 2022 13:13:53 +0200 Subject: [PATCH 065/129] comment --- libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 508381ee..2dd1f650 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -37,6 +37,7 @@ namespace libmpdataxx private: // helper variables for grid refinement + // TODO: replace with idx_t: mid_ijk_r2r ijk_r2r_h rng_t mid_ijk_r2r_0, mid_ijk_r2r_1, mid_ijk_r2r_2; // positions between already known values (to be filled during given iteration) rng_t ijk_r2r_0_h_with_halo, ijk_r2r_1_h, ijk_r2r_2_h; // all positions at resolution of given iteration int stride, hstride; // stride and half stride From 815377cd86290cb874bfb7a5def9c5e58e3dedcb Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Wed, 5 Oct 2022 13:29:40 +0200 Subject: [PATCH 066/129] formulae for averaging from refined to regular grid --- libmpdata++/formulae/refined_grid.hpp | 56 +++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 libmpdata++/formulae/refined_grid.hpp diff --git a/libmpdata++/formulae/refined_grid.hpp b/libmpdata++/formulae/refined_grid.hpp new file mode 100644 index 00000000..5771a93b --- /dev/null +++ b/libmpdata++/formulae/refined_grid.hpp @@ -0,0 +1,56 @@ +/** @file +* @copyright University of Warsaw +* @section LICENSE +* GPLv3+ (see the COPYING file or http://www.gnu.org/licenses/) +*/ + +#pragma once + +//#include + +namespace libmpdataxx +{ + namespace formulae + { + namespace refined + { + // calculate regular grid scalar as an average from the refined grid + // modifies the refined grid scalar! + // 3d version + template + void spatial_average_ref2reg( + arr_t arr, + const rng_t &i, // index of the refined point that overlaps with the regular point + const rng_t &j, // ditto + const rng_t &k, // ditto + const int &dist // average calculate from the box (i-dist, i+dist)(j-dist,j+dist)(k-dist.k+dist) + ) + { +// using idxperm::pis; + + // debug output + // std::cerr << "ranges in interpolation" << std::endl; + // if(d==0) + // std::cerr << "range<" << d << ">: " << i << " " << j << " " << k << std::endl; + // if(d==1) + // std::cerr << "range<" << d << ">: " << k << " " << i << " " << j << std::endl; + // if(d==2) + // std::cerr << "range<" << d << ">: " << j << " " << k << " " << i << std::endl; + + for(int id=-dist; id<=dist; ++id) + for(int jd=-dist; jd<=dist; ++jd) + for(int kd=-dist; kd<=dist; ++kd) + { + if(id==0 && jd==0 && kd==0) continue; // dont add the overlapping point + real_t w = 1; // weight of the point - points on edges of the averaging box have smaller weights + if(id==-dist || id == dist) w/=2.; + if(jd==-dist || jd == dist) w/=2.; + if(kd==-dist || kd == dist) w/=2.; + arr(i, j, k) += w * arr(i+id, j+jd, k+kd); + } + // divide by number of cells + arr(i, j, k) /= pow(2*dist+1, 3); + } + }; + }; +}; From a7e4a9f6c99f71cf7c7966c3639d0c18e39a2412 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Wed, 5 Oct 2022 15:29:42 +0200 Subject: [PATCH 067/129] formulae for averaging from refined to regular grid: working but without bcond --- libmpdata++/formulae/refined_grid.hpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/libmpdata++/formulae/refined_grid.hpp b/libmpdata++/formulae/refined_grid.hpp index 5771a93b..5487da71 100644 --- a/libmpdata++/formulae/refined_grid.hpp +++ b/libmpdata++/formulae/refined_grid.hpp @@ -17,13 +17,12 @@ namespace libmpdataxx // calculate regular grid scalar as an average from the refined grid // modifies the refined grid scalar! // 3d version - template + template void spatial_average_ref2reg( arr_t arr, - const rng_t &i, // index of the refined point that overlaps with the regular point - const rng_t &j, // ditto - const rng_t &k, // ditto - const int &dist // average calculate from the box (i-dist, i+dist)(j-dist,j+dist)(k-dist.k+dist) + const ijk_t &ijk, // index of the refined point that overlaps with the regular point + const int &dist, // average calculate from the box (i-dist, i+dist)(j-dist,j+dist)(k-dist.k+dist) + const bool volume_avg // should averaging account weights proportional to cell volume (refined cells at box edges are only partially within the regular cell) ) { // using idxperm::pis; @@ -37,19 +36,20 @@ namespace libmpdataxx // if(d==2) // std::cerr << "range<" << d << ">: " << j << " " << k << " " << i << std::endl; + real_t norm = 0; for(int id=-dist; id<=dist; ++id) for(int jd=-dist; jd<=dist; ++jd) for(int kd=-dist; kd<=dist; ++kd) { + real_t w = 1; // weight if(id==0 && jd==0 && kd==0) continue; // dont add the overlapping point - real_t w = 1; // weight of the point - points on edges of the averaging box have smaller weights if(id==-dist || id == dist) w/=2.; if(jd==-dist || jd == dist) w/=2.; if(kd==-dist || kd == dist) w/=2.; - arr(i, j, k) += w * arr(i+id, j+jd, k+kd); + arr(ijk) += w * arr(ijk[0]+id, ijk[1]+jd, ijk[2]+kd); + norm += w; } - // divide by number of cells - arr(i, j, k) /= pow(2*dist+1, 3); + arr(ijk) /= norm+1; // +1 because the center cell was not added to norm } }; }; From 9217fd42897509508ffd56e5c398ea0b4806804d Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Tue, 18 Oct 2022 16:03:42 +0200 Subject: [PATCH 068/129] averaging from refined to regular: fixes for edge cells --- libmpdata++/formulae/refined_grid.hpp | 57 ++++++++++--------- .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 2 + 2 files changed, 33 insertions(+), 26 deletions(-) diff --git a/libmpdata++/formulae/refined_grid.hpp b/libmpdata++/formulae/refined_grid.hpp index 5487da71..4093b1d5 100644 --- a/libmpdata++/formulae/refined_grid.hpp +++ b/libmpdata++/formulae/refined_grid.hpp @@ -20,36 +20,41 @@ namespace libmpdataxx template void spatial_average_ref2reg( arr_t arr, - const ijk_t &ijk, // index of the refined point that overlaps with the regular point - const int &dist, // average calculate from the box (i-dist, i+dist)(j-dist,j+dist)(k-dist.k+dist) - const bool volume_avg // should averaging account weights proportional to cell volume (refined cells at box edges are only partially within the regular cell) + const ijk_t &ijk, // index of the refined point that overlaps with the regular point + const int &dist, // average calculate from the box (i-dist, i+dist)(j-dist,j+dist)(k-dist.k+dist) + const std::array grid_size, // number of points in each direction (total, not for this MPI process), needed for averaging on edges so that it does not go out of the domain; refined arrays have no halos on domain edgs, only halos between MPI subdomains + const bool volume_avg // should averaging account weights proportional to cell volume (refined cells at box edges are only partially within the regular cell) ) { -// using idxperm::pis; - - // debug output - // std::cerr << "ranges in interpolation" << std::endl; - // if(d==0) - // std::cerr << "range<" << d << ">: " << i << " " << j << " " << k << std::endl; - // if(d==1) - // std::cerr << "range<" << d << ">: " << k << " " << i << " " << j << std::endl; - // if(d==2) - // std::cerr << "range<" << d << ">: " << j << " " << k << " " << i << std::endl; - - real_t norm = 0; - for(int id=-dist; id<=dist; ++id) - for(int jd=-dist; jd<=dist; ++jd) - for(int kd=-dist; kd<=dist; ++kd) + // WARNING: very bad LOOPS + for(int i = ijk[0].first(); i <= ijk[0].last(); i+= ijk[0].stride()) + for(int j = ijk[1].first(); j <= ijk[1].last(); j+= ijk[1].stride()) + for(int k = ijk[2].first(); k <= ijk[2].last(); k+= ijk[2].stride()) { - real_t w = 1; // weight - if(id==0 && jd==0 && kd==0) continue; // dont add the overlapping point - if(id==-dist || id == dist) w/=2.; - if(jd==-dist || jd == dist) w/=2.; - if(kd==-dist || kd == dist) w/=2.; - arr(ijk) += w * arr(ijk[0]+id, ijk[1]+jd, ijk[2]+kd); - norm += w; + real_t norm = 0; + for(int id=-dist; id<=dist; ++id) + for(int jd=-dist; jd<=dist; ++jd) + for(int kd=-dist; kd<=dist; ++kd) + { + if(id==0 && jd==0 && kd==0) continue; // dont add the overlapping point + // dont add points outside of the domain + if(i+id < 0 || i+id > grid_size[0]-1 || + j+jd < 0 || j+jd > grid_size[1]-1 || + k+kd < 0 || k+kd > grid_size[2]-1) + continue; + + real_t w = 1; // weight + if(volume_avg) + { + if(id==-dist || id == dist) w/=2.; + if(jd==-dist || jd == dist) w/=2.; + if(kd==-dist || kd == dist) w/=2.; + } + arr(i,j,k) += w * arr(i+id,j+jd,k+kd); + norm += w; + } + arr(i,j,k) /= norm+1; // +1 because the center cell was not added to norm } - arr(ijk) /= norm+1; // +1 because the center cell was not added to norm } }; }; diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 2dd1f650..23e1bc0e 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -201,6 +201,8 @@ namespace libmpdataxx //this->mem->barrier(); //std::cerr << "mem->grid_size_ref[0]: " << this->mem->grid_size_ref[0] << std::endl; + //std::cerr << "mem->grid_size_ref[1]: " << this->mem->grid_size_ref[1] << std::endl; + //std::cerr << "mem->grid_size_ref[2]: " << this->mem->grid_size_ref[2] << std::endl; fill_refinee_distmem_halos(e, halo_size); From 04ce94df2fc1d2e8abd48974136ed958d65424f5 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Wed, 19 Oct 2022 10:44:13 +0200 Subject: [PATCH 069/129] hdf output: record_aux_refined --- libmpdata++/output/hdf5.hpp | 21 ++++++++++++++++----- libmpdata++/output/hdf5_xdmf.hpp | 9 +++++++++ 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/libmpdata++/output/hdf5.hpp b/libmpdata++/output/hdf5.hpp index d9d918bc..859f2d77 100644 --- a/libmpdata++/output/hdf5.hpp +++ b/libmpdata++/output/hdf5.hpp @@ -373,20 +373,26 @@ namespace libmpdataxx } // data is assumed to be contiguous and in the same layout as hdf variable and in the C-style storage order - void record_aux_hlpr(const std::string &name, typename solver_t::real_t *data, H5::H5File hdf) + void record_aux_hlpr(const std::string &name, typename solver_t::real_t *data, H5::H5File hdf, bool refined = false) { assert(this->rank == 0); + const auto _shape(refined ? shape_ref : shape); + const auto _offst(refined ? offst_ref : offst); + if(refined) params.setChunk(parent_t::n_dims, chunk_ref.data()); auto aux = hdf.createDataSet( name, flttype_output, - sspace, + refined ? sspace_ref : sspace, params ); auto space = aux.getSpace(); - space.selectHyperslab(H5S_SELECT_SET, shape.data(), offst.data()); - aux.write(data, flttype_solver, H5::DataSpace(parent_t::n_dims, shape.data()), space, dxpl_id); + space.selectHyperslab(H5S_SELECT_SET, _shape.data(), _offst.data()); + aux.write(data, flttype_solver, H5::DataSpace(parent_t::n_dims, _shape.data()), space, dxpl_id); + + // revert to default chunk + if(refined) params.setChunk(parent_t::n_dims, chunk.data()); } // for discontiguous array with halos @@ -460,7 +466,12 @@ namespace libmpdataxx // data is assumed to be contiguous and in the same layout as hdf variable and in the C-style storage order void record_aux(const std::string &name, typename solver_t::real_t *data) { - record_aux_hlpr(name, data, *hdfp); + record_aux_hlpr(name, data, *hdfp, false); + } + + void record_aux_refined(const std::string &name, typename solver_t::real_t *data) + { + record_aux_hlpr(name, data, *hdfp, true); } void record_aux_dsc(const std::string &name, const typename solver_t::arr_t &arr, bool srfc = false) diff --git a/libmpdata++/output/hdf5_xdmf.hpp b/libmpdata++/output/hdf5_xdmf.hpp index 2eb3452a..9e1d7404 100644 --- a/libmpdata++/output/hdf5_xdmf.hpp +++ b/libmpdata++/output/hdf5_xdmf.hpp @@ -97,6 +97,15 @@ namespace libmpdataxx parent_t::record_aux(name, data); } + void record_aux_refined(const std::string &name, typename solver_t::real_t *data) + { +#if defined(USE_MPI) + if (this->mem->distmem.rank() == 0) +#endif + xdmfw_ref.add_attribute(name, this->hdf_name(), this->mem->distmem.grid_size_ref.data()); + parent_t::record_aux_refined(name, data); + } + void record_aux_dsc(const std::string &name, const typename solver_t::arr_t &arr, bool srfc = false) { auto shape = this->mem->distmem.grid_size; From bf7e3cac1a9d7730f9712944fa185ce991560874 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Wed, 19 Oct 2022 14:47:53 +0200 Subject: [PATCH 070/129] refined arrays: add halo size 1 --- libmpdata++/formulae/refined_grid.hpp | 6 +++++- .../solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 6 +++--- .../detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp | 11 ++++++----- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/libmpdata++/formulae/refined_grid.hpp b/libmpdata++/formulae/refined_grid.hpp index 4093b1d5..5f75f613 100644 --- a/libmpdata++/formulae/refined_grid.hpp +++ b/libmpdata++/formulae/refined_grid.hpp @@ -17,12 +17,16 @@ namespace libmpdataxx // calculate regular grid scalar as an average from the refined grid // modifies the refined grid scalar! // 3d version + // averaged only from points within the domain, halos outside of the modeled domain are not used + // however, halos between MPI domains are used; + // it is assumed that these are halos are large enough! + // currently, the halo between MPI domains is n_ref+n_ref/2 template void spatial_average_ref2reg( arr_t arr, const ijk_t &ijk, // index of the refined point that overlaps with the regular point const int &dist, // average calculate from the box (i-dist, i+dist)(j-dist,j+dist)(k-dist.k+dist) - const std::array grid_size, // number of points in each direction (total, not for this MPI process), needed for averaging on edges so that it does not go out of the domain; refined arrays have no halos on domain edgs, only halos between MPI subdomains + const std::array grid_size, // number of cells in the domain in each direction (total, not for this MPI process), needed for averaging on edges so that it does not go out of the domain const bool volume_avg // should averaging account weights proportional to cell volume (refined cells at box edges are only partially within the regular cell) ) { diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 23e1bc0e..5d408aa7 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -239,9 +239,9 @@ namespace libmpdataxx for (int n = 0; n < n_arr; ++n) mem->tmp[__file__].back().push_back(mem->old(new typename parent_t::arr_t( - parent_t::rng_ref_distmem_halo(mem->grid_size_ref[0], mem->n_ref, mem->distmem.rank(), mem->distmem.size()), - mem->grid_size_ref[1], - srfc ? rng_t(0, 0) : mem->grid_size_ref[2], + parent_t::rng_ref_distmem_halo(mem->grid_size_ref[0]^parent_t::halo_ref, mem->n_ref, mem->distmem.rank(), mem->distmem.size()), + mem->grid_size_ref[1]^parent_t::halo_ref, + srfc ? rng_t(0, 0) : mem->grid_size_ref[2]^parent_t::halo_ref, arr3D_storage ))); } diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp index a01780cf..cd7cdac3 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp @@ -38,9 +38,9 @@ namespace libmpdataxx const int n_ref, // number of refinements; refined resolution is dx / n_ref n_fra_iter; // number of iterations of grid refinement - -// const int halo_ref; // size of halos of refined scalar (and vector?) arrays + static const int halo_ref = 1; // halo size of refined scalar arrays + const std::array ix_r2r; constexpr std::array get_ix_r2r() { @@ -141,11 +141,12 @@ namespace libmpdataxx } // reconstruction based on 3 points, we need up to 2 resolved points between MPI domains - static rng_t rng_ref_distmem_halo(const rng_t &rng, const int &n_ref, const int rank = 0, const int size = 1) + static rng_t rng_ref_distmem_halo(const rng_t &rng, const int &n_ref, const int rank, const int size) { + const int halo = max(halo_ref, n_ref + n_ref/2); return rng_t( - rank == 0 ? rng.first() : rng.first() - (n_ref + n_ref/2), - rank < size - 1 ? rng.last() + (n_ref + n_ref/2) : rng.last(), + rank == 0 ? rng.first() : rng.first() - halo, + rank < size - 1 ? rng.last() + halo : rng.last(), rng.stride()); } From d97fc3ec204329d53e7f1438346352f7ea299829 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Fri, 21 Oct 2022 12:51:38 +0200 Subject: [PATCH 071/129] additional set of bconds for refined arrays --- libmpdata++/concurr/detail/concurr_common.hpp | 508 +++--------------- .../concurr/detail/concurr_common_hlpr.hpp | 475 ++++++++++++++++ .../detail/concurr_ref_common_hlpr.hpp | 91 ++++ .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 63 ++- .../mpdata_rhs_vip_prs_sgs_fra_common.hpp | 8 +- libmpdata++/solvers/detail/solver_3d.hpp | 6 +- libmpdata++/solvers/detail/solver_common.hpp | 10 +- 7 files changed, 725 insertions(+), 436 deletions(-) create mode 100644 libmpdata++/concurr/detail/concurr_common_hlpr.hpp create mode 100644 libmpdata++/concurr/detail/concurr_ref_common_hlpr.hpp diff --git a/libmpdata++/concurr/detail/concurr_common.hpp b/libmpdata++/concurr/detail/concurr_common.hpp index 5981803a..0806766b 100644 --- a/libmpdata++/concurr/detail/concurr_common.hpp +++ b/libmpdata++/concurr/detail/concurr_common.hpp @@ -6,35 +6,7 @@ #pragma once -// ensuring thread-safe versions of system headers are used -#if !defined(_REENTRANT) -# error _REENTRANT not defined, please use something like -pthread flag for gcc -#endif - -#include -#include - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include +#include namespace libmpdataxx { @@ -42,422 +14,114 @@ namespace libmpdataxx { namespace detail { - // helpers for setting remote bcond - template < - class real_t, - bcond::drctn_e dir, - int dim, - int n_dims, - int halo, - class bcp_t, - class mem_t - > - struct bc_set_remote_impl + /* + template + struct has_solver_family : std::false_type { }; + + template + struct has_solver_family : std::true_type { }; + */ + + + // helpers to find solvers that have the grid_refinement + /* + template< class, class = void > + struct has_grid_refinement : std::false_type { }; + template< class T > + struct has_grid_refinement> : std::true_type { }; + */ + +/* + template + constexpr bool is_derived_from_fra() { - static void _( - bcp_t &bcp, - const std::unique_ptr &mem, - const int thread_rank, - const int thread_size - ) - { - bcp.reset( - new bcond::bcond( - mem->slab(mem->grid_size[dim]), - mem->distmem.grid_size, - thread_rank, - thread_size - ) - ); - } - }; + if(!has_parent_t::value) return false; + return std::is_same_v ? true : is_derived_from_fra(); + } + */ - // 3d specialization - template < - class real_t, - bcond::drctn_e dir, - int dim, - int halo, - class bcp_t, - class mem_t + template< + class solver_t_, + bcond::bcond_e bcxl, bcond::bcond_e bcxr, + bcond::bcond_e bcyl, bcond::bcond_e bcyr, + bcond::bcond_e bczl, bcond::bcond_e bczr, + class enableif = void > - struct bc_set_remote_impl + class concurr_common : public concurr_ref_common_hlpr { - static void _( - bcp_t &bcp, - const std::unique_ptr &mem, - const int thread_rank, - const int thread_size - ) + public: + using parent_t = concurr_ref_common_hlpr; + + // ctor + concurr_common( + const typename solver_t_::rt_params_t &p, + typename solver_t_::mem_t *mem_p, + const int &size + ) : + parent_t(p, mem_p, size) { - bcp.reset( - new bcond::bcond( - mem->slab(mem->grid_size[dim]), - mem->distmem.grid_size, - mem->slab(mem->grid_size[1], thread_rank, thread_size), // NOTE: we assume here remote 3d bcond is only on the edges perpendicular to x - thread_rank, - thread_size - ) - ); + // allocate per-thread structures + this->init(p, mem_p->grid_size, size); } }; - - template < - class real_t, - bcond::drctn_e dir, - int dim, - int n_dims, - int halo, - class bcp_t, - class mem_t +/* + // specialization for solvers that do not have grid refinement + template< + class solver_t_, + bcond::bcond_e bcxl, bcond::bcond_e bcxr, + bcond::bcond_e bcyl, bcond::bcond_e bcyr, + bcond::bcond_e bczl, bcond::bcond_e bczr + //typename std::enable_if_t> + //typename std::enable_if_t()> > - void bc_set_remote( - bcp_t &bcp, - const std::unique_ptr &mem, - const int thread_rank, - const int thread_size - ) - { - bc_set_remote_impl::_(bcp, mem, thread_rank, thread_size); - } - + class concurr_common < + solver_t_, + bcxl, bcxr, + bcyl, bcyr, + bczl, bczr, + //typename std::enable_if_t + //typename std::enable_if_t()> + //typename std::enable_if_t()> + typename std::enable_if_t + > : public concurr_common_hlpr + {}; + */ + + // specialzie for solvers with refined grid template< class solver_t_, bcond::bcond_e bcxl, bcond::bcond_e bcxr, bcond::bcond_e bcyl, bcond::bcond_e bcyr, bcond::bcond_e bczl, bcond::bcond_e bczr + //typename std::enable_if_t> + //typename std::enable_if_t()> > - class concurr_common : public any + class concurr_common < + solver_t_, + bcxl, bcxr, + bcyl, bcyr, + bczl, bczr, + //typename std::enable_if_t + //typename std::enable_if_t()> + //typename std::enable_if_t()> + typename std::enable_if_t + > : public concurr_ref_common_hlpr { public: - - typedef solver_t_ solver_t; - - static_assert( - (solver_t::n_dims == 3) || - (solver_t::n_dims == 2 - && bczl == bcond::null - && bczr == bcond::null - ) || - (solver_t::n_dims == 1 - && bczl == bcond::null - && bczr == bcond::null - && bcyl == bcond::null - && bcyr == bcond::null - ) - , - "more boundary conditions than dimensions" - ); - - protected: - - using mem_t = typename solver_t::mem_t; - - // member fields - boost::ptr_vector algos; - std::unique_ptr mem; - timer tmr; - - public: - - typedef typename solver_t::real_t real_t; - using advance_arg_t = typename solver_t::advance_arg_t; - - // dtor - virtual ~concurr_common() - { - tmr.print(); - } + using parent_t = concurr_ref_common_hlpr; // ctor concurr_common( - const typename solver_t::rt_params_t &p, - mem_t *mem_p, + const typename solver_t_::rt_params_t &p, + typename solver_t_::mem_t *mem_p, const int &size - ) { - // allocate the memory to be shared by multiple threads - mem.reset(mem_p); - solver_t::alloc(mem.get(), p.n_iters); - - // allocate per-thread structures - init(p, mem->grid_size, size); - } - - private: - - template < - bcond::bcond_e type, - bcond::drctn_e dir, - int dim - > - void bc_set( - typename solver_t::bcp_t &bcp, - const int thread_rank = 0, // required only by 3D remote (MPI) and open bconds - const int thread_size = 0 // required only by 3D remote (MPI) and open bconds - ) - { - // sanity check - polar coords do not work with MPI yet - if (type == bcond::polar && mem->distmem.size() > 1) - throw std::runtime_error("Polar boundary conditions do not work with MPI."); - - // distmem overrides - if (mem->distmem.size() > 1 && dim == 0) - { - if ( - // distmem domain interior - (dir == bcond::left && mem->distmem.rank() > 0) - || - (dir == bcond::rght && mem->distmem.rank() != mem->distmem.size() - 1) - // cyclic condition for distmem domain (note: will not work if a non-cyclic condition is on the other end) - || - (type == bcond::cyclic) - ) - { - // bc allocation, all mpi routines called by the remote bcnd ctor are thread-safe (?) - bc_set_remote( - bcp, - mem, - thread_rank, - thread_size - ); - return; - } - } - - // 3d open bcond needs to know thread rank and size, because it zeroes perpendicular vectors - if (type == bcond::open && solver_t::n_dims == 3) - { - bcp.reset( - new bcond::bcond( - mem->slab(mem->grid_size[dim]), - mem->distmem.grid_size, - false, - thread_rank, - thread_size - ) - ); - return; - } - - // else: not remote and not open_3d - bcp.reset( - new bcond::bcond( - mem->slab(mem->grid_size[dim]), - mem->distmem.grid_size - ) - ); - } - - // 1D version - void init( - const typename solver_t::rt_params_t &p, - const std::array &grid_size, const int &n0 - ) - { - typename solver_t::bcp_t bxl, bxr, shrdl, shrdr; - - // NOTE: for remote bcond, thread_rank set to 0 on purpose in 1D to have propre left/right message tags - bc_set(bxl); - bc_set(bxr); - - for (int i0 = 0; i0 < n0; ++i0) - { - shrdl.reset(new bcond::shared()); - shrdr.reset(new bcond::shared()); - - algos.push_back( - new solver_t( - typename solver_t::ctor_args_t({ - i0, - mem.get(), - i0 == 0 ? bxl : shrdl, - i0 == n0 - 1 ? bxr : shrdr, - mem->slab(grid_size[0], i0, n0) - }), - p - ) - ); - } - } - - // 2D version - // TODO: assert parallelisation in the right dimensions! (blitz::assertContiguous) - void init( - const typename solver_t::rt_params_t &p, - const std::array &grid_size, - const int &n0, const int &n1 = 1 - ) { - for (int i0 = 0; i0 < n0; ++i0) - { - for (int i1 = 0; i1 < n1; ++i1) - { - typename solver_t::bcp_t bxl, bxr, byl, byr, shrdl, shrdr; - - // NOTE: for remote bcond, thread_rank set to 0 on purpose in 2D to have propre left/right message tags - bc_set(bxl); - bc_set(bxr); - - bc_set(byl); - bc_set(byr); - - shrdl.reset(new bcond::shared()); // TODO: shrdy if n1 != 1 - shrdr.reset(new bcond::shared()); // TODO: shrdy if n1 != 1 - - algos.push_back( - new solver_t( - typename solver_t::ctor_args_t({ - i0, - mem.get(), - i0 == 0 ? bxl : shrdl, - i0 == n0 - 1 ? bxr : shrdr, - byl, byr, - mem->slab(grid_size[0], i0, n0), - mem->slab(grid_size[1], i1, n1) - }), - p - ) - ); - } - } - } - - // 3D version, note sharedmem in y direction! - void init( - const typename solver_t::rt_params_t &p, - const std::array &grid_size, - const int &n1, const int &n0 = 1, const int &n2 = 1 - ) { - typename solver_t::bcp_t bxl, bxr, byl, byr, bzl, bzr, shrdl, shrdr; - - // TODO: renew pointers only if invalid ? - for (int i0 = 0; i0 < n0; ++i0) - { - for (int i1 = 0; i1 < n1; ++i1) - { - for (int i2 = 0; i2 < n2; ++i2) - { - // i1 is the local thread rank, n1 is the number of threads. These are needed by remote bcond, because only rank=0 does mpi communication - bc_set(bxl, i1, n1); - bc_set(bxr, i1, n1); - - bc_set(byl); - bc_set(byr); - - bc_set(bzl); - bc_set(bzr); - - shrdl.reset(new bcond::shared()); // TODO: shrdy if n1 != 1 - shrdr.reset(new bcond::shared()); // TODO: shrdy if n1 != 1 - - algos.push_back( - new solver_t( - typename solver_t::ctor_args_t({ - i1, - mem.get(), - bxl, bxr, - i1 == 0 ? byl : shrdl, - i1 == n1 - 1 ? byr : shrdr, - bzl, bzr, - mem->slab(grid_size[0], i0, n0), - mem->slab(grid_size[1], i1, n1), - mem->slab(grid_size[2], i2, n2) - }), - p - ) - ); - } - } - } - } - - virtual void solve(advance_arg_t nt) = 0; - - public: - - void advance(advance_arg_t nt) final - { - tmr.resume(); - solve(nt); - tmr.stop(); - } - - typename solver_t::arr_t advectee(int e = 0) final - { - return mem->advectee(e); - } - - const typename solver_t::arr_t advectee_global(int e = 0) final - { -#if defined(USE_MPI) - return mem->advectee_global(e); -#else - return advectee(e); -#endif - } - - void advectee_global_set(const typename solver_t::arr_t arr, int e = 0) final - { -#if defined(USE_MPI) - mem->advectee_global_set(arr, e); -#else - advectee(e) = arr; -#endif - } - - typename solver_t::arr_t advector(int d = 0) final + ) : + parent_t(p, mem_p, size) { - return mem->advector(d); - } - - typename solver_t::arr_t g_factor() final - { - return mem->g_factor(); - } - - typename solver_t::arr_t vab_coefficient() final - { - return mem->vab_coefficient(); - } - - typename solver_t::arr_t vab_relaxed_state(int d = 0) final - { - return mem->vab_relaxed_state(d); - } - - typename solver_t::arr_t sclr_array(const std::string &name, int n = 0) final - { - return mem->sclr_array(name, n); - } - - bool *panic_ptr() final - { - return &this->mem->panic; - } - - const real_t time() const final - { - return algos[0].time_(); - } - - const real_t min(int e = 0) const final - { - return mem->min(mem->advectee(e)); - } - - const real_t max(int e = 0) const final - { - return mem->max(mem->advectee(e)); + // allocate per-thread structures + this->init(p, mem_p->grid_size, size); } }; - - template< class mem_t, class solver_t> - mem_t* mem_factory(const typename solver_t::rt_params_t &p) - { - if constexpr (solvers::detail::slvr_with_frac_recn()) - return new mem_t(p.grid_size, pow(2, p.n_fra_iter)); - else - return new mem_t(p.grid_size); - } } // namespace detail } // namespace concurr } // namespace libmpdataxx diff --git a/libmpdata++/concurr/detail/concurr_common_hlpr.hpp b/libmpdata++/concurr/detail/concurr_common_hlpr.hpp new file mode 100644 index 00000000..8d03e945 --- /dev/null +++ b/libmpdata++/concurr/detail/concurr_common_hlpr.hpp @@ -0,0 +1,475 @@ +/** @file + * @copyright University of Warsaw + * @section LICENSE + * GPLv3+ (see the COPYING file or http://www.gnu.org/licenses/) + */ + +#pragma once + +// ensuring thread-safe versions of system headers are used +#if !defined(_REENTRANT) +# error _REENTRANT not defined, please use something like -pthread flag for gcc +#endif + +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace libmpdataxx +{ + namespace concurr + { + namespace detail + { + // helpers for setting remote bcond + template < + class real_t, + bcond::drctn_e dir, + int dim, + int n_dims, + int halo, + class bcp_t, + class mem_t + > + struct bc_set_remote_impl + { + static void _( + bcp_t &bcp, + const std::unique_ptr &mem, + const int thread_rank, + const int thread_size + ) + { + bcp.reset( + new bcond::bcond( + mem->slab(mem->grid_size[dim]), + mem->distmem.grid_size, + thread_rank, + thread_size + ) + ); + } + }; + + // 3d specialization + template < + class real_t, + bcond::drctn_e dir, + int dim, + int halo, + class bcp_t, + class mem_t + > + struct bc_set_remote_impl + { + static void _( + bcp_t &bcp, + const std::unique_ptr &mem, + const int thread_rank, + const int thread_size + ) + { + bcp.reset( + new bcond::bcond( + mem->slab(mem->grid_size[dim]), + mem->distmem.grid_size, + mem->slab(mem->grid_size[1], thread_rank, thread_size), // NOTE: we assume here remote 3d bcond is only on the edges perpendicular to x + thread_rank, + thread_size + ) + ); + } + }; + + template < + class real_t, + bcond::drctn_e dir, + int dim, + int n_dims, + int halo, + class bcp_t, + class mem_t + > + void bc_set_remote( + bcp_t &bcp, + const std::unique_ptr &mem, + const int thread_rank, + const int thread_size + ) + { + bc_set_remote_impl::_(bcp, mem, thread_rank, thread_size); + } + + template< + class solver_t_, + bcond::bcond_e bcxl, bcond::bcond_e bcxr, + bcond::bcond_e bcyl, bcond::bcond_e bcyr, + bcond::bcond_e bczl, bcond::bcond_e bczr + > + class concurr_common_hlpr : public any + { + public: + + typedef solver_t_ solver_t; + + static_assert( + (solver_t::n_dims == 3) || + (solver_t::n_dims == 2 + && bczl == bcond::null + && bczr == bcond::null + ) || + (solver_t::n_dims == 1 + && bczl == bcond::null + && bczr == bcond::null + && bcyl == bcond::null + && bcyr == bcond::null + ) + , + "more boundary conditions than dimensions" + ); + + protected: + + using mem_t = typename solver_t::mem_t; + + // member fields + boost::ptr_vector algos; + std::unique_ptr mem; + timer tmr; + + typename solver_t::bcp_t bxl, bxr, byl, byr, bzl, bzr, shrdl, shrdr; + + public: + + typedef typename solver_t::real_t real_t; + using advance_arg_t = typename solver_t::advance_arg_t; + + // dtor + virtual ~concurr_common_hlpr() + { + tmr.print(); + } + + // ctor + concurr_common_hlpr( + const typename solver_t::rt_params_t &p, + mem_t *mem_p, + const int &size + ) { + // allocate the memory to be shared by multiple threads + mem.reset(mem_p); + solver_t::alloc(mem.get(), p.n_iters); + + // allocate per-thread structures +// this->init(p, mem->grid_size, size); + } + + protected: + + template < + bcond::bcond_e type, + bcond::drctn_e dir, + int dim, + int halo, + class bcp_t, + class grid_size_t, + class distmem_grid_size_t + > + void bc_set( + bcp_t &bcp, + const grid_size_t grid_size, + const distmem_grid_size_t distmem_grid_size, + const int thread_rank = 0, // required only by 3D remote (MPI) and open bconds + const int thread_size = 0 // required only by 3D remote (MPI) and open bconds + ) + { + // sanity check - polar coords do not work with MPI yet + if (type == bcond::polar && mem->distmem.size() > 1) + throw std::runtime_error("Polar boundary conditions do not work with MPI."); + + // distmem overrides + if (mem->distmem.size() > 1 && dim == 0) + { + if ( + // distmem domain interior + (dir == bcond::left && mem->distmem.rank() > 0) + || + (dir == bcond::rght && mem->distmem.rank() != mem->distmem.size() - 1) + // cyclic condition for distmem domain (note: will not work if a non-cyclic condition is on the other end) + || + (type == bcond::cyclic) + ) + { + // bc allocation, all mpi routines called by the remote bcnd ctor are thread-safe (?) + bc_set_remote( + bcp, + mem, + thread_rank, + thread_size + ); + return; + } + } + + // 3d open bcond needs to know thread rank and size, because it zeroes perpendicular vectors + if (type == bcond::open && solver_t::n_dims == 3) + { + bcp.reset( + new bcond::bcond( + mem->slab(grid_size[dim]), + distmem_grid_size, + false, + thread_rank, + thread_size + ) + ); + return; + } + + // else: not remote and not open_3d + bcp.reset( + new bcond::bcond( + mem->slab(grid_size[dim]), + distmem_grid_size + ) + ); + } + + // 1D version + virtual void init( + const typename solver_t::rt_params_t &p, + const std::array &grid_size, const int &n0 + ) + { + typename solver_t::bcp_t bxl, bxr, shrdl, shrdr; + + // NOTE: for remote bcond, thread_rank set to 0 on purpose in 1D to have propre left/right message tags + bc_set(bxl, mem->grid_size, mem->distmem.grid_size); + bc_set(bxr, mem->grid_size, mem->distmem.grid_size); + + for (int i0 = 0; i0 < n0; ++i0) + { + shrdl.reset(new bcond::shared()); + shrdr.reset(new bcond::shared()); + + algos.push_back( + new solver_t( + typename solver_t::ctor_args_t({ + i0, + mem.get(), + i0 == 0 ? bxl : shrdl, + i0 == n0 - 1 ? bxr : shrdr, + mem->slab(grid_size[0], i0, n0) + }), + p + ) + ); + } + } + + // 2D version + // TODO: assert parallelisation in the right dimensions! (blitz::assertContiguous) + virtual void init( + const typename solver_t::rt_params_t &p, + const std::array &grid_size, + const int &n0, const int &n1 = 1 + ) { + for (int i0 = 0; i0 < n0; ++i0) + { + for (int i1 = 0; i1 < n1; ++i1) + { + typename solver_t::bcp_t bxl, bxr, byl, byr, shrdl, shrdr; + + // NOTE: for remote bcond, thread_rank set to 0 on purpose in 2D to have propre left/right message tags + bc_set(bxl, mem->grid_size, mem->distmem.grid_size); + bc_set(bxr, mem->grid_size, mem->distmem.grid_size); + + bc_set(byl, mem->grid_size, mem->distmem.grid_size); + bc_set(byr, mem->grid_size, mem->distmem.grid_size); + + shrdl.reset(new bcond::shared()); // TODO: shrdy if n1 != 1 + shrdr.reset(new bcond::shared()); // TODO: shrdy if n1 != 1 + + algos.push_back( + new solver_t( + typename solver_t::ctor_args_t({ + i0, + mem.get(), + i0 == 0 ? bxl : shrdl, + i0 == n0 - 1 ? bxr : shrdr, + byl, byr, + mem->slab(grid_size[0], i0, n0), + mem->slab(grid_size[1], i1, n1) + }), + p + ) + ); + } + } + } + + // 3D version, note sharedmem in y direction! + void init_bcs(const int &i1, const int &n1) + { + // i1 is the local thread rank, n1 is the number of threads. These are needed by remote bcond, because only rank=0 does mpi communication + bc_set(bxl, mem->grid_size, mem->distmem.grid_size, i1, n1); + bc_set(bxr, mem->grid_size, mem->distmem.grid_size, i1, n1); + + bc_set(byl, mem->grid_size, mem->distmem.grid_size); + bc_set(byr, mem->grid_size, mem->distmem.grid_size); + + bc_set(bzl, mem->grid_size, mem->distmem.grid_size); + bc_set(bzr, mem->grid_size, mem->distmem.grid_size); + + shrdl.reset(new bcond::shared()); // TODO: shrdy if n1 != 1 + shrdr.reset(new bcond::shared()); // TODO: shrdy if n1 != 1 + } + + virtual void init( + const typename solver_t::rt_params_t &p, + const std::array &grid_size, + const int &n1, const int &n0 = 1, const int &n2 = 1 + ) { + // TODO: renew pointers only if invalid ? + for (int i0 = 0; i0 < n0; ++i0) + { + for (int i1 = 0; i1 < n1; ++i1) + { + for (int i2 = 0; i2 < n2; ++i2) + { + init_bcs(i1, n1); + + algos.push_back( + new solver_t( + typename solver_t::ctor_args_t({ + i1, + mem.get(), + bxl, bxr, + i1 == 0 ? byl : shrdl, + i1 == n1 - 1 ? byr : shrdr, + bzl, bzr, + mem->slab(grid_size[0], i0, n0), + mem->slab(grid_size[1], i1, n1), + mem->slab(grid_size[2], i2, n2) + }), + p + ) + ); + } + } + } + } + + private: + virtual void solve(advance_arg_t nt) = 0; + + public: + + void advance(advance_arg_t nt) final + { + tmr.resume(); + solve(nt); + tmr.stop(); + } + + typename solver_t::arr_t advectee(int e = 0) final + { + return mem->advectee(e); + } + + const typename solver_t::arr_t advectee_global(int e = 0) final + { +#if defined(USE_MPI) + return mem->advectee_global(e); +#else + return advectee(e); +#endif + } + + void advectee_global_set(const typename solver_t::arr_t arr, int e = 0) final + { +#if defined(USE_MPI) + mem->advectee_global_set(arr, e); +#else + advectee(e) = arr; +#endif + } + + typename solver_t::arr_t advector(int d = 0) final + { + return mem->advector(d); + } + + typename solver_t::arr_t g_factor() final + { + return mem->g_factor(); + } + + typename solver_t::arr_t vab_coefficient() final + { + return mem->vab_coefficient(); + } + + typename solver_t::arr_t vab_relaxed_state(int d = 0) final + { + return mem->vab_relaxed_state(d); + } + + typename solver_t::arr_t sclr_array(const std::string &name, int n = 0) final + { + return mem->sclr_array(name, n); + } + + bool *panic_ptr() final + { + return &this->mem->panic; + } + + const real_t time() const final + { + return algos[0].time_(); + } + + const real_t min(int e = 0) const final + { + return mem->min(mem->advectee(e)); + } + + const real_t max(int e = 0) const final + { + return mem->max(mem->advectee(e)); + } + }; + + template< class mem_t, class solver_t> + mem_t* mem_factory(const typename solver_t::rt_params_t &p) + { + if constexpr (solvers::detail::slvr_with_frac_recn()) + return new mem_t(p.grid_size, pow(2, p.n_fra_iter)); + else + return new mem_t(p.grid_size); + } + } // namespace detail + } // namespace concurr +} // namespace libmpdataxx diff --git a/libmpdata++/concurr/detail/concurr_ref_common_hlpr.hpp b/libmpdata++/concurr/detail/concurr_ref_common_hlpr.hpp new file mode 100644 index 00000000..8f5def7d --- /dev/null +++ b/libmpdata++/concurr/detail/concurr_ref_common_hlpr.hpp @@ -0,0 +1,91 @@ +/** @file + * @copyright University of Warsaw + * @section LICENSE + * GPLv3+ (see the COPYING file or http://www.gnu.org/licenses/) + */ + +#pragma once + +#include + +namespace libmpdataxx +{ + namespace concurr + { + namespace detail + { + template< + class solver_t_, + bcond::bcond_e bcxl, bcond::bcond_e bcxr, + bcond::bcond_e bcyl, bcond::bcond_e bcyr, + bcond::bcond_e bczl, bcond::bcond_e bczr + > + class concurr_ref_common_hlpr : public concurr_common_hlpr + { + private: + + using parent_t = concurr_common_hlpr; + using parent_t::parent_t; + + protected: + // 3D version, note sharedmem in y direction! + virtual void init( + const typename solver_t_::rt_params_t &p, + const std::array &grid_size, + const int &n1, const int &n0 = 1, const int &n2 = 1 + ) override { + + typename solver_t_::bcp_ref_t bxl_ref, bxr_ref, byl_ref, byr_ref, bzl_ref, bzr_ref, shrdl_ref, shrdr_ref; + + // TODO: renew pointers only if invalid ? + for (int i0 = 0; i0 < n0; ++i0) + { + for (int i1 = 0; i1 < n1; ++i1) + { + for (int i2 = 0; i2 < n2; ++i2) + { + // i1 is the local thread rank, n1 is the number of threads. These are needed by remote bcond, because only rank=0 does mpi communication + this->init_bcs(i1, n1); + + this->template bc_set(bxl_ref, this->mem->grid_size_ref, this->mem->distmem.grid_size_ref, i1, n1); + this->template bc_set(bxr_ref, this->mem->grid_size_ref, this->mem->distmem.grid_size_ref, i1, n1); + + this->template bc_set(byl_ref, this->mem->grid_size_ref, this->mem->distmem.grid_size_ref); + this->template bc_set(byr_ref, this->mem->grid_size_ref, this->mem->distmem.grid_size_ref); + + this->template bc_set(bzl_ref, this->mem->grid_size_ref, this->mem->distmem.grid_size_ref); + this->template bc_set(bzr_ref, this->mem->grid_size_ref, this->mem->distmem.grid_size_ref); + + shrdl_ref.reset(new bcond::shared()); // TODO: shrdy if n1 != 1 + shrdr_ref.reset(new bcond::shared()); // TODO: shrdy if n1 != 1 + + this->algos.push_back( + new solver_t_( + typename solver_t_::ctor_args_t{ + { + i1, + this->mem.get(), + this->bxl, this->bxr, + i1 == 0 ? this->byl : this->shrdl, + i1 == n1 - 1 ? this->byr : this->shrdr, + this->bzl, this->bzr, + this->mem->slab(grid_size[0], i0, n0), + this->mem->slab(grid_size[1], i1, n1), + this->mem->slab(grid_size[2], i2, n2) + }, + bxl_ref, bxr_ref, + i1 == 0 ? byl_ref : shrdl_ref, + i1 == n1 - 1 ? byr_ref : shrdr_ref, + bzl_ref, bzr_ref, + }, + p + ) + ); + } + } + } + } + }; + } // namespace detail + } // namespace concurr +} // namespace libmpdataxx diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 5d408aa7..3cba9414 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -31,9 +31,18 @@ namespace libmpdataxx public: using parent_t = detail::mpdata_rhs_vip_prs_sgs_fra_common; - using parent_t::parent_t; // inheriting constructors +// using parent_t::parent_t; // inheriting constructors using real_t = typename ct_params_t::real_t; + struct ctor_args_t : parent_t::ctor_args_t + { + typename parent_t::bcp_ref_t + &bcxl_ref, &bcxr_ref, + &bcyl_ref, &bcyr_ref, + &bczl_ref, &bczr_ref; + }; + + private: // helper variables for grid refinement @@ -42,9 +51,10 @@ namespace libmpdataxx rng_t ijk_r2r_0_h_with_halo, ijk_r2r_1_h, ijk_r2r_2_h; // all positions at resolution of given iteration int stride, hstride; // stride and half stride - // fill distmem halos of refinee + // fill distmem halos of refinee, only at points overlapping with resolved points + // and not outside of the modeled domain (e.g. in periodic case) // TODO: move to bcond or sth? would be filled only by remote bcond - void fill_refinee_distmem_halos(const int e, const int halo_size) + void fill_refinee_r2r_distmem_halos(const int e, const int halo_size) { const int e_ref = this->ix_r2r.at(e); // TODO: we only need to xchng along distmem direction (x) @@ -109,6 +119,23 @@ namespace libmpdataxx } } + virtual void xchng_sclr_ref(typename parent_t::arr_t &arr, + const idx_t<3> &range_ijk + ) final // for a given array + { + this->mem->barrier(); + for (auto &bc : this->bcs_ref[1]) bc->fill_halos_sclr(arr, range_ijk[2], range_ijk[0]); + for (auto &bc : this->bcs_ref[2]) bc->fill_halos_sclr(arr, range_ijk[0], range_ijk[1]); + for (auto &bc : this->bcs_ref[0]) bc->fill_halos_sclr(arr, range_ijk[1], range_ijk[2]); + this->mem->barrier(); + } + + void xchng_ref(int e) + { + const int e_ref = this->ix_r2r.at(e); + this->xchng_sclr_ref(this->mem->psi_ref[e_ref], this->ijk_ref, this->halo_ref); + } + void refinement_ranges(const int iter, const int halo_size) { // messy, because in domain decomposition (sharedmem and distmem) some refined scalars are on the edge of the subdomain... @@ -171,7 +198,7 @@ namespace libmpdataxx //this->mem->barrier(); - fill_refinee_distmem_halos(e, halo_size); + fill_refinee_r2r_distmem_halos(e, halo_size); // fill refined array at position where it overlaps with the resolved array this->mem->refinee(e_ref)(this->ijk_r2r) = this->mem->advectee(e)(this->ijk); @@ -204,7 +231,7 @@ namespace libmpdataxx //std::cerr << "mem->grid_size_ref[1]: " << this->mem->grid_size_ref[1] << std::endl; //std::cerr << "mem->grid_size_ref[2]: " << this->mem->grid_size_ref[2] << std::endl; - fill_refinee_distmem_halos(e, halo_size); + fill_refinee_r2r_distmem_halos(e, halo_size); // fill refined array at position where it overlaps with the resolved array this->mem->refinee(e_ref)(this->ijk_r2r) = this->mem->advectee(e)(this->ijk); @@ -225,6 +252,32 @@ namespace libmpdataxx public: + mpdata_rhs_vip_prs_sgs_fra_dim( + ctor_args_t args, + const typename parent_t::rt_params_t &p + ) : + parent_t(args, p) + { + this->set_bcs(this->bcs_ref, 0, args.bcxl_ref, args.bcxr_ref); + this->set_bcs(this->bcs_ref, 1, args.bcyl_ref, args.bcyr_ref); + this->set_bcs(this->bcs_ref, 2, args.bczl_ref, args.bczr_ref); + + /* + this->bcs_ref[0][0].reset( + new bcond::bcond( + this->ijk_ref[0], + this->mem->distmem.grid_size_ref + ) + ); + */ + + /* + this->set_bcs(this->bcs_ref, 0, args.bcxl_ref, args.bcxr_ref); + this->set_bcs(this->bcs_ref, 1, args.bcyl_ref, args.bcyr_ref); + this->set_bcs(this->bcs_ref, 2, args.bczl_ref, args.bczr_ref); + */ + } + // helper method to allocate n_arr refined scalar temporary arrays static void alloc_tmp_sclr_ref( typename parent_t::mem_t *mem, diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp index cd7cdac3..84130b91 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp @@ -29,7 +29,10 @@ namespace libmpdataxx public: + static constexpr int halo_ref = 1; // halo size of refined scalar arrays using real_t = typename ct_params_t::real_t; + using bcp_ref_t = std::unique_ptr>; + using bcs_ref_t = std::array, ct_params_t::n_dims>; protected: @@ -38,8 +41,6 @@ namespace libmpdataxx const int n_ref, // number of refinements; refined resolution is dx / n_ref n_fra_iter; // number of iterations of grid refinement - - static const int halo_ref = 1; // halo size of refined scalar arrays const std::array ix_r2r; constexpr std::array get_ix_r2r() @@ -59,6 +60,9 @@ namespace libmpdataxx ijk_ref_with_halo; // same but with a halo in x direction between MPI processes const idxs_t ijk_r2r; // resolved to refined; refined scalars at the same position as resolved scalars + // boundary conditions for refined arrays + bcs_ref_t bcs_ref; + // range modifying methods used in grid refinement // TODO: unify diff --git a/libmpdata++/solvers/detail/solver_3d.hpp b/libmpdata++/solvers/detail/solver_3d.hpp index 3391dd2b..158b9bcc 100644 --- a/libmpdata++/solvers/detail/solver_3d.hpp +++ b/libmpdata++/solvers/detail/solver_3d.hpp @@ -371,9 +371,9 @@ namespace libmpdataxx this->dj = p.dj; this->dk = p.dk; this->dijk = {p.di, p.dj, p.dk}; - this->set_bcs(0, args.bcxl, args.bcxr); - this->set_bcs(1, args.bcyl, args.bcyr); - this->set_bcs(2, args.bczl, args.bczr); + this->set_bcs(this->bcs, 0, args.bcxl, args.bcxr); + this->set_bcs(this->bcs, 1, args.bcyl, args.bcyr); + this->set_bcs(this->bcs, 2, args.bczl, args.bczr); } diff --git a/libmpdata++/solvers/detail/solver_common.hpp b/libmpdata++/solvers/detail/solver_common.hpp index 90126ab2..65db9f8a 100644 --- a/libmpdata++/solvers/detail/solver_common.hpp +++ b/libmpdata++/solvers/detail/solver_common.hpp @@ -58,7 +58,8 @@ namespace libmpdataxx static constexpr bool div3_mpdata = opts::isset(ct_params_t::opts, opts::div_3rd) || opts::isset(ct_params_t::opts, opts::div_3rd_dt) ; - std::array, n_dims> bcs; + using bcs_t = std::array, n_dims>; + bcs_t bcs; const int rank; @@ -111,7 +112,8 @@ namespace libmpdataxx virtual void xchng_vctr_alng(arrvec_t&, const bool ad = false, const bool cyclic = false) = 0; - void set_bcs(const int &d, bcp_t &bcl, bcp_t &bcr) + template + void set_bcs(bcs_t_ &_bcs, const int &d, bcp_t_ &bcl, bcp_t_ &bcr) { // with distributed memory and cyclic boundary conditions, // leftmost node must send left first, as @@ -119,8 +121,8 @@ namespace libmpdataxx if (d == 0 && this->mem->distmem.size() > 0 && this->mem->distmem.rank() == 0) std::swap(bcl, bcr); - bcs[d][0] = std::move(bcl); - bcs[d][1] = std::move(bcr); + _bcs[d][0] = std::move(bcl); + _bcs[d][1] = std::move(bcr); } virtual real_t courant_number(const arrvec_t&) = 0; From 45e0dff9195abba687be8fab30e6113d1ae74942 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Fri, 21 Oct 2022 12:52:33 +0200 Subject: [PATCH 072/129] TEMP: add refinement to pbl --- tests/sandbox/pbl/pbl.hpp | 30 ++++++++++++++++++++++++------ tests/sandbox/pbl/pbl_test_def.hpp | 2 +- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/tests/sandbox/pbl/pbl.hpp b/tests/sandbox/pbl/pbl.hpp index 7ff0e255..e16744f1 100644 --- a/tests/sandbox/pbl/pbl.hpp +++ b/tests/sandbox/pbl/pbl.hpp @@ -9,20 +9,21 @@ #include #include +#include #include template class pbl : public libmpdataxx::output::hdf5_xdmf> { - using parent_t = libmpdataxx::output::hdf5_xdmf>; using ix = typename ct_params_t::ix; public: using real_t = typename ct_params_t::real_t; + using parent_t = libmpdataxx::output::hdf5_xdmf>; private: real_t hscale, iles_cdrag; - typename parent_t::arr_t &tke; + typename parent_t::arr_t &tke, &r2r_avg; void multiply_sgs_visc() { @@ -59,11 +60,12 @@ class pbl : public libmpdataxx::output::hdf5_xdmftimestep % static_cast(this->outfreq) == 0) { + if (this->rank == 0) std::cout << this->timestep << std::endl; + + // output tht refined with fractal reconstruction //this->reconstruct_refinee(ix::w); this->reconstruct_refinee(ix::tht); - if (this->rank == 0) std::cout << this->timestep << std::endl; - this->mem->barrier(); if (this->rank == 0) { @@ -77,6 +79,7 @@ class pbl : public libmpdataxx::output::hdf5_xdmfmem->barrier(); + // output tht refined with linear interpolation this->interpolate_refinee(ix::tht); this->mem->barrier(); @@ -85,6 +88,20 @@ class pbl : public libmpdataxx::output::hdf5_xdmfrecord_aux_dsc_refined("tht interpolated", this->mem->refinee(this->ix_r2r.at(ix::tht))); } this->mem->barrier(); + + // output tht on refined grid after averaging from interpolated refined tht to regular grid tht + // TODO: fill refined grid before this average! + // TODO: this will make avg_edge_sclr obsolete? + libmpdataxx::formulae::refined::spatial_average_ref2reg(this->mem->refinee(this->ix_r2r.at(ix::tht)), this->ijk_r2r, this->mem->n_ref/2, this->mem->distmem.grid_size_ref, true); + this->r2r_avg(this->ijk) = this->mem->refinee(this->ix_r2r.at(ix::tht))(this->ijk_r2r); + + this->mem->barrier(); + if (this->rank == 0) + { + //this->record_aux_dsc_refined("tht interpolated and averaged", this->mem->refinee(this->ix_r2r.at(ix::tht))); + this->record_aux_dsc("tht averaged from interpolated refined grid", this->r2r_avg); + } + this->mem->barrier(); } } @@ -103,7 +120,8 @@ class pbl : public libmpdataxx::output::hdf5_xdmftmp[__FILE__][0][0]) + tke(args.mem->tmp[__FILE__][0][0]), + r2r_avg(args.mem->tmp[__FILE__][0][1]) {} static void alloc( @@ -111,6 +129,6 @@ class pbl : public libmpdataxx::output::hdf5_xdmf void set_sgs_specific(params_t &p, iles_tag) { p.iles_cdrag = 0.1; - p.n_fra_iter = 2; + p.n_fra_iter = 1; } // smagorinsky From f5adbd8642a442ced1d71902acd1371da1a66d6f Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Fri, 21 Oct 2022 15:21:00 +0200 Subject: [PATCH 073/129] concurr_reg for separate init() --- libmpdata++/concurr/detail/concurr_common.hpp | 19 +-- .../concurr/detail/concurr_common_hlpr.hpp | 115 +----------------- ...common_hlpr.hpp => concurr_common_ref.hpp} | 17 +-- 3 files changed, 22 insertions(+), 129 deletions(-) rename libmpdata++/concurr/detail/{concurr_ref_common_hlpr.hpp => concurr_common_ref.hpp} (88%) diff --git a/libmpdata++/concurr/detail/concurr_common.hpp b/libmpdata++/concurr/detail/concurr_common.hpp index 0806766b..b69238ca 100644 --- a/libmpdata++/concurr/detail/concurr_common.hpp +++ b/libmpdata++/concurr/detail/concurr_common.hpp @@ -6,7 +6,8 @@ #pragma once -#include +#include +#include namespace libmpdataxx { @@ -40,6 +41,8 @@ namespace libmpdataxx } */ + + // default - for solvers with regular grid only template< class solver_t_, bcond::bcond_e bcxl, bcond::bcond_e bcxr, @@ -47,10 +50,10 @@ namespace libmpdataxx bcond::bcond_e bczl, bcond::bcond_e bczr, class enableif = void > - class concurr_common : public concurr_ref_common_hlpr + class concurr_common : public concurr_common_reg { public: - using parent_t = concurr_ref_common_hlpr; + using parent_t = concurr_common_reg; // ctor concurr_common( @@ -61,7 +64,7 @@ namespace libmpdataxx parent_t(p, mem_p, size) { // allocate per-thread structures - this->init(p, mem_p->grid_size, size); + this->init(p, size); } }; /* @@ -104,11 +107,11 @@ namespace libmpdataxx //typename std::enable_if_t //typename std::enable_if_t()> //typename std::enable_if_t()> - typename std::enable_if_t - > : public concurr_ref_common_hlpr + typename std::enable_if_t<(solver_t_::halo_ref > 0)> + > : public concurr_common_ref { public: - using parent_t = concurr_ref_common_hlpr; + using parent_t = concurr_common_ref; // ctor concurr_common( @@ -119,7 +122,7 @@ namespace libmpdataxx parent_t(p, mem_p, size) { // allocate per-thread structures - this->init(p, mem_p->grid_size, size); + this->init(p, size); } }; } // namespace detail diff --git a/libmpdata++/concurr/detail/concurr_common_hlpr.hpp b/libmpdata++/concurr/detail/concurr_common_hlpr.hpp index 8d03e945..cb1fe67b 100644 --- a/libmpdata++/concurr/detail/concurr_common_hlpr.hpp +++ b/libmpdata++/concurr/detail/concurr_common_hlpr.hpp @@ -180,9 +180,6 @@ namespace libmpdataxx // allocate the memory to be shared by multiple threads mem.reset(mem_p); solver_t::alloc(mem.get(), p.n_iters); - - // allocate per-thread structures -// this->init(p, mem->grid_size, size); } protected: @@ -256,81 +253,8 @@ namespace libmpdataxx ); } - // 1D version - virtual void init( - const typename solver_t::rt_params_t &p, - const std::array &grid_size, const int &n0 - ) - { - typename solver_t::bcp_t bxl, bxr, shrdl, shrdr; - - // NOTE: for remote bcond, thread_rank set to 0 on purpose in 1D to have propre left/right message tags - bc_set(bxl, mem->grid_size, mem->distmem.grid_size); - bc_set(bxr, mem->grid_size, mem->distmem.grid_size); - - for (int i0 = 0; i0 < n0; ++i0) - { - shrdl.reset(new bcond::shared()); - shrdr.reset(new bcond::shared()); - - algos.push_back( - new solver_t( - typename solver_t::ctor_args_t({ - i0, - mem.get(), - i0 == 0 ? bxl : shrdl, - i0 == n0 - 1 ? bxr : shrdr, - mem->slab(grid_size[0], i0, n0) - }), - p - ) - ); - } - } - - // 2D version - // TODO: assert parallelisation in the right dimensions! (blitz::assertContiguous) - virtual void init( - const typename solver_t::rt_params_t &p, - const std::array &grid_size, - const int &n0, const int &n1 = 1 - ) { - for (int i0 = 0; i0 < n0; ++i0) - { - for (int i1 = 0; i1 < n1; ++i1) - { - typename solver_t::bcp_t bxl, bxr, byl, byr, shrdl, shrdr; - - // NOTE: for remote bcond, thread_rank set to 0 on purpose in 2D to have propre left/right message tags - bc_set(bxl, mem->grid_size, mem->distmem.grid_size); - bc_set(bxr, mem->grid_size, mem->distmem.grid_size); - - bc_set(byl, mem->grid_size, mem->distmem.grid_size); - bc_set(byr, mem->grid_size, mem->distmem.grid_size); - - shrdl.reset(new bcond::shared()); // TODO: shrdy if n1 != 1 - shrdr.reset(new bcond::shared()); // TODO: shrdy if n1 != 1 - - algos.push_back( - new solver_t( - typename solver_t::ctor_args_t({ - i0, - mem.get(), - i0 == 0 ? bxl : shrdl, - i0 == n0 - 1 ? bxr : shrdr, - byl, byr, - mem->slab(grid_size[0], i0, n0), - mem->slab(grid_size[1], i1, n1) - }), - p - ) - ); - } - } - } - - // 3D version, note sharedmem in y direction! - void init_bcs(const int &i1, const int &n1) + // 3D version + void init_bcs_3d(const int &i1, const int &n1) { // i1 is the local thread rank, n1 is the number of threads. These are needed by remote bcond, because only rank=0 does mpi communication bc_set(bxl, mem->grid_size, mem->distmem.grid_size, i1, n1); @@ -346,41 +270,6 @@ namespace libmpdataxx shrdr.reset(new bcond::shared()); // TODO: shrdy if n1 != 1 } - virtual void init( - const typename solver_t::rt_params_t &p, - const std::array &grid_size, - const int &n1, const int &n0 = 1, const int &n2 = 1 - ) { - // TODO: renew pointers only if invalid ? - for (int i0 = 0; i0 < n0; ++i0) - { - for (int i1 = 0; i1 < n1; ++i1) - { - for (int i2 = 0; i2 < n2; ++i2) - { - init_bcs(i1, n1); - - algos.push_back( - new solver_t( - typename solver_t::ctor_args_t({ - i1, - mem.get(), - bxl, bxr, - i1 == 0 ? byl : shrdl, - i1 == n1 - 1 ? byr : shrdr, - bzl, bzr, - mem->slab(grid_size[0], i0, n0), - mem->slab(grid_size[1], i1, n1), - mem->slab(grid_size[2], i2, n2) - }), - p - ) - ); - } - } - } - } - private: virtual void solve(advance_arg_t nt) = 0; diff --git a/libmpdata++/concurr/detail/concurr_ref_common_hlpr.hpp b/libmpdata++/concurr/detail/concurr_common_ref.hpp similarity index 88% rename from libmpdata++/concurr/detail/concurr_ref_common_hlpr.hpp rename to libmpdata++/concurr/detail/concurr_common_ref.hpp index 8f5def7d..e5cdfb7d 100644 --- a/libmpdata++/concurr/detail/concurr_ref_common_hlpr.hpp +++ b/libmpdata++/concurr/detail/concurr_common_ref.hpp @@ -4,6 +4,8 @@ * GPLv3+ (see the COPYING file or http://www.gnu.org/licenses/) */ +// for solvers with both a refined and regular grid + #pragma once #include @@ -20,7 +22,7 @@ namespace libmpdataxx bcond::bcond_e bcyl, bcond::bcond_e bcyr, bcond::bcond_e bczl, bcond::bcond_e bczr > - class concurr_ref_common_hlpr : public concurr_common_hlpr + class concurr_common_ref : public concurr_common_hlpr { private: @@ -29,11 +31,10 @@ namespace libmpdataxx protected: // 3D version, note sharedmem in y direction! - virtual void init( + void init( const typename solver_t_::rt_params_t &p, - const std::array &grid_size, const int &n1, const int &n0 = 1, const int &n2 = 1 - ) override { + ) { typename solver_t_::bcp_ref_t bxl_ref, bxr_ref, byl_ref, byr_ref, bzl_ref, bzr_ref, shrdl_ref, shrdr_ref; @@ -45,7 +46,7 @@ namespace libmpdataxx for (int i2 = 0; i2 < n2; ++i2) { // i1 is the local thread rank, n1 is the number of threads. These are needed by remote bcond, because only rank=0 does mpi communication - this->init_bcs(i1, n1); + this->init_bcs_3d(i1, n1); this->template bc_set(bxl_ref, this->mem->grid_size_ref, this->mem->distmem.grid_size_ref, i1, n1); this->template bc_set(bxr_ref, this->mem->grid_size_ref, this->mem->distmem.grid_size_ref, i1, n1); @@ -69,9 +70,9 @@ namespace libmpdataxx i1 == 0 ? this->byl : this->shrdl, i1 == n1 - 1 ? this->byr : this->shrdr, this->bzl, this->bzr, - this->mem->slab(grid_size[0], i0, n0), - this->mem->slab(grid_size[1], i1, n1), - this->mem->slab(grid_size[2], i2, n2) + this->mem->slab(this->mem->grid_size[0], i0, n0), + this->mem->slab(this->mem->grid_size[1], i1, n1), + this->mem->slab(this->mem->grid_size[2], i2, n2) }, bxl_ref, bxr_ref, i1 == 0 ? byl_ref : shrdl_ref, From bbeb89bc97965753da17a2655f2e0131d26238d6 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Mon, 24 Oct 2022 20:30:14 +0200 Subject: [PATCH 074/129] refined courant spatial interpolation --- .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 75 ++++++++++++++----- .../mpdata_rhs_vip_prs_sgs_fra_common.hpp | 8 +- libmpdata++/solvers/mpdata_rhs_vip.hpp | 5 +- 3 files changed, 67 insertions(+), 21 deletions(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 3cba9414..cd4a30d0 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -119,23 +119,6 @@ namespace libmpdataxx } } - virtual void xchng_sclr_ref(typename parent_t::arr_t &arr, - const idx_t<3> &range_ijk - ) final // for a given array - { - this->mem->barrier(); - for (auto &bc : this->bcs_ref[1]) bc->fill_halos_sclr(arr, range_ijk[2], range_ijk[0]); - for (auto &bc : this->bcs_ref[2]) bc->fill_halos_sclr(arr, range_ijk[0], range_ijk[1]); - for (auto &bc : this->bcs_ref[0]) bc->fill_halos_sclr(arr, range_ijk[1], range_ijk[2]); - this->mem->barrier(); - } - - void xchng_ref(int e) - { - const int e_ref = this->ix_r2r.at(e); - this->xchng_sclr_ref(this->mem->psi_ref[e_ref], this->ijk_ref, this->halo_ref); - } - void refinement_ranges(const int iter, const int halo_size) { // messy, because in domain decomposition (sharedmem and distmem) some refined scalars are on the edge of the subdomain... @@ -187,6 +170,7 @@ namespace libmpdataxx protected: + // calculate refined points using (tri?)linear interpolation void interpolate_refinee(const int e = 0) { assert(opts::isset(ct_params_t::fractal_recon, opts::bit(e))); @@ -250,6 +234,36 @@ namespace libmpdataxx this->mem->barrier(); } + virtual void xchng_vctr_ref( + arrvec_t &arrvec, + const idx_t<3> &range_ijk + ) final + { + this->mem->barrier(); + for (auto &bc : this->bcs_ref[1]) bc->fill_halos_sclr(arrvec[1], range_ijk[2], range_ijk[0]); + for (auto &bc : this->bcs_ref[2]) bc->fill_halos_sclr(arrvec[2], range_ijk[0], range_ijk[1]); + for (auto &bc : this->bcs_ref[0]) bc->fill_halos_sclr(arrvec[0], range_ijk[1], range_ijk[2]); + this->mem->barrier(); + } + +// void xchng_ref(int e) +// { +// const int e_ref = this->ix_r2r.at(e); +// this->xchng_sclr_ref(this->mem->psi_ref[e_ref], this->ijk_ref); +// } + + // calculate courant numbers at refined cell edges using a + // linear interpolation of a refined uvw (src) from cell centers to cell edges (dste + void interpolate_refined_courants(arrvec_t &dst, + arrvec_t &src) + { + this->xchng_vctr_ref(src, this->ijk_ref); + this->template intrp<0>(dst[0], src[0], this->ijk_ref[0]^h, this->ijk_ref[1], this->ijk_ref[2], this->dijk_ref[0], true); + this->template intrp<1>(dst[1], src[1], this->ijk_ref[1]^h, this->ijk_ref[2], this->ijk_ref[0], this->dijk_ref[1], true); + this->template intrp<2>(dst[2], src[2], this->ijk_ref[2]^h, this->ijk_ref[0], this->ijk_ref[1], this->dijk_ref[2], true); + this->mem->barrier(); + } + public: mpdata_rhs_vip_prs_sgs_fra_dim( @@ -258,6 +272,12 @@ namespace libmpdataxx ) : parent_t(args, p) { + this->dijk_ref = {this->di / this->n_ref, this->dj / this->n_ref, this->dk / this->n_ref}; + this->ijkm_ref = this->ijk_ref; + this->ijkm_ref.lbound(0) = this->ijkm_ref.lbound(0) - 1; + this->ijkm_ref.lbound(1) = this->ijkm_ref.lbound(1) - 1; + this->ijkm_ref.lbound(2) = this->ijkm_ref.lbound(2) - 1; + this->set_bcs(this->bcs_ref, 0, args.bcxl_ref, args.bcxr_ref); this->set_bcs(this->bcs_ref, 1, args.bcyl_ref, args.bcyr_ref); this->set_bcs(this->bcs_ref, 2, args.bczl_ref, args.bczr_ref); @@ -298,6 +318,27 @@ namespace libmpdataxx arr3D_storage ))); } + + static void alloc_tmp_vctr_ref( + typename parent_t::mem_t *mem, + const char * __file__ + ) + { + const int n_arr = 3; + const std::vector> stgr{{true, false, false}, {false, true, false}, {false, false, true}}; + + mem->tmp[__file__].push_back(new arrvec_t()); + for (int n = 0; n < n_arr; ++n) + { + mem->tmp[__file__].back().push_back(mem->old(new typename parent_t::arr_t( + stgr[n][0] ? mem->grid_size_ref[0]^h : mem->grid_size_ref[0], + stgr[n][1] ? mem->grid_size_ref[1]^h : mem->grid_size_ref[1], + stgr[n][2] ? mem->grid_size_ref[2]^h : mem->grid_size_ref[2], + arr3D_storage + ))); + } + } + }; } // namespace detail } // namespace solvers diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp index 84130b91..c3e4f667 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp @@ -56,13 +56,17 @@ namespace libmpdataxx // herlper ranges // TODO: make these const! - idx_t ijk_ref, // range of refinee handled by given solver - ijk_ref_with_halo; // same but with a halo in x direction between MPI processes + idx_t ijk_ref, // range of refinee handled by given solver + ijkm_ref, // same but starting at 1 to the left + ijk_ref_with_halo; // as ijk_ref but with a halo in x direction between MPI processes const idxs_t ijk_r2r; // resolved to refined; refined scalars at the same position as resolved scalars // boundary conditions for refined arrays bcs_ref_t bcs_ref; + std::array dijk_ref; + + // range modifying methods used in grid refinement // TODO: unify diff --git a/libmpdata++/solvers/mpdata_rhs_vip.hpp b/libmpdata++/solvers/mpdata_rhs_vip.hpp index a037049e..e523d091 100644 --- a/libmpdata++/solvers/mpdata_rhs_vip.hpp +++ b/libmpdata++/solvers/mpdata_rhs_vip.hpp @@ -212,14 +212,15 @@ namespace libmpdataxx const rng_t &i, const rng_t &j, const rng_t &k, - const typename ct_params_t::real_t &di + const typename ct_params_t::real_t &di, + const bool &noG = false ) { using idxperm::pi; using namespace arakawa_c; using real_t = typename ct_params_t::real_t; - if (!this->mem->G) + if (!this->mem->G || noG) { dst(pi(i+h, j, k)) = this->dt / di * real_t(.5) * ( src(pi(i, j, k)) + From 199d9c9c0ec1a4af03829407ed559ef05bf843b5 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Tue, 25 Oct 2022 13:43:56 +0200 Subject: [PATCH 075/129] reconstruction: barrier after stretching parameters are created --- libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index cd4a30d0..7095e538 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -221,6 +221,7 @@ namespace libmpdataxx this->mem->refinee(e_ref)(this->ijk_r2r) = this->mem->advectee(e)(this->ijk); generate_stretching_parameters(); + this->mem->barrier(); for(int i=0; in_fra_iter; ++i) { @@ -228,7 +229,7 @@ namespace libmpdataxx formulae::fractal::rcnstrct<0, real_t>(this->mem->psi_ref[e_ref], this->rng_dbl_stride(mid_ijk_r2r_0), ijk_r2r_1_h, ijk_r2r_2_h, hstride, this->c_j, this->d_j, this->f_j); this->mem->barrier(); - formulae::fractal::rcnstrct<1, real_t>(this->mem->psi_ref[e_ref], this->rng_dbl_stride(mid_ijk_r2r_1) , ijk_r2r_2_h, ijk_r2r_0_h_with_halo, hstride, this->c_j, this->d_j, this->f_j); + formulae::fractal::rcnstrct<1, real_t>(this->mem->psi_ref[e_ref], this->rng_dbl_stride(mid_ijk_r2r_1) , ijk_r2r_2_h, ijk_r2r_0_h_with_halo, hstride, this->c_j, this->d_j, this->f_j); // NOTE: rng_dbl_stride(mid_ijk_r2r_1) gives overlapping ranges between thread subdomains... however it seems to work and naive fixes didnt work formulae::fractal::rcnstrct<2, real_t>(this->mem->psi_ref[e_ref], this->rng_dbl_stride(mid_ijk_r2r_2) , ijk_r2r_0_h_with_halo, this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride, this->c_j, this->d_j, this->f_j); } this->mem->barrier(); From 7194a954d1ff212e43b7966e066bd19b8c552db6 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Wed, 26 Oct 2022 15:19:19 +0200 Subject: [PATCH 076/129] fix remote refined bconds --- .../concurr/detail/concurr_common_hlpr.hpp | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/libmpdata++/concurr/detail/concurr_common_hlpr.hpp b/libmpdata++/concurr/detail/concurr_common_hlpr.hpp index cb1fe67b..ffc76112 100644 --- a/libmpdata++/concurr/detail/concurr_common_hlpr.hpp +++ b/libmpdata++/concurr/detail/concurr_common_hlpr.hpp @@ -50,21 +50,25 @@ namespace libmpdataxx int n_dims, int halo, class bcp_t, - class mem_t + class mem_t, + class grid_size_t, + class distmem_grid_size_t > struct bc_set_remote_impl { static void _( bcp_t &bcp, const std::unique_ptr &mem, + const grid_size_t grid_size, + const distmem_grid_size_t distmem_grid_size, const int thread_rank, const int thread_size ) { bcp.reset( new bcond::bcond( - mem->slab(mem->grid_size[dim]), - mem->distmem.grid_size, + mem->slab(grid_size[dim]), + distmem_grid_size, thread_rank, thread_size ) @@ -79,22 +83,26 @@ namespace libmpdataxx int dim, int halo, class bcp_t, - class mem_t + class mem_t, + class grid_size_t, + class distmem_grid_size_t > - struct bc_set_remote_impl + struct bc_set_remote_impl { static void _( bcp_t &bcp, const std::unique_ptr &mem, + const grid_size_t grid_size, + const distmem_grid_size_t distmem_grid_size, const int thread_rank, const int thread_size ) { bcp.reset( new bcond::bcond( - mem->slab(mem->grid_size[dim]), - mem->distmem.grid_size, - mem->slab(mem->grid_size[1], thread_rank, thread_size), // NOTE: we assume here remote 3d bcond is only on the edges perpendicular to x + mem->slab(grid_size[dim]), + distmem_grid_size, + mem->slab(grid_size[1], thread_rank, thread_size), // NOTE: we assume here remote 3d bcond is only on the edges perpendicular to x thread_rank, thread_size ) @@ -109,16 +117,20 @@ namespace libmpdataxx int n_dims, int halo, class bcp_t, - class mem_t + class mem_t, + class grid_size_t, + class distmem_grid_size_t > void bc_set_remote( bcp_t &bcp, const std::unique_ptr &mem, + const grid_size_t grid_size, + const distmem_grid_size_t distmem_grid_size, const int thread_rank, const int thread_size ) { - bc_set_remote_impl::_(bcp, mem, thread_rank, thread_size); + bc_set_remote_impl::_(bcp, mem, grid_size, distmem_grid_size, thread_rank, thread_size); } template< @@ -222,6 +234,8 @@ namespace libmpdataxx bc_set_remote( bcp, mem, + grid_size, + distmem_grid_size, thread_rank, thread_size ); From 3577129fe649e71829d1c9af1c83f4ec7b39b328 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Thu, 27 Oct 2022 15:04:35 +0200 Subject: [PATCH 077/129] mpi grid refinemet wip --- libmpdata++/bcond/detail/remote_3d_common.hpp | 7 ++- libmpdata++/bcond/detail/remote_common.hpp | 42 +++++++------ .../concurr/detail/concurr_common_hlpr.hpp | 62 ++++++++++++++----- .../concurr/detail/concurr_common_ref.hpp | 30 ++++++++- .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 4 ++ 5 files changed, 106 insertions(+), 39 deletions(-) diff --git a/libmpdata++/bcond/detail/remote_3d_common.hpp b/libmpdata++/bcond/detail/remote_3d_common.hpp index fa370053..fa0d9eb1 100644 --- a/libmpdata++/bcond/detail/remote_3d_common.hpp +++ b/libmpdata++/bcond/detail/remote_3d_common.hpp @@ -32,10 +32,10 @@ namespace libmpdataxx // based on the difference between idx to be sent by this thread and idx of this process idx_t extend_idx(idx_t idx) { -//std::cerr << "extend idx start idx(1): " << idx.lbound(1) << ", " << idx.ubound(1) << std::endl; +std::cerr << "extend idx start idx(1): " << idx.lbound(1) << ", " << idx.ubound(1) << std::endl; idx.lbound(1) = 0 + idx.lbound(1) - thread_j.first(); // does it have to start at 0? idx.ubound(1) = (grid_size_y - 1) + idx.ubound(1) - thread_j.last(); // does it have to end at grid_size_y - 1? -//std::cerr << "extend idx end idx(1): " << idx.lbound(1) << ", " << idx.ubound(1) << std::endl; +std::cerr << "extend idx end idx(1): " << idx.lbound(1) << ", " << idx.ubound(1) << std::endl; return idx; } @@ -68,13 +68,14 @@ namespace libmpdataxx // ctor remote_3d_common( + boost::mpi::communicator &mpic, const rng_t &i, const std::array &distmem_grid_size, const rng_t _thread_j, const int thread_rank, const int thread_size ) : - parent_t(i, distmem_grid_size, true, thread_rank, thread_size), // true indicating that this is a bcond done with a single thread + parent_t(mpic, i, distmem_grid_size, true, thread_rank, thread_size), // true indicating that this is a bcond done with a single thread thread_j(_thread_j), grid_size_y(distmem_grid_size[1]) { diff --git a/libmpdata++/bcond/detail/remote_common.hpp b/libmpdata++/bcond/detail/remote_common.hpp index ac90d41d..7be11176 100644 --- a/libmpdata++/bcond/detail/remote_common.hpp +++ b/libmpdata++/bcond/detail/remote_common.hpp @@ -78,16 +78,19 @@ namespace libmpdataxx const int msg_send = dir == left ? left : rght; -// std::cerr << "send_hlpr idx dir " << dir << " : " -// << " (" << idx_send.lbound(0) << ", " << idx_send.ubound(0) << ")" -// << " (" << idx_send.lbound(1) << ", " << idx_send.ubound(1) << ")" -// << " (" << idx_send.lbound(2) << ", " << idx_send.ubound(2) << ")" -// << std::endl; + std::cerr << "send_hlpr idx dir " << dir << " : " + << " (" << idx_send.lbound(0) << ", " << idx_send.ubound(0) << ")" + << " (" << idx_send.lbound(1) << ", " << idx_send.ubound(1) << ")" + << " (" << idx_send.lbound(2) << ", " << idx_send.ubound(2) << ")" + << std::endl; + + std::cerr << "buf_send: " << buf_send << std::endl; // arr_send references part of the send buffer that will be used arr_t arr_send(buf_send, a(idx_send).shape(), blitz::neverDeleteData); // copying data to be sent arr_send = a(idx_send); + std::cerr << "arr_send: " << arr_send << std::endl; // launching async data transfer if(arr_send.size()!=0) @@ -118,11 +121,11 @@ namespace libmpdataxx const int msg_recv = dir == left ? rght : left; -// std::cerr << "recv_hlpr idx dir " << dir << " : " -// << " (" << idx_recv.lbound(0) << ", " << idx_recv.ubound(0) << ")" -// << " (" << idx_recv.lbound(1) << ", " << idx_recv.ubound(1) << ")" -// << " (" << idx_recv.lbound(2) << ", " << idx_recv.ubound(2) << ")" -// << std::endl; + std::cerr << "recv_hlpr idx dir " << dir << " : " + << " (" << idx_recv.lbound(0) << ", " << idx_recv.ubound(0) << ")" + << " (" << idx_recv.lbound(1) << ", " << idx_recv.ubound(1) << ")" + << " (" << idx_recv.lbound(2) << ", " << idx_recv.ubound(2) << ")" + << std::endl; // launching async data transfer @@ -218,24 +221,27 @@ namespace libmpdataxx // ctor remote_common( + boost::mpi::communicator &mpic, const rng_t &i, const std::array &distmem_grid_size, bool single_threaded = false, const int thread_rank = -1, - const int thread_size = -1 + const int thread_size = -1 ) : parent_t(i, distmem_grid_size, single_threaded, thread_rank, thread_size) + //,mpicom(mpic, boost::mpi::comm_take_ownership) + ,mpicom(mpic, boost::mpi::comm_attach) { #if defined(USE_MPI) const int slice_size = n_dims==1 ? 1 : (n_dims==2? distmem_grid_size[1]+6 : (distmem_grid_size[1]+6) * (distmem_grid_size[2]+6) ); // 3 is the max halo size (?), so 6 on both sides -//std::cerr << "remote_common ctor, " -// << " distmem_grid_size[0]: " << distmem_grid_size[0] -// << " distmem_grid_size[1]: " << distmem_grid_size[1] -// << " distmem_grid_size[2]: " << distmem_grid_size[2] -// << " slice_size: " << slice_size -// << " halo: " << halo -// << std::endl; +std::cerr << "remote_common ctor, " + << " distmem_grid_size[0]: " << distmem_grid_size[0] + << " distmem_grid_size[1]: " << distmem_grid_size[1] + << " distmem_grid_size[2]: " << distmem_grid_size[2] + << " slice_size: " << slice_size + << " halo: " << halo + << std::endl; // allocate enough memory in buffers to store largest halos to be sent buf_send = (real_t *) malloc(halo * slice_size * sizeof(real_t)); buf_recv = (real_t *) malloc(halo * slice_size * sizeof(real_t)); diff --git a/libmpdata++/concurr/detail/concurr_common_hlpr.hpp b/libmpdata++/concurr/detail/concurr_common_hlpr.hpp index ffc76112..034dedea 100644 --- a/libmpdata++/concurr/detail/concurr_common_hlpr.hpp +++ b/libmpdata++/concurr/detail/concurr_common_hlpr.hpp @@ -62,11 +62,13 @@ namespace libmpdataxx const grid_size_t grid_size, const distmem_grid_size_t distmem_grid_size, const int thread_rank, - const int thread_size + const int thread_size, + boost::mpi::communicator &mpic ) { bcp.reset( new bcond::bcond( + mpic, mem->slab(grid_size[dim]), distmem_grid_size, thread_rank, @@ -95,11 +97,13 @@ namespace libmpdataxx const grid_size_t grid_size, const distmem_grid_size_t distmem_grid_size, const int thread_rank, - const int thread_size + const int thread_size, + boost::mpi::communicator &mpic ) { bcp.reset( new bcond::bcond( + mpic, mem->slab(grid_size[dim]), distmem_grid_size, mem->slab(grid_size[1], thread_rank, thread_size), // NOTE: we assume here remote 3d bcond is only on the edges perpendicular to x @@ -127,10 +131,11 @@ namespace libmpdataxx const grid_size_t grid_size, const distmem_grid_size_t distmem_grid_size, const int thread_rank, - const int thread_size + const int thread_size, + boost::mpi::communicator &mpic ) { - bc_set_remote_impl::_(bcp, mem, grid_size, distmem_grid_size, thread_rank, thread_size); + bc_set_remote_impl::_(bcp, mem, grid_size, distmem_grid_size, thread_rank, thread_size, mpic); } template< @@ -203,14 +208,16 @@ namespace libmpdataxx int halo, class bcp_t, class grid_size_t, - class distmem_grid_size_t + class distmem_grid_size_t, + class mpicomm_t > void bc_set( bcp_t &bcp, const grid_size_t grid_size, const distmem_grid_size_t distmem_grid_size, - const int thread_rank = 0, // required only by 3D remote (MPI) and open bconds - const int thread_size = 0 // required only by 3D remote (MPI) and open bconds + mpicomm_t &mpic, + const int thread_rank = 0, // required only by 3D remote (MPI) and open bconds + const int thread_size = 0 // required only by 3D remote (MPI) and open bconds ) { // sanity check - polar coords do not work with MPI yet @@ -237,7 +244,8 @@ namespace libmpdataxx grid_size, distmem_grid_size, thread_rank, - thread_size + thread_size, + mpic ); return; } @@ -268,20 +276,40 @@ namespace libmpdataxx } // 3D version - void init_bcs_3d(const int &i1, const int &n1) + template + void init_bcs_3d( + bcp_t &bxl, bcp_t &bxr, bcp_t &byl, bcp_t &byr, bcp_t &bzl, bcp_t &bzr, bcp_t &shrdl, bcp_t &shrdr, + const grid_size_t &grid_size, + const distmem_grid_size_t &distmem_grid_size, + const int &i1, const int &n1, + boost::mpi::communicator &mpic) { // i1 is the local thread rank, n1 is the number of threads. These are needed by remote bcond, because only rank=0 does mpi communication - bc_set(bxl, mem->grid_size, mem->distmem.grid_size, i1, n1); - bc_set(bxr, mem->grid_size, mem->distmem.grid_size, i1, n1); +#if defined(USE_MPI) + //boost::mpi::communicator mpic(MPI_COMM_WORLD, boost::mpi::comm_duplicate); + //boost::mpi::communicator mpic(MPI_COMM_WORLD, boost::mpi::comm_attach); + + bc_set(bxl, grid_size, distmem_grid_size, mpic, i1, n1); + bc_set(bxr, grid_size, distmem_grid_size, mpic, i1, n1); + + bc_set(byl, grid_size, distmem_grid_size, mpic); + bc_set(byr, grid_size, distmem_grid_size, mpic); + + bc_set(bzl, grid_size, distmem_grid_size, mpic); + bc_set(bzr, grid_size, distmem_grid_size, mpic); +#else + bc_set(bxl, grid_size, distmem_grid_size, i1, n1); + bc_set(bxr, grid_size, distmem_grid_size, i1, n1); - bc_set(byl, mem->grid_size, mem->distmem.grid_size); - bc_set(byr, mem->grid_size, mem->distmem.grid_size); + bc_set(byl, grid_size, distmem_grid_size); + bc_set(byr, grid_size, distmem_grid_size); - bc_set(bzl, mem->grid_size, mem->distmem.grid_size); - bc_set(bzr, mem->grid_size, mem->distmem.grid_size); + bc_set(bzl, grid_size, distmem_grid_size); + bc_set(bzr, grid_size, distmem_grid_size); +#endif - shrdl.reset(new bcond::shared()); // TODO: shrdy if n1 != 1 - shrdr.reset(new bcond::shared()); // TODO: shrdy if n1 != 1 + shrdl.reset(new bcond::shared()); // TODO: shrdy if n1 != 1 + shrdr.reset(new bcond::shared()); // TODO: shrdy if n1 != 1 } private: diff --git a/libmpdata++/concurr/detail/concurr_common_ref.hpp b/libmpdata++/concurr/detail/concurr_common_ref.hpp index e5cdfb7d..cfa124d9 100644 --- a/libmpdata++/concurr/detail/concurr_common_ref.hpp +++ b/libmpdata++/concurr/detail/concurr_common_ref.hpp @@ -27,7 +27,21 @@ namespace libmpdataxx private: using parent_t = concurr_common_hlpr; - using parent_t::parent_t; +// using parent_t::parent_t; + + boost::mpi::communicator mpic, mpic_ref; + + public: + // ctor + concurr_common_ref( + const typename solver_t_::rt_params_t &p, + typename solver_t_::mem_t *mem_p, + const int &size + ) : + parent_t(p, mem_p, size), + mpic(MPI_COMM_WORLD, boost::mpi::comm_duplicate), + mpic_ref(MPI_COMM_WORLD, boost::mpi::comm_duplicate) + {} protected: // 3D version, note sharedmem in y direction! @@ -46,6 +60,19 @@ namespace libmpdataxx for (int i2 = 0; i2 < n2; ++i2) { // i1 is the local thread rank, n1 is the number of threads. These are needed by remote bcond, because only rank=0 does mpi communication + this->template init_bcs_3d( + this->bxl, this->bxr, this->byl, this->byr, this->bzl, this->bzr, this->shrdl, this->shrdr, + this->mem->grid_size, + this->mem->distmem.grid_size, + i1, n1, mpic); + + this->template init_bcs_3d( + bxl_ref, bxr_ref, byl_ref, byr_ref, bzl_ref, bzr_ref, shrdl_ref, shrdr_ref, + this->mem->grid_size_ref, + this->mem->distmem.grid_size_ref, + i1, n1, mpic_ref); + + /* this->init_bcs_3d(i1, n1); this->template bc_set(bxl_ref, this->mem->grid_size_ref, this->mem->distmem.grid_size_ref, i1, n1); @@ -59,6 +86,7 @@ namespace libmpdataxx shrdl_ref.reset(new bcond::shared()); // TODO: shrdy if n1 != 1 shrdr_ref.reset(new bcond::shared()); // TODO: shrdy if n1 != 1 + */ this->algos.push_back( new solver_t_( diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 7095e538..fbd435b7 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -258,7 +258,11 @@ namespace libmpdataxx void interpolate_refined_courants(arrvec_t &dst, arrvec_t &src) { + std::cerr << "interpolate_refined_courants src[0](ijk_ref): " << src[0](this->ijk_ref) << std::endl; + std::cerr << "interpolate_refined_courants src[1](ijk_ref): " << src[1](this->ijk_ref) << std::endl; + std::cerr << "interpolate_refined_courants src[2](ijk_ref): " << src[2](this->ijk_ref) << std::endl; this->xchng_vctr_ref(src, this->ijk_ref); + std::cerr << "post xchng vctr ref" << std::endl; this->template intrp<0>(dst[0], src[0], this->ijk_ref[0]^h, this->ijk_ref[1], this->ijk_ref[2], this->dijk_ref[0], true); this->template intrp<1>(dst[1], src[1], this->ijk_ref[1]^h, this->ijk_ref[2], this->ijk_ref[0], this->dijk_ref[1], true); this->template intrp<2>(dst[2], src[2], this->ijk_ref[2]^h, this->ijk_ref[0], this->ijk_ref[1], this->dijk_ref[2], true); From 900c07cb10ff3e7724ea562c84b3da9a178a2c4b Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Fri, 28 Oct 2022 13:14:26 +0200 Subject: [PATCH 078/129] xchng_ref --- .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index fbd435b7..8401c130 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -235,10 +235,23 @@ namespace libmpdataxx this->mem->barrier(); } - virtual void xchng_vctr_ref( + void xchng_ref( + typename parent_t::arr_t &arr, + const idx_t<3> &range_ijk + ) + { + this->mem->barrier(); + for (auto &bc : this->bcs_ref[1]) bc->fill_halos_sclr(arr, range_ijk[2], range_ijk[0]); + for (auto &bc : this->bcs_ref[2]) bc->fill_halos_sclr(arr, range_ijk[0], range_ijk[1]); + for (auto &bc : this->bcs_ref[0]) bc->fill_halos_sclr(arr, range_ijk[1], range_ijk[2]); + this->mem->barrier(); + } + + + void xchng_vctr_ref( arrvec_t &arrvec, const idx_t<3> &range_ijk - ) final + ) { this->mem->barrier(); for (auto &bc : this->bcs_ref[1]) bc->fill_halos_sclr(arrvec[1], range_ijk[2], range_ijk[0]); From e4dc451295ad8d1695ffb0bdb4fc7f41b101ca5f Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Fri, 28 Oct 2022 13:26:09 +0200 Subject: [PATCH 079/129] delete debug output --- libmpdata++/bcond/detail/remote_3d_common.hpp | 2 -- libmpdata++/bcond/detail/remote_common.hpp | 23 ------------------- .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 4 ---- 3 files changed, 29 deletions(-) diff --git a/libmpdata++/bcond/detail/remote_3d_common.hpp b/libmpdata++/bcond/detail/remote_3d_common.hpp index fa0d9eb1..54fc4e8d 100644 --- a/libmpdata++/bcond/detail/remote_3d_common.hpp +++ b/libmpdata++/bcond/detail/remote_3d_common.hpp @@ -32,10 +32,8 @@ namespace libmpdataxx // based on the difference between idx to be sent by this thread and idx of this process idx_t extend_idx(idx_t idx) { -std::cerr << "extend idx start idx(1): " << idx.lbound(1) << ", " << idx.ubound(1) << std::endl; idx.lbound(1) = 0 + idx.lbound(1) - thread_j.first(); // does it have to start at 0? idx.ubound(1) = (grid_size_y - 1) + idx.ubound(1) - thread_j.last(); // does it have to end at grid_size_y - 1? -std::cerr << "extend idx end idx(1): " << idx.lbound(1) << ", " << idx.ubound(1) << std::endl; return idx; } diff --git a/libmpdata++/bcond/detail/remote_common.hpp b/libmpdata++/bcond/detail/remote_common.hpp index 7be11176..47a56ef2 100644 --- a/libmpdata++/bcond/detail/remote_common.hpp +++ b/libmpdata++/bcond/detail/remote_common.hpp @@ -78,19 +78,10 @@ namespace libmpdataxx const int msg_send = dir == left ? left : rght; - std::cerr << "send_hlpr idx dir " << dir << " : " - << " (" << idx_send.lbound(0) << ", " << idx_send.ubound(0) << ")" - << " (" << idx_send.lbound(1) << ", " << idx_send.ubound(1) << ")" - << " (" << idx_send.lbound(2) << ", " << idx_send.ubound(2) << ")" - << std::endl; - - std::cerr << "buf_send: " << buf_send << std::endl; - // arr_send references part of the send buffer that will be used arr_t arr_send(buf_send, a(idx_send).shape(), blitz::neverDeleteData); // copying data to be sent arr_send = a(idx_send); - std::cerr << "arr_send: " << arr_send << std::endl; // launching async data transfer if(arr_send.size()!=0) @@ -121,13 +112,6 @@ namespace libmpdataxx const int msg_recv = dir == left ? rght : left; - std::cerr << "recv_hlpr idx dir " << dir << " : " - << " (" << idx_recv.lbound(0) << ", " << idx_recv.ubound(0) << ")" - << " (" << idx_recv.lbound(1) << ", " << idx_recv.ubound(1) << ")" - << " (" << idx_recv.lbound(2) << ", " << idx_recv.ubound(2) << ")" - << std::endl; - - // launching async data transfer if(a(idx_recv).size()!=0) // TODO: test directly size of idx_recv { @@ -235,13 +219,6 @@ namespace libmpdataxx #if defined(USE_MPI) const int slice_size = n_dims==1 ? 1 : (n_dims==2? distmem_grid_size[1]+6 : (distmem_grid_size[1]+6) * (distmem_grid_size[2]+6) ); // 3 is the max halo size (?), so 6 on both sides -std::cerr << "remote_common ctor, " - << " distmem_grid_size[0]: " << distmem_grid_size[0] - << " distmem_grid_size[1]: " << distmem_grid_size[1] - << " distmem_grid_size[2]: " << distmem_grid_size[2] - << " slice_size: " << slice_size - << " halo: " << halo - << std::endl; // allocate enough memory in buffers to store largest halos to be sent buf_send = (real_t *) malloc(halo * slice_size * sizeof(real_t)); buf_recv = (real_t *) malloc(halo * slice_size * sizeof(real_t)); diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 8401c130..a99aaa27 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -271,11 +271,7 @@ namespace libmpdataxx void interpolate_refined_courants(arrvec_t &dst, arrvec_t &src) { - std::cerr << "interpolate_refined_courants src[0](ijk_ref): " << src[0](this->ijk_ref) << std::endl; - std::cerr << "interpolate_refined_courants src[1](ijk_ref): " << src[1](this->ijk_ref) << std::endl; - std::cerr << "interpolate_refined_courants src[2](ijk_ref): " << src[2](this->ijk_ref) << std::endl; this->xchng_vctr_ref(src, this->ijk_ref); - std::cerr << "post xchng vctr ref" << std::endl; this->template intrp<0>(dst[0], src[0], this->ijk_ref[0]^h, this->ijk_ref[1], this->ijk_ref[2], this->dijk_ref[0], true); this->template intrp<1>(dst[1], src[1], this->ijk_ref[1]^h, this->ijk_ref[2], this->ijk_ref[0], this->dijk_ref[1], true); this->template intrp<2>(dst[2], src[2], this->ijk_ref[2]^h, this->ijk_ref[0], this->ijk_ref[1], this->dijk_ref[2], true); From 804af0bb326770a50107e03c1eaf220d4578253e Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Thu, 24 Nov 2022 12:03:44 +0100 Subject: [PATCH 080/129] for doubles use mt19937_64 --- libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index a99aaa27..661e5f72 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -33,6 +33,7 @@ namespace libmpdataxx using parent_t = detail::mpdata_rhs_vip_prs_sgs_fra_common; // using parent_t::parent_t; // inheriting constructors using real_t = typename ct_params_t::real_t; + using gen_t = typename std::conditional_t, std::mt19937, std::mt19937_64>; // NOTE: we assume that if real_t is not float it has to be double struct ctor_args_t : parent_t::ctor_args_t { @@ -161,7 +162,7 @@ namespace libmpdataxx // also not all parameters in the halo are needed (but some are!) void generate_stretching_parameters(const int rng_seed = 44) { - std::mt19937 gen(rng_seed); + gen_t gen(rng_seed); std::uniform_real_distribution<> dis(-1, 1); // [-1,1), but whatever auto rand = std::bind(dis, gen); std::generate(this->d_j(this->ijk_ref_with_halo).begin(), this->d_j(this->ijk_ref_with_halo).end(), rand); From 30114abddd6f9e153bee7d4b0ae4c4acbac93ddd Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Thu, 24 Nov 2022 12:25:51 +0100 Subject: [PATCH 081/129] do not generate stretching parameters at each reconsteuction, make user do it --- .../solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 11 ++++++----- tests/sandbox/pbl/pbl.hpp | 2 ++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 661e5f72..48c41a8d 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -158,19 +158,20 @@ namespace libmpdataxx ); } + protected: + // TODO: stretching parameters at the overlaps of the reconstructed and resolved arrays (ijk_r2r) are not used, do not generate them // also not all parameters in the halo are needed (but some are!) - void generate_stretching_parameters(const int rng_seed = 44) + void generate_stretching_parameters(const typename gen_t::result_type rng_seed = 44) { gen_t gen(rng_seed); std::uniform_real_distribution<> dis(-1, 1); // [-1,1), but whatever auto rand = std::bind(dis, gen); std::generate(this->d_j(this->ijk_ref_with_halo).begin(), this->d_j(this->ijk_ref_with_halo).end(), rand); this->d_j(this->ijk_ref_with_halo) = formulae::fractal::d_of_CDF_fctr{}(this->d_j(this->ijk_ref_with_halo)); + this->mem->barrier(); } - protected: - // calculate refined points using (tri?)linear interpolation void interpolate_refinee(const int e = 0) { @@ -221,8 +222,8 @@ namespace libmpdataxx // fill refined array at position where it overlaps with the resolved array this->mem->refinee(e_ref)(this->ijk_r2r) = this->mem->advectee(e)(this->ijk); - generate_stretching_parameters(); - this->mem->barrier(); +// generate_stretching_parameters(); +// this->mem->barrier(); for(int i=0; in_fra_iter; ++i) { diff --git a/tests/sandbox/pbl/pbl.hpp b/tests/sandbox/pbl/pbl.hpp index e16744f1..9ea0b107 100644 --- a/tests/sandbox/pbl/pbl.hpp +++ b/tests/sandbox/pbl/pbl.hpp @@ -63,7 +63,9 @@ class pbl : public libmpdataxx::output::hdf5_xdmfrank == 0) std::cout << this->timestep << std::endl; // output tht refined with fractal reconstruction + //this->generate_stretching_parameters(std::random_device{}()); //this->reconstruct_refinee(ix::w); + this->generate_stretching_parameters(std::random_device{}()); this->reconstruct_refinee(ix::tht); this->mem->barrier(); From e61c94583c95ec74fcb41d9471fc3aa707c0c9b5 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Thu, 24 Nov 2022 12:25:51 +0100 Subject: [PATCH 082/129] do not generate stretching parameters at each reconsteuction, make user do it --- .../solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 11 ++++++----- tests/sandbox/pbl/pbl.hpp | 2 ++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 661e5f72..94dda036 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -158,19 +158,20 @@ namespace libmpdataxx ); } + protected: + // TODO: stretching parameters at the overlaps of the reconstructed and resolved arrays (ijk_r2r) are not used, do not generate them // also not all parameters in the halo are needed (but some are!) - void generate_stretching_parameters(const int rng_seed = 44) + void generate_stretching_parameters(const typename gen_t::result_type rng_seed) { gen_t gen(rng_seed); std::uniform_real_distribution<> dis(-1, 1); // [-1,1), but whatever auto rand = std::bind(dis, gen); std::generate(this->d_j(this->ijk_ref_with_halo).begin(), this->d_j(this->ijk_ref_with_halo).end(), rand); this->d_j(this->ijk_ref_with_halo) = formulae::fractal::d_of_CDF_fctr{}(this->d_j(this->ijk_ref_with_halo)); + this->mem->barrier(); } - protected: - // calculate refined points using (tri?)linear interpolation void interpolate_refinee(const int e = 0) { @@ -221,8 +222,8 @@ namespace libmpdataxx // fill refined array at position where it overlaps with the resolved array this->mem->refinee(e_ref)(this->ijk_r2r) = this->mem->advectee(e)(this->ijk); - generate_stretching_parameters(); - this->mem->barrier(); +// generate_stretching_parameters(); +// this->mem->barrier(); for(int i=0; in_fra_iter; ++i) { diff --git a/tests/sandbox/pbl/pbl.hpp b/tests/sandbox/pbl/pbl.hpp index e16744f1..9ea0b107 100644 --- a/tests/sandbox/pbl/pbl.hpp +++ b/tests/sandbox/pbl/pbl.hpp @@ -63,7 +63,9 @@ class pbl : public libmpdataxx::output::hdf5_xdmfrank == 0) std::cout << this->timestep << std::endl; // output tht refined with fractal reconstruction + //this->generate_stretching_parameters(std::random_device{}()); //this->reconstruct_refinee(ix::w); + this->generate_stretching_parameters(std::random_device{}()); this->reconstruct_refinee(ix::tht); this->mem->barrier(); From ddde107372e1cc2c18bc7727ae9f6a06fdbb6a3f Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Wed, 30 Nov 2022 12:04:48 +0100 Subject: [PATCH 083/129] track conc_reg --- .../concurr/detail/concurr_common_reg.hpp | 206 ++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 libmpdata++/concurr/detail/concurr_common_reg.hpp diff --git a/libmpdata++/concurr/detail/concurr_common_reg.hpp b/libmpdata++/concurr/detail/concurr_common_reg.hpp new file mode 100644 index 00000000..a3b13d41 --- /dev/null +++ b/libmpdata++/concurr/detail/concurr_common_reg.hpp @@ -0,0 +1,206 @@ +/** @file + * @copyright University of Warsaw + * @section LICENSE + * GPLv3+ (see the COPYING file or http://www.gnu.org/licenses/) + */ + +// for solver with regular grid only +// needed because regular and refined grid solvers have different ctors (ref needs bconds for ref grid) + +#pragma once + +#include + +namespace libmpdataxx +{ + namespace concurr + { + namespace detail + { + template< + class solver_t_, + bcond::bcond_e bcxl, bcond::bcond_e bcxr, + bcond::bcond_e bcyl, bcond::bcond_e bcyr, + bcond::bcond_e bczl, bcond::bcond_e bczr, + class enable = void + > + class concurr_common_reg + {}; + + // 1D version + template< + class solver_t_, + bcond::bcond_e bcxl, bcond::bcond_e bcxr, + bcond::bcond_e bcyl, bcond::bcond_e bcyr, + bcond::bcond_e bczl, bcond::bcond_e bczr + > + class concurr_common_reg< + solver_t_, + bcxl, bcxr, + bcyl, bcyr, + bczl, bczr, + typename std::enable_if_t + > : public concurr_common_hlpr + { + private: + + using parent_t = concurr_common_hlpr; + using parent_t::parent_t; + using real_t = typename solver_t_::real_t; + + protected: + + void init( + const typename solver_t_::rt_params_t &p, + const int &n0 + ) + { + // NOTE: for remote bcond, thread_rank set to 0 on purpose in 1D to have propre left/right message tags + this->template bc_set(this->bxl, this->mem->grid_size, this->mem->distmem.grid_size); + this->template bc_set(this->bxr, this->mem->grid_size, this->mem->distmem.grid_size); + + for (int i0 = 0; i0 < n0; ++i0) + { + this->shrdl.reset(new bcond::shared()); + this->shrdr.reset(new bcond::shared()); + + this->algos.push_back( + new solver_t_( + typename solver_t_::ctor_args_t({ + i0, + this->mem.get(), + i0 == 0 ? this->bxl : this->shrdl, + i0 == n0 - 1 ? this->bxr : this->shrdr, + this->mem->slab(this->mem->grid_size[0], i0, n0) + }), + p + ) + ); + } + } + }; + + + // 2D version + template< + class solver_t_, + bcond::bcond_e bcxl, bcond::bcond_e bcxr, + bcond::bcond_e bcyl, bcond::bcond_e bcyr, + bcond::bcond_e bczl, bcond::bcond_e bczr + > + class concurr_common_reg< + solver_t_, + bcxl, bcxr, + bcyl, bcyr, + bczl, bczr, + typename std::enable_if_t + > : public concurr_common_hlpr + { + private: + + using parent_t = concurr_common_hlpr; + using parent_t::parent_t; + using real_t = typename solver_t_::real_t; + + protected: + // TODO: assert parallelisation in the right dimensions! (blitz::assertContiguous) + void init( + const typename solver_t_::rt_params_t &p, + const int &n0, const int &n1 = 1 + ) { + for (int i0 = 0; i0 < n0; ++i0) + { + for (int i1 = 0; i1 < n1; ++i1) + { + // NOTE: for remote bcond, thread_rank set to 0 on purpose in 2D to have propre left/right message tags + this->template bc_set(this->bxl, this->mem->grid_size, this->mem->distmem.grid_size); + this->template bc_set(this->bxr, this->mem->grid_size, this->mem->distmem.grid_size); + + this->template bc_set(this->byl, this->mem->grid_size, this->mem->distmem.grid_size); + this->template bc_set(this->byr, this->mem->grid_size, this->mem->distmem.grid_size); + + this->shrdl.reset(new bcond::shared()); // TODO: shrdy if n1 != 1 + this->shrdr.reset(new bcond::shared()); // TODO: shrdy if n1 != 1 + + this->algos.push_back( + new solver_t_( + typename solver_t_::ctor_args_t({ + i0, + this->mem.get(), + i0 == 0 ? this->bxl : this->shrdl, + i0 == n0 - 1 ? this->bxr : this->shrdr, + this->byl, this->byr, + this->mem->slab(this->mem->grid_size[0], i0, n0), + this->mem->slab(this->mem->grid_size[1], i1, n1) + }), + p + ) + ); + } + } + } + }; + + // 3D version, note sharedmem in y direction! + template< + class solver_t_, + bcond::bcond_e bcxl, bcond::bcond_e bcxr, + bcond::bcond_e bcyl, bcond::bcond_e bcyr, + bcond::bcond_e bczl, bcond::bcond_e bczr + > + class concurr_common_reg< + solver_t_, + bcxl, bcxr, + bcyl, bcyr, + bczl, bczr, + typename std::enable_if_t + > : public concurr_common_hlpr + { + private: + + using parent_t = concurr_common_hlpr; + using parent_t::parent_t; + using real_t = typename solver_t_::real_t; + + protected: + void init( + const typename solver_t_::rt_params_t &p, + const int &n1, const int &n0 = 1, const int &n2 = 1 + ) { + // TODO: renew pointers only if invalid ? + for (int i0 = 0; i0 < n0; ++i0) + { + for (int i1 = 0; i1 < n1; ++i1) + { + for (int i2 = 0; i2 < n2; ++i2) + { + this->template init_bcs_3d( + this->bxl, this->bxr, this->byl, this->byr, this->bzl, this->bzr, this->shrdl, this->shrdr, + this->mem->grid_size, + this->mem->distmem.grid_size, + i1, n1); + + this->algos.push_back( + new solver_t_( + typename solver_t_::ctor_args_t({ + i1, + this->mem.get(), + this->bxl, this->bxr, + i1 == 0 ? this->byl : this->shrdl, + i1 == n1 - 1 ? this->byr : this->shrdr, + this->bzl, this->bzr, + this->mem->slab(this->mem->grid_size[0], i0, n0), + this->mem->slab(this->mem->grid_size[1], i1, n1), + this->mem->slab(this->mem->grid_size[2], i2, n2) + }), + p + ) + ); + } + } + } + } + }; + } // namespace detail + } // namespace concurr +} // namespace libmpdataxx From ab489d81138d28e8a0d95541554848a5ab340182 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Wed, 30 Nov 2022 15:20:42 +0100 Subject: [PATCH 084/129] fixes for non-mpi compilation --- .../concurr/detail/concurr_common_hlpr.hpp | 39 ++++++++++++++----- .../concurr/detail/concurr_common_ref.hpp | 22 ++++++++--- 2 files changed, 46 insertions(+), 15 deletions(-) diff --git a/libmpdata++/concurr/detail/concurr_common_hlpr.hpp b/libmpdata++/concurr/detail/concurr_common_hlpr.hpp index 034dedea..1a8f5036 100644 --- a/libmpdata++/concurr/detail/concurr_common_hlpr.hpp +++ b/libmpdata++/concurr/detail/concurr_common_hlpr.hpp @@ -29,9 +29,11 @@ #include #include #include -#include -#include -#include +#if defined(USE_MPI) + #include + #include + #include +#endif #include #include @@ -43,6 +45,7 @@ namespace libmpdataxx namespace detail { // helpers for setting remote bcond +#if defined(USE_MPI) template < class real_t, bcond::drctn_e dir, @@ -137,6 +140,7 @@ namespace libmpdataxx { bc_set_remote_impl::_(bcp, mem, grid_size, distmem_grid_size, thread_rank, thread_size, mpic); } +#endif template< class solver_t_, @@ -225,6 +229,7 @@ namespace libmpdataxx throw std::runtime_error("Polar boundary conditions do not work with MPI."); // distmem overrides +#if defined(USE_MPI) if (mem->distmem.size() > 1 && dim == 0) { if ( @@ -250,6 +255,7 @@ namespace libmpdataxx return; } } +#endif // 3d open bcond needs to know thread rank and size, because it zeroes perpendicular vectors if (type == bcond::open && solver_t::n_dims == 3) @@ -275,7 +281,7 @@ namespace libmpdataxx ); } - // 3D version +#if defined(USE_MPI) template void init_bcs_3d( bcp_t &bxl, bcp_t &bxr, bcp_t &byl, bcp_t &byr, bcp_t &bzl, bcp_t &bzr, bcp_t &shrdl, bcp_t &shrdr, @@ -285,7 +291,6 @@ namespace libmpdataxx boost::mpi::communicator &mpic) { // i1 is the local thread rank, n1 is the number of threads. These are needed by remote bcond, because only rank=0 does mpi communication -#if defined(USE_MPI) //boost::mpi::communicator mpic(MPI_COMM_WORLD, boost::mpi::comm_duplicate); //boost::mpi::communicator mpic(MPI_COMM_WORLD, boost::mpi::comm_attach); @@ -297,20 +302,34 @@ namespace libmpdataxx bc_set(bzl, grid_size, distmem_grid_size, mpic); bc_set(bzr, grid_size, distmem_grid_size, mpic); + + shrdl.reset(new bcond::shared()); // TODO: shrdy if n1 != 1 + shrdr.reset(new bcond::shared()); // TODO: shrdy if n1 != 1 + } + #else + + template + void init_bcs_3d( + bcp_t &bxl, bcp_t &bxr, bcp_t &byl, bcp_t &byr, bcp_t &bzl, bcp_t &bzr, bcp_t &shrdl, bcp_t &shrdr, + const grid_size_t &grid_size, + const distmem_grid_size_t &distmem_grid_size, + const int &i1, const int &n1) + { + // i1 is the local thread rank, n1 is the number of threads. These are needed by remote bcond, because only rank=0 does mpi communication bc_set(bxl, grid_size, distmem_grid_size, i1, n1); bc_set(bxr, grid_size, distmem_grid_size, i1, n1); - bc_set(byl, grid_size, distmem_grid_size); - bc_set(byr, grid_size, distmem_grid_size); + bc_set(byl, grid_size, distmem_grid_size, i1, n1); + bc_set(byr, grid_size, distmem_grid_size, i1, n1); - bc_set(bzl, grid_size, distmem_grid_size); - bc_set(bzr, grid_size, distmem_grid_size); -#endif + bc_set(bzl, grid_size, distmem_grid_size, i1, n1); + bc_set(bzr, grid_size, distmem_grid_size, i1, n1); shrdl.reset(new bcond::shared()); // TODO: shrdy if n1 != 1 shrdr.reset(new bcond::shared()); // TODO: shrdy if n1 != 1 } +#endif private: virtual void solve(advance_arg_t nt) = 0; diff --git a/libmpdata++/concurr/detail/concurr_common_ref.hpp b/libmpdata++/concurr/detail/concurr_common_ref.hpp index cfa124d9..26ec8d53 100644 --- a/libmpdata++/concurr/detail/concurr_common_ref.hpp +++ b/libmpdata++/concurr/detail/concurr_common_ref.hpp @@ -29,7 +29,9 @@ namespace libmpdataxx using parent_t = concurr_common_hlpr; // using parent_t::parent_t; +#if defined(USE_MPI) boost::mpi::communicator mpic, mpic_ref; +#endif public: // ctor @@ -38,9 +40,11 @@ namespace libmpdataxx typename solver_t_::mem_t *mem_p, const int &size ) : - parent_t(p, mem_p, size), - mpic(MPI_COMM_WORLD, boost::mpi::comm_duplicate), - mpic_ref(MPI_COMM_WORLD, boost::mpi::comm_duplicate) + parent_t(p, mem_p, size) +#if defined(USE_MPI) + ,mpic(MPI_COMM_WORLD, boost::mpi::comm_duplicate) + ,mpic_ref(MPI_COMM_WORLD, boost::mpi::comm_duplicate) +#endif {} protected: @@ -64,13 +68,21 @@ namespace libmpdataxx this->bxl, this->bxr, this->byl, this->byr, this->bzl, this->bzr, this->shrdl, this->shrdr, this->mem->grid_size, this->mem->distmem.grid_size, - i1, n1, mpic); + i1, n1 +#if defined(USE_MPI) + ,mpic +#endif + ); this->template init_bcs_3d( bxl_ref, bxr_ref, byl_ref, byr_ref, bzl_ref, bzr_ref, shrdl_ref, shrdr_ref, this->mem->grid_size_ref, this->mem->distmem.grid_size_ref, - i1, n1, mpic_ref); + i1, n1 +#if defined(USE_MPI) + ,mpic_ref +#endif + ); /* this->init_bcs_3d(i1, n1); From 18d254aa142977d7fc97b20b0f878f4d2ba29579 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Wed, 8 Mar 2023 12:20:29 +0100 Subject: [PATCH 085/129] comment --- libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 94dda036..54bd64d4 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -231,7 +231,7 @@ namespace libmpdataxx formulae::fractal::rcnstrct<0, real_t>(this->mem->psi_ref[e_ref], this->rng_dbl_stride(mid_ijk_r2r_0), ijk_r2r_1_h, ijk_r2r_2_h, hstride, this->c_j, this->d_j, this->f_j); this->mem->barrier(); - formulae::fractal::rcnstrct<1, real_t>(this->mem->psi_ref[e_ref], this->rng_dbl_stride(mid_ijk_r2r_1) , ijk_r2r_2_h, ijk_r2r_0_h_with_halo, hstride, this->c_j, this->d_j, this->f_j); // NOTE: rng_dbl_stride(mid_ijk_r2r_1) gives overlapping ranges between thread subdomains... however it seems to work and naive fixes didnt work + formulae::fractal::rcnstrct<1, real_t>(this->mem->psi_ref[e_ref], this->rng_dbl_stride(mid_ijk_r2r_1) , ijk_r2r_2_h, ijk_r2r_0_h_with_halo, hstride, this->c_j, this->d_j, this->f_j); // NOTE: rng_dbl_stride(mid_ijk_r2r_1) gives overlapping ranges between thread subdomains... however it seems to work and naive fixes didnt work; UPDATE: are the ranges really overlapping? it looks good at the second look... formulae::fractal::rcnstrct<2, real_t>(this->mem->psi_ref[e_ref], this->rng_dbl_stride(mid_ijk_r2r_2) , ijk_r2r_0_h_with_halo, this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride, this->c_j, this->d_j, this->f_j); } this->mem->barrier(); From e6dcd012af4252603ba1c1b03c575128d8b9a518 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Wed, 8 Mar 2023 12:21:45 +0100 Subject: [PATCH 086/129] comment --- libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 54bd64d4..4c183a65 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -203,6 +203,7 @@ namespace libmpdataxx this->mem->barrier(); } + // TODO: now that we have xchng_ref, use halo (other than distmem halos) in reconstruction? void reconstruct_refinee(const int e = 0) { assert(opts::isset(ct_params_t::fractal_recon, opts::bit(e))); From 22483f9cbb79f81572615ef6a8b5a22b4d611110 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Thu, 9 Mar 2023 16:19:07 +0100 Subject: [PATCH 087/129] negative reconstructuion dbg --- .../formulae/fractal_reconstruction.hpp | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/libmpdata++/formulae/fractal_reconstruction.hpp b/libmpdata++/formulae/fractal_reconstruction.hpp index 15fbacc6..0b60d60c 100644 --- a/libmpdata++/formulae/fractal_reconstruction.hpp +++ b/libmpdata++/formulae/fractal_reconstruction.hpp @@ -121,6 +121,30 @@ namespace libmpdataxx // new u, at j=2, between i=1 and i=2, result of w_j=2(u_i=1), Eq. (1) in Akinlabi et al. arr(pis(i_next, j, k)) = c_2 * 1 + d_2 * u_1 + f_2; + if(arr(pis(i , j, k)) < 0) + { + std::cerr << "negative reonstruction @ i: " << arr(pis(i , j, k)) << + "u_0: " << u_0 << + "u_1: " << u_1 << + "u_2: " << u_2 << + "c_1: " << c_1 << + "d_1: " << d_1 << + "f_1: " << f_1 << + std::endl; + } + + if(arr(pis(i_next, j, k)) < 0) + { + std::cerr << "negative reonstruction @ i_next: " << arr(pis(i_next, j, k)) << + "u_0: " << u_0 << + "u_1: " << u_1 << + "u_2: " << u_2 << + "c_2: " << c_2 << + "d_2: " << d_2 << + "f_2: " << f_2 << + std::endl; + } + // DEBUG: do interpolation, useful for testing if ranges are correct // arr(pis(i, j, k)) = real_t(.5) * (u_0 + u_1); // arr(pis(i_next, j, k)) = real_t(.5) * (u_1 + u_2); From b521dafd7a4316e0b96b9a27b59bd32f8318221b Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Thu, 9 Mar 2023 18:07:28 +0100 Subject: [PATCH 088/129] fix neg recon debug --- libmpdata++/formulae/fractal_reconstruction.hpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/libmpdata++/formulae/fractal_reconstruction.hpp b/libmpdata++/formulae/fractal_reconstruction.hpp index 0b60d60c..b64ad70b 100644 --- a/libmpdata++/formulae/fractal_reconstruction.hpp +++ b/libmpdata++/formulae/fractal_reconstruction.hpp @@ -121,9 +121,13 @@ namespace libmpdataxx // new u, at j=2, between i=1 and i=2, result of w_j=2(u_i=1), Eq. (1) in Akinlabi et al. arr(pis(i_next, j, k)) = c_2 * 1 + d_2 * u_1 + f_2; - if(arr(pis(i , j, k)) < 0) + if(blitz::min(arr(pis(i , j, k))) < 0) { - std::cerr << "negative reonstruction @ i: " << arr(pis(i , j, k)) << + std::cerr << "negative reonstruction @ i: " << + "i: " << i << + "j: " << j << + "k: " << k << + "arr: " << arr(pis(i , j, k)) << "u_0: " << u_0 << "u_1: " << u_1 << "u_2: " << u_2 << @@ -133,9 +137,13 @@ namespace libmpdataxx std::endl; } - if(arr(pis(i_next, j, k)) < 0) + if(blitz::min(arr(pis(i_next, j, k))) < 0) { - std::cerr << "negative reonstruction @ i_next: " << arr(pis(i_next, j, k)) << + std::cerr << "negative reonstruction @ i_next: " << + "i_next:" << i_next << + "j: " << j << + "k: " << k << + "arr: " << arr(pis(i_next, j, k)) << "u_0: " << u_0 << "u_1: " << u_1 << "u_2: " << u_2 << From 6b8f9077004911561f1bc9d0aedbac7575c60b97 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Fri, 10 Mar 2023 12:27:20 +0100 Subject: [PATCH 089/129] record prof const for refined --- libmpdata++/output/hdf5.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libmpdata++/output/hdf5.hpp b/libmpdata++/output/hdf5.hpp index 8a6cdfa5..fc2ff031 100644 --- a/libmpdata++/output/hdf5.hpp +++ b/libmpdata++/output/hdf5.hpp @@ -641,9 +641,9 @@ namespace libmpdataxx record_prof_hlpr(hdfcp, name, data, vctr, refined); } - void record_prof_const(const std::string &name, typename solver_t::real_t *data) + void record_prof_const(const std::string &name, typename solver_t::real_t *data, const bool refined = false) { - record_prof_const_hlpr(name, data, false); + record_prof_const_hlpr(name, data, false, refined); } void record_prof_vctr_const(const std::string &name, typename solver_t::real_t *data) From 2c070672c0e0ce2223337e5067da6b89ecc43787 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Mon, 13 Mar 2023 13:40:09 +0100 Subject: [PATCH 090/129] record prof hlpr assert fix --- libmpdata++/output/hdf5.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmpdata++/output/hdf5.hpp b/libmpdata++/output/hdf5.hpp index fc2ff031..63aed1cf 100644 --- a/libmpdata++/output/hdf5.hpp +++ b/libmpdata++/output/hdf5.hpp @@ -523,7 +523,7 @@ namespace libmpdataxx void record_prof_hlpr(H5::H5File hdff, const std::string &name, typename solver_t::real_t *data, const bool vctr, const bool refined) { assert(this->rank == 0); - assert((vctr && refined == false) && "record prof hlpr cant save refined vector profiles"); + assert(((vctr && refined) == false) && "record prof hlpr cant save refined vector profiles"); const auto _shape(refined ? shape_ref : vctr ? cshape : shape); const auto _offst(refined ? offst_ref : offst); From a540477c9a9c3b2347fb7d5c7ee5930d082cf273 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Tue, 14 Mar 2023 12:11:20 +0100 Subject: [PATCH 091/129] fractal: remove tabs --- .../formulae/fractal_reconstruction.hpp | 118 +++++++++--------- 1 file changed, 59 insertions(+), 59 deletions(-) diff --git a/libmpdata++/formulae/fractal_reconstruction.hpp b/libmpdata++/formulae/fractal_reconstruction.hpp index b64ad70b..ec63602d 100644 --- a/libmpdata++/formulae/fractal_reconstruction.hpp +++ b/libmpdata++/formulae/fractal_reconstruction.hpp @@ -82,7 +82,7 @@ namespace libmpdataxx arr_t f_j // parameter f ) { - // debug output + // debug output // std::cerr << "ranges in rcnstrct" << std::endl; // if(d==0) // std::cerr << "range<" << d << ">: " << i << " " << j << " " << k << std::endl; @@ -91,67 +91,67 @@ namespace libmpdataxx // if(d==2) // std::cerr << "range<" << d << ">: " << j << " " << k << " " << i << std::endl; - using idxperm::pis; + using idxperm::pis; - // second reconstructed position (j=2, between i=1 and i=2) - const rng_t i_next = i + 2*dist; + // second reconstructed position (j=2, between i=1 and i=2) + const rng_t i_next = i + 2*dist; - const int x[3] = {0, 1, 2}; + const int x[3] = {0, 1, 2}; - // helper references - const arr_t u_0 = arr(pis(i - dist, j, k)), - u_1 = arr(pis(i + dist, j, k)), - u_2 = arr(pis(i_next + dist, j, k)); - - arr_t c_1 = c_j(pis(i, j, k)), - c_2 = c_j(pis(i_next, j, k)), - d_1 = d_j(pis(i, j, k)), - d_2 = d_j(pis(i_next, j, k)), - f_1 = f_j(pis(i, j, k)), - f_2 = f_j(pis(i_next, j, k)); - - // Eqs. (5) and (6) Akinlabi et al. - c_1 = (u_1 - u_0) / (2.) - d_1 * (u_2 - u_0) / (2.); - c_2 = (u_2 - u_1) / (2.) - d_2 * (u_2 - u_0) / (2.); - f_1 = (x[2] * u_0 - x[0] * u_1) / (2.) - d_1 * (x[2] * u_0 - x[0] * u_2) / (2.); - f_2 = (x[2] * u_1 - x[0] * u_2) / (2.) - d_2 * (x[2] * u_0 - x[0] * u_2) / (2.); - - // new u, at j=1, between i=0 and i=1, result of w_j=1(u_i=1), Eq. (1) in Akinlabi et al. - arr(pis(i , j, k)) = c_1 * 1 + d_1 * u_1 + f_1; - // new u, at j=2, between i=1 and i=2, result of w_j=2(u_i=1), Eq. (1) in Akinlabi et al. - arr(pis(i_next, j, k)) = c_2 * 1 + d_2 * u_1 + f_2; - - if(blitz::min(arr(pis(i , j, k))) < 0) - { - std::cerr << "negative reonstruction @ i: " << - "i: " << i << - "j: " << j << - "k: " << k << - "arr: " << arr(pis(i , j, k)) << - "u_0: " << u_0 << - "u_1: " << u_1 << - "u_2: " << u_2 << - "c_1: " << c_1 << - "d_1: " << d_1 << - "f_1: " << f_1 << - std::endl; - } - - if(blitz::min(arr(pis(i_next, j, k))) < 0) - { - std::cerr << "negative reonstruction @ i_next: " << - "i_next:" << i_next << - "j: " << j << - "k: " << k << - "arr: " << arr(pis(i_next, j, k)) << - "u_0: " << u_0 << - "u_1: " << u_1 << - "u_2: " << u_2 << - "c_2: " << c_2 << - "d_2: " << d_2 << - "f_2: " << f_2 << - std::endl; - } + // helper references + const arr_t u_0 = arr(pis(i - dist, j, k)), + u_1 = arr(pis(i + dist, j, k)), + u_2 = arr(pis(i_next + dist, j, k)); + + arr_t c_1 = c_j(pis(i, j, k)), + c_2 = c_j(pis(i_next, j, k)), + d_1 = d_j(pis(i, j, k)), + d_2 = d_j(pis(i_next, j, k)), + f_1 = f_j(pis(i, j, k)), + f_2 = f_j(pis(i_next, j, k)); + + // Eqs. (5) and (6) Akinlabi et al. + c_1 = (u_1 - u_0) / (2.) - d_1 * (u_2 - u_0) / (2.); + c_2 = (u_2 - u_1) / (2.) - d_2 * (u_2 - u_0) / (2.); + f_1 = (x[2] * u_0 - x[0] * u_1) / (2.) - d_1 * (x[2] * u_0 - x[0] * u_2) / (2.); + f_2 = (x[2] * u_1 - x[0] * u_2) / (2.) - d_2 * (x[2] * u_0 - x[0] * u_2) / (2.); + + // new u, at j=1, between i=0 and i=1, result of w_j=1(u_i=1), Eq. (1) in Akinlabi et al. + arr(pis(i , j, k)) = c_1 * 1 + d_1 * u_1 + f_1; + // new u, at j=2, between i=1 and i=2, result of w_j=2(u_i=1), Eq. (1) in Akinlabi et al. + arr(pis(i_next, j, k)) = c_2 * 1 + d_2 * u_1 + f_2; + + if(blitz::min(arr(pis(i , j, k))) < 0) + { + std::cerr << "negative reonstruction @ i: " << + " i: " << i << + " j: " << j << + " k: " << k << + " arr: " << arr(pis(i , j, k)) << + " u_0: " << u_0 << + " u_1: " << u_1 << + " u_2: " << u_2 << + " c_1: " << c_1 << + " d_1: " << d_1 << + " f_1: " << f_1 << + std::endl; + } + + if(blitz::min(arr(pis(i_next, j, k))) < 0) + { + std::cerr << "negative reonstruction @ i_next: " << + " i_next:" << i_next << + " j: " << j << + " k: " << k << + " arr: " << arr(pis(i_next, j, k)) << + " u_0: " << u_0 << + " u_1: " << u_1 << + " u_2: " << u_2 << + " c_2: " << c_2 << + " d_2: " << d_2 << + " f_2: " << f_2 << + std::endl; + } // DEBUG: do interpolation, useful for testing if ranges are correct // arr(pis(i, j, k)) = real_t(.5) * (u_0 + u_1); From db06d953b30d71eaeec14b52928dad4dba0bc748 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Tue, 14 Mar 2023 12:25:02 +0100 Subject: [PATCH 092/129] fractal neg dbg: print d --- libmpdata++/formulae/fractal_reconstruction.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libmpdata++/formulae/fractal_reconstruction.hpp b/libmpdata++/formulae/fractal_reconstruction.hpp index ec63602d..6666f23e 100644 --- a/libmpdata++/formulae/fractal_reconstruction.hpp +++ b/libmpdata++/formulae/fractal_reconstruction.hpp @@ -124,6 +124,7 @@ namespace libmpdataxx if(blitz::min(arr(pis(i , j, k))) < 0) { std::cerr << "negative reonstruction @ i: " << + " d: " << d << " i: " << i << " j: " << j << " k: " << k << @@ -140,6 +141,7 @@ namespace libmpdataxx if(blitz::min(arr(pis(i_next, j, k))) < 0) { std::cerr << "negative reonstruction @ i_next: " << + " d: " << d << " i_next:" << i_next << " j: " << j << " k: " << k << From e78204b50d94a8ddd1e1adce20b929d9ce46f340 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Tue, 14 Mar 2023 12:30:42 +0100 Subject: [PATCH 093/129] fractal neg dbg: comment it out --- libmpdata++/formulae/fractal_reconstruction.hpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libmpdata++/formulae/fractal_reconstruction.hpp b/libmpdata++/formulae/fractal_reconstruction.hpp index 6666f23e..6cfba5cf 100644 --- a/libmpdata++/formulae/fractal_reconstruction.hpp +++ b/libmpdata++/formulae/fractal_reconstruction.hpp @@ -120,6 +120,9 @@ namespace libmpdataxx arr(pis(i , j, k)) = c_1 * 1 + d_1 * u_1 + f_1; // new u, at j=2, between i=1 and i=2, result of w_j=2(u_i=1), Eq. (1) in Akinlabi et al. arr(pis(i_next, j, k)) = c_2 * 1 + d_2 * u_1 + f_2; +/* + // Debug negative values in reconstructed arrays (e.g. negative rv) + // Didn't show errors in reconstruction, simply bad luck if(blitz::min(arr(pis(i , j, k))) < 0) { @@ -154,6 +157,7 @@ namespace libmpdataxx " f_2: " << f_2 << std::endl; } +*/ // DEBUG: do interpolation, useful for testing if ranges are correct // arr(pis(i, j, k)) = real_t(.5) * (u_0 + u_1); From c8a575982a6b736348da25930c326a6718b5bba1 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Wed, 22 Mar 2023 17:12:13 +0100 Subject: [PATCH 094/129] hdf and hdf5_xdmf: wip on specializations for solvers with refinemenet --- libmpdata++-config.cmake | 26 +++---- libmpdata++/output/hdf5.hpp | 112 ++++++++++++++++++++----------- libmpdata++/output/hdf5_xdmf.hpp | 96 +++++++++++++++++++------- 3 files changed, 156 insertions(+), 78 deletions(-) diff --git a/libmpdata++-config.cmake b/libmpdata++-config.cmake index 0415f1ec..ca5056a9 100644 --- a/libmpdata++-config.cmake +++ b/libmpdata++-config.cmake @@ -64,19 +64,19 @@ endif() ############################################################################################ # C++14 -include(CheckCXXSourceCompiles) -set(CMAKE_REQUIRED_FLAGS "-std=c++14") -check_cxx_source_compiles(" - #include - auto f() { return 1;} - template using ei=std::enable_if; - struct a {a(int){}};struct b:a {using a::a;}; - int main(){b i(1);} -" CXX14_SUPPORTED) -if (NOT CXX14_SUPPORTED) - message(FATAL_ERROR "C++14 compatibility test failed - please update your compiler or point CMake to another one with -DCMAKE_CXX_COMPILER=...") -endif() -unset(CMAKE_REQUIRED_FLAGS) +#include(CheckCXXSourceCompiles) +#set(CMAKE_REQUIRED_FLAGS "-std=c++14") +#check_cxx_source_compiles(" +# #include +# auto f() { return 1;} +# template using ei=std::enable_if; +# struct a {a(int){}};struct b:a {using a::a;}; +# int main(){b i(1);} +#" CXX14_SUPPORTED) +#if (NOT CXX14_SUPPORTED) +# message(FATAL_ERROR "C++14 compatibility test failed - please update your compiler or point CMake to another one with -DCMAKE_CXX_COMPILER=...") +#endif() +#unset(CMAKE_REQUIRED_FLAGS) ############################################################################################ diff --git a/libmpdata++/output/hdf5.hpp b/libmpdata++/output/hdf5.hpp index 63aed1cf..99e785ea 100644 --- a/libmpdata++/output/hdf5.hpp +++ b/libmpdata++/output/hdf5.hpp @@ -30,17 +30,16 @@ namespace libmpdataxx { namespace output { + // output class common for simulations with and without grid refinement template - class hdf5 : public detail::output_common + class hdf5_common : public detail::output_common { using parent_t = detail::output_common; protected: - using output_t = hdf5; - std::unique_ptr hdfp; - std::map dim_names, dim_names_ref; + std::map dim_names; const std::string const_name = "const.h5"; std::string const_file; const hsize_t zero = 0, one = 1; @@ -64,7 +63,7 @@ namespace libmpdataxx #endif hid_t dxpl_id; - void start(const typename parent_t::advance_arg_t nt) + virtual void start(const typename parent_t::advance_arg_t nt) { const_file = this->outdir + "/" + const_name; @@ -212,39 +211,6 @@ namespace libmpdataxx curr_dim.write(coord.data(), flttype_solver, H5::DataSpace(parent_t::n_dims, cshape.data()), dim_space, dxpl_id); } - // refined X, Y, Z, TODO: very similar to X, Y, Z - for (int i = 0; i < parent_t::n_dims; ++i) - { - - blitz::Array coord(cshape_ref); -#if defined(USE_MPI) - coord.reindexSelf(offst_ref); -#endif - std::string name; - switch (i) - { - case 0 : coord = this->di / 2. + this->di / this->mem->n_ref * (blitz::firstIndex() - .5); - name = "X refined"; - dim_names_ref[i] = name; - break; - case 1 : coord = this->dj / 2. + this->dj / this->mem->n_ref * (blitz::secondIndex() - .5); - name = "Y refined"; - dim_names_ref[i] = name; - break; - case 2 : coord = this->dk / 2. + this->dk / this->mem->n_ref * (blitz::thirdIndex() - .5); - name = "Z refined"; - dim_names_ref[i] = name; - break; - default : break; - } - - auto curr_dim = (*hdfp).createDataSet(name, flttype_output, cspace_ref); - - H5::DataSpace dim_space = curr_dim.getSpace(); - dim_space.selectHyperslab(H5S_SELECT_SET, cshape_ref.data(), offst_ref.data()); - curr_dim.write(coord.data(), flttype_solver, H5::DataSpace(parent_t::n_dims, cshape_ref.data()), dim_space, dxpl_id); - } - // T { const hsize_t @@ -831,7 +797,7 @@ namespace libmpdataxx public: // ctor - hdf5( + hdf5_common( typename parent_t::ctor_args_t args, const typename parent_t::rt_params_t &p ) : parent_t(args, p) @@ -848,7 +814,7 @@ namespace libmpdataxx } // dtor - virtual ~hdf5() + virtual ~hdf5_common() { H5Pclose(dxpl_id); #if defined(USE_MPI) @@ -856,5 +822,71 @@ namespace libmpdataxx #endif } }; + + + // for solvers without grid refinemenet + template + class hdf5 : public hdf5_common + { + protected: + + using parent_t = hdf5_common; + using parent_t::parent_t; + using output_t = hdf5; +// using n_dims = typename parent_t::n_dims; + }; + + // specialization for solvers with grid refinemenet + template + class hdf5 + > : public hdf5_common + { + protected: + + using parent_t = hdf5_common; + using parent_t::parent_t; + using output_t = hdf5; +// using n_dims = typename parent_t::n_dims; + + std::map dim_names_ref; + + void start(const typename parent_t::advance_arg_t nt) override + { + parent_t::start(nt); + + // refined X, Y, Z, TODO: very similar to X, Y, Z + for (int i = 0; i < parent_t::n_dims; ++i) + { + blitz::Array coord(this->cshape_ref); +#if defined(USE_MPI) + coord.reindexSelf(this->offst_ref); +#endif + std::string name; + switch (i) + { + case 0 : coord = this->di / 2. + this->di / this->mem->n_ref * (blitz::firstIndex() - .5); + name = "X refined"; + dim_names_ref[i] = name; + break; + case 1 : coord = this->dj / 2. + this->dj / this->mem->n_ref * (blitz::secondIndex() - .5); + name = "Y refined"; + dim_names_ref[i] = name; + break; + case 2 : coord = this->dk / 2. + this->dk / this->mem->n_ref * (blitz::thirdIndex() - .5); + name = "Z refined"; + dim_names_ref[i] = name; + break; + default : break; + } + auto curr_dim = (*(this->hdfp)).createDataSet(name, this->flttype_output, this->cspace_ref); + + H5::DataSpace dim_space = curr_dim.getSpace(); + dim_space.selectHyperslab(H5S_SELECT_SET, this->cshape_ref.data(), this->offst_ref.data()); + curr_dim.write(coord.data(), this->flttype_solver, H5::DataSpace(parent_t::n_dims, this->cshape_ref.data()), dim_space, this->dxpl_id); + } + } + }; + } // namespace output } // namespace libmpdataxx diff --git a/libmpdata++/output/hdf5_xdmf.hpp b/libmpdata++/output/hdf5_xdmf.hpp index 9e1d7404..abad6470 100644 --- a/libmpdata++/output/hdf5_xdmf.hpp +++ b/libmpdata++/output/hdf5_xdmf.hpp @@ -26,20 +26,19 @@ namespace libmpdataxx namespace output { template - class hdf5_xdmf : public hdf5 + class hdf5_xdmf_common : public hdf5 { protected: - using output_t = hdf5_xdmf; using parent_t = hdf5; static_assert(parent_t::n_dims > 1, "only 2D and 3D output supported"); std::vector timesteps; //xdmf writer, additional one for refined data (separate .xmf files describing the refined grid, TODO: use two grids in one file? Paraview AMR dataset could help?) - detail::xdmf_writer xdmfw, xdmfw_ref; + detail::xdmf_writer xdmfw; - void start(const typename parent_t::advance_arg_t nt) + void start(const typename parent_t::advance_arg_t nt) override { parent_t::start(nt); @@ -57,11 +56,10 @@ namespace libmpdataxx if (this->mem->G.get() != nullptr) xdmfw.add_const_attribute("G", this->const_name, this->mem->distmem.grid_size.data()); xdmfw.setup( this->const_name, this->dim_names, attr_names, this->mem->distmem.grid_size.data() ); - xdmfw_ref.setup(this->const_name, this->dim_names_ref, {}, this->mem->distmem.grid_size_ref.data()); } } - void write_xmfs() + virtual void write_xmfs() { #if defined(USE_MPI) if (this->mem->distmem.rank() == 0) @@ -71,14 +69,10 @@ namespace libmpdataxx std::string xmf_name = this->base_name() + ".xmf"; xdmfw.write(this->outdir + "/" + xmf_name, this->hdf_name(), this->record_time); - std::string xmf_ref_name = this->base_name() + "_ref.xmf"; - xdmfw_ref.write(this->outdir + "/" + xmf_ref_name, this->hdf_name(), this->record_time); - // save the xmf filename for temporal write timesteps.push_back(this->base_name()); // write temporal xmf xdmfw.write_temporal(this->outdir + "/temp.xmf", timesteps, ".xmf"); - xdmfw_ref.write_temporal(this->outdir + "/temp_ref.xmf", timesteps, "_ref.xmf"); } } @@ -97,24 +91,80 @@ namespace libmpdataxx parent_t::record_aux(name, data); } - void record_aux_refined(const std::string &name, typename solver_t::real_t *data) + void record_aux_dsc(const std::string &name, const typename solver_t::arr_t &arr, bool srfc = false) { + auto shape = this->mem->distmem.grid_size; + if(srfc) shape.at(parent_t::n_dims-1) = 1; #if defined(USE_MPI) if (this->mem->distmem.rank() == 0) #endif - xdmfw_ref.add_attribute(name, this->hdf_name(), this->mem->distmem.grid_size_ref.data()); - parent_t::record_aux_refined(name, data); + xdmfw.add_attribute(name, this->hdf_name(), shape.data()); + parent_t::record_aux_dsc(name, arr, srfc); } - void record_aux_dsc(const std::string &name, const typename solver_t::arr_t &arr, bool srfc = false) + public: + + // ctor + hdf5_xdmf_common( + typename parent_t::ctor_args_t args, + const typename parent_t::rt_params_t &p + ) : parent_t(args, p) + {} + }; + + // for solvers without grid refinemenet + template + class hdf5_xdmf : public hdf5_xdmf_common + { + protected: + using output_t = hdf5_xdmf; + }; + + // specialization for solvers with grid refinemenet + template + class hdf5_xdmf + > : public hdf5_xdmf_common + { + protected: + using output_t = hdf5_xdmf; + using parent_t = hdf5_xdmf_common; + + detail::xdmf_writer xdmfw_ref; + + void start(const typename parent_t::advance_arg_t nt) override { - auto shape = this->mem->distmem.grid_size; - if(srfc) shape.at(parent_t::n_dims-1) = 1; + parent_t::start(nt); + #if defined(USE_MPI) if (this->mem->distmem.rank() == 0) #endif - xdmfw.add_attribute(name, this->hdf_name(), shape.data()); - parent_t::record_aux_dsc(name, arr, srfc); + { + xdmfw_ref.setup(this->const_name, this->dim_names_ref, {}, this->mem->distmem.grid_size_ref.data()); + } + } + + void write_xmfs() override + { + parent_t::write_xmfs(); +#if defined(USE_MPI) + if (this->mem->distmem.rank() == 0) +#endif + { + std::string xmf_ref_name = this->base_name() + "_ref.xmf"; + xdmfw_ref.write(this->outdir + "/" + xmf_ref_name, this->hdf_name(), this->record_time); + + xdmfw_ref.write_temporal(this->outdir + "/temp_ref.xmf", this->timesteps, "_ref.xmf"); + } + } + + void record_aux_refined(const std::string &name, typename solver_t::real_t *data) + { +#if defined(USE_MPI) + if (this->mem->distmem.rank() == 0) +#endif + xdmfw_ref.add_attribute(name, this->hdf_name(), this->mem->distmem.grid_size_ref.data()); + parent_t::record_aux_refined(name, data); } void record_aux_dsc_refined(const std::string &name, const typename solver_t::arr_t &arr) @@ -128,13 +178,9 @@ namespace libmpdataxx } public: - - // ctor - hdf5_xdmf( - typename parent_t::ctor_args_t args, - const typename parent_t::rt_params_t &p - ) : parent_t(args, p) - {} + using parent_t::parent_t; }; + + } // namespace output } // namespace libmpdataxx From 71a1d20bb55496935b6631568f4ceb70b4145632 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Wed, 22 Mar 2023 18:17:55 +0100 Subject: [PATCH 095/129] hdf5_xdmf: ctor fix --- libmpdata++/output/hdf5_xdmf.hpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libmpdata++/output/hdf5_xdmf.hpp b/libmpdata++/output/hdf5_xdmf.hpp index abad6470..c988fcaf 100644 --- a/libmpdata++/output/hdf5_xdmf.hpp +++ b/libmpdata++/output/hdf5_xdmf.hpp @@ -118,6 +118,10 @@ namespace libmpdataxx { protected: using output_t = hdf5_xdmf; + using parent_t = hdf5_xdmf_common; + + public: + using parent_t::parent_t; }; // specialization for solvers with grid refinemenet From da6a8e689c2f477eb3e71d9070d6e94a55d6480c Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Wed, 22 Mar 2023 18:31:42 +0100 Subject: [PATCH 096/129] hdf5: move _ref output to ref specialization --- libmpdata++/output/hdf5.hpp | 77 ++++++++++++++++++++++++++----------- 1 file changed, 55 insertions(+), 22 deletions(-) diff --git a/libmpdata++/output/hdf5.hpp b/libmpdata++/output/hdf5.hpp index 99e785ea..1d63d198 100644 --- a/libmpdata++/output/hdf5.hpp +++ b/libmpdata++/output/hdf5.hpp @@ -54,10 +54,10 @@ namespace libmpdataxx H5::PredType::NATIVE_FLOAT, flttype_output = H5::PredType::NATIVE_FLOAT; // using floats not to waste disk space - blitz::TinyVector cshape, shape, chunk, srfcshape, srfcchunk, offst, shape_h, chunk_h, offst_h, shape_mem_h, offst_mem_h, shape_ref, cshape_ref, chunk_ref, offst_ref; // what if grid refinement is not done??? + blitz::TinyVector cshape, shape, chunk, srfcshape, srfcchunk, offst, shape_h, chunk_h, offst_h, shape_mem_h, offst_mem_h; H5::DSetCreatPropList params; - H5::DataSpace sspace, cspace, srfcspace, sspace_h, sspace_mem_h, sspace_ref, cspace_ref; + H5::DataSpace sspace, cspace, srfcspace, sspace_h, sspace_mem_h; #if defined(USE_MPI) hid_t fapl_id; #endif @@ -97,23 +97,19 @@ namespace libmpdataxx offst = 0; offst_h = 0; offst_mem_h = 0; - offst_ref = 0; for (int d = 0; d < parent_t::n_dims; ++d) { shape[d] = this->mem->distmem.grid_size[d]; // shape of arrays stored in file shape_h[d] = this->mem->distmem.grid_size[d] + 2 * this->halo; // shape of arrays with halos stored in files - shape_ref[d] = this->mem->distmem.grid_size_ref[d]; shape_mem_h[d] = this->mem->grid_size[d].length() + 2 * this->halo; // shape of the array with halo stored in memory of given MPI rank } chunk = shape; chunk_h = shape_h; - chunk_ref = shape_ref; // there is one more coordinate than cell index in each dimension cshape = shape + 1; - cshape_ref = shape_ref + 1; srfcshape = shape; *(srfcshape.end()-1) = 1; @@ -123,18 +119,12 @@ namespace libmpdataxx sspace_mem_h = H5::DataSpace(parent_t::n_dims, shape_mem_h.data()); srfcspace = H5::DataSpace(parent_t::n_dims, srfcshape.data()); cspace = H5::DataSpace(parent_t::n_dims, cshape.data()); - sspace_ref = H5::DataSpace(parent_t::n_dims, shape_ref.data()); - cspace_ref = H5::DataSpace(parent_t::n_dims, cshape_ref.data()); #if defined(USE_MPI) if (this->mem->distmem.size() > 1) { shape[0] = this->mem->grid_size[0].length(); cshape[0] = this->mem->grid_size[0].length(); - shape_ref[0] = this->mem->distmem.rank() < this->mem->distmem.size() - 1 ? - this->mem->grid_size_ref[0].length()-1 : // -1 to make ranges nonoverlapping - this->mem->grid_size_ref[0].length(); - cshape_ref[0] = shape_ref[0]; shape_h[0] = this->mem->distmem.rank() == 0 || this->mem->distmem.rank() == this->mem->distmem.size()-1 ? @@ -143,14 +133,10 @@ namespace libmpdataxx if (this->mem->distmem.rank() == this->mem->distmem.size() - 1) - { cshape[0] += 1; - cshape_ref[0] += 1; - } offst[0] = this->mem->grid_size[0].first(); offst_h[0] = this->mem->distmem.rank() == 0 ? 0 : this->mem->grid_size[0].first() + this->halo; - offst_ref[0] = this->mem->grid_size_ref[0].first(); if (this->mem->distmem.rank() > 0) offst_mem_h[0] = this->halo; @@ -159,7 +145,6 @@ namespace libmpdataxx // TODO: set to 1? Test performance... chunk[0] = ( (typename solver_t::real_t) (this->mem->distmem.grid_size[0])) / this->mem->distmem.size() + 0.5 ; chunk_h[0] = 1;//chunk[0]; - chunk_ref[0] = ( (typename solver_t::real_t) (this->mem->distmem.grid_size_ref[0])) / this->mem->distmem.size() + 0.5 ; } #endif @@ -330,13 +315,16 @@ namespace libmpdataxx }; } - void record_dsc_helper(const H5::DataSet &dset, const typename solver_t::arr_t &arr, const bool refined = false) + void record_dsc_helper( + const H5::DataSet &dset, const typename solver_t::arr_t &arr, + const std::array &_grid_size, + const blitz::TinyVector &_shape, + const blitz::TinyVector &_offst + ) { H5::DataSpace space = dset.getSpace(); - const auto _grid_size(refined ? this->mem->grid_size_ref : this->mem->grid_size); - const auto _shape(refined ? shape_ref : shape); - space.selectHyperslab(H5S_SELECT_SET, _shape.data(), refined ? offst_ref.data() : offst.data()); + space.selectHyperslab(H5S_SELECT_SET, _shape.data(), _offst.data()); // TODO: some permutation of grid_size instead of the switch switch (int(solver_t::n_dims)) @@ -365,6 +353,11 @@ namespace libmpdataxx }; } + void record_dsc_helper(const H5::DataSet &dset, const typename solver_t::arr_t &arr) + { + record_dsc_helper(dset, arr, this->mem->grid_size, shape, offst); + } + // data is assumed to be contiguous and in the same layout as hdf variable and in the C-style storage order void record_aux_hlpr(const std::string &name, typename solver_t::real_t *data, H5::H5File hdf, bool refined = false) { @@ -847,7 +840,9 @@ namespace libmpdataxx using parent_t = hdf5_common; using parent_t::parent_t; using output_t = hdf5; -// using n_dims = typename parent_t::n_dims; + + blitz::TinyVector shape_ref, cshape_ref, chunk_ref, offst_ref; + H5::DataSpace sspace_ref, cspace_ref; std::map dim_names_ref; @@ -855,6 +850,31 @@ namespace libmpdataxx { parent_t::start(nt); + offst_ref = 0; + for (int d = 0; d < parent_t::n_dims; ++d) + shape_ref[d] = this->mem->distmem.grid_size_ref[d]; + + chunk_ref = shape_ref; + cshape_ref = shape_ref + 1; + sspace_ref = H5::DataSpace(parent_t::n_dims, shape_ref.data()); + cspace_ref = H5::DataSpace(parent_t::n_dims, cshape_ref.data()); + +#if defined(USE_MPI) + if (this->mem->distmem.size() > 1) + { + shape_ref[0] = this->mem->distmem.rank() < this->mem->distmem.size() - 1 ? + this->mem->grid_size_ref[0].length()-1 : // -1 to make ranges nonoverlapping + this->mem->grid_size_ref[0].length(); + cshape_ref[0] = shape_ref[0]; + + if (this->mem->distmem.rank() == this->mem->distmem.size() - 1) + cshape_ref[0] += 1; + + offst_ref[0] = this->mem->grid_size_ref[0].first(); + chunk_ref[0] = ( (typename solver_t::real_t) (this->mem->distmem.grid_size_ref[0])) / this->mem->distmem.size() + 0.5 ; + } +#endif + // refined X, Y, Z, TODO: very similar to X, Y, Z for (int i = 0; i < parent_t::n_dims; ++i) { @@ -886,6 +906,19 @@ namespace libmpdataxx curr_dim.write(coord.data(), this->flttype_solver, H5::DataSpace(parent_t::n_dims, this->cshape_ref.data()), dim_space, this->dxpl_id); } } + + void record_dsc_helper(const H5::DataSet &dset, const typename solver_t::arr_t &arr, const bool refined=false) + { + const auto _grid_size(refined ? this->mem->grid_size_ref : this->mem->grid_size); + const auto _shape(refined ? shape_ref : shape); + space.selectHyperslab(H5S_SELECT_SET, _shape.data(), refined ? offst_ref.data() : offst.data()); + + record_dsc_helper(dset, arr, + refined ? this->mem->grid_size_ref : this->mem->grid_size, + refined ? shape_ref : this->shape, + refined ? offst_ref : this->offst, + ); + } }; } // namespace output From 25e8f0f5c9fed76875561da2b3311b36abef4289 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Thu, 23 Mar 2023 10:51:23 +0100 Subject: [PATCH 097/129] hdf5: move _ref output to ref specialization: aux_dsc --- libmpdata++/output/hdf5.hpp | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/libmpdata++/output/hdf5.hpp b/libmpdata++/output/hdf5.hpp index 1d63d198..033f8609 100644 --- a/libmpdata++/output/hdf5.hpp +++ b/libmpdata++/output/hdf5.hpp @@ -359,26 +359,30 @@ namespace libmpdataxx } // data is assumed to be contiguous and in the same layout as hdf variable and in the C-style storage order - void record_aux_hlpr(const std::string &name, typename solver_t::real_t *data, H5::H5File hdf, bool refined = false) + void record_aux_hlpr( + const std::string &name, typename solver_t::real_t *data, H5::H5File &hdf, + const H5::DataSpace &_sspace, + const blitz::TinyVector &_shape, + const blitz::TinyVector &_offst + ) { assert(this->rank == 0); - const auto _shape(refined ? shape_ref : shape); - const auto _offst(refined ? offst_ref : offst); - if(refined) params.setChunk(parent_t::n_dims, chunk_ref.data()); auto aux = hdf.createDataSet( name, flttype_output, - refined ? sspace_ref : sspace, + _sspace, params ); auto space = aux.getSpace(); space.selectHyperslab(H5S_SELECT_SET, _shape.data(), _offst.data()); aux.write(data, flttype_solver, H5::DataSpace(parent_t::n_dims, _shape.data()), space, dxpl_id); - - // revert to default chunk - if(refined) params.setChunk(parent_t::n_dims, chunk.data()); + } + + void record_aux_hlpr(const std::string &name, typename solver_t::real_t *data, H5::H5File &hdf) + { + record_aux_hlpr(name, data, hdf5, sspace, shape, offst); } // for discontiguous array with halos @@ -919,6 +923,20 @@ namespace libmpdataxx refined ? offst_ref : this->offst, ); } + + void record_aux_hlpr(const std::string &name, typename solver_t::real_t *data, H5::H5File &hdf, bool refined = false) + { + if(refined) this->params.setChunk(parent_t::n_dims, chunk_ref.data()); + + record_aux_hlpr(name, data, hdf5, + refined ? sspace_ref : this->sspace, + refined ? shape_ref : this->shape, + refined ? offst_ref : this->offst + ); + + // revert to default chunk + if(refined) this->params.setChunk(parent_t::n_dims, this->chunk.data()); + } }; } // namespace output From 155975fc67a18863873a30d3530e19d498dce697 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Thu, 23 Mar 2023 14:50:13 +0100 Subject: [PATCH 098/129] record aux dsc hlpr ref/nonref separation --- libmpdata++/output/hdf5.hpp | 69 ++++++++++++++++++++++++++++++++----- 1 file changed, 60 insertions(+), 9 deletions(-) diff --git a/libmpdata++/output/hdf5.hpp b/libmpdata++/output/hdf5.hpp index 033f8609..954cfe45 100644 --- a/libmpdata++/output/hdf5.hpp +++ b/libmpdata++/output/hdf5.hpp @@ -382,37 +382,46 @@ namespace libmpdataxx void record_aux_hlpr(const std::string &name, typename solver_t::real_t *data, H5::H5File &hdf) { - record_aux_hlpr(name, data, hdf5, sspace, shape, offst); + record_aux_hlpr(name, data, hdf, sspace, shape, offst); } // for discontiguous array with halos - void record_aux_dsc_hlpr(const std::string &name, const typename solver_t::arr_t &arr, H5::H5File hdf, bool srfc = false, bool refined = false) + void record_aux_dsc_hlpr( + const std::string &name, const typename solver_t::arr_t &arr, H5::H5File hdf, bool srfc, + std::function &_record_dsc_helper, + blitz::TinyVector _chunk, + const H5::DataSpace &_sspace + ) { assert(this->rank == 0); - assert(!(refined && srfc)); if(srfc) params.setChunk(parent_t::n_dims, srfcchunk.data()); - else if (refined) - params.setChunk(parent_t::n_dims, chunk_ref.data()); + else + params.setChunk(parent_t::n_dims, _chunk.data()); auto aux = hdf.createDataSet( name, flttype_output, - srfc ? srfcspace : refined ? sspace_ref : sspace, + srfc ? srfcspace : _sspace, params ); if(srfc) record_dsc_srfc_helper(aux, arr); else - record_dsc_helper(aux, arr, refined); + _record_dsc_helper(aux, arr); // revert to default chunk - if(srfc || refined) - params.setChunk(parent_t::n_dims, chunk.data()); + params.setChunk(parent_t::n_dims, chunk.data()); } + void record_aux_dsc_hlpr(const std::string &name, const typename solver_t::arr_t &arr, H5::H5File hdf, bool srfc = false) + { + record_aux_dsc_hlpr(name, arr, hdf, srfc, record_dsc_helper, chunk, sspace); + } + + // for array + halo void record_aux_halo_hlpr(const std::string &name, const typename solver_t::arr_t &arr, H5::H5File hdf) { @@ -937,6 +946,48 @@ namespace libmpdataxx // revert to default chunk if(refined) this->params.setChunk(parent_t::n_dims, this->chunk.data()); } + + // for discontiguous array with halos + void record_aux_dsc_hlpr( + const std::string &name, const typename solver_t::arr_t &arr, H5::H5File hdf, bool srfc, + blitz::TinyVector _chunk, + const H5::DataSpace &_sspace + { + assert(this->rank == 0); +// assert(!(refined && srfc)); + +/* + if(srfc) + params.setChunk(parent_t::n_dims, srfcchunk.data()); + else if (refined) + params.setChunk(parent_t::n_dims, chunk_ref.data()); + */ + if(srfc) + params.setChunk(parent_t::n_dims, srfcchunk.data()); + else + params.setChunk(parent_t::n_dims, _chunk.data()); + + auto aux = hdf.createDataSet( + name, + flttype_output, + srfc ? srfcspace : _sspace, + params + ); + + if(srfc) + record_dsc_srfc_helper(aux, arr); + else + record_dsc_helper(aux, arr); + //record_dsc_helper(aux, arr, refined); + + // revert to default chunk + params.setChunk(parent_t::n_dims, chunk.data()); + } + + void record_aux_dsc_hlpr(const std::string &name, const typename solver_t::arr_t &arr, H5::H5File hdf, bool refined = false) + { + parent_t::record_aux_dsc_hlpr(name, arr, hdf, false, std::bind(&record_dsc_helper, this, std::placeholders::_1, std::placeholders::_2, refined), refined ? chunk_ref : chunk, refined ? sspace_ref : sspace); + } }; } // namespace output From 2807e520fa56d42e060cb5685e2dddc225551ef9 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Thu, 23 Mar 2023 15:20:16 +0100 Subject: [PATCH 099/129] cd hdf5_ref ... --- libmpdata++/output/hdf5.hpp | 161 +------------------------------- libmpdata++/output/hdf5_ref.hpp | 136 +++++++++++++++++++++++++++ 2 files changed, 137 insertions(+), 160 deletions(-) create mode 100644 libmpdata++/output/hdf5_ref.hpp diff --git a/libmpdata++/output/hdf5.hpp b/libmpdata++/output/hdf5.hpp index 954cfe45..1b7b42fd 100644 --- a/libmpdata++/output/hdf5.hpp +++ b/libmpdata++/output/hdf5.hpp @@ -521,12 +521,7 @@ namespace libmpdataxx // data is assumed to be contiguous and in the same layout as hdf variable and in the C-style storage order void record_aux(const std::string &name, typename solver_t::real_t *data) { - record_aux_hlpr(name, data, *hdfp, false); - } - - void record_aux_refined(const std::string &name, typename solver_t::real_t *data) - { - record_aux_hlpr(name, data, *hdfp, true); + record_aux_hlpr(name, data, *hdfp); } void record_aux_dsc(const std::string &name, const typename solver_t::arr_t &arr, bool srfc = false) @@ -534,11 +529,6 @@ namespace libmpdataxx record_aux_dsc_hlpr(name, arr, *hdfp, srfc); } - void record_aux_dsc_refined(const std::string &name, const typename solver_t::arr_t &arr) - { - record_aux_dsc_hlpr(name, arr, *hdfp, false, true); - } - void record_aux_scalar(const std::string &name, const std::string &group_name, typename solver_t::real_t data) { record_scalar_hlpr(name, group_name, data, *hdfp); @@ -554,7 +544,6 @@ namespace libmpdataxx record_prof_hlpr(*hdfp, name, data, vctr, refined); } - // ---- functions for auxiliary output in const.h5 file ---- // has to be called after const file was created (i.e. after start()) @@ -842,153 +831,5 @@ namespace libmpdataxx // using n_dims = typename parent_t::n_dims; }; - // specialization for solvers with grid refinemenet - template - class hdf5 - > : public hdf5_common - { - protected: - - using parent_t = hdf5_common; - using parent_t::parent_t; - using output_t = hdf5; - - blitz::TinyVector shape_ref, cshape_ref, chunk_ref, offst_ref; - H5::DataSpace sspace_ref, cspace_ref; - - std::map dim_names_ref; - - void start(const typename parent_t::advance_arg_t nt) override - { - parent_t::start(nt); - - offst_ref = 0; - for (int d = 0; d < parent_t::n_dims; ++d) - shape_ref[d] = this->mem->distmem.grid_size_ref[d]; - - chunk_ref = shape_ref; - cshape_ref = shape_ref + 1; - sspace_ref = H5::DataSpace(parent_t::n_dims, shape_ref.data()); - cspace_ref = H5::DataSpace(parent_t::n_dims, cshape_ref.data()); - -#if defined(USE_MPI) - if (this->mem->distmem.size() > 1) - { - shape_ref[0] = this->mem->distmem.rank() < this->mem->distmem.size() - 1 ? - this->mem->grid_size_ref[0].length()-1 : // -1 to make ranges nonoverlapping - this->mem->grid_size_ref[0].length(); - cshape_ref[0] = shape_ref[0]; - - if (this->mem->distmem.rank() == this->mem->distmem.size() - 1) - cshape_ref[0] += 1; - - offst_ref[0] = this->mem->grid_size_ref[0].first(); - chunk_ref[0] = ( (typename solver_t::real_t) (this->mem->distmem.grid_size_ref[0])) / this->mem->distmem.size() + 0.5 ; - } -#endif - - // refined X, Y, Z, TODO: very similar to X, Y, Z - for (int i = 0; i < parent_t::n_dims; ++i) - { - blitz::Array coord(this->cshape_ref); -#if defined(USE_MPI) - coord.reindexSelf(this->offst_ref); -#endif - std::string name; - switch (i) - { - case 0 : coord = this->di / 2. + this->di / this->mem->n_ref * (blitz::firstIndex() - .5); - name = "X refined"; - dim_names_ref[i] = name; - break; - case 1 : coord = this->dj / 2. + this->dj / this->mem->n_ref * (blitz::secondIndex() - .5); - name = "Y refined"; - dim_names_ref[i] = name; - break; - case 2 : coord = this->dk / 2. + this->dk / this->mem->n_ref * (blitz::thirdIndex() - .5); - name = "Z refined"; - dim_names_ref[i] = name; - break; - default : break; - } - auto curr_dim = (*(this->hdfp)).createDataSet(name, this->flttype_output, this->cspace_ref); - - H5::DataSpace dim_space = curr_dim.getSpace(); - dim_space.selectHyperslab(H5S_SELECT_SET, this->cshape_ref.data(), this->offst_ref.data()); - curr_dim.write(coord.data(), this->flttype_solver, H5::DataSpace(parent_t::n_dims, this->cshape_ref.data()), dim_space, this->dxpl_id); - } - } - - void record_dsc_helper(const H5::DataSet &dset, const typename solver_t::arr_t &arr, const bool refined=false) - { - const auto _grid_size(refined ? this->mem->grid_size_ref : this->mem->grid_size); - const auto _shape(refined ? shape_ref : shape); - space.selectHyperslab(H5S_SELECT_SET, _shape.data(), refined ? offst_ref.data() : offst.data()); - - record_dsc_helper(dset, arr, - refined ? this->mem->grid_size_ref : this->mem->grid_size, - refined ? shape_ref : this->shape, - refined ? offst_ref : this->offst, - ); - } - - void record_aux_hlpr(const std::string &name, typename solver_t::real_t *data, H5::H5File &hdf, bool refined = false) - { - if(refined) this->params.setChunk(parent_t::n_dims, chunk_ref.data()); - - record_aux_hlpr(name, data, hdf5, - refined ? sspace_ref : this->sspace, - refined ? shape_ref : this->shape, - refined ? offst_ref : this->offst - ); - - // revert to default chunk - if(refined) this->params.setChunk(parent_t::n_dims, this->chunk.data()); - } - - // for discontiguous array with halos - void record_aux_dsc_hlpr( - const std::string &name, const typename solver_t::arr_t &arr, H5::H5File hdf, bool srfc, - blitz::TinyVector _chunk, - const H5::DataSpace &_sspace - { - assert(this->rank == 0); -// assert(!(refined && srfc)); - -/* - if(srfc) - params.setChunk(parent_t::n_dims, srfcchunk.data()); - else if (refined) - params.setChunk(parent_t::n_dims, chunk_ref.data()); - */ - if(srfc) - params.setChunk(parent_t::n_dims, srfcchunk.data()); - else - params.setChunk(parent_t::n_dims, _chunk.data()); - - auto aux = hdf.createDataSet( - name, - flttype_output, - srfc ? srfcspace : _sspace, - params - ); - - if(srfc) - record_dsc_srfc_helper(aux, arr); - else - record_dsc_helper(aux, arr); - //record_dsc_helper(aux, arr, refined); - - // revert to default chunk - params.setChunk(parent_t::n_dims, chunk.data()); - } - - void record_aux_dsc_hlpr(const std::string &name, const typename solver_t::arr_t &arr, H5::H5File hdf, bool refined = false) - { - parent_t::record_aux_dsc_hlpr(name, arr, hdf, false, std::bind(&record_dsc_helper, this, std::placeholders::_1, std::placeholders::_2, refined), refined ? chunk_ref : chunk, refined ? sspace_ref : sspace); - } - }; - } // namespace output } // namespace libmpdataxx diff --git a/libmpdata++/output/hdf5_ref.hpp b/libmpdata++/output/hdf5_ref.hpp new file mode 100644 index 00000000..57426396 --- /dev/null +++ b/libmpdata++/output/hdf5_ref.hpp @@ -0,0 +1,136 @@ +/** + * @file + * @copyright University of Warsaw + * @section LICENSE + * GPLv3+ (see the COPYING file or http://www.gnu.org/licenses/) + * @brief HDF5 output logic targetted at Paraview-xdmf reader + */ + + +#pragma once + +#include + +namespace libmpdataxx +{ + namespace output + { + // specialization for solvers with grid refinemenet + template + class hdf5 + > : public hdf5_common + { + protected: + + using parent_t = hdf5_common; + using parent_t::parent_t; + using output_t = hdf5; + + blitz::TinyVector shape_ref, cshape_ref, chunk_ref, offst_ref; + H5::DataSpace sspace_ref, cspace_ref; + + std::map dim_names_ref; + + void start(const typename parent_t::advance_arg_t nt) override + { + parent_t::start(nt); + + offst_ref = 0; + for (int d = 0; d < parent_t::n_dims; ++d) + shape_ref[d] = this->mem->distmem.grid_size_ref[d]; + + chunk_ref = shape_ref; + cshape_ref = shape_ref + 1; + sspace_ref = H5::DataSpace(parent_t::n_dims, shape_ref.data()); + cspace_ref = H5::DataSpace(parent_t::n_dims, cshape_ref.data()); + +#if defined(USE_MPI) + if (this->mem->distmem.size() > 1) + { + shape_ref[0] = this->mem->distmem.rank() < this->mem->distmem.size() - 1 ? + this->mem->grid_size_ref[0].length()-1 : // -1 to make ranges nonoverlapping + this->mem->grid_size_ref[0].length(); + cshape_ref[0] = shape_ref[0]; + + if (this->mem->distmem.rank() == this->mem->distmem.size() - 1) + cshape_ref[0] += 1; + + offst_ref[0] = this->mem->grid_size_ref[0].first(); + chunk_ref[0] = ( (typename solver_t::real_t) (this->mem->distmem.grid_size_ref[0])) / this->mem->distmem.size() + 0.5 ; + } +#endif + + // refined X, Y, Z, TODO: very similar to X, Y, Z + for (int i = 0; i < parent_t::n_dims; ++i) + { + blitz::Array coord(this->cshape_ref); +#if defined(USE_MPI) + coord.reindexSelf(this->offst_ref); +#endif + std::string name; + switch (i) + { + case 0 : coord = this->di / 2. + this->di / this->mem->n_ref * (blitz::firstIndex() - .5); + name = "X refined"; + dim_names_ref[i] = name; + break; + case 1 : coord = this->dj / 2. + this->dj / this->mem->n_ref * (blitz::secondIndex() - .5); + name = "Y refined"; + dim_names_ref[i] = name; + break; + case 2 : coord = this->dk / 2. + this->dk / this->mem->n_ref * (blitz::thirdIndex() - .5); + name = "Z refined"; + dim_names_ref[i] = name; + break; + default : break; + } + auto curr_dim = (*(this->hdfp)).createDataSet(name, this->flttype_output, this->cspace_ref); + + H5::DataSpace dim_space = curr_dim.getSpace(); + dim_space.selectHyperslab(H5S_SELECT_SET, this->cshape_ref.data(), this->offst_ref.data()); + curr_dim.write(coord.data(), this->flttype_solver, H5::DataSpace(parent_t::n_dims, this->cshape_ref.data()), dim_space, this->dxpl_id); + } + } + + void record_dsc_ref_helper(const H5::DataSet &dset, const typename solver_t::arr_t &arr) + { + parent_t::record_dsc_helper(dset, arr, this->mem->grid_size_ref, shape_ref, offst_ref); + } + + void record_aux_ref_hlpr(const std::string &name, typename solver_t::real_t *data, H5::H5File &hdf) + { + this->params.setChunk(parent_t::n_dims, chunk_ref.data()); + + record_aux_hlpr(name, data, hdf5, sspace_ref, shape_ref, offst_ref); + + // revert to default chunk + this->params.setChunk(parent_t::n_dims, this->chunk.data()); + } + + void record_aux_dsc_ref_hlpr(const std::string &name, const typename solver_t::arr_t &arr, H5::H5File hdf) + { + parent_t::record_aux_dsc_hlpr(name, arr, hdf, false, + record_dsc_ref_helper, + chunk_ref, + sspace_ref + ); + } + + + + + void record_aux_dsc_refined(const std::string &name, const typename solver_t::arr_t &arr) + { + record_aux_dsc_ref_hlpr(name, arr, *hdfp); + } + + void record_aux_refined(const std::string &name, typename solver_t::real_t *data) + { + record_aux_ref_hlpr(name, data, *hdfp); + } + + }; + + } // namespace output +} // namespace libmpdataxx From 23efe74c0d22b2d8b54a2963940d4bdceee0ba13 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Thu, 23 Mar 2023 15:55:14 +0100 Subject: [PATCH 100/129] cd hdf5_ref ... --- libmpdata++/output/hdf5.hpp | 47 ++++++++++++++++++++------------- libmpdata++/output/hdf5_ref.hpp | 6 +++++ 2 files changed, 35 insertions(+), 18 deletions(-) diff --git a/libmpdata++/output/hdf5.hpp b/libmpdata++/output/hdf5.hpp index 1b7b42fd..ce4684d1 100644 --- a/libmpdata++/output/hdf5.hpp +++ b/libmpdata++/output/hdf5.hpp @@ -387,7 +387,7 @@ namespace libmpdataxx // for discontiguous array with halos void record_aux_dsc_hlpr( - const std::string &name, const typename solver_t::arr_t &arr, H5::H5File hdf, bool srfc, + const std::string &name, const typename solver_t::arr_t &arr, H5::H5File hdf, std::function &_record_dsc_helper, blitz::TinyVector _chunk, const H5::DataSpace &_sspace @@ -395,30 +395,29 @@ namespace libmpdataxx { assert(this->rank == 0); - if(srfc) - params.setChunk(parent_t::n_dims, srfcchunk.data()); - else - params.setChunk(parent_t::n_dims, _chunk.data()); + params.setChunk(parent_t::n_dims, _chunk.data()); auto aux = hdf.createDataSet( name, flttype_output, - srfc ? srfcspace : _sspace, + _sspace, params ); - if(srfc) - record_dsc_srfc_helper(aux, arr); - else - _record_dsc_helper(aux, arr); + _record_dsc_helper(aux, arr); // revert to default chunk params.setChunk(parent_t::n_dims, chunk.data()); } - void record_aux_dsc_hlpr(const std::string &name, const typename solver_t::arr_t &arr, H5::H5File hdf, bool srfc = false) + void record_aux_dsc_hlpr(const std::string &name, const typename solver_t::arr_t &arr, H5::H5File hdf) + { + record_aux_dsc_hlpr(name, arr, hdf, record_dsc_helper, chunk, sspace); + } + + void record_aux_dsc_srfc_hlpr(const std::string &name, const typename solver_t::arr_t &arr, H5::H5File hdf) { - record_aux_dsc_hlpr(name, arr, hdf, srfc, record_dsc_helper, chunk, sspace); + record_aux_dsc_hlpr(name, arr, hdf, record_dsc_srfc_helper, srfcchunk, srfcsspace); } @@ -492,13 +491,12 @@ namespace libmpdataxx } // record 1D profiles, assumes that z is the last dimension - void record_prof_hlpr(H5::H5File hdff, const std::string &name, typename solver_t::real_t *data, const bool vctr, const bool refined) + void record_prof_hlpr(H5::H5File hdff, const std::string &name, typename solver_t::real_t *data, + const blitz::TinyVector &_shape, + const blitz::TinyVector &_offst + ) { assert(this->rank == 0); - assert(((vctr && refined) == false) && "record prof hlpr cant save refined vector profiles"); - - const auto _shape(refined ? shape_ref : vctr ? cshape : shape); - const auto _offst(refined ? offst_ref : offst); auto aux = hdff.createDataSet( name, @@ -516,6 +514,16 @@ namespace libmpdataxx } } + void record_prof_hlpr(H5::H5File hdff, const std::string &name, typename solver_t::real_t *data) + { + record_prof_hlpr(hdff, name, data, shape, offst); + } + + void record_prof_vctr_hlpr(H5::H5File hdff, const std::string &name, typename solver_t::real_t *data) + { + record_prof_hlpr(hdff, name, data, cshape, offst); + } + // ---- functions for auxiliary output in timestep files ---- // data is assumed to be contiguous and in the same layout as hdf variable and in the C-style storage order @@ -526,7 +534,10 @@ namespace libmpdataxx void record_aux_dsc(const std::string &name, const typename solver_t::arr_t &arr, bool srfc = false) { - record_aux_dsc_hlpr(name, arr, *hdfp, srfc); + if(srfc) + record_aux_dsc_hlpr(name, arr, *hdfp); + else + record_aux_dsc_srfc_hlpr(name, arr, *hdfp); } void record_aux_scalar(const std::string &name, const std::string &group_name, typename solver_t::real_t data) diff --git a/libmpdata++/output/hdf5_ref.hpp b/libmpdata++/output/hdf5_ref.hpp index 57426396..8a15ef9b 100644 --- a/libmpdata++/output/hdf5_ref.hpp +++ b/libmpdata++/output/hdf5_ref.hpp @@ -117,6 +117,12 @@ namespace libmpdataxx ); } + void record_prof_ref_hlpr(H5::H5File hdff, const std::string &name, typename solver_t::real_t *data) + { + record_prof_hlpr(hdff, name, data, shape_ref, offst_ref); + } + + From 8543f8d8d59772668fdd1ca8ae7878e58d9f246a Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Thu, 23 Mar 2023 16:51:27 +0100 Subject: [PATCH 101/129] record aux dsc hlpr ref/nonref separation --- libmpdata++/output/hdf5.hpp | 30 ++++++++++++++++++------------ libmpdata++/output/hdf5_ref.hpp | 15 +++++++++++++++ 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/libmpdata++/output/hdf5.hpp b/libmpdata++/output/hdf5.hpp index ce4684d1..ebe256af 100644 --- a/libmpdata++/output/hdf5.hpp +++ b/libmpdata++/output/hdf5.hpp @@ -216,7 +216,7 @@ namespace libmpdataxx if (this->mem->G.get() != nullptr) { auto g_set = (*hdfp).createDataSet("G", flttype_output, sspace); - record_dsc_helper(g_set, *this->mem->G); + record_dsc_hlpr(g_set, *this->mem->G); } // save selected compile and runtime parameters, the choice depends on the solver family @@ -271,7 +271,7 @@ namespace libmpdataxx ); // TODO: units attribute - record_dsc_helper(vars[v.first], this->out_data(v.first)); + record_dsc_hlpr(vars[v.first], this->out_data(v.first)); } } } @@ -353,7 +353,7 @@ namespace libmpdataxx }; } - void record_dsc_helper(const H5::DataSet &dset, const typename solver_t::arr_t &arr) + void record_dsc_hlpr(const H5::DataSet &dset, const typename solver_t::arr_t &arr) { record_dsc_helper(dset, arr, this->mem->grid_size, shape, offst); } @@ -388,7 +388,7 @@ namespace libmpdataxx // for discontiguous array with halos void record_aux_dsc_hlpr( const std::string &name, const typename solver_t::arr_t &arr, H5::H5File hdf, - std::function &_record_dsc_helper, + void(*_record_dsc_helper)(const H5::DataSet &, const typename solver_t::arr_t&) , blitz::TinyVector _chunk, const H5::DataSpace &_sspace ) @@ -412,12 +412,12 @@ namespace libmpdataxx void record_aux_dsc_hlpr(const std::string &name, const typename solver_t::arr_t &arr, H5::H5File hdf) { - record_aux_dsc_hlpr(name, arr, hdf, record_dsc_helper, chunk, sspace); + record_aux_dsc_hlpr(name, arr, hdf, &hdf5_common::record_dsc_hlpr, chunk, sspace); } void record_aux_dsc_srfc_hlpr(const std::string &name, const typename solver_t::arr_t &arr, H5::H5File hdf) { - record_aux_dsc_hlpr(name, arr, hdf, record_dsc_srfc_helper, srfcchunk, srfcsspace); + record_aux_dsc_hlpr(name, arr, hdf, &record_dsc_srfc_helper, srfcchunk, srfcspace); } @@ -550,9 +550,12 @@ namespace libmpdataxx record_aux_scalar(name, "/", data); } - void record_aux_prof(const std::string &name, typename solver_t::real_t *data, const bool vctr = false, const bool refined = false) + void record_aux_prof(const std::string &name, typename solver_t::real_t *data, const bool vctr = false) { - record_prof_hlpr(*hdfp, name, data, vctr, refined); + if(vctr) + record_prof_vctr_hlpr(*hdfp, name, data); + else + record_prof_hlpr(*hdfp, name, data); } // ---- functions for auxiliary output in const.h5 file ---- @@ -602,7 +605,7 @@ namespace libmpdataxx } // see above, also assumes that z is the last dimension - void record_prof_const_hlpr(const std::string &name, typename solver_t::real_t *data, const bool vctr, const bool refined = false) + void record_prof_const_hlpr(const std::string &name, typename solver_t::real_t *data, const bool vctr) { H5::H5File hdfcp(const_file, H5F_ACC_RDWR #if defined(USE_MPI) @@ -610,12 +613,15 @@ namespace libmpdataxx #endif ); // reopen the const file - record_prof_hlpr(hdfcp, name, data, vctr, refined); + if(vctr) + record_prof_vctr_hlpr(hdfcp, name, data); + else + record_prof_hlpr(hdfcp, name, data); } - void record_prof_const(const std::string &name, typename solver_t::real_t *data, const bool refined = false) + void record_prof_const(const std::string &name, typename solver_t::real_t *data) { - record_prof_const_hlpr(name, data, false, refined); + record_prof_const_hlpr(name, data, false); } void record_prof_vctr_const(const std::string &name, typename solver_t::real_t *data) diff --git a/libmpdata++/output/hdf5_ref.hpp b/libmpdata++/output/hdf5_ref.hpp index 8a15ef9b..0c33aea1 100644 --- a/libmpdata++/output/hdf5_ref.hpp +++ b/libmpdata++/output/hdf5_ref.hpp @@ -136,6 +136,21 @@ namespace libmpdataxx record_aux_ref_hlpr(name, data, *hdfp); } + void record_aux_prof_refined(const std::string &name, typename solver_t::real_t *data) + { + record_prof_ref_hlpr(*hdfp, name, data); + } + + void record_prof_refined_const(const std::string &name, typename solver_t::real_t *data) + { + H5::H5File hdfcp(const_file, H5F_ACC_RDWR +#if defined(USE_MPI) + , H5P_DEFAULT, fapl_id +#endif + ); // reopen the const file + + record_prof_ref_hlpr(hdfcp, name, data); + } }; } // namespace output From d0e4f6425d32f2ab5d0c27b1def2b9ec5d718696 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Fri, 24 Mar 2023 17:47:53 +0100 Subject: [PATCH 102/129] record aux dsc hlpr ref/nonref separation: compiling version (3D only) --- libmpdata++/output/hdf5.hpp | 14 +++++++++----- libmpdata++/output/hdf5_ref.hpp | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/libmpdata++/output/hdf5.hpp b/libmpdata++/output/hdf5.hpp index ebe256af..78dae9e1 100644 --- a/libmpdata++/output/hdf5.hpp +++ b/libmpdata++/output/hdf5.hpp @@ -279,7 +279,7 @@ namespace libmpdataxx // ---- output helpers ---- - void record_dsc_srfc_helper(const H5::DataSet &dset, const typename solver_t::arr_t &arr) + void record_dsc_srfc_hlpr(const H5::DataSet &dset, const typename solver_t::arr_t &arr) { H5::DataSpace space = dset.getSpace(); space.selectHyperslab(H5S_SELECT_SET, srfcshape.data(), offst.data()); @@ -386,9 +386,11 @@ namespace libmpdataxx } // for discontiguous array with halos + template void record_aux_dsc_hlpr( const std::string &name, const typename solver_t::arr_t &arr, H5::H5File hdf, - void(*_record_dsc_helper)(const H5::DataSet &, const typename solver_t::arr_t&) , + out_t *out, + record_dsc_helper_t _record_dsc_helper, blitz::TinyVector _chunk, const H5::DataSpace &_sspace ) @@ -404,20 +406,22 @@ namespace libmpdataxx params ); - _record_dsc_helper(aux, arr); + (out->*_record_dsc_helper)(aux, arr); // revert to default chunk params.setChunk(parent_t::n_dims, chunk.data()); } + void foo() {} + void record_aux_dsc_hlpr(const std::string &name, const typename solver_t::arr_t &arr, H5::H5File hdf) { - record_aux_dsc_hlpr(name, arr, hdf, &hdf5_common::record_dsc_hlpr, chunk, sspace); + record_aux_dsc_hlpr(name, arr, hdf, this, &hdf5_common::record_dsc_hlpr, chunk, sspace); } void record_aux_dsc_srfc_hlpr(const std::string &name, const typename solver_t::arr_t &arr, H5::H5File hdf) { - record_aux_dsc_hlpr(name, arr, hdf, &record_dsc_srfc_helper, srfcchunk, srfcspace); + record_aux_dsc_hlpr(name, arr, hdf, this, &hdf5_common::record_dsc_srfc_hlpr, srfcchunk, srfcspace); } diff --git a/libmpdata++/output/hdf5_ref.hpp b/libmpdata++/output/hdf5_ref.hpp index 0c33aea1..a272d8e3 100644 --- a/libmpdata++/output/hdf5_ref.hpp +++ b/libmpdata++/output/hdf5_ref.hpp @@ -111,7 +111,7 @@ namespace libmpdataxx void record_aux_dsc_ref_hlpr(const std::string &name, const typename solver_t::arr_t &arr, H5::H5File hdf) { parent_t::record_aux_dsc_hlpr(name, arr, hdf, false, - record_dsc_ref_helper, + this, &hdf5::record_dsc_ref_helper, chunk_ref, sspace_ref ); From b8ed80116ecec24228178f54585b2cb69e75ad5a Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Mon, 27 Mar 2023 14:57:03 +0200 Subject: [PATCH 103/129] ref hdf5 fixes --- libmpdata++/output/hdf5.hpp | 6 ++---- libmpdata++/output/hdf5_ref.hpp | 22 +++++++++------------- libmpdata++/output/hdf5_xdmf.hpp | 15 ++++++++++++--- 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/libmpdata++/output/hdf5.hpp b/libmpdata++/output/hdf5.hpp index 78dae9e1..cd1ef331 100644 --- a/libmpdata++/output/hdf5.hpp +++ b/libmpdata++/output/hdf5.hpp @@ -412,8 +412,6 @@ namespace libmpdataxx params.setChunk(parent_t::n_dims, chunk.data()); } - void foo() {} - void record_aux_dsc_hlpr(const std::string &name, const typename solver_t::arr_t &arr, H5::H5File hdf) { record_aux_dsc_hlpr(name, arr, hdf, this, &hdf5_common::record_dsc_hlpr, chunk, sspace); @@ -539,9 +537,9 @@ namespace libmpdataxx void record_aux_dsc(const std::string &name, const typename solver_t::arr_t &arr, bool srfc = false) { if(srfc) - record_aux_dsc_hlpr(name, arr, *hdfp); - else record_aux_dsc_srfc_hlpr(name, arr, *hdfp); + else + record_aux_dsc_hlpr(name, arr, *hdfp); } void record_aux_scalar(const std::string &name, const std::string &group_name, typename solver_t::real_t data) diff --git a/libmpdata++/output/hdf5_ref.hpp b/libmpdata++/output/hdf5_ref.hpp index a272d8e3..815e8224 100644 --- a/libmpdata++/output/hdf5_ref.hpp +++ b/libmpdata++/output/hdf5_ref.hpp @@ -18,7 +18,7 @@ namespace libmpdataxx // specialization for solvers with grid refinemenet template class hdf5 + typename std::enable_if_t<(int)solver_t::ct_params_t_::fractal_recon != (int)0> > : public hdf5_common { protected: @@ -102,7 +102,7 @@ namespace libmpdataxx { this->params.setChunk(parent_t::n_dims, chunk_ref.data()); - record_aux_hlpr(name, data, hdf5, sspace_ref, shape_ref, offst_ref); + parent_t::record_aux_hlpr(name, data, hdf, sspace_ref, shape_ref, offst_ref); // revert to default chunk this->params.setChunk(parent_t::n_dims, this->chunk.data()); @@ -110,7 +110,7 @@ namespace libmpdataxx void record_aux_dsc_ref_hlpr(const std::string &name, const typename solver_t::arr_t &arr, H5::H5File hdf) { - parent_t::record_aux_dsc_hlpr(name, arr, hdf, false, + parent_t::record_aux_dsc_hlpr(name, arr, hdf, this, &hdf5::record_dsc_ref_helper, chunk_ref, sspace_ref @@ -119,33 +119,29 @@ namespace libmpdataxx void record_prof_ref_hlpr(H5::H5File hdff, const std::string &name, typename solver_t::real_t *data) { - record_prof_hlpr(hdff, name, data, shape_ref, offst_ref); + this->record_prof_hlpr(hdff, name, data, shape_ref, offst_ref); } - - - - void record_aux_dsc_refined(const std::string &name, const typename solver_t::arr_t &arr) { - record_aux_dsc_ref_hlpr(name, arr, *hdfp); + record_aux_dsc_ref_hlpr(name, arr, *this->hdfp); } void record_aux_refined(const std::string &name, typename solver_t::real_t *data) { - record_aux_ref_hlpr(name, data, *hdfp); + record_aux_ref_hlpr(name, data, *this->hdfp); } void record_aux_prof_refined(const std::string &name, typename solver_t::real_t *data) { - record_prof_ref_hlpr(*hdfp, name, data); + record_prof_ref_hlpr(*this->hdfp, name, data); } void record_prof_refined_const(const std::string &name, typename solver_t::real_t *data) { - H5::H5File hdfcp(const_file, H5F_ACC_RDWR + H5::H5File hdfcp(this->const_file, H5F_ACC_RDWR #if defined(USE_MPI) - , H5P_DEFAULT, fapl_id + , H5P_DEFAULT, this->fapl_id #endif ); // reopen the const file diff --git a/libmpdata++/output/hdf5_xdmf.hpp b/libmpdata++/output/hdf5_xdmf.hpp index c988fcaf..2bf4e328 100644 --- a/libmpdata++/output/hdf5_xdmf.hpp +++ b/libmpdata++/output/hdf5_xdmf.hpp @@ -9,7 +9,7 @@ #pragma once -#include +#include #include #include @@ -111,10 +111,19 @@ namespace libmpdataxx ) : parent_t(args, p) {} }; + + //template + //class hdf5_xdmf; // for solvers without grid refinemenet + //template + //class hdf5_xdmf> + //> + template - class hdf5_xdmf : public hdf5_xdmf_common + class hdf5_xdmf + : public hdf5_xdmf_common { protected: using output_t = hdf5_xdmf; @@ -127,7 +136,7 @@ namespace libmpdataxx // specialization for solvers with grid refinemenet template class hdf5_xdmf + typename std::enable_if_t<(int)solver_t::ct_params_t_::fractal_recon != (int)0> > : public hdf5_xdmf_common { protected: From 38c72dbd9529591c20ef560ba85efd4198d11cae Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Mon, 3 Apr 2023 10:40:00 +0200 Subject: [PATCH 104/129] fill refinee distmem only if distmem size > 1 --- libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 4c183a65..c883f3e7 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -57,6 +57,8 @@ namespace libmpdataxx // TODO: move to bcond or sth? would be filled only by remote bcond void fill_refinee_r2r_distmem_halos(const int e, const int halo_size) { + if(this->mem->distmem.size() == 1) return; // no distmem + const int e_ref = this->ix_r2r.at(e); // TODO: we only need to xchng along distmem direction (x) this->xchng(e); From 11579de8cf450874666dadde8963ecc1a4ca4cad Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Mon, 3 Apr 2023 15:10:47 +0200 Subject: [PATCH 105/129] xchng_ref after generating d_j --- libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index c883f3e7..5591c626 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -171,7 +171,7 @@ namespace libmpdataxx auto rand = std::bind(dis, gen); std::generate(this->d_j(this->ijk_ref_with_halo).begin(), this->d_j(this->ijk_ref_with_halo).end(), rand); this->d_j(this->ijk_ref_with_halo) = formulae::fractal::d_of_CDF_fctr{}(this->d_j(this->ijk_ref_with_halo)); - this->mem->barrier(); + xchng_ref(this->d_j, this->ijk_ref); // xchng to have the same values of d_j in the distmem halo region } // calculate refined points using (tri?)linear interpolation @@ -212,6 +212,8 @@ namespace libmpdataxx const int halo_size = 2; const int e_ref = this->ix_r2r.at(e); +// std::cerr << "reconstructing: e = " << e << " e_ref = " << e_ref << std::endl; + //// TEMPORARY //this->mem->psi_ref[e_ref] = -1000; //this->mem->barrier(); From ff8c2a4fa1e3d8a2cf06f5a1cf9570c78238e9b3 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Wed, 5 Apr 2023 12:34:34 +0200 Subject: [PATCH 106/129] use halo in spatial_avergae_ref_2_reg; TODO: if it works, grid_size is not needed --- libmpdata++/formulae/refined_grid.hpp | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/libmpdata++/formulae/refined_grid.hpp b/libmpdata++/formulae/refined_grid.hpp index 5f75f613..d91a4c99 100644 --- a/libmpdata++/formulae/refined_grid.hpp +++ b/libmpdata++/formulae/refined_grid.hpp @@ -17,15 +17,14 @@ namespace libmpdataxx // calculate regular grid scalar as an average from the refined grid // modifies the refined grid scalar! // 3d version - // averaged only from points within the domain, halos outside of the modeled domain are not used - // however, halos between MPI domains are used; + // assumes refined halos are filled // it is assumed that these are halos are large enough! // currently, the halo between MPI domains is n_ref+n_ref/2 template void spatial_average_ref2reg( arr_t arr, const ijk_t &ijk, // index of the refined point that overlaps with the regular point - const int &dist, // average calculate from the box (i-dist, i+dist)(j-dist,j+dist)(k-dist.k+dist) + const int &dist, // average is calculated from the box (i-dist, i+dist)(j-dist,j+dist)(k-dist.k+dist) const std::array grid_size, // number of cells in the domain in each direction (total, not for this MPI process), needed for averaging on edges so that it does not go out of the domain const bool volume_avg // should averaging account weights proportional to cell volume (refined cells at box edges are only partially within the regular cell) ) @@ -42,10 +41,10 @@ namespace libmpdataxx { if(id==0 && jd==0 && kd==0) continue; // dont add the overlapping point // dont add points outside of the domain - if(i+id < 0 || i+id > grid_size[0]-1 || - j+jd < 0 || j+jd > grid_size[1]-1 || - k+kd < 0 || k+kd > grid_size[2]-1) - continue; + //if(i+id < 0 || i+id > grid_size[0]-1 || + // j+jd < 0 || j+jd > grid_size[1]-1 || + // k+kd < 0 || k+kd > grid_size[2]-1) + // continue; real_t w = 1; // weight if(volume_avg) From 85700371527dc135ef5098a4e9ab7691215cc647 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Thu, 6 Apr 2023 10:44:41 +0200 Subject: [PATCH 107/129] Revert "use halo in spatial_avergae_ref_2_reg; TODO: if it works, grid_size is not needed" This reverts commit ff8c2a4fa1e3d8a2cf06f5a1cf9570c78238e9b3. --- libmpdata++/formulae/refined_grid.hpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/libmpdata++/formulae/refined_grid.hpp b/libmpdata++/formulae/refined_grid.hpp index d91a4c99..5f75f613 100644 --- a/libmpdata++/formulae/refined_grid.hpp +++ b/libmpdata++/formulae/refined_grid.hpp @@ -17,14 +17,15 @@ namespace libmpdataxx // calculate regular grid scalar as an average from the refined grid // modifies the refined grid scalar! // 3d version - // assumes refined halos are filled + // averaged only from points within the domain, halos outside of the modeled domain are not used + // however, halos between MPI domains are used; // it is assumed that these are halos are large enough! // currently, the halo between MPI domains is n_ref+n_ref/2 template void spatial_average_ref2reg( arr_t arr, const ijk_t &ijk, // index of the refined point that overlaps with the regular point - const int &dist, // average is calculated from the box (i-dist, i+dist)(j-dist,j+dist)(k-dist.k+dist) + const int &dist, // average calculate from the box (i-dist, i+dist)(j-dist,j+dist)(k-dist.k+dist) const std::array grid_size, // number of cells in the domain in each direction (total, not for this MPI process), needed for averaging on edges so that it does not go out of the domain const bool volume_avg // should averaging account weights proportional to cell volume (refined cells at box edges are only partially within the regular cell) ) @@ -41,10 +42,10 @@ namespace libmpdataxx { if(id==0 && jd==0 && kd==0) continue; // dont add the overlapping point // dont add points outside of the domain - //if(i+id < 0 || i+id > grid_size[0]-1 || - // j+jd < 0 || j+jd > grid_size[1]-1 || - // k+kd < 0 || k+kd > grid_size[2]-1) - // continue; + if(i+id < 0 || i+id > grid_size[0]-1 || + j+jd < 0 || j+jd > grid_size[1]-1 || + k+kd < 0 || k+kd > grid_size[2]-1) + continue; real_t w = 1; // weight if(volume_avg) From 6ca6059d5898d68018285a35faa3dbcbf5a69da6 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Thu, 6 Apr 2023 11:05:33 +0200 Subject: [PATCH 108/129] spatial average r2r weighting: account for domain edges --- libmpdata++/formulae/refined_grid.hpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libmpdata++/formulae/refined_grid.hpp b/libmpdata++/formulae/refined_grid.hpp index 5f75f613..662a6f46 100644 --- a/libmpdata++/formulae/refined_grid.hpp +++ b/libmpdata++/formulae/refined_grid.hpp @@ -50,6 +50,12 @@ namespace libmpdataxx real_t w = 1; // weight if(volume_avg) { + // refined cells that are on the edge of the domain + if(i+id == 0 || i+id == grid_size[0]-1) w/=2.; + if(j+jd == 0 || j+jd == grid_size[1]-1) w/=2.; + if(k+kd == 0 || k+kd == grid_size[2]-1) w/=2.; + + // refined cells that are half-between two resolved cells if(id==-dist || id == dist) w/=2.; if(jd==-dist || jd == dist) w/=2.; if(kd==-dist || kd == dist) w/=2.; From ee3a1f43ad2905b99cfb23dad797eb3ab219140b Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Fri, 7 Apr 2023 12:13:34 +0200 Subject: [PATCH 109/129] add avg_edge_sclr_ref and spatial_average_ref2reg --- .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 38 +++++++++++++++++++ .../mpdata_rhs_vip_prs_sgs_fra_common.hpp | 1 + 2 files changed, 39 insertions(+) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 5591c626..be6b36ab 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -122,6 +122,42 @@ namespace libmpdataxx } } + void avg_edge_sclr_ref(typename parent_t::arr_t &arr, + const idx_t<3> &range_ijk + ) + { + this->mem->barrier(); + + for (auto &bc : this->bcs_ref[0]) bc->copy_edge_sclr_to_halo1_cyclic(arr, range_ijk[1], range_ijk[2]); +// barrier_if_single_threaded_bc0(); // not necessary? + for (auto &bc : this->bcs_ref[1]) bc->copy_edge_sclr_to_halo1_cyclic(arr, range_ijk[2], range_ijk[0]); + for (auto &bc : this->bcs_ref[2]) bc->copy_edge_sclr_to_halo1_cyclic(arr, range_ijk[0], range_ijk[1]); + this->mem->barrier(); // wait for all threads to copy edge to halo before modifying edge. + // NOTE: averaging this way gives different weights to different points in corners depending on the order (corners are averaged twice and each averaging is dividing by 2) + for (auto &bc : this->bcs_ref[0]) bc->avg_edge_and_halo1_sclr_cyclic(arr, range_ijk[1], range_ijk[2]); + for (auto &bc : this->bcs_ref[1]) bc->avg_edge_and_halo1_sclr_cyclic(arr, range_ijk[2], range_ijk[0]); + for (auto &bc : this->bcs_ref[2]) bc->avg_edge_and_halo1_sclr_cyclic(arr, range_ijk[0], range_ijk[1]); + + this->mem->barrier(); + } + + // compute arr_reg as average from cells of arr_ref that are overlapping with the arr_reg cell + // NOTE: modifies arr_ref too! + void spatial_average_ref2reg(typename parent_t::arr_t &arr_ref, typename parent_t::arr_t &arr_reg) + { + this->xchng_ref(arr_ref, this->ijk_ref); // fill distmem halos of arr_ref + // std::cerr << "rx_ref(ijk_ref) after xchng_ref: " << rx_ref(this->ijk_ref) << std::endl; + // std::cerr << "rx_ref after xchng_ref: " << rx_ref << std::endl; + libmpdataxx::formulae::refined::spatial_average_ref2reg(arr_ref, this->ijk_r2r, this->mem->n_ref/2, this->mem->distmem.grid_size_ref, true); // calculate the average and store in arr_ref + // std::cerr << "rx_ref(ijk_ref) after spatial average: " << rx_ref(this->ijk_ref) << std::endl; + arr_reg(this->ijk) = arr_ref(this->ijk_r2r); // copy to arr_reg + // std::cerr << "rx(ijk) after copy from refined: " << rx(this->ijk) << std::endl; + nancheck(arr_reg(this->ijk), "arr_reg after copying from arr_ref in spatial_average_ref2reg"); +// rx(this->ijk) *= 4./3. * 1000. * 3.14159; // get mixing ratio [kg/kg] + this->mem->barrier(); + this->avg_edge_sclr(arr_reg, this->ijk); // in case of cyclic bcond, arr_reg on edges needs to be the same + } + void refinement_ranges(const int iter, const int halo_size) { // messy, because in domain decomposition (sharedmem and distmem) some refined scalars are on the edge of the subdomain... @@ -203,6 +239,7 @@ namespace libmpdataxx formulae::fractal::intrp<2, real_t>(this->mem->psi_ref[e_ref], mid_ijk_r2r_2, ijk_r2r_0_h_with_halo, this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); } this->mem->barrier(); +// avg_edge_sclr_ref(this->mem->psi_ref[e_ref], this->ijk_ref); // just to make sure that with cyclic bconds, corresponding refined values are the same; TODO: not needed? } // TODO: now that we have xchng_ref, use halo (other than distmem halos) in reconstruction? @@ -240,6 +277,7 @@ namespace libmpdataxx formulae::fractal::rcnstrct<2, real_t>(this->mem->psi_ref[e_ref], this->rng_dbl_stride(mid_ijk_r2r_2) , ijk_r2r_0_h_with_halo, this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride, this->c_j, this->d_j, this->f_j); } this->mem->barrier(); +// avg_edge_sclr_ref(this->mem->psi_ref[e_ref], this->ijk_ref); // just to make sure that with cyclic bconds, corresponding refined values are the same; TODO: not needed? } void xchng_ref( diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp index c3e4f667..7652e700 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp @@ -11,6 +11,7 @@ #pragma once #include +#include //#include //#include #include From d35ae7121a58fd1bc7fe4352efaad456cc68b61c Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Fri, 7 Apr 2023 12:24:16 +0200 Subject: [PATCH 110/129] avg_edge_sclr_ref and spatial_average protected --- .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 72 +++++++++---------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index be6b36ab..6866a0a1 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -122,42 +122,6 @@ namespace libmpdataxx } } - void avg_edge_sclr_ref(typename parent_t::arr_t &arr, - const idx_t<3> &range_ijk - ) - { - this->mem->barrier(); - - for (auto &bc : this->bcs_ref[0]) bc->copy_edge_sclr_to_halo1_cyclic(arr, range_ijk[1], range_ijk[2]); -// barrier_if_single_threaded_bc0(); // not necessary? - for (auto &bc : this->bcs_ref[1]) bc->copy_edge_sclr_to_halo1_cyclic(arr, range_ijk[2], range_ijk[0]); - for (auto &bc : this->bcs_ref[2]) bc->copy_edge_sclr_to_halo1_cyclic(arr, range_ijk[0], range_ijk[1]); - this->mem->barrier(); // wait for all threads to copy edge to halo before modifying edge. - // NOTE: averaging this way gives different weights to different points in corners depending on the order (corners are averaged twice and each averaging is dividing by 2) - for (auto &bc : this->bcs_ref[0]) bc->avg_edge_and_halo1_sclr_cyclic(arr, range_ijk[1], range_ijk[2]); - for (auto &bc : this->bcs_ref[1]) bc->avg_edge_and_halo1_sclr_cyclic(arr, range_ijk[2], range_ijk[0]); - for (auto &bc : this->bcs_ref[2]) bc->avg_edge_and_halo1_sclr_cyclic(arr, range_ijk[0], range_ijk[1]); - - this->mem->barrier(); - } - - // compute arr_reg as average from cells of arr_ref that are overlapping with the arr_reg cell - // NOTE: modifies arr_ref too! - void spatial_average_ref2reg(typename parent_t::arr_t &arr_ref, typename parent_t::arr_t &arr_reg) - { - this->xchng_ref(arr_ref, this->ijk_ref); // fill distmem halos of arr_ref - // std::cerr << "rx_ref(ijk_ref) after xchng_ref: " << rx_ref(this->ijk_ref) << std::endl; - // std::cerr << "rx_ref after xchng_ref: " << rx_ref << std::endl; - libmpdataxx::formulae::refined::spatial_average_ref2reg(arr_ref, this->ijk_r2r, this->mem->n_ref/2, this->mem->distmem.grid_size_ref, true); // calculate the average and store in arr_ref - // std::cerr << "rx_ref(ijk_ref) after spatial average: " << rx_ref(this->ijk_ref) << std::endl; - arr_reg(this->ijk) = arr_ref(this->ijk_r2r); // copy to arr_reg - // std::cerr << "rx(ijk) after copy from refined: " << rx(this->ijk) << std::endl; - nancheck(arr_reg(this->ijk), "arr_reg after copying from arr_ref in spatial_average_ref2reg"); -// rx(this->ijk) *= 4./3. * 1000. * 3.14159; // get mixing ratio [kg/kg] - this->mem->barrier(); - this->avg_edge_sclr(arr_reg, this->ijk); // in case of cyclic bcond, arr_reg on edges needs to be the same - } - void refinement_ranges(const int iter, const int halo_size) { // messy, because in domain decomposition (sharedmem and distmem) some refined scalars are on the edge of the subdomain... @@ -198,6 +162,42 @@ namespace libmpdataxx protected: + void avg_edge_sclr_ref(typename parent_t::arr_t &arr, + const idx_t<3> &range_ijk + ) + { + this->mem->barrier(); + + for (auto &bc : this->bcs_ref[0]) bc->copy_edge_sclr_to_halo1_cyclic(arr, range_ijk[1], range_ijk[2]); +// barrier_if_single_threaded_bc0(); // not necessary? + for (auto &bc : this->bcs_ref[1]) bc->copy_edge_sclr_to_halo1_cyclic(arr, range_ijk[2], range_ijk[0]); + for (auto &bc : this->bcs_ref[2]) bc->copy_edge_sclr_to_halo1_cyclic(arr, range_ijk[0], range_ijk[1]); + this->mem->barrier(); // wait for all threads to copy edge to halo before modifying edge. + // NOTE: averaging this way gives different weights to different points in corners depending on the order (corners are averaged twice and each averaging is dividing by 2) + for (auto &bc : this->bcs_ref[0]) bc->avg_edge_and_halo1_sclr_cyclic(arr, range_ijk[1], range_ijk[2]); + for (auto &bc : this->bcs_ref[1]) bc->avg_edge_and_halo1_sclr_cyclic(arr, range_ijk[2], range_ijk[0]); + for (auto &bc : this->bcs_ref[2]) bc->avg_edge_and_halo1_sclr_cyclic(arr, range_ijk[0], range_ijk[1]); + + this->mem->barrier(); + } + + // compute arr_reg as average from cells of arr_ref that are overlapping with the arr_reg cell + // NOTE: modifies arr_ref too! + void spatial_average_ref2reg(typename parent_t::arr_t &arr_ref, typename parent_t::arr_t &arr_reg) + { + this->xchng_ref(arr_ref, this->ijk_ref); // fill distmem halos of arr_ref + // std::cerr << "rx_ref(ijk_ref) after xchng_ref: " << rx_ref(this->ijk_ref) << std::endl; + // std::cerr << "rx_ref after xchng_ref: " << rx_ref << std::endl; + libmpdataxx::formulae::refined::spatial_average_ref2reg(arr_ref, this->ijk_r2r, this->mem->n_ref/2, this->mem->distmem.grid_size_ref, true); // calculate the average and store in arr_ref + // std::cerr << "rx_ref(ijk_ref) after spatial average: " << rx_ref(this->ijk_ref) << std::endl; + arr_reg(this->ijk) = arr_ref(this->ijk_r2r); // copy to arr_reg + // std::cerr << "rx(ijk) after copy from refined: " << rx(this->ijk) << std::endl; + nancheck(arr_reg(this->ijk), "arr_reg after copying from arr_ref in spatial_average_ref2reg"); +// rx(this->ijk) *= 4./3. * 1000. * 3.14159; // get mixing ratio [kg/kg] + this->mem->barrier(); + this->avg_edge_sclr(arr_reg, this->ijk); // in case of cyclic bcond, arr_reg on edges needs to be the same + } + // TODO: stretching parameters at the overlaps of the reconstructed and resolved arrays (ijk_r2r) are not used, do not generate them // also not all parameters in the halo are needed (but some are!) void generate_stretching_parameters(const typename gen_t::result_type rng_seed) From dd892893db311a88b173966e9856052623bfc84b Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Fri, 7 Apr 2023 12:30:07 +0200 Subject: [PATCH 111/129] remove nancheck --- libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 6866a0a1..7384ece2 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -192,7 +192,6 @@ namespace libmpdataxx // std::cerr << "rx_ref(ijk_ref) after spatial average: " << rx_ref(this->ijk_ref) << std::endl; arr_reg(this->ijk) = arr_ref(this->ijk_r2r); // copy to arr_reg // std::cerr << "rx(ijk) after copy from refined: " << rx(this->ijk) << std::endl; - nancheck(arr_reg(this->ijk), "arr_reg after copying from arr_ref in spatial_average_ref2reg"); // rx(this->ijk) *= 4./3. * 1000. * 3.14159; // get mixing ratio [kg/kg] this->mem->barrier(); this->avg_edge_sclr(arr_reg, this->ijk); // in case of cyclic bcond, arr_reg on edges needs to be the same From 72f8051753051abe4efbb30ec729c9866fb1ab8e Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Tue, 11 Apr 2023 12:54:19 +0200 Subject: [PATCH 112/129] DBG: dont do spatial averaging ref_2_reg, just copy corresponding cells --- .../solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 7384ece2..cfa3ebd4 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -185,10 +185,10 @@ namespace libmpdataxx // NOTE: modifies arr_ref too! void spatial_average_ref2reg(typename parent_t::arr_t &arr_ref, typename parent_t::arr_t &arr_reg) { - this->xchng_ref(arr_ref, this->ijk_ref); // fill distmem halos of arr_ref - // std::cerr << "rx_ref(ijk_ref) after xchng_ref: " << rx_ref(this->ijk_ref) << std::endl; - // std::cerr << "rx_ref after xchng_ref: " << rx_ref << std::endl; - libmpdataxx::formulae::refined::spatial_average_ref2reg(arr_ref, this->ijk_r2r, this->mem->n_ref/2, this->mem->distmem.grid_size_ref, true); // calculate the average and store in arr_ref +// this->xchng_ref(arr_ref, this->ijk_ref); // fill distmem halos of arr_ref +// // std::cerr << "rx_ref(ijk_ref) after xchng_ref: " << rx_ref(this->ijk_ref) << std::endl; +// // std::cerr << "rx_ref after xchng_ref: " << rx_ref << std::endl; +// libmpdataxx::formulae::refined::spatial_average_ref2reg(arr_ref, this->ijk_r2r, this->mem->n_ref/2, this->mem->distmem.grid_size_ref, true); // calculate the average and store in arr_ref // std::cerr << "rx_ref(ijk_ref) after spatial average: " << rx_ref(this->ijk_ref) << std::endl; arr_reg(this->ijk) = arr_ref(this->ijk_r2r); // copy to arr_reg // std::cerr << "rx(ijk) after copy from refined: " << rx(this->ijk) << std::endl; From a11ee8e2fdbbcdd44ec8d821bd2ac26b13d9ca5c Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Tue, 11 Apr 2023 13:03:55 +0200 Subject: [PATCH 113/129] Revert "DBG: dont do spatial averaging ref_2_reg, just copy corresponding cells" This reverts commit 72f8051753051abe4efbb30ec729c9866fb1ab8e. --- .../solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index cfa3ebd4..7384ece2 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -185,10 +185,10 @@ namespace libmpdataxx // NOTE: modifies arr_ref too! void spatial_average_ref2reg(typename parent_t::arr_t &arr_ref, typename parent_t::arr_t &arr_reg) { -// this->xchng_ref(arr_ref, this->ijk_ref); // fill distmem halos of arr_ref -// // std::cerr << "rx_ref(ijk_ref) after xchng_ref: " << rx_ref(this->ijk_ref) << std::endl; -// // std::cerr << "rx_ref after xchng_ref: " << rx_ref << std::endl; -// libmpdataxx::formulae::refined::spatial_average_ref2reg(arr_ref, this->ijk_r2r, this->mem->n_ref/2, this->mem->distmem.grid_size_ref, true); // calculate the average and store in arr_ref + this->xchng_ref(arr_ref, this->ijk_ref); // fill distmem halos of arr_ref + // std::cerr << "rx_ref(ijk_ref) after xchng_ref: " << rx_ref(this->ijk_ref) << std::endl; + // std::cerr << "rx_ref after xchng_ref: " << rx_ref << std::endl; + libmpdataxx::formulae::refined::spatial_average_ref2reg(arr_ref, this->ijk_r2r, this->mem->n_ref/2, this->mem->distmem.grid_size_ref, true); // calculate the average and store in arr_ref // std::cerr << "rx_ref(ijk_ref) after spatial average: " << rx_ref(this->ijk_ref) << std::endl; arr_reg(this->ijk) = arr_ref(this->ijk_r2r); // copy to arr_reg // std::cerr << "rx(ijk) after copy from refined: " << rx(this->ijk) << std::endl; From d9d299c5e43beae928448365a9f45879c39c6f48 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Tue, 11 Apr 2023 17:09:33 +0200 Subject: [PATCH 114/129] Revert "Revert "DBG: dont do spatial averaging ref_2_reg, just copy corresponding cells"" This reverts commit a11ee8e2fdbbcdd44ec8d821bd2ac26b13d9ca5c. --- .../solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 7384ece2..cfa3ebd4 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -185,10 +185,10 @@ namespace libmpdataxx // NOTE: modifies arr_ref too! void spatial_average_ref2reg(typename parent_t::arr_t &arr_ref, typename parent_t::arr_t &arr_reg) { - this->xchng_ref(arr_ref, this->ijk_ref); // fill distmem halos of arr_ref - // std::cerr << "rx_ref(ijk_ref) after xchng_ref: " << rx_ref(this->ijk_ref) << std::endl; - // std::cerr << "rx_ref after xchng_ref: " << rx_ref << std::endl; - libmpdataxx::formulae::refined::spatial_average_ref2reg(arr_ref, this->ijk_r2r, this->mem->n_ref/2, this->mem->distmem.grid_size_ref, true); // calculate the average and store in arr_ref +// this->xchng_ref(arr_ref, this->ijk_ref); // fill distmem halos of arr_ref +// // std::cerr << "rx_ref(ijk_ref) after xchng_ref: " << rx_ref(this->ijk_ref) << std::endl; +// // std::cerr << "rx_ref after xchng_ref: " << rx_ref << std::endl; +// libmpdataxx::formulae::refined::spatial_average_ref2reg(arr_ref, this->ijk_r2r, this->mem->n_ref/2, this->mem->distmem.grid_size_ref, true); // calculate the average and store in arr_ref // std::cerr << "rx_ref(ijk_ref) after spatial average: " << rx_ref(this->ijk_ref) << std::endl; arr_reg(this->ijk) = arr_ref(this->ijk_r2r); // copy to arr_reg // std::cerr << "rx(ijk) after copy from refined: " << rx(this->ijk) << std::endl; From 8565eb0fcf8d67d9e60eee910341e7e4839bb532 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Wed, 12 Apr 2023 12:47:00 +0200 Subject: [PATCH 115/129] do avg_edge_sclr_ref after refinemenet/interpolation --- libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index cfa3ebd4..afb82e23 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -238,7 +238,7 @@ namespace libmpdataxx formulae::fractal::intrp<2, real_t>(this->mem->psi_ref[e_ref], mid_ijk_r2r_2, ijk_r2r_0_h_with_halo, this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); } this->mem->barrier(); -// avg_edge_sclr_ref(this->mem->psi_ref[e_ref], this->ijk_ref); // just to make sure that with cyclic bconds, corresponding refined values are the same; TODO: not needed? + avg_edge_sclr_ref(this->mem->psi_ref[e_ref], this->ijk_ref); // just to make sure that with cyclic bconds, corresponding refined values are the same; TODO: not needed? } // TODO: now that we have xchng_ref, use halo (other than distmem halos) in reconstruction? @@ -276,7 +276,7 @@ namespace libmpdataxx formulae::fractal::rcnstrct<2, real_t>(this->mem->psi_ref[e_ref], this->rng_dbl_stride(mid_ijk_r2r_2) , ijk_r2r_0_h_with_halo, this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride, this->c_j, this->d_j, this->f_j); } this->mem->barrier(); -// avg_edge_sclr_ref(this->mem->psi_ref[e_ref], this->ijk_ref); // just to make sure that with cyclic bconds, corresponding refined values are the same; TODO: not needed? + avg_edge_sclr_ref(this->mem->psi_ref[e_ref], this->ijk_ref); // just to make sure that with cyclic bconds, corresponding refined values are the same; TODO: not needed? } void xchng_ref( From f0e93f0612336c87baaae3cdc3fa98b576a5a9fb Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Thu, 13 Apr 2023 12:01:43 +0200 Subject: [PATCH 116/129] more barriers in refine/interpolate --- .../solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index afb82e23..4e49bdf6 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -232,10 +232,13 @@ namespace libmpdataxx { refinement_ranges(i, halo_size); + this->mem->barrier(); formulae::fractal::intrp<0, real_t>(this->mem->psi_ref[e_ref], mid_ijk_r2r_0, ijk_r2r_1_h, ijk_r2r_2_h, hstride); this->mem->barrier(); formulae::fractal::intrp<1, real_t>(this->mem->psi_ref[e_ref], mid_ijk_r2r_1, ijk_r2r_2_h, ijk_r2r_0_h_with_halo, hstride); + this->mem->barrier(); formulae::fractal::intrp<2, real_t>(this->mem->psi_ref[e_ref], mid_ijk_r2r_2, ijk_r2r_0_h_with_halo, this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride); + this->mem->barrier(); } this->mem->barrier(); avg_edge_sclr_ref(this->mem->psi_ref[e_ref], this->ijk_ref); // just to make sure that with cyclic bconds, corresponding refined values are the same; TODO: not needed? @@ -270,10 +273,13 @@ namespace libmpdataxx { refinement_ranges(i, halo_size); + this->mem->barrier(); formulae::fractal::rcnstrct<0, real_t>(this->mem->psi_ref[e_ref], this->rng_dbl_stride(mid_ijk_r2r_0), ijk_r2r_1_h, ijk_r2r_2_h, hstride, this->c_j, this->d_j, this->f_j); this->mem->barrier(); formulae::fractal::rcnstrct<1, real_t>(this->mem->psi_ref[e_ref], this->rng_dbl_stride(mid_ijk_r2r_1) , ijk_r2r_2_h, ijk_r2r_0_h_with_halo, hstride, this->c_j, this->d_j, this->f_j); // NOTE: rng_dbl_stride(mid_ijk_r2r_1) gives overlapping ranges between thread subdomains... however it seems to work and naive fixes didnt work; UPDATE: are the ranges really overlapping? it looks good at the second look... + this->mem->barrier(); formulae::fractal::rcnstrct<2, real_t>(this->mem->psi_ref[e_ref], this->rng_dbl_stride(mid_ijk_r2r_2) , ijk_r2r_0_h_with_halo, this->rng_merge(ijk_r2r_1_h, mid_ijk_r2r_1), hstride, this->c_j, this->d_j, this->f_j); + this->mem->barrier(); } this->mem->barrier(); avg_edge_sclr_ref(this->mem->psi_ref[e_ref], this->ijk_ref); // just to make sure that with cyclic bconds, corresponding refined values are the same; TODO: not needed? From 08a30e60cd7460190645d5ba96f12c2d09b960cb Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Tue, 18 Apr 2023 15:24:00 +0200 Subject: [PATCH 117/129] full spatial_ref_2_reg --- libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 4e49bdf6..5edf62d1 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -185,10 +185,10 @@ namespace libmpdataxx // NOTE: modifies arr_ref too! void spatial_average_ref2reg(typename parent_t::arr_t &arr_ref, typename parent_t::arr_t &arr_reg) { -// this->xchng_ref(arr_ref, this->ijk_ref); // fill distmem halos of arr_ref + this->xchng_ref(arr_ref, this->ijk_ref); // fill distmem halos of arr_ref // // std::cerr << "rx_ref(ijk_ref) after xchng_ref: " << rx_ref(this->ijk_ref) << std::endl; // // std::cerr << "rx_ref after xchng_ref: " << rx_ref << std::endl; -// libmpdataxx::formulae::refined::spatial_average_ref2reg(arr_ref, this->ijk_r2r, this->mem->n_ref/2, this->mem->distmem.grid_size_ref, true); // calculate the average and store in arr_ref + libmpdataxx::formulae::refined::spatial_average_ref2reg(arr_ref, this->ijk_r2r, this->mem->n_ref/2, this->mem->distmem.grid_size_ref, true); // calculate the average and store in arr_ref // std::cerr << "rx_ref(ijk_ref) after spatial average: " << rx_ref(this->ijk_ref) << std::endl; arr_reg(this->ijk) = arr_ref(this->ijk_r2r); // copy to arr_reg // std::cerr << "rx(ijk) after copy from refined: " << rx(this->ijk) << std::endl; From 312af4f90ec169efe34646c55837635edb579b49 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Wed, 19 Apr 2023 16:41:25 +0200 Subject: [PATCH 118/129] Revert "full spatial_ref_2_reg" This reverts commit 08a30e60cd7460190645d5ba96f12c2d09b960cb. --- libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 5edf62d1..4e49bdf6 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -185,10 +185,10 @@ namespace libmpdataxx // NOTE: modifies arr_ref too! void spatial_average_ref2reg(typename parent_t::arr_t &arr_ref, typename parent_t::arr_t &arr_reg) { - this->xchng_ref(arr_ref, this->ijk_ref); // fill distmem halos of arr_ref +// this->xchng_ref(arr_ref, this->ijk_ref); // fill distmem halos of arr_ref // // std::cerr << "rx_ref(ijk_ref) after xchng_ref: " << rx_ref(this->ijk_ref) << std::endl; // // std::cerr << "rx_ref after xchng_ref: " << rx_ref << std::endl; - libmpdataxx::formulae::refined::spatial_average_ref2reg(arr_ref, this->ijk_r2r, this->mem->n_ref/2, this->mem->distmem.grid_size_ref, true); // calculate the average and store in arr_ref +// libmpdataxx::formulae::refined::spatial_average_ref2reg(arr_ref, this->ijk_r2r, this->mem->n_ref/2, this->mem->distmem.grid_size_ref, true); // calculate the average and store in arr_ref // std::cerr << "rx_ref(ijk_ref) after spatial average: " << rx_ref(this->ijk_ref) << std::endl; arr_reg(this->ijk) = arr_ref(this->ijk_r2r); // copy to arr_reg // std::cerr << "rx(ijk) after copy from refined: " << rx(this->ijk) << std::endl; From 2d2958341964e7d6882c4c74431826382e41aa94 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Fri, 21 Apr 2023 12:19:02 +0200 Subject: [PATCH 119/129] Revert "Revert "full spatial_ref_2_reg"" This reverts commit 312af4f90ec169efe34646c55837635edb579b49. --- libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 4e49bdf6..5edf62d1 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -185,10 +185,10 @@ namespace libmpdataxx // NOTE: modifies arr_ref too! void spatial_average_ref2reg(typename parent_t::arr_t &arr_ref, typename parent_t::arr_t &arr_reg) { -// this->xchng_ref(arr_ref, this->ijk_ref); // fill distmem halos of arr_ref + this->xchng_ref(arr_ref, this->ijk_ref); // fill distmem halos of arr_ref // // std::cerr << "rx_ref(ijk_ref) after xchng_ref: " << rx_ref(this->ijk_ref) << std::endl; // // std::cerr << "rx_ref after xchng_ref: " << rx_ref << std::endl; -// libmpdataxx::formulae::refined::spatial_average_ref2reg(arr_ref, this->ijk_r2r, this->mem->n_ref/2, this->mem->distmem.grid_size_ref, true); // calculate the average and store in arr_ref + libmpdataxx::formulae::refined::spatial_average_ref2reg(arr_ref, this->ijk_r2r, this->mem->n_ref/2, this->mem->distmem.grid_size_ref, true); // calculate the average and store in arr_ref // std::cerr << "rx_ref(ijk_ref) after spatial average: " << rx_ref(this->ijk_ref) << std::endl; arr_reg(this->ijk) = arr_ref(this->ijk_r2r); // copy to arr_reg // std::cerr << "rx(ijk) after copy from refined: " << rx(this->ijk) << std::endl; From 8a27891a753e9ede6bd03f5e4a241262593a2994 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Tue, 25 Apr 2023 12:32:09 +0200 Subject: [PATCH 120/129] stretching parameters from E. Akinlabi: move the to separate namespace --- .../formulae/fractal_reconstruction.hpp | 47 ++++++++++--------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/libmpdata++/formulae/fractal_reconstruction.hpp b/libmpdata++/formulae/fractal_reconstruction.hpp index 6cfba5cf..5532981c 100644 --- a/libmpdata++/formulae/fractal_reconstruction.hpp +++ b/libmpdata++/formulae/fractal_reconstruction.hpp @@ -14,29 +14,34 @@ namespace libmpdataxx { namespace fractal { - // CDF of stretching parameter d; assuming CDF = A d^B + C under condition that CDF(0.5)=0 and CDF(1)=1 - template - static real_t CDF_of_d(const real_t &d) + namespace stretching_parameters { - const real_t B = -0.39091161; // comes from a fit to the E. Akinlabi data - return (pow(d,B) - pow(0.5,B)) / (1. - pow(0.5,B)); - } - // inverse function: d of a CDF - template - static real_t d_of_CDF(const real_t &CDF) - { - const real_t B = -0.39091161; // comes from a fit to the E. Akinlabi data - return pow((1.-pow(0.5,B))*CDF + pow(0.5,B), 1./B); - } - - template - struct d_of_CDF_fctr - { - real_t operator()(const real_t &CDF) const + namespace Akinlabi // stretching parameters from a fit to the E. Akinlabi data (these are from DNS?) { - return CDF < 0 ? -d_of_CDF(-CDF) : d_of_CDF(CDF); - } - BZ_DECLARE_FUNCTOR(d_of_CDF_fctr); + // CDF of stretching parameter d; assuming CDF = A d^B + C under condition that CDF(0.5)=0 and CDF(1)=1 + template + static real_t CDF_of_d(const real_t &d) + { + const real_t B = -0.39091161; + return (pow(d,B) - pow(0.5,B)) / (1. - pow(0.5,B)); + } + // inverse function: d of a CDF + template + static real_t d_of_CDF(const real_t &CDF) + { + const real_t B = -0.39091161; + return pow((1.-pow(0.5,B))*CDF + pow(0.5,B), 1./B); + } + template + struct d_of_CDF_fctr + { + real_t operator()(const real_t &CDF) const + { + return CDF < 0 ? -d_of_CDF(-CDF) : d_of_CDF(CDF); + } + BZ_DECLARE_FUNCTOR(d_of_CDF_fctr); + }; + }; }; // (tri-)linear interpolation From 753f46ea378895201e7d2d234f0df064bb4a02cb Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Tue, 25 Apr 2023 12:43:53 +0200 Subject: [PATCH 121/129] for reference, include the stretching parameter data from E. Akinlabi --- libmpdata++/formulae/fractal_reconstruction.hpp | 3 ++- .../formulae/stretching_parameter_data/Akinlabi_DNS.dat | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 libmpdata++/formulae/stretching_parameter_data/Akinlabi_DNS.dat diff --git a/libmpdata++/formulae/fractal_reconstruction.hpp b/libmpdata++/formulae/fractal_reconstruction.hpp index 5532981c..78f6b104 100644 --- a/libmpdata++/formulae/fractal_reconstruction.hpp +++ b/libmpdata++/formulae/fractal_reconstruction.hpp @@ -16,7 +16,8 @@ namespace libmpdataxx { namespace stretching_parameters { - namespace Akinlabi // stretching parameters from a fit to the E. Akinlabi data (these are from DNS?) + // stretching parameters from a fit to the E. Akinlabi data (the data is in the stretching_parameter_data subdirectory) + namespace Akinlabi { // CDF of stretching parameter d; assuming CDF = A d^B + C under condition that CDF(0.5)=0 and CDF(1)=1 template diff --git a/libmpdata++/formulae/stretching_parameter_data/Akinlabi_DNS.dat b/libmpdata++/formulae/stretching_parameter_data/Akinlabi_DNS.dat new file mode 100644 index 00000000..6a128356 --- /dev/null +++ b/libmpdata++/formulae/stretching_parameter_data/Akinlabi_DNS.dat @@ -0,0 +1,4 @@ +# (C) Emmanuel Akinlabi and Marta Waclawczyk +# Values of the stretching parameter for velocity obtained from DNS. DNS performed by Juan-Pedro Mellado. +# +0.928188,0.583540,0.948110,0.658857,0.833426,0.553489,0.587012,0.533077,0.528606,0.983013,0.543593,0.689402,0.761547,0.999972,0.645752,0.681821,0.589935,0.746706,0.610497,0.726029,0.680001,0.522556,0.502408,0.794410,0.782129,0.731786,0.679089,0.566662,0.579960,0.564932,0.680816,0.831542,0.976304,0.507234,0.699972,0.584431,0.686725,0.903092,0.639804,0.523677,0.967422,0.719968,0.826814,0.861091,0.534377,0.984072,0.798542,0.929810,0.663973,0.728403,0.570432,0.660802,0.527268,0.625718,0.988942,0.506717,0.869579,0.792927,0.956549,0.606465,0.615446,0.719184,0.516141,0.814920,0.703817,0.792067,0.579353,0.703599,0.505176,0.531601,0.733007,0.728759,0.635139,0.790075,0.915458,0.831659,0.540985,0.600800,0.632518,0.630528,0.769810,0.729514,0.678372,0.672672,0.538290,0.660828,0.595974,0.540796,0.743143,0.502059,0.837850,0.516149,0.531527,0.508546,0.605417,0.816713,0.937519,0.541440,0.715081,0.802366,0.798980,0.532520,0.666980,0.923555,0.882623,0.585536,0.640778,0.800545,0.638900,0.993778,0.653466,0.583052,0.552837,0.767368,0.982635,0.624085,0.583776,0.830837,0.622762,0.527004,0.801880,0.702646,0.521712,0.583722,0.878671,0.815654,0.939814,0.547953,0.548921,0.523109,0.762445,0.670564,0.638700,0.654343,0.505427,0.712077,0.612065,0.667715,0.997623,0.556282,0.558834,0.701715,0.915206,0.521808,0.899526,0.562892,0.568104,0.686890,0.637472,0.732995,0.719923,0.691018,0.683959,0.832811,0.647972,0.794778,0.751160,0.961217,0.543085,0.578248,0.570722,0.699265,0.838868,0.918103,0.750664,0.958559,0.701433,0.558768,0.976060,0.756556,0.921188,0.535581,0.566711,0.531120,0.900850,0.596717,0.687366,0.519502,0.936240,0.716113,0.683708,0.798779,0.911025,0.660259,0.657628,0.621469,0.765009,0.609787,0.607912,0.622329,0.702246,0.615946,0.611661,0.545893,0.563874,0.958919,0.670193,0.965381,0.866727,0.532914,0.548599,0.578303,0.658374,0.766836,0.882454,0.765757,0.786982,0.780744,0.662478,0.557110,0.732416,0.539099,0.978442,0.717231,0.949068,0.722187,0.980237,0.503968,0.515282,0.673559,0.748143,0.585892,0.539034,0.862322,0.533356,0.532139,0.974545,0.546693,0.619908,0.544022,0.595218,0.718816,0.507508,0.688010,0.607981,0.946285,0.520250,0.720476,0.506939,0.772208,0.550529,0.583232,0.576427,0.625992,0.612868,0.607185,0.582017,0.801453,0.538176,0.593398,0.660350,0.922723,0.729415,0.687159,0.790384,0.992785,0.849348,0.718927,0.868585,0.674569,0.529971,0.586896,0.587175,0.712353,0.708296,0.838311,0.774908,0.844933,0.695100,0.827330,0.597876,0.700107,0.756918,0.730000,0.772246,0.794858,0.767508,0.715310,0.615449,0.503056,0.998817,0.502969,0.624396,0.565177,0.885881,0.786426,0.888256,0.518823,0.706926,0.947891,0.636329,0.967777,0.940591,0.997540,0.760969,0.908156,0.558747,0.577276,0.779363,0.562991,0.519122,0.514257,0.607777,0.544104,0.696100,0.586027,0.926320,0.647712,0.723841,0.632630,0.511955,0.650926,0.518949,0.853989,0.653037,0.523329,0.647604,0.525642,0.537893,0.732815,0.971892,0.564871,0.644118,0.687982,0.921082,0.922892,0.851250,0.503387,0.919854,0.987893,0.507305,0.965672,0.680800,0.717239,0.520399,0.856189,0.836498,0.844400,0.735221,0.927618,0.699267,0.594598,0.694775,0.961700,0.527527,0.594096,0.593230,0.834462,0.710435,0.739907,0.861947,0.500822,0.551738,0.834945,0.656470,0.756866,0.782595,0.546130,0.604777,0.726657,0.940359,0.631606,0.574592,0.986073,0.739502,0.530270,0.901750,0.955133,0.518719,0.657903,0.767784,0.638086,0.874120,0.670144,0.536129,0.892110,0.553681,0.578045,0.705009,0.768860,0.548494,0.530204,0.617246,0.971689,0.951160,0.838485,0.675292,0.547413,0.644558,0.549313,0.784990,0.606135,0.789028,0.669353,0.619064,0.577611,0.588602,0.911252,0.873516,0.693113,0.641413,0.652551,0.515486,0.839485,0.775869,0.632157,0.868428,0.871882,0.820807,0.980948,0.577146,0.758768,0.549291,0.626668,0.894834,0.872663,0.553898,0.775510,0.797701,0.589206,0.713672,0.951542,0.671541,0.534561,0.965806,0.562452,0.864114,0.501961,0.503758,0.570199,0.520611,0.779388,0.531945,0.607231,0.907509,0.736731,0.686557,0.909934,0.606678,0.659768,0.749942,0.668531,0.577880,0.562204,0.643321,0.971687,0.685725,0.977851,0.983837,0.672451,0.696440,0.647733,0.851303,0.685686,0.537645,0.558121,0.508027,0.555406,0.786735,0.984664,0.897789,0.758280,0.500082,0.989086,0.874562,0.634631,0.501143,0.831816,0.715762,0.746899,0.654864,0.533727,0.670223,0.550462,0.923873,0.907971,0.549556,0.743616,0.649501,0.504579,0.570118,0.838138,0.853352,0.678333,0.946071,0.529044,0.503831,0.664711,0.740646,0.904086,0.800980,0.757564,0.734745,0.957340,0.679774,0.871055,0.760400,0.516930,0.635656,0.533623,0.768882,0.519929,0.516674,0.556330,0.616309,0.947704,0.838735,0.816590,0.518450,0.962553,0.669221,0.560858,0.864644,0.500487,0.956091,0.978171,0.930104,0.505535,0.787237,0.740607,0.854267,0.591379,0.761240,0.926281,0.696899,0.995477,0.793036,0.701188,0.702350,0.702451,0.729325,0.910849,0.858262,0.770285,0.963165,0.631278,0.524033,0.922262,0.516305,0.887021,0.939349,0.779983,0.854984,0.638343,0.727711,0.516145,0.882660,0.731661,0.616553,0.536576,0.907622,0.616980,0.569614,0.770154,0.706314,0.612900,0.930049,0.503178,0.940580,0.811689,0.734624,0.577023,0.732267,0.683153,0.926507,0.756991,0.915699,0.652213,0.526750,0.751828,0.509700,0.550238,0.647622,0.628129,0.721245,0.666492,0.530836,0.802917,0.551976,0.742373,0.946802,0.897744,0.988343,0.624864,0.889650,0.728555,0.900389,0.580433,0.733072,0.994271,0.675932,0.680180,0.884088,0.513236,0.570215,0.747084,0.661880,0.675841,0.961880,0.614220,0.592704,0.622081,0.544844,0.812371,0.904942,0.529091,0.733609,0.805665,0.965633,0.753582,0.890947,0.504407,0.724673,0.779461,0.883889,0.602288,0.910531,0.883592,0.984481,0.739054,0.752000,0.507989,0.951486,0.503663,0.788067,0.813819,0.713040,0.627320,0.722474,0.949906,0.641327,0.535614,0.950493,0.545281,0.629039,0.730393,0.500929,0.564052,0.777866,0.634277,0.626865,0.666335,0.515070,0.759528,0.577695,0.632267,0.894789,0.992370,0.794144,0.520456,0.978274,0.532193,0.699150,0.794411,0.601853,0.781037,0.506155,0.760170,0.973428,0.563804,0.703980,0.604636,0.660288,0.517721,0.771834,0.538447,0.713924,0.902767,0.528627,0.581040,0.534580,0.758549,0.557946,0.536281,0.657922,0.695411,0.981967,0.682607,0.880248,0.643869,0.846507,0.924227,0.694626,0.925165,0.959020,0.886652,0.827727,0.615758,0.813228,0.906883,0.972288,0.807312,0.892188,0.968920,0.503408,0.710068,0.818160,0.674362,0.898101,0.577553,0.555172,0.544204,0.502656,0.585411,0.653096,0.820654,0.582728,0.805214,0.707752,0.562404,0.949856,0.714311,0.626750,0.534335,0.672242,0.717541,0.680352,0.754656,0.799331,0.573405,0.672132,0.958145,0.613803,0.856402,0.558951,0.628229,0.651509,0.667431,0.618581,0.609058,0.810879,0.600575,0.927949,0.782410,0.529541,0.967787,0.523150,0.538258,0.779733,0.679859,0.984800,0.664552,0.636896,0.743266,0.906798,0.559574,0.575589,0.980727,0.590427,0.612770,0.945777,0.634667,0.995334,0.788522,0.929092,0.839269,0.916904,0.722456,0.578265,0.502318,0.626688,0.904697,0.789482,0.884768,0.608574,0.982005,0.505544,0.685172,0.833768,0.609457,0.561120,0.620634,0.606898,0.904157,0.768534,0.505619,0.509490,0.562908,0.811013,0.526471,0.882042,0.504725,0.687393,0.552064,0.601212,0.684452,0.663647,0.804682,0.942139,0.578553,0.756559,0.656053,0.791384,0.602045,0.680025,0.737986,0.769291,0.630289,0.602536,0.977759,0.984134,0.857471,0.929167,0.929021,0.537184,0.848451,0.576291,0.525298,0.567455,0.771165,0.862576,0.975656,0.846442,0.947654,0.950701,0.554402,0.534369,0.934735,0.963066,0.698086,0.911773,0.880946,0.922473,0.734720,0.513433,0.592506,0.963051,0.647017,0.527094,0.580756,0.546838,0.541945,0.717337,0.694845,0.537714,0.934149,0.989119,0.510064,0.715691,0.866101,0.526372,0.624233,0.683600,0.502973,0.508436,0.510107,0.845663,0.624035,0.672671,0.539342,0.557893,0.835382,0.639684,0.701721,0.589790,0.975023,0.798179,0.546462,0.741915,0.580320,0.577838,0.763609,0.689767,0.953994,0.820315,0.880602,0.587173,0.588447,0.921081,0.737535,0.694377,0.766530,0.527514,0.643626,0.634669,0.665157,0.555037,0.576079,0.744376,0.897700,0.643335,0.889435,0.822922,0.534125,0.945544,0.546930,0.742081,0.805479,0.941489,0.530436,0.726816,0.970840,0.903910,0.850062,0.658026,0.657692,0.665722,0.590482,0.572728,0.612830,0.616901,0.542787,0.571107,0.621769,0.616805,0.604151,0.718938,0.987268,0.514598,0.812175,0.917548,0.644682,0.516647,0.780496,0.509688,0.509862,0.611877,0.804102,0.541415,0.666780,0.528851,0.501797,0.505558,0.854935,0.750334,0.722608,0.722503,0.732509,0.738455,0.897216,0.531204,0.750874,0.752175,0.719768,0.903924,0.867757,0.961049,0.547907,0.568740,0.604902,0.899392,0.632488,0.547650,0.662464,0.857542,0.703475,0.773521,0.706256,0.562722,0.562806,0.697124,0.532060,0.514217,0.613290,0.533865,0.669763,0.756182,0.540986,0.710988,0.928635,0.674976,0.504555,0.872098,0.710203,0.728316,0.708497,0.607065,0.674910,0.645743,0.547787,0.546732,0.636850,0.676124,0.720039,0.636670,0.574241,0.698715,0.548455,0.573445,0.972556,0.515903,0.988508,0.988091,0.630787,0.962722,0.919878,0.704702,0.570066,0.516056,0.512238,0.673336,0.523711,0.642584,0.879291,0.661785,0.715095,0.749864,0.675079,0.980596,0.961858,0.693245,0.825774,0.746055,0.815934,0.887501,0.944383,0.527834,0.542906,0.871172,0.787540,0.553904,0.886696,0.546721,0.531604,0.633693,0.654352,0.885733,0.673538,0.543469,0.631016,0.546816,0.539775,0.657590,0.648337,0.934234,0.870692,0.995976,0.705011,0.789938,0.827041,0.529209,0.603429,0.573229,0.603412,0.564200,0.745591,0.512016,0.728011,0.780338,0.859835,0.835662,0.551308,0.779862,0.809512,0.941287,0.511305,0.596361,0.854391,0.549820,0.531254,0.651758,0.588131,0.571058,0.697495,0.526543,0.523557,0.790585,0.620275,0.517250,0.795246,0.690504,0.770354,0.637940,0.977086,0.749085,0.888822,0.676350,0.557848,0.860558,0.709506,0.696614,0.542427,0.592401,0.510444,0.823108,0.600427,0.835102,0.581201,0.823807,0.999473,0.513198,0.567939,0.831077,0.516338,0.842055,0.603308,0.546323,0.578939,0.610086,0.968159,0.967805,0.925351,0.770547,0.512520,0.837472,0.545989,0.986042,0.859519,0.945983,0.545442,0.855633,0.778646,0.891541,0.553102,0.980902,0.989598,0.810673,0.554406,0.654398,0.568894,0.748270,0.539565,0.537622,0.975861,0.668411,0.793270,0.998570,0.886033,0.607269,0.730238,0.606335,0.584767,0.586749,0.759098,0.965617,0.676758,0.652377,0.906646,0.762261,0.609454,0.631381,0.742159,0.790471,0.802440,0.540251,0.582258,0.675883,0.560283,0.651704,0.564193,0.634000,0.519040,0.950591,0.585597,0.950559,0.545658,0.706142,0.703448,0.519744,0.696222,0.550396,0.919340,0.824431,0.507794,0.508892,0.982089,0.575623,0.524455,0.582652,0.997597,0.974950,0.581165,0.712696,0.693177,0.524697,0.568564,0.845766,0.564382,0.540494,0.913049,0.563625,0.746063,0.804135,0.552278,0.765111,0.535641,0.829125,0.606301,0.772889,0.756454,0.773182,0.772114,0.715897,0.617716,0.531255,0.524650,0.763198,0.615759,0.578727,0.837471,0.945598,0.888558,0.502167,0.840618,0.611173,0.509615,0.752660,0.911650,0.642728,0.652983,0.598348,0.657640,0.588903,0.733034,0.597756,0.565425,0.978447,0.892437,0.544584,0.638671,0.585269,0.501367,0.563020,0.558580,0.962317,0.744235,0.632106,0.658801,0.835555,0.901206,0.770111,0.608252,0.702784,0.778407,0.602358,0.537184,0.883202,0.530486,0.660979,0.992049,0.970779,0.838641,0.852422,0.616366,0.906474,0.640948,0.556517,0.784963,0.596903,0.537115,0.786243,0.787777,0.715732,0.768789,0.664671,0.556719,0.643556,0.593388,0.671305,0.771135,0.750043,0.726033,0.676687,0.789050,0.881890,0.714437,0.686531,0.553898,0.656284,0.503052,0.646596,0.784671,0.590282,0.745673,0.541053,0.655652,0.548088,0.615151,0.506856,0.573427,0.509998,0.806376,0.750175,0.580470,0.657547,0.976161,0.669510,0.735147,0.587691,0.657182,0.738603,0.651367,0.677506,0.634564,0.688350,0.508771,0.653353,0.532246,0.900842,0.609946,0.794471,0.829065,0.666829,0.850736,0.658477,0.803733,0.850219,0.743368,0.630195,0.789197,0.865385,0.794146,0.553253,0.560044,0.965708,0.819595,0.937584,0.961433,0.841795,0.571552,0.908824,0.686417,0.780878,0.557674,0.605546,0.636630,0.784562,0.558953,0.896062,0.710596,0.717437,0.720059,0.795971,0.932511,0.726023,0.749702,0.694716,0.600705,0.696629,0.503208,0.646540,0.542768,0.718054,0.501031,0.515085,0.671505,0.517852,0.568901,0.636961,0.599486,0.747295,0.956406,0.958947,0.569597,0.811381,0.515359,0.658221,0.724733,0.582533,0.938545,0.708190,0.816972,0.755125,0.635111,0.661189,0.735839,0.661238,0.769043,0.646755,0.713892,0.529229,0.953283,0.635797,0.977464,0.723985,0.745768,0.602518,0.777990,0.552703,0.849009,0.702064,0.780669,0.925177,0.740012,0.792665,0.959078,0.640756,0.988704,0.764430,0.881190,0.923104,0.784546,0.867840,0.658838,0.870414,0.626651,0.777713,0.543852,0.567751,0.853081,0.728654,0.680771,0.614564,0.669792,0.585450,0.908155,0.861462,0.709705,0.897387,0.912583,0.521607,0.788885,0.574131,0.560885,0.549405,0.782330,0.509905,0.692571,0.811036,0.616726,0.832734,0.905029,0.582505,0.669051,0.721511,0.841660,0.620405,0.815945,0.756493,0.671365,0.711264,0.758746,0.621110,0.608757,0.978991,0.577331,0.692996,0.636665,0.909927,0.516488,0.776921,0.530202,0.603350,0.578853,0.663832,0.615662,0.665951,0.764975,0.938547,0.762186,0.763162,0.700439,0.863070,0.641379,0.778106,0.850111,0.569242,0.603260,0.914635,0.663766,0.625344,0.833115,0.710607,0.671436,0.616805,0.619027,0.676808,0.889479,0.767420,0.677749,0.628484,0.808778,0.669626,0.824734,0.909437,0.600274,0.515944,0.577566,0.557090,0.538550,0.862820,0.998235,0.891663,0.629223,0.916144,0.829969,0.970272,0.829981,0.713170,0.822874,0.788627,0.960342,0.619216,0.851703,0.705226,0.732219,0.638527,0.753755,0.526846,0.829079,0.535644,0.529300,0.527676,0.635320,0.925285,0.585920,0.828979,0.986995,0.796984,0.740338,0.515297,0.729711,0.504391,0.562037,0.749890,0.728650,0.898758,0.925305,0.733378,0.531565,0.613702,0.777106,0.792651,0.629937,0.572777,0.517354,0.770669,0.928131,0.581899,0.776025,0.952651,0.612863,0.567478,0.625280,0.908288,0.592045,0.938520,0.565014,0.991462,0.603571,0.761990,0.999231,0.738066,0.861880,0.713882,0.634618,0.994141,0.742346,0.668481,0.783336,0.783182,0.770805,0.763614,0.907394,0.975978,0.695032,0.869334,0.694037,0.642480,0.920570,0.968619,0.773036,0.864195,0.571072,0.849177,0.545113,0.979494,0.626816,0.577992,0.742616,0.975630,0.795427,0.596091,0.715563,0.610981,0.630928,0.645581,0.718442,0.606874,0.844893,0.561767,0.861362,0.738925,0.594492,0.715225,0.508592,0.605809,0.562830,0.561423,0.584801,0.517436,0.706947,0.847551,0.649295,0.532087,0.763136,0.839221,0.543266,0.719910,0.572359,0.659739,0.681790,0.735606,0.880807,0.664982,0.547151,0.553071,0.630078,0.576131,0.653297,0.504617,0.747664,0.710729,0.824573,0.660530,0.879975,0.990241,0.676202,0.694125,0.722181,0.959016,0.861737,0.805577,0.631851,0.822542,0.557936,0.926947,0.510470,0.629041,0.536445,0.850405,0.609232,0.558719,0.904782,0.756776,0.755721,0.692137,0.504154,0.960560,0.895549,0.701156,0.981685,0.768634,0.883968,0.960403,0.738585,0.562109,0.543898,0.560634,0.677686,0.599472,0.647294,0.620831,0.544970,0.824566,0.541496,0.763596,0.768443,0.737768,0.795299,0.698629,0.615867,0.950754,0.907574,0.758663,0.874996,0.598082,0.789051,0.953269,0.529012,0.612476,0.714452,0.571793,0.607520,0.665793,0.620388,0.753582,0.836803,0.513618,0.754874,0.974478,0.882250,0.878641,0.643700,0.569081,0.630460,0.714197,0.761952,0.605564,0.816883,0.724422,0.761278,0.573907,0.867466,0.542927,0.614533,0.635149,0.652516,0.648063,0.791254,0.690586,0.947244,0.673178,0.971278,0.730564,0.824221,0.684386,0.978145,0.736326,0.575236,0.529382,0.563922,0.510148,0.861408,0.638264,0.519400,0.682126,0.542032,0.641871,0.881417,0.897835,0.550140,0.789331,0.640467,0.820884,0.780879,0.973352,0.795772,0.511009,0.800712,0.684037,0.870859,0.564602,0.693452,0.942184,0.828786,0.579544,0.898087,0.503672,0.719475,0.527170,0.576589,0.519127,0.823610,0.593377,0.918230,0.698986,0.593257,0.602482,0.535693,0.686589,0.617678,0.796488,0.510719,0.567741,0.889778,0.697624,0.593570,0.805353,0.575345,0.640800,0.812863,0.762597,0.655006,0.609294,0.661135,0.881953,0.696703,0.599785,0.880854,0.723921,0.555854,0.648728,0.990534,0.508663,0.578480,0.673667,0.686053,0.546128,0.698319,0.724521,0.598907,0.762537,0.942804,0.525069,0.822627,0.540539,0.816143,0.766265,0.505171,0.905313,0.781264,0.782085,0.859082,0.639930,0.597976,0.530489,0.833801,0.642786,0.791161,0.987883,0.525335,0.532479,0.617355,0.643456,0.753097,0.743028,0.907631,0.540250,0.655885,0.616560,0.742547,0.597631,0.643877,0.907085,0.509960,0.521895,0.657614,0.835355,0.517833,0.587212,0.512048,0.527962,0.570642,0.684637,0.873183,0.674696,0.818283,0.511965,0.932055,0.508167,0.603844,0.756021,0.565758,0.804006,0.707313,0.764453,0.765165,0.684917,0.839989,0.897108,0.555998,0.864521,0.793741,0.602175,0.529091,0.847956,0.835228,0.558133,0.822131,0.724488,0.531883,0.853021,0.681985,0.857935,0.792633,0.911617,0.565229,0.968289,0.938044,0.838222,0.693355,0.746537,0.977783,0.660983,0.637870,0.890951,0.667913,0.578607,0.560366,0.638903,0.641126,0.680351,0.592313,0.643797,0.780419,0.663828,0.654573,0.610679,0.668183,0.701280,0.542028,0.953947,0.522878,0.598540,0.691763,0.650320,0.894822,0.869081,0.574185,0.714866,0.768040,0.619736,0.912048,0.790534,0.707579,0.756177,0.777171,0.909792,0.762517,0.501244,0.904893,0.693435,0.598494,0.894027,0.646962,0.609360,0.757595,0.594083,0.556016,0.853086,0.959987,0.733822,0.729166,0.696741,0.883046,0.725899,0.686600,0.597707,0.610046,0.553410,0.635394,0.665861,0.621706,0.531065,0.566344,0.640805,0.759445,0.501591,0.681180,0.806974,0.947883,0.578341,0.510406,0.502046,0.515975,0.660658,0.631685,0.584158,0.992788,0.548304,0.708479,0.892746,0.716284,0.561103,0.708420,0.761035,0.537646,0.738375,0.517863,0.648907,0.535448,0.562426,0.950148,0.855853,0.728672,0.846352,0.694382,0.570374,0.704791,0.807043,0.826793,0.556769,0.899388,0.884616,0.618021,0.978696,0.720861,0.939338,0.676886,0.752874,0.648305,0.829512,0.819722,0.604258,0.843476,0.644275,0.558085,0.531449,0.635799,0.631354,0.688638,0.565111,0.931778,0.838741,0.519348,0.635394,0.653208,0.510617,0.518150,0.711304,0.567923,0.959286,0.847994,0.812316,0.862450,0.559722,0.715792,0.562197,0.711941,0.646154,0.587664,0.655908,0.594836,0.591836,0.933502,0.815335,0.999544,0.580727,0.807227,0.791540,0.534542,0.907110,0.514128,0.793477,0.656211,0.609939,0.761305,0.624112,0.561896,0.526462,0.781789,0.725663,0.972488,0.721700,0.908853,0.695997,0.945175,0.547043,0.631457,0.890713,0.504299,0.698140,0.914777,0.920277,0.829808,0.938134,0.871619,0.782519,0.617931,0.789865,0.662757,0.702327,0.825629,0.583772,0.600717,0.965047,0.946190,0.649922,0.767637,0.677248,0.607262,0.828377,0.615029,0.961885,0.979755,0.953101,0.697816,0.508611,0.547542,0.607491,0.656798,0.616356,0.708364,0.509800,0.837054,0.656741,0.552255,0.627873,0.677176,0.634317,0.533816,0.755110,0.876975,0.783894,0.785466,0.687447,0.632740,0.573820,0.655089,0.616252,0.584531,0.814354,0.627380,0.878907,0.628586,0.892792,0.863526,0.606428,0.562391,0.539054,0.656787,0.774719,0.782450,0.764421,0.629300,0.940367,0.960101,0.641562,0.606915,0.742370,0.666226,0.695823,0.960407,0.690453,0.848967,0.742261,0.709067,0.863650,0.782045,0.773513,0.733037,0.935119,0.548180,0.771943,0.653251,0.807825,0.743658,0.746716,0.912837,0.975718,0.950856,0.639265,0.847404,0.776222,0.578733,0.880719,0.544833,0.531288,0.614499,0.609294,0.863960,0.855610,0.531720,0.979629,0.663906,0.673915,0.585485,0.731686,0.750195,0.548254,0.552096,0.618237,0.704425,0.818449,0.543875,0.890489,0.973936,0.590885,0.560696,0.811088,0.971805,0.607558,0.830644,0.761721,0.606764,0.611632,0.754633,0.745837,0.700377,0.557990,0.607314,0.880917,0.505011,0.763344,0.709548,0.553290,0.963127,0.922868,0.660156,0.729101,0.539132,0.803761,0.780639,0.842490,0.805555,0.891907,0.900447,0.821960,0.513090,0.603265,0.639667,0.526563,0.978095,0.710204,0.751507,0.641922,0.534686,0.974123,0.513797,0.565118,0.746750,0.768530,0.951973,0.844764,0.561436,0.874001,0.595323,0.955345,0.884829,0.853603,0.635301,0.687766,0.780608,0.565965,0.963595,0.567219,0.681274,0.800018,0.773108,0.653101,0.681811,0.938686,0.688011,0.538951,0.774141,0.668027,0.512102,0.606396,0.979472,0.827480,0.613170,0.874021,0.726340,0.777840,0.557455,0.635589,0.999501,0.500891,0.507437,0.680781,0.634921,0.867877,0.992029,0.969110,0.698228,0.853823,0.625607,0.583995,0.856465,0.820547,0.795924,0.993534,0.615718,0.891191,0.530268,0.638345,0.713829,0.593671,0.931141,0.532470,0.750206,0.604292,0.734513,0.745585,0.757750,0.526765,0.850982,0.924612,0.522083,0.816320,0.878753,0.760186,0.631695,0.547345,0.781984,0.568914,0.561627,0.765174,0.606648,0.766793,0.657845,0.517812,0.694047,0.927188,0.662792,0.650151,0.571336,0.700111,0.871202,0.586757,0.592578,0.744734,0.792214,0.546842,0.708244,0.935581,0.542741,0.513189,0.828113,0.837604,0.533472,0.589082,0.594612,0.868838,0.528512,0.601520,0.725755,0.924759,0.579673,0.937955,0.576260,0.826000,0.821028,0.688932,0.665048,0.559738,0.593078,0.813787,0.659141,0.580502,0.540684,0.558342,0.624874,0.621606,0.558398,0.982711,0.696743,0.857141,0.764547,0.739665,0.848779,0.703453,0.520665,0.894715,0.694764,0.624376,0.781002,0.583301,0.541399,0.769026,0.717518,0.941633,0.644063,0.689410,0.599699,0.666345,0.766581,0.739963,0.823858,0.569719,0.711446,0.797499,0.889195,0.501985,0.507477,0.577465,0.789017,0.670993,0.874241,0.987639,0.649345,0.585005,0.633797,0.948832,0.532396,0.534452,0.514853,0.687094,0.903812,0.506076,0.820709,0.503740,0.744675,0.887762,0.804068,0.690056,0.744402,0.522927,0.736943,0.781757,0.595055,0.530749,0.669456,0.569818,0.748179,0.729961,0.672250,0.867633,0.944573,0.602389,0.749168,0.821532,0.786779,0.789528,0.895016,0.792138,0.715956,0.643266,0.568034,0.958545,0.549210,0.677855,0.934435,0.667448,0.610734,0.778122,0.602467,0.931494,0.522184,0.703404,0.823645,0.990283,0.824933,0.569239,0.634704,0.547768,0.723603,0.554407,0.562192,0.746688,0.746765,0.539413,0.533827,0.556527,0.891679,0.913015,0.781070,0.830360,0.647513,0.552893,0.753382,0.576557,0.752373,0.624881,0.877274,0.901574,0.739765,0.530374,0.789952,0.543580,0.694663,0.811103,0.825727,0.521989,0.523747,0.937671,0.591607,0.740035,0.638943,0.612081,0.879336,0.518553,0.842439,0.819175,0.689149,0.592671,0.669114,0.601961,0.735906,0.672085,0.589134,0.773928,0.619723,0.786834,0.506664,0.571382,0.916844,0.854844,0.532124,0.908809,0.738943,0.573907,0.548354,0.815267,0.714159,0.872471,0.784513,0.516088,0.727280,0.542830,0.616424,0.669733,0.880105,0.941291,0.795437,0.843306,0.911737,0.739804,0.844528,0.691923,0.748648,0.791877,0.577987,0.627019,0.546821,0.697912,0.700559,0.941749,0.560266,0.511887,0.652654,0.867219,0.964486,0.868032,0.809649,0.965908,0.538175,0.729911,0.838043,0.661564,0.624302,0.588635,0.759432,0.673446,0.799963,0.694836,0.558706,0.769275,0.548657,0.828661,0.811948,0.628381,0.618864,0.991796,0.780020,0.526902,0.734473,0.917088,0.761408,0.574937,0.537843,0.518061,0.998490,0.813925,0.657841,0.622080,0.527658,0.737647,0.988876,0.857821,0.788403,0.621274,0.598246,0.650240,0.769957,0.566584,0.727212,0.809099,0.577927,0.767088,0.804055,0.594580,0.928182,0.674128,0.648224,0.645784,0.621240,0.882313,0.711124,0.788495,0.520684,0.532037,0.501963,0.542194,0.596804,0.840173,0.634863,0.755537,0.900765,0.643421,0.757631,0.725135,0.519398,0.976508,0.765094,0.602774,0.632133,0.551794,0.623232,0.836215,0.899493,0.633598,0.580606,0.543721,0.797780,0.810380,0.980964,0.595836,0.595789,0.635020,0.687879,0.798803,0.506982,0.565449,0.532690,0.770651,0.921431,0.860108,0.912713,0.950582,0.526367,0.524598,0.989446,0.644640,0.738990,0.790800,0.801008,0.954955,0.540684,0.726009,0.733736,0.503836,0.773013,0.686552,0.690523,0.794466,0.597002,0.779635,0.509672,0.648739,0.653735,0.517116,0.748908,0.675414,0.544862,0.775295,0.929312,0.599451,0.863402,0.783806,0.620476,0.860768,0.596778,0.990209,0.546856,0.607804,0.956267,0.548186,0.518059,0.817627,0.609937,0.832955,0.868467,0.847307,0.953227,0.565917,0.758396,0.821014,0.566523,0.761629,0.614087,0.601996,0.601311,0.643934,0.576138,0.687318,0.550343,0.913167,0.869971,0.904230,0.565051,0.524281,0.553170,0.534954,0.788830,0.511468,0.596412,0.850093,0.936143,0.954478,0.726941,0.681867,0.705363,0.741473,0.828501,0.625915,0.770171,0.720949,0.500928,0.965296,0.625618,0.755069,0.648025,0.865048,0.923701,0.634461,0.726817,0.657784,0.948322,0.586391,0.830217,0.705325,0.637326,0.554817,0.745945,0.746756,0.557574,0.626723,0.772166,0.586945,0.562504,0.934211,0.621799,0.777895,0.640420,0.793495,0.713517,0.716956,0.757123,0.579956,0.563758,0.686217,0.616899,0.847853,0.825539,0.650863,0.576836,0.797725,0.920386,0.960892,0.665201,0.540963,0.539791,0.545920,0.586675,0.517031,0.849537,0.766562,0.568400,0.598483,0.878358,0.715379,0.867329,0.708583,0.765870,0.710340,0.731705,0.913743,0.694337,0.546253,0.701292,0.736316,0.640695,0.669322,0.551320,0.863180,0.861977,0.581276,0.803958,0.619418,0.699200,0.585923,0.987801,0.839994,0.952154,0.833924,0.717350,0.719129,0.565481,0.616350,0.696564,0.831099,0.671391,0.934181,0.746966,0.514218,0.507572,0.530960,0.721377,0.658445,0.783403,0.609847,0.799343,0.560541,0.887187,0.724859,0.790426,0.796142,0.530922,0.804175,0.554595,0.527269,0.565028,0.690507,0.660449,0.641726,0.652119,0.544223,0.802531,0.784214,0.689923,0.719698,0.911845,0.727469,0.641230,0.783963,0.740846,0.692221,0.876317,0.585811,0.764701,0.531170,0.856801,0.978090,0.615488,0.784929,0.974480,0.876293,0.546031,0.746502,0.583441,0.977550,0.801347,0.644519,0.574941,0.761041,0.730361,0.763926,0.660637,0.815211,0.676535,0.544766,0.731984,0.539785,0.873394,0.685704,0.725017,0.662750,0.572566,0.931189,0.761115,0.695413,0.667631,0.514185,0.908476,0.732064,0.917460,0.745830,0.839662,0.630350,0.713378,0.936297,0.670819,0.680871,0.655211,0.621783,0.560828,0.855301,0.518117,0.591245,0.514587,0.784060,0.674451,0.919522,0.700814,0.902425,0.563029,0.989031,0.748106,0.951468,0.528042,0.842332,0.717948,0.562935,0.816557,0.588255,0.796695,0.897731,0.564630,0.742096,0.609995,0.929576,0.915147,0.960720,0.979900,0.592317,0.910006,0.833376,0.799916,0.511169,0.551286,0.578279,0.573938,0.761782,0.563224,0.912228,0.654349,0.523774,0.678749,0.976342,0.582944,0.605361,0.745855,0.614385,0.924249,0.734252,0.634064,0.686958,0.912691,0.690799,0.528415,0.770354,0.807496,0.724479,0.709447,0.851673,0.538257,0.883622,0.702839,0.830396,0.721289,0.619717,0.565501,0.699222,0.680967,0.628445,0.573475,0.530769,0.715437,0.555545,0.691253,0.513145,0.627646,0.579384,0.865664,0.643954,0.881583,0.559250,0.543639,0.775036,0.971304,0.657555,0.884318,0.586189,0.730165,0.908489,0.877558,0.679378,0.735464,0.814154,0.655558,0.535526,0.666641,0.750009,0.718029,0.806466,0.501297,0.791014,0.618554,0.607407,0.855409,0.907471,0.844094,0.558513,0.821691,0.539173,0.808849,0.963425,0.637117,0.512318,0.687417,0.654807,0.501041,0.537554,0.693330,0.753568,0.698852,0.738733,0.706611,0.989538,0.793546,0.919014,0.632581,0.788752,0.563560,0.631021,0.857753,0.742640,0.501018,0.933960,0.985400,0.562888,0.997879,0.683763,0.568492,0.558956,0.553022,0.517123,0.873400,0.593824,0.816444,0.896312,0.535728,0.540876,0.515654,0.684114,0.621204,0.870325,0.509257,0.587120,0.677728,0.529233,0.778821,0.956899,0.819174,0.904892,0.691493,0.797443,0.932159,0.822558,0.674797,0.729615,0.769472,0.698361,0.649076,0.903355,0.619660,0.561401,0.507566,0.531374,0.636382,0.720522,0.871171,0.628263,0.809968,0.772511,0.596380,0.536413,0.917560,0.535367,0.546147,0.924954,0.676985,0.533864,0.796937,0.769697,0.699200,0.711421,0.817599,0.844256,0.565255,0.843906,0.569141,0.619138,0.523133,0.913849,0.684350,0.911076,0.920733,0.746656,0.631007,0.594120,0.672583,0.575794,0.564863,0.934738,0.526787,0.774725,0.537625,0.639158,0.810917,0.629817,0.532580,0.507492,0.692121,0.763390,0.899951,0.904519,0.992745,0.921759,0.888366,0.828628,0.943621,0.970302,0.616009,0.646034,0.922148,0.945787,0.844201,0.611320,0.789916,0.579780,0.677308,0.654464,0.770439,0.635881,0.538781,0.624009,0.694511,0.913314,0.535229,0.529660,0.502937,0.887212,0.719489,0.735455,0.766724,0.542260,0.620614,0.664302,0.951258,0.871787,0.561915,0.726509,0.851760,0.552717,0.897867,0.735030,0.745155,0.900232,0.968740,0.763237,0.734872,0.787639,0.628759,0.653394,0.540955,0.676301,0.746386,0.540094,0.546813,0.914837,0.592918,0.719028,0.631627,0.604592,0.728448,0.865827,0.576126,0.550233,0.897981,0.802275,0.524458,0.755668,0.805128,0.772519,0.848506,0.550238,0.992840,0.560071,0.564977,0.570782,0.934486,0.620038,0.670436,0.641969,0.753539,0.568021,0.685610,0.734552,0.859957,0.813942,0.634595,0.791656,0.909858,0.539757,0.974693,0.827957,0.531404,0.968851,0.781194,0.565692,0.580127,0.597054,0.963576,0.864452,0.735357,0.824926,0.525493,0.805982,0.885425,0.535855,0.587956,0.705372,0.971510,0.661627,0.748531,0.620154,0.501166,0.650815,0.775197,0.590263,0.571401,0.518664,0.574981,0.644921,0.516024,0.599359,0.663319,0.987801,0.604616,0.584044,0.784061,0.587800,0.731859,0.677561,0.570458,0.788514,0.691803,0.680111,0.503250,0.778566,0.905595,0.507472,0.507739,0.677353,0.616244,0.714841,0.774324,0.721333,0.702919,0.558294,0.641163,0.838465,0.916651,0.549295,0.654615,0.574675,0.861285,0.637333,0.505617,0.508693,0.818644,0.902183,0.523785,0.623412,0.847006,0.810196,0.979953,0.565706,0.779192,0.730816,0.584523,0.647849,0.820897,0.649645,0.692717,0.575260,0.689374,0.782224,0.805383,0.716032,0.504323,0.916992,0.601992,0.685741,0.549516,0.516464,0.501752,0.757215,0.529470,0.513164,0.616127,0.598171,0.517539,0.600269,0.899305,0.583445,0.627333,0.680702,0.563512,0.963616,0.878914,0.585631,0.906290,0.574998,0.795875,0.848544,0.992387,0.761311,0.686295,0.704362,0.646207,0.860467,0.956738,0.563166,0.797669,0.980118,0.897051,0.611796,0.631555,0.773536,0.518296,0.958788,0.819500,0.707338,0.751469,0.691950,0.726020,0.645129,0.714194,0.550625,0.849116,0.513020,0.545766,0.647162,0.554635,0.725885,0.764813,0.558260,0.516648,0.869828,0.906753,0.689215,0.648040,0.848999,0.540975,0.947075,0.840353,0.613307,0.705985,0.525144,0.921432,0.671041,0.857659,0.831083,0.650149,0.581678,0.633040,0.617567,0.538811,0.592434,0.590442,0.911167,0.696681,0.634718,0.558405,0.533123,0.534539,0.878539,0.533611,0.573707,0.805299,0.765318,0.904834,0.621707,0.755023,0.635033,0.868332,0.586710,0.738242,0.772171,0.706887,0.939854,0.596299,0.608863,0.627196,0.637837,0.537068,0.671993,0.525177,0.934357,0.568933,0.610802,0.811974,0.726521,0.858441,0.505469,0.900295,0.763506,0.610165,0.754635,0.955949,0.846762,0.501394,0.976131,0.583381,0.599122,0.806218,0.587981,0.700249,0.786173,0.766172,0.896259,0.763035,0.638358,0.630131,0.665613,0.573924,0.552921,0.503465,0.600898,0.718336,0.639702,0.616744,0.791057,0.566314,0.743477,0.656289,0.947222,0.754516,0.506022,0.853161,0.687110,0.704349,0.609615,0.863631,0.625952,0.583638,0.628018,0.576453,0.814551,0.522468,0.814535,0.532120,0.720774,0.951435,0.534622,0.580003,0.551000,0.714105,0.652394,0.563731,0.618389,0.527610,0.691063,0.714033,0.807953,0.799789,0.585959,0.990550,0.926734,0.785873,0.625466,0.727710,0.801530,0.501515,0.669674,0.584900,0.732357,0.615466,0.648418,0.866522,0.633343,0.936758,0.584979,0.520406,0.648128,0.579901,0.888480,0.702988,0.807619,0.709114,0.663920,0.630545,0.897125,0.855184,0.928245,0.802636,0.987920,0.513606,0.907653,0.723192,0.904884,0.520073,0.601298,0.629585,0.600699,0.842269,0.693373,0.541137,0.541937,0.743327,0.985860,0.630815,0.702404,0.858550,0.893605,0.664499,0.922855,0.699078,0.574376,0.556792,0.864483,0.561096,0.737366,0.516365,0.698943,0.870877,0.580794,0.575622,0.869757,0.765497,0.909466,0.879962,0.603664,0.575031,0.702975,0.576123,0.979509,0.522324,0.772274,0.620574,0.682942,0.670665,0.735034,0.668151,0.959395,0.577149,0.617958,0.805412,0.718815,0.681533,0.545199,0.773777,0.889976,0.760177,0.575994,0.800569,0.515484,0.797144,0.811493,0.787628,0.983201,0.960590,0.669055,0.703527,0.906601,0.534203,0.633765,0.807278,0.814102,0.637388,0.529851,0.731381,0.768256,0.602944,0.987838,0.817431,0.885523,0.799711,0.970802,0.949626,0.633862,0.506597,0.652366,0.721010,0.656414,0.747711,0.746276,0.964385,0.732154,0.796709,0.836141,0.592732,0.579133,0.977482,0.824386,0.960995,0.717472,0.689316,0.561107,0.758716,0.736607,0.988942,0.834485,0.996300,0.526828,0.959901,0.909355,0.726951,0.829632,0.794011,0.587597,0.947590,0.969642,0.737268,0.868969,0.785004,0.946651,0.781378,0.674755,0.581882,0.896913,0.758556,0.559500,0.613898,0.683084,0.739118,0.666709,0.707687,0.546366,0.844138,0.694529,0.759735,0.534079,0.815331,0.607516,0.935789,0.957024,0.513081,0.570623,0.913196,0.733190,0.657290,0.678903,0.868214,0.824113,0.581411,0.810885,0.576717,0.583218,0.636769,0.914170,0.822585,0.822743,0.716328,0.890997,0.820153,0.807550,0.898041,0.898141,0.710832,0.973063,0.883919,0.588432,0.989688,0.922420,0.657651,0.834045,0.873304,0.565683,0.923990,0.527418,0.816247,0.720939,0.759608,0.957187,0.756302,0.796800,0.754076,0.518688,0.653012,0.786813,0.606785,0.685836,0.596868,0.724644,0.846550,0.738636,0.682452,0.843337,0.628878,0.783034,0.809868,0.801323,0.544536,0.665937,0.653150,0.567891,0.752780,0.872438,0.925562,0.739348,0.745190,0.517626,0.845435,0.556491,0.811511,0.568176,0.689671,0.695095,0.729599,0.617302,0.580365,0.548948,0.503301,0.705396,0.843487,0.821554,0.535730,0.752830,0.600191,0.595047,0.508180,0.912376,0.511918,0.717871,0.956860,0.908497,0.664923,0.967379,0.960821,0.615942,0.886267,0.921032,0.531175,0.650751,0.648983,0.550923,0.726516,0.869905,0.725002,0.549924,0.511835,0.796583,0.637459,0.558978,0.577017,0.538453,0.584530,0.631910,0.846009,0.861492,0.920115,0.554190,0.757677,0.549874,0.743929,0.587901,0.652381,0.649014,0.979689,0.517910,0.765974,0.549085,0.845770,0.812745,0.652454,0.945076,0.857057,0.704512,0.624376,0.633109,0.524883,0.598354,0.786481,0.682581,0.915881,0.639760,0.876014,0.801566,0.706789,0.626305,0.687549,0.975452,0.963608,0.626786,0.516977,0.719240,0.893107,0.649635,0.591977,0.569155,0.937819,0.619166,0.522842,0.532158,0.717875,0.511644,0.608418,0.810215,0.527658,0.701616,0.984773,0.579542,0.617346,0.526095,0.866133,0.523738,0.607291,0.599187,0.811998,0.761729,0.679344,0.669019,0.913994,0.662787,0.722868,0.567220,0.959818,0.879011,0.570604,0.724018,0.541978,0.780577,0.652870,0.689205,0.773709,0.785418,0.660290,0.700256,0.716929,0.911312,0.558059,0.883991,0.617640,0.941410,0.911969,0.707566,0.905517,0.921882,0.538921,0.551143,0.800336,0.541609,0.798739,0.690523,0.618467,0.955921,0.768915,0.526266,0.695853,0.940488,0.656990,0.502749,0.578283,0.884474,0.737258,0.611146,0.539536,0.691678,0.840554,0.848613,0.820029,0.630379,0.577733,0.719815,0.791284,0.783784,0.889998,0.563112,0.814038,0.880033,0.671839,0.661053,0.976043,0.603308,0.679555,0.507920,0.626989,0.909662,0.793802,0.624976,0.721661,0.899119,0.775286,0.741037,0.824018,0.548584,0.688211,0.852723,0.619819,0.867770,0.833645,0.945512,0.695160,0.901645,0.813393,0.885248,0.574713,0.993170,0.954603,0.581949,0.925629,0.714059,0.823600,0.866481,0.559324,0.825354,0.836439,0.992462,0.735671,0.619229,0.688595,0.605511,0.732764,0.569361,0.750579,0.520670,0.749907,0.677871,0.682511,0.833573,0.672906,0.531544,0.519510,0.530838,0.823825,0.574004,0.896347,0.568262,0.653377,0.846705,0.896162,0.874642,0.526376,0.553613,0.575912,0.561003,0.830599,0.736154,0.504603,0.836920,0.679348,0.978184,0.663562,0.555975,0.762379,0.966555,0.818513,0.723235,0.556646,0.852715,0.826031,0.631472,0.800399,0.780354,0.806562,0.684926,0.875235,0.748174,0.623419,0.501436,0.884290,0.692898,0.770705,0.784958,0.684620,0.603496,0.831277,0.680540,0.910229,0.514009,0.686433,0.717697,0.621436,0.880890,0.850200,0.709163,0.582151,0.719973,0.671871,0.534554,0.902099,0.586502,0.751592,0.672627,0.630667,0.528904,0.557460,0.844951,0.917605,0.974049,0.961142,0.852822,0.519102,0.619509,0.774367,0.700782,0.556258,0.592018,0.843182,0.913022,0.571736,0.813730,0.529513,0.861855,0.793211,0.781936,0.757474,0.646160,0.617849,0.516181,0.771855,0.650843,0.737366,0.748345,0.687090,0.679284,0.817031,0.712533,0.778822,0.911754,0.573496,0.733258,0.819076,0.587576,0.972865,0.796353,0.773382,0.689114,0.567986,0.597776,0.828931,0.545620,0.641087,0.737434,0.610996,0.734980,0.688513,0.897304,0.545758,0.810712,0.731496,0.538971,0.807172,0.928819,0.591798,0.683250,0.527008,0.931663,0.822638,0.836829,0.841882,0.843808,0.508763,0.750399,0.971181,0.771048,0.568349,0.708172,0.813642,0.623339,0.730285,0.672561,0.811854,0.806355,0.596661,0.657674,0.715929,0.923250,0.584588,0.623223,0.797957,0.793700,0.724734,0.958498,0.512110,0.693264,0.676728,0.531005,0.675961,0.734538,0.547514,0.581499,0.681215,0.721023,0.698680,0.615477,0.965789,0.860210,0.653603,0.942204,0.848041,0.879217,0.750309,0.734865,0.641847,0.563920,0.772484,0.988844,0.635294,0.591430,0.632008,0.742842,0.638663,0.673157,0.611251,0.788287,0.517876,0.912567,0.513694,0.694214,0.604285,0.728526,0.520669,0.761905,0.590028,0.677948,0.586221,0.560191,0.643665,0.621242,0.533688,0.852258,0.672630,0.812766,0.820206,0.788138,0.643317,0.602427,0.973348,0.922317,0.731681,0.600226,0.830744,0.821933,0.964333,0.888095,0.569637,0.886881,0.849357,0.572088,0.982741,0.560107,0.509052,0.805611,0.970161,0.941949,0.615475,0.708864,0.597394,0.612084,0.736335,0.773163,0.694760,0.594681,0.701788,0.589915,0.957270,0.681744,0.600219,0.806653,0.675480,0.910125,0.760698,0.820581,0.704728,0.817673,0.751279,0.558584,0.556948,0.914316,0.556598,0.662819,0.533269,0.777687,0.768954,0.642280,0.525662,0.531686,0.629297,0.928749,0.618444,0.570585,0.775031,0.646314,0.917890,0.786302,0.792846,0.534260,0.791408,0.549454,0.932787,0.947575,0.754772,0.603364,0.738512,0.574619,0.955548,0.821690,0.727157,0.629340,0.997537,0.811930,0.861481,0.786743,0.700262,0.928633,0.547048,0.933051,0.528694,0.761255,0.518599,0.956373,0.881884,0.797394,0.983389,0.550730,0.748233,0.702795,0.539625,0.748399,0.751125,0.725266,0.718775,0.562550,0.869913,0.602157,0.970556,0.550854,0.816255,0.631328,0.559780,0.909733,0.602344,0.635350,0.810071,0.959580,0.583818,0.755319,0.896365,0.534262,0.657499,0.635423,0.958383,0.827977,0.808857,0.528096,0.950581,0.530610,0.819978,0.543296,0.818013,0.808890,0.544116,0.886599,0.885350,0.582846,0.950598,0.678614,0.745845,0.804690,0.845577,0.673146,0.684381,0.525611,0.985422,0.680248,0.783097,0.765919,0.678143,0.848123,0.730389,0.785575,0.712067,0.532069,0.936072,0.527643,0.709395,0.583044,0.958379,0.506232,0.529913,0.580964,0.526098,0.788286,0.734971,0.507334,0.765056,0.572104,0.790302,0.851893,0.797614,0.872340,0.775905,0.625444,0.779510,0.748068,0.516113,0.777070,0.916025,0.592524,0.804850,0.618925,0.835853,0.514631,0.886762,0.686407,0.934694,0.918590,0.716536,0.973836,0.792157,0.515224,0.657477,0.746028,0.951194,0.565596,0.626611,0.735262,0.560969,0.518077,0.614600,0.656096,0.500163,0.956798,0.889978,0.703148,0.996482,0.690816,0.740396,0.575936,0.618171,0.517238,0.763139,0.821143,0.632452,0.929115,0.756735,0.662257,0.587805,0.829867,0.939363,0.655382,0.684219,0.967888,0.625424,0.586643,0.701271,0.827446,0.515474,0.545134,0.702809,0.676665,0.544586,0.757736,0.862262,0.807152,0.850112,0.865008,0.789688,0.741213,0.540799,0.599624,0.902239,0.644383,0.739793,0.503104,0.532967,0.590794,0.845418,0.693502,0.790884,0.885606,0.807435,0.502392,0.663400,0.875805,0.544340,0.541529,0.713744,0.951974,0.665610,0.957465,0.544534,0.588313,0.567754,0.985800,0.639520,0.588021,0.577110,0.645505,0.537671,0.522762,0.550865,0.879919,0.524439,0.851206,0.790563,0.639778,0.655743,0.773125,0.766991,0.959244,0.902945,0.821054,0.648036,0.536814,0.625034,0.610746,0.767837,0.980627,0.689937,0.952619,0.780337,0.841002,0.991293,0.844877,0.775886,0.579029,0.825071,0.859581,0.655667,0.545208,0.691890,0.788949,0.619448,0.850375,0.790991,0.766646,0.676400,0.614599,0.884112,0.941767,0.528970,0.804967,0.770956,0.605935,0.867950,0.516154,0.507800,0.664316,0.513553,0.944943,0.540805,0.565406,0.523452,0.594614,0.995808,0.649553,0.789561,0.547845,0.907971,0.650027,0.739723,0.633827,0.501248,0.739885,0.846222,0.919670,0.866197,0.547870,0.610222,0.760435,0.543733,0.831654,0.723954,0.503118,0.639165,0.873751,0.570313,0.914135,0.546993,0.899345,0.596767,0.511781,0.664953,0.511946,0.721502,0.930315,0.519869,0.739830,0.509652,0.698391,0.764034,0.597365,0.742278,0.530128,0.576110,0.696202,0.551621,0.780804,0.573469,0.983471,0.748026,0.731981,0.773276,0.783022,0.549698,0.643784,0.569552,0.930227,0.675539,0.588482,0.698599,0.854256,0.887234,0.524486,0.724449,0.950078,0.826652,0.612374,0.750545,0.741077,0.506226,0.807544,0.578553,0.860437,0.522549,0.796916,0.596094,0.902331,0.686230,0.991731,0.711694,0.593048,0.794332,0.648489,0.573727,0.662096,0.540078,0.732526,0.917416,0.642960,0.601451,0.627773,0.619197,0.690123,0.599519,0.899918,0.833188,0.636083,0.653921,0.819807,0.992075,0.630667,0.534623,0.664220,0.776074,0.574274,0.630525,0.795060,0.643596,0.716612,0.544086,0.727143,0.650509,0.580411,0.902734,0.500690,0.993069,0.655095,0.506523,0.951459,0.886073,0.595993,0.855351,0.553574,0.924624,0.750024,0.570677,0.617995,0.869897,0.978181,0.575715,0.618258,0.573848,0.519201,0.598530,0.814159,0.784323,0.629300,0.611306,0.989451,0.626344,0.682309,0.605456,0.680193,0.572431,0.595928,0.624493,0.929459,0.914140,0.598285,0.602193,0.683011,0.859203,0.527538,0.531235,0.846786,0.510933,0.657201,0.876427,0.616174,0.609434,0.653530,0.955549,0.676789,0.925281,0.618773,0.708178,0.961594,0.577823,0.560789,0.749644,0.572365,0.868992,0.785325,0.687884,0.757399,0.555040,0.575694,0.540548,0.732346,0.590817,0.823829,0.639873,0.507884,0.852589,0.862726,0.939594,0.596819,0.705644,0.533679,0.626767,0.800033,0.906382,0.597867,0.582194,0.508017,0.645570,0.696168,0.529233,0.698945,0.820484,0.529500,0.863863,0.768972,0.697563,0.762235,0.846690,0.733811,0.554854,0.561515,0.507731,0.918554,0.505316,0.658202,0.544890,0.996508,0.571442,0.938826,0.575211,0.631077,0.774541,0.575557,0.543825,0.663285,0.519048,0.630014,0.700404,0.969786,0.709192,0.579900,0.872045,0.965339,0.737167,0.596193,0.957214,0.576292,0.793452,0.640977,0.820770,0.643418,0.598370,0.548185,0.508839,0.601817,0.502676,0.604645,0.561167,0.776865,0.530896,0.520172,0.610181,0.946606,0.784966,0.535809,0.984917,0.835240,0.707461,0.678284,0.837997,0.786343,0.678015,0.587768,0.807991,0.913159,0.614342,0.645802,0.908670,0.517585,0.838668,0.816546,0.606560,0.872659,0.991883,0.837927,0.919420,0.821962,0.790527,0.567867,0.689342,0.641067,0.680758,0.665690,0.544285,0.933831,0.632920,0.796561,0.724286,0.642430,0.733453,0.595156,0.696845,0.597326,0.913878,0.860544,0.520556,0.591649,0.514253,0.910124,0.695327,0.746686,0.531227,0.563334,0.878178,0.637542,0.527452,0.541664,0.592918,0.857940,0.538898,0.788893,0.557869,0.555854,0.654658,0.981851,0.557831,0.627036,0.736635,0.774530,0.681089,0.793289,0.511622,0.725716,0.617445,0.866505,0.945197,0.833034,0.849060,0.535246,0.597148,0.761352,0.738088,0.738906,0.718186,0.584743,0.616256,0.978268,0.742617,0.613796,0.596885,0.580697,0.847417,0.752444,0.646995,0.640934,0.504575,0.837448,0.855624,0.621469,0.800894,0.688983,0.659076,0.667728,0.569365,0.796272,0.993683,0.504641,0.537560,0.512690,0.881517,0.823361,0.786771,0.699069,0.634008,0.609963,0.609253,0.775543,0.514775,0.549154,0.766743,0.741847,0.785474,0.585827,0.954076,0.613271,0.938105,0.523867,0.625692,0.500841,0.640083,0.941915,0.679935,0.770196,0.950213,0.618858,0.515998,0.678916,0.848682,0.695372,0.989589,0.862430,0.936881,0.798927,0.716464,0.989890,0.635544,0.562399,0.656562,0.698984,0.814685,0.758166,0.858394,0.555358,0.786140,0.696261,0.944867,0.597138,0.591836,0.904131,0.616994,0.666340,0.790589,0.951087,0.745082,0.876532,0.627027,0.555760,0.529748,0.612258,0.581472,0.517935,0.949452,0.746778,0.562403,0.598918,0.884217,0.891969,0.999615,0.564074,0.977330,0.509595,0.734009,0.515758,0.502730,0.506388,0.990118,0.678764,0.706190,0.692077,0.555773,0.532330,0.855944,0.639419,0.510437,0.660649,0.644333,0.630756,0.868477,0.822563,0.536445,0.554400,0.641723,0.761453,0.956484,0.639205,0.735892,0.925733,0.715011,0.554425,0.564788,0.621915,0.826873,0.548229,0.508671,0.559585,0.870337,0.963642,0.788479,0.909439,0.612140,0.558817,0.837165,0.574162,0.807326,0.808057,0.683644,0.901232,0.875295,0.635560,0.537776,0.534079,0.940256,0.931331,0.979836,0.584100,0.643845,0.712790,0.918606,0.951270,0.602717,0.976821,0.931031,0.777327,0.786403,0.616284,0.986166,0.644716,0.893594,0.652083,0.577190,0.600548,0.538585,0.526391,0.806243,0.505562,0.756358,0.670935,0.563842,0.505002,0.506221,0.663380,0.888800,0.655689,0.687509,0.685775,0.845773,0.576926,0.981041,0.517624,0.751769,0.579911,0.818153,0.616352,0.994026,0.815442,0.885199,0.586524,0.621621,0.915294,0.608482,0.746718,0.764307,0.941740,0.571486,0.728687,0.538213,0.970740,0.996697,0.638900,0.902156,0.594455,0.861059,0.768940,0.784274,0.770961,0.572239,0.967075,0.921336,0.823918,0.925284,0.541242,0.558156,0.873053,0.873291,0.736353,0.813581,0.827841,0.758372,0.768725,0.999147,0.973952,0.717400,0.979132,0.641750,0.752197,0.711181,0.667314,0.526520,0.596137,0.764303,0.512296,0.532848,0.566777,0.636097,0.541815,0.550246,0.657142,0.702477,0.615830,0.566191,0.506674,0.637781,0.608783,0.505086,0.699084,0.997870,0.505095,0.730768,0.673800,0.936200,0.551669,0.591472,0.509498,0.603360,0.840346,0.859060,0.935622,0.830605,0.811161,0.945015,0.518859,0.905584,0.502499,0.636128,0.564376,0.659330,0.629769,0.938711,0.539380,0.533862,0.862423,0.968619,0.540057,0.893904,0.536351,0.732951,0.578220,0.539420,0.546431,0.746623,0.719791,0.801284,0.576256,0.511658,0.834659,0.513695,0.778630,0.696062,0.922597,0.585971,0.851532,0.520402,0.503103,0.869720,0.765688,0.549789,0.810853,0.945636,0.769209,0.557265,0.887193,0.539201,0.613904,0.584833,0.803226,0.592610,0.730744,0.561245,0.725799,0.636277,0.577990,0.511029,0.583451,0.646840,0.563069,0.808889,0.715842,0.912571,0.585398,0.662905,0.891925,0.592828,0.786839,0.564493,0.602731,0.728474,0.893492,0.628474,0.771900,0.846210,0.893400,0.870803,0.521566,0.609196,0.606952,0.697326,0.590637,0.542593,0.797625,0.525967,0.610330,0.829968,0.742887,0.714756,0.984481,0.588275,0.629794,0.556188,0.866001,0.715531,0.626655,0.699312,0.971098,0.624122,0.929284,0.551866,0.765611,0.532878,0.919097,0.582137,0.854596,0.888147,0.899045,0.861968,0.541531,0.762225,0.939340,0.824324,0.549275,0.944234,0.557338,0.907472,0.531069,0.588830,0.870179,0.957171,0.642719,0.901087,0.739379,0.615856,0.533172,0.954113,0.520827,0.641890,0.507059,0.592937,0.716930,0.597284,0.611939,0.647637,0.911544,0.997156,0.683302,0.766656,0.874204,0.612509,0.845584,0.751163,0.569061,0.615750,0.743070,0.567462,0.829979,0.713875,0.677184,0.577960,0.528613,0.657423,0.602071,0.785290,0.531148,0.710869,0.596645,0.600038,0.696800,0.764354,0.529816,0.632395,0.972411,0.577837,0.697138,0.751266,0.566869,0.571092,0.746040,0.873589,0.663530,0.664138,0.526285,0.711953,0.647577,0.575988,0.938110,0.585856,0.775273,0.572160,0.587344,0.807710,0.657922,0.642077,0.973331,0.756012,0.979770,0.558106,0.525552,0.539318,0.717373,0.567686,0.603936,0.611312,0.625941,0.592208,0.883791,0.824243,0.546568,0.716456,0.842917,0.814299,0.546205,0.548810,0.605444,0.682097,0.605412,0.624696,0.539465,0.606622,0.600089,0.550705,0.717695,0.611942,0.743910,0.741889,0.622628,0.534918,0.569137,0.894513,0.552851,0.727120,0.743907,0.518930,0.531222,0.746931,0.514599,0.633452,0.548003,0.890160,0.622098,0.668221,0.605439,0.586655,0.676157,0.583415,0.982134,0.807437,0.854839,0.570603,0.728618,0.775516,0.564473,0.640001,0.612092,0.641176,0.576950,0.559935,0.763614,0.677501,0.593286,0.697892,0.512345,0.794315,0.600602,0.537981,0.680429,0.763786,0.670861,0.550615,0.693529,0.938163,0.649501,0.782586,0.506046,0.980950,0.564651,0.516781,0.542896,0.732785,0.921399,0.623048,0.774510,0.742417,0.644837,0.703639,0.931655,0.821816,0.572425,0.538037,0.619265,0.795266,0.819784,0.602754,0.621019,0.640647,0.715158,0.572436,0.844914,0.661986,0.570204,0.633178,0.589837,0.504358,0.503880,0.953028,0.941021,0.773477,0.617100,0.524447,0.959514,0.616920,0.994051,0.622047,0.579707,0.977863,0.883965,0.517498,0.919436,0.721954,0.605339,0.656205,0.736761,0.550144,0.540722,0.535584,0.703545,0.724732,0.725526,0.911513,0.953702,0.540633,0.878921,0.718954,0.937759,0.764276,0.745127,0.753253,0.720893,0.692553,0.830545,0.523685,0.586318,0.507544,0.754548,0.831747,0.633086,0.792771,0.764605,0.579202,0.563945,0.569500,0.945606,0.967706,0.734840,0.799773,0.729552,0.586663,0.696194,0.552002,0.549386,0.737554,0.573064,0.635246,0.550218,0.545450,0.814658,0.556983,0.877550,0.702733,0.655006,0.597104,0.862428,0.562633,0.737207,0.705347,0.677806,0.673170,0.530696,0.503456,0.919680,0.537953,0.789869,0.731602,0.660987,0.531278,0.893108,0.597234,0.668801,0.940121,0.560166,0.721393,0.952557,0.638505,0.583341,0.534981,0.741675,0.547381,0.686663,0.832752,0.656556,0.866257,0.576155,0.730997,0.739197,0.762546,0.853754,0.634256,0.540695,0.781926,0.768475,0.642464,0.803636,0.560546,0.655668,0.639358,0.587461,0.833858,0.667485,0.751041,0.614454,0.501477,0.542779,0.654859,0.512048,0.740756,0.639222,0.947222,0.652517,0.683474,0.631296,0.713726,0.928438,0.525630,0.777055,0.740661,0.669964,0.607008,0.686129,0.706080,0.795072,0.727700,0.753978,0.818293,0.750181,0.843698,0.612887,0.599698,0.596474,0.949384,0.819921,0.760735,0.595322,0.676244,0.580016,0.739181,0.531922,0.537842,0.533550,0.551602,0.916437,0.998584,0.901399,0.609518,0.767376,0.548626,0.758281,0.554672,0.830929,0.628931,0.661256,0.530605,0.709235,0.814150,0.721754,0.652166,0.812356,0.506760,0.712792,0.944952,0.931070,0.751725,0.839644,0.875722,0.713893,0.602822,0.777270,0.830624,0.738518,0.759083,0.677767,0.618090,0.866429,0.801718,0.592444,0.744471,0.981102,0.825311,0.875200,0.999738,0.833124,0.999138,0.513867,0.789865,0.902023,0.928744,0.811836,0.606700,0.677559,0.587987,0.685891,0.583729,0.746867,0.615306,0.550220,0.987629,0.573667,0.810867,0.630285,0.665694,0.971517,0.857226,0.623194,0.949102,0.874894,0.674204,0.673621,0.578675,0.850481,0.686084,0.663580,0.823978,0.926396,0.587889,0.920390,0.801222,0.946478,0.799800,0.884052,0.545823,0.944066,0.739675,0.619121,0.833233,0.999526,0.752286,0.903799,0.998019,0.905744,0.978997,0.874618,0.726551,0.678615,0.630007,0.763051,0.755087,0.641164,0.712223,0.940395,0.669515,0.944850,0.927721,0.621186,0.546176,0.634426,0.672634,0.556565,0.726741,0.738384,0.702573,0.544261,0.519451,0.559755,0.876897,0.666823,0.920377,0.703857,0.729300,0.714329,0.554554,0.671730,0.532928,0.522928,0.557358,0.549319,0.682384,0.509919,0.971035,0.889882,0.658129,0.678316,0.842956,0.761665,0.555988,0.955824,0.971379,0.588132,0.759916,0.665853,0.568994,0.735206,0.803837,0.988307,0.830325,0.555386,0.689477,0.761461,0.884233,0.627987,0.672966,0.643474,0.665934,0.529350,0.956812,0.808407,0.910650,0.697560,0.669906,0.880325,0.741548,0.673563,0.659283,0.522732,0.710561,0.973810,0.779828,0.668022,0.570385,0.536956,0.699539,0.714519,0.683073,0.988460,0.967468,0.693715,0.504834,0.633235,0.655916,0.981442,0.644848,0.777673,0.766192,0.693822,0.930579,0.635827,0.698063,0.873100,0.550790,0.615053,0.569593,0.520723,0.910724,0.779356,0.535926,0.683598,0.640648,0.777543,0.756653,0.516816,0.738956,0.983663,0.662003,0.659520,0.572166,0.833343,0.698550,0.856914,0.821988,0.556409,0.876196,0.523745,0.708343,0.631208,0.520808,0.652548,0.835610,0.833537,0.979394,0.843396,0.514704,0.531751,0.815277,0.510524,0.861111,0.650909,0.848665,0.589516,0.606538,0.915326,0.802093,0.840071,0.901106,0.983619,0.714243,0.549946,0.969613,0.617826,0.724555,0.917766,0.626923,0.556719,0.652977,0.713006,0.934746,0.803009,0.849931,0.698445,0.806710,0.702876,0.684261,0.806820,0.744627,0.590790,0.800063,0.591266,0.647744,0.678779,0.532063,0.777036,0.712662,0.779354,0.825681,0.632651,0.795715,0.544807,0.690138,0.807886,0.848097,0.718658,0.579235,0.723836,0.858499,0.571416,0.531400,0.595108,0.577173,0.507547,0.541393,0.911451,0.562464,0.507352,0.606041,0.646313,0.505811,0.563897,0.732594,0.749048,0.955679,0.718085,0.620835,0.736802,0.603288,0.759591,0.980143,0.736771,0.528342,0.501921,0.959594,0.917018,0.527886,0.902407,0.729783,0.653212,0.665446,0.540767,0.883835,0.605450,0.573140,0.798384,0.725279,0.577633,0.657742,0.811060,0.514478,0.686857,0.632476,0.700978,0.679783,0.734708,0.736272,0.757078,0.542637,0.737452,0.992745,0.712802,0.679536,0.694931,0.501518,0.585430,0.650969,0.858287,0.507007,0.625359,0.537798,0.964571,0.705347,0.522781,0.644189,0.508280,0.614452,0.508777,0.666098,0.656876,0.644247,0.806135,0.978321,0.655593,0.514662,0.591694,0.805105,0.937802,0.603570,0.671646,0.754277,0.563175,0.685307,0.924375,0.533537,0.611140,0.737830,0.501465,0.643778,0.875589,0.563945,0.631215,0.661369,0.552283,0.517107,0.784905,0.795549,0.789633,0.618027,0.724874,0.812983,0.693754,0.763033,0.756895,0.599886,0.572449,0.888209,0.557466,0.897873,0.973833,0.576161,0.643498,0.645189,0.618540,0.839385,0.577761,0.672440,0.551960,0.625908,0.636451,0.934988,0.630766,0.653313,0.544492,0.900004,0.697702,0.626810,0.582827,0.863164,0.721953,0.677732,0.806922,0.799017,0.820725,0.689509,0.719139,0.851470,0.958030,0.562277,0.557857,0.678543,0.536398,0.713357,0.532537,0.519844,0.633664,0.655192,0.574732,0.506720,0.508877,0.634064,0.770144,0.580996,0.911460,0.605195,0.748450,0.894362,0.647111,0.783863,0.778454,0.663157,0.684950,0.570893,0.831708,0.959313,0.885839,0.579009,0.838303,0.608178,0.620024,0.807940,0.686692,0.636201,0.707419,0.573843,0.983536,0.503431,0.916724,0.595554,0.742341,0.844338,0.654132,0.759138,0.521186,0.583152,0.533216,0.521754,0.530076,0.951813,0.576690,0.935914,0.678676,0.529591,0.694508,0.616697,0.815301,0.722301,0.685978,0.505641,0.961408,0.598097,0.846949,0.594977,0.589853,0.690954,0.658390,0.585464,0.944905,0.542378,0.741924,0.514391,0.505233,0.852577,0.542500,0.593977,0.768848,0.566317,0.916713,0.594405,0.788041,0.890531,0.910257,0.817671,0.743112,0.844536,0.859790,0.783155,0.883356,0.843849,0.729207,0.514572,0.651207,0.511500,0.643327,0.771066,0.898742,0.677583,0.870113,0.725814,0.602045,0.863056,0.679501,0.857567,0.780977,0.873899,0.686802,0.559026,0.511755,0.665161,0.621276,0.708349,0.837126,0.991592,0.573552,0.836662,0.615766,0.820591,0.627798,0.568010,0.585083,0.866799,0.664486,0.610770,0.883880,0.523724,0.972571,0.935591,0.652894,0.726912,0.557181,0.740890,0.500232,0.612517,0.583598,0.563989,0.873727,0.619920,0.729842,0.548214,0.899497,0.805314,0.611794,0.577746,0.538465,0.584074,0.782151,0.971612,0.858645,0.902246,0.559142,0.540935,0.765801,0.989426,0.734659,0.863871,0.837640,0.965785,0.743858,0.744570,0.631519,0.516083,0.675020,0.738280,0.700001,0.670336,0.514426,0.556168,0.726172,0.633724,0.772538,0.617750,0.995205,0.726602,0.851744,0.741028,0.919109,0.969972,0.653029,0.604762,0.889257,0.854989,0.891409,0.742480,0.954394,0.573903,0.612549,0.718215,0.530028,0.586764,0.533998,0.533325,0.579709,0.678006,0.839645,0.596877,0.995256,0.615991,0.615398,0.917392,0.533670,0.853401,0.928604,0.711693,0.628619,0.665355,0.588135,0.580370,0.981683,0.563483,0.630975,0.641461,0.904809,0.569494,0.571201,0.574758,0.782834,0.806949,0.792373,0.875412,0.663244,0.508612,0.715533,0.560327,0.906138,0.598049,0.924428,0.920379,0.708396,0.931449,0.570919,0.659348,0.689566,0.557917,0.557153,0.704723,0.721234,0.704228,0.749758,0.883031,0.711300,0.664978,0.757232,0.664238,0.677848,0.678099,0.512839,0.837224,0.697616,0.598829,0.579431,0.925551,0.969886,0.661225,0.887977,0.979205,0.659487,0.912933,0.578682,0.648454,0.547886,0.546753,0.775229,0.623283,0.668829,0.759840,0.979887,0.653012,0.848758,0.773502,0.519394,0.691132,0.955303,0.940122,0.756775,0.755412,0.636193,0.639415,0.853408,0.735443,0.731176,0.581963,0.907290,0.597015,0.568916,0.654551,0.884493,0.537199,0.579400,0.564886,0.763542,0.712592,0.852928,0.679759,0.520140,0.651576,0.663517,0.947741,0.739012,0.961551,0.869485,0.674340,0.779161,0.812023,0.592393,0.561127,0.755205,0.603725,0.885709,0.727335,0.734018,0.692297,0.687369,0.691643,0.824072,0.634836,0.526812,0.944132,0.638664,0.707831,0.811512,0.917666,0.733221,0.516776,0.867709,0.968079,0.585601,0.871882,0.589578,0.784592,0.601503,0.522228,0.527570,0.827620,0.552926,0.858683,0.807749,0.667479,0.821181,0.733385,0.840469,0.516018,0.873647,0.731054,0.722466,0.866298,0.891842,0.902227,0.709711,0.906681,0.545765,0.665147,0.758984,0.562616,0.808766,0.685160,0.559434,0.553559,0.716868,0.680506,0.640577,0.847962,0.579610,0.570897,0.599804,0.819618,0.918574,0.860108,0.815996,0.975244,0.641230,0.676544,0.803879,0.572593,0.815404,0.667610,0.584007,0.742261,0.798428,0.666454,0.661443,0.710188,0.709715,0.841350,0.754566,0.561633,0.714632,0.596170,0.536680,0.880736,0.886229,0.517098,0.636266,0.784435,0.664498,0.505485,0.824438,0.870196,0.590463,0.954252,0.829065,0.557833,0.935244,0.863607,0.928383,0.577509,0.521375,0.834089,0.610002,0.913727,0.561974,0.572586,0.515178,0.784830,0.602657,0.514392,0.578813,0.541398,0.790016,0.965736,0.722083,0.582409,0.746435,0.612439,0.914109,0.528504,0.808418,0.529925,0.787473,0.739842,0.675037,0.567862,0.766163,0.998391,0.513532,0.540791,0.638090,0.737855,0.998566,0.501686,0.969428,0.869150,0.829185,0.620842,0.579800,0.995788,0.823361,0.817223,0.813248,0.713392,0.586401,0.763321,0.961320,0.609004,0.614993,0.739218,0.685794,0.749017,0.883856,0.772277,0.674400,0.750968,0.966812,0.629230,0.509023,0.766307,0.639149,0.718462,0.594866,0.605212,0.676291,0.538206,0.927312,0.561683,0.544496,0.649181,0.507267,0.843374,0.576635,0.728932,0.667112,0.925818,0.773585,0.964012,0.567711,0.741909,0.880494,0.949148,0.697389,0.800953,0.648329,0.591723,0.540570,0.999282,0.607876,0.601718,0.823303,0.641120,0.502331,0.843018,0.502449,0.994730,0.538622,0.758998,0.584873,0.849183,0.594003,0.805337,0.858554,0.726913,0.808361,0.910592,0.619173,0.593353,0.696454,0.734280,0.887143,0.631785,0.500034,0.527213,0.528349,0.977336,0.914789,0.664639,0.623364,0.597752,0.950006,0.760690,0.713936,0.658820,0.713928,0.959254,0.577103,0.579882,0.991914,0.960717,0.999268,0.693675,0.926908,0.754997,0.583943,0.563817,0.966540,0.623486,0.716265,0.714320,0.547513,0.875084,0.928688,0.914219,0.741007,0.531051,0.553399,0.553616,0.813321,0.626984,0.838849,0.850641,0.535355,0.698473,0.523779,0.562085,0.520656,0.911717,0.855013,0.885419,0.773076,0.733119,0.737374,0.860978,0.995190,0.517962,0.983629,0.637102,0.790214,0.708869,0.786036,0.552572,0.923412,0.659958,0.710759,0.640291,0.888194,0.685508,0.686746,0.799612,0.663829,0.875263,0.734080,0.572014,0.643784,0.591386,0.700059,0.989654,0.596260,0.512155,0.640846,0.854920,0.974212,0.509565,0.569699,0.655896,0.871739,0.660980,0.647543,0.691377,0.666229,0.959819,0.544469,0.669325,0.562469,0.989326,0.921676,0.646172,0.650243,0.995987,0.631665,0.849863,0.755567,0.534809,0.665446,0.883112,0.614851,0.705338,0.577884,0.718760,0.822746,0.609558,0.863524,0.589265,0.503916,0.566956,0.520792,0.978713,0.792200,0.646343,0.965608,0.578727,0.634483,0.603305,0.549199,0.950767,0.609499,0.504564,0.750652,0.795426,0.591725,0.576612,0.709484,0.672946,0.608648,0.931809,0.623002,0.636168,0.707310,0.752800,0.746362,0.613409,0.891440,0.892629,0.560354,0.754741,0.759308,0.654941,0.667417,0.584534,0.908306,0.661293,0.701652,0.516890,0.758899,0.945687,0.618471,0.521663,0.724159,0.657570,0.649417,0.868077,0.690567,0.900238,0.513246,0.980215,0.763667,0.737894,0.558297,0.619883,0.754328,0.615510,0.956569,0.506236,0.510048,0.947502,0.557613,0.965641,0.658059,0.934239,0.952932,0.505193,0.770287,0.754192,0.848794,0.703100,0.611856,0.688641,0.682286,0.618208,0.786553,0.629621,0.588185,0.526500,0.884067,0.777585,0.628005,0.614358,0.575600,0.758855,0.533402,0.991082,0.804317,0.733627,0.766442,0.575666,0.746326,0.556711,0.651503,0.504193,0.724125,0.605875,0.885287,0.883759,0.547549,0.639262,0.865984,0.984982,0.523507,0.909274,0.905247,0.501590,0.835563,0.850758,0.580158,0.836755,0.671568,0.813416,0.920563,0.855790,0.654985,0.640622,0.920451,0.656424,0.702052,0.849196,0.540764,0.836412,0.606608,0.764145,0.784002,0.738536,0.666934,0.534426,0.934726,0.959931,0.639761,0.520150,0.620306,0.783163,0.557239,0.944380,0.537247,0.560159,0.949896,0.605944,0.617995,0.712733,0.935938,0.692214,0.514907,0.753204,0.609096,0.760504,0.618197,0.612355,0.669274,0.563986,0.566544,0.554738,0.805921,0.970880,0.773282,0.935873,0.618984,0.557433,0.834106,0.774337,0.584301,0.849172,0.636223,0.520293,0.517547,0.721654,0.672216,0.543783,0.651748,0.746242,0.791217,0.930841,0.750706,0.517192,0.539871,0.944925,0.949728,0.552847,0.984105,0.964609,0.563890,0.889150,0.657723,0.700857,0.939795,0.996494,0.989278,0.989058,0.776400,0.585383,0.727463,0.701573,0.885241,0.962304,0.666143,0.974592,0.549890,0.676569,0.877568,0.747263,0.588107,0.667132,0.882108,0.771906,0.643425,0.793242,0.855509,0.917481,0.572451,0.562213,0.927125,0.551868,0.621848,0.658733,0.755661,0.576843,0.597985,0.614166,0.649640,0.519741,0.652219,0.557949,0.669167,0.518694,0.876547,0.775198,0.838530,0.660168,0.606466,0.824417,0.809831,0.534045,0.580908,0.913441,0.640894,0.985380,0.889219,0.554894,0.504138,0.731453,0.714061,0.636856,0.758591,0.867202,0.519219,0.867679,0.965750,0.658391,0.566936,0.622809,0.755698,0.603984,0.921385,0.506342,0.774752,0.763488,0.974031,0.510902,0.691972,0.516127,0.856206,0.817338,0.610604,0.668508,0.976724,0.606187,0.977790,0.553762,0.815766,0.820205,0.984786,0.844373,0.682433,0.798325,0.870105,0.613076,0.529185,0.552937,0.883118,0.993197,0.598587,0.562008,0.908539,0.849930,0.764904,0.633707,0.615352,0.631268,0.517136,0.970065,0.827282,0.738403,0.558952,0.503508,0.650657,0.848403,0.743858,0.712588,0.676665,0.861572,0.712415,0.627873,0.902920,0.743667,0.653521,0.821057,0.709997,0.652995,0.684754,0.874867,0.554613,0.942979,0.931895,0.601492,0.923847,0.515800,0.682734,0.908876,0.918473,0.608141,0.933847,0.562846,0.814976,0.966520,0.615639,0.669677,0.901504,0.691657,0.995515,0.552257,0.850222,0.722721,0.926997,0.537554,0.949123,0.553945,0.524594,0.941226,0.746874,0.619498,0.843324,0.672436,0.552869,0.636027,0.731731,0.798249,0.799978,0.659709,0.661360,0.812652,0.522329,0.807589,0.581918,0.513490,0.915426,0.762949,0.596433,0.510195,0.737116,0.909240,0.784177,0.738464,0.614148,0.614930,0.544939,0.790718,0.744584,0.519749,0.504701,0.609621,0.647541,0.778374,0.777979,0.946161,0.872328,0.599260,0.819554,0.595809,0.770631,0.846355,0.642494,0.566638,0.708626,0.723686,0.596289,0.542224,0.825448,0.574739,0.543944,0.801142,0.814434,0.716670,0.741363,0.729707,0.834717,0.595104,0.625872,0.927751,0.817022,0.840772,0.676946,0.703277,0.591272,0.514247,0.908231,0.630469,0.832443,0.797987,0.589232,0.948397,0.520177,0.939082,0.962957,0.564949,0.662485,0.541167,0.775355,0.670228,0.742819,0.781511,0.785852,0.538826,0.530851,0.706817,0.711860,0.903144,0.699524,0.994407,0.679979,0.607387,0.832652,0.922925,0.587472,0.748882,0.816677,0.811118,0.969121,0.608888,0.600922,0.598310,0.839745,0.725001,0.520716,0.680782,0.761908,0.834019,0.576319,0.545592,0.857279,0.912769,0.883056,0.509440,0.619490,0.559129,0.748489,0.912947,0.503602,0.700375,0.987868,0.519053,0.708596,0.709334,0.985695,0.599141,0.873984,0.647199,0.608123,0.692995,0.560984,0.582659,0.565236,0.731415,0.597860,0.763322,0.944641,0.719295,0.666370,0.526266,0.616460,0.779492,0.579155,0.903077,0.750264,0.704487,0.708576,0.787001,0.673809,0.836395,0.511639,0.933364,0.609936,0.681194,0.552071,0.977883,0.505176,0.701659,0.842014,0.827086,0.568152,0.669831,0.752071,0.724402,0.822357,0.752089,0.865838,0.595330,0.767238,0.546489,0.568870,0.936890,0.753102,0.764146,0.676273,0.718721,0.680869,0.600867,0.934059,0.621526,0.866256,0.598016,0.589279,0.675622,0.893511,0.961221,0.551944,0.938293,0.502787,0.940036,0.550489,0.942418,0.512653,0.695426,0.894325,0.524181,0.589791,0.922541,0.564094,0.888467,0.711839,0.509231,0.819081,0.712164,0.700123,0.565950,0.923861,0.589866,0.693763,0.638285,0.543972,0.708237,0.570788,0.553742,0.750884,0.571509,0.857574,0.585055,0.722576,0.920969,0.517325,0.696868,0.509328,0.523987,0.705519,0.865278,0.805067,0.597140,0.556465,0.510387,0.757550,0.587819,0.560782,0.590369,0.636102,0.850402,0.690639,0.731809,0.821140,0.758244,0.872635,0.510334,0.840642,0.722912,0.531033,0.575819,0.957558,0.891161,0.512455,0.938847,0.580250,0.579720,0.776771,0.505235,0.955708,0.598116,0.598638,0.874306,0.658390,0.723479,0.557795,0.505378,0.838391,0.577471,0.760692,0.716115,0.520707,0.673937,0.799757,0.838535,0.803496,0.503638,0.548817,0.853991,0.768521,0.724814,0.788611,0.756867,0.752704,0.610884,0.527099,0.743726,0.582118,0.785642,0.626627,0.724835,0.581080,0.530371,0.574272,0.561689,0.565583,0.611392,0.811315,0.543161,0.721275,0.541108,0.567941,0.955085,0.682581,0.530669,0.570970,0.856312,0.603365,0.796838,0.620969,0.866427,0.966626,0.843111,0.921620,0.755568,0.582521,0.948686,0.525564,0.554881,0.571646,0.897834,0.540540,0.655894,0.546205,0.986921,0.654328,0.824836,0.627441,0.584384,0.527154,0.611653,0.605053,0.741931,0.544735,0.722284,0.528621,0.549392,0.502486,0.606220,0.926970,0.927355,0.569932,0.764842,0.913455,0.593394,0.659964,0.532539,0.651870,0.525209,0.754308,0.619127,0.675847,0.919913,0.715868,0.739221,0.551437,0.673114,0.937618,0.960209,0.958402,0.687132,0.615115,0.540387,0.704407,0.762349,0.783990,0.703846,0.615541,0.691434,0.643841,0.891835,0.613532,0.976263,0.669761,0.835632,0.502207,0.533547,0.894128,0.849395,0.777132,0.697874,0.663253,0.575280,0.522006,0.564095,0.543547,0.669247,0.716258,0.731933,0.861892,0.630005,0.996409,0.716337,0.785728,0.592344,0.952118,0.536859,0.665883,0.640653,0.730467,0.533468,0.986323,0.764256,0.675848,0.763326,0.503790,0.827139,0.751071,0.616367,0.799525,0.839094,0.791436,0.653649,0.690015,0.539330,0.780384,0.688062,0.949983,0.508742,0.693994,0.861768,0.920694,0.548549,0.612650,0.852520,0.771478,0.586034,0.539043,0.869452,0.909943,0.843261,0.648204,0.708105,0.706423,0.795206,0.673224,0.805719,0.547554,0.529790,0.774566,0.780466,0.779569,0.773657,0.895749,0.505619,0.894808,0.787595,0.850597,0.728140,0.512593,0.632804,0.836719,0.555839,0.504710,0.763780,0.508833,0.883677,0.570279,0.781533,0.744480,0.696473,0.553134,0.739616,0.541674,0.740717,0.535553,0.909158,0.890952,0.971524,0.971544,0.725047,0.625883,0.804351,0.689999,0.722617,0.715495,0.562062,0.819769,0.795018,0.621154,0.762839,0.550259,0.658394,0.716728,0.523950,0.700942,0.517613,0.803633,0.877365,0.567400,0.658366,0.644213,0.770339,0.827218,0.853795,0.991772,0.663370,0.635370,0.701364,0.533248,0.838556,0.628542,0.890067,0.649037,0.860719,0.865858,0.509145,0.979432,0.794132,0.624574,0.596074,0.849369,0.719293,0.696932,0.770111,0.545950,0.782424,0.789451,0.985264,0.937913,0.843813,0.689682,0.514098,0.753929,0.763085,0.861457,0.511343,0.824940,0.533537,0.752991,0.535063,0.831005,0.548546,0.836809,0.750531,0.889228,0.719911,0.905267,0.632672,0.586642,0.566524,0.778032,0.500898,0.618645,0.700105,0.871279,0.728331,0.587320,0.869332,0.957668,0.924286,0.513152,0.667513,0.610247,0.586974,0.913741,0.691536,0.635485,0.680732,0.755957,0.852593,0.940250,0.646417,0.573805,0.843758,0.729819,0.572511,0.872488,0.746959,0.630049,0.946575,0.682988,0.529514,0.604104,0.506792,0.966661,0.920138,0.630057,0.802504,0.502546,0.592717,0.749225,0.808493,0.933668,0.815700,0.712437,0.586381,0.864184,0.763567,0.787908,0.534593,0.764608,0.707185,0.890938,0.893887,0.506315,0.612433,0.697811,0.839731,0.646801,0.584790,0.919585,0.582396,0.502419,0.727713,0.767170,0.532109,0.752621,0.652219,0.503301,0.639108,0.552483,0.501054,0.734127,0.516213,0.503943,0.623942,0.714829,0.699234,0.518069,0.572523,0.739190,0.609959,0.924651,0.884236,0.503854,0.708727,0.669853,0.833531,0.551854,0.702788,0.530451,0.954487,0.506835,0.596228,0.505565,0.721148,0.834890,0.618940,0.744900,0.561004,0.524148,0.514697,0.981204,0.705061,0.895331,0.969275,0.967858,0.506036,0.500625,0.537085,0.533476,0.534902,0.515487,0.762495,0.560231,0.996683,0.767157,0.654595,0.585332,0.707975,0.753409,0.841647,0.950101,0.644966,0.610084,0.985839,0.646356,0.563155,0.516905,0.824844,0.851842,0.523721,0.714301,0.971837,0.722321,0.813511,0.623808,0.829967,0.976837,0.921969,0.621651,0.507200,0.849974,0.725270,0.677225,0.603105,0.818601,0.783102,0.526708,0.675574,0.546724,0.556926,0.670646,0.530074,0.897612,0.683825,0.517911,0.674712,0.759494,0.538674,0.653336,0.760355,0.625351,0.980106,0.774639,0.838296,0.510698,0.821907,0.500992,0.644927,0.554877,0.523254,0.699796,0.835125,0.583804,0.991995,0.512944,0.560377,0.533173,0.676009,0.839052,0.656084,0.510315,0.576778,0.505118,0.590188,0.846397,0.527700,0.557404,0.521196,0.991595,0.625693,0.732337,0.696672,0.589049,0.915172,0.878431,0.910804,0.515930,0.568950,0.963937,0.847623,0.533205,0.826535,0.643164,0.773575,0.574473,0.868914,0.936814,0.517682,0.642729,0.809113,0.723897,0.534002,0.630933,0.749379,0.551759,0.532680,0.793257,0.530681,0.553759,0.948849,0.916699,0.764604,0.577344,0.730440,0.742401,0.760682,0.610268,0.602803,0.800620,0.518761,0.601746,0.910843,0.689963,0.802196,0.516581,0.585352,0.631632,0.508305,0.725961,0.824693,0.803859,0.630367,0.521441,0.993709,0.681941,0.664433,0.822994,0.802408,0.693378,0.543853,0.697275,0.863476,0.547124,0.538197,0.601209,0.542555,0.815934,0.658789,0.778716,0.653416,0.601716,0.502133,0.628962,0.577337,0.902744,0.932941,0.511284,0.823623,0.835918,0.663038,0.648433,0.872804,0.673192,0.598753,0.984780,0.632324,0.882814,0.524108,0.913428,0.852440,0.588927,0.564025,0.786222,0.650956,0.688242,0.700321,0.767203,0.644779,0.597510,0.943279,0.866921,0.895979,0.805695,0.794968,0.526035,0.718327,0.830291,0.627117,0.819575,0.508012,0.532876,0.881097,0.531464,0.902403,0.991488,0.899855,0.851200,0.773968,0.581412,0.503783,0.535875,0.573791,0.733626,0.892296,0.893650,0.875138,0.622181,0.842651,0.703699,0.622178,0.869453,0.545711,0.890036,0.972176,0.569314,0.713792,0.912685,0.528044,0.634236,0.512362,0.557148,0.628930,0.644361,0.543973,0.698697,0.589832,0.645332,0.714787,0.712286,0.592855,0.686610,0.575320,0.519729,0.642361,0.574733,0.733034,0.755392,0.847070,0.548175,0.913389,0.506499,0.926268,0.865123,0.708382,0.708208,0.944866,0.525124,0.543796,0.991822,0.678961,0.859482,0.553653,0.797197,0.764444,0.628739,0.698762,0.933248,0.739314,0.751483,0.615579,0.968856,0.787372,0.928314,0.572451,0.617848,0.583705,0.678702,0.886644,0.674474,0.839833,0.705326,0.520668,0.873808,0.564115,0.837796,0.507237,0.797424,0.967884,0.559393,0.539089,0.521019,0.567915,0.683274,0.663163,0.926174,0.851034,0.598099,0.520940,0.766174,0.604052,0.735472,0.957765,0.508439,0.710358,0.737125,0.687163,0.540075,0.624261,0.647395,0.789566,0.568910,0.519065,0.552592,0.640954,0.745107,0.673517,0.801523,0.633326,0.516008,0.523768,0.982776,0.560056,0.930737,0.702838,0.525950,0.602564,0.814611,0.861874,0.968981,0.700073,0.803754,0.621713,0.717008,0.576157,0.874678,0.693695,0.602471,0.604209,0.720199,0.704320,0.832420,0.608645,0.735825,0.868325,0.509553,0.958016,0.640976,0.935672,0.774649,0.861274,0.906233,0.933088,0.588045,0.726342,0.665768,0.726555,0.783580,0.562261,0.603685,0.831492,0.503901,0.866014,0.535379,0.559088,0.561127,0.612476,0.548385,0.655123,0.734153,0.968885,0.528489,0.922616,0.874805,0.727397,0.534924,0.590262,0.576748,0.543916,0.853998,0.537000,0.686157,0.597628,0.634543,0.522308,0.823159,0.739561,0.577562,0.525232,0.750939,0.868440,0.567283,0.708184,0.684699,0.730530,0.543251,0.757555,0.619784,0.664937,0.572125,0.561445,0.590038,0.695405,0.741337,0.741017,0.665725,0.875879,0.579453,0.761831,0.844328,0.542773,0.612663,0.986666,0.590166,0.952122,0.883781,0.904736,0.594011,0.646713,0.508855,0.822593,0.835552,0.627068,0.648592,0.676146,0.629317,0.512879,0.841108,0.570068,0.820399,0.503736,0.679845,0.731545,0.683780,0.923741,0.558218,0.763625,0.532928,0.549872,0.758491,0.503663,0.591768,0.892024,0.741213,0.650270,0.624999,0.870105,0.585898,0.903017,0.898445,0.593073,0.760248,0.526555,0.813661,0.510501,0.759692,0.577047,0.609790,0.629221,0.657641,0.500862,0.985577,0.848149,0.904250,0.501575,0.891844,0.549764,0.501485,0.512287,0.676472,0.885835,0.758037,0.628885,0.876900,0.530900,0.891333,0.938688,0.785915,0.545012,0.594979,0.636700,0.948734,0.874254,0.907222,0.591149,0.997653,0.573888,0.586976,0.703652,0.703566,0.742099,0.827900,0.505013,0.692390,0.990055,0.719260,0.616231,0.801038,0.843222,0.586871,0.816404,0.597735,0.795063,0.537714,0.520337,0.606663,0.643655,0.547929,0.917751,0.703245,0.984928,0.594555,0.708408,0.609666,0.640518,0.526543,0.613522,0.675160,0.902954,0.503332,0.589793,0.767484,0.542982,0.729093,0.829103,0.552915,0.540976,0.600182,0.765056,0.965796,0.719174,0.750932,0.753438,0.756787,0.621977,0.717951,0.769307,0.595148,0.840284,0.590798,0.586413,0.693035,0.726953,0.931432,0.822536,0.940611,0.522581,0.505513,0.964710,0.538210,0.529173,0.967194,0.781071,0.781881,0.514050,0.882665,0.691283,0.528601,0.734897,0.919531,0.932379,0.633531,0.982776,0.827220,0.641821,0.975984,0.935706,0.525425,0.909217,0.653934,0.647047,0.821308,0.551635,0.527476,0.803593,0.569967,0.749579,0.617751,0.955456,0.680666,0.815584,0.709871,0.772261,0.583934,0.924597,0.992132,0.783178,0.598349,0.656441,0.932045,0.620222,0.607042,0.743315,0.623003,0.501665,0.732991,0.590135,0.858500,0.574110,0.969271,0.658262,0.604312,0.705263,0.772812,0.562962,0.775995,0.554305,0.594028,0.820726,0.632997,0.584816,0.517962,0.631286,0.956190,0.705766,0.760285,0.862133,0.510429,0.739567,0.730664,0.886141,0.780403,0.532663,0.866574,0.895377,0.633876,0.982282,0.608560,0.559136,0.842374,0.605851,0.865479,0.664878,0.630675,0.649198,0.905436,0.774400,0.691177,0.999356,0.567669,0.706635,0.842205,0.575875,0.614601,0.712521,0.693030,0.570587,0.761850,0.840463,0.538686,0.951071,0.511244,0.560311,0.762092,0.525586,0.836494,0.689068,0.971885,0.846765,0.573102,0.733133,0.709465,0.581890,0.654889,0.648199,0.921699,0.666277,0.654544,0.767004,0.531336,0.698021,0.630931,0.672496,0.922999,0.925655,0.669907,0.588356,0.901754,0.568475,0.981187,0.616911,0.617111,0.986815,0.963154,0.804798,0.607994,0.677128,0.998507,0.970268,0.797521,0.955057,0.575571,0.595654,0.872737,0.769483,0.855150,0.602823,0.814494,0.529876,0.824324,0.543149,0.643310,0.938658,0.873784,0.866073,0.676973,0.778422,0.857828,0.610829,0.839593,0.952412,0.534827,0.778072,0.535892,0.916031,0.574852,0.508783,0.517790,0.926070,0.530911,0.748270,0.894247,0.690395,0.654130,0.745797,0.704368,0.735838,0.595397,0.751895,0.505512,0.501034,0.619694,0.520997,0.719866,0.817649,0.538961,0.686948,0.526390,0.632108,0.699758,0.595855,0.563690,0.526779,0.539282,0.724077,0.545442,0.989132,0.830829,0.727480,0.939462,0.592442,0.638393,0.918422,0.713641,0.670559,0.631573,0.809208,0.670035,0.612256,0.630092,0.566846,0.853431,0.832513,0.541846,0.843974,0.866694,0.868594,0.656614,0.806210,0.855191,0.883944,0.508385,0.995325,0.736055,0.744737,0.588954,0.891410,0.582891,0.793675,0.641863,0.645822,0.975255,0.884621,0.533285,0.554773,0.718786,0.856548,0.886925,0.550277,0.729608,0.533782,0.548847,0.660589,0.809596,0.960383,0.584846,0.719890,0.720830,0.606947,0.790727,0.503022,0.554059,0.583930,0.518872,0.805535,0.643806,0.528050,0.653397,0.690622,0.661002,0.934629,0.661735,0.687278,0.603281,0.671266,0.522181,0.694461,0.852089,0.547539,0.524773,0.614049,0.611058,0.846746,0.556046,0.511449,0.985240,0.613508,0.588394,0.978595,0.875505,0.713726,0.663581,0.834052,0.866005,0.518319,0.923396,0.530666,0.769289,0.508762,0.553037,0.703708,0.653238,0.727881,0.672764,0.837122,0.574256,0.780866,0.547939,0.715182,0.600175,0.640939,0.524759,0.703772,0.549132,0.802218,0.913902,0.964720,0.785720,0.660087,0.592634,0.730902,0.622222,0.533492,0.637865,0.545771,0.733157,0.579887,0.538513,0.561288,0.613353,0.694805,0.581956,0.568000,0.591514,0.728142,0.528379,0.931939,0.832382,0.741379,0.639575,0.779374,0.991595,0.617615,0.513360,0.993432,0.591399,0.891234,0.751954,0.791084,0.943363,0.699569,0.880563,0.783962,0.525004,0.986367,0.758320,0.660129,0.741181,0.748828,0.560043,0.554868,0.942524,0.958691,0.846642,0.911698,0.860783,0.804322,0.972885,0.711682,0.548586,0.749442,0.879445,0.598196,0.541249,0.705937,0.712772,0.780354,0.521083,0.678400,0.966816,0.531289,0.933411,0.698979,0.639995,0.995052,0.625717,0.727666,0.643421,0.618136,0.535584,0.578388,0.901604,0.746425,0.715121,0.744285,0.635932,0.503484,0.526595,0.644816,0.882600,0.796886,0.618334,0.853331,0.706714,0.505949,0.948771,0.577283,0.991625,0.520418,0.523480,0.561733,0.791824,0.887423,0.871868,0.559470,0.991475,0.638538,0.508198,0.522453,0.633542,0.594091,0.580778,0.888820,0.650140,0.752504,0.669706,0.800865,0.667639,0.620238,0.896773,0.748695,0.823375,0.907559,0.535090,0.502358,0.609952,0.537294,0.521898,0.890650,0.567356,0.855854,0.546796,0.656991,0.859524,0.898317,0.535944,0.503861,0.940333,0.587773,0.823000,0.709298,0.673800,0.701603,0.597045,0.905748,0.915979,0.617221,0.567182,0.773657,0.647862,0.712839,0.846974,0.604959,0.861297,0.794886,0.554334,0.910586,0.571237,0.615031,0.624547,0.876397,0.902600,0.987466,0.520339,0.688482,0.555585,0.629575,0.584518,0.741682,0.583048,0.504346,0.532636,0.542888,0.623151,0.600516,0.840554,0.752566,0.912683,0.902901,0.866023,0.826644,0.612020,0.618835,0.723139,0.858559,0.660833,0.593293,0.641925,0.757006,0.935361,0.797139,0.864090,0.529002,0.503821,0.937301,0.672503,0.718872,0.780866,0.597374,0.607489,0.562950,0.681147,0.605597,0.639079,0.530455,0.845369,0.664009,0.690303,0.722610,0.884456,0.598942,0.751670,0.522942,0.619676,0.882877,0.577417,0.880086,0.587696,0.552771,0.792081,0.683724,0.931340,0.504968,0.938227,0.551686,0.570230,0.798123,0.503045,0.664594,0.656871,0.567401,0.612085,0.701374,0.519190,0.646064,0.835288,0.795865,0.724316,0.993900,0.607573,0.849724,0.997273,0.887558,0.613299,0.598762,0.979516,0.642393,0.812220,0.587138,0.588440,0.824383,0.856927,0.616001,0.634195,0.661657,0.597741,0.701252,0.559979,0.569491,0.691765,0.620046,0.610959,0.788201,0.853826,0.573900,0.665297,0.558946,0.947330,0.703870,0.586973,0.664554,0.622999,0.736716,0.542611,0.695847,0.903319,0.599733,0.541403,0.831317,0.977330,0.704392,0.576486,0.856579,0.655995,0.907617,0.752086,0.730215,0.936123,0.562392,0.765915,0.658113,0.560591,0.861414,0.698492,0.532813,0.505424,0.807790,0.586176,0.748718,0.501171,0.529915,0.763391,0.510786,0.678165,0.792158,0.971131,0.744016,0.601769,0.626346,0.964918,0.702361,0.500664,0.513710,0.779256,0.574104,0.633677,0.574520,0.522348,0.926332,0.656733,0.603527,0.607038,0.802441,0.878028,0.510788,0.899877,0.920924,0.518988,0.824570,0.502740,0.775012,0.622436,0.802007,0.514715,0.577805,0.892609,0.927842,0.501025,0.759873,0.527326,0.787753,0.555493,0.597539,0.900066,0.606560,0.564726,0.837835,0.805289,0.749516,0.527802,0.996586,0.748230,0.886556,0.641538,0.699470,0.890604,0.730793,0.789583,0.674770,0.879601,0.536830,0.804789,0.532576,0.746898,0.624952,0.743176,0.622723,0.537225,0.600017,0.515730,0.586268,0.907031,0.774724,0.737757,0.609426,0.630836,0.736058,0.505561,0.639638,0.571114,0.665348,0.522129,0.962189,0.980879,0.985835,0.976423,0.573519,0.882854,0.653846,0.911451,0.579944,0.509623,0.891063,0.542906,0.867386,0.535978,0.823943,0.502704,0.873774,0.618123,0.559131,0.820570,0.582463,0.636877,0.622019,0.722473,0.792715,0.746932,0.995336,0.605859,0.650811,0.608006,0.892361,0.760591,0.631238,0.784112,0.557774,0.567332,0.747537,0.747738,0.557074,0.648227,0.949921,0.527858,0.916803,0.914040,0.551715,0.553257,0.806064,0.810426,0.699413,0.633265,0.651288,0.607693,0.714544,0.620668,0.887568,0.967978,0.621418,0.654867,0.890060,0.883210,0.560911,0.925882,0.560081,0.551179,0.856080,0.824980,0.635669,0.677594,0.577193,0.801147,0.814570,0.554954,0.781254,0.546548,0.602344,0.650814,0.665870,0.872365,0.962416,0.959274,0.846559,0.980017,0.520162,0.868620,0.786744,0.912585,0.565840,0.588289,0.760543,0.858157,0.767373,0.627463,0.805537,0.816600,0.926516,0.966794,0.734947,0.989566,0.668298,0.590165,0.839151,0.507756,0.668435,0.632610,0.501010,0.516954,0.858246,0.935069,0.821209,0.982668,0.516325,0.736359,0.605903,0.898352,0.554739,0.925030,0.628923,0.694345,0.898362,0.755994,0.656751,0.552580,0.532583,0.644214,0.738877,0.776388,0.526944,0.844726,0.550493,0.515667,0.962965,0.950083,0.628112,0.666081,0.902720,0.556827,0.814731,0.808404,0.501220,0.564053,0.618742,0.773889,0.912097,0.620526,0.780170,0.656386,0.920203,0.633810,0.642327,0.504452,0.760573,0.691819,0.828159,0.648643,0.511671,0.630054,0.592250,0.572452,0.611795,0.853130,0.767857,0.803244,0.977559,0.637011,0.812853,0.687590,0.933577,0.537788,0.573612,0.867448,0.753871,0.893371,0.970312,0.671797,0.574914,0.681439,0.764636,0.568759,0.825038,0.744251,0.844906,0.896856,0.848703,0.949312,0.952169,0.551644,0.922527,0.605632,0.715088,0.825380,0.679452,0.501199,0.517814,0.555344,0.678142,0.686950,0.511173,0.623131,0.770302,0.874953,0.760712,0.984289,0.714458,0.509199,0.671047,0.887910,0.625711,0.504687,0.914575,0.813477,0.595948,0.626936,0.556167,0.988033,0.713675,0.722819,0.681335,0.663196,0.582542,0.555701,0.630556,0.667470,0.763603,0.722386,0.514648,0.893373,0.744990,0.866000,0.577401,0.767882,0.986483,0.533872,0.769178,0.757103,0.921029,0.578268,0.667310,0.846560,0.711200,0.595666,0.704175,0.984345,0.789939,0.601630,0.931843,0.795176,0.963952,0.986044,0.859394,0.769914,0.722864,0.646328,0.536825,0.522202,0.838180,0.717185,0.654856,0.694982,0.631812,0.558267,0.541740,0.831908,0.714662,0.734696,0.732807,0.901100,0.645397,0.704412,0.592432,0.923267,0.590038,0.983785,0.694431,0.966500,0.636938,0.988502,0.575287,0.763118,0.573206,0.686574,0.536539,0.558432,0.705054,0.689271,0.933615,0.571641,0.589522,0.652384,0.832367,0.677376,0.681758,0.765260,0.669155,0.919904,0.827689,0.842088,0.881507,0.669451,0.835810,0.921674,0.588850,0.930537,0.573017,0.604362,0.519932,0.792955,0.741997,0.612589,0.656914,0.522889,0.790293,0.940584,0.707690,0.784240,0.660622,0.981658,0.696395,0.638986,0.728662,0.602797,0.847992,0.670187,0.986568,0.543832,0.769118,0.600370,0.524375,0.895289,0.664344,0.585000,0.840934,0.517712,0.941836,0.911025,0.755612,0.795634,0.724436,0.716641,0.563844,0.814602,0.726244,0.648466,0.929938,0.566022,0.741295,0.809492,0.518140,0.996476,0.740893,0.800401,0.695274,0.765695,0.604894,0.644040,0.720671,0.672833,0.782528,0.939418,0.861290,0.593484,0.723639,0.845435,0.700228,0.662457,0.701139,0.715643,0.816431,0.778013,0.777753,0.934843,0.619195,0.604217,0.741449,0.896439,0.642311,0.956796,0.734709,0.543165,0.530472,0.529282,0.901945,0.686974,0.856870,0.603491,0.996925,0.819111,0.874764,0.613391,0.580148,0.503746,0.831862,0.762600,0.573336,0.662707,0.519838,0.964840,0.947635,0.785431,0.645472,0.542366,0.534648,0.809984,0.878806,0.781143,0.729606,0.509551,0.771064,0.707590,0.552170,0.523510,0.534125,0.792108,0.790945,0.539648,0.552340,0.607819,0.596086,0.735058,0.729101,0.716835,0.760717,0.823004,0.725685,0.729855,0.791387,0.569295,0.713830,0.971551,0.964946,0.515218,0.818450,0.613782,0.639322,0.548355,0.995822,0.851593,0.588010,0.918754,0.715647,0.735091,0.868172,0.548503,0.936189,0.589121,0.921727,0.611306,0.824551,0.803661,0.772861,0.661276,0.566211,0.549106,0.702373,0.914691,0.532663,0.815191,0.795662,0.535288,0.594291,0.503004,0.584799,0.501762,0.695143,0.956835,0.841954,0.783563,0.762761,0.625602,0.779535,0.905997,0.753714,0.660363,0.536616,0.520081,0.571861,0.969600,0.766632,0.783935,0.832014,0.708656,0.704004,0.739020,0.947174,0.571615,0.851102,0.787682,0.863594,0.597561,0.638423,0.795116,0.791801,0.805823,0.540431,0.884098,0.711055,0.511283,0.980106,0.690066,0.887023,0.518697,0.925754,0.920203,0.800403,0.617754,0.802436,0.502705,0.630736,0.656112,0.559891,0.546631,0.648119,0.530671,0.841626,0.784243,0.533393,0.797151,0.827299,0.537188,0.634083,0.895924,0.817609,0.715855,0.560232,0.515467,0.846161,0.676210,0.828768,0.583253,0.503894,0.534882,0.513893,0.536097,0.809973,0.748895,0.916232,0.692102,0.605492,0.729712,0.635098,0.730347,0.759175,0.582578,0.922413,0.575679,0.776235,0.589030,0.523719,0.758997,0.850897,0.716142,0.531322,0.667935,0.528473,0.589682,0.507148,0.920401,0.522798,0.931285,0.896363,0.566378,0.681869,0.559284,0.820106,0.595764,0.748448,0.557892,0.584946,0.686948,0.707208,0.565563,0.541156,0.757063,0.943204,0.796998,0.618892,0.726773,0.573147,0.581142,0.930637,0.838346,0.648609,0.509306,0.624778,0.570131,0.644147,0.803133,0.523877,0.666822,0.596197,0.909249,0.655023,0.715852,0.995154,0.688149,0.500884,0.673045,0.903521,0.506407,0.687173,0.581654,0.552293,0.608045,0.711095,0.817617,0.729938,0.546042,0.790341,0.989153,0.636034,0.652572,0.998487,0.976481,0.577199,0.866138,0.937551,0.987504,0.516145,0.862654,0.856197,0.594153,0.926652,0.711389,0.846429,0.703676,0.565031,0.834080,0.510284,0.546199,0.818750,0.779002,0.878454,0.598712,0.693525,0.993578,0.798442,0.808256,0.824244,0.975373,0.609827,0.511064,0.940273,0.846381,0.544938,0.584951,0.883289,0.731614,0.544038,0.617705,0.512818,0.663474,0.650680,0.684816,0.671346,0.554552,0.570929,0.700815,0.506752,0.632783,0.677756,0.778342,0.526537,0.830841,0.890641,0.935026,0.751061,0.662222,0.783248,0.753437,0.559008,0.562841,0.848974,0.851310,0.738167,0.667285,0.960948,0.873546,0.729443,0.592555,0.941249,0.745292,0.772211,0.561163,0.824234,0.733623,0.916648,0.867891,0.556269,0.616550,0.539736,0.905501,0.507367,0.661353,0.765503,0.526919,0.895461,0.800940,0.514371,0.525104,0.667796,0.801707,0.563362,0.822812,0.679997,0.914096,0.631084,0.817163,0.647241,0.752219,0.560569,0.930055,0.834736,0.520884,0.625024,0.543325,0.621299,0.877088,0.618402,0.735748,0.502211,0.664625,0.663070,0.592084,0.712748,0.965420,0.580138,0.701834,0.594536,0.762197,0.790649,0.538661,0.992411,0.584338,0.717152,0.931922,0.847092,0.612607,0.747159,0.789276,0.833404,0.738081,0.584440,0.868719,0.751519,0.816657,0.818875,0.640550,0.890810,0.698826,0.944877,0.584640,0.615767,0.980533,0.593867,0.793962,0.672081,0.874252,0.529105,0.810864,0.518002,0.958638,0.502057,0.917334,0.599973,0.628934,0.989873,0.663220,0.600942,0.684163,0.805066,0.782103,0.669933,0.706688,0.572238,0.925448,0.889566,0.738164,0.806295,0.685654,0.530402,0.743988,0.914269,0.792178,0.644023,0.716208,0.536182,0.507412,0.693128,0.568273,0.905650,0.813044,0.991052,0.683148,0.925816,0.671328,0.903051,0.694553,0.926293,0.945767,0.857548,0.571300,0.838779,0.613383,0.564771,0.819985,0.796975,0.508441,0.562602,0.709307,0.842429,0.738479,0.596694,0.935439,0.611024,0.863879,0.674015,0.593312,0.870915,0.592112,0.809536,0.545751,0.527561,0.955994,0.610053,0.533567,0.857786,0.506040,0.615511,0.792531,0.669659,0.863842,0.518302,0.570791,0.534796,0.998099,0.500737,0.811896,0.692687,0.542513,0.845770,0.897821,0.662616,0.762024,0.579293,0.805909,0.630673,0.544612,0.997462,0.542846,0.883759,0.504872,0.943012,0.726078,0.975128,0.976535,0.919034,0.610470,0.709504,0.777809,0.781708,0.813268,0.572767,0.663678,0.513508,0.713937,0.749949,0.711512,0.894823,0.991579,0.880106,0.762709,0.865610,0.951857,0.558750,0.558304,0.923324,0.808498,0.523339,0.682316,0.974865,0.746097,0.777043,0.548319,0.512660,0.669996,0.922692,0.535742,0.521703,0.637262,0.640653,0.839856,0.604149,0.900306,0.535248,0.758652,0.559566,0.575804,0.988342,0.807343,0.503999,0.764860,0.530762,0.627494,0.843222,0.713743,0.549141,0.768604,0.840751,0.570813,0.689754,0.643873,0.503292,0.737086,0.681105,0.926724,0.744176,0.563590,0.730313,0.801688,0.645069,0.940630,0.527844,0.759297,0.650633,0.773906,0.741698,0.950938,0.996453,0.766691,0.585911,0.915639,0.926730,0.909426,0.577285,0.658794,0.603036,0.520552,0.692172,0.543310,0.779506,0.618509,0.986604,0.760301,0.961254,0.830599,0.953030,0.541930,0.773382,0.981214,0.951369,0.699733,0.780053,0.581571,0.629805,0.586243,0.841237,0.669399,0.515333,0.964625,0.795985,0.534854,0.921469,0.831371,0.814430,0.579237,0.606824,0.697272,0.545011,0.630688,0.962503,0.777487,0.564272,0.954786,0.684601,0.812098,0.664588,0.956129,0.754850,0.507658,0.652507,0.588211,0.799644,0.835996,0.881497,0.790564,0.686533,0.948045,0.550375,0.522491,0.852227,0.843977,0.949739,0.858756,0.682459,0.632863,0.790467,0.536427,0.888238,0.649979,0.789175,0.545461,0.600018,0.847187,0.816264,0.584757,0.829891,0.761308,0.963173,0.847245,0.964584,0.671686,0.801393,0.610241,0.589611,0.628400,0.691734,0.575844,0.908351,0.726707,0.949400,0.567326,0.630131,0.652507,0.545050,0.625116,0.820674,0.886282,0.783389,0.844282,0.872118,0.511857,0.548439,0.985094,0.990749,0.672943,0.753010,0.771683,0.545495,0.741132,0.950366,0.569342,0.611619,0.963303,0.802776,0.960054,0.567637,0.534809,0.877646,0.563558,0.569817,0.630337,0.945647,0.512658,0.572804,0.568068,0.669196,0.560970,0.795417,0.630156,0.805944,0.711365,0.932515,0.913919,0.508272,0.742026,0.717258,0.537893,0.599257,0.955425,0.724357,0.801719,0.654062,0.838277,0.677526,0.550200,0.649493,0.656978,0.701823,0.620186,0.840498,0.621366,0.614860,0.994380,0.666967,0.558168,0.629195,0.526585,0.610277,0.810767,0.790826,0.602425,0.533721,0.758117,0.514064,0.764082,0.822564,0.567051,0.722774,0.808514,0.661737,0.605222,0.573738,0.868542,0.881263,0.573802,0.745094,0.521774,0.799955,0.972470,0.582704,0.830070,0.571810,0.974951,0.696390,0.723608,0.831990,0.513376,0.696564,0.563498,0.969434,0.705360,0.664412,0.618468,0.525846,0.521671,0.658248,0.813107,0.607097,0.511260,0.903615,0.957757,0.841863,0.876386,0.647548,0.937798,0.581232,0.558364,0.640371,0.888307,0.836950,0.518058,0.704168,0.512431,0.671421,0.695421,0.645942,0.762468,0.755845,0.521253,0.604079,0.564020,0.667574,0.626918,0.586115,0.751755,0.685064,0.755129,0.584941,0.739550,0.900325,0.830306,0.957085,0.512569,0.569123,0.693350,0.822129,0.585004,0.664623,0.711538,0.856079,0.880599,0.592103,0.634957,0.583258,0.767484,0.990206,0.826490,0.659648,0.530264,0.741673,0.504949,0.811954,0.876032,0.654255,0.766135,0.677520,0.559610,0.582553,0.810014,0.949324,0.850021,0.569475,0.907205,0.881121,0.664036,0.772071,0.925975,0.788126,0.988398,0.981062,0.831467,0.596221,0.980100,0.829146,0.664881,0.740300,0.875086,0.509139,0.626576,0.947405,0.738884,0.898763,0.674864,0.671608,0.651994,0.676151,0.894163,0.554325,0.594849,0.645789,0.553554,0.677751,0.926189,0.587111,0.898995,0.580089,0.630067,0.729369,0.802003,0.817686,0.603073,0.506778,0.599228,0.740794,0.672209,0.962957,0.903176,0.585807,0.800997,0.604428,0.587852,0.924836,0.689828,0.940054,0.670357,0.629518,0.523192,0.927452,0.804485,0.859688,0.716854,0.760826,0.992945,0.675351,0.903672,0.921728,0.675664,0.812096,0.792637,0.523208,0.887809,0.588286,0.763753,0.882605,0.542029,0.819541,0.553363,0.889609,0.732751,0.740620,0.623535,0.610706,0.672431,0.522146,0.633147,0.639742,0.583827,0.596593,0.661118,0.824514,0.685394,0.513829,0.946734,0.620501,0.849611,0.913148,0.505912,0.824887,0.667287,0.729099,0.667020,0.543656,0.535564,0.778349,0.628591,0.777491,0.852181,0.910923,0.935095,0.677714,0.731874,0.792442,0.628045,0.569207,0.972876,0.859780,0.965621,0.741341,0.567515,0.514272,0.559925,0.648669,0.555493,0.747381,0.917946,0.700593,0.972591,0.814834,0.959063,0.772875,0.857897,0.606929,0.711714,0.669363,0.522616,0.509161,0.969696,0.763739,0.937843,0.927172,0.676613,0.804684,0.776122,0.515339,0.838117,0.641166,0.809910,0.874857,0.592783,0.669597,0.562971,0.585259,0.612925,0.509746,0.589644,0.652251,0.777700,0.963499,0.589823,0.744631,0.713258,0.593509,0.785127,0.774852,0.695250,0.739914,0.581392,0.679543,0.761967,0.582322,0.658788,0.946152,0.770838,0.669988,0.667983,0.548030,0.700293,0.548073,0.659360,0.935427,0.609787,0.948389,0.739989,0.699261,0.920716,0.615826,0.705265,0.956499,0.994820,0.507988,0.677875,0.529380,0.509963,0.737149,0.561965,0.893009,0.618788,0.658694,0.784650,0.539401,0.819114,0.543091,0.878901,0.573276,0.570736,0.670666,0.601055,0.533710,0.576444,0.587707,0.788588,0.611004,0.675409,0.545044,0.725126,0.970239,0.557258,0.886280,0.823006,0.522563,0.595330,0.720009,0.642257,0.714072,0.760388,0.532192,0.700318,0.699484,0.695547,0.535063,0.579193,0.932221,0.857976,0.611557,0.726679,0.527907,0.634005,0.790209,0.866308,0.661703,0.833934,0.591358,0.666436,0.719085,0.676269,0.974973,0.896516,0.935602,0.656267,0.880531,0.944351,0.548110,0.643001,0.599064,0.607015,0.610301,0.551900,0.551499,0.733530,0.995832,0.525517,0.822901,0.506959,0.511310,0.833795,0.954765,0.562131,0.552978,0.751046,0.677941,0.713695,0.644156,0.535071,0.862458,0.971832,0.726509,0.567311,0.759257,0.846360,0.818170,0.962956,0.648915,0.516464,0.892380,0.920939,0.616270,0.645833,0.505403,0.726421,0.787753,0.853416,0.541565,0.828379,0.598100,0.527054,0.988394,0.892430,0.930749,0.552690,0.807182,0.507627,0.895175,0.584979,0.540734,0.675915,0.721309,0.720495,0.779033,0.615981,0.713130,0.868835,0.826071,0.935486,0.536669,0.673370,0.942912,0.994038,0.733512,0.548481,0.707502,0.622554,0.545123,0.573862,0.519910,0.645967,0.873917,0.754949,0.868087,0.640424,0.569749,0.903597,0.759299,0.512728,0.632406,0.532036,0.684580,0.690643,0.791029,0.505099,0.702367,0.965686,0.854271,0.731317,0.845652,0.931481,0.505008,0.850300,0.693339,0.570678,0.969774,0.798161,0.642472,0.561273,0.943079,0.737552,0.819524,0.827931,0.577195,0.681209,0.929293,0.857183,0.830402,0.981974,0.913397,0.546363,0.991508,0.764742,0.652054,0.577299,0.553361,0.779742,0.500783,0.763895,0.929006,0.677231,0.520737,0.726685,0.996574,0.913142,0.895445,0.817682,0.780836,0.679078,0.547914,0.627538,0.956667,0.718916,0.945008,0.755785,0.560330,0.876218,0.570243,0.668226,0.635441,0.606531,0.936135,0.714976,0.981092,0.642442,0.787257,0.913966,0.710586,0.510037,0.943594,0.994296,0.537089,0.546244,0.705041,0.579730,0.787997,0.839070,0.990617,0.966135,0.940795,0.577573,0.742571,0.669939,0.596807,0.756760,0.660204,0.775692,0.754699,0.856798,0.681870,0.769201,0.579181,0.538577,0.858476,0.626437,0.793425,0.501319,0.713266,0.753831,0.977386,0.795867,0.704985,0.519381,0.715664,0.540096,0.752474,0.924033,0.811387,0.588941,0.759824,0.803245,0.921426,0.597686,0.905971,0.912066,0.818612,0.953203,0.593574,0.815287,0.876614,0.769924,0.500044,0.614655,0.707312,0.858198,0.551576,0.638666,0.658349,0.798535,0.847358,0.901754,0.594429,0.634502,0.715071,0.917408,0.874328,0.837881,0.674123,0.884394,0.540987,0.620383,0.642457,0.595547,0.595574,0.753668,0.770844,0.660191,0.603622,0.787031,0.912797,0.601871,0.555911,0.874438,0.575015,0.657069,0.570886,0.672164,0.645704,0.569324,0.884489,0.506373,0.850975,0.849338,0.967288,0.504017,0.744447,0.723253,0.919413,0.876028,0.670259,0.800464,0.725290,0.971724,0.507987,0.531848,0.520288,0.725138,0.711625,0.530945,0.564667,0.542017,0.987205,0.784040,0.799382,0.560196,0.898234,0.743382,0.881410,0.662013,0.743656,0.714120,0.544848,0.634442,0.603752,0.644260,0.513521,0.585110,0.617299,0.555567,0.654289,0.589015,0.504530,0.579619,0.554223,0.797297,0.568603,0.639983,0.847220,0.569541,0.507058,0.595616,0.682234,0.530731,0.615569,0.593840,0.998962,0.821897,0.808062,0.663693,0.565687,0.584590,0.813622,0.641976,0.662210,0.740035,0.754626,0.540868,0.522191,0.913669,0.839373,0.775849,0.864922,0.654603,0.816976,0.906530,0.827605,0.643984,0.666205,0.983832,0.866123,0.694881,0.998147,0.634516,0.760145,0.789597,0.686578,0.639836,0.701344,0.888691,0.554886,0.697503,0.607975,0.504073,0.583037,0.566342,0.784580,0.559168,0.745396,0.820257,0.740193,0.733496,0.606297,0.867297,0.950851,0.559337,0.607164,0.599887,0.895107,0.536052,0.978872,0.524363,0.977266,0.760565,0.776423,0.515025,0.675477,0.786221,0.780123,0.655065,0.905001,0.715682,0.928184,0.509616,0.615659,0.587599,0.664556,0.711692,0.847424,0.566202,0.621996,0.802222,0.944974,0.820096,0.959715,0.617401,0.734346,0.613888,0.541120,0.660864,0.664252,0.661503,0.845896,0.519436,0.586115,0.849937,0.533735,0.997145,0.857758,0.689064,0.573177,0.734540,0.536208,0.601955,0.528915,0.722346,0.975757,0.589142,0.506454,0.552815,0.990233,0.619527,0.816355,0.910264,0.976690,0.952000,0.760411,0.934603,0.791151,0.892175,0.878397,0.817536,0.850283,0.889824,0.976865,0.555503,0.633895,0.534304,0.832021,0.522796,0.545179,0.543658,0.730114,0.581775,0.926679,0.757358,0.694059,0.633196,0.680687,0.888247,0.542841,0.997704,0.719114,0.608517,0.568532,0.743903,0.780911,0.618447,0.701478,0.913741,0.605189,0.639035,0.928978,0.804004,0.746509,0.673835,0.928209,0.523178,0.552926,0.626205,0.605621,0.537443,0.745820,0.592070,0.844203,0.997126,0.824756,0.567526,0.596079,0.677193,0.993654,0.564315,0.884754,0.931453,0.926750,0.783107,0.996047,0.939516,0.509974,0.767742,0.778620,0.580520,0.865568,0.860574,0.620675,0.838600,0.785441,0.771146,0.909119,0.898594,0.863945,0.580379,0.842371,0.652543,0.616149,0.904909,0.633209,0.509921,0.674529,0.748722,0.522135,0.712163,0.567084,0.723833,0.713751,0.519622,0.686944,0.945276,0.954940,0.549596,0.784703,0.510674,0.581575,0.813018,0.836544,0.646222,0.965097,0.714340,0.809505,0.711982,0.538824,0.556919,0.708667,0.867768,0.949889,0.711832,0.862047,0.550436,0.989956,0.931972,0.904772,0.600213,0.982915,0.817921,0.984239,0.501699,0.578157,0.876896,0.828889,0.770305,0.561775,0.747259,0.684883,0.926013,0.541553,0.921172,0.557689,0.544905,0.796640,0.594668,0.624226,0.844893,0.746903,0.715427,0.661805,0.603395,0.774726,0.699350,0.590618,0.592555,0.709403,0.586714,0.550831,0.517310,0.725511,0.531423,0.547672,0.691517,0.656381,0.812526,0.736568,0.545398,0.560139,0.855715,0.794367,0.884985,0.692018,0.647200,0.626600,0.981445,0.506642,0.650531,0.847614,0.869646,0.796813,0.711353,0.823704,0.589588,0.777397,0.876584,0.889914,0.531701,0.963290,0.861024,0.921236,0.529747,0.713110,0.600308,0.735805,0.509412,0.985852,0.674505,0.680098,0.652066,0.597328,0.901926,0.800450,0.747974,0.767182,0.819430,0.635456,0.574815,0.522848,0.606306,0.983266,0.650452,0.660653,0.861041,0.723710,0.573890,0.852714,0.707091,0.705901,0.528009,0.510856,0.556208,0.872260,0.705816,0.574141,0.533506,0.521962,0.501054,0.736913,0.610821,0.679981,0.580656,0.644565,0.921320,0.732191,0.570588,0.927323,0.753421,0.745972,0.876412,0.703431,0.779303,0.520012,0.584907,0.752741,0.538965,0.828275,0.886889,0.664816,0.980145,0.910218,0.831057,0.605491,0.911241,0.675984,0.575846,0.590376,0.605923,0.805020,0.891075,0.766312,0.552780,0.623953,0.954965,0.968790,0.520384,0.920212,0.857376,0.678804,0.570384,0.515219,0.630960,0.890796,0.610334,0.836735,0.937518,0.829881,0.929699,0.523890,0.985092,0.624640,0.734114,0.990989,0.692877,0.641799,0.548233,0.631637,0.914152,0.666997,0.629007,0.521774,0.568857,0.512392,0.836444,0.609969,0.555419,0.767893,0.910296,0.547895,0.879169,0.905911,0.861965,0.571830,0.910435,0.598505,0.579185,0.609398,0.714379,0.631827,0.779636,0.502825,0.900845,0.946936,0.956088,0.576796,0.963878,0.733009,0.751099,0.621213,0.681980,0.590763,0.525396,0.998203,0.591697,0.867456,0.700929,0.944628,0.875302,0.932461,0.646606,0.870017,0.800697,0.585610,0.900619,0.562201,0.899149,0.566319,0.541007,0.872132,0.923889,0.819025,0.874889,0.670856,0.781763,0.576150,0.669480,0.717041,0.755050,0.613519,0.867726,0.649442,0.513835,0.506665,0.600133,0.586694,0.645695,0.853754,0.729885,0.713864,0.794784,0.754156,0.537649,0.844148,0.731228,0.974146,0.542306,0.634609,0.512586,0.853929,0.934309,0.728563,0.519737,0.936788,0.602661,0.744470,0.636912,0.500984,0.758126,0.907246,0.568252,0.679532,0.500444,0.765161,0.735422,0.518931,0.949523,0.799157,0.957380,0.640287,0.532777,0.557939,0.619566,0.674982,0.998389,0.896388,0.833221,0.518561,0.552332,0.665616,0.701830,0.503602,0.675323,0.637970,0.592261,0.678172,0.587533,0.996467,0.535758,0.845782,0.550767,0.696916,0.651618,0.626743,0.507187,0.651496,0.687425,0.647088,0.926307,0.805841,0.645809,0.505107,0.828625,0.540494,0.535098,0.769920,0.507442,0.642566,0.675958,0.535678,0.549780,0.935674,0.543135,0.831569,0.695028,0.903621,0.564966,0.625263,0.984881,0.639310,0.504924,0.556750,0.553820,0.905563,0.986306,0.869224,0.809542,0.694077,0.848021,0.795371,0.779761,0.548068,0.632324,0.860051,0.532681,0.545756,0.603336,0.546705,0.903630,0.776499,0.601060,0.597137,0.669697,0.792308,0.992305,0.939844,0.661056,0.913354,0.573369,0.641453,0.636077,0.688542,0.806948,0.594143,0.632074,0.707053,0.738271,0.518928,0.538773,0.618995,0.590200,0.796221,0.552961,0.840944,0.641350,0.722470,0.948153,0.515471,0.949267,0.623965,0.551099,0.899190,0.585513,0.843720,0.546595,0.804313,0.578011,0.568570,0.542617,0.505626,0.831036,0.609327,0.525859,0.678140,0.774075,0.591988,0.937489,0.567213,0.703876,0.732052,0.770499,0.745657,0.538565,0.773898,0.919221,0.700330,0.749997,0.583970,0.943059,0.817947,0.760341,0.586072,0.595185,0.656753,0.904015,0.915434,0.522855,0.842240,0.518513,0.619611,0.782033,0.734672,0.863221,0.726071,0.512879,0.962890,0.755914,0.798384,0.577936,0.761516,0.905620,0.840895,0.673426,0.601519,0.500634,0.584005,0.574837,0.566546,0.748117,0.821367,0.957535,0.800483,0.528118,0.520010,0.711395,0.512241,0.573065,0.508392,0.754232,0.621676,0.746010,0.808630,0.619836,0.504926,0.679405,0.678283,0.947641,0.600227,0.964162,0.727345,0.748887,0.650419,0.804564,0.729897,0.609150,0.680826,0.593541,0.653149,0.937333,0.831637,0.910616,0.796190,0.615575,0.534491,0.559200,0.553410,0.584469,0.960770,0.654646,0.670185,0.636879,0.803070,0.877327,0.682990,0.701815,0.956147,0.511877,0.864065,0.653660,0.928322,0.530249,0.582562,0.557034,0.972267,0.857305,0.518377,0.583189,0.853307,0.596234,0.956992,0.810237,0.906086,0.607132,0.950221,0.591758,0.583129,0.607108,0.534349,0.610971,0.692550,0.562832,0.706092,0.500586,0.597586,0.515805,0.670802,0.556971,0.505996,0.776310,0.922621,0.765072,0.748845,0.604240,0.542476,0.833088,0.778208,0.943793,0.534798,0.516722,0.809132,0.640561,0.911820,0.718529,0.857134,0.989083,0.886107,0.576645,0.532334,0.623044,0.830608,0.797340,0.862184,0.560263,0.599765,0.855165,0.545082,0.517519,0.657867,0.994107,0.960396,0.609008,0.548763,0.678895,0.888725,0.858273,0.811329,0.569155,0.686319,0.885015,0.582811,0.516997,0.538352,0.777864,0.686984,0.733174,0.865822,0.640229,0.540369,0.511312,0.642166,0.549298,0.893115,0.611762,0.607545,0.607748,0.580346,0.715733,0.542024,0.686950,0.661205,0.738353,0.955051,0.560273,0.743225,0.939080,0.688985,0.663420,0.691807,0.749988,0.732600,0.779829,0.933181,0.812880,0.708833,0.536566,0.901288,0.815533,0.869900,0.504505,0.610531,0.591732,0.691965,0.968115,0.673833,0.618782,0.938810,0.815148,0.544521,0.737441,0.819419,0.792507,0.729546,0.519935,0.772089,0.999608,0.872112,0.637655,0.605082,0.608504,0.537000,0.783597,0.507979,0.734196,0.583504,0.557637,0.907410,0.542112,0.500776,0.638866,0.500199,0.604961,0.509025,0.534808,0.678069,0.589324,0.872511,0.619723,0.644792,0.787053,0.745095,0.900959,0.916313,0.716678,0.855436,0.727244,0.640560,0.853408,0.793272,0.806103,0.535107,0.502207,0.703762,0.886552,0.920118,0.742867,0.752505,0.994096,0.805021,0.764461,0.816844,0.640086,0.712920,0.879333,0.563596,0.616464,0.858898,0.520925,0.814398,0.811633,0.608329,0.861073,0.575944,0.667069,0.984609,0.779384,0.819055,0.514238,0.647318,0.994790,0.591117,0.598041,0.798256,0.982941,0.610551,0.684247,0.744867,0.875367,0.915385,0.635173,0.527286,0.584425,0.624296,0.701451,0.878589,0.716008,0.716210,0.662328,0.924498,0.979298,0.627572,0.934147,0.522418,0.931483,0.569909,0.654228,0.859124,0.796629,0.710153,0.700936,0.699350,0.542302,0.700566,0.582288,0.990474,0.664156,0.888170,0.832841,0.584215,0.512331,0.565804,0.564469,0.857440,0.771963,0.826961,0.513561,0.716123,0.540118,0.566947,0.652433,0.863442,0.594032,0.927096,0.605371,0.639453,0.700753,0.770129,0.762897,0.788207,0.966175,0.906036,0.807554,0.752617,0.828875,0.754953,0.976472,0.693897,0.613598,0.743194,0.981506,0.745907,0.671103,0.709442,0.546761,0.766129,0.984275,0.919098,0.825765,0.739057,0.561900,0.859566,0.619681,0.664784,0.564775,0.672191,0.753519,0.678625,0.835311,0.664696,0.731696,0.795590,0.698763,0.518520,0.566398,0.964043,0.509333,0.629683,0.628763,0.691329,0.724848,0.636902,0.756391,0.740132,0.539784,0.753590,0.746428,0.624170,0.570466,0.564870,0.923238,0.552054,0.873049,0.627658,0.684118,0.610850,0.598181,0.586873,0.562323,0.997431,0.873420,0.596066,0.724794,0.598019,0.749623,0.588095,0.767176,0.506187,0.820275,0.507361,0.547418,0.706944,0.929074,0.983109,0.974042,0.730045,0.719034,0.915020,0.594692,0.849784,0.576667,0.987013,0.539462,0.747930,0.902922,0.727059,0.612158,0.785308,0.537054,0.567183,0.570045,0.784817,0.526801,0.703555,0.583732,0.867591,0.721794,0.602711,0.871316,0.996933,0.991413,0.516419,0.562473,0.532084,0.564874,0.765441,0.755158,0.997775,0.655693,0.530899,0.653873,0.968098,0.509463,0.839845,0.661355,0.736012,0.522669,0.516685,0.749914,0.712308,0.652678,0.727276,0.611263,0.769737,0.551967,0.773584,0.511294,0.955978,0.582596,0.634558,0.709436,0.862858,0.619994,0.998660,0.645496,0.575972,0.611938,0.805731,0.607342,0.776938,0.724855,0.844138,0.919177,0.514300,0.568122,0.580478,0.574084,0.575790,0.527952,0.956526,0.559294,0.756229,0.688580,0.944492,0.565070,0.929566,0.832488,0.751471,0.517764,0.650339,0.648812,0.724621,0.970049,0.843229,0.713484,0.802671,0.514460,0.589786,0.509260,0.639452,0.977873,0.692653,0.748166,0.780838,0.795219,0.505040,0.762129,0.623844,0.699609,0.657991,0.716797,0.630844,0.928848,0.532742,0.528413,0.866460,0.561150,0.645681,0.836755,0.548712,0.847197,0.812734,0.901603,0.541812,0.610066,0.591000,0.642006,0.772005,0.554566,0.795861,0.534270,0.900442,0.581379,0.686950,0.752144,0.602521,0.648574,0.818194,0.792525,0.915929,0.913117,0.798694,0.899999,0.522319,0.570848,0.790092,0.613441,0.941189,0.814209,0.660269,0.732856,0.629514,0.614526,0.730686,0.534345,0.892220,0.933841,0.577302,0.742102,0.717702,0.547873,0.527969,0.596020,0.575530,0.949365,0.574280,0.929663,0.609610,0.933135,0.935951,0.683344,0.970738,0.836572,0.956066,0.707307,0.586881,0.718618,0.662447,0.681343,0.716811,0.658134,0.568033,0.843251,0.766652,0.859843,0.703952,0.841977,0.767368,0.772262,0.941832,0.800506,0.786477,0.562048,0.528106,0.516584,0.644394,0.684257,0.853861,0.804584,0.951967,0.664943,0.596150,0.869392,0.600477,0.678437,0.893399,0.507819,0.840764,0.707857,0.894429,0.589745,0.689967,0.782292,0.591311,0.536307,0.753839,0.776242,0.650866,0.620438,0.837932,0.655002,0.582169,0.534040,0.627726,0.511907,0.868091,0.697606,0.658929,0.758577,0.650817,0.669287,0.587732,0.568259,0.812007,0.833128,0.788014,0.590476,0.598019,0.904572,0.664329,0.822483,0.645754,0.816898,0.822713,0.778708,0.572970,0.726209,0.653781,0.933437,0.633104,0.887305,0.647973,0.908090,0.522342,0.703282,0.604050,0.920374,0.527308,0.769640,0.895523,0.524289,0.641869,0.583217,0.817396,0.983869,0.501193,0.790772,0.743065,0.691086,0.597943,0.980851,0.831538,0.665782,0.638866,0.600973,0.552304,0.516726,0.772771,0.644880,0.546274,0.630044,0.938729,0.931867,0.922410,0.901613,0.568881,0.648307,0.763372,0.592094,0.575429,0.881482,0.839082,0.556612,0.642611,0.992993,0.793054,0.787415,0.652623,0.552740,0.956644,0.872940,0.916246,0.618950,0.994678,0.893927,0.653370,0.868354,0.989396,0.972723,0.519347,0.994054,0.513829,0.881279,0.913673,0.592663,0.936641,0.968462,0.517731,0.541250,0.802611,0.717408,0.926090,0.596204,0.574584,0.853712,0.860227,0.623819,0.585754,0.871491,0.703998,0.721397,0.506342,0.569465,0.639888,0.996212,0.888356,0.593835,0.620276,0.519691,0.698661,0.907637,0.853877,0.504246,0.688093,0.835332,0.548434,0.564201,0.637446,0.827482,0.566638,0.581180,0.888194,0.840167,0.549564,0.699470,0.511723,0.702209,0.841045,0.585441,0.753624,0.681925,0.517167,0.511383,0.907418,0.646776,0.709070,0.998452,0.564685,0.877075,0.786830,0.861392,0.761505,0.530373,0.627104,0.573889,0.955873,0.876233,0.568686,0.794580,0.541005,0.679358,0.885283,0.515750,0.583409,0.555353,0.764454,0.520517,0.879512,0.567333,0.673240,0.834759,0.800820,0.595298,0.540247,0.594655,0.796139,0.551200,0.885226,0.702180,0.747386,0.810625,0.849648,0.570133,0.511809,0.640703,0.692266,0.570486,0.586101,0.773229,0.679996,0.774477,0.564693,0.765324,0.525917,0.654868,0.517284,0.625803,0.664955,0.697762,0.720588,0.724268,0.925688,0.834938,0.775574,0.994034,0.811344,0.807284,0.991897,0.943894,0.657463,0.512622,0.829749,0.991600,0.862391,0.657587,0.584737,0.563287,0.846788,0.572954,0.503287,0.551186,0.970908,0.802517,0.659771,0.560779,0.512910,0.653619,0.818403,0.756429,0.718515,0.515820,0.703316,0.796982,0.709871,0.517650,0.994663,0.915019,0.854542,0.894024,0.904693,0.539540,0.901531,0.905037,0.808099,0.626652,0.929849,0.626860,0.510117,0.526828,0.684423,0.554237,0.592031,0.584070,0.675606,0.597442,0.835228,0.517158,0.643584,0.672261,0.861994,0.688372,0.776893,0.759997,0.634003,0.529642,0.943854,0.828276,0.534548,0.791426,0.737610,0.512276,0.649822,0.836294,0.778367,0.896606,0.658312,0.553821,0.850939,0.641979,0.525574,0.971052,0.741453,0.835095,0.692444,0.640108,0.745393,0.664306,0.597138,0.751886,0.982659,0.582017,0.536611,0.530587,0.622760,0.845203,0.773943,0.721110,0.575520,0.538903,0.716389,0.624502,0.570886,0.599981,0.640966,0.901882,0.576076,0.690823,0.523499,0.528461,0.894090,0.657448,0.831220,0.661010,0.603349,0.648814,0.505572,0.799769,0.543244,0.514170,0.585516,0.779370,0.501124,0.792748,0.753786,0.832793,0.672809,0.943261,0.684224,0.521557,0.787720,0.732756,0.976909,0.949560,0.686320,0.556468,0.936084,0.998206,0.640754,0.672463,0.689237,0.646134,0.936023,0.739088,0.832361,0.567859,0.565626,0.523284,0.868640,0.580899,0.921092,0.865663,0.719644,0.724588,0.665118,0.543287,0.523065,0.762831,0.597224,0.576843,0.684350,0.955384,0.636666,0.676602,0.693418,0.765377,0.628411,0.864324,0.599131,0.750699,0.707922,0.953722,0.538215,0.808435,0.901138,0.873333,0.910395,0.621412,0.534400,0.535360,0.632967,0.995534,0.788185,0.502101,0.922630,0.747867,0.919959,0.907577,0.633191,0.550903,0.646458,0.629290,0.870621,0.872979,0.539642,0.660053,0.598051,0.941666,0.650106,0.556910,0.514176,0.758368,0.941234,0.667613,0.518680,0.915863,0.693221,0.799000,0.548971,0.525108,0.959043,0.715195,0.717484,0.972548,0.567228,0.576837,0.578105,0.758961,0.506575,0.603764,0.713254,0.984006,0.690644,0.958502,0.719935,0.520365,0.895156,0.601730,0.842185,0.563722,0.530446,0.707666,0.636588,0.942544,0.710231,0.726850,0.719597,0.584203,0.739767,0.680781,0.727663,0.645470,0.790693,0.608717,0.576496,0.509278,0.506740,0.671136,0.703962,0.920845,0.853925,0.822413,0.580601,0.673314,0.740317,0.544603,0.859029,0.822409,0.860752,0.799300,0.750887,0.707170,0.517991,0.794421,0.557041,0.676321,0.554253,0.615605,0.589809,0.814068,0.504221,0.671235,0.562987,0.718994,0.517871,0.977337,0.534106,0.829973,0.820673,0.515042,0.819345,0.576179,0.691452,0.792485,0.919869,0.594382,0.505481,0.711781,0.966622,0.654825,0.685986,0.667013,0.714571,0.976913,0.725741,0.816605,0.506838,0.789156,0.609057,0.577720,0.738367,0.550799,0.914941,0.819393,0.664602,0.570064,0.888462,0.676202,0.733677,0.788244,0.807913,0.735473,0.708832,0.722424,0.595908,0.535284,0.680940,0.630502,0.548385,0.982358,0.802587,0.653890,0.528302,0.970141,0.841688,0.818794,0.884059,0.877743,0.759497,0.955210,0.803317,0.784601,0.965509,0.513589,0.652943,0.682399,0.722262,0.647648,0.707360,0.928946,0.605210,0.916239,0.582574,0.655763,0.706764,0.520802,0.958577,0.643899,0.577092,0.741681,0.640695,0.984912,0.595516,0.649358,0.906628,0.879049,0.604142,0.540894,0.760354,0.900376,0.774237,0.753704,0.683374,0.929470,0.728538,0.525900,0.857736,0.538638,0.922666,0.766573,0.689924,0.556576,0.555895,0.559094,0.657502,0.584330,0.751016,0.749353,0.853098,0.539295,0.829022,0.610677,0.733901,0.850450,0.504753,0.554719,0.805981,0.546831,0.693925,0.769473,0.511728,0.845762,0.541788,0.903117,0.744118,0.552526,0.650464,0.525112,0.778631,0.785329,0.789559,0.651778,0.514215,0.608547,0.702911,0.766575,0.634663,0.526414,0.640864,0.615011,0.963022,0.624140,0.637129,0.569127,0.925030,0.585467,0.586943,0.847197,0.599184,0.578127,0.727041,0.820332,0.852213,0.625661,0.848768,0.691507,0.941224,0.891197,0.627558,0.791214,0.536111,0.658478,0.538326,0.591082,0.789639,0.783883,0.629730,0.509989,0.846045,0.879798,0.565297,0.546904,0.563602,0.843299,0.556271,0.907298,0.735969,0.769401,0.779191,0.921603,0.620985,0.934690,0.752074,0.982551,0.780178,0.744929,0.501387,0.851876,0.867122,0.534106,0.603322,0.897668,0.924535,0.659916,0.560686,0.879714,0.695188,0.560652,0.532837,0.835995,0.509838,0.626510,0.536753,0.887196,0.656019,0.502808,0.515930,0.505509,0.622747,0.591734,0.852214,0.554346,0.512667,0.967645,0.820416,0.585437,0.601257,0.571524,0.758070,0.944312,0.811723,0.554090,0.753232,0.618937,0.820112,0.895908,0.893419,0.719373,0.680177,0.642176,0.651426,0.531357,0.522966,0.655873,0.651556,0.995616,0.667052,0.518145,0.790183,0.839425,0.755706,0.952325,0.625806,0.511590,0.786187,0.501688,0.718104,0.798235,0.608319,0.559247,0.666355,0.613224,0.820705,0.735982,0.748550,0.542152,0.958931,0.708324,0.830900,0.665138,0.984805,0.789104,0.959106,0.597751,0.549106,0.911123,0.978554,0.540741,0.509326,0.591851,0.768523,0.939595,0.954378,0.678015,0.970581,0.822161,0.975662,0.592235,0.537943,0.806334,0.585824,0.646677,0.671146,0.820290,0.656744,0.506451,0.664269,0.950436,0.511592,0.955036,0.960909,0.692345,0.795330,0.565647,0.606465,0.627568,0.981791,0.664140,0.524617,0.701901,0.688071,0.877008,0.803566,0.814768,0.586732,0.535795,0.745429,0.742834,0.510465,0.773837,0.979101,0.954799,0.657725,0.749651,0.928115,0.781977,0.853242,0.800695,0.751907,0.593453,0.691295,0.591447,0.523981,0.669929,0.712024,0.540918,0.539880,0.654689,0.963070,0.563725,0.851003,0.689685,0.784630,0.650280,0.535716,0.865311,0.616551,0.965134,0.509125,0.573782,0.603565,0.665112,0.814786,0.793069,0.540696,0.633900,0.666592,0.796688,0.754264,0.641133,0.606923,0.544403,0.815461,0.934353,0.921859,0.980313,0.654409,0.633853,0.781533,0.613423,0.769545,0.544974,0.595209,0.567024,0.987752,0.509312,0.844367,0.596309,0.570907,0.577886,0.971780,0.587065,0.848131,0.914958,0.507619,0.588698,0.853058,0.952478,0.500567,0.598595,0.994885,0.689442,0.521357,0.550349,0.993103,0.828645,0.674893,0.947074,0.670328,0.877253,0.670945,0.584093,0.813829,0.541609,0.510275,0.881665,0.511882,0.694230,0.504823,0.904612,0.768013,0.634666,0.811408,0.571701,0.555759,0.994834,0.912671,0.527528,0.735082,0.672333,0.824759,0.551969,0.551470,0.839960,0.893956,0.591859,0.688416,0.545463,0.863951,0.972678,0.541780,0.501690,0.685363,0.898034,0.688284,0.633686,0.942671,0.558687,0.964561,0.614255,0.794104,0.740145,0.528009,0.574141,0.670862,0.554800,0.761757,0.873808,0.537109,0.812930,0.847042,0.560973,0.970346,0.600047,0.877434,0.579988,0.588108,0.662907,0.597131,0.580713,0.625020,0.557073,0.986750,0.754717,0.520548,0.554922,0.571256,0.738519,0.551455,0.568513,0.910664,0.845472,0.673165,0.519851,0.843394,0.523101,0.622554,0.690769,0.598639,0.987716,0.526464,0.513631,0.501749,0.738579,0.529628,0.563036,0.594630,0.582659,0.865778,0.542508,0.705743,0.964473,0.625513,0.641992,0.772268,0.564346,0.638781,0.518296,0.855675,0.704274,0.503598,0.551968,0.921308,0.776650,0.846990,0.537423,0.595136,0.792875,0.513443,0.776044,0.840006,0.878546,0.805291,0.888021,0.868013,0.835350,0.548918,0.854311,0.944015,0.882145,0.563431,0.800740,0.508480,0.635339,0.739733,0.687591,0.775064,0.865222,0.778084,0.608267,0.906087,0.739144,0.577113,0.850238,0.723505,0.643606,0.655789,0.637058,0.701792,0.869078,0.574417,0.532296,0.548077,0.657327,0.769948,0.543100,0.628325,0.729417,0.991757,0.569861,0.662674,0.639544,0.567568,0.521123,0.784326,0.546282,0.781037,0.658207,0.534997,0.783326,0.889166,0.924513,0.502149,0.538964,0.815658,0.723779,0.503034,0.736788,0.791889,0.580107,0.673167,0.746174,0.708779,0.915280,0.529388,0.913727,0.615578,0.977091,0.777676,0.562557,0.603584,0.686694,0.705352,0.601589,0.533340,0.626288,0.978605,0.558041,0.876779,0.776600,0.579998,0.676346,0.724453,0.519269,0.831281,0.601656,0.754510,0.578148,0.669000,0.533016,0.583623,0.658569,0.566934,0.846059,0.779116,0.746367,0.543027,0.992637,0.754989,0.981883,0.523933,0.697991,0.543301,0.552975,0.914961,0.546330,0.501470,0.596596,0.524482,0.653467,0.849454,0.521885,0.554390,0.663264,0.688448,0.553121,0.997060,0.612336,0.961522,0.506692,0.658701,0.716226,0.859751,0.573759,0.627533,0.757422,0.546061,0.592960,0.717055,0.723465,0.525009,0.872149,0.559244,0.887589,0.755645,0.658778,0.743100,0.757326,0.757846,0.609230,0.985601,0.788106,0.519093,0.791445,0.726777,0.631896,0.836330,0.973800,0.993777,0.697162,0.798884,0.759512,0.625056,0.658552,0.819736,0.891913,0.733467,0.544289,0.781031,0.941382,0.767150,0.511585,0.979383,0.973453,0.517009,0.934115,0.516334,0.517392,0.528331,0.605811,0.780156,0.544138,0.581161,0.937294,0.608825,0.626076,0.629266,0.680434,0.635234,0.528616,0.588036,0.949912,0.805446,0.540164,0.921768,0.984133,0.720428,0.510247,0.518345,0.833947,0.747667,0.874761,0.744297,0.536223,0.991044,0.572023,0.841311,0.649376,0.956987,0.853689,0.811675,0.822716,0.898869,0.531866,0.731365,0.866532,0.546850,0.937941,0.935905,0.906476,0.731864,0.593406,0.892456,0.697103,0.583999,0.857888,0.664393,0.974828,0.807463,0.997645,0.668188,0.521947,0.940431,0.558937,0.810854,0.878903,0.918585,0.841008,0.988223,0.623194,0.551941,0.820933,0.590883,0.510377,0.709258,0.573614,0.620206,0.558224,0.556749,0.681571,0.683110,0.812499,0.793884,0.772041,0.551377,0.526430,0.737482,0.534008,0.525652,0.606270,0.796720,0.905705,0.546985,0.706483,0.993718,0.936230,0.749023,0.900784,0.716180,0.531636,0.616201,0.590953,0.825566,0.568627,0.970331,0.886014,0.824972,0.671418,0.996006,0.655468,0.804979,0.770390,0.899023,0.737403,0.835108,0.796196,0.844554,0.579568,0.593331,0.942646,0.650620,0.511368,0.529156,0.801562,0.507687,0.897328,0.861281,0.649071,0.844523,0.855461,0.803194,0.875645,0.504134,0.642563,0.502422,0.636483,0.610993,0.675066,0.845496,0.510067,0.686558,0.510527,0.874054,0.613790,0.575910,0.540233,0.641555,0.987322,0.680273,0.697702,0.806276,0.516486,0.555593,0.915399,0.672122,0.884491,0.503395,0.974491,0.738843,0.790792,0.510301,0.693442,0.585563,0.718237,0.554150,0.834180,0.517501,0.554713,0.866581,0.544046,0.573452,0.692370,0.863160,0.872659,0.592778,0.775783,0.862082,0.647548,0.908044,0.851950,0.757757,0.704899,0.833221,0.693650,0.847179,0.642036,0.816687,0.934495,0.722139,0.802059,0.802361,0.519495,0.530349,0.679605,0.913116,0.771578,0.752094,0.739061,0.570549,0.653674,0.837758,0.750303,0.644323,0.637127,0.699057,0.509763,0.603065,0.923102,0.754957,0.859276,0.878384,0.756949,0.571167,0.621138,0.805274,0.683582,0.694341,0.542913,0.598867,0.674017,0.513904,0.832731,0.716123,0.709693,0.652620,0.828061,0.682651,0.797351,0.797145,0.864813,0.885815,0.897490,0.802265,0.586151,0.827706,0.500051,0.722592,0.675856,0.552155,0.516218,0.812120,0.799988,0.802545,0.559879,0.961745,0.575330,0.561541,0.623595,0.816250,0.832238,0.642515,0.654844,0.595405,0.579190,0.881963,0.537343,0.522032,0.526546,0.695669,0.736223,0.850075,0.712316,0.584219,0.817464,0.864344,0.633729,0.710802,0.536295,0.943028,0.728656,0.759773,0.830004,0.687208,0.558382,0.645108,0.574658,0.915998,0.513363,0.542715,0.516435,0.610521,0.683951,0.648808,0.773850,0.758449,0.663621,0.991528,0.733351,0.781144,0.917293,0.741588,0.812484,0.502497,0.661446,0.790325,0.809505,0.680103,0.669009,0.872601,0.693826,0.537270,0.976226,0.677208,0.561476,0.719396,0.666680,0.652938,0.576826,0.792412,0.914218,0.569879,0.940044,0.597469,0.681981,0.786092,0.523908,0.727741,0.674424,0.976086,0.510650,0.570687,0.669007,0.558759,0.508094,0.602433,0.678015,0.893753,0.929703,0.636093,0.532703,0.828170,0.519082,0.552239,0.547878,0.701855,0.725624,0.887341,0.961715,0.510792,0.645649,0.602595,0.562146,0.825282,0.714374,0.678038,0.710720,0.992824,0.711658,0.664369,0.804032,0.805019,0.572398,0.747197,0.770139,0.541808,0.608269,0.867779,0.513428,0.559180,0.902469,0.779981,0.542897,0.772075,0.994739,0.508500,0.608825,0.663356,0.776482,0.961255,0.855761,0.649339,0.551294,0.740675,0.643461,0.715404,0.850624,0.646835,0.605706,0.710514,0.549390,0.619909,0.944544,0.653021,0.521050,0.508560,0.826137,0.998272,0.629768,0.604382,0.567127,0.729046,0.638960,0.541105,0.720256,0.784436,0.939745,0.923380,0.893308,0.692107,0.681491,0.696999,0.905217,0.861531,0.532469,0.520833,0.990818,0.571516,0.581867,0.877366,0.836366,0.950981,0.516334,0.755344,0.951429,0.715872,0.946513,0.559793,0.879511,0.753393,0.522394,0.954702,0.569136,0.946254,0.551865,0.730093,0.638956,0.837348,0.908429,0.674373,0.813161,0.933297,0.731052,0.760894,0.581798,0.577627,0.980918,0.782793,0.587540,0.501732,0.597089,0.562998,0.539410,0.665398,0.535865,0.544262,0.764349,0.529212,0.791410,0.841856,0.642653,0.505785,0.502970,0.683058,0.586955,0.889638,0.643132,0.604521,0.970226,0.742909,0.589394,0.657283,0.743569,0.756374,0.755395,0.603828,0.640732,0.998112,0.689277,0.729321,0.849542,0.590013,0.508213,0.673762,0.526672,0.696962,0.656317,0.706134,0.654771,0.538441,0.820907,0.824795,0.587558,0.706244,0.544868,0.538611,0.975015,0.929142,0.755065,0.667545,0.521780,0.683270,0.697068,0.552206,0.524083,0.517092,0.536699,0.616250,0.518893,0.739848,0.708736,0.832118,0.654694,0.594878,0.698828,0.591074,0.522038,0.521034,0.932671,0.812662,0.605788,0.791795,0.699269,0.722147,0.627107,0.849721,0.620221,0.737855,0.719890,0.738599,0.506612,0.639823,0.699545,0.762780,0.526431,0.726968,0.593298,0.710075,0.643868,0.690746,0.610168,0.592116,0.511553,0.846168,0.910580,0.561468,0.562758,0.514541,0.640671,0.588674,0.901803,0.969300,0.557635,0.831512,0.850278,0.754512,0.538939,0.932870,0.603331,0.573214,0.522839,0.868462,0.897014,0.806727,0.681117,0.602414,0.551141,0.545271,0.539262,0.982869,0.531140,0.609197,0.719595,0.893810,0.801685,0.651000,0.757649,0.514182,0.872734,0.556082,0.548954,0.571021,0.742183,0.773949,0.776650,0.922810,0.560186,0.818335,0.939459,0.690663,0.551190,0.729164,0.514250,0.737406,0.995697,0.572856,0.736620,0.551522,0.731851,0.951263,0.806709,0.526593,0.657548,0.509599,0.722452,0.998758,0.651652,0.915200,0.595612,0.781031,0.809418,0.801259,0.615761,0.630582,0.532057,0.632876,0.811707,0.915967,0.589237,0.973849,0.527962,0.576973,0.555554,0.773950,0.585989,0.610808,0.679071,0.883611,0.677763,0.658193,0.947070,0.643314,0.717288,0.754304,0.543173,0.549304,0.595338,0.854583,0.537364,0.500817,0.915142,0.528938,0.826293,0.641547,0.583277,0.710587,0.692118,0.502741,0.786837,0.532451,0.842316,0.855105,0.846303,0.552904,0.725896,0.630698,0.519669,0.869807,0.774384,0.586298,0.772340,0.966879,0.974980,0.623998,0.792377,0.758512,0.734531,0.980308,0.559520,0.578400,0.797863,0.691882,0.826666,0.719801,0.855467,0.909618,0.519803,0.619681,0.565767,0.569347,0.636934,0.812589,0.617350,0.673375,0.721900,0.971785,0.708588,0.587516,0.863446,0.601015,0.654972,0.696809,0.582666,0.575645,0.910978,0.773092,0.614648,0.873425,0.736815,0.981468,0.508282,0.586380,0.920301,0.630272,0.826426,0.931093,0.754133,0.619336,0.506840,0.559789,0.621685,0.687914,0.921095,0.710790,0.833109,0.553858,0.899205,0.753651,0.703960,0.504891,0.756162,0.607050,0.718726,0.630267,0.887040,0.566303,0.522364,0.871261,0.710370,0.781152,0.906601,0.692843,0.634771,0.671920,0.972982,0.585325,0.569149,0.801673,0.507461,0.954769,0.720589,0.880921,0.508057,0.704246,0.544326,0.597169,0.727600,0.697322,0.909581,0.933227,0.517479,0.520838,0.677432,0.562682,0.685549,0.532909,0.632742,0.707024,0.740781,0.535559,0.916608,0.564452,0.981444,0.684123,0.707582,0.946597,0.514625,0.798438,0.689149,0.846416,0.926405,0.585695,0.633003,0.729189,0.833686,0.829165,0.543878,0.631400,0.597646,0.736771,0.622140,0.536209,0.607671,0.660473,0.658006,0.993598,0.865988,0.613933,0.898786,0.548622,0.548382,0.650892,0.656378,0.885992,0.529412,0.560341,0.535084,0.912178,0.537124,0.710926,0.511684,0.812466,0.935328,0.574646,0.751693,0.809376,0.868463,0.625155,0.773151,0.714124,0.605980,0.817419,0.850320,0.510111,0.760575,0.897752,0.521316,0.777571,0.590699,0.615322,0.968345,0.576601,0.507771,0.695867,0.600894,0.618397,0.523798,0.755256,0.534705,0.736480,0.795605,0.658717,0.785222,0.762176,0.574308,0.895687,0.626608,0.565067,0.834399,0.766534,0.602714,0.874883,0.995365,0.690536,0.676708,0.589876,0.668300,0.567977,0.777392,0.644831,0.534448,0.617775,0.533126,0.584634,0.691196,0.920305,0.609523,0.874683,0.708153,0.853968,0.754508,0.740006,0.553178,0.876902,0.996070,0.541051,0.515394,0.912314,0.548198,0.954933,0.694873,0.667925,0.960690,0.602207,0.598370,0.822358,0.640801,0.844833,0.742328,0.844019,0.755439,0.650223,0.991238,0.871403,0.798602,0.633724,0.544017,0.775922,0.866009,0.772136,0.871613,0.694493,0.523546,0.855935,0.792701,0.742838,0.696122,0.912944,0.754674,0.873528,0.736644,0.643278,0.885581,0.844596,0.710682,0.824637,0.717381,0.775028,0.854000,0.880536,0.738665,0.750623,0.868874,0.681087,0.509487,0.595440,0.612051,0.559839,0.839078,0.707770,0.825205,0.505598,0.545051,0.844360,0.827790,0.784469,0.618587,0.541276,0.756535,0.537173,0.508350,0.684960,0.585571,0.590560,0.614851,0.568496,0.620226,0.550350,0.561708,0.772357,0.799193,0.960240,0.901462,0.881594,0.669344,0.928728,0.805475,0.503197,0.782717,0.898265,0.827687,0.987576,0.585304,0.643699,0.957884,0.543592,0.733479,0.987695,0.521822,0.971892,0.852461,0.509785,0.793448,0.749384,0.701428,0.991589,0.750770,0.821885,0.734826,0.951747,0.918460,0.568818,0.988408,0.755151,0.786513,0.605645,0.655681,0.516581,0.997840,0.723773,0.551735,0.842871,0.694938,0.547518,0.503389,0.708672,0.761691,0.605536,0.526722,0.522715,0.751234,0.951921,0.708808,0.529503,0.955133,0.511660,0.601114,0.502888,0.869190,0.736323,0.640056,0.815241,0.646951,0.534506,0.720571,0.927548,0.597129,0.990817,0.686143,0.510154,0.519094,0.767843,0.953716,0.559939,0.559032,0.863063,0.578498,0.839484,0.634477,0.503706,0.999982,0.736545,0.932891,0.522616,0.908533,0.531780,0.842540,0.831777,0.584540,0.949665,0.782712,0.974116,0.633900,0.859465,0.827999,0.981802,0.789868,0.997084,0.551687,0.782113,0.901313,0.644274,0.765089,0.501776,0.562205,0.696368,0.704879,0.757343,0.530999,0.893684,0.570843,0.639991,0.526093,0.923774,0.747004,0.831162,0.996636,0.740328,0.883403,0.698190,0.669865,0.825329,0.615726,0.978939,0.544666,0.607050,0.531806,0.541318,0.530917,0.769201,0.799153,0.500053,0.888597,0.648978,0.739956,0.674201,0.707737,0.515520,0.804519,0.714543,0.862303,0.523054,0.744522,0.944788,0.557218,0.508573,0.835685,0.972999,0.999104,0.621478,0.899015,0.749149,0.852944,0.518445,0.838402,0.933675,0.926568,0.820206,0.735323,0.704311,0.625339,0.951294,0.827280,0.797728,0.675719,0.644925,0.951784,0.521341,0.548208,0.584593,0.761760,0.832772,0.984368,0.980188,0.658032,0.765256,0.637662,0.629930,0.862773,0.860885,0.791990,0.505578,0.554254,0.583720,0.571220,0.597957,0.593226,0.746206,0.642333,0.762822,0.593192,0.773213,0.521037,0.733147,0.526163,0.520424,0.659938,0.676735,0.764329,0.526263,0.933829,0.588476,0.592131,0.770888,0.871327,0.990462,0.796595,0.567680,0.735786,0.687423,0.773013,0.514152,0.766985,0.987181,0.674907,0.842533,0.766720,0.515584,0.694631,0.550444,0.896061,0.816929,0.796234,0.689412,0.652546,0.505444,0.579894,0.606544,0.804889,0.869103,0.544146,0.845217,0.969947,0.842121,0.990227,0.610498,0.838033,0.571005,0.575754,0.921690,0.586028,0.755491,0.597600,0.657538,0.519731,0.635062,0.596456,0.638090,0.507328,0.827361,0.666335,0.517087,0.649489,0.647307,0.671169,0.869074,0.572877,0.502058,0.696546,0.636138,0.506707,0.521815,0.535984,0.625662,0.654805,0.937884,0.566246,0.916130,0.855074,0.630212,0.678481,0.667853,0.575130,0.771752,0.960478,0.631401,0.909769,0.897858,0.994784,0.695056,0.541018,0.616891,0.850094,0.531591,0.993900,0.599032,0.835446,0.822199,0.989910,0.931894,0.632638,0.534147,0.511930,0.667653,0.701962,0.517804,0.713052,0.593686,0.915901,0.799743,0.637696,0.887687,0.518618,0.667699,0.504392,0.989027,0.755247,0.507035,0.588080,0.570632,0.824169,0.684690,0.613710,0.796068,0.717462,0.539233,0.670027,0.625508,0.571625,0.546037,0.541209,0.954967,0.603349,0.531441,0.898625,0.612724,0.664820,0.547059,0.525745,0.584947,0.559556,0.550702,0.665660,0.720052,0.649888,0.723310,0.687752,0.764790,0.548493,0.699985,0.934859,0.927048,0.970871,0.684912,0.544493,0.532972,0.649644,0.799985,0.628643,0.543038,0.589438,0.704377,0.694331,0.833837,0.638244,0.762190,0.817400,0.504710,0.879470,0.621307,0.550830,0.589284,0.708913,0.620884,0.504348,0.639654,0.532875,0.610197,0.994898,0.588430,0.847008,0.514385,0.648444,0.527207,0.977908,0.896678,0.683831,0.602155,0.615183,0.517513,0.700810,0.973818,0.731839,0.619314,0.930762,0.530528,0.706565,0.655981,0.796373,0.753699,0.633826,0.605561,0.533239,0.514420,0.769771,0.507440,0.636387,0.585059,0.541371,0.528105,0.662926,0.674438,0.745438,0.521926,0.580132,0.762384,0.847495,0.648271,0.917568,0.914922,0.583701,0.569042,0.700507,0.658303,0.579379,0.510790,0.523399,0.732905,0.925892,0.786286,0.663303,0.673170,0.904323,0.676849,0.809418,0.833083,0.619561,0.623331,0.555919,0.749539,0.692165,0.613517,0.907137,0.695893,0.502521,0.609231,0.916349,0.507780,0.518081,0.934467,0.605616,0.589309,0.667493,0.976929,0.616712,0.603274,0.823043,0.744430,0.801926,0.619082,0.762820,0.539308,0.825737,0.998461,0.775645,0.608738,0.994613,0.982428,0.542336,0.755402,0.902579,0.503491,0.727392,0.581446,0.777081,0.676455,0.586064,0.624136,0.582518,0.777616,0.934944,0.727566,0.672697,0.845923,0.921186,0.515906,0.840513,0.614691,0.905827,0.875308,0.939214,0.580035,0.653281,0.505326,0.542697,0.677769,0.748363,0.543957,0.764414,0.626780,0.707452,0.672523,0.569726,0.627977,0.637397,0.805961,0.805164,0.888426,0.529266,0.642463,0.548984,0.630816,0.523342,0.511265,0.606366,0.761732,0.559863,0.822309,0.687902,0.688727,0.780257,0.863852,0.758774,0.695084,0.909382,0.568356,0.886549,0.947809,0.562631,0.950875,0.966153,0.814210,0.576669,0.878697,0.740583,0.525051,0.757664,0.651491,0.659505,0.821995,0.648843,0.502871,0.546610,0.540147,0.578806,0.532779,0.672056,0.671327,0.999666,0.713817,0.914622,0.978290,0.750442,0.685443,0.593009,0.892802,0.595096,0.833506,0.837130,0.845230,0.981803,0.643846,0.512942,0.657684,0.521255,0.948590,0.872529,0.607566,0.501860,0.651581,0.665852,0.755116,0.812989,0.778757,0.539323,0.525411,0.552357,0.679194,0.529282,0.561345,0.853763,0.634922,0.531281,0.618376,0.516775,0.611285,0.629417,0.773855,0.778612,0.805502,0.978140,0.741627,0.907596,0.572201,0.634343,0.947442,0.897793,0.969085,0.590229,0.958903,0.588292,0.968972,0.843706,0.518320,0.564079,0.875747,0.694106,0.990221,0.875898,0.920631,0.501938,0.551076,0.522468,0.637228,0.743938,0.849212,0.856815,0.884474,0.630468,0.935494,0.927323,0.588977,0.576226,0.883715,0.725975,0.791826,0.760358,0.670480,0.559452,0.986323,0.831311,0.685287,0.601258,0.502478,0.542135,0.579940,0.640328,0.970145,0.833693,0.746223,0.748303,0.548581,0.512047,0.727335,0.503251,0.554190,0.508963,0.568207,0.796605,0.565299,0.636109,0.591262,0.914741,0.790194,0.730401,0.778161,0.893333,0.936393,0.625688,0.799690,0.938275,0.890370,0.854713,0.678324,0.617997,0.985710,0.618751,0.974028,0.517171,0.510833,0.858156,0.961503,0.692891,0.558232,0.990233,0.947764,0.511420,0.521984,0.800238,0.852362,0.874746,0.734004,0.655251,0.512492,0.779579,0.598312,0.851905,0.941446,0.527269,0.563802,0.874312,0.891141,0.523934,0.746870,0.511513,0.578385,0.533393,0.604702,0.741865,0.900006,0.623588,0.740721,0.732344,0.555103,0.588670,0.819931,0.580727,0.781145,0.652346,0.974981,0.869477,0.588116,0.705763,0.960397,0.533450,0.553158,0.771390,0.645034,0.579930,0.981266,0.827531,0.567357,0.615390,0.654195,0.650458,0.721021,0.969759,0.631389,0.855625,0.835519,0.524605,0.662122,0.717324,0.731020,0.723979,0.862138,0.599484,0.504372,0.986955,0.658355,0.502688,0.934914,0.725443,0.802507,0.529978,0.850715,0.935278,0.615048,0.829694,0.548532,0.745486,0.639048,0.728972,0.616766,0.708786,0.988703,0.856883,0.550152,0.863368,0.692096,0.699513,0.660889,0.705308,0.501628,0.881604,0.738094,0.538130,0.590393,0.561588,0.588636,0.831177,0.840019,0.540983,0.584641,0.651478,0.555317,0.928757,0.921286,0.850458,0.660188,0.518267,0.541588,0.783912,0.798753,0.568088,0.522695,0.806137,0.706367,0.680672,0.670158,0.703898,0.580473,0.837404,0.984484,0.954021,0.503722,0.509664,0.641643,0.740110,0.560966,0.924932,0.599408,0.697176,0.548490,0.648768,0.731503,0.529511,0.945404,0.664312,0.955580,0.752362,0.713500,0.602006,0.888815,0.842335,0.554548,0.816557,0.512535,0.944487,0.608075,0.824045,0.991047,0.593139,0.696806,0.581185,0.750605,0.921244,0.753818,0.552301,0.967761,0.765432,0.986221,0.533942,0.857394,0.586820,0.703595,0.818639,0.657359,0.815679,0.788270,0.620484,0.610517,0.532440,0.560671,0.542689,0.506741,0.710190,0.878899,0.506058,0.553741,0.590296,0.774945,0.918265,0.989926,0.615895,0.733525,0.862230,0.526636,0.520431,0.642624,0.671680,0.576052,0.757476,0.682486,0.769754,0.706646,0.602619,0.607957,0.768863,0.700947,0.999924,0.698276,0.837919,0.852222,0.606482,0.644577,0.747808,0.844603,0.892477,0.935087,0.793442,0.716877,0.717539,0.725380,0.560216,0.585452,0.756100,0.656099,0.940945,0.936949,0.776221,0.731310,0.613999,0.696315,0.567190,0.776074,0.519307,0.576075,0.972431,0.673703,0.894731,0.606134,0.507816,0.859223,0.631116,0.877838,0.621581,0.955822,0.831675,0.539001,0.698549,0.758669,0.639768,0.630661,0.874080,0.678055,0.737568,0.500822,0.611543,0.609030,0.932105,0.580289,0.584616,0.609368,0.586277,0.736052,0.577884,0.508663,0.838899,0.688758,0.510422,0.890671,0.565883,0.621620,0.725048,0.710609,0.875606,0.798221,0.803677,0.724361,0.586797,0.600077,0.796914,0.673611,0.912028,0.847317,0.549839,0.601966,0.630491,0.624388,0.676011,0.818423,0.895172,0.575562,0.816010,0.621219,0.819288,0.580414,0.578811,0.727946,0.792332,0.694512,0.603851,0.577530,0.605761,0.587664,0.818102,0.893104,0.877421,0.856330,0.602388,0.585805,0.877627,0.726036,0.809464,0.584188,0.739578,0.734821,0.733825,0.613463,0.672914,0.828011,0.956294,0.559146,0.872805,0.716518,0.643434,0.808435,0.514895,0.911438,0.918487,0.684706,0.898852,0.957349,0.693664,0.858065,0.837022,0.620041,0.702250,0.501348,0.798423,0.896063,0.818418,0.586243,0.577335,0.875052,0.712171,0.501818,0.514795,0.978166,0.783419,0.708712,0.674368,0.586545,0.799104,0.668787,0.504443,0.645566,0.574826,0.616315,0.527366,0.519725,0.770806,0.702916,0.685907,0.649327,0.529185,0.512529,0.757030,0.753907,0.612826,0.916599,0.781338,0.601709,0.715037,0.974721,0.993098,0.787630,0.556656,0.935547,0.848745,0.852636,0.885359,0.731899,0.897839,0.555000,0.728948,0.979640,0.557500,0.756229,0.794459,0.989101,0.731993,0.686708,0.875063,0.512018,0.812636,0.998332,0.663925,0.639140,0.649925,0.984207,0.508543,0.803851,0.939376,0.850798,0.603917,0.544951,0.754737,0.660806,0.581413,0.777496,0.613673,0.504954,0.798262,0.740725,0.677605,0.622701,0.578266,0.835890,0.555398,0.518886,0.801348,0.893305,0.980327,0.644942,0.594148,0.713906,0.792593,0.956231,0.716061,0.598375,0.836914,0.521309,0.783961,0.505867,0.947577,0.977031,0.570878,0.761856,0.519195,0.730788,0.676131,0.543132,0.996523,0.504237,0.580823,0.686449,0.819277,0.668840,0.861149,0.519941,0.719035,0.948029,0.546673,0.761287,0.616773,0.607656,0.874240,0.708625,0.894797,0.543998,0.556153,0.888213,0.823087,0.902875,0.800624,0.844213,0.933312,0.503695,0.631218,0.913133,0.720361,0.533850,0.619605,0.905741,0.833178,0.944016,0.556623,0.760549,0.689530,0.502309,0.506973,0.683479,0.798389,0.623273,0.724861,0.900165,0.595732,0.697387,0.692323,0.987134,0.722538,0.528103,0.589098,0.956343,0.642710,0.796312,0.548132,0.514264,0.531582,0.935970,0.713563,0.873027,0.762236,0.502364,0.555820,0.558386,0.685787,0.883182,0.634488,0.627962,0.820222,0.584952,0.620739,0.615732,0.745667,0.585236,0.738861,0.526100,0.785468,0.858423,0.550516,0.512899,0.711504,0.570892,0.538410,0.537410,0.642684,0.555164,0.942709,0.548877,0.500434,0.551128,0.771290,0.807422,0.659927,0.744191,0.729301,0.655682,0.649127,0.530424,0.587538,0.752111,0.590900,0.629953,0.902282,0.970535,0.937806,0.834513,0.881627,0.933409,0.630706,0.528624,0.565233,0.749831,0.930709,0.811258,0.807408,0.511417,0.809253,0.697639,0.845606,0.580197,0.590518,0.740791,0.763993,0.862730,0.603633,0.575307,0.657142,0.753376,0.543286,0.869058,0.542591,0.996731,0.660783,0.884895,0.752862,0.580286,0.516586,0.962978,0.731075,0.561283,0.908109,0.794905,0.796023,0.635368,0.673061,0.664922,0.606238,0.548956,0.770109,0.619483,0.735414,0.969561,0.827113,0.572565,0.808138,0.574609,0.634662,0.809542,0.622060,0.731209,0.771064,0.863895,0.814018,0.501044,0.632932,0.697060,0.903389,0.559840,0.522714,0.781566,0.676673,0.730919,0.652115,0.579118,0.717169,0.847750,0.638007,0.656789,0.685980,0.887371,0.731736,0.630901,0.563076,0.779159,0.854939,0.611531,0.831700,0.502551,0.542439,0.691086,0.658205,0.725770,0.800949,0.861065,0.502117,0.737788,0.582676,0.667137,0.599757,0.809998,0.689936,0.672738,0.793652,0.530068,0.659663,0.639903,0.797041,0.519165,0.865233,0.867439,0.537552,0.756226,0.730873,0.600931,0.581739,0.780833,0.662777,0.910704,0.914581,0.532624,0.715068,0.941105,0.756698,0.729649,0.635011,0.632727,0.889013,0.707691,0.792811,0.645107,0.743013,0.623911,0.713610,0.810594,0.653560,0.813769,0.651226,0.836193,0.831320,0.604327,0.969364,0.842281,0.883617,0.759313,0.666081,0.715743,0.858044,0.644729,0.809482,0.745911,0.765986,0.550473,0.852990,0.625647,0.821657,0.513640,0.656931,0.989777,0.860060,0.771700,0.552814,0.500589,0.715626,0.541177,0.679031,0.848431,0.538602,0.583251,0.650136,0.647290,0.801974,0.823856,0.723498,0.683552,0.808998,0.594068,0.845152,0.841382,0.833383,0.936224,0.535681,0.789678,0.648096,0.525377,0.941968,0.639013,0.787738,0.982999,0.555404,0.549361,0.540144,0.591867,0.854775,0.658847,0.603892,0.551836,0.773903,0.763422,0.535849,0.945936,0.809019,0.660377,0.751715,0.617757,0.923056,0.543954,0.673365,0.667644,0.685077,0.816916,0.860351,0.567508,0.992052,0.703983,0.916273,0.513891,0.542093,0.920282,0.906676,0.913082,0.569481,0.827879,0.899785,0.575688,0.739332,0.832582,0.559951,0.626435,0.777901,0.524380,0.846770,0.520846,0.560721,0.535911,0.775304,0.701910,0.868013,0.925201,0.607223,0.959901,0.980936,0.517753,0.949987,0.516737,0.799121,0.582966,0.541880,0.541080,0.660766,0.969994,0.881101,0.736904,0.526457,0.873865,0.557032,0.850885,0.848046,0.676014,0.552957,0.529843,0.875778,0.676251,0.559581,0.655171,0.846095,0.572619,0.669415,0.734838,0.734761,0.792672,0.910564,0.806705,0.665510,0.750727,0.789604,0.782189,0.923602,0.555903,0.706073,0.875297,0.941903,0.847265,0.844474,0.501005,0.868540,0.628905,0.609982,0.673890,0.511607,0.508854,0.728008,0.683330,0.805454,0.714943,0.746397,0.810626,0.969726,0.518802,0.903781,0.515083,0.517442,0.583149,0.810727,0.602541,0.859240,0.903833,0.637798,0.530045,0.561247,0.921752,0.858217,0.887269,0.735608,0.874181,0.739138,0.519588,0.864053,0.522521,0.591396,0.512134,0.525769,0.661718,0.834868,0.688349,0.693651,0.566329,0.828841,0.537195,0.771665,0.854400,0.882395,0.749353,0.806124,0.769955,0.795389,0.923487,0.629925,0.858404,0.598578,0.660212,0.818232,0.670554,0.769810,0.662365,0.733015,0.917434,0.574156,0.741873,0.696112,0.738379,0.722541,0.716300,0.744625,0.572345,0.885096,0.808002,0.904559,0.509477,0.581569,0.608621,0.539325,0.688787,0.820980,0.603054,0.612924,0.735515,0.919898,0.601882,0.839151,0.503590,0.658264,0.657286,0.587917,0.817281,0.593499,0.873404,0.934447,0.987963,0.797185,0.665936,0.622636,0.544143,0.838759,0.596550,0.821263,0.934020,0.799945,0.526542,0.623069,0.714288,0.932608,0.971033,0.922834,0.502571,0.620128,0.554609,0.830360,0.689162,0.653658,0.699469,0.707333,0.819542,0.561162,0.776994,0.509410,0.654390,0.638006,0.526328,0.793624,0.533622,0.666777,0.535756,0.582289,0.898176,0.708275,0.627306,0.877476,0.639642,0.513186,0.821204,0.531367,0.564502,0.635477,0.822664,0.611818,0.743092,0.986173,0.928351,0.782946,0.626098,0.806564,0.758296,0.724642,0.656964,0.650630,0.555250,0.755524,0.683128,0.517723,0.777373,0.962884,0.769994,0.567944,0.855900,0.615319,0.808248,0.629423,0.956048,0.780990,0.656428,0.568203,0.616771,0.836831,0.955539,0.814893,0.572216,0.965354,0.641888,0.659219,0.704362,0.526729,0.812511,0.622536,0.556602,0.692465,0.594904,0.756356,0.959264,0.774708,0.861485,0.770125,0.783352,0.530122,0.885427,0.618175,0.670888,0.601395,0.996287,0.630968,0.591256,0.644273,0.958458,0.705384,0.636943,0.867338,0.856449,0.780906,0.523941,0.717676,0.538842,0.568052,0.537682,0.777026,0.742616,0.927121,0.768437,0.525671,0.723926,0.862594,0.697519,0.672509,0.590965,0.884288,0.654134,0.814080,0.724664,0.931676,0.766653,0.800265,0.628728,0.623660,0.592829,0.848759,0.598473,0.565660,0.803008,0.955101,0.597563,0.502372,0.598532,0.776290,0.831754,0.830237,0.787313,0.959353,0.945380,0.590273,0.874870,0.548073,0.652293,0.803262,0.555501,0.531693,0.812628,0.502136,0.580071,0.649914,0.717669,0.534798,0.629836,0.579692,0.980504,0.508402,0.825753,0.527312,0.601987,0.898234,0.930205,0.986488,0.745389,0.502149,0.838333,0.865322,0.582062,0.759116,0.649292,0.718393,0.579964,0.512604,0.635735,0.996514,0.529675,0.533743,0.766986,0.539438,0.851574,0.776035,0.538438,0.696515,0.969881,0.569624,0.710211,0.641357,0.615054,0.601111,0.884585,0.813605,0.655370,0.905394,0.571799,0.952737,0.513550,0.510538,0.943465,0.862480,0.703343,0.753186,0.720346,0.726524,0.887081,0.605942,0.604796,0.987702,0.673190,0.566442,0.534405,0.569714,0.731906,0.557059,0.604496,0.518977,0.576824,0.676872,0.865193,0.839642,0.838175,0.517942,0.754758,0.616427,0.793998,0.947192,0.868474,0.502090,0.813587,0.936909,0.659534,0.752121,0.883732,0.584768,0.644390,0.628238,0.878098,0.545098,0.697209,0.814538,0.721862,0.846358,0.824830,0.709517,0.904612,0.572788,0.800831,0.524292,0.634848,0.650004,0.549310,0.560726,0.531332,0.670015,0.601226,0.599593,0.630792,0.917096,0.523529,0.566242,0.666209,0.544930,0.916476,0.760987,0.970941,0.907925,0.502282,0.527752,0.524123,0.820057,0.629406,0.704139,0.611287,0.601319,0.804138,0.795621,0.917504,0.952990,0.685928,0.516811,0.706296,0.663429,0.873867,0.537118,0.522470,0.583829,0.721374,0.512259,0.646674,0.685597,0.811596,0.986597,0.906176,0.522506,0.557926,0.867147,0.766122,0.679680,0.517271,0.764907,0.772851,0.876211,0.986136,0.856163,0.745119,0.962305,0.522858,0.569040,0.598483,0.536607,0.816288,0.719321,0.651095,0.974975,0.587288,0.629615,0.798373,0.981979,0.942213,0.567103,0.668302,0.641068,0.587705,0.517397,0.905279,0.702254,0.578908,0.536822,0.703649,0.595605,0.682084,0.647141,0.552591,0.514381,0.638081,0.854772,0.907785,0.957288,0.693044,0.766459,0.515491,0.591798,0.784854,0.513829,0.620272,0.561791,0.552145,0.593376,0.907150,0.748495,0.841848,0.783350,0.667667,0.877476,0.759813,0.521626,0.907334,0.521705,0.922786,0.756269,0.546088,0.523396,0.932756,0.985156,0.667093,0.793121,0.884192,0.950256,0.573058,0.699518,0.948114,0.642278,0.953544,0.580295,0.926561,0.664511,0.652956,0.594566,0.921034,0.545735,0.567153,0.921460,0.626171,0.539752,0.547248,0.588394,0.738820,0.518242,0.571615,0.531735,0.597339,0.920880,0.694725,0.880045,0.511092,0.592373,0.864188,0.543325,0.724998,0.627735,0.981093,0.503580,0.598809,0.533241,0.819404,0.931236,0.547602,0.514186,0.887285,0.914366,0.515487,0.516541,0.664377,0.744714,0.712695,0.951141,0.543890,0.799988,0.649512,0.961532,0.655058,0.595430,0.589790,0.655273,0.892404,0.792799,0.907202,0.858765,0.756596,0.526942,0.521592,0.627512,0.614410,0.798236,0.558201,0.804871,0.538589,0.943311,0.818492,0.592670,0.757129,0.631640,0.651825,0.967948,0.678014,0.616453,0.819178,0.649749,0.535981,0.612999,0.504951,0.959976,0.796340,0.978246,0.834010,0.690139,0.551948,0.500574,0.533869,0.580171,0.964318,0.577564,0.819667,0.683397,0.764440,0.648680,0.695616,0.512548,0.727481,0.934724,0.793195,0.961298,0.633192,0.997592,0.779651,0.656693,0.575389,0.674822,0.637307,0.934936,0.631992,0.538962,0.936951,0.705079,0.774006,0.994058,0.812889,0.681124,0.731034,0.589578,0.662161,0.757208,0.514401,0.840659,0.999523,0.597627,0.700651,0.542357,0.922455,0.959050,0.630821,0.720612,0.907385,0.876381,0.836925,0.903930,0.577774,0.594511,0.747816,0.864407,0.911511,0.713302,0.658292,0.525886,0.546630,0.675536,0.738602,0.520918,0.962083,0.527055,0.733999,0.679273,0.546696,0.563202,0.797266,0.525155,0.624817,0.558715,0.755023,0.626240,0.900992,0.730735,0.551093,0.733273,0.907526,0.752106,0.778760,0.576344,0.527703,0.733544,0.995071,0.873803,0.989194,0.679983,0.781727,0.694804,0.633583,0.534137,0.731501,0.856876,0.734433,0.707676,0.550087,0.701808,0.676923,0.526173,0.722860,0.612822,0.879180,0.531644,0.563380,0.614628,0.811296,0.870941,0.592964,0.868760,0.517407,0.534143,0.664012,0.593066,0.677863,0.533044,0.886325,0.734748,0.636060,0.525294,0.832142,0.962566,0.780751,0.946009,0.846432,0.768984,0.944354,0.547960,0.884148,0.608825,0.662053,0.510917,0.688505,0.516917,0.641621,0.664297,0.556877,0.731704,0.910529,0.529603,0.711141,0.971048,0.774049,0.966236,0.574199,0.697390,0.777186,0.847382,0.949963,0.639943,0.678106,0.985030,0.702417,0.876669,0.918476,0.881468,0.513097,0.603667,0.703071,0.988481,0.766745,0.908862,0.612352,0.781570,0.580856,0.519208,0.953208,0.726387,0.814568,0.790610,0.755762,0.702003,0.740442,0.943051,0.514490,0.584752,0.618702,0.971948,0.540742,0.744404,0.701059,0.887716,0.600322,0.507084,0.691449,0.568190,0.767943,0.707786,0.609983,0.889510,0.508391,0.670640,0.849269,0.938811,0.905700,0.535405,0.552757,0.649627,0.756137,0.590933,0.912134,0.818627,0.842855,0.777709,0.519365,0.740618,0.968778,0.902594,0.878080,0.700585,0.672850,0.524630,0.868125,0.986275,0.886795,0.625186,0.609152,0.939310,0.770118,0.813192,0.874382,0.950422,0.782678,0.684597,0.982857,0.722397,0.933034,0.723118,0.858528,0.535008,0.613674,0.695540,0.603426,0.577195,0.938003,0.861614,0.773997,0.579503,0.507163,0.772436,0.602998,0.582944,0.686769,0.666497,0.540059,0.630365,0.690834,0.676201,0.623079,0.766327,0.641041,0.754231,0.613563,0.869060,0.722022,0.640045,0.558057,0.680771,0.775266,0.948610,0.832076,0.762216,0.799539,0.524885,0.566192,0.541207,0.686805,0.662924,0.693666,0.540389,0.807052,0.850896,0.802419,0.764745,0.915362,0.565974,0.534872,0.951867,0.504910,0.602322,0.952640,0.952455,0.743874,0.586379,0.532182,0.634302,0.549725,0.905137,0.762213,0.761484,0.753711,0.543464,0.935204,0.628934,0.738237,0.666100,0.933554,0.519329,0.612436,0.765581,0.526645,0.504895,0.724544,0.877843,0.969042,0.511294,0.710077,0.507317,0.644846,0.608281,0.807920,0.671861,0.929072,0.615315,0.849925,0.621134,0.603563,0.990415,0.805904,0.842550,0.568122,0.576338,0.543098,0.711848,0.512548,0.811958,0.588507,0.567218,0.777978,0.836104,0.745088,0.574362,0.684500,0.717699,0.670573,0.516770,0.846673,0.848528,0.818887,0.806122,0.622146,0.975286,0.671003,0.533713,0.806603,0.627552,0.661315,0.968061,0.546136,0.735653,0.759261,0.655812,0.821249,0.866790,0.546233,0.754677,0.997480,0.970250,0.977926,0.564993,0.767416,0.598283,0.722777,0.867023,0.602152,0.785077,0.667822,0.517920,0.785227,0.574460,0.633123,0.530697,0.538861,0.563330,0.793125,0.997545,0.747193,0.676691,0.547788,0.627515,0.556395,0.544627,0.733449,0.792654,0.756420,0.832643,0.646632,0.723594,0.585811,0.829522,0.884159,0.866533,0.528459,0.517403,0.671067,0.625982,0.912381,0.525751,0.781263,0.918113,0.903777,0.659277,0.711542,0.880399,0.525747,0.723561,0.521300,0.558622,0.520294,0.573721,0.797170,0.826289,0.974216,0.972409,0.802835,0.881678,0.757818,0.529368,0.660308,0.809298,0.504572,0.967124,0.589245,0.772685,0.673561,0.847728,0.718319,0.576560,0.528835,0.778984,0.786845,0.864106,0.655591,0.559213,0.848574,0.973305,0.615751,0.750869,0.919275,0.704782,0.546664,0.723548,0.505885,0.664665,0.681885,0.527988,0.690640,0.931556,0.770883,0.676107,0.739414,0.990263,0.526758,0.803151,0.755577,0.519285,0.651408,0.579548,0.815681,0.808461,0.780289,0.963707,0.823551,0.837232,0.870994,0.685411,0.732408,0.816893,0.836872,0.521351,0.988781,0.797811,0.666158,0.721335,0.903962,0.606521,0.577610,0.809888,0.659900,0.772731,0.905189,0.781233,0.673604,0.958737,0.586118,0.947453,0.545545,0.779273,0.557006,0.518678,0.611822,0.711271,0.689686,0.649489,0.667755,0.688478,0.714452,0.600087,0.711760,0.542153,0.517526,0.618300,0.620739,0.908347,0.959226,0.973229,0.858343,0.535337,0.802977,0.509272,0.586337,0.583590,0.656988,0.957368,0.783322,0.617165,0.804354,0.577568,0.969019,0.721683,0.649760,0.748392,0.689448,0.938781,0.539871,0.992309,0.793312,0.652136,0.836228,0.579961,0.523614,0.568230,0.557748,0.753804,0.959386,0.557352,0.777523,0.885060,0.972555,0.786595,0.628896,0.545238,0.774388,0.507616,0.536061,0.646790,0.651994,0.618944,0.792668,0.513760,0.688557,0.894651,0.784617,0.847734,0.851630,0.922811,0.800614,0.823667,0.558800,0.536233,0.511043,0.806926,0.735320,0.523164,0.589349,0.853361,0.884956,0.546243,0.524893,0.690843,0.677840,0.902865,0.579885,0.696035,0.874986,0.807053,0.933614,0.986441,0.659882,0.570946,0.986885,0.564218,0.868239,0.856827,0.792757,0.654426,0.539971,0.562609,0.773865,0.611180,0.633394,0.911320,0.531138,0.576307,0.852148,0.768302,0.612582,0.617195,0.811884,0.829552,0.992684,0.677022,0.943288,0.814619,0.640038,0.963546,0.941464,0.937250,0.945131,0.674796,0.517691,0.545181,0.533435,0.680439,0.621076,0.527166,0.791186,0.906130,0.709009,0.747947,0.867353,0.696893,0.709585,0.618257,0.565447,0.970091,0.812497,0.782451,0.630877,0.557485,0.679371,0.569310,0.755302,0.551162,0.507784,0.688392,0.733960,0.694207,0.565967,0.781634,0.644818,0.855249,0.909132,0.645869,0.852271,0.688274,0.653951,0.575172,0.937524,0.653934,0.766016,0.836849,0.819013,0.720447,0.576508,0.854367,0.706190,0.902101,0.841910,0.988218,0.644157,0.737659,0.765790,0.505468,0.900567,0.892969,0.775499,0.547487,0.933229,0.592206,0.562101,0.625464,0.543155,0.600188,0.511027,0.557672,0.913206,0.960435,0.611410,0.751755,0.625338,0.737918,0.996158,0.999046,0.817626,0.694102,0.627443,0.659896,0.555962,0.659869,0.964388,0.696452,0.500502,0.527900,0.808420,0.908285,0.514726,0.779829,0.651999,0.762977,0.670726,0.748007,0.816631,0.892496,0.797148,0.688367,0.762133,0.858397,0.769675,0.750082,0.553684,0.811823,0.617999,0.634207,0.692550,0.755107,0.789600,0.812745,0.579907,0.633300,0.601400,0.865762,0.753429,0.636737,0.573034,0.625319,0.882890,0.543245,0.595228,0.738142,0.532041,0.916494,0.550977,0.685462,0.614682,0.965681,0.509041,0.527192,0.707045,0.812955,0.860876,0.596524,0.976012,0.880029,0.819320,0.647154,0.643128,0.736393,0.527192,0.581047,0.943390,0.579206,0.853974,0.916232,0.864797,0.771428,0.632115,0.578171,0.625597,0.679507,0.551041,0.975958,0.814171,0.903456,0.623403,0.788706,0.780216,0.800869,0.681935,0.906333,0.532319,0.721203,0.538744,0.930204,0.635712,0.507391,0.991068,0.532208,0.830932,0.663279,0.607895,0.554318,0.682207,0.808921,0.516673,0.532116,0.609587,0.851776,0.571890,0.823054,0.642931,0.690903,0.984067,0.619007,0.519201,0.621084,0.718120,0.927017,0.568717,0.731853,0.815345,0.629473,0.673145,0.801976,0.506980,0.779882,0.935274,0.785995,0.979844,0.687904,0.922795,0.683351,0.890099,0.759690,0.511131,0.962537,0.590673,0.567171,0.622891,0.602238,0.691470,0.753359,0.802804,0.947687,0.713822,0.693781,0.526903,0.977892,0.876379,0.991375,0.796893,0.701629,0.834318,0.960675,0.781065,0.723872,0.866424,0.518247,0.899663,0.636783,0.675342,0.936374,0.681975,0.666922,0.636554,0.510975,0.567190,0.524301,0.607701,0.576452,0.595220,0.950574,0.775297,0.986357,0.764377,0.545514,0.500498,0.596233,0.805595,0.634509,0.715056,0.867353,0.888389,0.834744,0.711669,0.728962,0.814500,0.792697,0.852091,0.580534,0.532151,0.865168,0.563297,0.553885,0.746392,0.537801,0.552789,0.538718,0.525104,0.560651,0.835111,0.836418,0.519772,0.693652,0.518952,0.533692,0.526621,0.675306,0.635418,0.810841,0.953015,0.616389,0.815837,0.673901,0.871905,0.533767,0.778731,0.708153,0.746670,0.828630,0.617226,0.523215,0.803129,0.834098,0.839974,0.652903,0.742616,0.829989,0.613423,0.529316,0.650780,0.572112,0.531290,0.735341,0.545286,0.914991,0.861187,0.972074,0.976978,0.516684,0.791926,0.658857,0.768909,0.937149,0.528303,0.626121,0.703946,0.724878,0.989786,0.890797,0.634176,0.942696,0.759834,0.631104,0.603589,0.732703,0.568830,0.825745,0.515785,0.946110,0.517426,0.981730,0.548097,0.636195,0.550403,0.540576,0.720921,0.601985,0.830086,0.856126,0.628529,0.622936,0.640618,0.543982,0.678602,0.772053,0.804881,0.838964,0.521393,0.595151,0.530912,0.782887,0.698751,0.626772,0.652146,0.922596,0.649527,0.586564,0.884536,0.579604,0.700154,0.714309,0.630792,0.823999,0.935959,0.950364,0.937703,0.593134,0.776359,0.701134,0.919948,0.836775,0.842291,0.656972,0.607422,0.687675,0.674119,0.883978,0.981000,0.517640,0.697771,0.643154,0.722328,0.604514,0.961697,0.770940,0.563391,0.557374,0.767905,0.505043,0.583874,0.589386,0.521453,0.794480,0.963764,0.559902,0.779504,0.832980,0.604873,0.548067,0.517578,0.901855,0.704107,0.533680,0.632667,0.636857,0.674332,0.783210,0.778220,0.513025,0.739847,0.569535,0.867611,0.535077,0.623892,0.611228,0.930927,0.571931,0.642047,0.677308,0.715778,0.895504,0.935068,0.848209,0.892720,0.886579,0.724778,0.889999,0.512744,0.759842,0.761322,0.814572,0.921580,0.512297,0.701626,0.810256,0.647137,0.665717,0.647252,0.836030,0.762666,0.672590,0.717640,0.644152,0.871444,0.574899,0.664166,0.593961,0.548813,0.631052,0.557455,0.555745,0.758497,0.596439,0.813843,0.834151,0.972768,0.821738,0.700193,0.770206,0.906248,0.701324,0.838696,0.674240,0.767910,0.730396,0.703230,0.643513,0.672103,0.664193,0.585831,0.836539,0.557842,0.551724,0.848500,0.706698,0.827151,0.525901,0.904455,0.789129,0.674027,0.928108,0.512949,0.572296,0.742500,0.840448,0.701157,0.549829,0.527526,0.503463,0.980940,0.764821,0.637832,0.638762,0.831416,0.819155,0.696333,0.720176,0.685798,0.790374,0.832705,0.587415,0.809085,0.693661,0.539169,0.584615,0.948101,0.539539,0.991621,0.834426,0.680044,0.903382,0.739246,0.984270,0.922348,0.797679,0.600702,0.810853,0.822607,0.722417,0.661752,0.664520,0.721451,0.654973,0.822649,0.815557,0.672833,0.539080,0.722161,0.768904,0.610284,0.766823,0.559243,0.654424,0.724576,0.686743,0.529281,0.802399,0.675500,0.521321,0.766596,0.760943,0.517733,0.676391,0.539098,0.614676,0.723710,0.940043,0.592242,0.595754,0.600188,0.503529,0.638805,0.648311,0.869708,0.643762,0.867374,0.819384,0.509751,0.734319,0.731799,0.726974,0.723296,0.780025,0.622136,0.584250,0.774145,0.526043,0.605504,0.812999,0.997080,0.798411,0.852464,0.757868,0.958008,0.625645,0.757385,0.828875,0.870992,0.764248,0.783388,0.599017,0.614905,0.809220,0.783436,0.696470,0.646058,0.646148,0.562527,0.578594,0.647649,0.655229,0.809443,0.881083,0.861615,0.959513,0.526923,0.566257,0.845514,0.502601,0.752235,0.613315,0.533329,0.504337,0.689099,0.906858,0.575256,0.707992,0.722427,0.875915,0.554678,0.504327,0.921554,0.932782,0.928310,0.550911,0.775105,0.630058,0.702339,0.666150,0.676495,0.710646,0.678326,0.965053,0.774705,0.590509,0.507590,0.925081,0.609542,0.816248,0.751789,0.765152,0.668009,0.654106,0.879425,0.873107,0.505960,0.617528,0.697923,0.975314,0.672723,0.828193,0.975535,0.874624,0.681810,0.682942,0.675944,0.823533,0.725432,0.791874,0.731464,0.788545,0.651969,0.630257,0.559038,0.548269,0.676953,0.588346,0.625131,0.809372,0.810620,0.930589,0.724746,0.939417,0.900278,0.951399,0.502645,0.772284,0.816669,0.505123,0.627214,0.914891,0.961476,0.578139,0.532953,0.651377,0.633287,0.540972,0.699662,0.552504,0.570710,0.612636,0.677491,0.529726,0.793302,0.775615,0.760111,0.869273,0.734111,0.791433,0.908945,0.875368,0.576071,0.784130,0.641382,0.732183,0.976970,0.780093,0.797557,0.623386,0.551181,0.683753,0.796679,0.992539,0.538820,0.557095,0.621434,0.590349,0.978415,0.579110,0.531633,0.747834,0.902652,0.773731,0.871185,0.689887,0.534565,0.783425,0.685362,0.690965,0.852128,0.621173,0.627798,0.880818,0.656871,0.647404,0.601647,0.985446,0.644483,0.537300,0.988959,0.658421,0.591980,0.794620,0.584641,0.860166,0.564033,0.606517,0.605942,0.901721,0.648683,0.595545,0.715257,0.518223,0.681108,0.999108,0.545047,0.513351,0.562441,0.706557,0.649106,0.645885,0.522601,0.937290,0.864261,0.825256,0.722114,0.830312,0.669363,0.900486,0.814485,0.822030,0.590299,0.720198,0.707423,0.558447,0.711282,0.802582,0.595038,0.618671,0.945632,0.995851,0.600638,0.931453,0.811144,0.751991,0.648411,0.621231,0.739955,0.913421,0.890233,0.626855,0.645660,0.624628,0.572440,0.605940,0.551706,0.592427,0.908986,0.590853,0.815042,0.769657,0.808411,0.763450,0.795399,0.710297,0.666418,0.847329,0.623551,0.720637,0.856090,0.538802,0.797080,0.817735,0.527917,0.617341,0.619205,0.786335,0.624395,0.602088,0.614028,0.967577,0.605773,0.786384,0.783228,0.567089,0.639131,0.847402,0.837132,0.558684,0.628764,0.737625,0.913711,0.579070,0.613675,0.610412,0.560605,0.993904,0.968854,0.946580,0.660467,0.728944,0.619040,0.528709,0.590650,0.863435,0.680417,0.687391,0.540554,0.575407,0.565987,0.533324,0.624685,0.645389,0.655549,0.531031,0.964267,0.739725,0.831894,0.954533,0.834422,0.571238,0.762884,0.520661,0.517364,0.837977,0.504803,0.875674,0.593969,0.501561,0.511115,0.591685,0.954963,0.768244,0.722802,0.997263,0.822409,0.760457,0.921089,0.758357,0.873037,0.842188,0.514763,0.687141,0.765480,0.607839,0.606698,0.597209,0.506956,0.797643,0.773800,0.637177,0.508031,0.975299,0.930723,0.508569,0.920518,0.713481,0.601264,0.760707,0.803061,0.783108,0.843519,0.899961,0.589434,0.755943,0.944777,0.590204,0.529381,0.701600,0.672488,0.791731,0.730020,0.721627,0.593142,0.659251,0.756566,0.883378,0.678115,0.871533,0.611696,0.902213,0.710262,0.650528,0.913204,0.581300,0.633181,0.563257,0.947900,0.696565,0.736924,0.608794,0.570741,0.932569,0.857562,0.549515,0.933963,0.546904,0.520199,0.704883,0.738515,0.640637,0.882337,0.838511,0.720120,0.798342,0.936745,0.569001,0.579221,0.567663,0.698395,0.795316,0.813502,0.564285,0.584355,0.867796,0.878899,0.768817,0.600719,0.705216,0.598068,0.994778,0.927990,0.691487,0.712320,0.860566,0.604767,0.584372,0.861354,0.659801,0.889253,0.783769,0.774051,0.616149,0.735792,0.941804,0.969652,0.836715,0.955788,0.721824,0.626531,0.914193,0.642115,0.864278,0.500267,0.624751,0.864877,0.787152,0.709948,0.561375,0.807038,0.876409,0.940972,0.517750,0.796966,0.924457,0.713493,0.864373,0.828488,0.724241,0.840559,0.746413,0.505047,0.556704,0.634647,0.540105,0.542066,0.508314,0.578331,0.863897,0.598498,0.674558,0.803982,0.503748,0.804327,0.522158,0.516824,0.731444,0.704409,0.611162,0.580370,0.780405,0.831782,0.789553,0.789926,0.787811,0.553415,0.541141,0.784450,0.793978,0.603495,0.501679,0.692783,0.507894,0.802989,0.704871,0.825727,0.774237,0.752927,0.756360,0.697058,0.896919,0.556644,0.581801,0.692601,0.598635,0.534064,0.593314,0.773909,0.812831,0.569922,0.822953,0.815428,0.631729,0.658917,0.809959,0.683379,0.812826,0.815135,0.705596,0.899936,0.610934,0.530448,0.713696,0.547227,0.752522,0.615261,0.725913,0.509636,0.606128,0.714365,0.918744,0.711166,0.641123,0.862994,0.508522,0.620914,0.592016,0.938236,0.629172,0.729596,0.857226,0.704091,0.574482,0.752658,0.738110,0.781456,0.560405,0.564771,0.650374,0.766764,0.717364,0.953001,0.616909,0.775694,0.796475,0.844763,0.647308,0.747241,0.585362,0.716806,0.958574,0.645684,0.770923,0.663049,0.626292,0.543042,0.916928,0.607837,0.631365,0.993930,0.772851,0.714780,0.517876,0.921976,0.700889,0.648641,0.533357,0.698205,0.716704,0.802781,0.579611,0.673104,0.529038,0.985817,0.890059,0.580042,0.760247,0.502093,0.807028,0.706036,0.678520,0.618343,0.660003,0.513064,0.960611,0.993552,0.724392,0.687694,0.512367,0.715873,0.734274,0.606751,0.785113,0.890196,0.680471,0.506815,0.978772,0.896080,0.725553,0.879281,0.667774,0.748204,0.656372,0.620903,0.563650,0.658003,0.856397,0.561567,0.564523,0.769549,0.781777,0.889362,0.539336,0.550742,0.728584,0.797371,0.783261,0.954404,0.836387,0.667707,0.863557,0.850241,0.780104,0.657459,0.501613,0.893346,0.709043,0.860310,0.909533,0.727658,0.628843,0.867414,0.607616,0.610948,0.914218,0.590852,0.794644,0.571461,0.655008,0.628052,0.611878,0.858736,0.971591,0.912428,0.624766,0.610707,0.613253,0.523520,0.744494,0.913965,0.741209,0.750008,0.816499,0.609808,0.865594,0.875682,0.768662,0.500711,0.522243,0.643225,0.862391,0.681931,0.536829,0.511141,0.667523,0.592143,0.696561,0.814107,0.672074,0.603899,0.706647,0.608842,0.785732,0.536692,0.674475,0.775166,0.731861,0.882369,0.889555,0.746680,0.850639,0.950797,0.762270,0.800145,0.531060,0.670368,0.640008,0.973525,0.976621,0.715902,0.761201,0.593024,0.995660,0.845244,0.690414,0.971408,0.792111,0.552715,0.733705,0.536667,0.572882,0.501220,0.528626,0.906066,0.794358,0.833182,0.600175,0.699517,0.583456,0.573134,0.771435,0.791954,0.886014,0.876891,0.581903,0.594071,0.585108,0.546163,0.580357,0.773620,0.716259,0.757024,0.826836,0.508788,0.761434,0.529427,0.925008,0.651552,0.574993,0.524756,0.620547,0.713073,0.935817,0.768950,0.505867,0.526405,0.632732,0.675498,0.862625,0.600909,0.820698,0.567047,0.736746,0.534942,0.818801,0.779600,0.656206,0.829713,0.778567,0.623288,0.989052,0.903183,0.746932,0.677435,0.665977,0.568721,0.575190,0.994026,0.694156,0.829797,0.720961,0.516200,0.795988,0.550795,0.664759,0.522216,0.642773,0.534773,0.597110,0.660873,0.964576,0.537887,0.537077,0.576182,0.649143,0.937778,0.806531,0.515520,0.533531,0.560614,0.667864,0.974781,0.989681,0.543172,0.963754,0.848081,0.708310,0.541207,0.878170,0.568867,0.644144,0.569406,0.581977,0.533227,0.691863,0.892392,0.766660,0.917697,0.527869,0.949615,0.535617,0.618970,0.617197,0.860291,0.636153,0.565559,0.909752,0.798770,0.617016,0.851811,0.532056,0.705432,0.531780,0.635291,0.544304,0.784602,0.566945,0.744237,0.686726,0.864510,0.956550,0.943975,0.543150,0.563776,0.557248,0.609597,0.877221,0.530316,0.600910,0.912049,0.513880,0.844888,0.897135,0.747578,0.758477,0.656735,0.519747,0.726921,0.568907,0.627427,0.911610,0.509976,0.632951,0.621711,0.583848,0.615336,0.811855,0.831109,0.526393,0.818340,0.779442,0.689119,0.525365,0.773716,0.535490,0.589389,0.839267,0.558169,0.629584,0.606666,0.613559,0.642926,0.930193,0.796667,0.766468,0.543181,0.772366,0.956436,0.570473,0.636356,0.632665,0.532513,0.514218,0.587803,0.501512,0.997548,0.659403,0.765479,0.502496,0.686738,0.700888,0.871467,0.553919,0.799779,0.640146,0.619533,0.510986,0.664522,0.957737,0.998327,0.962172,0.662209,0.708621,0.557281,0.713074,0.812859,0.552857,0.813484,0.729383,0.602311,0.648147,0.785275,0.678740,0.513009,0.553664,0.560894,0.997556,0.782211,0.616539,0.905910,0.525627,0.892425,0.890565,0.641958,0.638630,0.732932,0.868952,0.519782,0.524252,0.868505,0.576812,0.757459,0.598128,0.594867,0.610084,0.695988,0.656442,0.545483,0.686245,0.688498,0.606163,0.826280,0.711567,0.667241,0.756115,0.553319,0.947338,0.749272,0.753720,0.633774,0.654356,0.975138,0.539099,0.557040,0.752649,0.950943,0.513388,0.741639,0.569371,0.676004,0.837024,0.556096,0.912616,0.844477,0.528995,0.577383,0.849188,0.635782,0.955184,0.599654,0.732424,0.583443,0.909829,0.581047,0.804535,0.509313,0.705934,0.601907,0.762427,0.638983,0.831101,0.856328,0.703809,0.769767,0.976351,0.533147,0.703074,0.760023,0.883925,0.646283,0.919408,0.617296,0.926789,0.558549,0.733955,0.951331,0.675207,0.630307,0.643248,0.918489,0.567428,0.530895,0.700681,0.749660,0.886168,0.924110,0.628978,0.582829,0.642214,0.629045,0.684739,0.554013,0.528639,0.748166,0.779498,0.569854,0.739314,0.625385,0.738421,0.557230,0.936780,0.922071,0.782934,0.899085,0.615377,0.663993,0.967644,0.562654,0.504345,0.555233,0.796474,0.595354,0.977912,0.791613,0.897235,0.669856,0.969457,0.544362,0.684141,0.696944,0.953001,0.948261,0.558466,0.784916,0.930857,0.946892,0.671441,0.826819,0.630638,0.875829,0.530779,0.598714,0.659850,0.708143,0.713253,0.828852,0.916288,0.520021,0.698405,0.566051,0.639231,0.675569,0.816916,0.991161,0.693968,0.549806,0.621632,0.972945,0.534812,0.536606,0.507575,0.626532,0.571188,0.878212,0.568386,0.630759,0.584829,0.525024,0.953796,0.563698,0.509178,0.515149,0.638982,0.509511,0.662925,0.624307,0.646845,0.942348,0.505402,0.878612,0.609155,0.655923,0.600553,0.559955,0.676072,0.768253,0.579994,0.508233,0.811458,0.883329,0.528880,0.668297,0.524158,0.717236,0.556033,0.776952,0.880400,0.801671,0.890926,0.998487,0.584281,0.570579,0.650813,0.587460,0.852336,0.868469,0.738729,0.776911,0.730279,0.913151,0.622085,0.666358,0.511608,0.935355,0.795176,0.586840,0.858094,0.561186,0.831422,0.917732,0.905046,0.958574,0.868902,0.608721,0.706901,0.933023,0.548308,0.720138,0.511184,0.996740,0.679004,0.596249,0.509103,0.630474,0.517762,0.523960,0.638962,0.869711,0.938974,0.652151,0.614172,0.622510,0.596918,0.604267,0.542081,0.866056,0.796282,0.786999,0.540529,0.508313,0.526715,0.932461,0.593779,0.623418,0.556125,0.610194,0.719397,0.635875,0.509131,0.708087,0.906351,0.745291,0.697769,0.531676,0.921170,0.717375,0.504435,0.637454,0.924668,0.873302,0.615790,0.695511,0.560159,0.621254,0.650621,0.823823,0.760243,0.818039,0.547349,0.669757,0.616147,0.618458,0.576396,0.533063,0.585363,0.739983,0.739764,0.926782,0.859281,0.919975,0.880918,0.664604,0.605078,0.853424,0.986415,0.822593,0.585089,0.531226,0.895822,0.774733,0.980245,0.562687,0.776350,0.667521,0.500504,0.840456,0.526590,0.637939,0.685450,0.642199,0.591025,0.953172,0.716292,0.976767,0.545638,0.670218,0.614812,0.645128,0.694490,0.593873,0.973590,0.745004,0.705622,0.542318,0.787687,0.623524,0.595532,0.707006,0.761872,0.822243,0.728298,0.738555,0.544779,0.593112,0.941433,0.762449,0.577530,0.931848,0.579663,0.674875,0.667305,0.684492,0.552896,0.755599,0.720379,0.683563,0.515586,0.762883,0.568444,0.663113,0.544569,0.934072,0.781503,0.514556,0.804453,0.664802,0.855204,0.933201,0.830570,0.630839,0.919954,0.555931,0.964511,0.577415,0.825991,0.529806,0.688455,0.564059,0.567682,0.710430,0.517472,0.772864,0.768066,0.731840,0.810063,0.532272,0.649009,0.814283,0.706228,0.596464,0.681020,0.808642,0.659450,0.740408,0.846640,0.616316,0.612821,0.559918,0.631271,0.810289,0.777704,0.820939,0.680188,0.657115,0.726890,0.781342,0.502139,0.985019,0.621067,0.553969,0.655578,0.879374,0.679683,0.686187,0.915711,0.601791,0.846424,0.517313,0.733918,0.577938,0.848496,0.684990,0.532641,0.526035,0.750167,0.575588,0.668045,0.632437,0.994358,0.693882,0.875497,0.699889,0.990904,0.658726,0.557465,0.884368,0.597523,0.870597,0.965720,0.783985,0.728076,0.765923,0.656492,0.670367,0.665123,0.508167,0.885817,0.771173,0.726789,0.885146,0.716113,0.712982,0.568227,0.705962,0.952064,0.731153,0.919935,0.724643,0.560570,0.666310,0.787752,0.532031,0.509724,0.530298,0.668639,0.921913,0.785724,0.606556,0.616891,0.764463,0.788534,0.535010,0.684326,0.636992,0.755844,0.625836,0.800374,0.668534,0.600388,0.654030,0.799634,0.551059,0.838963,0.506155,0.926923,0.853535,0.572219,0.594822,0.504878,0.899179,0.905739,0.547017,0.628844,0.513859,0.709803,0.725245,0.823422,0.675598,0.786854,0.795996,0.579365,0.954448,0.794004,0.590392,0.625174,0.913478,0.802569,0.940659,0.782666,0.785651,0.759394,0.596059,0.552112,0.977769,0.805464,0.956742,0.894503,0.699969,0.509973,0.844677,0.694153,0.616371,0.686165,0.532725,0.719161,0.793132,0.549265,0.614951,0.536430,0.594857,0.886194,0.564910,0.606799,0.713125,0.797028,0.537691,0.568184,0.920424,0.556902,0.708472,0.563777,0.993981,0.961909,0.610549,0.521070,0.505668,0.618554,0.598133,0.735084,0.880405,0.521678,0.618667,0.682604,0.515263,0.537447,0.538533,0.696187,0.808487,0.558556,0.702270,0.886886,0.512833,0.555680,0.728296,0.848690,0.689965,0.744825,0.647182,0.587556,0.688085,0.512867,0.566823,0.559309,0.914567,0.502864,0.661131,0.824621,0.957745,0.748431,0.779431,0.622522,0.946170,0.921763,0.543803,0.618960,0.836724,0.905666,0.760191,0.563088,0.732906,0.983714,0.556921,0.627885,0.593899,0.719748,0.820260,0.521175,0.523991,0.694446,0.634089,0.573813,0.965761,0.640700,0.770070,0.812704,0.508861,0.874608,0.570705,0.621770,0.600160,0.816326,0.509692,0.549793,0.599692,0.801566,0.575890,0.569686,0.947940,0.593111,0.544818,0.837256,0.837146,0.706849,0.556059,0.807401,0.662539,0.533448,0.664592,0.703913,0.572695,0.575037,0.535498,0.526098,0.909167,0.712916,0.544875,0.877388,0.509060,0.701319,0.866232,0.773048,0.651767,0.661126,0.513483,0.596089,0.800321,0.660290,0.595730,0.701537,0.545768,0.671468,0.709172,0.789535,0.516763,0.504570,0.705634,0.773060,0.844978,0.828081,0.711975,0.536072,0.521945,0.608012,0.511761,0.707935,0.934091,0.505551,0.608145,0.579903,0.504556,0.746394,0.526563,0.733490,0.917050,0.562196,0.663394,0.640502,0.789385,0.685794,0.531345,0.842860,0.512701,0.906044,0.718156,0.521810,0.535089,0.813839,0.777767,0.550970,0.658909,0.547533,0.699604,0.619223,0.928839,0.784614,0.697927,0.807519,0.974459,0.798536,0.894148,0.673350,0.589845,0.514062,0.784465,0.664839,0.764810,0.690686,0.714930,0.717523,0.813549,0.692378,0.702739,0.952183,0.883275,0.514536,0.772319,0.655822,0.944299,0.834485,0.899125,0.977803,0.600814,0.556522,0.668834,0.685166,0.653113,0.515513,0.566796,0.674593,0.559051,0.738071,0.739097,0.963589,0.695277,0.655222,0.511416,0.662425,0.739045,0.744835,0.742927,0.985660,0.834099,0.552163,0.729478,0.539289,0.721721,0.531837,0.963828,0.625377,0.516718,0.567412,0.529374,0.597376,0.639584,0.534676,0.991980,0.508647,0.514033,0.919632,0.919847,0.595273,0.549479,0.674016,0.921259,0.578297,0.554459,0.817696,0.705828,0.712951,0.872548,0.603253,0.695336,0.842695,0.592921,0.878269,0.636925,0.519854,0.820809,0.709773,0.577891,0.817570,0.952545,0.504262,0.595929,0.701752,0.503174,0.505751,0.994770,0.701796,0.537061,0.537859,0.768510,0.952430,0.759529,0.641420,0.788427,0.510475,0.821542,0.793129,0.796221,0.697605,0.874435,0.844725,0.762467,0.536418,0.575852,0.633602,0.698291,0.572679,0.862268,0.573643,0.728601,0.638191,0.585824,0.527366,0.845739,0.677891,0.563208,0.990229,0.942611,0.536689,0.516382,0.991143,0.939803,0.925295,0.829655,0.658506,0.990644,0.931916,0.638440,0.670006,0.773279,0.850257,0.651092,0.839379,0.532263,0.630903,0.818227,0.522459,0.506256,0.693548,0.866997,0.822144,0.757939,0.894174,0.572326,0.682357,0.844500,0.601712,0.714109,0.858066,0.660809,0.515256,0.653590,0.726925,0.514339,0.968288,0.904002,0.580998,0.584540,0.863069,0.929000,0.789620,0.995069,0.770653,0.673513,0.801171,0.884332,0.762219,0.648253,0.990352,0.906917,0.685048,0.823644,0.617191,0.549250,0.871275,0.814095,0.910801,0.673852,0.602749,0.602847,0.800787,0.954105,0.875339,0.700157,0.645102,0.957879,0.962229,0.549650,0.808254,0.568296,0.651025,0.524898,0.947093,0.820799,0.910047,0.541508,0.649681,0.739260,0.729088,0.895572,0.505825,0.657904,0.564681,0.643635,0.617394,0.502057,0.671869,0.617300,0.670260,0.637569,0.943294,0.856473,0.529000,0.510960,0.885706,0.943654,0.643714,0.702052,0.713775,0.780625,0.685062,0.687190,0.753067,0.957421,0.917758,0.658163,0.820697,0.760328,0.892596,0.897329,0.558820,0.649955,0.537996,0.651667,0.580386,0.904091,0.618034,0.684409,0.748248,0.778595,0.565436,0.537825,0.757830,0.932445,0.979372,0.993330,0.586948,0.759516,0.565929,0.677804,0.509819,0.807875,0.696255,0.528437,0.576365,0.609878,0.517593,0.614247,0.784622,0.540649,0.646941,0.599771,0.610077,0.532685,0.548485,0.967275,0.827962,0.568444,0.818504,0.800273,0.593139,0.866763,0.850007,0.855828,0.564040,0.538979,0.522194,0.503478,0.888568,0.627929,0.603830,0.576902,0.734064,0.724548,0.892317,0.557254,0.538161,0.616539,0.889204,0.670553,0.619482,0.892154,0.701216,0.814234,0.607779,0.706902,0.698497,0.631344,0.935364,0.578215,0.507448,0.883396,0.518828,0.536065,0.966647,0.744874,0.907762,0.520148,0.904917,0.917800,0.995891,0.652280,0.629244,0.841981,0.719366,0.689734,0.585939,0.536045,0.532420,0.789757,0.786950,0.506408,0.566894,0.555142,0.552042,0.500571,0.647775,0.897589,0.625554,0.961969,0.697193,0.687740,0.756907,0.586355,0.515372,0.684896,0.884405,0.726303,0.591795,0.811263,0.756743,0.720267,0.717782,0.581554,0.519736,0.936315,0.629285,0.797706,0.769228,0.506003,0.677431,0.676315,0.549616,0.796874,0.509974,0.509933,0.646540,0.542039,0.587066,0.844516,0.795931,0.955630,0.574608,0.558423,0.759267,0.514491,0.642194,0.873118,0.816133,0.847273,0.692979,0.907742,0.689552,0.541802,0.584463,0.916623,0.747118,0.736252,0.860643,0.605860,0.586899,0.943726,0.563348,0.819332,0.557874,0.748326,0.553676,0.734284,0.842125,0.548039,0.599779,0.962073,0.641689,0.834343,0.764202,0.738026,0.643471,0.548542,0.514164,0.576929,0.830258,0.626687,0.512505,0.736050,0.677136,0.631695,0.600261,0.604420,0.610131,0.522884,0.865352,0.867135,0.946782,0.567122,0.764681,0.716939,0.971889,0.890879,0.513177,0.709075,0.720952,0.958160,0.778665,0.857372,0.965646,0.872163,0.506815,0.750164,0.650009,0.513304,0.540140,0.654970,0.904661,0.612721,0.758762,0.570229,0.596248,0.931195,0.816434,0.662996,0.992104,0.514334,0.552695,0.588291,0.501462,0.503237,0.569588,0.691768,0.584605,0.668910,0.957771,0.628487,0.510490,0.759708,0.559615,0.521465,0.625050,0.690137,0.837748,0.763125,0.776028,0.643366,0.606884,0.556347,0.502340,0.821703,0.966122,0.641525,0.623252,0.674612,0.558336,0.507151,0.794684,0.941284,0.623211,0.816518,0.861461,0.996281,0.792174,0.510243,0.658499,0.975154,0.916828,0.899886,0.828823,0.723861,0.908215,0.898427,0.912499,0.659517,0.704768,0.654161,0.561852,0.610066,0.519481,0.968742,0.953297,0.522423,0.569722,0.584882,0.822615,0.964304,0.871781,0.614787,0.607643,0.630497,0.607541,0.630613,0.582166,0.924674,0.519140,0.554253,0.533868,0.601065,0.926424,0.550923,0.914722,0.776762,0.869379,0.713486,0.727549,0.503535,0.529643,0.850315,0.957635,0.845097,0.544301,0.599106,0.714061,0.526858,0.626310,0.750253,0.836031,0.972896,0.910516,0.503663,0.960983,0.527308,0.648679,0.691708,0.964632,0.743821,0.855696,0.522555,0.651801,0.654014,0.605256,0.712806,0.979311,0.852538,0.817130,0.573639,0.715842,0.916516,0.585741,0.618651,0.697359,0.551589,0.783565,0.589727,0.538605,0.544690,0.581379,0.740521,0.791031,0.903568,0.860799,0.797349,0.734042,0.609878,0.843812,0.788876,0.891639,0.606396,0.844188,0.580488,0.629102,0.583566,0.514335,0.546761,0.829721,0.746045,0.644841,0.539161,0.958520,0.871115,0.678417,0.947319,0.600787,0.822226,0.815607,0.941400,0.608068,0.648945,0.529617,0.913740,0.879490,0.904582,0.506482,0.783693,0.967064,0.680774,0.504585,0.927323,0.570324,0.983768,0.619263,0.725790,0.699976,0.593407,0.938164,0.972324,0.535802,0.572960,0.560850,0.588223,0.648312,0.814898,0.884361,0.541445,0.995758,0.747722,0.999146,0.716364,0.971118,0.760042,0.954440,0.818582,0.650537,0.527121,0.776173,0.541546,0.544825,0.557386,0.804324,0.862456,0.642043,0.956755,0.673584,0.783637,0.636283,0.787967,0.942585,0.904809,0.574248,0.638321,0.934634,0.545379,0.518112,0.829745,0.744770,0.501522,0.972649,0.519060,0.789517,0.781544,0.810094,0.941498,0.564890,0.954509,0.501102,0.608373,0.739821,0.840791,0.646134,0.550597,0.811646,0.513934,0.799630,0.799140,0.749547,0.526631,0.583168,0.582057,0.873764,0.853703,0.842925,0.614132,0.614434,0.732627,0.799197,0.851720,0.593019,0.812904,0.825931,0.675241,0.792807,0.740719,0.659407,0.652703,0.648377,0.604310,0.764734,0.525881,0.690618,0.849491,0.740214,0.947802,0.936639,0.664694,0.657034,0.908360,0.776030,0.814289,0.542585,0.621361,0.948881,0.864658,0.706806,0.558438,0.683671,0.564402,0.896262,0.555032,0.997206,0.978045,0.869402,0.568648,0.537885,0.948197,0.536068,0.586100,0.748356,0.964358,0.537291,0.806758,0.963281,0.727657,0.689028,0.753981,0.574150,0.766271,0.679319,0.733229,0.989413,0.890001,0.510296,0.545686,0.513182,0.638246,0.614083,0.910794,0.817467,0.682002,0.544247,0.996509,0.851084,0.995854,0.567731,0.804406,0.935504,0.800753,0.950802,0.890224,0.554006,0.549831,0.568011,0.741366,0.971441,0.699856,0.713932,0.722241,0.911040,0.719584,0.944745,0.751529,0.998840,0.832466,0.927735,0.641315,0.743732,0.558595,0.761817,0.537185,0.792211,0.649677,0.889115,0.505435,0.596862,0.575711,0.981672,0.829405,0.818435,0.594735,0.585271,0.720011,0.576479,0.902687,0.704118,0.937779,0.582732,0.696734,0.969003,0.547940,0.553997,0.746711,0.938918,0.547660,0.676991,0.689165,0.813521,0.584486,0.800368,0.578217,0.958355,0.601397,0.743691,0.592963,0.589188,0.858832,0.560710,0.676112,0.991029,0.840305,0.760967,0.679703,0.792821,0.876231,0.723959,0.650743,0.795787,0.703063,0.586439,0.590642,0.796095,0.806947,0.934424,0.654126,0.909555,0.763091,0.824290,0.577800,0.611140,0.788931,0.923168,0.894790,0.599215,0.522524,0.543199,0.611620,0.606582,0.832956,0.507050,0.605021,0.562909,0.847733,0.654398,0.793056,0.820272,0.842569,0.609877,0.516375,0.577613,0.747240,0.527183,0.785764,0.981465,0.574907,0.607471,0.849202,0.529644,0.955416,0.514874,0.724852,0.531907,0.520782,0.606180,0.501157,0.894520,0.559983,0.654549,0.565381,0.537191,0.873200,0.790061,0.615632,0.580550,0.636218,0.601263,0.780870,0.769210,0.792682,0.936127,0.862643,0.993802,0.688186,0.638950,0.526765,0.836001,0.501964,0.967038,0.926161,0.942890,0.551864,0.748115,0.662458,0.800814,0.636642,0.586862,0.583709,0.642453,0.848286,0.541880,0.639778,0.645417,0.873798,0.516839,0.790312,0.774548,0.811064,0.578936,0.759350,0.579030,0.652643,0.813417,0.525763,0.978111,0.627632,0.997671,0.960442,0.639334,0.997225,0.893052,0.851773,0.678220,0.599482,0.567419,0.976751,0.812488,0.825280,0.537383,0.819492,0.971624,0.508132,0.956703,0.512748,0.984307,0.541946,0.601868,0.883899,0.564907,0.721469,0.852472,0.868443,0.675041,0.906989,0.795145,0.701579,0.760789,0.563115,0.513256,0.556303,0.900752,0.542431,0.789888,0.516731,0.748637,0.731546,0.538568,0.650449,0.896516,0.910623,0.670684,0.563171,0.890046,0.773447,0.669984,0.614229,0.510919,0.986003,0.671331,0.849331,0.612468,0.632438,0.545240,0.953899,0.806455,0.933986,0.536221,0.706887,0.584298,0.650931,0.803015,0.667296,0.886233,0.684495,0.595441,0.812109,0.929549,0.605286,0.637901,0.690594,0.597488,0.649071,0.536777,0.607390,0.572341,0.725794,0.600122,0.856522,0.545843,0.710632,0.654768,0.900865,0.513434,0.765454,0.725159,0.980522,0.764012,0.709293,0.700653,0.711830,0.717932,0.947376,0.697732,0.504204,0.530593,0.818113,0.539412,0.612099,0.529404,0.858799,0.509153,0.839858,0.525380,0.793561,0.983297,0.638699,0.566054,0.636808,0.778193,0.954757,0.951791,0.926678,0.615095,0.725429,0.881723,0.889430,0.771157,0.882046,0.738637,0.525873,0.904224,0.604754,0.728837,0.529785,0.630580,0.567363,0.667309,0.879209,0.714498,0.976157,0.541979,0.873449,0.689318,0.886880,0.583294,0.525880,0.975160,0.729321,0.878964,0.683180,0.549938,0.607559,0.716638,0.968064,0.591055,0.883221,0.990542,0.824638,0.717709,0.586689,0.571457,0.796225,0.597346,0.940395,0.547895,0.514185,0.654538,0.562760,0.924060,0.655758,0.681784,0.803392,0.726583,0.608247,0.519741,0.851822,0.968233,0.732868,0.824711,0.851833,0.884608,0.792067,0.515393,0.797009,0.799512,0.904134,0.839701,0.881456,0.920833,0.672848,0.726308,0.808797,0.512652,0.822702,0.543994,0.862821,0.628520,0.552670,0.726212,0.909685,0.570478,0.628600,0.598753,0.882492,0.602077,0.779726,0.846260,0.601267,0.510284,0.675774,0.610751,0.906560,0.510399,0.511915,0.702987,0.857053,0.606227,0.850133,0.796052,0.892608,0.602703,0.733900,0.688237,0.612910,0.782544,0.732536,0.864748,0.576176,0.912487,0.949473,0.595508,0.800945,0.595820,0.673824,0.706932,0.671063,0.518879,0.577683,0.534679,0.927813,0.529805,0.538166,0.693394,0.517516,0.541461,0.935715,0.906030,0.966941,0.764912,0.985043,0.948124,0.981616,0.811270,0.647792,0.543291,0.540301,0.799565,0.675859,0.952147,0.540531,0.915800,0.935107,0.679213,0.902132,0.542664,0.912856,0.902434,0.938081,0.622910,0.823295,0.681459,0.983962,0.769662,0.543372,0.500430,0.602802,0.682903,0.767274,0.746148,0.731418,0.690873,0.505700,0.793013,0.748966,0.835130,0.808891,0.656493,0.777209,0.852180,0.505012,0.795151,0.679251,0.575210,0.853657,0.909520,0.991140,0.797593,0.609390,0.950503,0.531053,0.573750,0.634889,0.890279,0.757265,0.569120,0.661211,0.525689,0.869993,0.911377,0.568887,0.947211,0.784998,0.570972,0.503855,0.506023,0.734914,0.773585,0.649660,0.838600,0.727961,0.568000,0.740862,0.669951,0.587751,0.588074,0.844191,0.726881,0.532564,0.509756,0.691447,0.945253,0.693449,0.852024,0.660792,0.803018,0.656171,0.519773,0.661292,0.663901,0.875993,0.649548,0.518112,0.851262,0.575828,0.526120,0.679570,0.764308,0.570727,0.945107,0.932720,0.702572,0.780966,0.940549,0.573515,0.737372,0.815847,0.578476,0.943694,0.522694,0.613903,0.764855,0.708964,0.607392,0.869205,0.549727,0.757488,0.864751,0.626523,0.922733,0.669623,0.519922,0.529717,0.829465,0.500162,0.618995,0.739451,0.821696,0.578830,0.543649,0.855582,0.655825,0.539263,0.906728,0.794546,0.686416,0.730991,0.562836,0.626379,0.591304,0.579303,0.895564,0.533998,0.615944,0.626644,0.946362,0.785776,0.754390,0.657432,0.514028,0.635093,0.612808,0.639274,0.590524,0.970986,0.635005,0.783883,0.696030,0.877065,0.537016,0.555509,0.780773,0.611831,0.715989,0.507607,0.750474,0.794206,0.993427,0.630038,0.696742,0.907362,0.553709,0.656465,0.908866,0.593068,0.714406,0.571115,0.512082,0.937484,0.682515,0.989869,0.674692,0.529059,0.697134,0.968854,0.912810,0.852157,0.574601,0.652586,0.548251,0.847649,0.534926,0.654981,0.933354,0.654496,0.603296,0.981069,0.569340,0.542378,0.641482,0.827808,0.959566,0.993948,0.669480,0.921359,0.599747,0.551156,0.935788,0.939900,0.860251,0.624297,0.561523,0.891458,0.752596,0.931229,0.819028,0.621068,0.770301,0.873071,0.556275,0.552373,0.616845,0.778634,0.643554,0.796127,0.620970,0.646985,0.779175,0.502258,0.530243,0.799518,0.534259,0.566826,0.798532,0.815860,0.675154,0.682385,0.970553,0.565435,0.612554,0.561974,0.511246,0.780199,0.889106,0.564428,0.628005,0.632312,0.569918,0.617976,0.928775,0.772112,0.523808,0.845314,0.639537,0.645481,0.524666,0.652541,0.762593,0.764171,0.596705,0.552216,0.625814,0.819797,0.996181,0.617550,0.908005,0.594021,0.727095,0.877181,0.914611,0.740175,0.885414,0.604525,0.598595,0.603036,0.972779,0.723719,0.922152,0.871662,0.583105,0.504803,0.715570,0.888761,0.561628,0.920244,0.672066,0.778454,0.839217,0.683391,0.942041,0.921665,0.783574,0.581621,0.808345,0.885694,0.834457,0.648004,0.601046,0.592503,0.975674,0.867191,0.575301,0.645156,0.676737,0.821383,0.822810,0.953423,0.590909,0.637139,0.844485,0.843005,0.712923,0.677157,0.719408,0.791991,0.838716,0.674748,0.559235,0.999740,0.539123,0.891273,0.557199,0.617341,0.540986,0.543938,0.935147,0.920597,0.522516,0.707620,0.505240,0.629630,0.807778,0.767358,0.802598,0.572802,0.507426,0.673140,0.980738,0.607977,0.713566,0.945611,0.643082,0.781666,0.819443,0.803414,0.821705,0.726047,0.527907,0.569079,0.988232,0.788571,0.522059,0.914343,0.831829,0.621450,0.563390,0.551581,0.874741,0.824570,0.693723,0.586207,0.560442,0.947024,0.939039,0.934927,0.689524,0.936692,0.712667,0.682295,0.566909,0.511554,0.562209,0.980010,0.953325,0.594491,0.668424,0.551257,0.774550,0.854869,0.628852,0.753170,0.886198,0.870779,0.747726,0.920311,0.632474,0.778533,0.606928,0.812924,0.913626,0.537675,0.692599,0.619318,0.509015,0.514114,0.996911,0.520981,0.606690,0.625468,0.569905,0.867119,0.813602,0.725866,0.772089,0.563728,0.715384,0.799667,0.847364,0.939476,0.971415,0.665410,0.811117,0.752457,0.696247,0.907450,0.855083,0.778938,0.537928,0.818759,0.746291,0.842763,0.881991,0.506955,0.904450,0.828655,0.652523,0.700019,0.975833,0.911205,0.732936,0.749440,0.695492,0.605692,0.503999,0.921610,0.686609,0.504192,0.621823,0.519042,0.904363,0.835397,0.959961,0.534337,0.522613,0.884789,0.986067,0.865368,0.620267,0.647190,0.704089,0.591237,0.903210,0.633747,0.617612,0.922742,0.894070,0.783298,0.653224,0.502132,0.864028,0.532959,0.653220,0.846834,0.528746,0.669715,0.949733,0.512219,0.770238,0.921511,0.630716,0.892099,0.921685,0.529537,0.529211,0.626520,0.948675,0.802061,0.881379,0.528642,0.607119,0.750619,0.616090,0.528521,0.935126,0.651345,0.820101,0.676062,0.591548,0.999944,0.562878,0.987784,0.594579,0.507865,0.889201,0.762143,0.847360,0.562576,0.883463,0.659755,0.663694,0.957247,0.957649,0.648572,0.932086,0.576943,0.794546,0.545177,0.640646,0.817773,0.529118,0.810668,0.937093,0.968293,0.560721,0.538737,0.552526,0.805567,0.506995,0.695580,0.713697,0.743322,0.983352,0.578390,0.900341,0.593329,0.559025,0.518993,0.956438,0.912216,0.823736,0.617360,0.924661,0.613075,0.721065,0.696453,0.551498,0.559656,0.986225,0.871858,0.651514,0.779820,0.942421,0.537871,0.546123,0.629221,0.562812,0.775015,0.530038,0.923688,0.504023,0.636284,0.777112,0.694361,0.663970,0.783868,0.603960,0.818519,0.593885,0.714508,0.958049,0.758082,0.664703,0.979292,0.627507,0.617979,0.837896,0.532721,0.635482,0.645268,0.607891,0.970462,0.677510,0.569732,0.686533,0.790841,0.605488,0.505679,0.527665,0.974823,0.511035,0.950619,0.698413,0.675805,0.561315,0.910244,0.514108,0.714500,0.548519,0.527406,0.795368,0.784209,0.511213,0.896304,0.794216,0.507144,0.994548,0.667875,0.993702,0.930929,0.874534,0.725570,0.961065,0.734433,0.733987,0.516862,0.548117,0.712729,0.684165,0.687732,0.822756,0.555935,0.564655,0.630999,0.636695,0.820674,0.918882,0.736989,0.581002,0.643554,0.571096,0.653127,0.793457,0.919428,0.655247,0.532568,0.812950,0.509719,0.582265,0.880413,0.830189,0.750586,0.941588,0.588464,0.673525,0.766918,0.804844,0.881304,0.603355,0.789059,0.500790,0.519224,0.744933,0.656599,0.685345,0.616346,0.523825,0.501539,0.675547,0.577765,0.568325,0.895754,0.796753,0.728997,0.599203,0.626508,0.780740,0.621265,0.562648,0.504809,0.597332,0.816118,0.508158,0.919095,0.836307,0.605962,0.957073,0.581987,0.644902,0.908427,0.962858,0.945659,0.788076,0.979818,0.934079,0.762277,0.511713,0.997465,0.985605,0.623273,0.735442,0.951529,0.616982,0.516044,0.649588,0.549808,0.582230,0.631303,0.680289,0.603369,0.737050,0.682507,0.509773,0.846772,0.920678,0.951173,0.658718,0.660915,0.797817,0.555767,0.504911,0.936157,0.804741,0.551628,0.715600,0.693992,0.981596,0.611346,0.515748,0.827875,0.600837,0.728414,0.688339,0.538102,0.561809,0.554785,0.909177,0.655345,0.502688,0.718189,0.781499,0.670298,0.601547,0.514031,0.656128,0.586145,0.569437,0.707297,0.709133,0.887876,0.626778,0.523003,0.690234,0.622616,0.976513,0.542185,0.550441,0.507460,0.678025,0.599165,0.627048,0.922399,0.680950,0.831668,0.604926,0.699817,0.993501,0.984315,0.503000,0.777500,0.997371,0.960100,0.858035,0.560726,0.818800,0.757759,0.563545,0.534116,0.525408,0.598441,0.635472,0.696596,0.621549,0.605215,0.607609,0.525445,0.544220,0.531470,0.779760,0.889016,0.662042,0.790074,0.529512,0.936598,0.538382,0.614422,0.792154,0.784472,0.771380,0.567056,0.787854,0.806351,0.683728,0.615377,0.783695,0.627418,0.526285,0.926258,0.501219,0.978131,0.506302,0.557988,0.749699,0.885410,0.751474,0.548343,0.596009,0.668308,0.522228,0.519344,0.574195,0.623459,0.569610,0.761224,0.919378,0.833549,0.814096,0.615729,0.513542,0.520163,0.768430,0.726914,0.750581,0.733728,0.970810,0.723229,0.844912,0.705193,0.831924,0.933677,0.969286,0.776468,0.851997,0.833907,0.906013,0.571333,0.948612,0.599138,0.522271,0.556720,0.956479,0.612937,0.644000,0.701056,0.545522,0.505678,0.683897,0.521797,0.617323,0.958477,0.763900,0.657142,0.772047,0.585533,0.828390,0.538107,0.773043,0.989926,0.738459,0.721773,0.881839,0.876448,0.829859,0.624836,0.667960,0.616380,0.844309,0.607280,0.550919,0.600996,0.723894,0.663516,0.606869,0.873836,0.982186,0.936039,0.825659,0.865523,0.627123,0.530989,0.524845,0.509474,0.525012,0.652310,0.563531,0.551904,0.689811,0.971048,0.534770,0.945082,0.703222,0.656178,0.960848,0.862917,0.858051,0.518287,0.956969,0.899799,0.828093,0.767472,0.630891,0.520222,0.551064,0.502776,0.716967,0.671671,0.926836,0.757305,0.588623,0.694574,0.854316,0.611851,0.762067,0.624523,0.644159,0.965234,0.713393,0.734972,0.628478,0.662180,0.647939,0.652511,0.934103,0.645850,0.744087,0.645667,0.725431,0.688815,0.508058,0.745995,0.737988,0.632600,0.988975,0.559404,0.886339,0.975285,0.504693,0.790930,0.657022,0.715801,0.598462,0.608949,0.921046,0.646979,0.535846,0.748446,0.977166,0.547507,0.578693,0.711441,0.873682,0.744342,0.508562,0.539644,0.647185,0.587197,0.583369,0.817965,0.517248,0.834048,0.512129,0.592226,0.668619,0.957743,0.989806,0.856472,0.664606,0.794371,0.790390,0.553672,0.985606,0.550841,0.982545,0.585961,0.911837,0.888538,0.528472,0.867098,0.578580,0.597237,0.857586,0.561699,0.719284,0.589219,0.829451,0.508730,0.747709,0.656372,0.631834,0.629057,0.577537,0.751862,0.760064,0.683070,0.515021,0.755190,0.604400,0.680870,0.506661,0.556028,0.815868,0.627989,0.710667,0.710409,0.584697,0.522944,0.966294,0.543182,0.709916,0.926035,0.792977,0.557033,0.876387,0.622174,0.508457,0.755989,0.589376,0.589160,0.762165,0.843879,0.525735,0.720161,0.882387,0.855439,0.832476,0.776157,0.707123,0.609936,0.509449,0.896327,0.642758,0.590241,0.598886,0.922224,0.524067,0.921452,0.832311,0.500233,0.810878,0.692641,0.899427,0.537216,0.813582,0.563333,0.522340,0.667277,0.594555,0.863544,0.772458,0.808227,0.651574,0.772765,0.850508,0.662519,0.521683,0.840391,0.507955,0.725533,0.607025,0.588420,0.634770,0.610555,0.830531,0.760765,0.714548,0.674189,0.759741,0.750089,0.950955,0.791505,0.681104,0.697300,0.603990,0.690054,0.783632,0.825676,0.516195,0.663883,0.758584,0.874939,0.846694,0.533462,0.573569,0.591833,0.557604,0.977585,0.862451,0.701313,0.842157,0.506925,0.911333,0.686937,0.992240,0.514565,0.626734,0.549615,0.762675,0.514403,0.652465,0.630978,0.569602,0.578941,0.972452,0.518837,0.794286,0.999234,0.539839,0.849008,0.618120,0.923138,0.646887,0.690098,0.628402,0.536539,0.665504,0.688219,0.576767,0.700375,0.792481,0.719885,0.685205,0.589483,0.886719,0.506226,0.891332,0.845086,0.577072,0.747849,0.635430,0.604457,0.952174,0.738181,0.513613,0.711418,0.981885,0.670989,0.575817,0.574514,0.706911,0.870879,0.608622,0.704545,0.579176,0.606527,0.833722,0.725151,0.842220,0.847889,0.681110,0.592986,0.524705,0.811020,0.553791,0.565262,0.516330,0.587599,0.512165,0.898049,0.615821,0.888781,0.505181,0.999162,0.605360,0.523536,0.510869,0.950603,0.548073,0.944306,0.538502,0.604799,0.713882,0.719730,0.620793,0.524620,0.598826,0.908644,0.789528,0.875095,0.746118,0.830178,0.510261,0.773567,0.916208,0.641146,0.837014,0.780858,0.602203,0.895666,0.507443,0.503255,0.628850,0.983065,0.520940,0.685752,0.780026,0.506673,0.544344,0.824163,0.504728,0.638354,0.616204,0.691342,0.841374,0.758643,0.656327,0.517213,0.539368,0.774021,0.973650,0.717599,0.999089,0.632639,0.660726,0.534763,0.792096,0.831150,0.823707,0.548879,0.572741,0.527239,0.716798,0.624407,0.638148,0.521635,0.972079,0.680653,0.774405,0.524358,0.561336,0.855962,0.563134,0.963798,0.886804,0.734423,0.623813,0.607184,0.674091,0.907548,0.520348,0.888905,0.866706,0.869994,0.941261,0.599185,0.812394,0.556684,0.892364,0.937126,0.859336,0.581402,0.955735,0.614730,0.501924,0.830402,0.878804,0.569547,0.567279,0.526142,0.922781,0.863150,0.816424,0.506428,0.554708,0.555568,0.547736,0.593839,0.665229,0.504409,0.836312,0.630158,0.649931,0.634077,0.503705,0.622674,0.927835,0.985019,0.640300,0.651687,0.885948,0.820337,0.941068,0.641214,0.592669,0.754506,0.739237,0.791194,0.563779,0.662359,0.518271,0.533987,0.613491,0.649305,0.787984,0.963870,0.940115,0.711695,0.907629,0.672304,0.775374,0.892343,0.745884,0.500393,0.669821,0.568357,0.714650,0.563764,0.505084,0.774364,0.801707,0.873776,0.644364,0.935469,0.587760,0.747238,0.702792,0.867156,0.759377,0.873980,0.755989,0.668735,0.705006,0.524148,0.862052,0.565267,0.639300,0.777031,0.766896,0.948933,0.504069,0.630922,0.609925,0.569003,0.818131,0.868998,0.676260,0.639066,0.584637,0.538965,0.930899,0.861120,0.606366,0.624200,0.806357,0.926962,0.997282,0.930262,0.783775,0.511214,0.685406,0.845423,0.636272,0.669454,0.725694,0.751900,0.630942,0.862789,0.513940,0.798336,0.596018,0.518411,0.833022,0.587730,0.853452,0.956869,0.513684,0.616907,0.695414,0.952697,0.795864,0.805826,0.602824,0.562086,0.679271,0.569570,0.544538,0.656335,0.667472,0.679451,0.599692,0.727436,0.603214,0.522565,0.847130,0.560032,0.960590,0.616701,0.729949,0.969873,0.882868,0.911741,0.565732,0.876822,0.959797,0.888382,0.820912,0.866107,0.516610,0.687991,0.628523,0.685954,0.601167,0.544313,0.683305,0.799250,0.720701,0.631899,0.586952,0.983187,0.854331,0.716214,0.554547,0.608182,0.965678,0.818435,0.966376,0.526267,0.684195,0.919219,0.803420,0.584851,0.723040,0.542628,0.539447,0.927826,0.666557,0.703263,0.604459,0.958836,0.624003,0.793849,0.693447,0.687897,0.949626,0.663950,0.976047,0.648670,0.532517,0.504417,0.832918,0.595380,0.542262,0.873623,0.511002,0.743149,0.771676,0.633980,0.551568,0.971212,0.846036,0.883247,0.958915,0.904905,0.982795,0.766534,0.833693,0.713602,0.660791,0.524672,0.831744,0.559376,0.696458,0.519915,0.948303,0.613241,0.723006,0.603797,0.774026,0.794309,0.782034,0.625776,0.501780,0.756632,0.650690,0.716929,0.712820,0.524178,0.591648,0.717702,0.554959,0.640095,0.591177,0.969208,0.920080,0.526398,0.666673,0.742584,0.780510,0.763355,0.542406,0.901044,0.718822,0.881062,0.597187,0.745060,0.672818,0.581548,0.674365,0.799685,0.671803,0.729562,0.539055,0.659952,0.746018,0.503173,0.735548,0.974483,0.848918,0.731729,0.964089,0.866120,0.787763,0.586934,0.754319,0.581976,0.573239,0.566798,0.598632,0.551236,0.917230,0.632455,0.753222,0.561269,0.987208,0.648908,0.794146,0.508920,0.656145,0.877857,0.785364,0.853744,0.632452,0.764287,0.906927,0.511827,0.896621,0.550230,0.553513,0.573367,0.501932,0.616420,0.664283,0.829970,0.861538,0.990994,0.975892,0.573514,0.685102,0.997157,0.875685,0.519762,0.511541,0.680436,0.650560,0.527078,0.649091,0.944901,0.523681,0.745751,0.970534,0.663176,0.565050,0.627372,0.902128,0.800729,0.758373,0.633510,0.855138,0.544983,0.823005,0.566118,0.556940,0.989396,0.863284,0.536288,0.751749,0.755905,0.610563,0.517502,0.808594,0.739305,0.634577,0.539401,0.652187,0.554198,0.721286,0.530193,0.534712,0.591043,0.699925,0.986725,0.549754,0.738201,0.923785,0.684329,0.522073,0.563399,0.551658,0.549669,0.584569,0.936421,0.650163,0.683264,0.979083,0.553113,0.527734,0.965976,0.989678,0.653250,0.963870,0.587546,0.807435,0.651533,0.640228,0.611510,0.526516,0.934326,0.775981,0.862309,0.976617,0.542933,0.515813,0.611755,0.954436,0.997426,0.675102,0.552047,0.764478,0.748437,0.667142,0.817041,0.880722,0.614686,0.940306,0.744653,0.570987,0.833361,0.512240,0.992665,0.683404,0.637152,0.559428,0.975192,0.741274,0.566428,0.730348,0.513099,0.657913,0.975453,0.824110,0.869710,0.854174,0.787010,0.768766,0.554222,0.524865,0.837578,0.680833,0.676228,0.579480,0.929842,0.960729,0.730727,0.943404,0.892718,0.502184,0.638837,0.810768,0.753376,0.865520,0.707762,0.836668,0.524375,0.580107,0.726624,0.610150,0.525686,0.819600,0.764412,0.617298,0.734287,0.655564,0.565485,0.777651,0.895548,0.904915,0.630172,0.659363,0.589195,0.927086,0.694654,0.726545,0.643916,0.675897,0.653768,0.605131,0.970749,0.574245,0.868230,0.714064,0.644860,0.537959,0.900854,0.878481,0.667558,0.737371,0.580858,0.965690,0.542218,0.737494,0.570037,0.991942,0.672058,0.560321,0.599491,0.653186,0.538720,0.611330,0.622485,0.750618,0.992108,0.885407,0.756515,0.848222,0.838846,0.769057,0.782972,0.789586,0.624290,0.584763,0.577318,0.850276,0.890422,0.544027,0.506213,0.775178,0.718934,0.972023,0.565219,0.607240,0.676042,0.698938,0.560471,0.655339,0.790811,0.634037,0.676726,0.562020,0.612938,0.966023,0.745435,0.691339,0.693902,0.754597,0.659697,0.507607,0.876664,0.648437,0.519851,0.594409,0.582961,0.926395,0.848658,0.802230,0.516761,0.508799,0.711201,0.703312,0.587279,0.930881,0.994343,0.773386,0.670724,0.820272,0.595652,0.909638,0.708671,0.527833,0.755815,0.964299,0.720655,0.588478,0.516242,0.872939,0.838275,0.736366,0.891799,0.606288,0.951852,0.758897,0.925550,0.632865,0.973481,0.664744,0.525844,0.951628,0.969895,0.602963,0.869936,0.978898,0.639097,0.529852,0.824801,0.866705,0.694395,0.570971,0.600522,0.927640,0.726254,0.747107,0.818886,0.691731,0.597715,0.540469,0.790789,0.669693,0.999719,0.562666,0.548695,0.799858,0.848750,0.625506,0.865644,0.888861,0.662923,0.897347,0.764615,0.652246,0.711766,0.847675,0.596782,0.734631,0.738677,0.625085,0.920440,0.810332,0.678078,0.693989,0.878564,0.600104,0.893909,0.692218,0.638069,0.545036,0.603318,0.858400,0.933421,0.896839,0.869659,0.560898,0.833109,0.775360,0.655269,0.539656,0.852111,0.791222,0.698529,0.520011,0.773203,0.554406,0.507907,0.712303,0.609791,0.903773,0.624233,0.576073,0.852112,0.889557,0.806364,0.528309,0.765857,0.502307,0.835160,0.598338,0.673043,0.952944,0.763540,0.698842,0.571592,0.525217,0.683793,0.723578,0.843817,0.715202,0.797059,0.611478,0.914176,0.681837,0.850890,0.621441,0.509370,0.664520,0.649080,0.526295,0.704719,0.961086,0.607244,0.524724,0.702992,0.785626,0.749509,0.651324,0.876694,0.614520,0.588612,0.740002,0.912075,0.591585,0.912571,0.782258,0.534469,0.682354,0.503786,0.642300,0.729791,0.804722,0.953042,0.547120,0.543062,0.636925,0.508984,0.609689,0.573336,0.850561,0.684819,0.610002,0.717957,0.566211,0.889006,0.785850,0.549481,0.728109,0.531422,0.652692,0.561433,0.922483,0.912028,0.704051,0.647493,0.838455,0.790884,0.966989,0.764089,0.575330,0.606820,0.547313,0.766132,0.532767,0.599880,0.657669,0.802034,0.838826,0.953814,0.898970,0.625300,0.534949,0.505365,0.915711,0.921629,0.583265,0.521061,0.907114,0.835032,0.867160,0.739313,0.669265,0.814273,0.501247,0.806538,0.926158,0.569552,0.943893,0.789013,0.765771,0.538819,0.608746,0.894653,0.610602,0.613328,0.650687,0.503489,0.575188,0.702972,0.908810,0.530938,0.535389,0.550443,0.507315,0.526728,0.847511,0.738501,0.817978,0.696358,0.552266,0.873408,0.920722,0.968668,0.532560,0.655709,0.614673,0.958596,0.979528,0.534040,0.529719,0.598504,0.573192,0.857987,0.657859,0.750645,0.955404,0.737076,0.602885,0.718807,0.811085,0.776328,0.565815,0.709446,0.544836,0.503196,0.610596,0.550053,0.996107,0.616910,0.564838,0.846753,0.594300,0.878861,0.620051,0.856775,0.983669,0.713930,0.684030,0.749260,0.857950,0.928121,0.535790,0.969166,0.739304,0.956942,0.702348,0.755386,0.664257,0.998401,0.761423,0.818893,0.736314,0.642532,0.624958,0.597073,0.549236,0.703872,0.503625,0.732862,0.598519,0.800029,0.524188,0.519861,0.826792,0.955997,0.656075,0.688271,0.888775,0.586129,0.540968,0.633743,0.645707,0.817168,0.831238,0.721316,0.747623,0.667678,0.762184,0.689232,0.979279,0.812968,0.597321,0.675192,0.623980,0.553391,0.985668,0.812567,0.826353,0.743152,0.540478,0.798327,0.624317,0.560050,0.876700,0.831578,0.574992,0.606517,0.634891,0.537130,0.768793,0.898166,0.804518,0.679671,0.700959,0.694333,0.758521,0.924775,0.505848,0.981725,0.661008,0.708061,0.683273,0.840130,0.893691,0.891416,0.881250,0.631865,0.531454,0.824300,0.652346,0.849734,0.568411,0.810935,0.573722,0.507613,0.637822,0.637770,0.808482,0.509651,0.509744,0.548628,0.576399,0.605506,0.517712,0.655679,0.517789,0.642443,0.650394,0.588268,0.783503,0.743125,0.512959,0.778914,0.508384,0.983419,0.778625,0.552724,0.692233,0.682886,0.850212,0.925642,0.704940,0.553525,0.552540,0.646592,0.835265,0.950705,0.927582,0.535507,0.838113,0.969584,0.997743,0.839708,0.648264,0.666517,0.512957,0.541074,0.568895,0.506100,0.668069,0.550696,0.620054,0.520232,0.921460,0.746273,0.642297,0.724321,0.656994,0.547196,0.567582,0.921860,0.648226,0.987480,0.577485,0.590680,0.723497,0.949013,0.787003,0.981367,0.624328,0.629941,0.698370,0.821215,0.712385,0.667327,0.828741,0.827126,0.566167,0.573625,0.851237,0.566209,0.688436,0.578686,0.758611,0.937539,0.809142,0.681844,0.707627,0.545200,0.641133,0.547587,0.836237,0.702401,0.548139,0.655725,0.875587,0.707964,0.648073,0.569920,0.610075,0.794080,0.825773,0.628510,0.516334,0.559980,0.696791,0.605838,0.615070,0.846971,0.801556,0.828980,0.535363,0.502362,0.858466,0.782849,0.714582,0.757440,0.647326,0.510340,0.607526,0.767375,0.642488,0.678919,0.940837,0.606312,0.756745,0.639968,0.693873,0.782334,0.925026,0.791925,0.997545,0.500952,0.583197,0.892093,0.682093,0.611856,0.743276,0.677224,0.982601,0.769150,0.781986,0.629411,0.845768,0.587670,0.873884,0.999368,0.718335,0.673631,0.743641,0.772052,0.506946,0.674759,0.910769,0.520058,0.981306,0.711343,0.999203,0.823930,0.907248,0.901952,0.641237,0.538151,0.570808,0.641332,0.730185,0.619400,0.514021,0.623929,0.502077,0.539431,0.596119,0.689055,0.852839,0.678887,0.692687,0.597093,0.516438,0.647074,0.751763,0.949070,0.822008,0.748801,0.900089,0.915956,0.883377,0.762928,0.685490,0.665450,0.698770,0.599739,0.771656,0.574570,0.799110,0.642217,0.828214,0.681521,0.939931,0.530331,0.914743,0.744722,0.579968,0.574753,0.860664,0.555705,0.579910,0.698586,0.550581,0.868476,0.551201,0.903088,0.657308,0.897883,0.778492,0.699647,0.839051,0.763299,0.782295,0.537023,0.784508,0.789848,0.670116,0.530795,0.551161,0.701016,0.998687,0.554518,0.626472,0.891781,0.805126,0.632043,0.502695,0.758450,0.645707,0.855105,0.595140,0.646097,0.583001,0.564012,0.555650,0.559006,0.896457,0.845151,0.681895,0.706962,0.698712,0.579113,0.889179,0.538362,0.695287,0.623270,0.661186,0.505862,0.653325,0.548581,0.849636,0.659134,0.969485,0.652262,0.971006,0.566913,0.992028,0.977227,0.659978,0.627521,0.705105,0.980778,0.710349,0.621339,0.921229,0.864445,0.577110,0.543462,0.819511,0.784912,0.805702,0.871288,0.886852,0.577985,0.588140,0.916351,0.905126,0.699064,0.598978,0.961486,0.534913,0.520508,0.539460,0.677720,0.587486,0.947655,0.806197,0.503288,0.509116,0.827898,0.812051,0.975470,0.852838,0.545111,0.748891,0.646455,0.522842,0.868665,0.777391,0.553933,0.533645,0.872842,0.626860,0.770925,0.638776,0.631004,0.591896,0.678850,0.558401,0.529184,0.903359,0.682845,0.971215,0.501759,0.564595,0.661772,0.807901,0.641413,0.688371,0.556724,0.564656,0.847843,0.697092,0.570462,0.891666,0.716796,0.511656,0.595570,0.892360,0.672228,0.928040,0.762544,0.718549,0.529775,0.960349,0.664537,0.720710,0.741201,0.526919,0.634152,0.802539,0.542354,0.612087,0.521282,0.665806,0.904550,0.990229,0.974709,0.845082,0.525427,0.750816,0.596382,0.725861,0.811728,0.531804,0.552193,0.926073,0.547331,0.983954,0.890424,0.735294,0.917595,0.557123,0.653601,0.519172,0.516982,0.953248,0.704025,0.714798,0.804947,0.554383,0.515580,0.937386,0.998379,0.901065,0.716472,0.603038,0.587422,0.545965,0.660760,0.511439,0.950669,0.728491,0.683798,0.866259,0.935758,0.730459,0.937228,0.566518,0.932341,0.695572,0.805590,0.902695,0.690231,0.779238,0.716513,0.964721,0.661017,0.508505,0.564336,0.536164,0.595040,0.629877,0.553192,0.537906,0.808191,0.995826,0.975261,0.831253,0.533870,0.585745,0.784030,0.569380,0.616208,0.712692,0.937937,0.720409,0.985268,0.655116,0.856887,0.655729,0.682966,0.952668,0.512477,0.991363,0.714042,0.759967,0.506295,0.708778,0.813339,0.643843,0.809784,0.837948,0.505496,0.766456,0.522229,0.792430,0.879009,0.605865,0.553888,0.933487,0.663593,0.615258,0.800269,0.564509,0.816063,0.884532,0.527456,0.753277,0.639421,0.853265,0.743804,0.893111,0.977758,0.676473,0.638762,0.644996,0.603242,0.805391,0.593955,0.905010,0.956889,0.811417,0.569106,0.785886,0.808203,0.776674,0.972195,0.516312,0.880450,0.987015,0.818507,0.522971,0.809778,0.514434,0.978767,0.623892,0.841375,0.986586,0.550252,0.515656,0.707544,0.651627,0.687927,0.654749,0.894922,0.647973,0.990722,0.812558,0.569580,0.654843,0.505017,0.669496,0.507551,0.886498,0.701202,0.541262,0.903414,0.710072,0.604956,0.821187,0.751541,0.842403,0.532712,0.695227,0.944180,0.802540,0.715189,0.521092,0.667105,0.694518,0.669972,0.594634,0.618042,0.648444,0.674021,0.662196,0.511437,0.673467,0.512293,0.979143,0.936409,0.917440,0.536163,0.786338,0.630798,0.790847,0.964152,0.881962,0.518732,0.598143,0.666798,0.718602,0.679690,0.599191,0.529156,0.564206,0.857895,0.725364,0.989491,0.524823,0.684200,0.750457,0.581889,0.632032,0.769963,0.524997,0.822237,0.928885,0.646528,0.583508,0.845296,0.600475,0.952157,0.949027,0.730284,0.983673,0.750560,0.798615,0.631307,0.775775,0.628094,0.999048,0.730844,0.713318,0.713040,0.769191,0.707532,0.573601,0.528448,0.551735,0.840848,0.634255,0.903490,0.702433,0.683760,0.579048,0.782373,0.976028,0.681073,0.870581,0.902637,0.870466,0.618330,0.757312,0.719792,0.735298,0.810908,0.992033,0.795210,0.629799,0.854181,0.571473,0.832777,0.715630,0.850160,0.500879,0.515676,0.719063,0.603238,0.513739,0.841032,0.536322,0.647035,0.822696,0.745132,0.638561,0.850643,0.582506,0.604008,0.867121,0.516209,0.644970,0.650542,0.990448,0.503977,0.850720,0.899548,0.940088,0.638771,0.863110,0.662263,0.546974,0.571921,0.740670,0.579199,0.891270,0.938780,0.697514,0.505106,0.617877,0.638414,0.715190,0.788022,0.512138,0.591781,0.856309,0.540023,0.844816,0.870364,0.511453,0.778410,0.615873,0.502545,0.724411,0.707041,0.700925,0.952772,0.547039,0.660593,0.818301,0.832384,0.688116,0.726062,0.837034,0.653668,0.749840,0.543696,0.568553,0.727871,0.902410,0.691492,0.840769,0.646443,0.510143,0.962875,0.604839,0.821492,0.729013,0.546526,0.981299,0.601579,0.615841,0.855978,0.856884,0.680246,0.526388,0.591954,0.576572,0.831221,0.666133,0.948604,0.652693,0.815691,0.684659,0.688928,0.567775,0.652351,0.991458,0.567060,0.736928,0.929384,0.983900,0.550069,0.894793,0.657752,0.506328,0.672585,0.520816,0.640076,0.669194,0.911842,0.587404,0.676326,0.576544,0.609377,0.719048,0.551406,0.533580,0.512235,0.579030,0.693655,0.506983,0.778758,0.581469,0.583997,0.738485,0.724181,0.906773,0.946355,0.762281,0.701699,0.814233,0.744253,0.694751,0.979530,0.516874,0.579505,0.560979,0.692191,0.518568,0.564925,0.633151,0.662290,0.936087,0.540046,0.514972,0.981738,0.886853,0.821385,0.650882,0.762091,0.636794,0.554583,0.961812,0.556343,0.838131,0.821007,0.812458,0.882487,0.777016,0.594745,0.705980,0.524623,0.697515,0.655491,0.598276,0.812156,0.510017,0.716363,0.788599,0.602929,0.940243,0.632403,0.626387,0.908873,0.686973,0.663569,0.887624,0.804935,0.501937,0.692904,0.601569,0.503271,0.558674,0.636339,0.554278,0.664650,0.697460,0.615118,0.925947,0.591272,0.946129,0.608269,0.568511,0.590179,0.960895,0.935607,0.735104,0.738439,0.742461,0.724656,0.895887,0.893175,0.633160,0.567656,0.648290,0.820862,0.786283,0.665963,0.612328,0.766128,0.636357,0.622813,0.769668,0.651030,0.897286,0.504444,0.605633,0.753289,0.786569,0.513460,0.556980,0.563588,0.949028,0.522127,0.775136,0.530384,0.611835,0.780826,0.989539,0.866724,0.838367,0.688689,0.838566,0.504542,0.513619,0.727193,0.527635,0.953298,0.829592,0.684648,0.526718,0.883606,0.977669,0.581461,0.840000,0.947183,0.773751,0.603652,0.540225,0.521352,0.828356,0.525455,0.753172,0.631564,0.718032,0.909361,0.749337,0.814534,0.834843,0.751580,0.706348,0.735654,0.678851,0.733784,0.700006,0.674606,0.687543,0.562344,0.755090,0.851677,0.810828,0.642589,0.753321,0.559279,0.974717,0.555659,0.559925,0.863397,0.521663,0.585173,0.771790,0.520902,0.687010,0.925330,0.635269,0.893463,0.815641,0.882243,0.553873,0.566874,0.579376,0.942025,0.514037,0.680524,0.533122,0.762329,0.640248,0.794806,0.659797,0.938230,0.601249,0.719526,0.587935,0.502782,0.539801,0.670740,0.710270,0.868390,0.845423,0.544586,0.755368,0.536580,0.641166,0.795544,0.626126,0.524746,0.693759,0.839092,0.529836,0.890257,0.804932,0.623138,0.604988,0.662887,0.771306,0.875878,0.651352,0.730960,0.538747,0.710172,0.648171,0.516341,0.566862,0.594883,0.524488,0.899685,0.812787,0.627429,0.916613,0.662988,0.772115,0.750635,0.959662,0.562262,0.543865,0.999871,0.656391,0.774539,0.513446,0.752256,0.798634,0.725824,0.955485,0.575158,0.922059,0.830346,0.606028,0.607925,0.931800,0.553734,0.580856,0.540438,0.871800,0.562463,0.937522,0.878497,0.672845,0.691211,0.626475,0.597819,0.580959,0.707464,0.861962,0.735738,0.520404,0.640713,0.715372,0.641701,0.688542,0.869186,0.539712,0.918447,0.530203,0.780794,0.821457,0.511060,0.674790,0.707459,0.995315,0.529918,0.886511,0.592086,0.673070,0.573993,0.510641,0.717461,0.951867,0.792537,0.621864,0.804875,0.527347,0.713949,0.860219,0.611440,0.523531,0.518773,0.959927,0.901008,0.540774,0.936613,0.682068,0.642426,0.851495,0.776451,0.541869,0.827166,0.645620,0.917945,0.526454,0.639277,0.653128,0.882745,0.662732,0.501065,0.935336,0.642117,0.694386,0.564423,0.724852,0.983725,0.833804,0.833465,0.703330,0.642074,0.642737,0.719083,0.673680,0.606051,0.687327,0.856664,0.650270,0.501084,0.501946,0.928953,0.516995,0.888814,0.912816,0.691428,0.542328,0.951691,0.987213,0.782806,0.503625,0.512325,0.611054,0.519299,0.744456,0.741312,0.845617,0.541218,0.919159,0.561308,0.557713,0.628415,0.628148,0.505255,0.881500,0.776601,0.500749,0.994006,0.759475,0.624399,0.685754,0.832468,0.980133,0.661115,0.630416,0.513711,0.777707,0.596080,0.875177,0.867070,0.930081,0.810897,0.654738,0.589675,0.894863,0.670705,0.716152,0.732366,0.842915,0.950928,0.985421,0.684192,0.641032,0.669037,0.793666,0.692170,0.575495,0.639485,0.874456,0.609713,0.527262,0.663612,0.844726,0.585181,0.598943,0.561473,0.966359,0.868035,0.543435,0.522733,0.917040,0.771380,0.962151,0.982149,0.702596,0.627059,0.530896,0.509660,0.584999,0.696354,0.523455,0.641154,0.580751,0.638588,0.650471,0.580892,0.538944,0.780466,0.740109,0.746358,0.631676,0.963455,0.725620,0.630587,0.936244,0.676473,0.698188,0.700856,0.525178,0.692866,0.662120,0.508591,0.641115,0.541595,0.669508,0.987354,0.514098,0.901014,0.744293,0.874094,0.702089,0.796273,0.610880,0.677967,0.558960,0.576992,0.610361,0.577598,0.704784,0.879590,0.558212,0.793262,0.749618,0.854295,0.643524,0.620237,0.594738,0.894783,0.798058,0.642309,0.821975,0.535741,0.693131,0.679166,0.645495,0.833871,0.549755,0.731738,0.720492,0.586105,0.642059,0.849883,0.917293,0.599339,0.843846,0.588257,0.916190,0.903859,0.660695,0.533854,0.601227,0.562844,0.640101,0.638288,0.985606,0.577543,0.805104,0.789104,0.697877,0.634803,0.554849,0.715566,0.734428,0.544507,0.627253,0.737859,0.526135,0.516564,0.548187,0.932081,0.999047,0.708472,0.581881,0.867421,0.846785,0.733553,0.892178,0.596459,0.975651,0.639092,0.696672,0.611360,0.625258,0.600540,0.696533,0.743080,0.568417,0.588108,0.568296,0.643669,0.619747,0.788721,0.952441,0.685151,0.829344,0.895200,0.695266,0.974174,0.956968,0.675459,0.501566,0.526294,0.543436,0.794885,0.558262,0.534976,0.748813,0.570210,0.661195,0.683434,0.566440,0.501742,0.649561,0.579104,0.684907,0.675938,0.632725,0.587779,0.714952,0.737653,0.562502,0.577586,0.908815,0.821501,0.589515,0.736912,0.928516,0.718433,0.947622,0.938860,0.896940,0.989123,0.518238,0.620049,0.884460,0.970658,0.879848,0.502514,0.579790,0.763229,0.589889,0.514762,0.751054,0.585394,0.991998,0.905681,0.581316,0.555529,0.679950,0.652988,0.629822,0.626653,0.545385,0.579081,0.996091,0.538455,0.586939,0.748305,0.857161,0.541709,0.778744,0.628958,0.778477,0.881517,0.868758,0.629310,0.619007,0.500918,0.546927,0.870815,0.796781,0.651766,0.872794,0.650137,0.777901,0.508167,0.660835,0.610801,0.997156,0.656066,0.622896,0.542670,0.699295,0.711698,0.569749,0.540892,0.793370,0.911199,0.911607,0.758392,0.690922,0.893964,0.620240,0.634914,0.738999,0.598941,0.913776,0.698155,0.788167,0.577250,0.612610,0.931936,0.504403,0.740169,0.855805,0.514967,0.563120,0.778501,0.673986,0.622848,0.691356,0.536023,0.500742,0.529801,0.628789,0.831361,0.511381,0.535977,0.704891,0.745169,0.663061,0.868538,0.689374,0.577770,0.747837,0.744090,0.809785,0.959509,0.968108,0.849894,0.565075,0.800769,0.713153,0.940964,0.888714,0.635007,0.623503,0.875738,0.596851,0.544878,0.987437,0.635876,0.607052,0.670486,0.655817,0.559402,0.512589,0.673396,0.516876,0.627946,0.546343,0.586271,0.673633,0.632177,0.586240,0.547595,0.718348,0.970252,0.819518,0.807449,0.717988,0.729042,0.812032,0.634870,0.958233,0.910609,0.659063,0.589546,0.782822,0.724457,0.580220,0.588742,0.720436,0.661927,0.617986,0.559294,0.701173,0.552840,0.568542,0.607812,0.971589,0.800346,0.620543,0.522158,0.646264,0.710640,0.505283,0.984426,0.841744,0.648770,0.839040,0.542811,0.555747,0.755019,0.576142,0.965502,0.991413,0.614818,0.589219,0.633826,0.576330,0.501382,0.529452,0.550866,0.730264,0.796019,0.523845,0.914752,0.824487,0.687189,0.620330,0.715561,0.861076,0.667445,0.954957,0.824955,0.893809,0.811004,0.541826,0.770085,0.655725,0.844995,0.993742,0.627932,0.721390,0.934957,0.521655,0.807040,0.606962,0.645539,0.594869,0.621640,0.559308,0.505583,0.535522,0.501895,0.703561,0.647201,0.745471,0.532820,0.781071,0.824799,0.908570,0.982228,0.566768,0.742339,0.637898,0.865020,0.861642,0.629905,0.600058,0.503683,0.634517,0.506415,0.594408,0.573472,0.596409,0.983941,0.929749,0.609350,0.532872,0.786460,0.576904,0.830111,0.529416,0.715177,0.609213,0.899579,0.757668,0.522297,0.849115,0.994026,0.557688,0.834914,0.506241,0.860404,0.736543,0.821116,0.922520,0.602726,0.573282,0.919252,0.595742,0.826545,0.694996,0.637990,0.596632,0.591779,0.890666,0.585157,0.993903,0.546589,0.797348,0.577106,0.872273,0.904799,0.556223,0.634311,0.990950,0.630228,0.986891,0.577605,0.988054,0.714468,0.670318,0.876412,0.671128,0.971402,0.578816,0.561666,0.561404,0.649987,0.813916,0.675406,0.576594,0.771038,0.554281,0.831453,0.611691,0.997060,0.537375,0.573823,0.510673,0.706349,0.891176,0.963893,0.882981,0.801026,0.636878,0.603296,0.782801,0.692453,0.569602,0.731978,0.575501,0.899152,0.511820,0.544206,0.545681,0.552789,0.552490,0.719876,0.765973,0.554822,0.523012,0.998705,0.763077,0.503546,0.703360,0.590228,0.889189,0.626820,0.750067,0.581136,0.599498,0.977448,0.573739,0.551931,0.584182,0.528300,0.772244,0.821472,0.940127,0.709852,0.568983,0.655589,0.921407,0.617897,0.634518,0.688167,0.643677,0.976698,0.965186,0.531228,0.816872,0.704808,0.912053,0.706052,0.917519,0.607725,0.549439,0.573986,0.936736,0.850733,0.909060,0.524374,0.516303,0.821547,0.763896,0.591887,0.584226,0.539057,0.915249,0.878072,0.520078,0.933524,0.537957,0.553096,0.571868,0.828835,0.655668,0.520408,0.676935,0.763824,0.595193,0.538877,0.834803,0.533499,0.561364,0.942907,0.556090,0.836835,0.836996,0.857591,0.633768,0.575844,0.810559,0.811599,0.990826,0.833516,0.812057,0.907754,0.609815,0.616256,0.560036,0.561004,0.798397,0.585951,0.866251,0.512224,0.896698,0.563459,0.585287,0.941364,0.701611,0.655571,0.545674,0.720315,0.830219,0.932199,0.948558,0.814749,0.691682,0.518211,0.545236,0.674721,0.744880,0.956156,0.674330,0.569606,0.771254,0.642974,0.999677,0.551991,0.632145,0.522661,0.891296,0.860933,0.540960,0.693031,0.943089,0.722590,0.516509,0.942200,0.967690,0.918901,0.727065,0.964804,0.676778,0.939128,0.981744,0.655532,0.533887,0.547515,0.746745,0.543219,0.576389,0.686221,0.837948,0.889909,0.934960,0.727711,0.847791,0.876245,0.642182,0.582727,0.567334,0.563046,0.596476,0.533560,0.544521,0.918906,0.571109,0.700566,0.984766,0.545720,0.512154,0.726681,0.552090,0.809309,0.579681,0.850396,0.867566,0.574149,0.530666,0.636437,0.660169,0.757155,0.594507,0.593678,0.925044,0.698024,0.693148,0.640704,0.904097,0.798967,0.506105,0.858469,0.737425,0.836872,0.859974,0.673072,0.938247,0.591864,0.880907,0.768769,0.949084,0.606105,0.654065,0.790676,0.930331,0.864100,0.571492,0.516712,0.791721,0.567189,0.780748,0.819535,0.969535,0.581926,0.928003,0.504903,0.945862,0.627477,0.612238,0.601840,0.594112,0.625946,0.867006,0.878291,0.893318,0.532110,0.950584,0.981718,0.563422,0.649627,0.578774,0.932372,0.926896,0.551025,0.941520,0.944650,0.593531,0.829884,0.847833,0.897376,0.573989,0.668014,0.533364,0.853094,0.632811,0.928166,0.597198,0.824557,0.967283,0.615504,0.559486,0.614249,0.622406,0.618545,0.538961,0.605209,0.547913,0.739306,0.855719,0.503626,0.952089,0.876871,0.748149,0.612594,0.913260,0.676546,0.561558,0.502742,0.768874,0.596414,0.766588,0.674890,0.561592,0.600116,0.717677,0.908678,0.777738,0.683002,0.784489,0.671732,0.653439,0.722262,0.666550,0.771748,0.561200,0.746195,0.829699,0.576557,0.780917,0.815024,0.681489,0.634567,0.741911,0.790713,0.931981,0.602017,0.567139,0.954661,0.950873,0.525667,0.838059,0.610191,0.600879,0.923207,0.686260,0.910441,0.599355,0.549271,0.877206,0.896429,0.515123,0.934788,0.642955,0.815898,0.746000,0.904673,0.720603,0.564637,0.824863,0.659563,0.580979,0.714993,0.964919,0.920848,0.744941,0.935541,0.964433,0.656224,0.508725,0.547047,0.632785,0.651735,0.666509,0.800308,0.745662,0.990272,0.925355,0.666337,0.937803,0.925541,0.921930,0.782149,0.851601,0.928739,0.594725,0.667170,0.839987,0.578476,0.691051,0.598971,0.966148,0.657478,0.610163,0.724884,0.845754,0.662623,0.558864,0.957766,0.503244,0.735770,0.742505,0.557787,0.887553,0.935474,0.691740,0.780819,0.550387,0.911934,0.817227,0.674464,0.882294,0.537265,0.843634,0.580767,0.708395,0.589293,0.664039,0.618766,0.781533,0.599870,0.773583,0.766579,0.839166,0.809721,0.577682,0.528576,0.628056,0.552182,0.629370,0.715675,0.714528,0.846209,0.940321,0.970216,0.675320,0.817751,0.742381,0.754887,0.507088,0.791781,0.583402,0.631550,0.680978,0.838276,0.626189,0.519142,0.894701,0.614547,0.996010,0.570068,0.965440,0.740503,0.675306,0.949253,0.602419,0.815346,0.668733,0.872727,0.611097,0.718068,0.513017,0.669405,0.586637,0.905697,0.657879,0.631372,0.953359,0.585075,0.628594,0.524914,0.634152,0.514972,0.734080,0.559643,0.672002,0.555194,0.762136,0.899700,0.973357,0.725367,0.867716,0.965034,0.595546,0.517124,0.507697,0.785428,0.597213,0.570916,0.654344,0.771258,0.604689,0.645688,0.681984,0.659864,0.795650,0.724896,0.969407,0.617231,0.697304,0.800066,0.956989,0.715057,0.670235,0.700140,0.931991,0.553009,0.588247,0.507094,0.679536,0.645401,0.817454,0.923681,0.589724,0.687574,0.628752,0.613479,0.995624,0.909761,0.791292,0.904678,0.670111,0.716169,0.958090,0.777756,0.871683,0.575814,0.596007,0.521398,0.561753,0.626495,0.917661,0.558139,0.649721,0.723317,0.836526,0.706030,0.759076,0.576163,0.570286,0.776377,0.771692,0.586729,0.630583,0.525018,0.611447,0.652375,0.893527,0.610475,0.666110,0.643586,0.913653,0.538554,0.655661,0.590343,0.544766,0.744394,0.616626,0.533961,0.793290,0.660272,0.591549,0.587584,0.648817,0.739713,0.737099,0.672530,0.577076,0.739331,0.982768,0.778587,0.539852,0.782098,0.702941,0.588214,0.518331,0.790044,0.580120,0.565186,0.963020,0.503198,0.717855,0.570891,0.727070,0.828122,0.612976,0.921926,0.932545,0.844915,0.708246,0.667560,0.897104,0.518507,0.882591,0.997087,0.578373,0.901385,0.675277,0.940565,0.897636,0.504597,0.985701,0.553987,0.933621,0.520705,0.593200,0.737070,0.554316,0.521388,0.638383,0.763999,0.690429,0.763957,0.526211,0.688051,0.768552,0.509863,0.585757,0.969551,0.511853,0.573694,0.939491,0.872959,0.513715,0.801764,0.981124,0.606864,0.589543,0.542270,0.616781,0.953786,0.732103,0.989192,0.888060,0.929556,0.679502,0.524077,0.652243,0.836864,0.706525,0.551825,0.527741,0.565086,0.975700,0.667922,0.958060,0.946556,0.919301,0.508011,0.680223,0.912853,0.884446,0.887316,0.997142,0.913332,0.624662,0.684045,0.789943,0.733565,0.735604,0.898509,0.692392,0.996391,0.611288,0.664634,0.563066,0.741813,0.965457,0.858986,0.734418,0.522413,0.715867,0.519252,0.500294,0.521414,0.960944,0.681462,0.709686,0.575600,0.990164,0.810907,0.761050,0.502019,0.513773,0.585396,0.977358,0.515913,0.505917,0.935063,0.763553,0.544669,0.994520,0.577426,0.601003,0.512481,0.587635,0.768735,0.559131,0.890293,0.566005,0.556979,0.939849,0.527996,0.525334,0.723853,0.551447,0.679395,0.725264,0.641496,0.976783,0.813527,0.747475,0.570669,0.514303,0.521259,0.863358,0.901658,0.918053,0.555784,0.593666,0.840169,0.628858,0.914019,0.530814,0.662337,0.519034,0.753052,0.764866,0.559192,0.500688,0.910217,0.616334,0.641662,0.840271,0.991830,0.920424,0.887921,0.760201,0.595978,0.581687,0.521064,0.594386,0.571434,0.592672,0.619378,0.633644,0.922715,0.961897,0.510973,0.538874,0.603265,0.904116,0.936182,0.607329,0.540239,0.891971,0.675262,0.848770,0.622735,0.557444,0.636904,0.502046,0.562672,0.844211,0.894147,0.862276,0.516065,0.604812,0.653693,0.661821,0.751529,0.569373,0.922613,0.715917,0.545514,0.704557,0.516406,0.649891,0.522283,0.613924,0.594551,0.696291,0.869200,0.698699,0.923446,0.925937,0.660692,0.849465,0.920010,0.527854,0.741303,0.759903,0.975938,0.876017,0.900721,0.920897,0.815162,0.743617,0.819095,0.680274,0.670971,0.532763,0.606761,0.909167,0.596308,0.741582,0.993842,0.661027,0.529633,0.890435,0.679263,0.516987,0.604808,0.842414,0.635061,0.687388,0.528662,0.510579,0.691405,0.520556,0.851168,0.601724,0.587430,0.627972,0.880592,0.662352,0.587751,0.809901,0.606798,0.616868,0.847450,0.535823,0.626599,0.985940,0.541726,0.749823,0.517000,0.542167,0.783107,0.582564,0.965952,0.895353,0.841401,0.983168,0.538870,0.773694,0.739161,0.533647,0.500898,0.786462,0.585847,0.909300,0.611555,0.851641,0.724617,0.586562,0.531828,0.538778,0.508967,0.547659,0.578569,0.734576,0.739429,0.839815,0.546244,0.701464,0.974253,0.594616,0.881078,0.978967,0.560581,0.828618,0.628143,0.652006,0.771193,0.913525,0.684593,0.690266,0.722467,0.891055,0.954217,0.556137,0.909357,0.955095,0.544121,0.878276,0.547042,0.785574,0.723861,0.689769,0.907539,0.624332,0.824807,0.562309,0.733024,0.535893,0.775944,0.538638,0.661196,0.646549,0.828775,0.691914,0.627237,0.699959,0.588240,0.789635,0.583800,0.518808,0.782851,0.894603,0.527446,0.993823,0.571355,0.524863,0.746454,0.511632,0.526751,0.792600,0.942733,0.830808,0.753487,0.566505,0.708020,0.821059,0.639422,0.826345,0.584504,0.679725,0.821720,0.525028,0.657916,0.785508,0.780876,0.542794,0.844049,0.524620,0.648661,0.620983,0.670424,0.532364,0.716214,0.972622,0.918470,0.661316,0.864274,0.733399,0.581475,0.852578,0.527433,0.535256,0.890873,0.577353,0.859245,0.865422,0.797804,0.580526,0.996173,0.597696,0.852243,0.542492,0.642756,0.715109,0.526329,0.760123,0.669668,0.573784,0.983280,0.896610,0.713385,0.744607,0.512660,0.696635,0.645708,0.764743,0.650706,0.774294,0.800121,0.721467,0.647497,0.580083,0.611302,0.665941,0.512846,0.563207,0.795006,0.778292,0.512219,0.606611,0.535692,0.716270,0.508208,0.816778,0.819995,0.784536,0.768588,0.855815,0.841268,0.645514,0.597850,0.640601,0.874743,0.634447,0.745716,0.868164,0.558257,0.677087,0.591791,0.508594,0.823270,0.555443,0.946044,0.698296,0.563310,0.671371,0.922480,0.728587,0.941366,0.829682,0.740838,0.740033,0.662760,0.826798,0.590974,0.553167,0.824210,0.776843,0.679169,0.830487,0.617590,0.555443,0.645098,0.600907,0.823495,0.538670,0.782496,0.762187,0.501204,0.725047,0.612729,0.801381,0.917129,0.639673,0.632799,0.807610,0.783702,0.633869,0.626403,0.710001,0.621604,0.610633,0.678038,0.631043,0.823020,0.781777,0.785883,0.882351,0.599879,0.544362,0.573197,0.629169,0.812968,0.820446,0.786599,0.848559,0.906835,0.511087,0.507990,0.577877,0.830160,0.570864,0.612837,0.839199,0.828069,0.946986,0.736411,0.548725,0.718050,0.568125,0.840515,0.666592,0.867477,0.624402,0.604390,0.852274,0.535932,0.630466,0.904799,0.797484,0.809277,0.634604,0.545589,0.764705,0.696235,0.609546,0.519966,0.801274,0.501929,0.712613,0.896831,0.605970,0.528838,0.687240,0.523524,0.670304,0.777610,0.519201,0.563398,0.848038,0.924318,0.578022,0.606787,0.718794,0.706344,0.651815,0.809936,0.600840,0.546176,0.682266,0.881390,0.548675,0.802734,0.821736,0.735163,0.501501,0.967958,0.682454,0.524112,0.656329,0.936639,0.566951,0.511720,0.685516,0.946703,0.935587,0.935830,0.509566,0.685054,0.524775,0.800780,0.591532,0.729698,0.567571,0.721921,0.878973,0.787683,0.563891,0.797589,0.761584,0.750814,0.568721,0.526334,0.553074,0.609258,0.739140,0.982446,0.524488,0.713107,0.900058,0.783862,0.696298,0.575797,0.578947,0.576660,0.524957,0.952846,0.866745,0.823877,0.986701,0.563773,0.838333,0.648856,0.989331,0.970104,0.784188,0.536439,0.979748,0.574157,0.878275,0.701988,0.760383,0.679243,0.788709,0.776119,0.535970,0.671697,0.755442,0.783050,0.886280,0.703639,0.674831,0.723373,0.753551,0.994362,0.984544,0.949043,0.523934,0.948553,0.812828,0.614012,0.732254,0.621593,0.781813,0.620407,0.807953,0.886060,0.590869,0.900421,0.504359,0.670932,0.519739,0.506995,0.804531,0.880894,0.540995,0.822815,0.537692,0.599900,0.796954,0.886931,0.778623,0.916430,0.521155,0.568642,0.556594,0.860426,0.875852,0.545841,0.716876,0.599279,0.596512,0.622241,0.940868,0.533837,0.535784,0.557119,0.771931,0.668716,0.789097,0.687751,0.582486,0.992195,0.684818,0.586337,0.937220,0.771888,0.681248,0.979252,0.688619,0.679308,0.504654,0.760317,0.854794,0.816298,0.968829,0.942129,0.558268,0.562852,0.917729,0.925402,0.955176,0.699558,0.624779,0.666379,0.778040,0.629281,0.892755,0.532659,0.729602,0.603778,0.557493,0.597529,0.508636,0.900292,0.515880,0.611196,0.550912,0.697473,0.874505,0.842362,0.773377,0.539620,0.643806,0.824235,0.620323,0.629228,0.696514,0.610813,0.908931,0.545400,0.549230,0.572401,0.600774,0.709906,0.568896,0.680222,0.618859,0.808827,0.672098,0.568019,0.536581,0.902202,0.729783,0.764485,0.686044,0.666791,0.561016,0.632242,0.875579,0.520124,0.912732,0.971225,0.836761,0.504338,0.614900,0.833580,0.927315,0.832212,0.768085,0.681343,0.593616,0.506225,0.731822,0.744508,0.801778,0.639783,0.624675,0.571791,0.685831,0.589582,0.524562,0.527736,0.699705,0.680668,0.500028,0.815860,0.933054,0.794935,0.764674,0.826463,0.999667,0.991923,0.686769,0.959084,0.707161,0.596730,0.647826,0.611405,0.891245,0.985165,0.667711,0.588904,0.713365,0.540266,0.613822,0.614771,0.586802,0.536660,0.844045,0.628524,0.630942,0.678280,0.763121,0.624049,0.809676,0.932919,0.761022,0.692030,0.637257,0.577629,0.507071,0.689128,0.723373,0.579280,0.860372,0.825340,0.560305,0.564175,0.679583,0.570825,0.963399,0.628463,0.743669,0.807102,0.550667,0.824251,0.701620,0.996611,0.730089,0.537521,0.743609,0.577651,0.570529,0.518220,0.877631,0.504969,0.599422,0.951605,0.954772,0.759637,0.758003,0.523153,0.896884,0.889050,0.835544,0.687184,0.526584,0.768375,0.634123,0.650246,0.760763,0.618240,0.880229,0.577661,0.822837,0.669581,0.723938,0.930400,0.658259,0.586131,0.701521,0.924866,0.557082,0.589376,0.784932,0.559275,0.996258,0.724439,0.550122,0.770639,0.735753,0.704522,0.889594,0.594172,0.770479,0.635027,0.669225,0.736847,0.552865,0.594212,0.535447,0.786659,0.520583,0.932222,0.691025,0.647693,0.781455,0.752962,0.624842,0.867666,0.662879,0.811708,0.605176,0.948122,0.660992,0.727481,0.667286,0.755904,0.532055,0.630287,0.952783,0.538918,0.522664,0.920659,0.731770,0.767166,0.850172,0.559780,0.759418,0.760294,0.669713,0.514922,0.573612,0.521293,0.654718,0.634686,0.607263,0.614123,0.625576,0.863440,0.819603,0.901295,0.769139,0.530508,0.845559,0.848486,0.553869,0.893997,0.812356,0.544330,0.990342,0.803168,0.527555,0.724754,0.966637,0.729822,0.903773,0.548704,0.743115,0.804553,0.907811,0.569179,0.568950,0.695590,0.562188,0.872039,0.558519,0.771841,0.825703,0.826339,0.595856,0.591963,0.787891,0.675077,0.628500,0.528134,0.609580,0.766452,0.631785,0.899679,0.711844,0.696967,0.833314,0.895100,0.526003,0.502359,0.559055,0.539837,0.753379,0.816612,0.706242,0.811248,0.673092,0.530447,0.691721,0.717506,0.887825,0.717082,0.635429,0.999161,0.624081,0.603512,0.928001,0.519820,0.628619,0.825640,0.906530,0.880498,0.878005,0.902325,0.848791,0.810784,0.873120,0.841328,0.784752,0.866428,0.570826,0.757614,0.774873,0.617974,0.812682,0.513470,0.833469,0.554250,0.798518,0.512849,0.584363,0.593305,0.602399,0.824990,0.941466,0.886247,0.656045,0.525748,0.532051,0.747390,0.764121,0.607291,0.864104,0.504195,0.562114,0.556575,0.650280,0.630412,0.746419,0.761863,0.614804,0.754439,0.829157,0.523533,0.986157,0.591602,0.565900,0.539924,0.561470,0.511142,0.998220,0.614495,0.637819,0.686361,0.910899,0.794486,0.801160,0.554325,0.665287,0.545699,0.667017,0.513551,0.691831,0.876151,0.887716,0.529295,0.562368,0.623588,0.515848,0.753025,0.552270,0.626501,0.666424,0.626753,0.711361,0.622557,0.869799,0.950066,0.815733,0.734916,0.889989,0.547263,0.964126,0.589920,0.503558,0.810850,0.612928,0.936074,0.971432,0.932977,0.535048,0.756594,0.500795,0.866812,0.524949,0.901601,0.809873,0.712581,0.579271,0.646027,0.792000,0.919173,0.716115,0.744823,0.698840,0.616243,0.865892,0.600772,0.728755,0.519803,0.693129,0.667471,0.506755,0.990251,0.641686,0.733603,0.800031,0.811583,0.783138,0.886437,0.596671,0.603988,0.663242,0.826838,0.574530,0.545462,0.806711,0.519641,0.916755,0.740016,0.683032,0.981567,0.848833,0.938232,0.814919,0.969200,0.543363,0.671891,0.671043,0.533645,0.678721,0.621841,0.930737,0.989863,0.516567,0.607510,0.569629,0.639886,0.718393,0.993801,0.691221,0.969155,0.791472,0.664057,0.844404,0.583921,0.567766,0.861961,0.895085,0.610000,0.567601,0.742905,0.700572,0.718261,0.507611,0.519448,0.528965,0.863974,0.941258,0.506331,0.649810,0.553752,0.694971,0.794765,0.913790,0.740057,0.645447,0.963674,0.614826,0.633392,0.569310,0.912693,0.783104,0.544862,0.862457,0.544647,0.774745,0.852933,0.765609,0.841538,0.600220,0.730751,0.956319,0.618693,0.513747,0.521393,0.858127,0.860784,0.710827,0.705676,0.737027,0.526307,0.687642,0.911471,0.535844,0.676254,0.534374,0.894940,0.961902,0.834269,0.577890,0.628073,0.839036,0.995647,0.969720,0.566683,0.771466,0.614146,0.784754,0.603025,0.752335,0.557216,0.619435,0.501666,0.615212,0.692274,0.920926,0.552866,0.570261,0.688297,0.522683,0.571673,0.950014,0.607901,0.509948,0.743534,0.981633,0.888057,0.889437,0.861019,0.730467,0.715772,0.528130,0.896129,0.591197,0.524188,0.659710,0.563890,0.934650,0.914068,0.576342,0.618045,0.696219,0.563243,0.548442,0.753640,0.772388,0.956384,0.686717,0.815844,0.928781,0.517332,0.810167,0.978908,0.507688,0.785605,0.610647,0.596551,0.611855,0.674756,0.525376,0.682360,0.900842,0.624970,0.816355,0.810647,0.782858,0.532873,0.682018,0.732670,0.846054,0.616872,0.920082,0.837536,0.615012,0.769301,0.556373,0.514589,0.653168,0.921942,0.517041,0.663797,0.920464,0.893090,0.779335,0.515267,0.666196,0.515992,0.530920,0.659625,0.932108,0.584509,0.778931,0.910553,0.757225,0.701712,0.764619,0.547042,0.643598,0.749798,0.510578,0.767896,0.989832,0.500078,0.534730,0.587367,0.604881,0.751524,0.882172,0.798512,0.729088,0.508963,0.762827,0.879101,0.529921,0.650450,0.929013,0.952178,0.847850,0.948085,0.507129,0.673343,0.525864,0.596235,0.508124,0.694152,0.943813,0.520416,0.852721,0.623081,0.554471,0.748103,0.726046,0.632981,0.968898,0.793832,0.553711,0.712451,0.675636,0.747584,0.504292,0.947513,0.686770,0.656828,0.726020,0.941658,0.626604,0.711223,0.538395,0.522267,0.783744,0.566537,0.667410,0.704571,0.824970,0.695523,0.698817,0.524662,0.646183,0.611311,0.530084,0.788724,0.895887,0.577157,0.786090,0.978896,0.867886,0.621074,0.656291,0.626001,0.532042,0.747969,0.500962,0.899171,0.558757,0.660943,0.504598,0.713143,0.984147,0.632769,0.786975,0.958885,0.627554,0.783195,0.990325,0.620993,0.698092,0.549716,0.613321,0.866081,0.708382,0.708565,0.659832,0.506670,0.695152,0.546375,0.698513,0.669917,0.920367,0.636035,0.746900,0.829710,0.538538,0.691678,0.653758,0.959257,0.659679,0.622030,0.927170,0.518830,0.603600,0.984208,0.882055,0.930213,0.744683,0.876811,0.708143,0.551491,0.672311,0.602222,0.623163,0.704253,0.518112,0.977081,0.848976,0.534009,0.583465,0.751472,0.884930,0.848075,0.535680,0.586460,0.500373,0.577953,0.849083,0.644785,0.514251,0.770570,0.738343,0.618905,0.853571,0.573923,0.750723,0.684322,0.561278,0.505427,0.558612,0.566666,0.751734,0.722036,0.523968,0.561636,0.575050,0.708268,0.714544,0.805343,0.545355,0.524267,0.853517,0.501286,0.592235,0.658906,0.573183,0.889092,0.582018,0.900329,0.721460,0.719083,0.787511,0.756572,0.684085,0.755041,0.614388,0.798367,0.516984,0.733283,0.795003,0.601246,0.601899,0.531811,0.663334,0.795835,0.847275,0.613574,0.626918,0.567328,0.766176,0.518194,0.846137,0.698283,0.526860,0.852421,0.567545,0.572878,0.526741,0.650752,0.500135,0.593904,0.594218,0.522865,0.932276,0.581655,0.599887,0.902004,0.732406,0.531682,0.511842,0.914892,0.682047,0.973459,0.833667,0.686228,0.654771,0.918985,0.884790,0.519158,0.634581,0.598853,0.635443,0.537205,0.810537,0.778242,0.621094,0.546958,0.856893,0.701531,0.838418,0.669977,0.721519,0.988901,0.661459,0.903718,0.844941,0.732339,0.665326,0.632022,0.624660,0.508984,0.510979,0.714499,0.862186,0.888016,0.714052,0.736718,0.640711,0.922479,0.545911,0.811474,0.571855,0.593534,0.694244,0.566409,0.535362,0.513522,0.698715,0.595673,0.952707,0.692370,0.523566,0.737651,0.609914,0.628375,0.630305,0.753622,0.541863,0.733078,0.512725,0.824798,0.937541,0.765026,0.740423,0.925773,0.738418,0.563808,0.706606,0.906668,0.606384,0.903498,0.988752,0.781289,0.779036,0.802752,0.628684,0.655024,0.719609,0.674204,0.681746,0.888010,0.744954,0.646030,0.889543,0.643220,0.795739,0.908867,0.548661,0.833956,0.566923,0.559209,0.650333,0.516440,0.575925,0.594031,0.664757,0.588422,0.659745,0.702565,0.976303,0.667613,0.613646,0.608993,0.705266,0.515626,0.588630,0.580691,0.737189,0.657653,0.585104,0.773461,0.618710,0.580297,0.798514,0.614643,0.756183,0.536392,0.545116,0.568594,0.785774,0.688631,0.666214,0.770631,0.529845,0.669493,0.990343,0.558940,0.896953,0.769332,0.613169,0.520914,0.872251,0.559130,0.757218,0.729300,0.672027,0.596592,0.926428,0.875851,0.678219,0.876351,0.602403,0.625813,0.839692,0.509112,0.744403,0.510275,0.590074,0.725443,0.962339,0.670440,0.888327,0.865459,0.600178,0.769657,0.525184,0.564156,0.995976,0.621214,0.500604,0.760863,0.671061,0.731543,0.589022,0.603678,0.560488,0.832913,0.553894,0.613987,0.944450,0.586661,0.882597,0.740862,0.706217,0.680770,0.837992,0.969088,0.762899,0.654142,0.547704,0.823629,0.527220,0.507967,0.750251,0.898333,0.545642,0.522815,0.562116,0.655787,0.536353,0.772373,0.798463,0.577479,0.572072,0.971078,0.790133,0.783199,0.837207,0.971173,0.702171,0.842324,0.676323,0.574631,0.666292,0.741769,0.719085,0.692798,0.519963,0.570332,0.789204,0.945166,0.584749,0.810520,0.520259,0.529379,0.616048,0.681629,0.980854,0.632814,0.703009,0.515662,0.571940,0.775472,0.938897,0.806743,0.721887,0.804377,0.549630,0.782857,0.546115,0.648967,0.571376,0.841694,0.515169,0.586397,0.659210,0.985085,0.903196,0.888610,0.685178,0.912344,0.556012,0.840208,0.711784,0.536404,0.812350,0.673010,0.636397,0.884426,0.530678,0.975178,0.850549,0.571212,0.748741,0.888074,0.682208,0.527528,0.878497,0.762675,0.883409,0.577464,0.509145,0.527016,0.722763,0.629183,0.926943,0.582844,0.941971,0.973599,0.933314,0.860450,0.941430,0.625626,0.790418,0.596003,0.526198,0.535454,0.847388,0.951087,0.985564,0.743691,0.776169,0.723750,0.799335,0.646436,0.534538,0.652392,0.681931,0.655501,0.510316,0.650785,0.978070,0.984566,0.707864,0.996973,0.554205,0.529674,0.673796,0.696528,0.651922,0.731223,0.817425,0.973973,0.688400,0.678785,0.782771,0.968425,0.728306,0.612630,0.657107,0.720618,0.578577,0.838448,0.811048,0.618279,0.960321,0.633201,0.945317,0.542565,0.816299,0.828429,0.531272,0.612723,0.827016,0.592312,0.729872,0.987411,0.618148,0.540796,0.562891,0.679527,0.884357,0.535983,0.553611,0.845836,0.645360,0.977125,0.836892,0.650576,0.567968,0.518095,0.528659,0.892751,0.978203,0.928513,0.941531,0.684712,0.891935,0.839573,0.902856,0.702000,0.665943,0.722609,0.812530,0.612483,0.922135,0.685888,0.594782,0.575626,0.858930,0.575476,0.767738,0.737651,0.616490,0.786890,0.669780,0.696984,0.778638,0.967491,0.773141,0.617465,0.879091,0.701442,0.648213,0.639249,0.996380,0.880988,0.541875,0.821121,0.937481,0.645732,0.679253,0.639934,0.705142,0.572416,0.539725,0.669333,0.617146,0.683706,0.516525,0.562720,0.761063,0.856522,0.574527,0.971197,0.561755,0.526694,0.519628,0.914027,0.843047,0.870459,0.631316,0.805758,0.944062,0.975098,0.694901,0.538188,0.945180,0.654106,0.590262,0.653815,0.763500,0.864407,0.601791,0.518509,0.776375,0.522389,0.867733,0.840321,0.504367,0.897209,0.867082,0.511564,0.653252,0.635808,0.571261,0.785266,0.882434,0.820118,0.566763,0.623604,0.648014,0.528727,0.634221,0.526750,0.828045,0.902995,0.897492,0.990610,0.696627,0.556810,0.617599,0.808950,0.872763,0.591359,0.578471,0.748975,0.714083,0.590507,0.684486,0.693214,0.538209,0.516428,0.767391,0.870341,0.996875,0.518415,0.540587,0.897143,0.754880,0.777176,0.579589,0.850419,0.546840,0.735374,0.583775,0.534167,0.752431,0.619336,0.723675,0.895064,0.920127,0.586878,0.603338,0.541932,0.520180,0.996755,0.972088,0.646620,0.922643,0.512931,0.636257,0.592635,0.516764,0.988477,0.960651,0.892366,0.513130,0.940139,0.654644,0.504940,0.855846,0.862446,0.740275,0.624672,0.948857,0.796111,0.912829,0.933900,0.910646,0.550556,0.968373,0.585260,0.562012,0.821210,0.758415,0.525165,0.837909,0.627092,0.627375,0.537510,0.710968,0.892082,0.521186,0.768968,0.500507,0.536952,0.829254,0.668474,0.671038,0.510433,0.525149,0.690195,0.892882,0.503265,0.788019,0.851589,0.569516,0.810336,0.583594,0.938695,0.531644,0.724670,0.563170,0.544833,0.564233,0.541785,0.956988,0.538254,0.619701,0.563775,0.599756,0.637706,0.515042,0.769554,0.991232,0.539130,0.938736,0.700897,0.628086,0.783395,0.656801,0.623399,0.954020,0.611065,0.953265,0.788498,0.714478,0.932083,0.854117,0.535630,0.581698,0.550722,0.517841,0.905494,0.587134,0.818307,0.671402,0.784886,0.576211,0.851645,0.519697,0.665696,0.604359,0.595978,0.503807,0.901191,0.722955,0.672589,0.558299,0.551966,0.709464,0.585516,0.561225,0.735043,0.997741,0.676225,0.505100,0.853222,0.670964,0.857665,0.944374,0.597383,0.554338,0.521241,0.714246,0.552563,0.521363,0.877404,0.648211,0.508015,0.979367,0.615882,0.603811,0.569244,0.547944,0.666845,0.595970,0.974849,0.846991,0.710656,0.895166,0.653001,0.874722,0.646488,0.653415,0.803600,0.808300,0.641843,0.505091,0.516922,0.733864,0.781224,0.631360,0.671856,0.547716,0.841401,0.694438,0.605627,0.789180,0.771130,0.912021,0.894467,0.534663,0.673540,0.541079,0.699143,0.628059,0.662396,0.511454,0.905713,0.621205,0.542130,0.587337,0.654495,0.634208,0.524656,0.642203,0.559214,0.504057,0.989782,0.506550,0.931119,0.518583,0.687754,0.766179,0.621095,0.574685,0.920507,0.509515,0.766504,0.622764,0.694726,0.588762,0.859177,0.526765,0.545838,0.705870,0.926516,0.570227,0.704849,0.655358,0.725467,0.536304,0.643008,0.749379,0.847723,0.958621,0.905291,0.549484,0.535445,0.745095,0.647599,0.831985,0.727715,0.897930,0.557200,0.952109,0.612614,0.543197,0.526726,0.511390,0.718896,0.707194,0.716659,0.617556,0.659647,0.769205,0.516188,0.732399,0.710538,0.517773,0.645315,0.971456,0.844989,0.607370,0.509310,0.582856,0.977692,0.644252,0.528032,0.661024,0.590401,0.566610,0.928749,0.908668,0.715803,0.780274,0.574526,0.555134,0.559844,0.718841,0.864896,0.885182,0.588239,0.602394,0.661934,0.603770,0.603678,0.546162,0.849849,0.606198,0.947642,0.623087,0.615110,0.601933,0.746615,0.874262,0.586397,0.539628,0.740610,0.650137,0.794147,0.852964,0.511228,0.690761,0.773648,0.573353,0.717107,0.845967,0.782277,0.707619,0.508267,0.830305,0.548262,0.956790,0.511176,0.536207,0.526534,0.725305,0.780716,0.766551,0.817428,0.610698,0.892684,0.571096,0.692353,0.514640,0.978861,0.526362,0.806440,0.546862,0.667092,0.561673,0.932505,0.696180,0.516127,0.696774,0.555247,0.739961,0.745412,0.888617,0.560898,0.993027,0.575068,0.593832,0.719174,0.806216,0.692319,0.905546,0.808166,0.774236,0.701524,0.816133,0.531779,0.882165,0.962116,0.508362,0.865080,0.591201,0.826954,0.518466,0.645051,0.726503,0.516434,0.628080,0.765002,0.631793,0.813060,0.801634,0.611204,0.719745,0.657698,0.940890,0.507166,0.594017,0.829830,0.645363,0.507132,0.829677,0.689622,0.874244,0.775390,0.599691,0.665898,0.540806,0.570351,0.999784,0.785347,0.790230,0.555902,0.784805,0.698371,0.570528,0.684400,0.936718,0.729246,0.522903,0.948607,0.500829,0.733086,0.683763,0.578956,0.550413,0.819719,0.732336,0.529121,0.592867,0.682611,0.547527,0.734015,0.585869,0.652851,0.723894,0.764818,0.589690,0.879307,0.926907,0.687288,0.901396,0.664023,0.987689,0.803878,0.547508,0.551601,0.610893,0.541561,0.675682,0.503921,0.544449,0.724474,0.839051,0.810896,0.619729,0.912645,0.515261,0.836155,0.672984,0.858614,0.824519,0.617397,0.777206,0.562039,0.986388,0.675070,0.638211,0.633309,0.837715,0.827819,0.533500,0.576654,0.592076,0.602880,0.725864,0.774745,0.612533,0.796931,0.913326,0.645764,0.711220,0.685831,0.533301,0.916178,0.663378,0.618469,0.683906,0.565405,0.790613,0.695340,0.898848,0.589217,0.521369,0.914786,0.540433,0.559706,0.533950,0.586008,0.742555,0.747808,0.754681,0.926597,0.501380,0.688425,0.931010,0.880750,0.716625,0.668387,0.925797,0.948417,0.982437,0.763590,0.999967,0.514122,0.518593,0.568452,0.730919,0.526133,0.539183,0.818314,0.612406,0.819225,0.771919,0.753214,0.656515,0.971739,0.609383,0.840641,0.554481,0.571578,0.975907,0.623753,0.515625,0.591482,0.698189,0.563700,0.718050,0.848953,0.947920,0.771906,0.595499,0.561052,0.941564,0.519719,0.543831,0.887000,0.562004,0.692138,0.659173,0.525268,0.659197,0.761768,0.502982,0.771357,0.676939,0.728581,0.938795,0.519602,0.573524,0.869172,0.801793,0.786596,0.734455,0.546745,0.947344,0.757147,0.733345,0.516255,0.673992,0.870791,0.919631,0.739350,0.600321,0.795519,0.528756,0.947470,0.726360,0.798598,0.510587,0.737209,0.617149,0.901434,0.516786,0.650352,0.644058,0.724321,0.502773,0.657889,0.557045,0.873645,0.623232,0.629792,0.691007,0.811446,0.541658,0.574225,0.510365,0.646330,0.761348,0.551482,0.867424,0.512901,0.589167,0.711970,0.810280,0.610494,0.600501,0.700860,0.866224,0.852913,0.532124,0.652990,0.537856,0.842600,0.610082,0.706320,0.655727,0.503697,0.546297,0.894494,0.564612,0.861146,0.588893,0.882144,0.621671,0.580512,0.779763,0.723640,0.773895,0.862592,0.724184,0.516628,0.519566,0.504929,0.930087,0.640934,0.870974,0.523086,0.557877,0.812179,0.572487,0.726110,0.968808,0.899588,0.501523,0.844209,0.948362,0.656329,0.625544,0.642460,0.513421,0.509878,0.531959,0.516761,0.511079,0.538034,0.635040,0.721028,0.542965,0.919475,0.552648,0.871559,0.536723,0.589566,0.558985,0.549435,0.891636,0.741803,0.869537,0.946150,0.556161,0.713811,0.713899,0.997959,0.758014,0.847081,0.501466,0.690970,0.522565,0.965135,0.846153,0.548538,0.546160,0.956054,0.871672,0.952043,0.998313,0.669413,0.808738,0.715886,0.641259,0.666927,0.614089,0.569646,0.526504,0.582710,0.694612,0.959215,0.516042,0.857547,0.628153,0.739145,0.743762,0.575022,0.619908,0.538384,0.629296,0.578420,0.564515,0.642259,0.747961,0.519728,0.805649,0.962651,0.514566,0.766804,0.799465,0.543705,0.924942,0.565683,0.523797,0.831865,0.886634,0.945017,0.552312,0.655240,0.868968,0.777392,0.508799,0.582681,0.986568,0.820437,0.596183,0.820506,0.522065,0.658020,0.779590,0.605564,0.850872,0.566425,0.752353,0.996780,0.718577,0.943606,0.748193,0.590638,0.653667,0.868321,0.543224,0.514857,0.766690,0.512524,0.505431,0.955294,0.912576,0.671270,0.786280,0.518809,0.733280,0.865552,0.909513,0.682219,0.722138,0.504539,0.800803,0.522635,0.592032,0.549953,0.872860,0.620021,0.553295,0.862993,0.775731,0.730274,0.913610,0.516771,0.884966,0.537600,0.937859,0.521694,0.593168,0.540069,0.506609,0.692310,0.762418,0.770428,0.964218,0.763207,0.705612,0.668527,0.553799,0.781605,0.961894,0.940866,0.609620,0.790445,0.522508,0.764153,0.612654,0.567035,0.942301,0.767920,0.688270,0.926322,0.741106,0.943991,0.576362,0.827993,0.724890,0.557257,0.545529,0.863851,0.515773,0.702077,0.869563,0.709787,0.892173,0.573527,0.541293,0.734833,0.803548,0.575709,0.502235,0.706841,0.729296,0.715153,0.984015,0.517356,0.973277,0.576395,0.890511,0.631240,0.613432,0.787797,0.696260,0.868802,0.965445,0.678339,0.929119,0.752584,0.530601,0.554171,0.761354,0.638222,0.582652,0.981907,0.595732,0.744251,0.535561,0.512854,0.565364,0.933875,0.841059,0.830860,0.733159,0.616575,0.838376,0.789208,0.514617,0.533024,0.526945,0.762155,0.953329,0.602258,0.572841,0.535257,0.994965,0.526443,0.827505,0.655682,0.762735,0.690934,0.804963,0.631152,0.509612,0.588790,0.746354,0.609585,0.632486,0.954681,0.591875,0.591101,0.925712,0.986551,0.548425,0.591438,0.755972,0.742987,0.563190,0.589349,0.624527,0.649010,0.678276,0.589409,0.517427,0.531966,0.509712,0.650372,0.925243,0.999820,0.581257,0.671433,0.552724,0.574366,0.893153,0.665003,0.661050,0.749314,0.736003,0.543965,0.728004,0.846682,0.560789,0.737022,0.607503,0.969307,0.501913,0.593343,0.542410,0.923154,0.992621,0.531158,0.515013,0.868322,0.502873,0.501790,0.691250,0.556834,0.636185,0.841382,0.516715,0.557131,0.707103,0.987091,0.820940,0.588204,0.569496,0.792993,0.852825,0.582641,0.619308,0.514313,0.864465,0.627817,0.561089,0.765792,0.755099,0.736045,0.680384,0.580011,0.559314,0.569590,0.645147,0.886666,0.538632,0.531876,0.700263,0.766275,0.597521,0.881351,0.595482,0.738585,0.805204,0.619179,0.559866,0.828941,0.653829,0.640393,0.922646,0.749207,0.606526,0.682175,0.614317,0.790487,0.677099,0.892748,0.950629,0.666488,0.522867,0.549624,0.604465,0.675133,0.589666,0.747297,0.554239,0.573700,0.524916,0.756624,0.757941,0.562858,0.625984,0.665854,0.553295,0.768195,0.504999,0.890734,0.873652,0.956340,0.580782,0.886156,0.566621,0.593174,0.895787,0.583844,0.746682,0.719358,0.977321,0.596714,0.681226,0.814269,0.875138,0.508025,0.612014,0.981122,0.514427,0.647960,0.574852,0.562274,0.524952,0.622125,0.552127,0.509311,0.704145,0.687542,0.796543,0.660314,0.724358,0.520354,0.957930,0.814134,0.517181,0.585988,0.681261,0.524900,0.958259,0.508771,0.601426,0.832930,0.821677,0.667474,0.558989,0.901758,0.670159,0.718597,0.510581,0.577131,0.898209,0.507547,0.549466,0.732668,0.966685,0.621634,0.788297,0.593376,0.754348,0.520735,0.759074,0.754181,0.689730,0.617414,0.596945,0.519325,0.688134,0.610743,0.759726,0.593735,0.907813,0.930304,0.664188,0.688024,0.871329,0.946330,0.833437,0.906460,0.547717,0.608281,0.676006,0.584172,0.703582,0.681999,0.765000,0.522115,0.687656,0.983903,0.624032,0.920767,0.823401,0.719694,0.759684,0.951994,0.984690,0.512169,0.584313,0.639603,0.512626,0.610488,0.999407,0.997224,0.661697,0.605510,0.656202,0.724242,0.727380,0.846561,0.536933,0.611675,0.691265,0.518838,0.561275,0.865856,0.904994,0.690546,0.616706,0.955574,0.961905,0.927374,0.876159,0.540895,0.544522,0.653586,0.903598,0.569227,0.501914,0.551659,0.536179,0.612625,0.622968,0.597424,0.674602,0.937518,0.742961,0.719491,0.929969,0.820424,0.782855,0.728160,0.583284,0.589243,0.607732,0.956402,0.700085,0.524816,0.585635,0.965505,0.794115,0.947863,0.634522,0.837576,0.549132,0.738558,0.869594,0.520766,0.818413,0.786904,0.784516,0.670278,0.790740,0.885246,0.802705,0.526299,0.509555,0.552559,0.653136,0.847405,0.596298,0.727310,0.686961,0.989010,0.914913,0.705062,0.647829,0.692471,0.746535,0.758759,0.758500,0.501992,0.598385,0.666748,0.511778,0.525301,0.558613,0.955983,0.595753,0.612333,0.691965,0.682895,0.557548,0.524423,0.806287,0.542554,0.587030,0.564540,0.702402,0.584593,0.664170,0.583794,0.579854,0.593391,0.895914,0.837041,0.782485,0.984597,0.691024,0.749672,0.592595,0.624774,0.563570,0.575882,0.718665,0.909073,0.578012,0.778702,0.548125,0.844893,0.583974,0.784744,0.747380,0.767340,0.601876,0.774131,0.866479,0.795041,0.537232,0.599219,0.699054,0.757375,0.932865,0.859054,0.730646,0.707537,0.851078,0.742933,0.800640,0.734603,0.556245,0.664404,0.851617,0.833993,0.583178,0.720692,0.602776,0.756827,0.772947,0.704209,0.653297,0.733128,0.886876,0.613693,0.525679,0.540575,0.858915,0.619677,0.608931,0.521527,0.776529,0.594767,0.524377,0.797166,0.999141,0.648447,0.663957,0.876993,0.995698,0.912326,0.531311,0.570045,0.851545,0.502992,0.529289,0.729559,0.581853,0.766178,0.952823,0.897727,0.582182,0.750701,0.674224,0.539773,0.813032,0.732317,0.531899,0.841485,0.560723,0.573193,0.585249,0.792930,0.634300,0.725669,0.626705,0.508552,0.538482,0.862666,0.690328,0.967554,0.788667,0.605362,0.730365,0.828191,0.788849,0.906561,0.774887,0.862256,0.713014,0.752895,0.819113,0.674078,0.547365,0.948901,0.651741,0.576450,0.756788,0.796963,0.984742,0.779223,0.841516,0.968781,0.695004,0.800913,0.760495,0.743496,0.518373,0.507502,0.950358,0.606400,0.556047,0.521557,0.616145,0.805423,0.535993,0.815627,0.983089,0.780604,0.519715,0.787422,0.650157,0.692536,0.805255,0.742943,0.794342,0.687483,0.523350,0.517691,0.778103,0.936313,0.566274,0.513003,0.597366,0.571483,0.610493,0.831607,0.724603,0.524771,0.500024,0.743275,0.703869,0.771346,0.534637,0.830384,0.815167,0.708665,0.702677,0.520086,0.970406,0.769915,0.568821,0.569663,0.765233,0.683994,0.992121,0.606009,0.965831,0.807490,0.913907,0.785862,0.705380,0.640245,0.756556,0.712141,0.774056,0.571580,0.722062,0.995816,0.764623,0.775338,0.710952,0.724925,0.860168,0.656528,0.801490,0.502331,0.511388,0.615576,0.604383,0.566939,0.589818,0.769629,0.822023,0.960015,0.577757,0.662213,0.967952,0.971636,0.567095,0.760093,0.549101,0.870619,0.538930,0.935549,0.584816,0.508072,0.553540,0.735770,0.926702,0.531263,0.790859,0.762147,0.521120,0.599911,0.546655,0.754509,0.758423,0.777167,0.699754,0.503926,0.933391,0.785094,0.503278,0.884885,0.512402,0.735374,0.726379,0.775789,0.634462,0.839746,0.758493,0.883225,0.781617,0.582307,0.570197,0.735365,0.674592,0.512332,0.892965,0.775838,0.640861,0.736523,0.572896,0.645030,0.636693,0.750462,0.902893,0.567750,0.697331,0.552826,0.634866,0.898827,0.602602,0.589238,0.530478,0.556497,0.973196,0.763412,0.956741,0.936002,0.590763,0.846266,0.519869,0.966144,0.801982,0.857013,0.926640,0.515822,0.566869,0.964789,0.783894,0.755448,0.585374,0.552963,0.979916,0.800787,0.668506,0.752369,0.761730,0.541107,0.689577,0.517139,0.609195,0.943818,0.768690,0.525164,0.764185,0.818907,0.528822,0.765215,0.860815,0.683358,0.560767,0.642288,0.518755,0.673364,0.873778,0.965012,0.723716,0.821916,0.659547,0.822126,0.768158,0.543933,0.689072,0.763953,0.601979,0.992002,0.566666,0.524933,0.637246,0.612230,0.778650,0.924335,0.529767,0.739039,0.820693,0.825871,0.601187,0.596408,0.683647,0.650108,0.577914,0.985239,0.812145,0.938686,0.686339,0.623302,0.932797,0.759836,0.856318,0.543993,0.605323,0.846988,0.621831,0.991652,0.595780,0.646736,0.576373,0.574668,0.795940,0.787494,0.634196,0.873599,0.506653,0.581255,0.684755,0.657651,0.623021,0.534026,0.669181,0.849718,0.580657,0.787369,0.560862,0.752891,0.874523,0.963578,0.903209,0.739967,0.758080,0.586940,0.649690,0.605971,0.963677,0.591457,0.616909,0.979925,0.535989,0.738567,0.751638,0.642703,0.922666,0.966014,0.840492,0.612332,0.696665,0.522260,0.549861,0.553143,0.805502,0.630349,0.817512,0.603110,0.891424,0.811833,0.922670,0.653099,0.760369,0.681733,0.827677,0.696907,0.641558,0.600832,0.584681,0.779678,0.711008,0.614495,0.740561,0.799465,0.745200,0.686622,0.617058,0.782200,0.571327,0.661951,0.835915,0.661181,0.838456,0.668979,0.523744,0.676554,0.785315,0.641505,0.626323,0.715886,0.538263,0.555229,0.778435,0.500378,0.771861,0.506157,0.786107,0.714511,0.809510,0.810595,0.600137,0.743382,0.928693,0.522012,0.916196,0.772950,0.600282,0.815975,0.536145,0.521461,0.547411,0.695390,0.758177,0.638879,0.738550,0.566515,0.979287,0.632090,0.704967,0.616510,0.588241,0.505375,0.692984,0.511497,0.774112,0.589248,0.848275,0.897824,0.780603,0.669286,0.744811,0.522642,0.746686,0.602613,0.538655,0.857930,0.627494,0.786464,0.591774,0.573352,0.991759,0.808807,0.820428,0.716940,0.631562,0.774643,0.738584,0.691521,0.591801,0.609458,0.691652,0.750761,0.946338,0.570549,0.683677,0.638746,0.668766,0.528149,0.968851,0.709955,0.697583,0.653927,0.930004,0.523256,0.964376,0.844407,0.736960,0.798566,0.836965,0.643535,0.748502,0.970606,0.695213,0.604839,0.886143,0.724408,0.541327,0.565779,0.903953,0.565265,0.792456,0.604118,0.526919,0.732253,0.638743,0.576170,0.688595,0.605158,0.658873,0.630007,0.529071,0.564593,0.665600,0.569668,0.886054,0.850586,0.886160,0.707091,0.607474,0.515664,0.956836,0.587131,0.566162,0.677642,0.762563,0.803169,0.585426,0.584415,0.944659,0.580175,0.601185,0.688382,0.970730,0.514613,0.550259,0.580098,0.913367,0.568382,0.644673,0.512026,0.602233,0.713478,0.607259,0.617503,0.824444,0.730031,0.855950,0.538495,0.789771,0.528897,0.552564,0.729730,0.807325,0.864078,0.691467,0.776903,0.994821,0.912369,0.502676,0.601397,0.752343,0.554492,0.632991,0.968857,0.521100,0.662154,0.937014,0.590366,0.730527,0.668173,0.862977,0.561600,0.962581,0.589639,0.503271,0.665791,0.676178,0.535398,0.931394,0.638724,0.599338,0.675091,0.500591,0.631685,0.903931,0.877455,0.926831,0.782608,0.601336,0.938137,0.682383,0.615748,0.505861,0.798969,0.594306,0.523407,0.782476,0.562311,0.823028,0.535392,0.815883,0.600518,0.816985,0.817433,0.600737,0.918716,0.859092,0.629841,0.736039,0.673698,0.974799,0.817093,0.879159,0.688200,0.956989,0.907226,0.608863,0.884115,0.545287,0.876908,0.687423,0.833448,0.523600,0.889675,0.744377,0.832035,0.999801,0.815219,0.866238,0.801816,0.753220,0.795477,0.728050,0.934329,0.552194,0.793205,0.630309,0.755474,0.686458,0.729453,0.650127,0.551138,0.770701,0.996995,0.519414,0.628095,0.722855,0.764066,0.702216,0.547895,0.697057,0.611492,0.977535,0.515611,0.696702,0.718207,0.923026,0.903212,0.650487,0.513759,0.974666,0.627245,0.629077,0.538278,0.852402,0.600228,0.929837,0.599457,0.594136,0.866081,0.660950,0.865891,0.639006,0.955458,0.585635,0.717421,0.720907,0.684313,0.576467,0.973997,0.733607,0.569391,0.716945,0.618811,0.622791,0.912641,0.509537,0.682483,0.612017,0.877252,0.577255,0.913500,0.802983,0.910274,0.986925,0.577883,0.601611,0.797717,0.769235,0.832171,0.946358,0.745945,0.679136,0.539383,0.726076,0.772895,0.728485,0.603103,0.561885,0.743835,0.720435,0.536229,0.542422,0.763000,0.749394,0.608295,0.766198,0.782981,0.689212,0.596879,0.765370,0.851238,0.778238,0.602413,0.748295,0.572956,0.798219,0.641495,0.514348,0.687574,0.729962,0.616704,0.767780,0.728672,0.884893,0.811423,0.585766,0.862031,0.527670,0.962489,0.799262,0.657053,0.523648,0.766838,0.961398,0.695097,0.640923,0.802772,0.635826,0.688119,0.576559,0.869402,0.519599,0.515782,0.939020,0.634746,0.938977,0.868830,0.962599,0.732981,0.764648,0.795737,0.744724,0.893012,0.919938,0.576927,0.626042,0.898016,0.647761,0.730672,0.549688,0.690645,0.585838,0.836503,0.530780,0.543187,0.785743,0.837823,0.683142,0.838612,0.520437,0.557917,0.994641,0.760255,0.744014,0.649449,0.538359,0.780422,0.646515,0.672493,0.657809,0.594439,0.563258,0.730965,0.957255,0.706518,0.700382,0.535628,0.753544,0.576632,0.835932,0.608863,0.650595,0.647382,0.778692,0.795683,0.674857,0.536810,0.578622,0.865568,0.771563,0.507697,0.688416,0.973192,0.945519,0.937772,0.899341,0.770429,0.953689,0.767466,0.661208,0.815687,0.670033,0.851734,0.701027,0.739956,0.616331,0.765830,0.700948,0.528056,0.777250,0.772326,0.909870,0.580813,0.612351,0.793085,0.720013,0.620885,0.517693,0.510842,0.707901,0.587096,0.984851,0.675248,0.545047,0.988217,0.704223,0.682344,0.614540,0.635729,0.602092,0.710606,0.806712,0.648649,0.732534,0.937321,0.565473,0.808729,0.841454,0.580748,0.501349,0.604508,0.772681,0.671846,0.658534,0.563222,0.553574,0.931090,0.646035,0.588396,0.520469,0.723767,0.926181,0.609341,0.721007,0.527917,0.856805,0.681532,0.520160,0.855593,0.832765,0.606919,0.821444,0.771106,0.759338,0.647094,0.889626,0.725693,0.862916,0.580787,0.555105,0.874958,0.972185,0.658998,0.750936,0.533374,0.834653,0.692538,0.694333,0.952050,0.878171,0.791072,0.560052,0.516771,0.973920,0.539072,0.913891,0.681887,0.527421,0.618938,0.720360,0.739740,0.789207,0.549746,0.786993,0.522640,0.541623,0.845853,0.525912,0.670003,0.770070,0.577744,0.635815,0.577572,0.554159,0.515039,0.756350,0.628410,0.543843,0.721649,0.944315,0.877439,0.599878,0.682738,0.578920,0.987951,0.802856,0.537977,0.848557,0.556146,0.792832,0.987413,0.555100,0.858033,0.680494,0.790492,0.545208,0.672324,0.846208,0.782256,0.567113,0.513133,0.502580,0.629641,0.575625,0.990691,0.588477,0.768906,0.506048,0.737781,0.708293,0.880423,0.889525,0.608941,0.894481,0.538901,0.571984,0.610511,0.554599,0.766899,0.607840,0.527214,0.767946,0.518099,0.683563,0.780761,0.515322,0.605283,0.576858,0.783339,0.581545,0.525725,0.667820,0.536881,0.514553,0.516779,0.505894,0.951947,0.503842,0.615974,0.864599,0.518080,0.754212,0.821174,0.647216,0.509900,0.794245,0.948495,0.833900,0.598596,0.687450,0.704873,0.634880,0.516713,0.560245,0.859968,0.516306,0.757943,0.502853,0.677463,0.721189,0.772892,0.686344,0.875663,0.546692,0.519854,0.505265,0.596358,0.612440,0.799415,0.870915,0.735725,0.504171,0.992662,0.803554,0.580108,0.867771,0.980822,0.625368,0.938368,0.804005,0.629313,0.500516,0.561888,0.925995,0.565469,0.680802,0.673418,0.917957,0.588880,0.956959,0.673932,0.881716,0.689090,0.889098,0.564227,0.573113,0.533219,0.975820,0.690563,0.537024,0.977769,0.599487,0.889172,0.586542,0.505422,0.690052,0.606709,0.656891,0.611210,0.964057,0.582054,0.937234,0.526974,0.541189,0.560314,0.931663,0.720570,0.525106,0.642743,0.730035,0.559024,0.949862,0.596762,0.586486,0.584053,0.543176,0.595382,0.629188,0.610050,0.619355,0.650776,0.502122,0.821487,0.965713,0.945053,0.598770,0.672976,0.891115,0.997379,0.817373,0.571801,0.560032,0.540727,0.556211,0.719786,0.838444,0.737678,0.679346,0.504711,0.584875,0.773318,0.821481,0.563019,0.885047,0.770593,0.893758,0.729904,0.901954,0.589978,0.685786,0.900808,0.524428,0.827431,0.656271,0.904996,0.827399,0.956072,0.531159,0.560182,0.680448,0.538581,0.657140,0.747973,0.531357,0.643055,0.656523,0.812881,0.979015,0.847419,0.616744,0.673460,0.828664,0.665598,0.578663,0.624689,0.733863,0.688858,0.605687,0.664546,0.677829,0.597893,0.571834,0.788605,0.817655,0.635006,0.966663,0.998317,0.555179,0.823913,0.934868,0.759277,0.601017,0.877596,0.800863,0.924431,0.564763,0.799738,0.515552,0.579471,0.686531,0.929455,0.875705,0.681114,0.561151,0.647189,0.915486,0.725545,0.584134,0.500515,0.593347,0.644699,0.900959,0.919326,0.821379,0.851440,0.538378,0.862861,0.854603,0.812955,0.932042,0.732706,0.670442,0.749207,0.541311,0.661365,0.997944,0.705223,0.602291,0.699044,0.694276,0.921245,0.778554,0.651492,0.869902,0.687197,0.739970,0.603732,0.676499,0.612192,0.731881,0.518426,0.593143,0.567522,0.815376,0.878222,0.792005,0.805894,0.665137,0.533247,0.515193,0.900907,0.672295,0.830775,0.545520,0.531001,0.806235,0.931536,0.607677,0.580720,0.567807,0.503686,0.983183,0.721167,0.805173,0.606710,0.743800,0.550563,0.598531,0.759746,0.745191,0.932181,0.573578,0.562806,0.547839,0.725596,0.765227,0.686918,0.586577,0.530132,0.886627,0.525877,0.778349,0.533238,0.816982,0.882040,0.871318,0.947500,0.837843,0.983131,0.666592,0.717294,0.554639,0.558846,0.511709,0.636678,0.918908,0.523937,0.570534,0.907977,0.539680,0.742986,0.598282,0.994841,0.648285,0.851731,0.675440,0.651420,0.660037,0.542947,0.851082,0.582157,0.723919,0.644249,0.725395,0.569093,0.750430,0.827962,0.758303,0.995413,0.986408,0.741194,0.555212,0.758112,0.667256,0.968272,0.830022,0.504072,0.775689,0.944073,0.604494,0.520352,0.883060,0.649870,0.668090,0.692546,0.610189,0.653488,0.592004,0.626939,0.766027,0.931072,0.590946,0.646056,0.995338,0.535622,0.560228,0.684776,0.641615,0.604379,0.515469,0.798628,0.754242,0.566230,0.973545,0.789602,0.718846,0.573696,0.659848,0.697217,0.642417,0.784983,0.849680,0.718749,0.841344,0.572213,0.620701,0.787328,0.659770,0.861140,0.976703,0.571362,0.610085,0.846907,0.793412,0.661275,0.552978,0.519274,0.843655,0.777141,0.532411,0.634724,0.584417,0.618497,0.711037,0.719061,0.511763,0.791169,0.678583,0.812757,0.587074,0.957418,0.707269,0.708454,0.557309,0.739866,0.963035,0.699168,0.730036,0.552885,0.702492,0.702135,0.596759,0.568537,0.547066,0.639043,0.662346,0.742478,0.884594,0.865450,0.662856,0.816535,0.641434,0.830363,0.770009,0.536079,0.885709,0.643619,0.606308,0.556470,0.873695,0.646892,0.529238,0.623948,0.704239,0.590936,0.547513,0.762047,0.828163,0.526454,0.856991,0.785099,0.517274,0.596538,0.879717,0.720180,0.695716,0.522174,0.640990,0.584035,0.816975,0.856232,0.881341,0.793578,0.637260,0.707935,0.702710,0.674213,0.521062,0.566953,0.617331,0.510802,0.601996,0.694708,0.690095,0.728535,0.988870,0.849426,0.887778,0.628746,0.656652,0.523983,0.695895,0.612708,0.703357,0.985767,0.828214,0.986455,0.614594,0.905152,0.902545,0.681087,0.518479,0.694079,0.698879,0.513641,0.723254,0.785885,0.727800,0.788322,0.637744,0.661672,0.528034,0.756651,0.660021,0.514580,0.869625,0.528041,0.743176,0.955561,0.842045,0.619064,0.588114,0.769508,0.788529,0.775152,0.509873,0.690540,0.753765,0.813781,0.876685,0.871572,0.659855,0.595547,0.744744,0.577655,0.589259,0.719259,0.559286,0.781025,0.786101,0.599671,0.937386,0.584776,0.632033,0.516577,0.963274,0.638168,0.516280,0.612563,0.605790,0.844543,0.675512,0.536673,0.513660,0.887722,0.819365,0.521663,0.714515,0.829178,0.653451,0.670830,0.607448,0.739911,0.764460,0.762368,0.728090,0.674744,0.708448,0.500723,0.769919,0.729010,0.586052,0.531449,0.808580,0.659848,0.544652,0.925778,0.643908,0.697960,0.839355,0.840434,0.612581,0.983078,0.564108,0.926016,0.525313,0.871611,0.759727,0.802273,0.655075,0.872164,0.536276,0.613396,0.744134,0.652630,0.803045,0.535328,0.595083,0.717699,0.501492,0.945736,0.546453,0.766240,0.567517,0.577100,0.567587,0.744208,0.662754,0.631645,0.599600,0.653434,0.670866,0.780393,0.717675,0.862259,0.849960,0.598272,0.687826,0.546664,0.664150,0.643075,0.976775,0.593894,0.578310,0.853879,0.739207,0.914685,0.692860,0.744876,0.859801,0.764008,0.680490,0.520450,0.635917,0.502926,0.588352,0.943713,0.708867,0.659100,0.744803,0.871970,0.765432,0.662505,0.595324,0.940075,0.521411,0.771450,0.786250,0.845105,0.606009,0.641061,0.724434,0.708643,0.886847,0.511286,0.757452,0.624464,0.544141,0.867276,0.954762,0.510999,0.619307,0.687899,0.795168,0.811308,0.622032,0.598312,0.638557,0.847193,0.628923,0.858533,0.673142,0.888265,0.701917,0.912872,0.566311,0.603210,0.837548,0.618920,0.640863,0.616756,0.993274,0.719444,0.675459,0.680744,0.757123,0.633850,0.575480,0.571812,0.566781,0.648398,0.904359,0.691493,0.639495,0.667386,0.928567,0.596493,0.576972,0.892060,0.730175,0.931536,0.543920,0.602164,0.770248,0.764685,0.784023,0.778847,0.647415,0.870308,0.666629,0.687963,0.752933,0.962344,0.543204,0.990566,0.512816,0.558542,0.813149,0.556104,0.557455,0.559373,0.635726,0.751133,0.825358,0.735585,0.867527,0.585498,0.605534,0.666500,0.685888,0.603052,0.656578,0.926956,0.642045,0.520319,0.757292,0.580092,0.589048,0.658733,0.672917,0.541849,0.573296,0.649143,0.647418,0.591955,0.742595,0.589118,0.766486,0.765242,0.838436,0.620253,0.710714,0.733821,0.881629,0.597593,0.635936,0.608075,0.502751,0.745260,0.564195,0.838939,0.916588,0.740776,0.648932,0.680543,0.736365,0.740905,0.693392,0.696812,0.500504,0.678228,0.635363,0.781921,0.671692,0.550140,0.760630,0.546176,0.768023,0.544898,0.792919,0.645536,0.597123,0.993074,0.586854,0.766119,0.528312,0.626215,0.689060,0.894546,0.806605,0.577047,0.501181,0.904608,0.832183,0.967078,0.599753,0.676144,0.701343,0.747746,0.652523,0.548532,0.868796,0.570914,0.656880,0.519407,0.985862,0.612134,0.804264,0.714993,0.549210,0.877599,0.639097,0.863347,0.865834,0.632377,0.569094,0.520356,0.561685,0.791625,0.634350,0.527486,0.973332,0.882778,0.799335,0.533478,0.698355,0.611512,0.672497,0.715124,0.571701,0.636218,0.688934,0.859579,0.814273,0.954736,0.676950,0.569359,0.638700,0.792764,0.974898,0.956555,0.522335,0.512020,0.724145,0.895435,0.659190,0.673628,0.698144,0.507324,0.541275,0.661718,0.551830,0.803212,0.879095,0.748188,0.891491,0.648731,0.580427,0.577856,0.633331,0.528763,0.971525,0.816672,0.698317,0.579664,0.908319,0.673571,0.518024,0.913430,0.741834,0.698522,0.628092,0.604269,0.867572,0.532465,0.609536,0.548393,0.508596,0.518069,0.569210,0.775352,0.940468,0.631681,0.775326,0.631602,0.698185,0.701365,0.608587,0.682620,0.790445,0.855521,0.613722,0.634075,0.681217,0.849689,0.607422,0.780593,0.541936,0.809091,0.583188,0.679003,0.896856,0.913029,0.741735,0.500461,0.565879,0.707797,0.527717,0.654808,0.705690,0.742074,0.676831,0.744102,0.832975,0.895119,0.596287,0.600791,0.834069,0.553205,0.544315,0.898137,0.853170,0.598793,0.802500,0.517359,0.960821,0.619563,0.664201,0.772229,0.971871,0.511352,0.870654,0.699160,0.979698,0.558814,0.535730,0.620014,0.627658,0.617129,0.585116,0.838907,0.507564,0.547440,0.680509,0.932778,0.701820,0.591418,0.637613,0.778117,0.775395,0.703976,0.852606,0.739481,0.591040,0.531373,0.958348,0.678084,0.529211,0.704809,0.547913,0.659667,0.867770,0.634328,0.603238,0.960849,0.570935,0.704686,0.696497,0.836945,0.984536,0.892700,0.547286,0.635350,0.755867,0.984753,0.504415,0.810187,0.860166,0.605886,0.639622,0.617336,0.961016,0.787611,0.672550,0.500235,0.609561,0.651588,0.522703,0.627369,0.776711,0.682516,0.628181,0.582674,0.949160,0.994831,0.555722,0.535474,0.500807,0.738953,0.866727,0.516211,0.727343,0.921027,0.779182,0.598778,0.607151,0.780766,0.609460,0.598265,0.705885,0.703466,0.578426,0.653534,0.793026,0.567543,0.691743,0.752920,0.670135,0.621862,0.737572,0.838672,0.556033,0.588016,0.837240,0.622468,0.912893,0.941396,0.535683,0.594952,0.733441,0.568753,0.979999,0.662446,0.794808,0.912397,0.656159,0.888583,0.603119,0.707949,0.795865,0.900914,0.931253,0.724467,0.714658,0.501368,0.568955,0.568153,0.926921,0.979396,0.752473,0.866102,0.577285,0.643570,0.860939,0.544165,0.667798,0.814749,0.975425,0.528902,0.862434,0.754362,0.766438,0.597077,0.806796,0.857126,0.751998,0.789037,0.511182,0.949747,0.851046,0.694989,0.776209,0.813629,0.539268,0.619122,0.825856,0.965212,0.518054,0.730025,0.716726,0.837863,0.512952,0.808569,0.818565,0.712471,0.684501,0.589817,0.849223,0.584369,0.634642,0.998065,0.656885,0.643050,0.979910,0.688684,0.862399,0.826078,0.755002,0.780902,0.567111,0.896266,0.720715,0.751005,0.602222,0.786816,0.877463,0.520211,0.670266,0.602682,0.901194,0.770872,0.819183,0.884902,0.510452,0.988388,0.504439,0.634120,0.526521,0.528431,0.812818,0.596657,0.839901,0.726518,0.899915,0.606313,0.552852,0.675109,0.685085,0.576632,0.651101,0.665527,0.803890,0.561734,0.517493,0.898507,0.745440,0.650407,0.523554,0.605349,0.546243,0.998462,0.970486,0.580223,0.890262,0.998532,0.767206,0.736876,0.697948,0.613619,0.597022,0.672278,0.691541,0.795489,0.608422,0.883191,0.834887,0.800542,0.941183,0.910096,0.977234,0.632846,0.512876,0.686830,0.735951,0.661075,0.728170,0.745731,0.556304,0.517017,0.557470,0.603960,0.842035,0.726534,0.793874,0.820535,0.871631,0.579076,0.848698,0.752024,0.589591,0.745232,0.643507,0.807946,0.819777,0.902554,0.508215,0.552746,0.507488,0.935503,0.808917,0.577604,0.508043,0.665823,0.702369,0.711447,0.669414,0.786598,0.712392,0.988191,0.577772,0.593284,0.526604,0.932091,0.586718,0.917019,0.655771,0.609132,0.653598,0.598620,0.678485,0.841740,0.966454,0.847008,0.627563,0.620638,0.928661,0.779699,0.537444,0.993942,0.779261,0.787300,0.968947,0.916980,0.924938,0.633376,0.558212,0.815985,0.827226,0.820852,0.942653,0.593210,0.620580,0.633554,0.717831,0.750894,0.532448,0.580164,0.936256,0.562485,0.573242,0.701495,0.942694,0.996466,0.913203,0.889081,0.683104,0.713720,0.853192,0.741617,0.858972,0.772326,0.802466,0.792918,0.925455,0.514796,0.664494,0.719959,0.777947,0.550591,0.507276,0.504952,0.614239,0.712480,0.749152,0.894262,0.900070,0.707724,0.922965,0.574726,0.965312,0.799395,0.702221,0.850265,0.926533,0.628702,0.942809,0.687656,0.641757,0.862563,0.949839,0.741404,0.804914,0.952950,0.630896,0.591043,0.626435,0.948410,0.613471,0.614313,0.914606,0.713793,0.756915,0.682668,0.623983,0.910652,0.812288,0.700136,0.885869,0.528967,0.694831,0.793816,0.741369,0.891866,0.737196,0.601219,0.761913,0.749211,0.724149,0.590243,0.618768,0.622252,0.644404,0.923500,0.810359,0.979619,0.980536,0.941147,0.584952,0.741078,0.733715,0.936864,0.533068,0.780030,0.804341,0.583050,0.569441,0.549977,0.514414,0.512422,0.555558,0.737425,0.829330,0.999294,0.677315,0.793999,0.981480,0.567857,0.708018,0.936937,0.841407,0.866749,0.918526,0.735495,0.587106,0.575710,0.632034,0.567192,0.709761,0.785276,0.964268,0.508604,0.969097,0.893295,0.502897,0.777491,0.535060,0.840863,0.540867,0.981880,0.561370,0.765900,0.571412,0.634624,0.735599,0.636614,0.744134,0.674445,0.987060,0.856181,0.576898,0.885165,0.852598,0.805294,0.658646,0.779420,0.689862,0.568029,0.711288,0.558904,0.961268,0.951922,0.528270,0.999411,0.879147,0.881291,0.622902,0.637441,0.931461,0.555301,0.520280,0.647265,0.773686,0.526117,0.961653,0.676980,0.573670,0.516849,0.523343,0.503919,0.792470,0.948246,0.594725,0.637755,0.598429,0.882846,0.723528,0.644443,0.574504,0.998348,0.731019,0.657573,0.594179,0.549485,0.766481,0.619120,0.785462,0.997853,0.549881,0.529471,0.694725,0.969789,0.766644,0.849386,0.984264,0.711161,0.755681,0.579027,0.660010,0.778941,0.769155,0.659826,0.694336,0.615051,0.942161,0.837899,0.509275,0.548532,0.515967,0.983549,0.528345,0.910525,0.573212,0.570953,0.784793,0.704435,0.840201,0.895473,0.970212,0.616346,0.525879,0.571650,0.501744,0.908727,0.589095,0.612395,0.508690,0.573308,0.801226,0.619588,0.504883,0.506882,0.877584,0.530888,0.588615,0.512395,0.600714,0.831856,0.658763,0.525112,0.525945,0.510161,0.518383,0.886362,0.537907,0.807906,0.705953,0.629687,0.681877,0.999551,0.646104,0.975289,0.674534,0.607160,0.971777,0.729615,0.929457,0.763984,0.677903,0.754918,0.666135,0.565540,0.837166,0.716254,0.516289,0.652529,0.579415,0.983877,0.855522,0.926135,0.842770,0.526787,0.990106,0.667375,0.518990,0.905207,0.708854,0.867359,0.639098,0.729548,0.605510,0.528478,0.958916,0.676502,0.688081,0.533485,0.513868,0.988941,0.637723,0.590388,0.770938,0.514252,0.511631,0.684128,0.782656,0.571928,0.562584,0.756185,0.556913,0.952875,0.619821,0.506358,0.658701,0.855557,0.865319,0.737519,0.748850,0.530808,0.905871,0.510402,0.608715,0.623477,0.647085,0.610874,0.787434,0.658930,0.979702,0.909482,0.686503,0.530963,0.844495,0.673273,0.883226,0.820481,0.590348,0.635388,0.550975,0.663278,0.871365,0.995518,0.622092,0.584032,0.557586,0.762233,0.808092,0.804696,0.707658,0.864905,0.703255,0.508229,0.576338,0.781862,0.552297,0.875512,0.730163,0.537669,0.660150,0.848836,0.625635,0.519691,0.676605,0.629778,0.554940,0.984320,0.721762,0.628075,0.888655,0.598905,0.744777,0.845701,0.852081,0.872373,0.648485,0.921115,0.879734,0.987977,0.875766,0.931749,0.852041,0.964897,0.534651,0.842375,0.624691,0.587054,0.838247,0.772625,0.561416,0.628533,0.773009,0.855939,0.500774,0.785565,0.514635,0.844816,0.978939,0.979651,0.558473,0.533701,0.632695,0.721813,0.952966,0.673137,0.576672,0.603830,0.651940,0.919811,0.640106,0.718568,0.795285,0.801325,0.514267,0.890442,0.567060,0.756685,0.861126,0.900979,0.701424,0.929327,0.534467,0.755630,0.547641,0.703615,0.571812,0.654040,0.824461,0.621084,0.589322,0.941760,0.767766,0.691020,0.894315,0.707340,0.509321,0.575585,0.641760,0.969805,0.768466,0.591245,0.942644,0.507036,0.529280,0.869649,0.530410,0.624380,0.859835,0.985248,0.612965,0.736124,0.642801,0.528724,0.994639,0.515195,0.825091,0.611961,0.527648,0.893849,0.928200,0.647768,0.721941,0.511862,0.840809,0.818135,0.971588,0.850033,0.888370,0.743204,0.580054,0.824126,0.862705,0.610152,0.899902,0.904731,0.547886,0.990866,0.586585,0.967738,0.949417,0.731733,0.904426,0.732571,0.640614,0.938817,0.548535,0.540726,0.682214,0.503215,0.659417,0.643019,0.754105,0.771895,0.588279,0.713882,0.597843,0.749011,0.519064,0.846919,0.566824,0.531181,0.531230,0.626351,0.756957,0.984824,0.501889,0.543938,0.594766,0.616097,0.632590,0.507371,0.674860,0.511435,0.612121,0.866340,0.851409,0.925737,0.988983,0.595842,0.528678,0.771551,0.743858,0.697760,0.995558,0.795333,0.751812,0.823122,0.766008,0.811157,0.908602,0.922472,0.878462,0.745900,0.918978,0.802281,0.519701,0.574525,0.642318,0.606784,0.978285,0.512425,0.571901,0.615555,0.652781,0.938703,0.931222,0.506817,0.732072,0.863522,0.612986,0.818803,0.673981,0.512647,0.815952,0.556485,0.640111,0.569485,0.750749,0.617016,0.669461,0.865413,0.593364,0.668213,0.786075,0.998484,0.784695,0.882287,0.809746,0.557208,0.665173,0.924521,0.656730,0.622831,0.901106,0.517590,0.795166,0.940362,0.509470,0.517017,0.774018,0.500602,0.593693,0.756642,0.719946,0.810327,0.555176,0.547852,0.882943,0.572145,0.537287,0.552599,0.939133,0.668980,0.876005,0.891268,0.633650,0.992353,0.987358,0.880535,0.698132,0.607435,0.539407,0.943904,0.582235,0.742095,0.880508,0.524036,0.572846,0.999295,0.966109,0.599419,0.753693,0.604550,0.964746,0.858988,0.671886,0.568224,0.684502,0.629870,0.937216,0.567764,0.652239,0.675827,0.726190,0.601242,0.685252,0.723834,0.574728,0.579147,0.544383,0.795915,0.733131,0.570544,0.515225,0.564332,0.655289,0.847705,0.595280,0.562781,0.880683,0.659823,0.601637,0.646089,0.867422,0.767884,0.864374,0.589259,0.706108,0.644455,0.606740,0.976049,0.909752,0.737465,0.995148,0.846089,0.621570,0.588971,0.980349,0.904230,0.900027,0.790265,0.512317,0.839665,0.570559,0.680750,0.555725,0.650637,0.668839,0.585150,0.967386,0.587203,0.704447,0.522711,0.739296,0.948651,0.660324,0.928181,0.768204,0.609220,0.610323,0.710860,0.646639,0.903152,0.840941,0.691904,0.504630,0.512627,0.917805,0.922308,0.925280,0.570663,0.957124,0.988114,0.525526,0.772469,0.753156,0.949351,0.535513,0.627397,0.690265,0.944173,0.556237,0.521481,0.540331,0.648723,0.799994,0.603073,0.638881,0.686311,0.589840,0.503353,0.769994,0.953824,0.588886,0.813823,0.941894,0.609993,0.625096,0.590576,0.857718,0.645898,0.986056,0.624237,0.684873,0.751309,0.835839,0.771368,0.823983,0.677192,0.616232,0.571085,0.608019,0.757861,0.690578,0.923714,0.560534,0.782051,0.540540,0.520432,0.503889,0.612297,0.641399,0.550844,0.693515,0.825424,0.637416,0.920628,0.767982,0.856000,0.893752,0.677355,0.530494,0.673756,0.796021,0.807334,0.764452,0.866223,0.718749,0.665958,0.806340,0.976973,0.595676,0.760848,0.886295,0.561825,0.863250,0.653226,0.901152,0.635962,0.692983,0.706221,0.997866,0.721504,0.602449,0.857725,0.922522,0.608899,0.611595,0.803215,0.602429,0.932288,0.643904,0.555923,0.664311,0.613495,0.929099,0.601526,0.678864,0.615509,0.603412,0.786260,0.584520,0.657682,0.860057,0.802585,0.582291,0.884489,0.760365,0.609654,0.575404,0.889542,0.985102,0.631967,0.640068,0.706699,0.609820,0.503323,0.605267,0.613627,0.537029,0.525688,0.751979,0.642618,0.543448,0.716210,0.823737,0.557152,0.735807,0.977056,0.744523,0.879134,0.605415,0.695701,0.952994,0.728171,0.960200,0.815791,0.954446,0.785855,0.526238,0.607988,0.661635,0.519766,0.672135,0.533640,0.623952,0.859469,0.546830,0.758823,0.577954,0.952214,0.609010,0.865016,0.746465,0.766784,0.942048,0.891413,0.739265,0.807617,0.538438,0.519168,0.879080,0.528696,0.582641,0.977319,0.707663,0.651493,0.509345,0.640576,0.528805,0.627910,0.513628,0.615379,0.580016,0.697928,0.549163,0.606820,0.616685,0.685658,0.867403,0.969479,0.761698,0.707473,0.606309,0.545683,0.798843,0.505613,0.742700,0.693063,0.522936,0.939722,0.630959,0.535036,0.501512,0.682610,0.657272,0.715651,0.560351,0.863734,0.585454,0.570181,0.626884,0.845956,0.523760,0.694201,0.902938,0.619142,0.879357,0.551858,0.806840,0.772798,0.760983,0.969370,0.963539,0.693470,0.672168,0.569058,0.512941,0.669612,0.590428,0.894042,0.919913,0.535429,0.707887,0.625923,0.562408,0.608802,0.842378,0.871389,0.851935,0.770016,0.740480,0.638224,0.941221,0.796211,0.504083,0.962661,0.629510,0.582512,0.973636,0.825351,0.641716,0.573999,0.670114,0.544325,0.941401,0.651816,0.607120,0.548564,0.621746,0.517214,0.871483,0.822795,0.942872,0.703993,0.958490,0.710132,0.655181,0.657516,0.611299,0.657167,0.644229,0.886731,0.600254,0.554535,0.846762,0.641602,0.501369,0.702586,0.649615,0.510496,0.844038,0.971405,0.625774,0.968772,0.668303,0.514131,0.730941,0.709621,0.676100,0.870081,0.889441,0.519116,0.669891,0.554447,0.807991,0.647033,0.784323,0.984185,0.656220,0.999289,0.995511,0.694456,0.613352,0.502870,0.952063,0.615514,0.681736,0.766140,0.555088,0.926977,0.501769,0.511997,0.642274,0.836924,0.952920,0.647677,0.726459,0.845741,0.903876,0.818075,0.560242,0.517973,0.890060,0.756382,0.595905,0.952045,0.998471,0.514466,0.553444,0.633806,0.575069,0.640296,0.701223,0.549592,0.573785,0.527107,0.653904,0.589323,0.633433,0.564138,0.712819,0.506208,0.549885,0.606567,0.562324,0.630946,0.519664,0.808347,0.532984,0.990376,0.783198,0.635179,0.645052,0.603495,0.633634,0.572880,0.858568,0.749266,0.703543,0.736660,0.881715,0.688180,0.896384,0.599977,0.904555,0.511609,0.541776,0.632776,0.835704,0.860396,0.872430,0.545044,0.556440,0.553528,0.647120,0.562840,0.644408,0.666417,0.663074,0.689318,0.529772,0.504869,0.694882,0.661128,0.739123,0.582892,0.515311,0.689880,0.855206,0.533104,0.701532,0.968803,0.901159,0.865348,0.767863,0.505300,0.530287,0.737402,0.931435,0.619531,0.927614,0.640319,0.626491,0.639521,0.920220,0.710100,0.733376,0.709347,0.665415,0.592100,0.628366,0.625423,0.743500,0.544755,0.562169,0.522119,0.712110,0.556828,0.503513,0.742157,0.591524,0.913651,0.615285,0.928150,0.675424,0.663362,0.526918,0.576343,0.546299,0.736248,0.598822,0.562213,0.519170,0.624156,0.799175,0.750067,0.610854,0.784071,0.742900,0.728661,0.981157,0.728941,0.920072,0.533732,0.707204,0.859797,0.511834,0.784890,0.506332,0.858822,0.605547,0.821329,0.696993,0.685692,0.540641,0.545988,0.935241,0.666176,0.604311,0.722122,0.908744,0.517303,0.670812,0.923964,0.560722,0.999508,0.926553,0.931829,0.591636,0.625059,0.738372,0.507476,0.570511,0.670512,0.522747,0.652627,0.909057,0.835669,0.698260,0.676903,0.784518,0.766807,0.977665,0.968812,0.705318,0.973235,0.954257,0.569579,0.595995,0.527644,0.685903,0.986880,0.514657,0.528388,0.757272,0.916066,0.764035,0.510060,0.885042,0.706629,0.651828,0.610578,0.696777,0.550480,0.743545,0.527214,0.716483,0.682670,0.518361,0.873856,0.687091,0.538490,0.633857,0.773259,0.797520,0.865473,0.931340,0.588389,0.773959,0.885688,0.693356,0.731279,0.629576,0.632560,0.911451,0.695159,0.669818,0.542605,0.774028,0.854765,0.653193,0.560932,0.685187,0.526694,0.718362,0.530462,0.639416,0.838130,0.595656,0.972926,0.979529,0.655032,0.810065,0.526549,0.708331,0.833002,0.703449,0.902851,0.736104,0.745230,0.862952,0.802975,0.530769,0.921208,0.930685,0.630781,0.838791,0.647945,0.608678,0.736957,0.550140,0.985074,0.931696,0.837124,0.538515,0.937300,0.777707,0.825592,0.903407,0.762856,0.683822,0.658464,0.510160,0.517945,0.515103,0.574173,0.500183,0.724488,0.637578,0.982247,0.591369,0.754249,0.600394,0.758333,0.926548,0.594804,0.903845,0.933211,0.676416,0.831097,0.918819,0.886125,0.694829,0.922614,0.944416,0.785839,0.643566,0.912287,0.647957,0.644531,0.876541,0.731086,0.751912,0.774672,0.533982,0.599661,0.748839,0.597343,0.684718,0.536525,0.611351,0.758705,0.578693,0.933991,0.834218,0.799582,0.913339,0.713931,0.822946,0.942133,0.844918,0.709384,0.700948,0.721255,0.544281,0.652417,0.645184,0.611192,0.567770,0.617969,0.842238,0.759021,0.622872,0.808450,0.871601,0.940832,0.786138,0.618052,0.558752,0.897972,0.500340,0.679831,0.527238,0.660602,0.606381,0.577671,0.797483,0.665851,0.645004,0.504848,0.919644,0.796101,0.876061,0.962129,0.651418,0.839364,0.842072,0.842325,0.973160,0.941067,0.637357,0.835703,0.573392,0.612778,0.855875,0.778503,0.929917,0.710156,0.589249,0.788499,0.852859,0.678886,0.965052,0.790757,0.646915,0.640867,0.539093,0.588217,0.689432,0.545647,0.906880,0.609747,0.946253,0.628571,0.858970,0.971268,0.550380,0.562461,0.897290,0.933892,0.555633,0.666865,0.972024,0.539599,0.832261,0.587622,0.781664,0.675806,0.589803,0.527789,0.828498,0.935756,0.502265,0.575264,0.593442,0.707692,0.529816,0.579421,0.790384,0.557740,0.585083,0.581862,0.503864,0.772778,0.883764,0.942117,0.943740,0.746464,0.904856,0.587019,0.796845,0.706616,0.784212,0.727281,0.612707,0.959313,0.719695,0.503170,0.575293,0.963026,0.914398,0.826507,0.838549,0.859194,0.695459,0.868265,0.814562,0.682569,0.598262,0.540875,0.569558,0.631404,0.532999,0.522169,0.887650,0.789434,0.811172,0.551153,0.675247,0.663283,0.966012,0.680996,0.866126,0.918264,0.718922,0.506075,0.645070,0.615199,0.945591,0.741235,0.580521,0.536995,0.884557,0.548554,0.531947,0.533075,0.630475,0.574680,0.573264,0.861380,0.977895,0.616458,0.573122,0.977858,0.913717,0.714673,0.916443,0.561592,0.688249,0.605573,0.818848,0.799960,0.557986,0.914294,0.505614,0.510560,0.988674,0.794862,0.665056,0.735161,0.570028,0.773725,0.598610,0.531148,0.537769,0.722566,0.683116,0.890849,0.854868,0.880276,0.535958,0.642036,0.954186,0.762426,0.819979,0.703415,0.558371,0.526345,0.585760,0.543350,0.552914,0.580119,0.535727,0.518701,0.858991,0.541636,0.936675,0.694782,0.766722,0.851958,0.986407,0.701313,0.592918,0.859845,0.775194,0.706657,0.616576,0.620541,0.537276,0.961933,0.609453,0.671800,0.695598,0.711310,0.984281,0.976964,0.573442,0.587879,0.679300,0.639488,0.871882,0.981165,0.571626,0.840035,0.509095,0.520732,0.649764,0.914665,0.690749,0.814084,0.720864,0.667238,0.904399,0.577760,0.917895,0.829758,0.699823,0.677907,0.512251,0.596780,0.585795,0.615293,0.864219,0.681064,0.541237,0.949356,0.665708,0.758584,0.705825,0.520277,0.826351,0.583335,0.558718,0.709423,0.546685,0.534427,0.528998,0.508624,0.535119,0.815139,0.603327,0.539079,0.544640,0.859595,0.559414,0.873601,0.542919,0.838042,0.873044,0.732668,0.687727,0.686265,0.579126,0.930473,0.736025,0.533363,0.790808,0.818609,0.788338,0.522972,0.989588,0.678737,0.988053,0.576138,0.652473,0.747726,0.630129,0.832010,0.581382,0.519004,0.523453,0.584054,0.848045,0.592771,0.541855,0.877446,0.970884,0.653633,0.831004,0.899246,0.788473,0.607291,0.530375,0.638723,0.509468,0.661409,0.831370,0.839967,0.564596,0.605410,0.504330,0.689786,0.726888,0.758514,0.677251,0.793007,0.548660,0.511166,0.561855,0.769373,0.631990,0.848816,0.686360,0.975602,0.978494,0.989335,0.683161,0.776692,0.680247,0.594663,0.511167,0.500007,0.778401,0.922890,0.636489,0.669808,0.930821,0.561013,0.711951,0.840840,0.717231,0.774028,0.942456,0.602961,0.566722,0.982818,0.513509,0.526807,0.946047,0.979457,0.834335,0.526464,0.730169,0.719409,0.917599,0.529415,0.697194,0.977883,0.795574,0.884565,0.591514,0.660229,0.904603,0.516170,0.708326,0.823140,0.547302,0.581573,0.738279,0.630125,0.712953,0.830803,0.502475,0.975281,0.902631,0.519632,0.831153,0.598260,0.972352,0.503535,0.517864,0.506477,0.696004,0.633877,0.886806,0.514982,0.585136,0.679198,0.853605,0.611823,0.642863,0.534126,0.749531,0.956421,0.821051,0.724105,0.763336,0.716321,0.636059,0.617593,0.957596,0.537147,0.908793,0.525470,0.771502,0.808281,0.585215,0.580846,0.515427,0.503750,0.597715,0.762450,0.884131,0.837261,0.605655,0.623084,0.619980,0.904192,0.656499,0.811968,0.544736,0.577733,0.729961,0.611864,0.665977,0.571828,0.863552,0.678134,0.841302,0.648378,0.831892,0.920516,0.891202,0.768887,0.954132,0.572884,0.589591,0.566484,0.558903,0.877896,0.666928,0.527245,0.912480,0.772115,0.554577,0.500563,0.752805,0.573431,0.690961,0.679657,0.667586,0.759873,0.824134,0.778040,0.815442,0.680610,0.874882,0.971970,0.684005,0.564701,0.885369,0.523036,0.682328,0.586831,0.938411,0.836036,0.661770,0.593546,0.561482,0.927048,0.604203,0.581004,0.557185,0.704497,0.786630,0.603106,0.559142,0.785809,0.545755,0.547415,0.745800,0.590545,0.751746,0.655350,0.643084,0.680333,0.954780,0.875715,0.668757,0.568280,0.950874,0.805025,0.864218,0.694462,0.519134,0.977536,0.866056,0.951757,0.646935,0.518247,0.558658,0.753843,0.928582,0.651412,0.651842,0.712982,0.985754,0.511914,0.886550,0.897122,0.511313,0.712227,0.732902,0.720305,0.723750,0.669549,0.777767,0.732685,0.710651,0.676441,0.562077,0.647767,0.545488,0.552589,0.915665,0.840417,0.588914,0.753736,0.663548,0.738028,0.754901,0.903940,0.808402,0.919678,0.899484,0.543812,0.701405,0.660228,0.743696,0.533156,0.627103,0.689899,0.766540,0.561818,0.505065,0.785725,0.573748,0.738946,0.911035,0.826225,0.540472,0.660449,0.588666,0.744202,0.802810,0.795561,0.703911,0.630976,0.701748,0.637217,0.610139,0.508409,0.598759,0.552048,0.787623,0.701488,0.765435,0.618901,0.906877,0.770951,0.735474,0.529009,0.727310,0.659846,0.872532,0.647505,0.618427,0.597508,0.964630,0.792642,0.741398,0.611935,0.602562,0.572144,0.574047,0.787256,0.927048,0.960257,0.563516,0.710322,0.561871,0.837553,0.527623,0.748102,0.887875,0.751835,0.712427,0.808668,0.845867,0.794660,0.745128,0.613813,0.686601,0.612016,0.749350,0.614844,0.851012,0.562000,0.790456,0.660380,0.631131,0.641730,0.694333,0.523797,0.882342,0.870661,0.584539,0.685373,0.810858,0.536734,0.603245,0.537280,0.567604,0.550484,0.639983,0.603286,0.587816,0.665412,0.850810,0.738263,0.914415,0.594170,0.534636,0.944696,0.963598,0.851442,0.889462,0.582070,0.970216,0.614331,0.714487,0.790086,0.857131,0.757396,0.546452,0.588382,0.508068,0.589762,0.559002,0.646739,0.624272,0.913247,0.833693,0.865720,0.980721,0.678589,0.715707,0.562161,0.676116,0.987382,0.624423,0.633874,0.555828,0.880680,0.861474,0.800892,0.917836,0.516568,0.967348,0.739978,0.547666,0.817098,0.702020,0.735101,0.944092,0.547544,0.554518,0.776711,0.801340,0.501430,0.836387,0.555057,0.687392,0.809664,0.584644,0.847717,0.591427,0.576328,0.555652,0.594139,0.605990,0.908368,0.660760,0.605122,0.572009,0.979864,0.701452,0.762089,0.536471,0.860816,0.927378,0.775321,0.661100,0.501947,0.511984,0.987622,0.989027,0.801420,0.566830,0.729123,0.658348,0.545784,0.924554,0.758618,0.519898,0.542471,0.829154,0.587250,0.708786,0.899056,0.662052,0.873972,0.598076,0.921709,0.521558,0.574604,0.867755,0.566890,0.851307,0.520662,0.564821,0.833153,0.738486,0.890866,0.815809,0.724042,0.800197,0.914319,0.598311,0.511991,0.670743,0.685127,0.862299,0.545403,0.591938,0.791710,0.538258,0.576294,0.554423,0.902383,0.582358,0.552376,0.643019,0.795068,0.604330,0.506192,0.840922,0.593801,0.774273,0.568144,0.997046,0.766014,0.704416,0.515452,0.825976,0.824359,0.856353,0.724061,0.585532,0.931097,0.831264,0.509120,0.654331,0.549468,0.716588,0.647381,0.994561,0.746649,0.622503,0.679467,0.617009,0.565495,0.625514,0.762086,0.664039,0.973780,0.997216,0.535575,0.896138,0.868943,0.722822,0.552218,0.675504,0.500514,0.507789,0.531074,0.668164,0.819308,0.878311,0.787788,0.529756,0.528575,0.887245,0.638328,0.548098,0.544980,0.538535,0.814134,0.635879,0.588369,0.970352,0.618741,0.529923,0.566847,0.504946,0.555341,0.702605,0.844484,0.628261,0.965514,0.685461,0.514313,0.775703,0.840890,0.601475,0.778478,0.548702,0.849081,0.680607,0.690538,0.653515,0.811807,0.629631,0.626011,0.994032,0.673246,0.828094,0.676133,0.548764,0.517693,0.819108,0.887279,0.722929,0.697782,0.958085,0.562328,0.793036,0.660097,0.645099,0.808813,0.948713,0.920773,0.501415,0.527527,0.844297,0.745911,0.645659,0.889001,0.543563,0.530394,0.504194,0.645130,0.656754,0.636546,0.864410,0.693801,0.830270,0.726786,0.543754,0.671127,0.543017,0.657629,0.969737,0.670477,0.948293,0.867441,0.653800,0.590192,0.794768,0.817182,0.944212,0.782877,0.540034,0.735181,0.566478,0.780120,0.564226,0.706706,0.862629,0.941797,0.879832,0.526929,0.768440,0.580880,0.768958,0.618546,0.588234,0.634548,0.712711,0.609551,0.712323,0.842471,0.660984,0.552044,0.588901,0.925513,0.527452,0.791654,0.707761,0.536920,0.724026,0.970317,0.855572,0.688692,0.856534,0.542210,0.764478,0.765705,0.627567,0.848349,0.613501,0.540841,0.711560,0.516528,0.765467,0.591234,0.718817,0.883287,0.569894,0.509960,0.504712,0.802565,0.641865,0.516323,0.786053,0.623670,0.504015,0.626531,0.761761,0.650476,0.836696,0.614359,0.934947,0.732544,0.578255,0.692411,0.569273,0.501999,0.566438,0.600992,0.589354,0.881836,0.772015,0.676901,0.632832,0.836287,0.753188,0.828591,0.734617,0.608675,0.636108,0.702905,0.594768,0.556867,0.733375,0.964899,0.831817,0.840764,0.740701,0.984263,0.591714,0.922894,0.864612,0.846726,0.645450,0.517823,0.850944,0.639273,0.885603,0.585723,0.886523,0.774563,0.841011,0.564260,0.742216,0.908132,0.706993,0.870453,0.993887,0.733546,0.502731,0.503165,0.968957,0.912085,0.574506,0.875873,0.614416,0.823729,0.592997,0.690057,0.575496,0.674775,0.554389,0.539006,0.524233,0.534417,0.549214,0.948821,0.626684,0.667442,0.641640,0.884784,0.948216,0.629566,0.697145,0.539745,0.821108,0.509272,0.729807,0.693893,0.567655,0.591513,0.749750,0.508315,0.520207,0.741066,0.846427,0.750910,0.757355,0.814140,0.514402,0.594535,0.539520,0.518919,0.542243,0.571300,0.958308,0.713453,0.952744,0.889352,0.619769,0.772431,0.768591,0.957085,0.603834,0.849182,0.778092,0.629421,0.999883,0.584462,0.964937,0.605315,0.598656,0.901277,0.600276,0.582126,0.628254,0.511305,0.981955,0.571589,0.947646,0.509348,0.511616,0.614280,0.668673,0.895582,0.787053,0.807380,0.503763,0.949691,0.769882,0.503498,0.697046,0.534640,0.522900,0.818794,0.625700,0.535057,0.700243,0.638499,0.550917,0.775060,0.572909,0.728792,0.618442,0.604022,0.900467,0.670267,0.684770,0.773992,0.595010,0.587479,0.538271,0.887208,0.500723,0.630104,0.631414,0.856686,0.682998,0.844744,0.652679,0.593066,0.821394,0.667579,0.850952,0.655395,0.911168,0.587064,0.880127,0.581355,0.593363,0.582758,0.655130,0.685250,0.982042,0.882672,0.500258,0.670173,0.856028,0.557744,0.659586,0.800654,0.783754,0.515289,0.756825,0.968516,0.593574,0.969060,0.653965,0.914325,0.751884,0.696842,0.761560,0.871988,0.804672,0.631860,0.580415,0.886407,0.660714,0.811344,0.528356,0.631588,0.810734,0.583111,0.546883,0.664099,0.580640,0.738914,0.778868,0.652343,0.690770,0.898897,0.707099,0.606090,0.660829,0.735306,0.834193,0.696489,0.724479,0.532111,0.756064,0.764533,0.752537,0.934644,0.552685,0.681591,0.966208,0.794428,0.724095,0.711339,0.857080,0.548101,0.853807,0.561956,0.628767,0.555232,0.601833,0.526801,0.809682,0.879113,0.948071,0.763403,0.597603,0.540875,0.787687,0.868913,0.803597,0.779216,0.509631,0.757413,0.651588,0.634777,0.845850,0.967350,0.554653,0.518577,0.558877,0.909521,0.612087,0.503300,0.715433,0.520331,0.779929,0.552121,0.622087,0.515006,0.909917,0.544289,0.760273,0.754831,0.621818,0.590683,0.804221,0.552681,0.791761,0.808030,0.694457,0.699647,0.954310,0.979200,0.581010,0.583847,0.875325,0.899414,0.561694,0.643973,0.824989,0.872373,0.874157,0.777254,0.504877,0.649423,0.586959,0.750274,0.801312,0.864851,0.545510,0.686229,0.542829,0.927014,0.566931,0.777758,0.507654,0.672113,0.516031,0.630268,0.762242,0.790257,0.952597,0.751103,0.698472,0.844521,0.679241,0.664326,0.774952,0.546595,0.632232,0.798054,0.672352,0.534711,0.500047,0.640532,0.842934,0.523419,0.684906,0.677183,0.787021,0.721933,0.557302,0.577199,0.534366,0.607312,0.717011,0.877763,0.529674,0.516852,0.516463,0.883531,0.709546,0.679919,0.510819,0.530335,0.780747,0.662668,0.734890,0.692913,0.980522,0.735754,0.512585,0.525393,0.894532,0.841201,0.774609,0.952029,0.615568,0.594510,0.781762,0.768193,0.741852,0.994615,0.944892,0.533015,0.757691,0.539006,0.604033,0.788862,0.595905,0.585286,0.658131,0.704516,0.734372,0.559477,0.856332,0.632093,0.562457,0.905632,0.897655,0.659018,0.786625,0.812557,0.868975,0.737105,0.632726,0.581023,0.728997,0.912687,0.860258,0.914011,0.611681,0.648356,0.561623,0.527495,0.644974,0.663989,0.643145,0.998443,0.684309,0.789033,0.644282,0.956184,0.501846,0.884249,0.632705,0.645532,0.926075,0.617252,0.540445,0.595737,0.844405,0.508163,0.676622,0.554717,0.953146,0.994032,0.577162,0.624229,0.539252,0.818203,0.787046,0.619506,0.714241,0.706563,0.544766,0.553622,0.753920,0.534286,0.677322,0.687208,0.523357,0.512032,0.827120,0.535715,0.584993,0.639029,0.850853,0.682276,0.505902,0.684585,0.989522,0.509961,0.526448,0.643698,0.702530,0.673909,0.740473,0.673324,0.629788,0.923730,0.589491,0.961209,0.700333,0.856681,0.595371,0.882504,0.637054,0.918396,0.853514,0.551101,0.527344,0.740772,0.974283,0.671298,0.975565,0.570511,0.585992,0.554389,0.694289,0.708258,0.539643,0.787144,0.841910,0.798117,0.721709,0.540309,0.726230,0.852283,0.943104,0.655910,0.911033,0.556285,0.700122,0.623381,0.933212,0.725105,0.589904,0.991996,0.932155,0.720133,0.560201,0.789670,0.813596,0.594543,0.669559,0.753976,0.689024,0.526053,0.608046,0.517751,0.514951,0.586566,0.868998,0.523798,0.672788,0.975144,0.763781,0.714117,0.592829,0.570020,0.773006,0.506225,0.593630,0.688754,0.908112,0.625636,0.629973,0.788027,0.869018,0.866539,0.547021,0.662501,0.655447,0.829412,0.548546,0.998361,0.686963,0.733419,0.738316,0.517005,0.877859,0.846255,0.806443,0.519421,0.503419,0.679063,0.736078,0.829876,0.611193,0.588174,0.695702,0.750184,0.508503,0.517310,0.630168,0.559030,0.621868,0.607373,0.551578,0.757732,0.502916,0.997155,0.653543,0.911830,0.836737,0.664251,0.523716,0.877660,0.909599,0.549689,0.760993,0.853481,0.875448,0.721681,0.557563,0.536171,0.578683,0.502423,0.506190,0.604208,0.855379,0.824231,0.706468,0.830921,0.556859,0.753334,0.690172,0.621169,0.598635,0.551286,0.681033,0.679128,0.527227,0.690263,0.605804,0.713599,0.665601,0.509678,0.656047,0.567467,0.635049,0.638170,0.838345,0.731659,0.585750,0.683414,0.731099,0.859808,0.912613,0.627450,0.739619,0.957328,0.803695,0.615131,0.523735,0.569415,0.719476,0.510916,0.559187,0.633225,0.729340,0.680661,0.874199,0.517360,0.891352,0.514971,0.620532,0.575369,0.649265,0.522793,0.948074,0.507206,0.544973,0.986376,0.843459,0.972996,0.515960,0.740055,0.906032,0.617895,0.744997,0.780497,0.641446,0.572475,0.570248,0.776741,0.656806,0.741185,0.875124,0.716215,0.813938,0.762620,0.651220,0.506856,0.785121,0.503707,0.665109,0.687482,0.626043,0.576218,0.615078,0.989739,0.853681,0.936782,0.754011,0.645099,0.653825,0.533264,0.629207,0.520719,0.579400,0.725118,0.738090,0.710575,0.679182,0.587862,0.699506,0.847948,0.569183,0.943637,0.535079,0.654079,0.799483,0.638886,0.841425,0.514295,0.521553,0.514505,0.581678,0.601638,0.563545,0.746515,0.950223,0.502843,0.995639,0.539731,0.890840,0.617031,0.563150,0.942301,0.819735,0.865678,0.550674,0.727480,0.969772,0.906965,0.524406,0.588920,0.605841,0.560839,0.705016,0.653359,0.505921,0.661108,0.548461,0.777737,0.888696,0.951623,0.885624,0.919898,0.806652,0.645703,0.694164,0.627499,0.592710,0.830626,0.921117,0.531256,0.558866,0.840158,0.585529,0.973198,0.549608,0.930742,0.519816,0.883168,0.979287,0.519518,0.798658,0.730154,0.731729,0.576148,0.763555,0.553433,0.514950,0.568457,0.704512,0.538809,0.630573,0.610613,0.647011,0.562935,0.716368,0.571290,0.598297,0.682463,0.640770,0.830249,0.743710,0.828050,0.520800,0.630247,0.900424,0.636985,0.871498,0.683221,0.526617,0.641528,0.572978,0.658054,0.837993,0.868649,0.546945,0.537641,0.732824,0.539984,0.981100,0.713716,0.580878,0.879625,0.789466,0.754818,0.889699,0.717948,0.740162,0.702038,0.847797,0.766010,0.722036,0.554819,0.817107,0.910165,0.856910,0.585628,0.821117,0.594018,0.944574,0.557658,0.716018,0.558173,0.553637,0.861892,0.588647,0.558321,0.671895,0.513141,0.698360,0.981673,0.777258,0.537191,0.762234,0.800935,0.623474,0.844576,0.528808,0.564898,0.884992,0.570103,0.725544,0.771264,0.743264,0.593117,0.707779,0.669292,0.633092,0.667694,0.961299,0.594728,0.924544,0.676193,0.502015,0.555185,0.681121,0.914300,0.879453,0.536963,0.538625,0.693806,0.529588,0.519448,0.835814,0.765122,0.916479,0.519941,0.789250,0.800119,0.521579,0.503327,0.708809,0.594923,0.784596,0.600029,0.808482,0.743516,0.653276,0.706309,0.955439,0.849036,0.819361,0.755828,0.896289,0.860392,0.680765,0.765113,0.843496,0.809714,0.511001,0.859830,0.652558,0.573559,0.618146,0.901162,0.625543,0.636437,0.812558,0.505343,0.637803,0.688027,0.585255,0.858775,0.580139,0.680211,0.721021,0.716546,0.740670,0.984819,0.948062,0.897738,0.920687,0.680325,0.681314,0.532014,0.698642,0.548048,0.665103,0.589149,0.758241,0.966808,0.550053,0.573411,0.860265,0.733672,0.820449,0.586039,0.894096,0.974078,0.518628,0.738741,0.505255,0.863058,0.832725,0.621191,0.738671,0.681701,0.566079,0.721180,0.550969,0.864184,0.860888,0.859082,0.744085,0.674639,0.720706,0.909373,0.540491,0.605130,0.561066,0.779873,0.953141,0.518408,0.905300,0.650005,0.741904,0.602477,0.603243,0.860346,0.517139,0.513480,0.670696,0.578951,0.973989,0.821322,0.603677,0.666124,0.983364,0.935618,0.686227,0.837101,0.838880,0.659755,0.595781,0.684134,0.639064,0.883209,0.557471,0.552449,0.751796,0.832358,0.658251,0.919565,0.766137,0.739805,0.798030,0.976707,0.858319,0.643732,0.678086,0.605971,0.646826,0.886577,0.672607,0.532406,0.531504,0.856161,0.616943,0.879773,0.692710,0.939542,0.536185,0.589765,0.967304,0.616994,0.661161,0.739818,0.981398,0.903329,0.875471,0.568599,0.643069,0.598621,0.858811,0.638815,0.568402,0.604484,0.856426,0.743438,0.551262,0.770651,0.719849,0.647660,0.556122,0.517948,0.994990,0.687923,0.641882,0.627393,0.598365,0.912343,0.940002,0.620601,0.560717,0.739856,0.530309,0.950200,0.788235,0.995293,0.986639,0.929706,0.983241,0.853449,0.666237,0.632287,0.521117,0.533373,0.821602,0.741439,0.506378,0.878675,0.615521,0.820541,0.512090,0.517634,0.693842,0.696090,0.677445,0.732495,0.683163,0.693052,0.592231,0.892530,0.990264,0.590580,0.621875,0.534198,0.734779,0.634461,0.503792,0.754255,0.608281,0.558251,0.998825,0.617179,0.602301,0.951953,0.558945,0.821478,0.554557,0.639875,0.530459,0.551530,0.812609,0.910208,0.797725,0.742340,0.712534,0.571245,0.978688,0.509601,0.698074,0.602466,0.606662,0.912836,0.740099,0.958436,0.784322,0.512189,0.747504,0.730494,0.521703,0.808186,0.687154,0.604687,0.906023,0.808052,0.610147,0.939337,0.899741,0.550409,0.908545,0.854772,0.525901,0.782569,0.566489,0.504891,0.768187,0.631540,0.915552,0.652520,0.857014,0.820451,0.522627,0.786801,0.748330,0.578004,0.899379,0.960650,0.545881,0.522638,0.542549,0.702466,0.837811,0.627009,0.624769,0.510119,0.507164,0.805499,0.611761,0.682176,0.685689,0.697791,0.938944,0.999943,0.678040,0.631707,0.544970,0.503810,0.870571,0.988149,0.627081,0.721693,0.689939,0.599727,0.503680,0.816811,0.707030,0.577095,0.695340,0.509973,0.635585,0.642438,0.862561,0.871525,0.972755,0.784744,0.903761,0.532203,0.665691,0.782309,0.558540,0.715799,0.705359,0.596817,0.809355,0.540739,0.739549,0.607001,0.728105,0.807842,0.810794,0.848827,0.726413,0.631819,0.531227,0.845011,0.921999,0.688383,0.658843,0.821338,0.610454,0.874169,0.513411,0.569805,0.865738,0.802512,0.552062,0.762271,0.544682,0.882542,0.969372,0.559102,0.684089,0.587827,0.809651,0.537729,0.571501,0.846141,0.561335,0.641206,0.774653,0.903152,0.943443,0.670565,0.617679,0.618896,0.860495,0.571783,0.601535,0.509767,0.542928,0.978812,0.523960,0.625942,0.540365,0.637787,0.924174,0.768191,0.565354,0.621643,0.571994,0.721129,0.826159,0.564521,0.671848,0.844046,0.554715,0.910020,0.599517,0.567970,0.720446,0.608531,0.729506,0.657208,0.980351,0.952012,0.534219,0.699416,0.882231,0.506035,0.759388,0.786124,0.568044,0.947138,0.663616,0.655374,0.784355,0.502270,0.746736,0.569631,0.870183,0.948933,0.620941,0.913484,0.849590,0.515165,0.641389,0.588003,0.554386,0.658802,0.567635,0.696677,0.530389,0.689968,0.923853,0.748107,0.700264,0.612052,0.899016,0.683367,0.779529,0.949334,0.678817,0.655813,0.751234,0.831805,0.540286,0.704530,0.515581,0.836468,0.836380,0.523875,0.622790,0.597630,0.532668,0.789617,0.633134,0.574732,0.642896,0.732720,0.638204,0.556389,0.661845,0.568897,0.788543,0.683882,0.950768,0.661160,0.619054,0.715668,0.645081,0.515778,0.722056,0.877809,0.582701,0.598521,0.538568,0.664468,0.706806,0.605405,0.960747,0.689831,0.529203,0.928482,0.835476,0.700677,0.517984,0.925153,0.889198,0.543185,0.566298,0.573941,0.550995,0.822002,0.955348,0.803089,0.915526,0.807781,0.697139,0.768121,0.551261,0.620166,0.939374,0.671057,0.943369,0.690180,0.615226,0.671943,0.764419,0.700653,0.948274,0.507986,0.674123,0.818982,0.624414,0.517159,0.633287,0.747406,0.977911,0.757121,0.587882,0.681987,0.589449,0.742969,0.683045,0.599767,0.805502,0.599947,0.775369,0.668431,0.922711,0.539972,0.548269,0.814304,0.812028,0.821569,0.717086,0.904774,0.697958,0.618874,0.543346,0.678460,0.700275,0.794364,0.648901,0.818513,0.789471,0.508687,0.769480,0.594664,0.896574,0.928359,0.638146,0.519283,0.791015,0.797242,0.581047,0.914770,0.514147,0.895421,0.734292,0.670353,0.750882,0.737576,0.819211,0.607157,0.955174,0.840009,0.963337,0.575236,0.919445,0.517489,0.740238,0.915856,0.662031,0.745728,0.810257,0.690616,0.934630,0.501133,0.635036,0.740891,0.719368,0.770317,0.845143,0.589386,0.666993,0.657213,0.704464,0.944730,0.752354,0.739559,0.832606,0.538029,0.792632,0.526195,0.663347,0.505635,0.821863,0.747991,0.665716,0.642067,0.875418,0.578386,0.796762,0.626991,0.574402,0.691575,0.752392,0.710714,0.621860,0.737400,0.747817,0.755175,0.678706,0.831461,0.975673,0.707711,0.837771,0.678847,0.557071,0.501690,0.507513,0.547200,0.968405,0.514503,0.702826,0.655623,0.519709,0.540006,0.838699,0.977410,0.724565,0.988354,0.878455,0.745884,0.652016,0.615007,0.605843,0.525981,0.649339,0.942155,0.992509,0.513775,0.690104,0.675626,0.911251,0.699056,0.627170,0.887443,0.570414,0.909173,0.864130,0.564610,0.702340,0.762589,0.804081,0.656461,0.912476,0.648082,0.595954,0.734599,0.516109,0.584765,0.802485,0.797099,0.613964,0.704262,0.843244,0.622057,0.685841,0.737906,0.696739,0.725633,0.650209,0.606300,0.667517,0.720820,0.653301,0.766241,0.955538,0.747785,0.766984,0.700348,0.592163,0.544758,0.831072,0.853677,0.898127,0.676016,0.657760,0.713477,0.645795,0.827724,0.692283,0.574520,0.934495,0.825625,0.600517,0.631821,0.734499,0.574581,0.724486,0.971557,0.647330,0.539327,0.968114,0.981933,0.672912,0.516267,0.875877,0.510323,0.737927,0.680578,0.776306,0.873313,0.799325,0.699191,0.863428,0.919259,0.680869,0.750912,0.582871,0.580755,0.578267,0.859137,0.672221,0.740857,0.749452,0.846535,0.604187,0.819449,0.819108,0.617083,0.595063,0.600064,0.602341,0.707766,0.546549,0.853753,0.876946,0.665903,0.803685,0.582504,0.592437,0.594247,0.900814,0.515099,0.930317,0.509053,0.539443,0.765016,0.636468,0.748199,0.796362,0.720036,0.767344,0.579526,0.820957,0.570086,0.944296,0.804500,0.610647,0.969645,0.800874,0.570167,0.515045,0.775683,0.830067,0.609870,0.628766,0.809672,0.926867,0.540339,0.652523,0.797619,0.944582,0.688133,0.578744,0.837984,0.788866,0.732607,0.719694,0.778050,0.851429,0.750560,0.931577,0.894980,0.850757,0.722208,0.827853,0.571995,0.911024,0.950787,0.539431,0.977575,0.601653,0.543627,0.789545,0.973542,0.970583,0.700972,0.940033,0.801730,0.594936,0.804048,0.828610,0.667056,0.565721,0.999371,0.648565,0.875201,0.724818,0.847516,0.624933,0.755389,0.905317,0.689700,0.791906,0.708529,0.580342,0.875127,0.521643,0.735634,0.565872,0.549486,0.985046,0.928087,0.597988,0.745841,0.516319,0.558072,0.701575,0.620177,0.630333,0.832317,0.570002,0.921445,0.865221,0.929373,0.689598,0.648557,0.891558,0.955484,0.648120,0.782756,0.528715,0.585873,0.881788,0.930319,0.609616,0.932296,0.601938,0.860071,0.903928,0.699046,0.823005,0.924768,0.520674,0.722739,0.537306,0.748194,0.816556,0.512995,0.706240,0.500075,0.611246,0.826267,0.932905,0.932827,0.599599,0.609604,0.712326,0.891660,0.985503,0.601826,0.651449,0.702638,0.907891,0.581757,0.506641,0.614182,0.568860,0.583674,0.647632,0.762496,0.539869,0.570330,0.687838,0.620887,0.887334,0.506892,0.592792,0.907976,0.975354,0.631750,0.809165,0.741964,0.908603,0.835673,0.641306,0.517427,0.537726,0.706651,0.644151,0.705276,0.994217,0.885267,0.620875,0.861380,0.827317,0.902357,0.711928,0.931338,0.521116,0.646319,0.621057,0.503154,0.641822,0.722404,0.613407,0.853875,0.683177,0.765900,0.798650,0.779896,0.697298,0.523091,0.979087,0.824484,0.589674,0.613819,0.670655,0.510443,0.975926,0.782240,0.595852,0.973738,0.874843,0.928162,0.659140,0.611111,0.556799,0.839089,0.586250,0.778208,0.517538,0.775342,0.945065,0.644477,0.737438,0.627094,0.956871,0.968213,0.596002,0.693721,0.847740,0.572196,0.783392,0.720785,0.646879,0.814239,0.733369,0.620949,0.925019,0.569006,0.833920,0.628588,0.808981,0.819016,0.917940,0.753710,0.749582,0.607750,0.620228,0.533676,0.632421,0.562744,0.512237,0.560433,0.528668,0.888737,0.826084,0.742771,0.694827,0.793228,0.950090,0.564933,0.765583,0.630458,0.580107,0.508306,0.759228,0.659215,0.503892,0.539766,0.895390,0.940728,0.547925,0.588176,0.670408,0.500038,0.575403,0.780608,0.662309,0.928650,0.919057,0.766371,0.650050,0.883143,0.832769,0.682937,0.508839,0.552040,0.515233,0.677860,0.877391,0.556971,0.846890,0.675646,0.964187,0.546688,0.898612,0.658422,0.984547,0.773200,0.620811,0.768497,0.745113,0.830006,0.833799,0.575073,0.762178,0.740744,0.594235,0.831070,0.578273,0.896818,0.884861,0.712860,0.787002,0.832448,0.535686,0.877912,0.589574,0.662236,0.548567,0.629366,0.542206,0.511965,0.947202,0.878515,0.689855,0.555641,0.792906,0.592946,0.836387,0.708445,0.837497,0.951231,0.776972,0.689473,0.675372,0.881745,0.740096,0.582948,0.763024,0.596731,0.626581,0.882342,0.929837,0.588783,0.967115,0.842383,0.939975,0.784895,0.506296,0.705250,0.583221,0.734953,0.593944,0.645405,0.947994,0.536099,0.837796,0.564615,0.825374,0.707737,0.537701,0.523544,0.516267,0.819228,0.867209,0.535737,0.696224,0.939379,0.771094,0.681690,0.528791,0.646554,0.502706,0.940546,0.822370,0.544868,0.755949,0.601610,0.574495,0.519451,0.504600,0.972705,0.506251,0.946334,0.699178,0.512356,0.625866,0.845594,0.527238,0.923742,0.699163,0.998089,0.761416,0.935813,0.815194,0.881883,0.507219,0.770868,0.743028,0.768030,0.667484,0.829277,0.518683,0.872148,0.750408,0.722318,0.514095,0.743135,0.983421,0.578150,0.819613,0.748081,0.591816,0.635379,0.511531,0.705465,0.663582,0.824378,0.780797,0.934508,0.767530,0.568547,0.547652,0.640668,0.798786,0.606194,0.919105,0.620134,0.729870,0.985512,0.533747,0.632268,0.546139,0.657647,0.634164,0.873429,0.833594,0.955947,0.859683,0.667615,0.575082,0.567435,0.535155,0.540812,0.679789,0.931620,0.509487,0.531345,0.510749,0.622067,0.508540,0.868980,0.814367,0.760682,0.749378,0.806374,0.674275,0.608842,0.884509,0.872332,0.605576,0.510192,0.553570,0.546369,0.692951,0.622883,0.523243,0.863836,0.755527,0.685080,0.527179,0.904141,0.570493,0.547207,0.679613,0.737559,0.822780,0.782089,0.919983,0.512008,0.639457,0.833816,0.698922,0.682276,0.707729,0.790168,0.715505,0.531997,0.533885,0.836758,0.976400,0.880598,0.935257,0.863133,0.970445,0.743829,0.608008,0.610206,0.572859,0.876730,0.890670,0.544206,0.508675,0.644608,0.546149,0.747868,0.647378,0.978311,0.719675,0.875011,0.596028,0.560521,0.758498,0.883857,0.686425,0.554035,0.543507,0.563859,0.740696,0.769732,0.538366,0.553490,0.633558,0.636654,0.609673,0.534652,0.517667,0.659424,0.658390,0.786063,0.853027,0.920034,0.766521,0.581890,0.867709,0.577445,0.756812,0.567848,0.537376,0.925333,0.535335,0.682073,0.506363,0.694578,0.894269,0.819872,0.651188,0.641626,0.598475,0.809722,0.620602,0.558982,0.511647,0.523010,0.547522,0.865829,0.589758,0.880564,0.995627,0.852400,0.757113,0.576467,0.945646,0.546346,0.685447,0.791173,0.681116,0.947035,0.530259,0.520163,0.645544,0.861225,0.752151,0.555668,0.508224,0.872937,0.713847,0.894771,0.547740,0.575263,0.814781,0.712833,0.622291,0.826901,0.803768,0.863089,0.642810,0.507832,0.840972,0.873962,0.863547,0.730425,0.551874,0.939015,0.600479,0.529042,0.813911,0.504312,0.815166,0.554295,0.567446,0.683472,0.532189,0.848005,0.869312,0.688619,0.657524,0.607948,0.661540,0.681748,0.727611,0.590029,0.668318,0.891498,0.555273,0.582479,0.500036,0.689846,0.543396,0.689077,0.537890,0.788757,0.538296,0.513451,0.826851,0.771590,0.632648,0.858979,0.705986,0.634073,0.863769,0.624757,0.537683,0.808978,0.799066,0.537319,0.922478,0.985077,0.585810,0.587309,0.803302,0.656777,0.795428,0.915204,0.541880,0.595739,0.633621,0.550967,0.871674,0.854028,0.848659,0.669350,0.710101,0.738735,0.575484,0.596099,0.831203,0.512384,0.908694,0.688019,0.983486,0.586659,0.945447,0.934471,0.561195,0.782070,0.980915,0.806390,0.543750,0.848520,0.712167,0.656831,0.599724,0.951160,0.960737,0.611547,0.512970,0.736807,0.656674,0.869225,0.638683,0.603457,0.714121,0.534276,0.630194,0.528468,0.581607,0.861022,0.759663,0.513078,0.686110,0.529982,0.740883,0.780207,0.844804,0.760057,0.600821,0.889347,0.943182,0.561090,0.736228,0.828697,0.510918,0.774045,0.787498,0.588324,0.578675,0.878887,0.858961,0.935582,0.759223,0.843603,0.718372,0.672537,0.551411,0.750257,0.631307,0.578815,0.810443,0.801392,0.555654,0.879445,0.558937,0.962311,0.866388,0.605484,0.770651,0.560481,0.793923,0.524702,0.958016,0.850224,0.684870,0.682216,0.922432,0.596202,0.745348,0.938390,0.590997,0.878611,0.540976,0.510267,0.780891,0.773409,0.975330,0.666349,0.723766,0.719759,0.641699,0.575856,0.953787,0.812494,0.644167,0.931260,0.776956,0.523981,0.569145,0.714657,0.782929,0.560585,0.831577,0.661458,0.846533,0.637556,0.935009,0.673340,0.931265,0.599051,0.846452,0.524615,0.552203,0.765511,0.663106,0.542618,0.864710,0.903775,0.714709,0.905897,0.929202,0.590249,0.778114,0.514443,0.515116,0.652971,0.845289,0.707448,0.578306,0.625641,0.566532,0.781839,0.939178,0.543122,0.850610,0.589324,0.793259,0.989048,0.806092,0.542132,0.595724,0.737620,0.559655,0.846121,0.636926,0.578684,0.655492,0.735113,0.722144,0.945633,0.766504,0.980193,0.852850,0.645362,0.708527,0.500164,0.841768,0.598481,0.533885,0.978954,0.741611,0.604973,0.601136,0.648841,0.748879,0.645627,0.909274,0.720192,0.525830,0.713799,0.616032,0.540850,0.756657,0.699723,0.627719,0.571840,0.664197,0.996262,0.870388,0.585174,0.724097,0.953383,0.663153,0.575176,0.596163,0.576779,0.872767,0.604634,0.682482,0.558841,0.650998,0.670450,0.897361,0.675891,0.789872,0.931530,0.604927,0.676739,0.503796,0.733711,0.567717,0.853536,0.962983,0.641278,0.559850,0.618512,0.985044,0.545389,0.562815,0.880683,0.510254,0.693768,0.639094,0.689095,0.808717,0.652641,0.550849,0.662257,0.876308,0.723716,0.792778,0.600246,0.748632,0.596496,0.895325,0.767711,0.690938,0.605201,0.670371,0.632639,0.543882,0.730656,0.840114,0.726259,0.698270,0.583901,0.568932,0.598922,0.840509,0.639497,0.555614,0.591222,0.858038,0.947042,0.789067,0.627350,0.615338,0.843034,0.574533,0.674236,0.621038,0.741340,0.584257,0.524038,0.674355,0.584683,0.993550,0.866476,0.640565,0.647136,0.699896,0.546429,0.554659,0.857396,0.745682,0.598629,0.527781,0.873166,0.850193,0.539507,0.665953,0.612275,0.795175,0.943232,0.566588,0.951659,0.617844,0.916930,0.559438,0.554005,0.807088,0.889678,0.513391,0.675926,0.502032,0.583555,0.984011,0.507944,0.805702,0.737073,0.803216,0.514422,0.651314,0.593401,0.615964,0.695072,0.586526,0.751752,0.604626,0.716253,0.768352,0.702277,0.676063,0.549146,0.882623,0.878248,0.837079,0.570277,0.578118,0.542909,0.610233,0.850281,0.977470,0.566326,0.676020,0.537069,0.686483,0.939722,0.585751,0.778250,0.815663,0.825575,0.621007,0.823149,0.835059,0.768556,0.719530,0.563782,0.881233,0.673889,0.511591,0.859552,0.513243,0.513560,0.638934,0.742707,0.751950,0.672864,0.582841,0.526955,0.996169,0.621576,0.653303,0.796126,0.720659,0.574593,0.628228,0.938972,0.569015,0.607956,0.715335,0.953847,0.897912,0.693749,0.501357,0.792989,0.871852,0.626832,0.889525,0.704097,0.542038,0.547039,0.896610,0.734387,0.967742,0.912303,0.875963,0.661225,0.580190,0.558133,0.592388,0.828828,0.833067,0.866592,0.536475,0.703800,0.694917,0.662169,0.880901,0.649020,0.881529,0.515141,0.558578,0.850199,0.647062,0.542775,0.552943,0.902267,0.941214,0.892425,0.870192,0.505272,0.982902,0.671685,0.816773,0.512205,0.886470,0.944022,0.904530,0.667370,0.657392,0.704641,0.524369,0.714674,0.824980,0.692016,0.861167,0.578332,0.974868,0.940726,0.581272,0.663276,0.529169,0.551110,0.531122,0.642476,0.601507,0.751443,0.568088,0.805066,0.813797,0.769682,0.650448,0.688534,0.981660,0.629394,0.689181,0.747822,0.781955,0.937559,0.592181,0.592177,0.925751,0.657590,0.905534,0.865092,0.738430,0.664585,0.500499,0.849564,0.636838,0.758610,0.596771,0.511954,0.692515,0.696458,0.621056,0.615624,0.540762,0.616414,0.780230,0.793051,0.592266,0.751449,0.720333,0.891404,0.702723,0.786762,0.713958,0.726181,0.785018,0.948607,0.718895,0.982853,0.596074,0.557102,0.678036,0.965515,0.964247,0.861723,0.648435,0.825773,0.597753,0.550636,0.653099,0.579608,0.948209,0.521900,0.553011,0.695917,0.525097,0.766259,0.628023,0.719772,0.843433,0.729124,0.649818,0.576760,0.877194,0.512738,0.563181,0.858096,0.709697,0.604029,0.635111,0.882892,0.587354,0.563911,0.961549,0.506780,0.702015,0.627300,0.510375,0.600771,0.579626,0.672262,0.610249,0.508186,0.682708,0.551394,0.548947,0.569149,0.517351,0.577369,0.715507,0.838189,0.537132,0.724807,0.643721,0.899688,0.623620,0.846314,0.500633,0.504943,0.640716,0.738424,0.575344,0.661466,0.946687,0.537471,0.771984,0.887991,0.736763,0.560027,0.643939,0.924893,0.735856,0.554460,0.822526,0.859021,0.700483,0.759973,0.587829,0.547549,0.569657,0.640707,0.525106,0.577459,0.777420,0.673103,0.660072,0.781191,0.634138,0.648253,0.590986,0.806840,0.578199,0.677801,0.804458,0.854739,0.695379,0.573087,0.664748,0.792984,0.529399,0.892515,0.951106,0.613374,0.575333,0.629199,0.552682,0.551439,0.741668,0.746333,0.678385,0.713665,0.637238,0.907642,0.835163,0.572003,0.520518,0.552564,0.675354,0.526950,0.558958,0.513660,0.538903,0.672786,0.863039,0.656761,0.937584,0.720212,0.526639,0.788209,0.580134,0.543509,0.712051,0.915210,0.652607,0.649045,0.905032,0.724200,0.827413,0.784142,0.613418,0.827040,0.531426,0.537432,0.605692,0.647998,0.894825,0.708373,0.999388,0.849581,0.836560,0.561207,0.698234,0.766863,0.508828,0.560489,0.505220,0.571684,0.638530,0.580760,0.598193,0.703127,0.839651,0.746656,0.812293,0.536339,0.698464,0.632547,0.670173,0.575362,0.810850,0.551129,0.504321,0.667631,0.785101,0.751576,0.691648,0.771469,0.503106,0.670925,0.681877,0.604325,0.960415,0.793661,0.901733,0.908245,0.656321,0.776002,0.682802,0.822969,0.611916,0.624656,0.555294,0.649132,0.671334,0.576521,0.707591,0.519734,0.643300,0.970193,0.704300,0.596988,0.542200,0.736881,0.664395,0.630948,0.935040,0.806086,0.870018,0.708728,0.580987,0.832039,0.954372,0.888417,0.684896,0.713566,0.597576,0.552740,0.541063,0.506759,0.564291,0.624909,0.511836,0.734721,0.693543,0.640151,0.827001,0.998371,0.829231,0.998719,0.668071,0.591072,0.866075,0.572570,0.933932,0.549474,0.741078,0.645470,0.648913,0.692864,0.985578,0.795948,0.597231,0.561998,0.943706,0.801989,0.873452,0.798919,0.900354,0.710389,0.651635,0.920770,0.565786,0.926411,0.567644,0.615119,0.720653,0.751490,0.761182,0.630410,0.721747,0.592576,0.767768,0.580964,0.670616,0.582008,0.553325,0.958367,0.621302,0.510200,0.550509,0.580050,0.878639,0.796333,0.889662,0.949306,0.693330,0.742287,0.505653,0.746373,0.739554,0.664312,0.767394,0.746496,0.536371,0.847067,0.956279,0.920548,0.595434,0.768721,0.712663,0.583119,0.699977,0.562718,0.519145,0.524016,0.758971,0.800887,0.580452,0.578926,0.603124,0.712466,0.727544,0.793919,0.775638,0.576068,0.916870,0.792222,0.503317,0.717856,0.735410,0.721095,0.805863,0.994152,0.884160,0.897776,0.743541,0.586885,0.551910,0.589377,0.772216,0.612750,0.675771,0.616399,0.761408,0.505002,0.723549,0.882378,0.690212,0.564903,0.629871,0.844608,0.728336,0.586799,0.635272,0.918438,0.718785,0.671782,0.579799,0.671935,0.958194,0.602987,0.659538,0.545190,0.518178,0.680505,0.585395,0.614834,0.754187,0.864890,0.621083,0.654187,0.798988,0.602521,0.704655,0.515314,0.854878,0.500186,0.793160,0.664947,0.816100,0.742038,0.516745,0.770783,0.756803,0.611543,0.563377,0.730089,0.576883,0.839289,0.742714,0.524769,0.594549,0.628041,0.566901,0.740858,0.763758,0.805155,0.506537,0.673501,0.521574,0.785439,0.857635,0.756810,0.788927,0.681454,0.665358,0.616753,0.599376,0.552269,0.687881,0.648137,0.683078,0.644199,0.607968,0.579856,0.637221,0.708077,0.538177,0.580171,0.606230,0.733621,0.538502,0.780961,0.512886,0.534199,0.956075,0.725551,0.517426,0.682817,0.515941,0.909870,0.672267,0.571454,0.940081,0.871593,0.506237,0.982361,0.586385,0.864844,0.706031,0.849783,0.631475,0.961640,0.783671,0.833996,0.753428,0.618075,0.819804,0.694757,0.974902,0.566345,0.579199,0.593976,0.590361,0.975504,0.609605,0.736756,0.739741,0.731267,0.617472,0.775130,0.957949,0.975726,0.851812,0.853477,0.886929,0.534889,0.518783,0.614890,0.560723,0.669251,0.668998,0.670615,0.653501,0.594032,0.529164,0.537066,0.841506,0.822350,0.762996,0.762403,0.947341,0.590261,0.553608,0.780998,0.719100,0.797657,0.597270,0.982537,0.612438,0.589389,0.948445,0.524769,0.552310,0.537291,0.955157,0.573063,0.943642,0.719873,0.627505,0.785216,0.626226,0.831825,0.559008,0.753643,0.731125,0.552765,0.669195,0.557212,0.534108,0.558394,0.571692,0.561916,0.616654,0.696034,0.862027,0.902008,0.915244,0.521894,0.873014,0.585876,0.797346,0.705322,0.623839,0.726885,0.555693,0.712990,0.524774,0.811152,0.509967,0.687556,0.692184,0.513012,0.591827,0.854785,0.606995,0.549151,0.659502,0.503664,0.580762,0.770855,0.621224,0.808224,0.707830,0.976548,0.739390,0.523924,0.787383,0.666386,0.616342,0.858960,0.886807,0.770617,0.590336,0.520018,0.514817,0.772841,0.578451,0.677735,0.694364,0.793253,0.535978,0.594024,0.754226,0.816268,0.833263,0.596806,0.720045,0.848849,0.704478,0.669237,0.817389,0.586828,0.774122,0.660712,0.660301,0.537235,0.855746,0.561053,0.932609,0.714091,0.804791,0.590843,0.944688,0.559345,0.735940,0.733523,0.806964,0.661551,0.595704,0.811448,0.639226,0.614282,0.531977,0.647766,0.963624,0.633375,0.639766,0.532203,0.997891,0.856813,0.505439,0.709506,0.594721,0.708817,0.837622,0.626832,0.755892,0.581924,0.872122,0.726490,0.611497,0.543975,0.502402,0.653163,0.919094,0.545700,0.779922,0.548498,0.700131,0.542139,0.725938,0.526641,0.591708,0.597574,0.906403,0.928160,0.515169,0.798093,0.565550,0.850063,0.988714,0.863399,0.930796,0.539690,0.694856,0.805280,0.772167,0.644364,0.512607,0.557169,0.759685,0.538016,0.672674,0.755617,0.522008,0.941643,0.630970,0.941018,0.540061,0.649745,0.577755,0.778251,0.876819,0.678831,0.735589,0.847606,0.806344,0.578665,0.610132,0.592426,0.930766,0.762878,0.576514,0.593992,0.885659,0.685352,0.690762,0.872793,0.564989,0.862213,0.626446,0.729967,0.536468,0.911106,0.635956,0.710822,0.767350,0.725826,0.684526,0.678477,0.657443,0.963810,0.752929,0.768925,0.564524,0.709007,0.895771,0.503319,0.736640,0.515285,0.849481,0.512060,0.568378,0.900766,0.851508,0.585805,0.816201,0.651691,0.853770,0.938575,0.983794,0.564543,0.745038,0.818853,0.852964,0.556922,0.621128,0.613490,0.598977,0.990940,0.561622,0.729959,0.750864,0.579145,0.695113,0.684576,0.734771,0.614401,0.635214,0.846864,0.530343,0.558174,0.901063,0.839815,0.698118,0.770529,0.778374,0.693965,0.881830,0.589113,0.814646,0.612406,0.761832,0.519775,0.556300,0.824666,0.650621,0.912234,0.673390,0.626583,0.701083,0.562031,0.880077,0.834929,0.954657,0.954137,0.541849,0.714988,0.778158,0.675492,0.737971,0.946717,0.831912,0.852961,0.626401,0.661460,0.999543,0.710673,0.599824,0.858677,0.667548,0.630103,0.766114,0.523888,0.802766,0.620033,0.782536,0.728569,0.936524,0.683786,0.604877,0.532939,0.576164,0.642227,0.679909,0.535473,0.716464,0.724549,0.872425,0.933930,0.925653,0.981245,0.978900,0.523856,0.898577,0.767198,0.811337,0.883975,0.663296,0.590828,0.965517,0.574045,0.661881,0.896026,0.571371,0.936638,0.624513,0.909756,0.514817,0.612875,0.694464,0.532673,0.919318,0.749505,0.589835,0.533833,0.674965,0.841192,0.631774,0.857024,0.938192,0.526073,0.528733,0.985627,0.808848,0.893657,0.978474,0.900572,0.600847,0.629714,0.664533,0.613254,0.945792,0.821672,0.601755,0.764045,0.886421,0.560956,0.503473,0.667568,0.860413,0.531492,0.624762,0.728781,0.673221,0.711301,0.511364,0.537562,0.877627,0.920221,0.760270,0.548345,0.825340,0.623394,0.762689,0.915220,0.962714,0.769390,0.840269,0.756142,0.937096,0.803066,0.673132,0.777618,0.577838,0.782682,0.863581,0.780995,0.547731,0.553784,0.946617,0.592254,0.887951,0.782280,0.555294,0.811170,0.820156,0.684502,0.565709,0.598905,0.578971,0.543304,0.821071,0.912644,0.517984,0.635159,0.777323,0.715536,0.503801,0.999235,0.550176,0.508293,0.787728,0.833174,0.565410,0.539123,0.842923,0.607031,0.728275,0.976778,0.545099,0.806712,0.841748,0.922345,0.619727,0.709801,0.628531,0.870153,0.548130,0.711407,0.811067,0.571223,0.681195,0.581311,0.580767,0.825611,0.921662,0.993871,0.568133,0.650131,0.760040,0.765478,0.992494,0.666914,0.727857,0.502210,0.795089,0.623052,0.785584,0.752883,0.769498,0.735951,0.804514,0.911004,0.657542,0.669385,0.510175,0.789888,0.938058,0.612929,0.509000,0.732080,0.662704,0.910886,0.674498,0.708691,0.632232,0.577553,0.545058,0.666769,0.918583,0.859745,0.565090,0.684326,0.895735,0.522196,0.926947,0.551563,0.520974,0.593304,0.898536,0.913326,0.531462,0.623630,0.665908,0.785659,0.829590,0.854398,0.501797,0.871296,0.795712,0.765920,0.699387,0.515206,0.517268,0.791786,0.702784,0.593872,0.758535,0.808368,0.641204,0.760509,0.871118,0.714999,0.635517,0.550215,0.649899,0.557074,0.619339,0.776553,0.515062,0.764698,0.960669,0.748662,0.615079,0.651171,0.641747,0.797364,0.675584,0.725174,0.786208,0.629620,0.535107,0.756352,0.510523,0.568997,0.938779,0.632383,0.668566,0.567277,0.819484,0.608986,0.821905,0.650501,0.902594,0.746894,0.711189,0.514342,0.719263,0.611829,0.715131,0.859032,0.664992,0.536657,0.755834,0.564745,0.869045,0.853020,0.536533,0.586701,0.615671,0.501840,0.751591,0.776629,0.595428,0.943790,0.902844,0.540428,0.912842,0.968071,0.730808,0.654327,0.540234,0.633889,0.715539,0.833495,0.522100,0.561998,0.624502,0.520068,0.720129,0.915015,0.904579,0.579001,0.528875,0.714127,0.652091,0.641776,0.729599,0.635911,0.804327,0.724217,0.601907,0.608066,0.525322,0.587119,0.889764,0.818936,0.605692,0.641776,0.936304,0.550566,0.600617,0.946182,0.647402,0.931129,0.764881,0.763703,0.593823,0.564519,0.957267,0.981996,0.973463,0.936078,0.699795,0.528773,0.516807,0.546272,0.849872,0.690988,0.656954,0.524999,0.985957,0.811406,0.692503,0.599169,0.766363,0.515193,0.667900,0.760469,0.663844,0.981667,0.800302,0.878114,0.536400,0.755999,0.857419,0.658547,0.576982,0.941952,0.624557,0.767137,0.639016,0.866618,0.557453,0.757524,0.864744,0.666309,0.532537,0.539676,0.947260,0.519314,0.650986,0.770319,0.565442,0.696079,0.632966,0.688109,0.602199,0.792529,0.504212,0.694053,0.615738,0.588302,0.783495,0.550958,0.859546,0.855298,0.529347,0.564488,0.584834,0.772107,0.959140,0.655965,0.965941,0.737582,0.600321,0.735616,0.565969,0.723188,0.832599,0.667507,0.651822,0.984003,0.720047,0.870751,0.665982,0.504031,0.605313,0.945942,0.763104,0.621903,0.926074,0.888720,0.654574,0.999339,0.630217,0.698788,0.878563,0.870421,0.721016,0.571890,0.948285,0.647124,0.619955,0.528684,0.682975,0.718301,0.550731,0.895814,0.952190,0.545052,0.507841,0.932249,0.560182,0.549986,0.932034,0.544244,0.967039,0.868732,0.839145,0.817885,0.686725,0.682862,0.577867,0.658024,0.516182,0.903828,0.655299,0.641996,0.594462,0.594877,0.698916,0.942262,0.869953,0.609607,0.573654,0.985663,0.518627,0.978718,0.657328,0.845781,0.544887,0.734415,0.600328,0.540311,0.780646,0.587738,0.980441,0.810301,0.609625,0.596303,0.989272,0.962536,0.558282,0.516309,0.508088,0.889226,0.928021,0.649952,0.803903,0.702039,0.902048,0.595446,0.695954,0.860437,0.863448,0.607194,0.853123,0.657727,0.924902,0.634843,0.548617,0.568515,0.609303,0.721240,0.971633,0.993904,0.773726,0.665779,0.553730,0.616041,0.516878,0.958973,0.762876,0.869287,0.791441,0.649799,0.645999,0.547002,0.615530,0.605573,0.665184,0.635866,0.624879,0.790527,0.503616,0.961312,0.663872,0.626801,0.983903,0.713105,0.621276,0.626089,0.680686,0.653179,0.652025,0.794176,0.659688,0.540258,0.748209,0.582206,0.577685,0.526362,0.882404,0.600049,0.801430,0.551166,0.954394,0.833731,0.560168,0.548377,0.808806,0.753947,0.921238,0.917334,0.672707,0.508381,0.849420,0.959892,0.728940,0.874080,0.637874,0.891332,0.602171,0.550093,0.519746,0.643648,0.588689,0.914807,0.917404,0.854374,0.645064,0.837733,0.553807,0.902228,0.894802,0.701639,0.939774,0.868500,0.634662,0.657584,0.983813,0.673736,0.789699,0.518472,0.544929,0.674143,0.717903,0.809452,0.920297,0.826708,0.908863,0.562186,0.766856,0.575229,0.617635,0.779921,0.828371,0.753552,0.641890,0.796994,0.823658,0.828414,0.579830,0.775656,0.729426,0.535407,0.791976,0.516233,0.904184,0.605883,0.813571,0.943240,0.554818,0.723213,0.650869,0.532643,0.650730,0.748575,0.516366,0.557105,0.593149,0.865363,0.985448,0.774149,0.798640,0.676588,0.800383,0.644640,0.902285,0.570597,0.969290,0.518365,0.603165,0.786621,0.734583,0.555401,0.551623,0.791865,0.642134,0.552614,0.607001,0.570282,0.800083,0.760170,0.673605,0.906482,0.651942,0.691551,0.555764,0.963441,0.621630,0.533522,0.642247,0.858994,0.684809,0.870621,0.511242,0.736752,0.503545,0.623974,0.598960,0.960857,0.723930,0.629354,0.809033,0.500078,0.604788,0.817904,0.937346,0.605707,0.804542,0.947540,0.845229,0.542570,0.875524,0.780795,0.611113,0.775137,0.553128,0.848594,0.864202,0.792867,0.882604,0.631317,0.550876,0.804827,0.605786,0.780093,0.672350,0.666432,0.934835,0.866897,0.552113,0.780841,0.704408,0.732635,0.716309,0.674028,0.814765,0.631765,0.878414,0.552317,0.558649,0.640908,0.916345,0.510866,0.976236,0.551779,0.755690,0.680374,0.622813,0.655606,0.888260,0.724375,0.735578,0.979672,0.754053,0.708161,0.780277,0.581468,0.541194,0.504733,0.665397,0.671006,0.516759,0.568422,0.616647,0.765999,0.771711,0.737763,0.501525,0.607867,0.881029,0.614037,0.686758,0.679278,0.825088,0.628751,0.690268,0.940064,0.639156,0.706245,0.917143,0.504508,0.617565,0.905907,0.542634,0.905771,0.546108,0.994451,0.595399,0.537740,0.810585,0.868588,0.731099,0.774775,0.797701,0.979998,0.533466,0.738828,0.603305,0.776419,0.511174,0.623176,0.690781,0.665513,0.699637,0.789015,0.804778,0.732750,0.583750,0.723917,0.522261,0.552224,0.606214,0.608204,0.850226,0.576414,0.633114,0.802536,0.912289,0.771379,0.846355,0.570965,0.557160,0.854354,0.610724,0.516487,0.539132,0.571288,0.618722,0.533210,0.531257,0.605507,0.628795,0.542631,0.561846,0.567411,0.588203,0.975998,0.586013,0.532296,0.723416,0.526501,0.612306,0.696479,0.802430,0.847932,0.639947,0.879900,0.835835,0.507819,0.860716,0.613855,0.591771,0.623146,0.877620,0.601152,0.621200,0.759556,0.982486,0.604627,0.788571,0.702495,0.783069,0.871728,0.639740,0.649028,0.512649,0.804092,0.547013,0.512845,0.699547,0.671049,0.555597,0.779037,0.702725,0.819504,0.614381,0.889604,0.928269,0.567520,0.519908,0.518823,0.554386,0.519680,0.576690,0.779173,0.505903,0.610199,0.801047,0.656242,0.789698,0.707789,0.686664,0.594799,0.684753,0.978698,0.666308,0.607655,0.889503,0.520908,0.576064,0.524935,0.573772,0.542608,0.502475,0.593012,0.637191,0.939429,0.555133,0.597551,0.509724,0.623763,0.551153,0.784679,0.646473,0.969498,0.818864,0.625683,0.528533,0.882715,0.569354,0.705092,0.591373,0.579968,0.888122,0.731172,0.828099,0.839153,0.969284,0.783266,0.814508,0.629714,0.584760,0.888758,0.639567,0.840427,0.646031,0.733677,0.745860,0.531372,0.829784,0.585164,0.710106,0.745830,0.898046,0.534777,0.790546,0.890732,0.624432,0.996685,0.762519,0.864850,0.588854,0.500998,0.533802,0.558485,0.598444,0.542771,0.797464,0.585916,0.713662,0.742739,0.854457,0.693545,0.969431,0.569491,0.882783,0.723884,0.539731,0.510679,0.591142,0.501878,0.515843,0.532026,0.616343,0.866702,0.500257,0.617327,0.594851,0.532898,0.514348,0.664183,0.504428,0.554365,0.674180,0.541853,0.521058,0.892655,0.970330,0.619399,0.850653,0.519146,0.617385,0.957755,0.944465,0.684441,0.696229,0.981453,0.877785,0.685579,0.522402,0.891874,0.524164,0.602347,0.586811,0.847334,0.866735,0.628667,0.751312,0.838012,0.537656,0.543491,0.682331,0.563352,0.817142,0.637979,0.503711,0.564783,0.759471,0.603013,0.892729,0.645872,0.810447,0.511406,0.936323,0.506199,0.638533,0.939591,0.620315,0.680840,0.588935,0.719288,0.699553,0.732170,0.513855,0.662310,0.599346,0.514553,0.567806,0.891727,0.718688,0.680925,0.541730,0.671590,0.992706,0.942928,0.581548,0.693103,0.856909,0.640860,0.639146,0.771618,0.658137,0.638470,0.616817,0.866968,0.515977,0.709094,0.580480,0.638952,0.767724,0.504701,0.998998,0.953274,0.569695,0.569656,0.530236,0.568966,0.590737,0.942854,0.895837,0.678977,0.976893,0.948322,0.880125,0.770156,0.700792,0.540075,0.564723,0.573735,0.587795,0.619676,0.651409,0.755073,0.857425,0.501504,0.545320,0.804660,0.587892,0.698947,0.565932,0.862790,0.577947,0.938600,0.797820,0.638892,0.523393,0.688124,0.612231,0.557032,0.620049,0.527177,0.777829,0.954536,0.670008,0.627040,0.644228,0.541413,0.606905,0.717921,0.518271,0.651691,0.676366,0.591926,0.742633,0.520905,0.897670,0.606817,0.865639,0.727015,0.628867,0.501137,0.520589,0.776675,0.670331,0.716321,0.516610,0.615166,0.543147,0.863641,0.733184,0.697139,0.943392,0.721443,0.762120,0.586753,0.585898,0.788748,0.639614,0.639783,0.989365,0.590377,0.676407,0.528829,0.568128,0.913996,0.572848,0.823283,0.630134,0.529808,0.519834,0.697148,0.634070,0.590701,0.934253,0.696525,0.676390,0.634799,0.692410,0.602258,0.631499,0.574641,0.869481,0.669958,0.915017,0.967705,0.731670,0.684174,0.677637,0.575669,0.547785,0.606348,0.539515,0.707415,0.658385,0.819129,0.864357,0.521307,0.584349,0.838134,0.520876,0.917848,0.801323,0.655937,0.521083,0.521769,0.554441,0.851001,0.671305,0.809349,0.642503,0.918139,0.601471,0.881930,0.923748,0.654711,0.534826,0.688219,0.911033,0.720207,0.876929,0.506072,0.666516,0.590065,0.743387,0.569176,0.528176,0.741421,0.999742,0.534010,0.563993,0.695335,0.664863,0.623903,0.843356,0.807786,0.857161,0.583246,0.690469,0.821091,0.562838,0.670948,0.722466,0.703218,0.538568,0.732509,0.913498,0.757403,0.764714,0.626931,0.665829,0.782957,0.594283,0.732952,0.601813,0.752481,0.587253,0.640161,0.503519,0.938822,0.969756,0.982336,0.691684,0.690820,0.752152,0.620124,0.834845,0.779699,0.706220,0.610146,0.721825,0.619657,0.719006,0.739565,0.828105,0.660481,0.780449,0.660388,0.953713,0.789402,0.878774,0.588465,0.609775,0.502862,0.851465,0.512947,0.685305,0.536842,0.609040,0.749099,0.692646,0.704834,0.668169,0.760696,0.971505,0.522415,0.609458,0.552706,0.626611,0.738337,0.772330,0.614310,0.760602,0.636174,0.603983,0.707574,0.841771,0.563620,0.579567,0.523293,0.926037,0.526431,0.936257,0.930124,0.521240,0.739526,0.867966,0.802041,0.764554,0.749321,0.946185,0.669029,0.957807,0.694094,0.745234,0.639038,0.724036,0.856168,0.645024,0.725151,0.805231,0.657244,0.568374,0.835824,0.776732,0.576889,0.638250,0.611480,0.622069,0.540334,0.656033,0.842413,0.989521,0.908499,0.827570,0.815993,0.627352,0.867931,0.670235,0.885003,0.558793,0.539018,0.648127,0.716896,0.993499,0.636511,0.707319,0.754219,0.731337,0.536323,0.687273,0.513506,0.617324,0.563103,0.709447,0.575528,0.984867,0.840814,0.621863,0.886190,0.638264,0.560792,0.534778,0.879488,0.847715,0.727818,0.519791,0.730729,0.515753,0.564220,0.929188,0.888959,0.716594,0.536988,0.558775,0.990774,0.788471,0.791273,0.547808,0.502678,0.576466,0.677542,0.738148,0.782582,0.658935,0.762053,0.786436,0.768329,0.716070,0.946861,0.724309,0.682213,0.867457,0.618272,0.609223,0.571151,0.516833,0.812668,0.749751,0.844340,0.653843,0.536439,0.538031,0.598828,0.512543,0.641391,0.921345,0.522847,0.768477,0.788013,0.638088,0.995549,0.828762,0.509612,0.524533,0.678808,0.760921,0.553802,0.500729,0.543894,0.789276,0.553008,0.644950,0.541206,0.670758,0.511184,0.744014,0.886329,0.779143,0.811879,0.812563,0.566388,0.622470,0.544484,0.832394,0.857524,0.994963,0.827661,0.969492,0.824363,0.958014,0.787605,0.525768,0.815802,0.822547,0.641224,0.599938,0.516621,0.675357,0.535738,0.629472,0.962085,0.857687,0.958982,0.559133,0.581485,0.545614,0.635164,0.893362,0.844349,0.767353,0.505890,0.881930,0.662607,0.662143,0.793854,0.626351,0.588451,0.615630,0.995492,0.538004,0.579736,0.561851,0.644303,0.832506,0.582861,0.540280,0.554250,0.801689,0.729305,0.511802,0.754484,0.543986,0.669074,0.534759,0.875770,0.904269,0.580231,0.563656,0.567954,0.889244,0.862936,0.574319,0.502358,0.925125,0.940887,0.771771,0.871546,0.601309,0.811133,0.735902,0.642903,0.507493,0.670363,0.657382,0.943962,0.678442,0.534061,0.565232,0.887769,0.988268,0.667613,0.523229,0.541342,0.730766,0.674820,0.917368,0.672850,0.605966,0.892535,0.515285,0.703478,0.840852,0.661205,0.724013,0.606722,0.789054,0.762818,0.771584,0.598225,0.577503,0.822883,0.562272,0.634291,0.632571,0.827648,0.664395,0.591649,0.925182,0.765310,0.664340,0.518365,0.757114,0.547899,0.647459,0.603758,0.533081,0.943770,0.997845,0.587326,0.506439,0.584503,0.502223,0.768609,0.818142,0.651037,0.503509,0.810501,0.685246,0.627706,0.782105,0.840199,0.685045,0.727170,0.765686,0.657470,0.881331,0.524901,0.774820,0.580460,0.606229,0.794846,0.750819,0.589411,0.877326,0.566044,0.510787,0.626315,0.695142,0.647483,0.777304,0.780828,0.529689,0.515945,0.856170,0.527727,0.529276,0.892104,0.716670,0.815420,0.754809,0.647914,0.909869,0.618805,0.779869,0.512237,0.532592,0.502295,0.784238,0.553519,0.621971,0.833299,0.712889,0.544907,0.973686,0.597052,0.593966,0.734212,0.776596,0.501804,0.893826,0.753455,0.875307,0.689573,0.767166,0.544164,0.511725,0.769811,0.637624,0.903441,0.591973,0.921716,0.986361,0.727517,0.903219,0.959830,0.722373,0.723417,0.539236,0.592395,0.883365,0.585578,0.585928,0.565729,0.717520,0.758197,0.580215,0.853994,0.601345,0.569804,0.540540,0.525656,0.620870,0.515152,0.815219,0.810378,0.606655,0.889246,0.696877,0.796352,0.855791,0.713675,0.965375,0.772823,0.848556,0.897981,0.653827,0.551646,0.540376,0.826628,0.544947,0.561446,0.606105,0.565049,0.915125,0.618801,0.803576,0.872826,0.615727,0.597266,0.565573,0.518627,0.585250,0.721928,0.646476,0.632685,0.697568,0.784395,0.774645,0.740124,0.652414,0.506903,0.641720,0.690837,0.880318,0.777732,0.842313,0.937430,0.572893,0.725059,0.875439,0.711247,0.685269,0.817063,0.932747,0.510726,0.552898,0.567451,0.662855,0.698379,0.577977,0.824705,0.614652,0.738563,0.684254,0.582531,0.537952,0.832830,0.787458,0.759270,0.570230,0.695235,0.932501,0.859377,0.764637,0.971327,0.590467,0.796989,0.776985,0.746534,0.786840,0.646835,0.562082,0.821110,0.621120,0.572494,0.875777,0.572513,0.598850,0.703752,0.779604,0.645494,0.597531,0.661750,0.988388,0.993624,0.547038,0.597986,0.507446,0.761787,0.697308,0.521124,0.643064,0.991724,0.712641,0.871858,0.580707,0.966252,0.801886,0.858243,0.755756,0.741638,0.822340,0.851899,0.780107,0.756477,0.732020,0.637721,0.851497,0.598520,0.619820,0.597461,0.580617,0.793220,0.597645,0.535241,0.543450,0.828455,0.653079,0.678201,0.563335,0.939876,0.539391,0.792513,0.738701,0.558328,0.669962,0.634541,0.731088,0.720971,0.726253,0.546048,0.908408,0.768546,0.519808,0.607874,0.703062,0.999393,0.714308,0.662248,0.702067,0.591528,0.922779,0.902007,0.587665,0.589567,0.800295,0.706854,0.695817,0.683575,0.599839,0.920352,0.806448,0.760821,0.789504,0.570751,0.566923,0.818745,0.820293,0.709489,0.923375,0.527858,0.573384,0.568884,0.677887,0.911464,0.786283,0.645511,0.652289,0.720575,0.668314,0.670558,0.786525,0.618261,0.927261,0.727224,0.692701,0.520232,0.655429,0.583450,0.583999,0.766636,0.543624,0.786915,0.596232,0.530979,0.804246,0.561254,0.594396,0.507131,0.697355,0.882015,0.609479,0.902602,0.712022,0.924529,0.515180,0.517831,0.913552,0.662496,0.891678,0.598055,0.651149,0.643518,0.843582,0.661118,0.849015,0.795637,0.590754,0.958566,0.772772,0.608318,0.571483,0.553000,0.908270,0.557997,0.618571,0.723501,0.609873,0.505947,0.674606,0.931634,0.614764,0.588833,0.706933,0.992153,0.828358,0.737867,0.533294,0.502100,0.729905,0.620398,0.500611,0.869340,0.570926,0.627068,0.563345,0.628220,0.792458,0.550787,0.620289,0.938336,0.507749,0.576052,0.554932,0.741787,0.756631,0.552176,0.585032,0.751755,0.897932,0.746803,0.505157,0.765509,0.529631,0.805602,0.751635,0.885905,0.937622,0.535120,0.534463,0.661203,0.576280,0.638362,0.569424,0.728012,0.617297,0.751120,0.897328,0.565544,0.975359,0.686668,0.636394,0.544817,0.738943,0.540665,0.756275,0.632596,0.560147,0.585602,0.591564,0.525051,0.870340,0.603710,0.798930,0.766984,0.704245,0.710740,0.658995,0.587835,0.825762,0.509917,0.566947,0.858853,0.541275,0.695959,0.983631,0.529252,0.823470,0.539993,0.636322,0.772630,0.546597,0.660762,0.676016,0.517673,0.503857,0.590178,0.891326,0.526769,0.582575,0.751640,0.907888,0.719029,0.555676,0.939209,0.515138,0.511702,0.553545,0.973970,0.500223,0.661455,0.968547,0.635119,0.576072,0.555876,0.995189,0.515191,0.968611,0.624347,0.674872,0.511713,0.983559,0.773890,0.503759,0.671896,0.595899,0.533628,0.569853,0.508360,0.697805,0.929408,0.737708,0.779203,0.502322,0.705372,0.714791,0.916582,0.678964,0.932696,0.663103,0.534848,0.617639,0.645014,0.683090,0.740713,0.607176,0.559943,0.848513,0.683442,0.516349,0.626602,0.509354,0.577791,0.766359,0.930591,0.507797,0.671225,0.652336,0.880783,0.694429,0.535471,0.573547,0.793541,0.664517,0.923004,0.555494,0.532026,0.994133,0.532183,0.938231,0.576622,0.728340,0.589301,0.865785,0.775573,0.886814,0.731387,0.624183,0.512854,0.952323,0.523139,0.608138,0.907218,0.797231,0.574954,0.530832,0.652410,0.775186,0.713257,0.622297,0.873434,0.540780,0.905075,0.519505,0.574810,0.830430,0.913962,0.622901,0.654810,0.733205,0.503677,0.754547,0.709729,0.714796,0.528352,0.594695,0.794508,0.503898,0.534615,0.705368,0.522389,0.665081,0.500611,0.571356,0.795821,0.829100,0.993584,0.526126,0.626690,0.742428,0.501411,0.513629,0.752283,0.633791,0.767146,0.932600,0.627509,0.694571,0.637922,0.590524,0.628229,0.649653,0.547893,0.783927,0.515242,0.888320,0.538533,0.804580,0.560140,0.508523,0.755534,0.729674,0.767555,0.816122,0.649096,0.658054,0.517385,0.781803,0.972281,0.591993,0.551433,0.759451,0.728662,0.952293,0.726923,0.627405,0.578799,0.705158,0.553156,0.772087,0.650861,0.623990,0.801805,0.883302,0.783407,0.841254,0.838409,0.596990,0.859493,0.700748,0.556549,0.510074,0.502586,0.783772,0.589662,0.561993,0.836875,0.682693,0.935868,0.837655,0.542419,0.824827,0.503852,0.659998,0.622644,0.959327,0.742983,0.705761,0.589520,0.562856,0.840175,0.743668,0.712422,0.637484,0.540962,0.502928,0.587652,0.827219,0.607755,0.519737,0.963066,0.633362,0.874192,0.598542,0.502734,0.641488,0.597998,0.851292,0.624255,0.546949,0.825492,0.507186,0.917893,0.963579,0.614604,0.974001,0.546134,0.964073,0.617933,0.764239,0.935902,0.973531,0.699906,0.540684,0.689041,0.781205,0.605351,0.954665,0.959364,0.553936,0.909881,0.690686,0.696297,0.589070,0.677581,0.923515,0.790540,0.995352,0.895855,0.778011,0.663234,0.771010,0.829377,0.686008,0.542097,0.744853,0.796961,0.817318,0.882577,0.594131,0.971372,0.665237,0.681986,0.571211,0.919787,0.655899,0.502324,0.939077,0.584171,0.751150,0.792210,0.505831,0.798030,0.796714,0.747988,0.890531,0.551762,0.897027,0.666815,0.535122,0.787402,0.548600,0.548243,0.506757,0.554532,0.908448,0.786370,0.518318,0.960867,0.758849,0.774996,0.544638,0.597735,0.674612,0.628019,0.956154,0.597557,0.530747,0.979404,0.530967,0.989363,0.756651,0.785792,0.611454,0.669247,0.604367,0.601205,0.528232,0.788211,0.744072,0.561609,0.627843,0.669768,0.787397,0.881523,0.671449,0.503456,0.679843,0.856129,0.759487,0.681429,0.871907,0.729482,0.549145,0.980576,0.952727,0.779268,0.964027,0.542858,0.940629,0.577939,0.635524,0.599865,0.929407,0.874325,0.619046,0.922699,0.651827,0.997493,0.625436,0.990224,0.503823,0.642875,0.627196,0.600806,0.639144,0.500184,0.952997,0.834434,0.933351,0.791225,0.941530,0.767409,0.535524,0.958326,0.592402,0.501630,0.793549,0.807989,0.833348,0.543880,0.506840,0.527082,0.988031,0.534074,0.913076,0.697115,0.619579,0.536527,0.588640,0.600667,0.507479,0.690261,0.679250,0.847794,0.576860,0.713905,0.631142,0.833654,0.859891,0.687601,0.507691,0.645776,0.951192,0.733897,0.690119,0.867796,0.799373,0.845942,0.860322,0.780153,0.547664,0.580872,0.619225,0.933268,0.862692,0.658465,0.552451,0.584633,0.658966,0.544431,0.779011,0.873852,0.596795,0.732193,0.725492,0.916806,0.813863,0.502426,0.661904,0.759649,0.855987,0.858921,0.549814,0.577826,0.506930,0.737718,0.588728,0.982416,0.712702,0.741878,0.895619,0.576307,0.534951,0.955715,0.836129,0.604166,0.633692,0.706198,0.516338,0.516362,0.534557,0.539683,0.524894,0.509140,0.505453,0.603995,0.969815,0.883605,0.595262,0.893174,0.563034,0.791787,0.627084,0.703124,0.785312,0.817643,0.566780,0.722539,0.500335,0.676251,0.695664,0.622260,0.551654,0.562056,0.940328,0.815924,0.854598,0.628825,0.620564,0.777471,0.501088,0.538574,0.622448,0.939622,0.669404,0.893301,0.853623,0.647166,0.632616,0.678989,0.523930,0.559600,0.926365,0.681508,0.841236,0.839527,0.578037,0.641835,0.706658,0.589627,0.896723,0.888583,0.653890,0.616803,0.834269,0.540594,0.543803,0.921715,0.791219,0.883363,0.790838,0.591047,0.638614,0.878967,0.884804,0.802288,0.719900,0.503852,0.513814,0.711884,0.518423,0.760812,0.855222,0.866019,0.660655,0.581075,0.996019,0.696142,0.859045,0.510702,0.540003,0.617020,0.591980,0.507138,0.570685,0.596714,0.506882,0.789789,0.532693,0.515910,0.965182,0.517110,0.562836,0.609198,0.683833,0.949785,0.963510,0.925688,0.689858,0.506648,0.656861,0.800585,0.768520,0.851666,0.637425,0.631371,0.808933,0.842262,0.742868,0.535464,0.672674,0.658243,0.790837,0.610519,0.502790,0.808469,0.708110,0.667049,0.943232,0.602866,0.639267,0.744143,0.760959,0.805761,0.876768,0.977895,0.722271,0.679515,0.912705,0.864136,0.939834,0.896360,0.808141,0.965417,0.651914,0.907783,0.662297,0.657329,0.533285,0.790004,0.869490,0.903352,0.829295,0.753996,0.835043,0.512115,0.593614,0.747249,0.704797,0.824241,0.568975,0.832333,0.510282,0.686658,0.736159,0.897809,0.502456,0.912749,0.926008,0.503773,0.686410,0.563576,0.782423,0.854025,0.693670,0.852686,0.632731,0.536195,0.556501,0.871471,0.781203,0.767486,0.551517,0.846874,0.698688,0.628002,0.607412,0.581303,0.522080,0.943876,0.777477,0.565933,0.828284,0.897371,0.566531,0.963924,0.597503,0.864368,0.679732,0.833831,0.808143,0.869474,0.958057,0.867784,0.868805,0.646469,0.540046,0.581174,0.765379,0.913042,0.529135,0.678193,0.991950,0.719022,0.798795,0.643075,0.750473,0.980772,0.826230,0.902500,0.633590,0.723221,0.794045,0.739469,0.855001,0.602749,0.685655,0.695989,0.644760,0.693983,0.913018,0.600478,0.690261,0.853159,0.639985,0.690197,0.724387,0.744951,0.546760,0.702922,0.833634,0.864619,0.904624,0.561215,0.758394,0.585101,0.854144,0.998468,0.512420,0.821347,0.714293,0.986188,0.872457,0.692768,0.645172,0.654584,0.508287,0.658410,0.730538,0.749697,0.709037,0.595819,0.787895,0.692002,0.502153,0.771671,0.500008,0.775639,0.545162,0.575835,0.724359,0.798331,0.729337,0.750845,0.637176,0.948918,0.706678,0.809973,0.656738,0.511493,0.796280,0.840250,0.617799,0.692904,0.536947,0.584605,0.805077,0.508679,0.964467,0.801342,0.767901,0.850266,0.668952,0.883884,0.911999,0.655823,0.513921,0.656874,0.684197,0.527933,0.893218,0.685402,0.534399,0.859459,0.719326,0.710484,0.568331,0.689354,0.941311,0.713542,0.689350,0.809877,0.967892,0.757818,0.833630,0.908949,0.898704,0.993026,0.602173,0.743304,0.526184,0.892878,0.637513,0.706140,0.515341,0.529887,0.656267,0.641218,0.929181,0.700839,0.789499,0.566304,0.648360,0.521297,0.835488,0.636787,0.513319,0.698763,0.595889,0.518541,0.826581,0.725874,0.547710,0.836645,0.735265,0.612974,0.947230,0.569567,0.527715,0.769098,0.647437,0.756509,0.686251,0.572240,0.610762,0.949104,0.700269,0.510219,0.743128,0.703949,0.697066,0.885125,0.662278,0.681030,0.926939,0.848805,0.732732,0.859351,0.997715,0.729740,0.956977,0.513570,0.589357,0.613069,0.502950,0.506379,0.657586,0.590355,0.588871,0.595309,0.838555,0.518917,0.539379,0.879064,0.788891,0.808189,0.823983,0.527838,0.821184,0.751402,0.505561,0.992717,0.610236,0.612467,0.569704,0.802143,0.561141,0.896989,0.581179,0.812464,0.557743,0.551540,0.776086,0.834836,0.839310,0.576352,0.636156,0.657133,0.967660,0.754699,0.623323,0.528822,0.541549,0.515167,0.552258,0.880061,0.548082,0.840635,0.569687,0.694700,0.505381,0.705196,0.977727,0.768484,0.722216,0.644591,0.824136,0.682104,0.583619,0.852077,0.688143,0.851237,0.841375,0.576410,0.574929,0.616075,0.527880,0.556387,0.527587,0.526956,0.616671,0.633301,0.712438,0.620162,0.959444,0.764069,0.745935,0.535692,0.797692,0.579410,0.554946,0.565443,0.862099,0.788745,0.563639,0.802392,0.591672,0.894276,0.513266,0.950557,0.930410,0.535179,0.723439,0.506019,0.591832,0.672370,0.910382,0.597470,0.572763,0.933707,0.589804,0.713639,0.637469,0.536782,0.747065,0.544276,0.707809,0.678465,0.675809,0.659146,0.743873,0.628506,0.720859,0.816015,0.670609,0.757026,0.924263,0.750880,0.537563,0.837139,0.782453,0.849533,0.990369,0.683091,0.990191,0.921779,0.511800,0.523276,0.596521,0.556798,0.540773,0.627526,0.894750,0.628756,0.647524,0.659687,0.634260,0.808727,0.913237,0.613229,0.726952,0.952861,0.571445,0.807645,0.756727,0.526592,0.593939,0.938883,0.718152,0.643527,0.974816,0.838533,0.785165,0.689225,0.708054,0.693244,0.664878,0.533340,0.899246,0.660917,0.763640,0.806501,0.901184,0.615017,0.680421,0.562380,0.736286,0.686791,0.612139,0.800931,0.602885,0.616950,0.560276,0.827257,0.956027,0.840426,0.909409,0.550542,0.512503,0.661911,0.972547,0.800288,0.505973,0.883160,0.633216,0.975144,0.556159,0.615358,0.522940,0.974427,0.540395,0.569083,0.770299,0.847837,0.701461,0.520330,0.553529,0.685833,0.634739,0.807138,0.817080,0.733864,0.838532,0.587098,0.947332,0.580400,0.614584,0.513402,0.976353,0.706995,0.864282,0.808562,0.538044,0.735775,0.796183,0.862213,0.648258,0.572262,0.623156,0.649686,0.511958,0.738618,0.876444,0.930493,0.506826,0.636883,0.752833,0.594373,0.570586,0.852701,0.534729,0.626092,0.820219,0.824446,0.606956,0.523183,0.737930,0.606475,0.662932,0.984246,0.622039,0.581541,0.563715,0.901891,0.756654,0.520621,0.970171,0.907504,0.706419,0.544650,0.629171,0.772478,0.619187,0.990964,0.943923,0.518152,0.729208,0.578212,0.931853,0.766806,0.664330,0.558601,0.895521,0.780558,0.809526,0.619648,0.544746,0.506071,0.838962,0.914785,0.665683,0.555489,0.510696,0.878287,0.637488,0.606121,0.771718,0.555821,0.530079,0.531642,0.596883,0.921528,0.589125,0.679800,0.514333,0.817552,0.824834,0.705950,0.689554,0.552768,0.846737,0.502669,0.500656,0.521266,0.925423,0.575899,0.536258,0.706809,0.533099,0.830309,0.550856,0.607381,0.857987,0.717355,0.976104,0.819148,0.684019,0.864771,0.563058,0.606644,0.842334,0.542718,0.580195,0.569209,0.904474,0.917647,0.515461,0.516476,0.592842,0.604769,0.983678,0.859878,0.835728,0.584204,0.743623,0.624779,0.839801,0.893272,0.989018,0.679716,0.710384,0.810389,0.826217,0.809558,0.921460,0.901526,0.940381,0.568036,0.583955,0.794081,0.670610,0.651057,0.723840,0.738520,0.514506,0.930793,0.748299,0.554748,0.821492,0.526072,0.912082,0.535939,0.752356,0.748627,0.801747,0.596492,0.564294,0.982847,0.716179,0.798749,0.616145,0.694614,0.669306,0.550434,0.511872,0.651464,0.844063,0.518417,0.573808,0.924849,0.570221,0.557670,0.874342,0.666064,0.508140,0.708659,0.600315,0.518697,0.571125,0.615867,0.735882,0.923469,0.823049,0.937571,0.885584,0.649565,0.651782,0.660409,0.823560,0.515200,0.783863,0.741507,0.512090,0.772936,0.503154,0.560891,0.685037,0.627490,0.567622,0.823163,0.827815,0.958175,0.896332,0.542243,0.530353,0.858028,0.602780,0.835645,0.551349,0.766682,0.645251,0.903314,0.797792,0.867819,0.570747,0.547079,0.934726,0.651109,0.666712,0.912233,0.566558,0.910214,0.926721,0.691546,0.907075,0.909205,0.960894,0.697938,0.714982,0.693484,0.845468,0.513969,0.532423,0.580789,0.608505,0.886181,0.962399,0.539615,0.845191,0.535890,0.597576,0.600487,0.628520,0.880167,0.623394,0.663713,0.506260,0.963748,0.704693,0.634235,0.505481,0.902168,0.713463,0.958533,0.685707,0.668019,0.646978,0.865451,0.890660,0.765068,0.840308,0.627692,0.542609,0.815253,0.689591,0.531548,0.521685,0.647012,0.555164,0.864437,0.707707,0.565673,0.662897,0.865762,0.967061,0.505976,0.734168,0.639814,0.640112,0.715740,0.827321,0.980196,0.848475,0.519598,0.758401,0.569039,0.929210,0.731610,0.834324,0.646282,0.533371,0.540926,0.774865,0.526046,0.715828,0.598695,0.592163,0.670203,0.690322,0.644293,0.510073,0.813004,0.572436,0.985116,0.654802,0.599950,0.631468,0.503219,0.741092,0.912336,0.899448,0.593703,0.743210,0.725113,0.792688,0.503050,0.814281,0.520060,0.905712,0.518647,0.644410,0.519839,0.564632,0.709261,0.587655,0.833115,0.627280,0.545186,0.919989,0.969215,0.891897,0.534855,0.596102,0.668190,0.740586,0.918970,0.988638,0.908692,0.812439,0.702248,0.792739,0.941984,0.810183,0.555533,0.533473,0.520841,0.811709,0.765001,0.781193,0.757392,0.671157,0.507437,0.569010,0.771108,0.511734,0.982475,0.858525,0.743703,0.769646,0.553260,0.775082,0.624268,0.511572,0.670231,0.980547,0.678765,0.670432,0.562497,0.623430,0.523288,0.696808,0.757189,0.871609,0.904663,0.549852,0.699082,0.824284,0.708473,0.905499,0.517981,0.933663,0.971767,0.542188,0.588541,0.609045,0.525602,0.584765,0.668139,0.765614,0.589272,0.536603,0.981875,0.784939,0.967739,0.640188,0.830084,0.837290,0.889788,0.911091,0.989939,0.714822,0.540590,0.537921,0.889403,0.924561,0.811688,0.638627,0.762925,0.706249,0.734386,0.848806,0.767948,0.623205,0.793175,0.683909,0.837936,0.628551,0.828440,0.626037,0.615466,0.961476,0.630619,0.737916,0.963869,0.939557,0.739016,0.689918,0.605085,0.518854,0.645985,0.823126,0.622577,0.599833,0.828437,0.518701,0.994929,0.613613,0.504307,0.721466,0.652096,0.685188,0.924202,0.500321,0.697936,0.558540,0.618078,0.519193,0.743478,0.663355,0.549194,0.528479,0.685727,0.530505,0.839001,0.966670,0.697591,0.937198,0.718544,0.565150,0.949297,0.780917,0.822537,0.663463,0.662363,0.664933,0.598779,0.831252,0.925915,0.863358,0.742891,0.951714,0.692379,0.809739,0.561660,0.869773,0.815503,0.567338,0.895912,0.672035,0.560009,0.784244,0.913684,0.740429,0.672140,0.704198,0.805914,0.546572,0.514265,0.634618,0.664247,0.618043,0.712108,0.752905,0.516089,0.840874,0.801775,0.991030,0.508495,0.552129,0.922524,0.898502,0.512576,0.757396,0.825361,0.561160,0.780623,0.599975,0.853879,0.555171,0.763875,0.544141,0.573034,0.652033,0.850903,0.620830,0.513301,0.520215,0.639307,0.712727,0.889229,0.978233,0.763288,0.714353,0.587338,0.723149,0.697072,0.579305,0.503303,0.555481,0.736140,0.638001,0.945382,0.979087,0.851782,0.525822,0.534330,0.614681,0.902906,0.876689,0.834587,0.986227,0.654110,0.826434,0.954720,0.543649,0.942767,0.690066,0.757447,0.519513,0.703839,0.588697,0.878978,0.632716,0.689176,0.786706,0.501695,0.994070,0.971234,0.884059,0.728613,0.792458,0.762975,0.694245,0.557559,0.606450,0.514600,0.828963,0.849573,0.555911,0.697620,0.889889,0.765987,0.999416,0.584204,0.804512,0.587060,0.687590,0.833076,0.971464,0.572038,0.511163,0.887142,0.897811,0.635812,0.552993,0.589638,0.802888,0.778250,0.832245,0.582724,0.720892,0.977099,0.697114,0.708517,0.532714,0.984704,0.769950,0.872977,0.771892,0.691919,0.589122,0.507944,0.781885,0.983734,0.867816,0.973126,0.660644,0.938269,0.506346,0.872609,0.658066,0.861606,0.692677,0.731200,0.732683,0.601762,0.611636,0.969781,0.614403,0.632393,0.878060,0.660771,0.735423,0.805596,0.502820,0.569998,0.882271,0.676932,0.603095,0.967001,0.838754,0.511536,0.834361,0.907214,0.528693,0.530182,0.501227,0.771642,0.777275,0.617077,0.574527,0.600665,0.722550,0.726767,0.691768,0.522390,0.506171,0.501544,0.500867,0.753229,0.740100,0.580675,0.676752,0.603243,0.714782,0.708729,0.606917,0.549322,0.902877,0.766926,0.922566,0.595677,0.758416,0.877597,0.680259,0.941206,0.866991,0.833968,0.604659,0.516728,0.974060,0.515119,0.560098,0.799218,0.859614,0.533249,0.720543,0.728385,0.921619,0.875461,0.860915,0.901507,0.644519,0.916504,0.770224,0.686738,0.526902,0.624971,0.965702,0.507666,0.941155,0.956323,0.625161,0.644833,0.653197,0.629707,0.578685,0.751802,0.771523,0.726709,0.575366,0.947334,0.520761,0.536021,0.578086,0.917505,0.703141,0.771336,0.825684,0.622704,0.872283,0.542492,0.584884,0.613473,0.755808,0.582125,0.508345,0.538984,0.606521,0.678615,0.557553,0.764427,0.836144,0.693727,0.776671,0.737049,0.993072,0.611296,0.574666,0.974594,0.588730,0.738014,0.912756,0.855369,0.654077,0.795788,0.724114,0.599587,0.636379,0.936303,0.551302,0.644834,0.820063,0.558571,0.834000,0.647129,0.605194,0.615495,0.653296,0.759486,0.978869,0.845270,0.782165,0.790035,0.865989,0.764025,0.654441,0.693493,0.751533,0.560182,0.780085,0.501994,0.797121,0.617453,0.667058,0.706045,0.609746,0.592197,0.895344,0.790264,0.539090,0.733035,0.981569,0.889334,0.615309,0.529008,0.800414,0.989695,0.529987,0.891025,0.617398,0.639139,0.576219,0.773260,0.514025,0.515608,0.507275,0.830339,0.948435,0.625080,0.786987,0.514658,0.596119,0.823375,0.522814,0.640034,0.811251,0.664414,0.954790,0.883794,0.938942,0.515272,0.572268,0.939704,0.615851,0.898428,0.838511,0.608223,0.752605,0.626721,0.760586,0.749608,0.815947,0.630171,0.585352,0.522938,0.555585,0.754316,0.966230,0.524049,0.510878,0.829619,0.590486,0.772009,0.995414,0.573659,0.566669,0.718884,0.617860,0.969431,0.657141,0.671601,0.576946,0.712277,0.969523,0.615203,0.610716,0.692453,0.564087,0.569859,0.778186,0.636742,0.860933,0.957847,0.529953,0.668238,0.731126,0.550414,0.577507,0.601909,0.635181,0.816048,0.765811,0.564600,0.916360,0.620460,0.760240,0.570101,0.639452,0.680892,0.561255,0.924276,0.766688,0.773852,0.791068,0.542555,0.769103,0.566200,0.918376,0.867228,0.777287,0.614600,0.569705,0.818302,0.555005,0.582320,0.942585,0.546435,0.817598,0.857339,0.981167,0.646646,0.569737,0.986680,0.982094,0.671305,0.748206,0.571285,0.706302,0.663458,0.833788,0.816796,0.730953,0.742086,0.631233,0.530474,0.570492,0.837006,0.969455,0.502280,0.734629,0.874027,0.990441,0.576405,0.524045,0.514121,0.506786,0.643670,0.843154,0.577545,0.603665,0.755977,0.602251,0.638933,0.543540,0.528129,0.554923,0.632037,0.981164,0.686503,0.805535,0.515470,0.645952,0.733428,0.830514,0.554501,0.982567,0.984940,0.727961,0.938873,0.653076,0.626078,0.699293,0.653797,0.954297,0.636510,0.652862,0.574502,0.810561,0.861167,0.681222,0.588448,0.658265,0.557277,0.903984,0.834712,0.597074,0.839914,0.824404,0.912222,0.511302,0.602098,0.922984,0.616517,0.661964,0.937560,0.558220,0.653086,0.596964,0.589982,0.795830,0.610344,0.917650,0.826060,0.564802,0.575969,0.650881,0.868700,0.587888,0.681683,0.920926,0.638973,0.810843,0.641706,0.600474,0.569400,0.579757,0.554706,0.851419,0.863613,0.522048,0.861104,0.681475,0.847367,0.506139,0.689748,0.832966,0.940200,0.640749,0.754091,0.783569,0.738307,0.764238,0.694994,0.697225,0.796731,0.682115,0.985396,0.531369,0.903758,0.586616,0.804715,0.942406,0.557756,0.898947,0.661217,0.816952,0.948905,0.791427,0.691971,0.515589,0.615847,0.762195,0.792494,0.886021,0.898877,0.737556,0.511954,0.533631,0.662644,0.806255,0.692855,0.545015,0.505217,0.734725,0.548374,0.898844,0.884703,0.969954,0.810149,0.942285,0.546015,0.665549,0.799700,0.694363,0.808419,0.714584,0.970869,0.522909,0.657753,0.893413,0.734257,0.817392,0.920364,0.819595,0.508191,0.681618,0.514768,0.693970,0.556084,0.951136,0.768437,0.955630,0.566527,0.757824,0.673512,0.665366,0.695541,0.706647,0.640679,0.566618,0.657967,0.617096,0.951216,0.730804,0.943158,0.735712,0.767761,0.832778,0.660000,0.573982,0.790568,0.544242,0.988464,0.696316,0.737318,0.910474,0.818907,0.560920,0.581922,0.822870,0.978978,0.603139,0.514198,0.985550,0.577319,0.554292,0.507653,0.821290,0.719375,0.589733,0.929635,0.718352,0.624180,0.534246,0.549263,0.705859,0.917711,0.672675,0.638118,0.747290,0.731458,0.930493,0.541007,0.804428,0.788034,0.805918,0.675106,0.640412,0.578294,0.944380,0.864556,0.803897,0.665637,0.918420,0.768340,0.865890,0.856264,0.705872,0.727714,0.521809,0.957347,0.514923,0.557492,0.756344,0.966770,0.876032,0.824639,0.536858,0.951290,0.656409,0.643844,0.939688,0.509219,0.769048,0.771473,0.789490,0.999181,0.982072,0.994389,0.531177,0.757885,0.580390,0.556098,0.527459,0.884218,0.712089,0.518168,0.674389,0.786836,0.696663,0.523450,0.652871,0.620573,0.569799,0.712581,0.501382,0.540638,0.613001,0.577955,0.994218,0.736010,0.524680,0.934659,0.587929,0.721076,0.506280,0.728900,0.616777,0.603861,0.946074,0.958262,0.694145,0.906673,0.537849,0.532926,0.648638,0.940041,0.773592,0.659245,0.754743,0.593786,0.516401,0.675682,0.945073,0.711082,0.898996,0.850771,0.594923,0.785451,0.575584,0.564158,0.887591,0.905378,0.760976,0.643532,0.790651,0.657775,0.647603,0.935364,0.892667,0.810454,0.673593,0.822371,0.524348,0.877698,0.868284,0.704028,0.685720,0.973210,0.631970,0.504677,0.768445,0.777671,0.721157,0.504104,0.661135,0.634128,0.681146,0.584294,0.516049,0.607105,0.552995,0.815231,0.626489,0.587551,0.528074,0.742666,0.519153,0.598533,0.515355,0.912555,0.816790,0.787442,0.670953,0.678033,0.528863,0.855032,0.975671,0.602977,0.914829,0.530105,0.939852,0.602408,0.679759,0.811555,0.729848,0.603417,0.537429,0.815140,0.584551,0.688140,0.974635,0.638523,0.973787,0.897153,0.712152,0.931772,0.502226,0.861136,0.832447,0.684843,0.654181,0.935083,0.585141,0.794838,0.672834,0.682738,0.962258,0.665060,0.519802,0.674748,0.699604,0.536976,0.564796,0.608110,0.623556,0.866926,0.952861,0.814271,0.801842,0.745307,0.520313,0.993781,0.788491,0.676620,0.637367,0.677804,0.978380,0.734369,0.692291,0.897614,0.899925,0.667310,0.728767,0.540061,0.846365,0.618768,0.602123,0.540769,0.573761,0.542450,0.551001,0.533806,0.594581,0.994887,0.584887,0.734104,0.584286,0.789093,0.596151,0.507386,0.882392,0.810845,0.905976,0.552367,0.701567,0.527351,0.768637,0.994197,0.512025,0.677038,0.709790,0.771664,0.606230,0.792760,0.638926,0.702036,0.615163,0.956764,0.833050,0.691457,0.962441,0.507967,0.534703,0.591584,0.738174,0.834580,0.627551,0.659811,0.663361,0.594087,0.619881,0.807289,0.674489,0.548323,0.695737,0.940800,0.517396,0.633979,0.689560,0.899858,0.591520,0.896033,0.557551,0.776102,0.668716,0.548190,0.546598,0.551334,0.571108,0.516259,0.918135,0.870273,0.933316,0.797049,0.778825,0.858276,0.551109,0.731660,0.558837,0.629368,0.553292,0.570004,0.798780,0.577814,0.538442,0.542914,0.510081,0.583105,0.949896,0.624367,0.682051,0.820423,0.707944,0.646891,0.914055,0.502858,0.764720,0.787200,0.933777,0.581625,0.560032,0.671904,0.586146,0.898711,0.645500,0.622689,0.508812,0.796572,0.742689,0.920640,0.645122,0.927844,0.675600,0.715380,0.677209,0.698101,0.762512,0.529878,0.990274,0.809558,0.830839,0.817179,0.692775,0.535579,0.517526,0.583096,0.644409,0.682455,0.569785,0.691961,0.924021,0.670208,0.748741,0.741419,0.669013,0.811376,0.566437,0.924694,0.659760,0.801303,0.635517,0.551248,0.633944,0.691728,0.515302,0.530774,0.624389,0.765149,0.691153,0.718973,0.559147,0.672501,0.534403,0.575134,0.579602,0.544695,0.958279,0.524649,0.966062,0.640890,0.757626,0.959639,0.552628,0.888555,0.648558,0.665841,0.698651,0.503226,0.679286,0.779830,0.831599,0.874544,0.725706,0.642331,0.622112,0.629383,0.791859,0.882187,0.546225,0.657825,0.536229,0.797985,0.712336,0.787393,0.694016,0.725627,0.502436,0.827754,0.704810,0.592739,0.562416,0.849953,0.594419,0.834995,0.777622,0.911961,0.553843,0.511110,0.685430,0.572228,0.662891,0.729454,0.583920,0.552009,0.645266,0.570993,0.797640,0.843370,0.545410,0.977566,0.788593,0.663330,0.545831,0.887052,0.629114,0.543090,0.625322,0.751384,0.793462,0.521653,0.871467,0.883720,0.772674,0.677656,0.545107,0.502232,0.597437,0.784787,0.543626,0.779243,0.556196,0.560238,0.676432,0.846966,0.521160,0.835306,0.731015,0.634926,0.991825,0.519924,0.864947,0.818994,0.658723,0.645193,0.611842,0.734916,0.508423,0.618546,0.764883,0.834354,0.972957,0.735892,0.519381,0.700085,0.512397,0.589341,0.564970,0.899880,0.812047,0.723338,0.696444,0.730267,0.566294,0.649935,0.986194,0.922178,0.632927,0.932200,0.846652,0.908397,0.880515,0.702592,0.924051,0.517768,0.581564,0.679738,0.545187,0.614054,0.694566,0.588203,0.902422,0.607244,0.528776,0.944032,0.938267,0.963095,0.605775,0.530499,0.826614,0.627309,0.582548,0.594175,0.788207,0.602799,0.603162,0.527696,0.524074,0.582524,0.854984,0.574180,0.608282,0.616585,0.543637,0.731461,0.536416,0.719729,0.791521,0.720978,0.891399,0.991762,0.505493,0.683367,0.744806,0.654379,0.513323,0.549679,0.746186,0.686466,0.516488,0.573784,0.769672,0.979504,0.919853,0.509961,0.908712,0.618998,0.758552,0.738173,0.594561,0.594687,0.561195,0.640166,0.533538,0.704407,0.954116,0.722399,0.686806,0.727575,0.695096,0.590939,0.797219,0.983782,0.888195,0.779726,0.797589,0.554568,0.630230,0.660613,0.570204,0.808583,0.551580,0.543321,0.517859,0.786806,0.976218,0.544852,0.525311,0.522045,0.897338,0.791266,0.536538,0.768672,0.506002,0.695556,0.522921,0.506790,0.606112,0.556216,0.835484,0.617945,0.709681,0.766113,0.830225,0.857074,0.505018,0.546638,0.643318,0.643159,0.573410,0.615476,0.822208,0.729317,0.516762,0.519818,0.668461,0.633734,0.509655,0.712167,0.523763,0.534205,0.941019,0.989133,0.742201,0.544321,0.734171,0.975028,0.554021,0.742703,0.532860,0.619875,0.517420,0.576525,0.698751,0.708663,0.701093,0.777528,0.669888,0.717030,0.786247,0.521918,0.526587,0.999318,0.651312,0.626715,0.659568,0.714421,0.706070,0.659291,0.622109,0.570337,0.922460,0.536715,0.508893,0.579280,0.511562,0.968072,0.920185,0.767288,0.521542,0.770241,0.620447,0.597605,0.688526,0.760857,0.551268,0.633439,0.667333,0.947790,0.682025,0.606153,0.630926,0.523892,0.835144,0.762286,0.550580,0.591180,0.595000,0.523690,0.623153,0.524906,0.873542,0.665701,0.845775,0.751717,0.865103,0.637297,0.920762,0.701131,0.512484,0.638573,0.681812,0.742622,0.949704,0.631135,0.880194,0.562190,0.651426,0.699293,0.709721,0.547076,0.555480,0.986500,0.929227,0.881653,0.888086,0.550966,0.614808,0.532984,0.526814,0.924731,0.505296,0.508378,0.615610,0.666292,0.559766,0.651443,0.590140,0.672662,0.544921,0.505219,0.723391,0.987773,0.890035,0.960028,0.802350,0.526929,0.642455,0.533690,0.662137,0.559095,0.565664,0.787711,0.638564,0.705760,0.702835,0.566761,0.896739,0.695667,0.588421,0.702566,0.844166,0.939488,0.602177,0.514992,0.537638,0.527600,0.513125,0.531619,0.658824,0.551977,0.714377,0.554553,0.877775,0.518557,0.922991,0.619926,0.658366,0.500874,0.631707,0.779059,0.811973,0.638405,0.677318,0.545907,0.624274,0.612513,0.771565,0.783125,0.751824,0.813123,0.514679,0.581086,0.693925,0.548767,0.773776,0.545685,0.768059,0.648912,0.829601,0.624125,0.863289,0.987367,0.981917,0.655685,0.770932,0.752078,0.924920,0.602176,0.532654,0.531767,0.632085,0.730030,0.779690,0.771626,0.547535,0.816384,0.871460,0.712371,0.798854,0.874483,0.540476,0.778107,0.848606,0.931097,0.665209,0.696852,0.519830,0.555140,0.602545,0.978823,0.659245,0.851035,0.920615,0.990521,0.751255,0.793120,0.719350,0.679788,0.594149,0.501818,0.623516,0.828617,0.987011,0.638655,0.656911,0.713282,0.659123,0.692149,0.667829,0.548452,0.748970,0.769215,0.923634,0.557207,0.718384,0.593139,0.789695,0.611092,0.913036,0.693703,0.648770,0.780078,0.909571,0.844516,0.638705,0.609644,0.848226,0.532538,0.780293,0.800526,0.880309,0.512348,0.717122,0.578994,0.723827,0.562123,0.717739,0.967901,0.854844,0.760932,0.532425,0.832964,0.532763,0.731544,0.510791,0.528485,0.509289,0.552526,0.848697,0.842137,0.637663,0.896218,0.686330,0.678505,0.562170,0.692338,0.666596,0.895296,0.661246,0.642728,0.540255,0.808510,0.687766,0.683273,0.573999,0.914409,0.533839,0.683827,0.583470,0.662299,0.996099,0.647904,0.704463,0.554613,0.688166,0.667467,0.634404,0.737330,0.500375,0.560652,0.740979,0.521953,0.983352,0.870446,0.516183,0.641645,0.503722,0.637478,0.585290,0.658373,0.613691,0.632951,0.517986,0.645471,0.992817,0.946122,0.572910,0.798033,0.714063,0.528047,0.792822,0.864044,0.599560,0.583365,0.596681,0.904282,0.847976,0.903948,0.952766,0.634689,0.608027,0.892981,0.758376,0.982528,0.727062,0.632283,0.880523,0.896448,0.685692,0.723785,0.947271,0.749264,0.581775,0.614046,0.711530,0.687905,0.706514,0.593917,0.531007,0.686798,0.542248,0.649659,0.508881,0.616104,0.612325,0.511700,0.930088,0.878101,0.749117,0.692520,0.549218,0.576035,0.528333,0.501834,0.607371,0.577728,0.548207,0.969492,0.597926,0.735848,0.841298,0.568404,0.604187,0.572133,0.618504,0.697800,0.886685,0.860388,0.974550,0.832127,0.589669,0.874713,0.651042,0.531972,0.933322,0.960181,0.614574,0.948403,0.632662,0.805168,0.882747,0.672358,0.577961,0.540454,0.536815,0.526249,0.762864,0.821748,0.567960,0.735149,0.901840,0.713996,0.655950,0.576685,0.941557,0.972037,0.543811,0.959219,0.609126,0.846156,0.932300,0.563894,0.591751,0.708345,0.555674,0.511722,0.908770,0.537835,0.927420,0.938930,0.619008,0.640650,0.872247,0.918715,0.675599,0.622576,0.509771,0.619981,0.607121,0.789019,0.835211,0.875103,0.558917,0.649079,0.574789,0.978616,0.919767,0.806355,0.575415,0.961667,0.923530,0.507516,0.919425,0.827742,0.632351,0.537846,0.514043,0.830192,0.909633,0.588824,0.811940,0.613953,0.975533,0.738014,0.913050,0.649809,0.643300,0.516472,0.832117,0.508198,0.923099,0.964566,0.966428,0.584551,0.931334,0.583666,0.629497,0.592979,0.946712,0.557817,0.536525,0.620178,0.594796,0.573992,0.596996,0.891455,0.870582,0.646068,0.559044,0.883468,0.510300,0.755707,0.908002,0.819938,0.819989,0.656003,0.673477,0.560619,0.886449,0.768728,0.877871,0.558933,0.623413,0.697841,0.741670,0.501070,0.831713,0.791552,0.780452,0.524542,0.610575,0.653839,0.772226,0.635660,0.853931,0.512601,0.636133,0.845295,0.890411,0.726680,0.983402,0.761055,0.535167,0.712937,0.876868,0.825818,0.863109,0.758582,0.616141,0.833389,0.766108,0.978175,0.671584,0.785079,0.562625,0.587195,0.753619,0.892993,0.576775,0.552069,0.664659,0.876123,0.699018,0.824955,0.672741,0.507264,0.860665,0.872828,0.875471,0.645615,0.534452,0.645587,0.806005,0.694296,0.827410,0.829343,0.502059,0.558193,0.739916,0.826337,0.516145,0.612632,0.743944,0.960263,0.739507,0.566534,0.513586,0.738738,0.599275,0.598748,0.838095,0.664418,0.669337,0.895417,0.578945,0.737592,0.745182,0.806497,0.794974,0.655829,0.696257,0.603515,0.551818,0.898044,0.839481,0.509381,0.513047,0.714429,0.775612,0.618422,0.687804,0.500828,0.566802,0.576093,0.500991,0.861424,0.675397,0.739996,0.889239,0.799517,0.682526,0.550961,0.842974,0.556267,0.556375,0.723871,0.855436,0.612256,0.714245,0.838650,0.697745,0.817892,0.564115,0.923700,0.584278,0.616974,0.633112,0.819412,0.673336,0.904192,0.950917,0.533612,0.702059,0.681527,0.591155,0.590867,0.579597,0.698790,0.947700,0.693420,0.598228,0.820138,0.568886,0.805965,0.809784,0.537136,0.679962,0.593628,0.687051,0.584550,0.908635,0.833825,0.561267,0.544233,0.517493,0.609464,0.610031,0.692748,0.588608,0.537417,0.838505,0.879801,0.591546,0.600638,0.527493,0.518618,0.669121,0.860059,0.815756,0.936363,0.793101,0.657470,0.544994,0.801257,0.512326,0.993877,0.530979,0.896696,0.584245,0.632070,0.595182,0.882610,0.580219,0.545903,0.532899,0.526675,0.519765,0.584660,0.622480,0.527506,0.992263,0.757620,0.751699,0.678665,0.626465,0.763028,0.558436,0.669355,0.761168,0.999856,0.581894,0.758691,0.853373,0.817125,0.513182,0.737606,0.725947,0.868673,0.818934,0.880900,0.919227,0.757261,0.608800,0.781478,0.866959,0.616456,0.644712,0.502801,0.945276,0.604868,0.537209,0.604185,0.521673,0.669286,0.765195,0.722956,0.530312,0.630173,0.878306,0.833932,0.731524,0.806112,0.575899,0.754603,0.575137,0.982243,0.894621,0.696350,0.619424,0.510844,0.536831,0.747418,0.536703,0.658459,0.588447,0.502215,0.816545,0.544740,0.780556,0.908641,0.703523,0.896561,0.509545,0.594007,0.634455,0.589897,0.640035,0.829027,0.929855,0.565002,0.571532,0.535349,0.969330,0.634179,0.710121,0.803922,0.839298,0.541716,0.702163,0.893726,0.909204,0.843649,0.980611,0.596455,0.772972,0.639307,0.941095,0.676530,0.626984,0.947744,0.697355,0.795078,0.817290,0.507904,0.667130,0.609868,0.869333,0.529649,0.910995,0.702293,0.839886,0.675910,0.791250,0.534177,0.710201,0.585354,0.928913,0.803338,0.572296,0.661815,0.608233,0.656405,0.590541,0.717460,0.629987,0.579074,0.503049,0.534517,0.515844,0.802936,0.504431,0.571439,0.860208,0.502086,0.594442,0.535106,0.794329,0.683728,0.572394,0.521739,0.867598,0.517310,0.630415,0.613321,0.880140,0.858420,0.535489,0.746726,0.575846,0.534580,0.925240,0.689042,0.586380,0.597501,0.628043,0.897703,0.897582,0.859543,0.680124,0.745165,0.683359,0.569518,0.822369,0.953845,0.815589,0.954318,0.712782,0.898677,0.532372,0.821874,0.964946,0.610489,0.521733,0.900528,0.687458,0.894757,0.584525,0.609358,0.993129,0.670347,0.532234,0.891700,0.730415,0.719836,0.609272,0.590296,0.566899,0.758366,0.655521,0.656316,0.559478,0.598757,0.683144,0.701055,0.542841,0.682559,0.551246,0.925318,0.866177,0.577701,0.841740,0.560973,0.989597,0.564939,0.683741,0.589147,0.913622,0.787658,0.756309,0.617475,0.627253,0.568533,0.856240,0.555307,0.524703,0.890355,0.756757,0.687874,0.765980,0.704819,0.738518,0.744330,0.786112,0.772932,0.539244,0.948652,0.525768,0.635056,0.553684,0.935317,0.619300,0.874358,0.505898,0.854615,0.726760,0.966782,0.974917,0.602840,0.711491,0.804812,0.630254,0.850205,0.666399,0.780373,0.527828,0.994786,0.539608,0.536165,0.782036,0.672048,0.741264,0.627415,0.510430,0.618449,0.564999,0.865035,0.686342,0.727815,0.563134,0.924293,0.719184,0.667259,0.532777,0.988724,0.842170,0.723024,0.560712,0.519846,0.871338,0.620464,0.757492,0.566815,0.647685,0.930836,0.716215,0.587169,0.672158,0.723907,0.729189,0.880398,0.993617,0.534566,0.884257,0.606307,0.910855,0.666199,0.792056,0.932039,0.869906,0.552504,0.634945,0.716268,0.623482,0.612355,0.642101,0.587182,0.873647,0.923003,0.658805,0.779746,0.958695,0.938793,0.515003,0.829384,0.814136,0.950100,0.765833,0.825197,0.834635,0.585122,0.797837,0.598383,0.835855,0.694742,0.841783,0.841400,0.541773,0.789137,0.686870,0.841706,0.712247,0.841181,0.806257,0.853577,0.544187,0.530371,0.623923,0.994601,0.652853,0.534361,0.588812,0.623167,0.737228,0.570350,0.898118,0.521930,0.761639,0.684273,0.864673,0.546584,0.933806,0.641469,0.772825,0.820961,0.653215,0.529578,0.980726,0.782312,0.900929,0.612532,0.884115,0.588875,0.836864,0.722442,0.514867,0.621510,0.822640,0.644489,0.705741,0.520483,0.656402,0.856650,0.573131,0.850359,0.922483,0.781142,0.650504,0.517320,0.790673,0.790346,0.525060,0.708708,0.730530,0.934805,0.972426,0.984432,0.567031,0.953753,0.919123,0.530801,0.736021,0.517827,0.935760,0.603931,0.613237,0.512608,0.634149,0.557339,0.717346,0.580268,0.780764,0.703727,0.964587,0.817447,0.724281,0.605964,0.522025,0.622546,0.910109,0.911862,0.504410,0.898550,0.780780,0.584522,0.729039,0.666069,0.755488,0.537778,0.527593,0.924413,0.562532,0.913409,0.545811,0.530812,0.762436,0.704208,0.883273,0.542926,0.572286,0.632369,0.534895,0.563043,0.529436,0.547992,0.501942,0.550369,0.808405,0.533193,0.779308,0.508069,0.704477,0.839470,0.583873,0.500258,0.932043,0.658299,0.759832,0.559789,0.617256,0.799390,0.993381,0.800238,0.664349,0.952440,0.539653,0.668247,0.518358,0.529864,0.961625,0.620353,0.656377,0.633078,0.519646,0.759027,0.720200,0.943784,0.657675,0.623819,0.505316,0.652226,0.756948,0.563017,0.711028,0.593333,0.661112,0.586769,0.585768,0.832693,0.835787,0.510191,0.904203,0.591200,0.988997,0.818890,0.516735,0.612646,0.679674,0.848853,0.868819,0.782420,0.750282,0.539338,0.580821,0.584123,0.627436,0.739397,0.513599,0.863289,0.693866,0.953712,0.549846,0.894575,0.587264,0.859917,0.952309,0.501403,0.525883,0.525747,0.806724,0.704435,0.719669,0.937695,0.926365,0.666959,0.598345,0.624459,0.548570,0.525354,0.548261,0.686292,0.616857,0.727101,0.729916,0.630925,0.545468,0.761062,0.629367,0.963022,0.560292,0.767381,0.529339,0.993524,0.590917,0.681550,0.523932,0.692539,0.912764,0.874456,0.692972,0.676876,0.541228,0.665072,0.829814,0.623042,0.962697,0.566986,0.570447,0.920588,0.626048,0.789109,0.636124,0.673190,0.931352,0.874773,0.799152,0.551268,0.697888,0.619751,0.528606,0.985379,0.511251,0.600445,0.613939,0.643314,0.518701,0.763626,0.982194,0.939501,0.527052,0.881104,0.848612,0.600703,0.531217,0.639374,0.502605,0.817233,0.726248,0.769259,0.688568,0.533569,0.727021,0.631741,0.907207,0.575617,0.726012,0.505727,0.846368,0.892139,0.546883,0.560584,0.591703,0.761043,0.796271,0.991654,0.766411,0.836291,0.764756,0.520865,0.646544,0.993187,0.839180,0.566034,0.726608,0.810738,0.806495,0.500114,0.762925,0.540559,0.542776,0.501205,0.696142,0.675447,0.995894,0.536824,0.852376,0.956970,0.855390,0.774421,0.723295,0.621738,0.776799,0.673389,0.790836,0.627820,0.866701,0.823583,0.709899,0.544138,0.688214,0.869284,0.544708,0.671073,0.843416,0.500025,0.733597,0.517465,0.996358,0.722442,0.562807,0.720343,0.646413,0.616268,0.709548,0.731968,0.651231,0.589715,0.807276,0.529787,0.548317,0.544531,0.756102,0.613900,0.524450,0.854834,0.891176,0.553692,0.790720,0.814412,0.909962,0.882104,0.550020,0.568313,0.998435,0.940931,0.672146,0.506648,0.993033,0.554675,0.752493,0.730921,0.592049,0.922885,0.530669,0.588007,0.657881,0.631699,0.738406,0.529402,0.783745,0.566176,0.890325,0.937325,0.612079,0.686946,0.569215,0.926434,0.813177,0.841131,0.584576,0.556842,0.564864,0.532450,0.805640,0.586781,0.562662,0.621531,0.929833,0.623090,0.602296,0.717005,0.678334,0.640450,0.664946,0.682579,0.585713,0.690073,0.716073,0.680035,0.786117,0.523665,0.521576,0.642703,0.502225,0.921224,0.889663,0.971759,0.738604,0.850597,0.771059,0.712410,0.546114,0.565487,0.547161,0.611372,0.787204,0.506167,0.826206,0.712441,0.814837,0.595521,0.814770,0.751325,0.899195,0.929380,0.612657,0.703498,0.576136,0.586314,0.544607,0.540876,0.918921,0.933814,0.501399,0.619953,0.837371,0.777115,0.956804,0.659142,0.645526,0.556782,0.886267,0.512728,0.703773,0.917850,0.556564,0.838164,0.632449,0.991582,0.713373,0.619865,0.695421,0.685882,0.623268,0.854422,0.648985,0.942072,0.781439,0.910429,0.680532,0.877612,0.633828,0.528014,0.685254,0.996085,0.669747,0.501741,0.747966,0.570784,0.842271,0.833407,0.651403,0.575349,0.582787,0.642273,0.749630,0.622495,0.530405,0.569362,0.539769,0.506735,0.521012,0.685728,0.762855,0.500798,0.989872,0.568201,0.610370,0.531330,0.769909,0.629330,0.819936,0.847229,0.999393,0.570695,0.691816,0.780152,0.732340,0.592083,0.937740,0.637085,0.722739,0.658681,0.577389,0.952036,0.526448,0.778559,0.565524,0.999907,0.508800,0.983398,0.568203,0.730324,0.856107,0.860845,0.622733,0.535582,0.626591,0.923520,0.740714,0.581845,0.616676,0.897479,0.553000,0.982286,0.957290,0.521129,0.569101,0.902155,0.769396,0.625350,0.856245,0.986515,0.706833,0.901950,0.803399,0.975705,0.636781,0.742913,0.592745,0.578619,0.608579,0.523327,0.560753,0.989759,0.755270,0.500222,0.665660,0.503391,0.714174,0.797576,0.701777,0.575785,0.756752,0.749397,0.692717,0.830958,0.865755,0.982452,0.763315,0.600078,0.599228,0.995584,0.528154,0.797493,0.521575,0.653908,0.708185,0.547343,0.858431,0.722819,0.587467,0.881551,0.787478,0.903494,0.546682,0.912385,0.502199,0.532288,0.791230,0.860788,0.715438,0.662377,0.727037,0.634995,0.625792,0.569208,0.850745,0.761915,0.800918,0.829074,0.693222,0.775192,0.735751,0.719312,0.996520,0.631545,0.600390,0.906481,0.663539,0.533644,0.607891,0.778355,0.768967,0.545952,0.793772,0.697089,0.737618,0.678167,0.731891,0.774581,0.505480,0.623881,0.519453,0.756938,0.551445,0.742161,0.512313,0.645119,0.773553,0.609224,0.926211,0.739054,0.929423,0.642507,0.543291,0.830771,0.986634,0.608191,0.794209,0.635890,0.924921,0.930495,0.842856,0.933783,0.512547,0.555099,0.569148 From 38dc714ef0ea89bd850cc4a49b7d6bec6a60d51a Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Tue, 25 Apr 2023 12:46:08 +0200 Subject: [PATCH 122/129] include the stretching parameter data from M. Waclawczyk --- .../vel_DNS.dat} | 0 .../Waclawczyk_LES/cdf_rv_subsaturated.dat | 51 +++++++++++++++++++ .../Waclawczyk_LES/cdf_rv_supersaturated.dat | 51 +++++++++++++++++++ .../Waclawczyk_LES/cdf_th_subsaturated.dat | 51 +++++++++++++++++++ .../Waclawczyk_LES/cdf_th_supersaturated.dat | 51 +++++++++++++++++++ 5 files changed, 204 insertions(+) rename libmpdata++/formulae/stretching_parameter_data/{Akinlabi_DNS.dat => Akinlabi_DNS/vel_DNS.dat} (100%) create mode 100644 libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_rv_subsaturated.dat create mode 100644 libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_rv_supersaturated.dat create mode 100644 libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_th_subsaturated.dat create mode 100644 libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_th_supersaturated.dat diff --git a/libmpdata++/formulae/stretching_parameter_data/Akinlabi_DNS.dat b/libmpdata++/formulae/stretching_parameter_data/Akinlabi_DNS/vel_DNS.dat similarity index 100% rename from libmpdata++/formulae/stretching_parameter_data/Akinlabi_DNS.dat rename to libmpdata++/formulae/stretching_parameter_data/Akinlabi_DNS/vel_DNS.dat diff --git a/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_rv_subsaturated.dat b/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_rv_subsaturated.dat new file mode 100644 index 00000000..db0c93c1 --- /dev/null +++ b/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_rv_subsaturated.dat @@ -0,0 +1,51 @@ + -1.9936882e+00 3.6607750e-03 + -1.9139081e+00 7.4978095e-03 + -1.8341280e+00 1.1724649e-02 + -1.7543479e+00 1.6198929e-02 + -1.6745678e+00 2.1068099e-02 + -1.5947877e+00 2.6342327e-02 + -1.5150077e+00 3.2182618e-02 + -1.4352276e+00 3.8644903e-02 + -1.3554475e+00 4.5646135e-02 + -1.2756674e+00 5.3330373e-02 + -1.1958873e+00 6.2131486e-02 + -1.1161072e+00 7.1863046e-02 + -1.0363271e+00 8.2509800e-02 + -9.5654705e-01 9.4719163e-02 + -8.7676697e-01 1.0841317e-01 + -7.9698688e-01 1.2429178e-01 + -7.1720679e-01 1.4261769e-01 + -6.3742671e-01 1.6377392e-01 + -5.5764662e-01 1.8794012e-01 + -4.7786654e-01 2.1629587e-01 + -3.9808645e-01 2.4917505e-01 + -3.1830636e-01 2.8725220e-01 + -2.3852628e-01 3.3044256e-01 + -1.5874619e-01 3.7904952e-01 + -7.8966106e-02 4.3228329e-01 + 8.1397964e-04 4.8916258e-01 + 8.0594066e-02 5.4704858e-01 + 1.6037415e-01 6.0301438e-01 + 2.4015424e-01 6.5454148e-01 + 3.1993432e-01 7.0126212e-01 + 3.9971441e-01 7.4248312e-01 + 4.7949450e-01 7.7761131e-01 + 5.5927458e-01 8.0739748e-01 + 6.3905467e-01 8.3276360e-01 + 7.1883475e-01 8.5513500e-01 + 7.9861484e-01 8.7462354e-01 + 8.7839493e-01 8.9117499e-01 + 9.5817501e-01 9.0566556e-01 + 1.0379551e+00 9.1841726e-01 + 1.1177352e+00 9.2990633e-01 + 1.1975153e+00 9.3971246e-01 + 1.2772954e+00 9.4867797e-01 + 1.3570754e+00 9.5673676e-01 + 1.4368555e+00 9.6387696e-01 + 1.5166356e+00 9.7051212e-01 + 1.5964157e+00 9.7658799e-01 + 1.6761958e+00 9.8208254e-01 + 1.7559759e+00 9.8706187e-01 + 1.8357560e+00 9.9180224e-01 + 1.9155360e+00 9.9600704e-01 + 1.9953161e+00 1.0000000e+00 diff --git a/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_rv_supersaturated.dat b/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_rv_supersaturated.dat new file mode 100644 index 00000000..567c9047 --- /dev/null +++ b/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_rv_supersaturated.dat @@ -0,0 +1,51 @@ + -1.9936882e+00 4.8301652e-03 + -1.9139081e+00 5.9987535e-03 + -1.8341280e+00 7.5568713e-03 + -1.7543479e+00 9.1928950e-03 + -1.6745678e+00 1.0751013e-02 + -1.5947877e+00 1.2153319e-02 + -1.5150077e+00 1.4568401e-02 + -1.4352276e+00 1.7450919e-02 + -1.3554475e+00 2.0956684e-02 + -1.2756674e+00 2.5397320e-02 + -1.1958873e+00 2.9760050e-02 + -1.1161072e+00 3.4278591e-02 + -1.0363271e+00 4.1290122e-02 + -9.5654705e-01 4.7366781e-02 + -8.7676697e-01 5.6793394e-02 + -7.9698688e-01 7.0894360e-02 + -7.1720679e-01 8.6553443e-02 + -6.3742671e-01 1.0330321e-01 + -5.5764662e-01 1.2394827e-01 + -4.7786654e-01 1.5207230e-01 + -3.9808645e-01 1.8518230e-01 + -3.1830636e-01 2.2351200e-01 + -2.3852628e-01 2.7648800e-01 + -1.5874619e-01 3.3476161e-01 + -7.8966106e-02 4.0986289e-01 + 8.1397964e-04 4.9353381e-01 + 8.0594066e-02 5.7517918e-01 + 1.6037415e-01 6.5269554e-01 + 2.4015424e-01 7.2210969e-01 + 3.1993432e-01 7.7827984e-01 + 3.9971441e-01 8.2003739e-01 + 4.7949450e-01 8.5454970e-01 + 5.5927458e-01 8.8205048e-01 + 6.3905467e-01 9.0511063e-01 + 7.1883475e-01 9.2178249e-01 + 7.9861484e-01 9.3487068e-01 + 8.7839493e-01 9.4476472e-01 + 9.5817501e-01 9.5255531e-01 + 1.0379551e+00 9.6057962e-01 + 1.1177352e+00 9.6727953e-01 + 1.1975153e+00 9.7327828e-01 + 1.2772954e+00 9.7701776e-01 + 1.3570754e+00 9.8028981e-01 + 1.4368555e+00 9.8247117e-01 + 1.5166356e+00 9.8480835e-01 + 1.5964157e+00 9.8667809e-01 + 1.6761958e+00 9.8924899e-01 + 1.7559759e+00 9.9150826e-01 + 1.8357560e+00 9.9298847e-01 + 1.9155360e+00 9.9478031e-01 + 1.9953161e+00 1.0000000e+00 diff --git a/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_th_subsaturated.dat b/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_th_subsaturated.dat new file mode 100644 index 00000000..c32ec8a7 --- /dev/null +++ b/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_th_subsaturated.dat @@ -0,0 +1,51 @@ + -1.9936882e+00 3.4883174e-03 + -1.9139081e+00 7.0606097e-03 + -1.8341280e+00 1.1074610e-02 + -1.7543479e+00 1.5182662e-02 + -1.6745678e+00 1.9700512e-02 + -1.5947877e+00 2.4839776e-02 + -1.5150077e+00 3.0205772e-02 + -1.4352276e+00 3.6371209e-02 + -1.3554475e+00 4.2986752e-02 + -1.2756674e+00 5.0331197e-02 + -1.1958873e+00 5.8685020e-02 + -1.1161072e+00 6.8086850e-02 + -1.0363271e+00 7.8600508e-02 + -9.5654705e-01 9.0432572e-02 + -8.7676697e-01 1.0396093e-01 + -7.9698688e-01 1.1918054e-01 + -7.1720679e-01 1.3642730e-01 + -6.3742671e-01 1.5645196e-01 + -5.5764662e-01 1.7986920e-01 + -4.7786654e-01 2.0715768e-01 + -3.9808645e-01 2.3872049e-01 + -3.1830636e-01 2.7536546e-01 + -2.3852628e-01 3.1744192e-01 + -1.5874619e-01 3.6609362e-01 + -7.8966106e-02 4.2029942e-01 + 8.1397964e-04 4.7868213e-01 + 8.0594066e-02 5.3950011e-01 + 1.6037415e-01 5.9885190e-01 + 2.4015424e-01 6.5395455e-01 + 3.1993432e-01 7.0323606e-01 + 3.9971441e-01 7.4501525e-01 + 4.7949450e-01 7.8131088e-01 + 5.5927458e-01 8.1190294e-01 + 6.3905467e-01 8.3772690e-01 + 7.1883475e-01 8.5983413e-01 + 7.9861484e-01 8.7850679e-01 + 8.7839493e-01 8.9485335e-01 + 9.5817501e-01 9.0928695e-01 + 1.0379551e+00 9.2176562e-01 + 1.1177352e+00 9.3254128e-01 + 1.1975153e+00 9.4239322e-01 + 1.2772954e+00 9.5113333e-01 + 1.3570754e+00 9.5888589e-01 + 1.4368555e+00 9.6588436e-01 + 1.5166356e+00 9.7212369e-01 + 1.5964157e+00 9.7781047e-01 + 1.6761958e+00 9.8294974e-01 + 1.7559759e+00 9.8775478e-01 + 1.8357560e+00 9.9215171e-01 + 1.9155360e+00 9.9620098e-01 + 1.9953161e+00 1.0000000e+00 diff --git a/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_th_supersaturated.dat b/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_th_supersaturated.dat new file mode 100644 index 00000000..c821ea47 --- /dev/null +++ b/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_th_supersaturated.dat @@ -0,0 +1,51 @@ + -1.9936882e+00 9.2096607e-03 + -1.9139081e+00 1.4472324e-02 + -1.8341280e+00 1.9734987e-02 + -1.7543479e+00 2.5091627e-02 + -1.6745678e+00 3.1106099e-02 + -1.5947877e+00 3.8718166e-02 + -1.5150077e+00 4.5484447e-02 + -1.4352276e+00 5.3660370e-02 + -1.3554475e+00 6.0896532e-02 + -1.2756674e+00 6.9542336e-02 + -1.1958873e+00 7.9033925e-02 + -1.1161072e+00 9.1156846e-02 + -1.0363271e+00 1.0450146e-01 + -9.5654705e-01 1.2151114e-01 + -8.7676697e-01 1.4171600e-01 + -7.9698688e-01 1.6173292e-01 + -7.1720679e-01 1.8560286e-01 + -6.3742671e-01 2.0928484e-01 + -5.5764662e-01 2.3663190e-01 + -4.7786654e-01 2.6632835e-01 + -3.9808645e-01 3.0335495e-01 + -3.1830636e-01 3.4282492e-01 + -2.3852628e-01 3.8577201e-01 + -1.5874619e-01 4.2890706e-01 + -7.8966106e-02 4.7927826e-01 + 8.1397964e-04 5.2936754e-01 + 8.0594066e-02 5.7908091e-01 + 1.6037415e-01 6.2578705e-01 + 2.4015424e-01 6.6751245e-01 + 3.1993432e-01 7.0717038e-01 + 3.9971441e-01 7.4128371e-01 + 4.7949450e-01 7.7135608e-01 + 5.5927458e-01 7.9860915e-01 + 6.3905467e-01 8.2238511e-01 + 7.1883475e-01 8.4033456e-01 + 7.9861484e-01 8.5894183e-01 + 8.7839493e-01 8.7801898e-01 + 9.5817501e-01 8.9484071e-01 + 1.0379551e+00 9.1034677e-01 + 1.1177352e+00 9.2152993e-01 + 1.1975153e+00 9.3205526e-01 + 1.2772954e+00 9.4164082e-01 + 1.3570754e+00 9.4944084e-01 + 1.4368555e+00 9.5573724e-01 + 1.5166356e+00 9.6306738e-01 + 1.5964157e+00 9.6964571e-01 + 1.6761958e+00 9.7490837e-01 + 1.7559759e+00 9.8082887e-01 + 1.8357560e+00 9.8609153e-01 + 1.9155360e+00 9.9116624e-01 + 1.9953161e+00 1.0000000e+00 From 392867cc57dc2b77eab8a10e2987a0a384e6adc5 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Tue, 25 Apr 2023 12:48:10 +0200 Subject: [PATCH 123/129] add a readme to stretching parameters from Marta --- .../formulae/stretching_parameter_data/Waclawczyk_LES/README | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/README diff --git a/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/README b/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/README new file mode 100644 index 00000000..025d8efa --- /dev/null +++ b/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/README @@ -0,0 +1,2 @@ +Files contain CDF of th/rv from LES of a marine cumulus (RICO setup) done using UWLCM. +Stertching parameters calculated by Marta Waclawczyk, simulations done by P. Dziekan. From c9588aee99ac92788fdb68550100eee799c4c914 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Tue, 25 Apr 2023 15:17:04 +0200 Subject: [PATCH 124/129] stretchin parameters from LES data: formulae for th subsat --- .../formulae/fractal_reconstruction.hpp | 49 ++++++++++++- .../Waclawczyk_LES/README | 7 +- .../Waclawczyk_LES/cdf_th_subsaturated.dat | 51 ------------- .../Waclawczyk_LES/cdf_th_subsaturated.hpp | 73 +++++++++++++++++++ 4 files changed, 124 insertions(+), 56 deletions(-) delete mode 100644 libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_th_subsaturated.dat create mode 100644 libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_th_subsaturated.hpp diff --git a/libmpdata++/formulae/fractal_reconstruction.hpp b/libmpdata++/formulae/fractal_reconstruction.hpp index 78f6b104..918426db 100644 --- a/libmpdata++/formulae/fractal_reconstruction.hpp +++ b/libmpdata++/formulae/fractal_reconstruction.hpp @@ -7,6 +7,7 @@ #pragma once #include +#include namespace libmpdataxx { @@ -16,8 +17,10 @@ namespace libmpdataxx { namespace stretching_parameters { - // stretching parameters from a fit to the E. Akinlabi data (the data is in the stretching_parameter_data subdirectory) - namespace Akinlabi + // stretching parameters for velocity from DNS + // the function used is a fit to the E. Akinlabi data + // the data is in the stretching_parameter_data subdirectory + namespace DNS_vel { // CDF of stretching parameter d; assuming CDF = A d^B + C under condition that CDF(0.5)=0 and CDF(1)=1 template @@ -34,13 +37,51 @@ namespace libmpdataxx return pow((1.-pow(0.5,B))*CDF + pow(0.5,B), 1./B); } template - struct d_of_CDF_fctr + struct d_of_CDF_fctr_DNS { real_t operator()(const real_t &CDF) const { return CDF < 0 ? -d_of_CDF(-CDF) : d_of_CDF(CDF); } - BZ_DECLARE_FUNCTOR(d_of_CDF_fctr); + BZ_DECLARE_FUNCTOR(d_of_CDF_fctr_DNS); + }; + }; + // stretching parameters for potential temperature in a LES of marine cumulus + namespace LES_th_rv + { + // available distributions of the stretching parameter + // TODO: move to libmpdata++ opts? + enum class d_distro {rv_subsaturated, rv_supersaturated, th_subsaturated, th_supersaturated}; + + template + class d_of_CDF_fctr_LES + { + std::vector> d_CDF_vctr; + + public: + + d_of_CDF_fctr_LES(d_distro dd) + { + switch(dd) + { + case th_subsaturated: + d_CDF_th_subsaturated(d_CDF_vctr); + break; + default: + std::runtime_error("libmpdata++: invalid d_distro type"); + } + }; + + real_t operator()(const real_t &CDF) const + { + auto pos = std::lower_bound(d_CDF_vctr.begin(), d_CDF_vctr.end(), CDF, + [](const std::pair &pair, real_t val) + { + return pair.second < val; + }); + return pos->first; + } + BZ_DECLARE_FUNCTOR(d_of_CDF_fctr_LES); }; }; }; diff --git a/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/README b/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/README index 025d8efa..22867769 100644 --- a/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/README +++ b/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/README @@ -1,2 +1,7 @@ -Files contain CDF of th/rv from LES of a marine cumulus (RICO setup) done using UWLCM. +Files contain CDF of th/rv from LES of a marine cumulus done using UWLCM. Stertching parameters calculated by Marta Waclawczyk, simulations done by P. Dziekan. + +setup: RICO +resolution: 50m x 50m x 20m +height at which parameters are calculated: near cloud base, 800m (?) + diff --git a/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_th_subsaturated.dat b/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_th_subsaturated.dat deleted file mode 100644 index c32ec8a7..00000000 --- a/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_th_subsaturated.dat +++ /dev/null @@ -1,51 +0,0 @@ - -1.9936882e+00 3.4883174e-03 - -1.9139081e+00 7.0606097e-03 - -1.8341280e+00 1.1074610e-02 - -1.7543479e+00 1.5182662e-02 - -1.6745678e+00 1.9700512e-02 - -1.5947877e+00 2.4839776e-02 - -1.5150077e+00 3.0205772e-02 - -1.4352276e+00 3.6371209e-02 - -1.3554475e+00 4.2986752e-02 - -1.2756674e+00 5.0331197e-02 - -1.1958873e+00 5.8685020e-02 - -1.1161072e+00 6.8086850e-02 - -1.0363271e+00 7.8600508e-02 - -9.5654705e-01 9.0432572e-02 - -8.7676697e-01 1.0396093e-01 - -7.9698688e-01 1.1918054e-01 - -7.1720679e-01 1.3642730e-01 - -6.3742671e-01 1.5645196e-01 - -5.5764662e-01 1.7986920e-01 - -4.7786654e-01 2.0715768e-01 - -3.9808645e-01 2.3872049e-01 - -3.1830636e-01 2.7536546e-01 - -2.3852628e-01 3.1744192e-01 - -1.5874619e-01 3.6609362e-01 - -7.8966106e-02 4.2029942e-01 - 8.1397964e-04 4.7868213e-01 - 8.0594066e-02 5.3950011e-01 - 1.6037415e-01 5.9885190e-01 - 2.4015424e-01 6.5395455e-01 - 3.1993432e-01 7.0323606e-01 - 3.9971441e-01 7.4501525e-01 - 4.7949450e-01 7.8131088e-01 - 5.5927458e-01 8.1190294e-01 - 6.3905467e-01 8.3772690e-01 - 7.1883475e-01 8.5983413e-01 - 7.9861484e-01 8.7850679e-01 - 8.7839493e-01 8.9485335e-01 - 9.5817501e-01 9.0928695e-01 - 1.0379551e+00 9.2176562e-01 - 1.1177352e+00 9.3254128e-01 - 1.1975153e+00 9.4239322e-01 - 1.2772954e+00 9.5113333e-01 - 1.3570754e+00 9.5888589e-01 - 1.4368555e+00 9.6588436e-01 - 1.5166356e+00 9.7212369e-01 - 1.5964157e+00 9.7781047e-01 - 1.6761958e+00 9.8294974e-01 - 1.7559759e+00 9.8775478e-01 - 1.8357560e+00 9.9215171e-01 - 1.9155360e+00 9.9620098e-01 - 1.9953161e+00 1.0000000e+00 diff --git a/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_th_subsaturated.hpp b/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_th_subsaturated.hpp new file mode 100644 index 00000000..5e5b5536 --- /dev/null +++ b/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_th_subsaturated.hpp @@ -0,0 +1,73 @@ +#pragma once + +namespace libmpdataxx +{ + namespace formulae + { + namespace fractal + { + namespace stretching_parameters + { + namespace LES_th_rv + { + template + void d_CDF_th_subsaturated(std::vector> &vec) + vec = + { + { -1.9936882e+00, 3.4883174e-03}, + { -1.9139081e+00, 7.0606097e-03}, + { -1.8341280e+00, 1.1074610e-02}, + { -1.7543479e+00, 1.5182662e-02}, + { -1.6745678e+00, 1.9700512e-02}, + { -1.5947877e+00, 2.4839776e-02}, + { -1.5150077e+00, 3.0205772e-02}, + { -1.4352276e+00, 3.6371209e-02}, + { -1.3554475e+00, 4.2986752e-02}, + { -1.2756674e+00, 5.0331197e-02}, + { -1.1958873e+00, 5.8685020e-02}, + { -1.1161072e+00, 6.8086850e-02}, + { -1.0363271e+00, 7.8600508e-02}, + { -9.5654705e-01, 9.0432572e-02}, + { -8.7676697e-01, 1.0396093e-01}, + { -7.9698688e-01, 1.1918054e-01}, + { -7.1720679e-01, 1.3642730e-01}, + { -6.3742671e-01, 1.5645196e-01}, + { -5.5764662e-01, 1.7986920e-01}, + { -4.7786654e-01, 2.0715768e-01}, + { -3.9808645e-01, 2.3872049e-01}, + { -3.1830636e-01, 2.7536546e-01}, + { -2.3852628e-01, 3.1744192e-01}, + { -1.5874619e-01, 3.6609362e-01}, + { -7.8966106e-02, 4.2029942e-01}, + { 8.1397964e-04, 4.7868213e-01}, + { 8.0594066e-02, 5.3950011e-01}, + { 1.6037415e-01, 5.9885190e-01}, + { 2.4015424e-01, 6.5395455e-01}, + { 3.1993432e-01, 7.0323606e-01}, + { 3.9971441e-01, 7.4501525e-01}, + { 4.7949450e-01, 7.8131088e-01}, + { 5.5927458e-01, 8.1190294e-01}, + { 6.3905467e-01, 8.3772690e-01}, + { 7.1883475e-01, 8.5983413e-01}, + { 7.9861484e-01, 8.7850679e-01}, + { 8.7839493e-01, 8.9485335e-01}, + { 9.5817501e-01, 9.0928695e-01}, + { 1.0379551e+00, 9.2176562e-01}, + { 1.1177352e+00, 9.3254128e-01}, + { 1.1975153e+00, 9.4239322e-01}, + { 1.2772954e+00, 9.5113333e-01}, + { 1.3570754e+00, 9.5888589e-01}, + { 1.4368555e+00, 9.6588436e-01}, + { 1.5166356e+00, 9.7212369e-01}, + { 1.5964157e+00, 9.7781047e-01}, + { 1.6761958e+00, 9.8294974e-01}, + { 1.7559759e+00, 9.8775478e-01}, + { 1.8357560e+00, 9.9215171e-01}, + { 1.9155360e+00, 9.9620098e-01}, + { 1.9953161e+00, 1.0000000e+00} + } + }; + }; + }; + }; +}; From b8deacd2a1317aded6450025e031dd723ab39827 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Tue, 25 Apr 2023 15:27:46 +0200 Subject: [PATCH 125/129] formulae: move stretchng parameters to separate file --- .../formulae/fractal_reconstruction.hpp | 71 --------------- .../formulae/stretching_parameters.hpp | 89 +++++++++++++++++++ 2 files changed, 89 insertions(+), 71 deletions(-) create mode 100644 libmpdata++/formulae/stretching_parameters.hpp diff --git a/libmpdata++/formulae/fractal_reconstruction.hpp b/libmpdata++/formulae/fractal_reconstruction.hpp index 918426db..427b1f18 100644 --- a/libmpdata++/formulae/fractal_reconstruction.hpp +++ b/libmpdata++/formulae/fractal_reconstruction.hpp @@ -15,77 +15,6 @@ namespace libmpdataxx { namespace fractal { - namespace stretching_parameters - { - // stretching parameters for velocity from DNS - // the function used is a fit to the E. Akinlabi data - // the data is in the stretching_parameter_data subdirectory - namespace DNS_vel - { - // CDF of stretching parameter d; assuming CDF = A d^B + C under condition that CDF(0.5)=0 and CDF(1)=1 - template - static real_t CDF_of_d(const real_t &d) - { - const real_t B = -0.39091161; - return (pow(d,B) - pow(0.5,B)) / (1. - pow(0.5,B)); - } - // inverse function: d of a CDF - template - static real_t d_of_CDF(const real_t &CDF) - { - const real_t B = -0.39091161; - return pow((1.-pow(0.5,B))*CDF + pow(0.5,B), 1./B); - } - template - struct d_of_CDF_fctr_DNS - { - real_t operator()(const real_t &CDF) const - { - return CDF < 0 ? -d_of_CDF(-CDF) : d_of_CDF(CDF); - } - BZ_DECLARE_FUNCTOR(d_of_CDF_fctr_DNS); - }; - }; - // stretching parameters for potential temperature in a LES of marine cumulus - namespace LES_th_rv - { - // available distributions of the stretching parameter - // TODO: move to libmpdata++ opts? - enum class d_distro {rv_subsaturated, rv_supersaturated, th_subsaturated, th_supersaturated}; - - template - class d_of_CDF_fctr_LES - { - std::vector> d_CDF_vctr; - - public: - - d_of_CDF_fctr_LES(d_distro dd) - { - switch(dd) - { - case th_subsaturated: - d_CDF_th_subsaturated(d_CDF_vctr); - break; - default: - std::runtime_error("libmpdata++: invalid d_distro type"); - } - }; - - real_t operator()(const real_t &CDF) const - { - auto pos = std::lower_bound(d_CDF_vctr.begin(), d_CDF_vctr.end(), CDF, - [](const std::pair &pair, real_t val) - { - return pair.second < val; - }); - return pos->first; - } - BZ_DECLARE_FUNCTOR(d_of_CDF_fctr_LES); - }; - }; - }; - // (tri-)linear interpolation // 3d version template diff --git a/libmpdata++/formulae/stretching_parameters.hpp b/libmpdata++/formulae/stretching_parameters.hpp new file mode 100644 index 00000000..6408fc67 --- /dev/null +++ b/libmpdata++/formulae/stretching_parameters.hpp @@ -0,0 +1,89 @@ +/** @file +* @copyright University of Warsaw +* @section LICENSE +* GPLv3+ (see the COPYING file or http://www.gnu.org/licenses/) +*/ + +#pragma once + +#include + +namespace libmpdataxx +{ + namespace formulae + { + namespace fractal + { + namespace stretching_parameters + { + // stretching parameters for velocity from DNS + // the function used is a fit to the E. Akinlabi data + // the data is in the stretching_parameter_data subdirectory + namespace DNS_vel + { + // CDF of stretching parameter d; assuming CDF = A d^B + C under condition that CDF(0.5)=0 and CDF(1)=1 + template + static real_t CDF_of_d(const real_t &d) + { + const real_t B = -0.39091161; + return (pow(d,B) - pow(0.5,B)) / (1. - pow(0.5,B)); + } + // inverse function: d of a CDF + template + static real_t d_of_CDF(const real_t &CDF) + { + const real_t B = -0.39091161; + return pow((1.-pow(0.5,B))*CDF + pow(0.5,B), 1./B); + } + template + struct d_of_CDF_fctr_DNS + { + real_t operator()(const real_t &CDF) const + { + return CDF < 0 ? -d_of_CDF(-CDF) : d_of_CDF(CDF); + } + BZ_DECLARE_FUNCTOR(d_of_CDF_fctr_DNS); + }; + }; + // stretching parameters for potential temperature in a LES of marine cumulus + namespace LES_th_rv + { + // available distributions of the stretching parameter + // TODO: move to libmpdata++ opts? + enum class d_distro {rv_subsaturated, rv_supersaturated, th_subsaturated, th_supersaturated}; + + template + class d_of_CDF_fctr_LES + { + std::vector> d_CDF_vctr; + + public: + + d_of_CDF_fctr_LES(d_distro dd) + { + switch(dd) + { + case th_subsaturated: + d_CDF_th_subsaturated(d_CDF_vctr); + break; + default: + std::runtime_error("libmpdata++: invalid d_distro type"); + } + }; + + real_t operator()(const real_t &CDF) const + { + auto pos = std::lower_bound(d_CDF_vctr.begin(), d_CDF_vctr.end(), CDF, + [](const std::pair &pair, real_t val) + { + return pair.second < val; + }); + return pos->first; + } + BZ_DECLARE_FUNCTOR(d_of_CDF_fctr_LES); + }; + }; + }; + }; + }; +}; From d7ca208db84ca2b2ae9bda2acbec45cac6c94847 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Tue, 25 Apr 2023 17:03:13 +0200 Subject: [PATCH 126/129] other stretching parameters WIP: check distribution of values, seems to bo a lot of extremes --- .../Waclawczyk_LES/cdf_th_subsaturated.hpp | 108 +++++++++--------- .../formulae/stretching_parameters.hpp | 21 ++-- libmpdata++/output/hdf5.hpp | 4 +- .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 24 +++- .../mpdata_rhs_vip_prs_sgs_fra_common.hpp | 11 +- tests/sandbox/pbl/pbl.hpp | 2 +- 6 files changed, 102 insertions(+), 68 deletions(-) diff --git a/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_th_subsaturated.hpp b/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_th_subsaturated.hpp index 5e5b5536..6b60283f 100644 --- a/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_th_subsaturated.hpp +++ b/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_th_subsaturated.hpp @@ -6,65 +6,67 @@ namespace libmpdataxx { namespace fractal { - namespace stretching_parameters + namespace stretch_params { namespace LES_th_rv { template void d_CDF_th_subsaturated(std::vector> &vec) - vec = { - { -1.9936882e+00, 3.4883174e-03}, - { -1.9139081e+00, 7.0606097e-03}, - { -1.8341280e+00, 1.1074610e-02}, - { -1.7543479e+00, 1.5182662e-02}, - { -1.6745678e+00, 1.9700512e-02}, - { -1.5947877e+00, 2.4839776e-02}, - { -1.5150077e+00, 3.0205772e-02}, - { -1.4352276e+00, 3.6371209e-02}, - { -1.3554475e+00, 4.2986752e-02}, - { -1.2756674e+00, 5.0331197e-02}, - { -1.1958873e+00, 5.8685020e-02}, - { -1.1161072e+00, 6.8086850e-02}, - { -1.0363271e+00, 7.8600508e-02}, - { -9.5654705e-01, 9.0432572e-02}, - { -8.7676697e-01, 1.0396093e-01}, - { -7.9698688e-01, 1.1918054e-01}, - { -7.1720679e-01, 1.3642730e-01}, - { -6.3742671e-01, 1.5645196e-01}, - { -5.5764662e-01, 1.7986920e-01}, - { -4.7786654e-01, 2.0715768e-01}, - { -3.9808645e-01, 2.3872049e-01}, - { -3.1830636e-01, 2.7536546e-01}, - { -2.3852628e-01, 3.1744192e-01}, - { -1.5874619e-01, 3.6609362e-01}, - { -7.8966106e-02, 4.2029942e-01}, - { 8.1397964e-04, 4.7868213e-01}, - { 8.0594066e-02, 5.3950011e-01}, - { 1.6037415e-01, 5.9885190e-01}, - { 2.4015424e-01, 6.5395455e-01}, - { 3.1993432e-01, 7.0323606e-01}, - { 3.9971441e-01, 7.4501525e-01}, - { 4.7949450e-01, 7.8131088e-01}, - { 5.5927458e-01, 8.1190294e-01}, - { 6.3905467e-01, 8.3772690e-01}, - { 7.1883475e-01, 8.5983413e-01}, - { 7.9861484e-01, 8.7850679e-01}, - { 8.7839493e-01, 8.9485335e-01}, - { 9.5817501e-01, 9.0928695e-01}, - { 1.0379551e+00, 9.2176562e-01}, - { 1.1177352e+00, 9.3254128e-01}, - { 1.1975153e+00, 9.4239322e-01}, - { 1.2772954e+00, 9.5113333e-01}, - { 1.3570754e+00, 9.5888589e-01}, - { 1.4368555e+00, 9.6588436e-01}, - { 1.5166356e+00, 9.7212369e-01}, - { 1.5964157e+00, 9.7781047e-01}, - { 1.6761958e+00, 9.8294974e-01}, - { 1.7559759e+00, 9.8775478e-01}, - { 1.8357560e+00, 9.9215171e-01}, - { 1.9155360e+00, 9.9620098e-01}, - { 1.9953161e+00, 1.0000000e+00} + vec = + { + { -1.9936882e+00, 3.4883174e-03}, + { -1.9139081e+00, 7.0606097e-03}, + { -1.8341280e+00, 1.1074610e-02}, + { -1.7543479e+00, 1.5182662e-02}, + { -1.6745678e+00, 1.9700512e-02}, + { -1.5947877e+00, 2.4839776e-02}, + { -1.5150077e+00, 3.0205772e-02}, + { -1.4352276e+00, 3.6371209e-02}, + { -1.3554475e+00, 4.2986752e-02}, + { -1.2756674e+00, 5.0331197e-02}, + { -1.1958873e+00, 5.8685020e-02}, + { -1.1161072e+00, 6.8086850e-02}, + { -1.0363271e+00, 7.8600508e-02}, + { -9.5654705e-01, 9.0432572e-02}, + { -8.7676697e-01, 1.0396093e-01}, + { -7.9698688e-01, 1.1918054e-01}, + { -7.1720679e-01, 1.3642730e-01}, + { -6.3742671e-01, 1.5645196e-01}, + { -5.5764662e-01, 1.7986920e-01}, + { -4.7786654e-01, 2.0715768e-01}, + { -3.9808645e-01, 2.3872049e-01}, + { -3.1830636e-01, 2.7536546e-01}, + { -2.3852628e-01, 3.1744192e-01}, + { -1.5874619e-01, 3.6609362e-01}, + { -7.8966106e-02, 4.2029942e-01}, + { 8.1397964e-04, 4.7868213e-01}, + { 8.0594066e-02, 5.3950011e-01}, + { 1.6037415e-01, 5.9885190e-01}, + { 2.4015424e-01, 6.5395455e-01}, + { 3.1993432e-01, 7.0323606e-01}, + { 3.9971441e-01, 7.4501525e-01}, + { 4.7949450e-01, 7.8131088e-01}, + { 5.5927458e-01, 8.1190294e-01}, + { 6.3905467e-01, 8.3772690e-01}, + { 7.1883475e-01, 8.5983413e-01}, + { 7.9861484e-01, 8.7850679e-01}, + { 8.7839493e-01, 8.9485335e-01}, + { 9.5817501e-01, 9.0928695e-01}, + { 1.0379551e+00, 9.2176562e-01}, + { 1.1177352e+00, 9.3254128e-01}, + { 1.1975153e+00, 9.4239322e-01}, + { 1.2772954e+00, 9.5113333e-01}, + { 1.3570754e+00, 9.5888589e-01}, + { 1.4368555e+00, 9.6588436e-01}, + { 1.5166356e+00, 9.7212369e-01}, + { 1.5964157e+00, 9.7781047e-01}, + { 1.6761958e+00, 9.8294974e-01}, + { 1.7559759e+00, 9.8775478e-01}, + { 1.8357560e+00, 9.9215171e-01}, + { 1.9155360e+00, 9.9620098e-01}, + { 1.9953161e+00, 1.0000000e+00} + }; } }; }; diff --git a/libmpdata++/formulae/stretching_parameters.hpp b/libmpdata++/formulae/stretching_parameters.hpp index 6408fc67..39f0e92c 100644 --- a/libmpdata++/formulae/stretching_parameters.hpp +++ b/libmpdata++/formulae/stretching_parameters.hpp @@ -14,8 +14,17 @@ namespace libmpdataxx { namespace fractal { - namespace stretching_parameters + namespace stretch_params { + // available distributions of the stretching parameter + enum class d_distro_t { + DNS_vel, + LES_rv_subsaturated, + LES_rv_supersaturated, + LES_th_subsaturated, + LES_th_supersaturated + }; + // stretching parameters for velocity from DNS // the function used is a fit to the E. Akinlabi data // the data is in the stretching_parameter_data subdirectory @@ -48,10 +57,6 @@ namespace libmpdataxx // stretching parameters for potential temperature in a LES of marine cumulus namespace LES_th_rv { - // available distributions of the stretching parameter - // TODO: move to libmpdata++ opts? - enum class d_distro {rv_subsaturated, rv_supersaturated, th_subsaturated, th_supersaturated}; - template class d_of_CDF_fctr_LES { @@ -59,15 +64,15 @@ namespace libmpdataxx public: - d_of_CDF_fctr_LES(d_distro dd) + d_of_CDF_fctr_LES(d_distro_t dd) { switch(dd) { - case th_subsaturated: + case d_distro_t::LES_th_subsaturated: d_CDF_th_subsaturated(d_CDF_vctr); break; default: - std::runtime_error("libmpdata++: invalid d_distro type"); + std::runtime_error("libmpdata++: invalid d_distro_t type in the constructor of d_of_CDF_fctr_LES"); } }; diff --git a/libmpdata++/output/hdf5.hpp b/libmpdata++/output/hdf5.hpp index cd1ef331..7d4a6674 100644 --- a/libmpdata++/output/hdf5.hpp +++ b/libmpdata++/output/hdf5.hpp @@ -789,7 +789,7 @@ namespace libmpdataxx } { auto dset = group.createDataSet("tht_e", flttype_output, sspace); - record_dsc_helper(dset, this->tht_e); + record_dsc_hlpr(dset, this->tht_e); } } @@ -804,7 +804,7 @@ namespace libmpdataxx } { auto dset = group.createDataSet("mix_len", flttype_output, sspace); - record_dsc_helper(dset, this->mix_len); + record_dsc_hlpr(dset, this->mix_len); } } diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 5edf62d1..230607f0 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -15,6 +15,8 @@ namespace libmpdataxx { namespace detail { + using formulae::fractal::stretch_params::d_distro_t; + template class mpdata_rhs_vip_prs_sgs_fra_dim { @@ -199,14 +201,30 @@ namespace libmpdataxx // TODO: stretching parameters at the overlaps of the reconstructed and resolved arrays (ijk_r2r) are not used, do not generate them // also not all parameters in the halo are needed (but some are!) - void generate_stretching_parameters(const typename gen_t::result_type rng_seed) + void generate_stretching_parameters(const typename gen_t::result_type rng_seed, const d_distro_t &dd = d_distro_t::DNS_vel) { gen_t gen(rng_seed); - std::uniform_real_distribution<> dis(-1, 1); // [-1,1), but whatever + std::uniform_real_distribution<> dis(dd == d_distro_t::DNS_vel ? -1 : 0, 1); // [min,max) instead of [min,max], but whatever auto rand = std::bind(dis, gen); std::generate(this->d_j(this->ijk_ref_with_halo).begin(), this->d_j(this->ijk_ref_with_halo).end(), rand); - this->d_j(this->ijk_ref_with_halo) = formulae::fractal::d_of_CDF_fctr{}(this->d_j(this->ijk_ref_with_halo)); + + // different stretchnig parameter distributions + switch(dd) + { + case d_distro_t::DNS_vel: + using formulae::fractal::stretch_params::DNS_vel::d_of_CDF_fctr_DNS; + this->d_j(this->ijk_ref_with_halo) = d_of_CDF_fctr_DNS{}(this->d_j(this->ijk_ref_with_halo)); + break; + case d_distro_t::LES_th_subsaturated: + this->d_j(this->ijk_ref_with_halo) = this->d_of_CDF_fctr_LES_th_subsaturated(this->d_j(this->ijk_ref_with_halo)); + break; + default: + std::runtime_error("libmpdata++: invalid d_distro_t type in generate_stretching_parameters"); + } + xchng_ref(this->d_j, this->ijk_ref); // xchng to have the same values of d_j in the distmem halo region + + std::cerr << "generated stretching params: " << this->d_j(this->ijk_ref_with_halo) << std::endl; } // calculate refined points using (tri?)linear interpolation diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp index 7652e700..65c9f49d 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp @@ -12,6 +12,8 @@ #include #include +#include + //#include //#include #include @@ -35,8 +37,14 @@ namespace libmpdataxx using bcp_ref_t = std::unique_ptr>; using bcs_ref_t = std::array, ct_params_t::n_dims>; + protected: + // types of stretching parameters that need to be instantiated at the beginning od the simulation + // all are instantiated even not or all used, but they do not use much memory + formulae::fractal::stretch_params::LES_th_rv::d_of_CDF_fctr_LES + d_of_CDF_fctr_LES_th_subsaturated; + typename parent_t::arr_t c_j, d_j, f_j; // parameters used in fractal reconstruction, Akinlabi et al. 2019 @@ -182,7 +190,8 @@ namespace libmpdataxx {this->ijk[0].first() * n_ref, this->ijk[1].first() * n_ref, this->ijk[2].first() * n_ref}, // lbound {this->ijk[0].last() * n_ref, this->ijk[1].last() * n_ref, this->ijk[2].last() * n_ref}, // ubound {n_ref, n_ref, n_ref}, // stride - } + }, + d_of_CDF_fctr_LES_th_subsaturated(formulae::fractal::stretch_params::d_distro_t::LES_th_subsaturated) { assert(p.n_fra_iter > 0); assert(n_ref % 2 == 0); diff --git a/tests/sandbox/pbl/pbl.hpp b/tests/sandbox/pbl/pbl.hpp index 9ea0b107..d9f0819c 100644 --- a/tests/sandbox/pbl/pbl.hpp +++ b/tests/sandbox/pbl/pbl.hpp @@ -65,7 +65,7 @@ class pbl : public libmpdataxx::output::hdf5_xdmfgenerate_stretching_parameters(std::random_device{}()); //this->reconstruct_refinee(ix::w); - this->generate_stretching_parameters(std::random_device{}()); + this->generate_stretching_parameters(std::random_device{}(), libmpdataxx::formulae::fractal::stretch_params::d_distro_t::LES_th_subsaturated); this->reconstruct_refinee(ix::tht); this->mem->barrier(); From c5be8a38f71e3c39eac2931e28226cb8684781ae Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Wed, 26 Apr 2023 11:13:37 +0200 Subject: [PATCH 127/129] Marta's stretch params: dat->hpp --- .../Waclawczyk_LES/cdf_rv_subsaturated.dat | 51 --------- .../Waclawczyk_LES/cdf_rv_subsaturated.hpp | 75 +++++++++++++ .../Waclawczyk_LES/cdf_rv_supersaturated.dat | 51 --------- .../Waclawczyk_LES/cdf_rv_supersaturated.hpp | 75 +++++++++++++ .../Waclawczyk_LES/cdf_th_subsaturated.hpp | 102 +++++++++--------- .../Waclawczyk_LES/cdf_th_supersaturated.dat | 51 --------- .../Waclawczyk_LES/cdf_th_supersaturated.hpp | 75 +++++++++++++ 7 files changed, 276 insertions(+), 204 deletions(-) delete mode 100644 libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_rv_subsaturated.dat create mode 100644 libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_rv_subsaturated.hpp delete mode 100644 libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_rv_supersaturated.dat create mode 100644 libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_rv_supersaturated.hpp delete mode 100644 libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_th_supersaturated.dat create mode 100644 libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_th_supersaturated.hpp diff --git a/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_rv_subsaturated.dat b/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_rv_subsaturated.dat deleted file mode 100644 index db0c93c1..00000000 --- a/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_rv_subsaturated.dat +++ /dev/null @@ -1,51 +0,0 @@ - -1.9936882e+00 3.6607750e-03 - -1.9139081e+00 7.4978095e-03 - -1.8341280e+00 1.1724649e-02 - -1.7543479e+00 1.6198929e-02 - -1.6745678e+00 2.1068099e-02 - -1.5947877e+00 2.6342327e-02 - -1.5150077e+00 3.2182618e-02 - -1.4352276e+00 3.8644903e-02 - -1.3554475e+00 4.5646135e-02 - -1.2756674e+00 5.3330373e-02 - -1.1958873e+00 6.2131486e-02 - -1.1161072e+00 7.1863046e-02 - -1.0363271e+00 8.2509800e-02 - -9.5654705e-01 9.4719163e-02 - -8.7676697e-01 1.0841317e-01 - -7.9698688e-01 1.2429178e-01 - -7.1720679e-01 1.4261769e-01 - -6.3742671e-01 1.6377392e-01 - -5.5764662e-01 1.8794012e-01 - -4.7786654e-01 2.1629587e-01 - -3.9808645e-01 2.4917505e-01 - -3.1830636e-01 2.8725220e-01 - -2.3852628e-01 3.3044256e-01 - -1.5874619e-01 3.7904952e-01 - -7.8966106e-02 4.3228329e-01 - 8.1397964e-04 4.8916258e-01 - 8.0594066e-02 5.4704858e-01 - 1.6037415e-01 6.0301438e-01 - 2.4015424e-01 6.5454148e-01 - 3.1993432e-01 7.0126212e-01 - 3.9971441e-01 7.4248312e-01 - 4.7949450e-01 7.7761131e-01 - 5.5927458e-01 8.0739748e-01 - 6.3905467e-01 8.3276360e-01 - 7.1883475e-01 8.5513500e-01 - 7.9861484e-01 8.7462354e-01 - 8.7839493e-01 8.9117499e-01 - 9.5817501e-01 9.0566556e-01 - 1.0379551e+00 9.1841726e-01 - 1.1177352e+00 9.2990633e-01 - 1.1975153e+00 9.3971246e-01 - 1.2772954e+00 9.4867797e-01 - 1.3570754e+00 9.5673676e-01 - 1.4368555e+00 9.6387696e-01 - 1.5166356e+00 9.7051212e-01 - 1.5964157e+00 9.7658799e-01 - 1.6761958e+00 9.8208254e-01 - 1.7559759e+00 9.8706187e-01 - 1.8357560e+00 9.9180224e-01 - 1.9155360e+00 9.9600704e-01 - 1.9953161e+00 1.0000000e+00 diff --git a/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_rv_subsaturated.hpp b/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_rv_subsaturated.hpp new file mode 100644 index 00000000..cddf5562 --- /dev/null +++ b/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_rv_subsaturated.hpp @@ -0,0 +1,75 @@ +#pragma once + +namespace libmpdataxx +{ + namespace formulae + { + namespace fractal + { + namespace stretch_params + { + namespace LES_th_rv + { + template + void d_CDF_rv_subsaturated(std::vector> &vec) + { + vec = + { + { -1.9936882e+00, 3.6607750e-03}, + { -1.9139081e+00, 7.4978095e-03}, + { -1.8341280e+00, 1.1724649e-02}, + { -1.7543479e+00, 1.6198929e-02}, + { -1.6745678e+00, 2.1068099e-02}, + { -1.5947877e+00, 2.6342327e-02}, + { -1.5150077e+00, 3.2182618e-02}, + { -1.4352276e+00, 3.8644903e-02}, + { -1.3554475e+00, 4.5646135e-02}, + { -1.2756674e+00, 5.3330373e-02}, + { -1.1958873e+00, 6.2131486e-02}, + { -1.1161072e+00, 7.1863046e-02}, + { -1.0363271e+00, 8.2509800e-02}, + { -9.5654705e-01, 9.4719163e-02}, + { -8.7676697e-01, 1.0841317e-01}, + { -7.9698688e-01, 1.2429178e-01}, + { -7.1720679e-01, 1.4261769e-01}, + { -6.3742671e-01, 1.6377392e-01}, + { -5.5764662e-01, 1.8794012e-01}, + { -4.7786654e-01, 2.1629587e-01}, + { -3.9808645e-01, 2.4917505e-01}, + { -3.1830636e-01, 2.8725220e-01}, + { -2.3852628e-01, 3.3044256e-01}, + { -1.5874619e-01, 3.7904952e-01}, + { -7.8966106e-02, 4.3228329e-01}, + { 8.1397964e-04, 4.8916258e-01}, + { 8.0594066e-02, 5.4704858e-01}, + { 1.6037415e-01, 6.0301438e-01}, + { 2.4015424e-01, 6.5454148e-01}, + { 3.1993432e-01, 7.0126212e-01}, + { 3.9971441e-01, 7.4248312e-01}, + { 4.7949450e-01, 7.7761131e-01}, + { 5.5927458e-01, 8.0739748e-01}, + { 6.3905467e-01, 8.3276360e-01}, + { 7.1883475e-01, 8.5513500e-01}, + { 7.9861484e-01, 8.7462354e-01}, + { 8.7839493e-01, 8.9117499e-01}, + { 9.5817501e-01, 9.0566556e-01}, + { 1.0379551e+00, 9.1841726e-01}, + { 1.1177352e+00, 9.2990633e-01}, + { 1.1975153e+00, 9.3971246e-01}, + { 1.2772954e+00, 9.4867797e-01}, + { 1.3570754e+00, 9.5673676e-01}, + { 1.4368555e+00, 9.6387696e-01}, + { 1.5166356e+00, 9.7051212e-01}, + { 1.5964157e+00, 9.7658799e-01}, + { 1.6761958e+00, 9.8208254e-01}, + { 1.7559759e+00, 9.8706187e-01}, + { 1.8357560e+00, 9.9180224e-01}, + { 1.9155360e+00, 9.9600704e-01}, + { 1.9953161e+00, 1.0000000e+00} + }; + } + }; + }; + }; + }; +}; diff --git a/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_rv_supersaturated.dat b/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_rv_supersaturated.dat deleted file mode 100644 index 567c9047..00000000 --- a/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_rv_supersaturated.dat +++ /dev/null @@ -1,51 +0,0 @@ - -1.9936882e+00 4.8301652e-03 - -1.9139081e+00 5.9987535e-03 - -1.8341280e+00 7.5568713e-03 - -1.7543479e+00 9.1928950e-03 - -1.6745678e+00 1.0751013e-02 - -1.5947877e+00 1.2153319e-02 - -1.5150077e+00 1.4568401e-02 - -1.4352276e+00 1.7450919e-02 - -1.3554475e+00 2.0956684e-02 - -1.2756674e+00 2.5397320e-02 - -1.1958873e+00 2.9760050e-02 - -1.1161072e+00 3.4278591e-02 - -1.0363271e+00 4.1290122e-02 - -9.5654705e-01 4.7366781e-02 - -8.7676697e-01 5.6793394e-02 - -7.9698688e-01 7.0894360e-02 - -7.1720679e-01 8.6553443e-02 - -6.3742671e-01 1.0330321e-01 - -5.5764662e-01 1.2394827e-01 - -4.7786654e-01 1.5207230e-01 - -3.9808645e-01 1.8518230e-01 - -3.1830636e-01 2.2351200e-01 - -2.3852628e-01 2.7648800e-01 - -1.5874619e-01 3.3476161e-01 - -7.8966106e-02 4.0986289e-01 - 8.1397964e-04 4.9353381e-01 - 8.0594066e-02 5.7517918e-01 - 1.6037415e-01 6.5269554e-01 - 2.4015424e-01 7.2210969e-01 - 3.1993432e-01 7.7827984e-01 - 3.9971441e-01 8.2003739e-01 - 4.7949450e-01 8.5454970e-01 - 5.5927458e-01 8.8205048e-01 - 6.3905467e-01 9.0511063e-01 - 7.1883475e-01 9.2178249e-01 - 7.9861484e-01 9.3487068e-01 - 8.7839493e-01 9.4476472e-01 - 9.5817501e-01 9.5255531e-01 - 1.0379551e+00 9.6057962e-01 - 1.1177352e+00 9.6727953e-01 - 1.1975153e+00 9.7327828e-01 - 1.2772954e+00 9.7701776e-01 - 1.3570754e+00 9.8028981e-01 - 1.4368555e+00 9.8247117e-01 - 1.5166356e+00 9.8480835e-01 - 1.5964157e+00 9.8667809e-01 - 1.6761958e+00 9.8924899e-01 - 1.7559759e+00 9.9150826e-01 - 1.8357560e+00 9.9298847e-01 - 1.9155360e+00 9.9478031e-01 - 1.9953161e+00 1.0000000e+00 diff --git a/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_rv_supersaturated.hpp b/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_rv_supersaturated.hpp new file mode 100644 index 00000000..dd46886d --- /dev/null +++ b/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_rv_supersaturated.hpp @@ -0,0 +1,75 @@ +#pragma once + +namespace libmpdataxx +{ + namespace formulae + { + namespace fractal + { + namespace stretch_params + { + namespace LES_th_rv + { + template + void d_CDF_rv_supersaturated(std::vector> &vec) + { + vec = + { + {-1.9936882e+00, 4.8301652e-03}, + {-1.9139081e+00, 5.9987535e-03}, + {-1.8341280e+00, 7.5568713e-03}, + {-1.7543479e+00, 9.1928950e-03}, + {-1.6745678e+00, 1.0751013e-02}, + {-1.5947877e+00, 1.2153319e-02}, + {-1.5150077e+00, 1.4568401e-02}, + {-1.4352276e+00, 1.7450919e-02}, + {-1.3554475e+00, 2.0956684e-02}, + {-1.2756674e+00, 2.5397320e-02}, + {-1.1958873e+00, 2.9760050e-02}, + {-1.1161072e+00, 3.4278591e-02}, + {-1.0363271e+00, 4.1290122e-02}, + {-9.5654705e-01, 4.7366781e-02}, + {-8.7676697e-01, 5.6793394e-02}, + {-7.9698688e-01, 7.0894360e-02}, + {-7.1720679e-01, 8.6553443e-02}, + {-6.3742671e-01, 1.0330321e-01}, + {-5.5764662e-01, 1.2394827e-01}, + {-4.7786654e-01, 1.5207230e-01}, + {-3.9808645e-01, 1.8518230e-01}, + {-3.1830636e-01, 2.2351200e-01}, + {-2.3852628e-01, 2.7648800e-01}, + {-1.5874619e-01, 3.3476161e-01}, + {-7.8966106e-02, 4.0986289e-01}, + { 8.1397964e-04, 4.9353381e-01}, + { 8.0594066e-02, 5.7517918e-01}, + { 1.6037415e-01, 6.5269554e-01}, + { 2.4015424e-01, 7.2210969e-01}, + { 3.1993432e-01, 7.7827984e-01}, + { 3.9971441e-01, 8.2003739e-01}, + { 4.7949450e-01, 8.5454970e-01}, + { 5.5927458e-01, 8.8205048e-01}, + { 6.3905467e-01, 9.0511063e-01}, + { 7.1883475e-01, 9.2178249e-01}, + { 7.9861484e-01, 9.3487068e-01}, + { 8.7839493e-01, 9.4476472e-01}, + { 9.5817501e-01, 9.5255531e-01}, + { 1.0379551e+00, 9.6057962e-01}, + { 1.1177352e+00, 9.6727953e-01}, + { 1.1975153e+00, 9.7327828e-01}, + { 1.2772954e+00, 9.7701776e-01}, + { 1.3570754e+00, 9.8028981e-01}, + { 1.4368555e+00, 9.8247117e-01}, + { 1.5166356e+00, 9.8480835e-01}, + { 1.5964157e+00, 9.8667809e-01}, + { 1.6761958e+00, 9.8924899e-01}, + { 1.7559759e+00, 9.9150826e-01}, + { 1.8357560e+00, 9.9298847e-01}, + { 1.9155360e+00, 9.9478031e-01}, + { 1.9953161e+00, 1.0000000e+00} + }; + } + }; + }; + }; + }; +}; diff --git a/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_th_subsaturated.hpp b/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_th_subsaturated.hpp index 6b60283f..4c504038 100644 --- a/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_th_subsaturated.hpp +++ b/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_th_subsaturated.hpp @@ -15,57 +15,57 @@ namespace libmpdataxx { vec = { - { -1.9936882e+00, 3.4883174e-03}, - { -1.9139081e+00, 7.0606097e-03}, - { -1.8341280e+00, 1.1074610e-02}, - { -1.7543479e+00, 1.5182662e-02}, - { -1.6745678e+00, 1.9700512e-02}, - { -1.5947877e+00, 2.4839776e-02}, - { -1.5150077e+00, 3.0205772e-02}, - { -1.4352276e+00, 3.6371209e-02}, - { -1.3554475e+00, 4.2986752e-02}, - { -1.2756674e+00, 5.0331197e-02}, - { -1.1958873e+00, 5.8685020e-02}, - { -1.1161072e+00, 6.8086850e-02}, - { -1.0363271e+00, 7.8600508e-02}, - { -9.5654705e-01, 9.0432572e-02}, - { -8.7676697e-01, 1.0396093e-01}, - { -7.9698688e-01, 1.1918054e-01}, - { -7.1720679e-01, 1.3642730e-01}, - { -6.3742671e-01, 1.5645196e-01}, - { -5.5764662e-01, 1.7986920e-01}, - { -4.7786654e-01, 2.0715768e-01}, - { -3.9808645e-01, 2.3872049e-01}, - { -3.1830636e-01, 2.7536546e-01}, - { -2.3852628e-01, 3.1744192e-01}, - { -1.5874619e-01, 3.6609362e-01}, - { -7.8966106e-02, 4.2029942e-01}, - { 8.1397964e-04, 4.7868213e-01}, - { 8.0594066e-02, 5.3950011e-01}, - { 1.6037415e-01, 5.9885190e-01}, - { 2.4015424e-01, 6.5395455e-01}, - { 3.1993432e-01, 7.0323606e-01}, - { 3.9971441e-01, 7.4501525e-01}, - { 4.7949450e-01, 7.8131088e-01}, - { 5.5927458e-01, 8.1190294e-01}, - { 6.3905467e-01, 8.3772690e-01}, - { 7.1883475e-01, 8.5983413e-01}, - { 7.9861484e-01, 8.7850679e-01}, - { 8.7839493e-01, 8.9485335e-01}, - { 9.5817501e-01, 9.0928695e-01}, - { 1.0379551e+00, 9.2176562e-01}, - { 1.1177352e+00, 9.3254128e-01}, - { 1.1975153e+00, 9.4239322e-01}, - { 1.2772954e+00, 9.5113333e-01}, - { 1.3570754e+00, 9.5888589e-01}, - { 1.4368555e+00, 9.6588436e-01}, - { 1.5166356e+00, 9.7212369e-01}, - { 1.5964157e+00, 9.7781047e-01}, - { 1.6761958e+00, 9.8294974e-01}, - { 1.7559759e+00, 9.8775478e-01}, - { 1.8357560e+00, 9.9215171e-01}, - { 1.9155360e+00, 9.9620098e-01}, - { 1.9953161e+00, 1.0000000e+00} + { -1.9936882e+00, 3.4883174e-03}, + { -1.9139081e+00, 7.0606097e-03}, + { -1.8341280e+00, 1.1074610e-02}, + { -1.7543479e+00, 1.5182662e-02}, + { -1.6745678e+00, 1.9700512e-02}, + { -1.5947877e+00, 2.4839776e-02}, + { -1.5150077e+00, 3.0205772e-02}, + { -1.4352276e+00, 3.6371209e-02}, + { -1.3554475e+00, 4.2986752e-02}, + { -1.2756674e+00, 5.0331197e-02}, + { -1.1958873e+00, 5.8685020e-02}, + { -1.1161072e+00, 6.8086850e-02}, + { -1.0363271e+00, 7.8600508e-02}, + { -9.5654705e-01, 9.0432572e-02}, + { -8.7676697e-01, 1.0396093e-01}, + { -7.9698688e-01, 1.1918054e-01}, + { -7.1720679e-01, 1.3642730e-01}, + { -6.3742671e-01, 1.5645196e-01}, + { -5.5764662e-01, 1.7986920e-01}, + { -4.7786654e-01, 2.0715768e-01}, + { -3.9808645e-01, 2.3872049e-01}, + { -3.1830636e-01, 2.7536546e-01}, + { -2.3852628e-01, 3.1744192e-01}, + { -1.5874619e-01, 3.6609362e-01}, + { -7.8966106e-02, 4.2029942e-01}, + { 8.1397964e-04, 4.7868213e-01}, + { 8.0594066e-02, 5.3950011e-01}, + { 1.6037415e-01, 5.9885190e-01}, + { 2.4015424e-01, 6.5395455e-01}, + { 3.1993432e-01, 7.0323606e-01}, + { 3.9971441e-01, 7.4501525e-01}, + { 4.7949450e-01, 7.8131088e-01}, + { 5.5927458e-01, 8.1190294e-01}, + { 6.3905467e-01, 8.3772690e-01}, + { 7.1883475e-01, 8.5983413e-01}, + { 7.9861484e-01, 8.7850679e-01}, + { 8.7839493e-01, 8.9485335e-01}, + { 9.5817501e-01, 9.0928695e-01}, + { 1.0379551e+00, 9.2176562e-01}, + { 1.1177352e+00, 9.3254128e-01}, + { 1.1975153e+00, 9.4239322e-01}, + { 1.2772954e+00, 9.5113333e-01}, + { 1.3570754e+00, 9.5888589e-01}, + { 1.4368555e+00, 9.6588436e-01}, + { 1.5166356e+00, 9.7212369e-01}, + { 1.5964157e+00, 9.7781047e-01}, + { 1.6761958e+00, 9.8294974e-01}, + { 1.7559759e+00, 9.8775478e-01}, + { 1.8357560e+00, 9.9215171e-01}, + { 1.9155360e+00, 9.9620098e-01}, + { 1.9953161e+00, 1.0000000e+00} }; } }; diff --git a/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_th_supersaturated.dat b/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_th_supersaturated.dat deleted file mode 100644 index c821ea47..00000000 --- a/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_th_supersaturated.dat +++ /dev/null @@ -1,51 +0,0 @@ - -1.9936882e+00 9.2096607e-03 - -1.9139081e+00 1.4472324e-02 - -1.8341280e+00 1.9734987e-02 - -1.7543479e+00 2.5091627e-02 - -1.6745678e+00 3.1106099e-02 - -1.5947877e+00 3.8718166e-02 - -1.5150077e+00 4.5484447e-02 - -1.4352276e+00 5.3660370e-02 - -1.3554475e+00 6.0896532e-02 - -1.2756674e+00 6.9542336e-02 - -1.1958873e+00 7.9033925e-02 - -1.1161072e+00 9.1156846e-02 - -1.0363271e+00 1.0450146e-01 - -9.5654705e-01 1.2151114e-01 - -8.7676697e-01 1.4171600e-01 - -7.9698688e-01 1.6173292e-01 - -7.1720679e-01 1.8560286e-01 - -6.3742671e-01 2.0928484e-01 - -5.5764662e-01 2.3663190e-01 - -4.7786654e-01 2.6632835e-01 - -3.9808645e-01 3.0335495e-01 - -3.1830636e-01 3.4282492e-01 - -2.3852628e-01 3.8577201e-01 - -1.5874619e-01 4.2890706e-01 - -7.8966106e-02 4.7927826e-01 - 8.1397964e-04 5.2936754e-01 - 8.0594066e-02 5.7908091e-01 - 1.6037415e-01 6.2578705e-01 - 2.4015424e-01 6.6751245e-01 - 3.1993432e-01 7.0717038e-01 - 3.9971441e-01 7.4128371e-01 - 4.7949450e-01 7.7135608e-01 - 5.5927458e-01 7.9860915e-01 - 6.3905467e-01 8.2238511e-01 - 7.1883475e-01 8.4033456e-01 - 7.9861484e-01 8.5894183e-01 - 8.7839493e-01 8.7801898e-01 - 9.5817501e-01 8.9484071e-01 - 1.0379551e+00 9.1034677e-01 - 1.1177352e+00 9.2152993e-01 - 1.1975153e+00 9.3205526e-01 - 1.2772954e+00 9.4164082e-01 - 1.3570754e+00 9.4944084e-01 - 1.4368555e+00 9.5573724e-01 - 1.5166356e+00 9.6306738e-01 - 1.5964157e+00 9.6964571e-01 - 1.6761958e+00 9.7490837e-01 - 1.7559759e+00 9.8082887e-01 - 1.8357560e+00 9.8609153e-01 - 1.9155360e+00 9.9116624e-01 - 1.9953161e+00 1.0000000e+00 diff --git a/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_th_supersaturated.hpp b/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_th_supersaturated.hpp new file mode 100644 index 00000000..3498d7cb --- /dev/null +++ b/libmpdata++/formulae/stretching_parameter_data/Waclawczyk_LES/cdf_th_supersaturated.hpp @@ -0,0 +1,75 @@ +#pragma once + +namespace libmpdataxx +{ + namespace formulae + { + namespace fractal + { + namespace stretch_params + { + namespace LES_th_rv + { + template + void d_CDF_th_supersaturated(std::vector> &vec) + { + vec = + { + { -1.9936882e+00, 9.2096607e-03}, + { -1.9139081e+00, 1.4472324e-02}, + { -1.8341280e+00, 1.9734987e-02}, + { -1.7543479e+00, 2.5091627e-02}, + { -1.6745678e+00, 3.1106099e-02}, + { -1.5947877e+00, 3.8718166e-02}, + { -1.5150077e+00, 4.5484447e-02}, + { -1.4352276e+00, 5.3660370e-02}, + { -1.3554475e+00, 6.0896532e-02}, + { -1.2756674e+00, 6.9542336e-02}, + { -1.1958873e+00, 7.9033925e-02}, + { -1.1161072e+00, 9.1156846e-02}, + { -1.0363271e+00, 1.0450146e-01}, + { -9.5654705e-01, 1.2151114e-01}, + { -8.7676697e-01, 1.4171600e-01}, + { -7.9698688e-01, 1.6173292e-01}, + { -7.1720679e-01, 1.8560286e-01}, + { -6.3742671e-01, 2.0928484e-01}, + { -5.5764662e-01, 2.3663190e-01}, + { -4.7786654e-01, 2.6632835e-01}, + { -3.9808645e-01, 3.0335495e-01}, + { -3.1830636e-01, 3.4282492e-01}, + { -2.3852628e-01, 3.8577201e-01}, + { -1.5874619e-01, 4.2890706e-01}, + { -7.8966106e-02, 4.7927826e-01}, + { 8.1397964e-04, 5.2936754e-01}, + { 8.0594066e-02, 5.7908091e-01}, + { 1.6037415e-01, 6.2578705e-01}, + { 2.4015424e-01, 6.6751245e-01}, + { 3.1993432e-01, 7.0717038e-01}, + { 3.9971441e-01, 7.4128371e-01}, + { 4.7949450e-01, 7.7135608e-01}, + { 5.5927458e-01, 7.9860915e-01}, + { 6.3905467e-01, 8.2238511e-01}, + { 7.1883475e-01, 8.4033456e-01}, + { 7.9861484e-01, 8.5894183e-01}, + { 8.7839493e-01, 8.7801898e-01}, + { 9.5817501e-01, 8.9484071e-01}, + { 1.0379551e+00, 9.1034677e-01}, + { 1.1177352e+00, 9.2152993e-01}, + { 1.1975153e+00, 9.3205526e-01}, + { 1.2772954e+00, 9.4164082e-01}, + { 1.3570754e+00, 9.4944084e-01}, + { 1.4368555e+00, 9.5573724e-01}, + { 1.5166356e+00, 9.6306738e-01}, + { 1.5964157e+00, 9.6964571e-01}, + { 1.6761958e+00, 9.7490837e-01}, + { 1.7559759e+00, 9.8082887e-01}, + { 1.8357560e+00, 9.8609153e-01}, + { 1.9155360e+00, 9.9116624e-01}, + { 1.9953161e+00, 1.0000000e+00} + }; + } + }; + }; + }; + }; +}; From 00e5135f9d07b68796cfc8d4ecc948f2dbde5d12 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Wed, 26 Apr 2023 11:39:20 +0200 Subject: [PATCH 128/129] add other stretching parameters from Marta --- libmpdata++/formulae/fractal_reconstruction.hpp | 1 - libmpdata++/formulae/stretching_parameters.hpp | 12 ++++++++++++ .../solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 11 +++++++++-- .../detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp | 10 ++++++++-- tests/sandbox/pbl/pbl.hpp | 1 + 5 files changed, 30 insertions(+), 5 deletions(-) diff --git a/libmpdata++/formulae/fractal_reconstruction.hpp b/libmpdata++/formulae/fractal_reconstruction.hpp index 427b1f18..00218b01 100644 --- a/libmpdata++/formulae/fractal_reconstruction.hpp +++ b/libmpdata++/formulae/fractal_reconstruction.hpp @@ -7,7 +7,6 @@ #pragma once #include -#include namespace libmpdataxx { diff --git a/libmpdata++/formulae/stretching_parameters.hpp b/libmpdata++/formulae/stretching_parameters.hpp index 39f0e92c..9c22a0d6 100644 --- a/libmpdata++/formulae/stretching_parameters.hpp +++ b/libmpdata++/formulae/stretching_parameters.hpp @@ -7,6 +7,9 @@ #pragma once #include +#include +#include +#include namespace libmpdataxx { @@ -71,6 +74,15 @@ namespace libmpdataxx case d_distro_t::LES_th_subsaturated: d_CDF_th_subsaturated(d_CDF_vctr); break; + case d_distro_t::LES_th_supersaturated: + d_CDF_th_supersaturated(d_CDF_vctr); + break; + case d_distro_t::LES_rv_subsaturated: + d_CDF_rv_subsaturated(d_CDF_vctr); + break; + case d_distro_t::LES_rv_supersaturated: + d_CDF_rv_supersaturated(d_CDF_vctr); + break; default: std::runtime_error("libmpdata++: invalid d_distro_t type in the constructor of d_of_CDF_fctr_LES"); } diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 230607f0..4b768485 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -218,13 +218,20 @@ namespace libmpdataxx case d_distro_t::LES_th_subsaturated: this->d_j(this->ijk_ref_with_halo) = this->d_of_CDF_fctr_LES_th_subsaturated(this->d_j(this->ijk_ref_with_halo)); break; + case d_distro_t::LES_th_supersaturated: + this->d_j(this->ijk_ref_with_halo) = this->d_of_CDF_fctr_LES_th_supersaturated(this->d_j(this->ijk_ref_with_halo)); + break; + case d_distro_t::LES_rv_subsaturated: + this->d_j(this->ijk_ref_with_halo) = this->d_of_CDF_fctr_LES_rv_subsaturated(this->d_j(this->ijk_ref_with_halo)); + break; + case d_distro_t::LES_rv_supersaturated: + this->d_j(this->ijk_ref_with_halo) = this->d_of_CDF_fctr_LES_rv_supersaturated(this->d_j(this->ijk_ref_with_halo)); + break; default: std::runtime_error("libmpdata++: invalid d_distro_t type in generate_stretching_parameters"); } xchng_ref(this->d_j, this->ijk_ref); // xchng to have the same values of d_j in the distmem halo region - - std::cerr << "generated stretching params: " << this->d_j(this->ijk_ref_with_halo) << std::endl; } // calculate refined points using (tri?)linear interpolation diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp index 65c9f49d..c64c2701 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp @@ -43,7 +43,10 @@ namespace libmpdataxx // types of stretching parameters that need to be instantiated at the beginning od the simulation // all are instantiated even not or all used, but they do not use much memory formulae::fractal::stretch_params::LES_th_rv::d_of_CDF_fctr_LES - d_of_CDF_fctr_LES_th_subsaturated; + d_of_CDF_fctr_LES_th_subsaturated, + d_of_CDF_fctr_LES_th_supersaturated, + d_of_CDF_fctr_LES_rv_subsaturated, + d_of_CDF_fctr_LES_rv_supersaturated; typename parent_t::arr_t c_j, d_j, f_j; // parameters used in fractal reconstruction, Akinlabi et al. 2019 @@ -191,7 +194,10 @@ namespace libmpdataxx {this->ijk[0].last() * n_ref, this->ijk[1].last() * n_ref, this->ijk[2].last() * n_ref}, // ubound {n_ref, n_ref, n_ref}, // stride }, - d_of_CDF_fctr_LES_th_subsaturated(formulae::fractal::stretch_params::d_distro_t::LES_th_subsaturated) + d_of_CDF_fctr_LES_th_subsaturated(formulae::fractal::stretch_params::d_distro_t::LES_th_subsaturated), + d_of_CDF_fctr_LES_th_supersaturated(formulae::fractal::stretch_params::d_distro_t::LES_th_supersaturated), + d_of_CDF_fctr_LES_rv_subsaturated(formulae::fractal::stretch_params::d_distro_t::LES_rv_subsaturated), + d_of_CDF_fctr_LES_rv_supersaturated(formulae::fractal::stretch_params::d_distro_t::LES_rv_supersaturated) { assert(p.n_fra_iter > 0); assert(n_ref % 2 == 0); diff --git a/tests/sandbox/pbl/pbl.hpp b/tests/sandbox/pbl/pbl.hpp index d9f0819c..da39da00 100644 --- a/tests/sandbox/pbl/pbl.hpp +++ b/tests/sandbox/pbl/pbl.hpp @@ -65,6 +65,7 @@ class pbl : public libmpdataxx::output::hdf5_xdmfgenerate_stretching_parameters(std::random_device{}()); //this->reconstruct_refinee(ix::w); +// this->generate_stretching_parameters(std::random_device{}(), libmpdataxx::formulae::fractal::stretch_params::d_distro_t::LES_rv_supersaturated); this->generate_stretching_parameters(std::random_device{}(), libmpdataxx::formulae::fractal::stretch_params::d_distro_t::LES_th_subsaturated); this->reconstruct_refinee(ix::tht); From cf806dee4466a1869257b37139befe8bbba5def8 Mon Sep 17 00:00:00 2001 From: Piotr Dziekan Date: Wed, 17 May 2023 11:36:28 +0200 Subject: [PATCH 129/129] new stretch params: LES supersat, but halved --- libmpdata++/formulae/stretching_parameters.hpp | 16 ++++++++++++---- .../detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp | 6 ++++++ .../detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp | 10 +++++++--- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/libmpdata++/formulae/stretching_parameters.hpp b/libmpdata++/formulae/stretching_parameters.hpp index 9c22a0d6..d35a1214 100644 --- a/libmpdata++/formulae/stretching_parameters.hpp +++ b/libmpdata++/formulae/stretching_parameters.hpp @@ -24,8 +24,10 @@ namespace libmpdataxx DNS_vel, LES_rv_subsaturated, LES_rv_supersaturated, + LES_rv_supersaturated_halved, LES_th_subsaturated, - LES_th_supersaturated + LES_th_supersaturated, + LES_th_supersaturated_halved }; // stretching parameters for velocity from DNS @@ -64,23 +66,26 @@ namespace libmpdataxx class d_of_CDF_fctr_LES { std::vector> d_CDF_vctr; + const d_distro_t dd; public: - d_of_CDF_fctr_LES(d_distro_t dd) + d_of_CDF_fctr_LES(d_distro_t _dd): dd(_dd) { - switch(dd) + switch(_dd) { case d_distro_t::LES_th_subsaturated: d_CDF_th_subsaturated(d_CDF_vctr); break; case d_distro_t::LES_th_supersaturated: + case d_distro_t::LES_th_supersaturated_halved: d_CDF_th_supersaturated(d_CDF_vctr); break; case d_distro_t::LES_rv_subsaturated: d_CDF_rv_subsaturated(d_CDF_vctr); break; case d_distro_t::LES_rv_supersaturated: + case d_distro_t::LES_rv_supersaturated_halved: d_CDF_rv_supersaturated(d_CDF_vctr); break; default: @@ -95,7 +100,10 @@ namespace libmpdataxx { return pair.second < val; }); - return pos->first; + if(dd == d_distro_t::LES_th_supersaturated_halved || dd == d_distro_t::LES_rv_supersaturated_halved) + return pos->first / real_t(2); + else + return pos->first; } BZ_DECLARE_FUNCTOR(d_of_CDF_fctr_LES); }; diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp index 4b768485..918d8b6a 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_3d.hpp @@ -221,12 +221,18 @@ namespace libmpdataxx case d_distro_t::LES_th_supersaturated: this->d_j(this->ijk_ref_with_halo) = this->d_of_CDF_fctr_LES_th_supersaturated(this->d_j(this->ijk_ref_with_halo)); break; + case d_distro_t::LES_th_supersaturated_halved: + this->d_j(this->ijk_ref_with_halo) = this->d_of_CDF_fctr_LES_th_supersaturated_halved(this->d_j(this->ijk_ref_with_halo)); + break; case d_distro_t::LES_rv_subsaturated: this->d_j(this->ijk_ref_with_halo) = this->d_of_CDF_fctr_LES_rv_subsaturated(this->d_j(this->ijk_ref_with_halo)); break; case d_distro_t::LES_rv_supersaturated: this->d_j(this->ijk_ref_with_halo) = this->d_of_CDF_fctr_LES_rv_supersaturated(this->d_j(this->ijk_ref_with_halo)); break; + case d_distro_t::LES_rv_supersaturated_halved: + this->d_j(this->ijk_ref_with_halo) = this->d_of_CDF_fctr_LES_rv_supersaturated_halved(this->d_j(this->ijk_ref_with_halo)); + break; default: std::runtime_error("libmpdata++: invalid d_distro_t type in generate_stretching_parameters"); } diff --git a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp index c64c2701..c7ca4ed5 100644 --- a/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp +++ b/libmpdata++/solvers/detail/mpdata_rhs_vip_prs_sgs_fra_common.hpp @@ -41,12 +41,14 @@ namespace libmpdataxx protected: // types of stretching parameters that need to be instantiated at the beginning od the simulation - // all are instantiated even not or all used, but they do not use much memory + // all are instantiated even if not or all used, but they do not use much memory and we don't know which will be used formulae::fractal::stretch_params::LES_th_rv::d_of_CDF_fctr_LES d_of_CDF_fctr_LES_th_subsaturated, d_of_CDF_fctr_LES_th_supersaturated, + d_of_CDF_fctr_LES_th_supersaturated_halved, d_of_CDF_fctr_LES_rv_subsaturated, - d_of_CDF_fctr_LES_rv_supersaturated; + d_of_CDF_fctr_LES_rv_supersaturated, + d_of_CDF_fctr_LES_rv_supersaturated_halved; typename parent_t::arr_t c_j, d_j, f_j; // parameters used in fractal reconstruction, Akinlabi et al. 2019 @@ -196,8 +198,10 @@ namespace libmpdataxx }, d_of_CDF_fctr_LES_th_subsaturated(formulae::fractal::stretch_params::d_distro_t::LES_th_subsaturated), d_of_CDF_fctr_LES_th_supersaturated(formulae::fractal::stretch_params::d_distro_t::LES_th_supersaturated), + d_of_CDF_fctr_LES_th_supersaturated_halved(formulae::fractal::stretch_params::d_distro_t::LES_th_supersaturated_halved), d_of_CDF_fctr_LES_rv_subsaturated(formulae::fractal::stretch_params::d_distro_t::LES_rv_subsaturated), - d_of_CDF_fctr_LES_rv_supersaturated(formulae::fractal::stretch_params::d_distro_t::LES_rv_supersaturated) + d_of_CDF_fctr_LES_rv_supersaturated(formulae::fractal::stretch_params::d_distro_t::LES_rv_supersaturated), + d_of_CDF_fctr_LES_rv_supersaturated_halved(formulae::fractal::stretch_params::d_distro_t::LES_rv_supersaturated_halved) { assert(p.n_fra_iter > 0); assert(n_ref % 2 == 0);